fux-engine 2.0.0-alpha.7

package/src/query/fuse.mjs

@@ -0,0 +1,49 @@
+/** Reciprocal-rank fusion for `-q` — several phrasings, one ranked list.
+ *  Twin of `fuse.py`. */
+import { pyRound9, cmpCodePoints } from "../compat/pyfloat.mjs";
+
+/** Cormack et al. 2009's constant. **Not tuned here and not a tune.toml key** —
+ *  a knob on it would be a knob on a published constant measured on TREC
+ *  collections, with nothing in this repo able to beat it at 10 documents. */
+export const K = 60;
+
+/** `id -> fused score`, from ranked id lists. Rank 0 is best.
+ *  A document absent from a list contributes nothing — **not a penalty**. */
+export function rrf(rankLists, k = K) {
+  const scores = {};
+  for (const ranks of rankLists) {
+    for (let i = 0; i < ranks.length; i++) {
+      scores[ranks[i]] = (scores[ranks[i]] ?? 0.0) + 1.0 / (k + i + 1);
+    }
+  }
+  return scores;
+}
+
+/** Fuse several ranked result lists into one, by rank.
+ *
+ * **The result object comes from the arm that ranked it best**, so its `loc`,
+ * `title` and `archived` are a real document's view, not a merge of several.
+ * Only `score` is replaced. Ordered by the same key `rank()` uses, so a fused
+ * list is as reproducible as an unfused one. */
+export function fuseResults(resultLists, top, k = K) {
+  if (!resultLists.length) return [];
+  if (resultLists.length === 1) return resultLists[0].slice(0, top);
+
+  const scores = rrf(resultLists.map((rs) => rs.map((r) => r.id)), k);
+
+  const bestSeen = new Map();
+  for (const results of resultLists) {
+    for (let i = 0; i < results.length; i++) {
+      const cur = bestSeen.get(results[i].id);
+      if (cur === undefined || i < cur[0]) bestSeen.set(results[i].id, [i, results[i]]);
+    }
+  }
+
+  const ordered = Object.keys(scores).sort((a, b) => {
+    const ra = -pyRound9(scores[a]), rb = -pyRound9(scores[b]);
+    if (ra !== rb) return ra < rb ? -1 : 1;
+    return cmpCodePoints(a, b);
+  });
+
+  return ordered.slice(0, top).map((id) => ({ ...bestSeen.get(id)[1], score: scores[id] }));
+}

package/src/query/headings.mjs

@@ -0,0 +1,30 @@
+/** Which of a record's headings match the query — DISPLAY ONLY.
+ *
+ * Twin of `headings.py`. Runs after `run_query` returns, over the record's
+ * committed `phrases`, so it can never reach a score. `[]` for a missing
+ * record, a `hashed` one, or a query no heading matches.
+ */
+import { tokenize } from "./tokenize.mjs";
+
+export const MAX_HEADINGS = 3;
+
+export function headingsFor(record, query, limit = MAX_HEADINGS) {
+  if (!record) return [];
+  const phrases = record.phrases || [];
+  if (!phrases.length) return [];
+
+  const wanted = new Set(tokenize(query));
+  if (!wanted.size) return [];
+
+  const scored = [];
+  for (let position = 0; position < phrases.length; position++) {
+    const phrase = phrases[position];
+    if (typeof phrase !== "string") continue;   // a record is data, not a promise
+    let matches = 0;
+    for (const t of new Set(tokenize(phrase))) if (wanted.has(t)) matches++;
+    if (matches) scored.push([-matches, position, phrase]);
+  }
+  // `position` ascending is the tie-break, so the key sorts directly.
+  scored.sort((a, b) => (a[0] - b[0]) || (a[1] - b[1]));
+  return scored.slice(0, limit).map((s) => s[2]);
+}

package/src/query/rank.mjs

@@ -0,0 +1,158 @@
+/** The scorer and the sort — ONE function, both candidate paths.
+ *
+ * Twin of `src/fux/query/rank.py`. This is why the differential law is
+ * achievable: float addition is not associative, so a second implementation
+ * that accumulates its own scores would diverge in the last bit. The
+ * accelerator produces candidates and statistics; this does every arithmetic
+ * operation, once, for both.
+ */
+import { DEFAULT_SCORING, deriveWlen, scoreRecord } from "./bm25f.mjs";
+import { displayTitle } from "../store/format.mjs";
+import { recencyMultiplier } from "../ingest/priors.mjs";
+import { pyRound9, cmpCodePoints } from "../compat/pyfloat.mjs";
+
+export class Corpus {
+  constructor(n = 0, totalWlen = 0.0, newestMtime = 0) {
+    this.n = n; this.totalWlen = totalWlen; this.newestMtime = newestMtime;
+  }
+  get avgWlen() { return this.n ? this.totalWlen / this.n : 0.0; }
+}
+
+/** `loc` is under one of the declared archived directories. */
+export function isArchivedLoc(loc, archivedDirs) {
+  for (const d of archivedDirs) {
+    if (loc === d || loc.startsWith(d.endsWith("/") ? d : d + "/")) return true;
+  }
+  return false;
+}
+
+function recordIsArchived(record, archivedDirs) {
+  if (record.archived) return true;
+  return archivedDirs.size > 0 && isArchivedLoc(record.loc, archivedDirs);
+}
+
+/** The score multiplier policy, in ONE place, for both candidate paths. */
+export class Weighting {
+  constructor({
+    archivedWeight = 1.0, archivedDirs = new Set(),
+    supersededWeight = 1.0, recencyHalfLifeDays = 0.0,
+    newestMtime = 0, priority = [],
+  } = {}) {
+    this.archivedWeight = archivedWeight;
+    this.archivedDirs = archivedDirs;
+    this.supersededWeight = supersededWeight;
+    this.recencyHalfLifeDays = recencyHalfLifeDays;
+    this.newestMtime = newestMtime;
+    this.priority = priority;   // [[prefix, weight], ...], longest-prefix wins
+    Object.freeze(this);
+  }
+  get trivial() {
+    return this.archivedWeight === 1.0
+      && this.supersededWeight === 1.0
+      && this.recencyHalfLifeDays === 0.0
+      && this.priority.length === 0;
+  }
+  priorityFor(loc) {
+    let best = 1.0, bestLen = -1;
+    for (const [prefix, w] of this.priority) {
+      if (loc === prefix || loc.startsWith(prefix.endsWith("/") ? prefix : prefix + "/")) {
+        if (prefix.length > bestLen) { best = w; bestLen = prefix.length; }
+      }
+    }
+    return best;
+  }
+  /** The multiplier for one record: archived x superseded x recency x priority. */
+  of(record) {
+    if (this.trivial) return 1.0;
+    let weight = 1.0;
+    if (this.archivedWeight !== 1.0 && recordIsArchived(record, this.archivedDirs)) {
+      weight *= this.archivedWeight;
+    }
+    if (this.supersededWeight !== 1.0 && record.superseded) weight *= this.supersededWeight;
+    if (this.recencyHalfLifeDays > 0) {
+      weight *= recencyMultiplier(record.mtime ?? null, this.newestMtime, this.recencyHalfLifeDays);
+    }
+    if (this.priority.length) weight *= this.priorityFor(record.loc || "");
+    return weight;
+  }
+  withNewestMtime(mtime) {
+    return new Weighting({
+      archivedWeight: this.archivedWeight, archivedDirs: this.archivedDirs,
+      supersededWeight: this.supersededWeight,
+      recencyHalfLifeDays: this.recencyHalfLifeDays,
+      newestMtime: mtime, priority: this.priority,
+    });
+  }
+}
+
+/** Rank candidates. `expansion` is `{hashes, required, weights}` or null. */
+export function rank(
+  candidates, queryHashes, df, corpus, top,
+  { weighting = null, scoring = DEFAULT_SCORING, statsOut = null, expansion = null } = {},
+) {
+  const termWeights = expansion && expansion.weights && Object.keys(expansion.weights).length
+    ? expansion.weights : null;
+  const required = expansion && expansion.required ? expansion.required : queryHashes;
+  const avgWlen = corpus.avgWlen;
+
+  let w = weighting || new Weighting();
+  if (w.recencyHalfLifeDays > 0 && corpus.newestMtime) w = w.withNewestMtime(corpus.newestMtime);
+  const demote = !w.trivial;
+
+  const scored = [];
+  for (const record of candidates) {
+    const terms = record.terms || {};
+    // 🔴 The hallucinated-citation guard, before anything is scored: a
+    // document matching ONLY expansion terms is not an answer to the question.
+    let matches = false;
+    for (const h of required) { if (h in terms) { matches = true; break; } }
+    if (!matches) continue;
+
+    let s = scoreRecord(
+      terms, record.flen || [], queryHashes, df, corpus.n, avgWlen, scoring, termWeights,
+    );
+    const archived = recordIsArchived(record, w.archivedDirs);
+    if (demote) s *= w.of(record);
+    if (s > 0) scored.push([record, s, archived]);
+  }
+
+  // W-111 — the DECLARED tie-break, in Arpit's ratified order:
+  //     superseded -> recency -> priority -> id
+  // Every one of these is already a Weighting multiplier and every one ships
+  // as a no-op; this reads the same FACTS, only where the rounded scores tie.
+  //
+  // 🔴 `id` is compared by CODE POINT, never with `<` — W-107 hazard H1.
+  scored.sort((A, B) => {
+    const ra = -pyRound9(A[1]), rb = -pyRound9(B[1]);
+    if (ra !== rb) return ra < rb ? -1 : 1;
+    const sa = A[0].superseded ? 1 : 0, sb = B[0].superseded ? 1 : 0;
+    if (sa !== sb) return sa - sb;
+    const ma = -(A[0].mtime || 0), mb = -(B[0].mtime || 0);
+    if (ma !== mb) return ma < mb ? -1 : 1;
+    const pa = -w.priorityFor(A[0].loc || ""), pb = -w.priorityFor(B[0].loc || "");
+    if (pa !== pb) return pa < pb ? -1 : 1;
+    return cmpCodePoints(A[0].id, B[0].id);
+  });
+
+  // Marked AFTER the sort and over the WHOLE list: a neighbour comparison on
+  // the truncated window would un-mark the last row, which is the one most
+  // likely to have been decided by a coin-toss.
+  const tied = new Set();
+  for (let i = 0; i < scored.length - 1; i++) {
+    if (pyRound9(scored[i][1]) === pyRound9(scored[i + 1][1])) { tied.add(i); tied.add(i + 1); }
+  }
+
+  if (statsOut !== null && scored.length) {
+    const topTerms = scored[0][0].terms || {};
+    statsOut.top_doc_hashes = queryHashes.filter((h) => h in topTerms);
+  }
+
+  return scored.slice(0, top).map(([record, s, archived], i) => ({
+    id: record.id,
+    title: displayTitle(record),
+    loc: record.loc,
+    score: s,
+    archived,
+    tie: tied.has(i),
+  }));
+}

package/src/query/run.mjs

@@ -0,0 +1,74 @@
+/** `run_query` — the shared spine of `ask`, `find` and `answer`.
+ *
+ * Twin of the pure half of `src/fux/query/__init__.py`. Everything that
+ * decides an answer lives here; the verbs are projections of it, which is why
+ * `find` cannot rank differently from `ask`.
+ */
+import { ask as scanAsk, queryTermHashes } from "./scan.mjs";
+import { rank } from "./rank.mjs";
+import { signals } from "./confidence.mjs";
+import { headingsFor } from "./headings.mjs";
+import { fuseResults } from "./fuse.mjs";
+import * as expandMod from "./expand.mjs";
+import { tokenizePairs } from "./tokenize.mjs";
+import { DEFAULT_SCORING } from "./bm25f.mjs";
+
+/** Retrieval depth when the proximity reranker is on. Off by default, so the
+ *  default depth is `top` and the extra 15 candidates are never scored. */
+export const RERANK_DEPTH = 20;
+
+/** One arm: scan, rank, band. `expand` is a term slot, not a second question. */
+export function runQuery(root, query, top, {
+  weighting = null, scoring = DEFAULT_SCORING, expand = null,
+  separationFloor = undefined, docCoverageFloor = undefined,
+  wantConfidence = false, rerankWeight = 0.0,
+} = {}) {
+  const queryHashes = queryTermHashes(query);
+  const expansion = expand
+    ? expandMod.build(queryHashes, queryTermHashes(expand), expand.weight ?? 0.2)
+    : null;
+
+  const depth = rerankWeight > 0 ? Math.max(top, RERANK_DEPTH) : top;
+  const statsOut = {};
+  const results = scanAsk(root, query, depth, { weighting, scoring, statsOut, expansion });
+
+  const out = results.slice(0, top);
+  let confidence = null;
+  if (wantConfidence) {
+    const opts = { topDocHashes: statsOut.top_doc_hashes ?? null };
+    if (separationFloor !== undefined) opts.separationFloor = separationFloor;
+    if (docCoverageFloor !== undefined) opts.docCoverageFloor = docCoverageFloor;
+    confidence = signals(
+      tokenizePairs(query), queryHashes, statsOut.df ?? {}, statsOut.n ?? 0,
+      results.map((r) => r.score), opts,
+    );
+  }
+  return { results: out, confidence, queryHashes, stats: statsOut };
+}
+
+/** Several phrasings, fused by rank. Arm 1 owns the confidence block.
+ *
+ * ⚠ **The band always describes ARM 1 only**, and that is deliberate: the
+ * `separation_floor` is calibrated against BM25F, and a fused top-2 differs by
+ * ~0.0003, so a band computed over fused scores would be measuring a different
+ * quantity under the same name. */
+export function runFused(root, queries, top, opts = {}) {
+  const unique = [...new Set(queries.filter((q) => q && q.trim()))];
+  if (unique.length <= 1) {
+    return { ...runQuery(root, unique[0] ?? "", top, opts), fused: false };
+  }
+  const first = runQuery(root, unique[0], top, opts);
+  const arms = [first.results];
+  for (const q of unique.slice(1)) {
+    arms.push(runQuery(root, q, top, { ...opts, wantConfidence: false }).results);
+  }
+  return {
+    results: fuseResults(arms, top),
+    confidence: first.confidence,
+    queryHashes: first.queryHashes,
+    stats: first.stats,
+    fused: true,
+  };
+}
+
+export { headingsFor, rank, expandMod as expand };

package/src/query/scan.mjs

@@ -0,0 +1,106 @@
+/** The reference scan — the DEFAULT path, and the one a bug report reproduces
+ *  against. Twin of `src/fux/query/scan.py`.
+ *
+ * Every line in the corpus is touched; only a line whose raw bytes contain a
+ * query hash is parsed. `flen` and `mtime` are read by regex from the bytes on
+ * every line, candidate or not, because parsing every record to sum a length
+ * is exactly what the prefilter exists to avoid.
+ */
+import { rawRecordLines, iterShardPaths } from "../store/reader.mjs";
+import { termHash, TF_FIELDS } from "../store/format.mjs";
+import { DEFAULT_SCORING, deriveWlen } from "./bm25f.mjs";
+import { Corpus, rank } from "./rank.mjs";
+import { tokenize } from "./tokenize.mjs";
+
+//: The byte-level oracle, over per-field counts. `wlen` is DERIVED from `flen`
+//: at the weights in force, so this parses the array and applies `deriveWlen`
+//: — the same function the scorer and the refer plane use, so they cannot drift.
+const FLEN_RE = /"flen":\[([0-9,\s]*)\]/;
+const MTIME_RE = /"mtime":(\d+)/;
+
+function flenFromLine(text) {
+  const m = FLEN_RE.exec(text);
+  if (!m) return null;
+  const inner = m[1].trim();
+  if (!inner) return [];
+  return inner.split(",").map((p) => parseInt(p, 10));
+}
+
+/** Query terms as index hashes, deduped, ORDER PRESERVED.
+ *  Order is load-bearing: `rank()` sums BM25F contributions in it, so both
+ *  candidate generators must derive it identically from the same string. */
+export function queryTermHashes(query) {
+  const seen = new Set();
+  const out = [];
+  for (const t of tokenize(query)) {
+    const h = termHash(t);
+    if (!seen.has(h)) { seen.add(h); out.push(h); }
+  }
+  return out;
+}
+
+/** The B2 pass: candidate records, `df`, and the corpus statistics. */
+export function scanCandidates(root, queryHashes, { scoring = DEFAULT_SCORING } = {}) {
+  const patterns = queryHashes.map((h) => Buffer.from(`"${h}"`, "ascii"));
+
+  let totalDocs = 0;
+  // Per-field totals, summed raw and weighted ONCE at the end. Summing
+  // `deriveWlen` per record would bake the weights into a running total the
+  // moment anything cached it — the accelerator's stats plane made exactly
+  // that mistake (ADR-TUNE, 2026-08-24).
+  const totalFlen = new Array(TF_FIELDS.length).fill(0);
+  let newestMtime = 0;
+  const df = {};
+  for (const h of queryHashes) df[h] = 0;
+  const candidates = [];
+
+  for (const path of iterShardPaths(root)) {
+    const [, lines] = rawRecordLines(path);
+    for (const line of lines) {
+      totalDocs++;
+      // latin1 keeps this a byte view: the regexes are ASCII-only and this
+      // avoids decoding UTF-8 for every line in the corpus.
+      const text = line.toString("latin1");
+      const flen = flenFromLine(text);
+      if (flen !== null) for (let i = 0; i < flen.length; i++) totalFlen[i] += flen[i];
+      const mt = MTIME_RE.exec(text);
+      if (mt !== null) {
+        const value = parseInt(mt[1], 10);
+        if (value > newestMtime) newestMtime = value;
+      }
+      // The substring check is a PREFILTER only: a query hash can appear as a
+      // literal 16-hex string outside `terms` (a title, an id, a sha) without
+      // the document containing that term. Once a line is worth parsing, `df`
+      // is counted from the parsed record's own `terms` keys, which is exact.
+      let hit = false;
+      for (const p of patterns) { if (line.includes(p)) { hit = true; break; } }
+      if (!hit) continue;
+
+      const record = JSON.parse(line.toString("utf8"));
+      const recordTerms = record.terms || {};
+      for (const h of queryHashes) if (h in recordTerms) df[h]++;
+      candidates.push(record);
+    }
+  }
+
+  return [candidates, df, new Corpus(totalDocs, deriveWlen(totalFlen, scoring), newestMtime)];
+}
+
+/** The reference path. */
+export function ask(root, query, top = 5, opts = {}) {
+  const { weighting = null, scoring = DEFAULT_SCORING, statsOut = null, expansion = null } = opts;
+  const queryHashes = queryTermHashes(query);
+  if (!queryHashes.length) {
+    // A query that tokenizes to nothing still owes the caller its corpus
+    // statistics, or confidence cannot tell "no terms" from "not run".
+    if (statsOut !== null) {
+      if (!("df" in statsOut)) statsOut.df = {};
+      if (!("n" in statsOut)) statsOut.n = 0;
+    }
+    return [];
+  }
+  const collect = expansion ? expansion.hashes : queryHashes;
+  const [candidates, df, corpus] = scanCandidates(root, collect, { scoring });
+  if (statsOut !== null) { statsOut.df = df; statsOut.n = corpus.n; }
+  return rank(candidates, collect, df, corpus, top, { weighting, scoring, statsOut, expansion });
+}

package/src/query/stem.mjs

@@ -0,0 +1,172 @@
+/** The Porter stemmer — analyzer v2's morphology step.
+ *
+ * Twin of `src/fux/query/stem.py`. **Transcribed, not reimplemented**: Porter
+ * is a fully specified algorithm with a published test vocabulary, and every
+ * departure fux makes from the textbook (see `shouldStem`) is a departure the
+ * Python side already made. A difference here is a silent no-match, not an
+ * error — the query hashes a string the index never wrote.
+ *
+ * Reference: M.F. Porter, "An algorithm for suffix stripping", Program 14(3),
+ * 1980. Step numbering follows the paper, as it does on the Python side.
+ */
+
+const VOWELS = new Set(["a", "e", "i", "o", "u"]);
+
+function isConsonant(word, i) {
+  const ch = word[i];
+  if (VOWELS.has(ch)) return false;
+  if (ch === "y") return i === 0 || !isConsonant(word, i - 1);
+  return true;
+}
+
+/** `m` — the number of VC sequences in [C](VC){m}[V]. */
+function measure(stemStr) {
+  let m = 0;
+  let i = 0;
+  const n = stemStr.length;
+  while (i < n && isConsonant(stemStr, i)) i++;
+  while (i < n) {
+    while (i < n && !isConsonant(stemStr, i)) i++;
+    if (i >= n) break;
+    m++;
+    while (i < n && isConsonant(stemStr, i)) i++;
+  }
+  return m;
+}
+
+function hasVowel(stemStr) {
+  for (let i = 0; i < stemStr.length; i++) if (!isConsonant(stemStr, i)) return true;
+  return false;
+}
+
+function doubleConsonantSuffix(word) {
+  return word.length >= 2
+    && word[word.length - 1] === word[word.length - 2]
+    && isConsonant(word, word.length - 1);
+}
+
+/** consonant-vowel-consonant where the final one is not w, x or y. */
+function cvc(word) {
+  if (word.length < 3) return false;
+  const n = word.length;
+  if (!(isConsonant(word, n - 3) && !isConsonant(word, n - 2) && isConsonant(word, n - 1))) {
+    return false;
+  }
+  return !"wxy".includes(word[n - 1]);
+}
+
+function replace(word, suffix, repl, minM) {
+  if (!word.endsWith(suffix)) return null;
+  const s = word.slice(0, word.length - suffix.length);
+  return measure(s) > minM ? s + repl : null;
+}
+
+const STEP2 = [
+  ["ational", "ate"], ["tional", "tion"], ["enci", "ence"], ["anci", "ance"],
+  ["izer", "ize"], ["bli", "ble"], ["alli", "al"], ["entli", "ent"],
+  ["eli", "e"], ["ousli", "ous"], ["ization", "ize"], ["ation", "ate"],
+  ["ator", "ate"], ["alism", "al"], ["iveness", "ive"], ["fulness", "ful"],
+  ["ousness", "ous"], ["aliti", "al"], ["iviti", "ive"], ["biliti", "ble"],
+  ["logi", "log"],
+];
+
+const STEP3 = [
+  ["icate", "ic"], ["ative", ""], ["alize", "al"], ["iciti", "ic"],
+  ["ical", "ic"], ["ful", ""], ["ness", ""],
+];
+
+const STEP4 = [
+  "al", "ance", "ence", "er", "ic", "able", "ible", "ant", "ement",
+  "ment", "ent", "ou", "ism", "ate", "iti", "ous", "ive", "ize",
+];
+
+/** Python's `str.isalpha()` — Unicode alphabetic, NOT `/^[a-z]+$/`.
+ *
+ * ⚠ This is a real divergence risk and the reason it is spelled out. Python's
+ * `isalpha` is true for `café` and `中文`; a naive `[a-zA-Z]` test is not, and
+ * would stem tokens Python leaves alone. `\p{Alphabetic}` is the Unicode
+ * property Python uses.
+ */
+const ALPHA_RE = /^\p{Alphabetic}+$/u;
+
+/** Words only: no digits, no underscores, at least three characters. */
+export function shouldStem(token) {
+  if (token.length < 3) return false;
+  return ALPHA_RE.test(token);
+}
+
+/** Porter-stem one already-lowercased token. */
+export function stem(word) {
+  if (!shouldStem(word)) return word;
+
+  // Step 1a — plurals
+  if (word.endsWith("sses")) word = word.slice(0, -2);
+  else if (word.endsWith("ies")) word = word.slice(0, -2);
+  else if (word.endsWith("ss")) { /* unchanged */ }
+  else if (word.endsWith("s")) word = word.slice(0, -1);
+
+  // Step 1b — -ed / -ing
+  let secondPass = false;
+  if (word.endsWith("eed")) {
+    if (measure(word.slice(0, -3)) > 0) word = word.slice(0, -1);
+  } else if (word.endsWith("ed") && hasVowel(word.slice(0, -2))) {
+    word = word.slice(0, -2);
+    secondPass = true;
+  } else if (word.endsWith("ing") && hasVowel(word.slice(0, -3))) {
+    word = word.slice(0, -3);
+    secondPass = true;
+  }
+  if (secondPass) {
+    if (word.endsWith("at") || word.endsWith("bl") || word.endsWith("iz")) {
+      word += "e";
+    } else if (
+      doubleConsonantSuffix(word)
+      && !(word.endsWith("l") || word.endsWith("s") || word.endsWith("z"))
+    ) {
+      word = word.slice(0, -1);
+    } else if (measure(word) === 1 && cvc(word)) {
+      word += "e";
+    }
+  }
+
+  // Step 1c — terminal y
+  if (word.endsWith("y") && hasVowel(word.slice(0, -1))) {
+    word = word.slice(0, -1) + "i";
+  }
+
+  // Step 2 / 3 — derivational suffixes
+  for (const [suffix, repl] of STEP2) {
+    const out = replace(word, suffix, repl, 0);
+    if (out !== null) { word = out; break; }
+  }
+  for (const [suffix, repl] of STEP3) {
+    const out = replace(word, suffix, repl, 0);
+    if (out !== null) { word = out; break; }
+  }
+
+  // Step 4 — strip when m > 1. Python's for/else: the `ion` branch runs only
+  // when NO suffix matched, which is why this needs an explicit flag.
+  let matched = false;
+  for (const suffix of STEP4) {
+    if (word.endsWith(suffix)) {
+      const s = word.slice(0, word.length - suffix.length);
+      matched = true;
+      if (suffix === "ion" && !(s.endsWith("s") || s.endsWith("t"))) continue;
+      if (measure(s) > 1) word = s;
+      break;
+    }
+  }
+  if (!matched && word.endsWith("ion")) {
+    const s = word.slice(0, -3);
+    if (measure(s) > 1 && (s.endsWith("s") || s.endsWith("t"))) word = s;
+  }
+
+  // Step 5a / 5b — terminal e and doubled l
+  if (word.endsWith("e")) {
+    const m = measure(word.slice(0, -1));
+    if (m > 1 || (m === 1 && !cvc(word.slice(0, -1)))) word = word.slice(0, -1);
+  }
+  if (word.endsWith("ll") && measure(word) > 1) word = word.slice(0, -1);
+
+  return word;
+}

package/src/query/tokenize.mjs

@@ -0,0 +1,11 @@
+/** The tokenizer both sides of a match share — analyzer v2.
+ *
+ * Twin of `src/fux/query/tokenize.py`. The pipeline lives in `analyzer.mjs`;
+ * this is the stable entry point, kept so every caller gets *the same*
+ * analysis by construction rather than by review.
+ */
+import { analyze, analyzePairs, STOPWORDS } from "./analyzer.mjs";
+
+export function tokenize(text) { return analyze(text); }
+export function tokenizePairs(text) { return analyzePairs(text); }
+export { STOPWORDS };

package/src/refer/assemble.mjs

@@ -0,0 +1,82 @@
+/** Fill a byte budget with the highest-value passages that fit.
+ *  Twin of `src/fux/refer/_assemble.py`.
+ *
+ * Greedy by **score per byte** — the question is not "what scores highest" but
+ * "what is the most answer per byte of the caller's window".
+ */
+import { cmpCodePoints } from "../compat/pyfloat.mjs";
+
+export const DEFAULT_BUDGET = 8000;
+export const PER_DOC_FRACTION = 0.5;
+/** Charged per citation, so the budget bounds the RENDERED answer and not
+ *  merely its payload. */
+export const CITATION_OVERHEAD = 80;
+
+const enc = new TextEncoder();
+
+export function assemble(scored, {
+  budget = DEFAULT_BUDGET, k = null, source = "fetched",
+  overhead = 0, perDocFraction = PER_DOC_FRACTION,
+} = {}) {
+  if (budget <= 0) throw new RangeError("budget must be positive");
+
+  const candidates = scored.filter((s) => s.score > 0);
+  if (!candidates.length) return { citations: [], budget, used: overhead, dropped: 0 };
+
+  const cmp = (a, b) => {
+    if (a.score !== b.score) return a.score > b.score ? -1 : 1;
+    const s = cmpCodePoints(a.sha, b.sha);
+    return s !== 0 ? s : cmpCodePoints(a.locator, b.locator);
+  };
+
+  // The floor: the best answer by ABSOLUTE score is seated first, so greedy
+  // score-per-byte cannot crowd it out with cheaper fragments.
+  let best = candidates[0];
+  for (const s of candidates) if (cmp(s, best) < 0) best = s;
+
+  const rest = candidates.filter((s) => s !== best);
+  rest.sort((a, b) => {
+    const ra = -(a.score / Math.max(a.nbytes, 1));
+    const rb = -(b.score / Math.max(b.nbytes, 1));
+    if (ra !== rb) return ra < rb ? -1 : 1;
+    const s = cmpCodePoints(a.sha, b.sha);
+    return s !== 0 ? s : cmpCodePoints(a.locator, b.locator);
+  });
+
+  // **The cap protects against competition, so with no competition there is
+  // nothing to protect.** Keyed on the candidates' own documents, not on `k`
+  // or the caller's intent.
+  const single = new Set(candidates.map((s) => s.doc_id)).size <= 1;
+  const perDocCap = single ? budget : Math.floor(budget * perDocFraction);
+
+  let used = overhead, dropped = 0;
+  const perDoc = new Map();
+  const chosen = [];
+
+  for (const s of [best, ...rest]) {
+    if (k !== null && chosen.length >= k) { dropped++; continue; }
+    const text = s.passage.text;
+    const citation = {
+      doc_id: s.doc_id, locator: s.locator, sha: s.sha,
+      heading: s.passage.heading, text, score: s.score, source,
+      // ⚠ The overhead is part of a citation's OWN size, not a separate line
+      // item: the locator line and separator will be rendered around every
+      // passage, so the budget bounds the rendered answer rather than the
+      // payload. Charging it here is what makes `used` mean what it says.
+      nbytes: enc.encode(text).length + CITATION_OVERHEAD,
+    };
+    const spent = perDoc.get(s.doc_id) ?? 0;
+    // A document's FIRST citation is exempt: the cap exists to stop a document
+    // DOMINATING, not to stop it appearing.
+    const capped = perDoc.has(s.doc_id) && spent + citation.nbytes > perDocCap;
+    if (used + citation.nbytes > budget || capped) { dropped++; continue; }
+    chosen.push(citation);
+    used += citation.nbytes;
+    perDoc.set(s.doc_id, spent + citation.nbytes);
+  }
+
+  // Presented in SCORE order — the caller reads the best answer first, even
+  // though selection ran on score-per-byte.
+  chosen.sort(cmp);
+  return { citations: chosen, budget, used, dropped };
+}