@konneal/engine 0.1.0

package/workers/worker_public/src/stages/termNudge.ts

@@ -0,0 +1,24 @@
+// Exact-term lookup: the understanding names the term; clause chunks
+// whose head IS the term get a decisive nudge (publication headers
+// contain the title words, so the reranker alone is unreliable here).
+import { THRESHOLDS } from "../config.ts";
+import type { Stage } from "./types.ts";
+
+export const termNudge: Stage = {
+  name: "term-nudge",
+  when: (c) => !!c.u?.term,
+  run: (c) => {
+    const scored = c.hits.map((h) => h.rerank_score ?? h.score);
+    const spread = Math.max(...scored) - Math.min(...scored);
+    if (spread > 0) {
+      const esc = c.u!.term!.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+      const termRe = new RegExp(`(^|[^a-z])${esc}([^a-z]|$)`, "i");
+      for (const h of c.hits) {
+        const body = h.text.split("\n").slice(1).join(" ").slice(0, 200);
+        const hay = `${h.metadata.clause_title || ""} ${body}`.toLowerCase();
+        if (termRe.test(hay)) h.rerank_score = (h.rerank_score ?? h.score) + spread * THRESHOLDS.termNudgeSpread;
+      }
+      c.hits.sort((a, b) => (b.rerank_score ?? -Infinity) - (a.rerank_score ?? -Infinity));
+    }
+  },
+};

package/workers/worker_public/src/stages/typedPin.ts

@@ -0,0 +1,131 @@
+// Answer contract v2 — typed-chunk pin (FINAL position): doc-scoped
+// queries get ONE typed unit chunk (table first) guaranteed a slot.
+// Prose outranks serialized tables under the cross-encoder AND the
+// per-doc diversity cap counts typed chunks against the same doc key —
+// without this guarantee the model never sees a unit id to reference.
+// Includes the small-to-big parent fetch: an embedded object answers
+// WITH its clause — if the parent clause's prose passage is not already
+// among the finals, one metadata-filtered fetch adds it. The typed unit
+// cites; the clause grounds.
+import { LIMITS, THRESHOLDS } from "../config.ts";
+import type { Hit } from "../../../shared/chunk";
+import type { Stage } from "./types.ts";
+
+/** Typed-chunk selection for the pin: among a doc's typed units pick the
+ *  one whose text best overlaps the QUERY (the first candidate is wrong as
+ *  often as right — annex example tables outrank nothing). Lexical-overlap
+ *  heuristic over title + serialized rows; tables, figures and formulas
+ *  compete on the same score so a figure question can pin the figure
+ *  (which then feeds multimodal generation), while table-value questions
+ *  still pin their table on overlap. When the query NAMES an artifact type
+ *  ("figure", "table", "formula"/"equation"), units of that type get a
+ *  dominating bonus — without it, clause units of the § tie with the
+ *  figure under rerank variance and the multimodal answer flips on luck. */
+function pickTypedChunk(query: string, candidates: Hit[], ranked: Hit[]): Hit | null {
+  if (!candidates.length) return null;
+  const pool = candidates;
+  const q = query.toLowerCase();
+  const terms = q.replace(/[^\p{L}\p{N}\s]/gu, " ").split(/\s+/).filter((t) => t.length > 2);
+  const typeBonus: Record<string, number> = {};
+  if (/\bfig(ure)?s?\b/.test(q)) typeBonus.figure = 1;
+  if (/\btables?\b/.test(q)) typeBonus.table = 1;
+  if (/\b(formulas?|equations?)\b/.test(q)) typeBonus.formula = 1;
+  // the top-ranked PROSE passage usually sits in the answer clause: a
+  // typed chunk from that same clause is the answering object, not a
+  // same-topic example from an annex
+  const topProse = ranked.find((h) => !h.metadata.unit_id);
+  const topAnchor = topProse?.metadata.clause_anchor ?? "";
+  let best: Hit | null = null;
+  let bestScore = -1;
+  for (const h of pool) {
+    const hay = `${h.metadata.clause_title ?? ""} ${h.text}`.toLowerCase();
+    let score = 0;
+    for (const t of terms) if (hay.includes(t)) score++;
+    if (typeBonus[h.metadata.block ?? ""]) score += terms.length * 2; // dominates
+    else if (topAnchor && h.metadata.clause_anchor === topAnchor) score += terms.length;
+    // blank annex FORMS (empty value cells) are not answer tables
+    const cells = h.text.split("|").map((x) => x.trim());
+    const filled = cells.filter((x) => x.length > 0).length;
+    const density = cells.length ? filled / cells.length : 0;
+    score += density * 2;
+    if (score > bestScore) {
+      bestScore = score;
+      best = h;
+    }
+  }
+  return best ?? pool[0];
+}
+
+export const typedPin: Stage = {
+  name: "typed-pin",
+  when: (c) => {
+    // family scope: the hard doc filter, else the understanding's
+    // family, else the UNION of the vocabulary link's candidate
+    // families — a value question can straddle families that define
+    // near-identical tables (n_LC class B = 5 000 exists in R 60-1 AND
+    // R 76-2); pickTypedChunk's overlap + top-prose-anchor scoring then
+    // picks the right table among them instead of the family choice
+    // deciding in advance
+    const glossaryFamilies = new Set<string>();
+    for (const g of c.glossary) if (g.doc_number) glossaryFamilies.add(g.doc_number.split("-")[0]);
+    const pinFamily = c.filters?.doc_number ?? c.u?.doc_number ?? null;
+    return new Set<string>(pinFamily ? [pinFamily.split("-")[0]] : [...glossaryFamilies]).size > 0;
+  },
+  run: async (c) => {
+    const { query, filters, u, glossary, hits, env, vector } = c;
+    const glossaryFamilies = new Set<string>();
+    for (const g of glossary) if (g.doc_number) glossaryFamilies.add(g.doc_number.split("-")[0]);
+    const pinFamily = filters?.doc_number ?? u?.doc_number ?? null;
+    const pinFamilies = new Set<string>(pinFamily ? [pinFamily.split("-")[0]] : [...glossaryFamilies]);
+    const base = (dn?: string) => String(dn ?? "").split("-")[0];
+    const sameDocTyped = (h: Hit) =>
+      !!h.metadata.unit_id && !!h.metadata.block && pinFamilies.has(base(h.metadata.doc_number));
+    {
+      // the pin guarantees the BEST query-overlap typed unit a slot — not
+      // merely "some" typed unit. Hard doc scope pins unconditionally
+      // (existing behavior); the glossary-family union (no hard scope)
+      // pins only on real query overlap — the picker always returns
+      // SOMETHING, and a near-zero-overlap table riding the window on
+      // every vocabulary-linked query would be pollution.
+      const typed = pickTypedChunk(query, hits.filter(sameDocTyped), hits);
+      const hardScope = !!pinFamily;
+      const overlap = (() => {
+        if (!typed || hardScope) return Infinity;
+        const terms = query.toLowerCase().replace(/[^\p{L}\p{N}\s]/gu, " ").split(/\s+/).filter((t) => t.length > 2);
+        const hay = `${typed.metadata.clause_title ?? ""} ${typed.text}`.toLowerCase();
+        return terms.filter((t) => hay.includes(t)).length;
+      })();
+      // tables are exempt from the overlap gate: a table in the window
+      // renders as a block and summarizes harmlessly — the pollution
+      // concern applies to other typed units, not to the one artifact
+      // the answer contract most needs to reach the user
+      const tableExempt = typed?.metadata.block === "table";
+      if (typed && (overlap >= 3 || tableExempt) && !c.finalHits.some((h) => h.id === typed.id)) {
+        c.finalHits = [...c.finalHits.slice(0, LIMITS.rerankKeep - 1), typed];
+        console.log("typed pin:", typed.metadata.docidentifier, "§", typed.metadata.clause_anchor, `(${typed.metadata.block})${hardScope ? "" : " [glossary families]"}`);
+
+        const anchor = typed.metadata.clause_anchor;
+        const docId = typed.metadata.doc_id;
+        const parentPresent = c.finalHits.some(
+          (h) => h.metadata.doc_id === docId && h.metadata.clause_anchor === anchor && !h.metadata.unit_id,
+        );
+        if (anchor && docId && !parentPresent) {
+          try {
+            const pv = await env.VECTORIZE.query(vector, {
+              topK: 4,
+              returnMetadata: "all",
+              filter: { $and: [{ doc_id: { $eq: docId } }, { clause_anchor: { $eq: anchor } }] },
+            });
+            const parent = (pv.matches ?? []).map((m: any) => ({ id: m.id, score: m.score, metadata: m.metadata, text: m.metadata?.chunk_text ?? "" })).find((h: any) => !h.metadata?.unit_id);
+            if (parent && !c.finalHits.some((h) => h.id === parent.id)) {
+              c.finalHits = [...c.finalHits, { ...parent, score: parent.score * THRESHOLDS.smallToBigDiscount }];
+              console.log("small-to-big: parent §", anchor, "of", typed.metadata.docidentifier, "added");
+            }
+          } catch {
+            // additive lane; primary results stand
+          }
+        }
+      }
+    }
+  },
+};

package/workers/worker_public/src/stages/types.ts

@@ -0,0 +1,112 @@
+// The pipeline stage contract (TODO.impl/02): every retrieval mechanism
+// is a self-contained stage implementing this interface, composed by the
+// registry in stages/index.ts. Adding a mechanism = adding a stage file
+// and one registry entry — no edits to existing stages, the runner, or
+// retrieve() (OCP). Contracts per stage: docs/spec-pipeline.md.
+
+import type { Hit } from "../../../shared/chunk";
+import type { QueryFilters } from "../selfquery.ts";
+import type { QueryUnderstanding } from "../understand";
+
+export interface RetrieveOptions {
+  prev?: string;
+  understanding?: QueryUnderstanding | null;
+  queryOverride?: string;
+  federate?: (query: string) => Promise<Hit[]>;
+  warmEmbed?: Promise<number[] | null>;
+  graphDocNumbers?: string[];
+  /** The declared context's HARD seal (TODO.ai-platform/02): when the
+   *  panel's chip declares a document scope, the CANDIDATE POOL is cut
+   *  to the publication family before rerank + the top-N cut — the
+   *  soft-steer widenings below (the full-corpus lexical union, the
+   *  sparse-filter widen, the sub-query lanes) can otherwise outscore
+   *  the filtered dense lane under the cross-encoder and push every
+   *  in-family passage out of the final hits, sealing the answer to
+   *  zero despite a healthy in-family pool. */
+  sealScope?: { doc_number: string; edition?: string } | null;
+  /** Option C: dense-lane results computed concurrently with
+   *  understanding (same folded-query vector, retrieve's exact query
+   *  parameters). With no filter they REPLACE the primary dense query;
+   *  with a filter they union in as discounted filter-miss cover. */
+  optimisticHits?: Hit[];
+  optimisticVec?: number[] | null;
+  /** Dataset scope (the sidebar toggles): the set of CORPUS values the
+   *  request allows. Present only when NARROWER than the default (all
+   *  permitted datasets) — a null scope means no filtering. The ask path
+   *  intersects the requested ids with session permissions before
+   *  building this set. */
+  datasetScope?: Set<string> | null;
+}
+
+export interface GlossaryEntry {
+  term: string;
+  definition: string;
+  docidentifier: string;
+  doc_number: string;
+  score: number;
+}
+
+/** The mutable state every stage reads and writes. Field ownership per
+ *  stage is specified in docs/spec-pipeline.md; in general: candidate
+ *  lanes append to `matches`, refinement stages rewrite `hits`, and only
+ *  the window-assembly stages touch `finalHits`. */
+export interface PipelineContext {
+  env: any;
+  query: string; // the user's original wording (rerank + pin score on it)
+  rq: string; // the retrieval query (folded / standalone / expanded)
+  folded: string; // the pre-understanding fold (optimistic-lane identity)
+  u: QueryUnderstanding | null;
+  filters: QueryFilters | null; // dense-stage may drop a guessed edition pin
+  filter: Record<string, string> | null | undefined; // toVectorizeFilter's output
+  vector: number[]; // embedding of rq
+  lexicalHits: Hit[]; // full-corpus BM25 ranking (already seal-filtered)
+  matches: any[]; // the candidate pool (Vectorize match shape, pre-Hit)
+  hits: Hit[]; // the ranked pool from poolOpen onward
+  finalHits: Hit[]; // the answer window
+  glossary: GlossaryEntry[]; // the vocabulary link (glossary stage owns)
+  opts: RetrieveOptions;
+  /** prefetch bag: stage-name → that stage's in-flight I/O promise (the
+   *  stage owns its key; see Stage.prefetch) */
+  lane: Record<string, Promise<unknown>>;
+}
+
+export interface Stage {
+  name: string;
+  /** Absent = always runs. Guards are pure reads of the context. */
+  when?: (c: PipelineContext) => boolean;
+  /** "additive": a throw is logged and the pipeline continues with the
+   *  context as the previous stage left it (the lane's results were not
+   *  written). "blocking" (default): the throw propagates to the caller. */
+  failure?: "additive" | "blocking";
+  /** Kick this stage's INDEPENDENT I/O off early (the runner invokes
+   *  every stage's prefetch before running any stage). Only for stages
+   *  whose I/O depends on pre-pipeline state (u, vector, opts) — never
+   *  on prior stages' output. The promise lands in c.lane[name]; run()
+   *  awaits it and merges. Merges stay in registry order — concurrency
+   *  changes when I/O completes, never the merge order (determinism). */
+  prefetch?: (c: PipelineContext) => void;
+  run: (c: PipelineContext) => Promise<void> | void;
+}
+
+/** Run the registry in order. Additive stages swallow their own throws —
+ *  replicating the per-lane try/catch the monolith carried inline. */
+export async function runStages(stages: Stage[], c: PipelineContext): Promise<void> {
+  for (const stage of stages) {
+    if (stage.prefetch && (!stage.when || stage.when(c))) stage.prefetch(c);
+  }
+  for (const stage of stages) {
+    if (stage.when && !stage.when(c)) continue;
+    if (stage.failure === "additive") {
+      try {
+        await stage.run(c);
+      } catch (e) {
+        console.log(`stage ${stage.name}: additive lane failed — primary results stand (${String(e).slice(0, 120)})`);
+      }
+    } else {
+      await stage.run(c);
+    }
+  }
+}
+
+// the shared match→Hit conversion (lib/hit.ts owns it)
+export { toHits } from "../lib/hit.ts";

package/workers/worker_public/src/stages/windowFloor.ts

@@ -0,0 +1,23 @@
+// Relevance-floored window (the evidence-budget principle): a full
+// window of near-miss passages costs tokens and attention while good
+// answers cite only what they need (measured: median 10 served, 1–2
+// cited). Serve what can matter: passages within a fraction of the top
+// score, plus every structurally-guaranteed unit — typed pins and the
+// passages appended after ranking (their scores are not cross-encoder
+// comparable). Never fewer than two.
+import { THRESHOLDS } from "../config.ts";
+import type { Stage } from "./types.ts";
+
+export const windowFloor: Stage = {
+  name: "window-floor",
+  run: (c) => {
+    const top = Math.max(...c.finalHits.map((h) => h.rerank_score ?? h.score));
+    const floored = c.finalHits.filter(
+      (h) => h.rerank_score === undefined || (h.rerank_score ?? h.score) >= THRESHOLDS.windowFloorFraction * top || !!h.metadata.unit_id || h.metadata.clause_anchor === "family",
+    );
+    if (floored.length >= 2) {
+      if (floored.length < c.finalHits.length) console.log("window floor:", c.finalHits.length, "→", floored.length, "passages");
+      c.finalHits = floored;
+    }
+  },
+};

package/workers/worker_public/src/structural.ts

@@ -0,0 +1,171 @@
+// Structural retrieval over the producer-native clause tree — serving-path
+// adaptations of FABLE/BEAR (arXiv:2601.18116) to a corpus that already HAS
+// the hierarchy: Metanorma clause anchors ARE the tree, so unlike FABLE no
+// LLM tree-builder runs at index time. Three techniques:
+//
+//   - structuralPropagation: TreeExpansion's relevance propagation — a
+//     clause blends its own score with its ancestors' (topic continuity)
+//     and descendants' (subtopic heat): sections whose clauses are
+//     collectively hot rise, hot sections lift their clauses.
+//   - positionOrder: NodeFusion's position-preserving ordering — evidence
+//     is presented in document reading order (per publication, groups by
+//     selection priority), because synthesis quality depends on
+//     arrangement, not just set membership.
+//   - ancestorDescendantDedup: near-duplicate chunks of the same clause
+//     chain (parent §3.1 vs child §3.1.2 repeating its heading + text)
+//     collapse to the stronger one before the window is cut.
+import type { Hit } from "./pipeline";
+
+/** "3.1.2" → [3,1,2]; null for everything else (annex labels, producer
+ *  UUIDs, overview/family, empty). */
+export function parseAnchor(anchor: string | undefined | null): number[] | null {
+  if (!anchor) return null;
+  const a = anchor.trim().replace(/\.$/, "");
+  if (!/^\d+(\.\d+)*$/.test(a)) return null;
+  return a.split(".").map(Number);
+}
+
+/** a is a PROPER ancestor of b ("3.1" ⊳ "3.1.2"). */
+export function isAncestorOf(a: number[], b: number[]): boolean {
+  return a.length < b.length && b.slice(0, a.length).every((s, i) => s === a[i]);
+}
+
+/** Document order for dotted numeric anchors ("3" < "3.1" < "3.1.2" < "3.2"). */
+export function anchorCompare(a: number[], b: number[]): number {
+  const n = Math.min(a.length, b.length);
+  for (let i = 0; i < n; i++) if (a[i] !== b[i]) return a[i] - b[i];
+  return a.length - b.length;
+}
+
+const scoreOf = (h: Hit) => h.rerank_score ?? h.score;
+
+/** TreeExpansion-style structural propagation (Eq. 7 of the paper):
+ *  S(v) = (self + inherited + childAgg)/3, blended into the live score as
+ *  a spread-scaled adjustment — same idiom as edition steering, so the
+ *  adjustment can never outrank the cross-encoder's own signal. */
+export function structuralPropagation(hits: Hit[]): Hit[] {
+  if (hits.length < 3) return hits;
+  const scored = hits.map(scoreOf);
+  const min = Math.min(...scored);
+  const max = Math.max(...scored);
+  const spread = max - min;
+  if (spread <= 0) return hits;
+
+  const byDoc = new Map<string, { h: Hit; a: number[]; n: number }[]>();
+  for (const h of hits) {
+    const a = parseAnchor(h.metadata.clause_anchor);
+    if (!a) continue;
+    const k = h.metadata.doc_id;
+    if (!byDoc.has(k)) byDoc.set(k, []);
+    byDoc.get(k)!.push({ h, a, n: (scoreOf(h) - min) / spread });
+  }
+
+  let adjusted = 0;
+  for (const nodes of byDoc.values()) {
+    if (nodes.length < 2) continue;
+    for (const nd of nodes) {
+      let inherited: number | null = null;
+      let childSum = 0;
+      let childN = 0;
+      for (const other of nodes) {
+        if (other === nd) continue;
+        if (isAncestorOf(other.a, nd.a)) inherited = Math.max(inherited ?? 0, other.n);
+        else if (isAncestorOf(nd.a, other.a)) {
+          childSum += other.n;
+          childN++;
+        }
+      }
+      if (inherited === null && childN === 0) continue;
+      const s = (nd.n + (inherited ?? nd.n) + (childN ? childSum / childN : nd.n)) / 3;
+      const adj = spread * 0.2 * (s - nd.n);
+      if (Math.abs(adj) < 1e-9) continue;
+      if (nd.h.rerank_score !== undefined) nd.h.rerank_score += adj;
+      else nd.h.score += adj;
+      adjusted++;
+    }
+  }
+  if (adjusted) {
+    console.log("structural propagation:", adjusted, "hits re-scored across the clause tree");
+    hits.sort((a, b) => scoreOf(b) - scoreOf(a));
+  }
+  return hits;
+}
+
+/** Position-preserving evidence order (NodeFusion, Algorithm 2): passages
+ *  of the same publication are fed in document order, publications ordered
+ *  by their best-ranked member. Document order is the PRODUCER'S ordinal
+ *  when the metadata carries one (metanorma-document#56) — a sort, never
+ *  an anchor parse; the anchor compare is the fallback for chunks whose
+ *  producer doesn't emit ordinals. Structural chunks (overview/family)
+ *  lead their doc; unnumbered passages follow the numbered ones. */
+export function positionOrder(hits: Hit[]): Hit[] {
+  if (hits.length < 3) return hits;
+  const idx = new Map(hits.map((h, i) => [h, i]));
+  const groups = new Map<string, Hit[]>();
+  for (const h of hits) {
+    const k = h.metadata.doc_id || h.id;
+    if (!groups.has(k)) groups.set(k, []);
+    groups.get(k)!.push(h);
+  }
+  const rank = (g: Hit[]) => Math.min(...g.map((h) => idx.get(h)!));
+  const structural = (h: Hit) => h.metadata.clause_anchor === "overview" || h.metadata.clause_anchor === "family";
+  const byOrig = (a: Hit, b: Hit) => idx.get(a)! - idx.get(b)!;
+  const byDocOrder = (a: Hit, b: Hit) => {
+    const oa = (a.metadata as any).ordinal;
+    const ob = (b.metadata as any).ordinal;
+    if (typeof oa === "number" && typeof ob === "number" && oa !== ob) return oa - ob;
+    const pa = parseAnchor(a.metadata.clause_anchor);
+    const pb = parseAnchor(b.metadata.clause_anchor);
+    if (pa && pb) return anchorCompare(pa, pb) || byOrig(a, b);
+    if (pa && !pb) return -1;
+    if (!pa && pb) return 1;
+    return byOrig(a, b);
+  };
+
+  const out: Hit[] = [];
+  for (const g of [...groups.values()].sort((a, b) => rank(a) - rank(b))) {
+    const head = g.filter(structural).sort(byOrig);
+    const ordered = g.filter((h) => !structural(h)).sort(byDocOrder);
+    out.push(...head, ...ordered);
+  }
+  return out;
+}
+
+const headText = (h: Hit) => h.text.replace(/\s+/g, " ").toLowerCase().slice(0, 600);
+
+function overlap(a: string, b: string): number {
+  const A = new Set(a.split(/[^a-z0-9°%]+/).filter((t) => t.length > 3));
+  const B = new Set(b.split(/[^a-z0-9°%]+/).filter((t) => t.length > 3));
+  if (!A.size || !B.size) return 0;
+  let inter = 0;
+  for (const t of A) if (B.has(t)) inter++;
+  return inter / (A.size + B.size - inter);
+}
+
+/** Same-chain near-duplicate collapse: when an ancestor chunk and a
+ *  descendant chunk of one clause chain carry substantially the same text,
+ *  the weaker one leaves the window (FABLE keeps the subtree, drops the
+ *  redundant node). Different-text relatives both stay — a parent clause
+ *  and a deep sub-clause are usually different content. */
+export function ancestorDescendantDedup(hits: Hit[]): Hit[] {
+  if (hits.length < 2) return hits;
+  const anchors = hits.map((h) => parseAnchor(h.metadata.clause_anchor));
+  const drop = new Set<Hit>();
+  for (let i = 0; i < hits.length; i++) {
+    if (!anchors[i] || drop.has(hits[i])) continue;
+    for (let j = i + 1; j < hits.length; j++) {
+      if (!anchors[j] || drop.has(hits[j])) continue;
+      if (hits[i].metadata.doc_id !== hits[j].metadata.doc_id) continue;
+      const chained = isAncestorOf(anchors[i]!, anchors[j]!) || isAncestorOf(anchors[j]!, anchors[i]!);
+      if (!chained) continue;
+      if (overlap(headText(hits[i]), headText(hits[j])) >= 0.5) {
+        drop.add(scoreOf(hits[i]) >= scoreOf(hits[j]) ? hits[j] : hits[i]);
+      }
+    }
+  }
+  if (drop.size) {
+    console.log("structural dedup:", drop.size, "same-chain near-duplicate(s) dropped");
+    return hits.filter((h) => !drop.has(h));
+  }
+  return hits;
+}

package/workers/worker_public/src/tablecontext.ts

@@ -0,0 +1,41 @@
+/** Schema-aware table context composition (TableRAG-class cell
+ *  selection) over the producer's typed table payload. Pure — the
+ *  interface is the test surface. */
+/** Schema-aware table context (TableRAG-class cell selection): the
+ *  producer payload (metadata.table) carries caption/columns/rows; the
+ *  consumer composes the model-facing serialization — columns whose
+ *  labels overlap the query, rows whose cells overlap the query or the
+ *  selected column labels. Full table stays available for rendering;
+ *  this only shapes the prompt context, and falls back to the stored
+ *  text when pruning matches nothing (never worse than baseline). */
+export function tableContext(meta: any, query: string): string | null {
+  const t: any = meta?.table;
+  if (!t || !Array.isArray(t.columns) || !Array.isArray(t.rows) || !t.rows.length) return null;
+  const terms = new Set(
+    query.toLowerCase().replace(/[^\p{L}\p{N}\s]/gu, " ").split(/\s+/).filter((w: string) => w.length > 2),
+  );
+  const termList = [...terms];
+  const label = (c: any): string => `${c?.label ?? ""} ${c?.unit ?? ""}`.toLowerCase();
+  const keepCols: number[] = [];
+  t.columns.forEach((c: any, i: number) => {
+    if (termList.some((term) => label(c).includes(term))) keepCols.push(i);
+  });
+  const colKeep: number[] = keepCols.length ? keepCols : t.columns.map((_: any, i: number) => i);
+  const rowHits: string[] = [];
+  for (const row of t.rows as string[]) {
+    const cells = String(row).split("|").map((c: string) => c.trim().toLowerCase());
+    const cellHit = cells.some((c: string) => c && termList.some((term) => c.includes(term)));
+    const colHit = keepCols.length > 0 && colKeep.some((i: number) => cells[i] && termList.some((term) => label(t.columns[i]).includes(term) && cells[i].length > 0));
+    if (cellHit || colHit) rowHits.push(row);
+  }
+  if (!rowHits.length) return null;
+  const CAP = 10;
+  const shown = rowHits.slice(0, CAP);
+  const header = `Table: ${t.caption ?? ""}\ncolumns: ${colKeep.map((i: number) => `${t.columns[i]?.label ?? ""}${t.columns[i]?.unit ? ` [${t.columns[i].unit}]` : ""}`).join(" | ")}`;
+  const lines = shown.map((r: string) => `row: ${r}`);
+  const elided =
+    rowHits.length > CAP || rowHits.length < t.rows.length
+      ? `\n(${shown.length} of ${t.rows.length} rows shown; ${t.rows.length - rowHits.length} rows did not match the question terms)`
+      : "";
+  return `${header}\n${lines.join("\n")}${elided}`;
+}

package/workers/worker_public/src/understand.ts

@@ -0,0 +1,72 @@
+import type { ModelRunner } from "./ports/model.ts";
+import { extractJson, type QueryUnderstanding } from "./understandContract.ts";
+export type { QueryUnderstanding };
+
+// Query understanding (TODO.impl/15 audit finding #1): a small, fast LLM
+// call replaces the regex heuristics as the decision-maker for what
+// retrieval should see. The regex path (selfquery.ts) remains as a
+// fallback when this call fails, errors, or times out — degraded mode,
+// never a hard failure.
+
+// The prompt is data (prompts/understanding.md), bundled as text.
+import SYSTEM from "../prompts/understanding.md";
+
+/** Understand the query with the cheap model. Null = use the regex fallback. */
+export async function understandQuery(
+  ai: ModelRunner,
+  model: string,
+  query: string,
+  history: Array<{ role: string; content: string }>,
+  entities: Array<{ entity: string; kind: string }> = [],
+): Promise<QueryUnderstanding | null> {
+  const convo = history
+    .slice(-6)
+    .map((h) => `${h.role === "user" ? "User" : "Assistant"}: ${h.content.slice(0, 600)}`)
+    .join("\n");
+  const entityLine = entities.length
+    ? `Entities already established in this conversation: ${entities.map((e) => e.entity).join("; ")}. Resolve pronouns and shorthand against these.\n\n`
+    : "";
+  const user = `${convo ? "Conversation so far:\n" + convo + "\n\n" : ""}${entityLine}Question: ${query}`;
+  const body = {
+    messages: [
+      { role: "system", content: SYSTEM },
+      { role: "user", content: user },
+    ],
+    // the model always reasons; reasoning tokens share this budget — too
+    // small and the JSON is never reached (understanding silently degrades).
+    // GLM-5 family defaults to reasoning_effort "max" when the parameter is
+    // not honored, so GLM needs headroom or reasoning starves the JSON.
+    max_tokens: model.includes("glm") ? 3072 : 1500,
+    reasoning_effort: "low",
+    // Qwen3 thinking-mode sampling (model card): greedy/1.0 sampling
+    // degrades into repetition loops — the 10s/5s timeout nulls were the
+    // budget being eaten by loops, not by reasoning
+    temperature: 0.6,
+    top_p: 0.95,
+    top_k: 20,
+  };
+  // each attempt issues a FRESH call — re-racing a timed-out promise would
+  // retry nothing. Generous first attempt: reasoning + the full JSON must
+  // fit inside the timeout or understanding silently degrades to vanilla
+  // retrieval (which refuses conversational turns).
+  const ATTEMPT_TIMEOUTS = [10000, 5000];
+  for (let attempt = 0; attempt < ATTEMPT_TIMEOUTS.length; attempt++) {
+    const call = (async () => {
+      const res = await ai.run({ model, messages: (body as any).messages, effort: (body as any).reasoning_effort, maxTokens: (body as any).max_tokens, temperature: (body as any).temperature, topP: (body as any).top_p, topK: (body as any).top_k });
+      const text = res?.text ?? null;
+      return typeof text === "string" ? extractJson(text) : null;
+    })();
+    const timeout = new Promise<null>((r) => setTimeout(() => r(null), ATTEMPT_TIMEOUTS[attempt]));
+    try {
+      const got = await Promise.race([call, timeout]);
+      if (got) return got;
+    } catch (e) {
+      // account rate-limited: a retry in the same minute will also fail —
+      // degrade to vanilla retrieval immediately instead of burning the
+      // second attempt (TTFT surgery)
+      if (String(e).includes("3021") || String(e).includes("rate")) return null;
+    }
+  }
+  console.warn("query understanding unavailable — vanilla retrieval");
+  return null;
+}

package/workers/worker_public/src/understandContract.ts

@@ -0,0 +1,67 @@
+// The understanding contract (TODO.impl/29): QueryUnderstanding and the
+// pure JSON extractor — no imports (the prompt file and the model call
+// stay in understand.ts; this module is the testable contract).
+
+export interface QueryUnderstanding {
+  /** conversational turn (greeting, identity, small talk) vs knowledge seek */
+  intent: "conversational" | "knowledge";
+  /** normalized document reference, e.g. "OIML R 76-2" — null when none */
+  docidentifier: string | null;
+  /** base document number for the Vectorize filter, e.g. "60" */
+  doc_number: string | null;
+  edition?: string | null;
+  language?: string | null;
+  /** the question is about a process around publications (certify, apply…) */
+  process_intent: boolean;
+  /** definition-style question whose subject is `term` */
+  term: string | null;
+  /** corpus-terminology mapping of everyday wording (drift→creep) */
+  defined_terms: string[];
+  /** self-contained retrieval query: follow-ups folded with context */
+  standalone_query: string;
+  complexity: "simple" | "complex";
+  query_variants: string[];
+  sub_queries: string[];
+  hypothetical_answer: string;
+  /** plausible next questions (conversational UX), in the user's language */
+  follow_ups: string[];
+}
+
+
+/** Model output text
+ *  → QueryUnderstanding (or null). Every silent coercion is pinned by
+ *  tests/understand.test.ts — change the prompt's JSON shape and the
+ *  test names what moved. */
+export function extractJson(text: string): QueryUnderstanding | null {
+  const m = text.match(/\{[\s\S]*\}/);
+  if (!m) return null;
+  try {
+    const raw = JSON.parse(m[0]);
+    const u: QueryUnderstanding = {
+      intent: raw.intent === "conversational" ? ("conversational" as const) : ("knowledge" as const),
+      docidentifier: typeof raw.docidentifier === "string" && raw.docidentifier.trim() ? raw.docidentifier.trim().slice(0, 60) : null,
+      doc_number: typeof raw.docnumber === "string" && /^\d{1,3}$/.test(raw.docnumber) ? raw.docnumber : null,
+      edition: typeof raw.edition === "string" && /^\d{4}$/.test(raw.edition) ? raw.edition : null,
+      language: typeof raw.language === "string" && /^[a-z]{2}$/.test(raw.language) ? raw.language : null,
+      process_intent: raw.process_intent === true,
+      term: typeof raw.term === "string" && raw.term.trim() ? raw.term.trim().slice(0, 60) : null,
+      defined_terms: Array.isArray(raw.defined_terms)
+        ? raw.defined_terms.filter((t: unknown) => typeof t === "string" && (t as string).trim()).map((t: string) => t.trim().slice(0, 60)).slice(0, 4)
+        : [],
+      standalone_query: typeof raw.standalone_query === "string" && raw.standalone_query.trim() ? raw.standalone_query.trim().slice(0, 400) : "",
+      complexity: raw.complexity === 'complex' ? ('complex' as const) : ('simple' as const),
+      query_variants: Array.isArray(raw.query_variants)
+        ? raw.query_variants.filter((q: unknown) => typeof q === 'string' && (q as string).trim()).map((q: string) => q.trim().slice(0, 300)).slice(0, 4)
+        : [],
+      hypothetical_answer: typeof raw.hypothetical_answer === "string" ? raw.hypothetical_answer.trim().slice(0, 300) : "",
+      sub_queries: Array.isArray(raw.sub_queries)
+        ? raw.sub_queries.filter((q: unknown) => typeof q === 'string' && (q as string).trim()).map((q: string) => q.trim().slice(0, 300)).slice(0, 5)
+        : [],
+      follow_ups: Array.isArray(raw.follow_ups)
+        ? raw.follow_ups.filter((q: unknown) => typeof q === 'string' && (q as string).trim()).map((q: string) => q.trim().slice(0, 200)).slice(0, 2)
+        : [],    };
+    return u;
+  } catch {
+    return null;
+  }
+}