@konneal/engine 0.1.0

package/workers/worker_public/src/research.ts

@@ -0,0 +1,126 @@
+// Deep-research dossier loop (G10): bounded agentic iterations,
+// members-only — the workflow-shaped research spend (TODO.impl/23).
+import { err, json, corsHeaders, readJson, validateQuery } from "./lib/http";
+import { portModelRunner } from "./env.ts";
+import { telemetry } from "./quota";
+import { LIMITS, MODELS, sha256Hex } from "./config";
+import { retrieve, buildMessages, citations } from "./pipeline";
+import { understandQuery } from "./understand";
+import { generateOnce } from "./ai";
+import { canonicalRefusal } from "./refusal";
+import { checkQuoteAnchors } from "./anchors";
+import { graphExpand, editionNote } from "./graph";
+import researchPromptText from "../prompts/research.md";
+import type { Env } from "./env";
+import type { Hit } from "../../shared/chunk.ts";
+
+/** Deep-research mode (G10 v1): bounded agentic loop for members —
+ *  retrieve → sufficiency judge → re-retrieve targeting the gap → answer
+ *  from the ACCUMULATED evidence. ≤ max_iterations rounds; every
+ *  iteration's retrieval goes through the same gated pipeline as a
+ *  normal ask. Workflows (durable, resumable) is the documented upgrade
+ *  path when runs outgrow a single request. */
+export async function handleResearch(env: Env, ctx: ExecutionContext, req: Request, session: any): Promise<Response> {
+  if (!session) {
+    return err(403, "forbidden", "Deep research is a member feature — sign in with your OIML SMART account.");
+  }
+  const body = await readJson(req);
+  const q = validateQuery(body);
+  if (!q) return err(400, "invalid_input", `query is required (1-${LIMITS.maxInputChars} chars)`);
+  const maxIters = Math.min(Math.max(Number(body?.max_iterations) || 3, 1), 3);
+
+  const started = Date.now();
+  const queryHash = await sha256Hex(q.query);
+  const understanding = await understandQuery(portModelRunner(env), MODELS.understand, q.query, [], []);
+  const graphDocNumbers = await graphExpand(env, understanding);
+  const eNote = await editionNote(env, understanding);
+
+  const accumulated = new Map<string, Hit>();
+  let iterations = 0;
+  let focus = understanding?.standalone_query?.trim() || q.query;
+  let judge: { sufficient: boolean; missing: string } | null = null;
+
+  for (let i = 0; i < maxIters; i++) {
+    iterations = i + 1;
+    let retrieved: { hits: Hit[] };
+    try {
+      retrieved = await retrieve(env, q.query, {
+        understanding: i === 0 ? understanding : ({ ...understanding, standalone_query: focus, query_variants: [], hypothetical_answer: undefined } as any),
+        graphDocNumbers,
+      });
+    } catch {
+      break;
+    }
+    for (const h of retrieved.hits.slice(0, LIMITS.rerankKeep)) {
+      if (!accumulated.has(h.id)) accumulated.set(h.id, h);
+    }
+    const passages = [...accumulated.values()];
+    // Hierarchical context management (GLM-5 report, their search agents):
+    // the judge re-reads the full evidence every round and its context
+    // grows without bound. Keep-recent-k: the k most recent findings at
+    // full length, everything older as one-line digests. The final ANSWER
+    // generation below still sees the full set within the token budget —
+    // folding is judge-context only.
+    const KEEP_RECENT = 10;
+    const older = passages.slice(0, Math.max(0, passages.length - KEEP_RECENT));
+    const recent = passages.slice(-KEEP_RECENT);
+    const digest = older.length
+      ? `Earlier evidence (digest, ${older.length} passages):\n${older.map((h) => `- ${h.metadata.docidentifier ?? ""} §${h.metadata.clause_anchor ?? ""}: ${h.text.replace(/\s+/g, " ").slice(0, 160)}`).join("\n")}\n\n`
+      : "";
+    judge = await (async () => {
+      try {
+        const res: any = await env.AI.run(MODELS.grader, {
+          messages: [
+            { role: "system", content: researchPromptText.trimEnd() },
+            { role: "user", content: `Research question: ${q.query}\n\n${digest}Collected passages (${recent.length}):\n${recent.map((h, n) => `[${n + 1}] ${h.metadata.docidentifier ?? ""} §${h.metadata.clause_anchor ?? ""}: ${h.text.slice(0, 700)}`).join("\n")}` },
+          ],
+          max_tokens: 3072,
+          reasoning_effort: "low",
+          temperature: 1.0,
+          top_p: 1.0,
+        });
+        const text = typeof res?.response === "string" ? res.response : res?.choices?.[0]?.message?.content;
+        let parsed: any = null;
+        for (const m of (text ?? "").matchAll(/\{[^{}]*\}/g)) {
+          try {
+            const obj = JSON.parse(m[0]);
+            if (typeof obj.sufficient === "boolean") parsed = obj;
+          } catch { /* keep scanning */ }
+        }
+        return parsed ? { sufficient: parsed.sufficient, missing: String(parsed.missing ?? "") } : null;
+      } catch {
+        return null;
+      }
+    })();
+    console.log("research iter", iterations, "passages", passages.length, "sufficient:", judge?.sufficient);
+    if (!judge || judge.sufficient || !judge.missing) break;
+    // fold, don't accumulate: appending every round's `missing` compounds
+    // stale wants; the next retrieval focuses on the ORIGINAL question plus
+    // what is still missing now
+    focus = `${understanding?.standalone_query?.trim() || q.query} ${judge.missing}`.slice(0, LIMITS.maxInputChars);
+  }
+
+  const used = [...accumulated.values()];
+  if (!used.length) {
+    return err(503, "retrieval_unavailable", "Search is briefly busy — please retry in a moment.");
+  }
+  const { messages, usedHits } = buildMessages(q.query, used, q.lang, [], eNote || undefined, undefined, LIMITS.inputTokenBudget);
+  let answer = await generateOnce(env, MODELS.research, messages);
+  if (answer === null) answer = await generateOnce(env, MODELS.fallback, messages);
+  if (answer === null) {
+    telemetry(env, ctx, "member", "research", MODELS.research, false, 0, queryHash, q.lang);
+    return err(502, "generation_failed", "The generation model is unavailable; please retry.");
+  }
+  answer = canonicalRefusal(answer);
+  const anchors = checkQuoteAnchors(answer, used.map((h: Hit) => h.text));
+  if (anchors.violations.length) console.log("research anchors:", anchors.violations.length, "unverified");
+  const out = {
+    answer,
+    citations: citations(usedHits),
+    model: MODELS.research,
+    query_hash: queryHash,
+    research: { iterations, passages: used.length, elapsed_ms: Date.now() - started, sufficient: judge?.sufficient ?? null },
+  };
+  telemetry(env, ctx, "member", "research", MODELS.research, true, answer.length, queryHash, q.lang);
+  return json({ ...out, ...corsHeaders(req) });
+}

package/workers/worker_public/src/search.ts

@@ -0,0 +1,56 @@
+// Passage search: retrieval without generation (TODO.impl/23).
+import { LIMITS, MODELS, num, sha256Hex } from "./config";
+import { portModelRunner } from "./env.ts";
+import type { Background } from "./ports/runtime.ts";
+import { retrieve } from "./pipeline";
+import { understandQuery } from "./understand";
+import { sessionFrom } from "./auth";
+import { err, json, corsHeaders, readJson, validateQuery, type ApiKey } from "./lib/http";
+import { checkQuota, clientIp, telemetry } from "./quota";
+import { graphExpand } from "./graph";
+import type { Env } from "./env";
+import type { Hit } from "../../shared/chunk.ts";
+
+export async function handleSearch(
+  env: Env,
+  ctx: Background,
+  req: Request,
+  tier: "anon" | "key" | "member",
+  key: ApiKey | null,
+): Promise<Response> {
+  const body = await readJson(req);
+  const q = validateQuery(body);
+  if (!q) return err(400, "invalid_input", `query is required (1-${LIMITS.maxInputChars} chars)`);
+
+  const member = tier === "member" ? await sessionFrom(req, env as any) : null;
+  const limit = tier === "key" || member ? Number.MAX_SAFE_INTEGER : num(env as any, "ANON_DAY_SEARCH", 50);
+  const bucketId = tier === "key" ? `key:${key!.id}` : member ? `sub:${member.sub}` : clientIp(req);
+  const quota = await checkQuota(env, "search", bucketId, limit);
+  if (!quota.ok) {
+    return err(429, "quota_exceeded", `Daily search limit reached (${quota.limit}). Try again tomorrow.`);
+  }
+
+  const understanding = await understandQuery(portModelRunner(env), MODELS.understand, q.query, []);
+  const graphDocNumbers = await graphExpand(env, understanding);
+  let retrieved;
+  try {
+    retrieved = await retrieve(env, q.query, { understanding, graphDocNumbers });
+  } catch {
+    return err(503, "retrieval_unavailable", "Search is briefly busy — please retry in a moment.");
+  }
+  const { hits, filters } = retrieved;
+  const results = hits.map((h: Hit) => ({
+    doc_id: h.metadata.doc_id,
+    docidentifier: h.metadata.docidentifier,
+    edition: h.metadata.edition,
+    language: h.metadata.language,
+    clause_anchor: h.metadata.clause_anchor,
+    clause_title: h.metadata.clause_title,
+    status: h.metadata.status ?? "unknown",
+    superseded_by: h.metadata.superseded_by || undefined,
+    text: h.text,
+    score: h.rerank_score ?? h.score,
+  }));
+  telemetry(env, ctx, tier, "search", MODELS.embed, true, 0, await sha256Hex(q.query), q.lang);
+  return json({ results, filters, quota, ...corsHeaders(req) });
+}

package/workers/worker_public/src/selfquery.ts

@@ -0,0 +1,25 @@
+// Filter types + Vectorize filter conversion for the LLM understanding
+// output. The regex filter floor that used to live here was removed:
+// doc-number/edition/process-intent are SEMANTIC judgments about the
+// user's query, and semantics are decided by query understanding
+// (understanding.ts) — never by string matching. When understanding is
+// unavailable, retrieval runs unfiltered rather than regex-filtered.
+
+export interface QueryFilters {
+  doctype?: string;
+  doc_number?: string;
+  edition?: string;
+  language?: string;
+}
+
+export function toVectorizeFilter(f: QueryFilters): Record<string, string> | undefined {
+  // a pinned doc number is near-selective on its own; including doctype
+  // hides the dirty-corpus docs whose identifiers lost the series letter
+  // ("OIML 106"), and the reranker resolves R/D number collisions
+  if (f.doc_number) {
+    const out: Record<string, string> = { doc_number: f.doc_number };
+    if (f.edition) out.edition = f.edition;
+    return out; // language filter omitted: the index is English-only
+  }
+  return undefined;
+}

package/workers/worker_public/src/session.ts

@@ -0,0 +1,4 @@
+// Compat re-export: the session implementation lives in
+// workers/shared/session.ts (TODO.impl/16) — the import surface of this
+// path is unchanged for worker_public's modules.
+export * from "../../shared/session.ts";

package/workers/worker_public/src/share.ts

@@ -0,0 +1,53 @@
+// Public read-only shared conversations (TODO.rag/09).
+
+import { LIMITS } from "./config";
+import { json, err } from "./lib/http";
+
+const SLUG_RE = /^[a-z0-9]{10}$/;
+
+function makeSlug(): string {
+  const bytes = crypto.getRandomValues(new Uint8Array(8));
+  return [...bytes].map((b) => b.toString(16).padStart(2, "0")).join("").slice(0, 10);
+}
+
+export async function handleShareConversation(
+  env: any,
+  ownerSub: string,
+  title: string,
+  messages: any[],
+): Promise<Response> {
+  if (messages.length === 0 || messages.length > 100) {
+    return err(400, "invalid_input", "Cannot share an empty or oversized conversation");
+  }
+  // rate limit: 10 shares/day per user
+  const day = new Date().toISOString().slice(0, 10);
+  const count = Number((await env.CACHE.get(`sh:${day}:${ownerSub.slice(0, 20)}`)) ?? "0");
+  if (count >= 10) return err(429, "rate_limited", "Daily share limit reached");
+  await env.CACHE.put(`sh:${day}:${ownerSub.slice(0, 20)}`, String(count + 1), { expirationTtl: 90000 });
+
+  const slug = makeSlug();
+  const cleanMessages = messages.slice(0, 50).map((m: any) => ({
+    role: m.role === "user" ? "user" : "assistant",
+    content: (m.content ?? "").slice(0, LIMITS.maxOutputTokens * 2),
+    citations: m.citations ? JSON.parse(m.citations) : null,
+  }));
+  await env.DB.prepare(
+    "INSERT INTO shared_conversations (slug, owner_sub, title, messages, created_at) VALUES (?1,?2,?3,?4,?5)",
+  )
+    .bind(slug, ownerSub, title.slice(0, 120), JSON.stringify(cleanMessages), new Date().toISOString())
+    .run();
+  return json({ slug, url: `/c/${slug}` }, 201);
+}
+
+export async function handleGetShared(env: any, slug: string): Promise<Response> {
+  if (!SLUG_RE.test(slug)) return err(400, "invalid_input", "Invalid link");
+  const row = await env.DB.prepare("SELECT title, messages, created_at FROM shared_conversations WHERE slug = ?1")
+    .bind(slug)
+    .first();
+  if (!row) return err(404, "not_found", "This shared link has expired or was removed");
+  return json({
+    title: (row as any).title,
+    created_at: (row as any).created_at,
+    messages: JSON.parse((row as any).messages),
+  });
+}

package/workers/worker_public/src/stages/conceptGraph.ts

@@ -0,0 +1,68 @@
+// Concept-graph lane (additive). The D1 projection (relaton structure +
+// Glossarist defines edges) resolves what the query's WORDS map to in
+// the corpus's own structure: a defined term → the documents that
+// define it; a named publication → its family/successors. Same query
+// vector, graph-filtered candidates — vocabulary mismatch stops
+// mattering when the graph carries the link. Path A (rag#137): the
+// concept link drives the SAME resolver — linked TERM → defining
+// documents. Unlike the reverted candidate-family routing (every chunk
+// of every candidate's family, which flooded the pool with the wrong
+// domain), this is narrow: the definitional clauses of the linked
+// concepts. "actual quantity" (defined by R 87) brings the prepackage
+// domain in without the weigh-labeler families.
+//
+// NOTE: candidate-family pool routing was measured and REVERTED (rag#137
+// experiment, 2026-09-05): the wrong-domain candidates' families (the
+// labeler concepts, R 51/R 45) merged at the same discount and DILUTED
+// the pool — canary fell 3/6 → 1/6. Candidate-level families cannot
+// adjudicate domains; only the answer model can (it does, when the
+// family's content reaches it).
+import { THRESHOLDS } from "../config.ts";
+import type { Stage } from "./types.ts";
+
+export const conceptGraph: Stage = {
+  name: "concept-graph",
+  failure: "additive",
+  when: (c) => c.glossary.length > 0 && !!c.env.DB && c.vector.length > 0,
+  run: async (c) => {
+    const numbers = new Set<string>();
+    // the terms are independent — resolve them in PARALLEL; a serial loop
+    // paid up to 3 sequential D1 round trips on the hot path
+    const termRows = await Promise.all(
+      c.glossary
+        .slice(0, 3)
+        .filter((gl) => gl.term.length >= 3)
+        .map((gl) =>
+          c.env.DB.prepare(
+            "SELECT e.src AS doc FROM graph_edges e JOIN graph_nodes c ON e.dst = c.id WHERE e.kind = 'defines' AND c.kind = 'concept' AND (c.label = ?1 OR c.label LIKE ?2) LIMIT 12",
+          )
+            .bind(gl.term, `%${gl.term}%`)
+            .all()
+            .catch(() => ({ results: [] })) as Promise<any>,
+        ),
+    );
+    for (const rows of termRows) {
+      for (const r of rows.results ?? []) {
+        const m = String(r.doc ?? "").match(/^doc:OIML-[A-Z]-(\d+)-/);
+        if (m) numbers.add(m[1]!);
+      }
+    }
+    if (numbers.size) {
+      const gc = await c.env.VECTORIZE.query(c.vector, {
+        topK: 12,
+        returnMetadata: "all",
+        filter: { doc_number: { $in: [...numbers] } },
+      });
+      const seenIds0 = new Set(c.matches.map((m: any) => m.id));
+      let merged0 = 0;
+      for (const m of (gc.matches ?? []).slice(0, 6)) {
+        if (!seenIds0.has(m.id)) {
+          c.matches.push({ id: m.id, score: m.score * THRESHOLDS.conceptGraphDiscount, metadata: m.metadata });
+          seenIds0.add(m.id);
+          merged0++;
+        }
+      }
+      if (merged0) console.log("concept graph:", [...numbers].join(","), "— merged", merged0);
+    }
+  },
+};

package/workers/worker_public/src/stages/conceptSteer.ts

@@ -0,0 +1,39 @@
+// Concept-anchored steering (rag#137): the vocabulary link names the
+// corpus's own concepts for the question's subject; their DEFINING
+// publications get a small rerank boost — the same spread-scaled
+// steering idiom as edition steering. Measured motivation: for
+// everyday-goods phrasing the right family's chunks sit at dense rank 3
+// and are then BURIED by the cross-encoder under instrument-markings
+// vocabulary (R 111/R 76 "label" clauses outrank R 79's overview for
+// "rules for the label on a bag of flour"). The concept link knows the
+// subject's family ("actual quantity" → R 87, "Prepackage" → R 79); the
+// boost is what lets that knowledge survive the cross-encoder. The
+// answer model still adjudicates among the boosted families' passages.
+import { THRESHOLDS } from "../config.ts";
+import type { Stage } from "./types.ts";
+
+export const conceptSteer: Stage = {
+  name: "concept-steer",
+  when: (c) => c.glossary.length > 0 && c.hits.length > 1,
+  run: (c) => {
+    const fams = new Set(c.glossary.map((g) => g.doc_number.split("-")[0]).filter(Boolean));
+    if (fams.size) {
+      const scored = c.hits.map((h) => h.rerank_score ?? h.score);
+      const spread = Math.max(...scored) - Math.min(...scored);
+      if (spread > 0) {
+        let boosted = 0;
+        for (const h of c.hits) {
+          const base = String(h.metadata.doc_number ?? "").split("-")[0];
+          if (fams.has(base)) {
+            h.rerank_score = (h.rerank_score ?? h.score) + spread * THRESHOLDS.conceptSteerSpread;
+            boosted++;
+          }
+        }
+        if (boosted) {
+          console.log("concept steering: +", boosted, "hits in", [...fams].join(","));
+          c.hits.sort((a, b) => (b.rerank_score ?? -Infinity) - (a.rerank_score ?? -Infinity));
+        }
+      }
+    }
+  },
+};

package/workers/worker_public/src/stages/corpusScope.ts

@@ -0,0 +1,25 @@
+// Dataset scope (the sidebar toggles): hits whose corpus lies outside
+// the request's enabled datasets drop from the pool before the window
+// is cut — disabling a database must actually disable it, not just hide
+// its UI row. Runs LAST of the pool-assembly stages: after lexicalRrf,
+// which unions the lexical lane's separate hit list into the pool AFTER
+// rerank (filtering earlier let lexical hits back in — measured). Corpora
+// the toggle model doesn't name (serving lanes, unknown values) pass
+// untouched: the toggles name DATASETS, not lanes.
+import type { Stage } from "./types.ts";
+
+const DATASET_CORPORA = new Set(["oiml", "dirty", "clean", "synthetic", "smart-model", "iso-internal"]);
+
+export const corpusScope: Stage = {
+  name: "corpus-scope",
+  when: (c) => !!c.opts.datasetScope && c.opts.datasetScope.size > 0,
+  run: (c) => {
+    const before = c.hits.length;
+    c.hits = c.hits.filter((h) => {
+      const corpus = h.metadata.corpus;
+      if (!corpus || !DATASET_CORPORA.has(corpus)) return true;
+      return c.opts.datasetScope!.has(corpus);
+    });
+    if (c.hits.length !== before) console.log("corpus scope:", before, "→", c.hits.length, "candidates");
+  },
+};

package/workers/worker_public/src/stages/dedup.ts

@@ -0,0 +1,10 @@
+// Same-chain near-duplicate collapse (FABLE ancestor-descendant dedup).
+import { ancestorDescendantDedup } from "../structural.ts";
+import type { Stage } from "./types.ts";
+
+export const dedup: Stage = {
+  name: "dedup",
+  run: (c) => {
+    c.finalHits = ancestorDescendantDedup(c.finalHits);
+  },
+};

package/workers/worker_public/src/stages/dense.ts

@@ -0,0 +1,73 @@
+// Dense lane (blocking): the primary Vectorize query over the folded
+// retrieval query. Three shapes: Option C optimistic reuse (identical
+// query, no filter — bit-for-bit same candidates, one serial round-trip
+// saved), the doc/edition-filtered query with its guessed-pin drop and
+// sparse-filter widen, and the plain unfiltered query.
+import { LIMITS } from "../config.ts";
+import { toVectorizeFilter } from "../selfquery.ts";
+import type { Stage } from "./types.ts";
+
+export const dense: Stage = {
+  name: "dense",
+  run: async (c) => {
+    const { env, filter, filters, vector, opts, rq, folded } = c;
+    const q: any = { topK: LIMITS.retrieveK, returnMetadata: "all" };
+    if (filter) q.filter = filter;
+    const optimistic = opts.optimisticHits ?? [];
+    const sameLane = rq === folded; // optimistic vector === this lane's vector
+    if (!filter && sameLane && optimistic.length) {
+      // Option C fast path: the unfiltered dense query already ran
+      // concurrently with understanding — same vector, same topK, no
+      // filter. Reusing it skips one serial Vectorize round-trip with a
+      // bit-for-bit identical candidate set.
+      c.matches = optimistic.map((h) => ({ id: h.id, score: h.score, metadata: h.metadata }));
+      console.log("optimistic lane: reused", c.matches.length, "dense hits (no re-query)");
+      // NOTE: when rq diverged (standalone_query / override) the
+      // optimistic hits are deliberately NOT unioned. Measured
+      // 2026-08-30: injecting the raw question's top-50 into a
+      // rewritten query's pool let topically close but wrong documents
+      // outscore the correct ones under the cross-encoder — recall@5
+      // fell 94.3% → 89.7% (golden ×3). The optimistic lane may only
+      // REPLACE an identical query, never dilute a better one.
+      return;
+    }
+    if (filter) {
+      let matches: any[] = (await env.VECTORIZE.query(vector, q)).matches ?? [];
+      // an edition pin corroborated by (almost) nothing means the pin
+      // was a guess — understanding emits editions for families it
+      // mixes up (observed: R 76 pinned @2021, an R 60 year; the corpus
+      // holds 1988/1992/2006). The index is the ground truth for which
+      // editions EXIST: a wrong pin starves the doc filter and the
+      // widen then floods the pool with superseded editions. Drop to
+      // the doc-only filter and let family-relative steering rank
+      // editions downstream. A pin the user actually asked for
+      // survives — its edition exists in the corpus.
+      if (filters && filters.edition && matches.length < 3) {
+        const docOnly = await env.VECTORIZE.query(vector, {
+          topK: LIMITS.retrieveK,
+          returnMetadata: "all",
+          filter: toVectorizeFilter({ doc_number: filters.doc_number }),
+        });
+        if ((docOnly.matches ?? []).length > matches.length) {
+          console.log("edition pin dropped:", filters.doc_number, "@", filters.edition, "→", docOnly.matches?.length ?? 0, "doc-scoped hits (edition not in corpus)");
+          matches = docOnly.matches ?? [];
+          filters.edition = undefined;
+        }
+      }
+      if (matches.length < LIMITS.rerankKeep) {
+        // sparse doc filter → widen with the unfiltered ranking. Same
+        // lane: the optimistic results ARE that ranking (identical
+        // vector, no filter) — reuse them; otherwise re-query with this
+        // lane's vector.
+        const unfiltered = sameLane && optimistic.length
+          ? optimistic.map((h) => ({ id: h.id, score: h.score, metadata: h.metadata }))
+          : (await env.VECTORIZE.query(vector, { topK: LIMITS.retrieveK, returnMetadata: "all" })).matches ?? [];
+        const seen = new Set(matches.map((m: any) => m.id));
+        matches = [...matches, ...unfiltered.filter((m: any) => !seen.has(m.id))];
+      }
+      c.matches = matches;
+      return;
+    }
+    c.matches = (await env.VECTORIZE.query(vector, q)).matches ?? [];
+  },
+};

package/workers/worker_public/src/stages/diversity.ts

@@ -0,0 +1,33 @@
+// Per-publication diversity, keyed by normalized identity: overview
+// chunks are near-duplicates across editions — at most ONE per
+// publication; clause chunks get a higher cap so content can fill slots.
+import { LIMITS } from "../config.ts";
+import type { Stage } from "./types.ts";
+
+export const diversity: Stage = {
+  name: "diversity",
+  run: (c) => {
+    const filters = c.filters;
+    const perDoc = new Map<string, number>();
+    let overviews = 0;
+    const diversified = [];
+    for (const h of c.hits) {
+      const isOverview = h.metadata.clause_anchor === "overview";
+      // a doc-number query matches every part (R 60-1/-2/Annexe A) —
+      // without a global overview cap their near-identical overviews
+      // crowd out the definition and clause chunks the answer needs
+      const ovCap = filters?.doc_number ? 6 : 2; // part overviews of the queried family are signal, not noise
+      if (isOverview && overviews >= ovCap) continue;
+      const key = `${h.metadata.docidentifier}|${h.metadata.language}`;
+      const n = perDoc.get(key) ?? 0;
+      const cap = isOverview ? 1 : filters?.doc_number ? 3 : 2;
+      if (n < cap) {
+        diversified.push(h);
+        perDoc.set(key, n + 1);
+        if (isOverview) overviews += 1;
+      }
+      if (diversified.length >= LIMITS.rerankKeep + 2) break;
+    }
+    c.finalHits = diversified.slice(0, LIMITS.rerankKeep);
+  },
+};

package/workers/worker_public/src/stages/editionCover.ts

@@ -0,0 +1,63 @@
+// Edition cover (the l6a guarantee): when a pool holds ONLY stale
+// editions of a document whose ACTIVE edition the registry knows, fetch
+// the current edition's chunks and add them at a discount — the current
+// edition must be ABLE to win: edition steering demotes stale siblings
+// only when the successor is in the pool ("R 60:2017 answers 'which
+// edition applies'" was the observed failure under rerank draws that
+// left every 2021 chunk out). One registry read per query; at most
+// `maxDocs` documents covered; additive — any failure skips.
+import { THRESHOLDS } from "../config.ts";
+import { toHits, type Stage } from "./types.ts";
+
+const maxDocs = 2;
+
+const familyOf = (di: string): string | null => {
+  const m = /^(?:OIML\s+)?([A-Z])\s?(\d{1,3})(?:[-–]([0-9A-Za-z]+))?/.exec(di);
+  return m ? `${m[1]}-${m[2]}` : null;
+};
+
+export const editionCover: Stage = {
+  name: "edition-cover",
+  failure: "additive",
+  when: (c) => !c.filters?.edition && c.hits.length > 1,
+  run: async (c) => {
+    const poolDocs = new Map<string, Set<string>>();
+    for (const h of c.hits) {
+      const di = h.metadata.docidentifier;
+      const ed = h.metadata.edition;
+      if (!di || !ed) continue;
+      if (!poolDocs.has(di)) poolDocs.set(di, new Set());
+      poolDocs.get(di)!.add(ed);
+    }
+    const families = [...new Set([...poolDocs.keys()].map(familyOf).filter(Boolean))] as string[];
+    if (!families.length) return;
+    const ph = families.map(() => "?").join(",");
+    const rows = (await c.env.DB.prepare(`SELECT docidentifier, edition FROM documents WHERE family IN (${ph}) AND active = 1`).bind(...families).all()).results ?? [];
+    // registry identifiers may carry the edition suffix; chunks never do
+    const want: Array<{ di: string; edition: string }> = [];
+    for (const r of rows as any[]) {
+      const di = String(r.docidentifier ?? "").replace(/:\d{4}$/, "");
+      const ed = String(r.edition ?? "");
+      if (di && /^\d{4}$/.test(ed) && poolDocs.has(di) && !poolDocs.get(di)!.has(ed)) want.push({ di, edition: ed });
+    }
+    if (!want.length) return;
+    const top = Math.max(...c.hits.map((h) => h.score));
+    let added = 0;
+    for (const w of want.slice(0, maxDocs)) {
+      try {
+        const q = await c.env.VECTORIZE.query(c.vector, {
+          topK: 3,
+          returnMetadata: "all",
+          filter: { $and: [{ docidentifier: { $eq: w.di } }, { edition: { $eq: w.edition } }] },
+        });
+        const hits = toHits((q as any).matches ?? []).map((h) => ({ ...h, score: top * THRESHOLDS.editionCoverDiscount }));
+        c.hits.push(...hits);
+        added += hits.length;
+        console.log("edition cover:", w.di, w.edition, `+${hits.length}`);
+      } catch {
+        // additive — retrieval stands without the cover
+      }
+    }
+    if (added) c.hits.sort((a, b) => b.score - a.score);
+  },
+};