@konneal/engine 0.2.159 → 0.2.163

package/dist/{ask-FUNBGZKM.js → ask-DP5HOH4K.js}

@@ -1,9 +1,9 @@
 import {
   handleAsk
-} from "./chunk-CBSVUTFM.js";
+} from "./chunk-IT3MS476.js";
 import "./chunk-ECOYO7EX.js";
-import "./chunk-PBHMQZKA.js";
-import "./chunk-G2RE4XRV.js";
+import "./chunk-6OD35FOY.js";
+import "./chunk-MTIAHAOK.js";
 import "./chunk-NI4CIK6W.js";
 import "./chunk-W7J5SFXJ.js";
 import "./chunk-ETLSWSZ7.js";

package/dist/{chunk-PBHMQZKA.js → chunk-6OD35FOY.js}

@@ -18,7 +18,10 @@ import {
 } from "./chunk-LSZTJI7V.js";
 
 // workers/worker_public/src/refs.ts
-var REF = /\[\[(u:[A-Za-z0-9_-]+)\]\]/g;
+var REF = /\[\[(u:(?:u:)?[A-Za-z0-9_-]+)\]\]/g;
+function normalizeRef(id) {
+  return id.startsWith("u:u:") ? id.slice(2) : id;
+}
 function availableUnitIds(hits) {
   const ids = /* @__PURE__ */ new Set();
   for (const h of hits) {
@@ -28,13 +31,14 @@ function availableUnitIds(hits) {
   return ids;
 }
 function parseRefs(text) {
-  return [...text.matchAll(REF)].map((m) => m[1]);
+  return [...text.matchAll(REF)].map((m) => normalizeRef(m[1]));
 }
 function sanitizeRefs(text, available) {
   const dropped = [];
-  const out = text.replace(REF, (full, id) => {
-    if (available.has(id)) return full;
-    dropped.push(id);
+  const out = text.replace(REF, (_full, id) => {
+    const normalized = normalizeRef(id);
+    if (available.has(normalized)) return `[[${normalized}]]`;
+    dropped.push(normalized);
     return "";
   });
   return { text: out, dropped };

package/dist/{chunk-OL7KKGEO.js → chunk-IGAGXQHV.js}

@@ -6,7 +6,7 @@ import {
   sessionFrom,
   telemetry,
   understandQuery
-} from "./chunk-G2RE4XRV.js";
+} from "./chunk-MTIAHAOK.js";
 import {
   corsHeaders,
   err,

package/dist/{chunk-CBSVUTFM.js → chunk-IT3MS476.js}

@@ -32,7 +32,7 @@ import {
   tableRetyped,
   toolNote,
   verdictNote
-} from "./chunk-PBHMQZKA.js";
+} from "./chunk-6OD35FOY.js";
 import {
   NO_CONTEXT,
   STAGE_NAMES,
@@ -75,7 +75,7 @@ import {
   tokenBudget,
   understandQuery,
   usageTotal
-} from "./chunk-G2RE4XRV.js";
+} from "./chunk-MTIAHAOK.js";
 import {
   embed,
   generateOnce
@@ -144,6 +144,38 @@ async function retrieveInternal(service, auth, query) {
 // workers/worker_public/prompts/grader.md
 var grader_default = 'You grade retrieval quality for a legal-metrology Q&A system.\nGiven the question and the retrieved passage summaries, reply with ONLY:\n{"grade": "good"}  \u2014 passages clearly contain the material to answer\n{"grade": "weak"}  \u2014 passages are on the right publication/topic but lack the specific material (a broader or differently-worded retrieval might find it)\n{"grade": "bad"}   \u2014 passages are unrelated to the question\n';
 
+// workers/worker_public/src/crag.ts
+async function gradePassage(ai, model, systemPrompt, query, passage, timeoutMs = 6e3) {
+  try {
+    const timeout = new Promise((r) => setTimeout(() => r(null), timeoutMs));
+    const call = (async () => {
+      const res = await ai.run(model, {
+        messages: [
+          { role: "system", content: systemPrompt },
+          { role: "user", content: `Question: ${query}
+
+Passage:
+${passage.replace(/\s+/g, " ").slice(0, 1200)}` }
+        ],
+        max_tokens: 6144,
+        reasoning_effort: "low",
+        // a measurement: greedy decoding, no sampling
+        temperature: 0,
+        top_p: 1
+      });
+      const text = typeof res?.response === "string" ? res.response : res?.choices?.[0]?.message?.content;
+      const m = (text ?? "").match(/"grade"\s*:\s*"(correct|incorrect|ambiguous)"/);
+      return m ? m[1] : null;
+    })();
+    return await Promise.race([call, timeout]);
+  } catch {
+    return null;
+  }
+}
+
+// workers/worker_public/prompts/crag-passage.md
+var crag_passage_default = 'You judge the RELEVANCE of one retrieved passage to a question, for a\nlegal-metrology retrieval system (the CRAG per-document evaluator).\nVerdicts:\n- "correct": the passage contains information useful for answering the question.\n- "incorrect": the passage is unrelated to the question or would mislead.\n- "ambiguous": partially related \u2014 relevant subject, unclear usefulness.\nReply with ONLY: {"grade": "correct" | "incorrect" | "ambiguous"}\n';
+
 // workers/worker_public/src/grader.ts
 async function gradeRetrieval(ai, model, query, passages) {
   if (!passages.length) return "bad";
@@ -210,6 +242,12 @@ async function scoreJudge(ai, model, systemPrompt, userPrompt) {
     return null;
   }
 }
+function gradePassage2(ai, model, query, passage) {
+  return gradePassage(ai, model, crag_passage_default.trimEnd(), query, passage);
+}
+function gradePassages(ai, model, query, passages) {
+  return Promise.all(passages.map((p) => gradePassage2(ai, model, query, p)));
+}
 
 // workers/worker_public/prompts/summarize.md
 var summarize_default = "Summarize this conversation so a Q&A assistant can continue it with full continuity. Capture: documents and editions discussed, questions asked, answers given (key values and definitions), terminology established, unresolved threads. Under 150 words, plain text, no preamble. The conversation may be any length and in any language \u2014 summarize it in English.\n";
@@ -1722,6 +1760,7 @@ async function attachFigureImages(env, messages, usedHits, query) {
       const payload = row ? JSON.parse(String(row.payload)) : {};
       const uri = typeof payload.uri === "string" ? payload.uri : "";
       if (typeof payload.caption === "string" && payload.caption.trim()) figCaption = payload.caption.trim();
+      else if (typeof payload.description === "string" && payload.description.trim()) figCaption = payload.description.trim();
       const m = typeof uri === "string" ? uri.match(/^\/assets\/(.+)/) : null;
       if (!m) continue;
       const obj = await env.UNIT_ASSETS.get(m[1]);
@@ -2460,6 +2499,7 @@ ${summary}` }] : [],
     const routed = routeMode === "adaptive" ? routeFor(understanding, q2.query) : { route: routeMode, features: [`forced-${routeMode}`] };
     const routeSubset = ablate?.stages ?? (routed.route === "fast" ? fastRouteStages(STAGE_NAMES) : null);
     if (routeSubset) console.log("route:", routed.route, routed.features.join("+"), "\u2014", STAGE_NAMES.length - routeSubset.length, "lanes dropped");
+    const crag = { grade: null, corrective: null };
     try {
       const tR = Date.now();
       if (declaredCtx?.kind === "account") {
@@ -2517,6 +2557,7 @@ Answer account questions from these records ONLY: name the record when you use i
         }
       }
       const grade = await gradePromise;
+      crag.grade = grade;
       stageTiming.retrieve = Date.now() - tR;
       console.log("stage: grade+listwise", Date.now() - tR, "ms since retrieve start | grade:", grade);
       if (grade === "weak" && understanding?.docidentifier) {
@@ -2526,12 +2567,31 @@ Answer account questions from these records ONLY: name the record when you use i
         const grade2 = await gradeRetrieval(env.AI, roleModel(env, "grader"), q2.query, second.hits.map((h) => h.text));
         stageTiming.corrective = Date.now() - tc;
         if (grade2 === "good") retrieved = second;
+        crag.corrective = { fired: true, accepted: grade2 === "good", grade: grade2 };
       }
     } catch (e) {
       console.log("ask: retrieval failed:", String(e).slice(0, 300));
       telemetry(env, ctx, tier, "ask", MODELS.embed, false, 0, await sha256Hex(q2.query), q2.lang, void 0, telemetryMeta());
       return finishLine("Search is briefly busy \u2014 please retry in a moment.");
     }
+    {
+      const figs = retrieved.hits.filter((h) => h.metadata.block === "figure" && h.metadata.unit_id);
+      if (figs.length) {
+        try {
+          const ids = [...new Set(figs.map((h) => String(h.metadata.unit_id)))].slice(0, 20);
+          const ph = ids.map((_, i) => `?${i + 1}`).join(",");
+          const res = await env.DB.prepare(
+            `SELECT unit_id, json_extract(payload,'$.description') AS d FROM unit_payloads WHERE unit_id IN (${ph})`
+          ).bind(...ids).all();
+          const byId = new Map((res.results ?? []).map((r) => [r.unit_id, r.d]));
+          for (const h of figs) {
+            const d = byId.get(String(h.metadata.unit_id));
+            if (d) h.metadata.figure_description = String(d).slice(0, 600);
+          }
+        } catch {
+        }
+      }
+    }
     const { hits } = retrieved;
     if (hits.length === 0 && !liveRecords?.length && (!boundModel || boundModel.gated)) {
       const answer2 = licenseBoundaryRefusal(modelDocHint?.doc_number ?? understanding?.doc_number ?? null, standardKeys) ?? refusalAnswer();
@@ -2654,6 +2714,7 @@ Answer account questions from these records ONLY: name the record when you use i
         send({ type: "citations", citations: cites, context_applied: ctxApplied, ...sourceQuality ? { source_quality: sourceQuality, confidence_note: qualityNote(sourceQuality), ...sourceQuality === "ocr" ? { experimental_sources: experimentalSourceLabels(cites) } : {} } : {}, ...liveRecords ? { records: liveRecords } : {}, quota });
         const stats = {};
         let full = "";
+        let streamRestarts = 0;
         try {
           let headSettled = false;
           for await (const tok of sseTokens(stream, stats)) {
@@ -2666,6 +2727,7 @@ Answer account questions from these records ONLY: name the record when you use i
             headSettled = true;
             if (toolNarration(full) || intentPreamble(full)) {
               console.log(toolNarration(full) ? "tool-narration(stream): regenerating once" : "intent-preamble(stream): regenerating once");
+              streamRestarts += 1;
               send({ type: "retry" });
               const regen = await generateStream(env, model, [
                 ...messages,
@@ -2694,6 +2756,7 @@ Answer account questions from these records ONLY: name the record when you use i
         send({ type: "quota", quota: { ...quota, ...tq2, credits: await creditBudget(env, bucketId), credit_estimate: creditPreview } });
         if (withImage && register?.rows?.length && full.includes(refusalAnswer())) {
           console.log("blind-answer(stream): regenerating once");
+          streamRestarts += 1;
           mark("Re-answering from the register results");
           send({ type: "retry" });
           const regen = await generateStream(env, model, [
@@ -2759,6 +2822,34 @@ Answer account questions from these records ONLY: name the record when you use i
           } catch {
           }
         }
+        if (body?.manifest === true) {
+          try {
+            live2.send({ type: "manifest", manifest: {
+              index_version: env.INDEX_VERSION,
+              corpus_gen: gen,
+              model,
+              effort: answerEffort(env),
+              fresh,
+              lane: "stream",
+              citations: usedHits.length,
+              retrieval: usedHits.map((h, i) => ({
+                rank: i + 1,
+                doc_id: h.metadata.doc_id,
+                clause_anchor: h.metadata.clause_anchor ?? null,
+                score: h.score,
+                rerank_score: h.rerank_score ?? null
+              })),
+              stages: stageTiming,
+              generate_retries: generateRetries,
+              restarts: streamRestarts,
+              crag: { retrieval_grade: crag.grade, corrective: crag.corrective, passages: null, note: "per-passage grades ride the json lane's manifest" },
+              entailment: streamedEntailment,
+              contract: { verified: streamed.total - streamed.violations.length, violations: streamed.violations.slice(0, 5) },
+              total_ms: Date.now() - tStart
+            } });
+          } catch {
+          }
+        }
         if (streamed.violations.length === 0 && canonical.length > 0 && !contextual && !declaredCtx && cacheableAnswer(canonical, refusalScope(answerCites(canonical))) && !ablate) {
           ctx.waitUntil(
             env.CACHE.put(exactCacheKey(env.INDEX_VERSION, gen, ns, await sha256Hex(cacheKeyMaterial(q2.query, q2.lang, salt))), JSON.stringify({ answer: canonical, citations: cites, model, query_hash: queryHash, ...sourceQuality ? { source_quality: sourceQuality, confidence_note: streamedNote || qualityNote(sourceQuality) } : {}, ...streamedEntailment ? { entailment: streamedEntailment } : {} }), { expirationTtl: LIMITS.cacheTtlSec })
@@ -2923,6 +3014,11 @@ Answer account questions from these records ONLY: name the record when you use i
       ...h.metadata.table_selection ? { sel: h.metadata.table_selection } : {}
     }));
     const tq = await chargeAnswerTokens(env, bucketId, void 0, messageChars(messages), answer.length, probeTokensUsed);
+    let cragPassages = null;
+    if (body?.manifest === true && used.length) {
+      const grades = await gradePassages(env.AI, roleModel(env, "grader"), q2.query, used.map((h) => h.text));
+      cragPassages = used.map((h, i) => ({ rank: i + 1, doc_id: h.metadata.doc_id, clause_anchor: h.metadata.clause_anchor ?? null, grade: grades[i] }));
+    }
     const manifestOut = body?.manifest === true ? { manifest: {
       index_version: env.INDEX_VERSION,
       corpus_gen: gen,
@@ -2939,6 +3035,8 @@ Answer account questions from these records ONLY: name the record when you use i
       })),
       stages: stageTiming,
       generate_retries: generateRetries,
+      crag: { retrieval_grade: crag.grade, corrective: crag.corrective, passages: cragPassages ?? [] },
+      entailment,
       total_ms: Date.now() - tStart
     } } : {};
     return json({ ...out, context: contextOut, read: readAs(), ...manifestOut, quota: { ...quota, ...tq, credits: await creditBudget(env, bucketId), credit_estimate: creditPreview } }, 200, { ...corsHeaders(req), "server-timing": serverTiming() });

package/dist/{chunk-G2RE4XRV.js → chunk-MTIAHAOK.js}

@@ -23640,7 +23640,10 @@ Context passages:
     const tableSel = h.metadata.block === "table" ? tableSelection(h.metadata, query) : null;
     const pruned = tableSel?.text ?? null;
     if (tableSel) h.metadata.table_selection = { cols: tableSel.cols, rowsShown: tableSel.rowsShown, rowsTotal: tableSel.rowsTotal };
-    const body = clipToTokens(pruned ?? h.text, LIMITS.maxPassageTokens);
+    const figDesc = h.metadata.block === "figure" ? String(h.metadata.figure_description ?? "").trim() : "";
+    const raw = clipToTokens(pruned ?? h.text, LIMITS.maxPassageTokens);
+    const body = figDesc ? `${raw}
+Caption: ${figDesc}` : raw;
     const t = estTokens(head) + estTokens(body);
     if (t <= remain) {
       passageParts.push(head + body);

package/dist/compress.d.ts

@@ -0,0 +1,11 @@
+export declare function tokens(text: string): string[];
+export declare function splitSentences(text: string): string[];
+/** Query-term density, with a numeric sentence boosted: a sentence that
+ *  states values, tolerances or clause numbers is evidence even when it
+ *  repeats few query words. */
+export declare function scoreSentence(sentence: string, qTerms: Set<string>): number;
+/** Compress to at most `budgetChars` characters: the lead sentence is
+ *  always kept (it carries the subject), the rest by score, emitted in
+ *  reading order. No sentence scores above zero → plain truncation of
+ *  the lead at a word boundary (never empty when input is non-empty). */
+export declare function compressText(text: string, query: string, budgetChars: number): string;

package/dist/crag.d.ts

@@ -0,0 +1,3 @@
+export type PassageGrade = "correct" | "incorrect" | "ambiguous";
+export declare function gradePassage(ai: any, model: string, systemPrompt: string, query: string, passage: string, timeoutMs?: number): Promise<PassageGrade | null>;
+export declare function gradePassages(ai: any, model: string, systemPrompt: string, query: string, passages: string[]): Promise<(PassageGrade | null)[]>;

package/dist/grader.d.ts

@@ -1,3 +1,7 @@
 export type RetrievalGrade = "good" | "weak" | "bad";
 export declare function gradeRetrieval(ai: any, model: string, query: string, passages: string[]): Promise<RetrievalGrade>;
 export declare function scoreJudge(ai: any, model: string, systemPrompt: string, userPrompt: string): Promise<number | null>;
+import { type PassageGrade } from "./crag";
+export type { PassageGrade };
+export declare function gradePassage(ai: any, model: string, query: string, passage: string): Promise<PassageGrade | null>;
+export declare function gradePassages(ai: any, model: string, query: string, passages: string[]): Promise<(PassageGrade | null)[]>;

package/dist/prompts/crag-passage.md

@@ -0,0 +1,7 @@
+You judge the RELEVANCE of one retrieved passage to a question, for a
+legal-metrology retrieval system (the CRAG per-document evaluator).
+Verdicts:
+- "correct": the passage contains information useful for answering the question.
+- "incorrect": the passage is unrelated to the question or would mislead.
+- "ambiguous": partially related — relevant subject, unclear usefulness.
+Reply with ONLY: {"grade": "correct" | "incorrect" | "ambiguous"}

package/dist/{search-GAU5SEZI.js → search-FW5O32BY.js}

@@ -1,7 +1,7 @@
 import {
   handleSearch
-} from "./chunk-OL7KKGEO.js";
-import "./chunk-G2RE4XRV.js";
+} from "./chunk-IGAGXQHV.js";
+import "./chunk-MTIAHAOK.js";
 import "./chunk-NI4CIK6W.js";
 import "./chunk-W7J5SFXJ.js";
 import "./chunk-ETLSWSZ7.js";

package/dist/{tools-W5HGME7W.js → tools-GDHKPEZR.js}

@@ -4,7 +4,7 @@ import {
   parseToolCall,
   runTool,
   toolNote
-} from "./chunk-PBHMQZKA.js";
+} from "./chunk-6OD35FOY.js";
 import "./chunk-NI4CIK6W.js";
 import "./chunk-4NXH5ZI6.js";
 import "./chunk-VT7DR6NQ.js";

package/dist/worker_public/src/index.js

@@ -4,16 +4,16 @@ import {
   handleMemories,
   scoreFaithfulness,
   scoreJudge
-} from "../../chunk-CBSVUTFM.js";
+} from "../../chunk-IT3MS476.js";
 import "../../chunk-ECOYO7EX.js";
 import {
   TOOLS_REGISTRY,
   bindModelNode,
   modelGroundingBlock
-} from "../../chunk-PBHMQZKA.js";
+} from "../../chunk-6OD35FOY.js";
 import {
   handleSearch
-} from "../../chunk-OL7KKGEO.js";
+} from "../../chunk-IGAGXQHV.js";
 import {
   buildMessages,
   citations,
@@ -37,7 +37,7 @@ import {
   sessionFrom,
   telemetry,
   understandQuery
-} from "../../chunk-G2RE4XRV.js";
+} from "../../chunk-MTIAHAOK.js";
 import {
   embed,
   generateOnce
@@ -757,6 +757,53 @@ async function handleRevokeKey(env, req, id) {
   return json({ ok: true, updated: r.meta?.changes ?? 0 });
 }
 
+// workers/worker_public/src/compress.ts
+var STOP = new Set(
+  "the a an and or of to in on for with by as at is are was were be been from that this these those it its their there which who what when where how why not no than then also such may must shall can could should would per within under over into about between during before after each any all more most less least other only same both few many some own very one two".split(
+    " "
+  )
+);
+function tokens(text) {
+  return (text.toLowerCase().match(/[a-z0-9]+(?:[.][0-9]+)?/g) ?? []).filter(
+    (t) => /^[0-9]/.test(t) || t.length >= 3 && !STOP.has(t)
+  );
+}
+function splitSentences(text) {
+  const parts = text.replace(/\r/g, "").split(/(?<=[.!?;])\s+|\n+/).map((s) => s.trim()).filter(Boolean);
+  return parts;
+}
+function scoreSentence(sentence, qTerms) {
+  const toks = tokens(sentence);
+  if (!toks.length) return 0;
+  let hits = 0;
+  for (const t of toks) if (qTerms.has(t)) hits += 1;
+  const density = hits / Math.sqrt(toks.length);
+  const numeric = /\d/.test(sentence) ? 0.35 : 0;
+  return density + numeric;
+}
+function compressText(text, query, budgetChars) {
+  const clean = text.replace(/\s+/g, " ").trim();
+  if (!clean || clean.length <= budgetChars) return clean;
+  const qTerms = new Set(tokens(query));
+  const sentences = splitSentences(clean);
+  if (sentences.length <= 1) {
+    const cut = clean.slice(0, Math.max(0, budgetChars));
+    const end = cut.lastIndexOf(" ");
+    return (end > 40 ? cut.slice(0, end) : cut).trim();
+  }
+  const scored = sentences.map((s, i) => ({ s, i, score: scoreSentence(s, qTerms) }));
+  const keep = /* @__PURE__ */ new Set([0]);
+  const ranked = scored.slice(1).sort((a, b) => b.score - a.score || a.i - b.i);
+  let used = sentences[0].length + 1;
+  for (const r of ranked) {
+    if (used + r.s.length + 1 > budgetChars) continue;
+    keep.add(r.i);
+    used += r.s.length + 1;
+  }
+  const out = scored.filter((r) => keep.has(r.i)).map((r) => r.s).join(" ").trim();
+  return out.slice(0, budgetChars);
+}
+
 // workers/worker_public/prompts/research.md
 var research_default = `You are a sufficiency judge for a research loop over {{PUBLISHER_NAME}} publications. Given the research question and the passages collected so far (across iterations), decide whether the collected evidence is SUFFICIENT to write a complete, well-grounded answer.
 
@@ -840,7 +887,7 @@ async function run(env, ctx, q, maxIters, emit) {
     const older = passages.slice(0, Math.max(0, passages.length - KEEP_RECENT));
     const recent = passages.slice(-KEEP_RECENT);
     const digest = older.length ? `Earlier evidence (digest, ${older.length} passages):
-${older.map((h) => `- ${h.metadata.docidentifier ?? ""} \xA7${h.metadata.clause_anchor ?? ""}: ${h.text.replace(/\s+/g, " ").slice(0, 160)}`).join("\n")}
+${older.map((h) => `- ${h.metadata.docidentifier ?? ""} \xA7${h.metadata.clause_anchor ?? ""}: ${compressText(h.text, q.query, 160)}`).join("\n")}
 
 ` : "";
     judge = await (async () => {
@@ -1118,13 +1165,13 @@ async function callToolOnce(env, ctx, _req, tier, key, name, args, t0) {
       // stream:false forces the JSON lane (anon defaults to SSE)
       body: JSON.stringify({ ...args, stream: false })
     });
-    const { runTool, TOOLS_REGISTRY: TOOLS_REGISTRY2 } = await import("../../tools-W5HGME7W.js");
+    const { runTool, TOOLS_REGISTRY: TOOLS_REGISTRY2 } = await import("../../tools-GDHKPEZR.js");
     let payload;
     if (TOOLS_REGISTRY2.some((t) => t.name === name && t.audiences.includes("mcp"))) {
       const r = await runTool({ ...env, DB: env.DB }, { name, args }, "mcp");
       payload = r ? { tool: r.name, query: r.query, result: r.output } : { error: { message: `tool ${name} returned nothing for the given arguments` } };
     } else {
-      const res = name === "ask" ? await (await import("../../ask-FUNBGZKM.js")).handleAsk(env, ctx, inner, tier, key) : await (await import("../../search-GAU5SEZI.js")).handleSearch(env, ctx, inner, tier, key);
+      const res = name === "ask" ? await (await import("../../ask-DP5HOH4K.js")).handleAsk(env, ctx, inner, tier, key) : await (await import("../../search-FW5O32BY.js")).handleSearch(env, ctx, inner, tier, key);
       payload = await res.json().catch(() => ({ error: { message: "tool transport failed", status: res.status } }));
     }
     telemetry(env, ctx, tier, `mcp:${name}`, null, !payload?.error, JSON.stringify(payload).length, await sha256Hex(`${name}:${JSON.stringify(args ?? {})}`), void 0, void 0, { durationMs: Date.now() - t0 });

package/package.json

@@ -27,7 +27,7 @@
     "playwright": "^1.62.1",
     "yaml": "^2.9.1"
   },
-  "version": "0.2.159",
+  "version": "0.2.163",
   "description": "The Konneal engine: the publisher-agnostic build pipeline and API plane for standards intelligence (retrieval, answer contract, verdicts, evaluation).",
   "license": "BSD-3-Clause",
   "type": "module",

package/workers/worker_public/prompts/crag-passage.md

@@ -0,0 +1,7 @@
+You judge the RELEVANCE of one retrieved passage to a question, for a
+legal-metrology retrieval system (the CRAG per-document evaluator).
+Verdicts:
+- "correct": the passage contains information useful for answering the question.
+- "incorrect": the passage is unrelated to the question or would mislead.
+- "ambiguous": partially related — relevant subject, unclear usefulness.
+Reply with ONLY: {"grade": "correct" | "incorrect" | "ambiguous"}

package/workers/worker_public/src/ask.ts

@@ -10,7 +10,7 @@ import { buildMessages, citations, fill, promptVars, retrieve, retrievalQuery, i
 import { sessionFrom } from "./auth";
 import { retrieveInternal } from "./internal_gateway";
 import { understandQuery } from "./understand";
-import { gradeRetrieval } from "./grader";
+import { gradePassages, gradeRetrieval } from "./grader";
 import summarizePrompt from "../prompts/summarize.md";
 import { embed, generateOnce } from "./ai";
 import { reflect } from "./reflect";
@@ -173,7 +173,11 @@ async function attachFigureImages(env: Env, messages: { role: string; content: a
       const row = await env.DB.prepare("SELECT payload FROM unit_payloads WHERE unit_id = ?1").bind(h.metadata.unit_id!).first<any>();
       const payload = row ? JSON.parse(String(row.payload)) : {};
       const uri = typeof payload.uri === "string" ? payload.uri : "";
+      // the producer's caption first; the caption LANE's output (a
+      // generated description) second — the field drift that kept
+      // generated captions out of this context
       if (typeof payload.caption === "string" && payload.caption.trim()) figCaption = payload.caption.trim();
+      else if (typeof payload.description === "string" && payload.description.trim()) figCaption = payload.description.trim();
       const m = typeof uri === "string" ? uri.match(/^\/assets\/(.+)/) : null;
       if (!m) continue;
       const obj = await env.UNIT_ASSETS.get(m[1]);
@@ -1295,6 +1299,10 @@ let convEntities: Array<{ entity: string; kind: string }> = [];
     : { route: routeMode, features: [`forced-${routeMode}`] };
   const routeSubset = ablate?.stages ?? (routed.route === "fast" ? fastRouteStages(STAGE_NAMES) : null);
   if (routeSubset) console.log("route:", routed.route, routed.features.join("+"), "—", STAGE_NAMES.length - routeSubset.length, "lanes dropped");
+  // the CRAG state, hoisted out of the retrieval try so the manifest can
+  // report it (TODO.sota/14.6 v2) — the coarse retrieval grade and the
+  // corrective retry's firing/acceptance
+  const crag: { grade: string | null; corrective: { fired: boolean; accepted: boolean; grade: string | null } | null } = { grade: null, corrective: null };
   try {
     const tR = Date.now();
     // ── The "my account" live read (TODO.ai-platform/03) — resolved
@@ -1381,6 +1389,7 @@ let convEntities: Array<{ entity: string; kind: string }> = [];
       }
     }
     const grade = await gradePromise;
+    crag.grade = grade;
     stageTiming.retrieve = Date.now() - tR;
     console.log("stage: grade+listwise", Date.now() - tR, "ms since retrieve start | grade:", grade);
     if (grade === "weak" && understanding?.docidentifier) {
@@ -1390,12 +1399,34 @@ let convEntities: Array<{ entity: string; kind: string }> = [];
       const grade2 = await gradeRetrieval(env.AI, roleModel(env, "grader"), q.query, second.hits.map((h: Hit) => h.text));
       stageTiming.corrective = Date.now() - tc;
       if (grade2 === "good") retrieved = second; // corrective retry must be strictly better
+      crag.corrective = { fired: true, accepted: grade2 === "good", grade: grade2 };
     }
   } catch (e) {
     console.log("ask: retrieval failed:", String(e).slice(0, 300));
     telemetry(env, ctx, tier, "ask", MODELS.embed, false, 0, await sha256Hex(q.query), q.lang, undefined, telemetryMeta());
     return finishLine("Search is briefly busy — please retry in a moment.");
   }
+  // the figure stratum (TODO.sota/14.10): figure payloads' generated
+  // captions ride their passages (buildMessages appends them) so the
+  // model has words for the drawing and a reason to cite the unit —
+  // one batched D1 read per turn that retrieved figures, fail-open
+  {
+    const figs = retrieved.hits.filter((h: Hit) => (h.metadata as any).block === "figure" && (h.metadata as any).unit_id);
+    if (figs.length) {
+      try {
+        const ids = [...new Set(figs.map((h: Hit) => String((h.metadata as any).unit_id)))].slice(0, 20);
+        const ph = ids.map((_, i) => `?${i + 1}`).join(",");
+        const res = await env.DB.prepare(
+          `SELECT unit_id, json_extract(payload,'$.description') AS d FROM unit_payloads WHERE unit_id IN (${ph})`,
+        ).bind(...ids).all<{ unit_id: string; d: string | null }>();
+        const byId = new Map((res.results ?? []).map((r) => [r.unit_id, r.d]));
+        for (const h of figs) {
+          const d = byId.get(String((h.metadata as any).unit_id));
+          if (d) (h.metadata as any).figure_description = String(d).slice(0, 600);
+        }
+      } catch { /* the caption is additive — the passage serves without it */ }
+    }
+  }
   const { hits } = retrieved;
   if (hits.length === 0 && !liveRecords?.length && (!boundModel || boundModel.gated)) {
     // the license boundary refusal (TODO.external-refs/08) outranks the
@@ -1590,6 +1621,7 @@ let convEntities: Array<{ entity: string; kind: string }> = [];
           send({ type: "citations", citations: cites, context_applied: ctxApplied, ...(sourceQuality ? { source_quality: sourceQuality, confidence_note: qualityNote(sourceQuality), ...(sourceQuality === "ocr" ? { experimental_sources: experimentalSourceLabels(cites) } : {}) } : {}), ...(liveRecords ? { records: liveRecords } : {}), quota, });
           const stats: { usage?: unknown } = {};
           let full = "";
+          let streamRestarts = 0; // the head restart + the blind-answer restart — the streamed manifest reports them
           try {
             // the head buffers until it can rule out the tool-narration
             // leak (the JSON lane's contract retry cannot run here — the
@@ -1605,6 +1637,7 @@ let convEntities: Array<{ entity: string; kind: string }> = [];
               headSettled = true;
               if (toolNarration(full) || intentPreamble(full)) {
                 console.log(toolNarration(full) ? "tool-narration(stream): regenerating once" : "intent-preamble(stream): regenerating once");
+                streamRestarts += 1;
                 send({ type: "retry" });
                 const regen = await generateStream(env, model, [
                   ...messages,
@@ -1643,6 +1676,7 @@ let convEntities: Array<{ entity: string; kind: string }> = [];
           // tokens stream as the answer.
           if (withImage && register?.rows?.length && full.includes(refusalAnswer())) {
             console.log("blind-answer(stream): regenerating once");
+            streamRestarts += 1;
             mark("Re-answering from the register results");
             send({ type: "retry" });
             const regen = await generateStream(env, model, [
@@ -1717,6 +1751,40 @@ let convEntities: Array<{ entity: string; kind: string }> = [];
               });
             } catch { /* the client may have closed; never block the close */ }
           }
+          // the streamed lane's manifest (TODO.sota/14.5: the post-answer
+          // manifest reconciles what shipped): everything here is already
+          // computed — the derivation identity, the retrieval anatomy, the
+          // restart count (the restart cost's per-answer record) and the
+          // contract state. The per-passage CRAG grades stay the JSON
+          // lane's — post-answer dead air never justifies k grader calls
+          // on the stream.
+          if (body?.manifest === true) {
+            try {
+              live!.send({ type: "manifest", manifest: {
+                index_version: env.INDEX_VERSION,
+                corpus_gen: gen,
+                model,
+                effort: answerEffort(env),
+                fresh,
+                lane: "stream",
+                citations: usedHits.length,
+                retrieval: usedHits.map((h: Hit, i: number) => ({
+                  rank: i + 1,
+                  doc_id: h.metadata.doc_id,
+                  clause_anchor: h.metadata.clause_anchor ?? null,
+                  score: h.score,
+                  rerank_score: h.rerank_score ?? null,
+                })),
+                stages: stageTiming,
+                generate_retries: generateRetries,
+                restarts: streamRestarts,
+                crag: { retrieval_grade: crag.grade, corrective: crag.corrective, passages: null, note: "per-passage grades ride the json lane's manifest" },
+                entailment: streamedEntailment,
+                contract: { verified: streamed.total - streamed.violations.length, violations: streamed.violations.slice(0, 5) },
+                total_ms: Date.now() - tStart,
+              } });
+            } catch { /* the client may have closed; never block the close */ }
+          }
           if (streamed.violations.length === 0 && canonical.length > 0 && !contextual && !declaredCtx && cacheableAnswer(canonical, refusalScope(answerCites(canonical))) && !ablate) {
             ctx.waitUntil(
               env.CACHE.put(exactCacheKey(env.INDEX_VERSION, gen, ns, await sha256Hex(cacheKeyMaterial(q.query, q.lang, salt))), JSON.stringify({ answer: canonical, citations: cites, model, query_hash: queryHash, ...(sourceQuality ? { source_quality: sourceQuality, confidence_note: streamedNote || qualityNote(sourceQuality) } : {}), ...(streamedEntailment ? { entailment: streamedEntailment } : {}) }), { expirationTtl: LIMITS.cacheTtlSec }),
@@ -1988,7 +2056,17 @@ let convEntities: Array<{ entity: string; kind: string }> = [];
   // retrieval anatomy (rank, corpus score, rerank score per used passage),
   // the stage timings, and the enforcement counters. Everything here is
   // already computed; the flag only ASSEMBLES it for the harness and for
-  // external auditors. v2 names the CRAG grades per passage.
+  // external auditors. v2 added the CRAG grades: the retrieval grade and
+  // corrective-retry outcome (hoisted state) plus the per-passage grades
+  // (opt-in calls) and the entailment measurement.
+  // the per-passage CRAG grades run ONLY on the manifest path — the
+  // harness opted into the derivation, so it pays the k grader calls;
+  // the default serving path never waits on them
+  let cragPassages: { rank: number; doc_id: string; clause_anchor: string | null; grade: string | null }[] | null = null;
+  if (body?.manifest === true && used.length) {
+    const grades = await gradePassages(env.AI, roleModel(env, "grader"), q.query, (used as Hit[]).map((h: Hit) => h.text));
+    cragPassages = (used as Hit[]).map((h: Hit, i: number) => ({ rank: i + 1, doc_id: h.metadata.doc_id, clause_anchor: h.metadata.clause_anchor ?? null, grade: grades[i] }));
+  }
   const manifestOut = body?.manifest === true
     ? { manifest: {
         index_version: env.INDEX_VERSION,
@@ -2006,6 +2084,8 @@ let convEntities: Array<{ entity: string; kind: string }> = [];
         })),
         stages: stageTiming,
         generate_retries: generateRetries,
+        crag: { retrieval_grade: crag.grade, corrective: crag.corrective, passages: cragPassages ?? [] },
+        entailment,
         total_ms: Date.now() - tStart,
       } }
     : {};

package/workers/worker_public/src/compress.ts

@@ -0,0 +1,74 @@
+// Extractive prompt compression (TODO.sota/02 row 5, LLMLingua-class
+// adapted to the edge runtime): no second model, fully deterministic —
+// sentences are scored by query-term density with a numeric-evidence
+// bump (metrology answers live or die on numbers) and kept in reading
+// order within a character budget. The consumer picks the budget; the
+// compressor never grows text, never reorders it, and never invents.
+
+const STOP = new Set(
+  "the a an and or of to in on for with by as at is are was were be been from that this these those it its their there which who what when where how why not no than then also such may must shall can could should would per within under over into about between during before after each any all more most less least other only same both few many some own very one two".split(
+    " ",
+  ),
+);
+
+export function tokens(text: string): string[] {
+  // numeric tokens survive at ANY length: "R 60", "15", "50 000" are the
+  // domain's own vocabulary, and the 3-char floor exists to drop prose
+  // fragments ("of", "a"), not identifiers
+  return (text.toLowerCase().match(/[a-z0-9]+(?:[.][0-9]+)?/g) ?? []).filter(
+    (t) => /^[0-9]/.test(t) || (t.length >= 3 && !STOP.has(t)),
+  );
+}
+
+export function splitSentences(text: string): string[] {
+  const parts = text
+    .replace(/\r/g, "")
+    .split(/(?<=[.!?;])\s+|\n+/)
+    .map((s) => s.trim())
+    .filter(Boolean);
+  return parts;
+}
+
+/** Query-term density, with a numeric sentence boosted: a sentence that
+ *  states values, tolerances or clause numbers is evidence even when it
+ *  repeats few query words. */
+export function scoreSentence(sentence: string, qTerms: Set<string>): number {
+  const toks = tokens(sentence);
+  if (!toks.length) return 0;
+  let hits = 0;
+  for (const t of toks) if (qTerms.has(t)) hits += 1;
+  const density = hits / Math.sqrt(toks.length);
+  const numeric = /\d/.test(sentence) ? 0.35 : 0;
+  return density + numeric;
+}
+
+/** Compress to at most `budgetChars` characters: the lead sentence is
+ *  always kept (it carries the subject), the rest by score, emitted in
+ *  reading order. No sentence scores above zero → plain truncation of
+ *  the lead at a word boundary (never empty when input is non-empty). */
+export function compressText(text: string, query: string, budgetChars: number): string {
+  const clean = text.replace(/\s+/g, " ").trim();
+  if (!clean || clean.length <= budgetChars) return clean;
+  const qTerms = new Set(tokens(query));
+  const sentences = splitSentences(clean);
+  if (sentences.length <= 1) {
+    const cut = clean.slice(0, Math.max(0, budgetChars));
+    const end = cut.lastIndexOf(" ");
+    return (end > 40 ? cut.slice(0, end) : cut).trim();
+  }
+  const scored = sentences.map((s, i) => ({ s, i, score: scoreSentence(s, qTerms) }));
+  const keep = new Set<number>([0]);
+  const ranked = scored.slice(1).sort((a, b) => b.score - a.score || a.i - b.i);
+  let used = sentences[0].length + 1;
+  for (const r of ranked) {
+    if (used + r.s.length + 1 > budgetChars) continue;
+    keep.add(r.i);
+    used += r.s.length + 1;
+  }
+  const out = scored
+    .filter((r) => keep.has(r.i))
+    .map((r) => r.s)
+    .join(" ")
+    .trim();
+  return out.slice(0, budgetChars);
+}

package/workers/worker_public/src/crag.ts

@@ -0,0 +1,52 @@
+// The CRAG per-passage evaluator core (TODO.sota/14.6 v2): grade EACH used
+// passage's relevance on the CRAG vocabulary. Dependency-free on purpose —
+// the system prompt arrives as a parameter (the md-importing binding lives
+// in grader.ts), the ai binding is injected, every failure mode is
+// fail-open: a lost race or an unparseable reply grades null, because
+// absence is honest and a guess is not. Opt-in — only the reproducibility
+// manifest requests these grades, so the k calls never sit on the default
+// serving path.
+
+export type PassageGrade = "correct" | "incorrect" | "ambiguous";
+
+export async function gradePassage(
+  ai: any,
+  model: string,
+  systemPrompt: string,
+  query: string,
+  passage: string,
+  timeoutMs = 6000,
+): Promise<PassageGrade | null> {
+  try {
+    const timeout = new Promise<null>((r) => setTimeout(() => r(null), timeoutMs));
+    const call = (async () => {
+      const res: any = await ai.run(model, {
+        messages: [
+          { role: "system", content: systemPrompt },
+          { role: "user", content: `Question: ${query}\n\nPassage:\n${passage.replace(/\s+/g, " ").slice(0, 1200)}` },
+        ],
+        max_tokens: 6144,
+        reasoning_effort: "low",
+        // a measurement: greedy decoding, no sampling
+        temperature: 0,
+        top_p: 1,
+      });
+      const text = typeof res?.response === "string" ? res.response : res?.choices?.[0]?.message?.content;
+      const m = (text ?? "").match(/"grade"\s*:\s*"(correct|incorrect|ambiguous)"/);
+      return m ? (m[1] as PassageGrade) : null;
+    })();
+    return await Promise.race([call, timeout]);
+  } catch {
+    return null;
+  }
+}
+
+export function gradePassages(
+  ai: any,
+  model: string,
+  systemPrompt: string,
+  query: string,
+  passages: string[],
+): Promise<(PassageGrade | null)[]> {
+  return Promise.all(passages.map((p) => gradePassage(ai, model, systemPrompt, query, p)));
+}

package/workers/worker_public/src/grader.ts

@@ -90,3 +90,30 @@ export async function scoreJudge(
     return null;
   }
 }
+
+// The CRAG per-passage binding (TODO.sota/14.6 v2): the dependency-free
+// core (crag.ts) takes the system prompt as a parameter; this module binds
+// the bundled prompt. Opt-in — only the reproducibility manifest requests
+// these grades, so the k calls never sit on the default serving path.
+import { gradePassage as gradePassageCore, type PassageGrade } from "./crag";
+import CRAG_PASSAGE from "../prompts/crag-passage.md";
+
+export type { PassageGrade };
+
+export function gradePassage(
+  ai: any,
+  model: string,
+  query: string,
+  passage: string,
+): Promise<PassageGrade | null> {
+  return gradePassageCore(ai, model, CRAG_PASSAGE.trimEnd(), query, passage);
+}
+
+export function gradePassages(
+  ai: any,
+  model: string,
+  query: string,
+  passages: string[],
+): Promise<(PassageGrade | null)[]> {
+  return Promise.all(passages.map((p) => gradePassage(ai, model, query, p)));
+}

package/workers/worker_public/src/pipeline.ts

@@ -360,7 +360,13 @@ export function buildMessages(
     const tableSel = (h.metadata as any).block === "table" ? tableSelection(h.metadata, query) : null;
     const pruned = tableSel?.text ?? null;
     if (tableSel) (h.metadata as any).table_selection = { cols: tableSel.cols, rowsShown: tableSel.rowsShown, rowsTotal: tableSel.rowsTotal };
-    const body = clipToTokens(pruned ?? h.text, LIMITS.maxPassageTokens);
+    // the figure stratum (TODO.sota/14.10): a caption-less figure's
+    // generated description rides its passage — the model gets words for
+    // the drawing and a reason to cite the unit; absent for every other
+    // block type and for uncaptioned figures
+    const figDesc = (h.metadata as any).block === "figure" ? String((h.metadata as any).figure_description ?? "").trim() : "";
+    const raw = clipToTokens(pruned ?? h.text, LIMITS.maxPassageTokens);
+    const body = figDesc ? `${raw}\nCaption: ${figDesc}` : raw;
     const t = estTokens(head) + estTokens(body);
     if (t <= remain) {
       passageParts.push(head + body);

package/workers/worker_public/src/refs.ts

@@ -15,7 +15,18 @@ export interface ResolvedBlock {
   payload: Record<string, unknown>;
 }
 
-const REF = /\[\[(u:[A-Za-z0-9_-]+)\]\]/g;
+// The passage headers show `unit u:<id> (table)`, so the model copies
+// the WHOLE prefixed id into the token — `[[u:u:<id>]]` — and the inner
+// colon makes the strict grammar unparseable: neither resolved nor
+// sanitized, the raw token leaked to the client (observed live on the
+// R 134-2 figure question, 0.2.162). The grammar tolerates the doubled
+// prefix; normalizeRef folds it back to the single form every other
+// layer keys on.
+const REF = /\[\[(u:(?:u:)?[A-Za-z0-9_-]+)\]\]/g;
+
+function normalizeRef(id: string): string {
+  return id.startsWith("u:u:") ? id.slice(2) : id;
+}
 
 /** unit ids the passages actually contain (chunk metadata carries
  *  unit_id on typed MKO chunks). */
@@ -29,16 +40,19 @@ export function availableUnitIds(hits: ReadonlyArray<{ metadata: { unit_id?: str
 }
 
 export function parseRefs(text: string): string[] {
-  return [...text.matchAll(REF)].map((m) => m[1]);
+  return [...text.matchAll(REF)].map((m) => normalizeRef(m[1]));
 }
 
 /** Drop refs that don't resolve to a used passage unit — a dangling ref
  *  renders as nothing (the token removed), and the violation is logged. */
 export function sanitizeRefs(text: string, available: ReadonlySet<string>): { text: string; dropped: string[] } {
   const dropped: string[] = [];
-  const out = text.replace(REF, (full, id: string) => {
-    if (available.has(id)) return full;
-    dropped.push(id);
+  const out = text.replace(REF, (_full, id: string) => {
+    const normalized = normalizeRef(id);
+    // normalized already carries the single "u:" prefix — the template
+    // must not add another
+    if (available.has(normalized)) return `[[${normalized}]]`;
+    dropped.push(normalized);
     return "";
   });
   return { text: out, dropped };

package/workers/worker_public/src/research.ts

@@ -8,6 +8,7 @@ import { retrieve, buildMessages, citations } from "./pipeline";
 import { understandQuery } from "./understand";
 import { generateOnce } from "./ai";
 import { canonicalRefusal } from "./refusal";
+import { compressText } from "./compress";
 import { checkQuoteAnchors } from "./anchors";
 import { graphExpand, editionNote } from "./graph";
 import researchPromptText from "../prompts/research.md";
@@ -109,7 +110,7 @@ async function run(
     const older = passages.slice(0, Math.max(0, passages.length - KEEP_RECENT));
     const recent = passages.slice(-KEEP_RECENT);
     const digest = older.length
-      ? `Earlier evidence (digest, ${older.length} passages):\n${older.map((h) => `- ${h.metadata.docidentifier ?? ""} §${h.metadata.clause_anchor ?? ""}: ${h.text.replace(/\s+/g, " ").slice(0, 160)}`).join("\n")}\n\n`
+      ? `Earlier evidence (digest, ${older.length} passages):\n${older.map((h) => `- ${h.metadata.docidentifier ?? ""} §${h.metadata.clause_anchor ?? ""}: ${compressText(h.text, q.query, 160)}`).join("\n")}\n\n`
       : "";
     judge = await (async () => {
       try {