@konneal/engine 0.2.162 → 0.2.164

package/dist/ablate.d.ts

@@ -13,6 +13,11 @@ export interface AblateConfig {
      *  diversified subset + one strong verifier. Off in serving until the
      *  gate promotes it; the grid arms it per ask. */
     speculative: boolean;
+    /** Extractive prompt compression (TODO.sota/02 row 5): each passage
+     *  body is query-weighted sentence selection at a fraction of its
+     *  budget before the answer model reads it. Off in serving; the grid
+     *  arms it per ask (the row's accuracy-at-token-budget gate). */
+    compress: boolean;
 }
 export type AblateParse = {
     ok: true;

package/dist/ask-B7XXKYZP.js

@@ -0,0 +1,16 @@
+import {
+  handleAsk
+} from "./chunk-GVCOGEJC.js";
+import "./chunk-ECOYO7EX.js";
+import "./chunk-K5WL3DKH.js";
+import "./chunk-E74MNLBC.js";
+import "./chunk-FHRGC6TP.js";
+import "./chunk-GJATVGBS.js";
+import "./chunk-ETLSWSZ7.js";
+import "./chunk-D6UUAL7I.js";
+import "./chunk-ZD6TYLET.js";
+import "./chunk-3FYJM7LH.js";
+import "./chunk-LSZTJI7V.js";
+export {
+  handleAsk
+};

package/dist/{chunk-4NXH5ZI6.js → chunk-D6UUAL7I.js}

@@ -1,7 +1,7 @@
 import {
   DATASETS,
   datasetAllowed
-} from "./chunk-VT7DR6NQ.js";
+} from "./chunk-ZD6TYLET.js";
 import {
   P
 } from "./chunk-3FYJM7LH.js";

package/dist/{chunk-MTIAHAOK.js → chunk-E74MNLBC.js}

@@ -1,11 +1,11 @@
 import {
   embed,
   rerank
-} from "./chunk-NI4CIK6W.js";
+} from "./chunk-FHRGC6TP.js";
 import {
   bubbleConfirmPage,
   isAllowedBubbleOrigin
-} from "./chunk-W7J5SFXJ.js";
+} from "./chunk-GJATVGBS.js";
 import {
   DATASETS,
   LIMITS,
@@ -15,7 +15,7 @@ import {
   processExpansion,
   sha256Hex,
   today
-} from "./chunk-VT7DR6NQ.js";
+} from "./chunk-ZD6TYLET.js";
 import {
   P
 } from "./chunk-3FYJM7LH.js";
@@ -1747,13 +1747,13 @@ var UnBuilder = class extends BaseBuilder {
     return key === "token";
   }
   build(data) {
-    const tokens = this.extractTokens(Array.isArray(data) ? data : data["token"]);
-    if (tokens.length === 0)
+    const tokens2 = this.extractTokens(Array.isArray(data) ? data : data["token"]);
+    if (tokens2.length === 0)
       throw new ParseFailed("UN identifier has no tokens", 0);
-    const path = tokens.slice(0, -1);
+    const path = tokens2.slice(0, -1);
     const yearToken = [...path].reverse().find((t) => /^\d{4}$/.test(t));
     const attrs = {
-      number: tokens[tokens.length - 1],
+      number: tokens2[tokens2.length - 1],
       path
     };
     if (yearToken)
@@ -3778,7 +3778,7 @@ function jisGrammarImplementation() {
 }
 
 // node_modules/@pubid/pubid/dist/flavors/iec/model.js
-var TS = (code, stageCode, typeCode, abbr, harmonized, tokens) => ({ code, stageCode, typeCode, abbr, harmonized, tokens });
+var TS = (code, stageCode, typeCode, abbr, harmonized, tokens2) => ({ code, stageCode, typeCode, abbr, harmonized, tokens: tokens2 });
 var TYPED_STAGES = [
   TS("is", "published", "is", ""),
   TS("tr", "published", "tr", "TR"),
@@ -4332,8 +4332,8 @@ function parseLanguages2(tree) {
   const raw = s(tree);
   if (raw === void 0)
     return [];
-  const tokens = raw.split(",").flatMap((chunk) => chunk.split("/"));
-  return tokens.map((token) => token.trim()).filter((token) => token !== "").map((original) => {
+  const tokens2 = raw.split(",").flatMap((chunk) => chunk.split("/"));
+  return tokens2.map((token) => token.trim()).filter((token) => token !== "").map((original) => {
     const code = original.length === 1 ? Language.CHAR_MAP[original] ?? original.toLowerCase() : original;
     return new Language({ code, originalCode: original });
   });
@@ -5582,8 +5582,8 @@ var TYPE_WORD_TO_CODE = (() => {
   return map;
 })();
 var LANGUAGE_ALIASES = { EN: "E", FR: "F" };
-function alternation(tokens) {
-  const sorted = [...tokens].sort((a, b) => b.length - a.length);
+function alternation(tokens2) {
+  const sorted = [...tokens2].sort((a, b) => b.length - a.length);
   return sorted.map((t) => str(t)).reduce((a, b) => a.or(b));
 }
 function buildRules24() {
@@ -6863,8 +6863,8 @@ function publishedStage(typeKey) {
 }
 function stageTokens2(typeKeys) {
   const stages = typeKeys === void 0 ? ISO_TYPED_STAGES : ISO_TYPED_STAGES.filter((s8) => typeKeys.includes(s8.typeKey));
-  const tokens = stages.flatMap((s8) => s8.abbr.filter((a) => a !== ""));
-  return [...new Set(tokens)].sort((a, b) => b.length - a.length);
+  const tokens2 = stages.flatMap((s8) => s8.abbr.filter((a) => a !== ""));
+  return [...new Set(tokens2)].sort((a, b) => b.length - a.length);
 }
 function stageAbbr(entry, formatLong, originalAbbr) {
   if (formatLong && entry.longAbbr !== void 0)
@@ -7697,8 +7697,8 @@ var WG_TYPES = [
   "JTAG",
   "JTG"
 ];
-function literalAlternation(tokens) {
-  const sorted = [...new Set(tokens)].sort((a, b) => b.length - a.length);
+function literalAlternation(tokens2) {
+  const sorted = [...new Set(tokens2)].sort((a, b) => b.length - a.length);
   return sorted.map((t) => str(t)).reduce((a, b) => a.or(b));
 }
 function buildRules25() {
@@ -8171,8 +8171,8 @@ var MONTHS = [
   "Nov",
   "Dec"
 ];
-function literalAlternation2(tokens) {
-  const sorted = [...new Set(tokens)].sort((a, b) => b.length - a.length);
+function literalAlternation2(tokens2) {
+  const sorted = [...new Set(tokens2)].sort((a, b) => b.length - a.length);
   return sorted.map((t) => str(t)).reduce((a, b) => a.or(b));
 }
 function buildRules26() {
@@ -10543,8 +10543,8 @@ var MONTHS2 = [
   "Nov",
   "Dec"
 ];
-function literalAlternation3(tokens) {
-  const sorted = [...new Set(tokens)].sort((a, b) => b.length - a.length);
+function literalAlternation3(tokens2) {
+  const sorted = [...new Set(tokens2)].sort((a, b) => b.length - a.length);
   return sorted.map((t) => str(t)).reduce((a, b) => a.or(b));
 }
 function buildRules27() {
@@ -14930,8 +14930,8 @@ var RM_ABBRS = [
   "RM"
 ];
 var SUPPLEMENT_ABBRS = ["AMD", "COR"];
-function abbrAlternation(tokens) {
-  const sorted = [...new Set(tokens)].sort((a, b) => b.length - a.length);
+function abbrAlternation(tokens2) {
+  const sorted = [...new Set(tokens2)].sort((a, b) => b.length - a.length);
   return sorted.map((t) => str(t)).reduce((a, b) => a.or(b));
 }
 function buildRules30() {
@@ -15857,9 +15857,9 @@ function adoptIdentifier(raw) {
 function splitPrefix(text) {
   if (text === void 0)
     return [void 0, void 0];
-  const tokens = text.trim().split(/\s+/);
-  const copublisher = normalizeCopublisher(tokens[0]);
-  const subtype = tokens.length > 1 ? normalizeSubtype(tokens.slice(1).join(" ")) : void 0;
+  const tokens2 = text.trim().split(/\s+/);
+  const copublisher = normalizeCopublisher(tokens2[0]);
+  const subtype = tokens2.length > 1 ? normalizeSubtype(tokens2.slice(1).join(" ")) : void 0;
   return [copublisher, subtype];
 }
 function normalizeCopublisher(raw) {
@@ -16710,7 +16710,7 @@ function stri(input) {
   }
   return parts.reduce((a, b) => a.then(b));
 }
-var alt = (tokens) => tokens.map((t) => str(t)).reduce((a, b) => a.or(b));
+var alt = (tokens2) => tokens2.map((t) => str(t)).reduce((a, b) => a.or(b));
 function buildRules34() {
   const rules = {};
   const rule = (name, build) => {
@@ -19512,9 +19512,9 @@ function parseCsa(input) {
       }
     }
     if (baseInput === void 0 || baseInput === "") {
-      const tokens = input.split(/\s+/);
+      const tokens2 = input.split(/\s+/);
       let acc = "";
-      for (const token of tokens) {
+      for (const token of tokens2) {
         if (/^PACKAGE$/i.test(token))
           break;
         const testInput = acc === "" ? token : `${acc} ${token}`;
@@ -19557,7 +19557,7 @@ function csaGrammarImplementation() {
 }
 
 // node_modules/@pubid/pubid/dist/flavors/asme/grammar.js
-var alt2 = (tokens) => tokens.map((t) => str(t)).reduce((a, b) => a.or(b));
+var alt2 = (tokens2) => tokens2.map((t) => str(t)).reduce((a, b) => a.or(b));
 function buildRules36() {
   const rules = {};
   const rule = (name, build) => {
@@ -23474,6 +23474,53 @@ columns: ${colKeep.map((i) => `${t.columns[i]?.label ?? ""}${t.columns[i]?.unit
 ${lines.join("\n")}${elided}`, cols: colKeep.map((i) => `${t.columns[i]?.label ?? ""}${t.columns[i]?.unit ? ` [${t.columns[i].unit}]` : ""}`), rowsShown: shown.length, rowsTotal: t.rows.length };
 }
 
+// workers/worker_public/src/compress.ts
+var STOP2 = new Set(
+  "the a an and or of to in on for with by as at is are was were be been from that this these those it its their there which who what when where how why not no than then also such may must shall can could should would per within under over into about between during before after each any all more most less least other only same both few many some own very one two".split(
+    " "
+  )
+);
+function tokens(text) {
+  return (text.toLowerCase().match(/[a-z0-9]+(?:[.][0-9]+)?/g) ?? []).filter(
+    (t) => /^[0-9]/.test(t) || t.length >= 3 && !STOP2.has(t)
+  );
+}
+function splitSentences(text) {
+  const parts = text.replace(/\r/g, "").split(/(?<=[.!?;])\s+|\n+/).map((s8) => s8.trim()).filter(Boolean);
+  return parts;
+}
+function scoreSentence(sentence, qTerms) {
+  const toks = tokens(sentence);
+  if (!toks.length) return 0;
+  let hits = 0;
+  for (const t of toks) if (qTerms.has(t)) hits += 1;
+  const density = hits / Math.sqrt(toks.length);
+  const numeric = /\d/.test(sentence) ? 0.35 : 0;
+  return density + numeric;
+}
+function compressText(text, query, budgetChars) {
+  const clean = text.replace(/\s+/g, " ").trim();
+  if (!clean || clean.length <= budgetChars) return clean;
+  const qTerms = new Set(tokens(query));
+  const sentences = splitSentences(clean);
+  if (sentences.length <= 1) {
+    const cut = clean.slice(0, Math.max(0, budgetChars));
+    const end = cut.lastIndexOf(" ");
+    return (end > 40 ? cut.slice(0, end) : cut).trim();
+  }
+  const scored = sentences.map((s8, i) => ({ s: s8, i, score: scoreSentence(s8, qTerms) }));
+  const keep = /* @__PURE__ */ new Set([0]);
+  const ranked = scored.slice(1).sort((a, b) => b.score - a.score || a.i - b.i);
+  let used = sentences[0].length + 1;
+  for (const r of ranked) {
+    if (used + r.s.length + 1 > budgetChars) continue;
+    keep.add(r.i);
+    used += r.s.length + 1;
+  }
+  const out = scored.filter((r) => keep.has(r.i)).map((r) => r.s).join(" ").trim();
+  return out.slice(0, budgetChars);
+}
+
 // workers/worker_public/src/pipeline.ts
 function promptVars(extra = {}) {
   const out = { PUBLISHER_NAME: P().publisher.name };
@@ -23596,7 +23643,7 @@ ${listing}` }
     return null;
   }
 }
-function buildMessages(query, hits, lang, history = [], retrievalNote, conversationSummary, budgetTokens = LIMITS.inputTokenBudget) {
+function buildMessages(query, hits, lang, history = [], retrievalNote, conversationSummary, budgetTokens = LIMITS.inputTokenBudget, compress) {
   const corpusNotes = DATASETS().filter(
     (d) => d.note && hits.some((h) => h.metadata.corpus === d.id)
   ).map((d) => d.note).join("\n");
@@ -23641,7 +23688,7 @@ Context passages:
     const pruned = tableSel?.text ?? null;
     if (tableSel) h.metadata.table_selection = { cols: tableSel.cols, rowsShown: tableSel.rowsShown, rowsTotal: tableSel.rowsTotal };
     const figDesc = h.metadata.block === "figure" ? String(h.metadata.figure_description ?? "").trim() : "";
-    const raw = clipToTokens(pruned ?? h.text, LIMITS.maxPassageTokens);
+    const raw = compress ? compressText(String(pruned ?? h.text ?? ""), query, Math.floor(LIMITS.maxPassageTokens * 4 * THRESHOLDS.compressShare)) : clipToTokens(pruned ?? h.text, LIMITS.maxPassageTokens);
     const body = figDesc ? `${raw}
 Caption: ${figDesc}` : raw;
     const t = estTokens(head) + estTokens(body);
@@ -24718,9 +24765,9 @@ async function tokenBudget(env, bucketId) {
   const used = Number(await env.CACHE.get(`t:${today()}:ask:${await sha256Hex(bucketId)}`) ?? "0");
   return { used, limit: tokenLimit(env, bucketId) };
 }
-async function chargeTokens(env, bucketId, tokens) {
-  if (!(tokens > 0)) return;
-  await kvIncr(env.CACHE, `t:${today()}:ask:${await sha256Hex(bucketId)}`, tokens);
+async function chargeTokens(env, bucketId, tokens2) {
+  if (!(tokens2 > 0)) return;
+  await kvIncr(env.CACHE, `t:${today()}:ask:${await sha256Hex(bucketId)}`, tokens2);
 }
 function usageTotal(usage) {
   if (!usage || typeof usage !== "object") return null;
@@ -24789,6 +24836,7 @@ Family summary (precomputed from the registry and the successor graph): ${summar
 }
 
 export {
+  compressText,
   refCodec,
   NO_CONTEXT,
   machineOffers,

package/dist/{chunk-NI4CIK6W.js → chunk-FHRGC6TP.js}

@@ -1,7 +1,7 @@
 import {
   answerEffort,
   effortBudget
-} from "./chunk-VT7DR6NQ.js";
+} from "./chunk-ZD6TYLET.js";
 
 // workers/worker_public/src/ai.ts
 var delay = (ms) => new Promise((r) => setTimeout(r, ms));

package/dist/{chunk-W7J5SFXJ.js → chunk-GJATVGBS.js}

@@ -1,7 +1,7 @@
 import {
   LIMITS,
   sha256Hex
-} from "./chunk-VT7DR6NQ.js";
+} from "./chunk-ZD6TYLET.js";
 import {
   P
 } from "./chunk-3FYJM7LH.js";

package/dist/{chunk-AEVDR7DA.js → chunk-GVCOGEJC.js}

@@ -32,7 +32,7 @@ import {
   tableRetyped,
   toolNote,
   verdictNote
-} from "./chunk-PBHMQZKA.js";
+} from "./chunk-K5WL3DKH.js";
 import {
   NO_CONTEXT,
   STAGE_NAMES,
@@ -75,18 +75,18 @@ import {
   tokenBudget,
   understandQuery,
   usageTotal
-} from "./chunk-MTIAHAOK.js";
+} from "./chunk-E74MNLBC.js";
 import {
   embed,
   generateOnce
-} from "./chunk-NI4CIK6W.js";
+} from "./chunk-FHRGC6TP.js";
 import {
   corsHeaders,
   err,
   json,
   readJson,
   validateQuery
-} from "./chunk-W7J5SFXJ.js";
+} from "./chunk-GJATVGBS.js";
 import {
   cacheableAnswer,
   canonicalRefusal,
@@ -99,7 +99,7 @@ import {
   requestSalt,
   resolveRequestScope,
   standardKeysFrom
-} from "./chunk-4NXH5ZI6.js";
+} from "./chunk-D6UUAL7I.js";
 import {
   LIMITS,
   MODELS,
@@ -110,7 +110,7 @@ import {
   requestEffort,
   roleModel,
   sha256Hex
-} from "./chunk-VT7DR6NQ.js";
+} from "./chunk-ZD6TYLET.js";
 import {
   P
 } from "./chunk-3FYJM7LH.js";
@@ -470,13 +470,19 @@ async function completeTables(db, answer, used) {
   }
   return blocks;
 }
-async function completeFigures(db, answer, alreadyAttached, used = []) {
+async function completeFigures(db, answer, alreadyAttached, used = [], query) {
   const unitForm = (answer.match(/u:fig[\w.-]*/g) ?? []).map((x) => x.replace(/[.,;:)]+$/, ""));
   const bareForm = (answer.match(/\bfig-[\w.-]+\b/g) ?? []).map((x) => x.replace(/[.,;:)]+$/, ""));
   let mentioned = [.../* @__PURE__ */ new Set([...unitForm, ...bareForm.map((x) => x.startsWith("u:") ? x : "u:" + x)])].slice(0, 6);
   const proseNums = new Set(
     (answer.match(/\bFig(?:ure|\.)s?\s*([0-9]{1,2}[a-z]?)/g) ?? []).map((x) => x.replace(/\bFig(?:ure|\.)s?\s*/i, "").toLowerCase())
   );
+  if (query && /\bfig(ure|s)?\b/i.test(query)) {
+    for (const h of used.slice(0, 12)) {
+      if (h.metadata.unit_id && h.metadata.block === "figure") mentioned.push(h.metadata.unit_id);
+    }
+    mentioned = [...new Set(mentioned)].slice(0, 6);
+  }
   if (proseNums.size) {
     const fams = [...new Set(used.map((h) => h.metadata.docidentifier).filter(Boolean))].slice(0, 3);
     for (const fam of fams) {
@@ -1397,9 +1403,10 @@ function parseAblate(body, adminToken, presentedToken, knownStages) {
   const noVerdict = body?.ablate_no_verdict === true;
   const rawRoute = body?.ablate_route;
   const speculative = body?.ablate_speculative === true;
+  const compress = body?.ablate_compress === true;
   const namesAblation = rawStages !== void 0 && rawStages !== null;
   const routeAblation = rawRoute !== void 0 && rawRoute !== null && rawRoute !== "adaptive";
-  if (!namesAblation && !noVerdict && !routeAblation && !speculative) return { ok: true, config: null };
+  if (!namesAblation && !noVerdict && !routeAblation && !speculative && !compress) return { ok: true, config: null };
   if (!adminToken || presentedToken !== adminToken) {
     return {
       ok: false,
@@ -1430,7 +1437,7 @@ function parseAblate(body, adminToken, presentedToken, knownStages) {
     }
     stages = [...new Set(rawStages)];
   }
-  return { ok: true, config: { stages, noVerdict, route, speculative } };
+  return { ok: true, config: { stages, noVerdict, route, speculative, compress } };
 }
 
 // workers/worker_public/src/route.ts
@@ -2636,7 +2643,8 @@ Answer account questions from these records ONLY: name the record when you use i
       // stage-extracted graph facts (GraphRAG) ride the same note channel
       [ops.note, processNote, eNote, regNote, contextNote(declaredCtx, docScope), accountNote, modelNote, vocabNote, memNote, machineNote, conditionNote, aggregationNote, boundaryNote, licenseNote, ...retrieved.notes ?? []].filter(Boolean).join("\n") || void 0,
       summary,
-      budget
+      budget,
+      ablate?.compress === true
     );
     await attachFigureImages(env, messages, usedHits, q2.query);
     const pendingTools = [];
@@ -2946,7 +2954,7 @@ Answer account questions from these records ONLY: name the record when you use i
           ...routeSubset ? { stageSubset: routeSubset ?? void 0 } : {}
         });
         if (retryRetrieve.hits.length > 0) {
-          const { messages: retryMessages, usedHits: retryUsed } = buildMessages(q2.query, retryRetrieve.hits, q2.lang, keptHistory, void 0, summary, budget);
+          const { messages: retryMessages, usedHits: retryUsed } = buildMessages(q2.query, retryRetrieve.hits, q2.lang, keptHistory, void 0, summary, budget, ablate?.compress === true);
           const retryAnswer = await generateOnce(env, model, retryMessages, routeEffort(routed.route));
           if (retryAnswer) {
             answer = canonicalRefusal(retryAnswer, refusalScope(answerCites(retryAnswer)));
@@ -2971,7 +2979,7 @@ Answer account questions from these records ONLY: name the record when you use i
       completionBlocks = await completeTables(env.DB, answer, used);
       if (completionBlocks.length) console.log("contract completion:", completionBlocks.length, "table block(s) attached server-side");
     }
-    completionBlocks.push(...await completeFigures(env.DB, answer, [...c2ns.blocks, ...completionBlocks], used));
+    completionBlocks.push(...await completeFigures(env.DB, answer, [...c2ns.blocks, ...completionBlocks], used, q2.query));
     if (withImage && register?.rows?.length && answer.includes(refusalAnswer())) {
       console.log("blind-answer: the register note carried rows but the answer refused \u2014 regenerating once");
       mark("Re-answering from the register results");

package/dist/{chunk-PBHMQZKA.js → chunk-K5WL3DKH.js}

@@ -1,12 +1,12 @@
 import {
   embed
-} from "./chunk-NI4CIK6W.js";
+} from "./chunk-FHRGC6TP.js";
 import {
   standardKeysFrom
-} from "./chunk-4NXH5ZI6.js";
+} from "./chunk-D6UUAL7I.js";
 import {
   THRESHOLDS
-} from "./chunk-VT7DR6NQ.js";
+} from "./chunk-ZD6TYLET.js";
 import {
   P
 } from "./chunk-3FYJM7LH.js";
@@ -18,7 +18,10 @@ import {
 } from "./chunk-LSZTJI7V.js";
 
 // workers/worker_public/src/refs.ts
-var REF = /\[\[(u:[A-Za-z0-9_-]+)\]\]/g;
+var REF = /\[\[(u:(?:u:)?[A-Za-z0-9_-]+)\]\]/g;
+function normalizeRef(id) {
+  return id.startsWith("u:u:") ? id.slice(2) : id;
+}
 function availableUnitIds(hits) {
   const ids = /* @__PURE__ */ new Set();
   for (const h of hits) {
@@ -28,13 +31,14 @@ function availableUnitIds(hits) {
   return ids;
 }
 function parseRefs(text) {
-  return [...text.matchAll(REF)].map((m) => m[1]);
+  return [...text.matchAll(REF)].map((m) => normalizeRef(m[1]));
 }
 function sanitizeRefs(text, available) {
   const dropped = [];
-  const out = text.replace(REF, (full, id) => {
-    if (available.has(id)) return full;
-    dropped.push(id);
+  const out = text.replace(REF, (_full, id) => {
+    const normalized = normalizeRef(id);
+    if (available.has(normalized)) return `[[${normalized}]]`;
+    dropped.push(normalized);
     return "";
   });
   return { text: out, dropped };

package/dist/{chunk-IGAGXQHV.js → chunk-K7UGBTBZ.js}

@@ -6,24 +6,24 @@ import {
   sessionFrom,
   telemetry,
   understandQuery
-} from "./chunk-MTIAHAOK.js";
+} from "./chunk-E74MNLBC.js";
 import {
   corsHeaders,
   err,
   json,
   readJson,
   validateQuery
-} from "./chunk-W7J5SFXJ.js";
+} from "./chunk-GJATVGBS.js";
 import {
   entitlementScope,
   standardKeysFrom
-} from "./chunk-4NXH5ZI6.js";
+} from "./chunk-D6UUAL7I.js";
 import {
   LIMITS,
   MODELS,
   num,
   sha256Hex
-} from "./chunk-VT7DR6NQ.js";
+} from "./chunk-ZD6TYLET.js";
 import {
   portModelRunner
 } from "./chunk-LSZTJI7V.js";

package/dist/{chunk-VT7DR6NQ.js → chunk-ZD6TYLET.js}

@@ -81,6 +81,9 @@ var THRESHOLDS = {
    *  then states a measurement — claim support in the cited passages —
    *  not only the source rung. Fails open (a slow or unparseable check
    *  leaves the source-rung note unchanged). */
+  // the extractive compression arm (TODO.sota/02 row 5): when ablated
+  // on, each passage body is compressed to this share of its budget
+  compressShare: 0.6,
   entailmentGate: true,
   entailmentBudgetMs: 6e3,
   entailmentSupportedFloor: 0.9,

package/dist/completion.d.ts

@@ -2,4 +2,4 @@ import type { StoreQuery } from "./ports/store.ts";
 import { type ResolvedBlock } from "./refs.ts";
 import type { Hit } from "./pipeline.ts";
 export declare function completeTables(db: StoreQuery, answer: string, used: Hit[]): Promise<ResolvedBlock[]>;
-export declare function completeFigures(db: StoreQuery, answer: string, alreadyAttached: ResolvedBlock[], used?: Hit[]): Promise<ResolvedBlock[]>;
+export declare function completeFigures(db: StoreQuery, answer: string, alreadyAttached: ResolvedBlock[], used?: Hit[], query?: string): Promise<ResolvedBlock[]>;

package/dist/config.d.ts

@@ -49,6 +49,7 @@ export declare const THRESHOLDS: {
      *  then states a measurement — claim support in the cited passages —
      *  not only the source rung. Fails open (a slow or unparseable check
      *  leaves the source-rung note unchanged). */
+    readonly compressShare: 0.6;
     readonly entailmentGate: true;
     readonly entailmentBudgetMs: 6000;
     readonly entailmentSupportedFloor: 0.9;

package/dist/pipeline.d.ts

@@ -50,7 +50,11 @@ export declare function splitHistory(history: HistoryTurn[], budgetTokens: numbe
  *  hard queries (cascade stage after the cross-encoder). Null = keep the
  *  incoming order (timeout/parse failure never blocks serving). */
 export declare function listwiseRerank(env: any, model: string, query: string, hits: Hit[]): Promise<Hit[] | null>;
-export declare function buildMessages(query: string, hits: Hit[], lang?: string, history?: HistoryTurn[], retrievalNote?: string, conversationSummary?: string, budgetTokens?: number): BuiltMessages;
+export declare function buildMessages(query: string, hits: Hit[], lang?: string, history?: HistoryTurn[], retrievalNote?: string, conversationSummary?: string, budgetTokens?: number, 
+/** the extractive compression arm (TODO.sota/02 row 5): ablate_compress
+ *  arms it per ask — each passage body becomes query-weighted sentence
+ *  selection at compressShare of its budget. Off in serving. */
+compress?: boolean): BuiltMessages;
 export declare function citations(hits: Hit[]): {
     doc_id: string;
     docidentifier: string;

package/dist/search-IXNVNOOY.js

@@ -0,0 +1,14 @@
+import {
+  handleSearch
+} from "./chunk-K7UGBTBZ.js";
+import "./chunk-E74MNLBC.js";
+import "./chunk-FHRGC6TP.js";
+import "./chunk-GJATVGBS.js";
+import "./chunk-ETLSWSZ7.js";
+import "./chunk-D6UUAL7I.js";
+import "./chunk-ZD6TYLET.js";
+import "./chunk-3FYJM7LH.js";
+import "./chunk-LSZTJI7V.js";
+export {
+  handleSearch
+};

package/dist/{tools-W5HGME7W.js → tools-JZY6GSYC.js}

@@ -4,10 +4,10 @@ import {
   parseToolCall,
   runTool,
   toolNote
-} from "./chunk-PBHMQZKA.js";
-import "./chunk-NI4CIK6W.js";
-import "./chunk-4NXH5ZI6.js";
-import "./chunk-VT7DR6NQ.js";
+} from "./chunk-K5WL3DKH.js";
+import "./chunk-FHRGC6TP.js";
+import "./chunk-D6UUAL7I.js";
+import "./chunk-ZD6TYLET.js";
 import "./chunk-3FYJM7LH.js";
 import "./chunk-LSZTJI7V.js";
 export {

package/dist/worker_mcp/src/index.js

@@ -1,7 +1,7 @@
 import {
   authenticate
-} from "../../chunk-W7J5SFXJ.js";
-import "../../chunk-VT7DR6NQ.js";
+} from "../../chunk-GJATVGBS.js";
+import "../../chunk-ZD6TYLET.js";
 import {
   P,
   setProfile

package/dist/worker_public/src/config.js

@@ -16,7 +16,7 @@ import {
   roleModel,
   sha256Hex,
   today
-} from "../../chunk-VT7DR6NQ.js";
+} from "../../chunk-ZD6TYLET.js";
 import "../../chunk-3FYJM7LH.js";
 export {
   DATASETS,

package/dist/worker_public/src/index.js

@@ -4,20 +4,21 @@ import {
   handleMemories,
   scoreFaithfulness,
   scoreJudge
-} from "../../chunk-AEVDR7DA.js";
+} from "../../chunk-GVCOGEJC.js";
 import "../../chunk-ECOYO7EX.js";
 import {
   TOOLS_REGISTRY,
   bindModelNode,
   modelGroundingBlock
-} from "../../chunk-PBHMQZKA.js";
+} from "../../chunk-K5WL3DKH.js";
 import {
   handleSearch
-} from "../../chunk-IGAGXQHV.js";
+} from "../../chunk-K7UGBTBZ.js";
 import {
   buildMessages,
   citations,
   clientIp,
+  compressText,
   creditBudget,
   editionNote,
   fill,
@@ -37,11 +38,11 @@ import {
   sessionFrom,
   telemetry,
   understandQuery
-} from "../../chunk-MTIAHAOK.js";
+} from "../../chunk-E74MNLBC.js";
 import {
   embed,
   generateOnce
-} from "../../chunk-NI4CIK6W.js";
+} from "../../chunk-FHRGC6TP.js";
 import {
   authenticate,
   corsHeaders,
@@ -50,14 +51,14 @@ import {
   readJson,
   validateQuery,
   withCors
-} from "../../chunk-W7J5SFXJ.js";
+} from "../../chunk-GJATVGBS.js";
 import {
   canonicalRefusal
 } from "../../chunk-ETLSWSZ7.js";
 import {
   entitlementScope,
   standardKeysFrom
-} from "../../chunk-4NXH5ZI6.js";
+} from "../../chunk-D6UUAL7I.js";
 import {
   LIMITS,
   MODELS,
@@ -68,7 +69,7 @@ import {
   roleModel,
   sha256Hex,
   today
-} from "../../chunk-VT7DR6NQ.js";
+} from "../../chunk-ZD6TYLET.js";
 import {
   P,
   setProfile
@@ -757,53 +758,6 @@ async function handleRevokeKey(env, req, id) {
   return json({ ok: true, updated: r.meta?.changes ?? 0 });
 }
 
-// workers/worker_public/src/compress.ts
-var STOP = new Set(
-  "the a an and or of to in on for with by as at is are was were be been from that this these those it its their there which who what when where how why not no than then also such may must shall can could should would per within under over into about between during before after each any all more most less least other only same both few many some own very one two".split(
-    " "
-  )
-);
-function tokens(text) {
-  return (text.toLowerCase().match(/[a-z0-9]+(?:[.][0-9]+)?/g) ?? []).filter(
-    (t) => /^[0-9]/.test(t) || t.length >= 3 && !STOP.has(t)
-  );
-}
-function splitSentences(text) {
-  const parts = text.replace(/\r/g, "").split(/(?<=[.!?;])\s+|\n+/).map((s) => s.trim()).filter(Boolean);
-  return parts;
-}
-function scoreSentence(sentence, qTerms) {
-  const toks = tokens(sentence);
-  if (!toks.length) return 0;
-  let hits = 0;
-  for (const t of toks) if (qTerms.has(t)) hits += 1;
-  const density = hits / Math.sqrt(toks.length);
-  const numeric = /\d/.test(sentence) ? 0.35 : 0;
-  return density + numeric;
-}
-function compressText(text, query, budgetChars) {
-  const clean = text.replace(/\s+/g, " ").trim();
-  if (!clean || clean.length <= budgetChars) return clean;
-  const qTerms = new Set(tokens(query));
-  const sentences = splitSentences(clean);
-  if (sentences.length <= 1) {
-    const cut = clean.slice(0, Math.max(0, budgetChars));
-    const end = cut.lastIndexOf(" ");
-    return (end > 40 ? cut.slice(0, end) : cut).trim();
-  }
-  const scored = sentences.map((s, i) => ({ s, i, score: scoreSentence(s, qTerms) }));
-  const keep = /* @__PURE__ */ new Set([0]);
-  const ranked = scored.slice(1).sort((a, b) => b.score - a.score || a.i - b.i);
-  let used = sentences[0].length + 1;
-  for (const r of ranked) {
-    if (used + r.s.length + 1 > budgetChars) continue;
-    keep.add(r.i);
-    used += r.s.length + 1;
-  }
-  const out = scored.filter((r) => keep.has(r.i)).map((r) => r.s).join(" ").trim();
-  return out.slice(0, budgetChars);
-}
-
 // workers/worker_public/prompts/research.md
 var research_default = `You are a sufficiency judge for a research loop over {{PUBLISHER_NAME}} publications. Given the research question and the passages collected so far (across iterations), decide whether the collected evidence is SUFFICIENT to write a complete, well-grounded answer.
 
@@ -1165,13 +1119,13 @@ async function callToolOnce(env, ctx, _req, tier, key, name, args, t0) {
       // stream:false forces the JSON lane (anon defaults to SSE)
       body: JSON.stringify({ ...args, stream: false })
     });
-    const { runTool, TOOLS_REGISTRY: TOOLS_REGISTRY2 } = await import("../../tools-W5HGME7W.js");
+    const { runTool, TOOLS_REGISTRY: TOOLS_REGISTRY2 } = await import("../../tools-JZY6GSYC.js");
     let payload;
     if (TOOLS_REGISTRY2.some((t) => t.name === name && t.audiences.includes("mcp"))) {
       const r = await runTool({ ...env, DB: env.DB }, { name, args }, "mcp");
       payload = r ? { tool: r.name, query: r.query, result: r.output } : { error: { message: `tool ${name} returned nothing for the given arguments` } };
     } else {
-      const res = name === "ask" ? await (await import("../../ask-ND63B3CF.js")).handleAsk(env, ctx, inner, tier, key) : await (await import("../../search-FW5O32BY.js")).handleSearch(env, ctx, inner, tier, key);
+      const res = name === "ask" ? await (await import("../../ask-B7XXKYZP.js")).handleAsk(env, ctx, inner, tier, key) : await (await import("../../search-IXNVNOOY.js")).handleSearch(env, ctx, inner, tier, key);
       payload = await res.json().catch(() => ({ error: { message: "tool transport failed", status: res.status } }));
     }
     telemetry(env, ctx, tier, `mcp:${name}`, null, !payload?.error, JSON.stringify(payload).length, await sha256Hex(`${name}:${JSON.stringify(args ?? {})}`), void 0, void 0, { durationMs: Date.now() - t0 });

package/dist/worker_public/src/requestScope.js

@@ -4,8 +4,8 @@ import {
   requestSalt,
   resolveRequestScope,
   standardKeysFrom
-} from "../../chunk-4NXH5ZI6.js";
-import "../../chunk-VT7DR6NQ.js";
+} from "../../chunk-D6UUAL7I.js";
+import "../../chunk-ZD6TYLET.js";
 import "../../chunk-3FYJM7LH.js";
 export {
   entitlementScope,

package/package.json

@@ -27,7 +27,7 @@
     "playwright": "^1.62.1",
     "yaml": "^2.9.1"
   },
-  "version": "0.2.162",
+  "version": "0.2.164",
   "description": "The Konneal engine: the publisher-agnostic build pipeline and API plane for standards intelligence (retrieval, answer contract, verdicts, evaluation).",
   "license": "BSD-3-Clause",
   "type": "module",

package/workers/worker_public/src/ablate.ts

@@ -34,6 +34,11 @@ export interface AblateConfig {
    *  diversified subset + one strong verifier. Off in serving until the
    *  gate promotes it; the grid arms it per ask. */
   speculative: boolean;
+  /** Extractive prompt compression (TODO.sota/02 row 5): each passage
+   *  body is query-weighted sentence selection at a fraction of its
+   *  budget before the answer model reads it. Off in serving; the grid
+   *  arms it per ask (the row's accuracy-at-token-budget gate). */
+  compress: boolean;
 }
 
 export type AblateParse =
@@ -55,10 +60,11 @@ export function parseAblate(
   const noVerdict = body?.ablate_no_verdict === true;
   const rawRoute = body?.ablate_route;
   const speculative = body?.ablate_speculative === true;
+  const compress = body?.ablate_compress === true;
   const namesAblation = rawStages !== undefined && rawStages !== null;
   const routeAblation = rawRoute !== undefined && rawRoute !== null && rawRoute !== "adaptive";
 
-  if (!namesAblation && !noVerdict && !routeAblation && !speculative) return { ok: true, config: null };
+  if (!namesAblation && !noVerdict && !routeAblation && !speculative && !compress) return { ok: true, config: null };
 
   if (!adminToken || presentedToken !== adminToken) {
     return {
@@ -93,5 +99,5 @@ export function parseAblate(
     stages = [...new Set(rawStages as string[])];
   }
 
-  return { ok: true, config: { stages, noVerdict, route, speculative } };
+  return { ok: true, config: { stages, noVerdict, route, speculative, compress } };
 }

package/workers/worker_public/src/ask.ts

@@ -1500,6 +1500,7 @@ let convEntities: Array<{ entity: string; kind: string }> = [];
     [ops.note, processNote, eNote, regNote, contextNote(declaredCtx, docScope), accountNote, modelNote, vocabNote, memNote, machineNote, conditionNote, aggregationNote, boundaryNote, licenseNote, ...(retrieved.notes ?? [])].filter(Boolean).join("\n") || undefined,
     summary,
     budget,
+    ablate?.compress === true,
   );
   await attachFigureImages(env, messages, usedHits, q.query);
   // the agent loop (the owner's 2026-09-28 direction): ONE bounded tool
@@ -1953,7 +1954,7 @@ let convEntities: Array<{ entity: string; kind: string }> = [];
         ...(routeSubset ? { stageSubset: routeSubset ?? undefined } : {}),
       });
       if (retryRetrieve.hits.length > 0) {
-        const { messages: retryMessages, usedHits: retryUsed } = buildMessages(q.query, retryRetrieve.hits, q.lang, keptHistory, undefined, summary, budget);
+        const { messages: retryMessages, usedHits: retryUsed } = buildMessages(q.query, retryRetrieve.hits, q.lang, keptHistory, undefined, summary, budget, ablate?.compress === true);
         const retryAnswer = await generateOnce(env, model, retryMessages, routeEffort(routed.route));
         // the answer now comes from the retry passages — citations must follow
         if (retryAnswer) {
@@ -1993,7 +1994,7 @@ let convEntities: Array<{ entity: string; kind: string }> = [];
   }
 
   // figure completion (#172) — see ./completion for the rationale
-  completionBlocks.push(...(await completeFigures(env.DB, answer!, [...c2ns.blocks, ...completionBlocks], used)));
+  completionBlocks.push(...(await completeFigures(env.DB, answer!, [...c2ns.blocks, ...completionBlocks], used, q.query)));
 
   // the blind-answer regeneration (2026-10-03, live on the photo case):
   // the multimodal call can answer AROUND an attached image — a 200 whose

package/workers/worker_public/src/completion.ts

@@ -59,6 +59,7 @@ export async function completeFigures(
   answer: string,
   alreadyAttached: ResolvedBlock[],
   used: Hit[] = [],
+  query?: string,
 ): Promise<ResolvedBlock[]> {
   // the model names figures BOTH ways: unit ids (u:fig-3) and bare
   // producer anchors (fig-2a of D 36) — collect both forms; bare
@@ -74,6 +75,18 @@ export async function completeFigures(
     (answer.match(/\bFig(?:ure|\.)s?\s*([0-9]{1,2}[a-z]?)/g) ?? [])
       .map((x) => x.replace(/\bFig(?:ure|\.)s?\s*/i, "").toLowerCase()),
   );
+  // the figure-SEEKING question (the MKO typed units carry UUID ids —
+  // no fig-N the model could name, and "sketch"/"drawing" prose the
+  // prose-number scan cannot see): when the question asks about a
+  // figure and a figure unit grounded the answer, the contract
+  // completes with THAT unit — the tables' mechanical philosophy, not
+  // a hope that the model names an unnameable id
+  if (query && /\bfig(ure|s)?\b/i.test(query)) {
+    for (const h of used.slice(0, 12)) {
+      if (h.metadata.unit_id && h.metadata.block === "figure") mentioned.push(h.metadata.unit_id);
+    }
+    mentioned = [...new Set(mentioned)].slice(0, 6);
+  }
   if (proseNums.size) {
     const fams = [...new Set(used.map((h) => h.metadata.docidentifier).filter(Boolean))].slice(0, 3);
     for (const fam of fams) {

package/workers/worker_public/src/config.ts

@@ -78,6 +78,9 @@ export const THRESHOLDS = {
    *  then states a measurement — claim support in the cited passages —
    *  not only the source rung. Fails open (a slow or unparseable check
    *  leaves the source-rung note unchanged). */
+  // the extractive compression arm (TODO.sota/02 row 5): when ablated
+  // on, each passage body is compressed to this share of its budget
+  compressShare: 0.6,
   entailmentGate: true,
   entailmentBudgetMs: 6000,
   entailmentSupportedFloor: 0.9,

package/workers/worker_public/src/pipeline.ts

@@ -5,6 +5,7 @@ import systemPromptText from "../prompts/system.md";
 import conversationalPromptText from "../prompts/conversational.md";
 import listwisePromptText from "../prompts/listwise.md";
 import { tableSelection } from "./tablecontext";
+import { compressText } from "./compress";
 
 // the pinned refusal sentence lives with the canonicalizer in ./refusal
 // (refusals are never cached: a refusal says "retrieval found nothing",
@@ -287,6 +288,10 @@ export function buildMessages(
   retrievalNote?: string,
   conversationSummary?: string,
   budgetTokens: number = LIMITS.inputTokenBudget,
+  /** the extractive compression arm (TODO.sota/02 row 5): ablate_compress
+   *  arms it per ask — each passage body becomes query-weighted sentence
+   *  selection at compressShare of its budget. Off in serving. */
+  compress?: boolean,
 ): BuiltMessages {
   // per-corpus guidance travels WITH the dataset (config.ts): every
   // dataset whose corpus appears in the passages contributes its note —
@@ -365,7 +370,12 @@ export function buildMessages(
     // the drawing and a reason to cite the unit; absent for every other
     // block type and for uncaptioned figures
     const figDesc = (h.metadata as any).block === "figure" ? String((h.metadata as any).figure_description ?? "").trim() : "";
-    const raw = clipToTokens(pruned ?? h.text, LIMITS.maxPassageTokens);
+    // the compression arm replaces clipping: the query-weighted extractor
+    // bounds the text itself (deterministic, never grows it); clipping
+    // stays the default
+    const raw = compress
+      ? compressText(String(pruned ?? h.text ?? ""), query, Math.floor(LIMITS.maxPassageTokens * 4 * THRESHOLDS.compressShare))
+      : clipToTokens(pruned ?? h.text, LIMITS.maxPassageTokens);
     const body = figDesc ? `${raw}\nCaption: ${figDesc}` : raw;
     const t = estTokens(head) + estTokens(body);
     if (t <= remain) {

package/workers/worker_public/src/refs.ts

@@ -15,7 +15,18 @@ export interface ResolvedBlock {
   payload: Record<string, unknown>;
 }
 
-const REF = /\[\[(u:[A-Za-z0-9_-]+)\]\]/g;
+// The passage headers show `unit u:<id> (table)`, so the model copies
+// the WHOLE prefixed id into the token — `[[u:u:<id>]]` — and the inner
+// colon makes the strict grammar unparseable: neither resolved nor
+// sanitized, the raw token leaked to the client (observed live on the
+// R 134-2 figure question, 0.2.162). The grammar tolerates the doubled
+// prefix; normalizeRef folds it back to the single form every other
+// layer keys on.
+const REF = /\[\[(u:(?:u:)?[A-Za-z0-9_-]+)\]\]/g;
+
+function normalizeRef(id: string): string {
+  return id.startsWith("u:u:") ? id.slice(2) : id;
+}
 
 /** unit ids the passages actually contain (chunk metadata carries
  *  unit_id on typed MKO chunks). */
@@ -29,16 +40,19 @@ export function availableUnitIds(hits: ReadonlyArray<{ metadata: { unit_id?: str
 }
 
 export function parseRefs(text: string): string[] {
-  return [...text.matchAll(REF)].map((m) => m[1]);
+  return [...text.matchAll(REF)].map((m) => normalizeRef(m[1]));
 }
 
 /** Drop refs that don't resolve to a used passage unit — a dangling ref
  *  renders as nothing (the token removed), and the violation is logged. */
 export function sanitizeRefs(text: string, available: ReadonlySet<string>): { text: string; dropped: string[] } {
   const dropped: string[] = [];
-  const out = text.replace(REF, (full, id: string) => {
-    if (available.has(id)) return full;
-    dropped.push(id);
+  const out = text.replace(REF, (_full, id: string) => {
+    const normalized = normalizeRef(id);
+    // normalized already carries the single "u:" prefix — the template
+    // must not add another
+    if (available.has(normalized)) return `[[${normalized}]]`;
+    dropped.push(normalized);
     return "";
   });
   return { text: out, dropped };

package/dist/ask-ND63B3CF.js

@@ -1,16 +0,0 @@
-import {
-  handleAsk
-} from "./chunk-AEVDR7DA.js";
-import "./chunk-ECOYO7EX.js";
-import "./chunk-PBHMQZKA.js";
-import "./chunk-MTIAHAOK.js";
-import "./chunk-NI4CIK6W.js";
-import "./chunk-W7J5SFXJ.js";
-import "./chunk-ETLSWSZ7.js";
-import "./chunk-4NXH5ZI6.js";
-import "./chunk-VT7DR6NQ.js";
-import "./chunk-3FYJM7LH.js";
-import "./chunk-LSZTJI7V.js";
-export {
-  handleAsk
-};

package/dist/search-FW5O32BY.js

@@ -1,14 +0,0 @@
-import {
-  handleSearch
-} from "./chunk-IGAGXQHV.js";
-import "./chunk-MTIAHAOK.js";
-import "./chunk-NI4CIK6W.js";
-import "./chunk-W7J5SFXJ.js";
-import "./chunk-ETLSWSZ7.js";
-import "./chunk-4NXH5ZI6.js";
-import "./chunk-VT7DR6NQ.js";
-import "./chunk-3FYJM7LH.js";
-import "./chunk-LSZTJI7V.js";
-export {
-  handleSearch
-};