@konneal/engine 0.1.3 → 0.2.0

package/workers/worker_public/src/pipeline.ts

@@ -4,7 +4,7 @@ import { LIMITS, MODELS, DATASETS, THRESHOLDS, processExpansion } from "./config
 import systemPromptText from "../prompts/system.md";
 import conversationalPromptText from "../prompts/conversational.md";
 import listwisePromptText from "../prompts/listwise.md";
-import { tableContext } from "./tablecontext";
+import { tableSelection } from "./tablecontext";
 
 // the pinned refusal sentence lives with the canonicalizer in ./refusal
 // (refusals are never cached: a refusal says "retrieval found nothing",
@@ -52,6 +52,9 @@ export interface Retrieved {
    *  among them (dense retrieval alone binds everyday words to the wrong
    *  term: measured "keeps drifting" → creep 0.69 vs durability 0.54) */
   glossary?: GlossaryEntry[];
+  /** structured facts stages extracted from the graph (GraphRAG) —
+   *  merged into the answer prompt's retrieval note */
+  notes?: string[];
 }
 
 // Short follow-ups are usually elliptical ("and the limits?") — fold the
@@ -130,13 +133,14 @@ export async function retrieve(
 
   const ctx: PipelineContext = {
     env, query, rq, folded, u, filters, filter, vector, lexicalHits,
-    matches: [], hits: [], finalHits: [], glossary: [], opts, lane: {},
+    matches: [], hits: [], finalHits: [], glossary: [], notes: [], opts, lane: {},
   };
   await runStages(STAGES, ctx);
   return {
     hits: ctx.finalHits,
     filters: ctx.filters ?? {},
     ...(ctx.glossary?.length ? { glossary: ctx.glossary } : {}),
+    ...(ctx.notes?.length ? { notes: ctx.notes } : {}),
   };
 }
 
@@ -333,7 +337,9 @@ export function buildMessages(
     const head = `[${usedHits.length + 1}] ${label}${unitTag} ${h.metadata.clause_title ? "— " + h.metadata.clause_title : ""}\n`;
     // tables: schema-aware pruning from the producer payload; the
     // stored text is the fallback (pruning never goes below baseline)
-    const pruned = (h.metadata as any).block === "table" ? tableContext(h.metadata, query) : null;
+    const tableSel = (h.metadata as any).block === "table" ? tableSelection(h.metadata, query) : null;
+    const pruned = tableSel?.text ?? null;
+    if (tableSel) (h.metadata as any).table_selection = { cols: tableSel.cols, rowsShown: tableSel.rowsShown, rowsTotal: tableSel.rowsTotal };
     const body = clipToTokens(pruned ?? h.text, LIMITS.maxPassageTokens);
     const t = estTokens(head) + estTokens(body);
     if (t <= remain) {

package/workers/worker_public/src/profile.gen.ts

@@ -104,6 +104,7 @@ export const PROFILE = {
   },
   "prompts": {
     "vars": {
+      "publisher_identity": "the Fixture Organization — a worldwide organization that publishes the fixture corpus",
       "assistant_identity": "the fixture assistant — a public service answering questions about the fixture publisher's documents",
       "refusal_sentence": "I don't have information on this in the indexed fixture documents.",
       "account_note_source": "the user's own fixture account",

package/workers/worker_public/src/projects.ts

@@ -30,8 +30,10 @@ export async function handleProjects(env: any, sub: string, req: Request, route:
   }
   if (method === "POST") {
     const body: any = await req.json().catch(() => null);
-    if (typeof body?.project_id === "string") {
-      // move a conversation in/out of a project (membership-as-move)
+    if (body && (typeof body.project_id === "string" || body.project_id === null)) {
+      // move a conversation in/out of a project (membership-as-move) —
+      // null UNFILES (the 2026-09-16 bug: the string-only guard let the
+      // null case fall through to create-project and 400)
       const convId = String(body.conversation_id ?? "");
       if (!/^[a-zA-Z0-9_-]{8,64}$/.test(convId)) return err(400, "invalid_input", "bad conversation id");
       const target = body.project_id === null ? null : String(body.project_id);

package/workers/worker_public/src/quota.ts

@@ -39,13 +39,15 @@ export function telemetry(
   answerChars: number,
   queryHash: string,
   lang?: string,
+  cache?: "exact" | "semantic",
+  meta?: { durationMs?: number; keyId?: string | null },
 ) {
   const day = today();
   ctx.waitUntil(
     env.DB.batch([
       env.DB.prepare(
-        "INSERT INTO queries (ts, day, tier, route, model, ok, answer_chars, query_hash, lang) VALUES (?1,?2,?3,?4,?5,?6,?7,?8,?9)",
-      ).bind(new Date().toISOString(), day, tier, route, model, ok ? 1 : 0, answerChars, queryHash, lang ?? null),
+        "INSERT INTO queries (ts, day, tier, route, model, ok, answer_chars, query_hash, lang, cache, duration_ms, key_id) VALUES (?1,?2,?3,?4,?5,?6,?7,?8,?9,?10,?11,?12)",
+      ).bind(new Date().toISOString(), day, tier, route, model, ok ? 1 : 0, answerChars, queryHash, lang ?? null, cache ?? null, meta?.durationMs ?? null, meta?.keyId ?? null),
       env.DB.prepare(
         "INSERT INTO spend (day, tier, model, requests) VALUES (?1,?2,?3,1) ON CONFLICT(day, tier, model) DO UPDATE SET requests = requests + 1",
       ).bind(day, tier, model ?? "none"),

package/workers/worker_public/src/research.ts

@@ -31,6 +31,47 @@ export async function handleResearch(env: Env, ctx: ExecutionContext, req: Reque
   if (!q) return err(400, "invalid_input", `query is required (1-${LIMITS.maxInputChars} chars)`);
   const maxIters = Math.min(Math.max(Number(body?.max_iterations) || 3, 1), 3);
 
+  // the passes, streamed when the caller asks: each retrieval and each
+  // sufficiency judgement is an event, so the reader watches the loop
+  // work instead of watching a spinner — the final event carries the
+  // same result object the JSON path returns
+  if (body?.stream === true) {
+    const enc = new TextEncoder();
+    const started = Date.now();
+    const stream = new ReadableStream({
+      async start(ctrl) {
+        const send = (e: unknown) => ctrl.enqueue(enc.encode(`data: ${JSON.stringify(e)}\n\n`));
+        try {
+          const out = await run(env, ctx, q, maxIters, send);
+          send({ type: "done", ...out });
+        } catch (e: any) {
+          send({ type: "error", message: String(e?.message ?? e).slice(0, 200) });
+        } finally {
+          ctrl.close();
+        }
+      },
+    });
+    return new Response(stream, {
+      headers: {
+        "content-type": "text/event-stream; charset=utf-8",
+        "cache-control": "no-cache",
+        "x-accel-buffering": "no",
+        ...corsHeaders(req),
+      },
+    });
+    void started;
+  }
+  const out = await run(env, ctx, q, maxIters);
+  return json({ ...out, ...corsHeaders(req) });
+}
+
+async function run(
+  env: Env,
+  ctx: ExecutionContext,
+  q: { query: string; lang?: string },
+  maxIters: number,
+  emit?: (e: unknown) => void,
+): Promise<{ answer: string; citations: unknown[]; model: string; query_hash: string; research: { iterations: number; passages: number; elapsed_ms: number; sufficient: boolean | null } }> {
   const started = Date.now();
   const queryHash = await sha256Hex(q.query);
   const understanding = await understandQuery(portModelRunner(env), MODELS.understand, q.query, [], []);
@@ -44,6 +85,7 @@ export async function handleResearch(env: Env, ctx: ExecutionContext, req: Reque
 
   for (let i = 0; i < maxIters; i++) {
     iterations = i + 1;
+    emit?.({ type: "pass", n: iterations, of: maxIters, phase: "retrieving" });
     let retrieved: { hits: Hit[] };
     try {
       retrieved = await retrieve(env, q.query, {
@@ -95,6 +137,15 @@ export async function handleResearch(env: Env, ctx: ExecutionContext, req: Reque
       }
     })();
     console.log("research iter", iterations, "passages", passages.length, "sufficient:", judge?.sufficient);
+    emit?.({
+      type: "pass",
+      n: iterations,
+      of: maxIters,
+      phase: "judged",
+      passages: passages.length,
+      sufficient: judge?.sufficient ?? null,
+      ...(judge && !judge.sufficient && judge.missing ? { missing: judge.missing.slice(0, 300) } : {}),
+    });
     if (!judge || judge.sufficient || !judge.missing) break;
     // fold, don't accumulate: appending every round's `missing` compounds
     // stale wants; the next retrieval focuses on the ORIGINAL question plus
@@ -104,14 +155,14 @@ export async function handleResearch(env: Env, ctx: ExecutionContext, req: Reque
 
   const used = [...accumulated.values()];
   if (!used.length) {
-    return err(503, "retrieval_unavailable", "Search is briefly busy — please retry in a moment.");
+    throw new Error("Search is briefly busy — please retry in a moment.");
   }
   const { messages, usedHits } = buildMessages(q.query, used, q.lang, [], eNote || undefined, undefined, LIMITS.inputTokenBudget);
   let answer = await generateOnce(env, MODELS.research, messages);
   if (answer === null) answer = await generateOnce(env, MODELS.fallback, messages);
   if (answer === null) {
     telemetry(env, ctx, "member", "research", MODELS.research, false, 0, queryHash, q.lang);
-    return err(502, "generation_failed", "The generation model is unavailable; please retry.");
+    throw new Error("The generation model is unavailable; please retry.");
   }
   answer = canonicalRefusal(answer);
   const anchors = checkQuoteAnchors(answer, used.map((h: Hit) => h.text));
@@ -124,5 +175,6 @@ export async function handleResearch(env: Env, ctx: ExecutionContext, req: Reque
     research: { iterations, passages: used.length, elapsed_ms: Date.now() - started, sufficient: judge?.sufficient ?? null },
   };
   telemetry(env, ctx, "member", "research", MODELS.research, true, answer.length, queryHash, q.lang);
-  return json({ ...out, ...corsHeaders(req) });
+  emit?.({ type: "pass", n: iterations, of: maxIters, phase: "writing", passages: used.length });
+  return out;
 }

package/workers/worker_public/src/stages/citationProbe.ts

@@ -0,0 +1,145 @@
+// Citation probe (deterministic, GraphRAG): questions asking what a
+// publication CITES/REFERENCES need the bibliography chunks in the
+// pool, but the bibliography's embedding rarely matches the question's
+// phrasing — and prior editions may carry references the current one
+// dropped. This stage pattern-matches the citation question shape and
+// does two things for the named document family (all editions):
+//   1. surfaces the bibliography SECTIONS as passages (FTS over the
+//      chunk store — no embedding similarity involved), and
+//   2. injects the graph's structured `cites` edges (built at ingest
+//      from the same bibliographies) as an authoritative note — the
+//      graph answers the STRUCTURE (the list of cited standards), the
+//      passages ground it verbatim.
+// Additive: results join the pool; nothing is filtered.
+import type { Stage } from "./types.ts";
+import { namedDocumentIn } from "../context.ts";
+import { refCodec } from "../codecs.ts";
+import type { Hit } from "../../../shared/chunk.ts";
+
+const CITE_PATTERN = /\b(?:cite[sd]?|citing|referenc(?:e|es|ed|ing)|list[s]?|quote[sd]?)\b/i;
+const REFS_PATTERN = /\b(?:standard|publication|document|normative|bibliograph)/i;
+
+export interface CiteRow {
+  docidentifier: string;
+  edition: string;
+  active: number;
+  label: string;
+}
+
+/** The authoritative citation note: per edition, the cited standards in
+ * bibliography order. docLabel is the codec-normalized identifier — a
+ * bare number never appears. */
+export function citationGraphNote(docLabel: string, rows: CiteRow[], cap = 30): string {
+  const bySrc = new Map<string, { active: boolean; labels: string[] }>();
+  for (const r of rows) {
+    const key = r.edition && !r.docidentifier.includes(r.edition) ? `${r.docidentifier}:${r.edition}` : r.docidentifier;
+    let e = bySrc.get(key);
+    if (!e) bySrc.set(key, (e = { active: !!r.active, labels: [] }));
+    if (e.labels.length < cap && !e.labels.includes(r.label)) e.labels.push(r.label);
+  }
+  if (!bySrc.size) return "";
+  const lines = [...bySrc.entries()]
+    .sort((a, b) => Number(b[1].active) - Number(a[1].active))
+    .map(([k, v]) => `- ${k}${v.active ? " (active edition)" : ""} cites: ${v.labels.join(", ")}`);
+  return [
+    `Citation graph (authoritative — extracted from the indexed bibliographies of ${docLabel}):`,
+    ...lines,
+    `When the question asks what ${docLabel} cites or references, answer from this list, name each standard exactly as listed, and cite the bibliography passage(s) provided in the context.`,
+  ].join("\n");
+}
+
+export const citationProbe: Stage = {
+  name: "citation-probe",
+  failure: "additive",
+  when: (c) => {
+    if (!CITE_PATTERN.test(c.query) || !REFS_PATTERN.test(c.query)) return false;
+    // the TEXT-derived naming (namedDocumentIn), never the LLM's
+    // extraction — the understand model may omit doc_number for this
+    // query shape (measured: it did)
+    const named = namedDocumentIn(c.query);
+    if (!named) return false;
+    (c as any).__citeDocNum = named.doc_number;
+    // the family key is codec-derived (label → family), never a bare number
+    (c as any).__citeFamily = refCodec().familyOf(named.label);
+    (c as any).__citeLabel = named.label;
+    (c as any).__citeEdition = named.edition ?? null;
+    return true;
+  },
+  prefetch: (c) => {
+    // deterministic: the chunk store's own bibliography-titled chunks for
+    // this document family — no embedding similarity involved (three
+    // vector-probe iterations measured: the embedding never matched
+    // reliably)
+    const docNum = String((c as any).__citeDocNum ?? c.u?.doc_number ?? "");
+    const family = (c as any).__citeFamily as string | null;
+    c.lane["citation-probe"] = Promise.all([
+      (async () => {
+        if (!docNum) return [] as Hit[];
+        try {
+          // chunks_fts is a virtual FTS5 table over chunks.fts_text — it
+          // has NO metadata columns. The doc_number and clause_title live
+          // in the chunks content table; join them.
+          const rows = await c.env.DB.prepare(
+            "SELECT c.id FROM chunks_fts f JOIN chunks c ON c.rowid = f.rowid WHERE chunks_fts MATCH ?1 AND c.doc_number = ?2 AND (c.clause_title LIKE '%ibliograph%' OR c.clause_title LIKE '%ormative reference%') LIMIT 8",
+          )
+            .bind("bibliography OR references", docNum)
+            .all() as { results?: Array<{ id: string }> };
+          const ids = (rows.results ?? []).map((r: { id: string }) => r.id).slice(0, 8);
+          if (!ids.length) return [] as Hit[];
+          const got = await c.env.VECTORIZE.getByIds(ids);
+          if (!got?.length) return [] as Hit[];
+          // getByIds returns metadata WITHOUT the chunk text — the
+          // usedHits builder calls h.text.clipToTokens and crashes on
+          // undefined (the live 503). Fetch the text from D1 and merge.
+          const ph = ids.map((_: string, i: number) => `?${i + 1}`).join(",");
+          const texts = await c.env.DB.prepare(`SELECT id, text FROM chunks WHERE id IN (${ph})`)
+            .bind(...ids)
+            .all() as { results?: Array<{ id: string; text: string }> };
+          const textById = new Map((texts.results ?? []).map((r) => [r.id, r.text]));
+          return got
+            .filter((h: any) => textById.has(h.id))
+            .map((h: any) => ({ ...h, score: 10, text: textById.get(h.id)! }));
+        } catch {
+          return [] as Hit[];
+        }
+      })(),
+      // the graph's cites edges for the family — structured, edition-keyed
+      (async () => {
+        if (!family) return [] as CiteRow[];
+        try {
+          const rows = await c.env.DB.prepare(
+            "SELECT d.docidentifier, d.edition, d.active, n.label FROM graph_edges e JOIN documents d ON e.src = d.canonical_id JOIN graph_nodes n ON e.dst = n.id WHERE e.kind = 'cites' AND d.family = ?1 ORDER BY d.active DESC, d.edition DESC LIMIT 120",
+          )
+            .bind(family)
+            .all() as { results?: CiteRow[] };
+          return rows.results ?? [];
+        } catch {
+          return [] as CiteRow[];
+        }
+      })(),
+    ]);
+  },
+  run: async (c) => {
+    const [probes, citeRows] = (await c.lane["citation-probe"]) as [Hit[], CiteRow[]];
+    const seen = new Set(c.hits.map((m: any) => m.id));
+    let added = 0;
+    // only bibliography-shaped chunks (clause title or text mentions it)
+    for (const h of probes) {
+      if (seen.has(h.id as any)) continue;
+      const title = String((h.metadata as any)?.clause_title ?? "");
+      const text = String(h.text ?? "");
+      if (/bibliograph|normative reference/i.test(title + " " + text.slice(0, 300))) {
+        c.hits.push(h);
+        seen.add(h.id as any);
+        added++;
+      }
+    }
+    // the structured note: an edition-scoped question sees only that
+    // edition's citations; the family question sees all editions
+    const edition = (c as any).__citeEdition as string | null;
+    const scoped = edition ? citeRows.filter((r) => r.edition === edition) : citeRows;
+    const note = citationGraphNote((c as any).__citeLabel as string, scoped);
+    if (note) c.notes.push(note);
+    console.log("citation-probe:", added, "passages,", note ? "graph note on" : "graph note off", `(${citeRows.length} cite rows)`);
+  },
+};

package/workers/worker_public/src/stages/index.ts

@@ -12,6 +12,7 @@ import type { Stage } from "./types.ts";
 export { runStages } from "./types.ts";
 export type { PipelineContext, RetrieveOptions, GlossaryEntry, Stage } from "./types.ts";
 import { dense } from "./dense.ts";
+import { citationProbe } from "./citationProbe.ts";
 import { hyde } from "./hyde.ts";
 import { glossary } from "./glossary.ts";
 import { conceptGraph } from "./conceptGraph.ts";
@@ -54,6 +55,7 @@ export const STAGES: Stage[] = [
   familyBoost,
   rerankStage,
   lexicalRrf,
+  citationProbe,
   corpusScope,
   editionCover,
   stdRefNudge,

package/workers/worker_public/src/stages/types.ts

@@ -64,6 +64,10 @@ export interface PipelineContext {
   hits: Hit[]; // the ranked pool from poolOpen onward
   finalHits: Hit[]; // the answer window
   glossary: GlossaryEntry[]; // the vocabulary link (glossary stage owns)
+  /** structured facts stages contribute to the answer prompt (the
+   *  GraphRAG seam: graph-derived notes ride the same channel the
+   *  vocabulary link does — ask.ts merges them into the retrieval note) */
+  notes: string[];
   opts: RetrieveOptions;
   /** prefetch bag: stage-name → that stage's in-flight I/O promise (the
    *  stage owns its key; see Stage.prefetch) */

package/workers/worker_public/src/tablecontext.ts

@@ -8,7 +8,14 @@
  *  selected column labels. Full table stays available for rendering;
  *  this only shapes the prompt context, and falls back to the stored
  *  text when pruning matches nothing (never worse than baseline). */
-export function tableContext(meta: any, query: string): string | null {
+export interface TableSelection {
+  text: string;
+  cols: string[];
+  rowsShown: number;
+  rowsTotal: number;
+}
+
+export function tableSelection(meta: any, query: string): TableSelection | null {
   const t: any = meta?.table;
   if (!t || !Array.isArray(t.columns) || !Array.isArray(t.rows) || !t.rows.length) return null;
   const terms = new Set(
@@ -37,5 +44,9 @@ export function tableContext(meta: any, query: string): string | null {
     rowHits.length > CAP || rowHits.length < t.rows.length
       ? `\n(${shown.length} of ${t.rows.length} rows shown; ${t.rows.length - rowHits.length} rows did not match the question terms)`
       : "";
-  return `${header}\n${lines.join("\n")}${elided}`;
+  return { text: `${header}\n${lines.join("\n")}${elided}`, cols: colKeep.map((i: number) => `${t.columns[i]?.label ?? ""}${t.columns[i]?.unit ? ` [${t.columns[i].unit}]` : ""}`), rowsShown: shown.length, rowsTotal: t.rows.length };
+}
+
+export function tableContext(meta: any, query: string): string | null {
+  return tableSelection(meta, query)?.text ?? null;
 }

package/workers/worker_public/src/verdict-parse.ts

@@ -0,0 +1,60 @@
+// Judge-verdict extraction, kept free of the prompt import so plain
+// node can load it for unit tests.
+export interface Verdict {
+  score: number; // 0-1, clamped
+  ungrounded_claims: string[];
+}
+
+function coerceVerdict(obj: unknown): Verdict | null {
+  if (obj === null || typeof obj !== "object" || Array.isArray(obj)) return null;
+  const raw = (obj as { score?: unknown }).score;
+  const score = typeof raw === "number" ? raw : typeof raw === "string" && raw.trim() !== "" ? Number(raw) : NaN;
+  if (!Number.isFinite(score)) return null;
+  const claims = (obj as { ungrounded_claims?: unknown }).ungrounded_claims;
+  return {
+    score: Math.max(0, Math.min(1, score)),
+    ungrounded_claims: Array.isArray(claims) ? claims.map(String).slice(0, 5) : [],
+  };
+}
+
+// Reasoning models emit drafts, prose, fenced blocks, and several {...}
+// fragments before the verdict; the verdict is the LAST well-formed
+// object carrying a numeric score. Braces can also appear INSIDE claim
+// strings, so the scan tracks string state instead of regexing flat
+// objects (the flat scan silently missed those shapes live).
+export function parseVerdict(text: string): Verdict | null {
+  const stripped = text.replace(/```[a-zA-Z]*\n?/g, "").replace(/```/g, "").trim();
+  try {
+    const whole = coerceVerdict(JSON.parse(stripped));
+    if (whole) return whole;
+  } catch {
+    // not one JSON document — scan candidates
+  }
+  let verdict: Verdict | null = null;
+  let depth = 0;
+  let start = -1;
+  let inString = false;
+  let escaped = false;
+  for (let i = 0; i < stripped.length; i++) {
+    const ch = stripped[i];
+    if (escaped) { escaped = false; continue; }
+    if (inString && ch === "\\") { escaped = true; continue; }
+    if (ch === '"') { inString = !inString; continue; }
+    if (inString) continue;
+    if (ch === "{") {
+      if (depth === 0) start = i;
+      depth++;
+    } else if (ch === "}" && depth > 0) {
+      depth--;
+      if (depth === 0 && start >= 0) {
+        try {
+          const v = coerceVerdict(JSON.parse(stripped.slice(start, i + 1)));
+          if (v) verdict = v;
+        } catch {
+          // malformed candidate — keep scanning
+        }
+      }
+    }
+  }
+  return verdict;
+}