@konneal/engine 0.1.4 → 0.2.1

package/workers/worker_public/src/pipeline.ts

@@ -4,7 +4,7 @@ import { LIMITS, MODELS, DATASETS, THRESHOLDS, processExpansion } from "./config
 import systemPromptText from "../prompts/system.md";
 import conversationalPromptText from "../prompts/conversational.md";
 import listwisePromptText from "../prompts/listwise.md";
-import { tableContext } from "./tablecontext";
+import { tableSelection } from "./tablecontext";
 
 // the pinned refusal sentence lives with the canonicalizer in ./refusal
 // (refusals are never cached: a refusal says "retrieval found nothing",
@@ -337,7 +337,9 @@ export function buildMessages(
     const head = `[${usedHits.length + 1}] ${label}${unitTag} ${h.metadata.clause_title ? "— " + h.metadata.clause_title : ""}\n`;
     // tables: schema-aware pruning from the producer payload; the
     // stored text is the fallback (pruning never goes below baseline)
-    const pruned = (h.metadata as any).block === "table" ? tableContext(h.metadata, query) : null;
+    const tableSel = (h.metadata as any).block === "table" ? tableSelection(h.metadata, query) : null;
+    const pruned = tableSel?.text ?? null;
+    if (tableSel) (h.metadata as any).table_selection = { cols: tableSel.cols, rowsShown: tableSel.rowsShown, rowsTotal: tableSel.rowsTotal };
     const body = clipToTokens(pruned ?? h.text, LIMITS.maxPassageTokens);
     const t = estTokens(head) + estTokens(body);
     if (t <= remain) {

package/workers/worker_public/src/profile.gen.ts

@@ -104,6 +104,7 @@ export const PROFILE = {
   },
   "prompts": {
     "vars": {
+      "publisher_identity": "the Fixture Organization — a worldwide organization that publishes the fixture corpus",
       "assistant_identity": "the fixture assistant — a public service answering questions about the fixture publisher's documents",
       "refusal_sentence": "I don't have information on this in the indexed fixture documents.",
       "account_note_source": "the user's own fixture account",

package/workers/worker_public/src/projects.ts

@@ -30,8 +30,10 @@ export async function handleProjects(env: any, sub: string, req: Request, route:
   }
   if (method === "POST") {
     const body: any = await req.json().catch(() => null);
-    if (typeof body?.project_id === "string") {
-      // move a conversation in/out of a project (membership-as-move)
+    if (body && (typeof body.project_id === "string" || body.project_id === null)) {
+      // move a conversation in/out of a project (membership-as-move) —
+      // null UNFILES (the 2026-09-16 bug: the string-only guard let the
+      // null case fall through to create-project and 400)
       const convId = String(body.conversation_id ?? "");
       if (!/^[a-zA-Z0-9_-]{8,64}$/.test(convId)) return err(400, "invalid_input", "bad conversation id");
       const target = body.project_id === null ? null : String(body.project_id);

package/workers/worker_public/src/quota.ts

@@ -39,13 +39,15 @@ export function telemetry(
   answerChars: number,
   queryHash: string,
   lang?: string,
+  cache?: "exact" | "semantic",
+  meta?: { durationMs?: number; keyId?: string | null },
 ) {
   const day = today();
   ctx.waitUntil(
     env.DB.batch([
       env.DB.prepare(
-        "INSERT INTO queries (ts, day, tier, route, model, ok, answer_chars, query_hash, lang) VALUES (?1,?2,?3,?4,?5,?6,?7,?8,?9)",
-      ).bind(new Date().toISOString(), day, tier, route, model, ok ? 1 : 0, answerChars, queryHash, lang ?? null),
+        "INSERT INTO queries (ts, day, tier, route, model, ok, answer_chars, query_hash, lang, cache, duration_ms, key_id) VALUES (?1,?2,?3,?4,?5,?6,?7,?8,?9,?10,?11,?12)",
+      ).bind(new Date().toISOString(), day, tier, route, model, ok ? 1 : 0, answerChars, queryHash, lang ?? null, cache ?? null, meta?.durationMs ?? null, meta?.keyId ?? null),
       env.DB.prepare(
         "INSERT INTO spend (day, tier, model, requests) VALUES (?1,?2,?3,1) ON CONFLICT(day, tier, model) DO UPDATE SET requests = requests + 1",
       ).bind(day, tier, model ?? "none"),

package/workers/worker_public/src/research.ts

@@ -31,6 +31,47 @@ export async function handleResearch(env: Env, ctx: ExecutionContext, req: Reque
   if (!q) return err(400, "invalid_input", `query is required (1-${LIMITS.maxInputChars} chars)`);
   const maxIters = Math.min(Math.max(Number(body?.max_iterations) || 3, 1), 3);
 
+  // the passes, streamed when the caller asks: each retrieval and each
+  // sufficiency judgement is an event, so the reader watches the loop
+  // work instead of watching a spinner — the final event carries the
+  // same result object the JSON path returns
+  if (body?.stream === true) {
+    const enc = new TextEncoder();
+    const started = Date.now();
+    const stream = new ReadableStream({
+      async start(ctrl) {
+        const send = (e: unknown) => ctrl.enqueue(enc.encode(`data: ${JSON.stringify(e)}\n\n`));
+        try {
+          const out = await run(env, ctx, q, maxIters, send);
+          send({ type: "done", ...out });
+        } catch (e: any) {
+          send({ type: "error", message: String(e?.message ?? e).slice(0, 200) });
+        } finally {
+          ctrl.close();
+        }
+      },
+    });
+    return new Response(stream, {
+      headers: {
+        "content-type": "text/event-stream; charset=utf-8",
+        "cache-control": "no-cache",
+        "x-accel-buffering": "no",
+        ...corsHeaders(req),
+      },
+    });
+    void started;
+  }
+  const out = await run(env, ctx, q, maxIters);
+  return json({ ...out, ...corsHeaders(req) });
+}
+
+async function run(
+  env: Env,
+  ctx: ExecutionContext,
+  q: { query: string; lang?: string },
+  maxIters: number,
+  emit?: (e: unknown) => void,
+): Promise<{ answer: string; citations: unknown[]; model: string; query_hash: string; research: { iterations: number; passages: number; elapsed_ms: number; sufficient: boolean | null } }> {
   const started = Date.now();
   const queryHash = await sha256Hex(q.query);
   const understanding = await understandQuery(portModelRunner(env), MODELS.understand, q.query, [], []);
@@ -44,6 +85,7 @@ export async function handleResearch(env: Env, ctx: ExecutionContext, req: Reque
 
   for (let i = 0; i < maxIters; i++) {
     iterations = i + 1;
+    emit?.({ type: "pass", n: iterations, of: maxIters, phase: "retrieving" });
     let retrieved: { hits: Hit[] };
     try {
       retrieved = await retrieve(env, q.query, {
@@ -95,6 +137,15 @@ export async function handleResearch(env: Env, ctx: ExecutionContext, req: Reque
       }
     })();
     console.log("research iter", iterations, "passages", passages.length, "sufficient:", judge?.sufficient);
+    emit?.({
+      type: "pass",
+      n: iterations,
+      of: maxIters,
+      phase: "judged",
+      passages: passages.length,
+      sufficient: judge?.sufficient ?? null,
+      ...(judge && !judge.sufficient && judge.missing ? { missing: judge.missing.slice(0, 300) } : {}),
+    });
     if (!judge || judge.sufficient || !judge.missing) break;
     // fold, don't accumulate: appending every round's `missing` compounds
     // stale wants; the next retrieval focuses on the ORIGINAL question plus
@@ -104,14 +155,14 @@ export async function handleResearch(env: Env, ctx: ExecutionContext, req: Reque
 
   const used = [...accumulated.values()];
   if (!used.length) {
-    return err(503, "retrieval_unavailable", "Search is briefly busy — please retry in a moment.");
+    throw new Error("Search is briefly busy — please retry in a moment.");
   }
   const { messages, usedHits } = buildMessages(q.query, used, q.lang, [], eNote || undefined, undefined, LIMITS.inputTokenBudget);
   let answer = await generateOnce(env, MODELS.research, messages);
   if (answer === null) answer = await generateOnce(env, MODELS.fallback, messages);
   if (answer === null) {
     telemetry(env, ctx, "member", "research", MODELS.research, false, 0, queryHash, q.lang);
-    return err(502, "generation_failed", "The generation model is unavailable; please retry.");
+    throw new Error("The generation model is unavailable; please retry.");
   }
   answer = canonicalRefusal(answer);
   const anchors = checkQuoteAnchors(answer, used.map((h: Hit) => h.text));
@@ -124,5 +175,6 @@ export async function handleResearch(env: Env, ctx: ExecutionContext, req: Reque
     research: { iterations, passages: used.length, elapsed_ms: Date.now() - started, sufficient: judge?.sufficient ?? null },
   };
   telemetry(env, ctx, "member", "research", MODELS.research, true, answer.length, queryHash, q.lang);
-  return json({ ...out, ...corsHeaders(req) });
+  emit?.({ type: "pass", n: iterations, of: maxIters, phase: "writing", passages: used.length });
+  return out;
 }

package/workers/worker_public/src/tablecontext.ts

@@ -8,7 +8,14 @@
  *  selected column labels. Full table stays available for rendering;
  *  this only shapes the prompt context, and falls back to the stored
  *  text when pruning matches nothing (never worse than baseline). */
-export function tableContext(meta: any, query: string): string | null {
+export interface TableSelection {
+  text: string;
+  cols: string[];
+  rowsShown: number;
+  rowsTotal: number;
+}
+
+export function tableSelection(meta: any, query: string): TableSelection | null {
   const t: any = meta?.table;
   if (!t || !Array.isArray(t.columns) || !Array.isArray(t.rows) || !t.rows.length) return null;
   const terms = new Set(
@@ -37,5 +44,9 @@ export function tableContext(meta: any, query: string): string | null {
     rowHits.length > CAP || rowHits.length < t.rows.length
       ? `\n(${shown.length} of ${t.rows.length} rows shown; ${t.rows.length - rowHits.length} rows did not match the question terms)`
       : "";
-  return `${header}\n${lines.join("\n")}${elided}`;
+  return { text: `${header}\n${lines.join("\n")}${elided}`, cols: colKeep.map((i: number) => `${t.columns[i]?.label ?? ""}${t.columns[i]?.unit ? ` [${t.columns[i].unit}]` : ""}`), rowsShown: shown.length, rowsTotal: t.rows.length };
+}
+
+export function tableContext(meta: any, query: string): string | null {
+  return tableSelection(meta, query)?.text ?? null;
 }

package/workers/worker_public/src/verdict-parse.ts

@@ -0,0 +1,60 @@
+// Judge-verdict extraction, kept free of the prompt import so plain
+// node can load it for unit tests.
+export interface Verdict {
+  score: number; // 0-1, clamped
+  ungrounded_claims: string[];
+}
+
+function coerceVerdict(obj: unknown): Verdict | null {
+  if (obj === null || typeof obj !== "object" || Array.isArray(obj)) return null;
+  const raw = (obj as { score?: unknown }).score;
+  const score = typeof raw === "number" ? raw : typeof raw === "string" && raw.trim() !== "" ? Number(raw) : NaN;
+  if (!Number.isFinite(score)) return null;
+  const claims = (obj as { ungrounded_claims?: unknown }).ungrounded_claims;
+  return {
+    score: Math.max(0, Math.min(1, score)),
+    ungrounded_claims: Array.isArray(claims) ? claims.map(String).slice(0, 5) : [],
+  };
+}
+
+// Reasoning models emit drafts, prose, fenced blocks, and several {...}
+// fragments before the verdict; the verdict is the LAST well-formed
+// object carrying a numeric score. Braces can also appear INSIDE claim
+// strings, so the scan tracks string state instead of regexing flat
+// objects (the flat scan silently missed those shapes live).
+export function parseVerdict(text: string): Verdict | null {
+  const stripped = text.replace(/```[a-zA-Z]*\n?/g, "").replace(/```/g, "").trim();
+  try {
+    const whole = coerceVerdict(JSON.parse(stripped));
+    if (whole) return whole;
+  } catch {
+    // not one JSON document — scan candidates
+  }
+  let verdict: Verdict | null = null;
+  let depth = 0;
+  let start = -1;
+  let inString = false;
+  let escaped = false;
+  for (let i = 0; i < stripped.length; i++) {
+    const ch = stripped[i];
+    if (escaped) { escaped = false; continue; }
+    if (inString && ch === "\\") { escaped = true; continue; }
+    if (ch === '"') { inString = !inString; continue; }
+    if (inString) continue;
+    if (ch === "{") {
+      if (depth === 0) start = i;
+      depth++;
+    } else if (ch === "}" && depth > 0) {
+      depth--;
+      if (depth === 0 && start >= 0) {
+        try {
+          const v = coerceVerdict(JSON.parse(stripped.slice(start, i + 1)));
+          if (v) verdict = v;
+        } catch {
+          // malformed candidate — keep scanning
+        }
+      }
+    }
+  }
+  return verdict;
+}