@konneal/engine 0.2.99 → 0.2.100

package/dist/{ask-W6NUC3MG.js → ask-KP42T6DH.js}

@@ -1,8 +1,8 @@
 import {
   handleAsk
-} from "./chunk-2SWMTCVX.js";
+} from "./chunk-GYCUMFOA.js";
 import "./chunk-QNLFNEI4.js";
-import "./chunk-TXPWLKCI.js";
+import "./chunk-BBKAETKM.js";
 import "./chunk-KNCRYAX3.js";
 import "./chunk-DTW4UW2E.js";
 import "./chunk-A3QHHUN5.js";

package/dist/{chunk-TXPWLKCI.js → chunk-BBKAETKM.js}

@@ -114,6 +114,69 @@ var STOP = /* @__PURE__ */ new Set([
   "their",
   "there"
 ]);
+function rowMeta(r) {
+  return {
+    doc_id: String(r.doc_id ?? ""),
+    docidentifier: String(r.docidentifier ?? ""),
+    doctype: String(r.doctype ?? ""),
+    doc_number: String(r.doc_number ?? ""),
+    edition: String(r.edition ?? ""),
+    language: String(r.language ?? "en"),
+    clause_anchor: String(r.clause_anchor ?? ""),
+    clause_title: String(r.clause_title ?? ""),
+    tier: String(r.tier ?? ""),
+    corpus: String(r.corpus ?? ""),
+    text_ref: "",
+    status: String(r.status ?? "unknown"),
+    superseded_by: String(r.superseded_by ?? ""),
+    // contract v2 over the lexical lane: typed chunks arriving via BM25
+    // keep their unit identity ([[u:…]] refs, typed pin, retyping check)
+    unit_id: String(r.unit_id ?? "") || void 0,
+    block: String(r.block ?? "") || void 0
+  };
+}
+function rowsToHits(rows) {
+  return rows.map((r) => {
+    const bm25 = typeof r.rank === "number" ? r.rank : 0;
+    return {
+      id: String(r.id),
+      score: 1 / (1 + Math.max(0, bm25)),
+      metadata: rowMeta(r),
+      text: String(r.text ?? "")
+    };
+  });
+}
+function rowsToMatches(rows) {
+  return rows.map((r) => {
+    const bm25 = typeof r.rank === "number" ? Math.max(0, r.rank) : 0;
+    return {
+      id: String(r.id),
+      score: 1 / (1 + bm25),
+      metadata: { ...rowMeta(r), chunk_text: String(r.text ?? "") }
+    };
+  });
+}
+async function lexicalWithin(env, query, docNumbers, k = 12) {
+  const match2 = ftsMatchQuery(query);
+  if (!match2 || !docNumbers.length) return [];
+  try {
+    const placeholders = docNumbers.map((_, i) => `?${i + 2}`).join(",");
+    const res = await env.DB.prepare(
+      `SELECT c.id, c.doc_id, c.docidentifier, c.doctype, c.doc_number, c.edition,
+              c.language, c.clause_anchor, c.clause_title, c.status, c.superseded_by,
+              c.corpus, c.tier, c.text, c.unit_id, c.block, bm25(chunks_fts) AS rank
+         FROM chunks_fts
+         JOIN chunks c ON c.rowid = chunks_fts.rowid
+        WHERE chunks_fts MATCH ?1 AND c.doc_number IN (${placeholders})
+        ORDER BY rank
+        LIMIT ?${docNumbers.length + 2}`
+    ).bind(match2, ...docNumbers, k).all();
+    return rowsToMatches(res.results ?? []);
+  } catch (e) {
+    console.log("lexical-within failed:", String(e).slice(0, 200));
+    return [];
+  }
+}
 async function lexicalPrefilter(env, query, k = LEXICAL_K) {
   const match2 = ftsMatchQuery(query);
   if (!match2) return [];
@@ -129,34 +192,7 @@ async function lexicalPrefilter(env, query, k = LEXICAL_K) {
         LIMIT ?2`
     ).bind(match2, k).all();
     const rows = res.results ?? [];
-    return rows.map((r, i) => {
-      const meta = {
-        doc_id: String(r.doc_id ?? ""),
-        docidentifier: String(r.docidentifier ?? ""),
-        doctype: String(r.doctype ?? ""),
-        doc_number: String(r.doc_number ?? ""),
-        edition: String(r.edition ?? ""),
-        language: String(r.language ?? "en"),
-        clause_anchor: String(r.clause_anchor ?? ""),
-        clause_title: String(r.clause_title ?? ""),
-        tier: String(r.tier ?? ""),
-        corpus: String(r.corpus ?? ""),
-        text_ref: "",
-        status: String(r.status ?? "unknown"),
-        superseded_by: String(r.superseded_by ?? ""),
-        // contract v2 over the lexical lane: typed chunks arriving via BM25
-        // keep their unit identity ([[u:…]] refs, typed pin, retyping check)
-        unit_id: String(r.unit_id ?? "") || void 0,
-        block: String(r.block ?? "") || void 0
-      };
-      const bm25 = typeof r.rank === "number" ? r.rank : i;
-      return {
-        id: String(r.id),
-        score: 1 / (1 + Math.max(0, bm25)),
-        metadata: meta,
-        text: String(r.text ?? "")
-      };
-    });
+    return rowsToHits(rows);
   } catch (e) {
     console.log("lexical prefilter failed:", String(e).slice(0, 200));
     return [];
@@ -21935,7 +21971,14 @@ var oimlPubid = {
 var plainSlug = {
   parse: () => null,
   scanQuestion: () => null,
-  graphDocNumber: () => null,
+  // the graph node id scheme (doc:OIML-<TYPE>-<NUM>-<EDITION>) is the
+  // GRAPH's own identity, estate-wide — not publisher-flavored — so the
+  // plain codec parses it too (a null here silenced the concept-graph
+  // lane's resolution for its whole life)
+  graphDocNumber: (nodeId) => {
+    const m = nodeId.match(/^doc:OIML-[A-Z]-(\d+)-/);
+    return m ? m[1] : null;
+  },
   familyOf: () => null
 };
 var REGISTRY = {
@@ -22477,14 +22520,10 @@ var conceptGraph = {
       }
     }
     if (numbers.size) {
-      const gc = await c.env.VECTORIZE.query(c.vector, {
-        topK: 12,
-        returnMetadata: "all",
-        filter: { doc_number: { $in: [...numbers] } }
-      });
+      const gc = await lexicalWithin(c.env, c.rq || c.query, [...numbers], 12);
       const seenIds0 = new Set(c.matches.map((m) => m.id));
       let merged0 = 0;
-      for (const m of (gc.matches ?? []).slice(0, 6)) {
+      for (const m of gc.slice(0, 6)) {
         if (!seenIds0.has(m.id)) {
           c.matches.push({ id: m.id, score: m.score * THRESHOLDS.conceptGraphDiscount, metadata: m.metadata });
           seenIds0.add(m.id);
@@ -22502,24 +22541,20 @@ var graphLane = {
   failure: "additive",
   when: (c) => !!c.opts.graphDocNumbers?.length && c.vector.length > 0,
   prefetch: (c) => {
-    c.lane["graph-lane"] = c.env.VECTORIZE.query(c.vector, {
-      topK: 15,
-      returnMetadata: "all",
-      filter: { doc_number: { $in: c.opts.graphDocNumbers } }
-    });
+    c.lane["graph-lane"] = lexicalWithin(c.env, c.rq || c.query, c.opts.graphDocNumbers, 15);
   },
   run: async (c) => {
     const g = await c.lane["graph-lane"];
     const seenIds = new Set(c.matches.map((m) => m.id));
     let merged = 0;
-    for (const m of (g.matches ?? []).slice(0, 10)) {
+    for (const m of (g ?? []).slice(0, 10)) {
       if (!seenIds.has(m.id)) {
         c.matches.push({ id: m.id, score: m.score * THRESHOLDS.graphLaneDiscount, metadata: m.metadata });
         seenIds.add(m.id);
         merged++;
       }
     }
-    console.log("graph lane:", g.matches?.length ?? 0, "hits,", merged, "merged");
+    console.log("graph lane:", g?.length ?? 0, "hits,", merged, "merged");
   }
 };
 

package/dist/{chunk-2SWMTCVX.js → chunk-GYCUMFOA.js}

@@ -56,7 +56,7 @@ import {
   tokenBudget,
   understandQuery,
   usageTotal
-} from "./chunk-TXPWLKCI.js";
+} from "./chunk-BBKAETKM.js";
 import {
   embed,
   generateOnce,

package/dist/{chunk-GY33RTEO.js → chunk-OREMF5LK.js}

@@ -6,7 +6,7 @@ import {
   sessionFrom,
   telemetry,
   understandQuery
-} from "./chunk-TXPWLKCI.js";
+} from "./chunk-BBKAETKM.js";
 import {
   portModelRunner
 } from "./chunk-KNCRYAX3.js";

package/dist/lexical.d.ts

@@ -2,6 +2,18 @@ import type { Hit } from "./pipeline";
 /** Build a safe FTS5 MATCH query from user text: alphanumeric tokens,
  *  joined with OR so jargon hits don't require full-phrase match. */
 export declare function ftsMatchQuery(query: string): string | null;
+/** Lexical ranking WITHIN a document set (the graph lanes' identity
+ *  resolution): BM25 over the corpus's own FTS, restricted to the
+ *  documents' doc_number values. This is the working substitute for
+ *  Vectorize metadata filtering, which is dead on this index (measured
+ *  2026-10-05: filters return empty for values proven present). */
+export declare function lexicalWithin(env: {
+    DB: D1Database;
+}, query: string, docNumbers: string[], k?: number): Promise<{
+    id: string;
+    score: number;
+    metadata: Record<string, unknown>;
+}[]>;
 export declare function lexicalPrefilter(env: {
     DB: D1Database;
 }, query: string, k?: number): Promise<Hit[]>;

package/dist/{search-EQAOPFCC.js → search-4CBKKNBJ.js}

@@ -1,7 +1,7 @@
 import {
   handleSearch
-} from "./chunk-GY33RTEO.js";
-import "./chunk-TXPWLKCI.js";
+} from "./chunk-OREMF5LK.js";
+import "./chunk-BBKAETKM.js";
 import "./chunk-KNCRYAX3.js";
 import "./chunk-DTW4UW2E.js";
 import "./chunk-A3QHHUN5.js";

package/dist/worker_public/src/index.js

@@ -4,7 +4,7 @@ import {
   handleMemories,
   scoreFaithfulness,
   scoreJudge
-} from "../../chunk-2SWMTCVX.js";
+} from "../../chunk-GYCUMFOA.js";
 import {
   TOOLS_REGISTRY,
   bindModelNode,
@@ -12,7 +12,7 @@ import {
 } from "../../chunk-QNLFNEI4.js";
 import {
   handleSearch
-} from "../../chunk-GY33RTEO.js";
+} from "../../chunk-OREMF5LK.js";
 import {
   buildMessages,
   citations,
@@ -34,7 +34,7 @@ import {
   sessionFrom,
   telemetry,
   understandQuery
-} from "../../chunk-TXPWLKCI.js";
+} from "../../chunk-BBKAETKM.js";
 import {
   cfBlobs,
   embed,
@@ -996,7 +996,7 @@ async function callToolOnce(env, ctx, _req, tier, key, name, args, t0) {
       const r = await runTool({ ...env, DB: env.DB }, { name, args }, "mcp");
       payload = r ? { tool: r.name, query: r.query, result: r.output } : { error: { message: `tool ${name} returned nothing for the given arguments` } };
     } else {
-      const res = name === "ask" ? await (await import("../../ask-W6NUC3MG.js")).handleAsk(env, ctx, inner, tier, key) : await (await import("../../search-EQAOPFCC.js")).handleSearch(env, ctx, inner, tier, key);
+      const res = name === "ask" ? await (await import("../../ask-KP42T6DH.js")).handleAsk(env, ctx, inner, tier, key) : await (await import("../../search-4CBKKNBJ.js")).handleSearch(env, ctx, inner, tier, key);
       payload = await res.json().catch(() => ({ error: { message: "tool transport failed", status: res.status } }));
     }
     telemetry(env, ctx, tier, `mcp:${name}`, null, !payload?.error, JSON.stringify(payload).length, await sha256Hex(`${name}:${JSON.stringify(args ?? {})}`), void 0, void 0, { durationMs: Date.now() - t0 });

package/package.json

@@ -27,7 +27,7 @@
     "playwright": "^1.62.1",
     "yaml": "^2.9.1"
   },
-  "version": "0.2.99",
+  "version": "0.2.100",
   "description": "The Konneal engine: the publisher-agnostic build pipeline and API plane for standards intelligence (retrieval, answer contract, verdicts, evaluation).",
   "license": "BSD-3-Clause",
   "type": "module",

package/workers/worker_public/src/codecs.ts

@@ -142,7 +142,14 @@ export const oimlPubid: RefCodec = {
 export const plainSlug: RefCodec = {
   parse: () => null,
   scanQuestion: () => null,
-  graphDocNumber: () => null,
+  // the graph node id scheme (doc:OIML-<TYPE>-<NUM>-<EDITION>) is the
+  // GRAPH's own identity, estate-wide — not publisher-flavored — so the
+  // plain codec parses it too (a null here silenced the concept-graph
+  // lane's resolution for its whole life)
+  graphDocNumber: (nodeId) => {
+    const m = nodeId.match(/^doc:OIML-[A-Z]-(\d+)-/);
+    return m ? m[1] : null;
+  },
   familyOf: () => null,
 };
 

package/workers/worker_public/src/lexical.ts

@@ -31,6 +31,89 @@ const STOP = new Set([
   "about", "into", "than", "then", "its", "it", "their", "there",
 ]);
 
+function rowMeta(r: any): ChunkMeta {
+  return {
+    doc_id: String(r.doc_id ?? ""),
+    docidentifier: String(r.docidentifier ?? ""),
+    doctype: String(r.doctype ?? ""),
+    doc_number: String(r.doc_number ?? ""),
+    edition: String(r.edition ?? ""),
+    language: String(r.language ?? "en"),
+    clause_anchor: String(r.clause_anchor ?? ""),
+    clause_title: String(r.clause_title ?? ""),
+    tier: String(r.tier ?? ""),
+    corpus: String(r.corpus ?? ""),
+    text_ref: "",
+    status: String(r.status ?? "unknown"),
+    superseded_by: String(r.superseded_by ?? ""),
+    // contract v2 over the lexical lane: typed chunks arriving via BM25
+    // keep their unit identity ([[u:…]] refs, typed pin, retyping check)
+    unit_id: String(r.unit_id ?? "") || undefined,
+    block: String(r.block ?? "") || undefined,
+  };
+}
+
+function rowsToHits(rows: any[]): Hit[] {
+  return rows.map((r: any) => {
+    // rank is bm25 (lower better) → convert to positive score that RRF can ignore
+    // (RRF uses rank position, not score). score kept for logging only.
+    const bm25 = typeof r.rank === "number" ? r.rank : 0;
+    return {
+      id: String(r.id),
+      score: 1 / (1 + Math.max(0, bm25)),
+      metadata: rowMeta(r),
+      text: String(r.text ?? ""),
+    } satisfies Hit;
+  });
+}
+
+/** Lexical matches for the lane merges: match-shaped, with the text
+ *  riding the metadata (toHits reads chunk_text out of it). */
+function rowsToMatches(rows: any[]): { id: string; score: number; metadata: Record<string, unknown> }[] {
+  return rows.map((r: any) => {
+    const bm25 = typeof r.rank === "number" ? Math.max(0, r.rank) : 0;
+    return {
+      id: String(r.id),
+      score: 1 / (1 + bm25),
+      metadata: { ...rowMeta(r), chunk_text: String(r.text ?? "") },
+    };
+  });
+}
+
+/** Lexical ranking WITHIN a document set (the graph lanes' identity
+ *  resolution): BM25 over the corpus's own FTS, restricted to the
+ *  documents' doc_number values. This is the working substitute for
+ *  Vectorize metadata filtering, which is dead on this index (measured
+ *  2026-10-05: filters return empty for values proven present). */
+export async function lexicalWithin(
+  env: { DB: D1Database },
+  query: string,
+  docNumbers: string[],
+  k = 12,
+): Promise<{ id: string; score: number; metadata: Record<string, unknown> }[]> {
+  const match = ftsMatchQuery(query);
+  if (!match || !docNumbers.length) return [];
+  try {
+    const placeholders = docNumbers.map((_, i) => `?${i + 2}`).join(",");
+    const res = await env.DB.prepare(
+      `SELECT c.id, c.doc_id, c.docidentifier, c.doctype, c.doc_number, c.edition,
+              c.language, c.clause_anchor, c.clause_title, c.status, c.superseded_by,
+              c.corpus, c.tier, c.text, c.unit_id, c.block, bm25(chunks_fts) AS rank
+         FROM chunks_fts
+         JOIN chunks c ON c.rowid = chunks_fts.rowid
+        WHERE chunks_fts MATCH ?1 AND c.doc_number IN (${placeholders})
+        ORDER BY rank
+        LIMIT ?${docNumbers.length + 2}`,
+    )
+      .bind(match, ...docNumbers, k)
+      .all();
+    return rowsToMatches(res.results ?? []);
+  } catch (e) {
+    console.log("lexical-within failed:", String(e).slice(0, 200));
+    return [];
+  }
+}
+
 export async function lexicalPrefilter(env: { DB: D1Database }, query: string, k = LEXICAL_K): Promise<Hit[]> {
   const match = ftsMatchQuery(query);
   if (!match) return [];
@@ -49,36 +132,7 @@ export async function lexicalPrefilter(env: { DB: D1Database }, query: string, k
       .bind(match, k)
       .all();
     const rows = res.results ?? [];
-    return rows.map((r: any, i: number) => {
-      const meta: ChunkMeta = {
-        doc_id: String(r.doc_id ?? ""),
-        docidentifier: String(r.docidentifier ?? ""),
-        doctype: String(r.doctype ?? ""),
-        doc_number: String(r.doc_number ?? ""),
-        edition: String(r.edition ?? ""),
-        language: String(r.language ?? "en"),
-        clause_anchor: String(r.clause_anchor ?? ""),
-        clause_title: String(r.clause_title ?? ""),
-        tier: String(r.tier ?? ""),
-        corpus: String(r.corpus ?? ""),
-        text_ref: "",
-        status: String(r.status ?? "unknown"),
-        superseded_by: String(r.superseded_by ?? ""),
-        // contract v2 over the lexical lane: typed chunks arriving via BM25
-        // keep their unit identity ([[u:…]] refs, typed pin, retyping check)
-        unit_id: String(r.unit_id ?? "") || undefined,
-        block: String(r.block ?? "") || undefined,
-      };
-      // rank is bm25 (lower better) → convert to positive score that RRF can ignore
-      // (RRF uses rank position, not score). score kept for logging only.
-      const bm25 = typeof r.rank === "number" ? r.rank : i;
-      return {
-        id: String(r.id),
-        score: 1 / (1 + Math.max(0, bm25)),
-        metadata: meta,
-        text: String(r.text ?? ""),
-      } satisfies Hit;
-    });
+    return rowsToHits(rows);
   } catch (e) {
     console.log("lexical prefilter failed:", String(e).slice(0, 200));
     return [];

package/workers/worker_public/src/stages/conceptGraph.ts

@@ -20,6 +20,7 @@
 import { THRESHOLDS } from "../config.ts";
 import type { Stage } from "./types.ts";
 import { refCodec } from "../codecs.ts";
+import { lexicalWithin } from "../lexical.ts";
 
 export const conceptGraph: Stage = {
   name: "concept-graph",
@@ -49,14 +50,14 @@ export const conceptGraph: Stage = {
       }
     }
     if (numbers.size) {
-      const gc = await c.env.VECTORIZE.query(c.vector, {
-        topK: 12,
-        returnMetadata: "all",
-        filter: { doc_number: { $in: [...numbers] } },
-      });
+      // identity resolution: the linked documents' query-relevant
+      // candidates rank by the corpus's own BM25 within the set — the
+      // index filter this lane used returned empty for its whole life
+      // (Vectorize metadata filtering is dead on this index)
+      const gc = await lexicalWithin(c.env, c.rq || c.query, [...numbers], 12);
       const seenIds0 = new Set(c.matches.map((m: any) => m.id));
       let merged0 = 0;
-      for (const m of (gc.matches ?? []).slice(0, 6)) {
+      for (const m of gc.slice(0, 6)) {
         if (!seenIds0.has(m.id)) {
           c.matches.push({ id: m.id, score: m.score * THRESHOLDS.conceptGraphDiscount, metadata: m.metadata });
           seenIds0.add(m.id);

package/workers/worker_public/src/stages/graphLane.ts

@@ -1,7 +1,15 @@
 // Graph lane (additive): the caller-resolved document numbers (relaton
-// family/successor resolution done in index.ts) merge their top
-// candidates at a discount.
+// family/successor resolution done in index.ts) merge their
+// query-relevant candidates at a discount.
+//
+// Identity resolution, not index filtering: the candidates rank by the
+// corpus's own BM25 WITHIN the document set (lexicalWithin) — Vectorize
+// metadata filtering is dead on this index (measured 2026-10-05:
+// filters return empty for values proven present), so the filter this
+// lane used for its whole life merged nothing. The lexical rows carry
+// their own text and metadata; no index round trip is needed at all.
 import { THRESHOLDS } from "../config.ts";
+import { lexicalWithin } from "../lexical.ts";
 import type { Stage } from "./types.ts";
 
 export const graphLane: Stage = {
@@ -9,23 +17,19 @@ export const graphLane: Stage = {
   failure: "additive",
   when: (c) => !!c.opts.graphDocNumbers?.length && c.vector.length > 0,
   prefetch: (c) => {
-    c.lane["graph-lane"] = c.env.VECTORIZE.query(c.vector, {
-      topK: 15,
-      returnMetadata: "all",
-      filter: { doc_number: { $in: c.opts.graphDocNumbers } },
-    });
+    c.lane["graph-lane"] = lexicalWithin(c.env, c.rq || c.query, c.opts.graphDocNumbers!, 15);
   },
   run: async (c) => {
     const g = (await c.lane["graph-lane"]!) as any;
     const seenIds = new Set(c.matches.map((m: any) => m.id));
     let merged = 0;
-    for (const m of (g.matches ?? []).slice(0, 10)) {
+    for (const m of (g ?? []).slice(0, 10)) {
       if (!seenIds.has(m.id)) {
         c.matches.push({ id: m.id, score: m.score * THRESHOLDS.graphLaneDiscount, metadata: m.metadata });
         seenIds.add(m.id);
         merged++;
       }
     }
-    console.log("graph lane:", g.matches?.length ?? 0, "hits,", merged, "merged");
+    console.log("graph lane:", g?.length ?? 0, "hits,", merged, "merged");
   },
 };