@konneal/engine 0.1.3 → 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/pipeline.d.ts +3 -0
- package/dist/prompts/understanding.md +1 -0
- package/dist/worker_public/src/index.js +229 -134
- package/docs/INGEST-ARCHITECTURE.md +1 -0
- package/docs/projects-design.md +7 -0
- package/docs/sota-mechanisms.md +15 -0
- package/docs/spec-pipeline.md +20 -13
- package/package.json +1 -1
- package/workers/worker_public/prompts/understanding.md +1 -0
- package/workers/worker_public/src/ask.ts +3 -2
- package/workers/worker_public/src/conversations.ts +1 -1
- package/workers/worker_public/src/pipeline.ts +5 -1
- package/workers/worker_public/src/stages/citationProbe.ts +145 -0
- package/workers/worker_public/src/stages/index.ts +2 -0
- package/workers/worker_public/src/stages/types.ts +4 -0
package/dist/pipeline.d.ts
CHANGED
|
@@ -18,6 +18,9 @@ export interface Retrieved {
|
|
|
18
18
|
* among them (dense retrieval alone binds everyday words to the wrong
|
|
19
19
|
* term: measured "keeps drifting" → creep 0.69 vs durability 0.54) */
|
|
20
20
|
glossary?: GlossaryEntry[];
|
|
21
|
+
/** structured facts stages extracted from the graph (GraphRAG) —
|
|
22
|
+
* merged into the answer prompt's retrieval note */
|
|
23
|
+
notes?: string[];
|
|
21
24
|
}
|
|
22
25
|
export declare function retrievalQuery(query: string, prev?: string): string;
|
|
23
26
|
export declare function retrieve(env: any, query: string, opts?: RetrieveOptions): Promise<Retrieved>;
|
|
@@ -6,6 +6,7 @@ Rules:
|
|
|
6
6
|
- docidentifier: the publication the user names, in any spelling ({{SPELLING_EXAMPLES}}, "the nonautomatic weighing instruments recommendation" → resolve to the {{PUBLISHER_NAME}} identifier you can infer; include the part ("-1", "-2") only when clearly meant). docnumber is the base number without part.
|
|
7
7
|
- edition: only when the user pins a year.
|
|
8
8
|
- language: only when the user asks for a specific answer language; otherwise null (the corpus is English; answering in the user's language is handled elsewhere).
|
|
9
|
+
- citation questions ("what does X cite/reference/list?", "which standards does X reference?"): ALWAYS include a query variant that names the document's bibliography or normative-references section explicitly, WITHOUT edition scoping (e.g. for "What ISO standards does R 60 cite?" generate BOTH "R 60 bibliography normative references" AND "R 60 2017 bibliography ISO IEC") — bibliographies embed differently than the query's phrasing, and prior editions may carry references the current edition dropped; set edition to null for these queries so retrieval covers the whole family.
|
|
9
10
|
- process_intent: true when the question is about the GOVERNING SYSTEM around publications rather than a publication's own technical content — HOW to get certified/apply/comply, OR which framework/vocabulary/{{PROCESS_VOCAB}}. Naming a Recommendation (e.g. "R 60") inside such a question does NOT make it a technical-content question: leave process_intent true and still emit docnumber when named, but the retrieval path must NOT seal to that document alone.
|
|
10
11
|
- term: the defined term when the question asks what something is ("what is an accuracy class" → "accuracy class"); otherwise null.
|
|
11
12
|
- defined_terms: the ESTABLISHED metrology / VIM terms this question is about, in the corpus's own terminology, EVEN WHEN the question uses everyday wording instead — match the TIME SCALE and sense carefully: "does the reading drift while a weight sits on it" (short-term, under load) → ["creep"]; "output keeps drifting over months of use" (long-term, in service) → ["span stability", "durability"]; "how many scale divisions is it allowed" → ["number of verification intervals"]. This is a terminology mapping, not a copy of the question's words. Empty when nothing maps.
|
|
@@ -432,6 +432,226 @@ var dense = {
|
|
|
432
432
|
}
|
|
433
433
|
};
|
|
434
434
|
|
|
435
|
+
// workers/worker_public/src/codecs.ts
|
|
436
|
+
var oimlPubid = {
|
|
437
|
+
parse(doc, edition) {
|
|
438
|
+
const m = doc.match(/^urn:oiml:pub:([rdbge]):(\d{1,3})(?:-[0-9A-Za-z]+)?(?::(\d{4}))?$/i) ?? doc.match(/^(?:OIML\s+)?([RDBGE])\s*(\d{1,3})(?:-[0-9A-Za-z]+)?(?::(\d{4}))?$/i);
|
|
439
|
+
if (!m) return null;
|
|
440
|
+
const type = m[1].toUpperCase();
|
|
441
|
+
const ed = edition ?? m[3] ?? void 0;
|
|
442
|
+
return { doc_number: m[2], ...ed ? { edition: ed } : {}, label: `OIML ${type} ${m[2]}${ed ? `:${ed}` : ""}` };
|
|
443
|
+
},
|
|
444
|
+
scanQuestion(query) {
|
|
445
|
+
const re = /\b(OIML\s+)?([RDBGE])(\s*)0*(\d{1,3})(?:\s*[-–]\s*\d+)?(?:\s*:\s*(\d{4}))?/gi;
|
|
446
|
+
for (const m of query.matchAll(re)) {
|
|
447
|
+
const [, oimlPrefix, letter, gap, digits, edition] = m;
|
|
448
|
+
if (digits.length === 1 && !oimlPrefix && !gap) continue;
|
|
449
|
+
const num2 = String(Number(digits));
|
|
450
|
+
const type = letter.toUpperCase();
|
|
451
|
+
return { doc_number: num2, ...edition ? { edition } : {}, label: `OIML ${type} ${num2}${edition ? `:${edition}` : ""}` };
|
|
452
|
+
}
|
|
453
|
+
return null;
|
|
454
|
+
},
|
|
455
|
+
graphDocNumber(nodeId) {
|
|
456
|
+
const m = nodeId.match(/^doc:OIML-[A-Z]-(\d+)-/);
|
|
457
|
+
return m ? m[1] : null;
|
|
458
|
+
},
|
|
459
|
+
familyOf(di) {
|
|
460
|
+
const m = /^(?:OIML\s+)?([A-Z])\s?(\d{1,3})(?:[-–]([0-9A-Za-z]+))?/.exec(di);
|
|
461
|
+
return m ? `${m[1]}-${m[2]}` : null;
|
|
462
|
+
}
|
|
463
|
+
};
|
|
464
|
+
var plainSlug = {
|
|
465
|
+
parse: () => null,
|
|
466
|
+
scanQuestion: () => null,
|
|
467
|
+
graphDocNumber: () => null,
|
|
468
|
+
familyOf: () => null
|
|
469
|
+
};
|
|
470
|
+
var REGISTRY = {
|
|
471
|
+
"oiml-pubid": oimlPubid,
|
|
472
|
+
"plain-slug": plainSlug
|
|
473
|
+
};
|
|
474
|
+
function refCodec() {
|
|
475
|
+
return REGISTRY[P().publisher.codec] ?? plainSlug;
|
|
476
|
+
}
|
|
477
|
+
|
|
478
|
+
// workers/worker_public/src/context.ts
|
|
479
|
+
var NO_CONTEXT = { kind: "none", scoped_to: null };
|
|
480
|
+
function parseContext(body) {
|
|
481
|
+
const c = body?.context;
|
|
482
|
+
if (!c || typeof c !== "object") return null;
|
|
483
|
+
if (c.kind !== "page" && c.kind !== "entity" && c.kind !== "document" && c.kind !== "account") return null;
|
|
484
|
+
const label = typeof c.label === "string" ? c.label.trim().slice(0, 120) : "";
|
|
485
|
+
const route = typeof c.route === "string" && c.route.trim() ? c.route.trim().slice(0, 200) : void 0;
|
|
486
|
+
const doc = typeof c.doc === "string" && c.doc.trim() ? c.doc.trim().slice(0, 80) : void 0;
|
|
487
|
+
const edition = typeof c.edition === "string" && /^\d{4}$/.test(c.edition.trim()) ? c.edition.trim() : void 0;
|
|
488
|
+
return { kind: c.kind, label, ...route ? { route } : {}, ...doc ? { doc } : {}, ...edition ? { edition } : {} };
|
|
489
|
+
}
|
|
490
|
+
function parseDocRef(doc, edition) {
|
|
491
|
+
return refCodec().parse(doc, edition);
|
|
492
|
+
}
|
|
493
|
+
function namedDocumentIn(query) {
|
|
494
|
+
return refCodec().scanQuestion(query);
|
|
495
|
+
}
|
|
496
|
+
async function resolveDocScope(env, ctx) {
|
|
497
|
+
if (!ctx.doc) return null;
|
|
498
|
+
const parsed = parseDocRef(ctx.doc, ctx.edition);
|
|
499
|
+
if (!parsed) return null;
|
|
500
|
+
try {
|
|
501
|
+
const type = parsed.label.split(" ")[1];
|
|
502
|
+
const row = await env.DB.prepare("SELECT 1 FROM documents WHERE family = ?1 LIMIT 1").bind(`${type}-${parsed.doc_number}`).first();
|
|
503
|
+
if (!row) return null;
|
|
504
|
+
} catch {
|
|
505
|
+
}
|
|
506
|
+
return parsed;
|
|
507
|
+
}
|
|
508
|
+
function appliedContext(declared, scope, note, live) {
|
|
509
|
+
if (!declared) return NO_CONTEXT;
|
|
510
|
+
return {
|
|
511
|
+
kind: declared.kind,
|
|
512
|
+
label: declared.label,
|
|
513
|
+
scoped_to: scope ? scope.label : null,
|
|
514
|
+
...note ? { note } : {},
|
|
515
|
+
...live ? { live } : {}
|
|
516
|
+
};
|
|
517
|
+
}
|
|
518
|
+
function parseAppliedContext(v) {
|
|
519
|
+
if (!v || typeof v !== "object") return null;
|
|
520
|
+
if (v.kind !== "page" && v.kind !== "entity" && v.kind !== "document" && v.kind !== "account" && v.kind !== "none") return null;
|
|
521
|
+
const label = typeof v.label === "string" && v.label.trim() ? v.label.trim().slice(0, 120) : void 0;
|
|
522
|
+
const scoped = typeof v.scoped_to === "string" && v.scoped_to.trim() ? v.scoped_to.trim().slice(0, 80) : null;
|
|
523
|
+
const note = v.note === "document-not-in-corpus" || v.note === "question-document-wins" || v.note === "sign-in-required" || v.note === "live-window-expired" || v.note === "live-unavailable" ? v.note : void 0;
|
|
524
|
+
const live = v.live && typeof v.live === "object" && typeof v.live.read_at === "string" && Array.isArray(v.live.stores) && typeof v.live.records === "number" ? { read_at: v.live.read_at.slice(0, 40), stores: v.live.stores.filter((s) => typeof s === "string").slice(0, 8), records: Math.min(Math.max(0, v.live.records), 999) } : void 0;
|
|
525
|
+
const model = v.model && typeof v.model === "object" && typeof v.model.node_id === "string" && typeof v.model.kind === "string" && typeof v.model.standard === "string" ? {
|
|
526
|
+
node_id: v.model.node_id.slice(0, 120),
|
|
527
|
+
kind: v.model.kind.slice(0, 40),
|
|
528
|
+
standard: v.model.standard.slice(0, 40),
|
|
529
|
+
...typeof v.model.clause === "string" && v.model.clause.trim() ? { clause: v.model.clause.slice(0, 120) } : {}
|
|
530
|
+
} : void 0;
|
|
531
|
+
return { kind: v.kind, ...label ? { label } : {}, scoped_to: scoped, ...note ? { note } : {}, ...live ? { live } : {}, ...model ? { model } : {} };
|
|
532
|
+
}
|
|
533
|
+
function contextNote(declared, scope) {
|
|
534
|
+
if (!declared) return void 0;
|
|
535
|
+
if (declared.kind === "account") {
|
|
536
|
+
return void 0;
|
|
537
|
+
}
|
|
538
|
+
if (declared.kind === "page") {
|
|
539
|
+
return `Context note: the user is viewing ${declared.label || "a page"}${declared.route ? ` (${declared.route})` : ""} in the ${P().publisher.product_name} platform. The passages come from the general corpus; frame procedural guidance for that page when relevant.`;
|
|
540
|
+
}
|
|
541
|
+
if (declared.kind === "entity") {
|
|
542
|
+
return scope ? `Context note: the user is asking about ${declared.label || "an entity"} \u2014 the passages are scoped to ${scope.label}, the publication that governs it. You do NOT have the entity's own data; answer what the publication requires and say when the question needs the record itself.` : `Context note: the user is asking about ${declared.label || "an entity"}. You do NOT have the entity's own data; answer from the corpus passages and say when the question needs the record itself.`;
|
|
543
|
+
}
|
|
544
|
+
return scope ? `Context note: the user scoped this question to ${scope.label} \u2014 the passages come from that publication. If they cannot answer the question, say so instead of drawing on other documents.` : `Context note: the user named ${declared.label || declared.doc || "a document"} as context, but it is not in the indexed corpus \u2014 answer from the general corpus and say the document was not found.`;
|
|
545
|
+
}
|
|
546
|
+
function syntheticUnderstanding(scope) {
|
|
547
|
+
return {
|
|
548
|
+
intent: "knowledge",
|
|
549
|
+
docidentifier: scope.label,
|
|
550
|
+
doc_number: scope.doc_number,
|
|
551
|
+
edition: scope.edition ?? null,
|
|
552
|
+
language: null,
|
|
553
|
+
process_intent: false,
|
|
554
|
+
term: null,
|
|
555
|
+
defined_terms: [],
|
|
556
|
+
standalone_query: "",
|
|
557
|
+
complexity: "simple",
|
|
558
|
+
query_variants: [],
|
|
559
|
+
sub_queries: [],
|
|
560
|
+
hypothetical_answer: "",
|
|
561
|
+
follow_ups: []
|
|
562
|
+
};
|
|
563
|
+
}
|
|
564
|
+
|
|
565
|
+
// workers/worker_public/src/stages/citationProbe.ts
|
|
566
|
+
var CITE_PATTERN = /\b(?:cite[sd]?|citing|referenc(?:e|es|ed|ing)|list[s]?|quote[sd]?)\b/i;
|
|
567
|
+
var REFS_PATTERN = /\b(?:standard|publication|document|normative|bibliograph)/i;
|
|
568
|
+
function citationGraphNote(docLabel, rows, cap = 30) {
|
|
569
|
+
const bySrc = /* @__PURE__ */ new Map();
|
|
570
|
+
for (const r of rows) {
|
|
571
|
+
const key = r.edition && !r.docidentifier.includes(r.edition) ? `${r.docidentifier}:${r.edition}` : r.docidentifier;
|
|
572
|
+
let e = bySrc.get(key);
|
|
573
|
+
if (!e) bySrc.set(key, e = { active: !!r.active, labels: [] });
|
|
574
|
+
if (e.labels.length < cap && !e.labels.includes(r.label)) e.labels.push(r.label);
|
|
575
|
+
}
|
|
576
|
+
if (!bySrc.size) return "";
|
|
577
|
+
const lines = [...bySrc.entries()].sort((a, b) => Number(b[1].active) - Number(a[1].active)).map(([k, v]) => `- ${k}${v.active ? " (active edition)" : ""} cites: ${v.labels.join(", ")}`);
|
|
578
|
+
return [
|
|
579
|
+
`Citation graph (authoritative \u2014 extracted from the indexed bibliographies of ${docLabel}):`,
|
|
580
|
+
...lines,
|
|
581
|
+
`When the question asks what ${docLabel} cites or references, answer from this list, name each standard exactly as listed, and cite the bibliography passage(s) provided in the context.`
|
|
582
|
+
].join("\n");
|
|
583
|
+
}
|
|
584
|
+
var citationProbe = {
|
|
585
|
+
name: "citation-probe",
|
|
586
|
+
failure: "additive",
|
|
587
|
+
when: (c) => {
|
|
588
|
+
if (!CITE_PATTERN.test(c.query) || !REFS_PATTERN.test(c.query)) return false;
|
|
589
|
+
const named = namedDocumentIn(c.query);
|
|
590
|
+
if (!named) return false;
|
|
591
|
+
c.__citeDocNum = named.doc_number;
|
|
592
|
+
c.__citeFamily = refCodec().familyOf(named.label);
|
|
593
|
+
c.__citeLabel = named.label;
|
|
594
|
+
c.__citeEdition = named.edition ?? null;
|
|
595
|
+
return true;
|
|
596
|
+
},
|
|
597
|
+
prefetch: (c) => {
|
|
598
|
+
const docNum = String(c.__citeDocNum ?? c.u?.doc_number ?? "");
|
|
599
|
+
const family = c.__citeFamily;
|
|
600
|
+
c.lane["citation-probe"] = Promise.all([
|
|
601
|
+
(async () => {
|
|
602
|
+
if (!docNum) return [];
|
|
603
|
+
try {
|
|
604
|
+
const rows = await c.env.DB.prepare(
|
|
605
|
+
"SELECT c.id FROM chunks_fts f JOIN chunks c ON c.rowid = f.rowid WHERE chunks_fts MATCH ?1 AND c.doc_number = ?2 AND (c.clause_title LIKE '%ibliograph%' OR c.clause_title LIKE '%ormative reference%') LIMIT 8"
|
|
606
|
+
).bind("bibliography OR references", docNum).all();
|
|
607
|
+
const ids = (rows.results ?? []).map((r) => r.id).slice(0, 8);
|
|
608
|
+
if (!ids.length) return [];
|
|
609
|
+
const got = await c.env.VECTORIZE.getByIds(ids);
|
|
610
|
+
if (!got?.length) return [];
|
|
611
|
+
const ph = ids.map((_, i) => `?${i + 1}`).join(",");
|
|
612
|
+
const texts = await c.env.DB.prepare(`SELECT id, text FROM chunks WHERE id IN (${ph})`).bind(...ids).all();
|
|
613
|
+
const textById = new Map((texts.results ?? []).map((r) => [r.id, r.text]));
|
|
614
|
+
return got.filter((h) => textById.has(h.id)).map((h) => ({ ...h, score: 10, text: textById.get(h.id) }));
|
|
615
|
+
} catch {
|
|
616
|
+
return [];
|
|
617
|
+
}
|
|
618
|
+
})(),
|
|
619
|
+
// the graph's cites edges for the family — structured, edition-keyed
|
|
620
|
+
(async () => {
|
|
621
|
+
if (!family) return [];
|
|
622
|
+
try {
|
|
623
|
+
const rows = await c.env.DB.prepare(
|
|
624
|
+
"SELECT d.docidentifier, d.edition, d.active, n.label FROM graph_edges e JOIN documents d ON e.src = d.canonical_id JOIN graph_nodes n ON e.dst = n.id WHERE e.kind = 'cites' AND d.family = ?1 ORDER BY d.active DESC, d.edition DESC LIMIT 120"
|
|
625
|
+
).bind(family).all();
|
|
626
|
+
return rows.results ?? [];
|
|
627
|
+
} catch {
|
|
628
|
+
return [];
|
|
629
|
+
}
|
|
630
|
+
})()
|
|
631
|
+
]);
|
|
632
|
+
},
|
|
633
|
+
run: async (c) => {
|
|
634
|
+
const [probes, citeRows] = await c.lane["citation-probe"];
|
|
635
|
+
const seen = new Set(c.hits.map((m) => m.id));
|
|
636
|
+
let added = 0;
|
|
637
|
+
for (const h of probes) {
|
|
638
|
+
if (seen.has(h.id)) continue;
|
|
639
|
+
const title = String(h.metadata?.clause_title ?? "");
|
|
640
|
+
const text = String(h.text ?? "");
|
|
641
|
+
if (/bibliograph|normative reference/i.test(title + " " + text.slice(0, 300))) {
|
|
642
|
+
c.hits.push(h);
|
|
643
|
+
seen.add(h.id);
|
|
644
|
+
added++;
|
|
645
|
+
}
|
|
646
|
+
}
|
|
647
|
+
const edition = c.__citeEdition;
|
|
648
|
+
const scoped = edition ? citeRows.filter((r) => r.edition === edition) : citeRows;
|
|
649
|
+
const note = citationGraphNote(c.__citeLabel, scoped);
|
|
650
|
+
if (note) c.notes.push(note);
|
|
651
|
+
console.log("citation-probe:", added, "passages,", note ? "graph note on" : "graph note off", `(${citeRows.length} cite rows)`);
|
|
652
|
+
}
|
|
653
|
+
};
|
|
654
|
+
|
|
435
655
|
// workers/worker_public/src/ports/cloudflare/adapters.ts
|
|
436
656
|
var EMBED_REQUEST_SHAPES = {
|
|
437
657
|
// "text" first: the verified request shape for qwen3-embedding-0.6b
|
|
@@ -618,49 +838,6 @@ var glossary = {
|
|
|
618
838
|
}
|
|
619
839
|
};
|
|
620
840
|
|
|
621
|
-
// workers/worker_public/src/codecs.ts
|
|
622
|
-
var oimlPubid = {
|
|
623
|
-
parse(doc, edition) {
|
|
624
|
-
const m = doc.match(/^urn:oiml:pub:([rdbge]):(\d{1,3})(?:-[0-9A-Za-z]+)?(?::(\d{4}))?$/i) ?? doc.match(/^(?:OIML\s+)?([RDBGE])\s*(\d{1,3})(?:-[0-9A-Za-z]+)?(?::(\d{4}))?$/i);
|
|
625
|
-
if (!m) return null;
|
|
626
|
-
const type = m[1].toUpperCase();
|
|
627
|
-
const ed = edition ?? m[3] ?? void 0;
|
|
628
|
-
return { doc_number: m[2], ...ed ? { edition: ed } : {}, label: `OIML ${type} ${m[2]}${ed ? `:${ed}` : ""}` };
|
|
629
|
-
},
|
|
630
|
-
scanQuestion(query) {
|
|
631
|
-
const re = /\b(OIML\s+)?([RDBGE])(\s*)0*(\d{1,3})(?:\s*[-–]\s*\d+)?(?:\s*:\s*(\d{4}))?/gi;
|
|
632
|
-
for (const m of query.matchAll(re)) {
|
|
633
|
-
const [, oimlPrefix, letter, gap, digits, edition] = m;
|
|
634
|
-
if (digits.length === 1 && !oimlPrefix && !gap) continue;
|
|
635
|
-
const num2 = String(Number(digits));
|
|
636
|
-
const type = letter.toUpperCase();
|
|
637
|
-
return { doc_number: num2, ...edition ? { edition } : {}, label: `OIML ${type} ${num2}${edition ? `:${edition}` : ""}` };
|
|
638
|
-
}
|
|
639
|
-
return null;
|
|
640
|
-
},
|
|
641
|
-
graphDocNumber(nodeId) {
|
|
642
|
-
const m = nodeId.match(/^doc:OIML-[A-Z]-(\d+)-/);
|
|
643
|
-
return m ? m[1] : null;
|
|
644
|
-
},
|
|
645
|
-
familyOf(di) {
|
|
646
|
-
const m = /^(?:OIML\s+)?([A-Z])\s?(\d{1,3})(?:[-–]([0-9A-Za-z]+))?/.exec(di);
|
|
647
|
-
return m ? `${m[1]}-${m[2]}` : null;
|
|
648
|
-
}
|
|
649
|
-
};
|
|
650
|
-
var plainSlug = {
|
|
651
|
-
parse: () => null,
|
|
652
|
-
scanQuestion: () => null,
|
|
653
|
-
graphDocNumber: () => null,
|
|
654
|
-
familyOf: () => null
|
|
655
|
-
};
|
|
656
|
-
var REGISTRY = {
|
|
657
|
-
"oiml-pubid": oimlPubid,
|
|
658
|
-
"plain-slug": plainSlug
|
|
659
|
-
};
|
|
660
|
-
function refCodec() {
|
|
661
|
-
return REGISTRY[P().publisher.codec] ?? plainSlug;
|
|
662
|
-
}
|
|
663
|
-
|
|
664
841
|
// workers/worker_public/src/stages/conceptGraph.ts
|
|
665
842
|
var conceptGraph = {
|
|
666
843
|
name: "concept-graph",
|
|
@@ -1316,6 +1493,7 @@ var STAGES = [
|
|
|
1316
1493
|
familyBoost,
|
|
1317
1494
|
rerankStage,
|
|
1318
1495
|
lexicalRrf,
|
|
1496
|
+
citationProbe,
|
|
1319
1497
|
corpusScope,
|
|
1320
1498
|
editionCover,
|
|
1321
1499
|
stdRefNudge,
|
|
@@ -1374,6 +1552,7 @@ async function retrieve(env, query, opts = {}) {
|
|
|
1374
1552
|
hits: [],
|
|
1375
1553
|
finalHits: [],
|
|
1376
1554
|
glossary: [],
|
|
1555
|
+
notes: [],
|
|
1377
1556
|
opts,
|
|
1378
1557
|
lane: {}
|
|
1379
1558
|
};
|
|
@@ -1381,7 +1560,8 @@ async function retrieve(env, query, opts = {}) {
|
|
|
1381
1560
|
return {
|
|
1382
1561
|
hits: ctx.finalHits,
|
|
1383
1562
|
filters: ctx.filters ?? {},
|
|
1384
|
-
...ctx.glossary?.length ? { glossary: ctx.glossary } : {}
|
|
1563
|
+
...ctx.glossary?.length ? { glossary: ctx.glossary } : {},
|
|
1564
|
+
...ctx.notes?.length ? { notes: ctx.notes } : {}
|
|
1385
1565
|
};
|
|
1386
1566
|
}
|
|
1387
1567
|
function estTokens(s) {
|
|
@@ -2185,93 +2365,6 @@ async function handleLogout(env, req) {
|
|
|
2185
2365
|
return new Response(null, { status: 302, headers });
|
|
2186
2366
|
}
|
|
2187
2367
|
|
|
2188
|
-
// workers/worker_public/src/context.ts
|
|
2189
|
-
var NO_CONTEXT = { kind: "none", scoped_to: null };
|
|
2190
|
-
function parseContext(body) {
|
|
2191
|
-
const c = body?.context;
|
|
2192
|
-
if (!c || typeof c !== "object") return null;
|
|
2193
|
-
if (c.kind !== "page" && c.kind !== "entity" && c.kind !== "document" && c.kind !== "account") return null;
|
|
2194
|
-
const label = typeof c.label === "string" ? c.label.trim().slice(0, 120) : "";
|
|
2195
|
-
const route = typeof c.route === "string" && c.route.trim() ? c.route.trim().slice(0, 200) : void 0;
|
|
2196
|
-
const doc = typeof c.doc === "string" && c.doc.trim() ? c.doc.trim().slice(0, 80) : void 0;
|
|
2197
|
-
const edition = typeof c.edition === "string" && /^\d{4}$/.test(c.edition.trim()) ? c.edition.trim() : void 0;
|
|
2198
|
-
return { kind: c.kind, label, ...route ? { route } : {}, ...doc ? { doc } : {}, ...edition ? { edition } : {} };
|
|
2199
|
-
}
|
|
2200
|
-
function parseDocRef(doc, edition) {
|
|
2201
|
-
return refCodec().parse(doc, edition);
|
|
2202
|
-
}
|
|
2203
|
-
function namedDocumentIn(query) {
|
|
2204
|
-
return refCodec().scanQuestion(query);
|
|
2205
|
-
}
|
|
2206
|
-
async function resolveDocScope(env, ctx) {
|
|
2207
|
-
if (!ctx.doc) return null;
|
|
2208
|
-
const parsed = parseDocRef(ctx.doc, ctx.edition);
|
|
2209
|
-
if (!parsed) return null;
|
|
2210
|
-
try {
|
|
2211
|
-
const type = parsed.label.split(" ")[1];
|
|
2212
|
-
const row = await env.DB.prepare("SELECT 1 FROM documents WHERE family = ?1 LIMIT 1").bind(`${type}-${parsed.doc_number}`).first();
|
|
2213
|
-
if (!row) return null;
|
|
2214
|
-
} catch {
|
|
2215
|
-
}
|
|
2216
|
-
return parsed;
|
|
2217
|
-
}
|
|
2218
|
-
function appliedContext(declared, scope, note, live) {
|
|
2219
|
-
if (!declared) return NO_CONTEXT;
|
|
2220
|
-
return {
|
|
2221
|
-
kind: declared.kind,
|
|
2222
|
-
label: declared.label,
|
|
2223
|
-
scoped_to: scope ? scope.label : null,
|
|
2224
|
-
...note ? { note } : {},
|
|
2225
|
-
...live ? { live } : {}
|
|
2226
|
-
};
|
|
2227
|
-
}
|
|
2228
|
-
function parseAppliedContext(v) {
|
|
2229
|
-
if (!v || typeof v !== "object") return null;
|
|
2230
|
-
if (v.kind !== "page" && v.kind !== "entity" && v.kind !== "document" && v.kind !== "account" && v.kind !== "none") return null;
|
|
2231
|
-
const label = typeof v.label === "string" && v.label.trim() ? v.label.trim().slice(0, 120) : void 0;
|
|
2232
|
-
const scoped = typeof v.scoped_to === "string" && v.scoped_to.trim() ? v.scoped_to.trim().slice(0, 80) : null;
|
|
2233
|
-
const note = v.note === "document-not-in-corpus" || v.note === "question-document-wins" || v.note === "sign-in-required" || v.note === "live-window-expired" || v.note === "live-unavailable" ? v.note : void 0;
|
|
2234
|
-
const live = v.live && typeof v.live === "object" && typeof v.live.read_at === "string" && Array.isArray(v.live.stores) && typeof v.live.records === "number" ? { read_at: v.live.read_at.slice(0, 40), stores: v.live.stores.filter((s) => typeof s === "string").slice(0, 8), records: Math.min(Math.max(0, v.live.records), 999) } : void 0;
|
|
2235
|
-
const model = v.model && typeof v.model === "object" && typeof v.model.node_id === "string" && typeof v.model.kind === "string" && typeof v.model.standard === "string" ? {
|
|
2236
|
-
node_id: v.model.node_id.slice(0, 120),
|
|
2237
|
-
kind: v.model.kind.slice(0, 40),
|
|
2238
|
-
standard: v.model.standard.slice(0, 40),
|
|
2239
|
-
...typeof v.model.clause === "string" && v.model.clause.trim() ? { clause: v.model.clause.slice(0, 120) } : {}
|
|
2240
|
-
} : void 0;
|
|
2241
|
-
return { kind: v.kind, ...label ? { label } : {}, scoped_to: scoped, ...note ? { note } : {}, ...live ? { live } : {}, ...model ? { model } : {} };
|
|
2242
|
-
}
|
|
2243
|
-
function contextNote(declared, scope) {
|
|
2244
|
-
if (!declared) return void 0;
|
|
2245
|
-
if (declared.kind === "account") {
|
|
2246
|
-
return void 0;
|
|
2247
|
-
}
|
|
2248
|
-
if (declared.kind === "page") {
|
|
2249
|
-
return `Context note: the user is viewing ${declared.label || "a page"}${declared.route ? ` (${declared.route})` : ""} in the ${P().publisher.product_name} platform. The passages come from the general corpus; frame procedural guidance for that page when relevant.`;
|
|
2250
|
-
}
|
|
2251
|
-
if (declared.kind === "entity") {
|
|
2252
|
-
return scope ? `Context note: the user is asking about ${declared.label || "an entity"} \u2014 the passages are scoped to ${scope.label}, the publication that governs it. You do NOT have the entity's own data; answer what the publication requires and say when the question needs the record itself.` : `Context note: the user is asking about ${declared.label || "an entity"}. You do NOT have the entity's own data; answer from the corpus passages and say when the question needs the record itself.`;
|
|
2253
|
-
}
|
|
2254
|
-
return scope ? `Context note: the user scoped this question to ${scope.label} \u2014 the passages come from that publication. If they cannot answer the question, say so instead of drawing on other documents.` : `Context note: the user named ${declared.label || declared.doc || "a document"} as context, but it is not in the indexed corpus \u2014 answer from the general corpus and say the document was not found.`;
|
|
2255
|
-
}
|
|
2256
|
-
function syntheticUnderstanding(scope) {
|
|
2257
|
-
return {
|
|
2258
|
-
intent: "knowledge",
|
|
2259
|
-
docidentifier: scope.label,
|
|
2260
|
-
doc_number: scope.doc_number,
|
|
2261
|
-
edition: scope.edition ?? null,
|
|
2262
|
-
language: null,
|
|
2263
|
-
process_intent: false,
|
|
2264
|
-
term: null,
|
|
2265
|
-
defined_terms: [],
|
|
2266
|
-
standalone_query: "",
|
|
2267
|
-
complexity: "simple",
|
|
2268
|
-
query_variants: [],
|
|
2269
|
-
sub_queries: [],
|
|
2270
|
-
hypothetical_answer: "",
|
|
2271
|
-
follow_ups: []
|
|
2272
|
-
};
|
|
2273
|
-
}
|
|
2274
|
-
|
|
2275
2368
|
// workers/worker_public/src/conversations.ts
|
|
2276
2369
|
var ID_RE = /^[a-zA-Z0-9_-]{8,64}$/;
|
|
2277
2370
|
async function ownedConversation(env, sub, id) {
|
|
@@ -2286,7 +2379,7 @@ async function handleConversations(env, sub, req, route) {
|
|
|
2286
2379
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
2287
2380
|
if (method === "GET" && !id) {
|
|
2288
2381
|
const rows = await env.DB.prepare(
|
|
2289
|
-
"SELECT c.id, c.title, c.updated_at, COUNT(m.id) AS messages FROM conversations c LEFT JOIN messages m ON m.conversation_id = c.id WHERE c.sub = ?1 GROUP BY c.id ORDER BY c.updated_at DESC LIMIT 50"
|
|
2382
|
+
"SELECT c.id, c.title, c.updated_at, c.project_id, COUNT(m.id) AS messages FROM conversations c LEFT JOIN messages m ON m.conversation_id = c.id WHERE c.sub = ?1 GROUP BY c.id ORDER BY c.updated_at DESC LIMIT 50"
|
|
2290
2383
|
).bind(sub).all();
|
|
2291
2384
|
return json({ conversations: rows.results ?? [] });
|
|
2292
2385
|
}
|
|
@@ -2621,6 +2714,7 @@ Rules:
|
|
|
2621
2714
|
- docidentifier: the publication the user names, in any spelling ({{SPELLING_EXAMPLES}}, "the nonautomatic weighing instruments recommendation" \u2192 resolve to the {{PUBLISHER_NAME}} identifier you can infer; include the part ("-1", "-2") only when clearly meant). docnumber is the base number without part.
|
|
2622
2715
|
- edition: only when the user pins a year.
|
|
2623
2716
|
- language: only when the user asks for a specific answer language; otherwise null (the corpus is English; answering in the user's language is handled elsewhere).
|
|
2717
|
+
- citation questions ("what does X cite/reference/list?", "which standards does X reference?"): ALWAYS include a query variant that names the document's bibliography or normative-references section explicitly, WITHOUT edition scoping (e.g. for "What ISO standards does R 60 cite?" generate BOTH "R 60 bibliography normative references" AND "R 60 2017 bibliography ISO IEC") \u2014 bibliographies embed differently than the query's phrasing, and prior editions may carry references the current edition dropped; set edition to null for these queries so retrieval covers the whole family.
|
|
2624
2718
|
- process_intent: true when the question is about the GOVERNING SYSTEM around publications rather than a publication's own technical content \u2014 HOW to get certified/apply/comply, OR which framework/vocabulary/{{PROCESS_VOCAB}}. Naming a Recommendation (e.g. "R 60") inside such a question does NOT make it a technical-content question: leave process_intent true and still emit docnumber when named, but the retrieval path must NOT seal to that document alone.
|
|
2625
2719
|
- term: the defined term when the question asks what something is ("what is an accuracy class" \u2192 "accuracy class"); otherwise null.
|
|
2626
2720
|
- defined_terms: the ESTABLISHED metrology / VIM terms this question is about, in the corpus's own terminology, EVEN WHEN the question uses everyday wording instead \u2014 match the TIME SCALE and sense carefully: "does the reading drift while a weight sits on it" (short-term, under load) \u2192 ["creep"]; "output keeps drifting over months of use" (long-term, in service) \u2192 ["span stability", "durability"]; "how many scale divisions is it allowed" \u2192 ["number of verification intervals"]. This is a terminology mapping, not a copy of the question's words. Empty when nothing maps.
|
|
@@ -4657,7 +4751,7 @@ Answer account questions from these records ONLY: name the record when you use i
|
|
|
4657
4751
|
const reordered = await listwiseRerank(env, MODELS.listwise, understanding?.standalone_query || q.query, retrieved.hits);
|
|
4658
4752
|
if (reordered) {
|
|
4659
4753
|
console.log("listwise: reordered", reordered[0]?.metadata?.docidentifier ?? "?", "to top");
|
|
4660
|
-
retrieved = { hits: reordered
|
|
4754
|
+
retrieved = { ...retrieved, hits: reordered };
|
|
4661
4755
|
}
|
|
4662
4756
|
}
|
|
4663
4757
|
const grade = await gradePromise;
|
|
@@ -4695,7 +4789,8 @@ Answer account questions from these records ONLY: name the record when you use i
|
|
|
4695
4789
|
hits,
|
|
4696
4790
|
q.lang,
|
|
4697
4791
|
keptHistory,
|
|
4698
|
-
|
|
4792
|
+
// stage-extracted graph facts (GraphRAG) ride the same note channel
|
|
4793
|
+
[processNote, eNote, contextNote(declaredCtx, docScope), accountNote, modelNote, vocabNote, memNote, machineNote, ...retrieved.notes ?? []].filter(Boolean).join("\n") || void 0,
|
|
4699
4794
|
summary,
|
|
4700
4795
|
budget
|
|
4701
4796
|
);
|
|
@@ -54,6 +54,7 @@ hand.
|
|
|
54
54
|
│ (derived status, active flags — the SSOT) │
|
|
55
55
|
│ structure typed nodes → ChunkRecordV2 (per block type) │
|
|
56
56
|
│ derivation registry + graph projection (relaton edges + │
|
|
57
|
+
│ citation edges from the indexed bibliographies │
|
|
57
58
|
│ glossarist defines edges) │
|
|
58
59
|
│ enrichment contextual contexts (quality-first lane, KV- │
|
|
59
60
|
│ cached, content-hash invalidated) │
|
package/docs/projects-design.md
CHANGED
|
@@ -2,6 +2,13 @@
|
|
|
2
2
|
|
|
3
3
|
> The question: should chats group into **projects** that share a
|
|
4
4
|
> per-project contextual memory? Yes — and the parts already exist.
|
|
5
|
+
>
|
|
6
|
+
> **Shipped (2026-09-15)**: membership is drag-and-drop — a chat drags
|
|
7
|
+
> onto a project row to file, onto the conversations list to unfile,
|
|
8
|
+
> with drop-target highlights and a live hint; a per-row picker covers
|
|
9
|
+
> touch/keyboard; chats carry a project badge. The conversations list
|
|
10
|
+
> carries `project_id` from the server (membership stays server-truth)
|
|
11
|
+
> and the move rides the server conversation id.
|
|
5
12
|
> This is the design; nothing is implemented yet.
|
|
6
13
|
|
|
7
14
|
## 1. Prior art — what the best got right and wrong
|
package/docs/sota-mechanisms.md
CHANGED
|
@@ -97,6 +97,21 @@ unit selection, so the pin lands the object the question is about
|
|
|
97
97
|
ranking answers from stale editions, guesses at doc scope, and never
|
|
98
98
|
surfaces a typed object.
|
|
99
99
|
|
|
100
|
+
### 7b. The citation graph (structured questions get graph answers)
|
|
101
|
+
|
|
102
|
+
Bibliographic citations become graph edges: what each edition's
|
|
103
|
+
bibliography cites is extracted at index time (the identifier grammar
|
|
104
|
+
lives in the publisher codec — ISO/IEC floor generic, publisher series
|
|
105
|
+
on top; ~300 cites edges across the OIML corpus), stored beside the
|
|
106
|
+
publication registry, and keyed by the same normalized pubids
|
|
107
|
+
everywhere ("OIML R 60", never a bare number). When a question asks
|
|
108
|
+
what a publication cites or references, the answer receives the
|
|
109
|
+
cited-standards list per edition as authoritative structured data
|
|
110
|
+
(the `notes` channel), while the bibliography section itself is
|
|
111
|
+
retrieved as the passage that grounds the answer verbatim. Structured
|
|
112
|
+
questions get graph answers; semantic questions get vector answers;
|
|
113
|
+
the hybrid class gets both, fused by the same rerank.
|
|
114
|
+
|
|
100
115
|
### 8. The answer contract (claims are checkable)
|
|
101
116
|
Inline citations on every claim; normative values quoted verbatim from
|
|
102
117
|
the cited passage; tables/formulas/figures rendered as typed objects
|
package/docs/spec-pipeline.md
CHANGED
|
@@ -51,18 +51,25 @@ candidate lanes → pool open → pool-level merges → refinement → window as
|
|
|
51
51
|
| 13 | `family-boost` | boost only under `filter.doc_number`; sort always | blocking | mutates `hits[].score`, sorts | Family chunks decisively boosted for doc-scoped queries; the sort establishes the rerank-failure fallback order. |
|
|
52
52
|
| 14 | `rerank` | `hits.length > 1` | additive | `hits[].rerank_score`, sorts, family pin | Cross-encoder scores; vector order is the designed fallback. Post-rerank family pin for doc-scoped queries. |
|
|
53
53
|
| 15 | `lexical-rrf` | `hits.length > 1 && lexicalHits.length` | blocking | REPLACES `hits` order | RRF fusion with the full-corpus lexical ranking. Runs even when rerank failed (additive semantics preserve this). |
|
|
54
|
-
| 16 | `
|
|
55
|
-
| 17 | `
|
|
56
|
-
| 18 | `
|
|
57
|
-
| 19 | `
|
|
58
|
-
| 20 | `
|
|
59
|
-
| 21 | `
|
|
60
|
-
| 22 | `
|
|
61
|
-
| 23 | `
|
|
62
|
-
|
|
|
63
|
-
|
|
|
64
|
-
|
|
|
65
|
-
|
|
|
54
|
+
| 16 | `citation-probe` | citation-question shape + a document named in the TEXT | additive | appends `hits`, appends `notes` | GraphRAG: bibliography-shaped questions get (a) the family's bibliography sections as passages — FTS over the chunk store, deterministic, text fetched from D1 (getByIds returns no text), pushed to `hits` at score 10 — and (b) the graph's `cites` edges for the family as an authoritative per-edition note on the `notes` channel, identifiers codec-normalized. The graph answers the STRUCTURE; the passages ground it verbatim. |
|
|
55
|
+
| 17 | `corpus-scope` | `opts.datasetScope` | blocking | filters `hits` | Dataset scope (the sidebar toggles): drops hits whose corpus the request excludes. LAST of the pool-assembly stages — the lexical union above refills the pool after rerank, so filtering earlier let excluded corpora back in. Corpora the toggle model doesn't name pass untouched. |
|
|
56
|
+
| 18 | `edition-cover` | `!filters.edition && hits.length > 1` | additive | appends `hits` | Registry-driven cover: when a pool holds only stale editions of a document whose ACTIVE edition the documents registry knows, fetch the current edition's chunks and add at `editionCoverDiscount` — steering needs the successor present to demote. |
|
|
57
|
+
| 19 | `std-ref-nudge` | query names ISO/IEC/ASTM/EN | blocking | `hits[].rerank_score`, sorts | Standard-reference nudge: chunks CARRYING such a citation get `stdRefNudgeSpread × spread` — the citing clause is the answer to "which standard does X invoke", and generic family prose otherwise fills the window (l5a measured). |
|
|
58
|
+
| 20 | `term-nudge` | `u.term` | blocking | `hits[].rerank_score`, sorts | Clause whose head IS the asked term gets `termNudgeSpread × spread` (decisive). |
|
|
59
|
+
| 21 | `concept-steer` | `glossary.length && hits.length > 1` | blocking | `hits[].rerank_score`, sorts | Vocabulary-link families boosted (`conceptSteerSpread`). |
|
|
60
|
+
| 22 | `edition-steer` | `!filters.edition && hits.length > 1` | blocking | `hits[].rerank_score`, sorts | Cross-pub recency boost + family-relative superseded-edition demotion (spread-scaled). |
|
|
61
|
+
| 23 | `structural-propagate` | — | blocking | REPLACES `hits` | FABLE TreeExpansion: score blends along the clause tree. |
|
|
62
|
+
| 24 | `diversity` | — | blocking | `finalHits` (from `hits`) | Per-publication caps (1 overview / 2–3 clauses; global overview cap 2/6); window cut to `rerankKeep`. FIRST writer of `finalHits`. |
|
|
63
|
+
| 25 | `typed-pin` | pin families resolvable | blocking (inner parent-fetch additive) | `finalHits` | Answer-contract v2: one typed unit guaranteed a slot (+ small-to-big parent fetch at `smallToBigDiscount`). |
|
|
64
|
+
| 26 | `section-descent` | a ranked depth-1 summary has children | additive | `finalHits` | Summary node → top child clauses at `sectionDescentDiscount`; the summary retires when children answer. |
|
|
65
|
+
| 27 | `dedup` | — | blocking | `finalHits` | FABLE ancestor-descendant same-chain collapse (≥0.5 text overlap). |
|
|
66
|
+
| 28 | `window-floor` | — | blocking | filters `finalHits` | Evidence-budget cut at `windowFloorFraction` of top; typed/family/unscored exempt; never fewer than two. |
|
|
67
|
+
|
|
68
|
+
The `notes` channel (the GraphRAG seam): stages may append structured
|
|
69
|
+
facts to `PipelineContext.notes`; `retrieve()` returns them and the ask
|
|
70
|
+
path merges them into the answer prompt's retrieval note — the same
|
|
71
|
+
channel the vocabulary link rides. `citation-probe` is the first
|
|
72
|
+
writer; any future graph-derived fact uses the same seam.
|
|
66
73
|
|
|
67
74
|
## Ordering dependencies (why the order is what it is)
|
|
68
75
|
|
|
@@ -77,7 +84,7 @@ candidate lanes → pool open → pool-level merges → refinement → window as
|
|
|
77
84
|
- **`seal` before `overview-demote`/`rerank`**: the declared context is
|
|
78
85
|
a hard scope, not a preference — steering and reranking happen WITHIN
|
|
79
86
|
it.
|
|
80
|
-
- **`rerank` before every steering stage** (
|
|
87
|
+
- **`rerank` before every steering stage** (19–22): steering is
|
|
81
88
|
spread-scaled over rerank scores; steering before rerank would be
|
|
82
89
|
erased by the re-sort.
|
|
83
90
|
- **`structural-propagate` after steering, before `diversity`**:
|
package/package.json
CHANGED
|
@@ -31,7 +31,7 @@
|
|
|
31
31
|
"@astrojs/markdown-satteri": "^0.4.1",
|
|
32
32
|
"@astrojs/mdx": "^4.3.14"
|
|
33
33
|
},
|
|
34
|
-
"version": "0.1.
|
|
34
|
+
"version": "0.1.4",
|
|
35
35
|
"description": "The Konneal engine: the publisher-agnostic build pipeline and API plane for standards intelligence (retrieval, answer contract, verdicts, evaluation).",
|
|
36
36
|
"license": "BSD-3-Clause",
|
|
37
37
|
"type": "module",
|
|
@@ -6,6 +6,7 @@ Rules:
|
|
|
6
6
|
- docidentifier: the publication the user names, in any spelling ({{SPELLING_EXAMPLES}}, "the nonautomatic weighing instruments recommendation" → resolve to the {{PUBLISHER_NAME}} identifier you can infer; include the part ("-1", "-2") only when clearly meant). docnumber is the base number without part.
|
|
7
7
|
- edition: only when the user pins a year.
|
|
8
8
|
- language: only when the user asks for a specific answer language; otherwise null (the corpus is English; answering in the user's language is handled elsewhere).
|
|
9
|
+
- citation questions ("what does X cite/reference/list?", "which standards does X reference?"): ALWAYS include a query variant that names the document's bibliography or normative-references section explicitly, WITHOUT edition scoping (e.g. for "What ISO standards does R 60 cite?" generate BOTH "R 60 bibliography normative references" AND "R 60 2017 bibliography ISO IEC") — bibliographies embed differently than the query's phrasing, and prior editions may carry references the current edition dropped; set edition to null for these queries so retrieval covers the whole family.
|
|
9
10
|
- process_intent: true when the question is about the GOVERNING SYSTEM around publications rather than a publication's own technical content — HOW to get certified/apply/comply, OR which framework/vocabulary/{{PROCESS_VOCAB}}. Naming a Recommendation (e.g. "R 60") inside such a question does NOT make it a technical-content question: leave process_intent true and still emit docnumber when named, but the retrieval path must NOT seal to that document alone.
|
|
10
11
|
- term: the defined term when the question asks what something is ("what is an accuracy class" → "accuracy class"); otherwise null.
|
|
11
12
|
- defined_terms: the ESTABLISHED metrology / VIM terms this question is about, in the corpus's own terminology, EVEN WHEN the question uses everyday wording instead — match the TIME SCALE and sense carefully: "does the reading drift while a weight sits on it" (short-term, under load) → ["creep"]; "output keeps drifting over months of use" (long-term, in service) → ["span stability", "durability"]; "how many scale divisions is it allowed" → ["number of verification intervals"]. This is a terminology mapping, not a copy of the question's words. Empty when nothing maps.
|
|
@@ -733,7 +733,7 @@ async function handleAsk(
|
|
|
733
733
|
const reordered = await listwiseRerank(env, MODELS.listwise, understanding?.standalone_query || q.query, retrieved.hits);
|
|
734
734
|
if (reordered) {
|
|
735
735
|
console.log("listwise: reordered", reordered[0]?.metadata?.docidentifier ?? "?", "to top");
|
|
736
|
-
retrieved = { hits: reordered
|
|
736
|
+
retrieved = { ...retrieved, hits: reordered };
|
|
737
737
|
}
|
|
738
738
|
}
|
|
739
739
|
const grade = await gradePromise;
|
|
@@ -787,7 +787,8 @@ async function handleAsk(
|
|
|
787
787
|
hits,
|
|
788
788
|
q.lang,
|
|
789
789
|
keptHistory,
|
|
790
|
-
|
|
790
|
+
// stage-extracted graph facts (GraphRAG) ride the same note channel
|
|
791
|
+
[processNote, eNote, contextNote(declaredCtx, docScope), accountNote, modelNote, vocabNote, memNote, machineNote, ...(retrieved.notes ?? [])].filter(Boolean).join("\n") || undefined,
|
|
791
792
|
summary,
|
|
792
793
|
budget,
|
|
793
794
|
);
|
|
@@ -38,7 +38,7 @@ export async function handleConversations(
|
|
|
38
38
|
|
|
39
39
|
if (method === "GET" && !id) {
|
|
40
40
|
const rows: any = await env.DB.prepare(
|
|
41
|
-
"SELECT c.id, c.title, c.updated_at, COUNT(m.id) AS messages FROM conversations c LEFT JOIN messages m ON m.conversation_id = c.id WHERE c.sub = ?1 GROUP BY c.id ORDER BY c.updated_at DESC LIMIT 50",
|
|
41
|
+
"SELECT c.id, c.title, c.updated_at, c.project_id, COUNT(m.id) AS messages FROM conversations c LEFT JOIN messages m ON m.conversation_id = c.id WHERE c.sub = ?1 GROUP BY c.id ORDER BY c.updated_at DESC LIMIT 50",
|
|
42
42
|
)
|
|
43
43
|
.bind(sub)
|
|
44
44
|
.all();
|
|
@@ -52,6 +52,9 @@ export interface Retrieved {
|
|
|
52
52
|
* among them (dense retrieval alone binds everyday words to the wrong
|
|
53
53
|
* term: measured "keeps drifting" → creep 0.69 vs durability 0.54) */
|
|
54
54
|
glossary?: GlossaryEntry[];
|
|
55
|
+
/** structured facts stages extracted from the graph (GraphRAG) —
|
|
56
|
+
* merged into the answer prompt's retrieval note */
|
|
57
|
+
notes?: string[];
|
|
55
58
|
}
|
|
56
59
|
|
|
57
60
|
// Short follow-ups are usually elliptical ("and the limits?") — fold the
|
|
@@ -130,13 +133,14 @@ export async function retrieve(
|
|
|
130
133
|
|
|
131
134
|
const ctx: PipelineContext = {
|
|
132
135
|
env, query, rq, folded, u, filters, filter, vector, lexicalHits,
|
|
133
|
-
matches: [], hits: [], finalHits: [], glossary: [], opts, lane: {},
|
|
136
|
+
matches: [], hits: [], finalHits: [], glossary: [], notes: [], opts, lane: {},
|
|
134
137
|
};
|
|
135
138
|
await runStages(STAGES, ctx);
|
|
136
139
|
return {
|
|
137
140
|
hits: ctx.finalHits,
|
|
138
141
|
filters: ctx.filters ?? {},
|
|
139
142
|
...(ctx.glossary?.length ? { glossary: ctx.glossary } : {}),
|
|
143
|
+
...(ctx.notes?.length ? { notes: ctx.notes } : {}),
|
|
140
144
|
};
|
|
141
145
|
}
|
|
142
146
|
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
// Citation probe (deterministic, GraphRAG): questions asking what a
|
|
2
|
+
// publication CITES/REFERENCES need the bibliography chunks in the
|
|
3
|
+
// pool, but the bibliography's embedding rarely matches the question's
|
|
4
|
+
// phrasing — and prior editions may carry references the current one
|
|
5
|
+
// dropped. This stage pattern-matches the citation question shape and
|
|
6
|
+
// does two things for the named document family (all editions):
|
|
7
|
+
// 1. surfaces the bibliography SECTIONS as passages (FTS over the
|
|
8
|
+
// chunk store — no embedding similarity involved), and
|
|
9
|
+
// 2. injects the graph's structured `cites` edges (built at ingest
|
|
10
|
+
// from the same bibliographies) as an authoritative note — the
|
|
11
|
+
// graph answers the STRUCTURE (the list of cited standards), the
|
|
12
|
+
// passages ground it verbatim.
|
|
13
|
+
// Additive: results join the pool; nothing is filtered.
|
|
14
|
+
import type { Stage } from "./types.ts";
|
|
15
|
+
import { namedDocumentIn } from "../context.ts";
|
|
16
|
+
import { refCodec } from "../codecs.ts";
|
|
17
|
+
import type { Hit } from "../../../shared/chunk.ts";
|
|
18
|
+
|
|
19
|
+
const CITE_PATTERN = /\b(?:cite[sd]?|citing|referenc(?:e|es|ed|ing)|list[s]?|quote[sd]?)\b/i;
|
|
20
|
+
const REFS_PATTERN = /\b(?:standard|publication|document|normative|bibliograph)/i;
|
|
21
|
+
|
|
22
|
+
export interface CiteRow {
|
|
23
|
+
docidentifier: string;
|
|
24
|
+
edition: string;
|
|
25
|
+
active: number;
|
|
26
|
+
label: string;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/** The authoritative citation note: per edition, the cited standards in
|
|
30
|
+
* bibliography order. docLabel is the codec-normalized identifier — a
|
|
31
|
+
* bare number never appears. */
|
|
32
|
+
export function citationGraphNote(docLabel: string, rows: CiteRow[], cap = 30): string {
|
|
33
|
+
const bySrc = new Map<string, { active: boolean; labels: string[] }>();
|
|
34
|
+
for (const r of rows) {
|
|
35
|
+
const key = r.edition && !r.docidentifier.includes(r.edition) ? `${r.docidentifier}:${r.edition}` : r.docidentifier;
|
|
36
|
+
let e = bySrc.get(key);
|
|
37
|
+
if (!e) bySrc.set(key, (e = { active: !!r.active, labels: [] }));
|
|
38
|
+
if (e.labels.length < cap && !e.labels.includes(r.label)) e.labels.push(r.label);
|
|
39
|
+
}
|
|
40
|
+
if (!bySrc.size) return "";
|
|
41
|
+
const lines = [...bySrc.entries()]
|
|
42
|
+
.sort((a, b) => Number(b[1].active) - Number(a[1].active))
|
|
43
|
+
.map(([k, v]) => `- ${k}${v.active ? " (active edition)" : ""} cites: ${v.labels.join(", ")}`);
|
|
44
|
+
return [
|
|
45
|
+
`Citation graph (authoritative — extracted from the indexed bibliographies of ${docLabel}):`,
|
|
46
|
+
...lines,
|
|
47
|
+
`When the question asks what ${docLabel} cites or references, answer from this list, name each standard exactly as listed, and cite the bibliography passage(s) provided in the context.`,
|
|
48
|
+
].join("\n");
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
export const citationProbe: Stage = {
|
|
52
|
+
name: "citation-probe",
|
|
53
|
+
failure: "additive",
|
|
54
|
+
when: (c) => {
|
|
55
|
+
if (!CITE_PATTERN.test(c.query) || !REFS_PATTERN.test(c.query)) return false;
|
|
56
|
+
// the TEXT-derived naming (namedDocumentIn), never the LLM's
|
|
57
|
+
// extraction — the understand model may omit doc_number for this
|
|
58
|
+
// query shape (measured: it did)
|
|
59
|
+
const named = namedDocumentIn(c.query);
|
|
60
|
+
if (!named) return false;
|
|
61
|
+
(c as any).__citeDocNum = named.doc_number;
|
|
62
|
+
// the family key is codec-derived (label → family), never a bare number
|
|
63
|
+
(c as any).__citeFamily = refCodec().familyOf(named.label);
|
|
64
|
+
(c as any).__citeLabel = named.label;
|
|
65
|
+
(c as any).__citeEdition = named.edition ?? null;
|
|
66
|
+
return true;
|
|
67
|
+
},
|
|
68
|
+
prefetch: (c) => {
|
|
69
|
+
// deterministic: the chunk store's own bibliography-titled chunks for
|
|
70
|
+
// this document family — no embedding similarity involved (three
|
|
71
|
+
// vector-probe iterations measured: the embedding never matched
|
|
72
|
+
// reliably)
|
|
73
|
+
const docNum = String((c as any).__citeDocNum ?? c.u?.doc_number ?? "");
|
|
74
|
+
const family = (c as any).__citeFamily as string | null;
|
|
75
|
+
c.lane["citation-probe"] = Promise.all([
|
|
76
|
+
(async () => {
|
|
77
|
+
if (!docNum) return [] as Hit[];
|
|
78
|
+
try {
|
|
79
|
+
// chunks_fts is a virtual FTS5 table over chunks.fts_text — it
|
|
80
|
+
// has NO metadata columns. The doc_number and clause_title live
|
|
81
|
+
// in the chunks content table; join them.
|
|
82
|
+
const rows = await c.env.DB.prepare(
|
|
83
|
+
"SELECT c.id FROM chunks_fts f JOIN chunks c ON c.rowid = f.rowid WHERE chunks_fts MATCH ?1 AND c.doc_number = ?2 AND (c.clause_title LIKE '%ibliograph%' OR c.clause_title LIKE '%ormative reference%') LIMIT 8",
|
|
84
|
+
)
|
|
85
|
+
.bind("bibliography OR references", docNum)
|
|
86
|
+
.all() as { results?: Array<{ id: string }> };
|
|
87
|
+
const ids = (rows.results ?? []).map((r: { id: string }) => r.id).slice(0, 8);
|
|
88
|
+
if (!ids.length) return [] as Hit[];
|
|
89
|
+
const got = await c.env.VECTORIZE.getByIds(ids);
|
|
90
|
+
if (!got?.length) return [] as Hit[];
|
|
91
|
+
// getByIds returns metadata WITHOUT the chunk text — the
|
|
92
|
+
// usedHits builder calls h.text.clipToTokens and crashes on
|
|
93
|
+
// undefined (the live 503). Fetch the text from D1 and merge.
|
|
94
|
+
const ph = ids.map((_: string, i: number) => `?${i + 1}`).join(",");
|
|
95
|
+
const texts = await c.env.DB.prepare(`SELECT id, text FROM chunks WHERE id IN (${ph})`)
|
|
96
|
+
.bind(...ids)
|
|
97
|
+
.all() as { results?: Array<{ id: string; text: string }> };
|
|
98
|
+
const textById = new Map((texts.results ?? []).map((r) => [r.id, r.text]));
|
|
99
|
+
return got
|
|
100
|
+
.filter((h: any) => textById.has(h.id))
|
|
101
|
+
.map((h: any) => ({ ...h, score: 10, text: textById.get(h.id)! }));
|
|
102
|
+
} catch {
|
|
103
|
+
return [] as Hit[];
|
|
104
|
+
}
|
|
105
|
+
})(),
|
|
106
|
+
// the graph's cites edges for the family — structured, edition-keyed
|
|
107
|
+
(async () => {
|
|
108
|
+
if (!family) return [] as CiteRow[];
|
|
109
|
+
try {
|
|
110
|
+
const rows = await c.env.DB.prepare(
|
|
111
|
+
"SELECT d.docidentifier, d.edition, d.active, n.label FROM graph_edges e JOIN documents d ON e.src = d.canonical_id JOIN graph_nodes n ON e.dst = n.id WHERE e.kind = 'cites' AND d.family = ?1 ORDER BY d.active DESC, d.edition DESC LIMIT 120",
|
|
112
|
+
)
|
|
113
|
+
.bind(family)
|
|
114
|
+
.all() as { results?: CiteRow[] };
|
|
115
|
+
return rows.results ?? [];
|
|
116
|
+
} catch {
|
|
117
|
+
return [] as CiteRow[];
|
|
118
|
+
}
|
|
119
|
+
})(),
|
|
120
|
+
]);
|
|
121
|
+
},
|
|
122
|
+
run: async (c) => {
|
|
123
|
+
const [probes, citeRows] = (await c.lane["citation-probe"]) as [Hit[], CiteRow[]];
|
|
124
|
+
const seen = new Set(c.hits.map((m: any) => m.id));
|
|
125
|
+
let added = 0;
|
|
126
|
+
// only bibliography-shaped chunks (clause title or text mentions it)
|
|
127
|
+
for (const h of probes) {
|
|
128
|
+
if (seen.has(h.id as any)) continue;
|
|
129
|
+
const title = String((h.metadata as any)?.clause_title ?? "");
|
|
130
|
+
const text = String(h.text ?? "");
|
|
131
|
+
if (/bibliograph|normative reference/i.test(title + " " + text.slice(0, 300))) {
|
|
132
|
+
c.hits.push(h);
|
|
133
|
+
seen.add(h.id as any);
|
|
134
|
+
added++;
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
// the structured note: an edition-scoped question sees only that
|
|
138
|
+
// edition's citations; the family question sees all editions
|
|
139
|
+
const edition = (c as any).__citeEdition as string | null;
|
|
140
|
+
const scoped = edition ? citeRows.filter((r) => r.edition === edition) : citeRows;
|
|
141
|
+
const note = citationGraphNote((c as any).__citeLabel as string, scoped);
|
|
142
|
+
if (note) c.notes.push(note);
|
|
143
|
+
console.log("citation-probe:", added, "passages,", note ? "graph note on" : "graph note off", `(${citeRows.length} cite rows)`);
|
|
144
|
+
},
|
|
145
|
+
};
|
|
@@ -12,6 +12,7 @@ import type { Stage } from "./types.ts";
|
|
|
12
12
|
export { runStages } from "./types.ts";
|
|
13
13
|
export type { PipelineContext, RetrieveOptions, GlossaryEntry, Stage } from "./types.ts";
|
|
14
14
|
import { dense } from "./dense.ts";
|
|
15
|
+
import { citationProbe } from "./citationProbe.ts";
|
|
15
16
|
import { hyde } from "./hyde.ts";
|
|
16
17
|
import { glossary } from "./glossary.ts";
|
|
17
18
|
import { conceptGraph } from "./conceptGraph.ts";
|
|
@@ -54,6 +55,7 @@ export const STAGES: Stage[] = [
|
|
|
54
55
|
familyBoost,
|
|
55
56
|
rerankStage,
|
|
56
57
|
lexicalRrf,
|
|
58
|
+
citationProbe,
|
|
57
59
|
corpusScope,
|
|
58
60
|
editionCover,
|
|
59
61
|
stdRefNudge,
|
|
@@ -64,6 +64,10 @@ export interface PipelineContext {
|
|
|
64
64
|
hits: Hit[]; // the ranked pool from poolOpen onward
|
|
65
65
|
finalHits: Hit[]; // the answer window
|
|
66
66
|
glossary: GlossaryEntry[]; // the vocabulary link (glossary stage owns)
|
|
67
|
+
/** structured facts stages contribute to the answer prompt (the
|
|
68
|
+
* GraphRAG seam: graph-derived notes ride the same channel the
|
|
69
|
+
* vocabulary link does — ask.ts merges them into the retrieval note) */
|
|
70
|
+
notes: string[];
|
|
67
71
|
opts: RetrieveOptions;
|
|
68
72
|
/** prefetch bag: stage-name → that stage's in-flight I/O promise (the
|
|
69
73
|
* stage owns its key; see Stage.prefetch) */
|