@konneal/engine 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +29 -0
- package/README.md +13 -0
- package/dist/admin.d.ts +26 -0
- package/dist/ai.d.ts +6 -0
- package/dist/anchors.d.ts +6 -0
- package/dist/answercache.d.ts +22 -0
- package/dist/ask.d.ts +5 -0
- package/dist/auth.d.ts +12 -0
- package/dist/bubble.d.ts +14 -0
- package/dist/chunk-LLWPT2XV.js +49 -0
- package/dist/chunk-MB74PTRM.js +114 -0
- package/dist/chunk-WOGQM7DJ.js +197 -0
- package/dist/chunk-WWNCWKKC.js +42 -0
- package/dist/completion.d.ts +5 -0
- package/dist/config.d.ts +154 -0
- package/dist/config.js +37 -0
- package/dist/context.d.ts +115 -0
- package/dist/conversations.d.ts +5 -0
- package/dist/drafts.d.ts +129 -0
- package/dist/env.d.ts +57 -0
- package/dist/faithfulness.d.ts +5 -0
- package/dist/grader.d.ts +3 -0
- package/dist/graph.d.ts +13 -0
- package/dist/hybrid.d.ts +7 -0
- package/dist/index.d.ts +9 -0
- package/dist/index.js +5373 -0
- package/dist/internal_gateway.d.ts +14 -0
- package/dist/lexical.d.ts +7 -0
- package/dist/livedata.d.ts +77 -0
- package/dist/memories.d.ts +10 -0
- package/dist/modelplane.d.ts +61 -0
- package/dist/oidc.d.ts +73 -0
- package/dist/pipeline.d.ts +57 -0
- package/dist/profile.d.ts +2 -0
- package/dist/profile.gen.d.ts +70 -0
- package/dist/profile.js +8 -0
- package/dist/projects.d.ts +8 -0
- package/dist/prompts/conversational.md +8 -0
- package/dist/prompts/enrichment.md +3 -0
- package/dist/prompts/faithfulness.md +1 -0
- package/dist/prompts/grader.md +5 -0
- package/dist/prompts/listwise.md +3 -0
- package/dist/prompts/precision.md +1 -0
- package/dist/prompts/reflect.md +1 -0
- package/dist/prompts/relevancy.md +1 -0
- package/dist/prompts/research.md +10 -0
- package/dist/prompts/section-summary.md +5 -0
- package/dist/prompts/summarize.md +1 -0
- package/dist/prompts/system.md +18 -0
- package/dist/prompts/understanding.md +17 -0
- package/dist/quota.d.ts +13 -0
- package/dist/reflect.d.ts +5 -0
- package/dist/refs.d.ts +40 -0
- package/dist/refusal.d.ts +9 -0
- package/dist/refusal.js +9 -0
- package/dist/requestScope.d.ts +26 -0
- package/dist/requestScope.js +10 -0
- package/dist/research.d.ts +8 -0
- package/dist/search.d.ts +4 -0
- package/dist/selfquery.d.ts +7 -0
- package/dist/session.d.ts +1 -0
- package/dist/share.d.ts +2 -0
- package/dist/structural.d.ts +27 -0
- package/dist/tablecontext.d.ts +11 -0
- package/dist/understand.d.ts +11 -0
- package/dist/understandContract.d.ts +29 -0
- package/dist/verdict.d.ts +24 -0
- package/docs/API.md +451 -0
- package/docs/ARCHITECTURE.md +302 -0
- package/docs/AUDIT-2026-08-24.md +71 -0
- package/docs/CONTRIBUTOR-AUDIT-2026-08-25.md +147 -0
- package/docs/INGEST-ARCHITECTURE.md +158 -0
- package/docs/MCP.md +92 -0
- package/docs/METANORMA-AI-SERIALIZATION.md +247 -0
- package/docs/MKO-EXPORT-PIPELINE.md +147 -0
- package/docs/REDESIGN-NORMATIVE-RAG-ETSI.md +485 -0
- package/docs/RESEARCH-SOTA-2026.md +243 -0
- package/docs/ROADMAP-SOTA.md +130 -0
- package/docs/SOTA-STAGE-SPECS.md +509 -0
- package/docs/annealment/F1-verdict.md +27 -0
- package/docs/annealment/F10-notes.md +19 -0
- package/docs/annealment/F11-composition.md +17 -0
- package/docs/annealment/F12-passport.md +17 -0
- package/docs/annealment/F2-counterfactual.md +20 -0
- package/docs/annealment/F3-absence.md +21 -0
- package/docs/annealment/F4-instance.md +18 -0
- package/docs/annealment/F5-workflow.md +21 -0
- package/docs/annealment/F6-impact.md +21 -0
- package/docs/annealment/F7-editions.md +18 -0
- package/docs/annealment/F8-selfverify.md +19 -0
- package/docs/annealment/F9-projection-qa.md +17 -0
- package/docs/annealment/L0-locate.md +19 -0
- package/docs/annealment/L1-extract.md +18 -0
- package/docs/annealment/L2-nomenclature.md +22 -0
- package/docs/annealment/L3-geometry.md +23 -0
- package/docs/annealment/L4-composition.md +21 -0
- package/docs/annealment/L5-cross-standard.md +20 -0
- package/docs/annealment/L6-diachrony.md +21 -0
- package/docs/annealment/L7-perception.md +20 -0
- package/docs/annealment/L8-computation.md +22 -0
- package/docs/annealment/L9-instance-process.md +23 -0
- package/docs/annealment/README.md +10 -0
- package/docs/guidelines-metanorma-ai-programme.md +279 -0
- package/docs/identity-onboarding-rag.md +65 -0
- package/docs/identity-service.md +219 -0
- package/docs/knowledge-annealment.md +273 -0
- package/docs/konneal-extraction-plan.md +481 -0
- package/docs/metanorma-for-ai.md +270 -0
- package/docs/mirror-plan.md +36 -0
- package/docs/multi-sdo-architecture.md +191 -0
- package/docs/paper-annealment-comparison.md +259 -0
- package/docs/paper-assets/architecture.svg +94 -0
- package/docs/paper-assets/contract-v2.svg +94 -0
- package/docs/paper-assets/mko-ingest.svg +91 -0
- package/docs/paper-oiml-bulletin.md +402 -0
- package/docs/paper-oiml-bulletin.mdx +419 -0
- package/docs/product-branding-options.md +172 -0
- package/docs/projects-design.md +88 -0
- package/docs/sota-mechanisms.md +184 -0
- package/docs/spec-api.md +77 -0
- package/docs/spec-pipeline.md +126 -0
- package/docs/vector-adapter.md +88 -0
- package/package.json +70 -0
- package/profile/corpora.yaml +5 -0
- package/profile/datasets.yaml +14 -0
- package/profile/prompts.yaml +5 -0
- package/profile/publisher.yaml +17 -0
- package/profile/retrieval.yaml +1 -0
- package/profile/sources.yaml +5 -0
- package/profile/ui.yaml +7 -0
- package/scripts/gen_profile.mjs +33 -0
- package/workers/shared/ai.ts +21 -0
- package/workers/shared/auth.ts +16 -0
- package/workers/shared/chunk.ts +108 -0
- package/workers/shared/oidc.ts +312 -0
- package/workers/shared/router.ts +45 -0
- package/workers/shared/session.ts +104 -0
- package/workers/worker_internal/src/index.ts +157 -0
- package/workers/worker_internal/tsconfig.json +15 -0
- package/workers/worker_internal/wrangler.toml +32 -0
- package/workers/worker_mcp/src/index.ts +175 -0
- package/workers/worker_mcp/tsconfig.json +13 -0
- package/workers/worker_mcp/wrangler.toml +18 -0
- package/workers/worker_public/migrations/0002_conversations.sql +22 -0
- package/workers/worker_public/migrations/0003_shared_conversations.sql +9 -0
- package/workers/worker_public/migrations/0004_graph.sql +16 -0
- package/workers/worker_public/migrations/0005_documents.sql +19 -0
- package/workers/worker_public/migrations/0006_conversation_entities.sql +11 -0
- package/workers/worker_public/migrations/0007_chunks_fts.sql +43 -0
- package/workers/worker_public/migrations/0008_unit_payloads.sql +15 -0
- package/workers/worker_public/migrations/0009_chunks_unit.sql +8 -0
- package/workers/worker_public/migrations/0009_message_context.sql +7 -0
- package/workers/worker_public/migrations/0010_model_nodes.sql +33 -0
- package/workers/worker_public/migrations/0011_schema_union.sql +38 -0
- package/workers/worker_public/migrations/0012_memories.sql +15 -0
- package/workers/worker_public/migrations/0013_projects.sql +21 -0
- package/workers/worker_public/node_modules/@cloudflare/workers-types/2021-11-03/index.d.ts +16306 -0
- package/workers/worker_public/node_modules/@cloudflare/workers-types/2021-11-03/index.ts +16261 -0
- package/workers/worker_public/node_modules/@cloudflare/workers-types/2022-01-31/index.d.ts +16373 -0
- package/workers/worker_public/node_modules/@cloudflare/workers-types/2022-01-31/index.ts +16328 -0
- package/workers/worker_public/node_modules/@cloudflare/workers-types/2022-03-21/index.d.ts +16382 -0
- package/workers/worker_public/node_modules/@cloudflare/workers-types/2022-03-21/index.ts +16337 -0
- package/workers/worker_public/node_modules/@cloudflare/workers-types/2022-08-04/index.d.ts +16383 -0
- package/workers/worker_public/node_modules/@cloudflare/workers-types/2022-08-04/index.ts +16338 -0
- package/workers/worker_public/node_modules/@cloudflare/workers-types/2022-10-31/index.d.ts +16403 -0
- package/workers/worker_public/node_modules/@cloudflare/workers-types/2022-10-31/index.ts +16358 -0
- package/workers/worker_public/node_modules/@cloudflare/workers-types/2022-11-30/index.d.ts +16408 -0
- package/workers/worker_public/node_modules/@cloudflare/workers-types/2022-11-30/index.ts +16363 -0
- package/workers/worker_public/node_modules/@cloudflare/workers-types/2023-03-01/index.d.ts +16414 -0
- package/workers/worker_public/node_modules/@cloudflare/workers-types/2023-03-01/index.ts +16369 -0
- package/workers/worker_public/node_modules/@cloudflare/workers-types/2023-07-01/index.d.ts +16414 -0
- package/workers/worker_public/node_modules/@cloudflare/workers-types/2023-07-01/index.ts +16369 -0
- package/workers/worker_public/node_modules/@cloudflare/workers-types/README.md +135 -0
- package/workers/worker_public/node_modules/@cloudflare/workers-types/entrypoints.svg +53 -0
- package/workers/worker_public/node_modules/@cloudflare/workers-types/experimental/index.d.ts +17095 -0
- package/workers/worker_public/node_modules/@cloudflare/workers-types/experimental/index.ts +17050 -0
- package/workers/worker_public/node_modules/@cloudflare/workers-types/index.d.ts +16306 -0
- package/workers/worker_public/node_modules/@cloudflare/workers-types/index.ts +16261 -0
- package/workers/worker_public/node_modules/@cloudflare/workers-types/latest/index.d.ts +16447 -0
- package/workers/worker_public/node_modules/@cloudflare/workers-types/latest/index.ts +16402 -0
- package/workers/worker_public/node_modules/@cloudflare/workers-types/oldest/index.d.ts +16306 -0
- package/workers/worker_public/node_modules/@cloudflare/workers-types/oldest/index.ts +16261 -0
- package/workers/worker_public/node_modules/@cloudflare/workers-types/package.json +11 -0
- package/workers/worker_public/package.json +13 -0
- package/workers/worker_public/prompts/conversational.md +8 -0
- package/workers/worker_public/prompts/enrichment.md +3 -0
- package/workers/worker_public/prompts/faithfulness.md +1 -0
- package/workers/worker_public/prompts/grader.md +5 -0
- package/workers/worker_public/prompts/listwise.md +3 -0
- package/workers/worker_public/prompts/precision.md +1 -0
- package/workers/worker_public/prompts/reflect.md +1 -0
- package/workers/worker_public/prompts/relevancy.md +1 -0
- package/workers/worker_public/prompts/research.md +10 -0
- package/workers/worker_public/prompts/section-summary.md +5 -0
- package/workers/worker_public/prompts/summarize.md +1 -0
- package/workers/worker_public/prompts/system.md +18 -0
- package/workers/worker_public/prompts/understanding.md +17 -0
- package/workers/worker_public/public/app.js +166 -0
- package/workers/worker_public/public/index.html +48 -0
- package/workers/worker_public/public/style.css +147 -0
- package/workers/worker_public/schema.sql +248 -0
- package/workers/worker_public/src/admin.ts +358 -0
- package/workers/worker_public/src/ai.ts +71 -0
- package/workers/worker_public/src/anchors.ts +41 -0
- package/workers/worker_public/src/answercache.ts +72 -0
- package/workers/worker_public/src/ask.ts +1094 -0
- package/workers/worker_public/src/auth.ts +252 -0
- package/workers/worker_public/src/bubble.ts +111 -0
- package/workers/worker_public/src/completion.ts +75 -0
- package/workers/worker_public/src/config.ts +238 -0
- package/workers/worker_public/src/context.ts +238 -0
- package/workers/worker_public/src/conversations.ts +162 -0
- package/workers/worker_public/src/drafts.ts +497 -0
- package/workers/worker_public/src/env.ts +90 -0
- package/workers/worker_public/src/faithfulness.ts +63 -0
- package/workers/worker_public/src/grader.ts +89 -0
- package/workers/worker_public/src/graph.ts +63 -0
- package/workers/worker_public/src/hybrid.ts +77 -0
- package/workers/worker_public/src/index.ts +441 -0
- package/workers/worker_public/src/internal_gateway.ts +41 -0
- package/workers/worker_public/src/lexical.ts +86 -0
- package/workers/worker_public/src/lib/hit.ts +4 -0
- package/workers/worker_public/src/lib/http.ts +83 -0
- package/workers/worker_public/src/lib/router.ts +4 -0
- package/workers/worker_public/src/livedata.ts +334 -0
- package/workers/worker_public/src/memories.ts +81 -0
- package/workers/worker_public/src/modelplane.ts +213 -0
- package/workers/worker_public/src/oidc.ts +333 -0
- package/workers/worker_public/src/pipeline.ts +377 -0
- package/workers/worker_public/src/ports/blobs.ts +7 -0
- package/workers/worker_public/src/ports/cloudflare/adapters.ts +177 -0
- package/workers/worker_public/src/ports/kv.ts +8 -0
- package/workers/worker_public/src/ports/model.ts +28 -0
- package/workers/worker_public/src/ports/runtime.ts +13 -0
- package/workers/worker_public/src/ports/store.ts +20 -0
- package/workers/worker_public/src/ports/vector.ts +26 -0
- package/workers/worker_public/src/profile.gen.ts +101 -0
- package/workers/worker_public/src/profile.ts +16 -0
- package/workers/worker_public/src/projects.ts +108 -0
- package/workers/worker_public/src/prompts.d.ts +6 -0
- package/workers/worker_public/src/quota.ts +54 -0
- package/workers/worker_public/src/reflect.ts +67 -0
- package/workers/worker_public/src/refs.ts +107 -0
- package/workers/worker_public/src/refusal.ts +65 -0
- package/workers/worker_public/src/requestScope.ts +71 -0
- package/workers/worker_public/src/research.ts +126 -0
- package/workers/worker_public/src/search.ts +56 -0
- package/workers/worker_public/src/selfquery.ts +25 -0
- package/workers/worker_public/src/session.ts +4 -0
- package/workers/worker_public/src/share.ts +53 -0
- package/workers/worker_public/src/stages/conceptGraph.ts +68 -0
- package/workers/worker_public/src/stages/conceptSteer.ts +39 -0
- package/workers/worker_public/src/stages/corpusScope.ts +25 -0
- package/workers/worker_public/src/stages/dedup.ts +10 -0
- package/workers/worker_public/src/stages/dense.ts +73 -0
- package/workers/worker_public/src/stages/diversity.ts +33 -0
- package/workers/worker_public/src/stages/editionCover.ts +63 -0
- package/workers/worker_public/src/stages/editionSteer.ts +88 -0
- package/workers/worker_public/src/stages/familyBoost.ts +22 -0
- package/workers/worker_public/src/stages/federate.ts +22 -0
- package/workers/worker_public/src/stages/glossary.ts +65 -0
- package/workers/worker_public/src/stages/graphLane.ts +31 -0
- package/workers/worker_public/src/stages/hyde.ts +29 -0
- package/workers/worker_public/src/stages/index.ts +69 -0
- package/workers/worker_public/src/stages/lexicalUnion.ts +21 -0
- package/workers/worker_public/src/stages/multiQuery.ts +57 -0
- package/workers/worker_public/src/stages/overviewDemote.ts +14 -0
- package/workers/worker_public/src/stages/poolOpen.ts +10 -0
- package/workers/worker_public/src/stages/propagate.ts +15 -0
- package/workers/worker_public/src/stages/rerank.ts +47 -0
- package/workers/worker_public/src/stages/seal.ts +16 -0
- package/workers/worker_public/src/stages/sectionDescent.ts +61 -0
- package/workers/worker_public/src/stages/stdRefNudge.ts +35 -0
- package/workers/worker_public/src/stages/subQuery.ts +42 -0
- package/workers/worker_public/src/stages/termNudge.ts +24 -0
- package/workers/worker_public/src/stages/typedPin.ts +131 -0
- package/workers/worker_public/src/stages/types.ts +112 -0
- package/workers/worker_public/src/stages/windowFloor.ts +23 -0
- package/workers/worker_public/src/structural.ts +171 -0
- package/workers/worker_public/src/tablecontext.ts +41 -0
- package/workers/worker_public/src/understand.ts +72 -0
- package/workers/worker_public/src/understandContract.ts +67 -0
- package/workers/worker_public/src/verdict.ts +255 -0
- package/workers/worker_public/tsconfig.json +18 -0
- package/workers/worker_public/wrangler.toml +104 -0
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
// Edition steering, family-relative: when the query does not pin an
|
|
2
|
+
// edition, chunks from an OLDER edition of a publication are demoted
|
|
3
|
+
// whenever a NEWER edition of the SAME publication is in the pool.
|
|
4
|
+
// Superseded editions match archaic phrasing strongly (their wording is
|
|
5
|
+
// what the question echoes) and the per-doc diversity cap then fills
|
|
6
|
+
// the publication's slots with them — observed: R 76-1:1992/1988
|
|
7
|
+
// passages displacing the current R 76-1:2006 on complex unfiltered
|
|
8
|
+
// queries. Cross-publication recency is deliberately NOT touched: a
|
|
9
|
+
// 1992 publication that is still current must not be demoted because
|
|
10
|
+
// some unrelated 2024 document exists. Scaled to the live rerank spread
|
|
11
|
+
// — the scores cluster within ~0.001.
|
|
12
|
+
import { THRESHOLDS } from "../config.ts";
|
|
13
|
+
import type { Stage } from "./types.ts";
|
|
14
|
+
|
|
15
|
+
export const editionSteer: Stage = {
|
|
16
|
+
name: "edition-steer",
|
|
17
|
+
when: (c) => !c.filters?.edition && c.hits.length > 1,
|
|
18
|
+
run: (c) => {
|
|
19
|
+
const year = (s?: string) => (/^(19|20)\d{2}$/.test(s ?? "") ? Number(s) : null);
|
|
20
|
+
// the FAMILY key (doctype + base number): "R 60", "R 60 (Annexes)"
|
|
21
|
+
// and "R 60-1/-2/-3" are one publication whose current shape is the
|
|
22
|
+
// 2021 PARTS — the 2017 monolith lives under other identifiers, so
|
|
23
|
+
// docidentifier-level steering never sees it against 2021
|
|
24
|
+
const family = (m: any) => `${m.doctype}|${String(m.doc_number ?? "").split("-")[0]}|${m.language}`;
|
|
25
|
+
const scored = c.hits.map((h) => h.rerank_score ?? h.score);
|
|
26
|
+
const spread = Math.max(...scored) - Math.min(...scored);
|
|
27
|
+
if (spread > 0) {
|
|
28
|
+
const newest = new Map<string, number>();
|
|
29
|
+
const famNewest = new Map<string, number>();
|
|
30
|
+
let anyYear = 0;
|
|
31
|
+
for (const h of c.hits) {
|
|
32
|
+
const y = year(h.metadata.edition);
|
|
33
|
+
if (!y || y < 1990) continue;
|
|
34
|
+
const k = `${h.metadata.docidentifier}|${h.metadata.language}`;
|
|
35
|
+
newest.set(k, Math.max(newest.get(k) ?? 0, y));
|
|
36
|
+
const fk = family(h.metadata);
|
|
37
|
+
famNewest.set(fk, Math.max(famNewest.get(fk) ?? 0, y));
|
|
38
|
+
anyYear = Math.max(anyYear, y);
|
|
39
|
+
}
|
|
40
|
+
// cross-publication tie-break: a current-edition publication ranks
|
|
41
|
+
// over stale ones (load-bearing — par-prepackaged: R 87:2004 must
|
|
42
|
+
// outrank 1990s texts); composed WITH the family-relative demotion
|
|
43
|
+
// below, which dominates for same-publication duplicates
|
|
44
|
+
if (anyYear > 1990) {
|
|
45
|
+
for (const h of c.hits) {
|
|
46
|
+
const y = year(h.metadata.edition);
|
|
47
|
+
if (y && y >= 1990) {
|
|
48
|
+
h.rerank_score = (h.rerank_score ?? h.score) + spread * THRESHOLDS.crossPubRecencySpread * ((y - 1990) / (anyYear - 1990));
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
let demoted = 0;
|
|
53
|
+
for (const h of c.hits) {
|
|
54
|
+
const y = year(h.metadata.edition);
|
|
55
|
+
// (a) same-identifier: the successor of THIS document is in the
|
|
56
|
+
// pool — FULL strength, because ordering is load-bearing here:
|
|
57
|
+
// the diversity cap keeps ONE overview per publication and
|
|
58
|
+
// near-tied editions otherwise hand the slot to the older one
|
|
59
|
+
// (observed: R 60's 2017 parts displacing the current 2021
|
|
60
|
+
// parts). A lone older edition (successor not in the pool)
|
|
61
|
+
// passes undemoted — superseded content stays citable when it
|
|
62
|
+
// alone carries the clause
|
|
63
|
+
const max = newest.get(`${h.metadata.docidentifier}|${h.metadata.language}`);
|
|
64
|
+
if (y && max && y < max) {
|
|
65
|
+
h.rerank_score = (h.rerank_score ?? h.score) - spread * THRESHOLDS.familyDemoteSpread;
|
|
66
|
+
demoted++;
|
|
67
|
+
continue;
|
|
68
|
+
}
|
|
69
|
+
// (b) family-wide, status-gated: a superseded/unknown-status
|
|
70
|
+
// edition demotes against the FAMILY's newest even under a
|
|
71
|
+
// different identifier — the corpus's own status carries the
|
|
72
|
+
// supersession the registry link missed. in-force documents are
|
|
73
|
+
// exempt: a newer part-2 must never demote a current part-1
|
|
74
|
+
const fmax = famNewest.get(family(h.metadata));
|
|
75
|
+
if (y && fmax && y < fmax && (h.metadata.status === "superseded" || h.metadata.status === "unknown")) {
|
|
76
|
+
h.rerank_score = (h.rerank_score ?? h.score) - spread * THRESHOLDS.familyDemoteSpread;
|
|
77
|
+
demoted++;
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
if (demoted) {
|
|
81
|
+
console.log("edition steering: demoted", demoted, "superseded-edition chunks (family-relative)");
|
|
82
|
+
c.hits.sort((a, b) => (b.rerank_score ?? -Infinity) - (a.rerank_score ?? -Infinity));
|
|
83
|
+
} else if (anyYear > 1990) {
|
|
84
|
+
c.hits.sort((a, b) => (b.rerank_score ?? -Infinity) - (a.rerank_score ?? -Infinity));
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
},
|
|
88
|
+
};
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
// Family boost + pool sort. Family chunks carry the multi-part structure
|
|
2
|
+
// (which parts/annexes exist) — they must reach the model for 'what is
|
|
3
|
+
// R 60' / 'how many parts' queries. Vector similarity alone won't rank
|
|
4
|
+
// them because they're short structural summaries competing with
|
|
5
|
+
// content-heavy clause text. Boost them decisively for doc-scoped
|
|
6
|
+
// queries, then sort the pool by score (the cross-encoder re-sorts
|
|
7
|
+
// after; this order is the rerank-failure fallback).
|
|
8
|
+
import type { Stage } from "./types.ts";
|
|
9
|
+
|
|
10
|
+
export const familyBoost: Stage = {
|
|
11
|
+
name: "family-boost",
|
|
12
|
+
run: (c) => {
|
|
13
|
+
if (c.filter?.doc_number) {
|
|
14
|
+
for (const h of c.hits) {
|
|
15
|
+
if (h.metadata.clause_anchor === "family") {
|
|
16
|
+
h.score = Math.max(h.score, ...c.hits.map((x) => x.score)) + 1;
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
c.hits.sort((a, b) => b.score - a.score);
|
|
21
|
+
},
|
|
22
|
+
};
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
// Federated ISO/IEC tier (additive by contract — the federate callback
|
|
2
|
+
// catches its own transport failures). Members get passages from the
|
|
3
|
+
// internal index (service binding) merged into the same candidate pool;
|
|
4
|
+
// the shared reranker + fusion below sort it out. Slight discount: the
|
|
5
|
+
// public corpus answers by default.
|
|
6
|
+
import { THRESHOLDS } from "../config.ts";
|
|
7
|
+
import type { Stage } from "./types.ts";
|
|
8
|
+
|
|
9
|
+
export const federate: Stage = {
|
|
10
|
+
name: "federate",
|
|
11
|
+
when: (c) => !!c.opts.federate,
|
|
12
|
+
run: async (c) => {
|
|
13
|
+
const fed = await c.opts.federate!(c.rq).catch(() => []);
|
|
14
|
+
const seen = new Set(c.hits.map((h) => h.id));
|
|
15
|
+
for (const h of fed) {
|
|
16
|
+
if (!seen.has(h.id)) {
|
|
17
|
+
c.hits.push({ ...h, score: h.score * THRESHOLDS.federateDiscount });
|
|
18
|
+
seen.add(h.id);
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
},
|
|
22
|
+
};
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
// Vocabulary link — the L2 nomenclature bridge (additive). Everyday
|
|
2
|
+
// words don't match defined terms — the one gap every comparison lane
|
|
3
|
+
// fails. Dense candidates + cross-encoder rerank, and the ANSWER model
|
|
4
|
+
// adjudicates among the top-3 (the entity-linking pattern: retrieval
|
|
5
|
+
// proposes, generation disambiguates — naive top-1 dense binding picks
|
|
6
|
+
// the wrong term). Computed BEFORE the pool closes, because the
|
|
7
|
+
// candidates' defining publications also ROUTE the pool: a question
|
|
8
|
+
// whose concept link lands in a family the primary lanes missed
|
|
9
|
+
// (colloquial "bag of flour sold by weight" → the prepackage domain)
|
|
10
|
+
// gets that family's passages merged like the graph lane does — the
|
|
11
|
+
// concept link is a family router, not just a note.
|
|
12
|
+
import { rerank } from "../ai.ts";
|
|
13
|
+
import { portModelRunner, portIndex, hasLane } from "../env.ts";
|
|
14
|
+
import { MODELS, THRESHOLDS } from "../config.ts";
|
|
15
|
+
import type { Stage } from "./types.ts";
|
|
16
|
+
|
|
17
|
+
interface GlossaryCand {
|
|
18
|
+
term: string;
|
|
19
|
+
definition: string;
|
|
20
|
+
docidentifier: string;
|
|
21
|
+
doc_number: string;
|
|
22
|
+
score: number;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export const glossary: Stage = {
|
|
26
|
+
name: "glossary",
|
|
27
|
+
failure: "additive",
|
|
28
|
+
when: (c) => hasLane(c.env, "glossary") && c.vector.length > 0,
|
|
29
|
+
prefetch: (c) => {
|
|
30
|
+
c.lane.glossary = (async () => {
|
|
31
|
+
const g = await portIndex(c.env, "glossary").query({ vector: c.vector, topK: 5 });
|
|
32
|
+
const cands = g.filter((m) => m.score >= THRESHOLDS.glossaryCosineFloor);
|
|
33
|
+
if (!cands.length) return [] as GlossaryCand[];
|
|
34
|
+
const texts = cands.map((m: any) => String(m.metadata?.chunk_text ?? ""));
|
|
35
|
+
const rs = await rerank(portModelRunner(c.env), MODELS.rerank, c.query, texts);
|
|
36
|
+
return cands
|
|
37
|
+
.map((m: any, i: number) => ({
|
|
38
|
+
term: String(m.metadata?.clause_title ?? "").trim(),
|
|
39
|
+
definition: String(m.metadata?.chunk_text ?? "").split(" — ").slice(1).join(" — ").slice(0, 300),
|
|
40
|
+
docidentifier: String(m.metadata?.docidentifier ?? ""),
|
|
41
|
+
doc_number: String(m.metadata?.doc_number ?? ""),
|
|
42
|
+
score: rs ? rs[i] : m.score,
|
|
43
|
+
}))
|
|
44
|
+
.filter((x: any) => x.term && x.definition);
|
|
45
|
+
})();
|
|
46
|
+
},
|
|
47
|
+
run: async (c) => {
|
|
48
|
+
const ranked = (await c.lane.glossary!) as GlossaryCand[];
|
|
49
|
+
// one entry per DISTINCT CONCEPT, and only candidates the
|
|
50
|
+
// cross-encoder actually deems relevant (score > 0). Distinctness
|
|
51
|
+
// is spelling-normalized: "Weigh labeler"/"Weigh labeller" are
|
|
52
|
+
// one concept in two spellings — without normalization the note's
|
|
53
|
+
// slots burn on variants while the domain-bridging concept one
|
|
54
|
+
// rank down ("actual quantity" — the PREPACKAGE domain) never
|
|
55
|
+
// reaches the adjudicator.
|
|
56
|
+
const norm = (t: string) => t.toLowerCase().replace(/[^a-z0-9 ]/g, "").replace(/labeler\b/g, "labeller").replace(/\s+/g, " ").trim();
|
|
57
|
+
const byTerm = new Map<string, GlossaryCand>();
|
|
58
|
+
for (const r of ranked) if (r.score > 0) {
|
|
59
|
+
const k = norm(r.term);
|
|
60
|
+
if (!byTerm.has(k)) byTerm.set(k, r);
|
|
61
|
+
}
|
|
62
|
+
c.glossary = [...byTerm.values()].sort((a, b) => b.score - a.score).slice(0, 3);
|
|
63
|
+
if (c.glossary.length) console.log("glossary link:", c.glossary.map((g2) => g2.term).join(", "));
|
|
64
|
+
},
|
|
65
|
+
};
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
// Graph lane (additive): the caller-resolved document numbers (relaton
|
|
2
|
+
// family/successor resolution done in index.ts) merge their top
|
|
3
|
+
// candidates at a discount.
|
|
4
|
+
import { THRESHOLDS } from "../config.ts";
|
|
5
|
+
import type { Stage } from "./types.ts";
|
|
6
|
+
|
|
7
|
+
export const graphLane: Stage = {
|
|
8
|
+
name: "graph-lane",
|
|
9
|
+
failure: "additive",
|
|
10
|
+
when: (c) => !!c.opts.graphDocNumbers?.length && c.vector.length > 0,
|
|
11
|
+
prefetch: (c) => {
|
|
12
|
+
c.lane["graph-lane"] = c.env.VECTORIZE.query(c.vector, {
|
|
13
|
+
topK: 15,
|
|
14
|
+
returnMetadata: "all",
|
|
15
|
+
filter: { doc_number: { $in: c.opts.graphDocNumbers } },
|
|
16
|
+
});
|
|
17
|
+
},
|
|
18
|
+
run: async (c) => {
|
|
19
|
+
const g = (await c.lane["graph-lane"]!) as any;
|
|
20
|
+
const seenIds = new Set(c.matches.map((m: any) => m.id));
|
|
21
|
+
let merged = 0;
|
|
22
|
+
for (const m of (g.matches ?? []).slice(0, 10)) {
|
|
23
|
+
if (!seenIds.has(m.id)) {
|
|
24
|
+
c.matches.push({ id: m.id, score: m.score * THRESHOLDS.graphLaneDiscount, metadata: m.metadata });
|
|
25
|
+
seenIds.add(m.id);
|
|
26
|
+
merged++;
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
console.log("graph lane:", g.matches?.length ?? 0, "hits,", merged, "merged");
|
|
30
|
+
},
|
|
31
|
+
};
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
// HyDE — Hypothetical Document Embeddings (additive): embed the
|
|
2
|
+
// hypothetical answer and search with it — its vocabulary matches the
|
|
3
|
+
// corpus better than the question's. Only for non-filtered queries (a
|
|
4
|
+
// filter would nullify the benefit).
|
|
5
|
+
// Ref: arXiv 2212.10496; arXiv 2507.16754 (adaptive HyDE)
|
|
6
|
+
import { embed } from "../ai.ts";
|
|
7
|
+
import { portModelRunner } from "../env.ts";
|
|
8
|
+
import { MODELS, THRESHOLDS } from "../config.ts";
|
|
9
|
+
import type { Stage } from "./types.ts";
|
|
10
|
+
|
|
11
|
+
export const hyde: Stage = {
|
|
12
|
+
name: "hyde",
|
|
13
|
+
failure: "additive",
|
|
14
|
+
when: (c) => !!c.u?.hypothetical_answer && !c.filter,
|
|
15
|
+
prefetch: (c) => {
|
|
16
|
+
c.lane.hyde = embed(portModelRunner(c.env), MODELS.embed, c.u!.hypothetical_answer!)
|
|
17
|
+
.then((hv) => c.env.VECTORIZE.query(hv, { topK: 20, returnMetadata: "all" }));
|
|
18
|
+
},
|
|
19
|
+
run: async (c) => {
|
|
20
|
+
const hres = (await c.lane.hyde!) as any;
|
|
21
|
+
const seenIds = new Set(c.matches.map((m: any) => m.id));
|
|
22
|
+
for (const m of (hres.matches ?? []).slice(0, 10)) {
|
|
23
|
+
if (!seenIds.has(m.id)) {
|
|
24
|
+
c.matches.push({ id: m.id, score: m.score * THRESHOLDS.hydeDiscount, metadata: m.metadata });
|
|
25
|
+
seenIds.add(m.id);
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
},
|
|
29
|
+
};
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
// The stage registry (TODO.impl/02): the pipeline is this ordered list.
|
|
2
|
+
// Adding a mechanism = writing a stage module and appending one entry —
|
|
3
|
+
// no edits to existing stages, the runner, or retrieve() (OCP). Ordering
|
|
4
|
+
// invariants and per-stage contracts: docs/spec-pipeline.md.
|
|
5
|
+
//
|
|
6
|
+
// Phase structure:
|
|
7
|
+
// candidate lanes (dense → glossary-graph lanes → fusion lanes)
|
|
8
|
+
// → pool open → pool-level merges (lexical/federate/seal)
|
|
9
|
+
// → refinement (demotion, rerank, steering, propagation, diversity)
|
|
10
|
+
// → window assembly (typed pin, section descent, dedup, floor)
|
|
11
|
+
import type { Stage } from "./types.ts";
|
|
12
|
+
export { runStages } from "./types.ts";
|
|
13
|
+
export type { PipelineContext, RetrieveOptions, GlossaryEntry, Stage } from "./types.ts";
|
|
14
|
+
import { dense } from "./dense.ts";
|
|
15
|
+
import { hyde } from "./hyde.ts";
|
|
16
|
+
import { glossary } from "./glossary.ts";
|
|
17
|
+
import { conceptGraph } from "./conceptGraph.ts";
|
|
18
|
+
import { graphLane } from "./graphLane.ts";
|
|
19
|
+
import { multiQuery } from "./multiQuery.ts";
|
|
20
|
+
import { subQuery } from "./subQuery.ts";
|
|
21
|
+
import { poolOpen } from "./poolOpen.ts";
|
|
22
|
+
import { lexicalUnion } from "./lexicalUnion.ts";
|
|
23
|
+
import { federate } from "./federate.ts";
|
|
24
|
+
import { seal } from "./seal.ts";
|
|
25
|
+
import { corpusScope } from "./corpusScope.ts";
|
|
26
|
+
import { editionCover } from "./editionCover.ts";
|
|
27
|
+
import { stdRefNudge } from "./stdRefNudge.ts";
|
|
28
|
+
import { overviewDemote } from "./overviewDemote.ts";
|
|
29
|
+
import { familyBoost } from "./familyBoost.ts";
|
|
30
|
+
import { rerankStage, lexicalRrf } from "./rerank.ts";
|
|
31
|
+
import { termNudge } from "./termNudge.ts";
|
|
32
|
+
import { conceptSteer } from "./conceptSteer.ts";
|
|
33
|
+
import { editionSteer } from "./editionSteer.ts";
|
|
34
|
+
import { propagate } from "./propagate.ts";
|
|
35
|
+
import { diversity } from "./diversity.ts";
|
|
36
|
+
import { typedPin } from "./typedPin.ts";
|
|
37
|
+
import { sectionDescent } from "./sectionDescent.ts";
|
|
38
|
+
import { dedup } from "./dedup.ts";
|
|
39
|
+
import { windowFloor } from "./windowFloor.ts";
|
|
40
|
+
|
|
41
|
+
export const STAGES: Stage[] = [
|
|
42
|
+
dense,
|
|
43
|
+
hyde,
|
|
44
|
+
glossary,
|
|
45
|
+
conceptGraph,
|
|
46
|
+
graphLane,
|
|
47
|
+
multiQuery,
|
|
48
|
+
subQuery,
|
|
49
|
+
poolOpen,
|
|
50
|
+
lexicalUnion,
|
|
51
|
+
federate,
|
|
52
|
+
seal,
|
|
53
|
+
overviewDemote,
|
|
54
|
+
familyBoost,
|
|
55
|
+
rerankStage,
|
|
56
|
+
lexicalRrf,
|
|
57
|
+
corpusScope,
|
|
58
|
+
editionCover,
|
|
59
|
+
stdRefNudge,
|
|
60
|
+
termNudge,
|
|
61
|
+
conceptSteer,
|
|
62
|
+
editionSteer,
|
|
63
|
+
propagate,
|
|
64
|
+
diversity,
|
|
65
|
+
typedPin,
|
|
66
|
+
sectionDescent,
|
|
67
|
+
dedup,
|
|
68
|
+
windowFloor,
|
|
69
|
+
];
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
// Lexical union: union full-corpus lexical hits that dense missed
|
|
2
|
+
// (G-ETSI-1). Prefer dense metadata/text when both sources return the
|
|
3
|
+
// same id.
|
|
4
|
+
import type { Stage } from "./types.ts";
|
|
5
|
+
|
|
6
|
+
export const lexicalUnion: Stage = {
|
|
7
|
+
name: "lexical-union",
|
|
8
|
+
when: (c) => c.lexicalHits.length > 0,
|
|
9
|
+
run: (c) => {
|
|
10
|
+
const seen = new Set(c.hits.map((h) => h.id));
|
|
11
|
+
let added = 0;
|
|
12
|
+
for (const h of c.lexicalHits) {
|
|
13
|
+
if (!seen.has(h.id)) {
|
|
14
|
+
c.hits.push(h);
|
|
15
|
+
seen.add(h.id);
|
|
16
|
+
added++;
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
if (added) console.log("lexical union:", added, "new candidates");
|
|
20
|
+
},
|
|
21
|
+
};
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
// Multi-Query RAG-Fusion: generate 2-3 alternative phrasings, retrieve
|
|
2
|
+
// for each, fuse via RRF. Different phrasings surface documents the
|
|
3
|
+
// original query misses.
|
|
4
|
+
// Ref: RAG-Fusion paper (Semantic Scholar b4d1da74); dev.to 2026 blueprint
|
|
5
|
+
import { embed } from "../ai.ts";
|
|
6
|
+
import { LIMITS, MODELS } from "../config.ts";
|
|
7
|
+
import { toHits, type Stage } from "./types.ts";
|
|
8
|
+
import type { Hit } from "../../../shared/chunk.ts";
|
|
9
|
+
|
|
10
|
+
const RRF_K = 60;
|
|
11
|
+
|
|
12
|
+
export const multiQuery: Stage = {
|
|
13
|
+
name: "multi-query",
|
|
14
|
+
when: (c) => !!c.u?.query_variants?.length,
|
|
15
|
+
prefetch: (c) => {
|
|
16
|
+
const { env, filter, u } = c;
|
|
17
|
+
// the variants are independent queries — embed + search them in
|
|
18
|
+
// PARALLEL (and ahead of the dense lane, via prefetch); a serial
|
|
19
|
+
// loop paid 2 x (embed + query) round trips on the hot path for
|
|
20
|
+
// zero quality difference (same candidate set)
|
|
21
|
+
c.lane["multi-query"] = Promise.all(
|
|
22
|
+
u!.query_variants!.slice(0, 3).map(async (variant) => {
|
|
23
|
+
try {
|
|
24
|
+
const vv = await embed(env.AI, MODELS.embed, variant);
|
|
25
|
+
const vres = await env.VECTORIZE.query(vv, { topK: 20, returnMetadata: "all", ...(filter ? { filter } : {}) });
|
|
26
|
+
return toHits(vres.matches ?? []);
|
|
27
|
+
} catch {
|
|
28
|
+
return [] as never[]; // variant retrieval failure — primary results stand
|
|
29
|
+
}
|
|
30
|
+
}),
|
|
31
|
+
);
|
|
32
|
+
},
|
|
33
|
+
run: async (c) => {
|
|
34
|
+
const variantResults = ((await c.lane["multi-query"]!) as Hit[][]).filter((r) => r.length > 0);
|
|
35
|
+
// RRF fuse: primary ranking + each variant ranking
|
|
36
|
+
if (variantResults.length > 0) {
|
|
37
|
+
const allRankings = [toHits(c.matches), ...variantResults];
|
|
38
|
+
// simple RRF across all rankings
|
|
39
|
+
const scores = new Map<string, number>();
|
|
40
|
+
const byId = new Map<string, (typeof allRankings)[number][number]>();
|
|
41
|
+
allRankings.forEach((ranking) => {
|
|
42
|
+
ranking.forEach((h, i) => {
|
|
43
|
+
scores.set(h.id, (scores.get(h.id) ?? 0) + 1 / (RRF_K + i + 1));
|
|
44
|
+
byId.set(h.id, h);
|
|
45
|
+
});
|
|
46
|
+
});
|
|
47
|
+
const fused = [...scores.entries()]
|
|
48
|
+
.sort((a, b) => b[1] - a[1])
|
|
49
|
+
.slice(0, LIMITS.retrieveK)
|
|
50
|
+
.map(([id]) => byId.get(id)!)
|
|
51
|
+
.filter(Boolean);
|
|
52
|
+
if (fused.length > 0) {
|
|
53
|
+
c.matches = fused.map((h) => ({ id: h.id, score: h.score, metadata: h.metadata }));
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
},
|
|
57
|
+
};
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
// Overview demotion: overview chunks repeat the title/doctype boilerplate
|
|
2
|
+
// and embed strongly for name-like queries, crowding clause chunks out of
|
|
3
|
+
// the rerank window.
|
|
4
|
+
import { THRESHOLDS } from "../config.ts";
|
|
5
|
+
import type { Stage } from "./types.ts";
|
|
6
|
+
|
|
7
|
+
export const overviewDemote: Stage = {
|
|
8
|
+
name: "overview-demote",
|
|
9
|
+
run: (c) => {
|
|
10
|
+
for (const h of c.hits) {
|
|
11
|
+
if (h.metadata.clause_anchor === "overview") h.score *= THRESHOLDS.overviewDemotion;
|
|
12
|
+
}
|
|
13
|
+
},
|
|
14
|
+
};
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
// Pool open: the candidate pool closes its lane-merge phase and becomes
|
|
2
|
+
// the ranked Hit list every refinement stage operates on.
|
|
3
|
+
import { toHits, type Stage } from "./types.ts";
|
|
4
|
+
|
|
5
|
+
export const poolOpen: Stage = {
|
|
6
|
+
name: "pool-open",
|
|
7
|
+
run: (c) => {
|
|
8
|
+
c.hits = toHits(c.matches);
|
|
9
|
+
},
|
|
10
|
+
};
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
// Structural propagation (FABLE TreeExpansion, arXiv:2601.18116): the
|
|
2
|
+
// corpus IS a tree — clause anchors chain parent→child, so a hit's
|
|
3
|
+
// score blends with its ancestors' (topic continuity) and descendants'
|
|
4
|
+
// (subtopic heat) — a section whose clauses are collectively hot rises,
|
|
5
|
+
// and a hot section lifts its clauses. Pure post-retrieval re-scoring
|
|
6
|
+
// over metadata the chunks already carry; no new index lane required.
|
|
7
|
+
import { structuralPropagation } from "../structural.ts";
|
|
8
|
+
import type { Stage } from "./types.ts";
|
|
9
|
+
|
|
10
|
+
export const propagate: Stage = {
|
|
11
|
+
name: "structural-propagate",
|
|
12
|
+
run: (c) => {
|
|
13
|
+
c.hits = structuralPropagation(c.hits);
|
|
14
|
+
},
|
|
15
|
+
};
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
// Cross-encoder rerank (additive — vector order is the fallback, by
|
|
2
|
+
// design) + the post-rerank family pin. The pre-rerank family boost does
|
|
3
|
+
// not survive re-sorting — short structural summaries always lose to
|
|
4
|
+
// content-heavy clauses under a cross-encoder. For doc-scoped queries
|
|
5
|
+
// pin family chunks AFTER rerank: 'what is R 60' must lead with the
|
|
6
|
+
// family summary, not an annex definition that happens to match the
|
|
7
|
+
// words.
|
|
8
|
+
//
|
|
9
|
+
// lexical-rrf: RRF with the FULL-CORPUS lexical ranking (not a re-score
|
|
10
|
+
// of the dense shortlist). ETSI §II-B7: dense+sparse fusion lifts
|
|
11
|
+
// precision and MRR on standards jargon without changing recall. Runs
|
|
12
|
+
// after rerank even when rerank failed (the monolith's try/catch left
|
|
13
|
+
// the fuse outside it — preserved by the runner's additive semantics).
|
|
14
|
+
import { rerank } from "../ai.ts";
|
|
15
|
+
import { portModelRunner } from "../env.ts";
|
|
16
|
+
import { LIMITS, MODELS } from "../config.ts";
|
|
17
|
+
import { rrfFuse } from "../hybrid.ts";
|
|
18
|
+
import type { Stage } from "./types.ts";
|
|
19
|
+
|
|
20
|
+
export const rerankStage: Stage = {
|
|
21
|
+
name: "rerank",
|
|
22
|
+
failure: "additive",
|
|
23
|
+
when: (c) => c.hits.length > 1,
|
|
24
|
+
run: async (c) => {
|
|
25
|
+
const tRerank = Date.now();
|
|
26
|
+
const scores = await rerank(portModelRunner(c.env), MODELS.rerank, c.query, c.hits.map((h) => h.text));
|
|
27
|
+
console.log("stage: rerank", Date.now() - tRerank, "ms over", c.hits.length, "candidates");
|
|
28
|
+
if (scores) {
|
|
29
|
+
c.hits.forEach((h, i) => (h.rerank_score = scores[i]));
|
|
30
|
+
c.hits.sort((a, b) => (b.rerank_score ?? -Infinity) - (a.rerank_score ?? -Infinity));
|
|
31
|
+
if (c.filter?.doc_number) {
|
|
32
|
+
const families = c.hits.filter((h) => h.metadata.clause_anchor === "family");
|
|
33
|
+
if (families.length) {
|
|
34
|
+
c.hits = [...families, ...c.hits.filter((h) => h.metadata.clause_anchor !== "family")];
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
},
|
|
39
|
+
};
|
|
40
|
+
|
|
41
|
+
export const lexicalRrf: Stage = {
|
|
42
|
+
name: "lexical-rrf",
|
|
43
|
+
when: (c) => c.hits.length > 1 && c.lexicalHits.length > 0,
|
|
44
|
+
run: (c) => {
|
|
45
|
+
c.hits = rrfFuse(c.hits, c.lexicalHits, LIMITS.retrieveK);
|
|
46
|
+
},
|
|
47
|
+
};
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
// The declared context's hard seal (TODO.ai-platform/02) — pool-level,
|
|
2
|
+
// after every lane has merged, before rerank + the top-N cut. Nothing
|
|
3
|
+
// outside the declared family competes for the window; everything
|
|
4
|
+
// inside it does.
|
|
5
|
+
import type { Stage } from "./types.ts";
|
|
6
|
+
|
|
7
|
+
export const seal: Stage = {
|
|
8
|
+
name: "seal",
|
|
9
|
+
when: (c) => !!c.opts.sealScope,
|
|
10
|
+
run: (c) => {
|
|
11
|
+
const before = c.hits.length;
|
|
12
|
+
const scope = c.opts.sealScope!;
|
|
13
|
+
c.hits = c.hits.filter((h) => h.metadata.doc_number === scope.doc_number && (!scope.edition || h.metadata.edition === scope.edition));
|
|
14
|
+
console.log("context seal:", before, "→", c.hits.length, "candidates within", `doc#${scope.doc_number}${scope.edition ? "@" + scope.edition : ""}`);
|
|
15
|
+
},
|
|
16
|
+
};
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
// Section-summary units → leaf evidence (FABLE multi-granularity): a
|
|
2
|
+
// depth-1 summary vector that ranked is a navigation node, not quotable
|
|
3
|
+
// evidence: fetch its top child clauses (metadata-filtered, same query
|
|
4
|
+
// vector) so the model gets source text to cite, then retire the
|
|
5
|
+
// synthetic summary — the answer contract grounds claims in source
|
|
6
|
+
// clauses, never in our own summaries. The summary stays only when no
|
|
7
|
+
// child answered (it is then the doc's sole representative).
|
|
8
|
+
import { THRESHOLDS } from "../config.ts";
|
|
9
|
+
import type { ChunkMeta } from "../../../shared/chunk";
|
|
10
|
+
import type { Hit } from "../../../shared/chunk";
|
|
11
|
+
import type { Stage } from "./types.ts";
|
|
12
|
+
|
|
13
|
+
export const sectionDescent: Stage = {
|
|
14
|
+
name: "section-descent",
|
|
15
|
+
failure: "additive",
|
|
16
|
+
when: (c) =>
|
|
17
|
+
!!c.finalHits.find((h) => h.metadata.section_summary === "1" && h.metadata.child_anchors && h.score > 0) &&
|
|
18
|
+
c.vector.length > 0,
|
|
19
|
+
run: async (c) => {
|
|
20
|
+
const sectionHit = c.finalHits.find(
|
|
21
|
+
(h) => h.metadata.section_summary === "1" && h.metadata.child_anchors && h.score > 0,
|
|
22
|
+
)!;
|
|
23
|
+
const kids = sectionHit
|
|
24
|
+
.metadata.child_anchors!.split(",")
|
|
25
|
+
.map((s) => s.trim())
|
|
26
|
+
.filter(Boolean)
|
|
27
|
+
.slice(0, 25);
|
|
28
|
+
if (kids.length) {
|
|
29
|
+
const cv = await c.env.VECTORIZE.query(c.vector, {
|
|
30
|
+
topK: 3,
|
|
31
|
+
returnMetadata: "all",
|
|
32
|
+
filter: {
|
|
33
|
+
$and: [
|
|
34
|
+
{ doc_id: { $eq: sectionHit.metadata.doc_id } },
|
|
35
|
+
{ clause_anchor: { $in: kids } },
|
|
36
|
+
],
|
|
37
|
+
},
|
|
38
|
+
});
|
|
39
|
+
const childHits = (cv.matches ?? [])
|
|
40
|
+
.filter((m: any) => !m.metadata?.section_summary)
|
|
41
|
+
.map((m: any) => ({
|
|
42
|
+
id: m.id,
|
|
43
|
+
score: m.score * THRESHOLDS.sectionDescentDiscount,
|
|
44
|
+
metadata: m.metadata as ChunkMeta,
|
|
45
|
+
text: (m.metadata?.chunk_text as string) ?? "",
|
|
46
|
+
}))
|
|
47
|
+
.filter((x: Hit) => !c.finalHits.some((h) => h.id === x.id))
|
|
48
|
+
.slice(0, 2);
|
|
49
|
+
if (childHits.length) {
|
|
50
|
+
c.finalHits = [...c.finalHits.filter((h) => h !== sectionHit), ...childHits];
|
|
51
|
+
console.log(
|
|
52
|
+
"section descent:",
|
|
53
|
+
sectionHit.metadata.docidentifier,
|
|
54
|
+
"§" + sectionHit.metadata.clause_anchor,
|
|
55
|
+
"→",
|
|
56
|
+
childHits.map((x: Hit) => "§" + x.metadata.clause_anchor).join(", "),
|
|
57
|
+
);
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
},
|
|
61
|
+
};
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
// Standard-reference nudge (L5): when the question asks which ISO/IEC
|
|
2
|
+
// standard a document invokes, the answer IS the clause carrying the
|
|
3
|
+
// citation — but generic family prose (definitions, certification
|
|
4
|
+
// passages) otherwise fills the window and the citing clause never
|
|
5
|
+
// reaches the answer (measured: l5a-iso-humidity flips ~1/6 runs).
|
|
6
|
+
// Lexical ranking signal, same design as term-nudge: query mentions the
|
|
7
|
+
// standard family, chunks whose text carries an ISO/IEC identifier get
|
|
8
|
+
// a spread-scaled boost. Typed table units (serialized payloads) compete
|
|
9
|
+
// on the same signal.
|
|
10
|
+
import { THRESHOLDS } from "../config.ts";
|
|
11
|
+
import type { Stage } from "./types.ts";
|
|
12
|
+
|
|
13
|
+
const ASKS_ABOUT_STD = /\b(iso|iec|astm|en\s?\d{2,5})\b/i;
|
|
14
|
+
const CITES_STD = /\b(?:ISO|IEC|ASTM|EN)[ /]?\d{3,6}(?:[-–]\d+)?\b/;
|
|
15
|
+
|
|
16
|
+
export const stdRefNudge: Stage = {
|
|
17
|
+
name: "std-ref-nudge",
|
|
18
|
+
when: (c) => ASKS_ABOUT_STD.test(c.query) && c.hits.length > 1,
|
|
19
|
+
run: (c) => {
|
|
20
|
+
const scored = c.hits.map((h) => h.rerank_score ?? h.score);
|
|
21
|
+
const spread = Math.max(...scored) - Math.min(...scored);
|
|
22
|
+
if (spread <= 0) return;
|
|
23
|
+
let nudged = 0;
|
|
24
|
+
for (const h of c.hits) {
|
|
25
|
+
if (CITES_STD.test(h.text) || CITES_STD.test(h.metadata.clause_title ?? "")) {
|
|
26
|
+
h.rerank_score = (h.rerank_score ?? h.score) + spread * THRESHOLDS.stdRefNudgeSpread;
|
|
27
|
+
nudged++;
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
if (nudged) {
|
|
31
|
+
c.hits.sort((a, b) => (b.rerank_score ?? -Infinity) - (a.rerank_score ?? -Infinity));
|
|
32
|
+
console.log("std-ref nudge:", nudged, "chunks carrying standard citations");
|
|
33
|
+
}
|
|
34
|
+
},
|
|
35
|
+
};
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
// Multi-hop decomposition for complex questions (additive per sub-query):
|
|
2
|
+
// retrieve for each sub-question and merge the top results.
|
|
3
|
+
// Ref: Agent-Orchestrated Adaptive RAG (arXiv 2606.05658)
|
|
4
|
+
import { embed } from "../ai.ts";
|
|
5
|
+
import { MODELS, THRESHOLDS } from "../config.ts";
|
|
6
|
+
import { toHits, type Stage } from "./types.ts";
|
|
7
|
+
import type { Hit } from "../../../shared/chunk.ts";
|
|
8
|
+
|
|
9
|
+
export const subQuery: Stage = {
|
|
10
|
+
name: "sub-query",
|
|
11
|
+
when: (c) => c.u?.complexity === "complex" && !!c.u?.sub_queries?.length,
|
|
12
|
+
prefetch: (c) => {
|
|
13
|
+
const { env, u } = c;
|
|
14
|
+
// sub-questions are independent — parallel rounds, ahead of the
|
|
15
|
+
// dense lane (prefetch), same as variants
|
|
16
|
+
c.lane["sub-query"] = Promise.all(
|
|
17
|
+
u!.sub_queries!.slice(0, 4).map(async (sub) => {
|
|
18
|
+
try {
|
|
19
|
+
const sv = await embed(env.AI, MODELS.embed, sub);
|
|
20
|
+
const sres = await env.VECTORIZE.query(sv, { topK: 15, returnMetadata: "all" });
|
|
21
|
+
return toHits(sres.matches ?? []);
|
|
22
|
+
} catch {
|
|
23
|
+
return [] as never[]; // sub-query failure — primary results stand
|
|
24
|
+
}
|
|
25
|
+
}),
|
|
26
|
+
);
|
|
27
|
+
},
|
|
28
|
+
run: async (c) => {
|
|
29
|
+
const subResults = ((await c.lane["sub-query"]!) as Hit[][]).filter((r) => r.length > 0);
|
|
30
|
+
// merge sub-results into the candidate pool (union, no RRF — these
|
|
31
|
+
// are complementary perspectives, not alternatives)
|
|
32
|
+
const seenIds = new Set(c.matches.map((m: any) => m.id));
|
|
33
|
+
for (const sr of subResults) {
|
|
34
|
+
for (const h of sr.slice(0, 8)) {
|
|
35
|
+
if (!seenIds.has(h.id)) {
|
|
36
|
+
c.matches.push({ id: h.id, score: h.score * THRESHOLDS.subQueryDiscount, metadata: h.metadata });
|
|
37
|
+
seenIds.add(h.id);
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
},
|
|
42
|
+
};
|