ko-qmd 2.8.3-ko.10 → 2.8.3-ko.12

package/CHANGELOG.md

@@ -2,6 +2,111 @@
 
 ## [Unreleased]
 
+## [2.8.3-ko.12] - 2026-11-14
+
+- `qmd mcp --http`: fix a memory leak in long-running daemons. The HTTP
+  bridge buffered every `/mcp` response to the end, so a
+  `subscriptions/listen` stream (which stays open) was never sent and was
+  never released when its client left. Every dead listener kept collecting
+  15-second keep-alive frames: one daemon reached 9.2 GB after 6 days and
+  hit the SDK's 1024-subscription cap, after which new listens were
+  refused. Responses now stream to the socket, and a client disconnect
+  closes the SDK stream (`/health` `listens` drops back). Under Bun 1.3 the
+  HTTP server gets no signal when a client hangs up on a POST, so a daemon
+  run with Bun still keeps dead listeners; run the daemon with Node.
+- Rerank input: a note that opens with an empty frontmatter block (`---`
+  twice) keeps its lead. The block used to run on to the next `---` rule in
+  the body, and when that span passed 350 characters the reranker saw only
+  the text after the rule. A `---` at the end of a frontmatter value no
+  longer closes the block either.
+- `--since` / `--until` (CLI, and `since` / `until` on the MCP `query` and
+  `grep` tools) reject a date the calendar does not have, such as
+  `2026-02-31` or a swapped `2026-31-01`, with `not a date`. They used to
+  roll it into a later month and search from there.
+- `qmd grep` (and the MCP `grep` tool) no longer says it stopped at the
+  limit when the limit-th match was the last document read; `--json`
+  `truncated` is false then.
+- Relevance judge: a note that cannot be read (renamed or deleted after the
+  bench) now fails only its query, recorded under that pass's `errors`.
+  It used to abort `scripts/judge/run.ts` before any judgments were written.
+- CI: `docs/vault-search/WINDOWS.md` lists the Windows test failures of
+  develop `f2694b1` (21 files; `test/metadata-search.test.ts` passes now).
+- `qmd grep` (and the MCP `grep` tool) matches CRLF notes like LF notes:
+  an end-anchored pattern such as `끝$` now matches, and the returned line
+  text no longer ends in a carriage return. Line numbers are unchanged.
+- `qmd update` and SDK `update()` no longer empty a collection whose root
+  directory is missing (a macOS volume that is not mounted, a moved or
+  renamed folder). The scan saw zero files and deactivated every document;
+  the root is now reported as skipped (`⚠ Skipped unreadable file: <root>
+  (ENOENT)`) and the index is left as it was. An empty mount point that is
+  still a directory is not caught. To drop a collection on purpose, use
+  `qmd collection remove`.
+- Relevance judge: `scripts/judge/pool.ts` reads CRLF notes (a Windows
+  checkout or a Windows-authored vault) and a leading BOM like LF notes. The
+  frontmatter was kept in the judged text and the aliases were lost, which
+  also failed `test/judge-run.test.ts` on the Windows CI job (run
+  41372458709, hidden by `continue-on-error`).
+- RAG eval: a `narrow` answer prompt (`RAG_PROMPT=narrow`) answers questions
+  that name a note's topic in other words, and abstains only when the asked
+  fact is in no chunk. On the 44 ko-vault questions (M3 Max, gemma3:4b, one
+  run, reference grades by Claude Opus 5.5, not people) it refuses 8 of 32
+  answerable questions against grounded's 12. It keeps 8 of 12 unanswerable
+  refusals. Faithfulness against baseline is +0.190 [0.024, 0.357]. The
+  faithfulness judge no longer sees inline "[n]" markers in the sentence it
+  grades, and `rag-ko.sh` keeps the reference grades of variants it does not
+  answer again. See `docs/eval/README.md`.
+- RAG answers with chunk citations and a faithfulness evaluation (eval stage
+  3). `scripts/rag/ask.ts` answers from a qmd index with `[n]` citations, or
+  says it does not know. `scripts/rag-ko.sh` measures faithfulness, citation
+  precision and abstention on 44 ko-vault questions, 12 of them unanswerable.
+  `gemma3:4b` generates and `qwen3:8b` judges. A stricter prompt raised
+  abstention on unanswerable questions from 0.333 to 0.667, but it also
+  refused 12 of 32 answerable ones. Its faithfulness change, +0.097
+  [-0.125, 0.333] on reference grades, is not distinguishable from zero. The
+  judge agreed with Claude Opus 5.5 reference grades (model, not human) at
+  kappa 0.35 only, so faithfulness is reported on the reference grades. See
+  `docs/eval/README.md`.
+- LLM-as-judge relevance grading for the ko-vault fixture (`scripts/judge-ko.sh`,
+  eval stage 2). A local Ollama model (`qwen3:8b`) grades each query's pool 0–3
+  on `test/fixtures/ko-vault/judge/rubric.md`. The pool is the union of the
+  bm25/hybrid/full top 10 plus missed answers. The report gives nDCG@10 per
+  backend with bootstrap intervals, a position-bias check and kappa against
+  labels sampled by `scripts/judge/label.ts`. Listwise grading changed 41% of
+  grades when only the order changed, so `JUDGE_MODE=pointwise` grades one
+  document at a time. Pointwise nDCG@10 on 50 goldset queries (M3 Max): bm25
+  0.726, hybrid 0.868, full 0.891. Against 250 reference labels by Claude
+  Opus 5.5 (model labels, not human), the pointwise judge's weighted kappa is
+  0.708 [0.646, 0.764]; it is lenient on same-field near misses. See
+  `docs/eval/README.md`.
+- CI gains a lex (BM25) evaluation gate, the `lex-gate` job. It runs
+  `scripts/bench-lex-gate.sh`, which benches bm25 over the ko-vault goldset five
+  times on fresh indexes with no model. It compares per-query recall@5 and
+  reciprocal rank with `test/fixtures/ko-vault/lex-baseline.json` by a seeded
+  paired bootstrap, and fails only when a 95% interval lies below zero. Baseline
+  (M3 Max, 5 reps): r5 0.8875, MRR 0.8169, no run-to-run difference. See
+  `docs/eval/README.md`.
+- `qmd bench --backend <name>` runs a subset of the bm25, vector, hybrid and
+  full backends (repeatable or a comma list). `--backend bm25` needs no
+  embedding or reranker model.
+
+## [2.8.3-ko.11] - 2026-10-31
+
+- Rerank now caps each document at 256 tokens by default
+  (`DEFAULT_RERANK_MAX_DOC_TOKENS` in `src/llm.ts`); `QMD_RERANK_MAX_DOC_TOKENS=0`
+  restores whole chunks, and an unset or invalid value takes the default. Replaying
+  29 rerank queries from the daemon query log with the rerank cache cleared
+  (M3 Max): query p50 5323 → 1682 ms, p90 8508 → 2668 ms, rerank stage p50
+  4968 → 1343 ms. ko-vault seam bench: `full_mrr` 0.9440 → 0.9451, hard line
+  unchanged (`full_r1`=0.6630). The cap is part of the rerank cache key, so
+  uncapped cache rows miss once after upgrading.
+- Release: the publish workflow runs the package smoke test before
+  `npm publish`, so a tarball that fails to install or start never ships.
+- CI: the Windows git-install job installs the pushed ref instead of `main`;
+  `docs/vault-search/WINDOWS.md` records the accepted Windows test failures.
+- Deps: zod 4.6.7, oxlint 1.88.0, actions/checkout and setup-node v7.
+- Docs: REFERENCE documents the memory MCP tools and `grep`, and corrects the
+  `multi_get` `maxBytes` default to 65536.
+
 ## [2.8.3-ko.10] - 2026-10-15
 
 ko-qmd 2.8.3-ko.10 adds a shared working memory for agents: three MCP tools

package/README.md

@@ -79,8 +79,9 @@ flowchart LR
   그 경로의 `score` 는 1/순위라 임계값을 걸 수 없기 때문이다.
 - **모델 상주** — `QMD_LLM_IDLE_TIMEOUT_MS=0` 이면 모델을 내리지 않는다. 기본은 5분 idle 뒤 내린다.
 - **리랭크 입력 cap** — 리랭커에 보내는 문서당 토큰을 자른다. 데몬 전체는 `QMD_RERANK_MAX_DOC_TOKENS`,
-  요청 하나는 REST `rerankMaxDocTokens` 로 건다(요청 값이 우선). KB 골드셋에서 128토큰 cap 은 품질을 지키면서
-  콜드 p90 을 3.6s → 0.9s 로 줄였다(Apple M3 Max, 후보 15).
+  요청 하나는 REST `rerankMaxDocTokens` 로 건다(요청 값이 우선). 기본은 256토큰이고 `QMD_RERANK_MAX_DOC_TOKENS=0` 이면
+  자르지 않는다. 요청 값은 양의 정수만 받으므로 요청 하나에서 cap 을 풀려면 큰 값(예: 100000)을 보낸다. 데몬 쿼리 29건 재생에서 256 cap 은 쿼리 p50 을 5.3s → 1.7s 로 줄였고 ko-vault bench 지표는 같거나
+  올랐다(Apple M3 Max, 리랭크 캐시 비움). KB 골드셋에서는 128토큰도 품질을 지켰다(콜드 p90 3.6s → 0.9s, 후보 15).
 
 ## 설치
 
@@ -171,7 +172,7 @@ API 전체: [docs/REFERENCE.md § SDK / Library Usage](docs/REFERENCE.md#sdk--li
 | `QMD_LLM_IDLE_TIMEOUT_MS` | 모델을 내리기까지 idle ms (`0` = 내리지 않음) |
 | `QMD_LLM_LOAD_TIMEOUT_MS` | 모델 로드 대기의 상한 ms (기본 120000, `0` = 무한 대기). 넘으면 그 단계 없이 답하고 로드는 백그라운드에서 계속된다 |
 | `QMD_RERANK_CASCADE` | `0`/`false`/`off`/`no` 면 리랭크 cascade를 끄고 후보 전체를 한 번에 채점한다 |
-| `QMD_RERANK_MAX_DOC_TOKENS` | 리랭커에 보내는 문서당 토큰 cap (요청의 `rerankMaxDocTokens` 가 우선) |
+| `QMD_RERANK_MAX_DOC_TOKENS` | 리랭커에 보내는 문서당 토큰 cap (기본 256, `0` = 자르지 않음, 요청의 `rerankMaxDocTokens` 가 우선) |
 | `QMD_RERANK_CONTEXT_SIZE` · `QMD_EMBED_CONTEXT_SIZE` · `QMD_EXPAND_CONTEXT_SIZE` | 역할별 context 크기 |
 | `QMD_EMBED_PARALLELISM` | 병렬 context 수 (높으면 RAM/VRAM 고갈) |
 | `QMD_LLAMA_GPU` · `QMD_FORCE_CPU` | GPU 백엔드 선택·끄기 |
@@ -201,7 +202,8 @@ npm run build                    # dist/ — `bun build --compile` 은 쓰지
 
 - 브랜치: `develop` 이 작업 브랜치(PR 은 여기로), `main` 은 배포 브랜치다. 릴리스는 `/release <version>`(→ `v*` 태그 → `publish.yml` 이 npm 발행).
   절차와 CHANGELOG 규칙: [skills/release/SKILL.md](skills/release/SKILL.md).
-- CI 는 업스트림 매트릭스에 `windows-latest` 와 Electron 스모크 잡을 더했다.
+- CI 는 업스트림 매트릭스에 `windows-latest` 와 Electron 스모크 잡을 더했다. `lex-gate` 잡은 모델 없이 ko-vault
+  goldset 으로 bm25 recall@5·MRR 저하를 막는다(방법·결과: [docs/eval/README.md](docs/eval/README.md)).
 
 ### dogfood — 작업 트리 빌드를 이 머신 데몬에 올리기
 
@@ -232,6 +234,29 @@ Qwen3-Embedding 기본값·질의 52 에서 bm25_r5 0.9519 · vector_r5 1.0000
 (exact 5·alias 4·semantic 3·topical 2·cross-domain 1, 저커버리지 토픽과 topics/ 입구 보강)과 hard 15건
 (sem-hard 5·multi 5·neg 5)을 더한 셋이다. hard 45 기준의 RESULT 참조선은 BASELINE.md 2026-10-10 절에 있다.
 
+### LLM 판정자 평가
+
+`bash scripts/judge-ko.sh`는 로컬 Ollama 모델(`qwen3:8b`)에게 검색 결과의 관련도를 0~3으로 매기게 한다. 그 등급으로
+정답 세트가 없는 질의에서도 nDCG@10을 잰다. 대상은 bm25·hybrid·full 상위 10의 합집합이고, 질의는 80개(정답 세트 50·정답 없음 30)다.
+판정자를 쓰기 전에 두 가지를 점검했다. 목록 하나를 통째로 채점하는 listwise 방식은 문서 순서만 섞어도 등급의 41%가 바뀌었다.
+그래서 문서를 하나씩 채점하는 pointwise 방식(`JUDGE_MODE=pointwise`)을 추가했다. pointwise는 정답 문서 59개 중 58개를 2 이상으로,
+off-corpus 문서 39개를 모두 0으로 채점했다.
+pointwise 기준 정답 세트 nDCG@10은 bm25 0.726·hybrid 0.868·full 0.891이고, full − bm25는 +0.164 [0.098, 0.234]다(M3 Max).
+판정자는 기준 라벨 250쌍과 weighted kappa 0.708 [0.646, 0.764]로 일치했다. 기준 라벨은 사람이 아니라 Claude Opus 5.5가
+판정자 등급을 보지 않고 매긴 것이다. 판정자는 관련 문서를 거의 놓치지 않지만 같은 분야의 다른 개념 문서에 높은 등급을 주는
+경향이 있다. 그래서 nDCG는 절대값보다 backend 간 비교로 읽는다. 방법과 수치는 [docs/eval/README.md](docs/eval/README.md)에 있다.
+
+### 인용 답변(RAG) 평가
+
+`npx tsx scripts/rag/ask.ts --index <index.sqlite> --config <index.yml> "질문"`은 검색한 청크 상위 5개로 답하고, 문장마다
+근거 청크 번호를 단다. 청크로 답할 수 없으면 "모른다"고 답한다. `bash scripts/rag-ko.sh`는 fixture 질문 44개(답 있음 32·
+주제는 있지만 사실이 없는 near-miss 8·무관 4)로 faithfulness·인용 정확도·"모른다" 비율을 잰다. 생성은 `gemma3:4b`, 판정은 `qwen3:8b`다.
+판정자는 Claude Opus 5.5 기준 등급(사람 아님)과 kappa 0.35로 일치해 믿기 어려웠다. 그래서 faithfulness는 기준 등급으로 보고한다.
+비교한 변경은 더 엄격한 프롬프트다. 이 프롬프트는 near-miss·무관 질문에서 "모른다"를 0.333에서 0.667로 올렸다.
+대신 답이 있는 질문 32개 중 12개도 거절했다. faithfulness 차이 +0.097 [−0.125, 0.333]은 0을 포함한다(M3 Max).
+거절 규칙만 좁힌 `narrow` 프롬프트(`ask.ts` 기본값)는 답 있는 질문 거절을 12개에서 8개로 줄이고, 답 없는 질문 거절은
+8/12로 유지했다. baseline 대비 faithfulness는 +0.190 [0.024, 0.357]이다. 방법과 수치는 [docs/eval/README.md](docs/eval/README.md)에 있다.
+
 ### Electron 내장
 
 앱은 SDK 를 워커(utilityProcess)에서 쓴다. CI `electron-smoke` 잡이 Electron 39(ABI 140)로 `better-sqlite3` 를 리빌드하고

package/dist/bench/bench.d.ts

@@ -14,6 +14,8 @@
  *   - full: Full hybrid pipeline with LLM reranking
  */
 import type { BackendSummaryByType, HardSummary, QueryResult, BenchmarkResult } from "./types.js";
+/** Backend names `runBenchmark` accepts in `options.backends`, in run order. */
+export declare const BENCH_BACKEND_NAMES: readonly string[];
 /** Per-type summary built from the existing computeSummary on each type's slice of results.
  *  A type with zero queries gets no entry — pooling must skip it, never divide by zero. */
 export declare const computeSummaryByType: (results: QueryResult[]) => BackendSummaryByType;

package/dist/bench/bench.js

@@ -1,12 +1,12 @@
-var v=Object.defineProperty;var i=(a,e)=>v(a,"name",{value:e,configurable:!0});import{readFileSync as x}from"node:fs";import{resolve as k}from"node:path";import{createStore as R,getDefaultDbPath as q}from"../index.js";import{scoreResults as S}from"./score.js";import{HARD_QUERY_TYPES as w}from"./types.js";function y(a){const e=a.split(`
-`).map((s,o)=>({trimmed:s.trim(),number:o+1})).filter(s=>s.trimmed.length>0);if(e.length===0)return;const t=/^(lex|vec|hyde):\s*/i,n=/^intent:\s*/i,r=[];let c;for(const s of e){if(n.test(s.trimmed)){if(c!==void 0)throw new Error(`Line ${s.number}: only one intent: line is allowed per benchmark query.`);if(c=s.trimmed.replace(n,"").trim(),!c)throw new Error(`Line ${s.number}: intent: must include text.`);continue}const o=s.trimmed.match(t);if(o){const f=o[1].toLowerCase(),m=s.trimmed.slice(o[0].length).trim();if(!m)throw new Error(`Line ${s.number} (${f}:) must include text.`);r.push({type:f,query:m,line:s.number});continue}if(e.length===1)return;throw new Error(`Line ${s.number} is missing a lex:/vec:/hyde:/intent: prefix.`)}if(c&&r.length===0)throw new Error("intent: cannot appear alone. Add at least one lex:, vec:, or hyde: line.");return r.length>0?{searches:r,intent:c}:void 0}i(y,"parseStructuredQuery");function b(a,e){const t=new Set,n=[];for(const r of a)if(!t.has(r)&&(t.add(r),n.push(r),n.length>=e))break;return n}i(b,"uniqueFiles");const g=[{name:"bm25",run:i(async(a,e,t,n)=>{const r=y(e.query),c=r?.searches.filter(o=>o.type==="lex");if(r){const o=[];for(const f of c??[]){const m=await a.searchLex(f.query,{limit:t,collection:n});o.push(...m.map(_=>_.filepath))}return b(o,t)}return(await a.searchLex(e.query,{limit:t,collection:n})).map(o=>o.filepath)},"run")},{name:"vector",run:i(async(a,e,t,n)=>{const r=y(e.query),c=r?.searches.filter(o=>o.type==="vec"||o.type==="hyde");if(r){const o=[];for(const f of c??[]){const m=await a.searchVector(f.query,{limit:t,collection:n});o.push(...m.map(_=>_.filepath))}return b(o,t)}return(await a.searchVector(e.query,{limit:t,collection:n})).map(o=>o.filepath)},"run")},{name:"hybrid",run:i(async(a,e,t,n)=>{const r=y(e.query);return(r?await a.search({queries:r.searches,intent:r.intent,limit:t,collection:n,rerank:!1}):await a.search({query:e.query,limit:t,collection:n,rerank:!1})).map(s=>s.file)},"run")},{name:"full",run:i(async(a,e,t,n)=>{const r=y(e.query);return(r?await a.search({queries:r.searches,intent:r.intent,limit:t,collection:n,rerank:!0}):await a.search({query:e.query,limit:t,collection:n,rerank:!0})).map(s=>s.file)},"run")}];async function E(a,e,t,n){const r=t.expected_in_top_k??t.expected_files.length,c=Math.max(Number.isFinite(r)?r:0,10),s=Date.now();let o;try{o=await e.run(a,t,c,n)}catch{return{precision_at_k:0,recall:0,recall_at_1:0,recall_at_3:0,recall_at_5:0,mrr:0,f1:0,hits_at_k:0,total_expected:t.expected_files.length,latency_ms:Date.now()-s,top_files:[],matched_files:[],unmatched_expected_files:t.expected_files}}const f=Date.now()-s;return{...S(o,t.expected_files,r),total_expected:t.expected_files.length,latency_ms:f,top_files:o.slice(0,10)}}i(E,"runQuery");function j(a){const e=[],t=i((r,c)=>r.slice(0,c).padEnd(c),"pad"),n=i(r=>r.toFixed(2).padStart(5),"num");e.push(`${t("Query",25)} ${t("Backend",8)} ${t("P@k",6)} ${t("R@1",6)} ${t("R@3",6)} ${t("R@5",6)} ${t("MRR",6)} ${t("F1",6)} ${t("ms",8)}`),e.push("-".repeat(88));for(const r of a){for(const[c,s]of Object.entries(r.backends))e.push(`${t(r.id,25)} ${t(c,8)} ${n(s.precision_at_k)} ${n(s.recall_at_1)} ${n(s.recall_at_3)} ${n(s.recall_at_5)} ${n(s.mrr)} ${n(s.f1)} ${String(Math.round(s.latency_ms)).padStart(7)}ms`);e.push("")}return e.join(`
-`)}i(j,"formatTable");function $(a){const e={},t=new Set;for(const n of a)for(const r of Object.keys(n.backends))t.add(r);for(const n of Array.from(t)){let r=0,c=0,s=0,o=0,f=0,m=0,_=0,p=0,u=0;for(const d of a){const l=d.backends[n];l&&(r+=l.precision_at_k,c+=l.recall,s+=l.recall_at_1,o+=l.recall_at_3,f+=l.recall_at_5,m+=l.mrr,_+=l.f1,p+=l.latency_ms,u++)}u>0&&(e[n]={avg_precision:r/u,avg_recall:c/u,avg_recall_at_1:s/u,avg_recall_at_3:o/u,avg_recall_at_5:f/u,avg_mrr:m/u,avg_f1:_/u,avg_latency_ms:p/u})}return e}i($,"computeSummary");const A=i(a=>{const e={},t=Array.from(new Set(a.map(n=>n.type))).sort();for(const n of t){const r=a.filter(c=>c.type===n);e[n]={};for(const[c,s]of Object.entries($(r)))e[n][c]={...s,count:r.length}}return e},"computeSummaryByType"),F=i(a=>{const e=[["hybrid_r1","hybrid","avg_recall_at_1"],["hybrid_mrr","hybrid","avg_mrr"],["full_r1","full","avg_recall_at_1"],["full_mrr","full","avg_mrr"]],t={hybrid_r1:null,hybrid_mrr:null,full_r1:null,full_mrr:null,n:0};for(const n of w)t.n+=Object.values(a[n]??{})[0]?.count??0;for(const[n,r,c]of e){let s=0,o=0;for(const f of w){const m=a[f]?.[r];m&&(s+=m[c]*m.count,o+=m.count)}t[n]=o>0?s/o:null}return t},"computeHardSummary");function O(a,e){if(e){const n=a.find(r=>r.name===e);if(!n){const r=a.map(s=>s.name),c=r.length>0?`Available: ${r.join(", ")}. Run 'qmd ls' to inspect.`:"Run 'qmd ls' to see available collections.";throw new Error(`Collection not found: ${e}
-${c}`)}if(n.active_count===0)throw new Error(`Collection '${e}' has no indexed documents.
-Run 'qmd update', then 'qmd ls ${e}' to confirm files are indexed before bench.`);return}const t=a.reduce((n,r)=>n+r.active_count,0);if(a.length===0||t===0)throw new Error(`No indexed documents found.
-Index a collection with 'qmd collection add' / 'qmd update' before running bench.`)}i(O,"assertBenchCollectionReady");function B(a){const e=Object.values(a);return e.length===0?!1:e.every(t=>t.avg_precision===0&&t.avg_recall===0&&t.avg_mrr===0)}i(B,"benchSummaryAllZero");function C(a){return`All benchmark scores were 0.00 \u2014 the collection is likely unindexed or the fixture's expected files are missing. Check with '${a?`qmd ls ${a}`:"qmd ls"}'.`}i(C,"allZeroBenchWarning");async function N(a,e={}){const t=x(k(a),"utf-8"),n=JSON.parse(t);if(!n.queries||!Array.isArray(n.queries))throw new Error("Invalid fixture: missing 'queries' array");if(n.queries.length===0)throw new Error(`Invalid fixture: '${a}' has no queries \u2014 nothing to run.
-Add at least one query; an empty fixture would otherwise report success with no results.`);const r=e.backends?g.filter(u=>e.backends.includes(u.name)):g;if(r.length===0){const u=(e.backends??[]).join(", ")||"(empty backend list)";throw new Error(`Unknown backend(s): ${u}.
-Available: ${g.map(d=>d.name).join(", ")}.`)}const c=await R({dbPath:e.dbPath??q(),...e.configPath?{configPath:e.configPath}:{},...e.config?{config:e.config}:{}}),s=e.collection??n.collection,o=[];try{const u=await c.listCollections();O(u,s);for(const d of n.queries){const l={};for(const h of r)e.json||process.stderr.write(`  ${d.id} / ${h.name}...`),l[h.name]=await E(c,h,d,s),e.json||process.stderr.write(` ${Math.round(l[h.name].latency_ms)}ms
-`);o.push({id:d.id,query:d.query,type:d.type,backends:l})}}finally{await c.close()}const f=$(o),m=new Date().toISOString().replace(/[:.]/g,"").slice(0,15),_=A(o),p={timestamp:m,fixture:a,results:o,summary:f,summary_by_type:_,summary_hard:F(_)};if(B(f)&&process.stderr.write(`
-${C(s)}
+var x=Object.defineProperty;var i=(a,e)=>x(a,"name",{value:e,configurable:!0});import{readFileSync as v}from"node:fs";import{resolve as k}from"node:path";import{createStore as R,getDefaultDbPath as q}from"../index.js";import{scoreResults as S}from"./score.js";import{HARD_QUERY_TYPES as w}from"./types.js";function y(a){const e=a.split(`
+`).map((s,o)=>({trimmed:s.trim(),number:o+1})).filter(s=>s.trimmed.length>0);if(e.length===0)return;const t=/^(lex|vec|hyde):\s*/i,r=/^intent:\s*/i,n=[];let c;for(const s of e){if(r.test(s.trimmed)){if(c!==void 0)throw new Error(`Line ${s.number}: only one intent: line is allowed per benchmark query.`);if(c=s.trimmed.replace(r,"").trim(),!c)throw new Error(`Line ${s.number}: intent: must include text.`);continue}const o=s.trimmed.match(t);if(o){const f=o[1].toLowerCase(),m=s.trimmed.slice(o[0].length).trim();if(!m)throw new Error(`Line ${s.number} (${f}:) must include text.`);n.push({type:f,query:m,line:s.number});continue}if(e.length===1)return;throw new Error(`Line ${s.number} is missing a lex:/vec:/hyde:/intent: prefix.`)}if(c&&n.length===0)throw new Error("intent: cannot appear alone. Add at least one lex:, vec:, or hyde: line.");return n.length>0?{searches:n,intent:c}:void 0}i(y,"parseStructuredQuery");function b(a,e){const t=new Set,r=[];for(const n of a)if(!t.has(n)&&(t.add(n),r.push(n),r.length>=e))break;return r}i(b,"uniqueFiles");const g=[{name:"bm25",run:i(async(a,e,t,r)=>{const n=y(e.query),c=n?.searches.filter(o=>o.type==="lex");if(n){const o=[];for(const f of c??[]){const m=await a.searchLex(f.query,{limit:t,collection:r});o.push(...m.map(h=>h.filepath))}return b(o,t)}return(await a.searchLex(e.query,{limit:t,collection:r})).map(o=>o.filepath)},"run")},{name:"vector",run:i(async(a,e,t,r)=>{const n=y(e.query),c=n?.searches.filter(o=>o.type==="vec"||o.type==="hyde");if(n){const o=[];for(const f of c??[]){const m=await a.searchVector(f.query,{limit:t,collection:r});o.push(...m.map(h=>h.filepath))}return b(o,t)}return(await a.searchVector(e.query,{limit:t,collection:r})).map(o=>o.filepath)},"run")},{name:"hybrid",run:i(async(a,e,t,r)=>{const n=y(e.query);return(n?await a.search({queries:n.searches,intent:n.intent,limit:t,collection:r,rerank:!1}):await a.search({query:e.query,limit:t,collection:r,rerank:!1})).map(s=>s.file)},"run")},{name:"full",run:i(async(a,e,t,r)=>{const n=y(e.query);return(n?await a.search({queries:n.searches,intent:n.intent,limit:t,collection:r,rerank:!0}):await a.search({query:e.query,limit:t,collection:r,rerank:!0})).map(s=>s.file)},"run")}],I=g.map(a=>a.name);async function E(a,e,t,r){const n=t.expected_in_top_k??t.expected_files.length,c=Math.max(Number.isFinite(n)?n:0,10),s=Date.now();let o;try{o=await e.run(a,t,c,r)}catch(h){return{error:h instanceof Error?h.message:String(h),precision_at_k:0,recall:0,recall_at_1:0,recall_at_3:0,recall_at_5:0,mrr:0,f1:0,hits_at_k:0,total_expected:t.expected_files.length,latency_ms:Date.now()-s,top_files:[],matched_files:[],unmatched_expected_files:t.expected_files}}const f=Date.now()-s;return{...S(o,t.expected_files,n),total_expected:t.expected_files.length,latency_ms:f,top_files:o.slice(0,10)}}i(E,"runQuery");function A(a){const e=[],t=i((n,c)=>n.slice(0,c).padEnd(c),"pad"),r=i(n=>n.toFixed(2).padStart(5),"num");e.push(`${t("Query",25)} ${t("Backend",8)} ${t("P@k",6)} ${t("R@1",6)} ${t("R@3",6)} ${t("R@5",6)} ${t("MRR",6)} ${t("F1",6)} ${t("ms",8)}`),e.push("-".repeat(88));for(const n of a){for(const[c,s]of Object.entries(n.backends))e.push(`${t(n.id,25)} ${t(c,8)} ${r(s.precision_at_k)} ${r(s.recall_at_1)} ${r(s.recall_at_3)} ${r(s.recall_at_5)} ${r(s.mrr)} ${r(s.f1)} ${String(Math.round(s.latency_ms)).padStart(7)}ms`);e.push("")}return e.join(`
+`)}i(A,"formatTable");function $(a){const e={},t=new Set;for(const r of a)for(const n of Object.keys(r.backends))t.add(n);for(const r of Array.from(t)){let n=0,c=0,s=0,o=0,f=0,m=0,h=0,p=0,u=0;for(const d of a){const l=d.backends[r];l&&(n+=l.precision_at_k,c+=l.recall,s+=l.recall_at_1,o+=l.recall_at_3,f+=l.recall_at_5,m+=l.mrr,h+=l.f1,p+=l.latency_ms,u++)}u>0&&(e[r]={avg_precision:n/u,avg_recall:c/u,avg_recall_at_1:s/u,avg_recall_at_3:o/u,avg_recall_at_5:f/u,avg_mrr:m/u,avg_f1:h/u,avg_latency_ms:p/u})}return e}i($,"computeSummary");const j=i(a=>{const e={},t=Array.from(new Set(a.map(r=>r.type))).sort();for(const r of t){const n=a.filter(c=>c.type===r);e[r]={};for(const[c,s]of Object.entries($(n)))e[r][c]={...s,count:n.length}}return e},"computeSummaryByType"),B=i(a=>{const e=[["hybrid_r1","hybrid","avg_recall_at_1"],["hybrid_mrr","hybrid","avg_mrr"],["full_r1","full","avg_recall_at_1"],["full_mrr","full","avg_mrr"]],t={hybrid_r1:null,hybrid_mrr:null,full_r1:null,full_mrr:null,n:0};for(const r of w)t.n+=Object.values(a[r]??{})[0]?.count??0;for(const[r,n,c]of e){let s=0,o=0;for(const f of w){const m=a[f]?.[n];m&&(s+=m[c]*m.count,o+=m.count)}t[r]=o>0?s/o:null}return t},"computeHardSummary");function C(a,e){if(e){const r=a.find(n=>n.name===e);if(!r){const n=a.map(s=>s.name),c=n.length>0?`Available: ${n.join(", ")}. Run 'qmd ls' to inspect.`:"Run 'qmd ls' to see available collections.";throw new Error(`Collection not found: ${e}
+${c}`)}if(r.active_count===0)throw new Error(`Collection '${e}' has no indexed documents.
+Run 'qmd update', then 'qmd ls ${e}' to confirm files are indexed before bench.`);return}const t=a.reduce((r,n)=>r+n.active_count,0);if(a.length===0||t===0)throw new Error(`No indexed documents found.
+Index a collection with 'qmd collection add' / 'qmd update' before running bench.`)}i(C,"assertBenchCollectionReady");function D(a){const e=Object.values(a);return e.length===0?!1:e.every(t=>t.avg_precision===0&&t.avg_recall===0&&t.avg_mrr===0)}i(D,"benchSummaryAllZero");function F(a){return`All benchmark scores were 0.00 \u2014 the collection is likely unindexed or the fixture's expected files are missing. Check with '${a?`qmd ls ${a}`:"qmd ls"}'.`}i(F,"allZeroBenchWarning");async function T(a,e={}){const t=v(k(a),"utf-8"),r=JSON.parse(t);if(!r.queries||!Array.isArray(r.queries))throw new Error("Invalid fixture: missing 'queries' array");if(r.queries.length===0)throw new Error(`Invalid fixture: '${a}' has no queries \u2014 nothing to run.
+Add at least one query; an empty fixture would otherwise report success with no results.`);const n=e.backends?g.filter(u=>e.backends.includes(u.name)):g;if(n.length===0){const u=(e.backends??[]).join(", ")||"(empty backend list)";throw new Error(`Unknown backend(s): ${u}.
+Available: ${g.map(d=>d.name).join(", ")}.`)}const c=await R({dbPath:e.dbPath??q(),...e.configPath?{configPath:e.configPath}:{},...e.config?{config:e.config}:{}}),s=e.collection??r.collection,o=[];try{const u=await c.listCollections();C(u,s);for(const d of r.queries){const l={};for(const _ of n)e.json||process.stderr.write(`  ${d.id} / ${_.name}...`),l[_.name]=await E(c,_,d,s),e.json||process.stderr.write(` ${Math.round(l[_.name].latency_ms)}ms
+`);o.push({id:d.id,query:d.query,type:d.type,backends:l})}}finally{await c.close()}const f=$(o),m=new Date().toISOString().replace(/[:.]/g,"").slice(0,15),h=j(o),p={timestamp:m,fixture:a,results:o,summary:f,summary_by_type:h,summary_hard:B(h)};if(D(f)&&process.stderr.write(`
+${F(s)}
 `),e.json)console.log(JSON.stringify(p,null,2));else{console.log(`
-`+j(o)),console.log("Summary:"),console.log("-".repeat(70));const u=i((l,h)=>l.slice(0,h).padEnd(h),"pad"),d=i(l=>l.toFixed(3).padStart(6),"num");for(const[l,h]of Object.entries(f))console.log(`  ${u(l,8)} P@k=${d(h.avg_precision)} R@1=${d(h.avg_recall_at_1)} R@3=${d(h.avg_recall_at_3)} R@5=${d(h.avg_recall_at_5)} MRR=${d(h.avg_mrr)} F1=${d(h.avg_f1)} Avg=${Math.round(h.avg_latency_ms)}ms`)}return p}i(N,"runBenchmark");export{C as allZeroBenchWarning,O as assertBenchCollectionReady,B as benchSummaryAllZero,F as computeHardSummary,A as computeSummaryByType,N as runBenchmark};
+`+A(o)),console.log("Summary:"),console.log("-".repeat(70));const u=i((l,_)=>l.slice(0,_).padEnd(_),"pad"),d=i(l=>l.toFixed(3).padStart(6),"num");for(const[l,_]of Object.entries(f))console.log(`  ${u(l,8)} P@k=${d(_.avg_precision)} R@1=${d(_.avg_recall_at_1)} R@3=${d(_.avg_recall_at_3)} R@5=${d(_.avg_recall_at_5)} MRR=${d(_.avg_mrr)} F1=${d(_.avg_f1)} Avg=${Math.round(_.avg_latency_ms)}ms`)}return p}i(T,"runBenchmark");export{I as BENCH_BACKEND_NAMES,F as allZeroBenchWarning,C as assertBenchCollectionReady,D as benchSummaryAllZero,B as computeHardSummary,j as computeSummaryByType,T as runBenchmark};

package/dist/bench/gate.d.ts

@@ -0,0 +1,92 @@
+/**
+ * Verdict logic for the lex CI gate (scripts/bench-lex-gate.sh).
+ *
+ * The gate benches bm25 over the ko-vault goldset and compares each query's
+ * recall@5 and reciprocal rank with a committed baseline. A paired bootstrap
+ * over the queries gives a 95% interval for the mean change of each metric:
+ *
+ *   FAIL — an interval lies wholly below zero (a drop the query sample cannot
+ *          explain), or the query set differs from the baseline's.
+ *   WARN — no such interval, but some query scores lower than its baseline.
+ *   PASS — no query scores lower.
+ *
+ * bm25 has no sampling step, so the interval measures query-set luck, not
+ * run-to-run noise. Run-to-run noise is measured separately by benching N
+ * fresh indexes and comparing them (noiseAcrossRuns).
+ *
+ * Pure functions only — the script owns I/O.
+ */
+import type { BenchmarkResult } from "./types.js";
+export interface QueryMetrics {
+    /** recall@5 */
+    r5: number;
+    /** reciprocal rank of the first relevant result */
+    rr: number;
+}
+/** Query id → metrics. */
+export type PerQuery = Record<string, QueryMetrics>;
+export interface BootstrapOptions {
+    iters?: number;
+    seed?: number;
+    /** Two-sided miss rate; 0.05 gives a 95% interval. */
+    alpha?: number;
+}
+export interface Interval {
+    mean: number;
+    lo: number;
+    hi: number;
+}
+export interface MetricVerdict {
+    baseline: number;
+    current: number;
+    /** current − baseline mean */
+    delta: number;
+    /** bootstrap interval of delta */
+    lo: number;
+    hi: number;
+}
+export interface DroppedQuery {
+    id: string;
+    r5: [baseline: number, current: number];
+    rr: [baseline: number, current: number];
+}
+/** The baseline and the current run cover different queries, so nothing is compared. */
+export interface StaleVerdict {
+    kind: "stale";
+    status: "FAIL";
+    reason: string;
+    /** baseline ids absent from the current run */
+    missing: string[];
+    /** current ids absent from the baseline */
+    added: string[];
+}
+export interface ComparedVerdict {
+    kind: "compared";
+    status: "PASS" | "WARN" | "FAIL";
+    reason: string;
+    r5: MetricVerdict;
+    mrr: MetricVerdict;
+    dropped: DroppedQuery[];
+    /** queries that score higher on some metric and lower on none */
+    improved: number;
+}
+export type GateVerdict = StaleVerdict | ComparedVerdict;
+export declare const DEFAULT_BOOTSTRAP: {
+    readonly iters: 10000;
+    readonly seed: 20261031;
+    readonly alpha: 0.05;
+};
+/** mulberry32 — small seeded PRNG so a CI rerun draws the same resamples. */
+export declare function mulberry32(seed: number): () => number;
+/** Percentile bootstrap interval of the mean of `deltas`, resampling queries with replacement. */
+export declare function pairedBootstrap(deltas: readonly number[], options?: BootstrapOptions): Interval;
+export declare function compareToBaseline(baseline: PerQuery, current: PerQuery, options?: BootstrapOptions): GateVerdict;
+/** Per-query recall@5 and reciprocal rank of one backend from a `qmd bench --json` result. */
+export declare function perQueryMetrics(bench: BenchmarkResult, backend: string): PerQuery;
+/** Queries whose metrics differ between repeated runs, and the largest such difference. */
+export declare function noiseAcrossRuns(runs: readonly PerQuery[]): {
+    differing: string[];
+    maxAbs: number;
+};
+/** Per-query mean over repeated runs — the gate's verdict input when runs disagree. */
+export declare function meanPerQuery(runs: readonly PerQuery[]): PerQuery;

package/dist/bench/gate.js

@@ -0,0 +1 @@
+var v=Object.defineProperty;var f=(e,t)=>v(e,"name",{value:t,configurable:!0});const j={iters:1e4,seed:20261031,alpha:.05},d=1e-9;function B(e){let t=e>>>0;return()=>{t=t+1831565813>>>0;let r=t;return r=Math.imul(r^r>>>15,r|1),r^=r+Math.imul(r^r>>>7,r|61),((r^r>>>14)>>>0)/4294967296}}f(B,"mulberry32");function b(e){let t=0;for(const r of e)t+=r;return t/e.length}f(b,"mean");function S(e,t={}){if(e.length===0)throw new Error("pairedBootstrap: no queries to resample");const{iters:r,seed:n,alpha:o}={...j,...t},c=B(n),a=e.length,i=new Float64Array(r);for(let l=0;l<r;l++){let u=0;for(let h=0;h<a;h++)u+=e[Math.floor(c()*a)];i[l]=u/a}return i.sort(),{mean:b(e),lo:i[Math.floor(o/2*(r-1))],hi:i[Math.ceil((1-o/2)*(r-1))]}}f(S,"pairedBootstrap");function g(e,t,r){for(const n of["r5","rr"]){const o=t?.[n];if(typeof o!="number"||!Number.isFinite(o))throw new Error(`${r}: query ${e} has no numeric ${n} (got ${JSON.stringify(o)})`)}return t}f(g,"assertMetrics");function q(e,t,r){const n=t.map((l,u)=>{const h=l-e[u];return Math.abs(h)<d?0:h}),{lo:o,hi:c}=S(n,r),a=b(e),i=b(t);return{baseline:a,current:i,delta:i-a,lo:o,hi:c}}f(q,"metricVerdict");function T(e,t,r={}){const n=Object.keys(e).sort(),o=n.filter(s=>!(s in t)),c=Object.keys(t).filter(s=>!(s in e)).sort();if(o.length>0||c.length>0)return{kind:"stale",status:"FAIL",reason:`baseline is stale: ${o.length} baseline queries missing, ${c.length} new \u2014 refresh it with --update-baseline`,missing:o,added:c};const a=n;if(a.length===0)throw new Error("compareToBaseline: the baseline has no queries");const i=a.map(s=>g(s,e[s],"baseline")),l=a.map(s=>g(s,t[s],"current")),u=q(i.map(s=>s.r5),l.map(s=>s.r5),r),h=q(i.map(s=>s.rr),l.map(s=>s.rr),r),w=[];let x=0;a.forEach((s,E)=>{const m=i[E],p=l[E],k=p.r5<m.r5-d||p.rr<m.rr-d,O=p.r5>m.r5+d||p.rr>m.rr+d;k?w.push({id:s,r5:[m.r5,p.r5],rr:[m.rr,p.rr]}):O&&x++});const M=[u.hi<0?"r5":"",h.hi<0?"mrr":""].filter(Boolean);let y,$;return M.length>0?(y="FAIL",$=`95% interval below zero for ${M.join(", ")}`):w.length>0?(y="WARN",$=`${w.length} queries dropped, but no interval lies below zero`):(y="PASS",$="no query dropped"),{kind:"compared",status:y,reason:$,r5:u,mrr:h,dropped:w,improved:x}}f(T,"compareToBaseline");function L(e,t){const r={};for(const n of e.results){const o=n.backends[t];if(!o)throw new Error(`bench result for query ${n.id} has no ${t} backend`);if(o.error!==void 0)throw new Error(`bench result for query ${n.id}: ${t} threw: ${o.error}`);r[n.id]=g(n.id,{r5:o.recall_at_5,rr:o.mrr},"bench")}return r}f(L,"perQueryMetrics");function A(e){if(e.length===0)throw new Error("no runs");const t=Object.keys(e[0]).sort();for(const r of e){const n=Object.keys(r).sort();if(n.length!==t.length||n.some((o,c)=>o!==t[c]))throw new Error("repeated runs must cover the same queries")}return t}f(A,"assertSameQueries");function N(e){const t=A(e),r=[];let n=0;for(const o of t){let c=!1;for(const a of["r5","rr"]){const i=e.map(u=>g(o,u[o],"run")[a]),l=Math.max(...i)-Math.min(...i);l>0&&(c=!0),n=Math.max(n,l)}c&&r.push(o)}return{differing:r,maxAbs:n}}f(N,"noiseAcrossRuns");function P(e){const t=A(e),r={};for(const n of t)r[n]={r5:b(e.map(o=>g(n,o[n],"run").r5)),rr:b(e.map(o=>o[n].rr))};return r}f(P,"meanPerQuery");export{j as DEFAULT_BOOTSTRAP,T as compareToBaseline,P as meanPerQuery,B as mulberry32,N as noiseAcrossRuns,S as pairedBootstrap,L as perQueryMetrics};

package/dist/bench/judge-metrics.d.ts

@@ -0,0 +1,33 @@
+/**
+ * Agreement and ranking metrics for LLM-as-judge relevance grades (0–3).
+ *
+ * Pure functions, no I/O. `scripts/judge/report.ts` feeds them the judge's grades, the
+ * hand labels and the bench rankings. Grades follow the rubric in
+ * test/fixtures/ko-vault/judge/rubric.md: 0 unrelated … 3 answers the query.
+ */
+import { type BootstrapOptions } from "./gate.js";
+export declare const GRADES = 4;
+/** Cohen's kappa: every disagreement costs the same. */
+export declare function cohenKappa(a: readonly number[], b: readonly number[]): number;
+/** Quadratic weighted kappa: a 1-grade miss costs 1/9 of a 3-grade miss. */
+export declare function weightedKappa(a: readonly number[], b: readonly number[]): number;
+/**
+ * nDCG@k with gain 2^g - 1. `ranked` is the grade at each rank of one system's list;
+ * `pool` is every judged grade for the query (all systems), whose best k form the ideal.
+ * NaN when the pool holds no relevant document — the query cannot tell systems apart.
+ */
+export declare function ndcgAt(ranked: readonly number[], pool: readonly number[], k?: number): number;
+/** Share of items whose grade differs between two judging passes. */
+export declare function flipRate(a: readonly number[], b: readonly number[]): number;
+export interface StatInterval {
+    value: number;
+    lo: number;
+    hi: number;
+    /** Resamples where the statistic was NaN (e.g. kappa on a one-category draw); left out of lo/hi. */
+    undefinedDraws: number;
+}
+/**
+ * Percentile bootstrap of any statistic over items resampled with replacement.
+ * Items are whole queries, so pairs from one query move together.
+ */
+export declare function bootstrapStatistic<T>(items: readonly T[], stat: (sample: T[]) => number, options?: BootstrapOptions): StatInterval;

package/dist/bench/judge-metrics.js

@@ -0,0 +1 @@
+var $=Object.defineProperty;var u=(t,e)=>$(t,"name",{value:e,configurable:!0});import{DEFAULT_BOOTSTRAP as x,mulberry32 as y}from"./gate.js";const g=4;function d(t,e,r){if(t.length!==e.length)throw new Error(`${r}: length mismatch (${t.length} vs ${e.length})`);if(t.length===0)throw new Error(`${r}: empty input`)}u(d,"assertPairs");function w(t,e){if(!Number.isInteger(t)||t<0||t>=g)throw new Error(`${e}: grade must be an integer 0-${g-1}, got ${t}`)}u(w,"assertGrade");function b(t,e,r,n){d(t,e,n);const c=g,a=Array.from({length:c},()=>new Array(c).fill(0));for(let o=0;o<t.length;o++)w(t[o],n),w(e[o],n),a[t[o]][e[o]]++;const m=a.map(o=>o.reduce((l,s)=>l+s,0)),h=a[0].map((o,l)=>a.reduce((s,p)=>s+p[l],0));let i=0,f=0;for(let o=0;o<c;o++)for(let l=0;l<c;l++){const s=r(o,l);i+=s*a[o][l],f+=s*(m[o]*h[l])/t.length}return f===0?i===0?Number.NaN:0:1-i/f}u(b,"kappa");function M(t,e){return b(t,e,(r,n)=>r===n?0:1,"cohenKappa")}u(M,"cohenKappa");function K(t,e){return b(t,e,(r,n)=>(r-n)**2/(g-1)**2,"weightedKappa")}u(K,"weightedKappa");function A(t,e){let r=0;for(let n=0;n<Math.min(e,t.length);n++)r+=(2**t[n]-1)/Math.log2(n+2);return r}u(A,"dcg");function R(t,e,r=10){const n=A([...e].sort((c,a)=>a-c),r);return n===0?Number.NaN:A(t,r)/n}u(R,"ndcgAt");function S(t,e){d(t,e,"flipRate");let r=0;for(let n=0;n<t.length;n++)t[n]!==e[n]&&r++;return r/t.length}u(S,"flipRate");function D(t,e,r={}){if(t.length===0)throw new Error("bootstrapStatistic: empty input");const{iters:n,seed:c,alpha:a}={...x,...r},m=y(c),h=t.length,i=[];let f=0;for(let l=0;l<n;l++){const s=new Array(h);for(let N=0;N<h;N++)s[N]=t[Math.floor(m()*h)];const p=e(s);Number.isNaN(p)?f++:i.push(p)}i.sort((l,s)=>l-s);const o=i.length-1;return{value:e([...t]),lo:i.length?i[Math.floor(a/2*o)]:Number.NaN,hi:i.length?i[Math.ceil((1-a/2)*o)]:Number.NaN,undefinedDraws:f}}u(D,"bootstrapStatistic");export{g as GRADES,D as bootstrapStatistic,M as cohenKappa,S as flipRate,R as ndcgAt,K as weightedKappa};

package/dist/bench/rag-metrics.d.ts

@@ -0,0 +1,82 @@
+/**
+ * Answer-level metrics for the RAG evaluation (eval stage 3). Pure functions — no I/O, no model.
+ *
+ * An answer is a list of sentences, each citing chunk numbers, or an abstention ("모른다").
+ * A judge grades every cited sentence: does the text of the cited chunks support it (full,
+ * partial, none), and does each cited chunk on its own support it. A sentence with no citation
+ * is never sent to the judge — it counts as unsupported and its missing citation is tallied.
+ */
+import { type BootstrapOptions, type Interval } from "./gate.js";
+import { type StatInterval } from "./judge-metrics.js";
+export type Support = "full" | "partial" | "none";
+export interface SentenceGrade {
+    support: Support;
+    /** Cited chunk number → whether that chunk alone supports the sentence. Empty when uncited. */
+    citations: Record<string, boolean>;
+    uncited: boolean;
+}
+export type QuestionType = "single" | "multi" | "near-miss" | "off-corpus";
+export interface QuestionOutcome {
+    id: string;
+    type: QuestionType;
+    /** True when the model said it cannot answer from the documents. */
+    abstained: boolean;
+    /** One grade per answer sentence; empty when abstained. */
+    sentences: SentenceGrade[];
+    /** Files the answer cited, as corpus paths. */
+    citedFiles: string[];
+    /** Files retrieved into the prompt, as corpus paths. */
+    contextFiles: string[];
+    expectedFiles: string[];
+}
+export declare const ANSWERABLE: ReadonlySet<QuestionType>;
+/**
+ * Share of this answer's sentences fully supported by their cited chunks. NaN when nothing was
+ * answered. Run-level faithfulness is the mean of this over answered questions (a macro average).
+ * It does not ask whether the answer is right: a near-miss answer built from true but irrelevant
+ * sentences scores high.
+ */
+export declare function faithfulness(o: QuestionOutcome): number;
+/** Share of citations whose chunk supports its sentence. NaN when the answer cites nothing. */
+export declare function citationPrecision(o: QuestionOutcome): number;
+/** 1 when an answered answerable question cites an expected file, 0 when not, NaN otherwise. */
+export declare function citesExpected(o: QuestionOutcome): number;
+/** 1 when an expected file reached the prompt, 0 when not, NaN for unanswerable questions. */
+export declare function contextRecall(o: QuestionOutcome): number;
+/** 1 when an unanswerable (near-miss, off-corpus) question was refused, 0 when answered, NaN otherwise. */
+export declare function abstainUnanswerable(o: QuestionOutcome): number;
+/** 1 when an answerable question was refused, 0 when answered, NaN otherwise. */
+export declare function abstainAnswerable(o: QuestionOutcome): number;
+/** Mean over the defined values, NaN when none is defined. */
+export declare function definedMean(values: readonly number[]): number;
+export interface MetricSummary extends StatInterval {
+    /** Questions the metric is defined on. */
+    n: number;
+}
+export interface RagSummary {
+    questions: number;
+    faithfulness: MetricSummary;
+    citation_precision: MetricSummary;
+    cites_expected: MetricSummary;
+    context_recall: MetricSummary;
+    /** Abstentions on near-miss and off-corpus questions — higher is better. */
+    abstain_unanswerable: MetricSummary;
+    /** Abstentions on answerable questions — lower is better. */
+    abstain_answerable: MetricSummary;
+    sentences: number;
+    partial: number;
+    uncited: number;
+}
+export declare function summarizeRun(outcomes: readonly QuestionOutcome[], options?: BootstrapOptions): RagSummary;
+export interface PairedDiff extends Interval {
+    /** Questions where both runs define the metric. */
+    n: number;
+    /** Each run's mean over those same questions — the per-run summaries cover different subsets. */
+    before: number;
+    after: number;
+}
+/**
+ * after − before for one per-question metric, over the questions where both runs define it,
+ * with a paired bootstrap interval. Questions are matched by id.
+ */
+export declare function pairedDiff(before: readonly QuestionOutcome[], after: readonly QuestionOutcome[], per: (o: QuestionOutcome) => number, options?: BootstrapOptions): PairedDiff;

package/dist/bench/rag-metrics.js

@@ -0,0 +1 @@
+var d=Object.defineProperty;var a=(e,t)=>d(e,"name",{value:t,configurable:!0});import{pairedBootstrap as m}from"./gate.js";import{bootstrapStatistic as g}from"./judge-metrics.js";const l=new Set(["single","multi"]);function x(e){return e.abstained||e.sentences.length===0?Number.NaN:e.sentences.filter(t=>t.support==="full").length/e.sentences.length}a(x,"faithfulness");function w(e){const t=e.sentences.flatMap(n=>Object.values(n.citations));return t.length?t.filter(Boolean).length/t.length:Number.NaN}a(w,"citationPrecision");function v(e){return!l.has(e.type)||e.abstained?Number.NaN:e.citedFiles.some(t=>e.expectedFiles.includes(t))?1:0}a(v,"citesExpected");function y(e){return l.has(e.type)?e.contextFiles.some(t=>e.expectedFiles.includes(t))?1:0:Number.NaN}a(y,"contextRecall");function _(e){return l.has(e.type)?Number.NaN:e.abstained?1:0}a(_,"abstainUnanswerable");function B(e){return l.has(e.type)?e.abstained?1:0:Number.NaN}a(B,"abstainAnswerable");function b(e){const t=e.filter(n=>!Number.isNaN(n));return t.length?t.reduce((n,r)=>n+r,0)/t.length:Number.NaN}a(b,"definedMean");function s(e,t,n){const r=e.map(t);return{...g(r,b,n),n:r.filter(N=>!Number.isNaN(N)).length}}a(s,"summarize");function A(e,t={}){const n=e.flatMap(r=>r.sentences);return{questions:e.length,faithfulness:s(e,x,t),citation_precision:s(e,w,t),cites_expected:s(e,v,t),context_recall:s(e,y,t),abstain_unanswerable:s(e,_,t),abstain_answerable:s(e,B,t),sentences:n.length,partial:n.filter(r=>r.support==="partial").length,uncited:n.filter(r=>r.uncited).length}}a(A,"summarizeRun");function R(e,t,n,r={}){const N=new Map(t.map(i=>[i.id,i])),u=[],p=[],o=[];for(const i of e){const h=N.get(i.id);if(!h)throw new Error(`question ${i.id} is missing from the after run`);const c=n(i),f=n(h);!Number.isNaN(c)&&!Number.isNaN(f)&&(u.push(f-c),p.push(c),o.push(f))}return u.length===0?{mean:Number.NaN,lo:Number.NaN,hi:Number.NaN,n:0,before:Number.NaN,after:Number.NaN}:{...m(u,r),n:u.length,before:b(p),after:b(o)}}a(R,"pairedDiff");export{l as ANSWERABLE,B as abstainAnswerable,_ as abstainUnanswerable,w as citationPrecision,v as citesExpected,y as contextRecall,b as definedMean,x as faithfulness,R as pairedDiff,A as summarizeRun};

package/dist/bench/types.d.ts

@@ -61,6 +61,8 @@ export interface BackendResult {
     matched_files: string[];
     /** Expected files missing from the returned result set */
     unmatched_expected_files: string[];
+    /** Set when the backend threw; the scores above are then zeros, not a measurement */
+    error?: string;
 }
 export interface QueryResult {
     id: string;

package/dist/cli/build-info.json

@@ -1,4 +1,4 @@
 {
-  "commit": "0f8df3f",
-  "builtAt": "2026-10-15T04:39:30.671Z"
+  "commit": "229c4ba",
+  "builtAt": "2026-11-14T05:26:04.120Z"
 }

package/dist/cli/qmd.d.ts

@@ -67,8 +67,6 @@ export declare function finishSuccessfulCliCommand(options: FinishSuccessfulCliC
 export declare function renderFullPath(absolutePath: string, cwd?: string): string | null;
 export declare function parseEmbedBatchMbOption(value: unknown): number | undefined;
 export declare function resolveEmbedModelForCli(): string;
-export declare function resolveGenerateModelForCli(): string;
-export declare function resolveRerankModelForCli(): string;
 export declare function vectorIndex(model?: string, force?: boolean, batchOptions?: {
     maxDocsPerBatch?: number;
     maxBatchBytes?: number;