@carlonicora/nestjs-neo4jsonapi 2.14.11 → 2.15.0

package/dist/eval/retrieval/retrieval-eval.types.d.ts

@@ -0,0 +1,84 @@
+import { z } from "zod";
+/** One question with its ground truth. Authored per repo, corrected by the owner. */
+export declare const evalQuestionSchema: z.ZodObject<{
+    id: z.ZodString;
+    corpus: z.ZodString;
+    question: z.ZodString;
+    mustRetrieve: z.ZodArray<z.ZodString>;
+    rubric: z.ZodString;
+    scopeId: z.ZodOptional<z.ZodString>;
+    scopeType: z.ZodOptional<z.ZodString>;
+    scopeLabel: z.ZodOptional<z.ZodString>;
+}, z.core.$strip>;
+export type EvalQuestion = z.infer<typeof evalQuestionSchema>;
+export declare const evalQuestionSetSchema: z.ZodObject<{
+    version: z.ZodLiteral<1>;
+    product: z.ZodString;
+    questions: z.ZodArray<z.ZodObject<{
+        id: z.ZodString;
+        corpus: z.ZodString;
+        question: z.ZodString;
+        mustRetrieve: z.ZodArray<z.ZodString>;
+        rubric: z.ZodString;
+        scopeId: z.ZodOptional<z.ZodString>;
+        scopeType: z.ZodOptional<z.ZodString>;
+        scopeLabel: z.ZodOptional<z.ZodString>;
+    }, z.core.$strip>>;
+}, z.core.$strip>;
+export type EvalQuestionSet = z.infer<typeof evalQuestionSetSchema>;
+/** Why a question failed its rubric. Closed set — see spec §6.5. */
+export type RubricFailureMode = "evidence-not-retrieved" | "retrieved-but-unused" | "contradicted-source" | "hedged-without-answering";
+export interface RubricVerdict {
+    passed: boolean;
+    failureMode?: RubricFailureMode;
+    explanation: string;
+}
+/** One question's result in retrieval-only mode. */
+export interface RetrievalSweepResult {
+    questionId: string;
+    evidenceRetrieved: number;
+    missingSnippets: string[];
+    chunksRead: number;
+    retrievalMs: number;
+    error?: string;
+}
+/** One question's result in end-to-end mode. Extends the retrieval facts with answer facts. */
+export interface EndToEndSweepResult extends RetrievalSweepResult {
+    evidenceCited: number;
+    uncitedSnippets: string[];
+    verdict: RubricVerdict;
+    inputTokens: number;
+    outputTokens: number;
+    answerMs: number;
+    tokenAccounting: TokenAccountingCheck;
+    /** Snippets found in the entries the answer model actually received (post-trim). */
+    evidenceRead: number;
+    /** Notebook entries that survived the trim; 0 when readObserved is false. */
+    chunksKept: number;
+    /** Kept entries that reached the model in core (unwidened) form; 0 when readObserved is false. */
+    chunksKeptCore: number;
+    /** False when the turn produced no keptChunkIds (failed turn / node skipped). */
+    readObserved: boolean;
+}
+/** Spec §6.5 — the harness validates its own token accounting. */
+export interface TokenAccountingCheck {
+    ledgerInput: number;
+    observedInput: number;
+    ledgerOutput: number;
+    observedOutput: number;
+    agrees: boolean;
+    /**
+     * Whether the comparison actually ran. A question that never produced a turn
+     * has nothing to compare — reporting `agrees: true` there would be the one
+     * direction this mechanism must never fail (§6.5).
+     */
+    checked: boolean;
+}
+export interface SweepSummary<T extends RetrievalSweepResult> {
+    product: string;
+    mode: "retrieval-only" | "end-to-end";
+    model: string;
+    startedAt: string;
+    results: T[];
+}
+//# sourceMappingURL=retrieval-eval.types.d.ts.map

package/dist/eval/retrieval/retrieval-eval.types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"retrieval-eval.types.d.ts","sourceRoot":"","sources":["../../../src/eval/retrieval/retrieval-eval.types.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,qFAAqF;AACrF,eAAO,MAAM,kBAAkB;;;;;;;;;iBAc7B,CAAC;AACH,MAAM,MAAM,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,kBAAkB,CAAC,CAAC;AAE9D,eAAO,MAAM,qBAAqB;;;;;;;;;;;;;iBAIhC,CAAC;AACH,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,qBAAqB,CAAC,CAAC;AAEpE,oEAAoE;AACpE,MAAM,MAAM,iBAAiB,GAC3B,wBAAwB,GAAG,sBAAsB,GAAG,qBAAqB,GAAG,0BAA0B,CAAC;AAEzG,MAAM,WAAW,aAAa;IAC5B,MAAM,EAAE,OAAO,CAAC;IAChB,WAAW,CAAC,EAAE,iBAAiB,CAAC;IAChC,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,oDAAoD;AACpD,MAAM,WAAW,oBAAoB;IACnC,UAAU,EAAE,MAAM,CAAC;IACnB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,+FAA+F;AAC/F,MAAM,WAAW,mBAAoB,SAAQ,oBAAoB;IAC/D,aAAa,EAAE,MAAM,CAAC;IACtB,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,OAAO,EAAE,aAAa,CAAC;IACvB,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,QAAQ,EAAE,MAAM,CAAC;IACjB,eAAe,EAAE,oBAAoB,CAAC;IACtC,oFAAoF;IACpF,YAAY,EAAE,MAAM,CAAC;IACrB,6EAA6E;IAC7E,UAAU,EAAE,MAAM,CAAC;IACnB,kGAAkG;IAClG,cAAc,EAAE,MAAM,CAAC;IACvB,iFAAiF;IACjF,YAAY,EAAE,OAAO,CAAC;CACvB;AAED,kEAAkE;AAClE,MAAM,WAAW,oBAAoB;IACnC,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,YAAY,EAAE,MAAM,CAAC;IACrB,cAAc,EAAE,MAAM,CAAC;IACvB,MAAM,EAAE,OAAO,CAAC;IAChB;;;;OAIG;IACH,OAAO,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,YAAY,CAAC,CAAC,SAAS,oBAAoB;IAC1D,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,gBAAgB,GAAG,YAAY,CAAC;IACtC,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,CAAC,EAAE,CAAC;CACd"}

package/dist/eval/retrieval/retrieval-eval.types.js

@@ -0,0 +1,25 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.evalQuestionSetSchema = exports.evalQuestionSchema = void 0;
+// src/eval/retrieval/retrieval-eval.types.ts
+const zod_1 = require("zod");
+/** One question with its ground truth. Authored per repo, corrected by the owner. */
+exports.evalQuestionSchema = zod_1.z.object({
+    id: zod_1.z.string().describe("Stable identifier, unique within the set"),
+    corpus: zod_1.z.string().describe("Which corpus this question is asked against, e.g. 'a360ai/P10'"),
+    question: zod_1.z.string().describe("The question exactly as a user would ask it"),
+    mustRetrieve: zod_1.z.array(zod_1.z.string()).describe("Distinctive text snippets a correct answer must be grounded in"),
+    rubric: zod_1.z.string().describe("What a correct answer must assert, in one or two sentences"),
+    scopeId: zod_1.z.string().optional().describe("Scope root id when the turn is scope-confined"),
+    scopeType: zod_1.z.string().optional().describe("JSON:API type of the scope root"),
+    scopeLabel: zod_1.z
+        .string()
+        .optional()
+        .describe("Neo4j label of the scope root, e.g. 'Campaign'. Required whenever scopeId is set: the scope guard needs the label, and deriving it would pull the graph catalog into the runner."),
+});
+exports.evalQuestionSetSchema = zod_1.z.object({
+    version: zod_1.z.literal(1),
+    product: zod_1.z.string().describe("Which product this set belongs to, e.g. 'a360ai'"),
+    questions: zod_1.z.array(exports.evalQuestionSchema).min(1),
+});
+//# sourceMappingURL=retrieval-eval.types.js.map

package/dist/eval/retrieval/retrieval-eval.types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"retrieval-eval.types.js","sourceRoot":"","sources":["../../../src/eval/retrieval/retrieval-eval.types.ts"],"names":[],"mappings":";;;AAAA,6CAA6C;AAC7C,6BAAwB;AAExB,qFAAqF;AACxE,QAAA,kBAAkB,GAAG,OAAC,CAAC,MAAM,CAAC;IACzC,EAAE,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,0CAA0C,CAAC;IACnE,MAAM,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,gEAAgE,CAAC;IAC7F,QAAQ,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,6CAA6C,CAAC;IAC5E,YAAY,EAAE,OAAC,CAAC,KAAK,CAAC,OAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,CAAC,gEAAgE,CAAC;IAC5G,MAAM,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,4DAA4D,CAAC;IACzF,OAAO,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,+CAA+C,CAAC;IACxF,SAAS,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,iCAAiC,CAAC;IAC5E,UAAU,EAAE,OAAC;SACV,MAAM,EAAE;SACR,QAAQ,EAAE;SACV,QAAQ,CACP,kLAAkL,CACnL;CACJ,CAAC,CAAC;AAGU,QAAA,qBAAqB,GAAG,OAAC,CAAC,MAAM,CAAC;IAC5C,OAAO,EAAE,OAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IACrB,OAAO,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,kDAAkD,CAAC;IAChF,SAAS,EAAE,OAAC,CAAC,KAAK,CAAC,0BAAkB,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;CAC9C,CAAC,CAAC"}

package/dist/eval/retrieval/retrieval-sweep.service.d.ts

@@ -0,0 +1,35 @@
+import { ChunkRepository } from "../../foundations/chunk/repositories/chunk.repository";
+import { EvidenceMatcher } from "./evidence.matcher";
+import { EvalQuestionSet, RetrievalSweepResult, SweepSummary } from "./retrieval-eval.types";
+/**
+ * The fast inner loop: asks the retrieval layer what it WOULD read for each
+ * question and stops there. No answer is generated, so the only provider call
+ * is embedding the question.
+ *
+ * Questions run SEQUENTIALLY on purpose — `retrievalMs` is a reported metric and
+ * concurrency would make it meaningless (spec §6.3).
+ */
+export declare class RetrievalSweepService {
+    private readonly chunkRepository;
+    private readonly evidence;
+    private readonly logger;
+    constructor(chunkRepository: ChunkRepository, evidence: EvidenceMatcher);
+    /**
+     * SCOPE COMES FROM CLS, NOT FROM A PARAMETER.
+     *
+     * `ChunkRepository` reads company scope from `ClsService` (via
+     * `Neo4jService.initQuery`, which pulls `companyId`/`userId` off CLS) and the
+     * run's scope root from `AGENT_SCOPE_CLS_KEY`
+     * (`common/repositories/agent-scope.filter.ts`), which fails closed. A caller
+     * that does not establish that context retrieves nothing — or, worse,
+     * retrieves unscoped. The entry point is responsible for it; see
+     * apps/api/src/eval/run-sweep.ts.
+     *
+     * This is why `run` takes no companyId: accepting one would imply it is
+     * honoured here, and it is not.
+     */
+    run(params: {
+        set: EvalQuestionSet;
+    }): Promise<SweepSummary<RetrievalSweepResult>>;
+}
+//# sourceMappingURL=retrieval-sweep.service.d.ts.map

package/dist/eval/retrieval/retrieval-sweep.service.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"retrieval-sweep.service.d.ts","sourceRoot":"","sources":["../../../src/eval/retrieval/retrieval-sweep.service.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,eAAe,EAAE,MAAM,uDAAuD,CAAC;AACxF,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,EAAE,eAAe,EAAE,oBAAoB,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAE7F;;;;;;;GAOG;AACH,qBACa,qBAAqB;IAI9B,OAAO,CAAC,QAAQ,CAAC,eAAe;IAChC,OAAO,CAAC,QAAQ,CAAC,QAAQ;IAJ3B,OAAO,CAAC,QAAQ,CAAC,MAAM,CAA0C;gBAG9C,eAAe,EAAE,eAAe,EAChC,QAAQ,EAAE,eAAe;IAG5C;;;;;;;;;;;;;OAaG;IACG,GAAG,CAAC,MAAM,EAAE;QAAE,GAAG,EAAE,eAAe,CAAA;KAAE,GAAG,OAAO,CAAC,YAAY,CAAC,oBAAoB,CAAC,CAAC;CA0CzF"}

package/dist/eval/retrieval/retrieval-sweep.service.js

@@ -0,0 +1,93 @@
+"use strict";
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+var RetrievalSweepService_1;
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RetrievalSweepService = void 0;
+const common_1 = require("@nestjs/common");
+const chunk_repository_1 = require("../../foundations/chunk/repositories/chunk.repository");
+const evidence_matcher_1 = require("./evidence.matcher");
+/**
+ * The fast inner loop: asks the retrieval layer what it WOULD read for each
+ * question and stops there. No answer is generated, so the only provider call
+ * is embedding the question.
+ *
+ * Questions run SEQUENTIALLY on purpose — `retrievalMs` is a reported metric and
+ * concurrency would make it meaningless (spec §6.3).
+ */
+let RetrievalSweepService = RetrievalSweepService_1 = class RetrievalSweepService {
+    constructor(chunkRepository, evidence) {
+        this.chunkRepository = chunkRepository;
+        this.evidence = evidence;
+        this.logger = new common_1.Logger(RetrievalSweepService_1.name);
+    }
+    /**
+     * SCOPE COMES FROM CLS, NOT FROM A PARAMETER.
+     *
+     * `ChunkRepository` reads company scope from `ClsService` (via
+     * `Neo4jService.initQuery`, which pulls `companyId`/`userId` off CLS) and the
+     * run's scope root from `AGENT_SCOPE_CLS_KEY`
+     * (`common/repositories/agent-scope.filter.ts`), which fails closed. A caller
+     * that does not establish that context retrieves nothing — or, worse,
+     * retrieves unscoped. The entry point is responsible for it; see
+     * apps/api/src/eval/run-sweep.ts.
+     *
+     * This is why `run` takes no companyId: accepting one would imply it is
+     * honoured here, and it is not.
+     */
+    async run(params) {
+        const startedAt = new Date().toISOString();
+        const results = [];
+        for (const question of params.set.questions) {
+            const began = Date.now();
+            try {
+                const chunks = await this.chunkRepository.findPotentialChunks({
+                    question: question.question,
+                    dataLimits: {},
+                });
+                const haystack = chunks.map((chunk) => chunk.content ?? "").join("\n\n");
+                const scored = this.evidence.score({ snippets: question.mustRetrieve, haystack });
+                results.push({
+                    questionId: question.id,
+                    evidenceRetrieved: scored.found,
+                    missingSnippets: scored.missing,
+                    chunksRead: chunks.length,
+                    retrievalMs: Date.now() - began,
+                });
+            }
+            catch (error) {
+                const message = error instanceof Error ? error.message : String(error);
+                this.logger.warn(`retrieval sweep: question ${question.id} failed: ${message}`);
+                results.push({
+                    questionId: question.id,
+                    evidenceRetrieved: 0,
+                    missingSnippets: question.mustRetrieve,
+                    chunksRead: 0,
+                    retrievalMs: Date.now() - began,
+                    error: message,
+                });
+            }
+        }
+        return {
+            product: params.set.product,
+            mode: "retrieval-only",
+            model: "n/a — retrieval only",
+            startedAt,
+            results,
+        };
+    }
+};
+exports.RetrievalSweepService = RetrievalSweepService;
+exports.RetrievalSweepService = RetrievalSweepService = RetrievalSweepService_1 = __decorate([
+    (0, common_1.Injectable)(),
+    __metadata("design:paramtypes", [chunk_repository_1.ChunkRepository,
+        evidence_matcher_1.EvidenceMatcher])
+], RetrievalSweepService);
+//# sourceMappingURL=retrieval-sweep.service.js.map

package/dist/eval/retrieval/retrieval-sweep.service.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"retrieval-sweep.service.js","sourceRoot":"","sources":["../../../src/eval/retrieval/retrieval-sweep.service.ts"],"names":[],"mappings":";;;;;;;;;;;;;AAAA,2CAAoD;AACpD,4FAAwF;AACxF,yDAAqD;AAGrD;;;;;;;GAOG;AAEI,IAAM,qBAAqB,6BAA3B,MAAM,qBAAqB;IAGhC,YACmB,eAAgC,EAChC,QAAyB;QADzB,oBAAe,GAAf,eAAe,CAAiB;QAChC,aAAQ,GAAR,QAAQ,CAAiB;QAJ3B,WAAM,GAAG,IAAI,eAAM,CAAC,uBAAqB,CAAC,IAAI,CAAC,CAAC;IAK9D,CAAC;IAEJ;;;;;;;;;;;;;OAaG;IACH,KAAK,CAAC,GAAG,CAAC,MAAgC;QACxC,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAC3C,MAAM,OAAO,GAA2B,EAAE,CAAC;QAE3C,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,GAAG,CAAC,SAAS,EAAE,CAAC;YAC5C,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACzB,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,mBAAmB,CAAC;oBAC5D,QAAQ,EAAE,QAAQ,CAAC,QAAQ;oBAC3B,UAAU,EAAE,EAAE;iBACf,CAAC,CAAC;gBACH,MAAM,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBACzE,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,QAAQ,EAAE,QAAQ,CAAC,YAAY,EAAE,QAAQ,EAAE,CAAC,CAAC;gBAClF,OAAO,CAAC,IAAI,CAAC;oBACX,UAAU,EAAE,QAAQ,CAAC,EAAE;oBACvB,iBAAiB,EAAE,MAAM,CAAC,KAAK;oBAC/B,eAAe,EAAE,MAAM,CAAC,OAAO;oBAC/B,UAAU,EAAE,MAAM,CAAC,MAAM;oBACzB,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;iBAChC,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;gBACvE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,6BAA6B,QAAQ,CAAC,EAAE,YAAY,OAAO,EAAE,CAAC,CAAC;gBAChF,OAAO,CAAC,IAAI,CAAC;oBACX,UAAU,EAAE,QAAQ,CAAC,EAAE;oBACvB,iBAAiB,EAAE,CAAC;oBACpB,eAAe,EAAE,QAAQ,CAAC,YAAY;oBACtC,UAAU,EAAE,CAAC;oBACb,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;oBAC/B,KAAK,EAAE,OAAO;iBACf,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,OAAO;YACL,OAAO,EAAE,MAAM,CAAC,GAAG,CAAC,OAAO;YAC3B,IAAI,EAAE,gBAAgB;YACtB,KAAK,EAAE,sBAAsB;YAC7B,SAAS;YACT,OAAO;SACR,CAAC;IACJ,CAAC;CACF,CAAA;AAhEY,sDAAqB;gCAArB,qBAAqB;IADjC,IAAA,mBAAU,GAAE;qCAKyB,kCAAe;QACtB,kCAAe;GALjC,qBAAqB,CAgEjC"}

package/dist/eval/retrieval/rubric-judge.service.d.ts

@@ -0,0 +1,30 @@
+import { LLMService } from "../../core/llm/services/llm.service";
+import { RubricVerdict } from "./retrieval-eval.types";
+export declare const RUBRIC_JUDGE_PROMPT = "\nYou are grading one answer produced by a retrieval-augmented assistant.\n\nYou are given the user's question, a rubric stating what a correct answer MUST\nassert, the answer that was produced, and how many pieces of required evidence\nthe answer cited.\n\nDecide ONE thing: does the answer satisfy the rubric?\n\nJudge only against the rubric. Do not reward length, tone, formatting or\nconfidence. An answer that is well written but does not assert what the rubric\nrequires has failed. An answer that asserts what the rubric requires in plain\nlanguage has passed, even if it is terse.\n\nWhen the answer FAILS, classify why, choosing exactly one:\n\n- evidence-not-retrieved: the answer could not have been written because the\n  required material never reached it. Typically the citation count is zero.\n- retrieved-but-unused: the required material was cited or is clearly present,\n  and the answer still does not assert what the rubric requires.\n- contradicted-source: the answer asserts something that conflicts with the\n  material it cites.\n- hedged-without-answering: the answer declines, defers, or describes what it\n  would need, instead of answering.\n\nBe strict and be consistent. Your explanation must be one sentence naming the\nspecific rubric requirement that was met or missed.\n";
+/**
+ * Grades one answer against its rubric.
+ *
+ * A judging failure degrades to a FAILED verdict for that question rather than
+ * killing the sweep — the graceful-degradation rule in 06-llm-calls.md rule 8,
+ * applied deliberately: an ungraded question must never silently count as a pass.
+ *
+ * ATTRIBUTION EXCEPTION (06-llm-calls.md rule 5). `tokenUsageType` and
+ * `metadata` are set, but `relationshipId` / `relationshipType` are deliberately
+ * absent: a sweep grades the harness itself, so there is no tenant entity whose
+ * spend this is. LLMService skips usage persistence unless BOTH are set, which
+ * is the intended outcome here — eval spend must not land on a customer's
+ * ledger. The call is still fully identifiable in dumps and telemetry through
+ * metadata.agentName / metadata.nodeName.
+ */
+export declare class RubricJudgeService {
+    private readonly llmService;
+    private readonly logger;
+    constructor(llmService: LLMService);
+    judge(params: {
+        question: string;
+        rubric: string;
+        answer: string;
+        evidenceCited: number;
+    }): Promise<RubricVerdict>;
+}
+//# sourceMappingURL=rubric-judge.service.d.ts.map

package/dist/eval/retrieval/rubric-judge.service.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"rubric-judge.service.d.ts","sourceRoot":"","sources":["../../../src/eval/retrieval/rubric-judge.service.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,UAAU,EAAE,MAAM,qCAAqC,CAAC;AACjE,OAAO,EAAqB,aAAa,EAAE,MAAM,wBAAwB,CAAC;AAE1E,eAAO,MAAM,mBAAmB,8xCA2B/B,CAAC;AAoCF;;;;;;;;;;;;;;GAcG;AACH,qBACa,kBAAkB;IAGjB,OAAO,CAAC,QAAQ,CAAC,UAAU;IAFvC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAuC;gBAEjC,UAAU,EAAE,UAAU;IAE7C,KAAK,CAAC,MAAM,EAAE;QAClB,QAAQ,EAAE,MAAM,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,MAAM,EAAE,MAAM,CAAC;QACf,aAAa,EAAE,MAAM,CAAC;KACvB,GAAG,OAAO,CAAC,aAAa,CAAC;CAuC3B"}

package/dist/eval/retrieval/rubric-judge.service.js

@@ -0,0 +1,138 @@
+"use strict";
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+var RubricJudgeService_1;
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RubricJudgeService = exports.RUBRIC_JUDGE_PROMPT = void 0;
+const common_1 = require("@nestjs/common");
+const zod_1 = require("zod");
+const llm_service_1 = require("../../core/llm/services/llm.service");
+exports.RUBRIC_JUDGE_PROMPT = `
+You are grading one answer produced by a retrieval-augmented assistant.
+
+You are given the user's question, a rubric stating what a correct answer MUST
+assert, the answer that was produced, and how many pieces of required evidence
+the answer cited.
+
+Decide ONE thing: does the answer satisfy the rubric?
+
+Judge only against the rubric. Do not reward length, tone, formatting or
+confidence. An answer that is well written but does not assert what the rubric
+requires has failed. An answer that asserts what the rubric requires in plain
+language has passed, even if it is terse.
+
+When the answer FAILS, classify why, choosing exactly one:
+
+- evidence-not-retrieved: the answer could not have been written because the
+  required material never reached it. Typically the citation count is zero.
+- retrieved-but-unused: the required material was cited or is clearly present,
+  and the answer still does not assert what the rubric requires.
+- contradicted-source: the answer asserts something that conflicts with the
+  material it cites.
+- hedged-without-answering: the answer declines, defers, or describes what it
+  would need, instead of answering.
+
+Be strict and be consistent. Your explanation must be one sentence naming the
+specific rubric requirement that was met or missed.
+`;
+/**
+ * The closed set of failure modes (spec §6.5). Declared once so the schema the
+ * model is constrained to and the runtime guard below can never drift apart.
+ */
+const FAILURE_MODES = [
+    "evidence-not-retrieved",
+    "retrieved-but-unused",
+    "contradicted-source",
+    "hedged-without-answering",
+];
+const outputSchema = zod_1.z.object({
+    passed: zod_1.z.boolean().describe("True only if the answer asserts everything the rubric requires"),
+    failureMode: zod_1.z
+        .enum(FAILURE_MODES)
+        .optional()
+        .describe("Present if and only if passed is false. One of the four named modes, verbatim."),
+    explanation: zod_1.z.string().describe("One sentence naming the specific rubric requirement that was met or missed"),
+});
+const inputSchema = zod_1.z.object({
+    question: zod_1.z.string().describe("The user's question, verbatim"),
+    rubric: zod_1.z
+        .string()
+        .describe("BINDING CRITERION. What a correct answer must assert. Judge against this and nothing else."),
+    answer: zod_1.z.string().describe("The answer under test. Grade it; do not improve or continue it."),
+    evidenceCited: zod_1.z
+        .number()
+        .describe("How many required evidence snippets the answer cited. Zero suggests evidence-not-retrieved."),
+});
+const isFailureMode = (value) => typeof value === "string" && FAILURE_MODES.includes(value);
+/**
+ * Grades one answer against its rubric.
+ *
+ * A judging failure degrades to a FAILED verdict for that question rather than
+ * killing the sweep — the graceful-degradation rule in 06-llm-calls.md rule 8,
+ * applied deliberately: an ungraded question must never silently count as a pass.
+ *
+ * ATTRIBUTION EXCEPTION (06-llm-calls.md rule 5). `tokenUsageType` and
+ * `metadata` are set, but `relationshipId` / `relationshipType` are deliberately
+ * absent: a sweep grades the harness itself, so there is no tenant entity whose
+ * spend this is. LLMService skips usage persistence unless BOTH are set, which
+ * is the intended outcome here — eval spend must not land on a customer's
+ * ledger. The call is still fully identifiable in dumps and telemetry through
+ * metadata.agentName / metadata.nodeName.
+ */
+let RubricJudgeService = RubricJudgeService_1 = class RubricJudgeService {
+    constructor(llmService) {
+        this.llmService = llmService;
+        this.logger = new common_1.Logger(RubricJudgeService_1.name);
+    }
+    async judge(params) {
+        try {
+            const response = await this.llmService.call({
+                inputSchema,
+                inputParams: {
+                    question: params.question,
+                    rubric: params.rubric,
+                    answer: params.answer,
+                    evidenceCited: params.evidenceCited,
+                },
+                outputSchema,
+                systemPrompts: [exports.RUBRIC_JUDGE_PROMPT],
+                // A grading pass is judgement, not creation: no sampling variance, so
+                // the same answer is graded the same way on every run.
+                temperature: 0,
+                // Deliberate for THIS call (06-llm-calls.md rule 6 / checkpoint step 5):
+                // stated here rather than inherited from the tier, so a deployment that
+                // retunes the default cannot silently change how the baseline is graded.
+                // "low" matches the model this baseline is measured on (gpt-5.6-luna).
+                reasoningEffort: "low",
+                tokenUsageType: "retrieval_eval",
+                metadata: { agentName: "retrieval-eval", nodeName: "rubric-judge" },
+            });
+            // The schema already constrains the model; this guard keeps the returned
+            // union honest even when the provider falls back to lenient parsing.
+            const failureMode = isFailureMode(response.failureMode) ? response.failureMode : undefined;
+            return {
+                passed: response.passed,
+                ...(failureMode ? { failureMode } : {}),
+                explanation: response.explanation,
+            };
+        }
+        catch (error) {
+            const message = error instanceof Error ? error.message : String(error);
+            this.logger.warn(`rubric judge failed, recording the question as failed: ${message}`);
+            return { passed: false, explanation: `Judge unavailable: ${message}` };
+        }
+    }
+};
+exports.RubricJudgeService = RubricJudgeService;
+exports.RubricJudgeService = RubricJudgeService = RubricJudgeService_1 = __decorate([
+    (0, common_1.Injectable)(),
+    __metadata("design:paramtypes", [llm_service_1.LLMService])
+], RubricJudgeService);
+//# sourceMappingURL=rubric-judge.service.js.map

package/dist/eval/retrieval/rubric-judge.service.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"rubric-judge.service.js","sourceRoot":"","sources":["../../../src/eval/retrieval/rubric-judge.service.ts"],"names":[],"mappings":";;;;;;;;;;;;;AAAA,2CAAoD;AACpD,6BAAwB;AACxB,qEAAiE;AAGpD,QAAA,mBAAmB,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;CA2BlC,CAAC;AAEF;;;GAGG;AACH,MAAM,aAAa,GAAG;IACpB,wBAAwB;IACxB,sBAAsB;IACtB,qBAAqB;IACrB,0BAA0B;CAClB,CAAC;AAEX,MAAM,YAAY,GAAG,OAAC,CAAC,MAAM,CAAC;IAC5B,MAAM,EAAE,OAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,CAAC,gEAAgE,CAAC;IAC9F,WAAW,EAAE,OAAC;SACX,IAAI,CAAC,aAAa,CAAC;SACnB,QAAQ,EAAE;SACV,QAAQ,CAAC,gFAAgF,CAAC;IAC7F,WAAW,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,4EAA4E,CAAC;CAC/G,CAAC,CAAC;AAEH,MAAM,WAAW,GAAG,OAAC,CAAC,MAAM,CAAC;IAC3B,QAAQ,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,+BAA+B,CAAC;IAC9D,MAAM,EAAE,OAAC;SACN,MAAM,EAAE;SACR,QAAQ,CAAC,4FAA4F,CAAC;IACzG,MAAM,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,iEAAiE,CAAC;IAC9F,aAAa,EAAE,OAAC;SACb,MAAM,EAAE;SACR,QAAQ,CAAC,6FAA6F,CAAC;CAC3G,CAAC,CAAC;AAEH,MAAM,aAAa,GAAG,CAAC,KAAc,EAA8B,EAAE,CACnE,OAAO,KAAK,KAAK,QAAQ,IAAK,aAAmC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;AAEpF;;;;;;;;;;;;;;GAcG;AAEI,IAAM,kBAAkB,0BAAxB,MAAM,kBAAkB;IAG7B,YAA6B,UAAsB;QAAtB,eAAU,GAAV,UAAU,CAAY;QAFlC,WAAM,GAAG,IAAI,eAAM,CAAC,oBAAkB,CAAC,IAAI,CAAC,CAAC;IAER,CAAC;IAEvD,KAAK,CAAC,KAAK,CAAC,MAKX;QACC,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,CAA+B;gBACxE,WAAW;gBACX,WAAW,EAAE;oBACX,QAAQ,EAAE,MAAM,CAAC,QAAQ;oBACzB,MAAM,EAAE,MAAM,CAAC,MAAM;oBACrB,MAAM,EAAE,MAAM,CAAC,MAAM;oBACrB,aAAa,EAAE,MAAM,CAAC,aAAa;iBACpC;gBACD,YAAY;gBACZ,aAAa,EAAE,CAAC,2BAAmB,CAAC;gBACpC,sEAAsE;gBACtE,uDAAuD;gBACvD,WAAW,EAAE,CAAC;gBACd,yEAAyE;gBACzE,wEAAwE;gBACxE,yEAAyE;gBACzE,uEAAuE;gBACvE,eAAe,EAAE,KAAK;gBACtB,cAAc,EAAE,gBAAgB;gBAChC,QAAQ,EAAE,EAAE,SAAS,EAAE,gBAAgB,EAAE,QAAQ,EAAE,cAAc,EAAE;aACpE,CAAC,CAAC;YAEH,yEAAyE;YACzE,qEAAqE;YACrE,MAAM,WAAW,GAAG,aAAa,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS,CAAC;YAE3F,OAAO;gBACL,MAAM,EAAE,QAAQ,CAAC,MAAM;gBACvB,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACvC,WAAW,EAAE,QAAQ,CAAC,WAAW;aAClC,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACvE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,0DAA0D,OAAO,EAAE,CAAC,CAAC;YACtF,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,sBAAsB,OAAO,EAAE,EAAE,CAAC;QACzE,CAAC;IACH,CAAC;CACF,CAAA;AAjDY,gDAAkB;6BAAlB,kBAAkB;IAD9B,IAAA,mBAAU,GAAE;qCAI8B,wBAAU;GAHxC,kBAAkB,CAiD9B"}

package/dist/eval/retrieval/sweep-report.d.ts

@@ -0,0 +1,11 @@
+import { RetrievalSweepResult, SweepSummary } from "./retrieval-eval.types";
+/**
+ * Renders a sweep to markdown, leading with the number that decides whether a
+ * change ships.
+ *
+ * A token-accounting mismatch is rendered as a banner, not a column: a sweep
+ * whose own arithmetic does not close cannot be used as a baseline, and that
+ * must not be discoverable only by reading a table.
+ */
+export declare function renderSweepReport(summary: SweepSummary<RetrievalSweepResult>): string;
+//# sourceMappingURL=sweep-report.d.ts.map

package/dist/eval/retrieval/sweep-report.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"sweep-report.d.ts","sourceRoot":"","sources":["../../../src/eval/retrieval/sweep-report.ts"],"names":[],"mappings":"AAAA,OAAO,EAAuB,oBAAoB,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AASjG;;;;;;;GAOG;AACH,wBAAgB,iBAAiB,CAAC,OAAO,EAAE,YAAY,CAAC,oBAAoB,CAAC,GAAG,MAAM,CA8ErF"}

package/dist/eval/retrieval/sweep-report.js

@@ -0,0 +1,79 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.renderSweepReport = renderSweepReport;
+const isEndToEnd = (result) => result.verdict !== undefined;
+/** Total `mustRetrieve` snippets for the question — same denominator `evidenceRetrieved` uses. */
+const questionSnippetTotal = (result) => result.evidenceRetrieved + result.missingSnippets.length;
+/**
+ * Renders a sweep to markdown, leading with the number that decides whether a
+ * change ships.
+ *
+ * A token-accounting mismatch is rendered as a banner, not a column: a sweep
+ * whose own arithmetic does not close cannot be used as a baseline, and that
+ * must not be discoverable only by reading a table.
+ */
+function renderSweepReport(summary) {
+    const lines = [];
+    const graded = summary.results.filter(isEndToEnd);
+    const passed = graded.filter((result) => result.verdict.passed).length;
+    lines.push(`# Retrieval sweep — ${summary.product} (${summary.mode})`);
+    lines.push("");
+    lines.push(`Model \`${summary.model}\` · started ${summary.startedAt}`);
+    lines.push("");
+    // `checked &&` on purpose: a question whose turn never completed carries
+    // `checked: false, agrees: false`, and it is already reported as a failure
+    // below. Counting it here too would announce a token mismatch that was never
+    // measured and cast doubt on a sweep whose arithmetic is fine.
+    const mismatched = graded.filter((result) => result.tokenAccounting.checked && !result.tokenAccounting.agrees);
+    if (mismatched.length > 0) {
+        lines.push(`> **TOKEN ACCOUNTING MISMATCH on ${mismatched.length} question(s).**`);
+        lines.push("> This sweep cannot be used as a baseline until the discrepancy is explained.");
+        for (const result of mismatched) {
+            lines.push(`> \`${result.questionId}\`: reported ${result.tokenAccounting.ledgerInput}/` +
+                `${result.tokenAccounting.ledgerOutput}, observed ` +
+                `${result.tokenAccounting.observedInput}/${result.tokenAccounting.observedOutput}`);
+        }
+        lines.push("");
+    }
+    if (graded.length > 0) {
+        const totalMustRetrieve = graded.reduce((sum, result) => sum + questionSnippetTotal(result), 0);
+        const totalEvidenceRead = graded.reduce((sum, result) => sum + result.evidenceRead, 0);
+        lines.push(`**Rubric passes: ${passed}/${graded.length}** · evidence read ${totalEvidenceRead}/${totalMustRetrieve}`);
+        lines.push("");
+        lines.push("| Question | Verdict | Failure mode | Evidence retrieved | Evidence read | Evidence cited | Chunks | Chunks kept | Chunks kept core | Tokens in/out | Answer ms |");
+        lines.push("|---|---|---|---:|---:|---:|---:|---:|---:|---:|---:|");
+        for (const result of graded) {
+            lines.push(`| \`${result.questionId}\` | ${result.verdict.passed ? "pass" : "**fail**"} | ` +
+                `${result.verdict.failureMode ?? "—"} | ` +
+                `${result.evidenceRetrieved}/${result.evidenceRetrieved + result.missingSnippets.length} | ` +
+                `${result.readObserved ? `${result.evidenceRead}/${questionSnippetTotal(result)}` : "n/a"} | ` +
+                `${result.evidenceCited}/${result.evidenceCited + result.uncitedSnippets.length} | ` +
+                `${result.chunksRead} | ${result.chunksKept} | ${result.chunksKeptCore} | ${result.inputTokens}/${result.outputTokens} | ${result.answerMs} |`);
+        }
+    }
+    else {
+        lines.push("| Question | Evidence retrieved | Chunks | Retrieval ms |");
+        lines.push("|---|---:|---:|---:|");
+        for (const result of summary.results) {
+            lines.push(`| \`${result.questionId}\` | ` +
+                `${result.evidenceRetrieved}/${result.evidenceRetrieved + result.missingSnippets.length} | ` +
+                `${result.chunksRead} | ${result.retrievalMs} |`);
+        }
+    }
+    const failures = graded.filter((result) => !result.verdict.passed);
+    if (failures.length > 0) {
+        lines.push("");
+        lines.push("## Failures");
+        for (const result of failures) {
+            lines.push("");
+            lines.push(`### \`${result.questionId}\` — ${result.verdict.failureMode ?? "ungraded"}`);
+            lines.push(result.verdict.explanation);
+            if (result.missingSnippets.length > 0) {
+                lines.push("");
+                lines.push(`Never retrieved: ${result.missingSnippets.map((s) => `"${s}"`).join(", ")}`);
+            }
+        }
+    }
+    return lines.join("\n");
+}
+//# sourceMappingURL=sweep-report.js.map

package/dist/eval/retrieval/sweep-report.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"sweep-report.js","sourceRoot":"","sources":["../../../src/eval/retrieval/sweep-report.ts"],"names":[],"mappings":";;AAiBA,8CA8EC;AA7FD,MAAM,UAAU,GAAG,CAAC,MAA4B,EAAiC,EAAE,CAChF,MAA8B,CAAC,OAAO,KAAK,SAAS,CAAC;AAExD,kGAAkG;AAClG,MAAM,oBAAoB,GAAG,CAAC,MAA2B,EAAU,EAAE,CACnE,MAAM,CAAC,iBAAiB,GAAG,MAAM,CAAC,eAAe,CAAC,MAAM,CAAC;AAE3D;;;;;;;GAOG;AACH,SAAgB,iBAAiB,CAAC,OAA2C;IAC3E,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;IAClD,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IAEvE,KAAK,CAAC,IAAI,CAAC,uBAAuB,OAAO,CAAC,OAAO,KAAK,OAAO,CAAC,IAAI,GAAG,CAAC,CAAC;IACvE,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,WAAW,OAAO,CAAC,KAAK,gBAAgB,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC;IACxE,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,yEAAyE;IACzE,2EAA2E;IAC3E,6EAA6E;IAC7E,+DAA+D;IAC/D,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,eAAe,CAAC,OAAO,IAAI,CAAC,MAAM,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC;IAC/G,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC1B,KAAK,CAAC,IAAI,CAAC,oCAAoC,UAAU,CAAC,MAAM,iBAAiB,CAAC,CAAC;QACnF,KAAK,CAAC,IAAI,CAAC,+EAA+E,CAAC,CAAC;QAC5F,KAAK,MAAM,MAAM,IAAI,UAAU,EAAE,CAAC;YAChC,KAAK,CAAC,IAAI,CACR,OAAO,MAAM,CAAC,UAAU,gBAAgB,MAAM,CAAC,eAAe,CAAC,WAAW,GAAG;gBAC3E,GAAG,MAAM,CAAC,eAAe,CAAC,YAAY,aAAa;gBACnD,GAAG,MAAM,CAAC,eAAe,CAAC,aAAa,IAAI,MAAM,CAAC,eAAe,CAAC,cAAc,EAAE,CACrF,CAAC;QACJ,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IAED,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtB,MAAM,iBAAiB,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE,CAAC,GAAG,GAAG,oBAAoB,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;QAChG,MAAM,iBAAiB,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE,CAAC,GAAG,GAAG,MAAM,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC;QACvF,KAAK,CAAC,IAAI,CACR,oBAAoB,MAAM,IAAI,MAAM,CAAC,MAAM,sBAAsB,iBAAiB,IAAI,iBAAiB,EAAE,CAC1G,CAAC;QACF,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAEf,KAAK,CAAC,IAAI,CACR,mKAAmK,CACpK,CAAC;QACF,KAAK,CAAC,IAAI,CAAC,uDAAuD,CAAC,CAAC;QACpE,KAAK,MAAM,MAAM,IAAI,MAAM,EAAE,CAAC;YAC5B,KAAK,CAAC,IAAI,CACR,OAAO,MAAM,CAAC,UAAU,QAAQ,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,UAAU,KAAK;gBAC9E,GAAG,MAAM,CAAC,OAAO,CAAC,WAAW,IAAI,GAAG,KAAK;gBACzC,GAAG,MAAM,CAAC,iBAAiB,IAAI,MAAM,CAAC,iBAAiB,GAAG,MAAM,CAAC,eAAe,CAAC,MAAM,KAAK;gBAC5F,GAAG,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,YAAY,IAAI,oBAAoB,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK;gBAC9F,GAAG,MAAM,CAAC,aAAa,IAAI,MAAM,CAAC,aAAa,GAAG,MAAM,CAAC,eAAe,CAAC,MAAM,KAAK;gBACpF,GAAG,MAAM,CAAC,UAAU,MAAM,MAAM,CAAC,UAAU,MAAM,MAAM,CAAC,cAAc,MAAM,MAAM,CAAC,WAAW,IAAI,MAAM,CAAC,YAAY,MAAM,MAAM,CAAC,QAAQ,IAAI,CACjJ,CAAC;QACJ,CAAC;IACH,CAAC;SAAM,CAAC;QACN,KAAK,CAAC,IAAI,CAAC,2DAA2D,CAAC,CAAC;QACxE,KAAK,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC;QACnC,KAAK,MAAM,MAAM,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;YACrC,KAAK,CAAC,IAAI,CACR,OAAO,MAAM,CAAC,UAAU,OAAO;gBAC7B,GAAG,MAAM,CAAC,iBAAiB,IAAI,MAAM,CAAC,iBAAiB,GAAG,MAAM,CAAC,eAAe,CAAC,MAAM,KAAK;gBAC5F,GAAG,MAAM,CAAC,UAAU,MAAM,MAAM,CAAC,WAAW,IAAI,CACnD,CAAC;QACJ,CAAC;IACH,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IACnE,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxB,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAC1B,KAAK,MAAM,MAAM,IAAI,QAAQ,EAAE,CAAC;YAC9B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACf,KAAK,CAAC,IAAI,CAAC,SAAS,MAAM,CAAC,UAAU,QAAQ,MAAM,CAAC,OAAO,CAAC,WAAW,IAAI,UAAU,EAAE,CAAC,CAAC;YACzF,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;YACvC,IAAI,MAAM,CAAC,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACtC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gBACf,KAAK,CAAC,IAAI,CAAC,oBAAoB,MAAM,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAC3F,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC"}

package/dist/eval/retrieval/token-accounting.check.d.ts

@@ -0,0 +1,39 @@
+import { TokenAccountingCheck } from "./retrieval-eval.types";
+/**
+ * Cross-checks a turn's reported token aggregate against the sum of the calls
+ * actually observed for it.
+ *
+ * Spec §6.5. The `2C + A` reducer bug inflated every archived chat figure and
+ * went unnoticed for a full measurement cycle; this check is what makes the
+ * same class of bug loud instead of silent. Provider-reported per-call figures
+ * are the source of truth.
+ */
+export declare class TokenAccountingChecker {
+    private readonly logger;
+    private observedInput;
+    private observedOutput;
+    /** Record one provider call's reported usage for the question in flight. */
+    observe(params: {
+        input: number;
+        output: number;
+    }): void;
+    /**
+     * Clears the accumulator WITHOUT producing a verdict.
+     *
+     * For the question that never completed a turn: it has nothing to compare, but
+     * it may still have deposited observations before it threw. Leaving them in
+     * place attributes them to the NEXT question — which either invents a mismatch
+     * that is not there or, worse, masks one that is. `check()` resets as a side
+     * effect of comparing; this is the same reset for the path that cannot compare.
+     */
+    reset(): void;
+    /** Compare and RESET, so the next question starts clean. */
+    check(params: {
+        questionId: string;
+        ledger: {
+            input: number;
+            output: number;
+        };
+    }): TokenAccountingCheck;
+}
+//# sourceMappingURL=token-accounting.check.d.ts.map

package/dist/eval/retrieval/token-accounting.check.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"token-accounting.check.d.ts","sourceRoot":"","sources":["../../../src/eval/retrieval/token-accounting.check.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,oBAAoB,EAAE,MAAM,wBAAwB,CAAC;AAK9D;;;;;;;;GAQG;AACH,qBACa,sBAAsB;IACjC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAA2C;IAClE,OAAO,CAAC,aAAa,CAAK;IAC1B,OAAO,CAAC,cAAc,CAAK;IAE3B,4EAA4E;IAC5E,OAAO,CAAC,MAAM,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI;IAKxD;;;;;;;;OAQG;IACH,KAAK,IAAI,IAAI;IAKb,4DAA4D;IAC5D,KAAK,CAAC,MAAM,EAAE;QAAE,UAAU,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE;YAAE,KAAK,EAAE,MAAM,CAAC;YAAC,MAAM,EAAE,MAAM,CAAA;SAAE,CAAA;KAAE,GAAG,oBAAoB;CA2BvG"}