@carlonicora/nestjs-neo4jsonapi 2.14.10 → 2.15.0

package/dist/eval/retrieval/evidence.matcher.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"evidence.matcher.js","sourceRoot":"","sources":["../../../src/eval/retrieval/evidence.matcher.ts"],"names":[],"mappings":";;;;;;;;;AAAA,2CAA4C;AAE5C;;;;;;;GAOG;AAEI,IAAM,eAAe,GAArB,MAAM,eAAe;IAClB,SAAS,CAAC,KAAa;QAC7B,OAAO,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IACzD,CAAC;IAED,OAAO,CAAC,MAA6C;QACnD,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAC9C,IAAI,MAAM,KAAK,EAAE;YAAE,OAAO,KAAK,CAAC;QAChC,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IAC1D,CAAC;IAED,KAAK,CAAC,MAAgD;QACpD,MAAM,OAAO,GAAG,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;QAC3G,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,QAAQ,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC;IACrE,CAAC;CACF,CAAA;AAfY,0CAAe;0BAAf,eAAe;IAD3B,IAAA,mBAAU,GAAE;GACA,eAAe,CAe3B"}

package/dist/eval/retrieval/index.d.ts

@@ -0,0 +1,10 @@
+export * from "./retrieval-eval.types";
+export { QuestionSetLoader } from "./question-set.loader";
+export { EvidenceMatcher } from "./evidence.matcher";
+export { RetrievalSweepService } from "./retrieval-sweep.service";
+export { EndToEndSweepService } from "./end-to-end-sweep.service";
+export { RubricJudgeService } from "./rubric-judge.service";
+export { TokenAccountingChecker } from "./token-accounting.check";
+export { renderSweepReport } from "./sweep-report";
+export { RetrievalEvalModule } from "./retrieval-eval.module";
+//# sourceMappingURL=index.d.ts.map

package/dist/eval/retrieval/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/eval/retrieval/index.ts"],"names":[],"mappings":"AAAA,cAAc,wBAAwB,CAAC;AACvC,OAAO,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAC;AAC1D,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,EAAE,qBAAqB,EAAE,MAAM,2BAA2B,CAAC;AAClE,OAAO,EAAE,oBAAoB,EAAE,MAAM,4BAA4B,CAAC;AAClE,OAAO,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAC;AAC5D,OAAO,EAAE,sBAAsB,EAAE,MAAM,0BAA0B,CAAC;AAClE,OAAO,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AACnD,OAAO,EAAE,mBAAmB,EAAE,MAAM,yBAAyB,CAAC"}

package/dist/eval/retrieval/index.js

@@ -0,0 +1,35 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RetrievalEvalModule = exports.renderSweepReport = exports.TokenAccountingChecker = exports.RubricJudgeService = exports.EndToEndSweepService = exports.RetrievalSweepService = exports.EvidenceMatcher = exports.QuestionSetLoader = void 0;
+__exportStar(require("./retrieval-eval.types"), exports);
+var question_set_loader_1 = require("./question-set.loader");
+Object.defineProperty(exports, "QuestionSetLoader", { enumerable: true, get: function () { return question_set_loader_1.QuestionSetLoader; } });
+var evidence_matcher_1 = require("./evidence.matcher");
+Object.defineProperty(exports, "EvidenceMatcher", { enumerable: true, get: function () { return evidence_matcher_1.EvidenceMatcher; } });
+var retrieval_sweep_service_1 = require("./retrieval-sweep.service");
+Object.defineProperty(exports, "RetrievalSweepService", { enumerable: true, get: function () { return retrieval_sweep_service_1.RetrievalSweepService; } });
+var end_to_end_sweep_service_1 = require("./end-to-end-sweep.service");
+Object.defineProperty(exports, "EndToEndSweepService", { enumerable: true, get: function () { return end_to_end_sweep_service_1.EndToEndSweepService; } });
+var rubric_judge_service_1 = require("./rubric-judge.service");
+Object.defineProperty(exports, "RubricJudgeService", { enumerable: true, get: function () { return rubric_judge_service_1.RubricJudgeService; } });
+var token_accounting_check_1 = require("./token-accounting.check");
+Object.defineProperty(exports, "TokenAccountingChecker", { enumerable: true, get: function () { return token_accounting_check_1.TokenAccountingChecker; } });
+var sweep_report_1 = require("./sweep-report");
+Object.defineProperty(exports, "renderSweepReport", { enumerable: true, get: function () { return sweep_report_1.renderSweepReport; } });
+var retrieval_eval_module_1 = require("./retrieval-eval.module");
+Object.defineProperty(exports, "RetrievalEvalModule", { enumerable: true, get: function () { return retrieval_eval_module_1.RetrievalEvalModule; } });
+//# sourceMappingURL=index.js.map

package/dist/eval/retrieval/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/eval/retrieval/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;AAAA,yDAAuC;AACvC,6DAA0D;AAAjD,wHAAA,iBAAiB,OAAA;AAC1B,uDAAqD;AAA5C,mHAAA,eAAe,OAAA;AACxB,qEAAkE;AAAzD,gIAAA,qBAAqB,OAAA;AAC9B,uEAAkE;AAAzD,gIAAA,oBAAoB,OAAA;AAC7B,+DAA4D;AAAnD,0HAAA,kBAAkB,OAAA;AAC3B,mEAAkE;AAAzD,gIAAA,sBAAsB,OAAA;AAC/B,+CAAmD;AAA1C,iHAAA,iBAAiB,OAAA;AAC1B,iEAA8D;AAArD,4HAAA,mBAAmB,OAAA"}

package/dist/eval/retrieval/question-set.loader.d.ts

@@ -0,0 +1,16 @@
+import { EvalQuestionSet } from "./retrieval-eval.types";
+/**
+ * Reads a question set from YAML and validates it.
+ *
+ * Fails loudly and specifically: a malformed ground truth that loads anyway
+ * would silently weaken every later comparison, which is the opposite of what
+ * this harness exists for.
+ *
+ * Mirrors the YAML-loading style of `core/model-manager/model-loader.ts`.
+ */
+export declare class QuestionSetLoader {
+    load(params: {
+        path: string;
+    }): EvalQuestionSet;
+}
+//# sourceMappingURL=question-set.loader.d.ts.map

package/dist/eval/retrieval/question-set.loader.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"question-set.loader.d.ts","sourceRoot":"","sources":["../../../src/eval/retrieval/question-set.loader.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,eAAe,EAAyB,MAAM,wBAAwB,CAAC;AAEhF;;;;;;;;GAQG;AACH,qBACa,iBAAiB;IAC5B,IAAI,CAAC,MAAM,EAAE;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,GAAG,eAAe;CAWhD"}

package/dist/eval/retrieval/question-set.loader.js

@@ -0,0 +1,73 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.QuestionSetLoader = void 0;
+const common_1 = require("@nestjs/common");
+const fs_1 = require("fs");
+const yaml = __importStar(require("js-yaml"));
+const retrieval_eval_types_1 = require("./retrieval-eval.types");
+/**
+ * Reads a question set from YAML and validates it.
+ *
+ * Fails loudly and specifically: a malformed ground truth that loads anyway
+ * would silently weaken every later comparison, which is the opposite of what
+ * this harness exists for.
+ *
+ * Mirrors the YAML-loading style of `core/model-manager/model-loader.ts`.
+ */
+let QuestionSetLoader = class QuestionSetLoader {
+    load(params) {
+        const raw = yaml.load((0, fs_1.readFileSync)(params.path, "utf8"));
+        const parsed = retrieval_eval_types_1.evalQuestionSetSchema.safeParse(raw);
+        if (!parsed.success) {
+            const issues = parsed.error.issues
+                .map((issue) => `${issue.path.join(".") || "<root>"}: ${issue.message}`)
+                .join("; ");
+            throw new Error(`Invalid question set at ${params.path} — ${issues}`);
+        }
+        return parsed.data;
+    }
+};
+exports.QuestionSetLoader = QuestionSetLoader;
+exports.QuestionSetLoader = QuestionSetLoader = __decorate([
+    (0, common_1.Injectable)()
+], QuestionSetLoader);
+//# sourceMappingURL=question-set.loader.js.map

package/dist/eval/retrieval/question-set.loader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"question-set.loader.js","sourceRoot":"","sources":["../../../src/eval/retrieval/question-set.loader.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,2CAA4C;AAC5C,2BAAkC;AAClC,8CAAgC;AAChC,iEAAgF;AAEhF;;;;;;;;GAQG;AAEI,IAAM,iBAAiB,GAAvB,MAAM,iBAAiB;IAC5B,IAAI,CAAC,MAAwB;QAC3B,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,IAAA,iBAAY,EAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC;QACzD,MAAM,MAAM,GAAG,4CAAqB,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;QACpD,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACpB,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM;iBAC/B,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,QAAQ,KAAK,KAAK,CAAC,OAAO,EAAE,CAAC;iBACvE,IAAI,CAAC,IAAI,CAAC,CAAC;YACd,MAAM,IAAI,KAAK,CAAC,2BAA2B,MAAM,CAAC,IAAI,MAAM,MAAM,EAAE,CAAC,CAAC;QACxE,CAAC;QACD,OAAO,MAAM,CAAC,IAAI,CAAC;IACrB,CAAC;CACF,CAAA;AAZY,8CAAiB;4BAAjB,iBAAiB;IAD7B,IAAA,mBAAU,GAAE;GACA,iBAAiB,CAY7B"}

package/dist/eval/retrieval/retrieval-eval.module.d.ts

@@ -0,0 +1,14 @@
+/**
+ * Evaluation tooling. NOT imported by AgentsModule or by bootstrap — a product
+ * that never runs a sweep must not pay for this module's dependency graph.
+ * Import it explicitly from a sweep entrypoint.
+ *
+ * The three imports carry exactly the providers this module's services inject:
+ * `LLMModule` exports `LLMService` (RubricJudgeService), `ChunkModule` exports
+ * `ChunkRepository` (RetrievalSweepService) and `ResponderModule` exports
+ * `ResponderService` (EndToEndSweepService). `TokenAccountingChecker`,
+ * `EvidenceMatcher` and `QuestionSetLoader` have no injected dependencies.
+ */
+export declare class RetrievalEvalModule {
+}
+//# sourceMappingURL=retrieval-eval.module.d.ts.map

package/dist/eval/retrieval/retrieval-eval.module.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"retrieval-eval.module.d.ts","sourceRoot":"","sources":["../../../src/eval/retrieval/retrieval-eval.module.ts"],"names":[],"mappings":"AAWA;;;;;;;;;;GAUG;AACH,qBAYa,mBAAmB;CAAG"}

package/dist/eval/retrieval/retrieval-eval.module.js

@@ -0,0 +1,48 @@
+"use strict";
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RetrievalEvalModule = void 0;
+const common_1 = require("@nestjs/common");
+const responder_module_1 = require("../../agents/responder/responder.module");
+const llm_module_1 = require("../../core/llm/llm.module");
+const chunk_module_1 = require("../../foundations/chunk/chunk.module");
+const end_to_end_sweep_service_1 = require("./end-to-end-sweep.service");
+const evidence_matcher_1 = require("./evidence.matcher");
+const question_set_loader_1 = require("./question-set.loader");
+const retrieval_sweep_service_1 = require("./retrieval-sweep.service");
+const rubric_judge_service_1 = require("./rubric-judge.service");
+const token_accounting_check_1 = require("./token-accounting.check");
+/**
+ * Evaluation tooling. NOT imported by AgentsModule or by bootstrap — a product
+ * that never runs a sweep must not pay for this module's dependency graph.
+ * Import it explicitly from a sweep entrypoint.
+ *
+ * The three imports carry exactly the providers this module's services inject:
+ * `LLMModule` exports `LLMService` (RubricJudgeService), `ChunkModule` exports
+ * `ChunkRepository` (RetrievalSweepService) and `ResponderModule` exports
+ * `ResponderService` (EndToEndSweepService). `TokenAccountingChecker`,
+ * `EvidenceMatcher` and `QuestionSetLoader` have no injected dependencies.
+ */
+let RetrievalEvalModule = class RetrievalEvalModule {
+};
+exports.RetrievalEvalModule = RetrievalEvalModule;
+exports.RetrievalEvalModule = RetrievalEvalModule = __decorate([
+    (0, common_1.Module)({
+        imports: [llm_module_1.LLMModule, chunk_module_1.ChunkModule, responder_module_1.ResponderModule],
+        providers: [
+            question_set_loader_1.QuestionSetLoader,
+            evidence_matcher_1.EvidenceMatcher,
+            retrieval_sweep_service_1.RetrievalSweepService,
+            end_to_end_sweep_service_1.EndToEndSweepService,
+            rubric_judge_service_1.RubricJudgeService,
+            token_accounting_check_1.TokenAccountingChecker,
+        ],
+        exports: [question_set_loader_1.QuestionSetLoader, retrieval_sweep_service_1.RetrievalSweepService, end_to_end_sweep_service_1.EndToEndSweepService],
+    })
+], RetrievalEvalModule);
+//# sourceMappingURL=retrieval-eval.module.js.map

package/dist/eval/retrieval/retrieval-eval.module.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"retrieval-eval.module.js","sourceRoot":"","sources":["../../../src/eval/retrieval/retrieval-eval.module.ts"],"names":[],"mappings":";;;;;;;;;AAAA,2CAAwC;AACxC,8EAA0E;AAC1E,0DAAsD;AACtD,uEAAmE;AACnE,yEAAkE;AAClE,yDAAqD;AACrD,+DAA0D;AAC1D,uEAAkE;AAClE,iEAA4D;AAC5D,qEAAkE;AAElE;;;;;;;;;;GAUG;AAaI,IAAM,mBAAmB,GAAzB,MAAM,mBAAmB;CAAG,CAAA;AAAtB,kDAAmB;8BAAnB,mBAAmB;IAZ/B,IAAA,eAAM,EAAC;QACN,OAAO,EAAE,CAAC,sBAAS,EAAE,0BAAW,EAAE,kCAAe,CAAC;QAClD,SAAS,EAAE;YACT,uCAAiB;YACjB,kCAAe;YACf,+CAAqB;YACrB,+CAAoB;YACpB,yCAAkB;YAClB,+CAAsB;SACvB;QACD,OAAO,EAAE,CAAC,uCAAiB,EAAE,+CAAqB,EAAE,+CAAoB,CAAC;KAC1E,CAAC;GACW,mBAAmB,CAAG"}

package/dist/eval/retrieval/retrieval-eval.types.d.ts

@@ -0,0 +1,84 @@
+import { z } from "zod";
+/** One question with its ground truth. Authored per repo, corrected by the owner. */
+export declare const evalQuestionSchema: z.ZodObject<{
+    id: z.ZodString;
+    corpus: z.ZodString;
+    question: z.ZodString;
+    mustRetrieve: z.ZodArray<z.ZodString>;
+    rubric: z.ZodString;
+    scopeId: z.ZodOptional<z.ZodString>;
+    scopeType: z.ZodOptional<z.ZodString>;
+    scopeLabel: z.ZodOptional<z.ZodString>;
+}, z.core.$strip>;
+export type EvalQuestion = z.infer<typeof evalQuestionSchema>;
+export declare const evalQuestionSetSchema: z.ZodObject<{
+    version: z.ZodLiteral<1>;
+    product: z.ZodString;
+    questions: z.ZodArray<z.ZodObject<{
+        id: z.ZodString;
+        corpus: z.ZodString;
+        question: z.ZodString;
+        mustRetrieve: z.ZodArray<z.ZodString>;
+        rubric: z.ZodString;
+        scopeId: z.ZodOptional<z.ZodString>;
+        scopeType: z.ZodOptional<z.ZodString>;
+        scopeLabel: z.ZodOptional<z.ZodString>;
+    }, z.core.$strip>>;
+}, z.core.$strip>;
+export type EvalQuestionSet = z.infer<typeof evalQuestionSetSchema>;
+/** Why a question failed its rubric. Closed set — see spec §6.5. */
+export type RubricFailureMode = "evidence-not-retrieved" | "retrieved-but-unused" | "contradicted-source" | "hedged-without-answering";
+export interface RubricVerdict {
+    passed: boolean;
+    failureMode?: RubricFailureMode;
+    explanation: string;
+}
+/** One question's result in retrieval-only mode. */
+export interface RetrievalSweepResult {
+    questionId: string;
+    evidenceRetrieved: number;
+    missingSnippets: string[];
+    chunksRead: number;
+    retrievalMs: number;
+    error?: string;
+}
+/** One question's result in end-to-end mode. Extends the retrieval facts with answer facts. */
+export interface EndToEndSweepResult extends RetrievalSweepResult {
+    evidenceCited: number;
+    uncitedSnippets: string[];
+    verdict: RubricVerdict;
+    inputTokens: number;
+    outputTokens: number;
+    answerMs: number;
+    tokenAccounting: TokenAccountingCheck;
+    /** Snippets found in the entries the answer model actually received (post-trim). */
+    evidenceRead: number;
+    /** Notebook entries that survived the trim; 0 when readObserved is false. */
+    chunksKept: number;
+    /** Kept entries that reached the model in core (unwidened) form; 0 when readObserved is false. */
+    chunksKeptCore: number;
+    /** False when the turn produced no keptChunkIds (failed turn / node skipped). */
+    readObserved: boolean;
+}
+/** Spec §6.5 — the harness validates its own token accounting. */
+export interface TokenAccountingCheck {
+    ledgerInput: number;
+    observedInput: number;
+    ledgerOutput: number;
+    observedOutput: number;
+    agrees: boolean;
+    /**
+     * Whether the comparison actually ran. A question that never produced a turn
+     * has nothing to compare — reporting `agrees: true` there would be the one
+     * direction this mechanism must never fail (§6.5).
+     */
+    checked: boolean;
+}
+export interface SweepSummary<T extends RetrievalSweepResult> {
+    product: string;
+    mode: "retrieval-only" | "end-to-end";
+    model: string;
+    startedAt: string;
+    results: T[];
+}
+//# sourceMappingURL=retrieval-eval.types.d.ts.map

package/dist/eval/retrieval/retrieval-eval.types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"retrieval-eval.types.d.ts","sourceRoot":"","sources":["../../../src/eval/retrieval/retrieval-eval.types.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,qFAAqF;AACrF,eAAO,MAAM,kBAAkB;;;;;;;;;iBAc7B,CAAC;AACH,MAAM,MAAM,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,kBAAkB,CAAC,CAAC;AAE9D,eAAO,MAAM,qBAAqB;;;;;;;;;;;;;iBAIhC,CAAC;AACH,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,qBAAqB,CAAC,CAAC;AAEpE,oEAAoE;AACpE,MAAM,MAAM,iBAAiB,GAC3B,wBAAwB,GAAG,sBAAsB,GAAG,qBAAqB,GAAG,0BAA0B,CAAC;AAEzG,MAAM,WAAW,aAAa;IAC5B,MAAM,EAAE,OAAO,CAAC;IAChB,WAAW,CAAC,EAAE,iBAAiB,CAAC;IAChC,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,oDAAoD;AACpD,MAAM,WAAW,oBAAoB;IACnC,UAAU,EAAE,MAAM,CAAC;IACnB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,+FAA+F;AAC/F,MAAM,WAAW,mBAAoB,SAAQ,oBAAoB;IAC/D,aAAa,EAAE,MAAM,CAAC;IACtB,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,OAAO,EAAE,aAAa,CAAC;IACvB,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,QAAQ,EAAE,MAAM,CAAC;IACjB,eAAe,EAAE,oBAAoB,CAAC;IACtC,oFAAoF;IACpF,YAAY,EAAE,MAAM,CAAC;IACrB,6EAA6E;IAC7E,UAAU,EAAE,MAAM,CAAC;IACnB,kGAAkG;IAClG,cAAc,EAAE,MAAM,CAAC;IACvB,iFAAiF;IACjF,YAAY,EAAE,OAAO,CAAC;CACvB;AAED,kEAAkE;AAClE,MAAM,WAAW,oBAAoB;IACnC,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,YAAY,EAAE,MAAM,CAAC;IACrB,cAAc,EAAE,MAAM,CAAC;IACvB,MAAM,EAAE,OAAO,CAAC;IAChB;;;;OAIG;IACH,OAAO,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,YAAY,CAAC,CAAC,SAAS,oBAAoB;IAC1D,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,gBAAgB,GAAG,YAAY,CAAC;IACtC,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,CAAC,EAAE,CAAC;CACd"}

package/dist/eval/retrieval/retrieval-eval.types.js

@@ -0,0 +1,25 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.evalQuestionSetSchema = exports.evalQuestionSchema = void 0;
+// src/eval/retrieval/retrieval-eval.types.ts
+const zod_1 = require("zod");
+/** One question with its ground truth. Authored per repo, corrected by the owner. */
+exports.evalQuestionSchema = zod_1.z.object({
+    id: zod_1.z.string().describe("Stable identifier, unique within the set"),
+    corpus: zod_1.z.string().describe("Which corpus this question is asked against, e.g. 'a360ai/P10'"),
+    question: zod_1.z.string().describe("The question exactly as a user would ask it"),
+    mustRetrieve: zod_1.z.array(zod_1.z.string()).describe("Distinctive text snippets a correct answer must be grounded in"),
+    rubric: zod_1.z.string().describe("What a correct answer must assert, in one or two sentences"),
+    scopeId: zod_1.z.string().optional().describe("Scope root id when the turn is scope-confined"),
+    scopeType: zod_1.z.string().optional().describe("JSON:API type of the scope root"),
+    scopeLabel: zod_1.z
+        .string()
+        .optional()
+        .describe("Neo4j label of the scope root, e.g. 'Campaign'. Required whenever scopeId is set: the scope guard needs the label, and deriving it would pull the graph catalog into the runner."),
+});
+exports.evalQuestionSetSchema = zod_1.z.object({
+    version: zod_1.z.literal(1),
+    product: zod_1.z.string().describe("Which product this set belongs to, e.g. 'a360ai'"),
+    questions: zod_1.z.array(exports.evalQuestionSchema).min(1),
+});
+//# sourceMappingURL=retrieval-eval.types.js.map

package/dist/eval/retrieval/retrieval-eval.types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"retrieval-eval.types.js","sourceRoot":"","sources":["../../../src/eval/retrieval/retrieval-eval.types.ts"],"names":[],"mappings":";;;AAAA,6CAA6C;AAC7C,6BAAwB;AAExB,qFAAqF;AACxE,QAAA,kBAAkB,GAAG,OAAC,CAAC,MAAM,CAAC;IACzC,EAAE,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,0CAA0C,CAAC;IACnE,MAAM,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,gEAAgE,CAAC;IAC7F,QAAQ,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,6CAA6C,CAAC;IAC5E,YAAY,EAAE,OAAC,CAAC,KAAK,CAAC,OAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,CAAC,gEAAgE,CAAC;IAC5G,MAAM,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,4DAA4D,CAAC;IACzF,OAAO,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,+CAA+C,CAAC;IACxF,SAAS,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,iCAAiC,CAAC;IAC5E,UAAU,EAAE,OAAC;SACV,MAAM,EAAE;SACR,QAAQ,EAAE;SACV,QAAQ,CACP,kLAAkL,CACnL;CACJ,CAAC,CAAC;AAGU,QAAA,qBAAqB,GAAG,OAAC,CAAC,MAAM,CAAC;IAC5C,OAAO,EAAE,OAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IACrB,OAAO,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,kDAAkD,CAAC;IAChF,SAAS,EAAE,OAAC,CAAC,KAAK,CAAC,0BAAkB,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;CAC9C,CAAC,CAAC"}

package/dist/eval/retrieval/retrieval-sweep.service.d.ts

@@ -0,0 +1,35 @@
+import { ChunkRepository } from "../../foundations/chunk/repositories/chunk.repository";
+import { EvidenceMatcher } from "./evidence.matcher";
+import { EvalQuestionSet, RetrievalSweepResult, SweepSummary } from "./retrieval-eval.types";
+/**
+ * The fast inner loop: asks the retrieval layer what it WOULD read for each
+ * question and stops there. No answer is generated, so the only provider call
+ * is embedding the question.
+ *
+ * Questions run SEQUENTIALLY on purpose — `retrievalMs` is a reported metric and
+ * concurrency would make it meaningless (spec §6.3).
+ */
+export declare class RetrievalSweepService {
+    private readonly chunkRepository;
+    private readonly evidence;
+    private readonly logger;
+    constructor(chunkRepository: ChunkRepository, evidence: EvidenceMatcher);
+    /**
+     * SCOPE COMES FROM CLS, NOT FROM A PARAMETER.
+     *
+     * `ChunkRepository` reads company scope from `ClsService` (via
+     * `Neo4jService.initQuery`, which pulls `companyId`/`userId` off CLS) and the
+     * run's scope root from `AGENT_SCOPE_CLS_KEY`
+     * (`common/repositories/agent-scope.filter.ts`), which fails closed. A caller
+     * that does not establish that context retrieves nothing — or, worse,
+     * retrieves unscoped. The entry point is responsible for it; see
+     * apps/api/src/eval/run-sweep.ts.
+     *
+     * This is why `run` takes no companyId: accepting one would imply it is
+     * honoured here, and it is not.
+     */
+    run(params: {
+        set: EvalQuestionSet;
+    }): Promise<SweepSummary<RetrievalSweepResult>>;
+}
+//# sourceMappingURL=retrieval-sweep.service.d.ts.map

package/dist/eval/retrieval/retrieval-sweep.service.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"retrieval-sweep.service.d.ts","sourceRoot":"","sources":["../../../src/eval/retrieval/retrieval-sweep.service.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,eAAe,EAAE,MAAM,uDAAuD,CAAC;AACxF,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,EAAE,eAAe,EAAE,oBAAoB,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAE7F;;;;;;;GAOG;AACH,qBACa,qBAAqB;IAI9B,OAAO,CAAC,QAAQ,CAAC,eAAe;IAChC,OAAO,CAAC,QAAQ,CAAC,QAAQ;IAJ3B,OAAO,CAAC,QAAQ,CAAC,MAAM,CAA0C;gBAG9C,eAAe,EAAE,eAAe,EAChC,QAAQ,EAAE,eAAe;IAG5C;;;;;;;;;;;;;OAaG;IACG,GAAG,CAAC,MAAM,EAAE;QAAE,GAAG,EAAE,eAAe,CAAA;KAAE,GAAG,OAAO,CAAC,YAAY,CAAC,oBAAoB,CAAC,CAAC;CA0CzF"}

package/dist/eval/retrieval/retrieval-sweep.service.js

@@ -0,0 +1,93 @@
+"use strict";
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+var RetrievalSweepService_1;
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RetrievalSweepService = void 0;
+const common_1 = require("@nestjs/common");
+const chunk_repository_1 = require("../../foundations/chunk/repositories/chunk.repository");
+const evidence_matcher_1 = require("./evidence.matcher");
+/**
+ * The fast inner loop: asks the retrieval layer what it WOULD read for each
+ * question and stops there. No answer is generated, so the only provider call
+ * is embedding the question.
+ *
+ * Questions run SEQUENTIALLY on purpose — `retrievalMs` is a reported metric and
+ * concurrency would make it meaningless (spec §6.3).
+ */
+let RetrievalSweepService = RetrievalSweepService_1 = class RetrievalSweepService {
+    constructor(chunkRepository, evidence) {
+        this.chunkRepository = chunkRepository;
+        this.evidence = evidence;
+        this.logger = new common_1.Logger(RetrievalSweepService_1.name);
+    }
+    /**
+     * SCOPE COMES FROM CLS, NOT FROM A PARAMETER.
+     *
+     * `ChunkRepository` reads company scope from `ClsService` (via
+     * `Neo4jService.initQuery`, which pulls `companyId`/`userId` off CLS) and the
+     * run's scope root from `AGENT_SCOPE_CLS_KEY`
+     * (`common/repositories/agent-scope.filter.ts`), which fails closed. A caller
+     * that does not establish that context retrieves nothing — or, worse,
+     * retrieves unscoped. The entry point is responsible for it; see
+     * apps/api/src/eval/run-sweep.ts.
+     *
+     * This is why `run` takes no companyId: accepting one would imply it is
+     * honoured here, and it is not.
+     */
+    async run(params) {
+        const startedAt = new Date().toISOString();
+        const results = [];
+        for (const question of params.set.questions) {
+            const began = Date.now();
+            try {
+                const chunks = await this.chunkRepository.findPotentialChunks({
+                    question: question.question,
+                    dataLimits: {},
+                });
+                const haystack = chunks.map((chunk) => chunk.content ?? "").join("\n\n");
+                const scored = this.evidence.score({ snippets: question.mustRetrieve, haystack });
+                results.push({
+                    questionId: question.id,
+                    evidenceRetrieved: scored.found,
+                    missingSnippets: scored.missing,
+                    chunksRead: chunks.length,
+                    retrievalMs: Date.now() - began,
+                });
+            }
+            catch (error) {
+                const message = error instanceof Error ? error.message : String(error);
+                this.logger.warn(`retrieval sweep: question ${question.id} failed: ${message}`);
+                results.push({
+                    questionId: question.id,
+                    evidenceRetrieved: 0,
+                    missingSnippets: question.mustRetrieve,
+                    chunksRead: 0,
+                    retrievalMs: Date.now() - began,
+                    error: message,
+                });
+            }
+        }
+        return {
+            product: params.set.product,
+            mode: "retrieval-only",
+            model: "n/a — retrieval only",
+            startedAt,
+            results,
+        };
+    }
+};
+exports.RetrievalSweepService = RetrievalSweepService;
+exports.RetrievalSweepService = RetrievalSweepService = RetrievalSweepService_1 = __decorate([
+    (0, common_1.Injectable)(),
+    __metadata("design:paramtypes", [chunk_repository_1.ChunkRepository,
+        evidence_matcher_1.EvidenceMatcher])
+], RetrievalSweepService);
+//# sourceMappingURL=retrieval-sweep.service.js.map

package/dist/eval/retrieval/retrieval-sweep.service.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"retrieval-sweep.service.js","sourceRoot":"","sources":["../../../src/eval/retrieval/retrieval-sweep.service.ts"],"names":[],"mappings":";;;;;;;;;;;;;AAAA,2CAAoD;AACpD,4FAAwF;AACxF,yDAAqD;AAGrD;;;;;;;GAOG;AAEI,IAAM,qBAAqB,6BAA3B,MAAM,qBAAqB;IAGhC,YACmB,eAAgC,EAChC,QAAyB;QADzB,oBAAe,GAAf,eAAe,CAAiB;QAChC,aAAQ,GAAR,QAAQ,CAAiB;QAJ3B,WAAM,GAAG,IAAI,eAAM,CAAC,uBAAqB,CAAC,IAAI,CAAC,CAAC;IAK9D,CAAC;IAEJ;;;;;;;;;;;;;OAaG;IACH,KAAK,CAAC,GAAG,CAAC,MAAgC;QACxC,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAC3C,MAAM,OAAO,GAA2B,EAAE,CAAC;QAE3C,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,GAAG,CAAC,SAAS,EAAE,CAAC;YAC5C,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACzB,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,mBAAmB,CAAC;oBAC5D,QAAQ,EAAE,QAAQ,CAAC,QAAQ;oBAC3B,UAAU,EAAE,EAAE;iBACf,CAAC,CAAC;gBACH,MAAM,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBACzE,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,QAAQ,EAAE,QAAQ,CAAC,YAAY,EAAE,QAAQ,EAAE,CAAC,CAAC;gBAClF,OAAO,CAAC,IAAI,CAAC;oBACX,UAAU,EAAE,QAAQ,CAAC,EAAE;oBACvB,iBAAiB,EAAE,MAAM,CAAC,KAAK;oBAC/B,eAAe,EAAE,MAAM,CAAC,OAAO;oBAC/B,UAAU,EAAE,MAAM,CAAC,MAAM;oBACzB,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;iBAChC,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;gBACvE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,6BAA6B,QAAQ,CAAC,EAAE,YAAY,OAAO,EAAE,CAAC,CAAC;gBAChF,OAAO,CAAC,IAAI,CAAC;oBACX,UAAU,EAAE,QAAQ,CAAC,EAAE;oBACvB,iBAAiB,EAAE,CAAC;oBACpB,eAAe,EAAE,QAAQ,CAAC,YAAY;oBACtC,UAAU,EAAE,CAAC;oBACb,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;oBAC/B,KAAK,EAAE,OAAO;iBACf,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,OAAO;YACL,OAAO,EAAE,MAAM,CAAC,GAAG,CAAC,OAAO;YAC3B,IAAI,EAAE,gBAAgB;YACtB,KAAK,EAAE,sBAAsB;YAC7B,SAAS;YACT,OAAO;SACR,CAAC;IACJ,CAAC;CACF,CAAA;AAhEY,sDAAqB;gCAArB,qBAAqB;IADjC,IAAA,mBAAU,GAAE;qCAKyB,kCAAe;QACtB,kCAAe;GALjC,qBAAqB,CAgEjC"}

package/dist/eval/retrieval/rubric-judge.service.d.ts

@@ -0,0 +1,30 @@
+import { LLMService } from "../../core/llm/services/llm.service";
+import { RubricVerdict } from "./retrieval-eval.types";
+export declare const RUBRIC_JUDGE_PROMPT = "\nYou are grading one answer produced by a retrieval-augmented assistant.\n\nYou are given the user's question, a rubric stating what a correct answer MUST\nassert, the answer that was produced, and how many pieces of required evidence\nthe answer cited.\n\nDecide ONE thing: does the answer satisfy the rubric?\n\nJudge only against the rubric. Do not reward length, tone, formatting or\nconfidence. An answer that is well written but does not assert what the rubric\nrequires has failed. An answer that asserts what the rubric requires in plain\nlanguage has passed, even if it is terse.\n\nWhen the answer FAILS, classify why, choosing exactly one:\n\n- evidence-not-retrieved: the answer could not have been written because the\n  required material never reached it. Typically the citation count is zero.\n- retrieved-but-unused: the required material was cited or is clearly present,\n  and the answer still does not assert what the rubric requires.\n- contradicted-source: the answer asserts something that conflicts with the\n  material it cites.\n- hedged-without-answering: the answer declines, defers, or describes what it\n  would need, instead of answering.\n\nBe strict and be consistent. Your explanation must be one sentence naming the\nspecific rubric requirement that was met or missed.\n";
+/**
+ * Grades one answer against its rubric.
+ *
+ * A judging failure degrades to a FAILED verdict for that question rather than
+ * killing the sweep — the graceful-degradation rule in 06-llm-calls.md rule 8,
+ * applied deliberately: an ungraded question must never silently count as a pass.
+ *
+ * ATTRIBUTION EXCEPTION (06-llm-calls.md rule 5). `tokenUsageType` and
+ * `metadata` are set, but `relationshipId` / `relationshipType` are deliberately
+ * absent: a sweep grades the harness itself, so there is no tenant entity whose
+ * spend this is. LLMService skips usage persistence unless BOTH are set, which
+ * is the intended outcome here — eval spend must not land on a customer's
+ * ledger. The call is still fully identifiable in dumps and telemetry through
+ * metadata.agentName / metadata.nodeName.
+ */
+export declare class RubricJudgeService {
+    private readonly llmService;
+    private readonly logger;
+    constructor(llmService: LLMService);
+    judge(params: {
+        question: string;
+        rubric: string;
+        answer: string;
+        evidenceCited: number;
+    }): Promise<RubricVerdict>;
+}
+//# sourceMappingURL=rubric-judge.service.d.ts.map

package/dist/eval/retrieval/rubric-judge.service.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"rubric-judge.service.d.ts","sourceRoot":"","sources":["../../../src/eval/retrieval/rubric-judge.service.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,UAAU,EAAE,MAAM,qCAAqC,CAAC;AACjE,OAAO,EAAqB,aAAa,EAAE,MAAM,wBAAwB,CAAC;AAE1E,eAAO,MAAM,mBAAmB,8xCA2B/B,CAAC;AAoCF;;;;;;;;;;;;;;GAcG;AACH,qBACa,kBAAkB;IAGjB,OAAO,CAAC,QAAQ,CAAC,UAAU;IAFvC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAuC;gBAEjC,UAAU,EAAE,UAAU;IAE7C,KAAK,CAAC,MAAM,EAAE;QAClB,QAAQ,EAAE,MAAM,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,MAAM,EAAE,MAAM,CAAC;QACf,aAAa,EAAE,MAAM,CAAC;KACvB,GAAG,OAAO,CAAC,aAAa,CAAC;CAuC3B"}