@carlonicora/nestjs-neo4jsonapi 2.14.10 → 2.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agents/contextualiser/contexts/contextualiser.context.d.ts +41 -0
- package/dist/agents/contextualiser/contexts/contextualiser.context.d.ts.map +1 -1
- package/dist/agents/contextualiser/contexts/contextualiser.context.js +17 -0
- package/dist/agents/contextualiser/contexts/contextualiser.context.js.map +1 -1
- package/dist/agents/contextualiser/contexts/notebook.context.d.ts +22 -0
- package/dist/agents/contextualiser/contexts/notebook.context.d.ts.map +1 -1
- package/dist/agents/contextualiser/contexts/notebook.context.js +14 -0
- package/dist/agents/contextualiser/contexts/notebook.context.js.map +1 -1
- package/dist/agents/contextualiser/factories/contextualiser.context.factory.d.ts.map +1 -1
- package/dist/agents/contextualiser/factories/contextualiser.context.factory.js +2 -0
- package/dist/agents/contextualiser/factories/contextualiser.context.factory.js.map +1 -1
- package/dist/agents/contextualiser/nodes/atomicfacts.node.service.d.ts.map +1 -1
- package/dist/agents/contextualiser/nodes/atomicfacts.node.service.js +17 -5
- package/dist/agents/contextualiser/nodes/atomicfacts.node.service.js.map +1 -1
- package/dist/agents/contextualiser/nodes/chunk.node.service.d.ts +1 -14
- package/dist/agents/contextualiser/nodes/chunk.node.service.d.ts.map +1 -1
- package/dist/agents/contextualiser/nodes/chunk.node.service.js +56 -206
- package/dist/agents/contextualiser/nodes/chunk.node.service.js.map +1 -1
- package/dist/agents/contextualiser/nodes/chunk.vector.node.service.d.ts +3 -12
- package/dist/agents/contextualiser/nodes/chunk.vector.node.service.d.ts.map +1 -1
- package/dist/agents/contextualiser/nodes/chunk.vector.node.service.js +44 -133
- package/dist/agents/contextualiser/nodes/chunk.vector.node.service.js.map +1 -1
- package/dist/agents/contextualiser/nodes/keyconcepts.node.service.d.ts +1 -2
- package/dist/agents/contextualiser/nodes/keyconcepts.node.service.d.ts.map +1 -1
- package/dist/agents/contextualiser/nodes/keyconcepts.node.service.js +60 -60
- package/dist/agents/contextualiser/nodes/keyconcepts.node.service.js.map +1 -1
- package/dist/agents/contextualiser/nodes/question.refiner.node.service.d.ts.map +1 -1
- package/dist/agents/contextualiser/nodes/question.refiner.node.service.js +7 -2
- package/dist/agents/contextualiser/nodes/question.refiner.node.service.js.map +1 -1
- package/dist/agents/contextualiser/nodes/rational.node.service.js +2 -1
- package/dist/agents/contextualiser/nodes/rational.node.service.js.map +1 -1
- package/dist/agents/contextualiser/services/contextualiser.service.d.ts.map +1 -1
- package/dist/agents/contextualiser/services/contextualiser.service.js +30 -22
- package/dist/agents/contextualiser/services/contextualiser.service.js.map +1 -1
- package/dist/agents/prompts/index.d.ts +0 -2
- package/dist/agents/prompts/index.d.ts.map +1 -1
- package/dist/agents/prompts/index.js +7 -5
- package/dist/agents/prompts/index.js.map +1 -1
- package/dist/agents/responder/contexts/responder.context.d.ts +96 -0
- package/dist/agents/responder/contexts/responder.context.d.ts.map +1 -1
- package/dist/agents/responder/interfaces/unified.trace.interface.d.ts +9 -0
- package/dist/agents/responder/interfaces/unified.trace.interface.d.ts.map +1 -1
- package/dist/agents/responder/nodes/__tests__/helpers/answer-node.harness.d.ts +38 -0
- package/dist/agents/responder/nodes/__tests__/helpers/answer-node.harness.d.ts.map +1 -0
- package/dist/agents/responder/nodes/__tests__/helpers/answer-node.harness.js +36 -0
- package/dist/agents/responder/nodes/__tests__/helpers/answer-node.harness.js.map +1 -0
- package/dist/agents/responder/nodes/graph.node.service.d.ts.map +1 -1
- package/dist/agents/responder/nodes/graph.node.service.js +18 -3
- package/dist/agents/responder/nodes/graph.node.service.js.map +1 -1
- package/dist/agents/responder/nodes/planner.node.service.d.ts.map +1 -1
- package/dist/agents/responder/nodes/planner.node.service.js +9 -0
- package/dist/agents/responder/nodes/planner.node.service.js.map +1 -1
- package/dist/agents/responder/nodes/responder.answer.node.service.d.ts +26 -1
- package/dist/agents/responder/nodes/responder.answer.node.service.d.ts.map +1 -1
- package/dist/agents/responder/nodes/responder.answer.node.service.js +127 -13
- package/dist/agents/responder/nodes/responder.answer.node.service.js.map +1 -1
- package/dist/agents/responder/services/responder.service.d.ts.map +1 -1
- package/dist/agents/responder/services/responder.service.js +4 -0
- package/dist/agents/responder/services/responder.service.js.map +1 -1
- package/dist/common/utils/settled.d.ts +33 -0
- package/dist/common/utils/settled.d.ts.map +1 -0
- package/dist/common/utils/settled.js +66 -0
- package/dist/common/utils/settled.js.map +1 -0
- package/dist/config/interfaces/config.prompts.interface.d.ts +12 -0
- package/dist/config/interfaces/config.prompts.interface.d.ts.map +1 -1
- package/dist/core/llm/services/model.service.d.ts.map +1 -1
- package/dist/core/llm/services/model.service.js +5 -2
- package/dist/core/llm/services/model.service.js.map +1 -1
- package/dist/core/llm/services/reasoning-content-fetch.d.ts +41 -0
- package/dist/core/llm/services/reasoning-content-fetch.d.ts.map +1 -0
- package/dist/core/llm/services/reasoning-content-fetch.js +143 -0
- package/dist/core/llm/services/reasoning-content-fetch.js.map +1 -0
- package/dist/eval/retrieval/end-to-end-sweep.service.d.ts +52 -0
- package/dist/eval/retrieval/end-to-end-sweep.service.d.ts.map +1 -0
- package/dist/eval/retrieval/end-to-end-sweep.service.js +194 -0
- package/dist/eval/retrieval/end-to-end-sweep.service.js.map +1 -0
- package/dist/eval/retrieval/evidence.matcher.d.ts +23 -0
- package/dist/eval/retrieval/evidence.matcher.d.ts.map +1 -0
- package/dist/eval/retrieval/evidence.matcher.js +38 -0
- package/dist/eval/retrieval/evidence.matcher.js.map +1 -0
- package/dist/eval/retrieval/index.d.ts +10 -0
- package/dist/eval/retrieval/index.d.ts.map +1 -0
- package/dist/eval/retrieval/index.js +35 -0
- package/dist/eval/retrieval/index.js.map +1 -0
- package/dist/eval/retrieval/question-set.loader.d.ts +16 -0
- package/dist/eval/retrieval/question-set.loader.d.ts.map +1 -0
- package/dist/eval/retrieval/question-set.loader.js +73 -0
- package/dist/eval/retrieval/question-set.loader.js.map +1 -0
- package/dist/eval/retrieval/retrieval-eval.module.d.ts +14 -0
- package/dist/eval/retrieval/retrieval-eval.module.d.ts.map +1 -0
- package/dist/eval/retrieval/retrieval-eval.module.js +48 -0
- package/dist/eval/retrieval/retrieval-eval.module.js.map +1 -0
- package/dist/eval/retrieval/retrieval-eval.types.d.ts +84 -0
- package/dist/eval/retrieval/retrieval-eval.types.d.ts.map +1 -0
- package/dist/eval/retrieval/retrieval-eval.types.js +25 -0
- package/dist/eval/retrieval/retrieval-eval.types.js.map +1 -0
- package/dist/eval/retrieval/retrieval-sweep.service.d.ts +35 -0
- package/dist/eval/retrieval/retrieval-sweep.service.d.ts.map +1 -0
- package/dist/eval/retrieval/retrieval-sweep.service.js +93 -0
- package/dist/eval/retrieval/retrieval-sweep.service.js.map +1 -0
- package/dist/eval/retrieval/rubric-judge.service.d.ts +30 -0
- package/dist/eval/retrieval/rubric-judge.service.d.ts.map +1 -0
- package/dist/eval/retrieval/rubric-judge.service.js +138 -0
- package/dist/eval/retrieval/rubric-judge.service.js.map +1 -0
- package/dist/eval/retrieval/sweep-report.d.ts +11 -0
- package/dist/eval/retrieval/sweep-report.d.ts.map +1 -0
- package/dist/eval/retrieval/sweep-report.js +79 -0
- package/dist/eval/retrieval/sweep-report.js.map +1 -0
- package/dist/eval/retrieval/token-accounting.check.d.ts +39 -0
- package/dist/eval/retrieval/token-accounting.check.d.ts.map +1 -0
- package/dist/eval/retrieval/token-accounting.check.js +75 -0
- package/dist/eval/retrieval/token-accounting.check.js.map +1 -0
- package/dist/foundations/atomicfact/repositories/atomicfact.repository.d.ts.map +1 -1
- package/dist/foundations/atomicfact/repositories/atomicfact.repository.js +7 -0
- package/dist/foundations/atomicfact/repositories/atomicfact.repository.js.map +1 -1
- package/dist/foundations/chunk/repositories/chunk.repository.d.ts +87 -1
- package/dist/foundations/chunk/repositories/chunk.repository.d.ts.map +1 -1
- package/dist/foundations/chunk/repositories/chunk.repository.js +239 -50
- package/dist/foundations/chunk/repositories/chunk.repository.js.map +1 -1
- package/dist/foundations/chunk/repositories/retrieval.constants.d.ts +69 -0
- package/dist/foundations/chunk/repositories/retrieval.constants.d.ts.map +1 -0
- package/dist/foundations/chunk/repositories/retrieval.constants.js +72 -0
- package/dist/foundations/chunk/repositories/retrieval.constants.js.map +1 -0
- package/dist/foundations/keyconcept/repositories/keyconcept.repository.d.ts +2 -0
- package/dist/foundations/keyconcept/repositories/keyconcept.repository.d.ts.map +1 -1
- package/dist/foundations/keyconcept/repositories/keyconcept.repository.js +9 -5
- package/dist/foundations/keyconcept/repositories/keyconcept.repository.js.map +1 -1
- package/package.json +1 -1
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
|
|
3
|
+
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
4
|
+
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
5
|
+
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
6
|
+
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
7
|
+
};
|
|
8
|
+
var __metadata = (this && this.__metadata) || function (k, v) {
|
|
9
|
+
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
|
|
10
|
+
};
|
|
11
|
+
var RubricJudgeService_1;
|
|
12
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
13
|
+
exports.RubricJudgeService = exports.RUBRIC_JUDGE_PROMPT = void 0;
|
|
14
|
+
const common_1 = require("@nestjs/common");
|
|
15
|
+
const zod_1 = require("zod");
|
|
16
|
+
const llm_service_1 = require("../../core/llm/services/llm.service");
|
|
17
|
+
exports.RUBRIC_JUDGE_PROMPT = `
|
|
18
|
+
You are grading one answer produced by a retrieval-augmented assistant.
|
|
19
|
+
|
|
20
|
+
You are given the user's question, a rubric stating what a correct answer MUST
|
|
21
|
+
assert, the answer that was produced, and how many pieces of required evidence
|
|
22
|
+
the answer cited.
|
|
23
|
+
|
|
24
|
+
Decide ONE thing: does the answer satisfy the rubric?
|
|
25
|
+
|
|
26
|
+
Judge only against the rubric. Do not reward length, tone, formatting or
|
|
27
|
+
confidence. An answer that is well written but does not assert what the rubric
|
|
28
|
+
requires has failed. An answer that asserts what the rubric requires in plain
|
|
29
|
+
language has passed, even if it is terse.
|
|
30
|
+
|
|
31
|
+
When the answer FAILS, classify why, choosing exactly one:
|
|
32
|
+
|
|
33
|
+
- evidence-not-retrieved: the answer could not have been written because the
|
|
34
|
+
required material never reached it. Typically the citation count is zero.
|
|
35
|
+
- retrieved-but-unused: the required material was cited or is clearly present,
|
|
36
|
+
and the answer still does not assert what the rubric requires.
|
|
37
|
+
- contradicted-source: the answer asserts something that conflicts with the
|
|
38
|
+
material it cites.
|
|
39
|
+
- hedged-without-answering: the answer declines, defers, or describes what it
|
|
40
|
+
would need, instead of answering.
|
|
41
|
+
|
|
42
|
+
Be strict and be consistent. Your explanation must be one sentence naming the
|
|
43
|
+
specific rubric requirement that was met or missed.
|
|
44
|
+
`;
|
|
45
|
+
/**
|
|
46
|
+
* The closed set of failure modes (spec §6.5). Declared once so the schema the
|
|
47
|
+
* model is constrained to and the runtime guard below can never drift apart.
|
|
48
|
+
*/
|
|
49
|
+
const FAILURE_MODES = [
|
|
50
|
+
"evidence-not-retrieved",
|
|
51
|
+
"retrieved-but-unused",
|
|
52
|
+
"contradicted-source",
|
|
53
|
+
"hedged-without-answering",
|
|
54
|
+
];
|
|
55
|
+
const outputSchema = zod_1.z.object({
|
|
56
|
+
passed: zod_1.z.boolean().describe("True only if the answer asserts everything the rubric requires"),
|
|
57
|
+
failureMode: zod_1.z
|
|
58
|
+
.enum(FAILURE_MODES)
|
|
59
|
+
.optional()
|
|
60
|
+
.describe("Present if and only if passed is false. One of the four named modes, verbatim."),
|
|
61
|
+
explanation: zod_1.z.string().describe("One sentence naming the specific rubric requirement that was met or missed"),
|
|
62
|
+
});
|
|
63
|
+
const inputSchema = zod_1.z.object({
|
|
64
|
+
question: zod_1.z.string().describe("The user's question, verbatim"),
|
|
65
|
+
rubric: zod_1.z
|
|
66
|
+
.string()
|
|
67
|
+
.describe("BINDING CRITERION. What a correct answer must assert. Judge against this and nothing else."),
|
|
68
|
+
answer: zod_1.z.string().describe("The answer under test. Grade it; do not improve or continue it."),
|
|
69
|
+
evidenceCited: zod_1.z
|
|
70
|
+
.number()
|
|
71
|
+
.describe("How many required evidence snippets the answer cited. Zero suggests evidence-not-retrieved."),
|
|
72
|
+
});
|
|
73
|
+
const isFailureMode = (value) => typeof value === "string" && FAILURE_MODES.includes(value);
|
|
74
|
+
/**
|
|
75
|
+
* Grades one answer against its rubric.
|
|
76
|
+
*
|
|
77
|
+
* A judging failure degrades to a FAILED verdict for that question rather than
|
|
78
|
+
* killing the sweep — the graceful-degradation rule in 06-llm-calls.md rule 8,
|
|
79
|
+
* applied deliberately: an ungraded question must never silently count as a pass.
|
|
80
|
+
*
|
|
81
|
+
* ATTRIBUTION EXCEPTION (06-llm-calls.md rule 5). `tokenUsageType` and
|
|
82
|
+
* `metadata` are set, but `relationshipId` / `relationshipType` are deliberately
|
|
83
|
+
* absent: a sweep grades the harness itself, so there is no tenant entity whose
|
|
84
|
+
* spend this is. LLMService skips usage persistence unless BOTH are set, which
|
|
85
|
+
* is the intended outcome here — eval spend must not land on a customer's
|
|
86
|
+
* ledger. The call is still fully identifiable in dumps and telemetry through
|
|
87
|
+
* metadata.agentName / metadata.nodeName.
|
|
88
|
+
*/
|
|
89
|
+
let RubricJudgeService = RubricJudgeService_1 = class RubricJudgeService {
|
|
90
|
+
constructor(llmService) {
|
|
91
|
+
this.llmService = llmService;
|
|
92
|
+
this.logger = new common_1.Logger(RubricJudgeService_1.name);
|
|
93
|
+
}
|
|
94
|
+
async judge(params) {
|
|
95
|
+
try {
|
|
96
|
+
const response = await this.llmService.call({
|
|
97
|
+
inputSchema,
|
|
98
|
+
inputParams: {
|
|
99
|
+
question: params.question,
|
|
100
|
+
rubric: params.rubric,
|
|
101
|
+
answer: params.answer,
|
|
102
|
+
evidenceCited: params.evidenceCited,
|
|
103
|
+
},
|
|
104
|
+
outputSchema,
|
|
105
|
+
systemPrompts: [exports.RUBRIC_JUDGE_PROMPT],
|
|
106
|
+
// A grading pass is judgement, not creation: no sampling variance, so
|
|
107
|
+
// the same answer is graded the same way on every run.
|
|
108
|
+
temperature: 0,
|
|
109
|
+
// Deliberate for THIS call (06-llm-calls.md rule 6 / checkpoint step 5):
|
|
110
|
+
// stated here rather than inherited from the tier, so a deployment that
|
|
111
|
+
// retunes the default cannot silently change how the baseline is graded.
|
|
112
|
+
// "low" matches the model this baseline is measured on (gpt-5.6-luna).
|
|
113
|
+
reasoningEffort: "low",
|
|
114
|
+
tokenUsageType: "retrieval_eval",
|
|
115
|
+
metadata: { agentName: "retrieval-eval", nodeName: "rubric-judge" },
|
|
116
|
+
});
|
|
117
|
+
// The schema already constrains the model; this guard keeps the returned
|
|
118
|
+
// union honest even when the provider falls back to lenient parsing.
|
|
119
|
+
const failureMode = isFailureMode(response.failureMode) ? response.failureMode : undefined;
|
|
120
|
+
return {
|
|
121
|
+
passed: response.passed,
|
|
122
|
+
...(failureMode ? { failureMode } : {}),
|
|
123
|
+
explanation: response.explanation,
|
|
124
|
+
};
|
|
125
|
+
}
|
|
126
|
+
catch (error) {
|
|
127
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
128
|
+
this.logger.warn(`rubric judge failed, recording the question as failed: ${message}`);
|
|
129
|
+
return { passed: false, explanation: `Judge unavailable: ${message}` };
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
};
|
|
133
|
+
exports.RubricJudgeService = RubricJudgeService;
|
|
134
|
+
exports.RubricJudgeService = RubricJudgeService = RubricJudgeService_1 = __decorate([
|
|
135
|
+
(0, common_1.Injectable)(),
|
|
136
|
+
__metadata("design:paramtypes", [llm_service_1.LLMService])
|
|
137
|
+
], RubricJudgeService);
|
|
138
|
+
//# sourceMappingURL=rubric-judge.service.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"rubric-judge.service.js","sourceRoot":"","sources":["../../../src/eval/retrieval/rubric-judge.service.ts"],"names":[],"mappings":";;;;;;;;;;;;;AAAA,2CAAoD;AACpD,6BAAwB;AACxB,qEAAiE;AAGpD,QAAA,mBAAmB,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;CA2BlC,CAAC;AAEF;;;GAGG;AACH,MAAM,aAAa,GAAG;IACpB,wBAAwB;IACxB,sBAAsB;IACtB,qBAAqB;IACrB,0BAA0B;CAClB,CAAC;AAEX,MAAM,YAAY,GAAG,OAAC,CAAC,MAAM,CAAC;IAC5B,MAAM,EAAE,OAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,CAAC,gEAAgE,CAAC;IAC9F,WAAW,EAAE,OAAC;SACX,IAAI,CAAC,aAAa,CAAC;SACnB,QAAQ,EAAE;SACV,QAAQ,CAAC,gFAAgF,CAAC;IAC7F,WAAW,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,4EAA4E,CAAC;CAC/G,CAAC,CAAC;AAEH,MAAM,WAAW,GAAG,OAAC,CAAC,MAAM,CAAC;IAC3B,QAAQ,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,+BAA+B,CAAC;IAC9D,MAAM,EAAE,OAAC;SACN,MAAM,EAAE;SACR,QAAQ,CAAC,4FAA4F,CAAC;IACzG,MAAM,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,iEAAiE,CAAC;IAC9F,aAAa,EAAE,OAAC;SACb,MAAM,EAAE;SACR,QAAQ,CAAC,6FAA6F,CAAC;CAC3G,CAAC,CAAC;AAEH,MAAM,aAAa,GAAG,CAAC,KAAc,EAA8B,EAAE,CACnE,OAAO,KAAK,KAAK,QAAQ,IAAK,aAAmC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;AAEpF;;;;;;;;;;;;;;GAcG;AAEI,IAAM,kBAAkB,0BAAxB,MAAM,kBAAkB;IAG7B,YAA6B,UAAsB;QAAtB,eAAU,GAAV,UAAU,CAAY;QAFlC,WAAM,GAAG,IAAI,eAAM,CAAC,oBAAkB,CAAC,IAAI,CAAC,CAAC;IAER,CAAC;IAEvD,KAAK,CAAC,KAAK,CAAC,MAKX;QACC,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,CAA+B;gBACxE,WAAW;gBACX,WAAW,EAAE;oBACX,QAAQ,EAAE,MAAM,CAAC,QAAQ;oBACzB,MAAM,EAAE,MAAM,CAAC,MAAM;oBACrB,MAAM,EAAE,MAAM,CAAC,MAAM;oBACrB,aAAa,EAAE,MAAM,CAAC,aAAa;iBACpC;gBACD,YAAY;gBACZ,aAAa,EAAE,CAAC,2BAAmB,CAAC;gBACpC,sEAAsE;gBACtE,uDAAuD;gBACvD,WAAW,EAAE,CAAC;gBACd,yEAAyE;gBACzE,wEAAwE;gBACxE,yEAAyE;gBACzE,uEAAuE;gBACvE,eAAe,EAAE,KAAK;gBACtB,cAAc,EAAE,gBAAgB;gBAChC,QAAQ,EAAE,EAAE,SAAS,EAAE,gBAAgB,EAAE,QAAQ,EAAE,cAAc,EAAE;aACpE,CAAC,CAAC;YAEH,yEAAyE;YACzE,qEAAqE;YACrE,MAAM,WAAW,GAAG,aAAa,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS,CAAC;YAE3F,OAAO;gBACL,MAAM,EAAE,QAAQ,CAAC,MAAM;gBACvB,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACvC,WAAW,EAAE,QAAQ,CAAC,WAAW;aAClC,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACvE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,0DAA0D,OAAO,EAAE,CAAC,CAAC;YACtF,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,sBAAsB,OAAO,EAAE,EAAE,CAAC;QACzE,CAAC;IACH,CAAC;CACF,CAAA;AAjDY,gDAAkB;6BAAlB,kBAAkB;IAD9B,IAAA,mBAAU,GAAE;qCAI8B,wBAAU;GAHxC,kBAAkB,CAiD9B"}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import { RetrievalSweepResult, SweepSummary } from "./retrieval-eval.types";
|
|
2
|
+
/**
|
|
3
|
+
* Renders a sweep to markdown, leading with the number that decides whether a
|
|
4
|
+
* change ships.
|
|
5
|
+
*
|
|
6
|
+
* A token-accounting mismatch is rendered as a banner, not a column: a sweep
|
|
7
|
+
* whose own arithmetic does not close cannot be used as a baseline, and that
|
|
8
|
+
* must not be discoverable only by reading a table.
|
|
9
|
+
*/
|
|
10
|
+
export declare function renderSweepReport(summary: SweepSummary<RetrievalSweepResult>): string;
|
|
11
|
+
//# sourceMappingURL=sweep-report.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"sweep-report.d.ts","sourceRoot":"","sources":["../../../src/eval/retrieval/sweep-report.ts"],"names":[],"mappings":"AAAA,OAAO,EAAuB,oBAAoB,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AASjG;;;;;;;GAOG;AACH,wBAAgB,iBAAiB,CAAC,OAAO,EAAE,YAAY,CAAC,oBAAoB,CAAC,GAAG,MAAM,CA8ErF"}
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.renderSweepReport = renderSweepReport;
|
|
4
|
+
const isEndToEnd = (result) => result.verdict !== undefined;
|
|
5
|
+
/** Total `mustRetrieve` snippets for the question — same denominator `evidenceRetrieved` uses. */
|
|
6
|
+
const questionSnippetTotal = (result) => result.evidenceRetrieved + result.missingSnippets.length;
|
|
7
|
+
/**
|
|
8
|
+
* Renders a sweep to markdown, leading with the number that decides whether a
|
|
9
|
+
* change ships.
|
|
10
|
+
*
|
|
11
|
+
* A token-accounting mismatch is rendered as a banner, not a column: a sweep
|
|
12
|
+
* whose own arithmetic does not close cannot be used as a baseline, and that
|
|
13
|
+
* must not be discoverable only by reading a table.
|
|
14
|
+
*/
|
|
15
|
+
function renderSweepReport(summary) {
|
|
16
|
+
const lines = [];
|
|
17
|
+
const graded = summary.results.filter(isEndToEnd);
|
|
18
|
+
const passed = graded.filter((result) => result.verdict.passed).length;
|
|
19
|
+
lines.push(`# Retrieval sweep — ${summary.product} (${summary.mode})`);
|
|
20
|
+
lines.push("");
|
|
21
|
+
lines.push(`Model \`${summary.model}\` · started ${summary.startedAt}`);
|
|
22
|
+
lines.push("");
|
|
23
|
+
// `checked &&` on purpose: a question whose turn never completed carries
|
|
24
|
+
// `checked: false, agrees: false`, and it is already reported as a failure
|
|
25
|
+
// below. Counting it here too would announce a token mismatch that was never
|
|
26
|
+
// measured and cast doubt on a sweep whose arithmetic is fine.
|
|
27
|
+
const mismatched = graded.filter((result) => result.tokenAccounting.checked && !result.tokenAccounting.agrees);
|
|
28
|
+
if (mismatched.length > 0) {
|
|
29
|
+
lines.push(`> **TOKEN ACCOUNTING MISMATCH on ${mismatched.length} question(s).**`);
|
|
30
|
+
lines.push("> This sweep cannot be used as a baseline until the discrepancy is explained.");
|
|
31
|
+
for (const result of mismatched) {
|
|
32
|
+
lines.push(`> \`${result.questionId}\`: reported ${result.tokenAccounting.ledgerInput}/` +
|
|
33
|
+
`${result.tokenAccounting.ledgerOutput}, observed ` +
|
|
34
|
+
`${result.tokenAccounting.observedInput}/${result.tokenAccounting.observedOutput}`);
|
|
35
|
+
}
|
|
36
|
+
lines.push("");
|
|
37
|
+
}
|
|
38
|
+
if (graded.length > 0) {
|
|
39
|
+
const totalMustRetrieve = graded.reduce((sum, result) => sum + questionSnippetTotal(result), 0);
|
|
40
|
+
const totalEvidenceRead = graded.reduce((sum, result) => sum + result.evidenceRead, 0);
|
|
41
|
+
lines.push(`**Rubric passes: ${passed}/${graded.length}** · evidence read ${totalEvidenceRead}/${totalMustRetrieve}`);
|
|
42
|
+
lines.push("");
|
|
43
|
+
lines.push("| Question | Verdict | Failure mode | Evidence retrieved | Evidence read | Evidence cited | Chunks | Chunks kept | Chunks kept core | Tokens in/out | Answer ms |");
|
|
44
|
+
lines.push("|---|---|---|---:|---:|---:|---:|---:|---:|---:|---:|");
|
|
45
|
+
for (const result of graded) {
|
|
46
|
+
lines.push(`| \`${result.questionId}\` | ${result.verdict.passed ? "pass" : "**fail**"} | ` +
|
|
47
|
+
`${result.verdict.failureMode ?? "—"} | ` +
|
|
48
|
+
`${result.evidenceRetrieved}/${result.evidenceRetrieved + result.missingSnippets.length} | ` +
|
|
49
|
+
`${result.readObserved ? `${result.evidenceRead}/${questionSnippetTotal(result)}` : "n/a"} | ` +
|
|
50
|
+
`${result.evidenceCited}/${result.evidenceCited + result.uncitedSnippets.length} | ` +
|
|
51
|
+
`${result.chunksRead} | ${result.chunksKept} | ${result.chunksKeptCore} | ${result.inputTokens}/${result.outputTokens} | ${result.answerMs} |`);
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
else {
|
|
55
|
+
lines.push("| Question | Evidence retrieved | Chunks | Retrieval ms |");
|
|
56
|
+
lines.push("|---|---:|---:|---:|");
|
|
57
|
+
for (const result of summary.results) {
|
|
58
|
+
lines.push(`| \`${result.questionId}\` | ` +
|
|
59
|
+
`${result.evidenceRetrieved}/${result.evidenceRetrieved + result.missingSnippets.length} | ` +
|
|
60
|
+
`${result.chunksRead} | ${result.retrievalMs} |`);
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
const failures = graded.filter((result) => !result.verdict.passed);
|
|
64
|
+
if (failures.length > 0) {
|
|
65
|
+
lines.push("");
|
|
66
|
+
lines.push("## Failures");
|
|
67
|
+
for (const result of failures) {
|
|
68
|
+
lines.push("");
|
|
69
|
+
lines.push(`### \`${result.questionId}\` — ${result.verdict.failureMode ?? "ungraded"}`);
|
|
70
|
+
lines.push(result.verdict.explanation);
|
|
71
|
+
if (result.missingSnippets.length > 0) {
|
|
72
|
+
lines.push("");
|
|
73
|
+
lines.push(`Never retrieved: ${result.missingSnippets.map((s) => `"${s}"`).join(", ")}`);
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
return lines.join("\n");
|
|
78
|
+
}
|
|
79
|
+
//# sourceMappingURL=sweep-report.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"sweep-report.js","sourceRoot":"","sources":["../../../src/eval/retrieval/sweep-report.ts"],"names":[],"mappings":";;AAiBA,8CA8EC;AA7FD,MAAM,UAAU,GAAG,CAAC,MAA4B,EAAiC,EAAE,CAChF,MAA8B,CAAC,OAAO,KAAK,SAAS,CAAC;AAExD,kGAAkG;AAClG,MAAM,oBAAoB,GAAG,CAAC,MAA2B,EAAU,EAAE,CACnE,MAAM,CAAC,iBAAiB,GAAG,MAAM,CAAC,eAAe,CAAC,MAAM,CAAC;AAE3D;;;;;;;GAOG;AACH,SAAgB,iBAAiB,CAAC,OAA2C;IAC3E,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;IAClD,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IAEvE,KAAK,CAAC,IAAI,CAAC,uBAAuB,OAAO,CAAC,OAAO,KAAK,OAAO,CAAC,IAAI,GAAG,CAAC,CAAC;IACvE,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,WAAW,OAAO,CAAC,KAAK,gBAAgB,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC;IACxE,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,yEAAyE;IACzE,2EAA2E;IAC3E,6EAA6E;IAC7E,+DAA+D;IAC/D,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,eAAe,CAAC,OAAO,IAAI,CAAC,MAAM,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC;IAC/G,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC1B,KAAK,CAAC,IAAI,CAAC,oCAAoC,UAAU,CAAC,MAAM,iBAAiB,CAAC,CAAC;QACnF,KAAK,CAAC,IAAI,CAAC,+EAA+E,CAAC,CAAC;QAC5F,KAAK,MAAM,MAAM,IAAI,UAAU,EAAE,CAAC;YAChC,KAAK,CAAC,IAAI,CACR,OAAO,MAAM,CAAC,UAAU,gBAAgB,MAAM,CAAC,eAAe,CAAC,WAAW,GAAG;gBAC3E,GAAG,MAAM,CAAC,eAAe,CAAC,YAAY,aAAa;gBACnD,GAAG,MAAM,CAAC,eAAe,CAAC,aAAa,IAAI,MAAM,CAAC,eAAe,CAAC,cAAc,EAAE,CACrF,CAAC;QACJ,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IAED,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtB,MAAM,iBAAiB,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE,CAAC,GAAG,GAAG,oBAAoB,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;QAChG,MAAM,iBAAiB,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE,CAAC,GAAG,GAAG,MAAM,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC;QACvF,KAAK,CAAC,IAAI,CACR,oBAAoB,MAAM,IAAI,MAAM,CAAC,MAAM,sBAAsB,iBAAiB,IAAI,iBAAiB,EAAE,CAC1G,CAAC;QACF,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAEf,KAAK,CAAC,IAAI,CACR,mKAAmK,CACpK,CAAC;QACF,KAAK,CAAC,IAAI,CAAC,uDAAuD,CAAC,CAAC;QACpE,KAAK,MAAM,MAAM,IAAI,MAAM,EAAE,CAAC;YAC5B,KAAK,CAAC,IAAI,CACR,OAAO,MAAM,CAAC,UAAU,QAAQ,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,UAAU,KAAK;gBAC9E,GAAG,MAAM,CAAC,OAAO,CAAC,WAAW,IAAI,GAAG,KAAK;gBACzC,GAAG,MAAM,CAAC,iBAAiB,IAAI,MAAM,CAAC,iBAAiB,GAAG,MAAM,CAAC,eAAe,CAAC,MAAM,KAAK;gBAC5F,GAAG,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,YAAY,IAAI,oBAAoB,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK;gBAC9F,GAAG,MAAM,CAAC,aAAa,IAAI,MAAM,CAAC,aAAa,GAAG,MAAM,CAAC,eAAe,CAAC,MAAM,KAAK;gBACpF,GAAG,MAAM,CAAC,UAAU,MAAM,MAAM,CAAC,UAAU,MAAM,MAAM,CAAC,cAAc,MAAM,MAAM,CAAC,WAAW,IAAI,MAAM,CAAC,YAAY,MAAM,MAAM,CAAC,QAAQ,IAAI,CACjJ,CAAC;QACJ,CAAC;IACH,CAAC;SAAM,CAAC;QACN,KAAK,CAAC,IAAI,CAAC,2DAA2D,CAAC,CAAC;QACxE,KAAK,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC;QACnC,KAAK,MAAM,MAAM,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;YACrC,KAAK,CAAC,IAAI,CACR,OAAO,MAAM,CAAC,UAAU,OAAO;gBAC7B,GAAG,MAAM,CAAC,iBAAiB,IAAI,MAAM,CAAC,iBAAiB,GAAG,MAAM,CAAC,eAAe,CAAC,MAAM,KAAK;gBAC5F,GAAG,MAAM,CAAC,UAAU,MAAM,MAAM,CAAC,WAAW,IAAI,CACnD,CAAC;QACJ,CAAC;IACH,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IACnE,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxB,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAC1B,KAAK,MAAM,MAAM,IAAI,QAAQ,EAAE,CAAC;YAC9B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACf,KAAK,CAAC,IAAI,CAAC,SAAS,MAAM,CAAC,UAAU,QAAQ,MAAM,CAAC,OAAO,CAAC,WAAW,IAAI,UAAU,EAAE,CAAC,CAAC;YACzF,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;YACvC,IAAI,MAAM,CAAC,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACtC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gBACf,KAAK,CAAC,IAAI,CAAC,oBAAoB,MAAM,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAC3F,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC"}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import { TokenAccountingCheck } from "./retrieval-eval.types";
|
|
2
|
+
/**
|
|
3
|
+
* Cross-checks a turn's reported token aggregate against the sum of the calls
|
|
4
|
+
* actually observed for it.
|
|
5
|
+
*
|
|
6
|
+
* Spec §6.5. The `2C + A` reducer bug inflated every archived chat figure and
|
|
7
|
+
* went unnoticed for a full measurement cycle; this check is what makes the
|
|
8
|
+
* same class of bug loud instead of silent. Provider-reported per-call figures
|
|
9
|
+
* are the source of truth.
|
|
10
|
+
*/
|
|
11
|
+
export declare class TokenAccountingChecker {
|
|
12
|
+
private readonly logger;
|
|
13
|
+
private observedInput;
|
|
14
|
+
private observedOutput;
|
|
15
|
+
/** Record one provider call's reported usage for the question in flight. */
|
|
16
|
+
observe(params: {
|
|
17
|
+
input: number;
|
|
18
|
+
output: number;
|
|
19
|
+
}): void;
|
|
20
|
+
/**
|
|
21
|
+
* Clears the accumulator WITHOUT producing a verdict.
|
|
22
|
+
*
|
|
23
|
+
* For the question that never completed a turn: it has nothing to compare, but
|
|
24
|
+
* it may still have deposited observations before it threw. Leaving them in
|
|
25
|
+
* place attributes them to the NEXT question — which either invents a mismatch
|
|
26
|
+
* that is not there or, worse, masks one that is. `check()` resets as a side
|
|
27
|
+
* effect of comparing; this is the same reset for the path that cannot compare.
|
|
28
|
+
*/
|
|
29
|
+
reset(): void;
|
|
30
|
+
/** Compare and RESET, so the next question starts clean. */
|
|
31
|
+
check(params: {
|
|
32
|
+
questionId: string;
|
|
33
|
+
ledger: {
|
|
34
|
+
input: number;
|
|
35
|
+
output: number;
|
|
36
|
+
};
|
|
37
|
+
}): TokenAccountingCheck;
|
|
38
|
+
}
|
|
39
|
+
//# sourceMappingURL=token-accounting.check.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"token-accounting.check.d.ts","sourceRoot":"","sources":["../../../src/eval/retrieval/token-accounting.check.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,oBAAoB,EAAE,MAAM,wBAAwB,CAAC;AAK9D;;;;;;;;GAQG;AACH,qBACa,sBAAsB;IACjC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAA2C;IAClE,OAAO,CAAC,aAAa,CAAK;IAC1B,OAAO,CAAC,cAAc,CAAK;IAE3B,4EAA4E;IAC5E,OAAO,CAAC,MAAM,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI;IAKxD;;;;;;;;OAQG;IACH,KAAK,IAAI,IAAI;IAKb,4DAA4D;IAC5D,KAAK,CAAC,MAAM,EAAE;QAAE,UAAU,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE;YAAE,KAAK,EAAE,MAAM,CAAC;YAAC,MAAM,EAAE,MAAM,CAAA;SAAE,CAAA;KAAE,GAAG,oBAAoB;CA2BvG"}
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
|
|
3
|
+
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
4
|
+
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
5
|
+
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
6
|
+
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
7
|
+
};
|
|
8
|
+
var TokenAccountingChecker_1;
|
|
9
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
10
|
+
exports.TokenAccountingChecker = void 0;
|
|
11
|
+
const common_1 = require("@nestjs/common");
|
|
12
|
+
/** Tokens may legitimately differ by rounding; anything larger is a defect. */
|
|
13
|
+
const TOLERANCE_TOKENS = 2;
|
|
14
|
+
/**
|
|
15
|
+
* Cross-checks a turn's reported token aggregate against the sum of the calls
|
|
16
|
+
* actually observed for it.
|
|
17
|
+
*
|
|
18
|
+
* Spec §6.5. The `2C + A` reducer bug inflated every archived chat figure and
|
|
19
|
+
* went unnoticed for a full measurement cycle; this check is what makes the
|
|
20
|
+
* same class of bug loud instead of silent. Provider-reported per-call figures
|
|
21
|
+
* are the source of truth.
|
|
22
|
+
*/
|
|
23
|
+
let TokenAccountingChecker = TokenAccountingChecker_1 = class TokenAccountingChecker {
|
|
24
|
+
constructor() {
|
|
25
|
+
this.logger = new common_1.Logger(TokenAccountingChecker_1.name);
|
|
26
|
+
this.observedInput = 0;
|
|
27
|
+
this.observedOutput = 0;
|
|
28
|
+
}
|
|
29
|
+
/** Record one provider call's reported usage for the question in flight. */
|
|
30
|
+
observe(params) {
|
|
31
|
+
this.observedInput += params.input;
|
|
32
|
+
this.observedOutput += params.output;
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* Clears the accumulator WITHOUT producing a verdict.
|
|
36
|
+
*
|
|
37
|
+
* For the question that never completed a turn: it has nothing to compare, but
|
|
38
|
+
* it may still have deposited observations before it threw. Leaving them in
|
|
39
|
+
* place attributes them to the NEXT question — which either invents a mismatch
|
|
40
|
+
* that is not there or, worse, masks one that is. `check()` resets as a side
|
|
41
|
+
* effect of comparing; this is the same reset for the path that cannot compare.
|
|
42
|
+
*/
|
|
43
|
+
reset() {
|
|
44
|
+
this.observedInput = 0;
|
|
45
|
+
this.observedOutput = 0;
|
|
46
|
+
}
|
|
47
|
+
/** Compare and RESET, so the next question starts clean. */
|
|
48
|
+
check(params) {
|
|
49
|
+
const result = {
|
|
50
|
+
ledgerInput: params.ledger.input,
|
|
51
|
+
observedInput: this.observedInput,
|
|
52
|
+
ledgerOutput: params.ledger.output,
|
|
53
|
+
observedOutput: this.observedOutput,
|
|
54
|
+
agrees: Math.abs(params.ledger.input - this.observedInput) <= TOLERANCE_TOKENS &&
|
|
55
|
+
Math.abs(params.ledger.output - this.observedOutput) <= TOLERANCE_TOKENS,
|
|
56
|
+
// The comparison ran. Only this method may set it — every other producer
|
|
57
|
+
// of a `TokenAccountingCheck` is describing a check that did NOT happen.
|
|
58
|
+
checked: true,
|
|
59
|
+
};
|
|
60
|
+
if (!result.agrees) {
|
|
61
|
+
this.logger.error(`token accounting MISMATCH on ${params.questionId}: reported ` +
|
|
62
|
+
`${result.ledgerInput}/${result.ledgerOutput} but observed ` +
|
|
63
|
+
`${result.observedInput}/${result.observedOutput}. The baseline is not trustworthy ` +
|
|
64
|
+
`until this is explained.`);
|
|
65
|
+
}
|
|
66
|
+
this.observedInput = 0;
|
|
67
|
+
this.observedOutput = 0;
|
|
68
|
+
return result;
|
|
69
|
+
}
|
|
70
|
+
};
|
|
71
|
+
exports.TokenAccountingChecker = TokenAccountingChecker;
|
|
72
|
+
exports.TokenAccountingChecker = TokenAccountingChecker = TokenAccountingChecker_1 = __decorate([
|
|
73
|
+
(0, common_1.Injectable)()
|
|
74
|
+
], TokenAccountingChecker);
|
|
75
|
+
//# sourceMappingURL=token-accounting.check.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"token-accounting.check.js","sourceRoot":"","sources":["../../../src/eval/retrieval/token-accounting.check.ts"],"names":[],"mappings":";;;;;;;;;;AAAA,2CAAoD;AAGpD,+EAA+E;AAC/E,MAAM,gBAAgB,GAAG,CAAC,CAAC;AAE3B;;;;;;;;GAQG;AAEI,IAAM,sBAAsB,8BAA5B,MAAM,sBAAsB;IAA5B;QACY,WAAM,GAAG,IAAI,eAAM,CAAC,wBAAsB,CAAC,IAAI,CAAC,CAAC;QAC1D,kBAAa,GAAG,CAAC,CAAC;QAClB,mBAAc,GAAG,CAAC,CAAC;IAkD7B,CAAC;IAhDC,4EAA4E;IAC5E,OAAO,CAAC,MAAyC;QAC/C,IAAI,CAAC,aAAa,IAAI,MAAM,CAAC,KAAK,CAAC;QACnC,IAAI,CAAC,cAAc,IAAI,MAAM,CAAC,MAAM,CAAC;IACvC,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK;QACH,IAAI,CAAC,aAAa,GAAG,CAAC,CAAC;QACvB,IAAI,CAAC,cAAc,GAAG,CAAC,CAAC;IAC1B,CAAC;IAED,4DAA4D;IAC5D,KAAK,CAAC,MAAyE;QAC7E,MAAM,MAAM,GAAyB;YACnC,WAAW,EAAE,MAAM,CAAC,MAAM,CAAC,KAAK;YAChC,aAAa,EAAE,IAAI,CAAC,aAAa;YACjC,YAAY,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM;YAClC,cAAc,EAAE,IAAI,CAAC,cAAc;YACnC,MAAM,EACJ,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,gBAAgB;gBACtE,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,cAAc,CAAC,IAAI,gBAAgB;YAC1E,yEAAyE;YACzE,yEAAyE;YACzE,OAAO,EAAE,IAAI;SACd,CAAC;QAEF,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YACnB,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,gCAAgC,MAAM,CAAC,UAAU,aAAa;gBAC5D,GAAG,MAAM,CAAC,WAAW,IAAI,MAAM,CAAC,YAAY,gBAAgB;gBAC5D,GAAG,MAAM,CAAC,aAAa,IAAI,MAAM,CAAC,cAAc,oCAAoC;gBACpF,0BAA0B,CAC7B,CAAC;QACJ,CAAC;QAED,IAAI,CAAC,aAAa,GAAG,CAAC,CAAC;QACvB,IAAI,CAAC,cAAc,GAAG,CAAC,CAAC;QACxB,OAAO,MAAM,CAAC;IAChB,CAAC;CACF,CAAA;AArDY,wDAAsB;iCAAtB,sBAAsB;IADlC,IAAA,mBAAU,GAAE;GACA,sBAAsB,CAqDlC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"atomicfact.repository.d.ts","sourceRoot":"","sources":["../../../../src/foundations/atomicfact/repositories/atomicfact.repository.ts"],"names":[],"mappings":"AAAA,OAAO,EAAsB,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAClE,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AACxC,OAAO,EAAE,uBAAuB,EAAE,MAAM,iDAAiD,CAAC;AAC1F,OAAO,EAAmB,qBAAqB,EAAE,MAAM,uDAAuD,CAAC;AAC/G,OAAO,EAAE,UAAU,EAAE,MAAM,mCAAmC,CAAC;AAC/D,OAAO,EAAE,YAAY,EAAE,MAAM,4CAA4C,CAAC;AAC1E,OAAO,EAAE,eAAe,EAAE,MAAM,kDAAkD,CAAC;AACnF,OAAO,EAAE,UAAU,EAAE,MAAM,8CAA8C,CAAC;
|
|
1
|
+
{"version":3,"file":"atomicfact.repository.d.ts","sourceRoot":"","sources":["../../../../src/foundations/atomicfact/repositories/atomicfact.repository.ts"],"names":[],"mappings":"AAAA,OAAO,EAAsB,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAClE,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AACxC,OAAO,EAAE,uBAAuB,EAAE,MAAM,iDAAiD,CAAC;AAC1F,OAAO,EAAmB,qBAAqB,EAAE,MAAM,uDAAuD,CAAC;AAC/G,OAAO,EAAE,UAAU,EAAE,MAAM,mCAAmC,CAAC;AAC/D,OAAO,EAAE,YAAY,EAAE,MAAM,4CAA4C,CAAC;AAC1E,OAAO,EAAE,eAAe,EAAE,MAAM,kDAAkD,CAAC;AACnF,OAAO,EAAE,UAAU,EAAE,MAAM,8CAA8C,CAAC;AAI1E,qBACa,oBAAqB,YAAW,YAAY;IAErD,OAAO,CAAC,QAAQ,CAAC,KAAK;IACtB,OAAO,CAAC,QAAQ,CAAC,eAAe;IAChC,OAAO,CAAC,QAAQ,CAAC,UAAU;IACF,OAAO,CAAC,QAAQ,CAAC,aAAa;IACvD,OAAO,CAAC,QAAQ,CAAC,UAAU;gBAJV,KAAK,EAAE,YAAY,EACnB,eAAe,EAAE,eAAe,EAChC,UAAU,EAAE,UAAU,EACG,aAAa,EAAE,qBAAqB,EAC7D,UAAU,EAAE,uBAAuB;IAGhD,YAAY;IAMZ,4BAA4B,CAAC,MAAM,EAAE;QACzC,WAAW,EAAE,MAAM,EAAE,CAAC;QACtB,YAAY,EAAE,MAAM,EAAE,CAAC;QACvB,iBAAiB,EAAE,MAAM,EAAE,CAAC;QAC5B,UAAU,EAAE,UAAU,CAAC;KACxB,GAAG,OAAO,CAAC,UAAU,EAAE,CAAC;IAwCnB,wBAAwB,CAAC,MAAM,EAAE;QAAE,OAAO,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,UAAU,EAAE,CAAC;IAgB5E,kBAAkB,CAAC,MAAM,EAAE;QAAE,YAAY,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,UAAU,CAAC;IAezE,6BAA6B,IAAI,OAAO,CAAC,IAAI,CAAC;IAY9C,gBAAgB,CAAC,MAAM,EAAE;QAAE,YAAY,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC;CAmB1G"}
|
|
@@ -20,6 +20,7 @@ const ai_source_query_provider_1 = require("../../../common/repositories/ai-sour
|
|
|
20
20
|
const neo4j_service_1 = require("../../../core/neo4j/services/neo4j.service");
|
|
21
21
|
const security_service_1 = require("../../../core/security/services/security.service");
|
|
22
22
|
const atomic_fact_model_1 = require("../../atomicfact/entities/atomic.fact.model");
|
|
23
|
+
const retrieval_constants_1 = require("../../chunk/repositories/retrieval.constants");
|
|
23
24
|
let AtomicFactRepository = class AtomicFactRepository {
|
|
24
25
|
constructor(neo4j, securityService, clsService, aiSourceQuery, agentScope) {
|
|
25
26
|
this.neo4j = neo4j;
|
|
@@ -51,9 +52,14 @@ let AtomicFactRepository = class AtomicFactRepository {
|
|
|
51
52
|
keyConcepts: params.keyConcepts,
|
|
52
53
|
skipChunkIds: params.skipChunkIds,
|
|
53
54
|
skipAtomicFactIds: params.skipAtomicFactIds,
|
|
55
|
+
maxAtomicFacts: retrieval_constants_1.MAX_ATOMIC_FACTS,
|
|
54
56
|
...scope.params,
|
|
55
57
|
...scopeFilter.params,
|
|
56
58
|
};
|
|
59
|
+
// Deliberate: the facts are NOT ordered before the cut. Atomic facts carry no
|
|
60
|
+
// embedding (spec §2.4), so there is no relevance signal to order by — this is
|
|
61
|
+
// a spend bound, not a ranking. Giving facts embeddings is out of scope and has
|
|
62
|
+
// its own spec.
|
|
57
63
|
query.query += `
|
|
58
64
|
${scope.cypher}
|
|
59
65
|
${scopeFilter.cypher}
|
|
@@ -62,6 +68,7 @@ let AtomicFactRepository = class AtomicFactRepository {
|
|
|
62
68
|
${params.skipChunkIds.length > 0 ? `AND NOT atomicfact_chunk.id IN $skipChunkIds` : ""}
|
|
63
69
|
${params.skipAtomicFactIds.length > 0 ? `AND NOT atomicfact.id IN $skipAtomicFactIds` : ""}
|
|
64
70
|
RETURN atomicfact, atomicfact_chunk
|
|
71
|
+
LIMIT toInteger($maxAtomicFacts)
|
|
65
72
|
`;
|
|
66
73
|
return this.neo4j.readMany(query);
|
|
67
74
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"atomicfact.repository.js","sourceRoot":"","sources":["../../../../src/foundations/atomicfact/repositories/atomicfact.repository.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;AAAA,2CAAkE;AAClE,2CAAwC;AACxC,wFAA0F;AAC1F,oGAA+G;AAE/G,8EAA0E;AAC1E,uFAAmF;AAEnF,mFAA8E;
|
|
1
|
+
{"version":3,"file":"atomicfact.repository.js","sourceRoot":"","sources":["../../../../src/foundations/atomicfact/repositories/atomicfact.repository.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;AAAA,2CAAkE;AAClE,2CAAwC;AACxC,wFAA0F;AAC1F,oGAA+G;AAE/G,8EAA0E;AAC1E,uFAAmF;AAEnF,mFAA8E;AAC9E,sFAAgF;AAGzE,IAAM,oBAAoB,GAA1B,MAAM,oBAAoB;IAC/B,YACmB,KAAmB,EACnB,eAAgC,EAChC,UAAsB,EACG,aAAoC,EAC7D,UAAmC;QAJnC,UAAK,GAAL,KAAK,CAAc;QACnB,oBAAe,GAAf,eAAe,CAAiB;QAChC,eAAU,GAAV,UAAU,CAAY;QACG,kBAAa,GAAb,aAAa,CAAuB;QAC7D,eAAU,GAAV,UAAU,CAAyB;IACnD,CAAC;IAEJ,KAAK,CAAC,YAAY;QAChB,MAAM,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC;YACxB,KAAK,EAAE,2GAA2G;SACnH,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,4BAA4B,CAAC,MAKlC;QACC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,EAAE,UAAU,EAAE,mCAAe,EAAE,CAAC,CAAC;QACpE,MAAM,KAAK,GAAG,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC;YACrC,UAAU,EAAE,MAAM,CAAC,UAAU;YAC7B,aAAa,EAAE,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC;YAC5C,eAAe,EAAE,IAAI,CAAC,eAAe;YACrC,WAAW,EAAE,IAAI;SAClB,CAAC,CAAC;QACH,0EAA0E;QAC1E,4EAA4E;QAC5E,wEAAwE;QACxE,0BAA0B;QAC1B,MAAM,WAAW,GAAG,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC;QAE5F,KAAK,CAAC,WAAW,GAAG;YAClB,GAAG,KAAK,CAAC,WAAW;YACpB,WAAW,EAAE,MAAM,CAAC,WAAW;YAC/B,YAAY,EAAE,MAAM,CAAC,YAAY;YACjC,iBAAiB,EAAE,MAAM,CAAC,iBAAiB;YAC3C,cAAc,EAAE,sCAAgB;YAChC,GAAG,KAAK,CAAC,MAAM;YACf,GAAG,WAAW,CAAC,MAAM;SACtB,CAAC;QACF,8EAA8E;QAC9E,+EAA+E;QAC/E,gFAAgF;QAChF,gBAAgB;QAChB,KAAK,CAAC,KAAK,IAAI;QACX,KAAK,CAAC,MAAM;QACZ,WAAW,CAAC,MAAM;;;QAGlB,MAAM,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,8CAA8C,CAAC,CAAC,CAAC,EAAE;QACpF,MAAM,CAAC,iBAAiB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,6CAA6C,CAAC,CAAC,CAAC,EAAE;;;GAG7F,CAAC;QACA,OAAO,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IACpC,CAAC;IAED,KAAK,CAAC,wBAAwB,CAAC,MAA2B;QACxD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,EAAE,UAAU,EAAE,mCAAe,EAAE,CAAC,CAAC;QAEpE,KAAK,CAAC,WAAW,GAAG;YAClB,GAAG,KAAK,CAAC,WAAW;YACpB,OAAO,EAAE,MAAM,CAAC,OAAO;SACxB,CAAC;QAEF,KAAK,CAAC,KAAK,IAAI;;;KAGd,CAAC;QAEF,OAAO,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IACpC,CAAC;IAED,KAAK,CAAC,kBAAkB,CAAC,MAAgC;QACvD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,EAAE,UAAU,EAAE,mCAAe,EAAE,CAAC,CAAC;QAEpE,KAAK,CAAC,WAAW,GAAG;YAClB,GAAG,KAAK,CAAC,WAAW;YACpB,YAAY,EAAE,MAAM,CAAC,YAAY;SAClC,CAAC;QAEF,KAAK,CAAC,KAAK,IAAI;;;KAGd,CAAC;QACF,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IACnC,CAAC;IAED,KAAK,CAAC,6BAA6B;QACjC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC;QAErC,KAAK,CAAC,KAAK,GAAG;;;;KAIb,CAAC;QAEF,MAAM,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IACnC,CAAC;IAED,KAAK,CAAC,gBAAgB,CAAC,MAAkE;QACvF,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC;QAErC,KAAK,CAAC,WAAW,GAAG;YAClB,GAAG,KAAK,CAAC,WAAW;YACpB,OAAO,EAAE,MAAM,CAAC,OAAO;YACvB,YAAY,EAAE,MAAM,CAAC,YAAY;YACjC,iBAAiB,EAAE,MAAM,CAAC,OAAO;SAClC,CAAC;QAEF,KAAK,CAAC,KAAK,IAAI;;;;;KAKd,CAAC;QAEF,MAAM,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IACnC,CAAC;CACF,CAAA;AA1HY,oDAAoB;+BAApB,oBAAoB;IADhC,IAAA,mBAAU,GAAE;IAMR,WAAA,IAAA,eAAM,EAAC,0CAAe,CAAC,CAAA;qCAHA,4BAAY;QACF,kCAAe;QACpB,uBAAU,UAEV,4CAAuB;GAN3C,oBAAoB,CA0HhC"}
|
|
@@ -39,13 +39,70 @@ export declare class ChunkRepository implements OnModuleInit {
|
|
|
39
39
|
* lives with the caller (the contextualiser knows which content the run is
|
|
40
40
|
* bound to), which is why it is passed down rather than derived here. Absent,
|
|
41
41
|
* `EmbedderService.persistUsage` records nothing.
|
|
42
|
+
*
|
|
43
|
+
* Each returned chunk carries `score`: the cosine similarity of that chunk against
|
|
44
|
+
* the question embedding. Both retrieval halves have to reach the notebook on ONE
|
|
45
|
+
* scale — the answer node orders entries best-score-first and fills a character
|
|
46
|
+
* budget, so an unscored half sorts last however good it is. The RRF score is NOT
|
|
47
|
+
* usable for that: it is rank-derived (the top hit scores ≈1/61 whether it is a
|
|
48
|
+
* perfect match or noise) and is not comparable with a cosine from the graph half.
|
|
42
49
|
*/
|
|
43
50
|
findPotentialChunks(params: {
|
|
44
51
|
question: string;
|
|
45
52
|
dataLimits: DataLimits;
|
|
46
53
|
attribution?: EmbedderAttribution;
|
|
47
|
-
|
|
54
|
+
/**
|
|
55
|
+
* Precomputed question embedding. When supplied the repository does NOT
|
|
56
|
+
* embed again — the same question is otherwise embedded twice per turn,
|
|
57
|
+
* once here and once in findPotentialKeyConcepts. Optional so existing
|
|
58
|
+
* callers are unaffected.
|
|
59
|
+
*/
|
|
60
|
+
queryEmbedding?: number[];
|
|
61
|
+
}): Promise<Array<Chunk & {
|
|
62
|
+
score?: number;
|
|
63
|
+
}>>;
|
|
64
|
+
/**
|
|
65
|
+
* Exact cosine over the scoped set. No recall cliff by construction: every
|
|
66
|
+
* in-scope chunk is scored, so a small tenant can never be crowded out of a
|
|
67
|
+
* global top-K by a large one.
|
|
68
|
+
*/
|
|
69
|
+
private vectorIdsByExactScan;
|
|
70
|
+
/**
|
|
71
|
+
* Index-backed fallback for scoped sets too large to scan exactly. Still
|
|
72
|
+
* post-filters, so it still has a cliff — the over-fetch is what pushes that
|
|
73
|
+
* cliff out of reach, and it is deliberately far above the previous 1,000.
|
|
74
|
+
*/
|
|
75
|
+
private vectorIdsByIndex;
|
|
76
|
+
/**
|
|
77
|
+
* Lexical branch, scoped the same way. Previously filtered against the
|
|
78
|
+
* client-side id list; now joined in the database like the vector branch.
|
|
79
|
+
*/
|
|
80
|
+
private lexicalIdsInScope;
|
|
81
|
+
/**
|
|
82
|
+
* INPUT ORDER IS PART OF THE CONTRACT here too: `WHERE chunk.id IN $ids` yields rows
|
|
83
|
+
* in store order, and the fused RRF order is what the caller means by "best first".
|
|
84
|
+
*
|
|
85
|
+
* `queryEmbedding` attaches the cosine score of each chunk against the question, on
|
|
86
|
+
* the same scale `findChunksByIds` puts on the graph half. No floor and no count cap
|
|
87
|
+
* are applied — the notebook's character budget decides what reaches the answer.
|
|
88
|
+
*/
|
|
48
89
|
private findChunksByIdsOrdered;
|
|
90
|
+
/**
|
|
91
|
+
* Attaches `vector.similarity.cosine(chunk.embedding, $queryEmbedding)` to already
|
|
92
|
+
* hydrated chunks, by id, in JS.
|
|
93
|
+
*
|
|
94
|
+
* Why a separate read rather than one extra column on the hydration query: the
|
|
95
|
+
* hydration goes through `readMany`, which maps every row with the descriptor's
|
|
96
|
+
* generated mapper, and that mapper only ever reads the descriptor's own fields,
|
|
97
|
+
* computed fields and virtual fields (`define-entity.ts`). A projected column such
|
|
98
|
+
* as `score` is therefore silently DROPPED before the caller ever sees it. Reading
|
|
99
|
+
* the id/score pairs on their own and merging them here keeps entity mapping on
|
|
100
|
+
* `readMany` — this repository never hand-maps raw Neo4j records.
|
|
101
|
+
*
|
|
102
|
+
* SCOPE: this read scores only ids that the caller's own scope-gated query already
|
|
103
|
+
* returned, so it cannot widen the scope by construction.
|
|
104
|
+
*/
|
|
105
|
+
private attachCosineScores;
|
|
49
106
|
findParentName(params: {
|
|
50
107
|
id: string;
|
|
51
108
|
nodeType: string;
|
|
@@ -67,6 +124,35 @@ export declare class ChunkRepository implements OnModuleInit {
|
|
|
67
124
|
chunkId: string;
|
|
68
125
|
dataLimits?: DataLimits;
|
|
69
126
|
}): Promise<Chunk>;
|
|
127
|
+
/**
|
|
128
|
+
* Hydrates many chunks in ONE query, with the same scope gate `findChunkById`
|
|
129
|
+
* applies to one. The caller previously looped with an `await` inside,
|
|
130
|
+
* costing one round trip per queued chunk. Returns only the chunks that
|
|
131
|
+
* exist and are in scope; the caller must not assume a 1:1 mapping with
|
|
132
|
+
* `chunkIds`.
|
|
133
|
+
*
|
|
134
|
+
* ORDER IS PART OF THE CONTRACT. `WHERE chunk.id IN $chunkIds` returns rows in
|
|
135
|
+
* whatever order the store yields them, but the loop this replaced hydrated in
|
|
136
|
+
* queue order, and that order reaches the contextualiser's per-chunk fan-out and
|
|
137
|
+
* the notebook entries it writes. Returning them shuffled changes what the answer
|
|
138
|
+
* node cites. The input order is therefore restored here, exactly as
|
|
139
|
+
* `findChunksByIdsOrdered` does for the retrieval path.
|
|
140
|
+
*
|
|
141
|
+
* `queryEmbedding` is OPTIONAL and, when given, attaches `score`: the cosine
|
|
142
|
+
* similarity of the chunk against the question. These chunks arrive from a fact
|
|
143
|
+
* join with no score of their own, which is precisely why an LLM used to have to
|
|
144
|
+
* judge them; cosine puts this half on the SAME scale as the document half, so the
|
|
145
|
+
* answer node can order both together. When it is absent the scoring clause is not
|
|
146
|
+
* in the Cypher at all. No floor is applied at any point: the notebook's character
|
|
147
|
+
* budget, not a threshold, decides what reaches the answer.
|
|
148
|
+
*/
|
|
149
|
+
findChunksByIds(params: {
|
|
150
|
+
chunkIds: string[];
|
|
151
|
+
dataLimits?: DataLimits;
|
|
152
|
+
queryEmbedding?: number[];
|
|
153
|
+
}): Promise<Array<Chunk & {
|
|
154
|
+
score?: number;
|
|
155
|
+
}>>;
|
|
70
156
|
/**
|
|
71
157
|
* Pipeline hydration of a content node's chunks — deliberately WITHOUT `embedding`.
|
|
72
158
|
*
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"chunk.repository.d.ts","sourceRoot":"","sources":["../../../../src/foundations/chunk/repositories/chunk.repository.ts"],"names":[],"mappings":"AAAA,OAAO,EAAsB,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAClE,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAExC,OAAO,EAAE,QAAQ,EAAE,MAAM,iCAAiC,CAAC;AAE3D,OAAO,EAAE,uBAAuB,EAAE,MAAM,iDAAiD,CAAC;AAC1F,OAAO,EAAmB,qBAAqB,EAAE,MAAM,uDAAuD,CAAC;AAC/G,OAAO,EAAE,UAAU,EAAE,MAAM,mCAAmC,CAAC;AAC/D,OAAO,EAAE,mBAAmB,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AACrE,OAAO,EAAE,YAAY,EAAE,MAAM,0CAA0C,CAAC;AACxE,OAAO,EAAE,YAAY,EAAE,MAAM,4CAA4C,CAAC;AAC1E,OAAO,EAAE,eAAe,EAAE,MAAM,kDAAkD,CAAC;AACnF,OAAO,EAAE,KAAK,EAAmB,MAAM,mCAAmC,CAAC;
|
|
1
|
+
{"version":3,"file":"chunk.repository.d.ts","sourceRoot":"","sources":["../../../../src/foundations/chunk/repositories/chunk.repository.ts"],"names":[],"mappings":"AAAA,OAAO,EAAsB,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAClE,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAExC,OAAO,EAAE,QAAQ,EAAE,MAAM,iCAAiC,CAAC;AAE3D,OAAO,EAAE,uBAAuB,EAAE,MAAM,iDAAiD,CAAC;AAC1F,OAAO,EAAmB,qBAAqB,EAAE,MAAM,uDAAuD,CAAC;AAC/G,OAAO,EAAE,UAAU,EAAE,MAAM,mCAAmC,CAAC;AAC/D,OAAO,EAAE,mBAAmB,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AACrE,OAAO,EAAE,YAAY,EAAE,MAAM,0CAA0C,CAAC;AACxE,OAAO,EAAE,YAAY,EAAE,MAAM,4CAA4C,CAAC;AAC1E,OAAO,EAAE,eAAe,EAAE,MAAM,kDAAkD,CAAC;AACnF,OAAO,EAAE,KAAK,EAAmB,MAAM,mCAAmC,CAAC;AAK3E,qBACa,eAAgB,YAAW,YAAY;IAWhD,OAAO,CAAC,QAAQ,CAAC,KAAK;IACtB,OAAO,CAAC,QAAQ,CAAC,YAAY;IAC7B,OAAO,CAAC,QAAQ,CAAC,eAAe;IAChC,OAAO,CAAC,QAAQ,CAAC,UAAU;IAC3B,OAAO,CAAC,QAAQ,CAAC,eAAe;IACP,OAAO,CAAC,QAAQ,CAAC,aAAa;IACvD,OAAO,CAAC,QAAQ,CAAC,UAAU;IAhB7B;;;;;;OAMG;IACH,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAM;gBAGtB,KAAK,EAAE,YAAY,EACnB,YAAY,EAAE,YAAY,EAC1B,eAAe,EAAE,eAAe,EAChC,UAAU,EAAE,UAAU,EACtB,eAAe,EAAE,eAAe,EACP,aAAa,EAAE,qBAAqB,EAC7D,UAAU,EAAE,uBAAuB;IAGhD,YAAY;IAuBZ,mBAAmB,IAAI,OAAO,CAAC,IAAI,CAAC;IAmBpC,aAAa,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;IAUjC,eAAe,CAAC,MAAM,EAAE;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,EAAE,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAatF;;;;;;;;;;;;;OAaG;IACG,mBAAmB,CAAC,MAAM,EAAE;QAChC,QAAQ,EAAE,MAAM,CAAC;QACjB,UAAU,EAAE,UAAU,CAAC;QACvB,WAAW,CAAC,EAAE,mBAAmB,CAAC;QAClC;;;;;WAKG;QACH,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;KAC3B,GAAG,OAAO,CAAC,KAAK,CAAC,KAAK,GAAG;QAAE,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAwD9C;;;;OAIG;YACW,oBAAoB;IA4BlC;;;;OAIG;YACW,gBAAgB;IA8B9B;;;OAGG;YACW,iBAAiB;IA6B/B;;;;;;;OAOG;YACW,sBAAsB;IA0BpC;;;;;;;;;;;;;;OAcG;YACW,kBAAkB;IA6B1B,cAAc,CAAC,MAAM,EAAE;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC;IAYrF,qBAAqB,CAAC,MAAM,EAAE;QAAE,OAAO,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,KAAK,CAAC;IAgBlE,mBAAmB,CAAC,MAAM,EAAE;QAAE,OAAO,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,KAAK,CAAC;IAgBtE;;;;;;OAMG;IACG,aAAa,CAAC,MAAM,EAAE;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,UAAU,CAAA;KAAE,GAAG,OAAO,CAAC,KAAK,CAAC;IAoBzF;;;;;;;;;;;;;;;;;;;;;OAqBG;IACG,eAAe,CAAC,MAAM,EAAE;QAC5B,QAAQ,EAAE,MAAM,EAAE,CAAC;QACnB,UAAU,CAAC,EAAE,UAAU,CAAC;QACxB,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;KAC3B,GAAG,OAAO,CAAC,KAAK,CAAC,KAAK,GAAG;QAAE,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IA2B9C;;;;;;;;;;;;;;;;;OAiBG;IACG,UAAU,CAAC,MAAM,EAAE;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC;IAkCtE,WAAW,CAAC,MAAM,EAAE;QACxB,EAAE,EAAE,MAAM,CAAC;QACX,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,EAAE,MAAM,CAAC;QACjB,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB,OAAO,EAAE,MAAM,CAAC;QAChB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,QAAQ,EAAE,MAAM,CAAC;KAClB,GAAG,OAAO,CAAC,IAAI,CAAC;IA2DX,YAAY,CAAC,MAAM,EAAE;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,QAAQ,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAiBvE,WAAW,CAAC,MAAM,EAAE;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAiB5E;;;;;;;OAOG;IACG,0BAA0B,CAC9B,KAAK,EAAE;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,eAAe,EAAE,MAAM,CAAC;QAAC,eAAe,CAAC,EAAE,MAAM,CAAA;KAAE,EAAE,EAC/E,WAAW,CAAC,EAAE,mBAAmB,GAChC,OAAO,CAAC,IAAI,CAAC;IAiDV,mBAAmB,CAAC,MAAM,EAAE;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAUlF;;;;;;;;;OASG;IACG,qBAAqB,CAAC,MAAM,EAAE;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,MAAM,CAAC;IAwBtF;;;;;;;;;;;;;;;;;OAiBG;IACG,wBAAwB,CAAC,MAAM,EAAE;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,OAAO,CAAC;IAgC1F;;;;;OAKG;IACG,sBAAsB,CAAC,MAAM,EAAE;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAU/E,mBAAmB,CAAC,MAAM,EAAE;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC;IAkB/E,sBAAsB,CAAC,MAAM,EAAE;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAgBvF,YAAY,CAAC,MAAM,EAAE;QAAE,QAAQ,EAAE,MAAM,EAAE,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAgB3D,wBAAwB,IAAI,OAAO,CAAC,IAAI,CAAC;IAYzC,sBAAsB,CAAC,MAAM,EAAE;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAe/E,kBAAkB,CAAC,MAAM,EAAE;QAC/B,QAAQ,EAAE,MAAM,EAAE,CAAC;QACnB,MAAM,EAAE,MAAM,CAAC;KAChB,GAAG,OAAO,CAAC;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,EAAE,CAAC;QAAC,KAAK,EAAE,MAAM,EAAE,CAAA;KAAE,EAAE,CAAC;IA8B/D,2BAA2B,CAAC,MAAM,EAAE;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC;CAe1F"}
|