@untangle-code/search 0.0.0-stage → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (140) hide show
  1. package/LICENSE +661 -0
  2. package/NOTICE +90 -0
  3. package/README.md +433 -2
  4. package/THIRD_PARTY_LICENSES.md +29 -0
  5. package/dist/benchmark-integrity.d.ts +12 -0
  6. package/dist/benchmark-integrity.d.ts.map +1 -0
  7. package/dist/benchmark-integrity.js +10 -0
  8. package/dist/benchmark-integrity.js.map +1 -0
  9. package/dist/bm25.d.ts +46 -0
  10. package/dist/bm25.d.ts.map +1 -0
  11. package/dist/bm25.js +105 -0
  12. package/dist/bm25.js.map +1 -0
  13. package/dist/deadline.d.ts +4 -0
  14. package/dist/deadline.d.ts.map +1 -0
  15. package/dist/deadline.js +54 -0
  16. package/dist/deadline.js.map +1 -0
  17. package/dist/dedupe.d.ts +48 -0
  18. package/dist/dedupe.d.ts.map +1 -0
  19. package/dist/dedupe.js +58 -0
  20. package/dist/dedupe.js.map +1 -0
  21. package/dist/diversify.d.ts +45 -0
  22. package/dist/diversify.d.ts.map +1 -0
  23. package/dist/diversify.js +30 -0
  24. package/dist/diversify.js.map +1 -0
  25. package/dist/engine.d.ts +157 -0
  26. package/dist/engine.d.ts.map +1 -0
  27. package/dist/engine.js +1498 -0
  28. package/dist/engine.js.map +1 -0
  29. package/dist/enrichment.d.ts +16 -0
  30. package/dist/enrichment.d.ts.map +1 -0
  31. package/dist/enrichment.js +92 -0
  32. package/dist/enrichment.js.map +1 -0
  33. package/dist/filter.d.ts +86 -0
  34. package/dist/filter.d.ts.map +1 -0
  35. package/dist/filter.js +166 -0
  36. package/dist/filter.js.map +1 -0
  37. package/dist/findability.d.ts +144 -0
  38. package/dist/findability.d.ts.map +1 -0
  39. package/dist/findability.js +393 -0
  40. package/dist/findability.js.map +1 -0
  41. package/dist/format.d.ts +28 -0
  42. package/dist/format.d.ts.map +1 -0
  43. package/dist/format.js +172 -0
  44. package/dist/format.js.map +1 -0
  45. package/dist/fusion.d.ts +47 -0
  46. package/dist/fusion.d.ts.map +1 -0
  47. package/dist/fusion.js +84 -0
  48. package/dist/fusion.js.map +1 -0
  49. package/dist/glob.d.ts +50 -0
  50. package/dist/glob.d.ts.map +1 -0
  51. package/dist/glob.js +144 -0
  52. package/dist/glob.js.map +1 -0
  53. package/dist/grep.d.ts +76 -0
  54. package/dist/grep.d.ts.map +1 -0
  55. package/dist/grep.js +475 -0
  56. package/dist/grep.js.map +1 -0
  57. package/dist/index.d.ts +42 -0
  58. package/dist/index.d.ts.map +1 -0
  59. package/dist/index.js +63 -0
  60. package/dist/index.js.map +1 -0
  61. package/dist/ledger.d.ts +45 -0
  62. package/dist/ledger.d.ts.map +1 -0
  63. package/dist/ledger.js +39 -0
  64. package/dist/ledger.js.map +1 -0
  65. package/dist/payload.d.ts +108 -0
  66. package/dist/payload.d.ts.map +1 -0
  67. package/dist/payload.js +263 -0
  68. package/dist/payload.js.map +1 -0
  69. package/dist/port.d.ts +112 -0
  70. package/dist/port.d.ts.map +1 -0
  71. package/dist/port.js +2 -0
  72. package/dist/port.js.map +1 -0
  73. package/dist/product-complexity.d.ts +89 -0
  74. package/dist/product-complexity.d.ts.map +1 -0
  75. package/dist/product-complexity.js +1367 -0
  76. package/dist/product-complexity.js.map +1 -0
  77. package/dist/regex-safety.d.ts +75 -0
  78. package/dist/regex-safety.d.ts.map +1 -0
  79. package/dist/regex-safety.js +902 -0
  80. package/dist/regex-safety.js.map +1 -0
  81. package/dist/relevance-prerequisites.d.ts +21 -0
  82. package/dist/relevance-prerequisites.d.ts.map +1 -0
  83. package/dist/relevance-prerequisites.js +47 -0
  84. package/dist/relevance-prerequisites.js.map +1 -0
  85. package/dist/relevance.d.ts +107 -0
  86. package/dist/relevance.d.ts.map +1 -0
  87. package/dist/relevance.js +199 -0
  88. package/dist/relevance.js.map +1 -0
  89. package/dist/retrieval.d.ts +46 -0
  90. package/dist/retrieval.d.ts.map +1 -0
  91. package/dist/retrieval.js +182 -0
  92. package/dist/retrieval.js.map +1 -0
  93. package/dist/scoring.d.ts +102 -0
  94. package/dist/scoring.d.ts.map +1 -0
  95. package/dist/scoring.js +175 -0
  96. package/dist/scoring.js.map +1 -0
  97. package/dist/sort.d.ts +21 -0
  98. package/dist/sort.d.ts.map +1 -0
  99. package/dist/sort.js +53 -0
  100. package/dist/sort.js.map +1 -0
  101. package/dist/symbol-match.d.ts +21 -0
  102. package/dist/symbol-match.d.ts.map +1 -0
  103. package/dist/symbol-match.js +72 -0
  104. package/dist/symbol-match.js.map +1 -0
  105. package/dist/tokenize.d.ts +41 -0
  106. package/dist/tokenize.d.ts.map +1 -0
  107. package/dist/tokenize.js +202 -0
  108. package/dist/tokenize.js.map +1 -0
  109. package/dist/types.d.ts +173 -0
  110. package/dist/types.d.ts.map +1 -0
  111. package/dist/types.js +42 -0
  112. package/dist/types.js.map +1 -0
  113. package/package.json +61 -3
  114. package/src/benchmark-integrity.ts +25 -0
  115. package/src/bm25.ts +135 -0
  116. package/src/deadline.ts +59 -0
  117. package/src/dedupe.ts +99 -0
  118. package/src/diversify.ts +71 -0
  119. package/src/engine.ts +1957 -0
  120. package/src/enrichment.ts +115 -0
  121. package/src/filter.ts +267 -0
  122. package/src/findability.ts +580 -0
  123. package/src/format.ts +228 -0
  124. package/src/fusion.ts +121 -0
  125. package/src/glob.ts +151 -0
  126. package/src/grep.ts +618 -0
  127. package/src/index.ts +83 -0
  128. package/src/ledger.ts +72 -0
  129. package/src/payload.ts +382 -0
  130. package/src/port.ts +133 -0
  131. package/src/product-complexity.ts +2168 -0
  132. package/src/regex-safety.ts +1002 -0
  133. package/src/relevance-prerequisites.ts +83 -0
  134. package/src/relevance.ts +328 -0
  135. package/src/retrieval.ts +299 -0
  136. package/src/scoring.ts +321 -0
  137. package/src/sort.ts +86 -0
  138. package/src/symbol-match.ts +87 -0
  139. package/src/tokenize.ts +213 -0
  140. package/src/types.ts +206 -0
package/dist/engine.js ADDED
@@ -0,0 +1,1498 @@
1
+ import { DEFAULT_SEARCH_CONFIG } from "@untangle-code/config";
2
+ import { buildSearchQueryFingerprint, canonicalJson, compareCodepoints, INDEX_CONTEXT_INCOMPLETE_WARNING_CODE, isUntangleError, normalizeSearchQuery, sha256Hex, sortByCodepoints, UntangleError, uniqueSortedByCodepoints, } from "@untangle-code/domain";
3
+ import { Bm25Ranker } from "./bm25.js";
4
+ import { dedupeCandidates } from "./dedupe.js";
5
+ import { assignDuplicatePenalties } from "./diversify.js";
6
+ import { SEARCH_GRAPH_RELATION_LIMIT, boundSearchGraphDetail, createSearchGraphRelations, unavailableSearchGraphRelations, } from "./enrichment.js";
7
+ import { buildPayloadFilter } from "./filter.js";
8
+ import { DEFAULT_BRANCH_WEIGHTS, fuseRankings } from "./fusion.js";
9
+ import { assertValidGlobPatterns, isPathSelected } from "./glob.js";
10
+ import { grepIndexedChunks } from "./grep.js";
11
+ import { SearchSkipLedger } from "./ledger.js";
12
+ import { candidateFromPoint, lexicalDocumentText, readRawString } from "./payload.js";
13
+ import { relevancePrerequisiteFailure } from "./relevance-prerequisites.js";
14
+ import { NO_SUFFICIENT_MATCH_WARNING_CODE, RELEVANCE_POLICY_NOT_EVALUATED_WARNING_CODE, decideResultRelevance, resolveRelevancePolicyBinding, selectFixedRelevanceEvidencePool, } from "./relevance.js";
15
+ import { assertCompletedRetrieval, branchesOfMode, retrieveIndependentBranches, rethrowCancellation, } from "./retrieval.js";
16
+ import { MAX_BOOST_SHARE, assembleScoreBreakdown } from "./scoring.js";
17
+ import { assignRanks, sortHits } from "./sort.js";
18
+ import { matchSymbolName } from "./symbol-match.js";
19
+ import { isLikelyGermanQuestion } from "./tokenize.js";
20
+ /** Halbwertszeit des Recency-Boosts, wenn keine gesetzt ist. */
21
+ export const DEFAULT_RECENCY_HALF_LIFE_DAYS = 30;
22
+ /** Harte Grenze fuer freie Providerwarnungen und Batch-Diagnose-IDs. */
23
+ export const SEARCH_GRAPH_DIAGNOSTIC_LIMIT = 32;
24
+ /** Providerwarnungen sind eine eigene Ausgabeachse und deshalb separat gedeckelt. */
25
+ export const SEARCH_GRAPH_PROVIDER_WARNING_LIMIT = 16;
26
+ function normalizeGraphProviderWarning(warning) {
27
+ const source = warning.details;
28
+ const details = {};
29
+ if (source !== undefined && source !== null && typeof source === "object") {
30
+ if (typeof source["workspaceId"] === "string") {
31
+ details["workspaceId"] = boundSearchGraphDetail(source["workspaceId"]);
32
+ }
33
+ if (typeof source["generationId"] === "string") {
34
+ details["generationId"] = boundSearchGraphDetail(source["generationId"]);
35
+ }
36
+ const rawIssues = source["issues"];
37
+ if (Array.isArray(rawIssues)) {
38
+ const issues = rawIssues.map(String).map(boundSearchGraphDetail).sort(compareCodepoints);
39
+ details["issues"] = issues.slice(0, SEARCH_GRAPH_PROVIDER_WARNING_LIMIT);
40
+ const providerOmitted = source["omittedIssueCount"];
41
+ const providerOmittedCount = typeof providerOmitted === "number" &&
42
+ Number.isSafeInteger(providerOmitted) &&
43
+ providerOmitted >= 0
44
+ ? providerOmitted
45
+ : 0;
46
+ details["omittedIssueCount"] =
47
+ providerOmittedCount + Math.max(0, issues.length - SEARCH_GRAPH_PROVIDER_WARNING_LIMIT);
48
+ }
49
+ const retainedKeys = new Set(["workspaceId", "generationId", "issues", "omittedIssueCount"]);
50
+ const omittedDetailFieldCount = Object.keys(source).filter((key) => !retainedKeys.has(key)).length;
51
+ if (omittedDetailFieldCount > 0) {
52
+ details["omittedDetailFieldCount"] = omittedDetailFieldCount;
53
+ }
54
+ }
55
+ return {
56
+ code: boundSearchGraphDetail(String(warning.code)),
57
+ message: boundSearchGraphDetail(String(warning.message)),
58
+ ...(Object.keys(details).length === 0 ? {} : { details }),
59
+ };
60
+ }
61
+ function boundedDiagnosticIds(values) {
62
+ const sorted = sortByCodepoints([...new Set(values)]);
63
+ return {
64
+ ids: sorted.slice(0, SEARCH_GRAPH_DIAGNOSTIC_LIMIT).map(boundSearchGraphDetail),
65
+ omittedCount: Math.max(0, sorted.length - SEARCH_GRAPH_DIAGNOSTIC_LIMIT),
66
+ };
67
+ }
68
+ export const DEFAULT_SEARCH_BRANCH_TIMEOUT_MS = 10_000;
69
+ /**
70
+ * Gibt dem PRAEZISIONSzweig bei einer zeichengleichen Symbolanfrage genug
71
+ * Basisgewicht, dass ein exakter Treffer selbst unter den erlaubten
72
+ * Boost-Grenzfaellen vor einem nicht-exakten Treffer bleibt.
73
+ *
74
+ * Bei MAX_BOOST_SHARE=0.5 kann ein nicht-exakter Basisscore hoechstens auf
75
+ * 1.5x steigen und ein exakter durch Strafen hoechstens auf 0.5x fallen. Das
76
+ * benoetigte Basisverhaeltnis ist daher (1+s)/(1-s)=3. Der kleine vorhandene
77
+ * Symbolanteil oberhalb dieser Grenze macht die Ungleichung strikt. Der
78
+ * Boost-Deckel selbst bleibt vollkommen unveraendert.
79
+ */
80
+ export function exactSymbolIntentBranchWeights(weights, active) {
81
+ if (!active.symbol || weights.symbol <= 0) {
82
+ return { ...weights };
83
+ }
84
+ const nonSymbolWeight = (active.dense ? Math.max(0, weights.dense) : 0) +
85
+ (active.lexical ? Math.max(0, weights.lexical) : 0);
86
+ const dominanceRatio = (1 + MAX_BOOST_SHARE) / (1 - MAX_BOOST_SHARE);
87
+ return {
88
+ ...weights,
89
+ symbol: Math.max(weights.symbol, nonSymbolWeight * dominanceRatio + weights.symbol),
90
+ };
91
+ }
92
+ /** Zahl dichter Spitzenplaetze, die einen lexikalischen Spitzenkandidaten bestaetigen. */
93
+ export const GERMAN_LEXICAL_CORROBORATION_DEPTH = 10;
94
+ /**
95
+ * Nur eine als deutsch erkannte Frage mit einem lexikalischen Spitzenreiter,
96
+ * der NICHT unter den zehn dicht besten Kandidaten vorkommt, verliert im
97
+ * Hybridmodus den lokalen BM25-Rang. Stimmen beide unabhaengigen Zweige
98
+ * ueberein, bleibt das nuetzliche lexikalische Signal erhalten.
99
+ */
100
+ export function shouldOmitUncorroboratedGermanLexicalBranch(queryText, denseRankedIds, lexicalRankedIds) {
101
+ if (!isLikelyGermanQuestion(queryText)) {
102
+ return false;
103
+ }
104
+ const lexicalTop = lexicalRankedIds[0];
105
+ if (lexicalTop === undefined) {
106
+ return true;
107
+ }
108
+ return !denseRankedIds.slice(0, GERMAN_LEXICAL_CORROBORATION_DEPTH).includes(lexicalTop);
109
+ }
110
+ /** Normalisiert die Zusatzfelder dieses Pakets (Gegenstueck zu `SearchQueryV1`). */
111
+ export function normalizeExtensions(request) {
112
+ const pathExcludes = uniqueSortedByCodepoints((request.pathExclude ?? []).map((value) => value.trim()).filter((value) => value !== ""));
113
+ return {
114
+ mode: request.mode ?? "hybrid",
115
+ abstention: request.abstention ?? "strict",
116
+ pathExcludes,
117
+ includeContent: request.includeContent ?? false,
118
+ complexityBias: request.complexityBias ?? "none",
119
+ sort: request.sort ?? "relevance",
120
+ };
121
+ }
122
+ /**
123
+ * Die Zusatzangaben, die den Fingerabdruck der Anfrage veraendern MUESSEN.
124
+ *
125
+ * Der Fingerabdruck beantwortet „ist das dieselbe Frage?". `mode`,
126
+ * `pathExcludes`, `complexityBias` und `sort` aendern die Treffermenge bzw.
127
+ * die Rangfolge — zwei Anfragen, die sich nur darin unterscheiden, sind NICHT
128
+ * dieselbe Frage. `includeContent` bleibt draussen: es aendert nur die
129
+ * Darstellung, genau wie `limit` und `candidateLimit` (siehe
130
+ * `buildSearchQueryFingerprint`).
131
+ *
132
+ * Nur ABWEICHUNGEN vom Standard gehen ein. Eine Anfrage ohne
133
+ * Paket-Erweiterungen behaelt damit exakt den Fingerabdruck, den sie ohne
134
+ * dieses Feld haette — Findability-Datensaetze aelterer Laeufe bleiben
135
+ * vergleichbar.
136
+ */
137
+ export function fingerprintRankingOptions(extensions) {
138
+ const options = {};
139
+ if (extensions.mode !== "hybrid") {
140
+ options["mode"] = extensions.mode;
141
+ }
142
+ if (extensions.pathExcludes.length > 0) {
143
+ options["pathExcludes"] = extensions.pathExcludes;
144
+ }
145
+ if (extensions.complexityBias !== "none") {
146
+ options["complexityBias"] = extensions.complexityBias;
147
+ }
148
+ if (extensions.sort !== "relevance") {
149
+ options["sort"] = extensions.sort;
150
+ }
151
+ // NUR bei Abweichung. `strict` traegt nichts bei, damit jeder bereits
152
+ // persistierte `queryFingerprint` byte-identisch bleibt — sonst waeren alle
153
+ // Findability-Datensaetze und Journey-Kohorten auf einen Schlag entwertet.
154
+ if (extensions.abstention !== "strict") {
155
+ options["abstention"] = extensions.abstention;
156
+ }
157
+ return Object.keys(options).length === 0 ? undefined : options;
158
+ }
159
+ /**
160
+ * Ergaenzt die Findability-Filter aus der Paketanfrage in die normalisierten
161
+ * Domain-Filter. `normalizeSearchQuery` kennt sie nicht, weil `SearchRequestV1`
162
+ * (spec.md §13.1) sie nicht kennt — `SearchFiltersV1` hat das Feld aber
163
+ * bereits, also wird es hier gefuellt statt neben der Query gefuehrt.
164
+ */
165
+ function withFindabilityFilter(query, request) {
166
+ const { findabilityMinScore, findabilityMaxScore } = request;
167
+ if (findabilityMinScore === undefined && findabilityMaxScore === undefined) {
168
+ return query;
169
+ }
170
+ if (findabilityMinScore !== undefined &&
171
+ findabilityMaxScore !== undefined &&
172
+ findabilityMinScore > findabilityMaxScore) {
173
+ throw new Error(`search query filter "findability" has min > max (${findabilityMinScore} > ${findabilityMaxScore})`);
174
+ }
175
+ const filters = {
176
+ ...query.filters,
177
+ findability: {
178
+ ...(findabilityMinScore === undefined ? {} : { minScore: findabilityMinScore }),
179
+ ...(findabilityMaxScore === undefined ? {} : { maxScore: findabilityMaxScore }),
180
+ },
181
+ };
182
+ return { ...query, filters };
183
+ }
184
+ /**
185
+ * WP-188 (E4): `searchConfig.diversifyByDefault` und `searchConfig.minScoreDefault`
186
+ * hatten KEINEN Leser.
187
+ *
188
+ * Beide standen in der Konfiguration, waren dokumentiert, wurden validiert —
189
+ * und veraenderten nichts. Eine Einstellung, die nichts tut, ist schlimmer als
190
+ * eine fehlende: sie behauptet eine Wirkung. Wer `minScoreDefault = 0.5`
191
+ * setzte und weiterhin Treffer mit 0.2 bekam, hatte keinen Grund, den Fehler
192
+ * beim Werkzeug zu suchen.
193
+ *
194
+ * Die Vorgaben greifen nur, wo die ANFRAGE nichts sagt — eine ausdrueckliche
195
+ * Angabe des Aufrufers gewinnt immer. Sie werden VOR dem Fingerabdruck
196
+ * angewandt, damit „gleiche Frage" auch dann gleich heisst, wenn ein Teil der
197
+ * Antwort aus der Konfiguration kommt.
198
+ */
199
+ function applyConfigDefaults(query, request, config) {
200
+ const diversify = request.diversify ?? config.diversifyByDefault;
201
+ const useConfigMinScore = request.minScore === undefined &&
202
+ config.minScoreDefault !== undefined &&
203
+ Number.isFinite(config.minScoreDefault);
204
+ return {
205
+ query: {
206
+ ...query,
207
+ diversify,
208
+ ...(useConfigMinScore ? { minScore: config.minScoreDefault } : {}),
209
+ },
210
+ minScoreFromConfig: useConfigMinScore,
211
+ };
212
+ }
213
+ /**
214
+ * Die Referenzimplementierung des {@link SearchPort}.
215
+ *
216
+ * Sie ist speicher-agnostisch: alles laeuft ueber den `VectorStore`-Port, also
217
+ * gegen Qdrant genauso wie gegen den In-Memory-Speicher des testkits.
218
+ */
219
+ export class HybridSearchEngine {
220
+ deps;
221
+ config;
222
+ branchWeights;
223
+ branchTimeoutMs;
224
+ relevancePolicies;
225
+ constructor(deps) {
226
+ this.deps = deps;
227
+ const seenProfileIds = new Set();
228
+ for (const target of deps.targets) {
229
+ if (target.embeddingProfileId !== target.embedder.profileId) {
230
+ throw new Error(`search target embeddingProfileId "${target.embeddingProfileId}" does not match its embedder.profileId "${target.embedder.profileId}"; collection/profile grouping and calibrated policies would be misattributed`);
231
+ }
232
+ if (seenProfileIds.has(target.embeddingProfileId)) {
233
+ throw new Error(`search target embeddingProfileId "${target.embeddingProfileId}" is configured more than once; one result group per embedding profile is required`);
234
+ }
235
+ seenProfileIds.add(target.embeddingProfileId);
236
+ }
237
+ this.config = deps.searchConfig ?? DEFAULT_SEARCH_CONFIG;
238
+ this.branchWeights = deps.branchWeights ?? { ...DEFAULT_BRANCH_WEIGHTS };
239
+ this.branchTimeoutMs = deps.branchTimeoutMs ?? DEFAULT_SEARCH_BRANCH_TIMEOUT_MS;
240
+ if (!Number.isSafeInteger(this.branchTimeoutMs) ||
241
+ this.branchTimeoutMs <= 0 ||
242
+ this.branchTimeoutMs > 2_147_483_647) {
243
+ throw new Error("branchTimeoutMs must be an integer between 1 and 2147483647");
244
+ }
245
+ // Die projektweite Config ist selbst eine vollwertige Verdrahtungsquelle.
246
+ // Das explizite Dependency-Feld bleibt fuer Library-Aufrufer und Tests,
247
+ // gewinnt bei gleichzeitiger Angabe aber eindeutig. Sonst waere eine
248
+ // geladene, validierte SearchConfig erneut ein wirkungsloser Wert.
249
+ const configuredRelevancePolicies = deps.relevancePolicies ?? this.config.relevancePolicies;
250
+ this.relevancePolicies = new Map(Object.entries(configuredRelevancePolicies ?? {}).map(([embeddingProfileId, configured]) => {
251
+ if (embeddingProfileId.trim() === "") {
252
+ throw new Error("relevancePolicies keys must be non-empty embedding profile ids");
253
+ }
254
+ return [embeddingProfileId, resolveRelevancePolicyBinding(configured)];
255
+ }));
256
+ }
257
+ async search(request, options) {
258
+ options?.signal?.throwIfAborted();
259
+ const extensions = normalizeExtensions(request);
260
+ const defaulted = applyConfigDefaults(withFindabilityFilter(normalizeSearchQuery(request, this.deps.defaults), request), request, this.config);
261
+ const query = defaulted.query;
262
+ // Der Ursprung reist als Parameter mit, NICHT als Feld der Instanz: zwei
263
+ // gleichzeitige `search()`-Aufrufe auf derselben Maschine wuerden sich
264
+ // sonst gegenseitig die Begruendung im Ledger umschreiben.
265
+ const minScoreOrigin = defaulted.minScoreFromConfig
266
+ ? "from searchConfig.minScoreDefault"
267
+ : "requested";
268
+ // Fehlerhafte Muster sind ein Eingabefehler des Nutzers und werden HIER
269
+ // abgelehnt. Faellt der Fehler erst beim Abgleich je Kandidat an, faengt
270
+ // ihn das Ziel-catch auf und meldet einen Speicherausfall — eine falsche
271
+ // Diagnose fuer ein falsches Zeichen im Suchmuster.
272
+ assertValidGlobPatterns(query.filters.pathGlobs, "pathGlob");
273
+ assertValidGlobPatterns(extensions.pathExcludes, "pathExclude");
274
+ const ledger = new SearchSkipLedger();
275
+ const warnings = [];
276
+ // Die Obergrenze aus der Konfiguration ist eine Grenze, kein Vorschlag:
277
+ // ohne sie zieht eine Anfrage mit candidateLimit=100000 drei Rundlaeufe
278
+ // ueber 100000 Punkte plus einen BM25-Aufbau darueber.
279
+ //
280
+ // WARUM AUCH `limit` GEDECKELT WIRD: `SearchQueryV1` verlangt
281
+ // `candidateLimit >= limit`, und `normalizeSearchQuery` weist alles andere
282
+ // ab. Wer die Obergrenze nur auf `candidateLimit` anwendet und den Wert
283
+ // anschliessend mit `Math.max(max, limit)` wieder anhebt, hat die Grenze
284
+ // damit fuer JEDE Anfrage mit grossem `limit` ausgeschaltet — die drei
285
+ // Speicherrundlaeufe ueber 100000 Punkte laufen dann wie zuvor. Die
286
+ // Obergrenze ist eine Aussage ueber die ARBEIT, nicht ueber ein einzelnes
287
+ // Feld: mehr als `maxCandidateLimit` Kandidaten werden nicht geholt, also
288
+ // koennen auch nicht mehr als `maxCandidateLimit` Treffer entstehen. Beide
289
+ // Deckelungen stehen einzeln und benannt im Ergebnis (E4).
290
+ const maxCandidateLimit = this.config.maxCandidateLimit;
291
+ const candidateCapActive = Number.isInteger(maxCandidateLimit) && maxCandidateLimit > 0;
292
+ let candidateLimit = query.candidateLimit;
293
+ let limit = query.limit;
294
+ if (candidateCapActive && candidateLimit > maxCandidateLimit) {
295
+ candidateLimit = maxCandidateLimit;
296
+ warnings.push({
297
+ code: "CANDIDATE_LIMIT_CLAMPED",
298
+ message: `candidateLimit ${query.candidateLimit} exceeds the configured maximum of ${maxCandidateLimit}; the search ran with ${candidateLimit} candidates per branch instead`,
299
+ details: {
300
+ requested: query.candidateLimit,
301
+ effective: candidateLimit,
302
+ maxCandidateLimit,
303
+ },
304
+ });
305
+ }
306
+ if (limit > candidateLimit) {
307
+ warnings.push({
308
+ code: "RESULT_LIMIT_CLAMPED",
309
+ message: `limit ${limit} exceeds the candidate pool of ${candidateLimit} (configured maximum ${maxCandidateLimit}); the search returned at most ${candidateLimit} hits per profile group instead`,
310
+ details: {
311
+ requested: limit,
312
+ effective: candidateLimit,
313
+ candidateLimit,
314
+ maxCandidateLimit,
315
+ },
316
+ });
317
+ limit = candidateLimit;
318
+ }
319
+ const effectiveQuery = {
320
+ ...query,
321
+ includeContent: extensions.includeContent,
322
+ candidateLimit,
323
+ limit,
324
+ };
325
+ // Die VOLLSTAENDIGE normalisierte Anfrage geht hinein, nicht eine von Hand
326
+ // gepflegte Auswahl daraus: `buildSearchQueryFingerprint` streicht selbst,
327
+ // was reine Darstellung ist (siehe SEARCH_FINGERPRINT_IGNORED_FIELDS).
328
+ // Wuerde hier eine Auswahl stehen, fiele jedes spaeter ergaenzte
329
+ // ergebnisveraendernde Feld wieder lautlos heraus — genau so verloren
330
+ // `workspaceId`, `minScore` und `diversify` ihren Platz im Fingerabdruck.
331
+ const rankingOptions = fingerprintRankingOptions(extensions);
332
+ const fingerprint = buildSearchQueryFingerprint({
333
+ ...effectiveQuery,
334
+ ...(rankingOptions === undefined ? {} : { rankingOptions }),
335
+ });
336
+ const rankingContextFingerprint = this.rankingContextFingerprint(effectiveQuery, extensions);
337
+ // Ein nicht verdrahteter Port bleibt derselbe ehrliche UNAVAILABLE-Fall
338
+ // wie bisher. Ist einer vorhanden, wird er erst NACH Ranking, Dedupe und
339
+ // Limit gerufen (unterhalb der Zielschleife).
340
+ if (effectiveQuery.includeGraphContext && this.deps.graphEnrichment === undefined) {
341
+ warnings.push({
342
+ code: "GRAPH_CONTEXT_UNAVAILABLE",
343
+ message: "the request asked for structural graph context (includeContext), but no graph enrichment provider is wired; the hits carry NO graph context",
344
+ });
345
+ }
346
+ if (effectiveQuery.includeLinked && this.deps.graphEnrichment === undefined) {
347
+ warnings.push({
348
+ code: "LINKED_ARTIFACTS_UNAVAILABLE",
349
+ message: "the request asked for linked symbols (includeLinked), but no graph enrichment provider is wired; the hits carry NO linked symbols",
350
+ });
351
+ }
352
+ if (effectiveQuery.filters.findability !== undefined && this.deps.findability === undefined) {
353
+ warnings.push({
354
+ code: "FINDABILITY_FILTER_UNAVAILABLE",
355
+ message: "the request asked for a findability filter, but no findability lookup is wired into the engine — the filter was NOT applied",
356
+ details: { filter: effectiveQuery.filters.findability },
357
+ });
358
+ }
359
+ if (branchesOfMode(extensions.mode).lexical && this.deps.sparseVectorizer === undefined) {
360
+ warnings.push({
361
+ code: "SPARSE_RECALL_UNAVAILABLE",
362
+ message: "no sparse vectorizer is wired into the engine; the requested lexical retrieval branch is unavailable",
363
+ });
364
+ }
365
+ const groups = [];
366
+ const failedTargets = [];
367
+ const refIdsByRepo = new Map();
368
+ for (const target of this.deps.targets) {
369
+ options?.signal?.throwIfAborted();
370
+ try {
371
+ const outcome = await this.searchTarget(target, effectiveQuery, extensions, ledger, minScoreOrigin, options);
372
+ groups.push(outcome.group);
373
+ warnings.push(...(outcome.branchWarnings ?? []));
374
+ if (outcome.relevanceWarning !== undefined) {
375
+ warnings.push(outcome.relevanceWarning);
376
+ }
377
+ if (outcome.indexContextWarning !== undefined) {
378
+ warnings.push(outcome.indexContextWarning);
379
+ }
380
+ for (const hit of outcome.group.hits) {
381
+ const set = refIdsByRepo.get(hit.repoId) ?? new Set();
382
+ set.add(hit.refId);
383
+ refIdsByRepo.set(hit.repoId, set);
384
+ }
385
+ }
386
+ catch (error) {
387
+ rethrowCancellation(error, options);
388
+ // E4: ein ausgefallenes Ziel wird gemeldet, nicht verschluckt. Die
389
+ // uebrigen Profile werden weiter durchsucht (spec.md §25.2: eine
390
+ // Lesende Operation degradiert sichtbar, statt ganz zu scheitern).
391
+ const message = error instanceof Error ? error.message : String(error);
392
+ const code = isUntangleError(error) ? error.code : "PROFILE_UNAVAILABLE";
393
+ failedTargets.push({ profileId: target.embeddingProfileId, error });
394
+ warnings.push({
395
+ code,
396
+ message: `profile "${target.embeddingProfileId}" could not be searched: ${message}`,
397
+ details: { collection: target.collection, repoIds: target.repoIds },
398
+ });
399
+ ledger.add({
400
+ subjectKind: "search_candidate",
401
+ subjectId: target.embeddingProfileId,
402
+ reason: "provider_unavailable",
403
+ detail: `collection "${target.collection}" did not answer: ${message}`,
404
+ recoverable: true,
405
+ });
406
+ this.deps.logger?.log({
407
+ timestamp: this.deps.clock?.nowIso() ?? "",
408
+ level: "warn",
409
+ component: "search",
410
+ event: "profile_search_failed",
411
+ context: { profileId: target.embeddingProfileId, message },
412
+ });
413
+ }
414
+ }
415
+ if (failedTargets.length > 0 &&
416
+ !groups.some((group) => group.notSearchedReason === undefined)) {
417
+ const first = failedTargets[0];
418
+ throw new UntangleError(isUntangleError(first.error) ? first.error.code : "QDRANT_UNAVAILABLE", `search failed: no requested profile completed a retrieval branch; ${failedTargets
419
+ .map(({ profileId, error }) => `profile "${profileId}": ${error instanceof Error ? error.message : String(error)}`)
420
+ .join("; ")}`, {
421
+ operation: "search",
422
+ retryable: failedTargets.some(({ error }) => !isUntangleError(error) || error.retryable),
423
+ context: { failedProfileIds: failedTargets.map(({ profileId }) => profileId) },
424
+ cause: first.error,
425
+ });
426
+ }
427
+ // Pipeline-Schritt 11: Nur die BEREITS final begrenzten und gerankten
428
+ // Treffer verlassen den Abrufkern in Richtung Graphadapter. Weder ein
429
+ // Graphsignal noch ein Ladefehler kann dadurch Ranking, Score, Dedupe,
430
+ // Fingerabdruck oder Trefferzahl veraendern.
431
+ //
432
+ // Diese Bedingung fragt, ob es ueberhaupt einen Treffer gibt, an den man
433
+ // Relationen haengen kann. Sie ist AUSDRUECKLICH KEINE Enthaltungspruefung:
434
+ // unter `evidence` behaelt eine von der Policy abgelehnte Gruppe ihre
435
+ // Treffer und wird folgerichtig angereichert. Anreicherung zu ueberspringen,
436
+ // weil die Policy ablehnte, waere eine zweite, verdeckte Bedeutung an
437
+ // derselben Stelle — genau die Verwechslung, die diese Welle beseitigt.
438
+ const hasHitsToEnrich = groups.some((group) => group.hits.length > 0);
439
+ if (this.deps.graphEnrichment !== undefined &&
440
+ (effectiveQuery.includeGraphContext || effectiveQuery.includeLinked) &&
441
+ hasHitsToEnrich) {
442
+ await this.enrichFinalHits(groups, effectiveQuery, warnings, options);
443
+ }
444
+ for (const [repoId, refIds] of refIdsByRepo) {
445
+ if (refIds.size > 1) {
446
+ warnings.push({
447
+ code: "MULTIPLE_REFS",
448
+ message: `repository "${repoId}" contributed hits from ${refIds.size} refs; results from different refs describe different states of the same code`,
449
+ details: { repoId, refIds: sortByCodepoints([...refIds]) },
450
+ });
451
+ }
452
+ }
453
+ const incompleteHits = groups.flatMap((group) => group.hits.filter((hit) => hit.incompletePayloadFields !== undefined));
454
+ if (incompleteHits.length > 0) {
455
+ warnings.push({
456
+ code: "INCOMPLETE_PAYLOAD",
457
+ message: `${incompleteHits.length} hit(s) carry payload fields that were missing in the index and filled with documented substitutes; see each hit's incompletePayloadFields`,
458
+ details: {
459
+ chunkIds: sortByCodepoints(incompleteHits.map((hit) => hit.chunkId)),
460
+ },
461
+ });
462
+ }
463
+ const profileSeparation = groups.length > 1;
464
+ if (profileSeparation) {
465
+ warnings.push({
466
+ code: "PROFILE_SEPARATION",
467
+ message: "results are grouped per embedding profile; scores are comparable only WITHIN a group (spec.md §13.4)",
468
+ details: { profileIds: groups.map((group) => group.embeddingProfileId) },
469
+ });
470
+ }
471
+ return {
472
+ schemaVersion: 1,
473
+ queryFingerprint: fingerprint,
474
+ rankingContextFingerprint,
475
+ query: effectiveQuery,
476
+ groups,
477
+ profileSeparation,
478
+ totalHits: groups.reduce((sum, group) => sum + group.hits.length, 0),
479
+ skipped: ledger.toArray(),
480
+ warnings,
481
+ };
482
+ }
483
+ rankingContextFingerprint(query, extensions) {
484
+ const targets = this.deps.targets
485
+ .map((target) => ({
486
+ embeddingProfileId: target.embeddingProfileId,
487
+ collection: target.collection,
488
+ repoIds: uniqueSortedByCodepoints(target.repoIds),
489
+ }))
490
+ .sort((left, right) => compareCodepoints(canonicalJson(left), canonicalJson(right)));
491
+ const relevancePolicies = [...this.relevancePolicies.entries()]
492
+ .sort(([left], [right]) => compareCodepoints(left, right))
493
+ .map(([embeddingProfileId, binding]) => ({ embeddingProfileId, ...binding }));
494
+ const repoPriorities = Object.entries(this.deps.repoPriorities ?? {}).sort(([left], [right]) => compareCodepoints(left, right));
495
+ return sha256Hex(canonicalJson({
496
+ schemaVersion: 1,
497
+ rankerVersion: query.rankerVersion,
498
+ mode: extensions.mode,
499
+ // Mode-selected recall and independent branch failure handling must
500
+ // never be cohort-compatible with the previous rank-only strategy.
501
+ retrievalStrategy: "independent-branches-v1",
502
+ ...(extensions.mode === "hybrid" ? { branchTimeoutMs: this.branchTimeoutMs } : {}),
503
+ // Ausdruecklicher bedingter Spread, NICHT `abstention: x ?? undefined`:
504
+ // der Hash darf nicht davon abhaengen, ob canonicalJson undefined
505
+ // wegwirft. Unter `strict` ist das gehashte Objekt buchstaeblich
506
+ // unveraendert, also bleiben persistierte Journey-Kohorten vergleichbar.
507
+ ...(extensions.abstention === "strict" ? {} : { abstention: extensions.abstention }),
508
+ sort: extensions.sort,
509
+ complexityBias: extensions.complexityBias,
510
+ diversify: query.diversify,
511
+ minScore: query.minScore ?? null,
512
+ branchWeights: this.branchWeights,
513
+ boostWeights: this.config.boostWeights,
514
+ recencyHalfLifeDays: this.deps.recencyHalfLifeDays ?? DEFAULT_RECENCY_HALF_LIFE_DAYS,
515
+ clockEnabled: this.deps.clock !== undefined,
516
+ sparseVectorizerEnabled: this.deps.sparseVectorizer !== undefined,
517
+ graphProximityEnabled: this.deps.graphProximity !== undefined,
518
+ findabilityEnabled: this.deps.findability !== undefined,
519
+ targets,
520
+ repoPriorities,
521
+ relevancePolicies,
522
+ }));
523
+ }
524
+ async enrichFinalHits(groups, query, warnings, options) {
525
+ const provider = this.deps.graphEnrichment;
526
+ if (provider === undefined)
527
+ return;
528
+ const targets = groups.flatMap((group, groupIndex) => group.hits.map((hit, hitIndex) => ({
529
+ targetId: `group:${String(groupIndex)}:hit:${String(hitIndex)}`,
530
+ workspaceId: query.workspaceId,
531
+ generationId: hit.indexGeneration,
532
+ repoId: hit.repoId,
533
+ refId: hit.refId,
534
+ fileId: hit.fileId,
535
+ filePath: hit.filePath,
536
+ ...(hit.symbolId === undefined ? {} : { symbolId: hit.symbolId }),
537
+ })));
538
+ try {
539
+ const result = await provider.enrich({
540
+ includeContext: query.includeGraphContext,
541
+ includeLinked: query.includeLinked,
542
+ relationLimit: SEARCH_GRAPH_RELATION_LIMIT,
543
+ targets,
544
+ }, options);
545
+ options?.signal?.throwIfAborted();
546
+ const expectedIds = new Set(targets.map((target) => target.targetId));
547
+ const entries = new Map();
548
+ const duplicateIds = new Set();
549
+ const unexpectedIds = new Set();
550
+ for (const entry of result.entries) {
551
+ if (!expectedIds.has(entry.targetId)) {
552
+ unexpectedIds.add(entry.targetId);
553
+ continue;
554
+ }
555
+ if (entries.has(entry.targetId))
556
+ duplicateIds.add(entry.targetId);
557
+ else
558
+ entries.set(entry.targetId, entry);
559
+ }
560
+ const missingIds = [];
561
+ for (let groupIndex = 0; groupIndex < groups.length; groupIndex++) {
562
+ const group = groups[groupIndex];
563
+ for (let hitIndex = 0; hitIndex < group.hits.length; hitIndex++) {
564
+ const hit = group.hits[hitIndex];
565
+ const targetId = `group:${String(groupIndex)}:hit:${String(hitIndex)}`;
566
+ const entry = duplicateIds.has(targetId) ? undefined : entries.get(targetId);
567
+ if (entry === undefined)
568
+ missingIds.push(targetId);
569
+ const detail = `graph enrichment provider returned no unique result for final hit "${hit.chunkId}"`;
570
+ if (query.includeGraphContext) {
571
+ hit.graphContext =
572
+ entry?.graphContext === undefined
573
+ ? unavailableSearchGraphRelations("provider_result_missing", detail, SEARCH_GRAPH_RELATION_LIMIT)
574
+ : createSearchGraphRelations(entry.graphContext.relations, entry.graphContext.omissions, SEARCH_GRAPH_RELATION_LIMIT);
575
+ }
576
+ if (query.includeLinked) {
577
+ hit.linkedSymbols =
578
+ entry?.linkedSymbols === undefined
579
+ ? unavailableSearchGraphRelations("provider_result_missing", detail, SEARCH_GRAPH_RELATION_LIMIT)
580
+ : createSearchGraphRelations(entry.linkedSymbols.relations, entry.linkedSymbols.omissions, SEARCH_GRAPH_RELATION_LIMIT);
581
+ }
582
+ }
583
+ }
584
+ const providerWarnings = [...result.warnings]
585
+ .map(normalizeGraphProviderWarning)
586
+ .sort((left, right) => {
587
+ const byCode = compareCodepoints(left.code, right.code);
588
+ if (byCode !== 0)
589
+ return byCode;
590
+ const byMessage = compareCodepoints(left.message, right.message);
591
+ return byMessage !== 0
592
+ ? byMessage
593
+ : compareCodepoints(JSON.stringify(left.details ?? {}), JSON.stringify(right.details ?? {}));
594
+ });
595
+ warnings.push(...providerWarnings.slice(0, SEARCH_GRAPH_PROVIDER_WARNING_LIMIT));
596
+ if (providerWarnings.length > SEARCH_GRAPH_PROVIDER_WARNING_LIMIT) {
597
+ warnings.push({
598
+ code: "GRAPH_ENRICHMENT_WARNINGS_TRUNCATED",
599
+ message: `${String(providerWarnings.length - SEARCH_GRAPH_PROVIDER_WARNING_LIMIT)} additional graph enrichment provider warning(s) were omitted by the response limit`,
600
+ details: {
601
+ warningLimit: SEARCH_GRAPH_PROVIDER_WARNING_LIMIT,
602
+ omittedWarningCount: providerWarnings.length - SEARCH_GRAPH_PROVIDER_WARNING_LIMIT,
603
+ },
604
+ });
605
+ }
606
+ if (missingIds.length > 0 || duplicateIds.size > 0 || unexpectedIds.size > 0) {
607
+ const missing = boundedDiagnosticIds(missingIds);
608
+ const duplicates = boundedDiagnosticIds([...duplicateIds]);
609
+ const unexpected = boundedDiagnosticIds([...unexpectedIds]);
610
+ warnings.push({
611
+ code: "GRAPH_ENRICHMENT_INCOMPLETE",
612
+ message: "the graph enrichment provider returned an incomplete or ambiguous batch; affected hits carry provider_result_missing omissions",
613
+ details: {
614
+ missingTargetIds: missing.ids,
615
+ omittedMissingTargetIdCount: missing.omittedCount,
616
+ duplicateTargetIds: duplicates.ids,
617
+ omittedDuplicateTargetIdCount: duplicates.omittedCount,
618
+ unexpectedTargetIds: unexpected.ids,
619
+ omittedUnexpectedTargetIdCount: unexpected.omittedCount,
620
+ },
621
+ });
622
+ }
623
+ }
624
+ catch (error) {
625
+ options?.signal?.throwIfAborted();
626
+ const message = error instanceof Error ? error.message : String(error);
627
+ const detail = boundSearchGraphDetail(`graph enrichment failed after ranking: ${message}`);
628
+ markFinalHitGraphUnavailable(groups, query, detail);
629
+ if (query.includeGraphContext) {
630
+ warnings.push({ code: "GRAPH_CONTEXT_UNAVAILABLE", message: detail });
631
+ }
632
+ if (query.includeLinked) {
633
+ warnings.push({ code: "LINKED_ARTIFACTS_UNAVAILABLE", message: detail });
634
+ }
635
+ this.deps.logger?.log({
636
+ timestamp: this.deps.clock?.nowIso() ?? "",
637
+ level: "warn",
638
+ component: "search",
639
+ event: "graph_enrichment_failed",
640
+ context: { message },
641
+ });
642
+ }
643
+ }
644
+ /**
645
+ * grep laeuft im selben Geltungsbereich wie `search`: Workspace aus den
646
+ * Vorgaben, wenn die Anfrage keinen nennt, und nur die Repositories der
647
+ * verdrahteten Ziele. Sonst faende dieselbe Oberflaeche mit demselben
648
+ * Aufrufer je nach Werkzeug verschiedene Mengen.
649
+ */
650
+ grep(request, options) {
651
+ return grepIndexedChunks(this.deps.vectorStore, uniqueSortedByCodepoints(this.deps.targets.map((target) => target.collection)), request, options, {
652
+ workspaceId: this.deps.defaults.workspaceId,
653
+ repoIds: uniqueSortedByCodepoints(this.deps.targets.flatMap((target) => target.repoIds)),
654
+ });
655
+ }
656
+ // -------------------------------------------------------------------------
657
+ // Ein Ziel (= ein Embedding-Profil, eine Collection)
658
+ // -------------------------------------------------------------------------
659
+ async searchTarget(target, query, extensions, ledger, minScoreOrigin, options) {
660
+ const scope = {
661
+ workspaceId: query.workspaceId,
662
+ targetRepoIds: target.repoIds,
663
+ };
664
+ const requestedFilter = buildPayloadFilter(query.filters, scope);
665
+ // A ref-less request must not turn a locally recorded current corpus into
666
+ // an unbounded scan of every historical ref still present in the shared
667
+ // collection. Use every recorded ref in the effective repo scope. Doing
668
+ // this at the port boundary keeps non-CLI callers honest as well.
669
+ const inferredRefIds = query.filters.refIds.length === 0 && requestedFilter.emptyScopeReason === undefined
670
+ ? uniqueSortedByCodepoints((target.indexContexts ?? [])
671
+ .filter((context) => requestedFilter.scopeRepoIds.length === 0 ||
672
+ requestedFilter.scopeRepoIds.includes(context.repoId))
673
+ .map((context) => context.refId))
674
+ : [];
675
+ const effectiveRefIds = query.filters.refIds.length === 0 ? inferredRefIds : query.filters.refIds;
676
+ const built = query.filters.refIds.length === 0 && inferredRefIds.length > 0
677
+ ? buildPayloadFilter({ ...query.filters, refIds: inferredRefIds }, scope)
678
+ : requestedFilter;
679
+ // WP-178: Die zweite Verteidigungslinie prueft ab hier den WIRKSAMEN
680
+ // Bereich (Ziel geschnitten mit dem Repo-Filter der Anfrage), nicht mehr
681
+ // nur die Zieldeklaration. Ein Speicher, der den Payloadfilter ignoriert,
682
+ // lieferte sonst einen Treffer aus einem Repository, das der Nutzer
683
+ // ausdruecklich NICHT angefragt hat — waehrend `group.repoIds` das
684
+ // Gegenteil behauptet.
685
+ const effectiveScope = {
686
+ workspaceId: query.workspaceId,
687
+ targetRepoIds: built.scopeRepoIds,
688
+ };
689
+ if (built.emptyScopeReason !== undefined) {
690
+ // E4: Der Speicher wird NICHT gefragt — eine leere `in`-Liste waere
691
+ // kein Filter, sondern gar keiner, und die Suche liefe ueber die ganze
692
+ // Collection. Der leere Bereich steht stattdessen im Ledger.
693
+ ledger.add({
694
+ subjectKind: "search_candidate",
695
+ subjectId: target.embeddingProfileId,
696
+ reason: "excluded_by_configuration",
697
+ detail: built.emptyScopeReason,
698
+ recoverable: true,
699
+ });
700
+ return {
701
+ group: {
702
+ embeddingProfileId: target.embeddingProfileId,
703
+ repoIds: [],
704
+ hits: [],
705
+ rankNormalized: false,
706
+ caveats: [built.emptyScopeReason],
707
+ // E4: „nicht gesucht" ist im Ergebnis von „gesucht, nichts gefunden"
708
+ // zu unterscheiden. Ohne diese Markierung sieht die Gruppe wie eine
709
+ // leere Trefferliste aus, und die Findability-Sonde macht daraus
710
+ // „das Symbol findet sich selbst nicht" — eine Aussage ueber den
711
+ // Code, die auf keiner Beobachtung steht.
712
+ notSearchedReason: built.emptyScopeReason,
713
+ },
714
+ refIds: new Set(),
715
+ };
716
+ }
717
+ // WP-176 (E4): Angefragte Repositories, die dieses Ziel gar nicht fuehrt,
718
+ // fielen bisher lautlos aus dem Durchschnitt. Nur der VOLLSTAENDIG leere
719
+ // Fall (oben) kam in den Ledger. Ein teilweiser Ausfall sieht damit wie
720
+ // eine vollstaendige Suche aus — die gefaehrlichere der beiden Formen,
721
+ // weil sie Treffer liefert und deshalb niemand nachfragt.
722
+ const outOfScope = built.outOfScopeRepoIds;
723
+ const caveats = built.residual.map((residual) => `filter "${residual.name}" was applied in-process, not in the store: ${residual.reason}`);
724
+ if (query.filters.refIds.length === 0 && inferredRefIds.length > 0) {
725
+ caveats.push(`the request omitted refs; this target was searched only across locally recorded refs [${inferredRefIds.join(", ")}]`);
726
+ }
727
+ if (outOfScope.length > 0) {
728
+ const detail = `the requested repositories [${outOfScope.join(", ")}] are not part of this target's repositories [${sortByCodepoints([...target.repoIds]).join(", ")}]; they were NOT searched here`;
729
+ caveats.push(detail);
730
+ for (const repoId of outOfScope) {
731
+ ledger.add({
732
+ subjectKind: "search_candidate",
733
+ subjectId: `${target.embeddingProfileId}:${repoId}`,
734
+ reason: "excluded_by_configuration",
735
+ detail,
736
+ recoverable: true,
737
+ });
738
+ }
739
+ }
740
+ const pool = new Map();
741
+ const rejectedPointIds = new Set();
742
+ const active = branchesOfMode(extensions.mode);
743
+ const retrieval = await retrieveIndependentBranches({
744
+ vectorStore: this.deps.vectorStore,
745
+ target,
746
+ text: query.text,
747
+ filter: built.filter,
748
+ candidateLimit: query.candidateLimit,
749
+ mode: extensions.mode,
750
+ branchTimeoutMs: this.branchTimeoutMs,
751
+ ...(this.deps.sparseVectorizer === undefined
752
+ ? {}
753
+ : { sparseVectorizer: this.deps.sparseVectorizer }),
754
+ }, options);
755
+ caveats.push(...retrieval.caveats);
756
+ for (const skipped of retrieval.skipped)
757
+ ledger.add(skipped);
758
+ assertCompletedRetrieval(retrieval, target);
759
+ const { densePoints, sparsePoints, symbolPage, completedBranches, warnings: branchWarnings, } = retrieval;
760
+ this.collect(densePoints, pool, rejectedPointIds, ledger, (candidate, point) => {
761
+ candidate.denseScore = point.denseScore ?? point.score;
762
+ });
763
+ this.collect(sparsePoints, pool, rejectedPointIds, ledger, () => undefined);
764
+ // --- Abruf: Symbolzweig ----------------------------------------------
765
+ let symbolPointCount = 0;
766
+ let symbolLimitReached = false;
767
+ if (symbolPage !== undefined) {
768
+ const symbolPoints = symbolPage.points.map((point) => ({
769
+ id: point.id,
770
+ score: 0,
771
+ payload: point.payload,
772
+ }));
773
+ symbolPointCount = symbolPoints.length;
774
+ symbolLimitReached =
775
+ symbolPage.nextOffset !== undefined || symbolPoints.length >= query.candidateLimit;
776
+ this.collect(symbolPoints, pool, rejectedPointIds, ledger, () => undefined);
777
+ if (symbolPage.nextOffset !== undefined) {
778
+ ledger.add({
779
+ subjectKind: "search_candidate",
780
+ subjectId: `${target.collection}:symbol_scroll`,
781
+ reason: "excluded_by_configuration",
782
+ detail: `exact symbol-name scroll stopped at candidateLimit=${query.candidateLimit}; more exact matches exist in "${target.collection}"`,
783
+ recoverable: true,
784
+ });
785
+ }
786
+ }
787
+ // --- Residuale Filter VOR jeder Rangfolge ------------------------------
788
+ // Ein Kandidat, der laut Anfrage ungueltig ist, darf weder Pool-IDF noch
789
+ // Zweigraenge, exakten Intent oder dessen Gewicht beeinflussen. Dadurch
790
+ // entsprechen alle erklaerten Raenge der Menge, die wirklich antworten
791
+ // durfte. `passesResidualFilters` wird genau einmal je Kandidat gerufen,
792
+ // also entsteht auch genau ein Ledger-Eintrag je Ausschluss.
793
+ const candidates = [...pool.values()].filter((candidate) => this.passesResidualFilters(candidate, query, extensions, effectiveScope, effectiveRefIds, ledger));
794
+ const eligibleIds = new Set(candidates.map((candidate) => candidate.pointId));
795
+ const denseRankedIds = [];
796
+ const denseSeen = new Set();
797
+ for (const point of densePoints) {
798
+ if (!eligibleIds.has(point.id) || denseSeen.has(point.id)) {
799
+ continue;
800
+ }
801
+ const candidate = pool.get(point.id);
802
+ if (candidate === undefined) {
803
+ continue;
804
+ }
805
+ denseSeen.add(point.id);
806
+ denseRankedIds.push(point.id);
807
+ candidate.denseRank = denseRankedIds.length;
808
+ }
809
+ // --- Rangfolge: lexikalisch (eigenes BM25 ueber den gueltigen Pool) ----
810
+ const lexicalRanker = new Bm25Ranker(candidates.map((candidate) => ({
811
+ id: candidate.pointId,
812
+ text: lexicalDocumentText(candidate.payload),
813
+ })));
814
+ const lexicalRanking = active.lexical && completedBranches.has("sparse") ? lexicalRanker.rank(query.text) : [];
815
+ const lexicalRankedIds = lexicalRanking.map((entry) => entry.id);
816
+ for (const entry of lexicalRanking) {
817
+ const candidate = pool.get(entry.id);
818
+ if (candidate !== undefined) {
819
+ candidate.lexicalScore = entry.score;
820
+ candidate.lexicalRank = entry.rank;
821
+ }
822
+ }
823
+ // --- Rangfolge: exakter Symbolname ------------------------------------
824
+ const symbolMatches = [];
825
+ for (const candidate of candidates) {
826
+ const match = matchSymbolName(candidate.payload, query.text);
827
+ if (match !== null) {
828
+ candidate.symbolMatchStrength = match.strength;
829
+ candidate.symbolMatchEvidence = match.evidence;
830
+ symbolMatches.push({
831
+ id: candidate.pointId,
832
+ strength: match.strength,
833
+ repoId: candidate.payload.repo_id,
834
+ refId: candidate.payload.ref_id,
835
+ filePath: candidate.payload.file_path,
836
+ startLine: candidate.payload.start_line,
837
+ });
838
+ }
839
+ }
840
+ symbolMatches.sort((a, b) => {
841
+ if (a.strength !== b.strength) {
842
+ return b.strength - a.strength;
843
+ }
844
+ const byRepo = compareCodepoints(a.repoId, b.repoId);
845
+ if (byRepo !== 0) {
846
+ return byRepo;
847
+ }
848
+ const byRef = compareCodepoints(a.refId, b.refId);
849
+ if (byRef !== 0) {
850
+ return byRef;
851
+ }
852
+ const byPath = compareCodepoints(a.filePath, b.filePath);
853
+ if (byPath !== 0) {
854
+ return byPath;
855
+ }
856
+ if (a.startLine !== b.startLine) {
857
+ return a.startLine - b.startLine;
858
+ }
859
+ return compareCodepoints(a.id, b.id);
860
+ });
861
+ // --- Fusion ------------------------------------------------------------
862
+ const available = {
863
+ dense: active.dense && completedBranches.has("dense"),
864
+ lexical: active.lexical && completedBranches.has("sparse"),
865
+ symbol: active.symbol && completedBranches.has("symbol"),
866
+ };
867
+ const inputs = [];
868
+ if (available.dense) {
869
+ inputs.push({ branch: "dense", rankedIds: denseRankedIds });
870
+ }
871
+ // WP-232: Bei klar deutsch formulierten NATUERLICHEN Fragen kann BM25 ein
872
+ // systematischer Gegenspieler des multilingualen Embeddings sein. Es wird
873
+ // aber nur dann weggelassen, wenn sein eigener Spitzenkandidat NICHT in
874
+ // den dichten Top 10 bestaetigt ist. So bleibt nuetzliche Uebereinstimmung
875
+ // (d1 und die deutsche INDEX-SHARING-Probe) erhalten, waehrend die
876
+ // gemessenen Fehlraenge d3/d4/d6 nicht mehr in die Fusion eingehen.
877
+ const omitLexicalForGermanQuestion = extensions.mode === "hybrid" &&
878
+ available.dense &&
879
+ available.lexical &&
880
+ shouldOmitUncorroboratedGermanLexicalBranch(query.text, denseRankedIds, lexicalRankedIds);
881
+ if (available.lexical && !omitLexicalForGermanQuestion) {
882
+ inputs.push({ branch: "lexical", rankedIds: lexicalRankedIds });
883
+ }
884
+ if (omitLexicalForGermanQuestion) {
885
+ caveats.push(`the lexical ranking branch did not participate because its top candidate was not corroborated in the dense top ${GERMAN_LEXICAL_CORROBORATION_DEPTH} for this German natural-language question; dense multilingual ranking remains active (WP-232)`);
886
+ }
887
+ const effectiveActive = {
888
+ ...available,
889
+ lexical: available.lexical && !omitLexicalForGermanQuestion,
890
+ };
891
+ const exactSymbolMatches = symbolMatches.filter((entry) => entry.strength === 1);
892
+ const exactSymbolIntent = available.symbol && exactSymbolMatches.length > 0 && this.branchWeights.symbol > 0;
893
+ const symbolRanking = exactSymbolIntent ? exactSymbolMatches : symbolMatches;
894
+ if (available.symbol) {
895
+ symbolRanking.forEach((entry, index) => {
896
+ const candidate = pool.get(entry.id);
897
+ if (candidate !== undefined) {
898
+ candidate.symbolRank = index + 1;
899
+ }
900
+ });
901
+ inputs.push({ branch: "symbol", rankedIds: symbolRanking.map((entry) => entry.id) });
902
+ }
903
+ const effectiveBranchWeights = exactSymbolIntent
904
+ ? exactSymbolIntentBranchWeights(this.branchWeights, effectiveActive)
905
+ : this.branchWeights;
906
+ if (exactSymbolIntent) {
907
+ caveats.push(`exact symbol-name intent: the symbol RRF branch contains only ${exactSymbolMatches.length} character-identical match(es) and uses weight ${effectiveBranchWeights.symbol}; heuristic boost clamping remains unchanged`);
908
+ }
909
+ const fused = fuseRankings(inputs, effectiveBranchWeights);
910
+ const fusionById = new Map(fused.map((entry) => [entry.id, entry]));
911
+ // Kandidaten, die in KEINEM aktiven Zweig auftauchen, sind fuer diesen
912
+ // Modus schlicht nicht relevant. Sie verschwinden trotzdem nicht
913
+ // stillschweigend (E4).
914
+ for (const candidate of candidates) {
915
+ if (!fusionById.has(candidate.pointId)) {
916
+ ledger.addCandidate({
917
+ subjectId: candidate.payload.chunk_id,
918
+ path: candidate.payload.file_path,
919
+ repoId: candidate.payload.repo_id,
920
+ refId: candidate.payload.ref_id,
921
+ reason: "excluded_by_configuration",
922
+ detail: `no active ranking branch (mode "${extensions.mode}") produced a rank for this candidate`,
923
+ });
924
+ }
925
+ }
926
+ // --- Kandidaten mit Basisrang ------------------------------------------
927
+ const scored = [];
928
+ for (const entry of fused) {
929
+ const candidate = pool.get(entry.id);
930
+ if (candidate === undefined) {
931
+ continue;
932
+ }
933
+ scored.push({ candidate, baseScore: entry.normalized });
934
+ }
935
+ // --- Dedupe -------------------------------------------------------------
936
+ const deduped = dedupeCandidates(scored);
937
+ for (const group of deduped) {
938
+ for (const absorbed of group.absorbed) {
939
+ ledger.addCandidate({
940
+ subjectId: absorbed.candidate.payload.chunk_id,
941
+ path: absorbed.candidate.payload.file_path,
942
+ repoId: absorbed.candidate.payload.repo_id,
943
+ refId: absorbed.candidate.payload.ref_id,
944
+ reason: "duplicate",
945
+ detail: `merged into chunk "${group.primary.candidate.payload.chunk_id}" as a related fragment (same metrics_scope_id "${absorbed.candidate.payload.metrics_scope_id}")`,
946
+ });
947
+ }
948
+ }
949
+ // --- Diversifizierung ---------------------------------------------------
950
+ const penalties = query.diversify
951
+ ? assignDuplicatePenalties(deduped, (entry) => entry.primary.candidate.payload)
952
+ : deduped.map((entry) => ({ item: entry, penaltyRawValue: 0 }));
953
+ // --- Score-Zerlegung ----------------------------------------------------
954
+ const scoringContext = {
955
+ queryText: query.text,
956
+ weights: this.config.boostWeights,
957
+ complexityBias: extensions.complexityBias,
958
+ repoPriorities: this.deps.repoPriorities ?? {},
959
+ graphProximity: this.buildProximityMap(deduped),
960
+ ...(this.deps.clock === undefined ? {} : { nowEpochMs: this.deps.clock.nowEpochMs() }),
961
+ recencyHalfLifeDays: this.deps.recencyHalfLifeDays ?? DEFAULT_RECENCY_HALF_LIFE_DAYS,
962
+ };
963
+ let hits = [];
964
+ for (const penalized of penalties) {
965
+ const group = penalized.item;
966
+ const fusion = fusionById.get(group.primary.candidate.pointId);
967
+ const breakdown = assembleScoreBreakdown(group.primary.candidate, fusion, scoringContext, penalized.penaltyRawValue);
968
+ const minScore = query.minScore;
969
+ if (minScore !== undefined && breakdown.final < minScore) {
970
+ ledger.addCandidate({
971
+ subjectId: group.primary.candidate.payload.chunk_id,
972
+ path: group.primary.candidate.payload.file_path,
973
+ repoId: group.primary.candidate.payload.repo_id,
974
+ refId: group.primary.candidate.payload.ref_id,
975
+ reason: "excluded_by_configuration",
976
+ detail: `final score ${breakdown.final} is below the effective minScore ${minScore} (${minScoreOrigin})`,
977
+ });
978
+ continue;
979
+ }
980
+ hits.push(this.buildHit(group.primary.candidate, breakdown, group.dedupeKey, group.relatedFragments, target, extensions));
981
+ }
982
+ hits = sortHits(hits, extensions.sort);
983
+ let relevanceWarning;
984
+ // Das Verdikt gehoert an die Gruppe, nicht nur in eine Warnung. Bis Welle
985
+ // 14 war es ausschliesslich in `relevanceWarning.details` flachgeklopft —
986
+ // deshalb lasen zehn Stellen im Repository die Enthaltung aus der
987
+ // Listenlaenge statt aus der Entscheidung.
988
+ let relevanceVerdict;
989
+ const relevanceBinding = this.relevancePolicies.get(target.embeddingProfileId);
990
+ if (relevanceBinding === undefined && this.relevancePolicies.size > 0) {
991
+ const configuredEmbeddingProfileIds = sortByCodepoints([...this.relevancePolicies.keys()]);
992
+ const detail = `relevance policy was not evaluated because active embedding profile "${target.embeddingProfileId}" has no configured binding; configured profile keys: ${configuredEmbeddingProfileIds.join(", ")}; results were left unchanged`;
993
+ caveats.push(detail);
994
+ relevanceWarning = {
995
+ code: RELEVANCE_POLICY_NOT_EVALUATED_WARNING_CODE,
996
+ message: detail,
997
+ details: {
998
+ status: "not_evaluated",
999
+ basis: "policy_profile_mismatch",
1000
+ actualEmbeddingProfileId: target.embeddingProfileId,
1001
+ configuredEmbeddingProfileIds,
1002
+ },
1003
+ };
1004
+ relevanceVerdict = {
1005
+ abstention: extensions.abstention,
1006
+ status: "not_evaluated",
1007
+ basis: "policy_profile_mismatch",
1008
+ };
1009
+ }
1010
+ else if (relevanceBinding !== undefined) {
1011
+ const exactSymbolIds = new Set(exactSymbolMatches.map((exact) => exact.id));
1012
+ const prerequisiteFailure = relevancePrerequisiteFailure({
1013
+ binding: relevanceBinding,
1014
+ embeddingProfileId: target.embeddingProfileId,
1015
+ workspaceId: query.workspaceId,
1016
+ collection: target.collection,
1017
+ completedBranches,
1018
+ exactSymbolEvidenceAvailable: active.symbol && exactSymbolIds.size > 0,
1019
+ abstention: extensions.abstention,
1020
+ });
1021
+ if (prerequisiteFailure !== undefined) {
1022
+ caveats.push(prerequisiteFailure.detail);
1023
+ relevanceWarning = prerequisiteFailure.warning;
1024
+ relevanceVerdict = prerequisiteFailure.verdict;
1025
+ }
1026
+ else {
1027
+ // Die Schwellen wurden gegen das feste Praefix von jeweils Rang 50
1028
+ // des dichten UND spaerlichen Abrufzweigs kalibriert. Der
1029
+ // Ranking-Pool darf groesser sein, die Abwesenheitsevidenz aber nicht:
1030
+ // sonst veraendert candidateLimit die Pool-IDF und damit bei
1031
+ // identischer Anfrage die Ja/Nein-Antwort. Der Symbolabruf gehoert
1032
+ // nicht zur IDF-Kalibrierung; ein exakter Name bleibt eigene direkte
1033
+ // Evidenz.
1034
+ // Residual ungueltige IDs sind bereits aus `eligibleIds` entfernt und
1035
+ // werden innerhalb des festen Praefixes bewusst NICHT durch tiefere
1036
+ // Kandidaten ersetzt.
1037
+ const evidencePool = selectFixedRelevanceEvidencePool([
1038
+ { branch: "dense", candidateIds: densePoints.map((point) => point.id) },
1039
+ { branch: "sparse", candidateIds: sparsePoints.map((point) => point.id) },
1040
+ ], eligibleIds, relevanceBinding.policy.denseReferenceRank);
1041
+ const evidenceIds = [...evidencePool.candidateIds];
1042
+ const evidenceIdSet = new Set(evidenceIds);
1043
+ // Der exakte Symbolpfad bleibt direkte Evidenz, auch wenn sehr viele
1044
+ // gleichartige Scroll-Treffer vor ihm liegen. Nur residual gueltige
1045
+ // `symbolMatches` koennen hier auftauchen.
1046
+ if (active.symbol) {
1047
+ for (const exactSymbolId of exactSymbolIds) {
1048
+ if (!evidenceIdSet.has(exactSymbolId)) {
1049
+ evidenceIdSet.add(exactSymbolId);
1050
+ evidenceIds.push(exactSymbolId);
1051
+ }
1052
+ }
1053
+ }
1054
+ const denseEvidenceIds = new Set(evidencePool.denseCandidateIds);
1055
+ const relevanceDecision = decideResultRelevance(query.text, evidenceIds.flatMap((candidateId) => {
1056
+ const candidate = pool.get(candidateId);
1057
+ if (candidate === undefined) {
1058
+ return [];
1059
+ }
1060
+ return [
1061
+ {
1062
+ ...(denseEvidenceIds.has(candidateId) && candidate.denseScore !== undefined
1063
+ ? { denseScore: candidate.denseScore }
1064
+ : {}),
1065
+ lexicalText: lexicalDocumentText(candidate.payload),
1066
+ exactSymbolMatch: active.symbol && exactSymbolIds.has(candidate.pointId),
1067
+ },
1068
+ ];
1069
+ }), relevanceBinding.policy);
1070
+ relevanceVerdict = {
1071
+ abstention: extensions.abstention,
1072
+ status: relevanceDecision.status,
1073
+ basis: relevanceDecision.basis,
1074
+ calibrationId: relevanceBinding.policy.calibrationId,
1075
+ };
1076
+ if (relevanceDecision.status === "rejected") {
1077
+ const observedReferenceRank = relevanceDecision.denseReferenceRankObserved;
1078
+ const detail = `candidate rejected by relevance policy "${relevanceBinding.policy.calibrationId}": top dense ${relevanceDecision.topDenseScore ?? "none"} (minimum ${relevanceBinding.policy.minimumDenseScore}), spread between rank 1 and observed rank ${observedReferenceRank} ${relevanceDecision.denseSpread ?? "unavailable"} (minimum ${relevanceBinding.policy.minimumDenseSpread}), fixed per-branch candidate-evidence pool IDF coverage ${relevanceDecision.candidateIdfCoverage} across ${relevanceDecision.candidateEvidenceCount} candidates (minimum ${relevanceBinding.policy.minimumCandidateIdfCoverage}); all three non-symbol conditions are required`;
1079
+ // NUR `strict` verwirft. Unter `evidence` bleiben die Treffer
1080
+ // stehen und tragen das Verdikt mit — und dann darf auch KEIN
1081
+ // Ledger-Eintrag geschrieben werden: ein Eintrag ueber einen
1082
+ // Gegenstand, der im Ergebnis steht, ist in diesem Repository ein
1083
+ // dokumentierter E4-Defekt (WP-142).
1084
+ if (extensions.abstention === "strict") {
1085
+ for (const dropped of hits) {
1086
+ ledger.addCandidate({
1087
+ subjectId: dropped.chunkId,
1088
+ path: dropped.filePath,
1089
+ repoId: dropped.repoId,
1090
+ refId: dropped.refId,
1091
+ // NICHT `excluded_by_configuration`: es wurde nichts
1092
+ // wegkonfiguriert, und der Kandidat ist nicht durch Filter
1093
+ // oder Budget gefallen. Ohne eigenen Grund liesse sich eine
1094
+ // Enthaltung nicht von einem Score-Schwellwert trennen (E4).
1095
+ reason: "relevance_policy_rejected",
1096
+ detail,
1097
+ });
1098
+ }
1099
+ hits = [];
1100
+ }
1101
+ // Vorbehalt und Warnung sind in BEIDEN Verhalten byte-identisch:
1102
+ // das Urteil aendert sich nicht, nur was mit den Treffern geschieht.
1103
+ caveats.push(detail);
1104
+ relevanceWarning = {
1105
+ code: NO_SUFFICIENT_MATCH_WARNING_CODE,
1106
+ message: `No sufficiently relevant result for embedding profile "${target.embeddingProfileId}": without an exact symbol name, the dense top score, gap to the fixed reference rank and IDF-weighted query coverage must jointly support the verdict within the fixed evidence pool`,
1107
+ details: {
1108
+ embeddingProfileId: target.embeddingProfileId,
1109
+ collection: target.collection,
1110
+ ...relevanceDecision,
1111
+ },
1112
+ };
1113
+ }
1114
+ else if (relevanceDecision.status === "not_evaluated") {
1115
+ const detail = `relevance policy "${relevanceBinding.policy.calibrationId}" was not evaluated: the deepest dense rank actually observed was ${relevanceDecision.deepestDenseRankObserved}, so calibrated reference rank ${relevanceBinding.policy.denseReferenceRank} was not observed; results were left unchanged`;
1116
+ caveats.push(detail);
1117
+ relevanceWarning = {
1118
+ code: RELEVANCE_POLICY_NOT_EVALUATED_WARNING_CODE,
1119
+ message: detail,
1120
+ details: {
1121
+ embeddingProfileId: target.embeddingProfileId,
1122
+ collection: target.collection,
1123
+ ...relevanceDecision,
1124
+ },
1125
+ };
1126
+ }
1127
+ }
1128
+ }
1129
+ const qualifiedHitCount = hits.length;
1130
+ const topScoreGap = hits.length < 2
1131
+ ? null
1132
+ : hits[0].score.final - hits[1].score.final;
1133
+ if (hits.length > query.limit) {
1134
+ for (const dropped of hits.slice(query.limit)) {
1135
+ ledger.addCandidate({
1136
+ subjectId: dropped.chunkId,
1137
+ path: dropped.filePath,
1138
+ repoId: dropped.repoId,
1139
+ refId: dropped.refId,
1140
+ reason: "excluded_by_configuration",
1141
+ detail: `final score ${dropped.score.final} ranked below the requested limit of ${query.limit} results`,
1142
+ });
1143
+ }
1144
+ hits = hits.slice(0, query.limit);
1145
+ }
1146
+ hits = assignRanks(hits);
1147
+ const uniqueEligibleFileCount = new Set(candidates.map((candidate) => `${candidate.payload.repo_id}\u001f${candidate.payload.ref_id}\u001f${candidate.payload.file_path}`)).size;
1148
+ const uniqueEligibleSymbolCount = new Set(candidates.flatMap((candidate) => candidate.payload.symbol_id === undefined
1149
+ ? []
1150
+ : [
1151
+ `${candidate.payload.repo_id}\u001f${candidate.payload.ref_id}\u001f${candidate.payload.symbol_id}`,
1152
+ ])).size;
1153
+ const eligibleFileDispersion = candidates.length <= 1 ? 0 : (uniqueEligibleFileCount - 1) / (candidates.length - 1);
1154
+ const { contexts: scopedIndexContexts, corpusRevisionFingerprint, warning: indexContextWarning, } = this.indexContextProvenance(target, built.scopeRepoIds, effectiveRefIds, candidates, query.filters.refIds.length === 0);
1155
+ // WP-189: Der Vertrag von `SearchHitGroupV1.repoIds` lautet „Repositories,
1156
+ // die zu dieser Gruppe BEITRAGEN". Hier stand der erlaubte Bereich — also
1157
+ // eine Aussage ueber die Anfrage, ausgegeben als Aussage ueber das
1158
+ // Ergebnis. Bei drei erlaubten und einem liefernden Repository nannte die
1159
+ // Gruppe drei Herkuenfte, von denen zwei nichts beigetragen hatten.
1160
+ const contributing = uniqueSortedByCodepoints(hits.map((hit) => hit.repoId));
1161
+ return {
1162
+ group: {
1163
+ embeddingProfileId: target.embeddingProfileId,
1164
+ // A completed local state proves provenance for an empty result, but
1165
+ // must never lend its generation to a returned legacy candidate that
1166
+ // did not carry `generation_id` itself. The display field may retain
1167
+ // the legacy dependency fallback; corpus provenance below never does.
1168
+ indexGenerations: uniqueSortedByCodepoints(candidates.length === 0
1169
+ ? scopedIndexContexts.map((context) => context.generationId)
1170
+ : candidates.map((candidate) => this.indexGenerationOf(candidate))),
1171
+ ...(corpusRevisionFingerprint === undefined ? {} : { corpusRevisionFingerprint }),
1172
+ repoIds: contributing,
1173
+ hits,
1174
+ rankNormalized: false,
1175
+ caveats,
1176
+ ...(relevanceVerdict === undefined ? {} : { relevanceDecision: relevanceVerdict }),
1177
+ retrieval: {
1178
+ branchCandidateCounts: {
1179
+ dense: densePoints.length,
1180
+ sparse: sparsePoints.length,
1181
+ symbol: symbolPointCount,
1182
+ },
1183
+ uniqueRetrievedCandidateCount: pool.size,
1184
+ eligibleCandidateCount: candidates.length,
1185
+ rankedCandidateCount: fused.length,
1186
+ deduplicatedCandidateCount: deduped.length,
1187
+ qualifiedHitCount,
1188
+ returnedHitCount: hits.length,
1189
+ uniqueEligibleFileCount,
1190
+ uniqueEligibleSymbolCount,
1191
+ eligibleFileDispersion,
1192
+ topScoreGap,
1193
+ candidateLimitReached: densePoints.length >= query.candidateLimit ||
1194
+ sparsePoints.length >= query.candidateLimit ||
1195
+ symbolLimitReached,
1196
+ },
1197
+ },
1198
+ refIds: new Set(hits.map((hit) => hit.refId)),
1199
+ ...(relevanceWarning === undefined ? {} : { relevanceWarning }),
1200
+ ...(indexContextWarning === undefined ? {} : { indexContextWarning }),
1201
+ ...(branchWarnings.length === 0 ? {} : { branchWarnings }),
1202
+ };
1203
+ }
1204
+ /** Uebernimmt Speicherpunkte in den Kandidatenpool; unbrauchbare landen im Ledger. */
1205
+ collect(points, pool, rejectedPointIds, ledger, onCandidate) {
1206
+ points.forEach((point) => {
1207
+ if (rejectedPointIds.has(point.id)) {
1208
+ return;
1209
+ }
1210
+ const existing = pool.get(point.id);
1211
+ if (existing !== undefined) {
1212
+ onCandidate(existing, point);
1213
+ return;
1214
+ }
1215
+ const result = candidateFromPoint(point);
1216
+ if (!result.ok) {
1217
+ rejectedPointIds.add(point.id);
1218
+ ledger.addCandidate({
1219
+ subjectId: point.id,
1220
+ reason: "missing_index_entry",
1221
+ detail: `point "${point.id}" could not be read as a code chunk: ${result.reason}`,
1222
+ });
1223
+ return;
1224
+ }
1225
+ pool.set(point.id, result.candidate);
1226
+ onCandidate(result.candidate, point);
1227
+ });
1228
+ }
1229
+ /** Pfad-Globs und Findability — die Filter, die der Speicher nicht kennt. */
1230
+ passesResidualFilters(candidate, query, extensions, scope, effectiveRefIds, ledger) {
1231
+ const { payload } = candidate;
1232
+ // Der Geltungsbereich steht bereits im Speicherfilter. Er wird hier ein
1233
+ // ZWEITES Mal geprueft, weil ein Speicher, der den Filter nicht oder
1234
+ // anders auswertet, sonst fremden Inhalt als eigenen ausliefern wuerde —
1235
+ // und das waere kein ungenaues Ranking, sondern ein Datenleck.
1236
+ if (payload.workspace_id !== scope.workspaceId) {
1237
+ ledger.addCandidate({
1238
+ subjectId: payload.chunk_id,
1239
+ path: payload.file_path,
1240
+ repoId: payload.repo_id,
1241
+ refId: payload.ref_id,
1242
+ reason: "excluded_by_configuration",
1243
+ detail: `candidate belongs to workspace "${payload.workspace_id}", but the query is scoped to workspace "${scope.workspaceId}"`,
1244
+ });
1245
+ return false;
1246
+ }
1247
+ const targetRepoIds = scope.targetRepoIds ?? [];
1248
+ if (targetRepoIds.length > 0 && !targetRepoIds.includes(payload.repo_id)) {
1249
+ ledger.addCandidate({
1250
+ subjectId: payload.chunk_id,
1251
+ path: payload.file_path,
1252
+ repoId: payload.repo_id,
1253
+ refId: payload.ref_id,
1254
+ reason: "excluded_by_configuration",
1255
+ detail: `candidate belongs to repository "${payload.repo_id}", which is not one of this target's repositories [${sortByCodepoints([...targetRepoIds]).join(", ")}]`,
1256
+ });
1257
+ return false;
1258
+ }
1259
+ if (effectiveRefIds.length > 0 && !effectiveRefIds.includes(payload.ref_id)) {
1260
+ ledger.addCandidate({
1261
+ subjectId: payload.chunk_id,
1262
+ path: payload.file_path,
1263
+ repoId: payload.repo_id,
1264
+ refId: payload.ref_id,
1265
+ reason: "excluded_by_configuration",
1266
+ detail: `candidate belongs to ref "${payload.ref_id}", which is not one of the effective refs [${sortByCodepoints([...effectiveRefIds]).join(", ")}]`,
1267
+ });
1268
+ return false;
1269
+ }
1270
+ if (!isPathSelected(payload.file_path, query.filters.pathGlobs, extensions.pathExcludes)) {
1271
+ ledger.addCandidate({
1272
+ subjectId: payload.chunk_id,
1273
+ path: payload.file_path,
1274
+ repoId: payload.repo_id,
1275
+ refId: payload.ref_id,
1276
+ reason: "excluded_by_configuration",
1277
+ detail: `path "${payload.file_path}" does not pass the include/exclude globs (include: [${query.filters.pathGlobs.join(", ")}], exclude: [${extensions.pathExcludes.join(", ")}])`,
1278
+ });
1279
+ return false;
1280
+ }
1281
+ const findabilityFilter = query.filters.findability;
1282
+ const lookup = this.deps.findability;
1283
+ if (findabilityFilter !== undefined && lookup !== undefined) {
1284
+ const summary = payload.symbol_id === undefined ? undefined : lookup(payload.symbol_id);
1285
+ if (summary === undefined) {
1286
+ ledger.addCandidate({
1287
+ subjectId: payload.chunk_id,
1288
+ path: payload.file_path,
1289
+ repoId: payload.repo_id,
1290
+ refId: payload.ref_id,
1291
+ reason: "missing_index_entry",
1292
+ detail: "a findability filter was requested but no findability record exists for this symbol",
1293
+ recoverable: true,
1294
+ });
1295
+ return false;
1296
+ }
1297
+ const { minScore, maxScore, causes } = findabilityFilter;
1298
+ const failsMin = minScore !== undefined && summary.score < minScore;
1299
+ const failsMax = maxScore !== undefined && summary.score > maxScore;
1300
+ const failsCause = causes !== undefined &&
1301
+ causes.length > 0 &&
1302
+ (summary.primaryCause === undefined || !causes.includes(summary.primaryCause));
1303
+ if (failsMin || failsMax || failsCause) {
1304
+ ledger.addCandidate({
1305
+ subjectId: payload.chunk_id,
1306
+ path: payload.file_path,
1307
+ repoId: payload.repo_id,
1308
+ refId: payload.ref_id,
1309
+ reason: "excluded_by_configuration",
1310
+ detail: `findability score ${summary.score} / cause "${summary.primaryCause ?? "none"}" does not satisfy the requested findability filter`,
1311
+ });
1312
+ return false;
1313
+ }
1314
+ }
1315
+ return true;
1316
+ }
1317
+ buildProximityMap(deduped) {
1318
+ const map = new Map();
1319
+ const lookup = this.deps.graphProximity;
1320
+ if (lookup === undefined) {
1321
+ return map;
1322
+ }
1323
+ for (const group of deduped) {
1324
+ const symbolId = group.primary.candidate.payload.symbol_id;
1325
+ if (symbolId === undefined || map.has(symbolId)) {
1326
+ continue;
1327
+ }
1328
+ const value = lookup(symbolId);
1329
+ if (value !== undefined) {
1330
+ map.set(symbolId, value);
1331
+ }
1332
+ }
1333
+ return map;
1334
+ }
1335
+ buildHit(candidate, score, dedupeKey, relatedFragments, target, extensions) {
1336
+ const { payload } = candidate;
1337
+ const findabilitySummary = payload.symbol_id === undefined ? undefined : this.deps.findability?.(payload.symbol_id);
1338
+ const contentRequested = extensions.includeContent;
1339
+ const hasContent = payload.content !== "";
1340
+ const complexity = this.complexityOf(candidate);
1341
+ return {
1342
+ schemaVersion: 1,
1343
+ // Der endgueltige Rang wird nach der Sortierung gesetzt (`assignRanks`);
1344
+ // bis dahin ist 0 ein bewusst ungueltiger Platzhalter, kein Rang 1.
1345
+ rank: 0,
1346
+ repoId: payload.repo_id,
1347
+ repoLabel: payload.repo_label,
1348
+ refId: payload.ref_id,
1349
+ refName: payload.ref_name,
1350
+ ...(payload.commit_sha === undefined ? {} : { commitSha: payload.commit_sha }),
1351
+ fileId: payload.file_id,
1352
+ filePath: payload.file_path,
1353
+ range: {
1354
+ startLine: payload.start_line,
1355
+ endLine: payload.end_line,
1356
+ startByte: payload.start_byte,
1357
+ endByte: payload.end_byte,
1358
+ },
1359
+ language: payload.language,
1360
+ parseQuality: payload.parse_quality,
1361
+ chunkId: payload.chunk_id,
1362
+ chunkKind: payload.chunk_kind,
1363
+ ...(payload.fragment_index === undefined ? {} : { fragmentIndex: payload.fragment_index }),
1364
+ ...(payload.fragment_count === undefined ? {} : { fragmentCount: payload.fragment_count }),
1365
+ ...(payload.symbol_id === undefined ? {} : { symbolId: payload.symbol_id }),
1366
+ ...(payload.symbol_name === undefined ? {} : { symbolName: payload.symbol_name }),
1367
+ ...(payload.symbol_fqn === undefined ? {} : { symbolFqn: payload.symbol_fqn }),
1368
+ ...(payload.symbol_kind === undefined ? {} : { symbolKind: payload.symbol_kind }),
1369
+ ...(payload.signature === undefined ? {} : { signature: payload.signature }),
1370
+ score,
1371
+ dedupeKey,
1372
+ relatedFragments,
1373
+ embeddingProfileId: target.embeddingProfileId,
1374
+ indexGeneration: this.indexGenerationOf(candidate),
1375
+ ...(complexity === undefined ? {} : { complexity }),
1376
+ ...(findabilitySummary === undefined ? {} : { findability: findabilitySummary }),
1377
+ ...(contentRequested && hasContent ? { content: payload.content } : {}),
1378
+ ...(contentRequested && !hasContent
1379
+ ? {
1380
+ contentOmittedReason: "the stored chunk payload carries no content (indexed without content, or content stripped at index time)",
1381
+ }
1382
+ : {}),
1383
+ ...(candidate.substitutedFields.length === 0
1384
+ ? {}
1385
+ : { incompletePayloadFields: [...candidate.substitutedFields] }),
1386
+ };
1387
+ }
1388
+ indexGenerationOf(candidate) {
1389
+ return this.rawIndexGenerationOf(candidate) ?? this.deps.indexGeneration ?? "unknown";
1390
+ }
1391
+ /** Persisted candidate provenance; display fallbacks are deliberately excluded. */
1392
+ rawIndexGenerationOf(candidate) {
1393
+ return readRawString(candidate.raw, "generation_id");
1394
+ }
1395
+ indexContextProvenance(target, scopeRepoIds, scopedRefIds, candidates, refScopeWasImplicit) {
1396
+ const contexts = (target.indexContexts ?? []).filter((context) => scopeRepoIds.includes(context.repoId) &&
1397
+ (scopedRefIds.length === 0 || scopedRefIds.includes(context.refId)));
1398
+ const completedContextExists = (repoId, refId, generationId) => contexts.some((context) => context.outcome === "completed" &&
1399
+ context.repoId === repoId &&
1400
+ (refId === undefined || context.refId === refId) &&
1401
+ (generationId === undefined || context.generationId === generationId));
1402
+ // An implicit scope represents "the recorded current ref(s) of every
1403
+ // selected repository", so every repository needs a completed context.
1404
+ // Explicit repo-qualified ref IDs instead form real repo/ref pairs. They
1405
+ // are not a Cartesian product: repo-a:main does not need to exist in
1406
+ // repo-b merely because both repos and both refs were selected.
1407
+ const hasCompleteScopeCoverage = contexts.length > 0 &&
1408
+ (refScopeWasImplicit
1409
+ ? scopeRepoIds.every((repoId) => completedContextExists(repoId))
1410
+ : scopedRefIds.every((refId) => contexts.some((context) => context.outcome === "completed" && context.refId === refId)));
1411
+ const candidatesHaveCompletedContext = candidates.every((candidate) => {
1412
+ const rawGenerationId = this.rawIndexGenerationOf(candidate);
1413
+ return (rawGenerationId !== undefined &&
1414
+ completedContextExists(candidate.payload.repo_id, candidate.payload.ref_id, rawGenerationId));
1415
+ });
1416
+ const incomplete = target.indexContexts !== undefined &&
1417
+ (!hasCompleteScopeCoverage ||
1418
+ !candidatesHaveCompletedContext ||
1419
+ contexts.some((context) => context.outcome !== "completed" || context.corpusRevisionFingerprint === undefined));
1420
+ if (!incomplete) {
1421
+ if (target.indexContexts === undefined)
1422
+ return { contexts };
1423
+ const corpusContexts = contexts
1424
+ .map((context) => ({
1425
+ repoId: context.repoId,
1426
+ refId: context.refId,
1427
+ generationId: context.generationId,
1428
+ corpusRevisionFingerprint: context.corpusRevisionFingerprint,
1429
+ }))
1430
+ .sort((left, right) => compareCodepoints(canonicalJson(left), canonicalJson(right)));
1431
+ return {
1432
+ contexts,
1433
+ corpusRevisionFingerprint: sha256Hex(canonicalJson({
1434
+ schemaVersion: 1,
1435
+ embeddingProfileId: target.embeddingProfileId,
1436
+ contexts: corpusContexts,
1437
+ })),
1438
+ };
1439
+ }
1440
+ return {
1441
+ contexts,
1442
+ warning: {
1443
+ code: INDEX_CONTEXT_INCOMPLETE_WARNING_CODE,
1444
+ message: `index completion could not be verified for embedding profile "${target.embeddingProfileId}"; ` +
1445
+ "hits remain usable, but this search is not healthy product-complexity evidence",
1446
+ details: {
1447
+ embeddingProfileId: target.embeddingProfileId,
1448
+ observedOutcomes: uniqueSortedByCodepoints(contexts.map((context) => context.outcome)),
1449
+ },
1450
+ },
1451
+ };
1452
+ }
1453
+ complexityOf(candidate) {
1454
+ const { payload } = candidate;
1455
+ const values = {
1456
+ ...(payload.cyclomatic_classic === undefined
1457
+ ? {}
1458
+ : { cyclomaticClassic: payload.cyclomatic_classic }),
1459
+ ...(payload.cyclomatic_extended === undefined
1460
+ ? {}
1461
+ : { cyclomaticExtended: payload.cyclomatic_extended }),
1462
+ ...(payload.cognitive === undefined ? {} : { cognitive: payload.cognitive }),
1463
+ ...(payload.max_nesting === undefined ? {} : { maxNesting: payload.max_nesting }),
1464
+ ...(payload.sloc === undefined ? {} : { sloc: payload.sloc }),
1465
+ ...(payload.halstead_volume === undefined ? {} : { halsteadVolume: payload.halstead_volume }),
1466
+ ...(payload.maintainability_index === undefined
1467
+ ? {}
1468
+ : { maintainabilityIndex: payload.maintainability_index }),
1469
+ ...(payload.risk_score === undefined ? {} : { riskScore: payload.risk_score }),
1470
+ };
1471
+ if (Object.keys(values).length === 0) {
1472
+ return undefined;
1473
+ }
1474
+ return {
1475
+ metricsScope: payload.metrics_scope,
1476
+ metricsScopeId: payload.metrics_scope_id,
1477
+ metricsVersion: payload.metrics_version,
1478
+ ...values,
1479
+ };
1480
+ }
1481
+ }
1482
+ /** Kleiner Bequemlichkeits-Konstruktor. */
1483
+ export function createHybridSearchEngine(deps) {
1484
+ return new HybridSearchEngine(deps);
1485
+ }
1486
+ function markFinalHitGraphUnavailable(groups, query, detail) {
1487
+ for (const group of groups) {
1488
+ for (const hit of group.hits) {
1489
+ if (query.includeGraphContext) {
1490
+ hit.graphContext = unavailableSearchGraphRelations("graph_unavailable", detail);
1491
+ }
1492
+ if (query.includeLinked) {
1493
+ hit.linkedSymbols = unavailableSearchGraphRelations("graph_unavailable", detail);
1494
+ }
1495
+ }
1496
+ }
1497
+ }
1498
+ //# sourceMappingURL=engine.js.map