@untangle-code/search 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +661 -0
- package/NOTICE +90 -0
- package/README.md +433 -2
- package/THIRD_PARTY_LICENSES.md +29 -0
- package/dist/benchmark-integrity.d.ts +12 -0
- package/dist/benchmark-integrity.d.ts.map +1 -0
- package/dist/benchmark-integrity.js +10 -0
- package/dist/benchmark-integrity.js.map +1 -0
- package/dist/bm25.d.ts +46 -0
- package/dist/bm25.d.ts.map +1 -0
- package/dist/bm25.js +105 -0
- package/dist/bm25.js.map +1 -0
- package/dist/deadline.d.ts +4 -0
- package/dist/deadline.d.ts.map +1 -0
- package/dist/deadline.js +54 -0
- package/dist/deadline.js.map +1 -0
- package/dist/dedupe.d.ts +48 -0
- package/dist/dedupe.d.ts.map +1 -0
- package/dist/dedupe.js +58 -0
- package/dist/dedupe.js.map +1 -0
- package/dist/diversify.d.ts +45 -0
- package/dist/diversify.d.ts.map +1 -0
- package/dist/diversify.js +30 -0
- package/dist/diversify.js.map +1 -0
- package/dist/engine.d.ts +157 -0
- package/dist/engine.d.ts.map +1 -0
- package/dist/engine.js +1498 -0
- package/dist/engine.js.map +1 -0
- package/dist/enrichment.d.ts +16 -0
- package/dist/enrichment.d.ts.map +1 -0
- package/dist/enrichment.js +92 -0
- package/dist/enrichment.js.map +1 -0
- package/dist/filter.d.ts +86 -0
- package/dist/filter.d.ts.map +1 -0
- package/dist/filter.js +166 -0
- package/dist/filter.js.map +1 -0
- package/dist/findability.d.ts +144 -0
- package/dist/findability.d.ts.map +1 -0
- package/dist/findability.js +393 -0
- package/dist/findability.js.map +1 -0
- package/dist/format.d.ts +28 -0
- package/dist/format.d.ts.map +1 -0
- package/dist/format.js +172 -0
- package/dist/format.js.map +1 -0
- package/dist/fusion.d.ts +47 -0
- package/dist/fusion.d.ts.map +1 -0
- package/dist/fusion.js +84 -0
- package/dist/fusion.js.map +1 -0
- package/dist/glob.d.ts +50 -0
- package/dist/glob.d.ts.map +1 -0
- package/dist/glob.js +144 -0
- package/dist/glob.js.map +1 -0
- package/dist/grep.d.ts +76 -0
- package/dist/grep.d.ts.map +1 -0
- package/dist/grep.js +475 -0
- package/dist/grep.js.map +1 -0
- package/dist/index.d.ts +42 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +63 -0
- package/dist/index.js.map +1 -0
- package/dist/ledger.d.ts +45 -0
- package/dist/ledger.d.ts.map +1 -0
- package/dist/ledger.js +39 -0
- package/dist/ledger.js.map +1 -0
- package/dist/payload.d.ts +108 -0
- package/dist/payload.d.ts.map +1 -0
- package/dist/payload.js +263 -0
- package/dist/payload.js.map +1 -0
- package/dist/port.d.ts +112 -0
- package/dist/port.d.ts.map +1 -0
- package/dist/port.js +2 -0
- package/dist/port.js.map +1 -0
- package/dist/product-complexity.d.ts +89 -0
- package/dist/product-complexity.d.ts.map +1 -0
- package/dist/product-complexity.js +1367 -0
- package/dist/product-complexity.js.map +1 -0
- package/dist/regex-safety.d.ts +75 -0
- package/dist/regex-safety.d.ts.map +1 -0
- package/dist/regex-safety.js +902 -0
- package/dist/regex-safety.js.map +1 -0
- package/dist/relevance-prerequisites.d.ts +21 -0
- package/dist/relevance-prerequisites.d.ts.map +1 -0
- package/dist/relevance-prerequisites.js +47 -0
- package/dist/relevance-prerequisites.js.map +1 -0
- package/dist/relevance.d.ts +107 -0
- package/dist/relevance.d.ts.map +1 -0
- package/dist/relevance.js +199 -0
- package/dist/relevance.js.map +1 -0
- package/dist/retrieval.d.ts +46 -0
- package/dist/retrieval.d.ts.map +1 -0
- package/dist/retrieval.js +182 -0
- package/dist/retrieval.js.map +1 -0
- package/dist/scoring.d.ts +102 -0
- package/dist/scoring.d.ts.map +1 -0
- package/dist/scoring.js +175 -0
- package/dist/scoring.js.map +1 -0
- package/dist/sort.d.ts +21 -0
- package/dist/sort.d.ts.map +1 -0
- package/dist/sort.js +53 -0
- package/dist/sort.js.map +1 -0
- package/dist/symbol-match.d.ts +21 -0
- package/dist/symbol-match.d.ts.map +1 -0
- package/dist/symbol-match.js +72 -0
- package/dist/symbol-match.js.map +1 -0
- package/dist/tokenize.d.ts +41 -0
- package/dist/tokenize.d.ts.map +1 -0
- package/dist/tokenize.js +202 -0
- package/dist/tokenize.js.map +1 -0
- package/dist/types.d.ts +173 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +42 -0
- package/dist/types.js.map +1 -0
- package/package.json +61 -3
- package/src/benchmark-integrity.ts +25 -0
- package/src/bm25.ts +135 -0
- package/src/deadline.ts +59 -0
- package/src/dedupe.ts +99 -0
- package/src/diversify.ts +71 -0
- package/src/engine.ts +1957 -0
- package/src/enrichment.ts +115 -0
- package/src/filter.ts +267 -0
- package/src/findability.ts +580 -0
- package/src/format.ts +228 -0
- package/src/fusion.ts +121 -0
- package/src/glob.ts +151 -0
- package/src/grep.ts +618 -0
- package/src/index.ts +83 -0
- package/src/ledger.ts +72 -0
- package/src/payload.ts +382 -0
- package/src/port.ts +133 -0
- package/src/product-complexity.ts +2168 -0
- package/src/regex-safety.ts +1002 -0
- package/src/relevance-prerequisites.ts +83 -0
- package/src/relevance.ts +328 -0
- package/src/retrieval.ts +299 -0
- package/src/scoring.ts +321 -0
- package/src/sort.ts +86 -0
- package/src/symbol-match.ts +87 -0
- package/src/tokenize.ts +213 -0
- package/src/types.ts +206 -0
package/src/engine.ts
ADDED
|
@@ -0,0 +1,1957 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Der hybride Abrufkern (spec.md §13.2; architecture.md §18; vision.md §1d).
|
|
3
|
+
*
|
|
4
|
+
* ABRUF UND RANGFOLGE SIND ZWEI VERSCHIEDENE DINGE — das ist die zentrale
|
|
5
|
+
* Bauentscheidung dieses Moduls und der Grund, warum sich der Suchmodus so
|
|
6
|
+
* einfach einschraenken laesst:
|
|
7
|
+
*
|
|
8
|
+
* - ABRUF (Recall) holt Kandidaten aus dem Speicher. Quellen: der dichte
|
|
9
|
+
* Zweig (`hybridQuery` mit Query-Embedding), der spaerliche Zweig
|
|
10
|
+
* (`hybridQuery` mit spaerlichem Vektor, sofern ein `SparseVectorizer`
|
|
11
|
+
* gestellt wurde) und der Symbolzweig (`scroll` mit Filter auf
|
|
12
|
+
* `symbol_name`).
|
|
13
|
+
* - RANGFOLGE entsteht danach IN DIESEM PROZESS: der dichte Rang aus den
|
|
14
|
+
* Cosine-Scores des Speichers, der lexikalische Rang aus dem eigenen BM25
|
|
15
|
+
* ueber den Kandidatenpool, der Symbolrang aus dem Namensvergleich. Diese
|
|
16
|
+
* drei Ranglisten werden per RRF fusioniert (`fusion.ts`).
|
|
17
|
+
*
|
|
18
|
+
* Der Suchmodus waehlt Abruf UND Rangfolge: `semantic` fragt nur den
|
|
19
|
+
* dichten Zweig, `lexical` nur den spaerlichen, `symbol` nur den
|
|
20
|
+
* Symbolspeicher. Im Hybridmodus laufen die drei Zweige unabhaengig;
|
|
21
|
+
* ausgefallene Zweige duerfen nutzbare Antworten anderer Zweige nicht
|
|
22
|
+
* verhindern. Jede solche Teilsuche bleibt als Degradation sichtbar.
|
|
23
|
+
*
|
|
24
|
+
* E4 durchgaengig: jeder verworfene Kandidat — Nachfilter, `minScore`,
|
|
25
|
+
* Dedupe, `limit`, unbrauchbarer Payload, ausgefallenes Ziel — bekommt einen
|
|
26
|
+
* Eintrag im Skip-Ledger des Ergebnisses. Kein `catch`, das schweigt.
|
|
27
|
+
*/
|
|
28
|
+
import type { SearchConfig } from "@untangle-code/config";
|
|
29
|
+
import { DEFAULT_SEARCH_CONFIG } from "@untangle-code/config";
|
|
30
|
+
import type {
|
|
31
|
+
Clock,
|
|
32
|
+
LoggerPort,
|
|
33
|
+
PayloadFilter,
|
|
34
|
+
PortCallOptions,
|
|
35
|
+
ScoredPointV1,
|
|
36
|
+
SearchFiltersV1,
|
|
37
|
+
SearchHitGroupV1,
|
|
38
|
+
SearchRelevanceVerdictV1,
|
|
39
|
+
SearchHitV1,
|
|
40
|
+
SearchQueryDefaults,
|
|
41
|
+
SearchQueryV1,
|
|
42
|
+
SearchResultSetV1,
|
|
43
|
+
ToolWarningV1,
|
|
44
|
+
VectorStore,
|
|
45
|
+
} from "@untangle-code/domain";
|
|
46
|
+
import {
|
|
47
|
+
buildSearchQueryFingerprint,
|
|
48
|
+
canonicalJson,
|
|
49
|
+
compareCodepoints,
|
|
50
|
+
INDEX_CONTEXT_INCOMPLETE_WARNING_CODE,
|
|
51
|
+
isUntangleError,
|
|
52
|
+
normalizeSearchQuery,
|
|
53
|
+
sha256Hex,
|
|
54
|
+
sortByCodepoints,
|
|
55
|
+
UntangleError,
|
|
56
|
+
uniqueSortedByCodepoints,
|
|
57
|
+
} from "@untangle-code/domain";
|
|
58
|
+
|
|
59
|
+
import { Bm25Ranker } from "./bm25.js";
|
|
60
|
+
import { dedupeCandidates } from "./dedupe.js";
|
|
61
|
+
import type { ScoredCandidate } from "./dedupe.js";
|
|
62
|
+
import { assignDuplicatePenalties } from "./diversify.js";
|
|
63
|
+
import {
|
|
64
|
+
SEARCH_GRAPH_RELATION_LIMIT,
|
|
65
|
+
boundSearchGraphDetail,
|
|
66
|
+
createSearchGraphRelations,
|
|
67
|
+
unavailableSearchGraphRelations,
|
|
68
|
+
} from "./enrichment.js";
|
|
69
|
+
import type { SearchScopeV1 } from "./filter.js";
|
|
70
|
+
import { buildPayloadFilter } from "./filter.js";
|
|
71
|
+
import type { BranchWeights, FusionInput, FusionResult } from "./fusion.js";
|
|
72
|
+
import { DEFAULT_BRANCH_WEIGHTS, fuseRankings } from "./fusion.js";
|
|
73
|
+
import { assertValidGlobPatterns, isPathSelected } from "./glob.js";
|
|
74
|
+
import { grepIndexedChunks } from "./grep.js";
|
|
75
|
+
import { SearchSkipLedger } from "./ledger.js";
|
|
76
|
+
import type { SearchCandidate } from "./payload.js";
|
|
77
|
+
import { candidateFromPoint, lexicalDocumentText, readRawString } from "./payload.js";
|
|
78
|
+
import type {
|
|
79
|
+
FindabilityLookup,
|
|
80
|
+
GraphProximityLookup,
|
|
81
|
+
ProfileTargetV1,
|
|
82
|
+
SearchGraphEnrichmentPort,
|
|
83
|
+
SearchPort,
|
|
84
|
+
SparseVectorizer,
|
|
85
|
+
} from "./port.js";
|
|
86
|
+
import { relevancePrerequisiteFailure } from "./relevance-prerequisites.js";
|
|
87
|
+
import type { RelevancePolicyBindingV1 } from "./relevance.js";
|
|
88
|
+
import {
|
|
89
|
+
NO_SUFFICIENT_MATCH_WARNING_CODE,
|
|
90
|
+
RELEVANCE_POLICY_NOT_EVALUATED_WARNING_CODE,
|
|
91
|
+
decideResultRelevance,
|
|
92
|
+
resolveRelevancePolicyBinding,
|
|
93
|
+
selectFixedRelevanceEvidencePool,
|
|
94
|
+
} from "./relevance.js";
|
|
95
|
+
import {
|
|
96
|
+
assertCompletedRetrieval,
|
|
97
|
+
branchesOfMode,
|
|
98
|
+
retrieveIndependentBranches,
|
|
99
|
+
rethrowCancellation,
|
|
100
|
+
} from "./retrieval.js";
|
|
101
|
+
import type { ScoringContext } from "./scoring.js";
|
|
102
|
+
import { MAX_BOOST_SHARE, assembleScoreBreakdown } from "./scoring.js";
|
|
103
|
+
import { assignRanks, sortHits } from "./sort.js";
|
|
104
|
+
import { matchSymbolName } from "./symbol-match.js";
|
|
105
|
+
import { isLikelyGermanQuestion } from "./tokenize.js";
|
|
106
|
+
import type {
|
|
107
|
+
GrepRequestV1,
|
|
108
|
+
GrepResultSetV1,
|
|
109
|
+
SearchQueryExtensionsV1,
|
|
110
|
+
UntangleSearchRequestV1,
|
|
111
|
+
} from "./types.js";
|
|
112
|
+
|
|
113
|
+
/** Halbwertszeit des Recency-Boosts, wenn keine gesetzt ist. */
|
|
114
|
+
export const DEFAULT_RECENCY_HALF_LIFE_DAYS = 30;
|
|
115
|
+
/** Harte Grenze fuer freie Providerwarnungen und Batch-Diagnose-IDs. */
|
|
116
|
+
export const SEARCH_GRAPH_DIAGNOSTIC_LIMIT = 32;
|
|
117
|
+
/** Providerwarnungen sind eine eigene Ausgabeachse und deshalb separat gedeckelt. */
|
|
118
|
+
export const SEARCH_GRAPH_PROVIDER_WARNING_LIMIT = 16;
|
|
119
|
+
|
|
120
|
+
function normalizeGraphProviderWarning(warning: ToolWarningV1): ToolWarningV1 {
|
|
121
|
+
const source = warning.details;
|
|
122
|
+
const details: Record<string, unknown> = {};
|
|
123
|
+
if (source !== undefined && source !== null && typeof source === "object") {
|
|
124
|
+
if (typeof source["workspaceId"] === "string") {
|
|
125
|
+
details["workspaceId"] = boundSearchGraphDetail(source["workspaceId"]);
|
|
126
|
+
}
|
|
127
|
+
if (typeof source["generationId"] === "string") {
|
|
128
|
+
details["generationId"] = boundSearchGraphDetail(source["generationId"]);
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
const rawIssues = source["issues"];
|
|
132
|
+
if (Array.isArray(rawIssues)) {
|
|
133
|
+
const issues = rawIssues.map(String).map(boundSearchGraphDetail).sort(compareCodepoints);
|
|
134
|
+
details["issues"] = issues.slice(0, SEARCH_GRAPH_PROVIDER_WARNING_LIMIT);
|
|
135
|
+
const providerOmitted = source["omittedIssueCount"];
|
|
136
|
+
const providerOmittedCount =
|
|
137
|
+
typeof providerOmitted === "number" &&
|
|
138
|
+
Number.isSafeInteger(providerOmitted) &&
|
|
139
|
+
providerOmitted >= 0
|
|
140
|
+
? providerOmitted
|
|
141
|
+
: 0;
|
|
142
|
+
details["omittedIssueCount"] =
|
|
143
|
+
providerOmittedCount + Math.max(0, issues.length - SEARCH_GRAPH_PROVIDER_WARNING_LIMIT);
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
const retainedKeys = new Set(["workspaceId", "generationId", "issues", "omittedIssueCount"]);
|
|
147
|
+
const omittedDetailFieldCount = Object.keys(source).filter(
|
|
148
|
+
(key) => !retainedKeys.has(key),
|
|
149
|
+
).length;
|
|
150
|
+
if (omittedDetailFieldCount > 0) {
|
|
151
|
+
details["omittedDetailFieldCount"] = omittedDetailFieldCount;
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
return {
|
|
155
|
+
code: boundSearchGraphDetail(String(warning.code)),
|
|
156
|
+
message: boundSearchGraphDetail(String(warning.message)),
|
|
157
|
+
...(Object.keys(details).length === 0 ? {} : { details }),
|
|
158
|
+
};
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
function boundedDiagnosticIds(values: readonly string[]): {
|
|
162
|
+
ids: string[];
|
|
163
|
+
omittedCount: number;
|
|
164
|
+
} {
|
|
165
|
+
const sorted = sortByCodepoints([...new Set(values)]);
|
|
166
|
+
return {
|
|
167
|
+
ids: sorted.slice(0, SEARCH_GRAPH_DIAGNOSTIC_LIMIT).map(boundSearchGraphDetail),
|
|
168
|
+
omittedCount: Math.max(0, sorted.length - SEARCH_GRAPH_DIAGNOSTIC_LIMIT),
|
|
169
|
+
};
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
export const DEFAULT_SEARCH_BRANCH_TIMEOUT_MS = 10_000;
|
|
173
|
+
|
|
174
|
+
/** Alles, was der Abrufkern braucht — reine Ports und Daten, keine Adapter. */
|
|
175
|
+
export interface HybridSearchEngineDeps {
|
|
176
|
+
vectorStore: VectorStore;
|
|
177
|
+
/** Ein Ziel je Embedding-Profil (spec.md §13.4). */
|
|
178
|
+
targets: ProfileTargetV1[];
|
|
179
|
+
/** Vorgaben aus der Konfiguration fuer `normalizeSearchQuery`. */
|
|
180
|
+
defaults: SearchQueryDefaults;
|
|
181
|
+
searchConfig?: SearchConfig;
|
|
182
|
+
/** Nur fuer den Recency-Boost noetig; ohne Clock entfaellt der Bestandteil. */
|
|
183
|
+
clock?: Clock;
|
|
184
|
+
logger?: LoggerPort;
|
|
185
|
+
/** Erzeugt spaerliche Anfragevektoren; siehe {@link SparseVectorizer}. */
|
|
186
|
+
sparseVectorizer?: SparseVectorizer;
|
|
187
|
+
/** Repo-Prioritaeten in [0, 1] fuer den `repo_priority_boost`. */
|
|
188
|
+
repoPriorities?: Record<string, number>;
|
|
189
|
+
graphProximity?: GraphProximityLookup;
|
|
190
|
+
/** Post-Ranking-Anreicherung; kennt als Port keine konkreten Graphpakete. */
|
|
191
|
+
graphEnrichment?: SearchGraphEnrichmentPort;
|
|
192
|
+
findability?: FindabilityLookup;
|
|
193
|
+
recencyHalfLifeDays?: number;
|
|
194
|
+
branchWeights?: BranchWeights;
|
|
195
|
+
/** Hybrid-only deadline per independent branch; single-mode uses caller cancellation. */
|
|
196
|
+
branchTimeoutMs?: number;
|
|
197
|
+
/**
|
|
198
|
+
* Profilgebundene Abwesenheitsentscheidungen. Eine leere Map behaelt das
|
|
199
|
+
* bisherige ungefilterte Verhalten. Ist die Map konfiguriert, aber fuer ein
|
|
200
|
+
* aktives Profil fehlt der Eintrag, bleibt die Trefferliste ebenfalls
|
|
201
|
+
* unveraendert und die Engine meldet die nicht ausgewertete Policy sichtbar.
|
|
202
|
+
*/
|
|
203
|
+
relevancePolicies?: Readonly<Record<string, RelevancePolicyBindingV1>>;
|
|
204
|
+
/** Display-only legacy fallback; never accepted as persisted provenance. */
|
|
205
|
+
indexGeneration?: string;
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
/**
|
|
209
|
+
* Gibt dem PRAEZISIONSzweig bei einer zeichengleichen Symbolanfrage genug
|
|
210
|
+
* Basisgewicht, dass ein exakter Treffer selbst unter den erlaubten
|
|
211
|
+
* Boost-Grenzfaellen vor einem nicht-exakten Treffer bleibt.
|
|
212
|
+
*
|
|
213
|
+
* Bei MAX_BOOST_SHARE=0.5 kann ein nicht-exakter Basisscore hoechstens auf
|
|
214
|
+
* 1.5x steigen und ein exakter durch Strafen hoechstens auf 0.5x fallen. Das
|
|
215
|
+
* benoetigte Basisverhaeltnis ist daher (1+s)/(1-s)=3. Der kleine vorhandene
|
|
216
|
+
* Symbolanteil oberhalb dieser Grenze macht die Ungleichung strikt. Der
|
|
217
|
+
* Boost-Deckel selbst bleibt vollkommen unveraendert.
|
|
218
|
+
*/
|
|
219
|
+
export function exactSymbolIntentBranchWeights(
|
|
220
|
+
weights: BranchWeights,
|
|
221
|
+
active: { dense: boolean; lexical: boolean; symbol: boolean },
|
|
222
|
+
): BranchWeights {
|
|
223
|
+
if (!active.symbol || weights.symbol <= 0) {
|
|
224
|
+
return { ...weights };
|
|
225
|
+
}
|
|
226
|
+
const nonSymbolWeight =
|
|
227
|
+
(active.dense ? Math.max(0, weights.dense) : 0) +
|
|
228
|
+
(active.lexical ? Math.max(0, weights.lexical) : 0);
|
|
229
|
+
const dominanceRatio = (1 + MAX_BOOST_SHARE) / (1 - MAX_BOOST_SHARE);
|
|
230
|
+
return {
|
|
231
|
+
...weights,
|
|
232
|
+
symbol: Math.max(weights.symbol, nonSymbolWeight * dominanceRatio + weights.symbol),
|
|
233
|
+
};
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
/** Zahl dichter Spitzenplaetze, die einen lexikalischen Spitzenkandidaten bestaetigen. */
|
|
237
|
+
export const GERMAN_LEXICAL_CORROBORATION_DEPTH = 10;
|
|
238
|
+
|
|
239
|
+
/**
|
|
240
|
+
* Nur eine als deutsch erkannte Frage mit einem lexikalischen Spitzenreiter,
|
|
241
|
+
* der NICHT unter den zehn dicht besten Kandidaten vorkommt, verliert im
|
|
242
|
+
* Hybridmodus den lokalen BM25-Rang. Stimmen beide unabhaengigen Zweige
|
|
243
|
+
* ueberein, bleibt das nuetzliche lexikalische Signal erhalten.
|
|
244
|
+
*/
|
|
245
|
+
export function shouldOmitUncorroboratedGermanLexicalBranch(
|
|
246
|
+
queryText: string,
|
|
247
|
+
denseRankedIds: readonly string[],
|
|
248
|
+
lexicalRankedIds: readonly string[],
|
|
249
|
+
): boolean {
|
|
250
|
+
if (!isLikelyGermanQuestion(queryText)) {
|
|
251
|
+
return false;
|
|
252
|
+
}
|
|
253
|
+
const lexicalTop = lexicalRankedIds[0];
|
|
254
|
+
if (lexicalTop === undefined) {
|
|
255
|
+
return true;
|
|
256
|
+
}
|
|
257
|
+
return !denseRankedIds.slice(0, GERMAN_LEXICAL_CORROBORATION_DEPTH).includes(lexicalTop);
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
/** Normalisiert die Zusatzfelder dieses Pakets (Gegenstueck zu `SearchQueryV1`). */
|
|
261
|
+
export function normalizeExtensions(request: UntangleSearchRequestV1): SearchQueryExtensionsV1 {
|
|
262
|
+
const pathExcludes = uniqueSortedByCodepoints(
|
|
263
|
+
(request.pathExclude ?? []).map((value) => value.trim()).filter((value) => value !== ""),
|
|
264
|
+
);
|
|
265
|
+
return {
|
|
266
|
+
mode: request.mode ?? "hybrid",
|
|
267
|
+
abstention: request.abstention ?? "strict",
|
|
268
|
+
pathExcludes,
|
|
269
|
+
includeContent: request.includeContent ?? false,
|
|
270
|
+
complexityBias: request.complexityBias ?? "none",
|
|
271
|
+
sort: request.sort ?? "relevance",
|
|
272
|
+
};
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
/**
|
|
276
|
+
* Die Zusatzangaben, die den Fingerabdruck der Anfrage veraendern MUESSEN.
|
|
277
|
+
*
|
|
278
|
+
* Der Fingerabdruck beantwortet „ist das dieselbe Frage?". `mode`,
|
|
279
|
+
* `pathExcludes`, `complexityBias` und `sort` aendern die Treffermenge bzw.
|
|
280
|
+
* die Rangfolge — zwei Anfragen, die sich nur darin unterscheiden, sind NICHT
|
|
281
|
+
* dieselbe Frage. `includeContent` bleibt draussen: es aendert nur die
|
|
282
|
+
* Darstellung, genau wie `limit` und `candidateLimit` (siehe
|
|
283
|
+
* `buildSearchQueryFingerprint`).
|
|
284
|
+
*
|
|
285
|
+
* Nur ABWEICHUNGEN vom Standard gehen ein. Eine Anfrage ohne
|
|
286
|
+
* Paket-Erweiterungen behaelt damit exakt den Fingerabdruck, den sie ohne
|
|
287
|
+
* dieses Feld haette — Findability-Datensaetze aelterer Laeufe bleiben
|
|
288
|
+
* vergleichbar.
|
|
289
|
+
*/
|
|
290
|
+
export function fingerprintRankingOptions(
|
|
291
|
+
extensions: SearchQueryExtensionsV1,
|
|
292
|
+
): Record<string, unknown> | undefined {
|
|
293
|
+
const options: Record<string, unknown> = {};
|
|
294
|
+
if (extensions.mode !== "hybrid") {
|
|
295
|
+
options["mode"] = extensions.mode;
|
|
296
|
+
}
|
|
297
|
+
if (extensions.pathExcludes.length > 0) {
|
|
298
|
+
options["pathExcludes"] = extensions.pathExcludes;
|
|
299
|
+
}
|
|
300
|
+
if (extensions.complexityBias !== "none") {
|
|
301
|
+
options["complexityBias"] = extensions.complexityBias;
|
|
302
|
+
}
|
|
303
|
+
if (extensions.sort !== "relevance") {
|
|
304
|
+
options["sort"] = extensions.sort;
|
|
305
|
+
}
|
|
306
|
+
// NUR bei Abweichung. `strict` traegt nichts bei, damit jeder bereits
|
|
307
|
+
// persistierte `queryFingerprint` byte-identisch bleibt — sonst waeren alle
|
|
308
|
+
// Findability-Datensaetze und Journey-Kohorten auf einen Schlag entwertet.
|
|
309
|
+
if (extensions.abstention !== "strict") {
|
|
310
|
+
options["abstention"] = extensions.abstention;
|
|
311
|
+
}
|
|
312
|
+
return Object.keys(options).length === 0 ? undefined : options;
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
/**
|
|
316
|
+
* Ergaenzt die Findability-Filter aus der Paketanfrage in die normalisierten
|
|
317
|
+
* Domain-Filter. `normalizeSearchQuery` kennt sie nicht, weil `SearchRequestV1`
|
|
318
|
+
* (spec.md §13.1) sie nicht kennt — `SearchFiltersV1` hat das Feld aber
|
|
319
|
+
* bereits, also wird es hier gefuellt statt neben der Query gefuehrt.
|
|
320
|
+
*/
|
|
321
|
+
function withFindabilityFilter(
|
|
322
|
+
query: SearchQueryV1,
|
|
323
|
+
request: UntangleSearchRequestV1,
|
|
324
|
+
): SearchQueryV1 {
|
|
325
|
+
const { findabilityMinScore, findabilityMaxScore } = request;
|
|
326
|
+
if (findabilityMinScore === undefined && findabilityMaxScore === undefined) {
|
|
327
|
+
return query;
|
|
328
|
+
}
|
|
329
|
+
if (
|
|
330
|
+
findabilityMinScore !== undefined &&
|
|
331
|
+
findabilityMaxScore !== undefined &&
|
|
332
|
+
findabilityMinScore > findabilityMaxScore
|
|
333
|
+
) {
|
|
334
|
+
throw new Error(
|
|
335
|
+
`search query filter "findability" has min > max (${findabilityMinScore} > ${findabilityMaxScore})`,
|
|
336
|
+
);
|
|
337
|
+
}
|
|
338
|
+
const filters: SearchFiltersV1 = {
|
|
339
|
+
...query.filters,
|
|
340
|
+
findability: {
|
|
341
|
+
...(findabilityMinScore === undefined ? {} : { minScore: findabilityMinScore }),
|
|
342
|
+
...(findabilityMaxScore === undefined ? {} : { maxScore: findabilityMaxScore }),
|
|
343
|
+
},
|
|
344
|
+
};
|
|
345
|
+
return { ...query, filters };
|
|
346
|
+
}
|
|
347
|
+
|
|
348
|
+
/**
|
|
349
|
+
* WP-188 (E4): `searchConfig.diversifyByDefault` und `searchConfig.minScoreDefault`
|
|
350
|
+
* hatten KEINEN Leser.
|
|
351
|
+
*
|
|
352
|
+
* Beide standen in der Konfiguration, waren dokumentiert, wurden validiert —
|
|
353
|
+
* und veraenderten nichts. Eine Einstellung, die nichts tut, ist schlimmer als
|
|
354
|
+
* eine fehlende: sie behauptet eine Wirkung. Wer `minScoreDefault = 0.5`
|
|
355
|
+
* setzte und weiterhin Treffer mit 0.2 bekam, hatte keinen Grund, den Fehler
|
|
356
|
+
* beim Werkzeug zu suchen.
|
|
357
|
+
*
|
|
358
|
+
* Die Vorgaben greifen nur, wo die ANFRAGE nichts sagt — eine ausdrueckliche
|
|
359
|
+
* Angabe des Aufrufers gewinnt immer. Sie werden VOR dem Fingerabdruck
|
|
360
|
+
* angewandt, damit „gleiche Frage" auch dann gleich heisst, wenn ein Teil der
|
|
361
|
+
* Antwort aus der Konfiguration kommt.
|
|
362
|
+
*/
|
|
363
|
+
function applyConfigDefaults(
|
|
364
|
+
query: SearchQueryV1,
|
|
365
|
+
request: UntangleSearchRequestV1,
|
|
366
|
+
config: SearchConfig,
|
|
367
|
+
): { query: SearchQueryV1; minScoreFromConfig: boolean } {
|
|
368
|
+
const diversify = request.diversify ?? config.diversifyByDefault;
|
|
369
|
+
const useConfigMinScore =
|
|
370
|
+
request.minScore === undefined &&
|
|
371
|
+
config.minScoreDefault !== undefined &&
|
|
372
|
+
Number.isFinite(config.minScoreDefault);
|
|
373
|
+
return {
|
|
374
|
+
query: {
|
|
375
|
+
...query,
|
|
376
|
+
diversify,
|
|
377
|
+
...(useConfigMinScore ? { minScore: config.minScoreDefault as number } : {}),
|
|
378
|
+
},
|
|
379
|
+
minScoreFromConfig: useConfigMinScore,
|
|
380
|
+
};
|
|
381
|
+
}
|
|
382
|
+
|
|
383
|
+
/** Woher der wirksame `minScore` stammt — Wortlaut fuer den Skip-Ledger. */
|
|
384
|
+
type MinScoreOrigin = "requested" | "from searchConfig.minScoreDefault";
|
|
385
|
+
|
|
386
|
+
interface TargetOutcome {
|
|
387
|
+
group: SearchHitGroupV1;
|
|
388
|
+
refIds: Set<string>;
|
|
389
|
+
relevanceWarning?: ToolWarningV1;
|
|
390
|
+
indexContextWarning?: ToolWarningV1;
|
|
391
|
+
branchWarnings?: ToolWarningV1[];
|
|
392
|
+
}
|
|
393
|
+
|
|
394
|
+
/**
|
|
395
|
+
* Die Referenzimplementierung des {@link SearchPort}.
|
|
396
|
+
*
|
|
397
|
+
* Sie ist speicher-agnostisch: alles laeuft ueber den `VectorStore`-Port, also
|
|
398
|
+
* gegen Qdrant genauso wie gegen den In-Memory-Speicher des testkits.
|
|
399
|
+
*/
|
|
400
|
+
export class HybridSearchEngine implements SearchPort {
|
|
401
|
+
private readonly config: SearchConfig;
|
|
402
|
+
private readonly branchWeights: BranchWeights;
|
|
403
|
+
private readonly branchTimeoutMs: number;
|
|
404
|
+
private readonly relevancePolicies: ReadonlyMap<string, RelevancePolicyBindingV1>;
|
|
405
|
+
|
|
406
|
+
public constructor(private readonly deps: HybridSearchEngineDeps) {
|
|
407
|
+
const seenProfileIds = new Set<string>();
|
|
408
|
+
for (const target of deps.targets) {
|
|
409
|
+
if (target.embeddingProfileId !== target.embedder.profileId) {
|
|
410
|
+
throw new Error(
|
|
411
|
+
`search target embeddingProfileId "${target.embeddingProfileId}" does not match its embedder.profileId "${target.embedder.profileId}"; collection/profile grouping and calibrated policies would be misattributed`,
|
|
412
|
+
);
|
|
413
|
+
}
|
|
414
|
+
if (seenProfileIds.has(target.embeddingProfileId)) {
|
|
415
|
+
throw new Error(
|
|
416
|
+
`search target embeddingProfileId "${target.embeddingProfileId}" is configured more than once; one result group per embedding profile is required`,
|
|
417
|
+
);
|
|
418
|
+
}
|
|
419
|
+
seenProfileIds.add(target.embeddingProfileId);
|
|
420
|
+
}
|
|
421
|
+
this.config = deps.searchConfig ?? DEFAULT_SEARCH_CONFIG;
|
|
422
|
+
this.branchWeights = deps.branchWeights ?? { ...DEFAULT_BRANCH_WEIGHTS };
|
|
423
|
+
this.branchTimeoutMs = deps.branchTimeoutMs ?? DEFAULT_SEARCH_BRANCH_TIMEOUT_MS;
|
|
424
|
+
if (
|
|
425
|
+
!Number.isSafeInteger(this.branchTimeoutMs) ||
|
|
426
|
+
this.branchTimeoutMs <= 0 ||
|
|
427
|
+
this.branchTimeoutMs > 2_147_483_647
|
|
428
|
+
) {
|
|
429
|
+
throw new Error("branchTimeoutMs must be an integer between 1 and 2147483647");
|
|
430
|
+
}
|
|
431
|
+
// Die projektweite Config ist selbst eine vollwertige Verdrahtungsquelle.
|
|
432
|
+
// Das explizite Dependency-Feld bleibt fuer Library-Aufrufer und Tests,
|
|
433
|
+
// gewinnt bei gleichzeitiger Angabe aber eindeutig. Sonst waere eine
|
|
434
|
+
// geladene, validierte SearchConfig erneut ein wirkungsloser Wert.
|
|
435
|
+
const configuredRelevancePolicies = deps.relevancePolicies ?? this.config.relevancePolicies;
|
|
436
|
+
this.relevancePolicies = new Map(
|
|
437
|
+
Object.entries(configuredRelevancePolicies ?? {}).map(([embeddingProfileId, configured]) => {
|
|
438
|
+
if (embeddingProfileId.trim() === "") {
|
|
439
|
+
throw new Error("relevancePolicies keys must be non-empty embedding profile ids");
|
|
440
|
+
}
|
|
441
|
+
return [embeddingProfileId, resolveRelevancePolicyBinding(configured)] as const;
|
|
442
|
+
}),
|
|
443
|
+
);
|
|
444
|
+
}
|
|
445
|
+
|
|
446
|
+
public async search(
|
|
447
|
+
request: UntangleSearchRequestV1,
|
|
448
|
+
options?: PortCallOptions,
|
|
449
|
+
): Promise<SearchResultSetV1> {
|
|
450
|
+
options?.signal?.throwIfAborted();
|
|
451
|
+
const extensions = normalizeExtensions(request);
|
|
452
|
+
const defaulted = applyConfigDefaults(
|
|
453
|
+
withFindabilityFilter(normalizeSearchQuery(request, this.deps.defaults), request),
|
|
454
|
+
request,
|
|
455
|
+
this.config,
|
|
456
|
+
);
|
|
457
|
+
const query = defaulted.query;
|
|
458
|
+
// Der Ursprung reist als Parameter mit, NICHT als Feld der Instanz: zwei
|
|
459
|
+
// gleichzeitige `search()`-Aufrufe auf derselben Maschine wuerden sich
|
|
460
|
+
// sonst gegenseitig die Begruendung im Ledger umschreiben.
|
|
461
|
+
const minScoreOrigin: MinScoreOrigin = defaulted.minScoreFromConfig
|
|
462
|
+
? "from searchConfig.minScoreDefault"
|
|
463
|
+
: "requested";
|
|
464
|
+
// Fehlerhafte Muster sind ein Eingabefehler des Nutzers und werden HIER
|
|
465
|
+
// abgelehnt. Faellt der Fehler erst beim Abgleich je Kandidat an, faengt
|
|
466
|
+
// ihn das Ziel-catch auf und meldet einen Speicherausfall — eine falsche
|
|
467
|
+
// Diagnose fuer ein falsches Zeichen im Suchmuster.
|
|
468
|
+
assertValidGlobPatterns(query.filters.pathGlobs, "pathGlob");
|
|
469
|
+
assertValidGlobPatterns(extensions.pathExcludes, "pathExclude");
|
|
470
|
+
|
|
471
|
+
const ledger = new SearchSkipLedger();
|
|
472
|
+
const warnings: ToolWarningV1[] = [];
|
|
473
|
+
|
|
474
|
+
// Die Obergrenze aus der Konfiguration ist eine Grenze, kein Vorschlag:
|
|
475
|
+
// ohne sie zieht eine Anfrage mit candidateLimit=100000 drei Rundlaeufe
|
|
476
|
+
// ueber 100000 Punkte plus einen BM25-Aufbau darueber.
|
|
477
|
+
//
|
|
478
|
+
// WARUM AUCH `limit` GEDECKELT WIRD: `SearchQueryV1` verlangt
|
|
479
|
+
// `candidateLimit >= limit`, und `normalizeSearchQuery` weist alles andere
|
|
480
|
+
// ab. Wer die Obergrenze nur auf `candidateLimit` anwendet und den Wert
|
|
481
|
+
// anschliessend mit `Math.max(max, limit)` wieder anhebt, hat die Grenze
|
|
482
|
+
// damit fuer JEDE Anfrage mit grossem `limit` ausgeschaltet — die drei
|
|
483
|
+
// Speicherrundlaeufe ueber 100000 Punkte laufen dann wie zuvor. Die
|
|
484
|
+
// Obergrenze ist eine Aussage ueber die ARBEIT, nicht ueber ein einzelnes
|
|
485
|
+
// Feld: mehr als `maxCandidateLimit` Kandidaten werden nicht geholt, also
|
|
486
|
+
// koennen auch nicht mehr als `maxCandidateLimit` Treffer entstehen. Beide
|
|
487
|
+
// Deckelungen stehen einzeln und benannt im Ergebnis (E4).
|
|
488
|
+
const maxCandidateLimit = this.config.maxCandidateLimit;
|
|
489
|
+
const candidateCapActive = Number.isInteger(maxCandidateLimit) && maxCandidateLimit > 0;
|
|
490
|
+
let candidateLimit = query.candidateLimit;
|
|
491
|
+
let limit = query.limit;
|
|
492
|
+
if (candidateCapActive && candidateLimit > maxCandidateLimit) {
|
|
493
|
+
candidateLimit = maxCandidateLimit;
|
|
494
|
+
warnings.push({
|
|
495
|
+
code: "CANDIDATE_LIMIT_CLAMPED",
|
|
496
|
+
message: `candidateLimit ${query.candidateLimit} exceeds the configured maximum of ${maxCandidateLimit}; the search ran with ${candidateLimit} candidates per branch instead`,
|
|
497
|
+
details: {
|
|
498
|
+
requested: query.candidateLimit,
|
|
499
|
+
effective: candidateLimit,
|
|
500
|
+
maxCandidateLimit,
|
|
501
|
+
},
|
|
502
|
+
});
|
|
503
|
+
}
|
|
504
|
+
if (limit > candidateLimit) {
|
|
505
|
+
warnings.push({
|
|
506
|
+
code: "RESULT_LIMIT_CLAMPED",
|
|
507
|
+
message: `limit ${limit} exceeds the candidate pool of ${candidateLimit} (configured maximum ${maxCandidateLimit}); the search returned at most ${candidateLimit} hits per profile group instead`,
|
|
508
|
+
details: {
|
|
509
|
+
requested: limit,
|
|
510
|
+
effective: candidateLimit,
|
|
511
|
+
candidateLimit,
|
|
512
|
+
maxCandidateLimit,
|
|
513
|
+
},
|
|
514
|
+
});
|
|
515
|
+
limit = candidateLimit;
|
|
516
|
+
}
|
|
517
|
+
|
|
518
|
+
const effectiveQuery: SearchQueryV1 = {
|
|
519
|
+
...query,
|
|
520
|
+
includeContent: extensions.includeContent,
|
|
521
|
+
candidateLimit,
|
|
522
|
+
limit,
|
|
523
|
+
};
|
|
524
|
+
// Die VOLLSTAENDIGE normalisierte Anfrage geht hinein, nicht eine von Hand
|
|
525
|
+
// gepflegte Auswahl daraus: `buildSearchQueryFingerprint` streicht selbst,
|
|
526
|
+
// was reine Darstellung ist (siehe SEARCH_FINGERPRINT_IGNORED_FIELDS).
|
|
527
|
+
// Wuerde hier eine Auswahl stehen, fiele jedes spaeter ergaenzte
|
|
528
|
+
// ergebnisveraendernde Feld wieder lautlos heraus — genau so verloren
|
|
529
|
+
// `workspaceId`, `minScore` und `diversify` ihren Platz im Fingerabdruck.
|
|
530
|
+
const rankingOptions = fingerprintRankingOptions(extensions);
|
|
531
|
+
const fingerprint = buildSearchQueryFingerprint({
|
|
532
|
+
...effectiveQuery,
|
|
533
|
+
...(rankingOptions === undefined ? {} : { rankingOptions }),
|
|
534
|
+
});
|
|
535
|
+
const rankingContextFingerprint = this.rankingContextFingerprint(effectiveQuery, extensions);
|
|
536
|
+
|
|
537
|
+
// Ein nicht verdrahteter Port bleibt derselbe ehrliche UNAVAILABLE-Fall
|
|
538
|
+
// wie bisher. Ist einer vorhanden, wird er erst NACH Ranking, Dedupe und
|
|
539
|
+
// Limit gerufen (unterhalb der Zielschleife).
|
|
540
|
+
if (effectiveQuery.includeGraphContext && this.deps.graphEnrichment === undefined) {
|
|
541
|
+
warnings.push({
|
|
542
|
+
code: "GRAPH_CONTEXT_UNAVAILABLE",
|
|
543
|
+
message:
|
|
544
|
+
"the request asked for structural graph context (includeContext), but no graph enrichment provider is wired; the hits carry NO graph context",
|
|
545
|
+
});
|
|
546
|
+
}
|
|
547
|
+
if (effectiveQuery.includeLinked && this.deps.graphEnrichment === undefined) {
|
|
548
|
+
warnings.push({
|
|
549
|
+
code: "LINKED_ARTIFACTS_UNAVAILABLE",
|
|
550
|
+
message:
|
|
551
|
+
"the request asked for linked symbols (includeLinked), but no graph enrichment provider is wired; the hits carry NO linked symbols",
|
|
552
|
+
});
|
|
553
|
+
}
|
|
554
|
+
|
|
555
|
+
if (effectiveQuery.filters.findability !== undefined && this.deps.findability === undefined) {
|
|
556
|
+
warnings.push({
|
|
557
|
+
code: "FINDABILITY_FILTER_UNAVAILABLE",
|
|
558
|
+
message:
|
|
559
|
+
"the request asked for a findability filter, but no findability lookup is wired into the engine — the filter was NOT applied",
|
|
560
|
+
details: { filter: effectiveQuery.filters.findability },
|
|
561
|
+
});
|
|
562
|
+
}
|
|
563
|
+
if (branchesOfMode(extensions.mode).lexical && this.deps.sparseVectorizer === undefined) {
|
|
564
|
+
warnings.push({
|
|
565
|
+
code: "SPARSE_RECALL_UNAVAILABLE",
|
|
566
|
+
message:
|
|
567
|
+
"no sparse vectorizer is wired into the engine; the requested lexical retrieval branch is unavailable",
|
|
568
|
+
});
|
|
569
|
+
}
|
|
570
|
+
|
|
571
|
+
const groups: SearchHitGroupV1[] = [];
|
|
572
|
+
const failedTargets: { profileId: string; error: unknown }[] = [];
|
|
573
|
+
const refIdsByRepo = new Map<string, Set<string>>();
|
|
574
|
+
for (const target of this.deps.targets) {
|
|
575
|
+
options?.signal?.throwIfAborted();
|
|
576
|
+
try {
|
|
577
|
+
const outcome = await this.searchTarget(
|
|
578
|
+
target,
|
|
579
|
+
effectiveQuery,
|
|
580
|
+
extensions,
|
|
581
|
+
ledger,
|
|
582
|
+
minScoreOrigin,
|
|
583
|
+
options,
|
|
584
|
+
);
|
|
585
|
+
groups.push(outcome.group);
|
|
586
|
+
warnings.push(...(outcome.branchWarnings ?? []));
|
|
587
|
+
if (outcome.relevanceWarning !== undefined) {
|
|
588
|
+
warnings.push(outcome.relevanceWarning);
|
|
589
|
+
}
|
|
590
|
+
if (outcome.indexContextWarning !== undefined) {
|
|
591
|
+
warnings.push(outcome.indexContextWarning);
|
|
592
|
+
}
|
|
593
|
+
for (const hit of outcome.group.hits) {
|
|
594
|
+
const set = refIdsByRepo.get(hit.repoId) ?? new Set<string>();
|
|
595
|
+
set.add(hit.refId);
|
|
596
|
+
refIdsByRepo.set(hit.repoId, set);
|
|
597
|
+
}
|
|
598
|
+
} catch (error) {
|
|
599
|
+
rethrowCancellation(error, options);
|
|
600
|
+
// E4: ein ausgefallenes Ziel wird gemeldet, nicht verschluckt. Die
|
|
601
|
+
// uebrigen Profile werden weiter durchsucht (spec.md §25.2: eine
|
|
602
|
+
// Lesende Operation degradiert sichtbar, statt ganz zu scheitern).
|
|
603
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
604
|
+
const code = isUntangleError(error) ? error.code : "PROFILE_UNAVAILABLE";
|
|
605
|
+
failedTargets.push({ profileId: target.embeddingProfileId, error });
|
|
606
|
+
warnings.push({
|
|
607
|
+
code,
|
|
608
|
+
message: `profile "${target.embeddingProfileId}" could not be searched: ${message}`,
|
|
609
|
+
details: { collection: target.collection, repoIds: target.repoIds },
|
|
610
|
+
});
|
|
611
|
+
ledger.add({
|
|
612
|
+
subjectKind: "search_candidate",
|
|
613
|
+
subjectId: target.embeddingProfileId,
|
|
614
|
+
reason: "provider_unavailable",
|
|
615
|
+
detail: `collection "${target.collection}" did not answer: ${message}`,
|
|
616
|
+
recoverable: true,
|
|
617
|
+
});
|
|
618
|
+
this.deps.logger?.log({
|
|
619
|
+
timestamp: this.deps.clock?.nowIso() ?? "",
|
|
620
|
+
level: "warn",
|
|
621
|
+
component: "search",
|
|
622
|
+
event: "profile_search_failed",
|
|
623
|
+
context: { profileId: target.embeddingProfileId, message },
|
|
624
|
+
});
|
|
625
|
+
}
|
|
626
|
+
}
|
|
627
|
+
|
|
628
|
+
if (
|
|
629
|
+
failedTargets.length > 0 &&
|
|
630
|
+
!groups.some((group) => group.notSearchedReason === undefined)
|
|
631
|
+
) {
|
|
632
|
+
const first = failedTargets[0] as (typeof failedTargets)[number];
|
|
633
|
+
throw new UntangleError(
|
|
634
|
+
isUntangleError(first.error) ? first.error.code : "QDRANT_UNAVAILABLE",
|
|
635
|
+
`search failed: no requested profile completed a retrieval branch; ${failedTargets
|
|
636
|
+
.map(
|
|
637
|
+
({ profileId, error }) =>
|
|
638
|
+
`profile "${profileId}": ${error instanceof Error ? error.message : String(error)}`,
|
|
639
|
+
)
|
|
640
|
+
.join("; ")}`,
|
|
641
|
+
{
|
|
642
|
+
operation: "search",
|
|
643
|
+
retryable: failedTargets.some(({ error }) => !isUntangleError(error) || error.retryable),
|
|
644
|
+
context: { failedProfileIds: failedTargets.map(({ profileId }) => profileId) },
|
|
645
|
+
cause: first.error,
|
|
646
|
+
},
|
|
647
|
+
);
|
|
648
|
+
}
|
|
649
|
+
|
|
650
|
+
// Pipeline-Schritt 11: Nur die BEREITS final begrenzten und gerankten
|
|
651
|
+
// Treffer verlassen den Abrufkern in Richtung Graphadapter. Weder ein
|
|
652
|
+
// Graphsignal noch ein Ladefehler kann dadurch Ranking, Score, Dedupe,
|
|
653
|
+
// Fingerabdruck oder Trefferzahl veraendern.
|
|
654
|
+
//
|
|
655
|
+
// Diese Bedingung fragt, ob es ueberhaupt einen Treffer gibt, an den man
|
|
656
|
+
// Relationen haengen kann. Sie ist AUSDRUECKLICH KEINE Enthaltungspruefung:
|
|
657
|
+
// unter `evidence` behaelt eine von der Policy abgelehnte Gruppe ihre
|
|
658
|
+
// Treffer und wird folgerichtig angereichert. Anreicherung zu ueberspringen,
|
|
659
|
+
// weil die Policy ablehnte, waere eine zweite, verdeckte Bedeutung an
|
|
660
|
+
// derselben Stelle — genau die Verwechslung, die diese Welle beseitigt.
|
|
661
|
+
const hasHitsToEnrich = groups.some((group) => group.hits.length > 0);
|
|
662
|
+
if (
|
|
663
|
+
this.deps.graphEnrichment !== undefined &&
|
|
664
|
+
(effectiveQuery.includeGraphContext || effectiveQuery.includeLinked) &&
|
|
665
|
+
hasHitsToEnrich
|
|
666
|
+
) {
|
|
667
|
+
await this.enrichFinalHits(groups, effectiveQuery, warnings, options);
|
|
668
|
+
}
|
|
669
|
+
|
|
670
|
+
for (const [repoId, refIds] of refIdsByRepo) {
|
|
671
|
+
if (refIds.size > 1) {
|
|
672
|
+
warnings.push({
|
|
673
|
+
code: "MULTIPLE_REFS",
|
|
674
|
+
message: `repository "${repoId}" contributed hits from ${refIds.size} refs; results from different refs describe different states of the same code`,
|
|
675
|
+
details: { repoId, refIds: sortByCodepoints([...refIds]) },
|
|
676
|
+
});
|
|
677
|
+
}
|
|
678
|
+
}
|
|
679
|
+
|
|
680
|
+
const incompleteHits = groups.flatMap((group) =>
|
|
681
|
+
group.hits.filter((hit) => hit.incompletePayloadFields !== undefined),
|
|
682
|
+
);
|
|
683
|
+
if (incompleteHits.length > 0) {
|
|
684
|
+
warnings.push({
|
|
685
|
+
code: "INCOMPLETE_PAYLOAD",
|
|
686
|
+
message: `${incompleteHits.length} hit(s) carry payload fields that were missing in the index and filled with documented substitutes; see each hit's incompletePayloadFields`,
|
|
687
|
+
details: {
|
|
688
|
+
chunkIds: sortByCodepoints(incompleteHits.map((hit) => hit.chunkId)),
|
|
689
|
+
},
|
|
690
|
+
});
|
|
691
|
+
}
|
|
692
|
+
|
|
693
|
+
const profileSeparation = groups.length > 1;
|
|
694
|
+
if (profileSeparation) {
|
|
695
|
+
warnings.push({
|
|
696
|
+
code: "PROFILE_SEPARATION",
|
|
697
|
+
message:
|
|
698
|
+
"results are grouped per embedding profile; scores are comparable only WITHIN a group (spec.md §13.4)",
|
|
699
|
+
details: { profileIds: groups.map((group) => group.embeddingProfileId) },
|
|
700
|
+
});
|
|
701
|
+
}
|
|
702
|
+
|
|
703
|
+
return {
|
|
704
|
+
schemaVersion: 1,
|
|
705
|
+
queryFingerprint: fingerprint,
|
|
706
|
+
rankingContextFingerprint,
|
|
707
|
+
query: effectiveQuery,
|
|
708
|
+
groups,
|
|
709
|
+
profileSeparation,
|
|
710
|
+
totalHits: groups.reduce((sum, group) => sum + group.hits.length, 0),
|
|
711
|
+
skipped: ledger.toArray(),
|
|
712
|
+
warnings,
|
|
713
|
+
};
|
|
714
|
+
}
|
|
715
|
+
|
|
716
|
+
private rankingContextFingerprint(
|
|
717
|
+
query: SearchQueryV1,
|
|
718
|
+
extensions: SearchQueryExtensionsV1,
|
|
719
|
+
): string {
|
|
720
|
+
const targets = this.deps.targets
|
|
721
|
+
.map((target) => ({
|
|
722
|
+
embeddingProfileId: target.embeddingProfileId,
|
|
723
|
+
collection: target.collection,
|
|
724
|
+
repoIds: uniqueSortedByCodepoints(target.repoIds),
|
|
725
|
+
}))
|
|
726
|
+
.sort((left, right) => compareCodepoints(canonicalJson(left), canonicalJson(right)));
|
|
727
|
+
const relevancePolicies = [...this.relevancePolicies.entries()]
|
|
728
|
+
.sort(([left], [right]) => compareCodepoints(left, right))
|
|
729
|
+
.map(([embeddingProfileId, binding]) => ({ embeddingProfileId, ...binding }));
|
|
730
|
+
const repoPriorities = Object.entries(this.deps.repoPriorities ?? {}).sort(([left], [right]) =>
|
|
731
|
+
compareCodepoints(left, right),
|
|
732
|
+
);
|
|
733
|
+
return sha256Hex(
|
|
734
|
+
canonicalJson({
|
|
735
|
+
schemaVersion: 1,
|
|
736
|
+
rankerVersion: query.rankerVersion,
|
|
737
|
+
mode: extensions.mode,
|
|
738
|
+
// Mode-selected recall and independent branch failure handling must
|
|
739
|
+
// never be cohort-compatible with the previous rank-only strategy.
|
|
740
|
+
retrievalStrategy: "independent-branches-v1",
|
|
741
|
+
...(extensions.mode === "hybrid" ? { branchTimeoutMs: this.branchTimeoutMs } : {}),
|
|
742
|
+
// Ausdruecklicher bedingter Spread, NICHT `abstention: x ?? undefined`:
|
|
743
|
+
// der Hash darf nicht davon abhaengen, ob canonicalJson undefined
|
|
744
|
+
// wegwirft. Unter `strict` ist das gehashte Objekt buchstaeblich
|
|
745
|
+
// unveraendert, also bleiben persistierte Journey-Kohorten vergleichbar.
|
|
746
|
+
...(extensions.abstention === "strict" ? {} : { abstention: extensions.abstention }),
|
|
747
|
+
sort: extensions.sort,
|
|
748
|
+
complexityBias: extensions.complexityBias,
|
|
749
|
+
diversify: query.diversify,
|
|
750
|
+
minScore: query.minScore ?? null,
|
|
751
|
+
branchWeights: this.branchWeights,
|
|
752
|
+
boostWeights: this.config.boostWeights,
|
|
753
|
+
recencyHalfLifeDays: this.deps.recencyHalfLifeDays ?? DEFAULT_RECENCY_HALF_LIFE_DAYS,
|
|
754
|
+
clockEnabled: this.deps.clock !== undefined,
|
|
755
|
+
sparseVectorizerEnabled: this.deps.sparseVectorizer !== undefined,
|
|
756
|
+
graphProximityEnabled: this.deps.graphProximity !== undefined,
|
|
757
|
+
findabilityEnabled: this.deps.findability !== undefined,
|
|
758
|
+
targets,
|
|
759
|
+
repoPriorities,
|
|
760
|
+
relevancePolicies,
|
|
761
|
+
}),
|
|
762
|
+
);
|
|
763
|
+
}
|
|
764
|
+
|
|
765
|
+
private async enrichFinalHits(
|
|
766
|
+
groups: SearchHitGroupV1[],
|
|
767
|
+
query: SearchQueryV1,
|
|
768
|
+
warnings: ToolWarningV1[],
|
|
769
|
+
options?: PortCallOptions,
|
|
770
|
+
): Promise<void> {
|
|
771
|
+
const provider = this.deps.graphEnrichment;
|
|
772
|
+
if (provider === undefined) return;
|
|
773
|
+
|
|
774
|
+
const targets = groups.flatMap((group, groupIndex) =>
|
|
775
|
+
group.hits.map((hit, hitIndex) => ({
|
|
776
|
+
targetId: `group:${String(groupIndex)}:hit:${String(hitIndex)}`,
|
|
777
|
+
workspaceId: query.workspaceId,
|
|
778
|
+
generationId: hit.indexGeneration,
|
|
779
|
+
repoId: hit.repoId,
|
|
780
|
+
refId: hit.refId,
|
|
781
|
+
fileId: hit.fileId,
|
|
782
|
+
filePath: hit.filePath,
|
|
783
|
+
...(hit.symbolId === undefined ? {} : { symbolId: hit.symbolId }),
|
|
784
|
+
})),
|
|
785
|
+
);
|
|
786
|
+
|
|
787
|
+
try {
|
|
788
|
+
const result = await provider.enrich(
|
|
789
|
+
{
|
|
790
|
+
includeContext: query.includeGraphContext,
|
|
791
|
+
includeLinked: query.includeLinked,
|
|
792
|
+
relationLimit: SEARCH_GRAPH_RELATION_LIMIT,
|
|
793
|
+
targets,
|
|
794
|
+
},
|
|
795
|
+
options,
|
|
796
|
+
);
|
|
797
|
+
options?.signal?.throwIfAborted();
|
|
798
|
+
|
|
799
|
+
const expectedIds = new Set(targets.map((target) => target.targetId));
|
|
800
|
+
const entries = new Map<string, (typeof result.entries)[number]>();
|
|
801
|
+
const duplicateIds = new Set<string>();
|
|
802
|
+
const unexpectedIds = new Set<string>();
|
|
803
|
+
for (const entry of result.entries) {
|
|
804
|
+
if (!expectedIds.has(entry.targetId)) {
|
|
805
|
+
unexpectedIds.add(entry.targetId);
|
|
806
|
+
continue;
|
|
807
|
+
}
|
|
808
|
+
if (entries.has(entry.targetId)) duplicateIds.add(entry.targetId);
|
|
809
|
+
else entries.set(entry.targetId, entry);
|
|
810
|
+
}
|
|
811
|
+
const missingIds: string[] = [];
|
|
812
|
+
|
|
813
|
+
for (let groupIndex = 0; groupIndex < groups.length; groupIndex++) {
|
|
814
|
+
const group = groups[groupIndex] as SearchHitGroupV1;
|
|
815
|
+
for (let hitIndex = 0; hitIndex < group.hits.length; hitIndex++) {
|
|
816
|
+
const hit = group.hits[hitIndex] as SearchHitV1;
|
|
817
|
+
const targetId = `group:${String(groupIndex)}:hit:${String(hitIndex)}`;
|
|
818
|
+
const entry = duplicateIds.has(targetId) ? undefined : entries.get(targetId);
|
|
819
|
+
if (entry === undefined) missingIds.push(targetId);
|
|
820
|
+
const detail = `graph enrichment provider returned no unique result for final hit "${hit.chunkId}"`;
|
|
821
|
+
|
|
822
|
+
if (query.includeGraphContext) {
|
|
823
|
+
hit.graphContext =
|
|
824
|
+
entry?.graphContext === undefined
|
|
825
|
+
? unavailableSearchGraphRelations(
|
|
826
|
+
"provider_result_missing",
|
|
827
|
+
detail,
|
|
828
|
+
SEARCH_GRAPH_RELATION_LIMIT,
|
|
829
|
+
)
|
|
830
|
+
: createSearchGraphRelations(
|
|
831
|
+
entry.graphContext.relations,
|
|
832
|
+
entry.graphContext.omissions,
|
|
833
|
+
SEARCH_GRAPH_RELATION_LIMIT,
|
|
834
|
+
);
|
|
835
|
+
}
|
|
836
|
+
if (query.includeLinked) {
|
|
837
|
+
hit.linkedSymbols =
|
|
838
|
+
entry?.linkedSymbols === undefined
|
|
839
|
+
? unavailableSearchGraphRelations(
|
|
840
|
+
"provider_result_missing",
|
|
841
|
+
detail,
|
|
842
|
+
SEARCH_GRAPH_RELATION_LIMIT,
|
|
843
|
+
)
|
|
844
|
+
: createSearchGraphRelations(
|
|
845
|
+
entry.linkedSymbols.relations,
|
|
846
|
+
entry.linkedSymbols.omissions,
|
|
847
|
+
SEARCH_GRAPH_RELATION_LIMIT,
|
|
848
|
+
);
|
|
849
|
+
}
|
|
850
|
+
}
|
|
851
|
+
}
|
|
852
|
+
|
|
853
|
+
const providerWarnings = [...result.warnings]
|
|
854
|
+
.map(normalizeGraphProviderWarning)
|
|
855
|
+
.sort((left, right) => {
|
|
856
|
+
const byCode = compareCodepoints(left.code, right.code);
|
|
857
|
+
if (byCode !== 0) return byCode;
|
|
858
|
+
const byMessage = compareCodepoints(left.message, right.message);
|
|
859
|
+
return byMessage !== 0
|
|
860
|
+
? byMessage
|
|
861
|
+
: compareCodepoints(
|
|
862
|
+
JSON.stringify(left.details ?? {}),
|
|
863
|
+
JSON.stringify(right.details ?? {}),
|
|
864
|
+
);
|
|
865
|
+
});
|
|
866
|
+
warnings.push(...providerWarnings.slice(0, SEARCH_GRAPH_PROVIDER_WARNING_LIMIT));
|
|
867
|
+
if (providerWarnings.length > SEARCH_GRAPH_PROVIDER_WARNING_LIMIT) {
|
|
868
|
+
warnings.push({
|
|
869
|
+
code: "GRAPH_ENRICHMENT_WARNINGS_TRUNCATED",
|
|
870
|
+
message: `${String(providerWarnings.length - SEARCH_GRAPH_PROVIDER_WARNING_LIMIT)} additional graph enrichment provider warning(s) were omitted by the response limit`,
|
|
871
|
+
details: {
|
|
872
|
+
warningLimit: SEARCH_GRAPH_PROVIDER_WARNING_LIMIT,
|
|
873
|
+
omittedWarningCount: providerWarnings.length - SEARCH_GRAPH_PROVIDER_WARNING_LIMIT,
|
|
874
|
+
},
|
|
875
|
+
});
|
|
876
|
+
}
|
|
877
|
+
if (missingIds.length > 0 || duplicateIds.size > 0 || unexpectedIds.size > 0) {
|
|
878
|
+
const missing = boundedDiagnosticIds(missingIds);
|
|
879
|
+
const duplicates = boundedDiagnosticIds([...duplicateIds]);
|
|
880
|
+
const unexpected = boundedDiagnosticIds([...unexpectedIds]);
|
|
881
|
+
warnings.push({
|
|
882
|
+
code: "GRAPH_ENRICHMENT_INCOMPLETE",
|
|
883
|
+
message:
|
|
884
|
+
"the graph enrichment provider returned an incomplete or ambiguous batch; affected hits carry provider_result_missing omissions",
|
|
885
|
+
details: {
|
|
886
|
+
missingTargetIds: missing.ids,
|
|
887
|
+
omittedMissingTargetIdCount: missing.omittedCount,
|
|
888
|
+
duplicateTargetIds: duplicates.ids,
|
|
889
|
+
omittedDuplicateTargetIdCount: duplicates.omittedCount,
|
|
890
|
+
unexpectedTargetIds: unexpected.ids,
|
|
891
|
+
omittedUnexpectedTargetIdCount: unexpected.omittedCount,
|
|
892
|
+
},
|
|
893
|
+
});
|
|
894
|
+
}
|
|
895
|
+
} catch (error) {
|
|
896
|
+
options?.signal?.throwIfAborted();
|
|
897
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
898
|
+
const detail = boundSearchGraphDetail(`graph enrichment failed after ranking: ${message}`);
|
|
899
|
+
markFinalHitGraphUnavailable(groups, query, detail);
|
|
900
|
+
if (query.includeGraphContext) {
|
|
901
|
+
warnings.push({ code: "GRAPH_CONTEXT_UNAVAILABLE", message: detail });
|
|
902
|
+
}
|
|
903
|
+
if (query.includeLinked) {
|
|
904
|
+
warnings.push({ code: "LINKED_ARTIFACTS_UNAVAILABLE", message: detail });
|
|
905
|
+
}
|
|
906
|
+
this.deps.logger?.log({
|
|
907
|
+
timestamp: this.deps.clock?.nowIso() ?? "",
|
|
908
|
+
level: "warn",
|
|
909
|
+
component: "search",
|
|
910
|
+
event: "graph_enrichment_failed",
|
|
911
|
+
context: { message },
|
|
912
|
+
});
|
|
913
|
+
}
|
|
914
|
+
}
|
|
915
|
+
|
|
916
|
+
/**
|
|
917
|
+
* grep laeuft im selben Geltungsbereich wie `search`: Workspace aus den
|
|
918
|
+
* Vorgaben, wenn die Anfrage keinen nennt, und nur die Repositories der
|
|
919
|
+
* verdrahteten Ziele. Sonst faende dieselbe Oberflaeche mit demselben
|
|
920
|
+
* Aufrufer je nach Werkzeug verschiedene Mengen.
|
|
921
|
+
*/
|
|
922
|
+
public grep(request: GrepRequestV1, options?: PortCallOptions): Promise<GrepResultSetV1> {
|
|
923
|
+
return grepIndexedChunks(
|
|
924
|
+
this.deps.vectorStore,
|
|
925
|
+
uniqueSortedByCodepoints(this.deps.targets.map((target) => target.collection)),
|
|
926
|
+
request,
|
|
927
|
+
options,
|
|
928
|
+
{
|
|
929
|
+
workspaceId: this.deps.defaults.workspaceId,
|
|
930
|
+
repoIds: uniqueSortedByCodepoints(this.deps.targets.flatMap((target) => target.repoIds)),
|
|
931
|
+
},
|
|
932
|
+
);
|
|
933
|
+
}
|
|
934
|
+
|
|
935
|
+
// -------------------------------------------------------------------------
|
|
936
|
+
// Ein Ziel (= ein Embedding-Profil, eine Collection)
|
|
937
|
+
// -------------------------------------------------------------------------
|
|
938
|
+
|
|
939
|
+
private async searchTarget(
|
|
940
|
+
target: ProfileTargetV1,
|
|
941
|
+
query: SearchQueryV1,
|
|
942
|
+
extensions: SearchQueryExtensionsV1,
|
|
943
|
+
ledger: SearchSkipLedger,
|
|
944
|
+
minScoreOrigin: MinScoreOrigin,
|
|
945
|
+
options?: PortCallOptions,
|
|
946
|
+
): Promise<TargetOutcome> {
|
|
947
|
+
const scope: SearchScopeV1 = {
|
|
948
|
+
workspaceId: query.workspaceId,
|
|
949
|
+
targetRepoIds: target.repoIds,
|
|
950
|
+
};
|
|
951
|
+
const requestedFilter = buildPayloadFilter(query.filters, scope);
|
|
952
|
+
// A ref-less request must not turn a locally recorded current corpus into
|
|
953
|
+
// an unbounded scan of every historical ref still present in the shared
|
|
954
|
+
// collection. Use every recorded ref in the effective repo scope. Doing
|
|
955
|
+
// this at the port boundary keeps non-CLI callers honest as well.
|
|
956
|
+
const inferredRefIds =
|
|
957
|
+
query.filters.refIds.length === 0 && requestedFilter.emptyScopeReason === undefined
|
|
958
|
+
? uniqueSortedByCodepoints(
|
|
959
|
+
(target.indexContexts ?? [])
|
|
960
|
+
.filter(
|
|
961
|
+
(context) =>
|
|
962
|
+
requestedFilter.scopeRepoIds.length === 0 ||
|
|
963
|
+
requestedFilter.scopeRepoIds.includes(context.repoId),
|
|
964
|
+
)
|
|
965
|
+
.map((context) => context.refId),
|
|
966
|
+
)
|
|
967
|
+
: [];
|
|
968
|
+
const effectiveRefIds =
|
|
969
|
+
query.filters.refIds.length === 0 ? inferredRefIds : query.filters.refIds;
|
|
970
|
+
const built =
|
|
971
|
+
query.filters.refIds.length === 0 && inferredRefIds.length > 0
|
|
972
|
+
? buildPayloadFilter({ ...query.filters, refIds: inferredRefIds }, scope)
|
|
973
|
+
: requestedFilter;
|
|
974
|
+
// WP-178: Die zweite Verteidigungslinie prueft ab hier den WIRKSAMEN
|
|
975
|
+
// Bereich (Ziel geschnitten mit dem Repo-Filter der Anfrage), nicht mehr
|
|
976
|
+
// nur die Zieldeklaration. Ein Speicher, der den Payloadfilter ignoriert,
|
|
977
|
+
// lieferte sonst einen Treffer aus einem Repository, das der Nutzer
|
|
978
|
+
// ausdruecklich NICHT angefragt hat — waehrend `group.repoIds` das
|
|
979
|
+
// Gegenteil behauptet.
|
|
980
|
+
const effectiveScope: SearchScopeV1 = {
|
|
981
|
+
workspaceId: query.workspaceId,
|
|
982
|
+
targetRepoIds: built.scopeRepoIds,
|
|
983
|
+
};
|
|
984
|
+
if (built.emptyScopeReason !== undefined) {
|
|
985
|
+
// E4: Der Speicher wird NICHT gefragt — eine leere `in`-Liste waere
|
|
986
|
+
// kein Filter, sondern gar keiner, und die Suche liefe ueber die ganze
|
|
987
|
+
// Collection. Der leere Bereich steht stattdessen im Ledger.
|
|
988
|
+
ledger.add({
|
|
989
|
+
subjectKind: "search_candidate",
|
|
990
|
+
subjectId: target.embeddingProfileId,
|
|
991
|
+
reason: "excluded_by_configuration",
|
|
992
|
+
detail: built.emptyScopeReason,
|
|
993
|
+
recoverable: true,
|
|
994
|
+
});
|
|
995
|
+
return {
|
|
996
|
+
group: {
|
|
997
|
+
embeddingProfileId: target.embeddingProfileId,
|
|
998
|
+
repoIds: [],
|
|
999
|
+
hits: [],
|
|
1000
|
+
rankNormalized: false,
|
|
1001
|
+
caveats: [built.emptyScopeReason],
|
|
1002
|
+
// E4: „nicht gesucht" ist im Ergebnis von „gesucht, nichts gefunden"
|
|
1003
|
+
// zu unterscheiden. Ohne diese Markierung sieht die Gruppe wie eine
|
|
1004
|
+
// leere Trefferliste aus, und die Findability-Sonde macht daraus
|
|
1005
|
+
// „das Symbol findet sich selbst nicht" — eine Aussage ueber den
|
|
1006
|
+
// Code, die auf keiner Beobachtung steht.
|
|
1007
|
+
notSearchedReason: built.emptyScopeReason,
|
|
1008
|
+
},
|
|
1009
|
+
refIds: new Set<string>(),
|
|
1010
|
+
};
|
|
1011
|
+
}
|
|
1012
|
+
// WP-176 (E4): Angefragte Repositories, die dieses Ziel gar nicht fuehrt,
|
|
1013
|
+
// fielen bisher lautlos aus dem Durchschnitt. Nur der VOLLSTAENDIG leere
|
|
1014
|
+
// Fall (oben) kam in den Ledger. Ein teilweiser Ausfall sieht damit wie
|
|
1015
|
+
// eine vollstaendige Suche aus — die gefaehrlichere der beiden Formen,
|
|
1016
|
+
// weil sie Treffer liefert und deshalb niemand nachfragt.
|
|
1017
|
+
const outOfScope = built.outOfScopeRepoIds;
|
|
1018
|
+
const caveats = built.residual.map(
|
|
1019
|
+
(residual) =>
|
|
1020
|
+
`filter "${residual.name}" was applied in-process, not in the store: ${residual.reason}`,
|
|
1021
|
+
);
|
|
1022
|
+
if (query.filters.refIds.length === 0 && inferredRefIds.length > 0) {
|
|
1023
|
+
caveats.push(
|
|
1024
|
+
`the request omitted refs; this target was searched only across locally recorded refs [${inferredRefIds.join(", ")}]`,
|
|
1025
|
+
);
|
|
1026
|
+
}
|
|
1027
|
+
if (outOfScope.length > 0) {
|
|
1028
|
+
const detail = `the requested repositories [${outOfScope.join(", ")}] are not part of this target's repositories [${sortByCodepoints([...target.repoIds]).join(", ")}]; they were NOT searched here`;
|
|
1029
|
+
caveats.push(detail);
|
|
1030
|
+
for (const repoId of outOfScope) {
|
|
1031
|
+
ledger.add({
|
|
1032
|
+
subjectKind: "search_candidate",
|
|
1033
|
+
subjectId: `${target.embeddingProfileId}:${repoId}`,
|
|
1034
|
+
reason: "excluded_by_configuration",
|
|
1035
|
+
detail,
|
|
1036
|
+
recoverable: true,
|
|
1037
|
+
});
|
|
1038
|
+
}
|
|
1039
|
+
}
|
|
1040
|
+
|
|
1041
|
+
const pool = new Map<string, SearchCandidate>();
|
|
1042
|
+
const rejectedPointIds = new Set<string>();
|
|
1043
|
+
const active = branchesOfMode(extensions.mode);
|
|
1044
|
+
const retrieval = await retrieveIndependentBranches(
|
|
1045
|
+
{
|
|
1046
|
+
vectorStore: this.deps.vectorStore,
|
|
1047
|
+
target,
|
|
1048
|
+
text: query.text,
|
|
1049
|
+
filter: built.filter,
|
|
1050
|
+
candidateLimit: query.candidateLimit,
|
|
1051
|
+
mode: extensions.mode,
|
|
1052
|
+
branchTimeoutMs: this.branchTimeoutMs,
|
|
1053
|
+
...(this.deps.sparseVectorizer === undefined
|
|
1054
|
+
? {}
|
|
1055
|
+
: { sparseVectorizer: this.deps.sparseVectorizer }),
|
|
1056
|
+
},
|
|
1057
|
+
options,
|
|
1058
|
+
);
|
|
1059
|
+
caveats.push(...retrieval.caveats);
|
|
1060
|
+
for (const skipped of retrieval.skipped) ledger.add(skipped);
|
|
1061
|
+
assertCompletedRetrieval(retrieval, target);
|
|
1062
|
+
const {
|
|
1063
|
+
densePoints,
|
|
1064
|
+
sparsePoints,
|
|
1065
|
+
symbolPage,
|
|
1066
|
+
completedBranches,
|
|
1067
|
+
warnings: branchWarnings,
|
|
1068
|
+
} = retrieval;
|
|
1069
|
+
this.collect(densePoints, pool, rejectedPointIds, ledger, (candidate, point) => {
|
|
1070
|
+
candidate.denseScore = point.denseScore ?? point.score;
|
|
1071
|
+
});
|
|
1072
|
+
this.collect(sparsePoints, pool, rejectedPointIds, ledger, () => undefined);
|
|
1073
|
+
|
|
1074
|
+
// --- Abruf: Symbolzweig ----------------------------------------------
|
|
1075
|
+
let symbolPointCount = 0;
|
|
1076
|
+
let symbolLimitReached = false;
|
|
1077
|
+
if (symbolPage !== undefined) {
|
|
1078
|
+
const symbolPoints = symbolPage.points.map((point) => ({
|
|
1079
|
+
id: point.id,
|
|
1080
|
+
score: 0,
|
|
1081
|
+
payload: point.payload,
|
|
1082
|
+
}));
|
|
1083
|
+
symbolPointCount = symbolPoints.length;
|
|
1084
|
+
symbolLimitReached =
|
|
1085
|
+
symbolPage.nextOffset !== undefined || symbolPoints.length >= query.candidateLimit;
|
|
1086
|
+
this.collect(symbolPoints, pool, rejectedPointIds, ledger, () => undefined);
|
|
1087
|
+
if (symbolPage.nextOffset !== undefined) {
|
|
1088
|
+
ledger.add({
|
|
1089
|
+
subjectKind: "search_candidate",
|
|
1090
|
+
subjectId: `${target.collection}:symbol_scroll`,
|
|
1091
|
+
reason: "excluded_by_configuration",
|
|
1092
|
+
detail: `exact symbol-name scroll stopped at candidateLimit=${query.candidateLimit}; more exact matches exist in "${target.collection}"`,
|
|
1093
|
+
recoverable: true,
|
|
1094
|
+
});
|
|
1095
|
+
}
|
|
1096
|
+
}
|
|
1097
|
+
|
|
1098
|
+
// --- Residuale Filter VOR jeder Rangfolge ------------------------------
|
|
1099
|
+
// Ein Kandidat, der laut Anfrage ungueltig ist, darf weder Pool-IDF noch
|
|
1100
|
+
// Zweigraenge, exakten Intent oder dessen Gewicht beeinflussen. Dadurch
|
|
1101
|
+
// entsprechen alle erklaerten Raenge der Menge, die wirklich antworten
|
|
1102
|
+
// durfte. `passesResidualFilters` wird genau einmal je Kandidat gerufen,
|
|
1103
|
+
// also entsteht auch genau ein Ledger-Eintrag je Ausschluss.
|
|
1104
|
+
const candidates = [...pool.values()].filter((candidate) =>
|
|
1105
|
+
this.passesResidualFilters(
|
|
1106
|
+
candidate,
|
|
1107
|
+
query,
|
|
1108
|
+
extensions,
|
|
1109
|
+
effectiveScope,
|
|
1110
|
+
effectiveRefIds,
|
|
1111
|
+
ledger,
|
|
1112
|
+
),
|
|
1113
|
+
);
|
|
1114
|
+
const eligibleIds = new Set(candidates.map((candidate) => candidate.pointId));
|
|
1115
|
+
const denseRankedIds: string[] = [];
|
|
1116
|
+
const denseSeen = new Set<string>();
|
|
1117
|
+
for (const point of densePoints) {
|
|
1118
|
+
if (!eligibleIds.has(point.id) || denseSeen.has(point.id)) {
|
|
1119
|
+
continue;
|
|
1120
|
+
}
|
|
1121
|
+
const candidate = pool.get(point.id);
|
|
1122
|
+
if (candidate === undefined) {
|
|
1123
|
+
continue;
|
|
1124
|
+
}
|
|
1125
|
+
denseSeen.add(point.id);
|
|
1126
|
+
denseRankedIds.push(point.id);
|
|
1127
|
+
candidate.denseRank = denseRankedIds.length;
|
|
1128
|
+
}
|
|
1129
|
+
|
|
1130
|
+
// --- Rangfolge: lexikalisch (eigenes BM25 ueber den gueltigen Pool) ----
|
|
1131
|
+
const lexicalRanker = new Bm25Ranker(
|
|
1132
|
+
candidates.map((candidate) => ({
|
|
1133
|
+
id: candidate.pointId,
|
|
1134
|
+
text: lexicalDocumentText(candidate.payload),
|
|
1135
|
+
})),
|
|
1136
|
+
);
|
|
1137
|
+
const lexicalRanking =
|
|
1138
|
+
active.lexical && completedBranches.has("sparse") ? lexicalRanker.rank(query.text) : [];
|
|
1139
|
+
const lexicalRankedIds = lexicalRanking.map((entry) => entry.id);
|
|
1140
|
+
for (const entry of lexicalRanking) {
|
|
1141
|
+
const candidate = pool.get(entry.id);
|
|
1142
|
+
if (candidate !== undefined) {
|
|
1143
|
+
candidate.lexicalScore = entry.score;
|
|
1144
|
+
candidate.lexicalRank = entry.rank;
|
|
1145
|
+
}
|
|
1146
|
+
}
|
|
1147
|
+
|
|
1148
|
+
// --- Rangfolge: exakter Symbolname ------------------------------------
|
|
1149
|
+
const symbolMatches: {
|
|
1150
|
+
id: string;
|
|
1151
|
+
strength: number;
|
|
1152
|
+
repoId: string;
|
|
1153
|
+
refId: string;
|
|
1154
|
+
filePath: string;
|
|
1155
|
+
startLine: number;
|
|
1156
|
+
}[] = [];
|
|
1157
|
+
for (const candidate of candidates) {
|
|
1158
|
+
const match = matchSymbolName(candidate.payload, query.text);
|
|
1159
|
+
if (match !== null) {
|
|
1160
|
+
candidate.symbolMatchStrength = match.strength;
|
|
1161
|
+
candidate.symbolMatchEvidence = match.evidence;
|
|
1162
|
+
symbolMatches.push({
|
|
1163
|
+
id: candidate.pointId,
|
|
1164
|
+
strength: match.strength,
|
|
1165
|
+
repoId: candidate.payload.repo_id,
|
|
1166
|
+
refId: candidate.payload.ref_id,
|
|
1167
|
+
filePath: candidate.payload.file_path,
|
|
1168
|
+
startLine: candidate.payload.start_line,
|
|
1169
|
+
});
|
|
1170
|
+
}
|
|
1171
|
+
}
|
|
1172
|
+
symbolMatches.sort((a, b) => {
|
|
1173
|
+
if (a.strength !== b.strength) {
|
|
1174
|
+
return b.strength - a.strength;
|
|
1175
|
+
}
|
|
1176
|
+
const byRepo = compareCodepoints(a.repoId, b.repoId);
|
|
1177
|
+
if (byRepo !== 0) {
|
|
1178
|
+
return byRepo;
|
|
1179
|
+
}
|
|
1180
|
+
const byRef = compareCodepoints(a.refId, b.refId);
|
|
1181
|
+
if (byRef !== 0) {
|
|
1182
|
+
return byRef;
|
|
1183
|
+
}
|
|
1184
|
+
const byPath = compareCodepoints(a.filePath, b.filePath);
|
|
1185
|
+
if (byPath !== 0) {
|
|
1186
|
+
return byPath;
|
|
1187
|
+
}
|
|
1188
|
+
if (a.startLine !== b.startLine) {
|
|
1189
|
+
return a.startLine - b.startLine;
|
|
1190
|
+
}
|
|
1191
|
+
return compareCodepoints(a.id, b.id);
|
|
1192
|
+
});
|
|
1193
|
+
// --- Fusion ------------------------------------------------------------
|
|
1194
|
+
const available = {
|
|
1195
|
+
dense: active.dense && completedBranches.has("dense"),
|
|
1196
|
+
lexical: active.lexical && completedBranches.has("sparse"),
|
|
1197
|
+
symbol: active.symbol && completedBranches.has("symbol"),
|
|
1198
|
+
};
|
|
1199
|
+
const inputs: FusionInput[] = [];
|
|
1200
|
+
if (available.dense) {
|
|
1201
|
+
inputs.push({ branch: "dense", rankedIds: denseRankedIds });
|
|
1202
|
+
}
|
|
1203
|
+
// WP-232: Bei klar deutsch formulierten NATUERLICHEN Fragen kann BM25 ein
|
|
1204
|
+
// systematischer Gegenspieler des multilingualen Embeddings sein. Es wird
|
|
1205
|
+
// aber nur dann weggelassen, wenn sein eigener Spitzenkandidat NICHT in
|
|
1206
|
+
// den dichten Top 10 bestaetigt ist. So bleibt nuetzliche Uebereinstimmung
|
|
1207
|
+
// (d1 und die deutsche INDEX-SHARING-Probe) erhalten, waehrend die
|
|
1208
|
+
// gemessenen Fehlraenge d3/d4/d6 nicht mehr in die Fusion eingehen.
|
|
1209
|
+
const omitLexicalForGermanQuestion =
|
|
1210
|
+
extensions.mode === "hybrid" &&
|
|
1211
|
+
available.dense &&
|
|
1212
|
+
available.lexical &&
|
|
1213
|
+
shouldOmitUncorroboratedGermanLexicalBranch(query.text, denseRankedIds, lexicalRankedIds);
|
|
1214
|
+
if (available.lexical && !omitLexicalForGermanQuestion) {
|
|
1215
|
+
inputs.push({ branch: "lexical", rankedIds: lexicalRankedIds });
|
|
1216
|
+
}
|
|
1217
|
+
if (omitLexicalForGermanQuestion) {
|
|
1218
|
+
caveats.push(
|
|
1219
|
+
`the lexical ranking branch did not participate because its top candidate was not corroborated in the dense top ${GERMAN_LEXICAL_CORROBORATION_DEPTH} for this German natural-language question; dense multilingual ranking remains active (WP-232)`,
|
|
1220
|
+
);
|
|
1221
|
+
}
|
|
1222
|
+
const effectiveActive = {
|
|
1223
|
+
...available,
|
|
1224
|
+
lexical: available.lexical && !omitLexicalForGermanQuestion,
|
|
1225
|
+
};
|
|
1226
|
+
const exactSymbolMatches = symbolMatches.filter((entry) => entry.strength === 1);
|
|
1227
|
+
const exactSymbolIntent =
|
|
1228
|
+
available.symbol && exactSymbolMatches.length > 0 && this.branchWeights.symbol > 0;
|
|
1229
|
+
const symbolRanking = exactSymbolIntent ? exactSymbolMatches : symbolMatches;
|
|
1230
|
+
if (available.symbol) {
|
|
1231
|
+
symbolRanking.forEach((entry, index) => {
|
|
1232
|
+
const candidate = pool.get(entry.id);
|
|
1233
|
+
if (candidate !== undefined) {
|
|
1234
|
+
candidate.symbolRank = index + 1;
|
|
1235
|
+
}
|
|
1236
|
+
});
|
|
1237
|
+
inputs.push({ branch: "symbol", rankedIds: symbolRanking.map((entry) => entry.id) });
|
|
1238
|
+
}
|
|
1239
|
+
const effectiveBranchWeights = exactSymbolIntent
|
|
1240
|
+
? exactSymbolIntentBranchWeights(this.branchWeights, effectiveActive)
|
|
1241
|
+
: this.branchWeights;
|
|
1242
|
+
if (exactSymbolIntent) {
|
|
1243
|
+
caveats.push(
|
|
1244
|
+
`exact symbol-name intent: the symbol RRF branch contains only ${exactSymbolMatches.length} character-identical match(es) and uses weight ${effectiveBranchWeights.symbol}; heuristic boost clamping remains unchanged`,
|
|
1245
|
+
);
|
|
1246
|
+
}
|
|
1247
|
+
const fused = fuseRankings(inputs, effectiveBranchWeights);
|
|
1248
|
+
const fusionById = new Map(fused.map((entry) => [entry.id, entry]));
|
|
1249
|
+
|
|
1250
|
+
// Kandidaten, die in KEINEM aktiven Zweig auftauchen, sind fuer diesen
|
|
1251
|
+
// Modus schlicht nicht relevant. Sie verschwinden trotzdem nicht
|
|
1252
|
+
// stillschweigend (E4).
|
|
1253
|
+
for (const candidate of candidates) {
|
|
1254
|
+
if (!fusionById.has(candidate.pointId)) {
|
|
1255
|
+
ledger.addCandidate({
|
|
1256
|
+
subjectId: candidate.payload.chunk_id,
|
|
1257
|
+
path: candidate.payload.file_path,
|
|
1258
|
+
repoId: candidate.payload.repo_id,
|
|
1259
|
+
refId: candidate.payload.ref_id,
|
|
1260
|
+
reason: "excluded_by_configuration",
|
|
1261
|
+
detail: `no active ranking branch (mode "${extensions.mode}") produced a rank for this candidate`,
|
|
1262
|
+
});
|
|
1263
|
+
}
|
|
1264
|
+
}
|
|
1265
|
+
|
|
1266
|
+
// --- Kandidaten mit Basisrang ------------------------------------------
|
|
1267
|
+
const scored: ScoredCandidate[] = [];
|
|
1268
|
+
for (const entry of fused) {
|
|
1269
|
+
const candidate = pool.get(entry.id);
|
|
1270
|
+
if (candidate === undefined) {
|
|
1271
|
+
continue;
|
|
1272
|
+
}
|
|
1273
|
+
scored.push({ candidate, baseScore: entry.normalized });
|
|
1274
|
+
}
|
|
1275
|
+
|
|
1276
|
+
// --- Dedupe -------------------------------------------------------------
|
|
1277
|
+
const deduped = dedupeCandidates(scored);
|
|
1278
|
+
for (const group of deduped) {
|
|
1279
|
+
for (const absorbed of group.absorbed) {
|
|
1280
|
+
ledger.addCandidate({
|
|
1281
|
+
subjectId: absorbed.candidate.payload.chunk_id,
|
|
1282
|
+
path: absorbed.candidate.payload.file_path,
|
|
1283
|
+
repoId: absorbed.candidate.payload.repo_id,
|
|
1284
|
+
refId: absorbed.candidate.payload.ref_id,
|
|
1285
|
+
reason: "duplicate",
|
|
1286
|
+
detail: `merged into chunk "${group.primary.candidate.payload.chunk_id}" as a related fragment (same metrics_scope_id "${absorbed.candidate.payload.metrics_scope_id}")`,
|
|
1287
|
+
});
|
|
1288
|
+
}
|
|
1289
|
+
}
|
|
1290
|
+
|
|
1291
|
+
// --- Diversifizierung ---------------------------------------------------
|
|
1292
|
+
const penalties = query.diversify
|
|
1293
|
+
? assignDuplicatePenalties(deduped, (entry) => entry.primary.candidate.payload)
|
|
1294
|
+
: deduped.map((entry) => ({ item: entry, penaltyRawValue: 0 }));
|
|
1295
|
+
|
|
1296
|
+
// --- Score-Zerlegung ----------------------------------------------------
|
|
1297
|
+
const scoringContext: ScoringContext = {
|
|
1298
|
+
queryText: query.text,
|
|
1299
|
+
weights: this.config.boostWeights,
|
|
1300
|
+
complexityBias: extensions.complexityBias,
|
|
1301
|
+
repoPriorities: this.deps.repoPriorities ?? {},
|
|
1302
|
+
graphProximity: this.buildProximityMap(deduped),
|
|
1303
|
+
...(this.deps.clock === undefined ? {} : { nowEpochMs: this.deps.clock.nowEpochMs() }),
|
|
1304
|
+
recencyHalfLifeDays: this.deps.recencyHalfLifeDays ?? DEFAULT_RECENCY_HALF_LIFE_DAYS,
|
|
1305
|
+
};
|
|
1306
|
+
|
|
1307
|
+
let hits: SearchHitV1[] = [];
|
|
1308
|
+
for (const penalized of penalties) {
|
|
1309
|
+
const group = penalized.item;
|
|
1310
|
+
const fusion = fusionById.get(group.primary.candidate.pointId) as FusionResult;
|
|
1311
|
+
const breakdown = assembleScoreBreakdown(
|
|
1312
|
+
group.primary.candidate,
|
|
1313
|
+
fusion,
|
|
1314
|
+
scoringContext,
|
|
1315
|
+
penalized.penaltyRawValue,
|
|
1316
|
+
);
|
|
1317
|
+
const minScore = query.minScore;
|
|
1318
|
+
if (minScore !== undefined && breakdown.final < minScore) {
|
|
1319
|
+
ledger.addCandidate({
|
|
1320
|
+
subjectId: group.primary.candidate.payload.chunk_id,
|
|
1321
|
+
path: group.primary.candidate.payload.file_path,
|
|
1322
|
+
repoId: group.primary.candidate.payload.repo_id,
|
|
1323
|
+
refId: group.primary.candidate.payload.ref_id,
|
|
1324
|
+
reason: "excluded_by_configuration",
|
|
1325
|
+
detail: `final score ${breakdown.final} is below the effective minScore ${minScore} (${minScoreOrigin})`,
|
|
1326
|
+
});
|
|
1327
|
+
continue;
|
|
1328
|
+
}
|
|
1329
|
+
hits.push(
|
|
1330
|
+
this.buildHit(
|
|
1331
|
+
group.primary.candidate,
|
|
1332
|
+
breakdown,
|
|
1333
|
+
group.dedupeKey,
|
|
1334
|
+
group.relatedFragments,
|
|
1335
|
+
target,
|
|
1336
|
+
extensions,
|
|
1337
|
+
),
|
|
1338
|
+
);
|
|
1339
|
+
}
|
|
1340
|
+
|
|
1341
|
+
hits = sortHits(hits, extensions.sort);
|
|
1342
|
+
let relevanceWarning: ToolWarningV1 | undefined;
|
|
1343
|
+
// Das Verdikt gehoert an die Gruppe, nicht nur in eine Warnung. Bis Welle
|
|
1344
|
+
// 14 war es ausschliesslich in `relevanceWarning.details` flachgeklopft —
|
|
1345
|
+
// deshalb lasen zehn Stellen im Repository die Enthaltung aus der
|
|
1346
|
+
// Listenlaenge statt aus der Entscheidung.
|
|
1347
|
+
let relevanceVerdict: SearchRelevanceVerdictV1 | undefined;
|
|
1348
|
+
const relevanceBinding = this.relevancePolicies.get(target.embeddingProfileId);
|
|
1349
|
+
if (relevanceBinding === undefined && this.relevancePolicies.size > 0) {
|
|
1350
|
+
const configuredEmbeddingProfileIds = sortByCodepoints([...this.relevancePolicies.keys()]);
|
|
1351
|
+
const detail = `relevance policy was not evaluated because active embedding profile "${target.embeddingProfileId}" has no configured binding; configured profile keys: ${configuredEmbeddingProfileIds.join(", ")}; results were left unchanged`;
|
|
1352
|
+
caveats.push(detail);
|
|
1353
|
+
relevanceWarning = {
|
|
1354
|
+
code: RELEVANCE_POLICY_NOT_EVALUATED_WARNING_CODE,
|
|
1355
|
+
message: detail,
|
|
1356
|
+
details: {
|
|
1357
|
+
status: "not_evaluated",
|
|
1358
|
+
basis: "policy_profile_mismatch",
|
|
1359
|
+
actualEmbeddingProfileId: target.embeddingProfileId,
|
|
1360
|
+
configuredEmbeddingProfileIds,
|
|
1361
|
+
},
|
|
1362
|
+
};
|
|
1363
|
+
relevanceVerdict = {
|
|
1364
|
+
abstention: extensions.abstention,
|
|
1365
|
+
status: "not_evaluated",
|
|
1366
|
+
basis: "policy_profile_mismatch",
|
|
1367
|
+
};
|
|
1368
|
+
} else if (relevanceBinding !== undefined) {
|
|
1369
|
+
const exactSymbolIds = new Set(exactSymbolMatches.map((exact) => exact.id));
|
|
1370
|
+
const prerequisiteFailure = relevancePrerequisiteFailure({
|
|
1371
|
+
binding: relevanceBinding,
|
|
1372
|
+
embeddingProfileId: target.embeddingProfileId,
|
|
1373
|
+
workspaceId: query.workspaceId,
|
|
1374
|
+
collection: target.collection,
|
|
1375
|
+
completedBranches,
|
|
1376
|
+
exactSymbolEvidenceAvailable: active.symbol && exactSymbolIds.size > 0,
|
|
1377
|
+
abstention: extensions.abstention,
|
|
1378
|
+
});
|
|
1379
|
+
if (prerequisiteFailure !== undefined) {
|
|
1380
|
+
caveats.push(prerequisiteFailure.detail);
|
|
1381
|
+
relevanceWarning = prerequisiteFailure.warning;
|
|
1382
|
+
relevanceVerdict = prerequisiteFailure.verdict;
|
|
1383
|
+
} else {
|
|
1384
|
+
// Die Schwellen wurden gegen das feste Praefix von jeweils Rang 50
|
|
1385
|
+
// des dichten UND spaerlichen Abrufzweigs kalibriert. Der
|
|
1386
|
+
// Ranking-Pool darf groesser sein, die Abwesenheitsevidenz aber nicht:
|
|
1387
|
+
// sonst veraendert candidateLimit die Pool-IDF und damit bei
|
|
1388
|
+
// identischer Anfrage die Ja/Nein-Antwort. Der Symbolabruf gehoert
|
|
1389
|
+
// nicht zur IDF-Kalibrierung; ein exakter Name bleibt eigene direkte
|
|
1390
|
+
// Evidenz.
|
|
1391
|
+
// Residual ungueltige IDs sind bereits aus `eligibleIds` entfernt und
|
|
1392
|
+
// werden innerhalb des festen Praefixes bewusst NICHT durch tiefere
|
|
1393
|
+
// Kandidaten ersetzt.
|
|
1394
|
+
const evidencePool = selectFixedRelevanceEvidencePool(
|
|
1395
|
+
[
|
|
1396
|
+
{ branch: "dense", candidateIds: densePoints.map((point) => point.id) },
|
|
1397
|
+
{ branch: "sparse", candidateIds: sparsePoints.map((point) => point.id) },
|
|
1398
|
+
],
|
|
1399
|
+
eligibleIds,
|
|
1400
|
+
relevanceBinding.policy.denseReferenceRank,
|
|
1401
|
+
);
|
|
1402
|
+
const evidenceIds = [...evidencePool.candidateIds];
|
|
1403
|
+
const evidenceIdSet = new Set(evidenceIds);
|
|
1404
|
+
// Der exakte Symbolpfad bleibt direkte Evidenz, auch wenn sehr viele
|
|
1405
|
+
// gleichartige Scroll-Treffer vor ihm liegen. Nur residual gueltige
|
|
1406
|
+
// `symbolMatches` koennen hier auftauchen.
|
|
1407
|
+
if (active.symbol) {
|
|
1408
|
+
for (const exactSymbolId of exactSymbolIds) {
|
|
1409
|
+
if (!evidenceIdSet.has(exactSymbolId)) {
|
|
1410
|
+
evidenceIdSet.add(exactSymbolId);
|
|
1411
|
+
evidenceIds.push(exactSymbolId);
|
|
1412
|
+
}
|
|
1413
|
+
}
|
|
1414
|
+
}
|
|
1415
|
+
const denseEvidenceIds = new Set(evidencePool.denseCandidateIds);
|
|
1416
|
+
const relevanceDecision = decideResultRelevance(
|
|
1417
|
+
query.text,
|
|
1418
|
+
evidenceIds.flatMap((candidateId) => {
|
|
1419
|
+
const candidate = pool.get(candidateId);
|
|
1420
|
+
if (candidate === undefined) {
|
|
1421
|
+
return [];
|
|
1422
|
+
}
|
|
1423
|
+
return [
|
|
1424
|
+
{
|
|
1425
|
+
...(denseEvidenceIds.has(candidateId) && candidate.denseScore !== undefined
|
|
1426
|
+
? { denseScore: candidate.denseScore }
|
|
1427
|
+
: {}),
|
|
1428
|
+
lexicalText: lexicalDocumentText(candidate.payload),
|
|
1429
|
+
exactSymbolMatch: active.symbol && exactSymbolIds.has(candidate.pointId),
|
|
1430
|
+
},
|
|
1431
|
+
];
|
|
1432
|
+
}),
|
|
1433
|
+
relevanceBinding.policy,
|
|
1434
|
+
);
|
|
1435
|
+
relevanceVerdict = {
|
|
1436
|
+
abstention: extensions.abstention,
|
|
1437
|
+
status: relevanceDecision.status,
|
|
1438
|
+
basis: relevanceDecision.basis,
|
|
1439
|
+
calibrationId: relevanceBinding.policy.calibrationId,
|
|
1440
|
+
};
|
|
1441
|
+
if (relevanceDecision.status === "rejected") {
|
|
1442
|
+
const observedReferenceRank = relevanceDecision.denseReferenceRankObserved as number;
|
|
1443
|
+
const detail = `candidate rejected by relevance policy "${relevanceBinding.policy.calibrationId}": top dense ${relevanceDecision.topDenseScore ?? "none"} (minimum ${relevanceBinding.policy.minimumDenseScore}), spread between rank 1 and observed rank ${observedReferenceRank} ${relevanceDecision.denseSpread ?? "unavailable"} (minimum ${relevanceBinding.policy.minimumDenseSpread}), fixed per-branch candidate-evidence pool IDF coverage ${relevanceDecision.candidateIdfCoverage} across ${relevanceDecision.candidateEvidenceCount} candidates (minimum ${relevanceBinding.policy.minimumCandidateIdfCoverage}); all three non-symbol conditions are required`;
|
|
1444
|
+
// NUR `strict` verwirft. Unter `evidence` bleiben die Treffer
|
|
1445
|
+
// stehen und tragen das Verdikt mit — und dann darf auch KEIN
|
|
1446
|
+
// Ledger-Eintrag geschrieben werden: ein Eintrag ueber einen
|
|
1447
|
+
// Gegenstand, der im Ergebnis steht, ist in diesem Repository ein
|
|
1448
|
+
// dokumentierter E4-Defekt (WP-142).
|
|
1449
|
+
if (extensions.abstention === "strict") {
|
|
1450
|
+
for (const dropped of hits) {
|
|
1451
|
+
ledger.addCandidate({
|
|
1452
|
+
subjectId: dropped.chunkId,
|
|
1453
|
+
path: dropped.filePath,
|
|
1454
|
+
repoId: dropped.repoId,
|
|
1455
|
+
refId: dropped.refId,
|
|
1456
|
+
// NICHT `excluded_by_configuration`: es wurde nichts
|
|
1457
|
+
// wegkonfiguriert, und der Kandidat ist nicht durch Filter
|
|
1458
|
+
// oder Budget gefallen. Ohne eigenen Grund liesse sich eine
|
|
1459
|
+
// Enthaltung nicht von einem Score-Schwellwert trennen (E4).
|
|
1460
|
+
reason: "relevance_policy_rejected",
|
|
1461
|
+
detail,
|
|
1462
|
+
});
|
|
1463
|
+
}
|
|
1464
|
+
hits = [];
|
|
1465
|
+
}
|
|
1466
|
+
// Vorbehalt und Warnung sind in BEIDEN Verhalten byte-identisch:
|
|
1467
|
+
// das Urteil aendert sich nicht, nur was mit den Treffern geschieht.
|
|
1468
|
+
caveats.push(detail);
|
|
1469
|
+
relevanceWarning = {
|
|
1470
|
+
code: NO_SUFFICIENT_MATCH_WARNING_CODE,
|
|
1471
|
+
message: `No sufficiently relevant result for embedding profile "${target.embeddingProfileId}": without an exact symbol name, the dense top score, gap to the fixed reference rank and IDF-weighted query coverage must jointly support the verdict within the fixed evidence pool`,
|
|
1472
|
+
details: {
|
|
1473
|
+
embeddingProfileId: target.embeddingProfileId,
|
|
1474
|
+
collection: target.collection,
|
|
1475
|
+
...relevanceDecision,
|
|
1476
|
+
},
|
|
1477
|
+
};
|
|
1478
|
+
} else if (relevanceDecision.status === "not_evaluated") {
|
|
1479
|
+
const detail = `relevance policy "${relevanceBinding.policy.calibrationId}" was not evaluated: the deepest dense rank actually observed was ${relevanceDecision.deepestDenseRankObserved}, so calibrated reference rank ${relevanceBinding.policy.denseReferenceRank} was not observed; results were left unchanged`;
|
|
1480
|
+
caveats.push(detail);
|
|
1481
|
+
relevanceWarning = {
|
|
1482
|
+
code: RELEVANCE_POLICY_NOT_EVALUATED_WARNING_CODE,
|
|
1483
|
+
message: detail,
|
|
1484
|
+
details: {
|
|
1485
|
+
embeddingProfileId: target.embeddingProfileId,
|
|
1486
|
+
collection: target.collection,
|
|
1487
|
+
...relevanceDecision,
|
|
1488
|
+
},
|
|
1489
|
+
};
|
|
1490
|
+
}
|
|
1491
|
+
}
|
|
1492
|
+
}
|
|
1493
|
+
const qualifiedHitCount = hits.length;
|
|
1494
|
+
const topScoreGap =
|
|
1495
|
+
hits.length < 2
|
|
1496
|
+
? null
|
|
1497
|
+
: (hits[0] as SearchHitV1).score.final - (hits[1] as SearchHitV1).score.final;
|
|
1498
|
+
if (hits.length > query.limit) {
|
|
1499
|
+
for (const dropped of hits.slice(query.limit)) {
|
|
1500
|
+
ledger.addCandidate({
|
|
1501
|
+
subjectId: dropped.chunkId,
|
|
1502
|
+
path: dropped.filePath,
|
|
1503
|
+
repoId: dropped.repoId,
|
|
1504
|
+
refId: dropped.refId,
|
|
1505
|
+
reason: "excluded_by_configuration",
|
|
1506
|
+
detail: `final score ${dropped.score.final} ranked below the requested limit of ${query.limit} results`,
|
|
1507
|
+
});
|
|
1508
|
+
}
|
|
1509
|
+
hits = hits.slice(0, query.limit);
|
|
1510
|
+
}
|
|
1511
|
+
hits = assignRanks(hits);
|
|
1512
|
+
|
|
1513
|
+
const uniqueEligibleFileCount = new Set(
|
|
1514
|
+
candidates.map(
|
|
1515
|
+
(candidate) =>
|
|
1516
|
+
`${candidate.payload.repo_id}\u001f${candidate.payload.ref_id}\u001f${candidate.payload.file_path}`,
|
|
1517
|
+
),
|
|
1518
|
+
).size;
|
|
1519
|
+
const uniqueEligibleSymbolCount = new Set(
|
|
1520
|
+
candidates.flatMap((candidate) =>
|
|
1521
|
+
candidate.payload.symbol_id === undefined
|
|
1522
|
+
? []
|
|
1523
|
+
: [
|
|
1524
|
+
`${candidate.payload.repo_id}\u001f${candidate.payload.ref_id}\u001f${candidate.payload.symbol_id}`,
|
|
1525
|
+
],
|
|
1526
|
+
),
|
|
1527
|
+
).size;
|
|
1528
|
+
const eligibleFileDispersion =
|
|
1529
|
+
candidates.length <= 1 ? 0 : (uniqueEligibleFileCount - 1) / (candidates.length - 1);
|
|
1530
|
+
const {
|
|
1531
|
+
contexts: scopedIndexContexts,
|
|
1532
|
+
corpusRevisionFingerprint,
|
|
1533
|
+
warning: indexContextWarning,
|
|
1534
|
+
} = this.indexContextProvenance(
|
|
1535
|
+
target,
|
|
1536
|
+
built.scopeRepoIds,
|
|
1537
|
+
effectiveRefIds,
|
|
1538
|
+
candidates,
|
|
1539
|
+
query.filters.refIds.length === 0,
|
|
1540
|
+
);
|
|
1541
|
+
// WP-189: Der Vertrag von `SearchHitGroupV1.repoIds` lautet „Repositories,
|
|
1542
|
+
// die zu dieser Gruppe BEITRAGEN". Hier stand der erlaubte Bereich — also
|
|
1543
|
+
// eine Aussage ueber die Anfrage, ausgegeben als Aussage ueber das
|
|
1544
|
+
// Ergebnis. Bei drei erlaubten und einem liefernden Repository nannte die
|
|
1545
|
+
// Gruppe drei Herkuenfte, von denen zwei nichts beigetragen hatten.
|
|
1546
|
+
const contributing = uniqueSortedByCodepoints(hits.map((hit) => hit.repoId));
|
|
1547
|
+
return {
|
|
1548
|
+
group: {
|
|
1549
|
+
embeddingProfileId: target.embeddingProfileId,
|
|
1550
|
+
// A completed local state proves provenance for an empty result, but
|
|
1551
|
+
// must never lend its generation to a returned legacy candidate that
|
|
1552
|
+
// did not carry `generation_id` itself. The display field may retain
|
|
1553
|
+
// the legacy dependency fallback; corpus provenance below never does.
|
|
1554
|
+
indexGenerations: uniqueSortedByCodepoints(
|
|
1555
|
+
candidates.length === 0
|
|
1556
|
+
? scopedIndexContexts.map((context) => context.generationId)
|
|
1557
|
+
: candidates.map((candidate) => this.indexGenerationOf(candidate)),
|
|
1558
|
+
),
|
|
1559
|
+
...(corpusRevisionFingerprint === undefined ? {} : { corpusRevisionFingerprint }),
|
|
1560
|
+
repoIds: contributing,
|
|
1561
|
+
hits,
|
|
1562
|
+
rankNormalized: false,
|
|
1563
|
+
caveats,
|
|
1564
|
+
...(relevanceVerdict === undefined ? {} : { relevanceDecision: relevanceVerdict }),
|
|
1565
|
+
retrieval: {
|
|
1566
|
+
branchCandidateCounts: {
|
|
1567
|
+
dense: densePoints.length,
|
|
1568
|
+
sparse: sparsePoints.length,
|
|
1569
|
+
symbol: symbolPointCount,
|
|
1570
|
+
},
|
|
1571
|
+
uniqueRetrievedCandidateCount: pool.size,
|
|
1572
|
+
eligibleCandidateCount: candidates.length,
|
|
1573
|
+
rankedCandidateCount: fused.length,
|
|
1574
|
+
deduplicatedCandidateCount: deduped.length,
|
|
1575
|
+
qualifiedHitCount,
|
|
1576
|
+
returnedHitCount: hits.length,
|
|
1577
|
+
uniqueEligibleFileCount,
|
|
1578
|
+
uniqueEligibleSymbolCount,
|
|
1579
|
+
eligibleFileDispersion,
|
|
1580
|
+
topScoreGap,
|
|
1581
|
+
candidateLimitReached:
|
|
1582
|
+
densePoints.length >= query.candidateLimit ||
|
|
1583
|
+
sparsePoints.length >= query.candidateLimit ||
|
|
1584
|
+
symbolLimitReached,
|
|
1585
|
+
},
|
|
1586
|
+
},
|
|
1587
|
+
refIds: new Set(hits.map((hit) => hit.refId)),
|
|
1588
|
+
...(relevanceWarning === undefined ? {} : { relevanceWarning }),
|
|
1589
|
+
...(indexContextWarning === undefined ? {} : { indexContextWarning }),
|
|
1590
|
+
...(branchWarnings.length === 0 ? {} : { branchWarnings }),
|
|
1591
|
+
};
|
|
1592
|
+
}
|
|
1593
|
+
|
|
1594
|
+
/** Uebernimmt Speicherpunkte in den Kandidatenpool; unbrauchbare landen im Ledger. */
|
|
1595
|
+
private collect(
|
|
1596
|
+
points: readonly ScoredPointV1[],
|
|
1597
|
+
pool: Map<string, SearchCandidate>,
|
|
1598
|
+
rejectedPointIds: Set<string>,
|
|
1599
|
+
ledger: SearchSkipLedger,
|
|
1600
|
+
onCandidate: (candidate: SearchCandidate, point: ScoredPointV1) => void,
|
|
1601
|
+
): void {
|
|
1602
|
+
points.forEach((point) => {
|
|
1603
|
+
if (rejectedPointIds.has(point.id)) {
|
|
1604
|
+
return;
|
|
1605
|
+
}
|
|
1606
|
+
const existing = pool.get(point.id);
|
|
1607
|
+
if (existing !== undefined) {
|
|
1608
|
+
onCandidate(existing, point);
|
|
1609
|
+
return;
|
|
1610
|
+
}
|
|
1611
|
+
const result = candidateFromPoint(point);
|
|
1612
|
+
if (!result.ok) {
|
|
1613
|
+
rejectedPointIds.add(point.id);
|
|
1614
|
+
ledger.addCandidate({
|
|
1615
|
+
subjectId: point.id,
|
|
1616
|
+
reason: "missing_index_entry",
|
|
1617
|
+
detail: `point "${point.id}" could not be read as a code chunk: ${result.reason}`,
|
|
1618
|
+
});
|
|
1619
|
+
return;
|
|
1620
|
+
}
|
|
1621
|
+
pool.set(point.id, result.candidate);
|
|
1622
|
+
onCandidate(result.candidate, point);
|
|
1623
|
+
});
|
|
1624
|
+
}
|
|
1625
|
+
|
|
1626
|
+
/** Pfad-Globs und Findability — die Filter, die der Speicher nicht kennt. */
|
|
1627
|
+
private passesResidualFilters(
|
|
1628
|
+
candidate: SearchCandidate,
|
|
1629
|
+
query: SearchQueryV1,
|
|
1630
|
+
extensions: SearchQueryExtensionsV1,
|
|
1631
|
+
scope: SearchScopeV1,
|
|
1632
|
+
effectiveRefIds: readonly string[],
|
|
1633
|
+
ledger: SearchSkipLedger,
|
|
1634
|
+
): boolean {
|
|
1635
|
+
const { payload } = candidate;
|
|
1636
|
+
// Der Geltungsbereich steht bereits im Speicherfilter. Er wird hier ein
|
|
1637
|
+
// ZWEITES Mal geprueft, weil ein Speicher, der den Filter nicht oder
|
|
1638
|
+
// anders auswertet, sonst fremden Inhalt als eigenen ausliefern wuerde —
|
|
1639
|
+
// und das waere kein ungenaues Ranking, sondern ein Datenleck.
|
|
1640
|
+
if (payload.workspace_id !== scope.workspaceId) {
|
|
1641
|
+
ledger.addCandidate({
|
|
1642
|
+
subjectId: payload.chunk_id,
|
|
1643
|
+
path: payload.file_path,
|
|
1644
|
+
repoId: payload.repo_id,
|
|
1645
|
+
refId: payload.ref_id,
|
|
1646
|
+
reason: "excluded_by_configuration",
|
|
1647
|
+
detail: `candidate belongs to workspace "${payload.workspace_id}", but the query is scoped to workspace "${scope.workspaceId}"`,
|
|
1648
|
+
});
|
|
1649
|
+
return false;
|
|
1650
|
+
}
|
|
1651
|
+
const targetRepoIds = scope.targetRepoIds ?? [];
|
|
1652
|
+
if (targetRepoIds.length > 0 && !targetRepoIds.includes(payload.repo_id)) {
|
|
1653
|
+
ledger.addCandidate({
|
|
1654
|
+
subjectId: payload.chunk_id,
|
|
1655
|
+
path: payload.file_path,
|
|
1656
|
+
repoId: payload.repo_id,
|
|
1657
|
+
refId: payload.ref_id,
|
|
1658
|
+
reason: "excluded_by_configuration",
|
|
1659
|
+
detail: `candidate belongs to repository "${payload.repo_id}", which is not one of this target's repositories [${sortByCodepoints([...targetRepoIds]).join(", ")}]`,
|
|
1660
|
+
});
|
|
1661
|
+
return false;
|
|
1662
|
+
}
|
|
1663
|
+
if (effectiveRefIds.length > 0 && !effectiveRefIds.includes(payload.ref_id)) {
|
|
1664
|
+
ledger.addCandidate({
|
|
1665
|
+
subjectId: payload.chunk_id,
|
|
1666
|
+
path: payload.file_path,
|
|
1667
|
+
repoId: payload.repo_id,
|
|
1668
|
+
refId: payload.ref_id,
|
|
1669
|
+
reason: "excluded_by_configuration",
|
|
1670
|
+
detail: `candidate belongs to ref "${payload.ref_id}", which is not one of the effective refs [${sortByCodepoints([...effectiveRefIds]).join(", ")}]`,
|
|
1671
|
+
});
|
|
1672
|
+
return false;
|
|
1673
|
+
}
|
|
1674
|
+
if (!isPathSelected(payload.file_path, query.filters.pathGlobs, extensions.pathExcludes)) {
|
|
1675
|
+
ledger.addCandidate({
|
|
1676
|
+
subjectId: payload.chunk_id,
|
|
1677
|
+
path: payload.file_path,
|
|
1678
|
+
repoId: payload.repo_id,
|
|
1679
|
+
refId: payload.ref_id,
|
|
1680
|
+
reason: "excluded_by_configuration",
|
|
1681
|
+
detail: `path "${payload.file_path}" does not pass the include/exclude globs (include: [${query.filters.pathGlobs.join(", ")}], exclude: [${extensions.pathExcludes.join(", ")}])`,
|
|
1682
|
+
});
|
|
1683
|
+
return false;
|
|
1684
|
+
}
|
|
1685
|
+
const findabilityFilter = query.filters.findability;
|
|
1686
|
+
const lookup = this.deps.findability;
|
|
1687
|
+
if (findabilityFilter !== undefined && lookup !== undefined) {
|
|
1688
|
+
const summary = payload.symbol_id === undefined ? undefined : lookup(payload.symbol_id);
|
|
1689
|
+
if (summary === undefined) {
|
|
1690
|
+
ledger.addCandidate({
|
|
1691
|
+
subjectId: payload.chunk_id,
|
|
1692
|
+
path: payload.file_path,
|
|
1693
|
+
repoId: payload.repo_id,
|
|
1694
|
+
refId: payload.ref_id,
|
|
1695
|
+
reason: "missing_index_entry",
|
|
1696
|
+
detail:
|
|
1697
|
+
"a findability filter was requested but no findability record exists for this symbol",
|
|
1698
|
+
recoverable: true,
|
|
1699
|
+
});
|
|
1700
|
+
return false;
|
|
1701
|
+
}
|
|
1702
|
+
const { minScore, maxScore, causes } = findabilityFilter;
|
|
1703
|
+
const failsMin = minScore !== undefined && summary.score < minScore;
|
|
1704
|
+
const failsMax = maxScore !== undefined && summary.score > maxScore;
|
|
1705
|
+
const failsCause =
|
|
1706
|
+
causes !== undefined &&
|
|
1707
|
+
causes.length > 0 &&
|
|
1708
|
+
(summary.primaryCause === undefined || !causes.includes(summary.primaryCause));
|
|
1709
|
+
if (failsMin || failsMax || failsCause) {
|
|
1710
|
+
ledger.addCandidate({
|
|
1711
|
+
subjectId: payload.chunk_id,
|
|
1712
|
+
path: payload.file_path,
|
|
1713
|
+
repoId: payload.repo_id,
|
|
1714
|
+
refId: payload.ref_id,
|
|
1715
|
+
reason: "excluded_by_configuration",
|
|
1716
|
+
detail: `findability score ${summary.score} / cause "${summary.primaryCause ?? "none"}" does not satisfy the requested findability filter`,
|
|
1717
|
+
});
|
|
1718
|
+
return false;
|
|
1719
|
+
}
|
|
1720
|
+
}
|
|
1721
|
+
return true;
|
|
1722
|
+
}
|
|
1723
|
+
|
|
1724
|
+
private buildProximityMap(
|
|
1725
|
+
deduped: readonly { primary: ScoredCandidate }[],
|
|
1726
|
+
): ReadonlyMap<string, number> {
|
|
1727
|
+
const map = new Map<string, number>();
|
|
1728
|
+
const lookup = this.deps.graphProximity;
|
|
1729
|
+
if (lookup === undefined) {
|
|
1730
|
+
return map;
|
|
1731
|
+
}
|
|
1732
|
+
for (const group of deduped) {
|
|
1733
|
+
const symbolId = group.primary.candidate.payload.symbol_id;
|
|
1734
|
+
if (symbolId === undefined || map.has(symbolId)) {
|
|
1735
|
+
continue;
|
|
1736
|
+
}
|
|
1737
|
+
const value = lookup(symbolId);
|
|
1738
|
+
if (value !== undefined) {
|
|
1739
|
+
map.set(symbolId, value);
|
|
1740
|
+
}
|
|
1741
|
+
}
|
|
1742
|
+
return map;
|
|
1743
|
+
}
|
|
1744
|
+
|
|
1745
|
+
private buildHit(
|
|
1746
|
+
candidate: SearchCandidate,
|
|
1747
|
+
score: SearchHitV1["score"],
|
|
1748
|
+
dedupeKey: string,
|
|
1749
|
+
relatedFragments: SearchHitV1["relatedFragments"],
|
|
1750
|
+
target: ProfileTargetV1,
|
|
1751
|
+
extensions: SearchQueryExtensionsV1,
|
|
1752
|
+
): SearchHitV1 {
|
|
1753
|
+
const { payload } = candidate;
|
|
1754
|
+
const findabilitySummary =
|
|
1755
|
+
payload.symbol_id === undefined ? undefined : this.deps.findability?.(payload.symbol_id);
|
|
1756
|
+
const contentRequested = extensions.includeContent;
|
|
1757
|
+
const hasContent = payload.content !== "";
|
|
1758
|
+
const complexity = this.complexityOf(candidate);
|
|
1759
|
+
return {
|
|
1760
|
+
schemaVersion: 1,
|
|
1761
|
+
// Der endgueltige Rang wird nach der Sortierung gesetzt (`assignRanks`);
|
|
1762
|
+
// bis dahin ist 0 ein bewusst ungueltiger Platzhalter, kein Rang 1.
|
|
1763
|
+
rank: 0,
|
|
1764
|
+
repoId: payload.repo_id,
|
|
1765
|
+
repoLabel: payload.repo_label,
|
|
1766
|
+
refId: payload.ref_id,
|
|
1767
|
+
refName: payload.ref_name,
|
|
1768
|
+
...(payload.commit_sha === undefined ? {} : { commitSha: payload.commit_sha }),
|
|
1769
|
+
fileId: payload.file_id,
|
|
1770
|
+
filePath: payload.file_path,
|
|
1771
|
+
range: {
|
|
1772
|
+
startLine: payload.start_line,
|
|
1773
|
+
endLine: payload.end_line,
|
|
1774
|
+
startByte: payload.start_byte,
|
|
1775
|
+
endByte: payload.end_byte,
|
|
1776
|
+
},
|
|
1777
|
+
language: payload.language,
|
|
1778
|
+
parseQuality: payload.parse_quality,
|
|
1779
|
+
chunkId: payload.chunk_id,
|
|
1780
|
+
chunkKind: payload.chunk_kind,
|
|
1781
|
+
...(payload.fragment_index === undefined ? {} : { fragmentIndex: payload.fragment_index }),
|
|
1782
|
+
...(payload.fragment_count === undefined ? {} : { fragmentCount: payload.fragment_count }),
|
|
1783
|
+
...(payload.symbol_id === undefined ? {} : { symbolId: payload.symbol_id }),
|
|
1784
|
+
...(payload.symbol_name === undefined ? {} : { symbolName: payload.symbol_name }),
|
|
1785
|
+
...(payload.symbol_fqn === undefined ? {} : { symbolFqn: payload.symbol_fqn }),
|
|
1786
|
+
...(payload.symbol_kind === undefined ? {} : { symbolKind: payload.symbol_kind }),
|
|
1787
|
+
...(payload.signature === undefined ? {} : { signature: payload.signature }),
|
|
1788
|
+
score,
|
|
1789
|
+
dedupeKey,
|
|
1790
|
+
relatedFragments,
|
|
1791
|
+
embeddingProfileId: target.embeddingProfileId,
|
|
1792
|
+
indexGeneration: this.indexGenerationOf(candidate),
|
|
1793
|
+
...(complexity === undefined ? {} : { complexity }),
|
|
1794
|
+
...(findabilitySummary === undefined ? {} : { findability: findabilitySummary }),
|
|
1795
|
+
...(contentRequested && hasContent ? { content: payload.content } : {}),
|
|
1796
|
+
...(contentRequested && !hasContent
|
|
1797
|
+
? {
|
|
1798
|
+
contentOmittedReason:
|
|
1799
|
+
"the stored chunk payload carries no content (indexed without content, or content stripped at index time)",
|
|
1800
|
+
}
|
|
1801
|
+
: {}),
|
|
1802
|
+
...(candidate.substitutedFields.length === 0
|
|
1803
|
+
? {}
|
|
1804
|
+
: { incompletePayloadFields: [...candidate.substitutedFields] }),
|
|
1805
|
+
};
|
|
1806
|
+
}
|
|
1807
|
+
|
|
1808
|
+
private indexGenerationOf(candidate: SearchCandidate): string {
|
|
1809
|
+
return this.rawIndexGenerationOf(candidate) ?? this.deps.indexGeneration ?? "unknown";
|
|
1810
|
+
}
|
|
1811
|
+
|
|
1812
|
+
/** Persisted candidate provenance; display fallbacks are deliberately excluded. */
|
|
1813
|
+
private rawIndexGenerationOf(candidate: SearchCandidate): string | undefined {
|
|
1814
|
+
return readRawString(candidate.raw, "generation_id");
|
|
1815
|
+
}
|
|
1816
|
+
|
|
1817
|
+
private indexContextProvenance(
|
|
1818
|
+
target: ProfileTargetV1,
|
|
1819
|
+
scopeRepoIds: readonly string[],
|
|
1820
|
+
scopedRefIds: readonly string[],
|
|
1821
|
+
candidates: readonly SearchCandidate[],
|
|
1822
|
+
refScopeWasImplicit: boolean,
|
|
1823
|
+
): {
|
|
1824
|
+
contexts: NonNullable<ProfileTargetV1["indexContexts"]>;
|
|
1825
|
+
corpusRevisionFingerprint?: string;
|
|
1826
|
+
warning?: ToolWarningV1;
|
|
1827
|
+
} {
|
|
1828
|
+
const contexts = (target.indexContexts ?? []).filter(
|
|
1829
|
+
(context) =>
|
|
1830
|
+
scopeRepoIds.includes(context.repoId) &&
|
|
1831
|
+
(scopedRefIds.length === 0 || scopedRefIds.includes(context.refId)),
|
|
1832
|
+
);
|
|
1833
|
+
const completedContextExists = (repoId: string, refId?: string, generationId?: string) =>
|
|
1834
|
+
contexts.some(
|
|
1835
|
+
(context) =>
|
|
1836
|
+
context.outcome === "completed" &&
|
|
1837
|
+
context.repoId === repoId &&
|
|
1838
|
+
(refId === undefined || context.refId === refId) &&
|
|
1839
|
+
(generationId === undefined || context.generationId === generationId),
|
|
1840
|
+
);
|
|
1841
|
+
// An implicit scope represents "the recorded current ref(s) of every
|
|
1842
|
+
// selected repository", so every repository needs a completed context.
|
|
1843
|
+
// Explicit repo-qualified ref IDs instead form real repo/ref pairs. They
|
|
1844
|
+
// are not a Cartesian product: repo-a:main does not need to exist in
|
|
1845
|
+
// repo-b merely because both repos and both refs were selected.
|
|
1846
|
+
const hasCompleteScopeCoverage =
|
|
1847
|
+
contexts.length > 0 &&
|
|
1848
|
+
(refScopeWasImplicit
|
|
1849
|
+
? scopeRepoIds.every((repoId) => completedContextExists(repoId))
|
|
1850
|
+
: scopedRefIds.every((refId) =>
|
|
1851
|
+
contexts.some((context) => context.outcome === "completed" && context.refId === refId),
|
|
1852
|
+
));
|
|
1853
|
+
const candidatesHaveCompletedContext = candidates.every((candidate) => {
|
|
1854
|
+
const rawGenerationId = this.rawIndexGenerationOf(candidate);
|
|
1855
|
+
return (
|
|
1856
|
+
rawGenerationId !== undefined &&
|
|
1857
|
+
completedContextExists(candidate.payload.repo_id, candidate.payload.ref_id, rawGenerationId)
|
|
1858
|
+
);
|
|
1859
|
+
});
|
|
1860
|
+
const incomplete =
|
|
1861
|
+
target.indexContexts !== undefined &&
|
|
1862
|
+
(!hasCompleteScopeCoverage ||
|
|
1863
|
+
!candidatesHaveCompletedContext ||
|
|
1864
|
+
contexts.some(
|
|
1865
|
+
(context) =>
|
|
1866
|
+
context.outcome !== "completed" || context.corpusRevisionFingerprint === undefined,
|
|
1867
|
+
));
|
|
1868
|
+
if (!incomplete) {
|
|
1869
|
+
if (target.indexContexts === undefined) return { contexts };
|
|
1870
|
+
const corpusContexts = contexts
|
|
1871
|
+
.map((context) => ({
|
|
1872
|
+
repoId: context.repoId,
|
|
1873
|
+
refId: context.refId,
|
|
1874
|
+
generationId: context.generationId,
|
|
1875
|
+
corpusRevisionFingerprint: context.corpusRevisionFingerprint as string,
|
|
1876
|
+
}))
|
|
1877
|
+
.sort((left, right) => compareCodepoints(canonicalJson(left), canonicalJson(right)));
|
|
1878
|
+
return {
|
|
1879
|
+
contexts,
|
|
1880
|
+
corpusRevisionFingerprint: sha256Hex(
|
|
1881
|
+
canonicalJson({
|
|
1882
|
+
schemaVersion: 1,
|
|
1883
|
+
embeddingProfileId: target.embeddingProfileId,
|
|
1884
|
+
contexts: corpusContexts,
|
|
1885
|
+
}),
|
|
1886
|
+
),
|
|
1887
|
+
};
|
|
1888
|
+
}
|
|
1889
|
+
return {
|
|
1890
|
+
contexts,
|
|
1891
|
+
warning: {
|
|
1892
|
+
code: INDEX_CONTEXT_INCOMPLETE_WARNING_CODE,
|
|
1893
|
+
message:
|
|
1894
|
+
`index completion could not be verified for embedding profile "${target.embeddingProfileId}"; ` +
|
|
1895
|
+
"hits remain usable, but this search is not healthy product-complexity evidence",
|
|
1896
|
+
details: {
|
|
1897
|
+
embeddingProfileId: target.embeddingProfileId,
|
|
1898
|
+
observedOutcomes: uniqueSortedByCodepoints(contexts.map((context) => context.outcome)),
|
|
1899
|
+
},
|
|
1900
|
+
},
|
|
1901
|
+
};
|
|
1902
|
+
}
|
|
1903
|
+
|
|
1904
|
+
private complexityOf(candidate: SearchCandidate): SearchHitV1["complexity"] {
|
|
1905
|
+
const { payload } = candidate;
|
|
1906
|
+
const values = {
|
|
1907
|
+
...(payload.cyclomatic_classic === undefined
|
|
1908
|
+
? {}
|
|
1909
|
+
: { cyclomaticClassic: payload.cyclomatic_classic }),
|
|
1910
|
+
...(payload.cyclomatic_extended === undefined
|
|
1911
|
+
? {}
|
|
1912
|
+
: { cyclomaticExtended: payload.cyclomatic_extended }),
|
|
1913
|
+
...(payload.cognitive === undefined ? {} : { cognitive: payload.cognitive }),
|
|
1914
|
+
...(payload.max_nesting === undefined ? {} : { maxNesting: payload.max_nesting }),
|
|
1915
|
+
...(payload.sloc === undefined ? {} : { sloc: payload.sloc }),
|
|
1916
|
+
...(payload.halstead_volume === undefined ? {} : { halsteadVolume: payload.halstead_volume }),
|
|
1917
|
+
...(payload.maintainability_index === undefined
|
|
1918
|
+
? {}
|
|
1919
|
+
: { maintainabilityIndex: payload.maintainability_index }),
|
|
1920
|
+
...(payload.risk_score === undefined ? {} : { riskScore: payload.risk_score }),
|
|
1921
|
+
};
|
|
1922
|
+
if (Object.keys(values).length === 0) {
|
|
1923
|
+
return undefined;
|
|
1924
|
+
}
|
|
1925
|
+
return {
|
|
1926
|
+
metricsScope: payload.metrics_scope,
|
|
1927
|
+
metricsScopeId: payload.metrics_scope_id,
|
|
1928
|
+
metricsVersion: payload.metrics_version,
|
|
1929
|
+
...values,
|
|
1930
|
+
};
|
|
1931
|
+
}
|
|
1932
|
+
}
|
|
1933
|
+
|
|
1934
|
+
/** Kleiner Bequemlichkeits-Konstruktor. */
|
|
1935
|
+
export function createHybridSearchEngine(deps: HybridSearchEngineDeps): HybridSearchEngine {
|
|
1936
|
+
return new HybridSearchEngine(deps);
|
|
1937
|
+
}
|
|
1938
|
+
|
|
1939
|
+
/** Wiederexport, damit Aufrufer den Filtertyp nicht aus zwei Paketen ziehen muessen. */
|
|
1940
|
+
export type { PayloadFilter };
|
|
1941
|
+
|
|
1942
|
+
function markFinalHitGraphUnavailable(
|
|
1943
|
+
groups: SearchHitGroupV1[],
|
|
1944
|
+
query: SearchQueryV1,
|
|
1945
|
+
detail: string,
|
|
1946
|
+
): void {
|
|
1947
|
+
for (const group of groups) {
|
|
1948
|
+
for (const hit of group.hits) {
|
|
1949
|
+
if (query.includeGraphContext) {
|
|
1950
|
+
hit.graphContext = unavailableSearchGraphRelations("graph_unavailable", detail);
|
|
1951
|
+
}
|
|
1952
|
+
if (query.includeLinked) {
|
|
1953
|
+
hit.linkedSymbols = unavailableSearchGraphRelations("graph_unavailable", detail);
|
|
1954
|
+
}
|
|
1955
|
+
}
|
|
1956
|
+
}
|
|
1957
|
+
}
|