@amsterdamdatalabs/enact-docs-search 0.1.2 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +29 -4
- package/dist/cli.js +158 -53
- package/dist/cli.js.map +1 -1
- package/dist/indexing/chunker.d.ts +6 -1
- package/dist/indexing/chunker.d.ts.map +1 -1
- package/dist/indexing/chunker.js +17 -11
- package/dist/indexing/chunker.js.map +1 -1
- package/dist/indexing/embedding-text.d.ts +22 -0
- package/dist/indexing/embedding-text.d.ts.map +1 -0
- package/dist/indexing/embedding-text.js +27 -0
- package/dist/indexing/embedding-text.js.map +1 -0
- package/dist/indexing/frontmatter.d.ts +48 -0
- package/dist/indexing/frontmatter.d.ts.map +1 -0
- package/dist/indexing/frontmatter.js +162 -0
- package/dist/indexing/frontmatter.js.map +1 -0
- package/dist/indexing/indexing-service.d.ts +1 -4
- package/dist/indexing/indexing-service.d.ts.map +1 -1
- package/dist/indexing/indexing-service.js +24 -12
- package/dist/indexing/indexing-service.js.map +1 -1
- package/dist/search/authority.d.ts +31 -0
- package/dist/search/authority.d.ts.map +1 -0
- package/dist/search/authority.js +50 -0
- package/dist/search/authority.js.map +1 -0
- package/dist/search/relevance.d.ts +80 -0
- package/dist/search/relevance.d.ts.map +1 -0
- package/dist/search/relevance.js +119 -0
- package/dist/search/relevance.js.map +1 -0
- package/dist/search/search-service.d.ts +44 -2
- package/dist/search/search-service.d.ts.map +1 -1
- package/dist/search/search-service.js +186 -23
- package/dist/search/search-service.js.map +1 -1
- package/dist/search/snippet.d.ts +11 -0
- package/dist/search/snippet.d.ts.map +1 -1
- package/dist/search/snippet.js +102 -17
- package/dist/search/snippet.js.map +1 -1
- package/dist/storage/document-store.d.ts +47 -5
- package/dist/storage/document-store.d.ts.map +1 -1
- package/dist/storage/document-store.js +94 -16
- package/dist/storage/document-store.js.map +1 -1
- package/dist/storage/persistence.d.ts +6 -10
- package/dist/storage/persistence.d.ts.map +1 -1
- package/dist/storage/persistence.js +20 -20
- package/dist/storage/persistence.js.map +1 -1
- package/dist/storage/vector-store.d.ts +2 -3
- package/dist/storage/vector-store.d.ts.map +1 -1
- package/dist/storage/vector-store.js +3 -11
- package/dist/storage/vector-store.js.map +1 -1
- package/dist/types.d.ts +2 -7
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js +2 -20
- package/dist/types.js.map +1 -1
- package/package.json +2 -2
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Relevance gating.
|
|
3
|
+
*
|
|
4
|
+
* Reciprocal rank fusion scores a result by its *position*, not by how well it
|
|
5
|
+
* matched, so the top hit for a nonsense query scores about as high as the top
|
|
6
|
+
* hit for a perfect one. Measured on this repo: gibberish topped out at 0.0664
|
|
7
|
+
* against 0.0815 for a well-matched query. Nothing downstream of fusion can
|
|
8
|
+
* tell those apart, so the gate has to run on the raw per-modality evidence
|
|
9
|
+
* before fusion throws it away.
|
|
10
|
+
*/
|
|
11
|
+
import { extractQueryTerms, containsTerm } from "./snippet.js";
|
|
12
|
+
/**
|
|
13
|
+
* Cosine floor for semantic hits.
|
|
14
|
+
*
|
|
15
|
+
* Re-measured against docs/ in enact-m5 (203 chunks), taking the top cosine
|
|
16
|
+
* over the whole corpus per query:
|
|
17
|
+
*
|
|
18
|
+
* junk zzqzz qqxxw 0.131 | florbnax vimtuzzle 0.134
|
|
19
|
+
* purple giraffe tapdance 0.103 | qwertyuiop asdfghjkl 0.132
|
|
20
|
+
* genuine invariants 0.260 | brownfield reference implementation 0.322
|
|
21
|
+
* database migrations 0.374 | websocket routes 0.520
|
|
22
|
+
* sandbox lifecycle 0.531 | coverage hooks 0.693
|
|
23
|
+
*
|
|
24
|
+
* So the real gap is [0.134, 0.260], and 0.20 sits in the middle of it with
|
|
25
|
+
* roughly equal headroom on each side. An earlier value of 0.35 was set from
|
|
26
|
+
* figures this measurement could not reproduce, and it silently discarded the
|
|
27
|
+
* best hit for real queries -- "invariants" returned nothing at all.
|
|
28
|
+
*
|
|
29
|
+
* Reproduce with:
|
|
30
|
+
* node dist/cli.js "<query>" --repo <repo> --mode semantic --min-score 0 --json
|
|
31
|
+
*/
|
|
32
|
+
export const DEFAULT_MIN_VECTOR_SCORE = 0.2;
|
|
33
|
+
/**
|
|
34
|
+
* Fraction of the query's *weight* a document must cover (see
|
|
35
|
+
* `idfWeightedCoverage`), not a fraction of its terms.
|
|
36
|
+
*
|
|
37
|
+
* Measured separation on this corpus:
|
|
38
|
+
* 0.09 "presence is not armedness" -- absent term dominates the weight
|
|
39
|
+
* 0.47 "mobile device" matching only the commoner term
|
|
40
|
+
* 0.53 "mobile device" matching only the rarer term
|
|
41
|
+
* 0.73 a five-term query missing one term
|
|
42
|
+
* 1.00 full match
|
|
43
|
+
*
|
|
44
|
+
* 0.5 sits below every genuine partial match and far above the junk floor.
|
|
45
|
+
*/
|
|
46
|
+
export const DEFAULT_MIN_TERM_COVERAGE = 0.5;
|
|
47
|
+
/**
|
|
48
|
+
* Fraction of the query's content terms that appear in `text`.
|
|
49
|
+
*
|
|
50
|
+
* Returns 1 for a query with no content terms (all stopwords or too short):
|
|
51
|
+
* there is nothing to be missing, so the gate must not reject on that basis.
|
|
52
|
+
*/
|
|
53
|
+
export function termCoverage(query, text) {
|
|
54
|
+
const terms = extractQueryTerms(query);
|
|
55
|
+
if (terms.length === 0)
|
|
56
|
+
return 1;
|
|
57
|
+
const matched = terms.filter((term) => containsTerm(text, term)).length;
|
|
58
|
+
return matched / terms.length;
|
|
59
|
+
}
|
|
60
|
+
/** Smoothed inverse document frequency. A term absent from the corpus scores
|
|
61
|
+
* highest: a query term nothing contains is the most load-bearing one there
|
|
62
|
+
* is, and a result that misses it has missed the point of the query. */
|
|
63
|
+
function idf(term, stats) {
|
|
64
|
+
const df = stats.documentFrequency.get(term) ?? 0;
|
|
65
|
+
return Math.log((stats.totalDocs + 1) / (df + 1)) + 1;
|
|
66
|
+
}
|
|
67
|
+
/**
|
|
68
|
+
* Normalise a raw query string.
|
|
69
|
+
*
|
|
70
|
+
* Agents type `mobile | devices` out of grep habit. The pipe is treated as a
|
|
71
|
+
* separator, exactly like a space -- `mobile | devices` and `mobile devices`
|
|
72
|
+
* are the same query. It is deliberately not an OR operator: one query syntax
|
|
73
|
+
* is easier to use correctly than two, and an unquoted pipe never reaches this
|
|
74
|
+
* process anyway -- the shell consumes it as a pipeline first.
|
|
75
|
+
*/
|
|
76
|
+
export function normalizeQuery(query) {
|
|
77
|
+
return query.replace(/\|/g, " ").replace(/\s+/g, " ").trim();
|
|
78
|
+
}
|
|
79
|
+
/**
|
|
80
|
+
* Coverage weighted by term rarity.
|
|
81
|
+
*
|
|
82
|
+
* Flat coverage treats every term as equally important, which makes a two-term
|
|
83
|
+
* query behave like AND -- "mobile device" then drops a document about a
|
|
84
|
+
* "mobile target" for lacking the word "device". Weighting by IDF lets the
|
|
85
|
+
* term that actually carries the query decide, in both directions: matching
|
|
86
|
+
* only the rare term passes, matching only the common one does not.
|
|
87
|
+
*
|
|
88
|
+
* Falls back to flat coverage when no stats are supplied.
|
|
89
|
+
*/
|
|
90
|
+
export function idfWeightedCoverage(query, text, stats) {
|
|
91
|
+
if (!stats)
|
|
92
|
+
return termCoverage(query, text);
|
|
93
|
+
const terms = extractQueryTerms(query);
|
|
94
|
+
if (terms.length === 0)
|
|
95
|
+
return 1;
|
|
96
|
+
let matchedWeight = 0;
|
|
97
|
+
let totalWeight = 0;
|
|
98
|
+
for (const term of terms) {
|
|
99
|
+
const weight = idf(term, stats);
|
|
100
|
+
totalWeight += weight;
|
|
101
|
+
if (containsTerm(text, term))
|
|
102
|
+
matchedWeight += weight;
|
|
103
|
+
}
|
|
104
|
+
return totalWeight === 0 ? 1 : matchedWeight / totalWeight;
|
|
105
|
+
}
|
|
106
|
+
/**
|
|
107
|
+
* Keyword-side gate. Uses term coverage rather than a BM25 threshold: BM25
|
|
108
|
+
* scores here are unnormalised and scale with query length and corpus size,
|
|
109
|
+
* so any absolute cutoff would be a fact about one repo. Coverage is a ratio
|
|
110
|
+
* and travels.
|
|
111
|
+
*/
|
|
112
|
+
export function passesKeywordGate(query, text, minCoverage = DEFAULT_MIN_TERM_COVERAGE, stats) {
|
|
113
|
+
return idfWeightedCoverage(query, text, stats) >= (minCoverage ?? DEFAULT_MIN_TERM_COVERAGE);
|
|
114
|
+
}
|
|
115
|
+
/** Semantic-side gate. Cosine is bounded, so an absolute floor is portable. */
|
|
116
|
+
export function passesVectorGate(score, minScore = DEFAULT_MIN_VECTOR_SCORE) {
|
|
117
|
+
return score >= minScore;
|
|
118
|
+
}
|
|
119
|
+
//# sourceMappingURL=relevance.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"relevance.js","sourceRoot":"","sources":["../../src/search/relevance.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AACH,OAAO,EAAE,iBAAiB,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAE/D;;;;;;;;;;;;;;;;;;;GAmBG;AACH,MAAM,CAAC,MAAM,wBAAwB,GAAG,GAAG,CAAC;AAE5C;;;;;;;;;;;;GAYG;AACH,MAAM,CAAC,MAAM,yBAAyB,GAAG,GAAG,CAAC;AAE7C;;;;;GAKG;AACH,MAAM,UAAU,YAAY,CAAC,KAAa,EAAE,IAAY;IACtD,MAAM,KAAK,GAAG,iBAAiB,CAAC,KAAK,CAAC,CAAC;IACvC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAEjC,MAAM,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,YAAY,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC;IACxE,OAAO,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC;AAChC,CAAC;AASD;;wEAEwE;AACxE,SAAS,GAAG,CAAC,IAAY,EAAE,KAAkB;IAC3C,MAAM,EAAE,GAAG,KAAK,CAAC,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAClD,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,SAAS,GAAG,CAAC,CAAC,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;AACxD,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,cAAc,CAAC,KAAa;IAC1C,OAAO,KAAK,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;AAC/D,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,UAAU,mBAAmB,CACjC,KAAa,EACb,IAAY,EACZ,KAAmB;IAEnB,IAAI,CAAC,KAAK;QAAE,OAAO,YAAY,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;IAE7C,MAAM,KAAK,GAAG,iBAAiB,CAAC,KAAK,CAAC,CAAC;IACvC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAEjC,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,IAAI,WAAW,GAAG,CAAC,CAAC;IAEpB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,MAAM,GAAG,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QAChC,WAAW,IAAI,MAAM,CAAC;QACtB,IAAI,YAAY,CAAC,IAAI,EAAE,IAAI,CAAC;YAAE,aAAa,IAAI,MAAM,CAAC;IACxD,CAAC;IAED,OAAO,WAAW,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,aAAa,GAAG,WAAW,CAAC;AAC7D,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,iBAAiB,CAC/B,KAAa,EACb,IAAY,EACZ,cAAsB,yBAAyB,EAC/C,KAAmB;IAEnB,OAAO,mBAAmB,CAAC,KAAK,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,WAAW,IAAI,yBAAyB,CAAC,CAAC;AAC/F,CAAC;AAED,+EAA+E;AAC/E,MAAM,UAAU,gBAAgB,CAC9B,KAAa,EACb,WAAmB,wBAAwB;IAE3C,OAAO,KAAK,IAAI,QAAQ,CAAC;AAC3B,CAAC"}
|
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
import { DocumentStore } from "../storage/document-store.js";
|
|
14
14
|
import { VectorStore } from "../storage/vector-store.js";
|
|
15
15
|
import { EmbeddingService } from "../embeddings/embedding-service.js";
|
|
16
|
-
import type {
|
|
16
|
+
import type { SearchMode } from "../types.js";
|
|
17
17
|
import type { HighlightMode } from "./snippet.js";
|
|
18
18
|
export interface HybridResult {
|
|
19
19
|
id: string;
|
|
@@ -27,10 +27,19 @@ export interface HybridResult {
|
|
|
27
27
|
content?: string;
|
|
28
28
|
headingBreadcrumb?: string[];
|
|
29
29
|
source: "text" | "vector" | "hybrid";
|
|
30
|
+
title?: string;
|
|
31
|
+
description?: string;
|
|
32
|
+
headings?: string;
|
|
33
|
+
status?: string;
|
|
34
|
+
documentType?: string;
|
|
35
|
+
lastUpdate?: string;
|
|
36
|
+
/** Fused score before authority weighting, kept so --explain can show both. */
|
|
37
|
+
baseScore?: number;
|
|
38
|
+
/** Authority multiplier applied to `baseScore` to produce the final order. */
|
|
39
|
+
authority?: number;
|
|
30
40
|
}
|
|
31
41
|
export interface SearchOptions {
|
|
32
42
|
mode: SearchMode;
|
|
33
|
-
scope: Scope;
|
|
34
43
|
limit?: number;
|
|
35
44
|
/** How matched terms should be marked in generated snippets. Defaults to
|
|
36
45
|
* "plain" (no markup) when omitted. */
|
|
@@ -39,15 +48,48 @@ export interface SearchOptions {
|
|
|
39
48
|
* snippet generation. When omitted, snippets fall back to chunk-only
|
|
40
49
|
* generation. */
|
|
41
50
|
repoRoot?: string;
|
|
51
|
+
/** Cosine floor for semantic hits. */
|
|
52
|
+
minVectorScore?: number;
|
|
53
|
+
/** Required fraction of query terms present, for keyword hits. */
|
|
54
|
+
minCoverage?: number;
|
|
55
|
+
/** Skip status/recency weighting, so raw retrieval order can be compared. */
|
|
56
|
+
rawRank?: boolean;
|
|
57
|
+
/** Clock for recency decay; injected so tests are not time-dependent. */
|
|
58
|
+
now?: number;
|
|
42
59
|
}
|
|
43
60
|
export declare class SearchService {
|
|
44
61
|
private docStore;
|
|
45
62
|
private vectorStore;
|
|
46
63
|
private embeddingService;
|
|
47
64
|
constructor(docStore: DocumentStore, vectorStore: VectorStore, embeddingService: EmbeddingService | null);
|
|
65
|
+
/**
|
|
66
|
+
* Fetch vector chunks and collapse to the best-scoring chunk per FILE,
|
|
67
|
+
* growing the request until `fileLimit` distinct files are found (or the
|
|
68
|
+
* store has no more chunks to give).
|
|
69
|
+
*
|
|
70
|
+
* `DocumentStore.search` collapses to one result per file before slicing
|
|
71
|
+
* to a limit (see its comment); a raw HNSW search can't do the equivalent
|
|
72
|
+
* because it has to be told a chunk budget upfront, and one large document
|
|
73
|
+
* can own most of that budget -- on the real corpus,
|
|
74
|
+
* docs/architecture/engine/lessons-learned.md supplied 26 of the top 30
|
|
75
|
+
* vector hits for one query, leaving only 4 chunk-slots (not file-slots)
|
|
76
|
+
* for every other document combined. Raising the chunk budget cannot fix
|
|
77
|
+
* this in general, since a bigger document just needs a bigger multiple.
|
|
78
|
+
* Instead ask for more chunks, geometrically, until enough distinct files
|
|
79
|
+
* have shown up -- the same "collapse before slice" shape the keyword side
|
|
80
|
+
* already gets from MiniSearch returning its full match list.
|
|
81
|
+
*/
|
|
82
|
+
private fetchVectorResultsByFile;
|
|
48
83
|
search(query: string, options: SearchOptions): Promise<HybridResult[]>;
|
|
49
84
|
private textSearch;
|
|
50
85
|
private semanticSearch;
|
|
51
86
|
private hybridSearch;
|
|
87
|
+
/**
|
|
88
|
+
* Apply authority weighting and produce the final page.
|
|
89
|
+
*
|
|
90
|
+
* Every mode ends here, so `--mode keyword` cannot become a quiet bypass of
|
|
91
|
+
* the same weighting hybrid results get.
|
|
92
|
+
*/
|
|
93
|
+
private finish;
|
|
52
94
|
}
|
|
53
95
|
//# sourceMappingURL=search-service.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"search-service.d.ts","sourceRoot":"","sources":["../../src/search/search-service.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AACH,OAAO,EAAE,aAAa,
|
|
1
|
+
{"version":3,"file":"search-service.d.ts","sourceRoot":"","sources":["../../src/search/search-service.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AACH,OAAO,EAAE,aAAa,EAAkB,MAAM,8BAA8B,CAAC;AAC7E,OAAO,EAAE,WAAW,EAAE,MAAM,4BAA4B,CAAC;AAEzD,OAAO,EAAE,gBAAgB,EAAE,MAAM,oCAAoC,CAAC;AACtE,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAE9C,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAUlD,MAAM,WAAW,YAAY;IAC3B,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC7B,MAAM,EAAE,MAAM,GAAG,QAAQ,GAAG,QAAQ,CAAC;IACrC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,+EAA+E;IAC/E,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,8EAA8E;IAC9E,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,UAAU,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;2CACuC;IACvC,SAAS,CAAC,EAAE,aAAa,CAAC;IAC1B;;qBAEiB;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,sCAAsC;IACtC,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,kEAAkE;IAClE,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,6EAA6E;IAC7E,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,yEAAyE;IACzE,GAAG,CAAC,EAAE,MAAM,CAAC;CACd;AAaD,qBAAa,aAAa;IAEtB,OAAO,CAAC,QAAQ;IAChB,OAAO,CAAC,WAAW;IACnB,OAAO,CAAC,gBAAgB;gBAFhB,QAAQ,EAAE,aAAa,EACvB,WAAW,EAAE,WAAW,EACxB,gBAAgB,EAAE,gBAAgB,GAAG,IAAI;IAGnD;;;;;;;;;;;;;;;;OAgBG;YACW,wBAAwB;IA6BhC,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;YA+B9D,UAAU;YAoCV,cAAc;YAuDd,YAAY;IAsH1B;;;;;OAKG;IACH,OAAO,CAAC,MAAM;CAyBf"}
|
|
@@ -1,5 +1,27 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Adapted from obsidian-qmd (MIT): https://github.com/thirteen37/obsidian-qmd
|
|
3
|
+
* Source: src/search/SearchService.ts. Carried over: the core idea of
|
|
4
|
+
* Reciprocal Rank Fusion over independently-ranked BM25 and vector result
|
|
5
|
+
* lists, with small positional bonuses for top ranks (RRF_K=60, same
|
|
6
|
+
* constant as upstream). Dropped: `QueryExpansionService` and
|
|
7
|
+
* `RerankingService` integration — both call out to an LLM, which is out of
|
|
8
|
+
* scope for this offline CLI (the brief's three dependencies are minisearch,
|
|
9
|
+
* @huggingface/transformers, and hnswlib-node; no LLM client). Also dropped:
|
|
10
|
+
* `groupByFile`/section dedup UI conveniences not needed by a line-oriented
|
|
11
|
+
* CLI, since callers get raw scored chunks that already carry a line number.
|
|
12
|
+
*/
|
|
13
|
+
import { searchableText } from "../storage/document-store.js";
|
|
1
14
|
import { extractQueryTerms, buildSnippet } from "./snippet.js";
|
|
15
|
+
import { passesKeywordGate, passesVectorGate, normalizeQuery, DEFAULT_MIN_TERM_COVERAGE, DEFAULT_MIN_VECTOR_SCORE, } from "./relevance.js";
|
|
16
|
+
import { authorityFactor } from "./authority.js";
|
|
2
17
|
const RRF_K = 60;
|
|
18
|
+
/**
|
|
19
|
+
* Retrieval over-fetch. The gate and the authority pass both remove or demote
|
|
20
|
+
* results, so each modality is asked for more than the caller wants -- without
|
|
21
|
+
* this a demoted document merely shuffles within the page instead of leaving
|
|
22
|
+
* it.
|
|
23
|
+
*/
|
|
24
|
+
const OVERFETCH = 3;
|
|
3
25
|
export class SearchService {
|
|
4
26
|
docStore;
|
|
5
27
|
vectorStore;
|
|
@@ -9,54 +31,142 @@ export class SearchService {
|
|
|
9
31
|
this.vectorStore = vectorStore;
|
|
10
32
|
this.embeddingService = embeddingService;
|
|
11
33
|
}
|
|
34
|
+
/**
|
|
35
|
+
* Fetch vector chunks and collapse to the best-scoring chunk per FILE,
|
|
36
|
+
* growing the request until `fileLimit` distinct files are found (or the
|
|
37
|
+
* store has no more chunks to give).
|
|
38
|
+
*
|
|
39
|
+
* `DocumentStore.search` collapses to one result per file before slicing
|
|
40
|
+
* to a limit (see its comment); a raw HNSW search can't do the equivalent
|
|
41
|
+
* because it has to be told a chunk budget upfront, and one large document
|
|
42
|
+
* can own most of that budget -- on the real corpus,
|
|
43
|
+
* docs/architecture/engine/lessons-learned.md supplied 26 of the top 30
|
|
44
|
+
* vector hits for one query, leaving only 4 chunk-slots (not file-slots)
|
|
45
|
+
* for every other document combined. Raising the chunk budget cannot fix
|
|
46
|
+
* this in general, since a bigger document just needs a bigger multiple.
|
|
47
|
+
* Instead ask for more chunks, geometrically, until enough distinct files
|
|
48
|
+
* have shown up -- the same "collapse before slice" shape the keyword side
|
|
49
|
+
* already gets from MiniSearch returning its full match list.
|
|
50
|
+
*/
|
|
51
|
+
async fetchVectorResultsByFile(embedding, fileLimit) {
|
|
52
|
+
// `fileLimit` already carries the caller's OVERFETCH (see semanticSearch
|
|
53
|
+
// / hybridSearch) -- starting the first request at `fileLimit` again
|
|
54
|
+
// multiplied by OVERFETCH asked for a chunk budget nobody wanted; the
|
|
55
|
+
// first request's budget is `fileLimit` itself.
|
|
56
|
+
let k = fileLimit;
|
|
57
|
+
let byFile = new Map();
|
|
58
|
+
for (;;) {
|
|
59
|
+
const raw = await this.vectorStore.search(embedding, k);
|
|
60
|
+
byFile = new Map();
|
|
61
|
+
for (const r of raw) {
|
|
62
|
+
const incumbent = byFile.get(r.filePath);
|
|
63
|
+
if (!incumbent || r.score > incumbent.score)
|
|
64
|
+
byFile.set(r.filePath, r);
|
|
65
|
+
}
|
|
66
|
+
// Stop once enough distinct files have surfaced, or once the store
|
|
67
|
+
// hands back fewer chunks than asked for -- a strictly smaller
|
|
68
|
+
// response is unambiguous proof there is nothing left to fetch. A
|
|
69
|
+
// response of exactly `k` is not that proof (the store may hold
|
|
70
|
+
// more), so that case keeps growing instead of stopping one
|
|
71
|
+
// (increasingly expensive) request early.
|
|
72
|
+
if (byFile.size >= fileLimit || raw.length < k)
|
|
73
|
+
break;
|
|
74
|
+
k *= 2;
|
|
75
|
+
}
|
|
76
|
+
return Array.from(byFile.values()).sort((a, b) => b.score - a.score);
|
|
77
|
+
}
|
|
12
78
|
async search(query, options) {
|
|
13
|
-
|
|
79
|
+
// The CLI normalizes (pipe-stripping, whitespace-collapsing) before
|
|
80
|
+
// calling in, but a library consumer of SearchService directly would
|
|
81
|
+
// otherwise get different pipe-handling than the CLI does for the exact
|
|
82
|
+
// same query string. Doing it here makes every caller's behaviour the
|
|
83
|
+
// same. Idempotent, so normalizing an already-normalized CLI query is a
|
|
84
|
+
// no-op, not a double-transform.
|
|
85
|
+
const trimmed = normalizeQuery(query);
|
|
14
86
|
if (!trimmed)
|
|
15
87
|
return [];
|
|
16
88
|
const limit = options.limit ?? 10;
|
|
89
|
+
// The CLI validates its flags, but this is also a library entry point. An
|
|
90
|
+
// unchecked limit reached hnswlib as a negative k and surfaced a native
|
|
91
|
+
// error describing the opposite problem; a fractional or NaN limit silently
|
|
92
|
+
// emptied the page via slice(). Fail here, where the message can be true.
|
|
93
|
+
if (!Number.isInteger(limit) || limit < 1) {
|
|
94
|
+
throw new Error(`limit must be a positive integer, got ${String(options.limit)}`);
|
|
95
|
+
}
|
|
17
96
|
const highlight = options.highlight ?? "plain";
|
|
18
97
|
const repoRoot = options.repoRoot;
|
|
19
98
|
switch (options.mode) {
|
|
20
99
|
case "keyword":
|
|
21
|
-
return this.textSearch(trimmed,
|
|
100
|
+
return this.finish(await this.textSearch(trimmed, limit, highlight, repoRoot, options), limit, options);
|
|
22
101
|
case "semantic":
|
|
23
|
-
return this.semanticSearch(trimmed,
|
|
102
|
+
return this.finish(await this.semanticSearch(trimmed, limit, highlight, repoRoot, options), limit, options);
|
|
24
103
|
case "hybrid":
|
|
25
|
-
return this.hybridSearch(trimmed,
|
|
104
|
+
return this.finish(await this.hybridSearch(trimmed, limit, highlight, repoRoot, options), limit, options);
|
|
26
105
|
}
|
|
27
106
|
}
|
|
28
|
-
async textSearch(query,
|
|
29
|
-
const
|
|
30
|
-
|
|
107
|
+
async textSearch(query, limit, highlight, repoRoot, options) {
|
|
108
|
+
const minCoverage = options.minCoverage ?? DEFAULT_MIN_TERM_COVERAGE;
|
|
109
|
+
const stats = this.docStore.corpusStats(extractQueryTerms(query));
|
|
110
|
+
const results = await this.docStore.search(query, {
|
|
111
|
+
limit: limit * OVERFETCH,
|
|
112
|
+
highlight,
|
|
113
|
+
repoRoot,
|
|
114
|
+
});
|
|
115
|
+
return results
|
|
116
|
+
.filter((r) => passesKeywordGate(query, searchableText(r), minCoverage, stats))
|
|
117
|
+
.map((r) => ({
|
|
31
118
|
id: r.id,
|
|
32
119
|
filePath: r.filePath,
|
|
33
120
|
chunkIndex: r.chunkIndex,
|
|
34
121
|
startLine: r.startLine,
|
|
35
122
|
textScore: r.score,
|
|
123
|
+
baseScore: r.score,
|
|
36
124
|
snippet: r.snippet,
|
|
37
125
|
content: r.content,
|
|
38
126
|
headingBreadcrumb: r.headingBreadcrumb,
|
|
127
|
+
title: r.title,
|
|
128
|
+
description: r.description,
|
|
129
|
+
headings: r.headings,
|
|
130
|
+
status: r.status,
|
|
131
|
+
documentType: r.documentType,
|
|
132
|
+
lastUpdate: r.lastUpdate,
|
|
39
133
|
source: "text",
|
|
40
134
|
}));
|
|
41
135
|
}
|
|
42
|
-
async semanticSearch(query,
|
|
136
|
+
async semanticSearch(query, limit, highlight, repoRoot, options) {
|
|
43
137
|
if (!this.embeddingService) {
|
|
44
138
|
throw new Error("Semantic search requires an embedding service");
|
|
45
139
|
}
|
|
140
|
+
const minVectorScore = options.minVectorScore ?? DEFAULT_MIN_VECTOR_SCORE;
|
|
46
141
|
const embedding = await this.embeddingService.embed(query);
|
|
47
|
-
const
|
|
142
|
+
const raw = await this.fetchVectorResultsByFile(embedding, limit * OVERFETCH);
|
|
143
|
+
const results = raw.filter((r) => passesVectorGate(r.score, minVectorScore));
|
|
48
144
|
const queryTerms = extractQueryTerms(query);
|
|
49
145
|
const hybridResults = [];
|
|
50
146
|
for (const r of results) {
|
|
51
147
|
const doc = await this.docStore.getDocument(r.id);
|
|
148
|
+
// The vector index and the document store are written separately, so a
|
|
149
|
+
// point can outlive its chunk. Without the chunk there is no line and no
|
|
150
|
+
// text to show -- it rendered as a citation to "file:0" with no snippet.
|
|
151
|
+
// Drop what we cannot substantiate rather than print a line that is not
|
|
152
|
+
// there.
|
|
153
|
+
if (!doc)
|
|
154
|
+
continue;
|
|
52
155
|
hybridResults.push({
|
|
53
156
|
id: r.id,
|
|
54
157
|
filePath: r.filePath,
|
|
55
158
|
chunkIndex: r.chunkIndex,
|
|
56
159
|
startLine: doc?.startLine ?? 0,
|
|
57
160
|
vectorScore: r.score,
|
|
161
|
+
baseScore: r.score,
|
|
58
162
|
content: doc?.content,
|
|
59
163
|
headingBreadcrumb: doc?.headingBreadcrumb,
|
|
164
|
+
title: doc?.title,
|
|
165
|
+
description: doc?.description,
|
|
166
|
+
headings: doc?.headings,
|
|
167
|
+
status: doc?.status,
|
|
168
|
+
documentType: doc?.documentType,
|
|
169
|
+
lastUpdate: doc?.lastUpdate,
|
|
60
170
|
snippet: doc
|
|
61
171
|
? await buildSnippet({ filePath: r.filePath, content: doc.content, startLine: doc.startLine }, queryTerms, highlight, repoRoot)
|
|
62
172
|
: undefined,
|
|
@@ -65,16 +175,28 @@ export class SearchService {
|
|
|
65
175
|
}
|
|
66
176
|
return hybridResults;
|
|
67
177
|
}
|
|
68
|
-
async hybridSearch(query,
|
|
69
|
-
const
|
|
70
|
-
|
|
178
|
+
async hybridSearch(query, limit, highlight, repoRoot, options) {
|
|
179
|
+
const minCoverage = options.minCoverage ?? DEFAULT_MIN_TERM_COVERAGE;
|
|
180
|
+
const minVectorScore = options.minVectorScore ?? DEFAULT_MIN_VECTOR_SCORE;
|
|
181
|
+
const [rawText, rawVector] = await Promise.all([
|
|
182
|
+
this.docStore.search(query, { limit: limit * OVERFETCH, highlight, repoRoot }),
|
|
71
183
|
this.embeddingService
|
|
72
|
-
? this.
|
|
184
|
+
? this.fetchVectorResultsByFile(await this.embeddingService.embed(query), limit * OVERFETCH)
|
|
73
185
|
: Promise.resolve([]),
|
|
74
186
|
]);
|
|
75
|
-
|
|
187
|
+
// Gate each list on its own evidence before fusion. Afterwards the scores
|
|
188
|
+
// are ranks, and a rank cannot tell a good match from a bad one.
|
|
189
|
+
const stats = this.docStore.corpusStats(extractQueryTerms(query));
|
|
190
|
+
const textResults = rawText.filter((r) => passesKeywordGate(query, searchableText(r), minCoverage, stats));
|
|
191
|
+
const vectorResults = rawVector.filter((r) => passesVectorGate(r.score, minVectorScore));
|
|
192
|
+
// Fuse per FILE, not per chunk. The keyword and vector sides routinely pick
|
|
193
|
+
// different chunks of the same document; keyed by chunk id they would
|
|
194
|
+
// compete as separate entries and one side's evidence would be thrown away
|
|
195
|
+
// at the final collapse. Keyed by file they reinforce, which is what a
|
|
196
|
+
// document matching both ways should mean.
|
|
197
|
+
const byFile = new Map();
|
|
76
198
|
textResults.forEach((r, index) => {
|
|
77
|
-
|
|
199
|
+
byFile.set(r.filePath, {
|
|
78
200
|
id: r.id,
|
|
79
201
|
filePath: r.filePath,
|
|
80
202
|
chunkIndex: r.chunkIndex,
|
|
@@ -84,20 +206,32 @@ export class SearchService {
|
|
|
84
206
|
snippet: r.snippet,
|
|
85
207
|
content: r.content,
|
|
86
208
|
headingBreadcrumb: r.headingBreadcrumb,
|
|
209
|
+
title: r.title,
|
|
210
|
+
description: r.description,
|
|
211
|
+
headings: r.headings,
|
|
212
|
+
status: r.status,
|
|
213
|
+
documentType: r.documentType,
|
|
214
|
+
lastUpdate: r.lastUpdate,
|
|
87
215
|
source: "text",
|
|
88
216
|
});
|
|
89
217
|
});
|
|
90
218
|
const queryTerms = extractQueryTerms(query);
|
|
91
219
|
for (const [index, r] of vectorResults.entries()) {
|
|
92
|
-
const existing =
|
|
220
|
+
const existing = byFile.get(r.filePath);
|
|
93
221
|
if (existing) {
|
|
94
|
-
|
|
95
|
-
|
|
222
|
+
// Keep the keyword side's chunk for display -- its snippet already has
|
|
223
|
+
// the query terms marked -- and take the vector rank as extra evidence
|
|
224
|
+
// for the same document.
|
|
225
|
+
existing.vectorScore = Math.max(existing.vectorScore ?? 0, r.score);
|
|
226
|
+
existing.vectorRank = Math.min(existing.vectorRank ?? Infinity, index + 1);
|
|
96
227
|
existing.source = "hybrid";
|
|
97
228
|
}
|
|
98
229
|
else {
|
|
99
230
|
const doc = await this.docStore.getDocument(r.id);
|
|
100
|
-
|
|
231
|
+
// Same orphan case as in semanticSearch: no chunk, nothing to cite.
|
|
232
|
+
if (!doc)
|
|
233
|
+
continue;
|
|
234
|
+
byFile.set(r.filePath, {
|
|
101
235
|
id: r.id,
|
|
102
236
|
filePath: r.filePath,
|
|
103
237
|
chunkIndex: r.chunkIndex,
|
|
@@ -106,6 +240,12 @@ export class SearchService {
|
|
|
106
240
|
vectorRank: index + 1,
|
|
107
241
|
content: doc?.content,
|
|
108
242
|
headingBreadcrumb: doc?.headingBreadcrumb,
|
|
243
|
+
title: doc?.title,
|
|
244
|
+
description: doc?.description,
|
|
245
|
+
headings: doc?.headings,
|
|
246
|
+
status: doc?.status,
|
|
247
|
+
documentType: doc?.documentType,
|
|
248
|
+
lastUpdate: doc?.lastUpdate,
|
|
109
249
|
snippet: doc
|
|
110
250
|
? await buildSnippet({ filePath: r.filePath, content: doc.content, startLine: doc.startLine }, queryTerms, highlight, repoRoot)
|
|
111
251
|
: undefined,
|
|
@@ -113,7 +253,7 @@ export class SearchService {
|
|
|
113
253
|
});
|
|
114
254
|
}
|
|
115
255
|
}
|
|
116
|
-
const results = Array.from(
|
|
256
|
+
const results = Array.from(byFile.values()).map((r) => {
|
|
117
257
|
let rrfScore = 0;
|
|
118
258
|
if (r.textRank) {
|
|
119
259
|
rrfScore += 1 / (RRF_K + r.textRank);
|
|
@@ -130,10 +270,33 @@ export class SearchService {
|
|
|
130
270
|
rrfScore += 0.02;
|
|
131
271
|
}
|
|
132
272
|
const { textRank, vectorRank, ...clean } = r;
|
|
133
|
-
return { ...clean, rrfScore };
|
|
273
|
+
return { ...clean, rrfScore, baseScore: rrfScore };
|
|
274
|
+
});
|
|
275
|
+
return results;
|
|
276
|
+
}
|
|
277
|
+
/**
|
|
278
|
+
* Apply authority weighting and produce the final page.
|
|
279
|
+
*
|
|
280
|
+
* Every mode ends here, so `--mode keyword` cannot become a quiet bypass of
|
|
281
|
+
* the same weighting hybrid results get.
|
|
282
|
+
*/
|
|
283
|
+
finish(results, limit, options) {
|
|
284
|
+
const now = options.now ?? Date.now();
|
|
285
|
+
const weighted = results.map((r) => {
|
|
286
|
+
const base = r.baseScore ?? r.rrfScore ?? r.textScore ?? r.vectorScore ?? 0;
|
|
287
|
+
const authority = options.rawRank ? 1 : authorityFactor(r, now);
|
|
288
|
+
return { ...r, baseScore: base, authority, finalScore: base * authority };
|
|
134
289
|
});
|
|
135
|
-
|
|
136
|
-
|
|
290
|
+
weighted.sort((a, b) => b.finalScore - a.finalScore);
|
|
291
|
+
// No per-file dedup here: every path into this method already returns
|
|
292
|
+
// at most one row per file. DocumentStore.search (used by both
|
|
293
|
+
// textSearch and hybridSearch's text side) and fetchVectorResultsByFile
|
|
294
|
+
// both collapse to the best-scoring chunk per file before their caller
|
|
295
|
+
// ever sees a result, and hybridSearch's fusion map is keyed by
|
|
296
|
+
// filePath. Verified with an instrumented dedup counter across the full
|
|
297
|
+
// test suite plus the exact "two chunks of one file both clear the
|
|
298
|
+
// gate" semantic-mode shape that used to need this -- it never fires.
|
|
299
|
+
return weighted.slice(0, limit).map(({ finalScore, ...r }) => r);
|
|
137
300
|
}
|
|
138
301
|
}
|
|
139
302
|
//# sourceMappingURL=search-service.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"search-service.js","sourceRoot":"","sources":["../../src/search/search-service.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"search-service.js","sourceRoot":"","sources":["../../src/search/search-service.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AACH,OAAO,EAAiB,cAAc,EAAE,MAAM,8BAA8B,CAAC;AAK7E,OAAO,EAAE,iBAAiB,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAE/D,OAAO,EACL,iBAAiB,EACjB,gBAAgB,EAChB,cAAc,EACd,yBAAyB,EACzB,wBAAwB,GACzB,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AA8CjD,MAAM,KAAK,GAAG,EAAE,CAAC;AAEjB;;;;;GAKG;AACH,MAAM,SAAS,GAAG,CAAC,CAAC;AAGpB,MAAM,OAAO,aAAa;IAEd;IACA;IACA;IAHV,YACU,QAAuB,EACvB,WAAwB,EACxB,gBAAyC;QAFzC,aAAQ,GAAR,QAAQ,CAAe;QACvB,gBAAW,GAAX,WAAW,CAAa;QACxB,qBAAgB,GAAhB,gBAAgB,CAAyB;IAChD,CAAC;IAEJ;;;;;;;;;;;;;;;;OAgBG;IACK,KAAK,CAAC,wBAAwB,CACpC,SAAmB,EACnB,SAAiB;QAEjB,yEAAyE;QACzE,qEAAqE;QACrE,sEAAsE;QACtE,gDAAgD;QAChD,IAAI,CAAC,GAAG,SAAS,CAAC;QAClB,IAAI,MAAM,GAAG,IAAI,GAAG,EAA8B,CAAC;QACnD,SAAS,CAAC;YACR,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;YACxD,MAAM,GAAG,IAAI,GAAG,EAA8B,CAAC;YAC/C,KAAK,MAAM,CAAC,IAAI,GAAG,EAAE,CAAC;gBACpB,MAAM,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;gBACzC,IAAI,CAAC,SAAS,IAAI,CAAC,CAAC,KAAK,GAAG,SAAS,CAAC,KAAK;oBAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;YACzE,CAAC;YACD,mEAAmE;YACnE,+DAA+D;YAC/D,kEAAkE;YAClE,gEAAgE;YAChE,4DAA4D;YAC5D,0CAA0C;YAC1C,IAAI,MAAM,CAAC,IAAI,IAAI,SAAS,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC;gBAAE,MAAM;YACtD,CAAC,IAAI,CAAC,CAAC;QACT,CAAC;QACD,OAAO,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;IACvE,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,KAAa,EAAE,OAAsB;QAChD,oEAAoE;QACpE,qEAAqE;QACrE,wEAAwE;QACxE,sEAAsE;QACtE,wEAAwE;QACxE,iCAAiC;QACjC,MAAM,OAAO,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC;QACtC,IAAI,CAAC,OAAO;YAAE,OAAO,EAAE,CAAC;QAExB,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,EAAE,CAAC;QAClC,0EAA0E;QAC1E,wEAAwE;QACxE,4EAA4E;QAC5E,0EAA0E;QAC1E,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;YAC1C,MAAM,IAAI,KAAK,CAAC,yCAAyC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QACpF,CAAC;QACD,MAAM,SAAS,GAAkB,OAAO,CAAC,SAAS,IAAI,OAAO,CAAC;QAC9D,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC;QAElC,QAAQ,OAAO,CAAC,IAAI,EAAE,CAAC;YACrB,KAAK,SAAS;gBACZ,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,OAAO,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;YAC1G,KAAK,UAAU;gBACb,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,CAAC,cAAc,CAAC,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,OAAO,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;YAC9G,KAAK,QAAQ;gBACX,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,OAAO,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;QAC9G,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,UAAU,CACtB,KAAa,EACb,KAAa,EACb,SAAwB,EACxB,QAA4B,EAC5B,OAAsB;QAEtB,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,yBAAyB,CAAC;QACrE,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC,CAAC;QAClE,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,EAAE;YAChD,KAAK,EAAE,KAAK,GAAG,SAAS;YACxB,SAAS;YACT,QAAQ;SACT,CAAC,CAAC;QACH,OAAO,OAAO;aACX,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,KAAK,EAAE,cAAc,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,KAAK,CAAC,CAAC;aAC9E,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACX,EAAE,EAAE,CAAC,CAAC,EAAE;YACR,QAAQ,EAAE,CAAC,CAAC,QAAQ;YACpB,UAAU,EAAE,CAAC,CAAC,UAAU;YACxB,SAAS,EAAE,CAAC,CAAC,SAAS;YACtB,SAAS,EAAE,CAAC,CAAC,KAAK;YAClB,SAAS,EAAE,CAAC,CAAC,KAAK;YAClB,OAAO,EAAE,CAAC,CAAC,OAAO;YAClB,OAAO,EAAE,CAAC,CAAC,OAAO;YAClB,iBAAiB,EAAE,CAAC,CAAC,iBAAiB;YACtC,KAAK,EAAE,CAAC,CAAC,KAAK;YACd,WAAW,EAAE,CAAC,CAAC,WAAW;YAC1B,QAAQ,EAAE,CAAC,CAAC,QAAQ;YACpB,MAAM,EAAE,CAAC,CAAC,MAAM;YAChB,YAAY,EAAE,CAAC,CAAC,YAAY;YAC5B,UAAU,EAAE,CAAC,CAAC,UAAU;YACxB,MAAM,EAAE,MAAe;SACxB,CAAC,CAAC,CAAC;IACR,CAAC;IAEO,KAAK,CAAC,cAAc,CAC1B,KAAa,EACb,KAAa,EACb,SAAwB,EACxB,QAA4B,EAC5B,OAAsB;QAEtB,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC3B,MAAM,IAAI,KAAK,CAAC,+CAA+C,CAAC,CAAC;QACnE,CAAC;QACD,MAAM,cAAc,GAAG,OAAO,CAAC,cAAc,IAAI,wBAAwB,CAAC;QAC1E,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAC3D,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,wBAAwB,CAAC,SAAS,EAAE,KAAK,GAAG,SAAS,CAAC,CAAC;QAC9E,MAAM,OAAO,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,gBAAgB,CAAC,CAAC,CAAC,KAAK,EAAE,cAAc,CAAC,CAAC,CAAC;QAE7E,MAAM,UAAU,GAAG,iBAAiB,CAAC,KAAK,CAAC,CAAC;QAE5C,MAAM,aAAa,GAAmB,EAAE,CAAC;QACzC,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;YACxB,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YAClD,uEAAuE;YACvE,yEAAyE;YACzE,yEAAyE;YACzE,wEAAwE;YACxE,SAAS;YACT,IAAI,CAAC,GAAG;gBAAE,SAAS;YACnB,aAAa,CAAC,IAAI,CAAC;gBACjB,EAAE,EAAE,CAAC,CAAC,EAAE;gBACR,QAAQ,EAAE,CAAC,CAAC,QAAQ;gBACpB,UAAU,EAAE,CAAC,CAAC,UAAU;gBACxB,SAAS,EAAE,GAAG,EAAE,SAAS,IAAI,CAAC;gBAC9B,WAAW,EAAE,CAAC,CAAC,KAAK;gBACpB,SAAS,EAAE,CAAC,CAAC,KAAK;gBAClB,OAAO,EAAE,GAAG,EAAE,OAAO;gBACrB,iBAAiB,EAAE,GAAG,EAAE,iBAAiB;gBACzC,KAAK,EAAE,GAAG,EAAE,KAAK;gBACjB,WAAW,EAAE,GAAG,EAAE,WAAW;gBAC7B,QAAQ,EAAE,GAAG,EAAE,QAAQ;gBACvB,MAAM,EAAE,GAAG,EAAE,MAAM;gBACnB,YAAY,EAAE,GAAG,EAAE,YAAY;gBAC/B,UAAU,EAAE,GAAG,EAAE,UAAU;gBAC3B,OAAO,EAAE,GAAG;oBACV,CAAC,CAAC,MAAM,YAAY,CAChB,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,OAAO,EAAE,GAAG,CAAC,OAAO,EAAE,SAAS,EAAE,GAAG,CAAC,SAAS,EAAE,EACxE,UAAU,EACV,SAAS,EACT,QAAQ,CACT;oBACH,CAAC,CAAC,SAAS;gBACb,MAAM,EAAE,QAAiB;aAC1B,CAAC,CAAC;QACL,CAAC;QACD,OAAO,aAAa,CAAC;IACvB,CAAC;IAEO,KAAK,CAAC,YAAY,CACxB,KAAa,EACb,KAAa,EACb,SAAwB,EACxB,QAA4B,EAC5B,OAAsB;QAEtB,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,yBAAyB,CAAC;QACrE,MAAM,cAAc,GAAG,OAAO,CAAC,cAAc,IAAI,wBAAwB,CAAC;QAE1E,MAAM,CAAC,OAAO,EAAE,SAAS,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YAC7C,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,EAAE,EAAE,KAAK,EAAE,KAAK,GAAG,SAAS,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC;YAC9E,IAAI,CAAC,gBAAgB;gBACnB,CAAC,CAAC,IAAI,CAAC,wBAAwB,CAAC,MAAM,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,KAAK,GAAG,SAAS,CAAC;gBAC5F,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;SACxB,CAAC,CAAC;QAEH,0EAA0E;QAC1E,iEAAiE;QACjE,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC,CAAC;QAClE,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CACvC,iBAAiB,CAAC,KAAK,EAAE,cAAc,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,KAAK,CAAC,CAChE,CAAC;QACF,MAAM,aAAa,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,gBAAgB,CAAC,CAAC,CAAC,KAAK,EAAE,cAAc,CAAC,CAAC,CAAC;QAEzF,4EAA4E;QAC5E,sEAAsE;QACtE,2EAA2E;QAC3E,uEAAuE;QACvE,2CAA2C;QAC3C,MAAM,MAAM,GAAG,IAAI,GAAG,EAGnB,CAAC;QAEJ,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE;YAC/B,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,EAAE;gBACrB,EAAE,EAAE,CAAC,CAAC,EAAE;gBACR,QAAQ,EAAE,CAAC,CAAC,QAAQ;gBACpB,UAAU,EAAE,CAAC,CAAC,UAAU;gBACxB,SAAS,EAAE,CAAC,CAAC,SAAS;gBACtB,SAAS,EAAE,CAAC,CAAC,KAAK;gBAClB,QAAQ,EAAE,KAAK,GAAG,CAAC;gBACnB,OAAO,EAAE,CAAC,CAAC,OAAO;gBAClB,OAAO,EAAE,CAAC,CAAC,OAAO;gBAClB,iBAAiB,EAAE,CAAC,CAAC,iBAAiB;gBACtC,KAAK,EAAE,CAAC,CAAC,KAAK;gBACd,WAAW,EAAE,CAAC,CAAC,WAAW;gBAC1B,QAAQ,EAAE,CAAC,CAAC,QAAQ;gBACpB,MAAM,EAAE,CAAC,CAAC,MAAM;gBAChB,YAAY,EAAE,CAAC,CAAC,YAAY;gBAC5B,UAAU,EAAE,CAAC,CAAC,UAAU;gBACxB,MAAM,EAAE,MAAM;aACf,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,MAAM,UAAU,GAAG,iBAAiB,CAAC,KAAK,CAAC,CAAC;QAE5C,KAAK,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,IAAI,aAAa,CAAC,OAAO,EAAE,EAAE,CAAC;YACjD,MAAM,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;YACxC,IAAI,QAAQ,EAAE,CAAC;gBACb,uEAAuE;gBACvE,uEAAuE;gBACvE,yBAAyB;gBACzB,QAAQ,CAAC,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,WAAW,IAAI,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC;gBACpE,QAAQ,CAAC,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,UAAU,IAAI,QAAQ,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;gBAC3E,QAAQ,CAAC,MAAM,GAAG,QAAQ,CAAC;YAC7B,CAAC;iBAAM,CAAC;gBACN,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;gBAClD,oEAAoE;gBACpE,IAAI,CAAC,GAAG;oBAAE,SAAS;gBACnB,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,EAAE;oBACrB,EAAE,EAAE,CAAC,CAAC,EAAE;oBACR,QAAQ,EAAE,CAAC,CAAC,QAAQ;oBACpB,UAAU,EAAE,CAAC,CAAC,UAAU;oBACxB,SAAS,EAAE,GAAG,EAAE,SAAS,IAAI,CAAC;oBAC9B,WAAW,EAAE,CAAC,CAAC,KAAK;oBACpB,UAAU,EAAE,KAAK,GAAG,CAAC;oBACrB,OAAO,EAAE,GAAG,EAAE,OAAO;oBACrB,iBAAiB,EAAE,GAAG,EAAE,iBAAiB;oBACzC,KAAK,EAAE,GAAG,EAAE,KAAK;oBACjB,WAAW,EAAE,GAAG,EAAE,WAAW;oBAC7B,QAAQ,EAAE,GAAG,EAAE,QAAQ;oBACvB,MAAM,EAAE,GAAG,EAAE,MAAM;oBACnB,YAAY,EAAE,GAAG,EAAE,YAAY;oBAC/B,UAAU,EAAE,GAAG,EAAE,UAAU;oBAC3B,OAAO,EAAE,GAAG;wBACV,CAAC,CAAC,MAAM,YAAY,CAChB,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,OAAO,EAAE,GAAG,CAAC,OAAO,EAAE,SAAS,EAAE,GAAG,CAAC,SAAS,EAAE,EACxE,UAAU,EACV,SAAS,EACT,QAAQ,CACT;wBACH,CAAC,CAAC,SAAS;oBACb,MAAM,EAAE,QAAQ;iBACjB,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;YACpD,IAAI,QAAQ,GAAG,CAAC,CAAC;YACjB,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC;gBACf,QAAQ,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC;gBACrC,IAAI,CAAC,CAAC,QAAQ,KAAK,CAAC;oBAAE,QAAQ,IAAI,IAAI,CAAC;qBAClC,IAAI,CAAC,CAAC,QAAQ,IAAI,CAAC;oBAAE,QAAQ,IAAI,IAAI,CAAC;YAC7C,CAAC;YACD,IAAI,CAAC,CAAC,UAAU,EAAE,CAAC;gBACjB,QAAQ,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC;gBACvC,IAAI,CAAC,CAAC,UAAU,KAAK,CAAC;oBAAE,QAAQ,IAAI,IAAI,CAAC;qBACpC,IAAI,CAAC,CAAC,UAAU,IAAI,CAAC;oBAAE,QAAQ,IAAI,IAAI,CAAC;YAC/C,CAAC;YACD,MAAM,EAAE,QAAQ,EAAE,UAAU,EAAE,GAAG,KAAK,EAAE,GAAG,CAAC,CAAC;YAC7C,OAAO,EAAE,GAAG,KAAK,EAAE,QAAQ,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC;QACrD,CAAC,CAAC,CAAC;QAEH,OAAO,OAAO,CAAC;IACjB,CAAC;IAED;;;;;OAKG;IACK,MAAM,CACZ,OAAuB,EACvB,KAAa,EACb,OAAsB;QAEtB,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;QAEtC,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;YACjC,MAAM,IAAI,GAAG,CAAC,CAAC,SAAS,IAAI,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,SAAS,IAAI,CAAC,CAAC,WAAW,IAAI,CAAC,CAAC;YAC5E,MAAM,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;YAChE,OAAO,EAAE,GAAG,CAAC,EAAE,SAAS,EAAE,IAAI,EAAE,SAAS,EAAE,UAAU,EAAE,IAAI,GAAG,SAAS,EAAE,CAAC;QAC5E,CAAC,CAAC,CAAC;QAEH,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC;QAErD,sEAAsE;QACtE,+DAA+D;QAC/D,wEAAwE;QACxE,uEAAuE;QACvE,gEAAgE;QAChE,wEAAwE;QACxE,mEAAmE;QACnE,sEAAsE;QACtE,OAAO,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,UAAU,EAAE,GAAG,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;IACnE,CAAC;CACF"}
|
package/dist/search/snippet.d.ts
CHANGED
|
@@ -16,6 +16,17 @@ export declare function extractQueryTerms(query: string): string[];
|
|
|
16
16
|
* the output programmatically, so escape codes would just be noise, and
|
|
17
17
|
* literal `**` clutter is exactly the bug being fixed). */
|
|
18
18
|
export type HighlightMode = "ansi" | "plain";
|
|
19
|
+
/**
|
|
20
|
+
* Does `text` contain `term` as a word?
|
|
21
|
+
*
|
|
22
|
+
* The single definition of "contains this term", shared by snippet
|
|
23
|
+
* highlighting, the relevance gate, and the document-frequency count behind
|
|
24
|
+
* it. Two implementations of this question disagree silently: a substring
|
|
25
|
+
* test credited a document about a "dialog box" with the term "log" while
|
|
26
|
+
* reporting that "log" appeared in no document at all, which handed it
|
|
27
|
+
* maximum IDF weight and inverted the ranking.
|
|
28
|
+
*/
|
|
29
|
+
export declare function containsTerm(text: string, term: string): boolean;
|
|
19
30
|
/** Context needed to resolve a chunk-local match position against the real,
|
|
20
31
|
* full source file rather than just the chunk's own text (fix #4 in the
|
|
21
32
|
* module doc-comment). `chunkStartLine` is the chunk's 1-based start line
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"snippet.d.ts","sourceRoot":"","sources":["../../src/search/snippet.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"snippet.d.ts","sourceRoot":"","sources":["../../src/search/snippet.ts"],"names":[],"mappings":"AAoFA;;;;;;;GAOG;AACH,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,EAAE,CASzD;AAMD;;;;;;;8DAO8D;AAC9D,MAAM,MAAM,aAAa,GAAG,MAAM,GAAG,OAAO,CAAC;AA+C7C;;;;;;;;;GASG;AACH,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAGhE;AAsMD;;;;kCAIkC;AAClC,MAAM,WAAW,WAAW;IAC1B,WAAW,EAAE,MAAM,CAAC;IACpB,cAAc,EAAE,MAAM,CAAC;CACxB;AAED;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,eAAe,CAC7B,OAAO,EAAE,MAAM,EACf,KAAK,EAAE,MAAM,EAAE,EACf,aAAa,GAAE,aAAuB,EACtC,WAAW,CAAC,EAAE,WAAW,GACxB,MAAM,CA4BR;AAuBD,gEAAgE;AAChE,MAAM,WAAW,UAAU;IACzB,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,sEAAsE;IACtE,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;;;;;;;;;;GAWG;AACH,wBAAsB,YAAY,CAChC,GAAG,EAAE,UAAU,EACf,KAAK,EAAE,MAAM,EAAE,EACf,aAAa,EAAE,aAAa,EAC5B,QAAQ,CAAC,EAAE,MAAM,GAChB,OAAO,CAAC,MAAM,CAAC,CAiBjB"}
|