@amsterdamdatalabs/enact-docs-search 0.1.2 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +29 -4
- package/dist/cli.js +158 -53
- package/dist/cli.js.map +1 -1
- package/dist/indexing/chunker.d.ts +6 -1
- package/dist/indexing/chunker.d.ts.map +1 -1
- package/dist/indexing/chunker.js +17 -11
- package/dist/indexing/chunker.js.map +1 -1
- package/dist/indexing/embedding-text.d.ts +22 -0
- package/dist/indexing/embedding-text.d.ts.map +1 -0
- package/dist/indexing/embedding-text.js +27 -0
- package/dist/indexing/embedding-text.js.map +1 -0
- package/dist/indexing/frontmatter.d.ts +48 -0
- package/dist/indexing/frontmatter.d.ts.map +1 -0
- package/dist/indexing/frontmatter.js +162 -0
- package/dist/indexing/frontmatter.js.map +1 -0
- package/dist/indexing/indexing-service.d.ts +1 -4
- package/dist/indexing/indexing-service.d.ts.map +1 -1
- package/dist/indexing/indexing-service.js +24 -12
- package/dist/indexing/indexing-service.js.map +1 -1
- package/dist/search/authority.d.ts +31 -0
- package/dist/search/authority.d.ts.map +1 -0
- package/dist/search/authority.js +50 -0
- package/dist/search/authority.js.map +1 -0
- package/dist/search/relevance.d.ts +80 -0
- package/dist/search/relevance.d.ts.map +1 -0
- package/dist/search/relevance.js +119 -0
- package/dist/search/relevance.js.map +1 -0
- package/dist/search/search-service.d.ts +44 -2
- package/dist/search/search-service.d.ts.map +1 -1
- package/dist/search/search-service.js +181 -23
- package/dist/search/search-service.js.map +1 -1
- package/dist/search/snippet.d.ts +11 -0
- package/dist/search/snippet.d.ts.map +1 -1
- package/dist/search/snippet.js +102 -17
- package/dist/search/snippet.js.map +1 -1
- package/dist/storage/document-store.d.ts +47 -5
- package/dist/storage/document-store.d.ts.map +1 -1
- package/dist/storage/document-store.js +94 -16
- package/dist/storage/document-store.js.map +1 -1
- package/dist/storage/persistence.d.ts +6 -10
- package/dist/storage/persistence.d.ts.map +1 -1
- package/dist/storage/persistence.js +20 -20
- package/dist/storage/persistence.js.map +1 -1
- package/dist/storage/vector-store.d.ts +2 -3
- package/dist/storage/vector-store.d.ts.map +1 -1
- package/dist/storage/vector-store.js +3 -11
- package/dist/storage/vector-store.js.map +1 -1
- package/dist/types.d.ts +2 -7
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js +2 -20
- package/dist/types.js.map +1 -1
- package/package.json +2 -2
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Embedding input construction.
|
|
3
|
+
*
|
|
4
|
+
* A bare chunk embeds as an orphan: the vector knows what the paragraph says
|
|
5
|
+
* but not which document it belongs to. Prefixing the document's title,
|
|
6
|
+
* description, and heading path puts that identity inside the vector, so the
|
|
7
|
+
* semantic half of the hybrid agrees with the field-weighted keyword half
|
|
8
|
+
* instead of competing with it.
|
|
9
|
+
*/
|
|
10
|
+
export interface EmbeddingTextInput {
|
|
11
|
+
title?: string;
|
|
12
|
+
description?: string;
|
|
13
|
+
headingBreadcrumb?: string[];
|
|
14
|
+
content: string;
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* Build the text that is actually embedded for a chunk. Falls back to the
|
|
18
|
+
* bare body when a document carries no frontmatter, so untitled files are
|
|
19
|
+
* neither enriched nor penalised.
|
|
20
|
+
*/
|
|
21
|
+
export declare function buildEmbeddingText(input: EmbeddingTextInput): string;
|
|
22
|
+
//# sourceMappingURL=embedding-text.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"embedding-text.d.ts","sourceRoot":"","sources":["../../src/indexing/embedding-text.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,MAAM,WAAW,kBAAkB;IACjC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC7B,OAAO,EAAE,MAAM,CAAC;CACjB;AAED;;;;GAIG;AACH,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,kBAAkB,GAAG,MAAM,CAYpE"}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Embedding input construction.
|
|
3
|
+
*
|
|
4
|
+
* A bare chunk embeds as an orphan: the vector knows what the paragraph says
|
|
5
|
+
* but not which document it belongs to. Prefixing the document's title,
|
|
6
|
+
* description, and heading path puts that identity inside the vector, so the
|
|
7
|
+
* semantic half of the hybrid agrees with the field-weighted keyword half
|
|
8
|
+
* instead of competing with it.
|
|
9
|
+
*/
|
|
10
|
+
/**
|
|
11
|
+
* Build the text that is actually embedded for a chunk. Falls back to the
|
|
12
|
+
* bare body when a document carries no frontmatter, so untitled files are
|
|
13
|
+
* neither enriched nor penalised.
|
|
14
|
+
*/
|
|
15
|
+
export function buildEmbeddingText(input) {
|
|
16
|
+
const parts = [];
|
|
17
|
+
const heading = [input.title, input.description].filter(Boolean).join(" — ");
|
|
18
|
+
if (heading)
|
|
19
|
+
parts.push(heading);
|
|
20
|
+
const breadcrumb = (input.headingBreadcrumb ?? []).filter(Boolean).join(" > ");
|
|
21
|
+
if (breadcrumb)
|
|
22
|
+
parts.push(breadcrumb);
|
|
23
|
+
if (parts.length === 0)
|
|
24
|
+
return input.content;
|
|
25
|
+
return `${parts.join("\n")}\n\n${input.content}`;
|
|
26
|
+
}
|
|
27
|
+
//# sourceMappingURL=embedding-text.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"embedding-text.js","sourceRoot":"","sources":["../../src/indexing/embedding-text.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AASH;;;;GAIG;AACH,MAAM,UAAU,kBAAkB,CAAC,KAAyB;IAC1D,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,MAAM,OAAO,GAAG,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,WAAW,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC7E,IAAI,OAAO;QAAE,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAEjC,MAAM,UAAU,GAAG,CAAC,KAAK,CAAC,iBAAiB,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC/E,IAAI,UAAU;QAAE,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IAEvC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC,OAAO,CAAC;IAE7C,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,KAAK,CAAC,OAAO,EAAE,CAAC;AACnD,CAAC"}
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* YAML frontmatter parsing for indexed documents.
|
|
3
|
+
*
|
|
4
|
+
* Frontmatter is metadata, not prose. Indexing it as body text pollutes both
|
|
5
|
+
* the keyword index and the embedding vector, and leaks raw YAML into result
|
|
6
|
+
* snippets. This module splits the two apart once, at parse time, so the rest
|
|
7
|
+
* of the pipeline only ever sees a body plus typed fields.
|
|
8
|
+
*
|
|
9
|
+
* The parser deliberately covers a flat scalar subset -- the fields that drive
|
|
10
|
+
* ranking and display. List and nested values are recognised well enough to be
|
|
11
|
+
* skipped, never well enough to be relied on.
|
|
12
|
+
*/
|
|
13
|
+
/** The document-level fields that ranking and display depend on. */
|
|
14
|
+
export interface DocMeta {
|
|
15
|
+
title?: string;
|
|
16
|
+
description?: string;
|
|
17
|
+
status?: string;
|
|
18
|
+
documentType?: string;
|
|
19
|
+
/** ISO-8601 timestamp as written in the source, unparsed. */
|
|
20
|
+
lastUpdate?: string;
|
|
21
|
+
}
|
|
22
|
+
export interface ParsedDocument {
|
|
23
|
+
meta: DocMeta;
|
|
24
|
+
/** The document with its frontmatter block removed. */
|
|
25
|
+
body: string;
|
|
26
|
+
/**
|
|
27
|
+
* Lines the frontmatter block occupied, delimiters included. Callers must
|
|
28
|
+
* add this to any line number derived from `body`, or every citation this
|
|
29
|
+
* tool prints will be short by exactly this much.
|
|
30
|
+
*/
|
|
31
|
+
frontmatterLineCount: number;
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* Strip a leading YAML frontmatter block and return it as typed fields.
|
|
35
|
+
*
|
|
36
|
+
* A `---` that is not on the very first line is a horizontal rule, not a
|
|
37
|
+
* frontmatter opener, and is left in the body untouched.
|
|
38
|
+
*/
|
|
39
|
+
/**
|
|
40
|
+
* One definition of "line ending", shared with the snippet layer.
|
|
41
|
+
*
|
|
42
|
+
* Two passes that normalise differently disagree about where a line is: the
|
|
43
|
+
* snippet layer split on "\n" only, so a CR-only document was a single line
|
|
44
|
+
* and blanking its frontmatter erased the entire file.
|
|
45
|
+
*/
|
|
46
|
+
export declare function normalizeLineEndings(content: string): string;
|
|
47
|
+
export declare function parseFrontmatter(content: string): ParsedDocument;
|
|
48
|
+
//# sourceMappingURL=frontmatter.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"frontmatter.d.ts","sourceRoot":"","sources":["../../src/indexing/frontmatter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,oEAAoE;AACpE,MAAM,WAAW,OAAO;IACtB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,6DAA6D;IAC7D,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,OAAO,CAAC;IACd,uDAAuD;IACvD,IAAI,EAAE,MAAM,CAAC;IACb;;;;OAIG;IACH,oBAAoB,EAAE,MAAM,CAAC;CAC9B;AAaD;;;;;GAKG;AACH;;;;;;GAMG;AACH,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAE5D;AAED,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,MAAM,GAAG,cAAc,CAmChE"}
|
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* YAML frontmatter parsing for indexed documents.
|
|
3
|
+
*
|
|
4
|
+
* Frontmatter is metadata, not prose. Indexing it as body text pollutes both
|
|
5
|
+
* the keyword index and the embedding vector, and leaks raw YAML into result
|
|
6
|
+
* snippets. This module splits the two apart once, at parse time, so the rest
|
|
7
|
+
* of the pipeline only ever sees a body plus typed fields.
|
|
8
|
+
*
|
|
9
|
+
* The parser deliberately covers a flat scalar subset -- the fields that drive
|
|
10
|
+
* ranking and display. List and nested values are recognised well enough to be
|
|
11
|
+
* skipped, never well enough to be relied on.
|
|
12
|
+
*/
|
|
13
|
+
const DELIMITER = "---";
|
|
14
|
+
/** Source key -> DocMeta field. Keys absent here are parsed and discarded. */
|
|
15
|
+
const FIELD_MAP = {
|
|
16
|
+
title: "title",
|
|
17
|
+
description: "description",
|
|
18
|
+
status: "status",
|
|
19
|
+
document_type: "documentType",
|
|
20
|
+
last_update: "lastUpdate",
|
|
21
|
+
};
|
|
22
|
+
/**
|
|
23
|
+
* Strip a leading YAML frontmatter block and return it as typed fields.
|
|
24
|
+
*
|
|
25
|
+
* A `---` that is not on the very first line is a horizontal rule, not a
|
|
26
|
+
* frontmatter opener, and is left in the body untouched.
|
|
27
|
+
*/
|
|
28
|
+
/**
|
|
29
|
+
* One definition of "line ending", shared with the snippet layer.
|
|
30
|
+
*
|
|
31
|
+
* Two passes that normalise differently disagree about where a line is: the
|
|
32
|
+
* snippet layer split on "\n" only, so a CR-only document was a single line
|
|
33
|
+
* and blanking its frontmatter erased the entire file.
|
|
34
|
+
*/
|
|
35
|
+
export function normalizeLineEndings(content) {
|
|
36
|
+
return content.replace(/\r\n/g, "\n").replace(/\r/g, "\n");
|
|
37
|
+
}
|
|
38
|
+
export function parseFrontmatter(content) {
|
|
39
|
+
// A leading UTF-8 BOM is invisible in an editor but fails `startsWith`
|
|
40
|
+
// below, which used to disable frontmatter detection outright: the whole
|
|
41
|
+
// YAML block fell through and got indexed as prose, embedding included.
|
|
42
|
+
// Strip it in the same pass as CRLF. It removes one character, not a
|
|
43
|
+
// line, so it cannot shift `frontmatterLineCount` or the body's alignment.
|
|
44
|
+
const normalized = content
|
|
45
|
+
.replace(/\r\n/g, "\n")
|
|
46
|
+
.replace(/\r/g, "\n")
|
|
47
|
+
.replace(/^\uFEFF/, "");
|
|
48
|
+
if (!normalized.startsWith(`${DELIMITER}\n`)) {
|
|
49
|
+
return { meta: {}, body: content, frontmatterLineCount: 0 };
|
|
50
|
+
}
|
|
51
|
+
const lines = normalized.split("\n");
|
|
52
|
+
let closing = -1;
|
|
53
|
+
for (let i = 1; i < lines.length; i++) {
|
|
54
|
+
if (lines[i].trimEnd() === DELIMITER) {
|
|
55
|
+
closing = i;
|
|
56
|
+
break;
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
// An unterminated block is not frontmatter. Treat the whole file as body
|
|
60
|
+
// rather than silently swallowing it.
|
|
61
|
+
if (closing === -1) {
|
|
62
|
+
return { meta: {}, body: content, frontmatterLineCount: 0 };
|
|
63
|
+
}
|
|
64
|
+
const meta = parseScalars(lines.slice(1, closing));
|
|
65
|
+
const frontmatterLineCount = closing + 1;
|
|
66
|
+
const body = lines.slice(frontmatterLineCount).join("\n");
|
|
67
|
+
return { meta, body, frontmatterLineCount };
|
|
68
|
+
}
|
|
69
|
+
// WHY single-line only: an earlier version treated a value whose first
|
|
70
|
+
// character was a quote but which didn't close on that same line as the
|
|
71
|
+
// start of a multi-line quoted scalar, and kept consuming following lines
|
|
72
|
+
// until one ended with the matching quote. That is not what a genuinely
|
|
73
|
+
// unterminated quote in hand-written frontmatter means -- it means the
|
|
74
|
+
// author forgot to close it. Treating it as "still open" ate every
|
|
75
|
+
// subsequent key (title/description/status/last_update all folded into one
|
|
76
|
+
// value) until either a later line happened to end with that quote
|
|
77
|
+
// character or the block ran out. Real multi-line YAML scalars use `|`/`>`
|
|
78
|
+
// block indicators or explicit indentation continuation, not an unterminated
|
|
79
|
+
// flow quote, so this parser -- which only ever covers the flat scalar
|
|
80
|
+
// subset -- does not need to support that shape at all. Parsing each line
|
|
81
|
+
// independently instead means a bad quote can only ever corrupt its own
|
|
82
|
+
// key's value, never a neighbour's: `no silent drops` for the fields that
|
|
83
|
+
// drive ranking.
|
|
84
|
+
function parseScalars(lines) {
|
|
85
|
+
const meta = {};
|
|
86
|
+
for (const line of lines) {
|
|
87
|
+
// List items and nested mappings belong to the key above them; skip.
|
|
88
|
+
if (/^\s/.test(line) || line.trimStart().startsWith("- "))
|
|
89
|
+
continue;
|
|
90
|
+
const separator = line.indexOf(":");
|
|
91
|
+
if (separator === -1)
|
|
92
|
+
continue;
|
|
93
|
+
const key = line.slice(0, separator).trim();
|
|
94
|
+
const field = FIELD_MAP[key];
|
|
95
|
+
if (!field)
|
|
96
|
+
continue;
|
|
97
|
+
const raw = line.slice(separator + 1).trim();
|
|
98
|
+
if (!raw)
|
|
99
|
+
continue;
|
|
100
|
+
meta[field] = unquote(raw);
|
|
101
|
+
}
|
|
102
|
+
return meta;
|
|
103
|
+
}
|
|
104
|
+
/**
|
|
105
|
+
* Parse `value` as a clean single-line YAML quoted scalar, or return `null`
|
|
106
|
+
* if it isn't one.
|
|
107
|
+
*
|
|
108
|
+
* YAML's two quote styles escape differently, and this walks the value
|
|
109
|
+
* applying whichever rule matches `quote`:
|
|
110
|
+
* - double quotes escape with a backslash (`\"`, `\\`);
|
|
111
|
+
* - single quotes escape a literal quote by doubling it (`''`), and have
|
|
112
|
+
* no backslash-escaping at all -- a backslash is just a character.
|
|
113
|
+
* An escaped quote is skipped, not counted as a delimiter, so a value like
|
|
114
|
+
* `"\"I need X\" the lookup"` (a description that itself quotes a phrase --
|
|
115
|
+
* a real shape in this project's docs corpus) parses cleanly even though it
|
|
116
|
+
* contains inner quote characters.
|
|
117
|
+
*
|
|
118
|
+
* What still returns `null`, deliberately: a value that only *starts* with
|
|
119
|
+
* a quoted phrase (`"why" not the what`) or carries an unescaped embedded
|
|
120
|
+
* quoted phrase (`"a" and "b"`) hits an unescaped quote before the value
|
|
121
|
+
* ends, which is ambiguous -- stripping first/last characters there would
|
|
122
|
+
* guess which quote is "the" closing one, and this parser does not guess.
|
|
123
|
+
* An unterminated value (`"Broken`) never finds a closing quote at all.
|
|
124
|
+
* Either way the raw value is left untouched by the caller.
|
|
125
|
+
*/
|
|
126
|
+
function parseQuotedScalar(value, quote) {
|
|
127
|
+
if (value.length < 2 || value[0] !== quote)
|
|
128
|
+
return null;
|
|
129
|
+
let content = "";
|
|
130
|
+
let i = 1;
|
|
131
|
+
while (i < value.length) {
|
|
132
|
+
const ch = value[i];
|
|
133
|
+
if (quote === '"' && ch === "\\" && i + 1 < value.length) {
|
|
134
|
+
const next = value[i + 1];
|
|
135
|
+
content += next === '"' || next === "\\" ? next : `\\${next}`;
|
|
136
|
+
i += 2;
|
|
137
|
+
continue;
|
|
138
|
+
}
|
|
139
|
+
if (quote === "'" && ch === "'" && value[i + 1] === "'") {
|
|
140
|
+
content += "'";
|
|
141
|
+
i += 2;
|
|
142
|
+
continue;
|
|
143
|
+
}
|
|
144
|
+
if (ch === quote) {
|
|
145
|
+
// An unescaped quote is only a legal closer at the very end.
|
|
146
|
+
return i === value.length - 1 ? content : null;
|
|
147
|
+
}
|
|
148
|
+
content += ch;
|
|
149
|
+
i++;
|
|
150
|
+
}
|
|
151
|
+
return null; // ran off the end without an unescaped closing quote
|
|
152
|
+
}
|
|
153
|
+
function unquote(value) {
|
|
154
|
+
const quote = value[0];
|
|
155
|
+
if (quote === '"' || quote === "'") {
|
|
156
|
+
const parsed = parseQuotedScalar(value, quote);
|
|
157
|
+
if (parsed !== null)
|
|
158
|
+
return parsed;
|
|
159
|
+
}
|
|
160
|
+
return value;
|
|
161
|
+
}
|
|
162
|
+
//# sourceMappingURL=frontmatter.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"frontmatter.js","sourceRoot":"","sources":["../../src/indexing/frontmatter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAwBH,MAAM,SAAS,GAAG,KAAK,CAAC;AAExB,8EAA8E;AAC9E,MAAM,SAAS,GAAkC;IAC/C,KAAK,EAAE,OAAO;IACd,WAAW,EAAE,aAAa;IAC1B,MAAM,EAAE,QAAQ;IAChB,aAAa,EAAE,cAAc;IAC7B,WAAW,EAAE,YAAY;CAC1B,CAAC;AAEF;;;;;GAKG;AACH;;;;;;GAMG;AACH,MAAM,UAAU,oBAAoB,CAAC,OAAe;IAClD,OAAO,OAAO,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;AAC7D,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,OAAe;IAC9C,uEAAuE;IACvE,yEAAyE;IACzE,wEAAwE;IACxE,qEAAqE;IACrE,2EAA2E;IAC3E,MAAM,UAAU,GAAG,OAAO;SACvB,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC;SACtB,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC;SACpB,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;IAE1B,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,GAAG,SAAS,IAAI,CAAC,EAAE,CAAC;QAC7C,OAAO,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,oBAAoB,EAAE,CAAC,EAAE,CAAC;IAC9D,CAAC;IAED,MAAM,KAAK,GAAG,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACrC,IAAI,OAAO,GAAG,CAAC,CAAC,CAAC;IACjB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,IAAI,KAAK,CAAC,CAAC,CAAE,CAAC,OAAO,EAAE,KAAK,SAAS,EAAE,CAAC;YACtC,OAAO,GAAG,CAAC,CAAC;YACZ,MAAM;QACR,CAAC;IACH,CAAC;IAED,yEAAyE;IACzE,sCAAsC;IACtC,IAAI,OAAO,KAAK,CAAC,CAAC,EAAE,CAAC;QACnB,OAAO,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,oBAAoB,EAAE,CAAC,EAAE,CAAC;IAC9D,CAAC;IAED,MAAM,IAAI,GAAG,YAAY,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC;IACnD,MAAM,oBAAoB,GAAG,OAAO,GAAG,CAAC,CAAC;IACzC,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE1D,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,oBAAoB,EAAE,CAAC;AAC9C,CAAC;AAED,uEAAuE;AACvE,wEAAwE;AACxE,0EAA0E;AAC1E,wEAAwE;AACxE,uEAAuE;AACvE,mEAAmE;AACnE,2EAA2E;AAC3E,mEAAmE;AACnE,2EAA2E;AAC3E,6EAA6E;AAC7E,uEAAuE;AACvE,0EAA0E;AAC1E,wEAAwE;AACxE,0EAA0E;AAC1E,iBAAiB;AACjB,SAAS,YAAY,CAAC,KAAe;IACnC,MAAM,IAAI,GAAY,EAAE,CAAC;IAEzB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,qEAAqE;QACrE,IAAI,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC;YAAE,SAAS;QAEpE,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QACpC,IAAI,SAAS,KAAK,CAAC,CAAC;YAAE,SAAS;QAE/B,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,IAAI,EAAE,CAAC;QAC5C,MAAM,KAAK,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC;QAC7B,IAAI,CAAC,KAAK;YAAE,SAAS;QAErB,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAC7C,IAAI,CAAC,GAAG;YAAE,SAAS;QAEnB,IAAI,CAAC,KAAK,CAAC,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;IAC7B,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,SAAS,iBAAiB,CAAC,KAAa,EAAE,KAAgB;IACxD,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,KAAK;QAAE,OAAO,IAAI,CAAC;IAExD,IAAI,OAAO,GAAG,EAAE,CAAC;IACjB,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;QACxB,MAAM,EAAE,GAAG,KAAK,CAAC,CAAC,CAAE,CAAC;QAErB,IAAI,KAAK,KAAK,GAAG,IAAI,EAAE,KAAK,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;YACzD,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,GAAG,CAAC,CAAE,CAAC;YAC3B,OAAO,IAAI,IAAI,KAAK,GAAG,IAAI,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YAC9D,CAAC,IAAI,CAAC,CAAC;YACP,SAAS;QACX,CAAC;QAED,IAAI,KAAK,KAAK,GAAG,IAAI,EAAE,KAAK,GAAG,IAAI,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,GAAG,EAAE,CAAC;YACxD,OAAO,IAAI,GAAG,CAAC;YACf,CAAC,IAAI,CAAC,CAAC;YACP,SAAS;QACX,CAAC;QAED,IAAI,EAAE,KAAK,KAAK,EAAE,CAAC;YACjB,6DAA6D;YAC7D,OAAO,CAAC,KAAK,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;QACjD,CAAC;QAED,OAAO,IAAI,EAAE,CAAC;QACd,CAAC,EAAE,CAAC;IACN,CAAC;IAED,OAAO,IAAI,CAAC,CAAC,qDAAqD;AACpE,CAAC;AAED,SAAS,OAAO,CAAC,KAAa;IAC5B,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;IACvB,IAAI,KAAK,KAAK,GAAG,IAAI,KAAK,KAAK,GAAG,EAAE,CAAC;QACnC,MAAM,MAAM,GAAG,iBAAiB,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;QAC/C,IAAI,MAAM,KAAK,IAAI;YAAE,OAAO,MAAM,CAAC;IACrC,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC"}
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import type { Scope } from "../types.js";
|
|
2
1
|
export interface IndexProgress {
|
|
3
2
|
phase: "scanning" | "text-indexing" | "embedding" | "done";
|
|
4
3
|
filesProcessed: number;
|
|
@@ -7,7 +6,6 @@ export interface IndexProgress {
|
|
|
7
6
|
}
|
|
8
7
|
export type ProgressCallback = (progress: IndexProgress) => void;
|
|
9
8
|
export interface IndexRunResult {
|
|
10
|
-
scope: Scope;
|
|
11
9
|
filesScanned: number;
|
|
12
10
|
filesChanged: number;
|
|
13
11
|
filesRemoved: number;
|
|
@@ -17,10 +15,9 @@ export interface IndexRunResult {
|
|
|
17
15
|
}
|
|
18
16
|
export interface IndexOptions {
|
|
19
17
|
repoPath: string;
|
|
20
|
-
scope: Scope;
|
|
21
18
|
/** Skip vector embedding entirely (keyword-only index, much faster). */
|
|
22
19
|
skipEmbeddings?: boolean;
|
|
23
|
-
/** Ignore the manifest and rebuild every file
|
|
20
|
+
/** Ignore the manifest and rebuild every file from scratch. */
|
|
24
21
|
force?: boolean;
|
|
25
22
|
/** Permit automatic model download when semantic indexing is required. */
|
|
26
23
|
allowRemoteModels?: boolean;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"indexing-service.d.ts","sourceRoot":"","sources":["../../src/indexing/indexing-service.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"indexing-service.d.ts","sourceRoot":"","sources":["../../src/indexing/indexing-service.ts"],"names":[],"mappings":"AAqCA,MAAM,WAAW,aAAa;IAC5B,KAAK,EAAE,UAAU,GAAG,eAAe,GAAG,WAAW,GAAG,MAAM,CAAC;IAC3D,cAAc,EAAE,MAAM,CAAC;IACvB,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,MAAM,gBAAgB,GAAG,CAAC,QAAQ,EAAE,aAAa,KAAK,IAAI,CAAC;AAEjE,MAAM,WAAW,cAAc;IAC7B,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,eAAe,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,YAAY;IAC3B,QAAQ,EAAE,MAAM,CAAC;IACjB,wEAAwE;IACxE,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,+DAA+D;IAC/D,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,0EAA0E;IAC1E,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAC5B,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,UAAU,CAAC,EAAE,gBAAgB,CAAC;CAC/B;AAKD,wBAAsB,QAAQ,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,cAAc,CAAC,CAyN7E"}
|
|
@@ -7,20 +7,22 @@
|
|
|
7
7
|
* keeps the plugin process alive). This CLI process exits after every
|
|
8
8
|
* command, so this version is built around a manifest-driven incremental
|
|
9
9
|
* diff (new/changed/removed files by mtime) against a store that is loaded
|
|
10
|
-
* from and saved back to disk on every run
|
|
10
|
+
* from and saved back to disk on every run for each target repo.
|
|
11
11
|
*/
|
|
12
12
|
import { DocumentStore } from "../storage/document-store.js";
|
|
13
13
|
import { VectorStore } from "../storage/vector-store.js";
|
|
14
14
|
import { EmbeddingService, EMBEDDING_DIMENSIONS } from "../embeddings/embedding-service.js";
|
|
15
|
+
import { parseFrontmatter } from "./frontmatter.js";
|
|
16
|
+
import { buildEmbeddingText } from "./embedding-text.js";
|
|
15
17
|
import { Chunker } from "./chunker.js";
|
|
16
18
|
import { listMarkdownFiles, readRepoFile } from "../adapter/repo-walker.js";
|
|
17
19
|
import { ensureRepoCache, readManifest, writeManifest, emptyManifest, readDocumentStoreExport, writeDocumentStoreExport, readVectorSidecar, writeVectorSidecar, removeVectorCache, vectorIndexExists, repoCacheDir, repoModelCacheDir, paths, EMBEDDING_MODEL_ID, } from "../storage/persistence.js";
|
|
18
|
-
import {
|
|
20
|
+
import { DOCS_ROOT } from "../types.js";
|
|
19
21
|
const CHUNK_SIZE = 3200;
|
|
20
22
|
const CHUNK_OVERLAP = 0.15;
|
|
21
23
|
export async function runIndex(options) {
|
|
22
24
|
const start = Date.now();
|
|
23
|
-
const { repoPath,
|
|
25
|
+
const { repoPath, onProgress } = options;
|
|
24
26
|
const cacheDir = repoCacheDir(repoPath);
|
|
25
27
|
await ensureRepoCache(repoPath);
|
|
26
28
|
const persistedManifest = await readManifest(cacheDir);
|
|
@@ -33,8 +35,7 @@ export async function runIndex(options) {
|
|
|
33
35
|
await removeVectorCache(cacheDir);
|
|
34
36
|
}
|
|
35
37
|
onProgress?.({ phase: "scanning", filesProcessed: 0, totalFiles: 0 });
|
|
36
|
-
const
|
|
37
|
-
const files = await listMarkdownFiles(repoPath, roots, { exclude: options.exclude });
|
|
38
|
+
const files = await listMarkdownFiles(repoPath, [DOCS_ROOT], { exclude: options.exclude });
|
|
38
39
|
const docStore = new DocumentStore();
|
|
39
40
|
await docStore.initialize(cacheRequiresRebuild ? undefined : (await readDocumentStoreExport(cacheDir)) ?? undefined);
|
|
40
41
|
let vectorStore = null;
|
|
@@ -67,11 +68,10 @@ export async function runIndex(options) {
|
|
|
67
68
|
if (textIsStale || vectorIsStale)
|
|
68
69
|
filesToProcess.push(f);
|
|
69
70
|
}
|
|
70
|
-
// Files that were indexed
|
|
71
|
+
// Files that were indexed previously but no longer exist under docs/.
|
|
71
72
|
const removed = Object.keys(manifest.files).filter((p) => {
|
|
72
|
-
const inScope = roots.some((r) => p === r || p.startsWith(r + "/"));
|
|
73
73
|
const entry = manifest.files[p];
|
|
74
|
-
return
|
|
74
|
+
return !currentPaths.has(p) &&
|
|
75
75
|
(entry.textMtimeMs !== null || entry.vectorMtimeMs !== null || entry.vectorStale);
|
|
76
76
|
});
|
|
77
77
|
onProgress?.({
|
|
@@ -113,7 +113,10 @@ export async function runIndex(options) {
|
|
|
113
113
|
await docStore.deleteByFilePath(file.path);
|
|
114
114
|
if (vectorStore)
|
|
115
115
|
await vectorStore.deleteByFilePath(file.path);
|
|
116
|
-
|
|
116
|
+
// Frontmatter is metadata, not prose: it is parsed into fields and removed
|
|
117
|
+
// from the body, so it never reaches the keyword index or the embedding.
|
|
118
|
+
const { meta, body, frontmatterLineCount } = parseFrontmatter(content);
|
|
119
|
+
const chunks = chunker.chunk(body, file.path, frontmatterLineCount);
|
|
117
120
|
const docs = chunks.map((chunk, idx) => ({
|
|
118
121
|
id: `${file.path}:${idx}`,
|
|
119
122
|
filePath: file.path,
|
|
@@ -122,6 +125,12 @@ export async function runIndex(options) {
|
|
|
122
125
|
mtimeMs: file.mtimeMs,
|
|
123
126
|
startLine: chunk.startLine,
|
|
124
127
|
headingBreadcrumb: chunk.headingBreadcrumb,
|
|
128
|
+
title: meta.title,
|
|
129
|
+
description: meta.description,
|
|
130
|
+
headings: chunk.headingBreadcrumb.join(" > "),
|
|
131
|
+
status: meta.status,
|
|
132
|
+
documentType: meta.documentType,
|
|
133
|
+
lastUpdate: meta.lastUpdate,
|
|
125
134
|
}));
|
|
126
135
|
await docStore.indexDocuments(docs);
|
|
127
136
|
chunksIndexed += docs.length;
|
|
@@ -162,7 +171,12 @@ export async function runIndex(options) {
|
|
|
162
171
|
const BATCH = 16;
|
|
163
172
|
for (let i = 0; i < changedDocsForEmbedding.length; i += BATCH) {
|
|
164
173
|
const batch = changedDocsForEmbedding.slice(i, i + BATCH);
|
|
165
|
-
const embeddings = await embeddingService.embedBatch(batch.map((d) =>
|
|
174
|
+
const embeddings = await embeddingService.embedBatch(batch.map((d) => buildEmbeddingText({
|
|
175
|
+
title: d.title,
|
|
176
|
+
description: d.description,
|
|
177
|
+
headingBreadcrumb: d.headingBreadcrumb,
|
|
178
|
+
content: d.content,
|
|
179
|
+
})));
|
|
166
180
|
for (let j = 0; j < batch.length; j++) {
|
|
167
181
|
const doc = batch[j];
|
|
168
182
|
await vectorStore.add({
|
|
@@ -191,13 +205,11 @@ export async function runIndex(options) {
|
|
|
191
205
|
const updatedManifest = {
|
|
192
206
|
...manifest,
|
|
193
207
|
embeddingModel: options.skipEmbeddings ? manifest.embeddingModel : EMBEDDING_MODEL_ID,
|
|
194
|
-
indexedScopes: Array.from(new Set([...manifest.indexedScopes, scope])),
|
|
195
208
|
lastIndexedAt: Date.now(),
|
|
196
209
|
};
|
|
197
210
|
await writeManifest(cacheDir, updatedManifest);
|
|
198
211
|
onProgress?.({ phase: "done", filesProcessed: files.length, totalFiles: files.length });
|
|
199
212
|
return {
|
|
200
|
-
scope,
|
|
201
213
|
filesScanned: files.length,
|
|
202
214
|
filesChanged: textChanged.length,
|
|
203
215
|
filesRemoved: removed.length,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"indexing-service.js","sourceRoot":"","sources":["../../src/indexing/indexing-service.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AACH,OAAO,EAAE,aAAa,EAAwB,MAAM,8BAA8B,CAAC;AACnF,OAAO,EAAE,WAAW,EAAE,MAAM,4BAA4B,CAAC;AACzD,OAAO,EAAE,gBAAgB,EAAE,oBAAoB,EAAE,MAAM,oCAAoC,CAAC;AAC5F,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,EAAE,iBAAiB,EAAE,YAAY,EAAiB,MAAM,2BAA2B,CAAC;AAC3F,OAAO,EACL,eAAe,EACf,YAAY,EACZ,aAAa,EACb,aAAa,EACb,uBAAuB,EACvB,wBAAwB,EACxB,iBAAiB,EACjB,kBAAkB,EAClB,iBAAiB,EACjB,iBAAiB,EACjB,YAAY,EACZ,iBAAiB,EACjB,KAAK,EACL,kBAAkB,GAEnB,MAAM,2BAA2B,CAAC;AACnC,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"indexing-service.js","sourceRoot":"","sources":["../../src/indexing/indexing-service.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AACH,OAAO,EAAE,aAAa,EAAwB,MAAM,8BAA8B,CAAC;AACnF,OAAO,EAAE,WAAW,EAAE,MAAM,4BAA4B,CAAC;AACzD,OAAO,EAAE,gBAAgB,EAAE,oBAAoB,EAAE,MAAM,oCAAoC,CAAC;AAC5F,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AACpD,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AACzD,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,EAAE,iBAAiB,EAAE,YAAY,EAAiB,MAAM,2BAA2B,CAAC;AAC3F,OAAO,EACL,eAAe,EACf,YAAY,EACZ,aAAa,EACb,aAAa,EACb,uBAAuB,EACvB,wBAAwB,EACxB,iBAAiB,EACjB,kBAAkB,EAClB,iBAAiB,EACjB,iBAAiB,EACjB,YAAY,EACZ,iBAAiB,EACjB,KAAK,EACL,kBAAkB,GAEnB,MAAM,2BAA2B,CAAC;AACnC,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAgCxC,MAAM,UAAU,GAAG,IAAI,CAAC;AACxB,MAAM,aAAa,GAAG,IAAI,CAAC;AAE3B,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,OAAqB;IAClD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACzB,MAAM,EAAE,QAAQ,EAAE,UAAU,EAAE,GAAG,OAAO,CAAC;IACzC,MAAM,QAAQ,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAC;IACxC,MAAM,eAAe,CAAC,QAAQ,CAAC,CAAC;IAEhC,MAAM,iBAAiB,GAAG,MAAM,YAAY,CAAC,QAAQ,CAAC,CAAC;IACvD,MAAM,QAAQ,GAAG,iBAAiB,IAAI,aAAa,EAAE,CAAC;IACtD,MAAM,oBAAoB,GAAG,iBAAiB,KAAK,IAAI,CAAC;IACxD,IAAI,oBAAoB,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;QACnD,wEAAwE;QACxE,sEAAsE;QACtE,uEAAuE;QACvE,MAAM,iBAAiB,CAAC,QAAQ,CAAC,CAAC;IACpC,CAAC;IAED,UAAU,EAAE,CAAC,EAAE,KAAK,EAAE,UAAU,EAAE,cAAc,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,CAAC,CAAC;IACtE,MAAM,KAAK,GAAG,MAAM,iBAAiB,CAAC,QAAQ,EAAE,CAAC,SAAS,CAAC,EAAE,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;IAE3F,MAAM,QAAQ,GAAG,IAAI,aAAa,EAAE,CAAC;IACrC,MAAM,QAAQ,CAAC,UAAU,CACvB,oBAAoB,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,MAAM,uBAAuB,CAAC,QAAQ,CAAC,CAAC,IAAI,SAAS,CAC1F,CAAC;IAEF,IAAI,WAAW,GAAuB,IAAI,CAAC;IAC3C,IAAI,OAAO,GAAG,IAAI,CAAC;IACnB,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC;QAC5B,WAAW,GAAG,IAAI,WAAW,CAAC,EAAE,UAAU,EAAE,oBAAoB,EAAE,CAAC,CAAC;QACpE,MAAM,kBAAkB,GAAG,MAAM,iBAAiB,CAAC,QAAQ,CAAC,CAAC;QAC7D,OAAO,GAAG,CAAC,oBAAoB,IAAI,kBAAkB;YACnD,CAAC,CAAC,MAAM,iBAAiB,CAAC,QAAQ,CAAC;YACnC,CAAC,CAAC,IAAI,CAAC;QACT,IAAI,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;YAC9B,MAAM,WAAW,CAAC,YAAY,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;QACvE,CAAC;aAAM,CAAC;YACN,MAAM,WAAW,CAAC,UAAU,EAAE,CAAC;QACjC,CAAC;IACH,CAAC;IAED,MAAM,OAAO,GAAG,IAAI,OAAO,CAAC,EAAE,YAAY,EAAE,UAAU,EAAE,cAAc,EAAE,aAAa,EAAE,CAAC,CAAC;IAEzF,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IACvD,MAAM,WAAW,GAAe,EAAE,CAAC;IACnC,MAAM,cAAc,GAAe,EAAE,CAAC;IACtC,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QACrC,MAAM,WAAW,GAAG,OAAO,CAAC,KAAK,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,WAAW,KAAK,CAAC,CAAC,OAAO,CAAC;QAC/E,MAAM,aAAa,GACjB,CAAC,OAAO,CAAC,cAAc;YACvB,CAAC,QAAQ,CAAC,cAAc,KAAK,kBAAkB;gBAC7C,CAAC,OAAO,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,WAAW,IAAI,KAAK,CAAC,aAAa,KAAK,CAAC,CAAC,OAAO,CAAC,CAAC;QAClF,IAAI,WAAW;YAAE,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACrC,IAAI,WAAW,IAAI,aAAa;YAAE,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC3D,CAAC;IAED,sEAAsE;IACtE,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE;QACvD,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC;QACjC,OAAO,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC;YACzB,CAAC,KAAK,CAAC,WAAW,KAAK,IAAI,IAAI,KAAK,CAAC,aAAa,KAAK,IAAI,IAAI,KAAK,CAAC,WAAW,CAAC,CAAC;IACtF,CAAC,CAAC,CAAC;IAEH,UAAU,EAAE,CAAC;QACX,KAAK,EAAE,eAAe;QACtB,cAAc,EAAE,CAAC;QACjB,UAAU,EAAE,cAAc,CAAC,MAAM;KAClC,CAAC,CAAC;IAEH,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,MAAM,uBAAuB,GAAsB,EAAE,CAAC;IAEtD,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE,CAAC;QAC3B,MAAM,QAAQ,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC;QACtC,IAAI,WAAW;YAAE,MAAM,WAAW,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC;QAC1D,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAE,CAAC;QACpC,KAAK,CAAC,WAAW,GAAG,IAAI,CAAC;QACzB,IAAI,WAAW,EAAE,CAAC;YAChB,KAAK,CAAC,aAAa,GAAG,IAAI,CAAC;YAC3B,KAAK,CAAC,WAAW,GAAG,KAAK,CAAC;QAC5B,CAAC;aAAM,IAAI,KAAK,CAAC,aAAa,KAAK,IAAI,EAAE,CAAC;YACxC,mEAAmE;YACnE,KAAK,CAAC,WAAW,GAAG,IAAI,CAAC;QAC3B,CAAC;aAAM,CAAC;YACN,KAAK,CAAC,WAAW,GAAG,KAAK,CAAC;QAC5B,CAAC;QACD,IAAI,KAAK,CAAC,WAAW,KAAK,IAAI,IAAI,KAAK,CAAC,aAAa,KAAK,IAAI;YAAE,OAAO,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC9F,CAAC;IAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,cAAc,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC/C,MAAM,IAAI,GAAG,cAAc,CAAC,CAAC,CAAE,CAAC;QAChC,UAAU,EAAE,CAAC;YACX,KAAK,EAAE,eAAe;YACtB,cAAc,EAAE,CAAC;YACjB,UAAU,EAAE,cAAc,CAAC,MAAM;YACjC,WAAW,EAAE,IAAI,CAAC,IAAI;SACvB,CAAC,CAAC;QAEH,MAAM,OAAO,GAAG,MAAM,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACjD,MAAM,QAAQ,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC3C,IAAI,WAAW;YAAE,MAAM,WAAW,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAE/D,2EAA2E;QAC3E,yEAAyE;QACzE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,oBAAoB,EAAE,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC;QAEvE,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,oBAAoB,CAAC,CAAC;QACpE,MAAM,IAAI,GAAsB,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC;YAC1D,EAAE,EAAE,GAAG,IAAI,CAAC,IAAI,IAAI,GAAG,EAAE;YACzB,QAAQ,EAAE,IAAI,CAAC,IAAI;YACnB,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,UAAU,EAAE,GAAG;YACf,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,SAAS,EAAE,KAAK,CAAC,SAAS;YAC1B,iBAAiB,EAAE,KAAK,CAAC,iBAAiB;YAC1C,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,QAAQ,EAAE,KAAK,CAAC,iBAAiB,CAAC,IAAI,CAAC,KAAK,CAAC;YAC7C,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,YAAY,EAAE,IAAI,CAAC,YAAY;YAC/B,UAAU,EAAE,IAAI,CAAC,UAAU;SAC5B,CAAC,CAAC,CAAC;QAEJ,MAAM,QAAQ,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;QACpC,aAAa,IAAI,IAAI,CAAC,MAAM,CAAC;QAC7B,IAAI,WAAW;YAAE,uBAAuB,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;QAEvD,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI;YACzC,WAAW,EAAE,IAAI;YACjB,aAAa,EAAE,IAAI;YACnB,WAAW,EAAE,KAAK;SACnB,CAAC;QACF,KAAK,CAAC,WAAW,GAAG,IAAI,CAAC,OAAO,CAAC;QACjC,IAAI,CAAC,WAAW,EAAE,CAAC;YACjB,wEAAwE;YACxE,wEAAwE;YACxE,mCAAmC;YACnC,KAAK,CAAC,WAAW,GAAG,IAAI,CAAC;QAC3B,CAAC;aAAM,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC7B,sEAAsE;YACtE,6DAA6D;YAC7D,KAAK,CAAC,aAAa,GAAG,IAAI,CAAC,OAAO,CAAC;YACnC,KAAK,CAAC,WAAW,GAAG,KAAK,CAAC;QAC5B,CAAC;QACD,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC;IACpC,CAAC;IAED,IAAI,mBAAmB,GAAG,CAAC,CAAC;IAC5B,IAAI,WAAW,IAAI,uBAAuB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtD,MAAM,gBAAgB,GAAG,IAAI,gBAAgB,CAAC;YAC5C,aAAa,EAAE,iBAAiB,CAAC,QAAQ,CAAC;YAC1C,iBAAiB,EAAE,OAAO,CAAC,iBAAiB,IAAI,IAAI;SACrD,CAAC,CAAC;QACH,MAAM,gBAAgB,CAAC,UAAU,EAAE,CAAC;QAEpC,UAAU,EAAE,CAAC;YACX,KAAK,EAAE,WAAW;YAClB,cAAc,EAAE,CAAC;YACjB,UAAU,EAAE,uBAAuB,CAAC,MAAM;SAC3C,CAAC,CAAC;QAEH,MAAM,KAAK,GAAG,EAAE,CAAC;QACjB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,uBAAuB,CAAC,MAAM,EAAE,CAAC,IAAI,KAAK,EAAE,CAAC;YAC/D,MAAM,KAAK,GAAG,uBAAuB,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,CAAC;YAC1D,MAAM,UAAU,GAAG,MAAM,gBAAgB,CAAC,UAAU,CAClD,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CACd,kBAAkB,CAAC;gBACjB,KAAK,EAAE,CAAC,CAAC,KAAK;gBACd,WAAW,EAAE,CAAC,CAAC,WAAW;gBAC1B,iBAAiB,EAAE,CAAC,CAAC,iBAAiB;gBACtC,OAAO,EAAE,CAAC,CAAC,OAAO;aACnB,CAAC,CACH,CACF,CAAC;YACF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACtC,MAAM,GAAG,GAAG,KAAK,CAAC,CAAC,CAAE,CAAC;gBACtB,MAAM,WAAW,CAAC,GAAG,CAAC;oBACpB,EAAE,EAAE,GAAG,CAAC,EAAE;oBACV,QAAQ,EAAE,GAAG,CAAC,QAAQ;oBACtB,UAAU,EAAE,GAAG,CAAC,UAAU;oBAC1B,SAAS,EAAE,UAAU,CAAC,CAAC,CAAE;iBAC1B,CAAC,CAAC;gBACH,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAE,CAAC,aAAa,GAAG,GAAG,CAAC,OAAO,CAAC;gBAC1D,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAE,CAAC,WAAW,GAAG,KAAK,CAAC;gBAClD,mBAAmB,EAAE,CAAC;YACxB,CAAC;YACD,UAAU,EAAE,CAAC;gBACX,KAAK,EAAE,WAAW;gBAClB,cAAc,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,KAAK,EAAE,uBAAuB,CAAC,MAAM,CAAC;gBACnE,UAAU,EAAE,uBAAuB,CAAC,MAAM;aAC3C,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,WAAW;IACX,MAAM,wBAAwB,CAAC,QAAQ,EAAE,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;IAC5D,IAAI,WAAW,EAAE,CAAC;QAChB,MAAM,cAAc,GAAG,MAAM,WAAW,CAAC,UAAU,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,WAAW,CAAC,CAAC;QACjF,MAAM,kBAAkB,CAAC,QAAQ,EAAE,cAAc,CAAC,CAAC;IACrD,CAAC;IAED,MAAM,eAAe,GAAa;QAChC,GAAG,QAAQ;QACX,cAAc,EAAE,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAC,CAAC,kBAAkB;QACrF,aAAa,EAAE,IAAI,CAAC,GAAG,EAAE;KAC1B,CAAC;IACF,MAAM,aAAa,CAAC,QAAQ,EAAE,eAAe,CAAC,CAAC;IAE/C,UAAU,EAAE,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,cAAc,EAAE,KAAK,CAAC,MAAM,EAAE,UAAU,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;IAExF,OAAO;QACL,YAAY,EAAE,KAAK,CAAC,MAAM;QAC1B,YAAY,EAAE,WAAW,CAAC,MAAM;QAChC,YAAY,EAAE,OAAO,CAAC,MAAM;QAC5B,aAAa;QACb,mBAAmB;QACnB,eAAe,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;KACpC,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Document authority: how much a document's own declared state should move it
|
|
3
|
+
* up or down the page, independent of how well it matched.
|
|
4
|
+
*
|
|
5
|
+
* Two signals, both taken from frontmatter the document wrote about itself:
|
|
6
|
+
* whether it claims to be stable, and when it was last touched. Both are
|
|
7
|
+
* multiplicative on the fused score, and both are neutral when the document
|
|
8
|
+
* makes no claim -- silence is not evidence of staleness.
|
|
9
|
+
*/
|
|
10
|
+
/** Age at which a document's score is halved. */
|
|
11
|
+
export declare const RECENCY_HALF_LIFE_DAYS = 180;
|
|
12
|
+
/** Decay never drives a document below this, however old it is. */
|
|
13
|
+
export declare const RECENCY_FLOOR = 0.25;
|
|
14
|
+
export interface AuthorityInput {
|
|
15
|
+
status?: string;
|
|
16
|
+
lastUpdate?: string;
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* Multiplier for a declared status. An unrecognised or absent status returns
|
|
20
|
+
* 1: only an explicit claim is acted on, so a document is never penalised for
|
|
21
|
+
* carrying no frontmatter.
|
|
22
|
+
*/
|
|
23
|
+
export declare function statusFactor(status?: string): number;
|
|
24
|
+
/**
|
|
25
|
+
* Exponential decay on document age, floored. An absent, unparseable, or
|
|
26
|
+
* future timestamp is neutral -- a clock skew must not hand out a bonus.
|
|
27
|
+
*/
|
|
28
|
+
export declare function recencyFactor(lastUpdate: string | undefined, now: number): number;
|
|
29
|
+
/** Combined multiplier applied to a fused score. */
|
|
30
|
+
export declare function authorityFactor(input: AuthorityInput, now: number): number;
|
|
31
|
+
//# sourceMappingURL=authority.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"authority.d.ts","sourceRoot":"","sources":["../../src/search/authority.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAQH,iDAAiD;AACjD,eAAO,MAAM,sBAAsB,MAAM,CAAC;AAE1C,mEAAmE;AACnE,eAAO,MAAM,aAAa,OAAO,CAAC;AAIlC,MAAM,WAAW,cAAc;IAC7B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;;;GAIG;AACH,wBAAgB,YAAY,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,MAAM,CAGpD;AAED;;;GAGG;AACH,wBAAgB,aAAa,CAAC,UAAU,EAAE,MAAM,GAAG,SAAS,EAAE,GAAG,EAAE,MAAM,GAAG,MAAM,CAWjF;AAED,oDAAoD;AACpD,wBAAgB,eAAe,CAAC,KAAK,EAAE,cAAc,EAAE,GAAG,EAAE,MAAM,GAAG,MAAM,CAE1E"}
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Document authority: how much a document's own declared state should move it
|
|
3
|
+
* up or down the page, independent of how well it matched.
|
|
4
|
+
*
|
|
5
|
+
* Two signals, both taken from frontmatter the document wrote about itself:
|
|
6
|
+
* whether it claims to be stable, and when it was last touched. Both are
|
|
7
|
+
* multiplicative on the fused score, and both are neutral when the document
|
|
8
|
+
* makes no claim -- silence is not evidence of staleness.
|
|
9
|
+
*/
|
|
10
|
+
/** Explicit status values and their multipliers. Anything else is neutral. */
|
|
11
|
+
const STATUS_FACTOR = {
|
|
12
|
+
stable: 1,
|
|
13
|
+
draft: 0.3,
|
|
14
|
+
};
|
|
15
|
+
/** Age at which a document's score is halved. */
|
|
16
|
+
export const RECENCY_HALF_LIFE_DAYS = 180;
|
|
17
|
+
/** Decay never drives a document below this, however old it is. */
|
|
18
|
+
export const RECENCY_FLOOR = 0.25;
|
|
19
|
+
const DAY_MS = 24 * 60 * 60 * 1000;
|
|
20
|
+
/**
|
|
21
|
+
* Multiplier for a declared status. An unrecognised or absent status returns
|
|
22
|
+
* 1: only an explicit claim is acted on, so a document is never penalised for
|
|
23
|
+
* carrying no frontmatter.
|
|
24
|
+
*/
|
|
25
|
+
export function statusFactor(status) {
|
|
26
|
+
if (!status)
|
|
27
|
+
return 1;
|
|
28
|
+
return STATUS_FACTOR[status.trim().toLowerCase()] ?? 1;
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* Exponential decay on document age, floored. An absent, unparseable, or
|
|
32
|
+
* future timestamp is neutral -- a clock skew must not hand out a bonus.
|
|
33
|
+
*/
|
|
34
|
+
export function recencyFactor(lastUpdate, now) {
|
|
35
|
+
if (!lastUpdate)
|
|
36
|
+
return 1;
|
|
37
|
+
const updatedAt = Date.parse(lastUpdate);
|
|
38
|
+
if (Number.isNaN(updatedAt))
|
|
39
|
+
return 1;
|
|
40
|
+
const ageDays = (now - updatedAt) / DAY_MS;
|
|
41
|
+
if (ageDays <= 0)
|
|
42
|
+
return 1;
|
|
43
|
+
const decayed = Math.pow(2, -ageDays / RECENCY_HALF_LIFE_DAYS);
|
|
44
|
+
return Math.max(RECENCY_FLOOR, decayed);
|
|
45
|
+
}
|
|
46
|
+
/** Combined multiplier applied to a fused score. */
|
|
47
|
+
export function authorityFactor(input, now) {
|
|
48
|
+
return statusFactor(input.status) * recencyFactor(input.lastUpdate, now);
|
|
49
|
+
}
|
|
50
|
+
//# sourceMappingURL=authority.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"authority.js","sourceRoot":"","sources":["../../src/search/authority.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,8EAA8E;AAC9E,MAAM,aAAa,GAA2B;IAC5C,MAAM,EAAE,CAAC;IACT,KAAK,EAAE,GAAG;CACX,CAAC;AAEF,iDAAiD;AACjD,MAAM,CAAC,MAAM,sBAAsB,GAAG,GAAG,CAAC;AAE1C,mEAAmE;AACnE,MAAM,CAAC,MAAM,aAAa,GAAG,IAAI,CAAC;AAElC,MAAM,MAAM,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;AAOnC;;;;GAIG;AACH,MAAM,UAAU,YAAY,CAAC,MAAe;IAC1C,IAAI,CAAC,MAAM;QAAE,OAAO,CAAC,CAAC;IACtB,OAAO,aAAa,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,IAAI,CAAC,CAAC;AACzD,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,aAAa,CAAC,UAA8B,EAAE,GAAW;IACvE,IAAI,CAAC,UAAU;QAAE,OAAO,CAAC,CAAC;IAE1B,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;IACzC,IAAI,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC;QAAE,OAAO,CAAC,CAAC;IAEtC,MAAM,OAAO,GAAG,CAAC,GAAG,GAAG,SAAS,CAAC,GAAG,MAAM,CAAC;IAC3C,IAAI,OAAO,IAAI,CAAC;QAAE,OAAO,CAAC,CAAC;IAE3B,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,OAAO,GAAG,sBAAsB,CAAC,CAAC;IAC/D,OAAO,IAAI,CAAC,GAAG,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC;AAC1C,CAAC;AAED,oDAAoD;AACpD,MAAM,UAAU,eAAe,CAAC,KAAqB,EAAE,GAAW;IAChE,OAAO,YAAY,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,aAAa,CAAC,KAAK,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC;AAC3E,CAAC"}
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Cosine floor for semantic hits.
|
|
3
|
+
*
|
|
4
|
+
* Re-measured against docs/ in enact-m5 (203 chunks), taking the top cosine
|
|
5
|
+
* over the whole corpus per query:
|
|
6
|
+
*
|
|
7
|
+
* junk zzqzz qqxxw 0.131 | florbnax vimtuzzle 0.134
|
|
8
|
+
* purple giraffe tapdance 0.103 | qwertyuiop asdfghjkl 0.132
|
|
9
|
+
* genuine invariants 0.260 | brownfield reference implementation 0.322
|
|
10
|
+
* database migrations 0.374 | websocket routes 0.520
|
|
11
|
+
* sandbox lifecycle 0.531 | coverage hooks 0.693
|
|
12
|
+
*
|
|
13
|
+
* So the real gap is [0.134, 0.260], and 0.20 sits in the middle of it with
|
|
14
|
+
* roughly equal headroom on each side. An earlier value of 0.35 was set from
|
|
15
|
+
* figures this measurement could not reproduce, and it silently discarded the
|
|
16
|
+
* best hit for real queries -- "invariants" returned nothing at all.
|
|
17
|
+
*
|
|
18
|
+
* Reproduce with:
|
|
19
|
+
* node dist/cli.js "<query>" --repo <repo> --mode semantic --min-score 0 --json
|
|
20
|
+
*/
|
|
21
|
+
export declare const DEFAULT_MIN_VECTOR_SCORE = 0.2;
|
|
22
|
+
/**
|
|
23
|
+
* Fraction of the query's *weight* a document must cover (see
|
|
24
|
+
* `idfWeightedCoverage`), not a fraction of its terms.
|
|
25
|
+
*
|
|
26
|
+
* Measured separation on this corpus:
|
|
27
|
+
* 0.09 "presence is not armedness" -- absent term dominates the weight
|
|
28
|
+
* 0.47 "mobile device" matching only the commoner term
|
|
29
|
+
* 0.53 "mobile device" matching only the rarer term
|
|
30
|
+
* 0.73 a five-term query missing one term
|
|
31
|
+
* 1.00 full match
|
|
32
|
+
*
|
|
33
|
+
* 0.5 sits below every genuine partial match and far above the junk floor.
|
|
34
|
+
*/
|
|
35
|
+
export declare const DEFAULT_MIN_TERM_COVERAGE = 0.5;
|
|
36
|
+
/**
|
|
37
|
+
* Fraction of the query's content terms that appear in `text`.
|
|
38
|
+
*
|
|
39
|
+
* Returns 1 for a query with no content terms (all stopwords or too short):
|
|
40
|
+
* there is nothing to be missing, so the gate must not reject on that basis.
|
|
41
|
+
*/
|
|
42
|
+
export declare function termCoverage(query: string, text: string): number;
|
|
43
|
+
/** Per-term document frequencies for the query, plus the corpus size. */
|
|
44
|
+
export interface CorpusStats {
|
|
45
|
+
totalDocs: number;
|
|
46
|
+
/** term -> number of indexed chunks containing it. 0 means absent entirely. */
|
|
47
|
+
documentFrequency: Map<string, number>;
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* Normalise a raw query string.
|
|
51
|
+
*
|
|
52
|
+
* Agents type `mobile | devices` out of grep habit. The pipe is treated as a
|
|
53
|
+
* separator, exactly like a space -- `mobile | devices` and `mobile devices`
|
|
54
|
+
* are the same query. It is deliberately not an OR operator: one query syntax
|
|
55
|
+
* is easier to use correctly than two, and an unquoted pipe never reaches this
|
|
56
|
+
* process anyway -- the shell consumes it as a pipeline first.
|
|
57
|
+
*/
|
|
58
|
+
export declare function normalizeQuery(query: string): string;
|
|
59
|
+
/**
|
|
60
|
+
* Coverage weighted by term rarity.
|
|
61
|
+
*
|
|
62
|
+
* Flat coverage treats every term as equally important, which makes a two-term
|
|
63
|
+
* query behave like AND -- "mobile device" then drops a document about a
|
|
64
|
+
* "mobile target" for lacking the word "device". Weighting by IDF lets the
|
|
65
|
+
* term that actually carries the query decide, in both directions: matching
|
|
66
|
+
* only the rare term passes, matching only the common one does not.
|
|
67
|
+
*
|
|
68
|
+
* Falls back to flat coverage when no stats are supplied.
|
|
69
|
+
*/
|
|
70
|
+
export declare function idfWeightedCoverage(query: string, text: string, stats?: CorpusStats): number;
|
|
71
|
+
/**
|
|
72
|
+
* Keyword-side gate. Uses term coverage rather than a BM25 threshold: BM25
|
|
73
|
+
* scores here are unnormalised and scale with query length and corpus size,
|
|
74
|
+
* so any absolute cutoff would be a fact about one repo. Coverage is a ratio
|
|
75
|
+
* and travels.
|
|
76
|
+
*/
|
|
77
|
+
export declare function passesKeywordGate(query: string, text: string, minCoverage?: number, stats?: CorpusStats): boolean;
|
|
78
|
+
/** Semantic-side gate. Cosine is bounded, so an absolute floor is portable. */
|
|
79
|
+
export declare function passesVectorGate(score: number, minScore?: number): boolean;
|
|
80
|
+
//# sourceMappingURL=relevance.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"relevance.d.ts","sourceRoot":"","sources":["../../src/search/relevance.ts"],"names":[],"mappings":"AAYA;;;;;;;;;;;;;;;;;;;GAmBG;AACH,eAAO,MAAM,wBAAwB,MAAM,CAAC;AAE5C;;;;;;;;;;;;GAYG;AACH,eAAO,MAAM,yBAAyB,MAAM,CAAC;AAE7C;;;;;GAKG;AACH,wBAAgB,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,CAMhE;AAED,yEAAyE;AACzE,MAAM,WAAW,WAAW;IAC1B,SAAS,EAAE,MAAM,CAAC;IAClB,+EAA+E;IAC/E,iBAAiB,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CACxC;AAUD;;;;;;;;GAQG;AACH,wBAAgB,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAEpD;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,mBAAmB,CACjC,KAAK,EAAE,MAAM,EACb,IAAI,EAAE,MAAM,EACZ,KAAK,CAAC,EAAE,WAAW,GAClB,MAAM,CAgBR;AAED;;;;;GAKG;AACH,wBAAgB,iBAAiB,CAC/B,KAAK,EAAE,MAAM,EACb,IAAI,EAAE,MAAM,EACZ,WAAW,GAAE,MAAkC,EAC/C,KAAK,CAAC,EAAE,WAAW,GAClB,OAAO,CAET;AAED,+EAA+E;AAC/E,wBAAgB,gBAAgB,CAC9B,KAAK,EAAE,MAAM,EACb,QAAQ,GAAE,MAAiC,GAC1C,OAAO,CAET"}
|