@gmickel/gno 2.10.1 → 2.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -11
- package/assets/skill/SKILL.md +27 -15
- package/assets/skill/cli-reference.md +6 -5
- package/assets/skill/examples.md +16 -12
- package/assets/skill/mcp-reference.md +2 -1
- package/assets/spa-production.json.gz +0 -0
- package/browser-extension/artifacts/{gno-browser-clipper-v2.10.1.zip → gno-browser-clipper-v2.12.0.zip} +0 -0
- package/browser-extension/artifacts/gno-browser-clipper-v2.12.0.zip.sha256 +1 -0
- package/browser-extension/dist/manifest.json +1 -1
- package/package.json +10 -5
- package/spec/cli.md +127 -35
- package/spec/evals-agentic.md +3 -1
- package/spec/mcp.md +36 -6
- package/spec/output-schemas/ask.schema.json +8 -0
- package/spec/output-schemas/claim-verification.schema.json +8 -2
- package/spec/output-schemas/context-capsule-v1.schema.json +6 -1
- package/spec/output-schemas/context-capsule-verification.schema.json +4 -2
- package/spec/output-schemas/get.schema.json +4 -0
- package/spec/output-schemas/metadata-summary.schema.json +60 -0
- package/spec/output-schemas/multi-get.schema.json +4 -0
- package/spec/output-schemas/search-result.schema.json +4 -0
- package/spec/output-schemas/search-results.schema.json +4 -0
- package/spec/output-schemas/status.schema.json +28 -0
- package/src/app/constants.ts +16 -0
- package/src/app/context-agent-projection.ts +10 -1
- package/src/app/context-format.ts +1 -0
- package/src/app/verified-ask.ts +2 -0
- package/src/cli/commands/agents/block.ts +10 -9
- package/src/cli/commands/agents/commands.ts +6 -5
- package/src/cli/commands/agents/engine.ts +15 -2
- package/src/cli/commands/agents/harnesses.ts +21 -14
- package/src/cli/commands/ask-format.ts +18 -1
- package/src/cli/commands/ask.ts +4 -5
- package/src/cli/commands/cleanup.ts +46 -6
- package/src/cli/commands/collection/remove.ts +1 -0
- package/src/cli/commands/collection/rename.ts +1 -0
- package/src/cli/commands/daemon.ts +5 -1
- package/src/cli/commands/doctor.ts +37 -0
- package/src/cli/commands/get.ts +11 -1
- package/src/cli/commands/mcp/paths.ts +2 -1
- package/src/cli/commands/metadata.ts +79 -0
- package/src/cli/commands/multi-get.ts +13 -0
- package/src/cli/commands/setup-semantic.ts +6 -1
- package/src/cli/commands/shared.ts +35 -8
- package/src/cli/commands/skill/paths.ts +20 -2
- package/src/cli/commands/status.ts +89 -15
- package/src/cli/format/search-results.ts +11 -0
- package/src/cli/options.ts +2 -0
- package/src/cli/program.ts +64 -3
- package/src/core/activation-probe-plan.ts +0 -1
- package/src/core/config-mutation.ts +72 -55
- package/src/core/context-capsule-schema.ts +2 -0
- package/src/core/context-capsule-verification.ts +2 -0
- package/src/core/context-compiler.ts +2 -5
- package/src/core/context-evidence.ts +1 -0
- package/src/core/context-verifier.ts +14 -1
- package/src/core/memory-remember.ts +0 -1
- package/src/core/metadata-discovery.ts +169 -0
- package/src/core/module-path.ts +33 -0
- package/src/core/runtime-entrypoint.ts +3 -1
- package/src/ingestion/file-processor.ts +6 -3
- package/src/ingestion/prepare-file.ts +14 -4
- package/src/ingestion/source-availability/root.ts +138 -0
- package/src/ingestion/sync.ts +35 -1
- package/src/ingestion/walker.ts +11 -3
- package/src/llm/native-worker/client.ts +8 -4
- package/src/llm/native-worker/errors.ts +26 -3
- package/src/mcp/http-egress.ts +1 -0
- package/src/mcp/http-transport.ts +7 -2
- package/src/mcp/server.ts +3 -0
- package/src/mcp/tool-descriptions-core.ts +3 -3
- package/src/mcp/tool-profile.ts +16 -1
- package/src/mcp/tools/get.ts +10 -0
- package/src/mcp/tools/index.ts +68 -4
- package/src/mcp/tools/multi-get.ts +15 -0
- package/src/mcp/tools/query.ts +5 -3
- package/src/mcp/tools/search.ts +5 -3
- package/src/mcp/tools/status.ts +17 -2
- package/src/mcp/tools/vsearch.ts +5 -3
- package/src/mcp/tools/workspace-write.ts +42 -34
- package/src/pipeline/answer-prompt.ts +15 -2
- package/src/pipeline/answer.ts +3 -0
- package/src/pipeline/claim-verification-schema.ts +6 -1
- package/src/pipeline/claim-verification.ts +54 -1
- package/src/pipeline/explain.ts +1 -1
- package/src/pipeline/fusion.ts +24 -4
- package/src/pipeline/hybrid.ts +5 -1
- package/src/pipeline/query-snippet.ts +266 -0
- package/src/pipeline/result-context.ts +29 -0
- package/src/pipeline/search.ts +154 -20
- package/src/pipeline/types.ts +4 -0
- package/src/pipeline/vsearch.ts +5 -1
- package/src/sdk/client.ts +46 -6
- package/src/sdk/documents.ts +5 -0
- package/src/sdk/types.ts +11 -1
- package/src/serve/public/components/ThoroughnessSelector.tsx +1 -1
- package/src/serve/public/globals.built.css +1 -1
- package/src/serve/public/pages/Collections.tsx +8 -0
- package/src/serve/public/pages/Search.tsx +9 -1
- package/src/serve/resident-request.ts +8 -0
- package/src/serve/resident-runtime.ts +25 -3
- package/src/serve/routes/api.ts +29 -1
- package/src/serve/server.ts +12 -0
- package/src/serve/spa-production-build.ts +7 -3
- package/src/serve/status-model.ts +2 -0
- package/src/serve/status.ts +29 -0
- package/src/sessions/config-refresh.ts +6 -0
- package/src/sessions/import-child.ts +4 -1
- package/src/store/sqlite/adapter.ts +241 -20
- package/src/store/types.ts +61 -7
- package/src/store/vector/variant-search.ts +112 -74
- package/browser-extension/artifacts/gno-browser-clipper-v2.10.1.zip.sha256 +0 -1
package/src/pipeline/fusion.ts
CHANGED
|
@@ -35,6 +35,15 @@ function rrfContribution(rank: number, k: number, weight: number): number {
|
|
|
35
35
|
return weight / (k + rank);
|
|
36
36
|
}
|
|
37
37
|
|
|
38
|
+
/**
|
|
39
|
+
* Candidates are documents: BM25 ranks whole documents while vector search
|
|
40
|
+
* ranks chunks, so a chunk-level key would split one document's votes across
|
|
41
|
+
* its chunks. Each list counts a document once, at its best rank.
|
|
42
|
+
*/
|
|
43
|
+
function fusionKey(result: RankedInput["results"][number]): string {
|
|
44
|
+
return `${result.documentId ?? result.mirrorHash}`;
|
|
45
|
+
}
|
|
46
|
+
|
|
38
47
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
39
48
|
// Fusion Implementation
|
|
40
49
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
@@ -51,7 +60,7 @@ export function rrfFuse(
|
|
|
51
60
|
inputs: RankedInput[],
|
|
52
61
|
config: RrfConfig
|
|
53
62
|
): FusionCandidate[] {
|
|
54
|
-
// Map:
|
|
63
|
+
// Map: document key -> FusionCandidate
|
|
55
64
|
const candidates = new Map<string, FusionCandidate>();
|
|
56
65
|
|
|
57
66
|
// Separate inputs by type for weight assignment
|
|
@@ -74,8 +83,11 @@ export function rrfFuse(
|
|
|
74
83
|
? config.bm25Weight * 2.0
|
|
75
84
|
: config.bm25Weight * 0.5;
|
|
76
85
|
|
|
86
|
+
const countedInInput = new Set<string>();
|
|
77
87
|
for (const result of input.results) {
|
|
78
|
-
const key =
|
|
88
|
+
const key = fusionKey(result);
|
|
89
|
+
if (countedInInput.has(key)) continue;
|
|
90
|
+
countedInInput.add(key);
|
|
79
91
|
let candidate = candidates.get(key);
|
|
80
92
|
|
|
81
93
|
if (!candidate) {
|
|
@@ -116,8 +128,11 @@ export function rrfFuse(
|
|
|
116
128
|
weight = config.vecWeight * 0.7;
|
|
117
129
|
}
|
|
118
130
|
|
|
131
|
+
const countedInInput = new Set<string>();
|
|
119
132
|
for (const result of input.results) {
|
|
120
|
-
const key =
|
|
133
|
+
const key = fusionKey(result);
|
|
134
|
+
if (countedInInput.has(key)) continue;
|
|
135
|
+
countedInInput.add(key);
|
|
121
136
|
let candidate = candidates.get(key);
|
|
122
137
|
|
|
123
138
|
if (!candidate) {
|
|
@@ -137,6 +152,8 @@ export function rrfFuse(
|
|
|
137
152
|
|
|
138
153
|
// Track best vector rank
|
|
139
154
|
if (candidate.vecRank === null || result.rank < candidate.vecRank) {
|
|
155
|
+
// The best-ranked chunk represents the document downstream.
|
|
156
|
+
candidate.seq = result.seq;
|
|
140
157
|
candidate.vecRank = result.rank;
|
|
141
158
|
}
|
|
142
159
|
|
|
@@ -152,8 +169,11 @@ export function rrfFuse(
|
|
|
152
169
|
for (const input of graphInputs) {
|
|
153
170
|
const weight = config.bm25Weight * 0.8;
|
|
154
171
|
|
|
172
|
+
const countedInInput = new Set<string>();
|
|
155
173
|
for (const result of input.results) {
|
|
156
|
-
const key =
|
|
174
|
+
const key = fusionKey(result);
|
|
175
|
+
if (countedInInput.has(key)) continue;
|
|
176
|
+
countedInInput.add(key);
|
|
157
177
|
let candidate = candidates.get(key);
|
|
158
178
|
|
|
159
179
|
if (!candidate) {
|
package/src/pipeline/hybrid.ts
CHANGED
|
@@ -73,7 +73,10 @@ import {
|
|
|
73
73
|
summarizeQueryModes,
|
|
74
74
|
} from "./query-modes";
|
|
75
75
|
import { rerankCandidates } from "./rerank";
|
|
76
|
-
import {
|
|
76
|
+
import {
|
|
77
|
+
attachSearchResultContexts,
|
|
78
|
+
attachSupersession,
|
|
79
|
+
} from "./result-context";
|
|
77
80
|
import { cleanDisplaySnippet } from "./snippet";
|
|
78
81
|
import {
|
|
79
82
|
isWithinTemporalRange,
|
|
@@ -1364,6 +1367,7 @@ async function searchHybridWithHydration(
|
|
|
1364
1367
|
}
|
|
1365
1368
|
: undefined;
|
|
1366
1369
|
await attachSearchResultContexts(store, finalResults);
|
|
1370
|
+
await attachSupersession(store, finalResults);
|
|
1367
1371
|
const lineageResult = await attachSearchResultEgressLineage(
|
|
1368
1372
|
store,
|
|
1369
1373
|
finalResults,
|
|
@@ -0,0 +1,266 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* BM25 result snippets built from stored chunks.
|
|
3
|
+
*
|
|
4
|
+
* FTS5 `snippet()` re-tokenizes the whole body of every matched document
|
|
5
|
+
* before LIMIT, which made long questions take seconds. Snippets are built
|
|
6
|
+
* for the returned rows only: pick the chunk that best covers the query terms
|
|
7
|
+
* (terms weighted by rarity across the returned chunks, so a chunk with
|
|
8
|
+
* "ladder" beats one with only "the" and "does"; lowest seq on ties), then
|
|
9
|
+
* its best 32-token window, with matching tokens wrapped in `<mark>`/`</mark>`
|
|
10
|
+
* and `...` where the window cuts the chunk, as `snippet()` did. Chunk choice
|
|
11
|
+
* uses substring presence of lightly stemmed terms; highlighting uses word
|
|
12
|
+
* prefixes. Both approximate the index tokenizer and affect display only,
|
|
13
|
+
* never ranking.
|
|
14
|
+
*
|
|
15
|
+
* @module src/pipeline/query-snippet
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
/** Tokens per snippet window (the old `snippet(..., 32)` budget). */
|
|
19
|
+
const WINDOW_TOKENS = 32;
|
|
20
|
+
|
|
21
|
+
/** Repeat hits only break ties between windows with equal term coverage. */
|
|
22
|
+
const HIT_TIEBREAK = 1e-6;
|
|
23
|
+
|
|
24
|
+
const TOKEN_REGEX = /[\p{L}\p{N}\p{M}]+/gu;
|
|
25
|
+
|
|
26
|
+
/** Quoted phrase (optionally negated) or a bare token, as the lexical grammar splits them. */
|
|
27
|
+
const QUERY_PART_REGEX = /(-?)"([^"]*)"|(\S+)/g;
|
|
28
|
+
|
|
29
|
+
const DIACRITIC_REGEX = /\p{M}/gu;
|
|
30
|
+
|
|
31
|
+
const NON_ASCII_REGEX = /[^\p{ASCII}]/u;
|
|
32
|
+
|
|
33
|
+
/** Final doubled consonant left by suffix removal (`embedd` -> `embed`). */
|
|
34
|
+
const DOUBLED_CONSONANT_REGEX = /([bdfgkmnprt])\1$/;
|
|
35
|
+
|
|
36
|
+
/** Suffixes removed from query terms, longest first. */
|
|
37
|
+
const STEM_SUFFIXES = [
|
|
38
|
+
"ational",
|
|
39
|
+
"ations",
|
|
40
|
+
"ation",
|
|
41
|
+
"ings",
|
|
42
|
+
"ions",
|
|
43
|
+
"ers",
|
|
44
|
+
"ied",
|
|
45
|
+
"ies",
|
|
46
|
+
"ing",
|
|
47
|
+
"ion",
|
|
48
|
+
"al",
|
|
49
|
+
"ed",
|
|
50
|
+
"er",
|
|
51
|
+
"ly",
|
|
52
|
+
"e",
|
|
53
|
+
"s",
|
|
54
|
+
"y",
|
|
55
|
+
];
|
|
56
|
+
|
|
57
|
+
const MIN_STEM_LENGTH = 3;
|
|
58
|
+
|
|
59
|
+
export interface SnippetTerms {
|
|
60
|
+
/** Lowercase term prefixes (stemmed query words and diacritic-free forms). */
|
|
61
|
+
prefixes: string[];
|
|
62
|
+
/** Query word id per prefix; variants of one word share an id. */
|
|
63
|
+
ids: number[];
|
|
64
|
+
/** Anchored prefix matcher for one lowercased token. */
|
|
65
|
+
token: RegExp;
|
|
66
|
+
idByPrefix: Map<string, number>;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/** Query word ids present in one chunk. */
|
|
70
|
+
export interface ChunkTermSet {
|
|
71
|
+
seq: number;
|
|
72
|
+
terms: Set<number>;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
interface Token {
|
|
76
|
+
start: number;
|
|
77
|
+
end: number;
|
|
78
|
+
term: number;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
const stripDiacritics = (text: string): string =>
|
|
82
|
+
NON_ASCII_REGEX.test(text)
|
|
83
|
+
? text.normalize("NFD").replace(DIACRITIC_REGEX, "")
|
|
84
|
+
: text;
|
|
85
|
+
|
|
86
|
+
function stemPrefix(term: string): string {
|
|
87
|
+
if (term.length <= MIN_STEM_LENGTH) return term;
|
|
88
|
+
for (const suffix of STEM_SUFFIXES) {
|
|
89
|
+
if (
|
|
90
|
+
term.endsWith(suffix) &&
|
|
91
|
+
term.length - suffix.length >= MIN_STEM_LENGTH
|
|
92
|
+
) {
|
|
93
|
+
const stem = term.slice(0, -suffix.length);
|
|
94
|
+
return DOUBLED_CONSONANT_REGEX.test(stem) ? stem.slice(0, -1) : stem;
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
return term;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/** Positive terms of a lexical query; null when there are none. */
|
|
101
|
+
export function createSnippetTerms(query: string): SnippetTerms | null {
|
|
102
|
+
const idByPrefix = new Map<string, number>();
|
|
103
|
+
let nextId = 0;
|
|
104
|
+
for (const part of query.matchAll(QUERY_PART_REGEX)) {
|
|
105
|
+
const bare = part[3];
|
|
106
|
+
if (part[1] === "-" || bare?.startsWith("-")) continue;
|
|
107
|
+
for (const [word] of (part[2] ?? bare ?? "").matchAll(TOKEN_REGEX)) {
|
|
108
|
+
const lower = word.toLowerCase();
|
|
109
|
+
for (const form of new Set([lower, stripDiacritics(lower)])) {
|
|
110
|
+
const prefix = stemPrefix(form);
|
|
111
|
+
if (!idByPrefix.has(prefix)) idByPrefix.set(prefix, nextId);
|
|
112
|
+
}
|
|
113
|
+
nextId += 1;
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
if (idByPrefix.size === 0) return null;
|
|
117
|
+
const prefixes = [...idByPrefix.keys()].sort((a, b) => b.length - a.length);
|
|
118
|
+
return {
|
|
119
|
+
prefixes,
|
|
120
|
+
ids: prefixes.map((prefix) => idByPrefix.get(prefix) ?? -1),
|
|
121
|
+
token: new RegExp(`^(?:${prefixes.join("|")})`, "u"),
|
|
122
|
+
idByPrefix,
|
|
123
|
+
};
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
/** In-memory equivalent of the store's chunk term presence query. */
|
|
127
|
+
export function chunkTermPresence(
|
|
128
|
+
text: string,
|
|
129
|
+
terms: SnippetTerms
|
|
130
|
+
): boolean[] {
|
|
131
|
+
const lower = text.toLowerCase();
|
|
132
|
+
return terms.prefixes.map((prefix) => lower.includes(prefix));
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
export function presentTermIds(
|
|
136
|
+
present: boolean[],
|
|
137
|
+
terms: SnippetTerms
|
|
138
|
+
): Set<number> {
|
|
139
|
+
const ids = new Set<number>();
|
|
140
|
+
for (const [index, flag] of present.entries()) {
|
|
141
|
+
const id = terms.ids[index];
|
|
142
|
+
if (flag && id !== undefined) ids.add(id);
|
|
143
|
+
}
|
|
144
|
+
return ids;
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
/** Rarity weight per query word across all candidate chunks. */
|
|
148
|
+
export function snippetTermWeights(
|
|
149
|
+
chunks: Iterable<ChunkTermSet>
|
|
150
|
+
): Map<number, number> {
|
|
151
|
+
let total = 0;
|
|
152
|
+
const frequency = new Map<number, number>();
|
|
153
|
+
for (const { terms } of chunks) {
|
|
154
|
+
total += 1;
|
|
155
|
+
for (const term of terms)
|
|
156
|
+
frequency.set(term, (frequency.get(term) ?? 0) + 1);
|
|
157
|
+
}
|
|
158
|
+
const weights = new Map<number, number>();
|
|
159
|
+
for (const [term, count] of frequency)
|
|
160
|
+
weights.set(term, Math.log(1 + total / count));
|
|
161
|
+
return weights;
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
/** Seq of the chunk with the best weighted coverage (lowest seq on ties). */
|
|
165
|
+
export function pickSnippetSeq(
|
|
166
|
+
chunks: ChunkTermSet[],
|
|
167
|
+
weights: Map<number, number>
|
|
168
|
+
): number | undefined {
|
|
169
|
+
let best: ChunkTermSet | undefined;
|
|
170
|
+
let bestScore = -1;
|
|
171
|
+
for (const chunk of chunks) {
|
|
172
|
+
let score = 0;
|
|
173
|
+
for (const term of chunk.terms) score += weights.get(term) ?? 0;
|
|
174
|
+
if (
|
|
175
|
+
score > bestScore ||
|
|
176
|
+
(score === bestScore && best && chunk.seq < best.seq)
|
|
177
|
+
) {
|
|
178
|
+
bestScore = score;
|
|
179
|
+
best = chunk;
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
return best?.seq;
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
function tokenize(text: string, terms: SnippetTerms): Token[] {
|
|
186
|
+
const tokens: Token[] = [];
|
|
187
|
+
for (const match of text.matchAll(TOKEN_REGEX)) {
|
|
188
|
+
const start = match.index;
|
|
189
|
+
const word = match[0];
|
|
190
|
+
const prefix = terms.token.exec(stripDiacritics(word.toLowerCase()));
|
|
191
|
+
tokens.push({
|
|
192
|
+
start,
|
|
193
|
+
end: start + word.length,
|
|
194
|
+
term: prefix ? (terms.idByPrefix.get(prefix[0]) ?? -1) : -1,
|
|
195
|
+
});
|
|
196
|
+
}
|
|
197
|
+
return tokens;
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
/** Start index of the window with the best weighted term coverage. */
|
|
201
|
+
function bestWindowStart(
|
|
202
|
+
tokens: Token[],
|
|
203
|
+
weights: Map<number, number>
|
|
204
|
+
): number {
|
|
205
|
+
if (tokens.length <= WINDOW_TOKENS) return 0;
|
|
206
|
+
const counts = new Map<number, number>();
|
|
207
|
+
let covered = 0;
|
|
208
|
+
let hits = 0;
|
|
209
|
+
const add = (term: number, delta: number) => {
|
|
210
|
+
if (term < 0) return;
|
|
211
|
+
const count = (counts.get(term) ?? 0) + delta;
|
|
212
|
+
counts.set(term, count);
|
|
213
|
+
hits += delta;
|
|
214
|
+
const weight = weights.get(term) ?? 1;
|
|
215
|
+
if (delta > 0 && count === 1) covered += weight;
|
|
216
|
+
if (delta < 0 && count === 0) covered -= weight;
|
|
217
|
+
};
|
|
218
|
+
let bestStart = 0;
|
|
219
|
+
let bestScore = -1;
|
|
220
|
+
for (let index = 0; index < tokens.length; index += 1) {
|
|
221
|
+
add(tokens[index]?.term ?? -1, 1);
|
|
222
|
+
const start = index - WINDOW_TOKENS + 1;
|
|
223
|
+
if (start > 0) add(tokens[start - 1]?.term ?? -1, -1);
|
|
224
|
+
if (start < 0) continue;
|
|
225
|
+
const score = covered + hits * HIT_TIEBREAK;
|
|
226
|
+
if (score > bestScore) {
|
|
227
|
+
bestScore = score;
|
|
228
|
+
bestStart = start;
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
// Center the hits inside the chosen window.
|
|
232
|
+
let first = -1;
|
|
233
|
+
let last = -1;
|
|
234
|
+
for (let index = bestStart; index < bestStart + WINDOW_TOKENS; index += 1) {
|
|
235
|
+
if ((tokens[index]?.term ?? -1) < 0) continue;
|
|
236
|
+
if (first < 0) first = index;
|
|
237
|
+
last = index;
|
|
238
|
+
}
|
|
239
|
+
if (first < 0) return bestStart;
|
|
240
|
+
const centered = first - Math.floor((WINDOW_TOKENS - (last - first + 1)) / 2);
|
|
241
|
+
return Math.max(0, Math.min(centered, tokens.length - WINDOW_TOKENS));
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
/** The best 32-token window of one chunk, query terms in `<mark>` tags. */
|
|
245
|
+
export function renderChunkSnippet(
|
|
246
|
+
text: string,
|
|
247
|
+
terms: SnippetTerms,
|
|
248
|
+
weights: Map<number, number>
|
|
249
|
+
): string {
|
|
250
|
+
const tokens = tokenize(text, terms);
|
|
251
|
+
if (tokens.length === 0) return text;
|
|
252
|
+
const start = bestWindowStart(tokens, weights);
|
|
253
|
+
const end = Math.min(tokens.length, start + WINDOW_TOKENS) - 1;
|
|
254
|
+
const atStart = start === 0;
|
|
255
|
+
const atEnd = end === tokens.length - 1;
|
|
256
|
+
let cursor = atStart ? 0 : (tokens[start]?.start ?? 0);
|
|
257
|
+
let out = atStart ? "" : "...";
|
|
258
|
+
for (let index = start; index <= end; index += 1) {
|
|
259
|
+
const token = tokens[index];
|
|
260
|
+
if (!token || token.term < 0) continue;
|
|
261
|
+
out += `${text.slice(cursor, token.start)}<mark>${text.slice(token.start, token.end)}</mark>`;
|
|
262
|
+
cursor = token.end;
|
|
263
|
+
}
|
|
264
|
+
out += text.slice(cursor, atEnd ? text.length : (tokens[end]?.end ?? 0));
|
|
265
|
+
return atEnd ? out : `${out}...`;
|
|
266
|
+
}
|
|
@@ -57,3 +57,32 @@ export async function attachSearchResultContexts(
|
|
|
57
57
|
// the historical result shape and are reported by config validation.
|
|
58
58
|
}
|
|
59
59
|
}
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* Mark results whose memory fact a newer fact superseded, naming the current
|
|
63
|
+
* successor. Fail-open like context guidance: a lookup failure marks nothing.
|
|
64
|
+
*/
|
|
65
|
+
export async function attachSupersession(
|
|
66
|
+
store: StorePort,
|
|
67
|
+
results: { uri: string; supersededBy?: string }[]
|
|
68
|
+
): Promise<void> {
|
|
69
|
+
if (results.length === 0 || !store.getSupersedingUris) return;
|
|
70
|
+
const successors = await store.getSupersedingUris(
|
|
71
|
+
results.map((result) => result.uri)
|
|
72
|
+
);
|
|
73
|
+
if (!successors.ok) return;
|
|
74
|
+
for (const result of results) {
|
|
75
|
+
const successor = successors.value.get(result.uri);
|
|
76
|
+
if (successor) result.supersededBy = successor;
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/** Current successor URI of one document's superseded memory fact, if any. */
|
|
81
|
+
export async function supersedingUri(
|
|
82
|
+
store: StorePort,
|
|
83
|
+
uri: string
|
|
84
|
+
): Promise<string | undefined> {
|
|
85
|
+
if (!store.getSupersedingUris) return undefined;
|
|
86
|
+
const successors = await store.getSupersedingUris([uri]);
|
|
87
|
+
return successors.ok ? successors.value.get(uri) : undefined;
|
|
88
|
+
}
|
package/src/pipeline/search.ts
CHANGED
|
@@ -8,6 +8,7 @@
|
|
|
8
8
|
import { join as pathJoin } from "node:path"; // No Bun path utils equivalent
|
|
9
9
|
|
|
10
10
|
import type { ChunkRow, FtsResult, StorePort } from "../store/types";
|
|
11
|
+
import type { ChunkTermSet, SnippetTerms } from "./query-snippet";
|
|
11
12
|
import type {
|
|
12
13
|
SearchOptions,
|
|
13
14
|
SearchResult,
|
|
@@ -31,7 +32,18 @@ import { typedMetadataWarnings } from "./filters";
|
|
|
31
32
|
import { selectBestChunkForSteering } from "./intent";
|
|
32
33
|
import { hasProjectAffinity } from "./project-affinity";
|
|
33
34
|
import { detectQueryLanguage } from "./query-language";
|
|
34
|
-
import {
|
|
35
|
+
import {
|
|
36
|
+
chunkTermPresence,
|
|
37
|
+
createSnippetTerms,
|
|
38
|
+
pickSnippetSeq,
|
|
39
|
+
presentTermIds,
|
|
40
|
+
renderChunkSnippet,
|
|
41
|
+
snippetTermWeights,
|
|
42
|
+
} from "./query-snippet";
|
|
43
|
+
import {
|
|
44
|
+
attachSearchResultContexts,
|
|
45
|
+
attachSupersession,
|
|
46
|
+
} from "./result-context";
|
|
35
47
|
import { cleanDisplaySnippet } from "./snippet";
|
|
36
48
|
import {
|
|
37
49
|
resolveRecencyTimestamp,
|
|
@@ -41,6 +53,9 @@ import {
|
|
|
41
53
|
import { attachSearchResultPlannerMetadata } from "./trace-metadata";
|
|
42
54
|
import { SEARCH_RESULT_PLANNER_METADATA } from "./types";
|
|
43
55
|
|
|
56
|
+
const passageHash = (text: string): string =>
|
|
57
|
+
new Bun.CryptoHasher("sha256").update(text).digest("hex");
|
|
58
|
+
|
|
44
59
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
45
60
|
// Score Normalization
|
|
46
61
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
@@ -121,11 +136,9 @@ function buildSearchResult(ctx: BuildResultContext): SearchResult {
|
|
|
121
136
|
snippet = chunk.text;
|
|
122
137
|
snippetRange = { startLine: chunk.startLine, endLine: chunk.endLine };
|
|
123
138
|
} else {
|
|
124
|
-
// Default:
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
chunk?.text
|
|
128
|
-
);
|
|
139
|
+
// Default: chunk text until attachChunkSnippets replaces it for the final
|
|
140
|
+
// rows, with leading frontmatter stripped
|
|
141
|
+
const cleaned = cleanDisplaySnippet(chunk?.text ?? "", chunk?.text);
|
|
129
142
|
snippet = cleaned.text;
|
|
130
143
|
if (chunk) {
|
|
131
144
|
line = chunk.startLine + cleaned.startLineOffset;
|
|
@@ -159,12 +172,132 @@ function buildSearchResult(ctx: BuildResultContext): SearchResult {
|
|
|
159
172
|
graphExpanded: false,
|
|
160
173
|
startLine: chunk.startLine,
|
|
161
174
|
endLine: chunk.endLine,
|
|
162
|
-
passageHash:
|
|
163
|
-
.update(chunk.text)
|
|
164
|
-
.digest("hex"),
|
|
175
|
+
passageHash: passageHash(chunk.text),
|
|
165
176
|
});
|
|
166
177
|
}
|
|
167
178
|
|
|
179
|
+
/** Query-word ids present in each chunk of the returned documents. */
|
|
180
|
+
async function readChunkTermSets(
|
|
181
|
+
store: StorePort,
|
|
182
|
+
hashes: string[],
|
|
183
|
+
terms: SnippetTerms,
|
|
184
|
+
loadedChunks: Map<string, ChunkRow[]> | undefined
|
|
185
|
+
): Promise<{
|
|
186
|
+
termSets: Map<string, ChunkTermSet[]>;
|
|
187
|
+
chunks?: Map<string, ChunkRow[]>;
|
|
188
|
+
} | null> {
|
|
189
|
+
const termSets = new Map<string, ChunkTermSet[]>();
|
|
190
|
+
const add = (mirrorHash: string, seq: number, present: boolean[]) => {
|
|
191
|
+
const list = termSets.get(mirrorHash) ?? [];
|
|
192
|
+
list.push({ seq, terms: presentTermIds(present, terms) });
|
|
193
|
+
termSets.set(mirrorHash, list);
|
|
194
|
+
};
|
|
195
|
+
// In-database presence keeps long documents from being loaded whole.
|
|
196
|
+
const presence =
|
|
197
|
+
!loadedChunks && store.getChunkTermPresence
|
|
198
|
+
? await store.getChunkTermPresence(hashes, terms.prefixes)
|
|
199
|
+
: undefined;
|
|
200
|
+
if (presence?.ok) {
|
|
201
|
+
for (const row of presence.value) add(row.mirrorHash, row.seq, row.present);
|
|
202
|
+
return { termSets };
|
|
203
|
+
}
|
|
204
|
+
let chunks = loadedChunks;
|
|
205
|
+
if (!chunks) {
|
|
206
|
+
const fetched = await store.getChunksBatch(hashes);
|
|
207
|
+
if (!fetched.ok) return null;
|
|
208
|
+
chunks = fetched.value;
|
|
209
|
+
}
|
|
210
|
+
for (const [mirrorHash, list] of chunks)
|
|
211
|
+
for (const chunk of list)
|
|
212
|
+
add(mirrorHash, chunk.seq, chunkTermPresence(chunk.text, terms));
|
|
213
|
+
return { termSets, chunks };
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
/**
|
|
217
|
+
* Default-mode snippets for the returned rows only, built from their stored
|
|
218
|
+
* chunks (FTS5 snippet() re-tokenized every matched document body). Only the
|
|
219
|
+
* chosen chunk of each row is loaded; with an intent, the steered chunk is
|
|
220
|
+
* kept. The snippet, its range and the planner chunk all name that chunk.
|
|
221
|
+
* Updates results in place so hidden ranking metadata stays attached.
|
|
222
|
+
*/
|
|
223
|
+
async function attachChunkSnippets(
|
|
224
|
+
store: StorePort,
|
|
225
|
+
results: SearchResult[],
|
|
226
|
+
query: string,
|
|
227
|
+
options: SearchOptions,
|
|
228
|
+
loadedChunks: Map<string, ChunkRow[]> | undefined
|
|
229
|
+
): Promise<void> {
|
|
230
|
+
const terms = createSnippetTerms(query);
|
|
231
|
+
const hashes = [
|
|
232
|
+
...new Set(
|
|
233
|
+
results
|
|
234
|
+
.map((result) => result.conversion?.mirrorHash)
|
|
235
|
+
.filter((hash): hash is string => Boolean(hash))
|
|
236
|
+
),
|
|
237
|
+
];
|
|
238
|
+
if (!terms || hashes.length === 0) return;
|
|
239
|
+
const read = await readChunkTermSets(store, hashes, terms, loadedChunks);
|
|
240
|
+
if (!read) return;
|
|
241
|
+
const weights = snippetTermWeights([...read.termSets.values()].flat());
|
|
242
|
+
|
|
243
|
+
const chosen = results.map((result) => {
|
|
244
|
+
const mirrorHash = result.conversion?.mirrorHash;
|
|
245
|
+
if (!mirrorHash) return undefined;
|
|
246
|
+
const steeredSeq = result[SEARCH_RESULT_PLANNER_METADATA]?.seq;
|
|
247
|
+
const candidates = read.termSets.get(mirrorHash) ?? [];
|
|
248
|
+
const seq = pickSnippetSeq(
|
|
249
|
+
options.intent
|
|
250
|
+
? candidates.filter((entry) => entry.seq === steeredSeq)
|
|
251
|
+
: candidates,
|
|
252
|
+
weights
|
|
253
|
+
);
|
|
254
|
+
return seq === undefined ? undefined : { mirrorHash, seq };
|
|
255
|
+
});
|
|
256
|
+
let chunks = read.chunks;
|
|
257
|
+
if (!chunks) {
|
|
258
|
+
const fetched = store.getChunksBySequenceBatch
|
|
259
|
+
? await store.getChunksBySequenceBatch(
|
|
260
|
+
chosen.filter((key) => key !== undefined)
|
|
261
|
+
)
|
|
262
|
+
: await store.getChunksBatch(hashes);
|
|
263
|
+
if (!fetched.ok) return;
|
|
264
|
+
chunks = fetched.value;
|
|
265
|
+
}
|
|
266
|
+
const getChunk = createChunkLookup(chunks);
|
|
267
|
+
|
|
268
|
+
for (const [index, result] of results.entries()) {
|
|
269
|
+
const key = chosen[index];
|
|
270
|
+
const chunk = key ? getChunk(key.mirrorHash, key.seq) : undefined;
|
|
271
|
+
if (!(key && chunk)) continue;
|
|
272
|
+
const cleaned = cleanDisplaySnippet(
|
|
273
|
+
renderChunkSnippet(chunk.text, terms, weights),
|
|
274
|
+
chunk.text
|
|
275
|
+
);
|
|
276
|
+
result.line = chunk.startLine + cleaned.startLineOffset;
|
|
277
|
+
result.snippet = cleaned.text;
|
|
278
|
+
result.snippetLanguage = chunk.language ?? undefined;
|
|
279
|
+
result.snippetRange = { startLine: result.line, endLine: chunk.endLine };
|
|
280
|
+
const coordinates = {
|
|
281
|
+
mirrorHash: key.mirrorHash,
|
|
282
|
+
seq: chunk.seq,
|
|
283
|
+
startLine: chunk.startLine,
|
|
284
|
+
endLine: chunk.endLine,
|
|
285
|
+
passageHash: passageHash(chunk.text),
|
|
286
|
+
};
|
|
287
|
+
const metadata = result[SEARCH_RESULT_PLANNER_METADATA];
|
|
288
|
+
if (metadata) {
|
|
289
|
+
Object.assign(metadata, coordinates);
|
|
290
|
+
} else {
|
|
291
|
+
attachSearchResultPlannerMetadata(result, {
|
|
292
|
+
retrievalRank: 0,
|
|
293
|
+
sources: ["bm25"],
|
|
294
|
+
graphExpanded: false,
|
|
295
|
+
...coordinates,
|
|
296
|
+
});
|
|
297
|
+
}
|
|
298
|
+
}
|
|
299
|
+
}
|
|
300
|
+
|
|
168
301
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
169
302
|
// Search Function
|
|
170
303
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
@@ -204,8 +337,7 @@ export async function searchBm25(
|
|
|
204
337
|
const detection = detectQueryLanguage(query);
|
|
205
338
|
const queryLanguage = options.lang ?? detection.bcp47;
|
|
206
339
|
|
|
207
|
-
// Run FTS search
|
|
208
|
-
// Disable FTS snippet when --full or --line-numbers (we use raw text instead)
|
|
340
|
+
// Run FTS search (ranking only; snippets come from chunks of the final rows)
|
|
209
341
|
const ftsResult = await store.searchFts(query, {
|
|
210
342
|
limit: retrievalLimit,
|
|
211
343
|
collection: options.collection,
|
|
@@ -213,7 +345,6 @@ export async function searchBm25(
|
|
|
213
345
|
exclude: options.exclude,
|
|
214
346
|
relPathPrefix: options.retrievalScope?.relPathPrefix,
|
|
215
347
|
language: options.lang,
|
|
216
|
-
snippet: !(options.full || options.lineNumbers),
|
|
217
348
|
tagsAll: options.tagsAll,
|
|
218
349
|
tagsAny: options.tagsAny,
|
|
219
350
|
since: temporalRange.since,
|
|
@@ -262,10 +393,9 @@ export async function searchBm25(
|
|
|
262
393
|
ftsResult.value.map((f) => f.mirrorHash).filter((h): h is string => !!h)
|
|
263
394
|
),
|
|
264
395
|
];
|
|
396
|
+
const needsAllChunks = Boolean(options.intent || options.exclude?.length);
|
|
265
397
|
const chunksMapResult =
|
|
266
|
-
store.getChunksBySequenceBatch &&
|
|
267
|
-
!options.intent &&
|
|
268
|
-
!options.exclude?.length
|
|
398
|
+
store.getChunksBySequenceBatch && !needsAllChunks
|
|
269
399
|
? await store.getChunksBySequenceBatch(
|
|
270
400
|
ftsResult.value
|
|
271
401
|
.filter((row) => !!row.mirrorHash)
|
|
@@ -275,6 +405,10 @@ export async function searchBm25(
|
|
|
275
405
|
const getChunk = chunksMapResult.ok
|
|
276
406
|
? createChunkLookup(chunksMapResult.value)
|
|
277
407
|
: () => undefined;
|
|
408
|
+
const allChunks =
|
|
409
|
+
chunksMapResult.ok && (needsAllChunks || !store.getChunksBySequenceBatch)
|
|
410
|
+
? chunksMapResult.value
|
|
411
|
+
: undefined;
|
|
278
412
|
|
|
279
413
|
// Dedup: multiple docs can share mirror_hash (content-addressed storage)
|
|
280
414
|
// Track seen uri+seq to eliminate duplicate rows from join fan-out
|
|
@@ -323,11 +457,7 @@ export async function searchBm25(
|
|
|
323
457
|
|
|
324
458
|
const excluded =
|
|
325
459
|
matchesExcludedText(
|
|
326
|
-
[
|
|
327
|
-
fts.title ?? "",
|
|
328
|
-
fts.recordSourcePath ?? fts.relPath ?? "",
|
|
329
|
-
fts.snippet ?? "",
|
|
330
|
-
],
|
|
460
|
+
[fts.title ?? "", fts.recordSourcePath ?? fts.relPath ?? ""],
|
|
331
461
|
options.exclude
|
|
332
462
|
) ||
|
|
333
463
|
matchesExcludedChunks(
|
|
@@ -461,11 +591,15 @@ export async function searchBm25(
|
|
|
461
591
|
}
|
|
462
592
|
|
|
463
593
|
const finalResults = filteredResults.slice(0, limit);
|
|
594
|
+
if (!(options.full || options.lineNumbers)) {
|
|
595
|
+
await attachChunkSnippets(store, finalResults, query, options, allChunks);
|
|
596
|
+
}
|
|
464
597
|
for (const [index, result] of finalResults.entries()) {
|
|
465
598
|
const metadata = result[SEARCH_RESULT_PLANNER_METADATA];
|
|
466
599
|
if (metadata) metadata.retrievalRank = index + 1;
|
|
467
600
|
}
|
|
468
601
|
await attachSearchResultContexts(store, finalResults);
|
|
602
|
+
await attachSupersession(store, finalResults);
|
|
469
603
|
const lineageResult = await attachSearchResultEgressLineage(
|
|
470
604
|
store,
|
|
471
605
|
finalResults,
|
package/src/pipeline/types.ts
CHANGED
|
@@ -94,6 +94,8 @@ export interface SearchResult {
|
|
|
94
94
|
record?: RecordEvidenceMetadata;
|
|
95
95
|
/** Complete, most-restrictive collection ownership for this derived result. */
|
|
96
96
|
egressLineage?: EgressLineage;
|
|
97
|
+
/** URI of the current fact when a newer memory fact superseded this one. */
|
|
98
|
+
supersededBy?: string;
|
|
97
99
|
[SEARCH_RESULT_PLANNER_METADATA]?: SearchResultPlannerMetadata;
|
|
98
100
|
}
|
|
99
101
|
|
|
@@ -455,6 +457,8 @@ export interface Citation {
|
|
|
455
457
|
uri: string;
|
|
456
458
|
startLine?: number;
|
|
457
459
|
endLine?: number;
|
|
460
|
+
/** URI of the current fact when a newer memory fact superseded the cited one. */
|
|
461
|
+
supersededBy?: string;
|
|
458
462
|
[CITATION_TRACE_METADATA]?: CitationTraceMetadata;
|
|
459
463
|
}
|
|
460
464
|
|
package/src/pipeline/vsearch.ts
CHANGED
|
@@ -38,7 +38,10 @@ import { typedMetadataWarnings, typedMetadataFilterReason } from "./filters";
|
|
|
38
38
|
import { selectBestChunkForSteering } from "./intent";
|
|
39
39
|
import { hasProjectAffinity } from "./project-affinity";
|
|
40
40
|
import { detectQueryLanguage } from "./query-language";
|
|
41
|
-
import {
|
|
41
|
+
import {
|
|
42
|
+
attachSearchResultContexts,
|
|
43
|
+
attachSupersession,
|
|
44
|
+
} from "./result-context";
|
|
42
45
|
import { cleanDisplaySnippet } from "./snippet";
|
|
43
46
|
import {
|
|
44
47
|
resolveRecencyTimestamp,
|
|
@@ -520,6 +523,7 @@ async function searchVectorWithEmbeddingOwned(
|
|
|
520
523
|
if (metadata) metadata.retrievalRank = index + 1;
|
|
521
524
|
}
|
|
522
525
|
await attachSearchResultContexts(store, finalResults);
|
|
526
|
+
await attachSupersession(store, finalResults);
|
|
523
527
|
const lineageResult = await attachSearchResultEgressLineage(
|
|
524
528
|
store,
|
|
525
529
|
finalResults,
|