@gmickel/gno 2.10.1 → 2.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (112) hide show
  1. package/README.md +11 -11
  2. package/assets/skill/SKILL.md +27 -15
  3. package/assets/skill/cli-reference.md +6 -5
  4. package/assets/skill/examples.md +16 -12
  5. package/assets/skill/mcp-reference.md +2 -1
  6. package/assets/spa-production.json.gz +0 -0
  7. package/browser-extension/artifacts/{gno-browser-clipper-v2.10.1.zip → gno-browser-clipper-v2.12.0.zip} +0 -0
  8. package/browser-extension/artifacts/gno-browser-clipper-v2.12.0.zip.sha256 +1 -0
  9. package/browser-extension/dist/manifest.json +1 -1
  10. package/package.json +10 -5
  11. package/spec/cli.md +127 -35
  12. package/spec/evals-agentic.md +3 -1
  13. package/spec/mcp.md +36 -6
  14. package/spec/output-schemas/ask.schema.json +8 -0
  15. package/spec/output-schemas/claim-verification.schema.json +8 -2
  16. package/spec/output-schemas/context-capsule-v1.schema.json +6 -1
  17. package/spec/output-schemas/context-capsule-verification.schema.json +4 -2
  18. package/spec/output-schemas/get.schema.json +4 -0
  19. package/spec/output-schemas/metadata-summary.schema.json +60 -0
  20. package/spec/output-schemas/multi-get.schema.json +4 -0
  21. package/spec/output-schemas/search-result.schema.json +4 -0
  22. package/spec/output-schemas/search-results.schema.json +4 -0
  23. package/spec/output-schemas/status.schema.json +28 -0
  24. package/src/app/constants.ts +16 -0
  25. package/src/app/context-agent-projection.ts +10 -1
  26. package/src/app/context-format.ts +1 -0
  27. package/src/app/verified-ask.ts +2 -0
  28. package/src/cli/commands/agents/block.ts +10 -9
  29. package/src/cli/commands/agents/commands.ts +6 -5
  30. package/src/cli/commands/agents/engine.ts +15 -2
  31. package/src/cli/commands/agents/harnesses.ts +21 -14
  32. package/src/cli/commands/ask-format.ts +18 -1
  33. package/src/cli/commands/ask.ts +4 -5
  34. package/src/cli/commands/cleanup.ts +46 -6
  35. package/src/cli/commands/collection/remove.ts +1 -0
  36. package/src/cli/commands/collection/rename.ts +1 -0
  37. package/src/cli/commands/daemon.ts +5 -1
  38. package/src/cli/commands/doctor.ts +37 -0
  39. package/src/cli/commands/get.ts +11 -1
  40. package/src/cli/commands/mcp/paths.ts +2 -1
  41. package/src/cli/commands/metadata.ts +79 -0
  42. package/src/cli/commands/multi-get.ts +13 -0
  43. package/src/cli/commands/setup-semantic.ts +6 -1
  44. package/src/cli/commands/shared.ts +35 -8
  45. package/src/cli/commands/skill/paths.ts +20 -2
  46. package/src/cli/commands/status.ts +89 -15
  47. package/src/cli/format/search-results.ts +11 -0
  48. package/src/cli/options.ts +2 -0
  49. package/src/cli/program.ts +64 -3
  50. package/src/core/activation-probe-plan.ts +0 -1
  51. package/src/core/config-mutation.ts +72 -55
  52. package/src/core/context-capsule-schema.ts +2 -0
  53. package/src/core/context-capsule-verification.ts +2 -0
  54. package/src/core/context-compiler.ts +2 -5
  55. package/src/core/context-evidence.ts +1 -0
  56. package/src/core/context-verifier.ts +14 -1
  57. package/src/core/memory-remember.ts +0 -1
  58. package/src/core/metadata-discovery.ts +169 -0
  59. package/src/core/module-path.ts +33 -0
  60. package/src/core/runtime-entrypoint.ts +3 -1
  61. package/src/ingestion/file-processor.ts +6 -3
  62. package/src/ingestion/prepare-file.ts +14 -4
  63. package/src/ingestion/source-availability/root.ts +138 -0
  64. package/src/ingestion/sync.ts +35 -1
  65. package/src/ingestion/walker.ts +11 -3
  66. package/src/llm/native-worker/client.ts +8 -4
  67. package/src/llm/native-worker/errors.ts +26 -3
  68. package/src/mcp/http-egress.ts +1 -0
  69. package/src/mcp/http-transport.ts +7 -2
  70. package/src/mcp/server.ts +3 -0
  71. package/src/mcp/tool-descriptions-core.ts +3 -3
  72. package/src/mcp/tool-profile.ts +16 -1
  73. package/src/mcp/tools/get.ts +10 -0
  74. package/src/mcp/tools/index.ts +68 -4
  75. package/src/mcp/tools/multi-get.ts +15 -0
  76. package/src/mcp/tools/query.ts +5 -3
  77. package/src/mcp/tools/search.ts +5 -3
  78. package/src/mcp/tools/status.ts +17 -2
  79. package/src/mcp/tools/vsearch.ts +5 -3
  80. package/src/mcp/tools/workspace-write.ts +42 -34
  81. package/src/pipeline/answer-prompt.ts +15 -2
  82. package/src/pipeline/answer.ts +3 -0
  83. package/src/pipeline/claim-verification-schema.ts +6 -1
  84. package/src/pipeline/claim-verification.ts +54 -1
  85. package/src/pipeline/explain.ts +1 -1
  86. package/src/pipeline/fusion.ts +24 -4
  87. package/src/pipeline/hybrid.ts +5 -1
  88. package/src/pipeline/query-snippet.ts +266 -0
  89. package/src/pipeline/result-context.ts +29 -0
  90. package/src/pipeline/search.ts +154 -20
  91. package/src/pipeline/types.ts +4 -0
  92. package/src/pipeline/vsearch.ts +5 -1
  93. package/src/sdk/client.ts +46 -6
  94. package/src/sdk/documents.ts +5 -0
  95. package/src/sdk/types.ts +11 -1
  96. package/src/serve/public/components/ThoroughnessSelector.tsx +1 -1
  97. package/src/serve/public/globals.built.css +1 -1
  98. package/src/serve/public/pages/Collections.tsx +8 -0
  99. package/src/serve/public/pages/Search.tsx +9 -1
  100. package/src/serve/resident-request.ts +8 -0
  101. package/src/serve/resident-runtime.ts +25 -3
  102. package/src/serve/routes/api.ts +29 -1
  103. package/src/serve/server.ts +12 -0
  104. package/src/serve/spa-production-build.ts +7 -3
  105. package/src/serve/status-model.ts +2 -0
  106. package/src/serve/status.ts +29 -0
  107. package/src/sessions/config-refresh.ts +6 -0
  108. package/src/sessions/import-child.ts +4 -1
  109. package/src/store/sqlite/adapter.ts +241 -20
  110. package/src/store/types.ts +61 -7
  111. package/src/store/vector/variant-search.ts +112 -74
  112. package/browser-extension/artifacts/gno-browser-clipper-v2.10.1.zip.sha256 +0 -1
@@ -35,6 +35,15 @@ function rrfContribution(rank: number, k: number, weight: number): number {
35
35
  return weight / (k + rank);
36
36
  }
37
37
 
38
+ /**
39
+ * Candidates are documents: BM25 ranks whole documents while vector search
40
+ * ranks chunks, so a chunk-level key would split one document's votes across
41
+ * its chunks. Each list counts a document once, at its best rank.
42
+ */
43
+ function fusionKey(result: RankedInput["results"][number]): string {
44
+ return `${result.documentId ?? result.mirrorHash}`;
45
+ }
46
+
38
47
  // ─────────────────────────────────────────────────────────────────────────────
39
48
  // Fusion Implementation
40
49
  // ─────────────────────────────────────────────────────────────────────────────
@@ -51,7 +60,7 @@ export function rrfFuse(
51
60
  inputs: RankedInput[],
52
61
  config: RrfConfig
53
62
  ): FusionCandidate[] {
54
- // Map: "mirrorHash:seq" -> FusionCandidate
63
+ // Map: document key -> FusionCandidate
55
64
  const candidates = new Map<string, FusionCandidate>();
56
65
 
57
66
  // Separate inputs by type for weight assignment
@@ -74,8 +83,11 @@ export function rrfFuse(
74
83
  ? config.bm25Weight * 2.0
75
84
  : config.bm25Weight * 0.5;
76
85
 
86
+ const countedInInput = new Set<string>();
77
87
  for (const result of input.results) {
78
- const key = `${result.mirrorHash}:${result.seq}:${result.documentId ?? ""}`;
88
+ const key = fusionKey(result);
89
+ if (countedInInput.has(key)) continue;
90
+ countedInInput.add(key);
79
91
  let candidate = candidates.get(key);
80
92
 
81
93
  if (!candidate) {
@@ -116,8 +128,11 @@ export function rrfFuse(
116
128
  weight = config.vecWeight * 0.7;
117
129
  }
118
130
 
131
+ const countedInInput = new Set<string>();
119
132
  for (const result of input.results) {
120
- const key = `${result.mirrorHash}:${result.seq}:${result.documentId ?? ""}`;
133
+ const key = fusionKey(result);
134
+ if (countedInInput.has(key)) continue;
135
+ countedInInput.add(key);
121
136
  let candidate = candidates.get(key);
122
137
 
123
138
  if (!candidate) {
@@ -137,6 +152,8 @@ export function rrfFuse(
137
152
 
138
153
  // Track best vector rank
139
154
  if (candidate.vecRank === null || result.rank < candidate.vecRank) {
155
+ // The best-ranked chunk represents the document downstream.
156
+ candidate.seq = result.seq;
140
157
  candidate.vecRank = result.rank;
141
158
  }
142
159
 
@@ -152,8 +169,11 @@ export function rrfFuse(
152
169
  for (const input of graphInputs) {
153
170
  const weight = config.bm25Weight * 0.8;
154
171
 
172
+ const countedInInput = new Set<string>();
155
173
  for (const result of input.results) {
156
- const key = `${result.mirrorHash}:${result.seq}:${result.documentId ?? ""}`;
174
+ const key = fusionKey(result);
175
+ if (countedInInput.has(key)) continue;
176
+ countedInInput.add(key);
157
177
  let candidate = candidates.get(key);
158
178
 
159
179
  if (!candidate) {
@@ -73,7 +73,10 @@ import {
73
73
  summarizeQueryModes,
74
74
  } from "./query-modes";
75
75
  import { rerankCandidates } from "./rerank";
76
- import { attachSearchResultContexts } from "./result-context";
76
+ import {
77
+ attachSearchResultContexts,
78
+ attachSupersession,
79
+ } from "./result-context";
77
80
  import { cleanDisplaySnippet } from "./snippet";
78
81
  import {
79
82
  isWithinTemporalRange,
@@ -1364,6 +1367,7 @@ async function searchHybridWithHydration(
1364
1367
  }
1365
1368
  : undefined;
1366
1369
  await attachSearchResultContexts(store, finalResults);
1370
+ await attachSupersession(store, finalResults);
1367
1371
  const lineageResult = await attachSearchResultEgressLineage(
1368
1372
  store,
1369
1373
  finalResults,
@@ -0,0 +1,266 @@
1
+ /**
2
+ * BM25 result snippets built from stored chunks.
3
+ *
4
+ * FTS5 `snippet()` re-tokenizes the whole body of every matched document
5
+ * before LIMIT, which made long questions take seconds. Snippets are built
6
+ * for the returned rows only: pick the chunk that best covers the query terms
7
+ * (terms weighted by rarity across the returned chunks, so a chunk with
8
+ * "ladder" beats one with only "the" and "does"; lowest seq on ties), then
9
+ * its best 32-token window, with matching tokens wrapped in `<mark>`/`</mark>`
10
+ * and `...` where the window cuts the chunk, as `snippet()` did. Chunk choice
11
+ * uses substring presence of lightly stemmed terms; highlighting uses word
12
+ * prefixes. Both approximate the index tokenizer and affect display only,
13
+ * never ranking.
14
+ *
15
+ * @module src/pipeline/query-snippet
16
+ */
17
+
18
+ /** Tokens per snippet window (the old `snippet(..., 32)` budget). */
19
+ const WINDOW_TOKENS = 32;
20
+
21
+ /** Repeat hits only break ties between windows with equal term coverage. */
22
+ const HIT_TIEBREAK = 1e-6;
23
+
24
+ const TOKEN_REGEX = /[\p{L}\p{N}\p{M}]+/gu;
25
+
26
+ /** Quoted phrase (optionally negated) or a bare token, as the lexical grammar splits them. */
27
+ const QUERY_PART_REGEX = /(-?)"([^"]*)"|(\S+)/g;
28
+
29
+ const DIACRITIC_REGEX = /\p{M}/gu;
30
+
31
+ const NON_ASCII_REGEX = /[^\p{ASCII}]/u;
32
+
33
+ /** Final doubled consonant left by suffix removal (`embedd` -> `embed`). */
34
+ const DOUBLED_CONSONANT_REGEX = /([bdfgkmnprt])\1$/;
35
+
36
+ /** Suffixes removed from query terms, longest first. */
37
+ const STEM_SUFFIXES = [
38
+ "ational",
39
+ "ations",
40
+ "ation",
41
+ "ings",
42
+ "ions",
43
+ "ers",
44
+ "ied",
45
+ "ies",
46
+ "ing",
47
+ "ion",
48
+ "al",
49
+ "ed",
50
+ "er",
51
+ "ly",
52
+ "e",
53
+ "s",
54
+ "y",
55
+ ];
56
+
57
+ const MIN_STEM_LENGTH = 3;
58
+
59
+ export interface SnippetTerms {
60
+ /** Lowercase term prefixes (stemmed query words and diacritic-free forms). */
61
+ prefixes: string[];
62
+ /** Query word id per prefix; variants of one word share an id. */
63
+ ids: number[];
64
+ /** Anchored prefix matcher for one lowercased token. */
65
+ token: RegExp;
66
+ idByPrefix: Map<string, number>;
67
+ }
68
+
69
+ /** Query word ids present in one chunk. */
70
+ export interface ChunkTermSet {
71
+ seq: number;
72
+ terms: Set<number>;
73
+ }
74
+
75
+ interface Token {
76
+ start: number;
77
+ end: number;
78
+ term: number;
79
+ }
80
+
81
+ const stripDiacritics = (text: string): string =>
82
+ NON_ASCII_REGEX.test(text)
83
+ ? text.normalize("NFD").replace(DIACRITIC_REGEX, "")
84
+ : text;
85
+
86
+ function stemPrefix(term: string): string {
87
+ if (term.length <= MIN_STEM_LENGTH) return term;
88
+ for (const suffix of STEM_SUFFIXES) {
89
+ if (
90
+ term.endsWith(suffix) &&
91
+ term.length - suffix.length >= MIN_STEM_LENGTH
92
+ ) {
93
+ const stem = term.slice(0, -suffix.length);
94
+ return DOUBLED_CONSONANT_REGEX.test(stem) ? stem.slice(0, -1) : stem;
95
+ }
96
+ }
97
+ return term;
98
+ }
99
+
100
+ /** Positive terms of a lexical query; null when there are none. */
101
+ export function createSnippetTerms(query: string): SnippetTerms | null {
102
+ const idByPrefix = new Map<string, number>();
103
+ let nextId = 0;
104
+ for (const part of query.matchAll(QUERY_PART_REGEX)) {
105
+ const bare = part[3];
106
+ if (part[1] === "-" || bare?.startsWith("-")) continue;
107
+ for (const [word] of (part[2] ?? bare ?? "").matchAll(TOKEN_REGEX)) {
108
+ const lower = word.toLowerCase();
109
+ for (const form of new Set([lower, stripDiacritics(lower)])) {
110
+ const prefix = stemPrefix(form);
111
+ if (!idByPrefix.has(prefix)) idByPrefix.set(prefix, nextId);
112
+ }
113
+ nextId += 1;
114
+ }
115
+ }
116
+ if (idByPrefix.size === 0) return null;
117
+ const prefixes = [...idByPrefix.keys()].sort((a, b) => b.length - a.length);
118
+ return {
119
+ prefixes,
120
+ ids: prefixes.map((prefix) => idByPrefix.get(prefix) ?? -1),
121
+ token: new RegExp(`^(?:${prefixes.join("|")})`, "u"),
122
+ idByPrefix,
123
+ };
124
+ }
125
+
126
+ /** In-memory equivalent of the store's chunk term presence query. */
127
+ export function chunkTermPresence(
128
+ text: string,
129
+ terms: SnippetTerms
130
+ ): boolean[] {
131
+ const lower = text.toLowerCase();
132
+ return terms.prefixes.map((prefix) => lower.includes(prefix));
133
+ }
134
+
135
+ export function presentTermIds(
136
+ present: boolean[],
137
+ terms: SnippetTerms
138
+ ): Set<number> {
139
+ const ids = new Set<number>();
140
+ for (const [index, flag] of present.entries()) {
141
+ const id = terms.ids[index];
142
+ if (flag && id !== undefined) ids.add(id);
143
+ }
144
+ return ids;
145
+ }
146
+
147
+ /** Rarity weight per query word across all candidate chunks. */
148
+ export function snippetTermWeights(
149
+ chunks: Iterable<ChunkTermSet>
150
+ ): Map<number, number> {
151
+ let total = 0;
152
+ const frequency = new Map<number, number>();
153
+ for (const { terms } of chunks) {
154
+ total += 1;
155
+ for (const term of terms)
156
+ frequency.set(term, (frequency.get(term) ?? 0) + 1);
157
+ }
158
+ const weights = new Map<number, number>();
159
+ for (const [term, count] of frequency)
160
+ weights.set(term, Math.log(1 + total / count));
161
+ return weights;
162
+ }
163
+
164
+ /** Seq of the chunk with the best weighted coverage (lowest seq on ties). */
165
+ export function pickSnippetSeq(
166
+ chunks: ChunkTermSet[],
167
+ weights: Map<number, number>
168
+ ): number | undefined {
169
+ let best: ChunkTermSet | undefined;
170
+ let bestScore = -1;
171
+ for (const chunk of chunks) {
172
+ let score = 0;
173
+ for (const term of chunk.terms) score += weights.get(term) ?? 0;
174
+ if (
175
+ score > bestScore ||
176
+ (score === bestScore && best && chunk.seq < best.seq)
177
+ ) {
178
+ bestScore = score;
179
+ best = chunk;
180
+ }
181
+ }
182
+ return best?.seq;
183
+ }
184
+
185
+ function tokenize(text: string, terms: SnippetTerms): Token[] {
186
+ const tokens: Token[] = [];
187
+ for (const match of text.matchAll(TOKEN_REGEX)) {
188
+ const start = match.index;
189
+ const word = match[0];
190
+ const prefix = terms.token.exec(stripDiacritics(word.toLowerCase()));
191
+ tokens.push({
192
+ start,
193
+ end: start + word.length,
194
+ term: prefix ? (terms.idByPrefix.get(prefix[0]) ?? -1) : -1,
195
+ });
196
+ }
197
+ return tokens;
198
+ }
199
+
200
+ /** Start index of the window with the best weighted term coverage. */
201
+ function bestWindowStart(
202
+ tokens: Token[],
203
+ weights: Map<number, number>
204
+ ): number {
205
+ if (tokens.length <= WINDOW_TOKENS) return 0;
206
+ const counts = new Map<number, number>();
207
+ let covered = 0;
208
+ let hits = 0;
209
+ const add = (term: number, delta: number) => {
210
+ if (term < 0) return;
211
+ const count = (counts.get(term) ?? 0) + delta;
212
+ counts.set(term, count);
213
+ hits += delta;
214
+ const weight = weights.get(term) ?? 1;
215
+ if (delta > 0 && count === 1) covered += weight;
216
+ if (delta < 0 && count === 0) covered -= weight;
217
+ };
218
+ let bestStart = 0;
219
+ let bestScore = -1;
220
+ for (let index = 0; index < tokens.length; index += 1) {
221
+ add(tokens[index]?.term ?? -1, 1);
222
+ const start = index - WINDOW_TOKENS + 1;
223
+ if (start > 0) add(tokens[start - 1]?.term ?? -1, -1);
224
+ if (start < 0) continue;
225
+ const score = covered + hits * HIT_TIEBREAK;
226
+ if (score > bestScore) {
227
+ bestScore = score;
228
+ bestStart = start;
229
+ }
230
+ }
231
+ // Center the hits inside the chosen window.
232
+ let first = -1;
233
+ let last = -1;
234
+ for (let index = bestStart; index < bestStart + WINDOW_TOKENS; index += 1) {
235
+ if ((tokens[index]?.term ?? -1) < 0) continue;
236
+ if (first < 0) first = index;
237
+ last = index;
238
+ }
239
+ if (first < 0) return bestStart;
240
+ const centered = first - Math.floor((WINDOW_TOKENS - (last - first + 1)) / 2);
241
+ return Math.max(0, Math.min(centered, tokens.length - WINDOW_TOKENS));
242
+ }
243
+
244
+ /** The best 32-token window of one chunk, query terms in `<mark>` tags. */
245
+ export function renderChunkSnippet(
246
+ text: string,
247
+ terms: SnippetTerms,
248
+ weights: Map<number, number>
249
+ ): string {
250
+ const tokens = tokenize(text, terms);
251
+ if (tokens.length === 0) return text;
252
+ const start = bestWindowStart(tokens, weights);
253
+ const end = Math.min(tokens.length, start + WINDOW_TOKENS) - 1;
254
+ const atStart = start === 0;
255
+ const atEnd = end === tokens.length - 1;
256
+ let cursor = atStart ? 0 : (tokens[start]?.start ?? 0);
257
+ let out = atStart ? "" : "...";
258
+ for (let index = start; index <= end; index += 1) {
259
+ const token = tokens[index];
260
+ if (!token || token.term < 0) continue;
261
+ out += `${text.slice(cursor, token.start)}<mark>${text.slice(token.start, token.end)}</mark>`;
262
+ cursor = token.end;
263
+ }
264
+ out += text.slice(cursor, atEnd ? text.length : (tokens[end]?.end ?? 0));
265
+ return atEnd ? out : `${out}...`;
266
+ }
@@ -57,3 +57,32 @@ export async function attachSearchResultContexts(
57
57
  // the historical result shape and are reported by config validation.
58
58
  }
59
59
  }
60
+
61
+ /**
62
+ * Mark results whose memory fact a newer fact superseded, naming the current
63
+ * successor. Fail-open like context guidance: a lookup failure marks nothing.
64
+ */
65
+ export async function attachSupersession(
66
+ store: StorePort,
67
+ results: { uri: string; supersededBy?: string }[]
68
+ ): Promise<void> {
69
+ if (results.length === 0 || !store.getSupersedingUris) return;
70
+ const successors = await store.getSupersedingUris(
71
+ results.map((result) => result.uri)
72
+ );
73
+ if (!successors.ok) return;
74
+ for (const result of results) {
75
+ const successor = successors.value.get(result.uri);
76
+ if (successor) result.supersededBy = successor;
77
+ }
78
+ }
79
+
80
+ /** Current successor URI of one document's superseded memory fact, if any. */
81
+ export async function supersedingUri(
82
+ store: StorePort,
83
+ uri: string
84
+ ): Promise<string | undefined> {
85
+ if (!store.getSupersedingUris) return undefined;
86
+ const successors = await store.getSupersedingUris([uri]);
87
+ return successors.ok ? successors.value.get(uri) : undefined;
88
+ }
@@ -8,6 +8,7 @@
8
8
  import { join as pathJoin } from "node:path"; // No Bun path utils equivalent
9
9
 
10
10
  import type { ChunkRow, FtsResult, StorePort } from "../store/types";
11
+ import type { ChunkTermSet, SnippetTerms } from "./query-snippet";
11
12
  import type {
12
13
  SearchOptions,
13
14
  SearchResult,
@@ -31,7 +32,18 @@ import { typedMetadataWarnings } from "./filters";
31
32
  import { selectBestChunkForSteering } from "./intent";
32
33
  import { hasProjectAffinity } from "./project-affinity";
33
34
  import { detectQueryLanguage } from "./query-language";
34
- import { attachSearchResultContexts } from "./result-context";
35
+ import {
36
+ chunkTermPresence,
37
+ createSnippetTerms,
38
+ pickSnippetSeq,
39
+ presentTermIds,
40
+ renderChunkSnippet,
41
+ snippetTermWeights,
42
+ } from "./query-snippet";
43
+ import {
44
+ attachSearchResultContexts,
45
+ attachSupersession,
46
+ } from "./result-context";
35
47
  import { cleanDisplaySnippet } from "./snippet";
36
48
  import {
37
49
  resolveRecencyTimestamp,
@@ -41,6 +53,9 @@ import {
41
53
  import { attachSearchResultPlannerMetadata } from "./trace-metadata";
42
54
  import { SEARCH_RESULT_PLANNER_METADATA } from "./types";
43
55
 
56
+ const passageHash = (text: string): string =>
57
+ new Bun.CryptoHasher("sha256").update(text).digest("hex");
58
+
44
59
  // ─────────────────────────────────────────────────────────────────────────────
45
60
  // Score Normalization
46
61
  // ─────────────────────────────────────────────────────────────────────────────
@@ -121,11 +136,9 @@ function buildSearchResult(ctx: BuildResultContext): SearchResult {
121
136
  snippet = chunk.text;
122
137
  snippetRange = { startLine: chunk.startLine, endLine: chunk.endLine };
123
138
  } else {
124
- // Default: FTS snippet or chunk text, with leading frontmatter stripped
125
- const cleaned = cleanDisplaySnippet(
126
- fts.snippet ?? chunk?.text ?? "",
127
- chunk?.text
128
- );
139
+ // Default: chunk text until attachChunkSnippets replaces it for the final
140
+ // rows, with leading frontmatter stripped
141
+ const cleaned = cleanDisplaySnippet(chunk?.text ?? "", chunk?.text);
129
142
  snippet = cleaned.text;
130
143
  if (chunk) {
131
144
  line = chunk.startLine + cleaned.startLineOffset;
@@ -159,12 +172,132 @@ function buildSearchResult(ctx: BuildResultContext): SearchResult {
159
172
  graphExpanded: false,
160
173
  startLine: chunk.startLine,
161
174
  endLine: chunk.endLine,
162
- passageHash: new Bun.CryptoHasher("sha256")
163
- .update(chunk.text)
164
- .digest("hex"),
175
+ passageHash: passageHash(chunk.text),
165
176
  });
166
177
  }
167
178
 
179
+ /** Query-word ids present in each chunk of the returned documents. */
180
+ async function readChunkTermSets(
181
+ store: StorePort,
182
+ hashes: string[],
183
+ terms: SnippetTerms,
184
+ loadedChunks: Map<string, ChunkRow[]> | undefined
185
+ ): Promise<{
186
+ termSets: Map<string, ChunkTermSet[]>;
187
+ chunks?: Map<string, ChunkRow[]>;
188
+ } | null> {
189
+ const termSets = new Map<string, ChunkTermSet[]>();
190
+ const add = (mirrorHash: string, seq: number, present: boolean[]) => {
191
+ const list = termSets.get(mirrorHash) ?? [];
192
+ list.push({ seq, terms: presentTermIds(present, terms) });
193
+ termSets.set(mirrorHash, list);
194
+ };
195
+ // In-database presence keeps long documents from being loaded whole.
196
+ const presence =
197
+ !loadedChunks && store.getChunkTermPresence
198
+ ? await store.getChunkTermPresence(hashes, terms.prefixes)
199
+ : undefined;
200
+ if (presence?.ok) {
201
+ for (const row of presence.value) add(row.mirrorHash, row.seq, row.present);
202
+ return { termSets };
203
+ }
204
+ let chunks = loadedChunks;
205
+ if (!chunks) {
206
+ const fetched = await store.getChunksBatch(hashes);
207
+ if (!fetched.ok) return null;
208
+ chunks = fetched.value;
209
+ }
210
+ for (const [mirrorHash, list] of chunks)
211
+ for (const chunk of list)
212
+ add(mirrorHash, chunk.seq, chunkTermPresence(chunk.text, terms));
213
+ return { termSets, chunks };
214
+ }
215
+
216
+ /**
217
+ * Default-mode snippets for the returned rows only, built from their stored
218
+ * chunks (FTS5 snippet() re-tokenized every matched document body). Only the
219
+ * chosen chunk of each row is loaded; with an intent, the steered chunk is
220
+ * kept. The snippet, its range and the planner chunk all name that chunk.
221
+ * Updates results in place so hidden ranking metadata stays attached.
222
+ */
223
+ async function attachChunkSnippets(
224
+ store: StorePort,
225
+ results: SearchResult[],
226
+ query: string,
227
+ options: SearchOptions,
228
+ loadedChunks: Map<string, ChunkRow[]> | undefined
229
+ ): Promise<void> {
230
+ const terms = createSnippetTerms(query);
231
+ const hashes = [
232
+ ...new Set(
233
+ results
234
+ .map((result) => result.conversion?.mirrorHash)
235
+ .filter((hash): hash is string => Boolean(hash))
236
+ ),
237
+ ];
238
+ if (!terms || hashes.length === 0) return;
239
+ const read = await readChunkTermSets(store, hashes, terms, loadedChunks);
240
+ if (!read) return;
241
+ const weights = snippetTermWeights([...read.termSets.values()].flat());
242
+
243
+ const chosen = results.map((result) => {
244
+ const mirrorHash = result.conversion?.mirrorHash;
245
+ if (!mirrorHash) return undefined;
246
+ const steeredSeq = result[SEARCH_RESULT_PLANNER_METADATA]?.seq;
247
+ const candidates = read.termSets.get(mirrorHash) ?? [];
248
+ const seq = pickSnippetSeq(
249
+ options.intent
250
+ ? candidates.filter((entry) => entry.seq === steeredSeq)
251
+ : candidates,
252
+ weights
253
+ );
254
+ return seq === undefined ? undefined : { mirrorHash, seq };
255
+ });
256
+ let chunks = read.chunks;
257
+ if (!chunks) {
258
+ const fetched = store.getChunksBySequenceBatch
259
+ ? await store.getChunksBySequenceBatch(
260
+ chosen.filter((key) => key !== undefined)
261
+ )
262
+ : await store.getChunksBatch(hashes);
263
+ if (!fetched.ok) return;
264
+ chunks = fetched.value;
265
+ }
266
+ const getChunk = createChunkLookup(chunks);
267
+
268
+ for (const [index, result] of results.entries()) {
269
+ const key = chosen[index];
270
+ const chunk = key ? getChunk(key.mirrorHash, key.seq) : undefined;
271
+ if (!(key && chunk)) continue;
272
+ const cleaned = cleanDisplaySnippet(
273
+ renderChunkSnippet(chunk.text, terms, weights),
274
+ chunk.text
275
+ );
276
+ result.line = chunk.startLine + cleaned.startLineOffset;
277
+ result.snippet = cleaned.text;
278
+ result.snippetLanguage = chunk.language ?? undefined;
279
+ result.snippetRange = { startLine: result.line, endLine: chunk.endLine };
280
+ const coordinates = {
281
+ mirrorHash: key.mirrorHash,
282
+ seq: chunk.seq,
283
+ startLine: chunk.startLine,
284
+ endLine: chunk.endLine,
285
+ passageHash: passageHash(chunk.text),
286
+ };
287
+ const metadata = result[SEARCH_RESULT_PLANNER_METADATA];
288
+ if (metadata) {
289
+ Object.assign(metadata, coordinates);
290
+ } else {
291
+ attachSearchResultPlannerMetadata(result, {
292
+ retrievalRank: 0,
293
+ sources: ["bm25"],
294
+ graphExpanded: false,
295
+ ...coordinates,
296
+ });
297
+ }
298
+ }
299
+ }
300
+
168
301
  // ─────────────────────────────────────────────────────────────────────────────
169
302
  // Search Function
170
303
  // ─────────────────────────────────────────────────────────────────────────────
@@ -204,8 +337,7 @@ export async function searchBm25(
204
337
  const detection = detectQueryLanguage(query);
205
338
  const queryLanguage = options.lang ?? detection.bcp47;
206
339
 
207
- // Run FTS search
208
- // Disable FTS snippet when --full or --line-numbers (we use raw text instead)
340
+ // Run FTS search (ranking only; snippets come from chunks of the final rows)
209
341
  const ftsResult = await store.searchFts(query, {
210
342
  limit: retrievalLimit,
211
343
  collection: options.collection,
@@ -213,7 +345,6 @@ export async function searchBm25(
213
345
  exclude: options.exclude,
214
346
  relPathPrefix: options.retrievalScope?.relPathPrefix,
215
347
  language: options.lang,
216
- snippet: !(options.full || options.lineNumbers),
217
348
  tagsAll: options.tagsAll,
218
349
  tagsAny: options.tagsAny,
219
350
  since: temporalRange.since,
@@ -262,10 +393,9 @@ export async function searchBm25(
262
393
  ftsResult.value.map((f) => f.mirrorHash).filter((h): h is string => !!h)
263
394
  ),
264
395
  ];
396
+ const needsAllChunks = Boolean(options.intent || options.exclude?.length);
265
397
  const chunksMapResult =
266
- store.getChunksBySequenceBatch &&
267
- !options.intent &&
268
- !options.exclude?.length
398
+ store.getChunksBySequenceBatch && !needsAllChunks
269
399
  ? await store.getChunksBySequenceBatch(
270
400
  ftsResult.value
271
401
  .filter((row) => !!row.mirrorHash)
@@ -275,6 +405,10 @@ export async function searchBm25(
275
405
  const getChunk = chunksMapResult.ok
276
406
  ? createChunkLookup(chunksMapResult.value)
277
407
  : () => undefined;
408
+ const allChunks =
409
+ chunksMapResult.ok && (needsAllChunks || !store.getChunksBySequenceBatch)
410
+ ? chunksMapResult.value
411
+ : undefined;
278
412
 
279
413
  // Dedup: multiple docs can share mirror_hash (content-addressed storage)
280
414
  // Track seen uri+seq to eliminate duplicate rows from join fan-out
@@ -323,11 +457,7 @@ export async function searchBm25(
323
457
 
324
458
  const excluded =
325
459
  matchesExcludedText(
326
- [
327
- fts.title ?? "",
328
- fts.recordSourcePath ?? fts.relPath ?? "",
329
- fts.snippet ?? "",
330
- ],
460
+ [fts.title ?? "", fts.recordSourcePath ?? fts.relPath ?? ""],
331
461
  options.exclude
332
462
  ) ||
333
463
  matchesExcludedChunks(
@@ -461,11 +591,15 @@ export async function searchBm25(
461
591
  }
462
592
 
463
593
  const finalResults = filteredResults.slice(0, limit);
594
+ if (!(options.full || options.lineNumbers)) {
595
+ await attachChunkSnippets(store, finalResults, query, options, allChunks);
596
+ }
464
597
  for (const [index, result] of finalResults.entries()) {
465
598
  const metadata = result[SEARCH_RESULT_PLANNER_METADATA];
466
599
  if (metadata) metadata.retrievalRank = index + 1;
467
600
  }
468
601
  await attachSearchResultContexts(store, finalResults);
602
+ await attachSupersession(store, finalResults);
469
603
  const lineageResult = await attachSearchResultEgressLineage(
470
604
  store,
471
605
  finalResults,
@@ -94,6 +94,8 @@ export interface SearchResult {
94
94
  record?: RecordEvidenceMetadata;
95
95
  /** Complete, most-restrictive collection ownership for this derived result. */
96
96
  egressLineage?: EgressLineage;
97
+ /** URI of the current fact when a newer memory fact superseded this one. */
98
+ supersededBy?: string;
97
99
  [SEARCH_RESULT_PLANNER_METADATA]?: SearchResultPlannerMetadata;
98
100
  }
99
101
 
@@ -455,6 +457,8 @@ export interface Citation {
455
457
  uri: string;
456
458
  startLine?: number;
457
459
  endLine?: number;
460
+ /** URI of the current fact when a newer memory fact superseded the cited one. */
461
+ supersededBy?: string;
458
462
  [CITATION_TRACE_METADATA]?: CitationTraceMetadata;
459
463
  }
460
464
 
@@ -38,7 +38,10 @@ import { typedMetadataWarnings, typedMetadataFilterReason } from "./filters";
38
38
  import { selectBestChunkForSteering } from "./intent";
39
39
  import { hasProjectAffinity } from "./project-affinity";
40
40
  import { detectQueryLanguage } from "./query-language";
41
- import { attachSearchResultContexts } from "./result-context";
41
+ import {
42
+ attachSearchResultContexts,
43
+ attachSupersession,
44
+ } from "./result-context";
42
45
  import { cleanDisplaySnippet } from "./snippet";
43
46
  import {
44
47
  resolveRecencyTimestamp,
@@ -520,6 +523,7 @@ async function searchVectorWithEmbeddingOwned(
520
523
  if (metadata) metadata.retrievalRank = index + 1;
521
524
  }
522
525
  await attachSearchResultContexts(store, finalResults);
526
+ await attachSupersession(store, finalResults);
523
527
  const lineageResult = await attachSearchResultEgressLineage(
524
528
  store,
525
529
  finalResults,