@gmickel/gno 2.11.0 → 2.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. package/README.md +1 -1
  2. package/assets/skill/SKILL.md +3 -2
  3. package/assets/spa-production.json.gz +0 -0
  4. package/browser-extension/artifacts/{gno-browser-clipper-v2.11.0.zip → gno-browser-clipper-v2.12.0.zip} +0 -0
  5. package/browser-extension/artifacts/gno-browser-clipper-v2.12.0.zip.sha256 +1 -0
  6. package/browser-extension/dist/manifest.json +1 -1
  7. package/package.json +8 -5
  8. package/spec/cli.md +100 -34
  9. package/spec/evals-agentic.md +3 -1
  10. package/spec/mcp.md +9 -3
  11. package/spec/output-schemas/status.schema.json +28 -0
  12. package/src/app/context-agent-projection.ts +10 -1
  13. package/src/cli/commands/agents/block.ts +2 -2
  14. package/src/cli/commands/agents/commands.ts +6 -5
  15. package/src/cli/commands/agents/engine.ts +15 -2
  16. package/src/cli/commands/agents/harnesses.ts +21 -14
  17. package/src/cli/commands/cleanup.ts +46 -6
  18. package/src/cli/commands/collection/remove.ts +1 -0
  19. package/src/cli/commands/collection/rename.ts +1 -0
  20. package/src/cli/commands/doctor.ts +37 -0
  21. package/src/cli/commands/mcp/paths.ts +2 -1
  22. package/src/cli/commands/setup-semantic.ts +6 -1
  23. package/src/cli/commands/shared.ts +26 -1
  24. package/src/cli/commands/skill/paths.ts +20 -2
  25. package/src/cli/commands/status.ts +89 -15
  26. package/src/cli/program.ts +19 -3
  27. package/src/core/activation-probe-plan.ts +0 -1
  28. package/src/core/config-mutation.ts +72 -55
  29. package/src/core/memory-remember.ts +0 -1
  30. package/src/core/module-path.ts +33 -0
  31. package/src/core/runtime-entrypoint.ts +3 -1
  32. package/src/ingestion/file-processor.ts +6 -3
  33. package/src/ingestion/prepare-file.ts +14 -4
  34. package/src/ingestion/source-availability/root.ts +138 -0
  35. package/src/ingestion/sync.ts +5 -19
  36. package/src/llm/native-worker/client.ts +8 -4
  37. package/src/llm/native-worker/errors.ts +26 -3
  38. package/src/mcp/http-transport.ts +7 -2
  39. package/src/mcp/server.ts +3 -0
  40. package/src/mcp/tool-descriptions-core.ts +2 -2
  41. package/src/mcp/tools/index.ts +30 -1
  42. package/src/mcp/tools/status.ts +17 -2
  43. package/src/pipeline/query-snippet.ts +266 -0
  44. package/src/pipeline/search.ts +149 -19
  45. package/src/sdk/client.ts +14 -2
  46. package/src/sdk/types.ts +6 -1
  47. package/src/serve/public/components/ThoroughnessSelector.tsx +1 -1
  48. package/src/serve/public/globals.built.css +1 -1
  49. package/src/serve/public/pages/Collections.tsx +8 -0
  50. package/src/serve/public/pages/Search.tsx +9 -1
  51. package/src/serve/resident-request.ts +8 -0
  52. package/src/serve/resident-runtime.ts +25 -3
  53. package/src/serve/spa-production-build.ts +7 -3
  54. package/src/serve/status-model.ts +2 -0
  55. package/src/serve/status.ts +29 -0
  56. package/src/sessions/config-refresh.ts +6 -0
  57. package/src/sessions/import-child.ts +4 -1
  58. package/src/store/sqlite/adapter.ts +159 -18
  59. package/src/store/types.ts +45 -7
  60. package/browser-extension/artifacts/gno-browser-clipper-v2.11.0.zip.sha256 +0 -1
@@ -37,6 +37,9 @@ import {
37
37
  assertInferenceActive,
38
38
  acquireInferencePermit,
39
39
  } from "../../llm/inference-scope";
40
+ import { envIsSet } from "../../llm/policy";
41
+ import { staleCollectionsMessage } from "../../store/sqlite/adapter";
42
+ import { MCP_ACTIVATION_VERIFICATION_ENV } from "../activation-verification-mode";
40
43
  import { exposesHostPaths } from "../context";
41
44
  import { profileToolDescription } from "../tool-descriptions-core";
42
45
  import {
@@ -175,7 +178,7 @@ export function normalizeTagFilters(tags?: string[]): string[] | undefined {
175
178
 
176
179
  export const MCP_TOOL_DESCRIPTIONS = {
177
180
  search:
178
- "BM25 keyword search. Fast exact-term lookup for names, identifiers, error text, and known phrases. Structured results include uri/docid, line when available, and optional user-configured context guidance; use gno_get with fromLine/lineCount or gno_multi_get for full context. For natural-language questions use gno_query (fast=true first).",
181
+ "BM25 keyword search. Fast lookup of exact text you already have verbatim: identifiers, filenames, error text, and quoted phrases. Structured results include uri/docid, line when available, and optional user-configured context guidance; use gno_get with fromLine/lineCount or gno_multi_get for full context. A question, or a name or topic taken from it, is not exact text: use gno_query (fast=true first).",
179
182
  vsearch:
180
183
  "Vector semantic search. Finds conceptually similar docs with different wording. Structured results preserve optional user-configured context guidance. Best after embeddings are current; use intent to disambiguate short terms. For questions, gno_query with fast=true adds keyword matching at about the same cost.",
181
184
  query:
@@ -1014,6 +1017,7 @@ export async function runTool<T>(
1014
1017
  // Sequential execution via mutex
1015
1018
  const release = await acquireInferencePermit(() => ctx.toolMutex.acquire());
1016
1019
  try {
1020
+ assertConfiguredCollections(ctx, name);
1017
1021
  assertInferenceActive();
1018
1022
  const raw = await (ctx.runWithSnapshot?.(fn) ?? fn());
1019
1023
  assertInferenceActive();
@@ -1084,6 +1088,31 @@ export async function runToolNoMutex<T>(
1084
1088
  }
1085
1089
  }
1086
1090
 
1091
+ /** Orientation tools that report the index state instead of reading it. */
1092
+ const STALE_CHECK_EXEMPT_TOOLS = new Set(["gno_status", "gno_peek"]);
1093
+
1094
+ /**
1095
+ * Refuse a tool call while the index holds collections this config omits
1096
+ * (another client indexed them after this server opened). Status and peek
1097
+ * still run, and a connector proof reads one collection through a subset
1098
+ * config.
1099
+ */
1100
+ function assertConfiguredCollections(ctx: ToolContext, name: string): void {
1101
+ // Store-free stand-ins (an uninitialized peek, test doubles) have no index.
1102
+ const store = ctx.store as Partial<ToolContext["store"]> | undefined;
1103
+ if (
1104
+ STALE_CHECK_EXEMPT_TOOLS.has(name) ||
1105
+ typeof store?.staleIndexedCollections !== "function" ||
1106
+ envIsSet(process.env, MCP_ACTIVATION_VERIFICATION_ENV)
1107
+ ) {
1108
+ return;
1109
+ }
1110
+ const stale = store.staleIndexedCollections(
1111
+ ctx.collections.map((collection) => collection.name)
1112
+ );
1113
+ if (stale.length > 0) throw new Error(staleCollectionsMessage(stale));
1114
+ }
1115
+
1087
1116
  function parseErrorMessage(message: string): { [x: string]: unknown } {
1088
1117
  const match = message.match(/^([A-Z_]+):\s*(.*)$/);
1089
1118
  if (match) {
@@ -12,6 +12,10 @@ import { formatChunkingStatus } from "../../core/chunking-status";
12
12
  import { OWNER_CONFIG_PATH_FIELDS, withoutFields } from "../../core/host-paths";
13
13
  import { formatLinkWorkspace } from "../../core/link-workspace";
14
14
  import { formatVectorPartitionLines } from "../../core/vector-partition-status";
15
+ import {
16
+ type CollectionRootError,
17
+ withCollectionRootErrors,
18
+ } from "../../ingestion/source-availability/root";
15
19
  import { resolveModelUri } from "../../llm/registry";
16
20
  import { createStandaloneResidentStatus } from "../../serve/resident-status";
17
21
  import { exposesHostPaths } from "../context";
@@ -23,7 +27,10 @@ type StatusInput = Record<string, never>;
23
27
  type StatusView = Omit<IndexStatus, "configPath" | "dbPath" | "collections"> &
24
28
  Partial<Pick<IndexStatus, "configPath" | "dbPath">> & {
25
29
  collections: Array<
26
- Omit<CollectionStatus, "path"> & Partial<Pick<CollectionStatus, "path">>
30
+ Omit<CollectionStatus, "path"> &
31
+ Partial<Pick<CollectionStatus, "path">> & {
32
+ rootError?: CollectionRootError;
33
+ }
27
34
  >;
28
35
  };
29
36
 
@@ -58,6 +65,11 @@ function formatStatus(status: StatusView): string {
58
65
  );
59
66
  const workspace = formatLinkWorkspace(c);
60
67
  if (workspace) lines.push(` Link workspace: ${workspace}`);
68
+ if (c.rootError) {
69
+ lines.push(
70
+ ` Root unavailable: [${c.rootError.code}] ${c.rootError.message}`
71
+ );
72
+ }
61
73
  }
62
74
  }
63
75
 
@@ -113,7 +125,10 @@ export function handleStatus(
113
125
 
114
126
  // Override configPath with actual path from context
115
127
  const status = {
116
- ...result.value,
128
+ ...(await withCollectionRootErrors(
129
+ result.value,
130
+ ctx.config.collections
131
+ )),
117
132
  configPath: ctx.actualConfigPath,
118
133
  contentTypeBoost: buildContentTypeBoostStatus(
119
134
  ctx.config.contentTypes ?? []
@@ -0,0 +1,266 @@
1
+ /**
2
+ * BM25 result snippets built from stored chunks.
3
+ *
4
+ * FTS5 `snippet()` re-tokenizes the whole body of every matched document
5
+ * before LIMIT, which made long questions take seconds. Snippets are built
6
+ * for the returned rows only: pick the chunk that best covers the query terms
7
+ * (terms weighted by rarity across the returned chunks, so a chunk with
8
+ * "ladder" beats one with only "the" and "does"; lowest seq on ties), then
9
+ * its best 32-token window, with matching tokens wrapped in `<mark>`/`</mark>`
10
+ * and `...` where the window cuts the chunk, as `snippet()` did. Chunk choice
11
+ * uses substring presence of lightly stemmed terms; highlighting uses word
12
+ * prefixes. Both approximate the index tokenizer and affect display only,
13
+ * never ranking.
14
+ *
15
+ * @module src/pipeline/query-snippet
16
+ */
17
+
18
+ /** Tokens per snippet window (the old `snippet(..., 32)` budget). */
19
+ const WINDOW_TOKENS = 32;
20
+
21
+ /** Repeat hits only break ties between windows with equal term coverage. */
22
+ const HIT_TIEBREAK = 1e-6;
23
+
24
+ const TOKEN_REGEX = /[\p{L}\p{N}\p{M}]+/gu;
25
+
26
+ /** Quoted phrase (optionally negated) or a bare token, as the lexical grammar splits them. */
27
+ const QUERY_PART_REGEX = /(-?)"([^"]*)"|(\S+)/g;
28
+
29
+ const DIACRITIC_REGEX = /\p{M}/gu;
30
+
31
+ const NON_ASCII_REGEX = /[^\p{ASCII}]/u;
32
+
33
+ /** Final doubled consonant left by suffix removal (`embedd` -> `embed`). */
34
+ const DOUBLED_CONSONANT_REGEX = /([bdfgkmnprt])\1$/;
35
+
36
+ /** Suffixes removed from query terms, longest first. */
37
+ const STEM_SUFFIXES = [
38
+ "ational",
39
+ "ations",
40
+ "ation",
41
+ "ings",
42
+ "ions",
43
+ "ers",
44
+ "ied",
45
+ "ies",
46
+ "ing",
47
+ "ion",
48
+ "al",
49
+ "ed",
50
+ "er",
51
+ "ly",
52
+ "e",
53
+ "s",
54
+ "y",
55
+ ];
56
+
57
+ const MIN_STEM_LENGTH = 3;
58
+
59
+ export interface SnippetTerms {
60
+ /** Lowercase term prefixes (stemmed query words and diacritic-free forms). */
61
+ prefixes: string[];
62
+ /** Query word id per prefix; variants of one word share an id. */
63
+ ids: number[];
64
+ /** Anchored prefix matcher for one lowercased token. */
65
+ token: RegExp;
66
+ idByPrefix: Map<string, number>;
67
+ }
68
+
69
+ /** Query word ids present in one chunk. */
70
+ export interface ChunkTermSet {
71
+ seq: number;
72
+ terms: Set<number>;
73
+ }
74
+
75
+ interface Token {
76
+ start: number;
77
+ end: number;
78
+ term: number;
79
+ }
80
+
81
+ const stripDiacritics = (text: string): string =>
82
+ NON_ASCII_REGEX.test(text)
83
+ ? text.normalize("NFD").replace(DIACRITIC_REGEX, "")
84
+ : text;
85
+
86
+ function stemPrefix(term: string): string {
87
+ if (term.length <= MIN_STEM_LENGTH) return term;
88
+ for (const suffix of STEM_SUFFIXES) {
89
+ if (
90
+ term.endsWith(suffix) &&
91
+ term.length - suffix.length >= MIN_STEM_LENGTH
92
+ ) {
93
+ const stem = term.slice(0, -suffix.length);
94
+ return DOUBLED_CONSONANT_REGEX.test(stem) ? stem.slice(0, -1) : stem;
95
+ }
96
+ }
97
+ return term;
98
+ }
99
+
100
+ /** Positive terms of a lexical query; null when there are none. */
101
+ export function createSnippetTerms(query: string): SnippetTerms | null {
102
+ const idByPrefix = new Map<string, number>();
103
+ let nextId = 0;
104
+ for (const part of query.matchAll(QUERY_PART_REGEX)) {
105
+ const bare = part[3];
106
+ if (part[1] === "-" || bare?.startsWith("-")) continue;
107
+ for (const [word] of (part[2] ?? bare ?? "").matchAll(TOKEN_REGEX)) {
108
+ const lower = word.toLowerCase();
109
+ for (const form of new Set([lower, stripDiacritics(lower)])) {
110
+ const prefix = stemPrefix(form);
111
+ if (!idByPrefix.has(prefix)) idByPrefix.set(prefix, nextId);
112
+ }
113
+ nextId += 1;
114
+ }
115
+ }
116
+ if (idByPrefix.size === 0) return null;
117
+ const prefixes = [...idByPrefix.keys()].sort((a, b) => b.length - a.length);
118
+ return {
119
+ prefixes,
120
+ ids: prefixes.map((prefix) => idByPrefix.get(prefix) ?? -1),
121
+ token: new RegExp(`^(?:${prefixes.join("|")})`, "u"),
122
+ idByPrefix,
123
+ };
124
+ }
125
+
126
+ /** In-memory equivalent of the store's chunk term presence query. */
127
+ export function chunkTermPresence(
128
+ text: string,
129
+ terms: SnippetTerms
130
+ ): boolean[] {
131
+ const lower = text.toLowerCase();
132
+ return terms.prefixes.map((prefix) => lower.includes(prefix));
133
+ }
134
+
135
+ export function presentTermIds(
136
+ present: boolean[],
137
+ terms: SnippetTerms
138
+ ): Set<number> {
139
+ const ids = new Set<number>();
140
+ for (const [index, flag] of present.entries()) {
141
+ const id = terms.ids[index];
142
+ if (flag && id !== undefined) ids.add(id);
143
+ }
144
+ return ids;
145
+ }
146
+
147
+ /** Rarity weight per query word across all candidate chunks. */
148
+ export function snippetTermWeights(
149
+ chunks: Iterable<ChunkTermSet>
150
+ ): Map<number, number> {
151
+ let total = 0;
152
+ const frequency = new Map<number, number>();
153
+ for (const { terms } of chunks) {
154
+ total += 1;
155
+ for (const term of terms)
156
+ frequency.set(term, (frequency.get(term) ?? 0) + 1);
157
+ }
158
+ const weights = new Map<number, number>();
159
+ for (const [term, count] of frequency)
160
+ weights.set(term, Math.log(1 + total / count));
161
+ return weights;
162
+ }
163
+
164
+ /** Seq of the chunk with the best weighted coverage (lowest seq on ties). */
165
+ export function pickSnippetSeq(
166
+ chunks: ChunkTermSet[],
167
+ weights: Map<number, number>
168
+ ): number | undefined {
169
+ let best: ChunkTermSet | undefined;
170
+ let bestScore = -1;
171
+ for (const chunk of chunks) {
172
+ let score = 0;
173
+ for (const term of chunk.terms) score += weights.get(term) ?? 0;
174
+ if (
175
+ score > bestScore ||
176
+ (score === bestScore && best && chunk.seq < best.seq)
177
+ ) {
178
+ bestScore = score;
179
+ best = chunk;
180
+ }
181
+ }
182
+ return best?.seq;
183
+ }
184
+
185
+ function tokenize(text: string, terms: SnippetTerms): Token[] {
186
+ const tokens: Token[] = [];
187
+ for (const match of text.matchAll(TOKEN_REGEX)) {
188
+ const start = match.index;
189
+ const word = match[0];
190
+ const prefix = terms.token.exec(stripDiacritics(word.toLowerCase()));
191
+ tokens.push({
192
+ start,
193
+ end: start + word.length,
194
+ term: prefix ? (terms.idByPrefix.get(prefix[0]) ?? -1) : -1,
195
+ });
196
+ }
197
+ return tokens;
198
+ }
199
+
200
+ /** Start index of the window with the best weighted term coverage. */
201
+ function bestWindowStart(
202
+ tokens: Token[],
203
+ weights: Map<number, number>
204
+ ): number {
205
+ if (tokens.length <= WINDOW_TOKENS) return 0;
206
+ const counts = new Map<number, number>();
207
+ let covered = 0;
208
+ let hits = 0;
209
+ const add = (term: number, delta: number) => {
210
+ if (term < 0) return;
211
+ const count = (counts.get(term) ?? 0) + delta;
212
+ counts.set(term, count);
213
+ hits += delta;
214
+ const weight = weights.get(term) ?? 1;
215
+ if (delta > 0 && count === 1) covered += weight;
216
+ if (delta < 0 && count === 0) covered -= weight;
217
+ };
218
+ let bestStart = 0;
219
+ let bestScore = -1;
220
+ for (let index = 0; index < tokens.length; index += 1) {
221
+ add(tokens[index]?.term ?? -1, 1);
222
+ const start = index - WINDOW_TOKENS + 1;
223
+ if (start > 0) add(tokens[start - 1]?.term ?? -1, -1);
224
+ if (start < 0) continue;
225
+ const score = covered + hits * HIT_TIEBREAK;
226
+ if (score > bestScore) {
227
+ bestScore = score;
228
+ bestStart = start;
229
+ }
230
+ }
231
+ // Center the hits inside the chosen window.
232
+ let first = -1;
233
+ let last = -1;
234
+ for (let index = bestStart; index < bestStart + WINDOW_TOKENS; index += 1) {
235
+ if ((tokens[index]?.term ?? -1) < 0) continue;
236
+ if (first < 0) first = index;
237
+ last = index;
238
+ }
239
+ if (first < 0) return bestStart;
240
+ const centered = first - Math.floor((WINDOW_TOKENS - (last - first + 1)) / 2);
241
+ return Math.max(0, Math.min(centered, tokens.length - WINDOW_TOKENS));
242
+ }
243
+
244
+ /** The best 32-token window of one chunk, query terms in `<mark>` tags. */
245
+ export function renderChunkSnippet(
246
+ text: string,
247
+ terms: SnippetTerms,
248
+ weights: Map<number, number>
249
+ ): string {
250
+ const tokens = tokenize(text, terms);
251
+ if (tokens.length === 0) return text;
252
+ const start = bestWindowStart(tokens, weights);
253
+ const end = Math.min(tokens.length, start + WINDOW_TOKENS) - 1;
254
+ const atStart = start === 0;
255
+ const atEnd = end === tokens.length - 1;
256
+ let cursor = atStart ? 0 : (tokens[start]?.start ?? 0);
257
+ let out = atStart ? "" : "...";
258
+ for (let index = start; index <= end; index += 1) {
259
+ const token = tokens[index];
260
+ if (!token || token.term < 0) continue;
261
+ out += `${text.slice(cursor, token.start)}<mark>${text.slice(token.start, token.end)}</mark>`;
262
+ cursor = token.end;
263
+ }
264
+ out += text.slice(cursor, atEnd ? text.length : (tokens[end]?.end ?? 0));
265
+ return atEnd ? out : `${out}...`;
266
+ }
@@ -8,6 +8,7 @@
8
8
  import { join as pathJoin } from "node:path"; // No Bun path utils equivalent
9
9
 
10
10
  import type { ChunkRow, FtsResult, StorePort } from "../store/types";
11
+ import type { ChunkTermSet, SnippetTerms } from "./query-snippet";
11
12
  import type {
12
13
  SearchOptions,
13
14
  SearchResult,
@@ -31,6 +32,14 @@ import { typedMetadataWarnings } from "./filters";
31
32
  import { selectBestChunkForSteering } from "./intent";
32
33
  import { hasProjectAffinity } from "./project-affinity";
33
34
  import { detectQueryLanguage } from "./query-language";
35
+ import {
36
+ chunkTermPresence,
37
+ createSnippetTerms,
38
+ pickSnippetSeq,
39
+ presentTermIds,
40
+ renderChunkSnippet,
41
+ snippetTermWeights,
42
+ } from "./query-snippet";
34
43
  import {
35
44
  attachSearchResultContexts,
36
45
  attachSupersession,
@@ -44,6 +53,9 @@ import {
44
53
  import { attachSearchResultPlannerMetadata } from "./trace-metadata";
45
54
  import { SEARCH_RESULT_PLANNER_METADATA } from "./types";
46
55
 
56
+ const passageHash = (text: string): string =>
57
+ new Bun.CryptoHasher("sha256").update(text).digest("hex");
58
+
47
59
  // ─────────────────────────────────────────────────────────────────────────────
48
60
  // Score Normalization
49
61
  // ─────────────────────────────────────────────────────────────────────────────
@@ -124,11 +136,9 @@ function buildSearchResult(ctx: BuildResultContext): SearchResult {
124
136
  snippet = chunk.text;
125
137
  snippetRange = { startLine: chunk.startLine, endLine: chunk.endLine };
126
138
  } else {
127
- // Default: FTS snippet or chunk text, with leading frontmatter stripped
128
- const cleaned = cleanDisplaySnippet(
129
- fts.snippet ?? chunk?.text ?? "",
130
- chunk?.text
131
- );
139
+ // Default: chunk text until attachChunkSnippets replaces it for the final
140
+ // rows, with leading frontmatter stripped
141
+ const cleaned = cleanDisplaySnippet(chunk?.text ?? "", chunk?.text);
132
142
  snippet = cleaned.text;
133
143
  if (chunk) {
134
144
  line = chunk.startLine + cleaned.startLineOffset;
@@ -162,10 +172,130 @@ function buildSearchResult(ctx: BuildResultContext): SearchResult {
162
172
  graphExpanded: false,
163
173
  startLine: chunk.startLine,
164
174
  endLine: chunk.endLine,
165
- passageHash: new Bun.CryptoHasher("sha256")
166
- .update(chunk.text)
167
- .digest("hex"),
175
+ passageHash: passageHash(chunk.text),
176
+ });
177
+ }
178
+
179
+ /** Query-word ids present in each chunk of the returned documents. */
180
+ async function readChunkTermSets(
181
+ store: StorePort,
182
+ hashes: string[],
183
+ terms: SnippetTerms,
184
+ loadedChunks: Map<string, ChunkRow[]> | undefined
185
+ ): Promise<{
186
+ termSets: Map<string, ChunkTermSet[]>;
187
+ chunks?: Map<string, ChunkRow[]>;
188
+ } | null> {
189
+ const termSets = new Map<string, ChunkTermSet[]>();
190
+ const add = (mirrorHash: string, seq: number, present: boolean[]) => {
191
+ const list = termSets.get(mirrorHash) ?? [];
192
+ list.push({ seq, terms: presentTermIds(present, terms) });
193
+ termSets.set(mirrorHash, list);
194
+ };
195
+ // In-database presence keeps long documents from being loaded whole.
196
+ const presence =
197
+ !loadedChunks && store.getChunkTermPresence
198
+ ? await store.getChunkTermPresence(hashes, terms.prefixes)
199
+ : undefined;
200
+ if (presence?.ok) {
201
+ for (const row of presence.value) add(row.mirrorHash, row.seq, row.present);
202
+ return { termSets };
203
+ }
204
+ let chunks = loadedChunks;
205
+ if (!chunks) {
206
+ const fetched = await store.getChunksBatch(hashes);
207
+ if (!fetched.ok) return null;
208
+ chunks = fetched.value;
209
+ }
210
+ for (const [mirrorHash, list] of chunks)
211
+ for (const chunk of list)
212
+ add(mirrorHash, chunk.seq, chunkTermPresence(chunk.text, terms));
213
+ return { termSets, chunks };
214
+ }
215
+
216
+ /**
217
+ * Default-mode snippets for the returned rows only, built from their stored
218
+ * chunks (FTS5 snippet() re-tokenized every matched document body). Only the
219
+ * chosen chunk of each row is loaded; with an intent, the steered chunk is
220
+ * kept. The snippet, its range and the planner chunk all name that chunk.
221
+ * Updates results in place so hidden ranking metadata stays attached.
222
+ */
223
+ async function attachChunkSnippets(
224
+ store: StorePort,
225
+ results: SearchResult[],
226
+ query: string,
227
+ options: SearchOptions,
228
+ loadedChunks: Map<string, ChunkRow[]> | undefined
229
+ ): Promise<void> {
230
+ const terms = createSnippetTerms(query);
231
+ const hashes = [
232
+ ...new Set(
233
+ results
234
+ .map((result) => result.conversion?.mirrorHash)
235
+ .filter((hash): hash is string => Boolean(hash))
236
+ ),
237
+ ];
238
+ if (!terms || hashes.length === 0) return;
239
+ const read = await readChunkTermSets(store, hashes, terms, loadedChunks);
240
+ if (!read) return;
241
+ const weights = snippetTermWeights([...read.termSets.values()].flat());
242
+
243
+ const chosen = results.map((result) => {
244
+ const mirrorHash = result.conversion?.mirrorHash;
245
+ if (!mirrorHash) return undefined;
246
+ const steeredSeq = result[SEARCH_RESULT_PLANNER_METADATA]?.seq;
247
+ const candidates = read.termSets.get(mirrorHash) ?? [];
248
+ const seq = pickSnippetSeq(
249
+ options.intent
250
+ ? candidates.filter((entry) => entry.seq === steeredSeq)
251
+ : candidates,
252
+ weights
253
+ );
254
+ return seq === undefined ? undefined : { mirrorHash, seq };
168
255
  });
256
+ let chunks = read.chunks;
257
+ if (!chunks) {
258
+ const fetched = store.getChunksBySequenceBatch
259
+ ? await store.getChunksBySequenceBatch(
260
+ chosen.filter((key) => key !== undefined)
261
+ )
262
+ : await store.getChunksBatch(hashes);
263
+ if (!fetched.ok) return;
264
+ chunks = fetched.value;
265
+ }
266
+ const getChunk = createChunkLookup(chunks);
267
+
268
+ for (const [index, result] of results.entries()) {
269
+ const key = chosen[index];
270
+ const chunk = key ? getChunk(key.mirrorHash, key.seq) : undefined;
271
+ if (!(key && chunk)) continue;
272
+ const cleaned = cleanDisplaySnippet(
273
+ renderChunkSnippet(chunk.text, terms, weights),
274
+ chunk.text
275
+ );
276
+ result.line = chunk.startLine + cleaned.startLineOffset;
277
+ result.snippet = cleaned.text;
278
+ result.snippetLanguage = chunk.language ?? undefined;
279
+ result.snippetRange = { startLine: result.line, endLine: chunk.endLine };
280
+ const coordinates = {
281
+ mirrorHash: key.mirrorHash,
282
+ seq: chunk.seq,
283
+ startLine: chunk.startLine,
284
+ endLine: chunk.endLine,
285
+ passageHash: passageHash(chunk.text),
286
+ };
287
+ const metadata = result[SEARCH_RESULT_PLANNER_METADATA];
288
+ if (metadata) {
289
+ Object.assign(metadata, coordinates);
290
+ } else {
291
+ attachSearchResultPlannerMetadata(result, {
292
+ retrievalRank: 0,
293
+ sources: ["bm25"],
294
+ graphExpanded: false,
295
+ ...coordinates,
296
+ });
297
+ }
298
+ }
169
299
  }
170
300
 
171
301
  // ─────────────────────────────────────────────────────────────────────────────
@@ -207,8 +337,7 @@ export async function searchBm25(
207
337
  const detection = detectQueryLanguage(query);
208
338
  const queryLanguage = options.lang ?? detection.bcp47;
209
339
 
210
- // Run FTS search
211
- // Disable FTS snippet when --full or --line-numbers (we use raw text instead)
340
+ // Run FTS search (ranking only; snippets come from chunks of the final rows)
212
341
  const ftsResult = await store.searchFts(query, {
213
342
  limit: retrievalLimit,
214
343
  collection: options.collection,
@@ -216,7 +345,6 @@ export async function searchBm25(
216
345
  exclude: options.exclude,
217
346
  relPathPrefix: options.retrievalScope?.relPathPrefix,
218
347
  language: options.lang,
219
- snippet: !(options.full || options.lineNumbers),
220
348
  tagsAll: options.tagsAll,
221
349
  tagsAny: options.tagsAny,
222
350
  since: temporalRange.since,
@@ -265,10 +393,9 @@ export async function searchBm25(
265
393
  ftsResult.value.map((f) => f.mirrorHash).filter((h): h is string => !!h)
266
394
  ),
267
395
  ];
396
+ const needsAllChunks = Boolean(options.intent || options.exclude?.length);
268
397
  const chunksMapResult =
269
- store.getChunksBySequenceBatch &&
270
- !options.intent &&
271
- !options.exclude?.length
398
+ store.getChunksBySequenceBatch && !needsAllChunks
272
399
  ? await store.getChunksBySequenceBatch(
273
400
  ftsResult.value
274
401
  .filter((row) => !!row.mirrorHash)
@@ -278,6 +405,10 @@ export async function searchBm25(
278
405
  const getChunk = chunksMapResult.ok
279
406
  ? createChunkLookup(chunksMapResult.value)
280
407
  : () => undefined;
408
+ const allChunks =
409
+ chunksMapResult.ok && (needsAllChunks || !store.getChunksBySequenceBatch)
410
+ ? chunksMapResult.value
411
+ : undefined;
281
412
 
282
413
  // Dedup: multiple docs can share mirror_hash (content-addressed storage)
283
414
  // Track seen uri+seq to eliminate duplicate rows from join fan-out
@@ -326,11 +457,7 @@ export async function searchBm25(
326
457
 
327
458
  const excluded =
328
459
  matchesExcludedText(
329
- [
330
- fts.title ?? "",
331
- fts.recordSourcePath ?? fts.relPath ?? "",
332
- fts.snippet ?? "",
333
- ],
460
+ [fts.title ?? "", fts.recordSourcePath ?? fts.relPath ?? ""],
334
461
  options.exclude
335
462
  ) ||
336
463
  matchesExcludedChunks(
@@ -464,6 +591,9 @@ export async function searchBm25(
464
591
  }
465
592
 
466
593
  const finalResults = filteredResults.slice(0, limit);
594
+ if (!(options.full || options.lineNumbers)) {
595
+ await attachChunkSnippets(store, finalResults, query, options, allChunks);
596
+ }
467
597
  for (const [index, result] of finalResults.entries()) {
468
598
  const metadata = result[SEARCH_RESULT_PLANNER_METADATA];
469
599
  if (metadata) metadata.retrievalRank = index + 1;
package/src/sdk/client.ts CHANGED
@@ -189,6 +189,7 @@ import {
189
189
  withContentTypeRules,
190
190
  } from "../ingestion";
191
191
  import { updateFrontmatterTags } from "../ingestion/frontmatter";
192
+ import { withCollectionRootErrors } from "../ingestion/source-availability/root";
192
193
  import {
193
194
  withInferenceScope,
194
195
  finishInferenceCleanup,
@@ -210,7 +211,10 @@ import { runAutomationProfile } from "../sessions/automation";
210
211
  import { assertSessionBinding } from "../sessions/binding";
211
212
  import { SessionsService } from "../sessions/service";
212
213
  import { SESSIONS_VALIDATION_CODES, SessionsError } from "../sessions/types";
213
- import { SqliteAdapter } from "../store/sqlite/adapter";
214
+ import {
215
+ SqliteAdapter,
216
+ staleCollectionsMessage,
217
+ } from "../store/sqlite/adapter";
214
218
  import { openScopedIndexStore } from "../store/sqlite/scoped-index";
215
219
  import { createVectorIndexPort } from "../store/vector";
216
220
  import {
@@ -462,6 +466,14 @@ class GnoClientImpl implements GnoClient {
462
466
  if (!this.isOpen()) {
463
467
  throw sdkError("RUNTIME", "GNO client is closed");
464
468
  }
469
+ // Another client may have indexed a collection this config omits since
470
+ // this one opened; never read it.
471
+ const stale = this.store.staleIndexedCollections(
472
+ this.config.collections.map(({ name }) => name)
473
+ );
474
+ if (stale.length > 0) {
475
+ throw sdkError("RUNTIME", staleCollectionsMessage(stale));
476
+ }
465
477
  }
466
478
 
467
479
  private getCollections(collection?: string) {
@@ -1541,7 +1553,7 @@ class GnoClientImpl implements GnoClient {
1541
1553
  })
1542
1554
  );
1543
1555
  return {
1544
- ...status,
1556
+ ...(await withCollectionRootErrors(status, this.config.collections)),
1545
1557
  contentTypeBoost: buildContentTypeBoostStatus(
1546
1558
  this.config.contentTypes ?? []
1547
1559
  ),
package/src/sdk/types.ts CHANGED
@@ -90,6 +90,7 @@ import type {
90
90
  SectionTargetV1,
91
91
  } from "../core/sections";
92
92
  import type { SyncResult } from "../ingestion";
93
+ import type { CollectionRootError } from "../ingestion/source-availability/root";
93
94
  import type { DownloadPolicy } from "../llm/policy";
94
95
  import type {
95
96
  AskOptions,
@@ -193,7 +194,11 @@ export type GnoVectorSearchOptions = Omit<
193
194
  export type GnoContextInput = Omit<ContextCapsuleBuildInput, "indexName"> &
194
195
  GnoProjectHintOptions;
195
196
  export type GnoContextResult = ContextCapsuleV1;
196
- export type GnoIndexStatus = IndexStatus & {
197
+ export type GnoIndexStatus = Omit<IndexStatus, "collections"> & {
198
+ /** `rootError` is set while a collection's folder is missing or unreadable. */
199
+ collections: Array<
200
+ IndexStatus["collections"][number] & { rootError?: CollectionRootError }
201
+ >;
197
202
  contentTypeBoost: ContentTypeBoostStatus;
198
203
  };
199
204
  export type GnoContextVerificationResult = ContextCapsuleVerification;
@@ -29,7 +29,7 @@ interface Option {
29
29
  }
30
30
 
31
31
  const options: Option[] = [
32
- { id: "fast", label: "Fast", timing: "BM25", icon: Rabbit },
32
+ { id: "fast", label: "Fast", timing: "<1s", icon: Rabbit },
33
33
  { id: "balanced", label: "Balanced", timing: "~2s", icon: Gauge },
34
34
  { id: "thorough", label: "Thorough", timing: "~5s", icon: Sparkles },
35
35
  ];