@gmickel/gno 2.11.0 → 2.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/assets/skill/SKILL.md +3 -2
- package/assets/spa-production.json.gz +0 -0
- package/browser-extension/artifacts/{gno-browser-clipper-v2.11.0.zip → gno-browser-clipper-v2.12.0.zip} +0 -0
- package/browser-extension/artifacts/gno-browser-clipper-v2.12.0.zip.sha256 +1 -0
- package/browser-extension/dist/manifest.json +1 -1
- package/package.json +8 -5
- package/spec/cli.md +100 -34
- package/spec/evals-agentic.md +3 -1
- package/spec/mcp.md +9 -3
- package/spec/output-schemas/status.schema.json +28 -0
- package/src/app/context-agent-projection.ts +10 -1
- package/src/cli/commands/agents/block.ts +2 -2
- package/src/cli/commands/agents/commands.ts +6 -5
- package/src/cli/commands/agents/engine.ts +15 -2
- package/src/cli/commands/agents/harnesses.ts +21 -14
- package/src/cli/commands/cleanup.ts +46 -6
- package/src/cli/commands/collection/remove.ts +1 -0
- package/src/cli/commands/collection/rename.ts +1 -0
- package/src/cli/commands/doctor.ts +37 -0
- package/src/cli/commands/mcp/paths.ts +2 -1
- package/src/cli/commands/setup-semantic.ts +6 -1
- package/src/cli/commands/shared.ts +26 -1
- package/src/cli/commands/skill/paths.ts +20 -2
- package/src/cli/commands/status.ts +89 -15
- package/src/cli/program.ts +19 -3
- package/src/core/activation-probe-plan.ts +0 -1
- package/src/core/config-mutation.ts +72 -55
- package/src/core/memory-remember.ts +0 -1
- package/src/core/module-path.ts +33 -0
- package/src/core/runtime-entrypoint.ts +3 -1
- package/src/ingestion/file-processor.ts +6 -3
- package/src/ingestion/prepare-file.ts +14 -4
- package/src/ingestion/source-availability/root.ts +138 -0
- package/src/ingestion/sync.ts +5 -19
- package/src/llm/native-worker/client.ts +8 -4
- package/src/llm/native-worker/errors.ts +26 -3
- package/src/mcp/http-transport.ts +7 -2
- package/src/mcp/server.ts +3 -0
- package/src/mcp/tool-descriptions-core.ts +2 -2
- package/src/mcp/tools/index.ts +30 -1
- package/src/mcp/tools/status.ts +17 -2
- package/src/pipeline/query-snippet.ts +266 -0
- package/src/pipeline/search.ts +149 -19
- package/src/sdk/client.ts +14 -2
- package/src/sdk/types.ts +6 -1
- package/src/serve/public/components/ThoroughnessSelector.tsx +1 -1
- package/src/serve/public/globals.built.css +1 -1
- package/src/serve/public/pages/Collections.tsx +8 -0
- package/src/serve/public/pages/Search.tsx +9 -1
- package/src/serve/resident-request.ts +8 -0
- package/src/serve/resident-runtime.ts +25 -3
- package/src/serve/spa-production-build.ts +7 -3
- package/src/serve/status-model.ts +2 -0
- package/src/serve/status.ts +29 -0
- package/src/sessions/config-refresh.ts +6 -0
- package/src/sessions/import-child.ts +4 -1
- package/src/store/sqlite/adapter.ts +159 -18
- package/src/store/types.ts +45 -7
- package/browser-extension/artifacts/gno-browser-clipper-v2.11.0.zip.sha256 +0 -1
package/src/mcp/tools/index.ts
CHANGED
|
@@ -37,6 +37,9 @@ import {
|
|
|
37
37
|
assertInferenceActive,
|
|
38
38
|
acquireInferencePermit,
|
|
39
39
|
} from "../../llm/inference-scope";
|
|
40
|
+
import { envIsSet } from "../../llm/policy";
|
|
41
|
+
import { staleCollectionsMessage } from "../../store/sqlite/adapter";
|
|
42
|
+
import { MCP_ACTIVATION_VERIFICATION_ENV } from "../activation-verification-mode";
|
|
40
43
|
import { exposesHostPaths } from "../context";
|
|
41
44
|
import { profileToolDescription } from "../tool-descriptions-core";
|
|
42
45
|
import {
|
|
@@ -175,7 +178,7 @@ export function normalizeTagFilters(tags?: string[]): string[] | undefined {
|
|
|
175
178
|
|
|
176
179
|
export const MCP_TOOL_DESCRIPTIONS = {
|
|
177
180
|
search:
|
|
178
|
-
"BM25 keyword search. Fast exact
|
|
181
|
+
"BM25 keyword search. Fast lookup of exact text you already have verbatim: identifiers, filenames, error text, and quoted phrases. Structured results include uri/docid, line when available, and optional user-configured context guidance; use gno_get with fromLine/lineCount or gno_multi_get for full context. A question, or a name or topic taken from it, is not exact text: use gno_query (fast=true first).",
|
|
179
182
|
vsearch:
|
|
180
183
|
"Vector semantic search. Finds conceptually similar docs with different wording. Structured results preserve optional user-configured context guidance. Best after embeddings are current; use intent to disambiguate short terms. For questions, gno_query with fast=true adds keyword matching at about the same cost.",
|
|
181
184
|
query:
|
|
@@ -1014,6 +1017,7 @@ export async function runTool<T>(
|
|
|
1014
1017
|
// Sequential execution via mutex
|
|
1015
1018
|
const release = await acquireInferencePermit(() => ctx.toolMutex.acquire());
|
|
1016
1019
|
try {
|
|
1020
|
+
assertConfiguredCollections(ctx, name);
|
|
1017
1021
|
assertInferenceActive();
|
|
1018
1022
|
const raw = await (ctx.runWithSnapshot?.(fn) ?? fn());
|
|
1019
1023
|
assertInferenceActive();
|
|
@@ -1084,6 +1088,31 @@ export async function runToolNoMutex<T>(
|
|
|
1084
1088
|
}
|
|
1085
1089
|
}
|
|
1086
1090
|
|
|
1091
|
+
/** Orientation tools that report the index state instead of reading it. */
|
|
1092
|
+
const STALE_CHECK_EXEMPT_TOOLS = new Set(["gno_status", "gno_peek"]);
|
|
1093
|
+
|
|
1094
|
+
/**
|
|
1095
|
+
* Refuse a tool call while the index holds collections this config omits
|
|
1096
|
+
* (another client indexed them after this server opened). Status and peek
|
|
1097
|
+
* still run, and a connector proof reads one collection through a subset
|
|
1098
|
+
* config.
|
|
1099
|
+
*/
|
|
1100
|
+
function assertConfiguredCollections(ctx: ToolContext, name: string): void {
|
|
1101
|
+
// Store-free stand-ins (an uninitialized peek, test doubles) have no index.
|
|
1102
|
+
const store = ctx.store as Partial<ToolContext["store"]> | undefined;
|
|
1103
|
+
if (
|
|
1104
|
+
STALE_CHECK_EXEMPT_TOOLS.has(name) ||
|
|
1105
|
+
typeof store?.staleIndexedCollections !== "function" ||
|
|
1106
|
+
envIsSet(process.env, MCP_ACTIVATION_VERIFICATION_ENV)
|
|
1107
|
+
) {
|
|
1108
|
+
return;
|
|
1109
|
+
}
|
|
1110
|
+
const stale = store.staleIndexedCollections(
|
|
1111
|
+
ctx.collections.map((collection) => collection.name)
|
|
1112
|
+
);
|
|
1113
|
+
if (stale.length > 0) throw new Error(staleCollectionsMessage(stale));
|
|
1114
|
+
}
|
|
1115
|
+
|
|
1087
1116
|
function parseErrorMessage(message: string): { [x: string]: unknown } {
|
|
1088
1117
|
const match = message.match(/^([A-Z_]+):\s*(.*)$/);
|
|
1089
1118
|
if (match) {
|
package/src/mcp/tools/status.ts
CHANGED
|
@@ -12,6 +12,10 @@ import { formatChunkingStatus } from "../../core/chunking-status";
|
|
|
12
12
|
import { OWNER_CONFIG_PATH_FIELDS, withoutFields } from "../../core/host-paths";
|
|
13
13
|
import { formatLinkWorkspace } from "../../core/link-workspace";
|
|
14
14
|
import { formatVectorPartitionLines } from "../../core/vector-partition-status";
|
|
15
|
+
import {
|
|
16
|
+
type CollectionRootError,
|
|
17
|
+
withCollectionRootErrors,
|
|
18
|
+
} from "../../ingestion/source-availability/root";
|
|
15
19
|
import { resolveModelUri } from "../../llm/registry";
|
|
16
20
|
import { createStandaloneResidentStatus } from "../../serve/resident-status";
|
|
17
21
|
import { exposesHostPaths } from "../context";
|
|
@@ -23,7 +27,10 @@ type StatusInput = Record<string, never>;
|
|
|
23
27
|
type StatusView = Omit<IndexStatus, "configPath" | "dbPath" | "collections"> &
|
|
24
28
|
Partial<Pick<IndexStatus, "configPath" | "dbPath">> & {
|
|
25
29
|
collections: Array<
|
|
26
|
-
Omit<CollectionStatus, "path"> &
|
|
30
|
+
Omit<CollectionStatus, "path"> &
|
|
31
|
+
Partial<Pick<CollectionStatus, "path">> & {
|
|
32
|
+
rootError?: CollectionRootError;
|
|
33
|
+
}
|
|
27
34
|
>;
|
|
28
35
|
};
|
|
29
36
|
|
|
@@ -58,6 +65,11 @@ function formatStatus(status: StatusView): string {
|
|
|
58
65
|
);
|
|
59
66
|
const workspace = formatLinkWorkspace(c);
|
|
60
67
|
if (workspace) lines.push(` Link workspace: ${workspace}`);
|
|
68
|
+
if (c.rootError) {
|
|
69
|
+
lines.push(
|
|
70
|
+
` Root unavailable: [${c.rootError.code}] ${c.rootError.message}`
|
|
71
|
+
);
|
|
72
|
+
}
|
|
61
73
|
}
|
|
62
74
|
}
|
|
63
75
|
|
|
@@ -113,7 +125,10 @@ export function handleStatus(
|
|
|
113
125
|
|
|
114
126
|
// Override configPath with actual path from context
|
|
115
127
|
const status = {
|
|
116
|
-
...
|
|
128
|
+
...(await withCollectionRootErrors(
|
|
129
|
+
result.value,
|
|
130
|
+
ctx.config.collections
|
|
131
|
+
)),
|
|
117
132
|
configPath: ctx.actualConfigPath,
|
|
118
133
|
contentTypeBoost: buildContentTypeBoostStatus(
|
|
119
134
|
ctx.config.contentTypes ?? []
|
|
@@ -0,0 +1,266 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* BM25 result snippets built from stored chunks.
|
|
3
|
+
*
|
|
4
|
+
* FTS5 `snippet()` re-tokenizes the whole body of every matched document
|
|
5
|
+
* before LIMIT, which made long questions take seconds. Snippets are built
|
|
6
|
+
* for the returned rows only: pick the chunk that best covers the query terms
|
|
7
|
+
* (terms weighted by rarity across the returned chunks, so a chunk with
|
|
8
|
+
* "ladder" beats one with only "the" and "does"; lowest seq on ties), then
|
|
9
|
+
* its best 32-token window, with matching tokens wrapped in `<mark>`/`</mark>`
|
|
10
|
+
* and `...` where the window cuts the chunk, as `snippet()` did. Chunk choice
|
|
11
|
+
* uses substring presence of lightly stemmed terms; highlighting uses word
|
|
12
|
+
* prefixes. Both approximate the index tokenizer and affect display only,
|
|
13
|
+
* never ranking.
|
|
14
|
+
*
|
|
15
|
+
* @module src/pipeline/query-snippet
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
/** Tokens per snippet window (the old `snippet(..., 32)` budget). */
|
|
19
|
+
const WINDOW_TOKENS = 32;
|
|
20
|
+
|
|
21
|
+
/** Repeat hits only break ties between windows with equal term coverage. */
|
|
22
|
+
const HIT_TIEBREAK = 1e-6;
|
|
23
|
+
|
|
24
|
+
const TOKEN_REGEX = /[\p{L}\p{N}\p{M}]+/gu;
|
|
25
|
+
|
|
26
|
+
/** Quoted phrase (optionally negated) or a bare token, as the lexical grammar splits them. */
|
|
27
|
+
const QUERY_PART_REGEX = /(-?)"([^"]*)"|(\S+)/g;
|
|
28
|
+
|
|
29
|
+
const DIACRITIC_REGEX = /\p{M}/gu;
|
|
30
|
+
|
|
31
|
+
const NON_ASCII_REGEX = /[^\p{ASCII}]/u;
|
|
32
|
+
|
|
33
|
+
/** Final doubled consonant left by suffix removal (`embedd` -> `embed`). */
|
|
34
|
+
const DOUBLED_CONSONANT_REGEX = /([bdfgkmnprt])\1$/;
|
|
35
|
+
|
|
36
|
+
/** Suffixes removed from query terms, longest first. */
|
|
37
|
+
const STEM_SUFFIXES = [
|
|
38
|
+
"ational",
|
|
39
|
+
"ations",
|
|
40
|
+
"ation",
|
|
41
|
+
"ings",
|
|
42
|
+
"ions",
|
|
43
|
+
"ers",
|
|
44
|
+
"ied",
|
|
45
|
+
"ies",
|
|
46
|
+
"ing",
|
|
47
|
+
"ion",
|
|
48
|
+
"al",
|
|
49
|
+
"ed",
|
|
50
|
+
"er",
|
|
51
|
+
"ly",
|
|
52
|
+
"e",
|
|
53
|
+
"s",
|
|
54
|
+
"y",
|
|
55
|
+
];
|
|
56
|
+
|
|
57
|
+
const MIN_STEM_LENGTH = 3;
|
|
58
|
+
|
|
59
|
+
export interface SnippetTerms {
|
|
60
|
+
/** Lowercase term prefixes (stemmed query words and diacritic-free forms). */
|
|
61
|
+
prefixes: string[];
|
|
62
|
+
/** Query word id per prefix; variants of one word share an id. */
|
|
63
|
+
ids: number[];
|
|
64
|
+
/** Anchored prefix matcher for one lowercased token. */
|
|
65
|
+
token: RegExp;
|
|
66
|
+
idByPrefix: Map<string, number>;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/** Query word ids present in one chunk. */
|
|
70
|
+
export interface ChunkTermSet {
|
|
71
|
+
seq: number;
|
|
72
|
+
terms: Set<number>;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
interface Token {
|
|
76
|
+
start: number;
|
|
77
|
+
end: number;
|
|
78
|
+
term: number;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
const stripDiacritics = (text: string): string =>
|
|
82
|
+
NON_ASCII_REGEX.test(text)
|
|
83
|
+
? text.normalize("NFD").replace(DIACRITIC_REGEX, "")
|
|
84
|
+
: text;
|
|
85
|
+
|
|
86
|
+
function stemPrefix(term: string): string {
|
|
87
|
+
if (term.length <= MIN_STEM_LENGTH) return term;
|
|
88
|
+
for (const suffix of STEM_SUFFIXES) {
|
|
89
|
+
if (
|
|
90
|
+
term.endsWith(suffix) &&
|
|
91
|
+
term.length - suffix.length >= MIN_STEM_LENGTH
|
|
92
|
+
) {
|
|
93
|
+
const stem = term.slice(0, -suffix.length);
|
|
94
|
+
return DOUBLED_CONSONANT_REGEX.test(stem) ? stem.slice(0, -1) : stem;
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
return term;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/** Positive terms of a lexical query; null when there are none. */
|
|
101
|
+
export function createSnippetTerms(query: string): SnippetTerms | null {
|
|
102
|
+
const idByPrefix = new Map<string, number>();
|
|
103
|
+
let nextId = 0;
|
|
104
|
+
for (const part of query.matchAll(QUERY_PART_REGEX)) {
|
|
105
|
+
const bare = part[3];
|
|
106
|
+
if (part[1] === "-" || bare?.startsWith("-")) continue;
|
|
107
|
+
for (const [word] of (part[2] ?? bare ?? "").matchAll(TOKEN_REGEX)) {
|
|
108
|
+
const lower = word.toLowerCase();
|
|
109
|
+
for (const form of new Set([lower, stripDiacritics(lower)])) {
|
|
110
|
+
const prefix = stemPrefix(form);
|
|
111
|
+
if (!idByPrefix.has(prefix)) idByPrefix.set(prefix, nextId);
|
|
112
|
+
}
|
|
113
|
+
nextId += 1;
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
if (idByPrefix.size === 0) return null;
|
|
117
|
+
const prefixes = [...idByPrefix.keys()].sort((a, b) => b.length - a.length);
|
|
118
|
+
return {
|
|
119
|
+
prefixes,
|
|
120
|
+
ids: prefixes.map((prefix) => idByPrefix.get(prefix) ?? -1),
|
|
121
|
+
token: new RegExp(`^(?:${prefixes.join("|")})`, "u"),
|
|
122
|
+
idByPrefix,
|
|
123
|
+
};
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
/** In-memory equivalent of the store's chunk term presence query. */
|
|
127
|
+
export function chunkTermPresence(
|
|
128
|
+
text: string,
|
|
129
|
+
terms: SnippetTerms
|
|
130
|
+
): boolean[] {
|
|
131
|
+
const lower = text.toLowerCase();
|
|
132
|
+
return terms.prefixes.map((prefix) => lower.includes(prefix));
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
export function presentTermIds(
|
|
136
|
+
present: boolean[],
|
|
137
|
+
terms: SnippetTerms
|
|
138
|
+
): Set<number> {
|
|
139
|
+
const ids = new Set<number>();
|
|
140
|
+
for (const [index, flag] of present.entries()) {
|
|
141
|
+
const id = terms.ids[index];
|
|
142
|
+
if (flag && id !== undefined) ids.add(id);
|
|
143
|
+
}
|
|
144
|
+
return ids;
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
/** Rarity weight per query word across all candidate chunks. */
|
|
148
|
+
export function snippetTermWeights(
|
|
149
|
+
chunks: Iterable<ChunkTermSet>
|
|
150
|
+
): Map<number, number> {
|
|
151
|
+
let total = 0;
|
|
152
|
+
const frequency = new Map<number, number>();
|
|
153
|
+
for (const { terms } of chunks) {
|
|
154
|
+
total += 1;
|
|
155
|
+
for (const term of terms)
|
|
156
|
+
frequency.set(term, (frequency.get(term) ?? 0) + 1);
|
|
157
|
+
}
|
|
158
|
+
const weights = new Map<number, number>();
|
|
159
|
+
for (const [term, count] of frequency)
|
|
160
|
+
weights.set(term, Math.log(1 + total / count));
|
|
161
|
+
return weights;
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
/** Seq of the chunk with the best weighted coverage (lowest seq on ties). */
|
|
165
|
+
export function pickSnippetSeq(
|
|
166
|
+
chunks: ChunkTermSet[],
|
|
167
|
+
weights: Map<number, number>
|
|
168
|
+
): number | undefined {
|
|
169
|
+
let best: ChunkTermSet | undefined;
|
|
170
|
+
let bestScore = -1;
|
|
171
|
+
for (const chunk of chunks) {
|
|
172
|
+
let score = 0;
|
|
173
|
+
for (const term of chunk.terms) score += weights.get(term) ?? 0;
|
|
174
|
+
if (
|
|
175
|
+
score > bestScore ||
|
|
176
|
+
(score === bestScore && best && chunk.seq < best.seq)
|
|
177
|
+
) {
|
|
178
|
+
bestScore = score;
|
|
179
|
+
best = chunk;
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
return best?.seq;
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
function tokenize(text: string, terms: SnippetTerms): Token[] {
|
|
186
|
+
const tokens: Token[] = [];
|
|
187
|
+
for (const match of text.matchAll(TOKEN_REGEX)) {
|
|
188
|
+
const start = match.index;
|
|
189
|
+
const word = match[0];
|
|
190
|
+
const prefix = terms.token.exec(stripDiacritics(word.toLowerCase()));
|
|
191
|
+
tokens.push({
|
|
192
|
+
start,
|
|
193
|
+
end: start + word.length,
|
|
194
|
+
term: prefix ? (terms.idByPrefix.get(prefix[0]) ?? -1) : -1,
|
|
195
|
+
});
|
|
196
|
+
}
|
|
197
|
+
return tokens;
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
/** Start index of the window with the best weighted term coverage. */
|
|
201
|
+
function bestWindowStart(
|
|
202
|
+
tokens: Token[],
|
|
203
|
+
weights: Map<number, number>
|
|
204
|
+
): number {
|
|
205
|
+
if (tokens.length <= WINDOW_TOKENS) return 0;
|
|
206
|
+
const counts = new Map<number, number>();
|
|
207
|
+
let covered = 0;
|
|
208
|
+
let hits = 0;
|
|
209
|
+
const add = (term: number, delta: number) => {
|
|
210
|
+
if (term < 0) return;
|
|
211
|
+
const count = (counts.get(term) ?? 0) + delta;
|
|
212
|
+
counts.set(term, count);
|
|
213
|
+
hits += delta;
|
|
214
|
+
const weight = weights.get(term) ?? 1;
|
|
215
|
+
if (delta > 0 && count === 1) covered += weight;
|
|
216
|
+
if (delta < 0 && count === 0) covered -= weight;
|
|
217
|
+
};
|
|
218
|
+
let bestStart = 0;
|
|
219
|
+
let bestScore = -1;
|
|
220
|
+
for (let index = 0; index < tokens.length; index += 1) {
|
|
221
|
+
add(tokens[index]?.term ?? -1, 1);
|
|
222
|
+
const start = index - WINDOW_TOKENS + 1;
|
|
223
|
+
if (start > 0) add(tokens[start - 1]?.term ?? -1, -1);
|
|
224
|
+
if (start < 0) continue;
|
|
225
|
+
const score = covered + hits * HIT_TIEBREAK;
|
|
226
|
+
if (score > bestScore) {
|
|
227
|
+
bestScore = score;
|
|
228
|
+
bestStart = start;
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
// Center the hits inside the chosen window.
|
|
232
|
+
let first = -1;
|
|
233
|
+
let last = -1;
|
|
234
|
+
for (let index = bestStart; index < bestStart + WINDOW_TOKENS; index += 1) {
|
|
235
|
+
if ((tokens[index]?.term ?? -1) < 0) continue;
|
|
236
|
+
if (first < 0) first = index;
|
|
237
|
+
last = index;
|
|
238
|
+
}
|
|
239
|
+
if (first < 0) return bestStart;
|
|
240
|
+
const centered = first - Math.floor((WINDOW_TOKENS - (last - first + 1)) / 2);
|
|
241
|
+
return Math.max(0, Math.min(centered, tokens.length - WINDOW_TOKENS));
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
/** The best 32-token window of one chunk, query terms in `<mark>` tags. */
|
|
245
|
+
export function renderChunkSnippet(
|
|
246
|
+
text: string,
|
|
247
|
+
terms: SnippetTerms,
|
|
248
|
+
weights: Map<number, number>
|
|
249
|
+
): string {
|
|
250
|
+
const tokens = tokenize(text, terms);
|
|
251
|
+
if (tokens.length === 0) return text;
|
|
252
|
+
const start = bestWindowStart(tokens, weights);
|
|
253
|
+
const end = Math.min(tokens.length, start + WINDOW_TOKENS) - 1;
|
|
254
|
+
const atStart = start === 0;
|
|
255
|
+
const atEnd = end === tokens.length - 1;
|
|
256
|
+
let cursor = atStart ? 0 : (tokens[start]?.start ?? 0);
|
|
257
|
+
let out = atStart ? "" : "...";
|
|
258
|
+
for (let index = start; index <= end; index += 1) {
|
|
259
|
+
const token = tokens[index];
|
|
260
|
+
if (!token || token.term < 0) continue;
|
|
261
|
+
out += `${text.slice(cursor, token.start)}<mark>${text.slice(token.start, token.end)}</mark>`;
|
|
262
|
+
cursor = token.end;
|
|
263
|
+
}
|
|
264
|
+
out += text.slice(cursor, atEnd ? text.length : (tokens[end]?.end ?? 0));
|
|
265
|
+
return atEnd ? out : `${out}...`;
|
|
266
|
+
}
|
package/src/pipeline/search.ts
CHANGED
|
@@ -8,6 +8,7 @@
|
|
|
8
8
|
import { join as pathJoin } from "node:path"; // No Bun path utils equivalent
|
|
9
9
|
|
|
10
10
|
import type { ChunkRow, FtsResult, StorePort } from "../store/types";
|
|
11
|
+
import type { ChunkTermSet, SnippetTerms } from "./query-snippet";
|
|
11
12
|
import type {
|
|
12
13
|
SearchOptions,
|
|
13
14
|
SearchResult,
|
|
@@ -31,6 +32,14 @@ import { typedMetadataWarnings } from "./filters";
|
|
|
31
32
|
import { selectBestChunkForSteering } from "./intent";
|
|
32
33
|
import { hasProjectAffinity } from "./project-affinity";
|
|
33
34
|
import { detectQueryLanguage } from "./query-language";
|
|
35
|
+
import {
|
|
36
|
+
chunkTermPresence,
|
|
37
|
+
createSnippetTerms,
|
|
38
|
+
pickSnippetSeq,
|
|
39
|
+
presentTermIds,
|
|
40
|
+
renderChunkSnippet,
|
|
41
|
+
snippetTermWeights,
|
|
42
|
+
} from "./query-snippet";
|
|
34
43
|
import {
|
|
35
44
|
attachSearchResultContexts,
|
|
36
45
|
attachSupersession,
|
|
@@ -44,6 +53,9 @@ import {
|
|
|
44
53
|
import { attachSearchResultPlannerMetadata } from "./trace-metadata";
|
|
45
54
|
import { SEARCH_RESULT_PLANNER_METADATA } from "./types";
|
|
46
55
|
|
|
56
|
+
const passageHash = (text: string): string =>
|
|
57
|
+
new Bun.CryptoHasher("sha256").update(text).digest("hex");
|
|
58
|
+
|
|
47
59
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
48
60
|
// Score Normalization
|
|
49
61
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
@@ -124,11 +136,9 @@ function buildSearchResult(ctx: BuildResultContext): SearchResult {
|
|
|
124
136
|
snippet = chunk.text;
|
|
125
137
|
snippetRange = { startLine: chunk.startLine, endLine: chunk.endLine };
|
|
126
138
|
} else {
|
|
127
|
-
// Default:
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
chunk?.text
|
|
131
|
-
);
|
|
139
|
+
// Default: chunk text until attachChunkSnippets replaces it for the final
|
|
140
|
+
// rows, with leading frontmatter stripped
|
|
141
|
+
const cleaned = cleanDisplaySnippet(chunk?.text ?? "", chunk?.text);
|
|
132
142
|
snippet = cleaned.text;
|
|
133
143
|
if (chunk) {
|
|
134
144
|
line = chunk.startLine + cleaned.startLineOffset;
|
|
@@ -162,10 +172,130 @@ function buildSearchResult(ctx: BuildResultContext): SearchResult {
|
|
|
162
172
|
graphExpanded: false,
|
|
163
173
|
startLine: chunk.startLine,
|
|
164
174
|
endLine: chunk.endLine,
|
|
165
|
-
passageHash:
|
|
166
|
-
|
|
167
|
-
|
|
175
|
+
passageHash: passageHash(chunk.text),
|
|
176
|
+
});
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
/** Query-word ids present in each chunk of the returned documents. */
|
|
180
|
+
async function readChunkTermSets(
|
|
181
|
+
store: StorePort,
|
|
182
|
+
hashes: string[],
|
|
183
|
+
terms: SnippetTerms,
|
|
184
|
+
loadedChunks: Map<string, ChunkRow[]> | undefined
|
|
185
|
+
): Promise<{
|
|
186
|
+
termSets: Map<string, ChunkTermSet[]>;
|
|
187
|
+
chunks?: Map<string, ChunkRow[]>;
|
|
188
|
+
} | null> {
|
|
189
|
+
const termSets = new Map<string, ChunkTermSet[]>();
|
|
190
|
+
const add = (mirrorHash: string, seq: number, present: boolean[]) => {
|
|
191
|
+
const list = termSets.get(mirrorHash) ?? [];
|
|
192
|
+
list.push({ seq, terms: presentTermIds(present, terms) });
|
|
193
|
+
termSets.set(mirrorHash, list);
|
|
194
|
+
};
|
|
195
|
+
// In-database presence keeps long documents from being loaded whole.
|
|
196
|
+
const presence =
|
|
197
|
+
!loadedChunks && store.getChunkTermPresence
|
|
198
|
+
? await store.getChunkTermPresence(hashes, terms.prefixes)
|
|
199
|
+
: undefined;
|
|
200
|
+
if (presence?.ok) {
|
|
201
|
+
for (const row of presence.value) add(row.mirrorHash, row.seq, row.present);
|
|
202
|
+
return { termSets };
|
|
203
|
+
}
|
|
204
|
+
let chunks = loadedChunks;
|
|
205
|
+
if (!chunks) {
|
|
206
|
+
const fetched = await store.getChunksBatch(hashes);
|
|
207
|
+
if (!fetched.ok) return null;
|
|
208
|
+
chunks = fetched.value;
|
|
209
|
+
}
|
|
210
|
+
for (const [mirrorHash, list] of chunks)
|
|
211
|
+
for (const chunk of list)
|
|
212
|
+
add(mirrorHash, chunk.seq, chunkTermPresence(chunk.text, terms));
|
|
213
|
+
return { termSets, chunks };
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
/**
|
|
217
|
+
* Default-mode snippets for the returned rows only, built from their stored
|
|
218
|
+
* chunks (FTS5 snippet() re-tokenized every matched document body). Only the
|
|
219
|
+
* chosen chunk of each row is loaded; with an intent, the steered chunk is
|
|
220
|
+
* kept. The snippet, its range and the planner chunk all name that chunk.
|
|
221
|
+
* Updates results in place so hidden ranking metadata stays attached.
|
|
222
|
+
*/
|
|
223
|
+
async function attachChunkSnippets(
|
|
224
|
+
store: StorePort,
|
|
225
|
+
results: SearchResult[],
|
|
226
|
+
query: string,
|
|
227
|
+
options: SearchOptions,
|
|
228
|
+
loadedChunks: Map<string, ChunkRow[]> | undefined
|
|
229
|
+
): Promise<void> {
|
|
230
|
+
const terms = createSnippetTerms(query);
|
|
231
|
+
const hashes = [
|
|
232
|
+
...new Set(
|
|
233
|
+
results
|
|
234
|
+
.map((result) => result.conversion?.mirrorHash)
|
|
235
|
+
.filter((hash): hash is string => Boolean(hash))
|
|
236
|
+
),
|
|
237
|
+
];
|
|
238
|
+
if (!terms || hashes.length === 0) return;
|
|
239
|
+
const read = await readChunkTermSets(store, hashes, terms, loadedChunks);
|
|
240
|
+
if (!read) return;
|
|
241
|
+
const weights = snippetTermWeights([...read.termSets.values()].flat());
|
|
242
|
+
|
|
243
|
+
const chosen = results.map((result) => {
|
|
244
|
+
const mirrorHash = result.conversion?.mirrorHash;
|
|
245
|
+
if (!mirrorHash) return undefined;
|
|
246
|
+
const steeredSeq = result[SEARCH_RESULT_PLANNER_METADATA]?.seq;
|
|
247
|
+
const candidates = read.termSets.get(mirrorHash) ?? [];
|
|
248
|
+
const seq = pickSnippetSeq(
|
|
249
|
+
options.intent
|
|
250
|
+
? candidates.filter((entry) => entry.seq === steeredSeq)
|
|
251
|
+
: candidates,
|
|
252
|
+
weights
|
|
253
|
+
);
|
|
254
|
+
return seq === undefined ? undefined : { mirrorHash, seq };
|
|
168
255
|
});
|
|
256
|
+
let chunks = read.chunks;
|
|
257
|
+
if (!chunks) {
|
|
258
|
+
const fetched = store.getChunksBySequenceBatch
|
|
259
|
+
? await store.getChunksBySequenceBatch(
|
|
260
|
+
chosen.filter((key) => key !== undefined)
|
|
261
|
+
)
|
|
262
|
+
: await store.getChunksBatch(hashes);
|
|
263
|
+
if (!fetched.ok) return;
|
|
264
|
+
chunks = fetched.value;
|
|
265
|
+
}
|
|
266
|
+
const getChunk = createChunkLookup(chunks);
|
|
267
|
+
|
|
268
|
+
for (const [index, result] of results.entries()) {
|
|
269
|
+
const key = chosen[index];
|
|
270
|
+
const chunk = key ? getChunk(key.mirrorHash, key.seq) : undefined;
|
|
271
|
+
if (!(key && chunk)) continue;
|
|
272
|
+
const cleaned = cleanDisplaySnippet(
|
|
273
|
+
renderChunkSnippet(chunk.text, terms, weights),
|
|
274
|
+
chunk.text
|
|
275
|
+
);
|
|
276
|
+
result.line = chunk.startLine + cleaned.startLineOffset;
|
|
277
|
+
result.snippet = cleaned.text;
|
|
278
|
+
result.snippetLanguage = chunk.language ?? undefined;
|
|
279
|
+
result.snippetRange = { startLine: result.line, endLine: chunk.endLine };
|
|
280
|
+
const coordinates = {
|
|
281
|
+
mirrorHash: key.mirrorHash,
|
|
282
|
+
seq: chunk.seq,
|
|
283
|
+
startLine: chunk.startLine,
|
|
284
|
+
endLine: chunk.endLine,
|
|
285
|
+
passageHash: passageHash(chunk.text),
|
|
286
|
+
};
|
|
287
|
+
const metadata = result[SEARCH_RESULT_PLANNER_METADATA];
|
|
288
|
+
if (metadata) {
|
|
289
|
+
Object.assign(metadata, coordinates);
|
|
290
|
+
} else {
|
|
291
|
+
attachSearchResultPlannerMetadata(result, {
|
|
292
|
+
retrievalRank: 0,
|
|
293
|
+
sources: ["bm25"],
|
|
294
|
+
graphExpanded: false,
|
|
295
|
+
...coordinates,
|
|
296
|
+
});
|
|
297
|
+
}
|
|
298
|
+
}
|
|
169
299
|
}
|
|
170
300
|
|
|
171
301
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
@@ -207,8 +337,7 @@ export async function searchBm25(
|
|
|
207
337
|
const detection = detectQueryLanguage(query);
|
|
208
338
|
const queryLanguage = options.lang ?? detection.bcp47;
|
|
209
339
|
|
|
210
|
-
// Run FTS search
|
|
211
|
-
// Disable FTS snippet when --full or --line-numbers (we use raw text instead)
|
|
340
|
+
// Run FTS search (ranking only; snippets come from chunks of the final rows)
|
|
212
341
|
const ftsResult = await store.searchFts(query, {
|
|
213
342
|
limit: retrievalLimit,
|
|
214
343
|
collection: options.collection,
|
|
@@ -216,7 +345,6 @@ export async function searchBm25(
|
|
|
216
345
|
exclude: options.exclude,
|
|
217
346
|
relPathPrefix: options.retrievalScope?.relPathPrefix,
|
|
218
347
|
language: options.lang,
|
|
219
|
-
snippet: !(options.full || options.lineNumbers),
|
|
220
348
|
tagsAll: options.tagsAll,
|
|
221
349
|
tagsAny: options.tagsAny,
|
|
222
350
|
since: temporalRange.since,
|
|
@@ -265,10 +393,9 @@ export async function searchBm25(
|
|
|
265
393
|
ftsResult.value.map((f) => f.mirrorHash).filter((h): h is string => !!h)
|
|
266
394
|
),
|
|
267
395
|
];
|
|
396
|
+
const needsAllChunks = Boolean(options.intent || options.exclude?.length);
|
|
268
397
|
const chunksMapResult =
|
|
269
|
-
store.getChunksBySequenceBatch &&
|
|
270
|
-
!options.intent &&
|
|
271
|
-
!options.exclude?.length
|
|
398
|
+
store.getChunksBySequenceBatch && !needsAllChunks
|
|
272
399
|
? await store.getChunksBySequenceBatch(
|
|
273
400
|
ftsResult.value
|
|
274
401
|
.filter((row) => !!row.mirrorHash)
|
|
@@ -278,6 +405,10 @@ export async function searchBm25(
|
|
|
278
405
|
const getChunk = chunksMapResult.ok
|
|
279
406
|
? createChunkLookup(chunksMapResult.value)
|
|
280
407
|
: () => undefined;
|
|
408
|
+
const allChunks =
|
|
409
|
+
chunksMapResult.ok && (needsAllChunks || !store.getChunksBySequenceBatch)
|
|
410
|
+
? chunksMapResult.value
|
|
411
|
+
: undefined;
|
|
281
412
|
|
|
282
413
|
// Dedup: multiple docs can share mirror_hash (content-addressed storage)
|
|
283
414
|
// Track seen uri+seq to eliminate duplicate rows from join fan-out
|
|
@@ -326,11 +457,7 @@ export async function searchBm25(
|
|
|
326
457
|
|
|
327
458
|
const excluded =
|
|
328
459
|
matchesExcludedText(
|
|
329
|
-
[
|
|
330
|
-
fts.title ?? "",
|
|
331
|
-
fts.recordSourcePath ?? fts.relPath ?? "",
|
|
332
|
-
fts.snippet ?? "",
|
|
333
|
-
],
|
|
460
|
+
[fts.title ?? "", fts.recordSourcePath ?? fts.relPath ?? ""],
|
|
334
461
|
options.exclude
|
|
335
462
|
) ||
|
|
336
463
|
matchesExcludedChunks(
|
|
@@ -464,6 +591,9 @@ export async function searchBm25(
|
|
|
464
591
|
}
|
|
465
592
|
|
|
466
593
|
const finalResults = filteredResults.slice(0, limit);
|
|
594
|
+
if (!(options.full || options.lineNumbers)) {
|
|
595
|
+
await attachChunkSnippets(store, finalResults, query, options, allChunks);
|
|
596
|
+
}
|
|
467
597
|
for (const [index, result] of finalResults.entries()) {
|
|
468
598
|
const metadata = result[SEARCH_RESULT_PLANNER_METADATA];
|
|
469
599
|
if (metadata) metadata.retrievalRank = index + 1;
|
package/src/sdk/client.ts
CHANGED
|
@@ -189,6 +189,7 @@ import {
|
|
|
189
189
|
withContentTypeRules,
|
|
190
190
|
} from "../ingestion";
|
|
191
191
|
import { updateFrontmatterTags } from "../ingestion/frontmatter";
|
|
192
|
+
import { withCollectionRootErrors } from "../ingestion/source-availability/root";
|
|
192
193
|
import {
|
|
193
194
|
withInferenceScope,
|
|
194
195
|
finishInferenceCleanup,
|
|
@@ -210,7 +211,10 @@ import { runAutomationProfile } from "../sessions/automation";
|
|
|
210
211
|
import { assertSessionBinding } from "../sessions/binding";
|
|
211
212
|
import { SessionsService } from "../sessions/service";
|
|
212
213
|
import { SESSIONS_VALIDATION_CODES, SessionsError } from "../sessions/types";
|
|
213
|
-
import {
|
|
214
|
+
import {
|
|
215
|
+
SqliteAdapter,
|
|
216
|
+
staleCollectionsMessage,
|
|
217
|
+
} from "../store/sqlite/adapter";
|
|
214
218
|
import { openScopedIndexStore } from "../store/sqlite/scoped-index";
|
|
215
219
|
import { createVectorIndexPort } from "../store/vector";
|
|
216
220
|
import {
|
|
@@ -462,6 +466,14 @@ class GnoClientImpl implements GnoClient {
|
|
|
462
466
|
if (!this.isOpen()) {
|
|
463
467
|
throw sdkError("RUNTIME", "GNO client is closed");
|
|
464
468
|
}
|
|
469
|
+
// Another client may have indexed a collection this config omits since
|
|
470
|
+
// this one opened; never read it.
|
|
471
|
+
const stale = this.store.staleIndexedCollections(
|
|
472
|
+
this.config.collections.map(({ name }) => name)
|
|
473
|
+
);
|
|
474
|
+
if (stale.length > 0) {
|
|
475
|
+
throw sdkError("RUNTIME", staleCollectionsMessage(stale));
|
|
476
|
+
}
|
|
465
477
|
}
|
|
466
478
|
|
|
467
479
|
private getCollections(collection?: string) {
|
|
@@ -1541,7 +1553,7 @@ class GnoClientImpl implements GnoClient {
|
|
|
1541
1553
|
})
|
|
1542
1554
|
);
|
|
1543
1555
|
return {
|
|
1544
|
-
...status,
|
|
1556
|
+
...(await withCollectionRootErrors(status, this.config.collections)),
|
|
1545
1557
|
contentTypeBoost: buildContentTypeBoostStatus(
|
|
1546
1558
|
this.config.contentTypes ?? []
|
|
1547
1559
|
),
|
package/src/sdk/types.ts
CHANGED
|
@@ -90,6 +90,7 @@ import type {
|
|
|
90
90
|
SectionTargetV1,
|
|
91
91
|
} from "../core/sections";
|
|
92
92
|
import type { SyncResult } from "../ingestion";
|
|
93
|
+
import type { CollectionRootError } from "../ingestion/source-availability/root";
|
|
93
94
|
import type { DownloadPolicy } from "../llm/policy";
|
|
94
95
|
import type {
|
|
95
96
|
AskOptions,
|
|
@@ -193,7 +194,11 @@ export type GnoVectorSearchOptions = Omit<
|
|
|
193
194
|
export type GnoContextInput = Omit<ContextCapsuleBuildInput, "indexName"> &
|
|
194
195
|
GnoProjectHintOptions;
|
|
195
196
|
export type GnoContextResult = ContextCapsuleV1;
|
|
196
|
-
export type GnoIndexStatus = IndexStatus & {
|
|
197
|
+
export type GnoIndexStatus = Omit<IndexStatus, "collections"> & {
|
|
198
|
+
/** `rootError` is set while a collection's folder is missing or unreadable. */
|
|
199
|
+
collections: Array<
|
|
200
|
+
IndexStatus["collections"][number] & { rootError?: CollectionRootError }
|
|
201
|
+
>;
|
|
197
202
|
contentTypeBoost: ContentTypeBoostStatus;
|
|
198
203
|
};
|
|
199
204
|
export type GnoContextVerificationResult = ContextCapsuleVerification;
|
|
@@ -29,7 +29,7 @@ interface Option {
|
|
|
29
29
|
}
|
|
30
30
|
|
|
31
31
|
const options: Option[] = [
|
|
32
|
-
{ id: "fast", label: "Fast", timing: "
|
|
32
|
+
{ id: "fast", label: "Fast", timing: "<1s", icon: Rabbit },
|
|
33
33
|
{ id: "balanced", label: "Balanced", timing: "~2s", icon: Gauge },
|
|
34
34
|
{ id: "thorough", label: "Thorough", timing: "~5s", icon: Sparkles },
|
|
35
35
|
];
|