@gmickel/gno 0.17.0 → 0.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +25 -1
- package/package.json +1 -1
- package/src/cli/commands/ask.ts +7 -0
- package/src/cli/commands/models/use.ts +1 -0
- package/src/cli/program.ts +42 -0
- package/src/config/types.ts +2 -0
- package/src/llm/nodeLlamaCpp/generation.ts +3 -1
- package/src/llm/registry.ts +1 -0
- package/src/llm/types.ts +2 -0
- package/src/mcp/tools/index.ts +7 -0
- package/src/mcp/tools/query.ts +6 -0
- package/src/mcp/tools/search.ts +4 -0
- package/src/mcp/tools/vsearch.ts +4 -0
- package/src/pipeline/exclude.ts +69 -0
- package/src/pipeline/expansion.ts +39 -4
- package/src/pipeline/hybrid.ts +59 -18
- package/src/pipeline/intent.ts +152 -0
- package/src/pipeline/rerank.ts +81 -44
- package/src/pipeline/search.ts +34 -1
- package/src/pipeline/types.ts +15 -0
- package/src/pipeline/vsearch.ts +41 -1
- package/src/serve/public/lib/retrieval-filters.ts +10 -0
- package/src/serve/public/pages/Ask.tsx +189 -1
- package/src/serve/public/pages/Search.tsx +78 -2
- package/src/serve/routes/api.ts +161 -48
package/README.md
CHANGED
|
@@ -32,7 +32,27 @@ GNO is a local knowledge engine that turns your documents into a searchable, con
|
|
|
32
32
|
|
|
33
33
|
---
|
|
34
34
|
|
|
35
|
-
## What's New in v0.
|
|
35
|
+
## What's New in v0.19
|
|
36
|
+
|
|
37
|
+
- **Exclusion Filters**: explicit `exclude` controls across CLI, API, Web, and MCP to hard-prune unwanted docs by title/path/body text
|
|
38
|
+
- **Ask Query-Mode Parity**: Ask now supports structured `term` / `intent` / `hyde` controls in both API and Web UI
|
|
39
|
+
|
|
40
|
+
### v0.18
|
|
41
|
+
|
|
42
|
+
- **Intent Steering**: optional `intent` control for ambiguous queries across CLI, API, Web, and MCP query flows
|
|
43
|
+
- **Rerank Controls**: `candidateLimit` lets you tune rerank cost vs. recall on slower or memory-constrained machines
|
|
44
|
+
- **Stability**: query expansion now uses a bounded configurable context size (`models.expandContextSize`, default `2048`)
|
|
45
|
+
- **Rerank Efficiency**: identical chunk texts are deduplicated before scoring and expanded back out deterministically
|
|
46
|
+
|
|
47
|
+
### v0.17
|
|
48
|
+
|
|
49
|
+
- **Structured Query Modes**: `term`, `intent`, and `hyde` controls across CLI, API, MCP, and Web
|
|
50
|
+
- **Temporal Retrieval Upgrades**: `since`/`until`, date-range parsing, and recency sorting with frontmatter-date fallback
|
|
51
|
+
- **Web Retrieval UX Polish**: richer advanced controls in Search and Ask (collection/date/category/author/tags + query modes)
|
|
52
|
+
- **Metadata-Aware Retrieval**: ingestion now materializes document metadata/date fields for better filtering and ranking
|
|
53
|
+
- **Migration Reliability**: SQLite-compatible migration path for existing indexes (including older SQLite engines)
|
|
54
|
+
|
|
55
|
+
### v0.15
|
|
36
56
|
|
|
37
57
|
- **HTTP Backends**: Offload embedding, reranking, and generation to remote GPU servers
|
|
38
58
|
- Simple URI config: `http://host:port/path#modelname`
|
|
@@ -146,6 +166,8 @@ gno search "handleAuth" # Find exact matches
|
|
|
146
166
|
gno vsearch "error handling patterns" # Semantic similarity
|
|
147
167
|
gno query "database optimization" # Full pipeline
|
|
148
168
|
gno query "meeting decisions" --since "last month" --category "meeting,notes" --author "gordon"
|
|
169
|
+
gno query "performance" --intent "web performance and latency"
|
|
170
|
+
gno query "performance" --exclude "reviews,hiring"
|
|
149
171
|
gno ask "what did we decide" --answer # AI synthesis
|
|
150
172
|
```
|
|
151
173
|
|
|
@@ -161,6 +183,8 @@ gno query "auth flow" --thorough
|
|
|
161
183
|
|
|
162
184
|
# Structured retrieval intent
|
|
163
185
|
gno query "auth flow" \
|
|
186
|
+
--intent "web authentication and token lifecycle" \
|
|
187
|
+
--candidate-limit 12 \
|
|
164
188
|
--query-mode term:"jwt refresh token -oauth1" \
|
|
165
189
|
--query-mode intent:"how refresh token rotation works" \
|
|
166
190
|
--query-mode hyde:"Refresh tokens rotate on each use and previous tokens are revoked." \
|
package/package.json
CHANGED
package/src/cli/commands/ask.ts
CHANGED
|
@@ -192,14 +192,17 @@ export async function ask(
|
|
|
192
192
|
limit,
|
|
193
193
|
collection: options.collection,
|
|
194
194
|
lang: options.lang,
|
|
195
|
+
intent: options.intent,
|
|
195
196
|
since: options.since,
|
|
196
197
|
until: options.until,
|
|
197
198
|
categories: options.categories,
|
|
198
199
|
author: options.author,
|
|
199
200
|
tagsAll: options.tagsAll,
|
|
200
201
|
tagsAny: options.tagsAny,
|
|
202
|
+
exclude: options.exclude,
|
|
201
203
|
noExpand: options.noExpand,
|
|
202
204
|
noRerank: options.noRerank,
|
|
205
|
+
candidateLimit: options.candidateLimit,
|
|
203
206
|
});
|
|
204
207
|
|
|
205
208
|
if (!searchResult.ok) {
|
|
@@ -258,6 +261,10 @@ export async function ask(
|
|
|
258
261
|
expanded: searchResult.value.meta.expanded ?? false,
|
|
259
262
|
reranked: searchResult.value.meta.reranked ?? false,
|
|
260
263
|
vectorsUsed: searchResult.value.meta.vectorsUsed ?? false,
|
|
264
|
+
intent: searchResult.value.meta.intent,
|
|
265
|
+
candidateLimit: searchResult.value.meta.candidateLimit,
|
|
266
|
+
exclude: searchResult.value.meta.exclude,
|
|
267
|
+
queryModes: searchResult.value.meta.queryModes,
|
|
261
268
|
answerGenerated,
|
|
262
269
|
totalResults: results.length,
|
|
263
270
|
answerContext,
|
|
@@ -58,6 +58,7 @@ export async function modelsUse(
|
|
|
58
58
|
presets: config.models?.presets ?? [],
|
|
59
59
|
loadTimeout: config.models?.loadTimeout ?? 60_000,
|
|
60
60
|
inferenceTimeout: config.models?.inferenceTimeout ?? 30_000,
|
|
61
|
+
expandContextSize: config.models?.expandContextSize ?? 2_048,
|
|
61
62
|
warmModelTtl: config.models?.warmModelTtl ?? 300_000,
|
|
62
63
|
},
|
|
63
64
|
};
|
package/src/cli/program.ts
CHANGED
|
@@ -224,6 +224,11 @@ function wireSearchCommands(program: Command): void {
|
|
|
224
224
|
)
|
|
225
225
|
.option("--category <values>", "require category match (comma-separated)")
|
|
226
226
|
.option("--author <text>", "filter by author (case-insensitive contains)")
|
|
227
|
+
.option("--intent <text>", "disambiguating context for ambiguous queries")
|
|
228
|
+
.option(
|
|
229
|
+
"--exclude <values>",
|
|
230
|
+
"exclude docs containing any term (comma-separated)"
|
|
231
|
+
)
|
|
227
232
|
.option("--tags-all <tags>", "require ALL tags (comma-separated)")
|
|
228
233
|
.option("--tags-any <tags>", "require ANY tag (comma-separated)")
|
|
229
234
|
.option("--full", "include full content")
|
|
@@ -269,6 +274,7 @@ function wireSearchCommands(program: Command): void {
|
|
|
269
274
|
? parsePositiveInt("limit", cmdOpts.limit)
|
|
270
275
|
: getDefaultLimit(format);
|
|
271
276
|
const categories = parseCsvValues(cmdOpts.category);
|
|
277
|
+
const exclude = parseCsvValues(cmdOpts.exclude);
|
|
272
278
|
|
|
273
279
|
const { search, formatSearch } = await import("./commands/search");
|
|
274
280
|
const result = await search(queryText, {
|
|
@@ -280,6 +286,8 @@ function wireSearchCommands(program: Command): void {
|
|
|
280
286
|
until: cmdOpts.until as string | undefined,
|
|
281
287
|
categories,
|
|
282
288
|
author: cmdOpts.author as string | undefined,
|
|
289
|
+
intent: cmdOpts.intent as string | undefined,
|
|
290
|
+
exclude,
|
|
283
291
|
tagsAll,
|
|
284
292
|
tagsAny,
|
|
285
293
|
full: Boolean(cmdOpts.full),
|
|
@@ -329,6 +337,11 @@ function wireSearchCommands(program: Command): void {
|
|
|
329
337
|
)
|
|
330
338
|
.option("--category <values>", "require category match (comma-separated)")
|
|
331
339
|
.option("--author <text>", "filter by author (case-insensitive contains)")
|
|
340
|
+
.option("--intent <text>", "disambiguating context for ambiguous queries")
|
|
341
|
+
.option(
|
|
342
|
+
"--exclude <values>",
|
|
343
|
+
"exclude docs containing any term (comma-separated)"
|
|
344
|
+
)
|
|
332
345
|
.option("--tags-all <tags>", "require ALL tags (comma-separated)")
|
|
333
346
|
.option("--tags-any <tags>", "require ANY tag (comma-separated)")
|
|
334
347
|
.option("--full", "include full content")
|
|
@@ -374,6 +387,7 @@ function wireSearchCommands(program: Command): void {
|
|
|
374
387
|
? parsePositiveInt("limit", cmdOpts.limit)
|
|
375
388
|
: getDefaultLimit(format);
|
|
376
389
|
const categories = parseCsvValues(cmdOpts.category);
|
|
390
|
+
const exclude = parseCsvValues(cmdOpts.exclude);
|
|
377
391
|
|
|
378
392
|
const { vsearch, formatVsearch } = await import("./commands/vsearch");
|
|
379
393
|
const result = await vsearch(queryText, {
|
|
@@ -385,6 +399,8 @@ function wireSearchCommands(program: Command): void {
|
|
|
385
399
|
until: cmdOpts.until as string | undefined,
|
|
386
400
|
categories,
|
|
387
401
|
author: cmdOpts.author as string | undefined,
|
|
402
|
+
intent: cmdOpts.intent as string | undefined,
|
|
403
|
+
exclude,
|
|
388
404
|
tagsAll,
|
|
389
405
|
tagsAny,
|
|
390
406
|
full: Boolean(cmdOpts.full),
|
|
@@ -429,6 +445,11 @@ function wireSearchCommands(program: Command): void {
|
|
|
429
445
|
)
|
|
430
446
|
.option("--category <values>", "require category match (comma-separated)")
|
|
431
447
|
.option("--author <text>", "filter by author (case-insensitive contains)")
|
|
448
|
+
.option("--intent <text>", "disambiguating context for ambiguous queries")
|
|
449
|
+
.option(
|
|
450
|
+
"--exclude <values>",
|
|
451
|
+
"exclude docs containing any term (comma-separated)"
|
|
452
|
+
)
|
|
432
453
|
.option("--tags-all <tags>", "require ALL tags (comma-separated)")
|
|
433
454
|
.option("--tags-any <tags>", "require ANY tag (comma-separated)")
|
|
434
455
|
.option("--full", "include full content")
|
|
@@ -443,6 +464,7 @@ function wireSearchCommands(program: Command): void {
|
|
|
443
464
|
(value: string, previous: string[] = []) => [...previous, value],
|
|
444
465
|
[]
|
|
445
466
|
)
|
|
467
|
+
.option("-C, --candidate-limit <num>", "max candidates passed to reranking")
|
|
446
468
|
.option("--explain", "include scoring explanation")
|
|
447
469
|
.option("--json", "JSON output")
|
|
448
470
|
.option("--md", "Markdown output")
|
|
@@ -495,7 +517,11 @@ function wireSearchCommands(program: Command): void {
|
|
|
495
517
|
const limit = cmdOpts.limit
|
|
496
518
|
? parsePositiveInt("limit", cmdOpts.limit)
|
|
497
519
|
: getDefaultLimit(format);
|
|
520
|
+
const candidateLimit = cmdOpts.candidateLimit
|
|
521
|
+
? parsePositiveInt("candidate-limit", cmdOpts.candidateLimit)
|
|
522
|
+
: undefined;
|
|
498
523
|
const categories = parseCsvValues(cmdOpts.category);
|
|
524
|
+
const exclude = parseCsvValues(cmdOpts.exclude);
|
|
499
525
|
|
|
500
526
|
// Determine expansion/rerank settings based on flags
|
|
501
527
|
// Priority: --fast > --thorough > --no-expand/--no-rerank > default
|
|
@@ -531,12 +557,15 @@ function wireSearchCommands(program: Command): void {
|
|
|
531
557
|
until: cmdOpts.until as string | undefined,
|
|
532
558
|
categories,
|
|
533
559
|
author: cmdOpts.author as string | undefined,
|
|
560
|
+
intent: cmdOpts.intent as string | undefined,
|
|
561
|
+
exclude,
|
|
534
562
|
tagsAll,
|
|
535
563
|
tagsAny,
|
|
536
564
|
full: Boolean(cmdOpts.full),
|
|
537
565
|
lineNumbers: Boolean(cmdOpts.lineNumbers),
|
|
538
566
|
noExpand,
|
|
539
567
|
noRerank,
|
|
568
|
+
candidateLimit,
|
|
540
569
|
queryModes,
|
|
541
570
|
explain: Boolean(cmdOpts.explain),
|
|
542
571
|
json: format === "json",
|
|
@@ -574,8 +603,14 @@ function wireSearchCommands(program: Command): void {
|
|
|
574
603
|
)
|
|
575
604
|
.option("--category <values>", "require category match (comma-separated)")
|
|
576
605
|
.option("--author <text>", "filter by author (case-insensitive contains)")
|
|
606
|
+
.option("--intent <text>", "disambiguating context for ambiguous queries")
|
|
607
|
+
.option(
|
|
608
|
+
"--exclude <values>",
|
|
609
|
+
"exclude docs containing any term (comma-separated)"
|
|
610
|
+
)
|
|
577
611
|
.option("--fast", "skip expansion and reranking (fastest)")
|
|
578
612
|
.option("--thorough", "enable query expansion (slower)")
|
|
613
|
+
.option("-C, --candidate-limit <num>", "max candidates passed to reranking")
|
|
579
614
|
.option("--answer", "generate short grounded answer")
|
|
580
615
|
.option("--no-answer", "force retrieval-only output")
|
|
581
616
|
.option("--max-answer-tokens <num>", "max answer tokens")
|
|
@@ -594,12 +629,16 @@ function wireSearchCommands(program: Command): void {
|
|
|
594
629
|
const limit = cmdOpts.limit
|
|
595
630
|
? parsePositiveInt("limit", cmdOpts.limit)
|
|
596
631
|
: getDefaultLimit(format);
|
|
632
|
+
const candidateLimit = cmdOpts.candidateLimit
|
|
633
|
+
? parsePositiveInt("candidate-limit", cmdOpts.candidateLimit)
|
|
634
|
+
: undefined;
|
|
597
635
|
|
|
598
636
|
// Parse max-answer-tokens (optional, defaults to 512 in command impl)
|
|
599
637
|
const maxAnswerTokens = cmdOpts.maxAnswerTokens
|
|
600
638
|
? parsePositiveInt("max-answer-tokens", cmdOpts.maxAnswerTokens)
|
|
601
639
|
: undefined;
|
|
602
640
|
const categories = parseCsvValues(cmdOpts.category);
|
|
641
|
+
const exclude = parseCsvValues(cmdOpts.exclude);
|
|
603
642
|
|
|
604
643
|
// Determine expansion/rerank settings based on flags
|
|
605
644
|
// Default: skip expansion (balanced mode)
|
|
@@ -624,8 +663,11 @@ function wireSearchCommands(program: Command): void {
|
|
|
624
663
|
until: cmdOpts.until as string | undefined,
|
|
625
664
|
categories,
|
|
626
665
|
author: cmdOpts.author as string | undefined,
|
|
666
|
+
intent: cmdOpts.intent as string | undefined,
|
|
667
|
+
exclude,
|
|
627
668
|
noExpand,
|
|
628
669
|
noRerank,
|
|
670
|
+
candidateLimit,
|
|
629
671
|
// Per spec: --answer defaults to false, --no-answer forces retrieval-only
|
|
630
672
|
// Commander creates separate cmdOpts.noAnswer for --no-answer flag
|
|
631
673
|
answer: Boolean(cmdOpts.answer),
|
package/src/config/types.ts
CHANGED
|
@@ -209,6 +209,8 @@ export const ModelConfigSchema = z.object({
|
|
|
209
209
|
loadTimeout: z.number().default(60_000),
|
|
210
210
|
/** Inference timeout in ms */
|
|
211
211
|
inferenceTimeout: z.number().default(30_000),
|
|
212
|
+
/** Context size used for query expansion generation */
|
|
213
|
+
expandContextSize: z.number().int().min(256).default(2_048),
|
|
212
214
|
/** Keep warm model TTL in ms (5 min) */
|
|
213
215
|
warmModelTtl: z.number().default(300_000),
|
|
214
216
|
});
|
|
@@ -56,7 +56,9 @@ export class NodeLlamaCppGeneration implements GenerationPort {
|
|
|
56
56
|
}
|
|
57
57
|
|
|
58
58
|
const llamaModel = model.value.model as LlamaModel;
|
|
59
|
-
const context = await llamaModel.createContext(
|
|
59
|
+
const context = await llamaModel.createContext(
|
|
60
|
+
params?.contextSize ? { contextSize: params.contextSize } : undefined
|
|
61
|
+
);
|
|
60
62
|
|
|
61
63
|
try {
|
|
62
64
|
// Import LlamaChatSession dynamically
|
package/src/llm/registry.ts
CHANGED
|
@@ -25,6 +25,7 @@ export function getModelConfig(config: Config): ModelConfig {
|
|
|
25
25
|
: DEFAULT_MODEL_PRESETS,
|
|
26
26
|
loadTimeout: config.models?.loadTimeout ?? 60_000,
|
|
27
27
|
inferenceTimeout: config.models?.inferenceTimeout ?? 30_000,
|
|
28
|
+
expandContextSize: config.models?.expandContextSize ?? 2_048,
|
|
28
29
|
warmModelTtl: config.models?.warmModelTtl ?? 300_000,
|
|
29
30
|
};
|
|
30
31
|
}
|
package/src/llm/types.ts
CHANGED
package/src/mcp/tools/index.ts
CHANGED
|
@@ -56,6 +56,8 @@ const searchInputSchema = z.object({
|
|
|
56
56
|
limit: z.number().int().min(1).max(100).default(5),
|
|
57
57
|
minScore: z.number().min(0).max(1).optional(),
|
|
58
58
|
lang: z.string().optional(),
|
|
59
|
+
intent: z.string().optional(),
|
|
60
|
+
exclude: z.array(z.string()).optional(),
|
|
59
61
|
since: z.string().optional(),
|
|
60
62
|
until: z.string().optional(),
|
|
61
63
|
categories: z.array(z.string()).optional(),
|
|
@@ -105,6 +107,8 @@ const vsearchInputSchema = z.object({
|
|
|
105
107
|
limit: z.number().int().min(1).max(100).default(5),
|
|
106
108
|
minScore: z.number().min(0).max(1).optional(),
|
|
107
109
|
lang: z.string().optional(),
|
|
110
|
+
intent: z.string().optional(),
|
|
111
|
+
exclude: z.array(z.string()).optional(),
|
|
108
112
|
since: z.string().optional(),
|
|
109
113
|
until: z.string().optional(),
|
|
110
114
|
categories: z.array(z.string()).optional(),
|
|
@@ -124,6 +128,9 @@ export const queryInputSchema = z.object({
|
|
|
124
128
|
limit: z.number().int().min(1).max(100).default(5),
|
|
125
129
|
minScore: z.number().min(0).max(1).optional(),
|
|
126
130
|
lang: z.string().optional(),
|
|
131
|
+
intent: z.string().optional(),
|
|
132
|
+
candidateLimit: z.number().int().min(1).max(100).optional(),
|
|
133
|
+
exclude: z.array(z.string()).optional(),
|
|
127
134
|
since: z.string().optional(),
|
|
128
135
|
until: z.string().optional(),
|
|
129
136
|
categories: z.array(z.string()).optional(),
|
package/src/mcp/tools/query.ts
CHANGED
|
@@ -36,6 +36,9 @@ interface QueryInput {
|
|
|
36
36
|
limit?: number;
|
|
37
37
|
minScore?: number;
|
|
38
38
|
lang?: string;
|
|
39
|
+
intent?: string;
|
|
40
|
+
candidateLimit?: number;
|
|
41
|
+
exclude?: string[];
|
|
39
42
|
since?: string;
|
|
40
43
|
until?: string;
|
|
41
44
|
categories?: string[];
|
|
@@ -247,6 +250,9 @@ export function handleQuery(
|
|
|
247
250
|
minScore: args.minScore,
|
|
248
251
|
collection: args.collection,
|
|
249
252
|
queryLanguageHint: args.lang, // Affects expansion prompt, not retrieval
|
|
253
|
+
intent: args.intent,
|
|
254
|
+
candidateLimit: args.candidateLimit,
|
|
255
|
+
exclude: args.exclude,
|
|
250
256
|
since: args.since,
|
|
251
257
|
until: args.until,
|
|
252
258
|
categories: args.categories,
|
package/src/mcp/tools/search.ts
CHANGED
|
@@ -19,6 +19,8 @@ interface SearchInput {
|
|
|
19
19
|
limit?: number;
|
|
20
20
|
minScore?: number;
|
|
21
21
|
lang?: string;
|
|
22
|
+
intent?: string;
|
|
23
|
+
exclude?: string[];
|
|
22
24
|
since?: string;
|
|
23
25
|
until?: string;
|
|
24
26
|
categories?: string[];
|
|
@@ -108,6 +110,8 @@ export function handleSearch(
|
|
|
108
110
|
minScore: args.minScore,
|
|
109
111
|
collection: args.collection,
|
|
110
112
|
lang: args.lang,
|
|
113
|
+
intent: args.intent,
|
|
114
|
+
exclude: args.exclude,
|
|
111
115
|
since: args.since,
|
|
112
116
|
until: args.until,
|
|
113
117
|
categories: args.categories,
|
package/src/mcp/tools/vsearch.ts
CHANGED
|
@@ -28,6 +28,8 @@ interface VsearchInput {
|
|
|
28
28
|
limit?: number;
|
|
29
29
|
minScore?: number;
|
|
30
30
|
lang?: string;
|
|
31
|
+
intent?: string;
|
|
32
|
+
exclude?: string[];
|
|
31
33
|
since?: string;
|
|
32
34
|
until?: string;
|
|
33
35
|
categories?: string[];
|
|
@@ -192,6 +194,8 @@ export function handleVsearch(
|
|
|
192
194
|
limit: args.limit ?? 5,
|
|
193
195
|
minScore: args.minScore,
|
|
194
196
|
collection: args.collection,
|
|
197
|
+
intent: args.intent,
|
|
198
|
+
exclude: args.exclude,
|
|
195
199
|
since: args.since,
|
|
196
200
|
until: args.until,
|
|
197
201
|
categories: args.categories,
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Explicit exclusion helpers for retrieval filters.
|
|
3
|
+
*
|
|
4
|
+
* @module src/pipeline/exclude
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
import type { ChunkRow } from "../store/types";
|
|
8
|
+
|
|
9
|
+
export function normalizeExcludeTerms(values: string[]): string[] {
|
|
10
|
+
const out: string[] = [];
|
|
11
|
+
const seen = new Set<string>();
|
|
12
|
+
|
|
13
|
+
for (const value of values) {
|
|
14
|
+
for (const part of value.split(",")) {
|
|
15
|
+
const trimmed = part.trim();
|
|
16
|
+
if (!trimmed) {
|
|
17
|
+
continue;
|
|
18
|
+
}
|
|
19
|
+
const key = trimmed.toLowerCase();
|
|
20
|
+
if (seen.has(key)) {
|
|
21
|
+
continue;
|
|
22
|
+
}
|
|
23
|
+
seen.add(key);
|
|
24
|
+
out.push(trimmed);
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
return out;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
function includesTerm(haystack: string, term: string): boolean {
|
|
32
|
+
return haystack.toLowerCase().includes(term.toLowerCase());
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
export function matchesExcludedText(
|
|
36
|
+
haystacks: string[],
|
|
37
|
+
excludeTerms: string[] | undefined
|
|
38
|
+
): boolean {
|
|
39
|
+
if (!excludeTerms?.length) {
|
|
40
|
+
return false;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
for (const haystack of haystacks) {
|
|
44
|
+
if (!haystack) {
|
|
45
|
+
continue;
|
|
46
|
+
}
|
|
47
|
+
for (const term of excludeTerms) {
|
|
48
|
+
if (includesTerm(haystack, term)) {
|
|
49
|
+
return true;
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
return false;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
export function matchesExcludedChunks(
|
|
58
|
+
chunks: ChunkRow[],
|
|
59
|
+
excludeTerms: string[] | undefined
|
|
60
|
+
): boolean {
|
|
61
|
+
if (!excludeTerms?.length || chunks.length === 0) {
|
|
62
|
+
return false;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
return matchesExcludedText(
|
|
66
|
+
chunks.map((chunk) => chunk.text),
|
|
67
|
+
excludeTerms
|
|
68
|
+
);
|
|
69
|
+
}
|
|
@@ -67,9 +67,16 @@ const STOPWORDS = new Set([
|
|
|
67
67
|
export function generateCacheKey(
|
|
68
68
|
modelUri: string,
|
|
69
69
|
query: string,
|
|
70
|
-
lang: string
|
|
70
|
+
lang: string,
|
|
71
|
+
intent?: string
|
|
71
72
|
): string {
|
|
72
|
-
const data = [
|
|
73
|
+
const data = [
|
|
74
|
+
EXPANSION_PROMPT_VERSION,
|
|
75
|
+
modelUri,
|
|
76
|
+
query,
|
|
77
|
+
lang,
|
|
78
|
+
intent?.trim() ?? "",
|
|
79
|
+
].join("\0");
|
|
73
80
|
return createHash("sha256").update(data).digest("hex");
|
|
74
81
|
}
|
|
75
82
|
|
|
@@ -150,6 +157,24 @@ function getPromptTemplate(lang?: string): string {
|
|
|
150
157
|
}
|
|
151
158
|
}
|
|
152
159
|
|
|
160
|
+
function buildPrompt(query: string, template: string, intent?: string): string {
|
|
161
|
+
const basePrompt = template.replace("{query}", query);
|
|
162
|
+
const trimmedIntent = intent?.trim();
|
|
163
|
+
if (!trimmedIntent) {
|
|
164
|
+
return basePrompt;
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
return basePrompt
|
|
168
|
+
.replace(
|
|
169
|
+
`Query: "${query}"\n`,
|
|
170
|
+
`Query: "${query}"\nQuery intent: "${trimmedIntent}"\n`
|
|
171
|
+
)
|
|
172
|
+
.replace(
|
|
173
|
+
`Anfrage: "${query}"\n`,
|
|
174
|
+
`Anfrage: "${query}"\nQuery intent: "${trimmedIntent}"\n`
|
|
175
|
+
);
|
|
176
|
+
}
|
|
177
|
+
|
|
153
178
|
interface QuerySignals {
|
|
154
179
|
quotedPhrases: string[];
|
|
155
180
|
negations: string[];
|
|
@@ -405,6 +430,10 @@ export interface ExpansionOptions {
|
|
|
405
430
|
lang?: string;
|
|
406
431
|
/** Timeout in milliseconds */
|
|
407
432
|
timeout?: number;
|
|
433
|
+
/** Optional context that steers expansion for ambiguous queries */
|
|
434
|
+
intent?: string;
|
|
435
|
+
/** Optional bounded context size override for expansion generation */
|
|
436
|
+
contextSize?: number;
|
|
408
437
|
}
|
|
409
438
|
|
|
410
439
|
/**
|
|
@@ -420,7 +449,7 @@ export async function expandQuery(
|
|
|
420
449
|
|
|
421
450
|
// Build prompt
|
|
422
451
|
const template = getPromptTemplate(options.lang);
|
|
423
|
-
const prompt =
|
|
452
|
+
const prompt = buildPrompt(query, template, options.intent);
|
|
424
453
|
|
|
425
454
|
// Run with timeout (clear timer to avoid resource leak)
|
|
426
455
|
let timeoutId: ReturnType<typeof setTimeout> | undefined;
|
|
@@ -434,6 +463,7 @@ export async function expandQuery(
|
|
|
434
463
|
temperature: 0,
|
|
435
464
|
seed: 42,
|
|
436
465
|
maxTokens: 512,
|
|
466
|
+
contextSize: options.contextSize,
|
|
437
467
|
}),
|
|
438
468
|
timeoutPromise,
|
|
439
469
|
]);
|
|
@@ -486,7 +516,12 @@ export async function expandQueryCached(
|
|
|
486
516
|
options: ExpansionOptions = {}
|
|
487
517
|
): Promise<StoreResult<ExpansionResult | null>> {
|
|
488
518
|
const lang = options.lang ?? "auto";
|
|
489
|
-
const cacheKey = generateCacheKey(
|
|
519
|
+
const cacheKey = generateCacheKey(
|
|
520
|
+
deps.genPort.modelUri,
|
|
521
|
+
query,
|
|
522
|
+
lang,
|
|
523
|
+
options.intent
|
|
524
|
+
);
|
|
490
525
|
|
|
491
526
|
// Check cache
|
|
492
527
|
const cached = await deps.getCache(cacheKey);
|
package/src/pipeline/hybrid.ts
CHANGED
|
@@ -21,6 +21,7 @@ import type {
|
|
|
21
21
|
import { err, ok } from "../store/types";
|
|
22
22
|
import { createChunkLookup } from "./chunk-lookup";
|
|
23
23
|
import { formatQueryForEmbedding } from "./contextual";
|
|
24
|
+
import { matchesExcludedChunks, matchesExcludedText } from "./exclude";
|
|
24
25
|
import { expandQuery } from "./expansion";
|
|
25
26
|
import {
|
|
26
27
|
buildExplainResults,
|
|
@@ -35,6 +36,7 @@ import {
|
|
|
35
36
|
explainVector,
|
|
36
37
|
} from "./explain";
|
|
37
38
|
import { type RankedInput, rrfFuse, toRankedInput } from "./fusion";
|
|
39
|
+
import { selectBestChunkForSteering } from "./intent";
|
|
38
40
|
import { detectQueryLanguage } from "./query-language";
|
|
39
41
|
import {
|
|
40
42
|
buildExpansionFromQueryModes,
|
|
@@ -329,16 +331,18 @@ export async function searchHybrid(
|
|
|
329
331
|
}
|
|
330
332
|
|
|
331
333
|
if (expansionStatus !== "provided" && shouldExpand) {
|
|
332
|
-
const hasStrongSignal =
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
334
|
+
const hasStrongSignal = options.intent?.trim()
|
|
335
|
+
? false
|
|
336
|
+
: await checkBm25Strength(store, query, {
|
|
337
|
+
collection: options.collection,
|
|
338
|
+
lang: options.lang,
|
|
339
|
+
tagsAll: options.tagsAll,
|
|
340
|
+
tagsAny: options.tagsAny,
|
|
341
|
+
since: temporalRange.since,
|
|
342
|
+
until: temporalRange.until,
|
|
343
|
+
categories: options.categories,
|
|
344
|
+
author: options.author,
|
|
345
|
+
});
|
|
342
346
|
|
|
343
347
|
if (hasStrongSignal) {
|
|
344
348
|
expansionStatus = "skipped_strong";
|
|
@@ -349,6 +353,8 @@ export async function searchHybrid(
|
|
|
349
353
|
// Use queryLanguage for prompt selection, NOT options.lang (retrieval filter)
|
|
350
354
|
lang: queryLanguage,
|
|
351
355
|
timeout: pipelineConfig.expansionTimeout,
|
|
356
|
+
intent: options.intent,
|
|
357
|
+
contextSize: deps.config.models?.expandContextSize,
|
|
352
358
|
});
|
|
353
359
|
if (expandResult.ok) {
|
|
354
360
|
expansion = expandResult.value;
|
|
@@ -496,13 +502,16 @@ export async function searchHybrid(
|
|
|
496
502
|
// 4. Reranking
|
|
497
503
|
// ─────────────────────────────────────────────────────────────────────────
|
|
498
504
|
const rerankStartedAt = performance.now();
|
|
505
|
+
const candidateLimit =
|
|
506
|
+
options.candidateLimit ?? pipelineConfig.rerankCandidates;
|
|
499
507
|
const rerankResult = await rerankCandidates(
|
|
500
508
|
{ rerankPort: options.noRerank ? null : rerankPort, store },
|
|
501
509
|
query,
|
|
502
510
|
fusedCandidates,
|
|
503
511
|
{
|
|
504
|
-
maxCandidates:
|
|
512
|
+
maxCandidates: candidateLimit,
|
|
505
513
|
blendingSchedule: pipelineConfig.blendingSchedule,
|
|
514
|
+
intent: options.intent,
|
|
506
515
|
}
|
|
507
516
|
);
|
|
508
517
|
if (rerankResult.fallbackReason === "disabled") {
|
|
@@ -513,10 +522,7 @@ export async function searchHybrid(
|
|
|
513
522
|
timings.rerankMs = performance.now() - rerankStartedAt;
|
|
514
523
|
|
|
515
524
|
explainLines.push(
|
|
516
|
-
explainRerank(
|
|
517
|
-
!options.noRerank && rerankPort !== null,
|
|
518
|
-
pipelineConfig.rerankCandidates
|
|
519
|
-
)
|
|
525
|
+
explainRerank(!options.noRerank && rerankPort !== null, candidateLimit)
|
|
520
526
|
);
|
|
521
527
|
|
|
522
528
|
// ─────────────────────────────────────────────────────────────────────────
|
|
@@ -665,6 +671,25 @@ export async function searchHybrid(
|
|
|
665
671
|
continue;
|
|
666
672
|
}
|
|
667
673
|
|
|
674
|
+
const excluded =
|
|
675
|
+
matchesExcludedText(
|
|
676
|
+
[
|
|
677
|
+
doc.title ?? "",
|
|
678
|
+
doc.relPath,
|
|
679
|
+
doc.author ?? "",
|
|
680
|
+
doc.contentType ?? "",
|
|
681
|
+
...(doc.categories ?? []),
|
|
682
|
+
],
|
|
683
|
+
options.exclude
|
|
684
|
+
) ||
|
|
685
|
+
matchesExcludedChunks(
|
|
686
|
+
chunksMap.get(candidate.mirrorHash) ?? [],
|
|
687
|
+
options.exclude
|
|
688
|
+
);
|
|
689
|
+
if (excluded) {
|
|
690
|
+
continue;
|
|
691
|
+
}
|
|
692
|
+
|
|
668
693
|
// For --full mode, de-dupe by docid (keep best scoring candidate per doc)
|
|
669
694
|
if (options.full && seenDocids.has(doc.docid)) {
|
|
670
695
|
continue;
|
|
@@ -692,10 +717,23 @@ export async function searchHybrid(
|
|
|
692
717
|
const collectionPath = collectionPaths.get(doc.collection);
|
|
693
718
|
|
|
694
719
|
// For --full mode, fetch full mirror content
|
|
695
|
-
|
|
720
|
+
const snippetChunk =
|
|
721
|
+
options.full || !options.intent?.trim()
|
|
722
|
+
? chunk
|
|
723
|
+
: (selectBestChunkForSteering(
|
|
724
|
+
chunksMap.get(candidate.mirrorHash) ?? [],
|
|
725
|
+
query,
|
|
726
|
+
options.intent,
|
|
727
|
+
{
|
|
728
|
+
preferredSeq: chunk.seq,
|
|
729
|
+
intentWeight: 0.3,
|
|
730
|
+
}
|
|
731
|
+
) ?? chunk);
|
|
732
|
+
|
|
733
|
+
let snippet = snippetChunk.text;
|
|
696
734
|
let snippetRange: { startLine: number; endLine: number } | undefined = {
|
|
697
|
-
startLine:
|
|
698
|
-
endLine:
|
|
735
|
+
startLine: snippetChunk.startLine,
|
|
736
|
+
endLine: snippetChunk.endLine,
|
|
699
737
|
};
|
|
700
738
|
|
|
701
739
|
if (options.full) {
|
|
@@ -791,12 +829,15 @@ export async function searchHybrid(
|
|
|
791
829
|
reranked: rerankResult.reranked,
|
|
792
830
|
vectorsUsed: vectorAvailable,
|
|
793
831
|
totalResults: finalResults.length,
|
|
832
|
+
intent: options.intent,
|
|
833
|
+
exclude: options.exclude,
|
|
794
834
|
collection: options.collection,
|
|
795
835
|
lang: options.lang,
|
|
796
836
|
since: temporalRange.since,
|
|
797
837
|
until: temporalRange.until,
|
|
798
838
|
categories: options.categories,
|
|
799
839
|
author: options.author,
|
|
840
|
+
candidateLimit,
|
|
800
841
|
queryLanguage,
|
|
801
842
|
queryModes: queryModeSummary,
|
|
802
843
|
explain: explainData,
|