@gmickel/gno 0.5.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/skill/cli-reference.md +10 -0
- package/assets/skill/examples.md +13 -7
- package/assets/skill/mcp-reference.md +15 -3
- package/package.json +6 -5
- package/src/cli/commands/ask.ts +18 -10
- package/src/cli/program.ts +45 -2
- package/src/index.ts +19 -5
- package/src/mcp/tools/index.ts +3 -1
- package/src/mcp/tools/query.ts +28 -6
- package/src/pipeline/rerank.ts +95 -83
- package/vendor/fts5-snowball/LICENSE +67 -0
- package/vendor/fts5-snowball/README.md +38 -0
- package/vendor/fts5-snowball/darwin-arm64/fts5stemmer.dylib +0 -0
- package/vendor/fts5-snowball/darwin-x64/fts5stemmer.dylib +0 -0
- package/vendor/fts5-snowball/linux-x64/fts5stemmer.so +0 -0
- package/vendor/fts5-snowball/windows-x64/fts5stemmer.dll +0 -0
|
@@ -130,6 +130,14 @@ Hybrid search with expansion and reranking.
|
|
|
130
130
|
gno query <query> [options]
|
|
131
131
|
```
|
|
132
132
|
|
|
133
|
+
**Search modes** (pick one):
|
|
134
|
+
|
|
135
|
+
| Flag | Time | Description |
|
|
136
|
+
|------|------|-------------|
|
|
137
|
+
| `--fast` | ~0.7s | Skip expansion and reranking |
|
|
138
|
+
| (default) | ~2-3s | Skip expansion, with reranking |
|
|
139
|
+
| `--thorough` | ~5-8s | Full pipeline with expansion |
|
|
140
|
+
|
|
133
141
|
Additional options:
|
|
134
142
|
|
|
135
143
|
| Option | Description |
|
|
@@ -148,6 +156,8 @@ gno ask <question> [options]
|
|
|
148
156
|
|
|
149
157
|
| Option | Description |
|
|
150
158
|
|--------|-------------|
|
|
159
|
+
| `--fast` | Skip expansion and reranking (fastest) |
|
|
160
|
+
| `--thorough` | Enable query expansion (better recall) |
|
|
151
161
|
| `--answer` | Generate grounded answer |
|
|
152
162
|
| `--no-answer` | Retrieval only |
|
|
153
163
|
| `--max-answer-tokens <n>` | Cap answer length |
|
package/assets/skill/examples.md
CHANGED
|
@@ -193,13 +193,19 @@ gno models pull
|
|
|
193
193
|
|
|
194
194
|
## Tips
|
|
195
195
|
|
|
196
|
-
###
|
|
197
|
-
|
|
198
|
-
| Command |
|
|
199
|
-
|
|
200
|
-
| `gno search` |
|
|
201
|
-
| `gno vsearch` |
|
|
202
|
-
| `gno query` |
|
|
196
|
+
### Search Modes
|
|
197
|
+
|
|
198
|
+
| Command | Time | Use When |
|
|
199
|
+
|---------|------|----------|
|
|
200
|
+
| `gno search` | instant | Exact keyword matching |
|
|
201
|
+
| `gno vsearch` | ~0.5s | Finding similar concepts |
|
|
202
|
+
| `gno query --fast` | ~0.7s | Quick lookups |
|
|
203
|
+
| `gno query` | ~2-3s | Default, balanced |
|
|
204
|
+
| `gno query --thorough` | ~5-8s | Best recall, complex queries |
|
|
205
|
+
|
|
206
|
+
**Agent retry strategy**: Use default mode first. If no results:
|
|
207
|
+
1. Rephrase the query (free, often helps)
|
|
208
|
+
2. Try `--thorough` for better recall
|
|
203
209
|
|
|
204
210
|
### Output formats
|
|
205
211
|
|
|
@@ -60,12 +60,24 @@ Hybrid search (best quality).
|
|
|
60
60
|
{
|
|
61
61
|
"query": "search terms",
|
|
62
62
|
"collection": "optional-collection",
|
|
63
|
-
"limit": 5
|
|
64
|
-
"expand": true,
|
|
65
|
-
"rerank": true
|
|
63
|
+
"limit": 5
|
|
66
64
|
}
|
|
67
65
|
```
|
|
68
66
|
|
|
67
|
+
**Search modes** (via parameters):
|
|
68
|
+
|
|
69
|
+
| Mode | Parameters | Time |
|
|
70
|
+
|------|------------|------|
|
|
71
|
+
| Fast | `fast: true` | ~0.7s |
|
|
72
|
+
| Default | (none) | ~2-3s |
|
|
73
|
+
| Thorough | `thorough: true` | ~5-8s |
|
|
74
|
+
|
|
75
|
+
Default skips expansion, with reranking. Use `thorough: true` for best recall.
|
|
76
|
+
|
|
77
|
+
**Agent retry strategy**: Use default mode first. If no relevant results:
|
|
78
|
+
1. Rephrase the query (free, often effective)
|
|
79
|
+
2. Then try `thorough: true` for better recall
|
|
80
|
+
|
|
69
81
|
### gno.get
|
|
70
82
|
|
|
71
83
|
Retrieve document by reference.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@gmickel/gno",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.6.0",
|
|
4
4
|
"description": "Local semantic search for your documents. Index Markdown, PDF, and Office files with hybrid BM25 + vector search.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"search",
|
|
@@ -29,7 +29,8 @@
|
|
|
29
29
|
},
|
|
30
30
|
"files": [
|
|
31
31
|
"src",
|
|
32
|
-
"assets"
|
|
32
|
+
"assets",
|
|
33
|
+
"vendor"
|
|
33
34
|
],
|
|
34
35
|
"engines": {
|
|
35
36
|
"bun": ">=1.0.0"
|
|
@@ -98,7 +99,7 @@
|
|
|
98
99
|
"streamdown": "^1.6.10",
|
|
99
100
|
"tailwind-merge": "^3.4.0",
|
|
100
101
|
"use-stick-to-bottom": "^1.1.1",
|
|
101
|
-
"zod": "^4.
|
|
102
|
+
"zod": "^4.3.4"
|
|
102
103
|
},
|
|
103
104
|
"devDependencies": {
|
|
104
105
|
"@biomejs/biome": "2.3.10",
|
|
@@ -111,12 +112,12 @@
|
|
|
111
112
|
"docx": "^9.5.1",
|
|
112
113
|
"evalite": "^1.0.0-beta.15",
|
|
113
114
|
"exceljs": "^4.4.0",
|
|
114
|
-
"lefthook": "^2.0.
|
|
115
|
+
"lefthook": "^2.0.13",
|
|
115
116
|
"oxlint-tsgolint": "^0.10.0",
|
|
116
117
|
"pdf-lib": "^1.17.1",
|
|
117
118
|
"pptxgenjs": "^4.0.1",
|
|
118
119
|
"tailwindcss": "^4.1.18",
|
|
119
|
-
"ultracite": "
|
|
120
|
+
"ultracite": "6.5.0"
|
|
120
121
|
},
|
|
121
122
|
"peerDependencies": {
|
|
122
123
|
"typescript": "^5"
|
package/src/cli/commands/ask.ts
CHANGED
|
@@ -89,18 +89,24 @@ export async function ask(
|
|
|
89
89
|
embedPort = embedResult.value;
|
|
90
90
|
}
|
|
91
91
|
|
|
92
|
-
// Create generation port (for expansion and answer)
|
|
93
|
-
|
|
94
|
-
const
|
|
95
|
-
if (
|
|
96
|
-
|
|
92
|
+
// Create generation port (for expansion and/or answer)
|
|
93
|
+
// Need genPort if: expansion enabled (!noExpand) OR answer requested
|
|
94
|
+
const needsGen = !options.noExpand || options.answer;
|
|
95
|
+
if (needsGen) {
|
|
96
|
+
const genUri = options.genModel ?? preset.gen;
|
|
97
|
+
const genResult = await llm.createGenerationPort(genUri);
|
|
98
|
+
if (genResult.ok) {
|
|
99
|
+
genPort = genResult.value;
|
|
100
|
+
}
|
|
97
101
|
}
|
|
98
102
|
|
|
99
|
-
// Create rerank port
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
103
|
+
// Create rerank port (unless --fast or --no-rerank)
|
|
104
|
+
if (!options.noRerank) {
|
|
105
|
+
const rerankUri = options.rerankModel ?? preset.rerank;
|
|
106
|
+
const rerankResult = await llm.createRerankPort(rerankUri);
|
|
107
|
+
if (rerankResult.ok) {
|
|
108
|
+
rerankPort = rerankResult.value;
|
|
109
|
+
}
|
|
104
110
|
}
|
|
105
111
|
|
|
106
112
|
// Create vector index
|
|
@@ -147,6 +153,8 @@ export async function ask(
|
|
|
147
153
|
limit,
|
|
148
154
|
collection: options.collection,
|
|
149
155
|
lang: options.lang,
|
|
156
|
+
noExpand: options.noExpand,
|
|
157
|
+
noRerank: options.noRerank,
|
|
150
158
|
});
|
|
151
159
|
|
|
152
160
|
if (!searchResult.ok) {
|
package/src/cli/program.ts
CHANGED
|
@@ -312,6 +312,8 @@ function wireSearchCommands(program: Command): void {
|
|
|
312
312
|
.option('--lang <code>', 'language hint (BCP-47)')
|
|
313
313
|
.option('--full', 'include full content')
|
|
314
314
|
.option('--line-numbers', 'include line numbers in output')
|
|
315
|
+
.option('--fast', 'skip expansion and reranking (fastest, ~0.7s)')
|
|
316
|
+
.option('--thorough', 'enable query expansion (slower, ~5-8s)')
|
|
315
317
|
.option('--no-expand', 'disable query expansion')
|
|
316
318
|
.option('--no-rerank', 'disable reranking')
|
|
317
319
|
.option('--explain', 'include scoring explanation')
|
|
@@ -339,6 +341,30 @@ function wireSearchCommands(program: Command): void {
|
|
|
339
341
|
? parsePositiveInt('limit', cmdOpts.limit)
|
|
340
342
|
: getDefaultLimit(format);
|
|
341
343
|
|
|
344
|
+
// Determine expansion/rerank settings based on flags
|
|
345
|
+
// Priority: --fast > --thorough > --no-expand/--no-rerank > default
|
|
346
|
+
// Default: skip expansion (balanced mode ~2-3s)
|
|
347
|
+
let noExpand = true; // Default: skip expansion
|
|
348
|
+
let noRerank = false; // Default: with reranking
|
|
349
|
+
|
|
350
|
+
if (cmdOpts.fast) {
|
|
351
|
+
// --fast: skip both (~0.7s)
|
|
352
|
+
noExpand = true;
|
|
353
|
+
noRerank = true;
|
|
354
|
+
} else if (cmdOpts.thorough) {
|
|
355
|
+
// --thorough: full pipeline (~5-8s)
|
|
356
|
+
noExpand = false;
|
|
357
|
+
noRerank = false;
|
|
358
|
+
} else {
|
|
359
|
+
// Check individual flags (override defaults)
|
|
360
|
+
if (cmdOpts.expand === false) {
|
|
361
|
+
noExpand = true;
|
|
362
|
+
}
|
|
363
|
+
if (cmdOpts.rerank === false) {
|
|
364
|
+
noRerank = true;
|
|
365
|
+
}
|
|
366
|
+
}
|
|
367
|
+
|
|
342
368
|
const { query, formatQuery } = await import('./commands/query');
|
|
343
369
|
const result = await query(queryText, {
|
|
344
370
|
limit,
|
|
@@ -347,8 +373,8 @@ function wireSearchCommands(program: Command): void {
|
|
|
347
373
|
lang: cmdOpts.lang as string | undefined,
|
|
348
374
|
full: Boolean(cmdOpts.full),
|
|
349
375
|
lineNumbers: Boolean(cmdOpts.lineNumbers),
|
|
350
|
-
noExpand
|
|
351
|
-
noRerank
|
|
376
|
+
noExpand,
|
|
377
|
+
noRerank,
|
|
352
378
|
explain: Boolean(cmdOpts.explain),
|
|
353
379
|
json: format === 'json',
|
|
354
380
|
md: format === 'md',
|
|
@@ -376,6 +402,8 @@ function wireSearchCommands(program: Command): void {
|
|
|
376
402
|
.option('-n, --limit <num>', 'max source results')
|
|
377
403
|
.option('-c, --collection <name>', 'filter by collection')
|
|
378
404
|
.option('--lang <code>', 'language hint (BCP-47)')
|
|
405
|
+
.option('--fast', 'skip expansion and reranking (fastest)')
|
|
406
|
+
.option('--thorough', 'enable query expansion (slower)')
|
|
379
407
|
.option('--answer', 'generate short grounded answer')
|
|
380
408
|
.option('--no-answer', 'force retrieval-only output')
|
|
381
409
|
.option('--max-answer-tokens <num>', 'max answer tokens')
|
|
@@ -400,12 +428,27 @@ function wireSearchCommands(program: Command): void {
|
|
|
400
428
|
? parsePositiveInt('max-answer-tokens', cmdOpts.maxAnswerTokens)
|
|
401
429
|
: undefined;
|
|
402
430
|
|
|
431
|
+
// Determine expansion/rerank settings based on flags
|
|
432
|
+
// Default: skip expansion (balanced mode)
|
|
433
|
+
let noExpand = true;
|
|
434
|
+
let noRerank = false;
|
|
435
|
+
|
|
436
|
+
if (cmdOpts.fast) {
|
|
437
|
+
noExpand = true;
|
|
438
|
+
noRerank = true;
|
|
439
|
+
} else if (cmdOpts.thorough) {
|
|
440
|
+
noExpand = false;
|
|
441
|
+
noRerank = false;
|
|
442
|
+
}
|
|
443
|
+
|
|
403
444
|
const { ask, formatAsk } = await import('./commands/ask');
|
|
404
445
|
const showSources = Boolean(cmdOpts.showSources);
|
|
405
446
|
const result = await ask(queryText, {
|
|
406
447
|
limit,
|
|
407
448
|
collection: cmdOpts.collection as string | undefined,
|
|
408
449
|
lang: cmdOpts.lang as string | undefined,
|
|
450
|
+
noExpand,
|
|
451
|
+
noRerank,
|
|
409
452
|
// Per spec: --answer defaults to false, --no-answer forces retrieval-only
|
|
410
453
|
// Commander creates separate cmdOpts.noAnswer for --no-answer flag
|
|
411
454
|
answer: Boolean(cmdOpts.answer),
|
package/src/index.ts
CHANGED
|
@@ -7,21 +7,35 @@
|
|
|
7
7
|
*/
|
|
8
8
|
|
|
9
9
|
import { runCli } from './cli/run';
|
|
10
|
+
import { resetModelManager } from './llm/nodeLlamaCpp/lifecycle';
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* Cleanup models and exit.
|
|
14
|
+
* Without this, llama.cpp native threads can keep the process alive.
|
|
15
|
+
*/
|
|
16
|
+
async function cleanupAndExit(code: number): Promise<never> {
|
|
17
|
+
await resetModelManager().catch(() => {
|
|
18
|
+
// Ignore cleanup errors on exit
|
|
19
|
+
});
|
|
20
|
+
process.exit(code);
|
|
21
|
+
}
|
|
10
22
|
|
|
11
23
|
// SIGINT handler for graceful shutdown
|
|
12
24
|
process.on('SIGINT', () => {
|
|
13
25
|
process.stderr.write('\nInterrupted\n');
|
|
14
|
-
|
|
26
|
+
cleanupAndExit(130).catch(() => {
|
|
27
|
+
// Ignore cleanup errors on exit
|
|
28
|
+
});
|
|
15
29
|
});
|
|
16
30
|
|
|
17
31
|
// Run CLI and exit
|
|
18
32
|
runCli(process.argv)
|
|
19
|
-
.then((code) =>
|
|
20
|
-
process.exit(code);
|
|
21
|
-
})
|
|
33
|
+
.then((code) => cleanupAndExit(code))
|
|
22
34
|
.catch((err) => {
|
|
23
35
|
process.stderr.write(
|
|
24
36
|
`Fatal error: ${err instanceof Error ? err.message : String(err)}\n`
|
|
25
37
|
);
|
|
26
|
-
|
|
38
|
+
cleanupAndExit(1).catch(() => {
|
|
39
|
+
// Ignore cleanup errors on exit
|
|
40
|
+
});
|
|
27
41
|
});
|
package/src/mcp/tools/index.ts
CHANGED
|
@@ -40,7 +40,9 @@ const queryInputSchema = z.object({
|
|
|
40
40
|
limit: z.number().int().min(1).max(100).default(5),
|
|
41
41
|
minScore: z.number().min(0).max(1).optional(),
|
|
42
42
|
lang: z.string().optional(),
|
|
43
|
-
|
|
43
|
+
fast: z.boolean().default(false),
|
|
44
|
+
thorough: z.boolean().default(false),
|
|
45
|
+
expand: z.boolean().default(false), // Default: skip expansion
|
|
44
46
|
rerank: z.boolean().default(true),
|
|
45
47
|
});
|
|
46
48
|
|
package/src/mcp/tools/query.ts
CHANGED
|
@@ -28,6 +28,8 @@ interface QueryInput {
|
|
|
28
28
|
limit?: number;
|
|
29
29
|
minScore?: number;
|
|
30
30
|
lang?: string;
|
|
31
|
+
fast?: boolean;
|
|
32
|
+
thorough?: boolean;
|
|
31
33
|
expand?: boolean;
|
|
32
34
|
rerank?: boolean;
|
|
33
35
|
}
|
|
@@ -138,9 +140,30 @@ export function handleQuery(
|
|
|
138
140
|
embedPort = embedResult.value;
|
|
139
141
|
}
|
|
140
142
|
|
|
143
|
+
// Determine noExpand/noRerank based on mode flags
|
|
144
|
+
// Priority: fast > thorough > expand/rerank params > defaults
|
|
145
|
+
// Default: noExpand=true (skip expansion), noRerank=false (with reranking)
|
|
146
|
+
let noExpand = true;
|
|
147
|
+
let noRerank = false;
|
|
148
|
+
|
|
149
|
+
if (args.fast) {
|
|
150
|
+
noExpand = true;
|
|
151
|
+
noRerank = true;
|
|
152
|
+
} else if (args.thorough) {
|
|
153
|
+
noExpand = false;
|
|
154
|
+
noRerank = false;
|
|
155
|
+
} else {
|
|
156
|
+
// Use explicit expand/rerank params if provided
|
|
157
|
+
if (args.expand === true) {
|
|
158
|
+
noExpand = false;
|
|
159
|
+
}
|
|
160
|
+
if (args.rerank === false) {
|
|
161
|
+
noRerank = true;
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
|
|
141
165
|
// Create generation port (for expansion) - optional
|
|
142
|
-
|
|
143
|
-
if (args.expand !== false) {
|
|
166
|
+
if (!noExpand) {
|
|
144
167
|
const genResult = await llm.createGenerationPort(preset.gen);
|
|
145
168
|
if (genResult.ok) {
|
|
146
169
|
genPort = genResult.value;
|
|
@@ -148,8 +171,7 @@ export function handleQuery(
|
|
|
148
171
|
}
|
|
149
172
|
|
|
150
173
|
// Create rerank port - optional
|
|
151
|
-
|
|
152
|
-
if (args.rerank !== false) {
|
|
174
|
+
if (!noRerank) {
|
|
153
175
|
const rerankResult = await llm.createRerankPort(preset.rerank);
|
|
154
176
|
if (rerankResult.ok) {
|
|
155
177
|
rerankPort = rerankResult.value;
|
|
@@ -189,8 +211,8 @@ export function handleQuery(
|
|
|
189
211
|
minScore: args.minScore,
|
|
190
212
|
collection: args.collection,
|
|
191
213
|
queryLanguageHint: args.lang, // Affects expansion prompt, not retrieval
|
|
192
|
-
noExpand
|
|
193
|
-
noRerank
|
|
214
|
+
noExpand,
|
|
215
|
+
noRerank,
|
|
194
216
|
});
|
|
195
217
|
|
|
196
218
|
if (!result.ok) {
|
package/src/pipeline/rerank.ts
CHANGED
|
@@ -66,6 +66,75 @@ function blend(
|
|
|
66
66
|
return fusionWeight * fusionScore + rerankWeight * rerankScore;
|
|
67
67
|
}
|
|
68
68
|
|
|
69
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
70
|
+
// Chunk Text Extraction
|
|
71
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
72
|
+
|
|
73
|
+
const MAX_CHUNK_CHARS = 4000;
|
|
74
|
+
|
|
75
|
+
interface BestChunkInfo {
|
|
76
|
+
candidate: FusionCandidate;
|
|
77
|
+
seq: number;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* Extract best chunk per document for efficient reranking.
|
|
82
|
+
*/
|
|
83
|
+
function selectBestChunks(
|
|
84
|
+
toRerank: FusionCandidate[]
|
|
85
|
+
): Map<string, BestChunkInfo> {
|
|
86
|
+
const bestChunkPerDoc = new Map<string, BestChunkInfo>();
|
|
87
|
+
for (const c of toRerank) {
|
|
88
|
+
const existing = bestChunkPerDoc.get(c.mirrorHash);
|
|
89
|
+
if (!existing || c.fusionScore > existing.candidate.fusionScore) {
|
|
90
|
+
bestChunkPerDoc.set(c.mirrorHash, { candidate: c, seq: c.seq });
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
return bestChunkPerDoc;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/**
|
|
97
|
+
* Fetch chunk texts for reranking.
|
|
98
|
+
*/
|
|
99
|
+
async function fetchChunkTexts(
|
|
100
|
+
store: StorePort,
|
|
101
|
+
bestChunkPerDoc: Map<string, BestChunkInfo>
|
|
102
|
+
): Promise<{ texts: string[]; hashToIndex: Map<string, number> }> {
|
|
103
|
+
const uniqueHashes = [...bestChunkPerDoc.keys()];
|
|
104
|
+
const chunkResults = await Promise.all(
|
|
105
|
+
uniqueHashes.map((hash) => store.getChunks(hash))
|
|
106
|
+
);
|
|
107
|
+
|
|
108
|
+
const chunkTexts = new Map<string, string>();
|
|
109
|
+
for (let i = 0; i < uniqueHashes.length; i++) {
|
|
110
|
+
const hash = uniqueHashes[i] as string;
|
|
111
|
+
const result = chunkResults[i];
|
|
112
|
+
const bestInfo = bestChunkPerDoc.get(hash);
|
|
113
|
+
|
|
114
|
+
if (result?.ok && result.value && bestInfo) {
|
|
115
|
+
const chunk = result.value.find((c) => c.seq === bestInfo.seq);
|
|
116
|
+
const text = chunk?.text ?? '';
|
|
117
|
+
chunkTexts.set(
|
|
118
|
+
hash,
|
|
119
|
+
text.length > MAX_CHUNK_CHARS
|
|
120
|
+
? `${text.slice(0, MAX_CHUNK_CHARS)}...`
|
|
121
|
+
: text
|
|
122
|
+
);
|
|
123
|
+
} else {
|
|
124
|
+
chunkTexts.set(hash, '');
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
const hashToIndex = new Map<string, number>();
|
|
129
|
+
const texts: string[] = [];
|
|
130
|
+
for (const hash of uniqueHashes) {
|
|
131
|
+
hashToIndex.set(hash, texts.length);
|
|
132
|
+
texts.push(chunkTexts.get(hash) ?? '');
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
return { texts, hashToIndex };
|
|
136
|
+
}
|
|
137
|
+
|
|
69
138
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
70
139
|
// Rerank Implementation
|
|
71
140
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
@@ -80,7 +149,6 @@ export async function rerankCandidates(
|
|
|
80
149
|
candidates: FusionCandidate[],
|
|
81
150
|
options: RerankOptions = {}
|
|
82
151
|
): Promise<RerankResult> {
|
|
83
|
-
// Early return for empty candidates
|
|
84
152
|
if (candidates.length === 0) {
|
|
85
153
|
return { candidates: [], reranked: false };
|
|
86
154
|
}
|
|
@@ -90,21 +158,20 @@ export async function rerankCandidates(
|
|
|
90
158
|
const schedule = options.blendingSchedule ?? DEFAULT_BLENDING_SCHEDULE;
|
|
91
159
|
|
|
92
160
|
// Normalize fusion scores to 0-1 range across ALL candidates for stability.
|
|
93
|
-
// This ensures blendedScore is always in [0,1] regardless of reranker availability.
|
|
94
161
|
const fusionScoresAll = candidates.map((c) => c.fusionScore);
|
|
95
162
|
const minFusionAll = Math.min(...fusionScoresAll);
|
|
96
163
|
const maxFusionAll = Math.max(...fusionScoresAll);
|
|
97
164
|
const fusionRangeAll = maxFusionAll - minFusionAll;
|
|
98
165
|
|
|
99
|
-
|
|
166
|
+
const normalizeFusionScore = (score: number): number => {
|
|
100
167
|
if (fusionRangeAll < 1e-9) {
|
|
101
|
-
return 1;
|
|
168
|
+
return 1;
|
|
102
169
|
}
|
|
103
170
|
const v = (score - minFusionAll) / fusionRangeAll;
|
|
104
171
|
return Math.max(0, Math.min(1, v));
|
|
105
|
-
}
|
|
172
|
+
};
|
|
106
173
|
|
|
107
|
-
//
|
|
174
|
+
// No reranker: return candidates with normalized fusion scores
|
|
108
175
|
if (!rerankPort) {
|
|
109
176
|
return {
|
|
110
177
|
candidates: candidates.map((c) => ({
|
|
@@ -116,52 +183,17 @@ export async function rerankCandidates(
|
|
|
116
183
|
};
|
|
117
184
|
}
|
|
118
185
|
|
|
119
|
-
// Limit candidates for reranking
|
|
120
186
|
const toRerank = candidates.slice(0, maxCandidates);
|
|
121
187
|
const remaining = candidates.slice(maxCandidates);
|
|
122
188
|
|
|
123
|
-
//
|
|
124
|
-
const
|
|
125
|
-
|
|
126
|
-
// Fetch full document content for each unique document (parallel)
|
|
127
|
-
// Max 128K chars per doc to fit in reranker context
|
|
128
|
-
const MAX_DOC_CHARS = 128_000;
|
|
129
|
-
const contentResults = await Promise.all(
|
|
130
|
-
uniqueHashes.map((hash) => store.getContent(hash))
|
|
131
|
-
);
|
|
132
|
-
const docContents = new Map<string, string>();
|
|
133
|
-
for (let i = 0; i < uniqueHashes.length; i++) {
|
|
134
|
-
const hash = uniqueHashes[i] as string;
|
|
135
|
-
const result = contentResults[i] as Awaited<
|
|
136
|
-
ReturnType<typeof store.getContent>
|
|
137
|
-
>;
|
|
138
|
-
if (result.ok && result.value) {
|
|
139
|
-
const content = result.value;
|
|
140
|
-
docContents.set(
|
|
141
|
-
hash,
|
|
142
|
-
content.length > MAX_DOC_CHARS
|
|
143
|
-
? `${content.slice(0, MAX_DOC_CHARS)}...`
|
|
144
|
-
: content
|
|
145
|
-
);
|
|
146
|
-
} else {
|
|
147
|
-
// Fallback to empty string if content not available
|
|
148
|
-
docContents.set(hash, '');
|
|
149
|
-
}
|
|
150
|
-
}
|
|
151
|
-
|
|
152
|
-
// Build texts array for reranking (one per unique document)
|
|
153
|
-
const hashToIndex = new Map<string, number>();
|
|
154
|
-
const texts: string[] = [];
|
|
155
|
-
for (const hash of uniqueHashes) {
|
|
156
|
-
hashToIndex.set(hash, texts.length);
|
|
157
|
-
texts.push(docContents.get(hash) ?? '');
|
|
158
|
-
}
|
|
189
|
+
// Extract best chunk per document for efficient reranking
|
|
190
|
+
const bestChunkPerDoc = selectBestChunks(toRerank);
|
|
191
|
+
const { texts, hashToIndex } = await fetchChunkTexts(store, bestChunkPerDoc);
|
|
159
192
|
|
|
160
|
-
// Run reranking on full
|
|
193
|
+
// Run reranking on best chunks (much faster than full docs)
|
|
161
194
|
const rerankResult = await rerankPort.rerank(query, texts);
|
|
162
195
|
|
|
163
196
|
if (!rerankResult.ok) {
|
|
164
|
-
// Graceful degradation - return normalized fusion scores
|
|
165
197
|
return {
|
|
166
198
|
candidates: candidates.map((c) => ({
|
|
167
199
|
...c,
|
|
@@ -172,37 +204,29 @@ export async function rerankCandidates(
|
|
|
172
204
|
};
|
|
173
205
|
}
|
|
174
206
|
|
|
175
|
-
//
|
|
176
|
-
// Note: We use normalizeFusionScore defined above (across ALL candidates)
|
|
177
|
-
// Build doc index->score map for O(1) lookup
|
|
178
|
-
// All chunks from same document share the same rerank score
|
|
207
|
+
// Normalize rerank scores using min-max
|
|
179
208
|
const scoreByDocIndex = new Map(
|
|
180
209
|
rerankResult.value.map((s) => [s.index, s.score])
|
|
181
210
|
);
|
|
182
|
-
|
|
183
|
-
// Normalize rerank scores using min-max (models return varying scales)
|
|
184
211
|
const rerankScores = rerankResult.value.map((s) => s.score);
|
|
185
212
|
const minRerank = Math.min(...rerankScores);
|
|
186
213
|
const maxRerank = Math.max(...rerankScores);
|
|
187
214
|
const rerankRange = maxRerank - minRerank;
|
|
188
215
|
|
|
189
|
-
|
|
216
|
+
const normalizeRerankScore = (score: number): number => {
|
|
190
217
|
if (rerankRange < 1e-9) {
|
|
191
|
-
return 1;
|
|
218
|
+
return 1;
|
|
192
219
|
}
|
|
193
220
|
return (score - minRerank) / rerankRange;
|
|
194
|
-
}
|
|
221
|
+
};
|
|
195
222
|
|
|
223
|
+
// Build reranked candidates with blended scores
|
|
196
224
|
const rerankedCandidates: RerankedCandidate[] = toRerank.map((c, i) => {
|
|
197
|
-
// Get document-level rerank score (shared by all chunks from same doc)
|
|
198
225
|
const docIndex = hashToIndex.get(c.mirrorHash) ?? -1;
|
|
199
226
|
const rerankScore = scoreByDocIndex.get(docIndex) ?? null;
|
|
200
|
-
|
|
201
|
-
// Normalize rerank score to 0-1 range using min-max
|
|
202
227
|
const normalizedRerankScore =
|
|
203
228
|
rerankScore !== null ? normalizeRerankScore(rerankScore) : null;
|
|
204
229
|
|
|
205
|
-
// Calculate blended score using normalized fusion score
|
|
206
230
|
const position = i + 1;
|
|
207
231
|
const normalizedFusion = normalizeFusionScore(c.fusionScore);
|
|
208
232
|
const blendedScore =
|
|
@@ -210,42 +234,30 @@ export async function rerankCandidates(
|
|
|
210
234
|
? blend(normalizedFusion, normalizedRerankScore, position, schedule)
|
|
211
235
|
: normalizedFusion;
|
|
212
236
|
|
|
213
|
-
return {
|
|
214
|
-
...c,
|
|
215
|
-
rerankScore: normalizedRerankScore,
|
|
216
|
-
blendedScore,
|
|
217
|
-
};
|
|
237
|
+
return { ...c, rerankScore: normalizedRerankScore, blendedScore };
|
|
218
238
|
});
|
|
219
239
|
|
|
220
|
-
// Add remaining candidates
|
|
221
|
-
// These get normalized fusion scores with penalty but clamped to [0,1]
|
|
240
|
+
// Add remaining candidates with penalty
|
|
222
241
|
const allCandidates: RerankedCandidate[] = [
|
|
223
242
|
...rerankedCandidates,
|
|
224
|
-
...remaining.map((c) => {
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
}),
|
|
243
|
+
...remaining.map((c) => ({
|
|
244
|
+
...c,
|
|
245
|
+
rerankScore: null,
|
|
246
|
+
blendedScore: Math.max(
|
|
247
|
+
0,
|
|
248
|
+
Math.min(1, normalizeFusionScore(c.fusionScore) * 0.5)
|
|
249
|
+
),
|
|
250
|
+
})),
|
|
233
251
|
];
|
|
234
252
|
|
|
235
|
-
// Sort by blended score
|
|
253
|
+
// Sort by blended score with deterministic tie-breaking
|
|
236
254
|
allCandidates.sort((a, b) => {
|
|
237
255
|
const scoreDiff = b.blendedScore - a.blendedScore;
|
|
238
256
|
if (Math.abs(scoreDiff) > 1e-9) {
|
|
239
257
|
return scoreDiff;
|
|
240
258
|
}
|
|
241
|
-
|
|
242
|
-
const aKey = `${a.mirrorHash}:${a.seq}`;
|
|
243
|
-
const bKey = `${b.mirrorHash}:${b.seq}`;
|
|
244
|
-
return aKey.localeCompare(bKey);
|
|
259
|
+
return `${a.mirrorHash}:${a.seq}`.localeCompare(`${b.mirrorHash}:${b.seq}`);
|
|
245
260
|
});
|
|
246
261
|
|
|
247
|
-
return {
|
|
248
|
-
candidates: allCandidates,
|
|
249
|
-
reranked: true,
|
|
250
|
-
};
|
|
262
|
+
return { candidates: allCandidates, reranked: true };
|
|
251
263
|
}
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
fts5-snowball - Snowball tokenizer for SQLite FTS5
|
|
2
|
+
==================================================
|
|
3
|
+
|
|
4
|
+
Copyright (c) 2016 Abilio Marques
|
|
5
|
+
https://github.com/abiliojr/fts5-snowball
|
|
6
|
+
|
|
7
|
+
BSD 3-Clause License
|
|
8
|
+
|
|
9
|
+
Redistribution and use in source and binary forms, with or without
|
|
10
|
+
modification, are permitted provided that the following conditions are met:
|
|
11
|
+
|
|
12
|
+
1. Redistributions of source code must retain the above copyright notice, this
|
|
13
|
+
list of conditions and the following disclaimer.
|
|
14
|
+
|
|
15
|
+
2. Redistributions in binary form must reproduce the above copyright notice,
|
|
16
|
+
this list of conditions and the following disclaimer in the documentation
|
|
17
|
+
and/or other materials provided with the distribution.
|
|
18
|
+
|
|
19
|
+
3. Neither the name of the copyright holder nor the names of its contributors
|
|
20
|
+
may be used to endorse or promote products derived from this software
|
|
21
|
+
without specific prior written permission.
|
|
22
|
+
|
|
23
|
+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
|
24
|
+
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
25
|
+
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
|
26
|
+
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
|
27
|
+
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
|
28
|
+
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
|
29
|
+
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
|
30
|
+
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
|
31
|
+
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
32
|
+
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
33
|
+
|
|
34
|
+
================================================================================
|
|
35
|
+
|
|
36
|
+
Snowball Stemmer Library
|
|
37
|
+
========================
|
|
38
|
+
|
|
39
|
+
Copyright (c) 2001-2025, Dr Martin Porter and Richard Boulton
|
|
40
|
+
https://github.com/snowballstem/snowball
|
|
41
|
+
|
|
42
|
+
BSD 3-Clause License
|
|
43
|
+
|
|
44
|
+
Redistribution and use in source and binary forms, with or without
|
|
45
|
+
modification, are permitted provided that the following conditions are met:
|
|
46
|
+
|
|
47
|
+
1. Redistributions of source code must retain the above copyright notice, this
|
|
48
|
+
list of conditions and the following disclaimer.
|
|
49
|
+
|
|
50
|
+
2. Redistributions in binary form must reproduce the above copyright notice,
|
|
51
|
+
this list of conditions and the following disclaimer in the documentation
|
|
52
|
+
and/or other materials provided with the distribution.
|
|
53
|
+
|
|
54
|
+
3. Neither the name of the Snowball project nor the names of its contributors
|
|
55
|
+
may be used to endorse or promote products derived from this software
|
|
56
|
+
without specific prior written permission.
|
|
57
|
+
|
|
58
|
+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
|
59
|
+
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
60
|
+
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
|
61
|
+
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
|
62
|
+
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
|
63
|
+
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
|
64
|
+
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
|
65
|
+
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
|
66
|
+
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
67
|
+
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
# fts5-snowball Vendored Binaries
|
|
2
|
+
|
|
3
|
+
Prebuilt [fts5-snowball](https://github.com/abiliojr/fts5-snowball) SQLite extension for multilingual FTS5 stemming.
|
|
4
|
+
|
|
5
|
+
## Supported Platforms
|
|
6
|
+
|
|
7
|
+
| Platform | File | Architecture |
|
|
8
|
+
|----------|------|--------------|
|
|
9
|
+
| Linux | `linux-x64/fts5stemmer.so` | x86_64 |
|
|
10
|
+
| macOS | `darwin-arm64/fts5stemmer.dylib` | ARM64 (Apple Silicon) |
|
|
11
|
+
| macOS | `darwin-x64/fts5stemmer.dylib` | x86_64 (Intel) |
|
|
12
|
+
| Windows | `windows-x64/fts5stemmer.dll` | x86_64 |
|
|
13
|
+
|
|
14
|
+
## Build Provenance
|
|
15
|
+
|
|
16
|
+
Built via GitHub Actions: `.github/workflows/build-fts5-snowball.yml`
|
|
17
|
+
|
|
18
|
+
Source: https://github.com/abiliojr/fts5-snowball (commit from main branch)
|
|
19
|
+
|
|
20
|
+
## Supported Languages
|
|
21
|
+
|
|
22
|
+
The Snowball stemmer supports: Arabic, Basque, Catalan, Danish, Dutch, English, Finnish, French, German, Greek, Hindi, Hungarian, Indonesian, Irish, Italian, Lithuanian, Nepali, Norwegian, Porter, Portuguese, Romanian, Russian, Serbian, Spanish, Swedish, Tamil, Turkish, Yiddish.
|
|
23
|
+
|
|
24
|
+
## Usage
|
|
25
|
+
|
|
26
|
+
```typescript
|
|
27
|
+
import { Database } from 'bun:sqlite';
|
|
28
|
+
|
|
29
|
+
// Load extension
|
|
30
|
+
db.loadExtension('vendor/fts5-snowball/darwin-arm64/fts5stemmer.dylib');
|
|
31
|
+
|
|
32
|
+
// Create FTS table with snowball tokenizer
|
|
33
|
+
db.exec(`CREATE VIRTUAL TABLE docs USING fts5(content, tokenize='snowball english')`);
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
## License
|
|
37
|
+
|
|
38
|
+
BSD-3-Clause. See LICENSE file.
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|