@gmickel/gno 0.5.0 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -130,6 +130,14 @@ Hybrid search with expansion and reranking.
130
130
  gno query <query> [options]
131
131
  ```
132
132
 
133
+ **Search modes** (pick one):
134
+
135
+ | Flag | Time | Description |
136
+ |------|------|-------------|
137
+ | `--fast` | ~0.7s | Skip expansion and reranking |
138
+ | (default) | ~2-3s | Skip expansion, with reranking |
139
+ | `--thorough` | ~5-8s | Full pipeline with expansion |
140
+
133
141
  Additional options:
134
142
 
135
143
  | Option | Description |
@@ -148,6 +156,8 @@ gno ask <question> [options]
148
156
 
149
157
  | Option | Description |
150
158
  |--------|-------------|
159
+ | `--fast` | Skip expansion and reranking (fastest) |
160
+ | `--thorough` | Enable query expansion (better recall) |
151
161
  | `--answer` | Generate grounded answer |
152
162
  | `--no-answer` | Retrieval only |
153
163
  | `--max-answer-tokens <n>` | Cap answer length |
@@ -193,13 +193,19 @@ gno models pull
193
193
 
194
194
  ## Tips
195
195
 
196
- ### Speed vs Quality
197
-
198
- | Command | Speed | Quality | Use When |
199
- |---------|-------|---------|----------|
200
- | `gno search` | Fast | Good for keywords | Exact phrase matching |
201
- | `gno vsearch` | Medium | Good for concepts | Finding similar meaning |
202
- | `gno query` | Slower | Best | Important queries |
196
+ ### Search Modes
197
+
198
+ | Command | Time | Use When |
199
+ |---------|------|----------|
200
+ | `gno search` | instant | Exact keyword matching |
201
+ | `gno vsearch` | ~0.5s | Finding similar concepts |
202
+ | `gno query --fast` | ~0.7s | Quick lookups |
203
+ | `gno query` | ~2-3s | Default, balanced |
204
+ | `gno query --thorough` | ~5-8s | Best recall, complex queries |
205
+
206
+ **Agent retry strategy**: Use default mode first. If no results:
207
+ 1. Rephrase the query (free, often helps)
208
+ 2. Try `--thorough` for better recall
203
209
 
204
210
  ### Output formats
205
211
 
@@ -60,12 +60,24 @@ Hybrid search (best quality).
60
60
  {
61
61
  "query": "search terms",
62
62
  "collection": "optional-collection",
63
- "limit": 5,
64
- "expand": true,
65
- "rerank": true
63
+ "limit": 5
66
64
  }
67
65
  ```
68
66
 
67
+ **Search modes** (via parameters):
68
+
69
+ | Mode | Parameters | Time |
70
+ |------|------------|------|
71
+ | Fast | `fast: true` | ~0.7s |
72
+ | Default | (none) | ~2-3s |
73
+ | Thorough | `thorough: true` | ~5-8s |
74
+
75
+ Default skips expansion, with reranking. Use `thorough: true` for best recall.
76
+
77
+ **Agent retry strategy**: Use default mode first. If no relevant results:
78
+ 1. Rephrase the query (free, often effective)
79
+ 2. Then try `thorough: true` for better recall
80
+
69
81
  ### gno.get
70
82
 
71
83
  Retrieve document by reference.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@gmickel/gno",
3
- "version": "0.5.0",
3
+ "version": "0.6.0",
4
4
  "description": "Local semantic search for your documents. Index Markdown, PDF, and Office files with hybrid BM25 + vector search.",
5
5
  "keywords": [
6
6
  "search",
@@ -29,7 +29,8 @@
29
29
  },
30
30
  "files": [
31
31
  "src",
32
- "assets"
32
+ "assets",
33
+ "vendor"
33
34
  ],
34
35
  "engines": {
35
36
  "bun": ">=1.0.0"
@@ -98,7 +99,7 @@
98
99
  "streamdown": "^1.6.10",
99
100
  "tailwind-merge": "^3.4.0",
100
101
  "use-stick-to-bottom": "^1.1.1",
101
- "zod": "^4.2.1"
102
+ "zod": "^4.3.4"
102
103
  },
103
104
  "devDependencies": {
104
105
  "@biomejs/biome": "2.3.10",
@@ -111,12 +112,12 @@
111
112
  "docx": "^9.5.1",
112
113
  "evalite": "^1.0.0-beta.15",
113
114
  "exceljs": "^4.4.0",
114
- "lefthook": "^2.0.12",
115
+ "lefthook": "^2.0.13",
115
116
  "oxlint-tsgolint": "^0.10.0",
116
117
  "pdf-lib": "^1.17.1",
117
118
  "pptxgenjs": "^4.0.1",
118
119
  "tailwindcss": "^4.1.18",
119
- "ultracite": "^6.5.0"
120
+ "ultracite": "6.5.0"
120
121
  },
121
122
  "peerDependencies": {
122
123
  "typescript": "^5"
@@ -89,18 +89,24 @@ export async function ask(
89
89
  embedPort = embedResult.value;
90
90
  }
91
91
 
92
- // Create generation port (for expansion and answer)
93
- const genUri = options.genModel ?? preset.gen;
94
- const genResult = await llm.createGenerationPort(genUri);
95
- if (genResult.ok) {
96
- genPort = genResult.value;
92
+ // Create generation port (for expansion and/or answer)
93
+ // Need genPort if: expansion enabled (!noExpand) OR answer requested
94
+ const needsGen = !options.noExpand || options.answer;
95
+ if (needsGen) {
96
+ const genUri = options.genModel ?? preset.gen;
97
+ const genResult = await llm.createGenerationPort(genUri);
98
+ if (genResult.ok) {
99
+ genPort = genResult.value;
100
+ }
97
101
  }
98
102
 
99
- // Create rerank port
100
- const rerankUri = options.rerankModel ?? preset.rerank;
101
- const rerankResult = await llm.createRerankPort(rerankUri);
102
- if (rerankResult.ok) {
103
- rerankPort = rerankResult.value;
103
+ // Create rerank port (unless --fast or --no-rerank)
104
+ if (!options.noRerank) {
105
+ const rerankUri = options.rerankModel ?? preset.rerank;
106
+ const rerankResult = await llm.createRerankPort(rerankUri);
107
+ if (rerankResult.ok) {
108
+ rerankPort = rerankResult.value;
109
+ }
104
110
  }
105
111
 
106
112
  // Create vector index
@@ -147,6 +153,8 @@ export async function ask(
147
153
  limit,
148
154
  collection: options.collection,
149
155
  lang: options.lang,
156
+ noExpand: options.noExpand,
157
+ noRerank: options.noRerank,
150
158
  });
151
159
 
152
160
  if (!searchResult.ok) {
@@ -312,6 +312,8 @@ function wireSearchCommands(program: Command): void {
312
312
  .option('--lang <code>', 'language hint (BCP-47)')
313
313
  .option('--full', 'include full content')
314
314
  .option('--line-numbers', 'include line numbers in output')
315
+ .option('--fast', 'skip expansion and reranking (fastest, ~0.7s)')
316
+ .option('--thorough', 'enable query expansion (slower, ~5-8s)')
315
317
  .option('--no-expand', 'disable query expansion')
316
318
  .option('--no-rerank', 'disable reranking')
317
319
  .option('--explain', 'include scoring explanation')
@@ -339,6 +341,30 @@ function wireSearchCommands(program: Command): void {
339
341
  ? parsePositiveInt('limit', cmdOpts.limit)
340
342
  : getDefaultLimit(format);
341
343
 
344
+ // Determine expansion/rerank settings based on flags
345
+ // Priority: --fast > --thorough > --no-expand/--no-rerank > default
346
+ // Default: skip expansion (balanced mode ~2-3s)
347
+ let noExpand = true; // Default: skip expansion
348
+ let noRerank = false; // Default: with reranking
349
+
350
+ if (cmdOpts.fast) {
351
+ // --fast: skip both (~0.7s)
352
+ noExpand = true;
353
+ noRerank = true;
354
+ } else if (cmdOpts.thorough) {
355
+ // --thorough: full pipeline (~5-8s)
356
+ noExpand = false;
357
+ noRerank = false;
358
+ } else {
359
+ // Check individual flags (override defaults)
360
+ if (cmdOpts.expand === false) {
361
+ noExpand = true;
362
+ }
363
+ if (cmdOpts.rerank === false) {
364
+ noRerank = true;
365
+ }
366
+ }
367
+
342
368
  const { query, formatQuery } = await import('./commands/query');
343
369
  const result = await query(queryText, {
344
370
  limit,
@@ -347,8 +373,8 @@ function wireSearchCommands(program: Command): void {
347
373
  lang: cmdOpts.lang as string | undefined,
348
374
  full: Boolean(cmdOpts.full),
349
375
  lineNumbers: Boolean(cmdOpts.lineNumbers),
350
- noExpand: cmdOpts.expand === false,
351
- noRerank: cmdOpts.rerank === false,
376
+ noExpand,
377
+ noRerank,
352
378
  explain: Boolean(cmdOpts.explain),
353
379
  json: format === 'json',
354
380
  md: format === 'md',
@@ -376,6 +402,8 @@ function wireSearchCommands(program: Command): void {
376
402
  .option('-n, --limit <num>', 'max source results')
377
403
  .option('-c, --collection <name>', 'filter by collection')
378
404
  .option('--lang <code>', 'language hint (BCP-47)')
405
+ .option('--fast', 'skip expansion and reranking (fastest)')
406
+ .option('--thorough', 'enable query expansion (slower)')
379
407
  .option('--answer', 'generate short grounded answer')
380
408
  .option('--no-answer', 'force retrieval-only output')
381
409
  .option('--max-answer-tokens <num>', 'max answer tokens')
@@ -400,12 +428,27 @@ function wireSearchCommands(program: Command): void {
400
428
  ? parsePositiveInt('max-answer-tokens', cmdOpts.maxAnswerTokens)
401
429
  : undefined;
402
430
 
431
+ // Determine expansion/rerank settings based on flags
432
+ // Default: skip expansion (balanced mode)
433
+ let noExpand = true;
434
+ let noRerank = false;
435
+
436
+ if (cmdOpts.fast) {
437
+ noExpand = true;
438
+ noRerank = true;
439
+ } else if (cmdOpts.thorough) {
440
+ noExpand = false;
441
+ noRerank = false;
442
+ }
443
+
403
444
  const { ask, formatAsk } = await import('./commands/ask');
404
445
  const showSources = Boolean(cmdOpts.showSources);
405
446
  const result = await ask(queryText, {
406
447
  limit,
407
448
  collection: cmdOpts.collection as string | undefined,
408
449
  lang: cmdOpts.lang as string | undefined,
450
+ noExpand,
451
+ noRerank,
409
452
  // Per spec: --answer defaults to false, --no-answer forces retrieval-only
410
453
  // Commander creates separate cmdOpts.noAnswer for --no-answer flag
411
454
  answer: Boolean(cmdOpts.answer),
package/src/index.ts CHANGED
@@ -7,21 +7,35 @@
7
7
  */
8
8
 
9
9
  import { runCli } from './cli/run';
10
+ import { resetModelManager } from './llm/nodeLlamaCpp/lifecycle';
11
+
12
+ /**
13
+ * Cleanup models and exit.
14
+ * Without this, llama.cpp native threads can keep the process alive.
15
+ */
16
+ async function cleanupAndExit(code: number): Promise<never> {
17
+ await resetModelManager().catch(() => {
18
+ // Ignore cleanup errors on exit
19
+ });
20
+ process.exit(code);
21
+ }
10
22
 
11
23
  // SIGINT handler for graceful shutdown
12
24
  process.on('SIGINT', () => {
13
25
  process.stderr.write('\nInterrupted\n');
14
- process.exit(130);
26
+ cleanupAndExit(130).catch(() => {
27
+ // Ignore cleanup errors on exit
28
+ });
15
29
  });
16
30
 
17
31
  // Run CLI and exit
18
32
  runCli(process.argv)
19
- .then((code) => {
20
- process.exit(code);
21
- })
33
+ .then((code) => cleanupAndExit(code))
22
34
  .catch((err) => {
23
35
  process.stderr.write(
24
36
  `Fatal error: ${err instanceof Error ? err.message : String(err)}\n`
25
37
  );
26
- process.exit(1);
38
+ cleanupAndExit(1).catch(() => {
39
+ // Ignore cleanup errors on exit
40
+ });
27
41
  });
@@ -40,7 +40,9 @@ const queryInputSchema = z.object({
40
40
  limit: z.number().int().min(1).max(100).default(5),
41
41
  minScore: z.number().min(0).max(1).optional(),
42
42
  lang: z.string().optional(),
43
- expand: z.boolean().default(true),
43
+ fast: z.boolean().default(false),
44
+ thorough: z.boolean().default(false),
45
+ expand: z.boolean().default(false), // Default: skip expansion
44
46
  rerank: z.boolean().default(true),
45
47
  });
46
48
 
@@ -28,6 +28,8 @@ interface QueryInput {
28
28
  limit?: number;
29
29
  minScore?: number;
30
30
  lang?: string;
31
+ fast?: boolean;
32
+ thorough?: boolean;
31
33
  expand?: boolean;
32
34
  rerank?: boolean;
33
35
  }
@@ -138,9 +140,30 @@ export function handleQuery(
138
140
  embedPort = embedResult.value;
139
141
  }
140
142
 
143
+ // Determine noExpand/noRerank based on mode flags
144
+ // Priority: fast > thorough > expand/rerank params > defaults
145
+ // Default: noExpand=true (skip expansion), noRerank=false (with reranking)
146
+ let noExpand = true;
147
+ let noRerank = false;
148
+
149
+ if (args.fast) {
150
+ noExpand = true;
151
+ noRerank = true;
152
+ } else if (args.thorough) {
153
+ noExpand = false;
154
+ noRerank = false;
155
+ } else {
156
+ // Use explicit expand/rerank params if provided
157
+ if (args.expand === true) {
158
+ noExpand = false;
159
+ }
160
+ if (args.rerank === false) {
161
+ noRerank = true;
162
+ }
163
+ }
164
+
141
165
  // Create generation port (for expansion) - optional
142
- // expand defaults to true per spec
143
- if (args.expand !== false) {
166
+ if (!noExpand) {
144
167
  const genResult = await llm.createGenerationPort(preset.gen);
145
168
  if (genResult.ok) {
146
169
  genPort = genResult.value;
@@ -148,8 +171,7 @@ export function handleQuery(
148
171
  }
149
172
 
150
173
  // Create rerank port - optional
151
- // rerank defaults to true per spec
152
- if (args.rerank !== false) {
174
+ if (!noRerank) {
153
175
  const rerankResult = await llm.createRerankPort(preset.rerank);
154
176
  if (rerankResult.ok) {
155
177
  rerankPort = rerankResult.value;
@@ -189,8 +211,8 @@ export function handleQuery(
189
211
  minScore: args.minScore,
190
212
  collection: args.collection,
191
213
  queryLanguageHint: args.lang, // Affects expansion prompt, not retrieval
192
- noExpand: args.expand === false,
193
- noRerank: args.rerank === false,
214
+ noExpand,
215
+ noRerank,
194
216
  });
195
217
 
196
218
  if (!result.ok) {
@@ -66,6 +66,75 @@ function blend(
66
66
  return fusionWeight * fusionScore + rerankWeight * rerankScore;
67
67
  }
68
68
 
69
+ // ─────────────────────────────────────────────────────────────────────────────
70
+ // Chunk Text Extraction
71
+ // ─────────────────────────────────────────────────────────────────────────────
72
+
73
+ const MAX_CHUNK_CHARS = 4000;
74
+
75
+ interface BestChunkInfo {
76
+ candidate: FusionCandidate;
77
+ seq: number;
78
+ }
79
+
80
+ /**
81
+ * Extract best chunk per document for efficient reranking.
82
+ */
83
+ function selectBestChunks(
84
+ toRerank: FusionCandidate[]
85
+ ): Map<string, BestChunkInfo> {
86
+ const bestChunkPerDoc = new Map<string, BestChunkInfo>();
87
+ for (const c of toRerank) {
88
+ const existing = bestChunkPerDoc.get(c.mirrorHash);
89
+ if (!existing || c.fusionScore > existing.candidate.fusionScore) {
90
+ bestChunkPerDoc.set(c.mirrorHash, { candidate: c, seq: c.seq });
91
+ }
92
+ }
93
+ return bestChunkPerDoc;
94
+ }
95
+
96
+ /**
97
+ * Fetch chunk texts for reranking.
98
+ */
99
+ async function fetchChunkTexts(
100
+ store: StorePort,
101
+ bestChunkPerDoc: Map<string, BestChunkInfo>
102
+ ): Promise<{ texts: string[]; hashToIndex: Map<string, number> }> {
103
+ const uniqueHashes = [...bestChunkPerDoc.keys()];
104
+ const chunkResults = await Promise.all(
105
+ uniqueHashes.map((hash) => store.getChunks(hash))
106
+ );
107
+
108
+ const chunkTexts = new Map<string, string>();
109
+ for (let i = 0; i < uniqueHashes.length; i++) {
110
+ const hash = uniqueHashes[i] as string;
111
+ const result = chunkResults[i];
112
+ const bestInfo = bestChunkPerDoc.get(hash);
113
+
114
+ if (result?.ok && result.value && bestInfo) {
115
+ const chunk = result.value.find((c) => c.seq === bestInfo.seq);
116
+ const text = chunk?.text ?? '';
117
+ chunkTexts.set(
118
+ hash,
119
+ text.length > MAX_CHUNK_CHARS
120
+ ? `${text.slice(0, MAX_CHUNK_CHARS)}...`
121
+ : text
122
+ );
123
+ } else {
124
+ chunkTexts.set(hash, '');
125
+ }
126
+ }
127
+
128
+ const hashToIndex = new Map<string, number>();
129
+ const texts: string[] = [];
130
+ for (const hash of uniqueHashes) {
131
+ hashToIndex.set(hash, texts.length);
132
+ texts.push(chunkTexts.get(hash) ?? '');
133
+ }
134
+
135
+ return { texts, hashToIndex };
136
+ }
137
+
69
138
  // ─────────────────────────────────────────────────────────────────────────────
70
139
  // Rerank Implementation
71
140
  // ─────────────────────────────────────────────────────────────────────────────
@@ -80,7 +149,6 @@ export async function rerankCandidates(
80
149
  candidates: FusionCandidate[],
81
150
  options: RerankOptions = {}
82
151
  ): Promise<RerankResult> {
83
- // Early return for empty candidates
84
152
  if (candidates.length === 0) {
85
153
  return { candidates: [], reranked: false };
86
154
  }
@@ -90,21 +158,20 @@ export async function rerankCandidates(
90
158
  const schedule = options.blendingSchedule ?? DEFAULT_BLENDING_SCHEDULE;
91
159
 
92
160
  // Normalize fusion scores to 0-1 range across ALL candidates for stability.
93
- // This ensures blendedScore is always in [0,1] regardless of reranker availability.
94
161
  const fusionScoresAll = candidates.map((c) => c.fusionScore);
95
162
  const minFusionAll = Math.min(...fusionScoresAll);
96
163
  const maxFusionAll = Math.max(...fusionScoresAll);
97
164
  const fusionRangeAll = maxFusionAll - minFusionAll;
98
165
 
99
- function normalizeFusionScore(score: number): number {
166
+ const normalizeFusionScore = (score: number): number => {
100
167
  if (fusionRangeAll < 1e-9) {
101
- return 1; // tie for best
168
+ return 1;
102
169
  }
103
170
  const v = (score - minFusionAll) / fusionRangeAll;
104
171
  return Math.max(0, Math.min(1, v));
105
- }
172
+ };
106
173
 
107
- // If no reranker, return candidates with normalized fusion scores
174
+ // No reranker: return candidates with normalized fusion scores
108
175
  if (!rerankPort) {
109
176
  return {
110
177
  candidates: candidates.map((c) => ({
@@ -116,52 +183,17 @@ export async function rerankCandidates(
116
183
  };
117
184
  }
118
185
 
119
- // Limit candidates for reranking
120
186
  const toRerank = candidates.slice(0, maxCandidates);
121
187
  const remaining = candidates.slice(maxCandidates);
122
188
 
123
- // Dedupe by document - multiple chunks from same doc use single full-doc rerank
124
- const uniqueHashes = [...new Set(toRerank.map((c) => c.mirrorHash))];
125
-
126
- // Fetch full document content for each unique document (parallel)
127
- // Max 128K chars per doc to fit in reranker context
128
- const MAX_DOC_CHARS = 128_000;
129
- const contentResults = await Promise.all(
130
- uniqueHashes.map((hash) => store.getContent(hash))
131
- );
132
- const docContents = new Map<string, string>();
133
- for (let i = 0; i < uniqueHashes.length; i++) {
134
- const hash = uniqueHashes[i] as string;
135
- const result = contentResults[i] as Awaited<
136
- ReturnType<typeof store.getContent>
137
- >;
138
- if (result.ok && result.value) {
139
- const content = result.value;
140
- docContents.set(
141
- hash,
142
- content.length > MAX_DOC_CHARS
143
- ? `${content.slice(0, MAX_DOC_CHARS)}...`
144
- : content
145
- );
146
- } else {
147
- // Fallback to empty string if content not available
148
- docContents.set(hash, '');
149
- }
150
- }
151
-
152
- // Build texts array for reranking (one per unique document)
153
- const hashToIndex = new Map<string, number>();
154
- const texts: string[] = [];
155
- for (const hash of uniqueHashes) {
156
- hashToIndex.set(hash, texts.length);
157
- texts.push(docContents.get(hash) ?? '');
158
- }
189
+ // Extract best chunk per document for efficient reranking
190
+ const bestChunkPerDoc = selectBestChunks(toRerank);
191
+ const { texts, hashToIndex } = await fetchChunkTexts(store, bestChunkPerDoc);
159
192
 
160
- // Run reranking on full documents
193
+ // Run reranking on best chunks (much faster than full docs)
161
194
  const rerankResult = await rerankPort.rerank(query, texts);
162
195
 
163
196
  if (!rerankResult.ok) {
164
- // Graceful degradation - return normalized fusion scores
165
197
  return {
166
198
  candidates: candidates.map((c) => ({
167
199
  ...c,
@@ -172,37 +204,29 @@ export async function rerankCandidates(
172
204
  };
173
205
  }
174
206
 
175
- // Map rerank scores to candidates
176
- // Note: We use normalizeFusionScore defined above (across ALL candidates)
177
- // Build doc index->score map for O(1) lookup
178
- // All chunks from same document share the same rerank score
207
+ // Normalize rerank scores using min-max
179
208
  const scoreByDocIndex = new Map(
180
209
  rerankResult.value.map((s) => [s.index, s.score])
181
210
  );
182
-
183
- // Normalize rerank scores using min-max (models return varying scales)
184
211
  const rerankScores = rerankResult.value.map((s) => s.score);
185
212
  const minRerank = Math.min(...rerankScores);
186
213
  const maxRerank = Math.max(...rerankScores);
187
214
  const rerankRange = maxRerank - minRerank;
188
215
 
189
- function normalizeRerankScore(score: number): number {
216
+ const normalizeRerankScore = (score: number): number => {
190
217
  if (rerankRange < 1e-9) {
191
- return 1; // All tied for best
218
+ return 1;
192
219
  }
193
220
  return (score - minRerank) / rerankRange;
194
- }
221
+ };
195
222
 
223
+ // Build reranked candidates with blended scores
196
224
  const rerankedCandidates: RerankedCandidate[] = toRerank.map((c, i) => {
197
- // Get document-level rerank score (shared by all chunks from same doc)
198
225
  const docIndex = hashToIndex.get(c.mirrorHash) ?? -1;
199
226
  const rerankScore = scoreByDocIndex.get(docIndex) ?? null;
200
-
201
- // Normalize rerank score to 0-1 range using min-max
202
227
  const normalizedRerankScore =
203
228
  rerankScore !== null ? normalizeRerankScore(rerankScore) : null;
204
229
 
205
- // Calculate blended score using normalized fusion score
206
230
  const position = i + 1;
207
231
  const normalizedFusion = normalizeFusionScore(c.fusionScore);
208
232
  const blendedScore =
@@ -210,42 +234,30 @@ export async function rerankCandidates(
210
234
  ? blend(normalizedFusion, normalizedRerankScore, position, schedule)
211
235
  : normalizedFusion;
212
236
 
213
- return {
214
- ...c,
215
- rerankScore: normalizedRerankScore,
216
- blendedScore,
217
- };
237
+ return { ...c, rerankScore: normalizedRerankScore, blendedScore };
218
238
  });
219
239
 
220
- // Add remaining candidates (not reranked)
221
- // These get normalized fusion scores with penalty but clamped to [0,1]
240
+ // Add remaining candidates with penalty
222
241
  const allCandidates: RerankedCandidate[] = [
223
242
  ...rerankedCandidates,
224
- ...remaining.map((c) => {
225
- const base = normalizeFusionScore(c.fusionScore);
226
- return {
227
- ...c,
228
- rerankScore: null,
229
- // Apply 0.5x penalty and clamp to [0,1]
230
- blendedScore: Math.max(0, Math.min(1, base * 0.5)),
231
- };
232
- }),
243
+ ...remaining.map((c) => ({
244
+ ...c,
245
+ rerankScore: null,
246
+ blendedScore: Math.max(
247
+ 0,
248
+ Math.min(1, normalizeFusionScore(c.fusionScore) * 0.5)
249
+ ),
250
+ })),
233
251
  ];
234
252
 
235
- // Sort by blended score
253
+ // Sort by blended score with deterministic tie-breaking
236
254
  allCandidates.sort((a, b) => {
237
255
  const scoreDiff = b.blendedScore - a.blendedScore;
238
256
  if (Math.abs(scoreDiff) > 1e-9) {
239
257
  return scoreDiff;
240
258
  }
241
- // Deterministic tie-breaking
242
- const aKey = `${a.mirrorHash}:${a.seq}`;
243
- const bKey = `${b.mirrorHash}:${b.seq}`;
244
- return aKey.localeCompare(bKey);
259
+ return `${a.mirrorHash}:${a.seq}`.localeCompare(`${b.mirrorHash}:${b.seq}`);
245
260
  });
246
261
 
247
- return {
248
- candidates: allCandidates,
249
- reranked: true,
250
- };
262
+ return { candidates: allCandidates, reranked: true };
251
263
  }
@@ -0,0 +1,67 @@
1
+ fts5-snowball - Snowball tokenizer for SQLite FTS5
2
+ ==================================================
3
+
4
+ Copyright (c) 2016 Abilio Marques
5
+ https://github.com/abiliojr/fts5-snowball
6
+
7
+ BSD 3-Clause License
8
+
9
+ Redistribution and use in source and binary forms, with or without
10
+ modification, are permitted provided that the following conditions are met:
11
+
12
+ 1. Redistributions of source code must retain the above copyright notice, this
13
+ list of conditions and the following disclaimer.
14
+
15
+ 2. Redistributions in binary form must reproduce the above copyright notice,
16
+ this list of conditions and the following disclaimer in the documentation
17
+ and/or other materials provided with the distribution.
18
+
19
+ 3. Neither the name of the copyright holder nor the names of its contributors
20
+ may be used to endorse or promote products derived from this software
21
+ without specific prior written permission.
22
+
23
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
24
+ AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
25
+ IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
26
+ DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
27
+ FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
28
+ DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
29
+ SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
30
+ CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
31
+ OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
32
+ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
33
+
34
+ ================================================================================
35
+
36
+ Snowball Stemmer Library
37
+ ========================
38
+
39
+ Copyright (c) 2001-2025, Dr Martin Porter and Richard Boulton
40
+ https://github.com/snowballstem/snowball
41
+
42
+ BSD 3-Clause License
43
+
44
+ Redistribution and use in source and binary forms, with or without
45
+ modification, are permitted provided that the following conditions are met:
46
+
47
+ 1. Redistributions of source code must retain the above copyright notice, this
48
+ list of conditions and the following disclaimer.
49
+
50
+ 2. Redistributions in binary form must reproduce the above copyright notice,
51
+ this list of conditions and the following disclaimer in the documentation
52
+ and/or other materials provided with the distribution.
53
+
54
+ 3. Neither the name of the Snowball project nor the names of its contributors
55
+ may be used to endorse or promote products derived from this software
56
+ without specific prior written permission.
57
+
58
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
59
+ AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
60
+ IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
61
+ DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
62
+ FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
63
+ DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
64
+ SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
65
+ CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
66
+ OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
67
+ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
@@ -0,0 +1,38 @@
1
+ # fts5-snowball Vendored Binaries
2
+
3
+ Prebuilt [fts5-snowball](https://github.com/abiliojr/fts5-snowball) SQLite extension for multilingual FTS5 stemming.
4
+
5
+ ## Supported Platforms
6
+
7
+ | Platform | File | Architecture |
8
+ |----------|------|--------------|
9
+ | Linux | `linux-x64/fts5stemmer.so` | x86_64 |
10
+ | macOS | `darwin-arm64/fts5stemmer.dylib` | ARM64 (Apple Silicon) |
11
+ | macOS | `darwin-x64/fts5stemmer.dylib` | x86_64 (Intel) |
12
+ | Windows | `windows-x64/fts5stemmer.dll` | x86_64 |
13
+
14
+ ## Build Provenance
15
+
16
+ Built via GitHub Actions: `.github/workflows/build-fts5-snowball.yml`
17
+
18
+ Source: https://github.com/abiliojr/fts5-snowball (commit from main branch)
19
+
20
+ ## Supported Languages
21
+
22
+ The Snowball stemmer supports: Arabic, Basque, Catalan, Danish, Dutch, English, Finnish, French, German, Greek, Hindi, Hungarian, Indonesian, Irish, Italian, Lithuanian, Nepali, Norwegian, Porter, Portuguese, Romanian, Russian, Serbian, Spanish, Swedish, Tamil, Turkish, Yiddish.
23
+
24
+ ## Usage
25
+
26
+ ```typescript
27
+ import { Database } from 'bun:sqlite';
28
+
29
+ // Load extension
30
+ db.loadExtension('vendor/fts5-snowball/darwin-arm64/fts5stemmer.dylib');
31
+
32
+ // Create FTS table with snowball tokenizer
33
+ db.exec(`CREATE VIRTUAL TABLE docs USING fts5(content, tokenize='snowball english')`);
34
+ ```
35
+
36
+ ## License
37
+
38
+ BSD-3-Clause. See LICENSE file.