@tobilu/qmd 1.1.2 → 1.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +46 -0
- package/README.md +75 -0
- package/dist/collections.d.ts +16 -2
- package/dist/collections.js +57 -8
- package/dist/formatter.d.ts +1 -0
- package/dist/formatter.js +4 -4
- package/dist/index.d.ts +129 -0
- package/dist/index.js +95 -0
- package/dist/llm.d.ts +1 -0
- package/dist/llm.js +4 -1
- package/dist/mcp.js +7 -2
- package/dist/qmd.js +42 -26
- package/dist/store.d.ts +18 -6
- package/dist/store.js +78 -19
- package/package.json +9 -1
package/CHANGELOG.md
CHANGED
|
@@ -2,6 +2,52 @@
|
|
|
2
2
|
|
|
3
3
|
## [Unreleased]
|
|
4
4
|
|
|
5
|
+
## [1.1.6] - 2026-03-09
|
|
6
|
+
|
|
7
|
+
QMD can now be used as a library. `import { createStore } from '@tobilu/qmd'`
|
|
8
|
+
gives you the full search and indexing API — hybrid query, BM25, structured
|
|
9
|
+
search, collection/context management — without shelling out to the CLI.
|
|
10
|
+
|
|
11
|
+
### Changes
|
|
12
|
+
|
|
13
|
+
- **SDK / library mode**: `createStore({ dbPath, config })` returns a
|
|
14
|
+
`QMDStore` with `query()`, `search()`, `structuredSearch()`, `get()`,
|
|
15
|
+
`multiGet()`, and collection/context management methods. Supports inline
|
|
16
|
+
config (no files needed) or a YAML config path.
|
|
17
|
+
- **Package exports**: `package.json` now declares `main`, `types`, and
|
|
18
|
+
`exports` so bundlers and TypeScript resolve `@tobilu/qmd` correctly.
|
|
19
|
+
|
|
20
|
+
## [1.1.5] - 2026-03-07
|
|
21
|
+
|
|
22
|
+
Ambiguous queries like "performance" now produce dramatically better results
|
|
23
|
+
when the caller knows what they mean. The new `intent` parameter steers all
|
|
24
|
+
five pipeline stages — expansion, strong-signal bypass, chunk selection,
|
|
25
|
+
reranking, and snippet extraction — without searching on its own. Design and
|
|
26
|
+
original implementation by Ilya Grigorik (@vyalamar) in #180.
|
|
27
|
+
|
|
28
|
+
### Changes
|
|
29
|
+
|
|
30
|
+
- **Intent parameter**: optional `intent` string disambiguates queries across
|
|
31
|
+
the entire search pipeline. Available via CLI (`--intent` flag or `intent:`
|
|
32
|
+
line in query documents), MCP (`intent` field on the query tool), and
|
|
33
|
+
programmatic API. Adapted from PR #180 (thanks @vyalamar).
|
|
34
|
+
- **Query expansion**: when intent is provided, the expansion LLM prompt
|
|
35
|
+
includes `Query intent: {intent}`, matching the finetune training data
|
|
36
|
+
format for better-aligned expansions.
|
|
37
|
+
- **Reranking**: intent is prepended to the rerank query so Qwen3-Reranker
|
|
38
|
+
scores with domain context.
|
|
39
|
+
- **Chunk selection**: intent terms scored at 0.5× weight alongside query
|
|
40
|
+
terms (1.0×) when selecting the best chunk per document for reranking.
|
|
41
|
+
- **Snippet extraction**: intent terms scored at 0.3× weight to nudge
|
|
42
|
+
snippets toward intent-relevant lines without overriding query anchoring.
|
|
43
|
+
- **Strong-signal bypass disabled with intent**: when intent is provided, the
|
|
44
|
+
BM25 strong-signal shortcut is skipped — the obvious keyword match may not
|
|
45
|
+
be what the caller wants.
|
|
46
|
+
- **MCP instructions**: callers are now guided to provide `intent` on every
|
|
47
|
+
search call for disambiguation.
|
|
48
|
+
- **Query document syntax**: `intent:` recognized as a line type. At most one
|
|
49
|
+
per document, cannot appear alone. Grammar updated in `docs/SYNTAX.md`.
|
|
50
|
+
|
|
5
51
|
## [1.1.2] - 2026-03-07
|
|
6
52
|
|
|
7
53
|
13 community PRs merged. GPU initialization replaced with node-llama-cpp's
|
package/README.md
CHANGED
|
@@ -137,6 +137,81 @@ LLM models stay loaded in VRAM across requests. Embedding/reranking contexts are
|
|
|
137
137
|
|
|
138
138
|
Point any MCP client at `http://localhost:8181/mcp` to connect.
|
|
139
139
|
|
|
140
|
+
### SDK / Library Usage
|
|
141
|
+
|
|
142
|
+
Use QMD as a library in your own Node.js or Bun applications:
|
|
143
|
+
|
|
144
|
+
```sh
|
|
145
|
+
npm install @tobilu/qmd
|
|
146
|
+
```
|
|
147
|
+
|
|
148
|
+
```typescript
|
|
149
|
+
import { createStore } from '@tobilu/qmd'
|
|
150
|
+
|
|
151
|
+
// Create a store with inline config (no config file needed)
|
|
152
|
+
const store = createStore({
|
|
153
|
+
dbPath: './my-index.sqlite',
|
|
154
|
+
config: {
|
|
155
|
+
collections: {
|
|
156
|
+
docs: { path: '/path/to/docs', pattern: '**/*.md' },
|
|
157
|
+
notes: { path: '/path/to/notes', pattern: '**/*.md' },
|
|
158
|
+
},
|
|
159
|
+
},
|
|
160
|
+
})
|
|
161
|
+
|
|
162
|
+
// Or reference a YAML config file
|
|
163
|
+
const store2 = createStore({
|
|
164
|
+
dbPath: './my-index.sqlite',
|
|
165
|
+
configPath: './qmd.yml',
|
|
166
|
+
})
|
|
167
|
+
```
|
|
168
|
+
|
|
169
|
+
**Search & retrieval:**
|
|
170
|
+
|
|
171
|
+
```typescript
|
|
172
|
+
// Hybrid search: BM25 + vector + query expansion + LLM reranking (best quality)
|
|
173
|
+
const results = await store.query("authentication flow", { limit: 5 })
|
|
174
|
+
|
|
175
|
+
// Fast BM25 keyword search (no LLM, synchronous)
|
|
176
|
+
const keywords = store.search("auth middleware", { limit: 10 })
|
|
177
|
+
|
|
178
|
+
// Structured search with pre-expanded queries (for LLM callers)
|
|
179
|
+
const structured = await store.structuredSearch([
|
|
180
|
+
{ type: 'lex', query: 'authentication' },
|
|
181
|
+
{ type: 'vec', query: 'how users log in' },
|
|
182
|
+
], { limit: 5 })
|
|
183
|
+
|
|
184
|
+
// Get a document by path or docid
|
|
185
|
+
const doc = store.get("docs/readme.md")
|
|
186
|
+
const byId = store.get("#abc123")
|
|
187
|
+
|
|
188
|
+
// Get multiple documents by glob
|
|
189
|
+
const { docs, errors } = store.multiGet("docs/**/*.md")
|
|
190
|
+
```
|
|
191
|
+
|
|
192
|
+
**Collection & context management:**
|
|
193
|
+
|
|
194
|
+
```typescript
|
|
195
|
+
// Add a collection
|
|
196
|
+
store.addCollection("myapp", { path: "/src/myapp", pattern: "**/*.ts" })
|
|
197
|
+
|
|
198
|
+
// Add context (improves search relevance)
|
|
199
|
+
store.addContext("myapp", "/auth", "Authentication and session management")
|
|
200
|
+
store.setGlobalContext("Internal engineering documentation")
|
|
201
|
+
|
|
202
|
+
// List everything
|
|
203
|
+
store.listCollections()
|
|
204
|
+
store.listContexts()
|
|
205
|
+
```
|
|
206
|
+
|
|
207
|
+
**Lifecycle:**
|
|
208
|
+
|
|
209
|
+
```typescript
|
|
210
|
+
store.close()
|
|
211
|
+
```
|
|
212
|
+
|
|
213
|
+
The SDK requires explicit `dbPath` and config — no defaults are assumed. This makes it safe to embed in any application without side effects.
|
|
214
|
+
|
|
140
215
|
## Architecture
|
|
141
216
|
|
|
142
217
|
```
|
package/dist/collections.d.ts
CHANGED
|
@@ -34,18 +34,32 @@ export interface CollectionConfig {
|
|
|
34
34
|
export interface NamedCollection extends Collection {
|
|
35
35
|
name: string;
|
|
36
36
|
}
|
|
37
|
+
/**
|
|
38
|
+
* Set the config source for SDK mode.
|
|
39
|
+
* - File path: load/save from a specific YAML file
|
|
40
|
+
* - Inline config: use an in-memory CollectionConfig (saveConfig updates in place, no file I/O)
|
|
41
|
+
* - undefined: reset to default file-based config
|
|
42
|
+
*/
|
|
43
|
+
export declare function setConfigSource(source?: {
|
|
44
|
+
configPath?: string;
|
|
45
|
+
config?: CollectionConfig;
|
|
46
|
+
}): void;
|
|
37
47
|
/**
|
|
38
48
|
* Set the current index name for config file lookup
|
|
39
49
|
* Config file will be ~/.config/qmd/{indexName}.yml
|
|
40
50
|
*/
|
|
41
51
|
export declare function setConfigIndexName(name: string): void;
|
|
42
52
|
/**
|
|
43
|
-
* Load configuration from
|
|
53
|
+
* Load configuration from the configured source.
|
|
54
|
+
* - Inline config: returns the in-memory object directly
|
|
55
|
+
* - File-based: reads from YAML file (default ~/.config/qmd/index.yml)
|
|
44
56
|
* Returns empty config if file doesn't exist
|
|
45
57
|
*/
|
|
46
58
|
export declare function loadConfig(): CollectionConfig;
|
|
47
59
|
/**
|
|
48
|
-
* Save configuration to
|
|
60
|
+
* Save configuration to the configured source.
|
|
61
|
+
* - Inline config: updates the in-memory object (no file I/O)
|
|
62
|
+
* - File-based: writes to YAML file (default ~/.config/qmd/index.yml)
|
|
49
63
|
*/
|
|
50
64
|
export declare function saveConfig(config: CollectionConfig): void;
|
|
51
65
|
/**
|
package/dist/collections.js
CHANGED
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
* Collections define which directories to index and their associated contexts.
|
|
6
6
|
*/
|
|
7
7
|
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs";
|
|
8
|
-
import { join } from "path";
|
|
8
|
+
import { join, dirname } from "path";
|
|
9
9
|
import { homedir } from "os";
|
|
10
10
|
import YAML from "yaml";
|
|
11
11
|
// ============================================================================
|
|
@@ -13,6 +13,33 @@ import YAML from "yaml";
|
|
|
13
13
|
// ============================================================================
|
|
14
14
|
// Current index name (default: "index")
|
|
15
15
|
let currentIndexName = "index";
|
|
16
|
+
// SDK mode: optional in-memory config or custom config path
|
|
17
|
+
let configSource = { type: 'file' };
|
|
18
|
+
/**
|
|
19
|
+
* Set the config source for SDK mode.
|
|
20
|
+
* - File path: load/save from a specific YAML file
|
|
21
|
+
* - Inline config: use an in-memory CollectionConfig (saveConfig updates in place, no file I/O)
|
|
22
|
+
* - undefined: reset to default file-based config
|
|
23
|
+
*/
|
|
24
|
+
export function setConfigSource(source) {
|
|
25
|
+
if (!source) {
|
|
26
|
+
configSource = { type: 'file' };
|
|
27
|
+
return;
|
|
28
|
+
}
|
|
29
|
+
if (source.config) {
|
|
30
|
+
// Ensure collections object exists
|
|
31
|
+
if (!source.config.collections) {
|
|
32
|
+
source.config.collections = {};
|
|
33
|
+
}
|
|
34
|
+
configSource = { type: 'inline', config: source.config };
|
|
35
|
+
}
|
|
36
|
+
else if (source.configPath) {
|
|
37
|
+
configSource = { type: 'file', path: source.configPath };
|
|
38
|
+
}
|
|
39
|
+
else {
|
|
40
|
+
configSource = { type: 'file' };
|
|
41
|
+
}
|
|
42
|
+
}
|
|
16
43
|
/**
|
|
17
44
|
* Set the current index name for config file lookup
|
|
18
45
|
* Config file will be ~/.config/qmd/{indexName}.yml
|
|
@@ -57,11 +84,18 @@ function ensureConfigDir() {
|
|
|
57
84
|
// Core functions
|
|
58
85
|
// ============================================================================
|
|
59
86
|
/**
|
|
60
|
-
* Load configuration from
|
|
87
|
+
* Load configuration from the configured source.
|
|
88
|
+
* - Inline config: returns the in-memory object directly
|
|
89
|
+
* - File-based: reads from YAML file (default ~/.config/qmd/index.yml)
|
|
61
90
|
* Returns empty config if file doesn't exist
|
|
62
91
|
*/
|
|
63
92
|
export function loadConfig() {
|
|
64
|
-
|
|
93
|
+
// SDK inline config mode
|
|
94
|
+
if (configSource.type === 'inline') {
|
|
95
|
+
return configSource.config;
|
|
96
|
+
}
|
|
97
|
+
// File-based config (SDK custom path or default)
|
|
98
|
+
const configPath = configSource.path || getConfigFilePath();
|
|
65
99
|
if (!existsSync(configPath)) {
|
|
66
100
|
return { collections: {} };
|
|
67
101
|
}
|
|
@@ -79,11 +113,21 @@ export function loadConfig() {
|
|
|
79
113
|
}
|
|
80
114
|
}
|
|
81
115
|
/**
|
|
82
|
-
* Save configuration to
|
|
116
|
+
* Save configuration to the configured source.
|
|
117
|
+
* - Inline config: updates the in-memory object (no file I/O)
|
|
118
|
+
* - File-based: writes to YAML file (default ~/.config/qmd/index.yml)
|
|
83
119
|
*/
|
|
84
120
|
export function saveConfig(config) {
|
|
85
|
-
|
|
86
|
-
|
|
121
|
+
// SDK inline config mode: update in place, no file I/O
|
|
122
|
+
if (configSource.type === 'inline') {
|
|
123
|
+
configSource.config = config;
|
|
124
|
+
return;
|
|
125
|
+
}
|
|
126
|
+
const configPath = configSource.path || getConfigFilePath();
|
|
127
|
+
const configDir = dirname(configPath);
|
|
128
|
+
if (!existsSync(configDir)) {
|
|
129
|
+
mkdirSync(configDir, { recursive: true });
|
|
130
|
+
}
|
|
87
131
|
try {
|
|
88
132
|
const yaml = YAML.stringify(config, {
|
|
89
133
|
indent: 2,
|
|
@@ -318,13 +362,18 @@ export function findContextForPath(collectionName, filePath) {
|
|
|
318
362
|
* Get the config file path (useful for error messages)
|
|
319
363
|
*/
|
|
320
364
|
export function getConfigPath() {
|
|
321
|
-
|
|
365
|
+
if (configSource.type === 'inline')
|
|
366
|
+
return '<inline>';
|
|
367
|
+
return configSource.path || getConfigFilePath();
|
|
322
368
|
}
|
|
323
369
|
/**
|
|
324
370
|
* Check if config file exists
|
|
325
371
|
*/
|
|
326
372
|
export function configExists() {
|
|
327
|
-
|
|
373
|
+
if (configSource.type === 'inline')
|
|
374
|
+
return true;
|
|
375
|
+
const path = configSource.path || getConfigFilePath();
|
|
376
|
+
return existsSync(path);
|
|
328
377
|
}
|
|
329
378
|
/**
|
|
330
379
|
* Validate a collection name
|
package/dist/formatter.d.ts
CHANGED
package/dist/formatter.js
CHANGED
|
@@ -55,7 +55,7 @@ export function searchResultsToJson(results, opts = {}) {
|
|
|
55
55
|
const output = results.map(row => {
|
|
56
56
|
const bodyStr = row.body || "";
|
|
57
57
|
let body = opts.full ? bodyStr : undefined;
|
|
58
|
-
let snippet = !opts.full ? extractSnippet(bodyStr, query, 300, row.chunkPos).snippet : undefined;
|
|
58
|
+
let snippet = !opts.full ? extractSnippet(bodyStr, query, 300, row.chunkPos, undefined, opts.intent).snippet : undefined;
|
|
59
59
|
if (opts.lineNumbers) {
|
|
60
60
|
if (body)
|
|
61
61
|
body = addLineNumbers(body);
|
|
@@ -82,7 +82,7 @@ export function searchResultsToCsv(results, opts = {}) {
|
|
|
82
82
|
const header = "docid,score,file,title,context,line,snippet";
|
|
83
83
|
const rows = results.map(row => {
|
|
84
84
|
const bodyStr = row.body || "";
|
|
85
|
-
const { line, snippet } = extractSnippet(bodyStr, query, 500, row.chunkPos);
|
|
85
|
+
const { line, snippet } = extractSnippet(bodyStr, query, 500, row.chunkPos, undefined, opts.intent);
|
|
86
86
|
let content = opts.full ? bodyStr : snippet;
|
|
87
87
|
if (opts.lineNumbers && content) {
|
|
88
88
|
content = addLineNumbers(content);
|
|
@@ -121,7 +121,7 @@ export function searchResultsToMarkdown(results, opts = {}) {
|
|
|
121
121
|
content = bodyStr;
|
|
122
122
|
}
|
|
123
123
|
else {
|
|
124
|
-
content = extractSnippet(bodyStr, query, 500, row.chunkPos).snippet;
|
|
124
|
+
content = extractSnippet(bodyStr, query, 500, row.chunkPos, undefined, opts.intent).snippet;
|
|
125
125
|
}
|
|
126
126
|
if (opts.lineNumbers) {
|
|
127
127
|
content = addLineNumbers(content);
|
|
@@ -138,7 +138,7 @@ export function searchResultsToXml(results, opts = {}) {
|
|
|
138
138
|
const items = results.map(row => {
|
|
139
139
|
const titleAttr = row.title ? ` title="${escapeXml(row.title)}"` : "";
|
|
140
140
|
const bodyStr = row.body || "";
|
|
141
|
-
let content = opts.full ? bodyStr : extractSnippet(bodyStr, query, 500, row.chunkPos).snippet;
|
|
141
|
+
let content = opts.full ? bodyStr : extractSnippet(bodyStr, query, 500, row.chunkPos, undefined, opts.intent).snippet;
|
|
142
142
|
if (opts.lineNumbers) {
|
|
143
143
|
content = addLineNumbers(content);
|
|
144
144
|
}
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* QMD SDK - Library mode for programmatic access to QMD search and indexing.
|
|
3
|
+
*
|
|
4
|
+
* Usage:
|
|
5
|
+
* import { createStore } from '@tobilu/qmd'
|
|
6
|
+
*
|
|
7
|
+
* const store = createStore({
|
|
8
|
+
* dbPath: './my-index.sqlite',
|
|
9
|
+
* config: {
|
|
10
|
+
* collections: {
|
|
11
|
+
* docs: { path: '/path/to/docs', pattern: '**\/*.md' }
|
|
12
|
+
* }
|
|
13
|
+
* }
|
|
14
|
+
* })
|
|
15
|
+
*
|
|
16
|
+
* const results = await store.query("how does auth work?")
|
|
17
|
+
* store.close()
|
|
18
|
+
*/
|
|
19
|
+
import { type Store as InternalStore, type DocumentResult, type DocumentNotFound, type SearchResult, type HybridQueryResult, type HybridQueryOptions, type HybridQueryExplain, type StructuredSubSearch, type StructuredSearchOptions, type MultiGetResult, type IndexStatus, type IndexHealthInfo, type ExpandedQuery, type SearchHooks } from "./store.js";
|
|
20
|
+
import { type Collection, type CollectionConfig, type NamedCollection, type ContextMap } from "./collections.js";
|
|
21
|
+
export type { DocumentResult, DocumentNotFound, SearchResult, HybridQueryResult, HybridQueryOptions, HybridQueryExplain, StructuredSubSearch, StructuredSearchOptions, MultiGetResult, IndexStatus, IndexHealthInfo, ExpandedQuery, SearchHooks, Collection, CollectionConfig, NamedCollection, ContextMap, };
|
|
22
|
+
/**
|
|
23
|
+
* Options for creating a QMD store.
|
|
24
|
+
* You must provide `dbPath` and either `configPath` (YAML file) or `config` (inline).
|
|
25
|
+
*/
|
|
26
|
+
export interface StoreOptions {
|
|
27
|
+
/** Path to the SQLite database file */
|
|
28
|
+
dbPath: string;
|
|
29
|
+
/** Path to a YAML config file (mutually exclusive with `config`) */
|
|
30
|
+
configPath?: string;
|
|
31
|
+
/** Inline collection config (mutually exclusive with `configPath`) */
|
|
32
|
+
config?: CollectionConfig;
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* The QMD SDK store — provides search, retrieval, collection management,
|
|
36
|
+
* context management, and indexing operations.
|
|
37
|
+
*/
|
|
38
|
+
export interface QMDStore {
|
|
39
|
+
/** The underlying internal store (for advanced use) */
|
|
40
|
+
readonly internal: InternalStore;
|
|
41
|
+
/** Path to the SQLite database */
|
|
42
|
+
readonly dbPath: string;
|
|
43
|
+
/** Hybrid search: BM25 + vector + query expansion + LLM reranking */
|
|
44
|
+
query(query: string, options?: HybridQueryOptions): Promise<HybridQueryResult[]>;
|
|
45
|
+
/** BM25 full-text keyword search (fast, no LLM) */
|
|
46
|
+
search(query: string, options?: {
|
|
47
|
+
limit?: number;
|
|
48
|
+
collection?: string;
|
|
49
|
+
}): SearchResult[];
|
|
50
|
+
/** Structured search with pre-expanded queries (for LLM callers) */
|
|
51
|
+
structuredSearch(searches: StructuredSubSearch[], options?: StructuredSearchOptions): Promise<HybridQueryResult[]>;
|
|
52
|
+
/** Get a single document by path or docid */
|
|
53
|
+
get(pathOrDocid: string, options?: {
|
|
54
|
+
includeBody?: boolean;
|
|
55
|
+
}): DocumentResult | DocumentNotFound;
|
|
56
|
+
/** Get multiple documents by glob pattern or comma-separated list */
|
|
57
|
+
multiGet(pattern: string, options?: {
|
|
58
|
+
includeBody?: boolean;
|
|
59
|
+
maxBytes?: number;
|
|
60
|
+
}): {
|
|
61
|
+
docs: MultiGetResult[];
|
|
62
|
+
errors: string[];
|
|
63
|
+
};
|
|
64
|
+
/** Add or update a collection */
|
|
65
|
+
addCollection(name: string, opts: {
|
|
66
|
+
path: string;
|
|
67
|
+
pattern?: string;
|
|
68
|
+
ignore?: string[];
|
|
69
|
+
}): void;
|
|
70
|
+
/** Remove a collection */
|
|
71
|
+
removeCollection(name: string): boolean;
|
|
72
|
+
/** Rename a collection */
|
|
73
|
+
renameCollection(oldName: string, newName: string): boolean;
|
|
74
|
+
/** List all collections with document stats */
|
|
75
|
+
listCollections(): {
|
|
76
|
+
name: string;
|
|
77
|
+
pwd: string;
|
|
78
|
+
glob_pattern: string;
|
|
79
|
+
doc_count: number;
|
|
80
|
+
active_count: number;
|
|
81
|
+
last_modified: string | null;
|
|
82
|
+
}[];
|
|
83
|
+
/** Add context for a path within a collection */
|
|
84
|
+
addContext(collectionName: string, pathPrefix: string, contextText: string): boolean;
|
|
85
|
+
/** Remove context from a collection path */
|
|
86
|
+
removeContext(collectionName: string, pathPrefix: string): boolean;
|
|
87
|
+
/** Set global context (applies to all collections) */
|
|
88
|
+
setGlobalContext(context: string | undefined): void;
|
|
89
|
+
/** Get global context */
|
|
90
|
+
getGlobalContext(): string | undefined;
|
|
91
|
+
/** List all contexts across all collections */
|
|
92
|
+
listContexts(): Array<{
|
|
93
|
+
collection: string;
|
|
94
|
+
path: string;
|
|
95
|
+
context: string;
|
|
96
|
+
}>;
|
|
97
|
+
/** Get index status (document counts, collections, embedding state) */
|
|
98
|
+
getStatus(): IndexStatus;
|
|
99
|
+
/** Get index health info (stale embeddings, etc.) */
|
|
100
|
+
getIndexHealth(): IndexHealthInfo;
|
|
101
|
+
/** Close the database connection */
|
|
102
|
+
close(): void;
|
|
103
|
+
}
|
|
104
|
+
/**
|
|
105
|
+
* Create a QMD store for programmatic access to search and indexing.
|
|
106
|
+
*
|
|
107
|
+
* @example
|
|
108
|
+
* ```typescript
|
|
109
|
+
* // With a YAML config file
|
|
110
|
+
* const store = createStore({
|
|
111
|
+
* dbPath: './index.sqlite',
|
|
112
|
+
* configPath: './qmd.yml',
|
|
113
|
+
* })
|
|
114
|
+
*
|
|
115
|
+
* // With inline config (no files needed besides the DB)
|
|
116
|
+
* const store = createStore({
|
|
117
|
+
* dbPath: './index.sqlite',
|
|
118
|
+
* config: {
|
|
119
|
+
* collections: {
|
|
120
|
+
* docs: { path: '/path/to/docs', pattern: '**\/*.md' }
|
|
121
|
+
* }
|
|
122
|
+
* }
|
|
123
|
+
* })
|
|
124
|
+
*
|
|
125
|
+
* const results = await store.query("authentication flow")
|
|
126
|
+
* store.close()
|
|
127
|
+
* ```
|
|
128
|
+
*/
|
|
129
|
+
export declare function createStore(options: StoreOptions): QMDStore;
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* QMD SDK - Library mode for programmatic access to QMD search and indexing.
|
|
3
|
+
*
|
|
4
|
+
* Usage:
|
|
5
|
+
* import { createStore } from '@tobilu/qmd'
|
|
6
|
+
*
|
|
7
|
+
* const store = createStore({
|
|
8
|
+
* dbPath: './my-index.sqlite',
|
|
9
|
+
* config: {
|
|
10
|
+
* collections: {
|
|
11
|
+
* docs: { path: '/path/to/docs', pattern: '**\/*.md' }
|
|
12
|
+
* }
|
|
13
|
+
* }
|
|
14
|
+
* })
|
|
15
|
+
*
|
|
16
|
+
* const results = await store.query("how does auth work?")
|
|
17
|
+
* store.close()
|
|
18
|
+
*/
|
|
19
|
+
import { createStore as createStoreInternal, hybridQuery, structuredSearch, listCollections as storeListCollections, } from "./store.js";
|
|
20
|
+
import { setConfigSource, loadConfig, addCollection as collectionsAddCollection, removeCollection as collectionsRemoveCollection, renameCollection as collectionsRenameCollection, listCollections as collectionsListCollections, addContext as collectionsAddContext, removeContext as collectionsRemoveContext, setGlobalContext as collectionsSetGlobalContext, getGlobalContext as collectionsGetGlobalContext, listAllContexts as collectionsListAllContexts, } from "./collections.js";
|
|
21
|
+
/**
|
|
22
|
+
* Create a QMD store for programmatic access to search and indexing.
|
|
23
|
+
*
|
|
24
|
+
* @example
|
|
25
|
+
* ```typescript
|
|
26
|
+
* // With a YAML config file
|
|
27
|
+
* const store = createStore({
|
|
28
|
+
* dbPath: './index.sqlite',
|
|
29
|
+
* configPath: './qmd.yml',
|
|
30
|
+
* })
|
|
31
|
+
*
|
|
32
|
+
* // With inline config (no files needed besides the DB)
|
|
33
|
+
* const store = createStore({
|
|
34
|
+
* dbPath: './index.sqlite',
|
|
35
|
+
* config: {
|
|
36
|
+
* collections: {
|
|
37
|
+
* docs: { path: '/path/to/docs', pattern: '**\/*.md' }
|
|
38
|
+
* }
|
|
39
|
+
* }
|
|
40
|
+
* })
|
|
41
|
+
*
|
|
42
|
+
* const results = await store.query("authentication flow")
|
|
43
|
+
* store.close()
|
|
44
|
+
* ```
|
|
45
|
+
*/
|
|
46
|
+
export function createStore(options) {
|
|
47
|
+
if (!options.dbPath) {
|
|
48
|
+
throw new Error("dbPath is required");
|
|
49
|
+
}
|
|
50
|
+
if (!options.configPath && !options.config) {
|
|
51
|
+
throw new Error("Either configPath or config is required");
|
|
52
|
+
}
|
|
53
|
+
if (options.configPath && options.config) {
|
|
54
|
+
throw new Error("Provide either configPath or config, not both");
|
|
55
|
+
}
|
|
56
|
+
// Inject config source into collections module
|
|
57
|
+
setConfigSource({
|
|
58
|
+
configPath: options.configPath,
|
|
59
|
+
config: options.config,
|
|
60
|
+
});
|
|
61
|
+
// Create the internal store
|
|
62
|
+
const internal = createStoreInternal(options.dbPath);
|
|
63
|
+
const store = {
|
|
64
|
+
internal,
|
|
65
|
+
dbPath: internal.dbPath,
|
|
66
|
+
// Search & Retrieval
|
|
67
|
+
query: (q, opts) => hybridQuery(internal, q, opts),
|
|
68
|
+
search: (q, opts) => internal.searchFTS(q, opts?.limit, opts?.collection),
|
|
69
|
+
structuredSearch: (searches, opts) => structuredSearch(internal, searches, opts),
|
|
70
|
+
get: (pathOrDocid, opts) => internal.findDocument(pathOrDocid, opts),
|
|
71
|
+
multiGet: (pattern, opts) => internal.findDocuments(pattern, opts),
|
|
72
|
+
// Collection Management
|
|
73
|
+
addCollection: (name, opts) => {
|
|
74
|
+
collectionsAddCollection(name, opts.path, opts.pattern);
|
|
75
|
+
},
|
|
76
|
+
removeCollection: (name) => collectionsRemoveCollection(name),
|
|
77
|
+
renameCollection: (oldName, newName) => collectionsRenameCollection(oldName, newName),
|
|
78
|
+
listCollections: () => storeListCollections(internal.db),
|
|
79
|
+
// Context Management
|
|
80
|
+
addContext: (collectionName, pathPrefix, contextText) => collectionsAddContext(collectionName, pathPrefix, contextText),
|
|
81
|
+
removeContext: (collectionName, pathPrefix) => collectionsRemoveContext(collectionName, pathPrefix),
|
|
82
|
+
setGlobalContext: (context) => collectionsSetGlobalContext(context),
|
|
83
|
+
getGlobalContext: () => collectionsGetGlobalContext(),
|
|
84
|
+
listContexts: () => collectionsListAllContexts(),
|
|
85
|
+
// Index Health
|
|
86
|
+
getStatus: () => internal.getStatus(),
|
|
87
|
+
getIndexHealth: () => internal.getIndexHealth(),
|
|
88
|
+
// Lifecycle
|
|
89
|
+
close: () => {
|
|
90
|
+
internal.close();
|
|
91
|
+
setConfigSource(undefined); // Reset config source
|
|
92
|
+
},
|
|
93
|
+
};
|
|
94
|
+
return store;
|
|
95
|
+
}
|
package/dist/llm.d.ts
CHANGED
|
@@ -330,6 +330,7 @@ export declare class LlamaCpp implements LLM {
|
|
|
330
330
|
expandQuery(query: string, options?: {
|
|
331
331
|
context?: string;
|
|
332
332
|
includeLexical?: boolean;
|
|
333
|
+
intent?: string;
|
|
333
334
|
}): Promise<Queryable[]>;
|
|
334
335
|
private static readonly RERANK_TEMPLATE_OVERHEAD;
|
|
335
336
|
private static readonly RERANK_TARGET_DOCS_PER_CONTEXT;
|
package/dist/llm.js
CHANGED
|
@@ -691,7 +691,10 @@ export class LlamaCpp {
|
|
|
691
691
|
content ::= [^\\n]+
|
|
692
692
|
`
|
|
693
693
|
});
|
|
694
|
-
const
|
|
694
|
+
const intent = options.intent;
|
|
695
|
+
const prompt = intent
|
|
696
|
+
? `/no_think Expand this search query: ${query}\nQuery intent: ${intent}`
|
|
697
|
+
: `/no_think Expand this search query: ${query}`;
|
|
695
698
|
// Create a bounded context for expansion to prevent large default VRAM allocations.
|
|
696
699
|
const genContext = await this.generateModel.createContext({
|
|
697
700
|
contextSize: this.expandContextSize,
|
package/dist/mcp.js
CHANGED
|
@@ -84,10 +84,13 @@ function buildInstructions(store) {
|
|
|
84
84
|
lines.push(" - type:'vec' — semantic vector search (meaning-based)");
|
|
85
85
|
lines.push(" - type:'hyde' — hypothetical document (write what the answer looks like)");
|
|
86
86
|
lines.push("");
|
|
87
|
+
lines.push(" Always provide `intent` on every search call to disambiguate and improve snippets.");
|
|
88
|
+
lines.push("");
|
|
87
89
|
lines.push("Examples:");
|
|
88
90
|
lines.push(" Quick keyword lookup: [{type:'lex', query:'error handling'}]");
|
|
89
91
|
lines.push(" Semantic search: [{type:'vec', query:'how to handle errors gracefully'}]");
|
|
90
92
|
lines.push(" Best results: [{type:'lex', query:'error'}, {type:'vec', query:'error handling best practices'}]");
|
|
93
|
+
lines.push(" With intent: searches=[{type:'lex', query:'performance'}], intent='web page load times'");
|
|
91
94
|
// --- Retrieval workflow ---
|
|
92
95
|
lines.push("");
|
|
93
96
|
lines.push("Retrieval:");
|
|
@@ -236,8 +239,9 @@ Intent-aware lex (C++ performance, not sports):
|
|
|
236
239
|
minScore: z.number().optional().default(0).describe("Min relevance 0-1 (default: 0)"),
|
|
237
240
|
candidateLimit: z.number().optional().describe("Maximum candidates to rerank (default: 40, lower = faster but may miss results)"),
|
|
238
241
|
collections: z.array(z.string()).optional().describe("Filter to collections (OR match)"),
|
|
242
|
+
intent: z.string().optional().describe("Background context to disambiguate the query. Example: query='performance', intent='web page load times and Core Web Vitals'. Does not search on its own."),
|
|
239
243
|
},
|
|
240
|
-
}, async ({ searches, limit, minScore, candidateLimit, collections }) => {
|
|
244
|
+
}, async ({ searches, limit, minScore, candidateLimit, collections, intent }) => {
|
|
241
245
|
// Map to internal format
|
|
242
246
|
const subSearches = searches.map(s => ({
|
|
243
247
|
type: s.type,
|
|
@@ -250,13 +254,14 @@ Intent-aware lex (C++ performance, not sports):
|
|
|
250
254
|
limit,
|
|
251
255
|
minScore,
|
|
252
256
|
candidateLimit,
|
|
257
|
+
intent,
|
|
253
258
|
});
|
|
254
259
|
// Use first lex or vec query for snippet extraction
|
|
255
260
|
const primaryQuery = searches.find(s => s.type === 'lex')?.query
|
|
256
261
|
|| searches.find(s => s.type === 'vec')?.query
|
|
257
262
|
|| searches[0]?.query || "";
|
|
258
263
|
const filtered = results.map(r => {
|
|
259
|
-
const { line, snippet } = extractSnippet(r.bestChunk, primaryQuery, 300);
|
|
264
|
+
const { line, snippet } = extractSnippet(r.bestChunk, primaryQuery, 300, undefined, undefined, intent);
|
|
260
265
|
return {
|
|
261
266
|
docid: `#${r.docid}`,
|
|
262
267
|
file: r.displayPath,
|
package/dist/qmd.js
CHANGED
|
@@ -1567,7 +1567,7 @@ function outputResults(results, query, opts) {
|
|
|
1567
1567
|
const output = filtered.map(row => {
|
|
1568
1568
|
const docid = row.docid || (row.hash ? row.hash.slice(0, 6) : undefined);
|
|
1569
1569
|
let body = opts.full ? row.body : undefined;
|
|
1570
|
-
let snippet = !opts.full ? extractSnippet(row.body, query, 300, row.chunkPos).snippet : undefined;
|
|
1570
|
+
let snippet = !opts.full ? extractSnippet(row.body, query, 300, row.chunkPos, undefined, opts.intent).snippet : undefined;
|
|
1571
1571
|
if (opts.lineNumbers) {
|
|
1572
1572
|
if (body)
|
|
1573
1573
|
body = addLineNumbers(body);
|
|
@@ -1600,7 +1600,7 @@ function outputResults(results, query, opts) {
|
|
|
1600
1600
|
const row = filtered[i];
|
|
1601
1601
|
if (!row)
|
|
1602
1602
|
continue;
|
|
1603
|
-
const { line, snippet } = extractSnippet(row.body, query, 500, row.chunkPos);
|
|
1603
|
+
const { line, snippet } = extractSnippet(row.body, query, 500, row.chunkPos, undefined, opts.intent);
|
|
1604
1604
|
const docid = row.docid || (row.hash ? row.hash.slice(0, 6) : undefined);
|
|
1605
1605
|
// Line 1: filepath with docid
|
|
1606
1606
|
const path = toQmdPath(row.displayPath);
|
|
@@ -1659,7 +1659,7 @@ function outputResults(results, query, opts) {
|
|
|
1659
1659
|
continue;
|
|
1660
1660
|
const heading = row.title || row.displayPath;
|
|
1661
1661
|
const docid = row.docid || (row.hash ? row.hash.slice(0, 6) : undefined);
|
|
1662
|
-
let content = opts.full ? row.body : extractSnippet(row.body, query, 500, row.chunkPos).snippet;
|
|
1662
|
+
let content = opts.full ? row.body : extractSnippet(row.body, query, 500, row.chunkPos, undefined, opts.intent).snippet;
|
|
1663
1663
|
if (opts.lineNumbers) {
|
|
1664
1664
|
content = addLineNumbers(content);
|
|
1665
1665
|
}
|
|
@@ -1673,7 +1673,7 @@ function outputResults(results, query, opts) {
|
|
|
1673
1673
|
const titleAttr = row.title ? ` title="${row.title.replace(/"/g, '"')}"` : "";
|
|
1674
1674
|
const contextAttr = row.context ? ` context="${row.context.replace(/"/g, '"')}"` : "";
|
|
1675
1675
|
const docid = row.docid || (row.hash ? row.hash.slice(0, 6) : "");
|
|
1676
|
-
let content = opts.full ? row.body : extractSnippet(row.body, query, 500, row.chunkPos).snippet;
|
|
1676
|
+
let content = opts.full ? row.body : extractSnippet(row.body, query, 500, row.chunkPos, undefined, opts.intent).snippet;
|
|
1677
1677
|
if (opts.lineNumbers) {
|
|
1678
1678
|
content = addLineNumbers(content);
|
|
1679
1679
|
}
|
|
@@ -1684,7 +1684,7 @@ function outputResults(results, query, opts) {
|
|
|
1684
1684
|
// CSV format
|
|
1685
1685
|
console.log("docid,score,file,title,context,line,snippet");
|
|
1686
1686
|
for (const row of filtered) {
|
|
1687
|
-
const { line, snippet } = extractSnippet(row.body, query, 500, row.chunkPos);
|
|
1687
|
+
const { line, snippet } = extractSnippet(row.body, query, 500, row.chunkPos, undefined, opts.intent);
|
|
1688
1688
|
let content = opts.full ? row.body : snippet;
|
|
1689
1689
|
if (opts.lineNumbers) {
|
|
1690
1690
|
content = addLineNumbers(content, line);
|
|
@@ -1727,21 +1727,6 @@ function filterByCollections(results, collectionNames) {
|
|
|
1727
1727
|
return prefixes.some(p => path.startsWith(p));
|
|
1728
1728
|
});
|
|
1729
1729
|
}
|
|
1730
|
-
/**
|
|
1731
|
-
* Parse structured search query syntax.
|
|
1732
|
-
* Lines starting with lex:, vec:, or hyde: are routed directly.
|
|
1733
|
-
* Plain lines without prefix go through query expansion.
|
|
1734
|
-
*
|
|
1735
|
-
* Returns null if this is a plain query (single line, no prefix).
|
|
1736
|
-
* Returns StructuredSubSearch[] if structured syntax detected.
|
|
1737
|
-
* Throws if multiple plain lines (ambiguous).
|
|
1738
|
-
*
|
|
1739
|
-
* Examples:
|
|
1740
|
-
* "CAP theorem" -> null (plain query, use expansion)
|
|
1741
|
-
* "lex: CAP theorem" -> [{ type: 'lex', query: 'CAP theorem' }]
|
|
1742
|
-
* "lex: CAP\nvec: consistency" -> [{ type: 'lex', ... }, { type: 'vec', ... }]
|
|
1743
|
-
* "CAP\nconsistency" -> throws (multiple plain lines)
|
|
1744
|
-
*/
|
|
1745
1730
|
function parseStructuredQuery(query) {
|
|
1746
1731
|
const rawLines = query.split('\n').map((line, idx) => ({
|
|
1747
1732
|
raw: line,
|
|
@@ -1752,7 +1737,9 @@ function parseStructuredQuery(query) {
|
|
|
1752
1737
|
return null;
|
|
1753
1738
|
const prefixRe = /^(lex|vec|hyde):\s*/i;
|
|
1754
1739
|
const expandRe = /^expand:\s*/i;
|
|
1740
|
+
const intentRe = /^intent:\s*/i;
|
|
1755
1741
|
const typed = [];
|
|
1742
|
+
let intent;
|
|
1756
1743
|
for (const line of rawLines) {
|
|
1757
1744
|
if (expandRe.test(line.trimmed)) {
|
|
1758
1745
|
if (rawLines.length > 1) {
|
|
@@ -1764,6 +1751,18 @@ function parseStructuredQuery(query) {
|
|
|
1764
1751
|
}
|
|
1765
1752
|
return null; // treat as standalone expand query
|
|
1766
1753
|
}
|
|
1754
|
+
// Parse intent: lines
|
|
1755
|
+
if (intentRe.test(line.trimmed)) {
|
|
1756
|
+
if (intent !== undefined) {
|
|
1757
|
+
throw new Error(`Line ${line.number}: only one intent: line is allowed per query document.`);
|
|
1758
|
+
}
|
|
1759
|
+
const text = line.trimmed.replace(intentRe, '').trim();
|
|
1760
|
+
if (!text) {
|
|
1761
|
+
throw new Error(`Line ${line.number}: intent: must include text.`);
|
|
1762
|
+
}
|
|
1763
|
+
intent = text;
|
|
1764
|
+
continue;
|
|
1765
|
+
}
|
|
1767
1766
|
const match = line.trimmed.match(prefixRe);
|
|
1768
1767
|
if (match) {
|
|
1769
1768
|
const type = match[1].toLowerCase();
|
|
@@ -1781,9 +1780,13 @@ function parseStructuredQuery(query) {
|
|
|
1781
1780
|
// Single plain line -> implicit expand
|
|
1782
1781
|
return null;
|
|
1783
1782
|
}
|
|
1784
|
-
throw new Error(`Line ${line.number} is missing a lex:/vec:/hyde: prefix. Each line in a query document must start with one.`);
|
|
1783
|
+
throw new Error(`Line ${line.number} is missing a lex:/vec:/hyde:/intent: prefix. Each line in a query document must start with one.`);
|
|
1785
1784
|
}
|
|
1786
|
-
|
|
1785
|
+
// intent: alone is not a valid query — must have at least one search
|
|
1786
|
+
if (intent && typed.length === 0) {
|
|
1787
|
+
throw new Error('intent: cannot appear alone. Add at least one lex:, vec:, or hyde: line.');
|
|
1788
|
+
}
|
|
1789
|
+
return typed.length > 0 ? { searches: typed, intent } : null;
|
|
1787
1790
|
}
|
|
1788
1791
|
function search(query, opts) {
|
|
1789
1792
|
const db = getDb();
|
|
@@ -1840,6 +1843,7 @@ async function vectorSearch(query, opts, _model = DEFAULT_EMBED_MODEL) {
|
|
|
1840
1843
|
collection: singleCollection,
|
|
1841
1844
|
limit: opts.all ? 500 : (opts.limit || 10),
|
|
1842
1845
|
minScore: opts.minScore || 0.3,
|
|
1846
|
+
intent: opts.intent,
|
|
1843
1847
|
hooks: {
|
|
1844
1848
|
onExpand: (original, expanded) => {
|
|
1845
1849
|
logExpansionTree(original, expanded);
|
|
@@ -1877,14 +1881,20 @@ async function querySearch(query, opts, _embedModel = DEFAULT_EMBED_MODEL, _rera
|
|
|
1877
1881
|
const collectionNames = resolveCollectionFilter(opts.collection, true);
|
|
1878
1882
|
const singleCollection = collectionNames.length === 1 ? collectionNames[0] : undefined;
|
|
1879
1883
|
checkIndexHealth(store.db);
|
|
1880
|
-
// Check for structured query syntax (lex:/vec:/hyde: prefixes)
|
|
1881
|
-
const
|
|
1884
|
+
// Check for structured query syntax (lex:/vec:/hyde:/intent: prefixes)
|
|
1885
|
+
const parsed = parseStructuredQuery(query);
|
|
1886
|
+
// Intent can come from --intent flag or from intent: line in query document
|
|
1887
|
+
const intent = opts.intent || parsed?.intent;
|
|
1882
1888
|
await withLLMSession(async () => {
|
|
1883
1889
|
let results;
|
|
1884
|
-
if (
|
|
1890
|
+
if (parsed) {
|
|
1891
|
+
const structuredQueries = parsed.searches;
|
|
1885
1892
|
// Structured search — user provided their own query expansions
|
|
1886
1893
|
const typeLabels = structuredQueries.map(s => s.type).join('+');
|
|
1887
1894
|
process.stderr.write(`${c.dim}Structured search: ${structuredQueries.length} queries (${typeLabels})${c.reset}\n`);
|
|
1895
|
+
if (intent) {
|
|
1896
|
+
process.stderr.write(`${c.dim}├─ intent: ${intent}${c.reset}\n`);
|
|
1897
|
+
}
|
|
1888
1898
|
// Log each sub-query
|
|
1889
1899
|
for (const s of structuredQueries) {
|
|
1890
1900
|
let preview = s.query.replace(/\n/g, ' ');
|
|
@@ -1899,6 +1909,7 @@ async function querySearch(query, opts, _embedModel = DEFAULT_EMBED_MODEL, _rera
|
|
|
1899
1909
|
minScore: opts.minScore || 0,
|
|
1900
1910
|
candidateLimit: opts.candidateLimit,
|
|
1901
1911
|
explain: !!opts.explain,
|
|
1912
|
+
intent,
|
|
1902
1913
|
hooks: {
|
|
1903
1914
|
onEmbedStart: (count) => {
|
|
1904
1915
|
process.stderr.write(`${c.dim}Embedding ${count} ${count === 1 ? 'query' : 'queries'}...${c.reset}`);
|
|
@@ -1925,6 +1936,7 @@ async function querySearch(query, opts, _embedModel = DEFAULT_EMBED_MODEL, _rera
|
|
|
1925
1936
|
minScore: opts.minScore || 0,
|
|
1926
1937
|
candidateLimit: opts.candidateLimit,
|
|
1927
1938
|
explain: !!opts.explain,
|
|
1939
|
+
intent,
|
|
1928
1940
|
hooks: {
|
|
1929
1941
|
onStrongSignal: (score) => {
|
|
1930
1942
|
process.stderr.write(`${c.dim}Strong BM25 signal (${score.toFixed(2)}) — skipping expansion${c.reset}\n`);
|
|
@@ -1967,6 +1979,7 @@ async function querySearch(query, opts, _embedModel = DEFAULT_EMBED_MODEL, _rera
|
|
|
1967
1979
|
return;
|
|
1968
1980
|
}
|
|
1969
1981
|
// Use first lex/vec query for output context, or original query
|
|
1982
|
+
const structuredQueries = parsed?.searches;
|
|
1970
1983
|
const displayQuery = structuredQueries
|
|
1971
1984
|
? (structuredQueries.find(s => s.type === 'lex')?.query || structuredQueries.find(s => s.type === 'vec')?.query || query)
|
|
1972
1985
|
: query;
|
|
@@ -2026,6 +2039,7 @@ function parseCLI() {
|
|
|
2026
2039
|
"line-numbers": { type: "boolean" }, // add line numbers to output
|
|
2027
2040
|
// Query options
|
|
2028
2041
|
"candidate-limit": { type: "string", short: "C" },
|
|
2042
|
+
intent: { type: "string" },
|
|
2029
2043
|
// MCP HTTP transport options
|
|
2030
2044
|
http: { type: "boolean" },
|
|
2031
2045
|
daemon: { type: "boolean" },
|
|
@@ -2066,6 +2080,7 @@ function parseCLI() {
|
|
|
2066
2080
|
lineNumbers: !!values["line-numbers"],
|
|
2067
2081
|
candidateLimit: values["candidate-limit"] ? parseInt(String(values["candidate-limit"]), 10) : undefined,
|
|
2068
2082
|
explain: !!values.explain,
|
|
2083
|
+
intent: values.intent,
|
|
2069
2084
|
};
|
|
2070
2085
|
return {
|
|
2071
2086
|
command: positionals[0] || "",
|
|
@@ -2124,7 +2139,8 @@ function showHelp() {
|
|
|
2124
2139
|
`query = expand_query | query_document ;`,
|
|
2125
2140
|
`expand_query = text | explicit_expand ;`,
|
|
2126
2141
|
`explicit_expand= "expand:" text ;`,
|
|
2127
|
-
`query_document = { typed_line } ;`,
|
|
2142
|
+
`query_document = [ intent_line ] { typed_line } ;`,
|
|
2143
|
+
`intent_line = "intent:" text newline ;`,
|
|
2128
2144
|
`typed_line = type ":" text newline ;`,
|
|
2129
2145
|
`type = "lex" | "vec" | "hyde" ;`,
|
|
2130
2146
|
`text = quoted_phrase | plain_text ;`,
|
package/dist/store.d.ts
CHANGED
|
@@ -202,11 +202,11 @@ export type Store = {
|
|
|
202
202
|
toVirtualPath: (absolutePath: string) => string | null;
|
|
203
203
|
searchFTS: (query: string, limit?: number, collectionName?: string) => SearchResult[];
|
|
204
204
|
searchVec: (query: string, model: string, limit?: number, collectionName?: string, session?: ILLMSession, precomputedEmbedding?: number[]) => Promise<SearchResult[]>;
|
|
205
|
-
expandQuery: (query: string, model?: string) => Promise<ExpandedQuery[]>;
|
|
205
|
+
expandQuery: (query: string, model?: string, intent?: string) => Promise<ExpandedQuery[]>;
|
|
206
206
|
rerank: (query: string, documents: {
|
|
207
207
|
file: string;
|
|
208
208
|
text: string;
|
|
209
|
-
}[], model?: string) => Promise<{
|
|
209
|
+
}[], model?: string, intent?: string) => Promise<{
|
|
210
210
|
file: string;
|
|
211
211
|
score: number;
|
|
212
212
|
}[]>;
|
|
@@ -598,11 +598,11 @@ export declare function clearAllEmbeddings(db: Database): void;
|
|
|
598
598
|
* The hash_seq key is formatted as "hash_seq" for the vectors_vec table.
|
|
599
599
|
*/
|
|
600
600
|
export declare function insertEmbedding(db: Database, hash: string, seq: number, pos: number, embedding: Float32Array, model: string, embeddedAt: string): void;
|
|
601
|
-
export declare function expandQuery(query: string, model: string | undefined, db: Database): Promise<ExpandedQuery[]>;
|
|
601
|
+
export declare function expandQuery(query: string, model: string | undefined, db: Database, intent?: string): Promise<ExpandedQuery[]>;
|
|
602
602
|
export declare function rerank(query: string, documents: {
|
|
603
603
|
file: string;
|
|
604
604
|
text: string;
|
|
605
|
-
}[], model: string | undefined, db: Database): Promise<{
|
|
605
|
+
}[], model: string | undefined, db: Database, intent?: string): Promise<{
|
|
606
606
|
file: string;
|
|
607
607
|
score: number;
|
|
608
608
|
}[]>;
|
|
@@ -650,7 +650,17 @@ export type SnippetResult = {
|
|
|
650
650
|
linesAfter: number;
|
|
651
651
|
snippetLines: number;
|
|
652
652
|
};
|
|
653
|
-
|
|
653
|
+
/** Weight for intent terms relative to query terms (1.0) in snippet scoring */
|
|
654
|
+
export declare const INTENT_WEIGHT_SNIPPET = 0.3;
|
|
655
|
+
/** Weight for intent terms relative to query terms (1.0) in chunk selection */
|
|
656
|
+
export declare const INTENT_WEIGHT_CHUNK = 0.5;
|
|
657
|
+
/**
|
|
658
|
+
* Extract meaningful terms from an intent string, filtering stop words and punctuation.
|
|
659
|
+
* Uses Unicode-aware punctuation stripping so domain terms like "API" survive.
|
|
660
|
+
* Returns lowercase terms suitable for text matching.
|
|
661
|
+
*/
|
|
662
|
+
export declare function extractIntentTerms(intent: string): string[];
|
|
663
|
+
export declare function extractSnippet(body: string, query: string, maxLen?: number, chunkPos?: number, chunkLen?: number, intent?: string): SnippetResult;
|
|
654
664
|
/**
|
|
655
665
|
* Add line numbers to text content.
|
|
656
666
|
* Each line becomes: "{lineNum}: {content}"
|
|
@@ -682,6 +692,7 @@ export interface HybridQueryOptions {
|
|
|
682
692
|
minScore?: number;
|
|
683
693
|
candidateLimit?: number;
|
|
684
694
|
explain?: boolean;
|
|
695
|
+
intent?: string;
|
|
685
696
|
hooks?: SearchHooks;
|
|
686
697
|
}
|
|
687
698
|
export interface HybridQueryResult {
|
|
@@ -719,6 +730,7 @@ export interface VectorSearchOptions {
|
|
|
719
730
|
collection?: string;
|
|
720
731
|
limit?: number;
|
|
721
732
|
minScore?: number;
|
|
733
|
+
intent?: string;
|
|
722
734
|
hooks?: Pick<SearchHooks, 'onExpand'>;
|
|
723
735
|
}
|
|
724
736
|
export interface VectorSearchResult {
|
|
@@ -758,7 +770,7 @@ export interface StructuredSearchOptions {
|
|
|
758
770
|
minScore?: number;
|
|
759
771
|
candidateLimit?: number;
|
|
760
772
|
explain?: boolean;
|
|
761
|
-
/**
|
|
773
|
+
/** Domain intent hint for disambiguation — steers reranking and chunk selection */
|
|
762
774
|
intent?: string;
|
|
763
775
|
hooks?: SearchHooks;
|
|
764
776
|
}
|
package/dist/store.js
CHANGED
|
@@ -667,8 +667,8 @@ export function createStore(dbPath) {
|
|
|
667
667
|
searchFTS: (query, limit, collectionName) => searchFTS(db, query, limit, collectionName),
|
|
668
668
|
searchVec: (query, model, limit, collectionName, session, precomputedEmbedding) => searchVec(db, query, model, limit, collectionName, session, precomputedEmbedding),
|
|
669
669
|
// Query expansion & reranking
|
|
670
|
-
expandQuery: (query, model) => expandQuery(query, model, db),
|
|
671
|
-
rerank: (query, documents, model) => rerank(query, documents, model, db),
|
|
670
|
+
expandQuery: (query, model, intent) => expandQuery(query, model, db, intent),
|
|
671
|
+
rerank: (query, documents, model, intent) => rerank(query, documents, model, db, intent),
|
|
672
672
|
// Document retrieval
|
|
673
673
|
findDocument: (filename, options) => findDocument(db, filename, options),
|
|
674
674
|
getDocumentBody: (doc, fromLine, maxLines) => getDocumentBody(db, doc, fromLine, maxLines),
|
|
@@ -1798,9 +1798,9 @@ export function insertEmbedding(db, hash, seq, pos, embedding, model, embeddedAt
|
|
|
1798
1798
|
// =============================================================================
|
|
1799
1799
|
// Query expansion
|
|
1800
1800
|
// =============================================================================
|
|
1801
|
-
export async function expandQuery(query, model = DEFAULT_QUERY_MODEL, db) {
|
|
1801
|
+
export async function expandQuery(query, model = DEFAULT_QUERY_MODEL, db, intent) {
|
|
1802
1802
|
// Check cache first — stored as JSON preserving types
|
|
1803
|
-
const cacheKey = getCacheKey("expandQuery", { query, model });
|
|
1803
|
+
const cacheKey = getCacheKey("expandQuery", { query, model, ...(intent && { intent }) });
|
|
1804
1804
|
const cached = getCachedResult(db, cacheKey);
|
|
1805
1805
|
if (cached) {
|
|
1806
1806
|
try {
|
|
@@ -1812,7 +1812,7 @@ export async function expandQuery(query, model = DEFAULT_QUERY_MODEL, db) {
|
|
|
1812
1812
|
}
|
|
1813
1813
|
const llm = getDefaultLlamaCpp();
|
|
1814
1814
|
// Note: LlamaCpp uses hardcoded model, model parameter is ignored
|
|
1815
|
-
const results = await llm.expandQuery(query);
|
|
1815
|
+
const results = await llm.expandQuery(query, { intent });
|
|
1816
1816
|
// Map Queryable[] → ExpandedQuery[] (same shape, decoupled from llm.ts internals).
|
|
1817
1817
|
// Filter out entries that duplicate the original query text.
|
|
1818
1818
|
const expanded = results
|
|
@@ -1826,7 +1826,9 @@ export async function expandQuery(query, model = DEFAULT_QUERY_MODEL, db) {
|
|
|
1826
1826
|
// =============================================================================
|
|
1827
1827
|
// Reranking
|
|
1828
1828
|
// =============================================================================
|
|
1829
|
-
export async function rerank(query, documents, model = DEFAULT_RERANK_MODEL, db) {
|
|
1829
|
+
export async function rerank(query, documents, model = DEFAULT_RERANK_MODEL, db, intent) {
|
|
1830
|
+
// Prepend intent to rerank query so the reranker scores with domain context
|
|
1831
|
+
const rerankQuery = intent ? `${intent}\n\n${query}` : query;
|
|
1830
1832
|
const cachedResults = new Map();
|
|
1831
1833
|
const uncachedDocsByChunk = new Map();
|
|
1832
1834
|
// Check cache for each document
|
|
@@ -1835,7 +1837,7 @@ export async function rerank(query, documents, model = DEFAULT_RERANK_MODEL, db)
|
|
|
1835
1837
|
// File path is excluded from the new cache key because the reranker score
|
|
1836
1838
|
// depends on the chunk content, not where it came from.
|
|
1837
1839
|
for (const doc of documents) {
|
|
1838
|
-
const cacheKey = getCacheKey("rerank", { query, model, chunk: doc.text });
|
|
1840
|
+
const cacheKey = getCacheKey("rerank", { query: rerankQuery, model, chunk: doc.text });
|
|
1839
1841
|
const legacyCacheKey = getCacheKey("rerank", { query, file: doc.file, model, chunk: doc.text });
|
|
1840
1842
|
const cached = getCachedResult(db, cacheKey) ?? getCachedResult(db, legacyCacheKey);
|
|
1841
1843
|
if (cached !== null) {
|
|
@@ -1849,12 +1851,12 @@ export async function rerank(query, documents, model = DEFAULT_RERANK_MODEL, db)
|
|
|
1849
1851
|
if (uncachedDocsByChunk.size > 0) {
|
|
1850
1852
|
const llm = getDefaultLlamaCpp();
|
|
1851
1853
|
const uncachedDocs = [...uncachedDocsByChunk.values()];
|
|
1852
|
-
const rerankResult = await llm.rerank(
|
|
1854
|
+
const rerankResult = await llm.rerank(rerankQuery, uncachedDocs, { model });
|
|
1853
1855
|
// Cache results by chunk text so identical chunks across files are scored once.
|
|
1854
1856
|
const textByFile = new Map(uncachedDocs.map(d => [d.file, d.text]));
|
|
1855
1857
|
for (const result of rerankResult.results) {
|
|
1856
1858
|
const chunk = textByFile.get(result.file) || "";
|
|
1857
|
-
const cacheKey = getCacheKey("rerank", { query, model, chunk });
|
|
1859
|
+
const cacheKey = getCacheKey("rerank", { query: rerankQuery, model, chunk });
|
|
1858
1860
|
setCachedResult(db, cacheKey, result.score.toString());
|
|
1859
1861
|
cachedResults.set(chunk, result.score);
|
|
1860
1862
|
}
|
|
@@ -2254,7 +2256,41 @@ export function getStatus(db) {
|
|
|
2254
2256
|
collections,
|
|
2255
2257
|
};
|
|
2256
2258
|
}
|
|
2257
|
-
|
|
2259
|
+
/** Weight for intent terms relative to query terms (1.0) in snippet scoring */
|
|
2260
|
+
export const INTENT_WEIGHT_SNIPPET = 0.3;
|
|
2261
|
+
/** Weight for intent terms relative to query terms (1.0) in chunk selection */
|
|
2262
|
+
export const INTENT_WEIGHT_CHUNK = 0.5;
|
|
2263
|
+
// Common stop words filtered from intent strings before tokenization.
|
|
2264
|
+
// Seeded from finetune/reward.py KEY_TERM_STOPWORDS, extended with common
|
|
2265
|
+
// 2-3 char function words so the length threshold can drop to >1 and let
|
|
2266
|
+
// short domain terms (API, SQL, LLM, CPU, CDN, …) survive.
|
|
2267
|
+
const INTENT_STOP_WORDS = new Set([
|
|
2268
|
+
// 2-char function words
|
|
2269
|
+
"am", "an", "as", "at", "be", "by", "do", "he", "if",
|
|
2270
|
+
"in", "is", "it", "me", "my", "no", "of", "on", "or", "so",
|
|
2271
|
+
"to", "up", "us", "we",
|
|
2272
|
+
// 3-char function words
|
|
2273
|
+
"all", "and", "any", "are", "but", "can", "did", "for", "get",
|
|
2274
|
+
"has", "her", "him", "his", "how", "its", "let", "may", "not",
|
|
2275
|
+
"our", "out", "the", "too", "was", "who", "why", "you",
|
|
2276
|
+
// 4+ char common words
|
|
2277
|
+
"also", "does", "find", "from", "have", "into", "more", "need",
|
|
2278
|
+
"show", "some", "tell", "that", "them", "this", "want", "what",
|
|
2279
|
+
"when", "will", "with", "your",
|
|
2280
|
+
// Search-context noise
|
|
2281
|
+
"about", "looking", "notes", "search", "where", "which",
|
|
2282
|
+
]);
|
|
2283
|
+
/**
|
|
2284
|
+
* Extract meaningful terms from an intent string, filtering stop words and punctuation.
|
|
2285
|
+
* Uses Unicode-aware punctuation stripping so domain terms like "API" survive.
|
|
2286
|
+
* Returns lowercase terms suitable for text matching.
|
|
2287
|
+
*/
|
|
2288
|
+
export function extractIntentTerms(intent) {
|
|
2289
|
+
return intent.toLowerCase().split(/\s+/)
|
|
2290
|
+
.map(t => t.replace(/^[^\p{L}\p{N}]+|[^\p{L}\p{N}]+$/gu, ""))
|
|
2291
|
+
.filter(t => t.length > 1 && !INTENT_STOP_WORDS.has(t));
|
|
2292
|
+
}
|
|
2293
|
+
export function extractSnippet(body, query, maxLen = 500, chunkPos, chunkLen, intent) {
|
|
2258
2294
|
const totalLines = body.split('\n').length;
|
|
2259
2295
|
let searchBody = body;
|
|
2260
2296
|
let lineOffset = 0;
|
|
@@ -2271,13 +2307,18 @@ export function extractSnippet(body, query, maxLen = 500, chunkPos, chunkLen) {
|
|
|
2271
2307
|
}
|
|
2272
2308
|
const lines = searchBody.split('\n');
|
|
2273
2309
|
const queryTerms = query.toLowerCase().split(/\s+/).filter(t => t.length > 0);
|
|
2310
|
+
const intentTerms = intent ? extractIntentTerms(intent) : [];
|
|
2274
2311
|
let bestLine = 0, bestScore = -1;
|
|
2275
2312
|
for (let i = 0; i < lines.length; i++) {
|
|
2276
2313
|
const lineLower = (lines[i] ?? "").toLowerCase();
|
|
2277
2314
|
let score = 0;
|
|
2278
2315
|
for (const term of queryTerms) {
|
|
2279
2316
|
if (lineLower.includes(term))
|
|
2280
|
-
score
|
|
2317
|
+
score += 1.0;
|
|
2318
|
+
}
|
|
2319
|
+
for (const term of intentTerms) {
|
|
2320
|
+
if (lineLower.includes(term))
|
|
2321
|
+
score += INTENT_WEIGHT_SNIPPET;
|
|
2281
2322
|
}
|
|
2282
2323
|
if (score > bestScore) {
|
|
2283
2324
|
bestScore = score;
|
|
@@ -2291,7 +2332,7 @@ export function extractSnippet(body, query, maxLen = 500, chunkPos, chunkLen) {
|
|
|
2291
2332
|
// If we focused on a chunk window and it produced an empty/whitespace-only snippet,
|
|
2292
2333
|
// fall back to a full-document snippet so we always show something useful.
|
|
2293
2334
|
if (chunkPos && chunkPos > 0 && snippetText.trim().length === 0) {
|
|
2294
|
-
return extractSnippet(body, query, maxLen, undefined);
|
|
2335
|
+
return extractSnippet(body, query, maxLen, undefined, undefined, intent);
|
|
2295
2336
|
}
|
|
2296
2337
|
if (snippetText.length > maxLen)
|
|
2297
2338
|
snippetText = snippetText.substring(0, maxLen - 3) + "...";
|
|
@@ -2340,17 +2381,21 @@ export async function hybridQuery(store, query, options) {
|
|
|
2340
2381
|
const candidateLimit = options?.candidateLimit ?? RERANK_CANDIDATE_LIMIT;
|
|
2341
2382
|
const collection = options?.collection;
|
|
2342
2383
|
const explain = options?.explain ?? false;
|
|
2384
|
+
const intent = options?.intent;
|
|
2343
2385
|
const hooks = options?.hooks;
|
|
2344
2386
|
const rankedLists = [];
|
|
2345
2387
|
const rankedListMeta = [];
|
|
2346
2388
|
const docidMap = new Map(); // filepath -> docid
|
|
2347
2389
|
const hasVectors = !!store.db.prepare(`SELECT name FROM sqlite_master WHERE type='table' AND name='vectors_vec'`).get();
|
|
2348
2390
|
// Step 1: BM25 probe — strong signal skips expensive LLM expansion
|
|
2391
|
+
// When intent is provided, disable strong-signal bypass — the obvious BM25
|
|
2392
|
+
// match may not be what the caller wants (e.g. "performance" with intent
|
|
2393
|
+
// "web page load times" should NOT shortcut to a sports-performance doc).
|
|
2349
2394
|
// Pass collection directly into FTS query (filter at SQL level, not post-hoc)
|
|
2350
2395
|
const initialFts = store.searchFTS(query, 20, collection);
|
|
2351
2396
|
const topScore = initialFts[0]?.score ?? 0;
|
|
2352
2397
|
const secondScore = initialFts[1]?.score ?? 0;
|
|
2353
|
-
const hasStrongSignal = initialFts.length > 0
|
|
2398
|
+
const hasStrongSignal = !intent && initialFts.length > 0
|
|
2354
2399
|
&& topScore >= STRONG_SIGNAL_MIN_SCORE
|
|
2355
2400
|
&& (topScore - secondScore) >= STRONG_SIGNAL_MIN_GAP;
|
|
2356
2401
|
if (hasStrongSignal)
|
|
@@ -2360,7 +2405,7 @@ export async function hybridQuery(store, query, options) {
|
|
|
2360
2405
|
const expandStart = Date.now();
|
|
2361
2406
|
const expanded = hasStrongSignal
|
|
2362
2407
|
? []
|
|
2363
|
-
: await store.expandQuery(query);
|
|
2408
|
+
: await store.expandQuery(query, undefined, intent);
|
|
2364
2409
|
hooks?.onExpand?.(query, expanded, Date.now() - expandStart);
|
|
2365
2410
|
// Seed with initial FTS results (avoid re-running original query FTS)
|
|
2366
2411
|
if (initialFts.length > 0) {
|
|
@@ -2440,6 +2485,7 @@ export async function hybridQuery(store, query, options) {
|
|
|
2440
2485
|
// Step 5: Chunk documents, pick best chunk per doc for reranking.
|
|
2441
2486
|
// Reranking full bodies is O(tokens) — the critical perf lesson that motivated this refactor.
|
|
2442
2487
|
const queryTerms = query.toLowerCase().split(/\s+/).filter(t => t.length > 2);
|
|
2488
|
+
const intentTerms = intent ? extractIntentTerms(intent) : [];
|
|
2443
2489
|
const chunksToRerank = [];
|
|
2444
2490
|
const docChunkMap = new Map();
|
|
2445
2491
|
for (const cand of candidates) {
|
|
@@ -2447,11 +2493,16 @@ export async function hybridQuery(store, query, options) {
|
|
|
2447
2493
|
if (chunks.length === 0)
|
|
2448
2494
|
continue;
|
|
2449
2495
|
// Pick chunk with most keyword overlap (fallback: first chunk)
|
|
2496
|
+
// Intent terms contribute at INTENT_WEIGHT_CHUNK (0.5) relative to query terms (1.0)
|
|
2450
2497
|
let bestIdx = 0;
|
|
2451
2498
|
let bestScore = -1;
|
|
2452
2499
|
for (let i = 0; i < chunks.length; i++) {
|
|
2453
2500
|
const chunkLower = chunks[i].text.toLowerCase();
|
|
2454
|
-
|
|
2501
|
+
let score = queryTerms.reduce((acc, term) => acc + (chunkLower.includes(term) ? 1 : 0), 0);
|
|
2502
|
+
for (const term of intentTerms) {
|
|
2503
|
+
if (chunkLower.includes(term))
|
|
2504
|
+
score += INTENT_WEIGHT_CHUNK;
|
|
2505
|
+
}
|
|
2455
2506
|
if (score > bestScore) {
|
|
2456
2507
|
bestScore = score;
|
|
2457
2508
|
bestIdx = i;
|
|
@@ -2463,7 +2514,7 @@ export async function hybridQuery(store, query, options) {
|
|
|
2463
2514
|
// Step 6: Rerank chunks (NOT full bodies)
|
|
2464
2515
|
hooks?.onRerankStart?.(chunksToRerank.length);
|
|
2465
2516
|
const rerankStart = Date.now();
|
|
2466
|
-
const reranked = await store.rerank(query, chunksToRerank);
|
|
2517
|
+
const reranked = await store.rerank(query, chunksToRerank, undefined, intent);
|
|
2467
2518
|
hooks?.onRerankDone?.(Date.now() - rerankStart);
|
|
2468
2519
|
// Step 7: Blend RRF position score with reranker score
|
|
2469
2520
|
// Position-aware weights: top retrieval results get more protection from reranker disagreement
|
|
@@ -2541,12 +2592,13 @@ export async function vectorSearchQuery(store, query, options) {
|
|
|
2541
2592
|
const limit = options?.limit ?? 10;
|
|
2542
2593
|
const minScore = options?.minScore ?? 0.3;
|
|
2543
2594
|
const collection = options?.collection;
|
|
2595
|
+
const intent = options?.intent;
|
|
2544
2596
|
const hasVectors = !!store.db.prepare(`SELECT name FROM sqlite_master WHERE type='table' AND name='vectors_vec'`).get();
|
|
2545
2597
|
if (!hasVectors)
|
|
2546
2598
|
return [];
|
|
2547
2599
|
// Expand query — filter to vec/hyde only (lex queries target FTS, not vector)
|
|
2548
2600
|
const expandStart = Date.now();
|
|
2549
|
-
const allExpanded = await store.expandQuery(query);
|
|
2601
|
+
const allExpanded = await store.expandQuery(query, undefined, intent);
|
|
2550
2602
|
const vecExpanded = allExpanded.filter(q => q.type !== 'lex');
|
|
2551
2603
|
options?.hooks?.onExpand?.(query, vecExpanded, Date.now() - expandStart);
|
|
2552
2604
|
// Run original + vec/hyde expanded through vector, sequentially — concurrent embed() hangs
|
|
@@ -2597,6 +2649,7 @@ export async function structuredSearch(store, searches, options) {
|
|
|
2597
2649
|
const minScore = options?.minScore ?? 0;
|
|
2598
2650
|
const candidateLimit = options?.candidateLimit ?? RERANK_CANDIDATE_LIMIT;
|
|
2599
2651
|
const explain = options?.explain ?? false;
|
|
2652
|
+
const intent = options?.intent;
|
|
2600
2653
|
const hooks = options?.hooks;
|
|
2601
2654
|
const collections = options?.collections;
|
|
2602
2655
|
if (searches.length === 0)
|
|
@@ -2696,6 +2749,7 @@ export async function structuredSearch(store, searches, options) {
|
|
|
2696
2749
|
|| searches.find(s => s.type === 'vec')?.query
|
|
2697
2750
|
|| searches[0]?.query || "";
|
|
2698
2751
|
const queryTerms = primaryQuery.toLowerCase().split(/\s+/).filter(t => t.length > 2);
|
|
2752
|
+
const intentTerms = intent ? extractIntentTerms(intent) : [];
|
|
2699
2753
|
const chunksToRerank = [];
|
|
2700
2754
|
const docChunkMap = new Map();
|
|
2701
2755
|
for (const cand of candidates) {
|
|
@@ -2703,11 +2757,16 @@ export async function structuredSearch(store, searches, options) {
|
|
|
2703
2757
|
if (chunks.length === 0)
|
|
2704
2758
|
continue;
|
|
2705
2759
|
// Pick chunk with most keyword overlap
|
|
2760
|
+
// Intent terms contribute at INTENT_WEIGHT_CHUNK (0.5) relative to query terms (1.0)
|
|
2706
2761
|
let bestIdx = 0;
|
|
2707
2762
|
let bestScore = -1;
|
|
2708
2763
|
for (let i = 0; i < chunks.length; i++) {
|
|
2709
2764
|
const chunkLower = chunks[i].text.toLowerCase();
|
|
2710
|
-
|
|
2765
|
+
let score = queryTerms.reduce((acc, term) => acc + (chunkLower.includes(term) ? 1 : 0), 0);
|
|
2766
|
+
for (const term of intentTerms) {
|
|
2767
|
+
if (chunkLower.includes(term))
|
|
2768
|
+
score += INTENT_WEIGHT_CHUNK;
|
|
2769
|
+
}
|
|
2711
2770
|
if (score > bestScore) {
|
|
2712
2771
|
bestScore = score;
|
|
2713
2772
|
bestIdx = i;
|
|
@@ -2719,7 +2778,7 @@ export async function structuredSearch(store, searches, options) {
|
|
|
2719
2778
|
// Step 5: Rerank chunks
|
|
2720
2779
|
hooks?.onRerankStart?.(chunksToRerank.length);
|
|
2721
2780
|
const rerankStart2 = Date.now();
|
|
2722
|
-
const reranked = await store.rerank(primaryQuery, chunksToRerank);
|
|
2781
|
+
const reranked = await store.rerank(primaryQuery, chunksToRerank, undefined, intent);
|
|
2723
2782
|
hooks?.onRerankDone?.(Date.now() - rerankStart2);
|
|
2724
2783
|
// Step 6: Blend RRF position score with reranker score
|
|
2725
2784
|
const candidateMap = new Map(candidates.map(c => [c.file, {
|
package/package.json
CHANGED
|
@@ -1,8 +1,16 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@tobilu/qmd",
|
|
3
|
-
"version": "1.1.
|
|
3
|
+
"version": "1.1.6",
|
|
4
4
|
"description": "Query Markup Documents - On-device hybrid search for markdown files with BM25, vector search, and LLM reranking",
|
|
5
5
|
"type": "module",
|
|
6
|
+
"main": "dist/index.js",
|
|
7
|
+
"types": "dist/index.d.ts",
|
|
8
|
+
"exports": {
|
|
9
|
+
".": {
|
|
10
|
+
"import": "./dist/index.js",
|
|
11
|
+
"types": "./dist/index.d.ts"
|
|
12
|
+
}
|
|
13
|
+
},
|
|
6
14
|
"bin": {
|
|
7
15
|
"qmd": "dist/qmd.js"
|
|
8
16
|
},
|