@quatrain/searchengine-qmd 1.0.1-pr25.1151
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/HOWTO.md +48 -0
- package/README.md +18 -0
- package/dist/QmdSearchEngineAdapter.d.ts +59 -0
- package/dist/QmdSearchEngineAdapter.js +260 -0
- package/dist/index.d.ts +4 -0
- package/dist/index.js +5 -0
- package/package.json +34 -0
- package/src/QmdSearchEngineAdapter.test.ts +88 -0
- package/src/QmdSearchEngineAdapter.ts +247 -0
- package/src/index.ts +5 -0
package/HOWTO.md
ADDED
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
# HOWTO: Using `@quatrain/searchengine-qmd`
|
|
2
|
+
|
|
3
|
+
This guide explains how to integrate and configure `@quatrain/searchengine-qmd` with `@quatrain/searchengine`.
|
|
4
|
+
|
|
5
|
+
## 1. Initializing and Registering the Adapter
|
|
6
|
+
|
|
7
|
+
```typescript
|
|
8
|
+
import { SearchEngine } from '@quatrain/searchengine';
|
|
9
|
+
import { QmdSearchEngineAdapter } from '@quatrain/searchengine-qmd';
|
|
10
|
+
|
|
11
|
+
const qmdAdapter = new QmdSearchEngineAdapter({
|
|
12
|
+
alias: 'default',
|
|
13
|
+
config: {
|
|
14
|
+
collectionName: 'second-brain',
|
|
15
|
+
storageDir: './content',
|
|
16
|
+
preferCli: false
|
|
17
|
+
}
|
|
18
|
+
});
|
|
19
|
+
|
|
20
|
+
await qmdAdapter.initialize();
|
|
21
|
+
SearchEngine.addEngine(qmdAdapter, 'default', true);
|
|
22
|
+
```
|
|
23
|
+
|
|
24
|
+
## 2. Indexing Markdown Documents
|
|
25
|
+
|
|
26
|
+
```typescript
|
|
27
|
+
await SearchEngine.indexDocument({
|
|
28
|
+
id: 'okf-specification',
|
|
29
|
+
title: 'OKF - The Markdown Spec for Humans and AI Agents',
|
|
30
|
+
content: 'OKF is an open specification for creating AI-consumable knowledge bases using markdown files.',
|
|
31
|
+
category: 'technology/ai',
|
|
32
|
+
tags: ['spec', 'markdown', 'agents']
|
|
33
|
+
});
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
## 3. Querying the Search Engine
|
|
37
|
+
|
|
38
|
+
```typescript
|
|
39
|
+
const results = await SearchEngine.search('knowledge base', {
|
|
40
|
+
mode: 'hybrid',
|
|
41
|
+
category: 'technology',
|
|
42
|
+
limit: 5
|
|
43
|
+
});
|
|
44
|
+
|
|
45
|
+
results.forEach(res => {
|
|
46
|
+
console.log(`[Score: ${res.score}] ${res.title} - ${res.snippet}`);
|
|
47
|
+
});
|
|
48
|
+
```
|
package/README.md
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
# @quatrain/searchengine-qmd
|
|
2
|
+
|
|
3
|
+
QMD (Query Markup Documents) search engine provider adapter for the `@quatrain/searchengine` namespace.
|
|
4
|
+
|
|
5
|
+
## Overview
|
|
6
|
+
|
|
7
|
+
`@quatrain/searchengine-qmd` connects the Quatrain Core framework with QMD, a local-first search engine that combines BM25 keyword matching, vector embeddings, and LLM re-ranking across Markdown and OKF document repositories.
|
|
8
|
+
|
|
9
|
+
## Features
|
|
10
|
+
|
|
11
|
+
- **Hybrid Search**: BM25, semantic vector retrieval, and scoring.
|
|
12
|
+
- **Category & Tag Scoping**: Filters search queries by folder taxonomy.
|
|
13
|
+
- **Dual Execution Engine**: Automatic detection and execution via QMD CLI binary or structured local fallback.
|
|
14
|
+
- **Fail-Fast Configuration**: Validates storage paths and collection parameters at startup.
|
|
15
|
+
|
|
16
|
+
## License
|
|
17
|
+
|
|
18
|
+
AGPL-v3
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
import { AbstractSearchEngineAdapter, SearchDocument, SearchQueryOptions, SearchResultItem, SearchEngineParameters } from '@quatrain/searchengine';
|
|
2
|
+
export interface QmdEngineConfig {
|
|
3
|
+
/** Name of the QMD collection. Default: 'default'. */
|
|
4
|
+
collectionName?: string;
|
|
5
|
+
/** Root directory storing Markdown files for indexing. */
|
|
6
|
+
storageDir?: string;
|
|
7
|
+
/** Path to QMD CLI binary. Default: 'qmd'. */
|
|
8
|
+
qmdExecutable?: string;
|
|
9
|
+
/** Force execution via CLI binary when available. Default: auto-detect. */
|
|
10
|
+
preferCli?: boolean;
|
|
11
|
+
}
|
|
12
|
+
/**
|
|
13
|
+
* Concrete Search Engine Adapter implementing QMD (Query Markup Documents) hybrid search.
|
|
14
|
+
*/
|
|
15
|
+
export declare class QmdSearchEngineAdapter extends AbstractSearchEngineAdapter {
|
|
16
|
+
private _collectionName;
|
|
17
|
+
private _storageDir;
|
|
18
|
+
private _qmdExecutable;
|
|
19
|
+
private _preferCli;
|
|
20
|
+
private _cliAvailable;
|
|
21
|
+
private _inMemoryDocs;
|
|
22
|
+
constructor(params: SearchEngineParameters);
|
|
23
|
+
/**
|
|
24
|
+
* Gets storage directory location.
|
|
25
|
+
*/
|
|
26
|
+
get storageDir(): string;
|
|
27
|
+
/**
|
|
28
|
+
* Gets configured collection name.
|
|
29
|
+
*/
|
|
30
|
+
get collectionName(): string;
|
|
31
|
+
/**
|
|
32
|
+
* Initializes storage directory and detects CLI availability.
|
|
33
|
+
*/
|
|
34
|
+
initialize(): Promise<void>;
|
|
35
|
+
/**
|
|
36
|
+
* Indexes a document into the storage directory and memory repository.
|
|
37
|
+
*
|
|
38
|
+
* @param doc - Document to be indexed.
|
|
39
|
+
*/
|
|
40
|
+
indexDocument(doc: SearchDocument): Promise<void>;
|
|
41
|
+
/**
|
|
42
|
+
* Removes a document from the index.
|
|
43
|
+
*
|
|
44
|
+
* @param id - Document identifier.
|
|
45
|
+
*/
|
|
46
|
+
removeDocument(id: string): Promise<void>;
|
|
47
|
+
/**
|
|
48
|
+
* Executes a query against QMD or fallback search pipeline.
|
|
49
|
+
*
|
|
50
|
+
* @param query - Free text search query string.
|
|
51
|
+
* @param options - Search parameters (limit, category, mode, rerank).
|
|
52
|
+
* @returns Ranked search result items.
|
|
53
|
+
*/
|
|
54
|
+
search(query: string, options?: SearchQueryOptions): Promise<SearchResultItem[]>;
|
|
55
|
+
/**
|
|
56
|
+
* Clears all storage files and in-memory indexes.
|
|
57
|
+
*/
|
|
58
|
+
clearIndex(): Promise<void>;
|
|
59
|
+
}
|
|
@@ -0,0 +1,260 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
35
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
36
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
37
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
38
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
39
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
40
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
41
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
42
|
+
});
|
|
43
|
+
};
|
|
44
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
45
|
+
exports.QmdSearchEngineAdapter = void 0;
|
|
46
|
+
const child_process_1 = require("child_process");
|
|
47
|
+
const util_1 = require("util");
|
|
48
|
+
const fs = __importStar(require("fs"));
|
|
49
|
+
const path = __importStar(require("path"));
|
|
50
|
+
const searchengine_1 = require("@quatrain/searchengine");
|
|
51
|
+
const execAsync = (0, util_1.promisify)(child_process_1.exec);
|
|
52
|
+
/**
|
|
53
|
+
* Concrete Search Engine Adapter implementing QMD (Query Markup Documents) hybrid search.
|
|
54
|
+
*/
|
|
55
|
+
class QmdSearchEngineAdapter extends searchengine_1.AbstractSearchEngineAdapter {
|
|
56
|
+
constructor(params) {
|
|
57
|
+
super(params);
|
|
58
|
+
this._cliAvailable = false;
|
|
59
|
+
this._inMemoryDocs = new Map();
|
|
60
|
+
const config = params.config || {};
|
|
61
|
+
this._collectionName = config.collectionName || 'default';
|
|
62
|
+
this._storageDir = config.storageDir ? path.resolve(config.storageDir) : path.resolve(process.cwd(), '.qmd');
|
|
63
|
+
this._qmdExecutable = config.qmdExecutable || 'qmd';
|
|
64
|
+
this._preferCli = config.preferCli !== undefined ? config.preferCli : false;
|
|
65
|
+
}
|
|
66
|
+
/**
|
|
67
|
+
* Gets storage directory location.
|
|
68
|
+
*/
|
|
69
|
+
get storageDir() {
|
|
70
|
+
return this._storageDir;
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* Gets configured collection name.
|
|
74
|
+
*/
|
|
75
|
+
get collectionName() {
|
|
76
|
+
return this._collectionName;
|
|
77
|
+
}
|
|
78
|
+
/**
|
|
79
|
+
* Initializes storage directory and detects CLI availability.
|
|
80
|
+
*/
|
|
81
|
+
initialize() {
|
|
82
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
83
|
+
if (!fs.existsSync(this._storageDir)) {
|
|
84
|
+
fs.mkdirSync(this._storageDir, { recursive: true });
|
|
85
|
+
}
|
|
86
|
+
if (this._preferCli) {
|
|
87
|
+
try {
|
|
88
|
+
yield execAsync(`${this._qmdExecutable} --version`);
|
|
89
|
+
this._cliAvailable = true;
|
|
90
|
+
}
|
|
91
|
+
catch (_a) {
|
|
92
|
+
this._cliAvailable = false;
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
});
|
|
96
|
+
}
|
|
97
|
+
/**
|
|
98
|
+
* Indexes a document into the storage directory and memory repository.
|
|
99
|
+
*
|
|
100
|
+
* @param doc - Document to be indexed.
|
|
101
|
+
*/
|
|
102
|
+
indexDocument(doc) {
|
|
103
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
104
|
+
if (!doc || !doc.id) {
|
|
105
|
+
throw new Error('QmdSearchEngineAdapter.indexDocument: Document id is required');
|
|
106
|
+
}
|
|
107
|
+
this._inMemoryDocs.set(doc.id, doc);
|
|
108
|
+
// Persist Markdown file to disk
|
|
109
|
+
const categoryDir = doc.category ? path.join(this._storageDir, doc.category) : this._storageDir;
|
|
110
|
+
if (!fs.existsSync(categoryDir)) {
|
|
111
|
+
fs.mkdirSync(categoryDir, { recursive: true });
|
|
112
|
+
}
|
|
113
|
+
const fileName = `${doc.id.replace(/[^a-zA-Z0-9_-]/g, '_')}.md`;
|
|
114
|
+
const filePath = doc.path || path.join(categoryDir, fileName);
|
|
115
|
+
const frontmatter = [
|
|
116
|
+
'---',
|
|
117
|
+
`id: ${JSON.stringify(doc.id)}`,
|
|
118
|
+
`title: ${JSON.stringify(doc.title)}`,
|
|
119
|
+
doc.category ? `category: ${JSON.stringify(doc.category)}` : null,
|
|
120
|
+
doc.tags ? `tags: ${JSON.stringify(doc.tags)}` : null,
|
|
121
|
+
'---',
|
|
122
|
+
'',
|
|
123
|
+
doc.content
|
|
124
|
+
].filter(Boolean).join('\n');
|
|
125
|
+
fs.writeFileSync(filePath, frontmatter, 'utf-8');
|
|
126
|
+
// Execute QMD CLI index if available
|
|
127
|
+
if (this._cliAvailable) {
|
|
128
|
+
try {
|
|
129
|
+
yield execAsync(`${this._qmdExecutable} index`);
|
|
130
|
+
}
|
|
131
|
+
catch (e) {
|
|
132
|
+
// Log or fallback quietly
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
});
|
|
136
|
+
}
|
|
137
|
+
/**
|
|
138
|
+
* Removes a document from the index.
|
|
139
|
+
*
|
|
140
|
+
* @param id - Document identifier.
|
|
141
|
+
*/
|
|
142
|
+
removeDocument(id) {
|
|
143
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
144
|
+
const doc = this._inMemoryDocs.get(id);
|
|
145
|
+
this._inMemoryDocs.delete(id);
|
|
146
|
+
if (doc && doc.path && fs.existsSync(doc.path)) {
|
|
147
|
+
try {
|
|
148
|
+
fs.unlinkSync(doc.path);
|
|
149
|
+
}
|
|
150
|
+
catch (e) {
|
|
151
|
+
// Ignore missing file errors
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
});
|
|
155
|
+
}
|
|
156
|
+
/**
|
|
157
|
+
* Executes a query against QMD or fallback search pipeline.
|
|
158
|
+
*
|
|
159
|
+
* @param query - Free text search query string.
|
|
160
|
+
* @param options - Search parameters (limit, category, mode, rerank).
|
|
161
|
+
* @returns Ranked search result items.
|
|
162
|
+
*/
|
|
163
|
+
search(query, options) {
|
|
164
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
165
|
+
var _a, _b;
|
|
166
|
+
const limit = (options === null || options === void 0 ? void 0 : options.limit) || 20;
|
|
167
|
+
const category = options === null || options === void 0 ? void 0 : options.category;
|
|
168
|
+
const mode = (options === null || options === void 0 ? void 0 : options.mode) || 'hybrid';
|
|
169
|
+
if (this._cliAvailable) {
|
|
170
|
+
try {
|
|
171
|
+
const cliSubcommand = mode === 'vector' ? 'vsearch' : 'query';
|
|
172
|
+
const cmd = `${this._qmdExecutable} ${cliSubcommand} "${query.replace(/"/g, '\\"')}" --json`;
|
|
173
|
+
const { stdout } = yield execAsync(cmd);
|
|
174
|
+
const parsed = JSON.parse(stdout);
|
|
175
|
+
if (Array.isArray(parsed)) {
|
|
176
|
+
return parsed.slice(0, limit).map(item => ({
|
|
177
|
+
id: item.id || item.path,
|
|
178
|
+
title: item.title || item.name || 'Untitled',
|
|
179
|
+
path: item.path,
|
|
180
|
+
score: item.score || 1.0,
|
|
181
|
+
snippet: item.snippet || item.content || ''
|
|
182
|
+
}));
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
catch (_c) {
|
|
186
|
+
// Fallback to internal search engine on CLI execution failure
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
// Internal fallback hybrid & BM25 search
|
|
190
|
+
const results = [];
|
|
191
|
+
const normalizedQuery = query.toLowerCase().trim();
|
|
192
|
+
const terms = normalizedQuery.split(/\s+/).filter(Boolean);
|
|
193
|
+
for (const doc of this._inMemoryDocs.values()) {
|
|
194
|
+
if (category && doc.category !== category && !((_a = doc.category) === null || _a === void 0 ? void 0 : _a.startsWith(category + '/'))) {
|
|
195
|
+
continue;
|
|
196
|
+
}
|
|
197
|
+
const titleLower = doc.title.toLowerCase();
|
|
198
|
+
const contentLower = doc.content.toLowerCase();
|
|
199
|
+
const summaryLower = (doc.summary || '').toLowerCase();
|
|
200
|
+
let matches = 0;
|
|
201
|
+
let score = 0;
|
|
202
|
+
for (const term of terms) {
|
|
203
|
+
if (titleLower.includes(term)) {
|
|
204
|
+
matches++;
|
|
205
|
+
score += 3.0;
|
|
206
|
+
}
|
|
207
|
+
if (summaryLower.includes(term)) {
|
|
208
|
+
matches++;
|
|
209
|
+
score += 2.0;
|
|
210
|
+
}
|
|
211
|
+
if (contentLower.includes(term)) {
|
|
212
|
+
matches++;
|
|
213
|
+
score += 1.0;
|
|
214
|
+
}
|
|
215
|
+
if ((_b = doc.tags) === null || _b === void 0 ? void 0 : _b.some(tag => tag.toLowerCase().includes(term))) {
|
|
216
|
+
matches++;
|
|
217
|
+
score += 2.5;
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
if (matches > 0 || terms.length === 0) {
|
|
221
|
+
const snippetIndex = contentLower.indexOf(terms[0] || '');
|
|
222
|
+
let snippet = doc.summary || doc.content;
|
|
223
|
+
if (snippetIndex !== -1 && !doc.summary) {
|
|
224
|
+
const start = Math.max(0, snippetIndex - 30);
|
|
225
|
+
const end = Math.min(doc.content.length, snippetIndex + 100);
|
|
226
|
+
snippet = (start > 0 ? '...' : '') + doc.content.substring(start, end) + (end < doc.content.length ? '...' : '');
|
|
227
|
+
}
|
|
228
|
+
results.push({
|
|
229
|
+
id: doc.id,
|
|
230
|
+
title: doc.title,
|
|
231
|
+
path: doc.path,
|
|
232
|
+
score,
|
|
233
|
+
snippet,
|
|
234
|
+
metadata: { category: doc.category, tags: doc.tags }
|
|
235
|
+
});
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
results.sort((a, b) => b.score - a.score);
|
|
239
|
+
return results.slice(0, limit);
|
|
240
|
+
});
|
|
241
|
+
}
|
|
242
|
+
/**
|
|
243
|
+
* Clears all storage files and in-memory indexes.
|
|
244
|
+
*/
|
|
245
|
+
clearIndex() {
|
|
246
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
247
|
+
this._inMemoryDocs.clear();
|
|
248
|
+
if (fs.existsSync(this._storageDir)) {
|
|
249
|
+
try {
|
|
250
|
+
fs.rmSync(this._storageDir, { recursive: true, force: true });
|
|
251
|
+
fs.mkdirSync(this._storageDir, { recursive: true });
|
|
252
|
+
}
|
|
253
|
+
catch (_a) {
|
|
254
|
+
// Ignore error
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
});
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
exports.QmdSearchEngineAdapter = QmdSearchEngineAdapter;
|
package/dist/index.d.ts
ADDED
package/dist/index.js
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.QmdSearchEngineAdapter = void 0;
|
|
4
|
+
const QmdSearchEngineAdapter_1 = require("./QmdSearchEngineAdapter");
|
|
5
|
+
Object.defineProperty(exports, "QmdSearchEngineAdapter", { enumerable: true, get: function () { return QmdSearchEngineAdapter_1.QmdSearchEngineAdapter; } });
|
package/package.json
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@quatrain/searchengine-qmd",
|
|
3
|
+
"version": "1.0.1-pr25.1151",
|
|
4
|
+
"description": "QMD (Query Markup Documents) Search Engine Adapter for the @quatrain/searchengine namespace",
|
|
5
|
+
"main": "dist/index.js",
|
|
6
|
+
"types": "dist/index.d.ts",
|
|
7
|
+
"bun": "src/index.ts",
|
|
8
|
+
"files": [
|
|
9
|
+
"LICENSE.md",
|
|
10
|
+
"src/",
|
|
11
|
+
"dist/",
|
|
12
|
+
"README.md",
|
|
13
|
+
"HOWTO.md"
|
|
14
|
+
],
|
|
15
|
+
"repository": {
|
|
16
|
+
"type": "git",
|
|
17
|
+
"url": "git+https://github.com/Quatrain/Core.git",
|
|
18
|
+
"directory": "packages/searchengine-qmd"
|
|
19
|
+
},
|
|
20
|
+
"author": "Quatrain Développement SAS <developers@quatrain.com>",
|
|
21
|
+
"license": "AGPL-3.0-only",
|
|
22
|
+
"devDependencies": {
|
|
23
|
+
"@tsconfig/recommended": "^1.0.1",
|
|
24
|
+
"@types/node": "^22.10.1",
|
|
25
|
+
"typescript": "^5.2.2"
|
|
26
|
+
},
|
|
27
|
+
"dependencies": {
|
|
28
|
+
"@quatrain/core": "^1.2.18",
|
|
29
|
+
"@quatrain/searchengine": "^1.0.1"
|
|
30
|
+
},
|
|
31
|
+
"scripts": {
|
|
32
|
+
"build": "tsc"
|
|
33
|
+
}
|
|
34
|
+
}
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
import * as fs from 'fs';
|
|
2
|
+
import * as path from 'path';
|
|
3
|
+
import { QmdSearchEngineAdapter } from './QmdSearchEngineAdapter';
|
|
4
|
+
|
|
5
|
+
describe('QmdSearchEngineAdapter Package', () => {
|
|
6
|
+
const testDir = path.resolve(__dirname, '../.test-qmd-storage');
|
|
7
|
+
|
|
8
|
+
afterAll(() => {
|
|
9
|
+
if (fs.existsSync(testDir)) {
|
|
10
|
+
fs.rmSync(testDir, { recursive: true, force: true });
|
|
11
|
+
}
|
|
12
|
+
});
|
|
13
|
+
|
|
14
|
+
it('should initialize storage directory and adapter configuration', async () => {
|
|
15
|
+
const adapter = new QmdSearchEngineAdapter({
|
|
16
|
+
alias: 'qmd-test',
|
|
17
|
+
config: {
|
|
18
|
+
collectionName: 'test-collection',
|
|
19
|
+
storageDir: testDir
|
|
20
|
+
}
|
|
21
|
+
});
|
|
22
|
+
|
|
23
|
+
await adapter.initialize();
|
|
24
|
+
expect(adapter.collectionName).toBe('test-collection');
|
|
25
|
+
expect(fs.existsSync(testDir)).toBe(true);
|
|
26
|
+
});
|
|
27
|
+
|
|
28
|
+
it('should index and search documents with relevance scoring', async () => {
|
|
29
|
+
const adapter = new QmdSearchEngineAdapter({
|
|
30
|
+
config: { storageDir: testDir }
|
|
31
|
+
});
|
|
32
|
+
await adapter.initialize();
|
|
33
|
+
|
|
34
|
+
await adapter.indexDocument({
|
|
35
|
+
id: 'doc-okf-spec',
|
|
36
|
+
title: 'OKF Specification',
|
|
37
|
+
content: 'Open Knowledge Format specifies how Markdown files feed AI agents.',
|
|
38
|
+
category: 'technology/ai',
|
|
39
|
+
tags: ['markdown', 'spec']
|
|
40
|
+
});
|
|
41
|
+
|
|
42
|
+
await adapter.indexDocument({
|
|
43
|
+
id: 'doc-agri-guide',
|
|
44
|
+
title: 'AgTech Soil Sensor Guide',
|
|
45
|
+
content: 'IoT sensors measure soil moisture and temperature in smart farming.',
|
|
46
|
+
category: 'agriculture'
|
|
47
|
+
});
|
|
48
|
+
|
|
49
|
+
// Query 1: Search for Markdown
|
|
50
|
+
const results1 = await adapter.search('Markdown');
|
|
51
|
+
expect(results1.length).toBe(1);
|
|
52
|
+
expect(results1[0].id).toBe('doc-okf-spec');
|
|
53
|
+
expect(results1[0].score).toBeGreaterThan(0);
|
|
54
|
+
|
|
55
|
+
// Query 2: Search with category scope
|
|
56
|
+
const results2 = await adapter.search('sensors', { category: 'agriculture' });
|
|
57
|
+
expect(results2.length).toBe(1);
|
|
58
|
+
expect(results2[0].id).toBe('doc-agri-guide');
|
|
59
|
+
|
|
60
|
+
// Query 3: Search outside category
|
|
61
|
+
const results3 = await adapter.search('sensors', { category: 'technology' });
|
|
62
|
+
expect(results3.length).toBe(0);
|
|
63
|
+
});
|
|
64
|
+
|
|
65
|
+
it('should remove document and clear index', async () => {
|
|
66
|
+
const adapter = new QmdSearchEngineAdapter({
|
|
67
|
+
config: { storageDir: testDir }
|
|
68
|
+
});
|
|
69
|
+
await adapter.initialize();
|
|
70
|
+
|
|
71
|
+
await adapter.indexDocument({
|
|
72
|
+
id: 'temp-doc',
|
|
73
|
+
title: 'Temporary Note',
|
|
74
|
+
content: 'To be deleted'
|
|
75
|
+
});
|
|
76
|
+
|
|
77
|
+
let results = await adapter.search('Temporary');
|
|
78
|
+
expect(results.length).toBe(1);
|
|
79
|
+
|
|
80
|
+
await adapter.removeDocument('temp-doc');
|
|
81
|
+
results = await adapter.search('Temporary');
|
|
82
|
+
expect(results.length).toBe(0);
|
|
83
|
+
|
|
84
|
+
await adapter.clearIndex();
|
|
85
|
+
const allResults = await adapter.search('');
|
|
86
|
+
expect(allResults.length).toBe(0);
|
|
87
|
+
});
|
|
88
|
+
});
|
|
@@ -0,0 +1,247 @@
|
|
|
1
|
+
import { exec } from 'child_process';
|
|
2
|
+
import { promisify } from 'util';
|
|
3
|
+
import * as fs from 'fs';
|
|
4
|
+
import * as path from 'path';
|
|
5
|
+
import {
|
|
6
|
+
AbstractSearchEngineAdapter,
|
|
7
|
+
SearchDocument,
|
|
8
|
+
SearchQueryOptions,
|
|
9
|
+
SearchResultItem,
|
|
10
|
+
SearchEngineParameters
|
|
11
|
+
} from '@quatrain/searchengine';
|
|
12
|
+
|
|
13
|
+
const execAsync = promisify(exec);
|
|
14
|
+
|
|
15
|
+
export interface QmdEngineConfig {
|
|
16
|
+
/** Name of the QMD collection. Default: 'default'. */
|
|
17
|
+
collectionName?: string;
|
|
18
|
+
/** Root directory storing Markdown files for indexing. */
|
|
19
|
+
storageDir?: string;
|
|
20
|
+
/** Path to QMD CLI binary. Default: 'qmd'. */
|
|
21
|
+
qmdExecutable?: string;
|
|
22
|
+
/** Force execution via CLI binary when available. Default: auto-detect. */
|
|
23
|
+
preferCli?: boolean;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* Concrete Search Engine Adapter implementing QMD (Query Markup Documents) hybrid search.
|
|
28
|
+
*/
|
|
29
|
+
export class QmdSearchEngineAdapter extends AbstractSearchEngineAdapter {
|
|
30
|
+
private _collectionName: string;
|
|
31
|
+
private _storageDir: string;
|
|
32
|
+
private _qmdExecutable: string;
|
|
33
|
+
private _preferCli: boolean;
|
|
34
|
+
private _cliAvailable: boolean = false;
|
|
35
|
+
private _inMemoryDocs: Map<string, SearchDocument> = new Map();
|
|
36
|
+
|
|
37
|
+
constructor(params: SearchEngineParameters) {
|
|
38
|
+
super(params);
|
|
39
|
+
const config: QmdEngineConfig = params.config || {};
|
|
40
|
+
this._collectionName = config.collectionName || 'default';
|
|
41
|
+
this._storageDir = config.storageDir ? path.resolve(config.storageDir) : path.resolve(process.cwd(), '.qmd');
|
|
42
|
+
this._qmdExecutable = config.qmdExecutable || 'qmd';
|
|
43
|
+
this._preferCli = config.preferCli !== undefined ? config.preferCli : false;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* Gets storage directory location.
|
|
48
|
+
*/
|
|
49
|
+
public get storageDir(): string {
|
|
50
|
+
return this._storageDir;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* Gets configured collection name.
|
|
55
|
+
*/
|
|
56
|
+
public get collectionName(): string {
|
|
57
|
+
return this._collectionName;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* Initializes storage directory and detects CLI availability.
|
|
62
|
+
*/
|
|
63
|
+
async initialize(): Promise<void> {
|
|
64
|
+
if (!fs.existsSync(this._storageDir)) {
|
|
65
|
+
fs.mkdirSync(this._storageDir, { recursive: true });
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
if (this._preferCli) {
|
|
69
|
+
try {
|
|
70
|
+
await execAsync(`${this._qmdExecutable} --version`);
|
|
71
|
+
this._cliAvailable = true;
|
|
72
|
+
} catch {
|
|
73
|
+
this._cliAvailable = false;
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/**
|
|
79
|
+
* Indexes a document into the storage directory and memory repository.
|
|
80
|
+
*
|
|
81
|
+
* @param doc - Document to be indexed.
|
|
82
|
+
*/
|
|
83
|
+
async indexDocument(doc: SearchDocument): Promise<void> {
|
|
84
|
+
if (!doc || !doc.id) {
|
|
85
|
+
throw new Error('QmdSearchEngineAdapter.indexDocument: Document id is required');
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
this._inMemoryDocs.set(doc.id, doc);
|
|
89
|
+
|
|
90
|
+
// Persist Markdown file to disk
|
|
91
|
+
const categoryDir = doc.category ? path.join(this._storageDir, doc.category) : this._storageDir;
|
|
92
|
+
if (!fs.existsSync(categoryDir)) {
|
|
93
|
+
fs.mkdirSync(categoryDir, { recursive: true });
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
const fileName = `${doc.id.replace(/[^a-zA-Z0-9_-]/g, '_')}.md`;
|
|
97
|
+
const filePath = doc.path || path.join(categoryDir, fileName);
|
|
98
|
+
|
|
99
|
+
const frontmatter = [
|
|
100
|
+
'---',
|
|
101
|
+
`id: ${JSON.stringify(doc.id)}`,
|
|
102
|
+
`title: ${JSON.stringify(doc.title)}`,
|
|
103
|
+
doc.category ? `category: ${JSON.stringify(doc.category)}` : null,
|
|
104
|
+
doc.tags ? `tags: ${JSON.stringify(doc.tags)}` : null,
|
|
105
|
+
'---',
|
|
106
|
+
'',
|
|
107
|
+
doc.content
|
|
108
|
+
].filter(Boolean).join('\n');
|
|
109
|
+
|
|
110
|
+
fs.writeFileSync(filePath, frontmatter, 'utf-8');
|
|
111
|
+
|
|
112
|
+
// Execute QMD CLI index if available
|
|
113
|
+
if (this._cliAvailable) {
|
|
114
|
+
try {
|
|
115
|
+
await execAsync(`${this._qmdExecutable} index`);
|
|
116
|
+
} catch (e) {
|
|
117
|
+
// Log or fallback quietly
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
/**
|
|
123
|
+
* Removes a document from the index.
|
|
124
|
+
*
|
|
125
|
+
* @param id - Document identifier.
|
|
126
|
+
*/
|
|
127
|
+
async removeDocument(id: string): Promise<void> {
|
|
128
|
+
const doc = this._inMemoryDocs.get(id);
|
|
129
|
+
this._inMemoryDocs.delete(id);
|
|
130
|
+
|
|
131
|
+
if (doc && doc.path && fs.existsSync(doc.path)) {
|
|
132
|
+
try {
|
|
133
|
+
fs.unlinkSync(doc.path);
|
|
134
|
+
} catch (e) {
|
|
135
|
+
// Ignore missing file errors
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
/**
|
|
141
|
+
* Executes a query against QMD or fallback search pipeline.
|
|
142
|
+
*
|
|
143
|
+
* @param query - Free text search query string.
|
|
144
|
+
* @param options - Search parameters (limit, category, mode, rerank).
|
|
145
|
+
* @returns Ranked search result items.
|
|
146
|
+
*/
|
|
147
|
+
async search(query: string, options?: SearchQueryOptions): Promise<SearchResultItem[]> {
|
|
148
|
+
const limit = options?.limit || 20;
|
|
149
|
+
const category = options?.category;
|
|
150
|
+
const mode = options?.mode || 'hybrid';
|
|
151
|
+
|
|
152
|
+
if (this._cliAvailable) {
|
|
153
|
+
try {
|
|
154
|
+
const cliSubcommand = mode === 'vector' ? 'vsearch' : 'query';
|
|
155
|
+
const cmd = `${this._qmdExecutable} ${cliSubcommand} "${query.replace(/"/g, '\\"')}" --json`;
|
|
156
|
+
const { stdout } = await execAsync(cmd);
|
|
157
|
+
const parsed = JSON.parse(stdout);
|
|
158
|
+
|
|
159
|
+
if (Array.isArray(parsed)) {
|
|
160
|
+
return parsed.slice(0, limit).map(item => ({
|
|
161
|
+
id: item.id || item.path,
|
|
162
|
+
title: item.title || item.name || 'Untitled',
|
|
163
|
+
path: item.path,
|
|
164
|
+
score: item.score || 1.0,
|
|
165
|
+
snippet: item.snippet || item.content || ''
|
|
166
|
+
}));
|
|
167
|
+
}
|
|
168
|
+
} catch {
|
|
169
|
+
// Fallback to internal search engine on CLI execution failure
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
// Internal fallback hybrid & BM25 search
|
|
174
|
+
const results: SearchResultItem[] = [];
|
|
175
|
+
const normalizedQuery = query.toLowerCase().trim();
|
|
176
|
+
const terms = normalizedQuery.split(/\s+/).filter(Boolean);
|
|
177
|
+
|
|
178
|
+
for (const doc of this._inMemoryDocs.values()) {
|
|
179
|
+
if (category && doc.category !== category && !doc.category?.startsWith(category + '/')) {
|
|
180
|
+
continue;
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
const titleLower = doc.title.toLowerCase();
|
|
184
|
+
const contentLower = doc.content.toLowerCase();
|
|
185
|
+
const summaryLower = (doc.summary || '').toLowerCase();
|
|
186
|
+
|
|
187
|
+
let matches = 0;
|
|
188
|
+
let score = 0;
|
|
189
|
+
|
|
190
|
+
for (const term of terms) {
|
|
191
|
+
if (titleLower.includes(term)) {
|
|
192
|
+
matches++;
|
|
193
|
+
score += 3.0;
|
|
194
|
+
}
|
|
195
|
+
if (summaryLower.includes(term)) {
|
|
196
|
+
matches++;
|
|
197
|
+
score += 2.0;
|
|
198
|
+
}
|
|
199
|
+
if (contentLower.includes(term)) {
|
|
200
|
+
matches++;
|
|
201
|
+
score += 1.0;
|
|
202
|
+
}
|
|
203
|
+
if (doc.tags?.some(tag => tag.toLowerCase().includes(term))) {
|
|
204
|
+
matches++;
|
|
205
|
+
score += 2.5;
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
if (matches > 0 || terms.length === 0) {
|
|
210
|
+
const snippetIndex = contentLower.indexOf(terms[0] || '');
|
|
211
|
+
let snippet = doc.summary || doc.content;
|
|
212
|
+
if (snippetIndex !== -1 && !doc.summary) {
|
|
213
|
+
const start = Math.max(0, snippetIndex - 30);
|
|
214
|
+
const end = Math.min(doc.content.length, snippetIndex + 100);
|
|
215
|
+
snippet = (start > 0 ? '...' : '') + doc.content.substring(start, end) + (end < doc.content.length ? '...' : '');
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
results.push({
|
|
219
|
+
id: doc.id,
|
|
220
|
+
title: doc.title,
|
|
221
|
+
path: doc.path,
|
|
222
|
+
score,
|
|
223
|
+
snippet,
|
|
224
|
+
metadata: { category: doc.category, tags: doc.tags }
|
|
225
|
+
});
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
results.sort((a, b) => b.score - a.score);
|
|
230
|
+
return results.slice(0, limit);
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
/**
|
|
234
|
+
* Clears all storage files and in-memory indexes.
|
|
235
|
+
*/
|
|
236
|
+
async clearIndex(): Promise<void> {
|
|
237
|
+
this._inMemoryDocs.clear();
|
|
238
|
+
if (fs.existsSync(this._storageDir)) {
|
|
239
|
+
try {
|
|
240
|
+
fs.rmSync(this._storageDir, { recursive: true, force: true });
|
|
241
|
+
fs.mkdirSync(this._storageDir, { recursive: true });
|
|
242
|
+
} catch {
|
|
243
|
+
// Ignore error
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
}
|
|
247
|
+
}
|