@terronex/aifbin-recall 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CONTRIBUTING.md +65 -0
- package/LICENSE +21 -0
- package/NOTICE +36 -0
- package/README.md +250 -0
- package/dist/cli.d.ts +6 -0
- package/dist/cli.d.ts.map +1 -0
- package/dist/cli.js +182 -0
- package/dist/cli.js.map +1 -0
- package/dist/db.d.ts +29 -0
- package/dist/db.d.ts.map +1 -0
- package/dist/db.js +252 -0
- package/dist/db.js.map +1 -0
- package/dist/embedder.d.ts +47 -0
- package/dist/embedder.d.ts.map +1 -0
- package/dist/embedder.js +152 -0
- package/dist/embedder.js.map +1 -0
- package/dist/index.d.ts +27 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +45 -0
- package/dist/index.js.map +1 -0
- package/dist/indexer.d.ts +34 -0
- package/dist/indexer.d.ts.map +1 -0
- package/dist/indexer.js +246 -0
- package/dist/indexer.js.map +1 -0
- package/dist/mcp.d.ts +7 -0
- package/dist/mcp.d.ts.map +1 -0
- package/dist/mcp.js +207 -0
- package/dist/mcp.js.map +1 -0
- package/dist/search.d.ts +27 -0
- package/dist/search.d.ts.map +1 -0
- package/dist/search.js +159 -0
- package/dist/search.js.map +1 -0
- package/dist/server.d.ts +13 -0
- package/dist/server.d.ts.map +1 -0
- package/dist/server.js +250 -0
- package/dist/server.js.map +1 -0
- package/dist/types.d.ts +79 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +20 -0
- package/dist/types.js.map +1 -0
- package/package.json +64 -0
- package/src/cli.ts +195 -0
- package/src/db.ts +295 -0
- package/src/embedder.ts +175 -0
- package/src/index.ts +46 -0
- package/src/indexer.ts +272 -0
- package/src/mcp.ts +244 -0
- package/src/search.ts +201 -0
- package/src/server.ts +270 -0
- package/src/types.ts +103 -0
- package/tsconfig.json +20 -0
package/dist/db.js
ADDED
|
@@ -0,0 +1,252 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* SQLite database management for AIF-BIN Recall
|
|
4
|
+
*/
|
|
5
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
6
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
7
|
+
};
|
|
8
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
9
|
+
exports.EngramDB = void 0;
|
|
10
|
+
const better_sqlite3_1 = __importDefault(require("better-sqlite3"));
|
|
11
|
+
const path_1 = __importDefault(require("path"));
|
|
12
|
+
const os_1 = __importDefault(require("os"));
|
|
13
|
+
const fs_1 = __importDefault(require("fs"));
|
|
14
|
+
const types_js_1 = require("./types.js");
|
|
15
|
+
class EngramDB {
|
|
16
|
+
db;
|
|
17
|
+
constructor(dbPath) {
|
|
18
|
+
const resolvedPath = this.resolvePath(dbPath || types_js_1.DEFAULT_CONFIG.index.path);
|
|
19
|
+
// Ensure directory exists
|
|
20
|
+
const dir = path_1.default.dirname(resolvedPath);
|
|
21
|
+
if (!fs_1.default.existsSync(dir)) {
|
|
22
|
+
fs_1.default.mkdirSync(dir, { recursive: true });
|
|
23
|
+
}
|
|
24
|
+
this.db = new better_sqlite3_1.default(resolvedPath);
|
|
25
|
+
this.init();
|
|
26
|
+
}
|
|
27
|
+
resolvePath(p) {
|
|
28
|
+
if (p.startsWith('~')) {
|
|
29
|
+
return path_1.default.join(os_1.default.homedir(), p.slice(1));
|
|
30
|
+
}
|
|
31
|
+
return path_1.default.resolve(p);
|
|
32
|
+
}
|
|
33
|
+
init() {
|
|
34
|
+
// Enable WAL mode for better concurrency
|
|
35
|
+
this.db.pragma('journal_mode = WAL');
|
|
36
|
+
// Create collections table
|
|
37
|
+
this.db.exec(`
|
|
38
|
+
CREATE TABLE IF NOT EXISTS collections (
|
|
39
|
+
id TEXT PRIMARY KEY,
|
|
40
|
+
name TEXT UNIQUE NOT NULL,
|
|
41
|
+
description TEXT,
|
|
42
|
+
file_count INTEGER DEFAULT 0,
|
|
43
|
+
chunk_count INTEGER DEFAULT 0,
|
|
44
|
+
created_at TEXT DEFAULT CURRENT_TIMESTAMP,
|
|
45
|
+
updated_at TEXT DEFAULT CURRENT_TIMESTAMP
|
|
46
|
+
)
|
|
47
|
+
`);
|
|
48
|
+
// Create chunks table
|
|
49
|
+
this.db.exec(`
|
|
50
|
+
CREATE TABLE IF NOT EXISTS chunks (
|
|
51
|
+
id TEXT PRIMARY KEY,
|
|
52
|
+
collection_id TEXT NOT NULL,
|
|
53
|
+
source_file TEXT NOT NULL,
|
|
54
|
+
chunk_index INTEGER NOT NULL,
|
|
55
|
+
text TEXT NOT NULL,
|
|
56
|
+
embedding BLOB NOT NULL,
|
|
57
|
+
metadata TEXT DEFAULT '{}',
|
|
58
|
+
created_at TEXT DEFAULT CURRENT_TIMESTAMP,
|
|
59
|
+
updated_at TEXT DEFAULT CURRENT_TIMESTAMP,
|
|
60
|
+
FOREIGN KEY (collection_id) REFERENCES collections(id) ON DELETE CASCADE
|
|
61
|
+
)
|
|
62
|
+
`);
|
|
63
|
+
// Create FTS5 virtual table for keyword search
|
|
64
|
+
this.db.exec(`
|
|
65
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts USING fts5(
|
|
66
|
+
text,
|
|
67
|
+
content='chunks',
|
|
68
|
+
content_rowid='rowid'
|
|
69
|
+
)
|
|
70
|
+
`);
|
|
71
|
+
// Triggers to keep FTS in sync
|
|
72
|
+
this.db.exec(`
|
|
73
|
+
CREATE TRIGGER IF NOT EXISTS chunks_ai AFTER INSERT ON chunks BEGIN
|
|
74
|
+
INSERT INTO chunks_fts(rowid, text) VALUES (NEW.rowid, NEW.text);
|
|
75
|
+
END
|
|
76
|
+
`);
|
|
77
|
+
this.db.exec(`
|
|
78
|
+
CREATE TRIGGER IF NOT EXISTS chunks_ad AFTER DELETE ON chunks BEGIN
|
|
79
|
+
INSERT INTO chunks_fts(chunks_fts, rowid, text) VALUES('delete', OLD.rowid, OLD.text);
|
|
80
|
+
END
|
|
81
|
+
`);
|
|
82
|
+
this.db.exec(`
|
|
83
|
+
CREATE TRIGGER IF NOT EXISTS chunks_au AFTER UPDATE ON chunks BEGIN
|
|
84
|
+
INSERT INTO chunks_fts(chunks_fts, rowid, text) VALUES('delete', OLD.rowid, OLD.text);
|
|
85
|
+
INSERT INTO chunks_fts(rowid, text) VALUES (NEW.rowid, NEW.text);
|
|
86
|
+
END
|
|
87
|
+
`);
|
|
88
|
+
// Indexes
|
|
89
|
+
this.db.exec(`CREATE INDEX IF NOT EXISTS idx_chunks_collection ON chunks(collection_id)`);
|
|
90
|
+
this.db.exec(`CREATE INDEX IF NOT EXISTS idx_chunks_source ON chunks(source_file)`);
|
|
91
|
+
}
|
|
92
|
+
// Collection operations
|
|
93
|
+
createCollection(name, description) {
|
|
94
|
+
const id = crypto.randomUUID();
|
|
95
|
+
const stmt = this.db.prepare(`
|
|
96
|
+
INSERT INTO collections (id, name, description)
|
|
97
|
+
VALUES (?, ?, ?)
|
|
98
|
+
`);
|
|
99
|
+
stmt.run(id, name, description || null);
|
|
100
|
+
return this.getCollection(name);
|
|
101
|
+
}
|
|
102
|
+
getCollection(name) {
|
|
103
|
+
const stmt = this.db.prepare(`SELECT * FROM collections WHERE name = ?`);
|
|
104
|
+
const row = stmt.get(name);
|
|
105
|
+
if (!row)
|
|
106
|
+
return null;
|
|
107
|
+
return {
|
|
108
|
+
id: row.id,
|
|
109
|
+
name: row.name,
|
|
110
|
+
description: row.description,
|
|
111
|
+
fileCount: row.file_count,
|
|
112
|
+
chunkCount: row.chunk_count,
|
|
113
|
+
createdAt: new Date(row.created_at),
|
|
114
|
+
updatedAt: new Date(row.updated_at),
|
|
115
|
+
};
|
|
116
|
+
}
|
|
117
|
+
getCollectionById(id) {
|
|
118
|
+
const stmt = this.db.prepare(`SELECT * FROM collections WHERE id = ?`);
|
|
119
|
+
const row = stmt.get(id);
|
|
120
|
+
if (!row)
|
|
121
|
+
return null;
|
|
122
|
+
return {
|
|
123
|
+
id: row.id,
|
|
124
|
+
name: row.name,
|
|
125
|
+
description: row.description,
|
|
126
|
+
fileCount: row.file_count,
|
|
127
|
+
chunkCount: row.chunk_count,
|
|
128
|
+
createdAt: new Date(row.created_at),
|
|
129
|
+
updatedAt: new Date(row.updated_at),
|
|
130
|
+
};
|
|
131
|
+
}
|
|
132
|
+
listCollections() {
|
|
133
|
+
const stmt = this.db.prepare(`SELECT * FROM collections ORDER BY name`);
|
|
134
|
+
const rows = stmt.all();
|
|
135
|
+
return rows.map(row => ({
|
|
136
|
+
id: row.id,
|
|
137
|
+
name: row.name,
|
|
138
|
+
description: row.description,
|
|
139
|
+
fileCount: row.file_count,
|
|
140
|
+
chunkCount: row.chunk_count,
|
|
141
|
+
createdAt: new Date(row.created_at),
|
|
142
|
+
updatedAt: new Date(row.updated_at),
|
|
143
|
+
}));
|
|
144
|
+
}
|
|
145
|
+
deleteCollection(name) {
|
|
146
|
+
const stmt = this.db.prepare(`DELETE FROM collections WHERE name = ?`);
|
|
147
|
+
const result = stmt.run(name);
|
|
148
|
+
return result.changes > 0;
|
|
149
|
+
}
|
|
150
|
+
// Chunk operations
|
|
151
|
+
insertChunk(chunk) {
|
|
152
|
+
const stmt = this.db.prepare(`
|
|
153
|
+
INSERT INTO chunks (id, collection_id, source_file, chunk_index, text, embedding, metadata)
|
|
154
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
155
|
+
`);
|
|
156
|
+
// Convert embedding array to binary buffer
|
|
157
|
+
const embeddingBuffer = Buffer.from(new Float32Array(chunk.embedding).buffer);
|
|
158
|
+
stmt.run(chunk.id, chunk.collectionId, chunk.sourceFile, chunk.chunkIndex, chunk.text, embeddingBuffer, JSON.stringify(chunk.metadata));
|
|
159
|
+
}
|
|
160
|
+
insertChunks(chunks) {
|
|
161
|
+
const insert = this.db.prepare(`
|
|
162
|
+
INSERT INTO chunks (id, collection_id, source_file, chunk_index, text, embedding, metadata)
|
|
163
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
164
|
+
`);
|
|
165
|
+
const insertMany = this.db.transaction((chunks) => {
|
|
166
|
+
for (const chunk of chunks) {
|
|
167
|
+
const embeddingBuffer = Buffer.from(new Float32Array(chunk.embedding).buffer);
|
|
168
|
+
insert.run(chunk.id, chunk.collectionId, chunk.sourceFile, chunk.chunkIndex, chunk.text, embeddingBuffer, JSON.stringify(chunk.metadata));
|
|
169
|
+
}
|
|
170
|
+
});
|
|
171
|
+
insertMany(chunks);
|
|
172
|
+
}
|
|
173
|
+
getChunk(id) {
|
|
174
|
+
const stmt = this.db.prepare(`SELECT * FROM chunks WHERE id = ?`);
|
|
175
|
+
const row = stmt.get(id);
|
|
176
|
+
if (!row)
|
|
177
|
+
return null;
|
|
178
|
+
return this.rowToChunk(row);
|
|
179
|
+
}
|
|
180
|
+
getChunksByCollection(collectionId) {
|
|
181
|
+
const stmt = this.db.prepare(`SELECT * FROM chunks WHERE collection_id = ?`);
|
|
182
|
+
const rows = stmt.all(collectionId);
|
|
183
|
+
return rows.map(row => this.rowToChunk(row));
|
|
184
|
+
}
|
|
185
|
+
deleteChunksBySource(sourceFile) {
|
|
186
|
+
const stmt = this.db.prepare(`DELETE FROM chunks WHERE source_file = ?`);
|
|
187
|
+
const result = stmt.run(sourceFile);
|
|
188
|
+
return result.changes;
|
|
189
|
+
}
|
|
190
|
+
// Search operations
|
|
191
|
+
getAllChunksWithEmbeddings(collectionId) {
|
|
192
|
+
let stmt;
|
|
193
|
+
if (collectionId) {
|
|
194
|
+
stmt = this.db.prepare(`SELECT * FROM chunks WHERE collection_id = ?`);
|
|
195
|
+
return stmt.all(collectionId).map(row => this.rowToChunk(row));
|
|
196
|
+
}
|
|
197
|
+
else {
|
|
198
|
+
stmt = this.db.prepare(`SELECT * FROM chunks`);
|
|
199
|
+
return stmt.all().map(row => this.rowToChunk(row));
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
keywordSearch(query, collectionId, limit = 10) {
|
|
203
|
+
// Escape special FTS5 characters and wrap in quotes for phrase matching
|
|
204
|
+
const escapedQuery = '"' + query.replace(/"/g, '""') + '"';
|
|
205
|
+
let sql = `
|
|
206
|
+
SELECT chunks.id, bm25(chunks_fts) as score
|
|
207
|
+
FROM chunks_fts
|
|
208
|
+
JOIN chunks ON chunks.rowid = chunks_fts.rowid
|
|
209
|
+
WHERE chunks_fts MATCH ?
|
|
210
|
+
`;
|
|
211
|
+
const params = [escapedQuery];
|
|
212
|
+
if (collectionId) {
|
|
213
|
+
sql += ` AND chunks.collection_id = ?`;
|
|
214
|
+
params.push(collectionId);
|
|
215
|
+
}
|
|
216
|
+
sql += ` ORDER BY score LIMIT ?`;
|
|
217
|
+
params.push(limit);
|
|
218
|
+
const stmt = this.db.prepare(sql);
|
|
219
|
+
return stmt.all(...params);
|
|
220
|
+
}
|
|
221
|
+
updateCollectionStats(collectionId) {
|
|
222
|
+
const stmt = this.db.prepare(`
|
|
223
|
+
UPDATE collections SET
|
|
224
|
+
file_count = (SELECT COUNT(DISTINCT source_file) FROM chunks WHERE collection_id = ?),
|
|
225
|
+
chunk_count = (SELECT COUNT(*) FROM chunks WHERE collection_id = ?),
|
|
226
|
+
updated_at = CURRENT_TIMESTAMP
|
|
227
|
+
WHERE id = ?
|
|
228
|
+
`);
|
|
229
|
+
stmt.run(collectionId, collectionId, collectionId);
|
|
230
|
+
}
|
|
231
|
+
rowToChunk(row) {
|
|
232
|
+
// Convert binary buffer back to Float32Array, then to regular array
|
|
233
|
+
const embeddingBuffer = row.embedding;
|
|
234
|
+
const embedding = Array.from(new Float32Array(embeddingBuffer.buffer, embeddingBuffer.byteOffset, embeddingBuffer.length / 4));
|
|
235
|
+
return {
|
|
236
|
+
id: row.id,
|
|
237
|
+
collectionId: row.collection_id,
|
|
238
|
+
sourceFile: row.source_file,
|
|
239
|
+
chunkIndex: row.chunk_index,
|
|
240
|
+
text: row.text,
|
|
241
|
+
embedding,
|
|
242
|
+
metadata: JSON.parse(row.metadata || '{}'),
|
|
243
|
+
createdAt: new Date(row.created_at),
|
|
244
|
+
updatedAt: new Date(row.updated_at),
|
|
245
|
+
};
|
|
246
|
+
}
|
|
247
|
+
close() {
|
|
248
|
+
this.db.close();
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
exports.EngramDB = EngramDB;
|
|
252
|
+
//# sourceMappingURL=db.js.map
|
package/dist/db.js.map
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"db.js","sourceRoot":"","sources":["../src/db.ts"],"names":[],"mappings":";AAAA;;GAEG;;;;;;AAEH,oEAAsC;AACtC,gDAAwB;AACxB,4CAAoB;AACpB,4CAAoB;AACpB,yCAAkG;AAElG,MAAa,QAAQ;IACX,EAAE,CAAoB;IAE9B,YAAY,MAAe;QACzB,MAAM,YAAY,GAAG,IAAI,CAAC,WAAW,CAAC,MAAM,IAAI,yBAAc,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAE3E,0BAA0B;QAC1B,MAAM,GAAG,GAAG,cAAI,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;QACvC,IAAI,CAAC,YAAE,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;YACxB,YAAE,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACzC,CAAC;QAED,IAAI,CAAC,EAAE,GAAG,IAAI,wBAAQ,CAAC,YAAY,CAAC,CAAC;QACrC,IAAI,CAAC,IAAI,EAAE,CAAC;IACd,CAAC;IAEO,WAAW,CAAC,CAAS;QAC3B,IAAI,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;YACtB,OAAO,cAAI,CAAC,IAAI,CAAC,YAAE,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QAC7C,CAAC;QACD,OAAO,cAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IACzB,CAAC;IAEO,IAAI;QACV,yCAAyC;QACzC,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,oBAAoB,CAAC,CAAC;QAErC,2BAA2B;QAC3B,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC;;;;;;;;;;KAUZ,CAAC,CAAC;QAEH,sBAAsB;QACtB,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC;;;;;;;;;;;;;KAaZ,CAAC,CAAC;QAEH,+CAA+C;QAC/C,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC;;;;;;KAMZ,CAAC,CAAC;QAEH,+BAA+B;QAC/B,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC;;;;KAIZ,CAAC,CAAC;QAEH,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC;;;;KAIZ,CAAC,CAAC;QAEH,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC;;;;;KAKZ,CAAC,CAAC;QAEH,UAAU;QACV,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,2EAA2E,CAAC,CAAC;QAC1F,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,qEAAqE,CAAC,CAAC;IACtF,CAAC;IAED,wBAAwB;IACxB,gBAAgB,CAAC,IAAY,EAAE,WAAoB;QACjD,MAAM,EAAE,GAAG,MAAM,CAAC,UAAU,EAAE,CAAC;QAC/B,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;;KAG5B,CAAC,CAAC;QACH,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,IAAI,EAAE,WAAW,IAAI,IAAI,CAAC,CAAC;QACxC,OAAO,IAAI,CAAC,aAAa,CAAC,IAAI,CAAE,CAAC;IACnC,CAAC;IAED,aAAa,CAAC,IAAY;QACxB,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,0CAA0C,CAAC,CAAC;QACzE,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAQ,CAAC;QAClC,IAAI,CAAC,GAAG;YAAE,OAAO,IAAI,CAAC;QACtB,OAAO;YACL,EAAE,EAAE,GAAG,CAAC,EAAE;YACV,IAAI,EAAE,GAAG,CAAC,IAAI;YACd,WAAW,EAAE,GAAG,CAAC,WAAW;YAC5B,SAAS,EAAE,GAAG,CAAC,UAAU;YACzB,UAAU,EAAE,GAAG,CAAC,WAAW;YAC3B,SAAS,EAAE,IAAI,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC;YACnC,SAAS,EAAE,IAAI,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC;SACpC,CAAC;IACJ,CAAC;IAED,iBAAiB,CAAC,EAAU;QAC1B,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,wCAAwC,CAAC,CAAC;QACvE,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,EAAE,CAAQ,CAAC;QAChC,IAAI,CAAC,GAAG;YAAE,OAAO,IAAI,CAAC;QACtB,OAAO;YACL,EAAE,EAAE,GAAG,CAAC,EAAE;YACV,IAAI,EAAE,GAAG,CAAC,IAAI;YACd,WAAW,EAAE,GAAG,CAAC,WAAW;YAC5B,SAAS,EAAE,GAAG,CAAC,UAAU;YACzB,UAAU,EAAE,GAAG,CAAC,WAAW;YAC3B,SAAS,EAAE,IAAI,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC;YACnC,SAAS,EAAE,IAAI,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC;SACpC,CAAC;IACJ,CAAC;IAED,eAAe;QACb,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,yCAAyC,CAAC,CAAC;QACxE,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,EAAW,CAAC;QACjC,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACtB,EAAE,EAAE,GAAG,CAAC,EAAE;YACV,IAAI,EAAE,GAAG,CAAC,IAAI;YACd,WAAW,EAAE,GAAG,CAAC,WAAW;YAC5B,SAAS,EAAE,GAAG,CAAC,UAAU;YACzB,UAAU,EAAE,GAAG,CAAC,WAAW;YAC3B,SAAS,EAAE,IAAI,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC;YACnC,SAAS,EAAE,IAAI,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC;SACpC,CAAC,CAAC,CAAC;IACN,CAAC;IAED,gBAAgB,CAAC,IAAY;QAC3B,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,wCAAwC,CAAC,CAAC;QACvE,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAC9B,OAAO,MAAM,CAAC,OAAO,GAAG,CAAC,CAAC;IAC5B,CAAC;IAED,mBAAmB;IACnB,WAAW,CAAC,KAAmD;QAC7D,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;;KAG5B,CAAC,CAAC;QAEH,2CAA2C;QAC3C,MAAM,eAAe,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,YAAY,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,CAAC;QAE9E,IAAI,CAAC,GAAG,CACN,KAAK,CAAC,EAAE,EACR,KAAK,CAAC,YAAY,EAClB,KAAK,CAAC,UAAU,EAChB,KAAK,CAAC,UAAU,EAChB,KAAK,CAAC,IAAI,EACV,eAAe,EACf,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,QAAQ,CAAC,CAC/B,CAAC;IACJ,CAAC;IAED,YAAY,CAAC,MAAsD;QACjE,MAAM,MAAM,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;;KAG9B,CAAC,CAAC;QAEH,MAAM,UAAU,GAAG,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,CAAC,MAAsD,EAAE,EAAE;YAChG,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;gBAC3B,MAAM,eAAe,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,YAAY,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,CAAC;gBAC9E,MAAM,CAAC,GAAG,CACR,KAAK,CAAC,EAAE,EACR,KAAK,CAAC,YAAY,EAClB,KAAK,CAAC,UAAU,EAChB,KAAK,CAAC,UAAU,EAChB,KAAK,CAAC,IAAI,EACV,eAAe,EACf,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,QAAQ,CAAC,CAC/B,CAAC;YACJ,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,UAAU,CAAC,MAAM,CAAC,CAAC;IACrB,CAAC;IAED,QAAQ,CAAC,EAAU;QACjB,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,mCAAmC,CAAC,CAAC;QAClE,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,EAAE,CAAQ,CAAC;QAChC,IAAI,CAAC,GAAG;YAAE,OAAO,IAAI,CAAC;QACtB,OAAO,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;IAC9B,CAAC;IAED,qBAAqB,CAAC,YAAoB;QACxC,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,8CAA8C,CAAC,CAAC;QAC7E,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,YAAY,CAAU,CAAC;QAC7C,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC;IAC/C,CAAC;IAED,oBAAoB,CAAC,UAAkB;QACrC,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,0CAA0C,CAAC,CAAC;QACzE,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QACpC,OAAO,MAAM,CAAC,OAAO,CAAC;IACxB,CAAC;IAED,oBAAoB;IACpB,0BAA0B,CAAC,YAAqB;QAC9C,IAAI,IAAI,CAAC;QACT,IAAI,YAAY,EAAE,CAAC;YACjB,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,8CAA8C,CAAC,CAAC;YACvE,OAAQ,IAAI,CAAC,GAAG,CAAC,YAAY,CAAW,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC;QAC5E,CAAC;aAAM,CAAC;YACN,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,sBAAsB,CAAC,CAAC;YAC/C,OAAQ,IAAI,CAAC,GAAG,EAAY,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC;QAChE,CAAC;IACH,CAAC;IAED,aAAa,CAAC,KAAa,EAAE,YAAqB,EAAE,QAAgB,EAAE;QACpE,wEAAwE;QACxE,MAAM,YAAY,GAAG,GAAG,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,GAAG,CAAC;QAE3D,IAAI,GAAG,GAAG;;;;;KAKT,CAAC;QAEF,MAAM,MAAM,GAAU,CAAC,YAAY,CAAC,CAAC;QAErC,IAAI,YAAY,EAAE,CAAC;YACjB,GAAG,IAAI,+BAA+B,CAAC;YACvC,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QAC5B,CAAC;QAED,GAAG,IAAI,yBAAyB,CAAC;QACjC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAEnB,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAClC,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,MAAM,CAAoC,CAAC;IAChE,CAAC;IAED,qBAAqB,CAAC,YAAoB;QACxC,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;;;;;KAM5B,CAAC,CAAC;QACH,IAAI,CAAC,GAAG,CAAC,YAAY,EAAE,YAAY,EAAE,YAAY,CAAC,CAAC;IACrD,CAAC;IAEO,UAAU,CAAC,GAAQ;QACzB,oEAAoE;QACpE,MAAM,eAAe,GAAG,GAAG,CAAC,SAAmB,CAAC;QAChD,MAAM,SAAS,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,YAAY,CAAC,eAAe,CAAC,MAAM,EAAE,eAAe,CAAC,UAAU,EAAE,eAAe,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;QAE/H,OAAO;YACL,EAAE,EAAE,GAAG,CAAC,EAAE;YACV,YAAY,EAAE,GAAG,CAAC,aAAa;YAC/B,UAAU,EAAE,GAAG,CAAC,WAAW;YAC3B,UAAU,EAAE,GAAG,CAAC,WAAW;YAC3B,IAAI,EAAE,GAAG,CAAC,IAAI;YACd,SAAS;YACT,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,IAAI,IAAI,CAAC;YAC1C,SAAS,EAAE,IAAI,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC;YACnC,SAAS,EAAE,IAAI,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC;SACpC,CAAC;IACJ,CAAC;IAED,KAAK;QACH,IAAI,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC;IAClB,CAAC;CACF;AA5RD,4BA4RC"}
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Local embedding generation for AIF-BIN Recall
|
|
3
|
+
* Uses @xenova/transformers to run sentence-transformers models locally
|
|
4
|
+
*/
|
|
5
|
+
export declare const EMBEDDING_MODELS: {
|
|
6
|
+
readonly minilm: "Xenova/all-MiniLM-L6-v2";
|
|
7
|
+
readonly mpnet: "Xenova/all-mpnet-base-v2";
|
|
8
|
+
readonly 'bge-small': "Xenova/bge-small-en-v1.5";
|
|
9
|
+
readonly 'bge-base': "Xenova/bge-base-en-v1.5";
|
|
10
|
+
readonly 'e5-small': "Xenova/e5-small-v2";
|
|
11
|
+
};
|
|
12
|
+
export type EmbeddingModelName = keyof typeof EMBEDDING_MODELS;
|
|
13
|
+
export declare class Embedder {
|
|
14
|
+
private model;
|
|
15
|
+
private embedder;
|
|
16
|
+
private loading;
|
|
17
|
+
constructor(model?: EmbeddingModelName);
|
|
18
|
+
/**
|
|
19
|
+
* Ensure the model is loaded
|
|
20
|
+
*/
|
|
21
|
+
init(): Promise<void>;
|
|
22
|
+
/**
|
|
23
|
+
* Embed a single text string
|
|
24
|
+
*/
|
|
25
|
+
embed(text: string): Promise<number[]>;
|
|
26
|
+
/**
|
|
27
|
+
* Embed multiple texts in batch
|
|
28
|
+
*/
|
|
29
|
+
embedBatch(texts: string[]): Promise<number[][]>;
|
|
30
|
+
/**
|
|
31
|
+
* Get the embedding dimension for the current model
|
|
32
|
+
*/
|
|
33
|
+
getDimension(): number;
|
|
34
|
+
/**
|
|
35
|
+
* Get the model name
|
|
36
|
+
*/
|
|
37
|
+
getModelName(): string;
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* Get the default embedder instance
|
|
41
|
+
*/
|
|
42
|
+
export declare function getDefaultEmbedder(model?: EmbeddingModelName): Embedder;
|
|
43
|
+
/**
|
|
44
|
+
* Quick helper to embed a single query
|
|
45
|
+
*/
|
|
46
|
+
export declare function embedQuery(text: string, model?: EmbeddingModelName): Promise<number[]>;
|
|
47
|
+
//# sourceMappingURL=embedder.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"embedder.d.ts","sourceRoot":"","sources":["../src/embedder.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAQH,eAAO,MAAM,gBAAgB;;;;;;CAMnB,CAAC;AAEX,MAAM,MAAM,kBAAkB,GAAG,MAAM,OAAO,gBAAgB,CAAC;AAkD/D,qBAAa,QAAQ;IACnB,OAAO,CAAC,KAAK,CAAqB;IAClC,OAAO,CAAC,QAAQ,CAAa;IAC7B,OAAO,CAAC,OAAO,CAA8B;gBAEjC,KAAK,GAAE,kBAA6B;IAIhD;;OAEG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAY3B;;OAEG;IACG,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAY5C;;OAEG;IACG,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;IAmBtD;;OAEG;IACH,YAAY,IAAI,MAAM;IActB;;OAEG;IACH,YAAY,IAAI,MAAM;CAGvB;AAKD;;GAEG;AACH,wBAAgB,kBAAkB,CAAC,KAAK,GAAE,kBAA6B,GAAG,QAAQ,CAKjF;AAED;;GAEG;AACH,wBAAsB,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,GAAE,kBAA6B,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAGtG"}
|
package/dist/embedder.js
ADDED
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Local embedding generation for AIF-BIN Recall
|
|
4
|
+
* Uses @xenova/transformers to run sentence-transformers models locally
|
|
5
|
+
*/
|
|
6
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
7
|
+
exports.Embedder = exports.EMBEDDING_MODELS = void 0;
|
|
8
|
+
exports.getDefaultEmbedder = getDefaultEmbedder;
|
|
9
|
+
exports.embedQuery = embedQuery;
|
|
10
|
+
const transformers_1 = require("@xenova/transformers");
|
|
11
|
+
// Disable local model check (download from HuggingFace)
|
|
12
|
+
transformers_1.env.allowLocalModels = false;
|
|
13
|
+
// Supported embedding models (same as AIF-BIN Pro)
|
|
14
|
+
exports.EMBEDDING_MODELS = {
|
|
15
|
+
minilm: 'Xenova/all-MiniLM-L6-v2', // 384 dims, fastest
|
|
16
|
+
mpnet: 'Xenova/all-mpnet-base-v2', // 768 dims, balanced
|
|
17
|
+
'bge-small': 'Xenova/bge-small-en-v1.5', // 384 dims, good quality
|
|
18
|
+
'bge-base': 'Xenova/bge-base-en-v1.5', // 768 dims, best quality
|
|
19
|
+
'e5-small': 'Xenova/e5-small-v2', // 384 dims
|
|
20
|
+
};
|
|
21
|
+
// Cache for loaded pipelines
|
|
22
|
+
const pipelineCache = new Map();
|
|
23
|
+
/**
|
|
24
|
+
* Get or create an embedding pipeline for the given model
|
|
25
|
+
*/
|
|
26
|
+
async function getEmbedder(model = 'minilm') {
|
|
27
|
+
const modelPath = exports.EMBEDDING_MODELS[model];
|
|
28
|
+
if (!pipelineCache.has(modelPath)) {
|
|
29
|
+
console.log(`Loading embedding model: ${model} (${modelPath})...`);
|
|
30
|
+
const embedder = await (0, transformers_1.pipeline)('feature-extraction', modelPath);
|
|
31
|
+
pipelineCache.set(modelPath, embedder);
|
|
32
|
+
console.log(`Model loaded: ${model}`);
|
|
33
|
+
}
|
|
34
|
+
return pipelineCache.get(modelPath);
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* Mean pooling for sentence embeddings
|
|
38
|
+
*/
|
|
39
|
+
function meanPool(embeddings) {
|
|
40
|
+
const dims = embeddings[0].length;
|
|
41
|
+
const result = new Array(dims).fill(0);
|
|
42
|
+
for (const embedding of embeddings) {
|
|
43
|
+
for (let i = 0; i < dims; i++) {
|
|
44
|
+
result[i] += embedding[i];
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
for (let i = 0; i < dims; i++) {
|
|
48
|
+
result[i] /= embeddings.length;
|
|
49
|
+
}
|
|
50
|
+
return result;
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* Normalize a vector to unit length
|
|
54
|
+
*/
|
|
55
|
+
function normalize(vector) {
|
|
56
|
+
const magnitude = Math.sqrt(vector.reduce((sum, val) => sum + val * val, 0));
|
|
57
|
+
if (magnitude === 0)
|
|
58
|
+
return vector;
|
|
59
|
+
return vector.map(val => val / magnitude);
|
|
60
|
+
}
|
|
61
|
+
class Embedder {
|
|
62
|
+
model;
|
|
63
|
+
embedder = null;
|
|
64
|
+
loading = null;
|
|
65
|
+
constructor(model = 'minilm') {
|
|
66
|
+
this.model = model;
|
|
67
|
+
}
|
|
68
|
+
/**
|
|
69
|
+
* Ensure the model is loaded
|
|
70
|
+
*/
|
|
71
|
+
async init() {
|
|
72
|
+
if (this.embedder)
|
|
73
|
+
return;
|
|
74
|
+
if (!this.loading) {
|
|
75
|
+
this.loading = (async () => {
|
|
76
|
+
this.embedder = await getEmbedder(this.model);
|
|
77
|
+
})();
|
|
78
|
+
}
|
|
79
|
+
await this.loading;
|
|
80
|
+
}
|
|
81
|
+
/**
|
|
82
|
+
* Embed a single text string
|
|
83
|
+
*/
|
|
84
|
+
async embed(text) {
|
|
85
|
+
await this.init();
|
|
86
|
+
const output = await this.embedder(text, {
|
|
87
|
+
pooling: 'mean',
|
|
88
|
+
normalize: true,
|
|
89
|
+
});
|
|
90
|
+
// Convert to regular array
|
|
91
|
+
return Array.from(output.data);
|
|
92
|
+
}
|
|
93
|
+
/**
|
|
94
|
+
* Embed multiple texts in batch
|
|
95
|
+
*/
|
|
96
|
+
async embedBatch(texts) {
|
|
97
|
+
await this.init();
|
|
98
|
+
const results = [];
|
|
99
|
+
// Process in batches to avoid memory issues
|
|
100
|
+
const batchSize = 32;
|
|
101
|
+
for (let i = 0; i < texts.length; i += batchSize) {
|
|
102
|
+
const batch = texts.slice(i, i + batchSize);
|
|
103
|
+
for (const text of batch) {
|
|
104
|
+
const embedding = await this.embed(text);
|
|
105
|
+
results.push(embedding);
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
return results;
|
|
109
|
+
}
|
|
110
|
+
/**
|
|
111
|
+
* Get the embedding dimension for the current model
|
|
112
|
+
*/
|
|
113
|
+
getDimension() {
|
|
114
|
+
switch (this.model) {
|
|
115
|
+
case 'minilm':
|
|
116
|
+
case 'bge-small':
|
|
117
|
+
case 'e5-small':
|
|
118
|
+
return 384;
|
|
119
|
+
case 'mpnet':
|
|
120
|
+
case 'bge-base':
|
|
121
|
+
return 768;
|
|
122
|
+
default:
|
|
123
|
+
return 384;
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
/**
|
|
127
|
+
* Get the model name
|
|
128
|
+
*/
|
|
129
|
+
getModelName() {
|
|
130
|
+
return this.model;
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
exports.Embedder = Embedder;
|
|
134
|
+
// Singleton instance for default model
|
|
135
|
+
let defaultEmbedder = null;
|
|
136
|
+
/**
|
|
137
|
+
* Get the default embedder instance
|
|
138
|
+
*/
|
|
139
|
+
function getDefaultEmbedder(model = 'minilm') {
|
|
140
|
+
if (!defaultEmbedder || defaultEmbedder.getModelName() !== model) {
|
|
141
|
+
defaultEmbedder = new Embedder(model);
|
|
142
|
+
}
|
|
143
|
+
return defaultEmbedder;
|
|
144
|
+
}
|
|
145
|
+
/**
|
|
146
|
+
* Quick helper to embed a single query
|
|
147
|
+
*/
|
|
148
|
+
async function embedQuery(text, model = 'minilm') {
|
|
149
|
+
const embedder = getDefaultEmbedder(model);
|
|
150
|
+
return embedder.embed(text);
|
|
151
|
+
}
|
|
152
|
+
//# sourceMappingURL=embedder.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"embedder.js","sourceRoot":"","sources":["../src/embedder.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;AA8JH,gDAKC;AAKD,gCAGC;AAzKD,uDAAqD;AAErD,wDAAwD;AACxD,kBAAG,CAAC,gBAAgB,GAAG,KAAK,CAAC;AAE7B,mDAAmD;AACtC,QAAA,gBAAgB,GAAG;IAC9B,MAAM,EAAE,yBAAyB,EAAY,oBAAoB;IACjE,KAAK,EAAE,0BAA0B,EAAY,qBAAqB;IAClE,WAAW,EAAE,0BAA0B,EAAM,yBAAyB;IACtE,UAAU,EAAE,yBAAyB,EAAQ,yBAAyB;IACtE,UAAU,EAAE,oBAAoB,EAAa,WAAW;CAChD,CAAC;AAIX,6BAA6B;AAC7B,MAAM,aAAa,GAAG,IAAI,GAAG,EAAe,CAAC;AAE7C;;GAEG;AACH,KAAK,UAAU,WAAW,CAAC,QAA4B,QAAQ;IAC7D,MAAM,SAAS,GAAG,wBAAgB,CAAC,KAAK,CAAC,CAAC;IAE1C,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;QAClC,OAAO,CAAC,GAAG,CAAC,4BAA4B,KAAK,KAAK,SAAS,MAAM,CAAC,CAAC;QACnE,MAAM,QAAQ,GAAG,MAAM,IAAA,uBAAQ,EAAC,oBAAoB,EAAE,SAAS,CAAC,CAAC;QACjE,aAAa,CAAC,GAAG,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;QACvC,OAAO,CAAC,GAAG,CAAC,iBAAiB,KAAK,EAAE,CAAC,CAAC;IACxC,CAAC;IAED,OAAO,aAAa,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;AACtC,CAAC;AAED;;GAEG;AACH,SAAS,QAAQ,CAAC,UAAsB;IACtC,MAAM,IAAI,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;IAClC,MAAM,MAAM,GAAG,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAEvC,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACnC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC;YAC9B,MAAM,CAAC,CAAC,CAAC,IAAI,SAAS,CAAC,CAAC,CAAC,CAAC;QAC5B,CAAC;IACH,CAAC;IAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC;QAC9B,MAAM,CAAC,CAAC,CAAC,IAAI,UAAU,CAAC,MAAM,CAAC;IACjC,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;GAEG;AACH,SAAS,SAAS,CAAC,MAAgB;IACjC,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,GAAG,GAAG,GAAG,GAAG,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;IAC7E,IAAI,SAAS,KAAK,CAAC;QAAE,OAAO,MAAM,CAAC;IACnC,OAAO,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,SAAS,CAAC,CAAC;AAC5C,CAAC;AAED,MAAa,QAAQ;IACX,KAAK,CAAqB;IAC1B,QAAQ,GAAQ,IAAI,CAAC;IACrB,OAAO,GAAyB,IAAI,CAAC;IAE7C,YAAY,QAA4B,QAAQ;QAC9C,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;IACrB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,IAAI;QACR,IAAI,IAAI,CAAC,QAAQ;YAAE,OAAO;QAE1B,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YAClB,IAAI,CAAC,OAAO,GAAG,CAAC,KAAK,IAAI,EAAE;gBACzB,IAAI,CAAC,QAAQ,GAAG,MAAM,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAChD,CAAC,CAAC,EAAE,CAAC;QACP,CAAC;QAED,MAAM,IAAI,CAAC,OAAO,CAAC;IACrB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK,CAAC,IAAY;QACtB,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAElB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE;YACvC,OAAO,EAAE,MAAM;YACf,SAAS,EAAE,IAAI;SAChB,CAAC,CAAC;QAEH,2BAA2B;QAC3B,OAAO,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACjC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU,CAAC,KAAe;QAC9B,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAElB,MAAM,OAAO,GAAe,EAAE,CAAC;QAE/B,4CAA4C;QAC5C,MAAM,SAAS,GAAG,EAAE,CAAC;QACrB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC;YACjD,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC;YAE5C,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBACzB,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBACzC,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAC1B,CAAC;QACH,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAED;;OAEG;IACH,YAAY;QACV,QAAQ,IAAI,CAAC,KAAK,EAAE,CAAC;YACnB,KAAK,QAAQ,CAAC;YACd,KAAK,WAAW,CAAC;YACjB,KAAK,UAAU;gBACb,OAAO,GAAG,CAAC;YACb,KAAK,OAAO,CAAC;YACb,KAAK,UAAU;gBACb,OAAO,GAAG,CAAC;YACb;gBACE,OAAO,GAAG,CAAC;QACf,CAAC;IACH,CAAC;IAED;;OAEG;IACH,YAAY;QACV,OAAO,IAAI,CAAC,KAAK,CAAC;IACpB,CAAC;CACF;AApFD,4BAoFC;AAED,uCAAuC;AACvC,IAAI,eAAe,GAAoB,IAAI,CAAC;AAE5C;;GAEG;AACH,SAAgB,kBAAkB,CAAC,QAA4B,QAAQ;IACrE,IAAI,CAAC,eAAe,IAAI,eAAe,CAAC,YAAY,EAAE,KAAK,KAAK,EAAE,CAAC;QACjE,eAAe,GAAG,IAAI,QAAQ,CAAC,KAAK,CAAC,CAAC;IACxC,CAAC;IACD,OAAO,eAAe,CAAC;AACzB,CAAC;AAED;;GAEG;AACI,KAAK,UAAU,UAAU,CAAC,IAAY,EAAE,QAA4B,QAAQ;IACjF,MAAM,QAAQ,GAAG,kBAAkB,CAAC,KAAK,CAAC,CAAC;IAC3C,OAAO,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;AAC9B,CAAC"}
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* AIF-BIN Recall - Local-first memory server for AI agents
|
|
3
|
+
*
|
|
4
|
+
* @example
|
|
5
|
+
* ```typescript
|
|
6
|
+
* import { EngramDB, SearchEngine, Indexer } from '@terronex/engram';
|
|
7
|
+
*
|
|
8
|
+
* const db = new EngramDB('~/.engram/index.db');
|
|
9
|
+
* const indexer = new Indexer(db);
|
|
10
|
+
* const search = new SearchEngine(db);
|
|
11
|
+
*
|
|
12
|
+
* // Index AIF-BIN files
|
|
13
|
+
* indexer.indexDirectory('./memories', { collection: 'my-project' });
|
|
14
|
+
*
|
|
15
|
+
* // Search with embedding
|
|
16
|
+
* const results = await search.search(queryEmbedding, { collection: 'my-project' });
|
|
17
|
+
* ```
|
|
18
|
+
*/
|
|
19
|
+
export { EngramDB } from './db.js';
|
|
20
|
+
export { SearchEngine, cosineSimilarity } from './search.js';
|
|
21
|
+
export { Indexer, parseAifBinFile, findAifBinFiles } from './indexer.js';
|
|
22
|
+
export { Embedder, embedQuery, getDefaultEmbedder, EMBEDDING_MODELS, type EmbeddingModelName } from './embedder.js';
|
|
23
|
+
export { createServer, startServer } from './server.js';
|
|
24
|
+
export { startMcpServer } from './mcp.js';
|
|
25
|
+
export type { MemoryChunk, Collection, SearchResult, SearchOptions, IndexOptions, EngramConfig, ServerConfig, IndexConfig, SearchConfig, AifBinFile, AifBinChunk, AifBinHeader, } from './types.js';
|
|
26
|
+
export { DEFAULT_CONFIG } from './types.js';
|
|
27
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAGH,OAAO,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AACnC,OAAO,EAAE,YAAY,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAC7D,OAAO,EAAE,OAAO,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AACzE,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,kBAAkB,EAAE,gBAAgB,EAAE,KAAK,kBAAkB,EAAE,MAAM,eAAe,CAAC;AAGpH,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AACxD,OAAO,EAAE,cAAc,EAAE,MAAM,UAAU,CAAC;AAG1C,YAAY,EACV,WAAW,EACX,UAAU,EACV,YAAY,EACZ,aAAa,EACb,YAAY,EACZ,YAAY,EACZ,YAAY,EACZ,WAAW,EACX,YAAY,EACZ,UAAU,EACV,WAAW,EACX,YAAY,GACb,MAAM,YAAY,CAAC;AAEpB,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC"}
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* AIF-BIN Recall - Local-first memory server for AI agents
|
|
4
|
+
*
|
|
5
|
+
* @example
|
|
6
|
+
* ```typescript
|
|
7
|
+
* import { EngramDB, SearchEngine, Indexer } from '@terronex/engram';
|
|
8
|
+
*
|
|
9
|
+
* const db = new EngramDB('~/.engram/index.db');
|
|
10
|
+
* const indexer = new Indexer(db);
|
|
11
|
+
* const search = new SearchEngine(db);
|
|
12
|
+
*
|
|
13
|
+
* // Index AIF-BIN files
|
|
14
|
+
* indexer.indexDirectory('./memories', { collection: 'my-project' });
|
|
15
|
+
*
|
|
16
|
+
* // Search with embedding
|
|
17
|
+
* const results = await search.search(queryEmbedding, { collection: 'my-project' });
|
|
18
|
+
* ```
|
|
19
|
+
*/
|
|
20
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
21
|
+
exports.DEFAULT_CONFIG = exports.startMcpServer = exports.startServer = exports.createServer = exports.EMBEDDING_MODELS = exports.getDefaultEmbedder = exports.embedQuery = exports.Embedder = exports.findAifBinFiles = exports.parseAifBinFile = exports.Indexer = exports.cosineSimilarity = exports.SearchEngine = exports.EngramDB = void 0;
|
|
22
|
+
// Core classes
|
|
23
|
+
var db_js_1 = require("./db.js");
|
|
24
|
+
Object.defineProperty(exports, "EngramDB", { enumerable: true, get: function () { return db_js_1.EngramDB; } });
|
|
25
|
+
var search_js_1 = require("./search.js");
|
|
26
|
+
Object.defineProperty(exports, "SearchEngine", { enumerable: true, get: function () { return search_js_1.SearchEngine; } });
|
|
27
|
+
Object.defineProperty(exports, "cosineSimilarity", { enumerable: true, get: function () { return search_js_1.cosineSimilarity; } });
|
|
28
|
+
var indexer_js_1 = require("./indexer.js");
|
|
29
|
+
Object.defineProperty(exports, "Indexer", { enumerable: true, get: function () { return indexer_js_1.Indexer; } });
|
|
30
|
+
Object.defineProperty(exports, "parseAifBinFile", { enumerable: true, get: function () { return indexer_js_1.parseAifBinFile; } });
|
|
31
|
+
Object.defineProperty(exports, "findAifBinFiles", { enumerable: true, get: function () { return indexer_js_1.findAifBinFiles; } });
|
|
32
|
+
var embedder_js_1 = require("./embedder.js");
|
|
33
|
+
Object.defineProperty(exports, "Embedder", { enumerable: true, get: function () { return embedder_js_1.Embedder; } });
|
|
34
|
+
Object.defineProperty(exports, "embedQuery", { enumerable: true, get: function () { return embedder_js_1.embedQuery; } });
|
|
35
|
+
Object.defineProperty(exports, "getDefaultEmbedder", { enumerable: true, get: function () { return embedder_js_1.getDefaultEmbedder; } });
|
|
36
|
+
Object.defineProperty(exports, "EMBEDDING_MODELS", { enumerable: true, get: function () { return embedder_js_1.EMBEDDING_MODELS; } });
|
|
37
|
+
// Server functions
|
|
38
|
+
var server_js_1 = require("./server.js");
|
|
39
|
+
Object.defineProperty(exports, "createServer", { enumerable: true, get: function () { return server_js_1.createServer; } });
|
|
40
|
+
Object.defineProperty(exports, "startServer", { enumerable: true, get: function () { return server_js_1.startServer; } });
|
|
41
|
+
var mcp_js_1 = require("./mcp.js");
|
|
42
|
+
Object.defineProperty(exports, "startMcpServer", { enumerable: true, get: function () { return mcp_js_1.startMcpServer; } });
|
|
43
|
+
var types_js_1 = require("./types.js");
|
|
44
|
+
Object.defineProperty(exports, "DEFAULT_CONFIG", { enumerable: true, get: function () { return types_js_1.DEFAULT_CONFIG; } });
|
|
45
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;;;;GAiBG;;;AAEH,eAAe;AACf,iCAAmC;AAA1B,iGAAA,QAAQ,OAAA;AACjB,yCAA6D;AAApD,yGAAA,YAAY,OAAA;AAAE,6GAAA,gBAAgB,OAAA;AACvC,2CAAyE;AAAhE,qGAAA,OAAO,OAAA;AAAE,6GAAA,eAAe,OAAA;AAAE,6GAAA,eAAe,OAAA;AAClD,6CAAoH;AAA3G,uGAAA,QAAQ,OAAA;AAAE,yGAAA,UAAU,OAAA;AAAE,iHAAA,kBAAkB,OAAA;AAAE,+GAAA,gBAAgB,OAAA;AAEnE,mBAAmB;AACnB,yCAAwD;AAA/C,yGAAA,YAAY,OAAA;AAAE,wGAAA,WAAW,OAAA;AAClC,mCAA0C;AAAjC,wGAAA,cAAc,OAAA;AAkBvB,uCAA4C;AAAnC,0GAAA,cAAc,OAAA"}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* AIF-BIN file indexer for AIF-BIN Recall
|
|
3
|
+
* Parses AIF-BIN v2 binary format
|
|
4
|
+
*/
|
|
5
|
+
import type { AifBinFile, IndexOptions } from './types.js';
|
|
6
|
+
import { EngramDB } from './db.js';
|
|
7
|
+
/**
|
|
8
|
+
* Parse an AIF-BIN v2 file
|
|
9
|
+
*/
|
|
10
|
+
export declare function parseAifBinFile(filePath: string): AifBinFile;
|
|
11
|
+
/**
|
|
12
|
+
* Find all .aif-bin files in a directory
|
|
13
|
+
*/
|
|
14
|
+
export declare function findAifBinFiles(dir: string, recursive?: boolean): string[];
|
|
15
|
+
export declare class Indexer {
|
|
16
|
+
private db;
|
|
17
|
+
constructor(db: EngramDB);
|
|
18
|
+
/**
|
|
19
|
+
* Index a single AIF-BIN file into a collection
|
|
20
|
+
*/
|
|
21
|
+
indexFile(filePath: string, collectionId: string): number;
|
|
22
|
+
/**
|
|
23
|
+
* Index a directory of AIF-BIN files
|
|
24
|
+
*/
|
|
25
|
+
indexDirectory(dir: string, options: IndexOptions): {
|
|
26
|
+
files: number;
|
|
27
|
+
chunks: number;
|
|
28
|
+
};
|
|
29
|
+
/**
|
|
30
|
+
* Remove a file from the index
|
|
31
|
+
*/
|
|
32
|
+
removeFile(filePath: string): number;
|
|
33
|
+
}
|
|
34
|
+
//# sourceMappingURL=indexer.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"indexer.d.ts","sourceRoot":"","sources":["../src/indexer.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAKH,OAAO,KAAK,EAAE,UAAU,EAA0C,YAAY,EAAE,MAAM,YAAY,CAAC;AACnG,OAAO,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAiBnC;;GAEG;AACH,wBAAgB,eAAe,CAAC,QAAQ,EAAE,MAAM,GAAG,UAAU,CAkI5D;AAED;;GAEG;AACH,wBAAgB,eAAe,CAAC,GAAG,EAAE,MAAM,EAAE,SAAS,GAAE,OAAc,GAAG,MAAM,EAAE,CAmBhF;AAED,qBAAa,OAAO;IAClB,OAAO,CAAC,EAAE,CAAW;gBAET,EAAE,EAAE,QAAQ;IAIxB;;OAEG;IACH,SAAS,CAAC,QAAQ,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,GAAG,MAAM;IAiCzD;;OAEG;IACH,cAAc,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,YAAY,GAAG;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE;IAkCrF;;OAEG;IACH,UAAU,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM;CAGrC"}
|