nexum-code 0.7.11 → 0.7.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/{app-BwutxHqk.mjs → app-Dn2QN6KQ.mjs} +3722 -2709
- package/dist/dist-CP92tKmm.mjs +2003 -0
- package/dist/dist-DC_zVbxT.mjs +7 -0
- package/dist/{dist-0bMQWc-B.mjs → dist-Dh_F9syl.mjs} +1 -1
- package/dist/{esm-BLHs2dah.mjs → esm-C7p5AwJU.mjs} +39 -39
- package/dist/{from-CKE2n10i.mjs → from-D_EZ04z8.mjs} +1 -1
- package/dist/{index.min-DtYFUEA3.mjs → index.min-Di7B-aJf.mjs} +138 -138
- package/dist/main.mjs +1 -1
- package/dist/{multipart-parser-BxHsVgPe.mjs → multipart-parser-uEw3PR6T.mjs} +2 -2
- package/dist/{chunk-apG1qJts.mjs → rolldown-runtime-CDCXy-ZS.mjs} +9 -2
- package/dist/{src-C6XAfwit.mjs → src-BAMmHkmC.mjs} +6 -6
- package/package.json +3 -1
|
@@ -0,0 +1,2003 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import { fileURLToPath as __cjsShimFileURLToPath } from 'node:url';
|
|
3
|
+
import { dirname as __cjsShimDirname } from 'node:path';
|
|
4
|
+
const __filename = __cjsShimFileURLToPath(import.meta.url);
|
|
5
|
+
const __dirname = __cjsShimDirname(__filename);
|
|
6
|
+
import { mkdir, readFile, readdir } from "node:fs/promises";
|
|
7
|
+
import { DatabaseSync } from "node:sqlite";
|
|
8
|
+
//#region ../../node_modules/.pnpm/pathe@2.0.3/node_modules/pathe/dist/shared/pathe.M-eThtNZ.mjs
|
|
9
|
+
const _DRIVE_LETTER_START_RE = /^[A-Za-z]:\//;
|
|
10
|
+
function normalizeWindowsPath(input = "") {
|
|
11
|
+
if (!input) return input;
|
|
12
|
+
return input.replace(/\\/g, "/").replace(_DRIVE_LETTER_START_RE, (r) => r.toUpperCase());
|
|
13
|
+
}
|
|
14
|
+
const _UNC_REGEX = /^[/\\]{2}/;
|
|
15
|
+
const _IS_ABSOLUTE_RE = /^[/\\](?![/\\])|^[/\\]{2}(?!\.)|^[A-Za-z]:[/\\]/;
|
|
16
|
+
const _DRIVE_LETTER_RE = /^[A-Za-z]:$/;
|
|
17
|
+
const _ROOT_FOLDER_RE = /^\/([A-Za-z]:)?$/;
|
|
18
|
+
const _EXTNAME_RE = /.(\.[^./]+|\.)$/;
|
|
19
|
+
const _PATH_ROOT_RE = /^[/\\]|^[a-zA-Z]:[/\\]/;
|
|
20
|
+
const normalize = function(path) {
|
|
21
|
+
if (path.length === 0) return ".";
|
|
22
|
+
path = normalizeWindowsPath(path);
|
|
23
|
+
const isUNCPath = path.match(_UNC_REGEX);
|
|
24
|
+
const isPathAbsolute = isAbsolute(path);
|
|
25
|
+
const trailingSeparator = path[path.length - 1] === "/";
|
|
26
|
+
path = normalizeString(path, !isPathAbsolute);
|
|
27
|
+
if (path.length === 0) {
|
|
28
|
+
if (isPathAbsolute) return "/";
|
|
29
|
+
return trailingSeparator ? "./" : ".";
|
|
30
|
+
}
|
|
31
|
+
if (trailingSeparator) path += "/";
|
|
32
|
+
if (_DRIVE_LETTER_RE.test(path)) path += "/";
|
|
33
|
+
if (isUNCPath) {
|
|
34
|
+
if (!isPathAbsolute) return `//./${path}`;
|
|
35
|
+
return `//${path}`;
|
|
36
|
+
}
|
|
37
|
+
return isPathAbsolute && !isAbsolute(path) ? `/${path}` : path;
|
|
38
|
+
};
|
|
39
|
+
const join = function(...segments) {
|
|
40
|
+
let path = "";
|
|
41
|
+
for (const seg of segments) {
|
|
42
|
+
if (!seg) continue;
|
|
43
|
+
if (path.length > 0) {
|
|
44
|
+
const pathTrailing = path[path.length - 1] === "/";
|
|
45
|
+
const segLeading = seg[0] === "/";
|
|
46
|
+
if (pathTrailing && segLeading) path += seg.slice(1);
|
|
47
|
+
else path += pathTrailing || segLeading ? seg : `/${seg}`;
|
|
48
|
+
} else path += seg;
|
|
49
|
+
}
|
|
50
|
+
return normalize(path);
|
|
51
|
+
};
|
|
52
|
+
function cwd() {
|
|
53
|
+
if (typeof process !== "undefined" && typeof process.cwd === "function") return process.cwd().replace(/\\/g, "/");
|
|
54
|
+
return "/";
|
|
55
|
+
}
|
|
56
|
+
const resolve = function(...arguments_) {
|
|
57
|
+
arguments_ = arguments_.map((argument) => normalizeWindowsPath(argument));
|
|
58
|
+
let resolvedPath = "";
|
|
59
|
+
let resolvedAbsolute = false;
|
|
60
|
+
for (let index = arguments_.length - 1; index >= -1 && !resolvedAbsolute; index--) {
|
|
61
|
+
const path = index >= 0 ? arguments_[index] : cwd();
|
|
62
|
+
if (!path || path.length === 0) continue;
|
|
63
|
+
resolvedPath = `${path}/${resolvedPath}`;
|
|
64
|
+
resolvedAbsolute = isAbsolute(path);
|
|
65
|
+
}
|
|
66
|
+
resolvedPath = normalizeString(resolvedPath, !resolvedAbsolute);
|
|
67
|
+
if (resolvedAbsolute && !isAbsolute(resolvedPath)) return `/${resolvedPath}`;
|
|
68
|
+
return resolvedPath.length > 0 ? resolvedPath : ".";
|
|
69
|
+
};
|
|
70
|
+
function normalizeString(path, allowAboveRoot) {
|
|
71
|
+
let res = "";
|
|
72
|
+
let lastSegmentLength = 0;
|
|
73
|
+
let lastSlash = -1;
|
|
74
|
+
let dots = 0;
|
|
75
|
+
let char = null;
|
|
76
|
+
for (let index = 0; index <= path.length; ++index) {
|
|
77
|
+
if (index < path.length) char = path[index];
|
|
78
|
+
else if (char === "/") break;
|
|
79
|
+
else char = "/";
|
|
80
|
+
if (char === "/") {
|
|
81
|
+
if (lastSlash === index - 1 || dots === 1);
|
|
82
|
+
else if (dots === 2) {
|
|
83
|
+
if (res.length < 2 || lastSegmentLength !== 2 || res[res.length - 1] !== "." || res[res.length - 2] !== ".") {
|
|
84
|
+
if (res.length > 2) {
|
|
85
|
+
const lastSlashIndex = res.lastIndexOf("/");
|
|
86
|
+
if (lastSlashIndex === -1) {
|
|
87
|
+
res = "";
|
|
88
|
+
lastSegmentLength = 0;
|
|
89
|
+
} else {
|
|
90
|
+
res = res.slice(0, lastSlashIndex);
|
|
91
|
+
lastSegmentLength = res.length - 1 - res.lastIndexOf("/");
|
|
92
|
+
}
|
|
93
|
+
lastSlash = index;
|
|
94
|
+
dots = 0;
|
|
95
|
+
continue;
|
|
96
|
+
} else if (res.length > 0) {
|
|
97
|
+
res = "";
|
|
98
|
+
lastSegmentLength = 0;
|
|
99
|
+
lastSlash = index;
|
|
100
|
+
dots = 0;
|
|
101
|
+
continue;
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
if (allowAboveRoot) {
|
|
105
|
+
res += res.length > 0 ? "/.." : "..";
|
|
106
|
+
lastSegmentLength = 2;
|
|
107
|
+
}
|
|
108
|
+
} else {
|
|
109
|
+
if (res.length > 0) res += `/${path.slice(lastSlash + 1, index)}`;
|
|
110
|
+
else res = path.slice(lastSlash + 1, index);
|
|
111
|
+
lastSegmentLength = index - lastSlash - 1;
|
|
112
|
+
}
|
|
113
|
+
lastSlash = index;
|
|
114
|
+
dots = 0;
|
|
115
|
+
} else if (char === "." && dots !== -1) ++dots;
|
|
116
|
+
else dots = -1;
|
|
117
|
+
}
|
|
118
|
+
return res;
|
|
119
|
+
}
|
|
120
|
+
const isAbsolute = function(p) {
|
|
121
|
+
return _IS_ABSOLUTE_RE.test(p);
|
|
122
|
+
};
|
|
123
|
+
const extname = function(p) {
|
|
124
|
+
if (p === "..") return "";
|
|
125
|
+
const match = _EXTNAME_RE.exec(normalizeWindowsPath(p));
|
|
126
|
+
return match && match[1] || "";
|
|
127
|
+
};
|
|
128
|
+
const relative = function(from, to) {
|
|
129
|
+
const _from = resolve(from).replace(_ROOT_FOLDER_RE, "$1").split("/");
|
|
130
|
+
const _to = resolve(to).replace(_ROOT_FOLDER_RE, "$1").split("/");
|
|
131
|
+
if (_to[0][1] === ":" && _from[0][1] === ":" && _from[0] !== _to[0]) return _to.join("/");
|
|
132
|
+
const _fromCopy = [..._from];
|
|
133
|
+
for (const segment of _fromCopy) {
|
|
134
|
+
if (_to[0] !== segment) break;
|
|
135
|
+
_from.shift();
|
|
136
|
+
_to.shift();
|
|
137
|
+
}
|
|
138
|
+
return [..._from.map(() => ".."), ..._to].join("/");
|
|
139
|
+
};
|
|
140
|
+
const dirname = function(p) {
|
|
141
|
+
const segments = normalizeWindowsPath(p).replace(/\/$/, "").split("/").slice(0, -1);
|
|
142
|
+
if (segments.length === 1 && _DRIVE_LETTER_RE.test(segments[0])) segments[0] += "/";
|
|
143
|
+
return segments.join("/") || (isAbsolute(p) ? "/" : ".");
|
|
144
|
+
};
|
|
145
|
+
const basename = function(p, extension) {
|
|
146
|
+
const segments = normalizeWindowsPath(p).split("/");
|
|
147
|
+
let lastSegment = "";
|
|
148
|
+
for (let i = segments.length - 1; i >= 0; i--) {
|
|
149
|
+
const val = segments[i];
|
|
150
|
+
if (val) {
|
|
151
|
+
lastSegment = val;
|
|
152
|
+
break;
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
return extension && lastSegment.endsWith(extension) ? lastSegment.slice(0, -extension.length) : lastSegment;
|
|
156
|
+
};
|
|
157
|
+
const parse = function(p) {
|
|
158
|
+
const root = _PATH_ROOT_RE.exec(p)?.[0]?.replace(/\\/g, "/") || "";
|
|
159
|
+
const base = basename(p);
|
|
160
|
+
const extension = extname(base);
|
|
161
|
+
return {
|
|
162
|
+
root,
|
|
163
|
+
dir: dirname(p),
|
|
164
|
+
base,
|
|
165
|
+
ext: extension,
|
|
166
|
+
name: base.slice(0, base.length - extension.length)
|
|
167
|
+
};
|
|
168
|
+
};
|
|
169
|
+
//#endregion
|
|
170
|
+
//#region ../../packages/knowledge/dist/index.mjs
|
|
171
|
+
const ENTITY_TYPES$1 = /* @__PURE__ */ new Set([
|
|
172
|
+
"person",
|
|
173
|
+
"organization",
|
|
174
|
+
"location",
|
|
175
|
+
"time",
|
|
176
|
+
"product",
|
|
177
|
+
"metric",
|
|
178
|
+
"action",
|
|
179
|
+
"work",
|
|
180
|
+
"group",
|
|
181
|
+
"subject",
|
|
182
|
+
"tags"
|
|
183
|
+
]);
|
|
184
|
+
function generateId(prefix) {
|
|
185
|
+
return `${prefix}-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 10)}`;
|
|
186
|
+
}
|
|
187
|
+
function normalizeName(name) {
|
|
188
|
+
return name.trim().toLowerCase().replaceAll(/\s+/g, " ");
|
|
189
|
+
}
|
|
190
|
+
function vectorToJson(vec) {
|
|
191
|
+
if (vec === null) return null;
|
|
192
|
+
return JSON.stringify(Array.from(vec));
|
|
193
|
+
}
|
|
194
|
+
function jsonToVector(json) {
|
|
195
|
+
if (json === null) return null;
|
|
196
|
+
try {
|
|
197
|
+
const arr = JSON.parse(json);
|
|
198
|
+
return new Float32Array(arr);
|
|
199
|
+
} catch {
|
|
200
|
+
return null;
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
function isEntityType(value) {
|
|
204
|
+
return ENTITY_TYPES$1.has(value);
|
|
205
|
+
}
|
|
206
|
+
var KnowledgeStore = class {
|
|
207
|
+
dbPath;
|
|
208
|
+
db;
|
|
209
|
+
initialized = false;
|
|
210
|
+
initPromise = null;
|
|
211
|
+
embeddingEngine;
|
|
212
|
+
constructor(projectDir) {
|
|
213
|
+
this.dbPath = join(projectDir, "knowledge", "knowledge.db");
|
|
214
|
+
}
|
|
215
|
+
async init() {
|
|
216
|
+
if (this.initPromise) return this.initPromise;
|
|
217
|
+
this.initPromise = this._doInit();
|
|
218
|
+
return this.initPromise;
|
|
219
|
+
}
|
|
220
|
+
async _doInit() {
|
|
221
|
+
if (this.initialized) return;
|
|
222
|
+
await mkdir(dirname(this.dbPath), { recursive: true });
|
|
223
|
+
this.db = new DatabaseSync(this.dbPath);
|
|
224
|
+
this.db.exec("PRAGMA journal_mode = WAL;");
|
|
225
|
+
this.db.exec("PRAGMA foreign_keys = ON;");
|
|
226
|
+
this.db.exec("PRAGMA busy_timeout = 5000;");
|
|
227
|
+
this.createSchema();
|
|
228
|
+
this.initialized = true;
|
|
229
|
+
}
|
|
230
|
+
close() {
|
|
231
|
+
if (this.db !== void 0) {
|
|
232
|
+
this.db.exec("PRAGMA wal_checkpoint(TRUNCATE)");
|
|
233
|
+
this.db.close();
|
|
234
|
+
this.db = void 0;
|
|
235
|
+
}
|
|
236
|
+
this.initialized = false;
|
|
237
|
+
this.initPromise = null;
|
|
238
|
+
}
|
|
239
|
+
setEmbeddingEngine(engine) {
|
|
240
|
+
this.embeddingEngine = engine;
|
|
241
|
+
}
|
|
242
|
+
getEmbeddingEngine() {
|
|
243
|
+
return this.embeddingEngine;
|
|
244
|
+
}
|
|
245
|
+
/**
|
|
246
|
+
* Proactively load the embedding model (downloads on first call).
|
|
247
|
+
* Call before ingestion so the user gets a clear download prompt instead
|
|
248
|
+
* of a mid-ingest failure.
|
|
249
|
+
*/
|
|
250
|
+
async ensureEmbeddingReady() {
|
|
251
|
+
if (this.embeddingEngine === void 0) return {
|
|
252
|
+
ok: false,
|
|
253
|
+
reason: "engine not set"
|
|
254
|
+
};
|
|
255
|
+
const ok = await this.embeddingEngine.ensureReady();
|
|
256
|
+
return {
|
|
257
|
+
ok,
|
|
258
|
+
reason: ok ? void 0 : "fastembed init failed"
|
|
259
|
+
};
|
|
260
|
+
}
|
|
261
|
+
/**
|
|
262
|
+
* Begin a write transaction. Use commitTransaction / rollbackTransaction
|
|
263
|
+
* to close it. Used by ingest to keep chunks/events/entities/edges atomic —
|
|
264
|
+
* a mid-ingest failure leaves no partial rows.
|
|
265
|
+
*/
|
|
266
|
+
beginTransaction() {
|
|
267
|
+
if (this.db === void 0) throw new Error("knowledge store not initialized");
|
|
268
|
+
this.db.exec("BEGIN");
|
|
269
|
+
}
|
|
270
|
+
commitTransaction() {
|
|
271
|
+
if (this.db === void 0) throw new Error("knowledge store not initialized");
|
|
272
|
+
this.db.exec("COMMIT");
|
|
273
|
+
}
|
|
274
|
+
rollbackTransaction() {
|
|
275
|
+
if (this.db === void 0) return;
|
|
276
|
+
try {
|
|
277
|
+
this.db.exec("ROLLBACK");
|
|
278
|
+
} catch {}
|
|
279
|
+
}
|
|
280
|
+
createSchema() {
|
|
281
|
+
if (this.db === void 0) return;
|
|
282
|
+
this.db.exec(`
|
|
283
|
+
CREATE TABLE IF NOT EXISTS knowledge_sources (
|
|
284
|
+
id TEXT PRIMARY KEY,
|
|
285
|
+
name TEXT NOT NULL,
|
|
286
|
+
file_path TEXT,
|
|
287
|
+
description TEXT,
|
|
288
|
+
created_at INTEGER NOT NULL
|
|
289
|
+
);
|
|
290
|
+
|
|
291
|
+
CREATE TABLE IF NOT EXISTS knowledge_documents (
|
|
292
|
+
id TEXT PRIMARY KEY,
|
|
293
|
+
source_id TEXT NOT NULL REFERENCES knowledge_sources(id) ON DELETE CASCADE,
|
|
294
|
+
title TEXT NOT NULL,
|
|
295
|
+
content TEXT,
|
|
296
|
+
status TEXT NOT NULL DEFAULT 'pending',
|
|
297
|
+
chunk_count INTEGER DEFAULT 0,
|
|
298
|
+
created_at INTEGER NOT NULL
|
|
299
|
+
);
|
|
300
|
+
|
|
301
|
+
CREATE INDEX IF NOT EXISTS idx_knowledge_documents_source ON knowledge_documents(source_id);
|
|
302
|
+
|
|
303
|
+
CREATE TABLE IF NOT EXISTS knowledge_chunks (
|
|
304
|
+
id TEXT PRIMARY KEY,
|
|
305
|
+
source_id TEXT NOT NULL REFERENCES knowledge_sources(id) ON DELETE CASCADE,
|
|
306
|
+
document_id TEXT NOT NULL REFERENCES knowledge_documents(id) ON DELETE CASCADE,
|
|
307
|
+
rank INTEGER NOT NULL,
|
|
308
|
+
heading TEXT,
|
|
309
|
+
content TEXT NOT NULL,
|
|
310
|
+
raw_content TEXT,
|
|
311
|
+
embedding_json TEXT,
|
|
312
|
+
created_at INTEGER NOT NULL
|
|
313
|
+
);
|
|
314
|
+
|
|
315
|
+
CREATE INDEX IF NOT EXISTS idx_knowledge_chunks_doc ON knowledge_chunks(document_id);
|
|
316
|
+
CREATE INDEX IF NOT EXISTS idx_knowledge_chunks_source ON knowledge_chunks(source_id);
|
|
317
|
+
|
|
318
|
+
CREATE TABLE IF NOT EXISTS knowledge_events (
|
|
319
|
+
id TEXT PRIMARY KEY,
|
|
320
|
+
source_id TEXT NOT NULL REFERENCES knowledge_sources(id) ON DELETE CASCADE,
|
|
321
|
+
document_id TEXT NOT NULL REFERENCES knowledge_documents(id) ON DELETE CASCADE,
|
|
322
|
+
chunk_id TEXT NOT NULL REFERENCES knowledge_chunks(id) ON DELETE CASCADE,
|
|
323
|
+
rank INTEGER NOT NULL,
|
|
324
|
+
title TEXT NOT NULL,
|
|
325
|
+
summary TEXT,
|
|
326
|
+
content TEXT NOT NULL,
|
|
327
|
+
category TEXT,
|
|
328
|
+
keywords TEXT,
|
|
329
|
+
title_embedding_json TEXT,
|
|
330
|
+
content_embedding_json TEXT,
|
|
331
|
+
created_at INTEGER NOT NULL
|
|
332
|
+
);
|
|
333
|
+
|
|
334
|
+
CREATE INDEX IF NOT EXISTS idx_knowledge_events_chunk ON knowledge_events(chunk_id);
|
|
335
|
+
CREATE INDEX IF NOT EXISTS idx_knowledge_events_doc ON knowledge_events(document_id);
|
|
336
|
+
CREATE INDEX IF NOT EXISTS idx_knowledge_events_source ON knowledge_events(source_id);
|
|
337
|
+
|
|
338
|
+
CREATE TABLE IF NOT EXISTS knowledge_entities (
|
|
339
|
+
id TEXT PRIMARY KEY,
|
|
340
|
+
source_id TEXT NOT NULL REFERENCES knowledge_sources(id) ON DELETE CASCADE,
|
|
341
|
+
type TEXT NOT NULL,
|
|
342
|
+
name TEXT NOT NULL,
|
|
343
|
+
normalized_name TEXT NOT NULL,
|
|
344
|
+
description TEXT,
|
|
345
|
+
embedding_json TEXT,
|
|
346
|
+
created_at INTEGER NOT NULL,
|
|
347
|
+
UNIQUE(source_id, type, normalized_name)
|
|
348
|
+
);
|
|
349
|
+
|
|
350
|
+
CREATE INDEX IF NOT EXISTS idx_knowledge_entities_source ON knowledge_entities(source_id);
|
|
351
|
+
CREATE INDEX IF NOT EXISTS idx_knowledge_entities_name ON knowledge_entities(normalized_name);
|
|
352
|
+
|
|
353
|
+
CREATE TABLE IF NOT EXISTS knowledge_event_entities (
|
|
354
|
+
id TEXT PRIMARY KEY,
|
|
355
|
+
event_id TEXT NOT NULL REFERENCES knowledge_events(id) ON DELETE CASCADE,
|
|
356
|
+
entity_id TEXT NOT NULL REFERENCES knowledge_entities(id) ON DELETE CASCADE,
|
|
357
|
+
weight REAL DEFAULT 1.0,
|
|
358
|
+
description TEXT,
|
|
359
|
+
embedding_json TEXT,
|
|
360
|
+
UNIQUE(event_id, entity_id)
|
|
361
|
+
);
|
|
362
|
+
|
|
363
|
+
CREATE INDEX IF NOT EXISTS idx_knowledge_event_entities_event ON knowledge_event_entities(event_id);
|
|
364
|
+
CREATE INDEX IF NOT EXISTS idx_knowledge_event_entities_entity ON knowledge_event_entities(entity_id);
|
|
365
|
+
|
|
366
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS knowledge_chunks_fts USING fts5(
|
|
367
|
+
heading, content, content='knowledge_chunks', content_rowid='rowid'
|
|
368
|
+
);
|
|
369
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS knowledge_events_fts USING fts5(
|
|
370
|
+
title, summary, content, content='knowledge_events', content_rowid='rowid'
|
|
371
|
+
);
|
|
372
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS knowledge_entities_fts USING fts5(
|
|
373
|
+
name, description, content='knowledge_entities', content_rowid='rowid'
|
|
374
|
+
);
|
|
375
|
+
`);
|
|
376
|
+
}
|
|
377
|
+
async createSource(params) {
|
|
378
|
+
await this.init();
|
|
379
|
+
if (this.db === void 0) throw new Error("knowledge store not initialized");
|
|
380
|
+
const id = generateId("src");
|
|
381
|
+
const createdAt = Date.now();
|
|
382
|
+
this.db.prepare("INSERT INTO knowledge_sources (id, name, file_path, description, created_at) VALUES (?, ?, ?, ?, ?)").run(id, params.name, params.filePath ?? null, params.description ?? null, createdAt);
|
|
383
|
+
return {
|
|
384
|
+
id,
|
|
385
|
+
name: params.name,
|
|
386
|
+
filePath: params.filePath ?? null,
|
|
387
|
+
description: params.description ?? null,
|
|
388
|
+
createdAt
|
|
389
|
+
};
|
|
390
|
+
}
|
|
391
|
+
async findSourceByFilePath(filePath) {
|
|
392
|
+
await this.init();
|
|
393
|
+
if (this.db === void 0) return void 0;
|
|
394
|
+
const row = this.db.prepare("SELECT * FROM knowledge_sources WHERE file_path = ?").get(filePath);
|
|
395
|
+
return row === void 0 ? void 0 : rowToSource(row);
|
|
396
|
+
}
|
|
397
|
+
async listSources() {
|
|
398
|
+
await this.init();
|
|
399
|
+
if (this.db === void 0) return [];
|
|
400
|
+
return this.db.prepare("SELECT * FROM knowledge_sources ORDER BY created_at DESC").all().map(rowToSource);
|
|
401
|
+
}
|
|
402
|
+
async getSource(id) {
|
|
403
|
+
await this.init();
|
|
404
|
+
if (this.db === void 0) return void 0;
|
|
405
|
+
const row = this.db.prepare("SELECT * FROM knowledge_sources WHERE id = ?").get(id);
|
|
406
|
+
return row === void 0 ? void 0 : rowToSource(row);
|
|
407
|
+
}
|
|
408
|
+
async deleteSource(id) {
|
|
409
|
+
await this.init();
|
|
410
|
+
if (this.db === void 0) return false;
|
|
411
|
+
return this.db.prepare("DELETE FROM knowledge_sources WHERE id = ?").run(id).changes > 0;
|
|
412
|
+
}
|
|
413
|
+
async createDocument(params) {
|
|
414
|
+
if (this.db === void 0) throw new Error("knowledge store not initialized");
|
|
415
|
+
const id = generateId("doc");
|
|
416
|
+
const createdAt = Date.now();
|
|
417
|
+
this.db.prepare("INSERT INTO knowledge_documents (id, source_id, title, content, status, chunk_count, created_at) VALUES (?, ?, ?, ?, ?, ?, ?)").run(id, params.sourceId, params.title, params.content ?? null, "pending", 0, createdAt);
|
|
418
|
+
return {
|
|
419
|
+
id,
|
|
420
|
+
sourceId: params.sourceId,
|
|
421
|
+
title: params.title,
|
|
422
|
+
content: params.content ?? null,
|
|
423
|
+
status: "pending",
|
|
424
|
+
chunkCount: 0,
|
|
425
|
+
createdAt
|
|
426
|
+
};
|
|
427
|
+
}
|
|
428
|
+
async updateDocumentStatus(documentId, status, chunkCount) {
|
|
429
|
+
if (this.db === void 0) return;
|
|
430
|
+
if (chunkCount !== void 0) this.db.prepare("UPDATE knowledge_documents SET status = ?, chunk_count = ? WHERE id = ?").run(status, chunkCount, documentId);
|
|
431
|
+
else this.db.prepare("UPDATE knowledge_documents SET status = ? WHERE id = ?").run(status, documentId);
|
|
432
|
+
}
|
|
433
|
+
async listDocuments() {
|
|
434
|
+
await this.init();
|
|
435
|
+
if (this.db === void 0) return [];
|
|
436
|
+
return this.db.prepare(`SELECT d.*, s.name AS source_name
|
|
437
|
+
FROM knowledge_documents d
|
|
438
|
+
JOIN knowledge_sources s ON s.id = d.source_id
|
|
439
|
+
ORDER BY d.created_at DESC`).all().map((row) => ({
|
|
440
|
+
...rowToDocument(row),
|
|
441
|
+
sourceName: String(row["source_name"])
|
|
442
|
+
}));
|
|
443
|
+
}
|
|
444
|
+
async getDocument(id) {
|
|
445
|
+
await this.init();
|
|
446
|
+
if (this.db === void 0) return void 0;
|
|
447
|
+
const row = this.db.prepare("SELECT * FROM knowledge_documents WHERE id = ?").get(id);
|
|
448
|
+
return row === void 0 ? void 0 : rowToDocument(row);
|
|
449
|
+
}
|
|
450
|
+
async insertChunk(params) {
|
|
451
|
+
if (this.db === void 0) throw new Error("knowledge store not initialized");
|
|
452
|
+
const id = generateId("chk");
|
|
453
|
+
const createdAt = Date.now();
|
|
454
|
+
const embeddingJson = vectorToJson(params.embedding);
|
|
455
|
+
this.db.prepare(`INSERT INTO knowledge_chunks
|
|
456
|
+
(id, source_id, document_id, rank, heading, content, raw_content, embedding_json, created_at)
|
|
457
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(id, params.sourceId, params.documentId, params.rank, params.heading, params.content, params.rawContent, embeddingJson, createdAt);
|
|
458
|
+
this.db.prepare(`INSERT INTO knowledge_chunks_fts (rowid, heading, content)
|
|
459
|
+
VALUES ((SELECT rowid FROM knowledge_chunks WHERE id = ?), ?, ?)`).run(id, params.heading ?? "", params.content);
|
|
460
|
+
return {
|
|
461
|
+
id,
|
|
462
|
+
sourceId: params.sourceId,
|
|
463
|
+
documentId: params.documentId,
|
|
464
|
+
rank: params.rank,
|
|
465
|
+
heading: params.heading,
|
|
466
|
+
content: params.content,
|
|
467
|
+
rawContent: params.rawContent,
|
|
468
|
+
embedding: params.embedding,
|
|
469
|
+
createdAt
|
|
470
|
+
};
|
|
471
|
+
}
|
|
472
|
+
async getChunk(id) {
|
|
473
|
+
await this.init();
|
|
474
|
+
if (this.db === void 0) return void 0;
|
|
475
|
+
const row = this.db.prepare("SELECT * FROM knowledge_chunks WHERE id = ?").get(id);
|
|
476
|
+
return row === void 0 ? void 0 : rowToChunk(row);
|
|
477
|
+
}
|
|
478
|
+
async listChunksByDocument(documentId) {
|
|
479
|
+
await this.init();
|
|
480
|
+
if (this.db === void 0) return [];
|
|
481
|
+
return this.db.prepare("SELECT * FROM knowledge_chunks WHERE document_id = ? ORDER BY rank ASC").all(documentId).map(rowToChunk);
|
|
482
|
+
}
|
|
483
|
+
/** Load all chunk embeddings + ids for vector search. */
|
|
484
|
+
async loadAllChunkEmbeddings() {
|
|
485
|
+
await this.init();
|
|
486
|
+
if (this.db === void 0) return [];
|
|
487
|
+
const rows = this.db.prepare("SELECT id, embedding_json FROM knowledge_chunks WHERE embedding_json IS NOT NULL").all();
|
|
488
|
+
const out = [];
|
|
489
|
+
for (const row of rows) {
|
|
490
|
+
const vec = jsonToVector(row.embedding_json);
|
|
491
|
+
if (vec !== null) out.push({
|
|
492
|
+
id: row.id,
|
|
493
|
+
embedding: vec
|
|
494
|
+
});
|
|
495
|
+
}
|
|
496
|
+
return out;
|
|
497
|
+
}
|
|
498
|
+
async insertEvent(params) {
|
|
499
|
+
if (this.db === void 0) throw new Error("knowledge store not initialized");
|
|
500
|
+
const id = generateId("evt");
|
|
501
|
+
const createdAt = Date.now();
|
|
502
|
+
this.db.prepare(`INSERT INTO knowledge_events
|
|
503
|
+
(id, source_id, document_id, chunk_id, rank, title, summary, content, category, keywords,
|
|
504
|
+
title_embedding_json, content_embedding_json, created_at)
|
|
505
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(id, params.sourceId, params.documentId, params.chunkId, params.rank, params.title, params.summary, params.content, params.category, JSON.stringify(params.keywords), vectorToJson(params.titleEmbedding), vectorToJson(params.contentEmbedding), createdAt);
|
|
506
|
+
this.db.prepare(`INSERT INTO knowledge_events_fts (rowid, title, summary, content)
|
|
507
|
+
VALUES ((SELECT rowid FROM knowledge_events WHERE id = ?), ?, ?, ?)`).run(id, params.title, params.summary ?? "", params.content);
|
|
508
|
+
return {
|
|
509
|
+
id,
|
|
510
|
+
sourceId: params.sourceId,
|
|
511
|
+
documentId: params.documentId,
|
|
512
|
+
chunkId: params.chunkId,
|
|
513
|
+
rank: params.rank,
|
|
514
|
+
title: params.title,
|
|
515
|
+
summary: params.summary,
|
|
516
|
+
content: params.content,
|
|
517
|
+
category: params.category,
|
|
518
|
+
keywords: params.keywords,
|
|
519
|
+
titleEmbedding: params.titleEmbedding,
|
|
520
|
+
contentEmbedding: params.contentEmbedding,
|
|
521
|
+
createdAt
|
|
522
|
+
};
|
|
523
|
+
}
|
|
524
|
+
async getEvent(id) {
|
|
525
|
+
await this.init();
|
|
526
|
+
if (this.db === void 0) return void 0;
|
|
527
|
+
const row = this.db.prepare("SELECT * FROM knowledge_events WHERE id = ?").get(id);
|
|
528
|
+
return row === void 0 ? void 0 : rowToEvent(row);
|
|
529
|
+
}
|
|
530
|
+
async listEventsByDocument(documentId) {
|
|
531
|
+
await this.init();
|
|
532
|
+
if (this.db === void 0) return [];
|
|
533
|
+
return this.db.prepare("SELECT * FROM knowledge_events WHERE document_id = ? ORDER BY rank ASC").all(documentId).map(rowToEvent);
|
|
534
|
+
}
|
|
535
|
+
/** Find events by title vector similarity. */
|
|
536
|
+
async findEventsByTitleVector(queryVec, options = {}) {
|
|
537
|
+
await this.init();
|
|
538
|
+
if (this.db === void 0) return [];
|
|
539
|
+
const limit = options.limit ?? 20;
|
|
540
|
+
const threshold = options.threshold ?? 0;
|
|
541
|
+
const rows = this.db.prepare("SELECT id, title_embedding_json FROM knowledge_events WHERE title_embedding_json IS NOT NULL").all();
|
|
542
|
+
const scored = [];
|
|
543
|
+
for (const row of rows) {
|
|
544
|
+
const vec = jsonToVector(row.title_embedding_json);
|
|
545
|
+
if (vec === null) continue;
|
|
546
|
+
const score = this.embeddingEngine?.cosineSimilarity(queryVec, vec) ?? 0;
|
|
547
|
+
if (score >= threshold) scored.push({
|
|
548
|
+
id: row.id,
|
|
549
|
+
score
|
|
550
|
+
});
|
|
551
|
+
}
|
|
552
|
+
scored.sort((a, b) => b.score - a.score);
|
|
553
|
+
const top = scored.slice(0, limit);
|
|
554
|
+
const out = [];
|
|
555
|
+
for (const item of top) {
|
|
556
|
+
const event = await this.getEvent(item.id);
|
|
557
|
+
if (event !== void 0) out.push({
|
|
558
|
+
event,
|
|
559
|
+
score: item.score
|
|
560
|
+
});
|
|
561
|
+
}
|
|
562
|
+
return out;
|
|
563
|
+
}
|
|
564
|
+
async upsertEntity(params) {
|
|
565
|
+
if (this.db === void 0) throw new Error("knowledge store not initialized");
|
|
566
|
+
if (!isEntityType(params.type)) throw new Error(`invalid entity type: ${params.type}`);
|
|
567
|
+
const normalizedName = normalizeName(params.name);
|
|
568
|
+
const existing = this.db.prepare("SELECT * FROM knowledge_entities WHERE source_id = ? AND type = ? AND normalized_name = ?").get(params.sourceId, params.type, normalizedName);
|
|
569
|
+
if (existing !== void 0) {
|
|
570
|
+
if (params.description !== null && existing["description"] === null) this.db.prepare("UPDATE knowledge_entities SET description = ? WHERE id = ?").run(params.description, String(existing["id"]));
|
|
571
|
+
if (params.embedding !== null && existing["embedding_json"] === null) this.db.prepare("UPDATE knowledge_entities SET embedding_json = ? WHERE id = ?").run(vectorToJson(params.embedding), String(existing["id"]));
|
|
572
|
+
return rowToEntity(existing);
|
|
573
|
+
}
|
|
574
|
+
const id = generateId("ent");
|
|
575
|
+
const createdAt = Date.now();
|
|
576
|
+
this.db.prepare(`INSERT INTO knowledge_entities
|
|
577
|
+
(id, source_id, type, name, normalized_name, description, embedding_json, created_at)
|
|
578
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`).run(id, params.sourceId, params.type, params.name, normalizedName, params.description, vectorToJson(params.embedding), createdAt);
|
|
579
|
+
this.db.prepare(`INSERT INTO knowledge_entities_fts (rowid, name, description)
|
|
580
|
+
VALUES ((SELECT rowid FROM knowledge_entities WHERE id = ?), ?, ?)`).run(id, params.name, params.description ?? "");
|
|
581
|
+
return {
|
|
582
|
+
id,
|
|
583
|
+
sourceId: params.sourceId,
|
|
584
|
+
type: params.type,
|
|
585
|
+
name: params.name,
|
|
586
|
+
normalizedName,
|
|
587
|
+
description: params.description,
|
|
588
|
+
embedding: params.embedding,
|
|
589
|
+
createdAt
|
|
590
|
+
};
|
|
591
|
+
}
|
|
592
|
+
async findEntitiesByName(name, sourceId) {
|
|
593
|
+
await this.init();
|
|
594
|
+
if (this.db === void 0) return [];
|
|
595
|
+
const normalized = normalizeName(name);
|
|
596
|
+
return (sourceId === void 0 ? this.db.prepare("SELECT * FROM knowledge_entities WHERE normalized_name = ?").all(normalized) : this.db.prepare("SELECT * FROM knowledge_entities WHERE normalized_name = ? AND source_id = ?").all(normalized, sourceId)).map(rowToEntity);
|
|
597
|
+
}
|
|
598
|
+
/** Find entities by name vector similarity. */
|
|
599
|
+
async findEntitiesByVector(queryVec, options = {}) {
|
|
600
|
+
await this.init();
|
|
601
|
+
if (this.db === void 0) return [];
|
|
602
|
+
const limit = options.limit ?? 20;
|
|
603
|
+
const threshold = options.threshold ?? 0;
|
|
604
|
+
const rows = this.db.prepare("SELECT id, embedding_json FROM knowledge_entities WHERE embedding_json IS NOT NULL").all();
|
|
605
|
+
const scored = [];
|
|
606
|
+
for (const row of rows) {
|
|
607
|
+
const vec = jsonToVector(row.embedding_json);
|
|
608
|
+
if (vec === null) continue;
|
|
609
|
+
const score = this.embeddingEngine?.cosineSimilarity(queryVec, vec) ?? 0;
|
|
610
|
+
if (score >= threshold) scored.push({
|
|
611
|
+
id: row.id,
|
|
612
|
+
score
|
|
613
|
+
});
|
|
614
|
+
}
|
|
615
|
+
scored.sort((a, b) => b.score - a.score);
|
|
616
|
+
const top = scored.slice(0, limit);
|
|
617
|
+
const out = [];
|
|
618
|
+
for (const item of top) {
|
|
619
|
+
const row = this.db.prepare("SELECT * FROM knowledge_entities WHERE id = ?").get(item.id);
|
|
620
|
+
if (row !== void 0) out.push({
|
|
621
|
+
entity: rowToEntity(row),
|
|
622
|
+
score: item.score
|
|
623
|
+
});
|
|
624
|
+
}
|
|
625
|
+
return out;
|
|
626
|
+
}
|
|
627
|
+
async insertEventEntity(params) {
|
|
628
|
+
if (this.db === void 0) throw new Error("knowledge store not initialized");
|
|
629
|
+
const id = generateId("ee");
|
|
630
|
+
this.db.prepare(`INSERT OR IGNORE INTO knowledge_event_entities
|
|
631
|
+
(id, event_id, entity_id, weight, description, embedding_json)
|
|
632
|
+
VALUES (?, ?, ?, ?, ?, ?)`).run(id, params.eventId, params.entityId, params.weight ?? 1, params.description, vectorToJson(params.embedding));
|
|
633
|
+
return {
|
|
634
|
+
id,
|
|
635
|
+
eventId: params.eventId,
|
|
636
|
+
entityId: params.entityId,
|
|
637
|
+
weight: params.weight ?? 1,
|
|
638
|
+
description: params.description,
|
|
639
|
+
embedding: params.embedding
|
|
640
|
+
};
|
|
641
|
+
}
|
|
642
|
+
async findEventsByEntity(entityId) {
|
|
643
|
+
await this.init();
|
|
644
|
+
if (this.db === void 0) return [];
|
|
645
|
+
return this.db.prepare(`SELECT e.* FROM knowledge_events e
|
|
646
|
+
JOIN knowledge_event_entities ee ON ee.event_id = e.id
|
|
647
|
+
WHERE ee.entity_id = ?`).all(entityId).map(rowToEvent);
|
|
648
|
+
}
|
|
649
|
+
async findEntitiesByEvent(eventId) {
|
|
650
|
+
await this.init();
|
|
651
|
+
if (this.db === void 0) return [];
|
|
652
|
+
return this.db.prepare(`SELECT en.* FROM knowledge_entities en
|
|
653
|
+
JOIN knowledge_event_entities ee ON ee.entity_id = en.id
|
|
654
|
+
WHERE ee.event_id = ?`).all(eventId).map(rowToEntity);
|
|
655
|
+
}
|
|
656
|
+
async listEntities(sourceId) {
|
|
657
|
+
await this.init();
|
|
658
|
+
if (this.db === void 0) return [];
|
|
659
|
+
const sql = sourceId ? `SELECT e.*, COUNT(ee.id) AS event_count
|
|
660
|
+
FROM knowledge_entities e
|
|
661
|
+
LEFT JOIN knowledge_event_entities ee ON ee.entity_id = e.id
|
|
662
|
+
WHERE e.source_id = ?
|
|
663
|
+
GROUP BY e.id
|
|
664
|
+
ORDER BY event_count DESC` : `SELECT e.*, COUNT(ee.id) AS event_count
|
|
665
|
+
FROM knowledge_entities e
|
|
666
|
+
LEFT JOIN knowledge_event_entities ee ON ee.entity_id = e.id
|
|
667
|
+
GROUP BY e.id
|
|
668
|
+
ORDER BY event_count DESC`;
|
|
669
|
+
return (sourceId ? this.db.prepare(sql).all(sourceId) : this.db.prepare(sql).all()).map((row) => ({
|
|
670
|
+
...rowToEntity(row),
|
|
671
|
+
eventCount: Number(row["event_count"] ?? 0)
|
|
672
|
+
}));
|
|
673
|
+
}
|
|
674
|
+
async listEvents(sourceId) {
|
|
675
|
+
await this.init();
|
|
676
|
+
if (this.db === void 0) return [];
|
|
677
|
+
const sql = sourceId ? "SELECT * FROM knowledge_events WHERE source_id = ? ORDER BY rank ASC" : "SELECT * FROM knowledge_events ORDER BY created_at DESC";
|
|
678
|
+
return (sourceId ? this.db.prepare(sql).all(sourceId) : this.db.prepare(sql).all()).map(rowToEvent);
|
|
679
|
+
}
|
|
680
|
+
async listEventEntities(sourceId) {
|
|
681
|
+
await this.init();
|
|
682
|
+
if (this.db === void 0) return [];
|
|
683
|
+
const sql = sourceId ? `SELECT ee.event_id, ee.entity_id
|
|
684
|
+
FROM knowledge_event_entities ee
|
|
685
|
+
JOIN knowledge_events e ON e.id = ee.event_id
|
|
686
|
+
WHERE e.source_id = ?` : "SELECT event_id, entity_id FROM knowledge_event_entities";
|
|
687
|
+
return (sourceId ? this.db.prepare(sql).all(sourceId) : this.db.prepare(sql).all()).map((row) => ({
|
|
688
|
+
eventId: asString(row["event_id"]),
|
|
689
|
+
entityId: asString(row["entity_id"])
|
|
690
|
+
}));
|
|
691
|
+
}
|
|
692
|
+
async ftsSearchChunks(query, limit = 50) {
|
|
693
|
+
await this.init();
|
|
694
|
+
if (this.db === void 0) return [];
|
|
695
|
+
const ftsQuery = buildFtsQuery(query);
|
|
696
|
+
if (ftsQuery === void 0) return [];
|
|
697
|
+
return this.db.prepare(`SELECT c.* FROM knowledge_chunks c
|
|
698
|
+
JOIN knowledge_chunks_fts f ON c.rowid = f.rowid
|
|
699
|
+
WHERE f.knowledge_chunks_fts MATCH ?
|
|
700
|
+
LIMIT ?`).all(ftsQuery, limit).map(rowToChunk);
|
|
701
|
+
}
|
|
702
|
+
async searchChunksByVector(queryVec, options = {}) {
|
|
703
|
+
await this.init();
|
|
704
|
+
if (this.db === void 0) return [];
|
|
705
|
+
const limit = options.limit ?? 50;
|
|
706
|
+
const threshold = options.threshold ?? 0;
|
|
707
|
+
const rows = this.db.prepare("SELECT id, embedding_json FROM knowledge_chunks WHERE embedding_json IS NOT NULL").all();
|
|
708
|
+
const scored = [];
|
|
709
|
+
for (const row of rows) {
|
|
710
|
+
const vec = jsonToVector(row.embedding_json);
|
|
711
|
+
if (vec === null) continue;
|
|
712
|
+
const score = this.embeddingEngine?.cosineSimilarity(queryVec, vec) ?? 0;
|
|
713
|
+
if (score >= threshold) scored.push({
|
|
714
|
+
id: row.id,
|
|
715
|
+
score
|
|
716
|
+
});
|
|
717
|
+
}
|
|
718
|
+
scored.sort((a, b) => b.score - a.score);
|
|
719
|
+
const top = scored.slice(0, limit);
|
|
720
|
+
const out = [];
|
|
721
|
+
for (const item of top) {
|
|
722
|
+
const chunk = await this.getChunk(item.id);
|
|
723
|
+
if (chunk !== void 0) out.push({
|
|
724
|
+
chunk,
|
|
725
|
+
score: item.score
|
|
726
|
+
});
|
|
727
|
+
}
|
|
728
|
+
return out;
|
|
729
|
+
}
|
|
730
|
+
async stats() {
|
|
731
|
+
await this.init();
|
|
732
|
+
if (this.db === void 0) return {
|
|
733
|
+
sources: 0,
|
|
734
|
+
documents: 0,
|
|
735
|
+
chunks: 0,
|
|
736
|
+
events: 0,
|
|
737
|
+
entities: 0
|
|
738
|
+
};
|
|
739
|
+
const count = (table) => {
|
|
740
|
+
return this.db.prepare(`SELECT COUNT(*) AS c FROM ${table}`).get()?.c ?? 0;
|
|
741
|
+
};
|
|
742
|
+
return {
|
|
743
|
+
sources: count("knowledge_sources"),
|
|
744
|
+
documents: count("knowledge_documents"),
|
|
745
|
+
chunks: count("knowledge_chunks"),
|
|
746
|
+
events: count("knowledge_events"),
|
|
747
|
+
entities: count("knowledge_entities")
|
|
748
|
+
};
|
|
749
|
+
}
|
|
750
|
+
async buildSearchResult(chunkId, score, eventId = null) {
|
|
751
|
+
if (this.db === void 0) return void 0;
|
|
752
|
+
const chunk = await this.getChunk(chunkId);
|
|
753
|
+
if (chunk === void 0) return void 0;
|
|
754
|
+
if (this.db.prepare("SELECT * FROM knowledge_documents WHERE id = ?").get(chunk.documentId) === void 0) return void 0;
|
|
755
|
+
const sourceRow = this.db.prepare("SELECT * FROM knowledge_sources WHERE id = ?").get(chunk.sourceId);
|
|
756
|
+
if (sourceRow === void 0) return void 0;
|
|
757
|
+
let eventTitle = null;
|
|
758
|
+
if (eventId !== null) eventTitle = (await this.getEvent(eventId))?.title ?? null;
|
|
759
|
+
return {
|
|
760
|
+
chunkId: chunk.id,
|
|
761
|
+
documentId: chunk.documentId,
|
|
762
|
+
sourceId: chunk.sourceId,
|
|
763
|
+
sourceName: String(sourceRow["name"]),
|
|
764
|
+
heading: chunk.heading,
|
|
765
|
+
content: chunk.content,
|
|
766
|
+
score,
|
|
767
|
+
eventId,
|
|
768
|
+
eventTitle
|
|
769
|
+
};
|
|
770
|
+
}
|
|
771
|
+
};
|
|
772
|
+
function asString(value) {
|
|
773
|
+
if (typeof value === "string") return value;
|
|
774
|
+
if (typeof value === "number" || typeof value === "bigint" || typeof value === "boolean") return String(value);
|
|
775
|
+
return "";
|
|
776
|
+
}
|
|
777
|
+
function asNullableString(value) {
|
|
778
|
+
if (value === null || value === void 0) return null;
|
|
779
|
+
if (typeof value === "string") return value;
|
|
780
|
+
if (typeof value === "number" || typeof value === "bigint" || typeof value === "boolean") return String(value);
|
|
781
|
+
return null;
|
|
782
|
+
}
|
|
783
|
+
function rowToSource(row) {
|
|
784
|
+
return {
|
|
785
|
+
id: asString(row["id"]),
|
|
786
|
+
name: asString(row["name"]),
|
|
787
|
+
filePath: asNullableString(row["file_path"]),
|
|
788
|
+
description: asNullableString(row["description"]),
|
|
789
|
+
createdAt: Number(row["created_at"])
|
|
790
|
+
};
|
|
791
|
+
}
|
|
792
|
+
function rowToDocument(row) {
|
|
793
|
+
return {
|
|
794
|
+
id: asString(row["id"]),
|
|
795
|
+
sourceId: asString(row["source_id"]),
|
|
796
|
+
title: asString(row["title"]),
|
|
797
|
+
content: asNullableString(row["content"]),
|
|
798
|
+
status: asString(row["status"]),
|
|
799
|
+
chunkCount: Number(row["chunk_count"] ?? 0),
|
|
800
|
+
createdAt: Number(row["created_at"])
|
|
801
|
+
};
|
|
802
|
+
}
|
|
803
|
+
function rowToChunk(row) {
|
|
804
|
+
return {
|
|
805
|
+
id: asString(row["id"]),
|
|
806
|
+
sourceId: asString(row["source_id"]),
|
|
807
|
+
documentId: asString(row["document_id"]),
|
|
808
|
+
rank: Number(row["rank"]),
|
|
809
|
+
heading: asNullableString(row["heading"]),
|
|
810
|
+
content: asString(row["content"]),
|
|
811
|
+
rawContent: asNullableString(row["raw_content"]),
|
|
812
|
+
embedding: jsonToVector(asNullableString(row["embedding_json"])),
|
|
813
|
+
createdAt: Number(row["created_at"])
|
|
814
|
+
};
|
|
815
|
+
}
|
|
816
|
+
function rowToEvent(row) {
|
|
817
|
+
let keywords = [];
|
|
818
|
+
try {
|
|
819
|
+
const parsed = JSON.parse(asString(row["keywords"] ?? "[]"));
|
|
820
|
+
if (Array.isArray(parsed)) keywords = parsed.filter((k) => typeof k === "string");
|
|
821
|
+
} catch {}
|
|
822
|
+
return {
|
|
823
|
+
id: asString(row["id"]),
|
|
824
|
+
sourceId: asString(row["source_id"]),
|
|
825
|
+
documentId: asString(row["document_id"]),
|
|
826
|
+
chunkId: asString(row["chunk_id"]),
|
|
827
|
+
rank: Number(row["rank"]),
|
|
828
|
+
title: asString(row["title"]),
|
|
829
|
+
summary: asNullableString(row["summary"]),
|
|
830
|
+
content: asString(row["content"]),
|
|
831
|
+
category: asNullableString(row["category"]),
|
|
832
|
+
keywords,
|
|
833
|
+
titleEmbedding: jsonToVector(asNullableString(row["title_embedding_json"])),
|
|
834
|
+
contentEmbedding: jsonToVector(asNullableString(row["content_embedding_json"])),
|
|
835
|
+
createdAt: Number(row["created_at"])
|
|
836
|
+
};
|
|
837
|
+
}
|
|
838
|
+
function rowToEntity(row) {
|
|
839
|
+
return {
|
|
840
|
+
id: asString(row["id"]),
|
|
841
|
+
sourceId: asString(row["source_id"]),
|
|
842
|
+
type: asString(row["type"]),
|
|
843
|
+
name: asString(row["name"]),
|
|
844
|
+
normalizedName: asString(row["normalized_name"]),
|
|
845
|
+
description: asNullableString(row["description"]),
|
|
846
|
+
embedding: jsonToVector(asNullableString(row["embedding_json"])),
|
|
847
|
+
createdAt: Number(row["created_at"])
|
|
848
|
+
};
|
|
849
|
+
}
|
|
850
|
+
/**
|
|
851
|
+
* Tokenize text so FTS5's unicode61 tokenizer can index mixed CJK/ASCII text.
|
|
852
|
+
* Mirrors the memory package's `toFtsText`.
|
|
853
|
+
*/
|
|
854
|
+
function toFtsText(text) {
|
|
855
|
+
const parts = text.toLowerCase().replaceAll(/([一-鿿㐀-䶿])([a-z0-9])/g, "$1 $2").replaceAll(/([a-z0-9])([一-鿿㐀-䶿])/g, "$1 $2").split(/[^a-z0-9一-鿿㐀-䶿]+/);
|
|
856
|
+
const tokens = [];
|
|
857
|
+
for (const part of parts) {
|
|
858
|
+
if (part.length === 0) continue;
|
|
859
|
+
if (/^[a-z0-9]+$/.test(part)) tokens.push(part);
|
|
860
|
+
else for (const ch of part) if (ch.length > 0) tokens.push(ch);
|
|
861
|
+
}
|
|
862
|
+
return tokens.join(" ");
|
|
863
|
+
}
|
|
864
|
+
function buildFtsQuery(search) {
|
|
865
|
+
const tokens = toFtsText(search).split(/\s+/).filter((t) => t.length > 0);
|
|
866
|
+
if (tokens.length === 0) return void 0;
|
|
867
|
+
return tokens.map((t) => `"${t.replaceAll("\"", "\"\"")}"`).join(" AND ");
|
|
868
|
+
}
|
|
869
|
+
const HEADING_RE = /^(#{1,6})\s+(.+?)\s*$/;
|
|
870
|
+
const CODE_FENCE_RE = /^(\s*)(`{3,}|~{3,})/;
|
|
871
|
+
/**
|
|
872
|
+
* Soft cap on chunk token count. BGE-small-zh-v1.5 (the fastembed model used
|
|
873
|
+
* here) has a 512-token sequence limit; chunks longer than that get truncated
|
|
874
|
+
* at the embedding layer, hurting retrieval quality. We split at ~480 to leave
|
|
875
|
+
* headroom for the heading line that gets prepended at embed time.
|
|
876
|
+
*/
|
|
877
|
+
const MAX_CHUNK_TOKENS = 480;
|
|
878
|
+
/**
|
|
879
|
+
* Split a markdown document into sections using heading_strict strategy:
|
|
880
|
+
* each section begins at a heading and contains all body content until the
|
|
881
|
+
* next heading. Consecutive headings collapse to the lowest-level one as the
|
|
882
|
+
* main heading. Content before the first heading is grouped as "Introduction".
|
|
883
|
+
*
|
|
884
|
+
* Mirrors SAG's `buildHeadingStrictSections`.
|
|
885
|
+
*/
|
|
886
|
+
function chunkMarkdown(content) {
|
|
887
|
+
const lines = content.split("\n");
|
|
888
|
+
const sections = [];
|
|
889
|
+
let buffer = [];
|
|
890
|
+
let currentHeading = null;
|
|
891
|
+
let currentLevel = null;
|
|
892
|
+
let rank = 0;
|
|
893
|
+
let inFence = false;
|
|
894
|
+
const flush = () => {
|
|
895
|
+
const raw = buffer.join("\n").trim();
|
|
896
|
+
const heading = currentHeading;
|
|
897
|
+
const level = currentLevel;
|
|
898
|
+
buffer = [];
|
|
899
|
+
if (raw.length === 0) return;
|
|
900
|
+
const body = stripMarkdown(raw);
|
|
901
|
+
if (body.length === 0 && heading === null) return;
|
|
902
|
+
const subSections = splitLargeSection({
|
|
903
|
+
heading,
|
|
904
|
+
headingLevel: level,
|
|
905
|
+
content: body,
|
|
906
|
+
rawContent: raw,
|
|
907
|
+
rank
|
|
908
|
+
});
|
|
909
|
+
for (const section of subSections) {
|
|
910
|
+
sections.push({
|
|
911
|
+
...section,
|
|
912
|
+
rank
|
|
913
|
+
});
|
|
914
|
+
rank += 1;
|
|
915
|
+
}
|
|
916
|
+
};
|
|
917
|
+
for (const line of lines) {
|
|
918
|
+
if (CODE_FENCE_RE.exec(line)) {
|
|
919
|
+
inFence = !inFence;
|
|
920
|
+
buffer.push(line);
|
|
921
|
+
continue;
|
|
922
|
+
}
|
|
923
|
+
if (inFence) {
|
|
924
|
+
buffer.push(line);
|
|
925
|
+
continue;
|
|
926
|
+
}
|
|
927
|
+
const headingMatch = HEADING_RE.exec(line);
|
|
928
|
+
if (headingMatch) {
|
|
929
|
+
flush();
|
|
930
|
+
currentHeading = headingMatch[2].trim();
|
|
931
|
+
currentLevel = headingMatch[1].length;
|
|
932
|
+
} else buffer.push(line);
|
|
933
|
+
}
|
|
934
|
+
flush();
|
|
935
|
+
return sections;
|
|
936
|
+
}
|
|
937
|
+
/**
|
|
938
|
+
* Strip markdown decorations that hurt embedding quality:
|
|
939
|
+
* code fences, inline code, images, links, emphasis markers.
|
|
940
|
+
*
|
|
941
|
+
* Mirrors SAG's `stripMarkdown`.
|
|
942
|
+
*/
|
|
943
|
+
function stripMarkdown(text) {
|
|
944
|
+
return text.replaceAll(/^(\s*)```[^\n]*$\n?/gm, "$1").replaceAll(/^(\s*)```[^\n]*$/gm, "").replaceAll(/`([^`]+)`/g, "$1").replaceAll(/!\[([^\]]*)\]\([^)]+\)/g, "$1").replaceAll(/\[([^\]]+)\]\([^)]+\)/g, "$1").replaceAll(/\*\*([^*]+)\*\*/g, "$1").replaceAll(/__([^_]+)__/g, "$1").replaceAll(/(?<!\w)\*([^*\n]+)\*(?!\w)/g, "$1").replaceAll(/(?<!\w)_([^_\n]+)_(?!\w)/g, "$1").replaceAll(/^#{1,6}\s+/gm, "").replaceAll(/\n{3,}/g, "\n\n").trim();
|
|
945
|
+
}
|
|
946
|
+
/**
|
|
947
|
+
* Split a plain text file into chunks by blank-line paragraphs.
|
|
948
|
+
* Each non-empty paragraph becomes one chunk with no heading.
|
|
949
|
+
* Long paragraphs are split at sentence boundaries to stay under the
|
|
950
|
+
* embedding model's token cap (same MAX_CHUNK_TOKENS as markdown).
|
|
951
|
+
*/
|
|
952
|
+
function chunkText(content) {
|
|
953
|
+
const MAX_PARA_TOKENS = MAX_CHUNK_TOKENS;
|
|
954
|
+
const MAX_PARA_CHARS = MAX_PARA_TOKENS * 4;
|
|
955
|
+
const paragraphs = content.split(/\n\s*\n/).map((p) => p.trim()).filter((p) => p.length > 0);
|
|
956
|
+
const sections = [];
|
|
957
|
+
let rank = 0;
|
|
958
|
+
for (const para of paragraphs) {
|
|
959
|
+
if (estimateTokens(para) <= MAX_PARA_TOKENS) {
|
|
960
|
+
sections.push({
|
|
961
|
+
heading: null,
|
|
962
|
+
headingLevel: null,
|
|
963
|
+
content: para,
|
|
964
|
+
rawContent: para,
|
|
965
|
+
rank: rank++
|
|
966
|
+
});
|
|
967
|
+
continue;
|
|
968
|
+
}
|
|
969
|
+
const sentences = para.match(/[^.!?。!?\n]+[.!?。!?\n]+|[^.!?。!?\n]+$/g) ?? [para];
|
|
970
|
+
let buffer = "";
|
|
971
|
+
const flushBuffer = () => {
|
|
972
|
+
const text = buffer.trim();
|
|
973
|
+
if (text.length === 0) return;
|
|
974
|
+
sections.push({
|
|
975
|
+
heading: null,
|
|
976
|
+
headingLevel: null,
|
|
977
|
+
content: text,
|
|
978
|
+
rawContent: text,
|
|
979
|
+
rank: rank++
|
|
980
|
+
});
|
|
981
|
+
buffer = "";
|
|
982
|
+
};
|
|
983
|
+
for (const sentence of sentences) {
|
|
984
|
+
const candidate = (buffer + " " + sentence).trim();
|
|
985
|
+
if (buffer.length > 0 && candidate.length > MAX_PARA_CHARS) flushBuffer();
|
|
986
|
+
buffer = buffer.length === 0 ? sentence : `${buffer} ${sentence}`;
|
|
987
|
+
}
|
|
988
|
+
flushBuffer();
|
|
989
|
+
}
|
|
990
|
+
return sections;
|
|
991
|
+
}
|
|
992
|
+
/** Rough token estimate — chars / 4 (English-leaning; CJK underestimates). */
|
|
993
|
+
function estimateTokens(text) {
|
|
994
|
+
return Math.ceil(text.length / 4);
|
|
995
|
+
}
|
|
996
|
+
/**
|
|
997
|
+
* Split a section that exceeds MAX_CHUNK_TOKENS into smaller sub-sections.
|
|
998
|
+
* Strategy (mirrors SAG's `splitLargeSection`):
|
|
999
|
+
* 1. Split into paragraphs by blank lines.
|
|
1000
|
+
* 2. Greedily pack paragraphs into chunks under the token cap.
|
|
1001
|
+
* 3. A paragraph that alone exceeds the cap is split at sentence boundaries;
|
|
1002
|
+
* a sentence that itself exceeds the cap is split by character limit.
|
|
1003
|
+
*
|
|
1004
|
+
* Sections under the cap are returned unchanged (single-element array).
|
|
1005
|
+
* The heading is preserved across all sub-sections; rank is reassigned by the
|
|
1006
|
+
* caller. rawContent is rebuilt from the sub-chunk's content paragraphs.
|
|
1007
|
+
*/
|
|
1008
|
+
function splitLargeSection(section) {
|
|
1009
|
+
if (estimateTokens(section.content) <= MAX_CHUNK_TOKENS) return [section];
|
|
1010
|
+
const paragraphs = section.rawContent.split(/\n{2,}/).map((p) => p.trim()).filter((p) => p.length > 0);
|
|
1011
|
+
if (paragraphs.length <= 1) return splitBySentences(section, section.rawContent);
|
|
1012
|
+
const result = [];
|
|
1013
|
+
let buffer = [];
|
|
1014
|
+
let bufferTokens = 0;
|
|
1015
|
+
const flushBuffer = () => {
|
|
1016
|
+
if (buffer.length === 0) return;
|
|
1017
|
+
const raw = buffer.join("\n\n").trim();
|
|
1018
|
+
if (raw.length === 0) {
|
|
1019
|
+
buffer = [];
|
|
1020
|
+
bufferTokens = 0;
|
|
1021
|
+
return;
|
|
1022
|
+
}
|
|
1023
|
+
result.push({
|
|
1024
|
+
heading: section.heading,
|
|
1025
|
+
headingLevel: section.headingLevel,
|
|
1026
|
+
content: stripMarkdown(raw),
|
|
1027
|
+
rawContent: raw,
|
|
1028
|
+
rank: 0
|
|
1029
|
+
});
|
|
1030
|
+
buffer = [];
|
|
1031
|
+
bufferTokens = 0;
|
|
1032
|
+
};
|
|
1033
|
+
for (const para of paragraphs) {
|
|
1034
|
+
const paraTokens = estimateTokens(para);
|
|
1035
|
+
if (paraTokens > MAX_CHUNK_TOKENS) {
|
|
1036
|
+
flushBuffer();
|
|
1037
|
+
for (const frag of splitBySentences(section, para)) result.push(frag);
|
|
1038
|
+
continue;
|
|
1039
|
+
}
|
|
1040
|
+
if (buffer.length > 0 && bufferTokens + paraTokens > MAX_CHUNK_TOKENS) flushBuffer();
|
|
1041
|
+
buffer.push(para);
|
|
1042
|
+
bufferTokens += paraTokens;
|
|
1043
|
+
}
|
|
1044
|
+
flushBuffer();
|
|
1045
|
+
return result.length > 0 ? result : [section];
|
|
1046
|
+
}
|
|
1047
|
+
/**
|
|
1048
|
+
* Split a long text into sub-sections at sentence boundaries.
|
|
1049
|
+
* Falls back to a hard character cut when a single sentence exceeds the cap.
|
|
1050
|
+
*/
|
|
1051
|
+
function splitBySentences(section, text) {
|
|
1052
|
+
const sentences = splitSentences(text);
|
|
1053
|
+
const result = [];
|
|
1054
|
+
let buffer = "";
|
|
1055
|
+
let bufferTokens = 0;
|
|
1056
|
+
const flush = () => {
|
|
1057
|
+
const trimmed = buffer.trim();
|
|
1058
|
+
if (trimmed.length === 0) {
|
|
1059
|
+
buffer = "";
|
|
1060
|
+
bufferTokens = 0;
|
|
1061
|
+
return;
|
|
1062
|
+
}
|
|
1063
|
+
result.push({
|
|
1064
|
+
heading: section.heading,
|
|
1065
|
+
headingLevel: section.headingLevel,
|
|
1066
|
+
content: stripMarkdown(trimmed),
|
|
1067
|
+
rawContent: trimmed,
|
|
1068
|
+
rank: 0
|
|
1069
|
+
});
|
|
1070
|
+
buffer = "";
|
|
1071
|
+
bufferTokens = 0;
|
|
1072
|
+
};
|
|
1073
|
+
for (const sentence of sentences) {
|
|
1074
|
+
const sentTokens = estimateTokens(sentence);
|
|
1075
|
+
if (sentTokens > MAX_CHUNK_TOKENS) {
|
|
1076
|
+
flush();
|
|
1077
|
+
for (const frag of splitByCharLimit(sentence)) result.push({
|
|
1078
|
+
heading: section.heading,
|
|
1079
|
+
headingLevel: section.headingLevel,
|
|
1080
|
+
content: stripMarkdown(frag),
|
|
1081
|
+
rawContent: frag,
|
|
1082
|
+
rank: 0
|
|
1083
|
+
});
|
|
1084
|
+
continue;
|
|
1085
|
+
}
|
|
1086
|
+
if (buffer.length > 0 && bufferTokens + sentTokens > MAX_CHUNK_TOKENS) flush();
|
|
1087
|
+
buffer = buffer.length === 0 ? sentence : `${buffer} ${sentence}`;
|
|
1088
|
+
bufferTokens += sentTokens;
|
|
1089
|
+
}
|
|
1090
|
+
flush();
|
|
1091
|
+
return result.length > 0 ? result : [{
|
|
1092
|
+
heading: section.heading,
|
|
1093
|
+
headingLevel: section.headingLevel,
|
|
1094
|
+
content: section.content,
|
|
1095
|
+
rawContent: section.rawContent,
|
|
1096
|
+
rank: 0
|
|
1097
|
+
}];
|
|
1098
|
+
}
|
|
1099
|
+
/** Split text into sentences using CJK + ASCII punctuation. */
|
|
1100
|
+
function splitSentences(text) {
|
|
1101
|
+
return text.match(/[^。!?!?.\n]+[。!?!?.\n]+|[^。!?!?.\n]+$/g) ?? [text];
|
|
1102
|
+
}
|
|
1103
|
+
/** Hard character cut for pathological single sentences longer than the cap. */
|
|
1104
|
+
function splitByCharLimit(text) {
|
|
1105
|
+
const maxChars = MAX_CHUNK_TOKENS * 4;
|
|
1106
|
+
const chunks = [];
|
|
1107
|
+
let remaining = text.trim();
|
|
1108
|
+
while (remaining.length > maxChars) {
|
|
1109
|
+
chunks.push(remaining.slice(0, maxChars));
|
|
1110
|
+
remaining = remaining.slice(maxChars).trimStart();
|
|
1111
|
+
}
|
|
1112
|
+
if (remaining.length > 0) chunks.push(remaining);
|
|
1113
|
+
return chunks;
|
|
1114
|
+
}
|
|
1115
|
+
/** Max LLM call attempts for extraction — fails fall back to a heading-derived event. */
|
|
1116
|
+
const EXTRACTION_MAX_ATTEMPTS = 3;
|
|
1117
|
+
/** Base delay (ms) for exponential backoff: 100, 200, 400. */
|
|
1118
|
+
const EXTRACTION_BACKOFF_BASE_MS = 100;
|
|
1119
|
+
function backoffDelay(attempt) {
|
|
1120
|
+
return EXTRACTION_BACKOFF_BASE_MS * 2 ** Math.max(0, attempt - 1);
|
|
1121
|
+
}
|
|
1122
|
+
async function callLlmWithBackoff(llm, systemPrompt, userPrompt) {
|
|
1123
|
+
let lastError;
|
|
1124
|
+
for (let attempt = 1; attempt <= EXTRACTION_MAX_ATTEMPTS; attempt += 1) try {
|
|
1125
|
+
return await llm.generate(systemPrompt, userPrompt);
|
|
1126
|
+
} catch (error) {
|
|
1127
|
+
lastError = error;
|
|
1128
|
+
if (attempt < EXTRACTION_MAX_ATTEMPTS) await new Promise((resolve) => setTimeout(resolve, backoffDelay(attempt)));
|
|
1129
|
+
}
|
|
1130
|
+
throw lastError;
|
|
1131
|
+
}
|
|
1132
|
+
const ENTITY_TYPES = [
|
|
1133
|
+
"person",
|
|
1134
|
+
"organization",
|
|
1135
|
+
"location",
|
|
1136
|
+
"time",
|
|
1137
|
+
"product",
|
|
1138
|
+
"metric",
|
|
1139
|
+
"action",
|
|
1140
|
+
"work",
|
|
1141
|
+
"group",
|
|
1142
|
+
"subject",
|
|
1143
|
+
"tags"
|
|
1144
|
+
];
|
|
1145
|
+
const EXTRACTION_SYSTEM_PROMPT = `You are a professional knowledge content extractor for a SAG-style event-centric knowledge graph.
|
|
1146
|
+
|
|
1147
|
+
Your task: given a markdown section (one chunk), extract exactly ONE fused event that captures the section's core informational content, plus the entities mentioned.
|
|
1148
|
+
|
|
1149
|
+
# Output Format
|
|
1150
|
+
|
|
1151
|
+
Return a JSON object with this exact shape:
|
|
1152
|
+
|
|
1153
|
+
{
|
|
1154
|
+
"items": [
|
|
1155
|
+
{
|
|
1156
|
+
"title": "short event title (≤80 chars, noun phrase or short sentence)",
|
|
1157
|
+
"summary": "one-sentence summary of what this section is about",
|
|
1158
|
+
"content": "the full informational content of this section, rephrased as a self-contained event description (1-4 sentences)",
|
|
1159
|
+
"category": "one of: definition, process, comparison, decision, history, reference, other",
|
|
1160
|
+
"keywords": ["3-5 short keywords or phrases"],
|
|
1161
|
+
"entities": [
|
|
1162
|
+
{ "type": "one of ENTITY_TYPES below", "name": "entity name", "description": "one-line context for the entity" }
|
|
1163
|
+
]
|
|
1164
|
+
}
|
|
1165
|
+
]
|
|
1166
|
+
}
|
|
1167
|
+
|
|
1168
|
+
# Entity Types
|
|
1169
|
+
|
|
1170
|
+
Use only these types:
|
|
1171
|
+
- person — named individuals
|
|
1172
|
+
- organization — companies, teams, institutions
|
|
1173
|
+
- location — places, geographic regions
|
|
1174
|
+
- time — dates, time periods, eras
|
|
1175
|
+
- product — products, services, software, tools
|
|
1176
|
+
- metric — numbers, statistics, measurements
|
|
1177
|
+
- action — processes, operations, activities
|
|
1178
|
+
- work — books, papers, projects, artworks
|
|
1179
|
+
- group — categories, classes, groups of things
|
|
1180
|
+
- subject — abstract topics, concepts, fields
|
|
1181
|
+
- tags — use sparingly for cross-cutting labels
|
|
1182
|
+
|
|
1183
|
+
# Rules
|
|
1184
|
+
|
|
1185
|
+
1. Produce EXACTLY ONE item in the "items" array.
|
|
1186
|
+
2. The event title should be specific enough to disambiguate from sibling sections.
|
|
1187
|
+
3. The content must be self-contained — a reader should understand it without the surrounding context.
|
|
1188
|
+
4. Extract 2-6 entities per chunk. Only include entities that are clearly referenced in the text.
|
|
1189
|
+
5. Use the entity's most specific name (e.g. "Anthropic" rather than "the company").
|
|
1190
|
+
6. Output language must follow the main input language. Chinese input requires Chinese title, summary, content, category, keywords, and entity descriptions. English input requires English fields.
|
|
1191
|
+
7. Output ONLY the JSON object — no prose, no markdown fences, no commentary.
|
|
1192
|
+
|
|
1193
|
+
# Example
|
|
1194
|
+
|
|
1195
|
+
Input section:
|
|
1196
|
+
"""
|
|
1197
|
+
## Model Context Protocol
|
|
1198
|
+
|
|
1199
|
+
The Model Context Protocol (MCP) is an open standard introduced by Anthropic in 2024. It defines a JSON-RPC interface between AI assistants and external tools. MCP servers expose resources, prompts, and tools that clients like Claude Desktop can consume.
|
|
1200
|
+
"""
|
|
1201
|
+
|
|
1202
|
+
Output:
|
|
1203
|
+
{
|
|
1204
|
+
"items": [
|
|
1205
|
+
{
|
|
1206
|
+
"title": "Model Context Protocol introduction",
|
|
1207
|
+
"summary": "MCP is an open standard for AI assistant / tool integration introduced by Anthropic in 2024.",
|
|
1208
|
+
"content": "The Model Context Protocol (MCP) is an open standard introduced by Anthropic in 2024 that defines a JSON-RPC interface between AI assistants and external tools. MCP servers expose resources, prompts, and tools that clients like Claude Desktop can consume.",
|
|
1209
|
+
"category": "definition",
|
|
1210
|
+
"keywords": ["MCP", "JSON-RPC", "Anthropic", "AI tools"],
|
|
1211
|
+
"entities": [
|
|
1212
|
+
{ "type": "product", "name": "Model Context Protocol", "description": "Open standard for AI assistant / tool integration" },
|
|
1213
|
+
{ "type": "organization", "name": "Anthropic", "description": "Company that introduced MCP in 2024" },
|
|
1214
|
+
{ "type": "time", "name": "2024", "description": "Year MCP was introduced" },
|
|
1215
|
+
{ "type": "product", "name": "Claude Desktop", "description": "An MCP client application" }
|
|
1216
|
+
]
|
|
1217
|
+
}
|
|
1218
|
+
]
|
|
1219
|
+
}`;
|
|
1220
|
+
/**
|
|
1221
|
+
* Build the user prompt for extracting events/entities from a chunk.
|
|
1222
|
+
* Includes a one-shot example to anchor the expected output shape.
|
|
1223
|
+
*/
|
|
1224
|
+
function buildExtractionUserPrompt(chunk) {
|
|
1225
|
+
return [
|
|
1226
|
+
"Extract the fused event and entities from the following markdown section.",
|
|
1227
|
+
"",
|
|
1228
|
+
`Section heading: ${chunk.heading ?? "(no heading)"}`,
|
|
1229
|
+
"",
|
|
1230
|
+
"Section content:",
|
|
1231
|
+
"\"\"\"",
|
|
1232
|
+
chunk.content,
|
|
1233
|
+
"\"\"\"",
|
|
1234
|
+
"",
|
|
1235
|
+
"Return only the JSON object per the schema."
|
|
1236
|
+
].join("\n");
|
|
1237
|
+
}
|
|
1238
|
+
/** Extract the first {...} JSON block from a text response. */
|
|
1239
|
+
function extractJsonFromText(text) {
|
|
1240
|
+
const match = text.match(/\{[\s\S]*\}/);
|
|
1241
|
+
return match === null ? null : match[0];
|
|
1242
|
+
}
|
|
1243
|
+
/** Parse LLM extraction output into structured events. Returns at most 1 event. */
|
|
1244
|
+
function parseExtractionResponse(text, fallbackChunk) {
|
|
1245
|
+
const jsonText = extractJsonFromText(text);
|
|
1246
|
+
if (jsonText === null) return fallbackEvent(fallbackChunk);
|
|
1247
|
+
try {
|
|
1248
|
+
const items = JSON.parse(jsonText)["items"];
|
|
1249
|
+
if (!Array.isArray(items) || items.length === 0) return fallbackEvent(fallbackChunk);
|
|
1250
|
+
const first = items[0];
|
|
1251
|
+
if (first === null || typeof first !== "object") return fallbackEvent(fallbackChunk);
|
|
1252
|
+
const obj = first;
|
|
1253
|
+
return {
|
|
1254
|
+
title: typeof obj["title"] === "string" && obj["title"].trim().length > 0 ? obj["title"].trim() : fallbackChunk.heading ?? "Untitled",
|
|
1255
|
+
summary: typeof obj["summary"] === "string" ? obj["summary"].trim() : "",
|
|
1256
|
+
content: typeof obj["content"] === "string" && obj["content"].trim().length > 0 ? obj["content"].trim() : fallbackChunk.content,
|
|
1257
|
+
category: typeof obj["category"] === "string" ? obj["category"].trim() : "other",
|
|
1258
|
+
keywords: parseKeywords(obj["keywords"]),
|
|
1259
|
+
entities: parseEntities(obj["entities"])
|
|
1260
|
+
};
|
|
1261
|
+
} catch {
|
|
1262
|
+
return fallbackEvent(fallbackChunk);
|
|
1263
|
+
}
|
|
1264
|
+
}
|
|
1265
|
+
function parseKeywords(value) {
|
|
1266
|
+
if (!Array.isArray(value)) return [];
|
|
1267
|
+
const out = [];
|
|
1268
|
+
for (const item of value) if (typeof item === "string" && item.trim().length > 0) out.push(item.trim());
|
|
1269
|
+
return out.slice(0, 10);
|
|
1270
|
+
}
|
|
1271
|
+
function parseEntities(value) {
|
|
1272
|
+
if (!Array.isArray(value)) return [];
|
|
1273
|
+
const out = [];
|
|
1274
|
+
const seen = /* @__PURE__ */ new Set();
|
|
1275
|
+
for (const item of value) {
|
|
1276
|
+
if (item === null || typeof item !== "object") continue;
|
|
1277
|
+
const obj = item;
|
|
1278
|
+
const type = typeof obj["type"] === "string" ? obj["type"] : "";
|
|
1279
|
+
const name = typeof obj["name"] === "string" ? obj["name"].trim() : "";
|
|
1280
|
+
const description = typeof obj["description"] === "string" ? obj["description"].trim() : "";
|
|
1281
|
+
if (name.length === 0) continue;
|
|
1282
|
+
if (!ENTITY_TYPES.includes(type)) continue;
|
|
1283
|
+
const key = `${type}|${name.toLowerCase()}`;
|
|
1284
|
+
if (seen.has(key)) continue;
|
|
1285
|
+
seen.add(key);
|
|
1286
|
+
out.push({
|
|
1287
|
+
type,
|
|
1288
|
+
name,
|
|
1289
|
+
description
|
|
1290
|
+
});
|
|
1291
|
+
}
|
|
1292
|
+
return out.slice(0, 20);
|
|
1293
|
+
}
|
|
1294
|
+
function fallbackEvent(chunk) {
|
|
1295
|
+
return {
|
|
1296
|
+
title: chunk.heading ?? "Untitled section",
|
|
1297
|
+
summary: "",
|
|
1298
|
+
content: chunk.content,
|
|
1299
|
+
category: "other",
|
|
1300
|
+
keywords: [],
|
|
1301
|
+
entities: []
|
|
1302
|
+
};
|
|
1303
|
+
}
|
|
1304
|
+
/**
|
|
1305
|
+
* Run extraction on a single chunk via the LLM caller.
|
|
1306
|
+
* Retries with exponential backoff on transient failures; falls back to a
|
|
1307
|
+
* heading-derived event only after all attempts fail.
|
|
1308
|
+
*/
|
|
1309
|
+
async function extractEventFromChunk(llm, chunk) {
|
|
1310
|
+
try {
|
|
1311
|
+
return parseExtractionResponse(await callLlmWithBackoff(llm, EXTRACTION_SYSTEM_PROMPT, buildExtractionUserPrompt(chunk)), chunk);
|
|
1312
|
+
} catch {
|
|
1313
|
+
return fallbackEvent(chunk);
|
|
1314
|
+
}
|
|
1315
|
+
}
|
|
1316
|
+
/**
|
|
1317
|
+
* Ask the LLM to rerank candidate events by relevance to a query.
|
|
1318
|
+
* Returns the candidate ids in reranked order (most relevant first).
|
|
1319
|
+
*
|
|
1320
|
+
* If the LLM fails or returns unparseable output, returns the input order.
|
|
1321
|
+
*/
|
|
1322
|
+
async function rerankEventsWithLlm(llm, query, candidates, topK) {
|
|
1323
|
+
if (candidates.length === 0) return [];
|
|
1324
|
+
if (candidates.length <= topK) return candidates.map((c) => c.id);
|
|
1325
|
+
const candidateList = candidates.map((c, i) => `[${i}] id=${c.id}\n title: ${c.title}\n summary: ${c.summary}`).join("\n");
|
|
1326
|
+
const system = `You are a precise relevance judge for a knowledge base retrieval system.
|
|
1327
|
+
|
|
1328
|
+
Given a user query and a list of candidate events, select the ${topK} event ids most useful for answering the query.
|
|
1329
|
+
|
|
1330
|
+
Return ONLY a JSON object: {"ids": ["id1", "id2", ...]}
|
|
1331
|
+
|
|
1332
|
+
The ids must come from the candidate list. Output at most ${topK} ids, most relevant first. No prose, no markdown.`;
|
|
1333
|
+
const user = `Query: ${query}
|
|
1334
|
+
|
|
1335
|
+
Candidates:
|
|
1336
|
+
${candidateList}
|
|
1337
|
+
|
|
1338
|
+
Return the JSON object now.`;
|
|
1339
|
+
try {
|
|
1340
|
+
const jsonText = extractJsonFromText(await llm.generate(system, user));
|
|
1341
|
+
if (jsonText === null) return candidates.slice(0, topK).map((c) => c.id);
|
|
1342
|
+
const parsed = JSON.parse(jsonText);
|
|
1343
|
+
if (!Array.isArray(parsed["ids"])) return candidates.slice(0, topK).map((c) => c.id);
|
|
1344
|
+
const validIds = new Set(candidates.map((c) => c.id));
|
|
1345
|
+
const result = [];
|
|
1346
|
+
for (const id of parsed["ids"]) {
|
|
1347
|
+
if (typeof id === "string" && validIds.has(id) && !result.includes(id)) result.push(id);
|
|
1348
|
+
if (result.length >= topK) break;
|
|
1349
|
+
}
|
|
1350
|
+
if (result.length === 0) return candidates.slice(0, topK).map((c) => c.id);
|
|
1351
|
+
return result;
|
|
1352
|
+
} catch {
|
|
1353
|
+
return candidates.slice(0, topK).map((c) => c.id);
|
|
1354
|
+
}
|
|
1355
|
+
}
|
|
1356
|
+
/**
|
|
1357
|
+
* Extract named entities from a query for entity-recall retrieval.
|
|
1358
|
+
* Returns a list of {type, name} pairs. Best-effort — falls back to empty.
|
|
1359
|
+
*/
|
|
1360
|
+
async function extractQueryEntities(llm, query) {
|
|
1361
|
+
const system = `Extract named entities from the user query. Return ONLY a JSON object: {"entities": [{"type": "...", "name": "..."}]}
|
|
1362
|
+
|
|
1363
|
+
Use these entity types: ${ENTITY_TYPES.join(", ")}.
|
|
1364
|
+
|
|
1365
|
+
Only include entities that explicitly appear in the query. If no clear entities, return an empty array.`;
|
|
1366
|
+
try {
|
|
1367
|
+
const jsonText = extractJsonFromText(await llm.generate(system, `Query: ${query}`));
|
|
1368
|
+
if (jsonText === null) return [];
|
|
1369
|
+
const parsed = JSON.parse(jsonText);
|
|
1370
|
+
if (!Array.isArray(parsed["entities"])) return [];
|
|
1371
|
+
const out = [];
|
|
1372
|
+
for (const item of parsed["entities"]) {
|
|
1373
|
+
if (item === null || typeof item !== "object") continue;
|
|
1374
|
+
const obj = item;
|
|
1375
|
+
const type = typeof obj["type"] === "string" ? obj["type"] : "";
|
|
1376
|
+
const name = typeof obj["name"] === "string" ? obj["name"].trim() : "";
|
|
1377
|
+
if (name.length === 0) continue;
|
|
1378
|
+
if (!ENTITY_TYPES.includes(type)) continue;
|
|
1379
|
+
out.push({
|
|
1380
|
+
type,
|
|
1381
|
+
name
|
|
1382
|
+
});
|
|
1383
|
+
}
|
|
1384
|
+
return out;
|
|
1385
|
+
} catch {
|
|
1386
|
+
return [];
|
|
1387
|
+
}
|
|
1388
|
+
}
|
|
1389
|
+
/**
|
|
1390
|
+
* Embed texts, returning null-array if embeddings are skipped or on failure.
|
|
1391
|
+
* Caller should set skipEmbedding=true when the model didn't load in time,
|
|
1392
|
+
* so the entire ingestion is consistent (all or nothing for embeddings).
|
|
1393
|
+
* Search falls back to FTS5 keyword match for chunks without embeddings.
|
|
1394
|
+
*/
|
|
1395
|
+
async function embedOrNulls(engine, texts, skipEmbedding) {
|
|
1396
|
+
if (engine === void 0 || texts.length === 0 || skipEmbedding) return texts.map(() => null);
|
|
1397
|
+
if (!engine.available) return texts.map(() => null);
|
|
1398
|
+
const vectors = await engine.embedBatch(texts);
|
|
1399
|
+
if (vectors === null || vectors.length !== texts.length) return texts.map(() => null);
|
|
1400
|
+
return vectors;
|
|
1401
|
+
}
|
|
1402
|
+
/** LLM concurrency for extraction — kept low to avoid rate-limit bursts. */
|
|
1403
|
+
const LLM_CONCURRENCY = 3;
|
|
1404
|
+
const SUPPORTED_EXTENSIONS = /* @__PURE__ */ new Set([
|
|
1405
|
+
".md",
|
|
1406
|
+
".markdown",
|
|
1407
|
+
".txt"
|
|
1408
|
+
]);
|
|
1409
|
+
/** Read a file and return its content. */
|
|
1410
|
+
async function readFileContent(filePath) {
|
|
1411
|
+
return readFile(filePath, "utf-8");
|
|
1412
|
+
}
|
|
1413
|
+
function getFileExtension(filePath) {
|
|
1414
|
+
const lastDot = filePath.lastIndexOf(".");
|
|
1415
|
+
return lastDot > 0 ? filePath.slice(lastDot).toLowerCase() : "";
|
|
1416
|
+
}
|
|
1417
|
+
function chunkContent(content, ext) {
|
|
1418
|
+
return ext === ".txt" ? chunkText(content) : chunkMarkdown(content);
|
|
1419
|
+
}
|
|
1420
|
+
function isSupportedFile(filePath) {
|
|
1421
|
+
return SUPPORTED_EXTENSIONS.has(getFileExtension(filePath));
|
|
1422
|
+
}
|
|
1423
|
+
/** Map async iterator values with a concurrency limit. */
|
|
1424
|
+
async function mapWithConcurrency(items, limit, fn, onProgress) {
|
|
1425
|
+
const results = Array.from({ length: items.length });
|
|
1426
|
+
let cursor = 0;
|
|
1427
|
+
let completed = 0;
|
|
1428
|
+
const total = items.length;
|
|
1429
|
+
async function worker() {
|
|
1430
|
+
while (cursor < items.length) {
|
|
1431
|
+
const idx = cursor;
|
|
1432
|
+
cursor += 1;
|
|
1433
|
+
const item = items[idx];
|
|
1434
|
+
const result = await fn(item, idx);
|
|
1435
|
+
results[idx] = result;
|
|
1436
|
+
completed += 1;
|
|
1437
|
+
onProgress?.(completed, total, result);
|
|
1438
|
+
}
|
|
1439
|
+
}
|
|
1440
|
+
const workers = Array.from({ length: Math.min(limit, items.length) }, () => worker());
|
|
1441
|
+
await Promise.all(workers);
|
|
1442
|
+
return results;
|
|
1443
|
+
}
|
|
1444
|
+
/**
|
|
1445
|
+
* Ingest a markdown file into the knowledge base.
|
|
1446
|
+
*
|
|
1447
|
+
* Steps:
|
|
1448
|
+
* 1. Dedupe by file_path — if a source already exists for this file, error.
|
|
1449
|
+
* 2. Read file → chunkMarkdown (heading_strict).
|
|
1450
|
+
* 3. Embed chunks ("{heading}\n{content}") → store knowledge_chunks.
|
|
1451
|
+
* 4. For each chunk (concurrency 3): LLM extract 1 event + N entities.
|
|
1452
|
+
* 5. Embed event title + content ("{title}\n\n{content}") → store knowledge_events.
|
|
1453
|
+
* 6. Upsert entities (dedupe by source_id+type+normalizedName) + embed entity name → store.
|
|
1454
|
+
* 7. For each (event, entity) pair: embed relation (entity.description || "{eventTitle} {entityName}") → store knowledge_event_entities.
|
|
1455
|
+
* 8. Update document status = 'completed'.
|
|
1456
|
+
*
|
|
1457
|
+
* All writes run inside a single transaction — a mid-ingest failure rolls back
|
|
1458
|
+
* every partial row (source/document/chunks/events/entities/edges).
|
|
1459
|
+
*
|
|
1460
|
+
* Reports progress via callback.
|
|
1461
|
+
*/
|
|
1462
|
+
async function ingestFile(store, llm, filePath, onProgress) {
|
|
1463
|
+
const engine = store.getEmbeddingEngine();
|
|
1464
|
+
let skipEmbed = engine === void 0;
|
|
1465
|
+
if (engine !== void 0) {
|
|
1466
|
+
onProgress?.({
|
|
1467
|
+
stage: "embedding-check",
|
|
1468
|
+
message: "检查向量模型..."
|
|
1469
|
+
});
|
|
1470
|
+
try {
|
|
1471
|
+
skipEmbed = !await Promise.race([engine.ensureReady(), new Promise((resolve) => setTimeout(() => resolve(false), 15e3))]);
|
|
1472
|
+
if (skipEmbed) onProgress?.({
|
|
1473
|
+
stage: "embedding-check",
|
|
1474
|
+
message: "向量模型未就绪,本次摄入跳过向量嵌入"
|
|
1475
|
+
});
|
|
1476
|
+
} catch {
|
|
1477
|
+
skipEmbed = true;
|
|
1478
|
+
onProgress?.({
|
|
1479
|
+
stage: "embedding-check",
|
|
1480
|
+
message: "向量模型未就绪,本次摄入跳过向量嵌入"
|
|
1481
|
+
});
|
|
1482
|
+
}
|
|
1483
|
+
}
|
|
1484
|
+
const existing = await store.findSourceByFilePath(filePath);
|
|
1485
|
+
if (existing !== void 0) {
|
|
1486
|
+
onProgress?.({
|
|
1487
|
+
stage: "error",
|
|
1488
|
+
message: `文件已摄入:${existing.name}(source id=${existing.id})`
|
|
1489
|
+
});
|
|
1490
|
+
throw new Error(`文件已摄入过: ${filePath} (source ${existing.id})`);
|
|
1491
|
+
}
|
|
1492
|
+
onProgress?.({
|
|
1493
|
+
stage: "chunking",
|
|
1494
|
+
message: `读取并切分文件: ${basename(filePath)}`
|
|
1495
|
+
});
|
|
1496
|
+
const content = await readFileContent(filePath);
|
|
1497
|
+
const sections = chunkContent(content, getFileExtension(filePath));
|
|
1498
|
+
if (sections.length === 0) {
|
|
1499
|
+
onProgress?.({
|
|
1500
|
+
stage: "error",
|
|
1501
|
+
message: "文件无有效内容"
|
|
1502
|
+
});
|
|
1503
|
+
throw new Error("文件无有效内容可摄入");
|
|
1504
|
+
}
|
|
1505
|
+
store.beginTransaction();
|
|
1506
|
+
try {
|
|
1507
|
+
const fileName = basename(filePath);
|
|
1508
|
+
const source = await store.createSource({
|
|
1509
|
+
name: fileName,
|
|
1510
|
+
filePath,
|
|
1511
|
+
description: null
|
|
1512
|
+
});
|
|
1513
|
+
const document = await store.createDocument({
|
|
1514
|
+
sourceId: source.id,
|
|
1515
|
+
title: fileName,
|
|
1516
|
+
content
|
|
1517
|
+
});
|
|
1518
|
+
onProgress?.({
|
|
1519
|
+
stage: "embedding-chunks",
|
|
1520
|
+
chunkIndex: 0,
|
|
1521
|
+
totalChunks: sections.length,
|
|
1522
|
+
message: `嵌入 chunks: 0/${sections.length}`
|
|
1523
|
+
});
|
|
1524
|
+
const chunkEmbeddings = await embedOrNulls(engine, sections.map((s) => s.heading !== null ? `${s.heading}\n${s.content}` : s.content), skipEmbed);
|
|
1525
|
+
const chunks = [];
|
|
1526
|
+
for (let i = 0; i < sections.length; i++) {
|
|
1527
|
+
const section = sections[i];
|
|
1528
|
+
const embedding = chunkEmbeddings[i] ?? null;
|
|
1529
|
+
const chunk = await store.insertChunk({
|
|
1530
|
+
sourceId: source.id,
|
|
1531
|
+
documentId: document.id,
|
|
1532
|
+
rank: section.rank,
|
|
1533
|
+
heading: section.heading,
|
|
1534
|
+
content: section.content,
|
|
1535
|
+
rawContent: section.rawContent,
|
|
1536
|
+
embedding
|
|
1537
|
+
});
|
|
1538
|
+
chunks.push(chunk);
|
|
1539
|
+
onProgress?.({
|
|
1540
|
+
stage: "embedding-chunks",
|
|
1541
|
+
chunkIndex: i + 1,
|
|
1542
|
+
totalChunks: sections.length,
|
|
1543
|
+
message: `嵌入 chunks: ${i + 1}/${sections.length}`
|
|
1544
|
+
});
|
|
1545
|
+
}
|
|
1546
|
+
onProgress?.({
|
|
1547
|
+
stage: "extracting",
|
|
1548
|
+
chunkIndex: 0,
|
|
1549
|
+
totalChunks: sections.length,
|
|
1550
|
+
message: `抽取事件: 0/${sections.length}`
|
|
1551
|
+
});
|
|
1552
|
+
const extractedEvents = await mapWithConcurrency(sections, LLM_CONCURRENCY, (section) => extractEventFromChunk(llm, section), (completed, total) => {
|
|
1553
|
+
onProgress?.({
|
|
1554
|
+
stage: "extracting",
|
|
1555
|
+
chunkIndex: completed,
|
|
1556
|
+
totalChunks: total,
|
|
1557
|
+
message: `抽取事件: ${completed}/${total}`
|
|
1558
|
+
});
|
|
1559
|
+
});
|
|
1560
|
+
onProgress?.({
|
|
1561
|
+
stage: "embedding-events",
|
|
1562
|
+
chunkIndex: 0,
|
|
1563
|
+
totalChunks: sections.length,
|
|
1564
|
+
message: `嵌入 events: 0/${sections.length}`
|
|
1565
|
+
});
|
|
1566
|
+
const titleTexts = extractedEvents.map((e) => e.title);
|
|
1567
|
+
const contentTexts = extractedEvents.map((e) => `${e.title}\n\n${e.content}`);
|
|
1568
|
+
const [titleEmbeddings, contentEmbeddings] = await Promise.all([embedOrNulls(engine, titleTexts, skipEmbed), embedOrNulls(engine, contentTexts, skipEmbed)]);
|
|
1569
|
+
const events = [];
|
|
1570
|
+
for (let i = 0; i < extractedEvents.length; i++) {
|
|
1571
|
+
const extracted = extractedEvents[i];
|
|
1572
|
+
const chunk = chunks[i];
|
|
1573
|
+
const event = await store.insertEvent({
|
|
1574
|
+
sourceId: source.id,
|
|
1575
|
+
documentId: document.id,
|
|
1576
|
+
chunkId: chunk.id,
|
|
1577
|
+
rank: i,
|
|
1578
|
+
title: extracted.title,
|
|
1579
|
+
summary: extracted.summary.length > 0 ? extracted.summary : null,
|
|
1580
|
+
content: extracted.content,
|
|
1581
|
+
category: extracted.category.length > 0 ? extracted.category : null,
|
|
1582
|
+
keywords: extracted.keywords,
|
|
1583
|
+
titleEmbedding: titleEmbeddings[i] ?? null,
|
|
1584
|
+
contentEmbedding: contentEmbeddings[i] ?? null
|
|
1585
|
+
});
|
|
1586
|
+
events.push(event);
|
|
1587
|
+
onProgress?.({
|
|
1588
|
+
stage: "embedding-events",
|
|
1589
|
+
chunkIndex: i + 1,
|
|
1590
|
+
totalChunks: sections.length,
|
|
1591
|
+
message: `嵌入 events: ${i + 1}/${sections.length}`
|
|
1592
|
+
});
|
|
1593
|
+
}
|
|
1594
|
+
onProgress?.({
|
|
1595
|
+
stage: "embedding-entities",
|
|
1596
|
+
message: "嵌入 entities..."
|
|
1597
|
+
});
|
|
1598
|
+
const entityMap = /* @__PURE__ */ new Map();
|
|
1599
|
+
for (const extracted of extractedEvents) for (const entity of extracted.entities) {
|
|
1600
|
+
const key = `${entity.type}|${entity.name.toLowerCase()}`;
|
|
1601
|
+
if (!entityMap.has(key)) entityMap.set(key, entity);
|
|
1602
|
+
}
|
|
1603
|
+
const uniqueEntities = Array.from(entityMap.values());
|
|
1604
|
+
const entityEmbeddings = uniqueEntities.length > 0 ? await embedOrNulls(engine, uniqueEntities.map((e) => e.name), skipEmbed) : [];
|
|
1605
|
+
const entityIdByEntityKey = /* @__PURE__ */ new Map();
|
|
1606
|
+
for (let i = 0; i < uniqueEntities.length; i++) {
|
|
1607
|
+
const entity = uniqueEntities[i];
|
|
1608
|
+
const embedding = entityEmbeddings[i] ?? null;
|
|
1609
|
+
const stored = await store.upsertEntity({
|
|
1610
|
+
sourceId: source.id,
|
|
1611
|
+
type: entity.type,
|
|
1612
|
+
name: entity.name,
|
|
1613
|
+
description: entity.description.length > 0 ? entity.description : null,
|
|
1614
|
+
embedding
|
|
1615
|
+
});
|
|
1616
|
+
entityIdByEntityKey.set(`${entity.type}|${entity.name.toLowerCase()}`, stored.id);
|
|
1617
|
+
}
|
|
1618
|
+
onProgress?.({
|
|
1619
|
+
stage: "embedding-relations",
|
|
1620
|
+
message: "嵌入关系..."
|
|
1621
|
+
});
|
|
1622
|
+
const relationPairs = [];
|
|
1623
|
+
for (let i = 0; i < extractedEvents.length; i++) {
|
|
1624
|
+
const extracted = extractedEvents[i];
|
|
1625
|
+
for (const entity of extracted.entities) relationPairs.push({
|
|
1626
|
+
eventIndex: i,
|
|
1627
|
+
entity
|
|
1628
|
+
});
|
|
1629
|
+
}
|
|
1630
|
+
const relationTexts = relationPairs.map(({ eventIndex, entity }) => {
|
|
1631
|
+
const event = events[eventIndex];
|
|
1632
|
+
return entity.description.length > 0 ? entity.description : `${event.title} ${entity.name}`;
|
|
1633
|
+
});
|
|
1634
|
+
const relationEmbeddings = relationPairs.length > 0 ? await embedOrNulls(engine, relationTexts, skipEmbed) : [];
|
|
1635
|
+
for (let i = 0; i < relationPairs.length; i++) {
|
|
1636
|
+
const pair = relationPairs[i];
|
|
1637
|
+
const event = events[pair.eventIndex];
|
|
1638
|
+
const entityKey = `${pair.entity.type}|${pair.entity.name.toLowerCase()}`;
|
|
1639
|
+
const entityId = entityIdByEntityKey.get(entityKey);
|
|
1640
|
+
if (entityId === void 0) continue;
|
|
1641
|
+
const embedding = relationEmbeddings?.[i] ?? null;
|
|
1642
|
+
await store.insertEventEntity({
|
|
1643
|
+
eventId: event.id,
|
|
1644
|
+
entityId,
|
|
1645
|
+
weight: 1,
|
|
1646
|
+
description: pair.entity.description.length > 0 ? pair.entity.description : null,
|
|
1647
|
+
embedding
|
|
1648
|
+
});
|
|
1649
|
+
}
|
|
1650
|
+
await store.updateDocumentStatus(document.id, "completed", chunks.length);
|
|
1651
|
+
store.commitTransaction();
|
|
1652
|
+
onProgress?.({
|
|
1653
|
+
stage: "completed",
|
|
1654
|
+
message: `摄入完成:${chunks.length} chunks, ${events.length} events, ${uniqueEntities.length} entities`
|
|
1655
|
+
});
|
|
1656
|
+
return {
|
|
1657
|
+
documentId: document.id,
|
|
1658
|
+
chunkCount: chunks.length,
|
|
1659
|
+
eventCount: events.length,
|
|
1660
|
+
entityCount: uniqueEntities.length
|
|
1661
|
+
};
|
|
1662
|
+
} catch (error) {
|
|
1663
|
+
store.rollbackTransaction();
|
|
1664
|
+
throw error;
|
|
1665
|
+
}
|
|
1666
|
+
}
|
|
1667
|
+
async function collectSupportedFiles(dirPath) {
|
|
1668
|
+
const files = [];
|
|
1669
|
+
const entries = await readdir(dirPath, { withFileTypes: true });
|
|
1670
|
+
for (const entry of entries) {
|
|
1671
|
+
const fullPath = join(dirPath, entry.name);
|
|
1672
|
+
if (entry.isDirectory()) files.push(...await collectSupportedFiles(fullPath));
|
|
1673
|
+
else if (entry.isFile() && isSupportedFile(fullPath)) files.push(fullPath);
|
|
1674
|
+
}
|
|
1675
|
+
return files.sort();
|
|
1676
|
+
}
|
|
1677
|
+
async function ingestDirectory(store, llm, dirPath, onProgress) {
|
|
1678
|
+
const files = await collectSupportedFiles(dirPath);
|
|
1679
|
+
if (files.length === 0) throw new Error(`目录下没有支持的文件 (.md, .markdown, .txt): ${dirPath}`);
|
|
1680
|
+
const errors = [];
|
|
1681
|
+
let succeeded = 0;
|
|
1682
|
+
let totalChunks = 0;
|
|
1683
|
+
let totalEvents = 0;
|
|
1684
|
+
let totalEntities = 0;
|
|
1685
|
+
for (let i = 0; i < files.length; i++) {
|
|
1686
|
+
const filePath = files[i];
|
|
1687
|
+
const fileProgress = (progress) => {
|
|
1688
|
+
const prefix = `批量摄入 ${i + 1}/${files.length}: ${basename(filePath)}`;
|
|
1689
|
+
onProgress?.({
|
|
1690
|
+
...progress,
|
|
1691
|
+
message: `${prefix} · ${progress.message}`
|
|
1692
|
+
});
|
|
1693
|
+
};
|
|
1694
|
+
try {
|
|
1695
|
+
const result = await ingestFile(store, llm, filePath, fileProgress);
|
|
1696
|
+
succeeded += 1;
|
|
1697
|
+
totalChunks += result.chunkCount;
|
|
1698
|
+
totalEvents += result.eventCount;
|
|
1699
|
+
totalEntities += result.entityCount;
|
|
1700
|
+
} catch (error) {
|
|
1701
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
1702
|
+
errors.push({
|
|
1703
|
+
filePath,
|
|
1704
|
+
message
|
|
1705
|
+
});
|
|
1706
|
+
onProgress?.({
|
|
1707
|
+
stage: "error",
|
|
1708
|
+
message: `批量摄入 ${i + 1}/${files.length}: ${basename(filePath)} · 失败: ${message}`
|
|
1709
|
+
});
|
|
1710
|
+
}
|
|
1711
|
+
}
|
|
1712
|
+
onProgress?.({
|
|
1713
|
+
stage: "completed",
|
|
1714
|
+
message: `批量摄入完成:${succeeded}/${files.length} 个文件成功,${totalChunks} chunks, ${totalEvents} events, ${totalEntities} entities`
|
|
1715
|
+
});
|
|
1716
|
+
return {
|
|
1717
|
+
succeeded,
|
|
1718
|
+
failed: errors.length,
|
|
1719
|
+
totalChunks,
|
|
1720
|
+
totalEvents,
|
|
1721
|
+
totalEntities,
|
|
1722
|
+
errors
|
|
1723
|
+
};
|
|
1724
|
+
}
|
|
1725
|
+
const DEFAULT_TOP_K = 5;
|
|
1726
|
+
const MAX_TOP_K = 20;
|
|
1727
|
+
const SEED_EVENT_LIMIT = 30;
|
|
1728
|
+
const EXPANDED_EVENT_LIMIT = 100;
|
|
1729
|
+
const COARSE_RANK_LIMIT = 50;
|
|
1730
|
+
const ENTITY_VECTOR_THRESHOLD = .7;
|
|
1731
|
+
const TITLE_VECTOR_THRESHOLD = .4;
|
|
1732
|
+
async function timed(onStep, step, detail, fn, payload) {
|
|
1733
|
+
const start = performance.now();
|
|
1734
|
+
try {
|
|
1735
|
+
const result = await fn();
|
|
1736
|
+
const traceStep = {
|
|
1737
|
+
step,
|
|
1738
|
+
detail,
|
|
1739
|
+
durationMs: Math.round((performance.now() - start) * 100) / 100,
|
|
1740
|
+
...payload !== void 0 ? { payload: payload(result) } : {}
|
|
1741
|
+
};
|
|
1742
|
+
onStep?.(traceStep);
|
|
1743
|
+
return result;
|
|
1744
|
+
} catch (error) {
|
|
1745
|
+
const durationMs = Math.round((performance.now() - start) * 100) / 100;
|
|
1746
|
+
onStep?.({
|
|
1747
|
+
step,
|
|
1748
|
+
detail: `${detail} 失败:${error instanceof Error ? error.message : String(error)}`,
|
|
1749
|
+
durationMs
|
|
1750
|
+
});
|
|
1751
|
+
throw error;
|
|
1752
|
+
}
|
|
1753
|
+
}
|
|
1754
|
+
/**
|
|
1755
|
+
* Multi-hop retrieval:
|
|
1756
|
+
* 1. Vectorize query.
|
|
1757
|
+
* 2. Entity recall — LLM extracts entities from query → match by name + vector.
|
|
1758
|
+
* 3. Seed events — events linked to recalled entities + events by title vector similarity.
|
|
1759
|
+
* 4. BFS expand (1 hop) — from seed events, walk to entities, then to other events.
|
|
1760
|
+
* 5. Coarse rank — score graph-reachable events by content embedding similarity.
|
|
1761
|
+
* Graph associativity is the gate; content vector orders within it.
|
|
1762
|
+
* 6. LLM rerank — pick top-K most relevant.
|
|
1763
|
+
* 7. Return corresponding chunks (deduped by chunk_id) with scores and provenance.
|
|
1764
|
+
* If rerank yields fewer than topK chunks, backfill by direct chunk vector search.
|
|
1765
|
+
*
|
|
1766
|
+
* Optional `onStep` callback receives a trace step at each phase — used by the
|
|
1767
|
+
* KnowledgeLookup tool to expose retrieval diagnostics to the agent.
|
|
1768
|
+
*/
|
|
1769
|
+
async function multiSearch(store, llm, query, options = {}) {
|
|
1770
|
+
return (await multiSearchWithTrace(store, llm, query, options)).results;
|
|
1771
|
+
}
|
|
1772
|
+
/** Like multiSearch but also returns the retrieval trace for diagnostics. */
|
|
1773
|
+
async function multiSearchWithTrace(store, llm, query, options = {}) {
|
|
1774
|
+
const topK = Math.min(options.topK ?? DEFAULT_TOP_K, MAX_TOP_K);
|
|
1775
|
+
const steps = [];
|
|
1776
|
+
const onStep = (step) => steps.push(step);
|
|
1777
|
+
const rerankedEventTitles = [];
|
|
1778
|
+
let fallbackReason = null;
|
|
1779
|
+
const engine = store.getEmbeddingEngine();
|
|
1780
|
+
if (engine === void 0 || !engine.available) {
|
|
1781
|
+
fallbackReason = "embedding engine unavailable; used FTS5 keyword fallback";
|
|
1782
|
+
return {
|
|
1783
|
+
results: await ftsFallback(store, query, topK),
|
|
1784
|
+
trace: {
|
|
1785
|
+
steps,
|
|
1786
|
+
rerankedEventTitles,
|
|
1787
|
+
fallbackReason
|
|
1788
|
+
}
|
|
1789
|
+
};
|
|
1790
|
+
}
|
|
1791
|
+
const queryEmbeddings = await timed(onStep, "queryEmbedding", "把用户问题转成向量,用于召回相关事件和切片。", async () => engine.embedBatch([query]));
|
|
1792
|
+
if (queryEmbeddings === null || queryEmbeddings.length === 0) {
|
|
1793
|
+
fallbackReason = "query embedding returned null; used FTS5 fallback";
|
|
1794
|
+
return {
|
|
1795
|
+
results: await ftsFallback(store, query, topK),
|
|
1796
|
+
trace: {
|
|
1797
|
+
steps,
|
|
1798
|
+
rerankedEventTitles,
|
|
1799
|
+
fallbackReason
|
|
1800
|
+
}
|
|
1801
|
+
};
|
|
1802
|
+
}
|
|
1803
|
+
const queryVec = queryEmbeddings[0];
|
|
1804
|
+
const recalledEntities = await timed(onStep, "entityRecall", "LLM 抽 query 实体 + 名字精确匹配 + 向量召回。", () => recallEntities(store, llm, query, queryVec), (entities) => ({ count: entities.length }));
|
|
1805
|
+
const seedEventIds = /* @__PURE__ */ new Set();
|
|
1806
|
+
for (const entity of recalledEntities) {
|
|
1807
|
+
const events = await store.findEventsByEntity(entity.id);
|
|
1808
|
+
for (const event of events) seedEventIds.add(event.id);
|
|
1809
|
+
}
|
|
1810
|
+
const titleMatches = await timed(onStep, "seedEventsByTitle", "按查询向量在事件标题向量上召回 seed events。", () => store.findEventsByTitleVector(queryVec, {
|
|
1811
|
+
limit: SEED_EVENT_LIMIT,
|
|
1812
|
+
threshold: TITLE_VECTOR_THRESHOLD
|
|
1813
|
+
}), (matches) => ({ count: matches.length }));
|
|
1814
|
+
for (const { event } of titleMatches) seedEventIds.add(event.id);
|
|
1815
|
+
if (seedEventIds.size === 0) {
|
|
1816
|
+
fallbackReason = "no seed events; used direct chunk vector search";
|
|
1817
|
+
return {
|
|
1818
|
+
results: await chunkVectorFallback(store, queryVec, topK),
|
|
1819
|
+
trace: {
|
|
1820
|
+
steps,
|
|
1821
|
+
rerankedEventTitles,
|
|
1822
|
+
fallbackReason
|
|
1823
|
+
}
|
|
1824
|
+
};
|
|
1825
|
+
}
|
|
1826
|
+
const expandedEventIds = await timed(onStep, "bfsExpand", "从 seed events 沿实体关系 1 跳扩展候选事件。", async () => {
|
|
1827
|
+
const ids = new Set(seedEventIds);
|
|
1828
|
+
let capped = false;
|
|
1829
|
+
for (const eventId of seedEventIds) {
|
|
1830
|
+
const entities = await store.findEntitiesByEvent(eventId);
|
|
1831
|
+
for (const entity of entities) {
|
|
1832
|
+
const neighborEvents = await store.findEventsByEntity(entity.id);
|
|
1833
|
+
for (const neighbor of neighborEvents) {
|
|
1834
|
+
ids.add(neighbor.id);
|
|
1835
|
+
if (ids.size >= EXPANDED_EVENT_LIMIT) {
|
|
1836
|
+
capped = true;
|
|
1837
|
+
break;
|
|
1838
|
+
}
|
|
1839
|
+
}
|
|
1840
|
+
if (ids.size >= EXPANDED_EVENT_LIMIT) break;
|
|
1841
|
+
}
|
|
1842
|
+
if (ids.size >= EXPANDED_EVENT_LIMIT) break;
|
|
1843
|
+
}
|
|
1844
|
+
return {
|
|
1845
|
+
ids,
|
|
1846
|
+
capped
|
|
1847
|
+
};
|
|
1848
|
+
}, (result) => ({
|
|
1849
|
+
expandedCount: result.ids.size,
|
|
1850
|
+
capped: result.capped
|
|
1851
|
+
}));
|
|
1852
|
+
const { candidates, topCandidates } = await timed(onStep, "coarseRank", "按 content 向量相似度对候选事件排序,取前 COARSE_RANK_LIMIT 个。", async () => {
|
|
1853
|
+
const cands = [];
|
|
1854
|
+
for (const eventId of expandedEventIds.ids) {
|
|
1855
|
+
const event = await store.getEvent(eventId);
|
|
1856
|
+
if (event === void 0) continue;
|
|
1857
|
+
const vec = event.contentEmbedding;
|
|
1858
|
+
const score = vec === null ? 0 : engine.cosineSimilarity(queryVec, vec);
|
|
1859
|
+
cands.push({
|
|
1860
|
+
id: event.id,
|
|
1861
|
+
title: event.title,
|
|
1862
|
+
summary: event.summary ?? "",
|
|
1863
|
+
score,
|
|
1864
|
+
chunkId: event.chunkId
|
|
1865
|
+
});
|
|
1866
|
+
}
|
|
1867
|
+
cands.sort((a, b) => b.score - a.score);
|
|
1868
|
+
return {
|
|
1869
|
+
candidates: cands,
|
|
1870
|
+
topCandidates: cands.slice(0, COARSE_RANK_LIMIT)
|
|
1871
|
+
};
|
|
1872
|
+
}, (result) => ({
|
|
1873
|
+
totalCandidates: result.candidates.length,
|
|
1874
|
+
kept: result.topCandidates.length,
|
|
1875
|
+
topScore: result.topCandidates[0]?.score ?? 0
|
|
1876
|
+
}));
|
|
1877
|
+
if (candidates.length === 0) {
|
|
1878
|
+
fallbackReason = "no graph-reachable events with content; used chunk vector fallback";
|
|
1879
|
+
return {
|
|
1880
|
+
results: await chunkVectorFallback(store, queryVec, topK),
|
|
1881
|
+
trace: {
|
|
1882
|
+
steps,
|
|
1883
|
+
rerankedEventTitles,
|
|
1884
|
+
fallbackReason
|
|
1885
|
+
}
|
|
1886
|
+
};
|
|
1887
|
+
}
|
|
1888
|
+
let rankedIds;
|
|
1889
|
+
if (options.skipRerank === true) {
|
|
1890
|
+
rankedIds = topCandidates.map((c) => c.id);
|
|
1891
|
+
onStep({
|
|
1892
|
+
step: "rerank",
|
|
1893
|
+
detail: "跳过 LLM rerank(skipRerank=true),直接用 coarse rank 顺序。",
|
|
1894
|
+
durationMs: 0,
|
|
1895
|
+
payload: { count: rankedIds.length }
|
|
1896
|
+
});
|
|
1897
|
+
} else if (topCandidates.length <= topK) {
|
|
1898
|
+
rankedIds = topCandidates.map((c) => c.id);
|
|
1899
|
+
onStep({
|
|
1900
|
+
step: "rerank",
|
|
1901
|
+
detail: `候选数 ${topCandidates.length} ≤ topK ${topK},无需 LLM rerank。`,
|
|
1902
|
+
durationMs: 0,
|
|
1903
|
+
payload: { count: rankedIds.length }
|
|
1904
|
+
});
|
|
1905
|
+
} else rankedIds = await timed(onStep, "rerank", `LLM 从 ${topCandidates.length} 个候选中选最相关的 ${topK} 个。`, () => rerankEventsWithLlm(llm, query, topCandidates.map((c) => ({
|
|
1906
|
+
id: c.id,
|
|
1907
|
+
title: c.title,
|
|
1908
|
+
summary: c.summary
|
|
1909
|
+
})), topK), (ids) => ({ count: ids.length }));
|
|
1910
|
+
for (const id of rankedIds) {
|
|
1911
|
+
const c = topCandidates.find((x) => x.id === id);
|
|
1912
|
+
if (c !== void 0) rerankedEventTitles.push(c.title);
|
|
1913
|
+
}
|
|
1914
|
+
const seen = /* @__PURE__ */ new Set();
|
|
1915
|
+
const results = [];
|
|
1916
|
+
for (const eventId of rankedIds) {
|
|
1917
|
+
const candidate = topCandidates.find((c) => c.id === eventId);
|
|
1918
|
+
if (candidate === void 0) continue;
|
|
1919
|
+
if (seen.has(candidate.chunkId)) continue;
|
|
1920
|
+
seen.add(candidate.chunkId);
|
|
1921
|
+
const result = await store.buildSearchResult(candidate.chunkId, candidate.score, eventId);
|
|
1922
|
+
if (result !== void 0) results.push(result);
|
|
1923
|
+
if (results.length >= topK) break;
|
|
1924
|
+
}
|
|
1925
|
+
if (results.length < topK) {
|
|
1926
|
+
let supplemented = 0;
|
|
1927
|
+
for (const candidate of topCandidates) {
|
|
1928
|
+
if (results.length >= topK) break;
|
|
1929
|
+
if (rankedIds.includes(candidate.id)) continue;
|
|
1930
|
+
if (seen.has(candidate.chunkId)) continue;
|
|
1931
|
+
seen.add(candidate.chunkId);
|
|
1932
|
+
const result = await store.buildSearchResult(candidate.chunkId, candidate.score, candidate.id);
|
|
1933
|
+
if (result !== void 0) {
|
|
1934
|
+
results.push(result);
|
|
1935
|
+
supplemented += 1;
|
|
1936
|
+
}
|
|
1937
|
+
}
|
|
1938
|
+
if (supplemented > 0) onStep({
|
|
1939
|
+
step: "supplementFromCoarse",
|
|
1940
|
+
detail: `rerank 结果不足,从 coarse rank 残余补 ${supplemented} 个。`,
|
|
1941
|
+
durationMs: 0,
|
|
1942
|
+
payload: { supplemented }
|
|
1943
|
+
});
|
|
1944
|
+
}
|
|
1945
|
+
if (results.length < topK) {
|
|
1946
|
+
const remaining = topK - results.length;
|
|
1947
|
+
const backfill = await timed(onStep, "backfill", `rerank 结果不足,回退到 chunk 向量搜索补 ${remaining} 个。`, () => store.searchChunksByVector(queryVec, { limit: remaining * 2 }), (matches) => ({ backfillCandidates: matches.length }));
|
|
1948
|
+
for (const { chunk, score } of backfill) {
|
|
1949
|
+
if (results.length >= topK) break;
|
|
1950
|
+
if (seen.has(chunk.id)) continue;
|
|
1951
|
+
seen.add(chunk.id);
|
|
1952
|
+
const result = await store.buildSearchResult(chunk.id, score);
|
|
1953
|
+
if (result !== void 0) results.push(result);
|
|
1954
|
+
}
|
|
1955
|
+
}
|
|
1956
|
+
return {
|
|
1957
|
+
results,
|
|
1958
|
+
trace: {
|
|
1959
|
+
steps,
|
|
1960
|
+
rerankedEventTitles,
|
|
1961
|
+
fallbackReason
|
|
1962
|
+
}
|
|
1963
|
+
};
|
|
1964
|
+
}
|
|
1965
|
+
/** Recall entities by name (LLM-extracted) and by vector similarity. */
|
|
1966
|
+
async function recallEntities(store, llm, query, queryVec) {
|
|
1967
|
+
const out = /* @__PURE__ */ new Map();
|
|
1968
|
+
const queryEntities = await extractQueryEntities(llm, query);
|
|
1969
|
+
for (const { name } of queryEntities) {
|
|
1970
|
+
const matches = await store.findEntitiesByName(name);
|
|
1971
|
+
for (const entity of matches) out.set(entity.id, { id: entity.id });
|
|
1972
|
+
}
|
|
1973
|
+
const vectorMatches = await store.findEntitiesByVector(queryVec, {
|
|
1974
|
+
limit: 20,
|
|
1975
|
+
threshold: ENTITY_VECTOR_THRESHOLD
|
|
1976
|
+
});
|
|
1977
|
+
for (const { entity } of vectorMatches) out.set(entity.id, { id: entity.id });
|
|
1978
|
+
return Array.from(out.values());
|
|
1979
|
+
}
|
|
1980
|
+
/** FTS5 keyword fallback when embeddings are unavailable. */
|
|
1981
|
+
async function ftsFallback(store, query, topK) {
|
|
1982
|
+
const chunks = await store.ftsSearchChunks(query, topK * 2);
|
|
1983
|
+
const results = [];
|
|
1984
|
+
for (const chunk of chunks) {
|
|
1985
|
+
const result = await store.buildSearchResult(chunk.id, 0);
|
|
1986
|
+
if (result !== void 0) results.push(result);
|
|
1987
|
+
if (results.length >= topK) break;
|
|
1988
|
+
}
|
|
1989
|
+
return results;
|
|
1990
|
+
}
|
|
1991
|
+
/** Direct chunk vector search fallback. */
|
|
1992
|
+
async function chunkVectorFallback(store, queryVec, topK) {
|
|
1993
|
+
const matches = await store.searchChunksByVector(queryVec, { limit: topK * 2 });
|
|
1994
|
+
const results = [];
|
|
1995
|
+
for (const { chunk, score } of matches) {
|
|
1996
|
+
const result = await store.buildSearchResult(chunk.id, score);
|
|
1997
|
+
if (result !== void 0) results.push(result);
|
|
1998
|
+
if (results.length >= topK) break;
|
|
1999
|
+
}
|
|
2000
|
+
return results;
|
|
2001
|
+
}
|
|
2002
|
+
//#endregion
|
|
2003
|
+
export { join as C, resolve as D, relative as E, isAbsolute as S, parse as T, rerankEventsWithLlm as _, chunkMarkdown as a, basename as b, extractEventFromChunk as c, ingestDirectory as d, ingestFile as f, parseExtractionResponse as g, multiSearchWithTrace as h, buildExtractionUserPrompt as i, extractJsonFromText as l, multiSearch as m, EXTRACTION_SYSTEM_PROMPT as n, chunkText as o, isSupportedFile as p, KnowledgeStore as r, estimateTokens as s, ENTITY_TYPES as t, extractQueryEntities as u, splitLargeSection as v, normalize as w, dirname as x, stripMarkdown as y };
|