@diegoaltoworks/chatter 0.63.0 → 0.64.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/core/knowledgeHealth.d.ts +70 -0
- package/dist/core/knowledgeHealth.d.ts.map +1 -0
- package/dist/index.js +137 -2
- package/dist/index.mjs +137 -2
- package/dist/mcp-server.d.ts +3 -1
- package/dist/mcp-server.d.ts.map +1 -1
- package/dist/mcp-server.js +127 -2
- package/dist/mcp-server.mjs +127 -2
- package/dist/server.d.ts.map +1 -1
- package/dist/server.js +123 -0
- package/dist/server.mjs +123 -0
- package/dist/types.d.ts +10 -0
- package/dist/types.d.ts.map +1 -1
- package/package.json +1 -1
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Post-boot and periodic health checks for the default `VectorStore`
|
|
3
|
+
* knowledge base, so a collapse is visible in logs the moment it happens
|
|
4
|
+
* instead of waiting for a user to notice broken answers - the gap that let
|
|
5
|
+
* two knowledge-base wipes go unnoticed until someone spotted broken
|
|
6
|
+
* answers.
|
|
7
|
+
*/
|
|
8
|
+
import type { Client as LibsqlClient } from "@libsql/client";
|
|
9
|
+
import { type BuildLock } from "./buildLock";
|
|
10
|
+
import { type Logger } from "./logger";
|
|
11
|
+
export interface KnowledgeHealth {
|
|
12
|
+
chunksByBucket: Record<string, number>;
|
|
13
|
+
totalChunks: number;
|
|
14
|
+
totalEmbeddings: number;
|
|
15
|
+
/** Embeddings rows with no matching chunks row. */
|
|
16
|
+
orphanedEmbeddings: number;
|
|
17
|
+
/** Chunks rows with no matching embeddings row (e.g. a build interrupted mid-embed). */
|
|
18
|
+
missingEmbeddings: number;
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* Read-only snapshot of chunks/embeddings consistency, logged at `info` -
|
|
22
|
+
* escalating to `warn` when `totalChunks === 0` (the exact signature of both
|
|
23
|
+
* knowledge-base wipes) or `orphanedEmbeddings > 0`. The query shape mirrors
|
|
24
|
+
* the diagnostic used live to investigate the second incident.
|
|
25
|
+
*/
|
|
26
|
+
export declare function checkKnowledgeHealth(db: LibsqlClient, logger: Logger): Promise<KnowledgeHealth>;
|
|
27
|
+
export interface KnowledgeHealthCheckConfig {
|
|
28
|
+
/** Run post-boot and periodic health checks. Default: true. */
|
|
29
|
+
enabled?: boolean;
|
|
30
|
+
/** How often to re-run the check after boot. Default: 30 minutes. */
|
|
31
|
+
intervalMs?: number;
|
|
32
|
+
/**
|
|
33
|
+
* Delete orphaned embeddings rows (no matching chunk) whenever a check
|
|
34
|
+
* finds them. Default: false - opt-in, since it's a delete even though a
|
|
35
|
+
* safe/idempotent one. Under normal operation orphans should be rare
|
|
36
|
+
* (`cleanupStaleChunks` already deletes a stale chunk and its embedding
|
|
37
|
+
* together); this exists chiefly to clean up historical damage and catch
|
|
38
|
+
* any future code path that doesn't maintain that invariant.
|
|
39
|
+
*/
|
|
40
|
+
autoCleanOrphanedEmbeddings?: boolean;
|
|
41
|
+
}
|
|
42
|
+
export declare const DEFAULT_KNOWLEDGE_HEALTH_INTERVAL_MS: number;
|
|
43
|
+
export interface KnowledgeHealthScheduler {
|
|
44
|
+
/** Runs one check immediately, applying auto-clean if configured. */
|
|
45
|
+
runOnce(): Promise<KnowledgeHealth>;
|
|
46
|
+
/** Stops periodic re-checks. Safe to call even if periodic checks never started. */
|
|
47
|
+
stop(): void;
|
|
48
|
+
}
|
|
49
|
+
export interface ScheduleKnowledgeHealthChecksOptions {
|
|
50
|
+
db: LibsqlClient;
|
|
51
|
+
config?: KnowledgeHealthCheckConfig;
|
|
52
|
+
/** Default: a console logger writing to stderr. */
|
|
53
|
+
logger?: Logger;
|
|
54
|
+
/** Default: a lock table in `db` (see buildLock.ts) - the same lock `VectorStore.build()` uses. */
|
|
55
|
+
buildLock?: BuildLock;
|
|
56
|
+
/** Identifies this process while it holds the build lock during cleanup. Default: a random id. */
|
|
57
|
+
instanceId?: string;
|
|
58
|
+
/** Injectable for tests; defaults to `setInterval`/`clearInterval`. */
|
|
59
|
+
schedule?: (fn: () => void, ms: number) => {
|
|
60
|
+
stop: () => void;
|
|
61
|
+
};
|
|
62
|
+
}
|
|
63
|
+
/**
|
|
64
|
+
* Runs {@link checkKnowledgeHealth} once immediately and, unless disabled,
|
|
65
|
+
* again on `intervalMs` via an unref'd timer - so it never blocks process
|
|
66
|
+
* exit or leaves an open handle in tests. `enabled: false` skips both the
|
|
67
|
+
* immediate run and the periodic one.
|
|
68
|
+
*/
|
|
69
|
+
export declare function scheduleKnowledgeHealthChecks(options: ScheduleKnowledgeHealthChecksOptions): Promise<KnowledgeHealthScheduler>;
|
|
70
|
+
//# sourceMappingURL=knowledgeHealth.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"knowledgeHealth.d.ts","sourceRoot":"","sources":["../../src/core/knowledgeHealth.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,KAAK,EAAE,MAAM,IAAI,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC7D,OAAO,EAGL,KAAK,SAAS,EAEf,MAAM,aAAa,CAAC;AACrB,OAAO,EAAuB,KAAK,MAAM,EAAE,MAAM,UAAU,CAAC;AAE5D,MAAM,WAAW,eAAe;IAC9B,cAAc,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACvC,WAAW,EAAE,MAAM,CAAC;IACpB,eAAe,EAAE,MAAM,CAAC;IACxB,mDAAmD;IACnD,kBAAkB,EAAE,MAAM,CAAC;IAC3B,wFAAwF;IACxF,iBAAiB,EAAE,MAAM,CAAC;CAC3B;AAED;;;;;GAKG;AACH,wBAAsB,oBAAoB,CACxC,EAAE,EAAE,YAAY,EAChB,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,eAAe,CAAC,CAuC1B;AAsCD,MAAM,WAAW,0BAA0B;IACzC,+DAA+D;IAC/D,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,qEAAqE;IACrE,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;;;;;;OAOG;IACH,2BAA2B,CAAC,EAAE,OAAO,CAAC;CACvC;AAED,eAAO,MAAM,oCAAoC,QAAc,CAAC;AAEhE,MAAM,WAAW,wBAAwB;IACvC,qEAAqE;IACrE,OAAO,IAAI,OAAO,CAAC,eAAe,CAAC,CAAC;IACpC,oFAAoF;IACpF,IAAI,IAAI,IAAI,CAAC;CACd;AAED,MAAM,WAAW,oCAAoC;IACnD,EAAE,EAAE,YAAY,CAAC;IACjB,MAAM,CAAC,EAAE,0BAA0B,CAAC;IACpC,mDAAmD;IACnD,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,mGAAmG;IACnG,SAAS,CAAC,EAAE,SAAS,CAAC;IACtB,kGAAkG;IAClG,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,uEAAuE;IACvE,QAAQ,CAAC,EAAE,CAAC,EAAE,EAAE,MAAM,IAAI,EAAE,EAAE,EAAE,MAAM,KAAK;QAAE,IAAI,EAAE,MAAM,IAAI,CAAA;KAAE,CAAC;CACjE;AAQD;;;;;GAKG;AACH,wBAAsB,6BAA6B,CACjD,OAAO,EAAE,oCAAoC,GAC5C,OAAO,CAAC,wBAAwB,CAAC,CAyCnC"}
|
package/dist/index.js
CHANGED
|
@@ -416,6 +416,119 @@ var init_retrieval = __esm({
|
|
|
416
416
|
}
|
|
417
417
|
});
|
|
418
418
|
|
|
419
|
+
// src/core/knowledgeHealth.ts
|
|
420
|
+
var knowledgeHealth_exports = {};
|
|
421
|
+
__export(knowledgeHealth_exports, {
|
|
422
|
+
DEFAULT_KNOWLEDGE_HEALTH_INTERVAL_MS: () => DEFAULT_KNOWLEDGE_HEALTH_INTERVAL_MS,
|
|
423
|
+
checkKnowledgeHealth: () => checkKnowledgeHealth,
|
|
424
|
+
scheduleKnowledgeHealthChecks: () => scheduleKnowledgeHealthChecks
|
|
425
|
+
});
|
|
426
|
+
async function checkKnowledgeHealth(db, logger) {
|
|
427
|
+
const [byBucket, totalChunksRes, totalEmbeddingsRes, orphanedRes, missingRes] = await Promise.all(
|
|
428
|
+
[
|
|
429
|
+
db.execute("SELECT bucket, COUNT(*) as count FROM chunks GROUP BY bucket"),
|
|
430
|
+
db.execute("SELECT COUNT(*) as count FROM chunks"),
|
|
431
|
+
db.execute("SELECT COUNT(*) as count FROM embeddings"),
|
|
432
|
+
db.execute(
|
|
433
|
+
"SELECT COUNT(*) as count FROM embeddings e LEFT JOIN chunks c ON c.id = e.id WHERE c.id IS NULL"
|
|
434
|
+
),
|
|
435
|
+
db.execute(
|
|
436
|
+
"SELECT COUNT(*) as count FROM chunks c LEFT JOIN embeddings e ON e.id = c.id WHERE e.id IS NULL"
|
|
437
|
+
)
|
|
438
|
+
]
|
|
439
|
+
);
|
|
440
|
+
const chunksByBucket = {};
|
|
441
|
+
for (const row of byBucket.rows) {
|
|
442
|
+
chunksByBucket[String(row.bucket)] = Number(row.count ?? 0);
|
|
443
|
+
}
|
|
444
|
+
const health = {
|
|
445
|
+
chunksByBucket,
|
|
446
|
+
totalChunks: Number(totalChunksRes.rows[0]?.count ?? 0),
|
|
447
|
+
totalEmbeddings: Number(totalEmbeddingsRes.rows[0]?.count ?? 0),
|
|
448
|
+
orphanedEmbeddings: Number(orphanedRes.rows[0]?.count ?? 0),
|
|
449
|
+
missingEmbeddings: Number(missingRes.rows[0]?.count ?? 0)
|
|
450
|
+
};
|
|
451
|
+
const summary = `chunksByBucket=${JSON.stringify(health.chunksByBucket)} totalChunks=${health.totalChunks} totalEmbeddings=${health.totalEmbeddings} orphanedEmbeddings=${health.orphanedEmbeddings} missingEmbeddings=${health.missingEmbeddings}`;
|
|
452
|
+
if (health.totalChunks === 0 || health.orphanedEmbeddings > 0) {
|
|
453
|
+
logger.warn(`Knowledge base health check: ${summary}`);
|
|
454
|
+
} else {
|
|
455
|
+
logger.info(`Knowledge base health check: ${summary}`);
|
|
456
|
+
}
|
|
457
|
+
return health;
|
|
458
|
+
}
|
|
459
|
+
async function cleanOrphanedEmbeddings(db, buildLock, instanceId, logger) {
|
|
460
|
+
const acquired = await buildLock.tryAcquire(
|
|
461
|
+
BUILD_LOCK_KEY,
|
|
462
|
+
instanceId,
|
|
463
|
+
Date.now(),
|
|
464
|
+
BUILD_LOCK_STALE_MS
|
|
465
|
+
);
|
|
466
|
+
if (!acquired) {
|
|
467
|
+
logger.warn(
|
|
468
|
+
"Knowledge health: skipping orphaned-embeddings cleanup - the build lock is held by another instance."
|
|
469
|
+
);
|
|
470
|
+
return;
|
|
471
|
+
}
|
|
472
|
+
try {
|
|
473
|
+
const result = await db.execute(
|
|
474
|
+
"DELETE FROM embeddings WHERE id NOT IN (SELECT id FROM chunks)"
|
|
475
|
+
);
|
|
476
|
+
if (result.rowsAffected > 0) {
|
|
477
|
+
logger.warn(`Knowledge health: deleted ${result.rowsAffected} orphaned embedding row(s).`);
|
|
478
|
+
}
|
|
479
|
+
} finally {
|
|
480
|
+
await buildLock.release(BUILD_LOCK_KEY, instanceId);
|
|
481
|
+
}
|
|
482
|
+
}
|
|
483
|
+
async function scheduleKnowledgeHealthChecks(options) {
|
|
484
|
+
const logger = options.logger ?? createConsoleLogger();
|
|
485
|
+
const enabled = options.config?.enabled !== false;
|
|
486
|
+
const intervalMs = options.config?.intervalMs ?? DEFAULT_KNOWLEDGE_HEALTH_INTERVAL_MS;
|
|
487
|
+
const autoClean = options.config?.autoCleanOrphanedEmbeddings === true;
|
|
488
|
+
const buildLock = options.buildLock ?? createTursoBuildLock(options.db);
|
|
489
|
+
const instanceId = options.instanceId ?? crypto.randomUUID();
|
|
490
|
+
const schedule = options.schedule ?? defaultSchedule;
|
|
491
|
+
async function runOnce() {
|
|
492
|
+
const health = await checkKnowledgeHealth(options.db, logger);
|
|
493
|
+
if (autoClean && health.orphanedEmbeddings > 0) {
|
|
494
|
+
await cleanOrphanedEmbeddings(options.db, buildLock, instanceId, logger);
|
|
495
|
+
}
|
|
496
|
+
return health;
|
|
497
|
+
}
|
|
498
|
+
let handle;
|
|
499
|
+
if (enabled) {
|
|
500
|
+
await runOnce();
|
|
501
|
+
if (intervalMs > 0) {
|
|
502
|
+
let ticking = false;
|
|
503
|
+
handle = schedule(() => {
|
|
504
|
+
if (ticking) return;
|
|
505
|
+
ticking = true;
|
|
506
|
+
runOnce().catch((error) => logger.error("Knowledge health check failed:", error)).finally(() => {
|
|
507
|
+
ticking = false;
|
|
508
|
+
});
|
|
509
|
+
}, intervalMs);
|
|
510
|
+
}
|
|
511
|
+
}
|
|
512
|
+
return {
|
|
513
|
+
runOnce,
|
|
514
|
+
stop: () => handle?.stop()
|
|
515
|
+
};
|
|
516
|
+
}
|
|
517
|
+
var DEFAULT_KNOWLEDGE_HEALTH_INTERVAL_MS, defaultSchedule;
|
|
518
|
+
var init_knowledgeHealth = __esm({
|
|
519
|
+
"src/core/knowledgeHealth.ts"() {
|
|
520
|
+
"use strict";
|
|
521
|
+
init_buildLock();
|
|
522
|
+
init_logger();
|
|
523
|
+
DEFAULT_KNOWLEDGE_HEALTH_INTERVAL_MS = 30 * 6e4;
|
|
524
|
+
defaultSchedule = (fn, ms) => {
|
|
525
|
+
const timer = setInterval(fn, ms);
|
|
526
|
+
if (typeof timer === "object" && "unref" in timer) timer.unref();
|
|
527
|
+
return { stop: () => clearInterval(timer) };
|
|
528
|
+
};
|
|
529
|
+
}
|
|
530
|
+
});
|
|
531
|
+
|
|
419
532
|
// src/index.ts
|
|
420
533
|
var index_exports = {};
|
|
421
534
|
__export(index_exports, {
|
|
@@ -1221,6 +1334,7 @@ async function createMCPServer(config) {
|
|
|
1221
1334
|
});
|
|
1222
1335
|
const model = config.openai.model || DEFAULT_MODEL;
|
|
1223
1336
|
let store;
|
|
1337
|
+
let db;
|
|
1224
1338
|
if (config.retriever) {
|
|
1225
1339
|
store = config.retriever;
|
|
1226
1340
|
} else {
|
|
@@ -1228,7 +1342,7 @@ async function createMCPServer(config) {
|
|
|
1228
1342
|
throw new Error(DATABASE_CONFIG_REQUIRED_MESSAGE);
|
|
1229
1343
|
}
|
|
1230
1344
|
const { DEFAULT_KNOWLEDGE_DIR: DEFAULT_KNOWLEDGE_DIR2, VectorStore: VectorStore2, createOpenAIEmbedder: createOpenAIEmbedder2, openLibsqlClient: openLibsqlClient2 } = await Promise.resolve().then(() => (init_retrieval(), retrieval_exports));
|
|
1231
|
-
|
|
1345
|
+
db = await openLibsqlClient2(config.database);
|
|
1232
1346
|
store = new VectorStore2(createOpenAIEmbedder2(client), {
|
|
1233
1347
|
databaseClient: db,
|
|
1234
1348
|
knowledgeDir: config.knowledgeDir || DEFAULT_KNOWLEDGE_DIR2,
|
|
@@ -1237,6 +1351,15 @@ async function createMCPServer(config) {
|
|
|
1237
1351
|
}
|
|
1238
1352
|
await store.build?.();
|
|
1239
1353
|
log.info("\u2705 Knowledge base ready");
|
|
1354
|
+
let knowledgeHealth;
|
|
1355
|
+
if (db) {
|
|
1356
|
+
const { scheduleKnowledgeHealthChecks: scheduleKnowledgeHealthChecks2 } = await Promise.resolve().then(() => (init_knowledgeHealth(), knowledgeHealth_exports));
|
|
1357
|
+
knowledgeHealth = await scheduleKnowledgeHealthChecks2({
|
|
1358
|
+
db,
|
|
1359
|
+
config: config.knowledgeHealthCheck,
|
|
1360
|
+
logger: log
|
|
1361
|
+
});
|
|
1362
|
+
}
|
|
1240
1363
|
const prompts = new PromptLoader(promptsDir, config.bot);
|
|
1241
1364
|
const server = new McpServer({
|
|
1242
1365
|
name: config.bot.name,
|
|
@@ -1433,7 +1556,9 @@ async function createMCPServer(config) {
|
|
|
1433
1556
|
log.info(
|
|
1434
1557
|
`\u2705 ${config.bot.name} MCP Server ready (transport: ${transportMode}, tools: ${enabledTools.join(", ") || "none"})`
|
|
1435
1558
|
);
|
|
1436
|
-
return server
|
|
1559
|
+
return Object.assign(server, {
|
|
1560
|
+
stopKnowledgeHealthChecks: () => knowledgeHealth?.stop()
|
|
1561
|
+
});
|
|
1437
1562
|
}
|
|
1438
1563
|
|
|
1439
1564
|
// src/middleware/apiKey.ts
|
|
@@ -2416,6 +2541,15 @@ async function createServer(config) {
|
|
|
2416
2541
|
}
|
|
2417
2542
|
await store.build?.();
|
|
2418
2543
|
logger.info("\u2705 Knowledge base ready");
|
|
2544
|
+
let knowledgeHealth;
|
|
2545
|
+
if (db && !config.retriever) {
|
|
2546
|
+
const { scheduleKnowledgeHealthChecks: scheduleKnowledgeHealthChecks2 } = await Promise.resolve().then(() => (init_knowledgeHealth(), knowledgeHealth_exports));
|
|
2547
|
+
knowledgeHealth = await scheduleKnowledgeHealthChecks2({
|
|
2548
|
+
db,
|
|
2549
|
+
config: config.knowledgeHealthCheck,
|
|
2550
|
+
logger
|
|
2551
|
+
});
|
|
2552
|
+
}
|
|
2419
2553
|
const prompts = new PromptLoader(promptsDir, config.bot);
|
|
2420
2554
|
let apiKeyManager;
|
|
2421
2555
|
const secret = config.auth?.secret || process.env.CHATTER_SECRET;
|
|
@@ -2546,6 +2680,7 @@ async function createServer(config) {
|
|
|
2546
2680
|
// (sync or async) is caught per-channel so one misbehaving channel can't
|
|
2547
2681
|
// stop the rest from cleaning up.
|
|
2548
2682
|
stopChannels: async () => {
|
|
2683
|
+
knowledgeHealth?.stop();
|
|
2549
2684
|
await Promise.allSettled(
|
|
2550
2685
|
started.map(async (channel) => {
|
|
2551
2686
|
try {
|
package/dist/index.mjs
CHANGED
|
@@ -393,6 +393,119 @@ var init_retrieval = __esm({
|
|
|
393
393
|
}
|
|
394
394
|
});
|
|
395
395
|
|
|
396
|
+
// src/core/knowledgeHealth.ts
|
|
397
|
+
var knowledgeHealth_exports = {};
|
|
398
|
+
__export(knowledgeHealth_exports, {
|
|
399
|
+
DEFAULT_KNOWLEDGE_HEALTH_INTERVAL_MS: () => DEFAULT_KNOWLEDGE_HEALTH_INTERVAL_MS,
|
|
400
|
+
checkKnowledgeHealth: () => checkKnowledgeHealth,
|
|
401
|
+
scheduleKnowledgeHealthChecks: () => scheduleKnowledgeHealthChecks
|
|
402
|
+
});
|
|
403
|
+
async function checkKnowledgeHealth(db, logger) {
|
|
404
|
+
const [byBucket, totalChunksRes, totalEmbeddingsRes, orphanedRes, missingRes] = await Promise.all(
|
|
405
|
+
[
|
|
406
|
+
db.execute("SELECT bucket, COUNT(*) as count FROM chunks GROUP BY bucket"),
|
|
407
|
+
db.execute("SELECT COUNT(*) as count FROM chunks"),
|
|
408
|
+
db.execute("SELECT COUNT(*) as count FROM embeddings"),
|
|
409
|
+
db.execute(
|
|
410
|
+
"SELECT COUNT(*) as count FROM embeddings e LEFT JOIN chunks c ON c.id = e.id WHERE c.id IS NULL"
|
|
411
|
+
),
|
|
412
|
+
db.execute(
|
|
413
|
+
"SELECT COUNT(*) as count FROM chunks c LEFT JOIN embeddings e ON e.id = c.id WHERE e.id IS NULL"
|
|
414
|
+
)
|
|
415
|
+
]
|
|
416
|
+
);
|
|
417
|
+
const chunksByBucket = {};
|
|
418
|
+
for (const row of byBucket.rows) {
|
|
419
|
+
chunksByBucket[String(row.bucket)] = Number(row.count ?? 0);
|
|
420
|
+
}
|
|
421
|
+
const health = {
|
|
422
|
+
chunksByBucket,
|
|
423
|
+
totalChunks: Number(totalChunksRes.rows[0]?.count ?? 0),
|
|
424
|
+
totalEmbeddings: Number(totalEmbeddingsRes.rows[0]?.count ?? 0),
|
|
425
|
+
orphanedEmbeddings: Number(orphanedRes.rows[0]?.count ?? 0),
|
|
426
|
+
missingEmbeddings: Number(missingRes.rows[0]?.count ?? 0)
|
|
427
|
+
};
|
|
428
|
+
const summary = `chunksByBucket=${JSON.stringify(health.chunksByBucket)} totalChunks=${health.totalChunks} totalEmbeddings=${health.totalEmbeddings} orphanedEmbeddings=${health.orphanedEmbeddings} missingEmbeddings=${health.missingEmbeddings}`;
|
|
429
|
+
if (health.totalChunks === 0 || health.orphanedEmbeddings > 0) {
|
|
430
|
+
logger.warn(`Knowledge base health check: ${summary}`);
|
|
431
|
+
} else {
|
|
432
|
+
logger.info(`Knowledge base health check: ${summary}`);
|
|
433
|
+
}
|
|
434
|
+
return health;
|
|
435
|
+
}
|
|
436
|
+
async function cleanOrphanedEmbeddings(db, buildLock, instanceId, logger) {
|
|
437
|
+
const acquired = await buildLock.tryAcquire(
|
|
438
|
+
BUILD_LOCK_KEY,
|
|
439
|
+
instanceId,
|
|
440
|
+
Date.now(),
|
|
441
|
+
BUILD_LOCK_STALE_MS
|
|
442
|
+
);
|
|
443
|
+
if (!acquired) {
|
|
444
|
+
logger.warn(
|
|
445
|
+
"Knowledge health: skipping orphaned-embeddings cleanup - the build lock is held by another instance."
|
|
446
|
+
);
|
|
447
|
+
return;
|
|
448
|
+
}
|
|
449
|
+
try {
|
|
450
|
+
const result = await db.execute(
|
|
451
|
+
"DELETE FROM embeddings WHERE id NOT IN (SELECT id FROM chunks)"
|
|
452
|
+
);
|
|
453
|
+
if (result.rowsAffected > 0) {
|
|
454
|
+
logger.warn(`Knowledge health: deleted ${result.rowsAffected} orphaned embedding row(s).`);
|
|
455
|
+
}
|
|
456
|
+
} finally {
|
|
457
|
+
await buildLock.release(BUILD_LOCK_KEY, instanceId);
|
|
458
|
+
}
|
|
459
|
+
}
|
|
460
|
+
async function scheduleKnowledgeHealthChecks(options) {
|
|
461
|
+
const logger = options.logger ?? createConsoleLogger();
|
|
462
|
+
const enabled = options.config?.enabled !== false;
|
|
463
|
+
const intervalMs = options.config?.intervalMs ?? DEFAULT_KNOWLEDGE_HEALTH_INTERVAL_MS;
|
|
464
|
+
const autoClean = options.config?.autoCleanOrphanedEmbeddings === true;
|
|
465
|
+
const buildLock = options.buildLock ?? createTursoBuildLock(options.db);
|
|
466
|
+
const instanceId = options.instanceId ?? crypto.randomUUID();
|
|
467
|
+
const schedule = options.schedule ?? defaultSchedule;
|
|
468
|
+
async function runOnce() {
|
|
469
|
+
const health = await checkKnowledgeHealth(options.db, logger);
|
|
470
|
+
if (autoClean && health.orphanedEmbeddings > 0) {
|
|
471
|
+
await cleanOrphanedEmbeddings(options.db, buildLock, instanceId, logger);
|
|
472
|
+
}
|
|
473
|
+
return health;
|
|
474
|
+
}
|
|
475
|
+
let handle;
|
|
476
|
+
if (enabled) {
|
|
477
|
+
await runOnce();
|
|
478
|
+
if (intervalMs > 0) {
|
|
479
|
+
let ticking = false;
|
|
480
|
+
handle = schedule(() => {
|
|
481
|
+
if (ticking) return;
|
|
482
|
+
ticking = true;
|
|
483
|
+
runOnce().catch((error) => logger.error("Knowledge health check failed:", error)).finally(() => {
|
|
484
|
+
ticking = false;
|
|
485
|
+
});
|
|
486
|
+
}, intervalMs);
|
|
487
|
+
}
|
|
488
|
+
}
|
|
489
|
+
return {
|
|
490
|
+
runOnce,
|
|
491
|
+
stop: () => handle?.stop()
|
|
492
|
+
};
|
|
493
|
+
}
|
|
494
|
+
var DEFAULT_KNOWLEDGE_HEALTH_INTERVAL_MS, defaultSchedule;
|
|
495
|
+
var init_knowledgeHealth = __esm({
|
|
496
|
+
"src/core/knowledgeHealth.ts"() {
|
|
497
|
+
"use strict";
|
|
498
|
+
init_buildLock();
|
|
499
|
+
init_logger();
|
|
500
|
+
DEFAULT_KNOWLEDGE_HEALTH_INTERVAL_MS = 30 * 6e4;
|
|
501
|
+
defaultSchedule = (fn, ms) => {
|
|
502
|
+
const timer = setInterval(fn, ms);
|
|
503
|
+
if (typeof timer === "object" && "unref" in timer) timer.unref();
|
|
504
|
+
return { stop: () => clearInterval(timer) };
|
|
505
|
+
};
|
|
506
|
+
}
|
|
507
|
+
});
|
|
508
|
+
|
|
396
509
|
// src/auth/apikeys.ts
|
|
397
510
|
import { randomUUID } from "node:crypto";
|
|
398
511
|
import { jwtVerify, SignJWT } from "jose";
|
|
@@ -1146,6 +1259,7 @@ async function createMCPServer(config) {
|
|
|
1146
1259
|
});
|
|
1147
1260
|
const model = config.openai.model || DEFAULT_MODEL;
|
|
1148
1261
|
let store;
|
|
1262
|
+
let db;
|
|
1149
1263
|
if (config.retriever) {
|
|
1150
1264
|
store = config.retriever;
|
|
1151
1265
|
} else {
|
|
@@ -1153,7 +1267,7 @@ async function createMCPServer(config) {
|
|
|
1153
1267
|
throw new Error(DATABASE_CONFIG_REQUIRED_MESSAGE);
|
|
1154
1268
|
}
|
|
1155
1269
|
const { DEFAULT_KNOWLEDGE_DIR: DEFAULT_KNOWLEDGE_DIR2, VectorStore: VectorStore2, createOpenAIEmbedder: createOpenAIEmbedder2, openLibsqlClient: openLibsqlClient2 } = await Promise.resolve().then(() => (init_retrieval(), retrieval_exports));
|
|
1156
|
-
|
|
1270
|
+
db = await openLibsqlClient2(config.database);
|
|
1157
1271
|
store = new VectorStore2(createOpenAIEmbedder2(client), {
|
|
1158
1272
|
databaseClient: db,
|
|
1159
1273
|
knowledgeDir: config.knowledgeDir || DEFAULT_KNOWLEDGE_DIR2,
|
|
@@ -1162,6 +1276,15 @@ async function createMCPServer(config) {
|
|
|
1162
1276
|
}
|
|
1163
1277
|
await store.build?.();
|
|
1164
1278
|
log.info("\u2705 Knowledge base ready");
|
|
1279
|
+
let knowledgeHealth;
|
|
1280
|
+
if (db) {
|
|
1281
|
+
const { scheduleKnowledgeHealthChecks: scheduleKnowledgeHealthChecks2 } = await Promise.resolve().then(() => (init_knowledgeHealth(), knowledgeHealth_exports));
|
|
1282
|
+
knowledgeHealth = await scheduleKnowledgeHealthChecks2({
|
|
1283
|
+
db,
|
|
1284
|
+
config: config.knowledgeHealthCheck,
|
|
1285
|
+
logger: log
|
|
1286
|
+
});
|
|
1287
|
+
}
|
|
1165
1288
|
const prompts = new PromptLoader(promptsDir, config.bot);
|
|
1166
1289
|
const server = new McpServer({
|
|
1167
1290
|
name: config.bot.name,
|
|
@@ -1358,7 +1481,9 @@ async function createMCPServer(config) {
|
|
|
1358
1481
|
log.info(
|
|
1359
1482
|
`\u2705 ${config.bot.name} MCP Server ready (transport: ${transportMode}, tools: ${enabledTools.join(", ") || "none"})`
|
|
1360
1483
|
);
|
|
1361
|
-
return server
|
|
1484
|
+
return Object.assign(server, {
|
|
1485
|
+
stopKnowledgeHealthChecks: () => knowledgeHealth?.stop()
|
|
1486
|
+
});
|
|
1362
1487
|
}
|
|
1363
1488
|
|
|
1364
1489
|
// src/middleware/apiKey.ts
|
|
@@ -2340,6 +2465,15 @@ async function createServer(config) {
|
|
|
2340
2465
|
}
|
|
2341
2466
|
await store.build?.();
|
|
2342
2467
|
logger.info("\u2705 Knowledge base ready");
|
|
2468
|
+
let knowledgeHealth;
|
|
2469
|
+
if (db && !config.retriever) {
|
|
2470
|
+
const { scheduleKnowledgeHealthChecks: scheduleKnowledgeHealthChecks2 } = await Promise.resolve().then(() => (init_knowledgeHealth(), knowledgeHealth_exports));
|
|
2471
|
+
knowledgeHealth = await scheduleKnowledgeHealthChecks2({
|
|
2472
|
+
db,
|
|
2473
|
+
config: config.knowledgeHealthCheck,
|
|
2474
|
+
logger
|
|
2475
|
+
});
|
|
2476
|
+
}
|
|
2343
2477
|
const prompts = new PromptLoader(promptsDir, config.bot);
|
|
2344
2478
|
let apiKeyManager;
|
|
2345
2479
|
const secret = config.auth?.secret || process.env.CHATTER_SECRET;
|
|
@@ -2470,6 +2604,7 @@ async function createServer(config) {
|
|
|
2470
2604
|
// (sync or async) is caught per-channel so one misbehaving channel can't
|
|
2471
2605
|
// stop the rest from cleaning up.
|
|
2472
2606
|
stopChannels: async () => {
|
|
2607
|
+
knowledgeHealth?.stop();
|
|
2473
2608
|
await Promise.allSettled(
|
|
2474
2609
|
started.map(async (channel) => {
|
|
2475
2610
|
try {
|
package/dist/mcp-server.d.ts
CHANGED
|
@@ -31,5 +31,7 @@ import type { MCPServerOptions } from "./mcp-server/types";
|
|
|
31
31
|
* await server.connect(transport);
|
|
32
32
|
* ```
|
|
33
33
|
*/
|
|
34
|
-
export declare function createMCPServer(config: MCPServerOptions): Promise<import("@modelcontextprotocol/sdk/server/mcp").McpServer
|
|
34
|
+
export declare function createMCPServer(config: MCPServerOptions): Promise<import("@modelcontextprotocol/sdk/server/mcp").McpServer & {
|
|
35
|
+
stopKnowledgeHealthChecks: () => void;
|
|
36
|
+
}>;
|
|
35
37
|
//# sourceMappingURL=mcp-server.d.ts.map
|
package/dist/mcp-server.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"mcp-server.d.ts","sourceRoot":"","sources":["../src/mcp-server.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;
|
|
1
|
+
{"version":3,"file":"mcp-server.d.ts","sourceRoot":"","sources":["../src/mcp-server.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAsBH,YAAY,EACV,WAAW,EACX,QAAQ,EACR,cAAc,EACd,gBAAgB,EAChB,aAAa,EACb,gBAAgB,GACjB,MAAM,oBAAoB,CAAC;AAE5B,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAE3D;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,wBAAsB,eAAe,CAAC,MAAM,EAAE,gBAAgB;qCA4U3B,IAAI;GAEtC"}
|
package/dist/mcp-server.js
CHANGED
|
@@ -411,6 +411,119 @@ var init_retrieval = __esm({
|
|
|
411
411
|
}
|
|
412
412
|
});
|
|
413
413
|
|
|
414
|
+
// src/core/knowledgeHealth.ts
|
|
415
|
+
var knowledgeHealth_exports = {};
|
|
416
|
+
__export(knowledgeHealth_exports, {
|
|
417
|
+
DEFAULT_KNOWLEDGE_HEALTH_INTERVAL_MS: () => DEFAULT_KNOWLEDGE_HEALTH_INTERVAL_MS,
|
|
418
|
+
checkKnowledgeHealth: () => checkKnowledgeHealth,
|
|
419
|
+
scheduleKnowledgeHealthChecks: () => scheduleKnowledgeHealthChecks
|
|
420
|
+
});
|
|
421
|
+
async function checkKnowledgeHealth(db, logger) {
|
|
422
|
+
const [byBucket, totalChunksRes, totalEmbeddingsRes, orphanedRes, missingRes] = await Promise.all(
|
|
423
|
+
[
|
|
424
|
+
db.execute("SELECT bucket, COUNT(*) as count FROM chunks GROUP BY bucket"),
|
|
425
|
+
db.execute("SELECT COUNT(*) as count FROM chunks"),
|
|
426
|
+
db.execute("SELECT COUNT(*) as count FROM embeddings"),
|
|
427
|
+
db.execute(
|
|
428
|
+
"SELECT COUNT(*) as count FROM embeddings e LEFT JOIN chunks c ON c.id = e.id WHERE c.id IS NULL"
|
|
429
|
+
),
|
|
430
|
+
db.execute(
|
|
431
|
+
"SELECT COUNT(*) as count FROM chunks c LEFT JOIN embeddings e ON e.id = c.id WHERE e.id IS NULL"
|
|
432
|
+
)
|
|
433
|
+
]
|
|
434
|
+
);
|
|
435
|
+
const chunksByBucket = {};
|
|
436
|
+
for (const row of byBucket.rows) {
|
|
437
|
+
chunksByBucket[String(row.bucket)] = Number(row.count ?? 0);
|
|
438
|
+
}
|
|
439
|
+
const health = {
|
|
440
|
+
chunksByBucket,
|
|
441
|
+
totalChunks: Number(totalChunksRes.rows[0]?.count ?? 0),
|
|
442
|
+
totalEmbeddings: Number(totalEmbeddingsRes.rows[0]?.count ?? 0),
|
|
443
|
+
orphanedEmbeddings: Number(orphanedRes.rows[0]?.count ?? 0),
|
|
444
|
+
missingEmbeddings: Number(missingRes.rows[0]?.count ?? 0)
|
|
445
|
+
};
|
|
446
|
+
const summary = `chunksByBucket=${JSON.stringify(health.chunksByBucket)} totalChunks=${health.totalChunks} totalEmbeddings=${health.totalEmbeddings} orphanedEmbeddings=${health.orphanedEmbeddings} missingEmbeddings=${health.missingEmbeddings}`;
|
|
447
|
+
if (health.totalChunks === 0 || health.orphanedEmbeddings > 0) {
|
|
448
|
+
logger.warn(`Knowledge base health check: ${summary}`);
|
|
449
|
+
} else {
|
|
450
|
+
logger.info(`Knowledge base health check: ${summary}`);
|
|
451
|
+
}
|
|
452
|
+
return health;
|
|
453
|
+
}
|
|
454
|
+
async function cleanOrphanedEmbeddings(db, buildLock, instanceId, logger) {
|
|
455
|
+
const acquired = await buildLock.tryAcquire(
|
|
456
|
+
BUILD_LOCK_KEY,
|
|
457
|
+
instanceId,
|
|
458
|
+
Date.now(),
|
|
459
|
+
BUILD_LOCK_STALE_MS
|
|
460
|
+
);
|
|
461
|
+
if (!acquired) {
|
|
462
|
+
logger.warn(
|
|
463
|
+
"Knowledge health: skipping orphaned-embeddings cleanup - the build lock is held by another instance."
|
|
464
|
+
);
|
|
465
|
+
return;
|
|
466
|
+
}
|
|
467
|
+
try {
|
|
468
|
+
const result = await db.execute(
|
|
469
|
+
"DELETE FROM embeddings WHERE id NOT IN (SELECT id FROM chunks)"
|
|
470
|
+
);
|
|
471
|
+
if (result.rowsAffected > 0) {
|
|
472
|
+
logger.warn(`Knowledge health: deleted ${result.rowsAffected} orphaned embedding row(s).`);
|
|
473
|
+
}
|
|
474
|
+
} finally {
|
|
475
|
+
await buildLock.release(BUILD_LOCK_KEY, instanceId);
|
|
476
|
+
}
|
|
477
|
+
}
|
|
478
|
+
async function scheduleKnowledgeHealthChecks(options) {
|
|
479
|
+
const logger = options.logger ?? createConsoleLogger();
|
|
480
|
+
const enabled = options.config?.enabled !== false;
|
|
481
|
+
const intervalMs = options.config?.intervalMs ?? DEFAULT_KNOWLEDGE_HEALTH_INTERVAL_MS;
|
|
482
|
+
const autoClean = options.config?.autoCleanOrphanedEmbeddings === true;
|
|
483
|
+
const buildLock = options.buildLock ?? createTursoBuildLock(options.db);
|
|
484
|
+
const instanceId = options.instanceId ?? crypto.randomUUID();
|
|
485
|
+
const schedule = options.schedule ?? defaultSchedule;
|
|
486
|
+
async function runOnce() {
|
|
487
|
+
const health = await checkKnowledgeHealth(options.db, logger);
|
|
488
|
+
if (autoClean && health.orphanedEmbeddings > 0) {
|
|
489
|
+
await cleanOrphanedEmbeddings(options.db, buildLock, instanceId, logger);
|
|
490
|
+
}
|
|
491
|
+
return health;
|
|
492
|
+
}
|
|
493
|
+
let handle;
|
|
494
|
+
if (enabled) {
|
|
495
|
+
await runOnce();
|
|
496
|
+
if (intervalMs > 0) {
|
|
497
|
+
let ticking = false;
|
|
498
|
+
handle = schedule(() => {
|
|
499
|
+
if (ticking) return;
|
|
500
|
+
ticking = true;
|
|
501
|
+
runOnce().catch((error) => logger.error("Knowledge health check failed:", error)).finally(() => {
|
|
502
|
+
ticking = false;
|
|
503
|
+
});
|
|
504
|
+
}, intervalMs);
|
|
505
|
+
}
|
|
506
|
+
}
|
|
507
|
+
return {
|
|
508
|
+
runOnce,
|
|
509
|
+
stop: () => handle?.stop()
|
|
510
|
+
};
|
|
511
|
+
}
|
|
512
|
+
var DEFAULT_KNOWLEDGE_HEALTH_INTERVAL_MS, defaultSchedule;
|
|
513
|
+
var init_knowledgeHealth = __esm({
|
|
514
|
+
"src/core/knowledgeHealth.ts"() {
|
|
515
|
+
"use strict";
|
|
516
|
+
init_buildLock();
|
|
517
|
+
init_logger();
|
|
518
|
+
DEFAULT_KNOWLEDGE_HEALTH_INTERVAL_MS = 30 * 6e4;
|
|
519
|
+
defaultSchedule = (fn, ms) => {
|
|
520
|
+
const timer = setInterval(fn, ms);
|
|
521
|
+
if (typeof timer === "object" && "unref" in timer) timer.unref();
|
|
522
|
+
return { stop: () => clearInterval(timer) };
|
|
523
|
+
};
|
|
524
|
+
}
|
|
525
|
+
});
|
|
526
|
+
|
|
414
527
|
// src/mcp-server.ts
|
|
415
528
|
var mcp_server_exports = {};
|
|
416
529
|
__export(mcp_server_exports, {
|
|
@@ -899,6 +1012,7 @@ async function createMCPServer(config) {
|
|
|
899
1012
|
});
|
|
900
1013
|
const model = config.openai.model || DEFAULT_MODEL;
|
|
901
1014
|
let store;
|
|
1015
|
+
let db;
|
|
902
1016
|
if (config.retriever) {
|
|
903
1017
|
store = config.retriever;
|
|
904
1018
|
} else {
|
|
@@ -906,7 +1020,7 @@ async function createMCPServer(config) {
|
|
|
906
1020
|
throw new Error(DATABASE_CONFIG_REQUIRED_MESSAGE);
|
|
907
1021
|
}
|
|
908
1022
|
const { DEFAULT_KNOWLEDGE_DIR: DEFAULT_KNOWLEDGE_DIR2, VectorStore: VectorStore2, createOpenAIEmbedder: createOpenAIEmbedder2, openLibsqlClient: openLibsqlClient2 } = await Promise.resolve().then(() => (init_retrieval(), retrieval_exports));
|
|
909
|
-
|
|
1023
|
+
db = await openLibsqlClient2(config.database);
|
|
910
1024
|
store = new VectorStore2(createOpenAIEmbedder2(client), {
|
|
911
1025
|
databaseClient: db,
|
|
912
1026
|
knowledgeDir: config.knowledgeDir || DEFAULT_KNOWLEDGE_DIR2,
|
|
@@ -915,6 +1029,15 @@ async function createMCPServer(config) {
|
|
|
915
1029
|
}
|
|
916
1030
|
await store.build?.();
|
|
917
1031
|
log.info("\u2705 Knowledge base ready");
|
|
1032
|
+
let knowledgeHealth;
|
|
1033
|
+
if (db) {
|
|
1034
|
+
const { scheduleKnowledgeHealthChecks: scheduleKnowledgeHealthChecks2 } = await Promise.resolve().then(() => (init_knowledgeHealth(), knowledgeHealth_exports));
|
|
1035
|
+
knowledgeHealth = await scheduleKnowledgeHealthChecks2({
|
|
1036
|
+
db,
|
|
1037
|
+
config: config.knowledgeHealthCheck,
|
|
1038
|
+
logger: log
|
|
1039
|
+
});
|
|
1040
|
+
}
|
|
918
1041
|
const prompts = new PromptLoader(promptsDir, config.bot);
|
|
919
1042
|
const server = new McpServer({
|
|
920
1043
|
name: config.bot.name,
|
|
@@ -1111,7 +1234,9 @@ async function createMCPServer(config) {
|
|
|
1111
1234
|
log.info(
|
|
1112
1235
|
`\u2705 ${config.bot.name} MCP Server ready (transport: ${transportMode}, tools: ${enabledTools.join(", ") || "none"})`
|
|
1113
1236
|
);
|
|
1114
|
-
return server
|
|
1237
|
+
return Object.assign(server, {
|
|
1238
|
+
stopKnowledgeHealthChecks: () => knowledgeHealth?.stop()
|
|
1239
|
+
});
|
|
1115
1240
|
}
|
|
1116
1241
|
// Annotate the CommonJS export names for ESM import in node:
|
|
1117
1242
|
0 && (module.exports = {
|
package/dist/mcp-server.mjs
CHANGED
|
@@ -388,6 +388,119 @@ var init_retrieval = __esm({
|
|
|
388
388
|
}
|
|
389
389
|
});
|
|
390
390
|
|
|
391
|
+
// src/core/knowledgeHealth.ts
|
|
392
|
+
var knowledgeHealth_exports = {};
|
|
393
|
+
__export(knowledgeHealth_exports, {
|
|
394
|
+
DEFAULT_KNOWLEDGE_HEALTH_INTERVAL_MS: () => DEFAULT_KNOWLEDGE_HEALTH_INTERVAL_MS,
|
|
395
|
+
checkKnowledgeHealth: () => checkKnowledgeHealth,
|
|
396
|
+
scheduleKnowledgeHealthChecks: () => scheduleKnowledgeHealthChecks
|
|
397
|
+
});
|
|
398
|
+
async function checkKnowledgeHealth(db, logger) {
|
|
399
|
+
const [byBucket, totalChunksRes, totalEmbeddingsRes, orphanedRes, missingRes] = await Promise.all(
|
|
400
|
+
[
|
|
401
|
+
db.execute("SELECT bucket, COUNT(*) as count FROM chunks GROUP BY bucket"),
|
|
402
|
+
db.execute("SELECT COUNT(*) as count FROM chunks"),
|
|
403
|
+
db.execute("SELECT COUNT(*) as count FROM embeddings"),
|
|
404
|
+
db.execute(
|
|
405
|
+
"SELECT COUNT(*) as count FROM embeddings e LEFT JOIN chunks c ON c.id = e.id WHERE c.id IS NULL"
|
|
406
|
+
),
|
|
407
|
+
db.execute(
|
|
408
|
+
"SELECT COUNT(*) as count FROM chunks c LEFT JOIN embeddings e ON e.id = c.id WHERE e.id IS NULL"
|
|
409
|
+
)
|
|
410
|
+
]
|
|
411
|
+
);
|
|
412
|
+
const chunksByBucket = {};
|
|
413
|
+
for (const row of byBucket.rows) {
|
|
414
|
+
chunksByBucket[String(row.bucket)] = Number(row.count ?? 0);
|
|
415
|
+
}
|
|
416
|
+
const health = {
|
|
417
|
+
chunksByBucket,
|
|
418
|
+
totalChunks: Number(totalChunksRes.rows[0]?.count ?? 0),
|
|
419
|
+
totalEmbeddings: Number(totalEmbeddingsRes.rows[0]?.count ?? 0),
|
|
420
|
+
orphanedEmbeddings: Number(orphanedRes.rows[0]?.count ?? 0),
|
|
421
|
+
missingEmbeddings: Number(missingRes.rows[0]?.count ?? 0)
|
|
422
|
+
};
|
|
423
|
+
const summary = `chunksByBucket=${JSON.stringify(health.chunksByBucket)} totalChunks=${health.totalChunks} totalEmbeddings=${health.totalEmbeddings} orphanedEmbeddings=${health.orphanedEmbeddings} missingEmbeddings=${health.missingEmbeddings}`;
|
|
424
|
+
if (health.totalChunks === 0 || health.orphanedEmbeddings > 0) {
|
|
425
|
+
logger.warn(`Knowledge base health check: ${summary}`);
|
|
426
|
+
} else {
|
|
427
|
+
logger.info(`Knowledge base health check: ${summary}`);
|
|
428
|
+
}
|
|
429
|
+
return health;
|
|
430
|
+
}
|
|
431
|
+
async function cleanOrphanedEmbeddings(db, buildLock, instanceId, logger) {
|
|
432
|
+
const acquired = await buildLock.tryAcquire(
|
|
433
|
+
BUILD_LOCK_KEY,
|
|
434
|
+
instanceId,
|
|
435
|
+
Date.now(),
|
|
436
|
+
BUILD_LOCK_STALE_MS
|
|
437
|
+
);
|
|
438
|
+
if (!acquired) {
|
|
439
|
+
logger.warn(
|
|
440
|
+
"Knowledge health: skipping orphaned-embeddings cleanup - the build lock is held by another instance."
|
|
441
|
+
);
|
|
442
|
+
return;
|
|
443
|
+
}
|
|
444
|
+
try {
|
|
445
|
+
const result = await db.execute(
|
|
446
|
+
"DELETE FROM embeddings WHERE id NOT IN (SELECT id FROM chunks)"
|
|
447
|
+
);
|
|
448
|
+
if (result.rowsAffected > 0) {
|
|
449
|
+
logger.warn(`Knowledge health: deleted ${result.rowsAffected} orphaned embedding row(s).`);
|
|
450
|
+
}
|
|
451
|
+
} finally {
|
|
452
|
+
await buildLock.release(BUILD_LOCK_KEY, instanceId);
|
|
453
|
+
}
|
|
454
|
+
}
|
|
455
|
+
async function scheduleKnowledgeHealthChecks(options) {
|
|
456
|
+
const logger = options.logger ?? createConsoleLogger();
|
|
457
|
+
const enabled = options.config?.enabled !== false;
|
|
458
|
+
const intervalMs = options.config?.intervalMs ?? DEFAULT_KNOWLEDGE_HEALTH_INTERVAL_MS;
|
|
459
|
+
const autoClean = options.config?.autoCleanOrphanedEmbeddings === true;
|
|
460
|
+
const buildLock = options.buildLock ?? createTursoBuildLock(options.db);
|
|
461
|
+
const instanceId = options.instanceId ?? crypto.randomUUID();
|
|
462
|
+
const schedule = options.schedule ?? defaultSchedule;
|
|
463
|
+
async function runOnce() {
|
|
464
|
+
const health = await checkKnowledgeHealth(options.db, logger);
|
|
465
|
+
if (autoClean && health.orphanedEmbeddings > 0) {
|
|
466
|
+
await cleanOrphanedEmbeddings(options.db, buildLock, instanceId, logger);
|
|
467
|
+
}
|
|
468
|
+
return health;
|
|
469
|
+
}
|
|
470
|
+
let handle;
|
|
471
|
+
if (enabled) {
|
|
472
|
+
await runOnce();
|
|
473
|
+
if (intervalMs > 0) {
|
|
474
|
+
let ticking = false;
|
|
475
|
+
handle = schedule(() => {
|
|
476
|
+
if (ticking) return;
|
|
477
|
+
ticking = true;
|
|
478
|
+
runOnce().catch((error) => logger.error("Knowledge health check failed:", error)).finally(() => {
|
|
479
|
+
ticking = false;
|
|
480
|
+
});
|
|
481
|
+
}, intervalMs);
|
|
482
|
+
}
|
|
483
|
+
}
|
|
484
|
+
return {
|
|
485
|
+
runOnce,
|
|
486
|
+
stop: () => handle?.stop()
|
|
487
|
+
};
|
|
488
|
+
}
|
|
489
|
+
var DEFAULT_KNOWLEDGE_HEALTH_INTERVAL_MS, defaultSchedule;
|
|
490
|
+
var init_knowledgeHealth = __esm({
|
|
491
|
+
"src/core/knowledgeHealth.ts"() {
|
|
492
|
+
"use strict";
|
|
493
|
+
init_buildLock();
|
|
494
|
+
init_logger();
|
|
495
|
+
DEFAULT_KNOWLEDGE_HEALTH_INTERVAL_MS = 30 * 6e4;
|
|
496
|
+
defaultSchedule = (fn, ms) => {
|
|
497
|
+
const timer = setInterval(fn, ms);
|
|
498
|
+
if (typeof timer === "object" && "unref" in timer) timer.unref();
|
|
499
|
+
return { stop: () => clearInterval(timer) };
|
|
500
|
+
};
|
|
501
|
+
}
|
|
502
|
+
});
|
|
503
|
+
|
|
391
504
|
// src/mcp-server.ts
|
|
392
505
|
import OpenAI from "openai";
|
|
393
506
|
|
|
@@ -871,6 +984,7 @@ async function createMCPServer(config) {
|
|
|
871
984
|
});
|
|
872
985
|
const model = config.openai.model || DEFAULT_MODEL;
|
|
873
986
|
let store;
|
|
987
|
+
let db;
|
|
874
988
|
if (config.retriever) {
|
|
875
989
|
store = config.retriever;
|
|
876
990
|
} else {
|
|
@@ -878,7 +992,7 @@ async function createMCPServer(config) {
|
|
|
878
992
|
throw new Error(DATABASE_CONFIG_REQUIRED_MESSAGE);
|
|
879
993
|
}
|
|
880
994
|
const { DEFAULT_KNOWLEDGE_DIR: DEFAULT_KNOWLEDGE_DIR2, VectorStore: VectorStore2, createOpenAIEmbedder: createOpenAIEmbedder2, openLibsqlClient: openLibsqlClient2 } = await Promise.resolve().then(() => (init_retrieval(), retrieval_exports));
|
|
881
|
-
|
|
995
|
+
db = await openLibsqlClient2(config.database);
|
|
882
996
|
store = new VectorStore2(createOpenAIEmbedder2(client), {
|
|
883
997
|
databaseClient: db,
|
|
884
998
|
knowledgeDir: config.knowledgeDir || DEFAULT_KNOWLEDGE_DIR2,
|
|
@@ -887,6 +1001,15 @@ async function createMCPServer(config) {
|
|
|
887
1001
|
}
|
|
888
1002
|
await store.build?.();
|
|
889
1003
|
log.info("\u2705 Knowledge base ready");
|
|
1004
|
+
let knowledgeHealth;
|
|
1005
|
+
if (db) {
|
|
1006
|
+
const { scheduleKnowledgeHealthChecks: scheduleKnowledgeHealthChecks2 } = await Promise.resolve().then(() => (init_knowledgeHealth(), knowledgeHealth_exports));
|
|
1007
|
+
knowledgeHealth = await scheduleKnowledgeHealthChecks2({
|
|
1008
|
+
db,
|
|
1009
|
+
config: config.knowledgeHealthCheck,
|
|
1010
|
+
logger: log
|
|
1011
|
+
});
|
|
1012
|
+
}
|
|
890
1013
|
const prompts = new PromptLoader(promptsDir, config.bot);
|
|
891
1014
|
const server = new McpServer({
|
|
892
1015
|
name: config.bot.name,
|
|
@@ -1083,7 +1206,9 @@ async function createMCPServer(config) {
|
|
|
1083
1206
|
log.info(
|
|
1084
1207
|
`\u2705 ${config.bot.name} MCP Server ready (transport: ${transportMode}, tools: ${enabledTools.join(", ") || "none"})`
|
|
1085
1208
|
);
|
|
1086
|
-
return server
|
|
1209
|
+
return Object.assign(server, {
|
|
1210
|
+
stopKnowledgeHealthChecks: () => knowledgeHealth?.stop()
|
|
1211
|
+
});
|
|
1087
1212
|
}
|
|
1088
1213
|
export {
|
|
1089
1214
|
createMCPServer
|
package/dist/server.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"server.d.ts","sourceRoot":"","sources":["../src/server.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAIH,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"server.d.ts","sourceRoot":"","sources":["../src/server.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAIH,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAiB5B,OAAO,KAAK,EAAE,aAAa,EAAsB,MAAM,SAAS,CAAC;AAEjE;;;;;;;;;;;;;;;GAeG;AACH,MAAM,MAAM,UAAU,GAAG,IAAI,GAAG;IAAE,YAAY,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAA;CAAE,CAAC;AAEtE;;;;;GAKG;AACH,wBAAsB,YAAY,CAAC,MAAM,EAAE,aAAa,GAAG,OAAO,CAAC,UAAU,CAAC,CA+P7E"}
|
package/dist/server.js
CHANGED
|
@@ -411,6 +411,119 @@ var init_retrieval = __esm({
|
|
|
411
411
|
}
|
|
412
412
|
});
|
|
413
413
|
|
|
414
|
+
// src/core/knowledgeHealth.ts
|
|
415
|
+
var knowledgeHealth_exports = {};
|
|
416
|
+
__export(knowledgeHealth_exports, {
|
|
417
|
+
DEFAULT_KNOWLEDGE_HEALTH_INTERVAL_MS: () => DEFAULT_KNOWLEDGE_HEALTH_INTERVAL_MS,
|
|
418
|
+
checkKnowledgeHealth: () => checkKnowledgeHealth,
|
|
419
|
+
scheduleKnowledgeHealthChecks: () => scheduleKnowledgeHealthChecks
|
|
420
|
+
});
|
|
421
|
+
async function checkKnowledgeHealth(db, logger) {
|
|
422
|
+
const [byBucket, totalChunksRes, totalEmbeddingsRes, orphanedRes, missingRes] = await Promise.all(
|
|
423
|
+
[
|
|
424
|
+
db.execute("SELECT bucket, COUNT(*) as count FROM chunks GROUP BY bucket"),
|
|
425
|
+
db.execute("SELECT COUNT(*) as count FROM chunks"),
|
|
426
|
+
db.execute("SELECT COUNT(*) as count FROM embeddings"),
|
|
427
|
+
db.execute(
|
|
428
|
+
"SELECT COUNT(*) as count FROM embeddings e LEFT JOIN chunks c ON c.id = e.id WHERE c.id IS NULL"
|
|
429
|
+
),
|
|
430
|
+
db.execute(
|
|
431
|
+
"SELECT COUNT(*) as count FROM chunks c LEFT JOIN embeddings e ON e.id = c.id WHERE e.id IS NULL"
|
|
432
|
+
)
|
|
433
|
+
]
|
|
434
|
+
);
|
|
435
|
+
const chunksByBucket = {};
|
|
436
|
+
for (const row of byBucket.rows) {
|
|
437
|
+
chunksByBucket[String(row.bucket)] = Number(row.count ?? 0);
|
|
438
|
+
}
|
|
439
|
+
const health = {
|
|
440
|
+
chunksByBucket,
|
|
441
|
+
totalChunks: Number(totalChunksRes.rows[0]?.count ?? 0),
|
|
442
|
+
totalEmbeddings: Number(totalEmbeddingsRes.rows[0]?.count ?? 0),
|
|
443
|
+
orphanedEmbeddings: Number(orphanedRes.rows[0]?.count ?? 0),
|
|
444
|
+
missingEmbeddings: Number(missingRes.rows[0]?.count ?? 0)
|
|
445
|
+
};
|
|
446
|
+
const summary = `chunksByBucket=${JSON.stringify(health.chunksByBucket)} totalChunks=${health.totalChunks} totalEmbeddings=${health.totalEmbeddings} orphanedEmbeddings=${health.orphanedEmbeddings} missingEmbeddings=${health.missingEmbeddings}`;
|
|
447
|
+
if (health.totalChunks === 0 || health.orphanedEmbeddings > 0) {
|
|
448
|
+
logger.warn(`Knowledge base health check: ${summary}`);
|
|
449
|
+
} else {
|
|
450
|
+
logger.info(`Knowledge base health check: ${summary}`);
|
|
451
|
+
}
|
|
452
|
+
return health;
|
|
453
|
+
}
|
|
454
|
+
async function cleanOrphanedEmbeddings(db, buildLock, instanceId, logger) {
|
|
455
|
+
const acquired = await buildLock.tryAcquire(
|
|
456
|
+
BUILD_LOCK_KEY,
|
|
457
|
+
instanceId,
|
|
458
|
+
Date.now(),
|
|
459
|
+
BUILD_LOCK_STALE_MS
|
|
460
|
+
);
|
|
461
|
+
if (!acquired) {
|
|
462
|
+
logger.warn(
|
|
463
|
+
"Knowledge health: skipping orphaned-embeddings cleanup - the build lock is held by another instance."
|
|
464
|
+
);
|
|
465
|
+
return;
|
|
466
|
+
}
|
|
467
|
+
try {
|
|
468
|
+
const result = await db.execute(
|
|
469
|
+
"DELETE FROM embeddings WHERE id NOT IN (SELECT id FROM chunks)"
|
|
470
|
+
);
|
|
471
|
+
if (result.rowsAffected > 0) {
|
|
472
|
+
logger.warn(`Knowledge health: deleted ${result.rowsAffected} orphaned embedding row(s).`);
|
|
473
|
+
}
|
|
474
|
+
} finally {
|
|
475
|
+
await buildLock.release(BUILD_LOCK_KEY, instanceId);
|
|
476
|
+
}
|
|
477
|
+
}
|
|
478
|
+
async function scheduleKnowledgeHealthChecks(options) {
|
|
479
|
+
const logger = options.logger ?? createConsoleLogger();
|
|
480
|
+
const enabled = options.config?.enabled !== false;
|
|
481
|
+
const intervalMs = options.config?.intervalMs ?? DEFAULT_KNOWLEDGE_HEALTH_INTERVAL_MS;
|
|
482
|
+
const autoClean = options.config?.autoCleanOrphanedEmbeddings === true;
|
|
483
|
+
const buildLock = options.buildLock ?? createTursoBuildLock(options.db);
|
|
484
|
+
const instanceId = options.instanceId ?? crypto.randomUUID();
|
|
485
|
+
const schedule = options.schedule ?? defaultSchedule;
|
|
486
|
+
async function runOnce() {
|
|
487
|
+
const health = await checkKnowledgeHealth(options.db, logger);
|
|
488
|
+
if (autoClean && health.orphanedEmbeddings > 0) {
|
|
489
|
+
await cleanOrphanedEmbeddings(options.db, buildLock, instanceId, logger);
|
|
490
|
+
}
|
|
491
|
+
return health;
|
|
492
|
+
}
|
|
493
|
+
let handle;
|
|
494
|
+
if (enabled) {
|
|
495
|
+
await runOnce();
|
|
496
|
+
if (intervalMs > 0) {
|
|
497
|
+
let ticking = false;
|
|
498
|
+
handle = schedule(() => {
|
|
499
|
+
if (ticking) return;
|
|
500
|
+
ticking = true;
|
|
501
|
+
runOnce().catch((error) => logger.error("Knowledge health check failed:", error)).finally(() => {
|
|
502
|
+
ticking = false;
|
|
503
|
+
});
|
|
504
|
+
}, intervalMs);
|
|
505
|
+
}
|
|
506
|
+
}
|
|
507
|
+
return {
|
|
508
|
+
runOnce,
|
|
509
|
+
stop: () => handle?.stop()
|
|
510
|
+
};
|
|
511
|
+
}
|
|
512
|
+
var DEFAULT_KNOWLEDGE_HEALTH_INTERVAL_MS, defaultSchedule;
|
|
513
|
+
var init_knowledgeHealth = __esm({
|
|
514
|
+
"src/core/knowledgeHealth.ts"() {
|
|
515
|
+
"use strict";
|
|
516
|
+
init_buildLock();
|
|
517
|
+
init_logger();
|
|
518
|
+
DEFAULT_KNOWLEDGE_HEALTH_INTERVAL_MS = 30 * 6e4;
|
|
519
|
+
defaultSchedule = (fn, ms) => {
|
|
520
|
+
const timer = setInterval(fn, ms);
|
|
521
|
+
if (typeof timer === "object" && "unref" in timer) timer.unref();
|
|
522
|
+
return { stop: () => clearInterval(timer) };
|
|
523
|
+
};
|
|
524
|
+
}
|
|
525
|
+
});
|
|
526
|
+
|
|
414
527
|
// src/server.ts
|
|
415
528
|
var server_exports = {};
|
|
416
529
|
__export(server_exports, {
|
|
@@ -1891,6 +2004,15 @@ async function createServer(config) {
|
|
|
1891
2004
|
}
|
|
1892
2005
|
await store.build?.();
|
|
1893
2006
|
logger.info("\u2705 Knowledge base ready");
|
|
2007
|
+
let knowledgeHealth;
|
|
2008
|
+
if (db && !config.retriever) {
|
|
2009
|
+
const { scheduleKnowledgeHealthChecks: scheduleKnowledgeHealthChecks2 } = await Promise.resolve().then(() => (init_knowledgeHealth(), knowledgeHealth_exports));
|
|
2010
|
+
knowledgeHealth = await scheduleKnowledgeHealthChecks2({
|
|
2011
|
+
db,
|
|
2012
|
+
config: config.knowledgeHealthCheck,
|
|
2013
|
+
logger
|
|
2014
|
+
});
|
|
2015
|
+
}
|
|
1894
2016
|
const prompts = new PromptLoader(promptsDir, config.bot);
|
|
1895
2017
|
let apiKeyManager;
|
|
1896
2018
|
const secret = config.auth?.secret || process.env.CHATTER_SECRET;
|
|
@@ -2021,6 +2143,7 @@ async function createServer(config) {
|
|
|
2021
2143
|
// (sync or async) is caught per-channel so one misbehaving channel can't
|
|
2022
2144
|
// stop the rest from cleaning up.
|
|
2023
2145
|
stopChannels: async () => {
|
|
2146
|
+
knowledgeHealth?.stop();
|
|
2024
2147
|
await Promise.allSettled(
|
|
2025
2148
|
started.map(async (channel) => {
|
|
2026
2149
|
try {
|
package/dist/server.mjs
CHANGED
|
@@ -388,6 +388,119 @@ var init_retrieval = __esm({
|
|
|
388
388
|
}
|
|
389
389
|
});
|
|
390
390
|
|
|
391
|
+
// src/core/knowledgeHealth.ts
|
|
392
|
+
var knowledgeHealth_exports = {};
|
|
393
|
+
__export(knowledgeHealth_exports, {
|
|
394
|
+
DEFAULT_KNOWLEDGE_HEALTH_INTERVAL_MS: () => DEFAULT_KNOWLEDGE_HEALTH_INTERVAL_MS,
|
|
395
|
+
checkKnowledgeHealth: () => checkKnowledgeHealth,
|
|
396
|
+
scheduleKnowledgeHealthChecks: () => scheduleKnowledgeHealthChecks
|
|
397
|
+
});
|
|
398
|
+
async function checkKnowledgeHealth(db, logger) {
|
|
399
|
+
const [byBucket, totalChunksRes, totalEmbeddingsRes, orphanedRes, missingRes] = await Promise.all(
|
|
400
|
+
[
|
|
401
|
+
db.execute("SELECT bucket, COUNT(*) as count FROM chunks GROUP BY bucket"),
|
|
402
|
+
db.execute("SELECT COUNT(*) as count FROM chunks"),
|
|
403
|
+
db.execute("SELECT COUNT(*) as count FROM embeddings"),
|
|
404
|
+
db.execute(
|
|
405
|
+
"SELECT COUNT(*) as count FROM embeddings e LEFT JOIN chunks c ON c.id = e.id WHERE c.id IS NULL"
|
|
406
|
+
),
|
|
407
|
+
db.execute(
|
|
408
|
+
"SELECT COUNT(*) as count FROM chunks c LEFT JOIN embeddings e ON e.id = c.id WHERE e.id IS NULL"
|
|
409
|
+
)
|
|
410
|
+
]
|
|
411
|
+
);
|
|
412
|
+
const chunksByBucket = {};
|
|
413
|
+
for (const row of byBucket.rows) {
|
|
414
|
+
chunksByBucket[String(row.bucket)] = Number(row.count ?? 0);
|
|
415
|
+
}
|
|
416
|
+
const health = {
|
|
417
|
+
chunksByBucket,
|
|
418
|
+
totalChunks: Number(totalChunksRes.rows[0]?.count ?? 0),
|
|
419
|
+
totalEmbeddings: Number(totalEmbeddingsRes.rows[0]?.count ?? 0),
|
|
420
|
+
orphanedEmbeddings: Number(orphanedRes.rows[0]?.count ?? 0),
|
|
421
|
+
missingEmbeddings: Number(missingRes.rows[0]?.count ?? 0)
|
|
422
|
+
};
|
|
423
|
+
const summary = `chunksByBucket=${JSON.stringify(health.chunksByBucket)} totalChunks=${health.totalChunks} totalEmbeddings=${health.totalEmbeddings} orphanedEmbeddings=${health.orphanedEmbeddings} missingEmbeddings=${health.missingEmbeddings}`;
|
|
424
|
+
if (health.totalChunks === 0 || health.orphanedEmbeddings > 0) {
|
|
425
|
+
logger.warn(`Knowledge base health check: ${summary}`);
|
|
426
|
+
} else {
|
|
427
|
+
logger.info(`Knowledge base health check: ${summary}`);
|
|
428
|
+
}
|
|
429
|
+
return health;
|
|
430
|
+
}
|
|
431
|
+
async function cleanOrphanedEmbeddings(db, buildLock, instanceId, logger) {
|
|
432
|
+
const acquired = await buildLock.tryAcquire(
|
|
433
|
+
BUILD_LOCK_KEY,
|
|
434
|
+
instanceId,
|
|
435
|
+
Date.now(),
|
|
436
|
+
BUILD_LOCK_STALE_MS
|
|
437
|
+
);
|
|
438
|
+
if (!acquired) {
|
|
439
|
+
logger.warn(
|
|
440
|
+
"Knowledge health: skipping orphaned-embeddings cleanup - the build lock is held by another instance."
|
|
441
|
+
);
|
|
442
|
+
return;
|
|
443
|
+
}
|
|
444
|
+
try {
|
|
445
|
+
const result = await db.execute(
|
|
446
|
+
"DELETE FROM embeddings WHERE id NOT IN (SELECT id FROM chunks)"
|
|
447
|
+
);
|
|
448
|
+
if (result.rowsAffected > 0) {
|
|
449
|
+
logger.warn(`Knowledge health: deleted ${result.rowsAffected} orphaned embedding row(s).`);
|
|
450
|
+
}
|
|
451
|
+
} finally {
|
|
452
|
+
await buildLock.release(BUILD_LOCK_KEY, instanceId);
|
|
453
|
+
}
|
|
454
|
+
}
|
|
455
|
+
async function scheduleKnowledgeHealthChecks(options) {
|
|
456
|
+
const logger = options.logger ?? createConsoleLogger();
|
|
457
|
+
const enabled = options.config?.enabled !== false;
|
|
458
|
+
const intervalMs = options.config?.intervalMs ?? DEFAULT_KNOWLEDGE_HEALTH_INTERVAL_MS;
|
|
459
|
+
const autoClean = options.config?.autoCleanOrphanedEmbeddings === true;
|
|
460
|
+
const buildLock = options.buildLock ?? createTursoBuildLock(options.db);
|
|
461
|
+
const instanceId = options.instanceId ?? crypto.randomUUID();
|
|
462
|
+
const schedule = options.schedule ?? defaultSchedule;
|
|
463
|
+
async function runOnce() {
|
|
464
|
+
const health = await checkKnowledgeHealth(options.db, logger);
|
|
465
|
+
if (autoClean && health.orphanedEmbeddings > 0) {
|
|
466
|
+
await cleanOrphanedEmbeddings(options.db, buildLock, instanceId, logger);
|
|
467
|
+
}
|
|
468
|
+
return health;
|
|
469
|
+
}
|
|
470
|
+
let handle;
|
|
471
|
+
if (enabled) {
|
|
472
|
+
await runOnce();
|
|
473
|
+
if (intervalMs > 0) {
|
|
474
|
+
let ticking = false;
|
|
475
|
+
handle = schedule(() => {
|
|
476
|
+
if (ticking) return;
|
|
477
|
+
ticking = true;
|
|
478
|
+
runOnce().catch((error) => logger.error("Knowledge health check failed:", error)).finally(() => {
|
|
479
|
+
ticking = false;
|
|
480
|
+
});
|
|
481
|
+
}, intervalMs);
|
|
482
|
+
}
|
|
483
|
+
}
|
|
484
|
+
return {
|
|
485
|
+
runOnce,
|
|
486
|
+
stop: () => handle?.stop()
|
|
487
|
+
};
|
|
488
|
+
}
|
|
489
|
+
var DEFAULT_KNOWLEDGE_HEALTH_INTERVAL_MS, defaultSchedule;
|
|
490
|
+
var init_knowledgeHealth = __esm({
|
|
491
|
+
"src/core/knowledgeHealth.ts"() {
|
|
492
|
+
"use strict";
|
|
493
|
+
init_buildLock();
|
|
494
|
+
init_logger();
|
|
495
|
+
DEFAULT_KNOWLEDGE_HEALTH_INTERVAL_MS = 30 * 6e4;
|
|
496
|
+
defaultSchedule = (fn, ms) => {
|
|
497
|
+
const timer = setInterval(fn, ms);
|
|
498
|
+
if (typeof timer === "object" && "unref" in timer) timer.unref();
|
|
499
|
+
return { stop: () => clearInterval(timer) };
|
|
500
|
+
};
|
|
501
|
+
}
|
|
502
|
+
});
|
|
503
|
+
|
|
391
504
|
// src/server.ts
|
|
392
505
|
import { relative } from "node:path";
|
|
393
506
|
import { Hono as Hono5 } from "hono";
|
|
@@ -1862,6 +1975,15 @@ async function createServer(config) {
|
|
|
1862
1975
|
}
|
|
1863
1976
|
await store.build?.();
|
|
1864
1977
|
logger.info("\u2705 Knowledge base ready");
|
|
1978
|
+
let knowledgeHealth;
|
|
1979
|
+
if (db && !config.retriever) {
|
|
1980
|
+
const { scheduleKnowledgeHealthChecks: scheduleKnowledgeHealthChecks2 } = await Promise.resolve().then(() => (init_knowledgeHealth(), knowledgeHealth_exports));
|
|
1981
|
+
knowledgeHealth = await scheduleKnowledgeHealthChecks2({
|
|
1982
|
+
db,
|
|
1983
|
+
config: config.knowledgeHealthCheck,
|
|
1984
|
+
logger
|
|
1985
|
+
});
|
|
1986
|
+
}
|
|
1865
1987
|
const prompts = new PromptLoader(promptsDir, config.bot);
|
|
1866
1988
|
let apiKeyManager;
|
|
1867
1989
|
const secret = config.auth?.secret || process.env.CHATTER_SECRET;
|
|
@@ -1992,6 +2114,7 @@ async function createServer(config) {
|
|
|
1992
2114
|
// (sync or async) is caught per-channel so one misbehaving channel can't
|
|
1993
2115
|
// stop the rest from cleaning up.
|
|
1994
2116
|
stopChannels: async () => {
|
|
2117
|
+
knowledgeHealth?.stop();
|
|
1995
2118
|
await Promise.allSettled(
|
|
1996
2119
|
started.map(async (channel) => {
|
|
1997
2120
|
try {
|
package/dist/types.d.ts
CHANGED
|
@@ -10,6 +10,7 @@ import type OpenAI from "openai";
|
|
|
10
10
|
import type { Channel, ChannelSenderRegistry } from "./channels";
|
|
11
11
|
import type { AnswerFn, TransformReply } from "./core/answer";
|
|
12
12
|
import type { BucketsFor } from "./core/buckets";
|
|
13
|
+
import type { KnowledgeHealthCheckConfig } from "./core/knowledgeHealth";
|
|
13
14
|
import type { Logger, LogLevel } from "./core/logger";
|
|
14
15
|
import type { FallbackFn, RerankContext, RewriteQuery } from "./core/pipeline";
|
|
15
16
|
import type { PromptLoader } from "./core/prompts";
|
|
@@ -208,6 +209,15 @@ export interface ChatterConfig extends BrainHooks {
|
|
|
208
209
|
* Unset: `database` is required, and the built-in `VectorStore` is used.
|
|
209
210
|
*/
|
|
210
211
|
retriever?: Retriever;
|
|
212
|
+
/**
|
|
213
|
+
* Post-boot and periodic consistency checks on the default `VectorStore`
|
|
214
|
+
* knowledge base (chunk/embedding counts, orphaned and missing rows),
|
|
215
|
+
* logged so a collapse is visible immediately instead of waiting for a
|
|
216
|
+
* user to notice broken answers. Ignored when `retriever` is set - the
|
|
217
|
+
* checks query the `chunks`/`embeddings` schema `VectorStore` itself
|
|
218
|
+
* owns. See [knowledgeHealth.ts](./core/knowledgeHealth.ts).
|
|
219
|
+
*/
|
|
220
|
+
knowledgeHealthCheck?: KnowledgeHealthCheckConfig;
|
|
211
221
|
auth?: {
|
|
212
222
|
/** Secret for JWT API key creation/verification */
|
|
213
223
|
secret?: string;
|
package/dist/types.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,MAAM,IAAI,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC7D,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AACjC,OAAO,KAAK,MAAM,MAAM,QAAQ,CAAC;AACjC,OAAO,KAAK,EAAE,OAAO,EAAE,qBAAqB,EAAE,MAAM,YAAY,CAAC;AACjE,OAAO,KAAK,EAAE,QAAQ,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAC9D,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AACjD,OAAO,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACtD,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC/E,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AACnD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAElD,YAAY,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAEtD;;;GAGG;AACH,MAAM,WAAW,YAAY;IAC3B,WAAW,EAAE,MAAM,CAAC;IACpB,eAAe,EAAE,MAAM,CAAC;CACzB;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,qDAAqD;IACrD,IAAI,EAAE,MAAM,CAAC;IACb,gDAAgD;IAChD,UAAU,EAAE,MAAM,CAAC;IACnB,2CAA2C;IAC3C,SAAS,EAAE,MAAM,CAAC;IAClB,6CAA6C;IAC7C,WAAW,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,2CAA2C;IAC3C,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,4CAA4C;IAC5C,mBAAmB,CAAC,EAAE,MAAM,CAAC;CAC9B;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,mCAAmC;IACnC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,sCAAsC;IACtC,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,oCAAoC;IACpC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,uCAAuC;IACvC,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED;;;;;;;;;;;GAWG;AACH,MAAM,WAAW,UAAU;IACzB;;;;;;;;;OASG;IACH,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB;;;;;;;;;;;;;OAaG;IACH,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB;;;;;;;;;;;;OAYG;IACH,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B;;;;;;;;OAQG;IACH,aAAa,CAAC,EAAE,aAAa,CAAC;IAC9B;;;;;;;;;;;;;;OAcG;IACH,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB;;;;;;;;;;;;;;OAcG;IACH,cAAc,CAAC,EAAE,cAAc,CAAC;CACjC;AAED;;GAEG;AACH,MAAM,WAAW,aAAc,SAAQ,UAAU;IAE/C,GAAG,EAAE,WAAW,CAAC;IACjB,QAAQ,CAAC,EAAE,WAAW,CAAC;IACvB,IAAI,CAAC,EAAE,aAAa,CAAC;IAGrB,+CAA+C;IAC/C,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,4DAA4D;IAC5D,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,mDAAmD;IACnD,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,uDAAuD;IACvD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,8FAA8F;IAC9F,SAAS,CAAC,EAAE,MAAM,CAAC;IAGnB,MAAM,EAAE;QACN,qBAAqB;QACrB,MAAM,EAAE,MAAM,CAAC;QACf,oCAAoC;QACpC,KAAK,CAAC,EAAE,MAAM,CAAC;QACf;;;;;WAKG;QACH,OAAO,CAAC,EAAE,YAAY,CAAC;KACxB,CAAC;IAGF;;;;;;OAMG;IACH,QAAQ,CAAC,EAAE;QACT,yBAAyB;QACzB,GAAG,EAAE,MAAM,CAAC;QACZ,uBAAuB;QACvB,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;IAEF;;;;;;;;;OASG;IACH,SAAS,CAAC,EAAE,SAAS,CAAC;
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,MAAM,IAAI,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC7D,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AACjC,OAAO,KAAK,MAAM,MAAM,QAAQ,CAAC;AACjC,OAAO,KAAK,EAAE,OAAO,EAAE,qBAAqB,EAAE,MAAM,YAAY,CAAC;AACjE,OAAO,KAAK,EAAE,QAAQ,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAC9D,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AACjD,OAAO,KAAK,EAAE,0BAA0B,EAAE,MAAM,wBAAwB,CAAC;AACzE,OAAO,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACtD,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC/E,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AACnD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAElD,YAAY,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAEtD;;;GAGG;AACH,MAAM,WAAW,YAAY;IAC3B,WAAW,EAAE,MAAM,CAAC;IACpB,eAAe,EAAE,MAAM,CAAC;CACzB;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,qDAAqD;IACrD,IAAI,EAAE,MAAM,CAAC;IACb,gDAAgD;IAChD,UAAU,EAAE,MAAM,CAAC;IACnB,2CAA2C;IAC3C,SAAS,EAAE,MAAM,CAAC;IAClB,6CAA6C;IAC7C,WAAW,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,2CAA2C;IAC3C,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,4CAA4C;IAC5C,mBAAmB,CAAC,EAAE,MAAM,CAAC;CAC9B;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,mCAAmC;IACnC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,sCAAsC;IACtC,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,oCAAoC;IACpC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,uCAAuC;IACvC,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED;;;;;;;;;;;GAWG;AACH,MAAM,WAAW,UAAU;IACzB;;;;;;;;;OASG;IACH,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB;;;;;;;;;;;;;OAaG;IACH,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB;;;;;;;;;;;;OAYG;IACH,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B;;;;;;;;OAQG;IACH,aAAa,CAAC,EAAE,aAAa,CAAC;IAC9B;;;;;;;;;;;;;;OAcG;IACH,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB;;;;;;;;;;;;;;OAcG;IACH,cAAc,CAAC,EAAE,cAAc,CAAC;CACjC;AAED;;GAEG;AACH,MAAM,WAAW,aAAc,SAAQ,UAAU;IAE/C,GAAG,EAAE,WAAW,CAAC;IACjB,QAAQ,CAAC,EAAE,WAAW,CAAC;IACvB,IAAI,CAAC,EAAE,aAAa,CAAC;IAGrB,+CAA+C;IAC/C,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,4DAA4D;IAC5D,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,mDAAmD;IACnD,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,uDAAuD;IACvD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,8FAA8F;IAC9F,SAAS,CAAC,EAAE,MAAM,CAAC;IAGnB,MAAM,EAAE;QACN,qBAAqB;QACrB,MAAM,EAAE,MAAM,CAAC;QACf,oCAAoC;QACpC,KAAK,CAAC,EAAE,MAAM,CAAC;QACf;;;;;WAKG;QACH,OAAO,CAAC,EAAE,YAAY,CAAC;KACxB,CAAC;IAGF;;;;;;OAMG;IACH,QAAQ,CAAC,EAAE;QACT,yBAAyB;QACzB,GAAG,EAAE,MAAM,CAAC;QACZ,uBAAuB;QACvB,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;IAEF;;;;;;;;;OASG;IACH,SAAS,CAAC,EAAE,SAAS,CAAC;IAEtB;;;;;;;OAOG;IACH,oBAAoB,CAAC,EAAE,0BAA0B,CAAC;IAGlD,IAAI,CAAC,EAAE;QACL,mDAAmD;QACnD,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,qEAAqE;QACrE,aAAa,CAAC,EAAE;YACd,MAAM,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC;gBAAE,KAAK,EAAE,OAAO,CAAC;gBAAC,OAAO,CAAC,EAAE,OAAO,CAAC;gBAAC,KAAK,CAAC,EAAE,MAAM,CAAA;aAAE,CAAC,CAAC;SAC3F,CAAC;QACF,6CAA6C;QAC7C,GAAG,CAAC,EAAE;YACJ,oCAAoC;YACpC,OAAO,CAAC,EAAE,MAAM,CAAC;YACjB,0BAA0B;YAC1B,MAAM,CAAC,EAAE,MAAM,CAAC;YAChB,4BAA4B;YAC5B,QAAQ,CAAC,EAAE,MAAM,CAAC;YAClB,2CAA2C;YAC3C,YAAY,CAAC,EAAE,MAAM,CAAC;SACvB,CAAC;QACF,0BAA0B;QAC1B,KAAK,CAAC,EAAE;YACN,4BAA4B;YAC5B,cAAc,CAAC,EAAE,MAAM,CAAC;YACxB,6BAA6B;YAC7B,WAAW,CAAC,EAAE,MAAM,CAAC;SACtB,CAAC;KACH,CAAC;IAGF,QAAQ,CAAC,EAAE;QACT,iDAAiD;QACjD,gBAAgB,CAAC,EAAE,OAAO,CAAC;QAC3B,kDAAkD;QAClD,iBAAiB,CAAC,EAAE,OAAO,CAAC;QAC5B,mDAAmD;QACnD,gBAAgB,CAAC,EAAE,OAAO,CAAC;QAC3B;;;;WAIG;QACH,kBAAkB,CAAC,EAAE,OAAO,CAAC;QAC7B;;;;WAIG;QACH,QAAQ,CAAC,EAAE,OAAO,CAAC;KACpB,CAAC;IAGF,SAAS,CAAC,EAAE;QACV,2DAA2D;QAC3D,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,6DAA6D;QAC7D,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB;;;;;;;;;WASG;QACH,UAAU,CAAC,EAAE,OAAO,CAAC;QACrB;;;;WAIG;QACH,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;KACxB,CAAC;IAGF,MAAM,CAAC,EAAE;QACP,iCAAiC;QACjC,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,iCAAiC;QACjC,IAAI,CAAC,EAAE,OAAO,CAAC;QACf;;;;;;;WAOG;QACH,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;QAC1B;;;;;;;;WAQG;QACH,kBAAkB,CAAC,EAAE,OAAO,CAAC;QAC7B;;;;;;;;WAQG;QACH,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB;;;;;;;;;;;;WAYG;QACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;QAC/B;;;;;;WAMG;QACH,uBAAuB,CAAC,EAAE,MAAM,GAAG,KAAK,CAAC;KAC1C,CAAC;IAGF,2FAA2F;IAC3F,YAAY,CAAC,EAAE,YAAY,CAAC;IAG5B;;;;;;;OAOG;IACH,QAAQ,CAAC,EAAE,OAAO,EAAE,CAAC;IAGrB;;;;;;;OAOG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,qGAAqG;IACrG,QAAQ,CAAC,EAAE,QAAQ,CAAC;CACrB;AAED;;;;;;;;;;;;;;GAcG;AACH,MAAM,MAAM,YAAY,GACpB,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,IAAI,EAAE,kBAAkB,KAAK,IAAI,CAAC,GAC/C,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,IAAI,EAAE,kBAAkB,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC;AAE7D;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,6BAA6B;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,+GAA+G;IAC/G,KAAK,EAAE,SAAS,CAAC;IACjB;;;;;;;;;;OAUG;IACH,EAAE,EAAE,YAAY,CAAC;IACjB,4BAA4B;IAC5B,MAAM,EAAE,aAAa,CAAC;IACtB,6BAA6B;IAC7B,OAAO,EAAE,YAAY,CAAC;IACtB,+CAA+C;IAC/C,aAAa,CAAC,EAAE;QACd,MAAM,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC;YAAE,KAAK,EAAE,OAAO,CAAC;YAAC,OAAO,CAAC,EAAE,OAAO,CAAC;YAAC,KAAK,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC,CAAC;KAC3F,CAAC;IACF;;;;;OAKG;IACH,OAAO,EAAE,qBAAqB,CAAC;IAC/B,wEAAwE;IACxE,MAAM,EAAE,MAAM,CAAC;CAChB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,gBAAgB;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,uBAAuB;IACvB,OAAO,EAAE,MAAM,CAAC;IAChB,wBAAwB;IACxB,QAAQ,EAAE;QACR,uBAAuB;QACvB,MAAM,EAAE,MAAM,CAAC;QACf,2CAA2C;QAC3C,IAAI,CAAC,EAAE,QAAQ,GAAG,SAAS,GAAG,MAAM,CAAC;QACrC,8BAA8B;QAC9B,YAAY,CAAC,EAAE,MAAM,CAAC;KACvB,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,oBAAoB;IACpB,OAAO,EAAE,MAAM,CAAC;IAChB,uBAAuB;IACvB,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,qBAAqB;IACrB,QAAQ,EAAE;QACR,uBAAuB;QACvB,MAAM,EAAE,MAAM,CAAC;QACf,qBAAqB;QACrB,IAAI,CAAC,EAAE,QAAQ,GAAG,SAAS,GAAG,MAAM,CAAC;QACrC,8BAA8B;QAC9B,WAAW,EAAE,MAAM,CAAC;KACrB,CAAC;CACH"}
|
package/package.json
CHANGED