@vdpeijl/kb-mcp 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +416 -0
- package/dist/scripts/postinstall.d.ts +3 -0
- package/dist/scripts/postinstall.d.ts.map +1 -0
- package/dist/scripts/postinstall.js +110 -0
- package/dist/scripts/postinstall.js.map +1 -0
- package/dist/src/cli.d.ts +3 -0
- package/dist/src/cli.d.ts.map +1 -0
- package/dist/src/cli.js +22 -0
- package/dist/src/cli.js.map +1 -0
- package/dist/src/commands/doctor.d.ts +3 -0
- package/dist/src/commands/doctor.d.ts.map +1 -0
- package/dist/src/commands/doctor.js +93 -0
- package/dist/src/commands/doctor.js.map +1 -0
- package/dist/src/commands/init.d.ts +3 -0
- package/dist/src/commands/init.d.ts.map +1 -0
- package/dist/src/commands/init.js +150 -0
- package/dist/src/commands/init.js.map +1 -0
- package/dist/src/commands/serve.d.ts +3 -0
- package/dist/src/commands/serve.d.ts.map +1 -0
- package/dist/src/commands/serve.js +22 -0
- package/dist/src/commands/serve.js.map +1 -0
- package/dist/src/commands/setup.d.ts +9 -0
- package/dist/src/commands/setup.d.ts.map +1 -0
- package/dist/src/commands/setup.js +115 -0
- package/dist/src/commands/setup.js.map +1 -0
- package/dist/src/commands/sources.d.ts +3 -0
- package/dist/src/commands/sources.d.ts.map +1 -0
- package/dist/src/commands/sources.js +258 -0
- package/dist/src/commands/sources.js.map +1 -0
- package/dist/src/commands/stats.d.ts +3 -0
- package/dist/src/commands/stats.d.ts.map +1 -0
- package/dist/src/commands/stats.js +48 -0
- package/dist/src/commands/stats.js.map +1 -0
- package/dist/src/commands/sync.d.ts +13 -0
- package/dist/src/commands/sync.d.ts.map +1 -0
- package/dist/src/commands/sync.js +106 -0
- package/dist/src/commands/sync.js.map +1 -0
- package/dist/src/config/index.d.ts +18 -0
- package/dist/src/config/index.d.ts.map +1 -0
- package/dist/src/config/index.js +67 -0
- package/dist/src/config/index.js.map +1 -0
- package/dist/src/config/paths.d.ts +21 -0
- package/dist/src/config/paths.d.ts.map +1 -0
- package/dist/src/config/paths.js +38 -0
- package/dist/src/config/paths.js.map +1 -0
- package/dist/src/config/schema.d.ts +133 -0
- package/dist/src/config/schema.d.ts.map +1 -0
- package/dist/src/config/schema.js +34 -0
- package/dist/src/config/schema.js.map +1 -0
- package/dist/src/db/articles.d.ts +39 -0
- package/dist/src/db/articles.d.ts.map +1 -0
- package/dist/src/db/articles.js +103 -0
- package/dist/src/db/articles.js.map +1 -0
- package/dist/src/db/chunks.d.ts +46 -0
- package/dist/src/db/chunks.d.ts.map +1 -0
- package/dist/src/db/chunks.js +129 -0
- package/dist/src/db/chunks.js.map +1 -0
- package/dist/src/db/index.d.ts +19 -0
- package/dist/src/db/index.d.ts.map +1 -0
- package/dist/src/db/index.js +86 -0
- package/dist/src/db/index.js.map +1 -0
- package/dist/src/db/schema.d.ts +10 -0
- package/dist/src/db/schema.d.ts.map +1 -0
- package/dist/src/db/schema.js +80 -0
- package/dist/src/db/schema.js.map +1 -0
- package/dist/src/db/sources.d.ts +48 -0
- package/dist/src/db/sources.d.ts.map +1 -0
- package/dist/src/db/sources.js +110 -0
- package/dist/src/db/sources.js.map +1 -0
- package/dist/src/index.d.ts +8 -0
- package/dist/src/index.d.ts.map +1 -0
- package/dist/src/index.js +87 -0
- package/dist/src/index.js.map +1 -0
- package/dist/src/search/embeddings.d.ts +22 -0
- package/dist/src/search/embeddings.d.ts.map +1 -0
- package/dist/src/search/embeddings.js +122 -0
- package/dist/src/search/embeddings.js.map +1 -0
- package/dist/src/search/index.d.ts +21 -0
- package/dist/src/search/index.d.ts.map +1 -0
- package/dist/src/search/index.js +50 -0
- package/dist/src/search/index.js.map +1 -0
- package/dist/src/sync/chunker.d.ts +15 -0
- package/dist/src/sync/chunker.d.ts.map +1 -0
- package/dist/src/sync/chunker.js +117 -0
- package/dist/src/sync/chunker.js.map +1 -0
- package/dist/src/sync/index.d.ts +24 -0
- package/dist/src/sync/index.d.ts.map +1 -0
- package/dist/src/sync/index.js +180 -0
- package/dist/src/sync/index.js.map +1 -0
- package/dist/src/sync/parser.d.ts +9 -0
- package/dist/src/sync/parser.d.ts.map +1 -0
- package/dist/src/sync/parser.js +91 -0
- package/dist/src/sync/parser.js.map +1 -0
- package/dist/src/sync/zendesk.d.ts +45 -0
- package/dist/src/sync/zendesk.d.ts.map +1 -0
- package/dist/src/sync/zendesk.js +161 -0
- package/dist/src/sync/zendesk.js.map +1 -0
- package/dist/src/utils/errors.d.ts +39 -0
- package/dist/src/utils/errors.d.ts.map +1 -0
- package/dist/src/utils/errors.js +62 -0
- package/dist/src/utils/errors.js.map +1 -0
- package/dist/src/utils/logger.d.ts +21 -0
- package/dist/src/utils/logger.d.ts.map +1 -0
- package/dist/src/utils/logger.js +25 -0
- package/dist/src/utils/logger.js.map +1 -0
- package/package.json +57 -0
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Insert or update an article
|
|
3
|
+
*/
|
|
4
|
+
export function upsertArticle(db, article) {
|
|
5
|
+
const stmt = db.prepare(`
|
|
6
|
+
INSERT INTO articles (id, source_id, title, url, section_name, category_name, updated_at, synced_at)
|
|
7
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
8
|
+
ON CONFLICT(id, source_id) DO UPDATE SET
|
|
9
|
+
title = excluded.title,
|
|
10
|
+
url = excluded.url,
|
|
11
|
+
section_name = excluded.section_name,
|
|
12
|
+
category_name = excluded.category_name,
|
|
13
|
+
updated_at = excluded.updated_at,
|
|
14
|
+
synced_at = excluded.synced_at
|
|
15
|
+
`);
|
|
16
|
+
stmt.run(article.id, article.sourceId, article.title, article.url, article.sectionName || null, article.categoryName || null, article.updatedAt.toISOString(), new Date().toISOString());
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* Get an article by ID and source
|
|
20
|
+
*/
|
|
21
|
+
export function getArticle(db, id, sourceId) {
|
|
22
|
+
const stmt = db.prepare('SELECT * FROM articles WHERE id = ? AND source_id = ?');
|
|
23
|
+
const row = stmt.get(id, sourceId);
|
|
24
|
+
if (!row) {
|
|
25
|
+
return null;
|
|
26
|
+
}
|
|
27
|
+
return {
|
|
28
|
+
id: row.id,
|
|
29
|
+
sourceId: row.source_id,
|
|
30
|
+
title: row.title,
|
|
31
|
+
url: row.url,
|
|
32
|
+
sectionName: row.section_name || undefined,
|
|
33
|
+
categoryName: row.category_name || undefined,
|
|
34
|
+
updatedAt: new Date(row.updated_at),
|
|
35
|
+
};
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* Get all articles for a source
|
|
39
|
+
*/
|
|
40
|
+
export function getArticlesBySource(db, sourceId) {
|
|
41
|
+
const stmt = db.prepare('SELECT * FROM articles WHERE source_id = ? ORDER BY updated_at DESC');
|
|
42
|
+
const rows = stmt.all(sourceId);
|
|
43
|
+
return rows.map(row => ({
|
|
44
|
+
id: row.id,
|
|
45
|
+
sourceId: row.source_id,
|
|
46
|
+
title: row.title,
|
|
47
|
+
url: row.url,
|
|
48
|
+
sectionName: row.section_name || undefined,
|
|
49
|
+
categoryName: row.category_name || undefined,
|
|
50
|
+
updatedAt: new Date(row.updated_at),
|
|
51
|
+
}));
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* Get articles that have been updated since last sync
|
|
55
|
+
* Returns articles from Zendesk that are newer than what we have
|
|
56
|
+
*/
|
|
57
|
+
export function getStaleArticleIds(db, sourceId, freshArticles) {
|
|
58
|
+
const staleIds = new Set();
|
|
59
|
+
// Get all current article IDs and their update times
|
|
60
|
+
const stmt = db.prepare('SELECT id, updated_at FROM articles WHERE source_id = ?');
|
|
61
|
+
const existingArticles = stmt.all(sourceId);
|
|
62
|
+
const existingMap = new Map(existingArticles.map(a => [a.id, new Date(a.updated_at)]));
|
|
63
|
+
// Check each fresh article
|
|
64
|
+
for (const fresh of freshArticles) {
|
|
65
|
+
const existing = existingMap.get(fresh.id);
|
|
66
|
+
const freshDate = new Date(fresh.updated_at);
|
|
67
|
+
// If article doesn't exist or has been updated, it's stale
|
|
68
|
+
if (!existing || freshDate > existing) {
|
|
69
|
+
staleIds.add(fresh.id);
|
|
70
|
+
}
|
|
71
|
+
// Remove from map so we can find deleted articles
|
|
72
|
+
existingMap.delete(fresh.id);
|
|
73
|
+
}
|
|
74
|
+
// Any remaining articles in the map have been deleted from Zendesk
|
|
75
|
+
// Add them to stale list so they get removed
|
|
76
|
+
for (const [id] of existingMap) {
|
|
77
|
+
staleIds.add(id);
|
|
78
|
+
}
|
|
79
|
+
return staleIds;
|
|
80
|
+
}
|
|
81
|
+
/**
|
|
82
|
+
* Delete an article and its chunks
|
|
83
|
+
*/
|
|
84
|
+
export function deleteArticle(db, id, sourceId) {
|
|
85
|
+
// Foreign key constraints will cascade delete chunks
|
|
86
|
+
const stmt = db.prepare('DELETE FROM articles WHERE id = ? AND source_id = ?');
|
|
87
|
+
stmt.run(id, sourceId);
|
|
88
|
+
}
|
|
89
|
+
/**
|
|
90
|
+
* Delete articles that no longer exist in Zendesk
|
|
91
|
+
*/
|
|
92
|
+
export function deleteOrphanedArticles(db, sourceId, validArticleIds) {
|
|
93
|
+
if (validArticleIds.length === 0) {
|
|
94
|
+
// Delete all articles for this source
|
|
95
|
+
db.prepare('DELETE FROM articles WHERE source_id = ?').run(sourceId);
|
|
96
|
+
return;
|
|
97
|
+
}
|
|
98
|
+
// Delete articles not in the valid list
|
|
99
|
+
const placeholders = validArticleIds.map(() => '?').join(',');
|
|
100
|
+
const stmt = db.prepare(`DELETE FROM articles WHERE source_id = ? AND id NOT IN (${placeholders})`);
|
|
101
|
+
stmt.run(sourceId, ...validArticleIds);
|
|
102
|
+
}
|
|
103
|
+
//# sourceMappingURL=articles.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"articles.js","sourceRoot":"","sources":["../../../src/db/articles.ts"],"names":[],"mappings":"AAuBA;;GAEG;AACH,MAAM,UAAU,aAAa,CAAC,EAAqB,EAAE,OAAgB;IACnE,MAAM,IAAI,GAAG,EAAE,CAAC,OAAO,CAAC;;;;;;;;;;GAUvB,CAAC,CAAC;IAEH,IAAI,CAAC,GAAG,CACN,OAAO,CAAC,EAAE,EACV,OAAO,CAAC,QAAQ,EAChB,OAAO,CAAC,KAAK,EACb,OAAO,CAAC,GAAG,EACX,OAAO,CAAC,WAAW,IAAI,IAAI,EAC3B,OAAO,CAAC,YAAY,IAAI,IAAI,EAC5B,OAAO,CAAC,SAAS,CAAC,WAAW,EAAE,EAC/B,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CACzB,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,UAAU,CAAC,EAAqB,EAAE,EAAU,EAAE,QAAgB;IAC5E,MAAM,IAAI,GAAG,EAAE,CAAC,OAAO,CAAC,uDAAuD,CAAC,CAAC;IACjF,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,QAAQ,CAA2B,CAAC;IAE7D,IAAI,CAAC,GAAG,EAAE,CAAC;QACT,OAAO,IAAI,CAAC;IACd,CAAC;IAED,OAAO;QACL,EAAE,EAAE,GAAG,CAAC,EAAE;QACV,QAAQ,EAAE,GAAG,CAAC,SAAS;QACvB,KAAK,EAAE,GAAG,CAAC,KAAK;QAChB,GAAG,EAAE,GAAG,CAAC,GAAG;QACZ,WAAW,EAAE,GAAG,CAAC,YAAY,IAAI,SAAS;QAC1C,YAAY,EAAE,GAAG,CAAC,aAAa,IAAI,SAAS;QAC5C,SAAS,EAAE,IAAI,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC;KACpC,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,mBAAmB,CAAC,EAAqB,EAAE,QAAgB;IACzE,MAAM,IAAI,GAAG,EAAE,CAAC,OAAO,CAAC,qEAAqE,CAAC,CAAC;IAC/F,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAiB,CAAC;IAEhD,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACtB,EAAE,EAAE,GAAG,CAAC,EAAE;QACV,QAAQ,EAAE,GAAG,CAAC,SAAS;QACvB,KAAK,EAAE,GAAG,CAAC,KAAK;QAChB,GAAG,EAAE,GAAG,CAAC,GAAG;QACZ,WAAW,EAAE,GAAG,CAAC,YAAY,IAAI,SAAS;QAC1C,YAAY,EAAE,GAAG,CAAC,aAAa,IAAI,SAAS;QAC5C,SAAS,EAAE,IAAI,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC;KACpC,CAAC,CAAC,CAAC;AACN,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,kBAAkB,CAChC,EAAqB,EACrB,QAAgB,EAChB,aAAwD;IAExD,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAC;IAEnC,qDAAqD;IACrD,MAAM,IAAI,GAAG,EAAE,CAAC,OAAO,CAAC,yDAAyD,CAAC,CAAC;IACnF,MAAM,gBAAgB,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,CAA8C,CAAC;IAEzF,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,IAAI,IAAI,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC;IAEvF,2BAA2B;IAC3B,KAAK,MAAM,KAAK,IAAI,aAAa,EAAE,CAAC;QAClC,MAAM,QAAQ,GAAG,WAAW,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAC3C,MAAM,SAAS,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;QAE7C,2DAA2D;QAC3D,IAAI,CAAC,QAAQ,IAAI,SAAS,GAAG,QAAQ,EAAE,CAAC;YACtC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QACzB,CAAC;QAED,kDAAkD;QAClD,WAAW,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IAC/B,CAAC;IAED,mEAAmE;IACnE,6CAA6C;IAC7C,KAAK,MAAM,CAAC,EAAE,CAAC,IAAI,WAAW,EAAE,CAAC;QAC/B,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IACnB,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,aAAa,CAAC,EAAqB,EAAE,EAAU,EAAE,QAAgB;IAC/E,qDAAqD;IACrD,MAAM,IAAI,GAAG,EAAE,CAAC,OAAO,CAAC,qDAAqD,CAAC,CAAC;IAC/E,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC;AACzB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,sBAAsB,CACpC,EAAqB,EACrB,QAAgB,EAChB,eAAyB;IAEzB,IAAI,eAAe,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACjC,sCAAsC;QACtC,EAAE,CAAC,OAAO,CAAC,0CAA0C,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QACrE,OAAO;IACT,CAAC;IAED,wCAAwC;IACxC,MAAM,YAAY,GAAG,eAAe,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC9D,MAAM,IAAI,GAAG,EAAE,CAAC,OAAO,CACrB,2DAA2D,YAAY,GAAG,CAC3E,CAAC;IAEF,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,GAAG,eAAe,CAAC,CAAC;AACzC,CAAC"}
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import type Database from 'better-sqlite3';
|
|
2
|
+
export interface Chunk {
|
|
3
|
+
id?: number;
|
|
4
|
+
articleId: number;
|
|
5
|
+
sourceId: string;
|
|
6
|
+
chunkIndex: number;
|
|
7
|
+
text: string;
|
|
8
|
+
tokenCount: number;
|
|
9
|
+
}
|
|
10
|
+
export interface ChunkWithEmbedding extends Chunk {
|
|
11
|
+
embedding: Float32Array;
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* Insert a chunk with its embedding
|
|
15
|
+
*/
|
|
16
|
+
export declare function insertChunk(db: Database.Database, chunk: ChunkWithEmbedding): number;
|
|
17
|
+
/**
|
|
18
|
+
* Insert multiple chunks in a transaction
|
|
19
|
+
*/
|
|
20
|
+
export declare function insertChunksBatch(db: Database.Database, chunks: ChunkWithEmbedding[]): void;
|
|
21
|
+
/**
|
|
22
|
+
* Get chunks for an article
|
|
23
|
+
*/
|
|
24
|
+
export declare function getChunksByArticle(db: Database.Database, articleId: number, sourceId: string): Chunk[];
|
|
25
|
+
/**
|
|
26
|
+
* Delete all chunks for an article
|
|
27
|
+
*/
|
|
28
|
+
export declare function deleteChunksByArticle(db: Database.Database, articleId: number, sourceId: string): void;
|
|
29
|
+
/**
|
|
30
|
+
* Delete all chunks for a source
|
|
31
|
+
*/
|
|
32
|
+
export declare function deleteChunksBySource(db: Database.Database, sourceId: string): void;
|
|
33
|
+
export interface SearchResult {
|
|
34
|
+
chunkId: number;
|
|
35
|
+
articleId: number;
|
|
36
|
+
sourceId: string;
|
|
37
|
+
title: string;
|
|
38
|
+
url: string;
|
|
39
|
+
text: string;
|
|
40
|
+
distance: number;
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* Search for similar chunks using vector similarity
|
|
44
|
+
*/
|
|
45
|
+
export declare function searchSimilarChunks(db: Database.Database, embedding: Float32Array, limit?: number, sourceIds?: string[]): SearchResult[];
|
|
46
|
+
//# sourceMappingURL=chunks.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"chunks.d.ts","sourceRoot":"","sources":["../../../src/db/chunks.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,QAAQ,MAAM,gBAAgB,CAAC;AAE3C,MAAM,WAAW,KAAK;IACpB,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,kBAAmB,SAAQ,KAAK;IAC/C,SAAS,EAAE,YAAY,CAAC;CACzB;AAWD;;GAEG;AACH,wBAAgB,WAAW,CAAC,EAAE,EAAE,QAAQ,CAAC,QAAQ,EAAE,KAAK,EAAE,kBAAkB,GAAG,MAAM,CA+BpF;AAED;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,EAAE,EAAE,QAAQ,CAAC,QAAQ,EAAE,MAAM,EAAE,kBAAkB,EAAE,GAAG,IAAI,CAQ3F;AAED;;GAEG;AACH,wBAAgB,kBAAkB,CAAC,EAAE,EAAE,QAAQ,CAAC,QAAQ,EAAE,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,KAAK,EAAE,CAiBtG;AAED;;GAEG;AACH,wBAAgB,qBAAqB,CAAC,EAAE,EAAE,QAAQ,CAAC,QAAQ,EAAE,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,IAAI,CAetG;AAED;;GAEG;AACH,wBAAgB,oBAAoB,CAAC,EAAE,EAAE,QAAQ,CAAC,QAAQ,EAAE,QAAQ,EAAE,MAAM,GAAG,IAAI,CAclF;AAED,MAAM,WAAW,YAAY;IAC3B,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED;;GAEG;AACH,wBAAgB,mBAAmB,CACjC,EAAE,EAAE,QAAQ,CAAC,QAAQ,EACrB,SAAS,EAAE,YAAY,EACvB,KAAK,GAAE,MAAU,EACjB,SAAS,CAAC,EAAE,MAAM,EAAE,GACnB,YAAY,EAAE,CAmDhB"}
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Insert a chunk with its embedding
|
|
3
|
+
*/
|
|
4
|
+
export function insertChunk(db, chunk) {
|
|
5
|
+
// Insert chunk
|
|
6
|
+
const insertChunk = db.prepare(`
|
|
7
|
+
INSERT INTO chunks (article_id, source_id, chunk_index, text, token_count)
|
|
8
|
+
VALUES (?, ?, ?, ?, ?)
|
|
9
|
+
`);
|
|
10
|
+
const result = insertChunk.run(chunk.articleId, chunk.sourceId, chunk.chunkIndex, chunk.text, chunk.tokenCount);
|
|
11
|
+
// Explicitly convert to integer (lastInsertRowid can be bigint)
|
|
12
|
+
const chunkId = Math.floor(Number(result.lastInsertRowid));
|
|
13
|
+
// Insert embedding into vector table
|
|
14
|
+
// vec0 uses rowid as the primary key
|
|
15
|
+
// Use CAST to ensure rowid is treated as INTEGER
|
|
16
|
+
const insertVec = db.prepare(`
|
|
17
|
+
INSERT INTO chunks_vec (rowid, embedding)
|
|
18
|
+
VALUES (CAST(? AS INTEGER), ?)
|
|
19
|
+
`);
|
|
20
|
+
// Convert Float32Array to buffer for sqlite-vec
|
|
21
|
+
const embeddingBuffer = Buffer.from(chunk.embedding.buffer);
|
|
22
|
+
insertVec.run(chunkId, embeddingBuffer);
|
|
23
|
+
return chunkId;
|
|
24
|
+
}
|
|
25
|
+
/**
|
|
26
|
+
* Insert multiple chunks in a transaction
|
|
27
|
+
*/
|
|
28
|
+
export function insertChunksBatch(db, chunks) {
|
|
29
|
+
const transaction = db.transaction((chunks) => {
|
|
30
|
+
for (const chunk of chunks) {
|
|
31
|
+
insertChunk(db, chunk);
|
|
32
|
+
}
|
|
33
|
+
});
|
|
34
|
+
transaction(chunks);
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* Get chunks for an article
|
|
38
|
+
*/
|
|
39
|
+
export function getChunksByArticle(db, articleId, sourceId) {
|
|
40
|
+
const stmt = db.prepare(`
|
|
41
|
+
SELECT * FROM chunks
|
|
42
|
+
WHERE article_id = ? AND source_id = ?
|
|
43
|
+
ORDER BY chunk_index
|
|
44
|
+
`);
|
|
45
|
+
const rows = stmt.all(articleId, sourceId);
|
|
46
|
+
return rows.map(row => ({
|
|
47
|
+
id: row.id,
|
|
48
|
+
articleId: row.article_id,
|
|
49
|
+
sourceId: row.source_id,
|
|
50
|
+
chunkIndex: row.chunk_index,
|
|
51
|
+
text: row.text,
|
|
52
|
+
tokenCount: row.token_count,
|
|
53
|
+
}));
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* Delete all chunks for an article
|
|
57
|
+
*/
|
|
58
|
+
export function deleteChunksByArticle(db, articleId, sourceId) {
|
|
59
|
+
// Get chunk IDs to delete from vector table
|
|
60
|
+
const chunkIds = db.prepare('SELECT id FROM chunks WHERE article_id = ? AND source_id = ?')
|
|
61
|
+
.all(articleId, sourceId);
|
|
62
|
+
// Delete from vector table
|
|
63
|
+
if (chunkIds.length > 0) {
|
|
64
|
+
const placeholders = chunkIds.map(() => '?').join(',');
|
|
65
|
+
db.prepare(`DELETE FROM chunks_vec WHERE rowid IN (${placeholders})`)
|
|
66
|
+
.run(...chunkIds.map(c => c.id));
|
|
67
|
+
}
|
|
68
|
+
// Delete from chunks table
|
|
69
|
+
db.prepare('DELETE FROM chunks WHERE article_id = ? AND source_id = ?')
|
|
70
|
+
.run(articleId, sourceId);
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* Delete all chunks for a source
|
|
74
|
+
*/
|
|
75
|
+
export function deleteChunksBySource(db, sourceId) {
|
|
76
|
+
// Get chunk IDs to delete from vector table
|
|
77
|
+
const chunkIds = db.prepare('SELECT id FROM chunks WHERE source_id = ?')
|
|
78
|
+
.all(sourceId);
|
|
79
|
+
// Delete from vector table
|
|
80
|
+
if (chunkIds.length > 0) {
|
|
81
|
+
const placeholders = chunkIds.map(() => '?').join(',');
|
|
82
|
+
db.prepare(`DELETE FROM chunks_vec WHERE rowid IN (${placeholders})`)
|
|
83
|
+
.run(...chunkIds.map(c => c.id));
|
|
84
|
+
}
|
|
85
|
+
// Delete from chunks table
|
|
86
|
+
db.prepare('DELETE FROM chunks WHERE source_id = ?').run(sourceId);
|
|
87
|
+
}
|
|
88
|
+
/**
|
|
89
|
+
* Search for similar chunks using vector similarity
|
|
90
|
+
*/
|
|
91
|
+
export function searchSimilarChunks(db, embedding, limit = 5, sourceIds) {
|
|
92
|
+
const embeddingBuffer = Buffer.from(embedding.buffer);
|
|
93
|
+
let query = `
|
|
94
|
+
SELECT
|
|
95
|
+
cv.rowid as chunk_id,
|
|
96
|
+
c.article_id,
|
|
97
|
+
c.source_id,
|
|
98
|
+
c.text,
|
|
99
|
+
a.title,
|
|
100
|
+
a.url,
|
|
101
|
+
cv.distance
|
|
102
|
+
FROM chunks_vec cv
|
|
103
|
+
INNER JOIN chunks c ON cv.rowid = c.id
|
|
104
|
+
INNER JOIN articles a ON c.article_id = a.id AND c.source_id = a.source_id
|
|
105
|
+
WHERE cv.embedding MATCH ? AND k = ?
|
|
106
|
+
`;
|
|
107
|
+
const params = [embeddingBuffer, limit];
|
|
108
|
+
// Add source filter if provided
|
|
109
|
+
if (sourceIds && sourceIds.length > 0) {
|
|
110
|
+
const placeholders = sourceIds.map(() => '?').join(',');
|
|
111
|
+
query += ` AND c.source_id IN (${placeholders})`;
|
|
112
|
+
params.push(...sourceIds);
|
|
113
|
+
}
|
|
114
|
+
query += `
|
|
115
|
+
ORDER BY cv.distance
|
|
116
|
+
`;
|
|
117
|
+
const stmt = db.prepare(query);
|
|
118
|
+
const rows = stmt.all(...params);
|
|
119
|
+
return rows.map(row => ({
|
|
120
|
+
chunkId: row.chunk_id,
|
|
121
|
+
articleId: row.article_id,
|
|
122
|
+
sourceId: row.source_id,
|
|
123
|
+
title: row.title,
|
|
124
|
+
url: row.url,
|
|
125
|
+
text: row.text,
|
|
126
|
+
distance: row.distance,
|
|
127
|
+
}));
|
|
128
|
+
}
|
|
129
|
+
//# sourceMappingURL=chunks.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"chunks.js","sourceRoot":"","sources":["../../../src/db/chunks.ts"],"names":[],"mappings":"AAwBA;;GAEG;AACH,MAAM,UAAU,WAAW,CAAC,EAAqB,EAAE,KAAyB;IAC1E,eAAe;IACf,MAAM,WAAW,GAAG,EAAE,CAAC,OAAO,CAAC;;;GAG9B,CAAC,CAAC;IAEH,MAAM,MAAM,GAAG,WAAW,CAAC,GAAG,CAC5B,KAAK,CAAC,SAAS,EACf,KAAK,CAAC,QAAQ,EACd,KAAK,CAAC,UAAU,EAChB,KAAK,CAAC,IAAI,EACV,KAAK,CAAC,UAAU,CACjB,CAAC;IAEF,gEAAgE;IAChE,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC,CAAC;IAE3D,qCAAqC;IACrC,qCAAqC;IACrC,iDAAiD;IACjD,MAAM,SAAS,GAAG,EAAE,CAAC,OAAO,CAAC;;;GAG5B,CAAC,CAAC;IAEH,gDAAgD;IAChD,MAAM,eAAe,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;IAC5D,SAAS,CAAC,GAAG,CAAC,OAAO,EAAE,eAAe,CAAC,CAAC;IAExC,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,iBAAiB,CAAC,EAAqB,EAAE,MAA4B;IACnF,MAAM,WAAW,GAAG,EAAE,CAAC,WAAW,CAAC,CAAC,MAA4B,EAAE,EAAE;QAClE,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,WAAW,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC;QACzB,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,WAAW,CAAC,MAAM,CAAC,CAAC;AACtB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,kBAAkB,CAAC,EAAqB,EAAE,SAAiB,EAAE,QAAgB;IAC3F,MAAM,IAAI,GAAG,EAAE,CAAC,OAAO,CAAC;;;;GAIvB,CAAC,CAAC;IAEH,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,QAAQ,CAAe,CAAC;IAEzD,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACtB,EAAE,EAAE,GAAG,CAAC,EAAE;QACV,SAAS,EAAE,GAAG,CAAC,UAAU;QACzB,QAAQ,EAAE,GAAG,CAAC,SAAS;QACvB,UAAU,EAAE,GAAG,CAAC,WAAW;QAC3B,IAAI,EAAE,GAAG,CAAC,IAAI;QACd,UAAU,EAAE,GAAG,CAAC,WAAW;KAC5B,CAAC,CAAC,CAAC;AACN,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,qBAAqB,CAAC,EAAqB,EAAE,SAAiB,EAAE,QAAgB;IAC9F,4CAA4C;IAC5C,MAAM,QAAQ,GAAG,EAAE,CAAC,OAAO,CAAC,8DAA8D,CAAC;SACxF,GAAG,CAAC,SAAS,EAAE,QAAQ,CAA0B,CAAC;IAErD,2BAA2B;IAC3B,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxB,MAAM,YAAY,GAAG,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACvD,EAAE,CAAC,OAAO,CAAC,0CAA0C,YAAY,GAAG,CAAC;aAClE,GAAG,CAAC,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACrC,CAAC;IAED,2BAA2B;IAC3B,EAAE,CAAC,OAAO,CAAC,2DAA2D,CAAC;SACpE,GAAG,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;AAC9B,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,oBAAoB,CAAC,EAAqB,EAAE,QAAgB;IAC1E,4CAA4C;IAC5C,MAAM,QAAQ,GAAG,EAAE,CAAC,OAAO,CAAC,2CAA2C,CAAC;SACrE,GAAG,CAAC,QAAQ,CAA0B,CAAC;IAE1C,2BAA2B;IAC3B,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxB,MAAM,YAAY,GAAG,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACvD,EAAE,CAAC,OAAO,CAAC,0CAA0C,YAAY,GAAG,CAAC;aAClE,GAAG,CAAC,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACrC,CAAC;IAED,2BAA2B;IAC3B,EAAE,CAAC,OAAO,CAAC,wCAAwC,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;AACrE,CAAC;AAYD;;GAEG;AACH,MAAM,UAAU,mBAAmB,CACjC,EAAqB,EACrB,SAAuB,EACvB,QAAgB,CAAC,EACjB,SAAoB;IAEpB,MAAM,eAAe,GAAG,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;IAEtD,IAAI,KAAK,GAAG;;;;;;;;;;;;;GAaX,CAAC;IAEF,MAAM,MAAM,GAAU,CAAC,eAAe,EAAE,KAAK,CAAC,CAAC;IAE/C,gCAAgC;IAChC,IAAI,SAAS,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtC,MAAM,YAAY,GAAG,SAAS,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACxD,KAAK,IAAI,wBAAwB,YAAY,GAAG,CAAC;QACjD,MAAM,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,CAAC;IAC5B,CAAC;IAED,KAAK,IAAI;;GAER,CAAC;IAEF,MAAM,IAAI,GAAG,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IAC/B,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,MAAM,CAQ7B,CAAC;IAEH,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACtB,OAAO,EAAE,GAAG,CAAC,QAAQ;QACrB,SAAS,EAAE,GAAG,CAAC,UAAU;QACzB,QAAQ,EAAE,GAAG,CAAC,SAAS;QACvB,KAAK,EAAE,GAAG,CAAC,KAAK;QAChB,GAAG,EAAE,GAAG,CAAC,GAAG;QACZ,IAAI,EAAE,GAAG,CAAC,IAAI;QACd,QAAQ,EAAE,GAAG,CAAC,QAAQ;KACvB,CAAC,CAAC,CAAC;AACN,CAAC"}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import Database from 'better-sqlite3';
|
|
2
|
+
/**
|
|
3
|
+
* Initialize and return a database connection
|
|
4
|
+
*/
|
|
5
|
+
export declare function getDatabase(): Promise<Database.Database>;
|
|
6
|
+
/**
|
|
7
|
+
* Close the database connection
|
|
8
|
+
*/
|
|
9
|
+
export declare function closeDatabase(): void;
|
|
10
|
+
/**
|
|
11
|
+
* Get database statistics
|
|
12
|
+
*/
|
|
13
|
+
export declare function getDatabaseStats(): Promise<{
|
|
14
|
+
sources: number;
|
|
15
|
+
articles: number;
|
|
16
|
+
chunks: number;
|
|
17
|
+
databaseSize: number;
|
|
18
|
+
}>;
|
|
19
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/db/index.ts"],"names":[],"mappings":"AAAA,OAAO,QAAQ,MAAM,gBAAgB,CAAC;AAqBtC;;GAEG;AACH,wBAAsB,WAAW,IAAI,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CA+C9D;AAED;;GAEG;AACH,wBAAgB,aAAa,IAAI,IAAI,CAKpC;AAED;;GAEG;AACH,wBAAsB,gBAAgB;;;;;GAkBrC"}
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
import Database from 'better-sqlite3';
|
|
2
|
+
import { join, dirname } from 'node:path';
|
|
3
|
+
import { fileURLToPath } from 'node:url';
|
|
4
|
+
import { createSchema } from './schema.js';
|
|
5
|
+
import { getXDGPaths, ensureDirectories } from '../config/paths.js';
|
|
6
|
+
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
7
|
+
let dbInstance = null;
|
|
8
|
+
/**
|
|
9
|
+
* Get the path to the sqlite-vec extension for the current platform
|
|
10
|
+
*/
|
|
11
|
+
function getSqliteVecPath() {
|
|
12
|
+
// The extracted file is always named vec0.{dylib|so|dll}
|
|
13
|
+
// Extension is in the native/ directory at the project root
|
|
14
|
+
// Note: better-sqlite3 automatically adds the platform extension (.dylib, .so, .dll)
|
|
15
|
+
return join(__dirname, '..', '..', 'native', 'vec0');
|
|
16
|
+
}
|
|
17
|
+
/**
|
|
18
|
+
* Initialize and return a database connection
|
|
19
|
+
*/
|
|
20
|
+
export async function getDatabase() {
|
|
21
|
+
if (dbInstance) {
|
|
22
|
+
return dbInstance;
|
|
23
|
+
}
|
|
24
|
+
const paths = await ensureDirectories();
|
|
25
|
+
const dbPath = paths.database;
|
|
26
|
+
// Create database connection
|
|
27
|
+
dbInstance = new Database(dbPath);
|
|
28
|
+
try {
|
|
29
|
+
// Load sqlite-vec extension
|
|
30
|
+
const vecPath = getSqliteVecPath();
|
|
31
|
+
dbInstance.loadExtension(vecPath);
|
|
32
|
+
// Verify the extension loaded
|
|
33
|
+
const versionQuery = dbInstance.prepare('SELECT vec_version() as version');
|
|
34
|
+
const result = versionQuery.get();
|
|
35
|
+
if (!result) {
|
|
36
|
+
throw new Error('sqlite-vec extension loaded but version query failed');
|
|
37
|
+
}
|
|
38
|
+
// Create schema if needed
|
|
39
|
+
createSchema(dbInstance);
|
|
40
|
+
return dbInstance;
|
|
41
|
+
}
|
|
42
|
+
catch (error) {
|
|
43
|
+
// Clean up on error
|
|
44
|
+
dbInstance.close();
|
|
45
|
+
dbInstance = null;
|
|
46
|
+
if (error instanceof Error) {
|
|
47
|
+
if (error.message.includes('cannot open shared object file')) {
|
|
48
|
+
throw new Error(`Failed to load sqlite-vec extension.\n\n` +
|
|
49
|
+
`Try reinstalling: npm install -g @vdpeijl/kb-mcp --force\n\n` +
|
|
50
|
+
`Or manually download from:\n` +
|
|
51
|
+
`https://github.com/asg017/sqlite-vec/releases`);
|
|
52
|
+
}
|
|
53
|
+
throw error;
|
|
54
|
+
}
|
|
55
|
+
throw new Error('Unknown error initializing database');
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
/**
|
|
59
|
+
* Close the database connection
|
|
60
|
+
*/
|
|
61
|
+
export function closeDatabase() {
|
|
62
|
+
if (dbInstance) {
|
|
63
|
+
dbInstance.close();
|
|
64
|
+
dbInstance = null;
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
/**
|
|
68
|
+
* Get database statistics
|
|
69
|
+
*/
|
|
70
|
+
export async function getDatabaseStats() {
|
|
71
|
+
const db = await getDatabase();
|
|
72
|
+
const paths = getXDGPaths();
|
|
73
|
+
const sourcesCount = db.prepare('SELECT COUNT(*) as count FROM sources').get();
|
|
74
|
+
const articlesCount = db.prepare('SELECT COUNT(*) as count FROM articles').get();
|
|
75
|
+
const chunksCount = db.prepare('SELECT COUNT(*) as count FROM chunks').get();
|
|
76
|
+
// Get database file size
|
|
77
|
+
const fs = await import('node:fs/promises');
|
|
78
|
+
const stats = await fs.stat(paths.database);
|
|
79
|
+
return {
|
|
80
|
+
sources: sourcesCount.count,
|
|
81
|
+
articles: articlesCount.count,
|
|
82
|
+
chunks: chunksCount.count,
|
|
83
|
+
databaseSize: stats.size,
|
|
84
|
+
};
|
|
85
|
+
}
|
|
86
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/db/index.ts"],"names":[],"mappings":"AAAA,OAAO,QAAQ,MAAM,gBAAgB,CAAC;AACtC,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAEzC,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EAAE,WAAW,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAEpE,MAAM,SAAS,GAAG,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AAE1D,IAAI,UAAU,GAA6B,IAAI,CAAC;AAEhD;;GAEG;AACH,SAAS,gBAAgB;IACvB,yDAAyD;IACzD,4DAA4D;IAC5D,qFAAqF;IACrF,OAAO,IAAI,CAAC,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC;AACvD,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW;IAC/B,IAAI,UAAU,EAAE,CAAC;QACf,OAAO,UAAU,CAAC;IACpB,CAAC;IAED,MAAM,KAAK,GAAG,MAAM,iBAAiB,EAAE,CAAC;IACxC,MAAM,MAAM,GAAG,KAAK,CAAC,QAAQ,CAAC;IAE9B,6BAA6B;IAC7B,UAAU,GAAG,IAAI,QAAQ,CAAC,MAAM,CAAC,CAAC;IAElC,IAAI,CAAC;QACH,4BAA4B;QAC5B,MAAM,OAAO,GAAG,gBAAgB,EAAE,CAAC;QACnC,UAAU,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;QAElC,8BAA8B;QAC9B,MAAM,YAAY,GAAG,UAAU,CAAC,OAAO,CAAC,iCAAiC,CAAC,CAAC;QAC3E,MAAM,MAAM,GAAG,YAAY,CAAC,GAAG,EAAqC,CAAC;QAErE,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CAAC,sDAAsD,CAAC,CAAC;QAC1E,CAAC;QAED,0BAA0B;QAC1B,YAAY,CAAC,UAAU,CAAC,CAAC;QAEzB,OAAO,UAAU,CAAC;IACpB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,oBAAoB;QACpB,UAAU,CAAC,KAAK,EAAE,CAAC;QACnB,UAAU,GAAG,IAAI,CAAC;QAElB,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;YAC3B,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,gCAAgC,CAAC,EAAE,CAAC;gBAC7D,MAAM,IAAI,KAAK,CACb,0CAA0C;oBAC1C,8DAA8D;oBAC9D,8BAA8B;oBAC9B,+CAA+C,CAChD,CAAC;YACJ,CAAC;YACD,MAAM,KAAK,CAAC;QACd,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;IACzD,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,aAAa;IAC3B,IAAI,UAAU,EAAE,CAAC;QACf,UAAU,CAAC,KAAK,EAAE,CAAC;QACnB,UAAU,GAAG,IAAI,CAAC;IACpB,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB;IACpC,MAAM,EAAE,GAAG,MAAM,WAAW,EAAE,CAAC;IAC/B,MAAM,KAAK,GAAG,WAAW,EAAE,CAAC;IAE5B,MAAM,YAAY,GAAG,EAAE,CAAC,OAAO,CAAC,uCAAuC,CAAC,CAAC,GAAG,EAAuB,CAAC;IACpG,MAAM,aAAa,GAAG,EAAE,CAAC,OAAO,CAAC,wCAAwC,CAAC,CAAC,GAAG,EAAuB,CAAC;IACtG,MAAM,WAAW,GAAG,EAAE,CAAC,OAAO,CAAC,sCAAsC,CAAC,CAAC,GAAG,EAAuB,CAAC;IAElG,yBAAyB;IACzB,MAAM,EAAE,GAAG,MAAM,MAAM,CAAC,kBAAkB,CAAC,CAAC;IAC5C,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IAE5C,OAAO;QACL,OAAO,EAAE,YAAY,CAAC,KAAK;QAC3B,QAAQ,EAAE,aAAa,CAAC,KAAK;QAC7B,MAAM,EAAE,WAAW,CAAC,KAAK;QACzB,YAAY,EAAE,KAAK,CAAC,IAAI;KACzB,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type Database from 'better-sqlite3';
|
|
2
|
+
/**
|
|
3
|
+
* Create all database tables and indexes
|
|
4
|
+
*/
|
|
5
|
+
export declare function createSchema(db: Database.Database): void;
|
|
6
|
+
/**
|
|
7
|
+
* Drop all tables (for testing or reset)
|
|
8
|
+
*/
|
|
9
|
+
export declare function dropSchema(db: Database.Database): void;
|
|
10
|
+
//# sourceMappingURL=schema.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../../../src/db/schema.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,QAAQ,MAAM,gBAAgB,CAAC;AAE3C;;GAEG;AACH,wBAAgB,YAAY,CAAC,EAAE,EAAE,QAAQ,CAAC,QAAQ,GAAG,IAAI,CA2ExD;AAED;;GAEG;AACH,wBAAgB,UAAU,CAAC,EAAE,EAAE,QAAQ,CAAC,QAAQ,GAAG,IAAI,CAKtD"}
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Create all database tables and indexes
|
|
3
|
+
*/
|
|
4
|
+
export function createSchema(db) {
|
|
5
|
+
// Enable foreign keys
|
|
6
|
+
db.pragma('foreign_keys = ON');
|
|
7
|
+
// Set WAL mode for better concurrency
|
|
8
|
+
db.pragma('journal_mode = WAL');
|
|
9
|
+
// Set busy timeout (in milliseconds)
|
|
10
|
+
db.pragma('busy_timeout = 5000');
|
|
11
|
+
// Create sources table
|
|
12
|
+
db.exec(`
|
|
13
|
+
CREATE TABLE IF NOT EXISTS sources (
|
|
14
|
+
id TEXT PRIMARY KEY,
|
|
15
|
+
name TEXT NOT NULL,
|
|
16
|
+
base_url TEXT NOT NULL,
|
|
17
|
+
locale TEXT NOT NULL,
|
|
18
|
+
last_synced_at TEXT,
|
|
19
|
+
enabled INTEGER DEFAULT 1
|
|
20
|
+
)
|
|
21
|
+
`);
|
|
22
|
+
// Create articles table
|
|
23
|
+
db.exec(`
|
|
24
|
+
CREATE TABLE IF NOT EXISTS articles (
|
|
25
|
+
id INTEGER NOT NULL,
|
|
26
|
+
source_id TEXT NOT NULL,
|
|
27
|
+
title TEXT NOT NULL,
|
|
28
|
+
url TEXT NOT NULL,
|
|
29
|
+
section_name TEXT,
|
|
30
|
+
category_name TEXT,
|
|
31
|
+
updated_at TEXT NOT NULL,
|
|
32
|
+
synced_at TEXT NOT NULL,
|
|
33
|
+
PRIMARY KEY (id, source_id),
|
|
34
|
+
FOREIGN KEY (source_id) REFERENCES sources(id) ON DELETE CASCADE
|
|
35
|
+
)
|
|
36
|
+
`);
|
|
37
|
+
// Create chunks table
|
|
38
|
+
db.exec(`
|
|
39
|
+
CREATE TABLE IF NOT EXISTS chunks (
|
|
40
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
41
|
+
article_id INTEGER NOT NULL,
|
|
42
|
+
source_id TEXT NOT NULL,
|
|
43
|
+
chunk_index INTEGER NOT NULL,
|
|
44
|
+
text TEXT NOT NULL,
|
|
45
|
+
token_count INTEGER,
|
|
46
|
+
FOREIGN KEY (article_id, source_id) REFERENCES articles(id, source_id) ON DELETE CASCADE
|
|
47
|
+
)
|
|
48
|
+
`);
|
|
49
|
+
// Create indexes for better query performance
|
|
50
|
+
db.exec(`
|
|
51
|
+
CREATE INDEX IF NOT EXISTS idx_chunks_source
|
|
52
|
+
ON chunks(source_id)
|
|
53
|
+
`);
|
|
54
|
+
db.exec(`
|
|
55
|
+
CREATE INDEX IF NOT EXISTS idx_chunks_article
|
|
56
|
+
ON chunks(article_id, source_id)
|
|
57
|
+
`);
|
|
58
|
+
db.exec(`
|
|
59
|
+
CREATE INDEX IF NOT EXISTS idx_articles_updated
|
|
60
|
+
ON articles(source_id, updated_at)
|
|
61
|
+
`);
|
|
62
|
+
// Create vector table for embeddings
|
|
63
|
+
// This uses sqlite-vec's vec0 virtual table
|
|
64
|
+
// Note: vec0 uses rowid as the implicit primary key
|
|
65
|
+
db.exec(`
|
|
66
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS chunks_vec USING vec0(
|
|
67
|
+
embedding float[768]
|
|
68
|
+
)
|
|
69
|
+
`);
|
|
70
|
+
}
|
|
71
|
+
/**
|
|
72
|
+
* Drop all tables (for testing or reset)
|
|
73
|
+
*/
|
|
74
|
+
export function dropSchema(db) {
|
|
75
|
+
db.exec('DROP TABLE IF EXISTS chunks_vec');
|
|
76
|
+
db.exec('DROP TABLE IF EXISTS chunks');
|
|
77
|
+
db.exec('DROP TABLE IF EXISTS articles');
|
|
78
|
+
db.exec('DROP TABLE IF EXISTS sources');
|
|
79
|
+
}
|
|
80
|
+
//# sourceMappingURL=schema.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"schema.js","sourceRoot":"","sources":["../../../src/db/schema.ts"],"names":[],"mappings":"AAEA;;GAEG;AACH,MAAM,UAAU,YAAY,CAAC,EAAqB;IAChD,sBAAsB;IACtB,EAAE,CAAC,MAAM,CAAC,mBAAmB,CAAC,CAAC;IAE/B,sCAAsC;IACtC,EAAE,CAAC,MAAM,CAAC,oBAAoB,CAAC,CAAC;IAEhC,qCAAqC;IACrC,EAAE,CAAC,MAAM,CAAC,qBAAqB,CAAC,CAAC;IAEjC,uBAAuB;IACvB,EAAE,CAAC,IAAI,CAAC;;;;;;;;;GASP,CAAC,CAAC;IAEH,wBAAwB;IACxB,EAAE,CAAC,IAAI,CAAC;;;;;;;;;;;;;GAaP,CAAC,CAAC;IAEH,sBAAsB;IACtB,EAAE,CAAC,IAAI,CAAC;;;;;;;;;;GAUP,CAAC,CAAC;IAEH,8CAA8C;IAC9C,EAAE,CAAC,IAAI,CAAC;;;GAGP,CAAC,CAAC;IAEH,EAAE,CAAC,IAAI,CAAC;;;GAGP,CAAC,CAAC;IAEH,EAAE,CAAC,IAAI,CAAC;;;GAGP,CAAC,CAAC;IAEH,qCAAqC;IACrC,4CAA4C;IAC5C,oDAAoD;IACpD,EAAE,CAAC,IAAI,CAAC;;;;GAIP,CAAC,CAAC;AACL,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,UAAU,CAAC,EAAqB;IAC9C,EAAE,CAAC,IAAI,CAAC,iCAAiC,CAAC,CAAC;IAC3C,EAAE,CAAC,IAAI,CAAC,6BAA6B,CAAC,CAAC;IACvC,EAAE,CAAC,IAAI,CAAC,+BAA+B,CAAC,CAAC;IACzC,EAAE,CAAC,IAAI,CAAC,8BAA8B,CAAC,CAAC;AAC1C,CAAC"}
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
import type Database from 'better-sqlite3';
|
|
2
|
+
import type { Source } from '../config/schema.js';
|
|
3
|
+
export interface SourceRow {
|
|
4
|
+
id: string;
|
|
5
|
+
name: string;
|
|
6
|
+
base_url: string;
|
|
7
|
+
locale: string;
|
|
8
|
+
last_synced_at: string | null;
|
|
9
|
+
enabled: number;
|
|
10
|
+
}
|
|
11
|
+
export interface SourceWithStats extends Source {
|
|
12
|
+
lastSyncedAt: Date | null;
|
|
13
|
+
articleCount: number;
|
|
14
|
+
chunkCount: number;
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* Insert or update a source
|
|
18
|
+
*/
|
|
19
|
+
export declare function upsertSource(db: Database.Database, source: Source): void;
|
|
20
|
+
/**
|
|
21
|
+
* Get a source by ID
|
|
22
|
+
*/
|
|
23
|
+
export declare function getSource(db: Database.Database, id: string): Source | null;
|
|
24
|
+
/**
|
|
25
|
+
* Get all sources
|
|
26
|
+
*/
|
|
27
|
+
export declare function getAllSources(db: Database.Database): Source[];
|
|
28
|
+
/**
|
|
29
|
+
* Get enabled sources only
|
|
30
|
+
*/
|
|
31
|
+
export declare function getEnabledSources(db: Database.Database): Source[];
|
|
32
|
+
/**
|
|
33
|
+
* Get sources with statistics
|
|
34
|
+
*/
|
|
35
|
+
export declare function getSourcesWithStats(db: Database.Database): SourceWithStats[];
|
|
36
|
+
/**
|
|
37
|
+
* Delete a source and all its data
|
|
38
|
+
*/
|
|
39
|
+
export declare function deleteSource(db: Database.Database, id: string): void;
|
|
40
|
+
/**
|
|
41
|
+
* Update source enabled status
|
|
42
|
+
*/
|
|
43
|
+
export declare function setSourceEnabled(db: Database.Database, id: string, enabled: boolean): void;
|
|
44
|
+
/**
|
|
45
|
+
* Update last synced timestamp
|
|
46
|
+
*/
|
|
47
|
+
export declare function updateLastSynced(db: Database.Database, id: string): void;
|
|
48
|
+
//# sourceMappingURL=sources.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"sources.d.ts","sourceRoot":"","sources":["../../../src/db/sources.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,QAAQ,MAAM,gBAAgB,CAAC;AAC3C,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAElD,MAAM,WAAW,SAAS;IACxB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,eAAgB,SAAQ,MAAM;IAC7C,YAAY,EAAE,IAAI,GAAG,IAAI,CAAC;IAC1B,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,wBAAgB,YAAY,CAAC,EAAE,EAAE,QAAQ,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI,CAkBxE;AAED;;GAEG;AACH,wBAAgB,SAAS,CAAC,EAAE,EAAE,QAAQ,CAAC,QAAQ,EAAE,EAAE,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAe1E;AAED;;GAEG;AACH,wBAAgB,aAAa,CAAC,EAAE,EAAE,QAAQ,CAAC,QAAQ,GAAG,MAAM,EAAE,CAW7D;AAED;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,EAAE,EAAE,QAAQ,CAAC,QAAQ,GAAG,MAAM,EAAE,CAWjE;AAED;;GAEG;AACH,wBAAgB,mBAAmB,CAAC,EAAE,EAAE,QAAQ,CAAC,QAAQ,GAAG,eAAe,EAAE,CAyB5E;AAED;;GAEG;AACH,wBAAgB,YAAY,CAAC,EAAE,EAAE,QAAQ,CAAC,QAAQ,EAAE,EAAE,EAAE,MAAM,GAAG,IAAI,CAIpE;AAED;;GAEG;AACH,wBAAgB,gBAAgB,CAAC,EAAE,EAAE,QAAQ,CAAC,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,GAAG,IAAI,CAG1F;AAED;;GAEG;AACH,wBAAgB,gBAAgB,CAAC,EAAE,EAAE,QAAQ,CAAC,QAAQ,EAAE,EAAE,EAAE,MAAM,GAAG,IAAI,CAGxE"}
|