@knowledgine/core 0.4.1 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/embedding/onnx-embedding-provider.d.ts +1 -0
- package/dist/embedding/onnx-embedding-provider.d.ts.map +1 -1
- package/dist/embedding/onnx-embedding-provider.js +9 -3
- package/dist/embedding/onnx-embedding-provider.js.map +1 -1
- package/dist/graph/entity-extractor.d.ts +11 -3
- package/dist/graph/entity-extractor.d.ts.map +1 -1
- package/dist/graph/entity-extractor.js +191 -4
- package/dist/graph/entity-extractor.js.map +1 -1
- package/dist/graph/graph-repository.d.ts +1 -0
- package/dist/graph/graph-repository.d.ts.map +1 -1
- package/dist/graph/graph-repository.js +7 -0
- package/dist/graph/graph-repository.js.map +1 -1
- package/dist/index.d.ts +5 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +7 -0
- package/dist/index.js.map +1 -1
- package/dist/search/hybrid-searcher.d.ts.map +1 -1
- package/dist/search/hybrid-searcher.js +5 -1
- package/dist/search/hybrid-searcher.js.map +1 -1
- package/dist/search/knowledge-searcher.d.ts +2 -2
- package/dist/search/knowledge-searcher.d.ts.map +1 -1
- package/dist/search/knowledge-searcher.js +33 -15
- package/dist/search/knowledge-searcher.js.map +1 -1
- package/dist/search/link-generator.d.ts +1 -0
- package/dist/search/link-generator.d.ts.map +1 -1
- package/dist/search/link-generator.js +44 -11
- package/dist/search/link-generator.js.map +1 -1
- package/dist/search/query-orchestrator.d.ts +2 -2
- package/dist/search/query-orchestrator.d.ts.map +1 -1
- package/dist/search/query-orchestrator.js +43 -23
- package/dist/search/query-orchestrator.js.map +1 -1
- package/dist/search/reasoning-reranker.d.ts +4 -4
- package/dist/search/reasoning-reranker.d.ts.map +1 -1
- package/dist/search/reasoning-reranker.js +9 -5
- package/dist/search/reasoning-reranker.js.map +1 -1
- package/dist/search/semantic-searcher.d.ts +2 -2
- package/dist/search/semantic-searcher.d.ts.map +1 -1
- package/dist/search/semantic-searcher.js +5 -1
- package/dist/search/semantic-searcher.js.map +1 -1
- package/dist/services/knowledge-service.d.ts +1 -0
- package/dist/services/knowledge-service.d.ts.map +1 -1
- package/dist/services/knowledge-service.js +12 -1
- package/dist/services/knowledge-service.js.map +1 -1
- package/dist/storage/database.d.ts.map +1 -1
- package/dist/storage/database.js +5 -0
- package/dist/storage/database.js.map +1 -1
- package/dist/storage/knowledge-repository.d.ts +42 -0
- package/dist/storage/knowledge-repository.d.ts.map +1 -1
- package/dist/storage/knowledge-repository.js +278 -145
- package/dist/storage/knowledge-repository.js.map +1 -1
- package/dist/storage/migrations/011_fts_unicode61.d.ts +10 -0
- package/dist/storage/migrations/011_fts_unicode61.d.ts.map +1 -0
- package/dist/storage/migrations/011_fts_unicode61.js +88 -0
- package/dist/storage/migrations/011_fts_unicode61.js.map +1 -0
- package/dist/storage/migrations/012_fts_trigram_cjk.d.ts +10 -0
- package/dist/storage/migrations/012_fts_trigram_cjk.d.ts.map +1 -0
- package/dist/storage/migrations/012_fts_trigram_cjk.js +53 -0
- package/dist/storage/migrations/012_fts_trigram_cjk.js.map +1 -0
- package/dist/storage/schema.d.ts +1 -1
- package/dist/storage/schema.d.ts.map +1 -1
- package/dist/storage/schema.js +1 -1
- package/dist/utils/semantic-readiness.d.ts +13 -0
- package/dist/utils/semantic-readiness.d.ts.map +1 -0
- package/dist/utils/semantic-readiness.js +21 -0
- package/dist/utils/semantic-readiness.js.map +1 -0
- package/package.json +1 -1
|
@@ -1,10 +1,25 @@
|
|
|
1
1
|
import { ValidationError, DatabaseError, KnowledgeNotFoundError } from "../errors.js";
|
|
2
2
|
import { createHash } from "crypto";
|
|
3
|
+
const SUMMARY_COLUMNS = `id, file_path, title, created_at, updated_at, content_hash,
|
|
4
|
+
version, supersedes, valid_from, deprecated, deprecation_reason,
|
|
5
|
+
extracted_at, code_location_json`;
|
|
3
6
|
export class KnowledgeRepository {
|
|
4
7
|
db;
|
|
8
|
+
_stmtCache = new Map();
|
|
5
9
|
constructor(db) {
|
|
6
10
|
this.db = db;
|
|
7
11
|
}
|
|
12
|
+
stmt(sql) {
|
|
13
|
+
let s = this._stmtCache.get(sql);
|
|
14
|
+
if (!s) {
|
|
15
|
+
s = this.db.prepare(sql);
|
|
16
|
+
this._stmtCache.set(sql, s);
|
|
17
|
+
}
|
|
18
|
+
return s;
|
|
19
|
+
}
|
|
20
|
+
clearStatementCache() {
|
|
21
|
+
this._stmtCache.clear();
|
|
22
|
+
}
|
|
8
23
|
computeHash(content) {
|
|
9
24
|
return createHash("sha256").update(content).digest("hex");
|
|
10
25
|
}
|
|
@@ -34,23 +49,21 @@ export class KnowledgeRepository {
|
|
|
34
49
|
if (existing.content_hash === contentHash) {
|
|
35
50
|
return existing.id;
|
|
36
51
|
}
|
|
37
|
-
|
|
52
|
+
this.stmt(`
|
|
38
53
|
UPDATE knowledge_notes
|
|
39
54
|
SET title = ?, content = ?, frontmatter_json = ?,
|
|
40
55
|
updated_at = ?, content_hash = ?, code_location_json = ?
|
|
41
56
|
WHERE id = ?
|
|
42
|
-
`);
|
|
43
|
-
stmt.run(data.title, data.content, frontmatterJson, now, contentHash, codeLocationJson, existing.id);
|
|
57
|
+
`).run(data.title, data.content, frontmatterJson, now, contentHash, codeLocationJson, existing.id);
|
|
44
58
|
return existing.id;
|
|
45
59
|
}
|
|
46
60
|
else {
|
|
47
|
-
const
|
|
61
|
+
const info = this.stmt(`
|
|
48
62
|
INSERT INTO knowledge_notes (
|
|
49
63
|
file_path, title, content, frontmatter_json,
|
|
50
64
|
created_at, updated_at, content_hash, code_location_json
|
|
51
65
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
52
|
-
`);
|
|
53
|
-
const info = stmt.run(data.filePath, data.title, data.content, frontmatterJson, data.createdAt || now, now, contentHash, codeLocationJson);
|
|
66
|
+
`).run(data.filePath, data.title, data.content, frontmatterJson, data.createdAt || now, now, contentHash, codeLocationJson);
|
|
54
67
|
return Number(info.lastInsertRowid);
|
|
55
68
|
}
|
|
56
69
|
}
|
|
@@ -61,12 +74,10 @@ export class KnowledgeRepository {
|
|
|
61
74
|
}
|
|
62
75
|
}
|
|
63
76
|
getNoteById(id) {
|
|
64
|
-
|
|
65
|
-
return stmt.get(id);
|
|
77
|
+
return this.stmt("SELECT * FROM knowledge_notes WHERE id = ?").get(id);
|
|
66
78
|
}
|
|
67
79
|
getNoteByPath(filePath) {
|
|
68
|
-
|
|
69
|
-
return stmt.get(filePath);
|
|
80
|
+
return this.stmt("SELECT * FROM knowledge_notes WHERE file_path = ?").get(filePath);
|
|
70
81
|
}
|
|
71
82
|
getNoteByIdOrThrow(id) {
|
|
72
83
|
const note = this.getNoteById(id);
|
|
@@ -82,26 +93,23 @@ export class KnowledgeRepository {
|
|
|
82
93
|
}
|
|
83
94
|
searchNotes(query, limit = 50) {
|
|
84
95
|
try {
|
|
85
|
-
|
|
96
|
+
return this.stmt(`
|
|
86
97
|
SELECT n.*
|
|
87
98
|
FROM knowledge_notes n
|
|
88
99
|
JOIN knowledge_notes_fts fts ON n.id = fts.rowid
|
|
89
100
|
WHERE knowledge_notes_fts MATCH ?
|
|
90
101
|
ORDER BY rank
|
|
91
102
|
LIMIT ?
|
|
92
|
-
`);
|
|
93
|
-
return stmt.all(query, limit);
|
|
103
|
+
`).all(query, limit);
|
|
94
104
|
}
|
|
95
105
|
catch {
|
|
96
106
|
// FTS5失敗時はLIKEフォールバック(不正なクエリ構文への耐性)
|
|
97
|
-
|
|
98
|
-
return stmt.all(`%${query}%`, `%${query}%`, limit);
|
|
107
|
+
return this.stmt(`SELECT * FROM knowledge_notes WHERE title LIKE ? OR content LIKE ? LIMIT ?`).all(`%${query}%`, `%${query}%`, limit);
|
|
99
108
|
}
|
|
100
109
|
}
|
|
101
110
|
deleteNoteById(id) {
|
|
102
111
|
try {
|
|
103
|
-
const
|
|
104
|
-
const info = stmt.run(id);
|
|
112
|
+
const info = this.stmt("DELETE FROM knowledge_notes WHERE id = ?").run(id);
|
|
105
113
|
return info.changes > 0;
|
|
106
114
|
}
|
|
107
115
|
catch (error) {
|
|
@@ -110,8 +118,7 @@ export class KnowledgeRepository {
|
|
|
110
118
|
}
|
|
111
119
|
deleteNoteByPath(path) {
|
|
112
120
|
try {
|
|
113
|
-
const
|
|
114
|
-
const info = stmt.run(path);
|
|
121
|
+
const info = this.stmt("DELETE FROM knowledge_notes WHERE file_path = ?").run(path);
|
|
115
122
|
return info.changes > 0;
|
|
116
123
|
}
|
|
117
124
|
catch (error) {
|
|
@@ -126,18 +133,20 @@ export class KnowledgeRepository {
|
|
|
126
133
|
throw new ValidationError("patterns", patterns, "Patterns must be an array");
|
|
127
134
|
}
|
|
128
135
|
try {
|
|
129
|
-
const deleteStmt = this.db.prepare("DELETE FROM extracted_patterns WHERE note_id = ?");
|
|
130
|
-
deleteStmt.run(noteId);
|
|
131
|
-
const insertStmt = this.db.prepare(`
|
|
132
|
-
INSERT INTO extracted_patterns (
|
|
133
|
-
note_id, pattern_type, content, confidence,
|
|
134
|
-
context, line_number, created_at
|
|
135
|
-
) VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
136
|
-
`);
|
|
137
136
|
const now = new Date().toISOString();
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
137
|
+
const savePatternsTransaction = this.db.transaction(() => {
|
|
138
|
+
this.stmt("DELETE FROM extracted_patterns WHERE note_id = ?").run(noteId);
|
|
139
|
+
const insertSql = `
|
|
140
|
+
INSERT INTO extracted_patterns (
|
|
141
|
+
note_id, pattern_type, content, confidence,
|
|
142
|
+
context, line_number, created_at
|
|
143
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
144
|
+
`;
|
|
145
|
+
for (const pattern of patterns) {
|
|
146
|
+
this.stmt(insertSql).run(noteId, pattern.type, pattern.content, pattern.confidence, pattern.context ?? null, pattern.lineNumber ?? null, now);
|
|
147
|
+
}
|
|
148
|
+
});
|
|
149
|
+
savePatternsTransaction();
|
|
141
150
|
}
|
|
142
151
|
catch (error) {
|
|
143
152
|
if (error instanceof ValidationError)
|
|
@@ -146,42 +155,38 @@ export class KnowledgeRepository {
|
|
|
146
155
|
}
|
|
147
156
|
}
|
|
148
157
|
getPatternsByNoteId(noteId) {
|
|
149
|
-
|
|
150
|
-
return stmt.all(noteId);
|
|
158
|
+
return this.stmt("SELECT * FROM extracted_patterns WHERE note_id = ? ORDER BY line_number").all(noteId);
|
|
151
159
|
}
|
|
152
160
|
saveProblemSolutionPairs(pairs) {
|
|
153
|
-
const stmt = this.db.prepare(`
|
|
154
|
-
INSERT INTO problem_solution_pairs (
|
|
155
|
-
problem_pattern_id, solution_pattern_id, relevance_score, created_at
|
|
156
|
-
) VALUES (?, ?, ?, ?)
|
|
157
|
-
`);
|
|
158
161
|
const now = new Date().toISOString();
|
|
159
162
|
for (const pair of pairs) {
|
|
160
|
-
stmt
|
|
163
|
+
this.stmt(`
|
|
164
|
+
INSERT INTO problem_solution_pairs (
|
|
165
|
+
problem_pattern_id, solution_pattern_id, relevance_score, created_at
|
|
166
|
+
) VALUES (?, ?, ?, ?)
|
|
167
|
+
`).run(pair.problemPatternId, pair.solutionPatternId, pair.relevanceScore, now);
|
|
161
168
|
}
|
|
162
169
|
}
|
|
163
170
|
saveNoteLinks(links) {
|
|
164
|
-
const stmt = this.db.prepare(`
|
|
165
|
-
INSERT OR REPLACE INTO note_links (
|
|
166
|
-
source_note_id, target_note_id, link_type, similarity, created_at
|
|
167
|
-
) VALUES (?, ?, ?, ?, ?)
|
|
168
|
-
`);
|
|
169
171
|
const now = new Date().toISOString();
|
|
170
172
|
for (const link of links) {
|
|
171
|
-
stmt
|
|
173
|
+
this.stmt(`
|
|
174
|
+
INSERT OR REPLACE INTO note_links (
|
|
175
|
+
source_note_id, target_note_id, link_type, similarity, created_at
|
|
176
|
+
) VALUES (?, ?, ?, ?, ?)
|
|
177
|
+
`).run(link.sourceNoteId, link.targetNoteId, link.linkType, link.similarity ?? null, now);
|
|
172
178
|
}
|
|
173
179
|
}
|
|
174
180
|
getNoteLinks(noteId) {
|
|
175
|
-
|
|
181
|
+
return this.stmt(`
|
|
176
182
|
SELECT target_note_id as targetNoteId, link_type as linkType, similarity
|
|
177
183
|
FROM note_links WHERE source_note_id = ?
|
|
178
184
|
ORDER BY similarity DESC
|
|
179
|
-
`);
|
|
180
|
-
return stmt.all(noteId);
|
|
185
|
+
`).all(noteId);
|
|
181
186
|
}
|
|
182
187
|
getProblemSolutionPairsByNoteId(noteId) {
|
|
183
188
|
// Join through extracted_patterns to find pairs associated with this note's patterns
|
|
184
|
-
|
|
189
|
+
return this.stmt(`
|
|
185
190
|
SELECT psp.id,
|
|
186
191
|
ep_problem.note_id as problemNoteId,
|
|
187
192
|
ep_solution.note_id as solutionNoteId,
|
|
@@ -192,8 +197,7 @@ export class KnowledgeRepository {
|
|
|
192
197
|
JOIN extracted_patterns ep_problem ON psp.problem_pattern_id = ep_problem.id
|
|
193
198
|
JOIN extracted_patterns ep_solution ON psp.solution_pattern_id = ep_solution.id
|
|
194
199
|
WHERE ep_problem.note_id = ? OR ep_solution.note_id = ?
|
|
195
|
-
`);
|
|
196
|
-
return stmt.all(noteId, noteId);
|
|
200
|
+
`).all(noteId, noteId);
|
|
197
201
|
}
|
|
198
202
|
findNotesByTagSimilarity(noteId, tags, limit) {
|
|
199
203
|
// Search notes that share tags via frontmatter_json
|
|
@@ -227,31 +231,36 @@ export class KnowledgeRepository {
|
|
|
227
231
|
return stmt.all(noteId, createdAt, days, createdAt, limit);
|
|
228
232
|
}
|
|
229
233
|
getStats() {
|
|
230
|
-
const totalNotes = this.
|
|
231
|
-
const totalPatterns = this.
|
|
232
|
-
const totalLinks = this.
|
|
233
|
-
const totalPairs = this.
|
|
234
|
-
const typeRows = this.
|
|
235
|
-
.prepare("SELECT pattern_type, COUNT(*) as count FROM extracted_patterns GROUP BY pattern_type")
|
|
236
|
-
.all();
|
|
234
|
+
const totalNotes = this.stmt("SELECT COUNT(*) as count FROM knowledge_notes").get().count;
|
|
235
|
+
const totalPatterns = this.stmt("SELECT COUNT(*) as count FROM extracted_patterns").get().count;
|
|
236
|
+
const totalLinks = this.stmt("SELECT COUNT(*) as count FROM note_links").get().count;
|
|
237
|
+
const totalPairs = this.stmt("SELECT COUNT(*) as count FROM problem_solution_pairs").get().count;
|
|
238
|
+
const typeRows = this.stmt("SELECT pattern_type, COUNT(*) as count FROM extracted_patterns GROUP BY pattern_type").all();
|
|
237
239
|
const patternsByType = {};
|
|
238
240
|
for (const row of typeRows) {
|
|
239
241
|
patternsByType[row.pattern_type] = row.count;
|
|
240
242
|
}
|
|
241
|
-
|
|
243
|
+
// Source breakdown (source_type may be NULL for pre-migration notes)
|
|
244
|
+
const sourceRows = this.db
|
|
245
|
+
.prepare("SELECT COALESCE(source_type, 'markdown') as source, COUNT(*) as count FROM knowledge_notes GROUP BY COALESCE(source_type, 'markdown')")
|
|
246
|
+
.all();
|
|
247
|
+
const notesBySource = {};
|
|
248
|
+
for (const row of sourceRows) {
|
|
249
|
+
notesBySource[row.source] = row.count;
|
|
250
|
+
}
|
|
251
|
+
return { totalNotes, totalPatterns, totalLinks, totalPairs, patternsByType, notesBySource };
|
|
242
252
|
}
|
|
243
253
|
/**
|
|
244
254
|
* コードファイルパスで code_location_json を持つノートを検索する
|
|
245
255
|
* path パラメータを含む code_location_json が NULL でないノートを返す
|
|
246
256
|
*/
|
|
247
257
|
searchByCodeLocation(path) {
|
|
248
|
-
|
|
258
|
+
return this.stmt(`
|
|
249
259
|
SELECT * FROM knowledge_notes
|
|
250
260
|
WHERE code_location_json IS NOT NULL
|
|
251
261
|
AND code_location_json LIKE ?
|
|
252
262
|
ORDER BY created_at DESC
|
|
253
|
-
`);
|
|
254
|
-
return stmt.all(`%${path}%`);
|
|
263
|
+
`).all(`%${path}%`);
|
|
255
264
|
}
|
|
256
265
|
/**
|
|
257
266
|
* ノートの埋め込みベクトルを保存する(upsert)
|
|
@@ -259,31 +268,29 @@ export class KnowledgeRepository {
|
|
|
259
268
|
saveEmbedding(noteId, embedding, modelName) {
|
|
260
269
|
try {
|
|
261
270
|
const now = new Date().toISOString();
|
|
262
|
-
const embBuf = Buffer.from(embedding.buffer);
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
.
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
}
|
|
284
|
-
|
|
285
|
-
// note_embeddings_vec が存在しない場合は無視(graceful degradation)
|
|
286
|
-
}
|
|
271
|
+
const embBuf = Buffer.from(embedding.buffer, embedding.byteOffset, embedding.byteLength);
|
|
272
|
+
const saveEmbeddingTransaction = this.db.transaction(() => {
|
|
273
|
+
// note_embeddings テーブルに upsert
|
|
274
|
+
this.stmt(`
|
|
275
|
+
INSERT INTO note_embeddings (note_id, embedding, model_name, dimensions, created_at, updated_at)
|
|
276
|
+
VALUES (?, ?, ?, ?, ?, ?)
|
|
277
|
+
ON CONFLICT(note_id) DO UPDATE SET
|
|
278
|
+
embedding = excluded.embedding,
|
|
279
|
+
model_name = excluded.model_name,
|
|
280
|
+
dimensions = excluded.dimensions,
|
|
281
|
+
updated_at = excluded.updated_at
|
|
282
|
+
`).run(noteId, embBuf, modelName, embedding.length, now, now);
|
|
283
|
+
// note_embeddings_vec (vec0) が存在する場合は手動で同期
|
|
284
|
+
// vec0 は ON CONFLICT をサポートしないため DELETE + INSERT を使う
|
|
285
|
+
try {
|
|
286
|
+
this.stmt("DELETE FROM note_embeddings_vec WHERE note_id = CAST(? AS INTEGER)").run(noteId);
|
|
287
|
+
this.stmt("INSERT INTO note_embeddings_vec(note_id, embedding) VALUES (CAST(? AS INTEGER), ?)").run(noteId, embBuf);
|
|
288
|
+
}
|
|
289
|
+
catch {
|
|
290
|
+
// note_embeddings_vec が存在しない場合は無視(graceful degradation)
|
|
291
|
+
}
|
|
292
|
+
});
|
|
293
|
+
saveEmbeddingTransaction();
|
|
287
294
|
}
|
|
288
295
|
catch (error) {
|
|
289
296
|
if (error instanceof DatabaseError)
|
|
@@ -291,21 +298,96 @@ export class KnowledgeRepository {
|
|
|
291
298
|
throw new DatabaseError("saveEmbedding", error, { noteId });
|
|
292
299
|
}
|
|
293
300
|
}
|
|
301
|
+
/**
|
|
302
|
+
* 複数ノートの埋め込みベクトルをバッチ保存する
|
|
303
|
+
* 100件単位のチャンクトランザクションで処理し、チャンク失敗時は1件ずつフォールバック
|
|
304
|
+
*/
|
|
305
|
+
saveEmbeddingBatch(items) {
|
|
306
|
+
if (items.length === 0)
|
|
307
|
+
return { saved: 0, failed: 0 };
|
|
308
|
+
// vec0 利用可否を事前に1回だけチェック
|
|
309
|
+
let vec0Available = true;
|
|
310
|
+
try {
|
|
311
|
+
this.db.prepare("SELECT COUNT(*) FROM note_embeddings_vec").get();
|
|
312
|
+
}
|
|
313
|
+
catch {
|
|
314
|
+
vec0Available = false;
|
|
315
|
+
}
|
|
316
|
+
const CHUNK_SIZE = 100;
|
|
317
|
+
let saved = 0;
|
|
318
|
+
let failed = 0;
|
|
319
|
+
const now = new Date().toISOString();
|
|
320
|
+
const upsertStmt = this.stmt(`
|
|
321
|
+
INSERT INTO note_embeddings (note_id, embedding, model_name, dimensions, created_at, updated_at)
|
|
322
|
+
VALUES (?, ?, ?, ?, ?, ?)
|
|
323
|
+
ON CONFLICT(note_id) DO UPDATE SET
|
|
324
|
+
embedding = excluded.embedding,
|
|
325
|
+
model_name = excluded.model_name,
|
|
326
|
+
dimensions = excluded.dimensions,
|
|
327
|
+
updated_at = excluded.updated_at
|
|
328
|
+
`);
|
|
329
|
+
const vecDeleteStmt = vec0Available
|
|
330
|
+
? this.stmt("DELETE FROM note_embeddings_vec WHERE note_id = CAST(? AS INTEGER)")
|
|
331
|
+
: null;
|
|
332
|
+
const vecInsertStmt = vec0Available
|
|
333
|
+
? this.stmt("INSERT INTO note_embeddings_vec(note_id, embedding) VALUES (CAST(? AS INTEGER), ?)")
|
|
334
|
+
: null;
|
|
335
|
+
const saveOne = (item) => {
|
|
336
|
+
const embBuf = Buffer.from(item.embedding.buffer, item.embedding.byteOffset, item.embedding.byteLength);
|
|
337
|
+
upsertStmt.run(item.noteId, embBuf, item.modelName, item.embedding.length, now, now);
|
|
338
|
+
if (vec0Available && vecDeleteStmt && vecInsertStmt) {
|
|
339
|
+
try {
|
|
340
|
+
vecDeleteStmt.run(item.noteId);
|
|
341
|
+
vecInsertStmt.run(item.noteId, embBuf);
|
|
342
|
+
}
|
|
343
|
+
catch {
|
|
344
|
+
// vec0 操作失敗は無視(graceful degradation)
|
|
345
|
+
}
|
|
346
|
+
}
|
|
347
|
+
};
|
|
348
|
+
for (let i = 0; i < items.length; i += CHUNK_SIZE) {
|
|
349
|
+
const chunk = items.slice(i, i + CHUNK_SIZE);
|
|
350
|
+
try {
|
|
351
|
+
const chunkTransaction = this.db.transaction(() => {
|
|
352
|
+
for (const item of chunk) {
|
|
353
|
+
saveOne(item);
|
|
354
|
+
}
|
|
355
|
+
});
|
|
356
|
+
chunkTransaction();
|
|
357
|
+
saved += chunk.length;
|
|
358
|
+
}
|
|
359
|
+
catch {
|
|
360
|
+
// チャンク失敗時は1件ずつフォールバック
|
|
361
|
+
for (const item of chunk) {
|
|
362
|
+
try {
|
|
363
|
+
const singleTransaction = this.db.transaction(() => {
|
|
364
|
+
saveOne(item);
|
|
365
|
+
});
|
|
366
|
+
singleTransaction();
|
|
367
|
+
saved++;
|
|
368
|
+
}
|
|
369
|
+
catch {
|
|
370
|
+
failed++;
|
|
371
|
+
}
|
|
372
|
+
}
|
|
373
|
+
}
|
|
374
|
+
}
|
|
375
|
+
return { saved, failed };
|
|
376
|
+
}
|
|
294
377
|
/**
|
|
295
378
|
* ベクトル類似度検索(note_embeddings_vec を使用)
|
|
296
379
|
* sqlite-vec が利用できない場合は空配列を返す
|
|
297
380
|
*/
|
|
298
381
|
searchByVector(embedding, limit = 10) {
|
|
299
382
|
try {
|
|
300
|
-
const buf = Buffer.from(embedding.buffer);
|
|
301
|
-
|
|
383
|
+
const buf = Buffer.from(embedding.buffer, embedding.byteOffset, embedding.byteLength);
|
|
384
|
+
return this.stmt(`
|
|
302
385
|
SELECT note_id, distance
|
|
303
386
|
FROM note_embeddings_vec
|
|
304
387
|
WHERE embedding MATCH ?
|
|
305
388
|
ORDER BY distance
|
|
306
389
|
LIMIT ?
|
|
307
|
-
`);
|
|
308
|
-
return stmt.all(buf, limit);
|
|
390
|
+
`).all(buf, limit);
|
|
309
391
|
}
|
|
310
392
|
catch {
|
|
311
393
|
// vec0テーブルが存在しない場合(sqlite-vec未ロード)は空を返す
|
|
@@ -324,72 +406,96 @@ export class KnowledgeRepository {
|
|
|
324
406
|
dateParams.push(dateFrom);
|
|
325
407
|
if (dateTo)
|
|
326
408
|
dateParams.push(dateTo);
|
|
409
|
+
// CJK文字を含むクエリはtrigramテーブルを優先使用(trigramは最低3文字必要)
|
|
410
|
+
const hasCjk = /[\u3040-\u309F\u30A0-\u30FF\u4E00-\u9FFF\uAC00-\uD7AF]/.test(query);
|
|
411
|
+
const useTrigram = hasCjk && query.length >= 3;
|
|
412
|
+
const ftsTable = useTrigram ? "knowledge_notes_fts_trigram" : "knowledge_notes_fts";
|
|
327
413
|
try {
|
|
328
|
-
const
|
|
329
|
-
SELECT n.*,
|
|
414
|
+
const rows = this.stmt(`
|
|
415
|
+
SELECT n.*, bm25(${ftsTable}, 10.0, 1.0) AS rank
|
|
330
416
|
FROM knowledge_notes n
|
|
331
|
-
JOIN
|
|
332
|
-
WHERE
|
|
417
|
+
JOIN ${ftsTable} fts ON n.id = fts.rowid
|
|
418
|
+
WHERE ${ftsTable} MATCH ?
|
|
333
419
|
${deprecatedFilter}
|
|
334
420
|
${dateFromFilter}
|
|
335
421
|
${dateToFilter}
|
|
336
422
|
ORDER BY rank
|
|
337
423
|
LIMIT ?
|
|
338
|
-
`);
|
|
339
|
-
|
|
424
|
+
`).all(query, ...dateParams, limit);
|
|
425
|
+
// CJKクエリでunicode61が0件の場合はLIKEフォールバック(短いCJKトークンの救済)
|
|
426
|
+
if (rows.length === 0 && hasCjk) {
|
|
427
|
+
return this.searchNotesWithLike(query, limit, includeDeprecated, dateFrom, dateTo);
|
|
428
|
+
}
|
|
340
429
|
return rows.map(({ rank, ...note }) => ({ note: note, rank }));
|
|
341
430
|
}
|
|
342
431
|
catch {
|
|
343
|
-
|
|
344
|
-
const deprecatedClause = includeDeprecated ? "" : "AND n.deprecated = 0";
|
|
345
|
-
const fallbackStmt = this.db.prepare(`SELECT n.* FROM knowledge_notes n WHERE (n.title LIKE ? OR n.content LIKE ?) ${deprecatedClause} ${dateFromFilter} ${dateToFilter} LIMIT ?`);
|
|
346
|
-
const fallbackRows = fallbackStmt.all(`%${query}%`, `%${query}%`, ...dateParams, limit);
|
|
347
|
-
return fallbackRows.map((row) => ({ note: row, rank: 0 }));
|
|
432
|
+
return this.searchNotesWithLike(query, limit, includeDeprecated, dateFrom, dateTo);
|
|
348
433
|
}
|
|
349
434
|
}
|
|
435
|
+
searchNotesWithLike(query, limit, includeDeprecated, dateFrom, dateTo) {
|
|
436
|
+
const deprecatedClause = includeDeprecated ? "" : "AND n.deprecated = 0";
|
|
437
|
+
const dateFromFilter = dateFrom ? "AND n.created_at >= ?" : "";
|
|
438
|
+
const dateToFilter = dateTo ? "AND n.created_at <= ?" : "";
|
|
439
|
+
const dateParams = [];
|
|
440
|
+
if (dateFrom)
|
|
441
|
+
dateParams.push(dateFrom);
|
|
442
|
+
if (dateTo)
|
|
443
|
+
dateParams.push(dateTo);
|
|
444
|
+
const fallbackRows = this.stmt(`SELECT n.* FROM knowledge_notes n WHERE (n.title LIKE ? OR n.content LIKE ?) ${deprecatedClause} ${dateFromFilter} ${dateToFilter} LIMIT ?`).all(`%${query}%`, `%${query}%`, ...dateParams, limit);
|
|
445
|
+
return fallbackRows.map((row) => ({ note: row, rank: 0 }));
|
|
446
|
+
}
|
|
350
447
|
/**
|
|
351
448
|
* 埋め込みがまだ生成されていないノートを取得する
|
|
352
449
|
*/
|
|
353
450
|
getNotesWithoutEmbeddings() {
|
|
354
|
-
|
|
451
|
+
return this.stmt(`
|
|
355
452
|
SELECT n.* FROM knowledge_notes n
|
|
356
453
|
WHERE NOT EXISTS (SELECT 1 FROM note_embeddings e WHERE e.note_id = n.id)
|
|
357
|
-
`);
|
|
358
|
-
return stmt.all();
|
|
454
|
+
`).all();
|
|
359
455
|
}
|
|
360
456
|
/**
|
|
361
457
|
* content_hashが変更されて埋め込みが古くなったノートを取得する
|
|
362
458
|
*/
|
|
363
459
|
getNotesWithStaleEmbeddings() {
|
|
364
460
|
// note_embeddingsのupdated_atとknowledge_notesのupdated_atを比較
|
|
365
|
-
|
|
461
|
+
return this.stmt(`
|
|
366
462
|
SELECT n.* FROM knowledge_notes n
|
|
367
463
|
JOIN note_embeddings e ON e.note_id = n.id
|
|
368
464
|
WHERE n.updated_at IS NOT NULL AND n.updated_at > e.created_at
|
|
369
|
-
`);
|
|
370
|
-
return stmt.all();
|
|
465
|
+
`).all();
|
|
371
466
|
}
|
|
372
467
|
/**
|
|
373
468
|
* 指定プレフィックスで始まる file_path を持つノートを取得する
|
|
374
469
|
*/
|
|
375
470
|
getNotesWithPrefix(prefix, limit = 100) {
|
|
376
|
-
|
|
377
|
-
return stmt.all(`${prefix}%`, limit);
|
|
471
|
+
return this.stmt("SELECT * FROM knowledge_notes WHERE file_path LIKE ? ORDER BY created_at DESC LIMIT ?").all(`${prefix}%`, limit);
|
|
378
472
|
}
|
|
379
473
|
/**
|
|
380
474
|
* 全ノートを取得する
|
|
381
475
|
*/
|
|
382
476
|
getAllNotes() {
|
|
383
|
-
|
|
384
|
-
|
|
477
|
+
return this.stmt("SELECT * FROM knowledge_notes").all();
|
|
478
|
+
}
|
|
479
|
+
/**
|
|
480
|
+
* 全ノートの ID のみを取得する(content をロードしないため OOM 対策)
|
|
481
|
+
*/
|
|
482
|
+
getAllNoteIds() {
|
|
483
|
+
return this.stmt("SELECT id FROM knowledge_notes").pluck().all();
|
|
484
|
+
}
|
|
485
|
+
/**
|
|
486
|
+
* 埋め込みがまだ生成されていないノートの ID のみを取得する(OOM 対策)
|
|
487
|
+
*/
|
|
488
|
+
getNotesWithoutEmbeddingIds() {
|
|
489
|
+
return this.stmt("SELECT n.id FROM knowledge_notes n WHERE NOT EXISTS (SELECT 1 FROM note_embeddings e WHERE e.note_id = n.id)")
|
|
490
|
+
.pluck()
|
|
491
|
+
.all();
|
|
385
492
|
}
|
|
386
493
|
/**
|
|
387
494
|
* 特定のソースプラグインで作成されたノートを取得する
|
|
388
495
|
* frontmatter_json 内の source_plugin フィールドで絞り込む
|
|
389
496
|
*/
|
|
390
497
|
getNotesBySourcePlugin(pluginId) {
|
|
391
|
-
|
|
392
|
-
return stmt.all(`%"source_plugin":"${pluginId}"%`);
|
|
498
|
+
return this.stmt(`SELECT * FROM knowledge_notes WHERE frontmatter_json LIKE ?`).all(`%"source_plugin":"${pluginId}"%`);
|
|
393
499
|
}
|
|
394
500
|
/**
|
|
395
501
|
* 指定IDリストのノートを一括削除する
|
|
@@ -398,8 +504,9 @@ export class KnowledgeRepository {
|
|
|
398
504
|
if (ids.length === 0)
|
|
399
505
|
return 0;
|
|
400
506
|
const placeholders = ids.map(() => "?").join(",");
|
|
401
|
-
const
|
|
402
|
-
|
|
507
|
+
const info = this.db
|
|
508
|
+
.prepare(`DELETE FROM knowledge_notes WHERE id IN (${placeholders})`)
|
|
509
|
+
.run(...ids);
|
|
403
510
|
return info.changes;
|
|
404
511
|
}
|
|
405
512
|
/**
|
|
@@ -407,68 +514,60 @@ export class KnowledgeRepository {
|
|
|
407
514
|
*/
|
|
408
515
|
updateExtractedAt(noteId) {
|
|
409
516
|
const now = new Date().toISOString();
|
|
410
|
-
this.
|
|
517
|
+
this.stmt("UPDATE knowledge_notes SET extracted_at = ? WHERE id = ?").run(now, noteId);
|
|
411
518
|
}
|
|
412
519
|
/**
|
|
413
520
|
* sourceUri プレフィックスでノートを取得する
|
|
414
521
|
* file_path が sourceUri として使われている(normalizer.ts の仕様)
|
|
415
522
|
*/
|
|
416
523
|
getNotesBySourceUriPrefix(prefix) {
|
|
417
|
-
|
|
418
|
-
return stmt.all(`${prefix}%`);
|
|
524
|
+
return this.stmt("SELECT * FROM knowledge_notes WHERE file_path LIKE ? ORDER BY created_at ASC").all(`${prefix}%`);
|
|
419
525
|
}
|
|
420
526
|
/**
|
|
421
527
|
* 既存リンクチェック後に note_links へ INSERT する(冪等)
|
|
422
528
|
* 同じ source/target ペアが既に存在する場合は false を返す
|
|
423
529
|
*/
|
|
424
530
|
saveNoteLinkIfNotExists(sourceNoteId, targetNoteId, linkType, similarity) {
|
|
425
|
-
const existing = this.
|
|
426
|
-
.prepare("SELECT id FROM note_links WHERE source_note_id = ? AND target_note_id = ?")
|
|
427
|
-
.get(sourceNoteId, targetNoteId);
|
|
531
|
+
const existing = this.stmt("SELECT id FROM note_links WHERE source_note_id = ? AND target_note_id = ?").get(sourceNoteId, targetNoteId);
|
|
428
532
|
if (existing)
|
|
429
533
|
return false;
|
|
430
534
|
const now = new Date().toISOString();
|
|
431
|
-
this.
|
|
432
|
-
.prepare("INSERT INTO note_links (source_note_id, target_note_id, link_type, similarity, created_at) VALUES (?, ?, ?, ?, ?)")
|
|
433
|
-
.run(sourceNoteId, targetNoteId, linkType, similarity ?? null, now);
|
|
535
|
+
this.stmt("INSERT INTO note_links (source_note_id, target_note_id, link_type, similarity, created_at) VALUES (?, ?, ?, ?, ?)").run(sourceNoteId, targetNoteId, linkType, similarity ?? null, now);
|
|
434
536
|
return true;
|
|
435
537
|
}
|
|
436
538
|
/**
|
|
437
539
|
* 指定ノートに関するリンクをすべて取得する(source または target)
|
|
438
540
|
*/
|
|
439
541
|
getLinksForNote(noteId) {
|
|
440
|
-
|
|
542
|
+
return this.stmt(`
|
|
441
543
|
SELECT id, source_note_id as sourceNoteId, target_note_id as targetNoteId,
|
|
442
544
|
link_type as linkType, similarity, created_at as createdAt
|
|
443
545
|
FROM note_links
|
|
444
546
|
WHERE source_note_id = ? OR target_note_id = ?
|
|
445
547
|
ORDER BY created_at DESC
|
|
446
|
-
`);
|
|
447
|
-
return stmt.all(noteId, noteId);
|
|
548
|
+
`).all(noteId, noteId);
|
|
448
549
|
}
|
|
449
550
|
/**
|
|
450
551
|
* サジェスト結果へのフィードバックを保存する
|
|
451
552
|
*/
|
|
452
553
|
saveSuggestFeedback(noteId, query, isUseful, context) {
|
|
453
554
|
const now = new Date().toISOString();
|
|
454
|
-
const
|
|
555
|
+
const info = this.stmt(`
|
|
455
556
|
INSERT INTO suggest_feedback (note_id, query, is_useful, context, created_at)
|
|
456
557
|
VALUES (?, ?, ?, ?, ?)
|
|
457
|
-
`);
|
|
458
|
-
const info = stmt.run(noteId, query, isUseful ? 1 : 0, context ?? null, now);
|
|
558
|
+
`).run(noteId, query, isUseful ? 1 : 0, context ?? null, now);
|
|
459
559
|
return Number(info.lastInsertRowid);
|
|
460
560
|
}
|
|
461
561
|
/**
|
|
462
562
|
* 特定ノートのサジェストフィードバック一覧を取得する(新しい順)
|
|
463
563
|
*/
|
|
464
564
|
getSuggestFeedbackForNote(noteId) {
|
|
465
|
-
const
|
|
565
|
+
const rows = this.stmt(`
|
|
466
566
|
SELECT id, query, is_useful as isUseful, context, created_at as createdAt
|
|
467
567
|
FROM suggest_feedback
|
|
468
568
|
WHERE note_id = ?
|
|
469
569
|
ORDER BY id DESC
|
|
470
|
-
`);
|
|
471
|
-
const rows = stmt.all(noteId);
|
|
570
|
+
`).all(noteId);
|
|
472
571
|
return rows.map((row) => ({ ...row, isUseful: row.isUseful === 1 }));
|
|
473
572
|
}
|
|
474
573
|
/**
|
|
@@ -478,9 +577,7 @@ export class KnowledgeRepository {
|
|
|
478
577
|
const note = this.getNoteById(noteId);
|
|
479
578
|
if (!note)
|
|
480
579
|
throw new KnowledgeNotFoundError(noteId, "id");
|
|
481
|
-
this.
|
|
482
|
-
.prepare("UPDATE knowledge_notes SET deprecated = 1, deprecation_reason = ? WHERE id = ?")
|
|
483
|
-
.run(reason, noteId);
|
|
580
|
+
this.stmt("UPDATE knowledge_notes SET deprecated = 1, deprecation_reason = ? WHERE id = ?").run(reason, noteId);
|
|
484
581
|
}
|
|
485
582
|
/**
|
|
486
583
|
* ノートの deprecated フラグを解除する
|
|
@@ -489,9 +586,7 @@ export class KnowledgeRepository {
|
|
|
489
586
|
const note = this.getNoteById(noteId);
|
|
490
587
|
if (!note)
|
|
491
588
|
throw new KnowledgeNotFoundError(noteId, "id");
|
|
492
|
-
this.
|
|
493
|
-
.prepare("UPDATE knowledge_notes SET deprecated = 0, deprecation_reason = NULL WHERE id = ?")
|
|
494
|
-
.run(noteId);
|
|
589
|
+
this.stmt("UPDATE knowledge_notes SET deprecated = 0, deprecation_reason = NULL WHERE id = ?").run(noteId);
|
|
495
590
|
}
|
|
496
591
|
/**
|
|
497
592
|
* 既存ノートの新バージョンを作成する
|
|
@@ -514,23 +609,61 @@ export class KnowledgeRepository {
|
|
|
514
609
|
const versionedPath = `${existing.file_path}#v${newVersion}`;
|
|
515
610
|
const createNewVersion = this.db.transaction(() => {
|
|
516
611
|
// deprecated にマーク
|
|
517
|
-
this.
|
|
518
|
-
.prepare("UPDATE knowledge_notes SET deprecated = 1, deprecation_reason = ? WHERE id = ?")
|
|
519
|
-
.run(`Superseded by version ${newVersion}`, noteId);
|
|
612
|
+
this.stmt("UPDATE knowledge_notes SET deprecated = 1, deprecation_reason = ? WHERE id = ?").run(`Superseded by version ${newVersion}`, noteId);
|
|
520
613
|
// 新バージョンを INSERT
|
|
521
|
-
const
|
|
614
|
+
const info = this.stmt(`
|
|
522
615
|
INSERT INTO knowledge_notes (
|
|
523
616
|
file_path, title, content, frontmatter_json,
|
|
524
617
|
created_at, updated_at, content_hash,
|
|
525
618
|
version, supersedes, valid_from, deprecated
|
|
526
619
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 0)
|
|
527
|
-
`);
|
|
528
|
-
const info = stmt.run(versionedPath, title, content, frontmatterJson, existing.created_at, now, contentHash, newVersion, noteId, now);
|
|
620
|
+
`).run(versionedPath, title, content, frontmatterJson, existing.created_at, now, contentHash, newVersion, noteId, now);
|
|
529
621
|
return Number(info.lastInsertRowid);
|
|
530
622
|
});
|
|
531
623
|
return createNewVersion();
|
|
532
624
|
}
|
|
625
|
+
/**
|
|
626
|
+
* 指定IDリストのノートをサマリー形式(content, frontmatter_jsonを除く)で一括取得する
|
|
627
|
+
* SQLITE_MAX_VARIABLE_NUMBER 対策として 500件ずつチャンク処理する
|
|
628
|
+
*
|
|
629
|
+
* **注意**: 返り順は入力 ids の順序と一致しない。順序が重要な場合は
|
|
630
|
+
* 呼び出し元で id→note の Map を構築してマッピングすること。
|
|
631
|
+
*/
|
|
632
|
+
getNotesSummaryByIds(ids) {
|
|
633
|
+
if (ids.length === 0)
|
|
634
|
+
return [];
|
|
635
|
+
const CHUNK = 500;
|
|
636
|
+
const results = [];
|
|
637
|
+
for (let i = 0; i < ids.length; i += CHUNK) {
|
|
638
|
+
const chunk = ids.slice(i, i + CHUNK);
|
|
639
|
+
const placeholders = chunk.map(() => "?").join(",");
|
|
640
|
+
const stmt = this.db.prepare(`SELECT ${SUMMARY_COLUMNS} FROM knowledge_notes WHERE id IN (${placeholders})`);
|
|
641
|
+
results.push(...stmt.all(...chunk));
|
|
642
|
+
}
|
|
643
|
+
return results;
|
|
644
|
+
}
|
|
645
|
+
/**
|
|
646
|
+
* 指定IDリストのノートを全カラムで一括取得する
|
|
647
|
+
* SQLITE_MAX_VARIABLE_NUMBER 対策として 500件ずつチャンク処理する
|
|
648
|
+
*
|
|
649
|
+
* **注意**: 返り順は入力 ids の順序と一致しない。順序が重要な場合は
|
|
650
|
+
* 呼び出し元で id→note の Map を構築してマッピングすること。
|
|
651
|
+
*/
|
|
652
|
+
getNotesByIds(ids) {
|
|
653
|
+
if (ids.length === 0)
|
|
654
|
+
return [];
|
|
655
|
+
const CHUNK = 500;
|
|
656
|
+
const results = [];
|
|
657
|
+
for (let i = 0; i < ids.length; i += CHUNK) {
|
|
658
|
+
const chunk = ids.slice(i, i + CHUNK);
|
|
659
|
+
const placeholders = chunk.map(() => "?").join(",");
|
|
660
|
+
const stmt = this.db.prepare(`SELECT * FROM knowledge_notes WHERE id IN (${placeholders})`);
|
|
661
|
+
results.push(...stmt.all(...chunk));
|
|
662
|
+
}
|
|
663
|
+
return results;
|
|
664
|
+
}
|
|
533
665
|
close() {
|
|
666
|
+
this._stmtCache.clear();
|
|
534
667
|
this.db.close();
|
|
535
668
|
}
|
|
536
669
|
}
|