@devflow-tools/knowledge-engine 0.17.8 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/dist/dependency-source-compiler.d.ts +1 -0
- package/dist/dependency-source-compiler.d.ts.map +1 -1
- package/dist/dependency-source-compiler.js +37 -87
- package/dist/dependency-source-compiler.js.map +1 -1
- package/dist/embedding-provider.d.ts +1 -1
- package/dist/embedding-provider.d.ts.map +1 -1
- package/dist/embedding-provider.js +54 -0
- package/dist/embedding-provider.js.map +1 -1
- package/dist/hybrid-search.d.ts +3 -1
- package/dist/hybrid-search.d.ts.map +1 -1
- package/dist/hybrid-search.js +26 -10
- package/dist/hybrid-search.js.map +1 -1
- package/dist/index.d.ts +8 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +4 -1
- package/dist/index.js.map +1 -1
- package/dist/knowledge-compiler.d.ts +1 -4
- package/dist/knowledge-compiler.d.ts.map +1 -1
- package/dist/knowledge-compiler.js +3 -13
- package/dist/knowledge-compiler.js.map +1 -1
- package/dist/knowledge-engine.d.ts +26 -58
- package/dist/knowledge-engine.d.ts.map +1 -1
- package/dist/knowledge-engine.js +159 -412
- package/dist/knowledge-engine.js.map +1 -1
- package/dist/knowledge-ingest-runtime.d.ts +4 -1
- package/dist/knowledge-ingest-runtime.d.ts.map +1 -1
- package/dist/knowledge-ingest-runtime.js +47 -33
- package/dist/knowledge-ingest-runtime.js.map +1 -1
- package/dist/knowledge-maintenance.d.ts.map +1 -1
- package/dist/knowledge-maintenance.js +38 -54
- package/dist/knowledge-maintenance.js.map +1 -1
- package/dist/knowledge-quality.js +1 -1
- package/dist/knowledge-quality.js.map +1 -1
- package/dist/knowledge-query-planner.d.ts +1 -1
- package/dist/knowledge-query-planner.d.ts.map +1 -1
- package/dist/knowledge-query-planner.js +4 -7
- package/dist/knowledge-query-planner.js.map +1 -1
- package/dist/knowledge-query-service.d.ts +62 -0
- package/dist/knowledge-query-service.d.ts.map +1 -0
- package/dist/knowledge-query-service.js +258 -0
- package/dist/knowledge-query-service.js.map +1 -0
- package/dist/knowledge-source-lifecycle.d.ts +91 -0
- package/dist/knowledge-source-lifecycle.d.ts.map +1 -0
- package/dist/knowledge-source-lifecycle.js +152 -0
- package/dist/knowledge-source-lifecycle.js.map +1 -0
- package/dist/knowledge-source-service.d.ts +66 -0
- package/dist/knowledge-source-service.d.ts.map +1 -0
- package/dist/knowledge-source-service.js +444 -0
- package/dist/knowledge-source-service.js.map +1 -0
- package/dist/knowledge-store.d.ts +59 -16
- package/dist/knowledge-store.d.ts.map +1 -1
- package/dist/knowledge-store.js +345 -133
- package/dist/knowledge-store.js.map +1 -1
- package/dist/markdown-converter.d.ts +0 -3
- package/dist/markdown-converter.d.ts.map +1 -1
- package/dist/markdown-converter.js +5 -5
- package/dist/markdown-converter.js.map +1 -1
- package/dist/semantic-output-contract.d.ts +3 -2
- package/dist/semantic-output-contract.d.ts.map +1 -1
- package/dist/semantic-output-contract.js +13 -0
- package/dist/semantic-output-contract.js.map +1 -1
- package/dist/source-catalog.d.ts +8 -6
- package/dist/source-catalog.d.ts.map +1 -1
- package/dist/source-catalog.js +26 -22
- package/dist/source-catalog.js.map +1 -1
- package/dist/source-readiness.d.ts +3 -2
- package/dist/source-readiness.d.ts.map +1 -1
- package/dist/source-readiness.js +10 -11
- package/dist/source-readiness.js.map +1 -1
- package/dist/source-selector.d.ts +1 -1
- package/dist/source-selector.d.ts.map +1 -1
- package/dist/source-selector.js +8 -3
- package/dist/source-selector.js.map +1 -1
- package/package.json +12 -4
package/dist/knowledge-store.js
CHANGED
|
@@ -6,6 +6,7 @@ import { isPollutedKnowledgeHeading, resolveKnowledgeTitle } from './result-qual
|
|
|
6
6
|
import { KNOWLEDGE_CHUNK_POLICY_VERSION } from './markdown-converter.js';
|
|
7
7
|
import { createHash } from 'node:crypto';
|
|
8
8
|
import { extractApiIdentifiers, extractCodeBlocks, inferKnowledgeChunkKind } from './markdown-converter.js';
|
|
9
|
+
import { createMiniLmEmbeddingProfile } from '@devflow-tools/sdk';
|
|
9
10
|
const KNOWLEDGE_SCHEMA = `
|
|
10
11
|
CREATE TABLE IF NOT EXISTS knowledge_docs (
|
|
11
12
|
id TEXT PRIMARY KEY,
|
|
@@ -31,7 +32,19 @@ CREATE TABLE IF NOT EXISTS knowledge_chunks (
|
|
|
31
32
|
framework TEXT,
|
|
32
33
|
api_identifiers TEXT NOT NULL DEFAULT '[]',
|
|
33
34
|
source_url TEXT,
|
|
34
|
-
content_hash TEXT NOT NULL DEFAULT ''
|
|
35
|
+
content_hash TEXT NOT NULL DEFAULT '',
|
|
36
|
+
page_id TEXT,
|
|
37
|
+
project_id TEXT NOT NULL DEFAULT '',
|
|
38
|
+
heading_breadcrumbs TEXT NOT NULL DEFAULT '[]',
|
|
39
|
+
embedding_text TEXT NOT NULL DEFAULT '',
|
|
40
|
+
original_start INTEGER,
|
|
41
|
+
original_end INTEGER,
|
|
42
|
+
entities TEXT NOT NULL DEFAULT '[]',
|
|
43
|
+
symbols TEXT NOT NULL DEFAULT '[]',
|
|
44
|
+
links TEXT NOT NULL DEFAULT '[]',
|
|
45
|
+
oversized_atomic INTEGER NOT NULL DEFAULT 0,
|
|
46
|
+
provenance TEXT NOT NULL DEFAULT '{}',
|
|
47
|
+
analysis_terms TEXT NOT NULL DEFAULT '[]'
|
|
35
48
|
);
|
|
36
49
|
|
|
37
50
|
CREATE VIRTUAL TABLE IF NOT EXISTS knowledge_fts USING fts5(
|
|
@@ -45,6 +58,15 @@ CREATE VIRTUAL TABLE IF NOT EXISTS knowledge_vec USING vec0(
|
|
|
45
58
|
embedding FLOAT[384]
|
|
46
59
|
);
|
|
47
60
|
|
|
61
|
+
CREATE TABLE IF NOT EXISTS knowledge_vector_profiles (
|
|
62
|
+
id TEXT PRIMARY KEY,
|
|
63
|
+
profile_hash TEXT NOT NULL,
|
|
64
|
+
generation INTEGER NOT NULL CHECK(generation > 0),
|
|
65
|
+
source_content_hash TEXT NOT NULL
|
|
66
|
+
);
|
|
67
|
+
CREATE INDEX IF NOT EXISTS idx_knowledge_vector_profiles_generation
|
|
68
|
+
ON knowledge_vector_profiles(profile_hash, generation);
|
|
69
|
+
|
|
48
70
|
CREATE TABLE IF NOT EXISTS knowledge_source_readiness (
|
|
49
71
|
source TEXT PRIMARY KEY,
|
|
50
72
|
expected_version TEXT,
|
|
@@ -60,7 +82,7 @@ CREATE TABLE IF NOT EXISTS knowledge_source_readiness (
|
|
|
60
82
|
`;
|
|
61
83
|
const COMPILED_KNOWLEDGE_SCHEMA = `
|
|
62
84
|
CREATE TABLE IF NOT EXISTS knowledge_project_profiles (
|
|
63
|
-
|
|
85
|
+
project_id TEXT PRIMARY KEY,
|
|
64
86
|
profile_hash TEXT NOT NULL,
|
|
65
87
|
profile_json TEXT NOT NULL,
|
|
66
88
|
generated_at INTEGER NOT NULL
|
|
@@ -81,7 +103,7 @@ CREATE INDEX IF NOT EXISTS idx_knowledge_source_schemas_source
|
|
|
81
103
|
|
|
82
104
|
CREATE TABLE IF NOT EXISTS knowledge_catalog_snapshots (
|
|
83
105
|
id TEXT PRIMARY KEY,
|
|
84
|
-
|
|
106
|
+
project_id TEXT NOT NULL,
|
|
85
107
|
profile_hash TEXT NOT NULL,
|
|
86
108
|
source_id TEXT NOT NULL,
|
|
87
109
|
schema_id TEXT NOT NULL,
|
|
@@ -90,14 +112,14 @@ CREATE TABLE IF NOT EXISTS knowledge_catalog_snapshots (
|
|
|
90
112
|
status TEXT NOT NULL CHECK(status IN ('active', 'inactive')),
|
|
91
113
|
created_at INTEGER NOT NULL,
|
|
92
114
|
activated_at INTEGER,
|
|
93
|
-
UNIQUE(
|
|
115
|
+
UNIQUE(project_id, source_id, content_hash)
|
|
94
116
|
);
|
|
95
117
|
CREATE UNIQUE INDEX IF NOT EXISTS idx_knowledge_catalog_active_source
|
|
96
|
-
ON knowledge_catalog_snapshots(
|
|
118
|
+
ON knowledge_catalog_snapshots(project_id, source_id) WHERE status = 'active';
|
|
97
119
|
|
|
98
120
|
CREATE TABLE IF NOT EXISTS knowledge_pages (
|
|
99
121
|
id TEXT PRIMARY KEY,
|
|
100
|
-
|
|
122
|
+
project_id TEXT NOT NULL,
|
|
101
123
|
source_id TEXT NOT NULL,
|
|
102
124
|
source_schema_id TEXT NOT NULL,
|
|
103
125
|
title TEXT NOT NULL,
|
|
@@ -113,7 +135,7 @@ CREATE TABLE IF NOT EXISTS knowledge_pages (
|
|
|
113
135
|
compiled_at INTEGER NOT NULL
|
|
114
136
|
);
|
|
115
137
|
CREATE INDEX IF NOT EXISTS idx_knowledge_pages_source
|
|
116
|
-
ON knowledge_pages(
|
|
138
|
+
ON knowledge_pages(project_id, source_id, status, content_hash);
|
|
117
139
|
|
|
118
140
|
CREATE TABLE IF NOT EXISTS knowledge_entity_links (
|
|
119
141
|
page_id TEXT NOT NULL REFERENCES knowledge_pages(id) ON DELETE CASCADE,
|
|
@@ -137,7 +159,7 @@ CREATE INDEX IF NOT EXISTS idx_knowledge_symbol_links_lookup
|
|
|
137
159
|
|
|
138
160
|
CREATE TABLE IF NOT EXISTS knowledge_refresh_jobs (
|
|
139
161
|
id TEXT PRIMARY KEY,
|
|
140
|
-
|
|
162
|
+
project_id TEXT NOT NULL,
|
|
141
163
|
source_id TEXT NOT NULL,
|
|
142
164
|
input_content_hash TEXT NOT NULL,
|
|
143
165
|
previous_snapshot_id TEXT,
|
|
@@ -146,14 +168,14 @@ CREATE TABLE IF NOT EXISTS knowledge_refresh_jobs (
|
|
|
146
168
|
error TEXT,
|
|
147
169
|
created_at INTEGER NOT NULL,
|
|
148
170
|
updated_at INTEGER NOT NULL,
|
|
149
|
-
UNIQUE(
|
|
171
|
+
UNIQUE(project_id, source_id, input_content_hash)
|
|
150
172
|
);
|
|
151
173
|
CREATE INDEX IF NOT EXISTS idx_knowledge_refresh_jobs_source
|
|
152
|
-
ON knowledge_refresh_jobs(
|
|
174
|
+
ON knowledge_refresh_jobs(project_id, source_id, updated_at DESC);
|
|
153
175
|
|
|
154
176
|
CREATE TABLE IF NOT EXISTS knowledge_coverage_gaps (
|
|
155
177
|
id TEXT PRIMARY KEY,
|
|
156
|
-
|
|
178
|
+
project_id TEXT NOT NULL,
|
|
157
179
|
source_id TEXT NOT NULL,
|
|
158
180
|
kind TEXT NOT NULL,
|
|
159
181
|
gap_json TEXT NOT NULL,
|
|
@@ -161,7 +183,7 @@ CREATE TABLE IF NOT EXISTS knowledge_coverage_gaps (
|
|
|
161
183
|
resolved_at INTEGER
|
|
162
184
|
);
|
|
163
185
|
CREATE INDEX IF NOT EXISTS idx_knowledge_coverage_gaps_active
|
|
164
|
-
ON knowledge_coverage_gaps(
|
|
186
|
+
ON knowledge_coverage_gaps(project_id, source_id, resolved_at, detected_at DESC);
|
|
165
187
|
|
|
166
188
|
CREATE TABLE IF NOT EXISTS knowledge_analysis_manifests (
|
|
167
189
|
source_id TEXT NOT NULL,
|
|
@@ -174,15 +196,60 @@ CREATE TABLE IF NOT EXISTS knowledge_analysis_manifests (
|
|
|
174
196
|
);
|
|
175
197
|
|
|
176
198
|
CREATE TABLE IF NOT EXISTS knowledge_raw_sources (
|
|
177
|
-
|
|
199
|
+
project_id TEXT NOT NULL,
|
|
178
200
|
source_id TEXT NOT NULL,
|
|
179
201
|
content_hash TEXT NOT NULL,
|
|
180
202
|
source_uri TEXT NOT NULL,
|
|
181
203
|
source_version TEXT,
|
|
182
204
|
metadata_json TEXT NOT NULL DEFAULT '{}',
|
|
183
205
|
retrieved_at INTEGER NOT NULL,
|
|
184
|
-
PRIMARY KEY(
|
|
206
|
+
PRIMARY KEY(project_id, source_id, content_hash)
|
|
207
|
+
);
|
|
208
|
+
|
|
209
|
+
CREATE TABLE IF NOT EXISTS knowledge_source_aggregates_v1 (
|
|
210
|
+
identity_hash TEXT PRIMARY KEY,
|
|
211
|
+
project_id TEXT NOT NULL,
|
|
212
|
+
source_id TEXT NOT NULL,
|
|
213
|
+
state TEXT NOT NULL,
|
|
214
|
+
revision INTEGER NOT NULL,
|
|
215
|
+
content_hash TEXT,
|
|
216
|
+
aggregate_hash TEXT NOT NULL,
|
|
217
|
+
aggregate_json TEXT NOT NULL,
|
|
218
|
+
updated_at INTEGER NOT NULL
|
|
219
|
+
);
|
|
220
|
+
CREATE INDEX IF NOT EXISTS idx_knowledge_source_aggregates_scope
|
|
221
|
+
ON knowledge_source_aggregates_v1(project_id, source_id, updated_at DESC);
|
|
222
|
+
|
|
223
|
+
CREATE TABLE IF NOT EXISTS knowledge_source_events_v1 (
|
|
224
|
+
event_id TEXT PRIMARY KEY,
|
|
225
|
+
identity_hash TEXT NOT NULL REFERENCES knowledge_source_aggregates_v1(identity_hash),
|
|
226
|
+
sequence INTEGER NOT NULL,
|
|
227
|
+
transition_json TEXT NOT NULL,
|
|
228
|
+
created_at INTEGER NOT NULL,
|
|
229
|
+
UNIQUE(identity_hash, sequence)
|
|
230
|
+
);
|
|
231
|
+
|
|
232
|
+
CREATE TABLE IF NOT EXISTS knowledge_claims_v1 (
|
|
233
|
+
claim_id TEXT PRIMARY KEY,
|
|
234
|
+
identity_hash TEXT NOT NULL REFERENCES knowledge_source_aggregates_v1(identity_hash),
|
|
235
|
+
source_id TEXT NOT NULL,
|
|
236
|
+
page_id TEXT NOT NULL,
|
|
237
|
+
status TEXT NOT NULL,
|
|
238
|
+
source_content_hash TEXT NOT NULL,
|
|
239
|
+
claim_hash TEXT NOT NULL,
|
|
240
|
+
claim_json TEXT NOT NULL,
|
|
241
|
+
created_at INTEGER NOT NULL
|
|
185
242
|
);
|
|
243
|
+
CREATE INDEX IF NOT EXISTS idx_knowledge_claims_source
|
|
244
|
+
ON knowledge_claims_v1(source_id, source_content_hash, status);
|
|
245
|
+
|
|
246
|
+
CREATE TABLE IF NOT EXISTS knowledge_maintenance_runs (
|
|
247
|
+
id TEXT PRIMARY KEY,
|
|
248
|
+
manifest_json TEXT NOT NULL,
|
|
249
|
+
created_at INTEGER NOT NULL
|
|
250
|
+
);
|
|
251
|
+
CREATE INDEX IF NOT EXISTS idx_knowledge_maintenance_runs_created
|
|
252
|
+
ON knowledge_maintenance_runs(created_at DESC);
|
|
186
253
|
`;
|
|
187
254
|
const FTS_TRIGGERS = `
|
|
188
255
|
CREATE TRIGGER IF NOT EXISTS kchunks_ai AFTER INSERT ON knowledge_chunks BEGIN
|
|
@@ -201,8 +268,10 @@ CREATE TRIGGER IF NOT EXISTS kchunks_au AFTER UPDATE ON knowledge_chunks BEGIN
|
|
|
201
268
|
END;
|
|
202
269
|
`;
|
|
203
270
|
export class KnowledgeStore {
|
|
271
|
+
embeddingProfile;
|
|
204
272
|
db;
|
|
205
|
-
constructor(dbPath) {
|
|
273
|
+
constructor(dbPath, embeddingProfile = createMiniLmEmbeddingProfile('knowledge_retrieval')) {
|
|
274
|
+
this.embeddingProfile = embeddingProfile;
|
|
206
275
|
const dir = dirname(dbPath);
|
|
207
276
|
if (!existsSync(dir))
|
|
208
277
|
mkdirSync(dir, { recursive: true });
|
|
@@ -211,66 +280,49 @@ export class KnowledgeStore {
|
|
|
211
280
|
this.db.pragma('busy_timeout = 5000');
|
|
212
281
|
this.db.pragma('foreign_keys = ON');
|
|
213
282
|
sqliteVec.load(this.db);
|
|
283
|
+
this.assertCanonicalSchemaOrEmpty();
|
|
214
284
|
this.db.exec(KNOWLEDGE_SCHEMA);
|
|
215
|
-
this.
|
|
216
|
-
this.migrateCompiledKnowledgeSchema();
|
|
217
|
-
this.ensureTrigramFts();
|
|
285
|
+
this.db.exec(COMPILED_KNOWLEDGE_SCHEMA);
|
|
218
286
|
this.db.exec(FTS_TRIGGERS);
|
|
219
287
|
}
|
|
220
|
-
|
|
221
|
-
const
|
|
222
|
-
if (
|
|
223
|
-
this.db.exec('ALTER TABLE knowledge_docs ADD COLUMN chunk_policy_version INTEGER NOT NULL DEFAULT 0');
|
|
224
|
-
}
|
|
225
|
-
const chunkColumns = this.db.pragma('table_info(knowledge_chunks)');
|
|
226
|
-
const addChunkColumn = (name, sql) => {
|
|
227
|
-
if (!chunkColumns.some(column => column.name === name))
|
|
228
|
-
this.db.exec(sql);
|
|
229
|
-
};
|
|
230
|
-
addChunkColumn('chunk_kind', "ALTER TABLE knowledge_chunks ADD COLUMN chunk_kind TEXT NOT NULL DEFAULT 'overview'");
|
|
231
|
-
addChunkColumn('framework', 'ALTER TABLE knowledge_chunks ADD COLUMN framework TEXT');
|
|
232
|
-
addChunkColumn('api_identifiers', "ALTER TABLE knowledge_chunks ADD COLUMN api_identifiers TEXT NOT NULL DEFAULT '[]'");
|
|
233
|
-
addChunkColumn('source_url', 'ALTER TABLE knowledge_chunks ADD COLUMN source_url TEXT');
|
|
234
|
-
addChunkColumn('content_hash', "ALTER TABLE knowledge_chunks ADD COLUMN content_hash TEXT NOT NULL DEFAULT ''");
|
|
235
|
-
}
|
|
236
|
-
migrateCompiledKnowledgeSchema() {
|
|
237
|
-
this.db.exec(COMPILED_KNOWLEDGE_SCHEMA);
|
|
238
|
-
const chunkColumns = this.db.pragma('table_info(knowledge_chunks)');
|
|
239
|
-
const add = (name, sql) => {
|
|
240
|
-
if (!chunkColumns.some(column => column.name === name))
|
|
241
|
-
this.db.exec(sql);
|
|
242
|
-
};
|
|
243
|
-
add('page_id', 'ALTER TABLE knowledge_chunks ADD COLUMN page_id TEXT');
|
|
244
|
-
add('project_root', "ALTER TABLE knowledge_chunks ADD COLUMN project_root TEXT NOT NULL DEFAULT ''");
|
|
245
|
-
add('heading_breadcrumbs', "ALTER TABLE knowledge_chunks ADD COLUMN heading_breadcrumbs TEXT NOT NULL DEFAULT '[]'");
|
|
246
|
-
add('embedding_text', "ALTER TABLE knowledge_chunks ADD COLUMN embedding_text TEXT NOT NULL DEFAULT ''");
|
|
247
|
-
add('original_start', 'ALTER TABLE knowledge_chunks ADD COLUMN original_start INTEGER');
|
|
248
|
-
add('original_end', 'ALTER TABLE knowledge_chunks ADD COLUMN original_end INTEGER');
|
|
249
|
-
add('entities', "ALTER TABLE knowledge_chunks ADD COLUMN entities TEXT NOT NULL DEFAULT '[]'");
|
|
250
|
-
add('symbols', "ALTER TABLE knowledge_chunks ADD COLUMN symbols TEXT NOT NULL DEFAULT '[]'");
|
|
251
|
-
add('links', "ALTER TABLE knowledge_chunks ADD COLUMN links TEXT NOT NULL DEFAULT '[]'");
|
|
252
|
-
add('oversized_atomic', 'ALTER TABLE knowledge_chunks ADD COLUMN oversized_atomic INTEGER NOT NULL DEFAULT 0');
|
|
253
|
-
add('provenance', "ALTER TABLE knowledge_chunks ADD COLUMN provenance TEXT NOT NULL DEFAULT '{}'");
|
|
254
|
-
add('analysis_terms', "ALTER TABLE knowledge_chunks ADD COLUMN analysis_terms TEXT NOT NULL DEFAULT '[]'");
|
|
255
|
-
}
|
|
256
|
-
ensureTrigramFts() {
|
|
257
|
-
const row = this.db.prepare("SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'knowledge_fts'").get();
|
|
258
|
-
if (/tokenize\s*=\s*['\"]?trigram/i.test(row?.sql ?? ''))
|
|
288
|
+
assertCanonicalSchemaOrEmpty() {
|
|
289
|
+
const docs = this.tableColumns('knowledge_docs');
|
|
290
|
+
if (docs.length === 0)
|
|
259
291
|
return;
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
292
|
+
const requiredColumns = new Map([
|
|
293
|
+
['knowledge_docs', ['source', 'url', 'title', 'version', 'content_hash', 'status', 'chunk_policy_version']],
|
|
294
|
+
['knowledge_chunks', [
|
|
295
|
+
'doc_id', 'chunk_index', 'heading_path', 'content', 'code_blocks', 'token_count',
|
|
296
|
+
'chunk_kind', 'framework', 'api_identifiers', 'source_url', 'content_hash', 'page_id',
|
|
297
|
+
'project_id', 'heading_breadcrumbs', 'embedding_text', 'original_start', 'original_end',
|
|
298
|
+
'entities', 'symbols', 'links', 'oversized_atomic', 'provenance', 'analysis_terms',
|
|
299
|
+
]],
|
|
300
|
+
['knowledge_vector_profiles', ['profile_hash', 'generation', 'source_content_hash']],
|
|
301
|
+
]);
|
|
302
|
+
const requiredTables = [
|
|
303
|
+
'knowledge_fts', 'knowledge_vec', 'knowledge_source_readiness', 'knowledge_project_profiles',
|
|
304
|
+
'knowledge_source_schemas', 'knowledge_catalog_snapshots', 'knowledge_pages',
|
|
305
|
+
'knowledge_entity_links', 'knowledge_symbol_links', 'knowledge_refresh_jobs',
|
|
306
|
+
'knowledge_coverage_gaps', 'knowledge_analysis_manifests', 'knowledge_raw_sources',
|
|
307
|
+
'knowledge_source_aggregates_v1', 'knowledge_source_events_v1', 'knowledge_claims_v1',
|
|
308
|
+
'knowledge_maintenance_runs',
|
|
309
|
+
];
|
|
310
|
+
const schemaRows = this.db.prepare("SELECT name, sql FROM sqlite_master WHERE type IN ('table', 'view')").all();
|
|
311
|
+
const tables = new Map(schemaRows.map(row => [row.name, row.sql ?? '']));
|
|
312
|
+
const invalid = [
|
|
313
|
+
...requiredTables.filter(table => !tables.has(table)).map(table => `table:${table}`),
|
|
314
|
+
...[...requiredColumns].flatMap(([table, columns]) => {
|
|
315
|
+
const actual = new Set(this.tableColumns(table));
|
|
316
|
+
return columns.filter(column => !actual.has(column)).map(column => `column:${table}.${column}`);
|
|
317
|
+
}),
|
|
318
|
+
...(/tokenize\s*=\s*['\"]?trigram/i.test(tables.get('knowledge_fts') ?? '')
|
|
319
|
+
? [] : ['table:knowledge_fts.tokenizer']),
|
|
320
|
+
];
|
|
321
|
+
if (invalid.length > 0)
|
|
322
|
+
throw new Error(`KNOWLEDGE_SCHEMA_UNSUPPORTED:${invalid.join(',')}`);
|
|
323
|
+
}
|
|
324
|
+
tableColumns(table) {
|
|
325
|
+
return this.db.pragma(`table_info(${table})`).map(row => row.name);
|
|
274
326
|
}
|
|
275
327
|
upsertDoc(doc) {
|
|
276
328
|
const title = resolveKnowledgeTitle({
|
|
@@ -282,10 +334,10 @@ export class KnowledgeStore {
|
|
|
282
334
|
throw new Error(`Knowledge document ${doc.id} has no meaningful title`);
|
|
283
335
|
const existing = this.db.prepare('SELECT id FROM knowledge_docs WHERE id = ?').get(doc.id);
|
|
284
336
|
if (existing) {
|
|
285
|
-
this.db.prepare("UPDATE knowledge_docs SET title=?, version=?, fetched_at=?, content_hash=?, chunk_policy_version=?, status='active' WHERE id=?").run(title, doc.version ?? null, Date.now(), doc.contentHash ?? null, doc.chunkPolicyVersion ?? KNOWLEDGE_CHUNK_POLICY_VERSION, doc.id);
|
|
337
|
+
this.db.prepare("UPDATE knowledge_docs SET title=?, version=?, fetched_at=?, content_hash=?, chunk_policy_version=?, status='active' WHERE id=?").run(title, doc.version ?? null, doc.fetchedAt ?? Date.now(), doc.contentHash ?? null, doc.chunkPolicyVersion ?? KNOWLEDGE_CHUNK_POLICY_VERSION, doc.id);
|
|
286
338
|
}
|
|
287
339
|
else {
|
|
288
|
-
this.db.prepare('INSERT INTO knowledge_docs (id, source, url, title, version, fetched_at, content_hash, chunk_policy_version) VALUES (?,?,?,?,?,?,?,?)').run(doc.id, doc.source, doc.url, title, doc.version ?? null, Date.now(), doc.contentHash ?? null, doc.chunkPolicyVersion ?? KNOWLEDGE_CHUNK_POLICY_VERSION);
|
|
340
|
+
this.db.prepare('INSERT INTO knowledge_docs (id, source, url, title, version, fetched_at, content_hash, chunk_policy_version) VALUES (?,?,?,?,?,?,?,?)').run(doc.id, doc.source, doc.url, title, doc.version ?? null, doc.fetchedAt ?? Date.now(), doc.contentHash ?? null, doc.chunkPolicyVersion ?? KNOWLEDGE_CHUNK_POLICY_VERSION);
|
|
289
341
|
}
|
|
290
342
|
}
|
|
291
343
|
clearChunks(docId) {
|
|
@@ -311,8 +363,12 @@ export class KnowledgeStore {
|
|
|
311
363
|
VALUES (?,?,?,?,?,?,?,?,?,?,?,?)`).run(chunk.id, chunk.docId, chunk.index, headingPath, chunk.content, chunk.codeBlocks ?? null, chunk.tokenCount ?? null, metadata.kind, metadata.framework ?? null, JSON.stringify(metadata.apiIdentifiers), metadata.sourceUrl ?? null, metadata.contentHash);
|
|
312
364
|
}
|
|
313
365
|
insertEmbedding(id, embedding) {
|
|
366
|
+
this.assertEmbeddingProfile(embedding);
|
|
314
367
|
const buffer = Buffer.from(embedding.buffer, embedding.byteOffset, embedding.byteLength);
|
|
315
|
-
this.db.
|
|
368
|
+
this.db.transaction(() => {
|
|
369
|
+
this.db.prepare('INSERT OR REPLACE INTO knowledge_vec(id, embedding) VALUES (?, ?)').run(id, buffer);
|
|
370
|
+
this.putEmbeddingProfile(id);
|
|
371
|
+
})();
|
|
316
372
|
}
|
|
317
373
|
insertEmbeddingBatch(entries) {
|
|
318
374
|
if (entries.length === 0)
|
|
@@ -320,7 +376,9 @@ export class KnowledgeStore {
|
|
|
320
376
|
const insert = this.db.prepare('INSERT OR REPLACE INTO knowledge_vec(id, embedding) VALUES (?, ?)');
|
|
321
377
|
this.db.transaction(() => {
|
|
322
378
|
for (const { id, embedding } of entries) {
|
|
379
|
+
this.assertEmbeddingProfile(embedding);
|
|
323
380
|
insert.run(id, Buffer.from(embedding.buffer, embedding.byteOffset, embedding.byteLength));
|
|
381
|
+
this.putEmbeddingProfile(id);
|
|
324
382
|
}
|
|
325
383
|
})();
|
|
326
384
|
}
|
|
@@ -338,14 +396,18 @@ export class KnowledgeStore {
|
|
|
338
396
|
const insertVector = this.db.prepare('INSERT INTO knowledge_vec(id, embedding) VALUES (?, ?)');
|
|
339
397
|
this.db.transaction(() => {
|
|
340
398
|
this.upsertDoc(normalized.doc);
|
|
341
|
-
for (const { id } of oldIds)
|
|
399
|
+
for (const { id } of oldIds) {
|
|
342
400
|
deleteVector.run(id);
|
|
401
|
+
this.deleteEmbeddingProfile(id);
|
|
402
|
+
}
|
|
343
403
|
this.clearChunks(input.doc.id);
|
|
344
404
|
for (let index = 0; index < normalized.chunks.length; index++) {
|
|
345
405
|
const chunk = normalized.chunks[index];
|
|
346
406
|
const embedding = input.embeddings[index];
|
|
407
|
+
this.assertEmbeddingProfile(embedding);
|
|
347
408
|
insertChunk.run(chunk.id, chunk.docId, chunk.index, chunk.headingPath, chunk.content, chunk.codeBlocks ?? null, chunk.tokenCount ?? null, chunk.kind, chunk.framework ?? null, JSON.stringify(chunk.apiIdentifiers ?? []), chunk.sourceUrl ?? null, chunk.contentHash);
|
|
348
409
|
insertVector.run(chunk.id, Buffer.from(embedding.buffer, embedding.byteOffset, embedding.byteLength));
|
|
410
|
+
this.putEmbeddingProfile(chunk.id);
|
|
349
411
|
}
|
|
350
412
|
})();
|
|
351
413
|
}
|
|
@@ -359,8 +421,10 @@ export class KnowledgeStore {
|
|
|
359
421
|
VALUES (?,?,?,?,?,?,?,?,?,?,?,?)`);
|
|
360
422
|
this.db.transaction(() => {
|
|
361
423
|
this.upsertDoc(normalized.doc);
|
|
362
|
-
for (const { id } of oldIds)
|
|
424
|
+
for (const { id } of oldIds) {
|
|
363
425
|
deleteVector.run(id);
|
|
426
|
+
this.deleteEmbeddingProfile(id);
|
|
427
|
+
}
|
|
364
428
|
this.clearChunks(input.doc.id);
|
|
365
429
|
for (const chunk of normalized.chunks) {
|
|
366
430
|
insertChunk.run(chunk.id, chunk.docId, chunk.index, chunk.headingPath, chunk.content, chunk.codeBlocks ?? null, chunk.tokenCount ?? null, chunk.kind, chunk.framework ?? null, JSON.stringify(chunk.apiIdentifiers ?? []), chunk.sourceUrl ?? null, chunk.contentHash);
|
|
@@ -432,14 +496,54 @@ export class KnowledgeStore {
|
|
|
432
496
|
const rows = this.db.prepare(`
|
|
433
497
|
SELECT v.id, vec_distance_cosine(v.embedding, ?) as distance
|
|
434
498
|
FROM knowledge_vec v
|
|
499
|
+
JOIN knowledge_vector_profiles p ON p.id = v.id
|
|
435
500
|
JOIN knowledge_chunks c ON c.id = v.id
|
|
436
501
|
JOIN knowledge_docs d ON d.id = c.doc_id
|
|
437
502
|
WHERE d.status = 'active'
|
|
503
|
+
AND p.profile_hash = ? AND p.generation = ?
|
|
504
|
+
AND p.source_content_hash = c.content_hash
|
|
438
505
|
${sourceFilter}
|
|
439
506
|
ORDER BY distance LIMIT ?
|
|
440
|
-
`).all(buffer, ...(sources ?? []), limit);
|
|
507
|
+
`).all(buffer, this.embeddingProfile.profileHash, this.embeddingProfile.generation, ...(sources ?? []), limit);
|
|
441
508
|
return rows;
|
|
442
509
|
}
|
|
510
|
+
getEmbeddingProfileState(source) {
|
|
511
|
+
const total = this.countChunks(source);
|
|
512
|
+
const sourceJoin = 'JOIN knowledge_vec v ON v.id = p.id JOIN knowledge_chunks c ON c.id = p.id JOIN knowledge_docs d ON d.id = c.doc_id';
|
|
513
|
+
const sourceWhere = source ? 'AND d.source = ?' : '';
|
|
514
|
+
const indexed = Number(this.db.prepare(`SELECT COUNT(*) AS count
|
|
515
|
+
FROM knowledge_vector_profiles p ${sourceJoin}
|
|
516
|
+
WHERE p.profile_hash = ? AND p.generation = ? AND p.source_content_hash = c.content_hash
|
|
517
|
+
AND d.status = 'active' ${sourceWhere}`).get(this.embeddingProfile.profileHash, this.embeddingProfile.generation, ...(source ? [source] : [])).count);
|
|
518
|
+
return {
|
|
519
|
+
status: this.embeddingProfile.dimension !== 384
|
|
520
|
+
? 'dimension_mismatch'
|
|
521
|
+
: indexed < total ? 'backfill_pending' : 'ready',
|
|
522
|
+
profileHash: this.embeddingProfile.profileHash,
|
|
523
|
+
generation: this.embeddingProfile.generation,
|
|
524
|
+
indexed,
|
|
525
|
+
total,
|
|
526
|
+
};
|
|
527
|
+
}
|
|
528
|
+
putEmbeddingProfile(id) {
|
|
529
|
+
const source = this.db.prepare('SELECT content_hash, content FROM knowledge_chunks WHERE id = ?')
|
|
530
|
+
.get(id);
|
|
531
|
+
if (!source)
|
|
532
|
+
throw new Error(`EMBEDDING_SOURCE_MISSING:knowledge:${id}`);
|
|
533
|
+
const sourceContentHash = source.content_hash
|
|
534
|
+
|| createHash('sha256').update(source.content).digest('hex');
|
|
535
|
+
this.db.prepare(`INSERT OR REPLACE INTO knowledge_vector_profiles
|
|
536
|
+
(id, profile_hash, generation, source_content_hash) VALUES (?, ?, ?, ?)`)
|
|
537
|
+
.run(id, this.embeddingProfile.profileHash, this.embeddingProfile.generation, sourceContentHash);
|
|
538
|
+
}
|
|
539
|
+
deleteEmbeddingProfile(id) {
|
|
540
|
+
this.db.prepare('DELETE FROM knowledge_vector_profiles WHERE id = ?').run(id);
|
|
541
|
+
}
|
|
542
|
+
assertEmbeddingProfile(embedding) {
|
|
543
|
+
if (embedding.length !== this.embeddingProfile.dimension || embedding.length !== 384) {
|
|
544
|
+
throw new Error(`EMBEDDING_DIMENSION_MISMATCH:${embedding.length}:${this.embeddingProfile.dimension}`);
|
|
545
|
+
}
|
|
546
|
+
}
|
|
443
547
|
getChunk(id) {
|
|
444
548
|
return this.db.prepare('SELECT * FROM knowledge_chunks WHERE id = ?').get(id);
|
|
445
549
|
}
|
|
@@ -473,8 +577,8 @@ export class KnowledgeStore {
|
|
|
473
577
|
`).all(...params, limit);
|
|
474
578
|
}
|
|
475
579
|
listMissingEmbeddingChunks(options = {}) {
|
|
476
|
-
const conditions = ["v.id IS NULL", "d.status = 'active'"];
|
|
477
|
-
const params = [];
|
|
580
|
+
const conditions = ["(v.id IS NULL OR p.id IS NULL OR p.source_content_hash <> c.content_hash)", "d.status = 'active'"];
|
|
581
|
+
const params = [this.embeddingProfile.profileHash, this.embeddingProfile.generation];
|
|
478
582
|
if (options.source) {
|
|
479
583
|
conditions.push('d.source = ?');
|
|
480
584
|
params.push(options.source);
|
|
@@ -488,6 +592,8 @@ export class KnowledgeStore {
|
|
|
488
592
|
FROM knowledge_chunks c
|
|
489
593
|
JOIN knowledge_docs d ON d.id = c.doc_id
|
|
490
594
|
LEFT JOIN knowledge_vec v ON v.id = c.id
|
|
595
|
+
LEFT JOIN knowledge_vector_profiles p ON p.id = v.id
|
|
596
|
+
AND p.profile_hash = ? AND p.generation = ?
|
|
491
597
|
WHERE ${conditions.join(' AND ')}
|
|
492
598
|
ORDER BY c.id
|
|
493
599
|
LIMIT ?
|
|
@@ -509,14 +615,21 @@ export class KnowledgeStore {
|
|
|
509
615
|
const row = source
|
|
510
616
|
? this.db.prepare(`
|
|
511
617
|
SELECT COUNT(*) AS c FROM knowledge_vec v
|
|
618
|
+
JOIN knowledge_vector_profiles p ON p.id = v.id
|
|
512
619
|
JOIN knowledge_chunks c ON c.id = v.id
|
|
513
|
-
JOIN knowledge_docs d ON d.id = c.doc_id
|
|
514
|
-
|
|
620
|
+
JOIN knowledge_docs d ON d.id = c.doc_id
|
|
621
|
+
WHERE p.profile_hash = ? AND p.generation = ?
|
|
622
|
+
AND p.source_content_hash = c.content_hash
|
|
623
|
+
AND d.source = ? AND d.status = 'active'
|
|
624
|
+
`).get(this.embeddingProfile.profileHash, this.embeddingProfile.generation, source)
|
|
515
625
|
: this.db.prepare(`
|
|
516
626
|
SELECT COUNT(*) AS c FROM knowledge_vec v
|
|
627
|
+
JOIN knowledge_vector_profiles p ON p.id = v.id
|
|
517
628
|
JOIN knowledge_chunks c ON c.id = v.id
|
|
518
|
-
JOIN knowledge_docs d ON d.id = c.doc_id
|
|
519
|
-
|
|
629
|
+
JOIN knowledge_docs d ON d.id = c.doc_id
|
|
630
|
+
WHERE p.profile_hash = ? AND p.generation = ?
|
|
631
|
+
AND p.source_content_hash = c.content_hash AND d.status = 'active'
|
|
632
|
+
`).get(this.embeddingProfile.profileHash, this.embeddingProfile.generation);
|
|
520
633
|
return row.c;
|
|
521
634
|
}
|
|
522
635
|
getCoverageBySource() {
|
|
@@ -530,7 +643,7 @@ export class KnowledgeStore {
|
|
|
530
643
|
`).all();
|
|
531
644
|
return rows;
|
|
532
645
|
}
|
|
533
|
-
refreshSourceReadiness(expectedVersions = {}) {
|
|
646
|
+
refreshSourceReadiness(expectedVersions = {}, operationTimestamp = Date.now()) {
|
|
534
647
|
const rows = this.db.prepare(`
|
|
535
648
|
SELECT d.source,
|
|
536
649
|
COUNT(DISTINCT d.id) AS documents,
|
|
@@ -569,7 +682,7 @@ export class KnowledgeStore {
|
|
|
569
682
|
: coverage < 0.95
|
|
570
683
|
? 'vector_coverage'
|
|
571
684
|
: row.chunks === 0 ? 'empty_source' : null;
|
|
572
|
-
upsert.run(row.source, expectedVersions[row.source] ?? null, row.documents, row.chunks, row.vectors, coverage, row.last_refresh, status, failure,
|
|
685
|
+
upsert.run(row.source, expectedVersions[row.source] ?? null, row.documents, row.chunks, row.vectors, coverage, row.last_refresh, status, failure, operationTimestamp);
|
|
573
686
|
}
|
|
574
687
|
})();
|
|
575
688
|
}
|
|
@@ -619,21 +732,21 @@ export class KnowledgeStore {
|
|
|
619
732
|
};
|
|
620
733
|
}
|
|
621
734
|
upsertProjectProfile(profile) {
|
|
622
|
-
const existing = this.db.prepare('SELECT profile_hash FROM knowledge_project_profiles WHERE
|
|
735
|
+
const existing = this.db.prepare('SELECT profile_hash FROM knowledge_project_profiles WHERE project_id = ?').get(profile.projectId);
|
|
623
736
|
if (existing?.profile_hash === profile.profileHash)
|
|
624
737
|
return false;
|
|
625
738
|
this.db.prepare(`
|
|
626
|
-
INSERT INTO knowledge_project_profiles (
|
|
739
|
+
INSERT INTO knowledge_project_profiles (project_id, profile_hash, profile_json, generated_at)
|
|
627
740
|
VALUES (?, ?, ?, ?)
|
|
628
|
-
ON CONFLICT(
|
|
741
|
+
ON CONFLICT(project_id) DO UPDATE SET
|
|
629
742
|
profile_hash = excluded.profile_hash,
|
|
630
743
|
profile_json = excluded.profile_json,
|
|
631
744
|
generated_at = excluded.generated_at
|
|
632
|
-
`).run(profile.
|
|
745
|
+
`).run(profile.projectId, profile.profileHash, JSON.stringify(profile), profile.generatedAt);
|
|
633
746
|
return true;
|
|
634
747
|
}
|
|
635
|
-
getProjectProfile(
|
|
636
|
-
const row = this.db.prepare('SELECT profile_json FROM knowledge_project_profiles WHERE
|
|
748
|
+
getProjectProfile(projectId) {
|
|
749
|
+
const row = this.db.prepare('SELECT profile_json FROM knowledge_project_profiles WHERE project_id = ?').get(projectId);
|
|
637
750
|
return row ? JSON.parse(row.profile_json) : null;
|
|
638
751
|
}
|
|
639
752
|
upsertSourceSchema(schema, now = Date.now()) {
|
|
@@ -665,7 +778,7 @@ export class KnowledgeStore {
|
|
|
665
778
|
activateCatalogEntry(input) {
|
|
666
779
|
const activatedAt = input.activatedAt ?? Date.now();
|
|
667
780
|
const snapshotId = `catalog:${createHash('sha256').update([
|
|
668
|
-
input.
|
|
781
|
+
input.projectId,
|
|
669
782
|
input.entry.sourceId,
|
|
670
783
|
input.entry.schemaId,
|
|
671
784
|
input.entry.contentHash,
|
|
@@ -673,12 +786,12 @@ export class KnowledgeStore {
|
|
|
673
786
|
return this.db.transaction(() => {
|
|
674
787
|
const existing = this.db.prepare(`
|
|
675
788
|
SELECT id, status, entry_json FROM knowledge_catalog_snapshots
|
|
676
|
-
WHERE
|
|
677
|
-
`).get(input.
|
|
789
|
+
WHERE project_id = ? AND source_id = ? AND content_hash = ?
|
|
790
|
+
`).get(input.projectId, input.entry.sourceId, input.entry.contentHash);
|
|
678
791
|
const active = this.db.prepare(`
|
|
679
792
|
SELECT id FROM knowledge_catalog_snapshots
|
|
680
|
-
WHERE
|
|
681
|
-
`).get(input.
|
|
793
|
+
WHERE project_id = ? AND source_id = ? AND status = 'active'
|
|
794
|
+
`).get(input.projectId, input.entry.sourceId);
|
|
682
795
|
if (existing?.status === 'active' && active?.id === existing.id) {
|
|
683
796
|
const entryJson = JSON.stringify(input.entry);
|
|
684
797
|
if (existing.entry_json !== entryJson) {
|
|
@@ -692,8 +805,8 @@ export class KnowledgeStore {
|
|
|
692
805
|
}
|
|
693
806
|
this.db.prepare(`
|
|
694
807
|
UPDATE knowledge_catalog_snapshots SET status = 'inactive'
|
|
695
|
-
WHERE
|
|
696
|
-
`).run(input.
|
|
808
|
+
WHERE project_id = ? AND source_id = ? AND status = 'active'
|
|
809
|
+
`).run(input.projectId, input.entry.sourceId);
|
|
697
810
|
if (existing) {
|
|
698
811
|
this.db.prepare(`
|
|
699
812
|
UPDATE knowledge_catalog_snapshots
|
|
@@ -704,19 +817,19 @@ export class KnowledgeStore {
|
|
|
704
817
|
}
|
|
705
818
|
this.db.prepare(`
|
|
706
819
|
INSERT INTO knowledge_catalog_snapshots
|
|
707
|
-
(id,
|
|
820
|
+
(id, project_id, profile_hash, source_id, schema_id, content_hash,
|
|
708
821
|
entry_json, status, created_at, activated_at)
|
|
709
822
|
VALUES (?, ?, ?, ?, ?, ?, ?, 'active', ?, ?)
|
|
710
|
-
`).run(snapshotId, input.
|
|
823
|
+
`).run(snapshotId, input.projectId, input.profileHash, input.entry.sourceId, input.entry.schemaId, input.entry.contentHash, JSON.stringify(input.entry), activatedAt, activatedAt);
|
|
711
824
|
return { snapshotId, changed: true };
|
|
712
825
|
})();
|
|
713
826
|
}
|
|
714
|
-
listActiveCatalogEntries(
|
|
827
|
+
listActiveCatalogEntries(projectId) {
|
|
715
828
|
return this.db.prepare(`
|
|
716
829
|
SELECT id, entry_json FROM knowledge_catalog_snapshots
|
|
717
|
-
WHERE
|
|
830
|
+
WHERE project_id = ? AND status = 'active'
|
|
718
831
|
ORDER BY source_id
|
|
719
|
-
`).all(
|
|
832
|
+
`).all(projectId).map(row => ({
|
|
720
833
|
...JSON.parse(row.entry_json),
|
|
721
834
|
activeRevisionId: row.id,
|
|
722
835
|
}));
|
|
@@ -724,7 +837,7 @@ export class KnowledgeStore {
|
|
|
724
837
|
beginRefreshJob(input) {
|
|
725
838
|
const now = input.now ?? Date.now();
|
|
726
839
|
const id = `refresh:${createHash('sha256').update([
|
|
727
|
-
input.
|
|
840
|
+
input.projectId,
|
|
728
841
|
input.sourceId,
|
|
729
842
|
input.inputContentHash,
|
|
730
843
|
].join('\0')).digest('hex').slice(0, 32)}`;
|
|
@@ -733,16 +846,16 @@ export class KnowledgeStore {
|
|
|
733
846
|
return { id, status: 'completed', resumed: false };
|
|
734
847
|
const active = this.db.prepare(`
|
|
735
848
|
SELECT id FROM knowledge_catalog_snapshots
|
|
736
|
-
WHERE
|
|
737
|
-
`).get(input.
|
|
849
|
+
WHERE project_id = ? AND source_id = ? AND status = 'active'
|
|
850
|
+
`).get(input.projectId, input.sourceId);
|
|
738
851
|
this.db.prepare(`
|
|
739
852
|
INSERT INTO knowledge_refresh_jobs
|
|
740
|
-
(id,
|
|
853
|
+
(id, project_id, source_id, input_content_hash, previous_snapshot_id,
|
|
741
854
|
status, created_at, updated_at)
|
|
742
855
|
VALUES (?, ?, ?, ?, ?, 'compiling', ?, ?)
|
|
743
856
|
ON CONFLICT(id) DO UPDATE SET
|
|
744
857
|
status = 'compiling', error = NULL, updated_at = excluded.updated_at
|
|
745
|
-
`).run(id, input.
|
|
858
|
+
`).run(id, input.projectId, input.sourceId, input.inputContentHash, active?.id ?? null, now, now);
|
|
746
859
|
return { id, status: 'compiling', resumed: existing !== undefined };
|
|
747
860
|
}
|
|
748
861
|
completeRefreshJob(id, outputSnapshotId, now = Date.now()) {
|
|
@@ -765,13 +878,13 @@ export class KnowledgeStore {
|
|
|
765
878
|
recordCoverageGap(gap) {
|
|
766
879
|
const result = this.db.prepare(`
|
|
767
880
|
INSERT INTO knowledge_coverage_gaps
|
|
768
|
-
(id,
|
|
881
|
+
(id, project_id, source_id, kind, gap_json, detected_at, resolved_at)
|
|
769
882
|
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
770
883
|
ON CONFLICT(id) DO UPDATE SET
|
|
771
884
|
gap_json = excluded.gap_json,
|
|
772
885
|
detected_at = excluded.detected_at,
|
|
773
886
|
resolved_at = excluded.resolved_at
|
|
774
|
-
`).run(gap.id, gap.
|
|
887
|
+
`).run(gap.id, gap.projectId, gap.sourceId, gap.kind, JSON.stringify(gap), gap.detectedAt, gap.resolvedAt ?? null);
|
|
775
888
|
return result.changes === 1;
|
|
776
889
|
}
|
|
777
890
|
resolveCoverageGap(id, resolvedAt = Date.now()) {
|
|
@@ -784,18 +897,18 @@ export class KnowledgeStore {
|
|
|
784
897
|
UPDATE knowledge_coverage_gaps SET resolved_at = ?, gap_json = ? WHERE id = ?
|
|
785
898
|
`).run(resolvedAt, JSON.stringify(gap), id).changes === 1;
|
|
786
899
|
}
|
|
787
|
-
listCoverageGaps(
|
|
900
|
+
listCoverageGaps(projectId, options = {}) {
|
|
788
901
|
const sql = options.includeResolved
|
|
789
|
-
? 'SELECT gap_json FROM knowledge_coverage_gaps WHERE
|
|
790
|
-
: 'SELECT gap_json FROM knowledge_coverage_gaps WHERE
|
|
791
|
-
return this.db.prepare(sql).all(
|
|
902
|
+
? 'SELECT gap_json FROM knowledge_coverage_gaps WHERE project_id = ? ORDER BY detected_at DESC, id'
|
|
903
|
+
: 'SELECT gap_json FROM knowledge_coverage_gaps WHERE project_id = ? AND resolved_at IS NULL ORDER BY detected_at DESC, id';
|
|
904
|
+
return this.db.prepare(sql).all(projectId).map(row => JSON.parse(row.gap_json));
|
|
792
905
|
}
|
|
793
906
|
upsertNormalizedPage(page) {
|
|
794
907
|
if (page.provenance.contentHash !== page.contentHash || page.provenance.immutable !== true) {
|
|
795
908
|
throw new Error(`Knowledge page ${page.id} has invalid immutable provenance`);
|
|
796
909
|
}
|
|
797
|
-
const existing = this.db.prepare('SELECT
|
|
798
|
-
if (existing && (existing.
|
|
910
|
+
const existing = this.db.prepare('SELECT project_id, source_id, content_hash FROM knowledge_pages WHERE id = ?').get(page.id);
|
|
911
|
+
if (existing && (existing.project_id !== page.projectId || existing.source_id !== page.sourceId)) {
|
|
799
912
|
throw new Error(`Knowledge page ${page.id} cannot move across project or source boundaries`);
|
|
800
913
|
}
|
|
801
914
|
if (existing?.content_hash === page.contentHash)
|
|
@@ -803,7 +916,7 @@ export class KnowledgeStore {
|
|
|
803
916
|
this.db.transaction(() => {
|
|
804
917
|
this.db.prepare(`
|
|
805
918
|
INSERT INTO knowledge_pages
|
|
806
|
-
(id,
|
|
919
|
+
(id, project_id, source_id, source_schema_id, title, markdown, frontmatter,
|
|
807
920
|
entities, symbols, links, version, content_hash, provenance, status, compiled_at)
|
|
808
921
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'active', ?)
|
|
809
922
|
ON CONFLICT(id) DO UPDATE SET
|
|
@@ -819,14 +932,14 @@ export class KnowledgeStore {
|
|
|
819
932
|
provenance = excluded.provenance,
|
|
820
933
|
status = 'active',
|
|
821
934
|
compiled_at = excluded.compiled_at
|
|
822
|
-
`).run(page.id, page.
|
|
935
|
+
`).run(page.id, page.projectId, page.sourceId, page.sourceSchemaId, page.title, page.markdown, JSON.stringify(page.frontmatter), JSON.stringify(page.entities), JSON.stringify(page.symbols), JSON.stringify(page.links), page.version ?? null, page.contentHash, JSON.stringify(page.provenance), page.compiledAt);
|
|
823
936
|
this.replacePageLinks(page);
|
|
824
937
|
})();
|
|
825
938
|
return true;
|
|
826
939
|
}
|
|
827
|
-
replaceCompiledPage(page, chunks) {
|
|
940
|
+
replaceCompiledPage(page, chunks, operationTimestamp = page.compiledAt) {
|
|
828
941
|
if (chunks.some(chunk => chunk.pageId !== page.id
|
|
829
|
-
|| chunk.
|
|
942
|
+
|| chunk.projectId !== page.projectId
|
|
830
943
|
|| chunk.sourceId !== page.sourceId
|
|
831
944
|
|| chunk.provenance.contentHash !== page.contentHash)) {
|
|
832
945
|
throw new Error(`Compiled chunks for ${page.id} do not share page provenance`);
|
|
@@ -837,7 +950,7 @@ export class KnowledgeStore {
|
|
|
837
950
|
INSERT INTO knowledge_chunks
|
|
838
951
|
(id, doc_id, chunk_index, heading_path, content, code_blocks, token_count,
|
|
839
952
|
chunk_kind, framework, api_identifiers, source_url, content_hash,
|
|
840
|
-
page_id,
|
|
953
|
+
page_id, project_id, heading_breadcrumbs, embedding_text, original_start,
|
|
841
954
|
original_end, entities, symbols, links, oversized_atomic, provenance, analysis_terms)
|
|
842
955
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
843
956
|
`);
|
|
@@ -851,9 +964,12 @@ export class KnowledgeStore {
|
|
|
851
964
|
version: page.version,
|
|
852
965
|
contentHash: page.contentHash,
|
|
853
966
|
chunkPolicyVersion: KNOWLEDGE_CHUNK_POLICY_VERSION,
|
|
967
|
+
fetchedAt: operationTimestamp,
|
|
854
968
|
});
|
|
855
|
-
for (const row of oldIds)
|
|
969
|
+
for (const row of oldIds) {
|
|
856
970
|
deleteVector.run(row.id);
|
|
971
|
+
this.deleteEmbeddingProfile(row.id);
|
|
972
|
+
}
|
|
857
973
|
this.db.prepare('DELETE FROM knowledge_chunks WHERE page_id = ? OR doc_id = ?')
|
|
858
974
|
.run(page.id, page.id);
|
|
859
975
|
const insertEntity = this.db.prepare(`
|
|
@@ -865,7 +981,7 @@ export class KnowledgeStore {
|
|
|
865
981
|
(page_id, chunk_id, symbol, normalized_symbol) VALUES (?, ?, ?, ?)
|
|
866
982
|
`);
|
|
867
983
|
for (const chunk of chunks) {
|
|
868
|
-
insert.run(chunk.id, page.id, chunk.index, chunk.headingBreadcrumbs.join(' > '), chunk.content, extractCodeBlocks(chunk.content) || null, estimateStoredTokens(chunk.content), chunk.kind, page.sourceId, JSON.stringify(chunk.symbols), page.provenance.sourceUri, chunk.contentHash, page.id, page.
|
|
984
|
+
insert.run(chunk.id, page.id, chunk.index, chunk.headingBreadcrumbs.join(' > '), chunk.content, extractCodeBlocks(chunk.content) || null, estimateStoredTokens(chunk.content), chunk.kind, page.sourceId, JSON.stringify(chunk.symbols), page.provenance.sourceUri, chunk.contentHash, page.id, page.projectId, JSON.stringify(chunk.headingBreadcrumbs), chunk.embeddingText, chunk.originalStart, chunk.originalEnd, JSON.stringify(chunk.entities), JSON.stringify(chunk.symbols), JSON.stringify(chunk.links), chunk.oversizedAtomic ? 1 : 0, JSON.stringify(chunk.provenance), JSON.stringify(chunk.analysisTerms));
|
|
869
985
|
for (const entity of uniqueStrings(chunk.entities)) {
|
|
870
986
|
insertEntity.run(page.id, chunk.id, entity, normalizeKnowledgeLookupKey(entity));
|
|
871
987
|
}
|
|
@@ -875,7 +991,7 @@ export class KnowledgeStore {
|
|
|
875
991
|
}
|
|
876
992
|
})();
|
|
877
993
|
}
|
|
878
|
-
replaceKnowledgeAnalysisManifest(manifest, quarantined) {
|
|
994
|
+
replaceKnowledgeAnalysisManifest(manifest, quarantined, operationTimestamp = Date.now()) {
|
|
879
995
|
this.db.prepare(`
|
|
880
996
|
INSERT INTO knowledge_analysis_manifests
|
|
881
997
|
(source_id, content_hash, version, manifest_json, quarantined_json, created_at)
|
|
@@ -884,38 +1000,38 @@ export class KnowledgeStore {
|
|
|
884
1000
|
version=excluded.version,
|
|
885
1001
|
manifest_json=excluded.manifest_json,
|
|
886
1002
|
quarantined_json=excluded.quarantined_json
|
|
887
|
-
`).run(manifest.sourceId, manifest.contentHash, manifest.version, JSON.stringify(manifest), JSON.stringify(quarantined),
|
|
1003
|
+
`).run(manifest.sourceId, manifest.contentHash, manifest.version, JSON.stringify(manifest), JSON.stringify(quarantined), operationTimestamp);
|
|
888
1004
|
}
|
|
889
1005
|
recordImmutableRawSource(input) {
|
|
890
1006
|
const metadata = JSON.stringify(input.metadata ?? {});
|
|
891
1007
|
const existing = this.db.prepare(`
|
|
892
1008
|
SELECT source_uri, source_version, metadata_json, retrieved_at
|
|
893
|
-
FROM knowledge_raw_sources WHERE
|
|
894
|
-
`).get(input.
|
|
1009
|
+
FROM knowledge_raw_sources WHERE project_id = ? AND source_id = ? AND content_hash = ?
|
|
1010
|
+
`).get(input.projectId, input.sourceId, input.contentHash);
|
|
895
1011
|
if (existing) {
|
|
896
1012
|
if (existing.source_uri !== input.sourceUri || existing.source_version !== (input.sourceVersion ?? null)
|
|
897
|
-
|| existing.metadata_json !== metadata
|
|
1013
|
+
|| existing.metadata_json !== metadata) {
|
|
898
1014
|
throw new Error('KNOWLEDGE_RAW_SOURCE_IMMUTABILITY_CONFLICT');
|
|
899
1015
|
}
|
|
900
1016
|
return false;
|
|
901
1017
|
}
|
|
902
1018
|
return this.db.prepare(`
|
|
903
1019
|
INSERT INTO knowledge_raw_sources
|
|
904
|
-
(
|
|
1020
|
+
(project_id, source_id, content_hash, source_uri, source_version, metadata_json, retrieved_at)
|
|
905
1021
|
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
906
|
-
`).run(input.
|
|
1022
|
+
`).run(input.projectId, input.sourceId, input.contentHash, input.sourceUri, input.sourceVersion ?? null, metadata, input.retrievedAt).changes === 1;
|
|
907
1023
|
}
|
|
908
1024
|
publishCompiledSource(input) {
|
|
909
1025
|
this.db.transaction(() => {
|
|
910
|
-
this.replaceKnowledgeAnalysisManifest(input.manifest, input.quarantined);
|
|
911
|
-
this.replaceCompiledPage(input.page, input.chunks);
|
|
1026
|
+
this.replaceKnowledgeAnalysisManifest(input.manifest, input.quarantined, input.operationTimestamp);
|
|
1027
|
+
this.replaceCompiledPage(input.page, input.chunks, input.operationTimestamp);
|
|
912
1028
|
})();
|
|
913
1029
|
}
|
|
914
1030
|
getNormalizedPage(id) {
|
|
915
1031
|
const row = this.db.prepare("SELECT * FROM knowledge_pages WHERE id = ? AND status = 'active'").get(id);
|
|
916
1032
|
return row ? {
|
|
917
1033
|
id: row.id,
|
|
918
|
-
|
|
1034
|
+
projectId: row.project_id,
|
|
919
1035
|
sourceId: row.source_id,
|
|
920
1036
|
sourceSchemaId: row.source_schema_id,
|
|
921
1037
|
title: row.title,
|
|
@@ -930,6 +1046,102 @@ export class KnowledgeStore {
|
|
|
930
1046
|
compiledAt: row.compiled_at,
|
|
931
1047
|
} : null;
|
|
932
1048
|
}
|
|
1049
|
+
transaction(operation) {
|
|
1050
|
+
return this.db.transaction(operation)();
|
|
1051
|
+
}
|
|
1052
|
+
getSourceAggregate(identityHash) {
|
|
1053
|
+
const row = this.db.prepare('SELECT aggregate_json FROM knowledge_source_aggregates_v1 WHERE identity_hash = ?').get(identityHash);
|
|
1054
|
+
return row ? JSON.parse(row.aggregate_json) : null;
|
|
1055
|
+
}
|
|
1056
|
+
appendSourceTransition(input) {
|
|
1057
|
+
return this.db.transaction(() => {
|
|
1058
|
+
const existingEvent = this.db.prepare('SELECT transition_json FROM knowledge_source_events_v1 WHERE event_id = ?').get(input.transition.eventId);
|
|
1059
|
+
const transitionJson = JSON.stringify(input.transition);
|
|
1060
|
+
if (existingEvent) {
|
|
1061
|
+
if (existingEvent.transition_json !== transitionJson) {
|
|
1062
|
+
throw new Error('KNOWLEDGE_SOURCE_EVENT_IDEMPOTENCY_CONFLICT');
|
|
1063
|
+
}
|
|
1064
|
+
return { replayed: true };
|
|
1065
|
+
}
|
|
1066
|
+
const current = this.getSourceAggregate(input.aggregate.identity.identityHash);
|
|
1067
|
+
if ((current?.revision ?? 0) + 1 !== input.aggregate.revision
|
|
1068
|
+
|| input.transition.sequence !== input.aggregate.revision) {
|
|
1069
|
+
throw new Error('KNOWLEDGE_SOURCE_SEQUENCE_CONFLICT');
|
|
1070
|
+
}
|
|
1071
|
+
this.db.prepare(`
|
|
1072
|
+
INSERT INTO knowledge_source_aggregates_v1
|
|
1073
|
+
(identity_hash, project_id, source_id, state, revision, content_hash,
|
|
1074
|
+
aggregate_hash, aggregate_json, updated_at)
|
|
1075
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
1076
|
+
ON CONFLICT(identity_hash) DO UPDATE SET
|
|
1077
|
+
state = excluded.state,
|
|
1078
|
+
revision = excluded.revision,
|
|
1079
|
+
content_hash = excluded.content_hash,
|
|
1080
|
+
aggregate_hash = excluded.aggregate_hash,
|
|
1081
|
+
aggregate_json = excluded.aggregate_json,
|
|
1082
|
+
updated_at = excluded.updated_at
|
|
1083
|
+
`).run(input.aggregate.identity.identityHash, input.aggregate.identity.projectId, input.aggregate.identity.sourceId, input.aggregate.state, input.aggregate.revision, input.aggregate.contentHash ?? null, input.aggregate.aggregateHash, JSON.stringify(input.aggregate), input.aggregate.updatedAt);
|
|
1084
|
+
this.db.prepare(`
|
|
1085
|
+
INSERT INTO knowledge_source_events_v1
|
|
1086
|
+
(event_id, identity_hash, sequence, transition_json, created_at)
|
|
1087
|
+
VALUES (?, ?, ?, ?, ?)
|
|
1088
|
+
`).run(input.transition.eventId, input.transition.identityHash, input.transition.sequence, transitionJson, input.transition.createdAt);
|
|
1089
|
+
return { replayed: false };
|
|
1090
|
+
})();
|
|
1091
|
+
}
|
|
1092
|
+
insertKnowledgeClaims(claims) {
|
|
1093
|
+
if (claims.length === 0)
|
|
1094
|
+
return 0;
|
|
1095
|
+
const insert = this.db.prepare(`
|
|
1096
|
+
INSERT OR IGNORE INTO knowledge_claims_v1
|
|
1097
|
+
(claim_id, identity_hash, source_id, page_id, status, source_content_hash,
|
|
1098
|
+
claim_hash, claim_json, created_at)
|
|
1099
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
1100
|
+
`);
|
|
1101
|
+
return this.db.transaction(() => claims.reduce((count, claim) => count + insert.run(claim.claimId, claim.identityHash, claim.sourceId, claim.pageId, claim.status, claim.sourceContentHash, claim.claimHash, JSON.stringify(claim), claim.effectiveAt).changes, 0))();
|
|
1102
|
+
}
|
|
1103
|
+
listKnowledgeClaims(sourceId) {
|
|
1104
|
+
const rows = (sourceId
|
|
1105
|
+
? this.db.prepare('SELECT claim_json FROM knowledge_claims_v1 WHERE source_id = ? ORDER BY created_at, claim_id').all(sourceId)
|
|
1106
|
+
: this.db.prepare('SELECT claim_json FROM knowledge_claims_v1 ORDER BY created_at, claim_id').all());
|
|
1107
|
+
return rows.map(row => JSON.parse(row.claim_json));
|
|
1108
|
+
}
|
|
1109
|
+
quarantineSourceRevision(input) {
|
|
1110
|
+
const updatedAt = input.updatedAt ?? Date.now();
|
|
1111
|
+
return this.db.transaction(() => ({
|
|
1112
|
+
pagesQuarantined: this.db.prepare(`
|
|
1113
|
+
UPDATE knowledge_pages SET status = 'quarantined'
|
|
1114
|
+
WHERE id = ? AND project_id = ? AND content_hash = ? AND status = 'active'
|
|
1115
|
+
`).run(input.pageId, input.projectId, input.contentHash).changes,
|
|
1116
|
+
documentsDeactivated: this.db.prepare(`
|
|
1117
|
+
UPDATE knowledge_docs SET status = 'inactive'
|
|
1118
|
+
WHERE id = ? AND content_hash = ? AND status = 'active'
|
|
1119
|
+
`).run(input.pageId, input.contentHash).changes,
|
|
1120
|
+
catalogSnapshotsDeactivated: this.db.prepare(`
|
|
1121
|
+
UPDATE knowledge_catalog_snapshots SET status = 'inactive'
|
|
1122
|
+
WHERE project_id = ? AND source_id = ? AND content_hash = ? AND status = 'active'
|
|
1123
|
+
`).run(input.projectId, input.sourceId, input.contentHash).changes,
|
|
1124
|
+
readinessSourcesDegraded: this.db.prepare(`
|
|
1125
|
+
INSERT INTO knowledge_source_readiness
|
|
1126
|
+
(source, expected_version, documents, chunks, vectors, coverage, last_refresh,
|
|
1127
|
+
status, failure_category, updated_at)
|
|
1128
|
+
VALUES (?, ?, 0, 0, 0, 0, NULL, 'degraded', ?, ?)
|
|
1129
|
+
ON CONFLICT(source) DO UPDATE SET
|
|
1130
|
+
status = 'degraded', failure_category = excluded.failure_category,
|
|
1131
|
+
updated_at = excluded.updated_at
|
|
1132
|
+
`).run(input.sourceId, input.expectedVersion ?? null, input.reason.slice(0, 500), updatedAt).changes,
|
|
1133
|
+
}))();
|
|
1134
|
+
}
|
|
1135
|
+
recordKnowledgeMaintenanceRun(input) {
|
|
1136
|
+
this.db.prepare(`
|
|
1137
|
+
INSERT INTO knowledge_maintenance_runs (id, manifest_json, created_at)
|
|
1138
|
+
VALUES (?, ?, ?)
|
|
1139
|
+
ON CONFLICT(id) DO UPDATE SET manifest_json = excluded.manifest_json
|
|
1140
|
+
`).run(input.id, JSON.stringify(input.manifest), input.createdAt);
|
|
1141
|
+
}
|
|
1142
|
+
rebuildFullTextIndex() {
|
|
1143
|
+
this.db.prepare("INSERT INTO knowledge_fts(knowledge_fts) VALUES('rebuild')").run();
|
|
1144
|
+
}
|
|
933
1145
|
replacePageLinks(page) {
|
|
934
1146
|
this.db.prepare('DELETE FROM knowledge_entity_links WHERE page_id = ?').run(page.id);
|
|
935
1147
|
this.db.prepare('DELETE FROM knowledge_symbol_links WHERE page_id = ?').run(page.id);
|