@cortexa/core 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +191 -0
- package/README.md +423 -0
- package/dist/chunk-6VDDIS65.js +21 -0
- package/dist/chunk-6VDDIS65.js.map +1 -0
- package/dist/chunk-T4NUBJ7T.js +13 -0
- package/dist/chunk-T4NUBJ7T.js.map +1 -0
- package/dist/cli/chunk-7HYSAZX4.js +15 -0
- package/dist/cli/chunk-7HYSAZX4.js.map +1 -0
- package/dist/cli/chunk-HVMZ6P54.js +23 -0
- package/dist/cli/chunk-HVMZ6P54.js.map +1 -0
- package/dist/cli/index.js +5921 -0
- package/dist/cli/index.js.map +1 -0
- package/dist/cli/mongodb-WTT5HVQH.js +145 -0
- package/dist/cli/mongodb-WTT5HVQH.js.map +1 -0
- package/dist/cli/mongodb-stream-Q5OBFWZP.js +93 -0
- package/dist/cli/mongodb-stream-Q5OBFWZP.js.map +1 -0
- package/dist/cli/mssql-KG7P3R2I.js +79 -0
- package/dist/cli/mssql-KG7P3R2I.js.map +1 -0
- package/dist/cli/mysql-stream-Z2MZS2KF.js +195 -0
- package/dist/cli/mysql-stream-Z2MZS2KF.js.map +1 -0
- package/dist/cli/postgres-stream-S4CRICEA.js +237 -0
- package/dist/cli/postgres-stream-S4CRICEA.js.map +1 -0
- package/dist/index.cjs +5733 -0
- package/dist/index.cjs.map +1 -0
- package/dist/index.d.cts +670 -0
- package/dist/index.d.ts +670 -0
- package/dist/index.js +4805 -0
- package/dist/index.js.map +1 -0
- package/dist/mongodb-M5UGJZTC.js +143 -0
- package/dist/mongodb-M5UGJZTC.js.map +1 -0
- package/dist/mongodb-stream-Q23UHLTM.js +92 -0
- package/dist/mongodb-stream-Q23UHLTM.js.map +1 -0
- package/dist/mssql-I27XEHQ2.js +77 -0
- package/dist/mssql-I27XEHQ2.js.map +1 -0
- package/dist/mysql-stream-YCNLPPPG.js +194 -0
- package/dist/mysql-stream-YCNLPPPG.js.map +1 -0
- package/dist/postgres-stream-A7EVYUX2.js +236 -0
- package/dist/postgres-stream-A7EVYUX2.js.map +1 -0
- package/package.json +105 -0
package/dist/index.js
ADDED
|
@@ -0,0 +1,4805 @@
|
|
|
1
|
+
import {
|
|
2
|
+
createLogger
|
|
3
|
+
} from "./chunk-T4NUBJ7T.js";
|
|
4
|
+
|
|
5
|
+
// src/index.ts
|
|
6
|
+
import { EventEmitter as EventEmitter2 } from "events";
|
|
7
|
+
|
|
8
|
+
// src/connectors/postgres.ts
|
|
9
|
+
import pg from "pg";
|
|
10
|
+
var { Pool } = pg;
|
|
11
|
+
var PostgresConnector = class {
|
|
12
|
+
pool = null;
|
|
13
|
+
async connect(config) {
|
|
14
|
+
if (config.url) {
|
|
15
|
+
this.pool = new Pool({ connectionString: config.url });
|
|
16
|
+
} else {
|
|
17
|
+
this.pool = new Pool({
|
|
18
|
+
host: config.host,
|
|
19
|
+
port: config.port,
|
|
20
|
+
database: config.database,
|
|
21
|
+
user: config.user,
|
|
22
|
+
password: config.password
|
|
23
|
+
});
|
|
24
|
+
}
|
|
25
|
+
const client = await this.pool.connect();
|
|
26
|
+
client.release();
|
|
27
|
+
}
|
|
28
|
+
async disconnect() {
|
|
29
|
+
if (this.pool) {
|
|
30
|
+
await this.pool.end();
|
|
31
|
+
this.pool = null;
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
async getTableCount() {
|
|
35
|
+
this.ensureConnected();
|
|
36
|
+
const result = await this.pool.query(
|
|
37
|
+
`SELECT COUNT(*) as count FROM information_schema.tables
|
|
38
|
+
WHERE table_schema = 'public' AND table_type = 'BASE TABLE'`
|
|
39
|
+
);
|
|
40
|
+
return parseInt(result.rows[0].count, 10);
|
|
41
|
+
}
|
|
42
|
+
async getDatabaseName() {
|
|
43
|
+
this.ensureConnected();
|
|
44
|
+
const result = await this.pool.query("SELECT current_database() as db");
|
|
45
|
+
return result.rows[0].db;
|
|
46
|
+
}
|
|
47
|
+
async isReadOnly() {
|
|
48
|
+
this.ensureConnected();
|
|
49
|
+
try {
|
|
50
|
+
await this.pool.query(
|
|
51
|
+
"CREATE TEMP TABLE _cortexa_ro_check (id int)"
|
|
52
|
+
);
|
|
53
|
+
return false;
|
|
54
|
+
} catch {
|
|
55
|
+
return true;
|
|
56
|
+
} finally {
|
|
57
|
+
try {
|
|
58
|
+
await this.pool.query(
|
|
59
|
+
"DROP TABLE IF EXISTS _cortexa_ro_check"
|
|
60
|
+
);
|
|
61
|
+
} catch {
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
async query(sql, params) {
|
|
66
|
+
this.ensureConnected();
|
|
67
|
+
const result = await this.pool.query(sql, params);
|
|
68
|
+
return {
|
|
69
|
+
rows: result.rows,
|
|
70
|
+
rowCount: result.rowCount ?? result.rows.length
|
|
71
|
+
};
|
|
72
|
+
}
|
|
73
|
+
isConnected() {
|
|
74
|
+
return this.pool !== null;
|
|
75
|
+
}
|
|
76
|
+
ensureConnected() {
|
|
77
|
+
if (!this.pool) {
|
|
78
|
+
throw new Error("Not connected. Call connect() first.");
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
};
|
|
82
|
+
|
|
83
|
+
// src/connectors/mysql.ts
|
|
84
|
+
import mysql from "mysql2/promise";
|
|
85
|
+
var MySQLConnector = class {
|
|
86
|
+
pool = null;
|
|
87
|
+
async connect(config) {
|
|
88
|
+
if (config.url) {
|
|
89
|
+
this.pool = mysql.createPool(config.url);
|
|
90
|
+
} else {
|
|
91
|
+
this.pool = mysql.createPool({
|
|
92
|
+
host: config.host,
|
|
93
|
+
port: config.port,
|
|
94
|
+
database: config.database,
|
|
95
|
+
user: config.user,
|
|
96
|
+
password: config.password
|
|
97
|
+
});
|
|
98
|
+
}
|
|
99
|
+
const conn = await this.pool.getConnection();
|
|
100
|
+
conn.release();
|
|
101
|
+
}
|
|
102
|
+
async disconnect() {
|
|
103
|
+
if (this.pool) {
|
|
104
|
+
await this.pool.end();
|
|
105
|
+
this.pool = null;
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
async getTableCount() {
|
|
109
|
+
this.ensureConnected();
|
|
110
|
+
const [rows] = await this.pool.query(
|
|
111
|
+
`SELECT COUNT(*) as count FROM information_schema.tables
|
|
112
|
+
WHERE table_schema = DATABASE() AND table_type = 'BASE TABLE'`
|
|
113
|
+
);
|
|
114
|
+
return rows[0].count;
|
|
115
|
+
}
|
|
116
|
+
async getDatabaseName() {
|
|
117
|
+
this.ensureConnected();
|
|
118
|
+
const [rows] = await this.pool.query("SELECT DATABASE() as db");
|
|
119
|
+
return rows[0].db;
|
|
120
|
+
}
|
|
121
|
+
async isReadOnly() {
|
|
122
|
+
this.ensureConnected();
|
|
123
|
+
try {
|
|
124
|
+
await this.pool.query(
|
|
125
|
+
"CREATE TEMPORARY TABLE _cortexa_ro_check (id INT)"
|
|
126
|
+
);
|
|
127
|
+
return false;
|
|
128
|
+
} catch {
|
|
129
|
+
return true;
|
|
130
|
+
} finally {
|
|
131
|
+
try {
|
|
132
|
+
await this.pool.query(
|
|
133
|
+
"DROP TEMPORARY TABLE IF EXISTS _cortexa_ro_check"
|
|
134
|
+
);
|
|
135
|
+
} catch {
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
async query(sql, params) {
|
|
140
|
+
this.ensureConnected();
|
|
141
|
+
const mysqlSql = sql.replace(/\$\d+/g, "?");
|
|
142
|
+
const [rows] = await this.pool.query(mysqlSql, params);
|
|
143
|
+
const rowArray = rows;
|
|
144
|
+
return {
|
|
145
|
+
rows: rowArray,
|
|
146
|
+
rowCount: rowArray.length
|
|
147
|
+
};
|
|
148
|
+
}
|
|
149
|
+
isConnected() {
|
|
150
|
+
return this.pool !== null;
|
|
151
|
+
}
|
|
152
|
+
ensureConnected() {
|
|
153
|
+
if (!this.pool) {
|
|
154
|
+
throw new Error("Not connected. Call connect() first.");
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
};
|
|
158
|
+
|
|
159
|
+
// src/connectors/sqlite.ts
|
|
160
|
+
import path from "path";
|
|
161
|
+
import Database from "better-sqlite3";
|
|
162
|
+
var SQLiteConnector = class {
|
|
163
|
+
db = null;
|
|
164
|
+
dbPath = "";
|
|
165
|
+
async connect(config) {
|
|
166
|
+
this.dbPath = config.path ?? config.database ?? ":memory:";
|
|
167
|
+
this.db = new Database(this.dbPath, { readonly: true });
|
|
168
|
+
}
|
|
169
|
+
async disconnect() {
|
|
170
|
+
if (this.db) {
|
|
171
|
+
this.db.close();
|
|
172
|
+
this.db = null;
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
async getTableCount() {
|
|
176
|
+
this.ensureConnected();
|
|
177
|
+
const row = this.db.prepare(
|
|
178
|
+
`SELECT COUNT(*) AS count FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'`
|
|
179
|
+
).get();
|
|
180
|
+
return row.count;
|
|
181
|
+
}
|
|
182
|
+
async getDatabaseName() {
|
|
183
|
+
return path.basename(this.dbPath, path.extname(this.dbPath));
|
|
184
|
+
}
|
|
185
|
+
async isReadOnly() {
|
|
186
|
+
return true;
|
|
187
|
+
}
|
|
188
|
+
async query(sql, params) {
|
|
189
|
+
this.ensureConnected();
|
|
190
|
+
const convertedSql = sql.replace(/\$(\d+)/g, "?");
|
|
191
|
+
const stmt = this.db.prepare(convertedSql);
|
|
192
|
+
if (convertedSql.trim().toUpperCase().startsWith("SELECT") || convertedSql.trim().toUpperCase().startsWith("PRAGMA")) {
|
|
193
|
+
const rows = params && params.length > 0 ? stmt.all(...params) : stmt.all();
|
|
194
|
+
return { rows, rowCount: rows.length };
|
|
195
|
+
}
|
|
196
|
+
const result = params && params.length > 0 ? stmt.run(...params) : stmt.run();
|
|
197
|
+
return { rows: [], rowCount: result.changes };
|
|
198
|
+
}
|
|
199
|
+
isConnected() {
|
|
200
|
+
return this.db !== null;
|
|
201
|
+
}
|
|
202
|
+
ensureConnected() {
|
|
203
|
+
if (!this.db) {
|
|
204
|
+
throw new Error("Not connected. Call connect() first.");
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
};
|
|
208
|
+
|
|
209
|
+
// src/core/connection.ts
|
|
210
|
+
async function createConnector(type) {
|
|
211
|
+
switch (type) {
|
|
212
|
+
case "postgres":
|
|
213
|
+
case "cockroachdb":
|
|
214
|
+
return new PostgresConnector();
|
|
215
|
+
case "mysql":
|
|
216
|
+
case "mariadb":
|
|
217
|
+
return new MySQLConnector();
|
|
218
|
+
case "sqlite":
|
|
219
|
+
return new SQLiteConnector();
|
|
220
|
+
case "mongodb": {
|
|
221
|
+
const { MongoDBConnector } = await import("./mongodb-M5UGJZTC.js");
|
|
222
|
+
return new MongoDBConnector();
|
|
223
|
+
}
|
|
224
|
+
case "mssql": {
|
|
225
|
+
const { MSSQLConnector } = await import("./mssql-I27XEHQ2.js");
|
|
226
|
+
return new MSSQLConnector();
|
|
227
|
+
}
|
|
228
|
+
default:
|
|
229
|
+
throw new Error(`Unsupported database type: ${type}`);
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
// src/core/storage.ts
|
|
234
|
+
import Database2 from "better-sqlite3";
|
|
235
|
+
import fs from "fs";
|
|
236
|
+
import path2 from "path";
|
|
237
|
+
var CortexaStorage = class {
|
|
238
|
+
db = null;
|
|
239
|
+
cortexaDir;
|
|
240
|
+
dbPath;
|
|
241
|
+
constructor(projectRoot) {
|
|
242
|
+
this.cortexaDir = path2.join(projectRoot, ".cortexa");
|
|
243
|
+
this.dbPath = path2.join(this.cortexaDir, "cortexa.db");
|
|
244
|
+
}
|
|
245
|
+
initialize() {
|
|
246
|
+
fs.mkdirSync(this.cortexaDir, { recursive: true });
|
|
247
|
+
this.db = new Database2(this.dbPath);
|
|
248
|
+
this.db.pragma("journal_mode = WAL");
|
|
249
|
+
this.createSchema();
|
|
250
|
+
}
|
|
251
|
+
setMeta(key, value) {
|
|
252
|
+
this.ensureInitialized();
|
|
253
|
+
this.db.prepare(
|
|
254
|
+
"INSERT OR REPLACE INTO meta (key, value) VALUES (?, ?)"
|
|
255
|
+
).run(key, value);
|
|
256
|
+
}
|
|
257
|
+
getMeta(key) {
|
|
258
|
+
this.ensureInitialized();
|
|
259
|
+
const row = this.db.prepare(
|
|
260
|
+
"SELECT value FROM meta WHERE key = ?"
|
|
261
|
+
).get(key);
|
|
262
|
+
return row?.value;
|
|
263
|
+
}
|
|
264
|
+
getDiscoveredTableCount() {
|
|
265
|
+
this.ensureInitialized();
|
|
266
|
+
const row = this.db.prepare(
|
|
267
|
+
"SELECT COUNT(*) as count FROM discovered_tables"
|
|
268
|
+
).get();
|
|
269
|
+
return row.count;
|
|
270
|
+
}
|
|
271
|
+
saveEntity(entity) {
|
|
272
|
+
this.ensureInitialized();
|
|
273
|
+
this.db.prepare(
|
|
274
|
+
`INSERT OR REPLACE INTO entities
|
|
275
|
+
(table_name, entity_label, entity_type, description, confidence, columns_classified)
|
|
276
|
+
VALUES (?, ?, ?, ?, ?, ?)`
|
|
277
|
+
).run(entity.tableName, entity.entityLabel, entity.entityType, entity.description, entity.confidence, JSON.stringify(entity.columns));
|
|
278
|
+
}
|
|
279
|
+
getEntities() {
|
|
280
|
+
this.ensureInitialized();
|
|
281
|
+
const rows = this.db.prepare("SELECT * FROM entities").all();
|
|
282
|
+
return rows.map((row) => ({
|
|
283
|
+
tableName: row.table_name,
|
|
284
|
+
entityLabel: row.entity_label,
|
|
285
|
+
entityType: row.entity_type,
|
|
286
|
+
description: row.description,
|
|
287
|
+
confidence: row.confidence,
|
|
288
|
+
columns: JSON.parse(row.columns_classified || "[]")
|
|
289
|
+
}));
|
|
290
|
+
}
|
|
291
|
+
getEntity(tableName) {
|
|
292
|
+
this.ensureInitialized();
|
|
293
|
+
const row = this.db.prepare("SELECT * FROM entities WHERE table_name = ?").get(tableName);
|
|
294
|
+
if (!row) return void 0;
|
|
295
|
+
return {
|
|
296
|
+
tableName: row.table_name,
|
|
297
|
+
entityLabel: row.entity_label,
|
|
298
|
+
entityType: row.entity_type,
|
|
299
|
+
description: row.description,
|
|
300
|
+
confidence: row.confidence,
|
|
301
|
+
columns: JSON.parse(row.columns_classified || "[]")
|
|
302
|
+
};
|
|
303
|
+
}
|
|
304
|
+
saveRelationship(rel) {
|
|
305
|
+
this.ensureInitialized();
|
|
306
|
+
this.db.prepare(
|
|
307
|
+
`INSERT OR REPLACE INTO relationships
|
|
308
|
+
(source_entity, target_entity, relationship, source_column, target_column, inferred, confidence)
|
|
309
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)`
|
|
310
|
+
).run(rel.sourceEntity, rel.targetEntity, rel.relationship, rel.sourceColumn, rel.targetColumn, rel.inferred ? 1 : 0, rel.confidence);
|
|
311
|
+
}
|
|
312
|
+
getRelationships() {
|
|
313
|
+
this.ensureInitialized();
|
|
314
|
+
const rows = this.db.prepare("SELECT * FROM relationships").all();
|
|
315
|
+
return rows.map((row) => ({
|
|
316
|
+
sourceEntity: row.source_entity,
|
|
317
|
+
targetEntity: row.target_entity,
|
|
318
|
+
relationship: row.relationship,
|
|
319
|
+
sourceColumn: row.source_column,
|
|
320
|
+
targetColumn: row.target_column,
|
|
321
|
+
inferred: row.inferred === 1,
|
|
322
|
+
confidence: row.confidence
|
|
323
|
+
}));
|
|
324
|
+
}
|
|
325
|
+
clearRelationships() {
|
|
326
|
+
this.ensureInitialized();
|
|
327
|
+
this.db.prepare("DELETE FROM relationships").run();
|
|
328
|
+
}
|
|
329
|
+
saveDiscoveredTable(table) {
|
|
330
|
+
this.ensureInitialized();
|
|
331
|
+
this.db.prepare(
|
|
332
|
+
`INSERT OR REPLACE INTO discovered_tables
|
|
333
|
+
(table_name, table_schema, row_count, columns_json, fk_json, indexes_json, sample_rows)
|
|
334
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)`
|
|
335
|
+
).run(table.tableName, table.tableSchema, table.rowCount, JSON.stringify(table.columns), JSON.stringify(table.foreignKeys), JSON.stringify(table.indexes), JSON.stringify(table.sampleRows));
|
|
336
|
+
}
|
|
337
|
+
setCache(key, response, model, tokensUsed) {
|
|
338
|
+
this.ensureInitialized();
|
|
339
|
+
this.db.prepare(
|
|
340
|
+
"INSERT OR REPLACE INTO llm_cache (cache_key, response, model, tokens_used) VALUES (?, ?, ?, ?)"
|
|
341
|
+
).run(key, response, model, tokensUsed);
|
|
342
|
+
}
|
|
343
|
+
getCache(key) {
|
|
344
|
+
this.ensureInitialized();
|
|
345
|
+
const row = this.db.prepare("SELECT response FROM llm_cache WHERE cache_key = ?").get(key);
|
|
346
|
+
return row?.response;
|
|
347
|
+
}
|
|
348
|
+
clearCache() {
|
|
349
|
+
this.ensureInitialized();
|
|
350
|
+
this.db.prepare("DELETE FROM llm_cache").run();
|
|
351
|
+
}
|
|
352
|
+
saveEvent(event) {
|
|
353
|
+
this.ensureInitialized();
|
|
354
|
+
this.db.prepare(
|
|
355
|
+
"INSERT INTO events (event_type, entity, table_name, operation, row_id, row_data) VALUES (?, ?, ?, ?, ?, ?)"
|
|
356
|
+
).run(event.eventType, event.entity, event.tableName, event.operation, event.rowId ?? null, event.rowData ?? null);
|
|
357
|
+
}
|
|
358
|
+
getEvents(filter) {
|
|
359
|
+
this.ensureInitialized();
|
|
360
|
+
let sql = "SELECT id, event_type, entity, table_name, operation, row_id, row_data, timestamp FROM events";
|
|
361
|
+
const conditions = [];
|
|
362
|
+
const params = [];
|
|
363
|
+
if (filter?.entity) {
|
|
364
|
+
conditions.push("entity = ?");
|
|
365
|
+
params.push(filter.entity);
|
|
366
|
+
}
|
|
367
|
+
if (filter?.last) {
|
|
368
|
+
conditions.push(`timestamp >= datetime('now', '-${filter.last} minutes')`);
|
|
369
|
+
}
|
|
370
|
+
if (filter?.operation) {
|
|
371
|
+
conditions.push("operation = ?");
|
|
372
|
+
params.push(filter.operation);
|
|
373
|
+
}
|
|
374
|
+
if (filter?.eventType) {
|
|
375
|
+
conditions.push("event_type = ?");
|
|
376
|
+
params.push(filter.eventType);
|
|
377
|
+
}
|
|
378
|
+
if (conditions.length > 0) {
|
|
379
|
+
sql += " WHERE " + conditions.join(" AND ");
|
|
380
|
+
}
|
|
381
|
+
sql += " ORDER BY timestamp DESC";
|
|
382
|
+
const rows = this.db.prepare(sql).all(...params);
|
|
383
|
+
return rows.map((r) => ({
|
|
384
|
+
id: r.id,
|
|
385
|
+
eventType: r.event_type,
|
|
386
|
+
entity: r.entity,
|
|
387
|
+
tableName: r.table_name,
|
|
388
|
+
operation: r.operation,
|
|
389
|
+
rowId: r.row_id,
|
|
390
|
+
rowData: r.row_data,
|
|
391
|
+
timestamp: r.timestamp
|
|
392
|
+
}));
|
|
393
|
+
}
|
|
394
|
+
countEvents(entity, operation, lastMinutes) {
|
|
395
|
+
this.ensureInitialized();
|
|
396
|
+
const row = this.db.prepare(
|
|
397
|
+
`SELECT COUNT(*) as count FROM events WHERE entity = ? AND operation = ? AND timestamp >= datetime('now', '-' || ? || ' minutes')`
|
|
398
|
+
).get(entity, operation, lastMinutes);
|
|
399
|
+
return row.count;
|
|
400
|
+
}
|
|
401
|
+
saveBaseline(baseline) {
|
|
402
|
+
this.ensureInitialized();
|
|
403
|
+
this.db.prepare(
|
|
404
|
+
"INSERT OR REPLACE INTO baselines (entity, metric, mean, stddev, sample_size) VALUES (?, ?, ?, ?, ?)"
|
|
405
|
+
).run(baseline.entity, baseline.metric, baseline.mean, baseline.stddev, baseline.sampleSize);
|
|
406
|
+
}
|
|
407
|
+
getBaselines() {
|
|
408
|
+
this.ensureInitialized();
|
|
409
|
+
const rows = this.db.prepare("SELECT entity, metric, mean, stddev, sample_size FROM baselines").all();
|
|
410
|
+
return rows.map((r) => ({
|
|
411
|
+
entity: r.entity,
|
|
412
|
+
metric: r.metric,
|
|
413
|
+
mean: r.mean,
|
|
414
|
+
stddev: r.stddev,
|
|
415
|
+
sampleSize: r.sample_size
|
|
416
|
+
}));
|
|
417
|
+
}
|
|
418
|
+
saveAnomaly(anomaly) {
|
|
419
|
+
this.ensureInitialized();
|
|
420
|
+
this.db.prepare(
|
|
421
|
+
"INSERT INTO anomalies (entity, anomaly_type, severity, expected, actual, message) VALUES (?, ?, ?, ?, ?, ?)"
|
|
422
|
+
).run(anomaly.entity, anomaly.anomalyType, anomaly.severity, anomaly.expected, anomaly.actual, anomaly.message);
|
|
423
|
+
}
|
|
424
|
+
getAnomalies(filter) {
|
|
425
|
+
this.ensureInitialized();
|
|
426
|
+
let sql = "SELECT id, entity, anomaly_type, severity, expected, actual, message, timestamp FROM anomalies";
|
|
427
|
+
const conditions = [];
|
|
428
|
+
const params = [];
|
|
429
|
+
if (filter?.last) {
|
|
430
|
+
conditions.push(`timestamp >= datetime('now', '-${filter.last} minutes')`);
|
|
431
|
+
}
|
|
432
|
+
if (filter?.entity) {
|
|
433
|
+
conditions.push("entity = ?");
|
|
434
|
+
params.push(filter.entity);
|
|
435
|
+
}
|
|
436
|
+
if (filter?.severity) {
|
|
437
|
+
conditions.push("severity = ?");
|
|
438
|
+
params.push(filter.severity);
|
|
439
|
+
}
|
|
440
|
+
if (filter?.anomalyType) {
|
|
441
|
+
conditions.push("anomaly_type = ?");
|
|
442
|
+
params.push(filter.anomalyType);
|
|
443
|
+
}
|
|
444
|
+
if (conditions.length > 0) {
|
|
445
|
+
sql += " WHERE " + conditions.join(" AND ");
|
|
446
|
+
}
|
|
447
|
+
sql += " ORDER BY timestamp DESC";
|
|
448
|
+
const rows = this.db.prepare(sql).all(...params);
|
|
449
|
+
return rows.map((r) => ({
|
|
450
|
+
id: r.id,
|
|
451
|
+
entity: r.entity,
|
|
452
|
+
anomalyType: r.anomaly_type,
|
|
453
|
+
severity: r.severity,
|
|
454
|
+
expected: r.expected,
|
|
455
|
+
actual: r.actual,
|
|
456
|
+
message: r.message,
|
|
457
|
+
timestamp: r.timestamp
|
|
458
|
+
}));
|
|
459
|
+
}
|
|
460
|
+
saveWatermark(wm) {
|
|
461
|
+
this.ensureInitialized();
|
|
462
|
+
this.db.prepare(
|
|
463
|
+
"INSERT OR REPLACE INTO watch_watermarks (table_name, table_schema, strategy, timestamp_column, pk_column, last_seen_value) VALUES (?, ?, ?, ?, ?, ?)"
|
|
464
|
+
).run(wm.tableName, wm.tableSchema, wm.strategy, wm.timestampColumn ?? null, wm.pkColumn ?? null, wm.lastSeenValue ?? null);
|
|
465
|
+
}
|
|
466
|
+
getWatermark(tableName, tableSchema) {
|
|
467
|
+
this.ensureInitialized();
|
|
468
|
+
const row = this.db.prepare(
|
|
469
|
+
"SELECT strategy, timestamp_column, pk_column, last_seen_value FROM watch_watermarks WHERE table_name = ? AND table_schema = ?"
|
|
470
|
+
).get(tableName, tableSchema);
|
|
471
|
+
if (!row) return void 0;
|
|
472
|
+
return {
|
|
473
|
+
strategy: row.strategy,
|
|
474
|
+
timestampColumn: row.timestamp_column,
|
|
475
|
+
pkColumn: row.pk_column,
|
|
476
|
+
lastSeenValue: row.last_seen_value
|
|
477
|
+
};
|
|
478
|
+
}
|
|
479
|
+
saveTransition(t) {
|
|
480
|
+
this.ensureInitialized();
|
|
481
|
+
this.db.prepare(
|
|
482
|
+
"INSERT INTO state_transitions (entity, table_name, primary_key, from_state, to_state, duration_ms, expected) VALUES (?, ?, ?, ?, ?, ?, ?)"
|
|
483
|
+
).run(t.entity, t.tableName, t.primaryKey, t.fromState, t.toState, t.durationMs, t.expected ? 1 : 0);
|
|
484
|
+
}
|
|
485
|
+
getTransitions(filter) {
|
|
486
|
+
this.ensureInitialized();
|
|
487
|
+
let sql = "SELECT entity, table_name, primary_key, from_state, to_state, duration_ms, expected, timestamp FROM state_transitions";
|
|
488
|
+
const params = [];
|
|
489
|
+
if (filter?.entity) {
|
|
490
|
+
sql += " WHERE entity = ?";
|
|
491
|
+
params.push(filter.entity);
|
|
492
|
+
}
|
|
493
|
+
sql += " ORDER BY timestamp DESC";
|
|
494
|
+
const rows = this.db.prepare(sql).all(...params);
|
|
495
|
+
return rows.map((r) => ({
|
|
496
|
+
entity: r.entity,
|
|
497
|
+
tableName: r.table_name,
|
|
498
|
+
primaryKey: r.primary_key,
|
|
499
|
+
fromState: r.from_state,
|
|
500
|
+
toState: r.to_state,
|
|
501
|
+
durationMs: r.duration_ms,
|
|
502
|
+
expected: r.expected === 1,
|
|
503
|
+
timestamp: r.timestamp
|
|
504
|
+
}));
|
|
505
|
+
}
|
|
506
|
+
getTransitionStats(entity) {
|
|
507
|
+
this.ensureInitialized();
|
|
508
|
+
let sql = "SELECT entity, from_state, to_state, COUNT(*) as count, AVG(duration_ms) as avg_duration, MIN(duration_ms) as min_duration, MAX(duration_ms) as max_duration FROM state_transitions";
|
|
509
|
+
const params = [];
|
|
510
|
+
if (entity) {
|
|
511
|
+
sql += " WHERE entity = ?";
|
|
512
|
+
params.push(entity);
|
|
513
|
+
}
|
|
514
|
+
sql += " GROUP BY entity, from_state, to_state";
|
|
515
|
+
const rows = this.db.prepare(sql).all(...params);
|
|
516
|
+
return rows.map((r) => ({
|
|
517
|
+
entity: r.entity,
|
|
518
|
+
fromState: r.from_state,
|
|
519
|
+
toState: r.to_state,
|
|
520
|
+
count: r.count,
|
|
521
|
+
avgDurationMs: Math.round(r.avg_duration),
|
|
522
|
+
minDurationMs: r.min_duration,
|
|
523
|
+
maxDurationMs: r.max_duration
|
|
524
|
+
}));
|
|
525
|
+
}
|
|
526
|
+
saveInsight(i) {
|
|
527
|
+
this.ensureInitialized();
|
|
528
|
+
this.db.prepare(
|
|
529
|
+
"INSERT INTO insights (entity, insight_type, severity, message, context_json) VALUES (?, ?, ?, ?, ?)"
|
|
530
|
+
).run(i.entity, i.insightType, i.severity, i.message, JSON.stringify(i.context));
|
|
531
|
+
}
|
|
532
|
+
getInsights(filter) {
|
|
533
|
+
this.ensureInitialized();
|
|
534
|
+
let sql = "SELECT id, entity, insight_type, severity, message, context_json, timestamp FROM insights";
|
|
535
|
+
const conditions = [];
|
|
536
|
+
const params = [];
|
|
537
|
+
if (filter?.entity) {
|
|
538
|
+
conditions.push("entity = ?");
|
|
539
|
+
params.push(filter.entity);
|
|
540
|
+
}
|
|
541
|
+
if (filter?.last !== void 0) {
|
|
542
|
+
conditions.push("timestamp >= datetime('now', '-' || ? || ' minutes')");
|
|
543
|
+
params.push(filter.last);
|
|
544
|
+
}
|
|
545
|
+
if (filter?.insightType) {
|
|
546
|
+
conditions.push("insight_type = ?");
|
|
547
|
+
params.push(filter.insightType);
|
|
548
|
+
}
|
|
549
|
+
if (filter?.severity) {
|
|
550
|
+
conditions.push("severity = ?");
|
|
551
|
+
params.push(filter.severity);
|
|
552
|
+
}
|
|
553
|
+
if (conditions.length > 0) {
|
|
554
|
+
sql += " WHERE " + conditions.join(" AND ");
|
|
555
|
+
}
|
|
556
|
+
sql += " ORDER BY timestamp DESC";
|
|
557
|
+
const rows = this.db.prepare(sql).all(...params);
|
|
558
|
+
return rows.map((r) => ({
|
|
559
|
+
id: r.id,
|
|
560
|
+
entity: r.entity,
|
|
561
|
+
insightType: r.insight_type,
|
|
562
|
+
severity: r.severity,
|
|
563
|
+
message: r.message,
|
|
564
|
+
context: JSON.parse(r.context_json || "{}"),
|
|
565
|
+
timestamp: r.timestamp
|
|
566
|
+
}));
|
|
567
|
+
}
|
|
568
|
+
getEventById(id) {
|
|
569
|
+
this.ensureInitialized();
|
|
570
|
+
const r = this.db.prepare("SELECT id, event_type, entity, table_name, operation, row_id, row_data, timestamp FROM events WHERE id = ?").get(id);
|
|
571
|
+
if (!r) return void 0;
|
|
572
|
+
return { id: r.id, eventType: r.event_type, entity: r.entity, tableName: r.table_name, operation: r.operation, rowId: r.row_id, rowData: r.row_data, timestamp: r.timestamp };
|
|
573
|
+
}
|
|
574
|
+
getAnomalyById(id) {
|
|
575
|
+
this.ensureInitialized();
|
|
576
|
+
const r = this.db.prepare("SELECT id, entity, anomaly_type, severity, expected, actual, message, timestamp FROM anomalies WHERE id = ?").get(id);
|
|
577
|
+
if (!r) return void 0;
|
|
578
|
+
return { id: r.id, entity: r.entity, anomalyType: r.anomaly_type, severity: r.severity, expected: r.expected, actual: r.actual, message: r.message, timestamp: r.timestamp };
|
|
579
|
+
}
|
|
580
|
+
getInsightById(id) {
|
|
581
|
+
this.ensureInitialized();
|
|
582
|
+
const r = this.db.prepare("SELECT id, entity, insight_type, severity, message, context_json, timestamp FROM insights WHERE id = ?").get(id);
|
|
583
|
+
if (!r) return void 0;
|
|
584
|
+
return { id: r.id, entity: r.entity, insightType: r.insight_type, severity: r.severity, message: r.message, context: JSON.parse(r.context_json || "{}"), timestamp: r.timestamp };
|
|
585
|
+
}
|
|
586
|
+
saveEntityState(s) {
|
|
587
|
+
this.ensureInitialized();
|
|
588
|
+
this.db.prepare(
|
|
589
|
+
"INSERT OR REPLACE INTO entity_states (table_name, primary_key, state, entered_at) VALUES (?, ?, ?, ?)"
|
|
590
|
+
).run(s.tableName, s.primaryKey, s.state, s.enteredAt);
|
|
591
|
+
}
|
|
592
|
+
getEntityState(tableName, primaryKey) {
|
|
593
|
+
this.ensureInitialized();
|
|
594
|
+
const row = this.db.prepare(
|
|
595
|
+
"SELECT state, entered_at FROM entity_states WHERE table_name = ? AND primary_key = ?"
|
|
596
|
+
).get(tableName, primaryKey);
|
|
597
|
+
if (!row) return void 0;
|
|
598
|
+
return {
|
|
599
|
+
state: row.state,
|
|
600
|
+
enteredAt: row.entered_at
|
|
601
|
+
};
|
|
602
|
+
}
|
|
603
|
+
getStuckEntityStates(tableName, thresholdMs) {
|
|
604
|
+
this.ensureInitialized();
|
|
605
|
+
const thresholdSeconds = Math.floor(thresholdMs / 1e3);
|
|
606
|
+
const rows = this.db.prepare(
|
|
607
|
+
"SELECT primary_key, state, entered_at FROM entity_states WHERE table_name = ? AND entered_at <= datetime('now', '-' || ? || ' seconds')"
|
|
608
|
+
).all(tableName, thresholdSeconds);
|
|
609
|
+
return rows.map((r) => ({
|
|
610
|
+
primaryKey: r.primary_key,
|
|
611
|
+
state: r.state,
|
|
612
|
+
enteredAt: r.entered_at
|
|
613
|
+
}));
|
|
614
|
+
}
|
|
615
|
+
saveCorrelationEvent(event) {
|
|
616
|
+
this.ensureInitialized();
|
|
617
|
+
this.db.prepare(
|
|
618
|
+
"INSERT INTO correlation_events (entity, event_type, primary_key) VALUES (?, ?, ?)"
|
|
619
|
+
).run(event.entity, event.eventType, event.primaryKey);
|
|
620
|
+
}
|
|
621
|
+
getCorrelationEvents(entity, lastMinutes) {
|
|
622
|
+
this.ensureInitialized();
|
|
623
|
+
const rows = this.db.prepare(
|
|
624
|
+
"SELECT entity, event_type, primary_key, timestamp FROM correlation_events WHERE entity = ? AND timestamp >= datetime('now', '-' || ? || ' minutes') ORDER BY timestamp DESC"
|
|
625
|
+
).all(entity, lastMinutes);
|
|
626
|
+
return rows.map((r) => ({
|
|
627
|
+
entity: r.entity,
|
|
628
|
+
eventType: r.event_type,
|
|
629
|
+
primaryKey: r.primary_key,
|
|
630
|
+
timestamp: r.timestamp
|
|
631
|
+
}));
|
|
632
|
+
}
|
|
633
|
+
saveCorrelationRate(rate) {
|
|
634
|
+
this.ensureInitialized();
|
|
635
|
+
this.db.prepare(
|
|
636
|
+
"INSERT OR REPLACE INTO correlation_rates (entity, bucket, event_count) VALUES (?, ?, ?)"
|
|
637
|
+
).run(rate.entity, rate.bucket, rate.eventCount);
|
|
638
|
+
}
|
|
639
|
+
getCorrelationRates(entity, lastHours) {
|
|
640
|
+
this.ensureInitialized();
|
|
641
|
+
const cutoff = new Date(Date.now() - lastHours * 36e5).toISOString();
|
|
642
|
+
const rows = this.db.prepare(
|
|
643
|
+
"SELECT entity, bucket, event_count FROM correlation_rates WHERE entity = ? AND bucket >= ? ORDER BY bucket DESC"
|
|
644
|
+
).all(entity, cutoff);
|
|
645
|
+
return rows.map((r) => ({
|
|
646
|
+
entity: r.entity,
|
|
647
|
+
bucket: r.bucket,
|
|
648
|
+
eventCount: r.event_count
|
|
649
|
+
}));
|
|
650
|
+
}
|
|
651
|
+
saveHistogramBucket(bucket) {
|
|
652
|
+
this.ensureInitialized();
|
|
653
|
+
this.db.prepare(
|
|
654
|
+
"INSERT OR REPLACE INTO distribution_histograms (entity, column_name, bucket_key, count, period) VALUES (?, ?, ?, ?, ?)"
|
|
655
|
+
).run(bucket.entity, bucket.columnName, bucket.bucketKey, bucket.count, bucket.period);
|
|
656
|
+
}
|
|
657
|
+
getHistogramBuckets(entity, columnName, period) {
|
|
658
|
+
this.ensureInitialized();
|
|
659
|
+
const rows = this.db.prepare(
|
|
660
|
+
"SELECT bucket_key, count FROM distribution_histograms WHERE entity = ? AND column_name = ? AND period = ? ORDER BY bucket_key"
|
|
661
|
+
).all(entity, columnName, period);
|
|
662
|
+
return rows.map((r) => ({
|
|
663
|
+
bucketKey: r.bucket_key,
|
|
664
|
+
count: r.count
|
|
665
|
+
}));
|
|
666
|
+
}
|
|
667
|
+
saveTemporalPattern(pattern) {
|
|
668
|
+
this.ensureInitialized();
|
|
669
|
+
this.db.prepare(
|
|
670
|
+
"INSERT OR REPLACE INTO temporal_patterns (entity, bucket_type, bucket_key, count, period) VALUES (?, ?, ?, ?, ?)"
|
|
671
|
+
).run(pattern.entity, pattern.bucketType, pattern.bucketKey, pattern.count, pattern.period);
|
|
672
|
+
}
|
|
673
|
+
getTemporalPatterns(entity, bucketType, period) {
|
|
674
|
+
this.ensureInitialized();
|
|
675
|
+
const rows = this.db.prepare(
|
|
676
|
+
"SELECT bucket_key, count FROM temporal_patterns WHERE entity = ? AND bucket_type = ? AND period = ? ORDER BY bucket_key"
|
|
677
|
+
).all(entity, bucketType, period);
|
|
678
|
+
return rows.map((r) => ({
|
|
679
|
+
bucketKey: r.bucket_key,
|
|
680
|
+
count: r.count
|
|
681
|
+
}));
|
|
682
|
+
}
|
|
683
|
+
saveAnalyticsBaseline(baseline) {
|
|
684
|
+
this.ensureInitialized();
|
|
685
|
+
this.db.prepare(
|
|
686
|
+
"INSERT OR REPLACE INTO analytics_baselines (key, mean, stddev, sample_size) VALUES (?, ?, ?, ?)"
|
|
687
|
+
).run(baseline.key, baseline.mean, baseline.stddev, baseline.sampleSize);
|
|
688
|
+
}
|
|
689
|
+
getAnalyticsBaseline(key) {
|
|
690
|
+
this.ensureInitialized();
|
|
691
|
+
const row = this.db.prepare(
|
|
692
|
+
"SELECT mean, stddev, sample_size FROM analytics_baselines WHERE key = ?"
|
|
693
|
+
).get(key);
|
|
694
|
+
if (!row) return void 0;
|
|
695
|
+
return {
|
|
696
|
+
mean: row.mean,
|
|
697
|
+
stddev: row.stddev,
|
|
698
|
+
sampleSize: row.sample_size
|
|
699
|
+
};
|
|
700
|
+
}
|
|
701
|
+
// ── Knowledge Graph ──────────────────────────────────────────
|
|
702
|
+
saveKgNode(node) {
|
|
703
|
+
this.ensureInitialized();
|
|
704
|
+
const info = this.db.prepare(
|
|
705
|
+
`INSERT INTO kg_nodes (node_type, ref_id, label, metadata) VALUES (?, ?, ?, ?)
|
|
706
|
+
ON CONFLICT(node_type, ref_id) DO UPDATE SET label = excluded.label, metadata = excluded.metadata, timestamp = datetime('now')`
|
|
707
|
+
).run(node.nodeType, node.refId, node.label, node.metadata);
|
|
708
|
+
if (info.changes === 0) {
|
|
709
|
+
const existing = this.getKgNodeByRef(node.nodeType, node.refId);
|
|
710
|
+
return existing.id;
|
|
711
|
+
}
|
|
712
|
+
return Number(info.lastInsertRowid);
|
|
713
|
+
}
|
|
714
|
+
getKgNode(id) {
|
|
715
|
+
this.ensureInitialized();
|
|
716
|
+
const row = this.db.prepare(
|
|
717
|
+
"SELECT id, node_type, ref_id, label, metadata, timestamp FROM kg_nodes WHERE id = ?"
|
|
718
|
+
).get(id);
|
|
719
|
+
if (!row) return void 0;
|
|
720
|
+
return { id: row.id, nodeType: row.node_type, refId: row.ref_id, label: row.label, metadata: row.metadata, timestamp: row.timestamp };
|
|
721
|
+
}
|
|
722
|
+
getKgNodeByRef(nodeType, refId) {
|
|
723
|
+
this.ensureInitialized();
|
|
724
|
+
const row = this.db.prepare(
|
|
725
|
+
"SELECT id, node_type, ref_id, label, metadata, timestamp FROM kg_nodes WHERE node_type = ? AND ref_id = ?"
|
|
726
|
+
).get(nodeType, refId);
|
|
727
|
+
if (!row) return void 0;
|
|
728
|
+
return { id: row.id, nodeType: row.node_type, refId: row.ref_id, label: row.label, metadata: row.metadata, timestamp: row.timestamp };
|
|
729
|
+
}
|
|
730
|
+
getKgNodesByType(nodeType) {
|
|
731
|
+
this.ensureInitialized();
|
|
732
|
+
const rows = this.db.prepare(
|
|
733
|
+
"SELECT id, node_type, ref_id, label, metadata, timestamp FROM kg_nodes WHERE node_type = ? ORDER BY timestamp DESC"
|
|
734
|
+
).all(nodeType);
|
|
735
|
+
return rows.map((r) => ({ id: r.id, nodeType: r.node_type, refId: r.ref_id, label: r.label, metadata: r.metadata, timestamp: r.timestamp }));
|
|
736
|
+
}
|
|
737
|
+
getAllKgNodes() {
|
|
738
|
+
this.ensureInitialized();
|
|
739
|
+
const rows = this.db.prepare(
|
|
740
|
+
"SELECT id, node_type, ref_id, label, metadata, timestamp FROM kg_nodes ORDER BY timestamp DESC"
|
|
741
|
+
).all();
|
|
742
|
+
return rows.map((r) => ({ id: r.id, nodeType: r.node_type, refId: r.ref_id, label: r.label, metadata: r.metadata, timestamp: r.timestamp }));
|
|
743
|
+
}
|
|
744
|
+
deleteKgNode(id) {
|
|
745
|
+
this.ensureInitialized();
|
|
746
|
+
this.db.prepare("DELETE FROM kg_edges WHERE source_id = ? OR target_id = ?").run(id, id);
|
|
747
|
+
this.db.prepare("DELETE FROM kg_nodes WHERE id = ?").run(id);
|
|
748
|
+
}
|
|
749
|
+
saveKgEdge(edge) {
|
|
750
|
+
this.ensureInitialized();
|
|
751
|
+
const info = this.db.prepare(
|
|
752
|
+
`INSERT INTO kg_edges (source_id, target_id, edge_type, weight, metadata) VALUES (?, ?, ?, ?, ?)
|
|
753
|
+
ON CONFLICT(source_id, target_id, edge_type) DO UPDATE SET weight = excluded.weight, metadata = excluded.metadata, timestamp = datetime('now')`
|
|
754
|
+
).run(edge.sourceId, edge.targetId, edge.edgeType, edge.weight, edge.metadata);
|
|
755
|
+
return Number(info.lastInsertRowid);
|
|
756
|
+
}
|
|
757
|
+
getKgEdgesFrom(nodeId, edgeType) {
|
|
758
|
+
this.ensureInitialized();
|
|
759
|
+
let sql = "SELECT id, source_id, target_id, edge_type, weight, metadata, timestamp FROM kg_edges WHERE source_id = ?";
|
|
760
|
+
const params = [nodeId];
|
|
761
|
+
if (edgeType) {
|
|
762
|
+
sql += " AND edge_type = ?";
|
|
763
|
+
params.push(edgeType);
|
|
764
|
+
}
|
|
765
|
+
sql += " ORDER BY timestamp DESC";
|
|
766
|
+
const rows = this.db.prepare(sql).all(...params);
|
|
767
|
+
return rows.map((r) => ({ id: r.id, sourceId: r.source_id, targetId: r.target_id, edgeType: r.edge_type, weight: r.weight, metadata: r.metadata, timestamp: r.timestamp }));
|
|
768
|
+
}
|
|
769
|
+
getKgEdgesTo(nodeId, edgeType) {
|
|
770
|
+
this.ensureInitialized();
|
|
771
|
+
let sql = "SELECT id, source_id, target_id, edge_type, weight, metadata, timestamp FROM kg_edges WHERE target_id = ?";
|
|
772
|
+
const params = [nodeId];
|
|
773
|
+
if (edgeType) {
|
|
774
|
+
sql += " AND edge_type = ?";
|
|
775
|
+
params.push(edgeType);
|
|
776
|
+
}
|
|
777
|
+
sql += " ORDER BY timestamp DESC";
|
|
778
|
+
const rows = this.db.prepare(sql).all(...params);
|
|
779
|
+
return rows.map((r) => ({ id: r.id, sourceId: r.source_id, targetId: r.target_id, edgeType: r.edge_type, weight: r.weight, metadata: r.metadata, timestamp: r.timestamp }));
|
|
780
|
+
}
|
|
781
|
+
getAllKgEdges() {
|
|
782
|
+
this.ensureInitialized();
|
|
783
|
+
const rows = this.db.prepare(
|
|
784
|
+
"SELECT id, source_id, target_id, edge_type, weight, metadata, timestamp FROM kg_edges ORDER BY timestamp DESC"
|
|
785
|
+
).all();
|
|
786
|
+
return rows.map((r) => ({ id: r.id, sourceId: r.source_id, targetId: r.target_id, edgeType: r.edge_type, weight: r.weight, metadata: r.metadata, timestamp: r.timestamp }));
|
|
787
|
+
}
|
|
788
|
+
getKgSummary() {
|
|
789
|
+
this.ensureInitialized();
|
|
790
|
+
const nodeCount = this.db.prepare("SELECT COUNT(*) as count FROM kg_nodes").get();
|
|
791
|
+
const edgeCount = this.db.prepare("SELECT COUNT(*) as count FROM kg_edges").get();
|
|
792
|
+
const entityCount = this.db.prepare("SELECT COUNT(*) as count FROM kg_nodes WHERE node_type = 'entity'").get();
|
|
793
|
+
const eventCount = this.db.prepare("SELECT COUNT(*) as count FROM kg_nodes WHERE node_type = 'event'").get();
|
|
794
|
+
const anomalyCount = this.db.prepare("SELECT COUNT(*) as count FROM kg_nodes WHERE node_type = 'anomaly'").get();
|
|
795
|
+
const insightCount = this.db.prepare("SELECT COUNT(*) as count FROM kg_nodes WHERE node_type = 'insight'").get();
|
|
796
|
+
return { totalNodes: nodeCount.count, totalEdges: edgeCount.count, entities: entityCount.count, events: eventCount.count, anomalies: anomalyCount.count, insights: insightCount.count };
|
|
797
|
+
}
|
|
798
|
+
// ── Recommendations ──────────────────────────────────────────
|
|
799
|
+
saveRecommendation(rec) {
|
|
800
|
+
this.ensureInitialized();
|
|
801
|
+
const info = this.db.prepare(
|
|
802
|
+
"INSERT INTO recommendations (action, reason, confidence, status, governance, insight_id, entity, context) VALUES (?, ?, ?, ?, ?, ?, ?, ?)"
|
|
803
|
+
).run(rec.action, rec.reason, rec.confidence, rec.status, rec.governance, rec.insightId, rec.entity, rec.context);
|
|
804
|
+
return Number(info.lastInsertRowid);
|
|
805
|
+
}
|
|
806
|
+
getRecommendation(id) {
|
|
807
|
+
this.ensureInitialized();
|
|
808
|
+
const row = this.db.prepare(
|
|
809
|
+
"SELECT id, action, reason, confidence, status, governance, insight_id, entity, context, created_at, resolved_at, resolved_by FROM recommendations WHERE id = ?"
|
|
810
|
+
).get(id);
|
|
811
|
+
if (!row) return void 0;
|
|
812
|
+
return {
|
|
813
|
+
id: row.id,
|
|
814
|
+
action: row.action,
|
|
815
|
+
reason: row.reason,
|
|
816
|
+
confidence: row.confidence,
|
|
817
|
+
status: row.status,
|
|
818
|
+
governance: row.governance,
|
|
819
|
+
insightId: row.insight_id,
|
|
820
|
+
entity: row.entity,
|
|
821
|
+
context: row.context,
|
|
822
|
+
createdAt: row.created_at,
|
|
823
|
+
resolvedAt: row.resolved_at,
|
|
824
|
+
resolvedBy: row.resolved_by
|
|
825
|
+
};
|
|
826
|
+
}
|
|
827
|
+
getRecommendations(filter) {
|
|
828
|
+
this.ensureInitialized();
|
|
829
|
+
let sql = "SELECT id, action, reason, confidence, status, governance, insight_id, entity, context, created_at, resolved_at, resolved_by FROM recommendations";
|
|
830
|
+
const conditions = [];
|
|
831
|
+
const params = [];
|
|
832
|
+
if (filter?.status) {
|
|
833
|
+
conditions.push("status = ?");
|
|
834
|
+
params.push(filter.status);
|
|
835
|
+
}
|
|
836
|
+
if (filter?.action) {
|
|
837
|
+
conditions.push("action = ?");
|
|
838
|
+
params.push(filter.action);
|
|
839
|
+
}
|
|
840
|
+
if (conditions.length > 0) {
|
|
841
|
+
sql += " WHERE " + conditions.join(" AND ");
|
|
842
|
+
}
|
|
843
|
+
sql += " ORDER BY created_at DESC";
|
|
844
|
+
if (filter?.last) {
|
|
845
|
+
sql += " LIMIT ?";
|
|
846
|
+
params.push(filter.last);
|
|
847
|
+
}
|
|
848
|
+
const rows = this.db.prepare(sql).all(...params);
|
|
849
|
+
return rows.map((row) => ({
|
|
850
|
+
id: row.id,
|
|
851
|
+
action: row.action,
|
|
852
|
+
reason: row.reason,
|
|
853
|
+
confidence: row.confidence,
|
|
854
|
+
status: row.status,
|
|
855
|
+
governance: row.governance,
|
|
856
|
+
insightId: row.insight_id,
|
|
857
|
+
entity: row.entity,
|
|
858
|
+
context: row.context,
|
|
859
|
+
createdAt: row.created_at,
|
|
860
|
+
resolvedAt: row.resolved_at,
|
|
861
|
+
resolvedBy: row.resolved_by
|
|
862
|
+
}));
|
|
863
|
+
}
|
|
864
|
+
updateRecommendationStatus(id, status, resolvedBy) {
|
|
865
|
+
this.ensureInitialized();
|
|
866
|
+
this.db.prepare(
|
|
867
|
+
"UPDATE recommendations SET status = ?, resolved_at = datetime('now'), resolved_by = ? WHERE id = ?"
|
|
868
|
+
).run(status, resolvedBy, id);
|
|
869
|
+
}
|
|
870
|
+
getRecommendationSummary() {
|
|
871
|
+
this.ensureInitialized();
|
|
872
|
+
const rows = this.db.prepare(
|
|
873
|
+
"SELECT status, COUNT(*) as count FROM recommendations GROUP BY status"
|
|
874
|
+
).all();
|
|
875
|
+
const counts = {};
|
|
876
|
+
let total = 0;
|
|
877
|
+
for (const row of rows) {
|
|
878
|
+
counts[row.status] = row.count;
|
|
879
|
+
total += row.count;
|
|
880
|
+
}
|
|
881
|
+
return {
|
|
882
|
+
total,
|
|
883
|
+
executed: counts["executed"] ?? 0,
|
|
884
|
+
pending: counts["pending"] ?? 0,
|
|
885
|
+
approved: counts["approved"] ?? 0,
|
|
886
|
+
rejected: counts["rejected"] ?? 0,
|
|
887
|
+
blocked: counts["blocked"] ?? 0,
|
|
888
|
+
failed: counts["failed"] ?? 0
|
|
889
|
+
};
|
|
890
|
+
}
|
|
891
|
+
close() {
|
|
892
|
+
if (this.db) {
|
|
893
|
+
this.db.close();
|
|
894
|
+
this.db = null;
|
|
895
|
+
}
|
|
896
|
+
}
|
|
897
|
+
createSchema() {
|
|
898
|
+
const createMetaTable = `
|
|
899
|
+
CREATE TABLE IF NOT EXISTS meta (
|
|
900
|
+
key TEXT PRIMARY KEY,
|
|
901
|
+
value TEXT
|
|
902
|
+
)
|
|
903
|
+
`;
|
|
904
|
+
const createDiscoveredTablesTable = `
|
|
905
|
+
CREATE TABLE IF NOT EXISTS discovered_tables (
|
|
906
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
907
|
+
table_name TEXT NOT NULL,
|
|
908
|
+
table_schema TEXT DEFAULT 'public',
|
|
909
|
+
row_count INTEGER,
|
|
910
|
+
columns_json TEXT,
|
|
911
|
+
fk_json TEXT,
|
|
912
|
+
indexes_json TEXT,
|
|
913
|
+
sample_rows TEXT,
|
|
914
|
+
discovered_at TEXT DEFAULT (datetime('now')),
|
|
915
|
+
UNIQUE(table_name, table_schema)
|
|
916
|
+
)
|
|
917
|
+
`;
|
|
918
|
+
const createEntitiesTable = `
|
|
919
|
+
CREATE TABLE IF NOT EXISTS entities (
|
|
920
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
921
|
+
table_name TEXT NOT NULL UNIQUE,
|
|
922
|
+
entity_label TEXT NOT NULL,
|
|
923
|
+
entity_type TEXT NOT NULL,
|
|
924
|
+
description TEXT,
|
|
925
|
+
confidence REAL DEFAULT 0.0,
|
|
926
|
+
columns_classified TEXT,
|
|
927
|
+
classified_at TEXT DEFAULT (datetime('now'))
|
|
928
|
+
)
|
|
929
|
+
`;
|
|
930
|
+
const createRelationshipsTable = `
|
|
931
|
+
CREATE TABLE IF NOT EXISTS relationships (
|
|
932
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
933
|
+
source_entity TEXT NOT NULL,
|
|
934
|
+
target_entity TEXT NOT NULL,
|
|
935
|
+
relationship TEXT NOT NULL,
|
|
936
|
+
source_column TEXT,
|
|
937
|
+
target_column TEXT,
|
|
938
|
+
inferred INTEGER DEFAULT 0,
|
|
939
|
+
confidence REAL DEFAULT 0.0,
|
|
940
|
+
UNIQUE(source_entity, target_entity, relationship)
|
|
941
|
+
)
|
|
942
|
+
`;
|
|
943
|
+
const createLlmCacheTable = `
|
|
944
|
+
CREATE TABLE IF NOT EXISTS llm_cache (
|
|
945
|
+
cache_key TEXT PRIMARY KEY,
|
|
946
|
+
response TEXT NOT NULL,
|
|
947
|
+
model TEXT,
|
|
948
|
+
tokens_used INTEGER,
|
|
949
|
+
cached_at TEXT DEFAULT (datetime('now'))
|
|
950
|
+
)
|
|
951
|
+
`;
|
|
952
|
+
const createEventsTable = `
|
|
953
|
+
CREATE TABLE IF NOT EXISTS events (
|
|
954
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
955
|
+
event_type TEXT NOT NULL,
|
|
956
|
+
entity TEXT NOT NULL,
|
|
957
|
+
table_name TEXT NOT NULL,
|
|
958
|
+
operation TEXT NOT NULL,
|
|
959
|
+
row_id TEXT,
|
|
960
|
+
row_data TEXT,
|
|
961
|
+
timestamp TEXT DEFAULT (datetime('now'))
|
|
962
|
+
)
|
|
963
|
+
`;
|
|
964
|
+
const createEventsEntityIndex = `CREATE INDEX IF NOT EXISTS idx_events_entity ON events(entity, timestamp)`;
|
|
965
|
+
const createEventsTypeIndex = `CREATE INDEX IF NOT EXISTS idx_events_type ON events(event_type, timestamp)`;
|
|
966
|
+
const createBaselinesTable = `
|
|
967
|
+
CREATE TABLE IF NOT EXISTS baselines (
|
|
968
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
969
|
+
entity TEXT NOT NULL,
|
|
970
|
+
metric TEXT NOT NULL,
|
|
971
|
+
mean REAL NOT NULL,
|
|
972
|
+
stddev REAL NOT NULL,
|
|
973
|
+
sample_size INTEGER NOT NULL,
|
|
974
|
+
updated_at TEXT DEFAULT (datetime('now')),
|
|
975
|
+
UNIQUE(entity, metric)
|
|
976
|
+
)
|
|
977
|
+
`;
|
|
978
|
+
const createAnomaliesTable = `
|
|
979
|
+
CREATE TABLE IF NOT EXISTS anomalies (
|
|
980
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
981
|
+
entity TEXT NOT NULL,
|
|
982
|
+
anomaly_type TEXT NOT NULL,
|
|
983
|
+
severity TEXT NOT NULL,
|
|
984
|
+
expected REAL,
|
|
985
|
+
actual REAL,
|
|
986
|
+
message TEXT,
|
|
987
|
+
timestamp TEXT DEFAULT (datetime('now'))
|
|
988
|
+
)
|
|
989
|
+
`;
|
|
990
|
+
const createWatchWatermarksTable = `
|
|
991
|
+
CREATE TABLE IF NOT EXISTS watch_watermarks (
|
|
992
|
+
table_name TEXT NOT NULL,
|
|
993
|
+
table_schema TEXT NOT NULL,
|
|
994
|
+
strategy TEXT NOT NULL,
|
|
995
|
+
timestamp_column TEXT,
|
|
996
|
+
pk_column TEXT,
|
|
997
|
+
last_seen_value TEXT,
|
|
998
|
+
updated_at TEXT DEFAULT (datetime('now')),
|
|
999
|
+
PRIMARY KEY(table_name, table_schema)
|
|
1000
|
+
)
|
|
1001
|
+
`;
|
|
1002
|
+
const createStateTransitionsTable = `
|
|
1003
|
+
CREATE TABLE IF NOT EXISTS state_transitions (
|
|
1004
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
1005
|
+
entity TEXT NOT NULL,
|
|
1006
|
+
table_name TEXT NOT NULL,
|
|
1007
|
+
primary_key TEXT NOT NULL,
|
|
1008
|
+
from_state TEXT NOT NULL,
|
|
1009
|
+
to_state TEXT NOT NULL,
|
|
1010
|
+
duration_ms INTEGER,
|
|
1011
|
+
expected INTEGER NOT NULL,
|
|
1012
|
+
timestamp TEXT DEFAULT (datetime('now'))
|
|
1013
|
+
)
|
|
1014
|
+
`;
|
|
1015
|
+
const createTransitionsEntityIndex = `CREATE INDEX IF NOT EXISTS idx_transitions_entity ON state_transitions(entity, timestamp)`;
|
|
1016
|
+
const createTransitionsPkIndex = `CREATE INDEX IF NOT EXISTS idx_transitions_pk ON state_transitions(table_name, primary_key)`;
|
|
1017
|
+
const createInsightsTable = `
|
|
1018
|
+
CREATE TABLE IF NOT EXISTS insights (
|
|
1019
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
1020
|
+
entity TEXT NOT NULL,
|
|
1021
|
+
insight_type TEXT NOT NULL,
|
|
1022
|
+
severity TEXT NOT NULL,
|
|
1023
|
+
message TEXT NOT NULL,
|
|
1024
|
+
context_json TEXT,
|
|
1025
|
+
timestamp TEXT DEFAULT (datetime('now'))
|
|
1026
|
+
)
|
|
1027
|
+
`;
|
|
1028
|
+
const createInsightsEntityIndex = `CREATE INDEX IF NOT EXISTS idx_insights_entity ON insights(entity, timestamp)`;
|
|
1029
|
+
const createEntityStatesTable = `
|
|
1030
|
+
CREATE TABLE IF NOT EXISTS entity_states (
|
|
1031
|
+
table_name TEXT NOT NULL,
|
|
1032
|
+
primary_key TEXT NOT NULL,
|
|
1033
|
+
state TEXT NOT NULL,
|
|
1034
|
+
entered_at TEXT NOT NULL,
|
|
1035
|
+
PRIMARY KEY(table_name, primary_key)
|
|
1036
|
+
)
|
|
1037
|
+
`;
|
|
1038
|
+
const createCorrelationEventsTable = `
|
|
1039
|
+
CREATE TABLE IF NOT EXISTS correlation_events (
|
|
1040
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
1041
|
+
entity TEXT NOT NULL,
|
|
1042
|
+
event_type TEXT NOT NULL,
|
|
1043
|
+
primary_key TEXT,
|
|
1044
|
+
timestamp TEXT DEFAULT (datetime('now'))
|
|
1045
|
+
)
|
|
1046
|
+
`;
|
|
1047
|
+
const createCorrelationEventsIndex = `CREATE INDEX IF NOT EXISTS idx_corr_events ON correlation_events(entity, timestamp)`;
|
|
1048
|
+
const createCorrelationRatesTable = `
|
|
1049
|
+
CREATE TABLE IF NOT EXISTS correlation_rates (
|
|
1050
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
1051
|
+
entity TEXT NOT NULL,
|
|
1052
|
+
bucket TEXT NOT NULL,
|
|
1053
|
+
event_count INTEGER NOT NULL,
|
|
1054
|
+
UNIQUE(entity, bucket)
|
|
1055
|
+
)
|
|
1056
|
+
`;
|
|
1057
|
+
const createDistributionHistogramsTable = `
|
|
1058
|
+
CREATE TABLE IF NOT EXISTS distribution_histograms (
|
|
1059
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
1060
|
+
entity TEXT NOT NULL,
|
|
1061
|
+
column_name TEXT NOT NULL,
|
|
1062
|
+
bucket_key TEXT NOT NULL,
|
|
1063
|
+
count INTEGER NOT NULL,
|
|
1064
|
+
period TEXT NOT NULL,
|
|
1065
|
+
UNIQUE(entity, column_name, bucket_key, period)
|
|
1066
|
+
)
|
|
1067
|
+
`;
|
|
1068
|
+
const createTemporalPatternsTable = `
|
|
1069
|
+
CREATE TABLE IF NOT EXISTS temporal_patterns (
|
|
1070
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
1071
|
+
entity TEXT NOT NULL,
|
|
1072
|
+
bucket_type TEXT NOT NULL,
|
|
1073
|
+
bucket_key INTEGER NOT NULL,
|
|
1074
|
+
count INTEGER NOT NULL,
|
|
1075
|
+
period TEXT NOT NULL,
|
|
1076
|
+
UNIQUE(entity, bucket_type, bucket_key, period)
|
|
1077
|
+
)
|
|
1078
|
+
`;
|
|
1079
|
+
const createAnalyticsBaselinesTable = `
|
|
1080
|
+
CREATE TABLE IF NOT EXISTS analytics_baselines (
|
|
1081
|
+
key TEXT PRIMARY KEY,
|
|
1082
|
+
mean REAL NOT NULL,
|
|
1083
|
+
stddev REAL NOT NULL,
|
|
1084
|
+
sample_size INTEGER NOT NULL,
|
|
1085
|
+
updated_at TEXT DEFAULT (datetime('now'))
|
|
1086
|
+
)
|
|
1087
|
+
`;
|
|
1088
|
+
const createKgNodesTable = `
|
|
1089
|
+
CREATE TABLE IF NOT EXISTS kg_nodes (
|
|
1090
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
1091
|
+
node_type TEXT NOT NULL,
|
|
1092
|
+
ref_id TEXT NOT NULL,
|
|
1093
|
+
label TEXT NOT NULL,
|
|
1094
|
+
metadata TEXT,
|
|
1095
|
+
timestamp TEXT DEFAULT (datetime('now')),
|
|
1096
|
+
UNIQUE(node_type, ref_id)
|
|
1097
|
+
)
|
|
1098
|
+
`;
|
|
1099
|
+
const createKgNodesTypeIndex = `CREATE INDEX IF NOT EXISTS idx_kg_nodes_type ON kg_nodes(node_type)`;
|
|
1100
|
+
const createKgNodesRefIndex = `CREATE INDEX IF NOT EXISTS idx_kg_nodes_ref ON kg_nodes(ref_id)`;
|
|
1101
|
+
const createKgEdgesTable = `
|
|
1102
|
+
CREATE TABLE IF NOT EXISTS kg_edges (
|
|
1103
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
1104
|
+
source_id INTEGER NOT NULL REFERENCES kg_nodes(id),
|
|
1105
|
+
target_id INTEGER NOT NULL REFERENCES kg_nodes(id),
|
|
1106
|
+
edge_type TEXT NOT NULL,
|
|
1107
|
+
weight REAL DEFAULT 1.0,
|
|
1108
|
+
metadata TEXT,
|
|
1109
|
+
timestamp TEXT DEFAULT (datetime('now')),
|
|
1110
|
+
UNIQUE(source_id, target_id, edge_type)
|
|
1111
|
+
)
|
|
1112
|
+
`;
|
|
1113
|
+
const createKgEdgesSourceIndex = `CREATE INDEX IF NOT EXISTS idx_kg_edges_source ON kg_edges(source_id)`;
|
|
1114
|
+
const createKgEdgesTargetIndex = `CREATE INDEX IF NOT EXISTS idx_kg_edges_target ON kg_edges(target_id)`;
|
|
1115
|
+
const createKgEdgesTypeIndex = `CREATE INDEX IF NOT EXISTS idx_kg_edges_type ON kg_edges(edge_type)`;
|
|
1116
|
+
const createRecommendationsTable = `
|
|
1117
|
+
CREATE TABLE IF NOT EXISTS recommendations (
|
|
1118
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
1119
|
+
action TEXT NOT NULL,
|
|
1120
|
+
reason TEXT NOT NULL,
|
|
1121
|
+
confidence REAL NOT NULL,
|
|
1122
|
+
status TEXT NOT NULL DEFAULT 'pending',
|
|
1123
|
+
governance TEXT NOT NULL,
|
|
1124
|
+
insight_id TEXT,
|
|
1125
|
+
entity TEXT NOT NULL,
|
|
1126
|
+
context TEXT NOT NULL DEFAULT '{}',
|
|
1127
|
+
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
|
1128
|
+
resolved_at TEXT,
|
|
1129
|
+
resolved_by TEXT
|
|
1130
|
+
)
|
|
1131
|
+
`;
|
|
1132
|
+
const createRecommendationsStatusIndex = `CREATE INDEX IF NOT EXISTS idx_recommendations_status ON recommendations(status)`;
|
|
1133
|
+
const createRecommendationsEntityIndex = `CREATE INDEX IF NOT EXISTS idx_recommendations_entity ON recommendations(entity)`;
|
|
1134
|
+
const createRecommendationsActionIndex = `CREATE INDEX IF NOT EXISTS idx_recommendations_action ON recommendations(action)`;
|
|
1135
|
+
this.db.prepare(createMetaTable).run();
|
|
1136
|
+
this.db.prepare(createDiscoveredTablesTable).run();
|
|
1137
|
+
this.db.prepare(createEntitiesTable).run();
|
|
1138
|
+
this.db.prepare(createRelationshipsTable).run();
|
|
1139
|
+
this.db.prepare(createLlmCacheTable).run();
|
|
1140
|
+
this.db.prepare(createEventsTable).run();
|
|
1141
|
+
this.db.prepare(createEventsEntityIndex).run();
|
|
1142
|
+
this.db.prepare(createEventsTypeIndex).run();
|
|
1143
|
+
this.db.prepare(createBaselinesTable).run();
|
|
1144
|
+
this.db.prepare(createAnomaliesTable).run();
|
|
1145
|
+
this.db.prepare(createWatchWatermarksTable).run();
|
|
1146
|
+
this.db.prepare(createStateTransitionsTable).run();
|
|
1147
|
+
this.db.prepare(createTransitionsEntityIndex).run();
|
|
1148
|
+
this.db.prepare(createTransitionsPkIndex).run();
|
|
1149
|
+
this.db.prepare(createInsightsTable).run();
|
|
1150
|
+
this.db.prepare(createInsightsEntityIndex).run();
|
|
1151
|
+
this.db.prepare(createEntityStatesTable).run();
|
|
1152
|
+
this.db.prepare(createCorrelationEventsTable).run();
|
|
1153
|
+
this.db.prepare(createCorrelationEventsIndex).run();
|
|
1154
|
+
this.db.prepare(createCorrelationRatesTable).run();
|
|
1155
|
+
this.db.prepare(createDistributionHistogramsTable).run();
|
|
1156
|
+
this.db.prepare(createTemporalPatternsTable).run();
|
|
1157
|
+
this.db.prepare(createAnalyticsBaselinesTable).run();
|
|
1158
|
+
this.db.prepare(createKgNodesTable).run();
|
|
1159
|
+
this.db.prepare(createKgNodesTypeIndex).run();
|
|
1160
|
+
this.db.prepare(createKgNodesRefIndex).run();
|
|
1161
|
+
this.db.prepare(createKgEdgesTable).run();
|
|
1162
|
+
this.db.prepare(createKgEdgesSourceIndex).run();
|
|
1163
|
+
this.db.prepare(createKgEdgesTargetIndex).run();
|
|
1164
|
+
this.db.prepare(createKgEdgesTypeIndex).run();
|
|
1165
|
+
this.db.prepare(createRecommendationsTable).run();
|
|
1166
|
+
this.db.prepare(createRecommendationsStatusIndex).run();
|
|
1167
|
+
this.db.prepare(createRecommendationsEntityIndex).run();
|
|
1168
|
+
this.db.prepare(createRecommendationsActionIndex).run();
|
|
1169
|
+
}
|
|
1170
|
+
ensureInitialized() {
|
|
1171
|
+
if (!this.db) {
|
|
1172
|
+
throw new Error("Storage not initialized. Call initialize() first.");
|
|
1173
|
+
}
|
|
1174
|
+
}
|
|
1175
|
+
};
|
|
1176
|
+
|
|
1177
|
+
// src/core/errors.ts
|
|
1178
|
+
var CortexaError = class extends Error {
|
|
1179
|
+
code;
|
|
1180
|
+
hint;
|
|
1181
|
+
constructor(message, code, hint) {
|
|
1182
|
+
super(message);
|
|
1183
|
+
this.name = "CortexaError";
|
|
1184
|
+
this.code = code;
|
|
1185
|
+
this.hint = hint;
|
|
1186
|
+
}
|
|
1187
|
+
};
|
|
1188
|
+
var ConfigError = class extends CortexaError {
|
|
1189
|
+
constructor(message, hint) {
|
|
1190
|
+
super(message, "CONFIG_ERROR", hint);
|
|
1191
|
+
this.name = "ConfigError";
|
|
1192
|
+
}
|
|
1193
|
+
};
|
|
1194
|
+
var LLMError = class extends CortexaError {
|
|
1195
|
+
statusCode;
|
|
1196
|
+
provider;
|
|
1197
|
+
constructor(message, options) {
|
|
1198
|
+
super(message, "LLM_ERROR", options?.hint);
|
|
1199
|
+
this.name = "LLMError";
|
|
1200
|
+
this.statusCode = options?.statusCode;
|
|
1201
|
+
this.provider = options?.provider;
|
|
1202
|
+
}
|
|
1203
|
+
};
|
|
1204
|
+
var LLMResponseError = class extends LLMError {
|
|
1205
|
+
rawContent;
|
|
1206
|
+
constructor(message, rawContent) {
|
|
1207
|
+
super(message, {
|
|
1208
|
+
hint: "The LLM returned an unparseable response. Try reducing batchSize or using a different model."
|
|
1209
|
+
});
|
|
1210
|
+
this.name = "LLMResponseError";
|
|
1211
|
+
this.rawContent = rawContent;
|
|
1212
|
+
}
|
|
1213
|
+
};
|
|
1214
|
+
|
|
1215
|
+
// src/llm/retry.ts
|
|
1216
|
+
var DEFAULT_RETRYABLE_CODES = /* @__PURE__ */ new Set([429, 500, 502, 503, 504]);
|
|
1217
|
+
function isRetryableError(error, retryableCodes) {
|
|
1218
|
+
if (error instanceof LLMError && error.statusCode) {
|
|
1219
|
+
return retryableCodes.has(error.statusCode);
|
|
1220
|
+
}
|
|
1221
|
+
if (error instanceof TypeError) {
|
|
1222
|
+
return true;
|
|
1223
|
+
}
|
|
1224
|
+
return false;
|
|
1225
|
+
}
|
|
1226
|
+
function sleep(ms) {
|
|
1227
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
1228
|
+
}
|
|
1229
|
+
async function withRetry(fn, config, logger10) {
|
|
1230
|
+
const maxRetries = config?.maxRetries ?? 3;
|
|
1231
|
+
const initialDelay = config?.initialDelayMs ?? 1e3;
|
|
1232
|
+
const maxDelay = config?.maxDelayMs ?? 3e4;
|
|
1233
|
+
const multiplier = config?.backoffMultiplier ?? 2;
|
|
1234
|
+
const retryableCodes = config?.retryableStatusCodes ? new Set(config.retryableStatusCodes) : DEFAULT_RETRYABLE_CODES;
|
|
1235
|
+
let lastError;
|
|
1236
|
+
for (let attempt = 0; attempt <= maxRetries; attempt++) {
|
|
1237
|
+
try {
|
|
1238
|
+
return await fn();
|
|
1239
|
+
} catch (error) {
|
|
1240
|
+
lastError = error;
|
|
1241
|
+
if (attempt === maxRetries) break;
|
|
1242
|
+
if (!isRetryableError(error, retryableCodes)) {
|
|
1243
|
+
throw error;
|
|
1244
|
+
}
|
|
1245
|
+
const delay = Math.min(initialDelay * Math.pow(multiplier, attempt), maxDelay);
|
|
1246
|
+
const jitteredDelay = delay * (0.75 + Math.random() * 0.5);
|
|
1247
|
+
logger10?.warn(
|
|
1248
|
+
{ attempt: attempt + 1, maxRetries, delayMs: Math.round(jitteredDelay) },
|
|
1249
|
+
`LLM call failed, retrying... (${lastError.message})`
|
|
1250
|
+
);
|
|
1251
|
+
await sleep(jitteredDelay);
|
|
1252
|
+
}
|
|
1253
|
+
}
|
|
1254
|
+
throw lastError;
|
|
1255
|
+
}
|
|
1256
|
+
|
|
1257
|
+
// src/llm/openai-compatible.ts
|
|
1258
|
+
var OpenAICompatibleProvider = class {
|
|
1259
|
+
apiKey;
|
|
1260
|
+
model;
|
|
1261
|
+
baseUrl;
|
|
1262
|
+
headers;
|
|
1263
|
+
retry;
|
|
1264
|
+
constructor(options) {
|
|
1265
|
+
this.apiKey = options.apiKey ?? "";
|
|
1266
|
+
this.model = options.model;
|
|
1267
|
+
this.baseUrl = options.baseUrl.replace(/\/+$/, "");
|
|
1268
|
+
this.headers = options.headers ?? {};
|
|
1269
|
+
this.retry = options.retry;
|
|
1270
|
+
}
|
|
1271
|
+
async chat(prompt, options) {
|
|
1272
|
+
return withRetry(() => this.doChat(prompt, options), this.retry);
|
|
1273
|
+
}
|
|
1274
|
+
async doChat(prompt, options) {
|
|
1275
|
+
const body = {
|
|
1276
|
+
model: this.model,
|
|
1277
|
+
messages: [{ role: "user", content: prompt }],
|
|
1278
|
+
max_tokens: options?.maxTokens ?? 4096
|
|
1279
|
+
};
|
|
1280
|
+
if (options?.jsonMode) {
|
|
1281
|
+
body.response_format = { type: "json_object" };
|
|
1282
|
+
}
|
|
1283
|
+
const headers = {
|
|
1284
|
+
"Content-Type": "application/json",
|
|
1285
|
+
...this.headers
|
|
1286
|
+
};
|
|
1287
|
+
if (this.apiKey) {
|
|
1288
|
+
headers["Authorization"] = `Bearer ${this.apiKey}`;
|
|
1289
|
+
}
|
|
1290
|
+
const url = `${this.baseUrl}/chat/completions`;
|
|
1291
|
+
const response = await fetch(url, {
|
|
1292
|
+
method: "POST",
|
|
1293
|
+
headers,
|
|
1294
|
+
body: JSON.stringify(body)
|
|
1295
|
+
});
|
|
1296
|
+
if (!response.ok) {
|
|
1297
|
+
const errorText = await response.text();
|
|
1298
|
+
throw new LLMError(
|
|
1299
|
+
`LLM API error (${response.status}): ${errorText}`,
|
|
1300
|
+
{
|
|
1301
|
+
statusCode: response.status,
|
|
1302
|
+
provider: this.model,
|
|
1303
|
+
hint: response.status === 401 ? "Check your API key in cortexa.config.ts." : response.status === 429 ? "Rate limited. Wait a moment and retry, or use a different model." : response.status >= 500 ? "The LLM provider returned a server error. This is usually temporary." : void 0
|
|
1304
|
+
}
|
|
1305
|
+
);
|
|
1306
|
+
}
|
|
1307
|
+
const data = await response.json();
|
|
1308
|
+
return {
|
|
1309
|
+
content: data.choices[0].message.content,
|
|
1310
|
+
tokensUsed: {
|
|
1311
|
+
prompt: data.usage?.prompt_tokens ?? 0,
|
|
1312
|
+
completion: data.usage?.completion_tokens ?? 0
|
|
1313
|
+
},
|
|
1314
|
+
model: data.model ?? this.model
|
|
1315
|
+
};
|
|
1316
|
+
}
|
|
1317
|
+
};
|
|
1318
|
+
|
|
1319
|
+
// src/llm/anthropic.ts
|
|
1320
|
+
var AnthropicProvider = class {
|
|
1321
|
+
apiKey;
|
|
1322
|
+
model;
|
|
1323
|
+
retry;
|
|
1324
|
+
constructor(apiKey, model = "claude-haiku-4-5-20251001", retry) {
|
|
1325
|
+
this.apiKey = apiKey;
|
|
1326
|
+
this.model = model;
|
|
1327
|
+
this.retry = retry;
|
|
1328
|
+
}
|
|
1329
|
+
async chat(prompt, options) {
|
|
1330
|
+
return withRetry(() => this.doChat(prompt, options), this.retry);
|
|
1331
|
+
}
|
|
1332
|
+
async doChat(prompt, options) {
|
|
1333
|
+
const body = {
|
|
1334
|
+
model: this.model,
|
|
1335
|
+
max_tokens: options?.maxTokens ?? 4096,
|
|
1336
|
+
messages: [{ role: "user", content: prompt }]
|
|
1337
|
+
};
|
|
1338
|
+
const response = await fetch("https://api.anthropic.com/v1/messages", {
|
|
1339
|
+
method: "POST",
|
|
1340
|
+
headers: {
|
|
1341
|
+
"Content-Type": "application/json",
|
|
1342
|
+
"x-api-key": this.apiKey,
|
|
1343
|
+
"anthropic-version": "2023-06-01"
|
|
1344
|
+
},
|
|
1345
|
+
body: JSON.stringify(body)
|
|
1346
|
+
});
|
|
1347
|
+
if (!response.ok) {
|
|
1348
|
+
const errorText = await response.text();
|
|
1349
|
+
throw new LLMError(
|
|
1350
|
+
`Anthropic API error (${response.status}): ${errorText}`,
|
|
1351
|
+
{
|
|
1352
|
+
statusCode: response.status,
|
|
1353
|
+
provider: "anthropic",
|
|
1354
|
+
hint: response.status === 401 ? "Check your Anthropic API key in cortexa.config.ts." : response.status === 429 ? "Rate limited. Wait a moment and retry, or use a different model." : response.status >= 500 ? "The Anthropic API returned a server error. This is usually temporary." : void 0
|
|
1355
|
+
}
|
|
1356
|
+
);
|
|
1357
|
+
}
|
|
1358
|
+
const data = await response.json();
|
|
1359
|
+
return {
|
|
1360
|
+
content: data.content[0].text,
|
|
1361
|
+
tokensUsed: {
|
|
1362
|
+
prompt: data.usage.input_tokens,
|
|
1363
|
+
completion: data.usage.output_tokens
|
|
1364
|
+
},
|
|
1365
|
+
model: data.model
|
|
1366
|
+
};
|
|
1367
|
+
}
|
|
1368
|
+
};
|
|
1369
|
+
|
|
1370
|
+
// src/llm/factory.ts
|
|
1371
|
+
var KNOWN_BASE_URLS = {
|
|
1372
|
+
openai: { baseUrl: "https://api.openai.com/v1", defaultModel: "gpt-4o-mini" },
|
|
1373
|
+
deepseek: { baseUrl: "https://api.deepseek.com", defaultModel: "deepseek-chat" }
|
|
1374
|
+
};
|
|
1375
|
+
function createLLMProvider(config) {
|
|
1376
|
+
if (config.provider === "anthropic") {
|
|
1377
|
+
if (!config.apiKey) {
|
|
1378
|
+
throw new ConfigError("Anthropic provider requires an apiKey.", "Add apiKey to llm config in cortexa.config.ts.");
|
|
1379
|
+
}
|
|
1380
|
+
return new AnthropicProvider(config.apiKey, config.model, config.retry);
|
|
1381
|
+
}
|
|
1382
|
+
const known = KNOWN_BASE_URLS[config.provider];
|
|
1383
|
+
const baseUrl = config.baseUrl ?? known?.baseUrl;
|
|
1384
|
+
if (!baseUrl) {
|
|
1385
|
+
throw new ConfigError(
|
|
1386
|
+
`Unknown provider "${config.provider}".`,
|
|
1387
|
+
`Provide a baseUrl for custom providers: { provider: "${config.provider}", baseUrl: "https://api.example.com/v1" }`
|
|
1388
|
+
);
|
|
1389
|
+
}
|
|
1390
|
+
const model = config.model ?? known?.defaultModel ?? config.provider;
|
|
1391
|
+
return new OpenAICompatibleProvider({
|
|
1392
|
+
apiKey: config.apiKey,
|
|
1393
|
+
model,
|
|
1394
|
+
baseUrl,
|
|
1395
|
+
headers: config.headers,
|
|
1396
|
+
retry: config.retry
|
|
1397
|
+
});
|
|
1398
|
+
}
|
|
1399
|
+
|
|
1400
|
+
// src/schema/introspector.ts
|
|
1401
|
+
var SchemaIntrospector = class {
|
|
1402
|
+
connector;
|
|
1403
|
+
dbType;
|
|
1404
|
+
options;
|
|
1405
|
+
constructor(connector, dbType, options = {}) {
|
|
1406
|
+
this.connector = connector;
|
|
1407
|
+
this.dbType = dbType;
|
|
1408
|
+
this.options = options;
|
|
1409
|
+
}
|
|
1410
|
+
async introspect() {
|
|
1411
|
+
const tableRows = await this.getTables();
|
|
1412
|
+
const tables = [];
|
|
1413
|
+
const excludeTables = new Set(this.options.excludeTables?.map((t) => t.toLowerCase()) ?? []);
|
|
1414
|
+
const excludeColumns = new Set(this.options.excludeColumns?.map((c) => c.toLowerCase()) ?? []);
|
|
1415
|
+
const includeSampleData = this.options.includeSampleData !== false;
|
|
1416
|
+
const sampleLimit = this.options.sampleRowLimit ?? 5;
|
|
1417
|
+
for (const row of tableRows) {
|
|
1418
|
+
const tableName = row.table_name;
|
|
1419
|
+
const tableSchema = row.table_schema;
|
|
1420
|
+
if (excludeTables.has(tableName.toLowerCase())) {
|
|
1421
|
+
continue;
|
|
1422
|
+
}
|
|
1423
|
+
const [allColumns, foreignKeys, indexes, sampleRows, rowCount] = await Promise.all([
|
|
1424
|
+
this.getColumns(tableName, tableSchema),
|
|
1425
|
+
this.getForeignKeys(tableName, tableSchema),
|
|
1426
|
+
this.getIndexes(tableName, tableSchema),
|
|
1427
|
+
includeSampleData ? this.getSampleRows(tableName, tableSchema, sampleLimit) : Promise.resolve([]),
|
|
1428
|
+
this.getRowCount(tableName, tableSchema)
|
|
1429
|
+
]);
|
|
1430
|
+
const columns = excludeColumns.size > 0 ? allColumns.filter((c) => !excludeColumns.has(c.columnName.toLowerCase())) : allColumns;
|
|
1431
|
+
const filteredSampleRows = excludeColumns.size > 0 && sampleRows.length > 0 ? sampleRows.map((row2) => {
|
|
1432
|
+
const filtered = {};
|
|
1433
|
+
for (const [key, value] of Object.entries(row2)) {
|
|
1434
|
+
if (!excludeColumns.has(key.toLowerCase())) {
|
|
1435
|
+
filtered[key] = value;
|
|
1436
|
+
}
|
|
1437
|
+
}
|
|
1438
|
+
return filtered;
|
|
1439
|
+
}) : sampleRows;
|
|
1440
|
+
tables.push({ tableName, tableSchema, columns, foreignKeys, indexes, sampleRows: filteredSampleRows, rowCount });
|
|
1441
|
+
}
|
|
1442
|
+
return { tables, discoveredAt: (/* @__PURE__ */ new Date()).toISOString() };
|
|
1443
|
+
}
|
|
1444
|
+
isPostgresLike() {
|
|
1445
|
+
return this.dbType === "postgres" || this.dbType === "cockroachdb";
|
|
1446
|
+
}
|
|
1447
|
+
isMysqlLike() {
|
|
1448
|
+
return this.dbType === "mysql" || this.dbType === "mariadb";
|
|
1449
|
+
}
|
|
1450
|
+
async getTables() {
|
|
1451
|
+
if (this.dbType === "sqlite") {
|
|
1452
|
+
const result2 = await this.connector.query(
|
|
1453
|
+
`SELECT name AS table_name, 'main' AS table_schema FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%' ORDER BY name`
|
|
1454
|
+
);
|
|
1455
|
+
return result2.rows;
|
|
1456
|
+
}
|
|
1457
|
+
if (this.dbType === "mongodb") {
|
|
1458
|
+
const result2 = await this.connector.query("COLLECTIONS");
|
|
1459
|
+
return result2.rows;
|
|
1460
|
+
}
|
|
1461
|
+
const schemaFilter = this.isPostgresLike() ? `table_schema NOT IN ('pg_catalog', 'information_schema')` : this.dbType === "mssql" ? `table_schema NOT IN ('sys', 'INFORMATION_SCHEMA')` : `table_schema = DATABASE()`;
|
|
1462
|
+
const result = await this.connector.query(
|
|
1463
|
+
`SELECT table_name AS table_name, table_schema AS table_schema FROM information_schema.tables WHERE table_type = 'BASE TABLE' AND ${schemaFilter} ORDER BY table_name`
|
|
1464
|
+
);
|
|
1465
|
+
return result.rows;
|
|
1466
|
+
}
|
|
1467
|
+
async getColumns(tableName, _tableSchema) {
|
|
1468
|
+
if (this.dbType === "sqlite") {
|
|
1469
|
+
const result2 = await this.connector.query(`PRAGMA table_info("${tableName}")`);
|
|
1470
|
+
return result2.rows.map((row) => ({
|
|
1471
|
+
columnName: row.name,
|
|
1472
|
+
dataType: row.type || "TEXT",
|
|
1473
|
+
isNullable: row.notnull === 0,
|
|
1474
|
+
columnDefault: row.dflt_value ?? null
|
|
1475
|
+
}));
|
|
1476
|
+
}
|
|
1477
|
+
if (this.dbType === "mongodb") {
|
|
1478
|
+
const result2 = await this.connector.query(`FIELDS:${tableName}:20`);
|
|
1479
|
+
return result2.rows.map((row) => ({
|
|
1480
|
+
columnName: row.column_name,
|
|
1481
|
+
dataType: row.data_type,
|
|
1482
|
+
isNullable: true,
|
|
1483
|
+
columnDefault: null
|
|
1484
|
+
}));
|
|
1485
|
+
}
|
|
1486
|
+
const result = await this.connector.query(
|
|
1487
|
+
`SELECT column_name AS column_name, data_type AS data_type, is_nullable AS is_nullable, column_default AS column_default FROM information_schema.columns WHERE table_name = $1 AND table_schema = $2 ORDER BY ordinal_position`,
|
|
1488
|
+
[tableName, _tableSchema]
|
|
1489
|
+
);
|
|
1490
|
+
return result.rows.map((row) => ({
|
|
1491
|
+
columnName: row.column_name,
|
|
1492
|
+
dataType: row.data_type,
|
|
1493
|
+
isNullable: row.is_nullable === "YES",
|
|
1494
|
+
columnDefault: row.column_default ?? null
|
|
1495
|
+
}));
|
|
1496
|
+
}
|
|
1497
|
+
async getForeignKeys(tableName, tableSchema) {
|
|
1498
|
+
if (this.dbType === "mongodb") {
|
|
1499
|
+
const columns = await this.getColumns(tableName, tableSchema);
|
|
1500
|
+
const fks = [];
|
|
1501
|
+
for (const col of columns) {
|
|
1502
|
+
if (col.columnName === "_id") continue;
|
|
1503
|
+
const match = col.columnName.match(/^(.+?)(?:_id|Id)$/);
|
|
1504
|
+
if (match) {
|
|
1505
|
+
fks.push({
|
|
1506
|
+
columnName: col.columnName,
|
|
1507
|
+
referencedTable: match[1] + "s",
|
|
1508
|
+
// naive pluralization
|
|
1509
|
+
referencedColumn: "_id"
|
|
1510
|
+
});
|
|
1511
|
+
}
|
|
1512
|
+
}
|
|
1513
|
+
return fks;
|
|
1514
|
+
}
|
|
1515
|
+
if (this.dbType === "sqlite") {
|
|
1516
|
+
const result2 = await this.connector.query(`PRAGMA foreign_key_list("${tableName}")`);
|
|
1517
|
+
return result2.rows.map((row) => ({
|
|
1518
|
+
columnName: row.from,
|
|
1519
|
+
referencedTable: row.table,
|
|
1520
|
+
referencedColumn: row.to
|
|
1521
|
+
}));
|
|
1522
|
+
}
|
|
1523
|
+
if (this.dbType === "mssql") {
|
|
1524
|
+
const sql2 = `SELECT COL_NAME(fkc.parent_object_id, fkc.parent_column_id) AS column_name,
|
|
1525
|
+
OBJECT_NAME(fkc.referenced_object_id) AS referenced_table,
|
|
1526
|
+
COL_NAME(fkc.referenced_object_id, fkc.referenced_column_id) AS referenced_column
|
|
1527
|
+
FROM sys.foreign_key_columns fkc
|
|
1528
|
+
JOIN sys.objects o ON o.object_id = fkc.parent_object_id
|
|
1529
|
+
JOIN sys.schemas s ON s.schema_id = o.schema_id
|
|
1530
|
+
WHERE o.name = $1 AND s.name = $2`;
|
|
1531
|
+
const result2 = await this.connector.query(sql2, [tableName, tableSchema]);
|
|
1532
|
+
return result2.rows.map((row) => ({
|
|
1533
|
+
columnName: row.column_name,
|
|
1534
|
+
referencedTable: row.referenced_table,
|
|
1535
|
+
referencedColumn: row.referenced_column
|
|
1536
|
+
}));
|
|
1537
|
+
}
|
|
1538
|
+
const sql = this.isPostgresLike() ? `SELECT a.attname AS column_name, cl2.relname AS referenced_table, a2.attname AS referenced_column
|
|
1539
|
+
FROM pg_constraint con
|
|
1540
|
+
JOIN pg_class cl ON con.conrelid = cl.oid
|
|
1541
|
+
JOIN pg_namespace ns ON cl.relnamespace = ns.oid
|
|
1542
|
+
JOIN pg_class cl2 ON con.confrelid = cl2.oid
|
|
1543
|
+
JOIN pg_attribute a ON a.attrelid = con.conrelid AND a.attnum = ANY(con.conkey)
|
|
1544
|
+
JOIN pg_attribute a2 ON a2.attrelid = con.confrelid AND a2.attnum = ANY(con.confkey)
|
|
1545
|
+
WHERE con.contype = 'f' AND cl.relname = $1 AND ns.nspname = $2` : `SELECT column_name AS column_name, referenced_table_name AS referenced_table, referenced_column_name AS referenced_column
|
|
1546
|
+
FROM information_schema.key_column_usage
|
|
1547
|
+
WHERE table_name = $1 AND table_schema = $2 AND referenced_table_name IS NOT NULL`;
|
|
1548
|
+
const result = await this.connector.query(sql, [tableName, tableSchema]);
|
|
1549
|
+
return result.rows.map((row) => ({
|
|
1550
|
+
columnName: row.column_name,
|
|
1551
|
+
referencedTable: row.referenced_table,
|
|
1552
|
+
referencedColumn: row.referenced_column
|
|
1553
|
+
}));
|
|
1554
|
+
}
|
|
1555
|
+
async getIndexes(tableName, tableSchema) {
|
|
1556
|
+
if (this.dbType === "mongodb") {
|
|
1557
|
+
const result2 = await this.connector.query(`INDEXES:${tableName}`);
|
|
1558
|
+
return result2.rows.map((row) => ({
|
|
1559
|
+
indexName: row.index_name,
|
|
1560
|
+
columns: row.columns,
|
|
1561
|
+
isUnique: Boolean(row.is_unique)
|
|
1562
|
+
}));
|
|
1563
|
+
}
|
|
1564
|
+
if (this.dbType === "sqlite") {
|
|
1565
|
+
const indexListResult = await this.connector.query(`PRAGMA index_list("${tableName}")`);
|
|
1566
|
+
const indexes = [];
|
|
1567
|
+
for (const idx of indexListResult.rows) {
|
|
1568
|
+
const idxName = idx.name;
|
|
1569
|
+
const infoResult = await this.connector.query(`PRAGMA index_info("${idxName}")`);
|
|
1570
|
+
indexes.push({
|
|
1571
|
+
indexName: idxName,
|
|
1572
|
+
columns: infoResult.rows.map((r) => r.name),
|
|
1573
|
+
isUnique: idx.unique === 1
|
|
1574
|
+
});
|
|
1575
|
+
}
|
|
1576
|
+
return indexes;
|
|
1577
|
+
}
|
|
1578
|
+
if (this.dbType === "mssql") {
|
|
1579
|
+
const sql2 = `SELECT i.name AS index_name, STRING_AGG(c.name, ',') WITHIN GROUP (ORDER BY ic.key_ordinal) AS columns, i.is_unique
|
|
1580
|
+
FROM sys.indexes i
|
|
1581
|
+
JOIN sys.index_columns ic ON ic.object_id = i.object_id AND ic.index_id = i.index_id
|
|
1582
|
+
JOIN sys.columns c ON c.object_id = ic.object_id AND c.column_id = ic.column_id
|
|
1583
|
+
JOIN sys.objects o ON o.object_id = i.object_id
|
|
1584
|
+
JOIN sys.schemas s ON s.schema_id = o.schema_id
|
|
1585
|
+
WHERE o.name = $1 AND s.name = $2 AND i.type > 0
|
|
1586
|
+
GROUP BY i.name, i.is_unique`;
|
|
1587
|
+
const result2 = await this.connector.query(sql2, [tableName, tableSchema]);
|
|
1588
|
+
return result2.rows.map((row) => ({
|
|
1589
|
+
indexName: row.index_name,
|
|
1590
|
+
columns: row.columns.split(","),
|
|
1591
|
+
isUnique: Boolean(row.is_unique)
|
|
1592
|
+
}));
|
|
1593
|
+
}
|
|
1594
|
+
const sql = this.isPostgresLike() ? `SELECT i.relname AS index_name, array_agg(a.attname ORDER BY k.n) AS columns, ix.indisunique AS is_unique
|
|
1595
|
+
FROM pg_index ix
|
|
1596
|
+
JOIN pg_class t ON t.oid = ix.indrelid
|
|
1597
|
+
JOIN pg_class i ON i.oid = ix.indexrelid
|
|
1598
|
+
JOIN pg_namespace n ON n.oid = t.relnamespace
|
|
1599
|
+
CROSS JOIN LATERAL unnest(ix.indkey) WITH ORDINALITY AS k(attnum, n)
|
|
1600
|
+
JOIN pg_attribute a ON a.attrelid = t.oid AND a.attnum = k.attnum
|
|
1601
|
+
WHERE t.relname = $1 AND n.nspname = $2
|
|
1602
|
+
GROUP BY i.relname, ix.indisunique` : `SELECT index_name AS index_name, GROUP_CONCAT(column_name ORDER BY seq_in_index) AS columns, NOT non_unique AS is_unique
|
|
1603
|
+
FROM information_schema.statistics
|
|
1604
|
+
WHERE table_name = $1 AND table_schema = $2
|
|
1605
|
+
GROUP BY index_name, non_unique`;
|
|
1606
|
+
const result = await this.connector.query(sql, [tableName, tableSchema]);
|
|
1607
|
+
return result.rows.map((row) => ({
|
|
1608
|
+
indexName: row.index_name,
|
|
1609
|
+
columns: Array.isArray(row.columns) ? row.columns : row.columns.split(","),
|
|
1610
|
+
isUnique: Boolean(row.is_unique)
|
|
1611
|
+
}));
|
|
1612
|
+
}
|
|
1613
|
+
async getSampleRows(tableName, tableSchema, limit = 5) {
|
|
1614
|
+
try {
|
|
1615
|
+
if (this.dbType === "mongodb") {
|
|
1616
|
+
return (await this.connector.query(`SAMPLE:${tableName}:${limit}`)).rows;
|
|
1617
|
+
}
|
|
1618
|
+
if (this.dbType === "mssql") {
|
|
1619
|
+
return (await this.connector.query(`SELECT TOP ${limit} * FROM [${tableSchema}].[${tableName}]`)).rows;
|
|
1620
|
+
}
|
|
1621
|
+
const qualifiedName = this.dbType === "sqlite" ? `"${tableName}"` : this.isPostgresLike() ? `"${tableSchema}"."${tableName}"` : `\`${tableSchema}\`.\`${tableName}\``;
|
|
1622
|
+
const result = await this.connector.query(`SELECT * FROM ${qualifiedName} LIMIT ${limit}`);
|
|
1623
|
+
return result.rows;
|
|
1624
|
+
} catch {
|
|
1625
|
+
return [];
|
|
1626
|
+
}
|
|
1627
|
+
}
|
|
1628
|
+
async getRowCount(tableName, tableSchema) {
|
|
1629
|
+
try {
|
|
1630
|
+
if (this.dbType === "mongodb") {
|
|
1631
|
+
const result2 = await this.connector.query(`COUNT:${tableName}`);
|
|
1632
|
+
return Number(result2.rows[0]?.count ?? 0);
|
|
1633
|
+
}
|
|
1634
|
+
if (this.dbType === "sqlite") {
|
|
1635
|
+
const result2 = await this.connector.query(`SELECT COUNT(*) AS count FROM "${tableName}"`);
|
|
1636
|
+
return Number(result2.rows[0]?.count ?? 0);
|
|
1637
|
+
}
|
|
1638
|
+
if (this.isPostgresLike()) {
|
|
1639
|
+
const result2 = await this.connector.query(
|
|
1640
|
+
`SELECT reltuples::bigint AS count FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace WHERE c.relname = $1 AND n.nspname = $2`,
|
|
1641
|
+
[tableName, tableSchema]
|
|
1642
|
+
);
|
|
1643
|
+
const count = Number(result2.rows[0]?.count ?? 0);
|
|
1644
|
+
return count >= 0 ? count : 0;
|
|
1645
|
+
}
|
|
1646
|
+
if (this.dbType === "mssql") {
|
|
1647
|
+
const result2 = await this.connector.query(
|
|
1648
|
+
`SELECT SUM(rows) AS count FROM sys.partitions WHERE object_id = OBJECT_ID($1) AND index_id IN (0, 1)`,
|
|
1649
|
+
[`${tableSchema}.${tableName}`]
|
|
1650
|
+
);
|
|
1651
|
+
return Number(result2.rows[0]?.count ?? 0);
|
|
1652
|
+
}
|
|
1653
|
+
const result = await this.connector.query(
|
|
1654
|
+
`SELECT table_rows AS count FROM information_schema.tables WHERE table_name = $1 AND table_schema = $2`,
|
|
1655
|
+
[tableName, tableSchema]
|
|
1656
|
+
);
|
|
1657
|
+
return Number(result.rows[0]?.count ?? 0);
|
|
1658
|
+
} catch {
|
|
1659
|
+
return 0;
|
|
1660
|
+
}
|
|
1661
|
+
}
|
|
1662
|
+
};
|
|
1663
|
+
|
|
1664
|
+
// src/schema/classifier.ts
|
|
1665
|
+
var DEFAULT_BATCH_SIZE = 5;
|
|
1666
|
+
var SchemaClassifier = class {
|
|
1667
|
+
llm;
|
|
1668
|
+
batchSize;
|
|
1669
|
+
constructor(llm, batchSize) {
|
|
1670
|
+
this.llm = llm;
|
|
1671
|
+
this.batchSize = batchSize ?? DEFAULT_BATCH_SIZE;
|
|
1672
|
+
}
|
|
1673
|
+
async classify(tables) {
|
|
1674
|
+
const batches = this.chunk(tables, this.batchSize);
|
|
1675
|
+
const results = [];
|
|
1676
|
+
for (const batch of batches) {
|
|
1677
|
+
const classified = await this.classifyBatch(batch);
|
|
1678
|
+
results.push(...classified);
|
|
1679
|
+
}
|
|
1680
|
+
return results;
|
|
1681
|
+
}
|
|
1682
|
+
async classifyBatch(tables) {
|
|
1683
|
+
const prompt = this.buildPrompt(tables);
|
|
1684
|
+
const chatOptions = { jsonMode: true, maxTokens: 8192 };
|
|
1685
|
+
let response = await this.llm.chat(prompt, chatOptions);
|
|
1686
|
+
try {
|
|
1687
|
+
return this.parseResponse(response.content);
|
|
1688
|
+
} catch {
|
|
1689
|
+
response = await this.llm.chat(prompt, chatOptions);
|
|
1690
|
+
return this.parseResponse(response.content);
|
|
1691
|
+
}
|
|
1692
|
+
}
|
|
1693
|
+
buildPrompt(tables) {
|
|
1694
|
+
const tablesContext = tables.map((t) => {
|
|
1695
|
+
const columnsStr = t.columns.map((c) => ` - ${c.columnName} (${c.dataType}, ${c.isNullable ? "nullable" : "not null"}${c.columnDefault ? `, default: ${c.columnDefault}` : ""})`).join("\n");
|
|
1696
|
+
const fkStr = t.foreignKeys.length > 0 ? `
|
|
1697
|
+
Foreign Keys:
|
|
1698
|
+
${t.foreignKeys.map((fk) => ` - ${fk.columnName} -> ${fk.referencedTable}.${fk.referencedColumn}`).join("\n")}` : "";
|
|
1699
|
+
const sampleStr = t.sampleRows.length > 0 ? `
|
|
1700
|
+
Sample Data (${t.sampleRows.length} rows): ${JSON.stringify(t.sampleRows[0])}` : "";
|
|
1701
|
+
return `Table: ${t.tableName} (schema: ${t.tableSchema}, ~${t.rowCount} rows)
|
|
1702
|
+
Columns:
|
|
1703
|
+
${columnsStr}${fkStr}${sampleStr}`;
|
|
1704
|
+
}).join("\n\n");
|
|
1705
|
+
return `You are a database schema analyst. Analyze the following database tables and classify each one.
|
|
1706
|
+
|
|
1707
|
+
For each table, return a JSON object with:
|
|
1708
|
+
- tableName: exact table name (must match input)
|
|
1709
|
+
- entityLabel: human-readable business name (e.g., "User Account", "Sales Order")
|
|
1710
|
+
- entityType: one of "master" (core business entities like Users, Products), "transaction" (events/activities like Orders, Payments), "reference" (lookup/config tables like Countries, Statuses), or "junction" (many-to-many link tables)
|
|
1711
|
+
- description: one sentence describing the table's business purpose
|
|
1712
|
+
- confidence: 0.0 to 1.0 indicating classification confidence
|
|
1713
|
+
- columns: array of { columnName, label, meaning } for each column
|
|
1714
|
+
|
|
1715
|
+
Return valid JSON in this exact format:
|
|
1716
|
+
{
|
|
1717
|
+
"tables": [
|
|
1718
|
+
{
|
|
1719
|
+
"tableName": "...",
|
|
1720
|
+
"entityLabel": "...",
|
|
1721
|
+
"entityType": "master|transaction|reference|junction",
|
|
1722
|
+
"description": "...",
|
|
1723
|
+
"confidence": 0.95,
|
|
1724
|
+
"columns": [
|
|
1725
|
+
{ "columnName": "...", "label": "...", "meaning": "..." }
|
|
1726
|
+
]
|
|
1727
|
+
}
|
|
1728
|
+
]
|
|
1729
|
+
}
|
|
1730
|
+
|
|
1731
|
+
DATABASE TABLES:
|
|
1732
|
+
|
|
1733
|
+
${tablesContext}`;
|
|
1734
|
+
}
|
|
1735
|
+
parseResponse(content) {
|
|
1736
|
+
let data;
|
|
1737
|
+
try {
|
|
1738
|
+
data = JSON.parse(content);
|
|
1739
|
+
} catch (err) {
|
|
1740
|
+
throw new LLMResponseError(
|
|
1741
|
+
`Failed to parse LLM response: ${err.message}`,
|
|
1742
|
+
content.slice(0, 500)
|
|
1743
|
+
);
|
|
1744
|
+
}
|
|
1745
|
+
if (!data.tables || !Array.isArray(data.tables)) {
|
|
1746
|
+
throw new LLMResponseError('Invalid LLM response: missing "tables" array', content.slice(0, 500));
|
|
1747
|
+
}
|
|
1748
|
+
return data.tables.map((t) => ({
|
|
1749
|
+
tableName: t.tableName,
|
|
1750
|
+
entityLabel: t.entityLabel,
|
|
1751
|
+
entityType: t.entityType,
|
|
1752
|
+
description: t.description,
|
|
1753
|
+
confidence: t.confidence,
|
|
1754
|
+
columns: (t.columns ?? []).map((c) => ({
|
|
1755
|
+
columnName: c.columnName,
|
|
1756
|
+
label: c.label,
|
|
1757
|
+
meaning: c.meaning
|
|
1758
|
+
}))
|
|
1759
|
+
}));
|
|
1760
|
+
}
|
|
1761
|
+
chunk(array, size) {
|
|
1762
|
+
const chunks = [];
|
|
1763
|
+
for (let i = 0; i < array.length; i += size) {
|
|
1764
|
+
chunks.push(array.slice(i, i + size));
|
|
1765
|
+
}
|
|
1766
|
+
return chunks;
|
|
1767
|
+
}
|
|
1768
|
+
};
|
|
1769
|
+
|
|
1770
|
+
// src/schema/graph-builder.ts
|
|
1771
|
+
var GraphBuilder = class {
|
|
1772
|
+
build(rawTables, entities) {
|
|
1773
|
+
const entityMap = new Map(entities.map((e) => [e.tableName, e]));
|
|
1774
|
+
const relationships = [];
|
|
1775
|
+
for (const table of rawTables) {
|
|
1776
|
+
for (const fk of table.foreignKeys) {
|
|
1777
|
+
const sourceEntity = entityMap.get(table.tableName);
|
|
1778
|
+
const targetEntity = entityMap.get(fk.referencedTable);
|
|
1779
|
+
const relationshipLabel = this.generateLabel(
|
|
1780
|
+
sourceEntity?.entityLabel ?? table.tableName,
|
|
1781
|
+
targetEntity?.entityLabel ?? fk.referencedTable,
|
|
1782
|
+
fk.columnName
|
|
1783
|
+
);
|
|
1784
|
+
relationships.push({
|
|
1785
|
+
sourceEntity: table.tableName,
|
|
1786
|
+
targetEntity: fk.referencedTable,
|
|
1787
|
+
relationship: relationshipLabel,
|
|
1788
|
+
sourceColumn: fk.columnName,
|
|
1789
|
+
targetColumn: fk.referencedColumn,
|
|
1790
|
+
inferred: false,
|
|
1791
|
+
confidence: 1
|
|
1792
|
+
});
|
|
1793
|
+
}
|
|
1794
|
+
}
|
|
1795
|
+
return { entities, relationships };
|
|
1796
|
+
}
|
|
1797
|
+
generateLabel(sourceLabel, targetLabel, columnName) {
|
|
1798
|
+
const cleaned = columnName.replace(/_id$/, "").replace(/_/g, " ").toUpperCase();
|
|
1799
|
+
if (cleaned.length > 0) {
|
|
1800
|
+
return `HAS_${cleaned}`;
|
|
1801
|
+
}
|
|
1802
|
+
return `REFERENCES_${targetLabel.replace(/\s+/g, "_").toUpperCase()}`;
|
|
1803
|
+
}
|
|
1804
|
+
};
|
|
1805
|
+
|
|
1806
|
+
// src/behavioral/change-detector.ts
|
|
1807
|
+
var TIMESTAMP_COLUMNS = ["updated_at", "modified_at", "last_modified", "updatedat", "modifiedat"];
|
|
1808
|
+
var AUTO_INCREMENT_PATTERNS = ["nextval", "auto_increment", "serial", "identity"];
|
|
1809
|
+
var ChangeDetector = class {
|
|
1810
|
+
connector;
|
|
1811
|
+
dbType;
|
|
1812
|
+
constructor(connector, dbType) {
|
|
1813
|
+
this.connector = connector;
|
|
1814
|
+
this.dbType = dbType;
|
|
1815
|
+
}
|
|
1816
|
+
static buildWatchConfigs(tables, excludeTables) {
|
|
1817
|
+
const excludeSet = new Set(excludeTables?.map((t) => t.toLowerCase()) ?? []);
|
|
1818
|
+
const configs = [];
|
|
1819
|
+
for (const table of tables) {
|
|
1820
|
+
if (excludeSet.has(table.tableName.toLowerCase())) continue;
|
|
1821
|
+
const timestampCol = table.columns.find(
|
|
1822
|
+
(c) => TIMESTAMP_COLUMNS.includes(c.columnName.toLowerCase()) && c.dataType.toLowerCase().includes("timestamp")
|
|
1823
|
+
);
|
|
1824
|
+
if (timestampCol) {
|
|
1825
|
+
const pkCol = table.columns.find(
|
|
1826
|
+
(c) => c.columnName.toLowerCase() === "id" && !c.isNullable
|
|
1827
|
+
);
|
|
1828
|
+
configs.push({
|
|
1829
|
+
tableName: table.tableName,
|
|
1830
|
+
tableSchema: table.tableSchema,
|
|
1831
|
+
strategy: "timestamp",
|
|
1832
|
+
timestampColumn: timestampCol.columnName,
|
|
1833
|
+
primaryKeyColumn: pkCol?.columnName ?? table.columns[0].columnName
|
|
1834
|
+
});
|
|
1835
|
+
continue;
|
|
1836
|
+
}
|
|
1837
|
+
const autoIncrementCol = table.columns.find(
|
|
1838
|
+
(c) => c.columnDefault !== null && AUTO_INCREMENT_PATTERNS.some((p) => (c.columnDefault ?? "").toLowerCase().includes(p))
|
|
1839
|
+
);
|
|
1840
|
+
if (autoIncrementCol) {
|
|
1841
|
+
configs.push({
|
|
1842
|
+
tableName: table.tableName,
|
|
1843
|
+
tableSchema: table.tableSchema,
|
|
1844
|
+
strategy: "id_watermark",
|
|
1845
|
+
primaryKeyColumn: autoIncrementCol.columnName
|
|
1846
|
+
});
|
|
1847
|
+
continue;
|
|
1848
|
+
}
|
|
1849
|
+
configs.push({
|
|
1850
|
+
tableName: table.tableName,
|
|
1851
|
+
tableSchema: table.tableSchema,
|
|
1852
|
+
strategy: "table_stats"
|
|
1853
|
+
});
|
|
1854
|
+
}
|
|
1855
|
+
return configs;
|
|
1856
|
+
}
|
|
1857
|
+
async poll(config) {
|
|
1858
|
+
if (config.strategy === "timestamp") {
|
|
1859
|
+
return this.pollTimestamp(config);
|
|
1860
|
+
}
|
|
1861
|
+
if (config.strategy === "id_watermark") {
|
|
1862
|
+
return this.pollIdWatermark(config);
|
|
1863
|
+
}
|
|
1864
|
+
if (config.strategy === "table_stats") {
|
|
1865
|
+
return this.pollTableStats(config);
|
|
1866
|
+
}
|
|
1867
|
+
return [];
|
|
1868
|
+
}
|
|
1869
|
+
async pollTimestamp(config) {
|
|
1870
|
+
const qualified = this.qualifyTable(config.tableName, config.tableSchema);
|
|
1871
|
+
const tsCol = config.timestampColumn;
|
|
1872
|
+
const pkCol = config.primaryKeyColumn ?? "id";
|
|
1873
|
+
const lastSeen = config.lastSeenValue ?? "1970-01-01T00:00:00Z";
|
|
1874
|
+
const result = await this.connector.query(
|
|
1875
|
+
`SELECT * FROM ${qualified} WHERE ${this.quoteIdent(tsCol)} > $1 ORDER BY ${this.quoteIdent(tsCol)} ASC`,
|
|
1876
|
+
[lastSeen]
|
|
1877
|
+
);
|
|
1878
|
+
return result.rows.map((row) => ({
|
|
1879
|
+
tableName: config.tableName,
|
|
1880
|
+
tableSchema: config.tableSchema,
|
|
1881
|
+
operation: "UPDATE",
|
|
1882
|
+
primaryKey: row[pkCol],
|
|
1883
|
+
newData: { ...row },
|
|
1884
|
+
detectedAt: /* @__PURE__ */ new Date()
|
|
1885
|
+
}));
|
|
1886
|
+
}
|
|
1887
|
+
async pollIdWatermark(config) {
|
|
1888
|
+
const qualified = this.qualifyTable(config.tableName, config.tableSchema);
|
|
1889
|
+
const pkCol = config.primaryKeyColumn;
|
|
1890
|
+
const lastSeen = config.lastSeenValue ?? "0";
|
|
1891
|
+
const result = await this.connector.query(
|
|
1892
|
+
`SELECT * FROM ${qualified} WHERE ${this.quoteIdent(pkCol)} > $1 ORDER BY ${this.quoteIdent(pkCol)} ASC`,
|
|
1893
|
+
[lastSeen]
|
|
1894
|
+
);
|
|
1895
|
+
return result.rows.map((row) => ({
|
|
1896
|
+
tableName: config.tableName,
|
|
1897
|
+
tableSchema: config.tableSchema,
|
|
1898
|
+
operation: "INSERT",
|
|
1899
|
+
primaryKey: row[pkCol],
|
|
1900
|
+
newData: { ...row },
|
|
1901
|
+
detectedAt: /* @__PURE__ */ new Date()
|
|
1902
|
+
}));
|
|
1903
|
+
}
|
|
1904
|
+
async pollTableStats(config) {
|
|
1905
|
+
const currentStats = await this.queryTableStats(config.tableName, config.tableSchema);
|
|
1906
|
+
if (!config.lastStats) {
|
|
1907
|
+
config.lastStats = currentStats;
|
|
1908
|
+
return [];
|
|
1909
|
+
}
|
|
1910
|
+
const prev = config.lastStats;
|
|
1911
|
+
const changes = [];
|
|
1912
|
+
const newInserts = currentStats.inserts - prev.inserts;
|
|
1913
|
+
const newUpdates = currentStats.updates - prev.updates;
|
|
1914
|
+
const newDeletes = currentStats.deletes - prev.deletes;
|
|
1915
|
+
config.lastStats = currentStats;
|
|
1916
|
+
if (newInserts === 0 && newUpdates === 0 && newDeletes === 0) return [];
|
|
1917
|
+
if (newInserts > 0) {
|
|
1918
|
+
changes.push({
|
|
1919
|
+
tableName: config.tableName,
|
|
1920
|
+
tableSchema: config.tableSchema,
|
|
1921
|
+
operation: "INSERT",
|
|
1922
|
+
primaryKey: "unknown",
|
|
1923
|
+
detectedAt: /* @__PURE__ */ new Date()
|
|
1924
|
+
});
|
|
1925
|
+
}
|
|
1926
|
+
if (newUpdates > 0) {
|
|
1927
|
+
changes.push({
|
|
1928
|
+
tableName: config.tableName,
|
|
1929
|
+
tableSchema: config.tableSchema,
|
|
1930
|
+
operation: "UPDATE",
|
|
1931
|
+
primaryKey: "unknown",
|
|
1932
|
+
detectedAt: /* @__PURE__ */ new Date()
|
|
1933
|
+
});
|
|
1934
|
+
}
|
|
1935
|
+
if (newDeletes > 0) {
|
|
1936
|
+
changes.push({
|
|
1937
|
+
tableName: config.tableName,
|
|
1938
|
+
tableSchema: config.tableSchema,
|
|
1939
|
+
operation: "DELETE",
|
|
1940
|
+
primaryKey: "unknown",
|
|
1941
|
+
detectedAt: /* @__PURE__ */ new Date()
|
|
1942
|
+
});
|
|
1943
|
+
}
|
|
1944
|
+
return changes;
|
|
1945
|
+
}
|
|
1946
|
+
async queryTableStats(tableName, tableSchema) {
|
|
1947
|
+
if (this.dbType === "postgres" || this.dbType === "cockroachdb") {
|
|
1948
|
+
const result2 = await this.connector.query(
|
|
1949
|
+
`SELECT COALESCE(n_tup_ins, 0) AS inserts, COALESCE(n_tup_upd, 0) AS updates, COALESCE(n_tup_del, 0) AS deletes
|
|
1950
|
+
FROM pg_stat_user_tables WHERE relname = $1 AND schemaname = $2`,
|
|
1951
|
+
[tableName, tableSchema]
|
|
1952
|
+
);
|
|
1953
|
+
if (result2.rows.length === 0) return { inserts: 0, updates: 0, deletes: 0 };
|
|
1954
|
+
const row2 = result2.rows[0];
|
|
1955
|
+
return {
|
|
1956
|
+
inserts: parseInt(String(row2.inserts), 10),
|
|
1957
|
+
updates: parseInt(String(row2.updates), 10),
|
|
1958
|
+
deletes: parseInt(String(row2.deletes), 10)
|
|
1959
|
+
};
|
|
1960
|
+
}
|
|
1961
|
+
if (this.dbType === "sqlite") {
|
|
1962
|
+
const result2 = await this.connector.query(
|
|
1963
|
+
`SELECT COUNT(*) AS row_count FROM "${tableName}"`
|
|
1964
|
+
);
|
|
1965
|
+
const rowCount2 = parseInt(String(result2.rows[0]?.row_count ?? 0), 10);
|
|
1966
|
+
return { inserts: rowCount2, updates: 0, deletes: 0 };
|
|
1967
|
+
}
|
|
1968
|
+
if (this.dbType === "mongodb") {
|
|
1969
|
+
const result2 = await this.connector.query(`COUNT:${tableName}`);
|
|
1970
|
+
const docCount = parseInt(String(result2.rows[0]?.count ?? 0), 10);
|
|
1971
|
+
return { inserts: docCount, updates: 0, deletes: 0 };
|
|
1972
|
+
}
|
|
1973
|
+
if (this.dbType === "mssql") {
|
|
1974
|
+
const result2 = await this.connector.query(
|
|
1975
|
+
`SELECT SUM(rows) AS row_count FROM sys.partitions WHERE object_id = OBJECT_ID($1) AND index_id IN (0, 1)`,
|
|
1976
|
+
[`${tableSchema}.${tableName}`]
|
|
1977
|
+
);
|
|
1978
|
+
const rowCount2 = parseInt(String(result2.rows[0]?.row_count ?? 0), 10);
|
|
1979
|
+
return { inserts: rowCount2, updates: 0, deletes: 0 };
|
|
1980
|
+
}
|
|
1981
|
+
const result = await this.connector.query(
|
|
1982
|
+
`SELECT TABLE_ROWS, UPDATE_TIME FROM information_schema.TABLES WHERE TABLE_NAME = $1 AND TABLE_SCHEMA = $2`,
|
|
1983
|
+
[tableName, tableSchema]
|
|
1984
|
+
);
|
|
1985
|
+
if (result.rows.length === 0) return { inserts: 0, updates: 0, deletes: 0 };
|
|
1986
|
+
const row = result.rows[0];
|
|
1987
|
+
const rowCount = parseInt(String(row.TABLE_ROWS ?? 0), 10);
|
|
1988
|
+
const updateTime = row.UPDATE_TIME ? new Date(String(row.UPDATE_TIME)).getTime() : 0;
|
|
1989
|
+
return { inserts: rowCount, updates: updateTime, deletes: 0 };
|
|
1990
|
+
}
|
|
1991
|
+
qualifyTable(tableName, tableSchema) {
|
|
1992
|
+
if (this.dbType === "postgres" || this.dbType === "cockroachdb") {
|
|
1993
|
+
return `"${tableSchema}"."${tableName}"`;
|
|
1994
|
+
}
|
|
1995
|
+
if (this.dbType === "sqlite") {
|
|
1996
|
+
return `"${tableName}"`;
|
|
1997
|
+
}
|
|
1998
|
+
if (this.dbType === "mssql") {
|
|
1999
|
+
return `[${tableSchema}].[${tableName}]`;
|
|
2000
|
+
}
|
|
2001
|
+
if (this.dbType === "mongodb") {
|
|
2002
|
+
return tableName;
|
|
2003
|
+
}
|
|
2004
|
+
return `\`${tableSchema}\`.\`${tableName}\``;
|
|
2005
|
+
}
|
|
2006
|
+
quoteIdent(name) {
|
|
2007
|
+
if (this.dbType === "postgres" || this.dbType === "cockroachdb" || this.dbType === "sqlite") {
|
|
2008
|
+
return `"${name}"`;
|
|
2009
|
+
}
|
|
2010
|
+
if (this.dbType === "mssql") {
|
|
2011
|
+
return `[${name}]`;
|
|
2012
|
+
}
|
|
2013
|
+
return `\`${name}\``;
|
|
2014
|
+
}
|
|
2015
|
+
};
|
|
2016
|
+
|
|
2017
|
+
// src/behavioral/event-classifier.ts
|
|
2018
|
+
var OPERATION_MAP = {
|
|
2019
|
+
INSERT: "created",
|
|
2020
|
+
UPDATE: "updated",
|
|
2021
|
+
DELETE: "deleted"
|
|
2022
|
+
};
|
|
2023
|
+
var EventClassifier = class {
|
|
2024
|
+
entityLookup;
|
|
2025
|
+
constructor(entityLookup) {
|
|
2026
|
+
this.entityLookup = entityLookup;
|
|
2027
|
+
}
|
|
2028
|
+
classify(change, options) {
|
|
2029
|
+
const entity = this.entityLookup.getEntity(change.tableName);
|
|
2030
|
+
const excludeSet = new Set(options?.excludeColumns?.map((c) => c.toLowerCase()) ?? []);
|
|
2031
|
+
const metadata = this.filterMetadata(change.newData ?? change.oldData ?? {}, excludeSet);
|
|
2032
|
+
return {
|
|
2033
|
+
entity: entity?.entityLabel ?? change.tableName,
|
|
2034
|
+
entityType: entity?.entityType ?? "reference",
|
|
2035
|
+
type: OPERATION_MAP[change.operation],
|
|
2036
|
+
table: change.tableName,
|
|
2037
|
+
primaryKey: change.primaryKey,
|
|
2038
|
+
timestamp: change.detectedAt,
|
|
2039
|
+
changedColumns: change.changedColumns,
|
|
2040
|
+
metadata
|
|
2041
|
+
};
|
|
2042
|
+
}
|
|
2043
|
+
filterMetadata(data, excludeSet) {
|
|
2044
|
+
if (excludeSet.size === 0) return { ...data };
|
|
2045
|
+
const filtered = {};
|
|
2046
|
+
for (const [key, value] of Object.entries(data)) {
|
|
2047
|
+
if (!excludeSet.has(key.toLowerCase())) {
|
|
2048
|
+
filtered[key] = value;
|
|
2049
|
+
}
|
|
2050
|
+
}
|
|
2051
|
+
return filtered;
|
|
2052
|
+
}
|
|
2053
|
+
};
|
|
2054
|
+
|
|
2055
|
+
// src/behavioral/pattern-engine.ts
|
|
2056
|
+
var MIN_SAMPLE_SIZE = 12;
|
|
2057
|
+
var DEFAULT_ANOMALY_THRESHOLD = 2;
|
|
2058
|
+
var PatternEngine = class _PatternEngine {
|
|
2059
|
+
storage;
|
|
2060
|
+
anomalyThreshold;
|
|
2061
|
+
constructor(storage, options) {
|
|
2062
|
+
this.storage = storage;
|
|
2063
|
+
this.anomalyThreshold = options?.anomalyThreshold ?? DEFAULT_ANOMALY_THRESHOLD;
|
|
2064
|
+
}
|
|
2065
|
+
updateBaselines(entities, operation) {
|
|
2066
|
+
const baselines = this.storage.getBaselines();
|
|
2067
|
+
const baselineMap = new Map(baselines.map((b) => [`${b.entity}:${b.metric}`, b]));
|
|
2068
|
+
for (const entity of entities) {
|
|
2069
|
+
const metric = `${operation}_per_hour`;
|
|
2070
|
+
const currentCount = this.storage.countEvents(entity, operation, 60);
|
|
2071
|
+
const key = `${entity}:${metric}`;
|
|
2072
|
+
const existing = baselineMap.get(key);
|
|
2073
|
+
if (!existing) {
|
|
2074
|
+
this.storage.saveBaseline({
|
|
2075
|
+
entity,
|
|
2076
|
+
metric,
|
|
2077
|
+
mean: currentCount,
|
|
2078
|
+
stddev: 0,
|
|
2079
|
+
sampleSize: 1
|
|
2080
|
+
});
|
|
2081
|
+
} else {
|
|
2082
|
+
const n = existing.sampleSize;
|
|
2083
|
+
const newN = n + 1;
|
|
2084
|
+
const newMean = existing.mean + (currentCount - existing.mean) / newN;
|
|
2085
|
+
const newStddev = Math.sqrt(
|
|
2086
|
+
((n - 1) * existing.stddev * existing.stddev + (currentCount - existing.mean) * (currentCount - newMean)) / Math.max(n, 1)
|
|
2087
|
+
);
|
|
2088
|
+
this.storage.saveBaseline({
|
|
2089
|
+
entity,
|
|
2090
|
+
metric,
|
|
2091
|
+
mean: newMean,
|
|
2092
|
+
stddev: newStddev,
|
|
2093
|
+
sampleSize: newN
|
|
2094
|
+
});
|
|
2095
|
+
}
|
|
2096
|
+
}
|
|
2097
|
+
}
|
|
2098
|
+
checkRateAnomalies(entities, operation) {
|
|
2099
|
+
const baselines = this.storage.getBaselines();
|
|
2100
|
+
const baselineMap = new Map(baselines.map((b) => [`${b.entity}:${b.metric}`, b]));
|
|
2101
|
+
const anomalies = [];
|
|
2102
|
+
for (const entity of entities) {
|
|
2103
|
+
const metric = `${operation}_per_hour`;
|
|
2104
|
+
const baseline = baselineMap.get(`${entity}:${metric}`);
|
|
2105
|
+
if (!baseline || baseline.sampleSize < MIN_SAMPLE_SIZE || baseline.stddev === 0) continue;
|
|
2106
|
+
const currentCount = this.storage.countEvents(entity, operation, 60);
|
|
2107
|
+
const deviation = Math.abs(currentCount - baseline.mean) / baseline.stddev;
|
|
2108
|
+
if (deviation >= this.anomalyThreshold) {
|
|
2109
|
+
const anomalyType = currentCount > baseline.mean ? "rate_spike" : "rate_drop";
|
|
2110
|
+
const severity = _PatternEngine.getSeverity(deviation);
|
|
2111
|
+
const anomaly = {
|
|
2112
|
+
entity,
|
|
2113
|
+
anomalyType,
|
|
2114
|
+
severity,
|
|
2115
|
+
expected: baseline.mean,
|
|
2116
|
+
actual: currentCount,
|
|
2117
|
+
message: `${entity} ${operation} rate ${anomalyType === "rate_spike" ? "spiked" : "dropped"} to ${currentCount}/hr (expected ${baseline.mean.toFixed(0)}\xB1${baseline.stddev.toFixed(0)})`,
|
|
2118
|
+
timestamp: /* @__PURE__ */ new Date()
|
|
2119
|
+
};
|
|
2120
|
+
this.storage.saveAnomaly({
|
|
2121
|
+
entity: anomaly.entity,
|
|
2122
|
+
anomalyType: anomaly.anomalyType,
|
|
2123
|
+
severity: anomaly.severity,
|
|
2124
|
+
expected: anomaly.expected,
|
|
2125
|
+
actual: anomaly.actual,
|
|
2126
|
+
message: anomaly.message
|
|
2127
|
+
});
|
|
2128
|
+
anomalies.push(anomaly);
|
|
2129
|
+
}
|
|
2130
|
+
}
|
|
2131
|
+
return anomalies;
|
|
2132
|
+
}
|
|
2133
|
+
static getSeverity(sigma) {
|
|
2134
|
+
if (sigma >= 5) return "critical";
|
|
2135
|
+
if (sigma >= 4) return "high";
|
|
2136
|
+
if (sigma >= 3) return "medium";
|
|
2137
|
+
return "low";
|
|
2138
|
+
}
|
|
2139
|
+
};
|
|
2140
|
+
|
|
2141
|
+
// src/behavioral/capability-detector.ts
|
|
2142
|
+
var CapabilityDetector = class {
|
|
2143
|
+
connector;
|
|
2144
|
+
dbType;
|
|
2145
|
+
constructor(connector, dbType) {
|
|
2146
|
+
this.connector = connector;
|
|
2147
|
+
this.dbType = dbType;
|
|
2148
|
+
}
|
|
2149
|
+
async detect() {
|
|
2150
|
+
switch (this.dbType) {
|
|
2151
|
+
case "postgres":
|
|
2152
|
+
return this.detectPostgres();
|
|
2153
|
+
case "cockroachdb":
|
|
2154
|
+
return { canStream: false, reason: "CockroachDB changefeed not yet supported. Polling mode will be used." };
|
|
2155
|
+
case "mysql":
|
|
2156
|
+
case "mariadb":
|
|
2157
|
+
return this.detectMysql();
|
|
2158
|
+
case "sqlite":
|
|
2159
|
+
return { canStream: false, reason: "SQLite does not support CDC streaming. Polling mode will be used." };
|
|
2160
|
+
case "mongodb":
|
|
2161
|
+
return this.detectMongodb();
|
|
2162
|
+
case "mssql":
|
|
2163
|
+
return { canStream: false, reason: "SQL Server CDC not yet supported. Polling mode will be used." };
|
|
2164
|
+
default:
|
|
2165
|
+
return { canStream: false, reason: `Unsupported database type: ${this.dbType}` };
|
|
2166
|
+
}
|
|
2167
|
+
}
|
|
2168
|
+
async detectMongodb() {
|
|
2169
|
+
try {
|
|
2170
|
+
const result = await this.connector.query('COMMAND:{"replSetGetStatus":1}');
|
|
2171
|
+
if (result.rows.length > 0) {
|
|
2172
|
+
return { canStream: true, reason: "MongoDB Change Streams supported (replica set detected)" };
|
|
2173
|
+
}
|
|
2174
|
+
return { canStream: false, reason: "MongoDB Change Streams require a replica set. Polling mode will be used." };
|
|
2175
|
+
} catch {
|
|
2176
|
+
return { canStream: false, reason: "MongoDB Change Streams require a replica set. Polling mode will be used." };
|
|
2177
|
+
}
|
|
2178
|
+
}
|
|
2179
|
+
async detectPostgres() {
|
|
2180
|
+
try {
|
|
2181
|
+
const walResult = await this.connector.query(
|
|
2182
|
+
"SELECT current_setting('wal_level') AS wal_level"
|
|
2183
|
+
);
|
|
2184
|
+
const walLevel = String(walResult.rows[0]?.wal_level ?? "").toLowerCase();
|
|
2185
|
+
if (walLevel !== "logical") {
|
|
2186
|
+
return {
|
|
2187
|
+
canStream: false,
|
|
2188
|
+
reason: "wal_level is not set to 'logical'. Set wal_level = logical in postgresql.conf and restart."
|
|
2189
|
+
};
|
|
2190
|
+
}
|
|
2191
|
+
const roleResult = await this.connector.query(
|
|
2192
|
+
"SELECT rolreplication FROM pg_roles WHERE rolname = current_user"
|
|
2193
|
+
);
|
|
2194
|
+
const rolreplication = roleResult.rows[0]?.rolreplication;
|
|
2195
|
+
if (rolreplication !== true) {
|
|
2196
|
+
return {
|
|
2197
|
+
canStream: false,
|
|
2198
|
+
reason: "Current user lacks REPLICATION privilege required for WAL streaming"
|
|
2199
|
+
};
|
|
2200
|
+
}
|
|
2201
|
+
return { canStream: true, reason: "WAL streaming supported" };
|
|
2202
|
+
} catch (err) {
|
|
2203
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
2204
|
+
return { canStream: false, reason: `Capability detection failed: ${message}` };
|
|
2205
|
+
}
|
|
2206
|
+
}
|
|
2207
|
+
async detectMysql() {
|
|
2208
|
+
try {
|
|
2209
|
+
const logBinResult = await this.connector.query("SHOW VARIABLES LIKE 'log_bin'");
|
|
2210
|
+
const logBinValue = String(logBinResult.rows[0]?.Value ?? "").toUpperCase();
|
|
2211
|
+
if (logBinValue !== "ON") {
|
|
2212
|
+
return {
|
|
2213
|
+
canStream: false,
|
|
2214
|
+
reason: "log_bin is not ON; binlog streaming requires log_bin = ON"
|
|
2215
|
+
};
|
|
2216
|
+
}
|
|
2217
|
+
const binlogFormatResult = await this.connector.query("SHOW VARIABLES LIKE 'binlog_format'");
|
|
2218
|
+
const binlogFormatValue = String(binlogFormatResult.rows[0]?.Value ?? "").toUpperCase();
|
|
2219
|
+
if (binlogFormatValue !== "ROW") {
|
|
2220
|
+
return {
|
|
2221
|
+
canStream: false,
|
|
2222
|
+
reason: `binlog_format is '${binlogFormatValue}', must be 'ROW' for binlog streaming`
|
|
2223
|
+
};
|
|
2224
|
+
}
|
|
2225
|
+
return { canStream: true, reason: "Binlog streaming supported" };
|
|
2226
|
+
} catch (err) {
|
|
2227
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
2228
|
+
return { canStream: false, reason: `Capability detection failed: ${message}` };
|
|
2229
|
+
}
|
|
2230
|
+
}
|
|
2231
|
+
};
|
|
2232
|
+
|
|
2233
|
+
// src/reasoning/types.ts
|
|
2234
|
+
var DURATION_REGEX = /^(\d+)(ms|s|m|h|d)$/;
|
|
2235
|
+
var UNIT_TO_MS = {
|
|
2236
|
+
ms: 1,
|
|
2237
|
+
s: 1e3,
|
|
2238
|
+
m: 6e4,
|
|
2239
|
+
h: 36e5,
|
|
2240
|
+
d: 864e5
|
|
2241
|
+
};
|
|
2242
|
+
function parseTransitionDuration(input) {
|
|
2243
|
+
const match = DURATION_REGEX.exec(input);
|
|
2244
|
+
if (!match) {
|
|
2245
|
+
throw new Error(`Invalid duration string: "${input}". Expected format: <number><unit> (ms|s|m|h|d)`);
|
|
2246
|
+
}
|
|
2247
|
+
const value = Number(match[1]);
|
|
2248
|
+
const unit = match[2];
|
|
2249
|
+
return value * UNIT_TO_MS[unit];
|
|
2250
|
+
}
|
|
2251
|
+
|
|
2252
|
+
// src/reasoning/transition-tracker.ts
|
|
2253
|
+
var logger = createLogger({ name: "transition-tracker" });
|
|
2254
|
+
var TransitionTracker = class {
|
|
2255
|
+
workflows;
|
|
2256
|
+
storage;
|
|
2257
|
+
onInsight;
|
|
2258
|
+
graphs;
|
|
2259
|
+
constructor(workflows, storage, onInsight) {
|
|
2260
|
+
this.workflows = workflows;
|
|
2261
|
+
this.storage = storage;
|
|
2262
|
+
this.onInsight = onInsight;
|
|
2263
|
+
this.graphs = /* @__PURE__ */ new Map();
|
|
2264
|
+
for (const [tableName, config] of Object.entries(workflows)) {
|
|
2265
|
+
this.graphs.set(tableName, this.parseGraph(config.expectedTransitions));
|
|
2266
|
+
}
|
|
2267
|
+
}
|
|
2268
|
+
handleChange(change) {
|
|
2269
|
+
if (change.operation !== "UPDATE") {
|
|
2270
|
+
return;
|
|
2271
|
+
}
|
|
2272
|
+
const tableName = change.tableName;
|
|
2273
|
+
const workflow = this.workflows[tableName];
|
|
2274
|
+
if (!workflow) {
|
|
2275
|
+
return;
|
|
2276
|
+
}
|
|
2277
|
+
if (change.changedColumns && !change.changedColumns.includes(workflow.stateColumn)) {
|
|
2278
|
+
return;
|
|
2279
|
+
}
|
|
2280
|
+
const newState = change.newData?.[workflow.stateColumn];
|
|
2281
|
+
if (newState === null || newState === void 0) {
|
|
2282
|
+
return;
|
|
2283
|
+
}
|
|
2284
|
+
const toState = String(newState);
|
|
2285
|
+
const primaryKey = String(change.primaryKey);
|
|
2286
|
+
const currentState = this.storage.getEntityState(tableName, primaryKey);
|
|
2287
|
+
if (!currentState) {
|
|
2288
|
+
this.storage.saveEntityState({
|
|
2289
|
+
tableName,
|
|
2290
|
+
primaryKey,
|
|
2291
|
+
state: toState,
|
|
2292
|
+
enteredAt: change.detectedAt.toISOString()
|
|
2293
|
+
});
|
|
2294
|
+
return;
|
|
2295
|
+
}
|
|
2296
|
+
const fromState = currentState.state;
|
|
2297
|
+
if (fromState === toState) {
|
|
2298
|
+
return;
|
|
2299
|
+
}
|
|
2300
|
+
const enteredAt = new Date(currentState.enteredAt);
|
|
2301
|
+
const durationMs = change.detectedAt.getTime() - enteredAt.getTime();
|
|
2302
|
+
const entityName = this.storage.getEntity(tableName)?.entityLabel ?? tableName;
|
|
2303
|
+
const graph = this.graphs.get(tableName);
|
|
2304
|
+
const validation = graph ? this.validateTransition(fromState, toState, graph) : { expected: false, insightType: "transition_unexpected", severity: "medium" };
|
|
2305
|
+
this.storage.saveTransition({
|
|
2306
|
+
entity: entityName,
|
|
2307
|
+
tableName,
|
|
2308
|
+
primaryKey,
|
|
2309
|
+
fromState,
|
|
2310
|
+
toState,
|
|
2311
|
+
durationMs,
|
|
2312
|
+
expected: validation.expected
|
|
2313
|
+
});
|
|
2314
|
+
this.storage.saveEntityState({
|
|
2315
|
+
tableName,
|
|
2316
|
+
primaryKey,
|
|
2317
|
+
state: toState,
|
|
2318
|
+
enteredAt: change.detectedAt.toISOString()
|
|
2319
|
+
});
|
|
2320
|
+
if (!validation.expected && validation.insightType && validation.severity) {
|
|
2321
|
+
const message = this.buildMessage(
|
|
2322
|
+
entityName,
|
|
2323
|
+
primaryKey,
|
|
2324
|
+
fromState,
|
|
2325
|
+
toState,
|
|
2326
|
+
validation.insightType,
|
|
2327
|
+
validation.expectedPath,
|
|
2328
|
+
durationMs
|
|
2329
|
+
);
|
|
2330
|
+
const insight = {
|
|
2331
|
+
entity: entityName,
|
|
2332
|
+
insightType: validation.insightType,
|
|
2333
|
+
severity: validation.severity,
|
|
2334
|
+
message,
|
|
2335
|
+
context: {
|
|
2336
|
+
primaryKey,
|
|
2337
|
+
fromState,
|
|
2338
|
+
toState,
|
|
2339
|
+
...validation.expectedPath ? { expectedPath: validation.expectedPath } : {},
|
|
2340
|
+
durationMs
|
|
2341
|
+
},
|
|
2342
|
+
timestamp: change.detectedAt
|
|
2343
|
+
};
|
|
2344
|
+
this.storage.saveInsight({
|
|
2345
|
+
entity: insight.entity,
|
|
2346
|
+
insightType: insight.insightType,
|
|
2347
|
+
severity: insight.severity,
|
|
2348
|
+
message: insight.message,
|
|
2349
|
+
context: insight.context
|
|
2350
|
+
});
|
|
2351
|
+
this.onInsight(insight);
|
|
2352
|
+
logger.warn(
|
|
2353
|
+
{ entity: entityName, primaryKey, fromState, toState, insightType: validation.insightType },
|
|
2354
|
+
message
|
|
2355
|
+
);
|
|
2356
|
+
}
|
|
2357
|
+
}
|
|
2358
|
+
checkStuck() {
|
|
2359
|
+
for (const [tableName, workflow] of Object.entries(this.workflows)) {
|
|
2360
|
+
if (!workflow.stuckThreshold) {
|
|
2361
|
+
continue;
|
|
2362
|
+
}
|
|
2363
|
+
const thresholdMs = parseTransitionDuration(workflow.stuckThreshold);
|
|
2364
|
+
const stuckRecords = this.storage.getStuckEntityStates(
|
|
2365
|
+
tableName,
|
|
2366
|
+
thresholdMs
|
|
2367
|
+
);
|
|
2368
|
+
if (stuckRecords.length === 0) {
|
|
2369
|
+
continue;
|
|
2370
|
+
}
|
|
2371
|
+
const entityName = this.storage.getEntity(tableName)?.entityLabel ?? tableName;
|
|
2372
|
+
const recentInsights = this.storage.getInsights({
|
|
2373
|
+
entity: entityName,
|
|
2374
|
+
last: 60
|
|
2375
|
+
});
|
|
2376
|
+
const recentlyAlertedKeys = new Set(
|
|
2377
|
+
recentInsights.filter((i) => i.insightType === "transition_stuck").map((i) => String(i.context.primaryKey))
|
|
2378
|
+
);
|
|
2379
|
+
for (const record of stuckRecords) {
|
|
2380
|
+
if (recentlyAlertedKeys.has(record.primaryKey)) {
|
|
2381
|
+
continue;
|
|
2382
|
+
}
|
|
2383
|
+
const enteredAt = new Date(record.enteredAt);
|
|
2384
|
+
const now = /* @__PURE__ */ new Date();
|
|
2385
|
+
const actualDurationMs = now.getTime() - enteredAt.getTime();
|
|
2386
|
+
const severity = this.computeStuckSeverity(actualDurationMs, thresholdMs);
|
|
2387
|
+
const message = this.buildMessage(
|
|
2388
|
+
entityName,
|
|
2389
|
+
record.primaryKey,
|
|
2390
|
+
record.state,
|
|
2391
|
+
void 0,
|
|
2392
|
+
"transition_stuck",
|
|
2393
|
+
void 0,
|
|
2394
|
+
actualDurationMs
|
|
2395
|
+
);
|
|
2396
|
+
const insight = {
|
|
2397
|
+
entity: entityName,
|
|
2398
|
+
insightType: "transition_stuck",
|
|
2399
|
+
severity,
|
|
2400
|
+
message,
|
|
2401
|
+
context: {
|
|
2402
|
+
primaryKey: record.primaryKey,
|
|
2403
|
+
fromState: record.state,
|
|
2404
|
+
durationMs: actualDurationMs
|
|
2405
|
+
},
|
|
2406
|
+
timestamp: now
|
|
2407
|
+
};
|
|
2408
|
+
this.storage.saveInsight({
|
|
2409
|
+
entity: insight.entity,
|
|
2410
|
+
insightType: insight.insightType,
|
|
2411
|
+
severity: insight.severity,
|
|
2412
|
+
message: insight.message,
|
|
2413
|
+
context: insight.context
|
|
2414
|
+
});
|
|
2415
|
+
this.onInsight(insight);
|
|
2416
|
+
logger.warn(
|
|
2417
|
+
{ entity: entityName, primaryKey: record.primaryKey, state: record.state, severity },
|
|
2418
|
+
message
|
|
2419
|
+
);
|
|
2420
|
+
}
|
|
2421
|
+
}
|
|
2422
|
+
}
|
|
2423
|
+
parseGraph(transitions) {
|
|
2424
|
+
const adjacency = /* @__PURE__ */ new Map();
|
|
2425
|
+
for (const transition of transitions) {
|
|
2426
|
+
const parts = transition.split("->").map((s) => s.trim());
|
|
2427
|
+
if (parts.length !== 2) {
|
|
2428
|
+
logger.warn({ transition }, "Invalid transition format, skipping");
|
|
2429
|
+
continue;
|
|
2430
|
+
}
|
|
2431
|
+
const [from, to] = parts;
|
|
2432
|
+
const neighbors = adjacency.get(from) ?? [];
|
|
2433
|
+
neighbors.push(to);
|
|
2434
|
+
adjacency.set(from, neighbors);
|
|
2435
|
+
}
|
|
2436
|
+
return { adjacency };
|
|
2437
|
+
}
|
|
2438
|
+
validateTransition(from, to, graph) {
|
|
2439
|
+
const directNeighbors = graph.adjacency.get(from) ?? [];
|
|
2440
|
+
if (directNeighbors.includes(to)) {
|
|
2441
|
+
return { expected: true };
|
|
2442
|
+
}
|
|
2443
|
+
const path3 = this.findPath(from, to, graph.adjacency);
|
|
2444
|
+
if (path3 && path3.length > 2) {
|
|
2445
|
+
return {
|
|
2446
|
+
expected: false,
|
|
2447
|
+
insightType: "transition_skip",
|
|
2448
|
+
severity: "high",
|
|
2449
|
+
expectedPath: path3
|
|
2450
|
+
};
|
|
2451
|
+
}
|
|
2452
|
+
return {
|
|
2453
|
+
expected: false,
|
|
2454
|
+
insightType: "transition_unexpected",
|
|
2455
|
+
severity: "medium"
|
|
2456
|
+
};
|
|
2457
|
+
}
|
|
2458
|
+
findPath(from, to, adjacency) {
|
|
2459
|
+
const visited = /* @__PURE__ */ new Set();
|
|
2460
|
+
const queue = [[from]];
|
|
2461
|
+
visited.add(from);
|
|
2462
|
+
while (queue.length > 0) {
|
|
2463
|
+
const currentPath = queue.shift();
|
|
2464
|
+
const current = currentPath[currentPath.length - 1];
|
|
2465
|
+
const neighbors = adjacency.get(current) ?? [];
|
|
2466
|
+
for (const neighbor of neighbors) {
|
|
2467
|
+
if (neighbor === to) {
|
|
2468
|
+
return [...currentPath, neighbor];
|
|
2469
|
+
}
|
|
2470
|
+
if (!visited.has(neighbor)) {
|
|
2471
|
+
visited.add(neighbor);
|
|
2472
|
+
queue.push([...currentPath, neighbor]);
|
|
2473
|
+
}
|
|
2474
|
+
}
|
|
2475
|
+
}
|
|
2476
|
+
return null;
|
|
2477
|
+
}
|
|
2478
|
+
computeStuckSeverity(durationMs, thresholdMs) {
|
|
2479
|
+
const multiplier = durationMs / thresholdMs;
|
|
2480
|
+
if (multiplier >= 8) {
|
|
2481
|
+
return "critical";
|
|
2482
|
+
}
|
|
2483
|
+
if (multiplier >= 4) {
|
|
2484
|
+
return "high";
|
|
2485
|
+
}
|
|
2486
|
+
if (multiplier >= 2) {
|
|
2487
|
+
return "medium";
|
|
2488
|
+
}
|
|
2489
|
+
return "low";
|
|
2490
|
+
}
|
|
2491
|
+
buildMessage(entity, primaryKey, fromState, toState, insightType, expectedPath, durationMs) {
|
|
2492
|
+
const formatted = this.formatDuration(durationMs);
|
|
2493
|
+
switch (insightType) {
|
|
2494
|
+
case "transition_skip":
|
|
2495
|
+
return `${entity} ${primaryKey} skipped states: ${fromState} -> ${toState} (expected path: ${expectedPath?.join(" -> ")}) after ${formatted}`;
|
|
2496
|
+
case "transition_unexpected":
|
|
2497
|
+
return `${entity} ${primaryKey} made unexpected transition: ${fromState} -> ${toState} after ${formatted}`;
|
|
2498
|
+
case "transition_stuck":
|
|
2499
|
+
return `${entity} ${primaryKey} stuck in "${fromState}" for ${formatted}`;
|
|
2500
|
+
default:
|
|
2501
|
+
return `${entity} ${primaryKey}: ${insightType} (${fromState}${toState ? " -> " + toState : ""}) after ${formatted}`;
|
|
2502
|
+
}
|
|
2503
|
+
}
|
|
2504
|
+
formatDuration(ms) {
|
|
2505
|
+
const totalMinutes = Math.floor(ms / 6e4);
|
|
2506
|
+
const hours = Math.floor(totalMinutes / 60);
|
|
2507
|
+
const minutes = totalMinutes % 60;
|
|
2508
|
+
if (hours > 0 && minutes > 0) {
|
|
2509
|
+
return `${hours}h ${minutes}m`;
|
|
2510
|
+
}
|
|
2511
|
+
if (hours > 0) {
|
|
2512
|
+
return `${hours}h`;
|
|
2513
|
+
}
|
|
2514
|
+
if (minutes > 0) {
|
|
2515
|
+
return `${minutes}m`;
|
|
2516
|
+
}
|
|
2517
|
+
const seconds = Math.floor(ms / 1e3);
|
|
2518
|
+
if (seconds > 0) {
|
|
2519
|
+
return `${seconds}s`;
|
|
2520
|
+
}
|
|
2521
|
+
return `${ms}ms`;
|
|
2522
|
+
}
|
|
2523
|
+
};
|
|
2524
|
+
|
|
2525
|
+
// src/analytics/types.ts
|
|
2526
|
+
var TIME_WINDOW_REGEX = /^(\d+)(s|m|h|d)$/;
|
|
2527
|
+
var UNIT_TO_MS2 = {
|
|
2528
|
+
s: 1e3,
|
|
2529
|
+
m: 6e4,
|
|
2530
|
+
h: 36e5,
|
|
2531
|
+
d: 864e5
|
|
2532
|
+
};
|
|
2533
|
+
function parseTimeWindow(input) {
|
|
2534
|
+
const match = TIME_WINDOW_REGEX.exec(input);
|
|
2535
|
+
if (!match) {
|
|
2536
|
+
throw new Error(`Invalid time window string: "${input}". Expected format: <number><unit> (s|m|h|d)`);
|
|
2537
|
+
}
|
|
2538
|
+
const value = Number(match[1]);
|
|
2539
|
+
const unit = match[2];
|
|
2540
|
+
return value * UNIT_TO_MS2[unit];
|
|
2541
|
+
}
|
|
2542
|
+
|
|
2543
|
+
// src/analytics/correlator.ts
|
|
2544
|
+
var logger2 = createLogger({ name: "correlator" });
|
|
2545
|
+
var MIN_SAMPLE_SIZE2 = 12;
|
|
2546
|
+
var ANOMALY_THRESHOLD = 2;
|
|
2547
|
+
var Correlator = class _Correlator {
|
|
2548
|
+
storage;
|
|
2549
|
+
onInsight;
|
|
2550
|
+
trackedEntities;
|
|
2551
|
+
parsedConfigs;
|
|
2552
|
+
constructor(configs, storage, onInsight) {
|
|
2553
|
+
this.storage = storage;
|
|
2554
|
+
this.onInsight = onInsight;
|
|
2555
|
+
this.trackedEntities = /* @__PURE__ */ new Set();
|
|
2556
|
+
this.parsedConfigs = [];
|
|
2557
|
+
for (const [name, config] of Object.entries(configs)) {
|
|
2558
|
+
this.trackedEntities.add(config.entities[0]);
|
|
2559
|
+
this.trackedEntities.add(config.entities[1]);
|
|
2560
|
+
this.parsedConfigs.push({
|
|
2561
|
+
name,
|
|
2562
|
+
entities: config.entities,
|
|
2563
|
+
timeWindowMs: parseTimeWindow(config.timeWindow)
|
|
2564
|
+
});
|
|
2565
|
+
}
|
|
2566
|
+
logger2.debug({ configCount: this.parsedConfigs.length }, "Correlator initialized");
|
|
2567
|
+
}
|
|
2568
|
+
recordEvent(entity, eventType, primaryKey) {
|
|
2569
|
+
if (!this.trackedEntities.has(entity)) {
|
|
2570
|
+
return;
|
|
2571
|
+
}
|
|
2572
|
+
this.storage.saveCorrelationEvent({ entity, eventType, primaryKey });
|
|
2573
|
+
}
|
|
2574
|
+
checkRateCorrelation() {
|
|
2575
|
+
for (const config of this.parsedConfigs) {
|
|
2576
|
+
const [entityA, entityB] = config.entities;
|
|
2577
|
+
const ratesA = this.storage.getCorrelationRates(entityA, 24);
|
|
2578
|
+
const ratesB = this.storage.getCorrelationRates(entityB, 24);
|
|
2579
|
+
const avgA = this.averageEventCount(ratesA);
|
|
2580
|
+
const avgB = this.averageEventCount(ratesB);
|
|
2581
|
+
if (avgA === 0 || avgB === 0) {
|
|
2582
|
+
continue;
|
|
2583
|
+
}
|
|
2584
|
+
const ratio = avgA / avgB;
|
|
2585
|
+
const baselineKey = `correlation:${config.name}:rate_ratio`;
|
|
2586
|
+
const existing = this.storage.getBaseline(baselineKey);
|
|
2587
|
+
if (!existing || existing.sampleSize < MIN_SAMPLE_SIZE2) {
|
|
2588
|
+
this.updateBaseline(baselineKey, ratio, existing);
|
|
2589
|
+
continue;
|
|
2590
|
+
}
|
|
2591
|
+
if (existing.stddev === 0) {
|
|
2592
|
+
this.updateBaseline(baselineKey, ratio, existing);
|
|
2593
|
+
continue;
|
|
2594
|
+
}
|
|
2595
|
+
const zScore = Math.abs(ratio - existing.mean) / existing.stddev;
|
|
2596
|
+
if (zScore >= ANOMALY_THRESHOLD) {
|
|
2597
|
+
const severity = _Correlator.getSeverity(zScore);
|
|
2598
|
+
const insight = {
|
|
2599
|
+
entity: config.name,
|
|
2600
|
+
insightType: "correlation_divergence",
|
|
2601
|
+
severity,
|
|
2602
|
+
message: `Rate ratio between ${entityA} and ${entityB} diverged to ${ratio.toFixed(2)} (expected ${existing.mean.toFixed(2)}\xB1${existing.stddev.toFixed(2)}, z=${zScore.toFixed(1)})`,
|
|
2603
|
+
context: {
|
|
2604
|
+
entityA,
|
|
2605
|
+
entityB,
|
|
2606
|
+
ratio,
|
|
2607
|
+
baselineMean: existing.mean,
|
|
2608
|
+
baselineStddev: existing.stddev,
|
|
2609
|
+
zScore
|
|
2610
|
+
},
|
|
2611
|
+
timestamp: /* @__PURE__ */ new Date()
|
|
2612
|
+
};
|
|
2613
|
+
this.storage.saveInsight({
|
|
2614
|
+
entity: insight.entity,
|
|
2615
|
+
insightType: insight.insightType,
|
|
2616
|
+
severity: insight.severity,
|
|
2617
|
+
message: insight.message,
|
|
2618
|
+
context: insight.context
|
|
2619
|
+
});
|
|
2620
|
+
this.onInsight(insight);
|
|
2621
|
+
logger2.info({ config: config.name, zScore, ratio }, "Rate correlation divergence detected");
|
|
2622
|
+
}
|
|
2623
|
+
this.updateBaseline(baselineKey, ratio, existing);
|
|
2624
|
+
}
|
|
2625
|
+
}
|
|
2626
|
+
checkCoOccurrence() {
|
|
2627
|
+
for (const config of this.parsedConfigs) {
|
|
2628
|
+
const [entityA, entityB] = config.entities;
|
|
2629
|
+
const timeWindowMinutes = Math.ceil(config.timeWindowMs / 6e4);
|
|
2630
|
+
const eventsA = this.storage.getCorrelationEvents(entityA, timeWindowMinutes);
|
|
2631
|
+
const eventsB = this.storage.getCorrelationEvents(entityB, timeWindowMinutes);
|
|
2632
|
+
const countA = eventsA.length;
|
|
2633
|
+
const countB = eventsB.length;
|
|
2634
|
+
if (countA === 0 && countB === 0) {
|
|
2635
|
+
continue;
|
|
2636
|
+
}
|
|
2637
|
+
const maxCount = Math.max(countA, countB);
|
|
2638
|
+
const minCount = Math.min(countA, countB);
|
|
2639
|
+
const coOccurrenceRate = maxCount === 0 ? 0 : minCount / maxCount;
|
|
2640
|
+
const baselineKey = `correlation:${config.name}:co_occurrence`;
|
|
2641
|
+
const existing = this.storage.getBaseline(baselineKey);
|
|
2642
|
+
if (!existing || existing.sampleSize < MIN_SAMPLE_SIZE2) {
|
|
2643
|
+
this.updateBaseline(baselineKey, coOccurrenceRate, existing);
|
|
2644
|
+
continue;
|
|
2645
|
+
}
|
|
2646
|
+
if (existing.stddev === 0) {
|
|
2647
|
+
this.updateBaseline(baselineKey, coOccurrenceRate, existing);
|
|
2648
|
+
continue;
|
|
2649
|
+
}
|
|
2650
|
+
const zScore = (existing.mean - coOccurrenceRate) / existing.stddev;
|
|
2651
|
+
if (zScore >= ANOMALY_THRESHOLD) {
|
|
2652
|
+
const severity = _Correlator.getSeverity(zScore);
|
|
2653
|
+
const insight = {
|
|
2654
|
+
entity: config.name,
|
|
2655
|
+
insightType: "correlation_timing",
|
|
2656
|
+
severity,
|
|
2657
|
+
message: `Co-occurrence between ${entityA} and ${entityB} dropped to ${(coOccurrenceRate * 100).toFixed(1)}% (expected ${(existing.mean * 100).toFixed(1)}%\xB1${(existing.stddev * 100).toFixed(1)}%, z=${zScore.toFixed(1)})`,
|
|
2658
|
+
context: {
|
|
2659
|
+
entityA,
|
|
2660
|
+
entityB,
|
|
2661
|
+
coOccurrenceRate,
|
|
2662
|
+
baselineMean: existing.mean,
|
|
2663
|
+
baselineStddev: existing.stddev,
|
|
2664
|
+
zScore,
|
|
2665
|
+
countA,
|
|
2666
|
+
countB
|
|
2667
|
+
},
|
|
2668
|
+
timestamp: /* @__PURE__ */ new Date()
|
|
2669
|
+
};
|
|
2670
|
+
this.storage.saveInsight({
|
|
2671
|
+
entity: insight.entity,
|
|
2672
|
+
insightType: insight.insightType,
|
|
2673
|
+
severity: insight.severity,
|
|
2674
|
+
message: insight.message,
|
|
2675
|
+
context: insight.context
|
|
2676
|
+
});
|
|
2677
|
+
this.onInsight(insight);
|
|
2678
|
+
logger2.info({ config: config.name, zScore, coOccurrenceRate }, "Co-occurrence timing violation detected");
|
|
2679
|
+
}
|
|
2680
|
+
this.updateBaseline(baselineKey, coOccurrenceRate, existing);
|
|
2681
|
+
}
|
|
2682
|
+
}
|
|
2683
|
+
static getSeverity(sigma) {
|
|
2684
|
+
if (sigma >= 5) return "critical";
|
|
2685
|
+
if (sigma >= 4) return "high";
|
|
2686
|
+
if (sigma >= 3) return "medium";
|
|
2687
|
+
return "low";
|
|
2688
|
+
}
|
|
2689
|
+
updateBaseline(key, currentValue, existing) {
|
|
2690
|
+
if (!existing) {
|
|
2691
|
+
this.storage.saveBaseline({
|
|
2692
|
+
key,
|
|
2693
|
+
mean: currentValue,
|
|
2694
|
+
stddev: 0,
|
|
2695
|
+
sampleSize: 1
|
|
2696
|
+
});
|
|
2697
|
+
return;
|
|
2698
|
+
}
|
|
2699
|
+
const newN = existing.sampleSize + 1;
|
|
2700
|
+
const delta = currentValue - existing.mean;
|
|
2701
|
+
const newMean = existing.mean + delta / newN;
|
|
2702
|
+
const delta2 = currentValue - newMean;
|
|
2703
|
+
const newVariance = newN <= 2 ? (existing.stddev * existing.stddev * Math.max(newN - 2, 0) + delta * delta2) / Math.max(newN - 1, 1) : (existing.stddev * existing.stddev * (newN - 2) + delta * delta2) / (newN - 1);
|
|
2704
|
+
const newStddev = Math.sqrt(Math.max(newVariance, 0));
|
|
2705
|
+
this.storage.saveBaseline({
|
|
2706
|
+
key,
|
|
2707
|
+
mean: newMean,
|
|
2708
|
+
stddev: newStddev,
|
|
2709
|
+
sampleSize: newN
|
|
2710
|
+
});
|
|
2711
|
+
}
|
|
2712
|
+
averageEventCount(rates) {
|
|
2713
|
+
if (rates.length === 0) return 0;
|
|
2714
|
+
const total = rates.reduce((sum, r) => sum + r.eventCount, 0);
|
|
2715
|
+
return total / rates.length;
|
|
2716
|
+
}
|
|
2717
|
+
};
|
|
2718
|
+
|
|
2719
|
+
// src/analytics/distribution-tracker.ts
|
|
2720
|
+
var logger3 = createLogger({ name: "distribution-tracker" });
|
|
2721
|
+
var MIN_SAMPLE_SIZE3 = 12;
|
|
2722
|
+
var NUM_BINS = 10;
|
|
2723
|
+
var DistributionTracker = class _DistributionTracker {
|
|
2724
|
+
storage;
|
|
2725
|
+
onInsight;
|
|
2726
|
+
entityConfigs;
|
|
2727
|
+
constructor(configs, storage, onInsight) {
|
|
2728
|
+
this.storage = storage;
|
|
2729
|
+
this.onInsight = onInsight;
|
|
2730
|
+
this.entityConfigs = /* @__PURE__ */ new Map();
|
|
2731
|
+
for (const [entity, config] of Object.entries(configs)) {
|
|
2732
|
+
this.entityConfigs.set(entity, {
|
|
2733
|
+
entity,
|
|
2734
|
+
columns: config.columns,
|
|
2735
|
+
temporalBuckets: config.temporalBuckets
|
|
2736
|
+
});
|
|
2737
|
+
}
|
|
2738
|
+
logger3.debug(
|
|
2739
|
+
{ entityCount: this.entityConfigs.size },
|
|
2740
|
+
"DistributionTracker initialized"
|
|
2741
|
+
);
|
|
2742
|
+
}
|
|
2743
|
+
recordValue(entity, data) {
|
|
2744
|
+
const config = this.entityConfigs.get(entity);
|
|
2745
|
+
if (!config) {
|
|
2746
|
+
return;
|
|
2747
|
+
}
|
|
2748
|
+
for (const column of config.columns) {
|
|
2749
|
+
const value = data[column];
|
|
2750
|
+
if (value === void 0 || value === null) {
|
|
2751
|
+
continue;
|
|
2752
|
+
}
|
|
2753
|
+
const bucketKey = _DistributionTracker.toBucketKey(value);
|
|
2754
|
+
if (bucketKey === null) {
|
|
2755
|
+
continue;
|
|
2756
|
+
}
|
|
2757
|
+
this.storage.saveHistogramBucket({
|
|
2758
|
+
entity,
|
|
2759
|
+
columnName: column,
|
|
2760
|
+
bucketKey,
|
|
2761
|
+
count: 1,
|
|
2762
|
+
period: "current"
|
|
2763
|
+
});
|
|
2764
|
+
}
|
|
2765
|
+
}
|
|
2766
|
+
recordTemporalEvent(entity, timestamp) {
|
|
2767
|
+
const config = this.entityConfigs.get(entity);
|
|
2768
|
+
if (!config) {
|
|
2769
|
+
return;
|
|
2770
|
+
}
|
|
2771
|
+
if (config.temporalBuckets === "hourly") {
|
|
2772
|
+
this.storage.saveTemporalPattern({
|
|
2773
|
+
entity,
|
|
2774
|
+
bucketType: "hour_of_day",
|
|
2775
|
+
bucketKey: timestamp.getUTCHours(),
|
|
2776
|
+
count: 1,
|
|
2777
|
+
period: "current"
|
|
2778
|
+
});
|
|
2779
|
+
} else {
|
|
2780
|
+
this.storage.saveTemporalPattern({
|
|
2781
|
+
entity,
|
|
2782
|
+
bucketType: "day_of_week",
|
|
2783
|
+
bucketKey: timestamp.getUTCDay(),
|
|
2784
|
+
count: 1,
|
|
2785
|
+
period: "current"
|
|
2786
|
+
});
|
|
2787
|
+
}
|
|
2788
|
+
}
|
|
2789
|
+
checkDistributionShift() {
|
|
2790
|
+
for (const config of this.entityConfigs.values()) {
|
|
2791
|
+
for (const column of config.columns) {
|
|
2792
|
+
const baselineKey = `distribution:${config.entity}:${column}:shift`;
|
|
2793
|
+
const baseline = this.storage.getBaseline(baselineKey);
|
|
2794
|
+
if (!baseline || baseline.sampleSize < MIN_SAMPLE_SIZE3) {
|
|
2795
|
+
continue;
|
|
2796
|
+
}
|
|
2797
|
+
const baselineBuckets = this.storage.getHistogramBuckets(
|
|
2798
|
+
config.entity,
|
|
2799
|
+
column,
|
|
2800
|
+
"baseline"
|
|
2801
|
+
);
|
|
2802
|
+
const currentBuckets = this.storage.getHistogramBuckets(
|
|
2803
|
+
config.entity,
|
|
2804
|
+
column,
|
|
2805
|
+
"current"
|
|
2806
|
+
);
|
|
2807
|
+
if (baselineBuckets.length === 0 || currentBuckets.length === 0) {
|
|
2808
|
+
continue;
|
|
2809
|
+
}
|
|
2810
|
+
const { observed, expected } = _DistributionTracker.alignAndNormalize(currentBuckets, baselineBuckets);
|
|
2811
|
+
if (observed.length === 0) {
|
|
2812
|
+
continue;
|
|
2813
|
+
}
|
|
2814
|
+
const chi2 = _DistributionTracker.chiSquared(observed, expected);
|
|
2815
|
+
const df = Math.max(observed.length - 1, 1);
|
|
2816
|
+
const threshold = _DistributionTracker.chiSquaredThreshold(df);
|
|
2817
|
+
if (chi2 > threshold) {
|
|
2818
|
+
const pValue = _DistributionTracker.chiSquaredPValue(chi2, df);
|
|
2819
|
+
const severity = _DistributionTracker.shiftSeverity(chi2, df);
|
|
2820
|
+
const insightType = "distribution_shift";
|
|
2821
|
+
const insight = {
|
|
2822
|
+
entity: config.entity,
|
|
2823
|
+
insightType,
|
|
2824
|
+
severity,
|
|
2825
|
+
message: `Distribution shift detected for ${config.entity}.${column}: chi2=${chi2.toFixed(2)}, df=${df}, p=${pValue.toFixed(4)}`,
|
|
2826
|
+
context: {
|
|
2827
|
+
column,
|
|
2828
|
+
chi2,
|
|
2829
|
+
df,
|
|
2830
|
+
pValue,
|
|
2831
|
+
threshold
|
|
2832
|
+
},
|
|
2833
|
+
timestamp: /* @__PURE__ */ new Date()
|
|
2834
|
+
};
|
|
2835
|
+
this.storage.saveInsight({
|
|
2836
|
+
entity: insight.entity,
|
|
2837
|
+
insightType: insight.insightType,
|
|
2838
|
+
severity: insight.severity,
|
|
2839
|
+
message: insight.message,
|
|
2840
|
+
context: insight.context
|
|
2841
|
+
});
|
|
2842
|
+
this.onInsight(insight);
|
|
2843
|
+
logger3.info(
|
|
2844
|
+
{ entity: config.entity, column, chi2, df, pValue },
|
|
2845
|
+
"Distribution shift detected"
|
|
2846
|
+
);
|
|
2847
|
+
}
|
|
2848
|
+
}
|
|
2849
|
+
}
|
|
2850
|
+
}
|
|
2851
|
+
checkTemporalShift() {
|
|
2852
|
+
for (const config of this.entityConfigs.values()) {
|
|
2853
|
+
const bucketType = config.temporalBuckets === "hourly" ? "hour_of_day" : "day_of_week";
|
|
2854
|
+
const baselineKey = `temporal:${config.entity}:${bucketType}:shift`;
|
|
2855
|
+
const baseline = this.storage.getBaseline(baselineKey);
|
|
2856
|
+
if (!baseline || baseline.sampleSize < MIN_SAMPLE_SIZE3) {
|
|
2857
|
+
continue;
|
|
2858
|
+
}
|
|
2859
|
+
const baselinePatterns = this.storage.getTemporalPatterns(
|
|
2860
|
+
config.entity,
|
|
2861
|
+
bucketType,
|
|
2862
|
+
"baseline"
|
|
2863
|
+
);
|
|
2864
|
+
const currentPatterns = this.storage.getTemporalPatterns(
|
|
2865
|
+
config.entity,
|
|
2866
|
+
bucketType,
|
|
2867
|
+
"current"
|
|
2868
|
+
);
|
|
2869
|
+
if (baselinePatterns.length === 0 || currentPatterns.length === 0) {
|
|
2870
|
+
continue;
|
|
2871
|
+
}
|
|
2872
|
+
const { observed, expected } = _DistributionTracker.alignAndNormalizeNumericBuckets(
|
|
2873
|
+
currentPatterns,
|
|
2874
|
+
baselinePatterns
|
|
2875
|
+
);
|
|
2876
|
+
if (observed.length === 0) {
|
|
2877
|
+
continue;
|
|
2878
|
+
}
|
|
2879
|
+
const chi2 = _DistributionTracker.chiSquared(observed, expected);
|
|
2880
|
+
const df = Math.max(observed.length - 1, 1);
|
|
2881
|
+
const threshold = _DistributionTracker.chiSquaredThreshold(df);
|
|
2882
|
+
if (chi2 > threshold) {
|
|
2883
|
+
const pValue = _DistributionTracker.chiSquaredPValue(chi2, df);
|
|
2884
|
+
const severity = _DistributionTracker.shiftSeverity(chi2, df);
|
|
2885
|
+
const insightType = "temporal_shift";
|
|
2886
|
+
const insight = {
|
|
2887
|
+
entity: config.entity,
|
|
2888
|
+
insightType,
|
|
2889
|
+
severity,
|
|
2890
|
+
message: `Temporal pattern shift detected for ${config.entity} (${bucketType}): chi2=${chi2.toFixed(2)}, df=${df}, p=${pValue.toFixed(4)}`,
|
|
2891
|
+
context: {
|
|
2892
|
+
bucketType,
|
|
2893
|
+
chi2,
|
|
2894
|
+
df,
|
|
2895
|
+
pValue,
|
|
2896
|
+
threshold
|
|
2897
|
+
},
|
|
2898
|
+
timestamp: /* @__PURE__ */ new Date()
|
|
2899
|
+
};
|
|
2900
|
+
this.storage.saveInsight({
|
|
2901
|
+
entity: insight.entity,
|
|
2902
|
+
insightType: insight.insightType,
|
|
2903
|
+
severity: insight.severity,
|
|
2904
|
+
message: insight.message,
|
|
2905
|
+
context: insight.context
|
|
2906
|
+
});
|
|
2907
|
+
this.onInsight(insight);
|
|
2908
|
+
logger3.info(
|
|
2909
|
+
{ entity: config.entity, bucketType, chi2, df, pValue },
|
|
2910
|
+
"Temporal pattern shift detected"
|
|
2911
|
+
);
|
|
2912
|
+
}
|
|
2913
|
+
}
|
|
2914
|
+
}
|
|
2915
|
+
static chiSquared(observed, expected) {
|
|
2916
|
+
let sum = 0;
|
|
2917
|
+
for (let i = 0; i < observed.length; i++) {
|
|
2918
|
+
const e = expected[i];
|
|
2919
|
+
if (e === 0) {
|
|
2920
|
+
continue;
|
|
2921
|
+
}
|
|
2922
|
+
const diff = observed[i] - e;
|
|
2923
|
+
sum += diff * diff / e;
|
|
2924
|
+
}
|
|
2925
|
+
return sum;
|
|
2926
|
+
}
|
|
2927
|
+
static chiSquaredThreshold(df) {
|
|
2928
|
+
const table = {
|
|
2929
|
+
1: 3.841,
|
|
2930
|
+
2: 5.991,
|
|
2931
|
+
3: 7.815,
|
|
2932
|
+
4: 9.488,
|
|
2933
|
+
5: 11.07
|
|
2934
|
+
};
|
|
2935
|
+
if (df in table) {
|
|
2936
|
+
return table[df];
|
|
2937
|
+
}
|
|
2938
|
+
const term = 1 - 2 / (9 * df);
|
|
2939
|
+
return df * Math.pow(term, 3);
|
|
2940
|
+
}
|
|
2941
|
+
static chiSquaredPValue(chi2, df) {
|
|
2942
|
+
if (df <= 0 || chi2 <= 0) return 1;
|
|
2943
|
+
const a = df / 2;
|
|
2944
|
+
const x = chi2 / 2;
|
|
2945
|
+
return 1 - _DistributionTracker.regularizedGammaP(a, x);
|
|
2946
|
+
}
|
|
2947
|
+
static regularizedGammaP(a, x) {
|
|
2948
|
+
if (x < 0) return 0;
|
|
2949
|
+
if (x === 0) return 0;
|
|
2950
|
+
if (x < a + 1) {
|
|
2951
|
+
return _DistributionTracker.gammaPSeries(a, x);
|
|
2952
|
+
}
|
|
2953
|
+
return 1 - _DistributionTracker.gammaPContinuedFraction(a, x);
|
|
2954
|
+
}
|
|
2955
|
+
static gammaPSeries(a, x) {
|
|
2956
|
+
const lnGammaA = _DistributionTracker.lnGamma(a);
|
|
2957
|
+
let sum = 1 / a;
|
|
2958
|
+
let term = 1 / a;
|
|
2959
|
+
for (let n = 1; n < 200; n++) {
|
|
2960
|
+
term *= x / (a + n);
|
|
2961
|
+
sum += term;
|
|
2962
|
+
if (Math.abs(term) < Math.abs(sum) * 1e-10) break;
|
|
2963
|
+
}
|
|
2964
|
+
return sum * Math.exp(-x + a * Math.log(x) - lnGammaA);
|
|
2965
|
+
}
|
|
2966
|
+
static gammaPContinuedFraction(a, x) {
|
|
2967
|
+
const lnGammaA = _DistributionTracker.lnGamma(a);
|
|
2968
|
+
let f = 1e-30;
|
|
2969
|
+
let c = 1e-30;
|
|
2970
|
+
let d = 1 / (x + 1 - a);
|
|
2971
|
+
let h = d;
|
|
2972
|
+
for (let n = 1; n < 200; n++) {
|
|
2973
|
+
const an = -n * (n - a);
|
|
2974
|
+
const bn = x + 2 * n + 1 - a;
|
|
2975
|
+
d = bn + an * d;
|
|
2976
|
+
if (Math.abs(d) < 1e-30) d = 1e-30;
|
|
2977
|
+
c = bn + an / c;
|
|
2978
|
+
if (Math.abs(c) < 1e-30) c = 1e-30;
|
|
2979
|
+
d = 1 / d;
|
|
2980
|
+
const delta = c * d;
|
|
2981
|
+
h *= delta;
|
|
2982
|
+
if (Math.abs(delta - 1) < 1e-10) break;
|
|
2983
|
+
}
|
|
2984
|
+
return Math.exp(-x + a * Math.log(x) - lnGammaA) * h;
|
|
2985
|
+
}
|
|
2986
|
+
static lnGamma(x) {
|
|
2987
|
+
const g = 7;
|
|
2988
|
+
const coef = [
|
|
2989
|
+
0.9999999999998099,
|
|
2990
|
+
676.5203681218851,
|
|
2991
|
+
-1259.1392167224028,
|
|
2992
|
+
771.3234287776531,
|
|
2993
|
+
-176.6150291621406,
|
|
2994
|
+
12.507343278686905,
|
|
2995
|
+
-0.13857109526572012,
|
|
2996
|
+
9984369578019572e-21,
|
|
2997
|
+
15056327351493116e-23
|
|
2998
|
+
];
|
|
2999
|
+
if (x < 0.5) {
|
|
3000
|
+
return Math.log(Math.PI / Math.sin(Math.PI * x)) - _DistributionTracker.lnGamma(1 - x);
|
|
3001
|
+
}
|
|
3002
|
+
x -= 1;
|
|
3003
|
+
let a = coef[0];
|
|
3004
|
+
const t = x + g + 0.5;
|
|
3005
|
+
for (let i = 1; i < coef.length; i++) {
|
|
3006
|
+
a += coef[i] / (x + i);
|
|
3007
|
+
}
|
|
3008
|
+
return 0.5 * Math.log(2 * Math.PI) + (x + 0.5) * Math.log(t) - t + Math.log(a);
|
|
3009
|
+
}
|
|
3010
|
+
static shiftSeverity(chi2, df) {
|
|
3011
|
+
const p = _DistributionTracker.chiSquaredPValue(chi2, df);
|
|
3012
|
+
if (p < 1e-3) return "critical";
|
|
3013
|
+
if (p < 0.01) return "high";
|
|
3014
|
+
if (p < 0.05) return "medium";
|
|
3015
|
+
return "low";
|
|
3016
|
+
}
|
|
3017
|
+
static toBucketKey(value) {
|
|
3018
|
+
if (typeof value === "number") {
|
|
3019
|
+
return `n:${value}`;
|
|
3020
|
+
}
|
|
3021
|
+
if (typeof value === "string") {
|
|
3022
|
+
return value;
|
|
3023
|
+
}
|
|
3024
|
+
return null;
|
|
3025
|
+
}
|
|
3026
|
+
static alignAndNormalize(currentBuckets, baselineBuckets) {
|
|
3027
|
+
const allBuckets = [...baselineBuckets, ...currentBuckets];
|
|
3028
|
+
const isNumeric = allBuckets.length > 0 && allBuckets.every((b) => b.bucketKey.startsWith("n:"));
|
|
3029
|
+
if (isNumeric) {
|
|
3030
|
+
return _DistributionTracker.alignNumericBins(currentBuckets, baselineBuckets);
|
|
3031
|
+
}
|
|
3032
|
+
return _DistributionTracker.alignCategoricalBuckets(currentBuckets, baselineBuckets);
|
|
3033
|
+
}
|
|
3034
|
+
static alignCategoricalBuckets(currentBuckets, baselineBuckets) {
|
|
3035
|
+
const allKeys = /* @__PURE__ */ new Set();
|
|
3036
|
+
for (const b of baselineBuckets) allKeys.add(b.bucketKey);
|
|
3037
|
+
for (const b of currentBuckets) allKeys.add(b.bucketKey);
|
|
3038
|
+
const baselineMap = new Map(baselineBuckets.map((b) => [b.bucketKey, b.count]));
|
|
3039
|
+
const currentMap = new Map(currentBuckets.map((b) => [b.bucketKey, b.count]));
|
|
3040
|
+
const keys = Array.from(allKeys).sort();
|
|
3041
|
+
const rawObserved = [];
|
|
3042
|
+
const rawExpected = [];
|
|
3043
|
+
for (const key of keys) {
|
|
3044
|
+
rawObserved.push(currentMap.get(key) ?? 0);
|
|
3045
|
+
rawExpected.push(baselineMap.get(key) ?? 0);
|
|
3046
|
+
}
|
|
3047
|
+
const totalObserved = rawObserved.reduce((s, v) => s + v, 0);
|
|
3048
|
+
const totalExpected = rawExpected.reduce((s, v) => s + v, 0);
|
|
3049
|
+
if (totalExpected === 0 || totalObserved === 0) {
|
|
3050
|
+
return { observed: [], expected: [] };
|
|
3051
|
+
}
|
|
3052
|
+
const scale = totalObserved / totalExpected;
|
|
3053
|
+
const expected = rawExpected.map((e) => e * scale);
|
|
3054
|
+
return { observed: rawObserved, expected };
|
|
3055
|
+
}
|
|
3056
|
+
static alignNumericBins(currentBuckets, baselineBuckets) {
|
|
3057
|
+
const parseValue = (key) => parseFloat(key.slice(2));
|
|
3058
|
+
const allValues = [];
|
|
3059
|
+
for (const b of baselineBuckets) allValues.push(parseValue(b.bucketKey));
|
|
3060
|
+
for (const b of currentBuckets) allValues.push(parseValue(b.bucketKey));
|
|
3061
|
+
if (allValues.length === 0) {
|
|
3062
|
+
return { observed: [], expected: [] };
|
|
3063
|
+
}
|
|
3064
|
+
const min = Math.min(...allValues);
|
|
3065
|
+
const max = Math.max(...allValues);
|
|
3066
|
+
const range = max - min;
|
|
3067
|
+
if (range === 0) {
|
|
3068
|
+
const totalObs = currentBuckets.reduce((s, b) => s + b.count, 0);
|
|
3069
|
+
const totalExp = baselineBuckets.reduce((s, b) => s + b.count, 0);
|
|
3070
|
+
if (totalObs === 0 || totalExp === 0) return { observed: [], expected: [] };
|
|
3071
|
+
return { observed: [totalObs], expected: [totalExp] };
|
|
3072
|
+
}
|
|
3073
|
+
const binWidth = range / NUM_BINS;
|
|
3074
|
+
const observed = new Array(NUM_BINS).fill(0);
|
|
3075
|
+
const rawExpected = new Array(NUM_BINS).fill(0);
|
|
3076
|
+
for (const b of currentBuckets) {
|
|
3077
|
+
const val = parseValue(b.bucketKey);
|
|
3078
|
+
const bin = Math.min(Math.floor((val - min) / binWidth), NUM_BINS - 1);
|
|
3079
|
+
observed[bin] += b.count;
|
|
3080
|
+
}
|
|
3081
|
+
for (const b of baselineBuckets) {
|
|
3082
|
+
const val = parseValue(b.bucketKey);
|
|
3083
|
+
const bin = Math.min(Math.floor((val - min) / binWidth), NUM_BINS - 1);
|
|
3084
|
+
rawExpected[bin] += b.count;
|
|
3085
|
+
}
|
|
3086
|
+
const totalObserved = observed.reduce((s, v) => s + v, 0);
|
|
3087
|
+
const totalExpected = rawExpected.reduce((s, v) => s + v, 0);
|
|
3088
|
+
if (totalObserved === 0 || totalExpected === 0) {
|
|
3089
|
+
return { observed: [], expected: [] };
|
|
3090
|
+
}
|
|
3091
|
+
const scale = totalObserved / totalExpected;
|
|
3092
|
+
const expected = rawExpected.map((e) => e * scale);
|
|
3093
|
+
return { observed, expected };
|
|
3094
|
+
}
|
|
3095
|
+
static alignAndNormalizeNumericBuckets(currentBuckets, baselineBuckets) {
|
|
3096
|
+
const allKeys = /* @__PURE__ */ new Set();
|
|
3097
|
+
for (const b of baselineBuckets) allKeys.add(b.bucketKey);
|
|
3098
|
+
for (const b of currentBuckets) allKeys.add(b.bucketKey);
|
|
3099
|
+
const baselineMap = new Map(baselineBuckets.map((b) => [b.bucketKey, b.count]));
|
|
3100
|
+
const currentMap = new Map(currentBuckets.map((b) => [b.bucketKey, b.count]));
|
|
3101
|
+
const keys = Array.from(allKeys).sort((a, b) => a - b);
|
|
3102
|
+
const rawObserved = [];
|
|
3103
|
+
const rawExpected = [];
|
|
3104
|
+
for (const key of keys) {
|
|
3105
|
+
rawObserved.push(currentMap.get(key) ?? 0);
|
|
3106
|
+
rawExpected.push(baselineMap.get(key) ?? 0);
|
|
3107
|
+
}
|
|
3108
|
+
const totalObserved = rawObserved.reduce((s, v) => s + v, 0);
|
|
3109
|
+
const totalExpected = rawExpected.reduce((s, v) => s + v, 0);
|
|
3110
|
+
if (totalExpected === 0 || totalObserved === 0) {
|
|
3111
|
+
return { observed: [], expected: [] };
|
|
3112
|
+
}
|
|
3113
|
+
const scale = totalObserved / totalExpected;
|
|
3114
|
+
const expected = rawExpected.map((e) => e * scale);
|
|
3115
|
+
return { observed: rawObserved, expected };
|
|
3116
|
+
}
|
|
3117
|
+
};
|
|
3118
|
+
|
|
3119
|
+
// src/analytics/analytics-engine.ts
|
|
3120
|
+
var logger4 = createLogger({ name: "analytics-engine" });
|
|
3121
|
+
var AnalyticsEngine = class {
|
|
3122
|
+
correlator;
|
|
3123
|
+
distributionTracker;
|
|
3124
|
+
constructor(config, storage, onInsight) {
|
|
3125
|
+
const correlationEntries = config.correlations ? Object.keys(config.correlations) : [];
|
|
3126
|
+
const distributionEntries = config.distributions ? Object.keys(config.distributions) : [];
|
|
3127
|
+
if (correlationEntries.length > 0 && config.correlations) {
|
|
3128
|
+
this.correlator = new Correlator(config.correlations, storage, onInsight);
|
|
3129
|
+
} else {
|
|
3130
|
+
this.correlator = null;
|
|
3131
|
+
}
|
|
3132
|
+
if (distributionEntries.length > 0 && config.distributions) {
|
|
3133
|
+
this.distributionTracker = new DistributionTracker(config.distributions, storage, onInsight);
|
|
3134
|
+
} else {
|
|
3135
|
+
this.distributionTracker = null;
|
|
3136
|
+
}
|
|
3137
|
+
logger4.debug(
|
|
3138
|
+
{ hasCorrelator: this.correlator !== null, hasDistributionTracker: this.distributionTracker !== null },
|
|
3139
|
+
"AnalyticsEngine initialized"
|
|
3140
|
+
);
|
|
3141
|
+
}
|
|
3142
|
+
handleEvent(entity, eventType, primaryKey, data, timestamp) {
|
|
3143
|
+
if (this.correlator) {
|
|
3144
|
+
this.correlator.recordEvent(entity, eventType, primaryKey);
|
|
3145
|
+
}
|
|
3146
|
+
if (this.distributionTracker) {
|
|
3147
|
+
this.distributionTracker.recordValue(entity, data);
|
|
3148
|
+
this.distributionTracker.recordTemporalEvent(entity, timestamp);
|
|
3149
|
+
}
|
|
3150
|
+
}
|
|
3151
|
+
runPeriodicChecks() {
|
|
3152
|
+
if (this.correlator) {
|
|
3153
|
+
this.correlator.checkRateCorrelation();
|
|
3154
|
+
this.correlator.checkCoOccurrence();
|
|
3155
|
+
}
|
|
3156
|
+
if (this.distributionTracker) {
|
|
3157
|
+
this.distributionTracker.checkDistributionShift();
|
|
3158
|
+
this.distributionTracker.checkTemporalShift();
|
|
3159
|
+
}
|
|
3160
|
+
}
|
|
3161
|
+
};
|
|
3162
|
+
|
|
3163
|
+
// src/knowledge/graph-store.ts
|
|
3164
|
+
var logger5 = createLogger({ name: "graph-store" });
|
|
3165
|
+
function toNode(raw) {
|
|
3166
|
+
return {
|
|
3167
|
+
id: raw.id,
|
|
3168
|
+
nodeType: raw.nodeType,
|
|
3169
|
+
refId: raw.refId,
|
|
3170
|
+
label: raw.label,
|
|
3171
|
+
metadata: JSON.parse(raw.metadata || "{}"),
|
|
3172
|
+
timestamp: raw.timestamp
|
|
3173
|
+
};
|
|
3174
|
+
}
|
|
3175
|
+
function toEdge(raw) {
|
|
3176
|
+
return {
|
|
3177
|
+
id: raw.id,
|
|
3178
|
+
sourceId: raw.sourceId,
|
|
3179
|
+
targetId: raw.targetId,
|
|
3180
|
+
edgeType: raw.edgeType,
|
|
3181
|
+
weight: raw.weight,
|
|
3182
|
+
metadata: JSON.parse(raw.metadata || "{}"),
|
|
3183
|
+
timestamp: raw.timestamp
|
|
3184
|
+
};
|
|
3185
|
+
}
|
|
3186
|
+
var GraphStore = class {
|
|
3187
|
+
storage;
|
|
3188
|
+
constructor(storage) {
|
|
3189
|
+
this.storage = storage;
|
|
3190
|
+
}
|
|
3191
|
+
upsertNode(nodeType, refId, label, metadata = {}) {
|
|
3192
|
+
const id = this.storage.saveKgNode({ nodeType, refId, label, metadata: JSON.stringify(metadata) });
|
|
3193
|
+
logger5.debug({ id, nodeType, refId }, "Upserted node");
|
|
3194
|
+
return id;
|
|
3195
|
+
}
|
|
3196
|
+
getNode(id) {
|
|
3197
|
+
const raw = this.storage.getKgNode(id);
|
|
3198
|
+
return raw ? toNode(raw) : void 0;
|
|
3199
|
+
}
|
|
3200
|
+
getNodeByRef(nodeType, refId) {
|
|
3201
|
+
const raw = this.storage.getKgNodeByRef(nodeType, refId);
|
|
3202
|
+
return raw ? toNode(raw) : void 0;
|
|
3203
|
+
}
|
|
3204
|
+
getNodesByType(nodeType) {
|
|
3205
|
+
return this.storage.getKgNodesByType(nodeType).map(toNode);
|
|
3206
|
+
}
|
|
3207
|
+
getAllNodes() {
|
|
3208
|
+
return this.storage.getAllKgNodes().map(toNode);
|
|
3209
|
+
}
|
|
3210
|
+
deleteNode(id) {
|
|
3211
|
+
this.storage.deleteKgNode(id);
|
|
3212
|
+
logger5.debug({ id }, "Deleted node");
|
|
3213
|
+
}
|
|
3214
|
+
addEdge(sourceId, targetId, edgeType, weight = 1, metadata = {}) {
|
|
3215
|
+
const id = this.storage.saveKgEdge({ sourceId, targetId, edgeType, weight, metadata: JSON.stringify(metadata) });
|
|
3216
|
+
logger5.debug({ id, sourceId, targetId, edgeType }, "Added edge");
|
|
3217
|
+
return id;
|
|
3218
|
+
}
|
|
3219
|
+
getEdgesFrom(nodeId, edgeType) {
|
|
3220
|
+
return this.storage.getKgEdgesFrom(nodeId, edgeType).map(toEdge);
|
|
3221
|
+
}
|
|
3222
|
+
getEdgesTo(nodeId, edgeType) {
|
|
3223
|
+
return this.storage.getKgEdgesTo(nodeId, edgeType).map(toEdge);
|
|
3224
|
+
}
|
|
3225
|
+
getAllEdges() {
|
|
3226
|
+
return this.storage.getAllKgEdges().map(toEdge);
|
|
3227
|
+
}
|
|
3228
|
+
neighbors(nodeId) {
|
|
3229
|
+
const outgoing = this.storage.getKgEdgesFrom(nodeId);
|
|
3230
|
+
const incoming = this.storage.getKgEdgesTo(nodeId);
|
|
3231
|
+
const neighborIds = /* @__PURE__ */ new Set();
|
|
3232
|
+
for (const e of outgoing) neighborIds.add(e.targetId);
|
|
3233
|
+
for (const e of incoming) neighborIds.add(e.sourceId);
|
|
3234
|
+
const nodes = [];
|
|
3235
|
+
for (const nid of neighborIds) {
|
|
3236
|
+
const raw = this.storage.getKgNode(nid);
|
|
3237
|
+
if (raw) nodes.push(toNode(raw));
|
|
3238
|
+
}
|
|
3239
|
+
return nodes;
|
|
3240
|
+
}
|
|
3241
|
+
getSummary() {
|
|
3242
|
+
return this.storage.getKgSummary();
|
|
3243
|
+
}
|
|
3244
|
+
};
|
|
3245
|
+
|
|
3246
|
+
// src/knowledge/graph-populator.ts
|
|
3247
|
+
var logger6 = createLogger({ name: "graph-populator" });
|
|
3248
|
+
var HIGH_SEVERITY = /* @__PURE__ */ new Set(["high", "critical"]);
|
|
3249
|
+
var eventCounter = 0;
|
|
3250
|
+
var insightCounter = 0;
|
|
3251
|
+
var GraphPopulator = class {
|
|
3252
|
+
store;
|
|
3253
|
+
constructor(store) {
|
|
3254
|
+
this.store = store;
|
|
3255
|
+
}
|
|
3256
|
+
populateFromSchema(tables) {
|
|
3257
|
+
for (const table of tables) {
|
|
3258
|
+
this.store.upsertNode("entity", table.tableName, table.tableName, {
|
|
3259
|
+
tableSchema: table.tableSchema,
|
|
3260
|
+
rowCount: table.rowCount
|
|
3261
|
+
});
|
|
3262
|
+
}
|
|
3263
|
+
for (const table of tables) {
|
|
3264
|
+
for (const fk of table.foreignKeys) {
|
|
3265
|
+
const sourceNode = this.store.getNodeByRef("entity", fk.referencedTable);
|
|
3266
|
+
const targetNode = this.store.getNodeByRef("entity", table.tableName);
|
|
3267
|
+
if (sourceNode && targetNode && sourceNode.id !== targetNode.id) {
|
|
3268
|
+
this.store.addEdge(sourceNode.id, targetNode.id, "contains", 1, {
|
|
3269
|
+
sourceColumn: fk.referencedColumn,
|
|
3270
|
+
targetColumn: fk.columnName
|
|
3271
|
+
});
|
|
3272
|
+
}
|
|
3273
|
+
}
|
|
3274
|
+
}
|
|
3275
|
+
logger6.info({ tables: tables.length }, "Graph populated from schema");
|
|
3276
|
+
}
|
|
3277
|
+
onEvent(entity, eventType, primaryKey, data) {
|
|
3278
|
+
const entityNode = this.store.getNodeByRef("entity", entity);
|
|
3279
|
+
if (!entityNode) return;
|
|
3280
|
+
eventCounter++;
|
|
3281
|
+
const refId = `event:${Date.now()}-${eventCounter}`;
|
|
3282
|
+
const label = `${entity} ${eventType}`;
|
|
3283
|
+
const eventNodeId = this.store.upsertNode("event", refId, label, {
|
|
3284
|
+
eventType,
|
|
3285
|
+
primaryKey,
|
|
3286
|
+
...data
|
|
3287
|
+
});
|
|
3288
|
+
this.store.addEdge(eventNodeId, entityNode.id, "detected_on");
|
|
3289
|
+
}
|
|
3290
|
+
onInsight(insight) {
|
|
3291
|
+
const entityNode = this.store.getNodeByRef("entity", insight.entity);
|
|
3292
|
+
if (!entityNode) return;
|
|
3293
|
+
insightCounter++;
|
|
3294
|
+
const nodeType = HIGH_SEVERITY.has(insight.severity) ? "anomaly" : "insight";
|
|
3295
|
+
const refId = `${nodeType}:${Date.now()}-${insightCounter}`;
|
|
3296
|
+
const nodeId = this.store.upsertNode(nodeType, refId, insight.message, {
|
|
3297
|
+
insightType: insight.insightType,
|
|
3298
|
+
severity: insight.severity,
|
|
3299
|
+
...insight.context
|
|
3300
|
+
});
|
|
3301
|
+
this.store.addEdge(nodeId, entityNode.id, "detected_on");
|
|
3302
|
+
}
|
|
3303
|
+
addCorrelation(entityA, entityB, weight = 1) {
|
|
3304
|
+
const nodeA = this.store.getNodeByRef("entity", entityA);
|
|
3305
|
+
const nodeB = this.store.getNodeByRef("entity", entityB);
|
|
3306
|
+
if (!nodeA || !nodeB) return;
|
|
3307
|
+
this.store.addEdge(nodeA.id, nodeB.id, "correlates", weight);
|
|
3308
|
+
}
|
|
3309
|
+
};
|
|
3310
|
+
|
|
3311
|
+
// src/actions/action-registry.ts
|
|
3312
|
+
var ActionRegistry = class {
|
|
3313
|
+
handlers = /* @__PURE__ */ new Map();
|
|
3314
|
+
governance;
|
|
3315
|
+
constructor(governance) {
|
|
3316
|
+
this.governance = governance ?? {};
|
|
3317
|
+
}
|
|
3318
|
+
register(name, handler, description) {
|
|
3319
|
+
if (this.handlers.has(name)) {
|
|
3320
|
+
throw new Error(`Action handler "${name}" is already registered`);
|
|
3321
|
+
}
|
|
3322
|
+
this.handlers.set(name, { name, handler, description });
|
|
3323
|
+
}
|
|
3324
|
+
getHandler(name) {
|
|
3325
|
+
return this.handlers.get(name);
|
|
3326
|
+
}
|
|
3327
|
+
has(name) {
|
|
3328
|
+
return this.handlers.has(name);
|
|
3329
|
+
}
|
|
3330
|
+
getNames() {
|
|
3331
|
+
return Array.from(this.handlers.keys());
|
|
3332
|
+
}
|
|
3333
|
+
getGovernance(actionName) {
|
|
3334
|
+
const level = this.governance[actionName];
|
|
3335
|
+
if (level === "auto_approve" || level === "require_approval" || level === "never_automate") {
|
|
3336
|
+
return level;
|
|
3337
|
+
}
|
|
3338
|
+
return "require_approval";
|
|
3339
|
+
}
|
|
3340
|
+
};
|
|
3341
|
+
|
|
3342
|
+
// src/actions/action-matcher.ts
|
|
3343
|
+
var DEFAULT_CONFIDENCE = 0.5;
|
|
3344
|
+
var ActionMatcher = class {
|
|
3345
|
+
rules;
|
|
3346
|
+
constructor(rules) {
|
|
3347
|
+
this.rules = rules;
|
|
3348
|
+
}
|
|
3349
|
+
match(insight, confidence) {
|
|
3350
|
+
const effectiveConfidence = confidence ?? DEFAULT_CONFIDENCE;
|
|
3351
|
+
const matched = [];
|
|
3352
|
+
for (const rule of this.rules) {
|
|
3353
|
+
const threshold = rule.confidence ?? DEFAULT_CONFIDENCE;
|
|
3354
|
+
if (effectiveConfidence < threshold) {
|
|
3355
|
+
continue;
|
|
3356
|
+
}
|
|
3357
|
+
if (!this.matchesConditions(rule, insight)) {
|
|
3358
|
+
continue;
|
|
3359
|
+
}
|
|
3360
|
+
matched.push(rule);
|
|
3361
|
+
}
|
|
3362
|
+
return matched;
|
|
3363
|
+
}
|
|
3364
|
+
matchesConditions(rule, insight) {
|
|
3365
|
+
const { when } = rule;
|
|
3366
|
+
if (when.severity && when.severity.length > 0) {
|
|
3367
|
+
if (!when.severity.includes(insight.severity)) {
|
|
3368
|
+
return false;
|
|
3369
|
+
}
|
|
3370
|
+
}
|
|
3371
|
+
if (when.insightType && when.insightType.length > 0) {
|
|
3372
|
+
if (!when.insightType.includes(insight.insightType)) {
|
|
3373
|
+
return false;
|
|
3374
|
+
}
|
|
3375
|
+
}
|
|
3376
|
+
if (when.entity && when.entity.length > 0) {
|
|
3377
|
+
if (!when.entity.includes(insight.entity)) {
|
|
3378
|
+
return false;
|
|
3379
|
+
}
|
|
3380
|
+
}
|
|
3381
|
+
return true;
|
|
3382
|
+
}
|
|
3383
|
+
};
|
|
3384
|
+
|
|
3385
|
+
// src/actions/action-executor.ts
|
|
3386
|
+
import { EventEmitter } from "events";
|
|
3387
|
+
var logger7 = createLogger({ name: "action-executor" });
|
|
3388
|
+
var ActionExecutor = class extends EventEmitter {
|
|
3389
|
+
registry;
|
|
3390
|
+
matcher;
|
|
3391
|
+
storage;
|
|
3392
|
+
constructor(registry, matcher, storage) {
|
|
3393
|
+
super();
|
|
3394
|
+
this.registry = registry;
|
|
3395
|
+
this.matcher = matcher;
|
|
3396
|
+
this.storage = storage;
|
|
3397
|
+
}
|
|
3398
|
+
async onInsight(insight) {
|
|
3399
|
+
const confidence = this.computeConfidence(insight);
|
|
3400
|
+
const matchedRules = this.matcher.match(insight, confidence);
|
|
3401
|
+
for (const rule of matchedRules) {
|
|
3402
|
+
await this.processRule(rule, insight, confidence);
|
|
3403
|
+
}
|
|
3404
|
+
}
|
|
3405
|
+
async processRule(rule, insight, confidence) {
|
|
3406
|
+
const governance = this.registry.getGovernance(rule.action);
|
|
3407
|
+
const handler = this.registry.getHandler(rule.action);
|
|
3408
|
+
let status = "pending";
|
|
3409
|
+
if (governance === "never_automate") {
|
|
3410
|
+
status = "blocked";
|
|
3411
|
+
} else if (governance === "auto_approve" && handler) {
|
|
3412
|
+
status = "executed";
|
|
3413
|
+
}
|
|
3414
|
+
const insightId = `insight:${insight.entity}:${insight.insightType}:${Date.now()}`;
|
|
3415
|
+
const recData = {
|
|
3416
|
+
action: rule.action,
|
|
3417
|
+
reason: insight.message,
|
|
3418
|
+
confidence,
|
|
3419
|
+
status: status === "executed" ? "executed" : status,
|
|
3420
|
+
governance,
|
|
3421
|
+
insightId,
|
|
3422
|
+
entity: insight.entity,
|
|
3423
|
+
context: JSON.stringify(insight.context)
|
|
3424
|
+
};
|
|
3425
|
+
const recId = this.storage.saveRecommendation(recData);
|
|
3426
|
+
const emitRec = {
|
|
3427
|
+
id: recId,
|
|
3428
|
+
action: rule.action,
|
|
3429
|
+
reason: insight.message,
|
|
3430
|
+
confidence,
|
|
3431
|
+
status,
|
|
3432
|
+
governance,
|
|
3433
|
+
insightId,
|
|
3434
|
+
entity: insight.entity,
|
|
3435
|
+
context: insight.context
|
|
3436
|
+
};
|
|
3437
|
+
if (status === "executed" && handler) {
|
|
3438
|
+
try {
|
|
3439
|
+
const actionContext = {
|
|
3440
|
+
insight,
|
|
3441
|
+
entity: insight.entity,
|
|
3442
|
+
recommendation: {
|
|
3443
|
+
id: recId,
|
|
3444
|
+
action: rule.action,
|
|
3445
|
+
reason: insight.message,
|
|
3446
|
+
confidence,
|
|
3447
|
+
status: "executed",
|
|
3448
|
+
governance,
|
|
3449
|
+
insightId,
|
|
3450
|
+
entity: insight.entity,
|
|
3451
|
+
context: insight.context,
|
|
3452
|
+
createdAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
3453
|
+
resolvedAt: null,
|
|
3454
|
+
resolvedBy: null
|
|
3455
|
+
}
|
|
3456
|
+
};
|
|
3457
|
+
await handler.handler(actionContext);
|
|
3458
|
+
this.emit("action:executed", emitRec);
|
|
3459
|
+
logger7.info({ action: rule.action, entity: insight.entity }, "Action executed");
|
|
3460
|
+
} catch (error) {
|
|
3461
|
+
this.storage.updateRecommendationStatus(recId, "failed", "auto");
|
|
3462
|
+
emitRec.status = "failed";
|
|
3463
|
+
this.emit("action:failed", { ...emitRec, error: error.message });
|
|
3464
|
+
logger7.error({ action: rule.action, error: error.message }, "Action failed");
|
|
3465
|
+
}
|
|
3466
|
+
}
|
|
3467
|
+
this.emit("recommendation", emitRec);
|
|
3468
|
+
}
|
|
3469
|
+
computeConfidence(insight) {
|
|
3470
|
+
const severityWeights = {
|
|
3471
|
+
critical: 0.95,
|
|
3472
|
+
high: 0.8,
|
|
3473
|
+
medium: 0.6,
|
|
3474
|
+
low: 0.4
|
|
3475
|
+
};
|
|
3476
|
+
return severityWeights[insight.severity] ?? 0.5;
|
|
3477
|
+
}
|
|
3478
|
+
};
|
|
3479
|
+
|
|
3480
|
+
// src/behavioral/watcher.ts
|
|
3481
|
+
var Watcher = class {
|
|
3482
|
+
config;
|
|
3483
|
+
detector;
|
|
3484
|
+
classifier;
|
|
3485
|
+
logger = createLogger({ name: "watcher" });
|
|
3486
|
+
patternEngine = null;
|
|
3487
|
+
transitionTracker = null;
|
|
3488
|
+
analyticsEngine = null;
|
|
3489
|
+
graphPopulator = null;
|
|
3490
|
+
actionExecutor = null;
|
|
3491
|
+
stuckInterval = null;
|
|
3492
|
+
analyticsInterval = null;
|
|
3493
|
+
interval = null;
|
|
3494
|
+
stream = null;
|
|
3495
|
+
_streaming = false;
|
|
3496
|
+
watchConfigs = [];
|
|
3497
|
+
constructor(config) {
|
|
3498
|
+
this.config = config;
|
|
3499
|
+
this.detector = new ChangeDetector(config.connector, config.dbType);
|
|
3500
|
+
this.classifier = new EventClassifier(config.storage);
|
|
3501
|
+
if (config.workflows && Object.keys(config.workflows).length > 0) {
|
|
3502
|
+
this.transitionTracker = new TransitionTracker(
|
|
3503
|
+
config.workflows,
|
|
3504
|
+
config.storage,
|
|
3505
|
+
(insight) => {
|
|
3506
|
+
config.onInsight?.(insight);
|
|
3507
|
+
this.graphPopulator?.onInsight(insight);
|
|
3508
|
+
this.actionExecutor?.onInsight(insight);
|
|
3509
|
+
}
|
|
3510
|
+
);
|
|
3511
|
+
}
|
|
3512
|
+
const hasAnalytics = config.analytics && (Object.keys(config.analytics.correlations ?? {}).length > 0 || Object.keys(config.analytics.distributions ?? {}).length > 0);
|
|
3513
|
+
if (hasAnalytics) {
|
|
3514
|
+
const analyticsStorage = {
|
|
3515
|
+
saveCorrelationEvent: config.storage.saveCorrelationEvent.bind(config.storage),
|
|
3516
|
+
getCorrelationEvents: config.storage.getCorrelationEvents.bind(config.storage),
|
|
3517
|
+
saveCorrelationRate: config.storage.saveCorrelationRate.bind(config.storage),
|
|
3518
|
+
getCorrelationRates: config.storage.getCorrelationRates.bind(config.storage),
|
|
3519
|
+
saveHistogramBucket: config.storage.saveHistogramBucket.bind(config.storage),
|
|
3520
|
+
getHistogramBuckets: config.storage.getHistogramBuckets.bind(config.storage),
|
|
3521
|
+
saveTemporalPattern: config.storage.saveTemporalPattern.bind(config.storage),
|
|
3522
|
+
getTemporalPatterns: config.storage.getTemporalPatterns.bind(config.storage),
|
|
3523
|
+
saveInsight: config.storage.saveInsight.bind(config.storage),
|
|
3524
|
+
getBaseline: config.storage.getAnalyticsBaseline.bind(config.storage),
|
|
3525
|
+
saveBaseline: config.storage.saveAnalyticsBaseline.bind(config.storage)
|
|
3526
|
+
};
|
|
3527
|
+
this.analyticsEngine = new AnalyticsEngine(
|
|
3528
|
+
config.analytics,
|
|
3529
|
+
analyticsStorage,
|
|
3530
|
+
(insight) => {
|
|
3531
|
+
config.onInsight?.(insight);
|
|
3532
|
+
this.graphPopulator?.onInsight(insight);
|
|
3533
|
+
this.actionExecutor?.onInsight(insight);
|
|
3534
|
+
}
|
|
3535
|
+
);
|
|
3536
|
+
}
|
|
3537
|
+
if (config.knowledge?.enabled) {
|
|
3538
|
+
const graphStore = new GraphStore({
|
|
3539
|
+
saveKgNode: config.storage.saveKgNode.bind(config.storage),
|
|
3540
|
+
getKgNode: config.storage.getKgNode.bind(config.storage),
|
|
3541
|
+
getKgNodeByRef: config.storage.getKgNodeByRef.bind(config.storage),
|
|
3542
|
+
getKgNodesByType: config.storage.getKgNodesByType.bind(config.storage),
|
|
3543
|
+
getAllKgNodes: config.storage.getAllKgNodes.bind(config.storage),
|
|
3544
|
+
deleteKgNode: config.storage.deleteKgNode.bind(config.storage),
|
|
3545
|
+
saveKgEdge: config.storage.saveKgEdge.bind(config.storage),
|
|
3546
|
+
getKgEdgesFrom: config.storage.getKgEdgesFrom.bind(config.storage),
|
|
3547
|
+
getKgEdgesTo: config.storage.getKgEdgesTo.bind(config.storage),
|
|
3548
|
+
getAllKgEdges: config.storage.getAllKgEdges.bind(config.storage),
|
|
3549
|
+
getKgSummary: config.storage.getKgSummary.bind(config.storage)
|
|
3550
|
+
});
|
|
3551
|
+
this.graphPopulator = new GraphPopulator(graphStore);
|
|
3552
|
+
this.graphPopulator.populateFromSchema(config.tables);
|
|
3553
|
+
}
|
|
3554
|
+
if (config.actions?.enabled && config.actions.rules && config.actions.rules.length > 0) {
|
|
3555
|
+
const actionRegistry = config.actionRegistry ?? new ActionRegistry(config.actions.governance);
|
|
3556
|
+
const actionMatcher = new ActionMatcher(config.actions.rules);
|
|
3557
|
+
const actionStorage = {
|
|
3558
|
+
saveRecommendation: config.storage.saveRecommendation.bind(config.storage),
|
|
3559
|
+
getRecommendation: config.storage.getRecommendation.bind(config.storage),
|
|
3560
|
+
updateRecommendationStatus: config.storage.updateRecommendationStatus.bind(config.storage)
|
|
3561
|
+
};
|
|
3562
|
+
this.actionExecutor = new ActionExecutor(actionRegistry, actionMatcher, actionStorage);
|
|
3563
|
+
this.actionExecutor.on("recommendation", (rec) => {
|
|
3564
|
+
config.onRecommendation?.(rec);
|
|
3565
|
+
});
|
|
3566
|
+
this.actionExecutor.on("action:executed", (rec) => {
|
|
3567
|
+
config.onActionExecuted?.(rec);
|
|
3568
|
+
});
|
|
3569
|
+
this.actionExecutor.on("action:failed", (rec) => {
|
|
3570
|
+
config.onActionFailed?.(rec);
|
|
3571
|
+
});
|
|
3572
|
+
}
|
|
3573
|
+
}
|
|
3574
|
+
async start(options) {
|
|
3575
|
+
if (this.interval || this.stream) throw new Error("Watcher already running");
|
|
3576
|
+
this.patternEngine = new PatternEngine(this.config.storage, {
|
|
3577
|
+
anomalyThreshold: options?.anomalyThreshold
|
|
3578
|
+
});
|
|
3579
|
+
if (options?.once) {
|
|
3580
|
+
this.watchConfigs = ChangeDetector.buildWatchConfigs(
|
|
3581
|
+
this.config.tables,
|
|
3582
|
+
options?.excludeTables
|
|
3583
|
+
);
|
|
3584
|
+
for (const wc of this.watchConfigs) {
|
|
3585
|
+
const saved = this.config.storage.getWatermark(wc.tableName, wc.tableSchema);
|
|
3586
|
+
if (saved?.lastSeenValue) {
|
|
3587
|
+
wc.lastSeenValue = saved.lastSeenValue;
|
|
3588
|
+
}
|
|
3589
|
+
}
|
|
3590
|
+
await this.pollCycle(options?.excludeColumns);
|
|
3591
|
+
return;
|
|
3592
|
+
}
|
|
3593
|
+
if (!options?.preferPolling && this.config.connectionConfig) {
|
|
3594
|
+
const started = await this.tryStartStreaming(options);
|
|
3595
|
+
if (started) {
|
|
3596
|
+
this.startStuckCheckInterval(options);
|
|
3597
|
+
return;
|
|
3598
|
+
}
|
|
3599
|
+
}
|
|
3600
|
+
this.watchConfigs = ChangeDetector.buildWatchConfigs(
|
|
3601
|
+
this.config.tables,
|
|
3602
|
+
options?.excludeTables
|
|
3603
|
+
);
|
|
3604
|
+
for (const config of this.watchConfigs) {
|
|
3605
|
+
const saved = this.config.storage.getWatermark(config.tableName, config.tableSchema);
|
|
3606
|
+
if (saved?.lastSeenValue) {
|
|
3607
|
+
config.lastSeenValue = saved.lastSeenValue;
|
|
3608
|
+
}
|
|
3609
|
+
}
|
|
3610
|
+
const excludeColumns = options?.excludeColumns;
|
|
3611
|
+
const intervalMs = options?.interval ?? 5e3;
|
|
3612
|
+
this.interval = setInterval(async () => {
|
|
3613
|
+
await this.pollCycle(excludeColumns);
|
|
3614
|
+
}, intervalMs);
|
|
3615
|
+
this.startStuckCheckInterval(options);
|
|
3616
|
+
}
|
|
3617
|
+
stop() {
|
|
3618
|
+
if (this.interval) {
|
|
3619
|
+
clearInterval(this.interval);
|
|
3620
|
+
this.interval = null;
|
|
3621
|
+
}
|
|
3622
|
+
if (this.stuckInterval) {
|
|
3623
|
+
clearInterval(this.stuckInterval);
|
|
3624
|
+
this.stuckInterval = null;
|
|
3625
|
+
}
|
|
3626
|
+
if (this.analyticsInterval) {
|
|
3627
|
+
clearInterval(this.analyticsInterval);
|
|
3628
|
+
this.analyticsInterval = null;
|
|
3629
|
+
}
|
|
3630
|
+
if (this.stream) {
|
|
3631
|
+
this.stream.stop().catch((err) => {
|
|
3632
|
+
this.logger.error({ error: err.message }, "Error stopping stream");
|
|
3633
|
+
});
|
|
3634
|
+
this.stream = null;
|
|
3635
|
+
}
|
|
3636
|
+
this._streaming = false;
|
|
3637
|
+
}
|
|
3638
|
+
async cleanup() {
|
|
3639
|
+
const stream = this.stream;
|
|
3640
|
+
this.stop();
|
|
3641
|
+
if (stream) {
|
|
3642
|
+
await stream.cleanup();
|
|
3643
|
+
}
|
|
3644
|
+
}
|
|
3645
|
+
isRunning() {
|
|
3646
|
+
return this.interval !== null || this.stream !== null && this.stream.isRunning();
|
|
3647
|
+
}
|
|
3648
|
+
isStreaming() {
|
|
3649
|
+
return this._streaming;
|
|
3650
|
+
}
|
|
3651
|
+
startStuckCheckInterval(options) {
|
|
3652
|
+
if (this.transitionTracker) {
|
|
3653
|
+
this.stuckInterval = setInterval(() => {
|
|
3654
|
+
this.transitionTracker?.checkStuck();
|
|
3655
|
+
}, options?.stuckCheckInterval ?? 6e4);
|
|
3656
|
+
}
|
|
3657
|
+
if (this.analyticsEngine) {
|
|
3658
|
+
this.analyticsInterval = setInterval(() => {
|
|
3659
|
+
this.analyticsEngine?.runPeriodicChecks();
|
|
3660
|
+
}, options?.stuckCheckInterval ?? 6e4);
|
|
3661
|
+
}
|
|
3662
|
+
}
|
|
3663
|
+
async tryStartStreaming(options) {
|
|
3664
|
+
const capDetector = new CapabilityDetector(this.config.connector, this.config.dbType);
|
|
3665
|
+
const capability = await capDetector.detect();
|
|
3666
|
+
if (!capability.canStream) {
|
|
3667
|
+
this.logger.info({ reason: capability.reason }, "Streaming not available, will use polling");
|
|
3668
|
+
return false;
|
|
3669
|
+
}
|
|
3670
|
+
try {
|
|
3671
|
+
const stream = await this.createStream();
|
|
3672
|
+
if (!stream) {
|
|
3673
|
+
this.logger.info("Streaming package not installed. Falling back to polling.");
|
|
3674
|
+
return false;
|
|
3675
|
+
}
|
|
3676
|
+
await stream.start((change) => {
|
|
3677
|
+
this.handleChange(change, options?.excludeColumns);
|
|
3678
|
+
});
|
|
3679
|
+
this.stream = stream;
|
|
3680
|
+
this._streaming = true;
|
|
3681
|
+
this.logger.info("Using streaming mode for change detection");
|
|
3682
|
+
return true;
|
|
3683
|
+
} catch (error) {
|
|
3684
|
+
this.logger.error({ error: error.message }, "Failed to start streaming, falling back to polling");
|
|
3685
|
+
return false;
|
|
3686
|
+
}
|
|
3687
|
+
}
|
|
3688
|
+
async createStream() {
|
|
3689
|
+
if (this.config.dbType === "postgres" || this.config.dbType === "cockroachdb") {
|
|
3690
|
+
try {
|
|
3691
|
+
const { PostgresStream } = await import("./postgres-stream-A7EVYUX2.js");
|
|
3692
|
+
return new PostgresStream(this.config.connectionConfig, this.config.connector);
|
|
3693
|
+
} catch {
|
|
3694
|
+
return null;
|
|
3695
|
+
}
|
|
3696
|
+
}
|
|
3697
|
+
if (this.config.dbType === "mysql" || this.config.dbType === "mariadb") {
|
|
3698
|
+
try {
|
|
3699
|
+
const { MySQLStream } = await import("./mysql-stream-YCNLPPPG.js");
|
|
3700
|
+
return new MySQLStream(this.config.connectionConfig, this.config.connector);
|
|
3701
|
+
} catch {
|
|
3702
|
+
return null;
|
|
3703
|
+
}
|
|
3704
|
+
}
|
|
3705
|
+
if (this.config.dbType === "mongodb") {
|
|
3706
|
+
try {
|
|
3707
|
+
const { MongoDBStream } = await import("./mongodb-stream-Q23UHLTM.js");
|
|
3708
|
+
return new MongoDBStream(this.config.connectionConfig);
|
|
3709
|
+
} catch {
|
|
3710
|
+
return null;
|
|
3711
|
+
}
|
|
3712
|
+
}
|
|
3713
|
+
return null;
|
|
3714
|
+
}
|
|
3715
|
+
handleChange(change, excludeColumns) {
|
|
3716
|
+
const event = this.classifier.classify(change, { excludeColumns });
|
|
3717
|
+
this.config.storage.saveEvent({
|
|
3718
|
+
eventType: `${event.entity} ${event.type}`,
|
|
3719
|
+
entity: event.entity,
|
|
3720
|
+
tableName: event.table,
|
|
3721
|
+
operation: change.operation,
|
|
3722
|
+
rowId: String(event.primaryKey),
|
|
3723
|
+
rowData: JSON.stringify(event.metadata)
|
|
3724
|
+
});
|
|
3725
|
+
this.config.onEvent?.(event);
|
|
3726
|
+
if (this.patternEngine) {
|
|
3727
|
+
this.patternEngine.updateBaselines([event.entity], change.operation);
|
|
3728
|
+
const anomalies = this.patternEngine.checkRateAnomalies([event.entity], change.operation);
|
|
3729
|
+
for (const anomaly of anomalies) {
|
|
3730
|
+
this.config.onAnomaly?.(anomaly);
|
|
3731
|
+
}
|
|
3732
|
+
}
|
|
3733
|
+
if (this.transitionTracker && change.operation === "UPDATE") {
|
|
3734
|
+
this.transitionTracker.handleChange(change);
|
|
3735
|
+
}
|
|
3736
|
+
if (this.analyticsEngine) {
|
|
3737
|
+
this.analyticsEngine.handleEvent(
|
|
3738
|
+
event.entity,
|
|
3739
|
+
event.type,
|
|
3740
|
+
String(event.primaryKey),
|
|
3741
|
+
event.metadata,
|
|
3742
|
+
change.detectedAt
|
|
3743
|
+
);
|
|
3744
|
+
}
|
|
3745
|
+
if (this.graphPopulator) {
|
|
3746
|
+
this.graphPopulator.onEvent(
|
|
3747
|
+
event.entity,
|
|
3748
|
+
event.type,
|
|
3749
|
+
String(event.primaryKey),
|
|
3750
|
+
event.metadata
|
|
3751
|
+
);
|
|
3752
|
+
}
|
|
3753
|
+
return event.entity;
|
|
3754
|
+
}
|
|
3755
|
+
async pollCycle(excludeColumns) {
|
|
3756
|
+
const allEntities = /* @__PURE__ */ new Set();
|
|
3757
|
+
for (const config of this.watchConfigs) {
|
|
3758
|
+
try {
|
|
3759
|
+
const changes = await this.detector.poll(config);
|
|
3760
|
+
for (const change of changes) {
|
|
3761
|
+
const entity = this.handleChange(change, excludeColumns);
|
|
3762
|
+
allEntities.add(entity);
|
|
3763
|
+
}
|
|
3764
|
+
if (changes.length > 0) {
|
|
3765
|
+
const lastChange = changes[changes.length - 1];
|
|
3766
|
+
const lastValue = config.strategy === "timestamp" ? String(lastChange.newData?.[config.timestampColumn] ?? "") : String(lastChange.primaryKey);
|
|
3767
|
+
config.lastSeenValue = lastValue;
|
|
3768
|
+
}
|
|
3769
|
+
this.config.storage.saveWatermark({
|
|
3770
|
+
tableName: config.tableName,
|
|
3771
|
+
tableSchema: config.tableSchema,
|
|
3772
|
+
strategy: config.strategy,
|
|
3773
|
+
timestampColumn: config.timestampColumn,
|
|
3774
|
+
pkColumn: config.primaryKeyColumn,
|
|
3775
|
+
lastSeenValue: config.lastSeenValue
|
|
3776
|
+
});
|
|
3777
|
+
} catch (error) {
|
|
3778
|
+
this.logger.error({ table: config.tableName, error: error.message }, "Poll error");
|
|
3779
|
+
}
|
|
3780
|
+
}
|
|
3781
|
+
}
|
|
3782
|
+
};
|
|
3783
|
+
|
|
3784
|
+
// src/knowledge/traversal.ts
|
|
3785
|
+
var DEFAULT_MAX_DEPTH = 5;
|
|
3786
|
+
var Traversal = class {
|
|
3787
|
+
store;
|
|
3788
|
+
constructor(store) {
|
|
3789
|
+
this.store = store;
|
|
3790
|
+
}
|
|
3791
|
+
causesOf(nodeId, maxDepth = DEFAULT_MAX_DEPTH) {
|
|
3792
|
+
return this.bfs(nodeId, "backward", maxDepth);
|
|
3793
|
+
}
|
|
3794
|
+
impactOf(nodeId, maxDepth = DEFAULT_MAX_DEPTH) {
|
|
3795
|
+
return this.bfs(nodeId, "forward", maxDepth);
|
|
3796
|
+
}
|
|
3797
|
+
timeline(nodeId) {
|
|
3798
|
+
const neighbors = this.store.neighbors(nodeId);
|
|
3799
|
+
return neighbors.sort((a, b) => {
|
|
3800
|
+
if (a.timestamp > b.timestamp) return -1;
|
|
3801
|
+
if (a.timestamp < b.timestamp) return 1;
|
|
3802
|
+
return 0;
|
|
3803
|
+
});
|
|
3804
|
+
}
|
|
3805
|
+
bfs(startId, direction, maxDepth) {
|
|
3806
|
+
const visited = /* @__PURE__ */ new Set();
|
|
3807
|
+
visited.add(startId);
|
|
3808
|
+
const result = [];
|
|
3809
|
+
let currentLevel = [startId];
|
|
3810
|
+
for (let depth = 0; depth < maxDepth && currentLevel.length > 0; depth++) {
|
|
3811
|
+
const nextLevel = [];
|
|
3812
|
+
for (const nodeId of currentLevel) {
|
|
3813
|
+
const edges = direction === "forward" ? this.store.getEdgesFrom(nodeId, "causes") : this.store.getEdgesTo(nodeId, "causes");
|
|
3814
|
+
for (const edge of edges) {
|
|
3815
|
+
const neighborId = direction === "forward" ? edge.targetId : edge.sourceId;
|
|
3816
|
+
if (!visited.has(neighborId)) {
|
|
3817
|
+
visited.add(neighborId);
|
|
3818
|
+
const node = this.store.getNode(neighborId);
|
|
3819
|
+
if (node) {
|
|
3820
|
+
result.push(node);
|
|
3821
|
+
nextLevel.push(neighborId);
|
|
3822
|
+
}
|
|
3823
|
+
}
|
|
3824
|
+
}
|
|
3825
|
+
}
|
|
3826
|
+
currentLevel = nextLevel;
|
|
3827
|
+
}
|
|
3828
|
+
return result;
|
|
3829
|
+
}
|
|
3830
|
+
};
|
|
3831
|
+
|
|
3832
|
+
// src/knowledge/entity-intelligence.ts
|
|
3833
|
+
var MAX_RECENT_INSIGHTS = 10;
|
|
3834
|
+
var EntityIntelligence = class {
|
|
3835
|
+
store;
|
|
3836
|
+
constructor(store) {
|
|
3837
|
+
this.store = store;
|
|
3838
|
+
}
|
|
3839
|
+
forEntity(refId) {
|
|
3840
|
+
const entityNode = this.store.getNodeByRef("entity", refId);
|
|
3841
|
+
if (!entityNode) return void 0;
|
|
3842
|
+
const containsEdges = this.store.getEdgesFrom(entityNode.id, "contains");
|
|
3843
|
+
const tableCount = containsEdges.length;
|
|
3844
|
+
const incomingEdges = this.store.getEdgesTo(entityNode.id, "detected_on");
|
|
3845
|
+
const connectedNodeIds = incomingEdges.map((e) => e.sourceId);
|
|
3846
|
+
let totalEvents = 0;
|
|
3847
|
+
let anomalyCount = 0;
|
|
3848
|
+
const recentInsights = [];
|
|
3849
|
+
for (const nid of connectedNodeIds) {
|
|
3850
|
+
const node = this.store.getNode(nid);
|
|
3851
|
+
if (!node) continue;
|
|
3852
|
+
if (node.nodeType === "event") totalEvents++;
|
|
3853
|
+
if (node.nodeType === "anomaly") anomalyCount++;
|
|
3854
|
+
if (node.nodeType === "insight") {
|
|
3855
|
+
recentInsights.push(node);
|
|
3856
|
+
}
|
|
3857
|
+
}
|
|
3858
|
+
recentInsights.sort((a, b) => a.timestamp > b.timestamp ? -1 : 1);
|
|
3859
|
+
const limitedInsights = recentInsights.slice(0, MAX_RECENT_INSIGHTS);
|
|
3860
|
+
const correlatesEdges = this.store.getEdgesFrom(entityNode.id, "correlates");
|
|
3861
|
+
const topCorrelations = correlatesEdges.map((edge) => {
|
|
3862
|
+
const targetNode = this.store.getNode(edge.targetId);
|
|
3863
|
+
return targetNode ? { entity: targetNode.label, weight: edge.weight } : null;
|
|
3864
|
+
}).filter((c) => c !== null).sort((a, b) => b.weight - a.weight);
|
|
3865
|
+
const riskScore = totalEvents > 0 ? anomalyCount / totalEvents : 0;
|
|
3866
|
+
return {
|
|
3867
|
+
entityName: entityNode.label,
|
|
3868
|
+
tableCount,
|
|
3869
|
+
totalEvents,
|
|
3870
|
+
anomalyCount,
|
|
3871
|
+
topCorrelations,
|
|
3872
|
+
recentInsights: limitedInsights,
|
|
3873
|
+
riskScore
|
|
3874
|
+
};
|
|
3875
|
+
}
|
|
3876
|
+
};
|
|
3877
|
+
|
|
3878
|
+
// src/knowledge/export.ts
|
|
3879
|
+
function exportGraph(store, rootNodeId) {
|
|
3880
|
+
if (rootNodeId === void 0) {
|
|
3881
|
+
return {
|
|
3882
|
+
nodes: store.getAllNodes(),
|
|
3883
|
+
edges: store.getAllEdges()
|
|
3884
|
+
};
|
|
3885
|
+
}
|
|
3886
|
+
const visited = /* @__PURE__ */ new Set();
|
|
3887
|
+
const queue = [rootNodeId];
|
|
3888
|
+
visited.add(rootNodeId);
|
|
3889
|
+
while (queue.length > 0) {
|
|
3890
|
+
const nodeId = queue.shift();
|
|
3891
|
+
const outgoing = store.getEdgesFrom(nodeId);
|
|
3892
|
+
const incoming = store.getEdgesTo(nodeId);
|
|
3893
|
+
for (const edge of [...outgoing, ...incoming]) {
|
|
3894
|
+
const neighborId = edge.sourceId === nodeId ? edge.targetId : edge.sourceId;
|
|
3895
|
+
if (!visited.has(neighborId)) {
|
|
3896
|
+
visited.add(neighborId);
|
|
3897
|
+
queue.push(neighborId);
|
|
3898
|
+
}
|
|
3899
|
+
}
|
|
3900
|
+
}
|
|
3901
|
+
const nodes = [];
|
|
3902
|
+
for (const id of visited) {
|
|
3903
|
+
const node = store.getNode(id);
|
|
3904
|
+
if (node) nodes.push(node);
|
|
3905
|
+
}
|
|
3906
|
+
const allEdges = store.getAllEdges();
|
|
3907
|
+
const edges = allEdges.filter(
|
|
3908
|
+
(e) => visited.has(e.sourceId) && visited.has(e.targetId)
|
|
3909
|
+
);
|
|
3910
|
+
return { nodes, edges };
|
|
3911
|
+
}
|
|
3912
|
+
|
|
3913
|
+
// src/notifications/formatters.ts
|
|
3914
|
+
var SEVERITY_COLORS = {
|
|
3915
|
+
critical: "#dc2626",
|
|
3916
|
+
high: "#ea580c",
|
|
3917
|
+
medium: "#ca8a04",
|
|
3918
|
+
low: "#2563eb"
|
|
3919
|
+
};
|
|
3920
|
+
function formatForSlack(trigger, payload) {
|
|
3921
|
+
const severity = payload.severity ?? "info";
|
|
3922
|
+
const color = SEVERITY_COLORS[severity] ?? "#6b7280";
|
|
3923
|
+
const entity = payload.entity ?? "unknown";
|
|
3924
|
+
const message = payload.message ?? JSON.stringify(payload);
|
|
3925
|
+
const timestamp = payload.timestamp ?? (/* @__PURE__ */ new Date()).toISOString();
|
|
3926
|
+
return {
|
|
3927
|
+
attachments: [
|
|
3928
|
+
{
|
|
3929
|
+
color,
|
|
3930
|
+
blocks: [
|
|
3931
|
+
{
|
|
3932
|
+
type: "section",
|
|
3933
|
+
text: {
|
|
3934
|
+
type: "mrkdwn",
|
|
3935
|
+
text: `*[${trigger.toUpperCase()}]* ${message}`
|
|
3936
|
+
}
|
|
3937
|
+
},
|
|
3938
|
+
{
|
|
3939
|
+
type: "context",
|
|
3940
|
+
elements: [
|
|
3941
|
+
{
|
|
3942
|
+
type: "mrkdwn",
|
|
3943
|
+
text: `Entity: *${entity}* | Severity: *${severity}* | ${timestamp}`
|
|
3944
|
+
}
|
|
3945
|
+
]
|
|
3946
|
+
}
|
|
3947
|
+
]
|
|
3948
|
+
}
|
|
3949
|
+
]
|
|
3950
|
+
};
|
|
3951
|
+
}
|
|
3952
|
+
function formatGeneric(trigger, payload) {
|
|
3953
|
+
return {
|
|
3954
|
+
trigger,
|
|
3955
|
+
timestamp: payload.timestamp ?? (/* @__PURE__ */ new Date()).toISOString(),
|
|
3956
|
+
data: payload
|
|
3957
|
+
};
|
|
3958
|
+
}
|
|
3959
|
+
|
|
3960
|
+
// src/notifications/notifier.ts
|
|
3961
|
+
var Notifier = class {
|
|
3962
|
+
rules;
|
|
3963
|
+
logger = createLogger({ name: "notifier" });
|
|
3964
|
+
constructor(rules) {
|
|
3965
|
+
this.rules = rules;
|
|
3966
|
+
}
|
|
3967
|
+
async notify(trigger, payload) {
|
|
3968
|
+
const matchingRules = this.rules.filter((rule) => {
|
|
3969
|
+
if (!rule.triggers.includes(trigger)) return false;
|
|
3970
|
+
if (rule.filter?.severity && payload.severity) {
|
|
3971
|
+
if (!rule.filter.severity.includes(payload.severity)) {
|
|
3972
|
+
return false;
|
|
3973
|
+
}
|
|
3974
|
+
}
|
|
3975
|
+
if (rule.filter?.entity && payload.entity) {
|
|
3976
|
+
if (!rule.filter.entity.includes(payload.entity)) {
|
|
3977
|
+
return false;
|
|
3978
|
+
}
|
|
3979
|
+
}
|
|
3980
|
+
return true;
|
|
3981
|
+
});
|
|
3982
|
+
const sends = [];
|
|
3983
|
+
for (const rule of matchingRules) {
|
|
3984
|
+
for (const target of rule.targets) {
|
|
3985
|
+
const body = target.type === "slack" ? formatForSlack(trigger, payload) : formatGeneric(trigger, payload);
|
|
3986
|
+
sends.push(this.send(target.url, body, target.headers));
|
|
3987
|
+
}
|
|
3988
|
+
}
|
|
3989
|
+
const results = await Promise.allSettled(sends);
|
|
3990
|
+
for (const result of results) {
|
|
3991
|
+
if (result.status === "rejected") {
|
|
3992
|
+
this.logger.error({ error: result.reason.message }, "Webhook delivery failed");
|
|
3993
|
+
}
|
|
3994
|
+
}
|
|
3995
|
+
}
|
|
3996
|
+
async send(url, body, headers) {
|
|
3997
|
+
const response = await fetch(url, {
|
|
3998
|
+
method: "POST",
|
|
3999
|
+
headers: {
|
|
4000
|
+
"Content-Type": "application/json",
|
|
4001
|
+
...headers
|
|
4002
|
+
},
|
|
4003
|
+
body: JSON.stringify(body)
|
|
4004
|
+
});
|
|
4005
|
+
if (!response.ok) {
|
|
4006
|
+
throw new Error(`Webhook returned ${response.status}: ${await response.text()}`);
|
|
4007
|
+
}
|
|
4008
|
+
}
|
|
4009
|
+
};
|
|
4010
|
+
|
|
4011
|
+
// src/explain/context-builder.ts
|
|
4012
|
+
var DEFAULT_MAX_EVENTS = 50;
|
|
4013
|
+
var DEFAULT_MAX_GRAPH_DEPTH = 3;
|
|
4014
|
+
var ExplainContextBuilder = class {
|
|
4015
|
+
storage;
|
|
4016
|
+
graphStore;
|
|
4017
|
+
traversal;
|
|
4018
|
+
constructor(storage, graphStore, traversal) {
|
|
4019
|
+
this.storage = storage;
|
|
4020
|
+
this.graphStore = graphStore ?? null;
|
|
4021
|
+
this.traversal = traversal ?? null;
|
|
4022
|
+
}
|
|
4023
|
+
buildContext(target, options) {
|
|
4024
|
+
const maxEvents = options?.maxContextEvents ?? DEFAULT_MAX_EVENTS;
|
|
4025
|
+
const maxGraphDepth = options?.maxGraphDepth ?? DEFAULT_MAX_GRAPH_DEPTH;
|
|
4026
|
+
switch (target.type) {
|
|
4027
|
+
case "anomaly":
|
|
4028
|
+
return this.buildAnomalyContext(target.id, maxEvents, maxGraphDepth);
|
|
4029
|
+
case "insight":
|
|
4030
|
+
return this.buildInsightContext(target.id, maxEvents, maxGraphDepth);
|
|
4031
|
+
case "event":
|
|
4032
|
+
return this.buildEventContext(target.id, maxEvents);
|
|
4033
|
+
case "entity":
|
|
4034
|
+
return this.buildEntityContext(target.id, maxEvents, maxGraphDepth);
|
|
4035
|
+
default:
|
|
4036
|
+
throw new Error(`Unknown explain target type: ${target.type}`);
|
|
4037
|
+
}
|
|
4038
|
+
}
|
|
4039
|
+
buildAnomalyContext(id, maxEvents, maxGraphDepth) {
|
|
4040
|
+
const anomaly = this.storage.getAnomalyById(id);
|
|
4041
|
+
if (!anomaly) {
|
|
4042
|
+
throw new Error(`Anomaly with id ${id} not found`);
|
|
4043
|
+
}
|
|
4044
|
+
const entity = anomaly.entity;
|
|
4045
|
+
const targetDescription = `Anomaly: ${anomaly.message} (type: ${anomaly.anomalyType}, severity: ${anomaly.severity}, expected: ${anomaly.expected}, actual: ${anomaly.actual}, at: ${anomaly.timestamp})`;
|
|
4046
|
+
return {
|
|
4047
|
+
targetDescription,
|
|
4048
|
+
entityInfo: this.getEntityInfo(entity),
|
|
4049
|
+
recentEvents: this.getRecentEvents(entity, maxEvents),
|
|
4050
|
+
baselines: this.getBaselines(entity),
|
|
4051
|
+
transitions: this.getTransitions(entity),
|
|
4052
|
+
graphContext: this.getGraphContext(entity, maxGraphDepth),
|
|
4053
|
+
correlations: this.getCorrelations(entity)
|
|
4054
|
+
};
|
|
4055
|
+
}
|
|
4056
|
+
buildInsightContext(id, maxEvents, maxGraphDepth) {
|
|
4057
|
+
const insight = this.storage.getInsightById(id);
|
|
4058
|
+
if (!insight) {
|
|
4059
|
+
throw new Error(`Insight with id ${id} not found`);
|
|
4060
|
+
}
|
|
4061
|
+
const entity = insight.entity;
|
|
4062
|
+
const targetDescription = `Insight: ${insight.message} (type: ${insight.insightType}, severity: ${insight.severity}, at: ${insight.timestamp}, context: ${JSON.stringify(insight.context)})`;
|
|
4063
|
+
return {
|
|
4064
|
+
targetDescription,
|
|
4065
|
+
entityInfo: this.getEntityInfo(entity),
|
|
4066
|
+
recentEvents: this.getRecentEvents(entity, maxEvents),
|
|
4067
|
+
baselines: this.getBaselines(entity),
|
|
4068
|
+
transitions: this.getTransitions(entity),
|
|
4069
|
+
graphContext: this.getGraphContext(entity, maxGraphDepth),
|
|
4070
|
+
correlations: this.getCorrelations(entity)
|
|
4071
|
+
};
|
|
4072
|
+
}
|
|
4073
|
+
buildEventContext(id, maxEvents) {
|
|
4074
|
+
const event = this.storage.getEventById(id);
|
|
4075
|
+
if (!event) {
|
|
4076
|
+
throw new Error(`Event with id ${id} not found`);
|
|
4077
|
+
}
|
|
4078
|
+
const entity = event.entity;
|
|
4079
|
+
const targetDescription = `Event: ${event.eventType} on ${event.tableName} (operation: ${event.operation}, row: ${event.rowId}, at: ${event.timestamp})`;
|
|
4080
|
+
return {
|
|
4081
|
+
targetDescription,
|
|
4082
|
+
entityInfo: this.getEntityInfo(entity),
|
|
4083
|
+
recentEvents: this.getRecentEvents(entity, maxEvents),
|
|
4084
|
+
baselines: this.getBaselines(entity),
|
|
4085
|
+
transitions: this.getTransitions(entity),
|
|
4086
|
+
graphContext: "",
|
|
4087
|
+
correlations: this.getCorrelations(entity)
|
|
4088
|
+
};
|
|
4089
|
+
}
|
|
4090
|
+
buildEntityContext(id, maxEvents, maxGraphDepth) {
|
|
4091
|
+
const entities = this.storage.getEntities();
|
|
4092
|
+
const entityRow = entities[id - 1];
|
|
4093
|
+
if (!entityRow) {
|
|
4094
|
+
throw new Error(`Entity with id ${id} not found`);
|
|
4095
|
+
}
|
|
4096
|
+
const entity = entityRow.tableName;
|
|
4097
|
+
const targetDescription = `Entity: ${entityRow.entityLabel} (${entity}, type: ${entityRow.entityType}, description: ${entityRow.description})`;
|
|
4098
|
+
const anomalies = this.storage.getAnomalies({ entity });
|
|
4099
|
+
const insights = this.storage.getInsights({ entity });
|
|
4100
|
+
const anomalySummary = anomalies.length > 0 ? `Recent anomalies (${anomalies.length}): ${anomalies.slice(0, 5).map((a) => `${a.anomalyType}:${a.severity} - ${a.message}`).join("; ")}` : "No anomalies detected";
|
|
4101
|
+
const insightSummary = insights.length > 0 ? `Recent insights (${insights.length}): ${insights.slice(0, 5).map((i) => `${i.insightType}:${i.severity} - ${i.message}`).join("; ")}` : "No insights generated";
|
|
4102
|
+
return {
|
|
4103
|
+
targetDescription: `${targetDescription}
|
|
4104
|
+
${anomalySummary}
|
|
4105
|
+
${insightSummary}`,
|
|
4106
|
+
entityInfo: this.getEntityInfo(entity),
|
|
4107
|
+
recentEvents: this.getRecentEvents(entity, maxEvents),
|
|
4108
|
+
baselines: this.getBaselines(entity),
|
|
4109
|
+
transitions: this.getTransitions(entity),
|
|
4110
|
+
graphContext: this.getGraphContext(entity, maxGraphDepth),
|
|
4111
|
+
correlations: this.getCorrelations(entity)
|
|
4112
|
+
};
|
|
4113
|
+
}
|
|
4114
|
+
getEntityInfo(entity) {
|
|
4115
|
+
const entityData = this.storage.getEntity(entity);
|
|
4116
|
+
if (!entityData) return `Entity: ${entity} (no classification data)`;
|
|
4117
|
+
return `Entity: ${entityData.entityLabel} (table: ${entity}, type: ${entityData.entityType}, description: ${entityData.description}, confidence: ${entityData.confidence})`;
|
|
4118
|
+
}
|
|
4119
|
+
getRecentEvents(entity, maxEvents) {
|
|
4120
|
+
const events = this.storage.getEvents({ entity });
|
|
4121
|
+
const limited = events.slice(0, maxEvents);
|
|
4122
|
+
if (limited.length === 0) return "No recent events";
|
|
4123
|
+
return limited.map((e) => `[${e.timestamp}] ${e.eventType} ${e.operation} (row: ${e.rowId})`).join("\n");
|
|
4124
|
+
}
|
|
4125
|
+
getBaselines(entity) {
|
|
4126
|
+
const baselines = this.storage.getBaselines();
|
|
4127
|
+
const entityBaselines = baselines.filter((b) => b.entity === entity);
|
|
4128
|
+
if (entityBaselines.length === 0) return "No baselines established";
|
|
4129
|
+
return entityBaselines.map((b) => `${b.metric}: mean=${b.mean.toFixed(2)}, stddev=${b.stddev.toFixed(2)}, samples=${b.sampleSize}`).join("\n");
|
|
4130
|
+
}
|
|
4131
|
+
getTransitions(entity) {
|
|
4132
|
+
const stats = this.storage.getTransitionStats(entity);
|
|
4133
|
+
if (stats.length === 0) return "No state transitions recorded";
|
|
4134
|
+
return stats.map((t) => `${t.fromState} -> ${t.toState}: count=${t.count}, avg=${t.avgDurationMs}ms, min=${t.minDurationMs}ms, max=${t.maxDurationMs}ms`).join("\n");
|
|
4135
|
+
}
|
|
4136
|
+
getGraphContext(entity, maxDepth) {
|
|
4137
|
+
if (!this.graphStore || !this.traversal) return "Knowledge graph not available";
|
|
4138
|
+
const entityNode = this.graphStore.getNodeByRef("entity", entity);
|
|
4139
|
+
if (!entityNode) return "Entity not found in knowledge graph";
|
|
4140
|
+
const neighbors = this.graphStore.neighbors(entityNode.id);
|
|
4141
|
+
const causes = this.traversal.causesOf(entityNode.id, maxDepth);
|
|
4142
|
+
const impacts = this.traversal.impactOf(entityNode.id, maxDepth);
|
|
4143
|
+
const parts = [];
|
|
4144
|
+
if (neighbors.length > 0) {
|
|
4145
|
+
parts.push(`Connected nodes (${neighbors.length}): ${neighbors.map((n) => `${n.nodeType}:${n.label}`).join(", ")}`);
|
|
4146
|
+
}
|
|
4147
|
+
if (causes.length > 0) {
|
|
4148
|
+
parts.push(`Causes (${causes.length}): ${causes.map((n) => `${n.nodeType}:${n.label}`).join(", ")}`);
|
|
4149
|
+
}
|
|
4150
|
+
if (impacts.length > 0) {
|
|
4151
|
+
parts.push(`Impacts (${impacts.length}): ${impacts.map((n) => `${n.nodeType}:${n.label}`).join(", ")}`);
|
|
4152
|
+
}
|
|
4153
|
+
return parts.length > 0 ? parts.join("\n") : "No graph connections found";
|
|
4154
|
+
}
|
|
4155
|
+
getCorrelations(entity) {
|
|
4156
|
+
const rates = this.storage.getCorrelationRates(entity, 24);
|
|
4157
|
+
if (rates.length === 0) return "No correlation data";
|
|
4158
|
+
return `Correlation rates (last 24h): ${rates.slice(0, 10).map((r) => `${r.bucket}:${r.eventCount}`).join(", ")}`;
|
|
4159
|
+
}
|
|
4160
|
+
};
|
|
4161
|
+
|
|
4162
|
+
// src/explain/explainer.ts
|
|
4163
|
+
var logger8 = createLogger({ name: "explainer" });
|
|
4164
|
+
var explainCounter = 0;
|
|
4165
|
+
function generateExplanationId(target) {
|
|
4166
|
+
explainCounter++;
|
|
4167
|
+
return `exp_${target.type}_${target.id}_${Date.now()}_${explainCounter}`;
|
|
4168
|
+
}
|
|
4169
|
+
var Explainer = class {
|
|
4170
|
+
llm;
|
|
4171
|
+
contextBuilder;
|
|
4172
|
+
constructor(llm, storage, graphStore, traversal) {
|
|
4173
|
+
this.llm = llm;
|
|
4174
|
+
this.contextBuilder = new ExplainContextBuilder(storage, graphStore, traversal);
|
|
4175
|
+
}
|
|
4176
|
+
async explain(target, options) {
|
|
4177
|
+
logger8.info({ target }, "Building explanation context");
|
|
4178
|
+
const context = this.contextBuilder.buildContext(target, options);
|
|
4179
|
+
const prompt = this.buildPrompt(target, context, options);
|
|
4180
|
+
const chatOptions = { jsonMode: true, maxTokens: 4096 };
|
|
4181
|
+
let response = await this.llm.chat(prompt, chatOptions);
|
|
4182
|
+
try {
|
|
4183
|
+
return this.parseResponse(response.content, target);
|
|
4184
|
+
} catch {
|
|
4185
|
+
logger8.warn("First LLM parse failed, retrying");
|
|
4186
|
+
response = await this.llm.chat(prompt, chatOptions);
|
|
4187
|
+
return this.parseResponse(response.content, target);
|
|
4188
|
+
}
|
|
4189
|
+
}
|
|
4190
|
+
buildPrompt(target, context, options) {
|
|
4191
|
+
const includeRecommendations = options?.includeRecommendations !== false;
|
|
4192
|
+
return `You are a database intelligence analyst. Given the following context about a detected ${target.type}, explain WHY it happened using root cause analysis.
|
|
4193
|
+
|
|
4194
|
+
TARGET:
|
|
4195
|
+
${context.targetDescription}
|
|
4196
|
+
|
|
4197
|
+
CONTEXT:
|
|
4198
|
+
|
|
4199
|
+
Entity Information:
|
|
4200
|
+
${context.entityInfo}
|
|
4201
|
+
|
|
4202
|
+
Recent Events:
|
|
4203
|
+
${context.recentEvents}
|
|
4204
|
+
|
|
4205
|
+
Baselines:
|
|
4206
|
+
${context.baselines}
|
|
4207
|
+
|
|
4208
|
+
State Transitions:
|
|
4209
|
+
${context.transitions}
|
|
4210
|
+
|
|
4211
|
+
Knowledge Graph:
|
|
4212
|
+
${context.graphContext}
|
|
4213
|
+
|
|
4214
|
+
Correlations:
|
|
4215
|
+
${context.correlations}
|
|
4216
|
+
|
|
4217
|
+
Analyze the data above and return a JSON object with your explanation. Be specific and reference actual data points from the context.
|
|
4218
|
+
|
|
4219
|
+
Return valid JSON in this exact format:
|
|
4220
|
+
{
|
|
4221
|
+
"summary": "1-2 sentence human-readable explanation of the root cause",
|
|
4222
|
+
"confidence": 0.85,
|
|
4223
|
+
"evidence": [
|
|
4224
|
+
{
|
|
4225
|
+
"type": "event|baseline|transition|correlation|graph_node",
|
|
4226
|
+
"description": "What this evidence shows",
|
|
4227
|
+
"data": {},
|
|
4228
|
+
"relevance": 0.9
|
|
4229
|
+
}
|
|
4230
|
+
],
|
|
4231
|
+
"causalChain": [
|
|
4232
|
+
{
|
|
4233
|
+
"order": 1,
|
|
4234
|
+
"description": "First thing that happened",
|
|
4235
|
+
"confidence": 0.8
|
|
4236
|
+
}
|
|
4237
|
+
]${includeRecommendations ? `,
|
|
4238
|
+
"recommendedActions": [
|
|
4239
|
+
{
|
|
4240
|
+
"action": "What to do",
|
|
4241
|
+
"reason": "Why this helps",
|
|
4242
|
+
"priority": "low|medium|high"
|
|
4243
|
+
}
|
|
4244
|
+
]` : ""}
|
|
4245
|
+
}`;
|
|
4246
|
+
}
|
|
4247
|
+
parseResponse(content, target) {
|
|
4248
|
+
let data;
|
|
4249
|
+
try {
|
|
4250
|
+
data = JSON.parse(content);
|
|
4251
|
+
} catch (err) {
|
|
4252
|
+
throw new LLMResponseError(
|
|
4253
|
+
`Failed to parse explain response: ${err.message}`,
|
|
4254
|
+
content.slice(0, 500)
|
|
4255
|
+
);
|
|
4256
|
+
}
|
|
4257
|
+
if (!data.summary || typeof data.summary !== "string") {
|
|
4258
|
+
throw new LLMResponseError('Invalid explain response: missing "summary"', content.slice(0, 500));
|
|
4259
|
+
}
|
|
4260
|
+
return {
|
|
4261
|
+
explanationId: generateExplanationId(target),
|
|
4262
|
+
target,
|
|
4263
|
+
summary: data.summary,
|
|
4264
|
+
confidence: typeof data.confidence === "number" ? data.confidence : 0.5,
|
|
4265
|
+
evidence: Array.isArray(data.evidence) ? data.evidence.map((e) => ({
|
|
4266
|
+
type: e.type ?? "event",
|
|
4267
|
+
description: e.description ?? "",
|
|
4268
|
+
data: e.data ?? {},
|
|
4269
|
+
relevance: typeof e.relevance === "number" ? e.relevance : 0.5
|
|
4270
|
+
})) : [],
|
|
4271
|
+
causalChain: Array.isArray(data.causalChain) ? data.causalChain.map((s, i) => ({
|
|
4272
|
+
order: s.order ?? i + 1,
|
|
4273
|
+
description: s.description ?? "",
|
|
4274
|
+
confidence: typeof s.confidence === "number" ? s.confidence : 0.5
|
|
4275
|
+
})) : [],
|
|
4276
|
+
recommendedActions: Array.isArray(data.recommendedActions) ? data.recommendedActions.map((a) => ({
|
|
4277
|
+
action: a.action ?? "",
|
|
4278
|
+
reason: a.reason ?? "",
|
|
4279
|
+
priority: ["low", "medium", "high"].includes(a.priority) ? a.priority : "medium"
|
|
4280
|
+
})) : [],
|
|
4281
|
+
generatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
4282
|
+
};
|
|
4283
|
+
}
|
|
4284
|
+
};
|
|
4285
|
+
|
|
4286
|
+
// src/ask/ask-context-builder.ts
|
|
4287
|
+
var DEFAULT_MAX_EVENTS2 = 50;
|
|
4288
|
+
var MAX_ANOMALIES = 20;
|
|
4289
|
+
var MAX_INSIGHTS = 20;
|
|
4290
|
+
var MAX_RECOMMENDATIONS = 10;
|
|
4291
|
+
var AskContextBuilder = class {
|
|
4292
|
+
storage;
|
|
4293
|
+
graphStore;
|
|
4294
|
+
constructor(storage, graphStore) {
|
|
4295
|
+
this.storage = storage;
|
|
4296
|
+
this.graphStore = graphStore ?? null;
|
|
4297
|
+
}
|
|
4298
|
+
buildContext(options) {
|
|
4299
|
+
const maxEvents = options?.maxContextEvents ?? DEFAULT_MAX_EVENTS2;
|
|
4300
|
+
const entityHint = options?.entityHint;
|
|
4301
|
+
return {
|
|
4302
|
+
entities: this.getEntities(),
|
|
4303
|
+
recentEvents: this.getRecentEvents(maxEvents, entityHint),
|
|
4304
|
+
baselines: this.getBaselines(),
|
|
4305
|
+
anomalies: this.getAnomalies(),
|
|
4306
|
+
insights: this.getInsights(),
|
|
4307
|
+
transitions: this.getTransitions(),
|
|
4308
|
+
graphSummary: this.getGraphSummary(),
|
|
4309
|
+
recommendations: this.getRecommendations()
|
|
4310
|
+
};
|
|
4311
|
+
}
|
|
4312
|
+
getEntities() {
|
|
4313
|
+
const entities = this.storage.getEntities();
|
|
4314
|
+
if (entities.length === 0) return "No entities discovered yet";
|
|
4315
|
+
return entities.map(
|
|
4316
|
+
(e) => `${e.tableName} (${e.entityLabel}, type: ${e.entityType}, ${e.description})`
|
|
4317
|
+
).join("\n");
|
|
4318
|
+
}
|
|
4319
|
+
getRecentEvents(maxEvents, entityHint) {
|
|
4320
|
+
const filter = {};
|
|
4321
|
+
if (entityHint) filter.entity = entityHint;
|
|
4322
|
+
const events = this.storage.getEvents(filter);
|
|
4323
|
+
const limited = events.slice(0, maxEvents);
|
|
4324
|
+
if (limited.length === 0) return "No recent events";
|
|
4325
|
+
return `${events.length} total events (showing ${limited.length}):
|
|
4326
|
+
` + limited.map(
|
|
4327
|
+
(e) => `[${e.timestamp}] ${e.entity} ${e.eventType} ${e.operation} (row: ${e.rowId})`
|
|
4328
|
+
).join("\n");
|
|
4329
|
+
}
|
|
4330
|
+
getBaselines() {
|
|
4331
|
+
const baselines = this.storage.getBaselines();
|
|
4332
|
+
if (baselines.length === 0) return "No baselines established";
|
|
4333
|
+
return baselines.map(
|
|
4334
|
+
(b) => `${b.entity}.${b.metric}: mean=${b.mean.toFixed(2)}, stddev=${b.stddev.toFixed(2)}, samples=${b.sampleSize}`
|
|
4335
|
+
).join("\n");
|
|
4336
|
+
}
|
|
4337
|
+
getAnomalies() {
|
|
4338
|
+
const anomalies = this.storage.getAnomalies();
|
|
4339
|
+
const limited = anomalies.slice(0, MAX_ANOMALIES);
|
|
4340
|
+
if (limited.length === 0) return "No anomalies detected";
|
|
4341
|
+
return `${anomalies.length} total anomalies (showing ${limited.length}):
|
|
4342
|
+
` + limited.map(
|
|
4343
|
+
(a) => `[${a.timestamp}] ${a.entity} ${a.anomalyType} (${a.severity}): ${a.message} (expected: ${a.expected}, actual: ${a.actual})`
|
|
4344
|
+
).join("\n");
|
|
4345
|
+
}
|
|
4346
|
+
getInsights() {
|
|
4347
|
+
const insights = this.storage.getInsights();
|
|
4348
|
+
const limited = insights.slice(0, MAX_INSIGHTS);
|
|
4349
|
+
if (limited.length === 0) return "No insights generated";
|
|
4350
|
+
return `${insights.length} total insights (showing ${limited.length}):
|
|
4351
|
+
` + limited.map(
|
|
4352
|
+
(i) => `[${i.timestamp}] ${i.entity} ${i.insightType} (${i.severity}): ${i.message}`
|
|
4353
|
+
).join("\n");
|
|
4354
|
+
}
|
|
4355
|
+
getTransitions() {
|
|
4356
|
+
const stats = this.storage.getTransitionStats();
|
|
4357
|
+
if (stats.length === 0) return "No state transitions recorded";
|
|
4358
|
+
return stats.map(
|
|
4359
|
+
(t) => `${t.entity}: ${t.fromState} -> ${t.toState} (count=${t.count}, avg=${t.avgDurationMs}ms)`
|
|
4360
|
+
).join("\n");
|
|
4361
|
+
}
|
|
4362
|
+
getGraphSummary() {
|
|
4363
|
+
if (!this.graphStore) return "Knowledge graph not available";
|
|
4364
|
+
const summary = this.graphStore.getSummary();
|
|
4365
|
+
return `Nodes: ${summary.totalNodes}, Edges: ${summary.totalEdges} (entities: ${summary.entities}, events: ${summary.events}, anomalies: ${summary.anomalies}, insights: ${summary.insights})`;
|
|
4366
|
+
}
|
|
4367
|
+
getRecommendations() {
|
|
4368
|
+
const recs = this.storage.getRecommendations({ last: MAX_RECOMMENDATIONS });
|
|
4369
|
+
if (recs.length === 0) return "No recommendations";
|
|
4370
|
+
return recs.map(
|
|
4371
|
+
(r) => `[${r.status}] ${r.action}: ${r.reason} (confidence: ${r.confidence}, entity: ${r.entity})`
|
|
4372
|
+
).join("\n");
|
|
4373
|
+
}
|
|
4374
|
+
};
|
|
4375
|
+
|
|
4376
|
+
// src/ask/asker.ts
|
|
4377
|
+
var logger9 = createLogger({ name: "asker" });
|
|
4378
|
+
var askCounter = 0;
|
|
4379
|
+
function generateAskId() {
|
|
4380
|
+
askCounter++;
|
|
4381
|
+
return `ask_${Date.now()}_${askCounter}`;
|
|
4382
|
+
}
|
|
4383
|
+
var VALID_SIGNAL_TYPES = /* @__PURE__ */ new Set([
|
|
4384
|
+
"baselineDeviation",
|
|
4385
|
+
"anomaly",
|
|
4386
|
+
"insight",
|
|
4387
|
+
"eventCluster",
|
|
4388
|
+
"transition",
|
|
4389
|
+
"correlation",
|
|
4390
|
+
"graphRelation"
|
|
4391
|
+
]);
|
|
4392
|
+
var Asker = class {
|
|
4393
|
+
llm;
|
|
4394
|
+
contextBuilder;
|
|
4395
|
+
constructor(llm, storage, graphStore) {
|
|
4396
|
+
this.llm = llm;
|
|
4397
|
+
this.contextBuilder = new AskContextBuilder(storage, graphStore);
|
|
4398
|
+
}
|
|
4399
|
+
async ask(question, options) {
|
|
4400
|
+
logger9.info({ question }, "Processing question");
|
|
4401
|
+
const context = this.contextBuilder.buildContext(options);
|
|
4402
|
+
const prompt = this.buildPrompt(question, context);
|
|
4403
|
+
const chatOptions = { jsonMode: true, maxTokens: 4096 };
|
|
4404
|
+
let response = await this.llm.chat(prompt, chatOptions);
|
|
4405
|
+
try {
|
|
4406
|
+
return this.parseResponse(response.content, question);
|
|
4407
|
+
} catch {
|
|
4408
|
+
logger9.warn("First LLM parse failed, retrying");
|
|
4409
|
+
response = await this.llm.chat(prompt, chatOptions);
|
|
4410
|
+
return this.parseResponse(response.content, question);
|
|
4411
|
+
}
|
|
4412
|
+
}
|
|
4413
|
+
buildPrompt(question, context) {
|
|
4414
|
+
return `You are a database intelligence analyst. You have access to a live monitoring system that tracks a database. Answer the user's question using ONLY the data provided below. Be specific \u2014 reference actual numbers, entity names, and timestamps from the data.
|
|
4415
|
+
|
|
4416
|
+
QUESTION: ${question}
|
|
4417
|
+
|
|
4418
|
+
SYSTEM STATE:
|
|
4419
|
+
|
|
4420
|
+
Entities:
|
|
4421
|
+
${context.entities}
|
|
4422
|
+
|
|
4423
|
+
Recent Events:
|
|
4424
|
+
${context.recentEvents}
|
|
4425
|
+
|
|
4426
|
+
Baselines:
|
|
4427
|
+
${context.baselines}
|
|
4428
|
+
|
|
4429
|
+
Anomalies:
|
|
4430
|
+
${context.anomalies}
|
|
4431
|
+
|
|
4432
|
+
Insights:
|
|
4433
|
+
${context.insights}
|
|
4434
|
+
|
|
4435
|
+
State Transitions:
|
|
4436
|
+
${context.transitions}
|
|
4437
|
+
|
|
4438
|
+
Knowledge Graph:
|
|
4439
|
+
${context.graphSummary}
|
|
4440
|
+
|
|
4441
|
+
Recommendations:
|
|
4442
|
+
${context.recommendations}
|
|
4443
|
+
|
|
4444
|
+
Analyze the data and answer the question. If the data doesn't contain enough information, say so honestly and lower your confidence score.
|
|
4445
|
+
|
|
4446
|
+
Return valid JSON in this exact format:
|
|
4447
|
+
{
|
|
4448
|
+
"answer": "Clear, specific answer referencing actual data points",
|
|
4449
|
+
"confidence": 0.85,
|
|
4450
|
+
"supportingSignals": [
|
|
4451
|
+
{
|
|
4452
|
+
"type": "baselineDeviation|anomaly|insight|eventCluster|transition|correlation|graphRelation",
|
|
4453
|
+
"entity": "entity name",
|
|
4454
|
+
"description": "What this signal shows",
|
|
4455
|
+
"data": {}
|
|
4456
|
+
}
|
|
4457
|
+
],
|
|
4458
|
+
"relatedEntities": ["entity1", "entity2"],
|
|
4459
|
+
"recommendedActions": ["Action 1", "Action 2"]
|
|
4460
|
+
}`;
|
|
4461
|
+
}
|
|
4462
|
+
parseResponse(content, question) {
|
|
4463
|
+
let data;
|
|
4464
|
+
try {
|
|
4465
|
+
data = JSON.parse(content);
|
|
4466
|
+
} catch (err) {
|
|
4467
|
+
throw new LLMResponseError(
|
|
4468
|
+
`Failed to parse ask response: ${err.message}`,
|
|
4469
|
+
content.slice(0, 500)
|
|
4470
|
+
);
|
|
4471
|
+
}
|
|
4472
|
+
if (!data.answer || typeof data.answer !== "string") {
|
|
4473
|
+
throw new LLMResponseError('Invalid ask response: missing "answer"', content.slice(0, 500));
|
|
4474
|
+
}
|
|
4475
|
+
return {
|
|
4476
|
+
askId: generateAskId(),
|
|
4477
|
+
question,
|
|
4478
|
+
answer: data.answer,
|
|
4479
|
+
confidence: typeof data.confidence === "number" ? data.confidence : 0.5,
|
|
4480
|
+
supportingSignals: Array.isArray(data.supportingSignals) ? data.supportingSignals.map((s) => ({
|
|
4481
|
+
type: VALID_SIGNAL_TYPES.has(s.type) ? s.type : "anomaly",
|
|
4482
|
+
entity: s.entity ?? "",
|
|
4483
|
+
description: s.description ?? "",
|
|
4484
|
+
data: s.data ?? {}
|
|
4485
|
+
})) : [],
|
|
4486
|
+
relatedEntities: Array.isArray(data.relatedEntities) ? data.relatedEntities.filter((e) => typeof e === "string") : [],
|
|
4487
|
+
recommendedActions: Array.isArray(data.recommendedActions) ? data.recommendedActions.filter((a) => typeof a === "string") : [],
|
|
4488
|
+
generatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
4489
|
+
};
|
|
4490
|
+
}
|
|
4491
|
+
};
|
|
4492
|
+
|
|
4493
|
+
// src/core/config.ts
|
|
4494
|
+
import { cosmiconfig } from "cosmiconfig";
|
|
4495
|
+
function defineConfig(config) {
|
|
4496
|
+
return config;
|
|
4497
|
+
}
|
|
4498
|
+
|
|
4499
|
+
// src/index.ts
|
|
4500
|
+
var KnowledgeGraph = class {
|
|
4501
|
+
store;
|
|
4502
|
+
traversal;
|
|
4503
|
+
entityIntelligence;
|
|
4504
|
+
constructor(store) {
|
|
4505
|
+
this.store = store;
|
|
4506
|
+
this.traversal = new Traversal(store);
|
|
4507
|
+
this.entityIntelligence = new EntityIntelligence(store);
|
|
4508
|
+
}
|
|
4509
|
+
causesOf(nodeId, maxDepth) {
|
|
4510
|
+
return this.traversal.causesOf(nodeId, maxDepth);
|
|
4511
|
+
}
|
|
4512
|
+
impactOf(nodeId, maxDepth) {
|
|
4513
|
+
return this.traversal.impactOf(nodeId, maxDepth);
|
|
4514
|
+
}
|
|
4515
|
+
timeline(nodeId) {
|
|
4516
|
+
return this.traversal.timeline(nodeId);
|
|
4517
|
+
}
|
|
4518
|
+
entity(refId) {
|
|
4519
|
+
return {
|
|
4520
|
+
intelligence: () => this.entityIntelligence.forEntity(refId)
|
|
4521
|
+
};
|
|
4522
|
+
}
|
|
4523
|
+
getNode(id) {
|
|
4524
|
+
return this.store.getNode(id);
|
|
4525
|
+
}
|
|
4526
|
+
getNodeByRef(nodeType, refId) {
|
|
4527
|
+
return this.store.getNodeByRef(nodeType, refId);
|
|
4528
|
+
}
|
|
4529
|
+
getEdges(nodeId, edgeType) {
|
|
4530
|
+
return this.store.getEdgesFrom(nodeId, edgeType);
|
|
4531
|
+
}
|
|
4532
|
+
neighbors(nodeId) {
|
|
4533
|
+
return this.store.neighbors(nodeId);
|
|
4534
|
+
}
|
|
4535
|
+
getSummary() {
|
|
4536
|
+
return this.store.getSummary();
|
|
4537
|
+
}
|
|
4538
|
+
export(rootNodeId) {
|
|
4539
|
+
return exportGraph(this.store, rootNodeId);
|
|
4540
|
+
}
|
|
4541
|
+
};
|
|
4542
|
+
var Cortexa = class extends EventEmitter2 {
|
|
4543
|
+
config;
|
|
4544
|
+
connector = null;
|
|
4545
|
+
storage;
|
|
4546
|
+
logger;
|
|
4547
|
+
llmProvider = null;
|
|
4548
|
+
watcher = null;
|
|
4549
|
+
knowledgeGraph = null;
|
|
4550
|
+
actionRegistry;
|
|
4551
|
+
discoveredTables = [];
|
|
4552
|
+
_status = { connected: false, tables: 0, db: "", watching: false };
|
|
4553
|
+
constructor(config) {
|
|
4554
|
+
super();
|
|
4555
|
+
this.config = config;
|
|
4556
|
+
this.logger = createLogger();
|
|
4557
|
+
this.storage = new CortexaStorage(config.projectRoot ?? process.cwd());
|
|
4558
|
+
this.actionRegistry = new ActionRegistry(config.actions?.governance);
|
|
4559
|
+
}
|
|
4560
|
+
async connect() {
|
|
4561
|
+
this.logger.info("Connecting to database...");
|
|
4562
|
+
this.storage.initialize();
|
|
4563
|
+
this.connector = await createConnector(this.config.connection.type);
|
|
4564
|
+
await this.connector.connect(this.config.connection);
|
|
4565
|
+
const tables = await this.connector.getTableCount();
|
|
4566
|
+
const db = await this.connector.getDatabaseName();
|
|
4567
|
+
const readOnly = await this.connector.isReadOnly();
|
|
4568
|
+
if (!readOnly) {
|
|
4569
|
+
this.logger.warn(
|
|
4570
|
+
"Connection is NOT read-only. Cortexa recommends using a read-only database user."
|
|
4571
|
+
);
|
|
4572
|
+
}
|
|
4573
|
+
if (this.config.llmProvider) {
|
|
4574
|
+
this.llmProvider = this.config.llmProvider;
|
|
4575
|
+
} else if (this.config.llm) {
|
|
4576
|
+
this.llmProvider = createLLMProvider(this.config.llm);
|
|
4577
|
+
}
|
|
4578
|
+
this.storage.setMeta("database_name", db);
|
|
4579
|
+
this.storage.setMeta("table_count", String(tables));
|
|
4580
|
+
this.storage.setMeta("connected_at", (/* @__PURE__ */ new Date()).toISOString());
|
|
4581
|
+
this._status = { connected: true, tables, db, watching: false };
|
|
4582
|
+
this.logger.info({ db, tables }, "Connected successfully");
|
|
4583
|
+
}
|
|
4584
|
+
async disconnect() {
|
|
4585
|
+
this.unwatch();
|
|
4586
|
+
if (this.connector) {
|
|
4587
|
+
await this.connector.disconnect();
|
|
4588
|
+
this.connector = null;
|
|
4589
|
+
}
|
|
4590
|
+
this.llmProvider = null;
|
|
4591
|
+
this.knowledgeGraph = null;
|
|
4592
|
+
this.storage.close();
|
|
4593
|
+
this._status = { connected: false, tables: 0, db: "", watching: false };
|
|
4594
|
+
this.logger.info("Disconnected");
|
|
4595
|
+
}
|
|
4596
|
+
async discover(options) {
|
|
4597
|
+
if (!this.llmProvider) {
|
|
4598
|
+
throw new Error("LLM configuration required for schema discovery. Add llm config to your Cortexa options.");
|
|
4599
|
+
}
|
|
4600
|
+
if (!this.connector) {
|
|
4601
|
+
throw new Error("Not connected. Call connect() first.");
|
|
4602
|
+
}
|
|
4603
|
+
this.logger.info("Starting schema discovery...");
|
|
4604
|
+
const introspector = new SchemaIntrospector(this.connector, this.config.connection.type, options);
|
|
4605
|
+
const rawSchema = await introspector.introspect();
|
|
4606
|
+
this.logger.info({ tables: rawSchema.tables.length }, "Schema introspected");
|
|
4607
|
+
const classifier = new SchemaClassifier(this.llmProvider, this.config.batchSize);
|
|
4608
|
+
const entities = await classifier.classify(rawSchema.tables);
|
|
4609
|
+
this.logger.info({ entities: entities.length }, "Entities classified");
|
|
4610
|
+
const graphBuilder = new GraphBuilder();
|
|
4611
|
+
const graph = graphBuilder.build(rawSchema.tables, entities);
|
|
4612
|
+
for (const entity of entities) {
|
|
4613
|
+
this.storage.saveEntity(entity);
|
|
4614
|
+
}
|
|
4615
|
+
this.storage.clearRelationships();
|
|
4616
|
+
for (const rel of graph.relationships) {
|
|
4617
|
+
this.storage.saveRelationship(rel);
|
|
4618
|
+
}
|
|
4619
|
+
for (const table of rawSchema.tables) {
|
|
4620
|
+
this.storage.saveDiscoveredTable(table);
|
|
4621
|
+
}
|
|
4622
|
+
this.discoveredTables = rawSchema.tables;
|
|
4623
|
+
if (this.config.knowledge?.enabled) {
|
|
4624
|
+
const graphStore = new GraphStore(this.storage);
|
|
4625
|
+
const populator = new GraphPopulator(graphStore);
|
|
4626
|
+
populator.populateFromSchema(rawSchema.tables);
|
|
4627
|
+
this.logger.info("Knowledge graph populated from schema");
|
|
4628
|
+
}
|
|
4629
|
+
this.logger.info("Discovery complete");
|
|
4630
|
+
return {
|
|
4631
|
+
entities: graph.entities,
|
|
4632
|
+
relationships: graph.relationships,
|
|
4633
|
+
raw: rawSchema
|
|
4634
|
+
};
|
|
4635
|
+
}
|
|
4636
|
+
entity(tableName) {
|
|
4637
|
+
return this.storage.getEntity(tableName);
|
|
4638
|
+
}
|
|
4639
|
+
exportGraph() {
|
|
4640
|
+
return {
|
|
4641
|
+
entities: this.storage.getEntities(),
|
|
4642
|
+
relationships: this.storage.getRelationships()
|
|
4643
|
+
};
|
|
4644
|
+
}
|
|
4645
|
+
async watch(options) {
|
|
4646
|
+
if (!this.connector) {
|
|
4647
|
+
throw new Error("Not connected. Call connect() first.");
|
|
4648
|
+
}
|
|
4649
|
+
this.watcher = new Watcher({
|
|
4650
|
+
connector: this.connector,
|
|
4651
|
+
dbType: this.config.connection.type,
|
|
4652
|
+
storage: this.storage,
|
|
4653
|
+
tables: this.discoveredTables,
|
|
4654
|
+
connectionConfig: this.config.connection,
|
|
4655
|
+
onEvent: (event) => this.emit("event", event),
|
|
4656
|
+
onAnomaly: (anomaly) => this.emit("anomaly", anomaly),
|
|
4657
|
+
onInsight: (insight) => this.emit("insight", insight),
|
|
4658
|
+
onRecommendation: (rec) => this.emit("recommendation", rec),
|
|
4659
|
+
onActionExecuted: (rec) => this.emit("action:executed", rec),
|
|
4660
|
+
onActionFailed: (rec) => this.emit("action:failed", rec),
|
|
4661
|
+
workflows: this.config.reasoning?.workflows,
|
|
4662
|
+
analytics: this.config.analytics,
|
|
4663
|
+
knowledge: this.config.knowledge,
|
|
4664
|
+
actions: this.config.actions,
|
|
4665
|
+
actionRegistry: this.actionRegistry
|
|
4666
|
+
});
|
|
4667
|
+
if (this.config.notifications?.enabled && this.config.notifications.rules?.length) {
|
|
4668
|
+
const notifier = new Notifier(this.config.notifications.rules);
|
|
4669
|
+
this.on("anomaly", (data) => notifier.notify("anomaly", data));
|
|
4670
|
+
this.on("insight", (data) => notifier.notify("insight", data));
|
|
4671
|
+
this.on("recommendation", (data) => notifier.notify("recommendation", data));
|
|
4672
|
+
this.on("action:executed", (data) => notifier.notify("action:executed", data));
|
|
4673
|
+
this.on("action:failed", (data) => notifier.notify("action:failed", data));
|
|
4674
|
+
}
|
|
4675
|
+
await this.watcher.start(options);
|
|
4676
|
+
if (options?.once) {
|
|
4677
|
+
this.watcher = null;
|
|
4678
|
+
this.logger.info("Single poll cycle complete");
|
|
4679
|
+
return;
|
|
4680
|
+
}
|
|
4681
|
+
this._status = { ...this._status, watching: true };
|
|
4682
|
+
this.logger.info("Watcher started");
|
|
4683
|
+
}
|
|
4684
|
+
unwatch() {
|
|
4685
|
+
if (this.watcher) {
|
|
4686
|
+
this.watcher.stop();
|
|
4687
|
+
this.watcher = null;
|
|
4688
|
+
this._status = { ...this._status, watching: false };
|
|
4689
|
+
this.logger.info("Watcher stopped");
|
|
4690
|
+
}
|
|
4691
|
+
}
|
|
4692
|
+
async cleanupStream() {
|
|
4693
|
+
if (this.watcher) {
|
|
4694
|
+
await this.watcher.cleanup();
|
|
4695
|
+
this.watcher = null;
|
|
4696
|
+
this._status = { ...this._status, watching: false };
|
|
4697
|
+
this.logger.info("Stream cleaned up");
|
|
4698
|
+
}
|
|
4699
|
+
}
|
|
4700
|
+
getEvents(filter) {
|
|
4701
|
+
return this.storage.getEvents(filter);
|
|
4702
|
+
}
|
|
4703
|
+
getBaselines() {
|
|
4704
|
+
return this.storage.getBaselines();
|
|
4705
|
+
}
|
|
4706
|
+
getAnomalies(filter) {
|
|
4707
|
+
return this.storage.getAnomalies(filter);
|
|
4708
|
+
}
|
|
4709
|
+
getInsights(filter) {
|
|
4710
|
+
return this.storage.getInsights(filter);
|
|
4711
|
+
}
|
|
4712
|
+
getTransitions(entity) {
|
|
4713
|
+
return this.storage.getTransitionStats(entity);
|
|
4714
|
+
}
|
|
4715
|
+
getCorrelations() {
|
|
4716
|
+
if (!this.config.analytics?.correlations) {
|
|
4717
|
+
return [];
|
|
4718
|
+
}
|
|
4719
|
+
const results = [];
|
|
4720
|
+
for (const [name, corrConfig] of Object.entries(this.config.analytics.correlations)) {
|
|
4721
|
+
const rateBaseline = this.storage.getAnalyticsBaseline(`correlation:${name}:rate_ratio`);
|
|
4722
|
+
const coOccBaseline = this.storage.getAnalyticsBaseline(`correlation:${name}:co_occurrence`);
|
|
4723
|
+
const rateRatio = rateBaseline ? rateBaseline.mean : 0;
|
|
4724
|
+
const coOccurrenceRate = coOccBaseline ? coOccBaseline.mean : 0;
|
|
4725
|
+
const healthy = !!(rateBaseline && coOccBaseline);
|
|
4726
|
+
results.push({
|
|
4727
|
+
name,
|
|
4728
|
+
entities: corrConfig.entities,
|
|
4729
|
+
rateRatio,
|
|
4730
|
+
baselineRatio: rateBaseline ? rateBaseline.mean : 0,
|
|
4731
|
+
coOccurrenceRate,
|
|
4732
|
+
baselineCoOccurrence: coOccBaseline ? coOccBaseline.mean : 0,
|
|
4733
|
+
healthy
|
|
4734
|
+
});
|
|
4735
|
+
}
|
|
4736
|
+
return results;
|
|
4737
|
+
}
|
|
4738
|
+
getDistributions(entity) {
|
|
4739
|
+
if (!this.config.analytics?.distributions) {
|
|
4740
|
+
return [];
|
|
4741
|
+
}
|
|
4742
|
+
const results = [];
|
|
4743
|
+
for (const [entityName, distConfig] of Object.entries(this.config.analytics.distributions)) {
|
|
4744
|
+
if (entity && entity !== entityName) continue;
|
|
4745
|
+
for (const col of distConfig.columns) {
|
|
4746
|
+
const buckets = this.storage.getHistogramBuckets(entityName, col, "baseline");
|
|
4747
|
+
const shiftBaseline = this.storage.getAnalyticsBaseline(`distribution:${entityName}:${col}:shift`);
|
|
4748
|
+
results.push({
|
|
4749
|
+
entity: entityName,
|
|
4750
|
+
columnName: col,
|
|
4751
|
+
bucketCount: buckets.length,
|
|
4752
|
+
baselineSamples: shiftBaseline ? shiftBaseline.sampleSize : 0,
|
|
4753
|
+
currentShift: shiftBaseline ? shiftBaseline.mean : null,
|
|
4754
|
+
shifted: shiftBaseline ? Math.abs(shiftBaseline.mean) > shiftBaseline.stddev * 2 : false
|
|
4755
|
+
});
|
|
4756
|
+
}
|
|
4757
|
+
}
|
|
4758
|
+
return results;
|
|
4759
|
+
}
|
|
4760
|
+
graph() {
|
|
4761
|
+
if (!this.knowledgeGraph) {
|
|
4762
|
+
const graphStore = new GraphStore(this.storage);
|
|
4763
|
+
this.knowledgeGraph = new KnowledgeGraph(graphStore);
|
|
4764
|
+
}
|
|
4765
|
+
return this.knowledgeGraph;
|
|
4766
|
+
}
|
|
4767
|
+
async explain(target, options) {
|
|
4768
|
+
if (!this.llmProvider) {
|
|
4769
|
+
throw new Error("LLM configuration required for explain(). Add llm config to your Cortexa options.");
|
|
4770
|
+
}
|
|
4771
|
+
const graphStore = new GraphStore(this.storage);
|
|
4772
|
+
const traversal = new Traversal(graphStore);
|
|
4773
|
+
const explainer = new Explainer(this.llmProvider, this.storage, graphStore, traversal);
|
|
4774
|
+
return explainer.explain(target, options);
|
|
4775
|
+
}
|
|
4776
|
+
async ask(question, options) {
|
|
4777
|
+
if (!this.llmProvider) {
|
|
4778
|
+
throw new Error("LLM configuration required for ask(). Add llm config to your Cortexa options.");
|
|
4779
|
+
}
|
|
4780
|
+
const graphStore = new GraphStore(this.storage);
|
|
4781
|
+
const asker = new Asker(this.llmProvider, this.storage, graphStore);
|
|
4782
|
+
return asker.ask(question, options);
|
|
4783
|
+
}
|
|
4784
|
+
registerAction(name, handler, description) {
|
|
4785
|
+
this.actionRegistry.register(name, handler, description);
|
|
4786
|
+
}
|
|
4787
|
+
approveRecommendation(id) {
|
|
4788
|
+
this.storage.updateRecommendationStatus(id, "approved", "user");
|
|
4789
|
+
}
|
|
4790
|
+
rejectRecommendation(id) {
|
|
4791
|
+
this.storage.updateRecommendationStatus(id, "rejected", "user");
|
|
4792
|
+
}
|
|
4793
|
+
getRecommendations(filter) {
|
|
4794
|
+
return this.storage.getRecommendations(filter);
|
|
4795
|
+
}
|
|
4796
|
+
get status() {
|
|
4797
|
+
return { ...this._status };
|
|
4798
|
+
}
|
|
4799
|
+
};
|
|
4800
|
+
export {
|
|
4801
|
+
Cortexa,
|
|
4802
|
+
KnowledgeGraph,
|
|
4803
|
+
defineConfig
|
|
4804
|
+
};
|
|
4805
|
+
//# sourceMappingURL=index.js.map
|