@prometheus-ai/memory 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +107 -0
- package/dist/types/cli.d.ts +35 -0
- package/dist/types/config.d.ts +77 -0
- package/dist/types/core/aaak.d.ts +55 -0
- package/dist/types/core/annotations.d.ts +75 -0
- package/dist/types/core/banks.d.ts +33 -0
- package/dist/types/core/beam/consolidate.d.ts +32 -0
- package/dist/types/core/beam/helpers.d.ts +76 -0
- package/dist/types/core/beam/index.d.ts +59 -0
- package/dist/types/core/beam/recall.d.ts +32 -0
- package/dist/types/core/beam/schema.d.ts +2 -0
- package/dist/types/core/beam/store.d.ts +35 -0
- package/dist/types/core/beam/types.d.ts +233 -0
- package/dist/types/core/binary-vectors.d.ts +54 -0
- package/dist/types/core/chat-normalize.d.ts +13 -0
- package/dist/types/core/content-sanitizer.d.ts +18 -0
- package/dist/types/core/cost-log.d.ts +13 -0
- package/dist/types/core/embeddings.d.ts +44 -0
- package/dist/types/core/entities.d.ts +7 -0
- package/dist/types/core/episodic-graph.d.ts +89 -0
- package/dist/types/core/extraction/client.d.ts +31 -0
- package/dist/types/core/extraction/diagnostics.d.ts +51 -0
- package/dist/types/core/extraction/prompts.d.ts +2 -0
- package/dist/types/core/extraction.d.ts +6 -0
- package/dist/types/core/index.d.ts +4 -0
- package/dist/types/core/llm-backends.d.ts +21 -0
- package/dist/types/core/local-llm.d.ts +15 -0
- package/dist/types/core/memory.d.ts +160 -0
- package/dist/types/core/migrations/e6-triplestore-split.d.ts +17 -0
- package/dist/types/core/migrations/index.d.ts +1 -0
- package/dist/types/core/mmr.d.ts +8 -0
- package/dist/types/core/orchestrator.d.ts +20 -0
- package/dist/types/core/patterns.d.ts +61 -0
- package/dist/types/core/plugins.d.ts +109 -0
- package/dist/types/core/polyphonic-recall.d.ts +66 -0
- package/dist/types/core/query-cache.d.ts +46 -0
- package/dist/types/core/query-intent.d.ts +20 -0
- package/dist/types/core/recall-diagnostics.d.ts +48 -0
- package/dist/types/core/runtime-options.d.ts +68 -0
- package/dist/types/core/shmr.d.ts +56 -0
- package/dist/types/core/streaming.d.ts +136 -0
- package/dist/types/core/synonyms.d.ts +46 -0
- package/dist/types/core/temporal-parser.d.ts +16 -0
- package/dist/types/core/token-counter.d.ts +8 -0
- package/dist/types/core/triples.d.ts +63 -0
- package/dist/types/core/typed-memory.d.ts +39 -0
- package/dist/types/core/vector-math.d.ts +1 -0
- package/dist/types/core/veracity-consolidation.d.ts +60 -0
- package/dist/types/core/weibull.d.ts +96 -0
- package/dist/types/db.d.ts +16 -0
- package/dist/types/diagnose.d.ts +24 -0
- package/dist/types/dr/index.d.ts +1 -0
- package/dist/types/dr/recovery.d.ts +68 -0
- package/dist/types/index.d.ts +5 -0
- package/dist/types/mcp-server.d.ts +40 -0
- package/dist/types/mcp-tools.d.ts +484 -0
- package/dist/types/migrations/e6-triplestore-split.d.ts +1 -0
- package/dist/types/migrations/index.d.ts +1 -0
- package/dist/types/types.d.ts +145 -0
- package/dist/types/util/datetime.d.ts +8 -0
- package/dist/types/util/env.d.ts +10 -0
- package/dist/types/util/ids.d.ts +3 -0
- package/dist/types/util/lru.d.ts +12 -0
- package/dist/types/util/regex.d.ts +10 -0
- package/package.json +85 -0
- package/src/cli.ts +398 -0
- package/src/config.ts +326 -0
- package/src/core/aaak.ts +142 -0
- package/src/core/annotations.ts +457 -0
- package/src/core/banks.ts +133 -0
- package/src/core/beam/consolidate.ts +965 -0
- package/src/core/beam/helpers.ts +977 -0
- package/src/core/beam/index.ts +353 -0
- package/src/core/beam/recall.ts +1100 -0
- package/src/core/beam/schema.ts +423 -0
- package/src/core/beam/store.ts +829 -0
- package/src/core/beam/types.ts +268 -0
- package/src/core/binary-vectors.ts +317 -0
- package/src/core/chat-normalize.ts +160 -0
- package/src/core/content-sanitizer.ts +136 -0
- package/src/core/cost-log.ts +103 -0
- package/src/core/embeddings.ts +423 -0
- package/src/core/entities.ts +259 -0
- package/src/core/episodic-graph.ts +708 -0
- package/src/core/extraction/client.ts +162 -0
- package/src/core/extraction/diagnostics.ts +193 -0
- package/src/core/extraction/prompts.ts +31 -0
- package/src/core/extraction.ts +335 -0
- package/src/core/index.ts +30 -0
- package/src/core/llm-backends.ts +51 -0
- package/src/core/local-llm.ts +436 -0
- package/src/core/memory.ts +630 -0
- package/src/core/migrations/e6-triplestore-split.ts +211 -0
- package/src/core/migrations/index.ts +1 -0
- package/src/core/mmr.ts +71 -0
- package/src/core/orchestrator.ts +62 -0
- package/src/core/patterns.ts +484 -0
- package/src/core/plugins.ts +375 -0
- package/src/core/polyphonic-recall.ts +563 -0
- package/src/core/query-cache.ts +354 -0
- package/src/core/query-intent.ts +139 -0
- package/src/core/recall-diagnostics.ts +157 -0
- package/src/core/runtime-options.ts +119 -0
- package/src/core/shmr.ts +460 -0
- package/src/core/streaming.ts +419 -0
- package/src/core/synonyms.ts +197 -0
- package/src/core/temporal-parser.ts +363 -0
- package/src/core/token-counter.ts +30 -0
- package/src/core/triples.ts +454 -0
- package/src/core/typed-memory.ts +407 -0
- package/src/core/vector-math.ts +23 -0
- package/src/core/veracity-consolidation.ts +477 -0
- package/src/core/weibull.ts +124 -0
- package/src/db.ts +128 -0
- package/src/diagnose.ts +174 -0
- package/src/dr/index.ts +1 -0
- package/src/dr/recovery.ts +405 -0
- package/src/index.ts +33 -0
- package/src/mcp-server.ts +155 -0
- package/src/mcp-tools.ts +970 -0
- package/src/migrations/e6-triplestore-split.ts +1 -0
- package/src/migrations/index.ts +1 -0
- package/src/types.ts +157 -0
- package/src/util/datetime.ts +69 -0
- package/src/util/env.ts +65 -0
- package/src/util/ids.ts +19 -0
- package/src/util/lru.ts +48 -0
- package/src/util/regex.ts +165 -0
package/src/db.ts
ADDED
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
import { Database } from "bun:sqlite";
|
|
2
|
+
import { mkdirSync } from "node:fs";
|
|
3
|
+
import { dirname } from "node:path";
|
|
4
|
+
import { dbPath } from "./config";
|
|
5
|
+
|
|
6
|
+
export type DatabasePath = string | ":memory:";
|
|
7
|
+
|
|
8
|
+
export interface OpenDatabaseOptions {
|
|
9
|
+
readonly create?: boolean;
|
|
10
|
+
readonly readwrite?: boolean;
|
|
11
|
+
readonly strict?: boolean;
|
|
12
|
+
readonly loadExtension?: string | readonly string[];
|
|
13
|
+
readonly pragmas?: boolean;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
interface TxState {
|
|
17
|
+
depth: number;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
const TX_STATE = Symbol("mnemopi.txState");
|
|
21
|
+
|
|
22
|
+
type TxDatabase = Database & { [TX_STATE]?: TxState };
|
|
23
|
+
type ExtensionDatabase = Database & { loadExtension(path: string): void };
|
|
24
|
+
|
|
25
|
+
export function openDatabase(path: DatabasePath = dbPath(), options: OpenDatabaseOptions = {}): Database {
|
|
26
|
+
if (path !== ":memory:") mkdirSync(dirname(path), { recursive: true });
|
|
27
|
+
const db = new Database(path, {
|
|
28
|
+
create: options.create ?? true,
|
|
29
|
+
readwrite: options.readwrite ?? true,
|
|
30
|
+
strict: options.strict ?? true,
|
|
31
|
+
});
|
|
32
|
+
if (options.pragmas !== false) enablePragmas(db, path);
|
|
33
|
+
if (options.loadExtension !== undefined) loadExtensions(db, options.loadExtension);
|
|
34
|
+
return db;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
export function enablePragmas(db: Database, path?: DatabasePath): void {
|
|
38
|
+
db.exec("PRAGMA foreign_keys=ON");
|
|
39
|
+
db.exec("PRAGMA busy_timeout=5000");
|
|
40
|
+
if (path !== ":memory:") db.exec("PRAGMA journal_mode=WAL");
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
export function loadExtensions(db: Database, extensions: string | readonly string[]): void {
|
|
44
|
+
if (typeof extensions === "string") {
|
|
45
|
+
if (extensions) (db as ExtensionDatabase).loadExtension(extensions);
|
|
46
|
+
return;
|
|
47
|
+
}
|
|
48
|
+
for (const extension of extensions) {
|
|
49
|
+
if (extension) (db as ExtensionDatabase).loadExtension(extension);
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
export function transaction<T>(db: Database, fn: () => T): T {
|
|
54
|
+
const txDb = db as TxDatabase;
|
|
55
|
+
let state = txDb[TX_STATE];
|
|
56
|
+
if (state !== undefined && state.depth > 0) {
|
|
57
|
+
state.depth++;
|
|
58
|
+
try {
|
|
59
|
+
return fn();
|
|
60
|
+
} finally {
|
|
61
|
+
state.depth--;
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
state = { depth: 1 };
|
|
66
|
+
txDb[TX_STATE] = state;
|
|
67
|
+
db.exec("BEGIN DEFERRED");
|
|
68
|
+
try {
|
|
69
|
+
const result = fn();
|
|
70
|
+
state.depth = 0;
|
|
71
|
+
db.exec("COMMIT");
|
|
72
|
+
return result;
|
|
73
|
+
} catch (error) {
|
|
74
|
+
state.depth = 0;
|
|
75
|
+
try {
|
|
76
|
+
db.exec("ROLLBACK");
|
|
77
|
+
} catch {
|
|
78
|
+
// Preserve the original error; rollback can fail if SQLite already closed the transaction.
|
|
79
|
+
}
|
|
80
|
+
throw error;
|
|
81
|
+
} finally {
|
|
82
|
+
delete txDb[TX_STATE];
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
export const deferredTransaction = transaction;
|
|
87
|
+
|
|
88
|
+
export async function transactionAsync<T>(db: Database, fn: () => Promise<T>): Promise<T> {
|
|
89
|
+
const txDb = db as TxDatabase;
|
|
90
|
+
let state = txDb[TX_STATE];
|
|
91
|
+
if (state !== undefined && state.depth > 0) {
|
|
92
|
+
state.depth++;
|
|
93
|
+
try {
|
|
94
|
+
return await fn();
|
|
95
|
+
} finally {
|
|
96
|
+
state.depth--;
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
state = { depth: 1 };
|
|
101
|
+
txDb[TX_STATE] = state;
|
|
102
|
+
db.exec("BEGIN DEFERRED");
|
|
103
|
+
try {
|
|
104
|
+
const result = await fn();
|
|
105
|
+
state.depth = 0;
|
|
106
|
+
db.exec("COMMIT");
|
|
107
|
+
return result;
|
|
108
|
+
} catch (error) {
|
|
109
|
+
state.depth = 0;
|
|
110
|
+
try {
|
|
111
|
+
db.exec("ROLLBACK");
|
|
112
|
+
} catch {
|
|
113
|
+
// Preserve the original error; rollback can fail if SQLite already closed the transaction.
|
|
114
|
+
}
|
|
115
|
+
throw error;
|
|
116
|
+
} finally {
|
|
117
|
+
delete txDb[TX_STATE];
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
export function closeQuietly(db: Database | undefined | null): void {
|
|
122
|
+
if (db === undefined || db === null) return;
|
|
123
|
+
try {
|
|
124
|
+
db.close();
|
|
125
|
+
} catch {
|
|
126
|
+
// Best-effort cleanup.
|
|
127
|
+
}
|
|
128
|
+
}
|
package/src/diagnose.ts
ADDED
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
import type { Database } from "bun:sqlite";
|
|
2
|
+
import { existsSync } from "node:fs";
|
|
3
|
+
import { dirname } from "node:path";
|
|
4
|
+
|
|
5
|
+
import { dataDir as configuredDataDir, dbPath as configuredDbPath } from "./config";
|
|
6
|
+
import { initBeam } from "./core/beam";
|
|
7
|
+
import { closeQuietly, openDatabase } from "./db";
|
|
8
|
+
|
|
9
|
+
export interface DiagnosticEntry {
|
|
10
|
+
readonly ts: string;
|
|
11
|
+
readonly category: string;
|
|
12
|
+
readonly check: string;
|
|
13
|
+
readonly status: string;
|
|
14
|
+
readonly detail?: string;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export interface DiagnosticSummary {
|
|
18
|
+
readonly checks_total: number;
|
|
19
|
+
readonly checks_passed: number;
|
|
20
|
+
readonly checks_failed: number;
|
|
21
|
+
readonly key_findings: string[];
|
|
22
|
+
readonly entries: DiagnosticEntry[];
|
|
23
|
+
readonly database: string;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
export interface DiagnosticOptions {
|
|
27
|
+
readonly db?: Database;
|
|
28
|
+
readonly dbPath?: string;
|
|
29
|
+
readonly dataDir?: string;
|
|
30
|
+
readonly initialize?: boolean;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
type CountRow = { count: number };
|
|
34
|
+
type IntegrityRow = { integrity_check: string };
|
|
35
|
+
type TableRow = { name: string };
|
|
36
|
+
type ColumnRow = { name: string };
|
|
37
|
+
|
|
38
|
+
const REQUIRED_TABLES = [
|
|
39
|
+
"working_memory",
|
|
40
|
+
"episodic_memory",
|
|
41
|
+
"scratchpad",
|
|
42
|
+
"fts_working",
|
|
43
|
+
"fts_episodes",
|
|
44
|
+
"memoria_facts",
|
|
45
|
+
"memoria_timelines",
|
|
46
|
+
"memoria_kg",
|
|
47
|
+
"memoria_instructions",
|
|
48
|
+
"memoria_preferences",
|
|
49
|
+
"consolidation_log",
|
|
50
|
+
"annotations",
|
|
51
|
+
"triples",
|
|
52
|
+
] as const;
|
|
53
|
+
|
|
54
|
+
const REQUIRED_COLUMNS: Readonly<Record<string, readonly string[]>> = {
|
|
55
|
+
working_memory: ["id", "content", "source", "timestamp", "session_id", "importance"],
|
|
56
|
+
episodic_memory: ["id", "content", "source", "timestamp", "session_id", "importance"],
|
|
57
|
+
scratchpad: ["id", "content", "session_id"],
|
|
58
|
+
triples: ["id", "subject", "predicate", "object"],
|
|
59
|
+
annotations: ["id", "memory_id", "kind", "value"],
|
|
60
|
+
};
|
|
61
|
+
|
|
62
|
+
function nowIso(): string {
|
|
63
|
+
return new Date().toISOString();
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
function hasTable(db: Database, table: string): boolean {
|
|
67
|
+
return (
|
|
68
|
+
(db
|
|
69
|
+
.query("SELECT name FROM sqlite_master WHERE type IN ('table', 'view') AND name = ? LIMIT 1")
|
|
70
|
+
.get(table) as TableRow | null) !== null
|
|
71
|
+
);
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
function tableColumns(db: Database, table: string): Set<string> {
|
|
75
|
+
return new Set((db.query(`PRAGMA table_info(${table})`).all() as ColumnRow[]).map(row => row.name));
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
function safeCount(db: Database, table: string): number | null {
|
|
79
|
+
if (!hasTable(db, table)) return null;
|
|
80
|
+
return (db.query(`SELECT COUNT(*) AS count FROM ${table}`).get() as CountRow).count;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
function safeEnv(name: string): string {
|
|
84
|
+
return process.env[name] ? "set" : "unset";
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
function passStatus(status: string): boolean {
|
|
88
|
+
return status === "OK" || status === "YES" || status === "set" || status === "0";
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
function failStatus(status: string): boolean {
|
|
92
|
+
return status === "MISSING" || status === "NO" || status === "ERROR" || status === "FAIL";
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
export function inspectDatabase(options: DiagnosticOptions = {}): DiagnosticSummary {
|
|
96
|
+
const path = options.dbPath ?? configuredDbPath();
|
|
97
|
+
const entries: DiagnosticEntry[] = [];
|
|
98
|
+
const log = (category: string, check: string, status: string, detail = ""): void => {
|
|
99
|
+
entries.push({ ts: nowIso(), category, check, status, detail });
|
|
100
|
+
};
|
|
101
|
+
|
|
102
|
+
log("env", "bun_version", Bun.version);
|
|
103
|
+
log("env", "platform", `${process.platform}-${process.arch}`);
|
|
104
|
+
log("env", "PROMETHEUS_MEMORY_DATA_DIR", safeEnv("PROMETHEUS_MEMORY_DATA_DIR"));
|
|
105
|
+
log("env", "PROMETHEUS_MEMORY_VEC_TYPE", safeEnv("PROMETHEUS_MEMORY_VEC_TYPE"));
|
|
106
|
+
log("db", "db_path", "OK", path);
|
|
107
|
+
log("db", "data_dir", "OK", options.dataDir ?? configuredDataDir());
|
|
108
|
+
log("db", "data_dir_parent", existsSync(dirname(path)) ? "OK" : "MISSING", dirname(path));
|
|
109
|
+
|
|
110
|
+
let db = options.db;
|
|
111
|
+
let owned = false;
|
|
112
|
+
try {
|
|
113
|
+
if (!db) {
|
|
114
|
+
db = openDatabase(path);
|
|
115
|
+
owned = true;
|
|
116
|
+
}
|
|
117
|
+
if (options.initialize !== false) initBeam(db);
|
|
118
|
+
|
|
119
|
+
const integrity = db.query("PRAGMA integrity_check").get() as IntegrityRow;
|
|
120
|
+
log("db", "integrity_check", integrity.integrity_check === "ok" ? "OK" : "FAIL", integrity.integrity_check);
|
|
121
|
+
|
|
122
|
+
for (const table of REQUIRED_TABLES) {
|
|
123
|
+
log("schema", `table:${table}`, hasTable(db, table) ? "OK" : "MISSING");
|
|
124
|
+
}
|
|
125
|
+
for (const table in REQUIRED_COLUMNS) {
|
|
126
|
+
if (!hasTable(db, table)) continue;
|
|
127
|
+
const columns = REQUIRED_COLUMNS[table];
|
|
128
|
+
if (!columns) continue;
|
|
129
|
+
const present = tableColumns(db, table);
|
|
130
|
+
const missing = columns.filter(column => !present.has(column));
|
|
131
|
+
log(
|
|
132
|
+
"schema",
|
|
133
|
+
`columns:${table}`,
|
|
134
|
+
missing.length === 0 ? "OK" : "MISSING",
|
|
135
|
+
missing.length === 0 ? `${present.size} columns` : `missing=${missing.join(",")}`,
|
|
136
|
+
);
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
for (const table of ["working_memory", "episodic_memory", "scratchpad", "triples", "annotations"] as const) {
|
|
140
|
+
const count = safeCount(db, table);
|
|
141
|
+
log("db", `${table}_count`, count === null ? "MISSING" : String(count));
|
|
142
|
+
}
|
|
143
|
+
} catch (error) {
|
|
144
|
+
log("db", "open_or_inspect", "ERROR", error instanceof Error ? error.message : String(error));
|
|
145
|
+
} finally {
|
|
146
|
+
if (owned) closeQuietly(db);
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
const keyFindings: string[] = [];
|
|
150
|
+
for (const entry of entries) {
|
|
151
|
+
if (entry.status === "MISSING") keyFindings.push(`${entry.check} missing`);
|
|
152
|
+
else if (entry.status === "FAIL" || entry.status === "ERROR") {
|
|
153
|
+
keyFindings.push(`${entry.check}: ${entry.detail ?? entry.status}`);
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
return {
|
|
158
|
+
checks_total: entries.length,
|
|
159
|
+
checks_passed: entries.filter(entry => passStatus(entry.status) || /^\d+$/.test(entry.status)).length,
|
|
160
|
+
checks_failed: entries.filter(entry => failStatus(entry.status)).length,
|
|
161
|
+
key_findings: keyFindings,
|
|
162
|
+
entries,
|
|
163
|
+
database: path,
|
|
164
|
+
};
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
export function runDiagnostics(options: DiagnosticOptions = {}): DiagnosticSummary {
|
|
168
|
+
return inspectDatabase(options);
|
|
169
|
+
}
|
|
170
|
+
if (import.meta.main) {
|
|
171
|
+
const summary = runDiagnostics();
|
|
172
|
+
console.log(JSON.stringify(summary, null, 2));
|
|
173
|
+
process.exit(summary.checks_failed === 0 ? 0 : 1);
|
|
174
|
+
}
|
package/src/dr/index.ts
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export * from "./recovery";
|
|
@@ -0,0 +1,405 @@
|
|
|
1
|
+
import { Database } from "bun:sqlite";
|
|
2
|
+
import { createHash } from "node:crypto";
|
|
3
|
+
import {
|
|
4
|
+
copyFileSync,
|
|
5
|
+
existsSync,
|
|
6
|
+
mkdirSync,
|
|
7
|
+
readdirSync,
|
|
8
|
+
readFileSync,
|
|
9
|
+
renameSync,
|
|
10
|
+
rmSync,
|
|
11
|
+
statSync,
|
|
12
|
+
unlinkSync,
|
|
13
|
+
writeFileSync,
|
|
14
|
+
} from "node:fs";
|
|
15
|
+
import { basename, dirname, extname, join } from "node:path";
|
|
16
|
+
import { gunzipSync, gzipSync } from "node:zlib";
|
|
17
|
+
import { dataDir as configuredDataDir, dbPath as configuredDbPath, type Env } from "../config";
|
|
18
|
+
import { closeQuietly, openDatabase } from "../db";
|
|
19
|
+
|
|
20
|
+
type SerializableDatabase = Database & { serialize(): Uint8Array };
|
|
21
|
+
const SQLITE_HEADER = new Uint8Array([83, 81, 76, 105, 116, 101, 32, 102, 111, 114, 109, 97, 116, 32, 51, 0]);
|
|
22
|
+
const SQLITE_SIDECAR_SUFFIXES = ["-wal", "-shm", "-journal"] as const;
|
|
23
|
+
let uniqueCounter = 0;
|
|
24
|
+
|
|
25
|
+
export interface RecoveryPaths {
|
|
26
|
+
readonly dataDir: string;
|
|
27
|
+
readonly backupDir: string;
|
|
28
|
+
readonly dbPath: string;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
export interface BackupMetadata {
|
|
32
|
+
readonly timestamp: string;
|
|
33
|
+
readonly original_size: number;
|
|
34
|
+
readonly backup_size: number;
|
|
35
|
+
readonly db_checksum: string;
|
|
36
|
+
readonly backup_checksum: string;
|
|
37
|
+
readonly compressed: true;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
export interface BackupResult extends BackupMetadata {
|
|
41
|
+
readonly backup_path: string;
|
|
42
|
+
readonly metadata_path: string;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
export interface RestoreResult {
|
|
46
|
+
readonly restored: true;
|
|
47
|
+
readonly backup_used: string;
|
|
48
|
+
readonly database_path: string;
|
|
49
|
+
readonly integrity_check: boolean;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
export interface EmergencyRestoreResult {
|
|
53
|
+
readonly restored: true;
|
|
54
|
+
readonly backup_used: string;
|
|
55
|
+
readonly attempts: number;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
export interface BackupInfo {
|
|
59
|
+
readonly file: string;
|
|
60
|
+
readonly name: string;
|
|
61
|
+
readonly size: number;
|
|
62
|
+
readonly modified: string;
|
|
63
|
+
readonly metadata?: BackupMetadata;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
export interface RotateBackupsResult {
|
|
67
|
+
readonly total_backups: number;
|
|
68
|
+
readonly kept: number;
|
|
69
|
+
readonly deleted: number;
|
|
70
|
+
readonly deleted_files: string[];
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
export interface HealthCheckResult {
|
|
74
|
+
readonly database: {
|
|
75
|
+
readonly exists: boolean;
|
|
76
|
+
readonly valid: boolean;
|
|
77
|
+
readonly path: string;
|
|
78
|
+
readonly message: string;
|
|
79
|
+
};
|
|
80
|
+
readonly backups: {
|
|
81
|
+
readonly total: number;
|
|
82
|
+
readonly latest: string | null;
|
|
83
|
+
readonly directory: string;
|
|
84
|
+
};
|
|
85
|
+
readonly status: "healthy" | "unhealthy";
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
function timestampForBackup(now = new Date()): string {
|
|
89
|
+
const pad = (value: number) => String(value).padStart(2, "0");
|
|
90
|
+
return `${now.getFullYear()}${pad(now.getMonth() + 1)}${pad(now.getDate())}_${pad(now.getHours())}${pad(now.getMinutes())}${pad(now.getSeconds())}`;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
function sha256Hex16(bytes: NodeJS.ArrayBufferView): string {
|
|
94
|
+
return createHash("sha256").update(bytes).digest("hex").slice(0, 16);
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
function nextUniqueToken(): string {
|
|
98
|
+
uniqueCounter = (uniqueCounter + 1) % 0x1fffffff;
|
|
99
|
+
return `${Date.now().toString(36)}_${process.pid.toString(36)}_${uniqueCounter.toString(36)}`;
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
function hasErrorCode(error: unknown, code: string): boolean {
|
|
103
|
+
return (
|
|
104
|
+
error !== null &&
|
|
105
|
+
typeof error === "object" &&
|
|
106
|
+
"code" in error &&
|
|
107
|
+
(error as { readonly code?: unknown }).code === code
|
|
108
|
+
);
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
function writeBackupFile(destinationDir: string, timestamp: string, bytes: Uint8Array): string {
|
|
112
|
+
for (let attempt = 0; attempt < 64; attempt += 1) {
|
|
113
|
+
const suffix = attempt === 0 ? "" : `_${nextUniqueToken()}`;
|
|
114
|
+
const backupPath = join(destinationDir, `prometheus_memory_backup_${timestamp}${suffix}.db.gz`);
|
|
115
|
+
try {
|
|
116
|
+
writeFileSync(backupPath, bytes, { flag: "wx" });
|
|
117
|
+
return backupPath;
|
|
118
|
+
} catch (error) {
|
|
119
|
+
if (hasErrorCode(error, "EEXIST")) continue;
|
|
120
|
+
throw error;
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
throw new Error(`Unable to allocate unique backup path in ${destinationDir}`);
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
function restoreTempPath(targetPath: string): string {
|
|
127
|
+
return join(dirname(targetPath), `.${basename(targetPath)}.${process.pid}.${nextUniqueToken()}.restore.tmp`);
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
function defaultBackupDir(env: Env = process.env): string {
|
|
131
|
+
const explicit = env.PROMETHEUS_MEMORY_BACKUP_DIR;
|
|
132
|
+
if (explicit !== undefined && explicit.length > 0) return explicit;
|
|
133
|
+
const dir = configuredDataDir(env);
|
|
134
|
+
return join(dirname(dir), "backups");
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
export function getDefaultPaths(env: Env = process.env): RecoveryPaths {
|
|
138
|
+
return {
|
|
139
|
+
dataDir: configuredDataDir(env),
|
|
140
|
+
backupDir: defaultBackupDir(env),
|
|
141
|
+
dbPath: configuredDbPath(env),
|
|
142
|
+
};
|
|
143
|
+
}
|
|
144
|
+
export function createBackup(dbPath?: string | null, backupDir?: string | null): BackupResult {
|
|
145
|
+
const paths = getDefaultPaths();
|
|
146
|
+
const sourcePath = dbPath ?? paths.dbPath;
|
|
147
|
+
const destinationDir = backupDir ?? paths.backupDir;
|
|
148
|
+
|
|
149
|
+
if (!existsSync(sourcePath)) throw new FileNotFoundError(`Database not found: ${sourcePath}`);
|
|
150
|
+
|
|
151
|
+
mkdirSync(destinationDir, { recursive: true });
|
|
152
|
+
const timestamp = timestampForBackup();
|
|
153
|
+
|
|
154
|
+
let snapshot: Uint8Array | null = null;
|
|
155
|
+
let sourceDb: Database | null = null;
|
|
156
|
+
try {
|
|
157
|
+
sourceDb = openDatabase(sourcePath, { create: false, readwrite: false, pragmas: false });
|
|
158
|
+
snapshot = (sourceDb as SerializableDatabase).serialize();
|
|
159
|
+
} finally {
|
|
160
|
+
closeQuietly(sourceDb);
|
|
161
|
+
}
|
|
162
|
+
if (snapshot === null) throw new Error(`Unable to serialize database backup: ${sourcePath}`);
|
|
163
|
+
const backupPath = writeBackupFile(destinationDir, timestamp, gzipSync(snapshot));
|
|
164
|
+
|
|
165
|
+
const dbBytes = readFileSync(sourcePath);
|
|
166
|
+
const backupBytes = readFileSync(backupPath);
|
|
167
|
+
const metadata: BackupMetadata = {
|
|
168
|
+
timestamp,
|
|
169
|
+
original_size: statSync(sourcePath).size,
|
|
170
|
+
backup_size: statSync(backupPath).size,
|
|
171
|
+
db_checksum: sha256Hex16(dbBytes),
|
|
172
|
+
backup_checksum: sha256Hex16(backupBytes),
|
|
173
|
+
compressed: true,
|
|
174
|
+
};
|
|
175
|
+
const metadataPath = `${backupPath.slice(0, -3)}.gz.json`;
|
|
176
|
+
writeFileSync(metadataPath, `${JSON.stringify(metadata, null, 2)}\n`);
|
|
177
|
+
|
|
178
|
+
return { backup_path: backupPath, metadata_path: metadataPath, ...metadata };
|
|
179
|
+
}
|
|
180
|
+
function isSqliteFile(bytes: Uint8Array): boolean {
|
|
181
|
+
if (bytes.length < SQLITE_HEADER.length) return false;
|
|
182
|
+
for (let i = 0; i < SQLITE_HEADER.length; i += 1) {
|
|
183
|
+
if (bytes[i] !== SQLITE_HEADER[i]) return false;
|
|
184
|
+
}
|
|
185
|
+
return true;
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
function writeGzippedSqlDump(sql: string, tempPath: string): void {
|
|
189
|
+
let db: Database | null = null;
|
|
190
|
+
try {
|
|
191
|
+
db = new Database(tempPath, { create: true, readwrite: true, strict: true });
|
|
192
|
+
db.exec(sql);
|
|
193
|
+
} finally {
|
|
194
|
+
closeQuietly(db);
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
function sqliteSidecarPath(dbPath: string, suffix: (typeof SQLITE_SIDECAR_SUFFIXES)[number]): string {
|
|
199
|
+
return `${dbPath}${suffix}`;
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
function removeSqliteSidecars(dbPath: string): void {
|
|
203
|
+
for (const suffix of SQLITE_SIDECAR_SUFFIXES) rmSync(sqliteSidecarPath(dbPath, suffix), { force: true });
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
function emergencyBackupPath(targetPath: string): string {
|
|
207
|
+
const ext = extname(targetPath);
|
|
208
|
+
if (ext.length === 0) return `${targetPath}.emergency_backup.db`;
|
|
209
|
+
return `${targetPath.slice(0, -ext.length)}.emergency_backup.db`;
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
function emergencyBackupSidecarPath(targetPath: string, suffix: (typeof SQLITE_SIDECAR_SUFFIXES)[number]): string {
|
|
213
|
+
return `${emergencyBackupPath(targetPath)}${suffix}`;
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
function snapshotCurrentDatabase(targetPath: string): void {
|
|
217
|
+
const mainBackup = emergencyBackupPath(targetPath);
|
|
218
|
+
rmSync(mainBackup, { force: true });
|
|
219
|
+
for (const suffix of SQLITE_SIDECAR_SUFFIXES)
|
|
220
|
+
rmSync(emergencyBackupSidecarPath(targetPath, suffix), { force: true });
|
|
221
|
+
if (existsSync(targetPath)) copyFileSync(targetPath, mainBackup);
|
|
222
|
+
for (const suffix of SQLITE_SIDECAR_SUFFIXES) {
|
|
223
|
+
const sidecar = sqliteSidecarPath(targetPath, suffix);
|
|
224
|
+
if (existsSync(sidecar)) copyFileSync(sidecar, emergencyBackupSidecarPath(targetPath, suffix));
|
|
225
|
+
}
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
function restoreCurrentDatabaseSnapshot(targetPath: string): void {
|
|
229
|
+
const mainBackup = emergencyBackupPath(targetPath);
|
|
230
|
+
if (!existsSync(mainBackup)) return;
|
|
231
|
+
copyFileSync(mainBackup, targetPath);
|
|
232
|
+
for (const suffix of SQLITE_SIDECAR_SUFFIXES) {
|
|
233
|
+
const sidecar = sqliteSidecarPath(targetPath, suffix);
|
|
234
|
+
rmSync(sidecar, { force: true });
|
|
235
|
+
const backupSidecar = emergencyBackupSidecarPath(targetPath, suffix);
|
|
236
|
+
if (existsSync(backupSidecar)) copyFileSync(backupSidecar, sidecar);
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
function writeRestoreCandidate(uncompressed: Buffer, tempPath: string): void {
|
|
241
|
+
if (isSqliteFile(uncompressed)) {
|
|
242
|
+
writeFileSync(tempPath, uncompressed, { flag: "wx" });
|
|
243
|
+
return;
|
|
244
|
+
}
|
|
245
|
+
writeGzippedSqlDump(uncompressed.toString("utf8"), tempPath);
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
export function restoreBackup(backupPath: string, dbPath?: string | null): RestoreResult {
|
|
249
|
+
const targetPath = dbPath ?? getDefaultPaths().dbPath;
|
|
250
|
+
if (!existsSync(backupPath)) throw new FileNotFoundError(`Backup not found: ${backupPath}`);
|
|
251
|
+
|
|
252
|
+
mkdirSync(dirname(targetPath), { recursive: true });
|
|
253
|
+
|
|
254
|
+
const uncompressed = gunzipSync(readFileSync(backupPath));
|
|
255
|
+
const tempPath = restoreTempPath(targetPath);
|
|
256
|
+
let replacedTarget = false;
|
|
257
|
+
try {
|
|
258
|
+
writeRestoreCandidate(uncompressed, tempPath);
|
|
259
|
+
if (!verifyIntegrity(tempPath)) throw new Error(`Backup failed integrity check: ${backupPath}`);
|
|
260
|
+
snapshotCurrentDatabase(targetPath);
|
|
261
|
+
renameSync(tempPath, targetPath);
|
|
262
|
+
replacedTarget = true;
|
|
263
|
+
removeSqliteSidecars(targetPath);
|
|
264
|
+
const integrity = verifyIntegrity(targetPath);
|
|
265
|
+
if (!integrity) throw new Error(`Restored database failed integrity check: ${backupPath}`);
|
|
266
|
+
return {
|
|
267
|
+
restored: true,
|
|
268
|
+
backup_used: backupPath,
|
|
269
|
+
database_path: targetPath,
|
|
270
|
+
integrity_check: integrity,
|
|
271
|
+
};
|
|
272
|
+
} catch (error) {
|
|
273
|
+
try {
|
|
274
|
+
rmSync(tempPath, { force: true });
|
|
275
|
+
} catch {
|
|
276
|
+
// Preserve the restore failure.
|
|
277
|
+
}
|
|
278
|
+
if (replacedTarget) {
|
|
279
|
+
try {
|
|
280
|
+
restoreCurrentDatabaseSnapshot(targetPath);
|
|
281
|
+
} catch {
|
|
282
|
+
// Preserve the restore failure.
|
|
283
|
+
}
|
|
284
|
+
}
|
|
285
|
+
throw error;
|
|
286
|
+
}
|
|
287
|
+
}
|
|
288
|
+
export function emergencyRestore(backupDir?: string | null, dbPath?: string | null): EmergencyRestoreResult {
|
|
289
|
+
const paths = getDefaultPaths();
|
|
290
|
+
const dir = backupDir ?? paths.backupDir;
|
|
291
|
+
const targetPath = dbPath ?? paths.dbPath;
|
|
292
|
+
const backups = existsSync(dir)
|
|
293
|
+
? readdirSync(dir)
|
|
294
|
+
.filter(name => /^prometheus_memory_backup_.*\.db\.gz$/.test(name))
|
|
295
|
+
.sort()
|
|
296
|
+
.reverse()
|
|
297
|
+
.map(name => join(dir, name))
|
|
298
|
+
: [];
|
|
299
|
+
|
|
300
|
+
if (backups.length === 0) throw new FileNotFoundError(`No backups found in ${dir}`);
|
|
301
|
+
|
|
302
|
+
let attempts = 0;
|
|
303
|
+
for (const backup of backups) {
|
|
304
|
+
attempts += 1;
|
|
305
|
+
try {
|
|
306
|
+
const result = restoreBackup(backup, targetPath);
|
|
307
|
+
if (result.integrity_check) return { restored: true, backup_used: backup, attempts };
|
|
308
|
+
} catch {
|
|
309
|
+
// Try the next backup, matching the Python recovery behavior.
|
|
310
|
+
}
|
|
311
|
+
}
|
|
312
|
+
throw new Error("All backups failed integrity check");
|
|
313
|
+
}
|
|
314
|
+
export function verifyIntegrity(dbPath?: string | null): boolean {
|
|
315
|
+
const targetPath = dbPath ?? getDefaultPaths().dbPath;
|
|
316
|
+
if (!existsSync(targetPath)) return false;
|
|
317
|
+
|
|
318
|
+
let db: Database | null = null;
|
|
319
|
+
try {
|
|
320
|
+
db = openDatabase(targetPath, { create: false, readwrite: false, pragmas: false });
|
|
321
|
+
const row = db.query("PRAGMA integrity_check").get() as { integrity_check: string } | null;
|
|
322
|
+
return row?.integrity_check === "ok";
|
|
323
|
+
} catch {
|
|
324
|
+
return false;
|
|
325
|
+
} finally {
|
|
326
|
+
closeQuietly(db);
|
|
327
|
+
}
|
|
328
|
+
}
|
|
329
|
+
export function listBackups(backupDir?: string | null): BackupInfo[] {
|
|
330
|
+
const dir = backupDir ?? getDefaultPaths().backupDir;
|
|
331
|
+
if (!existsSync(dir)) return [];
|
|
332
|
+
|
|
333
|
+
return readdirSync(dir)
|
|
334
|
+
.filter(name => /^prometheus_memory_backup_.*\.db\.gz$/.test(name))
|
|
335
|
+
.sort()
|
|
336
|
+
.reverse()
|
|
337
|
+
.map(name => {
|
|
338
|
+
const file = join(dir, name);
|
|
339
|
+
const stat = statSync(file);
|
|
340
|
+
const metaFile = `${file.slice(0, -3)}.gz.json`;
|
|
341
|
+
const info: BackupInfo = {
|
|
342
|
+
file,
|
|
343
|
+
name,
|
|
344
|
+
size: stat.size,
|
|
345
|
+
modified: stat.mtime.toISOString(),
|
|
346
|
+
};
|
|
347
|
+
if (!existsSync(metaFile)) return info;
|
|
348
|
+
return { ...info, metadata: JSON.parse(readFileSync(metaFile, "utf8")) as BackupMetadata };
|
|
349
|
+
});
|
|
350
|
+
}
|
|
351
|
+
export function rotateBackups(backupDir?: string | null, keep = 10): RotateBackupsResult {
|
|
352
|
+
const dir = backupDir ?? getDefaultPaths().backupDir;
|
|
353
|
+
const backups = existsSync(dir)
|
|
354
|
+
? readdirSync(dir)
|
|
355
|
+
.filter(name => /^prometheus_memory_backup_.*\.db\.gz$/.test(name))
|
|
356
|
+
.sort()
|
|
357
|
+
.map(name => join(dir, name))
|
|
358
|
+
: [];
|
|
359
|
+
const toDelete = backups.length > keep ? backups.slice(0, backups.length - keep) : [];
|
|
360
|
+
const deletedFiles: string[] = [];
|
|
361
|
+
for (const backup of toDelete) {
|
|
362
|
+
unlinkSync(backup);
|
|
363
|
+
const meta = `${backup.slice(0, -3)}.gz.json`;
|
|
364
|
+
if (existsSync(meta)) unlinkSync(meta);
|
|
365
|
+
deletedFiles.push(basename(backup));
|
|
366
|
+
}
|
|
367
|
+
return {
|
|
368
|
+
total_backups: backups.length,
|
|
369
|
+
kept: keep,
|
|
370
|
+
deleted: deletedFiles.length,
|
|
371
|
+
deleted_files: deletedFiles,
|
|
372
|
+
};
|
|
373
|
+
}
|
|
374
|
+
export function healthCheck(): HealthCheckResult {
|
|
375
|
+
const paths = getDefaultPaths();
|
|
376
|
+
const dbExists = existsSync(paths.dbPath);
|
|
377
|
+
const dbValid = dbExists ? verifyIntegrity(paths.dbPath) : false;
|
|
378
|
+
const backups = listBackups(paths.backupDir)
|
|
379
|
+
.map(backup => backup.file)
|
|
380
|
+
.sort();
|
|
381
|
+
return {
|
|
382
|
+
database: {
|
|
383
|
+
exists: dbExists,
|
|
384
|
+
valid: dbValid,
|
|
385
|
+
path: paths.dbPath,
|
|
386
|
+
message: dbValid ? "Database integrity verified" : "Database missing or corrupt",
|
|
387
|
+
},
|
|
388
|
+
backups: {
|
|
389
|
+
total: backups.length,
|
|
390
|
+
latest: backups.at(-1) ?? null,
|
|
391
|
+
directory: paths.backupDir,
|
|
392
|
+
},
|
|
393
|
+
status: dbValid ? "healthy" : "unhealthy",
|
|
394
|
+
};
|
|
395
|
+
}
|
|
396
|
+
export class FileNotFoundError extends Error {
|
|
397
|
+
constructor(message: string) {
|
|
398
|
+
super(message);
|
|
399
|
+
this.name = "FileNotFoundError";
|
|
400
|
+
}
|
|
401
|
+
}
|
|
402
|
+
|
|
403
|
+
export function resetRecoveryForTests(): void {
|
|
404
|
+
// Recovery has no module state; exported for test harness symmetry.
|
|
405
|
+
}
|