omp-fabric 1.4.4 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +26 -0
- package/dist/chunks/{chunk-TR3LQHO7.js → chunk-2ZEMWIHZ.js} +2 -2
- package/dist/chunks/chunk-3XPR2AZL.js +74 -0
- package/dist/chunks/chunk-3XPR2AZL.js.map +7 -0
- package/dist/chunks/{chunk-SZWF6C4V.js → chunk-4MVTUMKO.js} +3 -3
- package/dist/chunks/{chunk-SG4GVSWX.js → chunk-EQSL47QL.js} +10 -66
- package/dist/chunks/chunk-EQSL47QL.js.map +7 -0
- package/dist/chunks/{chunk-OW2LSF5R.js → chunk-IGNMZZQH.js} +3 -3
- package/dist/chunks/{chunk-LB2EC7FZ.js → chunk-JMYHQUJQ.js} +9 -105
- package/dist/chunks/chunk-JMYHQUJQ.js.map +7 -0
- package/dist/chunks/{chunk-6JPXLXDS.js → chunk-KHQJKGBC.js} +2 -2
- package/dist/chunks/{chunk-6JPXLXDS.js.map → chunk-KHQJKGBC.js.map} +1 -1
- package/dist/chunks/chunk-KTB4R3E3.js +51 -0
- package/dist/chunks/chunk-KTB4R3E3.js.map +7 -0
- package/dist/chunks/chunk-TQQPXV3I.js +58 -0
- package/dist/chunks/chunk-TQQPXV3I.js.map +7 -0
- package/dist/chunks/{chunk-2J2GOB4A.js → chunk-VDLZAORG.js} +5 -3
- package/dist/chunks/chunk-VDLZAORG.js.map +7 -0
- package/dist/chunks/chunk-ZDWBTQB6.js +107 -0
- package/dist/chunks/chunk-ZDWBTQB6.js.map +7 -0
- package/dist/compaction/hook.js +2 -1
- package/dist/compaction/lcm-loader.d.ts +12 -0
- package/dist/compaction/lcm-loader.d.ts.map +1 -0
- package/dist/compaction/lcm-maintenance.d.ts +2 -1
- package/dist/compaction/lcm-maintenance.d.ts.map +1 -1
- package/dist/compaction/lcm-runtime.d.ts +1 -0
- package/dist/compaction/lcm-runtime.d.ts.map +1 -1
- package/dist/compaction/lcm-runtime.js +1169 -0
- package/dist/compaction/lcm-runtime.js.map +7 -0
- package/dist/config.d.ts +1 -0
- package/dist/config.d.ts.map +1 -1
- package/dist/fabric-runtime-state.js +9 -7
- package/dist/fabric-runtime-state.js.map +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +135 -989
- package/dist/index.js.map +4 -4
- package/dist/memory/lcm-adapter.d.ts +1 -1
- package/dist/memory/lcm-adapter.d.ts.map +1 -1
- package/dist/providers/memory-provider.js +2 -2
- package/dist/residency/host.js +8 -6
- package/dist/residency/omp-entry.js +8 -6
- package/dist/residency/omp-entry.js.map +1 -1
- package/dist/storage/lcm-identity.d.ts +43 -0
- package/dist/storage/lcm-identity.d.ts.map +1 -0
- package/dist/storage/lcm-ledger.d.ts +7 -46
- package/dist/storage/lcm-ledger.d.ts.map +1 -1
- package/dist/storage/sqlite.d.ts +19 -0
- package/dist/storage/sqlite.d.ts.map +1 -0
- package/dist/ui/dashboard.js +3 -3
- package/dist/ui/settings.js +4 -4
- package/docs/compaction.md +8 -1
- package/package.json +1 -1
- package/dist/chunks/chunk-2J2GOB4A.js.map +0 -7
- package/dist/chunks/chunk-LB2EC7FZ.js.map +0 -7
- package/dist/chunks/chunk-MBOXJKF6.js +0 -311
- package/dist/chunks/chunk-MBOXJKF6.js.map +0 -7
- package/dist/chunks/chunk-SG4GVSWX.js.map +0 -7
- /package/dist/chunks/{chunk-TR3LQHO7.js.map → chunk-2ZEMWIHZ.js.map} +0 -0
- /package/dist/chunks/{chunk-SZWF6C4V.js.map → chunk-4MVTUMKO.js.map} +0 -0
- /package/dist/chunks/{chunk-OW2LSF5R.js.map → chunk-IGNMZZQH.js.map} +0 -0
|
@@ -0,0 +1,1169 @@
|
|
|
1
|
+
import {
|
|
2
|
+
sqliteDriver
|
|
3
|
+
} from "../chunks/chunk-TQQPXV3I.js";
|
|
4
|
+
import {
|
|
5
|
+
sessionDirNamesForCwd,
|
|
6
|
+
sessionsDirRoot
|
|
7
|
+
} from "../chunks/chunk-FX3XLQYI.js";
|
|
8
|
+
import "../chunks/chunk-6QBPSZJ5.js";
|
|
9
|
+
import {
|
|
10
|
+
LcmModelAdapter,
|
|
11
|
+
buildLcmPrompt,
|
|
12
|
+
emergencyReduce
|
|
13
|
+
} from "../chunks/chunk-ZDWBTQB6.js";
|
|
14
|
+
import {
|
|
15
|
+
canonicalLcmPayload,
|
|
16
|
+
canonicalProjectIdentity,
|
|
17
|
+
defaultLedgerPath,
|
|
18
|
+
hash,
|
|
19
|
+
hashLcmPayload
|
|
20
|
+
} from "../chunks/chunk-KTB4R3E3.js";
|
|
21
|
+
import "../chunks/chunk-3XPR2AZL.js";
|
|
22
|
+
import "../chunks/chunk-4HQDSTHN.js";
|
|
23
|
+
import "../chunks/chunk-T5ZA5YRU.js";
|
|
24
|
+
import "../chunks/chunk-6N4XGAZY.js";
|
|
25
|
+
|
|
26
|
+
// src/storage/lcm-ledger.ts
|
|
27
|
+
import fs from "node:fs";
|
|
28
|
+
import path from "node:path";
|
|
29
|
+
import crypto from "node:crypto";
|
|
30
|
+
var LCM_LEDGER_WARNING_BYTES = 8 * 1024 ** 3;
|
|
31
|
+
var LCM_LEDGER_MAINTENANCE_BYTES = 10 * 1024 ** 3;
|
|
32
|
+
var SCHEMA = `
|
|
33
|
+
CREATE TABLE IF NOT EXISTS schema_metadata (key TEXT PRIMARY KEY, value TEXT NOT NULL);
|
|
34
|
+
CREATE TABLE IF NOT EXISTS projects (project_key TEXT PRIMARY KEY, identity_json TEXT NOT NULL, created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL);
|
|
35
|
+
CREATE TABLE IF NOT EXISTS project_aliases (alias TEXT PRIMARY KEY, project_key TEXT NOT NULL REFERENCES projects(project_key));
|
|
36
|
+
CREATE TABLE IF NOT EXISTS sessions (project_key TEXT NOT NULL, session_id TEXT NOT NULL, created_at INTEGER NOT NULL, PRIMARY KEY(project_key, session_id));
|
|
37
|
+
CREATE TABLE IF NOT EXISTS raw_entries (project_key TEXT NOT NULL, session_id TEXT NOT NULL, entry_id TEXT NOT NULL, revision INTEGER NOT NULL, role TEXT NOT NULL, content TEXT NOT NULL, content_hash TEXT NOT NULL, payload_json TEXT NOT NULL, parent_entry_id TEXT, branch TEXT, created_at INTEGER NOT NULL, PRIMARY KEY(project_key, session_id, entry_id, revision), UNIQUE(project_key, session_id, entry_id, content_hash));
|
|
38
|
+
CREATE TABLE IF NOT EXISTS summary_nodes (node_id TEXT PRIMARY KEY, project_key TEXT NOT NULL, payload TEXT NOT NULL, created_at INTEGER NOT NULL);
|
|
39
|
+
CREATE TABLE IF NOT EXISTS summary_edges (parent_id TEXT NOT NULL, child_id TEXT NOT NULL, PRIMARY KEY(parent_id, child_id));
|
|
40
|
+
CREATE TABLE IF NOT EXISTS frontiers (project_key TEXT NOT NULL, frontier_id TEXT NOT NULL, node_id TEXT NOT NULL, created_at INTEGER NOT NULL, PRIMARY KEY(project_key, frontier_id, node_id));
|
|
41
|
+
CREATE TABLE IF NOT EXISTS maintenance_jobs (job_id TEXT PRIMARY KEY, project_key TEXT NOT NULL, status TEXT NOT NULL, payload TEXT NOT NULL, created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL);
|
|
42
|
+
CREATE TABLE IF NOT EXISTS maintenance_usage (project_key TEXT NOT NULL, day TEXT NOT NULL, session_id TEXT NOT NULL, calls INTEGER NOT NULL, input_tokens INTEGER NOT NULL, output_tokens INTEGER NOT NULL, cost REAL NOT NULL, wall_ms INTEGER NOT NULL, PRIMARY KEY(project_key,day,session_id));
|
|
43
|
+
CREATE INDEX IF NOT EXISTS raw_entries_lookup ON raw_entries(project_key, session_id, entry_id, revision);
|
|
44
|
+
`;
|
|
45
|
+
var fileHash = (file) => crypto.createHash("sha256").update(fs.readFileSync(file)).digest("hex");
|
|
46
|
+
var snapshotHash = (db) => {
|
|
47
|
+
const digest = crypto.createHash("sha256");
|
|
48
|
+
for (const [table, order] of [["schema_metadata", "key"], ["projects", "project_key"], ["project_aliases", "alias"], ["sessions", "project_key,session_id"], ["raw_entries", "project_key,session_id,entry_id,revision"], ["summary_nodes", "node_id"], ["summary_edges", "parent_id,child_id"], ["frontiers", "project_key,frontier_id,node_id"], ["maintenance_jobs", "job_id"], ["maintenance_usage", "project_key,day,session_id"]]) {
|
|
49
|
+
digest.update(`${table}\0`);
|
|
50
|
+
for (const row of db.prepare(`SELECT * FROM ${table} ORDER BY ${order}`).all()) digest.update(`${JSON.stringify(row)}\0`);
|
|
51
|
+
}
|
|
52
|
+
return digest.digest("hex");
|
|
53
|
+
};
|
|
54
|
+
var LedgerDegradedError = class extends Error {
|
|
55
|
+
constructor(message, cause) {
|
|
56
|
+
super(message);
|
|
57
|
+
this.cause = cause;
|
|
58
|
+
this.name = "LedgerDegradedError";
|
|
59
|
+
}
|
|
60
|
+
};
|
|
61
|
+
var LcmLedger = class {
|
|
62
|
+
db;
|
|
63
|
+
project;
|
|
64
|
+
degraded = false;
|
|
65
|
+
dbPath;
|
|
66
|
+
now;
|
|
67
|
+
warningBytes;
|
|
68
|
+
maintenanceBytes;
|
|
69
|
+
writeChain = Promise.resolve();
|
|
70
|
+
transactionDepth = 0;
|
|
71
|
+
closed = false;
|
|
72
|
+
constructor(options = {}) {
|
|
73
|
+
this.now = options.now ?? Date.now;
|
|
74
|
+
this.warningBytes = options.warningBytes ?? LCM_LEDGER_WARNING_BYTES;
|
|
75
|
+
this.maintenanceBytes = options.maintenanceBytes ?? LCM_LEDGER_MAINTENANCE_BYTES;
|
|
76
|
+
if (!Number.isSafeInteger(this.warningBytes) || !Number.isSafeInteger(this.maintenanceBytes) || this.warningBytes < 0 || this.maintenanceBytes < this.warningBytes) {
|
|
77
|
+
throw new Error("invalid ledger size thresholds");
|
|
78
|
+
}
|
|
79
|
+
this.project = canonicalProjectIdentity(options.project ?? { liveCwd: process.cwd() });
|
|
80
|
+
const dbPath = options.dbPath ?? defaultLedgerPath(options.rootDir, this.project.key);
|
|
81
|
+
fs.mkdirSync(path.dirname(dbPath), { recursive: true });
|
|
82
|
+
this.dbPath = dbPath;
|
|
83
|
+
this.db = new (sqliteDriver())(dbPath);
|
|
84
|
+
try {
|
|
85
|
+
this.db.exec("PRAGMA journal_mode=WAL; PRAGMA synchronous=FULL; PRAGMA busy_timeout=5000; PRAGMA wal_autocheckpoint=1000;");
|
|
86
|
+
this.db.exec("BEGIN IMMEDIATE;" + SCHEMA + "INSERT INTO schema_metadata(key,value) VALUES ('version','1') ON CONFLICT(key) DO UPDATE SET value='1'; COMMIT;");
|
|
87
|
+
const columns = this.db.prepare("PRAGMA table_info(raw_entries)").all();
|
|
88
|
+
if (!columns.some((column) => column.name === "payload_json")) {
|
|
89
|
+
if (Number(this.db.prepare("SELECT count(*) n FROM raw_entries").get().n)) throw new Error("raw payloads missing; reimport authoritative session entries");
|
|
90
|
+
this.db.exec("ALTER TABLE raw_entries ADD COLUMN payload_json TEXT NOT NULL DEFAULT ''");
|
|
91
|
+
}
|
|
92
|
+
const result = this.db.prepare("PRAGMA integrity_check").get();
|
|
93
|
+
if (result.integrity_check !== "ok") throw new Error(String(result.integrity_check));
|
|
94
|
+
this.registerProject(this.project);
|
|
95
|
+
} catch (error) {
|
|
96
|
+
this.degraded = true;
|
|
97
|
+
throw new LedgerDegradedError("ledger startup failed", error);
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
get isDegraded() {
|
|
101
|
+
return this.degraded;
|
|
102
|
+
}
|
|
103
|
+
get operationalState() {
|
|
104
|
+
if (this.degraded) return "degraded";
|
|
105
|
+
try {
|
|
106
|
+
const size = fs.statSync(this.dbPath).size;
|
|
107
|
+
if (size >= this.maintenanceBytes) return "maintenance";
|
|
108
|
+
if (size >= this.warningBytes) return "warning";
|
|
109
|
+
} catch {
|
|
110
|
+
}
|
|
111
|
+
return "healthy";
|
|
112
|
+
}
|
|
113
|
+
markDegraded(error) {
|
|
114
|
+
this.degraded = true;
|
|
115
|
+
return new LedgerDegradedError("ledger is degraded", error);
|
|
116
|
+
}
|
|
117
|
+
registerProject(identity) {
|
|
118
|
+
const now = this.now();
|
|
119
|
+
this.db.prepare("INSERT INTO projects(project_key,identity_json,created_at,updated_at) VALUES(?,?,?,?) ON CONFLICT(project_key) DO UPDATE SET identity_json=excluded.identity_json,updated_at=excluded.updated_at").run(identity.key, JSON.stringify(identity), now, now);
|
|
120
|
+
for (const alias of identity.aliases) {
|
|
121
|
+
const existing = this.db.prepare("SELECT project_key FROM project_aliases WHERE alias=?").get(alias);
|
|
122
|
+
if (existing && existing.project_key !== identity.key) throw new LedgerDegradedError(`ambiguous project alias: ${alias}`);
|
|
123
|
+
this.db.prepare("INSERT OR IGNORE INTO project_aliases(alias,project_key) VALUES(?,?)").run(alias, identity.key);
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
guard() {
|
|
127
|
+
if (this.degraded) throw new LedgerDegradedError("ledger is degraded");
|
|
128
|
+
}
|
|
129
|
+
assertProjectKey(projectKey) {
|
|
130
|
+
if (projectKey !== this.project.key) throw new Error("project key does not match ledger project");
|
|
131
|
+
}
|
|
132
|
+
async serialize(operation) {
|
|
133
|
+
const previous = this.writeChain;
|
|
134
|
+
let release;
|
|
135
|
+
this.writeChain = new Promise((resolve) => {
|
|
136
|
+
release = resolve;
|
|
137
|
+
});
|
|
138
|
+
await previous;
|
|
139
|
+
try {
|
|
140
|
+
this.guard();
|
|
141
|
+
return operation();
|
|
142
|
+
} finally {
|
|
143
|
+
release();
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
appendRaw(entry) {
|
|
147
|
+
this.guard();
|
|
148
|
+
if (entry.projectKey !== this.project.key) throw new Error("entry project does not match ledger project");
|
|
149
|
+
if (entry.entryId.trim().length === 0) throw new Error("entry ID is required");
|
|
150
|
+
if (typeof entry.payloadJson !== "string" || !entry.payloadJson.trim()) throw new Error("full payload JSON is required");
|
|
151
|
+
let payload;
|
|
152
|
+
try {
|
|
153
|
+
payload = JSON.parse(entry.payloadJson);
|
|
154
|
+
} catch {
|
|
155
|
+
throw new Error("invalid payload JSON");
|
|
156
|
+
}
|
|
157
|
+
if (!payload || typeof payload !== "object" || typeof payload.type !== "string" || typeof payload.id !== "string" || payload.id !== entry.entryId) throw new Error("payload must be a full SessionEntry");
|
|
158
|
+
const payloadJson = canonicalLcmPayload(payload);
|
|
159
|
+
const contentHash = hashLcmPayload(payload);
|
|
160
|
+
const ownsTransaction = this.transactionDepth === 0;
|
|
161
|
+
if (ownsTransaction) this.db.exec("BEGIN IMMEDIATE");
|
|
162
|
+
try {
|
|
163
|
+
const existing = this.db.prepare("SELECT revision,created_at FROM raw_entries WHERE project_key=? AND session_id=? AND entry_id=? AND content_hash=?").get(entry.projectKey, entry.sessionId, entry.entryId, contentHash);
|
|
164
|
+
if (existing) {
|
|
165
|
+
if (ownsTransaction) this.db.exec("COMMIT");
|
|
166
|
+
return { ...entry, payloadJson, payloadHash: contentHash, revision: existing.revision, contentHash, createdAt: existing.created_at };
|
|
167
|
+
}
|
|
168
|
+
const latest = this.db.prepare("SELECT COALESCE(MAX(revision),0) revision FROM raw_entries WHERE project_key=? AND session_id=? AND entry_id=?").get(entry.projectKey, entry.sessionId, entry.entryId);
|
|
169
|
+
const revision = latest.revision + 1;
|
|
170
|
+
const createdAt = entry.createdAt ?? this.now();
|
|
171
|
+
this.db.prepare("INSERT OR IGNORE INTO sessions(project_key,session_id,created_at) VALUES(?,?,?)").run(entry.projectKey, entry.sessionId, createdAt);
|
|
172
|
+
this.db.prepare("INSERT INTO raw_entries(project_key,session_id,entry_id,revision,role,content,content_hash,payload_json,parent_entry_id,branch,created_at) VALUES(?,?,?,?,?,?,?,?,?,?,?)").run(entry.projectKey, entry.sessionId, entry.entryId, revision, entry.role, entry.content, contentHash, payloadJson, entry.parentEntryId ?? null, entry.branch ?? null, createdAt);
|
|
173
|
+
if (ownsTransaction) this.db.exec("COMMIT");
|
|
174
|
+
return { ...entry, payloadJson, payloadHash: contentHash, revision, contentHash, createdAt };
|
|
175
|
+
} catch (error) {
|
|
176
|
+
if (ownsTransaction) this.db.exec("ROLLBACK");
|
|
177
|
+
const code = error.code;
|
|
178
|
+
if (code === "ENOSPC" || code === "SQLITE_FULL" || String(error).includes("database or disk is full")) this.degraded = true;
|
|
179
|
+
throw new LedgerDegradedError("ledger write failed", error);
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
readRaw(projectKey = this.project.key, sessionId) {
|
|
183
|
+
this.guard();
|
|
184
|
+
this.assertProjectKey(projectKey);
|
|
185
|
+
const rows = sessionId ? this.db.prepare("SELECT * FROM raw_entries WHERE project_key=? AND session_id=? ORDER BY created_at,revision").all(projectKey, sessionId) : this.db.prepare("SELECT * FROM raw_entries WHERE project_key=? ORDER BY created_at,session_id,entry_id,revision").all(projectKey);
|
|
186
|
+
return rows.map((row) => ({ projectKey: row.project_key, sessionId: row.session_id, entryId: row.entry_id, revision: row.revision, role: row.role, content: row.content, contentHash: row.content_hash, payloadHash: row.content_hash, payloadJson: row.payload_json, parentEntryId: row.parent_entry_id, branch: row.branch, createdAt: row.created_at }));
|
|
187
|
+
}
|
|
188
|
+
readRawPage(projectKey = this.project.key, sessionId, offset = 0, limit = 100) {
|
|
189
|
+
if (!Number.isSafeInteger(offset) || offset < 0 || !Number.isSafeInteger(limit) || limit < 1) throw new Error("invalid raw page");
|
|
190
|
+
this.guard();
|
|
191
|
+
this.assertProjectKey(projectKey);
|
|
192
|
+
const rows = sessionId ? this.db.prepare("SELECT * FROM raw_entries WHERE project_key=? AND session_id=? ORDER BY created_at,revision LIMIT ? OFFSET ?").all(projectKey, sessionId, limit, offset) : this.db.prepare("SELECT * FROM raw_entries WHERE project_key=? ORDER BY created_at,session_id,entry_id,revision LIMIT ? OFFSET ?").all(projectKey, limit, offset);
|
|
193
|
+
return rows.map((row) => ({ projectKey: row.project_key, sessionId: row.session_id, entryId: row.entry_id, revision: row.revision, role: row.role, content: row.content, contentHash: row.content_hash, payloadHash: row.content_hash, payloadJson: row.payload_json, parentEntryId: row.parent_entry_id, branch: row.branch, createdAt: row.created_at }));
|
|
194
|
+
}
|
|
195
|
+
readRawEntry(projectKey, sessionId, entryId, revision) {
|
|
196
|
+
this.guard();
|
|
197
|
+
this.assertProjectKey(projectKey);
|
|
198
|
+
const row = this.db.prepare("SELECT * FROM raw_entries WHERE project_key=? AND session_id=? AND entry_id=? AND revision=?").get(projectKey, sessionId, entryId, revision);
|
|
199
|
+
return row ? { projectKey: row.project_key, sessionId: row.session_id, entryId: row.entry_id, revision: row.revision, role: row.role, content: row.content, contentHash: row.content_hash, payloadHash: row.content_hash, payloadJson: row.payload_json, parentEntryId: row.parent_entry_id, branch: row.branch, createdAt: row.created_at } : void 0;
|
|
200
|
+
}
|
|
201
|
+
readOnly(fn) {
|
|
202
|
+
this.guard();
|
|
203
|
+
return fn(this.db);
|
|
204
|
+
}
|
|
205
|
+
transaction(fn) {
|
|
206
|
+
this.guard();
|
|
207
|
+
if (this.transactionDepth > 0) return fn(this.db);
|
|
208
|
+
this.db.exec("BEGIN IMMEDIATE");
|
|
209
|
+
this.transactionDepth = 1;
|
|
210
|
+
try {
|
|
211
|
+
const result = fn(this.db);
|
|
212
|
+
this.db.exec("COMMIT");
|
|
213
|
+
return result;
|
|
214
|
+
} catch (error) {
|
|
215
|
+
this.db.exec("ROLLBACK");
|
|
216
|
+
throw error;
|
|
217
|
+
} finally {
|
|
218
|
+
this.transactionDepth = 0;
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
checkpoint(mode = "passive") {
|
|
222
|
+
const row = this.db.prepare(`PRAGMA wal_checkpoint(${mode.toUpperCase()})`).get();
|
|
223
|
+
const busy = row.busy ?? 0;
|
|
224
|
+
const logPages = row.log ?? 0;
|
|
225
|
+
return { mode, busy, logPages, checkpointedPages: row.checkpointed ?? 0, truncated: mode === "truncate" && busy === 0 && logPages === 0 };
|
|
226
|
+
}
|
|
227
|
+
backup(destination) {
|
|
228
|
+
this.guard();
|
|
229
|
+
return this.serializeSync(() => {
|
|
230
|
+
const target = path.resolve(destination);
|
|
231
|
+
fs.mkdirSync(path.dirname(target), { recursive: true });
|
|
232
|
+
if (fs.existsSync(target)) throw new Error("backup destination exists");
|
|
233
|
+
this.db.exec(`VACUUM INTO '${target.replace(/'/g, "''")}'`);
|
|
234
|
+
fs.chmodSync(target, 384);
|
|
235
|
+
const copy = new (sqliteDriver())(target);
|
|
236
|
+
let integrity = "unknown";
|
|
237
|
+
let backupStateSha256 = "";
|
|
238
|
+
const rowCounts = {};
|
|
239
|
+
try {
|
|
240
|
+
integrity = copy.prepare("PRAGMA integrity_check").get().integrity_check ?? "unknown";
|
|
241
|
+
for (const table of ["projects", "sessions", "raw_entries", "summary_nodes", "summary_edges", "frontiers", "maintenance_jobs", "maintenance_usage"]) {
|
|
242
|
+
rowCounts[table] = Number(copy.prepare(`SELECT count(*) n FROM ${table}`).get().n);
|
|
243
|
+
}
|
|
244
|
+
backupStateSha256 = snapshotHash(copy);
|
|
245
|
+
} finally {
|
|
246
|
+
copy.close();
|
|
247
|
+
}
|
|
248
|
+
const manifest = { format: "lcm-ledger-backup", version: 1, source: this.dbPath, destination: target, sourceSha256: fileHash(this.dbPath), backupSha256: fileHash(target), sourceStateSha256: backupStateSha256, rowCounts, integrity, createdAt: this.now() };
|
|
249
|
+
fs.writeFileSync(`${target}.manifest.json`, JSON.stringify(manifest), { mode: 384 });
|
|
250
|
+
fs.chmodSync(`${target}.manifest.json`, 384);
|
|
251
|
+
return manifest;
|
|
252
|
+
});
|
|
253
|
+
}
|
|
254
|
+
exportBackup(destination) {
|
|
255
|
+
return this.backup(destination);
|
|
256
|
+
}
|
|
257
|
+
deleteProject(token2, backupManifest) {
|
|
258
|
+
this.guard();
|
|
259
|
+
if (token2.__brand !== "DeleteConfirmationToken" || token2.projectKey !== this.project.key || token2.value !== hash(`delete:${this.project.key}`)) throw new Error("invalid delete confirmation token");
|
|
260
|
+
const backupDb = fs.existsSync(backupManifest.destination) ? new (sqliteDriver())(backupManifest.destination) : void 0;
|
|
261
|
+
let backupStateSha256;
|
|
262
|
+
try {
|
|
263
|
+
if (backupDb) backupStateSha256 = snapshotHash(backupDb);
|
|
264
|
+
} finally {
|
|
265
|
+
backupDb?.close();
|
|
266
|
+
}
|
|
267
|
+
if (backupManifest.integrity !== "ok" || backupManifest.source !== this.dbPath || !backupManifest.backupSha256 || backupManifest.backupSha256 !== fileHash(backupManifest.destination) || !backupManifest.sourceStateSha256 || backupManifest.sourceStateSha256 !== backupStateSha256 || backupManifest.sourceSha256 !== fileHash(this.dbPath) || backupManifest.sourceStateSha256 !== snapshotHash(this.db)) throw new Error("backup verification failed");
|
|
268
|
+
this.transaction((db) => {
|
|
269
|
+
db.prepare("DELETE FROM summary_edges WHERE parent_id IN (SELECT node_id FROM summary_nodes WHERE project_key=?) OR child_id IN (SELECT node_id FROM summary_nodes WHERE project_key=?)").run(this.project.key, this.project.key);
|
|
270
|
+
for (const table of ["raw_entries", "sessions", "frontiers", "summary_nodes", "maintenance_jobs", "maintenance_usage"]) db.prepare(`DELETE FROM ${table} WHERE project_key=?`).run(this.project.key);
|
|
271
|
+
db.prepare("DELETE FROM project_aliases WHERE project_key=?").run(this.project.key);
|
|
272
|
+
db.prepare("DELETE FROM projects WHERE project_key=?").run(this.project.key);
|
|
273
|
+
});
|
|
274
|
+
const integrity = this.db.prepare("PRAGMA integrity_check").get().integrity_check;
|
|
275
|
+
if (integrity !== "ok") throw new Error(`post-delete integrity failed: ${integrity}`);
|
|
276
|
+
const audit = { format: "lcm-ledger-delete", version: 1, projectKey: this.project.key, deletedAt: this.now(), integrity, remaining: Number(this.db.prepare("SELECT count(*) n FROM projects WHERE project_key=?").get(this.project.key).n) };
|
|
277
|
+
fs.writeFileSync(`${backupManifest.destination}.delete-manifest.json`, JSON.stringify(audit), { mode: 384 });
|
|
278
|
+
}
|
|
279
|
+
serializeSync(operation) {
|
|
280
|
+
this.guard();
|
|
281
|
+
return operation();
|
|
282
|
+
}
|
|
283
|
+
close() {
|
|
284
|
+
if (this.closed) return;
|
|
285
|
+
this.db.close();
|
|
286
|
+
this.closed = true;
|
|
287
|
+
}
|
|
288
|
+
};
|
|
289
|
+
|
|
290
|
+
// src/storage/lcm-migration.ts
|
|
291
|
+
import crypto2 from "node:crypto";
|
|
292
|
+
import fs2 from "node:fs";
|
|
293
|
+
import path2 from "node:path";
|
|
294
|
+
var hash2 = (data) => crypto2.createHash("sha256").update(data).digest("hex");
|
|
295
|
+
var record = (value) => value !== null && typeof value === "object" && !Array.isArray(value) ? value : void 0;
|
|
296
|
+
var nonblank = (value) => typeof value === "string" && value.trim().length > 0;
|
|
297
|
+
var utc = (value) => {
|
|
298
|
+
if (!/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,3})?(?:Z|\+00:00)$/.test(value)) throw new Error("timestamps must be explicit UTC ISO-8601 values");
|
|
299
|
+
const at = Date.parse(value);
|
|
300
|
+
const normalized = value.replace("+00:00", "Z").replace(/Z$/, "");
|
|
301
|
+
if (!Number.isFinite(at) || new Date(at).toISOString().slice(0, 19) !== normalized.slice(0, 19)) throw new Error("invalid UTC timestamp");
|
|
302
|
+
return at;
|
|
303
|
+
};
|
|
304
|
+
function migrationWindow(options) {
|
|
305
|
+
const now = options.now ?? Date.now();
|
|
306
|
+
const since = options.since === void 0 ? now - 72 * 60 * 60 * 1e3 : utc(options.since);
|
|
307
|
+
const until = options.until === void 0 ? now : utc(options.until);
|
|
308
|
+
if (!Number.isFinite(since) || !Number.isFinite(until) || since > until) throw new Error("invalid migration window");
|
|
309
|
+
return { since, until };
|
|
310
|
+
}
|
|
311
|
+
var positive = (value, fallback) => {
|
|
312
|
+
const result = value ?? fallback;
|
|
313
|
+
if (!Number.isSafeInteger(result) || result < 1) throw new Error("scan limits must be positive safe integers");
|
|
314
|
+
return result;
|
|
315
|
+
};
|
|
316
|
+
function discoverMigrationSessions(options) {
|
|
317
|
+
const maxFiles = positive(options.maxFiles, 1e4);
|
|
318
|
+
const maxEntries = positive(options.maxDiscoveryEntries, 1e5);
|
|
319
|
+
const files = /* @__PURE__ */ new Set();
|
|
320
|
+
let incomplete = 0;
|
|
321
|
+
let entries = 0;
|
|
322
|
+
const add = (file) => {
|
|
323
|
+
if (files.size >= maxFiles && !files.has(path2.resolve(file))) {
|
|
324
|
+
incomplete++;
|
|
325
|
+
return;
|
|
326
|
+
}
|
|
327
|
+
files.add(path2.resolve(file));
|
|
328
|
+
};
|
|
329
|
+
if (options.files) {
|
|
330
|
+
for (const file of options.files.slice(0, maxFiles)) add(file);
|
|
331
|
+
if (options.files.length > maxFiles) incomplete++;
|
|
332
|
+
return { files: [...files].sort(), incomplete };
|
|
333
|
+
}
|
|
334
|
+
const list = (dir, visit) => {
|
|
335
|
+
let handle;
|
|
336
|
+
try {
|
|
337
|
+
handle = fs2.opendirSync(dir);
|
|
338
|
+
let entry;
|
|
339
|
+
while ((entry = handle.readSync()) !== null) {
|
|
340
|
+
if (++entries > maxEntries) {
|
|
341
|
+
incomplete++;
|
|
342
|
+
break;
|
|
343
|
+
}
|
|
344
|
+
visit(entry);
|
|
345
|
+
}
|
|
346
|
+
} catch {
|
|
347
|
+
incomplete++;
|
|
348
|
+
} finally {
|
|
349
|
+
handle?.closeSync();
|
|
350
|
+
}
|
|
351
|
+
};
|
|
352
|
+
const scan = (dir) => list(dir, (entry) => {
|
|
353
|
+
if (entry.isFile() && entry.name.endsWith(".jsonl")) add(path2.join(dir, entry.name));
|
|
354
|
+
else if (entry.isSymbolicLink() && entry.name.endsWith(".jsonl")) incomplete++;
|
|
355
|
+
});
|
|
356
|
+
if (options.candidateDirs) {
|
|
357
|
+
for (const dir of options.candidateDirs) {
|
|
358
|
+
if (entries >= maxEntries) {
|
|
359
|
+
incomplete++;
|
|
360
|
+
break;
|
|
361
|
+
}
|
|
362
|
+
scan(dir);
|
|
363
|
+
}
|
|
364
|
+
} else if (options.projectCwd) {
|
|
365
|
+
const names = sessionDirNamesForCwd(options.projectCwd);
|
|
366
|
+
let found = false;
|
|
367
|
+
for (const name of [names.canonical, ...names.legacy]) {
|
|
368
|
+
const dir = path2.join(sessionsDirRoot(options.agentDir), name);
|
|
369
|
+
try {
|
|
370
|
+
if (fs2.statSync(dir).isDirectory()) {
|
|
371
|
+
found = true;
|
|
372
|
+
scan(dir);
|
|
373
|
+
}
|
|
374
|
+
} catch (error) {
|
|
375
|
+
if (error.code !== "ENOENT") incomplete++;
|
|
376
|
+
}
|
|
377
|
+
}
|
|
378
|
+
if (!found) incomplete++;
|
|
379
|
+
} else {
|
|
380
|
+
const root = sessionsDirRoot(options.agentDir);
|
|
381
|
+
list(root, (entry) => {
|
|
382
|
+
if (entry.isDirectory()) scan(path2.join(root, entry.name));
|
|
383
|
+
else if (entry.isFile() && entry.name.endsWith(".jsonl")) add(path2.join(root, entry.name));
|
|
384
|
+
else if (entry.isSymbolicLink()) incomplete++;
|
|
385
|
+
});
|
|
386
|
+
}
|
|
387
|
+
return { files: [...files].sort(), incomplete };
|
|
388
|
+
}
|
|
389
|
+
var CHECKPOINT_SCHEMA = "CREATE TABLE IF NOT EXISTS migration_checkpoints (project_key TEXT NOT NULL, session_path TEXT NOT NULL, source_hash TEXT NOT NULL, line_ordinal INTEGER NOT NULL, entry_id TEXT NOT NULL, content_hash TEXT NOT NULL, PRIMARY KEY(project_key,session_path,source_hash,line_ordinal,entry_id,content_hash))";
|
|
390
|
+
function reconcileSession(options) {
|
|
391
|
+
return migrateSessions({ ...options, onDemand: true });
|
|
392
|
+
}
|
|
393
|
+
function migrateSessions(options) {
|
|
394
|
+
const window = migrationWindow(options);
|
|
395
|
+
const maxFileBytes = positive(options.maxFileBytes, 64 * 1024 ** 2);
|
|
396
|
+
const maxLineBytes = positive(options.maxLineBytes, 8 * 1024 ** 2);
|
|
397
|
+
const maxTotalBytes = positive(options.maxTotalBytes, 1024 ** 3);
|
|
398
|
+
const discovery = discoverMigrationSessions(options);
|
|
399
|
+
const result = {
|
|
400
|
+
mode: options.apply ? "apply" : "dry-run",
|
|
401
|
+
since: new Date(window.since).toISOString(),
|
|
402
|
+
until: new Date(window.until).toISOString(),
|
|
403
|
+
counts: { eligible: 0, imported: 0, skippedDuplicate: 0, skippedOutOfWindow: 0, malformed: 0, oversized: 0, incompleteDiscovery: discovery.incomplete, errors: 0 },
|
|
404
|
+
filesScanned: 0,
|
|
405
|
+
bytesScanned: 0,
|
|
406
|
+
generations: [],
|
|
407
|
+
exitCode: 0
|
|
408
|
+
};
|
|
409
|
+
const counts = result.counts;
|
|
410
|
+
const drySeen = /* @__PURE__ */ new Set();
|
|
411
|
+
for (const file of discovery.files) {
|
|
412
|
+
let data;
|
|
413
|
+
let fd;
|
|
414
|
+
try {
|
|
415
|
+
fd = fs2.openSync(file, "r");
|
|
416
|
+
const before = fs2.fstatSync(fd);
|
|
417
|
+
if (!before.isFile()) {
|
|
418
|
+
counts.incompleteDiscovery++;
|
|
419
|
+
continue;
|
|
420
|
+
}
|
|
421
|
+
if (before.size > maxFileBytes) {
|
|
422
|
+
counts.oversized++;
|
|
423
|
+
continue;
|
|
424
|
+
}
|
|
425
|
+
if (result.bytesScanned + before.size > maxTotalBytes) {
|
|
426
|
+
counts.incompleteDiscovery++;
|
|
427
|
+
break;
|
|
428
|
+
}
|
|
429
|
+
data = Buffer.allocUnsafe(before.size);
|
|
430
|
+
let offset = 0;
|
|
431
|
+
while (offset < data.length) {
|
|
432
|
+
const size = fs2.readSync(fd, data, offset, data.length - offset, offset);
|
|
433
|
+
if (size === 0) throw new Error("source shortened during read");
|
|
434
|
+
offset += size;
|
|
435
|
+
}
|
|
436
|
+
const after = fs2.fstatSync(fd);
|
|
437
|
+
if (after.size !== before.size || after.mtimeMs !== before.mtimeMs || after.ctimeMs !== before.ctimeMs) throw new Error("source changed during read");
|
|
438
|
+
} catch {
|
|
439
|
+
counts.errors++;
|
|
440
|
+
counts.incompleteDiscovery++;
|
|
441
|
+
continue;
|
|
442
|
+
} finally {
|
|
443
|
+
if (fd !== void 0) fs2.closeSync(fd);
|
|
444
|
+
}
|
|
445
|
+
result.filesScanned++;
|
|
446
|
+
result.bytesScanned += data.length;
|
|
447
|
+
const sourceHash = hash2(data);
|
|
448
|
+
result.generations.push({ sessionPath: file, sourceHash });
|
|
449
|
+
let header;
|
|
450
|
+
let ledger;
|
|
451
|
+
let database;
|
|
452
|
+
let sessionId = "";
|
|
453
|
+
let projectKey = "";
|
|
454
|
+
let cwd;
|
|
455
|
+
let ordinal = 0;
|
|
456
|
+
try {
|
|
457
|
+
for (let start = 0; start < data.length; ) {
|
|
458
|
+
ordinal++;
|
|
459
|
+
const newline = data.indexOf(10, start);
|
|
460
|
+
const end = newline < 0 ? data.length : newline;
|
|
461
|
+
const bytes = data.subarray(start, end);
|
|
462
|
+
start = newline < 0 ? data.length : end + 1;
|
|
463
|
+
if (bytes.length > maxLineBytes) {
|
|
464
|
+
counts.oversized++;
|
|
465
|
+
continue;
|
|
466
|
+
}
|
|
467
|
+
const content = bytes.toString("utf8").replace(/\r$/, "");
|
|
468
|
+
if (!content.trim()) continue;
|
|
469
|
+
let row;
|
|
470
|
+
try {
|
|
471
|
+
row = record(JSON.parse(content));
|
|
472
|
+
} catch {
|
|
473
|
+
counts.malformed++;
|
|
474
|
+
continue;
|
|
475
|
+
}
|
|
476
|
+
if (!row) {
|
|
477
|
+
counts.malformed++;
|
|
478
|
+
continue;
|
|
479
|
+
}
|
|
480
|
+
if (row.type === "message_end") continue;
|
|
481
|
+
if (!header) {
|
|
482
|
+
if (row.type !== "session") {
|
|
483
|
+
if (record(row.message)) counts.malformed++;
|
|
484
|
+
continue;
|
|
485
|
+
}
|
|
486
|
+
if (!nonblank(row.id)) {
|
|
487
|
+
counts.malformed++;
|
|
488
|
+
break;
|
|
489
|
+
}
|
|
490
|
+
header = row;
|
|
491
|
+
sessionId = row.id;
|
|
492
|
+
cwd = nonblank(row.cwd) ? row.cwd : options.liveCwd;
|
|
493
|
+
if (!cwd) {
|
|
494
|
+
counts.malformed++;
|
|
495
|
+
break;
|
|
496
|
+
}
|
|
497
|
+
const projectInput = options.liveCwd ? { recordedCwd: cwd, liveCwd: options.liveCwd } : { recordedCwd: cwd };
|
|
498
|
+
projectKey = canonicalProjectIdentity(projectInput).key;
|
|
499
|
+
if (options.ledger && options.ledger.project.key !== projectKey) {
|
|
500
|
+
counts.errors++;
|
|
501
|
+
break;
|
|
502
|
+
}
|
|
503
|
+
if (options.apply) {
|
|
504
|
+
if (options.ledger) ledger = options.ledger;
|
|
505
|
+
else if (options.ledgerRoot) ledger = new LcmLedger({ rootDir: options.ledgerRoot, project: { recordedCwd: cwd } });
|
|
506
|
+
else ledger = new LcmLedger({ project: { recordedCwd: cwd } });
|
|
507
|
+
database = ledger.db;
|
|
508
|
+
database.exec(CHECKPOINT_SCHEMA);
|
|
509
|
+
} else if (options.ledger) database = options.ledger.db;
|
|
510
|
+
else {
|
|
511
|
+
const dbPath = defaultLedgerPath(options.ledgerRoot, projectKey);
|
|
512
|
+
if (fs2.existsSync(dbPath)) database = new (sqliteDriver())(dbPath, { readOnly: true });
|
|
513
|
+
}
|
|
514
|
+
continue;
|
|
515
|
+
}
|
|
516
|
+
if (row.type === "session" || !nonblank(row.type) || !nonblank(row.id) || row.parentId !== null && row.parentId !== void 0 && typeof row.parentId !== "string") {
|
|
517
|
+
counts.malformed++;
|
|
518
|
+
continue;
|
|
519
|
+
}
|
|
520
|
+
const message = record(row.message);
|
|
521
|
+
if (row.type === "message" && (!message || !nonblank(message.role))) {
|
|
522
|
+
counts.malformed++;
|
|
523
|
+
continue;
|
|
524
|
+
}
|
|
525
|
+
let at;
|
|
526
|
+
try {
|
|
527
|
+
at = typeof row.timestamp === "string" ? utc(row.timestamp) : NaN;
|
|
528
|
+
} catch {
|
|
529
|
+
at = NaN;
|
|
530
|
+
}
|
|
531
|
+
if (!Number.isFinite(at)) {
|
|
532
|
+
counts.malformed++;
|
|
533
|
+
continue;
|
|
534
|
+
}
|
|
535
|
+
if (!options.onDemand && (at < window.since || at > window.until)) {
|
|
536
|
+
counts.skippedOutOfWindow++;
|
|
537
|
+
continue;
|
|
538
|
+
}
|
|
539
|
+
counts.eligible++;
|
|
540
|
+
const payloadJson = canonicalLcmPayload(row);
|
|
541
|
+
const contentHash = hashLcmPayload(row);
|
|
542
|
+
const identity = JSON.stringify([projectKey, sessionId, row.id, contentHash]);
|
|
543
|
+
const progress = { projectKey, sessionPath: file, sourceHash, lineOrdinal: ordinal, entryId: row.id, contentHash };
|
|
544
|
+
const duplicate = drySeen.has(identity) || database?.prepare("SELECT 1 FROM raw_entries WHERE project_key=? AND session_id=? AND entry_id=? AND content_hash=?").get(projectKey, sessionId, row.id, contentHash) !== void 0;
|
|
545
|
+
if (duplicate) counts.skippedDuplicate++;
|
|
546
|
+
else if (ledger) {
|
|
547
|
+
ledger.appendRaw({
|
|
548
|
+
projectKey,
|
|
549
|
+
sessionId,
|
|
550
|
+
entryId: row.id,
|
|
551
|
+
role: message && typeof message.role === "string" ? message.role : row.type,
|
|
552
|
+
content,
|
|
553
|
+
payloadJson,
|
|
554
|
+
parentEntryId: typeof row.parentId === "string" ? row.parentId : null,
|
|
555
|
+
branch: typeof row.branch === "string" ? row.branch : typeof row.branchId === "string" ? row.branchId : null,
|
|
556
|
+
...cwd ? { recordedCwd: cwd } : {},
|
|
557
|
+
createdAt: at
|
|
558
|
+
});
|
|
559
|
+
counts.imported++;
|
|
560
|
+
} else if (!duplicate) counts.imported++;
|
|
561
|
+
drySeen.add(identity);
|
|
562
|
+
if (ledger) database.prepare("INSERT OR IGNORE INTO migration_checkpoints(project_key,session_path,source_hash,line_ordinal,entry_id,content_hash) VALUES(?,?,?,?,?,?)").run(projectKey, file, sourceHash, ordinal, row.id, contentHash);
|
|
563
|
+
options.onProgress?.(progress);
|
|
564
|
+
}
|
|
565
|
+
if (!header) counts.malformed++;
|
|
566
|
+
} catch (error) {
|
|
567
|
+
counts.errors++;
|
|
568
|
+
if (options.onProgress) throw error;
|
|
569
|
+
} finally {
|
|
570
|
+
if (ledger && ledger !== options.ledger) ledger.close();
|
|
571
|
+
else if (!ledger && database && database !== options.ledger?.db) database.close();
|
|
572
|
+
}
|
|
573
|
+
}
|
|
574
|
+
result.exitCode = counts.errors || counts.malformed || counts.oversized || counts.incompleteDiscovery && !options.allowIncompleteDiscovery ? 1 : 0;
|
|
575
|
+
return result;
|
|
576
|
+
}
|
|
577
|
+
|
|
578
|
+
// src/compaction/lcm-maintenance.ts
|
|
579
|
+
import crypto3 from "node:crypto";
|
|
580
|
+
var DEFAULT_LCM_BUDGET = { calls: 32, inputTokens: 2e6, outputTokens: 128e3, cost: Number.POSITIVE_INFINITY, wallMs: 6e4, sessionCalls: 16 };
|
|
581
|
+
var hash3 = (v) => hashLcmPayload(v);
|
|
582
|
+
var parse = (v) => JSON.parse(String(v));
|
|
583
|
+
var rawHash = (e) => e.payloadHash;
|
|
584
|
+
var token = () => crypto3.randomUUID();
|
|
585
|
+
var LcmMaintenance = class {
|
|
586
|
+
constructor(ledger, options = {}) {
|
|
587
|
+
this.ledger = ledger;
|
|
588
|
+
this.projectKey = ledger.project.key;
|
|
589
|
+
this.now = options.now ?? Date.now;
|
|
590
|
+
this.ownerId = options.ownerId ?? crypto3.randomUUID();
|
|
591
|
+
this.policyHash = options.policyHash ?? hash3("lcm-policy-v1");
|
|
592
|
+
this.maxLeaf = options.maxLeafEntries ?? 8;
|
|
593
|
+
this.maxChildren = options.maxCondenseChildren ?? 4;
|
|
594
|
+
this.maxInputChars = options.maxInputChars ?? 48e3;
|
|
595
|
+
this.maxOutputChars = options.maxOutputChars ?? 4096;
|
|
596
|
+
this.budget = { ...DEFAULT_LCM_BUDGET, ...options.budget };
|
|
597
|
+
}
|
|
598
|
+
projectKey;
|
|
599
|
+
now;
|
|
600
|
+
ownerId;
|
|
601
|
+
policyHash;
|
|
602
|
+
maxLeaf;
|
|
603
|
+
maxChildren;
|
|
604
|
+
maxInputChars;
|
|
605
|
+
maxOutputChars;
|
|
606
|
+
budget;
|
|
607
|
+
listNodes(limit = 100) {
|
|
608
|
+
return this.ledger.readOnly((db) => db.prepare("SELECT payload FROM summary_nodes WHERE project_key=? ORDER BY created_at,node_id LIMIT ?").all(this.projectKey, limit).map((r) => {
|
|
609
|
+
const node = parse(r.payload);
|
|
610
|
+
if (node.projectKey !== this.projectKey) throw new Error("node project does not match ledger project");
|
|
611
|
+
return node;
|
|
612
|
+
}));
|
|
613
|
+
}
|
|
614
|
+
getNode(nodeId) {
|
|
615
|
+
return this.ledger.readOnly((db) => {
|
|
616
|
+
const row = db.prepare("SELECT payload FROM summary_nodes WHERE node_id=? AND project_key=?").get(nodeId, this.projectKey);
|
|
617
|
+
if (!row?.payload) return void 0;
|
|
618
|
+
const node = parse(row.payload);
|
|
619
|
+
if (node.projectKey !== this.projectKey) throw new Error("node project does not match ledger project");
|
|
620
|
+
return node;
|
|
621
|
+
});
|
|
622
|
+
}
|
|
623
|
+
getFrontier(sessionId, branch, activeSources) {
|
|
624
|
+
const nodes = this.listNodes(1e5).filter((n) => n.state === "ready" && n.policyHash === this.policyHash && (!sessionId || n.sessionId === sessionId) && (branch === void 0 || n.branch === branch || n.branch === null) && (!activeSources || n.sources.every((s) => activeSources.has(this.sourceKey(s)))));
|
|
625
|
+
const covered = new Set(nodes.flatMap((n) => n.children));
|
|
626
|
+
return nodes.filter((n) => !covered.has(n.nodeId));
|
|
627
|
+
}
|
|
628
|
+
listJobs(limit = 100) {
|
|
629
|
+
return this.ledger.readOnly((db) => db.prepare("SELECT payload FROM maintenance_jobs WHERE project_key=? ORDER BY created_at,job_id LIMIT ?").all(this.projectKey, limit).map((r) => {
|
|
630
|
+
const job = parse(r.payload);
|
|
631
|
+
if (job.projectKey !== this.projectKey) throw new Error("job project does not match ledger project");
|
|
632
|
+
return job;
|
|
633
|
+
}));
|
|
634
|
+
}
|
|
635
|
+
selectLeaf(entries = this.ledger.readRaw(this.projectKey), branch) {
|
|
636
|
+
const first = entries[0];
|
|
637
|
+
if (!first) return [];
|
|
638
|
+
const nodes = this.listNodes(1e5).filter((n) => n.state !== "failed" && n.policyHash === this.policyHash && n.sessionId === first.sessionId && (branch === void 0 || n.branch === branch || n.branch === null));
|
|
639
|
+
const covered = new Set(nodes.flatMap((n) => n.sources.map((s) => this.sourceKey(s))));
|
|
640
|
+
return entries.filter((e) => e.sessionId === first.sessionId && (branch === void 0 || e.branch === branch || e.branch === null) && !covered.has(this.sourceKey(e))).slice(0, this.maxLeaf);
|
|
641
|
+
}
|
|
642
|
+
selectCondensation(sessionId, activeSources, branch) {
|
|
643
|
+
const nodes = this.listNodes(1e5).filter((n) => n.state === "ready" && n.policyHash === this.policyHash && (!sessionId || n.sessionId === sessionId) && (!activeSources || n.sources.every((s) => activeSources.has(this.sourceKey(s)))) && (branch === void 0 || n.branch === branch || n.branch === null));
|
|
644
|
+
const eligible = new Set(nodes.map((n) => n.nodeId));
|
|
645
|
+
const edges = this.ledger.readOnly((db) => db.prepare("SELECT e.parent_id,e.child_id FROM summary_edges e JOIN summary_nodes parent ON parent.node_id=e.parent_id AND parent.project_key=? JOIN summary_nodes child ON child.node_id=e.child_id AND child.project_key=parent.project_key WHERE parent.project_key=?").all(this.projectKey, this.projectKey));
|
|
646
|
+
const consumed = new Set(edges.filter((e) => eligible.has(e.parent_id)).map((e) => e.child_id));
|
|
647
|
+
const candidates = nodes.filter((n) => !consumed.has(n.nodeId));
|
|
648
|
+
const first = candidates[0];
|
|
649
|
+
if (!first) return [];
|
|
650
|
+
const depth = Math.min(...candidates.map((n) => n.depth));
|
|
651
|
+
const sameDepth = candidates.filter((n) => n.depth === depth);
|
|
652
|
+
if (depth > 0 && sameDepth.length < 2) return [];
|
|
653
|
+
return candidates.filter((n) => n.projectKey === first.projectKey && n.sessionId === first.sessionId && n.depth === depth).sort((a, b) => a.nodeId.localeCompare(b.nodeId)).slice(0, this.maxChildren);
|
|
654
|
+
}
|
|
655
|
+
createLeaf(entries = this.selectLeaf()) {
|
|
656
|
+
if (!entries.length) return void 0;
|
|
657
|
+
const branches = new Set(entries.map((entry) => entry.branch).filter((value) => typeof value === "string"));
|
|
658
|
+
if (branches.size > 1) throw new Error("cross-branch ranges");
|
|
659
|
+
const branch = [...branches][0] ?? null;
|
|
660
|
+
const source = entries.map((e, index) => {
|
|
661
|
+
if (e.projectKey !== this.projectKey) throw new Error("entry project does not match ledger project");
|
|
662
|
+
const payload = parse(e.payloadJson);
|
|
663
|
+
if (typeof payload.type !== "string" || !payload.type || payload.id !== e.entryId) throw new Error("invalid raw payload");
|
|
664
|
+
if (hash3(payload) !== rawHash(e)) throw new Error("payload hash mismatch");
|
|
665
|
+
return { sessionId: e.sessionId, entryId: e.entryId, revision: e.revision, payloadHash: rawHash(e) };
|
|
666
|
+
});
|
|
667
|
+
this.validateSources(source);
|
|
668
|
+
const node = { nodeId: `leaf:${hash3({ projectKey: this.projectKey, source, branch, policyHash: this.policyHash })}`, projectKey: this.projectKey, sessionId: entries[0].sessionId, branch, kind: "leaf", sources: source, children: [], depth: 0, sourceHash: hash3(source.map((s) => s.payloadHash)), policyHash: this.policyHash, modelHash: "", state: "pending", createdAt: this.now() };
|
|
669
|
+
this.publish(node);
|
|
670
|
+
return node;
|
|
671
|
+
}
|
|
672
|
+
createCondensed(children = this.selectCondensation()) {
|
|
673
|
+
if (!children.length) return void 0;
|
|
674
|
+
if (children.some((c) => c.state !== "ready")) throw new Error("condensation requires ready children");
|
|
675
|
+
if (new Set(children.map((c) => c.sessionId)).size !== 1 || new Set(children.map((c) => c.depth)).size !== 1) throw new Error("cross-branch ranges");
|
|
676
|
+
const storedChildren = children.map((child) => this.getNode(child.nodeId));
|
|
677
|
+
if (storedChildren.some((child) => !child || child.projectKey !== this.projectKey)) throw new Error("child project does not match ledger project");
|
|
678
|
+
if (storedChildren.some((child, index) => JSON.stringify(child) !== JSON.stringify(children[index]))) throw new Error("child payload changed");
|
|
679
|
+
const concreteBranches = new Set(storedChildren.map((c) => c.branch).filter((value) => typeof value === "string"));
|
|
680
|
+
if (concreteBranches.size > 1) throw new Error("cross-branch ranges");
|
|
681
|
+
const branch = [...concreteBranches][0] ?? null;
|
|
682
|
+
const source = [...new Map(storedChildren.flatMap((c) => c.sources).map((value) => [this.sourceKey(value), value])).values()];
|
|
683
|
+
this.validateSources(source);
|
|
684
|
+
const node = { nodeId: `condensed:${hash3({ projectKey: this.projectKey, children: storedChildren.map((c) => c.nodeId), branch, policyHash: this.policyHash })}`, projectKey: this.projectKey, sessionId: children[0].sessionId, branch, kind: "condensed", sources: source, children: children.map((c) => c.nodeId), depth: Math.max(...children.map((c) => c.depth)) + 1, sourceHash: hash3(children.map((c) => c.sourceHash)), policyHash: this.policyHash, modelHash: "", state: "pending", createdAt: this.now() };
|
|
685
|
+
this.publish(node);
|
|
686
|
+
return node;
|
|
687
|
+
}
|
|
688
|
+
validateSources(source) {
|
|
689
|
+
const seen = /* @__PURE__ */ new Set();
|
|
690
|
+
for (const s of source) {
|
|
691
|
+
if (seen.has(this.sourceKey(s))) throw new Error("duplicate source identity");
|
|
692
|
+
seen.add(this.sourceKey(s));
|
|
693
|
+
}
|
|
694
|
+
if (new Set(source.map((s) => s.sessionId)).size !== 1) throw new Error("cross-session ranges");
|
|
695
|
+
}
|
|
696
|
+
sourceKey(source) {
|
|
697
|
+
return `${source.sessionId}:${source.entryId}:${source.revision}`;
|
|
698
|
+
}
|
|
699
|
+
publish(node) {
|
|
700
|
+
this.ledger.transaction((db) => {
|
|
701
|
+
if (node.children.includes(node.nodeId)) throw new Error("cycle");
|
|
702
|
+
for (const child of node.children) if (!db.prepare("SELECT 1 FROM summary_nodes WHERE node_id=? AND project_key=?").get(child, this.projectKey)) throw new Error("missing child");
|
|
703
|
+
db.prepare("INSERT OR IGNORE INTO summary_nodes(node_id,project_key,payload,created_at) VALUES(?,?,?,?)").run(node.nodeId, this.projectKey, JSON.stringify(node), node.createdAt);
|
|
704
|
+
for (const child of node.children) db.prepare("INSERT OR IGNORE INTO summary_edges(parent_id,child_id) VALUES(?,?)").run(node.nodeId, child);
|
|
705
|
+
const job = this.job(node);
|
|
706
|
+
db.prepare("INSERT OR IGNORE INTO maintenance_jobs(job_id,project_key,status,payload,created_at,updated_at) VALUES(?,?,?,?,?,?)").run(job.jobId, this.projectKey, job.state, JSON.stringify(job), job.createdAt, job.updatedAt);
|
|
707
|
+
});
|
|
708
|
+
}
|
|
709
|
+
job(node) {
|
|
710
|
+
const t = this.now();
|
|
711
|
+
return { jobId: `job:${node.nodeId}`, projectKey: this.projectKey, nodeId: node.nodeId, priority: node.kind === "condensed" ? 20 : 10, eligibleAt: t, state: "pending", attempts: 0, nextRetryAt: t, createdAt: t, updatedAt: t };
|
|
712
|
+
}
|
|
713
|
+
claim(jobId, ownerId = this.ownerId) {
|
|
714
|
+
return this.ledger.transaction((db) => {
|
|
715
|
+
const row = db.prepare("SELECT payload FROM maintenance_jobs WHERE job_id=? AND project_key=?").get(jobId, this.projectKey);
|
|
716
|
+
if (!row?.payload) throw new Error("job not found");
|
|
717
|
+
const old = parse(row.payload);
|
|
718
|
+
if (old.projectKey !== this.projectKey) throw new Error("job project does not match ledger project");
|
|
719
|
+
const t = this.now();
|
|
720
|
+
const expired = old.state === "running" && (old.leaseUntil ?? 0) <= t;
|
|
721
|
+
if (old.state === "running" && !expired) throw new Error("job lease held");
|
|
722
|
+
if (old.state !== "pending" && !expired || old.eligibleAt > t || old.nextRetryAt > t) throw new Error("job not eligible");
|
|
723
|
+
const active = db.prepare("SELECT payload FROM maintenance_jobs WHERE project_key=? AND status=?").all(this.projectKey, "running").map((r) => parse(r.payload)).filter((j) => j.jobId !== old.jobId && (j.leaseUntil ?? 0) > t);
|
|
724
|
+
if (active.length) throw new Error("project lease held");
|
|
725
|
+
const nodeRow = db.prepare("SELECT payload FROM summary_nodes WHERE node_id=? AND project_key=?").get(old.nodeId, this.projectKey);
|
|
726
|
+
const sessionId = nodeRow?.payload ? parse(nodeRow.payload).sessionId : "";
|
|
727
|
+
const day = this.day(t);
|
|
728
|
+
const projectUsage = this.usage(db, day);
|
|
729
|
+
const sessionUsage = this.usage(db, day, sessionId);
|
|
730
|
+
if (projectUsage.calls >= this.budget.calls || projectUsage.inputTokens >= this.budget.inputTokens || projectUsage.outputTokens >= this.budget.outputTokens || projectUsage.cost >= this.budget.cost || projectUsage.wallMs >= this.budget.wallMs || sessionUsage.calls >= this.budget.sessionCalls) throw new Error("budget exhausted");
|
|
731
|
+
const out = { ...old, state: "running", ownerId, leaseToken: token(), leaseUntil: t + 3e4, updatedAt: t };
|
|
732
|
+
db.prepare("UPDATE maintenance_jobs SET status=?,payload=?,updated_at=? WHERE job_id=? AND project_key=?").run(out.state, JSON.stringify(out), t, jobId, this.projectKey);
|
|
733
|
+
return out;
|
|
734
|
+
});
|
|
735
|
+
}
|
|
736
|
+
claimEmergency(jobId, ownerId = this.ownerId) {
|
|
737
|
+
return this.ledger.transaction((db) => {
|
|
738
|
+
const row = db.prepare("SELECT payload FROM maintenance_jobs WHERE job_id=? AND project_key=?").get(jobId, this.projectKey);
|
|
739
|
+
if (!row?.payload) throw new Error("job not found");
|
|
740
|
+
const old = parse(row.payload);
|
|
741
|
+
if (old.projectKey !== this.projectKey) throw new Error("job project does not match ledger project");
|
|
742
|
+
const t = this.now();
|
|
743
|
+
const expired = old.state === "running" && (old.leaseUntil ?? 0) <= t;
|
|
744
|
+
if (old.state === "running" && !expired) throw new Error("job lease held");
|
|
745
|
+
if (old.state === "completed") throw new Error("job already completed");
|
|
746
|
+
const out = { ...old, state: "running", ownerId, leaseToken: token(), leaseUntil: t + 3e4, updatedAt: t };
|
|
747
|
+
db.prepare("UPDATE maintenance_jobs SET status=?,payload=?,updated_at=? WHERE job_id=? AND project_key=?").run(out.state, JSON.stringify(out), t, jobId, this.projectKey);
|
|
748
|
+
return out;
|
|
749
|
+
});
|
|
750
|
+
}
|
|
751
|
+
renew(job) {
|
|
752
|
+
return this.fenced(job, (old) => ({ ...old, leaseUntil: this.now() + 3e4, updatedAt: this.now() }));
|
|
753
|
+
}
|
|
754
|
+
validateText(text) {
|
|
755
|
+
if (typeof text !== "string" || text.length === 0) throw new Error("invalid summary text");
|
|
756
|
+
let length = 0;
|
|
757
|
+
for (const _ of text) {
|
|
758
|
+
length += 1;
|
|
759
|
+
if (length > this.maxOutputChars) throw new Error("summary exceeds output bound");
|
|
760
|
+
}
|
|
761
|
+
}
|
|
762
|
+
complete(job, result) {
|
|
763
|
+
this.validateText(result.text);
|
|
764
|
+
return this.ledger.transaction((db) => {
|
|
765
|
+
const current = this.readJob(db, job.jobId);
|
|
766
|
+
this.assertLease(current, job);
|
|
767
|
+
if (current.nodeId !== job.nodeId) throw new Error("job node mismatch");
|
|
768
|
+
const row = db.prepare("SELECT payload FROM summary_nodes WHERE node_id=? AND project_key=?").get(job.nodeId, this.projectKey);
|
|
769
|
+
if (!row?.payload) throw new Error("node not found");
|
|
770
|
+
const node = parse(row.payload);
|
|
771
|
+
if (node.projectKey !== this.projectKey || node.nodeId !== current.nodeId) throw new Error("node project does not match ledger project");
|
|
772
|
+
for (const source of node.sources) {
|
|
773
|
+
const raw = db.prepare("SELECT payload_json,session_id FROM raw_entries WHERE project_key=? AND session_id=? AND entry_id=? AND revision=?").get(this.projectKey, source.sessionId, source.entryId, source.revision);
|
|
774
|
+
if (!raw?.payload_json || hash3(parse(raw.payload_json)) !== source.payloadHash || raw.session_id !== source.sessionId) throw new Error("stale source");
|
|
775
|
+
}
|
|
776
|
+
if (![result.inputTokens, result.outputTokens, result.cost, result.wallMs].every((value) => Number.isFinite(value) && value >= 0)) throw new Error("invalid model usage");
|
|
777
|
+
const projectUsage = this.usage(db, this.day(this.now()));
|
|
778
|
+
const sessionUsage = this.usage(db, this.day(this.now()), node.sessionId);
|
|
779
|
+
if (projectUsage.calls + 1 > this.budget.calls || projectUsage.inputTokens + result.inputTokens > this.budget.inputTokens || projectUsage.outputTokens + result.outputTokens > this.budget.outputTokens || projectUsage.cost + result.cost > this.budget.cost || projectUsage.wallMs + result.wallMs > this.budget.wallMs || sessionUsage.calls + 1 > this.budget.sessionCalls) throw new Error("budget exhausted");
|
|
780
|
+
const out = { ...node, state: "ready", text: result.text, modelHash: result.modelHash };
|
|
781
|
+
this.account(db, node.sessionId, result);
|
|
782
|
+
db.prepare("UPDATE summary_nodes SET payload=? WHERE node_id=? AND project_key=?").run(JSON.stringify(out), node.nodeId, this.projectKey);
|
|
783
|
+
const { ownerId: _ownerId, leaseToken: _leaseToken, leaseUntil: _leaseUntil, ...base } = current;
|
|
784
|
+
const done = { ...base, state: "completed", updatedAt: this.now() };
|
|
785
|
+
db.prepare("UPDATE maintenance_jobs SET status=?,payload=?,updated_at=? WHERE job_id=? AND project_key=?").run(done.state, JSON.stringify(done), done.updatedAt, done.jobId, this.projectKey);
|
|
786
|
+
return out;
|
|
787
|
+
});
|
|
788
|
+
}
|
|
789
|
+
completeEmergency(job, text) {
|
|
790
|
+
this.validateText(text);
|
|
791
|
+
return this.ledger.transaction((db) => {
|
|
792
|
+
const current = this.readJob(db, job.jobId);
|
|
793
|
+
this.assertLease(current, job);
|
|
794
|
+
if (current.nodeId !== job.nodeId) throw new Error("job node mismatch");
|
|
795
|
+
const row = db.prepare("SELECT payload FROM summary_nodes WHERE node_id=? AND project_key=?").get(job.nodeId, this.projectKey);
|
|
796
|
+
if (!row?.payload) throw new Error("node not found");
|
|
797
|
+
const node = parse(row.payload);
|
|
798
|
+
if (node.projectKey !== this.projectKey || node.nodeId !== current.nodeId) throw new Error("node project does not match ledger project");
|
|
799
|
+
for (const source of node.sources) {
|
|
800
|
+
const raw = db.prepare("SELECT payload_json,session_id FROM raw_entries WHERE project_key=? AND session_id=? AND entry_id=? AND revision=?").get(this.projectKey, source.sessionId, source.entryId, source.revision);
|
|
801
|
+
if (!raw?.payload_json || hash3(parse(raw.payload_json)) !== source.payloadHash || raw.session_id !== source.sessionId) throw new Error("stale source");
|
|
802
|
+
}
|
|
803
|
+
const out = { ...node, state: "ready", text, modelHash: "emergency" };
|
|
804
|
+
db.prepare("UPDATE summary_nodes SET payload=? WHERE node_id=? AND project_key=?").run(JSON.stringify(out), node.nodeId, this.projectKey);
|
|
805
|
+
const { ownerId: _ownerId, leaseToken: _leaseToken, leaseUntil: _leaseUntil, ...base } = current;
|
|
806
|
+
const done = { ...base, state: "completed", updatedAt: this.now() };
|
|
807
|
+
db.prepare("UPDATE maintenance_jobs SET status=?,payload=?,updated_at=? WHERE job_id=? AND project_key=?").run(done.state, JSON.stringify(done), done.updatedAt, done.jobId, this.projectKey);
|
|
808
|
+
return out;
|
|
809
|
+
});
|
|
810
|
+
}
|
|
811
|
+
fail(job, error) {
|
|
812
|
+
return this.fenced(job, (old) => {
|
|
813
|
+
const attempts = old.attempts + 1;
|
|
814
|
+
const delay = Math.min(9e5, 3e4 * 2 ** (attempts - 1));
|
|
815
|
+
const { ownerId: _ownerId, leaseToken: _leaseToken, leaseUntil: _leaseUntil, ...base } = old;
|
|
816
|
+
return { ...base, attempts, error, state: attempts >= 3 ? "failed" : "pending", eligibleAt: this.now() + delay, nextRetryAt: this.now() + delay, updatedAt: this.now() };
|
|
817
|
+
});
|
|
818
|
+
}
|
|
819
|
+
async run(job, model, input, signal = new AbortController().signal) {
|
|
820
|
+
let sources = [];
|
|
821
|
+
let claimed;
|
|
822
|
+
const controller = new AbortController();
|
|
823
|
+
let renew;
|
|
824
|
+
const timer = setTimeout(() => controller.abort(), 6e4);
|
|
825
|
+
const abort = () => controller.abort();
|
|
826
|
+
signal.addEventListener("abort", abort, { once: true });
|
|
827
|
+
try {
|
|
828
|
+
claimed = this.claim(job.jobId, job.ownerId ?? this.ownerId);
|
|
829
|
+
const node = this.getNode(claimed.nodeId);
|
|
830
|
+
sources = node?.sources ?? [];
|
|
831
|
+
renew = setInterval(() => {
|
|
832
|
+
try {
|
|
833
|
+
if (claimed) claimed = this.renew(claimed);
|
|
834
|
+
} catch {
|
|
835
|
+
}
|
|
836
|
+
}, 1e4);
|
|
837
|
+
const result = await Promise.race([model.generate({ prompt: buildLcmPrompt(node?.kind ?? "leaf", input, this.maxInputChars), sessionId: node?.sessionId ?? "", signal: controller.signal }), new Promise((_, reject) => controller.signal.addEventListener("abort", () => reject(new Error("LCM timeout")), { once: true }))]);
|
|
838
|
+
return this.complete(claimed, result);
|
|
839
|
+
} catch (error) {
|
|
840
|
+
if (!claimed) throw error;
|
|
841
|
+
const fallback = emergencyReduce(input, this.maxOutputChars, sources);
|
|
842
|
+
try {
|
|
843
|
+
return this.completeEmergency(claimed, fallback);
|
|
844
|
+
} catch {
|
|
845
|
+
try {
|
|
846
|
+
this.fail(claimed, String(error));
|
|
847
|
+
} catch {
|
|
848
|
+
}
|
|
849
|
+
throw error;
|
|
850
|
+
}
|
|
851
|
+
} finally {
|
|
852
|
+
clearTimeout(timer);
|
|
853
|
+
if (renew) clearInterval(renew);
|
|
854
|
+
signal.removeEventListener("abort", abort);
|
|
855
|
+
}
|
|
856
|
+
}
|
|
857
|
+
day(ms) {
|
|
858
|
+
return new Date(ms).toISOString().slice(0, 10);
|
|
859
|
+
}
|
|
860
|
+
usage(db, day, sessionId) {
|
|
861
|
+
const q = sessionId === void 0 ? "SELECT COALESCE(SUM(calls),0) calls,COALESCE(SUM(input_tokens),0) inputTokens,COALESCE(SUM(output_tokens),0) outputTokens,COALESCE(SUM(cost),0) cost,COALESCE(SUM(wall_ms),0) wallMs FROM maintenance_usage WHERE project_key=? AND day=?" : "SELECT COALESCE(SUM(calls),0) calls,COALESCE(SUM(input_tokens),0) inputTokens,COALESCE(SUM(output_tokens),0) outputTokens,COALESCE(SUM(cost),0) cost,COALESCE(SUM(wall_ms),0) wallMs FROM maintenance_usage WHERE project_key=? AND day=? AND session_id=?";
|
|
862
|
+
const r = sessionId === void 0 ? db.prepare(q).get(this.projectKey, day) : db.prepare(q).get(this.projectKey, day, sessionId);
|
|
863
|
+
return r;
|
|
864
|
+
}
|
|
865
|
+
account(db, sessionId, result) {
|
|
866
|
+
const d = this.day(this.now());
|
|
867
|
+
db.prepare("INSERT INTO maintenance_usage(project_key,day,session_id,calls,input_tokens,output_tokens,cost,wall_ms) VALUES(?,?,?,?,?,?,?,?) ON CONFLICT(project_key,day,session_id) DO UPDATE SET calls=calls+excluded.calls,input_tokens=input_tokens+excluded.input_tokens,output_tokens=output_tokens+excluded.output_tokens,cost=cost+excluded.cost,wall_ms=wall_ms+excluded.wall_ms").run(this.projectKey, d, sessionId, 1, result.inputTokens, result.outputTokens, result.cost, result.wallMs);
|
|
868
|
+
}
|
|
869
|
+
readJob(db, id) {
|
|
870
|
+
const row = db.prepare("SELECT payload FROM maintenance_jobs WHERE job_id=? AND project_key=?").get(id, this.projectKey);
|
|
871
|
+
if (!row) throw new Error("job not found");
|
|
872
|
+
const job = parse(row.payload);
|
|
873
|
+
if (job.projectKey !== this.projectKey) throw new Error("job project does not match ledger project");
|
|
874
|
+
return job;
|
|
875
|
+
}
|
|
876
|
+
assertLease(current, expected) {
|
|
877
|
+
if (current.ownerId !== expected.ownerId || current.leaseToken !== expected.leaseToken || current.state !== "running" || (current.leaseUntil ?? 0) < this.now()) throw new Error("lease fenced");
|
|
878
|
+
}
|
|
879
|
+
fenced(job, update) {
|
|
880
|
+
return this.ledger.transaction((db) => {
|
|
881
|
+
const old = this.readJob(db, job.jobId);
|
|
882
|
+
this.assertLease(old, job);
|
|
883
|
+
const out = update(old);
|
|
884
|
+
db.prepare("UPDATE maintenance_jobs SET status=?,payload=?,updated_at=? WHERE job_id=? AND project_key=?").run(out.state, JSON.stringify(out), out.updatedAt, out.jobId, this.projectKey);
|
|
885
|
+
return out;
|
|
886
|
+
});
|
|
887
|
+
}
|
|
888
|
+
};
|
|
889
|
+
|
|
890
|
+
// src/compaction/lcm-runtime.ts
|
|
891
|
+
var LcmRuntime = class {
|
|
892
|
+
ledger;
|
|
893
|
+
maintenance;
|
|
894
|
+
context;
|
|
895
|
+
resolveOptions;
|
|
896
|
+
abort = new AbortController();
|
|
897
|
+
writePending = Promise.resolve();
|
|
898
|
+
maintenancePending = Promise.resolve();
|
|
899
|
+
closed = false;
|
|
900
|
+
dirty = false;
|
|
901
|
+
degradedError;
|
|
902
|
+
activeSources = /* @__PURE__ */ new Set();
|
|
903
|
+
activeSessionId;
|
|
904
|
+
constructor(context, options = {}) {
|
|
905
|
+
this.context = context;
|
|
906
|
+
this.resolveOptions = typeof options === "function" ? options : () => options;
|
|
907
|
+
const initial = this.resolveOptions();
|
|
908
|
+
const recorded = context.sessionManager.getRecordedCwd?.();
|
|
909
|
+
const liveCwd = recorded || context.cwd;
|
|
910
|
+
const project = canonicalProjectIdentity({ liveCwd });
|
|
911
|
+
const ledgerOptions = initial.rootDir === void 0 ? { project: { liveCwd: project.canonicalPath ?? liveCwd } } : { rootDir: initial.rootDir, project: { liveCwd: project.canonicalPath ?? liveCwd } };
|
|
912
|
+
this.ledger = new LcmLedger(ledgerOptions);
|
|
913
|
+
this.maintenance = new LcmMaintenance(this.ledger, {
|
|
914
|
+
...initial,
|
|
915
|
+
...initial.lcmMaxInputChars === void 0 ? {} : { maxInputChars: initial.lcmMaxInputChars },
|
|
916
|
+
...initial.lcmMaxOutputChars === void 0 ? {} : { maxOutputChars: initial.lcmMaxOutputChars }
|
|
917
|
+
});
|
|
918
|
+
}
|
|
919
|
+
get options() {
|
|
920
|
+
return this.resolveOptions();
|
|
921
|
+
}
|
|
922
|
+
get projectKey() {
|
|
923
|
+
return this.ledger.project.key;
|
|
924
|
+
}
|
|
925
|
+
get signal() {
|
|
926
|
+
return this.abort.signal;
|
|
927
|
+
}
|
|
928
|
+
get status() {
|
|
929
|
+
return this.degradedError === void 0 ? "healthy" : "degraded";
|
|
930
|
+
}
|
|
931
|
+
get error() {
|
|
932
|
+
return this.degradedError;
|
|
933
|
+
}
|
|
934
|
+
markDirty() {
|
|
935
|
+
this.dirty = true;
|
|
936
|
+
}
|
|
937
|
+
enqueueWrite(operation) {
|
|
938
|
+
const run = this.writePending.then(operation);
|
|
939
|
+
this.writePending = run.catch((error) => {
|
|
940
|
+
this.degradedError = error;
|
|
941
|
+
});
|
|
942
|
+
return run;
|
|
943
|
+
}
|
|
944
|
+
refreshActiveSources(sessionId, entries) {
|
|
945
|
+
const ids = new Set(entries.map((entry) => entry.id));
|
|
946
|
+
const latest = /* @__PURE__ */ new Map();
|
|
947
|
+
for (const entry of this.ledger.readRaw(this.projectKey, sessionId)) {
|
|
948
|
+
if (!ids.has(entry.entryId)) continue;
|
|
949
|
+
const previous = latest.get(entry.entryId);
|
|
950
|
+
if (!previous || previous.revision < entry.revision) latest.set(entry.entryId, entry);
|
|
951
|
+
}
|
|
952
|
+
this.activeSources = new Set(
|
|
953
|
+
[...latest.values()].map((entry) => `${entry.sessionId}:${entry.entryId}:${entry.revision}`)
|
|
954
|
+
);
|
|
955
|
+
this.activeSessionId = sessionId;
|
|
956
|
+
}
|
|
957
|
+
reconcileSelectedSession() {
|
|
958
|
+
if (this.closed) return Promise.resolve();
|
|
959
|
+
return this.enqueueWrite(() => {
|
|
960
|
+
const getSessionFile = this.context.sessionManager.getSessionFile;
|
|
961
|
+
if (typeof getSessionFile !== "function") return;
|
|
962
|
+
const sessionFile = getSessionFile.call(this.context.sessionManager);
|
|
963
|
+
if (!sessionFile) return;
|
|
964
|
+
const recorded = this.context.sessionManager.getRecordedCwd?.();
|
|
965
|
+
const cwd = recorded || this.context.cwd;
|
|
966
|
+
const result = reconcileSession({
|
|
967
|
+
agentDir: this.options.rootDir ?? process.env.PI_CODING_AGENT_DIR ?? `${process.env.HOME ?? "."}/.omp/agent`,
|
|
968
|
+
ledger: this.ledger,
|
|
969
|
+
files: [sessionFile],
|
|
970
|
+
liveCwd: cwd,
|
|
971
|
+
projectCwd: cwd,
|
|
972
|
+
apply: true
|
|
973
|
+
});
|
|
974
|
+
this.degradedError = result.counts.errors > 0 ? new Error(`LCM session reconciliation reported ${result.counts.errors} error(s)`) : void 0;
|
|
975
|
+
const sessionId = this.context.sessionManager.getSessionId();
|
|
976
|
+
this.refreshActiveSources(sessionId, this.context.sessionManager.getBranch());
|
|
977
|
+
});
|
|
978
|
+
}
|
|
979
|
+
readback() {
|
|
980
|
+
if (this.closed) return Promise.resolve();
|
|
981
|
+
return this.enqueueWrite(() => {
|
|
982
|
+
if (this.closed) return;
|
|
983
|
+
const entries = this.context.sessionManager.getBranch();
|
|
984
|
+
const sessionId = this.context.sessionManager.getSessionId();
|
|
985
|
+
const branch = this.context.sessionManager.getLeafId();
|
|
986
|
+
const recorded = this.context.sessionManager.getRecordedCwd?.();
|
|
987
|
+
const active = /* @__PURE__ */ new Set();
|
|
988
|
+
for (const entry of entries) {
|
|
989
|
+
const payloadJson = canonicalLcmPayload(entry);
|
|
990
|
+
const message = entry.type === "message" ? entry.message : void 0;
|
|
991
|
+
const stored = this.ledger.appendRaw({
|
|
992
|
+
projectKey: this.projectKey,
|
|
993
|
+
sessionId,
|
|
994
|
+
entryId: entry.id,
|
|
995
|
+
role: message?.role ?? entry.type,
|
|
996
|
+
content: message ? JSON.stringify(message.content ?? "") : payloadJson,
|
|
997
|
+
payloadJson,
|
|
998
|
+
parentEntryId: entry.parentId ?? null,
|
|
999
|
+
branch,
|
|
1000
|
+
...recorded || this.context.cwd ? { recordedCwd: recorded || this.context.cwd } : {}
|
|
1001
|
+
});
|
|
1002
|
+
active.add(`${sessionId}:${stored.entryId}:${stored.revision}`);
|
|
1003
|
+
}
|
|
1004
|
+
this.activeSources = active;
|
|
1005
|
+
this.activeSessionId = sessionId;
|
|
1006
|
+
this.dirty = false;
|
|
1007
|
+
this.degradedError = void 0;
|
|
1008
|
+
});
|
|
1009
|
+
}
|
|
1010
|
+
async syncAndSchedule() {
|
|
1011
|
+
if (this.closed) return;
|
|
1012
|
+
await this.readback();
|
|
1013
|
+
this.scheduleMaintenance();
|
|
1014
|
+
}
|
|
1015
|
+
maintain() {
|
|
1016
|
+
return this.syncAndSchedule();
|
|
1017
|
+
}
|
|
1018
|
+
scheduleMaintenance() {
|
|
1019
|
+
if (this.closed) return;
|
|
1020
|
+
const run = this.maintenancePending.then(() => this.runMaintenance());
|
|
1021
|
+
this.maintenancePending = run.catch((error) => {
|
|
1022
|
+
this.degradedError = error;
|
|
1023
|
+
});
|
|
1024
|
+
}
|
|
1025
|
+
async runMaintenance() {
|
|
1026
|
+
if (this.closed || !this.activeSessionId || this.activeSources.size === 0) return;
|
|
1027
|
+
let model;
|
|
1028
|
+
try {
|
|
1029
|
+
model = new LcmModelAdapter(this.context, this.options.summaryModel, true, {
|
|
1030
|
+
...this.options.lcmMaxInputChars === void 0 ? {} : { maxInputChars: this.options.lcmMaxInputChars },
|
|
1031
|
+
...this.options.lcmMaxOutputTokens === void 0 ? {} : { maxOutputTokens: this.options.lcmMaxOutputTokens },
|
|
1032
|
+
...this.options.lcmMaxOutputChars === void 0 ? {} : { maxOutputChars: this.options.lcmMaxOutputChars }
|
|
1033
|
+
});
|
|
1034
|
+
} catch (error) {
|
|
1035
|
+
this.degradedError = error;
|
|
1036
|
+
model = { modelHash: "unavailable", generate: async () => {
|
|
1037
|
+
throw error instanceof Error ? error : new Error(String(error));
|
|
1038
|
+
} };
|
|
1039
|
+
}
|
|
1040
|
+
const fanIn = Math.max(2, this.options.maxCondenseChildren ?? 4);
|
|
1041
|
+
const passes = Math.max(1, this.options.maxMaintenancePasses ?? 4);
|
|
1042
|
+
const sessionId = this.activeSessionId;
|
|
1043
|
+
const raw = this.ledger.readRaw(this.projectKey, sessionId).filter(
|
|
1044
|
+
(entry) => this.activeSources.has(`${entry.sessionId}:${entry.entryId}:${entry.revision}`)
|
|
1045
|
+
);
|
|
1046
|
+
const inputFor = (node) => {
|
|
1047
|
+
if (node.kind === "condensed") return node.children.map((id) => this.maintenance.getNode(id)?.text ?? "").filter(Boolean).join("\n\n");
|
|
1048
|
+
return node.sources.map((source) => {
|
|
1049
|
+
const entry = this.ledger.readRawEntry(this.projectKey, source.sessionId, source.entryId, source.revision);
|
|
1050
|
+
if (!entry || entry.payloadHash !== source.payloadHash) throw new Error("stale source");
|
|
1051
|
+
return entry.payloadJson;
|
|
1052
|
+
}).join("\n");
|
|
1053
|
+
};
|
|
1054
|
+
const runJob = async (job, node) => {
|
|
1055
|
+
try {
|
|
1056
|
+
await this.maintenance.run(job, model, inputFor(node), this.signal);
|
|
1057
|
+
} catch {
|
|
1058
|
+
}
|
|
1059
|
+
};
|
|
1060
|
+
for (let pass = 0; pass < passes && !this.closed; pass += 1) {
|
|
1061
|
+
const leafCandidate = this.maintenance.createLeaf(this.maintenance.selectLeaf(raw, this.context.sessionManager.getLeafId()));
|
|
1062
|
+
const leaf = leafCandidate ? this.maintenance.getNode(leafCandidate.nodeId) : void 0;
|
|
1063
|
+
if (leaf?.state === "pending") {
|
|
1064
|
+
const job = this.maintenance.listJobs().find((item) => item.nodeId === leaf.nodeId);
|
|
1065
|
+
if (job) await runJob(job, leaf);
|
|
1066
|
+
}
|
|
1067
|
+
const branch = this.context.sessionManager.getLeafId();
|
|
1068
|
+
const pending = this.maintenance.listJobs().filter((job) => job.state === "pending").flatMap((job) => {
|
|
1069
|
+
const node = this.maintenance.getNode(job.nodeId);
|
|
1070
|
+
if (!node || node.sessionId !== sessionId || branch !== null && node.branch !== branch && node.branch !== null || !node.sources.every((source) => this.activeSources.has(`${source.sessionId}:${source.entryId}:${source.revision}`))) return [];
|
|
1071
|
+
return [{ job, node }];
|
|
1072
|
+
}).slice(0, passes);
|
|
1073
|
+
for (const item of pending) await runJob(item.job, item.node);
|
|
1074
|
+
const children = this.maintenance.selectCondensation(sessionId, this.activeSources, this.context.sessionManager.getLeafId());
|
|
1075
|
+
if (children.length >= fanIn) {
|
|
1076
|
+
const node = this.maintenance.createCondensed(children);
|
|
1077
|
+
if (node) {
|
|
1078
|
+
const job = this.maintenance.listJobs().find((item) => item.nodeId === node.nodeId);
|
|
1079
|
+
if (job) await runJob(job, node);
|
|
1080
|
+
}
|
|
1081
|
+
}
|
|
1082
|
+
if (!leaf && pending.length === 0 && children.length < fanIn) break;
|
|
1083
|
+
}
|
|
1084
|
+
}
|
|
1085
|
+
compact(input) {
|
|
1086
|
+
const persistedEntries = /* @__PURE__ */ new Map();
|
|
1087
|
+
for (const entry of input.branchEntries) {
|
|
1088
|
+
const message = entry.type === "message" ? entry.message : void 0;
|
|
1089
|
+
const payloadJson = canonicalLcmPayload(entry);
|
|
1090
|
+
const stored = this.ledger.appendRaw({
|
|
1091
|
+
projectKey: this.projectKey,
|
|
1092
|
+
sessionId: input.sessionId,
|
|
1093
|
+
entryId: entry.id,
|
|
1094
|
+
role: message?.role ?? entry.type,
|
|
1095
|
+
content: message ? JSON.stringify(message.content ?? "") : payloadJson,
|
|
1096
|
+
payloadJson,
|
|
1097
|
+
parentEntryId: entry.parentId ?? null,
|
|
1098
|
+
branch: input.branch
|
|
1099
|
+
});
|
|
1100
|
+
persistedEntries.set(`${entry.id}:${stored.payloadHash}`, stored);
|
|
1101
|
+
}
|
|
1102
|
+
const firstKeptIndex = input.firstKeptEntryId ? input.branchEntries.findIndex((entry) => entry.id === input.firstKeptEntryId) : input.branchEntries.length;
|
|
1103
|
+
const sourceEntries = input.branchEntries.slice(0, firstKeptIndex < 0 ? input.branchEntries.length : firstKeptIndex);
|
|
1104
|
+
const selectedStored = sourceEntries.map((entry) => {
|
|
1105
|
+
const payloadHash = hashLcmPayload(JSON.parse(canonicalLcmPayload(entry)));
|
|
1106
|
+
const stored = persistedEntries.get(`${entry.id}:${payloadHash}`);
|
|
1107
|
+
if (!stored) throw new Error("compaction source was not persisted");
|
|
1108
|
+
return stored;
|
|
1109
|
+
});
|
|
1110
|
+
const activeSources = new Set(selectedStored.map((entry) => `${entry.sessionId}:${entry.entryId}:${entry.revision}`));
|
|
1111
|
+
const frontier = this.maintenance.getFrontier(input.sessionId, input.branch, activeSources);
|
|
1112
|
+
const coveredSources = new Set(frontier.flatMap((node) => node.sources.map((source) => `${source.sessionId}:${source.entryId}:${source.revision}`)));
|
|
1113
|
+
const summary = frontier.map((node) => node.text ?? "").filter(Boolean).join("\n\n");
|
|
1114
|
+
if (summary && selectedStored.every((entry) => coveredSources.has(`${entry.sessionId}:${entry.entryId}:${entry.revision}`))) return { summary, firstKeptEntryId: input.firstKeptEntryId, tokensBefore: input.tokensBefore, source: "ready-frontier", branch: input.branch };
|
|
1115
|
+
if (selectedStored.length === 0) throw new Error("LCM emergency fallback has no persisted sources");
|
|
1116
|
+
const payloads = sourceEntries.map((entry) => canonicalLcmPayload(entry)).join("\n");
|
|
1117
|
+
const sources = selectedStored.map((stored) => ({ sessionId: stored.sessionId, entryId: stored.entryId, revision: stored.revision, payloadHash: stored.payloadHash }));
|
|
1118
|
+
const fallback = emergencyReduce(payloads, this.options.lcmMaxOutputChars ?? 4096, sources);
|
|
1119
|
+
const leaf = this.maintenance.createLeaf(selectedStored);
|
|
1120
|
+
if (!leaf) throw new Error("LCM emergency fallback node was not created");
|
|
1121
|
+
const persisted = this.maintenance.getNode(leaf.nodeId);
|
|
1122
|
+
if (persisted?.state === "ready") {
|
|
1123
|
+
if (persisted.branch !== input.branch || JSON.stringify(persisted.sources) !== JSON.stringify(sources)) throw new Error("LCM emergency fallback provenance mismatch");
|
|
1124
|
+
if (!persisted.text) throw new Error("LCM emergency fallback text is missing");
|
|
1125
|
+
return { summary: persisted.text, firstKeptEntryId: input.firstKeptEntryId, tokensBefore: input.tokensBefore, source: "emergency", branch: input.branch };
|
|
1126
|
+
}
|
|
1127
|
+
const job = this.maintenance.listJobs().find((item) => item.nodeId === leaf.nodeId);
|
|
1128
|
+
if (!job) throw new Error("LCM emergency fallback job was not created");
|
|
1129
|
+
const completed = this.maintenance.completeEmergency(this.maintenance.claimEmergency(job.jobId), fallback);
|
|
1130
|
+
return { summary: completed.text ?? fallback, firstKeptEntryId: input.firstKeptEntryId, tokensBefore: input.tokensBefore, source: "emergency", branch: input.branch };
|
|
1131
|
+
}
|
|
1132
|
+
memoryContext() {
|
|
1133
|
+
return {
|
|
1134
|
+
ledger: {
|
|
1135
|
+
projectKey: this.projectKey,
|
|
1136
|
+
readRaw: (sessionId) => this.ledger.readRaw(this.projectKey, sessionId),
|
|
1137
|
+
readRawPage: (sessionId, offset, limit) => this.ledger.readRawPage(this.projectKey, sessionId, offset, limit),
|
|
1138
|
+
readRawEntry: (sessionId, entryId, revision) => this.ledger.readRawEntry(this.projectKey, sessionId, entryId, revision)
|
|
1139
|
+
},
|
|
1140
|
+
...this.activeSessionId === void 0 ? {} : { currentSessionId: this.activeSessionId },
|
|
1141
|
+
summaries: {
|
|
1142
|
+
listNodes: ({ sessionId, branch, limit }) => this.maintenance.getFrontier(sessionId, branch, sessionId === this.activeSessionId ? this.activeSources : void 0).slice(0, limit).map((node) => ({ ...node, text: node.text ?? "", sources: node.sources.map((source) => ({ entryId: source.entryId, revision: source.revision, contentHash: source.payloadHash })) })),
|
|
1143
|
+
getNode: (nodeId) => {
|
|
1144
|
+
const node = this.maintenance.getNode(nodeId);
|
|
1145
|
+
return node ? { ...node, text: node.text ?? "", sources: node.sources.map((source) => ({ entryId: source.entryId, revision: source.revision, contentHash: source.payloadHash })) } : void 0;
|
|
1146
|
+
}
|
|
1147
|
+
},
|
|
1148
|
+
branchForSession: (sessionId) => sessionId === this.activeSessionId ? { branch: this.context.sessionManager.getLeafId(), activeSourceKeys: [...this.activeSources], ready: this.status === "healthy" } : void 0
|
|
1149
|
+
};
|
|
1150
|
+
}
|
|
1151
|
+
frontier(sessionId, branch) {
|
|
1152
|
+
return this.maintenance.getFrontier(sessionId, branch, sessionId === this.activeSessionId ? this.activeSources : void 0);
|
|
1153
|
+
}
|
|
1154
|
+
raw(sessionId) {
|
|
1155
|
+
return this.ledger.readRaw(this.projectKey, sessionId);
|
|
1156
|
+
}
|
|
1157
|
+
async shutdown() {
|
|
1158
|
+
if (this.closed) return;
|
|
1159
|
+
this.closed = true;
|
|
1160
|
+
this.abort.abort();
|
|
1161
|
+
await this.writePending.catch(() => void 0);
|
|
1162
|
+
await this.maintenancePending.catch(() => void 0);
|
|
1163
|
+
this.ledger.close();
|
|
1164
|
+
}
|
|
1165
|
+
};
|
|
1166
|
+
export {
|
|
1167
|
+
LcmRuntime
|
|
1168
|
+
};
|
|
1169
|
+
//# sourceMappingURL=lcm-runtime.js.map
|