@swestash/agent-lens 0.17.1 → 0.17.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-lens.js +229 -100
- package/dist/agent-lens.js.map +1 -1
- package/package.json +1 -1
- package/web/404.html +2 -2
- package/web/assets/{AboutView-Crn85usU.js → AboutView-cNKRITEW.js} +1 -1
- package/web/assets/{Dashboard-CH8nv5ZK.js → Dashboard-AjpnMF4y.js} +1 -1
- package/web/assets/{FileView-CtovgU1g.js → FileView-dsp2ReoN.js} +1 -1
- package/web/assets/{FilesView-38qLTWbj.js → FilesView-CedaWAxS.js} +1 -1
- package/web/assets/{SecurityView-CmKLXZP9.js → SecurityView-DDBov1tl.js} +1 -1
- package/web/assets/{SessionView-Bt-NmoU3.js → SessionView-Y7_R_3iF.js} +1 -1
- package/web/assets/{SkillView-DQEklf89.js → SkillView-CHYZYZwQ.js} +1 -1
- package/web/assets/{SkillsView-CNX1nAkf.js → SkillsView-DWMs97v0.js} +1 -1
- package/web/assets/{WorkflowView-CtFEKBzS.js → WorkflowView-BThiSvTY.js} +1 -1
- package/web/assets/{index-C5nCASCF.js → index-CLk0qZiT.js} +1 -1
- package/web/assets/{index-DYCDqLKI.js → index-CSqgNF0A.js} +2 -2
- package/web/assets/index-FnqCa_GV.css +1 -0
- package/web/assets/{jsonish-iCEt12Ty.js → jsonish-DTF7FYHm.js} +1 -1
- package/web/assets/{theme-DzMGF9VA.js → theme-BvznOeGO.js} +1 -1
- package/web/assets/{useResetOn-Bvmnyxvw.js → useResetOn-CnN8pwt1.js} +1 -1
- package/web/index.html +2 -2
- package/web/assets/index-DN9qB28e.css +0 -1
package/dist/agent-lens.js
CHANGED
|
@@ -592,6 +592,11 @@ var cac = (name = "") => new CAC(name);
|
|
|
592
592
|
|
|
593
593
|
// ../core/dist/schema.js
|
|
594
594
|
var SCHEMA_VERSION = 15;
|
|
595
|
+
var ANALYZE_SQL = "PRAGMA analysis_limit = 1000; ANALYZE;";
|
|
596
|
+
var BULK_DEFERRED = {
|
|
597
|
+
indexes: ["idx_events_type", "idx_events_ts", "idx_events_turn_reply", "idx_events_uuid_cover"],
|
|
598
|
+
triggers: ["events_ai", "events_ad", "events_au"]
|
|
599
|
+
};
|
|
595
600
|
var SCHEMA_SQL = (
|
|
596
601
|
/* sql */
|
|
597
602
|
`
|
|
@@ -694,9 +699,22 @@ CREATE TABLE IF NOT EXISTS events (
|
|
|
694
699
|
source_file TEXT -- archive path the canonical copy came from
|
|
695
700
|
);
|
|
696
701
|
CREATE INDEX IF NOT EXISTS idx_events_session ON events(session_id);
|
|
697
|
-
CREATE INDEX IF NOT EXISTS idx_events_turn ON events(turn_id);
|
|
698
702
|
CREATE INDEX IF NOT EXISTS idx_events_type ON events(type);
|
|
699
703
|
CREATE INDEX IF NOT EXISTS idx_events_ts ON events(timestamp);
|
|
704
|
+
-- Covering indexes for the dashboard joins. An events row averages ~3.4 KB (raw_json + text), so
|
|
705
|
+
-- about one row fits per page, and a join that reads the row instead of an index pays one random
|
|
706
|
+
-- page read per event: on the real corpus /dashboard/time took 31s from a cold cache. These hold
|
|
707
|
+
-- exactly the columns those joins read. The PK lookups (token_usage, tool_calls, findings,
|
|
708
|
+
-- file_changes \u2192 events by uuid) want timestamp and model, and model latency wants the first
|
|
709
|
+
-- assistant timestamp per turn. Both are index-only scans now, and the guard in
|
|
710
|
+
-- packages/server/test/query-plans.test.ts keeps them that way.
|
|
711
|
+
-- No SCHEMA_VERSION bump: indexes need no rebuild, and CREATE IF NOT EXISTS adds them on the next
|
|
712
|
+
-- incremental ingest. A bump would make that ingest refuse and demand --full.
|
|
713
|
+
-- An index's columns cannot be widened in place, so idx_events_turn is replaced under a new name.
|
|
714
|
+
-- idx_events_turn_reply serves every lookup the old one did, because turn_id is still its prefix.
|
|
715
|
+
DROP INDEX IF EXISTS idx_events_turn;
|
|
716
|
+
CREATE INDEX IF NOT EXISTS idx_events_turn_reply ON events(turn_id, role, timestamp);
|
|
717
|
+
CREATE INDEX IF NOT EXISTS idx_events_uuid_cover ON events(uuid, timestamp, model);
|
|
700
718
|
|
|
701
719
|
-- Tool invocations (incl. subagents / Task calls). -------------------------
|
|
702
720
|
CREATE TABLE IF NOT EXISTS tool_calls (
|
|
@@ -721,13 +739,32 @@ CREATE TABLE IF NOT EXISTS tool_calls (
|
|
|
721
739
|
input_json TEXT, -- tool input, verbatim
|
|
722
740
|
result_summary TEXT -- short result projection (no secrets dumped wholesale)
|
|
723
741
|
);
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
742
|
+
-- Covering indexes for the dashboard aggregates. A tool_calls row averages ~2.3 KB (input_json), and
|
|
743
|
+
-- the aggregates read only a handful of narrow columns. Each index below carries all of them, so
|
|
744
|
+
-- whether the planner drives from sessions (a selective source/date filter) or scans tool_calls
|
|
745
|
+
-- (no filter), it never reads the row. Before this, the errored-calls series scanned the whole
|
|
746
|
+
-- table, 7.5s cold on the real corpus. packages/server/test/query-plans.test.ts holds the line.
|
|
747
|
+
-- They replace the single-column session/tool/skill/skill_id/error_type indexes under new names, because
|
|
748
|
+
-- CREATE IF NOT EXISTS cannot widen an index that already exists. Each new index begins with the old
|
|
749
|
+
-- index's column, so every existing lookup still uses it. Same no-bump reasoning as the events
|
|
750
|
+
-- indexes above.
|
|
751
|
+
DROP INDEX IF EXISTS idx_tool_calls_session;
|
|
752
|
+
DROP INDEX IF EXISTS idx_tool_calls_tool;
|
|
753
|
+
DROP INDEX IF EXISTS idx_tool_calls_skill;
|
|
754
|
+
DROP INDEX IF EXISTS idx_tool_calls_skill_id;
|
|
755
|
+
DROP INDEX IF EXISTS idx_tool_calls_error_type;
|
|
756
|
+
CREATE INDEX IF NOT EXISTS idx_tool_calls_by_session
|
|
757
|
+
ON tool_calls(session_id, tool_name, status, error_type, skill_name, skill_id, agent_type, event_uuid);
|
|
758
|
+
CREATE INDEX IF NOT EXISTS idx_tool_calls_by_tool ON tool_calls(tool_name, session_id, status, error_type, event_uuid);
|
|
759
|
+
CREATE INDEX IF NOT EXISTS idx_tool_calls_by_skill ON tool_calls(skill_name, skill_id, session_id, event_uuid);
|
|
760
|
+
CREATE INDEX IF NOT EXISTS idx_tool_calls_by_skill_id ON tool_calls(skill_id, skill_name, session_id, event_uuid);
|
|
761
|
+
CREATE INDEX IF NOT EXISTS idx_tool_calls_by_error ON tool_calls(error_type, status, session_id, event_uuid);
|
|
728
762
|
CREATE INDEX IF NOT EXISTS idx_tool_calls_spawned ON tool_calls(spawned_session_id);
|
|
729
763
|
CREATE INDEX IF NOT EXISTS idx_tool_calls_workflow_run ON tool_calls(workflow_run_id);
|
|
730
|
-
|
|
764
|
+
-- Every column that REFERENCES turns(id) needs an index: deleting a turn makes SQLite look up its
|
|
765
|
+
-- referencing rows, and without one that is a full scan of this table per deleted turn. The
|
|
766
|
+
-- incremental rebuild deletes the dirty sessions' turns, so five changed files cost 12.9 GB of reads.
|
|
767
|
+
CREATE INDEX IF NOT EXISTS idx_tool_calls_turn ON tool_calls(turn_id);
|
|
731
768
|
|
|
732
769
|
-- Skill *versions* (content-addressed). A skill firing injects the full SKILL.md body into the
|
|
733
770
|
-- transcript as an isMeta user event (begins "Base directory for this skill: \u2026", then the body,
|
|
@@ -767,6 +804,8 @@ CREATE TABLE IF NOT EXISTS token_usage (
|
|
|
767
804
|
service_tier TEXT
|
|
768
805
|
);
|
|
769
806
|
CREATE INDEX IF NOT EXISTS idx_token_usage_session ON token_usage(session_id);
|
|
807
|
+
-- Foreign-key lookup for deleting turns; see idx_tool_calls_turn.
|
|
808
|
+
CREATE INDEX IF NOT EXISTS idx_token_usage_turn ON token_usage(turn_id);
|
|
770
809
|
-- Partial so rare usage rows lacking a message_id (e.g. <synthetic>) don't collide on NULL. --
|
|
771
810
|
CREATE UNIQUE INDEX IF NOT EXISTS idx_token_usage_msg
|
|
772
811
|
ON token_usage(session_id, message_id) WHERE message_id IS NOT NULL;
|
|
@@ -2282,9 +2321,11 @@ import { dirname as dirname6, join as join9 } from "node:path";
|
|
|
2282
2321
|
|
|
2283
2322
|
// ../ingest/dist/db.js
|
|
2284
2323
|
import { DatabaseSync } from "node:sqlite";
|
|
2324
|
+
var WAL_SIZE_LIMIT = 64 * 1024 * 1024;
|
|
2285
2325
|
function openRaw(file) {
|
|
2286
2326
|
const db = new DatabaseSync(file);
|
|
2287
2327
|
db.exec("PRAGMA journal_mode = WAL");
|
|
2328
|
+
db.exec(`PRAGMA journal_size_limit = ${WAL_SIZE_LIMIT}`);
|
|
2288
2329
|
db.exec("PRAGMA foreign_keys = ON");
|
|
2289
2330
|
return db;
|
|
2290
2331
|
}
|
|
@@ -2304,7 +2345,7 @@ function openDb(file) {
|
|
|
2304
2345
|
applySchema(db);
|
|
2305
2346
|
return db;
|
|
2306
2347
|
}
|
|
2307
|
-
function resetSchema(db) {
|
|
2348
|
+
function resetSchema(db, { bulk = false } = {}) {
|
|
2308
2349
|
db.exec("PRAGMA foreign_keys = OFF");
|
|
2309
2350
|
try {
|
|
2310
2351
|
db.exec(`
|
|
@@ -2331,10 +2372,20 @@ function resetSchema(db) {
|
|
|
2331
2372
|
DROP TABLE IF EXISTS meta;
|
|
2332
2373
|
`);
|
|
2333
2374
|
applySchema(db);
|
|
2375
|
+
if (bulk) {
|
|
2376
|
+
for (const name of BULK_DEFERRED.indexes)
|
|
2377
|
+
db.exec(`DROP INDEX IF EXISTS ${name}`);
|
|
2378
|
+
for (const name of BULK_DEFERRED.triggers)
|
|
2379
|
+
db.exec(`DROP TRIGGER IF EXISTS ${name}`);
|
|
2380
|
+
}
|
|
2334
2381
|
} finally {
|
|
2335
2382
|
db.exec("PRAGMA foreign_keys = ON");
|
|
2336
2383
|
}
|
|
2337
2384
|
}
|
|
2385
|
+
function finishBulkLoad(db) {
|
|
2386
|
+
db.exec(SCHEMA_SQL);
|
|
2387
|
+
db.exec("INSERT INTO events_fts(events_fts) VALUES('rebuild')");
|
|
2388
|
+
}
|
|
2338
2389
|
|
|
2339
2390
|
// ../ingest/dist/adapters/claude-code.js
|
|
2340
2391
|
import { readdirSync as readdirSync2 } from "node:fs";
|
|
@@ -3643,6 +3694,13 @@ function nearestGitRoot(path3, home) {
|
|
|
3643
3694
|
}
|
|
3644
3695
|
return null;
|
|
3645
3696
|
}
|
|
3697
|
+
function sessionCwd(first, last) {
|
|
3698
|
+
const f = strip(first);
|
|
3699
|
+
const l = strip(last);
|
|
3700
|
+
if (f === l || existsSync4(l))
|
|
3701
|
+
return last;
|
|
3702
|
+
return l.startsWith(f === "/" ? "/" : f + "/") ? first : last;
|
|
3703
|
+
}
|
|
3646
3704
|
function canonicalizeProjects(db, opts = {}) {
|
|
3647
3705
|
const home = strip(opts.homeDir ?? homedir3());
|
|
3648
3706
|
const rows = db.prepare("SELECT id, agent_id, path FROM projects").all();
|
|
@@ -3899,6 +3957,10 @@ function durationMs(start, end) {
|
|
|
3899
3957
|
return null;
|
|
3900
3958
|
return Math.max(0, b - a);
|
|
3901
3959
|
}
|
|
3960
|
+
function isTurnStart(e) {
|
|
3961
|
+
const body = e.text?.trim() ?? "";
|
|
3962
|
+
return e.type === "user" && e.is_meta === 0 && body.length > 0 && !isCommandResultCarrier(body);
|
|
3963
|
+
}
|
|
3902
3964
|
function buildTurns(sessionId, rows) {
|
|
3903
3965
|
rows.sort((x, y) => (x.timestamp ?? "").localeCompare(y.timestamp ?? "") || (x.seq ?? 0) - (y.seq ?? 0) || x.uuid.localeCompare(y.uuid));
|
|
3904
3966
|
const turns = [];
|
|
@@ -3906,9 +3968,7 @@ function buildTurns(sessionId, rows) {
|
|
|
3906
3968
|
let cur = null;
|
|
3907
3969
|
let seq = 0;
|
|
3908
3970
|
for (const e of rows) {
|
|
3909
|
-
|
|
3910
|
-
const isPrompt = e.type === "user" && e.is_meta === 0 && body.length > 0 && !isCommandResultCarrier(body);
|
|
3911
|
-
if (isPrompt) {
|
|
3971
|
+
if (isTurnStart(e)) {
|
|
3912
3972
|
cur = {
|
|
3913
3973
|
id: `${sessionId}:${seq}`,
|
|
3914
3974
|
session_id: sessionId,
|
|
@@ -3965,7 +4025,7 @@ function expandDirtyScope(db, dirty) {
|
|
|
3965
4025
|
}
|
|
3966
4026
|
}
|
|
3967
4027
|
function rebuildTurns(db, scope, sessionIds) {
|
|
3968
|
-
const selEvents = db.prepare("SELECT uuid, seq, type, is_sidechain, is_meta, timestamp, model, text FROM events WHERE session_id = ?");
|
|
4028
|
+
const selEvents = db.prepare("SELECT uuid, seq, type, is_sidechain, is_meta, timestamp, model, text, turn_id FROM events WHERE session_id = ?");
|
|
3969
4029
|
const insTurn = db.prepare(`INSERT INTO turns (id, session_id, seq, user_event_uuid, prompt_preview, model, started_at, ended_at, duration_ms)
|
|
3970
4030
|
VALUES (@id, @session_id, @seq, @user_event_uuid, @prompt_preview, @model, @started_at, @ended_at, @duration_ms)`);
|
|
3971
4031
|
const updEventTurn = db.prepare("UPDATE events SET turn_id = ? WHERE uuid = ?");
|
|
@@ -3975,8 +4035,11 @@ function rebuildTurns(db, scope, sessionIds) {
|
|
|
3975
4035
|
const { turns, eventTurn } = buildTurns(id, rows);
|
|
3976
4036
|
for (const t of turns)
|
|
3977
4037
|
runNamed(insTurn, t);
|
|
3978
|
-
for (const
|
|
3979
|
-
|
|
4038
|
+
for (const r of rows) {
|
|
4039
|
+
const turnId = eventTurn.get(r.uuid) ?? null;
|
|
4040
|
+
if (turnId !== r.turn_id)
|
|
4041
|
+
updEventTurn.run(turnId, r.uuid);
|
|
4042
|
+
}
|
|
3980
4043
|
}
|
|
3981
4044
|
});
|
|
3982
4045
|
tx();
|
|
@@ -4156,26 +4219,28 @@ function rebuildDerived(db, dirty) {
|
|
|
4156
4219
|
andId: incremental ? " AND id IN (SELECT id FROM _dirty)" : "",
|
|
4157
4220
|
andSelfId: incremental ? " AND sessions.id IN (SELECT id FROM _dirty)" : ""
|
|
4158
4221
|
};
|
|
4159
|
-
|
|
4160
|
-
|
|
4161
|
-
|
|
4162
|
-
|
|
4163
|
-
|
|
4164
|
-
|
|
4165
|
-
|
|
4166
|
-
|
|
4167
|
-
|
|
4168
|
-
|
|
4169
|
-
|
|
4170
|
-
|
|
4171
|
-
|
|
4172
|
-
|
|
4173
|
-
|
|
4174
|
-
|
|
4175
|
-
|
|
4176
|
-
|
|
4177
|
-
|
|
4178
|
-
|
|
4222
|
+
return transaction(db, () => {
|
|
4223
|
+
if (dirty != null)
|
|
4224
|
+
expandDirtyScope(db, dirty);
|
|
4225
|
+
db.exec("PRAGMA defer_foreign_keys = ON");
|
|
4226
|
+
db.exec(`UPDATE token_usage SET turn_id = NULL${scope.bySession}`);
|
|
4227
|
+
db.exec(`UPDATE tool_calls SET turn_id = NULL${scope.bySession}`);
|
|
4228
|
+
db.exec(`UPDATE tool_calls SET skill_id = NULL${scope.bySession}`);
|
|
4229
|
+
db.exec(`UPDATE sessions SET parent_turn_id = NULL${scope.byId}`);
|
|
4230
|
+
db.exec(`DELETE FROM turns${scope.bySession}`);
|
|
4231
|
+
const sessionIds = db.prepare(`SELECT id FROM sessions${scope.byId}`).all();
|
|
4232
|
+
rebuildTurns(db, scope, sessionIds);
|
|
4233
|
+
linkSkillVersions(db, sessionIds);
|
|
4234
|
+
linkParents(db, scope);
|
|
4235
|
+
recomputeSessionAggregates(db, scope);
|
|
4236
|
+
pruneOrphans(db, scope, incremental);
|
|
4237
|
+
if (incremental) {
|
|
4238
|
+
const expanded = new Set(db.prepare("SELECT id FROM _dirty").all().map((r) => r.id));
|
|
4239
|
+
db.exec("DROP TABLE IF EXISTS _dirty");
|
|
4240
|
+
return expanded;
|
|
4241
|
+
}
|
|
4242
|
+
return null;
|
|
4243
|
+
})();
|
|
4179
4244
|
}
|
|
4180
4245
|
function prepareStatements(db) {
|
|
4181
4246
|
return {
|
|
@@ -4232,9 +4297,11 @@ function prepareStatements(db) {
|
|
|
4232
4297
|
workflow_run_id = COALESCE(excluded.workflow_run_id, workflow_run_id)`)
|
|
4233
4298
|
};
|
|
4234
4299
|
}
|
|
4235
|
-
function ingestFile(db, stmts, adapter, file, lines, meta, now2, stats) {
|
|
4300
|
+
function ingestFile(db, stmts, adapter, file, lines, meta, now2, stats, opts = {}) {
|
|
4236
4301
|
const sessionMeta = {};
|
|
4237
4302
|
let eventsInFile = 0;
|
|
4303
|
+
let turnSeq = -1;
|
|
4304
|
+
let firstCwd = null;
|
|
4238
4305
|
const tx = transaction(db, () => {
|
|
4239
4306
|
stmts.insSessionStub.run(file.sessionId, adapter.agentId, file.sourceId);
|
|
4240
4307
|
let seq = 0;
|
|
@@ -4249,10 +4316,19 @@ function ingestFile(db, stmts, adapter, file, lines, meta, now2, stats) {
|
|
|
4249
4316
|
continue;
|
|
4250
4317
|
}
|
|
4251
4318
|
const parsed = adapter.parseLine(raw, file, seq++);
|
|
4252
|
-
if (parsed.meta)
|
|
4319
|
+
if (parsed.meta) {
|
|
4320
|
+
if (parsed.meta.cwd && firstCwd === null)
|
|
4321
|
+
firstCwd = parsed.meta.cwd;
|
|
4253
4322
|
Object.assign(sessionMeta, parsed.meta);
|
|
4323
|
+
}
|
|
4254
4324
|
if (parsed.event) {
|
|
4255
|
-
|
|
4325
|
+
let turnId = parsed.event.turn_id;
|
|
4326
|
+
if (opts.provisionalTurns) {
|
|
4327
|
+
if (isTurnStart(parsed.event))
|
|
4328
|
+
turnSeq++;
|
|
4329
|
+
turnId = turnSeq < 0 ? null : `${file.sessionId}:${turnSeq}`;
|
|
4330
|
+
}
|
|
4331
|
+
const info = stmts.insEvent.run({ ...parsed.event, turn_id: turnId, raw_json: packRaw(parsed.event.raw_json) });
|
|
4256
4332
|
if (info.changes > 0) {
|
|
4257
4333
|
stats.newEvents++;
|
|
4258
4334
|
eventsInFile++;
|
|
@@ -4281,11 +4357,12 @@ function ingestFile(db, stmts, adapter, file, lines, meta, now2, stats) {
|
|
|
4281
4357
|
}
|
|
4282
4358
|
let projectId2 = null;
|
|
4283
4359
|
if (sessionMeta.cwd) {
|
|
4284
|
-
|
|
4360
|
+
const cwd = sessionCwd(firstCwd ?? sessionMeta.cwd, sessionMeta.cwd);
|
|
4361
|
+
projectId2 = createHash5("sha1").update(`${adapter.agentId}\0${cwd}`).digest("hex").slice(0, 16);
|
|
4285
4362
|
stmts.upsertProject.run({
|
|
4286
4363
|
id: projectId2,
|
|
4287
4364
|
agent_id: adapter.agentId,
|
|
4288
|
-
path:
|
|
4365
|
+
path: cwd,
|
|
4289
4366
|
encoded_dir: file.encodedDir || null,
|
|
4290
4367
|
now: now2
|
|
4291
4368
|
});
|
|
@@ -4624,6 +4701,13 @@ function shouldReingest(path3, size, mtimeMs, prev) {
|
|
|
4624
4701
|
return { action: "skip-hash", hash: hash2 };
|
|
4625
4702
|
return { action: "ingest", hash: hash2, buf };
|
|
4626
4703
|
}
|
|
4704
|
+
function assertForeignKeys(db) {
|
|
4705
|
+
const bad = db.prepare("PRAGMA foreign_key_check").all();
|
|
4706
|
+
if (!bad.length)
|
|
4707
|
+
return;
|
|
4708
|
+
const edges = [...new Set(bad.map((b) => `${b.table} -> ${b.parent}`))].join(", ");
|
|
4709
|
+
throw new Error(`agent-lens-ingest: --full produced ${bad.length} foreign-key violations (${edges}); rolled back`);
|
|
4710
|
+
}
|
|
4627
4711
|
function printIngestReport(db, dbPath, r) {
|
|
4628
4712
|
const count = (sql) => db.prepare(sql).get().n;
|
|
4629
4713
|
const sessions = count("SELECT COUNT(*) n FROM sessions");
|
|
@@ -4670,9 +4754,7 @@ function runIngest(argv = process.argv.slice(2)) {
|
|
|
4670
4754
|
}
|
|
4671
4755
|
mkdirSync4(dirname6(dbPath), { recursive: true });
|
|
4672
4756
|
const db = args.full ? openRaw(dbPath) : openDb(dbPath);
|
|
4673
|
-
if (args.full) {
|
|
4674
|
-
resetSchema(db);
|
|
4675
|
-
} else {
|
|
4757
|
+
if (!args.full) {
|
|
4676
4758
|
const dbVer = readSchemaVersion(db);
|
|
4677
4759
|
if (dbVer != null && dbVer !== SCHEMA_VERSION) {
|
|
4678
4760
|
console.error(`agent-lens-ingest: schema mismatch \u2014 DB is v${dbVer}, this build expects v${SCHEMA_VERSION}. Run 'agent-lens ingest --full' to rebuild.`);
|
|
@@ -4686,62 +4768,81 @@ function runIngest(argv = process.argv.slice(2)) {
|
|
|
4686
4768
|
const now2 = (/* @__PURE__ */ new Date()).toISOString();
|
|
4687
4769
|
const excludePaths = loadExcludes();
|
|
4688
4770
|
const excludedDirs = parseExcludes(excludePaths.join(","));
|
|
4689
|
-
const pruned = pruneExcluded(db, excludePaths);
|
|
4690
|
-
|
|
4691
|
-
|
|
4692
|
-
|
|
4693
|
-
|
|
4694
|
-
|
|
4695
|
-
|
|
4696
|
-
|
|
4697
|
-
|
|
4698
|
-
|
|
4699
|
-
|
|
4700
|
-
|
|
4701
|
-
|
|
4702
|
-
|
|
4703
|
-
|
|
4704
|
-
|
|
4705
|
-
|
|
4706
|
-
|
|
4707
|
-
|
|
4708
|
-
|
|
4709
|
-
|
|
4710
|
-
|
|
4711
|
-
|
|
4712
|
-
|
|
4713
|
-
|
|
4714
|
-
|
|
4715
|
-
|
|
4716
|
-
|
|
4717
|
-
|
|
4718
|
-
|
|
4719
|
-
|
|
4720
|
-
|
|
4721
|
-
|
|
4722
|
-
|
|
4723
|
-
|
|
4724
|
-
|
|
4725
|
-
|
|
4726
|
-
|
|
4727
|
-
|
|
4728
|
-
|
|
4771
|
+
const pruned = args.full ? 0 : pruneExcluded(db, excludePaths);
|
|
4772
|
+
if (args.full)
|
|
4773
|
+
db.exec("PRAGMA foreign_keys = OFF");
|
|
4774
|
+
let report;
|
|
4775
|
+
try {
|
|
4776
|
+
report = transaction(db, () => {
|
|
4777
|
+
if (args.full)
|
|
4778
|
+
resetSchema(db, { bulk: true });
|
|
4779
|
+
const stmts = prepareStatements(db);
|
|
4780
|
+
const stats = newStats();
|
|
4781
|
+
const wfStats = newWorkflowStats();
|
|
4782
|
+
const metaStats = newMetaStats();
|
|
4783
|
+
const trStats = newToolResultStats();
|
|
4784
|
+
const dirty = /* @__PURE__ */ new Set();
|
|
4785
|
+
for (const source of sources) {
|
|
4786
|
+
const adapter = adapterById.get(source.agent);
|
|
4787
|
+
if (!adapter) {
|
|
4788
|
+
console.warn(`agent-lens-ingest: no adapter for agent '${source.agent}' (source '${source.label}') \u2014 skipping`);
|
|
4789
|
+
continue;
|
|
4790
|
+
}
|
|
4791
|
+
stmts.insAgent.run(adapter.agentId, adapter.agentName);
|
|
4792
|
+
stmts.insSource.run({ id: source.label, label: source.label, agent_id: adapter.agentId, config_dir: source.configDir });
|
|
4793
|
+
let files = adapter.discover(join9(archiveRoot, source.label), source.label);
|
|
4794
|
+
if (excludedDirs.length)
|
|
4795
|
+
files = files.filter((f) => !isExcludedArchivePath(f.path, excludedDirs));
|
|
4796
|
+
files.sort((a, b) => Number(a.isVersion) - Number(b.isVersion));
|
|
4797
|
+
for (const file of files) {
|
|
4798
|
+
stats.files++;
|
|
4799
|
+
const st = statSync3(file.path);
|
|
4800
|
+
const mtimeMs = Math.trunc(st.mtimeMs);
|
|
4801
|
+
const prev = args.full ? void 0 : stmts.getState.get(file.path);
|
|
4802
|
+
const decision = shouldReingest(file.path, st.size, mtimeMs, prev);
|
|
4803
|
+
if (decision.action === "skip-stat") {
|
|
4804
|
+
stats.skipped++;
|
|
4805
|
+
continue;
|
|
4806
|
+
}
|
|
4807
|
+
if (decision.action === "skip-hash") {
|
|
4808
|
+
stmts.setState.run({
|
|
4809
|
+
file_path: file.path,
|
|
4810
|
+
size: st.size,
|
|
4811
|
+
mtime_ms: mtimeMs,
|
|
4812
|
+
sha256: decision.hash,
|
|
4813
|
+
events_ingested: prev.events_ingested,
|
|
4814
|
+
ingested_at: now2
|
|
4815
|
+
});
|
|
4816
|
+
stats.skipped++;
|
|
4817
|
+
continue;
|
|
4818
|
+
}
|
|
4819
|
+
const lines = decision.buf ? decision.buf.toString("utf8").split("\n") : streamLines(file.path);
|
|
4820
|
+
ingestFile(db, stmts, adapter, file, lines, { size: st.size, mtimeMs, hash: decision.hash }, now2, stats, { provisionalTurns: args.full });
|
|
4821
|
+
dirty.add(file.sessionId);
|
|
4822
|
+
}
|
|
4823
|
+
ingestWorkflowResults(db, join9(archiveRoot, source.label), source.label, excludedDirs, now2, wfStats, args.full);
|
|
4824
|
+
ingestSubagentMeta(db, join9(archiveRoot, source.label), source.label, excludedDirs, now2, metaStats, args.full);
|
|
4825
|
+
ingestToolResults(db, join9(archiveRoot, source.label), excludedDirs, now2, trStats, args.full);
|
|
4729
4826
|
}
|
|
4730
|
-
const
|
|
4731
|
-
|
|
4732
|
-
|
|
4733
|
-
|
|
4734
|
-
|
|
4735
|
-
|
|
4736
|
-
|
|
4827
|
+
const expanded = rebuildDerived(db, args.full ? null : dirty);
|
|
4828
|
+
if (args.full)
|
|
4829
|
+
finishBulkLoad(db);
|
|
4830
|
+
canonicalizeProjects(db);
|
|
4831
|
+
const classified = classify(db, args.full ? null : expanded);
|
|
4832
|
+
const detected = detect(db, args.full ? null : expanded);
|
|
4833
|
+
classifyErrors(db, args.full ? null : expanded);
|
|
4834
|
+
const fileChanges = deriveFileChanges(db, args.full ? null : expanded);
|
|
4835
|
+
db.exec(ANALYZE_SQL);
|
|
4836
|
+
if (args.full)
|
|
4837
|
+
assertForeignKeys(db);
|
|
4838
|
+
return { stats, classified, detected, fileChanges, wfStats, metaStats, trStats };
|
|
4839
|
+
})();
|
|
4840
|
+
} finally {
|
|
4841
|
+
if (args.full)
|
|
4842
|
+
db.exec("PRAGMA foreign_keys = ON");
|
|
4737
4843
|
}
|
|
4738
|
-
|
|
4739
|
-
|
|
4740
|
-
const classified = classify(db, args.full ? null : expanded);
|
|
4741
|
-
const detected = detect(db, args.full ? null : expanded);
|
|
4742
|
-
classifyErrors(db, args.full ? null : expanded);
|
|
4743
|
-
const fileChanges = deriveFileChanges(db, args.full ? null : expanded);
|
|
4744
|
-
printIngestReport(db, dbPath, { stats, pruned, classified, detected, fileChanges, wfStats, metaStats, trStats });
|
|
4844
|
+
db.exec("PRAGMA wal_checkpoint(TRUNCATE)");
|
|
4845
|
+
printIngestReport(db, dbPath, { ...report, pruned });
|
|
4745
4846
|
}
|
|
4746
4847
|
function runMetrics(argv = process.argv.slice(2)) {
|
|
4747
4848
|
let dbArg = "";
|
|
@@ -4860,9 +4961,11 @@ function costOf(rows) {
|
|
|
4860
4961
|
}
|
|
4861
4962
|
|
|
4862
4963
|
// ../server/dist/db.js
|
|
4964
|
+
var READ_CACHE_KIB = 64 * 1024;
|
|
4863
4965
|
function openReadonly(file) {
|
|
4864
4966
|
const db = new DatabaseSync2(file, { readOnly: true });
|
|
4865
4967
|
db.exec("PRAGMA query_only = ON");
|
|
4968
|
+
db.exec(`PRAGMA cache_size = -${READ_CACHE_KIB}`);
|
|
4866
4969
|
return db;
|
|
4867
4970
|
}
|
|
4868
4971
|
function schemaStatus(db) {
|
|
@@ -6282,6 +6385,31 @@ function setPref(db, key, value) {
|
|
|
6282
6385
|
ON CONFLICT(key) DO UPDATE SET value = excluded.value, updated_at = excluded.updated_at`).run(key, value, (/* @__PURE__ */ new Date()).toISOString());
|
|
6283
6386
|
}
|
|
6284
6387
|
|
|
6388
|
+
// ../server/dist/response-cache.js
|
|
6389
|
+
function dataVersionCache(db, max = 64) {
|
|
6390
|
+
const version2 = db.prepare("PRAGMA data_version");
|
|
6391
|
+
const entries = /* @__PURE__ */ new Map();
|
|
6392
|
+
let seen;
|
|
6393
|
+
return (key, compute) => {
|
|
6394
|
+
const v = version2.get().data_version;
|
|
6395
|
+
if (v !== seen) {
|
|
6396
|
+
entries.clear();
|
|
6397
|
+
seen = v;
|
|
6398
|
+
}
|
|
6399
|
+
if (entries.has(key)) {
|
|
6400
|
+
const hit = entries.get(key);
|
|
6401
|
+
entries.delete(key);
|
|
6402
|
+
entries.set(key, hit);
|
|
6403
|
+
return hit;
|
|
6404
|
+
}
|
|
6405
|
+
const value = compute();
|
|
6406
|
+
entries.set(key, value);
|
|
6407
|
+
if (entries.size > max)
|
|
6408
|
+
entries.delete(entries.keys().next().value);
|
|
6409
|
+
return value;
|
|
6410
|
+
};
|
|
6411
|
+
}
|
|
6412
|
+
|
|
6285
6413
|
// ../server/dist/app.js
|
|
6286
6414
|
function notFound(reply, message = "not found") {
|
|
6287
6415
|
return reply.code(404).send({ error: { code: "NOT_FOUND", message } });
|
|
@@ -6360,19 +6488,20 @@ async function createApp(db, opts = {}) {
|
|
|
6360
6488
|
const q = req.query;
|
|
6361
6489
|
return { source: q.source, from: q.from, to: q.to, models: q.models?.split(",").filter(Boolean) };
|
|
6362
6490
|
};
|
|
6363
|
-
|
|
6491
|
+
const cached2 = dataVersionCache(db);
|
|
6492
|
+
app.get("/api/dashboard/overview", async (req) => cached2(req.url, () => dashboardOverview(db, dashFilters(req))));
|
|
6364
6493
|
app.get("/api/dashboard/timeseries", async (req) => {
|
|
6365
6494
|
const q = req.query;
|
|
6366
|
-
return dashboardTimeseries(db, dashFilters(req), q.bucket);
|
|
6495
|
+
return cached2(req.url, () => dashboardTimeseries(db, dashFilters(req), q.bucket));
|
|
6367
6496
|
});
|
|
6368
|
-
app.get("/api/dashboard/breakdowns", async (req) => dashboardBreakdowns(db, dashFilters(req)));
|
|
6497
|
+
app.get("/api/dashboard/breakdowns", async (req) => cached2(req.url, () => dashboardBreakdowns(db, dashFilters(req))));
|
|
6369
6498
|
app.get("/api/dashboard/time", async (req) => {
|
|
6370
6499
|
const q = req.query;
|
|
6371
|
-
return dashboardTime(db, dashFilters(req), q.bucket);
|
|
6500
|
+
return cached2(req.url, () => dashboardTime(db, dashFilters(req), q.bucket));
|
|
6372
6501
|
});
|
|
6373
6502
|
app.get("/api/dashboard/audit", async (req) => {
|
|
6374
6503
|
const q = req.query;
|
|
6375
|
-
return dashboardAudit(db, dashFilters(req), q.bucket);
|
|
6504
|
+
return cached2(req.url, () => dashboardAudit(db, dashFilters(req), q.bucket));
|
|
6376
6505
|
});
|
|
6377
6506
|
const PREF_KEY = /^[a-z0-9._-]{1,64}$/i;
|
|
6378
6507
|
app.get("/api/prefs/:key", async (req, reply) => {
|