@drewpayment/mink 0.11.0 → 0.12.0-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dashboard/out/404.html +1 -1
- package/dashboard/out/action-log.html +1 -1
- package/dashboard/out/action-log.txt +1 -1
- package/dashboard/out/activity.html +1 -1
- package/dashboard/out/activity.txt +1 -1
- package/dashboard/out/bugs.html +1 -1
- package/dashboard/out/bugs.txt +1 -1
- package/dashboard/out/capture.html +1 -1
- package/dashboard/out/capture.txt +1 -1
- package/dashboard/out/config.html +1 -1
- package/dashboard/out/config.txt +1 -1
- package/dashboard/out/daemon.html +1 -1
- package/dashboard/out/daemon.txt +1 -1
- package/dashboard/out/design.html +1 -1
- package/dashboard/out/design.txt +1 -1
- package/dashboard/out/discord.html +1 -1
- package/dashboard/out/discord.txt +1 -1
- package/dashboard/out/file-index.html +1 -1
- package/dashboard/out/file-index.txt +1 -1
- package/dashboard/out/index.html +1 -1
- package/dashboard/out/index.txt +1 -1
- package/dashboard/out/insights.html +1 -1
- package/dashboard/out/insights.txt +1 -1
- package/dashboard/out/learning.html +1 -1
- package/dashboard/out/learning.txt +1 -1
- package/dashboard/out/overview.html +1 -1
- package/dashboard/out/overview.txt +1 -1
- package/dashboard/out/scheduler.html +1 -1
- package/dashboard/out/scheduler.txt +1 -1
- package/dashboard/out/sync.html +1 -1
- package/dashboard/out/sync.txt +1 -1
- package/dashboard/out/tokens.html +1 -1
- package/dashboard/out/tokens.txt +1 -1
- package/dashboard/out/waste.html +1 -1
- package/dashboard/out/waste.txt +1 -1
- package/dashboard/out/wiki.html +1 -1
- package/dashboard/out/wiki.txt +1 -1
- package/dist/cli.bun.js +90615 -0
- package/dist/{cli.js → cli.node.js} +2227 -758
- package/package.json +14 -4
- package/scripts/build.mjs +47 -0
- package/src/commands/bug-search.ts +2 -4
- package/src/commands/detect-waste.ts +24 -32
- package/src/commands/post-read.ts +10 -11
- package/src/commands/post-write.ts +13 -19
- package/src/commands/pre-read.ts +19 -24
- package/src/commands/scan.ts +103 -40
- package/src/commands/status.ts +45 -26
- package/src/core/bug-memory.ts +32 -34
- package/src/core/dashboard-api.ts +44 -22
- package/src/core/index-store.ts +23 -0
- package/src/core/paths.ts +7 -0
- package/src/core/scanner.ts +8 -4
- package/src/core/state-aggregator.ts +64 -7
- package/src/core/state-counters.ts +11 -31
- package/src/core/sync-merge-drivers.ts +164 -1
- package/src/core/sync.ts +9 -0
- package/src/core/token-ledger.ts +50 -4
- package/src/repositories/bug-memory-repo.ts +268 -0
- package/src/repositories/counters-repo.ts +88 -0
- package/src/repositories/file-index-repo.ts +238 -0
- package/src/repositories/token-ledger-repo.ts +412 -0
- package/src/storage/db.ts +121 -0
- package/src/storage/driver.bun.ts +99 -0
- package/src/storage/driver.node.ts +107 -0
- package/src/storage/driver.ts +76 -0
- package/src/storage/migrate-json.ts +415 -0
- package/src/storage/schema.ts +207 -0
- package/src/types/file-index.ts +9 -0
- /package/dashboard/out/_next/static/{I7QxkFr_LXY4BWGjogOs1 → 9ElzGFcXpcjLq-QSQslWY}/_buildManifest.js +0 -0
- /package/dashboard/out/_next/static/{I7QxkFr_LXY4BWGjogOs1 → 9ElzGFcXpcjLq-QSQslWY}/_ssgManifest.js +0 -0
|
@@ -0,0 +1,238 @@
|
|
|
1
|
+
// File-index repository. Wraps the file_index table in `mink.db` behind a
|
|
2
|
+
// stable function-based API that the wrapper in src/core/index-store.ts
|
|
3
|
+
// delegates to. Hook hot paths (pre-read, post-write) call exactly one
|
|
4
|
+
// method per hook invocation — no full-index load.
|
|
5
|
+
//
|
|
6
|
+
// All writes attribute the calling device via device_id so the cross-device
|
|
7
|
+
// sync merge driver (mink-db-merge) can reconcile origin. Counters that
|
|
8
|
+
// were previously kept in file-index.json's header (lifetimeHits /
|
|
9
|
+
// lifetimeMisses) live in the `counters` table indexed by device_id.
|
|
10
|
+
|
|
11
|
+
import type { DbDriver } from "../storage/driver";
|
|
12
|
+
import type { FileIndexEntry, StalenessReport } from "../types/file-index";
|
|
13
|
+
import { openProjectDb } from "../storage/db";
|
|
14
|
+
import { getOrCreateDeviceId } from "../core/device";
|
|
15
|
+
|
|
16
|
+
interface FileIndexRow {
|
|
17
|
+
file_path: string;
|
|
18
|
+
description: string;
|
|
19
|
+
estimated_tokens: number;
|
|
20
|
+
last_modified: string;
|
|
21
|
+
last_indexed: string;
|
|
22
|
+
mtime_ms: number;
|
|
23
|
+
content_hash: string | null;
|
|
24
|
+
size_bytes: number | null;
|
|
25
|
+
device_id: string;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
function rowToEntry(row: FileIndexRow): FileIndexEntry {
|
|
29
|
+
return {
|
|
30
|
+
filePath: row.file_path,
|
|
31
|
+
description: row.description,
|
|
32
|
+
estimatedTokens: row.estimated_tokens,
|
|
33
|
+
lastModified: row.last_modified,
|
|
34
|
+
lastIndexed: row.last_indexed,
|
|
35
|
+
};
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
// Mirror of upsertEntry's semantics under the JSON store, expressed as a
|
|
39
|
+
// single SQL upsert. Conflict resolution picks the more recent
|
|
40
|
+
// last_modified — matches the merge driver's per-row rule so a hook that
|
|
41
|
+
// runs concurrently with sync converges deterministically.
|
|
42
|
+
const UPSERT_SQL = `
|
|
43
|
+
INSERT INTO file_index
|
|
44
|
+
(file_path, description, estimated_tokens, last_modified, last_indexed,
|
|
45
|
+
mtime_ms, content_hash, size_bytes, device_id)
|
|
46
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
47
|
+
ON CONFLICT(file_path) DO UPDATE SET
|
|
48
|
+
description = excluded.description,
|
|
49
|
+
estimated_tokens = excluded.estimated_tokens,
|
|
50
|
+
last_modified = excluded.last_modified,
|
|
51
|
+
last_indexed = excluded.last_indexed,
|
|
52
|
+
mtime_ms = excluded.mtime_ms,
|
|
53
|
+
content_hash = COALESCE(excluded.content_hash, file_index.content_hash),
|
|
54
|
+
size_bytes = COALESCE(excluded.size_bytes, file_index.size_bytes),
|
|
55
|
+
device_id = excluded.device_id
|
|
56
|
+
`;
|
|
57
|
+
|
|
58
|
+
export interface UpsertOptions {
|
|
59
|
+
mtimeMs?: number;
|
|
60
|
+
contentHash?: string | null;
|
|
61
|
+
sizeBytes?: number | null;
|
|
62
|
+
deviceId?: string;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
export interface IndexLookup {
|
|
66
|
+
lookupEntry(filePath: string): FileIndexEntry | null;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
export class FileIndexRepo implements IndexLookup {
|
|
70
|
+
constructor(private readonly db: DbDriver) {}
|
|
71
|
+
|
|
72
|
+
static for(cwd: string): FileIndexRepo {
|
|
73
|
+
return new FileIndexRepo(openProjectDb(cwd));
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
upsert(entry: FileIndexEntry, opts: UpsertOptions = {}): void {
|
|
77
|
+
const deviceId = opts.deviceId ?? getOrCreateDeviceId();
|
|
78
|
+
this.db.prepare(UPSERT_SQL).run(
|
|
79
|
+
entry.filePath,
|
|
80
|
+
entry.description,
|
|
81
|
+
entry.estimatedTokens,
|
|
82
|
+
entry.lastModified,
|
|
83
|
+
entry.lastIndexed,
|
|
84
|
+
opts.mtimeMs ?? 0,
|
|
85
|
+
opts.contentHash ?? null,
|
|
86
|
+
opts.sizeBytes ?? null,
|
|
87
|
+
deviceId
|
|
88
|
+
);
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
// Bulk upsert — used by `mink scan` to push hundreds-to-thousands of
|
|
92
|
+
// entries in a single transaction. ~50x faster than individual upserts
|
|
93
|
+
// because SQLite skips per-row WAL fsync.
|
|
94
|
+
upsertMany(entries: Array<{ entry: FileIndexEntry; opts?: UpsertOptions }>): void {
|
|
95
|
+
if (entries.length === 0) return;
|
|
96
|
+
const defaultDevice = getOrCreateDeviceId();
|
|
97
|
+
const stmt = this.db.prepare(UPSERT_SQL);
|
|
98
|
+
this.db.transaction(() => {
|
|
99
|
+
for (const { entry, opts } of entries) {
|
|
100
|
+
stmt.run(
|
|
101
|
+
entry.filePath,
|
|
102
|
+
entry.description,
|
|
103
|
+
entry.estimatedTokens,
|
|
104
|
+
entry.lastModified,
|
|
105
|
+
entry.lastIndexed,
|
|
106
|
+
opts?.mtimeMs ?? 0,
|
|
107
|
+
opts?.contentHash ?? null,
|
|
108
|
+
opts?.sizeBytes ?? null,
|
|
109
|
+
opts?.deviceId ?? defaultDevice
|
|
110
|
+
);
|
|
111
|
+
}
|
|
112
|
+
});
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
lookupEntry(filePath: string): FileIndexEntry | null {
|
|
116
|
+
const row = this.db
|
|
117
|
+
.prepare(
|
|
118
|
+
"SELECT file_path, description, estimated_tokens, last_modified, last_indexed, mtime_ms, content_hash, size_bytes, device_id FROM file_index WHERE file_path = ?"
|
|
119
|
+
)
|
|
120
|
+
.get(filePath);
|
|
121
|
+
if (!row) return null;
|
|
122
|
+
return rowToEntry(row as unknown as FileIndexRow);
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
remove(filePath: string): void {
|
|
126
|
+
this.db.prepare("DELETE FROM file_index WHERE file_path = ?").run(filePath);
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
// Remove every entry that's NOT in `keep`. Used by `mink scan` to prune
|
|
130
|
+
// orphaned entries for files that have been deleted from disk. Expressed
|
|
131
|
+
// as a single statement using a temp table to avoid the SQLite parameter
|
|
132
|
+
// limit (defaults to 999).
|
|
133
|
+
retainOnly(keep: Iterable<string>): number {
|
|
134
|
+
const keepArr = [...keep];
|
|
135
|
+
this.db.transaction(() => {
|
|
136
|
+
this.db.exec("CREATE TEMP TABLE IF NOT EXISTS _retain (path TEXT PRIMARY KEY)");
|
|
137
|
+
this.db.exec("DELETE FROM _retain");
|
|
138
|
+
const stmt = this.db.prepare("INSERT OR IGNORE INTO _retain VALUES (?)");
|
|
139
|
+
for (const p of keepArr) stmt.run(p);
|
|
140
|
+
});
|
|
141
|
+
const r = this.db
|
|
142
|
+
.prepare("DELETE FROM file_index WHERE file_path NOT IN (SELECT path FROM _retain)")
|
|
143
|
+
.run();
|
|
144
|
+
return Number(r.changes);
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
// Total count of indexed files. Cheap — backed by the PRIMARY KEY index.
|
|
148
|
+
totalFiles(): number {
|
|
149
|
+
const row = this.db.prepare("SELECT COUNT(*) AS n FROM file_index").get();
|
|
150
|
+
return Number((row as { n: number }).n);
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
// Bulk list — used by analytics, dashboard, and `mink status`. Stays
|
|
154
|
+
// off the hook hot path. Returns rows already shaped to the public
|
|
155
|
+
// FileIndexEntry type.
|
|
156
|
+
listAll(): FileIndexEntry[] {
|
|
157
|
+
const rows = this.db
|
|
158
|
+
.prepare(
|
|
159
|
+
"SELECT file_path, description, estimated_tokens, last_modified, last_indexed, mtime_ms, content_hash, size_bytes, device_id FROM file_index ORDER BY file_path"
|
|
160
|
+
)
|
|
161
|
+
.all();
|
|
162
|
+
return rows.map((r) => rowToEntry(r as unknown as FileIndexRow));
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
// For Phase 5's incremental scan. Returns the subset of `scanned` whose
|
|
166
|
+
// mtime differs from what we have stored — i.e. needs re-extraction.
|
|
167
|
+
// Done as one query per chunk to avoid a 20k-row IN list, but still much
|
|
168
|
+
// cheaper than reading every file's content.
|
|
169
|
+
staleSet(scanned: Array<{ relativePath: string; mtimeMs: number }>): string[] {
|
|
170
|
+
if (scanned.length === 0) return [];
|
|
171
|
+
const stmt = this.db.prepare(
|
|
172
|
+
"SELECT mtime_ms FROM file_index WHERE file_path = ?"
|
|
173
|
+
);
|
|
174
|
+
const stale: string[] = [];
|
|
175
|
+
for (const f of scanned) {
|
|
176
|
+
const row = stmt.get(f.relativePath);
|
|
177
|
+
if (!row) {
|
|
178
|
+
stale.push(f.relativePath); // never seen before
|
|
179
|
+
continue;
|
|
180
|
+
}
|
|
181
|
+
const storedMs = Number((row as { mtime_ms: number }).mtime_ms);
|
|
182
|
+
if (storedMs !== Math.floor(f.mtimeMs)) {
|
|
183
|
+
stale.push(f.relativePath);
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
return stale;
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
// Cheap content_hash lookup — used by `mink scan` to detect
|
|
190
|
+
// touch-without-edit cases (mtime changed, content didn't) and skip
|
|
191
|
+
// re-extraction in that case.
|
|
192
|
+
contentHashFor(filePath: string): string | null {
|
|
193
|
+
const row = this.db
|
|
194
|
+
.prepare("SELECT content_hash FROM file_index WHERE file_path = ?")
|
|
195
|
+
.get(filePath);
|
|
196
|
+
if (!row) return null;
|
|
197
|
+
const hash = (row as { content_hash: string | null }).content_hash;
|
|
198
|
+
return hash ?? null;
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
// Mirrors checkStaleness() under the JSON store: which files are on disk
|
|
202
|
+
// but not in the index (missing), and which are in the index but absent
|
|
203
|
+
// from disk (orphaned).
|
|
204
|
+
checkStaleness(scannedRelativePaths: string[]): StalenessReport {
|
|
205
|
+
const scannedSet = new Set(scannedRelativePaths);
|
|
206
|
+
const allIndexed = this.db
|
|
207
|
+
.prepare("SELECT file_path FROM file_index")
|
|
208
|
+
.all()
|
|
209
|
+
.map((r) => (r as { file_path: string }).file_path);
|
|
210
|
+
const indexedSet = new Set(allIndexed);
|
|
211
|
+
const missingFromIndex = scannedRelativePaths.filter((p) => !indexedSet.has(p));
|
|
212
|
+
const orphanedEntries = allIndexed.filter((p) => !scannedSet.has(p));
|
|
213
|
+
return {
|
|
214
|
+
missingFromIndex,
|
|
215
|
+
orphanedEntries,
|
|
216
|
+
isStale: missingFromIndex.length > 0 || orphanedEntries.length > 0,
|
|
217
|
+
};
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
// Header analogues. lastScanTimestamp is the only header field that's
|
|
221
|
+
// genuinely a project-wide state value; hit/miss counters live in the
|
|
222
|
+
// counters table and are per-device. Stored in the meta table.
|
|
223
|
+
setLastScanTimestamp(iso: string): void {
|
|
224
|
+
this.db
|
|
225
|
+
.prepare(
|
|
226
|
+
"INSERT INTO meta (key, value) VALUES ('last_scan_timestamp', ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value"
|
|
227
|
+
)
|
|
228
|
+
.run(iso);
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
getLastScanTimestamp(): string {
|
|
232
|
+
const row = this.db
|
|
233
|
+
.prepare("SELECT value FROM meta WHERE key = 'last_scan_timestamp'")
|
|
234
|
+
.get();
|
|
235
|
+
if (!row) return "";
|
|
236
|
+
return String((row as { value: string }).value);
|
|
237
|
+
}
|
|
238
|
+
}
|
|
@@ -0,0 +1,412 @@
|
|
|
1
|
+
// Token-ledger repository. Sessions + reads + writes + per-device lifetime
|
|
2
|
+
// counters all live in mink.db. The legacy archive file is subsumed by the
|
|
3
|
+
// `archived` column on ledger_sessions — "archive" becomes
|
|
4
|
+
// `UPDATE ledger_sessions SET archived = 1 WHERE ...`.
|
|
5
|
+
//
|
|
6
|
+
// Sessions are insert-only from the perspective of the merge driver
|
|
7
|
+
// (first-writer-wins keyed on session_id). The only post-insert mutations
|
|
8
|
+
// allowed are:
|
|
9
|
+
// - updateSession(): replaces totals/lists for the latest session before
|
|
10
|
+
// it's been seen by any other device — the in-process device knows
|
|
11
|
+
// its own session_id is exclusive until session_stop persists it.
|
|
12
|
+
// - archive(): flips `archived = 1` once the active-session count
|
|
13
|
+
// exceeds the retention threshold.
|
|
14
|
+
//
|
|
15
|
+
// Lifetime counters are also per-device; the merge driver MAX-merges them
|
|
16
|
+
// across shards so concurrent activity on different devices keeps the
|
|
17
|
+
// project-wide total monotonic.
|
|
18
|
+
|
|
19
|
+
import type { DbDriver } from "../storage/driver";
|
|
20
|
+
import type {
|
|
21
|
+
TokenLedger,
|
|
22
|
+
LedgerSession,
|
|
23
|
+
LifetimeCounters,
|
|
24
|
+
} from "../types/token-ledger";
|
|
25
|
+
import type { SessionSummary } from "../types/session";
|
|
26
|
+
import type { WasteFlag, WastePattern } from "../types/waste-detection";
|
|
27
|
+
import { openProjectDb } from "../storage/db";
|
|
28
|
+
import { getOrCreateDeviceId } from "../core/device";
|
|
29
|
+
|
|
30
|
+
function emptyLifetime(): LifetimeCounters {
|
|
31
|
+
return {
|
|
32
|
+
totalTokens: 0,
|
|
33
|
+
totalReads: 0,
|
|
34
|
+
totalWrites: 0,
|
|
35
|
+
totalSessions: 0,
|
|
36
|
+
totalFileIndexHits: 0,
|
|
37
|
+
totalFileIndexMisses: 0,
|
|
38
|
+
totalRepeatedReads: 0,
|
|
39
|
+
totalEstimatedSavings: 0,
|
|
40
|
+
};
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
export class TokenLedgerRepo {
|
|
44
|
+
constructor(private readonly db: DbDriver) {}
|
|
45
|
+
|
|
46
|
+
static for(cwd: string): TokenLedgerRepo {
|
|
47
|
+
return new TokenLedgerRepo(openProjectDb(cwd));
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
// ── Append / update sessions ──────────────────────────────────────────
|
|
51
|
+
|
|
52
|
+
// Insert a session as the active (archived = 0) ledger entry, append its
|
|
53
|
+
// reads/writes, and bump this device's lifetime counters. Wrapped in a
|
|
54
|
+
// transaction so a partial write never leaves the lifetime out of sync
|
|
55
|
+
// with the per-session rows.
|
|
56
|
+
appendSession(summary: SessionSummary, deviceId: string = getOrCreateDeviceId()): void {
|
|
57
|
+
this.db.transaction(() => {
|
|
58
|
+
this.insertSessionRow(summary, deviceId, 0);
|
|
59
|
+
this.appendChildRows(summary);
|
|
60
|
+
this.addToLifetime(deviceId, summary);
|
|
61
|
+
});
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
// Replace a previously-inserted active session. Used when a session is
|
|
65
|
+
// stopped multiple times — only the last stop's totals are authoritative
|
|
66
|
+
// for this device, so we subtract the old contribution and add the new.
|
|
67
|
+
// We don't allow updating an archived session (the merge driver assumes
|
|
68
|
+
// archived rows are immutable).
|
|
69
|
+
updateSession(summary: SessionSummary, deviceId: string = getOrCreateDeviceId()): void {
|
|
70
|
+
this.db.transaction(() => {
|
|
71
|
+
const existing = this.fetchSession(summary.sessionId);
|
|
72
|
+
if (!existing) {
|
|
73
|
+
this.insertSessionRow(summary, deviceId, 0);
|
|
74
|
+
this.appendChildRows(summary);
|
|
75
|
+
this.addToLifetime(deviceId, summary);
|
|
76
|
+
return;
|
|
77
|
+
}
|
|
78
|
+
this.subtractFromLifetime(existing.device_id, existing);
|
|
79
|
+
this.db.prepare(
|
|
80
|
+
"DELETE FROM ledger_reads WHERE session_id = ?"
|
|
81
|
+
).run(summary.sessionId);
|
|
82
|
+
this.db.prepare(
|
|
83
|
+
"DELETE FROM ledger_writes WHERE session_id = ?"
|
|
84
|
+
).run(summary.sessionId);
|
|
85
|
+
this.db.prepare(`
|
|
86
|
+
UPDATE ledger_sessions SET
|
|
87
|
+
start_timestamp = ?,
|
|
88
|
+
end_timestamp = ?,
|
|
89
|
+
read_count = ?,
|
|
90
|
+
write_count = ?,
|
|
91
|
+
estimated_tokens = ?,
|
|
92
|
+
repeated_reads = ?,
|
|
93
|
+
file_index_hits = ?,
|
|
94
|
+
file_index_misses = ?,
|
|
95
|
+
estimated_savings = ?
|
|
96
|
+
WHERE session_id = ?
|
|
97
|
+
`).run(
|
|
98
|
+
summary.startTimestamp, summary.endTimestamp,
|
|
99
|
+
summary.totals.readCount, summary.totals.writeCount,
|
|
100
|
+
summary.totals.estimatedTokens, summary.totals.repeatedReads,
|
|
101
|
+
summary.totals.fileIndexHits, summary.totals.fileIndexMisses,
|
|
102
|
+
summary.estimatedSavings, summary.sessionId
|
|
103
|
+
);
|
|
104
|
+
this.appendChildRows(summary);
|
|
105
|
+
this.addToLifetime(existing.device_id, summary);
|
|
106
|
+
});
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
// Archive everything past the retention threshold. Returns the number of
|
|
110
|
+
// sessions newly archived. We sort by start_timestamp ASC and flip the
|
|
111
|
+
// oldest ones so the most recent N stay active — same intent as the v1
|
|
112
|
+
// JSON archive flow.
|
|
113
|
+
archive(threshold: number = 1000): number {
|
|
114
|
+
if (threshold <= 0) return 0;
|
|
115
|
+
const active = Number(
|
|
116
|
+
(this.db.prepare(
|
|
117
|
+
"SELECT COUNT(*) AS n FROM ledger_sessions WHERE archived = 0"
|
|
118
|
+
).get() as { n: number }).n
|
|
119
|
+
);
|
|
120
|
+
if (active <= threshold) return 0;
|
|
121
|
+
const excess = active - threshold;
|
|
122
|
+
const r = this.db.prepare(`
|
|
123
|
+
UPDATE ledger_sessions SET archived = 1
|
|
124
|
+
WHERE session_id IN (
|
|
125
|
+
SELECT session_id FROM ledger_sessions
|
|
126
|
+
WHERE archived = 0
|
|
127
|
+
ORDER BY start_timestamp ASC
|
|
128
|
+
LIMIT ?
|
|
129
|
+
)
|
|
130
|
+
`).run(excess);
|
|
131
|
+
return Number(r.changes);
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
// ── Read ──────────────────────────────────────────────────────────────
|
|
135
|
+
|
|
136
|
+
lifetime(): LifetimeCounters {
|
|
137
|
+
// Sum across every device's row — gives the project-wide total.
|
|
138
|
+
const row = this.db.prepare(`
|
|
139
|
+
SELECT
|
|
140
|
+
COALESCE(SUM(total_tokens), 0) AS totalTokens,
|
|
141
|
+
COALESCE(SUM(total_reads), 0) AS totalReads,
|
|
142
|
+
COALESCE(SUM(total_writes), 0) AS totalWrites,
|
|
143
|
+
COALESCE(SUM(total_sessions), 0) AS totalSessions,
|
|
144
|
+
COALESCE(SUM(total_file_index_hits), 0) AS totalFileIndexHits,
|
|
145
|
+
COALESCE(SUM(total_file_index_misses), 0) AS totalFileIndexMisses,
|
|
146
|
+
COALESCE(SUM(total_repeated_reads), 0) AS totalRepeatedReads,
|
|
147
|
+
COALESCE(SUM(total_estimated_savings), 0) AS totalEstimatedSavings
|
|
148
|
+
FROM ledger_lifetime
|
|
149
|
+
`).get();
|
|
150
|
+
if (!row) return emptyLifetime();
|
|
151
|
+
const r = row as Record<string, number>;
|
|
152
|
+
return {
|
|
153
|
+
totalTokens: Number(r.totalTokens),
|
|
154
|
+
totalReads: Number(r.totalReads),
|
|
155
|
+
totalWrites: Number(r.totalWrites),
|
|
156
|
+
totalSessions: Number(r.totalSessions),
|
|
157
|
+
totalFileIndexHits: Number(r.totalFileIndexHits),
|
|
158
|
+
totalFileIndexMisses: Number(r.totalFileIndexMisses),
|
|
159
|
+
totalRepeatedReads: Number(r.totalRepeatedReads),
|
|
160
|
+
totalEstimatedSavings: Number(r.totalEstimatedSavings),
|
|
161
|
+
};
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
// All active (archived = 0) sessions, hydrated with their reads + writes.
|
|
165
|
+
// Sorted by start_timestamp to match the JSON aggregator's ordering.
|
|
166
|
+
activeSessions(): LedgerSession[] {
|
|
167
|
+
return this.hydrateSessions(
|
|
168
|
+
"SELECT * FROM ledger_sessions WHERE archived = 0 ORDER BY start_timestamp"
|
|
169
|
+
);
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
archivedSessions(): LedgerSession[] {
|
|
173
|
+
return this.hydrateSessions(
|
|
174
|
+
"SELECT * FROM ledger_sessions WHERE archived = 1 ORDER BY start_timestamp"
|
|
175
|
+
);
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
// Project-wide snapshot in the legacy TokenLedger shape — used by the
|
|
179
|
+
// dashboard, status, and detect-waste. wasteFlags are pulled from
|
|
180
|
+
// waste_flags and deduped by (pattern, detected_at).
|
|
181
|
+
snapshot(): TokenLedger {
|
|
182
|
+
const ledger: TokenLedger = {
|
|
183
|
+
lifetime: this.lifetime(),
|
|
184
|
+
sessions: this.activeSessions(),
|
|
185
|
+
};
|
|
186
|
+
const flagRows = this.db
|
|
187
|
+
.prepare(
|
|
188
|
+
"SELECT pattern, detected_at, details FROM waste_flags ORDER BY detected_at"
|
|
189
|
+
)
|
|
190
|
+
.all();
|
|
191
|
+
if (flagRows.length > 0) {
|
|
192
|
+
ledger.wasteFlags = flagRows.map((r) => {
|
|
193
|
+
const row = r as { pattern: string; detected_at: string; details: string | null };
|
|
194
|
+
const flag: WasteFlag = {
|
|
195
|
+
pattern: row.pattern as WastePattern,
|
|
196
|
+
detectedAt: row.detected_at,
|
|
197
|
+
description: "",
|
|
198
|
+
estimatedTokensWasted: 0,
|
|
199
|
+
suggestion: "",
|
|
200
|
+
};
|
|
201
|
+
if (row.details) {
|
|
202
|
+
try {
|
|
203
|
+
const parsed = JSON.parse(row.details) as Partial<WasteFlag>;
|
|
204
|
+
Object.assign(flag, parsed);
|
|
205
|
+
} catch {
|
|
206
|
+
// ignore bad JSON — keep base flag
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
return flag;
|
|
210
|
+
});
|
|
211
|
+
}
|
|
212
|
+
return ledger;
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
// Replace all waste_flags rows for THIS device with the provided set.
|
|
216
|
+
// detect-waste re-runs and overwrites every cycle, so we don't try to
|
|
217
|
+
// merge with previous flags.
|
|
218
|
+
replaceWasteFlagsForDevice(
|
|
219
|
+
deviceId: string,
|
|
220
|
+
flags: NonNullable<TokenLedger["wasteFlags"]>
|
|
221
|
+
): void {
|
|
222
|
+
this.db.transaction(() => {
|
|
223
|
+
this.db.prepare(
|
|
224
|
+
"DELETE FROM waste_flags WHERE device_id = ?"
|
|
225
|
+
).run(deviceId);
|
|
226
|
+
const stmt = this.db.prepare(
|
|
227
|
+
"INSERT OR REPLACE INTO waste_flags (pattern, detected_at, details, device_id) VALUES (?, ?, ?, ?)"
|
|
228
|
+
);
|
|
229
|
+
for (const flag of flags) {
|
|
230
|
+
const { pattern, detectedAt, ...rest } = flag;
|
|
231
|
+
stmt.run(pattern, detectedAt, JSON.stringify(rest), deviceId);
|
|
232
|
+
}
|
|
233
|
+
});
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
// ── Helpers ───────────────────────────────────────────────────────────
|
|
237
|
+
|
|
238
|
+
private insertSessionRow(
|
|
239
|
+
summary: SessionSummary,
|
|
240
|
+
deviceId: string,
|
|
241
|
+
archived: 0 | 1
|
|
242
|
+
): void {
|
|
243
|
+
this.db.prepare(`
|
|
244
|
+
INSERT OR REPLACE INTO ledger_sessions
|
|
245
|
+
(session_id, device_id, start_timestamp, end_timestamp,
|
|
246
|
+
read_count, write_count, estimated_tokens, repeated_reads,
|
|
247
|
+
file_index_hits, file_index_misses, estimated_savings, archived)
|
|
248
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
249
|
+
`).run(
|
|
250
|
+
summary.sessionId, deviceId,
|
|
251
|
+
summary.startTimestamp, summary.endTimestamp,
|
|
252
|
+
summary.totals.readCount, summary.totals.writeCount,
|
|
253
|
+
summary.totals.estimatedTokens, summary.totals.repeatedReads,
|
|
254
|
+
summary.totals.fileIndexHits, summary.totals.fileIndexMisses,
|
|
255
|
+
summary.estimatedSavings, archived
|
|
256
|
+
);
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
private appendChildRows(summary: SessionSummary): void {
|
|
260
|
+
const insertRead = this.db.prepare(
|
|
261
|
+
"INSERT INTO ledger_reads (session_id, file_path, estimated_tokens, read_count) VALUES (?, ?, ?, ?)"
|
|
262
|
+
);
|
|
263
|
+
for (const r of summary.reads ?? []) {
|
|
264
|
+
insertRead.run(summary.sessionId, r.filePath, r.estimatedTokens, r.readCount);
|
|
265
|
+
}
|
|
266
|
+
const insertWrite = this.db.prepare(
|
|
267
|
+
"INSERT INTO ledger_writes (session_id, file_path, estimated_tokens, action) VALUES (?, ?, ?, ?)"
|
|
268
|
+
);
|
|
269
|
+
for (const w of summary.writes ?? []) {
|
|
270
|
+
insertWrite.run(summary.sessionId, w.filePath, w.estimatedTokens, w.action);
|
|
271
|
+
}
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
private addToLifetime(deviceId: string, summary: SessionSummary): void {
|
|
275
|
+
this.adjustLifetime(deviceId, summary, +1);
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
private subtractFromLifetime(
|
|
279
|
+
deviceId: string,
|
|
280
|
+
existing: { estimated_tokens: number; read_count: number; write_count: number; file_index_hits: number; file_index_misses: number; repeated_reads: number; estimated_savings: number }
|
|
281
|
+
): void {
|
|
282
|
+
// Reconstruct a SessionSummary-shaped delta from the stored row.
|
|
283
|
+
const synthetic: SessionSummary = {
|
|
284
|
+
sessionId: "",
|
|
285
|
+
startTimestamp: "",
|
|
286
|
+
endTimestamp: "",
|
|
287
|
+
reads: [],
|
|
288
|
+
writes: [],
|
|
289
|
+
totals: {
|
|
290
|
+
readCount: existing.read_count,
|
|
291
|
+
writeCount: existing.write_count,
|
|
292
|
+
estimatedTokens: existing.estimated_tokens,
|
|
293
|
+
repeatedReads: existing.repeated_reads,
|
|
294
|
+
fileIndexHits: existing.file_index_hits,
|
|
295
|
+
fileIndexMisses: existing.file_index_misses,
|
|
296
|
+
},
|
|
297
|
+
estimatedSavings: existing.estimated_savings,
|
|
298
|
+
};
|
|
299
|
+
this.adjustLifetime(deviceId, synthetic, -1);
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
private adjustLifetime(
|
|
303
|
+
deviceId: string,
|
|
304
|
+
summary: SessionSummary,
|
|
305
|
+
sign: 1 | -1
|
|
306
|
+
): void {
|
|
307
|
+
const s = sign;
|
|
308
|
+
this.db.prepare(`
|
|
309
|
+
INSERT INTO ledger_lifetime
|
|
310
|
+
(device_id, total_tokens, total_reads, total_writes, total_sessions,
|
|
311
|
+
total_file_index_hits, total_file_index_misses, total_repeated_reads,
|
|
312
|
+
total_estimated_savings)
|
|
313
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
314
|
+
ON CONFLICT(device_id) DO UPDATE SET
|
|
315
|
+
total_tokens = ledger_lifetime.total_tokens + excluded.total_tokens,
|
|
316
|
+
total_reads = ledger_lifetime.total_reads + excluded.total_reads,
|
|
317
|
+
total_writes = ledger_lifetime.total_writes + excluded.total_writes,
|
|
318
|
+
total_sessions = ledger_lifetime.total_sessions + excluded.total_sessions,
|
|
319
|
+
total_file_index_hits = ledger_lifetime.total_file_index_hits + excluded.total_file_index_hits,
|
|
320
|
+
total_file_index_misses = ledger_lifetime.total_file_index_misses + excluded.total_file_index_misses,
|
|
321
|
+
total_repeated_reads = ledger_lifetime.total_repeated_reads + excluded.total_repeated_reads,
|
|
322
|
+
total_estimated_savings = ledger_lifetime.total_estimated_savings + excluded.total_estimated_savings
|
|
323
|
+
`).run(
|
|
324
|
+
deviceId,
|
|
325
|
+
s * summary.totals.estimatedTokens,
|
|
326
|
+
s * summary.totals.readCount,
|
|
327
|
+
s * summary.totals.writeCount,
|
|
328
|
+
s * 1,
|
|
329
|
+
s * summary.totals.fileIndexHits,
|
|
330
|
+
s * summary.totals.fileIndexMisses,
|
|
331
|
+
s * summary.totals.repeatedReads,
|
|
332
|
+
s * summary.estimatedSavings
|
|
333
|
+
);
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
private fetchSession(sessionId: string): {
|
|
337
|
+
session_id: string;
|
|
338
|
+
device_id: string;
|
|
339
|
+
estimated_tokens: number;
|
|
340
|
+
read_count: number;
|
|
341
|
+
write_count: number;
|
|
342
|
+
file_index_hits: number;
|
|
343
|
+
file_index_misses: number;
|
|
344
|
+
repeated_reads: number;
|
|
345
|
+
estimated_savings: number;
|
|
346
|
+
} | null {
|
|
347
|
+
const row = this.db
|
|
348
|
+
.prepare("SELECT * FROM ledger_sessions WHERE session_id = ?")
|
|
349
|
+
.get(sessionId);
|
|
350
|
+
if (!row) return null;
|
|
351
|
+
return row as never;
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
private hydrateSessions(sql: string): LedgerSession[] {
|
|
355
|
+
const rows = this.db.prepare(sql).all() as Array<Record<string, unknown>>;
|
|
356
|
+
if (rows.length === 0) return [];
|
|
357
|
+
const ids = rows.map((r) => String(r.session_id));
|
|
358
|
+
const readsBySession = this.groupChildren(
|
|
359
|
+
"SELECT session_id, file_path, estimated_tokens, read_count FROM ledger_reads WHERE session_id IN (" +
|
|
360
|
+
ids.map(() => "?").join(",") + ")",
|
|
361
|
+
ids
|
|
362
|
+
);
|
|
363
|
+
const writesBySession = this.groupChildren(
|
|
364
|
+
"SELECT session_id, file_path, estimated_tokens, action FROM ledger_writes WHERE session_id IN (" +
|
|
365
|
+
ids.map(() => "?").join(",") + ")",
|
|
366
|
+
ids
|
|
367
|
+
);
|
|
368
|
+
return rows.map((r) => {
|
|
369
|
+
const sid = String(r.session_id);
|
|
370
|
+
return {
|
|
371
|
+
sessionId: sid,
|
|
372
|
+
startTimestamp: String(r.start_timestamp),
|
|
373
|
+
endTimestamp: String(r.end_timestamp),
|
|
374
|
+
reads: (readsBySession.get(sid) ?? []).map((x) => ({
|
|
375
|
+
filePath: String(x.file_path),
|
|
376
|
+
estimatedTokens: Number(x.estimated_tokens),
|
|
377
|
+
readCount: Number(x.read_count),
|
|
378
|
+
})),
|
|
379
|
+
writes: (writesBySession.get(sid) ?? []).map((x) => ({
|
|
380
|
+
filePath: String(x.file_path),
|
|
381
|
+
estimatedTokens: Number(x.estimated_tokens),
|
|
382
|
+
action: x.action as "create" | "edit",
|
|
383
|
+
})),
|
|
384
|
+
totals: {
|
|
385
|
+
readCount: Number(r.read_count),
|
|
386
|
+
writeCount: Number(r.write_count),
|
|
387
|
+
estimatedTokens: Number(r.estimated_tokens),
|
|
388
|
+
repeatedReads: Number(r.repeated_reads),
|
|
389
|
+
fileIndexHits: Number(r.file_index_hits),
|
|
390
|
+
fileIndexMisses: Number(r.file_index_misses),
|
|
391
|
+
},
|
|
392
|
+
estimatedSavings: Number(r.estimated_savings),
|
|
393
|
+
};
|
|
394
|
+
});
|
|
395
|
+
}
|
|
396
|
+
|
|
397
|
+
private groupChildren(sql: string, ids: string[]): Map<string, Array<Record<string, unknown>>> {
|
|
398
|
+
const out = new Map<string, Array<Record<string, unknown>>>();
|
|
399
|
+
if (ids.length === 0) return out;
|
|
400
|
+
const rows = this.db.prepare(sql).all(...ids) as Array<Record<string, unknown>>;
|
|
401
|
+
for (const r of rows) {
|
|
402
|
+
const sid = String(r.session_id);
|
|
403
|
+
let list = out.get(sid);
|
|
404
|
+
if (!list) {
|
|
405
|
+
list = [];
|
|
406
|
+
out.set(sid, list);
|
|
407
|
+
}
|
|
408
|
+
list.push(r);
|
|
409
|
+
}
|
|
410
|
+
return out;
|
|
411
|
+
}
|
|
412
|
+
}
|