agentwrangler 0.1.0 → 0.1.1-next.2f6c1df
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +82 -94
- package/dist/apply/jobs.js +6 -2
- package/dist/daemon/index.js +9 -3
- package/dist/daemon/readiness.js +26 -1
- package/dist/daemon/router.js +4 -2
- package/dist/db/migrations/016_ingest_offset_file_version.sql +8 -0
- package/dist/db/migrations/017_ingest_metric_events.sql +22 -0
- package/dist/detector/detectors/d10_catalog_footprint.js +2 -2
- package/dist/detector/detectors/d1_ctx_always_loaded.js +9 -9
- package/dist/detector/detectors/d2_session_long_full_context.js +2 -2
- package/dist/detector/detectors/d4_model_mismatch.js +3 -3
- package/dist/detector/detectors/d6_tool_result_bloat.js +1 -1
- package/dist/detector/detectors/d7_loop_retry_waste.js +1 -1
- package/dist/detector/detectors/d8_cache_write_churn.js +2 -2
- package/dist/detector/detectors/d9_idle_background_session.js +1 -1
- package/dist/detector/measurement.js +3 -2
- package/dist/hook/precompact-checkpoint-hook.mjs +74 -25
- package/dist/ingest/ingestor.js +151 -62
- package/dist/ingest/pricing.js +15 -0
- package/dist/ingest/tail.js +55 -11
- package/dist/query/api/cost-per-success.js +13 -1
- package/dist/query/api/delivery.js +25 -3
- package/dist/query/api/effectiveness.js +30 -5
- package/dist/query/api/efficiency-headroom.js +12 -14
- package/dist/query/api/outcomes.js +2 -1
- package/dist/query/api/overview.js +90 -6
- package/dist/query/settings-store.js +2 -0
- package/dist/query/spend.js +64 -2
- package/dist/ui/assets/BarChart-BC-ciS9d.js +3 -0
- package/dist/ui/assets/BriefsPage-Dls6npt7.js +2 -0
- package/dist/ui/assets/CacheWriteSpikesChart-ncVwVoud.js +1 -0
- package/dist/ui/assets/CartesianChart-CRV6h2et.js +84 -0
- package/dist/ui/assets/Chip-EizRm13x.js +1 -0
- package/dist/ui/assets/ComposedChart-CrOxp6Ym.js +1 -0
- package/dist/ui/assets/EmptyState-9kcoimjj.js +1 -0
- package/dist/ui/assets/FlavorDecomposition-DRZxH0vQ.js +1 -0
- package/dist/ui/assets/FrictionCell-BGYB9UKh.js +2 -0
- package/dist/ui/assets/GlossaryPage-CRyQb1Hn.js +1 -0
- package/dist/ui/assets/HotSessionsPage-DfE90Kxi.js +1 -0
- package/dist/ui/assets/InfoTip-CmcYFNQt.js +1 -0
- package/dist/ui/assets/Legend-BCXsSaEM.js +5 -0
- package/dist/ui/assets/Line-2rUzs15v.js +1 -0
- package/dist/ui/assets/OverviewPage-Bm5KvXqa.js +2 -0
- package/dist/ui/assets/RecommendationsPage-CvSl3lzp.js +2 -0
- package/dist/ui/assets/Scatter-DpIZt_y5.js +1 -0
- package/dist/ui/assets/SessionDetailPage-w_lSgite.js +1 -0
- package/dist/ui/assets/SettingsPage-CfUoP8lB.js +25 -0
- package/dist/ui/assets/Skeleton-DjuxPm_D.js +1 -0
- package/dist/ui/assets/SpendPercentileChip-DYun7Wzh.js +1 -0
- package/dist/ui/assets/TrendChart-gyhHK9H_.js +1 -0
- package/dist/ui/assets/WorkspaceDetailPage-DacPKaKA.js +1 -0
- package/dist/ui/assets/WorkspacesPage-DiNQaAYG.js +1 -0
- package/dist/ui/assets/chart-theme-DnpNuyRt.js +1 -0
- package/dist/ui/assets/graphicalItemSelectors-CFhpQjLz.js +1 -0
- package/dist/ui/assets/index-D_IJsvkJ.css +1 -0
- package/dist/ui/assets/index-hbaLYHUX.js +41 -0
- package/dist/ui/assets/prompt-templates-CVzMex9L.js +13 -0
- package/dist/ui/assets/rec-sessions-ggz9MYgP.js +1 -0
- package/dist/ui/assets/useExperimentalActions-B7J8gIkb.js +1 -0
- package/dist/ui/assets/workspace-label-CysPJNc0.js +1 -0
- package/dist/ui/index.html +2 -2
- package/package.json +21 -5
- package/dist/ui/assets/index-DnRKgc21.css +0 -1
- package/dist/ui/assets/index-h1Q1wWq5.js +0 -168
|
@@ -11,6 +11,7 @@
|
|
|
11
11
|
* committed). The hook writes no transcript content to stdout/stderr.
|
|
12
12
|
*/
|
|
13
13
|
|
|
14
|
+
import { randomBytes } from "node:crypto";
|
|
14
15
|
import * as fs from "node:fs";
|
|
15
16
|
import * as os from "node:os";
|
|
16
17
|
import * as path from "node:path";
|
|
@@ -19,9 +20,11 @@ import { fileURLToPath } from "node:url";
|
|
|
19
20
|
|
|
20
21
|
const MAX_COUNT = 20;
|
|
21
22
|
const MAX_BYTES = 500 * 1024 * 1024; // 500 MiB across all snapshots
|
|
23
|
+
const MAX_COLLISION_ATTEMPTS = 16;
|
|
22
24
|
|
|
23
|
-
//
|
|
24
|
-
//
|
|
25
|
+
// A per-process counter preserves local write ordering. The random token prevents
|
|
26
|
+
// separately spawned hooks with the same session and millisecond from choosing the
|
|
27
|
+
// same name; exclusive creation below is the final no-overwrite guard.
|
|
25
28
|
let snapshotCounter = 0;
|
|
26
29
|
|
|
27
30
|
/** The directory snapshots are written to (overridable for tests). */
|
|
@@ -33,7 +36,8 @@ export function checkpointDir() {
|
|
|
33
36
|
export function snapshotName(sessionId, now) {
|
|
34
37
|
snapshotCounter = (snapshotCounter + 1) % 1_000_000;
|
|
35
38
|
const stamp = now.toISOString().replace(/[:.]/g, "-");
|
|
36
|
-
|
|
39
|
+
const counter = String(snapshotCounter).padStart(6, "0");
|
|
40
|
+
return `${encodeURIComponent(sessionId)}-${stamp}-${counter}-${randomBytes(6).toString("hex")}.jsonl`;
|
|
37
41
|
}
|
|
38
42
|
|
|
39
43
|
/**
|
|
@@ -43,14 +47,49 @@ export function snapshotName(sessionId, now) {
|
|
|
43
47
|
export function writeCheckpoint(transcriptPath, sessionId, dir, now = new Date()) {
|
|
44
48
|
if (typeof transcriptPath !== "string" || !fs.existsSync(transcriptPath)) return null;
|
|
45
49
|
fs.mkdirSync(dir, { recursive: true });
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
50
|
+
for (let attempt = 0; attempt < MAX_COLLISION_ATTEMPTS; attempt += 1) {
|
|
51
|
+
const dest = path.join(dir, snapshotName(sessionId, now));
|
|
52
|
+
try {
|
|
53
|
+
fs.copyFileSync(transcriptPath, dest, fs.constants.COPYFILE_EXCL);
|
|
54
|
+
} catch (error) {
|
|
55
|
+
if (error?.code === "EEXIST") continue;
|
|
56
|
+
throw error;
|
|
57
|
+
}
|
|
58
|
+
try {
|
|
59
|
+
fs.chmodSync(dest, 0o600);
|
|
60
|
+
} catch {
|
|
61
|
+
// chmod is a no-op boundary on Windows; the user-profile ACL is the equivalent guard.
|
|
62
|
+
}
|
|
63
|
+
return dest;
|
|
52
64
|
}
|
|
53
|
-
return
|
|
65
|
+
return null;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* Return the timestamp encoded by a snapshot filename, or null for files this hook did
|
|
70
|
+
* not create. Malformed .jsonl files are deliberately left alone by retention.
|
|
71
|
+
*/
|
|
72
|
+
function snapshotTimestamp(name) {
|
|
73
|
+
const match =
|
|
74
|
+
/-(\d{4})-(\d{2})-(\d{2})T(\d{2})-(\d{2})-(\d{2})-(\d{3})Z-\d{6}(?:-[a-f0-9]{12})?\.jsonl$/.exec(
|
|
75
|
+
name,
|
|
76
|
+
);
|
|
77
|
+
if (!match) return null;
|
|
78
|
+
const [, year, month, day, hour, minute, second, millisecond] = match;
|
|
79
|
+
const timestamp = Date.parse(
|
|
80
|
+
`${year}-${month}-${day}T${hour}:${minute}:${second}.${millisecond}Z`,
|
|
81
|
+
);
|
|
82
|
+
if (Number.isNaN(timestamp)) return null;
|
|
83
|
+
const parsed = new Date(timestamp);
|
|
84
|
+
return parsed.getUTCFullYear() === Number(year) &&
|
|
85
|
+
parsed.getUTCMonth() + 1 === Number(month) &&
|
|
86
|
+
parsed.getUTCDate() === Number(day) &&
|
|
87
|
+
parsed.getUTCHours() === Number(hour) &&
|
|
88
|
+
parsed.getUTCMinutes() === Number(minute) &&
|
|
89
|
+
parsed.getUTCSeconds() === Number(second) &&
|
|
90
|
+
parsed.getUTCMilliseconds() === Number(millisecond)
|
|
91
|
+
? timestamp
|
|
92
|
+
: null;
|
|
54
93
|
}
|
|
55
94
|
|
|
56
95
|
/** Prune oldest snapshots beyond the count cap, then beyond the total-byte cap. */
|
|
@@ -60,30 +99,40 @@ export function enforceRetention(dir, maxCount = MAX_COUNT, maxBytes = MAX_BYTES
|
|
|
60
99
|
entries = fs
|
|
61
100
|
.readdirSync(dir)
|
|
62
101
|
.filter((name) => name.endsWith(".jsonl"))
|
|
63
|
-
.sort() // timestamp-derived names sort chronologically
|
|
64
102
|
.map((name) => {
|
|
65
103
|
const full = path.join(dir, name);
|
|
66
|
-
|
|
67
|
-
|
|
104
|
+
const timestamp = snapshotTimestamp(name);
|
|
105
|
+
if (timestamp === null) return null;
|
|
106
|
+
try {
|
|
107
|
+
const stat = fs.statSync(full);
|
|
108
|
+
if (!stat.isFile()) return null;
|
|
109
|
+
return { full, name, size: stat.size, timestamp, mtimeMs: stat.mtimeMs };
|
|
110
|
+
} catch {
|
|
111
|
+
// A vanished or unreadable entry must not prevent pruning known snapshots.
|
|
112
|
+
return null;
|
|
113
|
+
}
|
|
114
|
+
})
|
|
115
|
+
.filter((entry) => entry !== null)
|
|
116
|
+
.sort(
|
|
117
|
+
(a, b) =>
|
|
118
|
+
a.timestamp - b.timestamp || a.mtimeMs - b.mtimeMs || a.name.localeCompare(b.name),
|
|
119
|
+
);
|
|
68
120
|
} catch {
|
|
69
121
|
return; // dir missing or unreadable — nothing to prune
|
|
70
122
|
}
|
|
71
123
|
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
124
|
+
let count = entries.length;
|
|
125
|
+
let total = entries.reduce((sum, entry) => sum + entry.size, 0);
|
|
126
|
+
for (const victim of entries) {
|
|
127
|
+
if (count <= maxCount && total <= maxBytes) break;
|
|
75
128
|
try {
|
|
76
129
|
fs.rmSync(victim.full);
|
|
77
|
-
} catch {
|
|
78
|
-
// A locked
|
|
130
|
+
} catch (error) {
|
|
131
|
+
// A locked snapshot still counts. Try each later candidate at most once.
|
|
132
|
+
if (error?.code !== "ENOENT") continue;
|
|
79
133
|
}
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
while (entries.length > maxCount) removeOldest();
|
|
83
|
-
let total = entries.reduce((sum, entry) => sum + entry.size, 0);
|
|
84
|
-
while (entries.length > 0 && total > maxBytes) {
|
|
85
|
-
total -= entries[0].size;
|
|
86
|
-
removeOldest();
|
|
134
|
+
count -= 1;
|
|
135
|
+
total -= victim.size;
|
|
87
136
|
}
|
|
88
137
|
}
|
|
89
138
|
|
package/dist/ingest/ingestor.js
CHANGED
|
@@ -18,7 +18,7 @@ import { Health } from "./health.js";
|
|
|
18
18
|
import { PARSER_VERSION, projectLine } from "./parser.js";
|
|
19
19
|
import { PricingSnapshotStore, seedListPrices } from "./pricing.js";
|
|
20
20
|
import { DEFAULT_RECONCILE_OPTIONS, reconcileSessions, } from "./reconcile.js";
|
|
21
|
-
import { loadOffset, saveOffset, tailFile } from "./tail.js";
|
|
21
|
+
import { fileVersion, loadOffset, sameFileVersion, saveOffset, tailFile, } from "./tail.js";
|
|
22
22
|
import { LONG_GAP_THRESHOLD_S } from "./types.js";
|
|
23
23
|
import { backfillDiscoveredCwd, defaultReadCwd, defaultReadRemote, resolveWorkspaceMappings, } from "./workspace-mapping.js";
|
|
24
24
|
export const DEFAULT_INGESTOR_OPTIONS = {
|
|
@@ -41,10 +41,9 @@ export class Ingestor {
|
|
|
41
41
|
countedResults = new Set(); // toolUseIds already summed
|
|
42
42
|
resultBytesByMsg = new Map(); // messageId → running byte sum
|
|
43
43
|
lineCursor = new Map(); // filePath → complete lines consumed
|
|
44
|
-
|
|
44
|
+
lastVersion = new Map(); // filePath → last-seen version
|
|
45
45
|
discoveryCache = createDiscoveryCache();
|
|
46
46
|
unresolvedRemotes = new Set();
|
|
47
|
-
userTurnTsBySession = new Map(); // sessionId → epoch-ms of user turns
|
|
48
47
|
// Prepared statements.
|
|
49
48
|
stInsertTurn;
|
|
50
49
|
stInsertSession;
|
|
@@ -52,7 +51,11 @@ export class Ingestor {
|
|
|
52
51
|
stBumpUserTurnCount;
|
|
53
52
|
stBumpFrictionCounts;
|
|
54
53
|
stSetGapAggregates;
|
|
54
|
+
stInsertMetricEvent;
|
|
55
|
+
stGetMetricTimestamps;
|
|
55
56
|
stGetGapN;
|
|
57
|
+
stGetMetricBaseline;
|
|
58
|
+
stSetMetricBaseline;
|
|
56
59
|
stInsertToolEvent;
|
|
57
60
|
stUpsertToolEventMetadata;
|
|
58
61
|
stSetToolResult;
|
|
@@ -93,7 +96,16 @@ export class Ingestor {
|
|
|
93
96
|
interrupt_count = interrupt_count + ?
|
|
94
97
|
WHERE session_id = ?`);
|
|
95
98
|
this.stSetGapAggregates = db.prepare("UPDATE sessions SET gap_median_s=?, gap_p90_s=?, long_gap_count=?, gap_n=? WHERE session_id=?");
|
|
96
|
-
this.
|
|
99
|
+
this.stInsertMetricEvent = db.prepare(`INSERT OR IGNORE INTO ingest_metric_events
|
|
100
|
+
(event_id, session_id, user_turn_ts, is_user_turn,
|
|
101
|
+
is_compact_summary, is_api_error, is_interrupt)
|
|
102
|
+
VALUES (?,?,?,?,?,?,?)`);
|
|
103
|
+
this.stGetMetricTimestamps = db.prepare(`SELECT user_turn_ts FROM ingest_metric_events
|
|
104
|
+
WHERE session_id = ? AND is_user_turn = 1 AND user_turn_ts IS NOT NULL`);
|
|
105
|
+
this.stGetGapN = db.prepare("SELECT gap_n FROM sessions WHERE session_id = ?");
|
|
106
|
+
this.stGetMetricBaseline = db.prepare("SELECT seeded_offset FROM ingest_metric_baselines WHERE file_path = ?");
|
|
107
|
+
this.stSetMetricBaseline = db.prepare(`INSERT INTO ingest_metric_baselines (file_path, seeded_offset) VALUES (?, ?)
|
|
108
|
+
ON CONFLICT(file_path) DO UPDATE SET seeded_offset = excluded.seeded_offset`);
|
|
97
109
|
this.stInsertToolEvent = db.prepare(`INSERT INTO tool_events
|
|
98
110
|
(event_id, session_id, ts, tool_name, input_bytes, result_bytes, input_hash, exit_class, commit_sha)
|
|
99
111
|
VALUES (?,?,?,?,?,?,?,?,?)
|
|
@@ -236,9 +248,8 @@ export class Ingestor {
|
|
|
236
248
|
this.countedResults.clear();
|
|
237
249
|
this.resultBytesByMsg.clear();
|
|
238
250
|
this.lineCursor.clear();
|
|
239
|
-
this.
|
|
251
|
+
this.lastVersion.clear();
|
|
240
252
|
this.unresolvedRemotes.clear();
|
|
241
|
-
this.userTurnTsBySession.clear();
|
|
242
253
|
}
|
|
243
254
|
// ── internals ──────────────────────────────────────────────────────────────
|
|
244
255
|
ingestAllKnown() {
|
|
@@ -278,31 +289,39 @@ export class Ingestor {
|
|
|
278
289
|
}
|
|
279
290
|
/** Tail one file from its stored offset and ingest the complete new lines. */
|
|
280
291
|
ingestFile(filePath, projectSlug) {
|
|
281
|
-
//
|
|
282
|
-
//
|
|
283
|
-
//
|
|
292
|
+
// Skip only when size, identity, and timestamps all match. This keeps the
|
|
293
|
+
// 2s tail tick from touching the DB for idle files while still noticing
|
|
294
|
+
// same-size rewrites and path replacement.
|
|
284
295
|
// every tick, which was starving the daemon's event loop (outcomes pass).
|
|
285
296
|
// Keep the per-file stat: directory mtimes only decide when to refresh paths;
|
|
286
297
|
// they cannot safely replace append/rotation change detection for each file.
|
|
287
|
-
let
|
|
298
|
+
let currentVersion;
|
|
288
299
|
try {
|
|
289
|
-
|
|
300
|
+
currentVersion = fileVersion(fs.statSync(filePath));
|
|
290
301
|
}
|
|
291
302
|
catch {
|
|
292
303
|
return; // file vanished between discovery and tail; nothing to ingest
|
|
293
304
|
}
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
305
|
+
const previousVersion = this.lastVersion.get(filePath);
|
|
306
|
+
if (previousVersion !== undefined && sameFileVersion(previousVersion, currentVersion))
|
|
307
|
+
return;
|
|
308
|
+
this.lastVersion.set(filePath, currentVersion);
|
|
297
309
|
this.health.fileSeen();
|
|
298
310
|
registerWorkspace(this.db, projectSlug);
|
|
299
311
|
const stored = loadOffset(this.db, filePath);
|
|
300
|
-
|
|
312
|
+
this.seedLegacyMetricPrefix(filePath, stored?.offset ?? null, currentVersion.size);
|
|
313
|
+
const result = tailFile(filePath, stored, currentVersion);
|
|
301
314
|
if (result.wasReset)
|
|
302
315
|
this.lineCursor.set(filePath, 0);
|
|
303
316
|
if (result.lines.length === 0) {
|
|
304
|
-
|
|
305
|
-
|
|
317
|
+
if (stored === null ||
|
|
318
|
+
stored.offset !== result.newOffset ||
|
|
319
|
+
stored.headHash !== result.newHeadHash ||
|
|
320
|
+
stored.fileVersion === null ||
|
|
321
|
+
!sameFileVersion(stored.fileVersion, currentVersion)) {
|
|
322
|
+
saveOffset(this.db, filePath, result.newOffset, result.newHeadHash, currentVersion);
|
|
323
|
+
}
|
|
324
|
+
this.stSetMetricBaseline.run(filePath, result.newOffset);
|
|
306
325
|
return;
|
|
307
326
|
}
|
|
308
327
|
const defaultSessionId = sessionStemFor(filePath);
|
|
@@ -317,9 +336,50 @@ export class Ingestor {
|
|
|
317
336
|
});
|
|
318
337
|
tx();
|
|
319
338
|
this.lineCursor.set(filePath, base + result.lines.length);
|
|
320
|
-
saveOffset(this.db, filePath, result.newOffset, result.newHeadHash);
|
|
339
|
+
saveOffset(this.db, filePath, result.newOffset, result.newHeadHash, currentVersion);
|
|
340
|
+
this.stSetMetricBaseline.run(filePath, result.newOffset);
|
|
321
341
|
this.health.fileParsed();
|
|
322
342
|
}
|
|
343
|
+
/**
|
|
344
|
+
* Upgrade bridge for offsets created before the metric-event ledger existed.
|
|
345
|
+
* Already-consumed complete lines are registered without advancing counters.
|
|
346
|
+
*/
|
|
347
|
+
seedLegacyMetricPrefix(filePath, storedOffset, currentSize) {
|
|
348
|
+
const baseline = this.stGetMetricBaseline.get(filePath);
|
|
349
|
+
if (baseline !== undefined)
|
|
350
|
+
return;
|
|
351
|
+
if (storedOffset === null || storedOffset <= 0 || storedOffset > currentSize) {
|
|
352
|
+
this.stSetMetricBaseline.run(filePath, 0);
|
|
353
|
+
return;
|
|
354
|
+
}
|
|
355
|
+
const fd = fs.openSync(filePath, "r");
|
|
356
|
+
try {
|
|
357
|
+
const buf = Buffer.alloc(storedOffset);
|
|
358
|
+
const bytesRead = fs.readSync(fd, buf, 0, storedOffset, 0);
|
|
359
|
+
let lastNl = -1;
|
|
360
|
+
for (let i = bytesRead - 1; i >= 0; i--) {
|
|
361
|
+
if (buf[i] === 0x0a) {
|
|
362
|
+
lastNl = i;
|
|
363
|
+
break;
|
|
364
|
+
}
|
|
365
|
+
}
|
|
366
|
+
const lines = lastNl < 0 ? [] : buf.subarray(0, lastNl).toString("utf8").split("\n");
|
|
367
|
+
const defaultSessionId = sessionStemFor(filePath);
|
|
368
|
+
this.db.transaction(() => {
|
|
369
|
+
for (const raw of lines) {
|
|
370
|
+
if (raw.length === 0)
|
|
371
|
+
continue;
|
|
372
|
+
const proj = projectLine(raw, { defaultSessionId });
|
|
373
|
+
if (proj.kind === "record")
|
|
374
|
+
this.recordMetricEvent(raw, proj, false);
|
|
375
|
+
}
|
|
376
|
+
this.stSetMetricBaseline.run(filePath, storedOffset);
|
|
377
|
+
})();
|
|
378
|
+
}
|
|
379
|
+
finally {
|
|
380
|
+
fs.closeSync(fd);
|
|
381
|
+
}
|
|
382
|
+
}
|
|
323
383
|
applyLine(raw, ctx, workspaceId, filePath, lineNo) {
|
|
324
384
|
const proj = projectLine(raw, ctx);
|
|
325
385
|
if (proj.kind === "quarantine") {
|
|
@@ -339,31 +399,13 @@ export class Ingestor {
|
|
|
339
399
|
if (lineSessionId !== null) {
|
|
340
400
|
this.ensureSession(lineSessionId, workspaceId, filePath, lineTs === "" ? null : lineTs);
|
|
341
401
|
}
|
|
342
|
-
if (
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
const epochMs = Date.parse(proj.ts);
|
|
346
|
-
if (Number.isFinite(epochMs)) {
|
|
347
|
-
const list = this.userTurnTsBySession.get(proj.sessionId);
|
|
348
|
-
if (list !== undefined) {
|
|
349
|
-
list.push(epochMs);
|
|
350
|
-
}
|
|
351
|
-
else {
|
|
352
|
-
this.userTurnTsBySession.set(proj.sessionId, [epochMs]);
|
|
353
|
-
}
|
|
354
|
-
this.applyGapAggregates(proj.sessionId);
|
|
355
|
-
}
|
|
356
|
-
}
|
|
357
|
-
}
|
|
358
|
-
// Friction counters (RV2a): compact summary, API error, interrupt.
|
|
359
|
-
if (proj.isCompactSummary || proj.isApiErrorMessage || proj.isInterrupt) {
|
|
360
|
-
// Ensure session row exists for lines that carry only top-level flags
|
|
361
|
-
// (no turn, command, or tool events — so lineSessionId would be null).
|
|
362
|
-
if (lineSessionId === null) {
|
|
363
|
-
this.ensureSession(proj.sessionId, workspaceId, filePath, null);
|
|
364
|
-
}
|
|
365
|
-
this.stBumpFrictionCounts.run(proj.isCompactSummary ? 1 : 0, proj.isApiErrorMessage ? 1 : 0, proj.isInterrupt ? 1 : 0, proj.sessionId);
|
|
402
|
+
if (lineSessionId === null &&
|
|
403
|
+
(proj.isCompactSummary || proj.isApiErrorMessage || proj.isInterrupt)) {
|
|
404
|
+
this.ensureSession(proj.sessionId, workspaceId, filePath, null);
|
|
366
405
|
}
|
|
406
|
+
// User/friction aggregates advance only for a structurally new source record.
|
|
407
|
+
// The persisted digest makes replay idempotent across restarts and files.
|
|
408
|
+
this.recordMetricEvent(raw, proj, true);
|
|
367
409
|
// Command markers → tool_events(local_command) for hygiene evaluation.
|
|
368
410
|
if (proj.command !== null) {
|
|
369
411
|
this.recordCommand(proj.command.sessionId, proj.command.ts, proj.command.command);
|
|
@@ -457,25 +499,83 @@ export class Ingestor {
|
|
|
457
499
|
.slice(0, 20)}`;
|
|
458
500
|
this.stInsertToolEvent.run(eventId, sessionId, ts, "local_command", null, null, command, null, null);
|
|
459
501
|
}
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
502
|
+
recordMetricEvent(raw, proj, applyAggregates) {
|
|
503
|
+
if (!proj.isUserTurn &&
|
|
504
|
+
!proj.isCompactSummary &&
|
|
505
|
+
!proj.isApiErrorMessage &&
|
|
506
|
+
!proj.isInterrupt) {
|
|
464
507
|
return;
|
|
465
508
|
}
|
|
466
|
-
const
|
|
509
|
+
const flags = [
|
|
510
|
+
proj.isUserTurn ? "u" : "",
|
|
511
|
+
proj.isCompactSummary ? "c" : "",
|
|
512
|
+
proj.isApiErrorMessage ? "a" : "",
|
|
513
|
+
proj.isInterrupt ? "i" : "",
|
|
514
|
+
].join("");
|
|
515
|
+
let sourceIdentity;
|
|
516
|
+
try {
|
|
517
|
+
const parsed = JSON.parse(raw);
|
|
518
|
+
if (typeof parsed.uuid === "string" && parsed.uuid.trim() !== "") {
|
|
519
|
+
sourceIdentity = `uuid:${parsed.uuid}`;
|
|
520
|
+
}
|
|
521
|
+
else {
|
|
522
|
+
const message = parsed.message;
|
|
523
|
+
const messageId = typeof message === "object" && message !== null
|
|
524
|
+
? message.id
|
|
525
|
+
: null;
|
|
526
|
+
if (typeof messageId === "string" && messageId.trim() !== "") {
|
|
527
|
+
sourceIdentity = `message:${String(parsed.type ?? "")}:${messageId}:${flags}`;
|
|
528
|
+
}
|
|
529
|
+
else {
|
|
530
|
+
sourceIdentity = `raw:${crypto.createHash("sha256").update(raw).digest("hex")}`;
|
|
531
|
+
}
|
|
532
|
+
}
|
|
533
|
+
}
|
|
534
|
+
catch {
|
|
535
|
+
// projectLine already parsed record lines; retain a safe fallback if that contract changes.
|
|
536
|
+
sourceIdentity = `raw:${crypto.createHash("sha256").update(raw).digest("hex")}`;
|
|
537
|
+
}
|
|
538
|
+
const eventId = crypto
|
|
539
|
+
.createHash("sha256")
|
|
540
|
+
.update(`${proj.sessionId}\0${sourceIdentity}`)
|
|
541
|
+
.digest("hex");
|
|
542
|
+
const validUserTs = proj.isUserTurn && typeof proj.ts === "string" && Number.isFinite(Date.parse(proj.ts))
|
|
543
|
+
? proj.ts
|
|
544
|
+
: null;
|
|
545
|
+
const inserted = this.stInsertMetricEvent.run(eventId, proj.sessionId, validUserTs, proj.isUserTurn ? 1 : 0, proj.isCompactSummary ? 1 : 0, proj.isApiErrorMessage ? 1 : 0, proj.isInterrupt ? 1 : 0);
|
|
546
|
+
if (!applyAggregates || inserted.changes === 0)
|
|
547
|
+
return;
|
|
548
|
+
if (proj.isUserTurn)
|
|
549
|
+
this.stBumpUserTurnCount.run(proj.sessionId);
|
|
550
|
+
if (proj.isCompactSummary || proj.isApiErrorMessage || proj.isInterrupt) {
|
|
551
|
+
this.stBumpFrictionCounts.run(proj.isCompactSummary ? 1 : 0, proj.isApiErrorMessage ? 1 : 0, proj.isInterrupt ? 1 : 0, proj.sessionId);
|
|
552
|
+
}
|
|
553
|
+
if (validUserTs !== null)
|
|
554
|
+
this.applyGapAggregates(proj.sessionId);
|
|
555
|
+
}
|
|
556
|
+
applyGapAggregates(sessionId) {
|
|
557
|
+
const rows = this.stGetMetricTimestamps.all(sessionId);
|
|
558
|
+
const sorted = rows
|
|
559
|
+
.map((row) => Date.parse(row.user_turn_ts))
|
|
560
|
+
.filter((epochMs) => Number.isFinite(epochMs))
|
|
561
|
+
.sort((a, b) => a - b);
|
|
467
562
|
const gaps = [];
|
|
468
563
|
for (let i = 1; i < sorted.length; i++) {
|
|
469
564
|
gaps.push((sorted[i] - sorted[i - 1]) / 1000);
|
|
470
565
|
}
|
|
471
566
|
const gapN = gaps.length;
|
|
567
|
+
const stored = this.stGetGapN.get(sessionId);
|
|
568
|
+
// During the 017 upgrade, a session's legacy timestamps can be spread across
|
|
569
|
+
// files that have not all established their metric-ledger baselines yet.
|
|
570
|
+
// Preserve the richer legacy aggregate until the ledger has caught up.
|
|
571
|
+
if (stored !== undefined && gapN < stored.gap_n)
|
|
572
|
+
return;
|
|
472
573
|
if (gapN === 0) {
|
|
473
|
-
this.
|
|
574
|
+
this.stSetGapAggregates.run(null, null, 0, 0, sessionId);
|
|
474
575
|
return;
|
|
475
576
|
}
|
|
476
|
-
//
|
|
477
|
-
//
|
|
478
|
-
// must not be suppressed just because it's smaller than a pre-restart gap_n.
|
|
577
|
+
// The persisted event ledger supplies the complete known history across
|
|
578
|
+
// restarts and source files, so this aggregate can be recomputed deterministically.
|
|
479
579
|
const g = [...gaps].sort((a, b) => a - b);
|
|
480
580
|
const mid = g.length;
|
|
481
581
|
const gapMedianS = mid % 2 === 1
|
|
@@ -486,17 +586,6 @@ export class Ingestor {
|
|
|
486
586
|
const longGapCount = g.filter((x) => x > LONG_GAP_THRESHOLD_S).length;
|
|
487
587
|
this.stSetGapAggregates.run(gapMedianS, gapP90S, longGapCount, gapN, sessionId);
|
|
488
588
|
}
|
|
489
|
-
// Write-guard for the "no computable gap" collapse (0 or 1 in-process user turns):
|
|
490
|
-
// the in-memory turn-timestamp map resets on every daemon restart, so this state is
|
|
491
|
-
// reached on the first post-restart user turn even when a prior cold runBackscan()
|
|
492
|
-
// already persisted a richer gap_n. Skip the null/zero write in that case so it
|
|
493
|
-
// doesn't clobber the richer aggregate.
|
|
494
|
-
clearGapAggregatesUnlessRicher(sessionId) {
|
|
495
|
-
const storedRow = this.stGetGapN.get(sessionId);
|
|
496
|
-
if ((storedRow?.gap_n ?? 0) > 0)
|
|
497
|
-
return;
|
|
498
|
-
this.stSetGapAggregates.run(null, null, 0, 0, sessionId);
|
|
499
|
-
}
|
|
500
589
|
quarantine(filePath, lineNo, errorClass) {
|
|
501
590
|
const qId = crypto
|
|
502
591
|
.createHash("sha1")
|
package/dist/ingest/pricing.js
CHANGED
|
@@ -62,6 +62,21 @@ export function modelTier(model) {
|
|
|
62
62
|
}
|
|
63
63
|
return null;
|
|
64
64
|
}
|
|
65
|
+
/**
|
|
66
|
+
* SQL predicate matching premium-priced models on a `model` column: Opus plus
|
|
67
|
+
* the Fable/Mythos families (Fable's $10/$50 base is 2× Opus). Shared by every
|
|
68
|
+
* premium-share aggregate so the family list can't drift between queries.
|
|
69
|
+
*/
|
|
70
|
+
export const PREMIUM_MODEL_SQL = "(model LIKE '%opus%' OR model LIKE '%fable%' OR model LIKE '%mythos%')";
|
|
71
|
+
/**
|
|
72
|
+
* Input list price ($/MTok) for a model, for ordering model lists
|
|
73
|
+
* most-expensive first. Derived from LIST_PRICES via modelTier so a new tier
|
|
74
|
+
* automatically sorts by its price; unpriceable models return 0 (sort last).
|
|
75
|
+
*/
|
|
76
|
+
export function modelInputListPrice(model) {
|
|
77
|
+
const tier = modelTier(model);
|
|
78
|
+
return tier === null ? 0 : (LIST_PRICES[tier]?.[0] ?? 0);
|
|
79
|
+
}
|
|
65
80
|
function addDaysIso(iso, days) {
|
|
66
81
|
const d = new Date(iso);
|
|
67
82
|
d.setUTCDate(d.getUTCDate() + days);
|
package/dist/ingest/tail.js
CHANGED
|
@@ -14,6 +14,22 @@
|
|
|
14
14
|
import * as crypto from "node:crypto";
|
|
15
15
|
import * as fs from "node:fs";
|
|
16
16
|
const HEAD_BYTES = 256; // bytes hashed for rotation detection
|
|
17
|
+
export function fileVersion(stat) {
|
|
18
|
+
return {
|
|
19
|
+
size: stat.size,
|
|
20
|
+
dev: String(stat.dev),
|
|
21
|
+
ino: String(stat.ino),
|
|
22
|
+
mtimeMs: stat.mtimeMs,
|
|
23
|
+
ctimeMs: stat.ctimeMs,
|
|
24
|
+
};
|
|
25
|
+
}
|
|
26
|
+
export function sameFileVersion(a, b) {
|
|
27
|
+
return (a.size === b.size &&
|
|
28
|
+
a.dev === b.dev &&
|
|
29
|
+
a.ino === b.ino &&
|
|
30
|
+
a.mtimeMs === b.mtimeMs &&
|
|
31
|
+
a.ctimeMs === b.ctimeMs);
|
|
32
|
+
}
|
|
17
33
|
/** SHA-256 of the first n bytes of a file (or fewer if the file is shorter). */
|
|
18
34
|
function headHashN(filePath, n) {
|
|
19
35
|
let fd;
|
|
@@ -51,12 +67,12 @@ export function headHash(filePath) {
|
|
|
51
67
|
* Tail one file from its stored offset. Returns complete lines only.
|
|
52
68
|
* See module header for the rotation/truncation contract.
|
|
53
69
|
*/
|
|
54
|
-
export function tailFile(filePath, stored) {
|
|
70
|
+
export function tailFile(filePath, stored, currentVersion) {
|
|
55
71
|
let storedOffset = stored?.offset ?? 0;
|
|
56
72
|
let storedHead = stored?.headHash ?? null;
|
|
57
|
-
let
|
|
73
|
+
let version = currentVersion;
|
|
58
74
|
try {
|
|
59
|
-
|
|
75
|
+
version ??= fileVersion(fs.statSync(filePath));
|
|
60
76
|
}
|
|
61
77
|
catch {
|
|
62
78
|
return {
|
|
@@ -67,7 +83,7 @@ export function tailFile(filePath, stored) {
|
|
|
67
83
|
wasReset: false,
|
|
68
84
|
};
|
|
69
85
|
}
|
|
70
|
-
const fileSize =
|
|
86
|
+
const fileSize = version.size;
|
|
71
87
|
let event = null;
|
|
72
88
|
let wasReset = false;
|
|
73
89
|
const curHead = fileSize > 0 ? headHash(filePath) : "";
|
|
@@ -84,6 +100,12 @@ export function tailFile(filePath, stored) {
|
|
|
84
100
|
}
|
|
85
101
|
// fileSize < storedN: size regression — headChanged stays false; handled below.
|
|
86
102
|
}
|
|
103
|
+
const storedVersion = stored?.fileVersion ?? null;
|
|
104
|
+
const identityChanged = storedVersion !== null &&
|
|
105
|
+
(storedVersion.dev !== version.dev || storedVersion.ino !== version.ino);
|
|
106
|
+
const equalSizeTimestampChanged = storedVersion !== null &&
|
|
107
|
+
storedVersion.size === version.size &&
|
|
108
|
+
(storedVersion.mtimeMs !== version.mtimeMs || storedVersion.ctimeMs !== version.ctimeMs);
|
|
87
109
|
if (storedOffset > fileSize) {
|
|
88
110
|
// Size regression always means truncation, regardless of head change.
|
|
89
111
|
event = "TRUNCATION";
|
|
@@ -91,7 +113,7 @@ export function tailFile(filePath, stored) {
|
|
|
91
113
|
wasReset = true;
|
|
92
114
|
storedHead = curHead;
|
|
93
115
|
}
|
|
94
|
-
else if (headChanged) {
|
|
116
|
+
else if (identityChanged || equalSizeTimestampChanged || headChanged) {
|
|
95
117
|
event = "ROTATION";
|
|
96
118
|
storedOffset = 0;
|
|
97
119
|
wasReset = true;
|
|
@@ -135,18 +157,40 @@ export function tailFile(filePath, stored) {
|
|
|
135
157
|
/** Read the persisted offset for a file, or null if never tailed. */
|
|
136
158
|
export function loadOffset(db, filePath) {
|
|
137
159
|
const row = db
|
|
138
|
-
.prepare("SELECT byte_offset, file_hash_head FROM ingest_offsets WHERE file_path = ?")
|
|
160
|
+
.prepare("SELECT byte_offset, file_hash_head, file_size, file_dev, file_ino, file_mtime_ms, file_ctime_ms FROM ingest_offsets WHERE file_path = ?")
|
|
139
161
|
.get(filePath);
|
|
140
162
|
if (row === undefined)
|
|
141
163
|
return null;
|
|
142
|
-
|
|
164
|
+
const hasVersion = row.file_size !== null &&
|
|
165
|
+
row.file_dev !== null &&
|
|
166
|
+
row.file_ino !== null &&
|
|
167
|
+
row.file_mtime_ms !== null &&
|
|
168
|
+
row.file_ctime_ms !== null;
|
|
169
|
+
return {
|
|
170
|
+
offset: row.byte_offset,
|
|
171
|
+
headHash: row.file_hash_head,
|
|
172
|
+
fileVersion: hasVersion
|
|
173
|
+
? {
|
|
174
|
+
size: row.file_size,
|
|
175
|
+
dev: row.file_dev,
|
|
176
|
+
ino: row.file_ino,
|
|
177
|
+
mtimeMs: row.file_mtime_ms,
|
|
178
|
+
ctimeMs: row.file_ctime_ms,
|
|
179
|
+
}
|
|
180
|
+
: null,
|
|
181
|
+
};
|
|
143
182
|
}
|
|
144
183
|
/** Upsert the persisted offset for a file. */
|
|
145
|
-
export function saveOffset(db, filePath, offset, headHash) {
|
|
146
|
-
db.prepare(`INSERT INTO ingest_offsets (file_path, byte_offset, file_hash_head, updated_at)
|
|
147
|
-
VALUES (
|
|
184
|
+
export function saveOffset(db, filePath, offset, headHash, version) {
|
|
185
|
+
db.prepare(`INSERT INTO ingest_offsets (file_path, byte_offset, file_hash_head, file_size, file_dev, file_ino, file_mtime_ms, file_ctime_ms, updated_at)
|
|
186
|
+
VALUES (?,?,?,?,?,?,?,?,?)
|
|
148
187
|
ON CONFLICT(file_path) DO UPDATE SET
|
|
149
188
|
byte_offset = excluded.byte_offset,
|
|
150
189
|
file_hash_head = excluded.file_hash_head,
|
|
151
|
-
|
|
190
|
+
file_size = excluded.file_size,
|
|
191
|
+
file_dev = excluded.file_dev,
|
|
192
|
+
file_ino = excluded.file_ino,
|
|
193
|
+
file_mtime_ms = excluded.file_mtime_ms,
|
|
194
|
+
file_ctime_ms = excluded.file_ctime_ms,
|
|
195
|
+
updated_at = excluded.updated_at`).run(filePath, offset, headHash, version.size, version.dev, version.ino, version.mtimeMs, version.ctimeMs, new Date().toISOString());
|
|
152
196
|
}
|
|
@@ -21,6 +21,7 @@ export function getCostPerSuccess(db, workspaceId, from, to) {
|
|
|
21
21
|
to,
|
|
22
22
|
workspaceId,
|
|
23
23
|
workspaceId,
|
|
24
|
+
workspaceId,
|
|
24
25
|
from,
|
|
25
26
|
to,
|
|
26
27
|
workspaceId,
|
|
@@ -46,6 +47,14 @@ export function getCostPerSuccess(db, workspaceId, from, to) {
|
|
|
46
47
|
JOIN merged_work_items mwi ON mwi.work_item_id = swl.work_item_id
|
|
47
48
|
WHERE 1 = 1${sessionFilter}
|
|
48
49
|
),
|
|
50
|
+
merged_session_links AS (
|
|
51
|
+
SELECT swl.session_id, COUNT(DISTINCT swl.work_item_id) AS merged_pr_links
|
|
52
|
+
FROM session_work_links swl
|
|
53
|
+
JOIN merged_work_items mwi ON mwi.work_item_id = swl.work_item_id
|
|
54
|
+
JOIN sessions s ON s.session_id = swl.session_id
|
|
55
|
+
WHERE 1 = 1${sessionFilter}
|
|
56
|
+
GROUP BY swl.session_id
|
|
57
|
+
),
|
|
49
58
|
in_window_sessions AS (
|
|
50
59
|
SELECT s.session_id, s.cost_equiv_u
|
|
51
60
|
FROM sessions s
|
|
@@ -64,6 +73,8 @@ export function getCostPerSuccess(db, workspaceId, from, to) {
|
|
|
64
73
|
SELECT
|
|
65
74
|
(SELECT COUNT(*) FROM merged_work_items) AS merged_pr_count,
|
|
66
75
|
(SELECT COUNT(*) FROM closed_unmerged_work_items) AS closed_unmerged_count,
|
|
76
|
+
(SELECT COUNT(*) FROM merged_session_links) AS unique_linked_session_count,
|
|
77
|
+
(SELECT COUNT(*) FROM merged_session_links WHERE merged_pr_links > 1) AS shared_linked_session_count,
|
|
67
78
|
CASE WHEN (SELECT COUNT(*) FROM merged_work_items) = 0 THEN NULL
|
|
68
79
|
ELSE (SELECT COALESCE(SUM(cost_equiv_u), 0) FROM merged_session_costs) * 1.0
|
|
69
80
|
/ (SELECT COUNT(*) FROM merged_work_items)
|
|
@@ -91,13 +102,14 @@ export function getCostPerSuccess(db, workspaceId, from, to) {
|
|
|
91
102
|
};
|
|
92
103
|
return buildResponse(data, {
|
|
93
104
|
claim_kind: "OBS_PROXY",
|
|
105
|
+
metric_definition_version: "esf-1",
|
|
94
106
|
n: data.merged_pr_count,
|
|
95
107
|
window: { from, to },
|
|
96
108
|
qualification: {
|
|
97
109
|
provisional_excluded: false,
|
|
98
110
|
unpriced_turns: 0,
|
|
99
111
|
claim_kinds_count: 1,
|
|
100
|
-
note: "Directional (OBS_PROXY):
|
|
112
|
+
note: "Directional (OBS_PROXY): Unique linked-session cost / merged PRs. The terminal-date PR cohort counts each linked session once, even when it links to multiple merged PRs; shared sessions are coverage, not per-PR allocation. Full linked-session lifecycle cost is used, while linkage_coverage_pct is a separate session-start coverage observation. Unknown pricing is not free. Survivorship bias and reviewer dependence remain.",
|
|
101
113
|
},
|
|
102
114
|
...(workspaceId === null ? {} : { drilldown_ids: { workspace_id: workspaceId } }),
|
|
103
115
|
});
|