@jitsusama/agentic-harness.core 0.1.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/cli.js +7 -0
- package/dist/bin/web.d.ts +30 -0
- package/dist/bin/web.js +97 -0
- package/dist/observability/index.d.ts +8 -0
- package/dist/observability/index.js +8 -0
- package/dist/observability/ledger/index.d.ts +14 -0
- package/dist/observability/ledger/index.js +13 -0
- package/dist/observability/ledger/store.d.ts +50 -0
- package/dist/observability/ledger/store.js +573 -0
- package/dist/observability/ledger/types.d.ts +197 -0
- package/dist/observability/ledger/types.js +1 -0
- package/dist/observability/recorder.d.ts +9 -2
- package/dist/observability/recorder.js +11 -18
- package/dist/observability/store.d.ts +5 -3
- package/dist/observability/store.js +152 -55
- package/dist/observability/types.d.ts +32 -4
- package/dist/web/audit/index.d.ts +1 -0
- package/dist/web/audit/index.js +1 -0
- package/dist/web/audit/motion.d.ts +87 -0
- package/dist/web/audit/motion.js +239 -0
- package/dist/web/design/index.d.ts +1 -0
- package/dist/web/design/index.js +1 -0
- package/dist/web/design/typography.d.ts +71 -0
- package/dist/web/design/typography.js +221 -0
- package/dist/web/hydration/capture.d.ts +37 -0
- package/dist/web/hydration/capture.js +96 -0
- package/dist/web/hydration/index.d.ts +10 -0
- package/dist/web/hydration/index.js +10 -0
- package/dist/web/hydration/judge.d.ts +56 -0
- package/dist/web/hydration/judge.js +191 -0
- package/dist/web/index.d.ts +1 -0
- package/dist/web/index.js +1 -0
- package/dist/web/perf/index.d.ts +1 -1
- package/dist/web/perf/index.js +1 -1
- package/dist/web/perf/view.js +19 -1
- package/dist/web/perf/vitals.d.ts +29 -0
- package/dist/web/perf/vitals.js +70 -0
- package/dist/web/session.d.ts +31 -2
- package/dist/web/session.js +75 -2
- package/package.json +12 -3
- package/dist/memory/db.d.ts +0 -15
- package/dist/memory/db.js +0 -25
|
@@ -0,0 +1,197 @@
|
|
|
1
|
+
import type { RunCost, RunTokens } from "../types.js";
|
|
2
|
+
/** What a turn was billed for. */
|
|
3
|
+
export type TurnKind = "assistant" | "compaction";
|
|
4
|
+
/**
|
|
5
|
+
* One billable turn read out of a session log.
|
|
6
|
+
*
|
|
7
|
+
* `cost` is null when the entry carried no usage. That is deliberately
|
|
8
|
+
* distinct from zero: a turn whose provider died before reporting cost an
|
|
9
|
+
* unknown amount, and recording it as free understates every total it
|
|
10
|
+
* appears in.
|
|
11
|
+
*/
|
|
12
|
+
export interface TurnRecord {
|
|
13
|
+
readonly entryId: string;
|
|
14
|
+
readonly sessionId: string;
|
|
15
|
+
/** ISO 8601, as the log wrote it. */
|
|
16
|
+
readonly timestamp: string;
|
|
17
|
+
readonly kind: TurnKind;
|
|
18
|
+
/** Resolved model id, or empty when the entry did not say. */
|
|
19
|
+
readonly model: string;
|
|
20
|
+
readonly tokens: RunTokens;
|
|
21
|
+
/** Null when unmetered. Never coerced to zero. */
|
|
22
|
+
readonly cost: RunCost | null;
|
|
23
|
+
/** Cache writes billed at the one-hour rate, for retention accounting. */
|
|
24
|
+
readonly cacheWrite1h: number;
|
|
25
|
+
/** Compaction only: resident tokens before the cut. */
|
|
26
|
+
readonly droppedBefore: number | null;
|
|
27
|
+
/** Compaction only: the first entry the cut kept, which bounds what it dropped. */
|
|
28
|
+
readonly firstKeptEntryId: string | null;
|
|
29
|
+
/**
|
|
30
|
+
* Content address of the turn's billable identity. Equal across forked
|
|
31
|
+
* sessions, which is what lets a corpus-wide total deduplicate the
|
|
32
|
+
* roughly ten percent of entries that forking copies verbatim.
|
|
33
|
+
*/
|
|
34
|
+
readonly digest: string;
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* One tool call, addressed by what it asked rather than what it got
|
|
38
|
+
* back, with no bytes of either kept.
|
|
39
|
+
*
|
|
40
|
+
* The digests are what make repetition visible: a call whose arguments
|
|
41
|
+
* digest to something already asked in the same session asked a
|
|
42
|
+
* question that session's context could already answer. That is waste
|
|
43
|
+
* with no value judgement in it, which is what makes it safe to act on.
|
|
44
|
+
*/
|
|
45
|
+
export interface ToolCallRecord {
|
|
46
|
+
/** Content address of the call, stable across a forked log. */
|
|
47
|
+
readonly digest: string;
|
|
48
|
+
readonly sessionId: string;
|
|
49
|
+
/** The assistant entry that made the call. */
|
|
50
|
+
readonly entryId: string;
|
|
51
|
+
/** The call's own id, which its result names. */
|
|
52
|
+
readonly callId: string;
|
|
53
|
+
readonly timestamp: string;
|
|
54
|
+
readonly name: string;
|
|
55
|
+
/** Digest of the arguments, never the arguments. */
|
|
56
|
+
readonly argsDigest: string;
|
|
57
|
+
/**
|
|
58
|
+
* What kind of verifier this call ran, if it ran one at all. Classified
|
|
59
|
+
* from the command text at scan time, before that text is digested
|
|
60
|
+
* away, so this is the one place any of it survives, and only as a
|
|
61
|
+
* category. A chained gate running more than one kind is `verify`
|
|
62
|
+
* rather than a pick of one, since one exit code cannot support the
|
|
63
|
+
* precision of naming a single kind.
|
|
64
|
+
*/
|
|
65
|
+
readonly verifierKind: VerifierKind | null;
|
|
66
|
+
/** The file the call declared, when it declared one. */
|
|
67
|
+
readonly path: string | null;
|
|
68
|
+
/** Characters the result came back with, or null if it never came. */
|
|
69
|
+
readonly resultChars: number | null;
|
|
70
|
+
/** Digest of the result text, or null if it never came. */
|
|
71
|
+
readonly resultDigest: string | null;
|
|
72
|
+
/** Whether the result came back an error. Null when it never came. */
|
|
73
|
+
readonly isError: boolean | null;
|
|
74
|
+
}
|
|
75
|
+
/**
|
|
76
|
+
* What kind of verifier a call ran. `verify` names a chained gate that
|
|
77
|
+
* ran more than one kind under a single exit code, which is a category
|
|
78
|
+
* of its own rather than a guess at which one kind mattered.
|
|
79
|
+
*/
|
|
80
|
+
export type VerifierKind = "test" | "build" | "typecheck" | "lint" | "verify";
|
|
81
|
+
/** How a kind of verifier fared across every call classified as it. */
|
|
82
|
+
export interface VerifierOutcome {
|
|
83
|
+
readonly kind: VerifierKind;
|
|
84
|
+
readonly passed: number;
|
|
85
|
+
readonly failed: number;
|
|
86
|
+
/** Calls whose result never arrived, so pass or fail is unknown. */
|
|
87
|
+
readonly unknown: number;
|
|
88
|
+
}
|
|
89
|
+
/**
|
|
90
|
+
* How real compactions compare against the payback test, replayed over
|
|
91
|
+
* turns already in the ledger rather than watched live. `evaluable`
|
|
92
|
+
* excludes a compaction with no turn after it (nothing to measure
|
|
93
|
+
* retention from) or whose model has no derivable rate yet.
|
|
94
|
+
*/
|
|
95
|
+
export interface PaybackReplay {
|
|
96
|
+
readonly compactions: number;
|
|
97
|
+
readonly evaluable: number;
|
|
98
|
+
/** The test would also have fired. */
|
|
99
|
+
readonly agreed: number;
|
|
100
|
+
/** The test would have declined what actually happened. */
|
|
101
|
+
readonly disagreed: number;
|
|
102
|
+
}
|
|
103
|
+
/**
|
|
104
|
+
* A tool call a compaction dropped from context, and where it happened.
|
|
105
|
+
*
|
|
106
|
+
* Recorded at the compaction rather than guessed at later, because the
|
|
107
|
+
* boundary a compaction drew is known precisely then and only then: the
|
|
108
|
+
* entry it names as first kept is a fact about that one event, not
|
|
109
|
+
* something a later query could reconstruct from the calls alone.
|
|
110
|
+
*/
|
|
111
|
+
export interface DroppedCallRecord {
|
|
112
|
+
/** The call that was dropped, addressing the same row in tool_calls. */
|
|
113
|
+
readonly callDigest: string;
|
|
114
|
+
readonly sessionId: string;
|
|
115
|
+
/** The compaction entry that dropped it. */
|
|
116
|
+
readonly droppedAtEntryId: string;
|
|
117
|
+
readonly droppedAtTimestamp: string;
|
|
118
|
+
}
|
|
119
|
+
/**
|
|
120
|
+
* Which calls a repeat or regret query is about, and what counts as a
|
|
121
|
+
* file changing underneath them. Both lists are the caller's, because
|
|
122
|
+
* which tools retrieve and which ones write is a fact about a harness's
|
|
123
|
+
* tool set, not about the ledger.
|
|
124
|
+
*/
|
|
125
|
+
export interface CallScope {
|
|
126
|
+
/**
|
|
127
|
+
* Tools whose result is information, so asking again means the
|
|
128
|
+
* information was needed again. Absent means every tool, which also
|
|
129
|
+
* counts re-issued actions and so overstates both measures.
|
|
130
|
+
*/
|
|
131
|
+
readonly retrieval?: readonly string[];
|
|
132
|
+
/**
|
|
133
|
+
* Tools that change the file a call declares. A read repeated after one
|
|
134
|
+
* of these touched the same path fetched something new, so it is
|
|
135
|
+
* neither a repeat nor a regret. Absent means no call is treated as a
|
|
136
|
+
* write.
|
|
137
|
+
*/
|
|
138
|
+
readonly writers?: readonly string[];
|
|
139
|
+
}
|
|
140
|
+
/**
|
|
141
|
+
* A dropped call that was asked again afterward: the earlier answer was
|
|
142
|
+
* discarded and the same question was put a second time. No claim about
|
|
143
|
+
* whether the second asking was necessary, only that it happened.
|
|
144
|
+
* Counted once per dropped call, at its first re-ask.
|
|
145
|
+
*/
|
|
146
|
+
export interface Regret {
|
|
147
|
+
readonly name: string;
|
|
148
|
+
readonly argsDigest: string;
|
|
149
|
+
readonly sessionId: string;
|
|
150
|
+
readonly droppedAtTimestamp: string;
|
|
151
|
+
readonly reAskedAtTimestamp: string;
|
|
152
|
+
readonly resultChars: number | null;
|
|
153
|
+
}
|
|
154
|
+
/**
|
|
155
|
+
* Regret with its denominator, since a count of re-asks means nothing
|
|
156
|
+
* without how many drops it could have been out of.
|
|
157
|
+
*/
|
|
158
|
+
export interface RegretReport {
|
|
159
|
+
/** Dropped calls within the scope asked about. */
|
|
160
|
+
readonly inScope: number;
|
|
161
|
+
/** Those of them asked again after the drop, earliest re-ask first. */
|
|
162
|
+
readonly reAsked: Regret[];
|
|
163
|
+
}
|
|
164
|
+
/**
|
|
165
|
+
* Arguments asked more than once inside one session, and what the
|
|
166
|
+
* repeats weighed.
|
|
167
|
+
*/
|
|
168
|
+
export interface RepeatedCall {
|
|
169
|
+
readonly argsDigest: string;
|
|
170
|
+
readonly name: string;
|
|
171
|
+
/** How many times these arguments were asked. */
|
|
172
|
+
readonly asked: number;
|
|
173
|
+
/** How many of those were repeats, so one less than asked. */
|
|
174
|
+
readonly repeated: number;
|
|
175
|
+
/** Characters the repeats re-admitted to the context. */
|
|
176
|
+
readonly repeatedChars: number;
|
|
177
|
+
}
|
|
178
|
+
/**
|
|
179
|
+
* What a session log says about itself.
|
|
180
|
+
*
|
|
181
|
+
* Every field but the id may be absent, and absent is kept rather than
|
|
182
|
+
* guessed: attributing spend to a repo or a quest the log never named
|
|
183
|
+
* would charge work that did not incur it.
|
|
184
|
+
*/
|
|
185
|
+
export interface SessionRecord {
|
|
186
|
+
readonly sessionId: string;
|
|
187
|
+
/** The working directory the log named, if it named one. */
|
|
188
|
+
readonly cwd: string | null;
|
|
189
|
+
/** The repo that directory belongs to, derived from it. */
|
|
190
|
+
readonly repo: string | null;
|
|
191
|
+
/** The quest the session was working under, if any. */
|
|
192
|
+
readonly quest: string | null;
|
|
193
|
+
/** Timestamp of the earliest billed turn. */
|
|
194
|
+
readonly firstSeen: string | null;
|
|
195
|
+
/** Timestamp of the latest billed turn. */
|
|
196
|
+
readonly lastSeen: string | null;
|
|
197
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -31,8 +31,15 @@ export interface RunRecordInput {
|
|
|
31
31
|
/**
|
|
32
32
|
* Build a {@link RunRecord} from a producer's per-subagent
|
|
33
33
|
* result. Verify outcome derives from the verification
|
|
34
|
-
* block when present
|
|
35
|
-
*
|
|
34
|
+
* block when present.
|
|
35
|
+
*
|
|
36
|
+
* A run that reported no usage (a crashed child, a round
|
|
37
|
+
* killed before billing, an older pi) still produces a record,
|
|
38
|
+
* because withholding the row would make the worst failures
|
|
39
|
+
* the ones the table has nothing to say about. But its tokens
|
|
40
|
+
* and cost are null rather than zero. The row is the claim
|
|
41
|
+
* that the run happened; a zero would be a second claim, that
|
|
42
|
+
* it was free, which nothing supports.
|
|
36
43
|
*/
|
|
37
44
|
export declare function runRecordFrom(input: RunRecordInput): RunRecord;
|
|
38
45
|
/**
|
|
@@ -1,23 +1,16 @@
|
|
|
1
1
|
import { processGlobal } from "../internal/process-global.js";
|
|
2
|
-
const ZERO_TOKENS = {
|
|
3
|
-
input: 0,
|
|
4
|
-
output: 0,
|
|
5
|
-
cacheRead: 0,
|
|
6
|
-
cacheWrite: 0,
|
|
7
|
-
total: 0,
|
|
8
|
-
};
|
|
9
|
-
const ZERO_COST = {
|
|
10
|
-
input: 0,
|
|
11
|
-
output: 0,
|
|
12
|
-
cacheRead: 0,
|
|
13
|
-
cacheWrite: 0,
|
|
14
|
-
total: 0,
|
|
15
|
-
};
|
|
16
2
|
/**
|
|
17
3
|
* Build a {@link RunRecord} from a producer's per-subagent
|
|
18
4
|
* result. Verify outcome derives from the verification
|
|
19
|
-
* block when present
|
|
20
|
-
*
|
|
5
|
+
* block when present.
|
|
6
|
+
*
|
|
7
|
+
* A run that reported no usage (a crashed child, a round
|
|
8
|
+
* killed before billing, an older pi) still produces a record,
|
|
9
|
+
* because withholding the row would make the worst failures
|
|
10
|
+
* the ones the table has nothing to say about. But its tokens
|
|
11
|
+
* and cost are null rather than zero. The row is the claim
|
|
12
|
+
* that the run happened; a zero would be a second claim, that
|
|
13
|
+
* it was free, which nothing supports.
|
|
21
14
|
*/
|
|
22
15
|
export function runRecordFrom(input) {
|
|
23
16
|
const { result } = input;
|
|
@@ -38,8 +31,8 @@ export function runRecordFrom(input) {
|
|
|
38
31
|
retriesToValid: Math.max(0, (result.verification?.attempts ?? 1) - 1),
|
|
39
32
|
warningCount: result.warnings.length,
|
|
40
33
|
exitCode: result.exitCode,
|
|
41
|
-
tokens: result.usage?.tokens ??
|
|
42
|
-
cost: result.usage?.cost ??
|
|
34
|
+
tokens: result.usage?.tokens ?? null,
|
|
35
|
+
cost: result.usage?.cost ?? null,
|
|
43
36
|
startedAt: input.startedAt,
|
|
44
37
|
};
|
|
45
38
|
}
|
|
@@ -8,9 +8,11 @@ export interface RunStore {
|
|
|
8
8
|
recordRun(record: RunRecord): Promise<void>;
|
|
9
9
|
queryRuns(filter?: RunQuery): Promise<RunRecord[]>;
|
|
10
10
|
summarizeRun(runId: string): Promise<RunSummary | null>;
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
11
|
+
/**
|
|
12
|
+
* Weekly per-model, per-persona summaries, computed from the rows
|
|
13
|
+
* rather than materialised beside them. Nothing has to run for this
|
|
14
|
+
* to be current, and no row is ever discarded to produce it.
|
|
15
|
+
*/
|
|
14
16
|
queryRollups(): Promise<RunRollup[]>;
|
|
15
17
|
close(): Promise<void>;
|
|
16
18
|
}
|
|
@@ -25,6 +25,11 @@ CREATE TABLE IF NOT EXISTS runs (
|
|
|
25
25
|
);
|
|
26
26
|
CREATE INDEX IF NOT EXISTS runs_run_id ON runs (run_id);
|
|
27
27
|
CREATE INDEX IF NOT EXISTS runs_started_at ON runs (started_at);
|
|
28
|
+
/**
|
|
29
|
+
* The rollups table is legacy and is never written to again. It holds
|
|
30
|
+
* the only surviving record of 2,199 runs whose raw rows an earlier
|
|
31
|
+
* retention pass deleted, so it is read and never dropped.
|
|
32
|
+
*/
|
|
28
33
|
CREATE TABLE IF NOT EXISTS rollups (
|
|
29
34
|
week_start INTEGER NOT NULL,
|
|
30
35
|
model TEXT NOT NULL,
|
|
@@ -48,8 +53,63 @@ export async function openRunStore(dbPath) {
|
|
|
48
53
|
const db = await openDb(dbPath);
|
|
49
54
|
await db.exec("PRAGMA journal_mode = WAL; PRAGMA busy_timeout = 5000;");
|
|
50
55
|
await db.exec(SCHEMA);
|
|
56
|
+
await migrate(db);
|
|
51
57
|
return new SqliteRunStore(db);
|
|
52
58
|
}
|
|
59
|
+
/**
|
|
60
|
+
* Bring a store forward to the current shape, in place and without
|
|
61
|
+
* losing a row.
|
|
62
|
+
*
|
|
63
|
+
* This is the only schema path: a new store is created at the original
|
|
64
|
+
* shape and migrated like any other, so every test that opens a fresh
|
|
65
|
+
* store also exercises the migration an existing file will meet.
|
|
66
|
+
* Every step is additive, a column or an index, so nothing a store
|
|
67
|
+
* already holds can be lost by opening it.
|
|
68
|
+
*/
|
|
69
|
+
async function migrate(db) {
|
|
70
|
+
const columns = new Set((await db.all("PRAGMA table_info(runs)")).map((column) => column.name));
|
|
71
|
+
if (!columns.has("metered")) {
|
|
72
|
+
await db.exec("ALTER TABLE runs ADD COLUMN metered INTEGER NOT NULL DEFAULT 1");
|
|
73
|
+
// Rows written before this column existed recorded an unmetered
|
|
74
|
+
// run as zero cost. Zero tokens identifies them: a run that did
|
|
75
|
+
// anything consumed some, and all 75 such rows in the real store
|
|
76
|
+
// had none. Only backfilled on the pass that adds the column, so
|
|
77
|
+
// a later metered run cannot be reclassified.
|
|
78
|
+
await db.exec("UPDATE runs SET metered = 0 WHERE tokens_total = 0 AND cost_total = 0");
|
|
79
|
+
}
|
|
80
|
+
for (const [name, type] of ATTRIBUTION_COLUMNS) {
|
|
81
|
+
if (!columns.has(name)) {
|
|
82
|
+
await db.exec(`ALTER TABLE runs ADD COLUMN ${name} ${type}`);
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
const indexes = await db.all("SELECT name FROM sqlite_master WHERE type = 'index' AND name = 'runs_identity'");
|
|
86
|
+
if (indexes.length === 0) {
|
|
87
|
+
// A subagent of a run is one row. A store written before that rule
|
|
88
|
+
// may hold duplicates, and the index cannot be added over them.
|
|
89
|
+
// They are captured before anything is removed, keeping the latest
|
|
90
|
+
// of each pair where it was: nothing is deleted that is not first
|
|
91
|
+
// copied somewhere it can be recovered from.
|
|
92
|
+
await db.exec(`CREATE TABLE IF NOT EXISTS runs_superseded AS SELECT * FROM runs WHERE 0;
|
|
93
|
+
INSERT INTO runs_superseded SELECT * FROM runs WHERE rowid NOT IN (
|
|
94
|
+
SELECT MAX(rowid) FROM runs GROUP BY run_id, subagent_id
|
|
95
|
+
);
|
|
96
|
+
DELETE FROM runs WHERE rowid NOT IN (
|
|
97
|
+
SELECT MAX(rowid) FROM runs GROUP BY run_id, subagent_id
|
|
98
|
+
);
|
|
99
|
+
CREATE UNIQUE INDEX runs_identity ON runs (run_id, subagent_id);`);
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
/**
|
|
103
|
+
* Where a run belonged, all nullable because a row written before these
|
|
104
|
+
* existed does not know, and a guess would charge work that did not
|
|
105
|
+
* incur the cost.
|
|
106
|
+
*/
|
|
107
|
+
const ATTRIBUTION_COLUMNS = [
|
|
108
|
+
["session_id", "TEXT"],
|
|
109
|
+
["cwd", "TEXT"],
|
|
110
|
+
["repo", "TEXT"],
|
|
111
|
+
["ended_at", "INTEGER"],
|
|
112
|
+
];
|
|
53
113
|
class SqliteRunStore {
|
|
54
114
|
db;
|
|
55
115
|
constructor(db) {
|
|
@@ -61,8 +121,27 @@ class SqliteRunStore {
|
|
|
61
121
|
retries_to_valid, warning_count, exit_code,
|
|
62
122
|
tokens_input, tokens_output, tokens_cache_read, tokens_cache_write, tokens_total,
|
|
63
123
|
cost_input, cost_output, cost_cache_read, cost_cache_write, cost_total,
|
|
64
|
-
started_at
|
|
65
|
-
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
124
|
+
started_at, metered, session_id, cwd, repo, ended_at
|
|
125
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
126
|
+
ON CONFLICT (run_id, subagent_id) DO UPDATE SET
|
|
127
|
+
kind = excluded.kind, model = excluded.model,
|
|
128
|
+
persona = excluded.persona, verify_outcome = excluded.verify_outcome,
|
|
129
|
+
retries_to_valid = excluded.retries_to_valid,
|
|
130
|
+
warning_count = excluded.warning_count,
|
|
131
|
+
exit_code = excluded.exit_code,
|
|
132
|
+
tokens_input = excluded.tokens_input,
|
|
133
|
+
tokens_output = excluded.tokens_output,
|
|
134
|
+
tokens_cache_read = excluded.tokens_cache_read,
|
|
135
|
+
tokens_cache_write = excluded.tokens_cache_write,
|
|
136
|
+
tokens_total = excluded.tokens_total,
|
|
137
|
+
cost_input = excluded.cost_input,
|
|
138
|
+
cost_output = excluded.cost_output,
|
|
139
|
+
cost_cache_read = excluded.cost_cache_read,
|
|
140
|
+
cost_cache_write = excluded.cost_cache_write,
|
|
141
|
+
cost_total = excluded.cost_total,
|
|
142
|
+
started_at = excluded.started_at, metered = excluded.metered,
|
|
143
|
+
session_id = excluded.session_id, cwd = excluded.cwd,
|
|
144
|
+
repo = excluded.repo, ended_at = excluded.ended_at`, [
|
|
66
145
|
record.runId,
|
|
67
146
|
record.subagentId,
|
|
68
147
|
record.kind,
|
|
@@ -72,17 +151,25 @@ class SqliteRunStore {
|
|
|
72
151
|
record.retriesToValid,
|
|
73
152
|
record.warningCount,
|
|
74
153
|
record.exitCode,
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
record.tokens
|
|
79
|
-
record.tokens
|
|
80
|
-
record.
|
|
81
|
-
record.
|
|
82
|
-
record.
|
|
83
|
-
record.cost
|
|
84
|
-
record.cost
|
|
154
|
+
// An unmetered run stores zeros, which leave every sum exactly
|
|
155
|
+
// as excluding it would, and the flag beside them is what says
|
|
156
|
+
// the zeros are unknown rather than free.
|
|
157
|
+
record.tokens?.input ?? 0,
|
|
158
|
+
record.tokens?.output ?? 0,
|
|
159
|
+
record.tokens?.cacheRead ?? 0,
|
|
160
|
+
record.tokens?.cacheWrite ?? 0,
|
|
161
|
+
record.tokens?.total ?? 0,
|
|
162
|
+
record.cost?.input ?? 0,
|
|
163
|
+
record.cost?.output ?? 0,
|
|
164
|
+
record.cost?.cacheRead ?? 0,
|
|
165
|
+
record.cost?.cacheWrite ?? 0,
|
|
166
|
+
record.cost?.total ?? 0,
|
|
85
167
|
record.startedAt,
|
|
168
|
+
record.cost ? 1 : 0,
|
|
169
|
+
record.sessionId ?? null,
|
|
170
|
+
record.cwd ?? null,
|
|
171
|
+
record.repo ?? null,
|
|
172
|
+
record.endedAt ?? null,
|
|
86
173
|
]);
|
|
87
174
|
}
|
|
88
175
|
async queryRuns(filter = {}) {
|
|
@@ -98,6 +185,7 @@ class SqliteRunStore {
|
|
|
98
185
|
SUM(CASE WHEN verify_outcome = 'failed' THEN 1 ELSE 0 END) AS failed,
|
|
99
186
|
SUM(retries_to_valid) AS total_retries,
|
|
100
187
|
SUM(warning_count) AS total_warnings,
|
|
188
|
+
SUM(CASE WHEN metered = 0 THEN 1 ELSE 0 END) AS unmetered,
|
|
101
189
|
SUM(tokens_input) AS tokens_input,
|
|
102
190
|
SUM(tokens_output) AS tokens_output,
|
|
103
191
|
SUM(tokens_cache_read) AS tokens_cache_read,
|
|
@@ -120,6 +208,7 @@ class SqliteRunStore {
|
|
|
120
208
|
failed: row.failed,
|
|
121
209
|
totalRetries: row.total_retries,
|
|
122
210
|
totalWarnings: row.total_warnings,
|
|
211
|
+
unmetered: row.unmetered,
|
|
123
212
|
tokens: {
|
|
124
213
|
input: row.tokens_input,
|
|
125
214
|
output: row.tokens_output,
|
|
@@ -137,36 +226,36 @@ class SqliteRunStore {
|
|
|
137
226
|
cacheReadRatio: freshInput === 0 ? 0 : row.tokens_cache_read / freshInput,
|
|
138
227
|
};
|
|
139
228
|
}
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
model, persona,
|
|
152
|
-
COUNT(*), SUM(retries_to_valid), SUM(warning_count),
|
|
153
|
-
SUM(tokens_total), SUM(cost_total),
|
|
154
|
-
SUM(tokens_cache_read), SUM(tokens_input + tokens_cache_read)
|
|
155
|
-
FROM runs WHERE started_at < ?
|
|
156
|
-
GROUP BY week_start, model, persona
|
|
157
|
-
ON CONFLICT(week_start, model, persona) DO UPDATE SET
|
|
158
|
-
run_count = run_count + excluded.run_count,
|
|
159
|
-
total_retries = total_retries + excluded.total_retries,
|
|
160
|
-
total_warnings = total_warnings + excluded.total_warnings,
|
|
161
|
-
tokens_total = tokens_total + excluded.tokens_total,
|
|
162
|
-
cost_total = cost_total + excluded.cost_total,
|
|
163
|
-
cache_read = cache_read + excluded.cache_read,
|
|
164
|
-
fresh_input = fresh_input + excluded.fresh_input`, [cutoffMs]);
|
|
165
|
-
await this.db.run("DELETE FROM runs WHERE started_at < ?", [cutoffMs]);
|
|
166
|
-
return { rolledRows };
|
|
167
|
-
}
|
|
229
|
+
/**
|
|
230
|
+
* Summaries over every row held, unioned with the legacy table.
|
|
231
|
+
*
|
|
232
|
+
* Computed rather than materialised, because the only reason to
|
|
233
|
+
* materialise was that the rows behind it were being deleted. They
|
|
234
|
+
* are not any more: the whole corpus is a rounding error on disk and
|
|
235
|
+
* discarding detail to save it was never a trade worth making.
|
|
236
|
+
*
|
|
237
|
+
* The legacy rows are unioned in rather than ignored, since for the
|
|
238
|
+
* period before this changed they are all that is left.
|
|
239
|
+
*/
|
|
168
240
|
async queryRollups() {
|
|
169
|
-
const rows = await this.db.all(
|
|
241
|
+
const rows = await this.db.all(`SELECT * FROM (
|
|
242
|
+
SELECT
|
|
243
|
+
(started_at / ${WEEK_MS}) * ${WEEK_MS} AS week_start,
|
|
244
|
+
model, persona,
|
|
245
|
+
COUNT(*) AS run_count,
|
|
246
|
+
SUM(retries_to_valid) AS total_retries,
|
|
247
|
+
SUM(warning_count) AS total_warnings,
|
|
248
|
+
SUM(tokens_total) AS tokens_total,
|
|
249
|
+
SUM(cost_total) AS cost_total,
|
|
250
|
+
SUM(tokens_cache_read) AS cache_read,
|
|
251
|
+
SUM(tokens_input + tokens_cache_read) AS fresh_input
|
|
252
|
+
FROM runs GROUP BY week_start, model, persona
|
|
253
|
+
UNION ALL
|
|
254
|
+
SELECT week_start, model, persona, run_count, total_retries,
|
|
255
|
+
total_warnings, tokens_total, cost_total, cache_read, fresh_input
|
|
256
|
+
FROM rollups
|
|
257
|
+
)
|
|
258
|
+
ORDER BY week_start ASC, model ASC, persona ASC`);
|
|
170
259
|
return rows.map((row) => ({
|
|
171
260
|
weekStart: row.week_start,
|
|
172
261
|
model: row.model,
|
|
@@ -194,20 +283,28 @@ function rowToRecord(row) {
|
|
|
194
283
|
retriesToValid: row.retries_to_valid,
|
|
195
284
|
warningCount: row.warning_count,
|
|
196
285
|
exitCode: row.exit_code,
|
|
197
|
-
tokens:
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
286
|
+
tokens: row.metered === 0
|
|
287
|
+
? null
|
|
288
|
+
: {
|
|
289
|
+
input: row.tokens_input,
|
|
290
|
+
output: row.tokens_output,
|
|
291
|
+
cacheRead: row.tokens_cache_read,
|
|
292
|
+
cacheWrite: row.tokens_cache_write,
|
|
293
|
+
total: row.tokens_total,
|
|
294
|
+
},
|
|
295
|
+
cost: row.metered === 0
|
|
296
|
+
? null
|
|
297
|
+
: {
|
|
298
|
+
input: row.cost_input,
|
|
299
|
+
output: row.cost_output,
|
|
300
|
+
cacheRead: row.cost_cache_read,
|
|
301
|
+
cacheWrite: row.cost_cache_write,
|
|
302
|
+
total: row.cost_total,
|
|
303
|
+
},
|
|
211
304
|
startedAt: row.started_at,
|
|
305
|
+
sessionId: row.session_id ?? null,
|
|
306
|
+
cwd: row.cwd ?? null,
|
|
307
|
+
repo: row.repo ?? null,
|
|
308
|
+
endedAt: row.ended_at ?? null,
|
|
212
309
|
};
|
|
213
310
|
}
|
|
@@ -43,12 +43,34 @@ export interface RunRecord {
|
|
|
43
43
|
readonly warningCount: number;
|
|
44
44
|
/** Process exit code. */
|
|
45
45
|
readonly exitCode: number;
|
|
46
|
-
/**
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
46
|
+
/**
|
|
47
|
+
* Token counts summed across the run's turns, or null when the run
|
|
48
|
+
* reported no usage at all.
|
|
49
|
+
*/
|
|
50
|
+
readonly tokens: RunTokens | null;
|
|
51
|
+
/**
|
|
52
|
+
* Cost summed across the run's turns, or null when the run reported
|
|
53
|
+
* no usage. Null is not zero: a run that died before it could report
|
|
54
|
+
* cost an unknown amount, and recording it as free makes it
|
|
55
|
+
* indistinguishable from one that genuinely cost nothing. Every one
|
|
56
|
+
* of the 75 zero-cost rows found in the real store had zero tokens
|
|
57
|
+
* too, so not one of them was actually free.
|
|
58
|
+
*/
|
|
59
|
+
readonly cost: RunCost | null;
|
|
50
60
|
/** When the run started, epoch milliseconds. */
|
|
51
61
|
readonly startedAt: number;
|
|
62
|
+
/**
|
|
63
|
+
* The parent session that dispatched the run, the directory it was
|
|
64
|
+
* working in and the repo that directory belongs to. Stamped by the
|
|
65
|
+
* sink rather than the producer, since the sink is what knows where
|
|
66
|
+
* it is. Null when not known, never guessed: without these, $12,825
|
|
67
|
+
* of fan-out could not be traced to the work that caused it.
|
|
68
|
+
*/
|
|
69
|
+
readonly sessionId?: string | null;
|
|
70
|
+
readonly cwd?: string | null;
|
|
71
|
+
readonly repo?: string | null;
|
|
72
|
+
/** When the run's record was written, epoch milliseconds. */
|
|
73
|
+
readonly endedAt?: number | null;
|
|
52
74
|
}
|
|
53
75
|
/** Aggregate view of one run across its subagents. */
|
|
54
76
|
export interface RunSummary {
|
|
@@ -58,6 +80,12 @@ export interface RunSummary {
|
|
|
58
80
|
readonly failed: number;
|
|
59
81
|
readonly totalRetries: number;
|
|
60
82
|
readonly totalWarnings: number;
|
|
83
|
+
/**
|
|
84
|
+
* Subagents that reported no usage. The totals beside this exclude
|
|
85
|
+
* them, so a total can say what it is missing rather than silently
|
|
86
|
+
* pricing the unknown at nothing.
|
|
87
|
+
*/
|
|
88
|
+
readonly unmetered: number;
|
|
61
89
|
readonly tokens: RunTokens;
|
|
62
90
|
readonly cost: RunCost;
|
|
63
91
|
/** cacheRead / (input + cacheRead); 0 when the denominator is 0. */
|
|
@@ -12,6 +12,7 @@ export { type AxFacts, buildStructure, selectorFor, } from "./capture.js";
|
|
|
12
12
|
export { composite, contrastRatio, deltaE, formatRgb, isOpaque, isTransparent, parseRgb, type Rgba, relativeLuminance, } from "./colour.js";
|
|
13
13
|
export { BOLD_WEIGHT, type ContrastLevel, type ContrastVerdict, isLargeText, judgeNonText, judgeText, LARGE_BOLD_PX, LARGE_TEXT_PX, NON_TEXT_MINIMUM, renderContrast, type TextSizing, textThreshold, undecidable, } from "./contrast.js";
|
|
14
14
|
export { overallOf, type Part, renderHealth } from "./health.js";
|
|
15
|
+
export { analyseMotion, BRIEF_MS, MOTION_CAPTURE, type MotionAnimation, type MotionCapture, type MotionVideo, PAUSE_STOP_HIDE_MS, } from "./motion.js";
|
|
15
16
|
export { foldPair, type PaintedSide, type PairReport, renderPair, } from "./pair.js";
|
|
16
17
|
export { TARGET_CAPTURE, visualCaptureSource } from "./probe.js";
|
|
17
18
|
export { MAX_LISTED_NODES, renderAudit, renderFinding, renderIndex, renderSummary, } from "./report.js";
|
package/dist/web/audit/index.js
CHANGED
|
@@ -12,6 +12,7 @@ export { buildStructure, selectorFor, } from "./capture.js";
|
|
|
12
12
|
export { composite, contrastRatio, deltaE, formatRgb, isOpaque, isTransparent, parseRgb, relativeLuminance, } from "./colour.js";
|
|
13
13
|
export { BOLD_WEIGHT, isLargeText, judgeNonText, judgeText, LARGE_BOLD_PX, LARGE_TEXT_PX, NON_TEXT_MINIMUM, renderContrast, textThreshold, undecidable, } from "./contrast.js";
|
|
14
14
|
export { overallOf, renderHealth } from "./health.js";
|
|
15
|
+
export { analyseMotion, BRIEF_MS, MOTION_CAPTURE, PAUSE_STOP_HIDE_MS, } from "./motion.js";
|
|
15
16
|
export { foldPair, renderPair, } from "./pair.js";
|
|
16
17
|
export { TARGET_CAPTURE, visualCaptureSource } from "./probe.js";
|
|
17
18
|
export { MAX_LISTED_NODES, renderAudit, renderFinding, renderIndex, renderSummary, } from "./report.js";
|