@jitsusama/agentic-harness.core 0.2.0 → 0.3.0

package/dist/observability/ledger/types.d.ts

@@ -0,0 +1,197 @@
+import type { RunCost, RunTokens } from "../types.js";
+/** What a turn was billed for. */
+export type TurnKind = "assistant" | "compaction";
+/**
+ * One billable turn read out of a session log.
+ *
+ * `cost` is null when the entry carried no usage. That is deliberately
+ * distinct from zero: a turn whose provider died before reporting cost an
+ * unknown amount, and recording it as free understates every total it
+ * appears in.
+ */
+export interface TurnRecord {
+    readonly entryId: string;
+    readonly sessionId: string;
+    /** ISO 8601, as the log wrote it. */
+    readonly timestamp: string;
+    readonly kind: TurnKind;
+    /** Resolved model id, or empty when the entry did not say. */
+    readonly model: string;
+    readonly tokens: RunTokens;
+    /** Null when unmetered. Never coerced to zero. */
+    readonly cost: RunCost | null;
+    /** Cache writes billed at the one-hour rate, for retention accounting. */
+    readonly cacheWrite1h: number;
+    /** Compaction only: resident tokens before the cut. */
+    readonly droppedBefore: number | null;
+    /** Compaction only: the first entry the cut kept, which bounds what it dropped. */
+    readonly firstKeptEntryId: string | null;
+    /**
+     * Content address of the turn's billable identity. Equal across forked
+     * sessions, which is what lets a corpus-wide total deduplicate the
+     * roughly ten percent of entries that forking copies verbatim.
+     */
+    readonly digest: string;
+}
+/**
+ * One tool call, addressed by what it asked rather than what it got
+ * back, with no bytes of either kept.
+ *
+ * The digests are what make repetition visible: a call whose arguments
+ * digest to something already asked in the same session asked a
+ * question that session's context could already answer. That is waste
+ * with no value judgement in it, which is what makes it safe to act on.
+ */
+export interface ToolCallRecord {
+    /** Content address of the call, stable across a forked log. */
+    readonly digest: string;
+    readonly sessionId: string;
+    /** The assistant entry that made the call. */
+    readonly entryId: string;
+    /** The call's own id, which its result names. */
+    readonly callId: string;
+    readonly timestamp: string;
+    readonly name: string;
+    /** Digest of the arguments, never the arguments. */
+    readonly argsDigest: string;
+    /**
+     * What kind of verifier this call ran, if it ran one at all. Classified
+     * from the command text at scan time, before that text is digested
+     * away, so this is the one place any of it survives, and only as a
+     * category. A chained gate running more than one kind is `verify`
+     * rather than a pick of one, since one exit code cannot support the
+     * precision of naming a single kind.
+     */
+    readonly verifierKind: VerifierKind | null;
+    /** The file the call declared, when it declared one. */
+    readonly path: string | null;
+    /** Characters the result came back with, or null if it never came. */
+    readonly resultChars: number | null;
+    /** Digest of the result text, or null if it never came. */
+    readonly resultDigest: string | null;
+    /** Whether the result came back an error. Null when it never came. */
+    readonly isError: boolean | null;
+}
+/**
+ * What kind of verifier a call ran. `verify` names a chained gate that
+ * ran more than one kind under a single exit code, which is a category
+ * of its own rather than a guess at which one kind mattered.
+ */
+export type VerifierKind = "test" | "build" | "typecheck" | "lint" | "verify";
+/** How a kind of verifier fared across every call classified as it. */
+export interface VerifierOutcome {
+    readonly kind: VerifierKind;
+    readonly passed: number;
+    readonly failed: number;
+    /** Calls whose result never arrived, so pass or fail is unknown. */
+    readonly unknown: number;
+}
+/**
+ * How real compactions compare against the payback test, replayed over
+ * turns already in the ledger rather than watched live. `evaluable`
+ * excludes a compaction with no turn after it (nothing to measure
+ * retention from) or whose model has no derivable rate yet.
+ */
+export interface PaybackReplay {
+    readonly compactions: number;
+    readonly evaluable: number;
+    /** The test would also have fired. */
+    readonly agreed: number;
+    /** The test would have declined what actually happened. */
+    readonly disagreed: number;
+}
+/**
+ * A tool call a compaction dropped from context, and where it happened.
+ *
+ * Recorded at the compaction rather than guessed at later, because the
+ * boundary a compaction drew is known precisely then and only then: the
+ * entry it names as first kept is a fact about that one event, not
+ * something a later query could reconstruct from the calls alone.
+ */
+export interface DroppedCallRecord {
+    /** The call that was dropped, addressing the same row in tool_calls. */
+    readonly callDigest: string;
+    readonly sessionId: string;
+    /** The compaction entry that dropped it. */
+    readonly droppedAtEntryId: string;
+    readonly droppedAtTimestamp: string;
+}
+/**
+ * Which calls a repeat or regret query is about, and what counts as a
+ * file changing underneath them. Both lists are the caller's, because
+ * which tools retrieve and which ones write is a fact about a harness's
+ * tool set, not about the ledger.
+ */
+export interface CallScope {
+    /**
+     * Tools whose result is information, so asking again means the
+     * information was needed again. Absent means every tool, which also
+     * counts re-issued actions and so overstates both measures.
+     */
+    readonly retrieval?: readonly string[];
+    /**
+     * Tools that change the file a call declares. A read repeated after one
+     * of these touched the same path fetched something new, so it is
+     * neither a repeat nor a regret. Absent means no call is treated as a
+     * write.
+     */
+    readonly writers?: readonly string[];
+}
+/**
+ * A dropped call that was asked again afterward: the earlier answer was
+ * discarded and the same question was put a second time. No claim about
+ * whether the second asking was necessary, only that it happened.
+ * Counted once per dropped call, at its first re-ask.
+ */
+export interface Regret {
+    readonly name: string;
+    readonly argsDigest: string;
+    readonly sessionId: string;
+    readonly droppedAtTimestamp: string;
+    readonly reAskedAtTimestamp: string;
+    readonly resultChars: number | null;
+}
+/**
+ * Regret with its denominator, since a count of re-asks means nothing
+ * without how many drops it could have been out of.
+ */
+export interface RegretReport {
+    /** Dropped calls within the scope asked about. */
+    readonly inScope: number;
+    /** Those of them asked again after the drop, earliest re-ask first. */
+    readonly reAsked: Regret[];
+}
+/**
+ * Arguments asked more than once inside one session, and what the
+ * repeats weighed.
+ */
+export interface RepeatedCall {
+    readonly argsDigest: string;
+    readonly name: string;
+    /** How many times these arguments were asked. */
+    readonly asked: number;
+    /** How many of those were repeats, so one less than asked. */
+    readonly repeated: number;
+    /** Characters the repeats re-admitted to the context. */
+    readonly repeatedChars: number;
+}
+/**
+ * What a session log says about itself.
+ *
+ * Every field but the id may be absent, and absent is kept rather than
+ * guessed: attributing spend to a repo or a quest the log never named
+ * would charge work that did not incur it.
+ */
+export interface SessionRecord {
+    readonly sessionId: string;
+    /** The working directory the log named, if it named one. */
+    readonly cwd: string | null;
+    /** The repo that directory belongs to, derived from it. */
+    readonly repo: string | null;
+    /** The quest the session was working under, if any. */
+    readonly quest: string | null;
+    /** Timestamp of the earliest billed turn. */
+    readonly firstSeen: string | null;
+    /** Timestamp of the latest billed turn. */
+    readonly lastSeen: string | null;
+}

package/dist/observability/ledger/types.js

@@ -0,0 +1 @@
+export {};

package/dist/observability/recorder.d.ts

@@ -31,8 +31,15 @@ export interface RunRecordInput {
 /**
  * Build a {@link RunRecord} from a producer's per-subagent
  * result. Verify outcome derives from the verification
- * block when present, and usage falls back to zeros for a
- * run that reported none (older pi, a crashed child).
+ * block when present.
+ *
+ * A run that reported no usage (a crashed child, a round
+ * killed before billing, an older pi) still produces a record,
+ * because withholding the row would make the worst failures
+ * the ones the table has nothing to say about. But its tokens
+ * and cost are null rather than zero. The row is the claim
+ * that the run happened; a zero would be a second claim, that
+ * it was free, which nothing supports.
  */
 export declare function runRecordFrom(input: RunRecordInput): RunRecord;
 /**

package/dist/observability/recorder.js

@@ -1,23 +1,16 @@
 import { processGlobal } from "../internal/process-global.js";
-const ZERO_TOKENS = {
-    input: 0,
-    output: 0,
-    cacheRead: 0,
-    cacheWrite: 0,
-    total: 0,
-};
-const ZERO_COST = {
-    input: 0,
-    output: 0,
-    cacheRead: 0,
-    cacheWrite: 0,
-    total: 0,
-};
 /**
  * Build a {@link RunRecord} from a producer's per-subagent
  * result. Verify outcome derives from the verification
- * block when present, and usage falls back to zeros for a
- * run that reported none (older pi, a crashed child).
+ * block when present.
+ *
+ * A run that reported no usage (a crashed child, a round
+ * killed before billing, an older pi) still produces a record,
+ * because withholding the row would make the worst failures
+ * the ones the table has nothing to say about. But its tokens
+ * and cost are null rather than zero. The row is the claim
+ * that the run happened; a zero would be a second claim, that
+ * it was free, which nothing supports.
  */
 export function runRecordFrom(input) {
     const { result } = input;
@@ -38,8 +31,8 @@ export function runRecordFrom(input) {
         retriesToValid: Math.max(0, (result.verification?.attempts ?? 1) - 1),
         warningCount: result.warnings.length,
         exitCode: result.exitCode,
-        tokens: result.usage?.tokens ?? ZERO_TOKENS,
-        cost: result.usage?.cost ?? ZERO_COST,
+        tokens: result.usage?.tokens ?? null,
+        cost: result.usage?.cost ?? null,
         startedAt: input.startedAt,
     };
 }

package/dist/observability/store.d.ts

@@ -8,9 +8,11 @@ export interface RunStore {
     recordRun(record: RunRecord): Promise<void>;
     queryRuns(filter?: RunQuery): Promise<RunRecord[]>;
     summarizeRun(runId: string): Promise<RunSummary | null>;
-    rollupBefore(cutoffMs: number): Promise<{
-        rolledRows: number;
-    }>;
+    /**
+     * Weekly per-model, per-persona summaries, computed from the rows
+     * rather than materialised beside them. Nothing has to run for this
+     * to be current, and no row is ever discarded to produce it.
+     */
     queryRollups(): Promise<RunRollup[]>;
     close(): Promise<void>;
 }

package/dist/observability/store.js

@@ -25,6 +25,11 @@ CREATE TABLE IF NOT EXISTS runs (
 );
 CREATE INDEX IF NOT EXISTS runs_run_id ON runs (run_id);
 CREATE INDEX IF NOT EXISTS runs_started_at ON runs (started_at);
+/**
+ * The rollups table is legacy and is never written to again. It holds
+ * the only surviving record of 2,199 runs whose raw rows an earlier
+ * retention pass deleted, so it is read and never dropped.
+ */
 CREATE TABLE IF NOT EXISTS rollups (
 	week_start INTEGER NOT NULL,
 	model TEXT NOT NULL,
@@ -48,8 +53,63 @@ export async function openRunStore(dbPath) {
     const db = await openDb(dbPath);
     await db.exec("PRAGMA journal_mode = WAL; PRAGMA busy_timeout = 5000;");
     await db.exec(SCHEMA);
+    await migrate(db);
     return new SqliteRunStore(db);
 }
+/**
+ * Bring a store forward to the current shape, in place and without
+ * losing a row.
+ *
+ * This is the only schema path: a new store is created at the original
+ * shape and migrated like any other, so every test that opens a fresh
+ * store also exercises the migration an existing file will meet.
+ * Every step is additive, a column or an index, so nothing a store
+ * already holds can be lost by opening it.
+ */
+async function migrate(db) {
+    const columns = new Set((await db.all("PRAGMA table_info(runs)")).map((column) => column.name));
+    if (!columns.has("metered")) {
+        await db.exec("ALTER TABLE runs ADD COLUMN metered INTEGER NOT NULL DEFAULT 1");
+        // Rows written before this column existed recorded an unmetered
+        // run as zero cost. Zero tokens identifies them: a run that did
+        // anything consumed some, and all 75 such rows in the real store
+        // had none. Only backfilled on the pass that adds the column, so
+        // a later metered run cannot be reclassified.
+        await db.exec("UPDATE runs SET metered = 0 WHERE tokens_total = 0 AND cost_total = 0");
+    }
+    for (const [name, type] of ATTRIBUTION_COLUMNS) {
+        if (!columns.has(name)) {
+            await db.exec(`ALTER TABLE runs ADD COLUMN ${name} ${type}`);
+        }
+    }
+    const indexes = await db.all("SELECT name FROM sqlite_master WHERE type = 'index' AND name = 'runs_identity'");
+    if (indexes.length === 0) {
+        // A subagent of a run is one row. A store written before that rule
+        // may hold duplicates, and the index cannot be added over them.
+        // They are captured before anything is removed, keeping the latest
+        // of each pair where it was: nothing is deleted that is not first
+        // copied somewhere it can be recovered from.
+        await db.exec(`CREATE TABLE IF NOT EXISTS runs_superseded AS SELECT * FROM runs WHERE 0;
+			INSERT INTO runs_superseded SELECT * FROM runs WHERE rowid NOT IN (
+				SELECT MAX(rowid) FROM runs GROUP BY run_id, subagent_id
+			);
+			DELETE FROM runs WHERE rowid NOT IN (
+				SELECT MAX(rowid) FROM runs GROUP BY run_id, subagent_id
+			);
+			CREATE UNIQUE INDEX runs_identity ON runs (run_id, subagent_id);`);
+    }
+}
+/**
+ * Where a run belonged, all nullable because a row written before these
+ * existed does not know, and a guess would charge work that did not
+ * incur the cost.
+ */
+const ATTRIBUTION_COLUMNS = [
+    ["session_id", "TEXT"],
+    ["cwd", "TEXT"],
+    ["repo", "TEXT"],
+    ["ended_at", "INTEGER"],
+];
 class SqliteRunStore {
     db;
     constructor(db) {
@@ -61,8 +121,27 @@ class SqliteRunStore {
 				retries_to_valid, warning_count, exit_code,
 				tokens_input, tokens_output, tokens_cache_read, tokens_cache_write, tokens_total,
 				cost_input, cost_output, cost_cache_read, cost_cache_write, cost_total,
-				started_at
-			) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, [
+				started_at, metered, session_id, cwd, repo, ended_at
+			) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+			ON CONFLICT (run_id, subagent_id) DO UPDATE SET
+				kind = excluded.kind, model = excluded.model,
+				persona = excluded.persona, verify_outcome = excluded.verify_outcome,
+				retries_to_valid = excluded.retries_to_valid,
+				warning_count = excluded.warning_count,
+				exit_code = excluded.exit_code,
+				tokens_input = excluded.tokens_input,
+				tokens_output = excluded.tokens_output,
+				tokens_cache_read = excluded.tokens_cache_read,
+				tokens_cache_write = excluded.tokens_cache_write,
+				tokens_total = excluded.tokens_total,
+				cost_input = excluded.cost_input,
+				cost_output = excluded.cost_output,
+				cost_cache_read = excluded.cost_cache_read,
+				cost_cache_write = excluded.cost_cache_write,
+				cost_total = excluded.cost_total,
+				started_at = excluded.started_at, metered = excluded.metered,
+				session_id = excluded.session_id, cwd = excluded.cwd,
+				repo = excluded.repo, ended_at = excluded.ended_at`, [
             record.runId,
             record.subagentId,
             record.kind,
@@ -72,17 +151,25 @@ class SqliteRunStore {
             record.retriesToValid,
             record.warningCount,
             record.exitCode,
-            record.tokens.input,
-            record.tokens.output,
-            record.tokens.cacheRead,
-            record.tokens.cacheWrite,
-            record.tokens.total,
-            record.cost.input,
-            record.cost.output,
-            record.cost.cacheRead,
-            record.cost.cacheWrite,
-            record.cost.total,
+            // An unmetered run stores zeros, which leave every sum exactly
+            // as excluding it would, and the flag beside them is what says
+            // the zeros are unknown rather than free.
+            record.tokens?.input ?? 0,
+            record.tokens?.output ?? 0,
+            record.tokens?.cacheRead ?? 0,
+            record.tokens?.cacheWrite ?? 0,
+            record.tokens?.total ?? 0,
+            record.cost?.input ?? 0,
+            record.cost?.output ?? 0,
+            record.cost?.cacheRead ?? 0,
+            record.cost?.cacheWrite ?? 0,
+            record.cost?.total ?? 0,
             record.startedAt,
+            record.cost ? 1 : 0,
+            record.sessionId ?? null,
+            record.cwd ?? null,
+            record.repo ?? null,
+            record.endedAt ?? null,
         ]);
     }
     async queryRuns(filter = {}) {
@@ -98,6 +185,7 @@ class SqliteRunStore {
 				SUM(CASE WHEN verify_outcome = 'failed' THEN 1 ELSE 0 END) AS failed,
 				SUM(retries_to_valid) AS total_retries,
 				SUM(warning_count) AS total_warnings,
+				SUM(CASE WHEN metered = 0 THEN 1 ELSE 0 END) AS unmetered,
 				SUM(tokens_input) AS tokens_input,
 				SUM(tokens_output) AS tokens_output,
 				SUM(tokens_cache_read) AS tokens_cache_read,
@@ -120,6 +208,7 @@ class SqliteRunStore {
             failed: row.failed,
             totalRetries: row.total_retries,
             totalWarnings: row.total_warnings,
+            unmetered: row.unmetered,
             tokens: {
                 input: row.tokens_input,
                 output: row.tokens_output,
@@ -137,36 +226,36 @@ class SqliteRunStore {
             cacheReadRatio: freshInput === 0 ? 0 : row.tokens_cache_read / freshInput,
         };
     }
-    async rollupBefore(cutoffMs) {
-        const counted = await this.db.all("SELECT COUNT(*) AS n FROM runs WHERE started_at < ?", [cutoffMs]);
-        const rolledRows = counted[0]?.n ?? 0;
-        if (rolledRows === 0)
-            return { rolledRows: 0 };
-        await this.db.run(`INSERT INTO rollups (
-				week_start, model, persona, run_count, total_retries, total_warnings,
-				tokens_total, cost_total, cache_read, fresh_input
-			)
-			SELECT
-				(started_at / ${WEEK_MS}) * ${WEEK_MS} AS week_start,
-				model, persona,
-				COUNT(*), SUM(retries_to_valid), SUM(warning_count),
-				SUM(tokens_total), SUM(cost_total),
-				SUM(tokens_cache_read), SUM(tokens_input + tokens_cache_read)
-			FROM runs WHERE started_at < ?
-			GROUP BY week_start, model, persona
-			ON CONFLICT(week_start, model, persona) DO UPDATE SET
-				run_count = run_count + excluded.run_count,
-				total_retries = total_retries + excluded.total_retries,
-				total_warnings = total_warnings + excluded.total_warnings,
-				tokens_total = tokens_total + excluded.tokens_total,
-				cost_total = cost_total + excluded.cost_total,
-				cache_read = cache_read + excluded.cache_read,
-				fresh_input = fresh_input + excluded.fresh_input`, [cutoffMs]);
-        await this.db.run("DELETE FROM runs WHERE started_at < ?", [cutoffMs]);
-        return { rolledRows };
-    }
+    /**
+     * Summaries over every row held, unioned with the legacy table.
+     *
+     * Computed rather than materialised, because the only reason to
+     * materialise was that the rows behind it were being deleted. They
+     * are not any more: the whole corpus is a rounding error on disk and
+     * discarding detail to save it was never a trade worth making.
+     *
+     * The legacy rows are unioned in rather than ignored, since for the
+     * period before this changed they are all that is left.
+     */
     async queryRollups() {
-        const rows = await this.db.all("SELECT * FROM rollups ORDER BY week_start ASC, model ASC, persona ASC");
+        const rows = await this.db.all(`SELECT * FROM (
+				SELECT
+					(started_at / ${WEEK_MS}) * ${WEEK_MS} AS week_start,
+					model, persona,
+					COUNT(*) AS run_count,
+					SUM(retries_to_valid) AS total_retries,
+					SUM(warning_count) AS total_warnings,
+					SUM(tokens_total) AS tokens_total,
+					SUM(cost_total) AS cost_total,
+					SUM(tokens_cache_read) AS cache_read,
+					SUM(tokens_input + tokens_cache_read) AS fresh_input
+				FROM runs GROUP BY week_start, model, persona
+				UNION ALL
+				SELECT week_start, model, persona, run_count, total_retries,
+					total_warnings, tokens_total, cost_total, cache_read, fresh_input
+				FROM rollups
+			)
+			ORDER BY week_start ASC, model ASC, persona ASC`);
         return rows.map((row) => ({
             weekStart: row.week_start,
             model: row.model,
@@ -194,20 +283,28 @@ function rowToRecord(row) {
         retriesToValid: row.retries_to_valid,
         warningCount: row.warning_count,
         exitCode: row.exit_code,
-        tokens: {
-            input: row.tokens_input,
-            output: row.tokens_output,
-            cacheRead: row.tokens_cache_read,
-            cacheWrite: row.tokens_cache_write,
-            total: row.tokens_total,
-        },
-        cost: {
-            input: row.cost_input,
-            output: row.cost_output,
-            cacheRead: row.cost_cache_read,
-            cacheWrite: row.cost_cache_write,
-            total: row.cost_total,
-        },
+        tokens: row.metered === 0
+            ? null
+            : {
+                input: row.tokens_input,
+                output: row.tokens_output,
+                cacheRead: row.tokens_cache_read,
+                cacheWrite: row.tokens_cache_write,
+                total: row.tokens_total,
+            },
+        cost: row.metered === 0
+            ? null
+            : {
+                input: row.cost_input,
+                output: row.cost_output,
+                cacheRead: row.cost_cache_read,
+                cacheWrite: row.cost_cache_write,
+                total: row.cost_total,
+            },
         startedAt: row.started_at,
+        sessionId: row.session_id ?? null,
+        cwd: row.cwd ?? null,
+        repo: row.repo ?? null,
+        endedAt: row.ended_at ?? null,
     };
 }

package/dist/observability/types.d.ts

@@ -43,12 +43,34 @@ export interface RunRecord {
     readonly warningCount: number;
     /** Process exit code. */
     readonly exitCode: number;
-    /** Token counts summed across the run's turns. */
-    readonly tokens: RunTokens;
-    /** Cost summed across the run's turns. */
-    readonly cost: RunCost;
+    /**
+     * Token counts summed across the run's turns, or null when the run
+     * reported no usage at all.
+     */
+    readonly tokens: RunTokens | null;
+    /**
+     * Cost summed across the run's turns, or null when the run reported
+     * no usage. Null is not zero: a run that died before it could report
+     * cost an unknown amount, and recording it as free makes it
+     * indistinguishable from one that genuinely cost nothing. Every one
+     * of the 75 zero-cost rows found in the real store had zero tokens
+     * too, so not one of them was actually free.
+     */
+    readonly cost: RunCost | null;
     /** When the run started, epoch milliseconds. */
     readonly startedAt: number;
+    /**
+     * The parent session that dispatched the run, the directory it was
+     * working in and the repo that directory belongs to. Stamped by the
+     * sink rather than the producer, since the sink is what knows where
+     * it is. Null when not known, never guessed: without these, $12,825
+     * of fan-out could not be traced to the work that caused it.
+     */
+    readonly sessionId?: string | null;
+    readonly cwd?: string | null;
+    readonly repo?: string | null;
+    /** When the run's record was written, epoch milliseconds. */
+    readonly endedAt?: number | null;
 }
 /** Aggregate view of one run across its subagents. */
 export interface RunSummary {
@@ -58,6 +80,12 @@ export interface RunSummary {
     readonly failed: number;
     readonly totalRetries: number;
     readonly totalWarnings: number;
+    /**
+     * Subagents that reported no usage. The totals beside this exclude
+     * them, so a total can say what it is missing rather than silently
+     * pricing the unknown at nothing.
+     */
+    readonly unmetered: number;
     readonly tokens: RunTokens;
     readonly cost: RunCost;
     /** cacheRead / (input + cacheRead); 0 when the denominator is 0. */