@jitsusama/agentic-harness.core 0.3.1 → 0.5.0

package/dist/observability/ledger/scan.d.ts

@@ -0,0 +1,11 @@
+import type { LedgerScan } from "./types.js";
+/**
+ * Read billable turns out of a session log's lines.
+ *
+ * Every line is offered to the parser and every outcome is counted, so a
+ * malformed line costs one entry rather than the remainder of the file.
+ * Both places a turn can carry usage are read: assistant turns hold it
+ * under `message`, and compactions hold it at the top level beside
+ * `type`.
+ */
+export declare function readTurns(sessionId: string, lines: Iterable<string>): LedgerScan;

package/dist/observability/ledger/scan.js

@@ -0,0 +1,159 @@
+import { createHash } from "node:crypto";
+import { SessionCollector } from "./session.js";
+/** Width of a stored content address. 96 bits is ample for a corpus of
+ * a few million turns and keeps the index small. */
+const DIGEST_CHARS = 24;
+/**
+ * Read billable turns out of a session log's lines.
+ *
+ * Every line is offered to the parser and every outcome is counted, so a
+ * malformed line costs one entry rather than the remainder of the file.
+ * Both places a turn can carry usage are read: assistant turns hold it
+ * under `message`, and compactions hold it at the top level beside
+ * `type`.
+ */
+export function readTurns(sessionId, lines) {
+    const turns = [];
+    const session = new SessionCollector(sessionId);
+    let count = 0;
+    let parsed = 0;
+    let unparseable = 0;
+    let billable = 0;
+    let unmetered = 0;
+    for (const line of lines) {
+        count += 1;
+        if (!line.trim())
+            continue;
+        let entry;
+        try {
+            const value = JSON.parse(line);
+            if (typeof value !== "object" || value === null) {
+                unparseable += 1;
+                continue;
+            }
+            entry = value;
+        }
+        catch {
+            // A truncated or interleaved write. Counted, never fatal: one
+            // unreadable line once reduced a corpus-wide total to a tenth
+            // of the truth by aborting the pipeline that met it.
+            unparseable += 1;
+            continue;
+        }
+        parsed += 1;
+        if (entry.type === "session") {
+            session.observeHeader(entry);
+            continue;
+        }
+        if (entry.customType === "quest-workflow") {
+            const data = asRecord(entry.data);
+            if (data)
+                session.observeWorkflow(data);
+            continue;
+        }
+        const turn = turnFrom(sessionId, entry);
+        if (!turn)
+            continue;
+        session.observeTurn(turn.timestamp);
+        turns.push(turn);
+        if (turn.cost)
+            billable += 1;
+        else
+            unmetered += 1;
+    }
+    return {
+        turns,
+        coverage: { lines: count, parsed, unparseable, billable, unmetered },
+        session: session.record(),
+    };
+}
+function turnFrom(sessionId, entry) {
+    const kind = kindOf(entry);
+    if (!kind)
+        return null;
+    const message = asRecord(entry.message);
+    const usage = asRecord(kind === "compaction" ? entry.usage : message?.usage);
+    const entryId = typeof entry.id === "string" ? entry.id : "";
+    const timestamp = typeof entry.timestamp === "string" ? entry.timestamp : "";
+    const model = typeof message?.model === "string" ? message.model : "";
+    return {
+        entryId,
+        sessionId,
+        timestamp,
+        kind,
+        model,
+        tokens: tokensFrom(usage),
+        cost: costFrom(usage),
+        cacheWrite1h: usage?.cacheWrite1h ?? 0,
+        droppedBefore: kind === "compaction" && typeof entry.tokensBefore === "number"
+            ? entry.tokensBefore
+            : null,
+        firstKeptEntryId: typeof entry.firstKeptEntryId === "string"
+            ? entry.firstKeptEntryId
+            : null,
+        digest: digestOf(entryId, timestamp, kind, usage),
+    };
+}
+/**
+ * Which turns are billable at all. An assistant turn and a compaction
+ * both cost money; a user message, a tool result and a state change do
+ * not. Enumerated rather than filtered, so a new entry type is ignored
+ * by omission instead of silently swept into a total.
+ */
+function kindOf(entry) {
+    if (entry.type === "compaction")
+        return "compaction";
+    const message = asRecord(entry.message);
+    if (message?.role === "assistant")
+        return "assistant";
+    return null;
+}
+function tokensFrom(usage) {
+    const input = usage?.input ?? 0;
+    const output = usage?.output ?? 0;
+    const cacheRead = usage?.cacheRead ?? 0;
+    const cacheWrite = usage?.cacheWrite ?? 0;
+    return {
+        input,
+        output,
+        cacheRead,
+        cacheWrite,
+        total: usage?.totalTokens ?? input + output + cacheRead + cacheWrite,
+    };
+}
+/** Null when the entry reported no cost, which is not the same as free. */
+function costFrom(usage) {
+    const cost = usage?.cost;
+    if (!cost || typeof cost.total !== "number")
+        return null;
+    return {
+        input: cost.input ?? 0,
+        output: cost.output ?? 0,
+        cacheRead: cost.cacheRead ?? 0,
+        cacheWrite: cost.cacheWrite ?? 0,
+        total: cost.total,
+    };
+}
+/**
+ * Address a turn by what it is rather than where it was found. Forking a
+ * session copies entries verbatim, ids included, so the same turn appears
+ * in several files and a naive count bills it more than once.
+ */
+function digestOf(entryId, timestamp, kind, usage) {
+    const canonical = JSON.stringify([
+        entryId,
+        timestamp,
+        kind,
+        usage?.cost?.total ?? null,
+        usage?.totalTokens ?? null,
+    ]);
+    return createHash("sha256")
+        .update(canonical)
+        .digest("hex")
+        .slice(0, DIGEST_CHARS);
+}
+function asRecord(value) {
+    return typeof value === "object" && value !== null
+        ? value
+        : null;
+}

package/dist/observability/ledger/session.d.ts

@@ -0,0 +1,38 @@
+import type { SessionRecord } from "./types.js";
+/**
+ * Name the repo a working directory belongs to, or nothing when the path
+ * names none.
+ *
+ * A monorepo zone is named by the zone rather than the worktree it was
+ * cut into, because two trees of the same zone are the same subject and
+ * naming the tree would split one zone's spend across every tree ever
+ * cut for it.
+ */
+export declare function repoOf(cwd: string | null): string | null;
+/**
+ * Accumulates what a log says about its session as the log is read, so
+ * one pass serves both the turns and their attribution.
+ */
+export declare class SessionCollector {
+    private readonly sessionId;
+    private cwd;
+    private quest;
+    private first;
+    private last;
+    constructor(sessionId: string);
+    /**
+     * Take the working directory from a session's header entry. Every log
+     * opens with one, which is what makes attribution complete rather than
+     * limited to the quarter of sessions that also name a quest.
+     */
+    observeHeader(entry: Record<string, unknown>): void;
+    /**
+     * Take the working directory and quest a workflow entry names. The
+     * last one wins, because a session can be re-pointed at another quest
+     * part way through and the later statement is the current one.
+     */
+    observeWorkflow(data: Record<string, unknown>): void;
+    /** Widen the span to include a billed turn. */
+    observeTurn(timestamp: string): void;
+    record(): SessionRecord;
+}

package/dist/observability/ledger/session.js

@@ -0,0 +1,89 @@
+/** Where a per-host checkout tree begins, as `src/{host}/{owner}/{repo}`. */
+const CHECKOUT_MARKER = "/src/";
+/** Where a monorepo worktree begins, as `world/trees/{tree}/src/{zone}`. */
+const MONOREPO_MARKER = "/world/trees/";
+/** Segments naming a repo under the checkout marker: host, owner, name. */
+const REPO_SEGMENTS = 3;
+/**
+ * Name the repo a working directory belongs to, or nothing when the path
+ * names none.
+ *
+ * A monorepo zone is named by the zone rather than the worktree it was
+ * cut into, because two trees of the same zone are the same subject and
+ * naming the tree would split one zone's spend across every tree ever
+ * cut for it.
+ */
+export function repoOf(cwd) {
+    if (!cwd)
+        return null;
+    const monorepo = cwd.indexOf(MONOREPO_MARKER);
+    if (monorepo >= 0) {
+        const tail = cwd.slice(monorepo + MONOREPO_MARKER.length);
+        const zone = tail.split("/src/")[1];
+        return zone ? `world/${zone}` : null;
+    }
+    const checkout = cwd.indexOf(CHECKOUT_MARKER);
+    if (checkout >= 0) {
+        const parts = cwd
+            .slice(checkout + CHECKOUT_MARKER.length)
+            .split("/")
+            .filter(Boolean);
+        if (parts.length >= REPO_SEGMENTS) {
+            return parts.slice(0, REPO_SEGMENTS).join("/");
+        }
+    }
+    return null;
+}
+/**
+ * Accumulates what a log says about its session as the log is read, so
+ * one pass serves both the turns and their attribution.
+ */
+export class SessionCollector {
+    sessionId;
+    cwd = null;
+    quest = null;
+    first = null;
+    last = null;
+    constructor(sessionId) {
+        this.sessionId = sessionId;
+    }
+    /**
+     * Take the working directory from a session's header entry. Every log
+     * opens with one, which is what makes attribution complete rather than
+     * limited to the quarter of sessions that also name a quest.
+     */
+    observeHeader(entry) {
+        if (typeof entry.cwd === "string")
+            this.cwd = entry.cwd;
+    }
+    /**
+     * Take the working directory and quest a workflow entry names. The
+     * last one wins, because a session can be re-pointed at another quest
+     * part way through and the later statement is the current one.
+     */
+    observeWorkflow(data) {
+        if (typeof data.cwd === "string")
+            this.cwd = data.cwd;
+        if (typeof data.questId === "string")
+            this.quest = data.questId;
+    }
+    /** Widen the span to include a billed turn. */
+    observeTurn(timestamp) {
+        if (!timestamp)
+            return;
+        if (!this.first || timestamp < this.first)
+            this.first = timestamp;
+        if (!this.last || timestamp > this.last)
+            this.last = timestamp;
+    }
+    record() {
+        return {
+            sessionId: this.sessionId,
+            cwd: this.cwd,
+            repo: repoOf(this.cwd),
+            quest: this.quest,
+            firstSeen: this.first,
+            lastSeen: this.last,
+        };
+    }
+}

package/dist/observability/ledger/store.d.ts

@@ -21,7 +21,7 @@ export interface RecordOutcome {
     readonly duplicates: number;
 }
 /** What a total or a slice may be narrowed to. */
-export type CostDimension = "model" | "session" | "kind" | "day" | "repo" | "quest";
+export type CostDimension = "model" | "session" | "kind" | "day" | "repo" | "quest" | "thinking";
 /** A content-addressed store of billable turns. */
 export interface TurnStore {
     recordTurns(turns: readonly TurnRecord[]): Promise<RecordOutcome>;
@@ -40,6 +40,8 @@ export interface TurnStore {
     paybackReplay(): Promise<PaybackReplay>;
     total(): Promise<LedgerTotal>;
     costBy(dimension: CostDimension): Promise<CostSlice[]>;
+    /** Every session the ledger holds, for another store to join to. */
+    sessions(): Promise<SessionRecord[]>;
     close(): Promise<void>;
 }
 /**

package/dist/observability/ledger/store.js

@@ -52,6 +52,8 @@ CREATE TABLE IF NOT EXISTS tool_calls (
 CREATE INDEX IF NOT EXISTS tool_calls_verifier ON tool_calls (verifier_kind);
 CREATE INDEX IF NOT EXISTS tool_calls_args ON tool_calls (session_id, args_digest);
 CREATE INDEX IF NOT EXISTS tool_calls_path ON tool_calls (path);
+CREATE INDEX IF NOT EXISTS tool_calls_session_verifier
+	ON tool_calls (session_id, timestamp) WHERE verifier_kind IS NOT NULL;
 CREATE TABLE IF NOT EXISTS dropped_calls (
 	digest TEXT PRIMARY KEY,
 	session_id TEXT NOT NULL,
@@ -80,6 +82,7 @@ const GROUP_BY = {
     // that quietly excluded most of the money.
     repo: "COALESCE(sessions.repo, '')",
     quest: "COALESCE(sessions.quest, '')",
+    thinking: "COALESCE(turns.thinking_level, '')",
 };
 /**
  * Open (creating if needed) a turn ledger at the given path. Safe to
@@ -90,8 +93,22 @@ export async function openTurnStore(dbPath) {
     const db = await openDb(dbPath);
     await db.exec("PRAGMA journal_mode = WAL; PRAGMA busy_timeout = 5000;");
     await db.exec(SCHEMA);
+    await migrate(db);
     return new SqliteTurnStore(db);
 }
+/**
+ * Bring a ledger written at an older shape up to this one. A new ledger
+ * is created at the original shape and migrated like any other, so every
+ * test that opens a fresh store also exercises the path an existing file
+ * meets. Every step is additive, so nothing a ledger holds can be lost by
+ * opening it.
+ */
+async function migrate(db) {
+    const columns = new Set((await db.all("PRAGMA table_info(turns)")).map((column) => column.name));
+    if (!columns.has("thinking_level")) {
+        await db.exec("ALTER TABLE turns ADD COLUMN thinking_level TEXT");
+    }
+}
 class SqliteTurnStore {
     db;
     constructor(db) {
@@ -128,6 +145,7 @@ class SqliteTurnStore {
                 // Seen before, so not billed again, but the sighting is still
                 // recorded: deduplicating must not hide that it happened.
                 await this.sight(t);
+                await this.fillThinkingLevel(t);
                 continue;
             }
             inserted += 1;
@@ -136,8 +154,8 @@ class SqliteTurnStore {
 					tokens_input, tokens_output, tokens_cache_read,
 					tokens_cache_write, tokens_total, cache_write_1h,
 					cost_input, cost_output, cost_cache_read, cost_cache_write,
-					cost_total, dropped_before, first_kept_entry_id
-				) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, [
+					cost_total, dropped_before, first_kept_entry_id, thinking_level
+				) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, [
                 t.digest,
                 t.entryId,
                 t.sessionId,
@@ -157,6 +175,7 @@ class SqliteTurnStore {
                 t.cost?.total ?? null,
                 t.droppedBefore,
                 t.firstKeptEntryId,
+                t.thinkingLevel,
             ]);
             await this.sight(t);
         }
@@ -284,18 +303,32 @@ class SqliteTurnStore {
         const changed = writtenBetween("o", "o.previous_at", "o.timestamp", scope.writers);
         const rows = await this.db.all(`WITH ordered AS (
 				SELECT c.session_id, c.args_digest, c.name, c.path,
-					c.timestamp, c.result_chars,
+					c.timestamp, c.result_chars, c.verifier_kind,
 					LAG(c.timestamp) OVER (
 						PARTITION BY c.session_id, c.args_digest
 						ORDER BY c.timestamp, c.digest
 					) AS previous_at
 				FROM tool_calls AS c
 				WHERE ${retrieval.sql}
+			),
+			repeats AS (
+				SELECT o.session_id, o.args_digest, o.name, o.result_chars,
+					(o.verifier_kind IS NOT NULL OR EXISTS (
+						SELECT 1 FROM tool_calls AS v
+						WHERE v.session_id = o.session_id
+							AND v.verifier_kind IS NOT NULL
+							AND v.timestamp > o.previous_at
+							AND v.timestamp < o.timestamp
+					)) AS is_appraisal
+				FROM ordered AS o
+				WHERE o.previous_at IS NOT NULL AND NOT (${changed.sql})
 			)
 			SELECT args_digest, name, COUNT(*) AS repeated,
-				SUM(COALESCE(result_chars, 0)) AS repeated_chars
-			FROM ordered AS o
-			WHERE o.previous_at IS NOT NULL AND NOT (${changed.sql})
+				SUM(COALESCE(result_chars, 0)) AS repeated_chars,
+				SUM(is_appraisal) AS appraisal,
+				SUM(CASE WHEN is_appraisal THEN COALESCE(result_chars, 0) ELSE 0 END)
+					AS appraisal_chars
+			FROM repeats
 			GROUP BY session_id, args_digest
 			ORDER BY repeated_chars DESC`, [...retrieval.params, ...changed.params]);
         return rows.map((row) => ({
@@ -304,6 +337,10 @@ class SqliteTurnStore {
             asked: row.repeated + 1,
             repeated: row.repeated,
             repeatedChars: row.repeated_chars ?? 0,
+            rework: row.repeated - row.appraisal,
+            reworkChars: (row.repeated_chars ?? 0) - (row.appraisal_chars ?? 0),
+            appraisal: row.appraisal,
+            appraisalChars: row.appraisal_chars ?? 0,
         }));
     }
     async recordDropped(dropped) {
@@ -498,9 +535,32 @@ class SqliteTurnStore {
             turns: r.turns,
         }));
     }
+    async sessions() {
+        const rows = await this.db.all(`SELECT session_id, cwd, repo, quest, first_seen, last_seen
+			FROM sessions ORDER BY session_id`);
+        return rows.map((r) => ({
+            sessionId: r.session_id,
+            cwd: r.cwd,
+            repo: r.repo,
+            quest: r.quest,
+            firstSeen: r.first_seen,
+            lastSeen: r.last_seen,
+        }));
+    }
     async close() {
         await this.db.close();
     }
+    /**
+     * Give a held turn the thinking level a later scan learned, when it
+     * had none. This is how a ledger indexed before the column existed
+     * gets it on the next rescan, without billing anything twice. A level
+     * already known is left alone: the same entry cannot have run at two.
+     */
+    async fillThinkingLevel(t) {
+        if (t.thinkingLevel === null)
+            return;
+        await this.db.run("UPDATE turns SET thinking_level = ? WHERE digest = ? AND thinking_level IS NULL", [t.thinkingLevel, t.digest]);
+    }
     async sight(t) {
         await this.db.run("INSERT OR IGNORE INTO sightings (digest, session_id) VALUES (?, ?)", [t.digest, t.sessionId]);
     }

package/dist/observability/ledger/types.d.ts

@@ -17,6 +17,13 @@ export interface TurnRecord {
     readonly kind: TurnKind;
     /** Resolved model id, or empty when the entry did not say. */
     readonly model: string;
+    /**
+     * The thinking level the turn ran at, as the log last set it before
+     * the turn, or null when the log never said. Unknown is not a level:
+     * guessing the harness default would put spend on a setting that may
+     * not have been in force.
+     */
+    readonly thinkingLevel: string | null;
     readonly tokens: RunTokens;
     /** Null when unmetered. Never coerced to zero. */
     readonly cost: RunCost | null;
@@ -174,6 +181,18 @@ export interface RepeatedCall {
     readonly repeated: number;
     /** Characters the repeats re-admitted to the context. */
     readonly repeatedChars: number;
+    /** Repeats with no verifier since the last ask: the waste bucket. */
+    readonly rework: number;
+    /** Characters the rework repeats re-admitted. */
+    readonly reworkChars: number;
+    /**
+     * Repeats that are themselves a verifier, or that follow one run
+     * since the last ask: the model checking its work, which is cost of
+     * quality rather than waste.
+     */
+    readonly appraisal: number;
+    /** Characters the appraisal repeats re-admitted. */
+    readonly appraisalChars: number;
 }
 /**
  * What a session log says about itself.

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@jitsusama/agentic-harness.core",
-	"version": "0.3.1",
+	"version": "0.5.0",
 	"description": "Pi-agnostic business logic for agentic-harness: state machines, guardian decisions, quest/TDD domain model.",
 	"license": "MIT",
 	"type": "module",