agentwrangler 0.1.0

package/dist/ingest/parser.js

@@ -0,0 +1,344 @@
+/**
+ * src/ingest/parser.ts — per-line SEC-101 projection.
+ *
+ * projectLine() parses one JSONL line and returns a structural projection:
+ * tokens, ids, sizes, model, and markers only. Content (message text, tool
+ * inputs/outputs) is read solely to measure size or extract a commit SHA, then
+ * discarded — nothing content-bearing is returned or stored.
+ *
+ * Failure handling (Spec §1.2): a line that fails JSON parse, or a usage record
+ * missing a required field (dedupe key or timestamp), becomes a quarantine
+ * pointer (no content). Unknown top-level fields are tolerated and counted.
+ */
+import * as crypto from "node:crypto";
+import { PARSER_VERSION, } from "./types.js";
+export { PARSER_VERSION };
+/** Top-level fields the parser recognises. Anything else is tolerated + counted. */
+const KNOWN_TOP_LEVEL = new Set([
+    "type",
+    "subtype",
+    "timestamp",
+    "uuid",
+    "parentUuid",
+    "logicalParentUuid",
+    "leafUuid",
+    "sessionId",
+    "session_id",
+    "isSidechain",
+    "message",
+    "command",
+    "name",
+    "content",
+    "summary",
+    "cwd",
+    "version",
+    "gitBranch",
+    "userType",
+    "requestId",
+    "toolUseResult",
+    "toolUseID",
+    "promptSource",
+    "isMeta",
+    "isCompactSummary",
+    "isApiErrorMessage",
+    "isVisibleInTranscriptOnly",
+    "level",
+]);
+const SHA_RE = /\b[0-9a-f]{40}\b/;
+const GIT_WRITE_RE = /\bgit\s+(commit|push)\b/;
+const TEST_COMMAND_RE = /^(?:(?:npm|pnpm|yarn)\s+(?:test|run\s+test(?::[\w.-]+)?)|bun\s+test|(?:npx\s+)?vitest(?:\s+run)?|pytest|python\s+-m\s+pytest|cargo\s+test|go\s+test|dotnet\s+test|mvn\s+test|gradle\s+test|\.\/gradlew\s+test)(?:\s|$)/;
+/** Project one JSONL line into its structural parts. */
+export function projectLine(rawLine, ctx) {
+    let d;
+    try {
+        d = JSON.parse(rawLine);
+    }
+    catch (e) {
+        return { kind: "quarantine", errorClass: classifyJsonError(e) };
+    }
+    if (typeof d !== "object" || d === null || Array.isArray(d)) {
+        return { kind: "quarantine", errorClass: "NOT_OBJECT" };
+    }
+    const rec = d;
+    const unknownFields = Object.keys(rec).filter((k) => !KNOWN_TOP_LEVEL.has(k));
+    const sessionId = resolveSessionId(rec, ctx);
+    const ts = typeof rec.timestamp === "string" ? rec.timestamp : null;
+    const cwd = typeof rec.cwd === "string" ? rec.cwd : null;
+    const message = isObject(rec.message) ? rec.message : null;
+    const isCompactSummary = rec.isCompactSummary === true;
+    const isApiErrorMessage = rec.isApiErrorMessage === true;
+    const isInterrupt = false;
+    const toolEvents = [];
+    const toolResults = [];
+    let command = extractCommand(rec, sessionId, ts);
+    // Tool blocks live in message.content arrays (assistant tool_use, user tool_result).
+    if (message && Array.isArray(message.content)) {
+        let idx = 0;
+        for (const block of message.content) {
+            if (!isObject(block)) {
+                idx++;
+                continue;
+            }
+            const b = block;
+            if (b.type === "tool_use") {
+                toolEvents.push(projectToolUse(b, sessionId, ts, idx, resolveMessageId(message, rec)));
+            }
+            else if (b.type === "tool_result") {
+                toolResults.push(projectToolResult(b, sessionId));
+            }
+            idx++;
+        }
+    }
+    const isUserTurn = rec.type === "user" &&
+        (rec.promptSource === "typed" || rec.promptSource === "queued") &&
+        toolResults.length === 0 &&
+        rec.isMeta !== true;
+    // A user record whose content is a bare slash command is also a hygiene marker.
+    if (command === null && rec.type === "user" && message && typeof message.content === "string") {
+        const c = message.content;
+        if (c === "/clear" || c === "/compact") {
+            command = { sessionId, ts: ts ?? "", command: c };
+        }
+    }
+    // Usage record → candidate turn.
+    const usage = message && isObject(message.usage) ? message.usage : null;
+    if (usage === null) {
+        return {
+            kind: "record",
+            sessionId,
+            turn: null,
+            toolEvents,
+            toolResults,
+            command,
+            synthetic: false,
+            isUserTurn,
+            isCompactSummary,
+            isApiErrorMessage,
+            isInterrupt,
+            ts,
+            unknownFields,
+            cwd,
+        };
+    }
+    const model = typeof message?.model === "string" ? message.model : null;
+    if (model === null || model === "" || model === "<synthetic>") {
+        // Synthetic / empty-model: excluded from cost, counted in health. Not a turn.
+        return {
+            kind: "record",
+            sessionId,
+            turn: null,
+            toolEvents,
+            toolResults,
+            command,
+            synthetic: true,
+            isUserTurn,
+            isCompactSummary,
+            isApiErrorMessage,
+            isInterrupt,
+            ts,
+            unknownFields,
+            cwd,
+        };
+    }
+    const dedupeId = typeof message?.id === "string" ? message.id : typeof rec.uuid === "string" ? rec.uuid : null;
+    if (dedupeId === null || ts === null) {
+        return { kind: "quarantine", errorClass: "MISSING_REQUIRED" };
+    }
+    const turn = projectTurn(dedupeId, sessionId, ts, model, rec, usage);
+    return {
+        kind: "record",
+        sessionId,
+        turn,
+        toolEvents,
+        toolResults,
+        command,
+        synthetic: false,
+        isUserTurn,
+        isCompactSummary,
+        isApiErrorMessage,
+        isInterrupt,
+        ts,
+        unknownFields,
+        cwd,
+    };
+}
+function projectTurn(messageId, sessionId, ts, model, rec, usage) {
+    const cacheCreation = isObject(usage.cache_creation)
+        ? usage.cache_creation
+        : null;
+    const c5 = num(cacheCreation?.ephemeral_5m_input_tokens);
+    const c1 = num(cacheCreation?.ephemeral_1h_input_tokens);
+    const cwTotal = num(usage.cache_creation_input_tokens);
+    const cwOther = Math.max(0, cwTotal - c5 - c1);
+    const outputTokensDetails = isObject(usage.output_tokens_details)
+        ? usage.output_tokens_details
+        : null;
+    const message = rec.message;
+    const effort = typeof message.effort === "string"
+        ? message.effort
+        : typeof rec.effort === "string"
+            ? rec.effort
+            : null;
+    return {
+        messageId,
+        sessionId,
+        ts,
+        model,
+        isSidechain: rec.isSidechain === true,
+        inputTokens: num(usage.input_tokens),
+        outputTokens: num(usage.output_tokens),
+        thinkingTokens: num(outputTokensDetails?.thinking_tokens, null),
+        cacheReadTokens: num(usage.cache_read_input_tokens),
+        cacheWrite5m: c5,
+        cacheWrite1h: c1,
+        cacheWriteOther: cwOther,
+        effort,
+    };
+}
+function projectToolUse(b, sessionId, ts, idx, ownerMessageId) {
+    const toolName = typeof b.name === "string" ? b.name : "";
+    const toolUseId = typeof b.id === "string" ? b.id : null;
+    const inputBytes = b.input != null ? JSON.stringify(b.input).length : null;
+    const inputHash = b.input != null ? hashSha256(canonicalJson(normalizeToolInputPaths(b.input))) : null;
+    const input = isObject(b.input) ? b.input : null;
+    const filePathHash = typeof input?.file_path === "string" ? hashSha256(normalizeFilePath(input.file_path)) : null;
+    let gitCommandHint = false;
+    let testCommandHint = false;
+    if (toolName === "Bash" && input !== null) {
+        const cmd = input.command;
+        if (typeof cmd === "string")
+            gitCommandHint = GIT_WRITE_RE.test(cmd);
+        if (typeof cmd === "string")
+            testCommandHint = isTestCommand(cmd);
+    }
+    const eventId = toolUseId ?? hashId(`${sessionId}|${ts ?? ""}|${toolName}|${idx}`);
+    return {
+        eventId,
+        toolUseId,
+        sessionId,
+        ts: ts ?? "",
+        toolName,
+        inputBytes,
+        inputHash,
+        filePathHash,
+        ownerMessageId,
+        blockIndex: idx,
+        gitCommandHint,
+        testCommandHint,
+    };
+}
+function projectToolResult(b, sessionId) {
+    const toolUseId = typeof b.tool_use_id === "string" ? b.tool_use_id : null;
+    let resultBytes = 0;
+    let commitSha = null;
+    const content = b.content;
+    if (typeof content === "string") {
+        resultBytes = content.length;
+        const m = SHA_RE.exec(content);
+        if (m)
+            commitSha = m[0];
+    }
+    else if (Array.isArray(content)) {
+        for (const c of content) {
+            if (isObject(c) && typeof c.text === "string") {
+                const text = c.text;
+                resultBytes += text.length;
+                if (commitSha === null) {
+                    const m = SHA_RE.exec(text);
+                    if (m)
+                        commitSha = m[0];
+                }
+            }
+        }
+    }
+    return { toolUseId, sessionId, resultBytes, commitSha, isError: b.is_error === true };
+}
+/** Deliberately anchored: prose and compound shell setup are not classified as tests. */
+export function isTestCommand(command) {
+    return TEST_COMMAND_RE.test(command.trim());
+}
+/** Stable JSON representation for privacy-safe near-duplicate hashes. */
+function canonicalJson(value) {
+    if (Array.isArray(value))
+        return `[${value.map((item) => canonicalJson(item ?? null)).join(",")}]`;
+    if (isObject(value)) {
+        return `{${Object.keys(value)
+            .filter((key) => value[key] !== undefined)
+            .sort()
+            .map((key) => `${JSON.stringify(key)}:${canonicalJson(value[key])}`)
+            .join(",")}}`;
+    }
+    return JSON.stringify(value) ?? "null";
+}
+/** Normalize path-bearing fields before hashing while retaining no raw projection. */
+function normalizeToolInputPaths(value) {
+    if (Array.isArray(value))
+        return value.map((item) => normalizeToolInputPaths(item));
+    if (!isObject(value))
+        return value;
+    const normalized = {};
+    for (const [key, item] of Object.entries(value)) {
+        normalized[key] =
+            key === "file_path" && typeof item === "string"
+                ? normalizeFilePath(item)
+                : normalizeToolInputPaths(item);
+    }
+    return normalized;
+}
+/** Lexically normalize separators and dot segments without retaining the path. */
+function normalizeFilePath(filePath) {
+    const slashPath = filePath.replace(/\\/g, "/");
+    const prefix = slashPath.startsWith("/") ? "/" : "";
+    const parts = [];
+    for (const part of slashPath.split("/")) {
+        if (part === "" || part === ".")
+            continue;
+        const atDriveRoot = parts.length === 1 && /^[A-Za-z]:$/.test(parts[0] ?? "");
+        if (part === ".." && parts.length > 0 && parts.at(-1) !== ".." && !atDriveRoot) {
+            parts.pop();
+        }
+        else if (part !== ".." || (prefix === "" && !atDriveRoot)) {
+            parts.push(part);
+        }
+    }
+    if (/^[A-Za-z]:$/.test(parts[0] ?? ""))
+        parts[0] = parts[0]?.toLowerCase() ?? "";
+    return `${prefix}${parts.join("/")}` || prefix;
+}
+function hashSha256(value) {
+    return crypto.createHash("sha256").update(value).digest("hex");
+}
+function resolveMessageId(message, rec) {
+    if (typeof message.id === "string")
+        return message.id;
+    return typeof rec.uuid === "string" ? rec.uuid : null;
+}
+function extractCommand(rec, sessionId, ts) {
+    if (rec.type === "system" &&
+        (rec.subtype === "local_command" || rec.subtype === "away_summary")) {
+        const raw = typeof rec.command === "string" ? rec.command : String(rec.subtype);
+        return { sessionId, ts: ts ?? "", command: raw };
+    }
+    return null;
+}
+function resolveSessionId(rec, ctx) {
+    // Prefer session_id, then sessionId, else the filename stem.
+    if (typeof rec.session_id === "string" && rec.session_id.length > 0)
+        return rec.session_id;
+    if (typeof rec.sessionId === "string" && rec.sessionId.length > 0)
+        return rec.sessionId;
+    return ctx.defaultSessionId;
+}
+function classifyJsonError(e) {
+    const msg = e instanceof Error ? e.message : String(e);
+    return `JSON_${msg.slice(0, 24).replace(/[^\w]/g, "_")}`.slice(0, 40);
+}
+function isObject(v) {
+    return typeof v === "object" && v !== null && !Array.isArray(v);
+}
+function num(v, fallback = 0) {
+    return typeof v === "number" && Number.isFinite(v) ? v : fallback;
+}
+function hashId(input) {
+    return crypto.createHash("sha1").update(input).digest("hex").slice(0, 24);
+}

package/dist/ingest/pricing.js

@@ -0,0 +1,153 @@
+/**
+ * src/ingest/pricing.ts — PricingSnapshotStore + per-turn pricing.
+ *
+ * Prices are stored as a snapshot per model tier in `pricing_snapshots`.
+ * unit_prices_json = [input, output, cacheRead, cacheWrite5m, cacheWrite1h] in
+ * $/MTok, which is numerically μUSD/token — so cost_equiv_u (integer micro-USD)
+ * is a direct token×price sum, no scaling. This matches the proven reference
+ * scripts (token-usage-analyze.mjs / s1-all.mjs) exactly.
+ *
+ * Pricing rules (Ingestion Spec §1.2):
+ *   - price each turn against the freshest NON-STALE snapshot for its tier;
+ *   - no snapshot for the tier ⇒ cost_equiv_u = NULL (tokens still counted);
+ *   - only a stale snapshot available ⇒ cost computed, cost_claim = LIST_EQUIV_STALE.
+ *
+ * PRICE SOURCE (BM0, re-fetched 2026-09-03 from platform.claude.com models
+ * overview — the authoritative Anthropic docs): current per-model list prices.
+ *   - Base $/MTok: opus 5/4.8 $5/$25 · sonnet-5 $2/$10 · sonnet-4.x $3/$15 ·
+ *     haiku-4.5 $1/$5 · fable 5/5.1 $10/$50.
+ *   - Cache multipliers (all families): 5m write 1.25×, 1h write 2.0×, read 0.1×
+ *     of base input — EXCEPT the Fable/Mythos 5.1 family, whose cache read is
+ *     0.025× (docs footnote), giving the split fable-5 vs fable-5-1 tiers.
+ * Tiers are per generation only where the corpus carries both AND prices differ:
+ *   - opus stays a single tier (4.8 and 5 are priced identically → no split);
+ *   - sonnet splits sonnet-5 vs sonnet-4 ($2 vs $3 base);
+ *   - fable splits fable-5-1 vs fable-5 (0.025× vs 0.1× cache read).
+ * Every haiku model maps to the single `haiku` tier (only 4.5 in the corpus).
+ * Re-pricing bumps snapshot ids to list-<tier>-v2; historical turns keep their
+ * old snapshot provenance until the next full re-scan (RV2a backfill stance).
+ */
+/**
+ * Canonical list-price table ($/MTok = μUSD/token), one row per model tier.
+ * [input, output, cacheRead, cacheWrite5m, cacheWrite1h].
+ */
+export const LIST_PRICES = {
+    opus: [5, 25, 0.5, 6.25, 10],
+    "sonnet-5": [2, 10, 0.2, 2.5, 4],
+    "sonnet-4": [3, 15, 0.3, 3.75, 6],
+    haiku: [1, 5, 0.1, 1.25, 2],
+    "fable-5": [10, 50, 1, 12.5, 20],
+    "fable-5-1": [10, 50, 0.25, 12.5, 20], // 0.025× cache read (5.1 family)
+};
+/** Default snapshot staleness horizon (days) when seeding list prices. */
+const DEFAULT_STALE_DAYS = 30;
+/**
+ * Resolve a model string to a pricing tier, or null if unpriceable.
+ * Generation-sensitive families (sonnet, fable) match the more specific
+ * generation first; opus and haiku are single-tier (see LIST_PRICES doc).
+ */
+export function modelTier(model) {
+    if (!model)
+        return null;
+    if (model.includes("opus"))
+        return "opus";
+    if (model.includes("sonnet"))
+        return model.includes("sonnet-5") ? "sonnet-5" : "sonnet-4";
+    if (model.includes("haiku"))
+        return "haiku";
+    // Fable and Mythos share the same per-token pricing per generation; the 5.1
+    // family (fable-5-1 / mythos-5-1) has the 0.025× cache-read exception.
+    if (model.includes("fable") || model.includes("mythos")) {
+        return model.includes("fable-5-1") || model.includes("mythos-5-1") ? "fable-5-1" : "fable-5";
+    }
+    return null;
+}
+function addDaysIso(iso, days) {
+    const d = new Date(iso);
+    d.setUTCDate(d.getUTCDate() + days);
+    return d.toISOString();
+}
+/**
+ * Seed `pricing_snapshots` with the canonical list-price table (one snapshot per
+ * tier) if not already present. Deterministic snapshot ids keep re-seeding and
+ * rebuild-equality idempotent. Returns the number of snapshots inserted.
+ */
+export function seedListPrices(db, capturedAtIso) {
+    const insert = db.prepare(`INSERT OR IGNORE INTO pricing_snapshots
+       (snapshot_id, model_tier, unit_prices_json, captured_at, stale_after)
+     VALUES (?,?,?,?,?)`);
+    const staleAfter = addDaysIso(capturedAtIso, DEFAULT_STALE_DAYS);
+    let inserted = 0;
+    for (const [tier, prices] of Object.entries(LIST_PRICES)) {
+        const res = insert.run(`list-${tier}-v2`, tier, JSON.stringify(prices), capturedAtIso, staleAfter);
+        inserted += res.changes;
+    }
+    return inserted;
+}
+/**
+ * Prices turns against the snapshots present in the DB at construction time.
+ * Load once, price many; the `now` reference fixes staleness for the batch.
+ */
+export class PricingSnapshotStore {
+    nowIso;
+    /** tier → chosen snapshot (freshest non-stale, else freshest stale). */
+    byTier = new Map();
+    constructor(db, nowIso = new Date().toISOString()) {
+        this.nowIso = nowIso;
+        const rows = db
+            .prepare(`SELECT snapshot_id, model_tier, unit_prices_json, stale_after
+         FROM pricing_snapshots`)
+            .all();
+        // Group by tier, then choose: prefer non-stale with max captured proxy.
+        // We select the freshest by stale_after (later horizon ⇒ newer capture).
+        const grouped = new Map();
+        for (const r of rows) {
+            const list = grouped.get(r.model_tier) ?? [];
+            list.push(r);
+            grouped.set(r.model_tier, list);
+        }
+        for (const [tier, list] of grouped) {
+            const nonStale = list.filter((r) => r.stale_after > this.nowIso);
+            const pool = nonStale.length > 0 ? nonStale : list;
+            // Freshest = latest stale_after horizon.
+            let chosen = pool[0];
+            if (chosen === undefined)
+                continue;
+            for (const r of pool) {
+                if (r.stale_after > chosen.stale_after)
+                    chosen = r;
+            }
+            this.byTier.set(tier, {
+                id: chosen.snapshot_id,
+                prices: parsePrices(chosen.unit_prices_json),
+                stale: nonStale.length === 0,
+            });
+        }
+    }
+    /** Price a single turn. Returns cost in micro-USD, the snapshot used, and the claim. */
+    price(model, tokens) {
+        const tier = modelTier(model);
+        if (tier === null) {
+            return { costU: null, snapshotId: null, claim: "LIST_EQUIV" };
+        }
+        const snap = this.byTier.get(tier);
+        if (snap === undefined) {
+            return { costU: null, snapshotId: null, claim: "LIST_EQUIV" };
+        }
+        const [pin, pout, pcr, pcw5, pcw1] = snap.prices;
+        const costU = Math.round(tokens.inputTokens * pin +
+            tokens.outputTokens * pout +
+            tokens.cacheReadTokens * pcr +
+            (tokens.cacheWrite5m + tokens.cacheWriteOther) * pcw5 +
+            tokens.cacheWrite1h * pcw1);
+        return {
+            costU,
+            snapshotId: snap.id,
+            claim: snap.stale ? "LIST_EQUIV_STALE" : "LIST_EQUIV",
+        };
+    }
+}
+function parsePrices(json) {
+    const arr = JSON.parse(json);
+    return [arr[0] ?? 0, arr[1] ?? 0, arr[2] ?? 0, arr[3] ?? 0, arr[4] ?? 0];
+}

package/dist/ingest/reconcile.js

@@ -0,0 +1,52 @@
+/**
+ * src/ingest/reconcile.ts — LIVE → RECONCILED session lifecycle.
+ *
+ * A session is LIVE while its last_turn_at is within the activity window of the
+ * reference time; otherwise it closes to RECONCILED. On close, provisional turn
+ * flags are cleared and hygiene flags are evaluated from settled DB state
+ * (deterministic, so rebuild-equality holds). Turns are inserted provisional=1
+ * by the ingestor; this pass is the sole authority that clears them.
+ *
+ * sessions.cost_equiv_u rollforward is maintained incrementally by the ingestor
+ * (sum of each session's turn costs) and is unaffected by reconciliation — the
+ * running total is the final total once the session settles.
+ */
+export const DEFAULT_RECONCILE_OPTIONS = {
+    longContextThreshold: 150_000,
+    longContextTurnCount: 20,
+};
+/**
+ * Reconcile every session against `cutoffIso` (= referenceTime − activityWindow).
+ * Sessions with last_turn_at ≥ cutoff stay LIVE (provisional); older sessions
+ * close to RECONCILED with provisional cleared and hygiene flags evaluated.
+ * Idempotent: re-running with the same cutoff yields the same state.
+ */
+export function reconcileSessions(db, cutoffIso, opts = DEFAULT_RECONCILE_OPTIONS) {
+    const sessions = db
+        .prepare("SELECT session_id, last_turn_at, turn_count FROM sessions WHERE state != 'RECONCILED' OR last_turn_at >= :cutoff")
+        .all({ cutoff: cutoffIso });
+    const setProvisional = db.prepare("UPDATE turns SET provisional = ? WHERE session_id = ?");
+    const setSession = db.prepare("UPDATE sessions SET state = ?, hygiene_flags = ? WHERE session_id = ?");
+    const countLong = db.prepare("SELECT COUNT(*) AS n FROM turns WHERE session_id = ? AND context_tokens > ?");
+    const countCompact = db.prepare("SELECT COUNT(*) AS n FROM tool_events WHERE session_id = ? AND tool_name = 'local_command' AND input_hash = '/compact'");
+    const tx = db.transaction(() => {
+        for (const s of sessions) {
+            const live = s.last_turn_at !== null && s.last_turn_at >= cutoffIso;
+            if (live) {
+                setProvisional.run(1, s.session_id);
+                setSession.run("LIVE", "[]", s.session_id);
+                continue;
+            }
+            setProvisional.run(0, s.session_id);
+            const flags = [];
+            const longN = countLong.get(s.session_id, opts.longContextThreshold).n;
+            if (longN > opts.longContextTurnCount)
+                flags.push("LONG_FULL_CONTEXT");
+            const compactN = countCompact.get(s.session_id).n;
+            if (compactN > 0 && s.turn_count > 1)
+                flags.push("COMPACT_MID_TASK");
+            setSession.run("RECONCILED", JSON.stringify(flags), s.session_id);
+        }
+    });
+    tx();
+}