@unerr-ai/unerr 0.2.1 → 0.2.2

package/dist/tracking/token-flow.js

@@ -1,276 +0,0 @@
-/**
- * Token Flow — unified observability + attribution for token savings.
- *
- * Layer 10: Every token that flows through unerr gets attributed to the mechanism
- * that handled it, aggregated across hierarchical scopes (operation → turn →
- * session → lifetime), and persisted to `.unerr/metrics.db` (token_flow_events
- * table).
- *
- * This replaces the fragmented tracking across EfficiencyTracker, ExplorationCost,
- * IntentTokenTracker, SessionStats, CompressionLog, and WeeklyAccumulator with a
- * single event-sourced model.
- *
- * Performance contract:
- *   - Token estimation: <0.01ms (delegated to token-estimator.ts)
- *   - SQLite insert: <0.05ms (better-sqlite3, prepared statement, WAL)
- *   - Total hot-path overhead: <0.2ms per tool call
- */
-import { openMetricsStore } from "./metrics-store.js";
-// ── Writer ──────────────────────────────────────────────────────────
-/** RC-5: Maximum in-memory session events before eviction. */
-const MAX_SESSION_EVENTS = 5000;
-function rowToEvent(r) {
-    let detail;
-    if (r.detail) {
-        try {
-            detail = JSON.parse(r.detail);
-        }
-        catch {
-            // corrupt payload — return the row without detail rather than throwing
-        }
-    }
-    return {
-        id: r.id,
-        ts: r.ts_iso,
-        session_id: r.session_id,
-        pid: r.pid,
-        turn: r.turn,
-        mechanism: r.mechanism,
-        tool: r.tool ?? null,
-        tokens_without: r.tokens_without,
-        tokens_with: r.tokens_with,
-        tokens_saved: r.tokens_saved,
-        detail,
-    };
-}
-export class TokenFlowWriter {
-    sessionId;
-    unerrDir;
-    /** In-memory buffer of this process's events for fast aggregation + SSE relay. */
-    sessionEvents = [];
-    constructor(unerrDir, sessionId) {
-        this.sessionId = sessionId;
-        this.unerrDir = unerrDir;
-        // Eager-open the store so first record() doesn't pay the bootstrap cost.
-        openMetricsStore(unerrDir);
-    }
-    /**
-     * Record a token flow event. Synchronous, <0.1ms.
-     * Inserts into `.unerr/metrics.db` (token_flow_events) and keeps an
-     * in-memory copy for session aggregation + SSE relay.
-     */
-    record(input) {
-        const now = new Date();
-        const tsIso = now.toISOString();
-        let rowId = 0;
-        try {
-            rowId = openMetricsStore(this.unerrDir).insertTokenFlow({
-                ts: now.getTime(),
-                ts_iso: tsIso,
-                session_id: input.session_id,
-                pid: process.pid,
-                turn: input.turn,
-                mechanism: input.mechanism,
-                tool: input.tool,
-                tokens_without: input.tokens_without,
-                tokens_with: input.tokens_with,
-                tokens_saved: input.tokens_saved,
-                detail: input.detail ? JSON.stringify(input.detail) : null,
-            });
-        }
-        catch {
-            /* best effort — never block the hot path */
-        }
-        const event = {
-            id: rowId,
-            ts: tsIso,
-            pid: process.pid,
-            ...input,
-        };
-        this.sessionEvents.push(event);
-        // RC-5: Evict oldest events if buffer exceeds limit
-        if (this.sessionEvents.length > MAX_SESSION_EVENTS) {
-            this.sessionEvents = this.sessionEvents.slice(-Math.floor(MAX_SESSION_EVENTS * 0.75));
-        }
-    }
-    /** Get all events this process has recorded for the current session. */
-    getSessionEvents() {
-        return this.sessionEvents;
-    }
-    /** Running total of tokens saved this session (in-memory, O(1) amortized). */
-    getSessionTokensSaved() {
-        let total = 0;
-        for (const e of this.sessionEvents)
-            total += e.tokens_saved;
-        return total;
-    }
-    /** Running session efficiency percentage. */
-    getSessionEfficiency() {
-        let totalWithout = 0;
-        let totalSaved = 0;
-        for (const e of this.sessionEvents) {
-            totalWithout += e.tokens_without;
-            totalSaved += e.tokens_saved;
-        }
-        if (totalWithout === 0)
-            return 0;
-        return Math.round((totalSaved / totalWithout) * 100);
-    }
-    /**
-     * Ingest an event produced by another process (relayed via the log-tailer's
-     * SQLite poll). Pushes to in-memory buffer for SSE streaming; does NOT
-     * re-insert into the DB (the originating process already did).
-     */
-    ingestExternal(event) {
-        if (event.pid === process.pid)
-            return;
-        this.sessionEvents.push(event);
-        if (this.sessionEvents.length > MAX_SESSION_EVENTS) {
-            this.sessionEvents = this.sessionEvents.slice(-Math.floor(MAX_SESSION_EVENTS * 0.75));
-        }
-    }
-}
-/**
- * Read token flow events from the SQLite store with optional filtering.
- * Used for CLI status, dashboard, and session summaries.
- * Performance: <1ms for 1000 events (indexed lookup + row mapping).
- */
-export function readTokenFlowEvents(unerrDir, filter) {
-    let rows;
-    try {
-        const store = openMetricsStore(unerrDir);
-        rows = filter?.session_id
-            ? store.tokenFlowBySession(filter.session_id)
-            : store.allTokenFlow();
-    }
-    catch {
-        return [];
-    }
-    const events = [];
-    for (const r of rows) {
-        if (filter?.mechanism && r.mechanism !== filter.mechanism)
-            continue;
-        if (filter?.from_ts && r.ts_iso < filter.from_ts)
-            continue;
-        if (filter?.to_ts && r.ts_iso > filter.to_ts)
-            continue;
-        events.push(rowToEvent(r));
-    }
-    return events;
-}
-// ── Aggregation ─────────────────────────────────────────────────────
-/**
- * Aggregate events into a session-level summary.
- * Computed on-read from the event stream (event sourcing principle).
- */
-export function aggregateSession(events, sessionId) {
-    const sessionEvents = events.filter((e) => e.session_id === sessionId);
-    const mechanismMap = new Map();
-    const turnMap = new Map();
-    let totalWithout = 0;
-    let totalWith = 0;
-    let totalSaved = 0;
-    const turnNumbers = new Set();
-    for (const e of sessionEvents) {
-        totalWithout += e.tokens_without;
-        totalWith += e.tokens_with;
-        totalSaved += e.tokens_saved;
-        turnNumbers.add(e.turn);
-        // Per-mechanism accumulation
-        const mech = mechanismMap.get(e.mechanism) ?? {
-            saved: 0,
-            delivered: 0,
-            count: 0,
-        };
-        mech.saved += e.tokens_saved;
-        mech.delivered += e.tokens_with;
-        mech.count++;
-        mechanismMap.set(e.mechanism, mech);
-        // Per-turn accumulation
-        const turn = turnMap.get(e.turn) ?? {
-            tool: e.tool ?? "unknown",
-            tokens_without: 0,
-            tokens_delivered: 0,
-            tokens_saved: 0,
-            mechanisms: new Map(),
-        };
-        turn.tokens_without += e.tokens_without;
-        turn.tokens_delivered += e.tokens_with;
-        turn.tokens_saved += e.tokens_saved;
-        turn.mechanisms.set(e.mechanism, (turn.mechanisms.get(e.mechanism) ?? 0) + e.tokens_saved);
-        if (e.tool)
-            turn.tool = e.tool;
-        turnMap.set(e.turn, turn);
-    }
-    // Build mechanism summary with percentage of total
-    const byMechanism = {};
-    for (const [mechanism, data] of mechanismMap) {
-        byMechanism[mechanism] = {
-            tokens_saved: data.saved,
-            tokens_delivered: data.delivered,
-            event_count: data.count,
-            pct_of_total: totalSaved > 0 ? Math.round((data.saved / totalSaved) * 1000) / 10 : 0,
-        };
-    }
-    // Build top turns sorted by tokens_saved descending, take top 5
-    const topTurns = [...turnMap.entries()]
-        .map(([turn, data]) => {
-        let primaryMechanism = "unknown";
-        let maxSaved = 0;
-        for (const [mech, saved] of data.mechanisms) {
-            if (saved > maxSaved) {
-                maxSaved = saved;
-                primaryMechanism = mech;
-            }
-        }
-        return {
-            turn,
-            tool: data.tool,
-            tokens_without: data.tokens_without,
-            tokens_delivered: data.tokens_delivered,
-            tokens_saved: data.tokens_saved,
-            primary_mechanism: primaryMechanism,
-        };
-    })
-        .sort((a, b) => b.tokens_saved - a.tokens_saved)
-        .slice(0, 5);
-    return {
-        session_id: sessionId,
-        total_turns: turnNumbers.size,
-        total_tokens_without: totalWithout,
-        total_tokens_with: totalWith,
-        total_tokens_saved: totalSaved,
-        efficiency_pct: totalWithout > 0 ? Math.round((totalSaved / totalWithout) * 100) : 0,
-        by_mechanism: byMechanism,
-        top_turns: topTurns,
-    };
-}
-/**
- * Aggregate events by mechanism across all sessions (for cross-session trends).
- */
-export function aggregateByMechanism(events) {
-    let totalSaved = 0;
-    const mechanismMap = new Map();
-    for (const e of events) {
-        totalSaved += e.tokens_saved;
-        const mech = mechanismMap.get(e.mechanism) ?? {
-            saved: 0,
-            delivered: 0,
-            count: 0,
-        };
-        mech.saved += e.tokens_saved;
-        mech.delivered += e.tokens_with;
-        mech.count++;
-        mechanismMap.set(e.mechanism, mech);
-    }
-    const result = {};
-    for (const [mechanism, data] of mechanismMap) {
-        result[mechanism] = {
-            tokens_saved: data.saved,
-            tokens_delivered: data.delivered,
-            event_count: data.count,
-            pct_of_total: totalSaved > 0 ? Math.round((data.saved / totalSaved) * 1000) / 10 : 0,
-        };
-    }
-    return result;
-}

package/dist/tracking/turn-segmenter.js

@@ -1,122 +0,0 @@
-/**
- * Turn Segmenter — assigns ledger entries to logical "turns" (one user→agent
- * round-trip). Embedded inside ShadowLedger.record(); does not write or read
- * files. Pure in-memory state, isolated per session.
- *
- * Boundary detection (in priority order):
- *  1. `closeTurn(sessionId)` — called by Stop hooks (Claude Code) or session
- *     shutdown. Anchors an exact boundary. Next observed entry opens a new
- *     turn with confidence "first_call".
- *  2. Idle gap > IDLE_GAP_MS between consecutive entries on the same session
- *     closes the previous turn and opens a new one with confidence "idle_gap".
- *  3. The very first entry on a session opens turn 1 with confidence
- *     "first_call".
- *
- * Subscribers get notified on every turn close via onTurnClose(listener).
- * The timeline subsystem uses this to upsert turn rollups into timeline.db
- * — but the segmenter itself has no dependency on that subsystem and is safe
- * to use standalone (e.g., in tests, or when UNERR_TIMELINE_V2=0).
- */
-import { randomBytes } from "node:crypto";
-const DEFAULT_IDLE_GAP_MS = 20_000;
-export class TurnSegmenter {
-    state = new Map();
-    listeners = [];
-    idleGapMs;
-    constructor(options = {}) {
-        this.idleGapMs = options.idleGapMs ?? DEFAULT_IDLE_GAP_MS;
-    }
-    /**
-     * Subscribe to turn-close events. Returns an unsubscribe function.
-     */
-    onTurnClose(listener) {
-        this.listeners.push(listener);
-        return () => {
-            const idx = this.listeners.indexOf(listener);
-            if (idx >= 0)
-                this.listeners.splice(idx, 1);
-        };
-    }
-    /**
-     * Observe an entry and stamp it with turn_id + turn_confidence in place.
-     * May emit a close event for the previous turn if an idle gap is detected.
-     */
-    observe(entry) {
-        const ts = Date.parse(entry.ts);
-        const s = this.state.get(entry.session_id) ?? this.initState();
-        if (s.currentTurnId === null) {
-            // Opening a new turn — openedBy is whatever the prior closeTurn or
-            // initState set ("first_call" for the very first or after stop_hook).
-            s.currentTurnId = generateTurnId();
-        }
-        else if (Number.isFinite(ts) &&
-            ts - s.lastEntryTs > this.idleGapMs &&
-            s.lastEntryTs > 0) {
-            this.emitClose({
-                turn_id: s.currentTurnId,
-                session_id: entry.session_id,
-                closed_at: s.lastEntryTs,
-                reason: "idle_gap",
-            });
-            s.currentTurnId = generateTurnId();
-            s.openedBy = "idle_gap";
-        }
-        entry.turn_id = s.currentTurnId;
-        entry.turn_confidence = s.openedBy;
-        if (Number.isFinite(ts))
-            s.lastEntryTs = ts;
-        this.state.set(entry.session_id, s);
-    }
-    /**
-     * Anchor an exact turn boundary. The next observed entry will open a new
-     * turn with confidence "first_call" (the boundary is known precisely).
-     * Idempotent — calling with no active turn is a no-op.
-     */
-    closeTurn(sessionId, reason = "stop_hook") {
-        const s = this.state.get(sessionId);
-        if (!s || s.currentTurnId === null)
-            return;
-        this.emitClose({
-            turn_id: s.currentTurnId,
-            session_id: sessionId,
-            closed_at: s.lastEntryTs,
-            reason,
-        });
-        s.currentTurnId = null;
-        s.openedBy = "first_call";
-        this.state.set(sessionId, s);
-    }
-    /**
-     * Inspect the current turn id for a session. Returns null if no turn is open.
-     * Intended for diagnostics / tests.
-     */
-    getCurrentTurnId(sessionId) {
-        return this.state.get(sessionId)?.currentTurnId ?? null;
-    }
-    /**
-     * Drop all state. For tests and clean shutdown.
-     */
-    reset() {
-        this.state.clear();
-    }
-    initState() {
-        return {
-            currentTurnId: null,
-            lastEntryTs: 0,
-            openedBy: "first_call",
-        };
-    }
-    emitClose(event) {
-        for (const l of this.listeners) {
-            try {
-                l(event);
-            }
-            catch (err) {
-                process.stderr.write(`[unerr:turn-segmenter] WARN: listener error: ${err instanceof Error ? err.message : String(err)}\n`);
-            }
-        }
-    }
-}
-function generateTurnId() {
-    return randomBytes(6).toString("hex");
-}

package/dist/tracking/weekly-accumulator.js

@@ -1,179 +0,0 @@
-/**
- * Weekly Summary Accumulator — unified cross-session stats persistence.
- *
- * S8.4: Persists session metrics to ~/.unerr/stats.json.
- * Replaces the separate cumulative-stats.json and cumulative-local-stats.json.
- *
- * Features:
- *   - Weekly auto-reset (Monday boundary)
- *   - All-time counters that persist across resets
- *   - Feeds into `unerr stats` command and session guard
- */
-function getWeekStart() {
-    const now = new Date();
-    const day = now.getDay();
-    const diff = now.getDate() - day + (day === 0 ? -6 : 1);
-    const monday = new Date(now);
-    monday.setDate(diff);
-    return monday.toISOString().slice(0, 10);
-}
-function getStatsPath() {
-    const home = process.env.HOME ?? process.env.USERPROFILE ?? process.cwd();
-    return `${home}/.unerr/stats.json`;
-}
-function createEmptyWeekly(weekStart) {
-    return {
-        weekStart,
-        sessions: 0,
-        tokensSaved: 0,
-        dollarsSaved: 0,
-        toolCalls: 0,
-        violationsCaught: 0,
-        chokepointWarnings: 0,
-        correctionsApplied: 0,
-        blastRadiusComputed: 0,
-        avgEfficiency: 0,
-        avgLatencyP50: 0,
-    };
-}
-function createEmptyAllTime() {
-    return {
-        firstSessionDate: new Date().toISOString(),
-        totalSessions: 0,
-        totalTokensSaved: 0,
-        totalDollarsSaved: 0,
-        totalViolationsCaught: 0,
-    };
-}
-export function loadStats() {
-    const currentWeek = getWeekStart();
-    try {
-        const { readFileSync } = require("node:fs");
-        const raw = JSON.parse(readFileSync(getStatsPath(), "utf-8"));
-        if (raw.version !== 1) {
-            return {
-                version: 1,
-                weekly: createEmptyWeekly(currentWeek),
-                allTime: createEmptyAllTime(),
-                lastUpdated: new Date().toISOString(),
-            };
-        }
-        // Reset weekly if new week, preserve all-time
-        if (raw.weekly.weekStart !== currentWeek) {
-            raw.weekly = createEmptyWeekly(currentWeek);
-        }
-        return raw;
-    }
-    catch {
-        return {
-            version: 1,
-            weekly: createEmptyWeekly(currentWeek),
-            allTime: createEmptyAllTime(),
-            lastUpdated: new Date().toISOString(),
-        };
-    }
-}
-/**
- * Accumulate a session's metrics into the unified stats file.
- * Returns the updated stats for display.
- */
-export function accumulateSession(input) {
-    const stats = loadStats();
-    // Weekly
-    stats.weekly.sessions += 1;
-    stats.weekly.tokensSaved += input.tokensSaved;
-    stats.weekly.dollarsSaved += input.dollarsSaved;
-    stats.weekly.toolCalls += input.toolCalls;
-    stats.weekly.violationsCaught += input.violationsCaught;
-    stats.weekly.chokepointWarnings += input.chokepointWarnings;
-    stats.weekly.correctionsApplied += input.correctionsApplied;
-    stats.weekly.blastRadiusComputed += input.blastRadiusComputed;
-    // Rolling average for efficiency and latency
-    const n = stats.weekly.sessions;
-    stats.weekly.avgEfficiency =
-        stats.weekly.avgEfficiency * ((n - 1) / n) + input.efficiency * (1 / n);
-    stats.weekly.avgLatencyP50 =
-        stats.weekly.avgLatencyP50 * ((n - 1) / n) + input.latencyP50 * (1 / n);
-    // Layer 10: Accumulate per-mechanism token savings
-    if (input.tokensByMechanism) {
-        if (!stats.weekly.tokensByMechanism)
-            stats.weekly.tokensByMechanism = {};
-        for (const [mech, saved] of Object.entries(input.tokensByMechanism)) {
-            stats.weekly.tokensByMechanism[mech] =
-                (stats.weekly.tokensByMechanism[mech] ?? 0) + saved;
-        }
-    }
-    // All-time
-    stats.allTime.totalSessions += 1;
-    stats.allTime.totalTokensSaved += input.tokensSaved;
-    stats.allTime.totalDollarsSaved += input.dollarsSaved;
-    stats.allTime.totalViolationsCaught += input.violationsCaught;
-    stats.lastUpdated = new Date().toISOString();
-    persistStats(stats);
-    return stats;
-}
-function persistStats(stats) {
-    try {
-        const fs = require("node:fs");
-        const path = require("node:path");
-        const filePath = getStatsPath();
-        fs.mkdirSync(path.dirname(filePath), { recursive: true });
-        fs.writeFileSync(filePath, JSON.stringify(stats, null, 2));
-    }
-    catch {
-        // Non-critical — don't break session shutdown
-    }
-}
-/**
- * Format stats for the `unerr stats` command output.
- */
-export function formatStatsReport(stats) {
-    const w = stats.weekly;
-    const a = stats.allTime;
-    const lines = [
-        "",
-        "── unerr stats ────────────────────────────────",
-        "",
-        "  This week:",
-        `    Sessions:       ${w.sessions}`,
-        `    Tool calls:     ${w.toolCalls}`,
-        `    Tokens saved:   ~${formatTokenCount(w.tokensSaved)}`,
-        `    Dollar savings: $${w.dollarsSaved.toFixed(2)}`,
-        `    Efficiency:     ${Math.round(w.avgEfficiency)}%`,
-        `    Violations:     ${w.violationsCaught} caught`,
-    ];
-    if (w.blastRadiusComputed > 0) {
-        lines.push(`    Blast radius:   ${w.blastRadiusComputed} computations`);
-    }
-    // Layer 10: Mechanism breakdown
-    if (w.tokensByMechanism && Object.keys(w.tokensByMechanism).length > 0) {
-        lines.push("");
-        lines.push("  By mechanism:");
-        const sorted = Object.entries(w.tokensByMechanism).sort(([, a], [, b]) => b - a);
-        const totalMech = sorted.reduce((s, [, v]) => s + v, 0);
-        for (const [mech, saved] of sorted) {
-            const pct = totalMech > 0 ? Math.round((saved / totalMech) * 100) : 0;
-            const barLen = Math.max(1, Math.round(pct / 5));
-            const bar = "█".repeat(barLen) + "░".repeat(20 - barLen);
-            lines.push(`    ${mech.padEnd(22)} ${formatTokenCount(saved).padStart(6)}  ${String(pct).padStart(3)}%  ${bar}`);
-        }
-    }
-    lines.push("");
-    lines.push("  All time:");
-    lines.push(`    Sessions:       ${a.totalSessions}`);
-    lines.push(`    Tokens saved:   ~${formatTokenCount(a.totalTokensSaved)}`);
-    lines.push(`    Dollar savings: $${a.totalDollarsSaved.toFixed(2)}`);
-    lines.push(`    Violations:     ${a.totalViolationsCaught} caught`);
-    lines.push(`    Since:          ${a.firstSessionDate.slice(0, 10)}`);
-    lines.push("");
-    lines.push("───────────────────────────────────────────────");
-    lines.push("");
-    return lines.join("\n");
-}
-function formatTokenCount(n) {
-    if (n >= 1_000_000)
-        return `${(n / 1_000_000).toFixed(1)}M`;
-    if (n >= 1_000)
-        return `${(n / 1_000).toFixed(1)}K`;
-    return String(n);
-}