@unerr-ai/unerr 0.0.1 → 0.1.1

package/dist/tracking/token-flow.js

@@ -0,0 +1,276 @@
+/**
+ * Token Flow — unified observability + attribution for token savings.
+ *
+ * Layer 10: Every token that flows through unerr gets attributed to the mechanism
+ * that handled it, aggregated across hierarchical scopes (operation → turn →
+ * session → lifetime), and persisted to `.unerr/metrics.db` (token_flow_events
+ * table).
+ *
+ * This replaces the fragmented tracking across EfficiencyTracker, ExplorationCost,
+ * IntentTokenTracker, SessionStats, CompressionLog, and WeeklyAccumulator with a
+ * single event-sourced model.
+ *
+ * Performance contract:
+ *   - Token estimation: <0.01ms (delegated to token-estimator.ts)
+ *   - SQLite insert: <0.05ms (better-sqlite3, prepared statement, WAL)
+ *   - Total hot-path overhead: <0.2ms per tool call
+ */
+import { openMetricsStore } from "./metrics-store.js";
+// ── Writer ──────────────────────────────────────────────────────────
+/** RC-5: Maximum in-memory session events before eviction. */
+const MAX_SESSION_EVENTS = 5000;
+function rowToEvent(r) {
+    let detail;
+    if (r.detail) {
+        try {
+            detail = JSON.parse(r.detail);
+        }
+        catch {
+            // corrupt payload — return the row without detail rather than throwing
+        }
+    }
+    return {
+        id: r.id,
+        ts: r.ts_iso,
+        session_id: r.session_id,
+        pid: r.pid,
+        turn: r.turn,
+        mechanism: r.mechanism,
+        tool: r.tool ?? null,
+        tokens_without: r.tokens_without,
+        tokens_with: r.tokens_with,
+        tokens_saved: r.tokens_saved,
+        detail,
+    };
+}
+export class TokenFlowWriter {
+    sessionId;
+    unerrDir;
+    /** In-memory buffer of this process's events for fast aggregation + SSE relay. */
+    sessionEvents = [];
+    constructor(unerrDir, sessionId) {
+        this.sessionId = sessionId;
+        this.unerrDir = unerrDir;
+        // Eager-open the store so first record() doesn't pay the bootstrap cost.
+        openMetricsStore(unerrDir);
+    }
+    /**
+     * Record a token flow event. Synchronous, <0.1ms.
+     * Inserts into `.unerr/metrics.db` (token_flow_events) and keeps an
+     * in-memory copy for session aggregation + SSE relay.
+     */
+    record(input) {
+        const now = new Date();
+        const tsIso = now.toISOString();
+        let rowId = 0;
+        try {
+            rowId = openMetricsStore(this.unerrDir).insertTokenFlow({
+                ts: now.getTime(),
+                ts_iso: tsIso,
+                session_id: input.session_id,
+                pid: process.pid,
+                turn: input.turn,
+                mechanism: input.mechanism,
+                tool: input.tool,
+                tokens_without: input.tokens_without,
+                tokens_with: input.tokens_with,
+                tokens_saved: input.tokens_saved,
+                detail: input.detail ? JSON.stringify(input.detail) : null,
+            });
+        }
+        catch {
+            /* best effort — never block the hot path */
+        }
+        const event = {
+            id: rowId,
+            ts: tsIso,
+            pid: process.pid,
+            ...input,
+        };
+        this.sessionEvents.push(event);
+        // RC-5: Evict oldest events if buffer exceeds limit
+        if (this.sessionEvents.length > MAX_SESSION_EVENTS) {
+            this.sessionEvents = this.sessionEvents.slice(-Math.floor(MAX_SESSION_EVENTS * 0.75));
+        }
+    }
+    /** Get all events this process has recorded for the current session. */
+    getSessionEvents() {
+        return this.sessionEvents;
+    }
+    /** Running total of tokens saved this session (in-memory, O(1) amortized). */
+    getSessionTokensSaved() {
+        let total = 0;
+        for (const e of this.sessionEvents)
+            total += e.tokens_saved;
+        return total;
+    }
+    /** Running session efficiency percentage. */
+    getSessionEfficiency() {
+        let totalWithout = 0;
+        let totalSaved = 0;
+        for (const e of this.sessionEvents) {
+            totalWithout += e.tokens_without;
+            totalSaved += e.tokens_saved;
+        }
+        if (totalWithout === 0)
+            return 0;
+        return Math.round((totalSaved / totalWithout) * 100);
+    }
+    /**
+     * Ingest an event produced by another process (relayed via the log-tailer's
+     * SQLite poll). Pushes to in-memory buffer for SSE streaming; does NOT
+     * re-insert into the DB (the originating process already did).
+     */
+    ingestExternal(event) {
+        if (event.pid === process.pid)
+            return;
+        this.sessionEvents.push(event);
+        if (this.sessionEvents.length > MAX_SESSION_EVENTS) {
+            this.sessionEvents = this.sessionEvents.slice(-Math.floor(MAX_SESSION_EVENTS * 0.75));
+        }
+    }
+}
+/**
+ * Read token flow events from the SQLite store with optional filtering.
+ * Used for CLI status, dashboard, and session summaries.
+ * Performance: <1ms for 1000 events (indexed lookup + row mapping).
+ */
+export function readTokenFlowEvents(unerrDir, filter) {
+    let rows;
+    try {
+        const store = openMetricsStore(unerrDir);
+        rows = filter?.session_id
+            ? store.tokenFlowBySession(filter.session_id)
+            : store.allTokenFlow();
+    }
+    catch {
+        return [];
+    }
+    const events = [];
+    for (const r of rows) {
+        if (filter?.mechanism && r.mechanism !== filter.mechanism)
+            continue;
+        if (filter?.from_ts && r.ts_iso < filter.from_ts)
+            continue;
+        if (filter?.to_ts && r.ts_iso > filter.to_ts)
+            continue;
+        events.push(rowToEvent(r));
+    }
+    return events;
+}
+// ── Aggregation ─────────────────────────────────────────────────────
+/**
+ * Aggregate events into a session-level summary.
+ * Computed on-read from the event stream (event sourcing principle).
+ */
+export function aggregateSession(events, sessionId) {
+    const sessionEvents = events.filter((e) => e.session_id === sessionId);
+    const mechanismMap = new Map();
+    const turnMap = new Map();
+    let totalWithout = 0;
+    let totalWith = 0;
+    let totalSaved = 0;
+    const turnNumbers = new Set();
+    for (const e of sessionEvents) {
+        totalWithout += e.tokens_without;
+        totalWith += e.tokens_with;
+        totalSaved += e.tokens_saved;
+        turnNumbers.add(e.turn);
+        // Per-mechanism accumulation
+        const mech = mechanismMap.get(e.mechanism) ?? {
+            saved: 0,
+            delivered: 0,
+            count: 0,
+        };
+        mech.saved += e.tokens_saved;
+        mech.delivered += e.tokens_with;
+        mech.count++;
+        mechanismMap.set(e.mechanism, mech);
+        // Per-turn accumulation
+        const turn = turnMap.get(e.turn) ?? {
+            tool: e.tool ?? "unknown",
+            tokens_without: 0,
+            tokens_delivered: 0,
+            tokens_saved: 0,
+            mechanisms: new Map(),
+        };
+        turn.tokens_without += e.tokens_without;
+        turn.tokens_delivered += e.tokens_with;
+        turn.tokens_saved += e.tokens_saved;
+        turn.mechanisms.set(e.mechanism, (turn.mechanisms.get(e.mechanism) ?? 0) + e.tokens_saved);
+        if (e.tool)
+            turn.tool = e.tool;
+        turnMap.set(e.turn, turn);
+    }
+    // Build mechanism summary with percentage of total
+    const byMechanism = {};
+    for (const [mechanism, data] of mechanismMap) {
+        byMechanism[mechanism] = {
+            tokens_saved: data.saved,
+            tokens_delivered: data.delivered,
+            event_count: data.count,
+            pct_of_total: totalSaved > 0 ? Math.round((data.saved / totalSaved) * 1000) / 10 : 0,
+        };
+    }
+    // Build top turns sorted by tokens_saved descending, take top 5
+    const topTurns = [...turnMap.entries()]
+        .map(([turn, data]) => {
+        let primaryMechanism = "unknown";
+        let maxSaved = 0;
+        for (const [mech, saved] of data.mechanisms) {
+            if (saved > maxSaved) {
+                maxSaved = saved;
+                primaryMechanism = mech;
+            }
+        }
+        return {
+            turn,
+            tool: data.tool,
+            tokens_without: data.tokens_without,
+            tokens_delivered: data.tokens_delivered,
+            tokens_saved: data.tokens_saved,
+            primary_mechanism: primaryMechanism,
+        };
+    })
+        .sort((a, b) => b.tokens_saved - a.tokens_saved)
+        .slice(0, 5);
+    return {
+        session_id: sessionId,
+        total_turns: turnNumbers.size,
+        total_tokens_without: totalWithout,
+        total_tokens_with: totalWith,
+        total_tokens_saved: totalSaved,
+        efficiency_pct: totalWithout > 0 ? Math.round((totalSaved / totalWithout) * 100) : 0,
+        by_mechanism: byMechanism,
+        top_turns: topTurns,
+    };
+}
+/**
+ * Aggregate events by mechanism across all sessions (for cross-session trends).
+ */
+export function aggregateByMechanism(events) {
+    let totalSaved = 0;
+    const mechanismMap = new Map();
+    for (const e of events) {
+        totalSaved += e.tokens_saved;
+        const mech = mechanismMap.get(e.mechanism) ?? {
+            saved: 0,
+            delivered: 0,
+            count: 0,
+        };
+        mech.saved += e.tokens_saved;
+        mech.delivered += e.tokens_with;
+        mech.count++;
+        mechanismMap.set(e.mechanism, mech);
+    }
+    const result = {};
+    for (const [mechanism, data] of mechanismMap) {
+        result[mechanism] = {
+            tokens_saved: data.saved,
+            tokens_delivered: data.delivered,
+            event_count: data.count,
+            pct_of_total: totalSaved > 0 ? Math.round((data.saved / totalSaved) * 1000) / 10 : 0,
+        };
+    }
+    return result;
+}

package/dist/tracking/turn-segmenter.js

@@ -0,0 +1,122 @@
+/**
+ * Turn Segmenter — assigns ledger entries to logical "turns" (one user→agent
+ * round-trip). Embedded inside ShadowLedger.record(); does not write or read
+ * files. Pure in-memory state, isolated per session.
+ *
+ * Boundary detection (in priority order):
+ *  1. `closeTurn(sessionId)` — called by Stop hooks (Claude Code) or session
+ *     shutdown. Anchors an exact boundary. Next observed entry opens a new
+ *     turn with confidence "first_call".
+ *  2. Idle gap > IDLE_GAP_MS between consecutive entries on the same session
+ *     closes the previous turn and opens a new one with confidence "idle_gap".
+ *  3. The very first entry on a session opens turn 1 with confidence
+ *     "first_call".
+ *
+ * Subscribers get notified on every turn close via onTurnClose(listener).
+ * The timeline subsystem uses this to upsert turn rollups into timeline.db
+ * — but the segmenter itself has no dependency on that subsystem and is safe
+ * to use standalone (e.g., in tests, or when UNERR_TIMELINE_V2=0).
+ */
+import { randomBytes } from "node:crypto";
+const DEFAULT_IDLE_GAP_MS = 20_000;
+export class TurnSegmenter {
+    state = new Map();
+    listeners = [];
+    idleGapMs;
+    constructor(options = {}) {
+        this.idleGapMs = options.idleGapMs ?? DEFAULT_IDLE_GAP_MS;
+    }
+    /**
+     * Subscribe to turn-close events. Returns an unsubscribe function.
+     */
+    onTurnClose(listener) {
+        this.listeners.push(listener);
+        return () => {
+            const idx = this.listeners.indexOf(listener);
+            if (idx >= 0)
+                this.listeners.splice(idx, 1);
+        };
+    }
+    /**
+     * Observe an entry and stamp it with turn_id + turn_confidence in place.
+     * May emit a close event for the previous turn if an idle gap is detected.
+     */
+    observe(entry) {
+        const ts = Date.parse(entry.ts);
+        const s = this.state.get(entry.session_id) ?? this.initState();
+        if (s.currentTurnId === null) {
+            // Opening a new turn — openedBy is whatever the prior closeTurn or
+            // initState set ("first_call" for the very first or after stop_hook).
+            s.currentTurnId = generateTurnId();
+        }
+        else if (Number.isFinite(ts) &&
+            ts - s.lastEntryTs > this.idleGapMs &&
+            s.lastEntryTs > 0) {
+            this.emitClose({
+                turn_id: s.currentTurnId,
+                session_id: entry.session_id,
+                closed_at: s.lastEntryTs,
+                reason: "idle_gap",
+            });
+            s.currentTurnId = generateTurnId();
+            s.openedBy = "idle_gap";
+        }
+        entry.turn_id = s.currentTurnId;
+        entry.turn_confidence = s.openedBy;
+        if (Number.isFinite(ts))
+            s.lastEntryTs = ts;
+        this.state.set(entry.session_id, s);
+    }
+    /**
+     * Anchor an exact turn boundary. The next observed entry will open a new
+     * turn with confidence "first_call" (the boundary is known precisely).
+     * Idempotent — calling with no active turn is a no-op.
+     */
+    closeTurn(sessionId, reason = "stop_hook") {
+        const s = this.state.get(sessionId);
+        if (!s || s.currentTurnId === null)
+            return;
+        this.emitClose({
+            turn_id: s.currentTurnId,
+            session_id: sessionId,
+            closed_at: s.lastEntryTs,
+            reason,
+        });
+        s.currentTurnId = null;
+        s.openedBy = "first_call";
+        this.state.set(sessionId, s);
+    }
+    /**
+     * Inspect the current turn id for a session. Returns null if no turn is open.
+     * Intended for diagnostics / tests.
+     */
+    getCurrentTurnId(sessionId) {
+        return this.state.get(sessionId)?.currentTurnId ?? null;
+    }
+    /**
+     * Drop all state. For tests and clean shutdown.
+     */
+    reset() {
+        this.state.clear();
+    }
+    initState() {
+        return {
+            currentTurnId: null,
+            lastEntryTs: 0,
+            openedBy: "first_call",
+        };
+    }
+    emitClose(event) {
+        for (const l of this.listeners) {
+            try {
+                l(event);
+            }
+            catch (err) {
+                process.stderr.write(`[unerr:turn-segmenter] WARN: listener error: ${err instanceof Error ? err.message : String(err)}\n`);
+            }
+        }
+    }
+}
+function generateTurnId() {
+    return randomBytes(6).toString("hex");
+}

package/dist/tracking/weekly-accumulator.js

@@ -0,0 +1,179 @@
+/**
+ * Weekly Summary Accumulator — unified cross-session stats persistence.
+ *
+ * S8.4: Persists session metrics to ~/.unerr/stats.json.
+ * Replaces the separate cumulative-stats.json and cumulative-local-stats.json.
+ *
+ * Features:
+ *   - Weekly auto-reset (Monday boundary)
+ *   - All-time counters that persist across resets
+ *   - Feeds into `unerr stats` command and session guard
+ */
+function getWeekStart() {
+    const now = new Date();
+    const day = now.getDay();
+    const diff = now.getDate() - day + (day === 0 ? -6 : 1);
+    const monday = new Date(now);
+    monday.setDate(diff);
+    return monday.toISOString().slice(0, 10);
+}
+function getStatsPath() {
+    const home = process.env.HOME ?? process.env.USERPROFILE ?? process.cwd();
+    return `${home}/.unerr/stats.json`;
+}
+function createEmptyWeekly(weekStart) {
+    return {
+        weekStart,
+        sessions: 0,
+        tokensSaved: 0,
+        dollarsSaved: 0,
+        toolCalls: 0,
+        violationsCaught: 0,
+        chokepointWarnings: 0,
+        correctionsApplied: 0,
+        blastRadiusComputed: 0,
+        avgEfficiency: 0,
+        avgLatencyP50: 0,
+    };
+}
+function createEmptyAllTime() {
+    return {
+        firstSessionDate: new Date().toISOString(),
+        totalSessions: 0,
+        totalTokensSaved: 0,
+        totalDollarsSaved: 0,
+        totalViolationsCaught: 0,
+    };
+}
+export function loadStats() {
+    const currentWeek = getWeekStart();
+    try {
+        const { readFileSync } = require("node:fs");
+        const raw = JSON.parse(readFileSync(getStatsPath(), "utf-8"));
+        if (raw.version !== 1) {
+            return {
+                version: 1,
+                weekly: createEmptyWeekly(currentWeek),
+                allTime: createEmptyAllTime(),
+                lastUpdated: new Date().toISOString(),
+            };
+        }
+        // Reset weekly if new week, preserve all-time
+        if (raw.weekly.weekStart !== currentWeek) {
+            raw.weekly = createEmptyWeekly(currentWeek);
+        }
+        return raw;
+    }
+    catch {
+        return {
+            version: 1,
+            weekly: createEmptyWeekly(currentWeek),
+            allTime: createEmptyAllTime(),
+            lastUpdated: new Date().toISOString(),
+        };
+    }
+}
+/**
+ * Accumulate a session's metrics into the unified stats file.
+ * Returns the updated stats for display.
+ */
+export function accumulateSession(input) {
+    const stats = loadStats();
+    // Weekly
+    stats.weekly.sessions += 1;
+    stats.weekly.tokensSaved += input.tokensSaved;
+    stats.weekly.dollarsSaved += input.dollarsSaved;
+    stats.weekly.toolCalls += input.toolCalls;
+    stats.weekly.violationsCaught += input.violationsCaught;
+    stats.weekly.chokepointWarnings += input.chokepointWarnings;
+    stats.weekly.correctionsApplied += input.correctionsApplied;
+    stats.weekly.blastRadiusComputed += input.blastRadiusComputed;
+    // Rolling average for efficiency and latency
+    const n = stats.weekly.sessions;
+    stats.weekly.avgEfficiency =
+        stats.weekly.avgEfficiency * ((n - 1) / n) + input.efficiency * (1 / n);
+    stats.weekly.avgLatencyP50 =
+        stats.weekly.avgLatencyP50 * ((n - 1) / n) + input.latencyP50 * (1 / n);
+    // Layer 10: Accumulate per-mechanism token savings
+    if (input.tokensByMechanism) {
+        if (!stats.weekly.tokensByMechanism)
+            stats.weekly.tokensByMechanism = {};
+        for (const [mech, saved] of Object.entries(input.tokensByMechanism)) {
+            stats.weekly.tokensByMechanism[mech] =
+                (stats.weekly.tokensByMechanism[mech] ?? 0) + saved;
+        }
+    }
+    // All-time
+    stats.allTime.totalSessions += 1;
+    stats.allTime.totalTokensSaved += input.tokensSaved;
+    stats.allTime.totalDollarsSaved += input.dollarsSaved;
+    stats.allTime.totalViolationsCaught += input.violationsCaught;
+    stats.lastUpdated = new Date().toISOString();
+    persistStats(stats);
+    return stats;
+}
+function persistStats(stats) {
+    try {
+        const fs = require("node:fs");
+        const path = require("node:path");
+        const filePath = getStatsPath();
+        fs.mkdirSync(path.dirname(filePath), { recursive: true });
+        fs.writeFileSync(filePath, JSON.stringify(stats, null, 2));
+    }
+    catch {
+        // Non-critical — don't break session shutdown
+    }
+}
+/**
+ * Format stats for the `unerr stats` command output.
+ */
+export function formatStatsReport(stats) {
+    const w = stats.weekly;
+    const a = stats.allTime;
+    const lines = [
+        "",
+        "── unerr stats ────────────────────────────────",
+        "",
+        "  This week:",
+        `    Sessions:       ${w.sessions}`,
+        `    Tool calls:     ${w.toolCalls}`,
+        `    Tokens saved:   ~${formatTokenCount(w.tokensSaved)}`,
+        `    Dollar savings: $${w.dollarsSaved.toFixed(2)}`,
+        `    Efficiency:     ${Math.round(w.avgEfficiency)}%`,
+        `    Violations:     ${w.violationsCaught} caught`,
+    ];
+    if (w.blastRadiusComputed > 0) {
+        lines.push(`    Blast radius:   ${w.blastRadiusComputed} computations`);
+    }
+    // Layer 10: Mechanism breakdown
+    if (w.tokensByMechanism && Object.keys(w.tokensByMechanism).length > 0) {
+        lines.push("");
+        lines.push("  By mechanism:");
+        const sorted = Object.entries(w.tokensByMechanism).sort(([, a], [, b]) => b - a);
+        const totalMech = sorted.reduce((s, [, v]) => s + v, 0);
+        for (const [mech, saved] of sorted) {
+            const pct = totalMech > 0 ? Math.round((saved / totalMech) * 100) : 0;
+            const barLen = Math.max(1, Math.round(pct / 5));
+            const bar = "█".repeat(barLen) + "░".repeat(20 - barLen);
+            lines.push(`    ${mech.padEnd(22)} ${formatTokenCount(saved).padStart(6)}  ${String(pct).padStart(3)}%  ${bar}`);
+        }
+    }
+    lines.push("");
+    lines.push("  All time:");
+    lines.push(`    Sessions:       ${a.totalSessions}`);
+    lines.push(`    Tokens saved:   ~${formatTokenCount(a.totalTokensSaved)}`);
+    lines.push(`    Dollar savings: $${a.totalDollarsSaved.toFixed(2)}`);
+    lines.push(`    Violations:     ${a.totalViolationsCaught} caught`);
+    lines.push(`    Since:          ${a.firstSessionDate.slice(0, 10)}`);
+    lines.push("");
+    lines.push("───────────────────────────────────────────────");
+    lines.push("");
+    return lines.join("\n");
+}
+function formatTokenCount(n) {
+    if (n >= 1_000_000)
+        return `${(n / 1_000_000).toFixed(1)}M`;
+    if (n >= 1_000)
+        return `${(n / 1_000).toFixed(1)}K`;
+    return String(n);
+}