agentwrangler 0.1.0

package/dist/detector/detectors/d4_model_mismatch.js

@@ -0,0 +1,258 @@
+/**
+ * src/detector/detectors/d4_model_mismatch.ts — D4 MODEL_MISMATCH.
+ *
+ * Fires PER-WORKSPACE when a sustained fraction of Opus turns in the trailing
+ * week match the "high-context, low-output" pattern — turns where Opus is reading
+ * a large context but producing little output, a signal that Sonnet could serve
+ * those turns at substantially lower cost.
+ *
+ * Conservative thresholds (all UNVALIDATED defaults, labeled in every rec):
+ *   min_opus_turns:      5     — cold-start guard; avoids firing on sparse data
+ *   mismatch_fraction:  40%    — sustained pattern, not isolated Opus use
+ *   min_context_tokens: 50,000 — well above trivial lookups; below Opus-4-8 avg 155K
+ *   max_output_tokens:    500  — short answer from large context ≈ lookup style
+ *
+ * False-positive guard: only turns where BOTH conditions hold (large context AND
+ * small output) are flagged. Opus turns with large output, or turns below the
+ * context threshold, are never counted as mismatch — preserving legitimate
+ * architecture/reasoning/security use of Opus.
+ *
+ * Savings model: per-turn cost differential (Opus − Sonnet) on mismatch turns,
+ * scaled by mismatch_turns_per_week × reduction_fraction (0.50 UNVALIDATED).
+ * Prices are derived from pricing_snapshots, not hard-coded. Falls back to
+ * LIST_PRICES when no snapshot is present for a tier.
+ *
+ * scopeKey: "D4|<workspace_id>" — one rec per workspace per window.
+ */
+import { LIST_PRICES } from "../../ingest/pricing.js";
+import { D4_MISMATCH_MAX_OUTPUT_TOKENS, D4_MISMATCH_MIN_CONTEXT_TOKENS, D4_MISMATCH_MIN_FRACTION, D4_OPUS_MIN_TURNS, D4_REDUCTION_FRACTION, d4Savings, } from "../savings.js";
+/**
+ * Fallback prices from LIST_PRICES when pricing_snapshots is missing a tier.
+ * Sonnet uses the sonnet-4 tier as the conservative routing target — its higher
+ * base rate ($3/$15 vs sonnet-5's $2/$10) yields the smaller Opus−Sonnet
+ * differential, so the modeled routing savings are never overstated (BM0).
+ */
+const OPUS_INPUT_FALLBACK = LIST_PRICES.opus?.[0] ?? 5;
+const OPUS_OUTPUT_FALLBACK = LIST_PRICES.opus?.[1] ?? 25;
+const SONNET_INPUT_FALLBACK = LIST_PRICES["sonnet-4"]?.[0] ?? 3;
+const SONNET_OUTPUT_FALLBACK = LIST_PRICES["sonnet-4"]?.[1] ?? 15;
+/** Read a fresh, well-formed /usage snapshot once; fail open on any problem. */
+function getPerModelSnapshot(db, now) {
+    try {
+        const row = db
+            .prepare("SELECT value FROM user_config WHERE key = 'per_model_snapshot'")
+            .get();
+        if (!row || typeof row.value !== "string")
+            return null;
+        const snapshot = JSON.parse(row.value);
+        if (!snapshot || typeof snapshot !== "object" || Array.isArray(snapshot))
+            return null;
+        const candidate = snapshot;
+        if (typeof candidate.captured_at !== "string" ||
+            typeof candidate.seven_day_util !== "number" ||
+            !Number.isFinite(candidate.seven_day_util) ||
+            typeof candidate.five_hour_util !== "number" ||
+            !Number.isFinite(candidate.five_hour_util) ||
+            !Array.isArray(candidate.per_model)) {
+            return null;
+        }
+        const capturedAtMs = Date.parse(candidate.captured_at);
+        if (!Number.isFinite(capturedAtMs) || now.getTime() - capturedAtMs > 24 * 60 * 60 * 1000) {
+            return null;
+        }
+        const perModel = [];
+        for (const entry of candidate.per_model) {
+            if (!entry || typeof entry !== "object" || Array.isArray(entry))
+                return null;
+            const model = entry;
+            if (typeof model.model !== "string" ||
+                typeof model.utilization !== "number" ||
+                !Number.isFinite(model.utilization)) {
+                return null;
+            }
+            perModel.push({ model: model.model, utilization: model.utilization });
+        }
+        return {
+            captured_at: candidate.captured_at,
+            seven_day_util: candidate.seven_day_util,
+            five_hour_util: candidate.five_hour_util,
+            per_model: perModel,
+        };
+    }
+    catch {
+        return null;
+    }
+}
+/** Parse a unit_prices_json array and return [input, output, cacheRead] prices. */
+function parsePrices(json) {
+    if (!json)
+        return [0, 0];
+    try {
+        const arr = JSON.parse(json);
+        return [arr[0] ?? 0, arr[1] ?? 0];
+    }
+    catch {
+        return [0, 0];
+    }
+}
+/** Get the freshest pricing snapshot for a given model tier, or null. */
+function getTierPrices(db, tier) {
+    const row = db
+        .prepare(`SELECT unit_prices_json FROM pricing_snapshots
+        WHERE model_tier = ?
+        ORDER BY stale_after DESC
+        LIMIT 1`)
+        .get(tier);
+    if (!row)
+        return null;
+    return parsePrices(row.unit_prices_json);
+}
+export const d4Detector = {
+    id: "D4",
+    name: "MODEL_MISMATCH",
+    evaluate(db, ctx) {
+        // Cap attribution is advisory metadata only; any read or parse failure fails open.
+        const perModelSnapshot = getPerModelSnapshot(db, ctx.now);
+        // All non-global workspaces with at least one turn in the window.
+        const workspaces = db
+            .prepare(`SELECT DISTINCT workspace_id
+           FROM turns
+          WHERE ts >= ? AND ts < ?
+            AND model LIKE '%opus%'
+            AND provisional = 0`)
+            .all(ctx.fromIso, ctx.toIso);
+        if (workspaces.length === 0) {
+            return {
+                fired: [],
+                status: "INACTIVE",
+                note: "no Opus turns in window",
+            };
+        }
+        // Resolve prices once (shared across all workspace evaluations).
+        const opusPrices = getTierPrices(db, "opus");
+        const sonnetPrices = getTierPrices(db, "sonnet-4");
+        const opusInputPrice = opusPrices?.[0] ?? OPUS_INPUT_FALLBACK;
+        const opusOutputPrice = opusPrices?.[1] ?? OPUS_OUTPUT_FALLBACK;
+        const sonnetInputPrice = sonnetPrices?.[0] ?? SONNET_INPUT_FALLBACK;
+        const sonnetOutputPrice = sonnetPrices?.[1] ?? SONNET_OUTPUT_FALLBACK;
+        const fired = [];
+        for (const { workspace_id } of workspaces) {
+            // Count ALL Opus turns for this workspace in the window (denominator for fraction).
+            const totalOpus = db
+                .prepare(`SELECT COUNT(*) AS n FROM turns
+              WHERE workspace_id = ?
+                AND model LIKE '%opus%'
+                AND ts >= ? AND ts < ?
+                AND provisional = 0`)
+                .get(workspace_id, ctx.fromIso, ctx.toIso).n;
+            // Cold-start guard: skip workspaces with too few Opus turns.
+            if (totalOpus < D4_OPUS_MIN_TURNS)
+                continue;
+            // Count mismatch turns (high context AND low output).
+            const mismatch = db
+                .prepare(`SELECT
+             COUNT(*) AS mismatch_count,
+             AVG(input_tokens) AS avg_input_tokens,
+             AVG(output_tokens) AS avg_output_tokens
+           FROM turns
+          WHERE workspace_id = ?
+            AND model LIKE '%opus%'
+            AND context_tokens >= ?
+            AND output_tokens <= ?
+            AND ts >= ? AND ts < ?
+            AND provisional = 0`)
+                .get(workspace_id, D4_MISMATCH_MIN_CONTEXT_TOKENS, D4_MISMATCH_MAX_OUTPUT_TOKENS, ctx.fromIso, ctx.toIso);
+            const mismatchCount = mismatch.mismatch_count ?? 0;
+            if (mismatchCount === 0)
+                continue;
+            const mismatchFraction = mismatchCount / totalOpus;
+            if (mismatchFraction < D4_MISMATCH_MIN_FRACTION)
+                continue;
+            const avgInputTokens = Math.round(mismatch.avg_input_tokens ?? 0);
+            const avgOutputTokens = Math.round(mismatch.avg_output_tokens ?? 0);
+            const { savingsU, formula } = d4Savings(avgInputTokens, avgOutputTokens, mismatchCount, opusInputPrice, opusOutputPrice, sonnetInputPrice, sonnetOutputPrice, D4_REDUCTION_FRACTION);
+            // Skip zero-savings recs (e.g., prices not loaded). The computed figure is
+            // retained as DIAGNOSTIC evidence only (see advisory gate below).
+            if (savingsU <= 0)
+                continue;
+            // This is evidence for an already-fired D4 recommendation, not another
+            // qualification rule. Keep its scope and time window aligned with D4.
+            const sidechainPremiumTurns = db
+                .prepare(`SELECT COUNT(*) AS n FROM turns
+              WHERE workspace_id = ?
+                AND model LIKE '%opus%'
+                AND is_sidechain = 1
+                AND ts >= ? AND ts < ?
+                AND provisional = 0`)
+                .get(workspace_id, ctx.fromIso, ctx.toIso).n;
+            const mismatchPct = Math.round(mismatchFraction * 100);
+            const sonnetEntries = perModelSnapshot?.per_model.filter((entry) => entry.model.toLowerCase().includes("sonnet"));
+            const bindingSonnet = sonnetEntries?.find((entry) => entry.utilization >= (perModelSnapshot?.seven_day_util ?? Number.POSITIVE_INFINITY));
+            const capEvidence = bindingSonnet && perModelSnapshot
+                ? {
+                    withheld: true,
+                    withheld_reason: `Sonnet weekly cap is the binding constraint (Sonnet util ${bindingSonnet.utilization} >= all-models ${perModelSnapshot.seven_day_util}) — routing Opus->Sonnet would worsen it`,
+                    title: `[withheld] Route Opus→Sonnet: ${mismatchPct}% of turns are high-context low-output`,
+                }
+                : perModelSnapshot && sonnetEntries && sonnetEntries.length > 0
+                    ? { cap_attribution: "all_models_or_opus_binds" }
+                    : undefined;
+            fired.push({
+                scopeKey: `D4|${workspace_id}`,
+                category: "MODEL",
+                scope_workspace_id: workspace_id,
+                // Advisory gate (W0.3): which cap binds is NOT inferable from JSONL. Conditional lever.
+                lever: "If your all-models / Opus / 5h cap is the one binding — check /usage — these high-context low-output Opus turns are Sonnet-movable. This does NOT help, and can hurt, if your Sonnet-specific weekly cap is the binding constraint.",
+                target_metric: "model_mix_opus_fraction",
+                // Advisory gate: suppress the crisp $/wk headline until live /usage cap-attribution exists.
+                modeled_savings_u_per_wk: null,
+                // Retain the computation as a DIAGNOSTIC formula (kind ADVISORY), not a headline.
+                // Destructure out result_usd_per_wk so the advisory formula carries no crisp $/wk figure.
+                modeled_formula: (({ result_usd_per_wk: _, ...rest }) => ({ ...rest, kind: "ADVISORY" }))(formula),
+                evidence: {
+                    title: `Route Opus→Sonnet: ${mismatchPct}% of turns are high-context low-output`,
+                    workspace_id,
+                    total_opus_turns_per_week: totalOpus,
+                    mismatch_turns_per_week: mismatchCount,
+                    sidechain_premium_turns: sidechainPremiumTurns,
+                    mismatch_fraction: Number(mismatchFraction.toFixed(4)),
+                    min_context_tokens_threshold: D4_MISMATCH_MIN_CONTEXT_TOKENS,
+                    max_output_tokens_threshold: D4_MISMATCH_MAX_OUTPUT_TOKENS,
+                    mismatch_fraction_threshold: D4_MISMATCH_MIN_FRACTION,
+                    min_opus_turns_threshold: D4_OPUS_MIN_TURNS,
+                    avg_input_tokens: avgInputTokens,
+                    avg_output_tokens: avgOutputTokens,
+                    opus_input_price_usd_per_mtok: opusInputPrice,
+                    opus_output_price_usd_per_mtok: opusOutputPrice,
+                    sonnet_input_price_usd_per_mtok: sonnetInputPrice,
+                    sonnet_output_price_usd_per_mtok: sonnetOutputPrice,
+                    reduction_fraction: D4_REDUCTION_FRACTION,
+                    // Advisory framing: crisp savings suppressed; computed figure is diagnostic only.
+                    advisory: true,
+                    requires_usage_cap_data: true,
+                    diagnostic_savings_u_per_wk_if_all_models_cap_binds: savingsU,
+                    advisory_note: "Routing savings are only real if the all-models/Opus/5h cap is binding — check /usage. No crisp $/wk is emitted from transcripts alone.",
+                    steps: [
+                        "Check /usage to see which cap (all-models/Opus/5h vs Sonnet weekly) is actually binding",
+                        "Only if the all-models/Opus cap binds: review the highest-Opus sessions in this workspace",
+                        "Identify turns where the task is lookup, summarization, or formatting (not deep reasoning)",
+                        "Route those turns to Sonnet; re-check this workspace's Opus fraction after 7 days",
+                    ],
+                    ...capEvidence,
+                },
+            });
+        }
+        if (fired.length === 0) {
+            return {
+                fired: [],
+                status: "INACTIVE",
+                note: `no workspace met the mismatch threshold (≥${D4_OPUS_MIN_TURNS} Opus turns, ≥${Math.round(D4_MISMATCH_MIN_FRACTION * 100)}% high-context-low-output)`,
+            };
+        }
+        return {
+            fired,
+            status: "ACTIVE",
+            note: `${fired.length} workspace(s) with sustained Opus mismatch pattern`,
+        };
+    },
+};

package/dist/detector/detectors/d5_limit_burn_forecast.js

@@ -0,0 +1,138 @@
+/**
+ * src/detector/detectors/d5_limit_burn_forecast.ts — D5 LIMIT_BURN_FORECAST.
+ *
+ * Warning-class detector (no savings model). Uses the existing forecastFromDb
+ * (ADR-107 §D-5) under the injected clock. Burn figures are CAP-WEIGHTED
+ * (cache reads × COEFF, unverified — see src/query/cap-weighted.ts); the
+ * caveat travels in the fired evidence via `token_metric`. Fires (global scope,
+ * LIMIT category) when the forecast state is WARNING or EXCEEDED. state OFF
+ * (no :limit_tokens) → degraded burn-trend row when history exists, otherwise
+ * BLOCKED; OK / NO_BURN / COLD_START → INACTIVE.
+ */
+import { capWeightExprSql } from "../../query/cap-weighted.js";
+import { forecastFromDb } from "../../query/forecast.js";
+import { d5Formula } from "../savings.js";
+const MS_PER_DAY = 24 * 60 * 60 * 1000;
+const RECENT_BURN_WINDOW_DAYS = 1;
+const BASELINE_BURN_WINDOW_DAYS = 7;
+const CALIBRATION_NUDGE = "Calibrate to get a real limit";
+function aggregateCapWeightedBurn(db, fromMs, toMs, coeff) {
+    const row = db
+        .prepare(`SELECT COALESCE(SUM(${capWeightExprSql("turns", coeff)}), 0) AS tokens,
+              COUNT(*) AS turns
+         FROM turns
+        WHERE ts >= ? AND ts < ?`)
+        .get(new Date(fromMs).toISOString(), new Date(toMs).toISOString());
+    return { tokens: row.tokens, turns: row.turns };
+}
+/** Compare the latest calendar day with the user's own preceding seven-day baseline. */
+function degradedBurnTrend(db, now, coeff) {
+    const recentToMs = now.getTime();
+    const recentFromMs = recentToMs - RECENT_BURN_WINDOW_DAYS * MS_PER_DAY;
+    const baselineFromMs = recentFromMs - BASELINE_BURN_WINDOW_DAYS * MS_PER_DAY;
+    const recent = aggregateCapWeightedBurn(db, recentFromMs, recentToMs, coeff);
+    const baseline = aggregateCapWeightedBurn(db, baselineFromMs, recentFromMs, coeff);
+    if (recent.turns === 0 && baseline.turns === 0)
+        return null;
+    const recentTokensPerDay = recent.tokens / RECENT_BURN_WINDOW_DAYS;
+    const baselineTokensPerDay = baseline.tokens / BASELINE_BURN_WINDOW_DAYS;
+    const deltaTokensPerDay = recentTokensPerDay - baselineTokensPerDay;
+    const direction = deltaTokensPerDay > 0 ? "RISING" : deltaTokensPerDay < 0 ? "FALLING" : "FLAT";
+    return {
+        direction,
+        recentTokens: recent.tokens,
+        recentTokensPerDay,
+        baselineTokens: baseline.tokens,
+        baselineTokensPerDay,
+        deltaTokensPerDay,
+    };
+}
+function directionLabel(direction) {
+    switch (direction) {
+        case "RISING":
+            return "rising";
+        case "FALLING":
+            return "falling";
+        case "FLAT":
+            return "flat";
+    }
+}
+export const d5Detector = {
+    id: "D5",
+    name: "LIMIT_BURN_FORECAST",
+    evaluate(db, ctx) {
+        const forecast = forecastFromDb(db, { now: ctx.now });
+        if (forecast.state === "OFF") {
+            const trend = degradedBurnTrend(db, ctx.now, forecast.cap_read_coeff);
+            if (trend === null) {
+                return { fired: [], status: "BLOCKED", note: "Weekly token limit is not configured" };
+            }
+            const fired = {
+                scopeKey: "D5|global|LIMIT_BURN_FORECAST",
+                category: "LIMIT",
+                scope_workspace_id: null,
+                lever: `${CALIBRATION_NUDGE}, then use this burn trend to guide workload.`,
+                target_metric: "burn_trend",
+                modeled_savings_u_per_wk: null,
+                modeled_formula: d5Formula(),
+                evidence: {
+                    title: `Burn trend ${directionLabel(trend.direction)}: ${CALIBRATION_NUDGE}`,
+                    state: "DEGRADED",
+                    forecast_state: forecast.state,
+                    tokens_used: forecast.tokens_used,
+                    limit_tokens: null,
+                    projected_exhaustion_jd: null,
+                    token_metric: forecast.token_metric,
+                    cap_weighted: forecast.cap_weighted,
+                    burn_trend: trend.direction,
+                    recent_cap_weighted_tokens: trend.recentTokens,
+                    recent_cap_weighted_tokens_per_day: trend.recentTokensPerDay,
+                    baseline_cap_weighted_tokens: trend.baselineTokens,
+                    baseline_cap_weighted_tokens_per_day: trend.baselineTokensPerDay,
+                    baseline_window_days: BASELINE_BURN_WINDOW_DAYS,
+                    trend_delta_cap_weighted_tokens_per_day: trend.deltaTokensPerDay,
+                    calibration_nudge: CALIBRATION_NUDGE,
+                },
+            };
+            return {
+                fired: [fired],
+                status: "ACTIVE",
+                note: `degraded burn trend ${directionLabel(trend.direction)}; calibrate to get a real limit`,
+            };
+        }
+        if (forecast.state === "WARNING" || forecast.state === "EXCEEDED") {
+            const stateLabel = forecast.state === "EXCEEDED" ? "EXCEEDED" : "warning";
+            const fired = {
+                scopeKey: "D5|global|LIMIT_BURN_FORECAST",
+                category: "LIMIT",
+                scope_workspace_id: null,
+                lever: "Scope work to the top-burn workspace/sessions before the weekly reset; reduce burn rate.",
+                target_metric: "forecast_margin",
+                modeled_savings_u_per_wk: null,
+                modeled_formula: d5Formula(),
+                evidence: {
+                    title: `Rate-limit ${stateLabel}: reduce burn before weekly reset`,
+                    state: forecast.state,
+                    tokens_used: forecast.tokens_used,
+                    limit_tokens: forecast.limit_tokens,
+                    projected_exhaustion_jd: forecast.projected_exhaustion_jd,
+                    token_metric: forecast.token_metric,
+                    cap_weighted: true,
+                    // Legacy limit-scale honesty flag (review P1): true when the stored
+                    // limit was calibrated under the old full-weight meter, so the fired
+                    // evidence carries the caveat instead of silently trusting the number.
+                    limit_scale_legacy: forecast.limit_scale_legacy,
+                    ...(forecast.limit_scale_note !== null
+                        ? { limit_scale_note: forecast.limit_scale_note }
+                        : {}),
+                },
+            };
+            return { fired: [fired], status: "ACTIVE", note: `burn forecast ${forecast.state}` };
+        }
+        return {
+            fired: [],
+            status: "INACTIVE",
+            note: `burn forecast ${forecast.state} (no exhaustion projected within the warn window)`,
+        };
+    },
+};