avorelo 0.1.0 → 0.3.0

package/scripts/lib/runtime-decision-policy.js

@@ -1,387 +0,0 @@
-"use strict";
-
-const { readAllMetrics, readSessionMetrics } = require("./metrics");
-const { readOutcomeEvents } = require("./unified-events");
-
-const POLICY_VERSION = 1;
-const SESSION_ROUTE_POLICY_VERSION = 1;
-const ESTIMATED_REROUTE_POLICY_VERSION = 1;
-
-const RECENT_FAILURE_WINDOW_MS = 15 * 60 * 1000;
-const SESSION_ROUTE_WINDOW_MS = 24 * 60 * 60 * 1000;
-const COUNTERFACTUAL_REASON_CODES = new Set(["FAIL_RETRY_GUARD", "LOOP_REPEATED_TOOL", "CTX_LARGE_OUTPUT"]);
-
-function numberOrNull(value) {
-  if (value === null || value === undefined || value === "") return null;
-  const number = Number(value);
-  return Number.isFinite(number) ? number : null;
-}
-
-function thresholdsForProfile(profile) {
-  if (profile === "cost-saver") {
-    return { cost: 0.03, tokens: 15000, latencyMs: 1500 };
-  }
-  if (profile === "quality") {
-    return { cost: 0.2, tokens: 60000, latencyMs: 4000 };
-  }
-  return { cost: 0.08, tokens: 30000, latencyMs: 2500 };
-}
-
-function heuristicThresholdsForProfile(profile) {
-  if (profile === "cost-saver") {
-    return { failureCount: 2, estimatedCounterfactual: 4 };
-  }
-  if (profile === "quality") {
-    return { failureCount: 2, estimatedCounterfactual: 8 };
-  }
-  return { failureCount: 2, estimatedCounterfactual: 6 };
-}
-
-function metricTs(metric) {
-  return Date.parse(metric?.ts || 0);
-}
-
-function eventTs(event) {
-  return Date.parse(event?.timestamp || 0);
-}
-
-function isRecent(ts, windowMs) {
-  return Number.isFinite(ts) && ts >= Date.now() - windowMs;
-}
-
-function triggerMatches(evidence, thresholds) {
-  const triggers = [];
-  if (numberOrNull(evidence.cost) !== null && Number(evidence.cost) >= thresholds.cost) triggers.push("cost");
-  if (numberOrNull(evidence.tokenUsage) !== null && Number(evidence.tokenUsage) >= thresholds.tokens) triggers.push("tokens");
-  if (numberOrNull(evidence.latencyMs) !== null && Number(evidence.latencyMs) >= thresholds.latencyMs) triggers.push("latency");
-  return triggers;
-}
-
-function evidenceSummary(evidence = {}) {
-  const parts = [];
-  if (evidence.provider || evidence.model) {
-    parts.push([evidence.provider, evidence.model].filter(Boolean).join(" "));
-  }
-  if (numberOrNull(evidence.tokenUsage) !== null) parts.push(`${Number(evidence.tokenUsage)} tokens`);
-  if (numberOrNull(evidence.cost) !== null) parts.push(`$${Number(evidence.cost).toFixed(2)}`);
-  if (numberOrNull(evidence.latencyMs) !== null) parts.push(`${Number(evidence.latencyMs)}ms`);
-  return parts.join(", ");
-}
-
-function extractFailureEvidence(metric = {}) {
-  const attribution = metric.attribution || {};
-  return {
-    provider: attribution.provider || null,
-    model: attribution.model || null,
-    tokenUsage: numberOrNull(attribution.tokenUsage),
-    cost: numberOrNull(attribution.cost),
-    latencyMs: numberOrNull(attribution.latencyMs),
-    executionPath: attribution.executionPath || null,
-  };
-}
-
-function toolLooksModelDriven(toolName, evidence = {}) {
-  return evidence.executionPath === "llm" || evidence.provider || evidence.model || toolName === "Agent";
-}
-
-function latestByTs(entries) {
-  return (entries || []).filter(Boolean).sort((a, b) => (b.ts || 0) - (a.ts || 0))[0] || null;
-}
-
-function findLatestExpensiveFailure(metrics, toolName, thresholds, options = {}) {
-  const windowMs = Number(options.windowMs || RECENT_FAILURE_WINDOW_MS);
-  const candidates = (Array.isArray(metrics) ? metrics : [])
-    .filter((metric) => metric?.event === "PostToolUseFailure" && String(metric?.tool || "") === String(toolName || ""))
-    .map((metric) => {
-      const ts = metricTs(metric);
-      const evidence = extractFailureEvidence(metric);
-      return {
-        metric,
-        ts,
-        evidence,
-        triggers: triggerMatches(evidence, thresholds),
-      };
-    })
-    .filter((entry) => isRecent(entry.ts, windowMs))
-    .filter((entry) => toolLooksModelDriven(toolName, entry.evidence))
-    .filter((entry) => entry.triggers.length > 0);
-
-  return latestByTs(candidates);
-}
-
-function buildCounterfactualSnapshot(cwd, sessionIds, windowMs) {
-  const events = readOutcomeEvents(cwd)
-    .map((event) => ({ event, ts: eventTs(event) }))
-    .filter((entry) => isRecent(entry.ts, windowMs))
-    .filter((entry) => sessionIds.has(String(entry.event.session_id || "")))
-    .filter((entry) => COUNTERFACTUAL_REASON_CODES.has(String(entry.event.reason_code || "")));
-
-  return {
-    estimatedCounterfactualTotal: events.reduce((sum, entry) => sum + Number(entry.event.estimated_counterfactual_value || 0), 0),
-    loopCount: events.filter((entry) => String(entry.event.reason_code || "") === "LOOP_REPEATED_TOOL").length,
-    largeOutputCount: events.filter((entry) => String(entry.event.reason_code || "") === "CTX_LARGE_OUTPUT").length,
-  };
-}
-
-function findEstimatedLowValuePattern(cwd, metrics, toolName, profile, windowMs, scope) {
-  const thresholds = heuristicThresholdsForProfile(profile);
-  const filtered = (Array.isArray(metrics) ? metrics : [])
-    .map((metric) => ({ metric, ts: metricTs(metric) }))
-    .filter((entry) => isRecent(entry.ts, windowMs));
-  const metricLoopCount = filtered.filter((entry) => (entry.metric?.reasonCodes || []).includes("LOOP_REPEATED_TOOL")).length;
-  const metricLargeOutputCount = filtered.filter((entry) => (entry.metric?.reasonCodes || []).includes("CTX_LARGE_OUTPUT")).length;
-
-  const modelDrivenFailures = filtered
-    .filter((entry) => entry.metric?.event === "PostToolUseFailure")
-    .filter((entry) => !toolName || String(entry.metric?.tool || "") === String(toolName))
-    .map((entry) => ({
-      metric: entry.metric,
-      ts: entry.ts,
-      evidence: extractFailureEvidence(entry.metric),
-    }))
-    .filter((entry) => toolLooksModelDriven(toolName || entry.metric?.tool, entry.evidence))
-    .filter((entry) => entry.evidence.cost === null);
-
-  if (modelDrivenFailures.length < thresholds.failureCount) {
-    return null;
-  }
-
-  const sessionIds = new Set(modelDrivenFailures.map((entry) => String(entry.metric?.sessionId || "")));
-  const counterfactual = buildCounterfactualSnapshot(cwd, sessionIds, windowMs);
-  const triggers = [];
-  if (counterfactual.estimatedCounterfactualTotal >= thresholds.estimatedCounterfactual) triggers.push("estimated_counterfactual");
-  if (counterfactual.loopCount > 0 || metricLoopCount > 0) triggers.push("loops");
-  if (counterfactual.largeOutputCount > 0 || metricLargeOutputCount > 0) triggers.push("large_output");
-
-  if (!triggers.length) {
-    return null;
-  }
-
-  const latestFailure = latestByTs(modelDrivenFailures);
-  return {
-    scope,
-    failureCount: modelDrivenFailures.length,
-    estimatedCounterfactualTotal: counterfactual.estimatedCounterfactualTotal,
-    loopCount: Math.max(counterfactual.loopCount, metricLoopCount),
-    largeOutputCount: Math.max(counterfactual.largeOutputCount, metricLargeOutputCount),
-    triggerSignals: triggers,
-    thresholds,
-    latestFailure,
-  };
-}
-
-function buildMeasuredRouteDecision(toolName, profile, requestedProfile, failure) {
-  const summary = evidenceSummary(failure.evidence);
-  const detail = summary ? ` after recent ${toolName} failures using ${summary}` : "";
-  return {
-    outputProfile: "terse",
-    reasonCodes: ["SMART_ROUTE_CHEAPER_SAFER"],
-    meta: {
-      decisionPolicy: `runtime_route_optimizer_v${SESSION_ROUTE_POLICY_VERSION}`,
-      evidenceBasis: "measured",
-      profile,
-      triggerSignals: failure.triggers,
-      thresholds: thresholdsForProfile(profile),
-      routeBefore: requestedProfile,
-      routeAfter: "terse",
-      automatic: true,
-      sourceTool: toolName,
-      priorFailureAt: failure.metric.ts || null,
-      priorRuntimeEvidence: {
-        provider: failure.evidence.provider,
-        model: failure.evidence.model,
-        tokenUsage: failure.evidence.tokenUsage,
-        cost: failure.evidence.cost,
-        latencyMs: failure.evidence.latencyMs,
-        costSource: failure.evidence.cost !== null ? "measured" : "unavailable",
-      },
-    },
-    intervention: {
-      eventName: "SessionStart",
-      kind: "runtime_route_downgrade",
-      whatHappened: `Wuz automatically started this session on a cheaper-safer route${detail}`,
-      nextAction: "Stay on the terse route first and re-expand only if the next step clearly needs deeper model work",
-      preservedState: "Launch stayed local-first and no tool ran before the route change",
-    },
-  };
-}
-
-function buildEstimatedRouteDecision(profile, requestedProfile, pattern) {
-  const detail = `${pattern.failureCount} recent failed model-driven attempt${pattern.failureCount === 1 ? "" : "s"}, estimated counterfactual ${Math.round(pattern.estimatedCounterfactualTotal * 100) / 100}, loops ${pattern.loopCount}`;
-  return {
-    outputProfile: "terse",
-    reasonCodes: ["SMART_ROUTE_CHEAPER_SAFER"],
-    meta: {
-      decisionPolicy: `runtime_route_optimizer_v${SESSION_ROUTE_POLICY_VERSION}`,
-      evidenceBasis: "estimated",
-      profile,
-      triggerSignals: pattern.triggerSignals,
-      thresholds: pattern.thresholds,
-      routeBefore: requestedProfile,
-      routeAfter: "terse",
-      automatic: true,
-      sourceTool: pattern.latestFailure?.metric?.tool || null,
-      priorFailureAt: pattern.latestFailure?.metric?.ts || null,
-      heuristicEvidence: {
-        failureCount: pattern.failureCount,
-        estimatedCounterfactualTotal: Math.round(pattern.estimatedCounterfactualTotal * 100) / 100,
-        loopCount: pattern.loopCount,
-        largeOutputCount: pattern.largeOutputCount,
-        costSource: "estimated",
-      },
-    },
-    intervention: {
-      eventName: "SessionStart",
-      kind: "runtime_route_downgrade",
-      whatHappened: `Wuz automatically started this session on a cheaper-safer route because recent model-driven retries already showed low expected value without measured cost (${detail})`,
-      nextAction: "Stay on the terse route first and prefer deterministic steps until the bounded path proves insufficient",
-      preservedState: "Launch stayed local-first and no tool ran before the route change",
-    },
-  };
-}
-
-function evaluateSessionStartRouteDecision(params = {}) {
-  const requestedProfile = String(params.requestedProfile || "standard");
-  if (!params.config?.smartRouting?.enabled) return null;
-  if (requestedProfile !== "standard") return null;
-
-  const profile = String(params.config?.profile || "balanced");
-  const allMetrics = readAllMetrics(params.cwd);
-  const thresholds = thresholdsForProfile(profile);
-
-  const candidates = [
-    findLatestExpensiveFailure(allMetrics, "Agent", thresholds, { windowMs: SESSION_ROUTE_WINDOW_MS }),
-    findLatestExpensiveFailure(allMetrics, "Skill", thresholds, { windowMs: SESSION_ROUTE_WINDOW_MS }),
-  ].filter(Boolean);
-
-  const measuredFailure = latestByTs(candidates);
-  if (measuredFailure) {
-    return buildMeasuredRouteDecision(String(measuredFailure.metric?.tool || "Agent"), profile, requestedProfile, measuredFailure);
-  }
-
-  const estimatedPattern = findEstimatedLowValuePattern(
-    params.cwd,
-    allMetrics,
-    null,
-    profile,
-    SESSION_ROUTE_WINDOW_MS,
-    "repo"
-  );
-  if (!estimatedPattern) return null;
-
-  return buildEstimatedRouteDecision(profile, requestedProfile, estimatedPattern);
-}
-
-function buildEstimatedRerouteDecision(toolName, profile, pattern) {
-  const failure = pattern.latestFailure;
-  const modelLine = evidenceSummary(failure?.evidence || {});
-  const detailParts = [];
-  if (modelLine) detailParts.push(modelLine);
-  detailParts.push(`${pattern.failureCount} failed attempt${pattern.failureCount === 1 ? "" : "s"}`);
-  detailParts.push(`estimated counterfactual ${Math.round(pattern.estimatedCounterfactualTotal * 100) / 100}`);
-  if (pattern.loopCount > 0) detailParts.push(`${pattern.loopCount} loop signal${pattern.loopCount === 1 ? "" : "s"}`);
-  const detail = detailParts.join(", ");
-
-  return {
-    action: "deny",
-    reasonCodes: ["EVIDENCE_ESTIMATED_LOW_VALUE_REROUTE"],
-    meta: {
-      decisionPolicy: `runtime_estimated_low_value_guard_v${ESTIMATED_REROUTE_POLICY_VERSION}`,
-      evidenceBasis: "estimated",
-      profile,
-      triggerSignals: pattern.triggerSignals,
-      thresholds: pattern.thresholds,
-      routeBefore: "model-driven",
-      routeAfter: "deterministic-first",
-      automatic: true,
-      sourceTool: toolName,
-      priorFailureAt: failure?.metric?.ts || null,
-      heuristicEvidence: {
-        provider: failure?.evidence?.provider || null,
-        model: failure?.evidence?.model || null,
-        failureCount: pattern.failureCount,
-        estimatedCounterfactualTotal: Math.round(pattern.estimatedCounterfactualTotal * 100) / 100,
-        loopCount: pattern.loopCount,
-        largeOutputCount: pattern.largeOutputCount,
-        costSource: "estimated",
-      },
-    },
-    intervention: {
-      eventName: "PreToolUse",
-      kind: "runtime_estimated_low_value_reroute",
-      whatHappened: `Wuz automatically kept this ${toolName} rerun off the model-driven path because recent retries already showed low expected value without measured cost (${detail})`,
-      nextAction: "Use a deterministic or smaller bounded step first, then retry the model-driven path only if that step proves it is still needed",
-      preservedState: `The ${toolName} rerun did not start`,
-    },
-  };
-}
-
-function evaluatePreToolRuntimeDecision(params = {}) {
-  const toolName = String(params.toolName || "");
-  if (!["Agent", "Skill"].includes(toolName)) return null;
-
-  const runtimeEvidence = params.runtimeEvidence || {};
-  const currentLooksModelDriven = toolLooksModelDriven(toolName, runtimeEvidence);
-  if (!currentLooksModelDriven) return null;
-
-  const profile = String(params.config?.profile || "balanced");
-  const thresholds = thresholdsForProfile(profile);
-  const metrics = readSessionMetrics(params.cwd, params.sessionId);
-  const recentFailure = findLatestExpensiveFailure(metrics, toolName, thresholds);
-  if (recentFailure) {
-    const summary = evidenceSummary(recentFailure.evidence);
-    const detail = summary ? ` after using ${summary}` : "";
-    const evidenceRecord = {
-      provider: recentFailure.evidence.provider,
-      model: recentFailure.evidence.model,
-      tokenUsage: recentFailure.evidence.tokenUsage,
-      cost: recentFailure.evidence.cost,
-      latencyMs: recentFailure.evidence.latencyMs,
-      costSource: recentFailure.evidence.cost !== null ? "measured" : "unavailable",
-    };
-
-    return {
-      action: "ask",
-      reasonCodes: ["EVIDENCE_RUNTIME_RETRY_REVIEW"],
-      meta: {
-        decisionPolicy: `runtime_evidence_retry_guard_v${POLICY_VERSION}`,
-        profile,
-        thresholds,
-        triggerSignals: recentFailure.triggers,
-        priorFailureAt: recentFailure.metric.ts || null,
-        priorTool: toolName,
-        priorRuntimeEvidence: evidenceRecord,
-      },
-      intervention: {
-        eventName: "PreToolUse",
-        kind: "runtime_evidence_retry_review",
-        whatHappened: `Wuz paused this ${toolName} rerun because the last attempt failed${detail}`,
-        nextAction: "Narrow scope, prefer a deterministic path if one exists, or confirm the stronger reason for paying this cost before rerunning",
-        preservedState: `The ${toolName} rerun did not start`,
-      },
-    };
-  }
-
-  if (!params.config?.smartRouting?.enabled) return null;
-  const estimatedPattern = findEstimatedLowValuePattern(
-    params.cwd,
-    metrics,
-    toolName,
-    profile,
-    RECENT_FAILURE_WINDOW_MS,
-    "session"
-  );
-  if (!estimatedPattern) return null;
-
-  return buildEstimatedRerouteDecision(toolName, profile, estimatedPattern);
-}
-
-module.exports = {
-  POLICY_VERSION,
-  SESSION_ROUTE_POLICY_VERSION,
-  ESTIMATED_REROUTE_POLICY_VERSION,
-  thresholdsForProfile,
-  heuristicThresholdsForProfile,
-  evaluatePreToolRuntimeDecision,
-  evaluateSessionStartRouteDecision,
-};