avorelo 0.1.0

package/scripts/lib/runtime-decision-policy.js

@@ -0,0 +1,387 @@
+"use strict";
+
+const { readAllMetrics, readSessionMetrics } = require("./metrics");
+const { readOutcomeEvents } = require("./unified-events");
+
+const POLICY_VERSION = 1;
+const SESSION_ROUTE_POLICY_VERSION = 1;
+const ESTIMATED_REROUTE_POLICY_VERSION = 1;
+
+const RECENT_FAILURE_WINDOW_MS = 15 * 60 * 1000;
+const SESSION_ROUTE_WINDOW_MS = 24 * 60 * 60 * 1000;
+const COUNTERFACTUAL_REASON_CODES = new Set(["FAIL_RETRY_GUARD", "LOOP_REPEATED_TOOL", "CTX_LARGE_OUTPUT"]);
+
+function numberOrNull(value) {
+  if (value === null || value === undefined || value === "") return null;
+  const number = Number(value);
+  return Number.isFinite(number) ? number : null;
+}
+
+function thresholdsForProfile(profile) {
+  if (profile === "cost-saver") {
+    return { cost: 0.03, tokens: 15000, latencyMs: 1500 };
+  }
+  if (profile === "quality") {
+    return { cost: 0.2, tokens: 60000, latencyMs: 4000 };
+  }
+  return { cost: 0.08, tokens: 30000, latencyMs: 2500 };
+}
+
+function heuristicThresholdsForProfile(profile) {
+  if (profile === "cost-saver") {
+    return { failureCount: 2, estimatedCounterfactual: 4 };
+  }
+  if (profile === "quality") {
+    return { failureCount: 2, estimatedCounterfactual: 8 };
+  }
+  return { failureCount: 2, estimatedCounterfactual: 6 };
+}
+
+function metricTs(metric) {
+  return Date.parse(metric?.ts || 0);
+}
+
+function eventTs(event) {
+  return Date.parse(event?.timestamp || 0);
+}
+
+function isRecent(ts, windowMs) {
+  return Number.isFinite(ts) && ts >= Date.now() - windowMs;
+}
+
+function triggerMatches(evidence, thresholds) {
+  const triggers = [];
+  if (numberOrNull(evidence.cost) !== null && Number(evidence.cost) >= thresholds.cost) triggers.push("cost");
+  if (numberOrNull(evidence.tokenUsage) !== null && Number(evidence.tokenUsage) >= thresholds.tokens) triggers.push("tokens");
+  if (numberOrNull(evidence.latencyMs) !== null && Number(evidence.latencyMs) >= thresholds.latencyMs) triggers.push("latency");
+  return triggers;
+}
+
+function evidenceSummary(evidence = {}) {
+  const parts = [];
+  if (evidence.provider || evidence.model) {
+    parts.push([evidence.provider, evidence.model].filter(Boolean).join(" "));
+  }
+  if (numberOrNull(evidence.tokenUsage) !== null) parts.push(`${Number(evidence.tokenUsage)} tokens`);
+  if (numberOrNull(evidence.cost) !== null) parts.push(`$${Number(evidence.cost).toFixed(2)}`);
+  if (numberOrNull(evidence.latencyMs) !== null) parts.push(`${Number(evidence.latencyMs)}ms`);
+  return parts.join(", ");
+}
+
+function extractFailureEvidence(metric = {}) {
+  const attribution = metric.attribution || {};
+  return {
+    provider: attribution.provider || null,
+    model: attribution.model || null,
+    tokenUsage: numberOrNull(attribution.tokenUsage),
+    cost: numberOrNull(attribution.cost),
+    latencyMs: numberOrNull(attribution.latencyMs),
+    executionPath: attribution.executionPath || null,
+  };
+}
+
+function toolLooksModelDriven(toolName, evidence = {}) {
+  return evidence.executionPath === "llm" || evidence.provider || evidence.model || toolName === "Agent";
+}
+
+function latestByTs(entries) {
+  return (entries || []).filter(Boolean).sort((a, b) => (b.ts || 0) - (a.ts || 0))[0] || null;
+}
+
+function findLatestExpensiveFailure(metrics, toolName, thresholds, options = {}) {
+  const windowMs = Number(options.windowMs || RECENT_FAILURE_WINDOW_MS);
+  const candidates = (Array.isArray(metrics) ? metrics : [])
+    .filter((metric) => metric?.event === "PostToolUseFailure" && String(metric?.tool || "") === String(toolName || ""))
+    .map((metric) => {
+      const ts = metricTs(metric);
+      const evidence = extractFailureEvidence(metric);
+      return {
+        metric,
+        ts,
+        evidence,
+        triggers: triggerMatches(evidence, thresholds),
+      };
+    })
+    .filter((entry) => isRecent(entry.ts, windowMs))
+    .filter((entry) => toolLooksModelDriven(toolName, entry.evidence))
+    .filter((entry) => entry.triggers.length > 0);
+
+  return latestByTs(candidates);
+}
+
+function buildCounterfactualSnapshot(cwd, sessionIds, windowMs) {
+  const events = readOutcomeEvents(cwd)
+    .map((event) => ({ event, ts: eventTs(event) }))
+    .filter((entry) => isRecent(entry.ts, windowMs))
+    .filter((entry) => sessionIds.has(String(entry.event.session_id || "")))
+    .filter((entry) => COUNTERFACTUAL_REASON_CODES.has(String(entry.event.reason_code || "")));
+
+  return {
+    estimatedCounterfactualTotal: events.reduce((sum, entry) => sum + Number(entry.event.estimated_counterfactual_value || 0), 0),
+    loopCount: events.filter((entry) => String(entry.event.reason_code || "") === "LOOP_REPEATED_TOOL").length,
+    largeOutputCount: events.filter((entry) => String(entry.event.reason_code || "") === "CTX_LARGE_OUTPUT").length,
+  };
+}
+
+function findEstimatedLowValuePattern(cwd, metrics, toolName, profile, windowMs, scope) {
+  const thresholds = heuristicThresholdsForProfile(profile);
+  const filtered = (Array.isArray(metrics) ? metrics : [])
+    .map((metric) => ({ metric, ts: metricTs(metric) }))
+    .filter((entry) => isRecent(entry.ts, windowMs));
+  const metricLoopCount = filtered.filter((entry) => (entry.metric?.reasonCodes || []).includes("LOOP_REPEATED_TOOL")).length;
+  const metricLargeOutputCount = filtered.filter((entry) => (entry.metric?.reasonCodes || []).includes("CTX_LARGE_OUTPUT")).length;
+
+  const modelDrivenFailures = filtered
+    .filter((entry) => entry.metric?.event === "PostToolUseFailure")
+    .filter((entry) => !toolName || String(entry.metric?.tool || "") === String(toolName))
+    .map((entry) => ({
+      metric: entry.metric,
+      ts: entry.ts,
+      evidence: extractFailureEvidence(entry.metric),
+    }))
+    .filter((entry) => toolLooksModelDriven(toolName || entry.metric?.tool, entry.evidence))
+    .filter((entry) => entry.evidence.cost === null);
+
+  if (modelDrivenFailures.length < thresholds.failureCount) {
+    return null;
+  }
+
+  const sessionIds = new Set(modelDrivenFailures.map((entry) => String(entry.metric?.sessionId || "")));
+  const counterfactual = buildCounterfactualSnapshot(cwd, sessionIds, windowMs);
+  const triggers = [];
+  if (counterfactual.estimatedCounterfactualTotal >= thresholds.estimatedCounterfactual) triggers.push("estimated_counterfactual");
+  if (counterfactual.loopCount > 0 || metricLoopCount > 0) triggers.push("loops");
+  if (counterfactual.largeOutputCount > 0 || metricLargeOutputCount > 0) triggers.push("large_output");
+
+  if (!triggers.length) {
+    return null;
+  }
+
+  const latestFailure = latestByTs(modelDrivenFailures);
+  return {
+    scope,
+    failureCount: modelDrivenFailures.length,
+    estimatedCounterfactualTotal: counterfactual.estimatedCounterfactualTotal,
+    loopCount: Math.max(counterfactual.loopCount, metricLoopCount),
+    largeOutputCount: Math.max(counterfactual.largeOutputCount, metricLargeOutputCount),
+    triggerSignals: triggers,
+    thresholds,
+    latestFailure,
+  };
+}
+
+function buildMeasuredRouteDecision(toolName, profile, requestedProfile, failure) {
+  const summary = evidenceSummary(failure.evidence);
+  const detail = summary ? ` after recent ${toolName} failures using ${summary}` : "";
+  return {
+    outputProfile: "terse",
+    reasonCodes: ["SMART_ROUTE_CHEAPER_SAFER"],
+    meta: {
+      decisionPolicy: `runtime_route_optimizer_v${SESSION_ROUTE_POLICY_VERSION}`,
+      evidenceBasis: "measured",
+      profile,
+      triggerSignals: failure.triggers,
+      thresholds: thresholdsForProfile(profile),
+      routeBefore: requestedProfile,
+      routeAfter: "terse",
+      automatic: true,
+      sourceTool: toolName,
+      priorFailureAt: failure.metric.ts || null,
+      priorRuntimeEvidence: {
+        provider: failure.evidence.provider,
+        model: failure.evidence.model,
+        tokenUsage: failure.evidence.tokenUsage,
+        cost: failure.evidence.cost,
+        latencyMs: failure.evidence.latencyMs,
+        costSource: failure.evidence.cost !== null ? "measured" : "unavailable",
+      },
+    },
+    intervention: {
+      eventName: "SessionStart",
+      kind: "runtime_route_downgrade",
+      whatHappened: `Wuz automatically started this session on a cheaper-safer route${detail}`,
+      nextAction: "Stay on the terse route first and re-expand only if the next step clearly needs deeper model work",
+      preservedState: "Launch stayed local-first and no tool ran before the route change",
+    },
+  };
+}
+
+function buildEstimatedRouteDecision(profile, requestedProfile, pattern) {
+  const detail = `${pattern.failureCount} recent failed model-driven attempt${pattern.failureCount === 1 ? "" : "s"}, estimated counterfactual ${Math.round(pattern.estimatedCounterfactualTotal * 100) / 100}, loops ${pattern.loopCount}`;
+  return {
+    outputProfile: "terse",
+    reasonCodes: ["SMART_ROUTE_CHEAPER_SAFER"],
+    meta: {
+      decisionPolicy: `runtime_route_optimizer_v${SESSION_ROUTE_POLICY_VERSION}`,
+      evidenceBasis: "estimated",
+      profile,
+      triggerSignals: pattern.triggerSignals,
+      thresholds: pattern.thresholds,
+      routeBefore: requestedProfile,
+      routeAfter: "terse",
+      automatic: true,
+      sourceTool: pattern.latestFailure?.metric?.tool || null,
+      priorFailureAt: pattern.latestFailure?.metric?.ts || null,
+      heuristicEvidence: {
+        failureCount: pattern.failureCount,
+        estimatedCounterfactualTotal: Math.round(pattern.estimatedCounterfactualTotal * 100) / 100,
+        loopCount: pattern.loopCount,
+        largeOutputCount: pattern.largeOutputCount,
+        costSource: "estimated",
+      },
+    },
+    intervention: {
+      eventName: "SessionStart",
+      kind: "runtime_route_downgrade",
+      whatHappened: `Wuz automatically started this session on a cheaper-safer route because recent model-driven retries already showed low expected value without measured cost (${detail})`,
+      nextAction: "Stay on the terse route first and prefer deterministic steps until the bounded path proves insufficient",
+      preservedState: "Launch stayed local-first and no tool ran before the route change",
+    },
+  };
+}
+
+function evaluateSessionStartRouteDecision(params = {}) {
+  const requestedProfile = String(params.requestedProfile || "standard");
+  if (!params.config?.smartRouting?.enabled) return null;
+  if (requestedProfile !== "standard") return null;
+
+  const profile = String(params.config?.profile || "balanced");
+  const allMetrics = readAllMetrics(params.cwd);
+  const thresholds = thresholdsForProfile(profile);
+
+  const candidates = [
+    findLatestExpensiveFailure(allMetrics, "Agent", thresholds, { windowMs: SESSION_ROUTE_WINDOW_MS }),
+    findLatestExpensiveFailure(allMetrics, "Skill", thresholds, { windowMs: SESSION_ROUTE_WINDOW_MS }),
+  ].filter(Boolean);
+
+  const measuredFailure = latestByTs(candidates);
+  if (measuredFailure) {
+    return buildMeasuredRouteDecision(String(measuredFailure.metric?.tool || "Agent"), profile, requestedProfile, measuredFailure);
+  }
+
+  const estimatedPattern = findEstimatedLowValuePattern(
+    params.cwd,
+    allMetrics,
+    null,
+    profile,
+    SESSION_ROUTE_WINDOW_MS,
+    "repo"
+  );
+  if (!estimatedPattern) return null;
+
+  return buildEstimatedRouteDecision(profile, requestedProfile, estimatedPattern);
+}
+
+function buildEstimatedRerouteDecision(toolName, profile, pattern) {
+  const failure = pattern.latestFailure;
+  const modelLine = evidenceSummary(failure?.evidence || {});
+  const detailParts = [];
+  if (modelLine) detailParts.push(modelLine);
+  detailParts.push(`${pattern.failureCount} failed attempt${pattern.failureCount === 1 ? "" : "s"}`);
+  detailParts.push(`estimated counterfactual ${Math.round(pattern.estimatedCounterfactualTotal * 100) / 100}`);
+  if (pattern.loopCount > 0) detailParts.push(`${pattern.loopCount} loop signal${pattern.loopCount === 1 ? "" : "s"}`);
+  const detail = detailParts.join(", ");
+
+  return {
+    action: "deny",
+    reasonCodes: ["EVIDENCE_ESTIMATED_LOW_VALUE_REROUTE"],
+    meta: {
+      decisionPolicy: `runtime_estimated_low_value_guard_v${ESTIMATED_REROUTE_POLICY_VERSION}`,
+      evidenceBasis: "estimated",
+      profile,
+      triggerSignals: pattern.triggerSignals,
+      thresholds: pattern.thresholds,
+      routeBefore: "model-driven",
+      routeAfter: "deterministic-first",
+      automatic: true,
+      sourceTool: toolName,
+      priorFailureAt: failure?.metric?.ts || null,
+      heuristicEvidence: {
+        provider: failure?.evidence?.provider || null,
+        model: failure?.evidence?.model || null,
+        failureCount: pattern.failureCount,
+        estimatedCounterfactualTotal: Math.round(pattern.estimatedCounterfactualTotal * 100) / 100,
+        loopCount: pattern.loopCount,
+        largeOutputCount: pattern.largeOutputCount,
+        costSource: "estimated",
+      },
+    },
+    intervention: {
+      eventName: "PreToolUse",
+      kind: "runtime_estimated_low_value_reroute",
+      whatHappened: `Wuz automatically kept this ${toolName} rerun off the model-driven path because recent retries already showed low expected value without measured cost (${detail})`,
+      nextAction: "Use a deterministic or smaller bounded step first, then retry the model-driven path only if that step proves it is still needed",
+      preservedState: `The ${toolName} rerun did not start`,
+    },
+  };
+}
+
+function evaluatePreToolRuntimeDecision(params = {}) {
+  const toolName = String(params.toolName || "");
+  if (!["Agent", "Skill"].includes(toolName)) return null;
+
+  const runtimeEvidence = params.runtimeEvidence || {};
+  const currentLooksModelDriven = toolLooksModelDriven(toolName, runtimeEvidence);
+  if (!currentLooksModelDriven) return null;
+
+  const profile = String(params.config?.profile || "balanced");
+  const thresholds = thresholdsForProfile(profile);
+  const metrics = readSessionMetrics(params.cwd, params.sessionId);
+  const recentFailure = findLatestExpensiveFailure(metrics, toolName, thresholds);
+  if (recentFailure) {
+    const summary = evidenceSummary(recentFailure.evidence);
+    const detail = summary ? ` after using ${summary}` : "";
+    const evidenceRecord = {
+      provider: recentFailure.evidence.provider,
+      model: recentFailure.evidence.model,
+      tokenUsage: recentFailure.evidence.tokenUsage,
+      cost: recentFailure.evidence.cost,
+      latencyMs: recentFailure.evidence.latencyMs,
+      costSource: recentFailure.evidence.cost !== null ? "measured" : "unavailable",
+    };
+
+    return {
+      action: "ask",
+      reasonCodes: ["EVIDENCE_RUNTIME_RETRY_REVIEW"],
+      meta: {
+        decisionPolicy: `runtime_evidence_retry_guard_v${POLICY_VERSION}`,
+        profile,
+        thresholds,
+        triggerSignals: recentFailure.triggers,
+        priorFailureAt: recentFailure.metric.ts || null,
+        priorTool: toolName,
+        priorRuntimeEvidence: evidenceRecord,
+      },
+      intervention: {
+        eventName: "PreToolUse",
+        kind: "runtime_evidence_retry_review",
+        whatHappened: `Wuz paused this ${toolName} rerun because the last attempt failed${detail}`,
+        nextAction: "Narrow scope, prefer a deterministic path if one exists, or confirm the stronger reason for paying this cost before rerunning",
+        preservedState: `The ${toolName} rerun did not start`,
+      },
+    };
+  }
+
+  if (!params.config?.smartRouting?.enabled) return null;
+  const estimatedPattern = findEstimatedLowValuePattern(
+    params.cwd,
+    metrics,
+    toolName,
+    profile,
+    RECENT_FAILURE_WINDOW_MS,
+    "session"
+  );
+  if (!estimatedPattern) return null;
+
+  return buildEstimatedRerouteDecision(toolName, profile, estimatedPattern);
+}
+
+module.exports = {
+  POLICY_VERSION,
+  SESSION_ROUTE_POLICY_VERSION,
+  ESTIMATED_REROUTE_POLICY_VERSION,
+  thresholdsForProfile,
+  heuristicThresholdsForProfile,
+  evaluatePreToolRuntimeDecision,
+  evaluateSessionStartRouteDecision,
+};