avorelo 0.1.0

package/scripts/lib/proof-metrics.js

@@ -0,0 +1,296 @@
+"use strict";
+
+/**
+ * Proof Metrics - 12 named effectiveness metric definitions.
+ */
+
+const { readAllMetrics } = require("./metrics");
+const { composeProofRecord } = require("./proof-record");
+
+const METRIC_DEFINITIONS = {
+  time_to_first_useful_result: {
+    id: "time_to_first_useful_result",
+    name: "Time to First Useful Result",
+    description: "Time from run start to the first truly usable output.",
+    unit: "seconds",
+    dimension: "outcome",
+    derivation: "started_at to first TASK_COMPLETED or meaningful artifact event",
+  },
+  completion_rate: {
+    id: "completion_rate",
+    name: "Completion Rate",
+    description: "Percentage of runs ending in completed task or safe stop with bounded next action.",
+    unit: "percent",
+    dimension: "outcome",
+    derivation: "sessions with canonical completion truth when present, otherwise TASK_COMPLETED or STOP_SNAPSHOT / total sessions",
+  },
+  evidence_backed_completion_rate: {
+    id: "evidence_backed_completion_rate",
+    name: "Evidence-Backed Completion Rate",
+    description:
+      "Of completions, how many have related security/trust evidence artifacts in the same session. This is a proxy for evidence presence, not a verification of task correctness.",
+    unit: "percent",
+    dimension: "outcome",
+    derivation: "canonical proof-aware completions with related evidence / total completions",
+  },
+  retries_per_task: {
+    id: "retries_per_task",
+    name: "Retries Per Task",
+    description: "Average retries, reruns, or recovery loops per task.",
+    unit: "count",
+    dimension: "efficiency",
+    derivation: "proof-record retries_count / sessions",
+  },
+  dead_end_rate: {
+    id: "dead_end_rate",
+    name: "Dead-End Rate",
+    description: "Runs ending without usable output, recovery path, or safe stop.",
+    unit: "percent",
+    dimension: "efficiency",
+    derivation: "sessions without completion, safe stop, or recovery / total sessions",
+  },
+  large_outputs_managed: {
+    id: "large_outputs_managed",
+    name: "Large Outputs Managed",
+    description:
+      "Count of large tool outputs stored out-of-band to reduce context pressure. This is a count, not a precise byte-level reduction measurement.",
+    unit: "count",
+    dimension: "efficiency",
+    derivation: "canonical material steps or CTX_LARGE_OUTPUT counts across sessions",
+  },
+  unnecessary_tool_load_avoided: {
+    id: "unnecessary_tool_load_avoided",
+    name: "Unnecessary Tool Load Avoided",
+    description: "Count of tool loads or scans intentionally skipped because unnecessary.",
+    unit: "count",
+    dimension: "efficiency",
+    derivation: "proof-record tools_skipped",
+  },
+  approval_interruptions_avoided: {
+    id: "approval_interruptions_avoided",
+    name: "Approval Interruptions Avoided",
+    description: "Count of policy-relevant interruptions avoided because execution stayed within policy.",
+    unit: "count",
+    dimension: "control",
+    derivation: "proof-record approval_requests_avoided",
+  },
+  risky_actions_blocked_or_downgraded: {
+    id: "risky_actions_blocked_or_downgraded",
+    name: "Risky Actions Blocked or Downgraded",
+    description: "Count of risky actions blocked, downgraded, or deferred.",
+    unit: "count",
+    dimension: "control",
+    derivation: "proof-record risky action counts, canonical proof first",
+  },
+  honest_fallback_rate: {
+    id: "honest_fallback_rate",
+    name: "Honest Fallback Rate",
+    description: "Rate at which Wuz stopped or limited claims honestly instead of fabricating certainty.",
+    unit: "percent",
+    dimension: "trust",
+    derivation: "safe-stop sessions / total sessions",
+  },
+  recovery_success_rate: {
+    id: "recovery_success_rate",
+    name: "Recovery Success Rate",
+    description: "Of runs that encountered blocks or failures, how many reached successful completion.",
+    unit: "percent",
+    dimension: "trust",
+    derivation: "sessions with retries that still reach completion / retry sessions",
+  },
+  evidence_coverage: {
+    id: "evidence_coverage",
+    name: "Evidence Coverage",
+    description: "Percentage of material sessions with evidence or traceable support.",
+    unit: "percent",
+    dimension: "trust",
+    derivation: "proof-record sessions with related evidence / total sessions",
+  },
+};
+
+function computeProofMetrics(cwd, options = {}) {
+  const days = Number(options.days || 7);
+  const cutoff = Date.now() - days * 24 * 60 * 60 * 1000;
+  const allMetrics = readAllMetrics(cwd);
+  const recent = allMetrics.filter((metric) => {
+    const ts = Date.parse(metric.ts || 0);
+    return Number.isFinite(ts) && ts >= cutoff;
+  });
+
+  const bySession = new Map();
+  recent.forEach((metric) => {
+    const id = metric.sessionId || "unknown";
+    if (!bySession.has(id)) bySession.set(id, []);
+    bySession.get(id).push(metric);
+  });
+
+  const totalSessions = bySession.size;
+  if (totalSessions === 0) return emptyMetrics();
+
+  let completedSessions = 0;
+  let safeStopSessions = 0;
+  let evidenceBackedSessions = 0;
+  let deadEndSessions = 0;
+  let blockedSessions = 0;
+  let blockedThenCompletedSessions = 0;
+  let evidenceSessions = 0;
+  let totalRetries = 0;
+  let totalLargeOutputBytes = 0;
+  let totalLargeOutputEvents = 0;
+  let totalScanSkips = 0;
+  let totalPermFatigueConsolidated = 0;
+  let totalRiskyBlocked = 0;
+  const ttfrValues = [];
+
+  for (const [sessionId, sessionMetrics] of bySession) {
+    const record = composeProofRecord(sessionId, sessionMetrics, []);
+    if (record.completion_status === "completed") completedSessions++;
+    if (record.completion_status === "safe_stop") safeStopSessions++;
+    if (record.outcome?.evidence_backed_completion) evidenceBackedSessions++;
+    if (
+      record.completion_status === "ended_without_completion" &&
+      !record.outcome?.next_bounded_action
+    ) {
+      deadEndSessions++;
+    }
+    if ((record.efficiency?.retries_count || 0) > 0) {
+      blockedSessions++;
+      if (record.completion_status === "completed") blockedThenCompletedSessions++;
+    }
+    if (record.trust?.has_related_evidence) evidenceSessions++;
+
+    totalRetries += record.efficiency?.retries_count || 0;
+    totalLargeOutputBytes += record.efficiency?.large_output_bytes_managed || 0;
+    totalLargeOutputEvents += record.efficiency?.large_output_events || 0;
+    totalScanSkips += record.efficiency?.tools_skipped || 0;
+    totalPermFatigueConsolidated += record.control?.approval_requests_avoided || 0;
+    totalRiskyBlocked +=
+      (record.control?.risky_actions_blocked || 0) +
+      (record.control?.risky_actions_downgraded || 0);
+
+    const timestamps = sessionMetrics
+      .map((metric) => Date.parse(metric.ts || 0))
+      .filter((value) => Number.isFinite(value));
+    const startTs = timestamps.length ? Math.min(...timestamps) : 0;
+    const completionMetric = sessionMetrics.find((metric) =>
+      (metric.reasonCodes || []).includes("TASK_COMPLETED")
+    );
+    if (completionMetric && startTs) {
+      const endTs = Date.parse(completionMetric.ts || 0);
+      if (Number.isFinite(endTs) && endTs > startTs) {
+        ttfrValues.push(Math.round((endTs - startTs) / 1000));
+      }
+    }
+  }
+
+  const completedOrStopped = completedSessions + safeStopSessions;
+  const completionRate = totalSessions > 0 ? Math.round((completedOrStopped / totalSessions) * 100) : 0;
+  const evidenceBackedRate =
+    completedSessions > 0 ? Math.round((evidenceBackedSessions / completedSessions) * 100) : 0;
+  const deadEndRate = totalSessions > 0 ? Math.round((deadEndSessions / totalSessions) * 100) : 0;
+  const retriesPerTask =
+    totalSessions > 0 ? Math.round((totalRetries / totalSessions) * 100) / 100 : 0;
+  const honestFallbackRate =
+    totalSessions > 0 ? Math.round((safeStopSessions / totalSessions) * 100) : 0;
+  const recoverySuccessRate =
+    blockedSessions > 0 ? Math.round((blockedThenCompletedSessions / blockedSessions) * 100) : 0;
+  const evidenceCoverage =
+    totalSessions > 0 ? Math.round((evidenceSessions / totalSessions) * 100) : 0;
+  const medianTtfr = ttfrValues.length > 0 ? median(ttfrValues) : null;
+
+  return {
+    period_days: days,
+    total_sessions: totalSessions,
+    time_to_first_useful_result: {
+      ...METRIC_DEFINITIONS.time_to_first_useful_result,
+      value: medianTtfr,
+      sample_size: ttfrValues.length,
+      confidence: ttfrValues.length >= 5 ? "medium" : "low",
+    },
+    completion_rate: {
+      ...METRIC_DEFINITIONS.completion_rate,
+      value: completionRate,
+      numerator: completedOrStopped,
+      denominator: totalSessions,
+    },
+    evidence_backed_completion_rate: {
+      ...METRIC_DEFINITIONS.evidence_backed_completion_rate,
+      value: evidenceBackedRate,
+      numerator: evidenceBackedSessions,
+      denominator: completedSessions,
+    },
+    retries_per_task: {
+      ...METRIC_DEFINITIONS.retries_per_task,
+      value: retriesPerTask,
+      total_retries: totalRetries,
+      sessions: totalSessions,
+    },
+    dead_end_rate: {
+      ...METRIC_DEFINITIONS.dead_end_rate,
+      value: deadEndRate,
+      numerator: deadEndSessions,
+      denominator: totalSessions,
+    },
+    large_outputs_managed: {
+      ...METRIC_DEFINITIONS.large_outputs_managed,
+      value: totalLargeOutputEvents,
+      total_bytes: totalLargeOutputBytes,
+    },
+    unnecessary_tool_load_avoided: {
+      ...METRIC_DEFINITIONS.unnecessary_tool_load_avoided,
+      value: totalScanSkips,
+    },
+    approval_interruptions_avoided: {
+      ...METRIC_DEFINITIONS.approval_interruptions_avoided,
+      value: totalPermFatigueConsolidated,
+    },
+    risky_actions_blocked_or_downgraded: {
+      ...METRIC_DEFINITIONS.risky_actions_blocked_or_downgraded,
+      value: totalRiskyBlocked,
+    },
+    honest_fallback_rate: {
+      ...METRIC_DEFINITIONS.honest_fallback_rate,
+      value: honestFallbackRate,
+      numerator: safeStopSessions,
+      denominator: totalSessions,
+    },
+    recovery_success_rate: {
+      ...METRIC_DEFINITIONS.recovery_success_rate,
+      value: recoverySuccessRate,
+      numerator: blockedThenCompletedSessions,
+      denominator: blockedSessions,
+      confidence: blockedSessions >= 3 ? "medium" : "low",
+    },
+    evidence_coverage: {
+      ...METRIC_DEFINITIONS.evidence_coverage,
+      value: evidenceCoverage,
+      numerator: evidenceSessions,
+      denominator: totalSessions,
+    },
+  };
+}
+
+function emptyMetrics() {
+  const result = { period_days: 0, total_sessions: 0 };
+  for (const [id, def] of Object.entries(METRIC_DEFINITIONS)) {
+    result[id] = {
+      ...def,
+      value: def.unit === "percent" ? 0 : def.unit === "count" ? 0 : null,
+    };
+  }
+  return result;
+}
+
+function median(arr) {
+  if (!arr.length) return 0;
+  const sorted = arr.slice().sort((a, b) => a - b);
+  const mid = Math.floor(sorted.length / 2);
+  return sorted.length % 2
+    ? sorted[mid]
+    : Math.round((sorted[mid - 1] + sorted[mid]) / 2);
+}
+
+module.exports = {
+  METRIC_DEFINITIONS,
+  computeProofMetrics,
+};

package/scripts/lib/proof-outcome-evidence.js

@@ -0,0 +1,134 @@
+"use strict";
+
+const fs = require("fs");
+const path = require("path");
+const { nowIso } = require("./fsx");
+
+const CONTRACT = "avorelo.proofOutcomeEvidence.v1";
+const SCHEMA_VERSION = 1;
+const PROOF_OUTCOME_DIR_REL = ".claude/cco/orchestration/seamless-outcome";
+const ARTIFACT_REL = `${PROOF_OUTCOME_DIR_REL}/latest-proof-outcome-evidence.json`;
+const VALUE_SUMMARY_REL = `${PROOF_OUTCOME_DIR_REL}/latest-value-summary.json`;
+const REALITY_GATE_REL = `${PROOF_OUTCOME_DIR_REL}/latest-reality-gate.json`;
+
+function safeReadJson(absPath) {
+  try {
+    if (!fs.existsSync(absPath)) return null;
+    return JSON.parse(fs.readFileSync(absPath, "utf8").replace(/^\uFEFF/, ""));
+  } catch {
+    return null;
+  }
+}
+
+function buildProofOutcomeEvidence(cwd) {
+  const valueSummary = safeReadJson(path.join(cwd, VALUE_SUMMARY_REL));
+  const realityGate = safeReadJson(path.join(cwd, REALITY_GATE_REL));
+
+  const latestProofAvailable = !!valueSummary;
+  const simulatedProofDetected = valueSummary && valueSummary.simulated === true;
+  const realTaskProofCount = valueSummary && simulatedProofDetected !== true ? 1 : 0;
+  const evidenceBacked = valueSummary ? valueSummary.evidenceBacked !== false : false;
+
+  let status = "warn";
+  let proofQuality = "missing";
+  const missingEvidence = [];
+  const safeNextActions = [];
+
+  if (!valueSummary) {
+    missingEvidence.push("No proof/value summary artifact is available from a completed real task.");
+    safeNextActions.push("Run a bounded real task in this repo.");
+    safeNextActions.push("Run: node bin/avorelo proof");
+    safeNextActions.push("Run: node bin/avorelo full-readiness --json");
+  } else {
+    proofQuality = evidenceBacked ? "evidence_backed" : "present_but_unverified";
+    if (simulatedProofDetected) {
+      proofQuality = "simulated_fixture_only";
+      missingEvidence.push("Latest proof artifact is marked simulated and does not count as real task evidence.");
+    }
+    if (!evidenceBacked) {
+      missingEvidence.push("Latest proof/value summary is present but not marked evidence-backed.");
+    }
+    if (!realityGate) {
+      missingEvidence.push("No reality gate artifact is available alongside the latest proof summary.");
+      safeNextActions.push("Run: node bin/avorelo outcome --gate --json");
+    }
+    if (evidenceBacked && realityGate && simulatedProofDetected !== true) {
+      status = "pass";
+    }
+  }
+
+  if (safeNextActions.length === 0) {
+    safeNextActions.push("Run: node bin/avorelo full-readiness --json");
+  }
+
+  return {
+    contract: CONTRACT,
+    schemaVersion: SCHEMA_VERSION,
+    createdAt: nowIso(),
+    status,
+    latestProofAvailable,
+    realTaskProofCount,
+    simulatedProofDetected: simulatedProofDetected === true,
+    proofQuality,
+    evidenceRefs: {
+      valueSummary: latestProofAvailable ? VALUE_SUMMARY_REL : null,
+      realityGate: realityGate ? REALITY_GATE_REL : null,
+    },
+    missingEvidence,
+    safeNextActions,
+    redacted: true,
+  };
+}
+
+function writeProofOutcomeEvidence(cwd, evidence) {
+  const dirAbs = path.join(cwd, PROOF_OUTCOME_DIR_REL);
+  fs.mkdirSync(dirAbs, { recursive: true });
+  fs.writeFileSync(path.join(cwd, ARTIFACT_REL), JSON.stringify(evidence, null, 2), "utf8");
+  return path.join(cwd, ARTIFACT_REL);
+}
+
+function buildProofOutcomeEvidenceSurface(cwd) {
+  const existing = safeReadJson(path.join(cwd, ARTIFACT_REL));
+  const evidence = existing || buildProofOutcomeEvidence(cwd);
+  if (!existing) writeProofOutcomeEvidence(cwd, evidence);
+  return {
+    status: evidence.status,
+    latestProofAvailable: evidence.latestProofAvailable,
+    realTaskProofCount: evidence.realTaskProofCount,
+    proofQuality: evidence.proofQuality,
+    artifactPath: ARTIFACT_REL,
+    redacted: true,
+  };
+}
+
+function formatProofOutcomeEvidenceText(evidence) {
+  const lines = [
+    `Proof outcome evidence: ${evidence.status}`,
+    `Latest proof available: ${evidence.latestProofAvailable === true ? "yes" : "no"}`,
+    `Real task proof count: ${evidence.realTaskProofCount || 0}`,
+    `Proof quality: ${evidence.proofQuality || "missing"}`,
+  ];
+  if (Array.isArray(evidence.missingEvidence) && evidence.missingEvidence.length) {
+    lines.push("");
+    lines.push("Missing evidence:");
+    evidence.missingEvidence.slice(0, 5).forEach((item) => lines.push(`  - ${item}`));
+  }
+  if (Array.isArray(evidence.safeNextActions) && evidence.safeNextActions.length) {
+    lines.push("");
+    lines.push("Next:");
+    evidence.safeNextActions.slice(0, 3).forEach((item) => lines.push(`  - ${item}`));
+  }
+  return lines.join("\n");
+}
+
+module.exports = {
+  CONTRACT,
+  SCHEMA_VERSION,
+  ARTIFACT_REL,
+  VALUE_SUMMARY_REL,
+  REALITY_GATE_REL,
+  buildProofOutcomeEvidence,
+  writeProofOutcomeEvidence,
+  buildProofOutcomeEvidenceSurface,
+  formatProofOutcomeEvidenceText,
+};