avorelo 0.1.0 → 0.3.0

package/scripts/lib/proof-metrics.js

@@ -1,296 +0,0 @@
-"use strict";
-
-/**
- * Proof Metrics - 12 named effectiveness metric definitions.
- */
-
-const { readAllMetrics } = require("./metrics");
-const { composeProofRecord } = require("./proof-record");
-
-const METRIC_DEFINITIONS = {
-  time_to_first_useful_result: {
-    id: "time_to_first_useful_result",
-    name: "Time to First Useful Result",
-    description: "Time from run start to the first truly usable output.",
-    unit: "seconds",
-    dimension: "outcome",
-    derivation: "started_at to first TASK_COMPLETED or meaningful artifact event",
-  },
-  completion_rate: {
-    id: "completion_rate",
-    name: "Completion Rate",
-    description: "Percentage of runs ending in completed task or safe stop with bounded next action.",
-    unit: "percent",
-    dimension: "outcome",
-    derivation: "sessions with canonical completion truth when present, otherwise TASK_COMPLETED or STOP_SNAPSHOT / total sessions",
-  },
-  evidence_backed_completion_rate: {
-    id: "evidence_backed_completion_rate",
-    name: "Evidence-Backed Completion Rate",
-    description:
-      "Of completions, how many have related security/trust evidence artifacts in the same session. This is a proxy for evidence presence, not a verification of task correctness.",
-    unit: "percent",
-    dimension: "outcome",
-    derivation: "canonical proof-aware completions with related evidence / total completions",
-  },
-  retries_per_task: {
-    id: "retries_per_task",
-    name: "Retries Per Task",
-    description: "Average retries, reruns, or recovery loops per task.",
-    unit: "count",
-    dimension: "efficiency",
-    derivation: "proof-record retries_count / sessions",
-  },
-  dead_end_rate: {
-    id: "dead_end_rate",
-    name: "Dead-End Rate",
-    description: "Runs ending without usable output, recovery path, or safe stop.",
-    unit: "percent",
-    dimension: "efficiency",
-    derivation: "sessions without completion, safe stop, or recovery / total sessions",
-  },
-  large_outputs_managed: {
-    id: "large_outputs_managed",
-    name: "Large Outputs Managed",
-    description:
-      "Count of large tool outputs stored out-of-band to reduce context pressure. This is a count, not a precise byte-level reduction measurement.",
-    unit: "count",
-    dimension: "efficiency",
-    derivation: "canonical material steps or CTX_LARGE_OUTPUT counts across sessions",
-  },
-  unnecessary_tool_load_avoided: {
-    id: "unnecessary_tool_load_avoided",
-    name: "Unnecessary Tool Load Avoided",
-    description: "Count of tool loads or scans intentionally skipped because unnecessary.",
-    unit: "count",
-    dimension: "efficiency",
-    derivation: "proof-record tools_skipped",
-  },
-  approval_interruptions_avoided: {
-    id: "approval_interruptions_avoided",
-    name: "Approval Interruptions Avoided",
-    description: "Count of policy-relevant interruptions avoided because execution stayed within policy.",
-    unit: "count",
-    dimension: "control",
-    derivation: "proof-record approval_requests_avoided",
-  },
-  risky_actions_blocked_or_downgraded: {
-    id: "risky_actions_blocked_or_downgraded",
-    name: "Risky Actions Blocked or Downgraded",
-    description: "Count of risky actions blocked, downgraded, or deferred.",
-    unit: "count",
-    dimension: "control",
-    derivation: "proof-record risky action counts, canonical proof first",
-  },
-  honest_fallback_rate: {
-    id: "honest_fallback_rate",
-    name: "Honest Fallback Rate",
-    description: "Rate at which Wuz stopped or limited claims honestly instead of fabricating certainty.",
-    unit: "percent",
-    dimension: "trust",
-    derivation: "safe-stop sessions / total sessions",
-  },
-  recovery_success_rate: {
-    id: "recovery_success_rate",
-    name: "Recovery Success Rate",
-    description: "Of runs that encountered blocks or failures, how many reached successful completion.",
-    unit: "percent",
-    dimension: "trust",
-    derivation: "sessions with retries that still reach completion / retry sessions",
-  },
-  evidence_coverage: {
-    id: "evidence_coverage",
-    name: "Evidence Coverage",
-    description: "Percentage of material sessions with evidence or traceable support.",
-    unit: "percent",
-    dimension: "trust",
-    derivation: "proof-record sessions with related evidence / total sessions",
-  },
-};
-
-function computeProofMetrics(cwd, options = {}) {
-  const days = Number(options.days || 7);
-  const cutoff = Date.now() - days * 24 * 60 * 60 * 1000;
-  const allMetrics = readAllMetrics(cwd);
-  const recent = allMetrics.filter((metric) => {
-    const ts = Date.parse(metric.ts || 0);
-    return Number.isFinite(ts) && ts >= cutoff;
-  });
-
-  const bySession = new Map();
-  recent.forEach((metric) => {
-    const id = metric.sessionId || "unknown";
-    if (!bySession.has(id)) bySession.set(id, []);
-    bySession.get(id).push(metric);
-  });
-
-  const totalSessions = bySession.size;
-  if (totalSessions === 0) return emptyMetrics();
-
-  let completedSessions = 0;
-  let safeStopSessions = 0;
-  let evidenceBackedSessions = 0;
-  let deadEndSessions = 0;
-  let blockedSessions = 0;
-  let blockedThenCompletedSessions = 0;
-  let evidenceSessions = 0;
-  let totalRetries = 0;
-  let totalLargeOutputBytes = 0;
-  let totalLargeOutputEvents = 0;
-  let totalScanSkips = 0;
-  let totalPermFatigueConsolidated = 0;
-  let totalRiskyBlocked = 0;
-  const ttfrValues = [];
-
-  for (const [sessionId, sessionMetrics] of bySession) {
-    const record = composeProofRecord(sessionId, sessionMetrics, []);
-    if (record.completion_status === "completed") completedSessions++;
-    if (record.completion_status === "safe_stop") safeStopSessions++;
-    if (record.outcome?.evidence_backed_completion) evidenceBackedSessions++;
-    if (
-      record.completion_status === "ended_without_completion" &&
-      !record.outcome?.next_bounded_action
-    ) {
-      deadEndSessions++;
-    }
-    if ((record.efficiency?.retries_count || 0) > 0) {
-      blockedSessions++;
-      if (record.completion_status === "completed") blockedThenCompletedSessions++;
-    }
-    if (record.trust?.has_related_evidence) evidenceSessions++;
-
-    totalRetries += record.efficiency?.retries_count || 0;
-    totalLargeOutputBytes += record.efficiency?.large_output_bytes_managed || 0;
-    totalLargeOutputEvents += record.efficiency?.large_output_events || 0;
-    totalScanSkips += record.efficiency?.tools_skipped || 0;
-    totalPermFatigueConsolidated += record.control?.approval_requests_avoided || 0;
-    totalRiskyBlocked +=
-      (record.control?.risky_actions_blocked || 0) +
-      (record.control?.risky_actions_downgraded || 0);
-
-    const timestamps = sessionMetrics
-      .map((metric) => Date.parse(metric.ts || 0))
-      .filter((value) => Number.isFinite(value));
-    const startTs = timestamps.length ? Math.min(...timestamps) : 0;
-    const completionMetric = sessionMetrics.find((metric) =>
-      (metric.reasonCodes || []).includes("TASK_COMPLETED")
-    );
-    if (completionMetric && startTs) {
-      const endTs = Date.parse(completionMetric.ts || 0);
-      if (Number.isFinite(endTs) && endTs > startTs) {
-        ttfrValues.push(Math.round((endTs - startTs) / 1000));
-      }
-    }
-  }
-
-  const completedOrStopped = completedSessions + safeStopSessions;
-  const completionRate = totalSessions > 0 ? Math.round((completedOrStopped / totalSessions) * 100) : 0;
-  const evidenceBackedRate =
-    completedSessions > 0 ? Math.round((evidenceBackedSessions / completedSessions) * 100) : 0;
-  const deadEndRate = totalSessions > 0 ? Math.round((deadEndSessions / totalSessions) * 100) : 0;
-  const retriesPerTask =
-    totalSessions > 0 ? Math.round((totalRetries / totalSessions) * 100) / 100 : 0;
-  const honestFallbackRate =
-    totalSessions > 0 ? Math.round((safeStopSessions / totalSessions) * 100) : 0;
-  const recoverySuccessRate =
-    blockedSessions > 0 ? Math.round((blockedThenCompletedSessions / blockedSessions) * 100) : 0;
-  const evidenceCoverage =
-    totalSessions > 0 ? Math.round((evidenceSessions / totalSessions) * 100) : 0;
-  const medianTtfr = ttfrValues.length > 0 ? median(ttfrValues) : null;
-
-  return {
-    period_days: days,
-    total_sessions: totalSessions,
-    time_to_first_useful_result: {
-      ...METRIC_DEFINITIONS.time_to_first_useful_result,
-      value: medianTtfr,
-      sample_size: ttfrValues.length,
-      confidence: ttfrValues.length >= 5 ? "medium" : "low",
-    },
-    completion_rate: {
-      ...METRIC_DEFINITIONS.completion_rate,
-      value: completionRate,
-      numerator: completedOrStopped,
-      denominator: totalSessions,
-    },
-    evidence_backed_completion_rate: {
-      ...METRIC_DEFINITIONS.evidence_backed_completion_rate,
-      value: evidenceBackedRate,
-      numerator: evidenceBackedSessions,
-      denominator: completedSessions,
-    },
-    retries_per_task: {
-      ...METRIC_DEFINITIONS.retries_per_task,
-      value: retriesPerTask,
-      total_retries: totalRetries,
-      sessions: totalSessions,
-    },
-    dead_end_rate: {
-      ...METRIC_DEFINITIONS.dead_end_rate,
-      value: deadEndRate,
-      numerator: deadEndSessions,
-      denominator: totalSessions,
-    },
-    large_outputs_managed: {
-      ...METRIC_DEFINITIONS.large_outputs_managed,
-      value: totalLargeOutputEvents,
-      total_bytes: totalLargeOutputBytes,
-    },
-    unnecessary_tool_load_avoided: {
-      ...METRIC_DEFINITIONS.unnecessary_tool_load_avoided,
-      value: totalScanSkips,
-    },
-    approval_interruptions_avoided: {
-      ...METRIC_DEFINITIONS.approval_interruptions_avoided,
-      value: totalPermFatigueConsolidated,
-    },
-    risky_actions_blocked_or_downgraded: {
-      ...METRIC_DEFINITIONS.risky_actions_blocked_or_downgraded,
-      value: totalRiskyBlocked,
-    },
-    honest_fallback_rate: {
-      ...METRIC_DEFINITIONS.honest_fallback_rate,
-      value: honestFallbackRate,
-      numerator: safeStopSessions,
-      denominator: totalSessions,
-    },
-    recovery_success_rate: {
-      ...METRIC_DEFINITIONS.recovery_success_rate,
-      value: recoverySuccessRate,
-      numerator: blockedThenCompletedSessions,
-      denominator: blockedSessions,
-      confidence: blockedSessions >= 3 ? "medium" : "low",
-    },
-    evidence_coverage: {
-      ...METRIC_DEFINITIONS.evidence_coverage,
-      value: evidenceCoverage,
-      numerator: evidenceSessions,
-      denominator: totalSessions,
-    },
-  };
-}
-
-function emptyMetrics() {
-  const result = { period_days: 0, total_sessions: 0 };
-  for (const [id, def] of Object.entries(METRIC_DEFINITIONS)) {
-    result[id] = {
-      ...def,
-      value: def.unit === "percent" ? 0 : def.unit === "count" ? 0 : null,
-    };
-  }
-  return result;
-}
-
-function median(arr) {
-  if (!arr.length) return 0;
-  const sorted = arr.slice().sort((a, b) => a - b);
-  const mid = Math.floor(sorted.length / 2);
-  return sorted.length % 2
-    ? sorted[mid]
-    : Math.round((sorted[mid - 1] + sorted[mid]) / 2);
-}
-
-module.exports = {
-  METRIC_DEFINITIONS,
-  computeProofMetrics,
-};

package/scripts/lib/proof-outcome-evidence.js

@@ -1,134 +0,0 @@
-"use strict";
-
-const fs = require("fs");
-const path = require("path");
-const { nowIso } = require("./fsx");
-
-const CONTRACT = "avorelo.proofOutcomeEvidence.v1";
-const SCHEMA_VERSION = 1;
-const PROOF_OUTCOME_DIR_REL = ".claude/cco/orchestration/seamless-outcome";
-const ARTIFACT_REL = `${PROOF_OUTCOME_DIR_REL}/latest-proof-outcome-evidence.json`;
-const VALUE_SUMMARY_REL = `${PROOF_OUTCOME_DIR_REL}/latest-value-summary.json`;
-const REALITY_GATE_REL = `${PROOF_OUTCOME_DIR_REL}/latest-reality-gate.json`;
-
-function safeReadJson(absPath) {
-  try {
-    if (!fs.existsSync(absPath)) return null;
-    return JSON.parse(fs.readFileSync(absPath, "utf8").replace(/^\uFEFF/, ""));
-  } catch {
-    return null;
-  }
-}
-
-function buildProofOutcomeEvidence(cwd) {
-  const valueSummary = safeReadJson(path.join(cwd, VALUE_SUMMARY_REL));
-  const realityGate = safeReadJson(path.join(cwd, REALITY_GATE_REL));
-
-  const latestProofAvailable = !!valueSummary;
-  const simulatedProofDetected = valueSummary && valueSummary.simulated === true;
-  const realTaskProofCount = valueSummary && simulatedProofDetected !== true ? 1 : 0;
-  const evidenceBacked = valueSummary ? valueSummary.evidenceBacked !== false : false;
-
-  let status = "warn";
-  let proofQuality = "missing";
-  const missingEvidence = [];
-  const safeNextActions = [];
-
-  if (!valueSummary) {
-    missingEvidence.push("No proof/value summary artifact is available from a completed real task.");
-    safeNextActions.push("Run a bounded real task in this repo.");
-    safeNextActions.push("Run: node bin/avorelo proof");
-    safeNextActions.push("Run: node bin/avorelo full-readiness --json");
-  } else {
-    proofQuality = evidenceBacked ? "evidence_backed" : "present_but_unverified";
-    if (simulatedProofDetected) {
-      proofQuality = "simulated_fixture_only";
-      missingEvidence.push("Latest proof artifact is marked simulated and does not count as real task evidence.");
-    }
-    if (!evidenceBacked) {
-      missingEvidence.push("Latest proof/value summary is present but not marked evidence-backed.");
-    }
-    if (!realityGate) {
-      missingEvidence.push("No reality gate artifact is available alongside the latest proof summary.");
-      safeNextActions.push("Run: node bin/avorelo outcome --gate --json");
-    }
-    if (evidenceBacked && realityGate && simulatedProofDetected !== true) {
-      status = "pass";
-    }
-  }
-
-  if (safeNextActions.length === 0) {
-    safeNextActions.push("Run: node bin/avorelo full-readiness --json");
-  }
-
-  return {
-    contract: CONTRACT,
-    schemaVersion: SCHEMA_VERSION,
-    createdAt: nowIso(),
-    status,
-    latestProofAvailable,
-    realTaskProofCount,
-    simulatedProofDetected: simulatedProofDetected === true,
-    proofQuality,
-    evidenceRefs: {
-      valueSummary: latestProofAvailable ? VALUE_SUMMARY_REL : null,
-      realityGate: realityGate ? REALITY_GATE_REL : null,
-    },
-    missingEvidence,
-    safeNextActions,
-    redacted: true,
-  };
-}
-
-function writeProofOutcomeEvidence(cwd, evidence) {
-  const dirAbs = path.join(cwd, PROOF_OUTCOME_DIR_REL);
-  fs.mkdirSync(dirAbs, { recursive: true });
-  fs.writeFileSync(path.join(cwd, ARTIFACT_REL), JSON.stringify(evidence, null, 2), "utf8");
-  return path.join(cwd, ARTIFACT_REL);
-}
-
-function buildProofOutcomeEvidenceSurface(cwd) {
-  const existing = safeReadJson(path.join(cwd, ARTIFACT_REL));
-  const evidence = existing || buildProofOutcomeEvidence(cwd);
-  if (!existing) writeProofOutcomeEvidence(cwd, evidence);
-  return {
-    status: evidence.status,
-    latestProofAvailable: evidence.latestProofAvailable,
-    realTaskProofCount: evidence.realTaskProofCount,
-    proofQuality: evidence.proofQuality,
-    artifactPath: ARTIFACT_REL,
-    redacted: true,
-  };
-}
-
-function formatProofOutcomeEvidenceText(evidence) {
-  const lines = [
-    `Proof outcome evidence: ${evidence.status}`,
-    `Latest proof available: ${evidence.latestProofAvailable === true ? "yes" : "no"}`,
-    `Real task proof count: ${evidence.realTaskProofCount || 0}`,
-    `Proof quality: ${evidence.proofQuality || "missing"}`,
-  ];
-  if (Array.isArray(evidence.missingEvidence) && evidence.missingEvidence.length) {
-    lines.push("");
-    lines.push("Missing evidence:");
-    evidence.missingEvidence.slice(0, 5).forEach((item) => lines.push(`  - ${item}`));
-  }
-  if (Array.isArray(evidence.safeNextActions) && evidence.safeNextActions.length) {
-    lines.push("");
-    lines.push("Next:");
-    evidence.safeNextActions.slice(0, 3).forEach((item) => lines.push(`  - ${item}`));
-  }
-  return lines.join("\n");
-}
-
-module.exports = {
-  CONTRACT,
-  SCHEMA_VERSION,
-  ARTIFACT_REL,
-  VALUE_SUMMARY_REL,
-  REALITY_GATE_REL,
-  buildProofOutcomeEvidence,
-  writeProofOutcomeEvidence,
-  buildProofOutcomeEvidenceSurface,
-  formatProofOutcomeEvidenceText,
-};