avorelo 0.1.0

package/scripts/lib/token-context-quality-gate.js

@@ -0,0 +1,370 @@
+"use strict";
+
+// ── Token Context Quality Gate ────────────────────────────────────────────────
+//
+// Validates that token reduction does not hurt task quality, proof, safety,
+// debuggability, or seamless UX.
+//
+// Contract: avorelo.tokenContextQualityGate.v1
+//
+// Pass: context bounded, proof available, no raw dump, secrets excluded,
+//       fallback available, debug artifacts available.
+// Warn: estimate unavailable but context bounded, repo map partial,
+//       cache readiness unknown, proof unavailable but next action clear.
+// Fail: full repo dump, raw huge output injected, proof hidden by compression,
+//       support bundle leaks raw code/secrets.
+
+const fs = require("node:fs");
+const path = require("node:path");
+const { nowIso } = require("./fsx");
+const { appendProductLearningEvent } = require("./product-learning-events");
+
+const CONTRACT = "avorelo.tokenContextQualityGate.v1";
+const SCHEMA_VERSION = 1;
+
+const QUALITY_GATE_DIR_REL = ".claude/cco/orchestration/token-efficiency";
+const LATEST_GATE_REL = `${QUALITY_GATE_DIR_REL}/latest-quality-gate.json`;
+
+// Thresholds
+const MAX_CONTEXT_TOKENS_DEFAULT = 16384;
+const FULL_REPO_TOKEN_THRESHOLD = 50000; // above this with no selection = dump
+
+// ── Individual checks ─────────────────────────────────────────────────────────
+
+function checkNoFullRepoDump(cwd, signals) {
+  const id = "no_full_repo_dump_by_default";
+  const { contextPlanStrategy, estimatedContextTokens } = signals;
+
+  if (contextPlanStrategy && !contextPlanStrategy.useFullFiles && !contextPlanStrategy.conservative) {
+    // Context plan explicitly avoids full files
+    return { id, status: "pass", detail: "Context plan does not load full repo by default." };
+  }
+  if (estimatedContextTokens && estimatedContextTokens > FULL_REPO_TOKEN_THRESHOLD) {
+    return { id, status: "fail", detail: `Estimated context ${estimatedContextTokens} tokens exceeds full-repo threshold (${FULL_REPO_TOKEN_THRESHOLD}). Full repo dump suspected.` };
+  }
+  if (!contextPlanStrategy) {
+    return { id, status: "warn", detail: "No context plan available. Cannot confirm no-full-repo-dump." };
+  }
+  return { id, status: "pass", detail: "Context tokens within expected range. No full repo dump detected." };
+}
+
+function checkRawToolOutputNotInjected(cwd, signals) {
+  const id = "raw_tool_output_not_injected";
+  const { toolOutputSandboxSummary } = signals;
+
+  if (!toolOutputSandboxSummary) {
+    return { id, status: "warn", detail: "No tool output sandbox summary. Cannot confirm raw output excluded from context." };
+  }
+  if (toolOutputSandboxSummary.oversized && !toolOutputSandboxSummary.artifactRef) {
+    return { id, status: "fail", detail: "Oversized tool output has no artifact ref — raw output may have entered context." };
+  }
+  return { id, status: "pass", detail: "Tool output sandbox summary available. Raw artifacts stored separately." };
+}
+
+function checkContextBounded(cwd, signals) {
+  const id = "selected_context_bounded";
+  const { estimatedContextTokens, contextBudget } = signals;
+
+  // estimatedContextTokens===0 with withinBudget:true means context plan ran but selected nothing —
+  // that is the smallest possible (and fully bounded) context; treat it as passing.
+  if (contextBudget && contextBudget.withinBudget === true && estimatedContextTokens != null) {
+    const max = contextBudget.maxTokens || MAX_CONTEXT_TOKENS_DEFAULT;
+    return { id, status: "pass", detail: `Context within budget: ~${estimatedContextTokens}/${max} tokens (withinBudget confirmed).` };
+  }
+  if (!estimatedContextTokens) {
+    return { id, status: "warn", detail: "Context token estimate not available. Cannot confirm bounded." };
+  }
+  const max = (contextBudget && contextBudget.maxTokens) || MAX_CONTEXT_TOKENS_DEFAULT;
+  if (estimatedContextTokens > max) {
+    return { id, status: "fail", detail: `Context estimated at ${estimatedContextTokens} tokens exceeds budget ${max}.` };
+  }
+  return { id, status: "pass", detail: `Context within budget: ~${estimatedContextTokens}/${max} tokens.` };
+}
+
+function checkProofAvailable(cwd, signals) {
+  const id = "proof_available_or_next_action_clear";
+  const { proofAvailable, proofNextAction } = signals;
+
+  if (proofAvailable === true) {
+    return { id, status: "pass", detail: "Proof artifact available." };
+  }
+  if (proofNextAction) {
+    return { id, status: "warn", detail: `Proof not yet run. Next action: ${proofNextAction}` };
+  }
+  return { id, status: "warn", detail: "Proof not available and no clear next action. Run `avorelo proof` after task." };
+}
+
+function checkRequiredEvidencePreserved(cwd, signals) {
+  const id = "required_evidence_preserved";
+  const { evidenceLevel, tokenEvidencePath } = signals;
+
+  if (!evidenceLevel || evidenceLevel === "not_available") {
+    return { id, status: "warn", detail: "Token efficiency evidence not available. Run after task to generate." };
+  }
+  return { id, status: "pass", detail: `Token efficiency evidence available (level: ${evidenceLevel}).` };
+}
+
+function checkFallbackAvailable(cwd, signals) {
+  const id = "fallback_available";
+  const { contextPlanStrategy, repoMapStatus } = signals;
+
+  if (contextPlanStrategy && contextPlanStrategy.conservative) {
+    return { id, status: "pass", detail: "Conservative strategy provides implicit fallback." };
+  }
+  if (repoMapStatus && repoMapStatus !== "error") {
+    return { id, status: "pass", detail: "Repo map available as fallback for context selection." };
+  }
+  return { id, status: "warn", detail: "No explicit fallback strategy confirmed. Repo map or conservative strategy recommended." };
+}
+
+function checkSecretsExcluded(cwd, signals) {
+  const id = "secrets_excluded";
+  const { repoMapSensitiveExcluded, contextPlanSecretsExcluded } = signals;
+
+  if (repoMapSensitiveExcluded === true || contextPlanSecretsExcluded === true) {
+    return { id, status: "pass", detail: "Sensitive/secret files excluded from context by repo map or plan." };
+  }
+  if (repoMapSensitiveExcluded === undefined && contextPlanSecretsExcluded === undefined) {
+    return { id, status: "warn", detail: "Cannot confirm secrets excluded. Ensure .env and sensitive files are not in context." };
+  }
+  return { id, status: "pass", detail: "Secrets exclusion confirmed." };
+}
+
+function checkDebugArtifactsAvailable(cwd, signals) {
+  const id = "debug_artifacts_available";
+  const toolOutputDir = path.join(cwd, ".claude/cco/orchestration/tool-output");
+  const hasToolArtifacts = fs.existsSync(toolOutputDir);
+  const supportBundlePath = path.join(cwd, ".claude/cco/support/latest-support-bundle.json");
+  const hasSupportBundle = fs.existsSync(supportBundlePath);
+
+  if (hasSupportBundle) {
+    return { id, status: "pass", detail: "Support bundle available for debug." };
+  }
+  if (hasToolArtifacts) {
+    return { id, status: "pass", detail: "Tool output artifacts available for debug." };
+  }
+  return { id, status: "warn", detail: "No debug artifacts found. Run `avorelo support-bundle` to generate." };
+}
+
+function checkSupportBundleNotLeaking(cwd, signals) {
+  const id = "support_bundle_not_leaking_raw";
+  const { supportBundleRedacted } = signals;
+
+  if (supportBundleRedacted === false) {
+    return { id, status: "fail", detail: "Support bundle is NOT marked redacted. Raw code/secrets may be exposed." };
+  }
+  // Check actual bundle file
+  try {
+    const bundlePath = path.join(cwd, ".claude/cco/support/latest-support-bundle.json");
+    if (fs.existsSync(bundlePath)) {
+      const bundle = JSON.parse(fs.readFileSync(bundlePath, "utf8"));
+      if (bundle.redacted === true) {
+        return { id, status: "pass", detail: "Support bundle is marked redacted." };
+      }
+      return { id, status: "warn", detail: "Support bundle exists but redacted flag not set." };
+    }
+  } catch {}
+  return { id, status: "pass", detail: "No support bundle generated yet; no leak possible." };
+}
+
+// ── Gate runner ───────────────────────────────────────────────────────────────
+
+function runTokenContextQualityGate(cwd, input = {}, options = {}) {
+  const {
+    contextPlanPath,
+    tokenEvidencePath,
+    toolOutputSandboxPath,
+    repoMapPath,
+    cacheReadinessPath,
+  } = input;
+
+  // Load signals from artifacts
+  const signals = {};
+
+  try {
+    const ep = path.join(cwd, tokenEvidencePath || ".claude/cco/orchestration/token-efficiency/latest-evidence.json");
+    if (fs.existsSync(ep)) {
+      const ev = JSON.parse(fs.readFileSync(ep, "utf8"));
+      signals.evidenceLevel = ev.evidenceLevel;
+      signals.estimatedContextTokens = ev.selectedContextTokens;
+      signals.tokenEvidencePath = tokenEvidencePath || ".claude/cco/orchestration/token-efficiency/latest-evidence.json";
+    }
+  } catch {}
+
+  try {
+    const cp = path.join(cwd, contextPlanPath || ".claude/cco/orchestration/context-acquisition/latest-plan.json");
+    if (fs.existsSync(cp)) {
+      const plan = JSON.parse(fs.readFileSync(cp, "utf8"));
+      signals.contextPlanStrategy = plan.strategy;
+      signals.contextBudget = plan.contextBudget;
+      if (plan.excluded) {
+        signals.contextPlanSecretsExcluded = plan.excluded.some((e) => e.reason && e.reason.includes("sensitive"));
+      }
+      if (!signals.estimatedContextTokens && plan.contextBudget) {
+        signals.estimatedContextTokens = plan.contextBudget.estimatedSelected;
+      }
+    }
+  } catch {}
+
+  try {
+    const tp = path.join(cwd, toolOutputSandboxPath || ".claude/cco/orchestration/tool-output/latest-summary.json");
+    if (fs.existsSync(tp)) {
+      signals.toolOutputSandboxSummary = JSON.parse(fs.readFileSync(tp, "utf8"));
+    }
+  } catch {}
+
+  try {
+    const rp = path.join(cwd, repoMapPath || ".claude/cco/orchestration/repo-map/latest-map.json");
+    if (fs.existsSync(rp)) {
+      const rm = JSON.parse(fs.readFileSync(rp, "utf8"));
+      signals.repoMapStatus = rm.status;
+      signals.repoMapSensitiveExcluded = rm.sensitiveExcluded > 0;
+    }
+  } catch {}
+
+  // Proof availability
+  try {
+    const proofPath = path.join(cwd, ".claude/cco/orchestration/safe-run/latest-run.json");
+    if (fs.existsSync(proofPath)) {
+      const run = JSON.parse(fs.readFileSync(proofPath, "utf8"));
+      signals.proofAvailable = run.decision === "prepared" || run.decision === "completed";
+    }
+  } catch {}
+
+  // Check support bundle redaction signal
+  try {
+    const bundlePath = path.join(cwd, ".claude/cco/support/latest-support-bundle.json");
+    if (fs.existsSync(bundlePath)) {
+      const bundle = JSON.parse(fs.readFileSync(bundlePath, "utf8"));
+      signals.supportBundleRedacted = bundle.redacted === true;
+    }
+  } catch {}
+
+  const checks = [
+    checkNoFullRepoDump(cwd, signals),
+    checkRawToolOutputNotInjected(cwd, signals),
+    checkContextBounded(cwd, signals),
+    checkProofAvailable(cwd, signals),
+    checkRequiredEvidencePreserved(cwd, signals),
+    checkFallbackAvailable(cwd, signals),
+    checkSecretsExcluded(cwd, signals),
+    checkDebugArtifactsAvailable(cwd, signals),
+    checkSupportBundleNotLeaking(cwd, signals),
+  ];
+
+  const failed = checks.filter((c) => c.status === "fail");
+  const warned = checks.filter((c) => c.status === "warn");
+
+  let status, verdict;
+  if (failed.length > 0) {
+    status = "fail";
+    verdict = "failed";
+  } else if (warned.length > 3) {
+    status = "warn";
+    verdict = "weak";
+  } else if (warned.length > 0) {
+    status = "warn";
+    verdict = "acceptable";
+  } else {
+    status = "pass";
+    verdict = "strong";
+  }
+
+  const nextAction = failed.length > 0
+    ? `Fix failures: ${failed.map((c) => c.id).join(", ")}`
+    : warned.length > 0
+    ? `Review warnings: ${warned.map((c) => c.id).join(", ")}`
+    : "Token context quality gate passed. Proceed.";
+
+  return {
+    contract: CONTRACT,
+    schemaVersion: SCHEMA_VERSION,
+    createdAt: nowIso(),
+    status,
+    verdict,
+    context: {
+      estimatedTokens: signals.estimatedContextTokens || null,
+      bounded: signals.contextBudget ? signals.contextBudget.withinBudget : null,
+    },
+    quality: {
+      checksPass: checks.filter((c) => c.status === "pass").length,
+      checksWarn: warned.length,
+      checksFail: failed.length,
+    },
+    latency: { evidenceLevel: "not_available" },
+    safety: {
+      secretsExcluded: signals.repoMapSensitiveExcluded || signals.contextPlanSecretsExcluded || null,
+      supportBundleRedacted: signals.supportBundleRedacted || null,
+    },
+    checks,
+    evidenceRefs: [
+      signals.tokenEvidencePath,
+      contextPlanPath || ".claude/cco/orchestration/context-acquisition/latest-plan.json",
+    ].filter(Boolean),
+    nextAction,
+    redacted: true,
+  };
+}
+
+function buildContextQualityVerdict(cwd, signals = {}, options = {}) {
+  return runTokenContextQualityGate(cwd, signals, options);
+}
+
+function writeTokenContextQualityGate(cwd, gate) {
+  const dir = path.join(cwd, QUALITY_GATE_DIR_REL);
+  fs.mkdirSync(dir, { recursive: true });
+  const abs = path.join(cwd, LATEST_GATE_REL);
+  fs.writeFileSync(abs, JSON.stringify(gate, null, 2), "utf8");
+  try {
+    appendProductLearningEvent(cwd, "token_context_quality_gate_run", {
+      status: gate.status,
+      verdict: gate.verdict,
+      checksPass: gate.quality.checksPass,
+      checksWarn: gate.quality.checksWarn,
+      checksFail: gate.quality.checksFail,
+    });
+  } catch {}
+  return abs;
+}
+
+function buildTokenContextQualitySurface(cwd, options = {}) {
+  const abs = path.join(cwd, LATEST_GATE_REL);
+  if (!fs.existsSync(abs)) {
+    return { status: "not_available", artifactPath: LATEST_GATE_REL };
+  }
+  try {
+    const gate = JSON.parse(fs.readFileSync(abs, "utf8"));
+    return {
+      status: gate.status,
+      verdict: gate.verdict,
+      checksPass: gate.quality.checksPass,
+      checksWarn: gate.quality.checksWarn,
+      checksFail: gate.quality.checksFail,
+      artifactPath: LATEST_GATE_REL,
+    };
+  } catch {
+    return { status: "error", artifactPath: LATEST_GATE_REL };
+  }
+}
+
+function formatTokenContextQualityText(gate, options = {}) {
+  const lines = [`Token Context Quality Gate: ${gate.status} (${gate.verdict})`];
+  lines.push(`  Checks: ${gate.quality.checksPass} pass, ${gate.quality.checksWarn} warn, ${gate.quality.checksFail} fail`);
+  for (const c of gate.checks) {
+    if (c.status !== "pass") lines.push(`  [${c.status}] ${c.id}: ${c.detail}`);
+  }
+  lines.push(`  Next: ${gate.nextAction}`);
+  return lines.join("\n");
+}
+
+module.exports = {
+  CONTRACT,
+  SCHEMA_VERSION,
+  QUALITY_GATE_DIR_REL,
+  LATEST_GATE_REL,
+  runTokenContextQualityGate,
+  buildContextQualityVerdict,
+  writeTokenContextQualityGate,
+  buildTokenContextQualitySurface,
+  formatTokenContextQualityText,
+};

package/scripts/lib/token-cost-capture.js

@@ -0,0 +1,187 @@
+"use strict";
+
+const fs = require("fs");
+const path = require("path");
+const crypto = require("crypto");
+
+const { nowIso } = require("./fsx");
+const { readOutcomeEvents, OUTCOME_EVENTS_REL } = require("./unified-events");
+const { TOKEN_COST_ITEM_CONTRACT, TOKEN_COST_CAPTURE_JSONL_REL, TOKEN_COST_CAPTURE_RECEIPT_REL } = require("./prelaunch-evidence-store");
+
+const CONTRACT = "avorelo.realUsageTokenCostCapture.v1";
+const SCHEMA_VERSION = 1;
+
+function sha256(value) {
+  return crypto.createHash("sha256").update(String(value || "")).digest("hex");
+}
+
+function sanitizeText(value, max = 64) {
+  if (value === null || value === undefined) return null;
+  const normalized = String(value).replace(/[\x00-\x08\x0b\x0c\x0e-\x1f]/g, "").trim();
+  if (!normalized) return null;
+  return normalized.slice(0, max);
+}
+
+function sanitizeNumber(value) {
+  if (value === null || value === undefined || value === "") return null;
+  const num = Number(value);
+  return Number.isFinite(num) ? num : null;
+}
+
+function sanitizeMode(mode) {
+  const normalized = sanitizeText(mode, 24);
+  if (!normalized) return null;
+  return ["measured", "estimated", "heuristic", "missing"].includes(normalized) ? normalized : null;
+}
+
+function safeReadJson(absPath) {
+  try {
+    if (!fs.existsSync(absPath)) return null;
+    return JSON.parse(fs.readFileSync(absPath, "utf8").replace(/^\uFEFF/, ""));
+  } catch {
+    return null;
+  }
+}
+
+function buildCapturedRow(event) {
+  if (!event || typeof event !== "object") return null;
+  const attribution = event.attribution && typeof event.attribution === "object" ? event.attribution : {};
+  const metadata = event.metadata && typeof event.metadata === "object" ? event.metadata : {};
+  const totalTokens = sanitizeNumber(attribution.tokenUsage ?? metadata.totalTokens);
+  const measuredCostUsd = sanitizeNumber(attribution.cost ?? metadata.totalCost);
+  const provider = sanitizeText(attribution.provider ?? metadata.provider, 24) || "unknown";
+  const model = sanitizeText(attribution.model ?? metadata.model, 64) || "unknown";
+  const capturedAt = sanitizeText(event.timestamp, 40) || nowIso();
+
+  if (totalTokens === null && measuredCostUsd === null) return null;
+
+  const evidenceMode = measuredCostUsd !== null && totalTokens !== null
+    ? "measured"
+    : sanitizeMode(metadata.evidenceMode) || "heuristic";
+
+  return {
+    contract: TOKEN_COST_ITEM_CONTRACT,
+    schemaVersion: 1,
+    importedAt: nowIso(),
+    sourceType: "runtime_outcome_event",
+    capturedAt,
+    runIdHash: sha256(event.session_id || event.event_id || capturedAt),
+    tool: sanitizeText(event.tool, 48) || "unknown",
+    taskType: sanitizeText(event.category || event.event_name || event.action, 48) || "unknown",
+    provider,
+    modelFamily: model,
+    modelProvider: provider,
+    modelName: model,
+    inputTokens: null,
+    outputTokens: null,
+    promptTokens: null,
+    completionTokens: null,
+    totalTokens,
+    estimatedCostUsd: null,
+    measuredCostUsd,
+    costMicros: measuredCostUsd !== null ? Math.round(measuredCostUsd * 1000000) : null,
+    currency: measuredCostUsd !== null ? "USD" : null,
+    evidenceMode,
+    confidence: evidenceMode === "measured" ? "measured" : "heuristic",
+    notesCategory: "runtime_capture",
+    cacheHit: false,
+    estimatedOnly: false,
+    heuristic: evidenceMode !== "measured",
+    redacted: true,
+  };
+}
+
+function dedupeRows(rows) {
+  const map = new Map();
+  for (const row of rows) {
+    if (!row) continue;
+    const key = `${row.runIdHash || "unknown"}|${row.capturedAt || "unknown"}|${row.tool || "unknown"}|${row.totalTokens || 0}|${row.measuredCostUsd || 0}`;
+    map.set(key, row);
+  }
+  return Array.from(map.values());
+}
+
+function writeCaptureRows(cwd, rows) {
+  const abs = path.join(cwd, TOKEN_COST_CAPTURE_JSONL_REL);
+  fs.mkdirSync(path.dirname(abs), { recursive: true });
+  const content = rows.length ? rows.map((row) => JSON.stringify(row)).join("\n") + "\n" : "";
+  fs.writeFileSync(abs, content, "utf8");
+  return abs;
+}
+
+function buildRealUsageTokenCostCapture(cwd) {
+  const events = readOutcomeEvents(cwd);
+  const rows = dedupeRows(events.map(buildCapturedRow).filter(Boolean));
+  const measuredRows = rows.filter((row) => row.evidenceMode === "measured" && typeof row.totalTokens === "number" && typeof row.measuredCostUsd === "number");
+  const heuristicRows = rows.filter((row) => row.evidenceMode === "heuristic");
+
+  let status = "warn";
+  if (measuredRows.length > 0) status = "pass";
+  else if (rows.length > 0) status = "warn";
+  else status = "info";
+
+  const missingEvidence = [];
+  const safeNextActions = [];
+
+  if (events.length === 0) {
+    missingEvidence.push("No local outcome events with token/cost attribution are available yet.");
+    safeNextActions.push("Complete a real local task that produces runtime attribution, then rerun: node bin/avorelo token-cost capture --json");
+  } else if (rows.length === 0) {
+    missingEvidence.push("Outcome events exist, but none include safe token/cost attribution fields.");
+    safeNextActions.push("Use a real local task path that records provider/model/token attribution, then rerun: node bin/avorelo token-cost capture --json");
+  }
+
+  if (measuredRows.length === 0) {
+    missingEvidence.push("No measured token/cost samples are available from captured local usage.");
+    safeNextActions.push("Import redacted measured usage rows if local runtime cost is unavailable: node bin/avorelo token-cost import --file <redacted-json-or-jsonl> --json");
+  }
+
+  if (safeNextActions.length === 0) {
+    safeNextActions.push("Run: node bin/avorelo token-cost summary --json");
+  }
+
+  return {
+    contract: CONTRACT,
+    schemaVersion: SCHEMA_VERSION,
+    createdAt: nowIso(),
+    status,
+    evidenceAvailable: measuredRows.length > 0,
+    capturedRowsCount: rows.length,
+    realUsageSamplesCount: measuredRows.length,
+    heuristicSamplesCount: heuristicRows.length,
+    evidenceMode: measuredRows.length > 0 ? "measured" : rows.length > 0 ? "heuristic" : "missing",
+    confidence: measuredRows.length > 0 ? "measured" : rows.length > 0 ? "heuristic" : "missing",
+    source: {
+      outcomeEventsPath: OUTCOME_EVENTS_REL,
+      capturePath: TOKEN_COST_CAPTURE_JSONL_REL,
+    },
+    missingEvidence,
+    safeNextActions,
+    redacted: true,
+    noSavingsClaimUnlessMeasured: true,
+    rows,
+  };
+}
+
+function writeRealUsageTokenCostCapture(cwd, capture) {
+  const payload = capture || buildRealUsageTokenCostCapture(cwd);
+  writeCaptureRows(cwd, payload.rows || []);
+  const stored = { ...payload };
+  delete stored.rows;
+  const abs = path.join(cwd, TOKEN_COST_CAPTURE_RECEIPT_REL);
+  fs.mkdirSync(path.dirname(abs), { recursive: true });
+  fs.writeFileSync(abs, JSON.stringify(stored, null, 2), "utf8");
+  return stored;
+}
+
+function readRealUsageTokenCostCapture(cwd) {
+  return safeReadJson(path.join(cwd, TOKEN_COST_CAPTURE_RECEIPT_REL));
+}
+
+module.exports = {
+  CONTRACT,
+  SCHEMA_VERSION,
+  buildRealUsageTokenCostCapture,
+  writeRealUsageTokenCostCapture,
+  readRealUsageTokenCostCapture,
+};