avorelo 0.1.0 → 0.3.0

package/scripts/lib/token-evidence.js

@@ -1,699 +0,0 @@
-"use strict";
-
-// ── Token Evidence v2 ─────────────────────────────────────────────────────────
-//
-// Upgrades token estimation with a priority-ordered evidence chain:
-//   1. provider_usage  — reads local artifact files for actual token counts
-//   2. local_tokenizer — uses gpt-tokenizer for text estimation
-//   3. chars_div_4     — fallback always kept
-//
-// No network. No secrets. No raw prompts in output.
-// chars_div_4 never removed. Activation still shows no savings.
-
-const fs = require("fs");
-const path = require("path");
-const crypto = require("crypto");
-
-const SCHEMA_VERSION = 1;
-const CONTRACT = "avorelo.tokenEvidence.v1";
-const TOKEN_EVIDENCE_DIR_REL = ".claude/cco/orchestration/token-evidence";
-const TOKEN_EVIDENCE_LATEST_REL = ".claude/cco/orchestration/token-evidence/latest-token-evidence.json";
-
-const CHARS_PER_TOKEN = 4;
-const RAW_TOKEN_CAP = 50000;
-
-// Receipt paths (mirrors operating-value.js receipt reader)
-const WORKSPACE_REGISTRY_REL = ".claude/cco/orchestration/workspace-registry/latest-registry.json";
-const SESSION_CONTEXT_REL = ".claude/cco/orchestration/session-context/latest-context.json";
-const BRAIN_PACK_REL = ".claude/cco/context/brain-pack/latest.json";
-const WORK_CONTROL_REL = ".claude/cco/orchestration/work-control/latest-receipt.json";
-
-// Provider usage artifact paths to scan (safe local paths only)
-const PROVIDER_USAGE_SCAN_PATHS = [
-  ".claude/cco/events",
-  ".avorelo/events",
-];
-const ORCHESTRATION_SCAN_PATHS = [
-  ".claude/cco/orchestration",
-];
-
-// gpt-tokenizer vendor path (installed in scripts/vendor to avoid root npm permission issues)
-const GPT_TOKENIZER_VENDOR_PATH = path.resolve(__dirname, "../vendor/node_modules/gpt-tokenizer/cjs/main.js");
-
-function shortId() {
-  return crypto.randomBytes(4).toString("hex");
-}
-
-function nowIso() {
-  return new Date().toISOString();
-}
-
-function ensureDirFor(filePath) {
-  const dir = path.dirname(filePath);
-  if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
-}
-
-function safeReadJson(filePath) {
-  try {
-    if (!fs.existsSync(filePath)) return null;
-    return JSON.parse(fs.readFileSync(filePath, "utf8"));
-  } catch {
-    return null;
-  }
-}
-
-function fileCharLen(filePath) {
-  try {
-    if (!fs.existsSync(filePath)) return 0;
-    return fs.statSync(filePath).size;
-  } catch {
-    return 0;
-  }
-}
-
-function charsToTokens(chars) {
-  return Math.round(chars / CHARS_PER_TOKEN);
-}
-
-// ── Provider Usage Evidence Extraction ────────────────────────────────────────
-// Scans safe local paths for token usage fields only.
-// Returns numeric usage fields without any raw content.
-
-const ANTHROPIC_USAGE_FIELDS = [
-  "input_tokens", "output_tokens",
-  "cache_read_input_tokens", "cache_creation_input_tokens",
-];
-const OPENAI_USAGE_FIELDS = [
-  "prompt_tokens", "completion_tokens", "total_tokens",
-];
-const GENERIC_USAGE_FIELDS = [
-  "usage.input_tokens", "usage.output_tokens",
-  "usage.prompt_tokens", "usage.completion_tokens",
-];
-
-function extractUsageFromObject(obj) {
-  if (!obj || typeof obj !== "object") return null;
-
-  const result = {};
-  let found = false;
-
-  // Anthropic-style
-  for (const field of ANTHROPIC_USAGE_FIELDS) {
-    if (typeof obj[field] === "number" && Number.isFinite(obj[field])) {
-      result[field] = obj[field];
-      found = true;
-    }
-  }
-
-  // OpenAI-style
-  for (const field of OPENAI_USAGE_FIELDS) {
-    if (typeof obj[field] === "number" && Number.isFinite(obj[field])) {
-      result[field] = obj[field];
-      found = true;
-    }
-  }
-
-  // Nested usage object
-  if (obj.usage && typeof obj.usage === "object") {
-    for (const field of [...ANTHROPIC_USAGE_FIELDS, ...OPENAI_USAGE_FIELDS]) {
-      if (typeof obj.usage[field] === "number" && Number.isFinite(obj.usage[field])) {
-        result[`usage.${field}`] = obj.usage[field];
-        found = true;
-      }
-    }
-  }
-
-  return found ? result : null;
-}
-
-function parseJsonlDefensively(content) {
-  const lines = content.split("\n");
-  const results = [];
-  for (const line of lines) {
-    const trimmed = line.trim();
-    if (!trimmed) continue;
-    try {
-      const parsed = JSON.parse(trimmed);
-      results.push(parsed);
-    } catch {
-      // Skip malformed lines
-    }
-  }
-  return results;
-}
-
-function collectProviderUsageEvidence(cwd) {
-  const sourcePaths = [];
-  const usageRecords = [];
-
-  // Scan .claude/cco/events and .avorelo/events for JSONL files
-  for (const dirRel of PROVIDER_USAGE_SCAN_PATHS) {
-    const dirAbs = path.resolve(cwd, dirRel);
-    if (!fs.existsSync(dirAbs)) continue;
-    let entries;
-    try { entries = fs.readdirSync(dirAbs); } catch { continue; }
-
-    for (const entry of entries) {
-      if (!entry.endsWith(".jsonl")) continue;
-      const fileAbs = path.join(dirAbs, entry);
-      const fileRel = path.relative(cwd, fileAbs).replace(/\\/g, "/");
-      let content;
-      try { content = fs.readFileSync(fileAbs, "utf8"); } catch { continue; }
-      const records = parseJsonlDefensively(content);
-      let fileHadUsage = false;
-      for (const record of records) {
-        const usage = extractUsageFromObject(record);
-        if (usage) {
-          usageRecords.push(usage);
-          fileHadUsage = true;
-        }
-      }
-      if (fileHadUsage) sourcePaths.push(fileRel);
-    }
-  }
-
-  // Scan .claude/cco/orchestration for JSON files
-  for (const dirRel of ORCHESTRATION_SCAN_PATHS) {
-    const dirAbs = path.resolve(cwd, dirRel);
-    if (!fs.existsSync(dirAbs)) continue;
-    scanDirForUsageJson(dirAbs, cwd, sourcePaths, usageRecords);
-  }
-
-  return { usageRecords, sourcePaths };
-}
-
-function scanDirForUsageJson(dirAbs, cwd, sourcePaths, usageRecords, depth = 0) {
-  if (depth > 3) return; // limit recursion
-  let entries;
-  try { entries = fs.readdirSync(dirAbs); } catch { return; }
-
-  for (const entry of entries) {
-    const entryAbs = path.join(dirAbs, entry);
-    let stat;
-    try { stat = fs.statSync(entryAbs); } catch { continue; }
-    if (stat.isDirectory()) {
-      scanDirForUsageJson(entryAbs, cwd, sourcePaths, usageRecords, depth + 1);
-    } else if (entry.endsWith(".json")) {
-      const data = safeReadJson(entryAbs);
-      if (!data) continue;
-      const usage = extractUsageFromObject(data);
-      if (usage) {
-        const fileRel = path.relative(cwd, entryAbs).replace(/\\/g, "/");
-        usageRecords.push(usage);
-        sourcePaths.push(fileRel);
-      }
-    }
-  }
-}
-
-function aggregateUsageRecords(records) {
-  if (!records || records.length === 0) return null;
-
-  // Prefer most recent / highest-count approach: sum all found records
-  // for candidate token counts; or just take the first record that has provider fields.
-  // For provider_usage, we report the summed total from all records.
-  let inputTokens = null;
-  let outputTokens = null;
-  let cacheReadInputTokens = null;
-  let cacheCreationInputTokens = null;
-  let promptTokens = null;
-  let completionTokens = null;
-  let totalTokens = null;
-
-  for (const rec of records) {
-    const add = (cur, val) => cur === null ? (typeof val === "number" ? val : null) : (typeof val === "number" ? cur + val : cur);
-    inputTokens = add(inputTokens, rec.input_tokens);
-    outputTokens = add(outputTokens, rec.output_tokens);
-    cacheReadInputTokens = add(cacheReadInputTokens, rec.cache_read_input_tokens);
-    cacheCreationInputTokens = add(cacheCreationInputTokens, rec.cache_creation_input_tokens);
-    promptTokens = add(promptTokens, rec.prompt_tokens);
-    completionTokens = add(completionTokens, rec.completion_tokens);
-    totalTokens = add(totalTokens, rec.total_tokens || rec["usage.total_tokens"]);
-    if (rec["usage.input_tokens"] !== undefined) inputTokens = add(inputTokens, rec["usage.input_tokens"]);
-    if (rec["usage.output_tokens"] !== undefined) outputTokens = add(outputTokens, rec["usage.output_tokens"]);
-    if (rec["usage.prompt_tokens"] !== undefined) promptTokens = add(promptTokens, rec["usage.prompt_tokens"]);
-    if (rec["usage.completion_tokens"] !== undefined) completionTokens = add(completionTokens, rec["usage.completion_tokens"]);
-  }
-
-  // Normalize: map OpenAI to Anthropic-style if Anthropic not present
-  const normalizedInput = inputTokens !== null ? inputTokens : promptTokens;
-  const normalizedOutput = outputTokens !== null ? outputTokens : completionTokens;
-  const normalizedCacheRead = cacheReadInputTokens;
-  const normalizedCacheWrite = cacheCreationInputTokens;
-
-  // totalProviderTokens
-  let total = totalTokens;
-  if (total === null && normalizedInput !== null && normalizedOutput !== null) {
-    total = normalizedInput + normalizedOutput;
-  }
-
-  const hasAny = normalizedInput !== null || normalizedOutput !== null || total !== null;
-  if (!hasAny) return null;
-
-  return {
-    inputTokens: normalizedInput,
-    outputTokens: normalizedOutput,
-    cachedInputTokens: normalizedCacheRead,
-    cacheWriteTokens: normalizedCacheWrite,
-    cacheReadTokens: normalizedCacheRead,
-    totalProviderTokens: total,
-  };
-}
-
-// ── Local Tokenizer ───────────────────────────────────────────────────────────
-// Uses gpt-tokenizer (pure TS, zero external deps) for text estimation.
-// Claude models: confidence=medium, caveat present.
-// Falls back gracefully if import fails.
-
-const CLAUDE_MODEL_CAVEAT = "Local tokenizer estimate may differ from Claude billing tokenizer.";
-
-function getTokenizerName(provider, model) {
-  if (!model) return "cl100k_base";
-  const m = model.toLowerCase();
-  if (m.includes("gpt-4") || m.includes("gpt4")) return "cl100k_base";
-  if (m.includes("gpt-3.5") || m.includes("gpt3.5")) return "cl100k_base";
-  if (m.includes("claude")) return "cl100k_base"; // Best approximation for Claude
-  return "cl100k_base"; // Default
-}
-
-function countTokensWithLocalTokenizer(text, { provider, model } = {}) {
-  const tokenizerName = getTokenizerName(provider, model);
-  const isClaudeModel = (model || "").toLowerCase().includes("claude");
-  const caveat = isClaudeModel ? CLAUDE_MODEL_CAVEAT : null;
-
-  try {
-    // Try main install first, then vendor path
-    let tokenizer = null;
-    try {
-      tokenizer = require("gpt-tokenizer");
-    } catch {
-      try {
-        tokenizer = require(GPT_TOKENIZER_VENDOR_PATH);
-      } catch {
-        return {
-          method: "chars_div_4",
-          tokens: charsToTokens(text.length),
-          tokenizerName: "chars_div_4",
-          model: model || null,
-          confidence: "low",
-          caveat: "gpt-tokenizer not available; fell back to chars÷4.",
-          fallbackReason: "gpt-tokenizer import failed",
-        };
-      }
-    }
-
-    const encodeFunction = tokenizer.encode || (tokenizer.default && tokenizer.default.encode);
-    if (typeof encodeFunction !== "function") {
-      return {
-        method: "chars_div_4",
-        tokens: charsToTokens(text.length),
-        tokenizerName: "chars_div_4",
-        model: model || null,
-        confidence: "low",
-        caveat: "gpt-tokenizer encode function not found; fell back to chars÷4.",
-        fallbackReason: "encode function not found",
-      };
-    }
-
-    const encoded = encodeFunction(text);
-    const count = Array.isArray(encoded) ? encoded.length : charsToTokens(text.length);
-
-    return {
-      method: "local_tokenizer",
-      tokens: count,
-      tokenizerName,
-      model: model || null,
-      confidence: isClaudeModel ? "medium" : "medium",
-      caveat,
-    };
-  } catch (err) {
-    return {
-      method: "chars_div_4",
-      tokens: charsToTokens(text.length),
-      tokenizerName: "chars_div_4",
-      model: model || null,
-      confidence: "low",
-      caveat: `Local tokenizer error; fell back to chars÷4. (${err && err.message ? err.message : "unknown error"})`,
-      fallbackReason: err && err.message ? err.message : "unknown error",
-    };
-  }
-}
-
-// ── Estimate raw candidate context tokens ─────────────────────────────────────
-// Mirrors operating-value.js estimateRawContextTokens — reads registry to
-// estimate what could have been loaded without Avorelo optimization.
-// Method: chars_div_4 (always, since we don't load actual token counts)
-
-function estimateRawCandidateTokens(cwd) {
-  const registryPath = path.resolve(cwd, WORKSPACE_REGISTRY_REL);
-  const registry = safeReadJson(registryPath);
-  const brainPackPath = path.resolve(cwd, BRAIN_PACK_REL);
-  const workControlPath = path.resolve(cwd, WORK_CONTROL_REL);
-
-  const sources = [];
-  let totalChars = 0;
-
-  if (registry && registry.summary) {
-    const assetChars = (registry.summary.totalAssets || 0) * 150;
-    totalChars += assetChars;
-    sources.push("workspace-registry");
-  }
-
-  const brainPackChars = fileCharLen(brainPackPath);
-  if (brainPackChars > 0) {
-    totalChars += brainPackChars;
-    sources.push("brain-pack");
-  }
-
-  const workControlChars = fileCharLen(workControlPath);
-  if (workControlChars > 0) {
-    totalChars += workControlChars;
-    sources.push("work-control");
-  }
-
-  if (totalChars === 0) {
-    return {
-      tokens: null,
-      method: "unavailable",
-      confidence: "unavailable",
-      sources: [],
-      caveat: "No source data to estimate raw candidate context.",
-    };
-  }
-
-  const rawTokens = Math.min(charsToTokens(totalChars), RAW_TOKEN_CAP);
-  return {
-    tokens: rawTokens,
-    method: "chars_div_4",
-    confidence: "estimated",
-    sources,
-    caveat: "Raw candidate context estimated from source file/receipt sizes via chars÷4. Not a measured model context window load.",
-  };
-}
-
-// ── Estimate compact handoff tokens ───────────────────────────────────────────
-// Reads session-context for estimatedTokens field (preferred).
-// Falls back to chars_div_4 of session-context file size.
-
-function estimateCompactHandoffTokens(cwd) {
-  const scPath = path.resolve(cwd, SESSION_CONTEXT_REL);
-  const sc = safeReadJson(scPath);
-
-  if (sc && sc.estimatedTokens && typeof sc.estimatedTokens === "number") {
-    return {
-      tokens: sc.estimatedTokens,
-      method: "session_context",
-      confidence: "measured",
-      basis: "session-context.estimatedTokens field",
-      caveat: null,
-    };
-  }
-
-  // Also check tokenDiscipline.estimatedTokens
-  if (sc && sc.tokenDiscipline && typeof sc.tokenDiscipline.estimatedTokens === "number") {
-    return {
-      tokens: sc.tokenDiscipline.estimatedTokens,
-      method: "session_context",
-      confidence: "measured",
-      basis: "session-context.tokenDiscipline.estimatedTokens field",
-      caveat: null,
-    };
-  }
-
-  const fileBytes = fileCharLen(scPath);
-  if (fileBytes > 0) {
-    return {
-      tokens: charsToTokens(fileBytes),
-      method: "chars_div_4",
-      confidence: "estimated",
-      basis: "session-context file size ÷ 4",
-      caveat: "Compact handoff token count estimated from file size via chars÷4. Session-context.estimatedTokens not available.",
-    };
-  }
-
-  return {
-    tokens: null,
-    method: "unavailable",
-    confidence: "unavailable",
-    basis: "session-context not available",
-    caveat: "Run `avorelo session-context --handoff --target claude` to generate compact handoff.",
-  };
-}
-
-// ── Build token evidence ───────────────────────────────────────────────────────
-
-function buildTokenEvidence(cwd, options = {}) {
-  const evidenceId = `te-${Date.now()}-${shortId()}`;
-  const createdAt = nowIso();
-
-  const rawEst = estimateRawCandidateTokens(cwd);
-  const compactEst = estimateCompactHandoffTokens(cwd);
-
-  const targetProvider = options.targetProvider || null;
-  const targetModel = options.targetModel || null;
-
-  // ── Priority 1: Provider usage from local artifacts ───────────────────────
-  let providerUsageResult = null;
-  let providerUsageSource = null;
-  let providerUsageSourcePaths = [];
-
-  try {
-    const { usageRecords, sourcePaths } = collectProviderUsageEvidence(cwd);
-    if (usageRecords.length > 0) {
-      const agg = aggregateUsageRecords(usageRecords);
-      if (agg) {
-        providerUsageResult = agg;
-        providerUsageSourcePaths = sourcePaths;
-        providerUsageSource = sourcePaths.length > 0 ? sourcePaths[0] : "local_artifact";
-      }
-    }
-  } catch {
-    // Non-blocking
-  }
-
-  // ── Priority 2: Local tokenizer ───────────────────────────────────────────
-  // We use local tokenizer for compact handoff text if no provider usage
-  let localTokenizerResult = null;
-  if (!providerUsageResult) {
-    try {
-      const scPath = path.resolve(cwd, SESSION_CONTEXT_REL);
-      if (fs.existsSync(scPath)) {
-        const scContent = fs.readFileSync(scPath, "utf8");
-        // Only use if session-context doesn't have estimatedTokens (avoid double-counting)
-        const sc = safeReadJson(scPath);
-        const hasEstimatedTokens = sc && (
-          (typeof sc.estimatedTokens === "number") ||
-          (sc.tokenDiscipline && typeof sc.tokenDiscipline.estimatedTokens === "number")
-        );
-        if (!hasEstimatedTokens && scContent.length > 0) {
-          localTokenizerResult = countTokensWithLocalTokenizer(scContent, {
-            provider: targetProvider,
-            model: targetModel,
-          });
-        }
-      }
-    } catch {
-      // Non-blocking
-    }
-  }
-
-  // ── Determine primary method ──────────────────────────────────────────────
-  let primaryMethod;
-  let primaryConfidence;
-  let tokenizerName;
-  let modelMatched = false;
-  let fallbackReason = null;
-  let localTokenizerUsed = false;
-  let fallbackUsed = false;
-
-  if (providerUsageResult) {
-    primaryMethod = "provider_usage";
-    primaryConfidence = "high";
-    tokenizerName = "provider_reported";
-    modelMatched = true;
-  } else if (localTokenizerResult && localTokenizerResult.method === "local_tokenizer") {
-    primaryMethod = "local_tokenizer";
-    primaryConfidence = "medium";
-    tokenizerName = localTokenizerResult.tokenizerName;
-    modelMatched = false;
-    localTokenizerUsed = true;
-  } else {
-    // Determine overall method for fallback
-    primaryMethod = compactEst.method === "session_context" && rawEst.method === "chars_div_4"
-      ? "mixed: session_context + chars_div_4"
-      : rawEst.method === "unavailable" || compactEst.method === "unavailable"
-        ? "unavailable"
-        : rawEst.method;
-    primaryConfidence = compactEst.confidence === "measured" ? "medium" : "low";
-    tokenizerName = "chars_div_4";
-    fallbackUsed = true;
-    fallbackReason = localTokenizerResult && localTokenizerResult.fallbackReason
-      ? localTokenizerResult.fallbackReason
-      : "no provider usage found; local tokenizer unavailable or not applicable";
-  }
-
-  // ── Measurements ──────────────────────────────────────────────────────────
-  let inputTokens = null;
-  let cachedInputTokens = null;
-  let outputTokens = null;
-  let cacheWriteTokens = null;
-  let cacheReadTokens = null;
-  let totalProviderTokens = null;
-
-  if (providerUsageResult) {
-    inputTokens = providerUsageResult.inputTokens;
-    cachedInputTokens = providerUsageResult.cachedInputTokens;
-    outputTokens = providerUsageResult.outputTokens;
-    cacheWriteTokens = providerUsageResult.cacheWriteTokens;
-    cacheReadTokens = providerUsageResult.cacheReadTokens;
-    totalProviderTokens = providerUsageResult.totalProviderTokens;
-  }
-
-  let candidateTokenReduction = null;
-  let contextCompressionRatio = null;
-  if (rawEst.tokens !== null && compactEst.tokens !== null && rawEst.tokens > 0) {
-    candidateTokenReduction = Math.max(0, rawEst.tokens - compactEst.tokens);
-    contextCompressionRatio = Math.round((compactEst.tokens / rawEst.tokens) * 100) / 100;
-  }
-
-  // ── Caveats ───────────────────────────────────────────────────────────────
-  const caveats = [];
-  caveats.push("Candidate context is estimated — not a guaranteed model behavior.");
-  if (rawEst.caveat) caveats.push(rawEst.caveat);
-  if (compactEst.caveat) caveats.push(compactEst.caveat);
-  if (localTokenizerResult && localTokenizerResult.caveat && !providerUsageResult) {
-    caveats.push(localTokenizerResult.caveat);
-  }
-
-  // ── Evidence sources ──────────────────────────────────────────────────────
-  const evidenceSources = [
-    rawEst.sources.length ? `raw from: ${rawEst.sources.join(", ")}` : null,
-    compactEst.basis ? `compact from: ${compactEst.basis}` : null,
-  ].filter(Boolean);
-
-  return {
-    schemaVersion: SCHEMA_VERSION,
-    contract: CONTRACT,
-    evidenceId,
-    createdAt,
-    cwd,
-    tokenizer: {
-      provider: targetProvider,
-      model: targetModel,
-      method: primaryMethod,
-      name: tokenizerName,
-      confidence: primaryConfidence,
-      modelMatched,
-      providerUsageSource,
-      fallbackReason,
-    },
-    measurements: {
-      rawCandidateContextTokens: rawEst.tokens,
-      compactHandoffTokens: compactEst.tokens,
-      inputTokens,
-      cachedInputTokens,
-      outputTokens,
-      cacheWriteTokens,
-      cacheReadTokens,
-      totalProviderTokens,
-      candidateTokenReduction,
-      contextCompressionRatio,
-    },
-    evidence: {
-      providerUsageEvidenceFound: providerUsageResult !== null,
-      localTokenizerUsed,
-      fallbackUsed,
-      sourcePaths: providerUsageSourcePaths,
-    },
-    basis: evidenceSources.join(". ") || "insufficient data",
-    caveats,
-    redacted: true,
-  };
-}
-
-// ── Write / read ───────────────────────────────────────────────────────────────
-
-function writeTokenEvidence(cwd, evidence) {
-  const latestPath = path.resolve(cwd, TOKEN_EVIDENCE_LATEST_REL);
-  ensureDirFor(latestPath);
-  fs.writeFileSync(latestPath, JSON.stringify(evidence, null, 2), "utf8");
-}
-
-function readTokenEvidence(cwd) {
-  const latestPath = path.resolve(cwd, TOKEN_EVIDENCE_LATEST_REL);
-  return safeReadJson(latestPath);
-}
-
-// ── Text format ────────────────────────────────────────────────────────────────
-
-function formatTokenEvidenceText(evidence) {
-  if (!evidence) return "Token evidence unavailable.\n";
-  const m = evidence.measurements;
-  const ev = evidence.evidence || {};
-  const lines = [
-    "Avorelo Token Evidence",
-    "",
-    `  Method:      ${evidence.tokenizer.method}`,
-    `  Confidence:  ${evidence.tokenizer.confidence}`,
-    evidence.tokenizer.name ? `  Tokenizer:   ${evidence.tokenizer.name}` : null,
-    "",
-    m.rawCandidateContextTokens !== null
-      ? `  Raw candidate context: ~${m.rawCandidateContextTokens.toLocaleString()} tokens (estimated)`
-      : "  Raw candidate context: unavailable",
-    m.compactHandoffTokens !== null
-      ? `  Compact handoff:       ~${m.compactHandoffTokens.toLocaleString()} tokens`
-      : "  Compact handoff:       unavailable",
-    m.candidateTokenReduction !== null
-      ? `  Candidate reduction:   ~${m.candidateTokenReduction.toLocaleString()} tokens not loaded`
-      : "  Candidate reduction:   unavailable",
-    m.contextCompressionRatio !== null
-      ? `  Compression ratio:     ${(m.contextCompressionRatio * 100).toFixed(0)}% of candidate loaded`
-      : null,
-    ev.providerUsageEvidenceFound && m.inputTokens !== null
-      ? `  Provider input tokens: ${m.inputTokens.toLocaleString()} (from local artifacts)`
-      : null,
-    ev.providerUsageEvidenceFound && m.outputTokens !== null
-      ? `  Provider output tokens: ${m.outputTokens.toLocaleString()} (from local artifacts)`
-      : null,
-    "",
-    `  Basis: ${evidence.basis}`,
-    evidence.caveats.length ? `  Caveats:` : null,
-    ...evidence.caveats.map((c) => `    - ${c}`),
-    "",
-  ].filter((l) => l !== null);
-  return lines.join("\n");
-}
-
-function formatTokenEvidenceDebugSources(evidence) {
-  if (!evidence) return "Token evidence unavailable.\n";
-  const ev = evidence.evidence || {};
-  const lines = [
-    "Avorelo Token Evidence — Debug Sources",
-    "",
-    `  Method:                ${evidence.tokenizer.method}`,
-    `  Provider usage found:  ${ev.providerUsageEvidenceFound}`,
-    `  Local tokenizer used:  ${ev.localTokenizerUsed}`,
-    `  Fallback used:         ${ev.fallbackUsed}`,
-    `  Fallback reason:       ${evidence.tokenizer.fallbackReason || "none"}`,
-    `  Provider source:       ${evidence.tokenizer.providerUsageSource || "none"}`,
-    "",
-    `  Source paths scanned (${(ev.sourcePaths || []).length}):`,
-    ...(ev.sourcePaths || []).map((p) => `    - ${p}`),
-    "",
-  ];
-  return lines.join("\n");
-}
-
-module.exports = {
-  CONTRACT,
-  TOKEN_EVIDENCE_LATEST_REL,
-  buildTokenEvidence,
-  writeTokenEvidence,
-  readTokenEvidence,
-  estimateRawCandidateTokens,
-  estimateCompactHandoffTokens,
-  countTokensWithLocalTokenizer,
-  collectProviderUsageEvidence,
-  formatTokenEvidenceText,
-  formatTokenEvidenceDebugSources,
-};