avorelo 0.3.8 → 0.4.0-rc.2

package/scripts/lib/failure-recovery.js

@@ -0,0 +1,419 @@
+"use strict";
+
+// ── Failure Recovery ──────────────────────────────────────────────────────────
+//
+// Contract: avorelo.failureRecovery.v1
+//
+// Classifies every failure mode with a safe next action, severity, and
+// recoverability flag. No raw logs, prompts, code, or secrets in receipts.
+
+const fs = require("fs");
+const path = require("path");
+const { ensureCcoDirs, nowIso } = require("./fsx");
+
+const CONTRACT = "avorelo.failureRecovery.v1";
+const SCHEMA_VERSION = 1;
+
+const RECOVERY_DIR_REL = ".claude/cco/orchestration/failure-recovery";
+const LATEST_RECOVERY_REL = `${RECOVERY_DIR_REL}/latest-recovery.json`;
+
+// ── Failure type constants ────────────────────────────────────────────────────
+
+const FAILURE_TYPES = Object.freeze({
+  CONFIG_INVALID: "CONFIG_INVALID",
+  CONFIG_BACKUP_MISSING: "CONFIG_BACKUP_MISSING",
+  HOOKS_NOT_APPLIED: "HOOKS_NOT_APPLIED",
+  HOOK_DOCTOR_FAILED: "HOOK_DOCTOR_FAILED",
+  RECURSION_GUARD_ACTIVE: "RECURSION_GUARD_ACTIVE",
+  PROJECT_PROFILE_UNKNOWN: "PROJECT_PROFILE_UNKNOWN",
+  SAFE_RUN_BLOCKED: "SAFE_RUN_BLOCKED",
+  PROOF_UNAVAILABLE: "PROOF_UNAVAILABLE",
+  PROOF_FAILED: "PROOF_FAILED",
+  TOKEN_GATE_FAILED: "TOKEN_GATE_FAILED",
+  SUPPORT_BUNDLE_REDACTION_FAILED: "SUPPORT_BUNDLE_REDACTION_FAILED",
+  STALE_ARTIFACTS: "STALE_ARTIFACTS",
+  CORRUPT_ARTIFACT: "CORRUPT_ARTIFACT",
+  PERMISSION_DENIED: "PERMISSION_DENIED",
+  MISSING_TOOL: "MISSING_TOOL",
+  PATH_NOT_FOUND: "PATH_NOT_FOUND",
+  WINDOWS_PATH_ISSUE: "WINDOWS_PATH_ISSUE",
+  MONOREPO_SCOPE_UNKNOWN: "MONOREPO_SCOPE_UNKNOWN",
+  FLAKY_TEST_SUSPECTED: "FLAKY_TEST_SUSPECTED",
+  DEPENDENCY_RESOLUTION_FAILED: "DEPENDENCY_RESOLUTION_FAILED",
+  NETWORK_DISABLED_EXPECTED: "NETWORK_DISABLED_EXPECTED",
+  ARTIFACT_WRITE_FAILED: "ARTIFACT_WRITE_FAILED",
+  JSON_OUTPUT_INVALID: "JSON_OUTPUT_INVALID",
+});
+
+// ── Failure classification table ──────────────────────────────────────────────
+
+const FAILURE_CLASSIFICATION = Object.freeze({
+  CONFIG_INVALID: {
+    status: "blocked",
+    severity: "error",
+    recoverable: false,
+    userMessage: "The Avorelo config is invalid or cannot be parsed.",
+    safeNextAction: "Run `avorelo doctor` to identify and fix config issues.",
+    debugHint: "Check .claude/cco/state/ for config artifacts.",
+  },
+  CONFIG_BACKUP_MISSING: {
+    status: "needs_review",
+    severity: "warn",
+    recoverable: true,
+    userMessage: "No config backup found before hook apply.",
+    safeNextAction: "Run `avorelo hooks apply --dry-run` to preview hook changes. Review backup artifacts before applying.",
+    debugHint: "Check .claude/cco/orchestration/hook-apply/ for backup artifacts.",
+  },
+  HOOKS_NOT_APPLIED: {
+    status: "recoverable",
+    severity: "warn",
+    recoverable: true,
+    userMessage: "Avorelo hooks are not applied to this project.",
+    safeNextAction: "Run `avorelo hooks apply --dry-run` to preview, then `avorelo hooks apply --yes` to apply.",
+    debugHint: "Check .claude/cco/orchestration/hook-apply/latest-apply.json.",
+  },
+  HOOK_DOCTOR_FAILED: {
+    status: "blocked",
+    severity: "error",
+    recoverable: false,
+    userMessage: "Hook validation failed. Avorelo hooks may not be functioning correctly.",
+    safeNextAction: "Run `avorelo hooks doctor` for details, then `avorelo hooks rollback` to restore previous config.",
+    debugHint: "Check .claude/cco/orchestration/hook-apply/latest-doctor.json.",
+  },
+  RECURSION_GUARD_ACTIVE: {
+    status: "recoverable",
+    severity: "warn",
+    recoverable: true,
+    userMessage: "Avorelo recursion guard prevented a nested invocation.",
+    safeNextAction: "Wait for the current Avorelo run to complete, then retry.",
+    debugHint: "Recursion guard is expected in nested Claude Code sessions.",
+  },
+  PROJECT_PROFILE_UNKNOWN: {
+    status: "recoverable",
+    severity: "warn",
+    recoverable: true,
+    userMessage: "Avorelo could not detect the project profile.",
+    safeNextAction: "Run `avorelo project-profile` to detect or configure the project profile.",
+    debugHint: "Check .claude/cco/orchestration/project-profile/latest-profile.json.",
+  },
+  SAFE_RUN_BLOCKED: {
+    status: "blocked",
+    severity: "warn",
+    recoverable: true,
+    userMessage: "A safe run step was blocked due to scope or risk level.",
+    safeNextAction: "Review the blocking reason in `avorelo run --json` output. Narrow scope or get approval.",
+    debugHint: "Check .claude/cco/orchestration/safe-run/latest-run.json.",
+  },
+  PROOF_UNAVAILABLE: {
+    status: "recoverable",
+    severity: "warn",
+    recoverable: true,
+    userMessage: "No proof artifact is available for this operation.",
+    safeNextAction: "Run `avorelo outcome` to generate a proof artifact.",
+    debugHint: "Check .claude/cco/orchestration/seamless-outcome/.",
+  },
+  PROOF_FAILED: {
+    status: "blocked",
+    severity: "error",
+    recoverable: false,
+    userMessage: "Proof validation failed. Outcome integrity cannot be confirmed.",
+    safeNextAction: "Run `avorelo outcome --json` to see the proof failure details. Review and rerun the failing step.",
+    debugHint: "Check .claude/cco/orchestration/seamless-outcome/latest-reality-gate.json.",
+  },
+  TOKEN_GATE_FAILED: {
+    status: "needs_review",
+    severity: "error",
+    recoverable: false,
+    userMessage: "Token context quality gate failed.",
+    safeNextAction: "Run `avorelo token-efficiency --json` to see the quality gate details and reduce context.",
+    debugHint: "Check .claude/cco/orchestration/token-efficiency/latest-quality-gate.json.",
+  },
+  SUPPORT_BUNDLE_REDACTION_FAILED: {
+    status: "blocked",
+    severity: "critical",
+    recoverable: false,
+    userMessage: "Support bundle redaction failed. Bundle may contain sensitive data.",
+    safeNextAction: "Do not share the support bundle. Contact support@avorelo.com for a safe bundle procedure.",
+    debugHint: "Check support-bundle.js redaction logic.",
+  },
+  STALE_ARTIFACTS: {
+    status: "recoverable",
+    severity: "info",
+    recoverable: true,
+    userMessage: "Some Avorelo artifacts are stale and may not reflect current state.",
+    safeNextAction: "Run `avorelo artifacts health` to review stale artifacts, then `avorelo artifacts cleanup --dry-run` to plan cleanup.",
+    debugHint: "Check .claude/cco/orchestration/artifact-health/latest-health.json.",
+  },
+  CORRUPT_ARTIFACT: {
+    status: "needs_review",
+    severity: "error",
+    recoverable: false,
+    userMessage: "A corrupt artifact was detected. The artifact cannot be parsed.",
+    safeNextAction: "Run `avorelo artifacts health` to identify corrupt artifacts. Delete and regenerate corrupt artifacts.",
+    debugHint: "Check .claude/cco/orchestration/artifact-health/latest-health.json for corrupt file paths.",
+  },
+  PERMISSION_DENIED: {
+    status: "blocked",
+    severity: "error",
+    recoverable: false,
+    userMessage: "Permission denied when accessing a required file or directory.",
+    safeNextAction: "Check file permissions for .claude/cco/ and ensure the current user can read/write this directory.",
+    debugHint: "Check OS-level permissions for .claude/cco/.",
+  },
+  MISSING_TOOL: {
+    status: "blocked",
+    severity: "error",
+    recoverable: false,
+    userMessage: "A required tool is missing from PATH.",
+    safeNextAction: "Install the required tool and ensure it is available in PATH. Run `avorelo doctor` to re-check.",
+    debugHint: "Run `avorelo doctor` to see which tools are required.",
+  },
+  PATH_NOT_FOUND: {
+    status: "blocked",
+    severity: "error",
+    recoverable: false,
+    userMessage: "A required path does not exist.",
+    safeNextAction: "Verify the path exists and the workspace is correctly configured. Run `avorelo doctor` to check.",
+    debugHint: "Check that cwd is set to the correct project root.",
+  },
+  WINDOWS_PATH_ISSUE: {
+    status: "needs_review",
+    severity: "warn",
+    recoverable: true,
+    userMessage: "A Windows path issue was detected (spaces, backslashes, or drive letter handling).",
+    safeNextAction: "Ensure all paths use path.join() and avoid string concatenation. Move the project to a path without spaces if needed.",
+    debugHint: "Check for hardcoded path separators or unquoted paths with spaces.",
+  },
+  MONOREPO_SCOPE_UNKNOWN: {
+    status: "recoverable",
+    severity: "warn",
+    recoverable: true,
+    userMessage: "Avorelo could not determine the monorepo scope.",
+    safeNextAction: "Run `avorelo project-profile` to detect the monorepo structure and configure scope.",
+    debugHint: "Check .claude/cco/orchestration/project-profile/latest-profile.json.",
+  },
+  FLAKY_TEST_SUSPECTED: {
+    status: "recoverable",
+    severity: "warn",
+    recoverable: true,
+    userMessage: "A flaky test failure was suspected. The test may pass on retry.",
+    safeNextAction: "Retry the failing test in isolation. If it fails consistently, investigate the test for environmental dependencies.",
+    debugHint: "Check test output for timing-related or environment-specific failures.",
+  },
+  DEPENDENCY_RESOLUTION_FAILED: {
+    status: "blocked",
+    severity: "error",
+    recoverable: false,
+    userMessage: "Dependency resolution failed. Package install may be needed.",
+    safeNextAction: "Run `npm install` (or the project package manager) to restore dependencies.",
+    debugHint: "Check package.json and lockfile for inconsistencies.",
+  },
+  NETWORK_DISABLED_EXPECTED: {
+    status: "recoverable",
+    severity: "info",
+    recoverable: true,
+    userMessage: "Network is disabled. Avorelo is running in local-only mode.",
+    safeNextAction: "No action needed — Avorelo operates fully locally. Network access is not required.",
+    debugHint: "All Avorelo operations are local-first. No network access is expected.",
+  },
+  ARTIFACT_WRITE_FAILED: {
+    status: "needs_review",
+    severity: "warn",
+    recoverable: true,
+    userMessage: "An Avorelo artifact could not be written.",
+    safeNextAction: "Check disk space and .claude/cco/ write permissions. Run `avorelo doctor` to check local state writability.",
+    debugHint: "Check for disk full or permission errors on .claude/cco/.",
+  },
+  JSON_OUTPUT_INVALID: {
+    status: "needs_review",
+    severity: "error",
+    recoverable: false,
+    userMessage: "A command produced invalid JSON output.",
+    safeNextAction: "Run the command again with --json flag. Check for mixed stdout/stderr contaminating JSON output.",
+    debugHint: "Ensure all log output goes to stderr, not stdout.",
+  },
+});
+
+// ── Helpers ───────────────────────────────────────────────────────────────────
+
+function safeReadJson(absPath) {
+  try {
+    if (!fs.existsSync(absPath)) return null;
+    return JSON.parse(fs.readFileSync(absPath, "utf8").replace(/^/, ""));
+  } catch {
+    return null;
+  }
+}
+
+function bestEffortWrite(absPath, obj) {
+  try {
+    fs.mkdirSync(path.dirname(absPath), { recursive: true });
+    fs.writeFileSync(absPath, JSON.stringify(obj, null, 2), "utf8");
+  } catch {}
+}
+
+// ── classifyFailure ───────────────────────────────────────────────────────────
+
+function classifyFailure(errorOrSignal, options = {}) {
+  const failureType = options.failureType || detectFailureType(errorOrSignal);
+  const classification = FAILURE_CLASSIFICATION[failureType] || {
+    status: "unknown",
+    severity: "warn",
+    recoverable: true,
+    userMessage: "An unknown error occurred.",
+    safeNextAction: "Run `avorelo doctor` for diagnostics.",
+    debugHint: null,
+  };
+
+  const exitCode = options.exitCode != null ? options.exitCode : (errorOrSignal?.exitCode ?? 1);
+
+  return {
+    contract: CONTRACT,
+    schemaVersion: SCHEMA_VERSION,
+    status: classification.status,
+    failureType,
+    severity: classification.severity,
+    recoverable: classification.recoverable,
+    userMessage: classification.userMessage,
+    safeNextAction: classification.safeNextAction,
+    debugHint: options.includeDebugHint !== false ? classification.debugHint : undefined,
+    artifactRefs: options.artifactRefs || [],
+    exitCode,
+    redacted: true,
+  };
+}
+
+function detectFailureType(errorOrSignal) {
+  if (!errorOrSignal) return "JSON_OUTPUT_INVALID";
+  const msg = String(errorOrSignal.message || errorOrSignal.code || errorOrSignal || "").toLowerCase();
+  if (/recursion.guard|recursion_guard/i.test(msg)) return FAILURE_TYPES.RECURSION_GUARD_ACTIVE;
+  if (/EACCES|permission.denied/i.test(msg)) return FAILURE_TYPES.PERMISSION_DENIED;
+  if (/ENOENT|path.not.found|no such file/i.test(msg)) return FAILURE_TYPES.PATH_NOT_FOUND;
+  if (/windows|backslash|drive.letter/i.test(msg)) return FAILURE_TYPES.WINDOWS_PATH_ISSUE;
+  if (/flaky|flaky.test/i.test(msg)) return FAILURE_TYPES.FLAKY_TEST_SUSPECTED;
+  if (/network|ECONNREFUSED|ENETUNREACH/i.test(msg)) return FAILURE_TYPES.NETWORK_DISABLED_EXPECTED;
+  if (/config.invalid|parse.error|SyntaxError/i.test(msg)) return FAILURE_TYPES.CONFIG_INVALID;
+  if (/hook.doctor/i.test(msg)) return FAILURE_TYPES.HOOK_DOCTOR_FAILED;
+  if (/proof.failed/i.test(msg)) return FAILURE_TYPES.PROOF_FAILED;
+  if (/token.gate/i.test(msg)) return FAILURE_TYPES.TOKEN_GATE_FAILED;
+  if (/redaction.failed/i.test(msg)) return FAILURE_TYPES.SUPPORT_BUNDLE_REDACTION_FAILED;
+  if (/corrupt/i.test(msg)) return FAILURE_TYPES.CORRUPT_ARTIFACT;
+  if (/stale/i.test(msg)) return FAILURE_TYPES.STALE_ARTIFACTS;
+  if (/missing.tool|command.not.found/i.test(msg)) return FAILURE_TYPES.MISSING_TOOL;
+  if (/dependency|npm.install/i.test(msg)) return FAILURE_TYPES.DEPENDENCY_RESOLUTION_FAILED;
+  if (/artifact.write/i.test(msg)) return FAILURE_TYPES.ARTIFACT_WRITE_FAILED;
+  return "JSON_OUTPUT_INVALID";
+}
+
+// ── buildRecoveryPlan ─────────────────────────────────────────────────────────
+
+function buildRecoveryPlan(cwd, failure, options = {}) {
+  const action = buildRecoveryAction(cwd, failure.failureType || "JSON_OUTPUT_INVALID", options);
+  const receipt = {
+    contract: CONTRACT,
+    schemaVersion: SCHEMA_VERSION,
+    status: failure.status || "unknown",
+    failureType: failure.failureType,
+    severity: failure.severity || "warn",
+    recoverable: failure.recoverable !== undefined ? failure.recoverable : true,
+    userMessage: failure.userMessage || "An error occurred.",
+    safeNextAction: failure.safeNextAction || action.command || "Run `avorelo doctor`.",
+    debugHint: failure.debugHint || null,
+    artifactRefs: failure.artifactRefs || [],
+    exitCode: failure.exitCode != null ? failure.exitCode : 1,
+    recoveryAction: action,
+    createdAt: nowIso(),
+    redacted: true,
+  };
+  return receipt;
+}
+
+// ── buildRecoveryAction ───────────────────────────────────────────────────────
+
+function buildRecoveryAction(cwd, failureType, options = {}) {
+  const classification = FAILURE_CLASSIFICATION[failureType];
+  if (!classification) {
+    return {
+      failureType,
+      command: "avorelo doctor",
+      rationale: "Run diagnostics to identify and recover from this failure.",
+      automated: false,
+    };
+  }
+
+  return {
+    failureType,
+    command: classification.safeNextAction,
+    rationale: classification.userMessage,
+    automated: false,
+    severity: classification.severity,
+    recoverable: classification.recoverable,
+  };
+}
+
+// ── writeFailureRecoveryReceipt ───────────────────────────────────────────────
+
+function writeFailureRecoveryReceipt(cwd, receipt) {
+  const absPath = path.join(cwd, LATEST_RECOVERY_REL);
+  bestEffortWrite(absPath, receipt);
+}
+
+// ── buildFailureRecoverySurface ───────────────────────────────────────────────
+
+function buildFailureRecoverySurface(cwd, options = {}) {
+  const absPath = path.join(cwd, LATEST_RECOVERY_REL);
+  const latest = safeReadJson(absPath);
+
+  if (!latest) {
+    return {
+      contract: CONTRACT,
+      status: "no_failure_recorded",
+      failureType: null,
+      severity: null,
+      recoverable: null,
+      safeNextAction: "No failure recovery receipt found. Run any avorelo command to generate state.",
+      latestRecoveryPath: null,
+    };
+  }
+
+  return {
+    contract: CONTRACT,
+    status: latest.status,
+    failureType: latest.failureType,
+    severity: latest.severity,
+    recoverable: latest.recoverable,
+    safeNextAction: latest.safeNextAction,
+    latestRecoveryPath: LATEST_RECOVERY_REL,
+  };
+}
+
+// ── formatFailureRecoveryText ─────────────────────────────────────────────────
+
+function formatFailureRecoveryText(receipt, options = {}) {
+  if (!receipt) return "No failure recovery data available.";
+  const lines = [];
+  lines.push(`Failure Recovery: ${receipt.failureType || "unknown"}`);
+  lines.push(`Status: ${receipt.status || "unknown"} | Severity: ${receipt.severity || "unknown"} | Recoverable: ${receipt.recoverable ? "yes" : "no"}`);
+  lines.push("");
+  lines.push(receipt.userMessage || "No details available.");
+  lines.push("");
+  lines.push(`Next step: ${receipt.safeNextAction || "Run avorelo doctor."}`);
+  if (options.debug && receipt.debugHint) {
+    lines.push(`Debug: ${receipt.debugHint}`);
+  }
+  return lines.join("\n");
+}
+
+module.exports = {
+  CONTRACT,
+  SCHEMA_VERSION,
+  FAILURE_TYPES,
+  FAILURE_CLASSIFICATION,
+  LATEST_RECOVERY_REL,
+  classifyFailure,
+  buildRecoveryPlan,
+  buildRecoveryAction,
+  writeFailureRecoveryReceipt,
+  buildFailureRecoverySurface,
+  formatFailureRecoveryText,
+};