avorelo 0.3.8 → 0.4.0-rc.2

package/scripts/lib/ai-efficiency-dogfood-pack.js

@@ -0,0 +1,437 @@
+"use strict";
+
+const DOGFOOD_SCENARIOS = [
+  {
+    id: "ai_efficiency_change",
+    name: "AI efficiency feature change",
+    input: {
+      changedFiles: [
+        "scripts/lib/ai-work-efficiency-routing.js",
+        "scripts/lib/context-budget.js",
+        "tests/ai-work-efficiency-routing.test.js",
+      ],
+    },
+    expected: {
+      taskTypes: ["implementation", "prompt_context", "value_reporting", "test"],
+      allowedRoutes: ["guarded", "proof_first", "test_first", "no_connect"],
+      forbiddenRoutes: [],
+      noFakeSavings: true,
+    },
+  },
+  {
+    id: "docs_only_change",
+    name: "Docs-only change",
+    input: {
+      changedFiles: [
+        "docs/ai-work-efficiency-routing.md",
+        "docs/architecture/00-product-principles.md",
+      ],
+    },
+    expected: {
+      taskTypes: ["documentation"],
+      allowedRoutes: ["docs_only", "no_connect"],
+      forbiddenRoutes: [],
+      noHeavyGuard: true,
+      compactOutput: true,
+    },
+  },
+  {
+    id: "dependency_package_change",
+    name: "Dependency/package change",
+    input: {
+      changedFiles: ["package.json", "package-lock.json"],
+    },
+    expected: {
+      taskTypes: ["dependency_change"],
+      allowedRoutes: ["production_confidence"],
+      forbiddenRoutes: [],
+      supplyChainProof: true,
+    },
+  },
+  {
+    id: "deployment_env_change",
+    name: "Deployment/env change",
+    input: {
+      changedFiles: [
+        ".github/workflows/deploy.yml",
+        ".env.production",
+        "scripts/lib/deployment-environment-guard.js",
+      ],
+    },
+    expected: {
+      taskTypes: ["deployment_environment"],
+      allowedRoutes: ["production_confidence"],
+      forbiddenRoutes: [],
+      deploymentGuardEvidence: true,
+    },
+  },
+  {
+    id: "observability_recovery_change",
+    name: "Observability and recovery change",
+    input: {
+      changedFiles: [
+        "scripts/lib/observability-recovery-guard.js",
+        "docs/production-confidence-guard-suite.md",
+      ],
+    },
+    expected: {
+      taskTypes: ["observability_recovery"],
+      allowedRoutes: ["recovery_first", "production_confidence"],
+      forbiddenRoutes: ["direct"],
+      proofExpected: true,
+      proofContains: "production_confidence_required",
+    },
+  },
+  {
+    id: "cost_abuse_change",
+    name: "Cost and abuse change",
+    input: {
+      changedFiles: [
+        "scripts/lib/cost-abuse-guard.js",
+        "tests/cost-abuse-guard.test.js",
+      ],
+    },
+    expected: {
+      taskTypes: ["cost_abuse"],
+      allowedRoutes: ["production_confidence", "security_review"],
+      forbiddenRoutes: ["direct"],
+      proofExpected: true,
+      proofContains: "production_confidence_required",
+    },
+  },
+  {
+    id: "webhook_idempotency_change",
+    name: "Webhook and idempotency change",
+    input: {
+      changedFiles: [
+        "scripts/lib/webhook-idempotency-guard.js",
+        "tests/webhook-idempotency-guard.test.js",
+      ],
+    },
+    expected: {
+      taskTypes: ["webhook_idempotency"],
+      allowedRoutes: ["production_confidence", "security_review"],
+      forbiddenRoutes: ["direct"],
+      proofExpected: true,
+      proofContains: "production_confidence_required",
+    },
+  },
+  {
+    id: "production_confidence_guard_suite_change",
+    name: "Combined production-confidence guard-suite change",
+    input: {
+      changedFiles: [
+        "scripts/lib/production-confidence.js",
+        "scripts/lib/observability-recovery-guard.js",
+        "scripts/lib/cost-abuse-guard.js",
+        "scripts/lib/webhook-idempotency-guard.js",
+      ],
+    },
+    expected: {
+      taskTypes: ["observability_recovery", "cost_abuse", "webhook_idempotency", "implementation"],
+      allowedRoutes: ["production_confidence", "recovery_first", "security_review"],
+      forbiddenRoutes: ["direct"],
+      proofExpected: true,
+      proofContains: "production_confidence_required",
+    },
+  },
+  {
+    id: "product_home_value_signals",
+    name: "Product Home/status/value-signals change",
+    input: {
+      changedFiles: [
+        "scripts/lib/product-home-receipt.js",
+        "scripts/lib/ai-value-ledger.js",
+      ],
+    },
+    expected: {
+      taskTypes: ["value_reporting", "implementation"],
+      allowedRoutes: ["proof_first", "no_connect", "guarded"],
+      forbiddenRoutes: [],
+    },
+  },
+  {
+    id: "guard_blocked_state",
+    name: "Guard blocked state",
+    input: {
+      changedFiles: ["scripts/lib/foo.js"],
+      guardOverride: [
+        { source: "deploymentEnvironment", status: "blocked", blocked: true, path: "mock" },
+      ],
+    },
+    expected: {
+      forbiddenRoutes: ["direct"],
+      noFakeGreen: true,
+      approvalNeeded: true,
+      humanReviewRequired: true,
+    },
+  },
+  {
+    id: "supply_chain_runtime_integration",
+    name: "Supply-chain runtime integration",
+    input: {
+      changedFiles: [
+        "package.json",
+        "package-lock.json",
+        "scripts/lib/supply-chain-guard.js",
+      ],
+    },
+    expected: {
+      taskTypes: ["dependency_change"],
+      allowedRoutes: ["production_confidence"],
+      forbiddenRoutes: ["direct"],
+      supplyChainProof: true,
+      proofExpected: true,
+      proofContains: "production_confidence_required",
+    },
+  },
+  {
+    id: "capability_mcp_tooling",
+    name: "Capability/MCP/tooling change",
+    input: {
+      changedFiles: [
+        "scripts/lib/capability-intake-quarantine.js",
+        "scripts/lib/skill-hygiene.js",
+      ],
+    },
+    expected: {
+      taskTypes: ["skill_mcp_capability"],
+      allowedRoutes: ["capability_quarantine_review", "security_review"],
+      forbiddenRoutes: [],
+      governedExposureSuppressed: true,
+    },
+  },
+  {
+    id: "oversized_noisy_context",
+    name: "Oversized context / noisy state",
+    input: {
+      changedFiles: Array.from({ length: 60 }, (_, i) => `src/file-${i}.js`),
+      stateArtifactOverride: { count: 250, bytes: 600000 },
+    },
+    expected: {
+      budgetClass: "oversized",
+      wastePresent: true,
+      compactFallback: true,
+      forbiddenRoutes: [],
+    },
+  },
+];
+
+function check(label, condition) {
+  return { label, passed: !!condition };
+}
+
+function runScenario(scenario, buildFn) {
+  const { id, name, input, expected } = scenario;
+  const report = buildFn({
+    changedFiles: input.changedFiles,
+    guardOverride: input.guardOverride || undefined,
+    stateArtifactOverride: input.stateArtifactOverride || undefined,
+    taskType: input.taskType || undefined,
+  });
+
+  const checks = [];
+
+  if (expected.taskTypes && expected.taskTypes.length > 0) {
+    checks.push(check(
+      `task type in [${expected.taskTypes.join(", ")}]`,
+      report.task && expected.taskTypes.includes(report.task.type)
+    ));
+  }
+
+  if (expected.allowedRoutes && expected.allowedRoutes.length > 0) {
+    const route = report.routingPlan && report.routingPlan.recommendedRoute;
+    checks.push(check(
+      `route "${route}" in [${expected.allowedRoutes.join(", ")}]`,
+      route && expected.allowedRoutes.includes(route)
+    ));
+  }
+
+  if (expected.forbiddenRoutes && expected.forbiddenRoutes.length > 0) {
+    const route = report.routingPlan && report.routingPlan.recommendedRoute;
+    checks.push(check(
+      `route "${route}" not in [${expected.forbiddenRoutes.join(", ")}]`,
+      !route || !expected.forbiddenRoutes.includes(route)
+    ));
+  }
+
+  checks.push(check(
+    "route confidence exists",
+    report.routingPlan && report.routingPlan.routeConfidence !== undefined && report.routingPlan.routeConfidence !== null
+  ));
+
+  checks.push(check(
+    "governed exposure status not undefined",
+    report.governedExposure && report.governedExposure.status !== undefined
+  ));
+
+  if (expected.proofExpected) {
+    const proofRefs = (report.governedExposure && report.governedExposure.proofExpectations) ||
+      (report.promptCompilerPlan && report.promptCompilerPlan.proofExpectations);
+    checks.push(check("proof expectations present", proofRefs && (Array.isArray(proofRefs) ? proofRefs.length > 0 : !!proofRefs)));
+    if (expected.proofContains) {
+      const flat = JSON.stringify(proofRefs || "");
+      checks.push(check(`proof contains "${expected.proofContains}"`, flat.includes(expected.proofContains)));
+    }
+  }
+
+  checks.push(check(
+    "measuredLocalValue exists",
+    report.measuredLocalValue !== undefined || (report.valueLedger && report.valueLedger.found !== undefined)
+  ));
+
+  checks.push(check(
+    "valueLedger has status",
+    report.valueLedger && report.valueLedger.status !== undefined
+  ));
+
+  checks.push(check(
+    "gate status valid",
+    report.status && ["pass", "warn", "blocked", "no_data"].includes(report.status)
+  ));
+
+  const summaryText = (report.summary && report.summary.text) || "";
+  checks.push(check("compact text output (< 500 chars)", summaryText.length < 500));
+
+  checks.push(check("no model calls", report.limits && report.limits.modelCalls === false));
+  checks.push(check("no network calls", report.limits && report.limits.networkCalls === false));
+  checks.push(check("no provider calls", report.limits && report.limits.providerCalls === false));
+  checks.push(check("no raw dump", report.limits && report.limits.rawPromptDump === false && report.limits.rawCodeDump === false && report.limits.rawLogDump === false));
+
+  if (expected.noFakeSavings) {
+    const avoided = (report.valueLedger && report.valueLedger.avoided) || [];
+    const proved = (report.valueLedger && report.valueLedger.proved) || [];
+    const hasFake = [...avoided, ...proved].some((v) => /fake|synthetic|mock savings/i.test(typeof v === "string" ? v : JSON.stringify(v)));
+    checks.push(check("no fake savings", !hasFake));
+  }
+
+  if (expected.noFakeGreen) {
+    const guardBlocked = input.guardOverride && input.guardOverride.some((g) => g.blocked);
+    checks.push(check("no fake green (blocked guard => status != pass)", !(guardBlocked && report.status === "pass")));
+  }
+
+  if (expected.approvalNeeded !== undefined) {
+    checks.push(check(
+      `approval needed = ${expected.approvalNeeded}`,
+      (report.routingPlan && report.routingPlan.approvalNeeded) === expected.approvalNeeded
+    ));
+  }
+
+  if (expected.humanReviewRequired) {
+    const hasHumanReview = (report.routingPlan && report.routingPlan.approvalNeeded) ||
+      (report.findings && report.findings.some((f) => /human.review|approval/i.test(f.reasonCode || f.message || "")));
+    checks.push(check("human review required", hasHumanReview));
+  }
+
+  if (expected.budgetClass) {
+    checks.push(check(
+      `budget class = "${expected.budgetClass}"`,
+      report.contextBudget && report.contextBudget.budgetClass === expected.budgetClass
+    ));
+  }
+
+  if (expected.wastePresent) {
+    const waste = (report.tokenEfficiency && report.tokenEfficiency.avoidableWasteSignals) || [];
+    const stale = (report.contextBudget && report.contextBudget.staleContextCount) || 0;
+    const dupe = (report.contextBudget && report.contextBudget.duplicateContextCount) || 0;
+    checks.push(check("waste/stale/duplicate signals detected", waste.length > 0 || stale > 0 || dupe > 0));
+  }
+
+  if (expected.noHeavyGuard) {
+    checks.push(check("no heavy guard activation", !(report.limits && report.limits.heavyDashboard)));
+  }
+
+  if (expected.compactOutput || expected.compactFallback) {
+    checks.push(check("compact output (summary text < 500)", summaryText.length < 500));
+  }
+
+  if (expected.supplyChainProof) {
+    const proofRefs = (report.governedExposure && report.governedExposure.proofExpectations) ||
+      (report.promptCompilerPlan && report.promptCompilerPlan.proofExpectations);
+    const contractReady = report.routingPlan && report.routingPlan.requiredProof;
+    checks.push(check("supply-chain proof expectation or contractReady caveat",
+      (proofRefs && JSON.stringify(proofRefs).length > 2) || !!contractReady
+    ));
+  }
+
+  if (expected.deploymentGuardEvidence) {
+    const guardRefs = report.governedExposure && report.governedExposure.guardEvidenceRefs;
+    checks.push(check("deployment guard evidence included", guardRefs !== undefined));
+  }
+
+  if (expected.governedExposureSuppressed) {
+    const suppressed = report.governedExposure && (
+      (report.governedExposure.suppressedCapabilityRefs && report.governedExposure.suppressedCapabilityRefs.length > 0) ||
+      (report.governedExposure.suppressedToolRefs && report.governedExposure.suppressedToolRefs.length > 0) ||
+      report.governedExposure.status === "restricted" ||
+      report.governedExposure.status === "quarantine"
+    );
+    checks.push(check("governed exposure suppresses broad capability/tool exposure", !!suppressed));
+  }
+
+  const allPassed = checks.every((c) => c.passed);
+  const hasBlock = expected.noFakeGreen && !checks.find((c) => c.label.includes("no fake green"))?.passed;
+
+  return { id, name, passed: allPassed, checks };
+}
+
+function runDogfoodPack(buildFn) {
+  const scenarios = [];
+  let scenariosPassed = 0;
+  let scenariosWarned = 0;
+  let scenariosBlocked = 0;
+  const reasonCodes = [];
+
+  for (const scenario of DOGFOOD_SCENARIOS) {
+    let result;
+    try {
+      result = runScenario(scenario, buildFn);
+    } catch (err) {
+      result = {
+        id: scenario.id,
+        name: scenario.name,
+        passed: false,
+        checks: [{ label: `error: ${err.message}`, passed: false }],
+      };
+    }
+    scenarios.push(result);
+
+    if (result.passed) {
+      scenariosPassed++;
+      reasonCodes.push("DOGFOOD_REAL_WORKFLOW_SCENARIO_PASSED");
+    } else {
+      const failedChecks = result.checks.filter((c) => !c.passed);
+      const hasBlock = failedChecks.some((c) => /blocked|fake green|approval/i.test(c.label));
+      if (hasBlock) {
+        scenariosBlocked++;
+        reasonCodes.push("DOGFOOD_REAL_WORKFLOW_SCENARIO_BLOCKED");
+      } else {
+        scenariosWarned++;
+        reasonCodes.push("DOGFOOD_REAL_WORKFLOW_SCENARIO_WARNED");
+      }
+    }
+  }
+
+  if (scenarios.some((s) => s.id === "dependency_package_change" && s.passed)) {
+    reasonCodes.push("DOGFOOD_SUPPLY_CHAIN_CONTRACT_READY");
+  }
+
+  const allPassed = scenariosBlocked === 0 && scenariosWarned === 0;
+  if (allPassed) {
+    reasonCodes.unshift("DOGFOOD_PACK_READY");
+  }
+
+  const status = scenariosBlocked > 0 ? "blocked" : scenariosWarned > 0 ? "warn" : allPassed ? "pass" : "no_data";
+
+  return {
+    status,
+    scenariosRun: scenarios.length,
+    scenariosPassed,
+    scenariosWarned,
+    scenariosBlocked,
+    scenarios,
+    reasonCodes,
+  };
+}
+
+module.exports = { runDogfoodPack, DOGFOOD_SCENARIOS };

package/scripts/lib/ai-efficiency-eval.js

@@ -0,0 +1,207 @@
+"use strict";
+
+const SCENARIOS = [
+  {
+    id: "docs_only",
+    name: "Docs-only update",
+    changedFiles: ["docs/README.md", "docs/getting-started.md"],
+    expected: { taskType: "documentation", routes: ["docs_only", "no_connect"] },
+  },
+  {
+    id: "source_no_tests",
+    name: "Source implementation without tests",
+    changedFiles: ["scripts/lib/new-feature.js"],
+    expected: { taskType: "implementation", routes: ["test_first", "guarded"], proofExpected: ["tests_required"] },
+  },
+  {
+    id: "dependency_change",
+    name: "Dependency/package change",
+    changedFiles: ["package.json", "package-lock.json"],
+    expected: { taskType: "dependency_change", routes: ["production_confidence"] },
+  },
+  {
+    id: "deployment_env",
+    name: "Deployment/env change",
+    changedFiles: [".github/workflows/deploy.yml", ".env.production"],
+    expected: { taskType: "deployment_environment", routes: ["production_confidence"] },
+  },
+  {
+    id: "data_permission",
+    name: "Data/permission change",
+    changedFiles: ["scripts/lib/data-permissions-guard.js"],
+    expected: { taskType: "data_permissions", routes: ["security_review", "production_confidence"] },
+  },
+  {
+    id: "frontend_public",
+    name: "Frontend/public surface change",
+    changedFiles: ["apps/public-web/src/index.html", "apps/public-web/src/styles.css"],
+    expected: { taskType: "public_web", routes: ["browser_proof"], proofExpected: ["browser_proof_required"] },
+  },
+  {
+    id: "capability_mcp",
+    name: "Capability/MCP/skill change",
+    changedFiles: ["scripts/lib/capability-intake-quarantine.js", "scripts/lib/skill-hygiene.js"],
+    expected: { taskType: "skill_mcp_capability", routes: ["capability_quarantine_review", "security_review"] },
+  },
+  {
+    id: "observability_recovery",
+    name: "Observability/recovery change",
+    changedFiles: ["scripts/lib/observability-monitor.js", "scripts/lib/recovery-handler.js"],
+    expected: { taskType: "observability_recovery", routes: ["recovery_first", "production_confidence"] },
+  },
+  {
+    id: "high_risk_mixed",
+    name: "High-risk mixed change",
+    changedFiles: [
+      "scripts/lib/auth-handler.js", "scripts/lib/data-permissions-guard.js",
+      ".env.production", "package.json", ".github/workflows/deploy.yml",
+      "scripts/lib/secret-manager.js", "apps/public-web/src/index.html",
+    ],
+    expected: { routes: ["human_review_required", "security_review"] },
+  },
+  {
+    id: "no_evidence",
+    name: "No relevant evidence",
+    changedFiles: [],
+    expected: { taskType: "unknown", status: "no_data" },
+  },
+  {
+    id: "oversized_context",
+    name: "Oversized noisy context",
+    changedFiles: Array.from({ length: 60 }, (_, i) => `src/file-${i}.js`),
+    stateArtifactOverride: { count: 250, bytes: 600000 },
+    expected: { budgetClass: "oversized", wastePresent: true },
+  },
+  {
+    id: "blocked_guard",
+    name: "Existing guard blocked",
+    changedFiles: ["scripts/lib/deployment-environment-guard.js"],
+    guardOverride: [{ source: "deploymentEnvironment", status: "blocked", blocked: true, path: "mock" }],
+    expected: { routeNot: ["direct"], approvalNeeded: true },
+  },
+];
+
+function runEvalScenario(scenario, buildFn) {
+  const result = buildFn({
+    changedFiles: scenario.changedFiles,
+    guardOverride: scenario.guardOverride,
+    stateArtifactOverride: scenario.stateArtifactOverride,
+  });
+
+  const checks = [];
+  const exp = scenario.expected;
+
+  if (exp.taskType) {
+    checks.push({
+      check: "task_type",
+      passed: result.task.type === exp.taskType,
+      expected: exp.taskType,
+      actual: result.task.type,
+    });
+  }
+
+  if (exp.routes) {
+    checks.push({
+      check: "route",
+      passed: exp.routes.includes(result.routingPlan.recommendedRoute),
+      expected: exp.routes,
+      actual: result.routingPlan.recommendedRoute,
+    });
+  }
+
+  if (exp.routeNot) {
+    checks.push({
+      check: "route_not",
+      passed: !exp.routeNot.includes(result.routingPlan.recommendedRoute),
+      expected: `not in [${exp.routeNot}]`,
+      actual: result.routingPlan.recommendedRoute,
+    });
+  }
+
+  if (exp.status) {
+    checks.push({
+      check: "status",
+      passed: result.status === exp.status,
+      expected: exp.status,
+      actual: result.status,
+    });
+  }
+
+  if (exp.budgetClass) {
+    checks.push({
+      check: "budget_class",
+      passed: result.contextBudget.budgetClass === exp.budgetClass,
+      expected: exp.budgetClass,
+      actual: result.contextBudget.budgetClass,
+    });
+  }
+
+  if (exp.budgetMax) {
+    const order = ["small", "medium", "large", "oversized"];
+    const actualIdx = order.indexOf(result.contextBudget.budgetClass);
+    const maxIdx = order.indexOf(exp.budgetMax);
+    checks.push({
+      check: "budget_max",
+      passed: actualIdx <= maxIdx,
+      expected: `<= ${exp.budgetMax}`,
+      actual: result.contextBudget.budgetClass,
+    });
+  }
+
+  if (exp.proofExpected) {
+    for (const proof of exp.proofExpected) {
+      checks.push({
+        check: `proof_${proof}`,
+        passed: (result.governedExposure.proofExpectations || []).includes(proof),
+        expected: proof,
+        actual: result.governedExposure.proofExpectations,
+      });
+    }
+  }
+
+  if (exp.wastePresent !== undefined) {
+    const hasWaste = (result.tokenEfficiency.avoidableWasteSignals || []).length > 0 ||
+      result.contextBudget.duplicateContextCount > 0 ||
+      result.contextBudget.staleContextCount > 0;
+    checks.push({
+      check: "waste_present",
+      passed: hasWaste === exp.wastePresent,
+      expected: exp.wastePresent,
+      actual: hasWaste,
+    });
+  }
+
+  if (exp.approvalNeeded !== undefined) {
+    checks.push({
+      check: "approval_needed",
+      passed: result.routingPlan.approvalNeeded === exp.approvalNeeded,
+      expected: exp.approvalNeeded,
+      actual: result.routingPlan.approvalNeeded,
+    });
+  }
+
+  return {
+    id: scenario.id,
+    name: scenario.name,
+    passed: checks.every((c) => c.passed),
+    checks,
+  };
+}
+
+function runAllEvalScenarios(buildFn) {
+  const startMs = Date.now();
+  const results = SCENARIOS.map((s) => runEvalScenario(s, buildFn));
+  const passed = results.filter((r) => r.passed).length;
+  const failed = results.filter((r) => !r.passed).length;
+
+  return {
+    status: failed === 0 ? "pass" : "warn",
+    total: results.length,
+    passed,
+    failed,
+    scenarios: results,
+    durationMs: Date.now() - startMs,
+  };
+}
+
+module.exports = { runAllEvalScenarios, runEvalScenario, SCENARIOS };