ccjk 13.3.6 → 13.3.7

package/dist/chunks/doctor.mjs

@@ -1,15 +1,15 @@
 import { existsSync, mkdirSync, statSync, writeFileSync, readFileSync, copyFileSync, readdirSync } from 'node:fs';
 import process__default from 'node:process';
-import a from './index3.mjs';
-import { i as inquirer } from './index4.mjs';
+import a from './index2.mjs';
+import { i as inquirer } from './index3.mjs';
 import { getApiProviderPresets } from './api-providers.mjs';
 import { SETTINGS_FILE, CLAUDE_DIR } from './constants.mjs';
-import { i18n } from './index6.mjs';
+import { i18n } from './index5.mjs';
 import { g as getPermissionManager } from '../shared/ccjk.SPoXMvZD.mjs';
 import { e as commandExists } from './platform.mjs';
 import { P as ProviderHealthMonitor } from '../shared/ccjk.J8YiPsOw.mjs';
 import { platform, userInfo, homedir } from 'node:os';
-import ora from './index8.mjs';
+import ora from './index7.mjs';
 import { exec as q } from './main.mjs';
 import { STATUS } from './banner.mjs';
 import { writeFileAtomic } from './fs-operations.mjs';

package/dist/chunks/eval.mjs

@@ -0,0 +1,502 @@
+import { mkdir, writeFile, readFile } from 'node:fs/promises';
+import { dirname, join, extname } from 'node:path';
+import { existsSync, readFileSync, mkdirSync, writeFileSync, readdirSync, statSync } from 'node:fs';
+import { spawn } from 'node:child_process';
+
+function gradeRun(scenario, result) {
+  const assertionResults = [];
+  for (const assertion of scenario.assertions) {
+    const assertionResult = evaluateAssertion(assertion, result);
+    assertionResults.push(assertionResult);
+  }
+  const success = assertionResults.every((r) => r.success);
+  return {
+    ...result,
+    success,
+    assertionResults
+  };
+}
+function evaluateAssertion(assertion, result) {
+  switch (assertion.type) {
+    case "exit_code":
+      return checkExitCode(assertion, result);
+    case "file_exists":
+      return checkFileExists(assertion);
+    case "contains_text":
+      return checkContainsText(assertion);
+    default:
+      return {
+        type: assertion.type,
+        success: false,
+        message: `Unknown assertion type: ${assertion.type}`
+      };
+  }
+}
+function checkExitCode(assertion, result) {
+  const expected = assertion.expected ?? 0;
+  const success = result.exitCode === expected;
+  return {
+    type: "exit_code",
+    success,
+    message: success ? `Exit code ${result.exitCode} matches expected ${expected}` : `Exit code ${result.exitCode} does not match expected ${expected}`
+  };
+}
+function checkFileExists(assertion) {
+  if (!assertion.path) {
+    return {
+      type: "file_exists",
+      success: false,
+      message: "Missing path for file_exists assertion"
+    };
+  }
+  const exists = existsSync(assertion.path);
+  return {
+    type: "file_exists",
+    success: exists,
+    message: exists ? `File exists: ${assertion.path}` : `File does not exist: ${assertion.path}`
+  };
+}
+function checkContainsText(assertion) {
+  if (!assertion.path) {
+    return {
+      type: "contains_text",
+      success: false,
+      message: "Missing path for contains_text assertion"
+    };
+  }
+  if (!assertion.value) {
+    return {
+      type: "contains_text",
+      success: false,
+      message: "Missing value for contains_text assertion"
+    };
+  }
+  if (!existsSync(assertion.path)) {
+    return {
+      type: "contains_text",
+      success: false,
+      message: `File does not exist: ${assertion.path}`
+    };
+  }
+  const content = readFileSync(assertion.path, "utf-8");
+  const contains = content.includes(assertion.value);
+  return {
+    type: "contains_text",
+    success: contains,
+    message: contains ? `File contains text: "${assertion.value}"` : `File does not contain text: "${assertion.value}"`
+  };
+}
+
+function writeHtmlDashboard(reports, outputPath) {
+  const html = generateHtml(reports);
+  mkdirSync(dirname(outputPath), { recursive: true });
+  writeFileSync(outputPath, html, "utf-8");
+}
+function generateHtml(reports) {
+  const timestamp = (/* @__PURE__ */ new Date()).toISOString();
+  const totalScenarios = reports.length;
+  const passedScenarios = reports.filter((r) => r.successRate === 1).length;
+  const failedScenarios = totalScenarios - passedScenarios;
+  const avgDuration = reports.reduce((sum, r) => sum + r.averageDurationMs, 0) / totalScenarios;
+  return `<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="UTF-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  <title>CCJK Eval Dashboard</title>
+  <style>
+    body {
+      font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', sans-serif;
+      max-width: 1200px;
+      margin: 0 auto;
+      padding: 20px;
+      background: #f5f5f5;
+    }
+    h1 { color: #333; }
+    .summary {
+      background: white;
+      padding: 20px;
+      border-radius: 8px;
+      margin-bottom: 20px;
+      box-shadow: 0 2px 4px rgba(0,0,0,0.1);
+    }
+    .summary-grid {
+      display: grid;
+      grid-template-columns: repeat(auto-fit, minmax(200px, 1fr));
+      gap: 15px;
+      margin-top: 15px;
+    }
+    .summary-item {
+      padding: 10px;
+      background: #f9f9f9;
+      border-radius: 4px;
+    }
+    .summary-item strong { display: block; font-size: 24px; margin-top: 5px; }
+    table {
+      width: 100%;
+      background: white;
+      border-collapse: collapse;
+      border-radius: 8px;
+      overflow: hidden;
+      box-shadow: 0 2px 4px rgba(0,0,0,0.1);
+    }
+    th, td { padding: 12px; text-align: left; }
+    th { background: #333; color: white; }
+    tr:nth-child(even) { background: #f9f9f9; }
+    .pass { color: #22c55e; font-weight: bold; }
+    .fail { color: #ef4444; font-weight: bold; }
+    .timestamp { color: #666; font-size: 14px; }
+  </style>
+</head>
+<body>
+  <h1>CCJK Eval Dashboard</h1>
+  <p class="timestamp">Generated: ${timestamp}</p>
+
+  <div class="summary">
+    <h2>Summary</h2>
+    <div class="summary-grid">
+      <div class="summary-item">
+        <div>Total Scenarios</div>
+        <strong>${totalScenarios}</strong>
+      </div>
+      <div class="summary-item">
+        <div>Passed</div>
+        <strong class="pass">${passedScenarios}</strong>
+      </div>
+      <div class="summary-item">
+        <div>Failed</div>
+        <strong class="fail">${failedScenarios}</strong>
+      </div>
+      <div class="summary-item">
+        <div>Avg Duration</div>
+        <strong>${Math.round(avgDuration)}ms</strong>
+      </div>
+    </div>
+  </div>
+
+  <table>
+    <thead>
+      <tr>
+        <th>Scenario</th>
+        <th>Suite</th>
+        <th>Success Rate</th>
+        <th>Passed/Total</th>
+        <th>Avg Duration</th>
+      </tr>
+    </thead>
+    <tbody>
+${reports.map((r) => `      <tr>
+        <td>${r.scenarioId}</td>
+        <td>${r.suite}</td>
+        <td class="${r.successRate === 1 ? "pass" : "fail"}">${(r.successRate * 100).toFixed(0)}%</td>
+        <td>${r.passedRuns}/${r.totalRuns}</td>
+        <td>${Math.round(r.averageDurationMs)}ms</td>
+      </tr>`).join("\n")}
+    </tbody>
+  </table>
+</body>
+</html>`;
+}
+
+async function runScenario(scenario) {
+  const runs = scenario.runs || 1;
+  const results = [];
+  for (let i = 0; i < runs; i++) {
+    const result = await runOnce(scenario, i);
+    results.push(result);
+  }
+  return results;
+}
+async function runOnce(scenario, runIndex) {
+  const startTime = Date.now();
+  return new Promise((resolve) => {
+    const [cmd, ...args] = scenario.command.split(/\s+/);
+    const proc = spawn(cmd, args, {
+      shell: true,
+      stdio: ["ignore", "pipe", "pipe"]
+    });
+    let stdout = "";
+    let stderr = "";
+    proc.stdout?.on("data", (data) => {
+      stdout += data.toString();
+    });
+    proc.stderr?.on("data", (data) => {
+      stderr += data.toString();
+    });
+    proc.on("close", (code) => {
+      const durationMs = Date.now() - startTime;
+      const exitCode = code ?? -1;
+      resolve({
+        scenarioId: scenario.id,
+        runIndex,
+        success: false,
+        // grader will set this
+        durationMs,
+        exitCode,
+        metrics: {
+          duration_ms: durationMs,
+          output_bytes: stdout.length + stderr.length
+        },
+        assertionResults: [],
+        stdout,
+        stderr
+      });
+    });
+    proc.on("error", (err) => {
+      const durationMs = Date.now() - startTime;
+      resolve({
+        scenarioId: scenario.id,
+        runIndex,
+        success: false,
+        durationMs,
+        exitCode: -1,
+        metrics: { duration_ms: durationMs },
+        assertionResults: [{
+          type: "spawn_error",
+          success: false,
+          message: `Failed to spawn process: ${err.message}`
+        }],
+        stderr: err.message
+      });
+    });
+  });
+}
+
+function loadScenarios(scenariosDir, filter) {
+  const scenarios = [];
+  function scanDir(dir) {
+    const entries = readdirSync(dir);
+    for (const entry of entries) {
+      const fullPath = join(dir, entry);
+      const stat = statSync(fullPath);
+      if (stat.isDirectory()) {
+        scanDir(fullPath);
+      } else if (extname(entry) === ".json") {
+        try {
+          const content = readFileSync(fullPath, "utf-8");
+          const scenario = JSON.parse(content);
+          validateScenario(scenario, fullPath);
+          if (filter?.suite && scenario.suite !== filter.suite)
+            continue;
+          if (filter?.id && scenario.id !== filter.id)
+            continue;
+          scenarios.push(scenario);
+        } catch (err) {
+          throw new Error(`Failed to load scenario from ${fullPath}: ${err}`);
+        }
+      }
+    }
+  }
+  scanDir(scenariosDir);
+  return scenarios;
+}
+function validateScenario(scenario, path) {
+  const required = ["id", "suite", "description", "command", "assertions"];
+  for (const field of required) {
+    if (!scenario[field]) {
+      throw new Error(`Scenario ${path} missing required field: ${field}`);
+    }
+  }
+  if (!Array.isArray(scenario.assertions)) {
+    throw new TypeError(`Scenario ${path} assertions must be an array`);
+  }
+  for (const assertion of scenario.assertions) {
+    if (!assertion.type) {
+      throw new Error(`Scenario ${path} has assertion without type`);
+    }
+  }
+}
+
+async function writeScenarioReport(report, options = {}) {
+  const outputPath = join(
+    process.cwd(),
+    "evals/reports",
+    report.suite,
+    `${report.scenarioId}.json`
+  );
+  const data = {
+    ...report,
+    results: report.results.map((run) => ({
+      ...run,
+      stdout: options.includeStdout ? run.stdout : void 0,
+      stderr: options.includeStderr ? run.stderr : void 0
+    }))
+  };
+  await mkdir(dirname(outputPath), { recursive: true });
+  await writeFile(outputPath, JSON.stringify(data, null, 2), "utf-8");
+  return outputPath;
+}
+function createSuiteSummary(suite, scenarios) {
+  const totalScenarios = scenarios.length;
+  const passedScenarios = scenarios.filter((s) => s.successRate === 1).length;
+  const failedScenarios = totalScenarios - passedScenarios;
+  const successRate = totalScenarios === 0 ? 0 : passedScenarios / totalScenarios;
+  const averageScore = totalScenarios === 0 ? 0 : scenarios.reduce((sum, s) => sum + (s.averageScore ?? s.successRate), 0) / totalScenarios;
+  return {
+    suite,
+    totalScenarios,
+    passedScenarios,
+    failedScenarios,
+    successRate,
+    averageScore,
+    scenarios
+  };
+}
+async function writeSuiteSummary(summary) {
+  const outputPath = join(
+    process.cwd(),
+    "evals/reports",
+    summary.suite,
+    "_summary.json"
+  );
+  await mkdir(dirname(outputPath), { recursive: true });
+  await writeFile(outputPath, JSON.stringify(summary, null, 2), "utf-8");
+  return outputPath;
+}
+function compareReports(baseline, candidate) {
+  const deltaSuccessRate = candidate.successRate - baseline.successRate;
+  const deltaDurationPercent = (candidate.averageDurationMs - baseline.averageDurationMs) / baseline.averageDurationMs * 100;
+  return {
+    scenarioId: baseline.scenarioId,
+    baselineSuccessRate: baseline.successRate,
+    candidateSuccessRate: candidate.successRate,
+    deltaSuccessRate,
+    baselineAverageDurationMs: baseline.averageDurationMs,
+    candidateAverageDurationMs: candidate.averageDurationMs,
+    deltaDurationPercent,
+    regression: deltaSuccessRate < -0.05
+  };
+}
+async function writeComparisonReport(comparisons) {
+  const outputPath = join(
+    process.cwd(),
+    "evals/reports",
+    `comparison-${Date.now()}.json`
+  );
+  await mkdir(dirname(outputPath), { recursive: true });
+  await writeFile(outputPath, JSON.stringify(comparisons, null, 2), "utf-8");
+  return outputPath;
+}
+
+async function evalRun(options = {}) {
+  const {
+    scenario,
+    suite,
+    runs = 1,
+    verbose = false,
+    html = true,
+    json = true
+  } = options;
+  console.log("\u{1F50D} Loading scenarios...");
+  const scenarios = loadScenarios("evals/scenarios", {
+    suite,
+    id: scenario
+  });
+  if (scenarios.length === 0) {
+    console.error("\u274C No scenarios found");
+    process.exit(1);
+  }
+  console.log(`\u{1F4CB} Found ${scenarios.length} scenario(s)
+`);
+  const reports = [];
+  for (const scenario2 of scenarios) {
+    console.log(`\u25B6\uFE0F  Running: ${scenario2.id}`);
+    const results = await runScenario({
+      ...scenario2,
+      runs
+    });
+    const gradedResults = results.map((r) => gradeRun(scenario2, r));
+    const passedRuns = gradedResults.filter((r) => r.success).length;
+    const totalRuns = gradedResults.length;
+    const avgDuration = gradedResults.reduce((sum, r) => sum + r.durationMs, 0) / totalRuns;
+    const avgScore = gradedResults.reduce((sum, r) => sum + (r.score ?? (r.success ? 1 : 0)), 0) / totalRuns;
+    const report = {
+      scenarioId: scenario2.id,
+      suite: scenario2.suite,
+      totalRuns,
+      passedRuns,
+      failedRuns: totalRuns - passedRuns,
+      averageDurationMs: avgDuration,
+      successRate: passedRuns / totalRuns,
+      averageScore: avgScore,
+      results: gradedResults
+    };
+    reports.push(report);
+    const status = report.successRate === 1 ? "\u2705" : "\u274C";
+    console.log(`${status} ${scenario2.id}: ${report.passedRuns}/${report.totalRuns} passed (${(report.successRate * 100).toFixed(0)}%)
+`);
+    if (json) {
+      await writeScenarioReport(report, { includeStdout: verbose, includeStderr: verbose });
+    }
+  }
+  const suiteMap = /* @__PURE__ */ new Map();
+  for (const report of reports) {
+    const suiteReports = suiteMap.get(report.suite) || [];
+    suiteReports.push(report);
+    suiteMap.set(report.suite, suiteReports);
+  }
+  for (const [suite2, suiteReports] of suiteMap) {
+    const summary = createSuiteSummary(suite2, suiteReports);
+    if (json) {
+      await writeSuiteSummary(summary);
+    }
+  }
+  if (html) {
+    const outputPath = join(process.cwd(), "evals/reports", `dashboard-${Date.now()}.html`);
+    writeHtmlDashboard(reports, outputPath);
+    console.log(`\u{1F4CA} Dashboard: ${outputPath}
+`);
+  }
+  console.log("\u{1F4CA} Summary:");
+  console.log(`   Total Scenarios: ${reports.length}`);
+  console.log(`   Passed: ${reports.filter((r) => r.successRate === 1).length}`);
+  console.log(`   Failed: ${reports.filter((r) => r.successRate < 1).length}`);
+  const overallSuccessRate = reports.reduce((sum, r) => sum + r.successRate, 0) / reports.length;
+  console.log(`   Success Rate: ${(overallSuccessRate * 100).toFixed(1)}%`);
+  if (reports.some((r) => r.successRate < 1)) {
+    process.exit(1);
+  }
+}
+async function evalCommand(options) {
+  if (options.compare) {
+    if (!options.baseline || !options.candidate) {
+      console.error("\u274C Both --baseline and --candidate are required for comparison");
+      process.exit(1);
+    }
+    await evalCompare({ baseline: options.baseline, candidate: options.candidate, verbose: options.verbose });
+    return;
+  }
+  await evalRun({
+    scenario: options.scenario,
+    suite: options.suite,
+    runs: options.runs,
+    verbose: options.verbose,
+    html: options.html,
+    json: options.json
+  });
+}
+async function evalCompare(options) {
+  const { baseline, candidate } = options;
+  console.log("\u{1F4CA} Loading reports...");
+  const baselineContent = await readFile(baseline, "utf-8");
+  const candidateContent = await readFile(candidate, "utf-8");
+  const baselineReport = JSON.parse(baselineContent);
+  const candidateReport = JSON.parse(candidateContent);
+  console.log(`Baseline: ${baseline}`);
+  console.log(`Candidate: ${candidate}
+`);
+  const comparison = compareReports(baselineReport, candidateReport);
+  console.log(`Scenario: ${comparison.scenarioId}`);
+  console.log(`Success Rate: ${(comparison.baselineSuccessRate * 100).toFixed(0)}% \u2192 ${(comparison.candidateSuccessRate * 100).toFixed(0)}% (${comparison.deltaSuccessRate >= 0 ? "+" : ""}${(comparison.deltaSuccessRate * 100).toFixed(1)}%)`);
+  console.log(`Duration: ${comparison.baselineAverageDurationMs.toFixed(0)}ms \u2192 ${comparison.candidateAverageDurationMs.toFixed(0)}ms (${comparison.deltaDurationPercent >= 0 ? "+" : ""}${comparison.deltaDurationPercent.toFixed(1)}%)
+`);
+  if (comparison.regression) {
+    console.log("\u274C REGRESSION DETECTED");
+    process.exit(1);
+  } else {
+    console.log("\u2705 No regression");
+  }
+  await writeComparisonReport([comparison]);
+}
+
+export { evalCommand, evalCompare, evalRun };