@kodax-ai/kodax-cli 0.7.38

package/dist/builtin/skill-creator/scripts/aggregate-benchmark.js

@@ -0,0 +1,209 @@
+#!/usr/bin/env node
+
+import { readFile, readdir, stat, writeFile } from 'node:fs/promises';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { calculateStats, formatDelta } from './utils.js';
+
+async function readJson(filePath) {
+  return JSON.parse(await readFile(filePath, 'utf8'));
+}
+
+async function pathExists(filePath) {
+  try {
+    await stat(filePath);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+async function listDirectories(dirPath) {
+  const entries = await readdir(dirPath, { withFileTypes: true }).catch(() => []);
+  return entries
+    .filter((entry) => entry.isDirectory())
+    .map((entry) => path.join(dirPath, entry.name))
+    .sort((left, right) => left.localeCompare(right));
+}
+
+export async function loadRunResults(iterationDir) {
+  const runsRoot = await pathExists(path.join(iterationDir, 'runs'))
+    ? path.join(iterationDir, 'runs')
+    : iterationDir;
+
+  const evalDirs = (await listDirectories(runsRoot))
+    .filter((dirPath) => path.basename(dirPath).startsWith('eval-'));
+
+  const configs = {};
+
+  for (const evalDir of evalDirs) {
+    const metadataPath = path.join(evalDir, 'eval_metadata.json');
+    const metadata = await readJson(metadataPath).catch(() => ({}));
+    const evalId = metadata.eval_id ?? path.basename(evalDir);
+
+    for (const configDir of await listDirectories(evalDir)) {
+      const configName = path.basename(configDir);
+      const runDirs = (await listDirectories(configDir))
+        .filter((dirPath) => path.basename(dirPath).startsWith('run-'));
+
+      if (runDirs.length === 0) {
+        continue;
+      }
+
+      configs[configName] ??= [];
+
+      for (const runDir of runDirs) {
+        const grading = await readJson(path.join(runDir, 'grading.json')).catch(() => null);
+        if (!grading) {
+          continue;
+        }
+
+        const timing = await readJson(path.join(runDir, 'timing.json')).catch(() => ({}));
+
+        configs[configName].push({
+          eval_id: evalId,
+          run_id: path.basename(runDir),
+          pass_rate: grading.summary?.pass_rate ?? 0,
+          passed: grading.summary?.passed ?? 0,
+          failed: grading.summary?.failed ?? 0,
+          total: grading.summary?.total ?? 0,
+          time_seconds: timing.total_duration_seconds ?? grading.timing?.total_duration_seconds ?? 0,
+          tokens: timing.total_tokens ?? grading.execution_metrics?.output_chars ?? 0,
+          tool_calls: grading.execution_metrics?.total_tool_calls ?? 0,
+          errors: grading.execution_metrics?.errors_encountered ?? 0,
+          expectations: Array.isArray(grading.expectations) ? grading.expectations : [],
+          notes: [
+            ...(grading.user_notes_summary?.uncertainties ?? []),
+            ...(grading.user_notes_summary?.needs_review ?? []),
+            ...(grading.user_notes_summary?.workarounds ?? []),
+          ],
+        });
+      }
+    }
+  }
+
+  return configs;
+}
+
+export function summarizeConfigs(configRuns) {
+  const summary = {};
+
+  for (const [configName, runs] of Object.entries(configRuns)) {
+    summary[configName] = {
+      pass_rate: calculateStats(runs.map((run) => Number(run.pass_rate ?? 0))),
+      time_seconds: calculateStats(runs.map((run) => Number(run.time_seconds ?? 0))),
+      tokens: calculateStats(runs.map((run) => Number(run.tokens ?? 0))),
+    };
+  }
+
+  const orderedConfigs = Object.keys(summary);
+  const primary = summary[orderedConfigs[0]] ?? {
+    pass_rate: { mean: 0 },
+    time_seconds: { mean: 0 },
+    tokens: { mean: 0 },
+  };
+  const baseline = summary[orderedConfigs[1]] ?? {
+    pass_rate: { mean: 0 },
+    time_seconds: { mean: 0 },
+    tokens: { mean: 0 },
+  };
+
+  return {
+    configs: summary,
+    delta: {
+      pass_rate: formatDelta(primary.pass_rate.mean - baseline.pass_rate.mean),
+      time_seconds: formatDelta(primary.time_seconds.mean - baseline.time_seconds.mean),
+      tokens: formatDelta(primary.tokens.mean - baseline.tokens.mean),
+    },
+  };
+}
+
+export function buildBenchmarkDocument(iterationDir, skillName, configRuns) {
+  const summary = summarizeConfigs(configRuns);
+
+  return {
+    skill_name: skillName,
+    generated_at: new Date().toISOString(),
+    workspace: path.resolve(iterationDir),
+    configs: summary.configs,
+    delta: summary.delta,
+    runs: configRuns,
+  };
+}
+
+export function renderBenchmarkMarkdown(benchmark) {
+  const lines = [
+    `# Benchmark: ${benchmark.skill_name}`,
+    '',
+    `Generated: ${benchmark.generated_at}`,
+    '',
+    '| Config | Pass Rate | Time (s) | Tokens |',
+    '| --- | --- | --- | --- |',
+  ];
+
+  for (const [configName, metrics] of Object.entries(benchmark.configs)) {
+    lines.push(
+      `| ${configName} | ${metrics.pass_rate.mean} ± ${metrics.pass_rate.stddev} | ${metrics.time_seconds.mean} ± ${metrics.time_seconds.stddev} | ${metrics.tokens.mean} ± ${metrics.tokens.stddev} |`
+    );
+  }
+
+  lines.push('');
+  lines.push('## Delta');
+  lines.push('');
+  lines.push(`- Pass rate: ${benchmark.delta.pass_rate}`);
+  lines.push(`- Time (s): ${benchmark.delta.time_seconds}`);
+  lines.push(`- Tokens: ${benchmark.delta.tokens}`);
+
+  return lines.join('\n');
+}
+
+function parseArgs(argv) {
+  const args = {
+    iterationDir: argv[2],
+    skillName: 'unknown-skill',
+  };
+
+  for (let index = 3; index < argv.length; index += 1) {
+    const token = argv[index];
+    if (token === '--skill-name' && argv[index + 1]) {
+      args.skillName = argv[index + 1];
+      index += 1;
+    }
+  }
+
+  return args;
+}
+
+async function main() {
+  const { iterationDir, skillName } = parseArgs(process.argv);
+  if (!iterationDir) {
+    console.error('Usage: node scripts/aggregate-benchmark.js <iteration-dir> --skill-name <name>');
+    process.exit(1);
+  }
+
+  const configRuns = await loadRunResults(iterationDir);
+  if (Object.keys(configRuns).length === 0) {
+    console.error(`No benchmark runs found in ${iterationDir}`);
+    process.exit(1);
+  }
+
+  const benchmark = buildBenchmarkDocument(iterationDir, skillName, configRuns);
+  const benchmarkJsonPath = path.join(iterationDir, 'benchmark.json');
+  const benchmarkMdPath = path.join(iterationDir, 'benchmark.md');
+
+  await writeFile(benchmarkJsonPath, JSON.stringify(benchmark, null, 2));
+  await writeFile(benchmarkMdPath, `${renderBenchmarkMarkdown(benchmark)}\n`);
+
+  console.log(`Wrote ${benchmarkJsonPath}`);
+  console.log(`Wrote ${benchmarkMdPath}`);
+}
+
+const isDirectRun = process.argv[1]
+  && fileURLToPath(import.meta.url) === path.resolve(process.argv[1]);
+
+if (isDirectRun) {
+  main().catch((error) => {
+    console.error(error instanceof Error ? error.message : String(error));
+    process.exit(1);
+  });
+}

package/dist/builtin/skill-creator/scripts/analyze-benchmark.d.ts

@@ -0,0 +1,46 @@
+export interface AnalyzeBenchmarkOptions {
+  workspaceDir: string;
+  benchmarkPath?: string;
+  outputPath?: string;
+  markdownPath?: string;
+  skillName?: string;
+  provider?: string;
+  model?: string;
+  reasoningMode?: string;
+  maxIter?: number;
+  cwd?: string;
+}
+
+export interface BenchmarkAnalysis {
+  skill_name: string;
+  generated_at: string;
+  workspace: string;
+  verdict: 'improves' | 'regresses' | 'mixed' | 'inconclusive';
+  release_readiness: 'ready' | 'needs_iteration' | 'needs_manual_review';
+  recommendation: string;
+  key_findings: string[];
+  variance_hotspots: string[];
+  suggested_actions: string[];
+  watchouts: string[];
+  supporting_metrics: {
+    pass_rate_delta: string;
+    time_seconds_delta: string;
+    tokens_delta: string;
+  };
+  failure_clusters: Record<string, unknown>;
+}
+
+export function buildAnalysisPrompt(input: Record<string, unknown>): string;
+
+export function renderAnalysisMarkdown(analysis: Record<string, any>): string;
+
+export function analyzeBenchmark(
+  options: AnalyzeBenchmarkOptions,
+  runner?: (prompt: string, options: Record<string, unknown>) => Promise<string>
+): Promise<{
+  analysis: BenchmarkAnalysis;
+  prompt: string;
+  rawResponse: string;
+  analysisJsonPath: string;
+  analysisMdPath: string;
+}>;

package/dist/builtin/skill-creator/scripts/analyze-benchmark.js

@@ -0,0 +1,289 @@
+#!/usr/bin/env node
+
+import { readFile, writeFile } from 'node:fs/promises';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import {
+  buildBenchmarkDocument,
+  loadRunResults,
+} from './aggregate-benchmark.js';
+import {
+  extractJsonObject,
+  loadKodaXSDK,
+  loadRelativeText,
+  readJsonFile,
+  truncateText,
+} from './utils.js';
+
+function normalizeStringArray(value) {
+  if (!Array.isArray(value)) {
+    return [];
+  }
+  return value
+    .map((item) => String(item ?? '').trim())
+    .filter(Boolean);
+}
+
+function summarizeFailureClusters(configRuns) {
+  const clusters = {};
+
+  for (const [configName, runs] of Object.entries(configRuns)) {
+    const failureCounts = new Map();
+    const notes = [];
+
+    for (const run of runs) {
+      for (const expectation of run.expectations ?? []) {
+        if (expectation?.passed === true) {
+          continue;
+        }
+        const text = String(expectation?.text ?? '').trim();
+        if (!text) {
+          continue;
+        }
+        failureCounts.set(text, (failureCounts.get(text) ?? 0) + 1);
+      }
+
+      for (const note of run.notes ?? []) {
+        const normalized = String(note ?? '').trim();
+        if (normalized) {
+          notes.push(normalized);
+        }
+      }
+    }
+
+    clusters[configName] = {
+      repeated_failures: Array.from(failureCounts.entries())
+        .sort((left, right) => right[1] - left[1])
+        .slice(0, 10)
+        .map(([text, count]) => ({ text, count })),
+      notes: notes.slice(0, 10),
+    };
+  }
+
+  return clusters;
+}
+
+function normalizeAnalysisResult(rawText, benchmark, failureClusters) {
+  const parsed = extractJsonObject(rawText) ?? {};
+
+  return {
+    skill_name: benchmark.skill_name,
+    generated_at: new Date().toISOString(),
+    workspace: benchmark.workspace,
+    verdict: ['improves', 'regresses', 'mixed', 'inconclusive'].includes(parsed.verdict)
+      ? parsed.verdict
+      : 'inconclusive',
+    release_readiness: ['ready', 'needs_iteration', 'needs_manual_review'].includes(parsed.release_readiness)
+      ? parsed.release_readiness
+      : 'needs_manual_review',
+    recommendation: String(parsed.recommendation ?? '').trim(),
+    key_findings: normalizeStringArray(parsed.key_findings),
+    variance_hotspots: normalizeStringArray(parsed.variance_hotspots),
+    suggested_actions: normalizeStringArray(parsed.suggested_actions),
+    watchouts: normalizeStringArray(parsed.watchouts),
+    supporting_metrics: {
+      pass_rate_delta: benchmark.delta?.pass_rate ?? 'n/a',
+      time_seconds_delta: benchmark.delta?.time_seconds ?? 'n/a',
+      tokens_delta: benchmark.delta?.tokens ?? 'n/a',
+    },
+    failure_clusters: failureClusters,
+  };
+}
+
+export function buildAnalysisPrompt(input) {
+  return `${input.agentInstructions.trim()}
+
+Return JSON with this shape:
+{
+  "verdict": "improves | regresses | mixed | inconclusive",
+  "release_readiness": "ready | needs_iteration | needs_manual_review",
+  "recommendation": "short recommendation",
+  "key_findings": [],
+  "variance_hotspots": [],
+  "suggested_actions": [],
+  "watchouts": []
+}
+
+## Benchmark Summary
+${truncateText(JSON.stringify({
+  skill_name: input.benchmark.skill_name,
+  configs: input.benchmark.configs,
+  delta: input.benchmark.delta,
+}, null, 2), 12000)}
+
+## Failure Clusters
+${truncateText(JSON.stringify(input.failureClusters, null, 2), 8000)}
+`;
+}
+
+async function defaultRunAnalyst(prompt, options) {
+  const { runKodaX } = await loadKodaXSDK();
+  const result = await runKodaX(
+    {
+      provider: options.provider ?? 'anthropic',
+      model: options.model,
+      maxIter: options.maxIter ?? 20,
+      reasoningMode: options.reasoningMode ?? 'balanced',
+      thinking: options.reasoningMode ? options.reasoningMode !== 'off' : true,
+      context: {
+        gitRoot: path.resolve(options.cwd ?? options.workspaceDir ?? process.cwd()),
+      },
+    },
+    prompt
+  );
+  return result.lastText;
+}
+
+export function renderAnalysisMarkdown(analysis) {
+  const lines = [
+    `# Benchmark Analysis: ${analysis.skill_name}`,
+    '',
+    `Generated: ${analysis.generated_at}`,
+    '',
+    `- Verdict: ${analysis.verdict}`,
+    `- Release readiness: ${analysis.release_readiness}`,
+    `- Recommendation: ${analysis.recommendation || 'n/a'}`,
+    '',
+  ];
+
+  const sections = [
+    ['key_findings', 'Key Findings'],
+    ['variance_hotspots', 'Variance Hotspots'],
+    ['suggested_actions', 'Suggested Actions'],
+    ['watchouts', 'Watchouts'],
+  ];
+
+  for (const [field, title] of sections) {
+    lines.push(`## ${title}`);
+    lines.push('');
+    const items = Array.isArray(analysis[field]) ? analysis[field] : [];
+    if (items.length === 0) {
+      lines.push('- None');
+    } else {
+      for (const item of items) {
+        lines.push(`- ${item}`);
+      }
+    }
+    lines.push('');
+  }
+
+  lines.push('## Supporting Metrics');
+  lines.push('');
+  lines.push(`- Pass rate delta: ${analysis.supporting_metrics.pass_rate_delta}`);
+  lines.push(`- Time delta: ${analysis.supporting_metrics.time_seconds_delta}`);
+  lines.push(`- Tokens delta: ${analysis.supporting_metrics.tokens_delta}`);
+
+  return `${lines.join('\n')}\n`;
+}
+
+export async function analyzeBenchmark(
+  options,
+  runner = defaultRunAnalyst
+) {
+  const workspaceDir = path.resolve(options.workspaceDir);
+  const benchmarkPath = path.resolve(options.benchmarkPath ?? path.join(workspaceDir, 'benchmark.json'));
+  let benchmark = await readJsonFile(benchmarkPath, null);
+
+  if (!benchmark) {
+    const configRuns = await loadRunResults(workspaceDir);
+    if (Object.keys(configRuns).length === 0) {
+      throw new Error(`No benchmark data found in ${workspaceDir}`);
+    }
+    benchmark = buildBenchmarkDocument(workspaceDir, options.skillName ?? path.basename(workspaceDir), configRuns);
+    await writeFile(benchmarkPath, `${JSON.stringify(benchmark, null, 2)}\n`, 'utf8');
+  }
+
+  const configRuns = await loadRunResults(workspaceDir);
+  const failureClusters = summarizeFailureClusters(configRuns);
+  const agentInstructions = await loadRelativeText(import.meta.url, '../agents/analyzer.md');
+  const prompt = buildAnalysisPrompt({
+    agentInstructions,
+    benchmark,
+    failureClusters,
+  });
+  const rawResponse = await runner(prompt, {
+    ...options,
+    workspaceDir,
+    benchmarkPath,
+    benchmark,
+  });
+  const analysis = normalizeAnalysisResult(rawResponse, benchmark, failureClusters);
+  const analysisJsonPath = path.resolve(options.outputPath ?? path.join(workspaceDir, 'analysis.json'));
+  const analysisMdPath = path.resolve(options.markdownPath ?? path.join(workspaceDir, 'analysis.md'));
+
+  await writeFile(analysisJsonPath, `${JSON.stringify(analysis, null, 2)}\n`, 'utf8');
+  await writeFile(analysisMdPath, renderAnalysisMarkdown(analysis), 'utf8');
+
+  return {
+    analysis,
+    prompt,
+    rawResponse,
+    analysisJsonPath,
+    analysisMdPath,
+  };
+}
+
+function parseArgs(argv) {
+  const args = {
+    workspaceDir: argv[2] ?? '',
+    benchmarkPath: undefined,
+    outputPath: undefined,
+    markdownPath: undefined,
+    skillName: undefined,
+    provider: 'anthropic',
+    model: undefined,
+    reasoningMode: 'balanced',
+    maxIter: 20,
+    cwd: process.cwd(),
+  };
+
+  for (let index = 3; index < argv.length; index += 1) {
+    const token = argv[index];
+    if (token === '--benchmark' && argv[index + 1]) {
+      args.benchmarkPath = argv[++index];
+    } else if (token === '--output' && argv[index + 1]) {
+      args.outputPath = argv[++index];
+    } else if (token === '--markdown' && argv[index + 1]) {
+      args.markdownPath = argv[++index];
+    } else if (token === '--skill-name' && argv[index + 1]) {
+      args.skillName = argv[++index];
+    } else if (token === '--provider' && argv[index + 1]) {
+      args.provider = argv[++index];
+    } else if (token === '--model' && argv[index + 1]) {
+      args.model = argv[++index];
+    } else if (token === '--reasoning' && argv[index + 1]) {
+      args.reasoningMode = argv[++index];
+    } else if (token === '--max-iter' && argv[index + 1]) {
+      args.maxIter = Number(argv[++index]);
+    } else if (token === '--cwd' && argv[index + 1]) {
+      args.cwd = argv[++index];
+    }
+  }
+
+  return args;
+}
+
+async function main() {
+  const args = parseArgs(process.argv);
+  if (!args.workspaceDir) {
+    console.error('Usage: node scripts/analyze-benchmark.js <workspace> [--benchmark benchmark.json] [--output analysis.json] [--markdown analysis.md]');
+    process.exit(1);
+  }
+
+  const result = await analyzeBenchmark(args);
+  process.stdout.write(`${JSON.stringify({
+    analysis: result.analysis,
+    analysis_json: result.analysisJsonPath,
+    analysis_md: result.analysisMdPath,
+  }, null, 2)}\n`);
+}
+
+const isDirectRun = process.argv[1]
+  && fileURLToPath(import.meta.url) === path.resolve(process.argv[1]);
+
+if (isDirectRun) {
+  main().catch((error) => {
+    console.error(error instanceof Error ? error.message : String(error));
+    process.exit(1);
+  });
+}

package/dist/builtin/skill-creator/scripts/compare-runs.d.ts

@@ -0,0 +1,62 @@
+export interface CompareWorkspaceOptions {
+  workspaceDir: string;
+  configA?: string;
+  configB?: string;
+  outputPath?: string;
+  markdownPath?: string;
+  provider?: string;
+  model?: string;
+  reasoningMode?: string;
+  maxIter?: number;
+  maxPairs?: number;
+  cwd?: string;
+}
+
+export function buildComparisonPrompt(input: Record<string, unknown>): string;
+
+export interface ComparisonSummary {
+  total_pairs: number;
+  config_a_wins: number;
+  config_b_wins: number;
+  ties: number;
+  inconclusive: number;
+}
+
+export interface ComparisonEntry {
+  index: number;
+  eval_id: string | number | null;
+  eval_name: string | null;
+  run_a: string;
+  run_b: string;
+  presented_as: {
+    A: string;
+    B: string;
+  };
+  winner_label: 'A' | 'B' | 'tie' | 'inconclusive';
+  winner_config: string;
+  confidence: number;
+  rationale: string;
+  strengths_a: string[];
+  strengths_b: string[];
+  risks: string[];
+}
+
+export interface ComparisonDocument {
+  workspace: string;
+  generated_at: string;
+  config_a: string;
+  config_b: string;
+  summary: ComparisonSummary;
+  comparisons: ComparisonEntry[];
+}
+
+export function renderComparisonMarkdown(document: ComparisonDocument): string;
+
+export function compareWorkspace(
+  options: CompareWorkspaceOptions,
+  runner?: (prompt: string, options: Record<string, unknown>) => Promise<string>
+): Promise<{
+  document: ComparisonDocument;
+  outputPath: string;
+  markdownPath: string;
+}>;