@kodax-ai/kodax 0.7.39

package/dist/builtin/skill-creator/references/schemas.md

@@ -0,0 +1,227 @@
+# Skill Creator Schemas
+
+这份参考文档定义 KodaX 版 `skill-creator` 默认使用的评测文件格式。它不是强制协议，但建议优先沿用，方便后续聚合、review 和自动分析。
+
+## `evals/evals.json`
+
+用于保存测试提示集合。
+
+```json
+{
+  "skill_name": "example-skill",
+  "evals": [
+    {
+      "id": 1,
+      "prompt": "User task prompt",
+      "expected_output": "What a good result should achieve",
+      "files": [],
+      "assertions": []
+    }
+  ]
+}
+```
+
+字段说明：
+- `skill_name`: skill 名称。
+- `evals`: 测试用例数组。
+- `id`: 用例唯一标识。
+- `prompt`: 给 agent 的任务文本。
+- `expected_output`: 对预期结果的简短说明。
+- `files`: 需要作为输入提供的文件列表。
+- `assertions`: 可选，后续 grading 用的断言定义。
+
+## `eval_metadata.json`
+
+用于单个 eval 目录，帮助 review 工具识别 prompt 和断言。
+
+```json
+{
+  "eval_id": 1,
+  "eval_name": "handles-empty-input",
+  "prompt": "Implement validation for empty input",
+  "expected_output": "Reject empty input with a clear message",
+  "assertions": [
+    {
+      "text": "rejects empty input with a clear message"
+    }
+  ]
+}
+```
+
+## `grading.json`
+
+由 `grade-evals.js` 生成，用于保存单次运行后的断言判定结果。
+
+```json
+{
+  "summary": {
+    "passed": 2,
+    "failed": 1,
+    "total": 3,
+    "pass_rate": 0.6667
+  },
+  "expectations": [
+    {
+      "text": "rejects empty input with a clear message",
+      "passed": true,
+      "evidence": "Observed in outputs/result.md"
+    }
+  ],
+  "execution_metrics": {
+    "total_tool_calls": 4,
+    "errors_encountered": 0,
+    "output_chars": 5120
+  },
+  "user_notes_summary": {
+    "uncertainties": [],
+    "needs_review": [],
+    "workarounds": []
+  },
+  "overall_summary": "Mostly correct, but edge cases need review.",
+  "timing": {
+    "total_tokens": 84852,
+    "total_duration_seconds": 23.3
+  },
+  "meta": {
+    "generated_at": "2026-03-17T12:00:00.000Z",
+    "eval_id": 1,
+    "eval_name": "handles-empty-input",
+    "config": "with_skill",
+    "run_id": "run-1"
+  }
+}
+```
+
+要求：
+- `expectations` 里的字段名固定为 `text`、`passed`、`evidence`。
+- `pass_rate` 建议是 `0..1` 之间的小数。
+
+## `timing.json`
+
+用于保存一次运行的耗时与 token 信息。
+
+```json
+{
+  "total_tokens": 84852,
+  "duration_ms": 23332,
+  "total_duration_seconds": 23.3
+}
+```
+
+## `benchmark.json`
+
+由 `aggregate-benchmark.js` 生成，用于总览不同配置的表现。
+
+```json
+{
+  "skill_name": "example-skill",
+  "generated_at": "2026-03-17T12:00:00.000Z",
+  "workspace": "/abs/path/to/iteration-1",
+  "configs": {
+    "with_skill": {
+      "pass_rate": { "mean": 0.9, "stddev": 0.1, "min": 0.8, "max": 1.0 },
+      "time_seconds": { "mean": 12.4, "stddev": 1.1, "min": 11.2, "max": 13.5 },
+      "tokens": { "mean": 4200, "stddev": 380, "min": 3900, "max": 4700 }
+    },
+    "without_skill": {
+      "pass_rate": { "mean": 0.6, "stddev": 0.2, "min": 0.4, "max": 0.8 },
+      "time_seconds": { "mean": 9.5, "stddev": 0.7, "min": 8.9, "max": 10.2 },
+      "tokens": { "mean": 3100, "stddev": 240, "min": 2900, "max": 3400 }
+    }
+  },
+  "delta": {
+    "pass_rate": "+0.3000",
+    "time_seconds": "+2.9000",
+    "tokens": "+1100.0000"
+  },
+  "runs": {
+    "with_skill": [],
+    "without_skill": []
+  }
+}
+```
+
+## `analysis.json`
+
+由 `analyze-benchmark.js` 生成，用于总结 benchmark 的稳定收益、方差热点和下一步建议。
+
+```json
+{
+  "skill_name": "example-skill",
+  "generated_at": "2026-03-17T12:15:00.000Z",
+  "workspace": "/abs/path/to/iteration-1",
+  "verdict": "improves",
+  "release_readiness": "needs_iteration",
+  "recommendation": "Keep the skill, but reduce variance before release.",
+  "key_findings": [
+    "with_skill materially improves pass rate"
+  ],
+  "variance_hotspots": [
+    "baseline repeatedly misses billing details"
+  ],
+  "suggested_actions": [
+    "tighten assertions around billing coverage"
+  ],
+  "watchouts": [
+    "token cost increased"
+  ],
+  "supporting_metrics": {
+    "pass_rate_delta": "+0.3000",
+    "time_seconds_delta": "+2.9000",
+    "tokens_delta": "+1100.0000"
+  },
+  "failure_clusters": {}
+}
+```
+
+## `comparison.json`
+
+由 `compare-runs.js` 生成，用于 blind comparison 两个 config 的输出质量。
+
+```json
+{
+  "workspace": "/abs/path/to/iteration-1",
+  "generated_at": "2026-03-17T12:20:00.000Z",
+  "config_a": "with_skill",
+  "config_b": "without_skill",
+  "summary": {
+    "total_pairs": 3,
+    "config_a_wins": 2,
+    "config_b_wins": 0,
+    "ties": 1,
+    "inconclusive": 0
+  },
+  "comparisons": [
+    {
+      "eval_id": 1,
+      "winner_label": "A",
+      "winner_config": "with_skill",
+      "confidence": 0.9,
+      "rationale": "Candidate A is more complete and specific."
+    }
+  ]
+}
+```
+
+## 推荐目录结构
+
+```text
+my-skill-workspace/
+└── iteration-1/
+    ├── eval-0/
+    │   ├── eval_metadata.json
+    │   ├── with_skill/
+    │   │   ├── outputs/
+    │   │   ├── grading.json
+    │   │   └── timing.json
+    │   └── without_skill/
+    │       ├── outputs/
+    │       ├── grading.json
+    │       └── timing.json
+    ├── benchmark.json
+    ├── benchmark.md
+    ├── analysis.json
+    ├── analysis.md
+    ├── comparison.json
+    └── comparison.md
+```

package/dist/builtin/skill-creator/scripts/aggregate-benchmark.d.ts

@@ -0,0 +1,46 @@
+export interface BenchmarkRun {
+  eval_id: string | number;
+  run_id: string;
+  pass_rate: number;
+  passed: number;
+  failed: number;
+  total: number;
+  time_seconds: number;
+  tokens: number;
+  tool_calls: number;
+  errors: number;
+  expectations: Array<Record<string, unknown>>;
+  notes: string[];
+}
+
+export interface StatsSummary {
+  mean: number;
+  stddev: number;
+  min: number;
+  max: number;
+}
+
+export interface BenchmarkDocument {
+  skill_name: string;
+  generated_at: string;
+  workspace: string;
+  configs: Record<string, {
+    pass_rate: StatsSummary;
+    time_seconds: StatsSummary;
+    tokens: StatsSummary;
+  }>;
+  delta: {
+    pass_rate: string;
+    time_seconds: string;
+    tokens: string;
+  };
+  runs: Record<string, BenchmarkRun[]>;
+}
+
+export function loadRunResults(iterationDir: string): Promise<Record<string, BenchmarkRun[]>>;
+export function buildBenchmarkDocument(
+  iterationDir: string,
+  skillName: string,
+  configRuns: Record<string, BenchmarkRun[]>
+): BenchmarkDocument;
+export function renderBenchmarkMarkdown(benchmark: BenchmarkDocument): string;

package/dist/builtin/skill-creator/scripts/aggregate-benchmark.js

@@ -0,0 +1,209 @@
+#!/usr/bin/env node
+
+import { readFile, readdir, stat, writeFile } from 'node:fs/promises';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { calculateStats, formatDelta } from './utils.js';
+
+async function readJson(filePath) {
+  return JSON.parse(await readFile(filePath, 'utf8'));
+}
+
+async function pathExists(filePath) {
+  try {
+    await stat(filePath);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+async function listDirectories(dirPath) {
+  const entries = await readdir(dirPath, { withFileTypes: true }).catch(() => []);
+  return entries
+    .filter((entry) => entry.isDirectory())
+    .map((entry) => path.join(dirPath, entry.name))
+    .sort((left, right) => left.localeCompare(right));
+}
+
+export async function loadRunResults(iterationDir) {
+  const runsRoot = await pathExists(path.join(iterationDir, 'runs'))
+    ? path.join(iterationDir, 'runs')
+    : iterationDir;
+
+  const evalDirs = (await listDirectories(runsRoot))
+    .filter((dirPath) => path.basename(dirPath).startsWith('eval-'));
+
+  const configs = {};
+
+  for (const evalDir of evalDirs) {
+    const metadataPath = path.join(evalDir, 'eval_metadata.json');
+    const metadata = await readJson(metadataPath).catch(() => ({}));
+    const evalId = metadata.eval_id ?? path.basename(evalDir);
+
+    for (const configDir of await listDirectories(evalDir)) {
+      const configName = path.basename(configDir);
+      const runDirs = (await listDirectories(configDir))
+        .filter((dirPath) => path.basename(dirPath).startsWith('run-'));
+
+      if (runDirs.length === 0) {
+        continue;
+      }
+
+      configs[configName] ??= [];
+
+      for (const runDir of runDirs) {
+        const grading = await readJson(path.join(runDir, 'grading.json')).catch(() => null);
+        if (!grading) {
+          continue;
+        }
+
+        const timing = await readJson(path.join(runDir, 'timing.json')).catch(() => ({}));
+
+        configs[configName].push({
+          eval_id: evalId,
+          run_id: path.basename(runDir),
+          pass_rate: grading.summary?.pass_rate ?? 0,
+          passed: grading.summary?.passed ?? 0,
+          failed: grading.summary?.failed ?? 0,
+          total: grading.summary?.total ?? 0,
+          time_seconds: timing.total_duration_seconds ?? grading.timing?.total_duration_seconds ?? 0,
+          tokens: timing.total_tokens ?? grading.execution_metrics?.output_chars ?? 0,
+          tool_calls: grading.execution_metrics?.total_tool_calls ?? 0,
+          errors: grading.execution_metrics?.errors_encountered ?? 0,
+          expectations: Array.isArray(grading.expectations) ? grading.expectations : [],
+          notes: [
+            ...(grading.user_notes_summary?.uncertainties ?? []),
+            ...(grading.user_notes_summary?.needs_review ?? []),
+            ...(grading.user_notes_summary?.workarounds ?? []),
+          ],
+        });
+      }
+    }
+  }
+
+  return configs;
+}
+
+export function summarizeConfigs(configRuns) {
+  const summary = {};
+
+  for (const [configName, runs] of Object.entries(configRuns)) {
+    summary[configName] = {
+      pass_rate: calculateStats(runs.map((run) => Number(run.pass_rate ?? 0))),
+      time_seconds: calculateStats(runs.map((run) => Number(run.time_seconds ?? 0))),
+      tokens: calculateStats(runs.map((run) => Number(run.tokens ?? 0))),
+    };
+  }
+
+  const orderedConfigs = Object.keys(summary);
+  const primary = summary[orderedConfigs[0]] ?? {
+    pass_rate: { mean: 0 },
+    time_seconds: { mean: 0 },
+    tokens: { mean: 0 },
+  };
+  const baseline = summary[orderedConfigs[1]] ?? {
+    pass_rate: { mean: 0 },
+    time_seconds: { mean: 0 },
+    tokens: { mean: 0 },
+  };
+
+  return {
+    configs: summary,
+    delta: {
+      pass_rate: formatDelta(primary.pass_rate.mean - baseline.pass_rate.mean),
+      time_seconds: formatDelta(primary.time_seconds.mean - baseline.time_seconds.mean),
+      tokens: formatDelta(primary.tokens.mean - baseline.tokens.mean),
+    },
+  };
+}
+
+export function buildBenchmarkDocument(iterationDir, skillName, configRuns) {
+  const summary = summarizeConfigs(configRuns);
+
+  return {
+    skill_name: skillName,
+    generated_at: new Date().toISOString(),
+    workspace: path.resolve(iterationDir),
+    configs: summary.configs,
+    delta: summary.delta,
+    runs: configRuns,
+  };
+}
+
+export function renderBenchmarkMarkdown(benchmark) {
+  const lines = [
+    `# Benchmark: ${benchmark.skill_name}`,
+    '',
+    `Generated: ${benchmark.generated_at}`,
+    '',
+    '| Config | Pass Rate | Time (s) | Tokens |',
+    '| --- | --- | --- | --- |',
+  ];
+
+  for (const [configName, metrics] of Object.entries(benchmark.configs)) {
+    lines.push(
+      `| ${configName} | ${metrics.pass_rate.mean} ± ${metrics.pass_rate.stddev} | ${metrics.time_seconds.mean} ± ${metrics.time_seconds.stddev} | ${metrics.tokens.mean} ± ${metrics.tokens.stddev} |`
+    );
+  }
+
+  lines.push('');
+  lines.push('## Delta');
+  lines.push('');
+  lines.push(`- Pass rate: ${benchmark.delta.pass_rate}`);
+  lines.push(`- Time (s): ${benchmark.delta.time_seconds}`);
+  lines.push(`- Tokens: ${benchmark.delta.tokens}`);
+
+  return lines.join('\n');
+}
+
+function parseArgs(argv) {
+  const args = {
+    iterationDir: argv[2],
+    skillName: 'unknown-skill',
+  };
+
+  for (let index = 3; index < argv.length; index += 1) {
+    const token = argv[index];
+    if (token === '--skill-name' && argv[index + 1]) {
+      args.skillName = argv[index + 1];
+      index += 1;
+    }
+  }
+
+  return args;
+}
+
+async function main() {
+  const { iterationDir, skillName } = parseArgs(process.argv);
+  if (!iterationDir) {
+    console.error('Usage: node scripts/aggregate-benchmark.js <iteration-dir> --skill-name <name>');
+    process.exit(1);
+  }
+
+  const configRuns = await loadRunResults(iterationDir);
+  if (Object.keys(configRuns).length === 0) {
+    console.error(`No benchmark runs found in ${iterationDir}`);
+    process.exit(1);
+  }
+
+  const benchmark = buildBenchmarkDocument(iterationDir, skillName, configRuns);
+  const benchmarkJsonPath = path.join(iterationDir, 'benchmark.json');
+  const benchmarkMdPath = path.join(iterationDir, 'benchmark.md');
+
+  await writeFile(benchmarkJsonPath, JSON.stringify(benchmark, null, 2));
+  await writeFile(benchmarkMdPath, `${renderBenchmarkMarkdown(benchmark)}\n`);
+
+  console.log(`Wrote ${benchmarkJsonPath}`);
+  console.log(`Wrote ${benchmarkMdPath}`);
+}
+
+const isDirectRun = process.argv[1]
+  && fileURLToPath(import.meta.url) === path.resolve(process.argv[1]);
+
+if (isDirectRun) {
+  main().catch((error) => {
+    console.error(error instanceof Error ? error.message : String(error));
+    process.exit(1);
+  });
+}

package/dist/builtin/skill-creator/scripts/analyze-benchmark.d.ts

@@ -0,0 +1,46 @@
+export interface AnalyzeBenchmarkOptions {
+  workspaceDir: string;
+  benchmarkPath?: string;
+  outputPath?: string;
+  markdownPath?: string;
+  skillName?: string;
+  provider?: string;
+  model?: string;
+  reasoningMode?: string;
+  maxIter?: number;
+  cwd?: string;
+}
+
+export interface BenchmarkAnalysis {
+  skill_name: string;
+  generated_at: string;
+  workspace: string;
+  verdict: 'improves' | 'regresses' | 'mixed' | 'inconclusive';
+  release_readiness: 'ready' | 'needs_iteration' | 'needs_manual_review';
+  recommendation: string;
+  key_findings: string[];
+  variance_hotspots: string[];
+  suggested_actions: string[];
+  watchouts: string[];
+  supporting_metrics: {
+    pass_rate_delta: string;
+    time_seconds_delta: string;
+    tokens_delta: string;
+  };
+  failure_clusters: Record<string, unknown>;
+}
+
+export function buildAnalysisPrompt(input: Record<string, unknown>): string;
+
+export function renderAnalysisMarkdown(analysis: Record<string, any>): string;
+
+export function analyzeBenchmark(
+  options: AnalyzeBenchmarkOptions,
+  runner?: (prompt: string, options: Record<string, unknown>) => Promise<string>
+): Promise<{
+  analysis: BenchmarkAnalysis;
+  prompt: string;
+  rawResponse: string;
+  analysisJsonPath: string;
+  analysisMdPath: string;
+}>;