@kodax-ai/kodax-cli 0.7.38

package/dist/builtin/skill-creator/scripts/run-loop.js

@@ -0,0 +1,243 @@
+#!/usr/bin/env node
+
+import { readFile, writeFile } from 'node:fs/promises';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { loadSkill, pathExists, writeSkill } from './utils.js';
+import { improveDescription } from './improve-description.js';
+import { runTriggerEval } from './run-trigger-eval.js';
+
+export function splitEvalSet(evals, holdout = 0.25, seed = 42) {
+  if (holdout <= 0 || evals.length < 4) {
+    return { train: [...evals], test: [] };
+  }
+
+  let random = seed;
+  const nextRandom = () => {
+    random = (random * 1664525 + 1013904223) % 4294967296;
+    return random / 4294967296;
+  };
+
+  const shuffle = (items) => {
+    const result = [...items];
+    for (let index = result.length - 1; index > 0; index -= 1) {
+      const swapIndex = Math.floor(nextRandom() * (index + 1));
+      [result[index], result[swapIndex]] = [result[swapIndex], result[index]];
+    }
+    return result;
+  };
+
+  const positives = evals.filter((item) => item.should_trigger === true);
+  const negatives = evals.filter((item) => item.should_trigger !== true);
+  const shuffledPositives = shuffle(positives);
+  const shuffledNegatives = shuffle(negatives);
+
+  const pickCount = (items) => {
+    if (items.length <= 1) {
+      return 0;
+    }
+    return Math.max(1, Math.floor(items.length * holdout));
+  };
+
+  const posCount = pickCount(shuffledPositives);
+  const negCount = pickCount(shuffledNegatives);
+
+  const test = [
+    ...shuffledPositives.slice(0, posCount),
+    ...shuffledNegatives.slice(0, negCount),
+  ];
+  const train = [
+    ...shuffledPositives.slice(posCount),
+    ...shuffledNegatives.slice(negCount),
+  ];
+
+  return { train, test };
+}
+
+async function writeTempEvalFile(workspaceDir, name, skillName, evals) {
+  const filePath = path.join(workspaceDir, `${name}.json`);
+  await writeFile(filePath, JSON.stringify({ skill_name: skillName, evals }, null, 2), 'utf8');
+  return filePath;
+}
+
+export async function runDescriptionLoop(
+  options,
+  dependencies = {
+    runTriggerEvalFn: runTriggerEval,
+    improveDescriptionFn: improveDescription,
+  }
+) {
+  const resolvedDependencies = {
+    runTriggerEvalFn: runTriggerEval,
+    improveDescriptionFn: improveDescription,
+    ...dependencies,
+  };
+  const skill = await loadSkill(options.skillPath);
+  const evalFile = JSON.parse(await readFile(options.evalsPath, 'utf8'));
+  const allEvals = Array.isArray(evalFile.evals) ? evalFile.evals : [];
+  const { train, test } = splitEvalSet(allEvals, options.holdout, options.seed);
+  const historyPath = path.join(options.workspaceDir, 'description-history.json');
+
+  let currentDescription = String(skill.frontmatter.description ?? '');
+  const history = [];
+  let best = null;
+
+  for (let iteration = 1; iteration <= options.maxIterations; iteration += 1) {
+    const trainFile = await writeTempEvalFile(options.workspaceDir, `train-iteration-${iteration}`, skill.frontmatter.name, train);
+    const trainResults = await resolvedDependencies.runTriggerEvalFn({
+      ...options,
+      evalsPath: trainFile,
+      descriptionOverride: currentDescription,
+    });
+
+    let testResults = null;
+    if (test.length > 0) {
+      const testFile = await writeTempEvalFile(options.workspaceDir, `test-iteration-${iteration}`, skill.frontmatter.name, test);
+      testResults = await resolvedDependencies.runTriggerEvalFn({
+        ...options,
+        evalsPath: testFile,
+        descriptionOverride: currentDescription,
+      });
+    }
+
+    const score = `${trainResults.summary.passed}/${trainResults.summary.total}`;
+    const record = {
+      iteration,
+      description: currentDescription,
+      score,
+      train: trainResults.summary,
+      test: testResults?.summary ?? null,
+      train_results: trainResults.results,
+      test_results: testResults?.results ?? [],
+    };
+    history.push(record);
+
+    const currentComparable = testResults?.summary.passed ?? trainResults.summary.passed;
+    const bestComparable = best?.comparable ?? -1;
+    if (currentComparable > bestComparable) {
+      best = {
+        comparable: currentComparable,
+        record,
+      };
+    }
+
+    if (trainResults.summary.failed === 0 || iteration === options.maxIterations) {
+      break;
+    }
+
+    const improveInputPath = path.join(options.workspaceDir, `iteration-${iteration}-train-results.json`);
+    await writeFile(improveInputPath, JSON.stringify(trainResults, null, 2), 'utf8');
+    if (!await pathExists(historyPath)) {
+      await writeFile(historyPath, JSON.stringify({ history }, null, 2), 'utf8');
+    }
+    const improved = await resolvedDependencies.improveDescriptionFn({
+      ...options,
+      evalResultsPath: improveInputPath,
+      historyPath,
+    });
+    currentDescription = improved.description;
+    await writeFile(
+      historyPath,
+      JSON.stringify({ history }, null, 2),
+      'utf8'
+    );
+  }
+
+  if (options.writeBest) {
+    const finalDescription = best?.record.description ?? currentDescription;
+    skill.frontmatter.description = finalDescription;
+    await writeSkill(options.skillPath, skill.frontmatter, skill.body);
+  }
+
+  return {
+    skill_name: skill.frontmatter.name,
+    original_description: skill.frontmatter.description,
+    final_description: currentDescription,
+    best_description: best?.record.description ?? currentDescription,
+    history,
+    train_size: train.length,
+    test_size: test.length,
+  };
+}
+
+function parseArgs(argv) {
+  const args = {
+    skillPath: '',
+    evalsPath: '',
+    workspaceDir: '',
+    provider: 'anthropic',
+    model: undefined,
+    maxIterations: 3,
+    runsPerQuery: 1,
+    triggerThreshold: 0.5,
+    holdout: 0.25,
+    seed: 42,
+    maxIter: 18,
+    reasoningMode: 'off',
+    output: undefined,
+    writeBest: false,
+  };
+
+  for (let index = 2; index < argv.length; index += 1) {
+    const token = argv[index];
+    if (token === '--skill-path' && argv[index + 1]) {
+      args.skillPath = argv[++index];
+    } else if (token === '--evals' && argv[index + 1]) {
+      args.evalsPath = argv[++index];
+    } else if (token === '--workspace' && argv[index + 1]) {
+      args.workspaceDir = argv[++index];
+    } else if (token === '--provider' && argv[index + 1]) {
+      args.provider = argv[++index];
+    } else if (token === '--model' && argv[index + 1]) {
+      args.model = argv[++index];
+    } else if (token === '--max-iterations' && argv[index + 1]) {
+      args.maxIterations = Number(argv[++index]);
+    } else if (token === '--runs-per-query' && argv[index + 1]) {
+      args.runsPerQuery = Number(argv[++index]);
+    } else if (token === '--trigger-threshold' && argv[index + 1]) {
+      args.triggerThreshold = Number(argv[++index]);
+    } else if (token === '--holdout' && argv[index + 1]) {
+      args.holdout = Number(argv[++index]);
+    } else if (token === '--seed' && argv[index + 1]) {
+      args.seed = Number(argv[++index]);
+    } else if (token === '--max-iter' && argv[index + 1]) {
+      args.maxIter = Number(argv[++index]);
+    } else if (token === '--reasoning' && argv[index + 1]) {
+      args.reasoningMode = argv[++index];
+    } else if (token === '--output' && argv[index + 1]) {
+      args.output = argv[++index];
+    } else if (token === '--write-best') {
+      args.writeBest = true;
+    }
+  }
+
+  return args;
+}
+
+async function main() {
+  const args = parseArgs(process.argv);
+  if (!args.skillPath || !args.evalsPath || !args.workspaceDir) {
+    console.error('Usage: node scripts/run-loop.js --skill-path <dir> --evals <evals.json> --workspace <dir> [--max-iterations 3]');
+    process.exit(1);
+  }
+
+  const report = await runDescriptionLoop(args);
+  const outputText = `${JSON.stringify(report, null, 2)}\n`;
+
+  if (args.output) {
+    await writeFile(args.output, outputText, 'utf8');
+    console.log(`Wrote ${path.resolve(args.output)}`);
+  } else {
+    process.stdout.write(outputText);
+  }
+}
+
+const isDirectRun = process.argv[1]
+  && fileURLToPath(import.meta.url) === path.resolve(process.argv[1]);
+
+if (isDirectRun) {
+  main().catch((error) => {
+    console.error(error instanceof Error ? error.message : String(error));
+    process.exit(1);
+  });
+}

package/dist/builtin/skill-creator/scripts/run-trigger-eval.d.ts

@@ -0,0 +1,58 @@
+export interface TriggerEvalAttempt {
+  trigger: boolean;
+  reason: string;
+}
+
+export interface TriggerEvalItemResult {
+  query: string;
+  should_trigger: boolean;
+  triggers: number;
+  runs: number;
+  trigger_rate: number;
+  predicted_trigger: boolean;
+  pass: boolean;
+  attempts: TriggerEvalAttempt[];
+}
+
+export interface TriggerEvalSummary {
+  passed: number;
+  failed: number;
+  total: number;
+  pass_rate: number;
+  precision: number;
+  recall: number;
+}
+
+export interface TriggerEvalReport {
+  skill_name: string;
+  description: string;
+  results: TriggerEvalItemResult[];
+  summary: TriggerEvalSummary;
+  meta: {
+    provider?: string;
+    model?: string | null;
+    runs_per_query: number;
+    trigger_threshold: number;
+    note: string;
+  };
+}
+
+export interface TriggerEvalOptions {
+  skillPath: string;
+  evalsPath: string;
+  provider?: string;
+  model?: string;
+  output?: string;
+  runsPerQuery?: number;
+  triggerThreshold?: number;
+  maxIter?: number;
+  reasoningMode?: string;
+  descriptionOverride?: string;
+}
+
+export function parseTriggerDecision(text: string): TriggerEvalAttempt;
+export function summarizeTriggerResults(results: TriggerEvalItemResult[]): TriggerEvalSummary;
+export function runTriggerEval(
+  options: TriggerEvalOptions,
+  runner?: (prompt: string, options: TriggerEvalOptions) => Promise<string>
+): Promise<TriggerEvalReport>;

package/dist/builtin/skill-creator/scripts/run-trigger-eval.js

@@ -0,0 +1,225 @@
+#!/usr/bin/env node
+
+import { readFile, writeFile } from 'node:fs/promises';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import {
+  extractJsonObject,
+  loadKodaXSDK,
+  loadSkill,
+} from './utils.js';
+
+function buildTriggerEvalPrompt(skillName, description, query) {
+  return [
+    'You are evaluating whether a KodaX skill description should trigger for a user request.',
+    'Decide whether the skill should be used based only on the skill name, description, and query.',
+    'Return JSON only with this exact shape:',
+    '{"trigger": true, "reason": "short explanation"}',
+    '',
+    `Skill name: ${skillName}`,
+    `Skill description: ${description}`,
+    `User query: ${query}`,
+  ].join('\n');
+}
+
+export function parseTriggerDecision(text) {
+  const parsed = extractJsonObject(text);
+  if (!parsed || typeof parsed !== 'object') {
+    throw new Error('Could not parse trigger decision JSON');
+  }
+
+  return {
+    trigger: parsed.trigger === true,
+    reason: typeof parsed.reason === 'string' ? parsed.reason.trim() : '',
+  };
+}
+
+async function defaultRunPrompt(prompt, options) {
+  const { runKodaX } = await loadKodaXSDK();
+  const result = await runKodaX(
+    {
+      provider: options.provider,
+      maxIter: options.maxIter ?? 12,
+      model: options.model,
+      reasoningMode: options.reasoningMode ?? 'off',
+      thinking: options.reasoningMode ? options.reasoningMode !== 'off' : false,
+    },
+    prompt
+  );
+
+  return result.lastText;
+}
+
+export function summarizeTriggerResults(results) {
+  const positives = results.filter((result) => result.should_trigger);
+  const negatives = results.filter((result) => !result.should_trigger);
+  const predictedPositives = results.filter((result) => result.predicted_trigger);
+  const truePositives = positives.filter((result) => result.predicted_trigger).length;
+  const falsePositives = negatives.filter((result) => result.predicted_trigger).length;
+
+  const summary = {
+    passed: 0,
+    failed: 0,
+    total: results.length,
+    pass_rate: 0,
+    precision: 0,
+    recall: 0,
+  };
+
+  for (const result of results) {
+    if (result.pass) {
+      summary.passed += 1;
+    } else {
+      summary.failed += 1;
+    }
+  }
+
+  summary.pass_rate = summary.total === 0 ? 0 : summary.passed / summary.total;
+  summary.precision = predictedPositives.length === 0
+    ? 0
+    : truePositives / predictedPositives.length;
+  summary.recall = positives.length === 0 ? 0 : truePositives / positives.length;
+
+  return summary;
+}
+
+export async function runTriggerEval(options, runner = defaultRunPrompt) {
+  const skill = await loadSkill(options.skillPath);
+  const currentDescription = options.descriptionOverride
+    ?? String(skill.frontmatter.description ?? '').trim();
+  const evalFile = JSON.parse(await readFile(options.evalsPath, 'utf8'));
+  const evals = Array.isArray(evalFile.evals) ? evalFile.evals : [];
+  const runsPerQuery = Number.isFinite(options.runsPerQuery) && options.runsPerQuery > 0
+    ? Math.floor(options.runsPerQuery)
+    : 1;
+  const threshold = Number.isFinite(options.triggerThreshold)
+    ? Math.min(Math.max(options.triggerThreshold, 0), 1)
+    : 0.5;
+  const results = [];
+
+  for (const item of evals) {
+    const query = String(item.query ?? item.prompt ?? '').trim();
+    if (!query) {
+      continue;
+    }
+
+    const shouldTrigger = item.should_trigger === true;
+    let triggers = 0;
+    const attempts = [];
+
+    for (let runIndex = 0; runIndex < runsPerQuery; runIndex += 1) {
+      const response = await runner(
+        buildTriggerEvalPrompt(skill.frontmatter.name, currentDescription, query),
+        options
+      );
+      const parsed = parseTriggerDecision(response);
+      if (parsed.trigger) {
+        triggers += 1;
+      }
+      attempts.push(parsed);
+    }
+
+    const triggerRate = triggers / runsPerQuery;
+    const predictedTrigger = triggerRate >= threshold;
+    const pass = shouldTrigger
+      ? predictedTrigger
+      : !predictedTrigger;
+
+    results.push({
+      query,
+      should_trigger: shouldTrigger,
+      triggers,
+      runs: runsPerQuery,
+      trigger_rate: triggerRate,
+      predicted_trigger: predictedTrigger,
+      pass,
+      attempts,
+    });
+  }
+
+  const summary = summarizeTriggerResults(results);
+
+  return {
+    skill_name: skill.frontmatter.name,
+    description: currentDescription,
+    results,
+    summary,
+    meta: {
+      provider: options.provider,
+      model: options.model ?? null,
+      runs_per_query: runsPerQuery,
+      trigger_threshold: threshold,
+      note: 'This is a KodaX-native description eval, not a Claude Code tool-trace replay.',
+    },
+  };
+}
+
+function parseArgs(argv) {
+  const args = {
+    skillPath: '',
+    evalsPath: '',
+    provider: 'anthropic',
+    model: undefined,
+    output: undefined,
+    runsPerQuery: 1,
+    triggerThreshold: 0.5,
+    maxIter: 12,
+    reasoningMode: 'off',
+    descriptionOverride: undefined,
+  };
+
+  for (let index = 2; index < argv.length; index += 1) {
+    const token = argv[index];
+    if (token === '--skill-path' && argv[index + 1]) {
+      args.skillPath = argv[++index];
+    } else if (token === '--evals' && argv[index + 1]) {
+      args.evalsPath = argv[++index];
+    } else if (token === '--provider' && argv[index + 1]) {
+      args.provider = argv[++index];
+    } else if (token === '--model' && argv[index + 1]) {
+      args.model = argv[++index];
+    } else if (token === '--output' && argv[index + 1]) {
+      args.output = argv[++index];
+    } else if (token === '--runs-per-query' && argv[index + 1]) {
+      args.runsPerQuery = Number(argv[++index]);
+    } else if (token === '--trigger-threshold' && argv[index + 1]) {
+      args.triggerThreshold = Number(argv[++index]);
+    } else if (token === '--max-iter' && argv[index + 1]) {
+      args.maxIter = Number(argv[++index]);
+    } else if (token === '--reasoning' && argv[index + 1]) {
+      args.reasoningMode = argv[++index];
+    } else if (token === '--description' && argv[index + 1]) {
+      args.descriptionOverride = argv[++index];
+    }
+  }
+
+  return args;
+}
+
+async function main() {
+  const args = parseArgs(process.argv);
+  if (!args.skillPath || !args.evalsPath) {
+    console.error('Usage: node scripts/run-trigger-eval.js --skill-path <dir> --evals <evals.json> [--provider anthropic] [--output results.json]');
+    process.exit(1);
+  }
+
+  const results = await runTriggerEval(args);
+  const outputText = `${JSON.stringify(results, null, 2)}\n`;
+
+  if (args.output) {
+    await writeFile(args.output, outputText, 'utf8');
+    console.log(`Wrote ${path.resolve(args.output)}`);
+  } else {
+    process.stdout.write(outputText);
+  }
+}
+
+const isDirectRun = process.argv[1]
+  && fileURLToPath(import.meta.url) === path.resolve(process.argv[1]);
+
+if (isDirectRun) {
+  main().catch((error) => {
+    console.error(error instanceof Error ? error.message : String(error));
+    process.exit(1);
+  });
+}