aiox-core 5.0.2 → 5.0.4

package/pro/squads/squad-creator-pro/scripts/model-tier-validator.cjs

@@ -0,0 +1,369 @@
+#!/usr/bin/env node
+/**
+ * Model Tier Validator for Squad Creator Token Economy
+ *
+ * Validates tasks against Opus/Sonnet/Haiku to determine optimal tier.
+ *
+ * Usage:
+ *   node model-tier-validator.cjs validate <task-name>
+ *   node model-tier-validator.cjs list
+ *   node model-tier-validator.cjs report
+ *   node model-tier-validator.cjs update-routing <task-name> <tier> [reason]
+ *
+ * Examples:
+ *   node model-tier-validator.cjs validate pv-axioma-assessment
+ *   node model-tier-validator.cjs list
+ *   node model-tier-validator.cjs report
+ */
+
+const fs = require('fs');
+const path = require('path');
+const yaml = require('js-yaml');
+
+// Paths
+const SQUAD_ROOT = path.join(__dirname, '..');
+const CONFIG_PATH = path.join(SQUAD_ROOT, 'config/scoring-rubric.yaml');
+const TEST_CASES_DIR = path.join(SQUAD_ROOT, 'test-cases');
+const ROUTING_CONFIG = path.join(SQUAD_ROOT, 'config/model-routing.yaml');
+// Note: Results are stored inside test-cases/{task-name}/ alongside test-case.yaml
+
+// Ensure test-cases directory exists
+if (!fs.existsSync(TEST_CASES_DIR)) {
+  fs.mkdirSync(TEST_CASES_DIR, { recursive: true });
+}
+
+// Load scoring rubric
+function loadRubric() {
+  try {
+    const content = fs.readFileSync(CONFIG_PATH, 'utf8');
+    return yaml.load(content);
+  } catch (e) {
+    console.error(`Error loading rubric: ${e.message}`);
+    return null;
+  }
+}
+
+// Load test case
+function loadTestCase(taskName) {
+  // New structure: test-cases/{task-name}/test-case.yaml
+  let testCasePath = path.join(TEST_CASES_DIR, taskName, 'test-case.yaml');
+
+  // Fallback to old structure: test-cases/{task-name}.yaml
+  if (!fs.existsSync(testCasePath)) {
+    testCasePath = path.join(TEST_CASES_DIR, `${taskName}.yaml`);
+  }
+
+  try {
+    const content = fs.readFileSync(testCasePath, 'utf8');
+    return yaml.load(content);
+  } catch (e) {
+    console.error(`Error loading test case: ${e.message}`);
+    return null;
+  }
+}
+
+// Load routing config
+function loadRouting() {
+  try {
+    const content = fs.readFileSync(ROUTING_CONFIG, 'utf8');
+    return yaml.load(content);
+  } catch (e) {
+    console.error(`Error loading routing config: ${e.message}`);
+    return null;
+  }
+}
+
+// Calculate score for an output against baseline
+function calculateScore(output, baseline, testCase, rubric) {
+  const scores = {};
+  let totalScore = 0;
+
+  // For each dimension in rubric
+  for (const [dimName, dimConfig] of Object.entries(rubric.dimensions)) {
+    // This would need actual comparison logic
+    // For now, placeholder scoring
+    const dimScore = evaluateDimension(dimName, output, baseline, testCase);
+    scores[dimName] = {
+      score: dimScore,
+      weight: dimConfig.weight,
+      weighted: dimScore * dimConfig.weight
+    };
+    totalScore += dimScore * dimConfig.weight;
+  }
+
+  return {
+    total: totalScore,
+    dimensions: scores,
+    percentage: (totalScore / 10) * 100
+  };
+}
+
+// Evaluate a single dimension
+// Uses pre-recorded scores from test case if available, otherwise returns null for manual scoring
+function evaluateDimension(dimName, output, baseline, testCase) {
+  // If test case has pre-recorded dimension scores, use them
+  if (testCase.results) {
+    for (const model of ['haiku', 'sonnet', 'opus']) {
+      if (testCase.results[model] && testCase.results[model].dimensions) {
+        const dimResult = testCase.results[model].dimensions[dimName];
+        if (dimResult && typeof dimResult.score === 'number') {
+          return dimResult.score;
+        }
+      }
+    }
+  }
+
+  // If overall score exists, distribute proportionally across dimensions
+  if (testCase.results && testCase.results.haiku && testCase.results.haiku.score) {
+    // Use the overall score as approximation for each dimension
+    return testCase.results.haiku.score;
+  }
+
+  // No pre-recorded data - return null to indicate manual scoring needed
+  return null;
+}
+
+// Determine tier recommendation
+function recommendTier(results, rubric) {
+  const { haiku, sonnet, opus } = results;
+
+  if (haiku && haiku.percentage >= rubric.thresholds.haiku * 100) {
+    return {
+      tier: 'haiku',
+      reason: `Haiku score (${haiku.percentage.toFixed(1)}%) >= threshold (${rubric.thresholds.haiku * 100}%)`,
+      savings: '93%'
+    };
+  }
+
+  if (sonnet && sonnet.percentage >= rubric.thresholds.sonnet * 100) {
+    return {
+      tier: 'sonnet',
+      reason: `Sonnet score (${sonnet.percentage.toFixed(1)}%) >= threshold (${rubric.thresholds.sonnet * 100}%)`,
+      savings: '80%'
+    };
+  }
+
+  return {
+    tier: 'opus',
+    reason: 'Neither Haiku nor Sonnet met quality threshold',
+    savings: '0%'
+  };
+}
+
+// Generate comparison report
+function generateReport(taskName, results, recommendation) {
+  const report = {
+    task: taskName,
+    timestamp: new Date().toISOString(),
+    results: {
+      haiku: results.haiku ? {
+        score: results.haiku.total.toFixed(2),
+        percentage: results.haiku.percentage.toFixed(1) + '%',
+        qualified: results.haiku.percentage >= 90
+      } : null,
+      sonnet: results.sonnet ? {
+        score: results.sonnet.total.toFixed(2),
+        percentage: results.sonnet.percentage.toFixed(1) + '%',
+        qualified: results.sonnet.percentage >= 95
+      } : null,
+      opus: {
+        score: '10.00',
+        percentage: '100%',
+        qualified: true
+      }
+    },
+    recommendation: recommendation,
+    compensation_needed: recommendation.tier !== 'haiku' && results.haiku && results.haiku.percentage < 90
+  };
+
+  return report;
+}
+
+// Print comparison table
+function printComparisonTable(reports) {
+  console.log('\n' + '='.repeat(80));
+  console.log('MODEL TIER QUALIFICATION REPORT');
+  console.log('='.repeat(80));
+  console.log('');
+  console.log('| Task | Haiku | Sonnet | Opus | Recommended | Savings |');
+  console.log('|------|-------|--------|------|-------------|---------|');
+
+  for (const report of reports) {
+    const haiku = report.results.haiku?.percentage != null ? `${report.results.haiku.percentage}%` : 'N/A';
+    const sonnet = report.results.sonnet?.percentage != null ? `${report.results.sonnet.percentage}%` : 'N/A';
+    const opus = report.results.opus?.percentage != null ? `${report.results.opus.percentage}%` : '100%';
+    const rec = report.recommendation.tier.toUpperCase();
+    const savings = report.recommendation.savings;
+
+    console.log(`| ${report.task.padEnd(20)} | ${haiku.padEnd(5)} | ${sonnet.padEnd(6)} | ${opus.padEnd(4)} | ${rec.padEnd(11)} | ${savings.padEnd(7)} |`);
+  }
+
+  console.log('');
+  console.log('='.repeat(80));
+}
+
+// List all test cases
+function listTestCases() {
+  const entries = fs.readdirSync(TEST_CASES_DIR, { withFileTypes: true });
+  const cases = [];
+
+  for (const entry of entries) {
+    if (entry.isDirectory() && !entry.name.startsWith('_')) {
+      // New structure: test-cases/{task-name}/test-case.yaml
+      const testCasePath = path.join(TEST_CASES_DIR, entry.name, 'test-case.yaml');
+      if (fs.existsSync(testCasePath)) {
+        cases.push(entry.name);
+      }
+    } else if (entry.isFile() && entry.name.endsWith('.yaml') && !entry.name.startsWith('_')) {
+      // Old structure: test-cases/{task-name}.yaml
+      cases.push(entry.name.replace('.yaml', ''));
+    }
+  }
+
+  return cases;
+}
+
+// Update routing config with validated tier
+function updateRoutingConfig(taskName, tier, reason) {
+  const routing = loadRouting();
+  if (!routing || !routing.tasks) return false;
+
+  const taskKey = `${taskName}.md`;
+  if (routing.tasks[taskKey]) {
+    routing.tasks[taskKey].tier = tier;
+    routing.tasks[taskKey].validated = true;
+    routing.tasks[taskKey].validation_reason = reason;
+    routing.tasks[taskKey].validated_date = new Date().toISOString().split('T')[0];
+
+    fs.writeFileSync(ROUTING_CONFIG, yaml.dump(routing, { lineWidth: 120 }));
+    return true;
+  }
+
+  return false;
+}
+
+// CLI handler
+const [,, command, ...args] = process.argv;
+
+switch (command) {
+  case 'validate':
+    const taskName = args[0];
+    if (!taskName) {
+      console.log('Usage: node model-tier-validator.cjs validate <task-name>');
+      process.exit(1);
+    }
+
+    const rubric = loadRubric();
+    const testCase = loadTestCase(taskName);
+
+    if (!rubric || !testCase) {
+      console.error('Failed to load required files');
+      process.exit(1);
+    }
+
+    console.log(`\nValidating: ${taskName}`);
+    console.log('Test case loaded:', testCase.test_case.name);
+    console.log('');
+
+    // Show test case info
+    console.log('Input:', testCase.input.target);
+    console.log('Expected sections:', testCase.expected_output.sections.join(', '));
+    console.log('');
+
+    // Show current results if available
+    if (testCase.results) {
+      console.log('--- Current Results ---');
+      if (testCase.results.haiku && testCase.results.haiku.score) {
+        console.log(`Haiku:  ${testCase.results.haiku.percentage}% - ${testCase.results.haiku.qualified ? '✅ Qualified' : '❌ Not qualified'}`);
+      }
+      if (testCase.results.sonnet && testCase.results.sonnet.score) {
+        console.log(`Sonnet: ${testCase.results.sonnet.percentage}% - ${testCase.results.sonnet.qualified ? '✅ Qualified' : '❌ Not qualified'}`);
+      }
+      console.log(`Opus:   ${testCase.results.opus.percentage}% - ✅ Baseline`);
+      console.log('');
+      console.log(`Final tier: ${testCase.results.final_tier || 'Not determined'}`);
+      console.log(`Recommendation: ${testCase.results.recommendation || 'None'}`);
+    }
+    break;
+
+  case 'list':
+    const tasks = listTestCases();
+    console.log('\nAvailable test cases:');
+    tasks.forEach(t => console.log(`  - ${t}`));
+    console.log(`\nTotal: ${tasks.length} test cases`);
+    break;
+
+  case 'report':
+    const allTasks = listTestCases();
+    const reports = [];
+
+    for (const task of allTasks) {
+      const tc = loadTestCase(task);
+      if (tc && tc.results && tc.results.final_tier) {
+        reports.push({
+          task,
+          results: tc.results,
+          recommendation: {
+            tier: tc.results.final_tier,
+            savings: tc.results.final_tier === 'haiku' ? '93%' :
+                     tc.results.final_tier === 'sonnet' ? '80%' : '0%'
+          }
+        });
+      }
+    }
+
+    if (reports.length === 0) {
+      console.log('\nNo validated test cases yet.');
+      console.log('Run validation first: node model-tier-validator.cjs validate <task-name>');
+    } else {
+      printComparisonTable(reports);
+    }
+    break;
+
+  case 'update-routing':
+    const updateTask = args[0];
+    const updateTier = args[1];
+    const updateReason = args.slice(2).join(' ') || 'Validated via model-tier-validator';
+
+    if (!updateTask || !updateTier) {
+      console.log('Usage: node model-tier-validator.cjs update-routing <task-name> <tier> [reason]');
+      process.exit(1);
+    }
+
+    if (updateRoutingConfig(updateTask, updateTier, updateReason)) {
+      console.log(`✅ Updated ${updateTask}.md to tier: ${updateTier}`);
+    } else {
+      console.log(`❌ Failed to update routing config for ${updateTask}`);
+    }
+    break;
+
+  default:
+    console.log(`
+Model Tier Validator - Squad Creator Token Economy
+
+Commands:
+  validate <task-name>
+    Show validation status for a task
+
+  list
+    List all available test cases
+
+  report
+    Generate comparison report for all validated tasks
+
+  update-routing <task-name> <tier> [reason]
+    Update model-routing.yaml with validated tier
+
+Examples:
+  node model-tier-validator.cjs validate pv-axioma-assessment
+  node model-tier-validator.cjs list
+  node model-tier-validator.cjs report
+  node model-tier-validator.cjs update-routing pv-axioma-assessment haiku "Scoring determinístico"
+
+Note:
+  Actual model comparison requires running tasks through Claude API.
+  This tool manages test cases and tracks results.
+  Use the Skill tool to run actual tests:
+    /pv-axioma-assessment <input>
+`);
+}

package/pro/squads/squad-creator-pro/scripts/model-usage-logger.cjs

@@ -0,0 +1,245 @@
+#!/usr/bin/env node
+/**
+ * Model Usage Logger for Squad Creator Token Economy
+ *
+ * Tracks which model tier is used for each task to prove/disprove
+ * the token optimization system is working.
+ *
+ * Usage:
+ *   node model-usage-logger.cjs log <task> <model> [tokens_in] [tokens_out] [latency_ms]
+ *   node model-usage-logger.cjs report [--days N]
+ *   node model-usage-logger.cjs validate <task>
+ *
+ * Examples:
+ *   node model-usage-logger.cjs log validate-squad.md haiku 1500 800 2300
+ *   node model-usage-logger.cjs report
+ *   node model-usage-logger.cjs validate validate-squad.md
+ */
+
+const fs = require('fs');
+const path = require('path');
+const yaml = require('js-yaml');
+
+// Paths
+const SQUAD_ROOT = path.join(__dirname, '..');
+const METRICS_DIR = path.join(SQUAD_ROOT, '../../outputs/metrics');
+const CONFIG_PATH = path.join(SQUAD_ROOT, 'config/model-routing.yaml');
+const LOG_PATH = path.join(METRICS_DIR, 'model-usage.jsonl');
+
+// Ensure metrics dir exists
+if (!fs.existsSync(METRICS_DIR)) {
+  fs.mkdirSync(METRICS_DIR, { recursive: true });
+}
+
+// Load routing config
+function loadConfig() {
+  try {
+    const content = fs.readFileSync(CONFIG_PATH, 'utf8');
+    return yaml.load(content);
+  } catch (e) {
+    console.error(`Error loading config: ${e.message}`);
+    return null;
+  }
+}
+
+// Get expected tier for a task
+function getExpectedTier(taskName, config) {
+  if (!config || !config.tasks) return 'opus'; // default
+  const task = config.tasks[taskName];
+  return task ? task.tier : 'opus';
+}
+
+// Log a model usage event
+function logUsage(task, model, tokensIn = 0, tokensOut = 0, latencyMs = 0) {
+  const config = loadConfig();
+  const expectedTier = getExpectedTier(task, config);
+  const isCorrect = model === expectedTier;
+
+  const entry = {
+    timestamp: new Date().toISOString(),
+    task,
+    model_used: model,
+    model_expected: expectedTier,
+    correct: isCorrect,
+    tokens_in: parseInt(tokensIn) || 0,
+    tokens_out: parseInt(tokensOut) || 0,
+    latency_ms: parseInt(latencyMs) || 0,
+    cost_actual: calculateCost(model, tokensIn, tokensOut),
+    cost_if_opus: calculateCost('opus', tokensIn, tokensOut),
+    savings: calculateCost('opus', tokensIn, tokensOut) - calculateCost(model, tokensIn, tokensOut)
+  };
+
+  // Append to JSONL log
+  fs.appendFileSync(LOG_PATH, JSON.stringify(entry) + '\n');
+
+  // Print result
+  const status = isCorrect ? '✅' : '❌';
+  console.log(`${status} Logged: ${task} → ${model} (expected: ${expectedTier})`);
+  if (entry.savings > 0) {
+    console.log(`   Savings: $${entry.savings.toFixed(4)} (${((entry.savings / entry.cost_if_opus) * 100).toFixed(1)}%)`);
+  }
+
+  return entry;
+}
+
+// Calculate cost based on model and tokens
+// Prices per 1M tokens - Opus 4.5/4.6 (Feb 2026)
+function calculateCost(model, tokensIn, tokensOut) {
+  const rates = {
+    haiku: { input: 1.00, output: 5.00 },
+    sonnet: { input: 3.00, output: 15.00 },
+    opus: { input: 5.00, output: 25.00 }
+  };
+  const rate = rates[model] || rates.opus;
+  return (tokensIn * rate.input + tokensOut * rate.output) / 1000000;
+}
+
+// Generate usage report
+function generateReport(days = 7) {
+  if (!fs.existsSync(LOG_PATH)) {
+    console.log('No usage data yet. Run some tasks first.');
+    return;
+  }
+
+  const lines = fs.readFileSync(LOG_PATH, 'utf8').trim().split('\n');
+  const cutoff = new Date(Date.now() - days * 24 * 60 * 60 * 1000);
+
+  const entries = lines
+    .map(line => { try { return JSON.parse(line); } catch { return null; } })
+    .filter(e => e && new Date(e.timestamp) > cutoff);
+
+  if (entries.length === 0) {
+    console.log(`No data in the last ${days} days.`);
+    return;
+  }
+
+  // Aggregate stats
+  const stats = {
+    total: entries.length,
+    correct: entries.filter(e => e.correct).length,
+    by_model: {},
+    by_task: {},
+    total_savings: 0,
+    total_cost_actual: 0,
+    total_cost_if_opus: 0
+  };
+
+  entries.forEach(e => {
+    // By model
+    stats.by_model[e.model_used] = (stats.by_model[e.model_used] || 0) + 1;
+
+    // By task
+    if (!stats.by_task[e.task]) {
+      stats.by_task[e.task] = { count: 0, correct: 0, savings: 0 };
+    }
+    stats.by_task[e.task].count++;
+    if (e.correct) stats.by_task[e.task].correct++;
+    stats.by_task[e.task].savings += e.savings || 0;
+
+    // Totals
+    stats.total_savings += e.savings || 0;
+    stats.total_cost_actual += e.cost_actual || 0;
+    stats.total_cost_if_opus += e.cost_if_opus || 0;
+  });
+
+  // Print report
+  console.log('\n' + '='.repeat(60));
+  console.log('MODEL ROUTING USAGE REPORT');
+  console.log('='.repeat(60));
+  console.log(`Period: Last ${days} days`);
+  console.log(`Total executions: ${stats.total}`);
+  console.log(`Correct routing: ${stats.correct}/${stats.total} (${((stats.correct/stats.total)*100).toFixed(1)}%)`);
+
+  console.log('\n--- By Model ---');
+  Object.entries(stats.by_model).forEach(([model, count]) => {
+    const pct = ((count / stats.total) * 100).toFixed(1);
+    console.log(`  ${model}: ${count} (${pct}%)`);
+  });
+
+  console.log('\n--- Cost Analysis ---');
+  console.log(`  Actual cost:    $${stats.total_cost_actual.toFixed(4)}`);
+  console.log(`  If all Opus:    $${stats.total_cost_if_opus.toFixed(4)}`);
+  console.log(`  Total savings:  $${stats.total_savings.toFixed(4)}`);
+  if (stats.total_cost_if_opus > 0) {
+    const savingsPct = ((stats.total_savings / stats.total_cost_if_opus) * 100).toFixed(1);
+    console.log(`  Savings rate:   ${savingsPct}%`);
+  }
+
+  console.log('\n--- Top Tasks (by execution count) ---');
+  const sortedTasks = Object.entries(stats.by_task)
+    .sort((a, b) => b[1].count - a[1].count)
+    .slice(0, 10);
+
+  sortedTasks.forEach(([task, data]) => {
+    const correctPct = ((data.correct / data.count) * 100).toFixed(0);
+    console.log(`  ${task}: ${data.count}x (${correctPct}% correct, $${data.savings.toFixed(4)} saved)`);
+  });
+
+  console.log('\n' + '='.repeat(60));
+}
+
+// Validate expected tier for a task
+function validateTask(taskName) {
+  const config = loadConfig();
+  const expected = getExpectedTier(taskName, config);
+  const taskConfig = config?.tasks?.[taskName];
+
+  console.log('\n--- Task Validation ---');
+  console.log(`Task: ${taskName}`);
+  console.log(`Expected tier: ${expected}`);
+
+  if (taskConfig) {
+    console.log(`Confidence: ${taskConfig.confidence || 'unknown'}`);
+    console.log(`Reason: ${taskConfig.reason || 'not specified'}`);
+  } else {
+    console.log('⚠️  Task not found in config - will use opus (default)');
+  }
+
+  return expected;
+}
+
+// CLI handler
+const [,, command, ...args] = process.argv;
+
+switch (command) {
+  case 'log':
+    if (args.length < 2) {
+      console.log('Usage: node model-usage-logger.cjs log <task> <model> [tokens_in] [tokens_out] [latency_ms]');
+      process.exit(1);
+    }
+    logUsage(...args);
+    break;
+
+  case 'report':
+    const days = args[0] === '--days' ? parseInt(args[1]) || 7 : 7;
+    generateReport(days);
+    break;
+
+  case 'validate':
+    if (!args[0]) {
+      console.log('Usage: node model-usage-logger.cjs validate <task>');
+      process.exit(1);
+    }
+    validateTask(args[0]);
+    break;
+
+  default:
+    console.log(`
+Model Usage Logger - Squad Creator Token Economy
+
+Commands:
+  log <task> <model> [tokens_in] [tokens_out] [latency_ms]
+    Log a model usage event
+
+  report [--days N]
+    Generate usage report (default: last 7 days)
+
+  validate <task>
+    Check expected tier for a task
+
+Examples:
+  node model-usage-logger.cjs log validate-squad.md haiku 1500 800 2300
+  node model-usage-logger.cjs report --days 30
+  node model-usage-logger.cjs validate qa-after-creation.md
+`);
+}