@hone-ai/cli 1.4.0 → 1.5.0

package/hone-cli.js

@@ -1299,6 +1299,138 @@ program
     console.log('Verification complete.');
   });
 
+// ── USAGE command (#251) ─────────────────────────────────────────────────────
+program
+  .command('usage')
+  .description('Show current month token usage and budget status')
+  .option('--format <fmt>', 'Output format: pretty | json', 'pretty')
+  .action(async (opts) => {
+    const config = getConfig();
+    const client = api(config);
+
+    try {
+      const { data } = await client.get('/usage/me');
+
+      if (opts.format === 'json') {
+        console.log(JSON.stringify(data, null, 2));
+        return;
+      }
+
+      console.log('');
+      console.log('Hone AI — Token Usage');
+      console.log('================================');
+      console.log(`Org:           ${data.org}`);
+      console.log(`Month:         ${data.current_month}`);
+      console.log(`Tokens used:   ${data.used_tokens.toLocaleString()}`);
+      console.log(`Cost (est):    $${data.used_cost_usd.toFixed(2)}`);
+
+      if (data.monthly_budget != null) {
+        console.log(`Budget:        ${data.monthly_budget.toLocaleString()} tokens`);
+        console.log(`Used:          ${data.budget_pct}%`);
+        console.log(`Remaining:     ${data.remaining.toLocaleString()}`);
+        if (data.exceeded) {
+          console.log(`Status:        EXCEEDED — resets ${data.resetsAt.split('T')[0]}`);
+        } else if (data.budget_pct >= data.budget_alert_pct) {
+          console.log(`Status:        WARNING — approaching budget (${data.budget_pct}% of ${data.budget_alert_pct}% alert threshold)`);
+        } else {
+          console.log(`Status:        OK`);
+        }
+      } else {
+        console.log(`Budget:        unlimited`);
+      }
+
+      if (data.by_job && data.by_job.length > 0) {
+        console.log('');
+        console.log('Recent derive jobs:');
+        for (const j of data.by_job.slice(0, 10)) {
+          console.log(`  ${j.date} | ${j.job_id} | ${j.tokens.toLocaleString()} tokens | $${Number(j.cost_usd).toFixed(2)}`);
+        }
+      }
+
+      console.log('');
+    } catch (e) {
+      if (e.response?.status === 401) {
+        console.error('Not authenticated. Run: hone init');
+      } else {
+        console.error(`Failed to fetch usage: ${e.message}`);
+      }
+      process.exit(1);
+    }
+  });
+
+// ── ADMIN-USAGE command ──────────────────────────────────────────────────────
+program
+  .command('admin-usage')
+  .description('Admin dashboard: cross-org token usage, budgets, alerts, trends')
+  .option('--format <fmt>', 'Output format: pretty | json', 'pretty')
+  .action(async (opts) => {
+    const rc = readRc();
+    const adminKey = process.env.HONE_ADMIN_KEY || rc.admin_key;
+    const apiUrl = process.env.HONE_API || rc.api || 'https://api.hone.ai';
+
+    if (!adminKey) {
+      console.error('Error: Admin key not found.');
+      console.error('Set HONE_ADMIN_KEY env var, or add "admin_key" to ~/.honerc');
+      process.exit(1);
+    }
+
+    try {
+      const { data } = await axios.get(`${apiUrl}/admin/usage`, {
+        headers: { 'x-admin-key': adminKey, 'User-Agent': `@hone-ai/cli/${pkg.version}` },
+        timeout: 15000,
+      });
+
+      if (opts.format === 'json') {
+        console.log(JSON.stringify(data, null, 2));
+        return;
+      }
+
+      console.log('');
+      console.log('Hone AI — Admin Dashboard');
+      console.log('================================');
+      console.log(`Month:         ${data.current_month}`);
+      console.log(`Total orgs:    ${data.platform_totals.total_orgs} (${data.platform_totals.active_orgs} active)`);
+      console.log(`Total tokens:  ${data.platform_totals.total_tokens.toLocaleString()}`);
+      console.log(`Total cost:    $${data.platform_totals.total_cost_usd.toFixed(2)}`);
+      console.log(`Total calls:   ${data.platform_totals.total_calls}`);
+
+      if (data.alerts.length > 0) {
+        console.log('');
+        console.log('Alerts:');
+        for (const a of data.alerts) {
+          const icon = a.level === 'critical' ? '!!' : a.level === 'warning' ? ' !' : ' i';
+          console.log(`  [${icon}] ${a.org}: ${a.message}`);
+        }
+      }
+
+      if (data.orgs.length > 0) {
+        console.log('');
+        console.log('Per-org usage:');
+        console.log('  Org                  Tier        Tokens         Cost    Budget%  Trend   Fails');
+        console.log('  ---                  ----        ------         ----    -------  -----   -----');
+        for (const o of data.orgs) {
+          const name = o.org.padEnd(20).slice(0, 20);
+          const tier = (o.tier || '').padEnd(10).slice(0, 10);
+          const tokens = String(o.total_tokens.toLocaleString()).padStart(12);
+          const cost = ('$' + o.total_cost_usd.toFixed(2)).padStart(8);
+          const pct = o.monthly_budget != null ? (o.budget_pct + '%').padStart(8) : '     n/a';
+          const trend = o.trend_pct != null ? ((o.trend_pct >= 0 ? '+' : '') + o.trend_pct + '%').padStart(6) : '   n/a';
+          const fails = String(o.failed_jobs).padStart(5);
+          console.log(`  ${name} ${tier} ${tokens} ${cost} ${pct} ${trend} ${fails}`);
+        }
+      }
+
+      console.log('');
+    } catch (e) {
+      if (e.response?.status === 401) {
+        console.error('Invalid admin key. Check HONE_ADMIN_KEY or ~/.honerc admin_key.');
+      } else {
+        console.error(`Failed to fetch admin dashboard: ${e.message}`);
+      }
+      process.exit(1);
+    }
+  });
+
 // ── SYNC command ──────────────────────────────────────────────────────────────
 program
   .command('sync')
@@ -3997,6 +4129,54 @@ program
     }, null, 2));
   });
 
+// ── HC-019d: Agent Eval Runner ────────────────────────────────────────────────
+program
+  .command('eval')
+  .description('Run eval scenarios against agent prompts (deterministic, zero LLM tokens)')
+  .option('--agent <name>', 'Run evals for a specific agent only')
+  .option('--tag <tag>', 'Filter scenarios by tag (e.g., smoke, regression)')
+  .option('--scenario <id>', 'Run a single scenario by ID')
+  .option('--format <fmt>', 'Output format: pretty | json', 'pretty')
+  .option('--evals-dir <path>', 'Override eval scenarios directory')
+  .option('--fail-fast', 'Stop on first failure')
+  .action(async (opts) => {
+    const path = require('path');
+    const fs = require('fs');
+    const yaml = require('js-yaml');
+    const { loadScenarios, runAllScenarios, formatResults } = require('./lib/eval-runner');
+
+    const evalDir = opts.evalsDir || path.resolve(__dirname, '..', 'evals');
+    if (!fs.existsSync(evalDir)) {
+      console.error(`Eval directory not found: ${evalDir}`);
+      process.exit(1);
+    }
+
+    // Load agent prompts from seed-agent-prompts.js
+    const seedPath = path.resolve(__dirname, '..', 'scripts', 'seed-agent-prompts.js');
+    const { AGENT_PROMPTS } = require(seedPath);
+
+    const scenarios = loadScenarios({
+      evalDir,
+      agent: opts.agent,
+      tag: opts.tag,
+      scenarioId: opts.scenario,
+      readFile: (p) => fs.readFileSync(p, 'utf8'),
+      listDir: (p) => fs.readdirSync(p),
+      isDir: (p) => fs.statSync(p).isDirectory(),
+      parseYaml: (text) => yaml.load(text),
+    });
+
+    if (scenarios.length === 0) {
+      console.log('No eval scenarios found matching filters.');
+      process.exit(0);
+    }
+
+    const results = runAllScenarios(scenarios, AGENT_PROMPTS, { failFast: opts.failFast });
+    console.log(formatResults(results, opts.format));
+
+    process.exit(results.failed + results.errors > 0 ? 1 : 0);
+  });
+
 // ── CLI setup ─────────────────────────────────────────────────────────────────
 program
   .name('hone')

package/lib/eval-graders.js

@@ -0,0 +1,99 @@
+'use strict';
+/**
+ * eval-graders.js — HC-019d deterministic grading checks for agent eval scenarios.
+ *
+ * Each grader is a pure function: (text, config) => { passed, detail }
+ * Zero LLM tokens — string/regex/structural checks only.
+ */
+
+function contains(text, { value, case_insensitive = false }) {
+  const haystack = case_insensitive ? text.toLowerCase() : text;
+  const needle = case_insensitive ? value.toLowerCase() : value;
+  const found = haystack.includes(needle);
+  return { passed: found, detail: found ? `found "${value}"` : `"${value}" NOT FOUND` };
+}
+
+function notContains(text, { value, case_insensitive = false }) {
+  const haystack = case_insensitive ? text.toLowerCase() : text;
+  const needle = case_insensitive ? value.toLowerCase() : value;
+  const found = haystack.includes(needle);
+  return { passed: !found, detail: found ? `"${value}" FOUND (should be absent)` : `"${value}" correctly absent` };
+}
+
+function regex(text, { pattern, flags = '' }) {
+  try {
+    const re = new RegExp(pattern, flags);
+    const match = re.test(text);
+    return { passed: match, detail: match ? `matched /${pattern}/` : `/${pattern}/ did NOT match` };
+  } catch (e) {
+    return { passed: false, detail: `invalid regex /${pattern}/: ${e.message}` };
+  }
+}
+
+function sectionExists(text, { heading }) {
+  const re = new RegExp(`^#{1,4}\\s+${escapeRegex(heading)}`, 'mi');
+  const found = re.test(text);
+  return { passed: found, detail: found ? `section "${heading}" found` : `section "${heading}" NOT FOUND` };
+}
+
+function wordCount(text, { min = 0, max = Infinity }) {
+  const count = text.split(/\s+/).filter(Boolean).length;
+  const passed = count >= min && count <= max;
+  return { passed, detail: `${count} words (expected ${min}-${max === Infinity ? '∞' : max})` };
+}
+
+function jsonValid(text) {
+  try {
+    JSON.parse(text);
+    return { passed: true, detail: 'valid JSON' };
+  } catch (e) {
+    return { passed: false, detail: `invalid JSON: ${e.message}` };
+  }
+}
+
+function yamlValid(text) {
+  try {
+    require('js-yaml').load(text);
+    return { passed: true, detail: 'valid YAML' };
+  } catch (e) {
+    return { passed: false, detail: `invalid YAML: ${e.message}` };
+  }
+}
+
+function lineCount(text, { min = 0, max = Infinity }) {
+  const count = text.split('\n').length;
+  const passed = count >= min && count <= max;
+  return { passed, detail: `${count} lines (expected ${min}-${max === Infinity ? '∞' : max})` };
+}
+
+// ── Dispatch ─────────────────────────────────────────────────────
+
+const GRADERS = {
+  contains,
+  not_contains: notContains,
+  regex,
+  section_exists: sectionExists,
+  word_count: wordCount,
+  json_valid: jsonValid,
+  yaml_valid: yamlValid,
+  line_count: lineCount,
+};
+
+/**
+ * Run a single grading check.
+ * @param {string} text — the text to grade (prompt content or LLM output)
+ * @param {{ type: string, ...config }} check
+ * @returns {{ type, passed, detail }}
+ */
+function runCheck(text, check) {
+  const grader = GRADERS[check.type];
+  if (!grader) return { type: check.type, passed: false, detail: `unknown grader type "${check.type}"` };
+  const result = grader(text, check);
+  return { type: check.type, ...result };
+}
+
+function escapeRegex(str) {
+  return str.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+}
+
+module.exports = { runCheck, GRADERS, contains, notContains, regex, sectionExists, wordCount, jsonValid, yamlValid, lineCount };

package/lib/eval-runner.js

@@ -0,0 +1,183 @@
+'use strict';
+/**
+ * eval-runner.js — HC-019d eval runner for agent prompt quality.
+ *
+ * Loads eval scenarios from evals/<agent>/*.eval.yml, runs deterministic
+ * grading checks against agent prompt text (zero LLM tokens).
+ *
+ * Pure helper with injected I/O (readFile, listDir).
+ */
+const { runCheck } = require('./eval-graders');
+
+/**
+ * Load eval scenarios from the evals directory.
+ * @param {object} opts
+ * @param {string} opts.evalDir — path to evals/ directory
+ * @param {string} [opts.agent] — filter by agent name
+ * @param {string} [opts.tag] — filter by tag
+ * @param {string} [opts.scenarioId] — run single scenario by ID
+ * @param {(path: string) => string} opts.readFile
+ * @param {(path: string) => string[]} opts.listDir
+ * @param {(path: string) => boolean} opts.isDir
+ * @returns {Array<object>} scenarios
+ */
+function loadScenarios({ evalDir, agent, tag, scenarioId, readFile, listDir, isDir, parseYaml }) {
+  const scenarios = [];
+  const seenIds = new Set();
+  const agentDirs = listDir(evalDir).filter(d => !d.startsWith('_'));
+
+  for (const dir of agentDirs) {
+    if (agent && dir !== agent) continue;
+    const dirPath = `${evalDir}/${dir}`;
+    if (!isDir(dirPath)) continue;
+
+    const files = listDir(dirPath).filter(f => f.endsWith('.eval.yml'));
+    for (const file of files) {
+      try {
+        const content = readFile(`${dirPath}/${file}`);
+        const scenario = parseYaml(content);
+        scenario.evalAgent = dir;
+        scenario.evalFile = file;
+
+        if (scenarioId && scenario.id !== scenarioId) continue;
+        if (tag && !(scenario.tags || []).includes(tag)) continue;
+
+        if (scenario.id && seenIds.has(scenario.id)) {
+          scenarios.push({
+            id: scenario.id, evalAgent: dir, evalFile: file,
+            loadError: `duplicate scenario ID "${scenario.id}" (first seen in another file)`,
+          });
+          continue;
+        }
+        if (scenario.id) seenIds.add(scenario.id);
+
+        scenarios.push(scenario);
+      } catch (e) {
+        scenarios.push({
+          id: file, evalAgent: dir, evalFile: file,
+          loadError: e.message,
+        });
+      }
+    }
+  }
+
+  return scenarios;
+}
+
+/**
+ * Run grading checks for a single scenario against prompt text.
+ * @param {object} scenario — parsed eval scenario
+ * @param {string} promptText — agent prompt content
+ * @returns {{ id, agent, name, result, checks, failures }}
+ */
+function runScenario(scenario, promptText) {
+  if (scenario.loadError) {
+    return {
+      id: scenario.id,
+      agent: scenario.evalAgent,
+      name: scenario.evalFile,
+      result: 'error',
+      checks: 0,
+      checks_passed: 0,
+      failures: [{ type: 'load', passed: false, detail: scenario.loadError }],
+    };
+  }
+
+  const checks = scenario.grading?.checks || [];
+  const results = checks.map(check => runCheck(promptText, check));
+  const passed = results.filter(r => r.passed).length;
+  const failures = results.filter(r => !r.passed);
+
+  return {
+    id: scenario.id,
+    agent: scenario.evalAgent,
+    name: scenario.name || scenario.id,
+    result: failures.length === 0 ? 'pass' : 'fail',
+    checks: checks.length,
+    checks_passed: passed,
+    failures,
+  };
+}
+
+/**
+ * Run all scenarios against their agent prompts.
+ * @param {Array<object>} scenarios
+ * @param {object} agentPrompts — { agentName: promptText }
+ * @param {object} [opts]
+ * @param {boolean} [opts.failFast] — stop on first failure
+ * @returns {{ total, passed, failed, errors, scenarios: Array }}
+ */
+function runAllScenarios(scenarios, agentPrompts, opts = {}) {
+  const results = [];
+
+  for (const scenario of scenarios) {
+    const agentName = scenario.evalAgent || scenario.agent;
+    const promptText = agentPrompts[agentName];
+
+    if (!promptText && !scenario.loadError) {
+      results.push({
+        id: scenario.id,
+        agent: agentName,
+        name: scenario.name || scenario.id,
+        result: 'error',
+        checks: 0,
+        checks_passed: 0,
+        failures: [{ type: 'missing_prompt', passed: false, detail: `agent "${agentName}" not found in AGENT_PROMPTS` }],
+      });
+      continue;
+    }
+
+    const result = runScenario(scenario, promptText || '');
+    results.push(result);
+
+    if (opts.failFast && result.result !== 'pass') break;
+  }
+
+  return {
+    total: results.length,
+    passed: results.filter(r => r.result === 'pass').length,
+    failed: results.filter(r => r.result === 'fail').length,
+    errors: results.filter(r => r.result === 'error').length,
+    scenarios: results,
+  };
+}
+
+/**
+ * Format results for display.
+ * @param {object} results — from runAllScenarios
+ * @param {'pretty'|'json'|'ci'} format
+ * @returns {string}
+ */
+function formatResults(results, format = 'pretty') {
+  if (format === 'json') return JSON.stringify(results, null, 2);
+
+  const lines = ['', 'Hone AI — Agent Eval Runner', '================================', ''];
+
+  // Group by agent
+  const byAgent = {};
+  for (const s of results.scenarios) {
+    if (!byAgent[s.agent]) byAgent[s.agent] = [];
+    byAgent[s.agent].push(s);
+  }
+
+  for (const [agent, scenarios] of Object.entries(byAgent)) {
+    lines.push(`${agent} (${scenarios.length} scenarios)`);
+    for (const s of scenarios) {
+      const icon = s.result === 'pass' ? 'PASS' : s.result === 'fail' ? 'FAIL' : 'ERR ';
+      lines.push(`  [${icon}] ${s.id}: ${s.name} (${s.checks_passed}/${s.checks} checks)`);
+      for (const f of s.failures) {
+        lines.push(`         x ${f.type}: ${f.detail}`);
+      }
+    }
+    lines.push('');
+  }
+
+  lines.push('----------------------------------');
+  lines.push(`Summary: ${results.total} scenarios | ${results.passed} passed | ${results.failed} failed | ${results.errors} errors`);
+  lines.push(`Exit code: ${results.failed + results.errors > 0 ? 1 : 0}`);
+  lines.push('');
+
+  return lines.join('\n');
+}
+
+module.exports = { loadScenarios, runScenario, runAllScenarios, formatResults };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hone-ai/cli",
-  "version": "1.4.0",
+  "version": "1.5.0",
   "description": "Hone AI — Enterprise SDLC Pipeline CLI",
   "main": "hone-cli.js",
   "bin": {