@princetheprogrammerbtw/husk 0.2.0 → 0.3.0

package/dist/cli/index.js

@@ -1,7 +1,9 @@
 #!/usr/bin/env node
+import { existsSync, statSync, promises } from 'fs';
+import { readdir } from 'fs/promises';
+import { resolve, extname, dirname, join } from 'path';
+import { pathToFileURL } from 'url';
 import { promisify, parseArgs } from 'util';
-import { promises } from 'fs';
-import { resolve, dirname, join } from 'path';
 import Anthropic from '@anthropic-ai/sdk';
 import OpenAI from 'openai';
 import { exec } from 'child_process';
@@ -1042,6 +1044,71 @@ function truncateOutput(output, limit) {
 ... (${lines.length - limit} more matches truncated)`;
 }
 
+// src/evals/runner.ts
+async function runSuite(suite, factory, options = {}) {
+  const start = Date.now();
+  const results = [];
+  let passed = 0;
+  for (const c of suite.cases) {
+    options.onCaseStart?.(c.name);
+    const caseResult = await runCase(c, factory);
+    results.push(caseResult);
+    if (caseResult.passed) passed += 1;
+    options.onCaseEnd?.(caseResult);
+    if (options.failFast && !caseResult.passed) {
+      break;
+    }
+  }
+  return {
+    suiteName: suite.name,
+    results,
+    passed,
+    total: suite.cases.length,
+    durationMs: Date.now() - start
+  };
+}
+async function runCase(c, factory) {
+  const start = Date.now();
+  const agent = await factory();
+  let agentResult;
+  try {
+    agentResult = await agent.run(c.input);
+  } catch (err) {
+    const message = err instanceof Error ? err.message : String(err);
+    const errorAssertionResult = {
+      pass: false,
+      name: "agent.run",
+      message: `agent.run threw: ${message}`
+    };
+    return {
+      caseName: c.name,
+      passed: false,
+      assertionResults: [errorAssertionResult],
+      agentResult: {
+        output: "",
+        messages: [],
+        iterations: 0,
+        usage: { inputTokens: 0, outputTokens: 0 },
+        durationMs: Date.now() - start
+      },
+      durationMs: Date.now() - start
+    };
+  }
+  const assertionResults = [];
+  for (const a of c.assertions) {
+    const r = await a(agentResult);
+    assertionResults.push(r);
+  }
+  const allPassed = assertionResults.every((r) => r.pass);
+  return {
+    caseName: c.name,
+    passed: allPassed,
+    assertionResults,
+    agentResult,
+    durationMs: Date.now() - start
+  };
+}
+
 // src/cli/index.ts
 var TOOL_REGISTRY = { read: Read, write: Write, edit: Edit, bash: Bash, grep: Grep };
 async function main() {
@@ -1054,6 +1121,10 @@ async function main() {
     await runCommand();
     return;
   }
+  if (subcommand === "eval") {
+    await evalCommand();
+    return;
+  }
   if (subcommand === "version" || subcommand === "--version" || subcommand === "-v") {
     console.log(`husk ${VERSION}`);
     return;
@@ -1079,7 +1150,7 @@ async function runCommand() {
     printHelp();
     return;
   }
-  const prompt = values.help === void 0 ? process.argv[3] : void 0;
+  const prompt = process.argv[3];
   if (!prompt) {
     console.error("Error: husk run requires a prompt argument.");
     console.error('Usage: husk run "your prompt here"');
@@ -1111,13 +1182,95 @@ async function runCommand() {
   console.log(result.output);
   process.exit(0);
 }
+async function evalCommand() {
+  const target = process.argv[3];
+  if (!target) {
+    console.error("Error: husk eval requires a path argument.");
+    console.error("Usage: husk eval <file-or-dir>");
+    process.exit(2);
+  }
+  const resolved = resolve(target);
+  if (!existsSync(resolved)) {
+    console.error(`Error: path not found: ${resolved}`);
+    process.exit(2);
+  }
+  const stat = statSync(resolved);
+  const files = [];
+  if (stat.isDirectory()) {
+    const entries = await readdir(resolved, { withFileTypes: true });
+    for (const e of entries) {
+      if (!e.isFile()) continue;
+      const ext = extname(e.name);
+      if (ext === ".ts" || ext === ".js" || ext === ".mjs") {
+        files.push(resolve(resolved, e.name));
+      }
+    }
+  } else {
+    files.push(resolved);
+  }
+  if (files.length === 0) {
+    console.error(`Error: no .ts/.js/.mjs files found in ${resolved}`);
+    process.exit(2);
+  }
+  let totalPassed = 0;
+  let totalCases = 0;
+  let anyFailed = false;
+  for (const file of files) {
+    console.log(`
+=== ${file} ===`);
+    try {
+      const mod = await import(pathToFileURL(file).href);
+      const suites = [];
+      for (const value of Object.values(mod)) {
+        if (value && typeof value === "object" && "name" in value && "cases" in value && Array.isArray(value.cases)) {
+          suites.push(value);
+        }
+      }
+      if (suites.length === 0) {
+        console.error(`  No EvalSuite found in ${file}`);
+        continue;
+      }
+      for (const suite of suites) {
+        const factory = () => Promise.resolve(makeDefaultAgent());
+        const result = await runSuite(suite, factory);
+        totalPassed += result.passed;
+        totalCases += result.total;
+        for (const r of result.results) {
+          const icon = r.passed ? "\u2713" : "\u2717";
+          console.log(`  ${icon} ${r.caseName}`);
+          if (!r.passed) {
+            anyFailed = true;
+            for (const a of r.assertionResults) {
+              console.log(`     \u2717 ${a.name}: ${a.message ?? "failed"}`);
+            }
+          }
+        }
+        console.log(`  ${result.passed}/${result.total} passed in ${result.durationMs}ms`);
+      }
+    } catch (err) {
+      const message = err instanceof Error ? err.message : String(err);
+      console.error(`  Error loading ${file}: ${message}`);
+      anyFailed = true;
+    }
+  }
+  console.log(`
+=== Total: ${totalPassed}/${totalCases} cases passed ===`);
+  process.exit(anyFailed ? 1 : 0);
+}
+function makeDefaultAgent() {
+  const providerName = process.env.HUSK_PROVIDER ?? "anthropic";
+  const modelId = process.env.HUSK_MODEL ?? "claude-opus-4-6";
+  const provider = providerName === "openai" ? new OpenAIProvider({ model: modelId, apiKey: process.env.OPENAI_API_KEY }) : new AnthropicProvider({ model: modelId, apiKey: process.env.ANTHROPIC_API_KEY });
+  return new Agent({ model: provider });
+}
 function printHelp() {
-  console.log(`husk \u2014 run an agent from the command line
+  console.log(`husk \u2014 run an agent or eval suite from the command line
 
 Usage:
   husk run "<prompt>" [options]
+  husk eval <file-or-dir>
 
-Options:
+Run options:
   --model <id>       Model id (default: claude-opus-4-6)
   --provider <name>  'anthropic' (default) or 'openai'
   --tools <list>     Comma-separated tool names: read,write,edit,bash,grep
@@ -1127,6 +1280,10 @@ Options:
   -h, --help         Show this help
   -v, --version      Show version
 
+Eval options:
+  <file>             A .ts/.js/.mjs file exporting one or more EvalSuite
+  <dir>              A directory; all *.ts/*.js/*.mjs files are loaded
+
 Environment:
   ANTHROPIC_API_KEY   Required for Anthropic provider
   OPENAI_API_KEY      Required for OpenAI provider
@@ -1137,9 +1294,10 @@ Examples:
   husk run "What is the capital of France?"
   husk run "Refactor src/foo.ts" --tools read,edit,write
   husk run "Summarize README.md" --provider openai --model gpt-5
+  husk eval ./evals/geography.ts
 `);
 }
-var VERSION = "0.1.0";
+var VERSION = "0.3.0-dev.0";
 await main();
 //# sourceMappingURL=index.js.map
 //# sourceMappingURL=index.js.map