@united-workforce/eval 0.1.7 → 0.2.1

package/README.md

@@ -0,0 +1,13 @@
+# @united-workforce/eval
+
+Evaluation harness for workflow runs. Installs the `uwf-eval` CLI binary.
+
+## Usage
+
+```bash
+uwf-eval run <task-dir> [options]
+```
+
+### Debugging
+
+`uwf-eval run` now cleans up `/tmp` workdirs after each run. Use `--keep-workdir` to retain them for debugging.

package/dist/.build-fingerprint

@@ -1 +1 @@
-f5c52636c23211ac74569ed6f0d9d1fcd247f2f17d9b8800fa47cbe82e05410f
+{"hash":"a185a72834e0bc03535f8eb1205c4670a48cdfe4f836cb10006da5b0dfe8c706","outputs":["cli.d.ts","cli.d.ts.map","cli.js","cli.js.map","commands/diff.d.ts","commands/diff.d.ts.map","commands/diff.js","commands/diff.js.map","commands/format.d.ts","commands/format.d.ts.map","commands/format.js","commands/format.js.map","commands/index.d.ts","commands/index.d.ts.map","commands/index.js","commands/index.js.map","commands/list.d.ts","commands/list.d.ts.map","commands/list.js","commands/list.js.map","commands/read.d.ts","commands/read.d.ts.map","commands/read.js","commands/read.js.map","commands/report.d.ts","commands/report.d.ts.map","commands/report.js","commands/report.js.map","commands/run.d.ts","commands/run.d.ts.map","commands/run.js","commands/run.js.map","commands/types.d.ts","commands/types.d.ts.map","commands/types.js","commands/types.js.map","index.d.ts","index.d.ts.map","index.js","index.js.map","judge/builtin/frontmatter.d.ts","judge/builtin/frontmatter.d.ts.map","judge/builtin/frontmatter.js","judge/builtin/frontmatter.js.map","judge/builtin/hallucination.d.ts","judge/builtin/hallucination.d.ts.map","judge/builtin/hallucination.js","judge/builtin/hallucination.js.map","judge/builtin/index.d.ts","judge/builtin/index.d.ts.map","judge/builtin/index.js","judge/builtin/index.js.map","judge/builtin/read-steps.d.ts","judge/builtin/read-steps.d.ts.map","judge/builtin/read-steps.js","judge/builtin/read-steps.js.map","judge/builtin/token-stats.d.ts","judge/builtin/token-stats.d.ts.map","judge/builtin/token-stats.js","judge/builtin/token-stats.js.map","judge/builtin/types.d.ts","judge/builtin/types.d.ts.map","judge/builtin/types.js","judge/builtin/types.js.map","judge/builtin/upstream.d.ts","judge/builtin/upstream.d.ts.map","judge/builtin/upstream.js","judge/builtin/upstream.js.map","judge/index.d.ts","judge/index.d.ts.map","judge/index.js","judge/index.js.map","judge/types.d.ts","judge/types.d.ts.map","judge/types.js","judge/types.js.map","runner/collect.d.ts","runner/collect.d.ts.map","runner/collect.js","runner/collect.js.map","runner/execute.d.ts","runner/execute.d.ts.map","runner/execute.js","runner/execute.js.map","runner/index.d.ts","runner/index.d.ts.map","runner/index.js","runner/index.js.map","runner/prepare.d.ts","runner/prepare.d.ts.map","runner/prepare.js","runner/prepare.js.map","runner/types.d.ts","runner/types.d.ts.map","runner/types.js","runner/types.js.map","storage/index.d.ts","storage/index.d.ts.map","storage/index.js","storage/index.js.map","storage/schemas.d.ts","storage/schemas.d.ts.map","storage/schemas.js","storage/schemas.js.map","storage/store.d.ts","storage/store.d.ts.map","storage/store.js","storage/store.js.map","storage/types.d.ts","storage/types.d.ts.map","storage/types.js","storage/types.js.map","task/index.d.ts","task/index.d.ts.map","task/index.js","task/index.js.map","task/loader.d.ts","task/loader.d.ts.map","task/loader.js","task/loader.js.map","task/types.d.ts","task/types.d.ts.map","task/types.js","task/types.js.map"]}

package/dist/commands/run.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../src/commands/run.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAgDzC,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI,CAiCzD"}
+{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../src/commands/run.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AA6DzC,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI,CAkCzD"}

package/dist/commands/run.js

@@ -1,40 +1,55 @@
+import { rm } from "node:fs/promises";
 import { resolve } from "node:path";
+import { createLogger } from "@united-workforce/util";
 import { collect, execute, getEngineVersion, prepare } from "../runner/index.js";
 import { createEvalStore } from "../storage/index.js";
-async function runOnce(taskDir, agent, model, engineVersion) {
+const log = createLogger({ sink: { kind: "stderr" } });
+const LOG_CLEANUP = "EVC1NP7Q";
+async function runOnce(taskDir, agent, model, engineVersion, keepWorkdir) {
     const prepared = await prepare(taskDir);
     const { manifest, workDir } = prepared;
-    const { threadId } = await execute({
-        workDir,
-        workflow: manifest.workflow,
-        prompt: manifest.prompt,
-        agent,
-        maxSteps: manifest.limits.maxSteps,
-    });
-    const evalStore = await createEvalStore();
-    const config = { agent, model, engineVersion };
-    const collected = await collect({
-        evalStore,
-        taskDir: prepared.taskDir,
-        workDir,
-        threadId,
-        manifest,
-        config,
-    });
-    return {
-        runHash: collected.runHash,
-        overall: collected.overall,
-        task: manifest.name,
-        judges: collected.judges,
-    };
+    try {
+        const { threadId } = await execute({
+            workDir,
+            workflow: manifest.workflow,
+            prompt: manifest.prompt,
+            agent,
+            maxSteps: manifest.limits.maxSteps,
+        });
+        const evalStore = await createEvalStore();
+        const config = { agent, model, engineVersion };
+        const collected = await collect({
+            evalStore,
+            taskDir: prepared.taskDir,
+            workDir,
+            threadId,
+            manifest,
+            config,
+        });
+        return {
+            runHash: collected.runHash,
+            overall: collected.overall,
+            task: manifest.name,
+            judges: collected.judges,
+        };
+    }
+    finally {
+        if (keepWorkdir) {
+            log(LOG_CLEANUP, `keeping workDir: ${workDir}`);
+        }
+        else {
+            await rm(workDir, { recursive: true, force: true });
+        }
+    }
 }
 export function registerRunCommand(program) {
     program
         .command("run <task>")
         .description("Run eval on a task directory or tarball")
-        .option("--agent <name>", "agent adapter to use", "uwf-hermes")
+        .option("--agent <name>", "agent adapter to use", "uwf-builtin")
         .option("--model <model>", "model override")
         .option("--count <n>", "number of eval runs", "1")
+        .option("--keep-workdir", "keep the temporary workDir after run (for debugging)", false)
         .action(async (task, opts) => {
         const taskDir = resolve(task);
         const agent = opts.agent;
@@ -49,7 +64,7 @@ export function registerRunCommand(program) {
         try {
             const results = [];
             for (let i = 0; i < count; i++) {
-                results.push(await runOnce(taskDir, agent, model, engineVersion));
+                results.push(await runOnce(taskDir, agent, model, engineVersion, opts.keepWorkdir));
             }
             const output = count === 1 ? results[0] : results;
             process.stdout.write(`${JSON.stringify(output)}\n`);

package/dist/commands/run.js.map

@@ -1 +1 @@
-{"version":3,"file":"run.js","sourceRoot":"","sources":["../../src/commands/run.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAIpC,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,gBAAgB,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAEjF,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAQtD,KAAK,UAAU,OAAO,CACpB,OAAe,EACf,KAAa,EACb,KAAa,EACb,aAAqB;IAErB,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,CAAC;IACxC,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,GAAG,QAAQ,CAAC;IAEvC,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,OAAO,CAAC;QACjC,OAAO;QACP,QAAQ,EAAE,QAAQ,CAAC,QAAQ;QAC3B,MAAM,EAAE,QAAQ,CAAC,MAAM;QACvB,KAAK;QACL,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,QAAQ;KACnC,CAAC,CAAC;IAEH,MAAM,SAAS,GAAG,MAAM,eAAe,EAAE,CAAC;IAC1C,MAAM,MAAM,GAAkB,EAAE,KAAK,EAAE,KAAK,EAAE,aAAa,EAAE,CAAC;IAC9D,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC;QAC9B,SAAS;QACT,OAAO,EAAE,QAAQ,CAAC,OAAO;QACzB,OAAO;QACP,QAAQ;QACR,QAAQ;QACR,MAAM;KACP,CAAC,CAAC;IAEH,OAAO;QACL,OAAO,EAAE,SAAS,CAAC,OAAO;QAC1B,OAAO,EAAE,SAAS,CAAC,OAAO;QAC1B,IAAI,EAAE,QAAQ,CAAC,IAAI;QACnB,MAAM,EAAE,SAAS,CAAC,MAAM;KACzB,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,OAAgB;IACjD,OAAO;SACJ,OAAO,CAAC,YAAY,CAAC;SACrB,WAAW,CAAC,yCAAyC,CAAC;SACtD,MAAM,CAAC,gBAAgB,EAAE,sBAAsB,EAAE,YAAY,CAAC;SAC9D,MAAM,CAAC,iBAAiB,EAAE,gBAAgB,CAAC;SAC3C,MAAM,CAAC,aAAa,EAAE,qBAAqB,EAAE,GAAG,CAAC;SACjD,MAAM,CAAC,KAAK,EAAE,IAAY,EAAE,IAAmB,EAAE,EAAE;QAClD,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;QAC9B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;QACzB,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC;QAC/B,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QAC9C,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;YAC1C,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,sCAAsC,CAAC,CAAC;YAC7D,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;YACrB,OAAO;QACT,CAAC;QAED,MAAM,aAAa,GAAG,gBAAgB,EAAE,CAAC;QAEzC,IAAI,CAAC;YACH,MAAM,OAAO,GAAgB,EAAE,CAAC;YAChC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC/B,OAAO,CAAC,IAAI,CAAC,MAAM,OAAO,CAAC,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,aAAa,CAAC,CAAC,CAAC;YACpE,CAAC;YACD,MAAM,MAAM,GAAG,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;YAClD,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QACtD,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,MAAM,OAAO,GAAG,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;YAC3D,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,OAAO,IAAI,CAAC,CAAC;YACrC,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;QACvB,CAAC;IACH,CAAC,CAAC,CAAC;AACP,CAAC"}
+{"version":3,"file":"run.js","sourceRoot":"","sources":["../../src/commands/run.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,EAAE,EAAE,MAAM,kBAAkB,CAAC;AACtC,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAGtD,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,gBAAgB,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAEjF,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAEtD,MAAM,GAAG,GAAG,YAAY,CAAC,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,CAAC,CAAC;AACvD,MAAM,WAAW,GAAG,UAAU,CAAC;AAS/B,KAAK,UAAU,OAAO,CACpB,OAAe,EACf,KAAa,EACb,KAAa,EACb,aAAqB,EACrB,WAAoB;IAEpB,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,CAAC;IACxC,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,GAAG,QAAQ,CAAC;IAEvC,IAAI,CAAC;QACH,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,OAAO,CAAC;YACjC,OAAO;YACP,QAAQ,EAAE,QAAQ,CAAC,QAAQ;YAC3B,MAAM,EAAE,QAAQ,CAAC,MAAM;YACvB,KAAK;YACL,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,QAAQ;SACnC,CAAC,CAAC;QAEH,MAAM,SAAS,GAAG,MAAM,eAAe,EAAE,CAAC;QAC1C,MAAM,MAAM,GAAkB,EAAE,KAAK,EAAE,KAAK,EAAE,aAAa,EAAE,CAAC;QAC9D,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC;YAC9B,SAAS;YACT,OAAO,EAAE,QAAQ,CAAC,OAAO;YACzB,OAAO;YACP,QAAQ;YACR,QAAQ;YACR,MAAM;SACP,CAAC,CAAC;QAEH,OAAO;YACL,OAAO,EAAE,SAAS,CAAC,OAAO;YAC1B,OAAO,EAAE,SAAS,CAAC,OAAO;YAC1B,IAAI,EAAE,QAAQ,CAAC,IAAI;YACnB,MAAM,EAAE,SAAS,CAAC,MAAM;SACzB,CAAC;IACJ,CAAC;YAAS,CAAC;QACT,IAAI,WAAW,EAAE,CAAC;YAChB,GAAG,CAAC,WAAW,EAAE,oBAAoB,OAAO,EAAE,CAAC,CAAC;QAClD,CAAC;aAAM,CAAC;YACN,MAAM,EAAE,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACtD,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,OAAgB;IACjD,OAAO;SACJ,OAAO,CAAC,YAAY,CAAC;SACrB,WAAW,CAAC,yCAAyC,CAAC;SACtD,MAAM,CAAC,gBAAgB,EAAE,sBAAsB,EAAE,aAAa,CAAC;SAC/D,MAAM,CAAC,iBAAiB,EAAE,gBAAgB,CAAC;SAC3C,MAAM,CAAC,aAAa,EAAE,qBAAqB,EAAE,GAAG,CAAC;SACjD,MAAM,CAAC,gBAAgB,EAAE,sDAAsD,EAAE,KAAK,CAAC;SACvF,MAAM,CAAC,KAAK,EAAE,IAAY,EAAE,IAAmB,EAAE,EAAE;QAClD,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;QAC9B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;QACzB,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC;QAC/B,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QAC9C,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;YAC1C,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,sCAAsC,CAAC,CAAC;YAC7D,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;YACrB,OAAO;QACT,CAAC;QAED,MAAM,aAAa,GAAG,gBAAgB,EAAE,CAAC;QAEzC,IAAI,CAAC;YACH,MAAM,OAAO,GAAgB,EAAE,CAAC;YAChC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC/B,OAAO,CAAC,IAAI,CAAC,MAAM,OAAO,CAAC,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,aAAa,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC;YACtF,CAAC;YACD,MAAM,MAAM,GAAG,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;YAClD,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QACtD,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,MAAM,OAAO,GAAG,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;YAC3D,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,OAAO,IAAI,CAAC,CAAC;YACrC,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;QACvB,CAAC;IACH,CAAC,CAAC,CAAC;AACP,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@united-workforce/eval",
-  "version": "0.1.7",
+  "version": "0.2.1",
   "private": false,
   "files": [
     "src",
@@ -22,8 +22,8 @@
     "@ocas/fs": "^0.4.1",
     "commander": "^14.0.3",
     "yaml": "^2.9.0",
-    "@united-workforce/protocol": "^0.4.0",
-    "@united-workforce/util": "^0.2.1"
+    "@united-workforce/protocol": "^0.5.0",
+    "@united-workforce/util": "^0.3.0"
   },
   "devDependencies": {
     "typescript": "^5.8.3"

package/src/commands/run.ts

@@ -1,15 +1,20 @@
+import { rm } from "node:fs/promises";
 import { resolve } from "node:path";
-
+import { createLogger } from "@united-workforce/util";
 import type { Command } from "commander";
 import type { RunResult } from "../runner/index.js";
 import { collect, execute, getEngineVersion, prepare } from "../runner/index.js";
 import type { EvalRunConfig } from "../storage/index.js";
 import { createEvalStore } from "../storage/index.js";
 
+const log = createLogger({ sink: { kind: "stderr" } });
+const LOG_CLEANUP = "EVC1NP7Q";
+
 type RunCliOptions = {
   agent: string;
   model: string | undefined;
   count: string;
+  keepWorkdir: boolean;
 };
 
 async function runOnce(
@@ -17,44 +22,54 @@ async function runOnce(
   agent: string,
   model: string,
   engineVersion: string,
+  keepWorkdir: boolean,
 ): Promise<RunResult> {
   const prepared = await prepare(taskDir);
   const { manifest, workDir } = prepared;
 
-  const { threadId } = await execute({
-    workDir,
-    workflow: manifest.workflow,
-    prompt: manifest.prompt,
-    agent,
-    maxSteps: manifest.limits.maxSteps,
-  });
+  try {
+    const { threadId } = await execute({
+      workDir,
+      workflow: manifest.workflow,
+      prompt: manifest.prompt,
+      agent,
+      maxSteps: manifest.limits.maxSteps,
+    });
 
-  const evalStore = await createEvalStore();
-  const config: EvalRunConfig = { agent, model, engineVersion };
-  const collected = await collect({
-    evalStore,
-    taskDir: prepared.taskDir,
-    workDir,
-    threadId,
-    manifest,
-    config,
-  });
+    const evalStore = await createEvalStore();
+    const config: EvalRunConfig = { agent, model, engineVersion };
+    const collected = await collect({
+      evalStore,
+      taskDir: prepared.taskDir,
+      workDir,
+      threadId,
+      manifest,
+      config,
+    });
 
-  return {
-    runHash: collected.runHash,
-    overall: collected.overall,
-    task: manifest.name,
-    judges: collected.judges,
-  };
+    return {
+      runHash: collected.runHash,
+      overall: collected.overall,
+      task: manifest.name,
+      judges: collected.judges,
+    };
+  } finally {
+    if (keepWorkdir) {
+      log(LOG_CLEANUP, `keeping workDir: ${workDir}`);
+    } else {
+      await rm(workDir, { recursive: true, force: true });
+    }
+  }
 }
 
 export function registerRunCommand(program: Command): void {
   program
     .command("run <task>")
     .description("Run eval on a task directory or tarball")
-    .option("--agent <name>", "agent adapter to use", "uwf-hermes")
+    .option("--agent <name>", "agent adapter to use", "uwf-builtin")
     .option("--model <model>", "model override")
     .option("--count <n>", "number of eval runs", "1")
+    .option("--keep-workdir", "keep the temporary workDir after run (for debugging)", false)
     .action(async (task: string, opts: RunCliOptions) => {
       const taskDir = resolve(task);
       const agent = opts.agent;
@@ -71,7 +86,7 @@ export function registerRunCommand(program: Command): void {
       try {
         const results: RunResult[] = [];
         for (let i = 0; i < count; i++) {
-          results.push(await runOnce(taskDir, agent, model, engineVersion));
+          results.push(await runOnce(taskDir, agent, model, engineVersion, opts.keepWorkdir));
         }
         const output = count === 1 ? results[0] : results;
         process.stdout.write(`${JSON.stringify(output)}\n`);