@united-workforce/eval 0.1.0

package/dist/runner/prepare.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"prepare.d.ts","sourceRoot":"","sources":["../../src/runner/prepare.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAiBhD;;;GAGG;AACH,wBAAsB,OAAO,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC,CAgBrE"}

package/dist/runner/prepare.js

@@ -0,0 +1,38 @@
+import { access, cp, mkdir, mkdtemp } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { createLogger } from "@united-workforce/util";
+import { loadTaskManifest } from "../task/index.js";
+const log = createLogger({ sink: { kind: "stderr" } });
+const LOG_PREPARE = "PRE4K2NQ";
+const LOG_FIXTURE = "PRE7M3VX";
+/** Check whether a path exists. */
+async function pathExists(path) {
+    try {
+        await access(path);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+/**
+ * Prepare a task for execution: read its manifest and copy the fixture
+ * directory into a fresh temp working directory.
+ */
+export async function prepare(taskDir) {
+    const manifest = await loadTaskManifest(taskDir);
+    log(LOG_PREPARE, `loaded task manifest name=${manifest.name} workflow=${manifest.workflow}`);
+    const workDir = await mkdtemp(join(tmpdir(), "uwf-eval-"));
+    const fixtureDir = join(taskDir, "fixture");
+    if (await pathExists(fixtureDir)) {
+        await cp(fixtureDir, workDir, { recursive: true });
+        log(LOG_FIXTURE, `copied fixture into workDir=${workDir}`);
+    }
+    else {
+        await mkdir(workDir, { recursive: true });
+        log(LOG_FIXTURE, `no fixture/ found, using empty workDir=${workDir}`);
+    }
+    return { taskDir, workDir, manifest };
+}
+//# sourceMappingURL=prepare.js.map

package/dist/runner/prepare.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"prepare.js","sourceRoot":"","sources":["../../src/runner/prepare.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAC9D,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACjC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAEjC,OAAO,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAEtD,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAGpD,MAAM,GAAG,GAAG,YAAY,CAAC,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,CAAC,CAAC;AAEvD,MAAM,WAAW,GAAG,UAAU,CAAC;AAC/B,MAAM,WAAW,GAAG,UAAU,CAAC;AAE/B,mCAAmC;AACnC,KAAK,UAAU,UAAU,CAAC,IAAY;IACpC,IAAI,CAAC;QACH,MAAM,MAAM,CAAC,IAAI,CAAC,CAAC;QACnB,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,OAAe;IAC3C,MAAM,QAAQ,GAAG,MAAM,gBAAgB,CAAC,OAAO,CAAC,CAAC;IACjD,GAAG,CAAC,WAAW,EAAE,6BAA6B,QAAQ,CAAC,IAAI,aAAa,QAAQ,CAAC,QAAQ,EAAE,CAAC,CAAC;IAE7F,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,WAAW,CAAC,CAAC,CAAC;IAE3D,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;IAC5C,IAAI,MAAM,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;QACjC,MAAM,EAAE,CAAC,UAAU,EAAE,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACnD,GAAG,CAAC,WAAW,EAAE,+BAA+B,OAAO,EAAE,CAAC,CAAC;IAC7D,CAAC;SAAM,CAAC;QACN,MAAM,KAAK,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC1C,GAAG,CAAC,WAAW,EAAE,0CAA0C,OAAO,EAAE,CAAC,CAAC;IACxE,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC;AACxC,CAAC"}

package/dist/runner/types.d.ts

@@ -0,0 +1,70 @@
+import type { JSONSchema } from "@ocas/core";
+import type { EvalRunConfig, EvalStore } from "../storage/index.js";
+import type { JudgeEntry, TaskManifest } from "../task/index.js";
+/** Result of the prepare phase: task dir, temp working dir, parsed manifest. */
+export type PrepareResult = {
+    taskDir: string;
+    workDir: string;
+    manifest: TaskManifest;
+};
+/** Input to the execute phase. */
+export type ExecuteInput = {
+    /** Working directory the workflow runs in (the prepared temp dir). */
+    workDir: string;
+    /** Workflow name or path (from task.yaml). */
+    workflow: string;
+    /** Initial prompt for the thread. */
+    prompt: string;
+    /** Agent adapter to use. */
+    agent: string;
+    /** Maximum number of steps to execute. */
+    maxSteps: number;
+};
+/** Result of the execute phase. */
+export type ExecuteResult = {
+    threadId: string;
+};
+/** Output produced by running a single judge. */
+export type JudgeRunOutput = {
+    score: number;
+    data: unknown;
+    /** Schema describing `data`, used when persisting to CAS. */
+    schema: JSONSchema;
+};
+/** Pluggable judge execution strategy (injectable for testing). */
+export type JudgeRunner = (taskDir: string, workDir: string, threadId: string, judge: JudgeEntry) => Promise<JudgeRunOutput>;
+/** Input to the collect phase. */
+export type CollectInput = {
+    evalStore: EvalStore;
+    taskDir: string;
+    workDir: string;
+    threadId: string;
+    manifest: TaskManifest;
+    config: EvalRunConfig;
+};
+/** A single judge's summarized result in the run output. */
+export type JudgeSummary = {
+    name: string;
+    score: number;
+    weight: number;
+};
+/** Result of the collect phase. */
+export type CollectResult = {
+    runHash: string;
+    overall: number;
+    judges: JudgeSummary[];
+};
+/** Options for a full eval run (from CLI flags). */
+export type RunOptions = {
+    agent: string;
+    model: string;
+    count: number;
+};
+/** Final result of a full eval run. */
+export type RunResult = {
+    runHash: string;
+    overall: number;
+    task: string;
+    judges: JudgeSummary[];
+};
+//# sourceMappingURL=types.d.ts.map

package/dist/runner/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/runner/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAE7C,OAAO,KAAK,EAAE,aAAa,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AACpE,OAAO,KAAK,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAEjE,gFAAgF;AAChF,MAAM,MAAM,aAAa,GAAG;IAC1B,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,YAAY,CAAC;CACxB,CAAC;AAEF,kCAAkC;AAClC,MAAM,MAAM,YAAY,GAAG;IACzB,sEAAsE;IACtE,OAAO,EAAE,MAAM,CAAC;IAChB,8CAA8C;IAC9C,QAAQ,EAAE,MAAM,CAAC;IACjB,qCAAqC;IACrC,MAAM,EAAE,MAAM,CAAC;IACf,4BAA4B;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,0CAA0C;IAC1C,QAAQ,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF,mCAAmC;AACnC,MAAM,MAAM,aAAa,GAAG;IAC1B,QAAQ,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF,iDAAiD;AACjD,MAAM,MAAM,cAAc,GAAG;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,OAAO,CAAC;IACd,6DAA6D;IAC7D,MAAM,EAAE,UAAU,CAAC;CACpB,CAAC;AAEF,mEAAmE;AACnE,MAAM,MAAM,WAAW,GAAG,CACxB,OAAO,EAAE,MAAM,EACf,OAAO,EAAE,MAAM,EACf,QAAQ,EAAE,MAAM,EAChB,KAAK,EAAE,UAAU,KACd,OAAO,CAAC,cAAc,CAAC,CAAC;AAE7B,kCAAkC;AAClC,MAAM,MAAM,YAAY,GAAG;IACzB,SAAS,EAAE,SAAS,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,YAAY,CAAC;IACvB,MAAM,EAAE,aAAa,CAAC;CACvB,CAAC;AAEF,4DAA4D;AAC5D,MAAM,MAAM,YAAY,GAAG;IACzB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF,mCAAmC;AACnC,MAAM,MAAM,aAAa,GAAG;IAC1B,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,YAAY,EAAE,CAAC;CACxB,CAAC;AAEF,oDAAoD;AACpD,MAAM,MAAM,UAAU,GAAG;IACvB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACf,CAAC;AAEF,uCAAuC;AACvC,MAAM,MAAM,SAAS,GAAG;IACtB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,YAAY,EAAE,CAAC;CACxB,CAAC"}

package/dist/runner/types.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=types.js.map

package/dist/runner/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/runner/types.ts"],"names":[],"mappings":""}

package/dist/storage/index.d.ts

@@ -0,0 +1,4 @@
+export { EVAL_JUDGE_FRONTMATTER_SCHEMA, EVAL_JUDGE_HALLUCINATION_SCHEMA, EVAL_JUDGE_TOKEN_STATS_SCHEMA, EVAL_JUDGE_UPSTREAM_SCHEMA, EVAL_RUN_SCHEMA, } from "./schemas.js";
+export { createEvalStore, setEvalLatest } from "./store.js";
+export type { EvalJudgeRecord, EvalRunConfig, EvalRunPayload, EvalStore } from "./types.js";
+//# sourceMappingURL=index.d.ts.map

package/dist/storage/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/storage/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,6BAA6B,EAC7B,+BAA+B,EAC/B,6BAA6B,EAC7B,0BAA0B,EAC1B,eAAe,GAChB,MAAM,cAAc,CAAC;AACtB,OAAO,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAC5D,YAAY,EAAE,eAAe,EAAE,aAAa,EAAE,cAAc,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC"}

package/dist/storage/index.js

@@ -0,0 +1,3 @@
+export { EVAL_JUDGE_FRONTMATTER_SCHEMA, EVAL_JUDGE_HALLUCINATION_SCHEMA, EVAL_JUDGE_TOKEN_STATS_SCHEMA, EVAL_JUDGE_UPSTREAM_SCHEMA, EVAL_RUN_SCHEMA, } from "./schemas.js";
+export { createEvalStore, setEvalLatest } from "./store.js";
+//# sourceMappingURL=index.js.map

package/dist/storage/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/storage/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,6BAA6B,EAC7B,+BAA+B,EAC/B,6BAA6B,EAC7B,0BAA0B,EAC1B,eAAe,GAChB,MAAM,cAAc,CAAC;AACtB,OAAO,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC"}

package/dist/storage/schemas.d.ts

@@ -0,0 +1,7 @@
+import type { JSONSchema } from "@ocas/core";
+export declare const EVAL_RUN_SCHEMA: JSONSchema;
+export declare const EVAL_JUDGE_FRONTMATTER_SCHEMA: JSONSchema;
+export declare const EVAL_JUDGE_UPSTREAM_SCHEMA: JSONSchema;
+export declare const EVAL_JUDGE_HALLUCINATION_SCHEMA: JSONSchema;
+export declare const EVAL_JUDGE_TOKEN_STATS_SCHEMA: JSONSchema;
+//# sourceMappingURL=schemas.d.ts.map

package/dist/storage/schemas.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"schemas.d.ts","sourceRoot":"","sources":["../../src/storage/schemas.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAE7C,eAAO,MAAM,eAAe,EAAE,UAgC7B,CAAC;AAEF,eAAO,MAAM,6BAA6B,EAAE,UAoB3C,CAAC;AAEF,eAAO,MAAM,0BAA0B,EAAE,UAmBxC,CAAC;AAEF,eAAO,MAAM,+BAA+B,EAAE,UAkB7C,CAAC;AAEF,eAAO,MAAM,6BAA6B,EAAE,UAuB3C,CAAC"}

package/dist/storage/schemas.js

@@ -0,0 +1,118 @@
+export const EVAL_RUN_SCHEMA = {
+    title: "@uwf/eval-run",
+    type: "object",
+    required: ["task", "config", "threadId", "judges", "overall", "timestamp"],
+    properties: {
+        task: { type: "string" },
+        config: {
+            type: "object",
+            required: ["agent", "model", "engineVersion"],
+            properties: {
+                agent: { type: "string" },
+                model: { type: "string" },
+                engineVersion: { type: "string" },
+            },
+        },
+        threadId: { type: "string" },
+        judges: {
+            type: "array",
+            items: {
+                type: "object",
+                required: ["name", "score", "weight", "dataHash"],
+                properties: {
+                    name: { type: "string" },
+                    score: { type: "number" },
+                    weight: { type: "number" },
+                    dataHash: { type: "string" },
+                },
+            },
+        },
+        overall: { type: "number" },
+        timestamp: { type: "integer" },
+    },
+};
+export const EVAL_JUDGE_FRONTMATTER_SCHEMA = {
+    title: "@uwf/eval-judge-frontmatter",
+    type: "object",
+    required: ["stepsTotal", "stepsValid", "invalidSteps"],
+    properties: {
+        stepsTotal: { type: "integer" },
+        stepsValid: { type: "integer" },
+        invalidSteps: {
+            type: "array",
+            items: {
+                type: "object",
+                required: ["stepIndex", "role", "errors"],
+                properties: {
+                    stepIndex: { type: "integer" },
+                    role: { type: "string" },
+                    errors: { type: "array", items: { type: "string" } },
+                },
+            },
+        },
+    },
+};
+export const EVAL_JUDGE_UPSTREAM_SCHEMA = {
+    title: "@uwf/eval-judge-upstream",
+    type: "object",
+    required: ["perStep"],
+    properties: {
+        perStep: {
+            type: "array",
+            items: {
+                type: "object",
+                required: ["role", "consumed", "missed", "score"],
+                properties: {
+                    role: { type: "string" },
+                    consumed: { type: "array", items: { type: "string" } },
+                    missed: { type: "array", items: { type: "string" } },
+                    score: { type: "number" },
+                },
+            },
+        },
+    },
+};
+export const EVAL_JUDGE_HALLUCINATION_SCHEMA = {
+    title: "@uwf/eval-judge-hallucination",
+    type: "object",
+    required: ["perStep"],
+    properties: {
+        perStep: {
+            type: "array",
+            items: {
+                type: "object",
+                required: ["role", "hallucinations", "score"],
+                properties: {
+                    role: { type: "string" },
+                    hallucinations: { type: "array", items: { type: "string" } },
+                    score: { type: "number" },
+                },
+            },
+        },
+    },
+};
+export const EVAL_JUDGE_TOKEN_STATS_SCHEMA = {
+    title: "@uwf/eval-judge-token-stats",
+    type: "object",
+    required: ["totalInput", "totalOutput", "totalTurns", "perStep"],
+    properties: {
+        totalInput: { type: "integer" },
+        totalOutput: { type: "integer" },
+        totalTurns: { type: "integer" },
+        perStep: {
+            type: "array",
+            items: {
+                type: "object",
+                required: ["role", "inputTokens", "outputTokens", "turns", "duration"],
+                properties: {
+                    role: { type: "string" },
+                    inputTokens: { type: "integer" },
+                    outputTokens: { type: "integer" },
+                    turns: { type: "integer" },
+                    duration: { type: "number" },
+                },
+            },
+        },
+    },
+};
+//# sourceMappingURL=schemas.js.map

package/dist/storage/schemas.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"schemas.js","sourceRoot":"","sources":["../../src/storage/schemas.ts"],"names":[],"mappings":"AAEA,MAAM,CAAC,MAAM,eAAe,GAAe;IACzC,KAAK,EAAE,eAAe;IACtB,IAAI,EAAE,QAAQ;IACd,QAAQ,EAAE,CAAC,MAAM,EAAE,QAAQ,EAAE,UAAU,EAAE,QAAQ,EAAE,SAAS,EAAE,WAAW,CAAC;IAC1E,UAAU,EAAE;QACV,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;QACxB,MAAM,EAAE;YACN,IAAI,EAAE,QAAQ;YACd,QAAQ,EAAE,CAAC,OAAO,EAAE,OAAO,EAAE,eAAe,CAAC;YAC7C,UAAU,EAAE;gBACV,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;gBACzB,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;gBACzB,aAAa,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;aAClC;SACF;QACD,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;QAC5B,MAAM,EAAE;YACN,IAAI,EAAE,OAAO;YACb,KAAK,EAAE;gBACL,IAAI,EAAE,QAAQ;gBACd,QAAQ,EAAE,CAAC,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,UAAU,CAAC;gBACjD,UAAU,EAAE;oBACV,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;oBACxB,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;oBACzB,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;oBAC1B,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;iBAC7B;aACF;SACF;QACD,OAAO,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;QAC3B,SAAS,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE;KAC/B;CACF,CAAC;AAEF,MAAM,CAAC,MAAM,6BAA6B,GAAe;IACvD,KAAK,EAAE,6BAA6B;IACpC,IAAI,EAAE,QAAQ;IACd,QAAQ,EAAE,CAAC,YAAY,EAAE,YAAY,EAAE,cAAc,CAAC;IACtD,UAAU,EAAE;QACV,UAAU,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE;QAC/B,UAAU,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE;QAC/B,YAAY,EAAE;YACZ,IAAI,EAAE,OAAO;YACb,KAAK,EAAE;gBACL,IAAI,EAAE,QAAQ;gBACd,QAAQ,EAAE,CAAC,WAAW,EAAE,MAAM,EAAE,QAAQ,CAAC;gBACzC,UAAU,EAAE;oBACV,SAAS,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE;oBAC9B,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;oBACxB,MAAM,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE;iBACrD;aACF;SACF;KACF;CACF,CAAC;AAEF,MAAM,CAAC,MAAM,0BAA0B,GAAe;IACpD,KAAK,EAAE,0BAA0B;IACjC,IAAI,EAAE,QAAQ;IACd,QAAQ,EAAE,CAAC,SAAS,CAAC;IACrB,UAAU,EAAE;QACV,OAAO,EAAE;YACP,IAAI,EAAE,OAAO;YACb,KAAK,EAAE;gBACL,IAAI,EAAE,QAAQ;gBACd,QAAQ,EAAE,CAAC,MAAM,EAAE,UAAU,EAAE,QAAQ,EAAE,OAAO,CAAC;gBACjD,UAAU,EAAE;oBACV,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;oBACxB,QAAQ,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE;oBACtD,MAAM,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE;oBACpD,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;iBAC1B;aACF;SACF;KACF;CACF,CAAC;AAEF,MAAM,CAAC,MAAM,+BAA+B,GAAe;IACzD,KAAK,EAAE,+BAA+B;IACtC,IAAI,EAAE,QAAQ;IACd,QAAQ,EAAE,CAAC,SAAS,CAAC;IACrB,UAAU,EAAE;QACV,OAAO,EAAE;YACP,IAAI,EAAE,OAAO;YACb,KAAK,EAAE;gBACL,IAAI,EAAE,QAAQ;gBACd,QAAQ,EAAE,CAAC,MAAM,EAAE,gBAAgB,EAAE,OAAO,CAAC;gBAC7C,UAAU,EAAE;oBACV,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;oBACxB,cAAc,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE;oBAC5D,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;iBAC1B;aACF;SACF;KACF;CACF,CAAC;AAEF,MAAM,CAAC,MAAM,6BAA6B,GAAe;IACvD,KAAK,EAAE,6BAA6B;IACpC,IAAI,EAAE,QAAQ;IACd,QAAQ,EAAE,CAAC,YAAY,EAAE,aAAa,EAAE,YAAY,EAAE,SAAS,CAAC;IAChE,UAAU,EAAE;QACV,UAAU,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE;QAC/B,WAAW,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE;QAChC,UAAU,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE;QAC/B,OAAO,EAAE;YACP,IAAI,EAAE,OAAO;YACb,KAAK,EAAE;gBACL,IAAI,EAAE,QAAQ;gBACd,QAAQ,EAAE,CAAC,MAAM,EAAE,aAAa,EAAE,cAAc,EAAE,OAAO,EAAE,UAAU,CAAC;gBACtE,UAAU,EAAE;oBACV,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;oBACxB,WAAW,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE;oBAChC,YAAY,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE;oBACjC,KAAK,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE;oBAC1B,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;iBAC7B;aACF;SACF;KACF;CACF,CAAC"}

package/dist/storage/store.d.ts

@@ -0,0 +1,10 @@
+import type { VarStore } from "@ocas/core";
+import type { EvalStore } from "./types.js";
+/**
+ * Open the unified OCAS store on the filesystem.
+ * Shares the same CAS + variable backend as the uwf CLI.
+ */
+export declare function createEvalStore(): Promise<EvalStore>;
+/** Set the `@uwf/eval/<task>/latest` variable to point at a run hash. */
+export declare function setEvalLatest(varStore: VarStore, taskName: string, runHash: string): void;
+//# sourceMappingURL=store.d.ts.map

package/dist/storage/store.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"store.d.ts","sourceRoot":"","sources":["../../src/storage/store.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAI3C,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAiB5C;;;GAGG;AACH,wBAAsB,eAAe,IAAI,OAAO,CAAC,SAAS,CAAC,CAQ1D;AAED,yEAAyE;AACzE,wBAAgB,aAAa,CAAC,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,IAAI,CAEzF"}

package/dist/storage/store.js

@@ -0,0 +1,36 @@
+import { mkdir } from "node:fs/promises";
+import { homedir } from "node:os";
+import { join } from "node:path";
+import { bootstrap } from "@ocas/core";
+import { createFsStore, createSqliteVarStore } from "@ocas/fs";
+/** Variable name prefix for eval run pointers (`@uwf/eval/<task>/latest`). */
+const EVAL_VAR_PREFIX = "@uwf/eval/";
+/**
+ * Resolve the global CAS directory shared by all uwf and ocas tools.
+ * Priority: `OCAS_HOME` → default ~/.ocas (matches uwf CLI's getGlobalCasDir).
+ */
+function getGlobalCasDir() {
+    const primary = process.env.OCAS_HOME;
+    if (primary !== undefined && primary !== "") {
+        return primary;
+    }
+    return join(homedir(), ".ocas");
+}
+/**
+ * Open the unified OCAS store on the filesystem.
+ * Shares the same CAS + variable backend as the uwf CLI.
+ */
+export async function createEvalStore() {
+    const casDir = getGlobalCasDir();
+    await mkdir(casDir, { recursive: true });
+    const cas = createFsStore(casDir);
+    const { var: varStore, tag } = createSqliteVarStore(join(casDir, "vars"), cas);
+    const store = { cas, var: varStore, tag };
+    bootstrap(store);
+    return { store, varStore };
+}
+/** Set the `@uwf/eval/<task>/latest` variable to point at a run hash. */
+export function setEvalLatest(varStore, taskName, runHash) {
+    varStore.set(`${EVAL_VAR_PREFIX}${taskName}/latest`, runHash);
+}
+//# sourceMappingURL=store.js.map

package/dist/storage/store.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"store.js","sourceRoot":"","sources":["../../src/storage/store.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACzC,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAClC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAEjC,OAAO,EAAE,SAAS,EAAc,MAAM,YAAY,CAAC;AACnD,OAAO,EAAE,aAAa,EAAE,oBAAoB,EAAE,MAAM,UAAU,CAAC;AAI/D,8EAA8E;AAC9E,MAAM,eAAe,GAAG,YAAY,CAAC;AAErC;;;GAGG;AACH,SAAS,eAAe;IACtB,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC;IACtC,IAAI,OAAO,KAAK,SAAS,IAAI,OAAO,KAAK,EAAE,EAAE,CAAC;QAC5C,OAAO,OAAO,CAAC;IACjB,CAAC;IACD,OAAO,IAAI,CAAC,OAAO,EAAE,EAAE,OAAO,CAAC,CAAC;AAClC,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe;IACnC,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;IACjC,MAAM,KAAK,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACzC,MAAM,GAAG,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC;IAClC,MAAM,EAAE,GAAG,EAAE,QAAQ,EAAE,GAAG,EAAE,GAAG,oBAAoB,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,GAAG,CAAC,CAAC;IAC/E,MAAM,KAAK,GAAU,EAAE,GAAG,EAAE,GAAG,EAAE,QAAQ,EAAE,GAAG,EAAE,CAAC;IACjD,SAAS,CAAC,KAAK,CAAC,CAAC;IACjB,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;AAC7B,CAAC;AAED,yEAAyE;AACzE,MAAM,UAAU,aAAa,CAAC,QAAkB,EAAE,QAAgB,EAAE,OAAe;IACjF,QAAQ,CAAC,GAAG,CAAC,GAAG,eAAe,GAAG,QAAQ,SAAS,EAAE,OAAO,CAAC,CAAC;AAChE,CAAC"}

package/dist/storage/types.d.ts

@@ -0,0 +1,30 @@
+import type { Store, VarStore } from "@ocas/core";
+import type { CasRef } from "@united-workforce/protocol";
+/** Handle to the OCAS store used for eval persistence. */
+export type EvalStore = {
+    store: Store;
+    varStore: VarStore;
+};
+/** A single judge result within an eval run. */
+export type EvalJudgeRecord = {
+    name: string;
+    score: number;
+    weight: number;
+    dataHash: CasRef;
+};
+/** Config snapshot for an eval run. */
+export type EvalRunConfig = {
+    agent: string;
+    model: string;
+    engineVersion: string;
+};
+/** Full eval run record stored in CAS. */
+export type EvalRunPayload = {
+    task: string;
+    config: EvalRunConfig;
+    threadId: string;
+    judges: EvalJudgeRecord[];
+    overall: number;
+    timestamp: number;
+};
+//# sourceMappingURL=types.d.ts.map

package/dist/storage/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/storage/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAClD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,4BAA4B,CAAC;AAEzD,0DAA0D;AAC1D,MAAM,MAAM,SAAS,GAAG;IACtB,KAAK,EAAE,KAAK,CAAC;IACb,QAAQ,EAAE,QAAQ,CAAC;CACpB,CAAC;AAEF,gDAAgD;AAChD,MAAM,MAAM,eAAe,GAAG;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF,uCAAuC;AACvC,MAAM,MAAM,aAAa,GAAG;IAC1B,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,aAAa,EAAE,MAAM,CAAC;CACvB,CAAC;AAEF,0CAA0C;AAC1C,MAAM,MAAM,cAAc,GAAG;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,aAAa,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,eAAe,EAAE,CAAC;IAC1B,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;CACnB,CAAC"}

package/dist/storage/types.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=types.js.map

package/dist/storage/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/storage/types.ts"],"names":[],"mappings":""}

package/dist/task/index.d.ts

@@ -0,0 +1,3 @@
+export { loadTaskManifest, parseTaskManifest } from "./loader.js";
+export type { JudgeEntry, TaskLimits, TaskManifest } from "./types.js";
+//# sourceMappingURL=index.d.ts.map

package/dist/task/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/task/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAClE,YAAY,EAAE,UAAU,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC"}

package/dist/task/index.js

@@ -0,0 +1,2 @@
+export { loadTaskManifest, parseTaskManifest } from "./loader.js";
+//# sourceMappingURL=index.js.map

package/dist/task/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/task/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC"}

package/dist/task/loader.d.ts

@@ -0,0 +1,6 @@
+import type { TaskManifest } from "./types.js";
+/** Parse and validate a task.yaml file into a TaskManifest. */
+export declare function parseTaskManifest(yamlText: string): TaskManifest;
+/** Load and parse task.yaml from a directory. */
+export declare function loadTaskManifest(taskDir: string): Promise<TaskManifest>;
+//# sourceMappingURL=loader.d.ts.map

package/dist/task/loader.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"loader.d.ts","sourceRoot":"","sources":["../../src/task/loader.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAA0B,YAAY,EAAE,MAAM,YAAY,CAAC;AAkCvE,+DAA+D;AAC/D,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,GAAG,YAAY,CA4BhE;AAED,iDAAiD;AACjD,wBAAsB,gBAAgB,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC,CAI7E"}

package/dist/task/loader.js

@@ -0,0 +1,69 @@
+import { readFile } from "node:fs/promises";
+import { join } from "node:path";
+import { parse as parseYaml } from "yaml";
+function isRecord(value) {
+    return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function parseJudgeEntry(raw, index) {
+    if (!isRecord(raw)) {
+        throw new Error(`judges[${index}]: expected object`);
+    }
+    const name = raw.name;
+    if (typeof name !== "string" || name === "") {
+        throw new Error(`judges[${index}]: name is required`);
+    }
+    const weight = typeof raw.weight === "number" ? raw.weight : 0;
+    const builtin = raw.builtin === true;
+    const entry = typeof raw.entry === "string" ? raw.entry : null;
+    const schema = typeof raw.schema === "string" ? raw.schema : null;
+    if (!builtin && entry === null) {
+        throw new Error(`judges[${index}] "${name}": non-builtin judge must have entry`);
+    }
+    return { name, weight, builtin, entry, schema };
+}
+function parseLimits(raw) {
+    if (!isRecord(raw)) {
+        return { maxSteps: 20, timeoutMinutes: 30 };
+    }
+    return {
+        maxSteps: typeof raw.maxSteps === "number" ? raw.maxSteps : 20,
+        timeoutMinutes: typeof raw.timeoutMinutes === "number" ? raw.timeoutMinutes : 30,
+    };
+}
+/** Parse and validate a task.yaml file into a TaskManifest. */
+export function parseTaskManifest(yamlText) {
+    const raw = parseYaml(yamlText);
+    if (!isRecord(raw)) {
+        throw new Error("task.yaml must be a YAML mapping");
+    }
+    const name = raw.name;
+    if (typeof name !== "string" || name === "") {
+        throw new Error("task.yaml: name is required");
+    }
+    const description = typeof raw.description === "string" ? raw.description : "";
+    const workflow = raw.workflow;
+    if (typeof workflow !== "string" || workflow === "") {
+        throw new Error("task.yaml: workflow is required");
+    }
+    const prompt = raw.prompt;
+    if (typeof prompt !== "string" || prompt === "") {
+        throw new Error("task.yaml: prompt is required");
+    }
+    const limits = parseLimits(raw.limits);
+    const judgesRaw = raw.judges;
+    if (!Array.isArray(judgesRaw) || judgesRaw.length === 0) {
+        throw new Error("task.yaml: at least one judge is required");
+    }
+    const judges = [];
+    for (let i = 0; i < judgesRaw.length; i++) {
+        judges.push(parseJudgeEntry(judgesRaw[i], i));
+    }
+    return { name, description, workflow, prompt, limits, judges };
+}
+/** Load and parse task.yaml from a directory. */
+export async function loadTaskManifest(taskDir) {
+    const yamlPath = join(taskDir, "task.yaml");
+    const text = await readFile(yamlPath, "utf8");
+    return parseTaskManifest(text);
+}
+//# sourceMappingURL=loader.js.map

package/dist/task/loader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"loader.js","sourceRoot":"","sources":["../../src/task/loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,KAAK,IAAI,SAAS,EAAE,MAAM,MAAM,CAAC;AAG1C,SAAS,QAAQ,CAAC,KAAc;IAC9B,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;AAC9E,CAAC;AAED,SAAS,eAAe,CAAC,GAAY,EAAE,KAAa;IAClD,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QACnB,MAAM,IAAI,KAAK,CAAC,UAAU,KAAK,oBAAoB,CAAC,CAAC;IACvD,CAAC;IACD,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC;IACtB,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,EAAE,EAAE,CAAC;QAC5C,MAAM,IAAI,KAAK,CAAC,UAAU,KAAK,qBAAqB,CAAC,CAAC;IACxD,CAAC;IACD,MAAM,MAAM,GAAG,OAAO,GAAG,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IAC/D,MAAM,OAAO,GAAG,GAAG,CAAC,OAAO,KAAK,IAAI,CAAC;IACrC,MAAM,KAAK,GAAG,OAAO,GAAG,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC;IAC/D,MAAM,MAAM,GAAG,OAAO,GAAG,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC;IAClE,IAAI,CAAC,OAAO,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;QAC/B,MAAM,IAAI,KAAK,CAAC,UAAU,KAAK,MAAM,IAAI,sCAAsC,CAAC,CAAC;IACnF,CAAC;IACD,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC;AAClD,CAAC;AAED,SAAS,WAAW,CAAC,GAAY;IAC/B,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QACnB,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,cAAc,EAAE,EAAE,EAAE,CAAC;IAC9C,CAAC;IACD,OAAO;QACL,QAAQ,EAAE,OAAO,GAAG,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE;QAC9D,cAAc,EAAE,OAAO,GAAG,CAAC,cAAc,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE;KACjF,CAAC;AACJ,CAAC;AAED,+DAA+D;AAC/D,MAAM,UAAU,iBAAiB,CAAC,QAAgB;IAChD,MAAM,GAAG,GAAG,SAAS,CAAC,QAAQ,CAAY,CAAC;IAC3C,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QACnB,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAC;IACtD,CAAC;IACD,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC;IACtB,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,EAAE,EAAE,CAAC;QAC5C,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC;IACjD,CAAC;IACD,MAAM,WAAW,GAAG,OAAO,GAAG,CAAC,WAAW,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,CAAC;IAC/E,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC;IAC9B,IAAI,OAAO,QAAQ,KAAK,QAAQ,IAAI,QAAQ,KAAK,EAAE,EAAE,CAAC;QACpD,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC;IACrD,CAAC;IACD,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC;IAC1B,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,EAAE,EAAE,CAAC;QAChD,MAAM,IAAI,KAAK,CAAC,+BAA+B,CAAC,CAAC;IACnD,CAAC;IACD,MAAM,MAAM,GAAG,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IACvC,MAAM,SAAS,GAAG,GAAG,CAAC,MAAM,CAAC;IAC7B,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxD,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;IAC/D,CAAC;IACD,MAAM,MAAM,GAAiB,EAAE,CAAC;IAChC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC1C,MAAM,CAAC,IAAI,CAAC,eAAe,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAChD,CAAC;IACD,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC;AACjE,CAAC;AAED,iDAAiD;AACjD,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,OAAe;IACpD,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;IAC5C,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IAC9C,OAAO,iBAAiB,CAAC,IAAI,CAAC,CAAC;AACjC,CAAC"}

package/dist/task/types.d.ts

@@ -0,0 +1,27 @@
+/** Judge entry in task.yaml */
+export type JudgeEntry = {
+    name: string;
+    weight: number;
+    builtin: boolean;
+    /** Path to judge entry script (relative to task root). Required for non-builtin judges. */
+    entry: string | null;
+    /** Path to OCAS schema JSON for judge data. Required for non-builtin judges. */
+    schema: string | null;
+};
+/** Limits for eval execution. */
+export type TaskLimits = {
+    maxSteps: number;
+    timeoutMinutes: number;
+};
+/** Parsed task.yaml manifest. */
+export type TaskManifest = {
+    name: string;
+    description: string;
+    /** Workflow name or relative path to .yaml file. */
+    workflow: string;
+    /** Initial prompt for thread start. */
+    prompt: string;
+    limits: TaskLimits;
+    judges: JudgeEntry[];
+};
+//# sourceMappingURL=types.d.ts.map

package/dist/task/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/task/types.ts"],"names":[],"mappings":"AAAA,+BAA+B;AAC/B,MAAM,MAAM,UAAU,GAAG;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,OAAO,CAAC;IACjB,2FAA2F;IAC3F,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,gFAAgF;IAChF,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;CACvB,CAAC;AAEF,iCAAiC;AACjC,MAAM,MAAM,UAAU,GAAG;IACvB,QAAQ,EAAE,MAAM,CAAC;IACjB,cAAc,EAAE,MAAM,CAAC;CACxB,CAAC;AAEF,iCAAiC;AACjC,MAAM,MAAM,YAAY,GAAG;IACzB,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,oDAAoD;IACpD,QAAQ,EAAE,MAAM,CAAC;IACjB,uCAAuC;IACvC,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,UAAU,CAAC;IACnB,MAAM,EAAE,UAAU,EAAE,CAAC;CACtB,CAAC"}

package/dist/task/types.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=types.js.map

package/dist/task/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/task/types.ts"],"names":[],"mappings":""}

package/package.json

@@ -0,0 +1,45 @@
+{
+  "name": "@united-workforce/eval",
+  "version": "0.1.0",
+  "private": false,
+  "files": [
+    "src",
+    "dist",
+    "package.json"
+  ],
+  "type": "module",
+  "bin": {
+    "uwf-eval": "./dist/cli.js"
+  },
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    }
+  },
+  "dependencies": {
+    "@ocas/core": "^0.3.0",
+    "@ocas/fs": "^0.3.0",
+    "commander": "^14.0.3",
+    "yaml": "^2.9.0",
+    "@united-workforce/protocol": "^0.1.0",
+    "@united-workforce/util": "^0.1.0"
+  },
+  "devDependencies": {
+    "typescript": "^5.8.3"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://git.shazhou.work/shazhou/united-workforce.git",
+    "directory": "packages/eval"
+  },
+  "homepage": "https://git.shazhou.work/shazhou/united-workforce#readme",
+  "bugs": {
+    "url": "https://git.shazhou.work/shazhou/united-workforce/issues"
+  },
+  "license": "MIT",
+  "scripts": {
+    "test": "vitest run __tests__/",
+    "test:ci": "vitest run __tests__/"
+  }
+}

package/src/cli.ts

@@ -0,0 +1,22 @@
+#!/usr/bin/env node
+import { Command } from "commander";
+import {
+  registerDiffCommand,
+  registerListCommand,
+  registerReportCommand,
+  registerRunCommand,
+} from "./commands/index.js";
+
+const program = new Command();
+
+program
+  .name("uwf-eval")
+  .description("Evaluate uwf workflow quality with real agents")
+  .version("0.1.0");
+
+registerRunCommand(program);
+registerReportCommand(program);
+registerDiffCommand(program);
+registerListCommand(program);
+
+program.parse();

package/src/commands/diff.ts

@@ -0,0 +1,38 @@
+import { createLogger } from "@united-workforce/util";
+import type { Command } from "commander";
+
+import { createEvalStore } from "../storage/index.js";
+import { formatDiff } from "./format.js";
+import { readEvalRun } from "./read.js";
+
+const log = createLogger({ sink: { kind: "stderr" } });
+const LOG_DIFF = "D3WZ8N5T";
+
+export function registerDiffCommand(program: Command): void {
+  program
+    .command("diff <hash1> <hash2>")
+    .description("Compare two eval runs side-by-side")
+    .action(async (hash1: string, hash2: string) => {
+      try {
+        const evalStore = await createEvalStore();
+        const payloadA = readEvalRun(evalStore, hash1);
+        if (payloadA === null) {
+          process.stderr.write(`eval run not found: ${hash1}\n`);
+          process.exitCode = 1;
+          return;
+        }
+        const payloadB = readEvalRun(evalStore, hash2);
+        if (payloadB === null) {
+          process.stderr.write(`eval run not found: ${hash2}\n`);
+          process.exitCode = 1;
+          return;
+        }
+        log(LOG_DIFF, `diff a=${hash1} b=${hash2}`);
+        process.stdout.write(formatDiff(payloadA, hash1, payloadB, hash2));
+      } catch (e) {
+        const message = e instanceof Error ? e.message : String(e);
+        process.stderr.write(`${message}\n`);
+        process.exitCode = 1;
+      }
+    });
+}