@united-workforce/eval 0.1.0

package/dist/judge/builtin/frontmatter.js

@@ -0,0 +1,75 @@
+import { createLogger } from "@united-workforce/util";
+import { parse as parseYaml } from "yaml";
+import { EVAL_JUDGE_FRONTMATTER_SCHEMA } from "../../storage/index.js";
+import { readThreadSteps } from "./read-steps.js";
+const log = createLogger({ sink: { kind: "stderr" } });
+const LOG_RESULT = "F2QH7R4M";
+const FENCE = "---";
+/**
+ * Extract the YAML frontmatter block from a step output. Returns the inner YAML
+ * string when the output starts with a `---\n` block closed by a `\n---` fence,
+ * otherwise null.
+ */
+function extractFrontmatterYaml(output) {
+    if (typeof output !== "string") {
+        return null;
+    }
+    if (!output.startsWith(`${FENCE}\n`)) {
+        return null;
+    }
+    const rest = output.slice(FENCE.length + 1);
+    const closeIndex = rest.indexOf(`\n${FENCE}`);
+    if (closeIndex === -1) {
+        return null;
+    }
+    return rest.slice(0, closeIndex);
+}
+/** Validate a single step's frontmatter, returning a list of errors (empty = valid). */
+function validateStepFrontmatter(output) {
+    const yaml = extractFrontmatterYaml(output);
+    if (yaml === null) {
+        return ["output does not begin with a valid '---' frontmatter block"];
+    }
+    let parsed;
+    try {
+        parsed = parseYaml(yaml);
+    }
+    catch (e) {
+        const message = e instanceof Error ? e.message : String(e);
+        return [`frontmatter YAML failed to parse: ${message}`];
+    }
+    if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
+        return ["frontmatter is not a YAML mapping"];
+    }
+    const status = parsed.$status;
+    if (typeof status !== "string" || status.trim() === "") {
+        return ["$status field is missing or not a non-empty string"];
+    }
+    return [];
+}
+/**
+ * Deterministic judge: every step's agent output must contain valid YAML
+ * frontmatter with a non-empty `$status` field. Score = stepsValid / stepsTotal
+ * (0 when there are no steps).
+ */
+export async function runFrontmatterJudge(threadId) {
+    const steps = readThreadSteps(threadId);
+    const invalidSteps = [];
+    for (let i = 0; i < steps.length; i++) {
+        const step = steps[i];
+        const errors = validateStepFrontmatter(step.output);
+        if (errors.length > 0) {
+            invalidSteps.push({ stepIndex: i, role: step.role, errors });
+        }
+    }
+    const stepsTotal = steps.length;
+    const stepsValid = stepsTotal - invalidSteps.length;
+    const score = stepsTotal > 0 ? stepsValid / stepsTotal : 0;
+    log(LOG_RESULT, `frontmatter thread=${threadId} valid=${stepsValid}/${stepsTotal}`);
+    return {
+        score,
+        data: { stepsTotal, stepsValid, invalidSteps },
+        schema: EVAL_JUDGE_FRONTMATTER_SCHEMA,
+    };
+}
+//# sourceMappingURL=frontmatter.js.map

package/dist/judge/builtin/frontmatter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"frontmatter.js","sourceRoot":"","sources":["../../../src/judge/builtin/frontmatter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AACtD,OAAO,EAAE,KAAK,IAAI,SAAS,EAAE,MAAM,MAAM,CAAC;AAE1C,OAAO,EAAE,6BAA6B,EAAE,MAAM,wBAAwB,CAAC;AACvE,OAAO,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAGlD,MAAM,GAAG,GAAG,YAAY,CAAC,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,CAAC,CAAC;AAEvD,MAAM,UAAU,GAAG,UAAU,CAAC;AAE9B,MAAM,KAAK,GAAG,KAAK,CAAC;AAQpB;;;;GAIG;AACH,SAAS,sBAAsB,CAAC,MAAe;IAC7C,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;QAC/B,OAAO,IAAI,CAAC;IACd,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,GAAG,KAAK,IAAI,CAAC,EAAE,CAAC;QACrC,OAAO,IAAI,CAAC;IACd,CAAC;IACD,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAC5C,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,KAAK,EAAE,CAAC,CAAC;IAC9C,IAAI,UAAU,KAAK,CAAC,CAAC,EAAE,CAAC;QACtB,OAAO,IAAI,CAAC;IACd,CAAC;IACD,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC;AACnC,CAAC;AAED,wFAAwF;AACxF,SAAS,uBAAuB,CAAC,MAAe;IAC9C,MAAM,IAAI,GAAG,sBAAsB,CAAC,MAAM,CAAC,CAAC;IAC5C,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;QAClB,OAAO,CAAC,4DAA4D,CAAC,CAAC;IACxE,CAAC;IAED,IAAI,MAAe,CAAC;IACpB,IAAI,CAAC;QACH,MAAM,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;IAC3B,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,MAAM,OAAO,GAAG,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAC3D,OAAO,CAAC,qCAAqC,OAAO,EAAE,CAAC,CAAC;IAC1D,CAAC;IAED,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,IAAI,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QAC3E,OAAO,CAAC,mCAAmC,CAAC,CAAC;IAC/C,CAAC;IAED,MAAM,MAAM,GAAI,MAAkC,CAAC,OAAO,CAAC;IAC3D,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;QACvD,OAAO,CAAC,oDAAoD,CAAC,CAAC;IAChE,CAAC;IAED,OAAO,EAAE,CAAC;AACZ,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,QAAgB;IACxD,MAAM,KAAK,GAAG,eAAe,CAAC,QAAQ,CAAC,CAAC;IAExC,MAAM,YAAY,GAAkB,EAAE,CAAC;IACvC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QACtB,MAAM,MAAM,GAAG,uBAAuB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACpD,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtB,YAAY,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC;QAC/D,CAAC;IACH,CAAC;IAED,MAAM,UAAU,GAAG,KAAK,CAAC,MAAM,CAAC;IAChC,MAAM,UAAU,GAAG,UAAU,GAAG,YAAY,CAAC,MAAM,CAAC;IACpD,MAAM,KAAK,GAAG,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC;IAE3D,GAAG,CAAC,UAAU,EAAE,sBAAsB,QAAQ,UAAU,UAAU,IAAI,UAAU,EAAE,CAAC,CAAC;IAEpF,OAAO;QACL,KAAK;QACL,IAAI,EAAE,EAAE,UAAU,EAAE,UAAU,EAAE,YAAY,EAAE;QAC9C,MAAM,EAAE,6BAA6B;KACtC,CAAC;AACJ,CAAC"}

package/dist/judge/builtin/hallucination.d.ts

@@ -0,0 +1,10 @@
+import type { BuiltinJudgeOutput } from "./types.js";
+/**
+ * LLM-as-judge: detects claims in each step's output that are not grounded in
+ * the available context (hallucinations).
+ *
+ * TODO: LLM-as-judge — needs provider config to call LLM API. Returns a stub
+ * (score 0, empty perStep) until the LLM call path is wired up.
+ */
+export declare function runHallucinationJudge(_threadId: string): Promise<BuiltinJudgeOutput>;
+//# sourceMappingURL=hallucination.d.ts.map

package/dist/judge/builtin/hallucination.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"hallucination.d.ts","sourceRoot":"","sources":["../../../src/judge/builtin/hallucination.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AAErD;;;;;;GAMG;AACH,wBAAsB,qBAAqB,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAM1F"}

package/dist/judge/builtin/hallucination.js

@@ -0,0 +1,16 @@
+import { EVAL_JUDGE_HALLUCINATION_SCHEMA } from "../../storage/index.js";
+/**
+ * LLM-as-judge: detects claims in each step's output that are not grounded in
+ * the available context (hallucinations).
+ *
+ * TODO: LLM-as-judge — needs provider config to call LLM API. Returns a stub
+ * (score 0, empty perStep) until the LLM call path is wired up.
+ */
+export async function runHallucinationJudge(_threadId) {
+    return {
+        score: 0,
+        data: { perStep: [] },
+        schema: EVAL_JUDGE_HALLUCINATION_SCHEMA,
+    };
+}
+//# sourceMappingURL=hallucination.js.map

package/dist/judge/builtin/hallucination.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"hallucination.js","sourceRoot":"","sources":["../../../src/judge/builtin/hallucination.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,+BAA+B,EAAE,MAAM,wBAAwB,CAAC;AAGzE;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,qBAAqB,CAAC,SAAiB;IAC3D,OAAO;QACL,KAAK,EAAE,CAAC;QACR,IAAI,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE;QACrB,MAAM,EAAE,+BAA+B;KACxC,CAAC;AACJ,CAAC"}

package/dist/judge/builtin/index.d.ts

@@ -0,0 +1,7 @@
+export { runFrontmatterJudge } from "./frontmatter.js";
+export { runHallucinationJudge } from "./hallucination.js";
+export { readThreadSteps } from "./read-steps.js";
+export { runTokenStatsJudge } from "./token-stats.js";
+export type { BuiltinJudge, BuiltinJudgeOutput } from "./types.js";
+export { runUpstreamJudge } from "./upstream.js";
+//# sourceMappingURL=index.d.ts.map

package/dist/judge/builtin/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/judge/builtin/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mBAAmB,EAAE,MAAM,kBAAkB,CAAC;AACvD,OAAO,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;AAC3D,OAAO,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAClD,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AACtD,YAAY,EAAE,YAAY,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AACnE,OAAO,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC"}

package/dist/judge/builtin/index.js

@@ -0,0 +1,6 @@
+export { runFrontmatterJudge } from "./frontmatter.js";
+export { runHallucinationJudge } from "./hallucination.js";
+export { readThreadSteps } from "./read-steps.js";
+export { runTokenStatsJudge } from "./token-stats.js";
+export { runUpstreamJudge } from "./upstream.js";
+//# sourceMappingURL=index.js.map

package/dist/judge/builtin/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/judge/builtin/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mBAAmB,EAAE,MAAM,kBAAkB,CAAC;AACvD,OAAO,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;AAC3D,OAAO,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAClD,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AAEtD,OAAO,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC"}

package/dist/judge/builtin/read-steps.d.ts

@@ -0,0 +1,4 @@
+import type { StepEntry } from "@united-workforce/protocol";
+/** Shell out to `uwf step list` and return the parsed step entries (excludes start entry). */
+export declare function readThreadSteps(threadId: string): StepEntry[];
+//# sourceMappingURL=read-steps.d.ts.map

package/dist/judge/builtin/read-steps.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"read-steps.d.ts","sourceRoot":"","sources":["../../../src/judge/builtin/read-steps.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,SAAS,EAAqB,MAAM,4BAA4B,CAAC;AAE/E,8FAA8F;AAC9F,wBAAgB,eAAe,CAAC,QAAQ,EAAE,MAAM,GAAG,SAAS,EAAE,CAQ7D"}

package/dist/judge/builtin/read-steps.js

@@ -0,0 +1,12 @@
+import { execFileSync } from "node:child_process";
+/** Shell out to `uwf step list` and return the parsed step entries (excludes start entry). */
+export function readThreadSteps(threadId) {
+    const stdout = execFileSync("uwf", ["step", "list", threadId], {
+        encoding: "utf8",
+        stdio: ["ignore", "pipe", "pipe"],
+    }).trim();
+    const parsed = JSON.parse(stdout);
+    // steps[0] is the StartEntry; the rest are StepEntry records.
+    return parsed.steps.slice(1);
+}
+//# sourceMappingURL=read-steps.js.map

package/dist/judge/builtin/read-steps.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"read-steps.js","sourceRoot":"","sources":["../../../src/judge/builtin/read-steps.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAIlD,8FAA8F;AAC9F,MAAM,UAAU,eAAe,CAAC,QAAgB;IAC9C,MAAM,MAAM,GAAG,YAAY,CAAC,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE;QAC7D,QAAQ,EAAE,MAAM;QAChB,KAAK,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC;KAClC,CAAC,CAAC,IAAI,EAAE,CAAC;IACV,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAsB,CAAC;IACvD,8DAA8D;IAC9D,OAAO,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAgB,CAAC;AAC9C,CAAC"}

package/dist/judge/builtin/token-stats.d.ts

@@ -0,0 +1,8 @@
+import type { BuiltinJudgeOutput } from "./types.js";
+/**
+ * Informational judge: aggregate token usage across every step. Always scores
+ * 1.0 — it never penalizes a run, it only reports usage. Steps with null usage
+ * contribute zeros.
+ */
+export declare function runTokenStatsJudge(threadId: string): Promise<BuiltinJudgeOutput>;
+//# sourceMappingURL=token-stats.d.ts.map

package/dist/judge/builtin/token-stats.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"token-stats.d.ts","sourceRoot":"","sources":["../../../src/judge/builtin/token-stats.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AAcrD;;;;GAIG;AACH,wBAAsB,kBAAkB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,kBAAkB,CAAC,CA6BtF"}

package/dist/judge/builtin/token-stats.js

@@ -0,0 +1,35 @@
+import { createLogger } from "@united-workforce/util";
+import { EVAL_JUDGE_TOKEN_STATS_SCHEMA } from "../../storage/index.js";
+import { readThreadSteps } from "./read-steps.js";
+const log = createLogger({ sink: { kind: "stderr" } });
+const LOG_RESULT = "T7KQ3M9P";
+/**
+ * Informational judge: aggregate token usage across every step. Always scores
+ * 1.0 — it never penalizes a run, it only reports usage. Steps with null usage
+ * contribute zeros.
+ */
+export async function runTokenStatsJudge(threadId) {
+    const steps = readThreadSteps(threadId);
+    let totalInput = 0;
+    let totalOutput = 0;
+    let totalTurns = 0;
+    const perStep = [];
+    for (const step of steps) {
+        const usage = step.usage;
+        const inputTokens = usage !== null ? usage.inputTokens : 0;
+        const outputTokens = usage !== null ? usage.outputTokens : 0;
+        const turns = usage !== null ? usage.turns : 0;
+        const duration = usage !== null ? usage.duration : 0;
+        totalInput += inputTokens;
+        totalOutput += outputTokens;
+        totalTurns += turns;
+        perStep.push({ role: step.role, inputTokens, outputTokens, turns, duration });
+    }
+    log(LOG_RESULT, `token-stats thread=${threadId} in=${totalInput} out=${totalOutput}`);
+    return {
+        score: 1.0,
+        data: { totalInput, totalOutput, totalTurns, perStep },
+        schema: EVAL_JUDGE_TOKEN_STATS_SCHEMA,
+    };
+}
+//# sourceMappingURL=token-stats.js.map

package/dist/judge/builtin/token-stats.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"token-stats.js","sourceRoot":"","sources":["../../../src/judge/builtin/token-stats.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAEtD,OAAO,EAAE,6BAA6B,EAAE,MAAM,wBAAwB,CAAC;AACvE,OAAO,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAGlD,MAAM,GAAG,GAAG,YAAY,CAAC,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,CAAC,CAAC;AAEvD,MAAM,UAAU,GAAG,UAAU,CAAC;AAU9B;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,QAAgB;IACvD,MAAM,KAAK,GAAG,eAAe,CAAC,QAAQ,CAAC,CAAC;IAExC,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,MAAM,OAAO,GAAmB,EAAE,CAAC;IAEnC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;QACzB,MAAM,WAAW,GAAG,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC;QAC3D,MAAM,YAAY,GAAG,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC;QAC7D,MAAM,KAAK,GAAG,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QAC/C,MAAM,QAAQ,GAAG,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;QAErD,UAAU,IAAI,WAAW,CAAC;QAC1B,WAAW,IAAI,YAAY,CAAC;QAC5B,UAAU,IAAI,KAAK,CAAC;QAEpB,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,WAAW,EAAE,YAAY,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC;IAChF,CAAC;IAED,GAAG,CAAC,UAAU,EAAE,sBAAsB,QAAQ,OAAO,UAAU,QAAQ,WAAW,EAAE,CAAC,CAAC;IAEtF,OAAO;QACL,KAAK,EAAE,GAAG;QACV,IAAI,EAAE,EAAE,UAAU,EAAE,WAAW,EAAE,UAAU,EAAE,OAAO,EAAE;QACtD,MAAM,EAAE,6BAA6B;KACtC,CAAC;AACJ,CAAC"}

package/dist/judge/builtin/types.d.ts

@@ -0,0 +1,15 @@
+import type { JSONSchema } from "@ocas/core";
+/**
+ * Output produced by a builtin judge. Structurally identical to the runner's
+ * `JudgeRunOutput`; defined locally to keep the judge module free of a
+ * dependency on the runner module.
+ */
+export type BuiltinJudgeOutput = {
+    score: number;
+    data: unknown;
+    /** Schema describing `data`, used when persisting to CAS. */
+    schema: JSONSchema;
+};
+/** A builtin judge analyzes a thread's steps and returns a scored result. */
+export type BuiltinJudge = (threadId: string) => Promise<BuiltinJudgeOutput>;
+//# sourceMappingURL=types.d.ts.map

package/dist/judge/builtin/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/judge/builtin/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAE7C;;;;GAIG;AACH,MAAM,MAAM,kBAAkB,GAAG;IAC/B,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,OAAO,CAAC;IACd,6DAA6D;IAC7D,MAAM,EAAE,UAAU,CAAC;CACpB,CAAC;AAEF,6EAA6E;AAC7E,MAAM,MAAM,YAAY,GAAG,CAAC,QAAQ,EAAE,MAAM,KAAK,OAAO,CAAC,kBAAkB,CAAC,CAAC"}

package/dist/judge/builtin/types.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=types.js.map

package/dist/judge/builtin/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../src/judge/builtin/types.ts"],"names":[],"mappings":""}

package/dist/judge/builtin/upstream.d.ts

@@ -0,0 +1,10 @@
+import type { BuiltinJudgeOutput } from "./types.js";
+/**
+ * LLM-as-judge: measures how well each role consumed the relevant outputs from
+ * upstream steps.
+ *
+ * TODO: LLM-as-judge — needs provider config to call LLM API. Returns a stub
+ * (score 0, empty perStep) until the LLM call path is wired up.
+ */
+export declare function runUpstreamJudge(_threadId: string): Promise<BuiltinJudgeOutput>;
+//# sourceMappingURL=upstream.d.ts.map

package/dist/judge/builtin/upstream.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"upstream.d.ts","sourceRoot":"","sources":["../../../src/judge/builtin/upstream.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AAErD;;;;;;GAMG;AACH,wBAAsB,gBAAgB,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAMrF"}

package/dist/judge/builtin/upstream.js

@@ -0,0 +1,16 @@
+import { EVAL_JUDGE_UPSTREAM_SCHEMA } from "../../storage/index.js";
+/**
+ * LLM-as-judge: measures how well each role consumed the relevant outputs from
+ * upstream steps.
+ *
+ * TODO: LLM-as-judge — needs provider config to call LLM API. Returns a stub
+ * (score 0, empty perStep) until the LLM call path is wired up.
+ */
+export async function runUpstreamJudge(_threadId) {
+    return {
+        score: 0,
+        data: { perStep: [] },
+        schema: EVAL_JUDGE_UPSTREAM_SCHEMA,
+    };
+}
+//# sourceMappingURL=upstream.js.map

package/dist/judge/builtin/upstream.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"upstream.js","sourceRoot":"","sources":["../../../src/judge/builtin/upstream.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,0BAA0B,EAAE,MAAM,wBAAwB,CAAC;AAGpE;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,SAAiB;IACtD,OAAO;QACL,KAAK,EAAE,CAAC;QACR,IAAI,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE;QACrB,MAAM,EAAE,0BAA0B;KACnC,CAAC;AACJ,CAAC"}

package/dist/judge/index.d.ts

@@ -0,0 +1,3 @@
+export { type BuiltinJudge, type BuiltinJudgeOutput, readThreadSteps, runFrontmatterJudge, runHallucinationJudge, runTokenStatsJudge, runUpstreamJudge, } from "./builtin/index.js";
+export type { JudgeInput, JudgeOutput } from "./types.js";
+//# sourceMappingURL=index.d.ts.map

package/dist/judge/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/judge/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,KAAK,YAAY,EACjB,KAAK,kBAAkB,EACvB,eAAe,EACf,mBAAmB,EACnB,qBAAqB,EACrB,kBAAkB,EAClB,gBAAgB,GACjB,MAAM,oBAAoB,CAAC;AAC5B,YAAY,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC"}

package/dist/judge/index.js

@@ -0,0 +1,2 @@
+export { readThreadSteps, runFrontmatterJudge, runHallucinationJudge, runTokenStatsJudge, runUpstreamJudge, } from "./builtin/index.js";
+//# sourceMappingURL=index.js.map

package/dist/judge/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/judge/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAGL,eAAe,EACf,mBAAmB,EACnB,qBAAqB,EACrB,kBAAkB,EAClB,gBAAgB,GACjB,MAAM,oBAAoB,CAAC"}

package/dist/judge/types.d.ts

@@ -0,0 +1,15 @@
+/** Output shape every judge must produce on stdout (JSON). */
+export type JudgeOutput<T = unknown> = {
+    /** Score between 0.0 and 1.0. */
+    score: number;
+    /** Judge-specific structured data, stored in CAS with its own schema. */
+    data: T;
+};
+/** Input context passed to judge scripts via argv. */
+export type JudgeInput = {
+    /** Working directory where the task was executed. */
+    cwd: string;
+    /** Thread ID of the eval run. */
+    threadId: string;
+};
+//# sourceMappingURL=types.d.ts.map

package/dist/judge/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/judge/types.ts"],"names":[],"mappings":"AAAA,8DAA8D;AAC9D,MAAM,MAAM,WAAW,CAAC,CAAC,GAAG,OAAO,IAAI;IACrC,iCAAiC;IACjC,KAAK,EAAE,MAAM,CAAC;IACd,yEAAyE;IACzE,IAAI,EAAE,CAAC,CAAC;CACT,CAAC;AAEF,sDAAsD;AACtD,MAAM,MAAM,UAAU,GAAG;IACvB,qDAAqD;IACrD,GAAG,EAAE,MAAM,CAAC;IACZ,iCAAiC;IACjC,QAAQ,EAAE,MAAM,CAAC;CAClB,CAAC"}

package/dist/judge/types.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=types.js.map

package/dist/judge/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/judge/types.ts"],"names":[],"mappings":""}

package/dist/runner/collect.d.ts

@@ -0,0 +1,16 @@
+import type { CollectInput, CollectResult, JudgeRunner } from "./types.js";
+/**
+ * Compute the weighted overall score. Judges with weight 0 are informational
+ * and do not affect the result (they contribute 0 to both numerator and
+ * denominator). Returns 0 when total weight is 0.
+ */
+export declare function computeOverall(judges: ReadonlyArray<{
+    score: number;
+    weight: number;
+}>): number;
+/**
+ * Run all judges, store their data and the overall eval-run record in CAS, then
+ * index the run under `@uwf/eval/<task>/latest`.
+ */
+export declare function collect(input: CollectInput, runJudge?: JudgeRunner): Promise<CollectResult>;
+//# sourceMappingURL=collect.d.ts.map

package/dist/runner/collect.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"collect.d.ts","sourceRoot":"","sources":["../../src/runner/collect.ts"],"names":[],"mappings":"AAmBA,OAAO,KAAK,EACV,YAAY,EACZ,aAAa,EACb,WAAW,EAGZ,MAAM,YAAY,CAAC;AAUpB;;;;GAIG;AACH,wBAAgB,cAAc,CAAC,MAAM,EAAE,aAAa,CAAC;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC,GAAG,MAAM,CAQ/F;AAkFD;;;GAGG;AACH,wBAAsB,OAAO,CAC3B,KAAK,EAAE,YAAY,EACnB,QAAQ,GAAE,WAAgC,GACzC,OAAO,CAAC,aAAa,CAAC,CAkCxB"}

package/dist/runner/collect.js

@@ -0,0 +1,129 @@
+import { execFileSync } from "node:child_process";
+import { readFile } from "node:fs/promises";
+import { resolve } from "node:path";
+import { putSchema } from "@ocas/core";
+import { createLogger } from "@united-workforce/util";
+import { runFrontmatterJudge, runHallucinationJudge, runTokenStatsJudge, runUpstreamJudge, } from "../judge/index.js";
+import { EVAL_RUN_SCHEMA, setEvalLatest } from "../storage/index.js";
+const log = createLogger({ sink: { kind: "stderr" } });
+const LOG_JUDGE = "CT6N3P2K";
+const LOG_STORED = "CT9V2Q7M";
+/** Permissive schema for judge data without a dedicated schema (e.g. builtin placeholders). */
+const GENERIC_DATA_SCHEMA = { type: "object" };
+/**
+ * Compute the weighted overall score. Judges with weight 0 are informational
+ * and do not affect the result (they contribute 0 to both numerator and
+ * denominator). Returns 0 when total weight is 0.
+ */
+export function computeOverall(judges) {
+    let totalWeight = 0;
+    let weighted = 0;
+    for (const judge of judges) {
+        totalWeight += judge.weight;
+        weighted += judge.score * judge.weight;
+    }
+    return totalWeight > 0 ? weighted / totalWeight : 0;
+}
+/** Run a task-provided judge script: `node <entry> <cwd> <threadId>`. */
+async function runTaskJudge(taskDir, workDir, threadId, judge) {
+    if (judge.entry === null) {
+        throw new Error(`judge "${judge.name}" is not builtin but has no entry`);
+    }
+    const entryPath = resolve(taskDir, judge.entry);
+    let stdout;
+    try {
+        stdout = execFileSync("node", [entryPath, workDir, threadId], {
+            encoding: "utf8",
+            stdio: ["ignore", "pipe", "pipe"],
+            maxBuffer: 50 * 1024 * 1024,
+        });
+    }
+    catch (e) {
+        const message = e instanceof Error ? e.message : String(e);
+        throw new Error(`judge "${judge.name}" failed: ${message}`);
+    }
+    const line = stdout.trim().split("\n").pop()?.trim() ?? "";
+    let parsed;
+    try {
+        parsed = JSON.parse(line);
+    }
+    catch {
+        throw new Error(`judge "${judge.name}" stdout is not valid JSON: ${line || "(empty)"}`);
+    }
+    const output = parsed;
+    if (typeof output.score !== "number") {
+        throw new Error(`judge "${judge.name}" output missing numeric score`);
+    }
+    const schema = judge.schema !== null ? await loadSchema(resolve(taskDir, judge.schema)) : GENERIC_DATA_SCHEMA;
+    return { score: output.score, data: output.data, schema };
+}
+/** Load and parse an OCAS JSON Schema file. */
+async function loadSchema(path) {
+    const text = await readFile(path, "utf8");
+    return JSON.parse(text);
+}
+/** Dispatch a builtin judge by name. Throws on an unknown builtin name. */
+async function runBuiltinJudge(name, threadId) {
+    switch (name) {
+        case "frontmatter-compliance":
+            return runFrontmatterJudge(threadId);
+        case "upstream-consumption":
+            return runUpstreamJudge(threadId);
+        case "hallucination":
+            return runHallucinationJudge(threadId);
+        case "token-stats":
+            return runTokenStatsJudge(threadId);
+        default:
+            throw new Error(`unknown builtin judge "${name}"`);
+    }
+}
+/**
+ * Default judge runner. Builtin judges are dispatched by name; task judges spawn
+ * their entry script.
+ */
+const defaultJudgeRunner = async (taskDir, workDir, threadId, judge) => {
+    if (judge.builtin) {
+        return runBuiltinJudge(judge.name, threadId);
+    }
+    return runTaskJudge(taskDir, workDir, threadId, judge);
+};
+/** Persist judge data to CAS under its schema and return the CAS hash. */
+async function storeJudgeData(store, schema, data) {
+    const schemaHash = await putSchema(store, schema);
+    return (await store.cas.put(schemaHash, data));
+}
+/**
+ * Run all judges, store their data and the overall eval-run record in CAS, then
+ * index the run under `@uwf/eval/<task>/latest`.
+ */
+export async function collect(input, runJudge = defaultJudgeRunner) {
+    const { evalStore, taskDir, workDir, threadId, manifest, config } = input;
+    const { store, varStore } = evalStore;
+    const records = [];
+    for (const judge of manifest.judges) {
+        const result = await runJudge(taskDir, workDir, threadId, judge);
+        const dataHash = await storeJudgeData(store, result.schema, result.data);
+        records.push({ name: judge.name, score: result.score, weight: judge.weight, dataHash });
+        log(LOG_JUDGE, `judge=${judge.name} score=${result.score} weight=${judge.weight}`);
+    }
+    const overall = computeOverall(records);
+    const payload = {
+        task: manifest.name,
+        config,
+        threadId,
+        judges: records,
+        overall,
+        timestamp: Date.now(),
+    };
+    const schemaHash = await putSchema(store, EVAL_RUN_SCHEMA);
+    const runHash = (await store.cas.put(schemaHash, payload));
+    setEvalLatest(varStore, manifest.name, runHash);
+    log(LOG_STORED, `stored eval-run task=${manifest.name} hash=${runHash} overall=${overall}`);
+    const judges = records.map((r) => ({
+        name: r.name,
+        score: r.score,
+        weight: r.weight,
+    }));
+    return { runHash, overall, judges };
+}
+//# sourceMappingURL=collect.js.map

package/dist/runner/collect.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"collect.js","sourceRoot":"","sources":["../../src/runner/collect.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAGpC,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAEvC,OAAO,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAGtD,OAAO,EACL,mBAAmB,EACnB,qBAAqB,EACrB,kBAAkB,EAClB,gBAAgB,GACjB,MAAM,mBAAmB,CAAC;AAE3B,OAAO,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAUrE,MAAM,GAAG,GAAG,YAAY,CAAC,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,CAAC,CAAC;AAEvD,MAAM,SAAS,GAAG,UAAU,CAAC;AAC7B,MAAM,UAAU,GAAG,UAAU,CAAC;AAE9B,+FAA+F;AAC/F,MAAM,mBAAmB,GAAe,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;AAE3D;;;;GAIG;AACH,MAAM,UAAU,cAAc,CAAC,MAAwD;IACrF,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,WAAW,IAAI,KAAK,CAAC,MAAM,CAAC;QAC5B,QAAQ,IAAI,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC;IACzC,CAAC;IACD,OAAO,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC;AACtD,CAAC;AAED,yEAAyE;AACzE,KAAK,UAAU,YAAY,CACzB,OAAe,EACf,OAAe,EACf,QAAgB,EAChB,KAAiB;IAEjB,IAAI,KAAK,CAAC,KAAK,KAAK,IAAI,EAAE,CAAC;QACzB,MAAM,IAAI,KAAK,CAAC,UAAU,KAAK,CAAC,IAAI,mCAAmC,CAAC,CAAC;IAC3E,CAAC;IACD,MAAM,SAAS,GAAG,OAAO,CAAC,OAAO,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC;IAEhD,IAAI,MAAc,CAAC;IACnB,IAAI,CAAC;QACH,MAAM,GAAG,YAAY,CAAC,MAAM,EAAE,CAAC,SAAS,EAAE,OAAO,EAAE,QAAQ,CAAC,EAAE;YAC5D,QAAQ,EAAE,MAAM;YAChB,KAAK,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC;YACjC,SAAS,EAAE,EAAE,GAAG,IAAI,GAAG,IAAI;SAC5B,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,MAAM,OAAO,GAAG,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAC3D,MAAM,IAAI,KAAK,CAAC,UAAU,KAAK,CAAC,IAAI,aAAa,OAAO,EAAE,CAAC,CAAC;IAC9D,CAAC;IAED,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;IAC3D,IAAI,MAAe,CAAC;IACpB,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC5B,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,KAAK,CAAC,UAAU,KAAK,CAAC,IAAI,+BAA+B,IAAI,IAAI,SAAS,EAAE,CAAC,CAAC;IAC1F,CAAC;IACD,MAAM,MAAM,GAAG,MAAqB,CAAC;IACrC,IAAI,OAAO,MAAM,CAAC,KAAK,KAAK,QAAQ,EAAE,CAAC;QACrC,MAAM,IAAI,KAAK,CAAC,UAAU,KAAK,CAAC,IAAI,gCAAgC,CAAC,CAAC;IACxE,CAAC;IAED,MAAM,MAAM,GACV,KAAK,CAAC,MAAM,KAAK,IAAI,CAAC,CAAC,CAAC,MAAM,UAAU,CAAC,OAAO,CAAC,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,mBAAmB,CAAC;IACjG,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC;AAC5D,CAAC;AAED,+CAA+C;AAC/C,KAAK,UAAU,UAAU,CAAC,IAAY;IACpC,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAC1C,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAe,CAAC;AACxC,CAAC;AAED,2EAA2E;AAC3E,KAAK,UAAU,eAAe,CAAC,IAAY,EAAE,QAAgB;IAC3D,QAAQ,IAAI,EAAE,CAAC;QACb,KAAK,wBAAwB;YAC3B,OAAO,mBAAmB,CAAC,QAAQ,CAAC,CAAC;QACvC,KAAK,sBAAsB;YACzB,OAAO,gBAAgB,CAAC,QAAQ,CAAC,CAAC;QACpC,KAAK,eAAe;YAClB,OAAO,qBAAqB,CAAC,QAAQ,CAAC,CAAC;QACzC,KAAK,aAAa;YAChB,OAAO,kBAAkB,CAAC,QAAQ,CAAC,CAAC;QACtC;YACE,MAAM,IAAI,KAAK,CAAC,0BAA0B,IAAI,GAAG,CAAC,CAAC;IACvD,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,MAAM,kBAAkB,GAAgB,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,EAAE;IAClF,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC;QAClB,OAAO,eAAe,CAAC,KAAK,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IAC/C,CAAC;IACD,OAAO,YAAY,CAAC,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC;AACzD,CAAC,CAAC;AAEF,0EAA0E;AAC1E,KAAK,UAAU,cAAc,CAAC,KAAY,EAAE,MAAkB,EAAE,IAAa;IAC3E,MAAM,UAAU,GAAG,MAAM,SAAS,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IAClD,OAAO,CAAC,MAAM,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,UAAU,EAAE,IAAI,CAAC,CAAW,CAAC;AAC3D,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,OAAO,CAC3B,KAAmB,EACnB,WAAwB,kBAAkB;IAE1C,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,KAAK,CAAC;IAC1E,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,GAAG,SAAS,CAAC;IAEtC,MAAM,OAAO,GAAsB,EAAE,CAAC;IACtC,KAAK,MAAM,KAAK,IAAI,QAAQ,CAAC,MAAM,EAAE,CAAC;QACpC,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC;QACjE,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC;QACzE,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC;QACxF,GAAG,CAAC,SAAS,EAAE,SAAS,KAAK,CAAC,IAAI,UAAU,MAAM,CAAC,KAAK,WAAW,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;IACrF,CAAC;IAED,MAAM,OAAO,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;IAExC,MAAM,OAAO,GAAmB;QAC9B,IAAI,EAAE,QAAQ,CAAC,IAAI;QACnB,MAAM;QACN,QAAQ;QACR,MAAM,EAAE,OAAO;QACf,OAAO;QACP,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;KACtB,CAAC;IAEF,MAAM,UAAU,GAAG,MAAM,SAAS,CAAC,KAAK,EAAE,eAAe,CAAC,CAAC;IAC3D,MAAM,OAAO,GAAG,CAAC,MAAM,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,UAAU,EAAE,OAAO,CAAC,CAAW,CAAC;IACrE,aAAa,CAAC,QAAQ,EAAE,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAChD,GAAG,CAAC,UAAU,EAAE,wBAAwB,QAAQ,CAAC,IAAI,SAAS,OAAO,YAAY,OAAO,EAAE,CAAC,CAAC;IAE5F,MAAM,MAAM,GAAmB,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACjD,IAAI,EAAE,CAAC,CAAC,IAAI;QACZ,KAAK,EAAE,CAAC,CAAC,KAAK;QACd,MAAM,EAAE,CAAC,CAAC,MAAM;KACjB,CAAC,CAAC,CAAC;IACJ,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;AACtC,CAAC"}

package/dist/runner/execute.d.ts

@@ -0,0 +1,9 @@
+import type { ExecuteInput, ExecuteResult } from "./types.js";
+/**
+ * Execute a workflow: create a thread, then run it for up to `maxSteps` steps.
+ * Shells out to the uwf CLI rather than importing it directly.
+ */
+export declare function execute(input: ExecuteInput): Promise<ExecuteResult>;
+/** Best-effort lookup of the uwf engine version (`uwf -V`); "unknown" on failure. */
+export declare function getEngineVersion(): string;
+//# sourceMappingURL=execute.d.ts.map

package/dist/runner/execute.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"execute.d.ts","sourceRoot":"","sources":["../../src/runner/execute.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAmD9D;;;GAGG;AACH,wBAAsB,OAAO,CAAC,KAAK,EAAE,YAAY,GAAG,OAAO,CAAC,aAAa,CAAC,CAezE;AAED,qFAAqF;AACrF,wBAAgB,gBAAgB,IAAI,MAAM,CASzC"}

package/dist/runner/execute.js

@@ -0,0 +1,72 @@
+import { execFileSync } from "node:child_process";
+import { createLogger } from "@united-workforce/util";
+const log = createLogger({ sink: { kind: "stderr" } });
+const LOG_START = "EX5M2T9V";
+const LOG_EXEC = "EX7Q4K2N";
+/** Resolve the uwf CLI binary. Override with `UWF_BIN` for testing. */
+function uwfBin() {
+    const override = process.env.UWF_BIN;
+    return override !== undefined && override !== "" ? override : "uwf";
+}
+/** Run a uwf subcommand and return trimmed stdout. */
+function runUwf(args, cwd) {
+    try {
+        return execFileSync(uwfBin(), args, {
+            encoding: "utf8",
+            stdio: ["ignore", "pipe", "pipe"],
+            maxBuffer: 50 * 1024 * 1024,
+            cwd,
+        }).trim();
+    }
+    catch (e) {
+        const err = e;
+        const stderr = err.stderr == null
+            ? ""
+            : typeof err.stderr === "string"
+                ? err.stderr
+                : err.stderr.toString("utf8");
+        const detail = stderr.trim() !== "" ? `: ${stderr.trim()}` : "";
+        throw new Error(`uwf ${args[0]} ${args[1]} failed${detail}`);
+    }
+}
+/** Parse the thread ID from `uwf thread start` JSON output (`{ workflow, thread }`). */
+function parseThreadId(stdout) {
+    let parsed;
+    try {
+        parsed = JSON.parse(stdout);
+    }
+    catch {
+        throw new Error(`uwf thread start did not emit valid JSON: ${stdout || "(empty)"}`);
+    }
+    const obj = parsed;
+    const thread = obj.thread;
+    if (typeof thread !== "string" || thread === "") {
+        throw new Error(`uwf thread start output missing thread id: ${stdout}`);
+    }
+    return thread;
+}
+/**
+ * Execute a workflow: create a thread, then run it for up to `maxSteps` steps.
+ * Shells out to the uwf CLI rather than importing it directly.
+ */
+export async function execute(input) {
+    const startOut = runUwf(["thread", "start", input.workflow, "-p", input.prompt, "--cwd", input.workDir], input.workDir);
+    const threadId = parseThreadId(startOut);
+    log(LOG_START, `thread started thread=${threadId} workflow=${input.workflow}`);
+    runUwf(["thread", "exec", threadId, "--agent", input.agent, "-c", String(input.maxSteps)], input.workDir);
+    log(LOG_EXEC, `thread executed thread=${threadId} maxSteps=${input.maxSteps}`);
+    return { threadId };
+}
+/** Best-effort lookup of the uwf engine version (`uwf -V`); "unknown" on failure. */
+export function getEngineVersion() {
+    try {
+        return execFileSync(uwfBin(), ["-V"], {
+            encoding: "utf8",
+            stdio: ["ignore", "pipe", "ignore"],
+        }).trim();
+    }
+    catch {
+        return "unknown";
+    }
+}
+//# sourceMappingURL=execute.js.map

package/dist/runner/execute.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"execute.js","sourceRoot":"","sources":["../../src/runner/execute.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAElD,OAAO,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAItD,MAAM,GAAG,GAAG,YAAY,CAAC,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,CAAC,CAAC;AAEvD,MAAM,SAAS,GAAG,UAAU,CAAC;AAC7B,MAAM,QAAQ,GAAG,UAAU,CAAC;AAE5B,uEAAuE;AACvE,SAAS,MAAM;IACb,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC;IACrC,OAAO,QAAQ,KAAK,SAAS,IAAI,QAAQ,KAAK,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC;AACtE,CAAC;AAED,sDAAsD;AACtD,SAAS,MAAM,CAAC,IAAc,EAAE,GAAW;IACzC,IAAI,CAAC;QACH,OAAO,YAAY,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE;YAClC,QAAQ,EAAE,MAAM;YAChB,KAAK,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC;YACjC,SAAS,EAAE,EAAE,GAAG,IAAI,GAAG,IAAI;YAC3B,GAAG;SACJ,CAAC,CAAC,IAAI,EAAE,CAAC;IACZ,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,MAAM,GAAG,GAAG,CAAgE,CAAC;QAC7E,MAAM,MAAM,GACV,GAAG,CAAC,MAAM,IAAI,IAAI;YAChB,CAAC,CAAC,EAAE;YACJ,CAAC,CAAC,OAAO,GAAG,CAAC,MAAM,KAAK,QAAQ;gBAC9B,CAAC,CAAC,GAAG,CAAC,MAAM;gBACZ,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACpC,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAChE,MAAM,IAAI,KAAK,CAAC,OAAO,IAAI,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,UAAU,MAAM,EAAE,CAAC,CAAC;IAC/D,CAAC;AACH,CAAC;AAED,wFAAwF;AACxF,SAAS,aAAa,CAAC,MAAc;IACnC,IAAI,MAAe,CAAC;IACpB,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IAC9B,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,KAAK,CAAC,6CAA6C,MAAM,IAAI,SAAS,EAAE,CAAC,CAAC;IACtF,CAAC;IACD,MAAM,GAAG,GAAG,MAAiC,CAAC;IAC9C,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC;IAC1B,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,EAAE,EAAE,CAAC;QAChD,MAAM,IAAI,KAAK,CAAC,8CAA8C,MAAM,EAAE,CAAC,CAAC;IAC1E,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,KAAmB;IAC/C,MAAM,QAAQ,GAAG,MAAM,CACrB,CAAC,QAAQ,EAAE,OAAO,EAAE,KAAK,CAAC,QAAQ,EAAE,IAAI,EAAE,KAAK,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,EAC/E,KAAK,CAAC,OAAO,CACd,CAAC;IACF,MAAM,QAAQ,GAAG,aAAa,CAAC,QAAQ,CAAC,CAAC;IACzC,GAAG,CAAC,SAAS,EAAE,yBAAyB,QAAQ,aAAa,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;IAE/E,MAAM,CACJ,CAAC,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,KAAK,CAAC,KAAK,EAAE,IAAI,EAAE,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,EAClF,KAAK,CAAC,OAAO,CACd,CAAC;IACF,GAAG,CAAC,QAAQ,EAAE,0BAA0B,QAAQ,aAAa,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;IAE/E,OAAO,EAAE,QAAQ,EAAE,CAAC;AACtB,CAAC;AAED,qFAAqF;AACrF,MAAM,UAAU,gBAAgB;IAC9B,IAAI,CAAC;QACH,OAAO,YAAY,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,CAAC,EAAE;YACpC,QAAQ,EAAE,MAAM;YAChB,KAAK,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,QAAQ,CAAC;SACpC,CAAC,CAAC,IAAI,EAAE,CAAC;IACZ,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC"}

package/dist/runner/index.d.ts

@@ -0,0 +1,5 @@
+export { collect, computeOverall } from "./collect.js";
+export { execute, getEngineVersion } from "./execute.js";
+export { prepare } from "./prepare.js";
+export type { CollectInput, CollectResult, ExecuteInput, ExecuteResult, JudgeRunner, JudgeRunOutput, JudgeSummary, PrepareResult, RunOptions, RunResult, } from "./types.js";
+//# sourceMappingURL=index.d.ts.map

package/dist/runner/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/runner/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AACvD,OAAO,EAAE,OAAO,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AACzD,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,YAAY,EACV,YAAY,EACZ,aAAa,EACb,YAAY,EACZ,aAAa,EACb,WAAW,EACX,cAAc,EACd,YAAY,EACZ,aAAa,EACb,UAAU,EACV,SAAS,GACV,MAAM,YAAY,CAAC"}

package/dist/runner/index.js

@@ -0,0 +1,4 @@
+export { collect, computeOverall } from "./collect.js";
+export { execute, getEngineVersion } from "./execute.js";
+export { prepare } from "./prepare.js";
+//# sourceMappingURL=index.js.map

package/dist/runner/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/runner/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AACvD,OAAO,EAAE,OAAO,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AACzD,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC"}

package/dist/runner/prepare.d.ts

@@ -0,0 +1,7 @@
+import type { PrepareResult } from "./types.js";
+/**
+ * Prepare a task for execution: read its manifest and copy the fixture
+ * directory into a fresh temp working directory.
+ */
+export declare function prepare(taskDir: string): Promise<PrepareResult>;
+//# sourceMappingURL=prepare.d.ts.map