agentic-engineering-harness 0.4.16 → 0.5.1

package/dist/distributed/queue.js

@@ -0,0 +1,140 @@
+import crypto from "node:crypto";
+import fs from "node:fs/promises";
+import http from "node:http";
+import path from "node:path";
+export async function submitDistributedJob(root, config, job) {
+    if (config.distributed?.provider === "http")
+        return submitHttp(config, job);
+    const dirs = await queueDirs(root, config);
+    await fs.writeFile(path.join(dirs.pending, `${safe(job.id)}.json`), `${JSON.stringify(job, null, 2)}\n`, { flag: "wx" });
+}
+export async function claimDistributedJob(root, config, workerId) {
+    if (config.distributed?.provider === "http")
+        return claimHttp(config, workerId);
+    const dirs = await queueDirs(root, config);
+    await reclaimExpired(dirs);
+    const files = (await fs.readdir(dirs.pending)).filter((name) => name.endsWith(".json")).sort();
+    for (const file of files) {
+        const source = path.join(dirs.pending, file);
+        const leaseId = `${workerId}-${crypto.randomUUID()}`;
+        const destination = path.join(dirs.leased, `${safe(leaseId)}.json`);
+        try {
+            const raw = await fs.readFile(source, "utf8");
+            const job = JSON.parse(raw);
+            const now = Date.now();
+            const envelope = { leaseId, leasedAt: new Date(now).toISOString(), expiresAt: new Date(now + (config.distributed?.leaseSeconds ?? 1800) * 1000).toISOString(), job };
+            await fs.rename(source, destination);
+            await fs.writeFile(destination, `${JSON.stringify(envelope, null, 2)}\n`);
+            return { job, leaseId };
+        }
+        catch (error) {
+            if (error?.code === "ENOENT")
+                continue;
+            throw error;
+        }
+    }
+    return undefined;
+}
+export async function completeDistributedJob(root, config, leaseId, result) {
+    if (config.distributed?.provider === "http")
+        return completeHttp(config, leaseId, result);
+    const dirs = await queueDirs(root, config);
+    const leaseFile = path.join(dirs.leased, `${safe(leaseId)}.json`);
+    await fs.writeFile(path.join(dirs.completed, `${safe(result.jobId)}.json`), `${JSON.stringify(result, null, 2)}\n`);
+    await fs.rm(leaseFile, { force: true });
+}
+export async function waitForDistributedResult(root, config, jobId, timeoutMs) {
+    const timeout = timeoutMs ?? (config.orchestration?.worker?.timeoutSeconds ?? 1800) * 1000;
+    const deadline = Date.now() + timeout;
+    const interval = config.distributed?.pollIntervalMs ?? 1000;
+    while (Date.now() < deadline) {
+        const result = config.distributed?.provider === "http" ? await getHttpResult(config, jobId) : await getFilesystemResult(root, config, jobId);
+        if (result)
+            return result;
+        await new Promise((resolve) => setTimeout(resolve, interval));
+    }
+    throw new Error(`Distributed job ${jobId} timed out after ${timeout}ms.`);
+}
+export async function serveDistributedQueue(root, config, options) {
+    if (config.distributed?.provider === "http")
+        throw new Error("Queue server must use its local filesystem backing store; configure provider=filesystem on the coordinator server.");
+    const token = config.distributed?.tokenEnv ? process.env[config.distributed.tokenEnv] : undefined;
+    const server = http.createServer(async (request, response) => {
+        try {
+            if (token && request.headers.authorization !== `Bearer ${token}`) {
+                json(response, 401, { error: "unauthorized" });
+                return;
+            }
+            const url = new URL(request.url ?? "/", `http://${request.headers.host ?? "localhost"}`);
+            if (request.method === "POST" && url.pathname === "/v1/jobs") {
+                const job = await readJsonBody(request);
+                await submitDistributedJob(root, config, job);
+                json(response, 202, { id: job.id });
+                return;
+            }
+            if (request.method === "POST" && url.pathname === "/v1/claim") {
+                const body = await readJsonBody(request);
+                const claimed = await claimDistributedJob(root, config, body.workerId);
+                json(response, claimed ? 200 : 204, claimed);
+                return;
+            }
+            const complete = url.pathname.match(/^\/v1\/leases\/([^/]+)\/complete$/);
+            if (request.method === "POST" && complete) {
+                const result = await readJsonBody(request);
+                await completeDistributedJob(root, config, decodeURIComponent(complete[1]), result);
+                json(response, 200, { ok: true });
+                return;
+            }
+            const resultMatch = url.pathname.match(/^\/v1\/jobs\/([^/]+)$/);
+            if (request.method === "GET" && resultMatch) {
+                const result = await getFilesystemResult(root, config, decodeURIComponent(resultMatch[1]));
+                json(response, result ? 200 : 404, result ?? { error: "not-ready" });
+                return;
+            }
+            json(response, 404, { error: "not-found" });
+        }
+        catch (error) {
+            json(response, 500, { error: String(error) });
+        }
+    });
+    await new Promise((resolve, reject) => { server.once("error", reject); server.listen(options.port, options.host ?? "127.0.0.1", () => resolve()); });
+    return server;
+}
+async function getFilesystemResult(root, config, jobId) { const dirs = await queueDirs(root, config); try {
+    return JSON.parse(await fs.readFile(path.join(dirs.completed, `${safe(jobId)}.json`), "utf8"));
+}
+catch {
+    return undefined;
+} }
+async function queueDirs(root, config) { const base = path.resolve(root, config.distributed?.queueDir ?? ".harness/distributed"); const dirs = { root: base, pending: path.join(base, "pending"), leased: path.join(base, "leased"), completed: path.join(base, "completed") }; await Promise.all([dirs.pending, dirs.leased, dirs.completed].map((dir) => fs.mkdir(dir, { recursive: true }))); return dirs; }
+async function reclaimExpired(dirs) { const files = (await fs.readdir(dirs.leased)).filter((name) => name.endsWith(".json")); for (const file of files) {
+    try {
+        const envelope = JSON.parse(await fs.readFile(path.join(dirs.leased, file), "utf8"));
+        if (Date.parse(envelope.expiresAt) <= Date.now()) {
+            await fs.writeFile(path.join(dirs.pending, `${safe(envelope.job.id)}.json`), `${JSON.stringify(envelope.job, null, 2)}\n`, { flag: "wx" }).catch(() => undefined);
+            await fs.rm(path.join(dirs.leased, file), { force: true });
+        }
+    }
+    catch { /* malformed lease stays visible for operator inspection */ }
+} }
+async function submitHttp(config, job) { const response = await request(config, "/v1/jobs", { method: "POST", body: JSON.stringify(job), headers: { "content-type": "application/json" } }); if (!response.ok)
+    throw new Error(`Distributed queue rejected job: HTTP ${response.status} ${await response.text()}`); }
+async function claimHttp(config, workerId) { const response = await request(config, "/v1/claim", { method: "POST", body: JSON.stringify({ workerId }), headers: { "content-type": "application/json" } }); if (response.status === 204)
+    return undefined; if (!response.ok)
+    throw new Error(`Distributed queue claim failed: HTTP ${response.status}`); return await response.json(); }
+async function completeHttp(config, leaseId, result) { const response = await request(config, `/v1/leases/${encodeURIComponent(leaseId)}/complete`, { method: "POST", body: JSON.stringify(result), headers: { "content-type": "application/json" } }); if (!response.ok)
+    throw new Error(`Distributed queue completion failed: HTTP ${response.status}`); }
+async function getHttpResult(config, jobId) { const response = await request(config, `/v1/jobs/${encodeURIComponent(jobId)}`); if (response.status === 404)
+    return undefined; if (!response.ok)
+    throw new Error(`Distributed queue result lookup failed: HTTP ${response.status}`); return await response.json(); }
+async function request(config, pathname, init = {}) { const endpoint = config.distributed?.endpoint; if (!endpoint)
+    throw new Error("distributed.endpoint is required for HTTP distributed queues."); const headers = new Headers(init.headers); const token = config.distributed?.tokenEnv ? process.env[config.distributed.tokenEnv] : undefined; if (token)
+    headers.set("authorization", `Bearer ${token}`); return fetch(new URL(pathname, endpoint).toString(), { ...init, headers }); }
+async function readJsonBody(request) { const chunks = []; for await (const chunk of request)
+    chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); return JSON.parse(Buffer.concat(chunks).toString("utf8")); }
+function json(response, status, value) { response.statusCode = status; if (value === undefined) {
+    response.end();
+    return;
+} response.setHeader("content-type", "application/json"); response.end(JSON.stringify(value)); }
+function safe(value) { return value.replace(/[^A-Za-z0-9._-]/g, "-"); }
+//# sourceMappingURL=queue.js.map

package/dist/distributed/queue.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"queue.js","sourceRoot":"","sources":["../../src/distributed/queue.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,aAAa,CAAC;AACjC,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAClC,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,IAAI,MAAM,WAAW,CAAC;AAM7B,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAC,IAAY,EAAE,MAA4B,EAAE,GAA6B;IAClH,IAAI,MAAM,CAAC,WAAW,EAAE,QAAQ,KAAK,MAAM;QAAE,OAAO,UAAU,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC5E,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAAC,MAAM,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,OAAO,CAAC,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;AACvK,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,IAAY,EAAE,MAA4B,EAAE,QAAgB;IACpG,IAAI,MAAM,CAAC,WAAW,EAAE,QAAQ,KAAK,MAAM;QAAE,OAAO,SAAS,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;IAChF,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAAC,MAAM,cAAc,CAAC,IAAI,CAAC,CAAC;IACvE,MAAM,KAAK,GAAG,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAC/F,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;QAAC,MAAM,OAAO,GAAG,GAAG,QAAQ,IAAI,MAAM,CAAC,UAAU,EAAE,EAAE,CAAC;QAAC,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QACxK,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;YAAC,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAA6B,CAAC;YAAC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAAC,MAAM,QAAQ,GAAkB,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,EAAE,SAAS,EAAE,IAAI,IAAI,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,WAAW,EAAE,YAAY,IAAI,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC,WAAW,EAAE,EAAE,GAAG,EAAE,CAAC;YACpT,MAAM,EAAE,CAAC,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;YAAC,MAAM,EAAE,CAAC,SAAS,CAAC,WAAW,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;YAAC,OAAO,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC;QAC3I,CAAC;QAAC,OAAO,KAAU,EAAE,CAAC;YAAC,IAAI,KAAK,EAAE,IAAI,KAAK,QAAQ;gBAAE,SAAS;YAAC,MAAM,KAAK,CAAC;QAAC,CAAC;IAC/E,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAAC,IAAY,EAAE,MAA4B,EAAE,OAAe,EAAE,MAAmC;IAC3I,IAAI,MAAM,CAAC,WAAW,EAAE,QAAQ,KAAK,MAAM;QAAE,OAAO,YAAY,CAAC,MAAM,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;IAC1F,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAAC,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IAC9G,MAAM,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;IAAC,MAAM,EAAE,CAAC,EAAE,CAAC,SAAS,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;AAC/J,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAAC,IAAY,EAAE,MAA4B,EAAE,KAAa,EAAE,SAAkB;IAC1H,MAAM,OAAO,GAAG,SAAS,IAAI,CAAC,MAAM,CAAC,aAAa,EAAE,MAAM,EAAE,cAAc,IAAI,IAAI,CAAC,GAAG,IAAI,CAAC;IAAC,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC;IAAC,MAAM,QAAQ,GAAG,MAAM,CAAC,WAAW,EAAE,cAAc,IAAI,IAAI,CAAC;IAC/L,OAAO,IAAI,CAAC,GAAG,EAAE,GAAG,QAAQ,EAAE,CAAC;QAC7B,MAAM,MAAM,GAAG,MAAM,CAAC,WAAW,EAAE,QAAQ,KAAK,MAAM,CAAC,CAAC,CAAC,MAAM,aAAa,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,mBAAmB,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC;QAC7I,IAAI,MAAM;YAAE,OAAO,MAAM,CAAC;QAC1B,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC;IAChE,CAAC;IACD,MAAM,IAAI,KAAK,CAAC,mBAAmB,KAAK,oBAAoB,OAAO,KAAK,CAAC,CAAC;AAC5E,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,qBAAqB,CAAC,IAAY,EAAE,MAA4B,EAAE,OAAwC;IAC9H,IAAI,MAAM,CAAC,WAAW,EAAE,QAAQ,KAAK,MAAM;QAAE,MAAM,IAAI,KAAK,CAAC,oHAAoH,CAAC,CAAC;IACnL,MAAM,KAAK,GAAG,MAAM,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAClG,MAAM,MAAM,GAAG,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE;QAC3D,IAAI,CAAC;YACH,IAAI,KAAK,IAAI,OAAO,CAAC,OAAO,CAAC,aAAa,KAAK,UAAU,KAAK,EAAE,EAAE,CAAC;gBAAC,IAAI,CAAC,QAAQ,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,cAAc,EAAE,CAAC,CAAC;gBAAC,OAAO;YAAC,CAAC;YAC7H,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,IAAI,GAAG,EAAE,UAAU,OAAO,CAAC,OAAO,CAAC,IAAI,IAAI,WAAW,EAAE,CAAC,CAAC;YACzF,IAAI,OAAO,CAAC,MAAM,KAAK,MAAM,IAAI,GAAG,CAAC,QAAQ,KAAK,UAAU,EAAE,CAAC;gBAAC,MAAM,GAAG,GAAG,MAAM,YAAY,CAA2B,OAAO,CAAC,CAAC;gBAAC,MAAM,oBAAoB,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,CAAC,CAAC;gBAAC,IAAI,CAAC,QAAQ,EAAE,GAAG,EAAE,EAAE,EAAE,EAAE,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC;gBAAC,OAAO;YAAC,CAAC;YAChO,IAAI,OAAO,CAAC,MAAM,KAAK,MAAM,IAAI,GAAG,CAAC,QAAQ,KAAK,WAAW,EAAE,CAAC;gBAAC,MAAM,IAAI,GAAG,MAAM,YAAY,CAAuB,OAAO,CAAC,CAAC;gBAAC,MAAM,OAAO,GAAG,MAAM,mBAAmB,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;gBAAC,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;gBAAC,OAAO;YAAC,CAAC;YAChQ,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,mCAAmC,CAAC,CAAC;YAAC,IAAI,OAAO,CAAC,MAAM,KAAK,MAAM,IAAI,QAAQ,EAAE,CAAC;gBAAC,MAAM,MAAM,GAAG,MAAM,YAAY,CAA8B,OAAO,CAAC,CAAC;gBAAC,MAAM,sBAAsB,CAAC,IAAI,EAAE,MAAM,EAAE,kBAAkB,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;gBAAC,IAAI,CAAC,QAAQ,EAAE,GAAG,EAAE,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC;gBAAC,OAAO;YAAC,CAAC;YACjU,MAAM,WAAW,GAAG,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,uBAAuB,CAAC,CAAC;YAAC,IAAI,OAAO,CAAC,MAAM,KAAK,KAAK,IAAI,WAAW,EAAE,CAAC;gBAAC,MAAM,MAAM,GAAG,MAAM,mBAAmB,CAAC,IAAI,EAAE,MAAM,EAAE,kBAAkB,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBAAC,IAAI,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,MAAM,IAAI,EAAE,KAAK,EAAE,WAAW,EAAE,CAAC,CAAC;gBAAC,OAAO;YAAC,CAAC;YAC3R,IAAI,CAAC,QAAQ,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,WAAW,EAAE,CAAC,CAAC;QAC9C,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAAC,IAAI,CAAC,QAAQ,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAAC,CAAC;IACpE,CAAC,CAAC,CAAC;IACH,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,IAAI,IAAI,WAAW,EAAE,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAAC,OAAO,MAAM,CAAC;AAC5K,CAAC;AAED,KAAK,UAAU,mBAAmB,CAAC,IAAY,EAAE,MAA4B,EAAE,KAAa,IAAsD,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC;IAAC,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC,CAAgC,CAAC;AAAC,CAAC;AAAC,MAAM,CAAC;IAAC,OAAO,SAAS,CAAC;AAAC,CAAC,CAAC,CAAC;AAClW,KAAK,UAAU,SAAS,CAAC,IAAY,EAAE,MAA4B,IAAmF,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC,WAAW,EAAE,QAAQ,IAAI,sBAAsB,CAAC,CAAC,CAAC,MAAM,IAAI,GAAG,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,SAAS,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,QAAQ,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,WAAW,CAAC,EAAE,CAAC,CAAC,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,CAAC;AAC5f,KAAK,UAAU,cAAc,CAAC,IAA2C,IAAmB,MAAM,KAAK,GAAG,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;IAAC,IAAI,CAAC;QAAC,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,CAAkB,CAAC;QAAC,IAAI,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;YAAC,MAAM,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,OAAO,CAAC,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;YAAC,MAAM,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QAAC,CAAC;IAAC,CAAC;IAAC,MAAM,CAAC,CAAC,2DAA2D,CAAC,CAAC;AAAC,CAAC,CAAC,CAAC;AAE7pB,KAAK,UAAU,UAAU,CAAC,MAA4B,EAAE,GAA6B,IAAmB,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,MAAM,EAAE,UAAU,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,OAAO,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE;IAAE,MAAM,IAAI,KAAK,CAAC,wCAAwC,QAAQ,CAAC,MAAM,IAAI,MAAM,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;AACpX,KAAK,UAAU,SAAS,CAAC,MAA4B,EAAE,QAAgB,IAAqC,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,MAAM,EAAE,WAAW,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,OAAO,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG;IAAE,OAAO,SAAS,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE;IAAE,MAAM,IAAI,KAAK,CAAC,wCAAwC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,OAAO,MAAM,QAAQ,CAAC,IAAI,EAAgB,CAAC,CAAC,CAAC;AACpc,KAAK,UAAU,YAAY,CAAC,MAA4B,EAAE,OAAe,EAAE,MAAmC,IAAmB,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,MAAM,EAAE,cAAc,kBAAkB,CAAC,OAAO,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE;IAAE,MAAM,IAAI,KAAK,CAAC,6CAA6C,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC;AACta,KAAK,UAAU,aAAa,CAAC,MAA4B,EAAE,KAAa,IAAsD,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,MAAM,EAAE,YAAY,kBAAkB,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG;IAAE,OAAO,SAAS,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE;IAAE,MAAM,IAAI,KAAK,CAAC,gDAAgD,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,OAAO,MAAM,QAAQ,CAAC,IAAI,EAAiC,CAAC,CAAC,CAAC;AACla,KAAK,UAAU,OAAO,CAAC,MAA4B,EAAE,QAAgB,EAAE,OAAoB,EAAE,IAAuB,MAAM,QAAQ,GAAG,MAAM,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC,IAAI,CAAC,QAAQ;IAAE,MAAM,IAAI,KAAK,CAAC,+DAA+D,CAAC,CAAC,CAAC,MAAM,OAAO,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,KAAK,GAAG,MAAM,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,IAAI,KAAK;IAAE,OAAO,CAAC,GAAG,CAAC,eAAe,EAAE,UAAU,KAAK,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,CAAC,IAAI,GAAG,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,QAAQ,EAAE,EAAE,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC;AAC3hB,KAAK,UAAU,YAAY,CAAI,OAA6B,IAAgB,MAAM,MAAM,GAAa,EAAE,CAAC,CAAC,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,OAAO;IAAE,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAM,CAAC,CAAC,CAAC;AAC/Q,SAAS,IAAI,CAAC,QAA6B,EAAE,MAAc,EAAE,KAAc,IAAU,QAAQ,CAAC,UAAU,GAAG,MAAM,CAAC,CAAC,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;IAAC,QAAQ,CAAC,GAAG,EAAE,CAAC;IAAC,OAAO;AAAC,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC,cAAc,EAAE,kBAAkB,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;AACtQ,SAAS,IAAI,CAAC,KAAa,IAAY,OAAO,KAAK,CAAC,OAAO,CAAC,kBAAkB,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC"}

package/dist/distributed/types.d.ts

@@ -0,0 +1,34 @@
+import type { AgentExecutionSelection } from "../agents/types.js";
+import type { DelegationTask } from "../agents/outputContracts.js";
+import type { HarnessProjectConfig, TaskContract, WorkerSession } from "../core/types.js";
+export interface DistributedDelegationJob {
+    version: 1;
+    id: string;
+    parentTaskId: string;
+    createdAt: string;
+    repositoryUrl: string;
+    baseRef: string;
+    controllerSha256?: string;
+    priorPatches: string[];
+    task: DelegationTask;
+    contract: TaskContract;
+    selection: AgentExecutionSelection;
+    config: HarnessProjectConfig;
+    prompt: string;
+}
+export interface DistributedDelegationResult {
+    version: 1;
+    jobId: string;
+    workerId: string;
+    startedAt: string;
+    finishedAt: string;
+    status: "PASS" | "FAIL";
+    session: WorkerSession;
+    changedFiles: string[];
+    patch: string;
+    message?: string;
+}
+export interface ClaimedJob {
+    job: DistributedDelegationJob;
+    leaseId: string;
+}

package/dist/distributed/types.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=types.js.map

package/dist/distributed/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/distributed/types.ts"],"names":[],"mappings":""}

package/dist/distributed/worker.d.ts

@@ -0,0 +1,21 @@
+import type { AgentExecutionSelection } from "../agents/types.js";
+import type { DelegationTask } from "../agents/outputContracts.js";
+import type { ControlPlaneSnapshot } from "../core/controlPlane.js";
+import type { HarnessProjectConfig, TaskContract } from "../core/types.js";
+import type { DistributedDelegationResult } from "./types.js";
+export declare function dispatchDistributedDelegation(input: {
+    root: string;
+    config: HarnessProjectConfig;
+    contract: TaskContract;
+    task: DelegationTask;
+    selection: AgentExecutionSelection;
+    prompt: string;
+    priorPatches?: string[];
+    controller?: ControlPlaneSnapshot;
+}): Promise<DistributedDelegationResult>;
+export declare function runDistributedWorkerOnce(root: string, config: HarnessProjectConfig, workerId?: string): Promise<DistributedDelegationResult | undefined>;
+export declare function runDistributedWorkerLoop(root: string, config: HarnessProjectConfig, options?: {
+    workerId?: string;
+    once?: boolean;
+    signal?: AbortSignal;
+}): Promise<void>;

package/dist/distributed/worker.js

@@ -0,0 +1,111 @@
+import crypto from "node:crypto";
+import fs from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import YAML from "yaml";
+import { minimatch } from "minimatch";
+import { executeAgentPrompt } from "../workers/agentPrompt.js";
+import { runProcess } from "../utils/process.js";
+import { claimDistributedJob, completeDistributedJob, submitDistributedJob, waitForDistributedResult } from "./queue.js";
+export async function dispatchDistributedDelegation(input) {
+    if (!input.config.distributed?.enabled)
+        throw new Error("Distributed execution is not enabled.");
+    const remote = await runProcess("git remote get-url origin", { cwd: input.root, timeoutMs: 10_000 });
+    if (remote.exitCode !== 0 || !remote.stdout.trim())
+        throw new Error("Distributed execution requires a Git remote named origin.");
+    const base = await runProcess("git rev-parse HEAD", { cwd: input.root, timeoutMs: 10_000 });
+    if (base.exitCode !== 0)
+        throw new Error("Distributed execution could not resolve the workspace HEAD.");
+    const selection = { ...input.selection, transport: "direct" };
+    const job = {
+        version: 1,
+        id: `${safe(input.contract.task.id)}-${safe(input.task.id)}-${crypto.randomUUID()}`,
+        parentTaskId: input.contract.task.id,
+        createdAt: new Date().toISOString(),
+        repositoryUrl: remote.stdout.trim(),
+        baseRef: base.stdout.trim(),
+        controllerSha256: input.controller?.compositeSha256,
+        priorPatches: input.priorPatches ?? [],
+        task: input.task,
+        contract: input.contract,
+        selection,
+        config: sanitizeRemoteConfig(input.config),
+        prompt: input.prompt
+    };
+    await submitDistributedJob(input.root, input.config, job);
+    return waitForDistributedResult(input.root, input.config, job.id);
+}
+export async function runDistributedWorkerOnce(root, config, workerId = config.distributed?.workerId ?? `worker-${process.pid}`) {
+    const claimed = await claimDistributedJob(root, config, workerId);
+    if (!claimed)
+        return undefined;
+    const result = await executeClaimedJob(claimed.job, workerId);
+    await completeDistributedJob(root, config, claimed.leaseId, result);
+    return result;
+}
+export async function runDistributedWorkerLoop(root, config, options = {}) {
+    const workerId = options.workerId ?? config.distributed?.workerId ?? `worker-${process.pid}`;
+    const interval = config.distributed?.pollIntervalMs ?? 1000;
+    do {
+        if (options.signal?.aborted)
+            return;
+        const result = await runDistributedWorkerOnce(root, config, workerId);
+        if (options.once)
+            return;
+        if (!result)
+            await new Promise((resolve) => setTimeout(resolve, interval));
+    } while (!options.signal?.aborted);
+}
+async function executeClaimedJob(job, workerId) {
+    const startedAt = new Date().toISOString();
+    const worktree = await fs.mkdtemp(path.join(os.tmpdir(), `aeh-remote-${safe(job.id)}-`));
+    let session = { provider: job.selection.runtimeAdapter, model: job.selection.modelName, logicalAgent: job.selection.logicalAgent, runtime: job.selection.runtimeName, exitCode: 1, stdout: "", stderr: "remote worker did not start" };
+    try {
+        const clone = await runProcess(`git clone --quiet --no-checkout ${quote(job.repositoryUrl)} ${quote(worktree)}`, { cwd: os.tmpdir(), timeoutMs: 300_000, toolchain: false });
+        if (clone.exitCode !== 0)
+            return failure(job, workerId, startedAt, session, `clone failed: ${clone.stderr || clone.stdout}`);
+        const checkout = await runProcess(`git checkout --quiet --detach ${quote(job.baseRef)}`, { cwd: worktree, timeoutMs: 120_000 });
+        if (checkout.exitCode !== 0)
+            return failure(job, workerId, startedAt, session, `checkout failed: ${checkout.stderr || checkout.stdout}`);
+        for (const patch of job.priorPatches ?? []) {
+            const applied = await runProcess("git apply --binary -", { cwd: worktree, timeoutMs: 60_000, stdin: patch });
+            if (applied.exitCode !== 0)
+                return failure(job, workerId, startedAt, session, `prior wave patch failed: ${applied.stderr || applied.stdout}`);
+        }
+        const contractDir = job.config.sdd?.contractsDir ?? ".harness/contracts";
+        const contractFile = path.join(worktree, contractDir, `${job.contract.task.id}.yaml`);
+        await fs.mkdir(path.dirname(contractFile), { recursive: true });
+        await fs.writeFile(contractFile, YAML.stringify(job.contract));
+        const baseline = await runProcess("git add -A && git -c user.name=aeh -c user.email=aeh@localhost commit --no-gpg-sign --allow-empty -m 'aeh distributed baseline'", { cwd: worktree, timeoutMs: 60_000 });
+        if (baseline.exitCode !== 0)
+            return failure(job, workerId, startedAt, session, `baseline commit failed: ${baseline.stderr || baseline.stdout}`);
+        session = await executeAgentPrompt(worktree, job.config, job.contract, job.selection, job.prompt);
+        if (session.exitCode !== 0)
+            return failure(job, workerId, startedAt, session, `agent exited with ${session.exitCode}`);
+        const status = await runProcess("git status --porcelain", { cwd: worktree, timeoutMs: 30_000 });
+        const untracked = status.stdout.split(/\r?\n/).filter((line) => line.startsWith("?? ")).map((line) => line.slice(3).trim()).filter(Boolean);
+        if (untracked.length)
+            await runProcess(`git add -N -- ${untracked.map(quote).join(" ")}`, { cwd: worktree, timeoutMs: 30_000 });
+        const names = await runProcess("git diff --name-only HEAD", { cwd: worktree, timeoutMs: 30_000 });
+        const changedFiles = names.stdout.split(/\r?\n/).map((line) => line.trim()).filter(Boolean);
+        const escaped = changedFiles.filter((file) => !job.task.scope.some((scope) => matches(file, scope)));
+        if (escaped.length)
+            return failure(job, workerId, startedAt, session, `remote task escaped scope: ${escaped.join(", ")}`, changedFiles);
+        const diff = await runProcess("git diff --binary --no-ext-diff HEAD", { cwd: worktree, timeoutMs: 60_000 });
+        if (diff.exitCode !== 0)
+            return failure(job, workerId, startedAt, session, diff.stderr || "unable to capture patch", changedFiles);
+        return { version: 1, jobId: job.id, workerId, startedAt, finishedAt: new Date().toISOString(), status: "PASS", session, changedFiles, patch: diff.stdout };
+    }
+    catch (error) {
+        return failure(job, workerId, startedAt, session, String(error));
+    }
+    finally {
+        await fs.rm(worktree, { recursive: true, force: true }).catch(() => undefined);
+    }
+}
+function failure(job, workerId, startedAt, session, message, changedFiles = []) { return { version: 1, jobId: job.id, workerId, startedAt, finishedAt: new Date().toISOString(), status: "FAIL", session: { ...session, exitCode: session.exitCode || 1, stderr: [session.stderr, message].filter(Boolean).join("\n") }, changedFiles, patch: "", message }; }
+function sanitizeRemoteConfig(config) { return { ...config, delivery: { ...config.delivery, github: { ...config.delivery?.github, enabled: false }, paseo: { ...config.delivery?.paseo, enabled: false, autoUseWorkspace: false } }, distributed: { ...config.distributed, enabled: false }, telemetry: { ...config.telemetry, exporter: "none" }, security: { ...config.security, sandbox: { ...config.security?.sandbox, required: false, forceForRisks: [] } } }; }
+function matches(file, scope) { return scope === "**" || minimatch(file, scope, { dot: true }) || file.startsWith(scope.split(/[?*\[]/, 1)[0].replace(/\/+$/, "")); }
+function safe(value) { return value.replace(/[^A-Za-z0-9._-]/g, "-"); }
+function quote(value) { return `'${value.replaceAll("'", "'\\''")}'`; }
+//# sourceMappingURL=worker.js.map

package/dist/distributed/worker.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"worker.js","sourceRoot":"","sources":["../../src/distributed/worker.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,aAAa,CAAC;AACjC,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAClC,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAKtC,OAAO,EAAE,kBAAkB,EAAE,MAAM,2BAA2B,CAAC;AAC/D,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AACjD,OAAO,EAAE,mBAAmB,EAAE,sBAAsB,EAAE,oBAAoB,EAAE,wBAAwB,EAAE,MAAM,YAAY,CAAC;AAGzH,MAAM,CAAC,KAAK,UAAU,6BAA6B,CAAC,KASnD;IACC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,WAAW,EAAE,OAAO;QAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;IACjG,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,2BAA2B,EAAE,EAAE,GAAG,EAAE,KAAK,CAAC,IAAI,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC;IAAC,IAAI,MAAM,CAAC,QAAQ,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE;QAAE,MAAM,IAAI,KAAK,CAAC,2DAA2D,CAAC,CAAC;IACvO,MAAM,IAAI,GAAG,MAAM,UAAU,CAAC,oBAAoB,EAAE,EAAE,GAAG,EAAE,KAAK,CAAC,IAAI,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC;IAAC,IAAI,IAAI,CAAC,QAAQ,KAAK,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,6DAA6D,CAAC,CAAC;IACrM,MAAM,SAAS,GAA4B,EAAE,GAAG,KAAK,CAAC,SAAS,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC;IACvF,MAAM,GAAG,GAA6B;QACpC,OAAO,EAAE,CAAC;QACV,EAAE,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,MAAM,CAAC,UAAU,EAAE,EAAE;QACnF,YAAY,EAAE,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE;QACpC,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,aAAa,EAAE,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE;QACnC,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE;QAC3B,gBAAgB,EAAE,KAAK,CAAC,UAAU,EAAE,eAAe;QACnD,YAAY,EAAE,KAAK,CAAC,YAAY,IAAI,EAAE;QACtC,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,QAAQ,EAAE,KAAK,CAAC,QAAQ;QACxB,SAAS;QACT,MAAM,EAAE,oBAAoB,CAAC,KAAK,CAAC,MAAM,CAAC;QAC1C,MAAM,EAAE,KAAK,CAAC,MAAM;KACrB,CAAC;IACF,MAAM,oBAAoB,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC1D,OAAO,wBAAwB,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;AACpE,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAAC,IAAY,EAAE,MAA4B,EAAE,QAAQ,GAAG,MAAM,CAAC,WAAW,EAAE,QAAQ,IAAI,UAAU,OAAO,CAAC,GAAG,EAAE;IAC3J,MAAM,OAAO,GAAG,MAAM,mBAAmB,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;IAAC,IAAI,CAAC,OAAO;QAAE,OAAO,SAAS,CAAC;IAClG,MAAM,MAAM,GAAG,MAAM,iBAAiB,CAAC,OAAO,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IAC9D,MAAM,sBAAsB,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IACpE,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAAC,IAAY,EAAE,MAA4B,EAAE,UAAuE,EAAE;IAClK,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,MAAM,CAAC,WAAW,EAAE,QAAQ,IAAI,UAAU,OAAO,CAAC,GAAG,EAAE,CAAC;IAAC,MAAM,QAAQ,GAAG,MAAM,CAAC,WAAW,EAAE,cAAc,IAAI,IAAI,CAAC;IAC1J,GAAG,CAAC;QACF,IAAI,OAAO,CAAC,MAAM,EAAE,OAAO;YAAE,OAAO;QACpC,MAAM,MAAM,GAAG,MAAM,wBAAwB,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;QACtE,IAAI,OAAO,CAAC,IAAI;YAAE,OAAO;QACzB,IAAI,CAAC,MAAM;YAAE,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC;IAC7E,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE;AACrC,CAAC;AAED,KAAK,UAAU,iBAAiB,CAAC,GAA6B,EAAE,QAAgB;IAC9E,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAAC,MAAM,QAAQ,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,cAAc,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IAAC,IAAI,OAAO,GAAkB,EAAE,QAAQ,EAAE,GAAG,CAAC,SAAS,CAAC,cAAc,EAAE,KAAK,EAAE,GAAG,CAAC,SAAS,CAAC,SAAS,EAAE,YAAY,EAAE,GAAG,CAAC,SAAS,CAAC,YAAY,EAAE,OAAO,EAAE,GAAG,CAAC,SAAS,CAAC,WAAW,EAAE,QAAQ,EAAE,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,6BAA6B,EAAE,CAAC;IAC5X,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,MAAM,UAAU,CAAC,mCAAmC,KAAK,CAAC,GAAG,CAAC,aAAa,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC;QAAC,IAAI,KAAK,CAAC,QAAQ,KAAK,CAAC;YAAE,OAAO,OAAO,CAAC,GAAG,EAAE,QAAQ,EAAE,SAAS,EAAE,OAAO,EAAE,iBAAiB,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;QAC3S,MAAM,QAAQ,GAAG,MAAM,UAAU,CAAC,iCAAiC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,EAAE,EAAE,GAAG,EAAE,QAAQ,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC,CAAC;QAAC,IAAI,QAAQ,CAAC,QAAQ,KAAK,CAAC;YAAE,OAAO,OAAO,CAAC,GAAG,EAAE,QAAQ,EAAE,SAAS,EAAE,OAAO,EAAE,oBAAoB,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;QAC1Q,KAAK,MAAM,KAAK,IAAI,GAAG,CAAC,YAAY,IAAI,EAAE,EAAE,CAAC;YAAC,MAAM,OAAO,GAAG,MAAM,UAAU,CAAC,sBAAsB,EAAE,EAAE,GAAG,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;YAAC,IAAI,OAAO,CAAC,QAAQ,KAAK,CAAC;gBAAE,OAAO,OAAO,CAAC,GAAG,EAAE,QAAQ,EAAE,SAAS,EAAE,OAAO,EAAE,4BAA4B,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;QAAC,CAAC;QAC5S,MAAM,WAAW,GAAG,GAAG,CAAC,MAAM,CAAC,GAAG,EAAE,YAAY,IAAI,oBAAoB,CAAC;QAAC,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,WAAW,EAAE,GAAG,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,OAAO,CAAC,CAAC;QAAC,MAAM,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAAC,MAAM,EAAE,CAAC,SAAS,CAAC,YAAY,EAAE,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC;QACjS,MAAM,QAAQ,GAAG,MAAM,UAAU,CAAC,iIAAiI,EAAE,EAAE,GAAG,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC;QAAC,IAAI,QAAQ,CAAC,QAAQ,KAAK,CAAC;YAAE,OAAO,OAAO,CAAC,GAAG,EAAE,QAAQ,EAAE,SAAS,EAAE,OAAO,EAAE,2BAA2B,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;QAC5V,OAAO,GAAG,MAAM,kBAAkB,CAAC,QAAQ,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,QAAQ,EAAE,GAAG,CAAC,SAAS,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;QAClG,IAAI,OAAO,CAAC,QAAQ,KAAK,CAAC;YAAE,OAAO,OAAO,CAAC,GAAG,EAAE,QAAQ,EAAE,SAAS,EAAE,OAAO,EAAE,qBAAqB,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC;QACvH,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,wBAAwB,EAAE,EAAE,GAAG,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC;QAAC,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAAC,IAAI,SAAS,CAAC,MAAM;YAAE,MAAM,UAAU,CAAC,iBAAiB,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,EAAE,GAAG,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC;QAC9W,MAAM,KAAK,GAAG,MAAM,UAAU,CAAC,2BAA2B,EAAE,EAAE,GAAG,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC;QAAC,MAAM,YAAY,GAAG,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAAC,MAAM,OAAO,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC;QAAC,IAAI,OAAO,CAAC,MAAM;YAAE,OAAO,OAAO,CAAC,GAAG,EAAE,QAAQ,EAAE,SAAS,EAAE,OAAO,EAAE,8BAA8B,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,YAAY,CAAC,CAAC;QAC9a,MAAM,IAAI,GAAG,MAAM,UAAU,CAAC,sCAAsC,EAAE,EAAE,GAAG,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC;QAAC,IAAI,IAAI,CAAC,QAAQ,KAAK,CAAC;YAAE,OAAO,OAAO,CAAC,GAAG,EAAE,QAAQ,EAAE,SAAS,EAAE,OAAO,EAAE,IAAI,CAAC,MAAM,IAAI,yBAAyB,EAAE,YAAY,CAAC,CAAC;QAChP,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,KAAK,EAAE,GAAG,CAAC,EAAE,EAAE,QAAQ,EAAE,SAAS,EAAE,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,YAAY,EAAE,KAAK,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC;IAC7J,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAAC,OAAO,OAAO,CAAC,GAAG,EAAE,QAAQ,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;IAAC,CAAC;YAC7E,CAAC;QAAC,MAAM,EAAE,CAAC,EAAE,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;IAAC,CAAC;AAC7F,CAAC;AAED,SAAS,OAAO,CAAC,GAA6B,EAAE,QAAgB,EAAE,SAAiB,EAAE,OAAsB,EAAE,OAAe,EAAE,eAAyB,EAAE,IAAiC,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,KAAK,EAAE,GAAG,CAAC,EAAE,EAAE,QAAQ,EAAE,SAAS,EAAE,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,EAAE,GAAG,OAAO,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,YAAY,EAAE,KAAK,EAAE,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC;AACtc,SAAS,oBAAoB,CAAC,MAA4B,IAA0B,OAAO,EAAE,GAAG,MAAM,EAAE,QAAQ,EAAE,EAAE,GAAG,MAAM,CAAC,QAAQ,EAAE,MAAM,EAAE,EAAE,GAAG,MAAM,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,KAAK,EAAE,EAAE,GAAG,MAAM,CAAC,QAAQ,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,gBAAgB,EAAE,KAAK,EAAE,EAAE,EAAE,WAAW,EAAE,EAAE,GAAG,MAAM,CAAC,WAAW,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,SAAS,EAAE,EAAE,GAAG,MAAM,CAAC,SAAS,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,QAAQ,EAAE,EAAE,GAAG,MAAM,CAAC,QAAQ,EAAE,OAAO,EAAE,EAAE,GAAG,MAAM,CAAC,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,aAAa,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC;AAClf,SAAS,OAAO,CAAC,IAAY,EAAE,KAAa,IAAa,OAAO,KAAK,KAAK,IAAI,IAAI,SAAS,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;AAC9L,SAAS,IAAI,CAAC,KAAa,IAAY,OAAO,KAAK,CAAC,OAAO,CAAC,kBAAkB,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;AACvF,SAAS,KAAK,CAAC,KAAa,IAAY,OAAO,IAAI,KAAK,CAAC,UAAU,CAAC,GAAG,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC"}

package/dist/entry.js

@@ -7,11 +7,22 @@ import { compileAgentTopology } from "./agents/compiler.js";
 import { compileToolchain, setupToolchain } from "./toolchain/setup.js";
 import { loadToolchainConfig } from "./toolchain/config.js";
 import { resolveToolchain } from "./toolchain/resolve.js";
+import { runDistributedWorkerLoop } from "./distributed/worker.js";
+import { serveDistributedQueue } from "./distributed/queue.js";
+import { resolveOrganizationPolicyBundles } from "./policy/bundles.js";
+import { benchmarkMcpCatalog } from "./mcp/benchmark.js";
+import { buildEvalDashboard, runRepeatedEval } from "./evals/statistics.js";
+import { startPaseoHarness } from "./paseo/start.js";
+const VERSION = "0.5.1";
 const args = process.argv.slice(2);
 if (args.length === 1 && ["--version", "-V"].includes(args[0])) {
-    console.log("0.4.16");
+    console.log(VERSION);
     process.exit(0);
 }
+if (args[0] === "start") {
+    await runStart(args.slice(1));
+    process.exit(process.exitCode ?? 0);
+}
 if (args[0] === "setup") {
     await runSetup(args.slice(1));
     process.exit(process.exitCode ?? 0);
@@ -24,7 +35,49 @@ if (args[0] === "init" && args.includes("--setup")) {
     await runInitSetup(args.slice(1));
     process.exit(process.exitCode ?? 0);
 }
+if (args[0] === "worker") {
+    await runWorker(args.slice(1));
+    process.exit(process.exitCode ?? 0);
+}
+if (args[0] === "policy" && args[1] === "sync") {
+    await runPolicySync(args.slice(2));
+    process.exit(process.exitCode ?? 0);
+}
+if (args[0] === "mcp" && args[1] === "benchmark") {
+    await runMcpBenchmark(args.slice(2));
+    process.exit(process.exitCode ?? 0);
+}
+if (args[0] === "eval" && ["repeat", "dashboard"].includes(args[1] ?? "")) {
+    await runStatisticalEval(args.slice(1));
+    process.exit(process.exitCode ?? 0);
+}
 await import("./cli.js");
+async function runStart(argv) {
+    const parsed = parseGeneric(argv, new Set(["lead", "title"]), new Set(["new", "no-web-ui", "no-setup"]));
+    if (parsed.positional.length > 1)
+        throw new Error(`aeh start accepts at most one project directory, received: ${parsed.positional.join(", ")}`);
+    const root = path.resolve(parsed.positional[0] ?? ".");
+    const config = await loadProjectConfig(root);
+    const entry = path.resolve(process.argv[1]);
+    const aehCommand = `${JSON.stringify(process.execPath)} ${JSON.stringify(entry)}`;
+    const result = await startPaseoHarness(root, config, {
+        autoSetup: parsed.flag("no-setup") ? false : undefined,
+        webUi: parsed.flag("no-web-ui") ? false : undefined,
+        forceNew: parsed.flag("new"),
+        leadAgent: parsed.value("lead"),
+        title: parsed.value("title"),
+        aehCommand
+    });
+    console.log(`AEH Paseo ready for ${config.project.name}.`);
+    console.log(`daemon=${result.daemonStarted ? "started" : "reused"}`);
+    console.log(`session=${result.session}`);
+    console.log(`lead=${result.leadAgent}`);
+    console.log(`provider=${result.provider}`);
+    console.log(`model=${result.model}`);
+    console.log(`agentId=${result.agentId}`);
+    console.log(`title=${result.title}`);
+    console.log(`Open Paseo and continue in '${result.title}'. Repository-changing prompts in that conversation now route through the Harness automatically.`);
+}
 async function runSetup(argv) {
     const parsed = parse(argv);
     const root = path.resolve(parsed.directory);
@@ -45,68 +98,79 @@ async function runSetup(argv) {
         process.exitCode = 1;
     }
 }
-async function runToolchain(argv) {
-    const sub = argv[0] ?? "show";
-    const parsed = parse(argv.slice(1));
-    const root = path.resolve(parsed.directory);
+async function runToolchain(argv) { const sub = argv[0] ?? "show"; const parsed = parse(argv.slice(1)); const root = path.resolve(parsed.directory); const config = await loadProjectConfig(root); if (sub === "compile") {
+    console.log(`Compiled toolchain: ${await compileToolchain(root, config, { profile: parsed.value("profile"), updateLock: parsed.flag("update-lock") })}`);
+    return;
+} if (sub === "show") {
+    const tc = await loadToolchainConfig(root, config);
+    console.log(JSON.stringify(await resolveToolchain(root, config, tc, { profile: parsed.value("profile") }), null, 2));
+    return;
+} if (sub === "setup") {
+    await runSetup(argv.slice(1));
+    return;
+} throw new Error(`Unknown toolchain command '${sub}'. Use compile, show or setup.`); }
+async function runInitSetup(argv) { const without = argv.filter((value) => value !== "--setup"); const parsed = parse(without); const root = path.resolve(parsed.directory); const created = await initializeProject(root); const config = await loadProjectConfig(root); if (config.agents) {
+    const compiled = await compileAgentTopology(root, config);
+    if (!compiled.ok)
+        throw new Error(compiled.issues.join("; "));
+} await compileToolchain(root, config); const setup = await setupToolchain(root, config, { profile: parsed.value("profile"), preferContainers: parsed.flag("prefer-containers") }); console.log(created.length ? `Created: ${created.join(", ")}` : "Harness already initialized."); console.log(`Toolchain ready: profile=${setup.profile}`); }
+async function runWorker(argv) {
+    const sub = argv[0] ?? "run";
+    const parsed = parseGeneric(argv.slice(1), new Set(["worker-id", "port", "host"]), new Set(["once"]));
+    const root = path.resolve(parsed.positional[0] ?? ".");
     const config = await loadProjectConfig(root);
-    if (sub === "compile") {
-        console.log(`Compiled toolchain: ${await compileToolchain(root, config, { profile: parsed.value("profile"), updateLock: parsed.flag("update-lock") })}`);
-        return;
-    }
-    if (sub === "show") {
-        const tc = await loadToolchainConfig(root, config);
-        console.log(JSON.stringify(await resolveToolchain(root, config, tc, { profile: parsed.value("profile") }), null, 2));
+    if (sub === "run") {
+        await runDistributedWorkerLoop(root, config, { workerId: parsed.value("worker-id"), once: parsed.flag("once") });
         return;
     }
-    if (sub === "setup") {
-        await runSetup(argv.slice(1));
+    if (sub === "serve") {
+        const port = Number(parsed.value("port") ?? "8787");
+        if (!Number.isInteger(port) || port <= 0 || port > 65535)
+            throw new Error("--port must be a valid TCP port.");
+        const localConfig = { ...config, distributed: { ...config.distributed, provider: "filesystem" } };
+        await serveDistributedQueue(root, localConfig, { port, host: parsed.value("host") });
+        console.log(`AEH distributed queue listening on ${parsed.value("host") ?? "127.0.0.1"}:${port}`);
+        await new Promise(() => undefined);
         return;
     }
-    throw new Error(`Unknown toolchain command '${sub}'. Use compile, show or setup.`);
-}
-async function runInitSetup(argv) {
-    const without = argv.filter((value) => value !== "--setup");
-    const parsed = parse(without);
-    const root = path.resolve(parsed.directory);
-    const created = await initializeProject(root);
-    const config = await loadProjectConfig(root);
-    if (config.agents) {
-        const compiled = await compileAgentTopology(root, config);
-        if (!compiled.ok)
-            throw new Error(compiled.issues.join("; "));
-    }
-    await compileToolchain(root, config);
-    const setup = await setupToolchain(root, config, { profile: parsed.value("profile"), preferContainers: parsed.flag("prefer-containers") });
-    console.log(created.length ? `Created: ${created.join(", ")}` : "Harness already initialized.");
-    console.log(`Toolchain ready: profile=${setup.profile}`);
+    throw new Error(`Unknown worker command '${sub}'. Use run or serve.`);
 }
-function parse(argv) {
-    const valueFlags = new Set(["profile"]);
-    const booleanFlags = new Set(["dry-run", "update-lock", "skip-project-deps", "prefer-containers"]);
+async function runPolicySync(argv) { const parsed = parseGeneric(argv, new Set(), new Set()); const root = path.resolve(parsed.positional[0] ?? "."); const config = await loadProjectConfig(root); const resolution = await resolveOrganizationPolicyBundles(root, config); console.log(JSON.stringify(resolution, null, 2)); if (resolution.issues.length && config.organization?.policyBundles?.required)
+    process.exitCode = 1; }
+async function runMcpBenchmark(argv) { const parsed = parseGeneric(argv, new Set(["server"]), new Set()); const root = path.resolve(parsed.positional[0] ?? "."); const config = await loadProjectConfig(root); const servers = parsed.values("server"); console.log(JSON.stringify(await benchmarkMcpCatalog(root, config, servers.length ? servers : undefined), null, 2)); }
+async function runStatisticalEval(argv) { const sub = argv[0]; const parsed = parseGeneric(argv.slice(1), new Set(["variant", "runs"]), new Set()); const caseId = parsed.positional[0]; if (!caseId)
+    throw new Error(`aeh eval ${sub} requires <caseId>.`); const root = path.resolve(parsed.positional[1] ?? "."); const config = await loadProjectConfig(root); if (sub === "repeat") {
+    const runs = parsed.value("runs") ? Number(parsed.value("runs")) : undefined;
+    console.log(JSON.stringify(await runRepeatedEval(root, config, caseId, parsed.value("variant"), runs), null, 2));
+    return;
+} console.log(JSON.stringify(await buildEvalDashboard(root, config, caseId), null, 2)); }
+function parse(argv) { const parsed = parseGeneric(argv, new Set(["profile"]), new Set(["dry-run", "update-lock", "skip-project-deps", "prefer-containers"])); if (parsed.positional.length > 1)
+    throw new Error(`Expected at most one project directory, received: ${parsed.positional.join(", ")}`); return { directory: parsed.positional[0] ?? ".", flag: parsed.flag, value: parsed.value }; }
+function parseGeneric(argv, valueFlags, booleanFlags) {
     const flags = new Map();
     const positional = [];
-    for (let i = 0; i < argv.length; i++) {
-        const value = argv[i];
-        if (!value.startsWith("--")) {
-            positional.push(value);
+    for (let i = 0; i < argv.length; i += 1) {
+        const token = argv[i];
+        if (!token.startsWith("--")) {
+            positional.push(token);
             continue;
         }
-        const name = value.slice(2);
+        const name = token.slice(2);
         if (valueFlags.has(name)) {
             const next = argv[++i];
             if (!next || next.startsWith("--"))
                 throw new Error(`--${name} requires a value.`);
-            flags.set(name, next);
+            const list = flags.get(name) ?? [];
+            list.push(next);
+            flags.set(name, list);
             continue;
         }
         if (!booleanFlags.has(name))
-            throw new Error(`Unknown toolchain option --${name}.`);
-        flags.set(name, true);
+            throw new Error(`Unknown option --${name}.`);
+        const list = flags.get(name) ?? [];
+        list.push(true);
+        flags.set(name, list);
     }
-    if (positional.length > 1)
-        throw new Error(`Expected at most one project directory, received: ${positional.join(", ")}`);
-    const directory = positional[0] ?? ".";
-    return { directory, flag: (name) => flags.get(name) === true, value: (name) => typeof flags.get(name) === "string" ? flags.get(name) : undefined };
+    return { positional, flag: (name) => flags.get(name)?.includes(true) ?? false, value: (name) => { const found = flags.get(name)?.find((item) => typeof item === "string"); return found; }, values: (name) => (flags.get(name) ?? []).filter((item) => typeof item === "string") };
 }
 //# sourceMappingURL=entry.js.map

package/dist/entry.js.map

@@ -1 +1 @@
-{"version":3,"file":"entry.js","sourceRoot":"","sources":["../src/entry.ts"],"names":[],"mappings":";AACA,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,OAAO,MAAM,cAAc,CAAC;AACnC,OAAO,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AACnD,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EAAE,oBAAoB,EAAE,MAAM,sBAAsB,CAAC;AAC5D,OAAO,EAAE,gBAAgB,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACxE,OAAO,EAAE,mBAAmB,EAAE,MAAM,uBAAuB,CAAC;AAC5D,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAE1D,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;AACnC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAAC,CAAC;AAE3G,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,OAAO,EAAE,CAAC;IAAC,MAAM,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,IAAI,CAAC,CAAC,CAAC;AAAC,CAAC;AAChG,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,WAAW,EAAE,CAAC;IAAC,MAAM,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,IAAI,CAAC,CAAC,CAAC;AAAC,CAAC;AACxG,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,MAAM,IAAI,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;IAAC,MAAM,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,IAAI,CAAC,CAAC,CAAC;AAAC,CAAC;AAE/H,MAAM,MAAM,CAAC,UAAU,CAAC,CAAC;AAEzB,KAAK,UAAU,QAAQ,CAAC,IAAc;IACpC,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC;IAAC,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IAAC,MAAM,MAAM,GAAG,MAAM,iBAAiB,CAAC,IAAI,CAAC,CAAC;IACtH,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,OAAO,EAAE,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,UAAU,EAAE,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,EAAE,uBAAuB,EAAE,MAAM,CAAC,IAAI,CAAC,mBAAmB,CAAC,EAAE,gBAAgB,EAAE,MAAM,CAAC,IAAI,CAAC,mBAAmB,CAAC,EAAE,CAAC,CAAC;IAC/Q,OAAO,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,sBAAsB,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC;IAC1F,OAAO,CAAC,GAAG,CAAC,cAAc,MAAM,CAAC,eAAe,EAAE,CAAC,CAAC;IAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;IAAC,OAAO,CAAC,GAAG,CAAC,SAAS,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC;IACtI,IAAI,MAAM,CAAC,SAAS,CAAC,MAAM;QAAE,OAAO,CAAC,GAAG,CAAC,SAAS,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjF,IAAI,MAAM,CAAC,UAAU,CAAC,MAAM;QAAE,OAAO,CAAC,GAAG,CAAC,cAAc,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACxF,IAAI,MAAM,CAAC,yBAAyB,CAAC,MAAM;QAAE,OAAO,CAAC,GAAG,CAAC,eAAe,MAAM,CAAC,yBAAyB,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IACzH,IAAI,MAAM,CAAC,aAAa,CAAC,MAAM,EAAE,CAAC;QAAC,OAAO,CAAC,KAAK,CAAC,iBAAiB,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAAC,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;IAAC,CAAC;AAC/H,CAAC;AAED,KAAK,UAAU,YAAY,CAAC,IAAc;IACxC,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC;IAAC,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IAAC,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IAAC,MAAM,MAAM,GAAG,MAAM,iBAAiB,CAAC,IAAI,CAAC,CAAC;IAC9J,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;QAAC,OAAO,CAAC,GAAG,CAAC,uBAAuB,MAAM,gBAAgB,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,OAAO,EAAE,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,EAAE,UAAU,EAAE,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;QAAC,OAAO;IAAC,CAAC;IAC5L,IAAI,GAAG,KAAK,MAAM,EAAE,CAAC;QAAC,MAAM,EAAE,GAAG,MAAM,mBAAmB,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,gBAAgB,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,EAAE,OAAO,EAAE,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAAC,OAAO;IAAC,CAAC;IACzM,IAAI,GAAG,KAAK,OAAO,EAAE,CAAC;QAAC,MAAM,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QAAC,OAAO;IAAC,CAAC;IAC/D,MAAM,IAAI,KAAK,CAAC,8BAA8B,GAAG,gCAAgC,CAAC,CAAC;AACrF,CAAC;AAED,KAAK,UAAU,YAAY,CAAC,IAAc;IACxC,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC;IAAC,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC;IAAC,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IAAC,MAAM,OAAO,GAAG,MAAM,iBAAiB,CAAC,IAAI,CAAC,CAAC;IAAC,MAAM,MAAM,GAAG,MAAM,iBAAiB,CAAC,IAAI,CAAC,CAAC;IACrO,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;QAAC,MAAM,QAAQ,GAAG,MAAM,oBAAoB,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QAAC,IAAI,CAAC,QAAQ,CAAC,EAAE;YAAE,MAAM,IAAI,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;IAAC,CAAC;IAChJ,MAAM,gBAAgB,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IACrC,MAAM,KAAK,GAAG,MAAM,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,OAAO,EAAE,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,EAAE,gBAAgB,EAAE,MAAM,CAAC,IAAI,CAAC,mBAAmB,CAAC,EAAE,CAAC,CAAC;IAC3I,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,YAAY,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,8BAA8B,CAAC,CAAC;IAChG,OAAO,CAAC,GAAG,CAAC,4BAA4B,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;AAC3D,CAAC;AAED,SAAS,KAAK,CAAC,IAAc;IAC3B,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC;IACxC,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,CAAC,SAAS,EAAE,aAAa,EAAE,mBAAmB,EAAE,mBAAmB,CAAC,CAAC,CAAC;IACnG,MAAM,KAAK,GAAG,IAAI,GAAG,EAAyB,CAAC;IAAC,MAAM,UAAU,GAAa,EAAE,CAAC;IAChF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACrC,MAAM,KAAK,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;QACtB,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;YAAC,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAAC,SAAS;QAAC,CAAC;QAClE,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAC5B,IAAI,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YACzB,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;YACvB,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;gBAAE,MAAM,IAAI,KAAK,CAAC,KAAK,IAAI,oBAAoB,CAAC,CAAC;YACnF,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;YACtB,SAAS;QACX,CAAC;QACD,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,8BAA8B,IAAI,GAAG,CAAC,CAAC;QACpF,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IACxB,CAAC;IACD,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,qDAAqD,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACzH,MAAM,SAAS,GAAG,UAAU,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC;IACvC,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,KAAK,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAW,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC;AAC/J,CAAC"}
+{"version":3,"file":"entry.js","sourceRoot":"","sources":["../src/entry.ts"],"names":[],"mappings":";AACA,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,OAAO,MAAM,cAAc,CAAC;AACnC,OAAO,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AACnD,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EAAE,oBAAoB,EAAE,MAAM,sBAAsB,CAAC;AAC5D,OAAO,EAAE,gBAAgB,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACxE,OAAO,EAAE,mBAAmB,EAAE,MAAM,uBAAuB,CAAC;AAC5D,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAC1D,OAAO,EAAE,wBAAwB,EAAE,MAAM,yBAAyB,CAAC;AACnE,OAAO,EAAE,qBAAqB,EAAE,MAAM,wBAAwB,CAAC;AAC/D,OAAO,EAAE,gCAAgC,EAAE,MAAM,qBAAqB,CAAC;AACvE,OAAO,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AACzD,OAAO,EAAE,kBAAkB,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAC5E,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAErD,MAAM,OAAO,GAAG,OAAO,CAAC;AACxB,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;AACnC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAAC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAAC,CAAC;AAE1G,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,OAAO,EAAE,CAAC;IAAC,MAAM,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,IAAI,CAAC,CAAC,CAAC;AAAC,CAAC;AAChG,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,OAAO,EAAE,CAAC;IAAC,MAAM,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,IAAI,CAAC,CAAC,CAAC;AAAC,CAAC;AAChG,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,WAAW,EAAE,CAAC;IAAC,MAAM,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,IAAI,CAAC,CAAC,CAAC;AAAC,CAAC;AACxG,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,MAAM,IAAI,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;IAAC,MAAM,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,IAAI,CAAC,CAAC,CAAC;AAAC,CAAC;AAC/H,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,QAAQ,EAAE,CAAC;IAAC,MAAM,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,IAAI,CAAC,CAAC,CAAC;AAAC,CAAC;AAClG,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,QAAQ,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,MAAM,EAAE,CAAC;IAAC,MAAM,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,IAAI,CAAC,CAAC,CAAC;AAAC,CAAC;AAC5H,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,KAAK,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,WAAW,EAAE,CAAC;IAAC,MAAM,eAAe,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,IAAI,CAAC,CAAC,CAAC;AAAC,CAAC;AAChI,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,MAAM,IAAI,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC;IAAC,MAAM,kBAAkB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,IAAI,CAAC,CAAC,CAAC;AAAC,CAAC;AAE5J,MAAM,MAAM,CAAC,UAAU,CAAC,CAAC;AAEzB,KAAK,UAAU,QAAQ,CAAC,IAAc;IACpC,MAAM,MAAM,GAAG,YAAY,CAAC,IAAI,EAAE,IAAI,GAAG,CAAC,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,EAAE,IAAI,GAAG,CAAC,CAAC,KAAK,EAAE,WAAW,EAAE,UAAU,CAAC,CAAC,CAAC,CAAC;IACzG,IAAI,MAAM,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,8DAA8D,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAChJ,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC;IACvD,MAAM,MAAM,GAAG,MAAM,iBAAiB,CAAC,IAAI,CAAC,CAAC;IAC7C,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAC5C,MAAM,UAAU,GAAG,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC;IAClF,MAAM,MAAM,GAAG,MAAM,iBAAiB,CAAC,IAAI,EAAE,MAAM,EAAE;QACnD,SAAS,EAAE,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS;QACtD,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS;QACnD,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC;QAC5B,SAAS,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC;QAC/B,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC;QAC5B,UAAU;KACX,CAAC,CAAC;IACH,OAAO,CAAC,GAAG,CAAC,uBAAuB,MAAM,CAAC,OAAO,CAAC,IAAI,GAAG,CAAC,CAAC;IAC3D,OAAO,CAAC,GAAG,CAAC,UAAU,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC;IACrE,OAAO,CAAC,GAAG,CAAC,WAAW,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC;IACzC,OAAO,CAAC,GAAG,CAAC,QAAQ,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC;IACxC,OAAO,CAAC,GAAG,CAAC,YAAY,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;IAC3C,OAAO,CAAC,GAAG,CAAC,SAAS,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC;IACrC,OAAO,CAAC,GAAG,CAAC,WAAW,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC;IACzC,OAAO,CAAC,GAAG,CAAC,SAAS,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC;IACrC,OAAO,CAAC,GAAG,CAAC,+BAA+B,MAAM,CAAC,KAAK,kGAAkG,CAAC,CAAC;AAC7J,CAAC;AAED,KAAK,UAAU,QAAQ,CAAC,IAAc;IACpC,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC;IAAC,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IAAC,MAAM,MAAM,GAAG,MAAM,iBAAiB,CAAC,IAAI,CAAC,CAAC;IACtH,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,OAAO,EAAE,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,UAAU,EAAE,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,EAAE,uBAAuB,EAAE,MAAM,CAAC,IAAI,CAAC,mBAAmB,CAAC,EAAE,gBAAgB,EAAE,MAAM,CAAC,IAAI,CAAC,mBAAmB,CAAC,EAAE,CAAC,CAAC;IAC/Q,OAAO,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,sBAAsB,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC;IAAC,OAAO,CAAC,GAAG,CAAC,cAAc,MAAM,CAAC,eAAe,EAAE,CAAC,CAAC;IAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;IAAC,OAAO,CAAC,GAAG,CAAC,SAAS,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC;IAAC,IAAI,MAAM,CAAC,SAAS,CAAC,MAAM;QAAE,OAAO,CAAC,GAAG,CAAC,SAAS,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAAC,IAAI,MAAM,CAAC,UAAU,CAAC,MAAM;QAAE,OAAO,CAAC,GAAG,CAAC,cAAc,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAAC,IAAI,MAAM,CAAC,yBAAyB,CAAC,MAAM;QAAE,OAAO,CAAC,GAAG,CAAC,eAAe,MAAM,CAAC,yBAAyB,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IAAC,IAAI,MAAM,CAAC,aAAa,CAAC,MAAM,EAAE,CAAC;QAAC,OAAO,CAAC,KAAK,CAAC,iBAAiB,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAAC,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;IAAC,CAAC;AACtoB,CAAC;AACD,KAAK,UAAU,YAAY,CAAC,IAAc,IAAmB,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,MAAM,GAAG,MAAM,iBAAiB,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;IAAC,OAAO,CAAC,GAAG,CAAC,uBAAuB,MAAM,gBAAgB,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,OAAO,EAAE,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,EAAE,UAAU,EAAE,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;IAAC,OAAO;AAAC,CAAC,CAAC,IAAI,GAAG,KAAK,MAAM,EAAE,CAAC;IAAC,MAAM,EAAE,GAAG,MAAM,mBAAmB,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,gBAAgB,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,EAAE,OAAO,EAAE,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IAAC,OAAO;AAAC,CAAC,CAAC,IAAI,GAAG,KAAK,OAAO,EAAE,CAAC;IAAC,MAAM,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IAAC,OAAO;AAAC,CAAC,CAAC,MAAM,IAAI,KAAK,CAAC,8BAA8B,GAAG,gCAAgC,CAAC,CAAC,CAAC,CAAC;AACxvB,KAAK,UAAU,YAAY,CAAC,IAAc,IAAmB,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,OAAO,GAAG,MAAM,iBAAiB,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,MAAM,GAAG,MAAM,iBAAiB,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;IAAC,MAAM,QAAQ,GAAG,MAAM,oBAAoB,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAAC,IAAI,CAAC,QAAQ,CAAC,EAAE;QAAE,MAAM,IAAI,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;AAAC,CAAC,CAAC,MAAM,gBAAgB,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,MAAM,KAAK,GAAG,MAAM,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,OAAO,EAAE,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,EAAE,gBAAgB,EAAE,MAAM,CAAC,IAAI,CAAC,mBAAmB,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,YAAY,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,8BAA8B,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,4BAA4B,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC;AAElwB,KAAK,UAAU,SAAS,CAAC,IAAc;IACrC,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC;IAAC,MAAM,MAAM,GAAG,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,IAAI,GAAG,CAAC,CAAC,WAAW,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,EAAE,IAAI,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IAAC,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC;IAAC,MAAM,MAAM,GAAG,MAAM,iBAAiB,CAAC,IAAI,CAAC,CAAC;IAC1O,IAAI,GAAG,KAAK,KAAK,EAAE,CAAC;QAAC,MAAM,wBAAwB,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAC,WAAW,CAAC,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QAAC,OAAO;IAAC,CAAC;IAChJ,IAAI,GAAG,KAAK,OAAO,EAAE,CAAC;QAAC,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,MAAM,CAAC,CAAC;QAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,IAAI,GAAG,KAAK;YAAE,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAC;QAAC,MAAM,WAAW,GAAG,EAAE,GAAG,MAAM,EAAE,WAAW,EAAE,EAAE,GAAG,MAAM,CAAC,WAAW,EAAE,QAAQ,EAAE,YAAqB,EAAE,EAAE,CAAC;QAAC,MAAM,qBAAqB,CAAC,IAAI,EAAE,WAAW,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QAAC,OAAO,CAAC,GAAG,CAAC,sCAAsC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,WAAW,IAAI,IAAI,EAAE,CAAC,CAAC;QAAC,MAAM,IAAI,OAAO,CAAO,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QAAC,OAAO;IAAC,CAAC;IAClhB,MAAM,IAAI,KAAK,CAAC,2BAA2B,GAAG,sBAAsB,CAAC,CAAC;AACxE,CAAC;AAED,KAAK,UAAU,aAAa,CAAC,IAAc,IAAmB,MAAM,MAAM,GAAG,YAAY,CAAC,IAAI,EAAE,IAAI,GAAG,EAAE,EAAE,IAAI,GAAG,EAAE,CAAC,CAAC,CAAC,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,MAAM,MAAM,GAAG,MAAM,iBAAiB,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,UAAU,GAAG,MAAM,gCAAgC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,UAAU,CAAC,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,YAAY,EAAE,aAAa,EAAE,QAAQ;IAAE,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC;AAC7b,KAAK,UAAU,eAAe,CAAC,IAAc,IAAmB,MAAM,MAAM,GAAG,YAAY,CAAC,IAAI,EAAE,IAAI,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,IAAI,GAAG,EAAE,CAAC,CAAC,CAAC,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,MAAM,MAAM,GAAG,MAAM,iBAAiB,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,mBAAmB,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AACxY,KAAK,UAAU,kBAAkB,CAAC,IAAc,IAAmB,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,MAAM,GAAG,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,IAAI,GAAG,CAAC,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC,EAAE,IAAI,GAAG,EAAE,CAAC,CAAC,CAAC,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM;IAAE,MAAM,IAAI,KAAK,CAAC,YAAY,GAAG,qBAAqB,CAAC,CAAC,CAAC,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,MAAM,MAAM,GAAG,MAAM,iBAAiB,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,GAAG,KAAK,QAAQ,EAAE,CAAC;IAAC,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,eAAe,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,EAAE,IAAI,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IAAC,OAAO;AAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,kBAAkB,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAErrB,SAAS,KAAK,CAAC,IAAc,IAAiG,MAAM,MAAM,GAAG,YAAY,CAAC,IAAI,EAAE,IAAI,GAAG,CAAC,CAAC,SAAS,CAAC,CAAC,EAAE,IAAI,GAAG,CAAC,CAAC,SAAS,EAAE,aAAa,EAAE,mBAAmB,EAAE,mBAAmB,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC;IAAE,MAAM,IAAI,KAAK,CAAC,qDAAqD,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,EAAE,SAAS,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,GAAG,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;AAC1e,SAAS,YAAY,CAAC,IAAc,EAAE,UAAuB,EAAE,YAAyB;IACtF,MAAM,KAAK,GAAG,IAAI,GAAG,EAAgC,CAAC;IAAC,MAAM,UAAU,GAAa,EAAE,CAAC;IACvF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;QAAC,MAAM,KAAK,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;QAAC,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;YAAC,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAAC,SAAS;QAAC,CAAC;QAAC,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAAC,IAAI,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YAAC,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;YAAC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;gBAAE,MAAM,IAAI,KAAK,CAAC,KAAK,IAAI,oBAAoB,CAAC,CAAC;YAAC,MAAM,IAAI,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;YAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;YAAC,SAAS;QAAC,CAAC;QAAC,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,oBAAoB,IAAI,GAAG,CAAC,CAAC;QAAC,MAAM,IAAI,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;QAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IAAC,CAAC;IAC1hB,OAAO,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,QAAQ,CAAC,IAAI,CAAC,IAAI,KAAK,EAAE,KAAK,EAAE,CAAC,IAAI,EAAE,EAAE,GAAG,MAAM,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,CAAC,IAAI,EAAkB,EAAE,CAAC,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,KAAK,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAkB,EAAE,CAAC,OAAO,IAAI,KAAK,QAAQ,CAAC,EAAE,CAAC;AACrT,CAAC"}

package/dist/evals/runner.js

@@ -77,9 +77,9 @@ export async function compareEvalCase(root, config, caseId) {
     const dir = path.resolve(root, config.evals?.resultsDir ?? ".harness/evals/results", safe(caseId));
     const files = await fs.readdir(dir).catch(() => []);
     const results = [];
-    for (const file of files.filter((name) => name.endsWith(".json"))) {
+    for (const file of files.filter((name) => name.endsWith(".json") && name !== "dashboard.json")) {
         const value = await readJson(path.join(dir, file));
-        if (value?.caseId === caseId)
+        if (value?.caseId === caseId && typeof value.variant === "string" && typeof value.score === "number" && (value.status === "PASS" || value.status === "FAIL"))
             results.push(value);
     }
     return rankEvalResults(results);