@oisincoveney/pipeline 2.5.0 → 2.6.0

package/dist/config/schemas.d.ts

@@ -221,8 +221,8 @@ declare const configSchema: z.ZodObject<{
     policy: z.ZodOptional<z.ZodObject<{
       commands: z.ZodOptional<z.ZodEnum<{
         allow: "allow";
-        "trusted-only": "trusted-only";
         deny: "deny";
+        "trusted-only": "trusted-only";
       }>>;
       modules: z.ZodOptional<z.ZodEnum<{
         allow: "allow";
@@ -246,8 +246,8 @@ declare const configSchema: z.ZodObject<{
     }, z.core.$strict>>>;
     default_profile: z.ZodOptional<z.ZodString>;
     mode: z.ZodEnum<{
-      hosted: "hosted";
       local: "local";
+      hosted: "hosted";
     }>;
     provider: z.ZodLiteral<"toolhive">;
     authorization_env: z.ZodDefault<z.ZodString>;
@@ -290,10 +290,10 @@ declare const configSchema: z.ZodObject<{
     }, z.core.$strict>>;
     output: z.ZodOptional<z.ZodObject<{
       format: z.ZodEnum<{
+        json_schema: "json_schema";
         text: "text";
         json: "json";
         jsonl: "jsonl";
-        json_schema: "json_schema";
       }>;
       repair: z.ZodOptional<z.ZodObject<{
         enabled: z.ZodOptional<z.ZodBoolean>;
@@ -311,6 +311,7 @@ declare const configSchema: z.ZodObject<{
     skills: z.ZodOptional<z.ZodArray<z.ZodString>>;
     timeout_ms: z.ZodOptional<z.ZodNumber>;
     tools: z.ZodOptional<z.ZodArray<z.ZodEnum<{
+      task: "task";
       read: "read";
       list: "list";
       grep: "grep";
@@ -318,7 +319,6 @@ declare const configSchema: z.ZodObject<{
       bash: "bash";
       edit: "edit";
       write: "write";
-      task: "task";
     }>>>;
   }, z.core.$strict>>>;
   runner_command: z.ZodDefault<z.ZodObject<{
@@ -362,14 +362,15 @@ declare const configSchema: z.ZodObject<{
         disabled: "disabled";
       }>>>;
       output_formats: z.ZodOptional<z.ZodArray<z.ZodEnum<{
+        json_schema: "json_schema";
         text: "text";
         json: "json";
         jsonl: "jsonl";
-        json_schema: "json_schema";
       }>>>;
       rules: z.ZodOptional<z.ZodBoolean>;
       skills: z.ZodOptional<z.ZodBoolean>;
       tools: z.ZodOptional<z.ZodArray<z.ZodEnum<{
+        task: "task";
         read: "read";
         list: "list";
         grep: "grep";
@@ -377,7 +378,6 @@ declare const configSchema: z.ZodObject<{
         bash: "bash";
         edit: "edit";
         write: "write";
-        task: "task";
       }>>>;
     }, z.core.$strict>;
     command: z.ZodOptional<z.ZodString>;
@@ -495,6 +495,7 @@ declare const configSchema: z.ZodObject<{
   best_of_n: z.ZodOptional<z.ZodObject<{
     categories: z.ZodDefault<z.ZodArray<z.ZodString>>;
     enabled: z.ZodDefault<z.ZodBoolean>;
+    judge_model: z.ZodOptional<z.ZodString>;
     n: z.ZodDefault<z.ZodNumber>;
   }, z.core.$strict>>;
   context_handoff: z.ZodOptional<z.ZodObject<{

package/dist/config/schemas.js

@@ -469,6 +469,7 @@ const parallelWorktreesSchema = z.object({ enabled: z.boolean().default(false) }
 const bestOfNSchema = z.object({
 	categories: z.array(z.string()).default(["green"]),
 	enabled: z.boolean().default(false),
+	judge_model: z.string().optional(),
 	n: z.number().int().positive().default(1)
 }).strict();
 const pipelineFileSchema = z.object({

package/dist/moka-submit.d.ts

@@ -5,13 +5,13 @@ import { z } from "zod";
 //#region src/moka-submit.d.ts
 declare const mokaSubmitDirectHooksSchema: z.ZodRecord<z.ZodEnum<{
   "workflow.start": "workflow.start";
-  "node.finish": "node.finish";
-  "node.start": "node.start";
   "workflow.success": "workflow.success";
   "workflow.failure": "workflow.failure";
   "workflow.complete": "workflow.complete";
+  "node.start": "node.start";
   "node.success": "node.success";
   "node.error": "node.error";
+  "node.finish": "node.finish";
   "gate.failure": "gate.failure";
 }> & z.core.$partial, z.ZodDiscriminatedUnion<[z.ZodObject<{
   failure: z.ZodDefault<z.ZodEnum<{
@@ -94,13 +94,13 @@ declare const mokaSubmitOptionsSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
   }, z.core.$strict>>;
   hooks: z.ZodOptional<z.ZodRecord<z.ZodEnum<{
     "workflow.start": "workflow.start";
-    "node.finish": "node.finish";
-    "node.start": "node.start";
     "workflow.success": "workflow.success";
     "workflow.failure": "workflow.failure";
     "workflow.complete": "workflow.complete";
+    "node.start": "node.start";
     "node.success": "node.success";
     "node.error": "node.error";
+    "node.finish": "node.finish";
     "gate.failure": "gate.failure";
   }> & z.core.$partial, z.ZodDiscriminatedUnion<[z.ZodObject<{
     failure: z.ZodDefault<z.ZodEnum<{
@@ -206,13 +206,13 @@ declare const mokaSubmitOptionsSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
   }, z.core.$strict>>;
   hooks: z.ZodOptional<z.ZodRecord<z.ZodEnum<{
     "workflow.start": "workflow.start";
-    "node.finish": "node.finish";
-    "node.start": "node.start";
     "workflow.success": "workflow.success";
     "workflow.failure": "workflow.failure";
     "workflow.complete": "workflow.complete";
+    "node.start": "node.start";
     "node.success": "node.success";
     "node.error": "node.error";
+    "node.finish": "node.finish";
     "gate.failure": "gate.failure";
   }> & z.core.$partial, z.ZodDiscriminatedUnion<[z.ZodObject<{
     failure: z.ZodDefault<z.ZodEnum<{

package/dist/runner-event-schema.d.ts

@@ -10,8 +10,8 @@ declare const runnerEventRecordSchema: z.ZodUnion<readonly [z.ZodObject<{
   at: z.ZodOptional<z.ZodString>;
   sequence: z.ZodNumber;
   type: z.ZodEnum<{
-    "workflow.planned": "workflow.planned";
     "workflow.start": "workflow.start";
+    "workflow.planned": "workflow.planned";
   }>;
   workflowPlan: z.ZodObject<{
     edges: z.ZodOptional<z.ZodArray<z.ZodObject<{
@@ -55,10 +55,10 @@ declare const runnerEventRecordSchema: z.ZodUnion<readonly [z.ZodObject<{
     }>;
   }, z.core.$strip>;
   type: z.ZodEnum<{
+    "node.start": "node.start";
+    "node.finish": "node.finish";
     "agent.finish": "agent.finish";
     "agent.start": "agent.start";
-    "node.finish": "node.finish";
-    "node.start": "node.start";
   }>;
 }, z.core.$strip>, z.ZodObject<{
   at: z.ZodOptional<z.ZodString>;
@@ -180,8 +180,8 @@ declare const runnerEventBatchSchema: z.ZodObject<{
     at: z.ZodOptional<z.ZodString>;
     sequence: z.ZodNumber;
     type: z.ZodEnum<{
-      "workflow.planned": "workflow.planned";
       "workflow.start": "workflow.start";
+      "workflow.planned": "workflow.planned";
     }>;
     workflowPlan: z.ZodObject<{
       edges: z.ZodOptional<z.ZodArray<z.ZodObject<{
@@ -225,10 +225,10 @@ declare const runnerEventBatchSchema: z.ZodObject<{
       }>;
     }, z.core.$strip>;
     type: z.ZodEnum<{
+      "node.start": "node.start";
+      "node.finish": "node.finish";
       "agent.finish": "agent.finish";
       "agent.start": "agent.start";
-      "node.finish": "node.finish";
-      "node.start": "node.start";
     }>;
   }, z.core.$strip>, z.ZodObject<{
     at: z.ZodOptional<z.ZodString>;

package/dist/runtime/select-candidate/select-candidate.js

@@ -1,13 +1,16 @@
+import { createRunnerLaunchPlan } from "../../runner.js";
+import { normalizeRunnerOutput } from "../../runner-output.js";
 import { parseJsonObject } from "../json-validation/json-validation.js";
 import "../json-validation/index.js";
 //#region src/runtime/select-candidate/select-candidate.ts
+const SCORE_RE = /-?\d+(?:\.\d+)?/;
 function selectBestCandidate(candidates) {
 	const passing = candidates.filter((candidate) => candidate.status === "PASS");
 	if (passing.length === 0) return null;
 	return passing.reduce((best, candidate) => (candidate.judgeScore ?? 0) > (best.judgeScore ?? 0) ? candidate : best);
 }
-function executeSelectCandidateBuiltin(context, node) {
-	const candidates = readCandidates(context, node?.needs.at(0) ?? null);
+async function executeSelectCandidateBuiltin(context, node) {
+	const candidates = await scoreCandidates(context, readCandidates(context, node?.needs.at(0) ?? null));
 	const selected = selectBestCandidate(candidates);
 	if (!selected) return {
 		evidence: [`select-candidate: no passing candidate among ${candidates.length}`, ...candidates.map((candidate) => `- ${candidate.nodeId}: FAIL`)],
@@ -20,6 +23,61 @@ function executeSelectCandidateBuiltin(context, node) {
 		output: selected.output
 	};
 }
+async function scoreCandidates(context, candidates) {
+	const model = context.config.best_of_n?.judge_model;
+	const runner = Object.keys(context.config.runners).at(0);
+	if (!(model && runner)) return candidates;
+	return await Promise.all(candidates.map((candidate) => scoreCandidate(context, candidate, runner, model)));
+}
+async function scoreCandidate(context, candidate, runner, model) {
+	const plan = judgePlan(context, candidate, runner, model);
+	context.agentInvocations.push(plan);
+	const judgeScore = parseScore(normalizeRunnerOutput(plan, (await context.executor(plan, { signal: context.signal })).stdout).output);
+	return judgeScore === null ? candidate : {
+		...candidate,
+		judgeScore
+	};
+}
+function judgePlan(context, candidate, runner, model) {
+	const profileId = `select-candidate:judge:${candidate.nodeId}`;
+	return createRunnerLaunchPlan({
+		...context.config,
+		profiles: {
+			...context.config.profiles,
+			[profileId]: {
+				filesystem: { mode: "read-only" },
+				instructions: { inline: "Score the candidate implementation." },
+				network: { mode: "disabled" },
+				output: { format: "text" },
+				runner,
+				tools: []
+			}
+		}
+	}, {
+		model,
+		nodeId: profileId,
+		profileId,
+		prompt: judgePrompt(context.task, candidate.output),
+		worktreePath: context.worktreePath
+	});
+}
+function judgePrompt(task, output) {
+	return [
+		"Score how well this candidate implementation satisfies the task.",
+		"Return ONLY a number between 0 and 1 (1 = best). No prose, no fences.",
+		"",
+		`Task: ${task}`,
+		"",
+		"Candidate result:",
+		output
+	].join("\n");
+}
+function parseScore(text) {
+	const match = SCORE_RE.exec(text);
+	if (!match) return null;
+	const value = Number(match[0]);
+	return Number.isFinite(value) ? Math.max(0, Math.min(1, value)) : null;
+}
 function readCandidates(context, upstreamNodeId) {
 	if (!upstreamNodeId) return [];
 	const upstream = context.plan.graph.node(upstreamNodeId);

package/package.json

@@ -121,7 +121,7 @@
     "prepack": "bun run build:cli"
   },
   "type": "module",
-  "version": "2.5.0",
+  "version": "2.6.0",
   "description": "Config-driven multi-agent pipeline runner for repository work",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",