@bastani/atomic 0.9.14-alpha.4 → 0.9.14-alpha.6

package/dist/builtin/workflows/builtin/ralph-core.ts

@@ -1,6 +1,7 @@
 import { mkdir, writeFile } from "node:fs/promises";
 import { dirname, join } from "node:path";
 import { Type } from "typebox";
+import { VERIFICATION_SCALE } from "./verification-criteria.js";
 import type { WorkflowTaskResult } from "../src/shared/types.js";
 import { createWorkflowArtifactDirectory } from "../src/shared/workflow-artifacts.js";
 import {
@@ -19,6 +20,8 @@ import {
   type ParsedReviewDecision,
   type ReviewConvergenceSummary,
 } from "./review-convergence.js";
+import type { ConvergenceEntry } from "./goal-convergence.js";
+import type { ReverifyAuditEntry } from "./goal-reverify.js";
 
 
 export const DEFAULT_MAX_LOOPS = 10;
@@ -104,6 +107,12 @@ export const reviewDecisionSchema = Type.Object(
     ]),
     overall_explanation: Type.String(),
     overall_confidence_score: Type.Number({ minimum: 0, maximum: 1 }),
+    criterion_scores: Type.Optional(
+      Type.Array(Type.Object({
+        criterion_id: Type.String(),
+        score: VERIFICATION_SCALE.schema,
+      }, { additionalProperties: false })),
+    ),
     requirements_traceability: Type.Array(requirementsTraceabilitySchema),
     stop_review_loop: Type.Boolean(),
     reviewer_error: Type.Optional(
@@ -294,7 +303,9 @@ type ReviewArtifact = {
 
 type ReviewRoundArtifact = {
   readonly convergence_decision: ReviewConvergenceSummary;
+  readonly convergence: readonly ConvergenceEntry[];
   readonly consolidated_findings?: readonly ConsolidatedFinding<ReviewFinding>[];
+  readonly reverification?: readonly ReverifyAuditEntry<ReviewFinding>[];
   readonly reviews: readonly {
     readonly reviewer: string;
     readonly artifact_path: string;

package/dist/builtin/workflows/builtin/ralph-review-gate.ts

@@ -64,6 +64,7 @@ export type RequirementTraceability = {
 
 export type ReviewDecision = {
   readonly findings: readonly ReviewFinding[];
+  readonly criterion_scores?: readonly { readonly criterion_id: string; readonly score: number }[];
   readonly overall_correctness: "patch is correct" | "patch is incorrect";
   readonly overall_explanation: string;
   readonly overall_confidence_score: number;

package/dist/builtin/workflows/builtin/ralph-reviewer-prompt.ts

@@ -7,6 +7,7 @@ import {
   REVIEW_CODE_DELTA_CONTRACT,
   REVIEWER_INDEPENDENT_VERIFICATION_CONTRACT,
   REVIEWER_INTERCOM_COORDINATION_PROTOCOL,
+  REVIEWER_CALIBRATION_RULES,
   REVIEWER_OVERIMPLEMENTATION_GUARD,
   REVIEWER_SPEC_VS_OBJECTIVE_GUARD,
   WORKTREE_DISCIPLINE_CONTRACT,
@@ -42,6 +43,7 @@ export function renderRalphReviewerPrompt(args: {
     ["literal_contract", LITERAL_OBJECTIVE_CONTRACT],
     ["acceptance_matrix", ACCEPTANCE_MATRIX_CONTRACT],
     ["independent_verification", REVIEWER_INDEPENDENT_VERIFICATION_CONTRACT],
+    ["calibration", REVIEWER_CALIBRATION_RULES],
     ["code_delta_review", REVIEW_CODE_DELTA_CONTRACT],
     ["worktree_discipline", WORKTREE_DISCIPLINE_CONTRACT],
     ["reviewer_coordination", REVIEWER_INTERCOM_COORDINATION_PROTOCOL],

package/dist/builtin/workflows/builtin/ralph-runner.ts

@@ -1,6 +1,12 @@
 import { existsSync } from "node:fs";
 import { join, resolve } from "node:path";
-import type { WorkflowRunContext, WorkflowTaskResult } from "../src/shared/types.js";
+import type { WorkflowRunContext, WorkflowTaskOptions, WorkflowTaskResult } from "../src/shared/types.js";
+import { fold_usage } from "./verification-usage.js";
+import {
+  convergence_escalation_evidence,
+  record_convergence,
+  type ConvergenceEntry,
+} from "./goal-convergence.js";
 import { createWorkflowArtifactDirectory } from "../src/shared/workflow-artifacts.js";
 import {
   ACCEPTANCE_MATRIX_CONTRACT,
@@ -41,6 +47,7 @@ import {
   type RalphWorkflowResult,
 } from "./ralph-core.js";
 import { consolidateFindingsBatch, summarizeReviewConvergence } from "./review-convergence.js";
+import { reverify_consolidated_batch } from "./goal-reverify.js";
 import {
   orchestratorModelConfig,
   promptEngineerModelConfig,
@@ -69,6 +76,7 @@ export async function runRalphWorkflow(
   let approved = false;
   let iterationsCompleted = 0;
   let previousResearchPromptRefinementSessionFile: string | undefined;
+  const convergenceEntries: ConvergenceEntry[] = [];
   let previousResearchSessionFile: string | undefined;
   let previousOrchestratorSessionFile: string | undefined;
   for (let iteration = 1; iteration <= maxLoops; iteration += 1) {
@@ -203,6 +211,7 @@ export async function runRalphWorkflow(
       createPr,
     });
     let reviews: WorkflowTaskResult[];
+    let reviewerBatchFailed = false;
     try {
       reviews = await ctx.parallel(
         [
@@ -234,6 +243,7 @@ export async function runRalphWorkflow(
         },
       );
     } catch (err) {
+      reviewerBatchFailed = true;
       reviews = [reviewerErrorResult(err)];
     }
     const reviewEntries = await Promise.all(reviews.map(async (review) => {
@@ -262,6 +272,7 @@ export async function runRalphWorkflow(
         convergence_decision: convergenceDecision,
       };
     }));
+    const roundProducedDecisions = reviewEntries.some((review) => review.convergence_decision.parsed);
     const approvalCount = reviewEntries.filter((review) =>
       review.convergence_decision.approved,
     ).length;
@@ -279,19 +290,54 @@ export async function runRalphWorkflow(
       finalActionRemaining: approved && createPr,
       diagnostics: reviewEntries.flatMap((review) => review.convergence_decision.diagnostics),
     });
+    const consolidatedFindings = consolidateFindingsBatch(
+      reviewEntries.map((review) => ({
+        reviewer: review.reviewer,
+        findings: review.decision.findings,
+      })),
+    );
+    const reverifyResults: WorkflowTaskResult[] = [];
+    const reverifyContext = {
+      task: async (name: string, taskOptions: WorkflowTaskOptions): Promise<WorkflowTaskResult> => {
+        const result = await ctx.task(name, taskOptions);
+        reverifyResults.push(result);
+        return result;
+      },
+    };
+    const reverified = approved
+      ? { batch: consolidatedFindings, audits: [] as const }
+      : await reverify_consolidated_batch(reverifyContext, {
+          batch: consolidatedFindings,
+          context: {
+            objective: workflowPrompt,
+            candidateRefs: [researchPath, implementationNotesPath, orchestratorReportPath],
+          },
+        });
+    const findings = reviewEntries.flatMap((review) => review.decision.findings);
+    const traceability = reviewEntries.flatMap((review) => review.decision.requirements_traceability);
+    // A thrown reviewer batch or an all-unparsed reviewer batch produced no
+    // decisions, so recording a zero-blocker round would fabricate progress
+    // and can suppress the escalation evidence on the very escalation it triggers.
+    if (!reviewerBatchFailed && roundProducedDecisions) {
+      convergenceEntries.push(record_convergence({
+        unresolvedBlockingCount: reverified.batch.filter((entry) => entry.blocking).length,
+        meanFindingConfidence: findings.length === 0
+          ? null
+          : findings.reduce((total, finding) => total + finding.confidence_score, 0) / findings.length,
+        fractionProven: traceability.length === 0
+          ? 0
+          : traceability.filter((entry) => entry.status === "proven").length / traceability.length,
+        demotions: reverified.audits.filter((audit) => audit.verdict === "demoted").length,
+        usage: fold_usage([orchestrator, ...reviews, ...reverifyResults]),
+      }));
+    }
     latestReviewReportPath = await writeJsonArtifact(
       join(artifactDir, "review-round-latest.json"),
       {
         convergence_decision: roundConvergenceDecision,
-        // Deduplicated cross-reviewer findings batch so the next research and
-        // orchestrator passes repair the round's findings together instead of
-        // one at a time.
-        consolidated_findings: consolidateFindingsBatch(
-          reviewEntries.map((review) => ({
-            reviewer: review.reviewer,
-            findings: review.decision.findings,
-          })),
-        ),
+        convergence: convergenceEntries,
+        consolidated_findings: reverified.batch,
+        reverification: reverified.audits,
         reviews: reviewEntries,
       },
     );
@@ -303,6 +349,9 @@ export async function runRalphWorkflow(
   // carrying the unresolved blocking findings. Without create_pr the workflow
   // never touches a PR, approved or not.
   const unapprovedHandoff = createPr === true && !approved;
+  const escalationEvidence = unapprovedHandoff
+    ? convergence_escalation_evidence(convergenceEntries)
+    : [];
   if (createPr === true) {
     const prResult = await ctx.task("pull-request", {
       prompt: taggedPrompt([
@@ -319,6 +368,7 @@ export async function runRalphWorkflow(
               : unapprovedHandoff
                 ? `Final unapproved review-round artifact: ${latestReviewReportPath}`
                 : `Approved review-round artifact: ${latestReviewReportPath}`,
+            ...escalationEvidence,
           ].join("\n"),
         ],
         ...(unapprovedHandoff

package/dist/builtin/workflows/builtin/selection-math.ts

@@ -0,0 +1,156 @@
+/** Pure, seeded selection math for the probabilistic pivot tournament. */
+import { VERIFICATION_SCALE } from "./verification-criteria.js";
+
+export interface DirectedPair {
+	a: number;
+	b: number;
+}
+
+export interface ScoringJob {
+	a: number;
+	b: number;
+	criterionId: string;
+	rep: number;
+	swapped: boolean;
+}
+
+export interface ComparisonPlan {
+	ring: DirectedPair[];
+	pivotRounds: (pivots: number[]) => DirectedPair[];
+	jobs: (pairs: readonly DirectedPair[], criterionIds: readonly string[]) => ScoringJob[];
+}
+
+export interface Preference {
+	a: number;
+	b: number;
+	p: number;
+}
+
+export interface RankingEntry {
+	index: number;
+	meanPreference: number;
+}
+
+export type Ranking = RankingEntry[];
+
+const UINT32_RANGE = 0x1_0000_0000;
+const MULBERRY_INCREMENT = 0x6d2b79f5;
+
+/** Return a deterministic Mulberry32-class random-float generator. */
+export function seeded_rng(seed: number): () => number {
+	let state = seed >>> 0;
+	return () => {
+		state = (state + MULBERRY_INCREMENT) >>> 0;
+		let t = state;
+		t = Math.imul(t ^ (t >>> 15), t | 1);
+		t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
+		return ((t ^ (t >>> 14)) >>> 0) / UINT32_RANGE;
+	};
+}
+
+function ring_cycle(n: number, rng: () => number): DirectedPair[] {
+	const permutation = Array.from({ length: n }, (_, index) => index);
+	for (let index = n - 1; index > 0; index -= 1) {
+		const other = Math.floor(rng() * (index + 1));
+		[permutation[index], permutation[other]] = [permutation[other]!, permutation[index]!];
+	}
+	return permutation.map((a, index) => ({ a, b: permutation[(index + 1) % n]! }));
+}
+
+function pair_key(a: number, b: number): string {
+	return a < b ? `${a}:${b}` : `${b}:${a}`;
+}
+
+
+/** Build a deterministic ring and expose the later pivot/job phases as pure closures. */
+export function plan_comparisons(input: {
+	n: number;
+	pivots: number;
+	repeats: number;
+	seed: number;
+}): ComparisonPlan {
+	const { n, pivots: pivotCount, repeats, seed } = input;
+	if (n < 2) throw new RangeError("n must be at least 2");
+	if (pivotCount < 1) throw new RangeError("pivots must be at least 1");
+	if (repeats < 1) throw new RangeError("repeats must be at least 1");
+
+	const ring = ring_cycle(n, seeded_rng(seed));
+	return {
+		ring,
+		pivotRounds: (pivots) => {
+			const scheduled = new Set(ring.map(({ a, b }) => pair_key(a, b)));
+			const pivotSet = new Set(pivots);
+			const pairs: DirectedPair[] = [];
+			const add = (a: number, b: number): void => {
+				if (a === b) return;
+				const key = pair_key(a, b);
+				if (scheduled.has(key)) return;
+				scheduled.add(key);
+				pairs.push({ a, b });
+			};
+			for (let candidate = 0; candidate < n; candidate += 1) {
+				if (pivotSet.has(candidate)) continue;
+				for (const pivot of pivots) add(candidate, pivot);
+			}
+			for (let first = 0; first < pivots.length; first += 1) {
+				for (let second = first + 1; second < pivots.length; second += 1) {
+					add(pivots[first]!, pivots[second]!);
+				}
+			}
+			return pairs;
+		},
+		jobs: (pairs, criterionIds) => {
+			const jobs: ScoringJob[] = [];
+			for (const pair of pairs) {
+				for (const criterionId of criterionIds) {
+					for (let rep = 0; rep < repeats; rep += 1) {
+						jobs.push({ a: pair.a, b: pair.b, criterionId, rep, swapped: rep % 2 === 1 });
+					}
+				}
+			}
+			return jobs;
+		},
+	};
+}
+
+/** Convert a verification score to the reward range used by Bradley-Terry. */
+function normalize_score(score: number): number {
+	return (score - VERIFICATION_SCALE.min) / (VERIFICATION_SCALE.max - VERIFICATION_SCALE.min);
+}
+
+/** Return the Bradley-Terry soft preference for the first score. */
+export function soft_win(scoreA: number, scoreB: number): number {
+	const difference = normalize_score(scoreA) - normalize_score(scoreB);
+	return 1 / (1 + Math.exp(-difference));
+}
+
+/** Add each preference to both candidates' win mass and comparison count. */
+export function accumulate(prefs: readonly Preference[], w: number[], c: number[]): void {
+	for (const { a, b, p } of prefs) {
+		w[a]! += p;
+		c[a]! += 1;
+		w[b]! += 1 - p;
+		c[b]! += 1;
+	}
+}
+
+function mean_preference(w: number[], c: number[], index: number): number {
+	return c[index] === 0 ? 0 : w[index]! / c[index]!;
+}
+
+function candidate_order(w: number[], c: number[]): number[] {
+	return Array.from({ length: w.length }, (_, index) => index).sort((left, right) => {
+		const difference = mean_preference(w, c, right) - mean_preference(w, c, left);
+		return difference === 0 ? left - right : difference;
+	});
+}
+
+/** Return the top-k candidate indices, resolving equal means by index. */
+export function select_pivots(w: number[], c: number[], k: number): number[] {
+	return candidate_order(w, c).slice(0, k);
+}
+
+/** Return every candidate ordered by mean preference, including unscored ones. */
+export function rank_candidates(w: number[], c: number[]): Ranking {
+	return candidate_order(w, c).map((index) => ({ index, meanPreference: mean_preference(w, c, index) }));
+}

package/dist/builtin/workflows/builtin/shared-prompts.ts

@@ -49,6 +49,11 @@ export function withSteeringPropagation(prompt: string): string {
   return `${prompt.slice(0, instructionAt)}\n\n${tagged}${prompt.slice(instructionAt)}`;
 }
 
+export const REVIEWER_CALIBRATION_RULES = [
+  "Trust observed output over the agent's narration.",
+  'Agent declarations of success ("done", "all tests pass") are ZERO evidence on their own.',
+].join("\n");
+
 export const WORKER_PREFLIGHT_CONTRACT = [
   "Before implementation delegation, infer the checkout's language, framework, build system, and setup requirements from repository evidence rather than ecosystem assumptions.",
   "Inspect source layout, setup docs, manifests, lockfiles, toolchain and codegen files, CI/workflow configuration, scripts, and generated-artifact conventions for missing dependencies, generated files, toolchains, submodules, or other initialization artifacts.",

package/dist/builtin/workflows/builtin/tournament-prompts.ts

@@ -3,86 +3,68 @@ type PromptSection = readonly [tag: string, content: string];
 const GROUNDED_REPORTING = "Before reporting progress, audit each claim against a tool result from this session. Report only work you can point to evidence for; say so explicitly when something is unverified.";
 const READABLE_REPORT = "Lead with the outcome. Keep facts, decisions, caveats, and next steps; drop background and repetition. Use complete, readable sentences rather than compressed fragments.";
 
+/** The default three-criterion decomposition of the tournament rubric. */
+export const DEFAULT_TOURNAMENT_CRITERIA = [
+	{ name: "Correctness", description: "Satisfies the task without material errors." },
+	{ name: "Completeness", description: "Covers required outcomes and important edge cases." },
+	{
+		name: "Evidence and task fit",
+		description: "Supports claims with observable evidence or checks and is directly usable without irrelevant work.",
+	},
+] as const;
+
 function taggedPrompt(sections: readonly PromptSection[]): string {
-  return sections
-    .map(([tag, content]) => `<${tag}>\n${content.trim()}\n</${tag}>`)
-    .join("\n\n");
+	return sections
+		.map(([tag, content]) => `<${tag}>\n${content.trim()}\n</${tag}>`)
+		.join("\n\n");
 }
 
 export function renderTournamentAttemptPrompt(task: string, attempt: number): string {
-  return taggedPrompt([
-    ["role", "You are an independent solution author competing on solution quality."],
-    ["attempt", `Produce attempt ${attempt} without assuming another attempt's approach or conclusions.`],
-    ["success_criteria", "A judge can evaluate this artifact directly against correctness, completeness, evidence, and task fit."],
-    ["requirements", [
-      "Deliver a complete, self-contained solution rather than commentary about how to solve it.",
-      "Ground important claims in observable evidence or executable checks.",
-      "State assumptions, limitations, and validation performed.",
-      "Optimize for correctness and usefulness, not length.",
-      GROUNDED_REPORTING,
-    ].join("\n")],
-    ["stop_rules", "Stop when the solution is complete, supported, validated where practical, and its residual risks are stated."],
-    ["output_format", `Markdown with Solution, Evidence and validation, Assumptions, and Residual risks. ${READABLE_REPORT}`],
-    ["objective", task],
-  ]);
-}
-
-export function renderPairwiseJudgePrompt(options: {
-  readonly task: string;
-  readonly firstLabel: string;
-  readonly secondLabel: string;
-  readonly firstPath: string;
-  readonly secondPath: string;
-}): string {
-  return taggedPrompt([
-    ["candidates", [
-      `First presentation: ${options.firstLabel} at ${options.firstPath}`,
-      `Second presentation: ${options.secondLabel} at ${options.secondPath}`,
-      "Read both files completely before deciding.",
-    ].join("\n")],
-    ["role", "You are an impartial pairwise judge. Evaluate only the supplied artifacts."],
-    ["rubric", [
-      "1. Correctness: satisfies the task without material errors.",
-      "2. Completeness: covers required outcomes and important edge cases.",
-      "3. Evidence: supports claims with observable evidence or checks.",
-      "4. Task fit: is directly usable and avoids irrelevant work.",
-    ].join("\n")],
-    ["decision_rules", [
-      "Choose exactly one presented candidate; do not merge or rewrite them.",
-      "Ignore presentation order, writing length, and stylistic polish unless they affect the rubric.",
-      "Cite observable evidence from both artifacts and give concise criteria-based justification.",
-      GROUNDED_REPORTING,
-    ].join("\n")],
-    ["success_criteria", "The selected winner is traceable to short rubric-grounded evidence from both candidates."],
-    ["stop_rules", "Stop after selecting one candidate and supporting the choice against every rubric criterion."],
-    ["output_format", `Return only the required structured decision with winner, rationale, and evidence. ${READABLE_REPORT}`],
-    ["objective", options.task],
-  ]);
+	return taggedPrompt([
+		["role", "You are an independent solution author competing on solution quality."],
+		["attempt", `Produce attempt ${attempt} without assuming another attempt's approach or conclusions.`],
+		["success_criteria", "A judge can evaluate this artifact directly against correctness, completeness, evidence, and task fit."],
+		["requirements", [
+			"Deliver a complete, self-contained solution rather than commentary about how to solve it.",
+			"Ground important claims in observable evidence or executable checks.",
+			"State assumptions, limitations, and validation performed.",
+			"Optimize for correctness and usefulness, not length.",
+			GROUNDED_REPORTING,
+		].join("\n")],
+		["stop_rules", "Stop when the solution is complete, supported, validated where practical, and its residual risks are stated."],
+		["output_format", `Markdown with Solution, Evidence and validation, Assumptions, and Residual risks. ${READABLE_REPORT}`],
+		["objective", task],
+	]);
 }
 
-export function renderBracketReducerPrompt(options: {
-  readonly task: string;
-  readonly bracketPath: string;
-  readonly winnerLabel: string;
-  readonly winnerPath: string;
+export function renderComparisonsReducerPrompt(options: {
+	readonly task: string;
+	readonly comparisonsPath: string;
+	readonly ranking: readonly { readonly label: string; readonly meanPreference: number }[];
+	readonly winnerLabel: string;
+	readonly winnerPath: string;
 }): string {
-  return taggedPrompt([
-    ["artifacts", [
-      `Bracket ledger: ${options.bracketPath}`,
-      `Winning artifact (${options.winnerLabel}): ${options.winnerPath}`,
-      "Read both files before reporting.",
-    ].join("\n")],
-    ["role", "You are the tournament bracket reducer and final reporter."],
-    ["requirements", [
-      "Return the winning solution faithfully; do not silently combine losing material into it.",
-      "Summarize why it advanced using the recorded pairwise rationale and evidence.",
-      "Call out bracket byes and any limitations recorded by judges.",
-      "Cite the bracket ledger and winning artifact paths.",
-      GROUNDED_REPORTING,
-    ].join("\n")],
-    ["success_criteria", "A reader can use the winning solution and audit every comparison that selected it."],
-    ["stop_rules", "Stop after faithfully presenting the winner and an auditable decision trail, including byes and limitations."],
-    ["output_format", `Markdown with Winner, Winning solution, Decision trail, Evidence, and Residual risks. ${READABLE_REPORT}`],
-    ["objective", options.task],
-  ]);
+	const rankingText = options.ranking
+		.map((entry, index) => `${index + 1}. ${entry.label} (mean preference ${entry.meanPreference})`)
+		.join("\n");
+	return taggedPrompt([
+		["artifacts", [
+			`Comparisons ledger: ${options.comparisonsPath}`,
+			`Winning artifact (${options.winnerLabel}): ${options.winnerPath}`,
+			"Read the ledger and winning artifact before reporting.",
+		].join("\n")],
+		["role", "You are the soft-scored tournament reducer and final reporter."],
+		["ranking", rankingText],
+		["requirements", [
+			"Return the winning solution faithfully; do not silently combine losing material into it.",
+			"Report the full ranking exactly as recorded in the comparisons ledger.",
+			"Identify notable score disagreements, invalid reports, or other limitations recorded by judges.",
+			"Cite the comparisons ledger and winning artifact paths.",
+			GROUNDED_REPORTING,
+		].join("\n")],
+		["success_criteria", "A reader can use the winning solution and recompute the ranking from the durable comparisons ledger."],
+		["stop_rules", "Stop after faithfully presenting the winner, full ranking, decision evidence, and residual risks."],
+		["output_format", `Markdown with Winner, Full ranking, Decision trail, Evidence, Notable disagreements, and Residual risks. ${READABLE_REPORT}`],
+		["objective", options.task],
+	]);
 }