@bastani/atomic 0.9.14-alpha.4 → 0.9.14-alpha.6

package/dist/builtin/workflows/builtin/generate-and-filter-runner.ts

@@ -1,4 +1,4 @@
-import { writeFile } from "node:fs/promises";
+import { readFile, writeFile } from "node:fs/promises";
 import { join } from "node:path";
 import { Type, type Static } from "typebox";
 import type { WorkflowRunContext, WorkflowSerializableValue } from "../src/shared/types.js";
@@ -61,19 +61,23 @@ export async function runGenerateAndFilter(ctx: WorkflowRunContext<Inputs>): Pro
   let shortlist = filteredShortlist.length > 0 ? filteredShortlist : fallbackShortlist;
   let judgePath: string | null = null;
   let decisionPath = filterPath;
-  if (ctx.inputs.use_judge) {
-    judgePath = join(root, "judge.json");
-    const judged = await ctx.task("judge", {
-      prompt: renderJudgePrompt(ctx.inputs.prompt, filterPath, shortlistLimit), context: "fresh",
-      reads: [filterPath, ...shortlist], schema: judgeSchema,
-    });
-    const judgedDecision = judgeDecision(judged.structured);
-    // Same inter-stage contract as the filter report above.
-    await writeFile(judgePath, `${JSON.stringify(judgedDecision ?? { shortlist: [], rationale: "Judge stage produced no valid structured decision." }, null, 2)}\n`);
-    const judgedShortlist = selectCandidates(judgedDecision?.shortlist ?? []);
-    shortlist = judgedShortlist.length > 0 ? judgedShortlist : shortlist;
-    decisionPath = judgePath;
-  }
+	if (ctx.inputs.use_judge) {
+		judgePath = join(root, "judge.json");
+		const judgeCandidates = await Promise.all(
+			shortlist.map(async (path) => ({ path, body: await readFile(path, "utf8") })),
+		);
+		const judgePrompt = renderJudgePrompt(ctx.inputs.prompt, filterPath, shortlistLimit, judgeCandidates);
+		const judged = await ctx.task("judge", {
+			prompt: judgePrompt.prompt, context: "fresh",
+			reads: judgePrompt.reads, schema: judgeSchema,
+		});
+		const judgedDecision = judgeDecision(judged.structured);
+		// Same inter-stage contract as the filter report above.
+		await writeFile(judgePath, `${JSON.stringify(judgedDecision ?? { shortlist: [], rationale: "Judge stage produced no valid structured decision." }, null, 2)}\n`);
+		const judgedShortlist = selectCandidates(judgedDecision?.shortlist ?? []);
+		shortlist = judgedShortlist.length > 0 ? judgedShortlist : shortlist;
+		decisionPath = judgePath;
+	}
   const finalPath = join(root, "shortlist.md");
   const finalShortlist = await ctx.task("final-shortlist", {
     prompt: renderFinalShortlistPrompt(ctx.inputs.prompt, decisionPath), context: "fresh",

package/dist/builtin/workflows/builtin/goal-artifacts.ts

@@ -3,8 +3,10 @@ import { join } from "node:path";
 import type { ReviewDecision, ReviewRecord } from "./goal-types.js";
 import {
   consolidateFindingsBatch,
+  type ConsolidatedFinding,
   type ReviewConvergenceSummary,
 } from "./review-convergence.js";
+import type { ReverifyAuditEntry, ReverifiableFinding } from "./goal-reverify.js";
 
 export function artifactSafeName(value: string): string {
   const safe = value
@@ -42,20 +44,19 @@ export async function writeReviewArtifact(
 export async function writeReviewRoundArtifact(
   artifactDir: string,
   reviews: readonly ReviewRecord[],
-): Promise<string> {
-  const artifactPath = join(artifactDir, "review-round-latest.json");
-  const visibleReviews = reviews.map(withoutTurn);
-  // Deduplicated cross-reviewer findings batch so the next orchestrator turn can
-  // plan and repair the round's findings together instead of one at a time.
-  const consolidatedFindings = consolidateFindingsBatch(
+  consolidatedFindings: readonly ConsolidatedFinding<ReverifiableFinding>[] = consolidateFindingsBatch(
     reviews.map((review) => ({
       reviewer: review.reviewer,
       findings: review.findings,
     })),
-  );
+  ),
+  reverification: readonly ReverifyAuditEntry<ReverifiableFinding>[] = [],
+): Promise<string> {
+  const artifactPath = join(artifactDir, "review-round-latest.json");
+  const visibleReviews = reviews.map(withoutTurn);
   await writeFile(
     artifactPath,
-    `${JSON.stringify({ reviews: visibleReviews, consolidated_findings: consolidatedFindings }, null, 2)}\n`,
+    `${JSON.stringify({ reviews: visibleReviews, consolidated_findings: consolidatedFindings, reverification }, null, 2)}\n`,
     { encoding: "utf8" },
   );
   return artifactPath;

package/dist/builtin/workflows/builtin/goal-convergence.ts

@@ -0,0 +1,87 @@
+/**
+ * Convergence evidence is observational only: it never approves, never
+ * terminates, never touches `stop_review_loop`, and feeds only
+ * `needs_human`-class escalation text.
+ */
+import type { UsageTotals } from "./verification-usage.js";
+import {
+	classify_trend,
+	DEFAULT_FALL_DELTA,
+	DEFAULT_RISE_DELTA,
+	type TrendResult,
+} from "./progress-scoring.js";
+import { VERIFICATION_SCALE } from "./verification-criteria.js";
+
+/**
+ * `fractionProven` remains in its natural [0, 1] units. V7's hysteresis
+ * thresholds span a proportional part of `VERIFICATION_SCALE`, so applying
+ * that same proportion to the unit interval keeps the two series comparable
+ * without hard-coding a fraction-specific magic number.
+ */
+export const FRACTION_TREND_RISE_DELTA =
+	DEFAULT_RISE_DELTA / (VERIFICATION_SCALE.max - VERIFICATION_SCALE.min);
+export const FRACTION_TREND_FALL_DELTA =
+	DEFAULT_FALL_DELTA / (VERIFICATION_SCALE.max - VERIFICATION_SCALE.min);
+
+export type ConvergenceEntry = {
+	readonly unresolvedBlockingCount: number;
+	readonly meanFindingConfidence: number | null;
+	readonly fractionProven: number;
+	readonly demotions: number;
+	readonly usage: UsageTotals;
+};
+
+/** Purely shape one already-folded review round into a ledger entry. */
+export function record_convergence(round: {
+	readonly unresolvedBlockingCount: number;
+	readonly meanFindingConfidence: number | null;
+	readonly fractionProven: number;
+	readonly demotions: number;
+	readonly usage: UsageTotals;
+}): ConvergenceEntry {
+	return {
+		unresolvedBlockingCount: round.unresolvedBlockingCount,
+		meanFindingConfidence: round.meanFindingConfidence,
+		fractionProven: round.fractionProven,
+		demotions: round.demotions,
+		usage: round.usage,
+	};
+}
+
+export function classify_convergence(entries: readonly ConvergenceEntry[]): {
+	blocking: TrendResult;
+	proven: TrendResult;
+} {
+	return {
+		blocking: classify_trend(entries.map((entry) => entry.unresolvedBlockingCount)),
+		proven: classify_trend(entries.map((entry) => entry.fractionProven), {
+			riseDelta: FRACTION_TREND_RISE_DELTA,
+			fallDelta: FRACTION_TREND_FALL_DELTA,
+		}),
+	};
+}
+
+/**
+ * A FALLING `unresolvedBlockingCount` (classified "regressing" on the raw
+ * series) and a RISING `fractionProven` are the converging directions. A
+ * rising blocking count is never suppressed because it is the worsening
+ * direction on the axis the objective names first.
+ */
+export function convergence_escalation_evidence(
+	entries: readonly ConvergenceEntry[],
+): readonly string[] {
+	if (entries.length === 0) return [];
+	const { blocking, proven } = classify_convergence(entries);
+	if (blocking.trend === "regressing" || (proven.trend === "rising" && blocking.trend !== "rising")) return [];
+
+	const latest = entries[entries.length - 1]?.meanFindingConfidence;
+	return [
+		`${entries.length} round${entries.length === 1 ? "" : "s"} recorded; no observed convergence: the blocking-count trend is ${blocking.trend}.`,
+		`Blocking-count trend: ${blocking.trend}; raw series: ${JSON.stringify(blocking.evidence.series)}.`,
+		`Fraction-proven trend: ${proven.trend}; raw series: ${JSON.stringify(proven.evidence.series)}.`,
+		latest === null
+			? "Latest mean finding confidence: no findings were filed."
+			: `Latest mean finding confidence: ${latest}.`,
+		"This is escalation EVIDENCE only; it never approves or terminates anything.",
+	];
+}

package/dist/builtin/workflows/builtin/goal-ledger.ts

@@ -34,6 +34,8 @@ function modelVisibleLedger(ledger: GoalLedger): ModelVisibleGoalLedger {
     blockers: ledger.blockers.map(withoutTurn),
     decisions: ledger.decisions.map(withoutTurn),
     lifecycle: ledger.lifecycle.map(withoutTurn),
+    reverification: ledger.reverification ?? [],
+    convergence: ledger.convergence ?? [],
   };
 }
 
@@ -73,6 +75,8 @@ export async function createGoalLedger(
     blockers: [],
     decisions: [],
     lifecycle: [],
+    reverification: [],
+    convergence: [],
   };
   appendLifecycleEvent(ledger, "created", "Goal created.", 0);
   const ledgerPath = join(artifactDir, LEDGER_FILENAME);

package/dist/builtin/workflows/builtin/goal-prompts.ts

@@ -9,6 +9,7 @@ import {
   REVIEW_CODE_DELTA_CONTRACT,
   REVIEWER_INDEPENDENT_VERIFICATION_CONTRACT,
   REVIEWER_INTERCOM_COORDINATION_PROTOCOL,
+  REVIEWER_CALIBRATION_RULES,
   REVIEWER_OVERIMPLEMENTATION_GUARD,
   REVIEWER_SPEC_VS_OBJECTIVE_GUARD,
   SCOPE_DISCIPLINE_CONTRACT,
@@ -159,6 +160,7 @@ export function renderReviewerPrompt(args: {
     ["literal_contract", LITERAL_OBJECTIVE_CONTRACT],
     ["acceptance_criteria", ACCEPTANCE_MATRIX_CONTRACT],
     ["independent_verification", REVIEWER_INDEPENDENT_VERIFICATION_CONTRACT],
+    ["calibration", REVIEWER_CALIBRATION_RULES],
     ["code_delta_review", REVIEW_CODE_DELTA_CONTRACT],
     ["reviewer_coordination", REVIEWER_INTERCOM_COORDINATION_PROTOCOL],
     ["regression_evidence", REGRESSION_EVIDENCE_CONTRACT],

package/dist/builtin/workflows/builtin/goal-reducer.ts

@@ -1,3 +1,5 @@
+import type { ConvergenceEntry } from "./goal-convergence.js";
+import { convergence_escalation_evidence } from "./goal-convergence.js";
 import type { BlockerObservation, GoalLedger, ReducerOutcome, ReviewRecord } from "./goal-types.js";
 import {
   summarizeReviewConvergence,
@@ -87,6 +89,7 @@ export function reduceGoalDecision(
     readonly reviewQuorum: number;
     readonly blockerThreshold: number;
     readonly nextActionOnComplete: ReviewNextAction;
+    readonly convergence?: readonly ConvergenceEntry[];
   },
 ): ReducerOutcome {
   const completeVotes = turnReviews.filter(
@@ -140,6 +143,8 @@ export function reduceGoalDecision(
   }
 
   if (options.turn >= options.maxTurns) {
+    const baseReason = `Orchestrator attempt budget reached without reviewer quorum. Remaining work: ${collectRemainingWork(turnReviews)}`;
+    const evidence = convergence_escalation_evidence(options.convergence ?? []);
     return {
       status: "needs_human",
       blockerObservation: observation,
@@ -147,7 +152,7 @@ export function reduceGoalDecision(
         ...reducerSummary(turnReviews, false, "needs_human"),
         turn: options.turn,
         decision: "needs_human",
-        reason: `Orchestrator attempt budget reached without reviewer quorum. Remaining work: ${collectRemainingWork(turnReviews)}`,
+        reason: [baseReason, ...evidence].join("\n"),
         complete_votes: completeVotes,
         review_quorum: options.reviewQuorum,
         ...(observation ? { blocker: observation.blocker } : {}),

package/dist/builtin/workflows/builtin/goal-reverify.ts

@@ -0,0 +1,305 @@
+import { Type } from "typebox";
+import type {
+	WorkflowRunContext,
+	WorkflowSerializableValue,
+	WorkflowTaskResult,
+} from "../src/shared/types.js";
+import {
+	build_scoring_prompt,
+	scoring_prompt_reads,
+	type ScoringCandidate,
+	type SharedHead,
+} from "./verification-prompts.js";
+import { VERIFICATION_SCALE } from "./verification-criteria.js";
+import type {
+	ConsolidatableFinding,
+	ConsolidatedFinding,
+} from "./review-convergence.js";
+
+/** Confidence values strictly below this threshold can enter re-verification. */
+export const DEFAULT_REVERIFY_THRESHOLD = 0.7;
+
+const STANDARD_CONFIRM_THRESHOLD = 10;
+const REQUIRED_CONFIRM_THRESHOLD = 6;
+const DEFAULT_REPEATS = 3;
+const REVERIFY_QUESTION =
+	"Assess this specific finding against the code it cites: is it a real, objective-relevant, currently-unresolved blocker?";
+
+/** The finding fields needed by the re-verification door. */
+export type ReverifiableFinding = ConsolidatableFinding & {
+	readonly body?: string;
+	readonly confidence_score?: number;
+};
+
+export type ReverifiableConsolidatedFinding = ConsolidatedFinding<ReverifiableFinding>;
+
+export type ReverifyResult = {
+	readonly verdict: "confirmed" | "demoted";
+	readonly meanScore: number;
+	readonly perRepeat: readonly (number | null)[];
+	readonly evidence: readonly string[];
+};
+
+/** Audit evidence retained alongside the original consolidated finding. */
+export type ReverifyAuditEntry<F extends ReverifiableFinding = ReverifiableFinding> = {
+	readonly finding: ConsolidatedFinding<F>;
+	readonly verdict: ReverifyResult["verdict"];
+	readonly meanScore: number;
+	readonly perRepeat: readonly (number | null)[];
+	readonly evidence: readonly string[];
+};
+
+export type ReverifyContext = Pick<WorkflowRunContext, "task">;
+
+export type ReverifyBatchResult<F extends ReverifiableFinding = ReverifiableFinding> = {
+	readonly batch: readonly ConsolidatedFinding<F>[];
+	readonly audits: readonly ReverifyAuditEntry<F>[];
+};
+
+export type ReverifyBatchInput<F extends ReverifiableFinding = ReverifiableFinding> = {
+	readonly batch: readonly ConsolidatedFinding<F>[];
+	readonly context: {
+		readonly objective: string;
+		readonly candidateRefs: readonly string[];
+	};
+	readonly repeats?: number;
+	readonly threshold?: number;
+};
+
+const reverifySchema = Type.Object(
+	{
+		score: VERIFICATION_SCALE.schema,
+		evidence: Type.Array(Type.String()),
+	},
+	{ additionalProperties: false },
+);
+
+type ReverifyReport = {
+	readonly score: number;
+	readonly evidence: readonly string[];
+};
+
+function isRecord(value: WorkflowSerializableValue | undefined): value is Record<string, WorkflowSerializableValue> {
+	return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+function isStringArray(value: WorkflowSerializableValue | undefined): value is readonly string[] {
+	return Array.isArray(value) && value.every((item) => typeof item === "string");
+}
+
+function parseReport(result: WorkflowTaskResult): ReverifyReport | undefined {
+	if (!isRecord(result.structured)) return undefined;
+	const score = result.structured.score;
+	if (
+		typeof score !== "number" ||
+		!Number.isInteger(score) ||
+		score < VERIFICATION_SCALE.min ||
+		score > VERIFICATION_SCALE.max
+	) {
+		return undefined;
+	}
+	const evidence = result.structured.evidence;
+	return {
+		score,
+		evidence: isStringArray(evidence) ? evidence : [],
+	};
+}
+
+function requiredByObjective(finding: ReverifiableConsolidatedFinding): boolean {
+	return finding.finding.objective_alignment === "required_by_objective";
+}
+
+/**
+ * Pure eligibility predicate. Missing confidence is deliberately ineligible:
+ * an absent graded signal is doubt, and doubt remains blocking.
+ */
+export function is_reverifiable(
+	finding: ReverifiableConsolidatedFinding,
+	threshold = DEFAULT_REVERIFY_THRESHOLD,
+): boolean {
+	const confidence = finding.finding.confidence_score;
+	return (
+		finding.blocking &&
+		finding.reviewers.length === 1 &&
+		typeof confidence === "number" &&
+		Number.isFinite(confidence) &&
+		confidence < threshold &&
+		finding.finding.objective_alignment !== "beyond_objective" &&
+		finding.finding.objective_alignment !== "contradicts_objective"
+	);
+}
+
+function readPaths(
+	finding: ReverifiableConsolidatedFinding,
+	candidateRefs: readonly string[],
+): readonly string[] {
+	const codePath = finding.finding.code_location?.absolute_file_path;
+	return [
+		...(codePath === undefined ? [] : [codePath]),
+		...candidateRefs,
+	];
+}
+
+function promptFor(
+	finding: ReverifiableConsolidatedFinding,
+	context: ReverifyBatchInput["context"],
+): { readonly prompt: string; readonly reads: readonly string[] } {
+	const codePath = finding.finding.code_location?.absolute_file_path;
+	const candidate: ScoringCandidate = {
+		...(codePath === undefined ? {} : { path: codePath }),
+		body: [
+			`Title: ${finding.finding.title}`,
+			`Body: ${finding.finding.body ?? ""}`,
+			`Code location: ${codePath ?? "not supplied"}`,
+		].join("\n"),
+	};
+	const head: SharedHead = {
+		task: `${REVERIFY_QUESTION}\n\nObjective:\n${context.objective}`,
+		groundTruthNote: `The objective is the contract. Verify the finding against observed code, not reviewer narration.`,
+		candidates: [candidate],
+		outputFormat: "Call structured_output with score (an integer from 1 to 20) and evidence (an array of observed, concise reasons).",
+	};
+	const prompt = build_scoring_prompt(head, {
+		id: "finding_reverification",
+		name: "Finding re-verification",
+		description: REVERIFY_QUESTION,
+	});
+	return {
+		prompt,
+		reads: [
+			...readPaths(finding, context.candidateRefs),
+			...scoring_prompt_reads(head),
+		],
+	};
+}
+
+async function runRepeat(
+	ctx: ReverifyContext,
+	finding: ReverifiableConsolidatedFinding,
+	context: ReverifyBatchInput["context"],
+	stageName: string,
+): Promise<ReverifyReport | undefined> {
+	try {
+		const rendered = promptFor(finding, context);
+		const options = {
+			prompt: rendered.prompt,
+			context: "fresh" as const,
+			reads: rendered.reads,
+			schema: reverifySchema,
+		};
+		return parseReport(await ctx.task(stageName, options));
+	} catch {
+		return undefined;
+	}
+}
+
+/**
+ * Re-score one eligible finding in K independent fresh contexts.
+ *
+ * A zero-valid-repeat mean is the deterministic sentinel `0`, matching the
+ * verification criteria module's empty-mean convention. It never authorizes a
+ * demotion because the quorum checks below require at least one valid score.
+ */
+export async function reverify_finding(
+	ctx: ReverifyContext,
+	input: {
+		readonly finding: ReverifiableConsolidatedFinding;
+		readonly context: {
+			readonly objective: string;
+			readonly candidateRefs: readonly string[];
+		};
+		repeats?: number;
+		threshold?: number;
+	},
+): Promise<ReverifyResult> {
+	if (!is_reverifiable(input.finding, input.threshold)) {
+		throw new Error("Cannot re-verify an ineligible finding.");
+	}
+
+	const repeatCount = Number.isFinite(input.repeats) && input.repeats !== undefined
+		? Math.max(0, Math.floor(input.repeats))
+		: DEFAULT_REPEATS;
+	const perRepeat: Array<number | null> = [];
+	const evidence: string[] = [];
+	const validScores: number[] = [];
+	for (let repeat = 1; repeat <= repeatCount; repeat += 1) {
+		let report = await runRepeat(ctx, input.finding, input.context, `reverify-${repeat}`);
+		if (report === undefined) {
+			report = await runRepeat(ctx, input.finding, input.context, `reverify-${repeat}-reask`);
+		}
+		if (report === undefined) {
+			perRepeat.push(null);
+			evidence.push(
+				`Re-verification repeat ${repeat} was invalid after one re-ask; doubt defaults to confirmed.`,
+			);
+			continue;
+		}
+		perRepeat.push(report.score);
+		validScores.push(report.score);
+		if (report.evidence.length === 0) {
+			evidence.push(`Re-verification repeat ${repeat} scored ${report.score}/20.`);
+		} else {
+			evidence.push(...report.evidence);
+		}
+	}
+
+	const meanScore = validScores.length === 0
+		? 0
+		: validScores.reduce((total, score) => total + score, 0) / validScores.length;
+	const validCount = validScores.length;
+	const requiredQuorum = Math.ceil(repeatCount / 2);
+	const demoted = requiredByObjective(input.finding)
+		? repeatCount > 0 && validCount === repeatCount && meanScore < REQUIRED_CONFIRM_THRESHOLD
+		: validCount > 0 && validCount >= requiredQuorum && meanScore < STANDARD_CONFIRM_THRESHOLD;
+	return {
+		verdict: demoted ? "demoted" : "confirmed",
+		meanScore,
+		perRepeat,
+		evidence,
+	};
+}
+
+function auditKey<F extends ReverifiableFinding>(finding: ConsolidatedFinding<F>): string {
+	return JSON.stringify([
+		finding.finding.code_location?.absolute_file_path ?? "",
+		finding.finding.title,
+		finding.reviewers,
+	]);
+}
+
+/** Apply only evidence-backed demotions; the original finding remains present. */
+export function apply_reverify_results<F extends ReverifiableFinding>(
+	batch: readonly ConsolidatedFinding<F>[],
+	audits: readonly ReverifyAuditEntry<F>[],
+): ReverifyBatchResult<F> {
+	const demoted = new Set(
+		audits
+			.filter((audit) => audit.verdict === "demoted")
+			.map((audit) => auditKey(audit.finding)),
+	);
+	return {
+		batch: batch.map((entry) =>
+			demoted.has(auditKey(entry)) ? { ...entry, blocking: false } : entry,
+		),
+		audits,
+	};
+}
+
+/** Re-verify only eligible entries and return the batch plus durable audit data. */
+export async function reverify_consolidated_batch<F extends ReverifiableFinding>(
+	ctx: ReverifyContext,
+	input: ReverifyBatchInput<F>,
+): Promise<ReverifyBatchResult<F>> {
+	const audits: ReverifyAuditEntry<F>[] = [];
+	for (const entry of input.batch) {
+		if (!is_reverifiable(entry, input.threshold)) continue;
+		const result = await reverify_finding(ctx, {
+			finding: entry,
+			context: input.context,
+			repeats: input.repeats,
+			threshold: input.threshold,
+		});
+		audits.push({ finding: entry, ...result });
+	}
+	return apply_reverify_results(input.batch, audits);
+}