headlesscode 1.0.3 → 1.2.1

package/src/rsi/mutation.ts

@@ -0,0 +1,77 @@
+import { exec as execCallback } from "node:child_process"
+import * as path from "node:path"
+import { promisify } from "node:util"
+import type { MutationRunner, RsiConfig, TrialResult, CandidateRecord } from "./types.js"
+import { DEFAULT_RSI_MODEL } from "./config.js"
+
+const exec = promisify(execCallback)
+
+export function mutationPrompt(candidate: CandidateRecord, config: RsiConfig): string {
+	const hypothesis = candidate.hypothesis
+	return [
+		"You are the bounded RSI worker for headlesscode.",
+		`Improve this candidate checkout for generation ${candidate.generation}.`,
+		`Mutation class: ${candidate.mutationKind ?? "corrective"}. Compute policy: ${config.computePolicy ?? "single"}.`,
+		`Objective: ${config.mutationTask}`,
+		hypothesis
+			? `Hypothesis: ${hypothesis.statement}\nExpected effect: ${hypothesis.expectedEffect}\nPotential downside: ${hypothesis.potentialDownside}${hypothesis.evidence ? `\nEvidence: ${hypothesis.evidence}` : ""}`
+			: "",
+		"You may change agent implementation, but you must not modify tests, evaluator/scoring/archive code, hidden evaluation commands, git metadata, or protected paths.",
+		"Inspect the existing repository first. Make a small, test-backed change. Run the relevant existing tests. Commit the candidate change before completing.",
+	].join("\n\n")
+}
+
+export const runMutation: MutationRunner = async (candidate, config) => {
+	const started = Date.now()
+	const workerRole = config.roles?.worker
+	const workerModel = workerRole?.model ?? config.model
+	const workerProvider = workerRole?.provider ?? "ollama"
+	const env: NodeJS.ProcessEnv = {
+		...process.env,
+		HEADLESSCODE_OPENROUTER_API_KEY: process.env.HEADLESSCODE_OPENROUTER_API_KEY ?? "rsi-local-placeholder",
+		HEADLESSCODE_CODE_MODE_BACKEND: workerProvider === "ollama" ? "ollama" : "openrouter",
+		HEADLESSCODE_LOCAL_BACKEND_MODES: "code",
+		HEADLESSCODE_CODE_MODE_MODEL: workerModel || DEFAULT_RSI_MODEL,
+		HEADLESSCODE_OLLAMA_THINK: "0",
+		HEADLESSCODE_CAPTURE_TRANSCRIPT_DIR: path.join(candidate.worktree, ".headlesscode", "rsi-transcripts"),
+	}
+	const command = [
+		"npx tsx src/cli.ts",
+		"--mode code",
+		`--max-iterations ${config.maxIterations}`,
+		"--no-checkpoints",
+		`--task ${JSON.stringify(mutationPrompt(candidate, config))}`,
+	].join(" ")
+	try {
+		const result = await exec(command, {
+			cwd: candidate.worktree,
+			env,
+			timeout: config.commandTimeoutMs,
+			maxBuffer: 16 * 1024 * 1024,
+		})
+		const trial: TrialResult = {
+			ok: true,
+			command,
+			exitCode: 0,
+			durationMs: Date.now() - started,
+			stdout: result.stdout,
+			stderr: result.stderr,
+		}
+		return { ok: true, result: trial }
+	} catch (error) {
+		const failure = error as { code?: number; killed?: boolean; stdout?: string; stderr?: string }
+		return {
+			ok: false,
+			result: {
+				ok: false,
+				command,
+				exitCode: typeof failure.code === "number" ? failure.code : null,
+				durationMs: Date.now() - started,
+				stdout: failure.stdout ?? "",
+				stderr: failure.stderr ?? String(error),
+				timedOut: failure.killed === true,
+			},
+			error: error instanceof Error ? error.message : String(error),
+		}
+	}
+}

package/src/rsi/reports.ts

@@ -0,0 +1,47 @@
+import type { CandidateRecord, Fitness, RsiConfig, RsiRunRecord } from "./types.js"
+
+export function formatCandidate(candidate: CandidateRecord): string {
+	const fitness = candidate.fitness
+	const parent = candidate.parentSelection ? `, parent=${candidate.parent} (${candidate.parentSelection.strategy})` : ""
+	const mutation = candidate.mutationKind ? `, mutation=${candidate.mutationKind}` : ""
+	const metrics = fitness ? `, metrics=${JSON.stringify(fitness.metrics)}` : ""
+	return `${candidate.id}: ${candidate.status}${parent}${mutation}${fitness ? `, score=${fitness.score}, ${fitness.reason}` : ""}${metrics}`
+}
+
+export function formatRunReport(run: RsiRunRecord, config: RsiConfig): string {
+	const lines = [
+		`# RSI Generation Report: ${run.runId}`,
+		"",
+		`- Model: ${run.model}`,
+		`- Base commit: ${run.baseCommit}`,
+		`- Generations: ${run.generations}`,
+		`- Started: ${run.startedAt}`,
+		`- Finished: ${run.finishedAt ?? "in progress"}`,
+		`- Baseline: ${run.baseline ? (run.baseline.ok ? "pass" : "fail") : "not run (dry-run)"}`,
+		`- Selected candidate: ${run.selected ?? "none"}`,
+		`- Pareto candidates: ${run.selectedCandidates?.join(", ") || "none"}`,
+		`- Parent policy: ${run.parentSelectionPolicy ?? config.parentSelectionPolicy ?? "champion-specialist-novelty"}`,
+		`- Model candidates: ${run.modelCandidates?.map((model) => `${model.id} (${model.status})`).join(", ") || "none"}`,
+		`- Experiment jobs: ${run.jobs?.length ?? 0}`,
+		`- Trajectories: ${run.trajectoryRefs?.length ?? 0}`,
+		"",
+		"## Candidates",
+		"",
+		...run.candidates.map((candidate) => `- ${formatCandidate(candidate)}${candidate.failure ? `; failure=${candidate.failure}` : ""}`),
+		"",
+		"## Guardrails",
+		"",
+		`- Visible evaluations: ${config.evalCommands.join("; ") || "none"}`,
+		`- Hidden evaluations: ${config.hiddenEvalCommands.length || 0}`,
+		`- Compute policy: ${config.computePolicy ?? "single"}`,
+		`- Protected paths: ${config.protectedPaths.join(", ")}`,
+		"",
+		"The archive is the durable machine-readable record for this run.",
+	]
+	return `${lines.join("\n")}\n`
+}
+
+export function fitnessSummary(fitness: Fitness | undefined): string {
+	if (!fitness) return "not evaluated"
+	return `${fitness.score} (${fitness.reason})`
+}

package/src/rsi/roles.ts

@@ -0,0 +1,37 @@
+import { DEFAULT_RSI_MODEL } from "./config.js"
+import type { ModelRole, RsiRoleConfig, RoleModelConfig } from "./types.js"
+
+const ROLE_ENV_NAMES: ModelRole[] = [
+	"worker",
+	"mutation-architect",
+	"failure-analyst",
+	"critic",
+	"adversary",
+	"reviewer",
+	"curriculum-designer",
+	"training-data-curator",
+]
+
+function envKey(role: ModelRole): string {
+	return `HEADLESSCODE_RSI_ROLE_${role.toUpperCase().replace(/-/g, "_")}_MODEL`
+}
+
+export function defaultRoles(workerModel = DEFAULT_RSI_MODEL): RsiRoleConfig {
+	return { worker: { provider: "ollama", model: workerModel } }
+}
+
+export function resolveRoles(
+	configured: RsiRoleConfig | undefined,
+	env: NodeJS.ProcessEnv = process.env,
+	workerModel = DEFAULT_RSI_MODEL,
+): RsiRoleConfig {
+	const roles: RsiRoleConfig = { ...defaultRoles(workerModel), ...configured }
+	for (const role of ROLE_ENV_NAMES) {
+		const model = env[envKey(role)]
+		if (model) {
+			const existing = roles[role]
+			roles[role] = { ...(existing ?? { provider: "ollama" }), model } as RoleModelConfig
+		}
+	}
+	return roles
+}

package/src/rsi/sandbox.ts

@@ -0,0 +1,10 @@
+import { findMatchingPattern } from "../permissions/protected-files.js"
+
+export function protectedPathViolations(changedFiles: string[], patterns: string[]): string[] {
+	return changedFiles.filter((file) => findMatchingPattern(file, patterns) !== null)
+}
+
+export function protectedPathReport(changedFiles: string[], patterns: string[]): string {
+	const violations = protectedPathViolations(changedFiles, patterns)
+	return violations.length === 0 ? "none" : violations.join(", ")
+}

package/src/rsi/search.ts

@@ -0,0 +1,32 @@
+import type { ComputePolicy } from "./types.js"
+
+export interface DifficultySignals {
+	taskLength: number
+	priorFailures: number
+	uncertainty: number
+		repeatedFailure: boolean
+}
+
+export interface SearchPlan {
+	policy: ComputePolicy
+	attempts: number
+	criticAfterFailure: boolean
+	reason: string
+}
+
+export function chooseComputePolicy(signals: DifficultySignals): SearchPlan {
+	if (signals.repeatedFailure || signals.priorFailures >= 2) {
+		return { policy: "critic-retry", attempts: 2, criticAfterFailure: true, reason: "repeated failure warrants diagnosis before another attempt" }
+	}
+	if (signals.taskLength >= 1200 || signals.uncertainty >= 0.75) {
+		return { policy: "planner-executors", attempts: 3, criticAfterFailure: false, reason: "long or uncertain task needs a plan plus independent implementations" }
+	}
+	if (signals.taskLength >= 500 || signals.uncertainty >= 0.4) {
+		return { policy: "independent", attempts: 2, criticAfterFailure: false, reason: "moderate difficulty merits independent trajectories" }
+	}
+	return { policy: "single", attempts: 1, criticAfterFailure: false, reason: "easy task does not justify extra inference" }
+}
+
+export function boundedSearchBudget(plan: SearchPlan, baseIterations: number): number {
+	return Math.max(1, Math.floor(baseIterations * plan.attempts))
+}

package/src/rsi/selection.ts

@@ -0,0 +1,132 @@
+import { randomUUID } from "node:crypto"
+import type {
+	CandidateRecord,
+	MetricVector,
+	ParentSelectionPolicy,
+	ParentSelectionReason,
+	RsiArchive,
+	ExperimentJob,
+	ExperimentJobKind,
+	ExperimentJobStatus,
+	ResourceRequirements,
+} from "./types.js"
+
+const SPECIALIST_METRICS: Array<keyof MetricVector> = ["correctness", "hidden", "efficiency", "recovery", "reliability"]
+
+function metrics(candidate: CandidateRecord): MetricVector {
+	return (
+		candidate.fitness?.metrics ?? {
+			correctness: candidate.fitness?.components.regression ?? 0,
+			reliability: candidate.fitness?.components.recovery ?? 0,
+			generalization: candidate.fitness?.components.visible ?? 0,
+			hidden: candidate.fitness?.components.hidden ?? 0,
+			efficiency: candidate.fitness?.components.efficiency ?? 0,
+			latency: 0,
+			tokenUse: 0,
+			recovery: candidate.fitness?.components.recovery ?? 0,
+			fabricationRate: 1,
+			complexityPenalty: 0,
+		}
+	)
+}
+
+function eligible(candidates: CandidateRecord[]): CandidateRecord[] {
+	return candidates.filter((candidate) => candidate.fitness && candidate.fitness.hardGates.committed !== false && candidate.commits[0])
+}
+
+export interface ParentChoice {
+	candidateId: string
+	baseCommit: string
+	reason: ParentSelectionReason
+}
+
+export function paretoDominates(left: MetricVector, right: MetricVector): boolean {
+	let strictlyBetter = false
+	for (const key of Object.keys(left) as Array<keyof MetricVector>) {
+		const leftValue = key === "fabricationRate" || key === "complexityPenalty" || key === "latency" || key === "tokenUse" ? -left[key] : left[key]
+		const rightValue = key === "fabricationRate" || key === "complexityPenalty" || key === "latency" || key === "tokenUse" ? -right[key] : right[key]
+		if (leftValue < rightValue) return false
+		if (leftValue > rightValue) strictlyBetter = true
+	}
+	return strictlyBetter
+}
+
+export function paretoFront(candidates: CandidateRecord[]): CandidateRecord[] {
+	const pool = eligible(candidates)
+	return pool.filter((candidate) => !pool.some((other) => other !== candidate && paretoDominates(metrics(other), metrics(candidate))))
+}
+
+function byScore(a: CandidateRecord, b: CandidateRecord): number {
+	return (b.fitness?.score ?? 0) - (a.fitness?.score ?? 0)
+}
+
+function choice(candidate: CandidateRecord, strategy: ParentSelectionReason["strategy"], reason: string): ParentChoice {
+	return {
+		candidateId: candidate.id,
+		baseCommit: candidate.commits[0] ?? candidate.baseCommit,
+		reason: { strategy, reason, metrics: metrics(candidate) },
+	}
+}
+
+export function selectParentChoices(
+	archive: RsiArchive,
+	policy: ParentSelectionPolicy,
+	count: number,
+	): ParentChoice[] {
+	const pool = eligible(archive.candidates).sort(byScore)
+	if (pool.length === 0 || count <= 0) return []
+	if (policy === "all-eligible") {
+		return Array.from({ length: count }, (_, index) => choice(pool[index % pool.length], "archive", "eligible archived candidate"))
+	}
+	if (policy === "pareto-front") {
+		const front = paretoFront(pool).sort(byScore)
+		return Array.from({ length: count }, (_, index) => choice(front[index % front.length], "pareto", "candidate retained on the multi-objective Pareto front"))
+	}
+
+	const selected: ParentChoice[] = []
+	const champion = pool[0]
+	selected.push(choice(champion, "champion", "highest hard-gated scalar fitness"))
+	for (const metric of SPECIALIST_METRICS) {
+		if (selected.length >= count) break
+		const specialist = [...pool].sort((a, b) => metrics(b)[metric] - metrics(a)[metric])[0]
+		if (specialist && !selected.some((entry) => entry.candidateId === specialist.id)) {
+			selected.push(choice(specialist, "specialist", `best archived candidate on ${metric}`))
+		}
+	}
+	for (const candidate of pool) {
+		if (selected.length >= count) break
+		if (!selected.some((entry) => entry.candidateId === candidate.id)) {
+			selected.push(choice(candidate, "novelty", "next eligible candidate preserves population diversity"))
+		}
+	}
+	return Array.from({ length: count }, (_, index) => selected[index % selected.length])
+}
+
+export function newExperimentJob(
+	kind: ExperimentJobKind,
+	resource: ResourceRequirements,
+	now: string,
+	candidateId?: string,
+): ExperimentJob {
+	return {
+		id: `job-${kind}-${randomUUID().slice(0, 8)}`,
+		kind,
+		status: "queued",
+		resource,
+		candidateId,
+		createdAt: now,
+		updatedAt: now,
+		attempts: 0,
+	}
+}
+
+export function transitionJob(job: ExperimentJob, status: ExperimentJobStatus, now: string, error?: string): ExperimentJob {
+	if (job.status === "completed" || job.status === "cancelled") return job
+	return {
+		...job,
+		status,
+		updatedAt: now,
+		attempts: status === "running" ? job.attempts + 1 : job.attempts,
+		...(error ? { error } : {}),
+	}
+}

package/src/rsi/trajectory.ts

@@ -0,0 +1,143 @@
+import { createHash } from "node:crypto"
+import * as fs from "node:fs/promises"
+import * as path from "node:path"
+import type { CandidateRecord, RsiConfig, TrajectoryMessage, TrajectoryRecord, TrajectorySummary } from "./types.js"
+
+interface CapturedCall {
+	messages?: TrajectoryMessage[]
+	response?: { message?: { tool_calls?: unknown[] } }
+	error?: string
+	provider?: string
+	model?: string
+}
+
+function failureClass(candidate: CandidateRecord): string | undefined {
+	if (candidate.failure) return candidate.failure.includes("Max iterations") ? "iteration-cap" : "mutation-failure"
+	if (candidate.result && !candidate.result.ok) {
+		const text = `${candidate.result.stderr}\n${candidate.result.stdout}`.toLowerCase()
+		if (candidate.result.timedOut) return "timeout"
+		if (text.includes("test") || text.includes("typecheck")) return "regression-failure"
+		return "evaluation-failure"
+	}
+	return undefined
+}
+
+async function readCapturedCalls(candidate: CandidateRecord): Promise<CapturedCall[]> {
+	const directory = path.join(candidate.worktree, ".headlesscode", "rsi-transcripts")
+	let names: string[]
+	try {
+		names = (await fs.readdir(directory)).filter((name) => name.endsWith(".jsonl")).sort()
+	} catch {
+		return []
+	}
+	const calls: CapturedCall[] = []
+	for (const name of names) {
+		const lines = (await fs.readFile(path.join(directory, name), "utf8")).split("\n").filter(Boolean)
+		for (const line of lines) {
+			try {
+				calls.push(JSON.parse(line) as CapturedCall)
+			} catch {
+				// A torn JSONL line is marked incomplete by the absence of a call.
+			}
+		}
+	}
+	return calls
+}
+
+export function trajectoryOutcome(candidate: CandidateRecord): TrajectoryRecord["outcome"] {
+	if (candidate.status === "accepted" && candidate.fitness?.hardGates.completed) return "success"
+	if (candidate.status === "failed" || candidate.status === "rejected") return "failure"
+	return "incomplete"
+}
+
+export async function captureCandidateTrajectory(
+	candidate: CandidateRecord,
+	config: RsiConfig,
+	now: string,
+): Promise<{ record: TrajectoryRecord; summary: TrajectorySummary }> {
+	const calls = await readCapturedCalls(candidate)
+	const messages = calls.flatMap((call) => call.messages ?? [])
+	const toolCalls = calls.reduce((count, call) => count + (call.response?.message?.tool_calls?.length ?? 0), 0)
+	const outcome = trajectoryOutcome(candidate)
+	const trusted = outcome === "success" && candidate.fitness?.hardGates.noProtectedPathViolation === true
+	const record: TrajectoryRecord = {
+		id: `trajectory-${candidate.id}`,
+		task: candidate.mutation,
+		environment: { repoRoot: config.repoRoot, baseCommit: candidate.baseCommit, generation: candidate.generation },
+		model: { id: candidate.model, candidateId: candidate.modelCandidateId },
+		harnessVersion: candidate.parentCommit ?? candidate.baseCommit,
+		promptConfig: {
+			mutationKind: candidate.mutationKind,
+			hypothesis: candidate.hypothesis,
+			parentSelection: candidate.parentSelection,
+		},
+		messages,
+		toolCalls,
+		outcome,
+		verification: {
+			verified: trusted,
+			regressionPass: candidate.fitness?.hardGates.regressionPass === true,
+			hiddenPass: candidate.fitness?.hardGates.hiddenEvalPass === true,
+		},
+		failureClassification: failureClass(candidate),
+		fitnessImpact: candidate.fitness?.score,
+		provenance: { source: "headlesscode-rsi", capturedAt: now, trusted },
+	}
+	const outputDir = config.trajectoryDir ?? path.join(config.archiveDir, "trajectories")
+	await fs.mkdir(outputDir, { recursive: true })
+	const outputPath = path.join(outputDir, `${candidate.id}.json`)
+	await fs.writeFile(outputPath, `${JSON.stringify(record, null, 2)}\n`, "utf8")
+	return {
+		record,
+		summary: {
+			path: outputPath,
+			messageCount: messages.length,
+			toolCalls,
+			trusted,
+			outcome,
+			failureClassification: record.failureClassification,
+		},
+	}
+}
+
+export async function readTrajectoryRecord(filePath: string): Promise<TrajectoryRecord | undefined> {
+	try {
+		return JSON.parse(await fs.readFile(filePath, "utf8")) as TrajectoryRecord
+	} catch {
+		return undefined
+	}
+}
+
+function jsonl(records: unknown[]): string {
+	return records.map((record) => JSON.stringify(record)).join("\n") + (records.length ? "\n" : "")
+}
+
+export async function exportTrajectoryDatasets(records: TrajectoryRecord[], outputDir: string): Promise<{
+	sft: number
+	preferences: number
+	failures: number
+	manifestPath: string
+}> {
+	await fs.mkdir(outputDir, { recursive: true })
+	const trusted = records.filter((record) => record.provenance.trusted && record.outcome === "success")
+	const failures = records.filter((record) => record.outcome === "failure")
+	const preferences: Array<{ task: string; chosen: TrajectoryRecord; rejected: TrajectoryRecord }> = []
+	for (const task of new Set(records.map((record) => record.task))) {
+		const successful = records.find((record) => record.task === task && record.provenance.trusted && record.outcome === "success")
+		const failed = records.find((record) => record.task === task && record.outcome === "failure")
+		if (successful && failed) preferences.push({ task, chosen: successful, rejected: failed })
+	}
+	await fs.writeFile(path.join(outputDir, "sft.jsonl"), jsonl(trusted), "utf8")
+	await fs.writeFile(path.join(outputDir, "preferences.jsonl"), jsonl(preferences), "utf8")
+	await fs.writeFile(path.join(outputDir, "failures.jsonl"), jsonl(failures), "utf8")
+	const manifest = {
+		schemaVersion: 1,
+		createdAt: new Date().toISOString(),
+		counts: { sft: trusted.length, preferences: preferences.length, failures: failures.length },
+		trustedOnly: true,
+		sha256: createHash("sha256").update(JSON.stringify(records)).digest("hex"),
+	}
+	const manifestPath = path.join(outputDir, "manifest.json")
+	await fs.writeFile(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`, "utf8")
+	return { ...manifest.counts, manifestPath }
+}