headlesscode 1.0.3 → 1.2.1

package/src/rsi/controller.ts

@@ -0,0 +1,268 @@
+import { randomUUID } from "node:crypto"
+import * as fs from "node:fs/promises"
+import { appendRun, checkpointRun, findActiveRun, readArchive } from "./archive.js"
+import { parseRsiArgs, rsiHelp } from "./config.js"
+import { generateCurriculumProposals, writeCurriculumProposals } from "./curriculum.js"
+import { evaluateCandidate, runCommand } from "./evaluator.js"
+import { computeFitness, compareFitness } from "./fitness.js"
+import { baseModelCandidate, createCombination } from "./models.js"
+import { runMutation } from "./mutation.js"
+import { formatRunReport } from "./reports.js"
+import { protectedPathViolations } from "./sandbox.js"
+import { resolveRoles } from "./roles.js"
+import { newExperimentJob, paretoFront, selectParentChoices, transitionJob } from "./selection.js"
+import { captureCandidateTrajectory, exportTrajectoryDatasets } from "./trajectory.js"
+import type { CandidateRecord, ExperimentJob, RsiArchive, RsiConfig, RsiHooks, RsiRunRecord, TrajectoryRecord } from "./types.js"
+import {
+	candidateCommits,
+	changedFiles,
+	createCandidateWorktree,
+	newCandidate,
+	removeCandidateWorktree,
+	resolveBaseCommit,
+} from "./workspace.js"
+
+function log(hooks: RsiHooks, line: string): void {
+	(hooks.log ?? ((message) => process.stdout.write(`${message}\n`)))(line)
+}
+
+function isTerminal(candidate: CandidateRecord): boolean {
+	return candidate.status === "accepted" || candidate.status === "rejected" || candidate.status === "failed"
+}
+
+function replaceJob(run: RsiRunRecord, job: ExperimentJob): void {
+	run.jobs ??= []
+	const index = run.jobs.findIndex((entry) => entry.id === job.id)
+	if (index >= 0) run.jobs[index] = job
+	else run.jobs.push(job)
+}
+
+function combinedArchive(archive: RsiArchive, run: RsiRunRecord): RsiArchive {
+	const currentIds = new Set(run.candidates.map((candidate) => candidate.id))
+	return { ...archive, candidates: [...archive.candidates.filter((candidate) => !currentIds.has(candidate.id)), ...run.candidates] }
+}
+
+function initialRun(config: RsiConfig, baseCommit: string, now: string): RsiRunRecord {
+	const modelCandidateId = config.modelCandidateId ?? "model-base"
+	return {
+		runId: `rsi-${now.replace(/[^0-9]/g, "").slice(0, 14)}-${randomUUID().slice(0, 8)}`,
+		startedAt: now,
+		model: config.model,
+		baseRef: config.baseRef,
+		baseCommit,
+		generations: config.generations,
+		parentSelectionPolicy: config.parentSelectionPolicy ?? "champion-specialist-novelty",
+		selectedCandidates: [],
+		parentChoices: {},
+		modelCandidates: [baseModelCandidate(config.model, now, modelCandidateId)],
+		combinations: [],
+		jobs: [],
+		trajectoryRefs: [],
+		curriculumTasks: [],
+		candidates: [],
+		reports: [],
+	}
+}
+
+function recoverInterruptedCandidates(run: RsiRunRecord, now: string): void {
+	for (const candidate of run.candidates) {
+		if (candidate.status === "mutating" || candidate.status === "evaluating") {
+			candidate.status = "failed"
+			candidate.failure = "run interrupted before candidate reached a terminal state"
+			candidate.updatedAt = now
+		}
+	}
+}
+
+async function captureCandidate(
+	candidate: CandidateRecord,
+	config: RsiConfig,
+	run: RsiRunRecord,
+	trajectories: TrajectoryRecord[],
+	now: string,
+	hooks: RsiHooks,
+): Promise<void> {
+	try {
+		const captured = await captureCandidateTrajectory(candidate, config, now)
+		candidate.trajectory = captured.summary
+		trajectories.push(captured.record)
+		run.trajectoryRefs ??= []
+		run.trajectoryRefs.push(captured.summary.path)
+	} catch (error) {
+		log(hooks, `[rsi] ${candidate.id}: trajectory capture failed: ${String(error)}`)
+	}
+}
+
+export async function runRsi(config: RsiConfig, hooks: RsiHooks = {}): Promise<RsiRunRecord> {
+	const now = hooks.now ?? (() => new Date().toISOString())
+	const archive = await readArchive(config.archiveDir)
+	const roles = resolveRoles(config.roles, process.env, config.model)
+	const workerModel = roles.worker?.model ?? config.model
+	const resumed = config.resumeRunId ? await findActiveRun(config.archiveDir, config.resumeRunId) : undefined
+	if (config.resumeRunId && !resumed) throw new Error(`no active RSI run found for --resume ${config.resumeRunId}`)
+	const baseCommit = resumed?.baseCommit ?? (await resolveBaseCommit(config))
+	const run = resumed ?? initialRun({ ...config, model: workerModel, roles }, baseCommit, now())
+	run.generations = config.generations
+	run.model = workerModel
+	run.parentSelectionPolicy = config.parentSelectionPolicy ?? run.parentSelectionPolicy ?? "champion-specialist-novelty"
+	run.modelCandidates ??= [baseModelCandidate(config.model, now(), config.modelCandidateId ?? "model-base")]
+	if (config.modelCandidates?.length) {
+		run.modelCandidates = [...run.modelCandidates, ...config.modelCandidates.filter((model) => !run.modelCandidates!.some((entry) => entry.id === model.id))]
+	}
+	run.combinations ??= []
+	run.jobs ??= []
+	run.trajectoryRefs ??= []
+	run.curriculumTasks ??= []
+	run.parentChoices ??= {}
+	if (resumed) recoverInterruptedCandidates(run, now())
+	const runConfig: RsiConfig = { ...config, model: workerModel, roles, seed: `${config.seed}-${run.runId.slice(-8)}` }
+	const trajectories: TrajectoryRecord[] = []
+
+	if (!config.dryRun && !run.baseline) {
+		run.baseline = await (hooks.runCommand ?? runCommand)("npm test", config.repoRoot, config.commandTimeoutMs)
+		log(hooks, `[rsi] baseline: ${run.baseline.ok ? "pass" : "fail"}`)
+		await checkpointRun(config.archiveDir, run)
+	}
+
+	for (let generation = 0; generation < config.generations; generation++) {
+		let candidates = run.candidates.filter((candidate) => candidate.generation === generation)
+		if (candidates.length === 0) {
+			const merged = combinedArchive(archive, run)
+			const choices = generation === 0
+				? []
+				: selectParentChoices(merged, run.parentSelectionPolicy ?? "champion-specialist-novelty", config.population)
+			const fallbackBase = run.selected
+				? run.candidates.find((candidate) => candidate.id === run.selected)?.commits[0] ?? run.baseCommit
+				: run.baseCommit
+			candidates = Array.from({ length: config.population }, (_, index) => {
+				const parentChoice = choices[index]
+				const candidate = newCandidate(runConfig, generation, index, parentChoice?.baseCommit ?? fallbackBase, now(), parentChoice)
+				if (parentChoice) run.parentChoices![candidate.id] = parentChoice.reason
+				return candidate
+			})
+			run.candidates.push(...candidates)
+			log(hooks, `[rsi] generation ${generation}: planning ${candidates.length} candidate(s) with policy ${run.parentSelectionPolicy}`)
+			if (!config.dryRun) await checkpointRun(config.archiveDir, run)
+		} else {
+			log(hooks, `[rsi] generation ${generation}: resuming ${candidates.length} persisted candidate(s)`)
+		}
+		if (config.dryRun) continue
+
+		for (const candidate of candidates) {
+			if (isTerminal(candidate)) continue
+			let job = run.jobs.find((entry) => entry.candidateId === candidate.id && entry.kind === "mutation")
+			if (!job) {
+				job = newExperimentJob("mutation", { class: "LOCAL_GPU", units: 1, concurrencyKey: "ollama" }, now(), candidate.id)
+				run.jobs.push(job)
+			}
+			candidate.status = "mutating"
+			candidate.updatedAt = now()
+			job = transitionJob(job, "running", now())
+			replaceJob(run, job)
+			await checkpointRun(config.archiveDir, run)
+			let worktreeCreated = false
+			try {
+				await createCandidateWorktree(runConfig, candidate)
+				worktreeCreated = true
+				const mutation = await (hooks.runMutation ?? runMutation)(candidate, runConfig)
+				if (!mutation.ok) {
+					candidate.status = "failed"
+					candidate.failure = mutation.error
+					candidate.result = mutation.result
+					candidate.updatedAt = now()
+					job = transitionJob(job, "failed", now(), mutation.error)
+					replaceJob(run, job)
+					await captureCandidate(candidate, runConfig, run, trajectories, now(), hooks)
+					log(hooks, `[rsi] ${candidate.id}: mutation failed${mutation.error ? `: ${mutation.error}` : ""}`)
+					continue
+				}
+				candidate.status = "evaluating"
+				candidate.changedFiles = await changedFiles(candidate.worktree, candidate.baseCommit)
+				candidate.commits = await candidateCommits(candidate.worktree, candidate.baseCommit)
+				candidate.protectedPathViolations = protectedPathViolations(candidate.changedFiles, config.protectedPaths)
+				const evaluation = await evaluateCandidate(runConfig, candidate.worktree, candidate.baseCommit, hooks.runCommand ?? runCommand)
+				candidate.changedFiles = evaluation.changedFiles
+				candidate.commits = candidate.commits.length > 0 ? candidate.commits : await candidateCommits(candidate.worktree, candidate.baseCommit)
+				candidate.result = evaluation.regression
+				candidate.fitness = computeFitness({ ...evaluation, protectedPathViolation: candidate.protectedPathViolations.length > 0 })
+				candidate.status = candidate.fitness.score > 0 ? "accepted" : "rejected"
+				candidate.updatedAt = now()
+				const combination = createCombination(candidate.id, candidate.modelCandidateId ?? "model-base", now())
+				combination.status = candidate.status === "accepted" ? "accepted" : "rejected"
+				combination.fitness = candidate.fitness
+				combination.metrics = candidate.fitness.metrics
+				run.combinations.push(combination)
+				job = transitionJob(job, "completed", now())
+				replaceJob(run, job)
+				await captureCandidate(candidate, runConfig, run, trajectories, now(), hooks)
+				log(hooks, `[rsi] ${candidate.id}: ${candidate.status}, score=${candidate.fitness.score}`)
+			} catch (error) {
+				candidate.status = "failed"
+				candidate.failure = error instanceof Error ? error.message : String(error)
+				candidate.updatedAt = now()
+				job = transitionJob(job, "failed", now(), candidate.failure)
+				replaceJob(run, job)
+				log(hooks, `[rsi] ${candidate.id}: lifecycle failed: ${candidate.failure}`)
+			} finally {
+				if (worktreeCreated && !config.keepWorktrees) {
+					try {
+						await removeCandidateWorktree(runConfig, candidate)
+					} catch (error) {
+						log(hooks, `[rsi] ${candidate.id}: worktree cleanup failed: ${String(error)}`)
+					}
+				}
+				await checkpointRun(config.archiveDir, run)
+			}
+		}
+
+		const accepted = candidates.filter((candidate) => candidate.status === "accepted")
+		const front = paretoFront(accepted)
+		const ranked = [...accepted].sort((a, b) => compareFitness(b.fitness, a.fitness))
+		run.selectedCandidates = front.map((candidate) => candidate.id)
+		if (ranked[0]) run.selected = ranked[0].id
+		await checkpointRun(config.archiveDir, run)
+	}
+
+	const merged = combinedArchive(archive, run)
+	const curriculum = generateCurriculumProposals(merged, now())
+	run.curriculumTasks = curriculum
+	if (!config.dryRun && curriculum.length > 0) {
+		await writeCurriculumProposals(curriculum, config.curriculumDir ?? `${config.archiveDir}/curriculum`)
+	}
+	if (!config.dryRun && trajectories.length > 0) {
+		await exportTrajectoryDatasets(trajectories, `${config.trajectoryDir ?? `${config.archiveDir}/trajectories`}/datasets`)
+	}
+
+	run.finishedAt = now()
+	const report = formatRunReport(run, config)
+	const reportPath = `${config.archiveDir}/${run.runId}.md`
+	if (!config.dryRun) {
+		await fs.mkdir(config.archiveDir, { recursive: true })
+		await fs.writeFile(reportPath, report, "utf8")
+		run.reports = [...new Set([...run.reports, reportPath])]
+		await appendRun(config.archiveDir, run)
+	} else {
+		log(hooks, report.trimEnd())
+	}
+	return run
+}
+
+export async function improveMain(argv: string[]): Promise<number> {
+	const parsed = parseRsiArgs(argv)
+	if (parsed.help) {
+		process.stdout.write(rsiHelp())
+		return 0
+	}
+	if (parsed.error || !parsed.config) {
+		process.stderr.write(`headlesscode improve: ${parsed.error ?? "invalid configuration"}\n`)
+		return 2
+	}
+	try {
+		const run = await runRsi(parsed.config)
+		process.stdout.write(`[rsi] complete: ${run.runId}; selected=${run.selected ?? "none"}\n`)
+		return run.selected || parsed.config.dryRun ? 0 : 1
+	} catch (error) {
+		process.stderr.write(`headlesscode improve: ${error instanceof Error ? error.message : String(error)}\n`)
+		return 1
+	}
+}

package/src/rsi/curriculum.ts

@@ -0,0 +1,68 @@
+import * as fs from "node:fs/promises"
+import * as path from "node:path"
+import type { CandidateRecord, CurriculumTask, RsiArchive } from "./types.js"
+
+const FAILURE_TASKS: Record<string, { capability: string; difficulty: CurriculumTask["difficulty"]; template: string }> = {
+	"iteration-cap": {
+		capability: "completion-discipline",
+		difficulty: 3,
+		template: "Complete a bounded implementation task and prove it with a final verification command before the iteration budget expires.",
+	},
+	"regression-failure": {
+		capability: "regression-recovery",
+		difficulty: 3,
+		template: "Repair a deliberately failing change while preserving the existing regression suite and report the verified result.",
+	},
+	"timeout": {
+		capability: "tool-efficiency",
+		difficulty: 4,
+		template: "Solve a repository task under a strict command-time budget without repeating unproductive exploration.",
+	},
+	"evaluation-failure": {
+		capability: "generalization",
+		difficulty: 4,
+		template: "Make a focused repository improvement that passes the visible suite and an adjacent executable behavior check.",
+	},
+}
+
+function candidateFailure(candidate: CandidateRecord): string | undefined {
+	if (candidate.failure?.includes("Max iterations")) return "iteration-cap"
+	if (candidate.result?.timedOut) return "timeout"
+	if (candidate.result && !candidate.result.ok) return "evaluation-failure"
+	return candidate.fitness?.hardGates.regressionPass === false ? "regression-failure" : undefined
+}
+
+export function generateCurriculumProposals(archive: RsiArchive, now = new Date().toISOString()): CurriculumTask[] {
+	const grouped = new Map<string, CandidateRecord[]>()
+	for (const candidate of archive.candidates) {
+		const failure = candidateFailure(candidate)
+		if (!failure) continue
+		const entries = grouped.get(failure) ?? []
+		entries.push(candidate)
+		grouped.set(failure, entries)
+	}
+	return [...grouped.entries()].flatMap(([failureClass, candidates]) => {
+		const definition = FAILURE_TASKS[failureClass] ?? FAILURE_TASKS["evaluation-failure"]
+		return [{
+			id: `curriculum-${failureClass}`,
+			task: definition.template,
+			difficulty: definition.difficulty,
+			capability: definition.capability,
+			groundTruthCommand: "npm test",
+			provenance: { sourceCandidateIds: candidates.map((candidate) => candidate.id), failureClass, generatedAt: now },
+			validated: false,
+		}]
+	})
+}
+
+export function validateCurriculumTask(task: CurriculumTask): boolean {
+	return task.task.trim() !== "" && task.groundTruthCommand.trim() !== "" && task.provenance.sourceCandidateIds.length > 0
+}
+
+export async function writeCurriculumProposals(tasks: CurriculumTask[], outputDir: string): Promise<string> {
+	await fs.mkdir(outputDir, { recursive: true })
+	const validated = tasks.map((task) => ({ ...task, validated: validateCurriculumTask(task) }))
+	const outputPath = path.join(outputDir, "proposals.json")
+	await fs.writeFile(outputPath, `${JSON.stringify(validated, null, 2)}\n`, "utf8")
+	return outputPath
+}

package/src/rsi/evaluator.ts

@@ -0,0 +1,106 @@
+import { exec as execCallback } from "node:child_process"
+import { promisify } from "node:util"
+import type { CommandRunner, EvaluationSummary, RsiConfig, TrialResult } from "./types.js"
+import { changedFiles, candidateCommits, gitOutput } from "./workspace.js"
+import { protectedPathViolations } from "./sandbox.js"
+
+const exec = promisify(execCallback)
+
+export const runCommand: CommandRunner = async (command, cwd, timeoutMs, env) => {
+	const started = Date.now()
+	try {
+		const result = await exec(command, { cwd, env, timeout: timeoutMs, maxBuffer: 8 * 1024 * 1024 })
+		return { ok: true, command, exitCode: 0, durationMs: Date.now() - started, stdout: result.stdout, stderr: result.stderr }
+	} catch (error) {
+		const failure = error as { code?: number | string; killed?: boolean; stdout?: string; stderr?: string }
+		return {
+			ok: false,
+			command,
+			exitCode: typeof failure.code === "number" ? failure.code : null,
+			durationMs: Date.now() - started,
+			stdout: failure.stdout ?? "",
+			stderr: failure.stderr ?? String(error),
+			timedOut: failure.killed === true,
+		}
+	}
+}
+
+export async function evaluateCandidate(
+	config: RsiConfig,
+	candidateRoot: string,
+	baseCommit: string,
+	runner: CommandRunner = runCommand,
+): Promise<EvaluationSummary> {
+	const files = await changedFiles(candidateRoot, baseCommit)
+	const commits = await candidateCommits(candidateRoot, baseCommit)
+	const complexity = await complexityMetrics(candidateRoot, baseCommit, files)
+	const violations = protectedPathViolations(files, config.protectedPaths)
+	// Refuse to execute a candidate that touched evaluator/scoring inputs. This
+	// check happens before any candidate-controlled test command runs.
+	if (violations.length > 0) {
+		const blocked: TrialResult = {
+			ok: false,
+			command: "protected-path-check",
+			exitCode: 1,
+			durationMs: 0,
+			stdout: "",
+			stderr: `protected paths changed: ${violations.join(", ")}`,
+		}
+		return {
+			regression: blocked,
+			visible: [],
+			hidden: [],
+			completed: false,
+			crashed: false,
+			protectedPathViolation: true,
+			changedFiles: files,
+			committed: commits.length > 0,
+			complexity,
+			failureClassification: "protected-path-violation",
+		}
+	}
+	const regression = await runner("npm test", candidateRoot, config.commandTimeoutMs)
+	const visible: TrialResult[] = []
+	for (const command of config.evalCommands) {
+		visible.push(await runner(command, candidateRoot, config.commandTimeoutMs))
+	}
+	const hidden: TrialResult[] = []
+	for (const command of config.hiddenEvalCommands) {
+		// Hidden commands belong to the supervisor checkout. They receive the
+		// candidate path explicitly so a candidate cannot replace the evaluator
+		// script or package metadata in the process that scores it.
+		hidden.push(
+			await runner(command, config.repoRoot, config.commandTimeoutMs, {
+				...process.env,
+				HEADLESSCODE_RSI_CANDIDATE_ROOT: candidateRoot,
+			}),
+		)
+	}
+	return {
+		regression,
+		visible,
+		hidden,
+		completed: regression.ok && visible.every((trial) => trial.ok) && hidden.every((trial) => trial.ok),
+		crashed: [regression, ...visible, ...hidden].some((trial) => trial.exitCode === null && !trial.timedOut),
+		protectedPathViolation: false,
+		changedFiles: files,
+		committed: commits.length > 0,
+		complexity,
+		failureClassification: [regression, ...visible, ...hidden].some((trial) => !trial.ok) ? "evaluation-failure" : undefined,
+	}
+}
+
+async function complexityMetrics(candidateRoot: string, baseCommit: string, files: string[]): Promise<NonNullable<EvaluationSummary["complexity"]>> {
+	try {
+		const numstat = await gitOutput(candidateRoot, ["diff", "--numstat", `${baseCommit}...HEAD`])
+		const diffLines = numstat.split("\n").reduce((total, line) => {
+			const [added, deleted] = line.split(/\s+/)
+			const a = Number(added)
+			const d = Number(deleted)
+			return total + (Number.isFinite(a) ? a : 0) + (Number.isFinite(d) ? d : 0)
+		}, 0)
+		return { diffLines, changedFiles: files.length, newDependencies: 0, additionalModelCalls: 0, runtimeOverheadMs: 0 }
+	} catch {
+		return { diffLines: 0, changedFiles: files.length, newDependencies: 0, additionalModelCalls: 0, runtimeOverheadMs: 0 }
+	}
+}

package/src/rsi/fitness.ts

@@ -0,0 +1,64 @@
+import type { EvaluationSummary, Fitness, HardGates } from "./types.js"
+
+function rate(passed: number, total: number): number {
+	return total === 0 ? 1 : passed / total
+}
+
+export function hardGatesFor(summary: EvaluationSummary): HardGates {
+	return {
+		regressionPass: summary.regression.ok,
+		visibleEvalPass: summary.visible.every((trial) => trial.ok),
+		hiddenEvalPass: summary.hidden.every((trial) => trial.ok),
+		noProtectedPathViolation: !summary.protectedPathViolation,
+		completed: summary.completed,
+		noCrash: !summary.crashed,
+		committed: summary.committed !== false,
+	}
+}
+
+export function computeFitness(summary: EvaluationSummary): Fitness {
+	const hardGates = hardGatesFor(summary)
+	const visibleRate = rate(summary.visible.filter((trial) => trial.ok).length, summary.visible.length)
+	const hiddenRate = rate(summary.hidden.filter((trial) => trial.ok).length, summary.hidden.length)
+	const regression = hardGates.regressionPass ? 1 : 0
+	const efficiency = Math.max(0, 1 - Math.min(1, summary.visible.reduce((sum, trial) => sum + trial.durationMs, 0) / 600_000))
+	const recovery = summary.visible.length === 0 ? 0 : visibleRate
+	const complexity = summary.complexity ?? {
+		diffLines: 0,
+		changedFiles: summary.changedFiles.length,
+		newDependencies: 0,
+		additionalModelCalls: 0,
+		runtimeOverheadMs: 0,
+	}
+	const latency = [summary.regression, ...summary.visible, ...summary.hidden].reduce((sum, trial) => sum + trial.durationMs, 0)
+	const complexityPenalty = Math.min(1, complexity.diffLines / 20_000 + complexity.newDependencies / 10 + complexity.additionalModelCalls / 20)
+	const metrics = {
+		correctness: regression,
+		reliability: hardGates.noCrash ? 1 : 0,
+		generalization: visibleRate,
+		hidden: hiddenRate,
+		efficiency,
+		latency: latency,
+		tokenUse: 0,
+		recovery,
+		fabricationRate: summary.completed ? 0 : 1,
+		complexityPenalty,
+	}
+	const components = { regression, visible: visibleRate, hidden: hiddenRate, efficiency, recovery }
+	const score = Math.round(
+		(regression * 45 + visibleRate * 25 + hiddenRate * 20 + efficiency * 5 + recovery * 5 - complexityPenalty * 5) * 100,
+	) / 100
+	const failedGate = Object.entries(hardGates).find(([, passed]) => !passed)?.[0]
+	return {
+		score: failedGate ? 0 : score,
+		components,
+		metrics,
+		complexity,
+		hardGates,
+		reason: failedGate ? `hard gate failed: ${failedGate}` : "all hard gates passed",
+	}
+}
+
+export function compareFitness(left: Fitness | undefined, right: Fitness | undefined): number {
+	return (left?.score ?? 0) - (right?.score ?? 0)
+}

package/src/rsi/index.ts

@@ -0,0 +1,16 @@
+export * from "./archive.js"
+export * from "./config.js"
+export * from "./controller.js"
+export * from "./evaluator.js"
+export * from "./fitness.js"
+export * from "./mutation.js"
+export * from "./models.js"
+export * from "./reports.js"
+export * from "./sandbox.js"
+export * from "./selection.js"
+export * from "./types.js"
+export * from "./workspace.js"
+export * from "./trajectory.js"
+export * from "./curriculum.js"
+export * from "./roles.js"
+export * from "./search.js"

package/src/rsi/models.ts

@@ -0,0 +1,89 @@
+import { createHash } from "node:crypto"
+import { exec as execCallback } from "node:child_process"
+import * as fs from "node:fs/promises"
+import { promisify } from "node:util"
+import type {
+	HarnessModelCombination,
+	ModelCandidate,
+	TrainingConfig,
+	TrialResult,
+} from "./types.js"
+
+const exec = promisify(execCallback)
+
+export function baseModelCandidate(model: string, now: string, id = "model-base"): ModelCandidate {
+	return {
+		id,
+		parentModel: model,
+		trainingMethod: "none",
+		trainingConfig: { method: "none" },
+		status: "base",
+		createdAt: now,
+		provenance: { source: "configured-worker-model" },
+	}
+}
+
+export function modelCombinationId(harnessCandidateId: string, modelCandidateId: string): string {
+	return `${harnessCandidateId}::${modelCandidateId}`
+}
+
+export function createCombination(
+	harnessCandidateId: string,
+	modelCandidateId: string,
+	now: string,
+): HarnessModelCombination {
+	return {
+		id: modelCombinationId(harnessCandidateId, modelCandidateId),
+		harnessCandidateId,
+		modelCandidateId,
+		status: "planned",
+		createdAt: now,
+	}
+}
+
+export function factorialCombinations(harnessIds: string[], modelIds: string[], now: string): HarnessModelCombination[] {
+	return harnessIds.flatMap((harnessId) => modelIds.map((modelId) => createCombination(harnessId, modelId, now)))
+}
+
+export interface TrainingBackend {
+	prepareDataset(datasetVersion: string, outputDir: string): Promise<void>
+	train(config: TrainingConfig, outputDir: string): Promise<TrialResult>
+	inspectArtifact(artifactPath: string): Promise<{ hash: string; bytes: number }>
+}
+
+export class ExternalTrainingBackend implements TrainingBackend {
+	constructor(private readonly timeoutMs = 60 * 60_000) {}
+
+	async prepareDataset(datasetVersion: string, outputDir: string): Promise<void> {
+		await fs.mkdir(outputDir, { recursive: true })
+		await fs.writeFile(`${outputDir}/dataset-version.txt`, `${datasetVersion}\n`, "utf8")
+	}
+
+	async train(config: TrainingConfig, outputDir: string): Promise<TrialResult> {
+		const command = config.command
+		if (!command) {
+			return { ok: false, command: "training-command", exitCode: 2, durationMs: 0, stdout: "", stderr: "no training command configured" }
+		}
+		const started = Date.now()
+		try {
+			const result = await exec(command, { cwd: outputDir, timeout: this.timeoutMs, maxBuffer: 16 * 1024 * 1024 })
+			return { ok: true, command, exitCode: 0, durationMs: Date.now() - started, stdout: result.stdout, stderr: result.stderr }
+		} catch (error) {
+			const failure = error as { code?: number; killed?: boolean; stdout?: string; stderr?: string }
+			return {
+				ok: false,
+				command,
+				exitCode: typeof failure.code === "number" ? failure.code : null,
+				durationMs: Date.now() - started,
+				stdout: failure.stdout ?? "",
+				stderr: failure.stderr ?? String(error),
+				timedOut: failure.killed === true,
+			}
+		}
+	}
+
+	async inspectArtifact(artifactPath: string): Promise<{ hash: string; bytes: number }> {
+		const bytes = await fs.readFile(artifactPath)
+		return { hash: createHash("sha256").update(bytes).digest("hex"), bytes: bytes.byteLength }
+	}
+}