headlesscode 1.2.0 → 1.2.2

package/src/monitoring/controller.ts

@@ -0,0 +1,73 @@
+import type {
+	FrozenPredictorArtifact,
+	InterventionDecision,
+	IterationInterventionTemplateId,
+	PilotArm,
+	PilotBoundary,
+	Prediction,
+} from "./types.js"
+import { predict, predictedIntervention } from "./predictor.js"
+
+export const SPECIFIC_CORRECTION =
+	"Check the latest tool result. Re-read the affected configuration if necessary, identify the failed precondition, and change the next action before retrying."
+export const GENERIC_REVIEW = "Pause and review your approach before continuing."
+export const TEMPLATE_TEXT: Readonly<Record<IterationInterventionTemplateId, string>> = {
+	"specific-correction": SPECIFIC_CORRECTION,
+	"generic-review": GENERIC_REVIEW,
+}
+
+export interface ControllerState {
+	readonly arm: PilotArm
+	readonly disarmed: boolean
+	readonly appliedInterventionIds: ReadonlySet<string>
+}
+
+export function initialControllerState(arm: PilotArm): ControllerState {
+	return { arm, disarmed: false, appliedInterventionIds: new Set() }
+}
+
+export function decide(
+	artifact: FrozenPredictorArtifact | undefined,
+	boundary: PilotBoundary,
+	state: ControllerState,
+	prediction?: Prediction,
+): { prediction?: Prediction; decision?: InterventionDecision; state: ControllerState } {
+	if (state.disarmed || state.arm === "B") return { prediction, state }
+	const nextPrediction = prediction ?? (artifact === undefined ? undefined : predict(artifact, boundary))
+	if (nextPrediction === undefined || artifact === undefined || !predictedIntervention(artifact, nextPrediction)) {
+		return { prediction: nextPrediction, state }
+	}
+	const templateId: IterationInterventionTemplateId = state.arm === "G" ? "generic-review" : "specific-correction"
+	const interventionId = `${boundary.attemptId}:${boundary.sequence}:${templateId}`
+	if (state.appliedInterventionIds.has(interventionId)) return { prediction: nextPrediction, state }
+	const applied = new Set(state.appliedInterventionIds)
+	applied.add(interventionId)
+	return {
+		prediction: nextPrediction,
+		decision: { interventionId, templateId, reason: "threshold" },
+		state: { ...state, disarmed: true, appliedInterventionIds: applied },
+	}
+}
+
+export function randomDecision(
+	boundary: PilotBoundary,
+	state: ControllerState,
+	probability: number,
+	random: () => number,
+): { decision?: InterventionDecision; state: ControllerState } {
+	if (state.disarmed || state.arm !== "R" || probability <= 0 || random() >= probability) return { state }
+	const interventionId = `${boundary.attemptId}:${boundary.sequence}:specific-correction`
+	if (state.appliedInterventionIds.has(interventionId)) return { state }
+	const applied = new Set(state.appliedInterventionIds)
+	applied.add(interventionId)
+	return {
+		decision: { interventionId, templateId: "specific-correction", reason: "random-schedule" },
+		state: { ...state, disarmed: true, appliedInterventionIds: applied },
+	}
+}
+
+export function applyFixedTemplate(messages: string[], decision: InterventionDecision): void {
+	const text = TEMPLATE_TEXT[decision.templateId]
+	if (typeof text !== "string") throw new Error("unknown intervention template")
+	messages.push(text)
+}

package/src/monitoring/features.ts

@@ -0,0 +1,103 @@
+import { createHash } from "node:crypto"
+import type { MonitorFeatureName, MonitorFeatureVector, PilotBoundary, PilotToolCall, PilotToolResult } from "./types.js"
+import { MONITOR_FEATURE_ORDER } from "./types.js"
+
+export function stableJson(value: unknown): string {
+	if (value === null || typeof value !== "object") return JSON.stringify(value)
+	if (Array.isArray(value)) return `[${value.map(stableJson).join(",")}]`
+	const record = value as Record<string, unknown>
+	return `{${Object.keys(record)
+		.sort()
+		.map((key) => `${JSON.stringify(key)}:${stableJson(record[key])}`)
+		.join(",")}}`
+}
+
+export function sha256(value: unknown): string {
+	return createHash("sha256").update(typeof value === "string" ? value : stableJson(value)).digest("hex")
+}
+
+function isReadOnlyCall(call: PilotToolCall): boolean {
+	return call.name === "read_file" || call.name === "list_files" || call.name === "execute_command"
+}
+
+function isWriteCall(call: PilotToolCall): boolean {
+	return call.name === "apply_diff"
+}
+
+function callSignature(call: PilotToolCall): string {
+	return `${call.name}:${stableJson(call.args)}`
+}
+
+function resultSignature(result: PilotToolResult): string {
+	return `${result.name}:${result.isError}:${result.resultHash}`
+}
+
+function boundedFraction(numerator: number, denominator: number): number {
+	return denominator > 0 ? numerator / denominator : 0
+}
+
+/** Extract only current and recent action/result state. Oracle fields never enter here. */
+export function extractMonitorFeatures(boundary: PilotBoundary): MonitorFeatureVector {
+	const turns = [...boundary.recentTurns].slice(-4)
+	const calls = turns.flatMap((turn) => turn.calls)
+	const results = turns.flatMap((turn) => turn.results)
+	const lastCall = boundary.calls.at(-1)
+	let identicalCallStreak = boundary.deterministicGuardrails.identicalCallStreak
+	if (lastCall !== undefined) {
+		identicalCallStreak = 0
+		for (const call of [...calls].reverse()) {
+			if (callSignature(call) !== callSignature(lastCall)) break
+			identicalCallStreak++
+		}
+	}
+	let sameToolErrorStreak = boundary.deterministicGuardrails.sameToolErrorStreak
+	const lastError = [...results].reverse().find((result) => result.isError)
+	if (lastError !== undefined) {
+		sameToolErrorStreak = 0
+		for (const result of [...results].reverse()) {
+			if (!result.isError || result.name !== lastError.name) break
+			sameToolErrorStreak++
+		}
+	}
+	const distinctCalls = new Set(calls.map(callSignature)).size
+	const errorCount = results.filter((result) => result.isError).length
+	const repeatedResults = results.length - new Set(results.map(resultSignature)).size
+	let turnsSinceSuccessfulVirtualWrite = 0
+	for (const turn of [...turns].reverse()) {
+		if (turn.successfulVirtualWrite) break
+		turnsSinceSuccessfulVirtualWrite++
+	}
+	const values: Record<MonitorFeatureName, number> = {
+		identicalCallStreak,
+		sameToolErrorStreak,
+		readOnlyStreak: boundary.deterministicGuardrails.readOnlyStreak,
+		uniqueCallFraction4: boundedFraction(distinctCalls, calls.length),
+		errorFraction4: boundedFraction(errorCount, results.length),
+		turnsSinceSuccessfulVirtualWrite,
+		repeatedResultFraction4: boundedFraction(repeatedResults, results.length),
+		iterationBudgetFraction: boundary.maxIterations > 0 ? boundary.iteration / boundary.maxIterations : 1,
+	}
+	return values
+}
+
+export function featureVector(features: MonitorFeatureVector): number[] {
+	return MONITOR_FEATURE_ORDER.map((name) => features[name])
+}
+
+export function featureHash(features: MonitorFeatureVector): string {
+	return sha256(featureVector(features))
+}
+
+export function assertFiniteFeatures(features: MonitorFeatureVector): void {
+	for (const name of MONITOR_FEATURE_ORDER) {
+		if (!Number.isFinite(features[name])) throw new Error(`non-finite feature: ${name}`)
+	}
+}
+
+export function currentTurnIsReadOnly(boundary: PilotBoundary): boolean {
+	return boundary.calls.length > 0 && boundary.calls.every(isReadOnlyCall)
+}
+
+export function currentTurnHasWrite(boundary: PilotBoundary): boolean {
+	return boundary.calls.some(isWriteCall) && boundary.results.some((result) => result.name === "apply_diff" && !result.isError)
+}

package/src/monitoring/index.ts

@@ -0,0 +1,4 @@
+export * from "./types.js"
+export * from "./features.js"
+export * from "./predictor.js"
+export * from "./controller.js"

package/src/monitoring/predictor.ts

@@ -0,0 +1,246 @@
+import type {
+	Calibration,
+	FrozenPredictorArtifact,
+	MonitorFeatureName,
+	MonitorFeatureVector,
+	PilotBoundary,
+	Prediction,
+	Standardization,
+} from "./types.js"
+import { MONITOR_FEATURE_ORDER } from "./types.js"
+import { assertFiniteFeatures, extractMonitorFeatures, featureHash, sha256 } from "./features.js"
+
+export interface LabeledWindow {
+	readonly features: MonitorFeatureVector
+	readonly label: 0 | 1
+	readonly trajectoryId: string
+	readonly boundarySequence?: number
+}
+
+export interface CalibrationPoint {
+	readonly score: number
+	readonly label: 0 | 1
+}
+
+export interface FitOptions {
+	readonly modelDigest: string
+	readonly templatesHash: string
+	readonly l2?: 1
+}
+
+function sigmoid(value: number): number {
+	if (value >= 0) {
+		const z = Math.exp(-value)
+		return 1 / (1 + z)
+	}
+	const z = Math.exp(value)
+	return z / (1 + z)
+}
+
+function emptyFeatureRecord(value: number): Record<MonitorFeatureName, number> {
+	return Object.fromEntries(MONITOR_FEATURE_ORDER.map((name) => [name, value])) as Record<MonitorFeatureName, number>
+}
+
+function standardize(rows: readonly LabeledWindow[]): Standardization {
+	const mean = emptyFeatureRecord(0)
+	const scale = emptyFeatureRecord(1)
+	for (const row of rows) {
+		for (const name of MONITOR_FEATURE_ORDER) mean[name] += row.features[name]
+	}
+	for (const name of MONITOR_FEATURE_ORDER) mean[name] /= Math.max(1, rows.length)
+	for (const row of rows) {
+		for (const name of MONITOR_FEATURE_ORDER) {
+			const delta = row.features[name] - mean[name]
+			scale[name] += delta * delta
+		}
+	}
+	for (const name of MONITOR_FEATURE_ORDER) scale[name] = Math.sqrt((scale[name] - 1) / Math.max(1, rows.length)) || 1
+	return { mean, scale }
+}
+
+function standardizedVector(features: MonitorFeatureVector, normalization: Standardization): number[] {
+	assertFiniteFeatures(features)
+	return MONITOR_FEATURE_ORDER.map((name) => (features[name] - normalization.mean[name]) / normalization.scale[name])
+}
+
+function fitLogistic(rows: readonly LabeledWindow[], normalization: Standardization): {
+	coefficients: Record<MonitorFeatureName, number>
+	intercept: number
+} {
+	const coefficients = emptyFeatureRecord(0)
+	let intercept = 0
+	// Fixed optimizer settings are part of the frozen baseline, not tunable data.
+	const learningRate = 0.08
+	for (let step = 0; step < 2_000; step++) {
+		const gradient = emptyFeatureRecord(0)
+		let interceptGradient = 0
+		for (const row of rows) {
+			const x = standardizedVector(row.features, normalization)
+			let logit = intercept
+			for (let i = 0; i < MONITOR_FEATURE_ORDER.length; i++) logit += coefficients[MONITOR_FEATURE_ORDER[i]] * x[i]
+			const error = sigmoid(logit) - row.label
+			interceptGradient += error
+			for (let i = 0; i < MONITOR_FEATURE_ORDER.length; i++) gradient[MONITOR_FEATURE_ORDER[i]] += error * x[i]
+		}
+		const denominator = Math.max(1, rows.length)
+		intercept -= learningRate * (interceptGradient / denominator)
+		for (const name of MONITOR_FEATURE_ORDER) {
+			// Fixed L2 coefficient 1.0; intercept is not regularized.
+			coefficients[name] -= learningRate * (gradient[name] / denominator + coefficients[name] / denominator)
+		}
+	}
+	return { coefficients, intercept }
+}
+
+function rawScore(artifact: Omit<FrozenPredictorArtifact, "artifactHash">, features: MonitorFeatureVector): number {
+	const x = standardizedVector(features, artifact.standardization)
+	let logit = artifact.intercept
+	for (let i = 0; i < MONITOR_FEATURE_ORDER.length; i++) logit += artifact.coefficients[MONITOR_FEATURE_ORDER[i]] * x[i]
+	return sigmoid(logit)
+}
+
+function fitCalibration(points: readonly CalibrationPoint[]): Calibration {
+	let intercept = 0
+	let slope = 1
+	for (let step = 0; step < 1_000; step++) {
+		let gi = 0
+		let gs = 0
+		for (const point of points) {
+			const logit = Math.log(Math.min(1 - 1e-6, Math.max(1e-6, point.score)) / Math.max(1e-6, 1 - point.score))
+			const probability = sigmoid(intercept + slope * logit)
+			const error = probability - point.label
+			gi += error
+			gs += error * logit
+		}
+		const denominator = Math.max(1, points.length)
+		intercept -= 0.05 * gi / denominator
+		slope -= 0.05 * gs / denominator
+	}
+	return { intercept, slope }
+}
+
+function calibratedScore(artifact: Omit<FrozenPredictorArtifact, "artifactHash">, features: MonitorFeatureVector): number {
+	const score = rawScore(artifact, features)
+	const logit = Math.log(Math.min(1 - 1e-6, Math.max(1e-6, score)) / Math.max(1e-6, 1 - score))
+	return sigmoid(artifact.calibration.intercept + artifact.calibration.slope * logit)
+}
+
+export function chooseThresholdFromScores(rows: readonly CalibrationPoint[]): number | undefined {
+	for (let step = 10; step <= 19; step++) {
+		const threshold = step / 20
+		const selected = rows.filter((row) => row.score >= threshold)
+		const truePositive = selected.filter((row) => row.label === 1).length
+		const falsePositive = selected.filter((row) => row.label === 0).length
+		const positives = rows.filter((row) => row.label === 1).length
+		const negatives = rows.filter((row) => row.label === 0).length
+		const precision = selected.length === 0 ? 1 : truePositive / selected.length
+		const falsePositiveRate = negatives === 0 ? 0 : falsePositive / negatives
+		if (precision >= 0.8 && falsePositiveRate <= 0.1 && positives >= 10) return threshold
+	}
+	return undefined
+}
+
+export function fitFrozenPredictor(
+	rows: readonly LabeledWindow[],
+	calibrationRows: readonly LabeledWindow[],
+	options: FitOptions,
+): FrozenPredictorArtifact {
+	if (rows.length === 0) throw new Error("cannot fit predictor without training rows")
+	const standardization = standardize(rows)
+	const model = fitLogistic(rows, standardization)
+	const provisional: Omit<FrozenPredictorArtifact, "artifactHash"> = {
+		schemaVersion: 1,
+		featureOrder: MONITOR_FEATURE_ORDER,
+		coefficients: model.coefficients,
+		intercept: model.intercept,
+		standardization,
+		calibration: fitCalibration(
+			calibrationRows.map((row) => ({
+				score: rawScore({
+					schemaVersion: 1,
+					featureOrder: MONITOR_FEATURE_ORDER,
+					coefficients: model.coefficients,
+					intercept: model.intercept,
+					standardization,
+					calibration: { intercept: 0, slope: 1 },
+					threshold: 0.5,
+					l2: 1,
+					modelDigest: options.modelDigest,
+					templatesHash: options.templatesHash,
+				}, row.features),
+				label: row.label,
+			})),
+		),
+		threshold: 0.5,
+		l2: 1,
+		modelDigest: options.modelDigest,
+		templatesHash: options.templatesHash,
+	}
+	const calibrationScores = calibrationRows.map((row) => ({
+		score: calibratedScore(provisional, row.features),
+		label: row.label,
+	}))
+	const threshold = chooseThresholdFromScores(calibrationScores)
+	if (threshold === undefined) throw new Error("STOP: no qualifying calibration threshold")
+	const frozen = { ...provisional, threshold }
+	return { ...frozen, artifactHash: sha256(frozen) }
+}
+
+export function predict(artifact: FrozenPredictorArtifact, boundary: PilotBoundary): Prediction {
+	const started = process.hrtime.bigint()
+	try {
+		return predictFromFeatures(artifact, extractMonitorFeatures(boundary))
+	} catch (error) {
+		const elapsedNs = Number(process.hrtime.bigint() - started)
+		const features = emptyFeatureRecord(Number.NaN)
+		return {
+			probability: 0,
+			threshold: artifact.threshold,
+			features,
+			featureHash: sha256(features),
+			predictorHash: artifact.artifactHash,
+			latencyNs: elapsedNs,
+			abstentionReason: error instanceof Error && error.message.includes("finite")
+				? "non-finite"
+				: error instanceof Error && error.message.includes("unknown model")
+					? "unknown-model"
+					: "malformed",
+		}
+	}
+}
+
+export function predictFromFeatures(artifact: FrozenPredictorArtifact, features: MonitorFeatureVector): Prediction {
+	const started = process.hrtime.bigint()
+	try {
+		assertFiniteFeatures(features)
+		if (artifact.modelDigest.trim() === "") throw new Error("unknown model")
+		const provisional = artifact as Omit<FrozenPredictorArtifact, "artifactHash">
+		const probability = calibratedScore(provisional, features)
+		return {
+			probability,
+			threshold: artifact.threshold,
+			features,
+			featureHash: featureHash(features),
+			predictorHash: artifact.artifactHash,
+			latencyNs: Number(process.hrtime.bigint() - started),
+		}
+	} catch (error) {
+		return {
+			probability: 0,
+			threshold: artifact.threshold,
+			features,
+			featureHash: sha256(features),
+			predictorHash: artifact.artifactHash,
+			latencyNs: Number(process.hrtime.bigint() - started),
+			abstentionReason: error instanceof Error && error.message.includes("finite")
+				? "non-finite"
+				: error instanceof Error && error.message.includes("unknown model")
+					? "unknown-model"
+					: "malformed",
+		}
+	}
+}
+
+export function predictedIntervention(artifact: FrozenPredictorArtifact, prediction: Prediction): boolean {
+	return prediction.abstentionReason === undefined && prediction.probability >= artifact.threshold
+}

package/src/monitoring/types.ts

@@ -0,0 +1,192 @@
+import type { IterationInterventionTemplateId } from "../engine/types.js"
+
+export { type IterationInterventionTemplateId }
+
+export const MONITOR_FEATURE_ORDER = [
+	"identicalCallStreak",
+	"sameToolErrorStreak",
+	"readOnlyStreak",
+	"uniqueCallFraction4",
+	"errorFraction4",
+	"turnsSinceSuccessfulVirtualWrite",
+	"repeatedResultFraction4",
+	"iterationBudgetFraction",
+] as const
+
+export type MonitorFeatureName = (typeof MONITOR_FEATURE_ORDER)[number]
+export type MonitorFeatureVector = Record<MonitorFeatureName, number>
+
+export type PilotArm = "B" | "M" | "R" | "G"
+export type FixtureFamily = "wrong-key-path" | "revision-mismatch" | "failed-validation" | "legitimate-investigation"
+export type FixtureSplit = "train" | "calibration" | "test"
+
+export interface PilotFileSet {
+	readonly [relativePath: string]: string
+}
+
+export interface PilotFixture {
+	readonly id: string
+	readonly family: FixtureFamily
+	readonly split: FixtureSplit
+	readonly task: string
+	readonly initialFiles: PilotFileSet
+	readonly targetFiles: PilotFileSet
+	readonly unrelatedPaths: readonly string[]
+	readonly validator: (files: PilotFileSet) => { ok: boolean; message: string; milestone: number }
+	readonly oracle: (files: PilotFileSet) => { success: boolean; reason: string; milestone: number }
+}
+
+export interface PilotToolCall {
+	readonly id: string
+	readonly name: string
+	readonly args: Readonly<Record<string, unknown>>
+}
+
+export interface PilotToolResult {
+	readonly callId: string
+	readonly name: string
+	readonly isError: boolean
+	readonly content: string
+	readonly normalizedError?: string
+	readonly resultHash: string
+}
+
+export interface PilotTurnSummary {
+	readonly iteration: number
+	readonly calls: readonly PilotToolCall[]
+	readonly results: readonly PilotToolResult[]
+	readonly readOnly: boolean
+	readonly successfulVirtualWrite: boolean
+	readonly milestone: number
+}
+
+/** Complete supervisor-owned observation. It contains no oracle labels. */
+export interface PilotBoundary {
+	readonly schemaVersion: 1
+	readonly experimentId: string
+	readonly runId: string
+	readonly attemptId: string
+	readonly epoch: number
+	readonly sequence: number
+	readonly iteration: number
+	readonly maxIterations: number
+	readonly calls: readonly PilotToolCall[]
+	readonly results: readonly PilotToolResult[]
+	readonly recentTurns: readonly PilotTurnSummary[]
+	readonly virtualRevisionHashes: Readonly<Record<string, string>>
+	readonly inputTokens: number
+	readonly outputTokens: number
+	readonly reasoningAvailable: boolean
+	readonly truncationFlags: readonly string[]
+	readonly deterministicGuardrails: Readonly<{
+		consecutiveMistakes: number
+		identicalCallStreak: number
+		readOnlyStreak: number
+		sameToolErrorStreak: number
+		repeatedToolFailureStreak: number
+	}>
+}
+
+export interface Standardization {
+	readonly mean: Record<MonitorFeatureName, number>
+	readonly scale: Record<MonitorFeatureName, number>
+}
+
+export interface Calibration {
+	readonly intercept: number
+	readonly slope: number
+}
+
+export interface FrozenPredictorArtifact {
+	readonly schemaVersion: 1
+	readonly featureOrder: readonly MonitorFeatureName[]
+	readonly coefficients: Record<MonitorFeatureName, number>
+	readonly intercept: number
+	readonly standardization: Standardization
+	readonly calibration: Calibration
+	readonly threshold: number
+	readonly l2: 1
+	readonly modelDigest: string
+	readonly templatesHash: string
+	readonly artifactHash: string
+}
+
+export interface Prediction {
+	readonly probability: number
+	readonly threshold: number
+	readonly features: MonitorFeatureVector
+	readonly featureHash: string
+	readonly predictorHash: string
+	readonly latencyNs: number
+	readonly abstentionReason?: "missing" | "non-finite" | "unknown-model" | "malformed"
+}
+
+export interface InterventionDecision {
+	readonly interventionId: string
+	readonly templateId: IterationInterventionTemplateId
+	readonly reason: "threshold" | "random-schedule"
+}
+
+export interface PilotInterventionRecord extends InterventionDecision {
+	readonly boundarySequence: number
+	readonly delivered: boolean
+}
+
+export interface PilotBudget {
+	readonly maxCalls: 16
+	readonly maxTokens: 20_000
+	readonly maxWallMs: number
+	readonly globalMaxTokens: number
+	readonly globalMaxWallMs: number
+}
+
+export interface PilotManifest {
+	readonly schemaVersion: 1
+	readonly experimentId: string
+	readonly createdAt: string
+	readonly modelDigest: string
+	readonly modelSettings: Readonly<Record<string, unknown>>
+	readonly fixtureHash: string
+	readonly splitHash: string
+	readonly templatesHash: string
+	readonly featureOrder: readonly MonitorFeatureName[]
+	readonly predictorHash?: string
+	readonly threshold?: number
+	readonly armAssignments: readonly PilotArm[]
+	readonly seeds: readonly number[]
+	readonly budget: PilotBudget
+	readonly randomProbability?: number
+	readonly randomBoundaryDistribution?: readonly Readonly<{ boundarySequence: number; weight: number }>[]
+}
+
+export interface PilotOutcome {
+	readonly experimentId: string
+	readonly runId: string
+	readonly attemptId: string
+	readonly epoch: number
+	readonly fixtureId: string
+	readonly split: FixtureSplit
+	readonly arm: PilotArm
+	readonly valid: boolean
+	readonly success: boolean
+	readonly finalArtifactHash: string
+	readonly oracleHash: string
+	readonly verifierHash: string
+	readonly milestoneCount: number
+	readonly redundantCalls: number
+	readonly invalidRetries: number
+	readonly forbiddenEffectAttempts: number
+	readonly experimentalInterventionAttempted: boolean
+	readonly correctionDelivered: boolean
+	readonly deterministicInterventionCount: number
+	readonly tokens: number
+	readonly calls: number
+	readonly elapsedMs: number
+	readonly stopReason: string
+	readonly predictionRecords: readonly Prediction[]
+	readonly interventions: readonly PilotInterventionRecord[]
+	readonly scheduledIntervention: boolean
+	readonly scheduledBoundarySequence?: number
+	/** Labels are retained only in the outcome analysis, never passed to prediction. */
+	readonly detectionLabels: readonly (0 | 1)[]
+}