@oh-my-pi/pi-coding-agent 13.18.0 → 14.0.2

package/src/autoresearch/prompt.md

@@ -8,7 +8,11 @@ Autoresearch mode is active.
 Primary goal:
 {{goal}}
 {{else}}
+{{#if has_autoresearch_md}}
 Primary goal is documented in `autoresearch.md` for this session.
+{{else}}
+There is no `autoresearch.md` yet. Infer what to optimize from the latest user message and the conversation; after you create `autoresearch.md`, keep it as the durable source of truth for goal and benchmark contract.
+{{/if}}
 {{/if}}
 
 Working directory:
@@ -63,7 +67,7 @@ An unlogged run artifact exists at `{{pending_run_directory}}`.
 
 - `init_experiment` — initialize or reset the experiment session for the current optimization target.
 - `run_experiment` — run a benchmark or experiment command with timing, output capture, structured metric parsing, and optional backpressure checks.
-- `log_experiment` — record the result, update the dashboard, persist JSONL history, auto-commit kept experiments, and auto-revert discarded or failed experiments.
+- `log_experiment` — record the result, update the dashboard, persist JSONL history, auto-commit kept experiments, and revert only run-modified files for discarded or failed experiments (pre-existing uncommitted changes are preserved).
 
 ### Operating protocol
 
@@ -83,6 +87,8 @@ An unlogged run artifact exists at `{{pending_run_directory}}`.
    - Use the same workload every run unless you intentionally re-initialize with a new segment.
    - Keep the measurement harness, evaluator, and fixed benchmark inputs stable unless you intentionally start a new segment and document the change.
 4. Initialize the loop with `init_experiment` before the first logged run of a segment.
+   - Pass `from_autoresearch_md: true` with only `name` to load the benchmark contract from `autoresearch.md` without mirroring every field in the tool call.
+   - Use `abandon_unlogged_runs: true` only when you intentionally discard unlogged run artifacts and need a fresh segment (for example after a bad or obsolete benchmark directory).
 5. Run a baseline first.
    - Establish the baseline metric before attempting optimizations.
    - Track secondary metrics only when they matter to correctness, quality, or obvious regressions.
@@ -90,7 +96,9 @@ An unlogged run artifact exists at `{{pending_run_directory}}`.
    - Make one coherent experiment at a time.
    - Run `run_experiment`.
    - Interpret the result honestly.
-   - Call `log_experiment` after every run.
+   - Call `log_experiment` after every run (it refreshes benchmark/scope fields from `autoresearch.md` before logging so keep validation matches the file on disk).
+   - Use `run_experiment` with `force: true` only when you must override the segment benchmark command or skip the direct-`autoresearch.sh` rule.
+   - On `log_experiment`, `force: true` relaxes ASI requirements and allows keeping a primary-metric regression; prefer normal logging when possible.
 7. Keep the primary metric as the decision maker.
    - `keep` when the primary metric improves.
    - `discard` when it regresses or stays flat.
@@ -137,7 +145,11 @@ Suggested structure:
 {{#if has_goal}}
 - {{goal}}
 {{else}}
+{{#if has_autoresearch_md}}
 - document the active target here before the first benchmark
+{{else}}
+- (derive from the user's messages, then record here)
+{{/if}}
 {{/if}}
 
 ## Benchmark
@@ -194,15 +206,20 @@ Resume from the existing notes:
 {{else}}
 ### Initial setup
 
-`autoresearch.md` does not exist yet.
+`autoresearch.md` does not exist yet. You decide the benchmark contract, harness, and scope from the user's messages and the repository—do not ask the user to re-type benchmark commands or metric names in a separate UI prompt.
+
+Before the first benchmark:
 
-Create the experiment workspace before the first benchmark:
+- Write `autoresearch.md` with goal, benchmark command (must be a **direct** invocation of `autoresearch.sh`, e.g. `bash autoresearch.sh`), primary metric name and unit, direction (`lower` or `higher`), tradeoff metrics if relevant, files in scope, off limits, and constraints.
+- Add a short preflight section: prerequisites, one-time setup, and the comparability invariant that must stay fixed across runs.
+- Mark ground-truth evaluators, fixed datasets, and other measurement-critical files as off limits or hard constraints when they define the benchmark contract.
+- Write or update `autoresearch.program.md` when you learn durable heuristics, failure patterns, or repo-specific strategy for later resume turns.
+- Create `autoresearch.sh` as the canonical benchmark entrypoint; print the primary metric as `METRIC <name>=<number>` and optional secondary metrics as additional `METRIC` lines.
+- Optionally add `autoresearch.checks.sh` if correctness or quality needs a hard gate.
+- Call `init_experiment` with arguments that match `autoresearch.md` exactly (benchmark command, metric, unit, direction, scope paths, off limits, constraints).
+- Run and log the baseline.
 
-- write `autoresearch.md`
-- write `autoresearch.sh`
-- optionally write `autoresearch.checks.sh`
-- run `init_experiment`
-- run and log the baseline
+Until `init_experiment` succeeds, only autoresearch control files (`autoresearch.md`, `autoresearch.sh`, `autoresearch.program.md`, `autoresearch.ideas.md`, `autoresearch.checks.sh`) may be edited; after initialization, respect Files in Scope from the contract.
 
 {{/if}}
 {{#if has_checks}}

package/src/autoresearch/state.ts

@@ -34,7 +34,6 @@ export function createExperimentState(): ExperimentState {
 		scopePaths: [],
 		offLimits: [],
 		constraints: [],
-		segmentFingerprint: null,
 	};
 }
 
@@ -203,8 +202,6 @@ export function reconstructStateFromJsonl(workDir: string): ReconstructedExperim
 			state.scopePaths = cloneStringArray(configEntry.scopePaths);
 			state.offLimits = cloneStringArray(configEntry.offLimits);
 			state.constraints = cloneStringArray(configEntry.constraints);
-			state.segmentFingerprint =
-				typeof configEntry.segmentFingerprint === "string" ? configEntry.segmentFingerprint : null;
 			state.secondaryMetrics = hydrateMetricDefs(configEntry.secondaryMetrics);
 			continue;
 		}
@@ -322,9 +319,6 @@ function parseConfigEntry(value: unknown): AutoresearchJsonConfigEntry | null {
 			candidate.constraints.filter((item): item is string => typeof item === "string"),
 		);
 	}
-	if (typeof candidate.segmentFingerprint === "string" && candidate.segmentFingerprint.trim().length > 0) {
-		config.segmentFingerprint = candidate.segmentFingerprint;
-	}
 	return config;
 }
 

package/src/autoresearch/tools/init-experiment.ts

@@ -6,15 +6,15 @@ import { Type } from "@sinclair/typebox";
 import type { ToolDefinition } from "../../extensibility/extensions";
 import type { Theme } from "../../modes/theme/theme";
 import { replaceTabs, truncateToWidth } from "../../tools/render-utils";
+import { applyAutoresearchContractToExperimentState } from "../apply-contract-to-state";
 import {
-	buildAutoresearchSegmentFingerprint,
 	contractListsEqual,
 	contractPathListsEqual,
 	loadAutoresearchScriptSnapshot,
 	readAutoresearchContract,
 } from "../contract";
 import {
-	inferMetricUnitFromName,
+	abandonUnloggedAutoresearchRuns,
 	isAutoresearchShCommand,
 	readMaxExperiments,
 	readPendingRunSummary,
@@ -28,9 +28,29 @@ const initExperimentSchema = Type.Object({
 	name: Type.String({
 		description: "Human-readable experiment name.",
 	}),
-	metric_name: Type.String({
-		description: "Primary metric name shown in the dashboard.",
-	}),
+	from_autoresearch_md: Type.Optional(
+		Type.Boolean({
+			description:
+				"When true, load benchmark command, metrics, scope, off-limits, and constraints from autoresearch.md instead of passing mirrored fields below.",
+		}),
+	),
+	abandon_unlogged_runs: Type.Optional(
+		Type.Boolean({
+			description:
+				"When true, mark all completed but unlogged run artifacts as abandoned so initialization can proceed without logging them first.",
+		}),
+	),
+	new_segment: Type.Optional(
+		Type.Boolean({
+			description:
+				"When true, force a new segment even when the contract fields have not changed. Without this, re-initialization with matching contract is a no-op.",
+		}),
+	),
+	metric_name: Type.Optional(
+		Type.String({
+			description: "Primary metric name shown in the dashboard. Required when from_autoresearch_md is false.",
+		}),
+	),
 	metric_unit: Type.Optional(
 		Type.String({
 			description: "Unit for the primary metric, for example µs, ms, s, kb, or empty.",
@@ -41,13 +61,17 @@ const initExperimentSchema = Type.Object({
 			description: "Whether lower or higher values are better. Defaults to lower.",
 		}),
 	),
-	benchmark_command: Type.String({
-		description: "Benchmark command recorded in autoresearch.md.",
-	}),
-	scope_paths: Type.Array(Type.String(), {
-		description: "Files in Scope from autoresearch.md. Must be non-empty.",
-		minItems: 1,
-	}),
+	benchmark_command: Type.Optional(
+		Type.String({
+			description: "Benchmark command recorded in autoresearch.md. Required when from_autoresearch_md is false.",
+		}),
+	),
+	scope_paths: Type.Optional(
+		Type.Array(Type.String(), {
+			description: "Files in Scope from autoresearch.md. Required when from_autoresearch_md is false.",
+			minItems: 1,
+		}),
+	),
 	off_limits: Type.Optional(
 		Type.Array(Type.String(), {
 			description: "Off Limits paths from autoresearch.md.",
@@ -86,25 +110,43 @@ export function createInitExperimentTool(
 			const state = runtime.state;
 			const isReinitializing = state.results.length > 0;
 			const workDir = resolveWorkDir(ctx.cwd);
-			const pendingRun = await readPendingRunSummary(workDir, collectLoggedRunNumbers(state.results));
+			const loggedRunNumbers = collectLoggedRunNumbers(state.results);
+
+			let abandonSummary = "";
+			if (params.abandon_unlogged_runs === true) {
+				const abandoned = await abandonUnloggedAutoresearchRuns(workDir, loggedRunNumbers);
+				if (abandoned > 0) {
+					abandonSummary =
+						abandoned === 1
+							? "Abandoned 1 unlogged run artifact.\n"
+							: `Abandoned ${abandoned} unlogged run artifacts.\n`;
+				}
+			}
+
+			const pendingRun = await readPendingRunSummary(workDir, loggedRunNumbers);
 			if (pendingRun) {
+				const metricInfo = pendingRun.parsedPrimary !== null ? `, metric=${pendingRun.parsedPrimary}` : "";
+				const passedInfo = pendingRun.passed ? "passed" : "failed";
 				return {
 					content: [
 						{
 							type: "text",
 							text:
-								`Error: run #${pendingRun.runNumber} has not been logged yet. ` +
-								"Call log_experiment before re-initializing the current segment.",
+								abandonSummary +
+								`Error: run #${pendingRun.runNumber} has not been logged yet.\n` +
+								`Pending: command="${pendingRun.command}"${metricInfo}, ${passedInfo}\n` +
+								"Call log_experiment before re-initializing, or pass abandon_unlogged_runs=true.",
 						},
 					],
 				};
 			}
+
 			const contractResult = readAutoresearchContract(workDir);
 			const scriptSnapshot = loadAutoresearchScriptSnapshot(workDir);
 			const errors = [...contractResult.errors, ...scriptSnapshot.errors];
 			if (errors.length > 0) {
 				return {
-					content: [{ type: "text", text: `Error: ${errors.join(" ")}` }],
+					content: [{ type: "text", text: `${abandonSummary}Error: ${errors.join(" ")}` }],
 				};
 			}
 
@@ -117,118 +159,161 @@ export function createInitExperimentTool(
 						{
 							type: "text",
 							text:
+								abandonSummary +
 								"Error: Benchmark.command in autoresearch.md must invoke `autoresearch.sh` directly. " +
 								"Move the real workload into `autoresearch.sh` and re-run init_experiment.",
 						},
 					],
 				};
 			}
-			if (benchmarkContract.command !== params.benchmark_command.trim()) {
-				return {
-					content: [
-						{
-							type: "text",
-							text:
-								"Error: benchmark_command does not match autoresearch.md. " +
-								`Expected: ${benchmarkContract.command ?? "(missing)"}\nReceived: ${params.benchmark_command}`,
-						},
-					],
-				};
-			}
-			if (benchmarkContract.primaryMetric !== params.metric_name.trim()) {
-				return {
-					content: [
-						{
-							type: "text",
-							text:
-								"Error: metric_name does not match autoresearch.md. " +
-								`Expected: ${benchmarkContract.primaryMetric ?? "(missing)"}\nReceived: ${params.metric_name}`,
-						},
-					],
-				};
-			}
-			if ((params.metric_unit ?? "") !== expectedMetricUnit) {
-				return {
-					content: [
-						{
-							type: "text",
-							text:
-								"Error: metric_unit does not match autoresearch.md. " +
-								`Expected: ${expectedMetricUnit || "(empty)"}\nReceived: ${params.metric_unit ?? "(empty)"}`,
-						},
-					],
-				};
-			}
-			if ((params.direction ?? "lower") !== expectedDirection) {
-				return {
-					content: [
-						{
-							type: "text",
-							text:
-								"Error: direction does not match autoresearch.md. " +
-								`Expected: ${expectedDirection}\nReceived: ${params.direction ?? "lower"}`,
-						},
-					],
-				};
-			}
-			if (!contractPathListsEqual(params.scope_paths, contractResult.contract.scopePaths)) {
-				return {
-					content: [
-						{
-							type: "text",
-							text:
-								"Error: scope_paths do not match autoresearch.md. " +
-								`Expected: ${contractResult.contract.scopePaths.join(", ")}`,
-						},
-					],
-				};
-			}
-			if (!contractPathListsEqual(params.off_limits ?? [], contractResult.contract.offLimits)) {
-				return {
-					content: [
-						{
-							type: "text",
-							text:
-								"Error: off_limits do not match autoresearch.md. " +
-								`Expected: ${contractResult.contract.offLimits.join(", ") || "(empty)"}`,
-						},
-					],
-				};
-			}
-			if (!contractListsEqual(params.constraints ?? [], contractResult.contract.constraints)) {
-				return {
-					content: [
-						{
-							type: "text",
-							text:
-								"Error: constraints do not match autoresearch.md. " +
-								`Expected: ${contractResult.contract.constraints.join(", ") || "(empty)"}`,
-						},
-					],
-				};
+
+			const fromMd = params.from_autoresearch_md === true;
+			if (!fromMd) {
+				const metricName = params.metric_name?.trim();
+				const benchmarkCommand = params.benchmark_command?.trim();
+				const scopePaths = params.scope_paths;
+				if (!metricName || !benchmarkCommand || !scopePaths || scopePaths.length === 0) {
+					return {
+						content: [
+							{
+								type: "text",
+								text:
+									abandonSummary +
+									"Error: when from_autoresearch_md is false or omitted, metric_name, benchmark_command, and scope_paths are required and must match autoresearch.md. " +
+									"Alternatively pass from_autoresearch_md=true with only name (plus optional flags).",
+							},
+						],
+					};
+				}
+				if (benchmarkContract.command !== benchmarkCommand) {
+					return {
+						content: [
+							{
+								type: "text",
+								text:
+									abandonSummary +
+									"Error: benchmark_command does not match autoresearch.md. " +
+									`Expected: ${benchmarkContract.command ?? "(missing)"}\nReceived: ${params.benchmark_command}`,
+							},
+						],
+					};
+				}
+				if (benchmarkContract.primaryMetric !== metricName) {
+					return {
+						content: [
+							{
+								type: "text",
+								text:
+									abandonSummary +
+									"Error: metric_name does not match autoresearch.md. " +
+									`Expected: ${benchmarkContract.primaryMetric ?? "(missing)"}\nReceived: ${params.metric_name}`,
+							},
+						],
+					};
+				}
+				if ((params.metric_unit ?? "") !== expectedMetricUnit) {
+					return {
+						content: [
+							{
+								type: "text",
+								text:
+									abandonSummary +
+									"Error: metric_unit does not match autoresearch.md. " +
+									`Expected: ${expectedMetricUnit || "(empty)"}\nReceived: ${params.metric_unit ?? "(empty)"}`,
+							},
+						],
+					};
+				}
+				if ((params.direction ?? "lower") !== expectedDirection) {
+					return {
+						content: [
+							{
+								type: "text",
+								text:
+									abandonSummary +
+									"Error: direction does not match autoresearch.md. " +
+									`Expected: ${expectedDirection}\nReceived: ${params.direction ?? "lower"}`,
+							},
+						],
+					};
+				}
+				if (!contractPathListsEqual(scopePaths, contractResult.contract.scopePaths)) {
+					return {
+						content: [
+							{
+								type: "text",
+								text:
+									abandonSummary +
+									"Error: scope_paths do not match autoresearch.md. " +
+									`Expected: ${contractResult.contract.scopePaths.join(", ")}`,
+							},
+						],
+					};
+				}
+				if (!contractPathListsEqual(params.off_limits ?? [], contractResult.contract.offLimits)) {
+					return {
+						content: [
+							{
+								type: "text",
+								text:
+									abandonSummary +
+									"Error: off_limits do not match autoresearch.md. " +
+									`Expected: ${contractResult.contract.offLimits.join(", ") || "(empty)"}`,
+							},
+						],
+					};
+				}
+				if (!contractListsEqual(params.constraints ?? [], contractResult.contract.constraints)) {
+					return {
+						content: [
+							{
+								type: "text",
+								text:
+									abandonSummary +
+									"Error: constraints do not match autoresearch.md. " +
+									`Expected: ${contractResult.contract.constraints.join(", ") || "(empty)"}`,
+							},
+						],
+					};
+				}
 			}
 
-			const segmentFingerprint = buildAutoresearchSegmentFingerprint(contractResult.contract, {
-				benchmarkScript: scriptSnapshot.benchmarkScript,
-				checksScript: scriptSnapshot.checksScript,
-			});
+			// Check if contract matches current state — if so, re-init is a no-op
+			if (isReinitializing && params.new_segment !== true) {
+				const contract = contractResult.contract;
+				const bm = contract.benchmark;
+				const contractMatches =
+					(bm.primaryMetric ?? "metric") === state.metricName &&
+					bm.metricUnit === state.metricUnit &&
+					(bm.direction ?? "lower") === state.bestDirection &&
+					(bm.command ?? null) === state.benchmarkCommand &&
+					contractPathListsEqual(contract.scopePaths, state.scopePaths) &&
+					contractPathListsEqual(contract.offLimits, state.offLimits) &&
+					contractListsEqual(contract.constraints, state.constraints);
+				if (contractMatches) {
+					runtime.autoresearchMode = true;
+					runtime.autoResumeArmed = true;
+					options.dashboard.updateWidget(ctx, runtime);
+					options.dashboard.requestRender();
+					return {
+						content: [
+							{
+								type: "text",
+								text:
+									abandonSummary +
+									`Experiment session already initialized with matching contract. Continuing segment ${state.currentSegment}.`,
+							},
+						],
+						details: { state: cloneExperimentState(state) },
+					};
+				}
+			}
 
+			applyAutoresearchContractToExperimentState(contractResult.contract, state);
 			state.name = params.name;
-			state.metricName = params.metric_name;
-			state.metricUnit = params.metric_unit ?? "";
-			state.bestDirection = params.direction ?? "lower";
 			state.maxExperiments = readMaxExperiments(ctx.cwd);
 			state.bestMetric = null;
 			state.confidence = null;
-			state.secondaryMetrics = benchmarkContract.secondaryMetrics.map(name => ({
-				name,
-				unit: inferMetricUnitFromName(name),
-			}));
-			state.benchmarkCommand = params.benchmark_command.trim();
-			state.scopePaths = [...contractResult.contract.scopePaths];
-			state.offLimits = [...contractResult.contract.offLimits];
-			state.constraints = [...contractResult.contract.constraints];
-			state.segmentFingerprint = segmentFingerprint;
 			if (isReinitializing) {
 				state.currentSegment += 1;
 			}
@@ -245,7 +330,6 @@ export function createInitExperimentTool(
 				scopePaths: state.scopePaths,
 				offLimits: state.offLimits,
 				constraints: state.constraints,
-				segmentFingerprint,
 			});
 
 			if (isReinitializing) {
@@ -267,6 +351,7 @@ export function createInitExperimentTool(
 			options.dashboard.requestRender();
 
 			const lines = [
+				abandonSummary.trimEnd(),
 				`Experiment initialized: ${state.name}`,
 				`Metric: ${state.metricName} (${state.metricUnit || "unitless"}, ${state.bestDirection} is better)`,
 				`Benchmark command: ${state.benchmarkCommand}`,
@@ -275,7 +360,7 @@ export function createInitExperimentTool(
 				isReinitializing
 					? "Previous results remain in history. This starts a new segment and requires a fresh baseline."
 					: "Now run the baseline experiment and log it.",
-			];
+			].filter(line => line.length > 0);
 			if (state.maxExperiments !== null) {
 				lines.push(`Max iterations: ${state.maxExperiments}`);
 			}