@f5-sales-demo/xcsh 19.51.2

package/src/autoresearch/index.ts

@@ -0,0 +1,402 @@
+import * as fs from "node:fs";
+import * as path from "node:path";
+import type { AutocompleteItem } from "@f5xc-salesdemos/pi-tui";
+import { prompt } from "@f5xc-salesdemos/pi-utils";
+import type { ExtensionContext, ExtensionFactory } from "../extensibility/extensions";
+import commandResumeTemplate from "./command-resume.md" with { type: "text" };
+import { pathMatchesContractPath } from "./contract";
+import { createDashboardController } from "./dashboard";
+import { ensureAutoresearchBranch } from "./git";
+import {
+	formatNum,
+	isAutoresearchCommittableFile,
+	isAutoresearchLocalStatePath,
+	normalizeAutoresearchPath,
+	readMaxExperiments,
+	readPendingRunSummary,
+	resolveWorkDir,
+	validateWorkDir,
+} from "./helpers";
+import promptTemplate from "./prompt.md" with { type: "text" };
+import resumeMessageTemplate from "./resume-message.md" with { type: "text" };
+import {
+	cloneExperimentState,
+	createExperimentState,
+	createRuntimeStore,
+	currentResults,
+	findBaselineMetric,
+	findBestKeptResult,
+	reconstructControlState,
+	reconstructStateFromJsonl,
+} from "./state";
+import { createInitExperimentTool } from "./tools/init-experiment";
+import { createLogExperimentTool } from "./tools/log-experiment";
+import { createRunExperimentTool } from "./tools/run-experiment";
+import type { AutoresearchRuntime, ExperimentResult, PendingRunSummary } from "./types";
+
+const EXPERIMENT_TOOL_NAMES = ["init_experiment", "run_experiment", "log_experiment"];
+const EXPERIMENT_TOOL_SET = new Set(EXPERIMENT_TOOL_NAMES);
+const addExperimentTools = (tools: string[]) => [...new Set([...tools, ...EXPERIMENT_TOOL_NAMES])];
+const removeExperimentTools = (tools: string[]) => tools.filter(name => !EXPERIMENT_TOOL_SET.has(name));
+export const createAutoresearchExtension: ExtensionFactory = api => {
+	const runtimeStore = createRuntimeStore();
+	const dashboard = createDashboardController();
+	const getSessionKey = (ctx: ExtensionContext): string => ctx.sessionManager.getSessionId();
+	const getRuntime = (ctx: ExtensionContext): AutoresearchRuntime => runtimeStore.ensure(getSessionKey(ctx));
+	const rehydrate = async (ctx: ExtensionContext): Promise<void> => {
+		const runtime = getRuntime(ctx);
+		const workDir = resolveWorkDir(ctx.cwd);
+		const reconstructed = reconstructStateFromJsonl(workDir);
+		const control = reconstructControlState(ctx.sessionManager.getBranch());
+		const loggedRunNumbers = collectLoggedRunNumbers(reconstructed.state.results);
+		runtime.state = cloneExperimentState(reconstructed.state);
+		runtime.state.maxExperiments = readMaxExperiments(ctx.cwd);
+		runtime.goal = control.goal;
+		runtime.autoresearchMode = control.autoresearchMode;
+		runtime.autoResumeArmed = false;
+		runtime.lastAutoResumePendingRunNumber = null;
+		applyPendingRunToRuntime(runtime, await readPendingRunSummary(workDir, loggedRunNumbers));
+		runtime.runningExperiment = null;
+		dashboard.updateWidget(ctx, runtime);
+		const activeTools = api.getActiveTools();
+
+		const nextActiveTools = runtime.autoresearchMode
+			? addExperimentTools(activeTools)
+			: removeExperimentTools(activeTools);
+		const toolsChanged =
+			nextActiveTools.length !== activeTools.length ||
+			nextActiveTools.some((name, index) => name !== activeTools[index]);
+		if (toolsChanged) await api.setActiveTools(nextActiveTools);
+	};
+	const setMode = (
+		ctx: ExtensionContext,
+		enabled: boolean,
+		goal: string | null,
+		mode: "on" | "off" | "clear",
+	): void => {
+		const runtime = getRuntime(ctx);
+		runtime.autoresearchMode = enabled;
+		runtime.autoResumeArmed = false;
+		runtime.goal = goal;
+		runtime.lastAutoResumePendingRunNumber = null;
+		api.appendEntry("autoresearch-control", goal ? { mode, goal } : { mode });
+	};
+	api.registerTool(createInitExperimentTool({ dashboard, getRuntime, pi: api }));
+	api.registerTool(createRunExperimentTool({ dashboard, getRuntime, pi: api }));
+	api.registerTool(createLogExperimentTool({ dashboard, getRuntime, pi: api }));
+	api.on("tool_call", (event, ctx) => {
+		const runtime = getRuntime(ctx);
+		if (!runtime.autoresearchMode) return;
+		if (event.toolName !== "write" && event.toolName !== "edit" && event.toolName !== "ast_edit") return;
+
+		const rawPaths =
+			event.toolName === "write" || event.toolName === "ast_edit"
+				? typeof event.input.path === "string"
+					? [event.input.path]
+					: null
+				: event.toolName === "edit"
+					? ["path", "rename", "move"].map(k => event.input[k]).filter((v): v is string => typeof v === "string")
+					: [];
+		if (rawPaths === null)
+			return {
+				block: true,
+				reason:
+					"Autoresearch requires an explicit target path for this editing tool so it can enforce Files in Scope and Off Limits before changes are made.",
+			};
+
+		const workDir = resolveWorkDir(ctx.cwd);
+		for (const rawPath of rawPaths) {
+			const relativePath = resolveAutoresearchRelativePath(workDir, rawPath);
+			if (!relativePath.ok) return { block: true, reason: relativePath.reason };
+			const validationError = validateEditableAutoresearchPath(relativePath.relativePath, runtime);
+			if (validationError)
+				return {
+					block: true,
+					reason: `Autoresearch blocked edits to ${relativePath.relativePath}: ${validationError}`,
+				};
+		}
+	});
+	api.registerCommand("autoresearch", {
+		description: "Toggle builtin autoresearch mode, or pass off / clear, or a goal message.",
+		getArgumentCompletions(argumentPrefix: string): AutocompleteItem[] | null {
+			if (argumentPrefix.includes(" ")) return null;
+			const normalized = argumentPrefix.trim().toLowerCase();
+			// No suggestions for an empty argument prefix so Tab after "/autoresearch " does not
+			// force-complete into off/clear; bare command submit toggles like /plan.
+			if (normalized.length === 0) return null;
+			const completions: AutocompleteItem[] = [
+				{ label: "off", value: "off", description: "Leave autoresearch mode" },
+				{ label: "clear", value: "clear", description: "Delete autoresearch.jsonl and leave autoresearch mode" },
+			];
+			const filtered = completions.filter(item => item.label.startsWith(normalized));
+			return filtered.length > 0 ? filtered : null;
+		},
+		async handler(args, ctx): Promise<void> {
+			const trimmed = args.trim();
+			const runtime = getRuntime(ctx);
+			const workDirError = validateWorkDir(ctx.cwd);
+			if (workDirError) {
+				ctx.ui.notify(workDirError, "error");
+				return;
+			}
+
+			if ((trimmed === "" && runtime.autoresearchMode) || trimmed === "off" || trimmed === "clear") {
+				if (trimmed === "clear") {
+					const workDir = resolveWorkDir(ctx.cwd);
+					fs.rmSync(path.join(workDir, "autoresearch.jsonl"), { force: true });
+					fs.rmSync(path.join(workDir, ".autoresearch"), { force: true, recursive: true });
+					runtime.state = createExperimentState();
+					runtime.state.maxExperiments = readMaxExperiments(ctx.cwd);
+					runtime.goal = null;
+					applyPendingRunToRuntime(runtime, null);
+				}
+				const mode = trimmed === "clear" ? "clear" : "off";
+				setMode(ctx, false, mode === "clear" ? null : runtime.goal, mode);
+				dashboard.updateWidget(ctx, runtime);
+				await api.setActiveTools(removeExperimentTools(api.getActiveTools()));
+				ctx.ui.notify(mode === "clear" ? "Autoresearch local state cleared" : "Autoresearch mode disabled", "info");
+				return;
+			}
+
+			const workDir = resolveWorkDir(ctx.cwd);
+			const autoresearchMdPath = path.join(workDir, "autoresearch.md");
+			const hasAutoresearchMd = fs.existsSync(autoresearchMdPath);
+			const controlState = reconstructControlState(ctx.sessionManager.getBranch());
+			const shouldResumeExistingNotes =
+				hasAutoresearchMd &&
+				(fs.existsSync(path.join(workDir, "autoresearch.jsonl")) ||
+					fs.existsSync(path.join(workDir, ".autoresearch")) ||
+					(controlState.lastMode !== "clear" && trimmed.length === 0));
+
+			const goal = shouldResumeExistingNotes
+				? (runtime.goal ?? runtime.state.name ?? null)
+				: trimmed.length > 0
+					? trimmed
+					: null;
+			const branchResult = await ensureAutoresearchBranch(api, workDir, goal);
+			if (!branchResult.ok) {
+				ctx.ui.notify(branchResult.error, "error");
+				return;
+			}
+
+			setMode(ctx, true, goal, "on");
+			dashboard.updateWidget(ctx, runtime);
+			await api.setActiveTools(addExperimentTools(api.getActiveTools()));
+
+			if (shouldResumeExistingNotes) {
+				api.sendUserMessage(
+					prompt.render(commandResumeTemplate, {
+						autoresearch_md_path: autoresearchMdPath,
+						branch_status_line: branchResult.created
+							? `Created and checked out dedicated git branch \`${branchResult.branchName}\` before resuming.`
+							: `Using dedicated git branch \`${branchResult.branchName}\`.`,
+						has_resume_context: trimmed.length > 0,
+						resume_context: trimmed,
+					}),
+				);
+			} else if (trimmed.length > 0) {
+				api.sendUserMessage(trimmed);
+			} else {
+				ctx.ui.notify("Autoresearch enabled\u2014describe what to optimize in your next message.", "info");
+			}
+		},
+	});
+	api.registerShortcut("ctrl+x", {
+		description: "Toggle autoresearch dashboard",
+		handler(ctx): void {
+			const runtime = getRuntime(ctx);
+			if (runtime.state.results.length === 0 && !runtime.runningExperiment) {
+				ctx.ui.notify("No autoresearch results yet", "info");
+				return;
+			}
+			runtime.dashboardExpanded = !runtime.dashboardExpanded;
+			dashboard.updateWidget(ctx, runtime);
+		},
+	});
+	api.registerShortcut("ctrl+shift+x", {
+		description: "Show autoresearch dashboard overlay",
+		handler(ctx): Promise<void> {
+			return dashboard.showOverlay(ctx, getRuntime(ctx));
+		},
+	});
+	api.on("session_start", (_event, ctx) => rehydrate(ctx));
+	api.on("session_switch", (_event, ctx) => rehydrate(ctx));
+	api.on("session_branch", (_event, ctx) => rehydrate(ctx));
+	api.on("session_tree", (_event, ctx) => rehydrate(ctx));
+	api.on("session_shutdown", (_event, ctx) => {
+		dashboard.clear(ctx);
+		runtimeStore.clear(getSessionKey(ctx));
+	});
+	api.on("agent_end", async (_event, ctx) => {
+		const runtime = getRuntime(ctx);
+		runtime.runningExperiment = null;
+		dashboard.updateWidget(ctx, runtime);
+		dashboard.requestRender();
+		if (!runtime.autoresearchMode) return;
+		if (ctx.hasPendingMessages()) {
+			runtime.autoResumeArmed = false;
+			return;
+		}
+		const workDir = resolveWorkDir(ctx.cwd);
+		const pendingRun = await refreshPendingRun(runtime, workDir);
+		const shouldResumePendingRun =
+			pendingRun !== null && runtime.lastAutoResumePendingRunNumber !== pendingRun.runNumber;
+		if (!shouldResumePendingRun && !runtime.autoResumeArmed) return;
+		runtime.autoResumeArmed = false;
+		runtime.lastAutoResumePendingRunNumber = pendingRun?.runNumber ?? null;
+		const autoresearchMdPath = path.join(workDir, "autoresearch.md");
+		const ideasPath = path.join(workDir, "autoresearch.ideas.md");
+		api.sendMessage(
+			{
+				customType: "autoresearch-resume",
+				content: prompt.render(resumeMessageTemplate, {
+					autoresearch_md_path: autoresearchMdPath,
+					has_ideas: fs.existsSync(ideasPath),
+					has_pending_run: Boolean(pendingRun),
+				}),
+				display: false,
+				attribution: "agent",
+			},
+			{ deliverAs: "nextTurn", triggerTurn: true },
+		);
+	});
+	api.on("before_agent_start", async (event, ctx) => {
+		const runtime = getRuntime(ctx);
+		if (!runtime.autoresearchMode) return;
+		const workDir = resolveWorkDir(ctx.cwd);
+		const autoresearchMdPath = path.join(workDir, "autoresearch.md");
+		const checksPath = path.join(workDir, "autoresearch.checks.sh");
+		const ideasPath = path.join(workDir, "autoresearch.ideas.md");
+		const pendingRun = await refreshPendingRun(runtime, workDir);
+		const currentSegmentResults = currentResults(runtime.state.results, runtime.state.currentSegment);
+		const baselineMetric = findBaselineMetric(runtime.state.results, runtime.state.currentSegment);
+		const bestKept = findBestKeptResult(runtime.state);
+		const goal = runtime.goal ?? runtime.state.name ?? "";
+		const recentResults = currentSegmentResults.slice(-3).map(result => {
+			const asiSummary = summarizeExperimentAsi(result);
+			return {
+				asi_summary: asiSummary,
+				description: result.description,
+				has_asi_summary: Boolean(asiSummary),
+				metric_display: formatNum(result.metric, runtime.state.metricUnit),
+				run_number: result.runNumber ?? runtime.state.results.indexOf(result) + 1,
+				status: result.status,
+			};
+		});
+		const hasPendingMetric = pendingRun?.parsedPrimary != null;
+		return {
+			systemPrompt: prompt.render(promptTemplate, {
+				base_system_prompt: event.systemPrompt,
+				has_goal: goal.trim().length > 0,
+				goal,
+				has_autoresearch_md: fs.existsSync(autoresearchMdPath),
+				working_dir: workDir,
+				default_metric_name: runtime.state.metricName,
+				metric_name: runtime.state.metricName,
+				autoresearch_md_path: autoresearchMdPath,
+				has_checks: fs.existsSync(checksPath),
+				checks_path: checksPath,
+				has_ideas: fs.existsSync(ideasPath),
+				ideas_path: ideasPath,
+				current_segment: runtime.state.currentSegment + 1,
+				current_segment_run_count: currentSegmentResults.length,
+				has_baseline_metric: baselineMetric !== null,
+				baseline_metric_display: formatNum(baselineMetric, runtime.state.metricUnit),
+				has_best_result: Boolean(bestKept),
+				best_metric_display: bestKept
+					? formatNum(bestKept.result.metric, runtime.state.metricUnit)
+					: formatNum(baselineMetric, runtime.state.metricUnit),
+				best_run_number: bestKept ? (bestKept.result.runNumber ?? bestKept.index + 1) : null,
+				has_recent_results: recentResults.length > 0,
+				recent_results: recentResults,
+				has_pending_run: Boolean(pendingRun),
+				pending_run_number: pendingRun?.runNumber,
+				pending_run_command: pendingRun?.command,
+				pending_run_directory: pendingRun?.runDirectory,
+				pending_run_passed: pendingRun?.passed ?? false,
+				has_pending_run_metric: hasPendingMetric,
+				pending_run_metric_display: hasPendingMetric
+					? formatNum(pendingRun!.parsedPrimary!, runtime.state.metricUnit)
+					: null,
+			}),
+		};
+	});
+};
+function summarizeExperimentAsi(result: ExperimentResult): string | null {
+	const parts = ["hypothesis", "rollback_reason", "next_action_hint"]
+		.map(k => (typeof result.asi?.[k] === "string" ? (result.asi[k] as string).trim() : ""))
+		.filter(Boolean);
+	return parts.length > 0 ? parts.join(" | ").slice(0, 220) : null;
+}
+function resolveAutoresearchRelativePath(
+	workDir: string,
+	rawPath: string,
+): { ok: false; reason: string } | { ok: true; relativePath: string } {
+	if (/^[a-z][a-z0-9+.-]*:\/\//i.test(rawPath))
+		return { ok: false, reason: `Autoresearch cannot validate internal URL paths during scoped editing: ${rawPath}` };
+	const resolvedPath = path.isAbsolute(rawPath) ? path.resolve(rawPath) : path.resolve(workDir, rawPath);
+	const canonicalWorkDir = canonicalizeExistingPath(workDir);
+	const canonicalTargetPath = canonicalizeTargetPath(resolvedPath);
+	const relativePath = path.relative(canonicalWorkDir, canonicalTargetPath);
+	if (relativePath === ".." || relativePath.startsWith(`..${path.sep}`) || path.isAbsolute(relativePath))
+		return { ok: false, reason: `Autoresearch blocked edits outside the working tree: ${rawPath}` };
+	return { ok: true, relativePath: relativePath.length === 0 ? "." : normalizeAutoresearchPath(relativePath) };
+}
+function validateEditableAutoresearchPath(relativePath: string, runtime: AutoresearchRuntime): string | null {
+	if (isAutoresearchLocalStatePath(relativePath))
+		return "autoresearch local state files are managed by the experiment tools and cannot be edited directly";
+	if (runtime.state.offLimits.some(spec => pathMatchesContractPath(relativePath, spec)))
+		return "this path is listed under Off Limits in autoresearch.md";
+	if (isAutoresearchCommittableFile(relativePath)) return null;
+	if (runtime.state.scopePaths.length === 0)
+		return "Files in Scope is not initialized yet; only autoresearch control files may be edited before init_experiment runs";
+	if (!runtime.state.scopePaths.some(spec => pathMatchesContractPath(relativePath, spec)))
+		return "this path is outside Files in Scope in autoresearch.md";
+	return null;
+}
+function collectLoggedRunNumbers(results: ExperimentResult[]): Set<number> {
+	return new Set(results.map(r => r.runNumber).filter((n): n is number => n !== null));
+}
+function applyPendingRunToRuntime(runtime: AutoresearchRuntime, summary: PendingRunSummary | null): void {
+	runtime.lastRunSummary = summary;
+	runtime.lastRunChecks = summaryToChecks(summary);
+	runtime.lastRunDuration = summary?.durationSeconds ?? null;
+	runtime.lastRunAsi = summary?.parsedAsi ?? null;
+	runtime.lastRunArtifactDir = summary?.runDirectory ?? null;
+	runtime.lastRunNumber = summary?.runNumber ?? null;
+}
+function summaryToChecks(summary: PendingRunSummary | null): AutoresearchRuntime["lastRunChecks"] {
+	if (!summary || summary.checksPass === null) return null;
+	return {
+		pass: summary.checksPass,
+		output: "",
+		duration: summary.checksDurationSeconds ?? 0,
+	};
+}
+function canonicalizeExistingPath(targetPath: string): string {
+	try {
+		return fs.realpathSync.native(targetPath);
+	} catch {
+		return path.resolve(targetPath);
+	}
+}
+function canonicalizeTargetPath(targetPath: string): string {
+	const pendingSegments: string[] = [];
+	let currentPath = path.resolve(targetPath);
+	while (!fs.existsSync(currentPath)) {
+		const parentPath = path.dirname(currentPath);
+		if (parentPath === currentPath) return currentPath;
+		pendingSegments.unshift(path.basename(currentPath));
+		currentPath = parentPath;
+	}
+	return path.resolve(canonicalizeExistingPath(currentPath), ...pendingSegments);
+}
+async function refreshPendingRun(runtime: AutoresearchRuntime, workDir: string): Promise<PendingRunSummary | null> {
+	const pendingRun =
+		runtime.lastRunSummary ?? (await readPendingRunSummary(workDir, collectLoggedRunNumbers(runtime.state.results)));
+	runtime.lastRunSummary = pendingRun;
+	runtime.lastRunChecks = summaryToChecks(pendingRun);
+	runtime.lastRunDuration = pendingRun?.durationSeconds ?? runtime.lastRunDuration;
+	runtime.lastRunAsi = pendingRun?.parsedAsi ?? runtime.lastRunAsi;
+	return pendingRun;
+}

package/src/autoresearch/prompt.md

@@ -0,0 +1,236 @@
+{{{base_system_prompt}}}
+
+## Autoresearch Mode
+
+Autoresearch mode is active.
+
+{{#if has_goal}}
+Primary goal:
+{{goal}}
+{{else}}
+{{#if has_autoresearch_md}}
+Primary goal is documented in `autoresearch.md` for this session.
+{{else}}
+There is no `autoresearch.md` yet. Infer what to optimize from the latest user message and the conversation; after you create `autoresearch.md`, keep it as the durable source of truth for goal and benchmark contract.
+{{/if}}
+{{/if}}
+
+Working directory:
+`{{working_dir}}`
+
+You are running an autonomous experiment loop. Keep iterating until the user interrupts you or the configured maximum iteration count is reached.
+{{#if has_recent_results}}
+
+### Current Segment Snapshot
+
+- segment: `{{current_segment}}`
+- runs in current segment: `{{current_segment_run_count}}`
+{{#if has_baseline_metric}}
+- baseline `{{metric_name}}`: `{{baseline_metric_display}}`
+{{/if}}
+{{#if has_best_result}}
+- best kept `{{metric_name}}`: `{{best_metric_display}}`{{#if best_run_number}} from run `#{{best_run_number}}`{{/if}}
+{{/if}}
+
+Recent runs:
+{{#each recent_results}}
+- run `#{{run_number}}`: `{{status}}` `{{metric_display}}` — {{description}}
+{{#if has_asi_summary}}
+  ASI: {{asi_summary}}
+{{/if}}
+{{/each}}
+{{/if}}
+{{#if has_pending_run}}
+
+### Pending Run
+
+An unlogged run artifact exists at `{{pending_run_directory}}`.
+
+- run: `#{{pending_run_number}}`
+- command: `{{pending_run_command}}`
+{{#if has_pending_run_metric}}
+- parsed `{{metric_name}}`: `{{pending_run_metric_display}}`
+{{/if}}
+- result status: {{#if pending_run_passed}}passed{{else}}failed{{/if}}
+- finish the `log_experiment` step before starting another benchmark
+{{/if}}
+
+### Available tools
+
+- `init_experiment` — initialize or reset the experiment session for the current optimization target.
+- `run_experiment` — run a benchmark or experiment command with timing, output capture, structured metric parsing, and optional backpressure checks.
+- `log_experiment` — record the result, update the dashboard, persist JSONL history, auto-commit kept experiments, and revert only run-modified files for discarded or failed experiments (pre-existing uncommitted changes are preserved).
+
+### Operating protocol
+
+1. Understand the target before touching code.
+   - Read the relevant source files.
+   - Identify the true bottleneck or quality constraint.
+   - Check existing scripts, benchmark harnesses, and config files.
+   - Verify prerequisites, one-time setup, and benchmark inputs before the first run of a segment.
+2. Keep your notes in `autoresearch.md`.
+   - Record the goal, the benchmark command, the primary metric, important secondary metrics, the files in scope, hard constraints, preflight requirements, and the benchmark comparability invariant.
+   - Update the notes whenever the strategy changes.
+   - Keep durable conclusions in `autoresearch.md`.
+   - Use `autoresearch.ideas.md` for deferred experiment ideas that are promising but not active yet.
+3. The benchmark command in `autoresearch.md` is the canonical entrypoint.
+   - If it does not exist yet, create a benchmark script.
+   - Make it print structured metric lines in the form `METRIC name=value`.
+   - Quality scores (`direction: higher`) are first-class — not every benchmark is a timing measurement.
+   - Use the same workload every run unless you intentionally re-initialize with a new segment.
+4. Initialize the loop with `init_experiment` before the first logged run of a segment.
+   - Pass `from_autoresearch_md: true` with only `name` to load the benchmark contract from `autoresearch.md` without mirroring every field in the tool call.
+   - Use `abandon_unlogged_runs: true` only when you intentionally discard unlogged run artifacts and need a fresh segment (for example after a bad or obsolete benchmark directory).
+5. Run a baseline first.
+   - Establish the baseline metric before attempting optimizations.
+   - Track secondary metrics only when they matter to correctness, quality, or obvious regressions.
+6. Iterate.
+   - Make one coherent experiment at a time.
+   - Run `run_experiment`.
+   - Interpret the result honestly.
+   - Call `log_experiment` after every run (it refreshes benchmark/scope fields from `autoresearch.md` before logging so keep validation matches the file on disk).
+   - Use `run_experiment` with `force: true` only when you must override the segment benchmark command.
+   - After any code change, verify with `bun check:ts` or the project test suite before logging. A kept experiment that breaks the build is worse than a discarded one.
+   - On `log_experiment`, `force: true` relaxes ASI requirements and allows keeping a primary-metric regression; prefer normal logging when possible.
+7. Keep the primary metric as the decision maker.
+   - `keep` when the primary metric improves.
+   - `discard` when it regresses or stays flat.
+   - `crash` when the run fails.
+   - `checks_failed` when the benchmark passes but backpressure checks fail.
+8. Record ASI on every `log_experiment` call.
+   - At minimum include `hypothesis`.
+   - On `discard`, `crash`, or `checks_failed`, also include `rollback_reason` and `next_action_hint`.
+   - Use ASI to capture what you learned, not just what you changed.
+9. Prefer simpler wins.
+   - Remove dead ends.
+   - Keep equal or near-equal results when they materially simplify the implementation.
+   - Do not keep ugly complexity for tiny gains unless the payoff is clearly worth it.
+   - Do not thrash between unrelated ideas without writing down the conclusion.
+10. When confidence is low, confirm.
+    - The dashboard confidence score compares the best observed improvement against the observed noise floor.
+    - Below `1.0x` usually means the improvement is within noise.
+    - Re-run promising changes when needed before keeping them.
+
+### Benchmark harness guidance
+
+Your benchmark script SHOULD:
+
+- match the benchmark command from `autoresearch.md`
+- run from `{{working_dir}}`
+- fail with a non-zero exit status on invalid runs
+- print the primary metric as `METRIC {{default_metric_name}}=<number>` or another explicit metric name chosen during initialization
+- print secondary metrics as additional `METRIC name=value` lines
+- avoid extra randomness when possible
+- use repeated samples and median-style summaries for fast benchmarks
+- preserve the comparability invariant for the current segment
+- keep the ground-truth evaluator and fixed benchmark inputs unchanged unless the segment is explicitly re-initialized
+
+### Notes file template
+
+Keep `autoresearch.md` concise and current.
+
+Suggested structure:
+
+```md
+# Autoresearch
+
+## Goal
+{{#if has_goal}}
+- {{goal}}
+{{else}}
+{{#if has_autoresearch_md}}
+- document the active target here before the first benchmark
+{{else}}
+- (derive from the user's messages, then record here)
+{{/if}}
+{{/if}}
+
+## Benchmark
+ - command:
+ - primary metric:
+ - metric unit:
+ - direction:
+ - secondary metrics: memory_mb, rss_mb
+
+## Files in Scope
+- path:
+
+## Off Limits
+- path:
+
+## Constraints
+- rule:
+
+## Baseline
+- metric:
+- notes:
+
+## Current best
+- metric:
+- why it won:
+
+## What's Been Tried
+- experiment:
+- lesson:
+```
+
+### Guardrails
+
+- Do not game the benchmark.
+- Do not overfit to synthetic inputs if the real workload is broader.
+- Preserve correctness.
+- Only modify files that are explicitly in scope for the current session.
+- Do not use the general shell tool for file mutations during autoresearch. Use `write`, `edit`, or `ast_edit` for scoped code changes and `run_experiment` for benchmark execution.
+- If you create `autoresearch.checks.sh`, treat it as a hard gate for `keep`.
+- If the user sends another message while a run is in progress, finish the current run and logging cycle first, then address the new input in the next iteration.
+
+{{#if has_autoresearch_md}}
+### Resume mode
+
+`autoresearch.md` already exists at `{{autoresearch_md_path}}`.
+
+Resume from the existing notes:
+
+- read `autoresearch.md`
+- inspect recent git history
+- inspect `autoresearch.jsonl`
+- continue from the most promising unfinished direction on the current protected branch
+
+{{else}}
+### Initial setup
+
+`autoresearch.md` does not exist yet. You decide the benchmark contract, harness, and scope from the user's messages and the repository—do not ask the user to re-type benchmark commands or metric names in a separate UI prompt.
+
+Before the first benchmark:
+
+- Write `autoresearch.md` with goal, benchmark command, primary metric name and unit, direction (`lower` or `higher`), tradeoff metrics if relevant, files in scope, off limits, and constraints.
+- Add a short preflight section: prerequisites, one-time setup, and the comparability invariant that must stay fixed across runs.
+- Mark ground-truth evaluators, fixed datasets, and other measurement-critical files as off limits or hard constraints when they define the benchmark contract.
+- Create a benchmark script; print the primary metric as `METRIC <name>=<number>` and optional secondary metrics as additional `METRIC` lines.
+- Optionally add `autoresearch.checks.sh` if correctness or quality needs a hard gate.
+- Call `init_experiment` with arguments that match `autoresearch.md` exactly (benchmark command, metric, unit, direction, scope paths, off limits, constraints).
+- Run and log the baseline.
+
+Until `init_experiment` succeeds, only autoresearch control files (`autoresearch.md`, `autoresearch.sh`, `autoresearch.ideas.md`, `autoresearch.checks.sh`) may be edited; after initialization, respect Files in Scope from the contract.
+
+{{/if}}
+{{#if has_checks}}
+### Backpressure checks
+
+`autoresearch.checks.sh` exists at `{{checks_path}}` and runs automatically after passing benchmark runs.
+
+Treat failing checks as a failed experiment:
+
+- do not `keep` a run when checks fail
+- log it as `checks_failed`
+- diagnose the regression before continuing
+
+{{/if}}
+{{#if has_ideas}}
+### Ideas backlog
+
+`autoresearch.ideas.md` exists at `{{ideas_path}}`.
+
+Use it to keep promising but deferred experiments. `autoresearch.md` should hold durable conclusions; `autoresearch.ideas.md` is the scratch backlog. Prune stale ideas when they are disproven or superseded.
+
+{{/if}}

package/src/autoresearch/resume-message.md

@@ -0,0 +1,16 @@
+Continue the autoresearch loop now.
+
+@{{autoresearch_md_path}}
+
+- Read `autoresearch.md` and `autoresearch.jsonl`.
+- Treat `autoresearch.md` as the source of truth for the current direction, scope, and constraints.
+- Inspect recent git history for context.
+{{#if has_pending_run}}
+- Inspect the latest unlogged `run.json` under `.autoresearch/runs/` and finish the pending `log_experiment` step before starting a new benchmark.
+{{/if}}
+- Continue from the most promising unfinished direction.
+{{#if has_ideas}}
+- Review `autoresearch.ideas.md` for deferred next steps and prune stale items.
+{{/if}}
+- Keep iterating until interrupted or until the configured iteration cap is reached.
+- Preserve correctness and do not game the benchmark.