@oh-my-pi/pi-coding-agent 13.14.0 → 13.15.2

package/src/autoresearch/prompt.md

@@ -0,0 +1,227 @@
+{{{base_system_prompt}}}
+
+## Autoresearch Mode
+
+Autoresearch mode is active.
+
+{{#if has_goal}}
+Primary goal:
+{{goal}}
+{{else}}
+Primary goal is documented in `autoresearch.md` for this session.
+{{/if}}
+
+Working directory:
+`{{working_dir}}`
+
+You are running an autonomous experiment loop. Keep iterating until the user interrupts you or the configured maximum iteration count is reached.
+{{#if has_program}}
+
+### Local Playbook
+
+`autoresearch.program.md` exists at `{{program_path}}`.
+
+Use it as a repo-local strategy overlay for this session. `autoresearch.md` remains the source of truth for benchmark, scope, and constraints.
+{{/if}}
+{{#if has_recent_results}}
+
+### Current Segment Snapshot
+
+- segment: `{{current_segment}}`
+- runs in current segment: `{{current_segment_run_count}}`
+{{#if has_baseline_metric}}
+- baseline `{{metric_name}}`: `{{baseline_metric_display}}`
+{{/if}}
+{{#if has_best_result}}
+- best kept `{{metric_name}}`: `{{best_metric_display}}`{{#if best_run_number}} from run `#{{best_run_number}}`{{/if}}
+{{/if}}
+
+Recent runs:
+{{#each recent_results}}
+- run `#{{run_number}}`: `{{status}}` `{{metric_display}}` — {{description}}
+{{#if has_asi_summary}}
+  ASI: {{asi_summary}}
+{{/if}}
+{{/each}}
+{{/if}}
+{{#if has_pending_run}}
+
+### Pending Run
+
+An unlogged run artifact exists at `{{pending_run_directory}}`.
+
+- run: `#{{pending_run_number}}`
+- command: `{{pending_run_command}}`
+{{#if has_pending_run_metric}}
+- parsed `{{metric_name}}`: `{{pending_run_metric_display}}`
+{{/if}}
+- result status: {{#if pending_run_passed}}passed{{else}}failed{{/if}}
+- finish the `log_experiment` step before starting another benchmark
+{{/if}}
+
+### Available tools
+
+- `init_experiment` — initialize or reset the experiment session for the current optimization target.
+- `run_experiment` — run a benchmark or experiment command with timing, output capture, structured metric parsing, and optional backpressure checks.
+- `log_experiment` — record the result, update the dashboard, persist JSONL history, auto-commit kept experiments, and auto-revert discarded or failed experiments.
+
+### Operating protocol
+
+1. Understand the target before touching code.
+   - Read the relevant source files.
+   - Identify the true bottleneck or quality constraint.
+   - Check existing scripts, benchmark harnesses, and config files.
+   - Verify prerequisites, one-time setup, and benchmark inputs before the first run of a segment.
+2. Keep your notes in `autoresearch.md`.
+   - Record the goal, the benchmark command, the primary metric, important secondary metrics, the files in scope, hard constraints, preflight requirements, and the benchmark comparability invariant.
+   - Update the notes whenever the strategy changes.
+   - Keep durable conclusions in `autoresearch.md`.
+   - Use `autoresearch.ideas.md` for deferred experiment ideas that are promising but not active yet.
+3. Use `autoresearch.sh` as the canonical benchmark entrypoint.
+   - If it does not exist yet, create it.
+   - Make it print structured metric lines in the form `METRIC name=value`.
+   - Use the same workload every run unless you intentionally re-initialize with a new segment.
+   - Keep the measurement harness, evaluator, and fixed benchmark inputs stable unless you intentionally start a new segment and document the change.
+4. Initialize the loop with `init_experiment` before the first logged run of a segment.
+5. Run a baseline first.
+   - Establish the baseline metric before attempting optimizations.
+   - Track secondary metrics only when they matter to correctness, quality, or obvious regressions.
+6. Iterate.
+   - Make one coherent experiment at a time.
+   - Run `run_experiment`.
+   - Interpret the result honestly.
+   - Call `log_experiment` after every run.
+7. Keep the primary metric as the decision maker.
+   - `keep` when the primary metric improves.
+   - `discard` when it regresses or stays flat.
+   - `crash` when the run fails.
+   - `checks_failed` when the benchmark passes but backpressure checks fail.
+8. Record ASI on every `log_experiment` call.
+   - At minimum include `hypothesis`.
+   - On `discard`, `crash`, or `checks_failed`, also include `rollback_reason` and `next_action_hint`.
+   - Use ASI to capture what you learned, not just what you changed.
+9. Prefer simpler wins.
+   - Remove dead ends.
+   - Keep equal or near-equal results when they materially simplify the implementation.
+   - Do not keep ugly complexity for tiny gains unless the payoff is clearly worth it.
+   - Do not thrash between unrelated ideas without writing down the conclusion.
+10. When confidence is low, confirm.
+    - The dashboard confidence score compares the best observed improvement against the observed noise floor.
+    - Below `1.0x` usually means the improvement is within noise.
+    - Re-run promising changes when needed before keeping them.
+
+### Benchmark harness guidance
+
+Your benchmark script SHOULD:
+
+- live at `autoresearch.sh`
+- run from `{{working_dir}}`
+- fail with a non-zero exit status on invalid runs
+- print the primary metric as `METRIC {{default_metric_name}}=<number>` or another explicit metric name chosen during initialization
+- print secondary metrics as additional `METRIC name=value` lines
+- avoid extra randomness when possible
+- use repeated samples and median-style summaries for fast benchmarks
+- preserve the comparability invariant for the current segment
+- keep the ground-truth evaluator and fixed benchmark inputs unchanged unless the segment is explicitly re-initialized
+
+### Notes file template
+
+Keep `autoresearch.md` concise and current.
+
+Suggested structure:
+
+```md
+# Autoresearch
+
+## Goal
+{{#if has_goal}}
+- {{goal}}
+{{else}}
+- document the active target here before the first benchmark
+{{/if}}
+
+## Benchmark
+ - command:
+ - primary metric:
+ - metric unit:
+ - direction:
+ - secondary metrics: memory_mb, rss_mb
+
+## Files in Scope
+- path:
+
+## Off Limits
+- path:
+
+## Constraints
+- rule:
+
+## Baseline
+- metric:
+- notes:
+
+## Current best
+- metric:
+- why it won:
+
+## What's Been Tried
+- experiment:
+- lesson:
+```
+
+### Guardrails
+
+- Do not game the benchmark.
+- Do not overfit to synthetic inputs if the real workload is broader.
+- Preserve correctness.
+- Only modify files that are explicitly in scope for the current session.
+- Do not use the general shell tool for file mutations during autoresearch. Use `write`, `edit`, or `ast_edit` for scoped code changes and `run_experiment` for benchmark execution.
+- If you create `autoresearch.checks.sh`, treat it as a hard gate for `keep`.
+- If the user sends another message while a run is in progress, finish the current run and logging cycle first, then address the new input in the next iteration.
+
+{{#if has_autoresearch_md}}
+### Resume mode
+
+`autoresearch.md` already exists at `{{autoresearch_md_path}}`.
+
+Resume from the existing notes:
+
+- read `autoresearch.md`
+- inspect recent git history
+- inspect `autoresearch.jsonl`
+- continue from the most promising unfinished direction on the current protected branch
+
+{{else}}
+### Initial setup
+
+`autoresearch.md` does not exist yet.
+
+Create the experiment workspace before the first benchmark:
+
+- write `autoresearch.md`
+- write `autoresearch.sh`
+- optionally write `autoresearch.checks.sh`
+- run `init_experiment`
+- run and log the baseline
+
+{{/if}}
+{{#if has_checks}}
+### Backpressure checks
+
+`autoresearch.checks.sh` exists at `{{checks_path}}` and runs automatically after passing benchmark runs.
+
+Treat failing checks as a failed experiment:
+
+- do not `keep` a run when checks fail
+- log it as `checks_failed`
+- diagnose the regression before continuing
+
+{{/if}}
+{{#if has_ideas}}
+### Ideas backlog
+
+`autoresearch.ideas.md` exists at `{{ideas_path}}`.
+
+Use it to keep promising but deferred experiments. `autoresearch.md` should hold durable conclusions; `autoresearch.ideas.md` is the scratch backlog. Prune stale ideas when they are disproven or superseded.
+
+{{/if}}

package/src/autoresearch/resume-message.md

@@ -0,0 +1,16 @@
+Continue the autoresearch loop now.
+
+@{{autoresearch_md_path}}
+
+- Read `autoresearch.md` and `autoresearch.jsonl`.
+- Treat `autoresearch.md` as the source of truth for the current direction, scope, and constraints.
+- Inspect recent git history for context.
+{{#if has_pending_run}}
+- Inspect the latest unlogged `run.json` under `.autoresearch/runs/` and finish the pending `log_experiment` step before starting a new benchmark.
+{{/if}}
+- Continue from the most promising unfinished direction.
+{{#if has_ideas}}
+- Review `autoresearch.ideas.md` for deferred next steps and prune stale items.
+{{/if}}
+- Keep iterating until interrupted or until the configured iteration cap is reached.
+- Preserve correctness and do not game the benchmark.

package/src/autoresearch/state.ts

@@ -0,0 +1,386 @@
+import * as fs from "node:fs";
+import * as path from "node:path";
+import type { SessionEntry } from "../session/session-manager";
+import { normalizeAutoresearchList, normalizeContractPathSpec } from "./contract";
+import { inferMetricUnitFromName, isBetter } from "./helpers";
+import type {
+	AutoresearchControlEntryData,
+	AutoresearchJsonConfigEntry,
+	AutoresearchJsonRunEntry,
+	AutoresearchRuntime,
+	ExperimentResult,
+	ExperimentState,
+	MetricDef,
+	MetricDirection,
+	NumericMetricMap,
+	ReconstructedControlState,
+	ReconstructedExperimentData,
+	RuntimeStore,
+} from "./types";
+
+export function createExperimentState(): ExperimentState {
+	return {
+		results: [],
+		bestMetric: null,
+		bestDirection: "lower",
+		metricName: "metric",
+		metricUnit: "",
+		secondaryMetrics: [],
+		name: null,
+		currentSegment: 0,
+		maxExperiments: null,
+		confidence: null,
+		benchmarkCommand: null,
+		scopePaths: [],
+		offLimits: [],
+		constraints: [],
+		segmentFingerprint: null,
+	};
+}
+
+export function createSessionRuntime(): AutoresearchRuntime {
+	return {
+		autoresearchMode: false,
+		autoResumeArmed: false,
+		dashboardExpanded: false,
+		lastAutoResumePendingRunNumber: null,
+		lastRunChecks: null,
+		lastRunDuration: null,
+		lastRunAsi: null,
+		lastRunArtifactDir: null,
+		lastRunNumber: null,
+		lastRunSummary: null,
+		runningExperiment: null,
+		state: createExperimentState(),
+		goal: null,
+	};
+}
+
+export function cloneExperimentState(state: ExperimentState): ExperimentState {
+	return {
+		...state,
+		results: state.results.map(result => ({
+			...result,
+			metrics: { ...result.metrics },
+			asi: result.asi ? structuredClone(result.asi) : undefined,
+		})),
+		secondaryMetrics: state.secondaryMetrics.map(metric => ({ ...metric })),
+		scopePaths: [...state.scopePaths],
+		offLimits: [...state.offLimits],
+		constraints: [...state.constraints],
+	};
+}
+
+export function currentResults(results: ExperimentResult[], segment: number): ExperimentResult[] {
+	return results.filter(result => result.segment === segment);
+}
+
+export function findBaselineResult(results: ExperimentResult[], segment: number): ExperimentResult | null {
+	return currentResults(results, segment).find(result => result.status === "keep") ?? null;
+}
+
+export function findBaselineMetric(results: ExperimentResult[], segment: number): number | null {
+	const baseline = findBaselineResult(results, segment);
+	return baseline ? baseline.metric : null;
+}
+
+export function findBestKeptMetric(
+	results: ExperimentResult[],
+	segment: number,
+	direction: MetricDirection,
+): number | null {
+	let best: number | null = null;
+	for (const result of currentResults(results, segment)) {
+		if (result.status !== "keep") continue;
+		if (best === null || isBetter(result.metric, best, direction)) {
+			best = result.metric;
+		}
+	}
+	return best;
+}
+
+export function findBaselineRunNumber(results: ExperimentResult[], segment: number): number | null {
+	const baseline = findBaselineResult(results, segment);
+	if (!baseline) return null;
+	if (baseline.runNumber !== null) return baseline.runNumber;
+	const index = results.indexOf(baseline);
+	return index >= 0 ? index + 1 : null;
+}
+
+export function findBaselineSecondary(
+	results: ExperimentResult[],
+	segment: number,
+	knownMetrics: MetricDef[],
+): NumericMetricMap {
+	const baseline = findBaselineResult(results, segment);
+	const values: NumericMetricMap = baseline ? { ...baseline.metrics } : {};
+	for (const metric of knownMetrics) {
+		if (values[metric.name] !== undefined) continue;
+		for (const result of currentResults(results, segment)) {
+			const value = result.metrics[metric.name];
+			if (value !== undefined) {
+				values[metric.name] = value;
+				break;
+			}
+		}
+	}
+	return values;
+}
+
+export function sortedMedian(values: number[]): number {
+	if (values.length === 0) return 0;
+	const sorted = [...values].sort((left, right) => left - right);
+	const midpoint = Math.floor(sorted.length / 2);
+	if (sorted.length % 2 === 0) {
+		return (sorted[midpoint - 1] + sorted[midpoint]) / 2;
+	}
+	return sorted[midpoint];
+}
+
+export function computeConfidence(
+	results: ExperimentResult[],
+	segment: number,
+	direction: MetricDirection,
+): number | null {
+	const current = currentResults(results, segment).filter(result => result.metric > 0);
+	if (current.length < 3) return null;
+
+	const values = current.map(result => result.metric);
+	const median = sortedMedian(values);
+	const mad = sortedMedian(values.map(value => Math.abs(value - median)));
+	if (mad === 0) return null;
+
+	const baseline = findBaselineMetric(results, segment);
+	if (baseline === null) return null;
+
+	let bestKept: number | null = null;
+	for (const result of current) {
+		if (result.status !== "keep" || result.metric <= 0) continue;
+		if (bestKept === null || isBetter(result.metric, bestKept, direction)) {
+			bestKept = result.metric;
+		}
+	}
+	if (bestKept === null || bestKept === baseline) return null;
+
+	return Math.abs(bestKept - baseline) / mad;
+}
+
+export function reconstructStateFromJsonl(workDir: string): ReconstructedExperimentData {
+	const state = createExperimentState();
+	const jsonlPath = path.join(workDir, "autoresearch.jsonl");
+	if (!fs.existsSync(jsonlPath)) {
+		return { hasLog: false, state };
+	}
+
+	const content = fs.readFileSync(jsonlPath, "utf8");
+	const lines = content
+		.split("\n")
+		.map(line => line.trim())
+		.filter(line => line.length > 0);
+
+	let segment = 0;
+	let sawConfig = false;
+	for (const line of lines) {
+		let parsed: unknown;
+		try {
+			parsed = JSON.parse(line) as unknown;
+		} catch {
+			continue;
+		}
+
+		const configEntry = parseConfigEntry(parsed);
+		if (configEntry) {
+			if (sawConfig || state.results.length > 0) {
+				segment += 1;
+			}
+			sawConfig = true;
+			state.currentSegment = segment;
+			if (configEntry.name) state.name = configEntry.name;
+			if (configEntry.metricName) state.metricName = configEntry.metricName;
+			if (configEntry.metricUnit !== undefined) state.metricUnit = configEntry.metricUnit;
+			if (configEntry.bestDirection) state.bestDirection = configEntry.bestDirection;
+			if (configEntry.benchmarkCommand !== undefined) state.benchmarkCommand = configEntry.benchmarkCommand;
+			state.scopePaths = cloneStringArray(configEntry.scopePaths);
+			state.offLimits = cloneStringArray(configEntry.offLimits);
+			state.constraints = cloneStringArray(configEntry.constraints);
+			state.segmentFingerprint =
+				typeof configEntry.segmentFingerprint === "string" ? configEntry.segmentFingerprint : null;
+			state.secondaryMetrics = hydrateMetricDefs(configEntry.secondaryMetrics);
+			continue;
+		}
+
+		if (!isRunEntry(parsed)) continue;
+		const result: ExperimentResult = {
+			runNumber: typeof parsed.run === "number" && Number.isFinite(parsed.run) ? parsed.run : null,
+			commit: typeof parsed.commit === "string" ? parsed.commit : "",
+			metric: typeof parsed.metric === "number" && Number.isFinite(parsed.metric) ? parsed.metric : 0,
+			metrics: cloneNumericMetrics(parsed.metrics),
+			status: isExperimentStatus(parsed.status) ? parsed.status : "keep",
+			description: typeof parsed.description === "string" ? parsed.description : "",
+			timestamp: typeof parsed.timestamp === "number" && Number.isFinite(parsed.timestamp) ? parsed.timestamp : 0,
+			segment,
+			confidence:
+				typeof parsed.confidence === "number" && Number.isFinite(parsed.confidence) ? parsed.confidence : null,
+			asi: cloneAsi(parsed.asi),
+		};
+		state.results.push(result);
+		if (segment !== state.currentSegment) continue;
+		registerSecondaryMetrics(state.secondaryMetrics, result.metrics);
+	}
+
+	state.bestMetric = findBaselineMetric(state.results, state.currentSegment);
+	state.confidence = computeConfidence(state.results, state.currentSegment, state.bestDirection);
+	return { hasLog: true, state };
+}
+
+export function reconstructControlState(entries: SessionEntry[]): ReconstructedControlState {
+	let autoresearchMode = false;
+	let goal: string | null = null;
+	let lastMode: ReconstructedControlState["lastMode"] = null;
+	for (const entry of entries) {
+		if (entry.type !== "custom" || entry.customType !== "autoresearch-control") continue;
+		const data = parseControlEntry(entry.data);
+		if (!data) continue;
+		lastMode = data.mode;
+		autoresearchMode = data.mode === "on";
+		goal = data.goal ?? goal;
+		if (data.mode === "clear") {
+			goal = null;
+		}
+	}
+	return { autoresearchMode, goal, lastMode };
+}
+
+export function createRuntimeStore(): RuntimeStore {
+	const runtimes = new Map<string, AutoresearchRuntime>();
+	return {
+		clear(sessionKey: string): void {
+			runtimes.delete(sessionKey);
+		},
+		ensure(sessionKey: string): AutoresearchRuntime {
+			const existing = runtimes.get(sessionKey);
+			if (existing) return existing;
+			const runtime = createSessionRuntime();
+			runtimes.set(sessionKey, runtime);
+			return runtime;
+		},
+	};
+}
+
+function registerSecondaryMetrics(metrics: MetricDef[], values: NumericMetricMap): void {
+	for (const name of Object.keys(values)) {
+		if (metrics.some(metric => metric.name === name)) continue;
+		metrics.push({
+			name,
+			unit: inferMetricUnitFromName(name),
+		});
+	}
+}
+
+function isConfigEntry(value: unknown): value is AutoresearchJsonConfigEntry {
+	if (typeof value !== "object" || value === null) return false;
+	const candidate = value as { type?: unknown };
+	return candidate.type === "config";
+}
+
+function parseConfigEntry(value: unknown): AutoresearchJsonConfigEntry | null {
+	if (!isConfigEntry(value)) return null;
+	const candidate = value as AutoresearchJsonConfigEntry;
+	const config: AutoresearchJsonConfigEntry = { type: "config" };
+	if (typeof candidate.name === "string" && candidate.name.trim().length > 0) {
+		config.name = candidate.name;
+	}
+	if (typeof candidate.metricName === "string" && candidate.metricName.trim().length > 0) {
+		config.metricName = candidate.metricName;
+	}
+	if (typeof candidate.metricUnit === "string") {
+		config.metricUnit = candidate.metricUnit;
+	}
+	if (candidate.bestDirection === "lower" || candidate.bestDirection === "higher") {
+		config.bestDirection = candidate.bestDirection;
+	}
+	if (typeof candidate.benchmarkCommand === "string" && candidate.benchmarkCommand.trim().length > 0) {
+		config.benchmarkCommand = candidate.benchmarkCommand;
+	}
+	if (Array.isArray(candidate.secondaryMetrics)) {
+		config.secondaryMetrics = normalizeAutoresearchList(
+			candidate.secondaryMetrics.filter((item): item is string => typeof item === "string"),
+		);
+	}
+	if (Array.isArray(candidate.scopePaths)) {
+		config.scopePaths = normalizeAutoresearchList(
+			candidate.scopePaths.filter((item): item is string => typeof item === "string").map(normalizeContractPathSpec),
+		);
+	}
+	if (Array.isArray(candidate.offLimits)) {
+		config.offLimits = normalizeAutoresearchList(
+			candidate.offLimits.filter((item): item is string => typeof item === "string").map(normalizeContractPathSpec),
+		);
+	}
+	if (Array.isArray(candidate.constraints)) {
+		config.constraints = normalizeAutoresearchList(
+			candidate.constraints.filter((item): item is string => typeof item === "string"),
+		);
+	}
+	if (typeof candidate.segmentFingerprint === "string" && candidate.segmentFingerprint.trim().length > 0) {
+		config.segmentFingerprint = candidate.segmentFingerprint;
+	}
+	return config;
+}
+
+function isRunEntry(value: unknown): value is AutoresearchJsonRunEntry {
+	if (typeof value !== "object" || value === null) return false;
+	const candidate = value as { type?: unknown };
+	return candidate.type === undefined || candidate.type === "run";
+}
+
+function isExperimentStatus(value: unknown): value is ExperimentResult["status"] {
+	return value === "keep" || value === "discard" || value === "crash" || value === "checks_failed";
+}
+
+function cloneNumericMetrics(value: unknown): NumericMetricMap {
+	if (typeof value !== "object" || value === null) return {};
+	const metrics = value as { [key: string]: unknown };
+	const clone: NumericMetricMap = {};
+	for (const [key, entryValue] of Object.entries(metrics)) {
+		if (key === "__proto__" || key === "constructor" || key === "prototype") continue;
+		if (typeof entryValue === "number" && Number.isFinite(entryValue)) {
+			clone[key] = entryValue;
+		}
+	}
+	return clone;
+}
+
+function cloneStringArray(value: unknown): string[] {
+	if (!Array.isArray(value)) return [];
+	return value.filter((item): item is string => typeof item === "string");
+}
+
+function hydrateMetricDefs(metricNames: string[] | undefined): MetricDef[] {
+	if (!metricNames) return [];
+	return metricNames.map(name => ({
+		name,
+		unit: inferMetricUnitFromName(name),
+	}));
+}
+
+function cloneAsi(value: unknown): ExperimentResult["asi"] {
+	if (typeof value !== "object" || value === null) return undefined;
+	const clone: { [key: string]: unknown } = {};
+	for (const [key, entryValue] of Object.entries(value)) {
+		if (key === "__proto__" || key === "constructor" || key === "prototype") continue;
+		clone[key] = structuredClone(entryValue);
+	}
+	return clone as ExperimentResult["asi"];
+}
+
+function parseControlEntry(value: unknown): AutoresearchControlEntryData | null {
+	if (typeof value !== "object" || value === null) return null;
+	const candidate = value as { goal?: unknown; mode?: unknown };
+	if (candidate.mode !== "on" && candidate.mode !== "off" && candidate.mode !== "clear") return null;
+	const data: AutoresearchControlEntryData = { mode: candidate.mode };
+	if (typeof candidate.goal === "string" && candidate.goal.trim().length > 0) {
+		data.goal = candidate.goal;
+	}
+	return data;
+}