@mainahq/core 0.2.0

package/src/prompts/__tests__/defaults.test.ts

@@ -0,0 +1,52 @@
+import { describe, expect, test } from "bun:test";
+import { loadDefault, type PromptTask } from "../defaults/index";
+
+const ALL_TASKS: PromptTask[] = [
+	"review",
+	"commit",
+	"tests",
+	"fix",
+	"explain",
+	"design",
+	"context",
+];
+
+describe("loadDefault", () => {
+	test("loadDefault('review') returns string containing '{{constitution}}'", async () => {
+		const template = await loadDefault("review");
+		expect(template).toContain("{{constitution}}");
+	});
+
+	test("loadDefault('commit') returns string containing '{{diff}}'", async () => {
+		const template = await loadDefault("commit");
+		expect(template).toContain("{{diff}}");
+	});
+
+	test("loadDefault('tests') returns string containing '{{plan}}'", async () => {
+		const template = await loadDefault("tests");
+		expect(template).toContain("{{plan}}");
+	});
+
+	test("every default prompt contains [NEEDS CLARIFICATION]", async () => {
+		for (const task of ALL_TASKS) {
+			const template = await loadDefault(task);
+			expect(template).toContain("[NEEDS CLARIFICATION");
+		}
+	});
+
+	test("loadDefault for all 7 task types returns non-empty strings", async () => {
+		for (const task of ALL_TASKS) {
+			const template = await loadDefault(task);
+			expect(typeof template).toBe("string");
+			expect(template.length).toBeGreaterThan(0);
+		}
+	});
+
+	test("loadDefault for unknown task returns fallback template", async () => {
+		// Cast to PromptTask to simulate unknown task
+		const template = await loadDefault("unknown" as PromptTask);
+		expect(typeof template).toBe("string");
+		expect(template.length).toBeGreaterThan(0);
+		expect(template).toContain("{{task}}");
+	});
+});

package/src/prompts/__tests__/engine.test.ts

@@ -0,0 +1,183 @@
+import { afterEach, beforeEach, describe, expect, test } from "bun:test";
+import { mkdirSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { hashContent } from "../../cache/keys";
+import { getFeedbackDb } from "../../db/index";
+import { buildSystemPrompt, getPromptStats, recordOutcome } from "../engine";
+
+let tmpDir: string;
+
+beforeEach(() => {
+	tmpDir = join(
+		import.meta.dir,
+		`tmp-engine-${Date.now()}-${Math.random().toString(36).slice(2)}`,
+	);
+	mkdirSync(tmpDir, { recursive: true });
+});
+
+afterEach(() => {
+	try {
+		const { rmSync } = require("node:fs");
+		rmSync(tmpDir, { recursive: true, force: true });
+	} catch {
+		// ignore
+	}
+});
+
+describe("buildSystemPrompt", () => {
+	test("includes constitution content when present", async () => {
+		const constitutionContent = "Always be concise.";
+		writeFileSync(join(tmpDir, "constitution.md"), constitutionContent);
+
+		// Use a user override with {{constitution}} to ensure test is independent
+		// of which default template loadDefault returns
+		const promptsDir = join(tmpDir, "prompts");
+		mkdirSync(promptsDir, { recursive: true });
+		writeFileSync(
+			join(promptsDir, "commit.md"),
+			"Task prompt.\n\n## Constitution\n{{constitution}}\n\n## Diff\n{{diff}}",
+		);
+
+		const result = await buildSystemPrompt("commit", tmpDir, {
+			diff: "some diff",
+		});
+
+		expect(result.prompt).toContain(constitutionContent);
+	});
+
+	test("replaces template variables", async () => {
+		// Use a user override with {{diff}} to ensure deterministic template
+		const promptsDir = join(tmpDir, "prompts");
+		mkdirSync(promptsDir, { recursive: true });
+		writeFileSync(
+			join(promptsDir, "commit.md"),
+			"Generate commit for:\n{{diff}}",
+		);
+
+		const result = await buildSystemPrompt("commit", tmpDir, {
+			diff: "my test diff content",
+		});
+
+		expect(result.prompt).toContain("my test diff content");
+		expect(result.prompt).not.toContain("{{diff}}");
+	});
+
+	test("returns consistent hash for same content", async () => {
+		// Use a user override so hash is deterministic regardless of default template
+		const promptsDir = join(tmpDir, "prompts");
+		mkdirSync(promptsDir, { recursive: true });
+		writeFileSync(join(promptsDir, "commit.md"), "Stable prompt: {{diff}}");
+
+		const context = { diff: "same diff" };
+
+		const result1 = await buildSystemPrompt("commit", tmpDir, context);
+		const result2 = await buildSystemPrompt("commit", tmpDir, context);
+
+		expect(result1.hash).toBe(result2.hash);
+		expect(result1.hash).toBe(hashContent(result1.prompt));
+	});
+
+	test("uses user override when available", async () => {
+		const promptsDir = join(tmpDir, "prompts");
+		mkdirSync(promptsDir, { recursive: true });
+		const customPrompt = "Custom commit prompt: {{diff}}";
+		writeFileSync(join(promptsDir, "commit.md"), customPrompt);
+
+		const result = await buildSystemPrompt("commit", tmpDir, {
+			diff: "override diff",
+		});
+
+		expect(result.prompt).toContain("Custom commit prompt:");
+		expect(result.prompt).toContain("override diff");
+	});
+});
+
+describe("recordOutcome", () => {
+	test("writes to feedback database", async () => {
+		const promptHash = "abc123hash";
+		const outcome = {
+			accepted: true,
+			command: "git commit -m 'feat: add feature'",
+			context: "some context",
+		};
+
+		recordOutcome(tmpDir, promptHash, outcome);
+
+		// verify by reading back from db
+		const dbResult = getFeedbackDb(tmpDir);
+		expect(dbResult.ok).toBe(true);
+		if (!dbResult.ok) return;
+
+		const { db } = dbResult.value;
+		const rows = db
+			.query("SELECT * FROM feedback WHERE prompt_hash = ?")
+			.all(promptHash) as Array<{
+			id: string;
+			prompt_hash: string;
+			command: string;
+			accepted: number;
+			context: string;
+			created_at: string;
+		}>;
+
+		expect(rows.length).toBe(1);
+		const row = rows[0];
+		expect(row).toBeDefined();
+		expect(row?.prompt_hash).toBe(promptHash);
+		expect(row?.command).toBe(outcome.command);
+		expect(row?.accepted).toBe(1);
+		expect(row?.context).toBe("some context");
+		expect(row?.created_at).toBeTruthy();
+	});
+
+	test("handles rejected outcome", async () => {
+		const promptHash = "rejected-hash";
+		const outcome = {
+			accepted: false,
+			command: "git commit -m 'bad commit'",
+		};
+
+		recordOutcome(tmpDir, promptHash, outcome);
+
+		const dbResult = getFeedbackDb(tmpDir);
+		expect(dbResult.ok).toBe(true);
+		if (!dbResult.ok) return;
+
+		const { db } = dbResult.value;
+		const rows = db
+			.query("SELECT * FROM feedback WHERE prompt_hash = ?")
+			.all(promptHash) as Array<{ accepted: number }>;
+
+		expect(rows.length).toBe(1);
+		expect(rows[0]?.accepted).toBe(0);
+	});
+});
+
+describe("getPromptStats", () => {
+	test("returns correct accept rates", async () => {
+		// record some outcomes
+		recordOutcome(tmpDir, "hash-A", { accepted: true, command: "cmd1" });
+		recordOutcome(tmpDir, "hash-A", { accepted: true, command: "cmd2" });
+		recordOutcome(tmpDir, "hash-A", { accepted: false, command: "cmd3" });
+		recordOutcome(tmpDir, "hash-B", { accepted: false, command: "cmd4" });
+
+		const stats = getPromptStats(tmpDir);
+
+		const statA = stats.find((s) => s.promptHash === "hash-A");
+		const statB = stats.find((s) => s.promptHash === "hash-B");
+
+		expect(statA).toBeDefined();
+		expect(statA?.totalUsage).toBe(3);
+		// 2 accepted out of 3
+		expect(statA?.acceptRate).toBeCloseTo(2 / 3, 5);
+
+		expect(statB).toBeDefined();
+		expect(statB?.totalUsage).toBe(1);
+		expect(statB?.acceptRate).toBe(0);
+	});
+
+	test("returns empty array when no feedback recorded", async () => {
+		const stats = getPromptStats(tmpDir);
+		expect(stats).toEqual([]);
+	});
+});

package/src/prompts/__tests__/evolution-resolve.test.ts

@@ -0,0 +1,169 @@
+import { afterEach, beforeEach, describe, expect, test } from "bun:test";
+import { mkdirSync } from "node:fs";
+import { join } from "node:path";
+import { recordOutcome } from "../engine";
+import { createCandidate, resolveABTests } from "../evolution";
+
+let tmpDir: string;
+
+beforeEach(() => {
+	tmpDir = join(
+		import.meta.dir,
+		`tmp-resolve-${Date.now()}-${Math.random().toString(36).slice(2)}`,
+	);
+	mkdirSync(tmpDir, { recursive: true });
+});
+
+afterEach(() => {
+	try {
+		const { rmSync } = require("node:fs");
+		rmSync(tmpDir, { recursive: true, force: true });
+	} catch {
+		// ignore
+	}
+});
+
+describe("resolveABTests", () => {
+	test("returns empty array when no candidates exist", () => {
+		const resolutions = resolveABTests(tmpDir);
+		expect(resolutions).toEqual([]);
+	});
+
+	test("promotes candidate that outperforms incumbent by >5%", () => {
+		// Create a candidate for "review"
+		const candidate = createCandidate(tmpDir, "review", "Better review prompt");
+
+		// Record 35 samples for the candidate hash: 30 accepted, 5 rejected (85.7% accept)
+		for (let i = 0; i < 30; i++) {
+			recordOutcome(tmpDir, candidate.hash, {
+				accepted: true,
+				command: "review",
+			});
+		}
+		for (let i = 0; i < 5; i++) {
+			recordOutcome(tmpDir, candidate.hash, {
+				accepted: false,
+				command: "review",
+			});
+		}
+
+		// Record 35 samples for the incumbent hash: 20 accepted, 15 rejected (57.1% accept)
+		for (let i = 0; i < 20; i++) {
+			recordOutcome(tmpDir, "incumbent-hash", {
+				accepted: true,
+				command: "review",
+			});
+		}
+		for (let i = 0; i < 15; i++) {
+			recordOutcome(tmpDir, "incumbent-hash", {
+				accepted: false,
+				command: "review",
+			});
+		}
+
+		const resolutions = resolveABTests(tmpDir);
+		expect(resolutions.length).toBe(1);
+		expect(resolutions[0]?.action).toBe("promoted");
+		expect(resolutions[0]?.task).toBe("review");
+		expect(resolutions[0]?.candidateAcceptRate).toBeGreaterThan(0.8);
+	});
+
+	test("retires candidate that underperforms incumbent by >5%", () => {
+		const candidate = createCandidate(tmpDir, "commit", "Worse commit prompt");
+
+		// Record 35 samples for the candidate: 10 accepted, 25 rejected (28.6% accept)
+		for (let i = 0; i < 10; i++) {
+			recordOutcome(tmpDir, candidate.hash, {
+				accepted: true,
+				command: "commit",
+			});
+		}
+		for (let i = 0; i < 25; i++) {
+			recordOutcome(tmpDir, candidate.hash, {
+				accepted: false,
+				command: "commit",
+			});
+		}
+
+		// Record 35 samples for the incumbent: 25 accepted, 10 rejected (71.4% accept)
+		for (let i = 0; i < 25; i++) {
+			recordOutcome(tmpDir, "incumbent-commit", {
+				accepted: true,
+				command: "commit",
+			});
+		}
+		for (let i = 0; i < 10; i++) {
+			recordOutcome(tmpDir, "incumbent-commit", {
+				accepted: false,
+				command: "commit",
+			});
+		}
+
+		const resolutions = resolveABTests(tmpDir);
+		expect(resolutions.length).toBe(1);
+		expect(resolutions[0]?.action).toBe("retired");
+		expect(resolutions[0]?.task).toBe("commit");
+		expect(resolutions[0]?.candidateAcceptRate).toBeLessThan(0.35);
+	});
+
+	test("continues testing with insufficient samples (<30)", () => {
+		const candidate = createCandidate(tmpDir, "fix", "Trial fix prompt");
+
+		// Only 10 samples for candidate
+		for (let i = 0; i < 8; i++) {
+			recordOutcome(tmpDir, candidate.hash, {
+				accepted: true,
+				command: "fix",
+			});
+		}
+		for (let i = 0; i < 2; i++) {
+			recordOutcome(tmpDir, candidate.hash, {
+				accepted: false,
+				command: "fix",
+			});
+		}
+
+		const resolutions = resolveABTests(tmpDir);
+		expect(resolutions.length).toBe(1);
+		expect(resolutions[0]?.action).toBe("continuing");
+		expect(resolutions[0]?.task).toBe("fix");
+		expect(resolutions[0]?.reason).toContain("sample");
+	});
+
+	test("continues testing when difference is within 5% margin", () => {
+		const candidate = createCandidate(tmpDir, "tests", "Similar tests prompt");
+
+		// Record 35 samples for candidate: 25 accepted, 10 rejected (71.4%)
+		for (let i = 0; i < 25; i++) {
+			recordOutcome(tmpDir, candidate.hash, {
+				accepted: true,
+				command: "tests",
+			});
+		}
+		for (let i = 0; i < 10; i++) {
+			recordOutcome(tmpDir, candidate.hash, {
+				accepted: false,
+				command: "tests",
+			});
+		}
+
+		// Record 35 samples for incumbent: 24 accepted, 11 rejected (68.6%)
+		for (let i = 0; i < 24; i++) {
+			recordOutcome(tmpDir, "incumbent-tests", {
+				accepted: true,
+				command: "tests",
+			});
+		}
+		for (let i = 0; i < 11; i++) {
+			recordOutcome(tmpDir, "incumbent-tests", {
+				accepted: false,
+				command: "tests",
+			});
+		}
+
+		const resolutions = resolveABTests(tmpDir);
+		expect(resolutions.length).toBe(1);
+		expect(resolutions[0]?.action).toBe("continuing");
+		expect(resolutions[0]?.reason).toContain("margin");
+	});
+});

package/src/prompts/__tests__/evolution.test.ts

@@ -0,0 +1,187 @@
+import { afterEach, beforeEach, describe, expect, it, test } from "bun:test";
+import { mkdirSync } from "node:fs";
+import { join } from "node:path";
+import { recordOutcome } from "../engine";
+import {
+	abTest,
+	analyseFeedback,
+	analyseWorkflowFeedback,
+	analyseWorkflowRuns,
+	createCandidate,
+	promote,
+	retire,
+} from "../evolution";
+
+let tmpDir: string;
+
+beforeEach(() => {
+	tmpDir = join(
+		import.meta.dir,
+		`tmp-evolution-${Date.now()}-${Math.random().toString(36).slice(2)}`,
+	);
+	mkdirSync(tmpDir, { recursive: true });
+});
+
+afterEach(() => {
+	try {
+		const { rmSync } = require("node:fs");
+		rmSync(tmpDir, { recursive: true, force: true });
+	} catch {
+		// ignore
+	}
+});
+
+describe("analyseFeedback", () => {
+	test("returns low accept rate for task with many rejections", () => {
+		// Record 30 rejections and 5 accepts for "review"
+		for (let i = 0; i < 30; i++) {
+			recordOutcome(tmpDir, "review-hash", {
+				accepted: false,
+				command: "review",
+			});
+		}
+		for (let i = 0; i < 5; i++) {
+			recordOutcome(tmpDir, "review-hash", {
+				accepted: true,
+				command: "review",
+			});
+		}
+
+		const analysis = analyseFeedback(tmpDir, "review");
+		expect(analysis.totalSamples).toBe(35);
+		expect(analysis.acceptRate).toBeCloseTo(5 / 35, 5);
+		expect(analysis.needsImprovement).toBe(true);
+	});
+
+	test("returns high accept rate for well-performing task", () => {
+		for (let i = 0; i < 20; i++) {
+			recordOutcome(tmpDir, "commit-hash", {
+				accepted: true,
+				command: "commit",
+			});
+		}
+		for (let i = 0; i < 2; i++) {
+			recordOutcome(tmpDir, "commit-hash", {
+				accepted: false,
+				command: "commit",
+			});
+		}
+
+		const analysis = analyseFeedback(tmpDir, "commit");
+		expect(analysis.acceptRate).toBeCloseTo(20 / 22, 5);
+		expect(analysis.needsImprovement).toBe(false);
+	});
+
+	test("returns empty analysis for task with no feedback", () => {
+		const analysis = analyseFeedback(tmpDir, "explain");
+		expect(analysis.totalSamples).toBe(0);
+		expect(analysis.acceptRate).toBe(0);
+		expect(analysis.needsImprovement).toBe(false);
+	});
+});
+
+describe("createCandidate", () => {
+	test("stores candidate prompt version", () => {
+		const candidate = createCandidate(
+			tmpDir,
+			"review",
+			"Improved review prompt content",
+		);
+
+		expect(candidate.task).toBe("review");
+		expect(candidate.content).toBe("Improved review prompt content");
+		expect(candidate.status).toBe("candidate");
+		expect(candidate.hash).toBeTruthy();
+	});
+});
+
+describe("abTest", () => {
+	test("returns active prompt when no candidate exists", () => {
+		const result = abTest(tmpDir, "commit");
+		expect(result.variant).toBe("active");
+	});
+
+	test("returns candidate ~20% of time when candidate exists", () => {
+		createCandidate(tmpDir, "review", "Candidate review prompt");
+
+		let candidateCount = 0;
+		const trials = 1000;
+		for (let i = 0; i < trials; i++) {
+			const result = abTest(tmpDir, "review");
+			if (result.variant === "candidate") {
+				candidateCount++;
+			}
+		}
+
+		// Should be roughly 20% ± tolerance
+		const ratio = candidateCount / trials;
+		expect(ratio).toBeGreaterThan(0.1);
+		expect(ratio).toBeLessThan(0.35);
+	});
+});
+
+describe("promote", () => {
+	test("promotes candidate to active", () => {
+		const candidate = createCandidate(tmpDir, "review", "Better review prompt");
+		const promoted = promote(tmpDir, candidate.hash);
+
+		expect(promoted).toBe(true);
+
+		// After promotion, abTest should always return active (no candidate left)
+		const result = abTest(tmpDir, "review");
+		expect(result.variant).toBe("active");
+	});
+});
+
+describe("retire", () => {
+	test("retires candidate without promoting", () => {
+		const candidate = createCandidate(tmpDir, "review", "Bad review prompt");
+		const retired = retire(tmpDir, candidate.hash);
+
+		expect(retired).toBe(true);
+
+		// No candidate should exist
+		const result = abTest(tmpDir, "review");
+		expect(result.variant).toBe("active");
+	});
+});
+
+describe("analyseWorkflowFeedback", () => {
+	it("should return an array", () => {
+		const result = analyseWorkflowFeedback(tmpDir);
+		expect(Array.isArray(result)).toBe(true);
+	});
+
+	it("should have step and acceptRate fields when data exists", () => {
+		const result = analyseWorkflowFeedback(tmpDir);
+		for (const entry of result) {
+			expect(typeof entry.step).toBe("string");
+			expect(typeof entry.totalSamples).toBe("number");
+			expect(typeof entry.acceptRate).toBe("number");
+			expect(typeof entry.needsImprovement).toBe("boolean");
+		}
+	});
+});
+
+describe("analyseWorkflowRuns", () => {
+	it("should return an array", () => {
+		const result = analyseWorkflowRuns(tmpDir);
+		expect(Array.isArray(result)).toBe(true);
+	});
+
+	it("should respect limit parameter", () => {
+		const result = analyseWorkflowRuns(tmpDir, 3);
+		expect(result.length).toBeLessThanOrEqual(3);
+	});
+
+	it("should have correct fields when data exists", () => {
+		const result = analyseWorkflowRuns(tmpDir);
+		for (const entry of result) {
+			expect(typeof entry.workflowId).toBe("string");
+			expect(typeof entry.totalSteps).toBe("number");
+			expect(typeof entry.passedSteps).toBe("number");
+			expect(typeof entry.successRate).toBe("number");
+			expect(typeof entry.createdAt).toBe("string");
+		}
+	});
+});

package/src/prompts/__tests__/loader.test.ts

@@ -0,0 +1,105 @@
+import { afterEach, beforeEach, describe, expect, test } from "bun:test";
+import { mkdirSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import {
+	loadConstitution,
+	loadUserOverride,
+	mergePrompts,
+	renderTemplate,
+} from "../loader";
+
+let tmpDir: string;
+
+beforeEach(() => {
+	tmpDir = join(
+		import.meta.dir,
+		`tmp-loader-${Date.now()}-${Math.random().toString(36).slice(2)}`,
+	);
+	mkdirSync(tmpDir, { recursive: true });
+});
+
+afterEach(() => {
+	// cleanup is best-effort
+	try {
+		const { rmSync } = require("node:fs");
+		rmSync(tmpDir, { recursive: true, force: true });
+	} catch {
+		// ignore
+	}
+});
+
+describe("loadConstitution", () => {
+	test("returns content when file exists", async () => {
+		const constitutionContent = "# My Constitution\nBe helpful.";
+		writeFileSync(join(tmpDir, "constitution.md"), constitutionContent);
+
+		const result = await loadConstitution(tmpDir);
+		expect(result).toBe(constitutionContent);
+	});
+
+	test("returns empty string when file does not exist", async () => {
+		const result = await loadConstitution(tmpDir);
+		expect(result).toBe("");
+	});
+});
+
+describe("loadUserOverride", () => {
+	test("returns content when file exists", async () => {
+		const promptsDir = join(tmpDir, "prompts");
+		mkdirSync(promptsDir, { recursive: true });
+		const overrideContent = "# Custom Review Prompt\nReview carefully.";
+		writeFileSync(join(promptsDir, "review.md"), overrideContent);
+
+		const result = await loadUserOverride(tmpDir, "review");
+		expect(result).toBe(overrideContent);
+	});
+
+	test("returns null when file does not exist", async () => {
+		const result = await loadUserOverride(tmpDir, "review");
+		expect(result).toBeNull();
+	});
+});
+
+describe("mergePrompts", () => {
+	test("returns default when no override", () => {
+		const defaultPrompt = "Default prompt content";
+		const result = mergePrompts(defaultPrompt, null);
+		expect(result).toBe(defaultPrompt);
+	});
+
+	test("returns override when provided (user has full control)", () => {
+		const defaultPrompt = "Default prompt content";
+		const userOverride = "User override content";
+		const result = mergePrompts(defaultPrompt, userOverride);
+		expect(result).toBe(userOverride);
+	});
+});
+
+describe("renderTemplate", () => {
+	test("replaces {{variables}} with values", () => {
+		const template = "Hello {{name}}, your task is {{task}}.";
+		const variables = { name: "Alice", task: "review" };
+		const result = renderTemplate(template, variables);
+		expect(result).toBe("Hello Alice, your task is review.");
+	});
+
+	test("leaves unreplaced variables as-is", () => {
+		const template = "Hello {{name}}, your {{missing}} is here.";
+		const variables = { name: "Alice" };
+		const result = renderTemplate(template, variables);
+		expect(result).toBe("Hello Alice, your {{missing}} is here.");
+	});
+
+	test("replaces multiple occurrences of same variable", () => {
+		const template = "{{task}} is the task. Do {{task}} well.";
+		const variables = { task: "review" };
+		const result = renderTemplate(template, variables);
+		expect(result).toBe("review is the task. Do review well.");
+	});
+
+	test("handles empty variables object", () => {
+		const template = "Hello {{name}}.";
+		const result = renderTemplate(template, {});
+		expect(result).toBe("Hello {{name}}.");
+	});
+});