@mainahq/core 0.2.0

package/src/prompts/engine.ts

@@ -0,0 +1,137 @@
+import { hashContent } from "../cache/keys";
+import { getFeedbackDb } from "../db/index";
+import { loadDefault, type PromptTask } from "./defaults/index";
+import { abTest } from "./evolution";
+import {
+	loadConstitution,
+	loadUserOverride,
+	mergePrompts,
+	renderTemplate,
+} from "./loader";
+
+export interface BuiltPrompt {
+	prompt: string;
+	hash: string;
+}
+
+export interface FeedbackOutcome {
+	accepted: boolean;
+	command: string;
+	context?: string;
+}
+
+export interface PromptStat {
+	promptHash: string;
+	totalUsage: number;
+	acceptRate: number;
+}
+
+/**
+ * Assembles a full system prompt for a given task.
+ * Loads default template, applies user overrides, injects constitution,
+ * renders template variables, and returns the prompt with its content hash.
+ */
+export async function buildSystemPrompt(
+	task: string,
+	mainaDir: string,
+	variables: Record<string, string> = {},
+): Promise<BuiltPrompt> {
+	// Load constitution (always injected)
+	const constitution = await loadConstitution(mainaDir);
+
+	// Load default prompt template
+	const defaultPrompt = await loadDefault(task as PromptTask);
+
+	// A/B test: check if a candidate prompt should be used (20% traffic)
+	const abResult = abTest(mainaDir, task);
+	if (abResult.variant === "candidate" && abResult.hash) {
+		// Candidate selected — load its content from prompt_versions
+		const dbResult = getFeedbackDb(mainaDir);
+		if (dbResult.ok) {
+			const row = dbResult.value.db
+				.query("SELECT content FROM prompt_versions WHERE hash = ? LIMIT 1")
+				.get(abResult.hash) as { content: string } | null;
+			if (row?.content) {
+				const allVariables: Record<string, string> = {
+					constitution,
+					...variables,
+				};
+				const prompt = renderTemplate(row.content, allVariables);
+				return { prompt, hash: hashContent(prompt) };
+			}
+		}
+	}
+
+	// Load user override from .maina/prompts/<task>.md
+	const userOverride = await loadUserOverride(mainaDir, task);
+
+	// Merge: user overrides replace default entirely if present
+	const merged = mergePrompts(defaultPrompt, userOverride);
+
+	// Render template variables — constitution is always available
+	const allVariables: Record<string, string> = {
+		constitution,
+		...variables,
+	};
+	const prompt = renderTemplate(merged, allVariables);
+
+	return {
+		prompt,
+		hash: hashContent(prompt),
+	};
+}
+
+/**
+ * Records a prompt outcome (accepted/rejected) to the feedback database.
+ * Used to drive prompt evolution via the `maina learn` command.
+ */
+export function recordOutcome(
+	mainaDir: string,
+	promptHash: string,
+	outcome: FeedbackOutcome,
+): void {
+	const dbResult = getFeedbackDb(mainaDir);
+	if (!dbResult.ok) return;
+
+	const { db } = dbResult.value;
+	const id = `${promptHash}-${Date.now()}-${Math.random().toString(36).slice(2)}`;
+
+	db.prepare(
+		`INSERT INTO feedback (id, prompt_hash, command, accepted, context, created_at)
+		 VALUES (?, ?, ?, ?, ?, ?)`,
+	).run(
+		id,
+		promptHash,
+		outcome.command,
+		outcome.accepted ? 1 : 0,
+		outcome.context ?? null,
+		new Date().toISOString(),
+	);
+}
+
+/**
+ * Returns per-prompt-hash statistics: total usage and accept rate.
+ */
+export function getPromptStats(mainaDir: string): PromptStat[] {
+	const dbResult = getFeedbackDb(mainaDir);
+	if (!dbResult.ok) return [];
+
+	const { db } = dbResult.value;
+
+	const rows = db
+		.query(
+			`SELECT prompt_hash, COUNT(*) as total, SUM(CASE WHEN accepted = 1 THEN 1 ELSE 0 END) as accepted_count
+			 FROM feedback GROUP BY prompt_hash`,
+		)
+		.all() as Array<{
+		prompt_hash: string;
+		total: number;
+		accepted_count: number;
+	}>;
+
+	return rows.map((row) => ({
+		promptHash: row.prompt_hash,
+		totalUsage: row.total,
+		acceptRate: row.total > 0 ? row.accepted_count / row.total : 0,
+	}));
+}

package/src/prompts/evolution.ts

@@ -0,0 +1,409 @@
+import { hashContent } from "../cache/keys";
+import { getFeedbackDb } from "../db/index";
+
+// ── Types ────────────────────────────────────────────────────────────────────
+
+export interface FeedbackAnalysis {
+	task: string;
+	totalSamples: number;
+	acceptRate: number;
+	needsImprovement: boolean;
+}
+
+export interface CandidatePrompt {
+	task: string;
+	content: string;
+	hash: string;
+	status: "candidate" | "active" | "retired";
+}
+
+export interface AbTestResult {
+	variant: "active" | "candidate";
+	hash?: string;
+}
+
+export interface ABResolution {
+	task: string;
+	action: "promoted" | "retired" | "continuing";
+	reason: string;
+	candidateAcceptRate?: number;
+	incumbentAcceptRate?: number;
+}
+
+export interface WorkflowStepAnalysis {
+	step: string;
+	totalSamples: number;
+	acceptRate: number;
+	needsImprovement: boolean;
+}
+
+export interface WorkflowRunSummary {
+	workflowId: string;
+	totalSteps: number;
+	passedSteps: number;
+	successRate: number;
+	createdAt: string;
+}
+
+// ── Constants ────────────────────────────────────────────────────────────────
+
+/** Accept rate below this threshold triggers needsImprovement flag */
+const IMPROVEMENT_THRESHOLD = 0.6;
+
+/** Minimum samples before we judge a prompt's performance */
+const MIN_SAMPLES = 10;
+
+/** Minimum samples for A/B test resolution */
+const MIN_AB_SAMPLES = 30;
+
+/** Accept rate margin: candidate must beat incumbent by this much to promote */
+const AB_MARGIN = 0.05;
+
+/** Candidate traffic split (20% goes to candidate) */
+const CANDIDATE_RATIO = 0.2;
+
+// ── Helpers ──────────────────────────────────────────────────────────────────
+
+function ensurePromptVersionsTable(mainaDir: string) {
+	const dbResult = getFeedbackDb(mainaDir);
+	if (!dbResult.ok) return null;
+	return dbResult.value;
+}
+
+// ── Public API ───────────────────────────────────────────────────────────────
+
+/**
+ * Analyses feedback for a specific task (command).
+ * Returns sample count, accept rate, and whether improvement is needed.
+ */
+export function analyseFeedback(
+	mainaDir: string,
+	task: string,
+): FeedbackAnalysis {
+	const handle = ensurePromptVersionsTable(mainaDir);
+	if (!handle) {
+		return { task, totalSamples: 0, acceptRate: 0, needsImprovement: false };
+	}
+
+	const { db } = handle;
+	const row = db
+		.query(
+			`SELECT COUNT(*) as total,
+			        SUM(CASE WHEN accepted = 1 THEN 1 ELSE 0 END) as accepted_count
+			 FROM feedback WHERE command = ?`,
+		)
+		.get(task) as { total: number; accepted_count: number } | null;
+
+	if (!row || row.total === 0) {
+		return { task, totalSamples: 0, acceptRate: 0, needsImprovement: false };
+	}
+
+	const acceptRate = row.accepted_count / row.total;
+	const needsImprovement =
+		row.total >= MIN_SAMPLES && acceptRate < IMPROVEMENT_THRESHOLD;
+
+	return {
+		task,
+		totalSamples: row.total,
+		acceptRate,
+		needsImprovement,
+	};
+}
+
+/**
+ * Analyse feedback grouped by workflow step.
+ * Returns per-step metrics using the workflow_step column.
+ */
+export function analyseWorkflowFeedback(
+	mainaDir: string,
+): WorkflowStepAnalysis[] {
+	const handle = ensurePromptVersionsTable(mainaDir);
+	if (!handle) return [];
+
+	const { db } = handle;
+
+	const rows = db
+		.query(
+			`SELECT workflow_step, COUNT(*) as total,
+			        SUM(CASE WHEN accepted = 1 THEN 1 ELSE 0 END) as accepted_count
+			 FROM feedback
+			 WHERE workflow_step IS NOT NULL
+			 GROUP BY workflow_step
+			 ORDER BY workflow_step`,
+		)
+		.all() as Array<{
+		workflow_step: string;
+		total: number;
+		accepted_count: number;
+	}>;
+
+	return rows.map((row) => {
+		const acceptRate = row.total > 0 ? row.accepted_count / row.total : 0;
+		return {
+			step: row.workflow_step,
+			totalSamples: row.total,
+			acceptRate,
+			needsImprovement:
+				row.total >= MIN_SAMPLES && acceptRate < IMPROVEMENT_THRESHOLD,
+		};
+	});
+}
+
+/**
+ * Analyse workflow runs grouped by workflow_id.
+ * Returns per-run summary showing how many steps passed.
+ */
+export function analyseWorkflowRuns(
+	mainaDir: string,
+	limit = 5,
+): WorkflowRunSummary[] {
+	const handle = ensurePromptVersionsTable(mainaDir);
+	if (!handle) return [];
+
+	const { db } = handle;
+
+	const rows = db
+		.query(
+			`SELECT workflow_id, COUNT(*) as total_steps,
+			        SUM(CASE WHEN accepted = 1 THEN 1 ELSE 0 END) as passed_steps,
+			        MIN(created_at) as created_at
+			 FROM feedback
+			 WHERE workflow_id IS NOT NULL
+			 GROUP BY workflow_id
+			 ORDER BY created_at DESC
+			 LIMIT ?`,
+		)
+		.all(limit) as Array<{
+		workflow_id: string;
+		total_steps: number;
+		passed_steps: number;
+		created_at: string;
+	}>;
+
+	return rows.map((row) => ({
+		workflowId: row.workflow_id,
+		totalSteps: row.total_steps,
+		passedSteps: row.passed_steps,
+		successRate: row.total_steps > 0 ? row.passed_steps / row.total_steps : 0,
+		createdAt: row.created_at,
+	}));
+}
+
+/**
+ * Creates a candidate prompt version for A/B testing.
+ * Stored in prompt_versions table with status 'candidate'.
+ */
+export function createCandidate(
+	mainaDir: string,
+	task: string,
+	content: string,
+): CandidatePrompt {
+	const hash = hashContent(content);
+	const handle = ensurePromptVersionsTable(mainaDir);
+
+	if (handle) {
+		const { db } = handle;
+		const id = `${task}-${Date.now()}-${Math.random().toString(36).slice(2)}`;
+
+		// Get next version number for this task
+		const maxRow = db
+			.query(
+				`SELECT MAX(version) as max_ver FROM prompt_versions WHERE task = ?`,
+			)
+			.get(task) as { max_ver: number | null } | null;
+		const nextVersion = (maxRow?.max_ver ?? 0) + 1;
+
+		db.prepare(
+			`INSERT INTO prompt_versions (id, task, hash, content, version, accept_rate, usage_count, created_at)
+			 VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
+		).run(
+			id,
+			task,
+			hash,
+			content,
+			nextVersion,
+			null,
+			0,
+			new Date().toISOString(),
+		);
+	}
+
+	return { task, content, hash, status: "candidate" };
+}
+
+/**
+ * A/B test: returns 'active' (80%) or 'candidate' (20%).
+ * If no candidate exists for the task, always returns 'active'.
+ */
+export function abTest(mainaDir: string, task: string): AbTestResult {
+	const handle = ensurePromptVersionsTable(mainaDir);
+	if (!handle) {
+		return { variant: "active" };
+	}
+
+	const { db } = handle;
+
+	// Find the most recent candidate for this task
+	const candidate = db
+		.query(
+			`SELECT hash, content FROM prompt_versions
+			 WHERE task = ? ORDER BY version DESC LIMIT 1`,
+		)
+		.get(task) as { hash: string; content: string } | null;
+
+	if (!candidate) {
+		return { variant: "active" };
+	}
+
+	// 80/20 split
+	if (Math.random() < CANDIDATE_RATIO) {
+		return { variant: "candidate", hash: candidate.hash };
+	}
+
+	return { variant: "active" };
+}
+
+/**
+ * Promotes a candidate to active by removing it from the candidates table.
+ * The promoted content should be written to .maina/prompts/<task>.md by the caller.
+ */
+export function promote(mainaDir: string, hash: string): boolean {
+	const handle = ensurePromptVersionsTable(mainaDir);
+	if (!handle) return false;
+
+	const { db } = handle;
+	const result = db
+		.prepare(`DELETE FROM prompt_versions WHERE hash = ?`)
+		.run(hash);
+	return result.changes > 0;
+}
+
+/**
+ * Retires a candidate without promoting it.
+ * Removes it from the candidates table.
+ */
+export function retire(mainaDir: string, hash: string): boolean {
+	const handle = ensurePromptVersionsTable(mainaDir);
+	if (!handle) return false;
+
+	const { db } = handle;
+	const result = db
+		.prepare(`DELETE FROM prompt_versions WHERE hash = ?`)
+		.run(hash);
+	return result.changes > 0;
+}
+
+/**
+ * Check all active A/B tests and promote/retire based on performance.
+ * Returns a list of actions taken.
+ *
+ * Logic:
+ * - For each task with a candidate in prompt_versions, query feedback
+ * - If candidate has < MIN_AB_SAMPLES samples: continue
+ * - If candidate accept rate > incumbent + AB_MARGIN: promote
+ * - If candidate accept rate < incumbent - AB_MARGIN: retire
+ * - Otherwise: continue (within margin)
+ */
+export function resolveABTests(mainaDir: string): ABResolution[] {
+	const handle = ensurePromptVersionsTable(mainaDir);
+	if (!handle) return [];
+
+	const { db } = handle;
+
+	// Find all candidates (one per task, most recent)
+	const candidates = db
+		.query(
+			`SELECT task, hash FROM prompt_versions
+			 ORDER BY version DESC`,
+		)
+		.all() as Array<{ task: string; hash: string }>;
+
+	if (candidates.length === 0) return [];
+
+	// Deduplicate: keep only the most recent candidate per task
+	const taskMap = new Map<string, string>();
+	for (const c of candidates) {
+		if (!taskMap.has(c.task)) {
+			taskMap.set(c.task, c.hash);
+		}
+	}
+
+	const resolutions: ABResolution[] = [];
+
+	for (const [task, candidateHash] of taskMap) {
+		// Get candidate feedback stats
+		const candidateRow = db
+			.query(
+				`SELECT COUNT(*) as total,
+				        SUM(CASE WHEN accepted = 1 THEN 1 ELSE 0 END) as accepted_count
+				 FROM feedback WHERE command = ? AND prompt_hash = ?`,
+			)
+			.get(task, candidateHash) as {
+			total: number;
+			accepted_count: number;
+		} | null;
+
+		const candidateTotal = candidateRow?.total ?? 0;
+		const candidateAccepted = candidateRow?.accepted_count ?? 0;
+
+		if (candidateTotal < MIN_AB_SAMPLES) {
+			resolutions.push({
+				task,
+				action: "continuing",
+				reason: `Insufficient samples (${candidateTotal}/${MIN_AB_SAMPLES})`,
+				candidateAcceptRate:
+					candidateTotal > 0 ? candidateAccepted / candidateTotal : 0,
+			});
+			continue;
+		}
+
+		const candidateAcceptRate = candidateAccepted / candidateTotal;
+
+		// Get recent incumbent feedback (last 100 samples, excluding candidate)
+		const incumbentRows = db
+			.prepare(
+				`SELECT accepted FROM feedback WHERE command = ? AND prompt_hash != ? ORDER BY created_at DESC LIMIT 100`,
+			)
+			.all(task, candidateHash) as Array<{ accepted: number }>;
+
+		const incumbentTotal = incumbentRows.length;
+		const incumbentAccepted = incumbentRows.filter(
+			(r) => r.accepted === 1,
+		).length;
+		const incumbentAcceptRate =
+			incumbentTotal > 0 ? incumbentAccepted / incumbentTotal : 0.5;
+
+		if (candidateAcceptRate > incumbentAcceptRate + AB_MARGIN) {
+			// Candidate outperforms — promote
+			promote(mainaDir, candidateHash);
+			resolutions.push({
+				task,
+				action: "promoted",
+				reason: `Candidate (${(candidateAcceptRate * 100).toFixed(1)}%) outperforms incumbent (${(incumbentAcceptRate * 100).toFixed(1)}%) by >${(AB_MARGIN * 100).toFixed(0)}%`,
+				candidateAcceptRate,
+				incumbentAcceptRate,
+			});
+		} else if (candidateAcceptRate < incumbentAcceptRate - AB_MARGIN) {
+			// Candidate underperforms — retire
+			retire(mainaDir, candidateHash);
+			resolutions.push({
+				task,
+				action: "retired",
+				reason: `Candidate (${(candidateAcceptRate * 100).toFixed(1)}%) underperforms incumbent (${(incumbentAcceptRate * 100).toFixed(1)}%) by >${(AB_MARGIN * 100).toFixed(0)}%`,
+				candidateAcceptRate,
+				incumbentAcceptRate,
+			});
+		} else {
+			// Within margin — continue
+			resolutions.push({
+				task,
+				action: "continuing",
+				reason: `Within margin: candidate ${(candidateAcceptRate * 100).toFixed(1)}% vs incumbent ${(incumbentAcceptRate * 100).toFixed(1)}%`,
+				candidateAcceptRate,
+				incumbentAcceptRate,
+			});
+		}
+	}
+
+	return resolutions;
+}

package/src/prompts/loader.ts

@@ -0,0 +1,71 @@
+import { join } from "node:path";
+
+/**
+ * Reads .maina/constitution.md, returns empty string if not found.
+ * Never throws.
+ */
+export async function loadConstitution(mainaDir: string): Promise<string> {
+	try {
+		const filePath = join(mainaDir, "constitution.md");
+		const file = Bun.file(filePath);
+		const exists = await file.exists();
+		if (!exists) {
+			return "";
+		}
+		return await file.text();
+	} catch {
+		return "";
+	}
+}
+
+/**
+ * Reads .maina/prompts/<task>.md, returns null if not found.
+ * Never throws.
+ */
+export async function loadUserOverride(
+	mainaDir: string,
+	task: string,
+): Promise<string | null> {
+	try {
+		const filePath = join(mainaDir, "prompts", `${task}.md`);
+		const file = Bun.file(filePath);
+		const exists = await file.exists();
+		if (!exists) {
+			return null;
+		}
+		return await file.text();
+	} catch {
+		return null;
+	}
+}
+
+/**
+ * Merges default prompt with user override.
+ * If userOverride is null, returns default.
+ * If userOverride exists, it REPLACES the default entirely (user has full control).
+ */
+export function mergePrompts(
+	defaultPrompt: string,
+	userOverride: string | null,
+): string {
+	if (userOverride === null) {
+		return defaultPrompt;
+	}
+	return userOverride;
+}
+
+/**
+ * Replaces all {{variableName}} placeholders with values from the variables object.
+ * Unreplaced variables are left as-is.
+ */
+export function renderTemplate(
+	template: string,
+	variables: Record<string, string>,
+): string {
+	return template.replace(/\{\{(\w+)\}\}/g, (match, key: string) => {
+		if (Object.hasOwn(variables, key)) {
+			return variables[key] ?? match;
+		}
+		return match;
+	});
+}