@ag-eco/agentplate-cli 0.13.0

package/src/tools/loam/utils/audit.ts

@@ -0,0 +1,667 @@
+import { existsSync } from "node:fs";
+import { readFile } from "node:fs/promises";
+import { basename, join, resolve } from "node:path";
+import {
+	type AuditConfig,
+	type AuditThresholds,
+	type LoamConfig,
+	resolveAuditThresholds,
+} from "../schemas/config.ts";
+import type { ExpertiseRecord } from "../schemas/record.ts";
+import { TRACKERS, type TrackerName } from "./active-work.ts";
+import { getExpertisePath } from "./config.ts";
+import { isRecordStale, readExpertiseFile } from "./expertise.ts";
+
+// Words that indicate a convention encodes a rule or rationale rather than
+// restating code structure. Mirrors the Python prototype's RULE_SIGNALS so
+// existing audit baselines transfer 1:1.
+//
+// Known limitation (V1_PLAN §7): the regex over-counts conventions about
+// Bun-isms ("avoid `process.exit`") and under-counts "we …" phrasings.
+// Threshold defaults assume this skew.
+export const RULE_SIGNAL_WORDS = [
+	"because",
+	"must not",
+	"do not",
+	"don't",
+	"avoid",
+	"always ",
+	"never ",
+	"prefer",
+	"required for",
+	"reason:",
+	"rationale",
+	"so that",
+	"otherwise",
+] as const;
+
+export function hasRuleSignal(text: string | undefined): boolean {
+	if (!text) return false;
+	const t = text.toLowerCase();
+	return RULE_SIGNAL_WORDS.some((k) => t.includes(k));
+}
+
+export type Verdict = "PASS" | "WARN" | "FAIL";
+
+export interface MetricVerdict {
+	verdict: Verdict;
+	value: number;
+	threshold: number;
+	warn_threshold?: number;
+}
+
+export interface DomainMix {
+	domain: string;
+	total: number;
+	type_counts: Record<string, number>;
+	high_value_pct: number; // (failure + decision) / total * 100
+	first_recorded_at: string | null;
+	last_recorded_at: string | null;
+}
+
+export interface TrackerCitation {
+	id: string;
+	tracker: TrackerName;
+	count: number;
+	status: string;
+	title: string;
+}
+
+export interface AuditReport {
+	repo: string;
+	total_records: number;
+	domains: string[];
+	ignored_domains: string[];
+	type_mix: Array<{ type: string; count: number; pct: number }>;
+	evidence: {
+		// Deprecated since v0.10.1 (loam-1334) — populated from `with_tracker.sprout`.
+		// Kept readable for one release so old JSON consumers don't break. Removed in v0.11.
+		with_sprout: number;
+		with_tracker: Record<TrackerName, number>;
+		with_any_tracker: number;
+		with_commit: number;
+		with_relates: number;
+		floaters: number;
+	};
+	convention_quality: {
+		total: number;
+		with_rule_signal: number;
+		likely_code_restatement: number;
+	} | null;
+	// Deprecated since v0.10.1 (loam-1334) — populated from sprout entries in
+	// `tracker_citations` for back-compat. Removed in v0.11. Read `tracker_citations`.
+	seed_citations: {
+		unique: number;
+		status_counts: Record<string, number>;
+		missing_in_sprout: number;
+		top_cited: Array<{ id: string; count: number; status: string; title: string }>;
+	};
+	tracker_citations: {
+		unique: number;
+		status_counts: Record<string, number>;
+		missing_in_index: number;
+		top_cited: TrackerCitation[];
+	};
+	by_domain: DomainMix[];
+	weak_domains: string[];
+	thresholds: Required<AuditThresholds>;
+	per_domain_thresholds: Record<string, Required<AuditThresholds>>;
+	signals: {
+		evidence_coverage: MetricVerdict;
+		rule_density: MetricVerdict | null;
+		floater_rate: MetricVerdict;
+		per_domain_max_records: Array<{ domain: string; count: number; verdict: Verdict }>;
+		per_domain_max_stale: Array<{ domain: string; count: number; verdict: Verdict }>;
+	};
+	failures: string[];
+	warnings: string[];
+}
+
+export interface AuditOptions {
+	cwd?: string;
+	domain?: string;
+	ignoreDomains?: string[];
+}
+
+interface RecordWithDomain {
+	record: ExpertiseRecord;
+	domain: string;
+}
+
+export interface TrackerRef {
+	tracker: TrackerName;
+	id: string;
+}
+
+export function extractTrackerRefs(record: ExpertiseRecord): TrackerRef[] {
+	const ev = record.evidence;
+	if (!ev) return [];
+	const refs: TrackerRef[] = [];
+	for (const tracker of TRACKERS) {
+		const value = ev[tracker];
+		if (typeof value === "string" && value.length > 0) {
+			refs.push({ tracker, id: value });
+		}
+	}
+	return refs;
+}
+
+function recordHasTracker(record: ExpertiseRecord): boolean {
+	const ev = record.evidence;
+	if (!ev) return false;
+	return Boolean(ev.sprout || ev.gh || ev.linear || ev.bead);
+}
+
+function recordIsFloater(record: ExpertiseRecord): boolean {
+	const ev = record.evidence;
+	if (!ev) {
+		return !(record.relates_to && record.relates_to.length > 0);
+	}
+	if (ev.sprout || ev.gh || ev.linear || ev.bead || ev.commit) return false;
+	if (record.relates_to && record.relates_to.length > 0) return false;
+	return true;
+}
+
+interface TrackerIndexEntry {
+	tracker: TrackerName;
+	id: string;
+	status?: string;
+	title?: string;
+}
+
+interface SeedRow {
+	id: string;
+	status?: string;
+	title?: string;
+}
+
+function trackerKey(tracker: TrackerName, id: string): string {
+	return `${tracker}:${id}`;
+}
+
+// Render a tracker ref for human-readable output. Sprout keep the bare id
+// (back-compat with the pre-loam-1334 top_cited shape); gh uses the
+// GitHub-native `gh#42` form (incoming `#42` is normalized to `gh#42`);
+// linear and bead use a `<tracker>:<id>` prefix.
+export function formatTrackerId(tracker: TrackerName, id: string): string {
+	if (tracker === "sprout") return id;
+	if (tracker === "gh") return `gh#${id.replace(/^#/, "")}`;
+	return `${tracker}:${id}`;
+}
+
+async function loadTrackerIndex(cwd: string): Promise<Map<string, TrackerIndexEntry>> {
+	const index = new Map<string, TrackerIndexEntry>();
+
+	// Sprout: read .sprout/issues.jsonl (only tracker with local resolution today).
+	const sproutPath = join(cwd, ".sprout", "issues.jsonl");
+	if (existsSync(sproutPath)) {
+		const content = await readFile(sproutPath, "utf-8");
+		for (const line of content.split("\n")) {
+			const trimmed = line.trim();
+			if (!trimmed || trimmed.startsWith("#")) continue;
+			try {
+				const row = JSON.parse(trimmed) as SeedRow;
+				if (row && typeof row.id === "string") {
+					index.set(trackerKey("sprout", row.id), {
+						tracker: "sprout",
+						id: row.id,
+						status: row.status,
+						title: row.title,
+					});
+				}
+			} catch {
+				// Skip malformed sprout rows — the audit is read-only and best-effort here.
+			}
+		}
+	}
+
+	// gh / linear / bead: no local resolution. Citations to these trackers fall
+	// through to `unresolved` in status_counts (per loam-1334 acceptance).
+	// A future seed can layer in networked status lookups (gh CLI, linear API).
+
+	return index;
+}
+
+function classifyEvidenceCoverage(
+	value: number,
+	thresholds: Required<AuditThresholds>,
+): MetricVerdict {
+	const verdict: Verdict =
+		value >= thresholds.evidence_coverage
+			? "PASS"
+			: value >= thresholds.evidence_coverage_warn
+				? "WARN"
+				: "FAIL";
+	return {
+		verdict,
+		value,
+		threshold: thresholds.evidence_coverage,
+		warn_threshold: thresholds.evidence_coverage_warn,
+	};
+}
+
+function classifyRuleDensity(value: number, thresholds: Required<AuditThresholds>): MetricVerdict {
+	const verdict: Verdict =
+		value >= thresholds.rule_density_min
+			? "PASS"
+			: value >= thresholds.rule_density_warn
+				? "WARN"
+				: "FAIL";
+	return {
+		verdict,
+		value,
+		threshold: thresholds.rule_density_min,
+		warn_threshold: thresholds.rule_density_warn,
+	};
+}
+
+function classifyFloaterRate(value: number, thresholds: Required<AuditThresholds>): MetricVerdict {
+	// Floater rate: PASS when ≤ floater_max. Above floater_max is WARN; above
+	// 2× floater_max is FAIL. The 2× FAIL band is asymmetric on purpose — V1_PLAN
+	// expects floaters to drop after attribution gates land in v0.11, so the
+	// FAIL band should fire only for corpora that are *flagrantly* unattributed.
+	const verdict: Verdict =
+		value <= thresholds.floater_max
+			? "PASS"
+			: value <= thresholds.floater_max * 2
+				? "WARN"
+				: "FAIL";
+	return {
+		verdict,
+		value,
+		threshold: thresholds.floater_max,
+	};
+}
+
+async function readDomainRecords(cwd: string, domains: string[]): Promise<RecordWithDomain[]> {
+	const out: RecordWithDomain[] = [];
+	for (const domain of domains) {
+		const filePath = getExpertisePath(domain, cwd);
+		if (!existsSync(filePath)) continue;
+		const records = await readExpertiseFile(filePath, { allowUnknownTypes: true });
+		for (const record of records) {
+			out.push({ record, domain });
+		}
+	}
+	return out;
+}
+
+export async function computeAudit(
+	config: LoamConfig,
+	options: AuditOptions = {},
+): Promise<AuditReport> {
+	const cwd = options.cwd ?? process.cwd();
+	const auditCfg: AuditConfig | undefined = config.audit;
+
+	const allDomains = Object.keys(config.domains);
+	const cfgIgnored = new Set(auditCfg?.ignore_domains ?? []);
+	const cliIgnored = new Set(options.ignoreDomains ?? []);
+	const ignored = [...new Set([...cfgIgnored, ...cliIgnored])];
+
+	let domains: string[];
+	if (options.domain) {
+		domains = allDomains.filter((d) => d === options.domain);
+	} else {
+		domains = allDomains.filter((d) => !cfgIgnored.has(d) && !cliIgnored.has(d));
+	}
+
+	const repo = basename(resolve(cwd)) || cwd;
+
+	const items = await readDomainRecords(cwd, domains);
+	const records = items.map((i) => i.record);
+	const total = records.length;
+
+	const thresholds = resolveAuditThresholds(auditCfg, options.domain);
+	const perDomainThresholds: Record<string, Required<AuditThresholds>> = {};
+	for (const d of domains) {
+		perDomainThresholds[d] = resolveAuditThresholds(auditCfg, d);
+	}
+
+	// Type mix
+	const typeCounts = new Map<string, number>();
+	for (const r of records) {
+		typeCounts.set(r.type, (typeCounts.get(r.type) ?? 0) + 1);
+	}
+	const type_mix = Array.from(typeCounts.entries())
+		.sort((a, b) => b[1] - a[1])
+		.map(([type, count]) => ({
+			type,
+			count,
+			pct: total > 0 ? Math.floor((100 * count) / total) : 0,
+		}));
+
+	// Evidence coverage (per-tracker breakdown + roll-ups)
+	const withTracker: Record<TrackerName, number> = { sprout: 0, gh: 0, linear: 0, bead: 0 };
+	let withCommit = 0;
+	let withRelates = 0;
+	let withAnyTracker = 0;
+	let floaters = 0;
+	for (const r of records) {
+		for (const ref of extractTrackerRefs(r)) {
+			withTracker[ref.tracker]++;
+		}
+		if (r.evidence?.commit) withCommit++;
+		if (r.relates_to && r.relates_to.length > 0) withRelates++;
+		if (recordHasTracker(r) || r.evidence?.commit) withAnyTracker++;
+		if (recordIsFloater(r)) floaters++;
+	}
+
+	// Tracker citations (generalized from seed_citations per loam-1334).
+	const trackerIndex = await loadTrackerIndex(cwd);
+	const refCounts = new Map<string, { tracker: TrackerName; id: string; count: number }>();
+	for (const r of records) {
+		for (const ref of extractTrackerRefs(r)) {
+			const key = trackerKey(ref.tracker, ref.id);
+			const existing = refCounts.get(key);
+			if (existing) {
+				existing.count++;
+			} else {
+				refCounts.set(key, { tracker: ref.tracker, id: ref.id, count: 1 });
+			}
+		}
+	}
+	const trackerStatusCounts: Record<string, number> = {};
+	let trackerMissing = 0;
+	for (const [key, entry] of refCounts.entries()) {
+		const hit = trackerIndex.get(key);
+		if (hit) {
+			const status = hit.status ?? "?";
+			trackerStatusCounts[status] = (trackerStatusCounts[status] ?? 0) + 1;
+		} else if (entry.tracker === "sprout") {
+			// Sprout is the one tracker with a local index — a missing entry means
+			// the citation points at a sprout id that isn't in .sprout/issues.jsonl.
+			trackerMissing++;
+		} else {
+			// gh/linear/bead have no local index yet — bucket into "unresolved"
+			// rather than "missing" so consumers don't confuse the two.
+			trackerStatusCounts.unresolved = (trackerStatusCounts.unresolved ?? 0) + 1;
+		}
+	}
+	const trackerTopCited: TrackerCitation[] = Array.from(refCounts.values())
+		.sort((a, b) => b.count - a.count)
+		.slice(0, 8)
+		.map(({ tracker, id, count }) => {
+			const hit = trackerIndex.get(trackerKey(tracker, id));
+			let status: string;
+			if (hit) {
+				status = hit.status ?? "?";
+			} else if (tracker === "sprout") {
+				status = "MISSING";
+			} else {
+				status = "unresolved";
+			}
+			return {
+				id: formatTrackerId(tracker, id),
+				tracker,
+				count,
+				status,
+				title: (hit?.title ?? "").slice(0, 55),
+			};
+		});
+
+	// Sprout-only back-compat shape (deprecated, populated from tracker_citations).
+	// Filter the tracker refs/index to sprout entries so old JSON consumers reading
+	// `report.seed_citations` still see exactly what they saw pre-loam-1334.
+	const sproutRefEntries = Array.from(refCounts.values()).filter((e) => e.tracker === "sprout");
+	const sproutStatusCounts: Record<string, number> = {};
+	let sproutMissing = 0;
+	for (const { tracker, id } of sproutRefEntries) {
+		const hit = trackerIndex.get(trackerKey(tracker, id));
+		if (hit) {
+			const status = hit.status ?? "?";
+			sproutStatusCounts[status] = (sproutStatusCounts[status] ?? 0) + 1;
+		} else {
+			sproutMissing++;
+		}
+	}
+	const sproutTopCited = sproutRefEntries
+		.sort((a, b) => b.count - a.count)
+		.slice(0, 8)
+		.map(({ id, count }) => {
+			const hit = trackerIndex.get(trackerKey("sprout", id));
+			return {
+				id,
+				count,
+				status: hit?.status ?? "MISSING",
+				title: (hit?.title ?? "MISSING").slice(0, 55),
+			};
+		});
+
+	// Convention quality
+	const conventions = records.filter((r) => r.type === "convention");
+	let conv_quality: AuditReport["convention_quality"] = null;
+	if (conventions.length > 0) {
+		const withSignal = conventions.filter((r) =>
+			hasRuleSignal((r as { content?: string }).content),
+		).length;
+		conv_quality = {
+			total: conventions.length,
+			with_rule_signal: withSignal,
+			likely_code_restatement: conventions.length - withSignal,
+		};
+	}
+
+	// Per-domain mix
+	const byDomain = new Map<string, RecordWithDomain[]>();
+	for (const item of items) {
+		const list = byDomain.get(item.domain) ?? [];
+		list.push(item);
+		byDomain.set(item.domain, list);
+	}
+	const by_domain: DomainMix[] = [];
+	for (const [domain, group] of byDomain.entries()) {
+		const counts: Record<string, number> = {};
+		let first: string | null = null;
+		let last: string | null = null;
+		for (const { record } of group) {
+			counts[record.type] = (counts[record.type] ?? 0) + 1;
+			const ts = record.recorded_at;
+			if (ts) {
+				if (!first || ts < first) first = ts;
+				if (!last || ts > last) last = ts;
+			}
+		}
+		const tot = group.length;
+		const hv = (counts.failure ?? 0) + (counts.decision ?? 0);
+		by_domain.push({
+			domain,
+			total: tot,
+			type_counts: counts,
+			high_value_pct: tot > 0 ? Math.floor((100 * hv) / tot) : 0,
+			first_recorded_at: first,
+			last_recorded_at: last,
+		});
+	}
+	by_domain.sort((a, b) => b.total - a.total);
+
+	const weak_domains = by_domain
+		.filter((d) => d.total >= 5 && d.high_value_pct < 20)
+		.map((d) => d.domain);
+
+	// Verdicts
+	const evidence_coverage_rate = total > 0 ? withAnyTracker / total : 0;
+	const floater_rate = total > 0 ? floaters / total : 0;
+	const rule_density_rate =
+		conv_quality && conv_quality.total > 0 ? conv_quality.with_rule_signal / conv_quality.total : 0;
+
+	const evidence_coverage = classifyEvidenceCoverage(evidence_coverage_rate, thresholds);
+	const floater_rate_verdict = classifyFloaterRate(floater_rate, thresholds);
+	const rule_density: MetricVerdict | null = conv_quality
+		? classifyRuleDensity(rule_density_rate, thresholds)
+		: null;
+
+	const now = new Date();
+	const shelfLife = config.classification_defaults.shelf_life;
+	const per_domain_max_records = by_domain.map((d) => {
+		const t = perDomainThresholds[d.domain] ?? thresholds;
+		return {
+			domain: d.domain,
+			count: d.total,
+			verdict: (d.total <= t.max_records_per_domain ? "PASS" : "WARN") as Verdict,
+		};
+	});
+	const per_domain_max_stale = by_domain.map((d) => {
+		const t = perDomainThresholds[d.domain] ?? thresholds;
+		const group = byDomain.get(d.domain) ?? [];
+		const stale = group.filter((i) => isRecordStale(i.record, now, shelfLife)).length;
+		return {
+			domain: d.domain,
+			count: stale,
+			verdict: (stale <= t.max_stale ? "PASS" : "WARN") as Verdict,
+		};
+	});
+
+	const failures: string[] = [];
+	const warnings: string[] = [];
+	const trackVerdict = (label: string, v: MetricVerdict | null) => {
+		if (!v) return;
+		if (v.verdict === "FAIL") failures.push(label);
+		else if (v.verdict === "WARN") warnings.push(label);
+	};
+	trackVerdict("evidence_coverage", evidence_coverage);
+	trackVerdict("rule_density", rule_density);
+	trackVerdict("floater_rate", floater_rate_verdict);
+	for (const row of per_domain_max_records) {
+		if (row.verdict === "WARN") warnings.push(`max_records_per_domain[${row.domain}]`);
+	}
+	for (const row of per_domain_max_stale) {
+		if (row.verdict === "WARN") warnings.push(`max_stale[${row.domain}]`);
+	}
+
+	return {
+		repo,
+		total_records: total,
+		domains,
+		ignored_domains: ignored,
+		type_mix,
+		evidence: {
+			with_sprout: withTracker.sprout,
+			with_tracker: withTracker,
+			with_any_tracker: withAnyTracker,
+			with_commit: withCommit,
+			with_relates: withRelates,
+			floaters,
+		},
+		convention_quality: conv_quality,
+		seed_citations: {
+			unique: sproutRefEntries.length,
+			status_counts: sproutStatusCounts,
+			missing_in_sprout: sproutMissing,
+			top_cited: sproutTopCited,
+		},
+		tracker_citations: {
+			unique: refCounts.size,
+			status_counts: trackerStatusCounts,
+			missing_in_index: trackerMissing,
+			top_cited: trackerTopCited,
+		},
+		by_domain,
+		weak_domains,
+		thresholds,
+		per_domain_thresholds: perDomainThresholds,
+		signals: {
+			evidence_coverage,
+			rule_density,
+			floater_rate: floater_rate_verdict,
+			per_domain_max_records,
+			per_domain_max_stale,
+		},
+		failures,
+		warnings,
+	};
+}
+
+// ---- Suggest helpers ----
+
+export interface SuggestionGroup {
+	action: "archive" | "revise" | "attribute";
+	headline: string;
+	rationale: string;
+	record_ids: string[];
+	commands: string[];
+}
+
+export interface SuggestPayload {
+	groups: SuggestionGroup[];
+}
+
+export async function buildSuggestions(
+	config: LoamConfig,
+	options: AuditOptions = {},
+): Promise<SuggestPayload> {
+	const cwd = options.cwd ?? process.cwd();
+	const auditCfg = config.audit;
+	const cfgIgnored = new Set(auditCfg?.ignore_domains ?? []);
+	const cliIgnored = new Set(options.ignoreDomains ?? []);
+	let domains = Object.keys(config.domains);
+	if (options.domain) {
+		domains = domains.filter((d) => d === options.domain);
+	} else {
+		domains = domains.filter((d) => !cfgIgnored.has(d) && !cliIgnored.has(d));
+	}
+
+	const items = await readDomainRecords(cwd, domains);
+	const now = new Date();
+	const shelfLife = config.classification_defaults.shelf_life;
+
+	const archive: string[] = [];
+	const revise: string[] = [];
+	const attribute: string[] = [];
+
+	for (const { record } of items) {
+		const id = record.id ?? "";
+		if (!id) continue;
+
+		if (record.classification !== "foundational" && isRecordStale(record, now, shelfLife)) {
+			archive.push(id);
+		}
+		if (record.type === "convention") {
+			const content = (record as { content?: string }).content;
+			if (!hasRuleSignal(content)) revise.push(id);
+		}
+		if (recordIsFloater(record)) {
+			attribute.push(id);
+		}
+	}
+
+	const groups: SuggestionGroup[] = [];
+	if (archive.length > 0) {
+		groups.push({
+			action: "archive",
+			headline: `${archive.length} stale records past shelf life`,
+			rationale:
+				"These records are past their classification shelf life and are not foundational. Soft-archive them with `lm prune` (or `lm archive` once it lands in v0.10).",
+			record_ids: archive,
+			commands: [
+				`lm prune --ids ${archive.join(",")} --dry-run`,
+				`lm prune --ids ${archive.join(",")}`,
+			],
+		});
+	}
+	if (revise.length > 0) {
+		groups.push({
+			action: "revise",
+			headline: `${revise.length} conventions lack rule-signal language`,
+			rationale:
+				"Conventions without rule-signal words (because, must not, avoid, always, never, …) often restate code rather than encode a rule. Revise the content or downgrade to a note.",
+			record_ids: revise,
+			commands: revise.slice(0, 10).map((id) => `lm edit ${id} --content "..."`),
+		});
+	}
+	if (attribute.length > 0) {
+		groups.push({
+			action: "attribute",
+			headline: `${attribute.length} records have no evidence or relates_to link`,
+			rationale:
+				"Floater records lack any tracker (sprout/gh/linear/bead), commit, or relates_to link. Attribute them so future audits and pruning can follow the trail.",
+			record_ids: attribute,
+			commands: attribute
+				.slice(0, 10)
+				.map(
+					(id) =>
+						`lm edit ${id} --evidence-sprout <id>   # or --evidence-commit <sha> / --relates-to <mx-id>`,
+				),
+		});
+	}
+	return { groups };
+}