@ag-eco/agentplate-cli 0.13.0

package/src/tools/loam/utils/bm25.ts

@@ -0,0 +1,238 @@
+import type { ExpertiseRecord } from "../schemas/record.ts";
+
+/**
+ * BM25 parameters (tuned for short document collections like expertise records)
+ */
+export interface BM25Params {
+	/** Controls non-linear term frequency normalization (typical: 1.2-2.0) */
+	k1: number;
+	/** Controls document length normalization (0 = no normalization, 1 = full normalization) */
+	b: number;
+}
+
+/**
+ * Default BM25 parameters optimized for expertise records
+ */
+export const DEFAULT_BM25_PARAMS: BM25Params = {
+	k1: 1.5,
+	b: 0.75,
+};
+
+/**
+ * Result of BM25 search
+ */
+export interface BM25Result {
+	record: ExpertiseRecord;
+	score: number;
+	/** Fields that matched the query */
+	matchedFields: string[];
+}
+
+/**
+ * Tokenize text into searchable terms
+ */
+export function tokenize(text: string): string[] {
+	return text
+		.toLowerCase()
+		.replace(/[^\w\s-]/g, " ") // Replace punctuation with spaces (keep hyphens in words)
+		.split(/\s+/)
+		.filter((token) => token.length > 0);
+}
+
+/**
+ * Extract searchable text from a record
+ */
+export function extractRecordText(record: ExpertiseRecord): {
+	allText: string;
+	fieldTexts: Record<string, string>;
+} {
+	const fieldTexts: Record<string, string> = {};
+	const allParts: string[] = [];
+
+	// Helper to add field text
+	const addField = (name: string, value: unknown): void => {
+		if (typeof value === "string" && value.trim().length > 0) {
+			fieldTexts[name] = value;
+			allParts.push(value);
+		} else if (Array.isArray(value)) {
+			const arrayText = value.filter((item) => typeof item === "string").join(" ");
+			if (arrayText.trim().length > 0) {
+				fieldTexts[name] = arrayText;
+				allParts.push(arrayText);
+			}
+		}
+	};
+
+	// Extract type-specific fields
+	switch (record.type) {
+		case "pattern":
+			addField("name", record.name);
+			addField("description", record.description);
+			addField("files", record.files);
+			break;
+		case "convention":
+			addField("content", record.content);
+			break;
+		case "failure":
+			addField("description", record.description);
+			addField("resolution", record.resolution);
+			break;
+		case "decision":
+			addField("title", record.title);
+			addField("rationale", record.rationale);
+			break;
+		case "reference":
+			addField("name", record.name);
+			addField("description", record.description);
+			addField("files", record.files);
+			break;
+		case "guide":
+			addField("name", record.name);
+			addField("description", record.description);
+			break;
+	}
+
+	// Add common fields
+	addField("tags", record.tags);
+
+	return {
+		allText: allParts.join(" "),
+		fieldTexts,
+	};
+}
+
+/**
+ * Calculate term frequency in a document
+ */
+function calculateTermFrequency(tokens: string[]): Map<string, number> {
+	const tf = new Map<string, number>();
+	for (const token of tokens) {
+		tf.set(token, (tf.get(token) || 0) + 1);
+	}
+	return tf;
+}
+
+/**
+ * Calculate inverse document frequency for all terms in the corpus
+ */
+function calculateIDF(corpus: Array<{ tokens: string[] }>): Map<string, number> {
+	const docCount = corpus.length;
+	const docFreq = new Map<string, number>();
+
+	// Count how many documents contain each term
+	for (const doc of corpus) {
+		const uniqueTerms = new Set(doc.tokens);
+		for (const term of uniqueTerms) {
+			docFreq.set(term, (docFreq.get(term) || 0) + 1);
+		}
+	}
+
+	// Calculate IDF for each term
+	const idf = new Map<string, number>();
+	for (const [term, freq] of docFreq.entries()) {
+		// IDF formula: log((N - df + 0.5) / (df + 0.5) + 1)
+		// The +1 ensures positive values for common terms
+		idf.set(term, Math.log((docCount - freq + 0.5) / (freq + 0.5) + 1));
+	}
+
+	return idf;
+}
+
+/**
+ * Calculate BM25 score for a single document against a query
+ */
+function calculateBM25Score(
+	queryTokens: string[],
+	docTokens: string[],
+	docLength: number,
+	avgDocLength: number,
+	idf: Map<string, number>,
+	params: BM25Params,
+): number {
+	const tf = calculateTermFrequency(docTokens);
+	let score = 0;
+
+	for (const queryTerm of queryTokens) {
+		const termFreq = tf.get(queryTerm) || 0;
+		const termIDF = idf.get(queryTerm) || 0;
+
+		// BM25 formula
+		const numerator = termFreq * (params.k1 + 1);
+		const denominator =
+			termFreq + params.k1 * (1 - params.b + params.b * (docLength / avgDocLength));
+
+		score += termIDF * (numerator / denominator);
+	}
+
+	return score;
+}
+
+/**
+ * Search records using BM25 ranking
+ */
+export function searchBM25(
+	records: ExpertiseRecord[],
+	query: string,
+	params: BM25Params = DEFAULT_BM25_PARAMS,
+): BM25Result[] {
+	if (records.length === 0 || query.trim().length === 0) {
+		return [];
+	}
+
+	const queryTokens = tokenize(query);
+	if (queryTokens.length === 0) {
+		return [];
+	}
+
+	// Extract and tokenize all documents
+	const docs = records.map((record) => {
+		const { allText, fieldTexts } = extractRecordText(record);
+		const tokens = tokenize(allText);
+		return { record, tokens, allText, fieldTexts };
+	});
+
+	// Calculate average document length
+	const totalLength = docs.reduce((sum, doc) => sum + doc.tokens.length, 0);
+	const avgDocLength = totalLength / docs.length;
+
+	// Calculate IDF for all terms
+	const idf = calculateIDF(docs);
+
+	// Score each document
+	const results: BM25Result[] = [];
+
+	for (const doc of docs) {
+		const score = calculateBM25Score(
+			queryTokens,
+			doc.tokens,
+			doc.tokens.length,
+			avgDocLength,
+			idf,
+			params,
+		);
+
+		// Only include results with score > 0
+		if (score > 0) {
+			// Determine which fields matched
+			const matchedFields: string[] = [];
+			for (const [fieldName, fieldText] of Object.entries(doc.fieldTexts)) {
+				const fieldTokens = tokenize(fieldText);
+				const hasMatch = queryTokens.some((qt) => fieldTokens.includes(qt));
+				if (hasMatch) {
+					matchedFields.push(fieldName);
+				}
+			}
+
+			results.push({
+				record: doc.record,
+				score,
+				matchedFields,
+			});
+		}
+	}
+
+	// Sort by score descending
+	results.sort((a, b) => b.score - a.score);
+
+	return results;
+}

package/src/tools/loam/utils/budget.ts

@@ -0,0 +1,142 @@
+import type { BuiltinRecordType, Classification, ExpertiseRecord } from "../schemas/record.ts";
+import { computeConfirmationScore, type ScoredRecord } from "./scoring.ts";
+
+export const DEFAULT_BUDGET = 4000;
+
+// Priority order for built-in types only. Custom types (Phase 2) sort after
+// built-ins (indexOf returns -1, which sorts ahead — so use length when missing).
+const TYPE_PRIORITY: BuiltinRecordType[] = [
+	"convention",
+	"decision",
+	"pattern",
+	"guide",
+	"failure",
+	"reference",
+];
+
+/** Priority order for classifications (lower index = higher priority) */
+const CLASSIFICATION_PRIORITY: Classification[] = ["foundational", "tactical", "observational"];
+
+export interface DomainRecords {
+	domain: string;
+	records: ScoredRecord[];
+}
+
+export interface BudgetResult {
+	/** Records kept, grouped by domain (preserves original domain order) */
+	kept: DomainRecords[];
+	/** Total number of records that were dropped */
+	droppedCount: number;
+	/** Number of domains that had records dropped */
+	droppedDomainCount: number;
+}
+
+/**
+ * Sort records by priority: type order, then classification, then confirmation score
+ * (higher score = higher priority), then recency (newest first).
+ */
+function recordSortKey(r: ScoredRecord): [number, number, number, number] {
+	const builtinIdx = TYPE_PRIORITY.indexOf(r.type as BuiltinRecordType);
+	// Custom types (-1 from indexOf) sort after all built-ins.
+	const typeIdx = builtinIdx === -1 ? TYPE_PRIORITY.length : builtinIdx;
+	const classIdx = CLASSIFICATION_PRIORITY.indexOf(r.classification);
+	const confirmationScore = computeConfirmationScore(r);
+	const time = r.recorded_at ? new Date(r.recorded_at).getTime() : 0;
+	return [typeIdx, classIdx, -confirmationScore, -time];
+}
+
+function compareRecords(a: ScoredRecord, b: ScoredRecord): number {
+	const [a0, a1, a2, a3] = recordSortKey(a);
+	const [b0, b1, b2, b3] = recordSortKey(b);
+	return a0 - b0 || a1 - b1 || a2 - b2 || a3 - b3;
+}
+
+/**
+ * Estimate token count from character count (chars / 4).
+ */
+export function estimateTokens(text: string): number {
+	return Math.ceil(text.length / 4);
+}
+
+/**
+ * Apply a token budget to records across multiple domains.
+ *
+ * Records are prioritized by type (conventions first, then decisions, etc.),
+ * then by classification (foundational > tactical > observational),
+ * then by confirmation score (higher = higher priority),
+ * then by recency (newest first).
+ *
+ * The formatRecord callback is used to estimate per-record token cost.
+ */
+export function applyBudget(
+	domains: DomainRecords[],
+	budget: number,
+	formatRecord: (record: ExpertiseRecord, domain: string) => string,
+): BudgetResult {
+	// Flatten all records with their domain, then sort by priority
+	const tagged: Array<{ domain: string; record: ScoredRecord }> = [];
+	for (const d of domains) {
+		for (const r of d.records) {
+			tagged.push({ domain: d.domain, record: r });
+		}
+	}
+	tagged.sort((a, b) => compareRecords(a.record, b.record));
+
+	const totalRecords = tagged.length;
+	let usedTokens = 0;
+	const kept = new Set<number>();
+
+	for (const [i, item] of tagged.entries()) {
+		const formatted = formatRecord(item.record, item.domain);
+		const cost = estimateTokens(formatted);
+		if (usedTokens + cost <= budget) {
+			usedTokens += cost;
+			kept.add(i);
+		}
+	}
+
+	// Rebuild domain groups preserving original domain order and record order
+	const domainOrder = domains.map((d) => d.domain);
+	const result: DomainRecords[] = [];
+	const droppedDomains = new Set<string>();
+
+	for (const domainName of domainOrder) {
+		const originalRecords = domains.find((d) => d.domain === domainName)?.records;
+		const keptRecords: ScoredRecord[] = [];
+
+		for (const rec of originalRecords ?? []) {
+			// Find this record's index in the tagged array
+			const idx = tagged.findIndex((t) => t.domain === domainName && t.record === rec);
+			if (idx !== -1 && kept.has(idx)) {
+				keptRecords.push(rec);
+			} else if (idx !== -1) {
+				droppedDomains.add(domainName);
+			}
+		}
+
+		if (keptRecords.length > 0) {
+			result.push({ domain: domainName, records: keptRecords });
+		} else if ((originalRecords ?? []).length > 0) {
+			droppedDomains.add(domainName);
+		}
+	}
+
+	const droppedCount = totalRecords - kept.size;
+
+	return {
+		kept: result,
+		droppedCount,
+		droppedDomainCount: droppedDomains.size,
+	};
+}
+
+/**
+ * Format the truncation summary line shown when records are dropped.
+ */
+export function formatBudgetSummary(droppedCount: number, droppedDomainCount: number): string {
+	const domainPart =
+		droppedDomainCount > 0
+			? ` across ${droppedDomainCount} domain${droppedDomainCount === 1 ? "" : "s"}`
+			: "";
+	return `... and ${droppedCount} more record${droppedCount === 1 ? "" : "s"}${domainPart} (use --budget <n> to show more)`;
+}