opencode-starterkit 1.0.1

package/baseline/plugin/lib/distill.ts

@@ -0,0 +1,376 @@
+/**
+ * Heuristic Distillation Module
+ *
+ * Compresses batches of temporal messages into distillations using
+ * TF-IDF term extraction and key sentence selection.
+ * No LLM dependency — pure heuristic, upgradeable later.
+ *
+ * Pipeline: temporal_messages → TF-IDF terms + key sentences → distillation
+ */
+
+import {
+	type DistillationInput,
+	getUndistilledMessageCount,
+	getUndistilledMessages,
+	MEMORY_CONFIG,
+	markMessagesDistilled,
+	storeDistillation,
+	type TemporalMessageRow,
+} from "./memory-db.js";
+
+// ============================================================================
+// TF-IDF Engine
+// ============================================================================
+
+/** Stop words to exclude from term extraction */
+const STOP_WORDS = new Set([
+	"the",
+	"be",
+	"to",
+	"of",
+	"and",
+	"a",
+	"in",
+	"that",
+	"have",
+	"i",
+	"it",
+	"for",
+	"not",
+	"on",
+	"with",
+	"he",
+	"as",
+	"you",
+	"do",
+	"at",
+	"this",
+	"but",
+	"his",
+	"by",
+	"from",
+	"they",
+	"we",
+	"say",
+	"her",
+	"she",
+	"or",
+	"an",
+	"will",
+	"my",
+	"one",
+	"all",
+	"would",
+	"there",
+	"their",
+	"what",
+	"so",
+	"up",
+	"out",
+	"if",
+	"about",
+	"who",
+	"get",
+	"which",
+	"go",
+	"me",
+	"when",
+	"make",
+	"can",
+	"like",
+	"time",
+	"no",
+	"just",
+	"him",
+	"know",
+	"take",
+	"people",
+	"into",
+	"year",
+	"your",
+	"good",
+	"some",
+	"could",
+	"them",
+	"see",
+	"other",
+	"than",
+	"then",
+	"now",
+	"look",
+	"only",
+	"come",
+	"its",
+	"over",
+	"think",
+	"also",
+	"back",
+	"after",
+	"use",
+	"two",
+	"how",
+	"our",
+	"work",
+	"first",
+	"well",
+	"way",
+	"even",
+	"new",
+	"want",
+	"because",
+	"any",
+	"these",
+	"give",
+	"day",
+	"most",
+	"us",
+	"is",
+	"are",
+	"was",
+	"were",
+	"been",
+	"being",
+	"has",
+	"had",
+	"did",
+	"does",
+	"doing",
+	"am",
+	// Code-specific stop words
+	"function",
+	"const",
+	"let",
+	"var",
+	"return",
+	"import",
+	"export",
+	"true",
+	"false",
+	"null",
+	"undefined",
+	"string",
+	"number",
+	"boolean",
+]);
+
+/**
+ * Tokenize text into normalized words.
+ */
+function tokenize(text: string): string[] {
+	return text
+		.toLowerCase()
+		.replace(/[^a-z0-9_\-/.]+/g, " ")
+		.split(/\s+/)
+		.filter((w) => w.length > 2 && !STOP_WORDS.has(w));
+}
+
+/**
+ * Compute term frequency for a document.
+ */
+function computeTF(words: string[]): Map<string, number> {
+	const tf = new Map<string, number>();
+	for (const word of words) {
+		tf.set(word, (tf.get(word) ?? 0) + 1);
+	}
+	// Normalize by total words
+	const total = words.length || 1;
+	for (const [word, count] of tf) {
+		tf.set(word, count / total);
+	}
+	return tf;
+}
+
+/**
+ * Compute inverse document frequency across multiple documents.
+ */
+function computeIDF(documents: string[][]): Map<string, number> {
+	const idf = new Map<string, number>();
+	const N = documents.length || 1;
+
+	// Count documents containing each term
+	const docFreq = new Map<string, number>();
+	for (const words of documents) {
+		const unique = new Set(words);
+		for (const word of unique) {
+			docFreq.set(word, (docFreq.get(word) ?? 0) + 1);
+		}
+	}
+
+	// IDF = log(N / df)
+	for (const [word, df] of docFreq) {
+		idf.set(word, Math.log(N / df));
+	}
+
+	return idf;
+}
+
+/**
+ * Extract top-N TF-IDF terms from a collection of messages.
+ */
+function extractTopTerms(
+	messages: TemporalMessageRow[],
+	topN: number,
+): string[] {
+	// Tokenize each message as a document
+	const documents = messages.map((m) => tokenize(m.content));
+
+	// Compute IDF across all documents
+	const idf = computeIDF(documents);
+
+	// Compute TF-IDF for the merged corpus
+	const allWords = documents.flat();
+	const tf = computeTF(allWords);
+
+	// Score each term
+	const scores: Array<[string, number]> = [];
+	for (const [word, tfScore] of tf) {
+		const idfScore = idf.get(word) ?? 0;
+		scores.push([word, tfScore * idfScore]);
+	}
+
+	// Sort by score descending, return top N
+	scores.sort((a, b) => b[1] - a[1]);
+	return scores.slice(0, topN).map(([term]) => term);
+}
+
+// ============================================================================
+// Key Sentence Selection
+// ============================================================================
+
+/**
+ * Select key sentences from messages based on term density.
+ * Prefers sentences that contain high-value TF-IDF terms.
+ */
+function selectKeySentences(
+	messages: TemporalMessageRow[],
+	topTerms: string[],
+	targetLength: number,
+): string {
+	const termSet = new Set(topTerms);
+
+	// Split all messages into sentences
+	interface ScoredSentence {
+		text: string;
+		score: number;
+		messageIndex: number;
+	}
+
+	const sentences: ScoredSentence[] = [];
+
+	for (let i = 0; i < messages.length; i++) {
+		const msg = messages[i];
+		// Split on sentence boundaries
+		const msgSentences = msg.content
+			.split(/(?<=[.!?])\s+|\n+/)
+			.map((s) => s.trim())
+			.filter((s) => s.length > 10 && s.length < 500);
+
+		for (const sentence of msgSentences) {
+			const words = tokenize(sentence);
+			const termHits = words.filter((w) => termSet.has(w)).length;
+			const density = termHits / (words.length || 1);
+
+			sentences.push({
+				text: sentence,
+				score: density * (1 + termHits), // Boost sentences with more term hits
+				messageIndex: i,
+			});
+		}
+	}
+
+	// Sort by score descending
+	sentences.sort((a, b) => b.score - a.score);
+
+	// Greedy-pack sentences up to target length
+	const selected: ScoredSentence[] = [];
+	let currentLength = 0;
+
+	for (const sentence of sentences) {
+		if (currentLength + sentence.text.length + 2 > targetLength) continue;
+		selected.push(sentence);
+		currentLength += sentence.text.length + 2; // +2 for separator
+	}
+
+	// Re-sort by original message order for coherence
+	selected.sort((a, b) => a.messageIndex - b.messageIndex);
+
+	return selected.map((s) => s.text).join("\n");
+}
+
+// ============================================================================
+// Distillation Pipeline
+// ============================================================================
+
+/**
+ * Run distillation for a session if enough undistilled messages exist.
+ *
+ * Returns the distillation ID if created, or null if no distillation was needed.
+ */
+export function distillSession(sessionId: string): number | null {
+	if (!MEMORY_CONFIG.distillation.enabled) return null;
+
+	const undistilledCount = getUndistilledMessageCount(sessionId);
+
+	if (undistilledCount < MEMORY_CONFIG.distillation.minMessages) {
+		return null; // Not enough messages yet
+	}
+
+	// Get undistilled messages (up to maxMessages)
+	const messages = getUndistilledMessages(
+		sessionId,
+		MEMORY_CONFIG.distillation.maxMessages,
+	);
+
+	if (messages.length < MEMORY_CONFIG.distillation.minMessages) {
+		return null;
+	}
+
+	// Extract top TF-IDF terms
+	const topTerms = extractTopTerms(
+		messages,
+		MEMORY_CONFIG.distillation.topTerms,
+	);
+
+	// Compute target length based on compression ratio
+	const totalChars = messages.reduce((sum, m) => sum + m.content.length, 0);
+	const targetLength = Math.max(
+		200,
+		Math.floor(totalChars * MEMORY_CONFIG.distillation.compressionTarget),
+	);
+
+	// Select key sentences
+	const distilledContent = selectKeySentences(messages, topTerms, targetLength);
+
+	if (!distilledContent || distilledContent.length < 50) {
+		return null; // Distillation too thin
+	}
+
+	// Compute compression ratio
+	const compressionRatio = distilledContent.length / (totalChars || 1);
+
+	// Time range
+	const timeStart = messages[0].time_created;
+	const timeEnd = messages[messages.length - 1].time_created;
+
+	// Store distillation
+	const input: DistillationInput = {
+		session_id: sessionId,
+		content: distilledContent,
+		terms: topTerms,
+		message_count: messages.length,
+		compression_ratio: compressionRatio,
+		time_start: timeStart,
+		time_end: timeEnd,
+	};
+
+	const distillationId = storeDistillation(input);
+
+	// Mark messages as distilled
+	const messageIds = messages.map((m) => m.id);
+	markMessagesDistilled(messageIds, distillationId);
+
+	return distillationId;
+}
+
+// Export term extraction for use by inject.ts
+export { extractTopTerms, tokenize };

package/baseline/plugin/lib/inject.ts

@@ -0,0 +1,126 @@
+/**
+ * LTM Injection Module
+ *
+ * Implements system.transform: searches observations + distillations using
+ * TF-IDF query terms extracted from recent conversation, scores results by
+ * BM25 * recency * confidence, and greedy-packs into a token budget.
+ *
+ * Injected into the system prompt on every turn for relevant context.
+ */
+
+import { tokenize } from "./distill.js";
+import {
+	estimateTokens,
+	getRelevantKnowledge,
+	MEMORY_CONFIG,
+} from "./memory-db.js";
+
+// ============================================================================
+// Query Term Extraction
+// ============================================================================
+
+/**
+ * Extract top query terms from the current system prompt context.
+ * Uses TF-IDF-like approach on the existing system prompt to find
+ * what the conversation is about.
+ */
+export function extractQueryTerms(
+	systemPrompt: string[],
+	topN?: number,
+): string[] {
+	const n = topN ?? MEMORY_CONFIG.injection.topTerms;
+
+	// Combine all system prompt segments
+	const fullText = systemPrompt.join("\n");
+
+	// Tokenize
+	const words = tokenize(fullText);
+	if (words.length === 0) return [];
+
+	// Compute term frequency
+	const tf = new Map<string, number>();
+	for (const word of words) {
+		tf.set(word, (tf.get(word) ?? 0) + 1);
+	}
+
+	// Sort by frequency descending
+	const sorted = [...tf.entries()].sort((a, b) => b[1] - a[1]);
+
+	// Return top N terms
+	return sorted.slice(0, n).map(([term]) => term);
+}
+
+// ============================================================================
+// Injection
+// ============================================================================
+
+/**
+ * Build the LTM injection block for system.transform.
+ *
+ * @param systemPrompt - Current system prompt segments
+ * @returns Additional system prompt segment with relevant knowledge, or null if empty
+ */
+export function buildInjection(systemPrompt: string[]): string | null {
+	if (!MEMORY_CONFIG.injection.enabled) return null;
+
+	// Extract query terms from current system context
+	const queryTerms = extractQueryTerms(systemPrompt);
+	if (queryTerms.length === 0) return null;
+
+	// Get relevant knowledge, scored and packed within budget
+	const knowledge = getRelevantKnowledge(queryTerms, {
+		tokenBudget: MEMORY_CONFIG.injection.tokenBudget,
+		minScore: MEMORY_CONFIG.injection.minScore,
+	});
+
+	if (knowledge.length === 0) return null;
+
+	// Format as injection block
+	const lines: string[] = [
+		"<memory_context>",
+		"Relevant knowledge from previous sessions:",
+		"",
+	];
+
+	for (const item of knowledge) {
+		const sourceTag =
+			item.source === "observation" ? `[${item.type}]` : "[distillation]";
+		const scoreTag = `(relevance: ${item.score.toFixed(2)})`;
+
+		lines.push(`### ${sourceTag} ${item.title} ${scoreTag}`);
+
+		// Truncate content to avoid dominating the injection
+		const maxContentChars = Math.floor(
+			(MEMORY_CONFIG.injection.tokenBudget * 4) / knowledge.length,
+		);
+		const content =
+			item.content.length > maxContentChars
+				? `${item.content.slice(0, maxContentChars)}...`
+				: item.content;
+
+		lines.push(content);
+		lines.push("");
+	}
+
+	lines.push("</memory_context>");
+
+	const injection = lines.join("\n");
+
+	// Final check: ensure we're within token budget
+	const tokens = estimateTokens(injection);
+	if (tokens > MEMORY_CONFIG.injection.tokenBudget * 1.2) {
+		// Over budget — trim to just titles
+		const trimmed = [
+			"<memory_context>",
+			"Relevant knowledge (summaries):",
+			...knowledge.map(
+				(k) =>
+					`- [${k.source === "observation" ? k.type : "distillation"}] ${k.title}`,
+			),
+			"</memory_context>",
+		].join("\n");
+		return trimmed;
+	}
+
+	return injection;
+}

package/baseline/plugin/lib/memory-admin-tools.ts

@@ -0,0 +1,188 @@
+/**
+ * Memory Plugin — Admin Tools
+ *
+ * memory-admin (9 operations).
+ *
+ * Uses factory pattern: createAdminTools(deps) returns tool definitions.
+ */
+
+import { readdir, readFile } from "node:fs/promises";
+import path from "node:path";
+import { tool } from "@opencode-ai/plugin/tool";
+import { curateFromDistillations } from "./curator.js";
+import { distillSession } from "./distill.js";
+import {
+	archiveOldObservations,
+	type ConfidenceLevel,
+	checkFTS5Available,
+	checkpointWAL,
+	getCaptureStats,
+	getDatabaseSizes,
+	getDistillationStats,
+	getMarkdownFilesInSqlite,
+	getObservationStats,
+	type ObservationType,
+	rebuildFTS5,
+	runFullMaintenance,
+	storeObservation,
+	vacuumDatabase,
+} from "./memory-db.js";
+
+interface AdminToolDeps {
+	directory: string;
+}
+
+export function createAdminTools(deps: AdminToolDeps) {
+	const { directory } = deps;
+
+	return {
+		"memory-admin": tool({
+			description: `Memory system administration: maintenance and migration.\n\nOperations:\n- "status": Storage stats and recommendations\n- "full": Full maintenance cycle (archive + checkpoint + vacuum)\n- "archive": Archive old observations (>90 days default)\n- "checkpoint": Checkpoint WAL file\n- "vacuum": Vacuum database\n- "migrate": Import .opencode/memory/observations/*.md into SQLite\n- "capture-stats": Temporal message capture statistics\n- "distill-now": Force distillation for current session\n- "curate-now": Force curator run\n\nExample:\nmemory-admin({ operation: "status" })\nmemory-admin({ operation: "migrate", dry_run: true })`,
+			args: {
+				operation: tool.schema
+					.string()
+					.optional()
+					.describe("Operation (default: status)"),
+				older_than_days: tool.schema
+					.number()
+					.optional()
+					.describe("Archive threshold (default: 90)"),
+				dry_run: tool.schema
+					.boolean()
+					.optional()
+					.describe("Preview without executing"),
+				force: tool.schema.boolean().optional().describe("Force re-migration"),
+			},
+			execute: async (args, ctx) => {
+				const op = args.operation ?? "status";
+				const dryRun = args.dry_run ?? false;
+				const olderThanDays = args.older_than_days ?? 90;
+
+				switch (op) {
+					case "status": {
+						const sizes = getDatabaseSizes();
+						const stats = getObservationStats();
+						const archivable = archiveOldObservations({
+							olderThanDays,
+							dryRun: true,
+						});
+						const captureStats = getCaptureStats();
+						const distillStats = getDistillationStats();
+						return [
+							"## Memory System Status\n",
+							`**Database**: ${(sizes.total / 1024).toFixed(1)} KB`,
+							`**FTS5**: ${checkFTS5Available() ? "Available (porter stemming)" : "Unavailable"}`,
+							`**Schema**: v2 (4-tier storage)\n`,
+							"### Observations",
+							...Object.entries(stats).map(([k, v]) => `  ${k}: ${v}`),
+							`  Archivable (>${olderThanDays}d): ${archivable}\n`,
+							"### Capture Pipeline",
+							`  Messages: ${captureStats.total} (undistilled: ${captureStats.undistilled})`,
+							`  Sessions: ${captureStats.sessions}\n`,
+							"### Distillations",
+							`  Total: ${distillStats.total} (${distillStats.sessions} sessions)`,
+							`  Avg compression: ${(distillStats.avgCompression * 100).toFixed(1)}%`,
+						].join("\n");
+					}
+					case "full": {
+						if (dryRun)
+							return `Dry run: would archive, purge, optimize, checkpoint, vacuum.`;
+						const r = runFullMaintenance({
+							olderThanDays,
+							includeSuperseded: true,
+						});
+						return `Done: archived ${r.archived}, purged ${r.purgedMessages} msgs, freed ${(r.freedBytes / 1024).toFixed(1)} KB.`;
+					}
+					case "archive": {
+						const c = archiveOldObservations({
+							olderThanDays,
+							includeSuperseded: true,
+							dryRun,
+						});
+						return dryRun
+							? `Would archive ${c} observations.`
+							: `Archived ${c} observations.`;
+					}
+					case "checkpoint": {
+						const r = checkpointWAL();
+						return r.checkpointed
+							? `WAL checkpointed (${r.walSize} pages).`
+							: "Checkpoint failed or busy.";
+					}
+					case "vacuum":
+						return vacuumDatabase() ? "Vacuumed." : "Vacuum failed.";
+					case "capture-stats":
+						return JSON.stringify(getCaptureStats(), null, 2);
+					case "distill-now": {
+						const sid = ctx?.sessionID;
+						if (!sid) return "Error: No session ID.";
+						const did = distillSession(sid);
+						return did
+							? `Distillation #${did} created.`
+							: "Not enough undistilled messages.";
+					}
+					case "curate-now": {
+						const r = curateFromDistillations();
+						return `Created ${r.created}, skipped ${r.skipped}. Patterns: ${JSON.stringify(r.patterns)}`;
+					}
+					case "migrate": {
+						const obsDir = path.join(
+							directory,
+							".opencode",
+							"memory",
+							"observations",
+						);
+						let mdFiles: string[] = [];
+						try {
+							mdFiles = (await readdir(obsDir)).filter((f) =>
+								f.endsWith(".md"),
+							);
+						} catch {
+							return "No observations directory found.";
+						}
+						if (mdFiles.length === 0) return "No files to migrate.";
+						const existing = new Set(getMarkdownFilesInSqlite());
+						const toMigrate = args.force
+							? mdFiles
+							: mdFiles.filter((f) => !existing.has(f));
+						if (toMigrate.length === 0) return "All files already migrated.";
+						if (dryRun) return `Would migrate ${toMigrate.length} files.`;
+						let migrated = 0;
+						for (const file of toMigrate) {
+							try {
+								const content = await readFile(
+									path.join(obsDir, file),
+									"utf-8",
+								);
+								const fmMatch = content.match(
+									/^---\n([\s\S]*?)\n---\n([\s\S]*)$/,
+								);
+								const body = fmMatch ? fmMatch[2].trim() : content.trim();
+								const fm = fmMatch ? fmMatch[1] : "";
+								storeObservation({
+									type: (fm.match(/type:\s*(\w+)/)?.[1] ??
+										"discovery") as ObservationType,
+									title:
+										fm.match(/title:\s*(.+)/)?.[1]?.trim() ??
+										file.replace(/\.md$/, ""),
+									narrative: body,
+									confidence: (fm.match(/confidence:\s*(\w+)/)?.[1] ??
+										"medium") as ConfidenceLevel,
+									markdown_file: file,
+									source: "imported",
+								});
+								migrated++;
+							} catch {
+								/* Skip failed files */
+							}
+						}
+						if (migrated > 0) rebuildFTS5();
+						return `Migrated ${migrated}/${toMigrate.length} files.`;
+					}
+					default:
+						return `Unknown operation: "${op}".`;
+				}
+			},
+		}),
+	};
+}