amalfa 1.0.37 → 1.0.38

package/README.md

@@ -244,7 +244,13 @@ Agents generate knowledge through structured reflection. Amalfa provides semanti
 - [ ] Git-based auditing for augmentations
 - [ ] Automated file watcher updates
 
-### 📋 Phase 2: Latent Space Organization (Planned)
+### 🚧 Phase 2: Ember Service (Automated Enrichment)
+- ✅ **Analyzer** - Louvain community detection & heuristics
+- ✅ **Sidecar Generator** - Safe proposal mechanism (`.ember.json`)
+- ✅ **Squasher** - Robust metadata merging (preserves user content)
+- ✅ **CLI** - `amalfa ember scan/squash` commands
+
+### 📋 Phase 3: Latent Space Organization (Planned)
 
 - [ ] Document clustering (HDBSCAN)
 - [ ] Cluster label generation

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "amalfa",
-	"version": "1.0.37",
+	"version": "1.0.38",
 	"description": "Local-first knowledge graph engine for AI agents. Transforms markdown into searchable memory with MCP protocol.",
 	"license": "MIT",
 	"homepage": "https://github.com/pjsvis/amalfa#readme",
@@ -69,6 +69,7 @@
 		"fastembed": "2.0.0",
 		"graphology": "0.26.0",
 		"graphology-library": "0.8.0",
+		"gray-matter": "^4.0.3",
 		"hono": "4.11.3",
 		"pino": "10.1.0"
 	}

package/src/cli.ts

@@ -50,6 +50,7 @@ Commands:
   daemon <action>    Manage file watcher (start|stop|status|restart)
   vector <action>    Manage vector daemon (start|stop|status|restart)
   sonar <action>     Manage Sonar AI agent (start|stop|status|restart)
+  ember <action>     Manage Ember enrichment service (scan|squash)
   scripts list       List available scripts and their descriptions
   servers [--dot]    Show status of all AMALFA services (--dot for graph)
 
@@ -821,6 +822,67 @@ async function cmdValidate() {
 	}
 }
 
+async function cmdEmber() {
+	const rawAction = args[1] || "help";
+	const action =
+		rawAction === "--help" || rawAction === "-h" ? "help" : rawAction;
+
+	if (action === "help") {
+		console.log(`
+EMBER - Automated Enrichment Service
+
+Usage:
+  amalfa ember scan [--dry-run]   Analyze files and generate sidecars
+  amalfa ember squash             Merge sidecars into markdown files
+  amalfa ember status             Show pending sidecars (TODO)
+`);
+		return;
+	}
+
+	const { ResonanceDB } = await import("./resonance/db");
+	const { EmberService } = await import("./ember/index");
+	const { loadConfig } = await import("./config/defaults");
+
+	// Check DB
+	const dbPath = await getDbPath();
+	if (!existsSync(dbPath)) {
+		console.error("❌ Database not found. Run 'amalfa init' first.");
+		process.exit(1);
+	}
+
+	const db = new ResonanceDB(dbPath);
+	const appConfig = await loadConfig();
+
+	const emberConfig = {
+		enabled: true,
+		sources: appConfig.sources || ["./docs"],
+		minConfidence: 0.7,
+		backupDir: ".amalfa/backups",
+		excludePatterns: appConfig.excludePatterns || [],
+	};
+
+	const ember = new EmberService(db, emberConfig);
+
+	try {
+		if (action === "scan") {
+			const dryRun = args.includes("--dry-run");
+			await ember.runFullSweep(dryRun);
+		} else if (action === "squash") {
+			await ember.squashAll();
+		} else if (action === "status") {
+			console.log("Checking pending sidecars... (Not yet implemented)");
+		} else {
+			console.error(`❌ Unknown action: ${action}`);
+			process.exit(1);
+		}
+	} catch (e) {
+		console.error("❌ Ember command failed:", e);
+		process.exit(1);
+	} finally {
+		db.close();
+	}
+}
+
 async function cmdDoctor() {
 	console.log("🩺 AMALFA Health Check\n");
 
@@ -944,6 +1006,10 @@ async function main() {
 			await cmdSonar();
 			break;
 
+		case "ember":
+			await cmdEmber();
+			break;
+
 		case "scripts":
 			await cmdScripts();
 			break;

package/src/ember/README.md

@@ -0,0 +1,23 @@
+
+# Ember Service
+
+Automated enrichment service for the Amalfa Knowledge Graph.
+
+## Stability Clause
+
+> **Warning**
+> This module is responsible for modifying user data (markdown files). 
+> 
+> *   **Do not modify** `squasher.ts` without explicit regression testing.
+> *   **Do not change** the sidecar format without updating `types.ts` and `generator.ts`.
+> *   **Always use** `safe-dump` equivalent (e.g., `gray-matter`) when writing back files.
+
+## Architecture
+
+- **Analyzer**: Scans graph for enrichment opportunities.
+- **Generator**: Writes changes to `.ember.json` sidecar files.
+- **Squasher**: Merges sidecars into `.md` files safely.
+
+## Usage
+
+Included in the main Amalfa daemon. Can be triggered via CLI.

package/src/ember/analyzer.ts

@@ -0,0 +1,120 @@
+import { GraphEngine } from "@src/core/GraphEngine";
+import type { ResonanceDB } from "@src/resonance/db";
+import { getLogger } from "@src/utils/Logger";
+import type { EmberSidecar } from "./types";
+
+export class EmberAnalyzer {
+	private log = getLogger("EmberAnalyzer");
+	private graphEngine: GraphEngine;
+	private communities: Record<string, number> | null = null;
+	private isGraphLoaded = false;
+
+	constructor(private db: ResonanceDB) {
+		this.graphEngine = new GraphEngine();
+	}
+
+	/**
+	 * Pre-load graph data for batch analysis
+	 */
+	async prepare() {
+		this.log.info("Loading graph engine for analysis...");
+		await this.graphEngine.load(this.db.getRawDb());
+		this.communities = this.graphEngine.detectCommunities();
+		this.isGraphLoaded = true;
+		this.log.info("Graph engine ready.");
+	}
+
+	/**
+	 * Analyze a file and generate enrichment proposals
+	 */
+	async analyze(
+		filePath: string,
+		content: string,
+	): Promise<EmberSidecar | null> {
+		this.log.info(`Analyzing ${filePath}...`);
+
+		// Lazy load if not ready
+		if (!this.isGraphLoaded) {
+			await this.prepare();
+		}
+
+		// 1. Identify Node in Graph
+		const filename = filePath.split("/").pop() || "unknown";
+		const id = filename
+			.replace(/\.(md|ts|js)$/, "")
+			.toLowerCase()
+			.replace(/[^a-z0-9-]/g, "-");
+
+		const node = this.db.getNode(id);
+		if (!node) {
+			this.log.warn(`Node ${id} not found in graph. Skipping analysis.`);
+			return null;
+		}
+
+		const proposedTags: string[] = [];
+		const proposedLinks: string[] = [];
+
+		// 2. Community-based Tag Suggestion
+		if (this.communities && this.communities[id] !== undefined) {
+			const communityId = this.communities[id];
+			const communityNodes = Object.entries(this.communities)
+				.filter(([_, comm]) => comm === communityId)
+				.map(([nId]) => nId);
+
+			// Only analyze if community is large enough
+			if (communityNodes.length > 2) {
+				const tagFreq = new Map<string, number>();
+				let neighborCount = 0;
+
+				// Analyze neighbors specifically (stronger signal than whole community)
+				const neighbors = this.graphEngine.getNeighbors(id);
+
+				for (const neighborId of neighbors) {
+					const neighbor = this.db.getNode(neighborId);
+					const nTags = (neighbor?.meta?.tags as string[]) || [];
+
+					for (const tag of nTags) {
+						tagFreq.set(tag, (tagFreq.get(tag) || 0) + 1);
+					}
+					neighborCount++;
+				}
+
+				// Suggest tags present in > 50% of neighbors
+				if (neighborCount > 0) {
+					for (const [tag, count] of tagFreq.entries()) {
+						if (count / neighborCount >= 0.5) {
+							const currentTags = (node.meta?.tags as string[]) || [];
+							if (!currentTags.includes(tag) && !proposedTags.includes(tag)) {
+								proposedTags.push(tag);
+							}
+						}
+					}
+				}
+			}
+		}
+
+		// 3. Heuristics (Stub detection)
+		const tags = (node.meta?.tags as string[]) || [];
+		if (content.length < 100 && !tags.includes("stub")) {
+			proposedTags.push("stub");
+		}
+
+		// If no meaningful changes, return null
+		if (proposedTags.length === 0 && proposedLinks.length === 0) {
+			return null;
+		}
+
+		// 4. Construct Sidecar
+		const sidecar: EmberSidecar = {
+			targetFile: filePath,
+			generatedAt: new Date().toISOString(),
+			confidence: 0.8,
+			changes: {
+				tags: proposedTags.length > 0 ? { add: proposedTags } : undefined,
+				links: proposedLinks.length > 0 ? { add: proposedLinks } : undefined,
+			},
+		};
+
+		return sidecar;
+	}
+}

package/src/ember/generator.ts

@@ -0,0 +1,25 @@
+import { getLogger } from "@src/utils/Logger";
+import type { EmberSidecar } from "./types";
+
+export class EmberGenerator {
+	private log = getLogger("EmberGenerator");
+
+	/**
+	 * Write the sidecar file to disk
+	 */
+	async generate(sidecar: EmberSidecar): Promise<string> {
+		const sidecarPath = `${sidecar.targetFile}.ember.json`;
+
+		try {
+			await Bun.write(sidecarPath, JSON.stringify(sidecar, null, 2));
+			this.log.info(`Generated sidecar: ${sidecarPath}`);
+			return sidecarPath;
+		} catch (error) {
+			this.log.error(
+				{ err: error, file: sidecarPath },
+				"Failed to write sidecar",
+			);
+			throw error;
+		}
+	}
+}

package/src/ember/index.ts

@@ -0,0 +1,106 @@
+import { join } from "node:path";
+import type { ResonanceDB } from "@src/resonance/db";
+import { getLogger } from "@src/utils/Logger";
+import { Glob } from "bun";
+import { EmberAnalyzer } from "./analyzer";
+import { EmberGenerator } from "./generator";
+import { EmberSquasher } from "./squasher";
+import type { EmberConfig } from "./types";
+
+export class EmberService {
+	private analyzer: EmberAnalyzer;
+	private generator: EmberGenerator;
+	private squasher: EmberSquasher;
+	private log = getLogger("EmberService");
+
+	constructor(
+		db: ResonanceDB,
+		private config: EmberConfig,
+	) {
+		this.analyzer = new EmberAnalyzer(db);
+		this.generator = new EmberGenerator();
+		this.squasher = new EmberSquasher();
+	}
+
+	/**
+	 * Run a full sweep of all configured sources
+	 */
+	async runFullSweep(dryRun = false) {
+		this.log.info("Starting full Ember sweep...");
+
+		const files = await this.discoverFiles();
+		let enrichedCount = 0;
+
+		for (const file of files) {
+			const content = await Bun.file(file).text();
+			const sidecar = await this.analyzer.analyze(file, content);
+
+			if (sidecar) {
+				if (dryRun) {
+					this.log.info(`[Dry Run] Would generate sidecar for ${file}`);
+					console.log(JSON.stringify(sidecar, null, 2));
+				} else {
+					await this.generator.generate(sidecar);
+					enrichedCount++;
+				}
+			}
+		}
+
+		this.log.info(`Sweep complete. Enriched ${enrichedCount} files.`);
+		return enrichedCount;
+	}
+
+	/**
+	 * Squash all pending sidecars
+	 */
+	async squashAll() {
+		this.log.info("Squashing all pending sidecars...");
+		let count = 0;
+
+		// Simpler scan:
+		const sidecars = await this.findSidecars();
+		for (const sidecarPath of sidecars) {
+			await this.squasher.squash(sidecarPath);
+			count++;
+		}
+
+		this.log.info(`Squashed ${count} sidecars.`);
+		return count;
+	}
+
+	private async findSidecars(): Promise<string[]> {
+		const sidecars: string[] = [];
+		const glob = new Glob("**/*.ember.json");
+		// Scan sources
+		for (const source of this.config.sources) {
+			// Assuming source is like "./docs"
+			const sourcePath = join(process.cwd(), source);
+			for (const file of glob.scanSync({ cwd: sourcePath })) {
+				sidecars.push(join(sourcePath, file));
+			}
+		}
+		return sidecars;
+	}
+
+	private async discoverFiles(): Promise<string[]> {
+		const files: string[] = [];
+		const glob = new Glob("**/*.{md,mdx}"); // Only markdown for now
+
+		for (const source of this.config.sources) {
+			const sourcePath = join(process.cwd(), source);
+			try {
+				for (const file of glob.scanSync({ cwd: sourcePath })) {
+					const shouldExclude = this.config.excludePatterns.some((p) =>
+						file.includes(p),
+					);
+					if (!shouldExclude) {
+						files.push(join(sourcePath, file));
+					}
+				}
+			} catch (e) {
+				this.log.warn({ source: sourcePath, err: e }, "Failed to scan source");
+			}
+		}
+		return files;
+	}
+}

package/src/ember/squasher.ts

@@ -0,0 +1,71 @@
+import { unlink } from "node:fs/promises";
+import { getLogger } from "@src/utils/Logger";
+import matter from "gray-matter";
+import type { EmberSidecar } from "./types";
+
+export class EmberSquasher {
+	private log = getLogger("EmberSquasher");
+
+	/**
+	 * Apply the sidecar changes to the target file
+	 */
+	async squash(sidecarPath: string): Promise<void> {
+		try {
+			// 1. Read Sidecar
+			const sidecarContent = await Bun.file(sidecarPath).text();
+			const sidecar: EmberSidecar = JSON.parse(sidecarContent);
+
+			const targetPath = sidecar.targetFile;
+
+			// 2. Read Target File
+			const fileContent = await Bun.file(targetPath).text();
+
+			// 3. Parse with gray-matter
+			const parsed = matter(fileContent);
+			const data = parsed.data || {};
+
+			// 4. Apply Changes
+			if (sidecar.changes.tags) {
+				const currentTags = (
+					Array.isArray(data.tags) ? data.tags : []
+				) as string[];
+				const toAdd = sidecar.changes.tags.add || [];
+				const toRemove = sidecar.changes.tags.remove || [];
+
+				const newTags = new Set(currentTags);
+				for (const t of toAdd) {
+					newTags.add(t);
+				}
+				for (const t of toRemove) {
+					newTags.delete(t);
+				}
+
+				data.tags = Array.from(newTags);
+			}
+
+			if (sidecar.changes.frontmatter) {
+				Object.assign(data, sidecar.changes.frontmatter);
+			}
+
+			if (sidecar.changes.summary) {
+				data.summary = sidecar.changes.summary;
+			}
+
+			// 5. Reconstruct File
+			const newContent = matter.stringify(parsed.content, data);
+
+			// 6. Write Back
+			await Bun.write(targetPath, newContent);
+			this.log.info(`Squashed sidecar into ${targetPath}`);
+
+			// 7. Cleanup Sidecar
+			await unlink(sidecarPath);
+		} catch (error) {
+			this.log.error(
+				{ err: error, file: sidecarPath },
+				"Failed to squash sidecar",
+			);
+			throw error;
+		}
+	}
+}

package/src/ember/types.ts

@@ -0,0 +1,26 @@
+export interface EmberSidecar {
+	targetFile: string;
+	generatedAt: string;
+	confidence: number;
+	changes: {
+		tags?: {
+			add: string[];
+			remove?: string[];
+		};
+		frontmatter?: Record<string, unknown>;
+		summary?: string;
+		links?: {
+			add: string[]; // List of IDs or Titles to add to 'related'
+		};
+	};
+}
+
+export interface EmberConfig {
+	enabled: boolean;
+	sources: string[];
+	minConfidence: number;
+	backupDir: string;
+	excludePatterns: string[];
+}
+
+export type EnrichmentType = "tag" | "link" | "summary" | "metadata";

package/src/pipeline/AmalfaIngestor.ts

@@ -12,6 +12,7 @@ import { Embedder } from "@src/resonance/services/embedder";
 import { SimpleTokenizerService as TokenizerService } from "@src/resonance/services/simpleTokenizer";
 import { getLogger } from "@src/utils/Logger";
 import { Glob } from "bun";
+import matter from "gray-matter";
 
 export interface IngestionResult {
 	success: boolean;
@@ -236,11 +237,12 @@ export class AmalfaIngestor {
 		tokenizer: TokenizerService,
 	): Promise<void> {
 		try {
-			const content = await Bun.file(filePath).text();
+			const rawContent = await Bun.file(filePath).text();
 
-			// Parse frontmatter
-			const fmMatch = content.match(/^---\n([\s\S]*?)\n---/);
-			const frontmatter = fmMatch?.[1] ? this.parseFrontmatter(fmMatch[1]) : {};
+			// Parse frontmatter with gray-matter
+			const parsed = matter(rawContent);
+			const frontmatter = parsed.data || {};
+			const content = parsed.content;
 
 			// Generate ID from filename
 			const filename = filePath.split("/").pop() || "unknown";
@@ -251,7 +253,7 @@ export class AmalfaIngestor {
 
 			// Skip if content unchanged (hash check)
 			const hasher = new Bun.CryptoHasher("md5");
-			hasher.update(content.trim());
+			hasher.update(rawContent.trim());
 			const currentHash = hasher.digest("hex");
 			const storedHash = this.db.getNodeHash(id);
 
@@ -268,6 +270,8 @@ export class AmalfaIngestor {
 			// Extract semantic tokens
 			const tokens = tokenizer.extract(content);
 
+			// Insert node
+
 			// Insert node
 			const node: Node = {
 				id,
@@ -295,18 +299,4 @@ export class AmalfaIngestor {
 			this.log.warn({ err: e, file: filePath }, "⚠️  Failed to process file");
 		}
 	}
-
-	/**
-	 * Parse YAML-like frontmatter
-	 */
-	private parseFrontmatter(text: string): Record<string, unknown> {
-		const meta: Record<string, unknown> = {};
-		text.split("\n").forEach((line) => {
-			const [key, ...vals] = line.split(":");
-			if (key && vals.length) {
-				meta[key.trim()] = vals.join(":").trim();
-			}
-		});
-		return meta;
-	}
 }