amalfa 0.0.0-reserved → 1.0.0

package/src/resonance/daemon.ts

@@ -0,0 +1,236 @@
+import { watch } from "node:fs";
+import { join } from "node:path";
+import settings from "@/polyvis.settings.json";
+import { Ingestor } from "../pipeline/Ingestor";
+import { EnvironmentVerifier } from "../utils/EnvironmentVerifier";
+import { getLogger } from "../utils/Logger";
+import { ServiceLifecycle } from "../utils/ServiceLifecycle";
+import { Embedder } from "./services/embedder";
+
+const args = process.argv.slice(2);
+const command = args[0] || "serve";
+const log = getLogger("Daemon");
+
+// --- Helper: Notifications ---
+
+async function notify(title: string, message: string) {
+	// Native macOS notifications via AppleScript
+	// Zero dependencies
+	try {
+		const script = `display notification "${message}" with title "${title}"`;
+		await Bun.spawn(["osascript", "-e", script]);
+	} catch (e) {
+		log.error({ err: e }, "Failed to send notification");
+	}
+}
+
+// --- Service Lifecycle ---
+
+const lifecycle = new ServiceLifecycle({
+	name: "Daemon",
+	pidFile: ".daemon.pid",
+	logFile: ".daemon.log",
+	entryPoint: "src/resonance/daemon.ts",
+});
+
+// --- Server Logic (The actual Daemon) ---
+
+async function main() {
+	// 0. Verify Environment
+	await EnvironmentVerifier.verifyOrExit();
+
+	// 1. Initialize Ingestion (Daemon Mode: Watch Enabled)
+	const PORT = parseInt(process.env.VECTOR_PORT || "3010", 10);
+
+	log.info({ port: PORT }, "🔌 Vector Daemon starting...");
+	log.info("Initializing Embedder...");
+
+	// 1. Initialize Embedder (Compute Node)
+	try {
+		const embedder = Embedder.getInstance();
+		await embedder.embed("warmup", true);
+		log.info("✅ Embedder Ready.");
+	} catch (e) {
+		log.fatal({ err: e }, "❌ Failed to initialize embedder");
+		process.exit(1);
+	}
+
+	// 2. Start HTTP Server
+	Bun.serve({
+		port: PORT,
+		async fetch(req) {
+			const url = new URL(req.url);
+
+			if (req.method === "GET" && url.pathname === "/health") {
+				return new Response(JSON.stringify({ status: "ok" }), {
+					headers: { "Content-Type": "application/json" },
+				});
+			}
+
+			if (req.method === "POST" && url.pathname === "/embed") {
+				try {
+					const body = (await req.json()) as { text: string };
+					if (!body.text || typeof body.text !== "string") {
+						return new Response("Bad Request: 'text' field required", {
+							status: 400,
+						});
+					}
+
+					const vector = await Embedder.getInstance().embed(body.text, true);
+
+					return new Response(JSON.stringify({ vector: Array.from(vector) }), {
+						headers: { "Content-Type": "application/json" },
+					});
+				} catch (e) {
+					log.error({ err: e }, "Embedder API Error");
+					return new Response("Internal Server Error", { status: 500 });
+				}
+			}
+
+			return new Response("Not Found", { status: 404 });
+		},
+	});
+
+	log.info(`🚀 Vector Daemon listening on http://localhost:${PORT}`);
+
+	// 3. Start The Watcher (Active Custodian)
+	startWatcher();
+
+	// Handle cleanup
+	process.on("SIGTERM", () => {
+		log.info("🛑 Received SIGTERM, shutting down...");
+		process.exit(0);
+	});
+}
+
+// --- Watcher Logic ---
+
+let debounceTimer: ReturnType<typeof setTimeout> | null = null;
+const DEBOUNCE_MS = 2000;
+const pendingFiles = new Set<string>();
+
+// Retry queue: Track failed ingestions with attempt counts
+const retryQueue = new Map<
+	string,
+	{ attempts: number; lastError: string; lastAttempt: number }
+>();
+const MAX_RETRIES = 3;
+const RETRY_BACKOFF_MS = 5000; // Wait 5 seconds before retry
+
+function startWatcher() {
+	// Dynamically load watch targets from settings
+	const rawSources = settings.paths.sources.experience;
+	const dirsToWatch = rawSources.map((s) => s.path);
+
+	log.info({ triggers: dirsToWatch }, "👀 Watching directories");
+
+	dirsToWatch.forEach((dir) => {
+		const path = join(process.cwd(), dir);
+		try {
+			watch(path, { recursive: true }, (event, filename) => {
+				// Ignore dotfiles and ensure markdown
+				if (filename && !filename.startsWith(".") && filename.endsWith(".md")) {
+					log.debug(
+						{ file: `${dir}/${filename}`, event },
+						"📝 Change detected",
+					);
+
+					// Add full path to pending set
+					const fullPath = join(process.cwd(), dir, filename);
+					pendingFiles.add(fullPath);
+
+					triggerIngestion();
+				}
+			});
+		} catch (e) {
+			log.warn({ dir, err: e }, "⚠️ Could not watch directory");
+		}
+	});
+}
+
+function triggerIngestion() {
+	if (debounceTimer) {
+		clearTimeout(debounceTimer);
+	}
+
+	debounceTimer = setTimeout(async () => {
+		const batchSize = pendingFiles.size;
+		if (batchSize === 0) return;
+
+		log.info({ batchSize }, "🔄 Debounce settle. Starting Batch Ingestion...");
+
+		// Drain the set
+		const batch = Array.from(pendingFiles);
+		pendingFiles.clear();
+
+		try {
+			// Re-instantiate DB/Ingestor for fresh context
+			const ingestor = new Ingestor();
+
+			// OPTIMIZATION: Pass only the changed files
+			await ingestor.run({ files: batch });
+
+			log.info("✅ Batch Ingestion Complete.");
+			// Clear retry counts for successful files
+			for (const file of batch) {
+				retryQueue.delete(file);
+			}
+			await notify("PolyVis Resonance", `Graph Updated (${batchSize} files).`);
+		} catch (e) {
+			const errorMsg = e instanceof Error ? e.message : String(e);
+			log.error({ err: e }, "❌ Ingestion Failed");
+
+			// Re-queue failed files with retry logic
+			const now = Date.now();
+			for (const file of batch) {
+				const retryInfo = retryQueue.get(file) || {
+					attempts: 0,
+					lastError: "",
+					lastAttempt: 0,
+				};
+
+				if (retryInfo.attempts < MAX_RETRIES) {
+					// Re-queue with exponential backoff
+					const nextAttempt = retryInfo.attempts + 1;
+					retryQueue.set(file, {
+						attempts: nextAttempt,
+						lastError: errorMsg,
+						lastAttempt: now,
+					});
+
+					// Re-add to pending files after backoff delay
+					setTimeout(() => {
+						pendingFiles.add(file);
+						triggerIngestion();
+					}, RETRY_BACKOFF_MS * nextAttempt);
+
+					log.warn(
+						{
+							file,
+							attempt: nextAttempt,
+							max: MAX_RETRIES,
+							delayMs: RETRY_BACKOFF_MS * nextAttempt,
+						},
+						"🔄 Scheduling Retry",
+					);
+				} else {
+					// Abandon after max retries
+					log.error(
+						{ file, lastError: retryInfo.lastError },
+						"⛔ ABANDONED: File failed max retries",
+					);
+					retryQueue.delete(file); // Remove from tracking
+				}
+			}
+
+			await notify(
+				"PolyVis Resonance",
+				`Ingestion Failed (${batch.length} files will retry)`,
+			);
+		}
+	}, DEBOUNCE_MS);
+}
+
+// --- Dispatch ---
+
+await lifecycle.run(command, main);

package/src/resonance/db.ts

@@ -0,0 +1,422 @@
+import type { Database } from "bun:sqlite";
+import { getLogger } from "@src/utils/Logger";
+import settings from "@/polyvis.settings.json";
+import { DatabaseFactory } from "./DatabaseFactory";
+import { CURRENT_SCHEMA_VERSION, MIGRATIONS } from "./schema";
+
+const log = getLogger("ResonanceDB");
+
+// Types matching Schema
+export interface Node {
+	id: string;
+	type: string;
+	label?: string; // stored as 'title'
+	content?: string;
+	domain?: string;
+	layer?: string;
+	embedding?: Float32Array;
+	hash?: string;
+	meta?: Record<string, unknown>; // JSON object for flexible metadata
+}
+
+export class ResonanceDB {
+	private db: Database;
+	// biome-ignore lint/correctness/noUnusedPrivateClassMembers: May be used for debugging/diagnostics
+	private dbPath: string;
+
+	/**
+	 * Factory method to load the default Resonance Graph based on settings.
+	 */
+	static init(): ResonanceDB {
+		return new ResonanceDB(settings.paths.database.resonance);
+	}
+
+	/**
+	 * @param dbPath - Absolute path to the SQLite database file
+	 *
+	 * Note: ResonanceDB always opens in read-write mode regardless of any options.
+	 * WAL mode requires write access to the -shm (shared memory) file even for readers.
+	 */
+	constructor(dbPath: string) {
+		this.dbPath = dbPath;
+		// Use DatabaseFactory to ensure compliant configuration (WAL mode + timeouts)
+		// Always read-write: WAL mode requires all connections to have write access to -shm file
+		this.db = DatabaseFactory.connect(dbPath, { readonly: false });
+
+		// Always check migration (it's safe now with locking)
+		this.migrate();
+	}
+
+	private migrate() {
+		const row = this.db.query("PRAGMA user_version").get() as {
+			user_version: number;
+		};
+		let currentVersion = row.user_version;
+
+		// Backward Compatibility for existing unversioned DBs
+		if (currentVersion === 0) {
+			const tables = this.db
+				.query(
+					"SELECT name FROM sqlite_master WHERE type='table' AND name='nodes'",
+				)
+				.get();
+			if (tables) {
+				// DB exists but has version 0. Detect schema state.
+				const cols = this.db.query("PRAGMA table_info(nodes)").all() as {
+					name: string;
+				}[];
+				const hasHash = cols.some((c) => c.name === "hash");
+				const hasMeta = cols.some((c) => c.name === "meta");
+
+				if (hasHash && hasMeta) {
+					currentVersion = 3;
+				} else if (hasHash) {
+					currentVersion = 2;
+				} else {
+					currentVersion = 1;
+				}
+				// Update the version on the file so we don't guess next time
+				this.db.run(`PRAGMA user_version = ${currentVersion}`);
+			}
+		}
+
+		if (currentVersion >= CURRENT_SCHEMA_VERSION) return;
+
+		log.info(
+			`📦 ResonanceDB: Migrating from v${currentVersion} to v${CURRENT_SCHEMA_VERSION}...`,
+		);
+
+		for (const migration of MIGRATIONS) {
+			if (migration.version > currentVersion) {
+				// console.log(`   Running Migration v${migration.version}: ${migration.description}`);
+				if (migration.sql) {
+					this.db.run(migration.sql);
+				}
+				if (migration.up) {
+					migration.up(this.db);
+				}
+				this.db.run(`PRAGMA user_version = ${migration.version}`);
+				currentVersion = migration.version;
+			}
+		}
+	}
+
+	insertNode(node: Node) {
+		// No inline migrations here anymore!
+
+		const stmt = this.db.prepare(`
+            INSERT OR REPLACE INTO nodes (id, type, title, content, domain, layer, embedding, hash, meta)
+            VALUES ($id, $type, $title, $content, $domain, $layer, $embedding, $hash, $meta)
+        `);
+
+		try {
+			// FAFCAS Protocol: Trust pre-normalized embeddings from Embedder/VectorEngine
+			// Embeddings are already normalized at generation boundary
+			const blob = node.embedding
+				? new Uint8Array(
+						node.embedding.buffer,
+						node.embedding.byteOffset,
+						node.embedding.byteLength,
+					)
+				: null;
+
+			stmt.run({
+				$id: String(node.id),
+				$type: String(node.type),
+				$title: node.label ? String(node.label) : null,
+				$content: node.content ? String(node.content) : null,
+				$domain: String(node.domain || "knowledge"),
+				$layer: String(node.layer || "experience"),
+				$embedding: blob,
+				$hash: node.hash ? String(node.hash) : null,
+				$meta: node.meta ? JSON.stringify(node.meta) : null,
+			});
+		} catch (err) {
+			log.error(
+				{
+					err,
+					id: node.id,
+					blobSize: node.embedding ? node.embedding.byteLength : 0,
+					blobType: node.embedding
+						? node.embedding instanceof Float32Array
+							? "F32"
+							: "Other"
+						: "Null",
+				},
+				"❌ Failed to insert node",
+			);
+			throw err;
+		}
+	}
+
+	getRawDb(): Database {
+		return this.db;
+	}
+
+	insertEdge(source: string, target: string, type: string = "related_to") {
+		this.db.run(
+			`
+            INSERT OR IGNORE INTO edges (source, target, type)
+            VALUES (?, ?, ?)
+        `,
+			[source, target, type],
+		);
+	}
+
+	/**
+	 * Insert a semantic edge with confidence and veracity metadata.
+	 * Used by the Sieve+Net harvester for extracted triples.
+	 */
+	insertSemanticEdge(
+		source: string,
+		target: string,
+		type: string,
+		confidence: number = 1.0,
+		veracity: number = 1.0,
+		contextSource?: string,
+	) {
+		this.db.run(
+			`
+            INSERT INTO edges (source, target, type, confidence, veracity, context_source)
+            VALUES (?, ?, ?, ?, ?, ?)
+            ON CONFLICT(source, target, type) DO UPDATE SET
+                confidence = excluded.confidence,
+                veracity = excluded.veracity,
+                context_source = excluded.context_source
+        `,
+			[source, target, type, confidence, veracity, contextSource ?? null],
+		);
+	}
+
+	// Typed Data Accessors
+
+	// Typed Data Accessors
+
+	/**
+	 * Fetch nodes with Safe Limits.
+	 * @param options.excludeContent If true, skips 'content' and 'embedding' columns (Large BLOBs).
+	 */
+	getNodes(
+		options: {
+			domain?: string;
+			type?: string;
+			limit?: number;
+			offset?: number;
+			excludeContent?: boolean; // FAFCAS: Optimization for metadata scans
+		} = {},
+	): Node[] {
+		// Safe Default Limit? User specifically asked to fix unbounded.
+		// But for backward compatibility with scripts that expect everything, we might need a high limit or explicit 'unlimited' flag.
+		// Let's default to unlimited IF not specified, but strongly encourage limits in docs.
+		// Actually, 'sloppy code' implies implicit SELECT * is bad.
+		// Let's implement options but keep default behavior 'all' to avoid breaking existing logic silently,
+		// BUT we will log a warning if count > 1000 and no limit?
+		// No, let's just implement the capabilities first. callers must opt-in to limits.
+
+		const cols = options.excludeContent
+			? "id, type, title, domain, layer, hash, meta" // No content, No embedding
+			: "*";
+
+		let sql = `SELECT ${cols} FROM nodes WHERE 1=1`;
+		const params: (string | number)[] = [];
+
+		if (options.domain) {
+			sql += " AND domain = ?";
+			params.push(options.domain);
+		}
+		if (options.type) {
+			sql += " AND type = ?";
+			params.push(options.type);
+		}
+
+		if (options.limit) {
+			sql += " LIMIT ?";
+			params.push(options.limit);
+		}
+		if (options.offset) {
+			sql += " OFFSET ?";
+			params.push(options.offset);
+		}
+
+		const rows = this.db.query(sql).all(...params) as Record<string, unknown>[];
+		return rows.map((row) => this.mapRowToNode(row));
+	}
+
+	getLexicon(): {
+		id: string;
+		label: string;
+		aliases: string[];
+		definition: string;
+	}[] {
+		// Optimized: Exclude content/embedding since we only need ID, Title, Aliases
+		const _sql =
+			"SELECT id, title, meta, content FROM nodes WHERE domain = 'lexicon' AND type = 'concept'";
+		// Note: content is 'definition', usually small. Keep it. Embedding is big.
+		// Actually, let's select specific columns to avoid embedding blob
+
+		const rows = this.db
+			.query(
+				"SELECT id, title, meta, content FROM nodes WHERE domain = 'lexicon' AND type = 'concept'",
+			)
+			.all() as { id: string; title: string; meta: string; content: string }[];
+
+		return rows.map((row) => {
+			const meta = row.meta ? JSON.parse(row.meta) : {};
+			return {
+				id: row.id,
+				label: row.title,
+				aliases: meta.aliases || [],
+				definition: row.content,
+				...meta,
+			};
+		});
+	}
+
+	// biome-ignore lint/suspicious/noExplicitAny: row is raw DB result
+	private mapRowToNode(row: any): Node {
+		return {
+			id: row.id,
+			type: row.type,
+			label: row.title,
+			content: row.content, // May be undefined if excluded
+			domain: row.domain,
+			layer: row.layer,
+			// Only hydrate embedding if it exists (was selected)
+			embedding: row.embedding
+				? new Float32Array(
+						row.embedding.buffer,
+						row.embedding.byteOffset,
+						row.embedding.byteLength / 4,
+					)
+				: undefined,
+			hash: row.hash,
+			meta: row.meta ? JSON.parse(row.meta) : {},
+		};
+	}
+
+	getNodeHash(id: string): string | null {
+		const row = this.db
+			.prepare("SELECT hash FROM nodes WHERE id = ?")
+			.get(id) as { hash: string } | undefined;
+		return row ? row.hash : null;
+	}
+
+	getStats() {
+		const nodesCount = (
+			this.db.query("SELECT COUNT(*) as c FROM nodes").get() as { c: number }
+		).c;
+		const edgesCount = (
+			this.db.query("SELECT COUNT(*) as c FROM edges").get() as { c: number }
+		).c;
+		const vectorsCount = (
+			this.db
+				.query("SELECT COUNT(*) as c FROM nodes WHERE embedding IS NOT NULL")
+				.get() as { c: number }
+		).c;
+		const semanticTokensCount = (
+			this.db
+				.query(
+					"SELECT COUNT(*) as c FROM nodes WHERE meta LIKE '%semantic_tokens%'",
+				)
+				.get() as { c: number }
+		).c;
+
+		return {
+			nodes: nodesCount,
+			edges: edgesCount,
+			vectors: vectorsCount,
+			semantic_tokens: semanticTokensCount,
+			db_size_bytes:
+				(this.db.query("PRAGMA page_count").get() as { page_count: number })
+					.page_count *
+				(this.db.query("PRAGMA page_size").get() as { page_size: number })
+					.page_size,
+		};
+	}
+
+	getNodesByType(type: string): Node[] {
+		// Forward to new method
+		return this.getNodes({ type });
+	}
+
+	// searchText method removed (Hollow Node Simplification)
+
+	/**
+	 * Transaction Management
+	 * Wraps database operations in transactions for atomicity and performance
+	 */
+	beginTransaction() {
+		this.db.run("BEGIN TRANSACTION");
+	}
+
+	commit() {
+		this.db.run("COMMIT");
+	}
+
+	rollback() {
+		this.db.run("ROLLBACK");
+	}
+
+	close() {
+		this.db.close();
+	}
+
+	checkpoint() {
+		this.db.run("PRAGMA wal_checkpoint(TRUNCATE);");
+	}
+}
+
+// Helper: Calculate magnitude (L2 norm) of a vector
+function magnitude(vec: Float32Array): number {
+	let sum = 0;
+	for (let i = 0; i < vec.length; i++) {
+		sum += (vec[i] || 0) * (vec[i] || 0);
+	}
+	return Math.sqrt(sum);
+}
+
+// FAFCAS Protocol: use Dot Product for normalized vectors
+// Source: playbooks/embeddings-and-fafcas-protocol-playbook.md
+//
+// Returns 0 for zero-magnitude vectors (failed embeddings) to prevent
+// false matches in search results.
+export function dotProduct(a: Float32Array, b: Float32Array): number {
+	// Check for zero vectors (failed embeddings)
+	const magA = magnitude(a);
+	const magB = magnitude(b);
+
+	if (magA < 1e-6 || magB < 1e-6) {
+		// log.warn("⚠️ Zero vector detected in dot product, skipping comparison"); // Too noisy for tight loops
+		return 0;
+	}
+
+	let sum = 0;
+	// Modern JS engines SIMD-optimize this loop automatically
+	for (let i = 0; i < a.length; i++) {
+		sum += (a[i] || 0) * (b[i] || 0);
+	}
+	return sum;
+}
+
+// Source: playbooks/embeddings-and-fafcas-protocol-playbook.md
+export function toFafcas(vector: Float32Array): Uint8Array {
+	// 1. Calculate Magnitude (L2 Norm)
+	let sum = 0;
+	for (let i = 0; i < vector.length; i++) {
+		const val = vector[i] || 0;
+		sum += val * val;
+	}
+	const magnitude = Math.sqrt(sum);
+
+	// 2. Normalize (Divide by Magnitude)
+	// Optimization: If magnitude is 0, return zero vector
+	if (magnitude > 1e-6) {
+		for (let i = 0; i < vector.length; i++) {
+			const val = vector[i] || 0;
+			vector[i] = val / magnitude;
+		}
+	}
+
+	// 3. Serialize to Raw Bytes (FAFCAS Blob)
+	return new Uint8Array(vector.buffer, vector.byteOffset, vector.byteLength);
+}

package/src/resonance/pipeline/README.md

@@ -0,0 +1,7 @@
+# 🔄 Resonance Pipeline
+
+Data processing and extraction steps for the Resonance Engine.
+
+## Contents
+- **`extract.ts`**: Extracts high-value terms from the knowledge graph for frontend use.
+- **`transform_docs.ts`**: Prepares markdown documents for ingestion.