amalfa 1.0.0 → 1.0.2

package/src/resonance/daemon.ts

@@ -1,236 +0,0 @@
-import { watch } from "node:fs";
-import { join } from "node:path";
-import settings from "@/polyvis.settings.json";
-import { Ingestor } from "../pipeline/Ingestor";
-import { EnvironmentVerifier } from "../utils/EnvironmentVerifier";
-import { getLogger } from "../utils/Logger";
-import { ServiceLifecycle } from "../utils/ServiceLifecycle";
-import { Embedder } from "./services/embedder";
-
-const args = process.argv.slice(2);
-const command = args[0] || "serve";
-const log = getLogger("Daemon");
-
-// --- Helper: Notifications ---
-
-async function notify(title: string, message: string) {
-	// Native macOS notifications via AppleScript
-	// Zero dependencies
-	try {
-		const script = `display notification "${message}" with title "${title}"`;
-		await Bun.spawn(["osascript", "-e", script]);
-	} catch (e) {
-		log.error({ err: e }, "Failed to send notification");
-	}
-}
-
-// --- Service Lifecycle ---
-
-const lifecycle = new ServiceLifecycle({
-	name: "Daemon",
-	pidFile: ".daemon.pid",
-	logFile: ".daemon.log",
-	entryPoint: "src/resonance/daemon.ts",
-});
-
-// --- Server Logic (The actual Daemon) ---
-
-async function main() {
-	// 0. Verify Environment
-	await EnvironmentVerifier.verifyOrExit();
-
-	// 1. Initialize Ingestion (Daemon Mode: Watch Enabled)
-	const PORT = parseInt(process.env.VECTOR_PORT || "3010", 10);
-
-	log.info({ port: PORT }, "🔌 Vector Daemon starting...");
-	log.info("Initializing Embedder...");
-
-	// 1. Initialize Embedder (Compute Node)
-	try {
-		const embedder = Embedder.getInstance();
-		await embedder.embed("warmup", true);
-		log.info("✅ Embedder Ready.");
-	} catch (e) {
-		log.fatal({ err: e }, "❌ Failed to initialize embedder");
-		process.exit(1);
-	}
-
-	// 2. Start HTTP Server
-	Bun.serve({
-		port: PORT,
-		async fetch(req) {
-			const url = new URL(req.url);
-
-			if (req.method === "GET" && url.pathname === "/health") {
-				return new Response(JSON.stringify({ status: "ok" }), {
-					headers: { "Content-Type": "application/json" },
-				});
-			}
-
-			if (req.method === "POST" && url.pathname === "/embed") {
-				try {
-					const body = (await req.json()) as { text: string };
-					if (!body.text || typeof body.text !== "string") {
-						return new Response("Bad Request: 'text' field required", {
-							status: 400,
-						});
-					}
-
-					const vector = await Embedder.getInstance().embed(body.text, true);
-
-					return new Response(JSON.stringify({ vector: Array.from(vector) }), {
-						headers: { "Content-Type": "application/json" },
-					});
-				} catch (e) {
-					log.error({ err: e }, "Embedder API Error");
-					return new Response("Internal Server Error", { status: 500 });
-				}
-			}
-
-			return new Response("Not Found", { status: 404 });
-		},
-	});
-
-	log.info(`🚀 Vector Daemon listening on http://localhost:${PORT}`);
-
-	// 3. Start The Watcher (Active Custodian)
-	startWatcher();
-
-	// Handle cleanup
-	process.on("SIGTERM", () => {
-		log.info("🛑 Received SIGTERM, shutting down...");
-		process.exit(0);
-	});
-}
-
-// --- Watcher Logic ---
-
-let debounceTimer: ReturnType<typeof setTimeout> | null = null;
-const DEBOUNCE_MS = 2000;
-const pendingFiles = new Set<string>();
-
-// Retry queue: Track failed ingestions with attempt counts
-const retryQueue = new Map<
-	string,
-	{ attempts: number; lastError: string; lastAttempt: number }
->();
-const MAX_RETRIES = 3;
-const RETRY_BACKOFF_MS = 5000; // Wait 5 seconds before retry
-
-function startWatcher() {
-	// Dynamically load watch targets from settings
-	const rawSources = settings.paths.sources.experience;
-	const dirsToWatch = rawSources.map((s) => s.path);
-
-	log.info({ triggers: dirsToWatch }, "👀 Watching directories");
-
-	dirsToWatch.forEach((dir) => {
-		const path = join(process.cwd(), dir);
-		try {
-			watch(path, { recursive: true }, (event, filename) => {
-				// Ignore dotfiles and ensure markdown
-				if (filename && !filename.startsWith(".") && filename.endsWith(".md")) {
-					log.debug(
-						{ file: `${dir}/${filename}`, event },
-						"📝 Change detected",
-					);
-
-					// Add full path to pending set
-					const fullPath = join(process.cwd(), dir, filename);
-					pendingFiles.add(fullPath);
-
-					triggerIngestion();
-				}
-			});
-		} catch (e) {
-			log.warn({ dir, err: e }, "⚠️ Could not watch directory");
-		}
-	});
-}
-
-function triggerIngestion() {
-	if (debounceTimer) {
-		clearTimeout(debounceTimer);
-	}
-
-	debounceTimer = setTimeout(async () => {
-		const batchSize = pendingFiles.size;
-		if (batchSize === 0) return;
-
-		log.info({ batchSize }, "🔄 Debounce settle. Starting Batch Ingestion...");
-
-		// Drain the set
-		const batch = Array.from(pendingFiles);
-		pendingFiles.clear();
-
-		try {
-			// Re-instantiate DB/Ingestor for fresh context
-			const ingestor = new Ingestor();
-
-			// OPTIMIZATION: Pass only the changed files
-			await ingestor.run({ files: batch });
-
-			log.info("✅ Batch Ingestion Complete.");
-			// Clear retry counts for successful files
-			for (const file of batch) {
-				retryQueue.delete(file);
-			}
-			await notify("PolyVis Resonance", `Graph Updated (${batchSize} files).`);
-		} catch (e) {
-			const errorMsg = e instanceof Error ? e.message : String(e);
-			log.error({ err: e }, "❌ Ingestion Failed");
-
-			// Re-queue failed files with retry logic
-			const now = Date.now();
-			for (const file of batch) {
-				const retryInfo = retryQueue.get(file) || {
-					attempts: 0,
-					lastError: "",
-					lastAttempt: 0,
-				};
-
-				if (retryInfo.attempts < MAX_RETRIES) {
-					// Re-queue with exponential backoff
-					const nextAttempt = retryInfo.attempts + 1;
-					retryQueue.set(file, {
-						attempts: nextAttempt,
-						lastError: errorMsg,
-						lastAttempt: now,
-					});
-
-					// Re-add to pending files after backoff delay
-					setTimeout(() => {
-						pendingFiles.add(file);
-						triggerIngestion();
-					}, RETRY_BACKOFF_MS * nextAttempt);
-
-					log.warn(
-						{
-							file,
-							attempt: nextAttempt,
-							max: MAX_RETRIES,
-							delayMs: RETRY_BACKOFF_MS * nextAttempt,
-						},
-						"🔄 Scheduling Retry",
-					);
-				} else {
-					// Abandon after max retries
-					log.error(
-						{ file, lastError: retryInfo.lastError },
-						"⛔ ABANDONED: File failed max retries",
-					);
-					retryQueue.delete(file); // Remove from tracking
-				}
-			}
-
-			await notify(
-				"PolyVis Resonance",
-				`Ingestion Failed (${batch.length} files will retry)`,
-			);
-		}
-	}, DEBOUNCE_MS);
-}
-
-// --- Dispatch ---
-
-await lifecycle.run(command, main);

package/src/resonance/pipeline/extract.ts

@@ -1,89 +0,0 @@
-import { existsSync, mkdirSync } from "node:fs";
-import { join } from "node:path";
-
-console.log("Starting term extraction...");
-
-// --- Path Resolution ---
-import settings from "@/polyvis.settings.json";
-
-// Reading from Resonance DB
-const dbPath = join(process.cwd(), settings.paths.database.resonance);
-const publicDir = join(process.cwd(), "public");
-const outputPath = join(publicDir, "terms.json");
-
-// --- Pre-flight Checks ---
-if (!existsSync(dbPath)) {
-	console.error(`❌ Error: Database not found at ${dbPath}`);
-	console.error("Please run 'bun run scripts/build_db.ts' first.");
-	process.exit(1);
-}
-
-// Ensure the 'public' directory exists before trying to write to it.
-if (!existsSync(publicDir)) {
-	console.log(`Creating 'public' directory at ${publicDir}...`);
-	mkdirSync(publicDir, { recursive: true });
-}
-
-import { DatabaseFactory } from "@src/resonance/DatabaseFactory";
-
-// --- Database Query ---
-// Open the database in read-only mode, using Factory for safe concurrency config
-const db = DatabaseFactory.connect(dbPath, { readonly: true });
-
-// This query identifies the most "valuable" terms by counting how many
-// connections (edges) they have. Terms that are more connected are likely
-// more central to the knowledge graph and will yield richer results.
-const query = `
-  SELECT
-    n.title
-  FROM
-    nodes n
-  JOIN (
-    -- First, for each node ID, count its number of DISTINCT neighbors
-    SELECT
-        id,
-        COUNT(DISTINCT neighbor) as neighbor_count
-    FROM (
-        -- Create a unified list of all connections (id -> neighbor)
-        SELECT source as id, target as neighbor FROM edges
-        UNION
-        SELECT target as id, source as neighbor FROM edges
-    )
-    GROUP BY id
-  ) AS counts ON n.id = counts.id
-  WHERE
-    -- Only include nodes that have 2 or more distinct neighbors
-    counts.neighbor_count >= 2
-  ORDER BY
-    counts.neighbor_count DESC, n.title ASC;
-`;
-
-try {
-	// --- Data Extraction & Transformation ---
-	console.log("Querying database for high-value terms...");
-	const results = db.query(query).all() as { title: string }[];
-
-	// We just want an array of the term labels.
-	const terms = results.map((row) => row.title);
-
-	if (terms.length === 0) {
-		console.warn(
-			"⚠️ Warning: Query returned no terms. The resulting file will be an empty array.",
-		);
-	}
-
-	// --- File Output ---
-	// Write the curated list of terms to a static JSON file.
-	// This file can be easily fetched by the frontend.
-	await Bun.write(outputPath, JSON.stringify(terms, null, 2));
-
-	console.log(
-		`✅ Successfully extracted and wrote ${terms.length} terms to ${outputPath}`,
-	);
-} catch (error) {
-	console.error(`❌ An error occurred: ${error}`);
-} finally {
-	// --- Finalization ---
-	db.close();
-	console.log("Term extraction complete.");
-}

package/src/resonance/pipeline/transform_docs.ts

@@ -1,60 +0,0 @@
-import { basename, join } from "node:path";
-import type { IngestionArtifact } from "@src/types/artifact";
-import { Glob } from "bun";
-import settings from "@/polyvis.settings.json";
-
-const artifacts: IngestionArtifact[] = [];
-const root = process.cwd();
-
-// --- Helpers ---
-function extractTitle(content: string, filename: string): string {
-	const match = content.match(/^#\s+(.+)$/m);
-	return match?.[1] ? match[1].trim() : filename;
-}
-
-// --- Transformation Loop ---
-let orderCounter = 0;
-
-for (const source of settings.paths.sources.experience) {
-	const sourceDirRelative = source.path;
-	const sourceDir = join(root, sourceDirRelative);
-	console.log(`Scanning ${sourceDir}...`);
-
-	const glob = new Glob("*.md");
-	// Sort logic is implicit in file system usually, but better to be explicit
-	// Array.from(glob.scanSync) gives unsorted?
-	const files = Array.from(glob.scanSync(sourceDir)).sort();
-
-	for (const file of files) {
-		const fullPath = join(sourceDir, file);
-		const content = await Bun.file(fullPath).text();
-		const id = basename(file, ".md");
-		const type: "playbook" | "debrief" = sourceDirRelative.includes("playbooks")
-			? "playbook"
-			: "debrief";
-
-		artifacts.push({
-			id,
-			type,
-			order_index: orderCounter++,
-			payload: {
-				title: extractTitle(content, id),
-				content: content,
-				domain: "knowledge",
-				layer: "experience",
-				metadata: { path: fullPath },
-			},
-		});
-	}
-}
-
-// --- Output ---
-const outDir = join(root, ".resonance", "artifacts");
-if (!require("node:fs").existsSync(outDir)) {
-	require("node:fs").mkdirSync(outDir, { recursive: true });
-}
-
-const outFile = join(outDir, "docs.json");
-await Bun.write(outFile, JSON.stringify(artifacts, null, 2));
-
-console.log(`✅ Transformed ${artifacts.length} docs to ${outFile}`);

package/src/resonance/services/tokenizer.ts

@@ -1,159 +0,0 @@
-import nlp from "compromise";
-
-export interface SemanticTags {
-	people: string[];
-	places: string[];
-	organizations: string[];
-	topics: string[];
-	dates?: string[];
-	money: string[];
-	protocols?: string[];
-	concepts?: string[];
-}
-
-export class TokenizerService {
-	private static instance: TokenizerService;
-	// Map of normalized_term -> tag inside class state
-	private vocabulary: Map<string, string> = new Map();
-	// Cache keys sorted by length (desc) for greedy matching
-	private searchKeys: string[] = [];
-
-	// Compromise instance (optional, keeping for 'people', 'places' currently)
-	// Could eventually remove if we go 100% custom.
-
-	private constructor() {}
-
-	public static getInstance(): TokenizerService {
-		if (!TokenizerService.instance) {
-			TokenizerService.instance = new TokenizerService();
-		}
-		return TokenizerService.instance;
-	}
-
-	/**
-	 * Extracts semantic entities.
-	 * 1. Uses Compromise for generic Named Entity Recognition (NER)
-	 * 2. Uses Custom "Zero Magic" Brute Force Scanner for Domain Vocabulary
-	 */
-	public extract(text: string): SemanticTags {
-		const doc = nlp(text);
-
-		// 1. Generic NLP (Keep for now as fallback/enrichment)
-		const result: SemanticTags = {
-			people: doc.people().out("array"),
-			places: doc.places().out("array"),
-			organizations: doc.organizations().out("array"),
-			topics: doc.topics().out("array"),
-			money: [],
-			protocols: [],
-			concepts: [],
-		};
-
-		// 2. Zero Magic Domain Scan (Brute Force)
-		// Optimization: Check text.includes() only if vocabulary is small?
-		// But for regex construction or Aho-Corasick, naive loop is fine for now on small text blocks (Bento boxes).
-
-		const lowerText = text.toLowerCase();
-
-		for (const term of this.searchKeys) {
-			// Simple subset check.
-			// Limitation: Matches "pro" in "process". Needs word boundary check.
-			// RegExp construction is costly inside loop?
-			// Better: Pre-build a massive Regex?
-			// Or simpler: \bSTR\b with indexof?
-
-			// Fast "includes" check first
-			if (lowerText.includes(term)) {
-				// Confirm Word Boundary to avoid partial matches
-				// Regex is expensive, but safer for accuracy.
-				// We construct regex only on 'hit' to save cycles?
-				const escaped = term.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
-				const boundaryRegex = new RegExp(`\\b${escaped}\\b`, "i");
-
-				if (boundaryRegex.test(text)) {
-					const tag = this.vocabulary.get(term);
-					// Retrieve Canonical Form (Original Case) if needed?
-					// For now, we normalize to the lowercase SEARCH key, but we might prefer the original.
-					// Given the user wants "OH-058" format, let's try to map back if possible.
-					// But we only stored 'tag' in the map.
-					// Let's store Normalized -> Original ID in a separate map if required?
-					// User Request: "no need to manage case" -> likely means "return standard ID".
-
-					// Since we are returning the matched string (which is usually the ID in lowercase in our loop),
-					// If we want UPPERCASE OH-058, we need to know it.
-					// Simplest fix: Just return the term as found (lowercase) and EdgeWeaver will handle lookup.
-					// EdgeWeaver expects slugified keys anyway.
-
-					// Actually, let's return the TERM as it appears in the text?
-					// boundaryRegex.match(text) would give us the real casing used in the doc (e.g. "OH-058").
-
-					const match = boundaryRegex.exec(text);
-					const realTerm = match ? match[0] : term;
-
-					if (tag === "Protocol") {
-						if (!result.protocols) result.protocols = [];
-						if (!result.protocols.includes(realTerm))
-							result.protocols.push(realTerm);
-					} else if (tag === "Concept") {
-						if (!result.concepts) result.concepts = [];
-						if (!result.concepts.includes(realTerm))
-							result.concepts.push(realTerm);
-					} else if (tag === "Organization") {
-						if (!result.organizations.includes(realTerm))
-							result.organizations.push(realTerm);
-					} else {
-						// Default to Concept if tag is unknown or not explicitly handled
-						if (!result.concepts) result.concepts = [];
-						if (!result.concepts.includes(realTerm))
-							result.concepts.push(realTerm);
-					}
-				}
-			}
-		}
-
-		return result;
-	}
-
-	public loadLexicon(
-		lexicon: { id: string; title: string; type?: string; category?: string }[],
-	) {
-		// Reset
-		this.vocabulary.clear();
-
-		for (const item of lexicon) {
-			let tag = "Concept";
-			if (item.type === "operational-heuristic") tag = "Protocol";
-			if (item.category === "Tool") tag = "Organization";
-
-			// Add Title
-			if (item.title) {
-				this.vocabulary.set(item.title.toLowerCase(), tag);
-			}
-			// Add ID
-			if (item.id) {
-				this.vocabulary.set(item.id.toLowerCase(), tag);
-				// Handle Hyphen Variants
-				if (item.id.includes("-")) {
-					this.vocabulary.set(item.id.toLowerCase().replace(/-/g, " "), tag);
-				}
-			}
-		}
-
-		// Sort keys by length desc to ensure "Web Standards" matches before "Web"
-		this.searchKeys = Array.from(this.vocabulary.keys()).sort(
-			(a, b) => b.length - a.length,
-		);
-
-		console.log(
-			`🧠 ZeroMagic Tokenizer learned ${this.vocabulary.size} terms from lexicon.`,
-		);
-	}
-
-	// Deprecated / No-Op
-	public extend(
-		_customWords: Record<string, string>,
-		_customPatterns: Record<string, string>,
-	) {
-		// No-op for brute force scanner
-	}
-}