@rkarim08/sia 1.0.0

package/src/ast/watcher.ts

@@ -0,0 +1,322 @@
+import { existsSync, watch as fsWatch, readdirSync, readFileSync, statSync } from "node:fs";
+import { join, relative, resolve } from "node:path";
+import { dispatchExtractionAsync } from "@/ast/extractors/tier-dispatch";
+import { getLanguageForFile } from "@/ast/languages";
+import { createIgnoreMatcher, detectPackagePath, toPosixPath } from "@/ast/path-utils";
+import { computeEdits } from "@/ast/tree-sitter/edit-computer";
+import { TreeSitterService } from "@/ast/tree-sitter/service";
+import { extractTrackA } from "@/capture/track-a-ast";
+import type { SiaDb } from "@/graph/db-interface";
+import { getActiveEdges, invalidateEdge } from "@/graph/edges";
+import { insertEntity, invalidateEntity } from "@/graph/entities";
+import type { SiaConfig } from "@/shared/config";
+import { getConfig } from "@/shared/config";
+
+let _watcherService: TreeSitterService | null = null;
+function getWatcherService(): TreeSitterService {
+	if (!_watcherService) {
+		const config = getConfig();
+		_watcherService = new TreeSitterService(config.treeSitter!);
+	}
+	return _watcherService;
+}
+
+export interface FileWatcher {
+	start(): void;
+	stop(): Promise<void>;
+	ready: Promise<void>;
+}
+
+interface TrackedEntity {
+	id: string;
+	name: string;
+	content: string;
+}
+
+async function getEntitiesForPath(db: SiaDb, relPath: string): Promise<TrackedEntity[]> {
+	const pattern = `%"${relPath}"%`;
+	const result = await db.execute(
+		"SELECT id, name, content FROM graph_nodes WHERE t_valid_until IS NULL AND archived_at IS NULL AND file_paths LIKE ?",
+		[pattern],
+	);
+	return result.rows as unknown as TrackedEntity[];
+}
+
+async function invalidateEdgesForEntity(db: SiaDb, entityId: string, ts: number): Promise<void> {
+	const edges = await getActiveEdges(db, entityId);
+	for (const edge of edges) {
+		await invalidateEdge(db, edge.id, ts);
+	}
+}
+
+async function handleDeletion(db: SiaDb, relPath: string): Promise<void> {
+	const existing = await getEntitiesForPath(db, relPath);
+	const ts = Date.now();
+	for (const entity of existing) {
+		await invalidateEntity(db, entity.id, ts);
+		await invalidateEdgesForEntity(db, entity.id, ts);
+	}
+}
+
+async function handleChange(db: SiaDb, relPath: string, content: string): Promise<void> {
+	// --- Tree-sitter incremental re-parse path ---
+	let treeSitterFacts: import("@/capture/types").CandidateFact[] | null = null;
+	try {
+		const langConfig = getLanguageForFile(relPath);
+		if (langConfig) {
+			const service = getWatcherService();
+			const cache = service.cache;
+			const cachedEntry = cache.get(relPath);
+
+			if (cachedEntry) {
+				// Incremental re-parse: compute edits, apply to old tree, parse
+				const edits = computeEdits(cachedEntry.source, content);
+				const treeAny = cachedEntry.tree as any;
+				for (const edit of edits) {
+					treeAny.edit?.(edit);
+				}
+				const newTree = await service.parse(content, langConfig.name, cachedEntry.tree);
+				if (newTree) {
+					cache.set(relPath, newTree, content);
+					// Use dispatchExtractionAsync for scoped extraction
+					treeSitterFacts = await dispatchExtractionAsync(
+						content,
+						relPath,
+						langConfig.tier,
+						langConfig.name,
+						langConfig.specialHandling,
+					);
+				}
+			} else {
+				// Full parse (populates cache for next time)
+				const tree = await service.parse(content, langConfig.name);
+				if (tree) {
+					cache.set(relPath, tree, content);
+					treeSitterFacts = await dispatchExtractionAsync(
+						content,
+						relPath,
+						langConfig.tier,
+						langConfig.name,
+						langConfig.specialHandling,
+					);
+				}
+			}
+		}
+	} catch {
+		// Tree-sitter failed — fall through to regex below
+		treeSitterFacts = null;
+	}
+
+	// --- Regex fallback (existing logic, used when tree-sitter is unavailable) ---
+	const facts = treeSitterFacts ?? extractTrackA(content, relPath);
+	const existing = await getEntitiesForPath(db, relPath);
+
+	const existingByName = new Map(existing.map((e) => [e.name, e]));
+	const newNames = new Set(facts.map((f) => f.name));
+
+	for (const fact of facts) {
+		const existingEntity = existingByName.get(fact.name);
+		if (existingEntity) {
+			// SUPERSEDE pattern: invalidate old, insert new
+			if (existingEntity.content !== fact.content) {
+				const ts = Date.now();
+				await invalidateEntity(db, existingEntity.id, ts);
+				await invalidateEdgesForEntity(db, existingEntity.id, ts);
+				await insertEntity(db, {
+					type: fact.type,
+					name: fact.name,
+					content: fact.content,
+					summary: fact.summary,
+					tags: JSON.stringify(fact.tags ?? []),
+					file_paths: JSON.stringify([relPath]),
+					trust_tier: fact.trust_tier,
+					confidence: fact.confidence,
+					package_path: detectPackagePath(relPath),
+					extraction_method: fact.extraction_method ?? null,
+				});
+			}
+			continue;
+		}
+
+		await insertEntity(db, {
+			type: fact.type,
+			name: fact.name,
+			content: fact.content,
+			summary: fact.summary,
+			tags: JSON.stringify(fact.tags ?? []),
+			file_paths: JSON.stringify([relPath]),
+			trust_tier: fact.trust_tier,
+			confidence: fact.confidence,
+			package_path: detectPackagePath(relPath),
+			extraction_method: fact.extraction_method ?? null,
+		});
+	}
+
+	// Invalidate removed entities + edges
+	const ts = Date.now();
+	for (const entity of existing) {
+		if (newNames.has(entity.name)) continue;
+		await invalidateEntity(db, entity.id, ts);
+		await invalidateEdgesForEntity(db, entity.id, ts);
+	}
+}
+
+type CloseableWatcher = {
+	close(): void | Promise<void>;
+};
+
+async function createUnderlyingWatcher(
+	root: string,
+	ignoreMatcher: ReturnType<typeof createIgnoreMatcher>,
+	onChange: (absPath: string) => void,
+	onDelete: (absPath: string) => void,
+	onReady: () => void,
+): Promise<CloseableWatcher> {
+	try {
+		const mod = await import("chokidar");
+		const chokidar = mod as unknown as {
+			default?: { watch: typeof import("chokidar").watch };
+			watch: typeof import("chokidar").watch;
+		};
+		const watchFn = chokidar.watch ?? chokidar.default?.watch;
+		if (!watchFn) {
+			throw new Error("chokidar watch not available");
+		}
+		const watcher = watchFn(root, {
+			ignoreInitial: true,
+			ignored: (path, stats) =>
+				ignoreMatcher.shouldIgnore(path ?? "", Boolean(stats?.isDirectory())),
+		});
+		watcher.on("change", onChange);
+		watcher.on("add", onChange);
+		watcher.on("unlink", onDelete);
+		(watcher as unknown as { on: (event: string, cb: () => void) => void }).on("ready", onReady);
+		return watcher;
+	} catch {
+		// Fallback: fs.watch (recursive on macOS/Windows)
+		const watcher = fsWatch(root, { recursive: true }, (eventType, filename) => {
+			if (!filename) return;
+			const absPath = join(root, filename.toString());
+			if (ignoreMatcher.shouldIgnore(absPath, false)) return;
+			if (eventType === "rename") {
+				if (existsSync(absPath)) {
+					onChange(absPath);
+				} else {
+					onDelete(absPath);
+				}
+				return;
+			}
+			onChange(absPath);
+		});
+		onReady();
+		return watcher;
+	}
+}
+
+export function createWatcher(repoRoot: string, db: SiaDb, config: SiaConfig): FileWatcher {
+	const root = resolve(repoRoot);
+	const ignoreMatcher = createIgnoreMatcher(root, config.excludePaths ?? []);
+
+	let closer: CloseableWatcher | null = null;
+	let readyPromise: Promise<void> = Promise.resolve();
+	let readyResolve: () => void = () => {};
+	// Track file mtimes for initial sync
+	const fileMtimes = new Map<string, number>();
+
+	function normalize(absPath: string): string | null {
+		const rel = toPosixPath(relative(root, absPath));
+		if (rel.startsWith("..")) return null;
+		return rel;
+	}
+
+	async function syncOnce(): Promise<void> {
+		const seen = new Set<string>();
+		const walk = async (dir: string): Promise<void> => {
+			let entries: import("node:fs").Dirent[];
+			try {
+				entries = readdirSync(dir, { withFileTypes: true }) as import("node:fs").Dirent[];
+			} catch {
+				return;
+			}
+			for (const entry of entries) {
+				const absPath = join(dir, entry.name as string);
+				const isDir = entry.isDirectory();
+				if (ignoreMatcher.shouldIgnore(absPath, isDir)) continue;
+				if (isDir) {
+					await walk(absPath);
+					continue;
+				}
+				const relPath = normalize(absPath);
+				if (!relPath) continue;
+				seen.add(relPath);
+				const stat = statSync(absPath);
+				const prev = fileMtimes.get(relPath);
+				if (prev === undefined || prev !== stat.mtimeMs) {
+					fileMtimes.set(relPath, stat.mtimeMs);
+					const content = readFileSync(absPath, "utf-8");
+					await handleChange(db, relPath, content);
+				}
+			}
+		};
+
+		await walk(root);
+
+		for (const path of [...fileMtimes.keys()]) {
+			if (!seen.has(path)) {
+				fileMtimes.delete(path);
+				await handleDeletion(db, path);
+			}
+		}
+	}
+
+	const start = (): void => {
+		readyPromise = new Promise<void>((resolveReady) => {
+			readyResolve = resolveReady;
+		});
+
+		void (async () => {
+			closer = await createUnderlyingWatcher(
+				root,
+				ignoreMatcher,
+				(absPath) => {
+					const relPath = normalize(absPath);
+					if (!relPath) return;
+					if (ignoreMatcher.shouldIgnore(absPath, false)) return;
+					const language = getLanguageForFile(absPath);
+					if (!language) return;
+					try {
+						const content = readFileSync(absPath, "utf-8");
+						void handleChange(db, relPath, content);
+					} catch {
+						void handleDeletion(db, relPath);
+					}
+				},
+				(absPath) => {
+					const relPath = normalize(absPath);
+					if (!relPath) return;
+					void handleDeletion(db, relPath);
+				},
+				() => {
+					// Initial sync after chokidar is ready
+					void syncOnce().then(() => readyResolve());
+				},
+			);
+		})();
+	};
+
+	const stop = async (): Promise<void> => {
+		await syncOnce();
+		if (closer) {
+			await closer.close();
+			closer = null;
+		}
+	};
+
+	return {
+		start,
+		stop,
+		get ready(): Promise<void> {
+			return readyPromise;
+		},
+	};
+}

package/src/capture/chunker.ts

@@ -0,0 +1,169 @@
+// Module: chunker — Convert HookPayload into CandidateFact[], applying
+// filtering, trust-tier assignment, and paranoid-capture quarantining.
+
+import { writeAuditEntry } from "@/graph/audit";
+import type { SiaDb } from "@/graph/db-interface";
+import type { SiaConfig } from "@/shared/config";
+import type { CandidateFact, HookPayload } from "./types";
+
+/** File extensions considered "recognised" source code for Tier 2 classification. */
+const RECOGNISED_EXTENSIONS = new Set([
+	".ts",
+	".tsx",
+	".js",
+	".jsx",
+	".mjs",
+	".cjs",
+	".json",
+	".md",
+	".yaml",
+	".yml",
+	".toml",
+	".py",
+	".rs",
+	".go",
+	".java",
+	".c",
+	".cpp",
+	".h",
+	".hpp",
+	".cs",
+	".rb",
+	".php",
+	".swift",
+	".kt",
+	".sh",
+	".bash",
+	".zsh",
+	".sql",
+	".html",
+	".css",
+	".scss",
+	".less",
+	".vue",
+	".svelte",
+]);
+
+/** Regex that matches http:// or https:// URLs. */
+const URL_RE = /https?:\/\/[^\s)>"']+/g;
+
+/**
+ * Return true when `url` points outside the project working directory.
+ * Localhost / 127.0.0.1 URLs are treated as internal.
+ */
+function isExternalUrl(url: string, cwd: string): boolean {
+	// file:// or paths starting with cwd are internal
+	if (url.startsWith(`file://${cwd}`)) return false;
+	try {
+		const parsed = new URL(url);
+		const host = parsed.hostname;
+		if (host === "localhost" || host === "127.0.0.1" || host === "::1") {
+			return false;
+		}
+	} catch {
+		// Malformed URL — treat as external to be safe
+		return true;
+	}
+	return true;
+}
+
+/**
+ * Determine whether a filePath has a recognised source-code extension.
+ */
+function hasRecognisedExtension(filePath: string): boolean {
+	const dot = filePath.lastIndexOf(".");
+	if (dot === -1) return false;
+	return RECOGNISED_EXTENSIONS.has(filePath.slice(dot).toLowerCase());
+}
+
+/**
+ * Convert a single HookPayload into zero or one CandidateFact entries.
+ *
+ * Filtering rules (returns []):
+ *  - empty content
+ *  - content shorter than 20 characters
+ *  - node_modules reads (toolName contains 'Read' AND filePath contains 'node_modules')
+ *
+ * Trust tier assignment:
+ *  - Tier 1: type === 'Stop'
+ *  - Tier 2: filePath with recognised source extension
+ *  - Tier 3: no filePath
+ *  - Tier 4: content contains external URLs (http/https not matching cwd)
+ *
+ * paranoidCapture + Tier 4: writes QUARANTINE audit entry and discards.
+ */
+export async function chunkPayload(
+	payload: HookPayload,
+	config: SiaConfig,
+	db?: SiaDb,
+): Promise<CandidateFact[]> {
+	// --- Filtering -----------------------------------------------------------
+
+	// Empty content
+	if (!payload.content || payload.content.trim().length === 0) {
+		return [];
+	}
+
+	// Content shorter than 20 chars
+	if (payload.content.length < 20) {
+		return [];
+	}
+
+	// node_modules reads
+	if (payload.toolName?.includes("Read") && payload.filePath?.includes("node_modules")) {
+		return [];
+	}
+
+	// --- Trust tier assignment -----------------------------------------------
+
+	let trust_tier: 1 | 2 | 3 | 4;
+
+	if (payload.type === "Stop") {
+		trust_tier = 1;
+	} else if (payload.filePath && hasRecognisedExtension(payload.filePath)) {
+		trust_tier = 2;
+	} else if (!payload.filePath) {
+		trust_tier = 3;
+	} else {
+		// filePath present but unrecognised extension — default to Tier 2
+		trust_tier = 2;
+	}
+
+	// Check for external URLs — promotes to Tier 4
+	const urls = payload.content.match(URL_RE) ?? [];
+	const hasExternal = urls.some((u) => isExternalUrl(u, payload.cwd));
+	if (hasExternal) {
+		trust_tier = 4;
+	}
+
+	// --- Paranoid capture quarantine -----------------------------------------
+
+	if (config.paranoidCapture && trust_tier === 4) {
+		if (db) {
+			await writeAuditEntry(db, "QUARANTINE", {
+				trust_tier: 4,
+				source_episode: payload.sessionId,
+			});
+		}
+		return [];
+	}
+
+	// --- Build CandidateFact -------------------------------------------------
+
+	const name = payload.content.trim().slice(0, 50).trim();
+	const summary = payload.content.trim().slice(0, 100).trim();
+
+	const fact: CandidateFact = {
+		type: "Concept",
+		name,
+		content: payload.content,
+		summary,
+		tags: [],
+		file_paths: payload.filePath ? [payload.filePath] : [],
+		trust_tier,
+		confidence: trust_tier <= 2 ? 0.8 : 0.5,
+		extraction_method: "chunker",
+	};
+
+	return [fact];
+}

package/src/capture/consolidate.ts

@@ -0,0 +1,127 @@
+// Module: consolidate — Two-phase consolidation of candidate facts into the graph
+
+import type { CandidateFact, ConsolidationResult } from "@/capture/types";
+import { writeAuditEntry } from "@/graph/audit";
+import type { SiaDb } from "@/graph/db-interface";
+import type { Entity } from "@/graph/entities";
+import { insertEntity, invalidateEntity, updateEntity } from "@/graph/entities";
+
+/**
+ * Compute the Jaccard similarity between two strings based on their word sets.
+ * Splits both strings on whitespace, computes |intersection| / |union|.
+ * Returns 0 if either string is empty.
+ */
+export function wordJaccard(a: string, b: string): number {
+	const setA = new Set(a.split(/\s+/).filter(Boolean));
+	const setB = new Set(b.split(/\s+/).filter(Boolean));
+
+	if (setA.size === 0 || setB.size === 0) return 0;
+
+	let intersectionSize = 0;
+	for (const word of setA) {
+		if (setB.has(word)) intersectionSize++;
+	}
+
+	const unionSize = setA.size + setB.size - intersectionSize;
+	if (unionSize === 0) return 0;
+
+	return intersectionSize / unionSize;
+}
+
+/**
+ * Consolidate an array of candidate facts into the graph database.
+ *
+ * For each candidate:
+ * 1. Query existing entities matching name + type that are still active.
+ * 2. If no match: ADD (insert new entity + audit).
+ * 3. If match found, compute wordJaccard on content:
+ *    - >0.8  => NOOP (skip, audit NOOP)
+ *    - 0.4-0.8 => UPDATE (update content, audit UPDATE)
+ *    - <0.4  => INVALIDATE old + ADD new (invalidate old, insert new, audit both)
+ * 4. All writes happen inside db.transaction() for atomicity.
+ * 5. Returns aggregate counts { added, updated, invalidated, noops }.
+ */
+export async function consolidate(
+	db: SiaDb,
+	candidates: CandidateFact[],
+): Promise<ConsolidationResult> {
+	const result: ConsolidationResult = {
+		added: 0,
+		updated: 0,
+		invalidated: 0,
+		noops: 0,
+	};
+
+	await db.transaction(async (tx) => {
+		for (const candidate of candidates) {
+			// 1. Query active entities with same name and type
+			const existing = await tx.execute(
+				"SELECT * FROM graph_nodes WHERE name = ? AND type = ? AND t_valid_until IS NULL AND archived_at IS NULL",
+				[candidate.name, candidate.type],
+			);
+
+			const match = existing.rows[0] as unknown as Entity | undefined;
+
+			if (!match) {
+				// 2. No match — ADD
+				const _inserted = await insertEntity(tx, {
+					type: candidate.type,
+					name: candidate.name,
+					content: candidate.content,
+					summary: candidate.summary,
+					tags: JSON.stringify(candidate.tags),
+					file_paths: JSON.stringify(candidate.file_paths),
+					trust_tier: candidate.trust_tier,
+					confidence: candidate.confidence,
+					extraction_method: candidate.extraction_method ?? null,
+					t_valid_from: candidate.t_valid_from ?? null,
+				});
+				// insertEntity already writes an ADD audit entry
+				result.added++;
+			} else {
+				// 3. Match found — compare content similarity
+				const similarity = wordJaccard(match.content, candidate.content);
+
+				if (similarity > 0.8) {
+					// NOOP — content is sufficiently similar
+					await writeAuditEntry(tx, "NOOP", { entity_id: match.id });
+					result.noops++;
+				} else if (similarity >= 0.4) {
+					// UPDATE — content is moderately similar
+					await updateEntity(tx, match.id, {
+						content: candidate.content,
+						summary: candidate.summary,
+						tags: JSON.stringify(candidate.tags),
+						file_paths: JSON.stringify(candidate.file_paths),
+						confidence: candidate.confidence,
+						extraction_method: candidate.extraction_method ?? null,
+					});
+					// updateEntity already writes an UPDATE audit entry
+					result.updated++;
+				} else {
+					// INVALIDATE old + ADD new — content is too different
+					await invalidateEntity(tx, match.id);
+					// invalidateEntity already writes an INVALIDATE audit entry
+
+					await insertEntity(tx, {
+						type: candidate.type,
+						name: candidate.name,
+						content: candidate.content,
+						summary: candidate.summary,
+						tags: JSON.stringify(candidate.tags),
+						file_paths: JSON.stringify(candidate.file_paths),
+						trust_tier: candidate.trust_tier,
+						confidence: candidate.confidence,
+						extraction_method: candidate.extraction_method ?? null,
+						t_valid_from: candidate.t_valid_from ?? null,
+					});
+					// insertEntity already writes an ADD audit entry
+					result.invalidated++;
+					result.added++;
+				}
+			}
+		}
+	});
+
+	return result;
+}