@rkarim08/sia 1.0.0

package/src/freshness/dirty-tracker.ts

@@ -0,0 +1,237 @@
+// Module: dirty-tracker — Salsa-inspired dirty propagation engine
+//
+// Coordination layer for the freshness system. Maintains an in-memory map
+// (NOT persisted to SQLite) of node dirty states. The push phase marks
+// nodes dirty when source files change; the pull phase checks dirty state
+// before serving query results.
+//
+// Key design decisions:
+// - In-memory only — rebuilt from source_deps at startup
+// - BFS propagation with firewall cutoff at high-fan-out nodes
+// - Early cutoff: markClean does NOT propagate (content hash unchanged)
+// - Durable nodes skip dirty when only volatile sources change
+
+import { getOutgoingNeighbors } from "@/freshness/firewall";
+import { getDependentsForFile } from "@/freshness/inverted-index";
+import type { SiaDb } from "@/graph/db-interface";
+
+export type DirtyState = "clean" | "dirty" | "maybe_dirty";
+export type Durability = "volatile" | "durable";
+
+/** Default BFS depth limit for dirty propagation. */
+const DEFAULT_MAX_DEPTH = 2;
+
+/** Default edge_count threshold above which a node is a firewall. */
+const DEFAULT_FIREWALL_THRESHOLD = 50;
+
+export class DirtyTracker {
+	/** In-memory only — rebuilt from source_deps at startup. */
+	private dirtyMap = new Map<string, DirtyState>();
+	private durabilityMap = new Map<string, Durability>();
+
+	/**
+	 * Get the dirty state of a node. Nodes not in the map are assumed clean.
+	 */
+	getState(nodeId: string): DirtyState {
+		return this.dirtyMap.get(nodeId) ?? "clean";
+	}
+
+	/**
+	 * Phase 1 — Push: Mark nodes as dirty when their source files change.
+	 * Called by Layer 1 (file-watcher) and Layer 2 (git-reconcile).
+	 *
+	 * Algorithm:
+	 * 1. Look up source_deps[changedFile] -> affected_node_ids
+	 * 2. For each affected node:
+	 *    a. Skip if node is durable and only volatile sources changed
+	 *    b. Set dirtyMap[nodeId] = 'dirty'
+	 *    c. BFS outgoing dependency edges up to maxDepth (default 2)
+	 *    d. For each neighbor:
+	 *       - If edge_count > firewallThreshold (50): set 'maybe_dirty', STOP
+	 *       - Else: set 'dirty', continue
+	 */
+	async markDirty(
+		db: SiaDb,
+		changedFile: string,
+		opts?: { maxDepth?: number; firewallThreshold?: number },
+	): Promise<string[]> {
+		const maxDepth = opts?.maxDepth ?? DEFAULT_MAX_DEPTH;
+		const firewallThreshold = opts?.firewallThreshold ?? DEFAULT_FIREWALL_THRESHOLD;
+
+		// Step 1: Look up affected nodes from inverted index
+		const deps = await getDependentsForFile(db, changedFile);
+		if (deps.length === 0) return [];
+
+		const newlyDirtied: string[] = [];
+
+		// Step 2: Mark each affected node
+		for (const dep of deps) {
+			const nodeId = dep.node_id;
+
+			// 2a: Skip durable nodes for volatile source changes
+			const durability = this.durabilityMap.get(nodeId);
+			if (durability === "durable") {
+				continue;
+			}
+
+			// 2b: Set dirty
+			this.dirtyMap.set(nodeId, "dirty");
+			newlyDirtied.push(nodeId);
+
+			// 2c-d: BFS outgoing edges
+			const bfsDirtied = await this.bfsPropagation(db, nodeId, maxDepth, firewallThreshold);
+			for (const id of bfsDirtied) {
+				newlyDirtied.push(id);
+			}
+		}
+
+		return newlyDirtied;
+	}
+
+	/**
+	 * Phase 2 — Pull: Check and resolve dirty state for a node.
+	 * Called by Layer 3 (stale-while-revalidate) before serving a query result.
+	 *
+	 * Returns:
+	 * - 'clean': serve immediately
+	 * - 'dirty': needs re-verification (caller must re-extract)
+	 * - 'maybe_dirty': needs mtime check (caller does stat())
+	 */
+	checkNode(nodeId: string): DirtyState {
+		return this.dirtyMap.get(nodeId) ?? "clean";
+	}
+
+	/**
+	 * Mark a node as clean after successful re-verification.
+	 * This is the early cutoff: if content hash unchanged, clear dirty
+	 * WITHOUT propagating to dependents.
+	 */
+	markClean(nodeId: string): void {
+		this.dirtyMap.delete(nodeId);
+	}
+
+	/**
+	 * Mark a node as clean and propagate dirty to its dependents.
+	 * Called when re-verification found the content actually changed.
+	 */
+	async markCleanAndPropagate(
+		db: SiaDb,
+		nodeId: string,
+		opts?: { maxDepth?: number; firewallThreshold?: number },
+	): Promise<string[]> {
+		const maxDepth = opts?.maxDepth ?? DEFAULT_MAX_DEPTH;
+		const firewallThreshold = opts?.firewallThreshold ?? DEFAULT_FIREWALL_THRESHOLD;
+
+		// Clear this node
+		this.dirtyMap.delete(nodeId);
+
+		// Propagate to dependents
+		return this.bfsPropagation(db, nodeId, maxDepth, firewallThreshold);
+	}
+
+	/**
+	 * Set durability for a node. Durable nodes skip dirty-checking
+	 * when only volatile sources change.
+	 */
+	setDurability(nodeId: string, durability: Durability): void {
+		this.durabilityMap.set(nodeId, durability);
+	}
+
+	/**
+	 * Clear all dirty state (used on restart or full reindex).
+	 */
+	reset(): void {
+		this.dirtyMap.clear();
+		this.durabilityMap.clear();
+	}
+
+	/**
+	 * Get counts for diagnostics.
+	 */
+	getStats(): {
+		clean: number;
+		dirty: number;
+		maybeDirty: number;
+		total: number;
+	} {
+		let dirty = 0;
+		let maybeDirty = 0;
+		let clean = 0;
+
+		for (const state of this.dirtyMap.values()) {
+			switch (state) {
+				case "dirty":
+					dirty++;
+					break;
+				case "maybe_dirty":
+					maybeDirty++;
+					break;
+				case "clean":
+					clean++;
+					break;
+			}
+		}
+
+		return {
+			clean,
+			dirty,
+			maybeDirty,
+			total: this.dirtyMap.size,
+		};
+	}
+
+	// -----------------------------------------------------------------
+	// Internal BFS propagation
+	// -----------------------------------------------------------------
+
+	/**
+	 * BFS propagation from a starting node through outgoing dependency edges.
+	 *
+	 * For each neighbor:
+	 * - If edge_count > firewallThreshold: set 'maybe_dirty', STOP (don't enqueue)
+	 * - Else: set 'dirty', continue BFS
+	 *
+	 * Returns list of newly-dirtied node IDs (excluding the start node).
+	 */
+	private async bfsPropagation(
+		db: SiaDb,
+		startNodeId: string,
+		maxDepth: number,
+		firewallThreshold: number,
+	): Promise<string[]> {
+		const newlyDirtied: string[] = [];
+		const visited = new Set<string>([startNodeId]);
+
+		// BFS queue: [nodeId, currentDepth]
+		const queue: Array<[string, number]> = [[startNodeId, 0]];
+
+		while (queue.length > 0) {
+			const entry = queue.shift();
+			if (!entry) break;
+			const [currentId, depth] = entry;
+
+			if (depth >= maxDepth) continue;
+
+			const neighbors = await getOutgoingNeighbors(db, currentId);
+
+			for (const neighbor of neighbors) {
+				if (visited.has(neighbor.nodeId)) continue;
+				visited.add(neighbor.nodeId);
+
+				if (neighbor.edgeCount > firewallThreshold) {
+					// Firewall node: mark maybe_dirty and STOP propagation
+					this.dirtyMap.set(neighbor.nodeId, "maybe_dirty");
+					newlyDirtied.push(neighbor.nodeId);
+					// Do NOT enqueue — BFS stops here
+				} else {
+					// Normal node: mark dirty and continue BFS
+					this.dirtyMap.set(neighbor.nodeId, "dirty");
+					newlyDirtied.push(neighbor.nodeId);
+					queue.push([neighbor.nodeId, depth + 1]);
+				}
+			}
+		}
+
+		return newlyDirtied;
+	}
+}

package/src/freshness/file-watcher-layer.ts

@@ -0,0 +1,119 @@
+// Module: file-watcher-layer — Layer 1 freshness invalidation
+//
+// Handles >90% of invalidation cases: code edits during active development.
+// Pipeline per file-save event:
+//   1. Cuckoo filter fast-reject (O(1) — skip files with no derived nodes)
+//   2. Inverted index lookup for affected node IDs
+//   3. DirtyTracker.markDirty with bounded BFS propagation
+//   4. For 'delete' events: invalidate nodes derived solely from that file
+//   5. For 'modify' events: trigger re-extraction for structural nodes
+//
+// Must complete in < 200ms per file save.
+
+import type { SiaDb } from "@/graph/db-interface";
+import type { CuckooFilter } from "./cuckoo-filter";
+import type { DirtyTracker } from "./dirty-tracker";
+import { getDependenciesForNode, getDependentsForFile } from "./inverted-index";
+
+export interface FileChangeEvent {
+	filePath: string; // relative path from repo root
+	type: "create" | "modify" | "delete";
+	mtime?: number; // file modification time (Unix ms)
+}
+
+/**
+ * Layer 1: Process a file change event from the file watcher.
+ *
+ * Pipeline:
+ * 1. Check Cuckoo filter — if file has no derived nodes, skip entirely
+ * 2. Look up inverted index for affected nodes
+ * 3. Mark affected nodes dirty via DirtyTracker
+ * 4. For 'delete' events, invalidate all nodes derived solely from this file
+ * 5. For 'modify' events, trigger re-extraction for structural nodes
+ *
+ * Returns the list of newly-dirtied node IDs.
+ */
+export async function handleFileChange(
+	db: SiaDb,
+	event: FileChangeEvent,
+	tracker: DirtyTracker,
+	filter: CuckooFilter,
+	_opts?: { debounceMs?: number },
+): Promise<string[]> {
+	// Step 1: Cuckoo filter fast-reject
+	if (!filter.contains(event.filePath)) {
+		return [];
+	}
+
+	// Step 2-3: Look up inverted index and mark dirty via DirtyTracker
+	// DirtyTracker.markDirty already does: inverted index lookup -> mark dirty -> BFS propagation
+	const dirtied = await tracker.markDirty(db, event.filePath);
+
+	// Step 4: For 'delete' events, invalidate nodes derived solely from this file
+	if (event.type === "delete") {
+		const deps = await getDependentsForFile(db, event.filePath);
+		const now = Date.now();
+
+		for (const dep of deps) {
+			// Check if the node has any OTHER source dependencies besides the deleted file
+			const allDeps = await getDependenciesForNode(db, dep.node_id);
+			const otherDeps = allDeps.filter((d) => d.source_path !== event.filePath);
+
+			if (otherDeps.length === 0) {
+				// Node derived solely from the deleted file — invalidate it
+				await db.execute(
+					"UPDATE graph_nodes SET t_valid_until = ?, t_expired = ? WHERE id = ? AND t_valid_until IS NULL",
+					[now, now, dep.node_id],
+				);
+			}
+		}
+
+		// Remove the deleted file from the Cuckoo filter
+		filter.remove(event.filePath);
+	}
+
+	return dirtied;
+}
+
+/**
+ * Create a debounced file change handler that coalesces rapid saves.
+ * Default debounce: 50ms. Per-file debouncing so that changes to
+ * different files are not blocked by each other.
+ */
+export function createDebouncedHandler(
+	db: SiaDb,
+	tracker: DirtyTracker,
+	filter: CuckooFilter,
+	debounceMs?: number,
+): (event: FileChangeEvent) => void {
+	const delay = debounceMs ?? 50;
+	const timers = new Map<string, ReturnType<typeof setTimeout>>();
+	const latestEvents = new Map<string, FileChangeEvent>();
+
+	return (event: FileChangeEvent) => {
+		// Store the latest event for this file path
+		latestEvents.set(event.filePath, event);
+
+		// Clear any existing timer for this file
+		const existing = timers.get(event.filePath);
+		if (existing != null) {
+			clearTimeout(existing);
+		}
+
+		// Set a new timer — when it fires, process the latest event
+		const timer = setTimeout(() => {
+			timers.delete(event.filePath);
+			const latest = latestEvents.get(event.filePath);
+			latestEvents.delete(event.filePath);
+
+			if (latest) {
+				// Fire and forget — errors are logged but not propagated
+				handleFileChange(db, latest, tracker, filter).catch(() => {
+					// Swallow errors in debounced handler
+				});
+			}
+		}, delay);
+
+		timers.set(event.filePath, timer);
+	};
+}

package/src/freshness/firewall.ts

@@ -0,0 +1,64 @@
+// Module: firewall — Detects high-fan-out nodes that stop dirty propagation
+//
+// Firewall nodes have more than `threshold` incoming edges. When BFS dirty
+// propagation reaches a firewall node, it marks the node as 'maybe_dirty'
+// and stops — preventing cascading invalidation through hub nodes like
+// utils/helpers.ts that are imported by hundreds of files.
+
+import type { SiaDb } from "@/graph/db-interface";
+
+/** Default edge_count threshold above which a node is a firewall. */
+const DEFAULT_FIREWALL_THRESHOLD = 50;
+
+/**
+ * Check if a node is a firewall — has more than threshold incoming edges.
+ * Firewall nodes stop dirty propagation to prevent cascading invalidation
+ * (e.g., utils/helpers.ts imported by 200 files shouldn't cascade to all 200).
+ *
+ * Uses the denormalized `edge_count` column on the entities table for O(1) lookup.
+ * Returns false for unknown nodes (not in the database).
+ */
+export async function isFirewallNode(
+	db: SiaDb,
+	nodeId: string,
+	threshold?: number,
+): Promise<boolean> {
+	const limit = threshold ?? DEFAULT_FIREWALL_THRESHOLD;
+
+	const { rows } = await db.execute("SELECT edge_count FROM graph_nodes WHERE id = ?", [nodeId]);
+
+	if (rows.length === 0) return false;
+
+	const edgeCount = rows[0].edge_count as number;
+	return edgeCount > limit;
+}
+
+/**
+ * Get the outgoing neighbors of a node for BFS propagation.
+ * Only follows active edges (t_valid_until IS NULL).
+ * Returns the neighbor node ID along with its edge_count (for firewall checks).
+ *
+ * "Outgoing" here means both directions — edges where this node is the source
+ * (from_id) or the target (to_id) — because dependency relationships are
+ * bidirectional for invalidation purposes.
+ */
+export async function getOutgoingNeighbors(
+	db: SiaDb,
+	nodeId: string,
+): Promise<Array<{ nodeId: string; edgeCount: number }>> {
+	const { rows } = await db.execute(
+		`SELECT
+			CASE WHEN e.from_id = ? THEN e.to_id ELSE e.from_id END AS neighbor_id,
+			ent.edge_count
+		 FROM graph_edges e
+		 JOIN graph_nodes ent ON ent.id = CASE WHEN e.from_id = ? THEN e.to_id ELSE e.from_id END
+		 WHERE (e.from_id = ? OR e.to_id = ?)
+		   AND e.t_valid_until IS NULL`,
+		[nodeId, nodeId, nodeId, nodeId],
+	);
+
+	return rows.map((r) => ({
+		nodeId: r.neighbor_id as string,
+		edgeCount: r.edge_count as number,
+	}));
+}

package/src/freshness/git-reconcile-layer.ts

@@ -0,0 +1,161 @@
+// Module: git-reconcile-layer — Layer 2 freshness invalidation
+//
+// Handles changes made outside the file watcher's scope: merges, rebases,
+// checkouts, stash pops, and pulls. Processes 5-50 files at once but still
+// targets O(affected nodes) not O(all nodes).
+//
+// Pipeline per git operation:
+//   1. Parse the list of changed files from the git operation
+//   2. For each file, call tracker.markDirty() with bounded BFS
+//   3. Firewall nodes (edge_count > 50) stop propagation
+
+import type { SiaDb } from "@/graph/db-interface";
+import type { DirtyTracker } from "./dirty-tracker";
+
+export interface GitOperation {
+	type: "commit" | "merge" | "rebase" | "checkout" | "stash_pop" | "pull";
+	changedFiles: string[]; // relative paths
+}
+
+/**
+ * Layer 2: Process a git operation by invalidating all affected files' nodes.
+ *
+ * Pipeline:
+ * 1. Parse the list of changed files from the git operation
+ * 2. For each file, call tracker.markDirty() with bounded BFS
+ * 3. Firewall nodes (edge_count > 50) stop propagation
+ *
+ * This is heavier than Layer 1 (may process 5-50 files at once for a merge)
+ * but still targets O(affected nodes), not O(all nodes).
+ */
+export async function handleGitOperation(
+	db: SiaDb,
+	op: GitOperation,
+	tracker: DirtyTracker,
+): Promise<{ filesProcessed: number; nodesDirtied: number }> {
+	const allDirtied = new Set<string>();
+
+	for (const filePath of op.changedFiles) {
+		const dirtied = await tracker.markDirty(db, filePath);
+		for (const nodeId of dirtied) {
+			allDirtied.add(nodeId);
+		}
+	}
+
+	return {
+		filesProcessed: op.changedFiles.length,
+		nodesDirtied: allDirtied.size,
+	};
+}
+
+/**
+ * Parse git diff output to extract list of changed files.
+ * Handles various diff formats from git commands:
+ *   - --name-only: bare file paths, one per line
+ *   - --name-status: "M\tpath" / "A\tpath" / "D\tpath" / "R100\told\tnew"
+ *   - --stat: " path | N ++--" with a summary line at the end
+ */
+export function parseGitDiff(diffOutput: string): string[] {
+	const lines = diffOutput.split("\n").filter((l) => l.trim().length > 0);
+	if (lines.length === 0) return [];
+
+	const files: string[] = [];
+
+	for (const line of lines) {
+		const trimmed = line.trim();
+
+		// Skip git --stat summary lines like "3 files changed, 12 insertions(+), 6 deletions(-)"
+		if (/^\d+\s+files?\s+changed/.test(trimmed)) {
+			continue;
+		}
+
+		// --name-status format: "M\tpath" or "R100\told\tnew"
+		if (/^[ACDMRTUX]\d*\t/.test(trimmed)) {
+			const parts = trimmed.split("\t");
+			// Skip the status prefix (parts[0])
+			for (let i = 1; i < parts.length; i++) {
+				const p = parts[i].trim();
+				if (p.length > 0) {
+					files.push(p);
+				}
+			}
+			continue;
+		}
+
+		// --stat format: " path | N ++--"
+		if (trimmed.includes("|")) {
+			const pipeIdx = trimmed.indexOf("|");
+			const path = trimmed.slice(0, pipeIdx).trim();
+			if (path.length > 0 && !path.includes(" changed")) {
+				files.push(path);
+			}
+			continue;
+		}
+
+		// --name-only format: bare file path (contains / or . but no spaces that look like a summary)
+		if ((trimmed.includes("/") || trimmed.includes(".")) && !trimmed.includes(" files changed")) {
+			files.push(trimmed);
+		}
+	}
+
+	return files;
+}
+
+/** Git subcommands that modify the working tree and should trigger Layer 2. */
+const GIT_OP_PATTERNS: Array<{
+	pattern: RegExp;
+	type: GitOperation["type"];
+}> = [
+	{ pattern: /^git\s+commit\b/, type: "commit" },
+	{ pattern: /^git\s+merge\b/, type: "merge" },
+	{ pattern: /^git\s+rebase\b/, type: "rebase" },
+	{ pattern: /^git\s+checkout\b/, type: "checkout" },
+	{ pattern: /^git\s+switch\b/, type: "checkout" },
+	{ pattern: /^git\s+stash\s+pop\b/, type: "stash_pop" },
+	{ pattern: /^git\s+stash\s+apply\b/, type: "stash_pop" },
+	{ pattern: /^git\s+pull\b/, type: "pull" },
+];
+
+/** Git subcommands that are read-only and should NOT trigger Layer 2. */
+const GIT_READONLY_PATTERNS: RegExp[] = [
+	/^git\s+status\b/,
+	/^git\s+log\b/,
+	/^git\s+diff\b/,
+	/^git\s+branch\b/,
+	/^git\s+show\b/,
+	/^git\s+remote\b/,
+	/^git\s+fetch\b/,
+	/^git\s+tag\b/,
+	/^git\s+blame\b/,
+	/^git\s+stash\s+list\b/,
+];
+
+/**
+ * Detect if a bash command is a git operation that should trigger Layer 2.
+ * Returns a GitOperation stub (with empty changedFiles — caller must populate
+ * from `git diff` output) or null if the command is not a relevant git op.
+ */
+export function isGitOperation(command: string): GitOperation | null {
+	const trimmed = command.trim();
+
+	// Not a git command at all
+	if (!trimmed.startsWith("git ")) {
+		return null;
+	}
+
+	// Explicitly skip read-only git commands
+	for (const ro of GIT_READONLY_PATTERNS) {
+		if (ro.test(trimmed)) {
+			return null;
+		}
+	}
+
+	// Match against known write operations
+	for (const { pattern, type } of GIT_OP_PATTERNS) {
+		if (pattern.test(trimmed)) {
+			return { type, changedFiles: [] };
+		}
+	}
+
+	return null;
+}