@rkarim08/sia 1.0.0

package/src/community/leiden.ts

@@ -0,0 +1,462 @@
+// Module: leiden — community detection via a simplified Louvain/Leiden pass
+
+import { randomUUID } from "node:crypto";
+import type { SiaDb } from "@/graph/db-interface";
+
+export interface LeidenOpts {
+	/** Custom resolution parameters per level (fine → coarse). Defaults: [2.0, 1.0, 0.5]. */
+	resolutions?: number[];
+}
+
+export interface CommunityResult {
+	/** Community counts per level (index = level). */
+	levels: number[];
+	/** Total communities created across all levels. */
+	totalCommunities: number;
+	/** Wall-clock duration for the run. */
+	durationMs: number;
+}
+
+interface EntityRow {
+	id: string;
+	package_path: string | null;
+	importance: number;
+}
+
+interface EdgeRow {
+	fromId: string;
+	toId: string;
+	type: string;
+	weight: number;
+}
+
+interface Unit {
+	id: string;
+	members: Set<string>;
+}
+
+interface DetectedCommunity {
+	id: string;
+	level: number;
+	members: Set<string>;
+	parentId: string | null;
+	packagePath: string | null;
+}
+
+const DEFAULT_RESOLUTIONS = [2.0, 1.0, 0.5];
+
+const EDGE_TYPE_WEIGHTS: Record<string, number> = {
+	calls: 0.5,
+	imports: 0.5,
+	inherits_from: 0.5,
+	contains: 0.5,
+	depends_on: 0.5,
+	co_occurrence: 0.3,
+	cochange: 0.2,
+	"git-cochange": 0.2,
+};
+
+function weightForEdge(type: string | null, base: number): number {
+	// Specification notes equal weights for now, but keep the map for future tuning.
+	const typeWeight = type ? (EDGE_TYPE_WEIGHTS[type] ?? 0.3) : 0.3;
+	return base * typeWeight;
+}
+
+function dedupeEdges(
+	edges: EdgeRow[],
+	validNodes: Set<string>,
+): Array<{ a: string; b: string; w: number }> {
+	const map = new Map<string, number>();
+	for (const edge of edges) {
+		if (!validNodes.has(edge.fromId) || !validNodes.has(edge.toId)) continue;
+		const keyParts = [edge.fromId, edge.toId].sort();
+		const key = `${keyParts[0]}::${keyParts[1]}`;
+		const baseWeight = typeof edge.weight === "number" ? edge.weight : 1;
+		const w = weightForEdge(edge.type, baseWeight);
+		map.set(key, (map.get(key) ?? 0) + w);
+	}
+	const result: Array<{ a: string; b: string; w: number }> = [];
+	for (const [key, w] of map) {
+		const [a, b] = key.split("::");
+		result.push({ a, b, w });
+	}
+	return result;
+}
+
+function buildUnitAdjacency(
+	edges: Array<{ a: string; b: string; w: number }>,
+	entityToUnit: Map<string, string>,
+): Map<string, Map<string, number>> {
+	const adj = new Map<string, Map<string, number>>();
+	const ensure = (id: string): Map<string, number> => {
+		let entry = adj.get(id);
+		if (!entry) {
+			entry = new Map();
+			adj.set(id, entry);
+		}
+		return entry;
+	};
+
+	for (const { a, b, w } of edges) {
+		const ua = entityToUnit.get(a);
+		const ub = entityToUnit.get(b);
+		if (!ua || !ub) continue;
+
+		if (ua === ub) {
+			const selfAdj = ensure(ua);
+			selfAdj.set(ua, (selfAdj.get(ua) ?? 0) + w);
+			continue;
+		}
+
+		const adjA = ensure(ua);
+		adjA.set(ub, (adjA.get(ub) ?? 0) + w);
+
+		const adjB = ensure(ub);
+		adjB.set(ua, (adjB.get(ua) ?? 0) + w);
+	}
+
+	return adj;
+}
+
+function louvain(
+	units: Unit[],
+	adjacency: Map<string, Map<string, number>>,
+	resolution: number,
+	maxIterations = 100,
+): Map<string, string> {
+	const community = new Map<string, string>();
+	const sumTot = new Map<string, number>();
+	const degrees = new Map<string, number>();
+
+	for (const unit of units) {
+		community.set(unit.id, unit.id);
+		const deg = Array.from(adjacency.get(unit.id)?.values() ?? []).reduce((a, b) => a + b, 0);
+		degrees.set(unit.id, deg);
+		sumTot.set(unit.id, deg);
+	}
+
+	const m2 = Array.from(degrees.values()).reduce((a, b) => a + b, 0) || 1; // 2 * total edge weight
+
+	let iterations = 0;
+	let moved = true;
+	while (moved && iterations < maxIterations) {
+		iterations++;
+		moved = false;
+		for (const unit of units) {
+			const node = unit.id;
+			const nodeComm = community.get(node) ?? node;
+			const nodeDegree = degrees.get(node) ?? 0;
+
+			// Temporarily remove node from its community
+			sumTot.set(nodeComm, (sumTot.get(nodeComm) ?? 0) - nodeDegree);
+
+			const neighborWeights = adjacency.get(node) ?? new Map<string, number>();
+			const communityWeights = new Map<string, number>();
+			for (const [neighbor, weight] of neighborWeights) {
+				const comm = community.get(neighbor);
+				if (!comm) continue;
+				communityWeights.set(comm, (communityWeights.get(comm) ?? 0) + weight);
+			}
+
+			let bestComm = nodeComm;
+			let bestGain = 0;
+			for (const [comm, kin] of communityWeights) {
+				const tot = sumTot.get(comm) ?? 0;
+				const gain = kin - (resolution * nodeDegree * tot) / m2;
+				if (gain > bestGain + 1e-9) {
+					bestGain = gain;
+					bestComm = comm;
+				}
+			}
+
+			sumTot.set(bestComm, (sumTot.get(bestComm) ?? 0) + nodeDegree);
+			community.set(node, bestComm);
+			if (bestComm !== nodeComm) {
+				moved = true;
+			}
+		}
+	}
+
+	// Normalize community identifiers to stable compact ids
+	const normalized = new Map<string, string>();
+	let idx = 0;
+	for (const comm of new Set(community.values())) {
+		normalized.set(comm, `c${idx++}`);
+	}
+
+	const assignment = new Map<string, string>();
+	for (const [node, comm] of community) {
+		assignment.set(node, normalized.get(comm) ?? comm);
+	}
+	return assignment;
+}
+
+function refinePartition(
+	assignment: Map<string, string>,
+	adjacency: Map<string, Map<string, number>>,
+): Map<string, string> {
+	// Group nodes by community
+	const communities = new Map<string, string[]>();
+	for (const [node, comm] of assignment) {
+		let members = communities.get(comm);
+		if (!members) {
+			members = [];
+			communities.set(comm, members);
+		}
+		members.push(node);
+	}
+
+	const refined = new Map<string, string>();
+	for (const [comm, members] of communities) {
+		if (members.length <= 1) {
+			for (const m of members) refined.set(m, comm);
+			continue;
+		}
+
+		// BFS to find connected components within this community
+		const memberSet = new Set(members);
+		const visited = new Set<string>();
+		let componentIdx = 0;
+
+		for (const start of members) {
+			if (visited.has(start)) continue;
+			const component: string[] = [];
+			const queue = [start];
+			visited.add(start);
+
+			while (queue.length > 0) {
+				const node = queue.shift();
+				if (!node) break;
+				component.push(node);
+				const neighbors = adjacency.get(node);
+				if (neighbors) {
+					for (const [neighbor] of neighbors) {
+						if (memberSet.has(neighbor) && !visited.has(neighbor)) {
+							visited.add(neighbor);
+							queue.push(neighbor);
+						}
+					}
+				}
+			}
+
+			const compId = componentIdx === 0 ? comm : `${comm}_${componentIdx}`;
+			for (const node of component) {
+				refined.set(node, compId);
+			}
+			componentIdx++;
+		}
+	}
+
+	return refined;
+}
+
+function determinePackagePath(
+	members: Set<string>,
+	entityPackages: Map<string, string | null>,
+): string | null {
+	let current: string | null | undefined;
+	for (const id of members) {
+		const pkg = entityPackages.get(id) ?? null;
+		if (pkg === null) return null;
+		if (current === undefined) {
+			current = pkg;
+		} else if (current !== pkg) {
+			return null;
+		}
+	}
+	return current ?? null;
+}
+
+function assignParents(levels: DetectedCommunity[][]): void {
+	for (let i = 0; i < levels.length - 1; i++) {
+		const parents = levels[i + 1];
+		for (const community of levels[i]) {
+			const parent = parents.find((p) => {
+				for (const member of community.members) {
+					if (!p.members.has(member)) return false;
+				}
+				return true;
+			});
+			community.parentId = parent?.id ?? null;
+		}
+	}
+}
+
+export async function detectCommunities(
+	db: SiaDb,
+	opts: LeidenOpts = {},
+): Promise<CommunityResult> {
+	const start = Date.now();
+	const resolutions = opts.resolutions ?? DEFAULT_RESOLUTIONS;
+
+	const entityResult = await db.execute(
+		`SELECT id, package_path, importance
+                 FROM graph_nodes
+                 WHERE t_valid_until IS NULL AND archived_at IS NULL`,
+	);
+	const entities = entityResult.rows as unknown as EntityRow[];
+	if (entities.length === 0) {
+		return { levels: resolutions.map(() => 0), totalCommunities: 0, durationMs: 0 };
+	}
+
+	const entityPackages = new Map<string, string | null>();
+	for (const entity of entities) {
+		entityPackages.set(entity.id, entity.package_path);
+	}
+	const validIds = new Set(entities.map((e) => e.id));
+
+	const edgeResult = await db.execute(
+		`SELECT from_id as fromId, to_id as toId, type, weight
+                 FROM graph_edges
+                 WHERE t_valid_until IS NULL`,
+	);
+	const edges = edgeResult.rows as unknown as EdgeRow[];
+	const undirectedEdges = dedupeEdges(edges, validIds);
+
+	let units: Unit[] = entities.map((e) => ({ id: e.id, members: new Set([e.id]) }));
+	const levelCommunities: DetectedCommunity[][] = [];
+
+	for (let level = 0; level < resolutions.length; level++) {
+		// Per-package Level 0: group entities by package_path, run separate louvain+refine per package
+		if (level === 0 && entityPackages.size > 0) {
+			const byPackage = new Map<string, Unit[]>();
+			for (const unit of units) {
+				const pkg = entityPackages.get([...unit.members][0]) ?? "__root__";
+				let pkgList = byPackage.get(pkg);
+				if (!pkgList) {
+					pkgList = [];
+					byPackage.set(pkg, pkgList);
+				}
+				pkgList.push(unit);
+			}
+
+			const allDetected: DetectedCommunity[] = [];
+			for (const [pkg, pkgUnits] of byPackage) {
+				const pkgEntityToUnit = new Map<string, string>();
+				for (const unit of pkgUnits) {
+					for (const member of unit.members) {
+						pkgEntityToUnit.set(member, unit.id);
+					}
+				}
+
+				const pkgAdj = buildUnitAdjacency(undirectedEdges, pkgEntityToUnit);
+				const pkgAssignment = louvain(pkgUnits, pkgAdj, resolutions[level]);
+				const pkgRefined = refinePartition(pkgAssignment, pkgAdj);
+
+				const communityMembers = new Map<string, Set<string>>();
+				for (const unit of pkgUnits) {
+					const commKey = pkgRefined.get(unit.id) ?? unit.id;
+					if (!communityMembers.has(commKey)) communityMembers.set(commKey, new Set());
+					for (const member of unit.members) {
+						communityMembers.get(commKey)?.add(member);
+					}
+				}
+
+				for (const [_key, members] of communityMembers) {
+					const id = randomUUID();
+					allDetected.push({
+						id,
+						level: 0,
+						members,
+						parentId: null,
+						packagePath: pkg === "__root__" ? null : pkg,
+					});
+				}
+			}
+
+			levelCommunities.push(allDetected);
+			units = allDetected.map((c) => ({ id: c.id, members: c.members }));
+			continue;
+		}
+
+		const entityToUnit = new Map<string, string>();
+		for (const unit of units) {
+			for (const member of unit.members) {
+				entityToUnit.set(member, unit.id);
+			}
+		}
+
+		const unitAdj = buildUnitAdjacency(undirectedEdges, entityToUnit);
+		const assignment = louvain(units, unitAdj, resolutions[level]);
+		const refinedAssignment = refinePartition(assignment, unitAdj);
+
+		const communityMembers = new Map<string, Set<string>>();
+		const communityUnits = new Map<string, Unit[]>();
+		for (const unit of units) {
+			const commKey = refinedAssignment.get(unit.id) ?? unit.id;
+			if (!communityMembers.has(commKey)) {
+				communityMembers.set(commKey, new Set());
+				communityUnits.set(commKey, []);
+			}
+			for (const member of unit.members) {
+				communityMembers.get(commKey)?.add(member);
+			}
+			communityUnits.get(commKey)?.push(unit);
+		}
+
+		const detected: DetectedCommunity[] = [];
+		for (const [_key, members] of communityMembers) {
+			const id = randomUUID();
+			const pkg = determinePackagePath(members, entityPackages);
+			detected.push({
+				id,
+				level,
+				members,
+				parentId: null,
+				packagePath: level === 0 ? pkg : null,
+			});
+		}
+
+		levelCommunities.push(detected);
+
+		// Prepare units for next level
+		units = detected.map((c) => ({ id: c.id, members: c.members }));
+	}
+
+	assignParents(levelCommunities);
+
+	// Persist communities and memberships
+	await db.transaction(async (tx) => {
+		await tx.execute("DELETE FROM community_members");
+		await tx.execute("DELETE FROM communities");
+
+		const now = Date.now();
+		const levelsDescending = [...levelCommunities].reverse();
+		for (const level of levelsDescending) {
+			for (const community of level) {
+				const memberCount = community.members.size;
+				await tx.execute(
+					`INSERT INTO communities (
+                                                id, level, parent_id, summary, summary_hash,
+                                                member_count, last_summary_member_count,
+                                                package_path, created_at, updated_at
+                                        ) VALUES (?, ?, ?, NULL, NULL, ?, 0, ?, ?, ?)`,
+					[
+						community.id,
+						community.level,
+						community.parentId,
+						memberCount,
+						community.packagePath,
+						now,
+						now,
+					],
+				);
+
+				for (const member of community.members) {
+					await tx.execute(
+						`INSERT INTO community_members (community_id, entity_id, level)
+                                                 VALUES (?, ?, ?)`,
+						[community.id, member, community.level],
+					);
+				}
+			}
+		}
+	});
+
+	const levels = levelCommunities.map((c) => c.length);
+	return {
+		levels,
+		totalCommunities: levels.reduce((a, b) => a + b, 0),
+		durationMs: Date.now() - start,
+	};
+}

package/src/community/raptor.ts

@@ -0,0 +1,210 @@
+// Module: raptor — RAPTOR summary tree construction
+
+import { createHash } from "node:crypto";
+import type { SiaDb } from "@/graph/db-interface";
+import type { LlmClient } from "@/shared/llm-client";
+
+interface EntityRow {
+	id: string;
+	content: string;
+	summary: string;
+	t_valid_until: number | null;
+}
+
+interface CommunityRow {
+	id: string;
+	summary: string | null;
+}
+
+interface SummaryRow {
+	id: string;
+	level: number;
+	scopeId: string;
+	content: string;
+	tokenCount: number;
+	contentHash: string;
+	expiresAt: number | null;
+}
+
+function wordCount(text: string): number {
+	const trimmed = text.trim();
+	if (!trimmed) return 0;
+	return trimmed.split(/\s+/).length;
+}
+
+function hashContent(content: string): string {
+	return createHash("sha256").update(content).digest("hex");
+}
+
+async function upsertSummary(db: SiaDb, row: SummaryRow, now: number): Promise<void> {
+	await db.execute(
+		`INSERT INTO summary_tree (id, level, scope_id, content, content_hash, token_count, created_at, expires_at)
+                 VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+                 ON CONFLICT(id) DO UPDATE SET
+                        level = excluded.level,
+                        scope_id = excluded.scope_id,
+                        content = excluded.content,
+                        content_hash = excluded.content_hash,
+                        token_count = excluded.token_count,
+                        created_at = excluded.created_at,
+                        expires_at = excluded.expires_at`,
+		[
+			row.id,
+			row.level,
+			row.scopeId,
+			row.content,
+			row.contentHash,
+			row.tokenCount,
+			now,
+			row.expiresAt,
+		],
+	);
+}
+
+export async function buildSummaryTree(db: SiaDb, llmClient?: LlmClient): Promise<void> {
+	const now = Date.now();
+
+	const entityResult = await db.execute(
+		`SELECT id, content, summary, t_valid_until
+		 FROM graph_nodes
+		 WHERE t_valid_until IS NULL AND archived_at IS NULL`,
+	);
+	const communityResult = await db.execute(`SELECT id, summary FROM communities`);
+
+	const entities = entityResult.rows as unknown as EntityRow[];
+	const communities = communityResult.rows as unknown as CommunityRow[];
+
+	await db.transaction(async (tx) => {
+		// Level 0 — raw entity content (eager)
+		for (const entity of entities) {
+			const level0: SummaryRow = {
+				id: `lvl0:${entity.id}`,
+				level: 0,
+				scopeId: entity.id,
+				content: entity.content,
+				tokenCount: wordCount(entity.content),
+				contentHash: hashContent(entity.content),
+				expiresAt: null,
+			};
+			await upsertSummary(tx, level0, now);
+		}
+
+		// Level 1 — lazy (generated on-demand via getOrCreateLevel1Summary)
+
+		// Level 2 — community/module summaries via LLM
+		for (const community of communities) {
+			let content: string;
+			if (llmClient && community.summary?.trim()) {
+				content = await llmClient.summarize(
+					`Rewrite this community summary as a coherent paragraph describing the module's purpose:\n\n${community.summary}`,
+				);
+			} else {
+				content = community.summary?.trim()
+					? community.summary
+					: `Community ${community.id} has no summary.`;
+			}
+			const level2: SummaryRow = {
+				id: `lvl2:${community.id}`,
+				level: 2,
+				scopeId: community.id,
+				content,
+				tokenCount: wordCount(content),
+				contentHash: hashContent(content),
+				expiresAt: null,
+			};
+			await upsertSummary(tx, level2, now);
+		}
+
+		// Level 3 — architectural overview (only regenerate weekly)
+		if (communities.length > 0) {
+			const existingLevel3 = await tx.execute(
+				"SELECT created_at FROM summary_tree WHERE id = 'lvl3:overview'",
+			);
+			const lastLevel3At = (existingLevel3.rows[0]?.created_at as number) ?? 0;
+			const oneWeek = 7 * 24 * 60 * 60 * 1000;
+			if (now - lastLevel3At < oneWeek && existingLevel3.rows.length > 0) {
+				// Skip Level 3 regeneration — generated less than 7 days ago
+			} else {
+				const overviewBody = communities
+					.map((c) => {
+						const text =
+							c.summary && c.summary.trim().length > 0
+								? c.summary.trim()
+								: `Community ${c.id} has no summary.`;
+						return `- ${text}`;
+					})
+					.join("\n");
+				const overview = `Architectural overview:\n${overviewBody}`;
+				const level3: SummaryRow = {
+					id: "lvl3:overview",
+					level: 3,
+					scopeId: "all",
+					content: overview,
+					tokenCount: wordCount(overview),
+					contentHash: hashContent(overview),
+					expiresAt: null,
+				};
+				await upsertSummary(tx, level3, now);
+			}
+		}
+	});
+}
+
+/**
+ * Lazily get or create a Level 1 summary for a single entity.
+ * Returns the summary text, or null if the entity does not exist.
+ */
+export async function getOrCreateLevel1Summary(
+	db: SiaDb,
+	entityId: string,
+	llmClient?: LlmClient,
+): Promise<string | null> {
+	// Check if Level 1 summary already exists and is not expired
+	const existing = await db.execute(
+		"SELECT content FROM summary_tree WHERE id = ? AND expires_at IS NULL",
+		[`lvl1:${entityId}`],
+	);
+	if (existing.rows.length > 0) {
+		return existing.rows[0].content as string;
+	}
+
+	// Fetch entity
+	const entityResult = await db.execute(
+		"SELECT id, content, summary, t_valid_until FROM graph_nodes WHERE id = ?",
+		[entityId],
+	);
+	if (entityResult.rows.length === 0) return null;
+	const entity = entityResult.rows[0] as {
+		id: string;
+		content: string;
+		summary: string;
+		t_valid_until: number | null;
+	};
+
+	// Generate via LLM or fallback
+	let summaryText: string;
+	if (llmClient) {
+		summaryText = await llmClient.summarize(
+			`Write a one-paragraph summary of this code entity:\n\nName: ${entity.id}\nContent: ${entity.content.slice(0, 500)}`,
+		);
+	} else {
+		summaryText = entity.summary?.trim() ? entity.summary : entity.content.slice(0, 240);
+	}
+
+	const now = Date.now();
+	await upsertSummary(
+		db,
+		{
+			id: `lvl1:${entityId}`,
+			level: 1,
+			scopeId: entityId,
+			content: summaryText,
+			tokenCount: wordCount(summaryText),
+			contentHash: hashContent(summaryText),
+			expiresAt: entity.t_valid_until ? now : null,
+		},
+		now,
+	);
+
+	return summaryText;
+}