@rkarim08/sia 1.0.0

package/src/ast/languages.ts

@@ -0,0 +1,408 @@
+// Module: languages — Language registry for AST extraction
+import { basename, extname } from "node:path";
+import type { AdditionalLanguage } from "@/shared/config";
+
+/** Extraction tier: A = full, B = partial, C = schema-only, D = manifest */
+export type ExtractionTier = "A" | "B" | "C" | "D";
+
+/** Special handling hints for certain languages */
+export type SpecialHandling =
+	| "c-include-paths"
+	| "csharp-project"
+	| "sql-schema"
+	| "prisma-schema"
+	| "project-manifest";
+
+/** Extractor capabilities for a language */
+export interface Extractors {
+	functions: boolean;
+	classes: boolean;
+	imports: boolean;
+	calls: boolean;
+}
+
+/** Full language configuration */
+export interface LanguageConfig {
+	name: string;
+	extensions: string[];
+	treeSitterGrammar: string;
+	tier: ExtractionTier;
+	extractors: Extractors;
+	specialHandling?: SpecialHandling;
+	nativePackage?: string;
+	wasmFile?: string;
+	queryDir?: string;
+	parserEntrypoint?: string;
+}
+
+/** LanguageConfig with all tree-sitter resolution fields populated */
+export interface ResolvedLanguageConfig extends LanguageConfig {
+	nativePackage: string;
+	wasmFile: string;
+	queryDir: string;
+}
+
+/**
+ * Resolve a LanguageConfig, deriving nativePackage, wasmFile, and queryDir
+ * from treeSitterGrammar if not explicitly provided.
+ */
+export function resolveLanguageConfig(lang: LanguageConfig): ResolvedLanguageConfig {
+	return {
+		...lang,
+		nativePackage: lang.nativePackage ?? lang.treeSitterGrammar,
+		wasmFile: lang.wasmFile ?? `${lang.treeSitterGrammar}.wasm`,
+		queryDir: lang.queryDir ?? lang.name,
+	};
+}
+
+/** The canonical language registry keyed by language name */
+export type LanguageRegistry = Record<string, LanguageConfig>;
+
+// ---------- Tier A (15 languages) ----------
+// All have functions: true, imports: true, calls: true
+// All have classes: true except go, rust, elixir
+
+const TIER_A: LanguageConfig[] = [
+	{
+		name: "typescript",
+		extensions: [".ts"],
+		treeSitterGrammar: "tree-sitter-typescript",
+		tier: "A",
+		extractors: { functions: true, classes: true, imports: true, calls: true },
+	},
+	{
+		name: "tsx",
+		extensions: [".tsx"],
+		treeSitterGrammar: "tree-sitter-tsx",
+		tier: "A",
+		extractors: { functions: true, classes: true, imports: true, calls: true },
+	},
+	{
+		name: "javascript",
+		extensions: [".js", ".mjs", ".cjs"],
+		treeSitterGrammar: "tree-sitter-javascript",
+		tier: "A",
+		extractors: { functions: true, classes: true, imports: true, calls: true },
+	},
+	{
+		name: "jsx",
+		extensions: [".jsx"],
+		treeSitterGrammar: "tree-sitter-javascript",
+		tier: "A",
+		extractors: { functions: true, classes: true, imports: true, calls: true },
+	},
+	{
+		name: "python",
+		extensions: [".py"],
+		treeSitterGrammar: "tree-sitter-python",
+		tier: "A",
+		extractors: { functions: true, classes: true, imports: true, calls: true },
+	},
+	{
+		name: "go",
+		extensions: [".go"],
+		treeSitterGrammar: "tree-sitter-go",
+		tier: "A",
+		extractors: { functions: true, classes: false, imports: true, calls: true },
+	},
+	{
+		name: "rust",
+		extensions: [".rs"],
+		treeSitterGrammar: "tree-sitter-rust",
+		tier: "A",
+		extractors: { functions: true, classes: false, imports: true, calls: true },
+	},
+	{
+		name: "java",
+		extensions: [".java"],
+		treeSitterGrammar: "tree-sitter-java",
+		tier: "A",
+		extractors: { functions: true, classes: true, imports: true, calls: true },
+	},
+	{
+		name: "kotlin",
+		extensions: [".kt", ".kts"],
+		treeSitterGrammar: "tree-sitter-kotlin",
+		tier: "A",
+		extractors: { functions: true, classes: true, imports: true, calls: true },
+	},
+	{
+		name: "swift",
+		extensions: [".swift"],
+		treeSitterGrammar: "tree-sitter-swift",
+		tier: "A",
+		extractors: { functions: true, classes: true, imports: true, calls: true },
+	},
+	{
+		name: "php",
+		extensions: [".php"],
+		treeSitterGrammar: "tree-sitter-php",
+		tier: "A",
+		extractors: { functions: true, classes: true, imports: true, calls: true },
+	},
+	{
+		name: "ruby",
+		extensions: [".rb"],
+		treeSitterGrammar: "tree-sitter-ruby",
+		tier: "A",
+		extractors: { functions: true, classes: true, imports: true, calls: true },
+	},
+	{
+		name: "scala",
+		extensions: [".scala"],
+		treeSitterGrammar: "tree-sitter-scala",
+		tier: "A",
+		extractors: { functions: true, classes: true, imports: true, calls: true },
+	},
+	{
+		name: "elixir",
+		extensions: [".ex", ".exs"],
+		treeSitterGrammar: "tree-sitter-elixir",
+		tier: "A",
+		extractors: { functions: true, classes: false, imports: true, calls: true },
+	},
+	{
+		name: "dart",
+		extensions: [".dart"],
+		treeSitterGrammar: "tree-sitter-dart",
+		tier: "A",
+		extractors: { functions: true, classes: true, imports: true, calls: true },
+	},
+];
+
+// ---------- Tier B (10 languages) ----------
+// All have calls: false. Most have classes: false except cpp and csharp.
+
+const TIER_B: LanguageConfig[] = [
+	{
+		name: "c",
+		extensions: [".c", ".h"],
+		treeSitterGrammar: "tree-sitter-c",
+		tier: "B",
+		extractors: { functions: true, classes: false, imports: true, calls: false },
+		specialHandling: "c-include-paths",
+	},
+	{
+		name: "cpp",
+		extensions: [".cpp", ".cc", ".cxx", ".hpp", ".hxx", ".h++"],
+		treeSitterGrammar: "tree-sitter-cpp",
+		tier: "B",
+		extractors: { functions: true, classes: true, imports: true, calls: false },
+		specialHandling: "c-include-paths",
+	},
+	{
+		name: "csharp",
+		extensions: [".cs"],
+		treeSitterGrammar: "tree-sitter-c-sharp",
+		tier: "B",
+		extractors: { functions: true, classes: true, imports: true, calls: false },
+		specialHandling: "csharp-project",
+	},
+	{
+		name: "bash",
+		extensions: [".sh", ".bash", ".zsh", ".fish"],
+		treeSitterGrammar: "tree-sitter-bash",
+		tier: "B",
+		extractors: { functions: true, classes: false, imports: true, calls: false },
+	},
+	{
+		name: "lua",
+		extensions: [".lua"],
+		treeSitterGrammar: "tree-sitter-lua",
+		tier: "B",
+		extractors: { functions: true, classes: false, imports: true, calls: false },
+	},
+	{
+		name: "zig",
+		extensions: [".zig"],
+		treeSitterGrammar: "tree-sitter-zig",
+		tier: "B",
+		extractors: { functions: true, classes: false, imports: true, calls: false },
+	},
+	{
+		name: "r",
+		extensions: [".r", ".R"],
+		treeSitterGrammar: "tree-sitter-r",
+		tier: "B",
+		extractors: { functions: true, classes: false, imports: true, calls: false },
+	},
+	{
+		name: "ocaml",
+		extensions: [".ml", ".mli"],
+		treeSitterGrammar: "tree-sitter-ocaml",
+		tier: "B",
+		extractors: { functions: true, classes: false, imports: true, calls: false },
+	},
+	{
+		name: "haskell",
+		extensions: [".hs", ".lhs"],
+		treeSitterGrammar: "tree-sitter-haskell",
+		tier: "B",
+		extractors: { functions: true, classes: false, imports: true, calls: false },
+	},
+	{
+		name: "perl",
+		extensions: [".pl", ".pm"],
+		treeSitterGrammar: "tree-sitter-perl",
+		tier: "B",
+		extractors: { functions: true, classes: false, imports: true, calls: false },
+	},
+];
+
+// ---------- Tier C (2 languages) ----------
+
+const TIER_C: LanguageConfig[] = [
+	{
+		name: "sql",
+		extensions: [".sql"],
+		treeSitterGrammar: "tree-sitter-sql",
+		tier: "C",
+		extractors: { functions: false, classes: false, imports: false, calls: false },
+		specialHandling: "sql-schema",
+	},
+	{
+		name: "prisma",
+		extensions: [".prisma"],
+		treeSitterGrammar: "tree-sitter-prisma",
+		tier: "C",
+		extractors: { functions: false, classes: false, imports: false, calls: false },
+		specialHandling: "prisma-schema",
+	},
+];
+
+// ---------- Tier D (3 languages) ----------
+
+const TIER_D: LanguageConfig[] = [
+	{
+		name: "cargo_toml",
+		extensions: ["Cargo.toml"],
+		treeSitterGrammar: "tree-sitter-toml",
+		tier: "D",
+		extractors: { functions: false, classes: false, imports: false, calls: false },
+		specialHandling: "project-manifest",
+	},
+	{
+		name: "go_mod",
+		extensions: ["go.mod"],
+		treeSitterGrammar: "tree-sitter-gomod",
+		tier: "D",
+		extractors: { functions: false, classes: false, imports: false, calls: false },
+		specialHandling: "project-manifest",
+	},
+	{
+		name: "pyproject",
+		extensions: ["pyproject.toml", "setup.py", "setup.cfg"],
+		treeSitterGrammar: "tree-sitter-toml",
+		tier: "D",
+		extractors: { functions: false, classes: false, imports: false, calls: false },
+		specialHandling: "project-manifest",
+	},
+];
+
+/** The complete language registry (30 languages) */
+export const LANGUAGE_REGISTRY: LanguageRegistry = {};
+
+// Populate the registry from the tier arrays
+for (const lang of [...TIER_A, ...TIER_B, ...TIER_C, ...TIER_D]) {
+	LANGUAGE_REGISTRY[lang.name] = lang;
+}
+
+// ---------- Extension lookup cache ----------
+
+let extensionCache: Map<string, LanguageConfig> | null = null;
+
+function buildExtensionCache(registry: LanguageRegistry): Map<string, LanguageConfig> {
+	const cache = new Map<string, LanguageConfig>();
+	for (const lang of Object.values(registry)) {
+		for (const ext of lang.extensions) {
+			cache.set(ext.toLowerCase(), lang);
+		}
+	}
+	return cache;
+}
+
+function getCache(registry: LanguageRegistry): Map<string, LanguageConfig> {
+	if (registry === LANGUAGE_REGISTRY && extensionCache !== null) {
+		return extensionCache;
+	}
+	const cache = buildExtensionCache(registry);
+	if (registry === LANGUAGE_REGISTRY) {
+		extensionCache = cache;
+	}
+	return cache;
+}
+
+/**
+ * Look up a LanguageConfig by file extension (e.g. ".ts", ".py").
+ * Extension should include the leading dot for normal extensions,
+ * or be a full filename for manifest files (e.g. "Cargo.toml").
+ *
+ * Accepts an optional registry for custom/merged registries.
+ */
+export function getLanguageByExtension(
+	ext: string,
+	registry: LanguageRegistry = LANGUAGE_REGISTRY,
+): LanguageConfig | null {
+	const cache = getCache(registry);
+	return cache.get(ext.toLowerCase()) ?? null;
+}
+
+/**
+ * Resolve a file path to its LanguageConfig (convenience wrapper).
+ * Handles both extension-based and filename-based lookups (e.g. "Cargo.toml").
+ */
+export function getLanguageForFile(
+	filePath: string,
+	registry: LanguageRegistry = LANGUAGE_REGISTRY,
+): LanguageConfig | null {
+	const base = basename(filePath).toLowerCase();
+	const cache = getCache(registry);
+
+	// Try full filename first (for manifests like Cargo.toml, go.mod)
+	if (cache.has(base)) {
+		return cache.get(base) ?? null;
+	}
+
+	// Fall back to extension
+	const ext = extname(base);
+	if (ext) {
+		return cache.get(ext.toLowerCase()) ?? null;
+	}
+
+	return null;
+}
+
+/**
+ * Merge additional languages into a registry.
+ * Does NOT overwrite languages that already exist (by name).
+ * Invalidates the extension cache so new languages are discoverable.
+ */
+export function mergeAdditionalLanguages(
+	registry: LanguageRegistry,
+	additions: AdditionalLanguage[],
+): void {
+	for (const lang of additions) {
+		// Do not overwrite existing languages
+		if (registry[lang.name]) {
+			continue;
+		}
+
+		const tier: ExtractionTier =
+			lang.tier === "A" || lang.tier === "B" || lang.tier === "C" || lang.tier === "D"
+				? (lang.tier as ExtractionTier)
+				: "D";
+
+		const config: LanguageConfig = {
+			name: lang.name,
+			extensions: lang.extensions.map((e) => e.toLowerCase()),
+			treeSitterGrammar: lang.grammar,
+			tier,
+			extractors: { functions: false, classes: false, imports: false, calls: false },
+		};
+
+		registry[lang.name] = config;
+	}
+
+	// Invalidate cache so new extensions are picked up
+	extensionCache = null;
+}

package/src/ast/pagerank-builder.ts

@@ -0,0 +1,125 @@
+import type { SiaDb } from "@/graph/db-interface";
+
+export interface PageRankResult {
+	iterations: number;
+	converged: boolean;
+	finalDelta: number;
+	nodesScored: number;
+}
+
+interface EdgeRow {
+	from_id: string;
+	to_id: string;
+}
+
+function buildTeleportVector(nodes: string[], activeFileIds: string[]): Map<string, number> {
+	const bias = new Set(activeFileIds ?? []);
+	if (bias.size === 0) {
+		const uniform = 1 / nodes.length;
+		return new Map(nodes.map((id) => [id, uniform]));
+	}
+
+	const epsilon = 0.01;
+	const activeWeight = (1 - epsilon) / bias.size;
+	const passiveWeight = epsilon / nodes.length;
+	return new Map(nodes.map((id) => [id, bias.has(id) ? activeWeight : passiveWeight]));
+}
+
+export async function computePageRank(
+	db: SiaDb,
+	activeFileIds: string[] = [],
+): Promise<PageRankResult> {
+	const { rows } = await db.execute(
+		"SELECT from_id, to_id FROM graph_edges WHERE t_valid_until IS NULL AND type IN ('calls','imports','inherits_from')",
+	);
+	const edges = rows as unknown as EdgeRow[];
+
+	const nodes = new Set<string>();
+	const outgoing = new Map<string, string[]>();
+	const incoming = new Map<string, string[]>();
+
+	for (const edge of edges) {
+		nodes.add(edge.from_id);
+		nodes.add(edge.to_id);
+
+		if (!outgoing.has(edge.from_id)) outgoing.set(edge.from_id, []);
+		outgoing.get(edge.from_id)?.push(edge.to_id);
+
+		if (!incoming.has(edge.to_id)) incoming.set(edge.to_id, []);
+		incoming.get(edge.to_id)?.push(edge.from_id);
+	}
+
+	if (nodes.size === 0) {
+		return { iterations: 0, converged: true, finalDelta: 0, nodesScored: 0 };
+	}
+
+	const nodeList = [...nodes];
+	const teleport = buildTeleportVector(
+		nodeList,
+		activeFileIds.filter((id) => nodes.has(id)),
+	);
+	const damping = 0.85;
+	const maxIter = 30;
+	const n = nodeList.length;
+
+	let scores = new Map<string, number>(nodeList.map((id) => [id, teleport.get(id) ?? 1 / n]));
+	let converged = false;
+	let finalDelta = 0;
+	let iterationCount = 0;
+
+	for (let iter = 0; iter < maxIter; iter++) {
+		iterationCount = iter + 1;
+		const newScores = new Map<string, number>();
+		const danglingSum = nodeList.reduce((sum, id) => {
+			const out = outgoing.get(id);
+			return out && out.length > 0 ? sum : sum + (scores.get(id) ?? 0);
+		}, 0);
+
+		for (const node of nodeList) {
+			const incomingNodes = incoming.get(node) ?? [];
+			let rank = danglingSum / n;
+
+			for (const source of incomingNodes) {
+				const out = outgoing.get(source) ?? [];
+				const weight = out.length === 0 ? 0 : 1 / out.length;
+				rank += (scores.get(source) ?? 0) * weight;
+			}
+
+			const teleportWeight = teleport.get(node) ?? 1 / n;
+			const value = (1 - damping) * teleportWeight + damping * rank;
+			newScores.set(node, value);
+		}
+
+		let delta = 0;
+		for (const node of nodeList) {
+			delta += Math.abs((newScores.get(node) ?? 0) - (scores.get(node) ?? 0));
+		}
+		scores = newScores;
+		finalDelta = delta;
+		if (delta < 1e-6) {
+			converged = true;
+			break;
+		}
+	}
+
+	if (!converged) {
+		console.warn(`PageRank did not converge after ${maxIter} iterations (delta=${finalDelta})`);
+	}
+
+	const BATCH_SIZE = 500;
+	for (let i = 0; i < nodeList.length; i += BATCH_SIZE) {
+		const batch = nodeList.slice(i, i + BATCH_SIZE);
+		const statements = batch.map((id) => ({
+			sql: "UPDATE graph_nodes SET importance = ? WHERE id = ?",
+			params: [scores.get(id) ?? 0, id],
+		}));
+		await db.executeMany(statements);
+	}
+
+	return {
+		iterations: iterationCount,
+		converged,
+		finalDelta,
+		nodesScored: nodeList.length,
+	};
+}

package/src/ast/path-utils.ts

@@ -0,0 +1,137 @@
+import { existsSync, readFileSync } from "node:fs";
+import { join, relative, resolve, sep } from "node:path";
+
+export interface IgnoreMatcher {
+	shouldIgnore(absPath: string, isDir: boolean): boolean;
+}
+
+export interface GitignoreRule {
+	regex: RegExp;
+	dirOnly: boolean;
+}
+
+/** Normalize a path to posix-style separators for consistent matching. */
+export function toPosixPath(path: string): string {
+	return path.split(sep).join("/");
+}
+
+function patternToRegExp(pattern: string): RegExp {
+	const trimmed = pattern.trim();
+	if (!trimmed) return /^$/; // unused
+
+	const escaped = trimmed.replace(/[.+?^${}()|[\]\\]/g, "\\$&");
+	const wildcarded = escaped.replace(/\\\*\\\*/g, ".*").replace(/\\\*/g, "[^/]*");
+
+	if (trimmed.startsWith("/")) {
+		return new RegExp(`^${wildcarded.slice(1)}(/.*)?$`);
+	}
+
+	return new RegExp(`(^|/)${wildcarded}(/.*)?$`);
+}
+
+function loadGitignorePatterns(repoRoot: string): {
+	ignore: GitignoreRule[];
+	negate: GitignoreRule[];
+} {
+	const gitignorePath = join(repoRoot, ".gitignore");
+	if (!existsSync(gitignorePath)) return { ignore: [], negate: [] };
+
+	const lines = readFileSync(gitignorePath, "utf-8")
+		.split("\n")
+		.map((line) => line.trim())
+		.filter((line) => line.length > 0 && !line.startsWith("#"));
+
+	const ignoreRules: GitignoreRule[] = [];
+	const negateRules: GitignoreRule[] = [];
+
+	for (const line of lines) {
+		if (line.startsWith("!")) {
+			const raw = line.slice(1);
+			const isDirOnly = raw.endsWith("/");
+			const cleanLine = isDirOnly ? raw.slice(0, -1) : raw;
+			negateRules.push({ regex: patternToRegExp(cleanLine), dirOnly: isDirOnly });
+		} else {
+			const isDirOnly = line.endsWith("/");
+			const cleanLine = isDirOnly ? line.slice(0, -1) : line;
+			ignoreRules.push({ regex: patternToRegExp(cleanLine), dirOnly: isDirOnly });
+		}
+	}
+
+	return { ignore: ignoreRules, negate: negateRules };
+}
+
+/**
+ * Build a matcher that applies default ignores, .gitignore rules, and config.excludePaths.
+ */
+export function createIgnoreMatcher(repoRoot: string, excludePaths: string[] = []): IgnoreMatcher {
+	const root = resolve(repoRoot);
+	const gitignoreResult = loadGitignorePatterns(root);
+	const defaultExcludes = [
+		"node_modules",
+		".git",
+		".idea",
+		".vscode",
+		".sia",
+		"dist",
+		"build",
+		"coverage",
+		"out",
+		"tmp",
+	].map((p) => toPosixPath(p));
+
+	const configuredExcludes = excludePaths.map((p) => toPosixPath(p.replace(/^\/+/, "")));
+
+	function isOutside(rel: string): boolean {
+		return rel.startsWith("..");
+	}
+
+	return {
+		shouldIgnore(absPath: string, isDir: boolean): boolean {
+			const rel = toPosixPath(relative(root, resolve(absPath)));
+			if (isOutside(rel)) return true;
+			if (rel === "" || rel === ".") return false;
+
+			for (const prefix of [...defaultExcludes, ...configuredExcludes]) {
+				if (rel === prefix || rel.startsWith(`${prefix}/`)) {
+					return true;
+				}
+			}
+
+			// Apply gitignore patterns with negation support
+			let gitIgnored = false;
+			for (const rule of gitignoreResult.ignore) {
+				if (rule.dirOnly && !isDir) continue;
+				if (rule.regex.test(rel)) {
+					gitIgnored = true;
+					break;
+				}
+			}
+			if (gitIgnored) {
+				for (const rule of gitignoreResult.negate) {
+					if (rule.dirOnly && !isDir) continue;
+					if (rule.regex.test(rel)) {
+						gitIgnored = false;
+						break;
+					}
+				}
+			}
+			if (gitIgnored) return true;
+
+			return false;
+		},
+	};
+}
+
+/** Best-effort monorepo package detector for paths like packages/foo/src/index.ts */
+export function detectPackagePath(relativePath: string): string | null {
+	const parts = relativePath.split("/");
+	const packagesIdx = parts.indexOf("packages");
+	if (packagesIdx !== -1 && packagesIdx + 1 < parts.length) {
+		return parts.slice(0, packagesIdx + 2).join("/");
+	}
+	const appsIdx = parts.indexOf("apps");
+	if (appsIdx !== -1 && appsIdx + 1 < parts.length) {
+		return parts.slice(0, appsIdx + 2).join("/");
+	}
+	return null;
+}