@rkarim08/sia 1.0.0

package/src/cli/commands/lead-report.ts

@@ -0,0 +1,241 @@
+// Module: lead-report — tech lead intelligence reports
+//
+// Usage:
+//   sia lead-report --type drift|knowledge-map|compliance
+
+import type { SiaDb } from "@/graph/db-interface";
+
+const ACTIVE_FILTER = "t_valid_until IS NULL AND archived_at IS NULL";
+
+// --- Drift Report ---
+
+export interface DriftEntity {
+	id: string;
+	name: string;
+	content: string;
+	filePaths: string[];
+	createdAt: string;
+	trustTier: number;
+}
+
+export interface DriftReport {
+	type: "drift";
+	decisions: DriftEntity[];
+	conventions: DriftEntity[];
+}
+
+async function generateDriftReport(db: SiaDb): Promise<DriftReport> {
+	const decisions = await queryEntitiesByType(db, "Decision");
+	const conventions = await queryEntitiesByType(db, "Convention");
+	return { type: "drift", decisions, conventions };
+}
+
+// --- Knowledge Map Report ---
+
+export interface KnowledgeMapReport {
+	type: "knowledge-map";
+	totalEntities: number;
+	byType: Record<string, number>;
+	byContributor: Record<string, number>;
+}
+
+async function generateKnowledgeMapReport(db: SiaDb): Promise<KnowledgeMapReport> {
+	const { rows: totalRows } = await db.execute(
+		`SELECT COUNT(*) AS cnt FROM graph_nodes WHERE ${ACTIVE_FILTER}`,
+	);
+	const totalEntities = (totalRows[0]?.cnt as number) ?? 0;
+
+	const byType: Record<string, number> = {};
+	const { rows: typeRows } = await db.execute(
+		`SELECT type, COUNT(*) AS cnt FROM graph_nodes WHERE ${ACTIVE_FILTER} GROUP BY type`,
+	);
+	for (const row of typeRows) {
+		byType[row.type as string] = row.cnt as number;
+	}
+
+	const byContributor: Record<string, number> = {};
+	const { rows: contribRows } = await db.execute(
+		`SELECT created_by, COUNT(*) AS cnt FROM graph_nodes WHERE ${ACTIVE_FILTER} GROUP BY created_by`,
+	);
+	for (const row of contribRows) {
+		byContributor[row.created_by as string] = row.cnt as number;
+	}
+
+	return { type: "knowledge-map", totalEntities, byType, byContributor };
+}
+
+// --- Compliance Report ---
+
+export interface ComplianceConvention {
+	id: string;
+	name: string;
+	content: string;
+	filePaths: string[];
+}
+
+export interface ComplianceReport {
+	type: "compliance";
+	conventions: ComplianceConvention[];
+}
+
+async function generateComplianceReport(db: SiaDb): Promise<ComplianceReport> {
+	const entities = await queryEntitiesByType(db, "Convention");
+	const conventions: ComplianceConvention[] = entities.map(({ id, name, content, filePaths }) => ({
+		id,
+		name,
+		content,
+		filePaths,
+	}));
+	return { type: "compliance", conventions };
+}
+
+// --- Shared ---
+
+async function queryEntitiesByType(db: SiaDb, entityType: string): Promise<DriftEntity[]> {
+	const { rows } = await db.execute(
+		`SELECT id, name, content, file_paths, created_at, trust_tier
+		 FROM graph_nodes
+		 WHERE type = ? AND ${ACTIVE_FILTER}
+		 ORDER BY created_at DESC`,
+		[entityType],
+	);
+	return rows.map((row) => ({
+		id: row.id as string,
+		name: row.name as string,
+		content: row.content as string,
+		filePaths: parseJsonArray(row.file_paths as string),
+		createdAt: new Date(row.created_at as number).toISOString(),
+		trustTier: row.trust_tier as number,
+	}));
+}
+
+function parseJsonArray(raw: string): string[] {
+	try {
+		const parsed = JSON.parse(raw);
+		return Array.isArray(parsed) ? parsed : [];
+	} catch {
+		return [];
+	}
+}
+
+// --- Public API ---
+
+export type LeadReport = DriftReport | KnowledgeMapReport | ComplianceReport;
+
+export interface LeadReportOptions {
+	type: "drift" | "knowledge-map" | "compliance";
+}
+
+export async function generateLeadReport(db: SiaDb, opts: LeadReportOptions): Promise<LeadReport> {
+	switch (opts.type) {
+		case "drift":
+			return generateDriftReport(db);
+		case "knowledge-map":
+			return generateKnowledgeMapReport(db);
+		case "compliance":
+			return generateComplianceReport(db);
+	}
+}
+
+export function formatLeadReport(report: LeadReport): string {
+	switch (report.type) {
+		case "drift":
+			return formatDriftReport(report);
+		case "knowledge-map":
+			return formatKnowledgeMapReport(report);
+		case "compliance":
+			return formatComplianceReport(report);
+	}
+}
+
+function formatDriftReport(report: DriftReport): string {
+	const lines: string[] = [];
+	lines.push("=== Architecture Drift Report ===");
+	lines.push("");
+
+	lines.push(`Decisions: ${report.decisions.length}`);
+	for (const d of report.decisions) {
+		lines.push(`  - ${d.name}`);
+		if (d.filePaths.length > 0) {
+			lines.push(`    Files: ${d.filePaths.join(", ")}`);
+		}
+	}
+
+	lines.push("");
+	lines.push(`Conventions: ${report.conventions.length}`);
+	for (const c of report.conventions) {
+		lines.push(`  - ${c.name}`);
+		if (c.filePaths.length > 0) {
+			lines.push(`    Files: ${c.filePaths.join(", ")}`);
+		}
+	}
+
+	return lines.join("\n");
+}
+
+function formatKnowledgeMapReport(report: KnowledgeMapReport): string {
+	const lines: string[] = [];
+	lines.push("=== Knowledge Distribution Map ===");
+	lines.push("");
+	lines.push(`Total entities: ${report.totalEntities}`);
+
+	lines.push("");
+	lines.push("--- By Type ---");
+	for (const [type, count] of Object.entries(report.byType).sort((a, b) => b[1] - a[1])) {
+		lines.push(`  ${type.padEnd(20)} ${count}`);
+	}
+
+	lines.push("");
+	lines.push("--- By Contributor ---");
+	for (const [contrib, count] of Object.entries(report.byContributor).sort((a, b) => b[1] - a[1])) {
+		lines.push(`  ${contrib.padEnd(20)} ${count}`);
+	}
+
+	return lines.join("\n");
+}
+
+function formatComplianceReport(report: ComplianceReport): string {
+	const lines: string[] = [];
+	lines.push("=== Convention Compliance ===");
+	lines.push("");
+	lines.push(`Conventions tracked: ${report.conventions.length}`);
+
+	for (const c of report.conventions) {
+		lines.push("");
+		lines.push(`  ${c.name}`);
+		if (c.filePaths.length > 0) {
+			lines.push(`    Referenced files: ${c.filePaths.join(", ")}`);
+		}
+	}
+
+	return lines.join("\n");
+}
+
+// --- CLI Entry Point ---
+
+export async function runLeadReport(args: string[]): Promise<void> {
+	const { resolveRepoHash } = await import("@/capture/hook");
+	const { openGraphDb } = await import("@/graph/semantic-db");
+
+	let type: LeadReportOptions["type"] = "drift";
+	for (let i = 0; i < args.length; i++) {
+		if (args[i] === "--type" && args[i + 1]) {
+			const val = args[i + 1];
+			if (val === "drift" || val === "knowledge-map" || val === "compliance") {
+				type = val;
+			} else {
+				console.error(`Unknown report type: ${val}. Use: drift, knowledge-map, compliance`);
+				return;
+			}
+		}
+	}
+
+	const repoHash = resolveRepoHash(process.cwd());
+	const db = openGraphDb(repoHash);
+	try {
+		const report = await generateLeadReport(db, { type });
+		console.log(formatLeadReport(report));
+	} finally {
+		await db.close();
+	}
+}

package/src/cli/commands/learn.ts

@@ -0,0 +1,321 @@
+// Module: learn — Full knowledge graph builder orchestrator.
+//
+// Usage:
+//   sia learn                    Full rebuild (default)
+//   sia learn --incremental      Only process changed files
+//   sia learn --force            Skip snapshot restore, rebuild everything
+//   sia learn --verbose          Phase-by-phase progress (default)
+//   sia learn --quiet            Summary only
+//   sia learn --interactive      Confirm after each phase
+
+import { readFileSync } from "node:fs";
+import { resolve } from "node:path";
+import { resolveRepoHash } from "@/capture/hook";
+import {
+	deleteProgress,
+	type LearnProgress,
+	readProgress,
+	runWithRetry,
+	writeProgress,
+} from "@/cli/learn-progress";
+import { getConfig, resolveSiaHome } from "@/shared/config";
+
+export type Verbosity = "verbose" | "quiet" | "interactive";
+
+export interface LearnOptions {
+	cwd?: string;
+	siaHome?: string;
+	incremental?: boolean;
+	force?: boolean;
+	verbosity?: Verbosity;
+}
+
+export interface LearnResult {
+	phasesCompleted: number[];
+	phasesFailed: number[];
+	codeEntities: number;
+	codeFiles: number;
+	codeCacheHits: number;
+	docsIngested: number;
+	docChunks: number;
+	externalRefs: number;
+	communities: number;
+	skippedFiles: Array<{ path: string; error: string }>;
+	durationMs: number;
+}
+
+function log(verbosity: Verbosity, msg: string): void {
+	if (verbosity !== "quiet") {
+		process.stderr.write(`${msg}\n`);
+	}
+}
+
+export async function siaLearn(opts: LearnOptions = {}): Promise<LearnResult | null> {
+	const start = Date.now();
+	const cwd = resolve(opts.cwd ?? process.cwd());
+	const siaHome = opts.siaHome ?? resolveSiaHome();
+	const verbosity = opts.verbosity ?? "verbose";
+	const config = getConfig(siaHome);
+
+	const result: LearnResult = {
+		phasesCompleted: [],
+		phasesFailed: [],
+		codeEntities: 0,
+		codeFiles: 0,
+		codeCacheHits: 0,
+		docsIngested: 0,
+		docChunks: 0,
+		externalRefs: 0,
+		communities: 0,
+		skippedFiles: [],
+		durationMs: 0,
+	};
+
+	// Check for resumable progress
+	const existingProgress = readProgress(cwd);
+	let phasesToSkip: Set<number> = new Set();
+	if (existingProgress && !opts.force) {
+		const repoHash = resolveRepoHash(cwd);
+		if (existingProgress.repo_hash === repoHash) {
+			phasesToSkip = new Set(existingProgress.phases_completed);
+			log(
+				verbosity,
+				`[sia-learn] Resuming — phases ${[...phasesToSkip].join(", ")} already complete`,
+			);
+		}
+	}
+
+	// Initialize progress file
+	const progress: LearnProgress = {
+		started_at: existingProgress?.started_at ?? Date.now(),
+		repo_hash: resolveRepoHash(cwd),
+		branch: "",
+		phases_completed: [...phasesToSkip],
+		files_indexed: existingProgress?.files_indexed ?? 0,
+		total_files: 0,
+		last_checkpoint_at: Date.now(),
+	};
+
+	// Try to get current branch
+	try {
+		const { execFileSync } = await import("node:child_process");
+		progress.branch = execFileSync("git", ["branch", "--show-current"], {
+			cwd,
+			encoding: "utf-8",
+			stdio: ["pipe", "pipe", "pipe"],
+		}).trim();
+	} catch {
+		progress.branch = "unknown";
+	}
+
+	// --- Phase 0: Install (idempotent) ---
+	if (!phasesToSkip.has(0)) {
+		log(verbosity, "[sia-learn] Phase 0: Installing SIA...");
+		const installResult = await runWithRetry("Phase 0: Install", async () => {
+			const { siaInstall } = await import("@/cli/commands/install");
+			await siaInstall({ cwd, siaHome });
+		});
+		if (installResult !== null) {
+			result.phasesCompleted.push(0);
+			progress.phases_completed.push(0);
+			writeProgress(cwd, progress);
+			log(verbosity, "[sia-learn] Phase 0: Done");
+		} else {
+			result.phasesFailed.push(0);
+			// Install failure is fatal — can't continue without databases
+			log(verbosity, "[sia-learn] Phase 0: FAILED — cannot continue without databases");
+			result.durationMs = Date.now() - start;
+			return result;
+		}
+	} else {
+		result.phasesCompleted.push(0);
+	}
+
+	// Open graph DB for remaining phases
+	const { openGraphDb } = await import("@/graph/semantic-db");
+	const repoHash = resolveRepoHash(cwd);
+	const db = openGraphDb(repoHash, siaHome);
+
+	try {
+		// --- Phase 1: AST code indexing (delegates to siaReindex) ---
+		if (!phasesToSkip.has(1)) {
+			log(verbosity, "[sia-learn] Phase 1: Indexing code...");
+			const indexResult = await runWithRetry("Phase 1: Code indexing", async () => {
+				const { siaReindex } = await import("@/cli/commands/reindex");
+				return siaReindex({ cwd, siaHome });
+			});
+			if (indexResult) {
+				result.codeEntities = indexResult.entitiesCreated;
+				result.codeFiles = indexResult.filesProcessed;
+				result.codeCacheHits = indexResult.cacheHits;
+				result.phasesCompleted.push(1);
+				progress.phases_completed.push(1);
+				progress.files_indexed = indexResult.filesProcessed;
+				writeProgress(cwd, progress);
+				log(
+					verbosity,
+					`[sia-learn] Phase 1: Done — ${indexResult.entitiesCreated} entities from ${indexResult.filesProcessed} files (${indexResult.cacheHits} cached)`,
+				);
+			} else {
+				result.phasesFailed.push(1);
+			}
+		} else {
+			result.phasesCompleted.push(1);
+		}
+
+		// --- Phase 2: Markdown doc ingestion ---
+		if (!phasesToSkip.has(2)) {
+			log(verbosity, "[sia-learn] Phase 2: Ingesting docs...");
+			const docResult = await runWithRetry("Phase 2: Doc ingestion", async () => {
+				const { discoverDocFiles } = await import("@/knowledge/discovery");
+				const { ingestDocument } = await import("@/knowledge/ingest");
+				const { detectExternalRefs } = await import("@/knowledge/external-refs");
+
+				const docFiles = discoverDocFiles(cwd);
+				let docsIngested = 0;
+				let chunksCreated = 0;
+				let refsFound = 0;
+
+				for (const doc of docFiles) {
+					try {
+						// Incremental: skip if file hasn't changed
+						if (opts.incremental && !opts.force) {
+							const { statSync } = await import("node:fs");
+							const fileMtime = statSync(doc.absolutePath).mtimeMs;
+							const existing = await db.execute(
+								`SELECT updated_at FROM graph_nodes
+								 WHERE file_paths LIKE ? AND type = 'FileNode'
+								 AND t_valid_until IS NULL AND archived_at IS NULL
+								 LIMIT 1`,
+								[`%${doc.relativePath}%`],
+							);
+							if (existing.rows.length > 0 && (existing.rows[0] as any).updated_at >= fileMtime) {
+								continue; // Skip unchanged file
+							}
+						}
+
+						const ingestResult = await ingestDocument(db, doc.absolutePath, doc.relativePath, {
+							tag: doc.pattern.tag,
+							trustTier: doc.pattern.trustTier as 1 | 2,
+							packagePath: doc.packagePath,
+						});
+						docsIngested++;
+						chunksCreated += ingestResult.chunksCreated;
+
+						// Detect external refs in doc content
+						try {
+							const content = readFileSync(doc.absolutePath, "utf-8");
+							const refs = detectExternalRefs(content);
+							refsFound += refs.length;
+						} catch {
+							// Non-fatal — skip external ref detection for this file
+						}
+					} catch (err) {
+						// Per-file error — log and continue
+						const msg = err instanceof Error ? err.message : String(err);
+						process.stderr.write(
+							`[sia-learn] Warning: Failed to ingest ${doc.relativePath}: ${msg}\n`,
+						);
+					}
+				}
+
+				return { docsIngested, chunksCreated, refsFound };
+			});
+
+			if (docResult) {
+				result.docsIngested = docResult.docsIngested;
+				result.docChunks = docResult.chunksCreated;
+				result.externalRefs = docResult.refsFound;
+				result.phasesCompleted.push(2);
+				progress.phases_completed.push(2);
+				writeProgress(cwd, progress);
+				log(
+					verbosity,
+					`[sia-learn] Phase 2: Done — ${docResult.chunksCreated} chunks from ${docResult.docsIngested} documents, ${docResult.refsFound} external refs`,
+				);
+			} else {
+				result.phasesFailed.push(2);
+			}
+		} else {
+			result.phasesCompleted.push(2);
+		}
+
+		// --- Phase 3: Community detection + summarization ---
+		if (!phasesToSkip.has(3)) {
+			log(verbosity, "[sia-learn] Phase 3: Detecting communities...");
+			const communityResult = await runWithRetry("Phase 3: Community detection", async () => {
+				const { detectCommunities } = await import("@/community/leiden");
+				const { summarizeCommunities } = await import("@/community/summarize");
+
+				const detectionResult = await detectCommunities(db);
+
+				// Summarize if we have communities
+				if (detectionResult.totalCommunities > 0) {
+					await summarizeCommunities(db, { airGapped: config.airGapped ?? false });
+				}
+
+				return detectionResult;
+			});
+
+			if (communityResult) {
+				result.communities = communityResult.totalCommunities;
+				result.phasesCompleted.push(3);
+				progress.phases_completed.push(3);
+				writeProgress(cwd, progress);
+				log(
+					verbosity,
+					`[sia-learn] Phase 3: Done — ${communityResult.totalCommunities} communities formed`,
+				);
+			} else {
+				result.phasesFailed.push(3);
+			}
+		} else {
+			result.phasesCompleted.push(3);
+		}
+
+		// --- Branch snapshot save (if Phase D.5 is available) ---
+		try {
+			// Check if branch_snapshots table exists
+			await db.execute("SELECT 1 FROM branch_snapshots LIMIT 0");
+			// Table exists — save snapshot for current branch
+			const { createBranchSnapshot } = await import("@/graph/snapshots");
+			const { execFileSync } = await import("node:child_process");
+			const branch = execFileSync("git", ["branch", "--show-current"], {
+				cwd,
+				encoding: "utf-8",
+				stdio: ["pipe", "pipe", "pipe"],
+			}).trim();
+			const commit = execFileSync("git", ["rev-parse", "--short", "HEAD"], {
+				cwd,
+				encoding: "utf-8",
+				stdio: ["pipe", "pipe", "pipe"],
+			}).trim();
+			if (branch) {
+				await createBranchSnapshot(db, branch, commit);
+				log(verbosity, `[sia-learn] Saved snapshot for branch '${branch}' at ${commit}`);
+			}
+		} catch {
+			// branch_snapshots table doesn't exist — Phase D.5 not implemented yet, skip silently
+		}
+
+		// --- Summary ---
+		result.durationMs = Date.now() - start;
+		deleteProgress(cwd); // Clean finish
+
+		const summary = `
+=== SIA Learn Complete ===
+Code entities:  ${result.codeEntities} (${result.codeFiles} files, ${result.codeCacheHits} cached)
+Doc chunks:     ${result.docChunks} (${result.docsIngested} documents)
+External refs:  ${result.externalRefs}
+Communities:    ${result.communities}
+Duration:       ${(result.durationMs / 1000).toFixed(1)}s
+${result.phasesFailed.length > 0 ? `Failed phases: ${result.phasesFailed.join(", ")}` : ""}
+${result.skippedFiles.length > 0 ? `Skipped files:  ${result.skippedFiles.length}` : ""}`;
+
+		process.stderr.write(`${summary.trim()}\n`);
+
+		return result;
+	} finally {
+		await db.close();
+	}
+}