@rkarim08/sia 1.0.0

package/src/community/scheduler.ts

@@ -0,0 +1,74 @@
+// Module: scheduler — decides when to run community detection
+
+import { detectCommunities } from "@/community/leiden";
+import { buildSummaryTree } from "@/community/raptor";
+import { summarizeCommunities } from "@/community/summarize";
+import type { SiaDb } from "@/graph/db-interface";
+import type { SiaConfig } from "@/shared/config";
+import type { LlmClient } from "@/shared/llm-client";
+
+async function countActiveEntities(db: SiaDb): Promise<number> {
+	const result = await db.execute(
+		`SELECT COUNT(*) as count
+		 FROM graph_nodes
+		 WHERE t_valid_until IS NULL AND archived_at IS NULL`,
+	);
+	return Number((result.rows[0] as { count: number }).count ?? 0);
+}
+
+async function lastRunAt(db: SiaDb): Promise<number> {
+	const result = await db.execute("SELECT MAX(updated_at) as ts FROM communities");
+	const ts = (result.rows[0] as { ts: number | null }).ts;
+	return typeof ts === "number" ? ts : 0;
+}
+
+async function newEntitiesSince(db: SiaDb, since: number): Promise<number> {
+	const result = await db.execute(
+		`SELECT COUNT(*) as count
+		 FROM graph_nodes
+		 WHERE t_valid_until IS NULL
+		   AND archived_at IS NULL
+		   AND (created_at > ? OR t_created > ?)`,
+		[since, since],
+	);
+	return Number((result.rows[0] as { count: number }).count ?? 0);
+}
+
+export async function shouldRunDetection(db: SiaDb, config: SiaConfig): Promise<boolean> {
+	const totalEntities = await countActiveEntities(db);
+	if (totalEntities < config.communityMinGraphSize) {
+		console.warn(
+			`Graph has fewer than ${config.communityMinGraphSize} entities (${totalEntities}) — skipping community detection`,
+		);
+		return false;
+	}
+
+	const lastRun = await lastRunAt(db);
+	const fresh = await newEntitiesSince(db, lastRun);
+	return fresh > config.communityTriggerNodeCount;
+}
+
+export class CommunityScheduler {
+	constructor(
+		private readonly db: SiaDb,
+		private readonly config: SiaConfig,
+		private readonly llmClient?: LlmClient,
+	) {}
+
+	async check(): Promise<boolean> {
+		return shouldRunDetection(this.db, this.config);
+	}
+
+	async run(): Promise<void> {
+		const shouldRun = await this.check();
+		if (!shouldRun) return;
+
+		await detectCommunities(this.db);
+		await summarizeCommunities(this.db, { airGapped: this.config.airGapped }, this.llmClient);
+		await buildSummaryTree(this.db, this.llmClient);
+	}
+
+	runInBackground(): void {
+		void this.run().catch((err) => console.error("Community detection failed:", err));
+	}
+}

package/src/community/summarize.ts

@@ -0,0 +1,115 @@
+// Module: summarize — community summary generation and caching
+
+import { createHash } from "node:crypto";
+import type { SiaDb } from "@/graph/db-interface";
+import type { LlmClient } from "@/shared/llm-client";
+
+interface CommunityRow {
+	id: string;
+	member_count: number;
+	last_summary_member_count: number;
+	summary: string | null;
+}
+
+interface TopEntityRow {
+	id: string;
+	name: string;
+	summary: string;
+	importance: number;
+}
+
+function sha256(value: string): string {
+	return createHash("sha256").update(value).digest("hex");
+}
+
+async function generateSummary(entities: TopEntityRow[], llmClient?: LlmClient): Promise<string> {
+	if (entities.length === 0) {
+		return "Community has no active members (all entities invalidated or archived).";
+	}
+
+	const entityDescriptions = entities
+		.map((e) => `${e.name}: ${e.summary || "No summary available."}`)
+		.join("\n");
+
+	if (!llmClient) {
+		return `Top members — ${entityDescriptions.replace(/\n/g, "; ")}`;
+	}
+
+	const prompt = `Summarize this code community in a single coherent paragraph (2-4 sentences). Describe what the community does, how its members relate, and what purpose it serves in the codebase.\n\nMembers:\n${entityDescriptions}`;
+	return llmClient.summarize(prompt);
+}
+
+async function loadCommunities(db: SiaDb): Promise<CommunityRow[]> {
+	const result = await db.execute(
+		`SELECT id, member_count, last_summary_member_count, summary
+                 FROM communities`,
+	);
+	return result.rows as unknown as CommunityRow[];
+}
+
+async function topEntities(db: SiaDb, communityId: string): Promise<TopEntityRow[]> {
+	const result = await db.execute(
+		`SELECT e.id, e.name, e.summary, e.importance
+                 FROM community_members cm
+                 JOIN graph_nodes e ON cm.entity_id = e.id
+                 WHERE cm.community_id = ?
+                   AND e.t_valid_until IS NULL
+                   AND e.archived_at IS NULL
+                 ORDER BY e.importance DESC
+                 LIMIT 5`,
+		[communityId],
+	);
+	return result.rows as unknown as TopEntityRow[];
+}
+
+async function memberIds(db: SiaDb, communityId: string): Promise<string[]> {
+	const result = await db.execute(
+		`SELECT entity_id
+                 FROM community_members
+                 WHERE community_id = ?
+                 ORDER BY entity_id`,
+		[communityId],
+	);
+	return (result.rows as Array<{ entity_id: string }>).map((r) => r.entity_id);
+}
+
+export async function summarizeCommunities(
+	db: SiaDb,
+	config: { airGapped: boolean },
+	llmClient?: LlmClient,
+): Promise<number> {
+	if (config.airGapped) {
+		return 0;
+	}
+
+	const communities = await loadCommunities(db);
+	let generated = 0;
+	const now = Date.now();
+
+	await db.transaction(async (tx) => {
+		for (const community of communities) {
+			const changeRatio =
+				Math.abs(community.member_count - community.last_summary_member_count) /
+				Math.max(community.last_summary_member_count, 1);
+			const needsSummary = !community.summary || changeRatio > 0.2;
+			if (!needsSummary) continue;
+
+			const entities = await topEntities(tx, community.id);
+			const summary = await generateSummary(entities, llmClient);
+			const ids = await memberIds(tx, community.id);
+			const summaryHash = sha256(ids.join(","));
+
+			await tx.execute(
+				`UPDATE communities
+                                 SET summary = ?, summary_hash = ?, last_summary_member_count = ?, updated_at = ?
+                                 WHERE id = ?`,
+				[summary, summaryHash, community.member_count, now, community.id],
+			);
+			generated++;
+		}
+	});
+
+	return generated;
+}
+
+export type { TopEntityRow };

package/src/decay/archiver.ts

@@ -0,0 +1,73 @@
+// Module: archiver — soft-archive decayed entities
+
+import type { BatchResult } from "@/decay/types";
+import type { SiaDb } from "@/graph/db-interface";
+import { archiveEntity } from "@/graph/entities";
+import type { SiaConfig } from "@/shared/config";
+
+/** Default batch size for archiveDecayedEntities loop. */
+const DEFAULT_BATCH_SIZE = 50;
+
+/** Inactivity window: 90 days in milliseconds. */
+const INACTIVE_DAYS_MS = 90 * 86400000;
+
+/**
+ * Archive a single batch of decayed entities.
+ *
+ * Selects entities matching ALL conditions:
+ * - importance below config.archiveThreshold (default 0.05)
+ * - zero edges (isolated node)
+ * - not accessed in 90 days
+ * - not bi-temporally invalidated (t_valid_until IS NULL)
+ * - not already archived (archived_at IS NULL)
+ *
+ * For each matched entity, calls archiveEntity which sets archived_at ONLY
+ * (never t_valid_until or t_expired).
+ */
+export async function archiveBatch(
+	db: SiaDb,
+	config: SiaConfig,
+	batchSize: number,
+): Promise<BatchResult> {
+	const cutoff = Date.now() - INACTIVE_DAYS_MS;
+
+	const { rows } = await db.execute(
+		`SELECT id FROM graph_nodes
+		 WHERE importance < ?
+		   AND edge_count = 0
+		   AND last_accessed < ?
+		   AND t_valid_until IS NULL
+		   AND archived_at IS NULL
+		 ORDER BY importance ASC
+		 LIMIT ?`,
+		[config.archiveThreshold, cutoff, batchSize],
+	);
+
+	for (const row of rows) {
+		await archiveEntity(db, row.id as string);
+	}
+
+	const processed = rows.length;
+	return { processed, remaining: processed === batchSize };
+}
+
+/**
+ * Archive all decayed entities across the entire graph.
+ *
+ * Repeatedly calls archiveBatch in batches of 50 until no more
+ * archivable entities remain.
+ *
+ * Returns the total number of entities archived.
+ */
+export async function archiveDecayedEntities(db: SiaDb, config: SiaConfig): Promise<number> {
+	let total = 0;
+	let hasMore = true;
+
+	while (hasMore) {
+		const result = await archiveBatch(db, config, DEFAULT_BATCH_SIZE);
+		total += result.processed;
+		hasMore = result.remaining;
+	}
+
+	return total;
+}

package/src/decay/bridge-orphan-cleanup.ts

@@ -0,0 +1,212 @@
+// Module: bridge-orphan-cleanup — invalidate cross-repo edges where source/target no longer active
+
+import type { BatchResult } from "@/decay/types";
+import type { SiaDb } from "@/graph/db-interface";
+
+/**
+ * Validate that a string is safe to use as a SQLite ATTACH alias.
+ * Only allows alphanumeric characters and underscores (no injection vectors).
+ */
+function isSafeAlias(s: string): boolean {
+	return /^[a-zA-Z0-9_]+$/.test(s);
+}
+
+/**
+ * Derive a safe ATTACH alias from a repo id (UUID or hash).
+ * Strips hyphens and takes the first 16 alphanumeric chars with a prefix.
+ */
+function repoAlias(repoId: string): string {
+	const safe = repoId.replace(/[^a-zA-Z0-9]/g, "").slice(0, 16);
+	return `peer_${safe}`;
+}
+
+/**
+ * Check whether an entity is "live" in a peer graph.db using ATTACH.
+ *
+ * Live = exists in entities table, archived_at IS NULL, t_valid_until IS NULL.
+ *
+ * Returns true if the entity is live, false if it's gone or the repo path isn't known.
+ */
+function checkEntityLivenessViaAttach(
+	rawSqlite: {
+		prepare: (sql: string) => {
+			get: (...args: unknown[]) => unknown;
+			run: (...args: unknown[]) => void;
+		};
+		exec?: (sql: string) => void;
+	},
+	graphDbPath: string,
+	entityId: string,
+	alias: string,
+): boolean {
+	if (!isSafeAlias(alias)) {
+		return true; // be conservative — don't invalidate if alias is unsafe
+	}
+
+	try {
+		// ATTACH database names cannot be parameterized in SQLite — use the validated alias
+		rawSqlite.prepare(`ATTACH DATABASE ? AS ${alias}`).run(graphDbPath);
+
+		const row = rawSqlite
+			.prepare(
+				`SELECT 1 FROM ${alias}.graph_nodes
+				 WHERE id = ?
+				   AND archived_at IS NULL
+				   AND t_valid_until IS NULL`,
+			)
+			.get(entityId);
+
+		rawSqlite.prepare(`DETACH DATABASE ${alias}`).run();
+
+		return row !== undefined && row !== null;
+	} catch {
+		// If ATTACH fails (e.g., file doesn't exist), try to DETACH and be conservative
+		try {
+			rawSqlite.prepare(`DETACH DATABASE ${alias}`).run();
+		} catch {
+			// ignore DETACH error
+		}
+		return true; // be conservative — don't invalidate if we can't verify
+	}
+}
+
+/**
+ * Find and invalidate orphaned cross-repo edges in bridge.db.
+ *
+ * An edge is orphaned when its source or target entity is no longer active
+ * in the respective repo's graph.db. We ATTACH each peer's graph.db to
+ * check entity liveness, then invalidate dead edges.
+ *
+ * When `metaDb` is provided AND `bridgeDb.rawSqlite()` returns a handle,
+ * this function uses SQLite's ATTACH to check entity liveness in each peer
+ * repo's graph.db. The graph.db path is derived from the repo's `path` column
+ * in metaDb's repos table (the path IS the graph.db file path as registered).
+ *
+ * Falls back to the simplified null-endpoint check when ATTACH isn't available
+ * (e.g., LibSqlDb where rawSqlite() returns null).
+ */
+export async function bridgeOrphanBatch(
+	bridgeDb: SiaDb,
+	batchSize: number,
+	metaDb?: SiaDb,
+): Promise<BatchResult> {
+	// Get active cross-repo edges that might be orphaned
+	const { rows } = await bridgeDb.execute(
+		`SELECT id, source_repo_id, source_entity_id, target_repo_id, target_entity_id
+		 FROM cross_repo_edges
+		 WHERE t_valid_until IS NULL
+		 LIMIT ?`,
+		[batchSize],
+	);
+
+	if (rows.length === 0) {
+		return { processed: 0, remaining: false };
+	}
+
+	let processed = 0;
+	const now = Date.now();
+
+	// Determine if we can use ATTACH-based verification
+	const rawSqlite = bridgeDb.rawSqlite();
+	const canUseAttach = rawSqlite !== null && metaDb !== undefined;
+
+	// Cache repo paths looked up from metaDb to avoid repeated queries
+	const repoPathCache = new Map<string, string | null>();
+
+	async function getRepoPath(repoId: string): Promise<string | null> {
+		if (repoPathCache.has(repoId)) {
+			return repoPathCache.get(repoId) ?? null;
+		}
+		const result = await metaDb?.execute("SELECT path FROM repos WHERE id = ?", [repoId]);
+		const path = (result?.rows[0]?.path as string) ?? null;
+		repoPathCache.set(repoId, path);
+		return path;
+	}
+
+	for (const row of rows) {
+		const edgeId = row.id as string;
+		const sourceId = row.source_entity_id as string;
+		const targetId = row.target_entity_id as string;
+		const sourceRepoId = row.source_repo_id as string;
+		const targetRepoId = row.target_repo_id as string;
+
+		// Check if source/target are null or empty — these are definitely orphaned
+		if (!sourceId || !targetId) {
+			await bridgeDb.execute(
+				"UPDATE cross_repo_edges SET t_valid_until = ?, t_expired = ? WHERE id = ?",
+				[now, now, edgeId],
+			);
+			processed++;
+			continue;
+		}
+
+		if (canUseAttach) {
+			// ATTACH-based liveness verification
+			const sourceRepoPath = await getRepoPath(sourceRepoId);
+			const targetRepoPath = await getRepoPath(targetRepoId);
+
+			let isOrphan = false;
+
+			if (sourceRepoPath) {
+				const sourceAlias = repoAlias(sourceRepoId);
+				const sourceLive = checkEntityLivenessViaAttach(
+					rawSqlite as Parameters<typeof checkEntityLivenessViaAttach>[0],
+					sourceRepoPath,
+					sourceId,
+					sourceAlias,
+				);
+				if (!sourceLive) {
+					isOrphan = true;
+				}
+			}
+
+			if (!isOrphan && targetRepoPath) {
+				const targetAlias = repoAlias(targetRepoId);
+				// If source and target are in the same repo, generate a distinct alias
+				const sourceAlias = repoAlias(sourceRepoId);
+				const targetAliasResolved = targetAlias === sourceAlias ? `${targetAlias}t` : targetAlias;
+
+				const targetLive = checkEntityLivenessViaAttach(
+					rawSqlite as Parameters<typeof checkEntityLivenessViaAttach>[0],
+					targetRepoPath,
+					targetId,
+					targetAliasResolved,
+				);
+				if (!targetLive) {
+					isOrphan = true;
+				}
+			}
+
+			if (isOrphan) {
+				await bridgeDb.execute(
+					"UPDATE cross_repo_edges SET t_valid_until = ?, t_expired = ? WHERE id = ?",
+					[now, now, edgeId],
+				);
+			}
+		}
+
+		// Mark as processed
+		processed++;
+	}
+
+	return { processed, remaining: processed === batchSize };
+}
+
+/**
+ * Full cleanup pass: invalidate all orphaned cross-repo edges.
+ * Processes in batches of 50.
+ *
+ * Optionally accepts a `metaDb` to enable ATTACH-based entity liveness
+ * verification against peer graph.db files.
+ */
+export async function cleanupBridgeOrphans(bridgeDb: SiaDb, metaDb?: SiaDb): Promise<number> {
+	let total = 0;
+
+	for (;;) {
+		const { processed, remaining } = await bridgeOrphanBatch(bridgeDb, 50, metaDb);
+		total += processed;
+		if (!remaining) break;
+	}
+
+	return total;
+}

package/src/decay/consolidation-sweep.ts

@@ -0,0 +1,112 @@
+// Module: consolidation-sweep — maintenance dedup of similar entities
+
+import { wordJaccard } from "@/capture/consolidate";
+import type { BatchResult } from "@/decay/types";
+import type { SiaDb } from "@/graph/db-interface";
+
+interface ActiveEntity {
+	id: string;
+	type: string;
+	name: string;
+	content: string;
+}
+
+/**
+ * Ensure canonical ordering: the smaller ID is always first.
+ */
+function canonicalPair(idA: string, idB: string): [string, string] {
+	return idA < idB ? [idA, idB] : [idB, idA];
+}
+
+/**
+ * Process a batch of entity pairs for deduplication.
+ *
+ * Finds active entity pairs of the same type that have not yet been checked,
+ * computes word Jaccard similarity, and records a decision in local_dedup_log:
+ *   - > 0.92  => 'merged'
+ *   - > 0.5   => 'related'
+ *   - else    => 'different'
+ */
+export async function consolidationSweepBatch(db: SiaDb, batchSize: number): Promise<BatchResult> {
+	// 1. Fetch all active entities grouped by type, name
+	const { rows } = await db.execute(
+		`SELECT id, type, name, content FROM graph_nodes
+		 WHERE t_valid_until IS NULL AND archived_at IS NULL
+		 ORDER BY type, name`,
+	);
+
+	const entities = rows as unknown as ActiveEntity[];
+
+	// 2. Group entities by type
+	const byType = new Map<string, ActiveEntity[]>();
+	for (const entity of entities) {
+		let group = byType.get(entity.type);
+		if (!group) {
+			group = [];
+			byType.set(entity.type, group);
+		}
+		group.push(entity);
+	}
+
+	// 3. Iterate pairs within each type group, up to batchSize
+	let pairsProcessed = 0;
+
+	for (const group of byType.values()) {
+		if (pairsProcessed >= batchSize) break;
+
+		for (let i = 0; i < group.length && pairsProcessed < batchSize; i++) {
+			for (let j = i + 1; j < group.length && pairsProcessed < batchSize; j++) {
+				const [aId, bId] = canonicalPair(group[i].id, group[j].id);
+
+				// Check if this pair was already processed
+				const existing = await db.execute(
+					"SELECT 1 FROM local_dedup_log WHERE entity_a_id = ? AND entity_b_id = ?",
+					[aId, bId],
+				);
+
+				if (existing.rows.length > 0) continue;
+
+				// Compute similarity
+				const similarity = wordJaccard(group[i].content, group[j].content);
+
+				let decision: string;
+				if (similarity > 0.92) {
+					decision = "merged";
+				} else if (similarity > 0.5) {
+					decision = "related";
+				} else {
+					decision = "different";
+				}
+
+				const now = Date.now();
+				await db.execute(
+					"INSERT INTO local_dedup_log (entity_a_id, entity_b_id, decision, checked_at) VALUES (?, ?, ?, ?)",
+					[aId, bId, decision, now],
+				);
+
+				pairsProcessed++;
+			}
+		}
+	}
+
+	return { processed: pairsProcessed, remaining: pairsProcessed === batchSize };
+}
+
+/**
+ * Run the full consolidation sweep across all entity pairs.
+ * Processes in batches of 50 until no work remains.
+ * Returns the total number of pairs processed.
+ */
+export async function runConsolidationSweep(db: SiaDb): Promise<number> {
+	const BATCH_SIZE = 50;
+	let total = 0;
+	let remaining = true;
+
+	while (remaining) {
+		const result = await consolidationSweepBatch(db, BATCH_SIZE);
+		total += result.processed;
+		remaining = result.remaining;
+	}
+
+	return total;
+}

package/src/decay/decay.ts

@@ -0,0 +1,116 @@
+// Module: decay — importance decay batch processing
+
+import type { BatchResult, DecayResult } from "@/decay/types";
+import { computeConfidence } from "@/freshness/confidence-decay";
+import type { SiaDb } from "@/graph/db-interface";
+import type { Entity } from "@/graph/entities";
+import { updateEntity } from "@/graph/entities";
+import type { DecayHalfLife, SiaConfig } from "@/shared/config";
+
+export type { BatchResult, DecayResult };
+
+const BATCH_SIZE = 500;
+const MS_PER_DAY = 86_400_000;
+
+/**
+ * Compute decayed importance for a single entity.
+ *
+ * Formula:
+ *   daysSinceAccess = (now - entity.last_accessed) / 86400000
+ *   halfLife = config.decayHalfLife[entity.type] ?? config.decayHalfLife.default
+ *   decayFactor = 0.5 ^ (daysSinceAccess / halfLife)
+ *   edgeBoost = min(entity.edge_count * 0.02, 0.3)
+ *   newImportance = max(entity.base_importance * decayFactor + edgeBoost, 0.01)
+ *
+ * Highly-connected entities (edge_count > 20) never drop below 0.25.
+ */
+function computeDecayedImportance(entity: Entity, config: SiaConfig, now: number): number {
+	const daysSinceAccess = (now - entity.last_accessed) / MS_PER_DAY;
+	const halfLife =
+		config.decayHalfLife[entity.type as keyof DecayHalfLife] ?? config.decayHalfLife.default;
+	const decayFactor = 0.5 ** (daysSinceAccess / halfLife);
+	const edgeBoost = Math.min(entity.edge_count * 0.02, 0.3);
+
+	let newImportance = Math.max(entity.base_importance * decayFactor + edgeBoost, 0.01);
+
+	// Highly-connected entities never drop below 0.25
+	if (entity.edge_count > 20 && newImportance < 0.25) {
+		newImportance = 0.25;
+	}
+
+	return newImportance;
+}
+
+/**
+ * Process a single batch of entities for importance decay.
+ *
+ * Queries active, non-invalidated entities ordered by least-recently accessed,
+ * applies the decay formula, and updates each entity's importance. Entities
+ * whose importance falls below `config.archiveThreshold` are archived.
+ */
+export async function decayBatch(
+	db: SiaDb,
+	config: SiaConfig,
+	batchSize: number,
+	offset: number,
+): Promise<BatchResult> {
+	const result = await db.execute(
+		`SELECT * FROM graph_nodes
+		 WHERE archived_at IS NULL AND t_valid_until IS NULL
+		 ORDER BY last_accessed ASC
+		 LIMIT ? OFFSET ?`,
+		[batchSize, offset],
+	);
+
+	const entities = result.rows as unknown as Entity[];
+	const now = Date.now();
+
+	for (const entity of entities) {
+		const newImportance = computeDecayedImportance(entity, config, now);
+		const daysSinceAccess = (now - entity.last_accessed) / MS_PER_DAY;
+
+		const newConfidence = computeConfidence(
+			entity.base_confidence,
+			entity.trust_tier as 1 | 2 | 3 | 4,
+			entity.type,
+			daysSinceAccess,
+		);
+
+		const updates: Record<string, unknown> = { importance: newImportance };
+		if (Math.abs(newConfidence - entity.confidence) > 0.01) {
+			updates.confidence = newConfidence;
+		}
+		await updateEntity(db, entity.id, updates);
+	}
+
+	return {
+		processed: entities.length,
+		remaining: entities.length === batchSize,
+	};
+}
+
+/**
+ * Run importance decay across all active entities.
+ *
+ * Iterates through the full set in batches of 500, applying `decayBatch`
+ * to each page. Returns the total number of entities processed and the
+ * wall-clock duration.
+ */
+export async function decayImportance(db: SiaDb, config: SiaConfig): Promise<DecayResult> {
+	const start = Date.now();
+	let total = 0;
+	let offset = 0;
+	let hasMore = true;
+
+	while (hasMore) {
+		const batch = await decayBatch(db, config, BATCH_SIZE, offset);
+		total += batch.processed;
+		hasMore = batch.remaining;
+		offset += BATCH_SIZE;
+	}
+
+	return {
+		processed: total,
+		durationMs: Date.now() - start,
+	};
+}