@rkarim08/sia 1.0.0

package/src/freshness/confidence-decay.ts

@@ -0,0 +1,122 @@
+/**
+ * confidence-decay.ts — Layer 4 of the freshness engine.
+ *
+ * Trust-tier-specific confidence decay with Bayesian re-observation reinforcement.
+ *
+ * Key insight:
+ *   - Tier 2 (AST-derived): event-driven invalidation only — no time decay.
+ *   - Tier 3 (LLM-inferred): exponential decay × Beta(α,β) Bayesian confidence.
+ *   - Tier 1 (User-stated): slow exponential decay, no Bayesian adjustment.
+ *   - Tier 4 (External): fast exponential decay, no Bayesian adjustment.
+ */
+
+export interface DecayParams {
+	halfLifeDays: number;
+	decayMultiplier: number;
+	reObservationBoost: number; // how much α increments per re-observation
+}
+
+/** Decay parameters by trust tier and entity type. */
+export const DECAY_PARAMS: Record<string, DecayParams> = {
+	tier2: { halfLifeDays: Infinity, decayMultiplier: 0, reObservationBoost: 0 }, // event-driven only
+	Decision: { halfLifeDays: 14, decayMultiplier: 1.0, reObservationBoost: 1 },
+	Convention: { halfLifeDays: 21, decayMultiplier: 1.0, reObservationBoost: 1 },
+	Bug: { halfLifeDays: 7, decayMultiplier: 1.5, reObservationBoost: 1 },
+	Solution: { halfLifeDays: 7, decayMultiplier: 1.5, reObservationBoost: 1 },
+	Concept: { halfLifeDays: 14, decayMultiplier: 1.0, reObservationBoost: 1 },
+	tier1_user: { halfLifeDays: 30, decayMultiplier: 0.5, reObservationBoost: 2 },
+	tier4_external: { halfLifeDays: 7, decayMultiplier: 3.0, reObservationBoost: 1 },
+	event: { halfLifeDays: 1 / 24, decayMultiplier: 1.0, reObservationBoost: 0 }, // 1 hour
+};
+
+/** Default tier-3 decay params used when the entity type is not explicitly listed. */
+const TIER3_DEFAULT = DECAY_PARAMS.Decision as DecayParams;
+
+export interface BayesianState {
+	alpha: number; // successful re-observations (starts at 1)
+	beta: number; // contradictions (starts at 0)
+}
+
+/**
+ * Get the appropriate decay params for a node given its trust tier and entity type.
+ */
+export function getDecayParams(trustTier: 1 | 2 | 3 | 4, entityType: string): DecayParams {
+	if (trustTier === 2) return DECAY_PARAMS.tier2 as DecayParams;
+	if (trustTier === 1) return DECAY_PARAMS.tier1_user as DecayParams;
+	if (trustTier === 4) return DECAY_PARAMS.tier4_external as DecayParams;
+
+	// Tier 3 — look up by entity type, fall back to Decision
+	return DECAY_PARAMS[entityType] ?? TIER3_DEFAULT;
+}
+
+/**
+ * Compute current confidence for a node based on its trust tier, type, and age.
+ *
+ * Tier 2 (AST-derived): binary — 1.0 if source unchanged, 0.0 if changed.
+ * Tier 3 (LLM-inferred): exponential decay × Bayesian re-observation (min of both).
+ * Tier 1 (User-stated): slow exponential decay only.
+ * Tier 4 (External): fast exponential decay only.
+ *
+ * Formula:
+ *   λ = ln(2) / halfLifeDays
+ *   decayed = baseConfidence × e^(-λ × decayMultiplier × daysSinceAccess)
+ *   bayesian = α / (α + β)
+ *   final (tier 3) = min(decayed, bayesian)
+ *   final (tier 1/4) = decayed
+ *   final (tier 2) = sourceUnchanged ? 1.0 : 0.0
+ */
+export function computeConfidence(
+	baseConfidence: number,
+	trustTier: 1 | 2 | 3 | 4,
+	entityType: string,
+	daysSinceAccess: number,
+	bayesian?: BayesianState,
+	sourceUnchanged?: boolean,
+): number {
+	// Tier 2: binary, event-driven
+	if (trustTier === 2) {
+		return sourceUnchanged === true ? 1.0 : 0.0;
+	}
+
+	const params = getDecayParams(trustTier, entityType);
+
+	// Compute exponential decay
+	let decayed: number;
+	if (params.halfLifeDays === Infinity || params.decayMultiplier === 0 || daysSinceAccess === 0) {
+		decayed = baseConfidence;
+	} else {
+		const lambda = Math.LN2 / params.halfLifeDays;
+		decayed = baseConfidence * Math.exp(-lambda * params.decayMultiplier * daysSinceAccess);
+	}
+
+	// Tier 3: combine decay with Bayesian state
+	if (trustTier === 3 && bayesian !== undefined) {
+		const bayes = bayesianConfidence(bayesian);
+		return Math.min(decayed, bayes);
+	}
+
+	return decayed;
+}
+
+/**
+ * Record a successful re-observation: increment α by the boost amount.
+ * Returns a new BayesianState (does not mutate the original).
+ */
+export function recordReObservation(state: BayesianState, boost = 1): BayesianState {
+	return { alpha: state.alpha + boost, beta: state.beta };
+}
+
+/**
+ * Record a contradiction: increment β by 1.
+ * Returns a new BayesianState (does not mutate the original).
+ */
+export function recordContradiction(state: BayesianState): BayesianState {
+	return { alpha: state.alpha, beta: state.beta + 1 };
+}
+
+/**
+ * Get the Bayesian confidence: α / (α + β).
+ */
+export function bayesianConfidence(state: BayesianState): number {
+	return state.alpha / (state.alpha + state.beta);
+}

package/src/freshness/cuckoo-filter.ts

@@ -0,0 +1,176 @@
+// Module: cuckoo-filter — Probabilistic set membership with deletion support
+//
+// Real fingerprint-based Cuckoo filter (Fan et al. 2014):
+// - 16-bit fingerprints in Uint16Array buckets (4 slots/bucket)
+// - FNV-1a hash: lower bits → bucket index, upper 16 bits → fingerprint
+// - Alternate bucket via XOR with prime-multiplied fingerprint
+// - Auxiliary Set<number> of 31-bit hashes for exact size tracking
+// - Max 500 eviction kicks before declaring full
+//
+// Memory: ~328KB for 50K paths (128KB filter + 200KB hash set)
+// FP rate: ~0.012% theoretical (16-bit fingerprints, 4 slots/bucket)
+
+import type { SiaDb } from "@/graph/db-interface";
+
+const BUCKET_SIZE = 4;
+const MAX_KICKS = 500;
+const MAX_BUCKETS = 65536;
+
+function fnv1a(str: string): number {
+	let hash = 0x811c9dc5;
+	for (let i = 0; i < str.length; i++) {
+		hash ^= str.charCodeAt(i);
+		hash = (hash * 0x01000193) >>> 0;
+	}
+	return hash;
+}
+
+function nextPowerOfTwo(n: number): number {
+	if (n <= 1) return 1;
+	let p = 1;
+	while (p < n) p <<= 1;
+	return p;
+}
+
+export class CuckooFilter {
+	private data: Uint16Array;
+	private numBuckets: number;
+	private hashes: Set<number>;
+	private _size: number;
+
+	constructor(capacity = 65536) {
+		this.numBuckets = nextPowerOfTwo(Math.ceil(Math.max(capacity, 1) / BUCKET_SIZE / 0.95));
+		if (this.numBuckets > MAX_BUCKETS) {
+			throw new Error(
+				`CuckooFilter capacity too large: ${capacity} requires ${this.numBuckets} buckets (max ${MAX_BUCKETS})`,
+			);
+		}
+		this.data = new Uint16Array(this.numBuckets * BUCKET_SIZE);
+		this.hashes = new Set();
+		this._size = 0;
+	}
+
+	add(path: string): boolean {
+		const hash = fnv1a(path);
+		const smiHash = hash >>> 1;
+		if (this.hashes.has(smiHash)) return true;
+
+		const fp = hash >>> 16 || 1;
+		const h1 = hash & (this.numBuckets - 1);
+		const h2 = (h1 ^ (((fp * 0x5bd1e995) >>> 0) & (this.numBuckets - 1))) & (this.numBuckets - 1);
+
+		if (this._bucketInsert(h1, fp)) {
+			this.hashes.add(smiHash);
+			this._size++;
+			return true;
+		}
+		if (this._bucketInsert(h2, fp)) {
+			this.hashes.add(smiHash);
+			this._size++;
+			return true;
+		}
+
+		let evictBucket = Math.random() < 0.5 ? h1 : h2;
+		let evictFp = fp;
+
+		for (let kick = 0; kick < MAX_KICKS; kick++) {
+			const slotIdx = Math.floor(Math.random() * BUCKET_SIZE);
+			const offset = evictBucket * BUCKET_SIZE + slotIdx;
+			const evicted = this.data[offset];
+			this.data[offset] = evictFp;
+			evictFp = evicted;
+
+			const altBucket =
+				(evictBucket ^ (((evictFp * 0x5bd1e995) >>> 0) & (this.numBuckets - 1))) &
+				(this.numBuckets - 1);
+
+			if (this._bucketInsert(altBucket, evictFp)) {
+				this.hashes.add(smiHash);
+				this._size++;
+				return true;
+			}
+			evictBucket = altBucket;
+		}
+
+		return false;
+	}
+
+	remove(path: string): boolean {
+		const hash = fnv1a(path);
+		const smiHash = hash >>> 1;
+		if (!this.hashes.has(smiHash)) return false;
+
+		const fp = hash >>> 16 || 1;
+		const h1 = hash & (this.numBuckets - 1);
+		const h2 = (h1 ^ (((fp * 0x5bd1e995) >>> 0) & (this.numBuckets - 1))) & (this.numBuckets - 1);
+
+		if (this._bucketRemove(h1, fp)) {
+			this.hashes.delete(smiHash);
+			this._size--;
+			return true;
+		}
+		if (this._bucketRemove(h2, fp)) {
+			this.hashes.delete(smiHash);
+			this._size--;
+			return true;
+		}
+		return false;
+	}
+
+	contains(path: string): boolean {
+		const hash = fnv1a(path);
+		const fp = hash >>> 16 || 1;
+		const h1 = hash & (this.numBuckets - 1);
+		const h2 = (h1 ^ (((fp * 0x5bd1e995) >>> 0) & (this.numBuckets - 1))) & (this.numBuckets - 1);
+		return this._bucketContains(h1, fp) || this._bucketContains(h2, fp);
+	}
+
+	clear(): void {
+		this.data.fill(0);
+		this.hashes.clear();
+		this._size = 0;
+	}
+
+	get size(): number {
+		return this._size;
+	}
+
+	private _bucketContains(bucket: number, fp: number): boolean {
+		const base = bucket * BUCKET_SIZE;
+		for (let i = 0; i < BUCKET_SIZE; i++) {
+			if (this.data[base + i] === fp) return true;
+		}
+		return false;
+	}
+
+	private _bucketInsert(bucket: number, fp: number): boolean {
+		const base = bucket * BUCKET_SIZE;
+		for (let i = 0; i < BUCKET_SIZE; i++) {
+			if (this.data[base + i] === 0) {
+				this.data[base + i] = fp;
+				return true;
+			}
+		}
+		return false;
+	}
+
+	private _bucketRemove(bucket: number, fp: number): boolean {
+		const base = bucket * BUCKET_SIZE;
+		for (let i = 0; i < BUCKET_SIZE; i++) {
+			if (this.data[base + i] === fp) {
+				this.data[base + i] = 0;
+				return true;
+			}
+		}
+		return false;
+	}
+
+	static async fromDatabase(db: SiaDb): Promise<CuckooFilter> {
+		const filter = new CuckooFilter();
+		const { rows } = await db.execute("SELECT DISTINCT source_path FROM source_deps");
+		for (const row of rows) {
+			filter.add(row.source_path as string);
+		}
+		return filter;
+	}
+}

package/src/freshness/deep-validation.ts

@@ -0,0 +1,345 @@
+/**
+ * deep-validation.ts — Layer 5 of the freshness engine.
+ *
+ * Periodic deep validation pipeline (nightly / weekly).
+ * Catches anything that real-time layers (1–4) missed:
+ *
+ *   (a) Documentation-vs-code cross-validation
+ *   (b) Low-confidence LLM-inferred claim re-verification
+ *   (c) PageRank importance score recomputation
+ *   (d) Version compaction (archived entity purge + FTS5 optimize)
+ *
+ * Must be run in a separate DB connection — never blocks the MCP server.
+ */
+
+import { existsSync, statSync } from "node:fs";
+import { computePageRank } from "@/ast/pagerank-builder";
+import type { SiaDb } from "@/graph/db-interface";
+import { invalidateEntity } from "@/graph/entities";
+import { updateImportanceScores } from "@/retrieval/pagerank";
+
+// ─── Public types ───────────────────────────────────────────────────────────
+
+export interface DeepValidationResult {
+	documentsChecked: number;
+	staleDocsFound: number;
+	claimsReVerified: number;
+	claimsInvalidated: number;
+	claimsConfirmed: number;
+	nodesScored: number; // PageRank
+	versionsCompacted: number;
+	ftsOptimized: boolean;
+	durationMs: number;
+}
+
+export interface DeepValidationConfig {
+	maxClaimsToVerify: number; // default 20
+	retentionDays: number; // default 90
+	eventRetentionDays: number; // default 30
+	archiveThreshold: number; // default 0.05
+}
+
+const DEFAULT_CONFIG: DeepValidationConfig = {
+	maxClaimsToVerify: 20,
+	retentionDays: 90,
+	eventRetentionDays: 30,
+	archiveThreshold: 0.05,
+};
+
+// ─── Main pipeline ──────────────────────────────────────────────────────────
+
+/**
+ * Run the full deep validation pipeline.
+ * Should be called nightly via the decay scheduler.
+ * Runs in a separate DB connection — never blocks the MCP server.
+ */
+export async function runDeepValidation(
+	db: SiaDb,
+	repoRoot: string,
+	config?: Partial<DeepValidationConfig>,
+): Promise<DeepValidationResult> {
+	const cfg: DeepValidationConfig = { ...DEFAULT_CONFIG, ...config };
+	const start = Date.now();
+
+	// (a) Documentation cross-validation
+	const { checked, staleFound } = await validateDocumentation(db, repoRoot);
+
+	// (b) Low-confidence claim re-verification
+	const { verified, invalidated, confirmed } = await identifyLowConfidenceClaims(
+		db,
+		cfg.maxClaimsToVerify,
+	);
+
+	// (c) PageRank recomputation
+	const { nodesScored } = await recomputePageRank(db);
+
+	// (d) Version compaction
+	const { compacted, ftsOptimized } = await compactVersions(db, cfg);
+
+	const durationMs = Date.now() - start;
+
+	return {
+		documentsChecked: checked,
+		staleDocsFound: staleFound,
+		claimsReVerified: verified,
+		claimsInvalidated: invalidated,
+		claimsConfirmed: confirmed,
+		nodesScored,
+		versionsCompacted: compacted,
+		ftsOptimized,
+		durationMs,
+	};
+}
+
+// ─── Sub-task (a): Documentation-vs-code cross-validation ──────────────────
+
+/**
+ * Sub-task (a): Documentation-vs-code cross-validation.
+ *
+ * For each entity with trust_tier 1 and type in ('CodeEntity','Convention','Decision')
+ * that has a non-empty file_paths array, stat() the referenced files.
+ * If any file was modified (mtime) after the entity's t_created, tag it as potentially-stale
+ * and reduce its confidence.
+ */
+export async function validateDocumentation(
+	db: SiaDb,
+	repoRoot: string,
+): Promise<{ checked: number; staleFound: number }> {
+	const { rows } = await db.execute(
+		`SELECT id, file_paths, t_created, confidence, tags
+		 FROM graph_nodes
+		 WHERE type IN ('CodeEntity', 'Convention', 'Decision')
+		   AND trust_tier = 1
+		   AND file_paths IS NOT NULL
+		   AND file_paths != '[]'
+		   AND t_valid_until IS NULL
+		   AND archived_at IS NULL`,
+	);
+
+	let checked = 0;
+	let staleFound = 0;
+
+	for (const row of rows) {
+		const entityId = row.id as string;
+		const tCreated = row.t_created as number;
+		const filePathsRaw = row.file_paths as string;
+		const currentTagsRaw = row.tags as string;
+		const currentConfidence = row.confidence as number;
+
+		let filePaths: string[];
+		try {
+			filePaths = JSON.parse(filePathsRaw) as string[];
+		} catch {
+			continue;
+		}
+
+		if (!Array.isArray(filePaths) || filePaths.length === 0) continue;
+
+		checked++;
+
+		let isStale = false;
+
+		for (const rawPath of filePaths) {
+			if (typeof rawPath !== "string" || rawPath.length === 0) continue;
+
+			// Resolve relative paths against repoRoot
+			const absPath = rawPath.startsWith("/") ? rawPath : `${repoRoot}/${rawPath}`;
+
+			if (!existsSync(absPath)) continue;
+
+			try {
+				const stat = statSync(absPath);
+				const mtimeMs = stat.mtimeMs;
+				if (mtimeMs > tCreated) {
+					isStale = true;
+					break;
+				}
+			} catch {
+				// Stat failed — skip this path
+			}
+		}
+
+		if (isStale) {
+			staleFound++;
+
+			// Tag entity as potentially-stale and reduce confidence
+			let tags: string[];
+			try {
+				tags = JSON.parse(currentTagsRaw) as string[];
+			} catch {
+				tags = [];
+			}
+			if (!tags.includes("potentially-stale")) {
+				tags.push("potentially-stale");
+			}
+
+			const newConfidence = Math.max(0.01, currentConfidence - 0.1);
+			await db.execute("UPDATE graph_nodes SET tags = ?, confidence = ? WHERE id = ?", [
+				JSON.stringify(tags),
+				newConfidence,
+				entityId,
+			]);
+		}
+	}
+
+	return { checked, staleFound };
+}
+
+// ─── Sub-task (b): Low-confidence claim re-verification ────────────────────
+
+/**
+ * Sub-task (b): Sample lowest-confidence LLM-inferred (tier-3) entities
+ * and flag them for re-verification.
+ *
+ * In production this would call an LLM (e.g. Haiku), but for now we identify
+ * candidates and update their Bayesian state based on whether source files
+ * still exist on disk.
+ *
+ * - If the entity has no file_paths or all referenced files still exist → confirmed
+ * - If any referenced file has been deleted → invalidated
+ */
+export async function identifyLowConfidenceClaims(
+	db: SiaDb,
+	maxClaims = 20,
+): Promise<{ verified: number; invalidated: number; confirmed: number }> {
+	const { rows } = await db.execute(
+		`SELECT id, file_paths, confidence
+		 FROM graph_nodes
+		 WHERE trust_tier = 3
+		   AND t_valid_until IS NULL
+		   AND archived_at IS NULL
+		 ORDER BY confidence ASC
+		 LIMIT ?`,
+		[maxClaims],
+	);
+
+	let verified = 0;
+	let invalidated = 0;
+	let confirmed = 0;
+
+	for (const row of rows) {
+		const entityId = row.id as string;
+		const filePathsRaw = row.file_paths as string;
+
+		let filePaths: string[];
+		try {
+			filePaths = JSON.parse(filePathsRaw) as string[];
+		} catch {
+			filePaths = [];
+		}
+
+		verified++;
+
+		if (!Array.isArray(filePaths) || filePaths.length === 0) {
+			// No source file to check — treat as confirmed
+			confirmed++;
+			await db.execute(
+				"UPDATE graph_nodes SET confidence = MIN(1.0, confidence + 0.05) WHERE id = ?",
+				[entityId],
+			);
+			continue;
+		}
+
+		// Check if any source file has been deleted
+		const hasDeletedFile = filePaths.some(
+			(p) => typeof p === "string" && p.length > 0 && !existsSync(p),
+		);
+
+		if (hasDeletedFile) {
+			invalidated++;
+			await invalidateEntity(db, entityId);
+		} else {
+			confirmed++;
+			// Bump confidence slightly as a re-observation
+			await db.execute(
+				"UPDATE graph_nodes SET confidence = MIN(1.0, confidence + 0.05) WHERE id = ?",
+				[entityId],
+			);
+		}
+	}
+
+	return { verified, invalidated, confirmed };
+}
+
+// ─── Sub-task (c): PageRank recomputation ──────────────────────────────────
+
+/**
+ * Sub-task (c): Recompute PageRank importance scores.
+ * Uses the existing computePageRank from ast/pagerank-builder.ts.
+ */
+export async function recomputePageRank(db: SiaDb): Promise<{ nodesScored: number }> {
+	const result = await computePageRank(db);
+
+	// computePageRank already writes importance scores directly.
+	// We call updateImportanceScores with an empty map to emit the audit log entry,
+	// but only if there were actually nodes to avoid a no-op audit spam.
+	if (result.nodesScored > 0) {
+		// Build score map from the computed results
+		const { rows } = await db.execute(
+			"SELECT id, importance FROM graph_nodes WHERE t_valid_until IS NULL AND archived_at IS NULL",
+		);
+		const scores = new Map<string, number>(
+			rows.map((r) => [r.id as string, r.importance as number]),
+		);
+		await updateImportanceScores(db, scores);
+	}
+
+	return { nodesScored: result.nodesScored };
+}
+
+// ─── Sub-task (d): Version compaction ──────────────────────────────────────
+
+/**
+ * Sub-task (d): Version compaction.
+ *
+ * 1. Hard-delete archived entities older than `retentionDays`.
+ * 2. Hard-delete archived event entities with low importance, zero edges,
+ *    and older than `eventRetentionDays`.
+ * 3. Run FTS5 optimize.
+ */
+export async function compactVersions(
+	db: SiaDb,
+	config?: Partial<DeepValidationConfig>,
+): Promise<{ compacted: number; ftsOptimized: boolean }> {
+	const cfg: DeepValidationConfig = { ...DEFAULT_CONFIG, ...config };
+
+	const now = Date.now();
+	const retentionCutoff = now - cfg.retentionDays * 86_400_000;
+	const eventRetentionCutoff = now - cfg.eventRetentionDays * 86_400_000;
+
+	// 1. Delete archived entities beyond the main retention window
+	const { rows: deletedRows } = await db.execute(
+		`DELETE FROM graph_nodes
+		 WHERE archived_at IS NOT NULL
+		   AND archived_at < ?
+		 RETURNING id`,
+		[retentionCutoff],
+	);
+	let compacted = deletedRows.length;
+
+	// 2. Delete archived event entities with low importance and zero edges
+	//    that are beyond the event retention window
+	const { rows: deletedEventRows } = await db.execute(
+		`DELETE FROM graph_nodes
+		 WHERE type LIKE '%Event'
+		   AND importance < ?
+		   AND edge_count = 0
+		   AND archived_at IS NOT NULL
+		   AND archived_at < ?
+		 RETURNING id`,
+		[cfg.archiveThreshold, eventRetentionCutoff],
+	);
+	compacted += deletedEventRows.length;
+
+	// 3. Optimize the FTS5 virtual table
+	let ftsOptimized = false;
+	try {
+		await db.execute("INSERT INTO graph_nodes_fts(graph_nodes_fts) VALUES('optimize')", []);
+		ftsOptimized = true;
+	} catch {
+		// FTS5 table may not exist in all test environments — treat as non-fatal
+		ftsOptimized = false;
+	}
+
+	return { compacted, ftsOptimized };
+}