@rkarim08/sia 1.0.0

package/src/decay/deep-validator.ts

@@ -0,0 +1,62 @@
+// Module: deep-validator — LLM re-verification of lowest-confidence Tier 3 entities
+
+import type { BatchResult } from "@/decay/types";
+import type { SiaDb } from "@/graph/db-interface";
+import { invalidateEntity, updateEntity } from "@/graph/entities";
+import type { LlmClient } from "@/shared/llm-client";
+
+/**
+ * Validate one low-confidence Tier 3 entity using LLM classification.
+ *
+ * Picks the active Tier 3 entity with the lowest confidence that hasn't
+ * been validated recently (last_accessed > 24h ago), asks the LLM whether
+ * the entity's content is still valid, and either boosts confidence or
+ * invalidates it.
+ *
+ * Rate-limited externally by the maintenance scheduler to 1 call per
+ * config.deepValidationRateMs (default 5s).
+ */
+export async function deepValidateBatch(
+	db: SiaDb,
+	llmClient: LlmClient,
+	batchSize: number,
+): Promise<BatchResult> {
+	const cutoff = Date.now() - 86_400_000; // 24h ago
+
+	const { rows } = await db.execute(
+		`SELECT id, name, type, content, confidence FROM graph_nodes
+		 WHERE trust_tier = 3
+		   AND t_valid_until IS NULL
+		   AND archived_at IS NULL
+		   AND last_accessed < ?
+		 ORDER BY confidence ASC
+		 LIMIT ?`,
+		[cutoff, batchSize],
+	);
+
+	let processed = 0;
+
+	for (const row of rows) {
+		const entityId = row.id as string;
+		const name = row.name as string;
+		const content = row.content as string;
+		const currentConfidence = row.confidence as number;
+
+		const prompt = `Is the following knowledge entity still likely to be accurate and relevant?\n\nName: ${name}\nContent: ${content}\n\nRespond with YES if it appears valid, or NO if it appears outdated or incorrect.`;
+
+		const verdict = await llmClient.classify(prompt, ["YES", "NO"]);
+
+		if (verdict === "YES") {
+			// Boost confidence slightly (capped at 0.9 for Tier 3)
+			const newConfidence = Math.min(currentConfidence + 0.1, 0.9);
+			await updateEntity(db, entityId, { confidence: newConfidence });
+		} else {
+			// Invalidate — the LLM thinks this is no longer valid
+			await invalidateEntity(db, entityId);
+		}
+
+		processed++;
+	}
+
+	return { processed, remaining: processed === batchSize };
+}

package/src/decay/episodic-promoter.ts

@@ -0,0 +1,132 @@
+// Module: episodic-promoter — re-process failed/incomplete sessions
+
+import type { CandidateFact } from "@/capture/types";
+import type { BatchResult } from "@/decay/types";
+import type { SiaDb } from "@/graph/db-interface";
+
+/**
+ * Gather session IDs that need (re-)processing:
+ * 1. Explicitly failed sessions (processing_status = 'failed')
+ * 2. Orphaned sessions (episodes exist but no sessions_processed row)
+ */
+async function findUnprocessedSessions(episodicDb: SiaDb): Promise<string[]> {
+	const failed = await episodicDb.execute(
+		"SELECT session_id FROM sessions_processed WHERE processing_status = 'failed'",
+	);
+
+	const orphaned = await episodicDb.execute(
+		"SELECT DISTINCT session_id FROM episodes WHERE session_id NOT IN (SELECT session_id FROM sessions_processed)",
+	);
+
+	const ids = new Set<string>();
+	for (const row of failed.rows) {
+		ids.add(row.session_id as string);
+	}
+	for (const row of orphaned.rows) {
+		ids.add(row.session_id as string);
+	}
+
+	return [...ids];
+}
+
+/**
+ * Convert raw episode rows into CandidateFact instances.
+ *
+ * Uses a simplified extraction approach: each episode becomes a single
+ * Concept candidate. Full Track A + Track B extraction is not invoked here
+ * because this is a maintenance sweep, not a real-time pipeline.
+ */
+function episodesToCandidates(episodes: Record<string, unknown>[]): CandidateFact[] {
+	const candidates: CandidateFact[] = [];
+
+	for (const ep of episodes) {
+		const content = (ep.content as string) ?? "";
+		if (content.trim().length === 0) continue;
+
+		const filePaths: string[] = [];
+		if (ep.file_path && typeof ep.file_path === "string") {
+			filePaths.push(ep.file_path);
+		}
+
+		const trustTier = (ep.trust_tier as 1 | 2 | 3 | 4) ?? 2;
+
+		candidates.push({
+			type: "Concept",
+			name: content.slice(0, 50),
+			content,
+			summary: content.slice(0, 80),
+			tags: [],
+			file_paths: filePaths,
+			trust_tier: trustTier,
+			confidence: 0.5,
+			extraction_method: "episodic-promoter",
+		});
+	}
+
+	return candidates;
+}
+
+/**
+ * Process up to `batchSize` failed or orphaned sessions, promoting their
+ * episodic content into the semantic graph via consolidation.
+ *
+ * Returns `{ processed, remaining }` where `remaining` is true when the
+ * batch was full (more work may exist).
+ */
+export async function promoteBatch(
+	graphDb: SiaDb,
+	episodicDb: SiaDb,
+	batchSize: number,
+): Promise<BatchResult> {
+	const sessionIds = await findUnprocessedSessions(episodicDb);
+	const toProcess = sessionIds.slice(0, batchSize);
+
+	// Dynamic import so capture internals are only loaded when actually needed
+	const { consolidate } = await import("@/capture/consolidate");
+
+	let processed = 0;
+
+	for (const sessionId of toProcess) {
+		const { rows: episodes } = await episodicDb.execute(
+			"SELECT content, trust_tier, tool_name, file_path FROM episodes WHERE session_id = ? ORDER BY ts ASC",
+			[sessionId],
+		);
+
+		const candidates = episodesToCandidates(episodes);
+
+		try {
+			await consolidate(graphDb, candidates);
+
+			await episodicDb.execute(
+				"INSERT OR REPLACE INTO sessions_processed (session_id, processing_status, processed_at, entity_count, pipeline_version) VALUES (?, 'complete', ?, ?, 'maintenance-sweep')",
+				[sessionId, Date.now(), candidates.length],
+			);
+
+			processed++;
+		} catch (_err) {
+			// Mark as failed so next sweep retries
+			await episodicDb.execute(
+				"INSERT OR REPLACE INTO sessions_processed (session_id, processing_status, processed_at, entity_count, pipeline_version) VALUES (?, 'failed', ?, 0, 'maintenance-sweep')",
+				[sessionId, Date.now()],
+			);
+		}
+	}
+
+	return { processed, remaining: processed === batchSize };
+}
+
+/**
+ * Drain all failed and orphaned sessions, processing one at a time.
+ * Returns the total number of sessions successfully promoted.
+ */
+export async function promoteFailedSessions(graphDb: SiaDb, episodicDb: SiaDb): Promise<number> {
+	let total = 0;
+
+	for (;;) {
+		const { processed, remaining } = await promoteBatch(graphDb, episodicDb, 1);
+		total += processed;
+		if (!remaining) break;
+	}
+
+	return total;
+}

package/src/decay/maintenance-scheduler.ts

@@ -0,0 +1,326 @@
+// Module: maintenance-scheduler — central orchestrator for decay/lifecycle work units
+//
+// Three trigger modes:
+// 1. Startup Catchup — if > maintenanceInterval since last sweep, run full sweep
+// 2. Idle Opportunistic — on 60s idle gap, run ONE batch from highest-priority unit
+// 3. Session-End Sweep — targeted dedup of current session's entities
+
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { archiveBatch } from "@/decay/archiver";
+import { bridgeOrphanBatch } from "@/decay/bridge-orphan-cleanup";
+import { consolidationSweepBatch } from "@/decay/consolidation-sweep";
+import { decayBatch } from "@/decay/decay";
+import { deepValidateBatch } from "@/decay/deep-validator";
+import { promoteBatch } from "@/decay/episodic-promoter";
+import { sweepSession } from "@/decay/session-sweeper";
+import type { BatchResult } from "@/decay/types";
+import type { SiaDb } from "@/graph/db-interface";
+import { SIA_HOME, type SiaConfig } from "@/shared/config";
+import type { LlmClient } from "@/shared/llm-client";
+
+// ---------------------------------------------------------------------------
+// State file persistence
+// ---------------------------------------------------------------------------
+
+interface MaintenanceState {
+	lastSweepAt: number;
+	lastSessionSweepAt: number;
+	pendingBatchOffset: number;
+}
+
+const DEFAULT_STATE: MaintenanceState = {
+	lastSweepAt: 0,
+	lastSessionSweepAt: 0,
+	pendingBatchOffset: 0,
+};
+
+function stateFilePath(repoHash: string, siaHome?: string): string {
+	return join(siaHome ?? SIA_HOME, "repos", repoHash, "maintenance.json");
+}
+
+export function loadMaintenanceState(repoHash: string, siaHome?: string): MaintenanceState {
+	const path = stateFilePath(repoHash, siaHome);
+	if (!existsSync(path)) return { ...DEFAULT_STATE };
+	try {
+		return JSON.parse(readFileSync(path, "utf-8")) as MaintenanceState;
+	} catch {
+		return { ...DEFAULT_STATE };
+	}
+}
+
+export function saveMaintenanceState(
+	repoHash: string,
+	state: MaintenanceState,
+	siaHome?: string,
+): void {
+	const path = stateFilePath(repoHash, siaHome);
+	const dir = join(siaHome ?? SIA_HOME, "repos", repoHash);
+	mkdirSync(dir, { recursive: true });
+	writeFileSync(path, JSON.stringify(state, null, 2), "utf-8");
+}
+
+// ---------------------------------------------------------------------------
+// Work unit registry
+// ---------------------------------------------------------------------------
+
+type WorkUnitFn = (batchSize: number) => Promise<BatchResult>;
+
+interface WorkUnit {
+	name: string;
+	fn: WorkUnitFn;
+}
+
+function buildWorkUnits(
+	graphDb: SiaDb,
+	episodicDb: SiaDb | null,
+	bridgeDb: SiaDb | null,
+	config: SiaConfig,
+	llmClient: LlmClient | null,
+): WorkUnit[] {
+	const units: WorkUnit[] = [
+		{
+			name: "decay",
+			fn: (batchSize) => decayBatch(graphDb, config, batchSize, 0),
+		},
+		{
+			name: "archival",
+			fn: (batchSize) => archiveBatch(graphDb, config, batchSize),
+		},
+		{
+			name: "consolidation",
+			fn: (batchSize) => consolidationSweepBatch(graphDb, batchSize),
+		},
+	];
+
+	if (episodicDb) {
+		units.push({
+			name: "episodic-promotion",
+			fn: (batchSize) => promoteBatch(graphDb, episodicDb, batchSize),
+		});
+	}
+
+	if (llmClient) {
+		units.push({
+			name: "deep-validation",
+			fn: (batchSize) => deepValidateBatch(graphDb, llmClient, batchSize),
+		});
+	}
+
+	if (bridgeDb) {
+		units.push({
+			name: "bridge-orphan",
+			fn: (batchSize) => bridgeOrphanBatch(bridgeDb, batchSize),
+		});
+	}
+
+	return units;
+}
+
+// ---------------------------------------------------------------------------
+// Scheduler implementation
+// ---------------------------------------------------------------------------
+
+export interface MaintenanceSchedulerOpts {
+	graphDb: SiaDb;
+	episodicDb?: SiaDb | null;
+	bridgeDb?: SiaDb | null;
+	config: SiaConfig;
+	repoHash: string;
+	llmClient?: LlmClient | null;
+	siaHome?: string;
+}
+
+export interface MaintenanceScheduler {
+	onStartup(repoHash: string): Promise<void>;
+	onPostToolUse(): void;
+	onSessionEnd(sessionId: string): Promise<void>;
+	stop(): void;
+}
+
+/**
+ * Create a maintenance scheduler instance.
+ *
+ * The scheduler manages three trigger modes:
+ * - Startup: full sweep if overdue (> maintenanceInterval)
+ * - Idle: one batch per idle cycle (60s gap in PostToolUse events)
+ * - Session-end: targeted dedup of session entities
+ */
+export function createMaintenanceScheduler(opts: MaintenanceSchedulerOpts): MaintenanceScheduler {
+	const { graphDb, config, repoHash, siaHome } = opts;
+	const episodicDb = opts.episodicDb ?? null;
+	const bridgeDb = opts.bridgeDb ?? null;
+	const llmClient = opts.llmClient ?? null;
+
+	let idleTimer: ReturnType<typeof setTimeout> | null = null;
+	let stopped = false;
+	let lastDeepValidation = 0;
+
+	const workUnits = buildWorkUnits(graphDb, episodicDb, bridgeDb, config, llmClient);
+
+	// -----------------------------------------------------------------------
+	// Startup catchup
+	// -----------------------------------------------------------------------
+
+	async function onStartup(): Promise<void> {
+		if (stopped) return;
+
+		const state = loadMaintenanceState(repoHash, siaHome);
+		const elapsed = Date.now() - state.lastSweepAt;
+
+		if (elapsed < config.maintenanceInterval) return;
+
+		// Run full sweep with large batches (500)
+		const STARTUP_BATCH = 500;
+
+		for (const unit of workUnits) {
+			if (stopped) break;
+
+			// Rate-limit deep validation
+			if (unit.name === "deep-validation") {
+				const sinceLast = Date.now() - lastDeepValidation;
+				if (sinceLast < config.deepValidationRateMs) continue;
+			}
+
+			let hasMore = true;
+			while (hasMore && !stopped) {
+				if (unit.name === "deep-validation") {
+					lastDeepValidation = Date.now();
+				}
+
+				const result = await unit.fn(STARTUP_BATCH);
+				hasMore = result.remaining;
+
+				// Yield to event loop between batches
+				await new Promise((r) => setTimeout(r, 0));
+			}
+		}
+
+		// FTS5 optimization after full sweep
+		try {
+			await graphDb.execute("INSERT INTO graph_nodes_fts(graph_nodes_fts) VALUES('optimize')");
+		} catch {
+			// FTS5 table may not exist in all configurations
+		}
+
+		state.lastSweepAt = Date.now();
+		state.pendingBatchOffset = 0;
+		saveMaintenanceState(repoHash, state, siaHome);
+	}
+
+	// -----------------------------------------------------------------------
+	// Idle opportunistic
+	// -----------------------------------------------------------------------
+
+	function scheduleIdleCheck(): void {
+		if (stopped || idleTimer) return;
+
+		idleTimer = setTimeout(async () => {
+			idleTimer = null;
+			if (stopped) return;
+
+			// Run ONE batch from the highest-priority unit that has work
+			for (const unit of workUnits) {
+				if (stopped) break;
+
+				// Rate-limit deep validation
+				if (unit.name === "deep-validation") {
+					const sinceLast = Date.now() - lastDeepValidation;
+					if (sinceLast < config.deepValidationRateMs) continue;
+					lastDeepValidation = Date.now();
+				}
+
+				const result = await unit.fn(50);
+				if (result.processed > 0) break; // Did some work, yield back
+			}
+		}, config.idleTimeoutMs);
+	}
+
+	function onPostToolUse(): void {
+		// Reset idle timer on each tool use
+		if (idleTimer) {
+			clearTimeout(idleTimer);
+			idleTimer = null;
+		}
+		scheduleIdleCheck();
+	}
+
+	// -----------------------------------------------------------------------
+	// Session-end sweep
+	// -----------------------------------------------------------------------
+
+	async function onSessionEnd(sessionId: string): Promise<void> {
+		if (stopped) return;
+
+		await sweepSession(graphDb, sessionId);
+
+		const state = loadMaintenanceState(repoHash, siaHome);
+		state.lastSessionSweepAt = Date.now();
+		saveMaintenanceState(repoHash, state, siaHome);
+	}
+
+	// -----------------------------------------------------------------------
+	// Stop
+	// -----------------------------------------------------------------------
+
+	function stop(): void {
+		stopped = true;
+		if (idleTimer) {
+			clearTimeout(idleTimer);
+			idleTimer = null;
+		}
+	}
+
+	return {
+		onStartup: () => onStartup(),
+		onPostToolUse,
+		onSessionEnd,
+		stop,
+	};
+}
+
+// ---------------------------------------------------------------------------
+// Standalone full sweep (for backward compatibility / CLI usage)
+// ---------------------------------------------------------------------------
+
+/**
+ * Run all maintenance jobs in sequence.
+ *
+ * This is the "old" runNightlyJobs interface, preserved for CLI invocation.
+ * Errors in one job do not prevent others from running.
+ */
+export async function runMaintenanceJobs(
+	config: SiaConfig,
+	graphDb: SiaDb,
+	episodicDb?: SiaDb,
+	bridgeDb?: SiaDb,
+	llmClient?: LlmClient,
+): Promise<void> {
+	const units = buildWorkUnits(
+		graphDb,
+		episodicDb ?? null,
+		bridgeDb ?? null,
+		config,
+		llmClient ?? null,
+	);
+
+	for (const unit of units) {
+		try {
+			let hasMore = true;
+			while (hasMore) {
+				const result = await unit.fn(500);
+				hasMore = result.remaining;
+			}
+			console.log(`[maintenance] ${unit.name}: complete`);
+		} catch (err) {
+			console.error(`[maintenance] ${unit.name}: failed`, err);
+		}
+	}
+
+	// FTS5 optimization
+	try {
+		await graphDb.execute("INSERT INTO graph_nodes_fts(graph_nodes_fts) VALUES('optimize')");
+	} catch {
+		// FTS5 may not exist
+	}
+}

package/src/decay/scheduler.ts

@@ -0,0 +1,6 @@
+// Module: scheduler — re-exports from maintenance-scheduler for backward compatibility
+//
+// The old "nightly scheduler" model has been replaced by the maintenance-scheduler
+// with startup-catchup + idle-opportunistic + session-end triggers.
+
+export { runMaintenanceJobs } from "@/decay/maintenance-scheduler";

package/src/decay/session-sweeper.ts

@@ -0,0 +1,79 @@
+// Module: session-sweeper — session-end focused dedup of current session's entities
+
+import { wordJaccard } from "@/capture/consolidate";
+import type { SiaDb } from "@/graph/db-interface";
+
+/**
+ * Targeted sweep of entities created during a specific session.
+ *
+ * On SessionEnd, deduplicates the session's output against the existing graph:
+ * 1. Query entities where source_episode = sessionId
+ * 2. For each, check local_dedup_log for existing pairs
+ * 3. Compare against same-type active entities (word Jaccard > 0.92)
+ * 4. Write results to local_dedup_log
+ *
+ * Typically completes in < 2s for a session's worth of entities (5-20).
+ */
+export async function sweepSession(db: SiaDb, sessionId: string): Promise<number> {
+	const { rows: sessionEntities } = await db.execute(
+		`SELECT id, type, content FROM graph_nodes
+		 WHERE source_episode = ?
+		   AND t_valid_until IS NULL
+		   AND archived_at IS NULL`,
+		[sessionId],
+	);
+
+	if (sessionEntities.length === 0) return 0;
+
+	let pairsProcessed = 0;
+	const now = Date.now();
+
+	for (const entity of sessionEntities) {
+		const entityId = entity.id as string;
+		const entityType = entity.type as string;
+		const entityContent = entity.content as string;
+
+		// Find same-type active entities (excluding self)
+		const { rows: candidates } = await db.execute(
+			`SELECT id, content FROM graph_nodes
+			 WHERE type = ?
+			   AND id != ?
+			   AND t_valid_until IS NULL
+			   AND archived_at IS NULL`,
+			[entityType, entityId],
+		);
+
+		for (const candidate of candidates) {
+			const candidateId = candidate.id as string;
+
+			// Canonical pair ordering
+			const [aId, bId] = entityId < candidateId ? [entityId, candidateId] : [candidateId, entityId];
+
+			// Skip if already checked
+			const existing = await db.execute(
+				"SELECT 1 FROM local_dedup_log WHERE entity_a_id = ? AND entity_b_id = ?",
+				[aId, bId],
+			);
+			if (existing.rows.length > 0) continue;
+
+			const similarity = wordJaccard(entityContent, candidate.content as string);
+
+			let decision: string;
+			if (similarity > 0.92) {
+				decision = "merged";
+			} else if (similarity > 0.5) {
+				decision = "related";
+			} else {
+				decision = "different";
+			}
+
+			await db.execute(
+				"INSERT INTO local_dedup_log (entity_a_id, entity_b_id, decision, checked_at) VALUES (?, ?, ?, ?)",
+				[aId, bId, decision, now],
+			);
+			pairsProcessed++;
+		}
+	}
+
+	return pairsProcessed;
+}

package/src/decay/types.ts

@@ -0,0 +1,17 @@
+// Module: decay/types — shared types for maintenance work units
+
+/** Result of processing a single batch within a work unit. */
+export interface BatchResult {
+	/** Number of items processed in this batch. */
+	processed: number;
+	/** Whether more work remains after this batch. */
+	remaining: boolean;
+}
+
+/** Result of a full importance decay run. */
+export interface DecayResult {
+	/** Number of entities processed. */
+	processed: number;
+	/** Wall-clock duration in milliseconds. */
+	durationMs: number;
+}