@rkarim08/sia 1.0.0

package/src/freshness/inverted-index.ts

@@ -0,0 +1,158 @@
+// Module: inverted-index — CRUD for the source_deps inverted dependency index
+//
+// Maps source files to the graph entities derived from them. When a file
+// changes, getDependentsForFile returns the exact set of entities that may
+// be stale and require revalidation.
+
+import type { SiaDb } from "@/graph/db-interface";
+
+/** A single source-file-to-entity dependency mapping. */
+export interface SourceDep {
+	source_path: string;
+	node_id: string;
+	dep_type: "defines" | "extracted_from" | "pertains_to" | "references";
+	source_mtime: number;
+}
+
+/**
+ * Add a source -> node dependency mapping.
+ * Uses INSERT OR REPLACE for idempotency — re-inserting the same
+ * (source_path, node_id) pair updates dep_type and source_mtime.
+ */
+export async function addDependency(db: SiaDb, dep: SourceDep): Promise<void> {
+	await db.execute(
+		`INSERT OR REPLACE INTO source_deps (source_path, node_id, dep_type, source_mtime)
+		 VALUES (?, ?, ?, ?)`,
+		[dep.source_path, dep.node_id, dep.dep_type, dep.source_mtime],
+	);
+}
+
+/** Remove a specific source -> node dependency. */
+export async function removeDependency(
+	db: SiaDb,
+	sourcePath: string,
+	nodeId: string,
+): Promise<void> {
+	await db.execute("DELETE FROM source_deps WHERE source_path = ? AND node_id = ?", [
+		sourcePath,
+		nodeId,
+	]);
+}
+
+/** Remove all dependencies for a node (used when a node is invalidated). */
+export async function removeDependenciesForNode(db: SiaDb, nodeId: string): Promise<void> {
+	await db.execute("DELETE FROM source_deps WHERE node_id = ?", [nodeId]);
+}
+
+/**
+ * Get all nodes derived from a source file.
+ * This is the core invalidation query — when file X changes, this returns
+ * every entity that may need revalidation.
+ */
+export async function getDependentsForFile(db: SiaDb, sourcePath: string): Promise<SourceDep[]> {
+	const { rows } = await db.execute(
+		"SELECT source_path, node_id, dep_type, source_mtime FROM source_deps WHERE source_path = ?",
+		[sourcePath],
+	);
+	return rows as unknown as SourceDep[];
+}
+
+/** Get all source files that an entity depends on. */
+export async function getDependenciesForNode(db: SiaDb, nodeId: string): Promise<SourceDep[]> {
+	const { rows } = await db.execute(
+		"SELECT source_path, node_id, dep_type, source_mtime FROM source_deps WHERE node_id = ?",
+		[nodeId],
+	);
+	return rows as unknown as SourceDep[];
+}
+
+/** Get all distinct source paths (used for Cuckoo filter rebuild). */
+export async function getAllSourcePaths(db: SiaDb): Promise<string[]> {
+	const { rows } = await db.execute("SELECT DISTINCT source_path FROM source_deps");
+	return rows.map((r) => r.source_path as string);
+}
+
+/**
+ * Rebuild the entire inverted index from existing graph data.
+ *
+ * - For each entity with a non-empty `file_paths` JSON array, creates
+ *   'defines' dependencies from each path to that entity.
+ * - For each active `pertains_to` edge whose target entity has file_paths,
+ *   creates 'pertains_to' dependencies from those target paths to the
+ *   source entity.
+ *
+ * Returns the total count of dependencies created.
+ */
+export async function rebuildFromGraph(db: SiaDb): Promise<number> {
+	// Clear existing deps
+	await db.execute("DELETE FROM source_deps", []);
+
+	let count = 0;
+	const now = Date.now();
+
+	// 1. Entities with file_paths -> 'defines' deps
+	const { rows: entities } = await db.execute(
+		`SELECT id, file_paths FROM graph_nodes
+		 WHERE archived_at IS NULL AND t_valid_until IS NULL
+		   AND file_paths IS NOT NULL AND file_paths != '[]'`,
+	);
+
+	for (const entity of entities) {
+		const id = entity.id as string;
+		const filePathsRaw = entity.file_paths as string;
+		let paths: string[];
+		try {
+			paths = JSON.parse(filePathsRaw);
+		} catch {
+			continue;
+		}
+		if (!Array.isArray(paths)) continue;
+
+		for (const p of paths) {
+			if (typeof p !== "string" || p.length === 0) continue;
+			await addDependency(db, {
+				source_path: p,
+				node_id: id,
+				dep_type: "defines",
+				source_mtime: now,
+			});
+			count++;
+		}
+	}
+
+	// 2. pertains_to edges: source entity depends on target's file_paths
+	const { rows: edges } = await db.execute(
+		`SELECT e.from_id, t.file_paths
+		 FROM graph_edges e
+		 JOIN graph_nodes t ON t.id = e.to_id
+		 WHERE e.type = 'pertains_to'
+		   AND e.t_valid_until IS NULL
+		   AND t.archived_at IS NULL AND t.t_valid_until IS NULL
+		   AND t.file_paths IS NOT NULL AND t.file_paths != '[]'`,
+	);
+
+	for (const edge of edges) {
+		const fromId = edge.from_id as string;
+		const filePathsRaw = edge.file_paths as string;
+		let paths: string[];
+		try {
+			paths = JSON.parse(filePathsRaw);
+		} catch {
+			continue;
+		}
+		if (!Array.isArray(paths)) continue;
+
+		for (const p of paths) {
+			if (typeof p !== "string" || p.length === 0) continue;
+			await addDependency(db, {
+				source_path: p,
+				node_id: fromId,
+				dep_type: "pertains_to",
+				source_mtime: now,
+			});
+			count++;
+		}
+	}
+
+	return count;
+}

package/src/freshness/stale-read-layer.ts

@@ -0,0 +1,222 @@
+// Module: stale-read-layer — Layer 3 per-query freshness validation
+//
+// Provides stale-while-revalidate semantics for node reads.
+// When a node is accessed during retrieval this layer determines whether
+// it is Fresh, Stale, or Rotten by comparing source file mtimes against
+// the node's t_created timestamp.
+//
+// Fresh:  source unchanged since extraction   → serve immediately (< 0.05ms)
+// Stale:  source modified within window       → serve + async re-validate
+// Rotten: source modified beyond window       → block until re-validated
+//
+// For nodes with multiple source files, only the most-recently-modified
+// source is stat()-checked — if it has not changed, none have (optimistic
+// fast path).
+
+import { createHash } from "node:crypto";
+import { existsSync, readFileSync, statSync } from "node:fs";
+import { join } from "node:path";
+import type { SiaDb } from "@/graph/db-interface";
+import type { DirtyTracker } from "./dirty-tracker";
+import { getDependenciesForNode } from "./inverted-index";
+
+export type FreshnessState = "fresh" | "stale" | "rotten";
+
+export interface FreshnessCheck {
+	state: FreshnessState;
+	sourcePath?: string; // the source file checked
+	sourceMtime?: number; // source file mtime (ms)
+	extractionTime?: number; // when the node was extracted (t_created ms)
+	divergenceSeconds?: number; // (mtime - t_created) / 1000
+}
+
+export interface StalenessConfig {
+	activeEditWindowMs: number; // default 30_000 (30s)
+	sessionCommitWindowMs: number; // default 300_000 (5min)
+	defaultWindowMs: number; // default Infinity (event-driven via Layers 1-2)
+}
+
+const DEFAULT_CONFIG: StalenessConfig = {
+	activeEditWindowMs: 30_000,
+	sessionCommitWindowMs: 300_000,
+	defaultWindowMs: Number.POSITIVE_INFINITY,
+};
+
+// ---------------------------------------------------------------------------
+// checkFreshness
+// ---------------------------------------------------------------------------
+
+/**
+ * Check the freshness of a node by comparing source file mtimes.
+ *
+ * Algorithm:
+ * 1. If tracker.checkNode(nodeId) is 'clean' → return Fresh (no stat() needed)
+ * 2. Get source files via getDependenciesForNode()
+ * 3. If no source files → return Fresh (no dependency to check)
+ * 4. stat() the most recently modified source file
+ * 5. Compare mtime against node's t_created timestamp
+ * 6. If mtime <= t_created → Fresh (source unchanged)
+ * 7. If mtime > t_created and within staleness window → Stale
+ * 8. If mtime > t_created and beyond staleness window → Rotten
+ */
+export async function checkFreshness(
+	db: SiaDb,
+	nodeId: string,
+	tracker: DirtyTracker,
+	repoRoot: string,
+	config?: Partial<StalenessConfig>,
+): Promise<FreshnessCheck> {
+	const cfg: StalenessConfig = { ...DEFAULT_CONFIG, ...config };
+
+	// Step 1: fast path — if tracker says clean, no stat() needed
+	const dirtyState = tracker.checkNode(nodeId);
+	if (dirtyState === "clean") {
+		return { state: "fresh" };
+	}
+
+	// Step 2: get source dependencies for this node
+	const deps = await getDependenciesForNode(db, nodeId);
+
+	// Step 3: no dependencies → treat as fresh (nothing to check)
+	if (deps.length === 0) {
+		return { state: "fresh" };
+	}
+
+	// Step 4: select the most recently modified source file recorded in source_deps.
+	// source_mtime is the recorded mtime at extraction time; the dep with the
+	// highest recorded mtime is most likely to have diverged.
+	const primaryDep = deps.reduce((best, d) => (d.source_mtime > best.source_mtime ? d : best));
+
+	const sourcePath = primaryDep.source_path;
+
+	// Step 4a: resolve path — if not absolute, join with repoRoot
+	const resolvedPath = sourcePath.startsWith("/") ? sourcePath : join(repoRoot, sourcePath);
+
+	// Handle missing file (deleted) → Rotten
+	if (!existsSync(resolvedPath)) {
+		return { state: "rotten", sourcePath };
+	}
+
+	let sourceMtime: number;
+	try {
+		sourceMtime = statSync(resolvedPath).mtimeMs;
+	} catch {
+		// stat error (permission denied, race condition, etc.) → Rotten
+		return { state: "rotten", sourcePath };
+	}
+
+	// Step 5: get node's t_created from the database
+	const { rows } = await db.execute("SELECT t_created FROM graph_nodes WHERE id = ?", [nodeId]);
+	if (rows.length === 0) {
+		// Node not found — treat as rotten
+		return { state: "rotten", sourcePath, sourceMtime };
+	}
+
+	const extractionTime = rows[0].t_created as number;
+
+	// Step 6: source unchanged → Fresh
+	if (sourceMtime <= extractionTime) {
+		return { state: "fresh", sourcePath, sourceMtime, extractionTime };
+	}
+
+	// Source was modified after extraction — compute divergence
+	const divergenceMs = sourceMtime - extractionTime;
+	const divergenceSeconds = divergenceMs / 1000;
+
+	// Step 7: within staleness window → Stale
+	if (divergenceMs <= cfg.activeEditWindowMs || divergenceMs <= cfg.sessionCommitWindowMs) {
+		return { state: "stale", sourcePath, sourceMtime, extractionTime, divergenceSeconds };
+	}
+
+	// Step 8: beyond staleness window → Rotten
+	return { state: "rotten", sourcePath, sourceMtime, extractionTime, divergenceSeconds };
+}
+
+// ---------------------------------------------------------------------------
+// readRepair
+// ---------------------------------------------------------------------------
+
+/**
+ * Perform read-repair: re-read source and compare content hash against the
+ * stored node content.
+ *
+ * Algorithm:
+ * 1. Get source files for the node
+ * 2. Re-read the primary source file content
+ * 3. Hash the content and compare against stored content hash
+ * 4. If content unchanged → tracker.markClean(nodeId) (early cutoff), return false
+ * 5. If changed → update node, tracker.markCleanAndPropagate(), return true
+ *
+ * Returns true if the node content changed (not an early cutoff).
+ */
+export async function readRepair(
+	db: SiaDb,
+	nodeId: string,
+	tracker: DirtyTracker,
+	repoRoot: string,
+): Promise<boolean> {
+	// Step 1: get source files for the node
+	const deps = await getDependenciesForNode(db, nodeId);
+
+	if (deps.length === 0) {
+		// No source to compare — mark clean (nothing to repair)
+		tracker.markClean(nodeId);
+		return false;
+	}
+
+	// Select the primary source (highest recorded mtime — most likely changed)
+	const primaryDep = deps.reduce((best, d) => (d.source_mtime > best.source_mtime ? d : best));
+
+	const sourcePath = primaryDep.source_path;
+	const resolvedPath = sourcePath.startsWith("/") ? sourcePath : join(repoRoot, sourcePath);
+
+	// Step 2: re-read the source file
+	let sourceContent: string;
+	try {
+		sourceContent = readFileSync(resolvedPath, "utf8");
+	} catch {
+		// File unreadable (deleted, permission error) — mark clean to avoid
+		// repeated repair attempts; caller should handle rotten state separately
+		tracker.markClean(nodeId);
+		return false;
+	}
+
+	// Step 3: hash the current source content
+	const sourceHash = sha256(sourceContent);
+
+	// Fetch the stored node content and hash it
+	const { rows } = await db.execute("SELECT content FROM graph_nodes WHERE id = ?", [nodeId]);
+
+	if (rows.length === 0) {
+		tracker.markClean(nodeId);
+		return false;
+	}
+
+	const storedContent = rows[0].content as string;
+	const storedHash = sha256(storedContent);
+
+	// Step 4: hashes match — content unchanged (early cutoff)
+	if (sourceHash === storedHash) {
+		tracker.markClean(nodeId);
+		return false;
+	}
+
+	// Step 5: content changed — update the node and propagate
+	const now = Date.now();
+	await db.execute("UPDATE graph_nodes SET content = ?, t_created = ? WHERE id = ?", [
+		sourceContent,
+		now,
+		nodeId,
+	]);
+
+	await tracker.markCleanAndPropagate(db, nodeId);
+	return true;
+}
+
+// ---------------------------------------------------------------------------
+// Internal helpers
+// ---------------------------------------------------------------------------
+
+function sha256(data: string): string {
+	return createHash("sha256").update(data).digest("hex");
+}

package/src/graph/audit.ts

@@ -0,0 +1,69 @@
+// Module: audit — Append-only audit log write layer
+
+import { randomUUID } from "node:crypto";
+import type { SiaDb } from "@/graph/db-interface";
+
+/** All supported audit operations (ARCHI v4.1 schema). */
+export type AuditOperation =
+	| "ADD"
+	| "UPDATE"
+	| "INVALIDATE"
+	| "NOOP"
+	| "STAGE"
+	| "PROMOTE"
+	| "QUARANTINE"
+	| "SYNC_RECV"
+	| "SYNC_SEND"
+	| "ARCHIVE"
+	| "VSS_REFRESH"
+	| "PAGERANK_UPDATE";
+
+/** Optional detail fields matching audit_log columns. */
+export interface AuditDetails {
+	entity_id?: string;
+	edge_id?: string;
+	source_episode?: string;
+	trust_tier?: number;
+	extraction_method?: string;
+	source_hash?: string;
+	developer_id?: string;
+	snapshot_id?: string;
+}
+
+/**
+ * Write a single append-only audit log entry.
+ *
+ * NEVER throws — any DB error is caught and logged to console.error.
+ * There is intentionally no update or delete export from this module.
+ */
+export async function writeAuditEntry(
+	db: SiaDb,
+	op: AuditOperation,
+	details: AuditDetails = {},
+): Promise<void> {
+	try {
+		await db.execute(
+			`INSERT INTO audit_log (
+				id, ts, operation,
+				entity_id, edge_id, source_episode,
+				trust_tier, extraction_method, source_hash,
+				developer_id, snapshot_id
+			) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+			[
+				randomUUID(),
+				Date.now(),
+				op,
+				details.entity_id ?? null,
+				details.edge_id ?? null,
+				details.source_episode ?? null,
+				details.trust_tier ?? null,
+				details.extraction_method ?? null,
+				details.source_hash ?? null,
+				details.developer_id ?? null,
+				details.snapshot_id ?? null,
+			],
+		);
+	} catch (err) {
+		console.error("writeAuditEntry failed:", err);
+	}
+}

package/src/graph/bridge-db.ts

@@ -0,0 +1,141 @@
+// Module: bridge-db — Bridge database opener and cross-repo edge CRUD
+
+import { randomUUID } from "node:crypto";
+import { join, resolve } from "node:path";
+import type { BunSqliteDb, SiaDb } from "@/graph/db-interface";
+import { runMigrations } from "@/graph/semantic-db";
+import { SIA_HOME } from "@/shared/config";
+
+/**
+ * Open (or create) the global bridge database.
+ * Resolves to `{siaHome}/bridge.db` (not under repos/) and applies
+ * migrations from the `migrations/bridge` directory.
+ */
+export function openBridgeDb(siaHome?: string): BunSqliteDb {
+	const home = siaHome ?? SIA_HOME;
+	const dbPath = join(home, "bridge.db");
+	const migrationsDir = resolve(import.meta.dirname, "../../migrations/bridge");
+	return runMigrations(dbPath, migrationsDir);
+}
+
+// ---------------------------------------------------------------------------
+// Cross-repo edge types
+// ---------------------------------------------------------------------------
+
+/** Shape accepted by insertCrossRepoEdge. Caller supplies endpoints and classification. */
+export interface NewCrossRepoEdge {
+	source_repo_id: string;
+	source_entity_id: string;
+	target_repo_id: string;
+	target_entity_id: string;
+	type: string;
+	weight?: number;
+	confidence?: number;
+	trust_tier?: number;
+	properties?: string | null;
+	t_valid_from?: number | null;
+	created_by?: string | null;
+}
+
+/** Row shape returned by cross-repo edge queries. */
+export interface CrossRepoEdgeRow {
+	id: string;
+	source_repo_id: string;
+	source_entity_id: string;
+	target_repo_id: string;
+	target_entity_id: string;
+	type: string;
+	weight: number;
+	confidence: number;
+	trust_tier: number;
+	properties: string | null;
+	t_created: number;
+	t_expired: number | null;
+	t_valid_from: number | null;
+	t_valid_until: number | null;
+	hlc_created: number | null;
+	hlc_modified: number | null;
+	created_by: string | null;
+}
+
+// ---------------------------------------------------------------------------
+// Cross-repo edge CRUD
+// ---------------------------------------------------------------------------
+
+/**
+ * Insert a new cross-repo edge into bridge.db.
+ *
+ * Generates a UUID for `id`, sets `t_created = Date.now()`.
+ * Returns the generated id.
+ */
+export async function insertCrossRepoEdge(db: SiaDb, edge: NewCrossRepoEdge): Promise<string> {
+	const id = randomUUID();
+	const now = Date.now();
+
+	await db.execute(
+		`INSERT INTO cross_repo_edges (
+			id, source_repo_id, source_entity_id,
+			target_repo_id, target_entity_id,
+			type, weight, confidence, trust_tier, properties,
+			t_created, t_expired, t_valid_from, t_valid_until,
+			created_by
+		) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, NULL, ?, NULL, ?)`,
+		[
+			id,
+			edge.source_repo_id,
+			edge.source_entity_id,
+			edge.target_repo_id,
+			edge.target_entity_id,
+			edge.type,
+			edge.weight ?? 1.0,
+			edge.confidence ?? 0.9,
+			edge.trust_tier ?? 2,
+			edge.properties ?? null,
+			now,
+			edge.t_valid_from ?? null,
+			edge.created_by ?? null,
+		],
+	);
+
+	return id;
+}
+
+/**
+ * Invalidate a cross-repo edge (soft-delete).
+ *
+ * Sets BOTH `t_valid_until` AND `t_expired` to the given timestamp (default: now).
+ * Never hard-deletes.
+ */
+export async function invalidateCrossRepoEdge(
+	db: SiaDb,
+	id: string,
+	tValidUntil?: number,
+): Promise<void> {
+	const ts = tValidUntil ?? Date.now();
+
+	await db.execute("UPDATE cross_repo_edges SET t_valid_until = ?, t_expired = ? WHERE id = ?", [
+		ts,
+		ts,
+		id,
+	]);
+}
+
+/**
+ * Get all currently active cross-repo edges for an entity (as source or target).
+ *
+ * Active = `t_valid_until IS NULL`.
+ */
+export async function getActiveCrossRepoEdgesFor(
+	db: SiaDb,
+	repoId: string,
+	entityId: string,
+): Promise<CrossRepoEdgeRow[]> {
+	const result = await db.execute(
+		`SELECT * FROM cross_repo_edges
+		 WHERE ((source_repo_id = ? AND source_entity_id = ?)
+		    OR  (target_repo_id = ? AND target_entity_id = ?))
+		   AND t_valid_until IS NULL`,
+		[repoId, entityId, repoId, entityId],
+	);
+	return result.rows as unknown as CrossRepoEdgeRow[];
+}