@rkarim08/sia 1.0.0

package/src/ontology/middleware.ts

@@ -0,0 +1,266 @@
+// Module: middleware — Typed factory methods enforcing co-creation and cardinality constraints
+
+import type { SiaDb } from "@/graph/db-interface";
+import { insertEdge } from "@/graph/edges";
+import type { Entity } from "@/graph/entities";
+import { insertEntity, invalidateEntity } from "@/graph/entities";
+import { validateEdge } from "@/ontology/constraints";
+import { OntologyError } from "@/ontology/errors";
+
+// ---------------------------------------------------------------------------
+// Shared helpers
+// ---------------------------------------------------------------------------
+
+/** Build a tags JSON string from an optional string array. */
+function tagsJson(tags?: string[]): string {
+	return JSON.stringify(tags ?? []);
+}
+
+// ---------------------------------------------------------------------------
+// createBug
+// ---------------------------------------------------------------------------
+
+export interface CreateBugOpts {
+	name: string;
+	content: string;
+	causedBy: string;
+	tags?: string[];
+	sessionId?: string;
+}
+
+/**
+ * Create a Bug entity together with a required `caused_by` edge.
+ *
+ * Throws OntologyError if `causedBy` is not provided.
+ */
+export async function createBug(db: SiaDb, opts: CreateBugOpts): Promise<Entity> {
+	if (!opts.causedBy) {
+		throw new OntologyError("createBug requires a causedBy target entity id");
+	}
+
+	let created!: Entity;
+	await db.transaction(async (tx) => {
+		created = await insertEntity(tx, {
+			type: "Bug",
+			name: opts.name,
+			content: opts.content,
+			summary: opts.content.slice(0, 120),
+			tags: tagsJson(opts.tags),
+			source_episode: opts.sessionId ?? null,
+		});
+
+		const valid = await validateEdge(tx, "Bug", "caused_by", "CodeEntity");
+		if (!valid) {
+			// Fall back: still allow if the constraint exists for any target type
+			const fallback = await validateEdge(tx, "Bug", "caused_by", "FileNode");
+			if (!fallback) {
+				throw new OntologyError("No edge_constraints entry for Bug→caused_by");
+			}
+		}
+
+		await insertEdge(tx, {
+			from_id: created.id,
+			to_id: opts.causedBy,
+			type: "caused_by",
+			source_episode: opts.sessionId ?? null,
+		});
+	});
+
+	return created;
+}
+
+// ---------------------------------------------------------------------------
+// createConvention
+// ---------------------------------------------------------------------------
+
+export interface CreateConventionOpts {
+	name: string;
+	content: string;
+	pertainsTo: string[];
+	tags?: string[];
+}
+
+/**
+ * Create a Convention entity together with one or more `pertains_to` edges.
+ *
+ * Throws OntologyError if `pertainsTo` is empty.
+ */
+export async function createConvention(db: SiaDb, opts: CreateConventionOpts): Promise<Entity> {
+	if (!opts.pertainsTo || opts.pertainsTo.length === 0) {
+		throw new OntologyError("createConvention requires at least one pertainsTo target");
+	}
+
+	let created!: Entity;
+	await db.transaction(async (tx) => {
+		created = await insertEntity(tx, {
+			type: "Convention",
+			name: opts.name,
+			content: opts.content,
+			summary: opts.content.slice(0, 120),
+			tags: tagsJson(opts.tags),
+		});
+
+		for (const targetId of opts.pertainsTo) {
+			await insertEdge(tx, {
+				from_id: created.id,
+				to_id: targetId,
+				type: "pertains_to",
+			});
+		}
+	});
+
+	return created;
+}
+
+// ---------------------------------------------------------------------------
+// createDecision
+// ---------------------------------------------------------------------------
+
+export interface CreateDecisionOpts {
+	name: string;
+	content: string;
+	pertainsTo?: string[];
+	supersedes?: string;
+	tags?: string[];
+}
+
+/**
+ * Create a Decision entity with optional `pertains_to` and `supersedes` edges.
+ *
+ * If `supersedes` is provided the old Decision is invalidated via
+ * `invalidateEntity`.
+ */
+export async function createDecision(db: SiaDb, opts: CreateDecisionOpts): Promise<Entity> {
+	let created!: Entity;
+	await db.transaction(async (tx) => {
+		created = await insertEntity(tx, {
+			type: "Decision",
+			name: opts.name,
+			content: opts.content,
+			summary: opts.content.slice(0, 120),
+			tags: tagsJson(opts.tags),
+		});
+
+		if (opts.pertainsTo) {
+			for (const targetId of opts.pertainsTo) {
+				await insertEdge(tx, {
+					from_id: created.id,
+					to_id: targetId,
+					type: "pertains_to",
+				});
+			}
+		}
+
+		if (opts.supersedes) {
+			await insertEdge(tx, {
+				from_id: created.id,
+				to_id: opts.supersedes,
+				type: "supersedes",
+			});
+			await invalidateEntity(tx, opts.supersedes);
+		}
+	});
+
+	return created;
+}
+
+// ---------------------------------------------------------------------------
+// createSolution
+// ---------------------------------------------------------------------------
+
+export interface CreateSolutionOpts {
+	name: string;
+	content: string;
+	solves: string;
+	pertainsTo?: string[];
+	tags?: string[];
+}
+
+/**
+ * Create a Solution entity together with a required `solves` edge and
+ * optional `pertains_to` edges.
+ *
+ * Throws OntologyError if `solves` is not provided.
+ */
+export async function createSolution(db: SiaDb, opts: CreateSolutionOpts): Promise<Entity> {
+	if (!opts.solves) {
+		throw new OntologyError("createSolution requires a solves target entity id");
+	}
+
+	let created!: Entity;
+	await db.transaction(async (tx) => {
+		created = await insertEntity(tx, {
+			type: "Solution",
+			name: opts.name,
+			content: opts.content,
+			summary: opts.content.slice(0, 120),
+			tags: tagsJson(opts.tags),
+		});
+
+		await insertEdge(tx, {
+			from_id: created.id,
+			to_id: opts.solves,
+			type: "solves",
+		});
+
+		if (opts.pertainsTo) {
+			for (const targetId of opts.pertainsTo) {
+				await insertEdge(tx, {
+					from_id: created.id,
+					to_id: targetId,
+					type: "pertains_to",
+				});
+			}
+		}
+	});
+
+	return created;
+}
+
+// ---------------------------------------------------------------------------
+// createConcept
+// ---------------------------------------------------------------------------
+
+export interface CreateConceptOpts {
+	name: string;
+	content: string;
+	pertainsTo?: string[];
+	elaborates?: string;
+	tags?: string[];
+}
+
+/**
+ * Create a Concept entity with optional `pertains_to` and `elaborates` edges.
+ */
+export async function createConcept(db: SiaDb, opts: CreateConceptOpts): Promise<Entity> {
+	let created!: Entity;
+	await db.transaction(async (tx) => {
+		created = await insertEntity(tx, {
+			type: "Concept",
+			name: opts.name,
+			content: opts.content,
+			summary: opts.content.slice(0, 120),
+			tags: tagsJson(opts.tags),
+		});
+
+		if (opts.pertainsTo) {
+			for (const targetId of opts.pertainsTo) {
+				await insertEdge(tx, {
+					from_id: created.id,
+					to_id: targetId,
+					type: "pertains_to",
+				});
+			}
+		}
+
+		if (opts.elaborates) {
+			await insertEdge(tx, {
+				from_id: created.id,
+				to_id: opts.elaborates,
+				type: "elaborates",
+			});
+		}
+	});
+
+	return created;
+}

package/src/retrieval/bm25-search.ts

@@ -0,0 +1,151 @@
+// Module: bm25-search — FTS5 MATCH keyword search with normalized ranking
+
+import type { SiaDb } from "@/graph/db-interface";
+
+/** A single BM25 search result. */
+export interface BM25Result {
+	entityId: string;
+	score: number;
+}
+
+/** Options for bm25Search. */
+export interface BM25SearchOpts {
+	/** Maximum number of results to return. Default 20. */
+	limit?: number;
+	/** When true, exclude entities with trust_tier = 4 (external/untrusted). */
+	paranoid?: boolean;
+	/** Filter results to a specific monorepo package path. */
+	packagePath?: string;
+}
+
+/**
+ * Sanitize a query string for FTS5 MATCH syntax.
+ *
+ * - Preserves double-quoted phrases intact (e.g. `"auth module"`)
+ * - Strips FTS5 special characters from unquoted parts
+ * - Splits unquoted parts on whitespace so each token is a separate term
+ * - Returns the sanitized query string ready for FTS5 MATCH
+ */
+export function sanitizeFts5Query(query: string): string {
+	const trimmed = query.trim();
+	if (!trimmed) return "";
+
+	const parts: string[] = [];
+	let remaining = trimmed;
+
+	// Extract quoted phrases and unquoted segments
+	while (remaining.length > 0) {
+		const quoteStart = remaining.indexOf('"');
+		if (quoteStart === -1) {
+			// No more quotes — process remaining as unquoted
+			const tokens = sanitizeUnquoted(remaining);
+			if (tokens) parts.push(tokens);
+			break;
+		}
+
+		// Process text before the quote
+		if (quoteStart > 0) {
+			const before = remaining.slice(0, quoteStart);
+			const tokens = sanitizeUnquoted(before);
+			if (tokens) parts.push(tokens);
+		}
+
+		// Find closing quote
+		const quoteEnd = remaining.indexOf('"', quoteStart + 1);
+		if (quoteEnd === -1) {
+			// Unclosed quote — treat rest as unquoted
+			const rest = remaining.slice(quoteStart + 1);
+			const tokens = sanitizeUnquoted(rest);
+			if (tokens) parts.push(tokens);
+			break;
+		}
+
+		// Extract the quoted phrase (including quotes)
+		const phrase = remaining.slice(quoteStart, quoteEnd + 1);
+		// Only add if the phrase has actual content between quotes
+		const inner = phrase.slice(1, -1).trim();
+		if (inner.length > 0) {
+			parts.push(`"${inner}"`);
+		}
+
+		remaining = remaining.slice(quoteEnd + 1);
+	}
+
+	return parts.join(" ");
+}
+
+/**
+ * Strip FTS5 special characters from an unquoted segment and return
+ * space-separated tokens. Returns empty string if no valid tokens remain.
+ */
+function sanitizeUnquoted(text: string): string {
+	// Remove FTS5 operators and special characters: * + - ^ ~ : ( ) { } < >
+	const cleaned = text.replace(/[*+\-^~:(){}|<>@!.,;'/\\[\]]/g, " ");
+	const tokens = cleaned.split(/\s+/).filter((t) => t.length > 0);
+	return tokens.join(" ");
+}
+
+/**
+ * Perform a BM25 keyword search against the graph_nodes_fts virtual table.
+ *
+ * Joins `graph_nodes_fts` with `entities` on rowid, filters for active entities
+ * (t_valid_until IS NULL AND archived_at IS NULL), applies optional paranoid
+ * and packagePath filters, and normalizes FTS5 rank to a 0–1 range.
+ *
+ * Returns an empty array for empty or invalid queries.
+ */
+export async function bm25Search(
+	db: SiaDb,
+	query: string,
+	opts?: BM25SearchOpts,
+): Promise<BM25Result[]> {
+	const sanitized = sanitizeFts5Query(query);
+	if (!sanitized) return [];
+
+	const limit = opts?.limit ?? 20;
+	const paranoid = opts?.paranoid ?? false;
+	const packagePath = opts?.packagePath;
+
+	// Build the query with optional filters
+	const conditions: string[] = ["e.t_valid_until IS NULL", "e.archived_at IS NULL"];
+	const params: unknown[] = [sanitized];
+
+	if (paranoid) {
+		conditions.push("e.trust_tier != 4");
+	}
+
+	if (packagePath) {
+		conditions.push("e.package_path = ?");
+		params.push(packagePath);
+	}
+
+	params.push(limit);
+
+	const whereClause = conditions.join(" AND ");
+
+	const sql = `
+		SELECT e.id AS entityId, -fts.rank AS rawRank
+		FROM graph_nodes_fts fts
+		JOIN graph_nodes e ON e.rowid = fts.rowid
+		WHERE graph_nodes_fts MATCH ?
+		  AND ${whereClause}
+		ORDER BY fts.rank
+		LIMIT ?
+	`;
+
+	const result = await db.execute(sql, params);
+	const rows = result.rows as Array<{ entityId: string; rawRank: number }>;
+
+	if (rows.length === 0) return [];
+
+	// Normalize rawRank to 0–1 range using min/max of the result set
+	const rawRanks = rows.map((r) => r.rawRank);
+	const minRank = Math.min(...rawRanks);
+	const maxRank = Math.max(...rawRanks);
+	const range = maxRank - minRank;
+
+	return rows.map((row) => ({
+		entityId: row.entityId,
+		score: range === 0 ? 1.0 : (row.rawRank - minRank) / range,
+	}));
+}

package/src/retrieval/context-assembly.ts

@@ -0,0 +1,76 @@
+// src/retrieval/context-assembly.ts
+
+import type { SiaSearchResult } from "@/graph/types";
+
+export interface AssemblyOptions {
+	includeProvenance?: boolean;
+}
+
+/**
+ * Map a raw entity DB row to a typed SiaSearchResult.
+ * - Parses JSON `tags` and `file_paths` (falls back to [] on parse failure)
+ * - Includes `extraction_method` only when opts.includeProvenance is true
+ * - Always includes `conflict_group_id` and `t_valid_from`
+ */
+export function assembleSearchResult(
+	row: Record<string, unknown>,
+	opts?: AssemblyOptions,
+): SiaSearchResult {
+	const result: SiaSearchResult = {
+		entity_id: row.id as string,
+		type: row.type as string,
+		name: row.name as string,
+		summary: row.summary as string,
+		content: row.content as string,
+		tags: safeParseJsonArray(row.tags as string),
+		file_paths: safeParseJsonArray(row.file_paths as string),
+		trust_tier: row.trust_tier as SiaSearchResult["trust_tier"],
+		confidence: row.confidence as number,
+		importance: row.importance as number,
+		conflict_group_id: (row.conflict_group_id as string) ?? null,
+		t_valid_from: (row.t_valid_from as number) ?? null,
+		t_valid_until: (row.t_valid_until as number) ?? null,
+	};
+
+	if (opts?.includeProvenance && row.extraction_method) {
+		result.extraction_method = row.extraction_method as string;
+	}
+
+	return result;
+}
+
+/**
+ * Enforce maxResponseTokens by estimating ~150 tokens per result.
+ * Returns whole results or nothing (no partial results).
+ */
+export function enforceResponseBudget(
+	results: SiaSearchResult[],
+	maxTokens: number,
+): { results: SiaSearchResult[]; truncated: boolean } {
+	const TOKENS_PER_RESULT = 150;
+
+	if (maxTokens <= 0) {
+		return { results: [], truncated: results.length > 0 };
+	}
+
+	const maxResults = Math.floor(maxTokens / TOKENS_PER_RESULT);
+
+	if (results.length <= maxResults) {
+		return { results, truncated: false };
+	}
+
+	return {
+		results: results.slice(0, maxResults),
+		truncated: true,
+	};
+}
+
+function safeParseJsonArray(value: string): string[] {
+	if (!value) return [];
+	try {
+		const parsed = JSON.parse(value);
+		return Array.isArray(parsed) ? parsed : [];
+	} catch {
+		return [];
+	}
+}

package/src/retrieval/graph-traversal.ts

@@ -0,0 +1,168 @@
+// Module: graph-traversal — Entity name extraction + 1-hop graph expansion search signal
+
+import type { SiaDb } from "@/graph/db-interface";
+
+/** A single result from the graph traversal search signal. */
+export interface GraphTraversalResult {
+	entityId: string;
+	score: number;
+}
+
+/** Options for graphTraversalSearch. */
+export interface GraphTraversalSearchOpts {
+	/** Maximum number of results to return. Default 20. */
+	limit?: number;
+	/** When true, exclude trust_tier >= 4 entities. */
+	paranoid?: boolean;
+}
+
+/**
+ * Extract candidate entity names from a natural-language query.
+ *
+ * Steps:
+ * 1. Split on whitespace, keep tokens >= 2 chars.
+ * 2. Try CamelCase splits on each token (e.g. "TokenStore" -> ["Token", "Store"]).
+ * 3. Generate two-word combinations from adjacent tokens:
+ *    - CamelCase joined ("token store" -> "TokenStore")
+ *    - snake_case joined ("token store" -> "token_store")
+ */
+export function extractQueryTerms(query: string): string[] {
+	const raw = query.split(/\s+/).filter((t) => t.length >= 2);
+	const terms = new Set<string>();
+
+	for (const token of raw) {
+		terms.add(token);
+
+		// CamelCase split: "TokenStore" -> ["Token", "Store"]
+		const camelParts = token.match(/[A-Z][a-z]+|[a-z]+|[A-Z]+(?=[A-Z][a-z]|\b)/g);
+		if (camelParts && camelParts.length > 1) {
+			for (const part of camelParts) {
+				if (part.length >= 2) {
+					terms.add(part);
+				}
+			}
+		}
+	}
+
+	// Two-word combinations from adjacent tokens
+	for (let i = 0; i < raw.length - 1; i++) {
+		const a = raw[i];
+		const b = raw[i + 1];
+		// CamelCase: capitalize first letter of each word
+		const camel =
+			a.charAt(0).toUpperCase() +
+			a.slice(1).toLowerCase() +
+			b.charAt(0).toUpperCase() +
+			b.slice(1).toLowerCase();
+		terms.add(camel);
+		// snake_case
+		terms.add(`${a.toLowerCase()}_${b.toLowerCase()}`);
+	}
+
+	return [...terms];
+}
+
+/**
+ * Graph traversal search signal.
+ *
+ * 1. Extract candidate terms from the query.
+ * 2. Direct lookup: exact name match on active entities (score 1.0).
+ * 3. LIKE partial match for terms >= 3 chars (score 0.9, limit 5 per term).
+ * 4. 1-hop expansion via active edges for each matched entity (score 0.7).
+ * 5. Deduplicate (highest score wins), sort by score DESC, cap at limit.
+ */
+export async function graphTraversalSearch(
+	db: SiaDb,
+	query: string,
+	opts?: GraphTraversalSearchOpts,
+): Promise<GraphTraversalResult[]> {
+	const limit = opts?.limit ?? 20;
+	const paranoid = opts?.paranoid ?? false;
+	const terms = extractQueryTerms(query);
+
+	if (terms.length === 0) {
+		return [];
+	}
+
+	// Map<entityId, score> — highest score wins
+	const scoreMap = new Map<string, number>();
+
+	function addScore(entityId: string, score: number): void {
+		const existing = scoreMap.get(entityId);
+		if (existing === undefined || score > existing) {
+			scoreMap.set(entityId, score);
+		}
+	}
+
+	// Collect direct-match entity IDs for 1-hop expansion
+	const directMatchIds: string[] = [];
+
+	// Stage 1: Direct exact lookup (score 1.0)
+	for (const term of terms) {
+		const paranoidClause = paranoid ? " AND trust_tier < 4" : "";
+		const result = await db.execute(
+			`SELECT id FROM graph_nodes WHERE name = ? AND t_valid_until IS NULL AND archived_at IS NULL${paranoidClause}`,
+			[term],
+		);
+		for (const row of result.rows) {
+			const id = row.id as string;
+			addScore(id, 1.0);
+			directMatchIds.push(id);
+		}
+	}
+
+	// Stage 2: LIKE partial match (score 0.9) for terms >= 3 chars
+	for (const term of terms) {
+		if (term.length < 3) continue;
+		const paranoidClause = paranoid ? " AND trust_tier < 4" : "";
+		const result = await db.execute(
+			`SELECT id FROM graph_nodes WHERE name LIKE ? AND t_valid_until IS NULL AND archived_at IS NULL${paranoidClause} LIMIT 5`,
+			[`%${term}%`],
+		);
+		for (const row of result.rows) {
+			const id = row.id as string;
+			addScore(id, 0.9);
+			if (!directMatchIds.includes(id)) {
+				directMatchIds.push(id);
+			}
+		}
+	}
+
+	// Stage 3: 1-hop expansion via edges (score 0.7)
+	for (const entityId of directMatchIds) {
+		const edgeResult = await db.execute(
+			"SELECT from_id, to_id FROM graph_edges WHERE (from_id = ? OR to_id = ?) AND t_valid_until IS NULL",
+			[entityId, entityId],
+		);
+		for (const row of edgeResult.rows) {
+			const fromId = row.from_id as string;
+			const toId = row.to_id as string;
+			const neighborId = fromId === entityId ? toId : fromId;
+
+			// Only add neighbor if it is an active, non-archived entity
+			if (paranoid) {
+				const check = await db.execute(
+					"SELECT id FROM graph_nodes WHERE id = ? AND t_valid_until IS NULL AND archived_at IS NULL AND trust_tier < 4",
+					[neighborId],
+				);
+				if (check.rows.length === 0) continue;
+			} else {
+				const check = await db.execute(
+					"SELECT id FROM graph_nodes WHERE id = ? AND t_valid_until IS NULL AND archived_at IS NULL",
+					[neighborId],
+				);
+				if (check.rows.length === 0) continue;
+			}
+
+			addScore(neighborId, 0.7);
+		}
+	}
+
+	// Build sorted result list
+	const results: GraphTraversalResult[] = [...scoreMap.entries()]
+		.map(([entityId, score]) => ({ entityId, score }))
+		.sort((a, b) => b.score - a.score)
+		.slice(0, limit);
+
+	return results;
+}

package/src/retrieval/pagerank.ts

@@ -0,0 +1,40 @@
+// Module: pagerank — Retrieval-layer PageRank wrapper
+
+import { writeAuditEntry } from "@/graph/audit";
+import type { SiaDb } from "@/graph/db-interface";
+
+export const EDGE_TYPE_WEIGHTS: Record<string, number> = {
+	calls: 0.5,
+	pertains_to: 0.4,
+	solves: 0.4,
+	relates_to: 0.3,
+	imports: 0.3,
+	caused_by: 0.3,
+	elaborates: 0.2,
+	supersedes: 0.2,
+	member_of: 0.1,
+};
+
+export async function getImportanceScore(db: SiaDb, nodeId: string): Promise<number> {
+	const { rows } = await db.execute("SELECT importance FROM graph_nodes WHERE id = ?", [nodeId]);
+	if (rows.length === 0) return 0.5;
+	return (rows[0] as { importance: number }).importance;
+}
+
+export async function updateImportanceScores(
+	db: SiaDb,
+	scores: Map<string, number>,
+): Promise<number> {
+	if (scores.size === 0) return 0;
+	const statements = [...scores.entries()].map(([id, score]) => ({
+		sql: "UPDATE graph_nodes SET importance = ? WHERE id = ?",
+		params: [score, id] as unknown[],
+	}));
+	await db.executeMany(statements);
+	await writeAuditEntry(db, "PAGERANK_UPDATE", {});
+	return scores.size;
+}
+
+export function getEdgeWeight(edgeType: string): number {
+	return EDGE_TYPE_WEIGHTS[edgeType] ?? 0.1;
+}