@rkarim08/sia 1.0.0

package/src/mcp/tools/sia-fetch-and-index.ts

@@ -0,0 +1,241 @@
+// Module: sia-fetch-and-index — Fetch a URL, convert to markdown, and index via contentTypeChunker
+
+import { randomUUID } from "node:crypto";
+import * as dns from "node:dns/promises";
+import TurndownService from "turndown";
+import { z } from "zod";
+import type { Embedder } from "@/capture/embedder";
+import type { SiaDb } from "@/graph/db-interface";
+import { contentTypeChunker } from "@/sandbox/context-mode";
+
+// ---------------------------------------------------------------------------
+// Input / Output types
+// ---------------------------------------------------------------------------
+
+export const SiaFetchAndIndexInput = z.object({
+	url: z.string(),
+	intent: z.string().optional(),
+	tags: z.array(z.string()).optional(),
+});
+
+export interface SiaFetchAndIndexResult {
+	indexed?: number;
+	contentType?: string;
+	sourceUrl?: string;
+	externalRefId?: string;
+	error?: string;
+}
+
+// ---------------------------------------------------------------------------
+// SSRF Protection
+// ---------------------------------------------------------------------------
+
+/**
+ * Returns true if the given IP address is a private/loopback/link-local address.
+ * Checks IPv4 ranges: 127.0.0.0/8, 10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16,
+ * 169.254.0.0/16, and IPv6 loopback ::1.
+ */
+export function isPrivateIp(ip: string): boolean {
+	// IPv4-mapped IPv6 (e.g. ::ffff:127.0.0.1)
+	if (ip.startsWith("::ffff:")) {
+		return isPrivateIp(ip.slice(7));
+	}
+
+	// IPv6 loopback
+	if (ip === "::1" || ip === "0:0:0:0:0:0:0:1") {
+		return true;
+	}
+
+	// Parse IPv4 octets
+	const parts = ip.split(".");
+	if (parts.length !== 4) {
+		// Non-IPv4 that isn't ::1 — treat as potentially private to be safe
+		return true;
+	}
+
+	const [a, b, _c] = parts.map(Number);
+
+	// 127.0.0.0/8 — loopback
+	if (a === 127) return true;
+	// 10.0.0.0/8 — private
+	if (a === 10) return true;
+	// 172.16.0.0/12 — private (172.16.x.x through 172.31.x.x)
+	if (a === 172 && b >= 16 && b <= 31) return true;
+	// 192.168.0.0/16 — private
+	if (a === 192 && b === 168) return true;
+	// 169.254.0.0/16 — link-local
+	if (a === 169 && b === 254) return true;
+	// 0.0.0.0/8 — this network
+	if (a === 0) return true;
+
+	return false;
+}
+
+// ---------------------------------------------------------------------------
+// handleSiaFetchAndIndex
+// ---------------------------------------------------------------------------
+
+export async function handleSiaFetchAndIndex(
+	db: SiaDb,
+	input: z.infer<typeof SiaFetchAndIndexInput>,
+	_embedder: Embedder,
+	sessionId: string,
+): Promise<SiaFetchAndIndexResult> {
+	const { url, tags } = input;
+
+	// 1. Parse URL — error if invalid
+	let parsed: URL;
+	try {
+		parsed = new URL(url);
+	} catch {
+		return { error: `Invalid URL: ${url}` };
+	}
+
+	// 2. Block non-HTTP(S) schemes
+	if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
+		return {
+			error: `Only HTTP and HTTPS URLs are supported. Got: ${parsed.protocol}`,
+		};
+	}
+
+	// 3. Resolve hostname via DNS → block private IPs
+	const hostname = parsed.hostname;
+	let resolvedIps: string[] = [];
+	try {
+		resolvedIps = await dns.resolve(hostname);
+	} catch {
+		// If DNS resolution fails entirely, attempt lookup as fallback
+		try {
+			const lookupResult = await dns.lookup(hostname);
+			resolvedIps = [lookupResult.address];
+		} catch (lookupErr) {
+			return {
+				error: `DNS resolution failed for host: ${hostname} (${(lookupErr as Error).message})`,
+			};
+		}
+	}
+
+	for (const ip of resolvedIps) {
+		if (isPrivateIp(ip)) {
+			return {
+				error: `Blocked: ${hostname} resolves to private IP ${ip}`,
+			};
+		}
+	}
+
+	// 4. Fetch with timeout (30s), User-Agent header
+	// Use the resolved IP directly to prevent DNS rebinding (TOCTOU attack):
+	// the hostname is sent via the Host header so the server responds correctly.
+	const MAX_CONTENT_LENGTH = 5 * 1024 * 1024; // 5 MB cap
+	const controller = new AbortController();
+	const timeout = setTimeout(() => controller.abort(), 30_000);
+
+	const resolvedUrl = new URL(url);
+	resolvedUrl.hostname = resolvedIps[0];
+
+	let response: Response;
+	try {
+		response = await fetch(resolvedUrl.toString(), {
+			signal: controller.signal,
+			headers: {
+				Host: hostname,
+				"User-Agent": "Sia/1.0 (knowledge graph indexer)",
+				Accept: "text/html,text/markdown,application/json,text/plain,*/*",
+			},
+		});
+	} catch (err) {
+		clearTimeout(timeout);
+		return { error: `Fetch failed: ${err instanceof Error ? err.message : String(err)}` };
+	} finally {
+		clearTimeout(timeout);
+	}
+
+	if (!response.ok) {
+		return { error: `HTTP ${response.status}: ${response.statusText}` };
+	}
+
+	// 5. Content-type detection & size cap
+	const rawContentType = response.headers.get("content-type") ?? "text/plain";
+	const contentType = rawContentType.split(";")[0].trim().toLowerCase();
+
+	// Check content-length header first
+	const contentLengthHeader = response.headers.get("content-length");
+	if (contentLengthHeader && Number(contentLengthHeader) > MAX_CONTENT_LENGTH) {
+		return {
+			error: `Response too large: ${contentLengthHeader} bytes (max ${MAX_CONTENT_LENGTH})`,
+		};
+	}
+
+	let rawBody: string;
+	try {
+		const buffer = await response.arrayBuffer();
+		if (buffer.byteLength > MAX_CONTENT_LENGTH) {
+			return {
+				error: `Response too large: ${buffer.byteLength} bytes (max ${MAX_CONTENT_LENGTH})`,
+			};
+		}
+		rawBody = new TextDecoder().decode(buffer);
+	} catch (err) {
+		return {
+			error: `Failed to read response body: ${err instanceof Error ? err.message : String(err)}`,
+		};
+	}
+
+	// 6. HTML → markdown via turndown
+	let processedContent: string;
+	if (contentType === "text/html") {
+		const td = new TurndownService({ headingStyle: "atx", codeBlockStyle: "fenced" });
+		processedContent = td.turndown(rawBody);
+	} else {
+		processedContent = rawBody;
+	}
+
+	// 7. Chunk content with contentTypeChunker and insert with trust_tier 4
+	const tagsJson = JSON.stringify(tags ?? []);
+	const now = Date.now();
+	const nowStr = String(now);
+
+	const rawChunks = contentTypeChunker.chunk(processedContent);
+	const chunkIds: string[] = [];
+
+	for (let i = 0; i < rawChunks.length; i++) {
+		const raw = rawChunks[i];
+		const nodeId = randomUUID();
+		const chunkName = `fetch-chunk-${sessionId}-${i}`;
+		const summary = raw.text.slice(0, 100);
+
+		await db.execute(
+			`INSERT INTO graph_nodes (id, type, name, summary, content, trust_tier, confidence, base_confidence, importance, base_importance, access_count, edge_count, tags, file_paths, t_created, t_valid_from, created_by, created_at, last_accessed)
+			 VALUES (?, 'ContentChunk', ?, ?, ?, 4, 0.7, 0.7, 0.4, 0.4, 0, 0, ?, '[]', ?, ?, 'sia-fetch-and-index', ?, ?)`,
+			[nodeId, chunkName, summary, raw.text, tagsJson, nowStr, nowStr, nowStr, nowStr],
+		);
+
+		chunkIds.push(nodeId);
+	}
+
+	// 8. Create ExternalRef node in graph_nodes
+	const externalRefId = randomUUID();
+	await db.execute(
+		`INSERT INTO graph_nodes (id, type, name, summary, content, trust_tier, confidence, base_confidence, importance, base_importance, access_count, edge_count, tags, file_paths, t_created, t_valid_from, created_by, created_at, last_accessed)
+		 VALUES (?, 'ExternalRef', ?, ?, ?, 4, 0.7, 0.7, 0.4, 0.4, 0, 0, ?, '[]', ?, ?, 'sia-fetch-and-index', ?, ?)`,
+		[
+			externalRefId,
+			url,
+			`External reference: ${url}`,
+			url,
+			tagsJson,
+			nowStr,
+			nowStr,
+			nowStr,
+			nowStr,
+		],
+	);
+
+	// 9. Return result
+	return {
+		indexed: chunkIds.length,
+		contentType,
+		sourceUrl: url,
+		externalRefId,
+	};
+}

package/src/mcp/tools/sia-flag.ts

@@ -0,0 +1,65 @@
+// Module: sia-flag — Flag the current session for human review
+
+import { v4 as uuid } from "uuid";
+import type { z } from "zod";
+import type { SiaDb } from "@/graph/db-interface";
+import type { SiaFlagInput as SiaFlagInputSchema } from "@/mcp/server";
+
+export type SiaFlagInput = z.infer<typeof SiaFlagInputSchema>;
+
+export interface SiaFlagConfig {
+	enableFlagging: boolean;
+	sessionId: string;
+}
+
+export interface SiaFlagResult {
+	flagged?: boolean;
+	id?: string;
+	error?: string;
+}
+
+/**
+ * Sanitize a flag reason string.
+ *
+ * Strips: < > { } [ ] \ " and control characters (0x00-0x1F, 0x7F).
+ * Keeps: `: backticks _ / # @ ( ) . , ' -` and all normal printable chars.
+ * Truncates to 100 characters after sanitization.
+ */
+export function sanitizeReason(raw: string): string {
+	// Remove < > { } [ ] \ " and control chars (0x00-0x1F, 0x7F)
+	// biome-ignore lint/suspicious/noControlCharactersInRegex: intentionally stripping control characters for sanitization
+	const cleaned = raw.replace(/[<>{}[\]\\"]/g, "").replace(/[\x00-\x1f\x7f]/g, "");
+	return cleaned.slice(0, 100);
+}
+
+/**
+ * Handle a sia_flag request: insert a flag into session_flags for human review.
+ */
+export async function handleSiaFlag(
+	db: SiaDb,
+	input: SiaFlagInput,
+	config: SiaFlagConfig,
+): Promise<SiaFlagResult> {
+	if (!config.enableFlagging) {
+		return {
+			error: "Flagging is disabled. Run 'npx sia enable-flagging' to enable.",
+		};
+	}
+
+	const reason = sanitizeReason(input.reason);
+
+	if (reason.length === 0) {
+		return { error: "Flag reason is empty after sanitization" };
+	}
+
+	const id = uuid();
+	const createdAt = Date.now();
+
+	await db.execute(
+		`INSERT INTO session_flags (id, session_id, reason, created_at, consumed)
+		 VALUES (?, ?, ?, ?, 0)`,
+		[id, config.sessionId, reason, createdAt],
+	);
+
+	return { flagged: true, id };
+}

package/src/mcp/tools/sia-index.ts

@@ -0,0 +1,111 @@
+// Module: sia-index — Index markdown/text content by chunking and scanning for entity references
+
+import { randomUUID } from "node:crypto";
+import { z } from "zod";
+import type { Embedder } from "@/capture/embedder";
+import type { SiaDb } from "@/graph/db-interface";
+import { insertEdge } from "@/graph/edges";
+import { headingChunker } from "@/sandbox/context-mode";
+
+// ---------------------------------------------------------------------------
+// Input / Output types
+// ---------------------------------------------------------------------------
+
+export const SiaIndexInput = z.object({
+	content: z.string(),
+	source: z.string().optional(),
+	tags: z.array(z.string()).optional(),
+});
+
+export interface SiaIndexResult {
+	indexed: number;
+	references: number;
+	chunkIds: string[];
+}
+
+// ---------------------------------------------------------------------------
+// handleSiaIndex
+// ---------------------------------------------------------------------------
+
+export async function handleSiaIndex(
+	db: SiaDb,
+	input: z.infer<typeof SiaIndexInput>,
+	_embedder: Embedder,
+	sessionId: string,
+): Promise<SiaIndexResult> {
+	const { content, source, tags } = input;
+
+	// 1. Empty content fast-path
+	if (!content || content.trim().length === 0) {
+		return { indexed: 0, references: 0, chunkIds: [] };
+	}
+
+	const now = Date.now();
+	const nowStr = String(now);
+	const tagsJson = JSON.stringify(tags ?? []);
+
+	// 2. Chunk content via headingChunker
+	const rawChunks = headingChunker.chunk(content);
+
+	// 3. Embed each chunk and store as ContentChunk node in graph_nodes
+	const chunkIds: string[] = [];
+
+	for (let i = 0; i < rawChunks.length; i++) {
+		const raw = rawChunks[i];
+		const nodeId = randomUUID();
+		const chunkName = source ? `chunk-${source}-${i}` : `chunk-${sessionId}-${i}`;
+		const summary = raw.text.slice(0, 100);
+
+		await db.execute(
+			`INSERT INTO graph_nodes (id, type, name, summary, content, trust_tier, confidence, base_confidence, importance, base_importance, access_count, edge_count, tags, file_paths, t_created, t_valid_from, created_by, created_at, last_accessed)
+			 VALUES (?, 'ContentChunk', ?, ?, ?, 3, 0.8, 0.8, 0.5, 0.5, 0, 0, ?, '[]', ?, ?, 'sia-index', ?, ?)`,
+			[nodeId, chunkName, summary, raw.text, tagsJson, nowStr, nowStr, nowStr, nowStr],
+		);
+
+		chunkIds.push(nodeId);
+	}
+
+	// 4. Scan each chunk for mentions of known entity names
+	//    Query graph_nodes for CodeSymbol and FileNode types
+	const { rows: knownEntities } = await db.execute(
+		`SELECT id, name FROM graph_nodes WHERE type IN ('CodeSymbol', 'FileNode') AND (t_expired IS NULL OR t_expired = '') AND (t_valid_until IS NULL OR t_valid_until = '')`,
+		[],
+	);
+
+	let referenceCount = 0;
+
+	if (knownEntities.length > 0) {
+		for (let ci = 0; ci < rawChunks.length; ci++) {
+			const chunkText = rawChunks[ci].text;
+			const chunkNodeId = chunkIds[ci];
+
+			for (const row of knownEntities) {
+				const entityId = row.id as string;
+				const entityName = row.name as string;
+				if (entityName && chunkText.includes(entityName)) {
+					try {
+						await insertEdge(db, {
+							from_id: chunkNodeId,
+							to_id: entityId,
+							type: "references",
+							weight: 1.0,
+							confidence: 0.7,
+							trust_tier: 3,
+						});
+						referenceCount++;
+					} catch (edgeErr) {
+						console.error(
+							`[sia-index] edge insert failed for ${chunkNodeId}->${entityId}: ${(edgeErr as Error).message}`,
+						);
+					}
+				}
+			}
+		}
+	}
+
+	return {
+		indexed: chunkIds.length,
+		references: referenceCount,
+		chunkIds,
+	};
+}

package/src/mcp/tools/sia-note.ts

@@ -0,0 +1,134 @@
+// Module: sia-note — Developer-authored knowledge entry via MCP
+
+import type { SiaDb } from "@/graph/db-interface";
+import type { Entity } from "@/graph/entities";
+import { OntologyError } from "@/ontology/errors";
+import {
+	createBug,
+	createConcept,
+	createConvention,
+	createDecision,
+	createSolution,
+} from "@/ontology/middleware";
+
+export interface SiaNoteInput {
+	kind: "Decision" | "Convention" | "Bug" | "Solution" | "Concept";
+	name: string;
+	content: string;
+	tags?: string[];
+	relates_to?: string[]; // entity IDs → pertains_to/caused_by edges
+	supersedes?: string; // entity ID this replaces
+}
+
+export interface SiaNoteResult {
+	node_id: string;
+	kind: string;
+	edges_created: number;
+}
+
+/**
+ * Create a developer-authored knowledge entry in the graph.
+ * Routes to the appropriate ontology middleware function based on kind.
+ *
+ * For Bug: first relates_to entry becomes the causedBy target.
+ * For Convention: all relates_to become pertainsTo targets (at least 1 required).
+ * For Decision: relates_to become pertainsTo, supersedes if provided.
+ * For Solution: first relates_to entry becomes the solves target, rest become pertainsTo.
+ * For Concept: relates_to become pertainsTo.
+ */
+export async function handleSiaNote(db: SiaDb, input: SiaNoteInput): Promise<SiaNoteResult> {
+	const relatesTo = input.relates_to ?? [];
+
+	try {
+		let entity: Entity;
+		let edgesCreated: number;
+
+		switch (input.kind) {
+			case "Bug": {
+				if (relatesTo.length === 0) {
+					throw new OntologyError(
+						"Bug requires at least one relates_to entry as the causedBy target",
+					);
+				}
+				entity = await createBug(db, {
+					name: input.name,
+					content: input.content,
+					causedBy: relatesTo[0],
+					tags: input.tags,
+				});
+				// Bug creates exactly 1 caused_by edge
+				edgesCreated = 1;
+				break;
+			}
+
+			case "Convention": {
+				if (relatesTo.length === 0) {
+					throw new OntologyError(
+						"Convention requires at least one relates_to entry as a pertainsTo target",
+					);
+				}
+				entity = await createConvention(db, {
+					name: input.name,
+					content: input.content,
+					pertainsTo: relatesTo,
+					tags: input.tags,
+				});
+				edgesCreated = relatesTo.length;
+				break;
+			}
+
+			case "Decision": {
+				entity = await createDecision(db, {
+					name: input.name,
+					content: input.content,
+					pertainsTo: relatesTo.length > 0 ? relatesTo : undefined,
+					supersedes: input.supersedes,
+					tags: input.tags,
+				});
+				edgesCreated = relatesTo.length + (input.supersedes ? 1 : 0);
+				break;
+			}
+
+			case "Solution": {
+				if (relatesTo.length === 0) {
+					throw new OntologyError(
+						"Solution requires at least one relates_to entry as the solves target",
+					);
+				}
+				const pertainsTo = relatesTo.length > 1 ? relatesTo.slice(1) : undefined;
+				entity = await createSolution(db, {
+					name: input.name,
+					content: input.content,
+					solves: relatesTo[0],
+					pertainsTo,
+					tags: input.tags,
+				});
+				// 1 solves edge + any remaining pertains_to edges
+				edgesCreated = 1 + (pertainsTo?.length ?? 0);
+				break;
+			}
+
+			case "Concept": {
+				entity = await createConcept(db, {
+					name: input.name,
+					content: input.content,
+					pertainsTo: relatesTo.length > 0 ? relatesTo : undefined,
+					tags: input.tags,
+				});
+				edgesCreated = relatesTo.length;
+				break;
+			}
+		}
+
+		return {
+			node_id: entity.id,
+			kind: input.kind,
+			edges_created: edgesCreated,
+		};
+	} catch (err) {
+		if (err instanceof OntologyError) {
+			throw new Error(`sia_note failed: ${err.message}`);
+		}
+		throw err;
+	}
+}

package/src/mcp/tools/sia-search.ts

@@ -0,0 +1,105 @@
+// Module: sia-search — Three-stage hybrid retrieval via BM25 + graph + vector
+//
+// Workspace routing is preserved from Phase 5.
+// Local search delegates to the three-stage pipeline in @/retrieval/search.
+
+import type { z } from "zod";
+import type { Embedder } from "@/capture/embedder";
+import type { SiaDb } from "@/graph/db-interface";
+import { annotateFreshness } from "@/mcp/freshness-annotator";
+import type { SiaSearchInput } from "@/mcp/server";
+import { hybridSearch } from "@/retrieval/search";
+import { workspaceSearch } from "@/retrieval/workspace-search";
+
+/** Shape returned for each entity hit in sia_search results. */
+export interface SiaSearchResult {
+	id: string;
+	type: string;
+	name: string;
+	summary: string;
+	content: string;
+	trust_tier: number;
+	confidence: number;
+	importance: number;
+	tags: string;
+	file_paths: string;
+	conflict_group_id: string | null;
+	t_valid_from: number | null;
+	source_repo_name: string | null;
+	extraction_method?: string | null;
+}
+
+/** Dependencies for workspace-scoped search. */
+export interface WorkspaceDeps {
+	metaDb: SiaDb;
+	bridgeDb: SiaDb;
+	workspaceId: string;
+	primaryRepoId: string;
+	siaHome?: string;
+}
+
+/** Maximum number of results sia_search will return regardless of input. */
+const MAX_LIMIT = 15;
+
+/** Default number of results when `limit` is not specified. */
+const DEFAULT_LIMIT = 5;
+
+/**
+ * Execute a simplified search against the entities table.
+ *
+ * Filters:
+ *  - Active only: t_valid_until IS NULL AND archived_at IS NULL
+ *  - paranoid mode: additionally excludes trust_tier = 4
+ *  - node_types: IN filter on type column
+ *  - package_path: exact match on package_path column
+ *
+ * Results are ordered by importance DESC and capped at `limit` (default 5, max 15).
+ * When `workspace: true` and workspaceDeps are provided, delegates to workspace search.
+ */
+export async function handleSiaSearch(
+	db: SiaDb,
+	input: z.infer<typeof SiaSearchInput>,
+	_embedder?: Embedder,
+	workspaceDeps?: WorkspaceDeps,
+): Promise<SiaSearchResult[]> {
+	// Workspace-scoped search
+	if (input.workspace && workspaceDeps) {
+		const result = await workspaceSearch({
+			primaryDb: db,
+			metaDb: workspaceDeps.metaDb,
+			bridgeDb: workspaceDeps.bridgeDb,
+			workspaceId: workspaceDeps.workspaceId,
+			primaryRepoId: workspaceDeps.primaryRepoId,
+			query: input.query,
+			siaHome: workspaceDeps.siaHome,
+			limit: input.limit,
+			paranoid: input.paranoid,
+			node_types: input.node_types,
+			package_path: input.package_path,
+		});
+		return (await annotateFreshness(
+			result.entities as unknown as Record<string, unknown>[],
+			db,
+		)) as unknown as SiaSearchResult[];
+	}
+
+	// Compute effective limit
+	const rawLimit = input.limit ?? DEFAULT_LIMIT;
+	const effectiveLimit = Math.min(Math.max(1, rawLimit), MAX_LIMIT);
+
+	// Local search via three-stage pipeline
+	const searchResult = await hybridSearch(db, _embedder ?? null, {
+		query: input.query,
+		taskType: input.task_type,
+		nodeTypes: input.node_types,
+		packagePath: input.package_path,
+		paranoid: input.paranoid,
+		limit: effectiveLimit,
+		includeProvenance: input.include_provenance,
+	});
+
+	return (await annotateFreshness(
+		searchResult.results as unknown as Record<string, unknown>[],
+		db,
+	)) as unknown as SiaSearchResult[];
+}