@rkarim08/sia 1.0.0

package/src/retrieval/vector-search.ts

@@ -0,0 +1,203 @@
+// Module: vector-search — ONNX embedder + cosine similarity (VSS fallback)
+
+import type { Embedder } from "@/capture/embedder";
+import type { SiaDb } from "@/graph/db-interface";
+
+/** A single vector search result: entity ID + similarity score. */
+export interface VectorResult {
+	entityId: string;
+	score: number;
+}
+
+/** Options for vectorSearch. */
+export interface VectorSearchOpts {
+	limit?: number;
+	paranoid?: boolean;
+	packagePath?: string;
+}
+
+/** Default similarity threshold below which results are discarded. */
+const SIMILARITY_THRESHOLD = 0.3;
+
+/** Maximum candidate entities to scan in brute-force fallback. */
+const BRUTE_FORCE_LIMIT = 1000;
+
+/**
+ * Compute cosine similarity between two Float32Arrays.
+ *
+ * Both vectors are assumed to be of equal length. Returns 0 if either
+ * has zero magnitude (degenerate case).
+ */
+function cosineSim(a: Float32Array, b: Float32Array): number {
+	let dot = 0;
+	let normA = 0;
+	let normB = 0;
+
+	for (let i = 0; i < a.length; i++) {
+		dot += a[i] * b[i];
+		normA += a[i] * a[i];
+		normB += b[i] * b[i];
+	}
+
+	const denom = Math.sqrt(normA) * Math.sqrt(normB);
+	if (denom === 0) return 0;
+	return dot / denom;
+}
+
+/**
+ * Search entities by vector similarity.
+ *
+ * 1. Embed query via the provided embedder.
+ * 2. Try sqlite-vss (`vss_search`) on the `graph_nodes_vss` virtual table.
+ * 3. If VSS is unavailable, fall back to brute-force cosine scan over
+ *    entities that have a non-NULL embedding column (capped at 1000).
+ *
+ * Results are filtered by an optional paranoid flag (excludes tier 4)
+ * and packagePath, then sorted by score descending and capped at `limit`.
+ */
+export async function vectorSearch(
+	db: SiaDb,
+	query: string,
+	embedder: Embedder,
+	opts?: VectorSearchOpts,
+): Promise<VectorResult[]> {
+	const limit = opts?.limit ?? 15;
+
+	// Step 1: Embed the query text
+	const queryEmbedding = await embedder.embed(query);
+	if (!queryEmbedding) return [];
+
+	// Step 2: Try sqlite-vss via rawSqlite()
+	const vssResults = tryVssSearch(db, queryEmbedding, limit, opts);
+	if (vssResults !== null) return vssResults;
+
+	// Step 3: Brute-force cosine scan fallback
+	return bruteForceCosineSearch(db, queryEmbedding, limit, opts);
+}
+
+/**
+ * Attempt to use sqlite-vss extension for fast approximate search.
+ * Returns null if VSS is not available (extension not loaded, table missing, etc.).
+ */
+function tryVssSearch(
+	db: SiaDb,
+	queryEmbedding: Float32Array,
+	limit: number,
+	opts?: VectorSearchOpts,
+): VectorResult[] | null {
+	const raw = db.rawSqlite();
+	if (!raw) return null;
+
+	try {
+		// Serialize embedding to JSON array for vss_search
+		const embeddingJson = JSON.stringify(Array.from(queryEmbedding));
+
+		// Use vss_search to get candidate rowids with distances
+		const vssRows = raw
+			.prepare(
+				`SELECT rowid, distance
+				 FROM vss_search(graph_nodes_vss, ?, ?)`,
+			)
+			.all(embeddingJson, limit * 2) as Array<{ rowid: number; distance: number }>;
+
+		if (!vssRows || vssRows.length === 0) return null;
+
+		// Map rowids back to entity IDs with filters
+		const results: VectorResult[] = [];
+		for (const vssRow of vssRows) {
+			// Convert distance to similarity score (VSS returns L2 distance)
+			const score = 1 / (1 + vssRow.distance);
+			if (score < SIMILARITY_THRESHOLD) continue;
+
+			// Look up entity to apply filters
+			const entity = raw
+				.prepare(
+					`SELECT id, trust_tier, package_path
+					 FROM graph_nodes
+					 WHERE rowid = ?
+					   AND t_valid_until IS NULL
+					   AND archived_at IS NULL`,
+				)
+				.get(vssRow.rowid) as
+				| { id: string; trust_tier: number; package_path: string | null }
+				| undefined;
+
+			if (!entity) continue;
+			if (opts?.paranoid && entity.trust_tier === 4) continue;
+			if (opts?.packagePath && entity.package_path !== opts.packagePath) continue;
+
+			results.push({ entityId: entity.id, score });
+		}
+
+		results.sort((a, b) => b.score - a.score);
+		return results.slice(0, limit);
+	} catch {
+		// VSS extension not loaded or table doesn't exist — fall through to brute-force
+		return null;
+	}
+}
+
+/**
+ * Brute-force cosine similarity scan.
+ *
+ * Queries up to BRUTE_FORCE_LIMIT entities that have a non-NULL embedding,
+ * computes cosine similarity against the query embedding, filters by
+ * threshold and optional constraints, then returns sorted top-N results.
+ */
+async function bruteForceCosineSearch(
+	db: SiaDb,
+	queryEmbedding: Float32Array,
+	limit: number,
+	opts?: VectorSearchOpts,
+): Promise<VectorResult[]> {
+	// Build WHERE clauses
+	const clauses: string[] = [
+		"embedding IS NOT NULL",
+		"t_valid_until IS NULL",
+		"archived_at IS NULL",
+	];
+	const params: unknown[] = [];
+
+	if (opts?.paranoid) {
+		clauses.push("trust_tier != 4");
+	}
+	if (opts?.packagePath) {
+		clauses.push("package_path = ?");
+		params.push(opts.packagePath);
+	}
+
+	params.push(BRUTE_FORCE_LIMIT);
+
+	const sql = `SELECT id, embedding FROM graph_nodes WHERE ${clauses.join(" AND ")} LIMIT ?`;
+	const { rows } = await db.execute(sql, params);
+
+	const results: VectorResult[] = [];
+
+	for (const row of rows) {
+		const embeddingBlob = row.embedding;
+		if (!embeddingBlob) continue;
+
+		// Convert stored BLOB to Float32Array
+		let storedEmbedding: Float32Array;
+		if (embeddingBlob instanceof Buffer || embeddingBlob instanceof Uint8Array) {
+			storedEmbedding = new Float32Array(
+				(embeddingBlob as Uint8Array).buffer,
+				(embeddingBlob as Uint8Array).byteOffset,
+				(embeddingBlob as Uint8Array).byteLength / 4,
+			);
+		} else if (embeddingBlob instanceof ArrayBuffer) {
+			storedEmbedding = new Float32Array(embeddingBlob);
+		} else {
+			// Unexpected type — skip
+			continue;
+		}
+
+		const score = cosineSim(queryEmbedding, storedEmbedding);
+		if (score < SIMILARITY_THRESHOLD) continue;
+
+		results.push({ entityId: row.id as string, score });
+	}
+
+	results.sort((a, b) => b.score - a.score);
+	return results.slice(0, limit);
+}

package/src/retrieval/workspace-search.ts

@@ -0,0 +1,130 @@
+// Module: workspace-search — Workspace-scoped search via ATTACH
+
+import type { SiaDb } from "@/graph/db-interface";
+import type { SiaSearchResult } from "@/mcp/tools/sia-search";
+import { SIA_HOME } from "@/shared/config";
+import { getPeerRepos } from "@/workspace/cross-repo";
+
+/** Options for workspaceSearch. */
+export interface WorkspaceSearchOpts {
+	primaryDb: SiaDb;
+	metaDb: SiaDb;
+	bridgeDb: SiaDb;
+	workspaceId: string;
+	primaryRepoId: string;
+	query: string;
+	siaHome?: string;
+	limit?: number;
+	paranoid?: boolean;
+	node_types?: string[];
+	package_path?: string;
+}
+
+/** Result of a workspace search. */
+export interface WorkspaceSearchResult {
+	entities: SiaSearchResult[];
+	missingRepos: string[];
+}
+
+/** Max peer repos to ATTACH (SQLite limit of 10 - main - bridge = 8) */
+const MAX_PEERS = 8;
+
+/**
+ * Perform workspace-scoped search across primary + peer repo databases.
+ *
+ * ATTACHes one peer at a time (ATTACH, query, DETACH, next) to stay safely
+ * within SQLite limits and simplify error handling. Results are merged and
+ * re-sorted after all peers are queried.
+ *
+ * Missing peers produce metadata entries, not errors.
+ * Does NOT set WAL pragma on attached read-only databases.
+ */
+export async function workspaceSearch(opts: WorkspaceSearchOpts): Promise<WorkspaceSearchResult> {
+	const siaHome = opts.siaHome ?? SIA_HOME;
+	const limit = opts.limit ?? 15;
+	const missingRepos: string[] = [];
+
+	// Get peers from meta.db
+	const allPeers = await getPeerRepos(opts.metaDb, opts.workspaceId, opts.primaryRepoId, siaHome);
+
+	// Cap at MAX_PEERS
+	const peers = allPeers.slice(0, MAX_PEERS);
+	if (allPeers.length > MAX_PEERS) {
+		for (let i = MAX_PEERS; i < allPeers.length; i++) {
+			missingRepos.push(allPeers[i].name ?? allPeers[i].repoId);
+		}
+	}
+
+	// Build WHERE clause
+	const clauses: string[] = ["t_valid_until IS NULL", "archived_at IS NULL"];
+	if (opts.paranoid) clauses.push("trust_tier != 4");
+	if (opts.node_types && opts.node_types.length > 0) {
+		const placeholders = opts.node_types.map(() => "?").join(", ");
+		clauses.push(`type IN (${placeholders})`);
+	}
+	if (opts.package_path) {
+		clauses.push("package_path = ?");
+	}
+	const whereClause = clauses.join(" AND ");
+
+	// Build params for WHERE clause (without limit)
+	const filterParams: unknown[] = [];
+	if (opts.node_types) filterParams.push(...opts.node_types);
+	if (opts.package_path) filterParams.push(opts.package_path);
+
+	// Query primary
+	const allEntities: SiaSearchResult[] = [];
+	const primarySql = `SELECT * FROM graph_nodes WHERE ${whereClause} ORDER BY importance DESC LIMIT ?`;
+	const primaryResult = await opts.primaryDb.execute(primarySql, [...filterParams, limit]);
+
+	for (const row of primaryResult.rows) {
+		allEntities.push(mapRow(row, null));
+	}
+
+	// Query each peer via ATTACH
+	for (const peer of peers) {
+		try {
+			await opts.primaryDb.execute("ATTACH DATABASE ? AS peer_db", [peer.graphDbPath]);
+
+			const peerSql = `SELECT * FROM peer_db.graph_nodes WHERE ${whereClause} ORDER BY importance DESC LIMIT ?`;
+			const peerResult = await opts.primaryDb.execute(peerSql, [...filterParams, limit]);
+
+			for (const row of peerResult.rows) {
+				allEntities.push(mapRow(row, peer.name));
+			}
+
+			await opts.primaryDb.execute("DETACH DATABASE peer_db", []);
+		} catch {
+			missingRepos.push(peer.name ?? peer.repoId);
+			try {
+				await opts.primaryDb.execute("DETACH DATABASE peer_db", []);
+			} catch {
+				/* already detached or never attached */
+			}
+		}
+	}
+
+	// Sort all by importance DESC, take top `limit`
+	allEntities.sort((a, b) => b.importance - a.importance);
+	const capped = allEntities.slice(0, limit);
+
+	return { entities: capped, missingRepos };
+}
+
+function mapRow(row: Record<string, unknown>, sourceRepoName: string | null): SiaSearchResult {
+	return {
+		id: row.id as string,
+		type: row.type as string,
+		name: row.name as string,
+		summary: (row.summary as string) ?? "",
+		content: (row.content as string) ?? "",
+		trust_tier: row.trust_tier as number,
+		confidence: row.confidence as number,
+		importance: row.importance as number,
+		tags: (row.tags as string) ?? "[]",
+		file_paths: (row.file_paths as string) ?? "[]",
+		conflict_group_id: (row.conflict_group_id as string | null) ?? null,
+		t_valid_from: (row.t_valid_from as number | null) ?? null,
+		source_repo_name: sourceRepoName,
+	};
+}

package/src/sandbox/context-mode.ts

@@ -0,0 +1,285 @@
+// Module: context-mode — Large output chunking with intent-based retrieval using strategy pattern
+
+import { randomUUID } from "node:crypto";
+import type { Embedder } from "@/capture/embedder";
+import type { SiaDb } from "@/graph/db-interface";
+
+// ---------------------------------------------------------------------------
+// Public interfaces
+// ---------------------------------------------------------------------------
+
+export interface RawChunk {
+	text: string;
+	metadata?: Record<string, unknown>;
+}
+
+export interface StoredChunk {
+	id: string;
+	text: string;
+	embedding: number[];
+	nodeId: string; // ContentChunk entity ID in graph
+}
+
+export interface ChunkStrategy {
+	name: string;
+	chunk(content: string): RawChunk[];
+	extraEdges?(chunk: StoredChunk, db: SiaDb): Promise<void>;
+}
+
+export interface ContextModeResult {
+	applied: boolean;
+	chunks: string[];
+	totalIndexed: number;
+	contextSavings: number;
+}
+
+// ---------------------------------------------------------------------------
+// headingChunker strategy — splits markdown by heading lines outside code blocks
+// ---------------------------------------------------------------------------
+
+export const headingChunker: ChunkStrategy = {
+	name: "headingChunker",
+
+	chunk(content: string): RawChunk[] {
+		const lines = content.split("\n");
+		const chunks: RawChunk[] = [];
+		let inCodeBlock = false;
+		let currentHeading = "";
+		let currentLines: string[] = [];
+
+		const flush = () => {
+			if (currentLines.length > 0) {
+				chunks.push({
+					text: currentLines.join("\n"),
+					metadata: { heading: currentHeading },
+				});
+			}
+		};
+
+		for (const line of lines) {
+			// Track code fence state
+			if (line.trimStart().startsWith("```")) {
+				inCodeBlock = !inCodeBlock;
+				currentLines.push(line);
+				continue;
+			}
+
+			// Only split on headings outside code blocks
+			if (!inCodeBlock && /^#{1,6} /.test(line)) {
+				flush();
+				currentHeading = line;
+				currentLines = [line];
+			} else {
+				currentLines.push(line);
+			}
+		}
+
+		flush();
+
+		// If no headings were found, return single chunk
+		if (chunks.length === 0) {
+			return [{ text: content, metadata: { heading: "" } }];
+		}
+
+		return chunks;
+	},
+};
+
+// ---------------------------------------------------------------------------
+// lineChunker strategy — groups newline-delimited lines into ~512-token (~2048 char) paragraphs
+// ---------------------------------------------------------------------------
+
+const LINE_CHUNK_SIZE = 2048;
+
+export const lineChunker: ChunkStrategy = {
+	name: "lineChunker",
+
+	chunk(content: string): RawChunk[] {
+		const lines = content.split("\n");
+		const chunks: RawChunk[] = [];
+		let current = "";
+		let currentStart = 0;
+
+		for (let i = 0; i < lines.length; i++) {
+			const line = lines[i];
+			const candidate = current.length === 0 ? line : `${current}\n${line}`;
+
+			if (candidate.length > LINE_CHUNK_SIZE && current.length > 0) {
+				chunks.push({
+					text: current,
+					metadata: { startLine: currentStart, endLine: i - 1 },
+				});
+				current = line;
+				currentStart = i;
+			} else {
+				current = candidate;
+			}
+		}
+
+		if (current.length > 0) {
+			chunks.push({
+				text: current,
+				metadata: { startLine: currentStart, endLine: lines.length - 1 },
+			});
+		}
+
+		return chunks;
+	},
+};
+
+// ---------------------------------------------------------------------------
+// contentTypeChunker strategy — detects content type and delegates to the right chunker
+// ---------------------------------------------------------------------------
+
+export const contentTypeChunker: ChunkStrategy = {
+	name: "contentTypeChunker",
+
+	chunk(content: string): RawChunk[] {
+		const trimmed = content.trimStart();
+
+		// Detect JSON (starts with { or [)
+		if (trimmed.startsWith("{") || trimmed.startsWith("[")) {
+			try {
+				const parsed = JSON.parse(content);
+				const formatted = JSON.stringify(parsed, null, 2);
+				return lineChunker.chunk(formatted);
+			} catch {
+				// Not valid JSON (possibly truncated by output cap) — fall through to heading/line detection
+			}
+		}
+
+		// Check for markdown headings
+		if (/^#{1,6} /m.test(content)) {
+			return headingChunker.chunk(content);
+		}
+
+		// Default: lineChunker
+		return lineChunker.chunk(content);
+	},
+};
+
+// ---------------------------------------------------------------------------
+// Cosine similarity between two number arrays
+// ---------------------------------------------------------------------------
+
+function cosineSimilarity(a: number[], b: number[]): number {
+	if (a.length === 0 || b.length === 0) return 0;
+	let dot = 0;
+	let normA = 0;
+	let normB = 0;
+	const len = Math.min(a.length, b.length);
+	for (let i = 0; i < len; i++) {
+		dot += a[i] * b[i];
+		normA += a[i] * a[i];
+		normB += b[i] * b[i];
+	}
+	const denom = Math.sqrt(normA) * Math.sqrt(normB);
+	return denom === 0 ? 0 : dot / denom;
+}
+
+// ---------------------------------------------------------------------------
+// applyContextMode — main entry point
+// ---------------------------------------------------------------------------
+
+/**
+ * Apply context mode to a large content string.
+ *
+ * If content.length <= threshold OR intent is undefined, returns raw content unchanged.
+ * Otherwise, chunks the content via the strategy, embeds each chunk into the graph DB
+ * as ContentChunk entities, embeds the intent, and returns the top-K chunks by
+ * cosine similarity to the intent embedding.
+ */
+export async function applyContextMode(
+	content: string,
+	intent: string | undefined,
+	strategy: ChunkStrategy,
+	db: SiaDb,
+	embedder: Embedder,
+	sessionId: string,
+	config: { threshold: number; topK: number },
+): Promise<ContextModeResult> {
+	// Short-circuit: below threshold or no intent
+	if (content.length <= config.threshold || intent === undefined) {
+		return {
+			applied: false,
+			chunks: [content],
+			totalIndexed: 0,
+			contextSavings: 0,
+		};
+	}
+
+	const now = Date.now();
+	const nowStr = String(now);
+
+	// 1. Chunk content via strategy
+	const rawChunks = strategy.chunk(content);
+
+	// 2. Embed each chunk and store as ContentChunk entity in the graph
+	const storedChunks: StoredChunk[] = [];
+
+	for (let i = 0; i < rawChunks.length; i++) {
+		const raw = rawChunks[i];
+		const nodeId = randomUUID();
+		const chunkName = `chunk-${sessionId}-${i}`;
+
+		// Embed the chunk text
+		const rawEmb = await embedder.embed(raw.text);
+		const embedding: number[] = rawEmb ? Array.from(rawEmb) : [];
+
+		// Store entity in graph DB (table is 'graph_nodes' after v5 migration)
+		await db.execute(
+			`INSERT INTO graph_nodes (id, type, name, summary, content, trust_tier, confidence, base_confidence, importance, base_importance, access_count, edge_count, tags, file_paths, t_created, t_valid_from, created_by, created_at, last_accessed)
+			 VALUES (?, 'ContentChunk', ?, ?, ?, 3, 0.8, 0.8, 0.5, 0.5, 0, 0, '[]', '[]', ?, ?, 'sia-context-mode', ?, ?)`,
+			[nodeId, chunkName, raw.text.slice(0, 100), raw.text, nowStr, nowStr, nowStr, nowStr],
+		);
+
+		const stored: StoredChunk = {
+			id: randomUUID(),
+			text: raw.text,
+			embedding,
+			nodeId,
+		};
+
+		storedChunks.push(stored);
+
+		// Call extraEdges if defined
+		if (strategy.extraEdges) {
+			await strategy.extraEdges(stored, db);
+		}
+	}
+
+	// 3. Embed the intent
+	const intentEmbRaw = await embedder.embed(intent);
+	if (!intentEmbRaw) {
+		// Embedder failed — cannot do intent-based retrieval, return all chunks
+		return {
+			applied: true,
+			chunks: storedChunks.map((s) => s.text).slice(0, config.topK),
+			totalIndexed: storedChunks.length,
+			contextSavings:
+				content.length -
+				storedChunks.slice(0, config.topK).reduce((sum, s) => sum + s.text.length, 0),
+		};
+	}
+	const intentEmbedding: number[] = Array.from(intentEmbRaw);
+
+	// 4. Cosine similarity between intent embedding and each stored chunk embedding
+	const scored = storedChunks.map((chunk) => ({
+		chunk,
+		score: cosineSimilarity(intentEmbedding, chunk.embedding),
+	}));
+
+	// Sort by similarity descending, take top-K
+	scored.sort((a, b) => b.score - a.score);
+	const topChunks = scored.slice(0, config.topK).map((s) => s.chunk.text);
+
+	const totalIndexed = storedChunks.length;
+	const contextSavings = content.length - topChunks.reduce((sum, c) => sum + c.length, 0);
+
+	return {
+		applied: true,
+		chunks: topChunks,
+		totalIndexed,
+		contextSavings: Math.max(0, contextSavings),
+	};
+}

package/src/sandbox/credential-pass.ts

@@ -0,0 +1,55 @@
+// Module: credential-pass — Build allowlisted env for sandbox subprocesses
+
+/** Exact env var names that always pass through. */
+const EXACT_ALLOWLIST = [
+	"PATH",
+	"HOME",
+	"USER",
+	"SHELL",
+	"LANG",
+	"TERM",
+	"KUBECONFIG",
+	"GH_TOKEN",
+	"GITHUB_TOKEN",
+	"NODE_PATH",
+	"BUN_INSTALL",
+] as const;
+
+/** Glob prefixes — any env var starting with these passes through. */
+const PREFIX_ALLOWLIST = ["AWS_", "GOOGLE_", "GCLOUD_", "CLOUDSDK_", "DOCKER_", "GITHUB_"] as const;
+
+/** Exported for test assertions. */
+export const ENV_ALLOWLIST = { exact: EXACT_ALLOWLIST, prefixes: PREFIX_ALLOWLIST };
+
+function isAllowlisted(key: string): boolean {
+	if ((EXACT_ALLOWLIST as readonly string[]).includes(key)) return true;
+	return PREFIX_ALLOWLIST.some((prefix) => key.startsWith(prefix));
+}
+
+/**
+ * Build a filtered env object for sandbox subprocess execution.
+ * Only allowlisted env vars from process.env pass through.
+ * `overrides` are merged last — user-provided values win.
+ * Never logs or persists any env values.
+ */
+export function buildSandboxEnv(overrides?: Record<string, string>): Record<string, string> {
+	const env: Record<string, string> = {};
+
+	for (const [key, value] of Object.entries(process.env)) {
+		if (value !== undefined && isAllowlisted(key)) {
+			env[key] = value;
+		}
+	}
+
+	if (overrides) {
+		for (const [key, value] of Object.entries(overrides)) {
+			if (isAllowlisted(key)) {
+				env[key] = value;
+			} else {
+				console.warn(`[sia-sandbox] env override "${key}" dropped: not in allowlist`);
+			}
+		}
+	}
+
+	return env;
+}