@rkarim08/sia 1.0.0

package/src/capture/tokenizer.ts

@@ -0,0 +1,147 @@
+// Module: tokenizer — Basic WordPiece tokenizer for HuggingFace tokenizer.json files
+
+import { readFileSync } from "node:fs";
+
+/** Special token IDs for BERT-style tokenizers. */
+const CLS_ID = 101;
+const SEP_ID = 102;
+const UNK_ID = 100;
+const PAD_ID = 0;
+
+/** Default maximum sequence length. */
+const DEFAULT_MAX_LENGTH = 128;
+
+/**
+ * A loaded tokenizer: just the vocabulary mapping word → token id.
+ */
+export interface Tokenizer {
+	vocab: Map<string, number>;
+}
+
+/**
+ * Tokenized output ready for ONNX inference.
+ */
+export interface TokenizedOutput {
+	inputIds: BigInt64Array;
+	attentionMask: BigInt64Array;
+}
+
+/**
+ * Load a HuggingFace tokenizer.json and extract its vocabulary.
+ *
+ * The file format has `model.vocab` as an object mapping token strings to ids.
+ */
+export function loadTokenizer(tokenizerJsonPath: string): Tokenizer {
+	const raw = readFileSync(tokenizerJsonPath, "utf-8");
+	const json = JSON.parse(raw);
+
+	const vocabObj: Record<string, number> = json?.model?.vocab ?? {};
+	const vocab = new Map<string, number>(Object.entries(vocabObj));
+
+	return { vocab };
+}
+
+/**
+ * Look up a single token string in the vocabulary, falling back to [UNK].
+ */
+function lookupToken(vocab: Map<string, number>, token: string): number {
+	return vocab.get(token) ?? UNK_ID;
+}
+
+/**
+ * Perform basic WordPiece-style tokenization:
+ *   1. Lowercase the text
+ *   2. Strip punctuation into separate tokens
+ *   3. Split on whitespace
+ *   4. For each word, try to find it in the vocabulary; if not found, split
+ *      into sub-word pieces using the ## prefix convention
+ *   5. Prepend [CLS], append [SEP], pad to maxLength
+ */
+export function tokenize(
+	tokenizer: Tokenizer,
+	text: string,
+	maxLength: number = DEFAULT_MAX_LENGTH,
+): TokenizedOutput {
+	const { vocab } = tokenizer;
+
+	// Lowercase and insert spaces around punctuation so they become separate tokens
+	const normalized = text
+		.toLowerCase()
+		.replace(/([^\w\s])/g, " $1 ")
+		.replace(/\s+/g, " ")
+		.trim();
+
+	const words = normalized.length > 0 ? normalized.split(" ") : [];
+
+	// Tokenize each word with basic WordPiece
+	const tokenIds: number[] = [CLS_ID];
+
+	for (const word of words) {
+		// Budget: leave room for trailing [SEP] + current position
+		if (tokenIds.length >= maxLength - 1) break;
+
+		if (vocab.has(word)) {
+			tokenIds.push(lookupToken(vocab, word));
+			continue;
+		}
+
+		// Attempt sub-word splitting
+		const subTokens = wordPieceSplit(vocab, word);
+		for (const st of subTokens) {
+			if (tokenIds.length >= maxLength - 1) break;
+			tokenIds.push(st);
+		}
+	}
+
+	tokenIds.push(SEP_ID);
+
+	// Build padded arrays
+	const inputIds = new BigInt64Array(maxLength);
+	const attentionMask = new BigInt64Array(maxLength);
+
+	for (let i = 0; i < maxLength; i++) {
+		if (i < tokenIds.length) {
+			inputIds[i] = BigInt(tokenIds[i]);
+			attentionMask[i] = 1n;
+		} else {
+			inputIds[i] = BigInt(PAD_ID);
+			attentionMask[i] = 0n;
+		}
+	}
+
+	return { inputIds, attentionMask };
+}
+
+/**
+ * Greedy left-to-right WordPiece splitting for a single word.
+ * Returns an array of token ids.
+ */
+function wordPieceSplit(vocab: Map<string, number>, word: string): number[] {
+	const ids: number[] = [];
+	let start = 0;
+
+	while (start < word.length) {
+		let end = word.length;
+		let matched = false;
+
+		while (start < end) {
+			const substr = start === 0 ? word.slice(start, end) : `##${word.slice(start, end)}`;
+
+			if (vocab.has(substr)) {
+				ids.push(lookupToken(vocab, substr));
+				matched = true;
+				start = end;
+				break;
+			}
+			end--;
+		}
+
+		if (!matched) {
+			// No sub-word match found — emit [UNK] for this character and advance
+			ids.push(UNK_ID);
+			start++;
+		}
+	}
+
+	return ids;
+}

package/src/capture/track-a-ast.ts

@@ -0,0 +1,93 @@
+// Module: track-a-ast — structural extraction of code entities via regex patterns
+
+import type { CandidateFact } from "@/capture/types";
+
+/** Map of file extension to an array of regex patterns that capture entity names. */
+const PATTERNS: Record<string, RegExp[]> = {
+	".ts": [
+		/export\s+(?:async\s+)?function\s+(\w+)/gm,
+		/export\s+class\s+(\w+)/gm,
+		/export\s+const\s+(\w+)/gm,
+	],
+	".tsx": [
+		/export\s+(?:async\s+)?function\s+(\w+)/gm,
+		/export\s+class\s+(\w+)/gm,
+		/export\s+const\s+(\w+)/gm,
+	],
+	".js": [
+		/export\s+(?:async\s+)?function\s+(\w+)/gm,
+		/export\s+class\s+(\w+)/gm,
+		/export\s+const\s+(\w+)/gm,
+	],
+	".jsx": [
+		/export\s+(?:async\s+)?function\s+(\w+)/gm,
+		/export\s+class\s+(\w+)/gm,
+		/export\s+const\s+(\w+)/gm,
+	],
+	".py": [/^def\s+(\w+)/gm, /^class\s+(\w+)/gm],
+	".go": [/^func\s+(\w+)/gm, /^type\s+(\w+)\s+struct/gm],
+	".rs": [/^pub\s+fn\s+(\w+)/gm, /^pub\s+struct\s+(\w+)/gm],
+};
+
+/**
+ * Given the surrounding lines around a match index, return a small context window.
+ */
+function surroundingLines(content: string, matchIndex: number): string {
+	const before = content.lastIndexOf("\n", matchIndex);
+	const lineStart = before === -1 ? 0 : before + 1;
+	let end = matchIndex;
+	for (let i = 0; i < 3; i++) {
+		const next = content.indexOf("\n", end + 1);
+		if (next === -1) {
+			end = content.length;
+			break;
+		}
+		end = next;
+	}
+	return content.slice(lineStart, end).trim();
+}
+
+/**
+ * Extract structural code entities from file content using regex-based pattern matching.
+ *
+ * @param content  The file text to scan.
+ * @param filePath Optional path used to determine language by extension.
+ * @returns An array of CandidateFact objects for every matched entity.
+ */
+export function extractTrackA(content: string, filePath?: string): CandidateFact[] {
+	if (!filePath) return [];
+
+	const dotIdx = filePath.lastIndexOf(".");
+	if (dotIdx === -1) return [];
+
+	const ext = filePath.slice(dotIdx);
+	const patterns = PATTERNS[ext];
+	if (!patterns) return [];
+
+	const facts: CandidateFact[] = [];
+
+	for (const pattern of patterns) {
+		// Reset lastIndex so the regex starts from the beginning each time
+		pattern.lastIndex = 0;
+
+		let match: RegExpExecArray | null = pattern.exec(content);
+		while (match !== null) {
+			const name = match[1];
+			const context = surroundingLines(content, match.index);
+			facts.push({
+				type: "CodeEntity",
+				name,
+				content: context,
+				summary: `CodeEntity: ${name}`,
+				tags: [],
+				file_paths: [filePath],
+				trust_tier: 2,
+				confidence: 0.92,
+				extraction_method: "regex-ast",
+			});
+			match = pattern.exec(content);
+		}
+	}
+
+	return facts;
+}

package/src/capture/track-b-llm.ts

@@ -0,0 +1,149 @@
+// Module: track-b-llm — semantic extraction via LLM or pattern matching (Track B)
+
+import { extractPrompt } from "@/capture/prompts/extract";
+import type { CandidateFact, EntityType, ProposedRelationship } from "@/capture/types";
+
+interface LlmMessageParam {
+	role: string;
+	content: string;
+}
+
+interface LlmCreateParams {
+	model: string;
+	max_tokens: number;
+	system: string;
+	messages: LlmMessageParam[];
+}
+
+interface LlmResponse {
+	content: Array<{ text: string }>;
+}
+
+interface LlmClient {
+	messages: {
+		create: (params: LlmCreateParams) => Promise<LlmResponse>;
+	};
+}
+
+interface TrackBConfig {
+	captureModel: string;
+	minExtractConfidence: number;
+	airGapped: boolean;
+	llmClient?: LlmClient;
+}
+
+interface PatternEntry {
+	pattern: RegExp;
+	type: EntityType;
+}
+
+interface RawFact {
+	type?: string;
+	name?: string;
+	content?: string;
+	summary?: string;
+	tags?: string[];
+	file_paths?: string[];
+	confidence?: number;
+	proposed_relationships?: ProposedRelationship[];
+}
+
+const PATTERNS: PatternEntry[] = [
+	{
+		pattern: /decided to|chose|we will use|going with|selected|opted for/i,
+		type: "Decision",
+	},
+	{
+		pattern: /bug|error|crash|broken|failing|exception|regression/i,
+		type: "Bug",
+	},
+	{
+		pattern: /always|never|must|convention|rule:|standard:/i,
+		type: "Convention",
+	},
+	{
+		pattern: /fix|solved|workaround|resolution|patch/i,
+		type: "Solution",
+	},
+];
+
+function splitSentences(content: string): string[] {
+	return content
+		.split(/\. |\n/)
+		.map((s) => s.trim())
+		.filter((s) => s.length > 0);
+}
+
+function patternMatch(content: string, minConfidence: number): CandidateFact[] {
+	const sentences = splitSentences(content);
+	const candidates: CandidateFact[] = [];
+
+	for (const sentence of sentences) {
+		for (const { pattern, type } of PATTERNS) {
+			if (pattern.test(sentence)) {
+				candidates.push({
+					type,
+					name: sentence.slice(0, 50),
+					content: sentence,
+					summary: sentence.slice(0, 80),
+					tags: [],
+					file_paths: [],
+					trust_tier: 3,
+					confidence: 0.7,
+					extraction_method: "pattern-match",
+				});
+				break;
+			}
+		}
+	}
+
+	return candidates.filter((c) => c.confidence >= minConfidence);
+}
+
+export async function extractTrackB(
+	content: string,
+	config: TrackBConfig,
+): Promise<CandidateFact[]> {
+	if (config.airGapped) {
+		return [];
+	}
+
+	// LLM path: attempt extraction via Anthropic API when client is provided
+	if (config.llmClient) {
+		try {
+			const { system, user } = extractPrompt(content);
+			const response = await config.llmClient.messages.create({
+				model: config.captureModel,
+				max_tokens: 1024,
+				system,
+				messages: [{ role: "user", content: user }],
+			});
+
+			const rawText: string = response.content[0].text;
+			const parsed: { facts?: RawFact[] } = JSON.parse(rawText) as { facts?: RawFact[] };
+			const facts: RawFact[] = Array.isArray(parsed.facts) ? parsed.facts : [];
+
+			const candidates: CandidateFact[] = facts.map((f: RawFact) => ({
+				type: (f.type ?? "Concept") as EntityType,
+				name: String(f.name ?? "").slice(0, 200),
+				content: String(f.content ?? ""),
+				summary: String(f.summary ?? ""),
+				tags: Array.isArray(f.tags) ? f.tags : [],
+				file_paths: Array.isArray(f.file_paths) ? f.file_paths : [],
+				trust_tier: 3,
+				confidence: typeof f.confidence === "number" ? f.confidence : 0.7,
+				extraction_method: "llm-haiku",
+				proposed_relationships: Array.isArray(f.proposed_relationships)
+					? f.proposed_relationships
+					: undefined,
+			}));
+
+			return candidates.filter((c) => c.confidence >= config.minExtractConfidence);
+		} catch {
+			// Fall through to pattern matching
+		}
+	}
+
+	// Pattern matching fallback
+	return patternMatch(content, config.minExtractConfidence);
+}

package/src/capture/types.ts

@@ -0,0 +1,64 @@
+// Module: types — shared types for the capture pipeline
+
+/** The kind of knowledge entity extracted from a conversation. */
+export type EntityType =
+	| "CodeEntity"
+	| "Concept"
+	| "Decision"
+	| "Bug"
+	| "Solution"
+	| "Convention"
+	| "Dependency";
+
+/** A proposed edge to another entity, resolved by name during consolidation. */
+export interface ProposedRelationship {
+	target_name: string;
+	type: string;
+	weight: number;
+}
+
+/** A candidate fact produced by extraction tracks before consolidation. */
+export interface CandidateFact {
+	type: EntityType;
+	name: string;
+	content: string;
+	summary: string;
+	tags: string[];
+	file_paths: string[];
+	trust_tier: 1 | 2 | 3 | 4;
+	confidence: number;
+	extraction_method?: string;
+	proposed_relationships?: ProposedRelationship[];
+	t_valid_from?: number;
+}
+
+/** Payload delivered by Claude-Code hooks into the capture pipeline. */
+export interface HookPayload {
+	cwd: string;
+	type: "PostToolUse" | "Stop";
+	sessionId: string;
+	content: string;
+	toolName?: string;
+	filePath?: string;
+}
+
+/** Operation the consolidator decides for a candidate. */
+export type ConsolidationOp = "ADD" | "UPDATE" | "INVALIDATE" | "NOOP";
+
+/** Aggregate counts returned after a consolidation pass. */
+export interface ConsolidationResult {
+	added: number;
+	updated: number;
+	invalidated: number;
+	noops: number;
+}
+
+/** End-to-end result of a single pipeline invocation. */
+export interface PipelineResult {
+	candidates: number;
+	consolidation: ConsolidationResult;
+	edgesCreated: number;
+	flagsProcessed: number;
+	durationMs: number;
+	circuitBreakerActive: boolean;
+}

package/src/cli/commands/community.ts

@@ -0,0 +1,137 @@
+// Module: community — CLI for printing community structure
+
+import { resolveRepoHash } from "@/capture/hook";
+import { detectCommunities } from "@/community/leiden";
+import { buildSummaryTree } from "@/community/raptor";
+import { summarizeCommunities } from "@/community/summarize";
+import type { SiaDb } from "@/graph/db-interface";
+import { openGraphDb } from "@/graph/semantic-db";
+import { getConfig } from "@/shared/config";
+
+interface CliCommunity {
+	id: string;
+	summary: string | null;
+	memberCount: number;
+	parentId: string | null;
+}
+
+interface CliEntity {
+	name: string;
+	importance: number;
+}
+
+export interface CommunityCliOptions {
+	packagePath?: string;
+}
+
+async function countCommunities(db: SiaDb): Promise<number> {
+	const result = await db.execute(`SELECT COUNT(*) as count FROM communities`);
+	return Number((result.rows[0] as { count: number }).count ?? 0);
+}
+
+async function loadCommunities(
+	db: SiaDb,
+	level: number,
+	packagePath?: string,
+): Promise<CliCommunity[]> {
+	const params: unknown[] = [level];
+	let wherePackage = "";
+	if (packagePath) {
+		wherePackage = "AND package_path = ?";
+		params.push(packagePath);
+	}
+	const result = await db.execute(
+		`SELECT id, summary, member_count as memberCount, parent_id as parentId
+                 FROM communities
+                 WHERE level = ?
+                 ${wherePackage}
+                 ORDER BY memberCount DESC`,
+		params,
+	);
+	return result.rows as unknown as CliCommunity[];
+}
+
+async function topEntities(db: SiaDb, communityId: string): Promise<CliEntity[]> {
+	const result = await db.execute(
+		`SELECT e.name, e.importance
+                 FROM community_members cm
+                 JOIN graph_nodes e ON cm.entity_id = e.id
+                 WHERE cm.community_id = ?
+                 ORDER BY e.importance DESC
+                 LIMIT 5`,
+		[communityId],
+	);
+	return result.rows as unknown as CliEntity[];
+}
+
+export async function formatCommunityTree(
+	db: SiaDb,
+	opts: CommunityCliOptions = {},
+): Promise<string> {
+	const level2 = await loadCommunities(db, 2, opts.packagePath);
+	const level1 = await loadCommunities(db, 1, opts.packagePath);
+	const level0 = await loadCommunities(db, 0, opts.packagePath);
+
+	if (level2.length === 0 && level1.length === 0) {
+		return "No communities yet. Run detection first.";
+	}
+
+	const lines: string[] = [];
+
+	for (const community of level2) {
+		const header = `Community ${community.id.slice(0, 8)} — members: ${community.memberCount}`;
+		lines.push(header);
+		if (community.summary) {
+			lines.push(`  ${community.summary}`);
+		}
+
+		const children = level1.filter((c) => c.parentId === community.id);
+		for (const child of children) {
+			const childTitle = `  - ${child.summary ?? child.id.slice(0, 8)} (${child.memberCount} members)`;
+			lines.push(childTitle);
+			const entities = await topEntities(db, child.id);
+			if (entities.length === 0) {
+				lines.push("    (no entities)");
+			} else {
+				for (const entity of entities) {
+					lines.push(`    - ${entity.name} [importance ${entity.importance.toFixed(2)}]`);
+				}
+			}
+
+			// Level 0 — briefly noted
+			const level0Children = level0.filter((c) => c.parentId === child.id);
+			if (level0Children.length > 0) {
+				lines.push(`    (${level0Children.length} fine-grained clusters)`);
+			}
+		}
+	}
+
+	return lines.join("\n");
+}
+
+export async function runCommunityCommand(args: string[]): Promise<void> {
+	let packagePath: string | undefined;
+	for (let i = 0; i < args.length; i++) {
+		if (args[i] === "--package" || args[i] === "-p") {
+			packagePath = args[i + 1];
+			i++;
+		}
+	}
+
+	const repoHash = resolveRepoHash(process.cwd());
+	const db = openGraphDb(repoHash);
+	const config = getConfig();
+
+	try {
+		const existing = await countCommunities(db);
+		if (existing === 0) {
+			await detectCommunities(db);
+			await summarizeCommunities(db, { airGapped: config.airGapped });
+			await buildSummaryTree(db);
+		}
+		const output = await formatCommunityTree(db, { packagePath });
+		console.log(output);
+	} finally {
+		await db.close();
+	}
+}

package/src/cli/commands/compare.ts

@@ -0,0 +1,123 @@
+// Module: compare — Compare knowledge graph state between two time points
+//
+// Usage:
+//   sia compare --since 2026-03-01 --until 2026-03-15
+//   sia compare --since "last week"
+
+import type { SiaDb } from "@/graph/db-interface";
+
+export interface CompareResult {
+	added: Array<{ id: string; type: string; name: string; created_at: number }>;
+	invalidated: Array<{ id: string; type: string; name: string; t_valid_until: number }>;
+	archived: Array<{ id: string; type: string; name: string; archived_at: number }>;
+	summary: { added: number; invalidated: number; archived: number };
+}
+
+export async function compareGraphState(
+	db: SiaDb,
+	since: number,
+	until: number,
+): Promise<CompareResult> {
+	// Entities created in the time range
+	const addedResult = await db.execute(
+		`SELECT id, type, name, created_at FROM graph_nodes
+		 WHERE created_at >= ? AND created_at <= ?
+		 ORDER BY created_at DESC`,
+		[since, until],
+	);
+
+	// Entities invalidated in the time range
+	const invalidatedResult = await db.execute(
+		`SELECT id, type, name, t_valid_until FROM graph_nodes
+		 WHERE t_valid_until >= ? AND t_valid_until <= ?
+		 ORDER BY t_valid_until DESC`,
+		[since, until],
+	);
+
+	// Entities archived in the time range
+	const archivedResult = await db.execute(
+		`SELECT id, type, name, archived_at FROM graph_nodes
+		 WHERE archived_at >= ? AND archived_at <= ?
+		 ORDER BY archived_at DESC`,
+		[since, until],
+	);
+
+	const added = addedResult.rows as CompareResult["added"];
+	const invalidated = invalidatedResult.rows as CompareResult["invalidated"];
+	const archived = archivedResult.rows as CompareResult["archived"];
+
+	return {
+		added,
+		invalidated,
+		archived,
+		summary: {
+			added: added.length,
+			invalidated: invalidated.length,
+			archived: archived.length,
+		},
+	};
+}
+
+export function formatComparison(result: CompareResult, since: number, until: number): string {
+	const lines: string[] = [];
+	const sinceDate = new Date(since).toISOString().split("T")[0];
+	const untilDate = new Date(until).toISOString().split("T")[0];
+
+	lines.push(`=== SIA Graph Comparison (${sinceDate} to ${untilDate}) ===\n`);
+	lines.push(`Added:       ${result.summary.added}`);
+	lines.push(`Invalidated: ${result.summary.invalidated}`);
+	lines.push(`Archived:    ${result.summary.archived}`);
+
+	if (result.added.length > 0) {
+		lines.push("\n--- Added ---");
+		for (const e of result.added.slice(0, 20)) {
+			lines.push(`  + [${e.type}] ${e.name}`);
+		}
+		if (result.added.length > 20) lines.push(`  ... and ${result.added.length - 20} more`);
+	}
+
+	if (result.invalidated.length > 0) {
+		lines.push("\n--- Invalidated (superseded) ---");
+		for (const e of result.invalidated.slice(0, 10)) {
+			lines.push(`  ~ [${e.type}] ${e.name}`);
+		}
+	}
+
+	if (result.archived.length > 0) {
+		lines.push("\n--- Archived (decayed) ---");
+		for (const e of result.archived.slice(0, 10)) {
+			lines.push(`  - [${e.type}] ${e.name}`);
+		}
+	}
+
+	return lines.join("\n");
+}
+
+export async function runCompare(args: string[]): Promise<void> {
+	const { resolveRepoHash } = await import("@/capture/hook");
+	const { openGraphDb } = await import("@/graph/semantic-db");
+	const { resolveSiaHome } = await import("@/shared/config");
+
+	let since = Date.now() - 7 * 86400000; // default: last 7 days
+	let until = Date.now();
+
+	for (let i = 0; i < args.length; i++) {
+		if (args[i] === "--since" && args[i + 1]) {
+			since = new Date(args[++i]).getTime();
+		} else if (args[i] === "--until" && args[i + 1]) {
+			until = new Date(args[++i]).getTime();
+		}
+	}
+
+	const cwd = process.cwd();
+	const repoHash = resolveRepoHash(cwd);
+	const siaHome = resolveSiaHome();
+	const db = openGraphDb(repoHash, siaHome);
+
+	try {
+		const result = await compareGraphState(db, since, until);
+		console.log(formatComparison(result, since, until));
+	} finally {
+		await db.close();
+	}
+}