@rkarim08/sia 1.0.0

package/src/knowledge/ingest.ts

@@ -0,0 +1,330 @@
+// Module: ingest — Heading-based markdown chunking and graph ingestion
+
+import { readFileSync } from "node:fs";
+import { basename } from "node:path";
+import type { SiaDb } from "@/graph/db-interface";
+import { insertEdge } from "@/graph/edges";
+import { insertEntity } from "@/graph/entities";
+
+/** A chunk extracted from a markdown document. */
+export interface DocChunk {
+	heading: string;
+	headingLevel: number;
+	headingPath: string[];
+	content: string;
+	codeBlocks: CodeBlock[];
+	internalLinks: InternalLink[];
+}
+
+export interface CodeBlock {
+	language: string;
+	code: string;
+}
+
+export interface InternalLink {
+	text: string;
+	target: string;
+	isAnchor: boolean;
+}
+
+export interface IngestResult {
+	fileNodeId: string;
+	chunksCreated: number;
+	edgesCreated: number;
+}
+
+// ---------------------------------------------------------------
+// Markdown parsing
+// ---------------------------------------------------------------
+
+/**
+ * Parse markdown content into heading-based chunks.
+ * Splits at heading boundaries (#, ##, ###), preserving heading hierarchy.
+ * Code blocks and lists are kept intact within their heading-scoped chunks.
+ */
+export function parseMarkdown(content: string): DocChunk[] {
+	const lines = content.split("\n");
+	const chunks: DocChunk[] = [];
+
+	// State for the current chunk being built
+	let currentHeading = "";
+	let currentLevel = 0;
+	let headingPath: string[] = [];
+	let contentLines: string[] = [];
+	let inCodeFence = false;
+
+	function flushChunk(): void {
+		const body = contentLines.join("\n");
+		// Only emit a chunk when there is a heading or non-empty content
+		if (currentHeading !== "" || body.trim().length > 0) {
+			chunks.push({
+				heading: currentHeading,
+				headingLevel: currentLevel,
+				headingPath: [...headingPath],
+				content: body.trimEnd(),
+				codeBlocks: extractCodeBlocks(body),
+				internalLinks: extractInternalLinks(body),
+			});
+		}
+		contentLines = [];
+	}
+
+	for (const line of lines) {
+		// Track code fences — content inside fences is never treated as headings
+		if (line.trimStart().startsWith("```")) {
+			inCodeFence = !inCodeFence;
+			contentLines.push(line);
+			continue;
+		}
+
+		if (inCodeFence) {
+			contentLines.push(line);
+			continue;
+		}
+
+		// Detect ATX-style headings: # H1, ## H2, ### H3 etc.
+		const headingMatch = line.match(/^(#{1,6})\s+(.+)$/);
+		if (headingMatch) {
+			// Save the previous chunk before starting a new one
+			flushChunk();
+
+			const level = headingMatch[1].length;
+			const heading = headingMatch[2].trim();
+
+			// Update heading path: keep entries up to the parent level, then add current
+			headingPath = headingPath.filter((_, i) => i < level - 1);
+			// Ensure path length matches: fill with empty if there are gaps
+			while (headingPath.length < level - 1) {
+				headingPath.push("");
+			}
+			headingPath[level - 1] = heading;
+			headingPath = headingPath.slice(0, level);
+
+			currentHeading = heading;
+			currentLevel = level;
+			continue;
+		}
+
+		contentLines.push(line);
+	}
+
+	// Flush the last chunk
+	flushChunk();
+
+	return chunks;
+}
+
+/**
+ * Extract fenced code blocks from markdown content.
+ * Recognises ```language ... ``` patterns.
+ */
+function extractCodeBlocks(content: string): CodeBlock[] {
+	const blocks: CodeBlock[] = [];
+	const lines = content.split("\n");
+	let inBlock = false;
+	let language = "";
+	let codeLines: string[] = [];
+
+	for (const line of lines) {
+		if (!inBlock && line.trimStart().startsWith("```")) {
+			inBlock = true;
+			language = line.trimStart().slice(3).trim();
+			codeLines = [];
+			continue;
+		}
+		if (inBlock && line.trimStart().startsWith("```")) {
+			blocks.push({ language, code: codeLines.join("\n") });
+			inBlock = false;
+			language = "";
+			codeLines = [];
+			continue;
+		}
+		if (inBlock) {
+			codeLines.push(line);
+		}
+	}
+
+	return blocks;
+}
+
+/**
+ * Extract internal links (markdown link syntax) from content.
+ * Internal links are those whose target is a relative path or an anchor (#).
+ * Absolute URLs (http://, https://) are excluded.
+ */
+function extractInternalLinks(content: string): InternalLink[] {
+	const links: InternalLink[] = [];
+	const linkRegex = /\[([^\]]+)\]\(([^)]+)\)/g;
+	let match = linkRegex.exec(content);
+
+	while (match !== null) {
+		const text = match[1];
+		const target = match[2];
+
+		// Skip absolute URLs
+		if (!/^https?:\/\//.test(target)) {
+			links.push({
+				text,
+				target,
+				isAnchor: target.startsWith("#"),
+			});
+		}
+
+		match = linkRegex.exec(content);
+	}
+
+	return links;
+}
+
+// ---------------------------------------------------------------
+// Frontmatter parsing
+// ---------------------------------------------------------------
+
+/**
+ * Parse YAML frontmatter from markdown content.
+ * Returns the frontmatter as key-value pairs and the remaining content.
+ */
+export function parseFrontmatter(content: string): {
+	frontmatter: Record<string, string>;
+	body: string;
+} {
+	const frontmatter: Record<string, string> = {};
+
+	if (!content.startsWith("---\n") && !content.startsWith("---\r\n")) {
+		return { frontmatter, body: content };
+	}
+
+	// Find the closing delimiter
+	const endIdx = content.indexOf("\n---", 4);
+	if (endIdx === -1) {
+		return { frontmatter, body: content };
+	}
+
+	const yamlBlock = content.slice(4, endIdx);
+	// Skip past the closing ---\n
+	const bodyStart = content.indexOf("\n", endIdx + 1);
+	const body = bodyStart === -1 ? "" : content.slice(bodyStart + 1);
+
+	// Simple key: value parser (no nested YAML)
+	for (const line of yamlBlock.split("\n")) {
+		const trimmed = line.trim();
+		if (trimmed === "" || trimmed.startsWith("#")) continue;
+
+		const colonIdx = trimmed.indexOf(":");
+		if (colonIdx === -1) continue;
+
+		const key = trimmed.slice(0, colonIdx).trim();
+		const value = trimmed.slice(colonIdx + 1).trim();
+		if (key) {
+			frontmatter[key] = value;
+		}
+	}
+
+	return { frontmatter, body };
+}
+
+// ---------------------------------------------------------------
+// Graph ingestion
+// ---------------------------------------------------------------
+
+/**
+ * Ingest a documentation file into the knowledge graph.
+ *
+ * 1. Create or reuse a FileNode entity for the file
+ * 2. Parse the markdown into chunks
+ * 3. Create ContentChunk entities for each chunk
+ * 4. Create child_of edges from chunks to the FileNode
+ * 5. Resolve internal links to references edges
+ */
+export async function ingestDocument(
+	db: SiaDb,
+	filePath: string,
+	relativePath: string,
+	opts?: {
+		tag?: string;
+		trustTier?: 1 | 2;
+		packagePath?: string | null;
+	},
+): Promise<IngestResult> {
+	const raw = readFileSync(filePath, "utf-8");
+	const { frontmatter, body } = parseFrontmatter(raw);
+	const chunks = parseMarkdown(body);
+
+	const tag = opts?.tag ?? "project-docs";
+	const trustTier = opts?.trustTier ?? 1;
+	const packagePath = opts?.packagePath ?? null;
+	const fileName = basename(relativePath);
+
+	let edgesCreated = 0;
+
+	// ---- Step 1: Find or create FileNode ----
+	let fileNodeId: string;
+
+	const existing = await db.execute(
+		"SELECT id FROM graph_nodes WHERE type = 'FileNode' AND file_paths LIKE ? AND t_valid_until IS NULL AND archived_at IS NULL",
+		[`%"${relativePath}"%`],
+	);
+
+	if (existing.rows.length > 0) {
+		fileNodeId = existing.rows[0].id as string;
+	} else {
+		const summary = frontmatter.description ?? frontmatter.title ?? raw.slice(0, 200).trim();
+
+		const fileNode = await insertEntity(db, {
+			type: "FileNode",
+			name: fileName,
+			content: summary,
+			summary: `Documentation file: ${relativePath}`,
+			package_path: packagePath,
+			tags: JSON.stringify([tag]),
+			file_paths: JSON.stringify([relativePath]),
+			trust_tier: trustTier,
+			confidence: 1.0,
+			extraction_method: "document-ingest",
+		});
+		fileNodeId = fileNode.id;
+	}
+
+	// ---- Step 2-4: Create ContentChunk entities and child_of edges ----
+	let chunksCreated = 0;
+
+	for (const chunk of chunks) {
+		const chunkName = chunk.heading !== "" ? chunk.heading : `${fileName} - Introduction`;
+
+		const contentPreview = chunk.content.slice(0, 150).trim();
+
+		const chunkTags: string[] = [tag];
+		if (chunk.headingLevel > 0) {
+			chunkTags.push(`h${chunk.headingLevel}`);
+		}
+
+		const chunkEntity = await insertEntity(db, {
+			type: "ContentChunk",
+			name: chunkName,
+			content: chunk.content,
+			summary: contentPreview,
+			package_path: packagePath,
+			tags: JSON.stringify(chunkTags),
+			file_paths: JSON.stringify([relativePath]),
+			trust_tier: trustTier,
+			confidence: 1.0,
+			extraction_method: "document-ingest",
+		});
+
+		// child_of edge: chunk -> FileNode
+		await insertEdge(db, {
+			from_id: chunkEntity.id,
+			to_id: fileNodeId,
+			type: "child_of",
+			extraction_method: "document-ingest",
+		});
+		edgesCreated++;
+		chunksCreated++;
+	}
+
+	return {
+		fileNodeId,
+		chunksCreated,
+		edgesCreated,
+	};
+}

package/src/knowledge/markdown-export.ts

@@ -0,0 +1,229 @@
+// Module: markdown-export — Export knowledge graph as markdown vault
+
+import { mkdirSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import type { SiaDb } from "@/graph/db-interface";
+
+export interface MarkdownExportOpts {
+	outputDir: string;
+	types?: string[];
+	includeCode?: boolean;
+}
+
+export interface MarkdownExportResult {
+	filesWritten: number;
+	entitiesExported: number;
+	outputDir: string;
+}
+
+/** Default semantic entity types to export (excludes CodeEntity and FileNode). */
+const DEFAULT_TYPES = ["Decision", "Convention", "Bug", "Solution", "Concept"];
+
+/** Related entity info resolved from edges. */
+interface RelatedEntity {
+	edgeType: string;
+	id: string;
+	entityType: string;
+	name: string;
+}
+
+/**
+ * Convert entity type to directory name.
+ * "Decision" -> "decisions", "CodeEntity" -> "code", etc.
+ */
+function typeToDir(type: string): string {
+	if (type === "CodeEntity") return "code";
+	if (type === "FileNode") return "files";
+	return `${type.toLowerCase()}s`;
+}
+
+/**
+ * Slugify a name for use as a filename.
+ * Lowercases, replaces whitespace/underscores with hyphens,
+ * strips non-alphanumeric characters (except hyphens), and collapses
+ * multiple consecutive hyphens.
+ */
+export function slugify(name: string): string {
+	return name
+		.toLowerCase()
+		.replace(/[\s_]+/g, "-")
+		.replace(/[^a-z0-9-]/g, "")
+		.replace(/-{2,}/g, "-")
+		.replace(/^-+|-+$/g, "");
+}
+
+/**
+ * Format a millisecond epoch timestamp as an ISO 8601 date string.
+ * Returns "unknown" if the timestamp is null or undefined.
+ */
+function toIsoDate(ts: number | null | undefined): string {
+	if (ts == null || ts === 0) return "unknown";
+	return new Date(ts).toISOString();
+}
+
+/**
+ * Parse a JSON tags string into an array of strings.
+ * Returns empty array for any parse failure.
+ */
+function parseTags(tags: unknown): string[] {
+	if (typeof tags !== "string") return [];
+	try {
+		const parsed = JSON.parse(tags);
+		return Array.isArray(parsed) ? parsed.map(String) : [];
+	} catch {
+		return [];
+	}
+}
+
+/**
+ * Escape YAML string values that may contain special characters.
+ * Wraps in double quotes if necessary.
+ */
+function yamlString(value: string): string {
+	if (/[:#{}[\],&*?|>!%@`]/.test(value) || value.includes('"') || value.includes("'")) {
+		return `"${value.replace(/\\/g, "\\\\").replace(/"/g, '\\"')}"`;
+	}
+	return value;
+}
+
+/**
+ * Export knowledge graph entities as markdown files organized by type.
+ * Each entity becomes a markdown file with YAML frontmatter.
+ * Wikilinks connect related entities for Obsidian compatibility.
+ */
+export async function exportAsMarkdown(
+	db: SiaDb,
+	opts: MarkdownExportOpts,
+): Promise<MarkdownExportResult> {
+	const types = opts.types ?? DEFAULT_TYPES;
+	const includeCode = opts.includeCode ?? false;
+
+	// Build the effective type list
+	const effectiveTypes = [...types];
+	if (includeCode && !effectiveTypes.includes("CodeEntity")) {
+		effectiveTypes.push("CodeEntity");
+	}
+
+	// Query active entities filtered by type
+	const placeholders = effectiveTypes.map(() => "?").join(", ");
+	const { rows: entityRows } = await db.execute(
+		`SELECT id, type, name, content, summary, importance, trust_tier, tags, created_at, t_valid_from
+		 FROM graph_nodes
+		 WHERE t_valid_until IS NULL AND archived_at IS NULL
+		   AND type IN (${placeholders})
+		 ORDER BY type, importance DESC`,
+		effectiveTypes,
+	);
+
+	// Create output directory structure
+	mkdirSync(opts.outputDir, { recursive: true });
+	const dirsCreated = new Set<string>();
+	for (const t of effectiveTypes) {
+		const dir = join(opts.outputDir, typeToDir(t));
+		mkdirSync(dir, { recursive: true });
+		dirsCreated.add(dir);
+	}
+
+	// For each entity, resolve related entities and write markdown file
+	let filesWritten = 0;
+	const typeCounts = new Map<string, number>();
+
+	for (const row of entityRows) {
+		const entity = row as Record<string, unknown>;
+		const entityId = entity.id as string;
+		const entityType = entity.type as string;
+		const entityName = entity.name as string;
+		const entityContent = entity.content as string;
+		const trustTier = entity.trust_tier as number;
+		const createdAt = entity.created_at as number | null;
+		const tags = parseTags(entity.tags);
+		const importance = entity.importance as number;
+
+		// Track type counts for index
+		typeCounts.set(entityType, (typeCounts.get(entityType) ?? 0) + 1);
+
+		// Resolve related entities via outgoing edges
+		const { rows: relatedRows } = await db.execute(
+			`SELECT e.type AS edge_type, ent.id, ent.type AS entity_type, ent.name
+			 FROM graph_edges e
+			 JOIN graph_nodes ent ON ent.id = e.to_id
+			 WHERE e.from_id = ? AND e.t_valid_until IS NULL
+			   AND ent.t_valid_until IS NULL AND ent.archived_at IS NULL`,
+			[entityId],
+		);
+
+		const related: RelatedEntity[] = (relatedRows as Record<string, unknown>[]).map((r) => ({
+			edgeType: r.edge_type as string,
+			id: r.id as string,
+			entityType: r.entity_type as string,
+			name: r.name as string,
+		}));
+
+		// Build YAML frontmatter
+		const tagsYaml = tags.length > 0 ? `[${tags.map((t) => yamlString(t)).join(", ")}]` : "[]";
+
+		const lines: string[] = [
+			"---",
+			`id: ${yamlString(entityId)}`,
+			`kind: ${entityType}`,
+			`trust_tier: ${trustTier}`,
+			`created_at: ${yamlString(toIsoDate(createdAt))}`,
+			`tags: ${tagsYaml}`,
+			`importance: ${importance}`,
+			"---",
+			"",
+			`# ${entityName}`,
+			"",
+			entityContent,
+		];
+
+		// Add related section if there are linked entities
+		if (related.length > 0) {
+			lines.push("", "## Related", "");
+			for (const rel of related) {
+				const targetDir = typeToDir(rel.entityType);
+				const targetSlug = slugify(rel.name);
+				lines.push(`- ${rel.edgeType}: [[${targetDir}/${targetSlug}]]`);
+			}
+		}
+
+		lines.push(""); // trailing newline
+
+		// Write file
+		const dir = typeToDir(entityType);
+		const filename = `${slugify(entityName)}.md`;
+		const filePath = join(opts.outputDir, dir, filename);
+		writeFileSync(filePath, lines.join("\n"), "utf-8");
+		filesWritten++;
+	}
+
+	// Generate index.md
+	const indexLines: string[] = [
+		"# Sia Knowledge Graph Export",
+		"",
+		`**Exported at:** ${new Date().toISOString()}`,
+		`**Entities:** ${entityRows.length}`,
+		"",
+		"## Summary",
+		"",
+		"| Type | Count |",
+		"|------|-------|",
+	];
+
+	for (const t of effectiveTypes) {
+		const count = typeCounts.get(t) ?? 0;
+		if (count > 0) {
+			indexLines.push(`| ${t} | ${count} |`);
+		}
+	}
+
+	indexLines.push(""); // trailing newline
+	writeFileSync(join(opts.outputDir, "index.md"), indexLines.join("\n"), "utf-8");
+	filesWritten++;
+
+	return {
+		filesWritten,
+		entitiesExported: entityRows.length,
+		outputDir: opts.outputDir,
+	};
+}