@rkarim08/sia 1.0.0

package/src/knowledge/discovery.ts

@@ -0,0 +1,238 @@
+// Module: discovery — Priority-ordered documentation file scanner
+
+import { existsSync, readdirSync, statSync } from "node:fs";
+import { dirname, join, relative, resolve } from "node:path";
+import { DISCOVERY_PATTERNS, type DiscoveryPattern, EXCLUDED_DIRS } from "@/knowledge/patterns";
+
+export interface DiscoveredFile {
+	absolutePath: string;
+	relativePath: string;
+	pattern: DiscoveryPattern;
+	packagePath: string | null;
+}
+
+/** Manifest files that indicate a package root. */
+const PACKAGE_MANIFESTS = ["package.json", "Cargo.toml", "go.mod", "pyproject.toml"];
+
+/**
+ * Discover documentation files in a repository root.
+ * Returns files sorted by priority (1 = highest).
+ * Excludes files in EXCLUDED_DIRS and respects .gitignore conceptually
+ * (by excluding common non-project directories).
+ *
+ * Also scans sub-package directories (detected by manifest files) so that
+ * monorepo packages have their documentation discovered automatically.
+ */
+export function discoverDocFiles(repoRoot: string): DiscoveredFile[] {
+	const seen = new Set<string>();
+	const results: DiscoveredFile[] = [];
+
+	const addMatches = (baseDir: string): void => {
+		for (const pattern of DISCOVERY_PATTERNS) {
+			const matches = resolvePattern(repoRoot, baseDir, pattern);
+			for (const file of matches) {
+				if (!seen.has(file.absolutePath)) {
+					seen.add(file.absolutePath);
+					results.push(file);
+				}
+			}
+		}
+	};
+
+	// Scan repo root
+	addMatches(repoRoot);
+
+	// Scan sub-package directories for additional documentation
+	const subPackages = findSubPackageDirs(repoRoot);
+	for (const pkgDir of subPackages) {
+		addMatches(pkgDir);
+	}
+
+	results.sort((a, b) => a.pattern.priority - b.pattern.priority);
+	return results;
+}
+
+/**
+ * Scan a specific subdirectory for documentation files.
+ * Used for JIT discovery when the agent accesses files in a new directory.
+ */
+export function discoverDocFilesInDir(repoRoot: string, subDir: string): DiscoveredFile[] {
+	const absSubDir = resolve(repoRoot, subDir);
+	const seen = new Set<string>();
+	const results: DiscoveredFile[] = [];
+
+	for (const pattern of DISCOVERY_PATTERNS) {
+		const matches = resolvePattern(repoRoot, absSubDir, pattern);
+		for (const file of matches) {
+			if (!seen.has(file.absolutePath)) {
+				seen.add(file.absolutePath);
+				results.push(file);
+			}
+		}
+	}
+
+	results.sort((a, b) => a.pattern.priority - b.pattern.priority);
+	return results;
+}
+
+/**
+ * Recursively find directories containing package manifests within the repo,
+ * excluding the repo root itself and excluded directories.
+ * Limited to a reasonable depth (4 levels) to avoid deep traversals.
+ */
+function findSubPackageDirs(repoRoot: string, maxDepth = 4): string[] {
+	const absRoot = resolve(repoRoot);
+	const dirs: string[] = [];
+
+	function walk(dir: string, depth: number): void {
+		if (depth > maxDepth) return;
+
+		let entries: import("node:fs").Dirent[];
+		try {
+			entries = readdirSync(dir, { withFileTypes: true }) as import("node:fs").Dirent[];
+		} catch {
+			return;
+		}
+
+		for (const entry of entries) {
+			if (!entry.isDirectory()) continue;
+			if (EXCLUDED_DIRS.has(entry.name as string)) continue;
+
+			const subDir = join(dir, entry.name as string);
+
+			// Check if this directory has a package manifest
+			const hasManifest = PACKAGE_MANIFESTS.some((m) => existsSync(join(subDir, m)));
+			if (hasManifest && subDir !== absRoot) {
+				dirs.push(subDir);
+			}
+
+			// Continue walking regardless of manifest presence
+			walk(subDir, depth + 1);
+		}
+	}
+
+	walk(absRoot, 1);
+	return dirs;
+}
+
+/**
+ * Walk up from file's directory to repoRoot, looking for package manifest files.
+ * If found at a directory OTHER than repoRoot, returns the relative path of that
+ * directory. Returns null if file is at root level.
+ */
+function detectPackagePath(filePath: string, repoRoot: string): string | null {
+	const absRoot = resolve(repoRoot);
+	let current = resolve(dirname(filePath));
+
+	while (current.length >= absRoot.length) {
+		// Skip the repo root itself — we only want sub-packages
+		if (current === absRoot) {
+			break;
+		}
+
+		for (const manifest of PACKAGE_MANIFESTS) {
+			if (existsSync(join(current, manifest))) {
+				return relative(absRoot, current);
+			}
+		}
+
+		const parent = dirname(current);
+		if (parent === current) break;
+		current = parent;
+	}
+
+	return null;
+}
+
+/**
+ * Check whether any path segment in a relative path is an excluded directory.
+ */
+function isExcludedPath(relativePath: string): boolean {
+	const segments = relativePath.split("/");
+	return segments.some((seg) => EXCLUDED_DIRS.has(seg));
+}
+
+/**
+ * Resolve a single DiscoveryPattern against a base directory, returning
+ * all matching DiscoveredFile entries.
+ *
+ * Handles two kinds of patterns:
+ * - Direct file paths (no wildcards): e.g. "AGENTS.md", ".claude/CLAUDE.md"
+ * - Single-star globs: e.g. "docs/adr/*.md", ".cursor/rules/*.mdc"
+ */
+function resolvePattern(
+	repoRoot: string,
+	baseDir: string,
+	pattern: DiscoveryPattern,
+): DiscoveredFile[] {
+	const results: DiscoveredFile[] = [];
+	const absRoot = resolve(repoRoot);
+
+	if (!pattern.glob.includes("*")) {
+		// Direct file — check existence
+		const absPath = resolve(baseDir, pattern.glob);
+		if (existsSync(absPath) && isFile(absPath)) {
+			const relPath = relative(absRoot, absPath);
+			if (!isExcludedPath(relPath)) {
+				results.push({
+					absolutePath: absPath,
+					relativePath: relPath,
+					pattern,
+					packagePath: detectPackagePath(absPath, repoRoot),
+				});
+			}
+		}
+	} else {
+		// Glob with wildcard — split into directory part and file filter
+		const starIdx = pattern.glob.indexOf("*");
+		const dirPart = pattern.glob.slice(0, starIdx);
+		const suffix = pattern.glob.slice(starIdx + 1);
+
+		// The directory containing the wildcard
+		const searchDir = resolve(baseDir, dirPart);
+
+		if (existsSync(searchDir) && isDirectory(searchDir)) {
+			let entries: import("node:fs").Dirent[];
+			try {
+				entries = readdirSync(searchDir, { withFileTypes: true }) as import("node:fs").Dirent[];
+			} catch {
+				return results;
+			}
+
+			for (const entry of entries) {
+				if (!entry.isFile()) continue;
+				if (!(entry.name as string).endsWith(suffix)) continue;
+
+				const absPath = join(searchDir, entry.name as string);
+				const relPath = relative(absRoot, absPath);
+
+				if (!isExcludedPath(relPath)) {
+					results.push({
+						absolutePath: absPath,
+						relativePath: relPath,
+						pattern,
+						packagePath: detectPackagePath(absPath, repoRoot),
+					});
+				}
+			}
+		}
+	}
+
+	return results;
+}
+
+function isFile(p: string): boolean {
+	try {
+		return statSync(p).isFile();
+	} catch {
+		return false;
+	}
+}
+
+function isDirectory(p: string): boolean {
+	try {
+		return statSync(p).isDirectory();
+	} catch {
+		return false;
+	}
+}

package/src/knowledge/external-refs.ts

@@ -0,0 +1,98 @@
+/**
+ * External Reference Detection — detect external service URLs in documentation content.
+ *
+ * Scans text for URLs matching known external service patterns (Notion, Confluence,
+ * Jira, GitHub, etc.) and classifies them by service type.
+ *
+ * Security: external links are NEVER auto-followed. This module only detects
+ * and classifies URLs — it performs no network requests.
+ */
+
+export interface ExternalServicePattern {
+	pattern: RegExp;
+	service: string;
+}
+
+export interface DetectedExternalRef {
+	url: string;
+	service: string;
+	lineNumber: number;
+}
+
+// ---------------------------------------------------------------------------
+// Known external service patterns
+// ---------------------------------------------------------------------------
+
+export const EXTERNAL_SERVICE_PATTERNS: ExternalServicePattern[] = [
+	{ pattern: /notion\.so\//, service: "notion" },
+	{ pattern: /[\w-]+\.atlassian\.net\/wiki\//, service: "confluence" },
+	{ pattern: /docs\.google\.com\//, service: "google-docs" },
+	{ pattern: /[\w-]+\.atlassian\.net\/browse\//, service: "jira" },
+	{ pattern: /linear\.app\//, service: "linear" },
+	{ pattern: /figma\.com\//, service: "figma" },
+	{ pattern: /miro\.com\//, service: "miro" },
+	{ pattern: /github\.com\/[\w-]+\/[\w-]+\/wiki/, service: "github-wiki" },
+	{ pattern: /github\.com\/[\w-]+\/[\w-]+\/issues\/\d+/, service: "github-issue" },
+	{ pattern: /github\.com\/[\w-]+\/[\w-]+\/pull\/\d+/, service: "github-pr" },
+	{ pattern: /stackoverflow\.com\/questions\/\d+/, service: "stackoverflow" },
+];
+
+// ---------------------------------------------------------------------------
+// URL extraction regex
+// ---------------------------------------------------------------------------
+
+const URL_REGEX = /https?:\/\/[^\s)>\]"']+/g;
+
+// ---------------------------------------------------------------------------
+// Public API
+// ---------------------------------------------------------------------------
+
+/**
+ * Check if a URL matches any known external service pattern.
+ * Returns the service name if matched, null otherwise.
+ */
+export function classifyUrl(url: string): string | null {
+	for (const entry of EXTERNAL_SERVICE_PATTERNS) {
+		if (entry.pattern.test(url)) {
+			return entry.service;
+		}
+	}
+	return null;
+}
+
+/**
+ * Detect external service URLs in text content.
+ * Does NOT follow or fetch any URLs — only detects and classifies them.
+ *
+ * Scans each line for URLs matching known external service patterns.
+ * Returns all detected references with their line numbers and service types.
+ */
+export function detectExternalRefs(content: string): DetectedExternalRef[] {
+	if (!content || content.trim().length === 0) {
+		return [];
+	}
+
+	const refs: DetectedExternalRef[] = [];
+	const lines = content.split("\n");
+
+	for (let i = 0; i < lines.length; i++) {
+		const line = lines[i];
+		const urls = line.match(URL_REGEX);
+		if (!urls) {
+			continue;
+		}
+
+		for (const url of urls) {
+			const service = classifyUrl(url);
+			if (service !== null) {
+				refs.push({
+					url,
+					service,
+					lineNumber: i + 1,
+				});
+			}
+		}
+	}
+
+	return refs;
+}

package/src/knowledge/freshness.ts

@@ -0,0 +1,221 @@
+// Module: freshness — Documentation freshness tracking via git metadata
+
+import { execSync } from "node:child_process";
+import type { SiaDb } from "@/graph/db-interface";
+import { updateEntity } from "@/graph/entities";
+
+export interface FreshnessResult {
+	entityId: string;
+	filePath: string;
+	docModifiedAt: number | null;
+	codeModifiedAt: number | null;
+	divergenceDays: number | null;
+	isStale: boolean;
+}
+
+export interface FreshnessConfig {
+	divergenceThreshold: number;
+	freshnessPenalty: number;
+}
+
+export const DEFAULT_FRESHNESS_CONFIG: FreshnessConfig = {
+	divergenceThreshold: 90,
+	freshnessPenalty: 0.15,
+};
+
+/** Documentation tags that identify FileNode entities as documentation. */
+const DOC_TAGS = ["ai-context", "architecture", "project-docs", "api-docs", "changelog"];
+
+/**
+ * Get the last git modification date for a file.
+ * Returns Unix milliseconds or null if not tracked.
+ *
+ * Note: Uses execSync with git CLI. The filePath argument comes from Sia's
+ * internal graph database (entity file_paths column), not from external user
+ * input, so shell injection risk is minimal. The path is quoted in the command.
+ */
+export function getGitModifiedAt(repoRoot: string, filePath: string): number | null {
+	try {
+		const output = execSync(`git log -1 --format=%at -- "${filePath}"`, {
+			cwd: repoRoot,
+			encoding: "utf-8",
+			timeout: 5000,
+		}).trim();
+		if (!output) return null;
+		return Number.parseInt(output, 10) * 1000;
+	} catch {
+		return null;
+	}
+}
+
+/**
+ * Check freshness of a documentation FileNode by comparing its last git modification
+ * date to the most recent modification of code files it references.
+ *
+ * Does NOT modify the database — returns freshness info for the caller to act on.
+ */
+export function checkDocFreshness(
+	repoRoot: string,
+	docFilePath: string,
+	referencedFilePaths: string[],
+	config?: FreshnessConfig,
+): FreshnessResult {
+	const cfg = config ?? DEFAULT_FRESHNESS_CONFIG;
+	const docModifiedAt = getGitModifiedAt(repoRoot, docFilePath);
+
+	if (docModifiedAt === null || referencedFilePaths.length === 0) {
+		return {
+			entityId: "",
+			filePath: docFilePath,
+			docModifiedAt,
+			codeModifiedAt: null,
+			divergenceDays: null,
+			isStale: false,
+		};
+	}
+
+	let latestCodeModified: number | null = null;
+	for (const codePath of referencedFilePaths) {
+		const ts = getGitModifiedAt(repoRoot, codePath);
+		if (ts !== null && (latestCodeModified === null || ts > latestCodeModified)) {
+			latestCodeModified = ts;
+		}
+	}
+
+	if (latestCodeModified === null) {
+		return {
+			entityId: "",
+			filePath: docFilePath,
+			docModifiedAt,
+			codeModifiedAt: null,
+			divergenceDays: null,
+			isStale: false,
+		};
+	}
+
+	const msPerDay = 86_400_000;
+	const divergenceDays = (latestCodeModified - docModifiedAt) / msPerDay;
+	const isStale = divergenceDays > 0 && divergenceDays > cfg.divergenceThreshold;
+
+	return {
+		entityId: "",
+		filePath: docFilePath,
+		docModifiedAt,
+		codeModifiedAt: latestCodeModified,
+		divergenceDays,
+		isStale,
+	};
+}
+
+/**
+ * Run freshness check on all documentation FileNodes in the graph that have
+ * reference edges to code entities.
+ *
+ * Returns freshness results for all checked documents.
+ * Optionally applies freshness penalty to stale documents.
+ */
+export async function checkAllDocFreshness(
+	db: SiaDb,
+	repoRoot: string,
+	config?: FreshnessConfig,
+	opts?: { applyPenalty?: boolean },
+): Promise<FreshnessResult[]> {
+	const cfg = config ?? DEFAULT_FRESHNESS_CONFIG;
+
+	// Build the LIKE conditions for documentation tags
+	const tagConditions = DOC_TAGS.map((tag) => `tags LIKE '%${tag}%'`).join(" OR ");
+
+	const docRows = await db.execute(
+		`SELECT id, file_paths, tags, importance FROM graph_nodes
+		 WHERE type = 'FileNode'
+		   AND (${tagConditions})
+		   AND t_valid_until IS NULL AND archived_at IS NULL`,
+	);
+
+	const results: FreshnessResult[] = [];
+
+	for (const row of docRows.rows) {
+		const entityId = row.id as string;
+		const filePaths = parseJsonArray(row.file_paths as string);
+		const importance = row.importance as number;
+
+		if (filePaths.length === 0) continue;
+
+		const docFilePath = filePaths[0] as string;
+
+		// Find referenced code files via edges
+		const referencedPaths = await getReferencedCodePaths(db, entityId);
+
+		const result = checkDocFreshness(repoRoot, docFilePath, referencedPaths, cfg);
+		result.entityId = entityId;
+
+		if (opts?.applyPenalty && result.isStale) {
+			const currentTags = parseJsonArray(row.tags as string);
+			if (!currentTags.includes("potentially-stale")) {
+				currentTags.push("potentially-stale");
+			}
+			const newImportance = Math.max(0.01, importance - cfg.freshnessPenalty);
+			await updateEntity(db, entityId, {
+				tags: JSON.stringify(currentTags),
+				importance: newImportance,
+			});
+		}
+
+		results.push(result);
+	}
+
+	return results;
+}
+
+/**
+ * Find all code file paths referenced by a documentation entity via edges.
+ * Includes both direct edges and edges from ContentChunk children.
+ */
+async function getReferencedCodePaths(db: SiaDb, docEntityId: string): Promise<string[]> {
+	// Direct edges from the doc entity to code entities
+	const directResult = await db.execute(
+		`SELECT DISTINCT e2.file_paths FROM graph_edges ed
+		 JOIN graph_nodes e2 ON e2.id = ed.to_id
+		 WHERE ed.from_id = ? AND ed.t_valid_until IS NULL
+		   AND e2.type IN ('CodeEntity', 'FileNode')
+		   AND e2.t_valid_until IS NULL AND e2.archived_at IS NULL`,
+		[docEntityId],
+	);
+
+	// Edges from ContentChunk children of this doc entity
+	const chunkResult = await db.execute(
+		`SELECT DISTINCT e3.file_paths FROM graph_edges parent_edge
+		 JOIN graph_nodes chunk ON chunk.id = parent_edge.to_id
+		 JOIN graph_edges child_edge ON child_edge.from_id = chunk.id
+		 JOIN graph_nodes e3 ON e3.id = child_edge.to_id
+		 WHERE parent_edge.from_id = ? AND parent_edge.t_valid_until IS NULL
+		   AND chunk.type = 'ContentChunk'
+		   AND chunk.t_valid_until IS NULL AND chunk.archived_at IS NULL
+		   AND child_edge.t_valid_until IS NULL
+		   AND e3.type IN ('CodeEntity', 'FileNode')
+		   AND e3.t_valid_until IS NULL AND e3.archived_at IS NULL`,
+		[docEntityId],
+	);
+
+	const paths = new Set<string>();
+
+	for (const row of [...directResult.rows, ...chunkResult.rows]) {
+		const filePaths = parseJsonArray(row.file_paths as string);
+		for (const p of filePaths) {
+			paths.add(p);
+		}
+	}
+
+	return [...paths];
+}
+
+/** Safely parse a JSON string array, returning [] on failure. */
+function parseJsonArray(json: string): string[] {
+	try {
+		const parsed: unknown = JSON.parse(json);
+		if (Array.isArray(parsed)) return parsed as string[];
+		return [];
+	} catch {
+		return [];
+	}
+}