@iowarp/clio-coder 0.3.1 → 0.3.2

package/src/domains/context/codewiki/indexer.ts

@@ -5,74 +5,52 @@ import { dirname, join, relative, resolve } from "node:path";
 import { classifyCHeaderLanguage, isAmbiguousHeaderPath } from "../../../core/c-header-language.js";
 import { safeResourceWrite } from "../../../core/safe-resource-write.js";
 import { enumerateWorkspaceFilesAsync, filterWorkspaceFileCandidates } from "../../../core/workspace-files.js";
-import type { ProjectType, SourceProjectType } from "../../session/workspace/project-type.js";
+import type { ProjectType } from "../../session/workspace/project-type.js";
 import { EXCLUDED_DIRS } from "../excluded-dirs.js";
 import { extractCMake, isCMakePath } from "./cmake.js";
 import { type CooperativeSlicer, createSlicer } from "./cooperative.js";
-import { createTreeSitterExtractor, type TreeSitterExtractor } from "./tree-sitter.js";
+import { isIndexablePath, languageForPath } from "./paths.js";
+import {
+	type BuildCodewikiInput,
+	CODEWIKI_VERSION,
+	type Codewiki,
+	type CodewikiEdge,
+	type CodewikiEntry,
+	type CodewikiExternalEdge,
+	type CodewikiFile,
+	type CodewikiFileRole,
+	type CodewikiInternalEdge,
+	type CodewikiLanguage,
+	type CodewikiReadFile,
+	type CodewikiSymbol,
+	type CodewikiSymbolKind,
+	type ExtractedSymbol,
+	type LanguageExtraction,
+	type LanguageExtractor,
+} from "./schema.js";
+import type { TreeSitterExtractor } from "./tree-sitter.js";
+
+export {
+	type BuildCodewikiInput,
+	CODEWIKI_VERSION,
+	type Codewiki,
+	type CodewikiEdge,
+	type CodewikiEntry,
+	type CodewikiExternalEdge,
+	type CodewikiFile,
+	type CodewikiFileRole,
+	type CodewikiInternalEdge,
+	type CodewikiLanguage,
+	type CodewikiReadFile,
+	type CodewikiSymbol,
+	type CodewikiSymbolKind,
+	type ExtractedSymbol,
+	type LanguageExtraction,
+	type LanguageExtractor,
+} from "./schema.js";
 
-export type CodewikiLanguage = SourceProjectType | "config";
-export type CodewikiFileRole = "entry" | "test" | "module" | "config";
-export type CodewikiSymbolKind = "func" | "class" | "method" | "type" | "const" | "var" | "trait" | "iface";
 const CODEWIKI_SYMBOL_KINDS_WITH_SIG = new Set<CodewikiSymbolKind>(["func", "class", "method", "type"]);
 
-export interface CodewikiFile {
-	id: string;
-	path: string;
-	lang: CodewikiLanguage;
-	loc: number;
-	role: CodewikiFileRole;
-	hash: string;
-	imports: string[];
-	summary?: string;
-}
-
-export interface CodewikiSymbol {
-	name: string;
-	kind: CodewikiSymbolKind;
-	fileId: string;
-	line: number;
-	sig?: string;
-}
-
-export interface CodewikiInternalEdge {
-	fileId: string;
-	toFileId: string;
-}
-
-export interface CodewikiExternalEdge {
-	fileId: string;
-	externalModule: string;
-}
-
-export type CodewikiEdge = CodewikiInternalEdge | CodewikiExternalEdge;
-
-export const CODEWIKI_VERSION = 5 as const;
-
-export interface Codewiki {
-	version: typeof CODEWIKI_VERSION;
-	language: ProjectType;
-	files: CodewikiFile[];
-	symbols: CodewikiSymbol[];
-	edges: CodewikiEdge[];
-}
-
-export interface CodewikiEntry {
-	path: string;
-	exports: string[];
-	imports: string[];
-	kind: "entry-point" | "test" | "module";
-	summary?: string;
-}
-
-export interface BuildCodewikiInput {
-	cwd: string;
-	language: ProjectType;
-	generatedAt?: string;
-}
-
-export type CodewikiReadFile = (path: string) => string | null;
-
 export interface CodewikiBuildOptions {
 	readFile?: CodewikiReadFile;
 	/**
@@ -83,71 +61,15 @@ export interface CodewikiBuildOptions {
 	slicer?: CooperativeSlicer;
 }
 
-export interface ExtractedSymbol {
-	name: string;
-	kind: CodewikiSymbolKind;
-	line: number;
-	sig?: string;
-}
-
-export interface LanguageExtraction {
-	symbols: ExtractedSymbol[];
-	imports: string[];
-}
-
-export interface LanguageExtractor {
-	langs: ReadonlyArray<CodewikiLanguage>;
-	extractImports?(path: string, text: string): string[];
-	extract(path: string, text: string): LanguageExtraction;
-}
-
 let treeSitterExtractorPromise: Promise<TreeSitterExtractor> | null = null;
 
 function loadTreeSitterExtractor(): Promise<TreeSitterExtractor> {
-	treeSitterExtractorPromise ??= createTreeSitterExtractor();
+	treeSitterExtractorPromise ??= import("./tree-sitter.js").then(({ createTreeSitterExtractor }) =>
+		createTreeSitterExtractor(),
+	);
 	return treeSitterExtractorPromise;
 }
 
-const SOURCE_EXTENSIONS = new Map<string, SourceProjectType>([
-	[".ts", "typescript"],
-	[".tsx", "typescript"],
-	[".mts", "typescript"],
-	[".cts", "typescript"],
-	[".js", "javascript"],
-	[".jsx", "javascript"],
-	[".mjs", "javascript"],
-	[".cjs", "javascript"],
-	[".py", "python"],
-	[".pyw", "python"],
-	[".rs", "rust"],
-	[".go", "go"],
-	[".c", "c"],
-	[".h", "c"],
-	[".cc", "c++"],
-	[".cpp", "c++"],
-	[".cxx", "c++"],
-	[".hpp", "c++"],
-	[".hh", "c++"],
-	[".hxx", "c++"],
-	[".cu", "c++"],
-	[".cuh", "c++"],
-	[".java", "java"],
-	[".rb", "ruby"],
-	[".cs", "c#"],
-]);
-
-const CONFIG_FILE_NAMES = new Set([
-	"package.json",
-	"pyproject.toml",
-	"setup.py",
-	"Cargo.toml",
-	"go.mod",
-	"pom.xml",
-	"CMakeLists.txt",
-	"compile_commands.json",
-	"Gemfile",
-]);
-
 const RESOLUTION_EXTENSIONS = [
 	".ts",
 	".tsx",
@@ -195,11 +117,6 @@ function defaultReadFile(path: string): string | null {
 	}
 }
 
-function extensionOf(name: string): string {
-	const index = name.lastIndexOf(".");
-	return index === -1 ? "" : name.slice(index);
-}
-
 function normalizeRel(cwd: string, filePath: string): string {
 	return relative(cwd, filePath).split("\\").join("/");
 }
@@ -208,24 +125,7 @@ function normalizeInputPath(path: string): string {
 	return path.split("\\").join("/").replace(/^\.\//, "");
 }
 
-function sourceLanguageForPath(relPath: string): SourceProjectType | null {
-	if (relPath.endsWith(".d.ts")) return null;
-	return SOURCE_EXTENSIONS.get(extensionOf(relPath)) ?? null;
-}
-
-function languageForPath(relPath: string): CodewikiLanguage | null {
-	const source = sourceLanguageForPath(relPath);
-	if (source) return source;
-	const name = relPath.split("/").pop() ?? relPath;
-	return CONFIG_FILE_NAMES.has(name) || name.endsWith(".csproj") || name.toLowerCase().endsWith(".cmake")
-		? "config"
-		: null;
-}
-
-export function isIndexablePath(relPath: string): boolean {
-	if (relPath.split("/").some((segment) => EXCLUDED_DIRS.has(segment))) return false;
-	return languageForPath(relPath) !== null;
-}
+export { isIndexablePath } from "./paths.js";
 
 function lineCount(text: string): number {
 	if (text.length === 0) return 0;
@@ -856,6 +756,20 @@ export function fallbackExtraction(language: CodewikiLanguage, relPath: string,
 	return extractWithExtractors(fallbackExtractors, language, relPath, text);
 }
 
+/**
+ * Largest source text handed to tree-sitter. The cooperative slicer yields
+ * between files, never inside one, so a single parse is the longest stretch the
+ * event loop can be held for; measured on this machine that is roughly 0.5 ms
+ * per KiB of ordinary source and 2 to 3 ms per KiB of a minified bundle, which
+ * made one 9 MiB vendored bundle a 25 s freeze that no shutdown budget could
+ * interrupt (issue #99). Nothing hand-written is this big; anything above the
+ * cap is generated or vendored, and the regex extractor covers it in tens of
+ * milliseconds. Under the cap one parse of ordinary source is a few hundred
+ * milliseconds at most; a minified file right at the cap can still take about a
+ * second, which is bounded, and the loop gets its turn at the next file.
+ */
+export const MAX_TREE_SITTER_SOURCE_CHARS = 512 * 1024;
+
 function extractSourceFile(
 	language: CodewikiLanguage,
 	relPath: string,
@@ -863,6 +777,7 @@ function extractSourceFile(
 	treeSitterExtractor: LanguageExtractor,
 ): LanguageExtraction {
 	if (!treeSitterExtractor.langs.includes(language)) return fallbackExtraction(language, relPath, text);
+	if (text.length > MAX_TREE_SITTER_SOURCE_CHARS) return fallbackExtraction(language, relPath, text);
 	try {
 		return mergeTreeSitterWithRegexImports(language, relPath, text, treeSitterExtractor.extract(relPath, text));
 	} catch {

package/src/domains/context/codewiki/paths.ts

@@ -0,0 +1,67 @@
+import type { SourceProjectType } from "../../session/workspace/project-type.js";
+import { EXCLUDED_DIRS } from "../excluded-dirs.js";
+import type { CodewikiLanguage } from "./schema.js";
+
+const SOURCE_EXTENSIONS = new Map<string, SourceProjectType>([
+	[".ts", "typescript"],
+	[".tsx", "typescript"],
+	[".mts", "typescript"],
+	[".cts", "typescript"],
+	[".js", "javascript"],
+	[".jsx", "javascript"],
+	[".mjs", "javascript"],
+	[".cjs", "javascript"],
+	[".py", "python"],
+	[".pyw", "python"],
+	[".rs", "rust"],
+	[".go", "go"],
+	[".c", "c"],
+	[".h", "c"],
+	[".cc", "c++"],
+	[".cpp", "c++"],
+	[".cxx", "c++"],
+	[".hpp", "c++"],
+	[".hh", "c++"],
+	[".hxx", "c++"],
+	[".cu", "c++"],
+	[".cuh", "c++"],
+	[".java", "java"],
+	[".rb", "ruby"],
+	[".cs", "c#"],
+]);
+
+const CONFIG_FILE_NAMES = new Set([
+	"package.json",
+	"pyproject.toml",
+	"setup.py",
+	"Cargo.toml",
+	"go.mod",
+	"pom.xml",
+	"CMakeLists.txt",
+	"compile_commands.json",
+	"Gemfile",
+]);
+
+function extensionOf(name: string): string {
+	const index = name.lastIndexOf(".");
+	return index === -1 ? "" : name.slice(index);
+}
+
+function sourceLanguageForPath(relPath: string): SourceProjectType | null {
+	if (relPath.endsWith(".d.ts")) return null;
+	return SOURCE_EXTENSIONS.get(extensionOf(relPath)) ?? null;
+}
+
+export function languageForPath(relPath: string): CodewikiLanguage | null {
+	const source = sourceLanguageForPath(relPath);
+	if (source) return source;
+	const name = relPath.split("/").pop() ?? relPath;
+	return CONFIG_FILE_NAMES.has(name) || name.endsWith(".csproj") || name.toLowerCase().endsWith(".cmake")
+		? "config"
+		: null;
+}
+
+export function isIndexablePath(relPath: string): boolean {
+	if (relPath.split("/").some((segment) => EXCLUDED_DIRS.has(segment))) return false;
+	return languageForPath(relPath) !== null;
+}

package/src/domains/context/codewiki/schema.ts

@@ -0,0 +1,80 @@
+import type { ProjectType, SourceProjectType } from "../../session/workspace/project-type.js";
+
+export type CodewikiLanguage = SourceProjectType | "config";
+export type CodewikiFileRole = "entry" | "test" | "module" | "config";
+export type CodewikiSymbolKind = "func" | "class" | "method" | "type" | "const" | "var" | "trait" | "iface";
+
+export interface CodewikiFile {
+	id: string;
+	path: string;
+	lang: CodewikiLanguage;
+	loc: number;
+	role: CodewikiFileRole;
+	hash: string;
+	imports: string[];
+	summary?: string;
+}
+
+export interface CodewikiSymbol {
+	name: string;
+	kind: CodewikiSymbolKind;
+	fileId: string;
+	line: number;
+	sig?: string;
+}
+
+export interface CodewikiInternalEdge {
+	fileId: string;
+	toFileId: string;
+}
+
+export interface CodewikiExternalEdge {
+	fileId: string;
+	externalModule: string;
+}
+
+export type CodewikiEdge = CodewikiInternalEdge | CodewikiExternalEdge;
+
+export const CODEWIKI_VERSION = 5 as const;
+
+export interface Codewiki {
+	version: typeof CODEWIKI_VERSION;
+	language: ProjectType;
+	files: CodewikiFile[];
+	symbols: CodewikiSymbol[];
+	edges: CodewikiEdge[];
+}
+
+export interface CodewikiEntry {
+	path: string;
+	exports: string[];
+	imports: string[];
+	kind: "entry-point" | "test" | "module";
+	summary?: string;
+}
+
+export interface BuildCodewikiInput {
+	cwd: string;
+	language: ProjectType;
+	generatedAt?: string;
+}
+
+export type CodewikiReadFile = (path: string) => string | null;
+
+export interface ExtractedSymbol {
+	name: string;
+	kind: CodewikiSymbolKind;
+	line: number;
+	sig?: string;
+}
+
+export interface LanguageExtraction {
+	symbols: ExtractedSymbol[];
+	imports: string[];
+}
+
+export interface LanguageExtractor {
+	langs: ReadonlyArray<CodewikiLanguage>;
+	extractImports?(path: string, text: string): string[];
+	extract(path: string, text: string): LanguageExtraction;
+}

package/src/domains/context/codewiki/tree-sitter.ts

@@ -13,7 +13,7 @@ import type {
 	ExtractedSymbol,
 	LanguageExtraction,
 	LanguageExtractor,
-} from "./indexer.js";
+} from "./schema.js";
 
 const { Language, Parser } = TreeSitter;
 

package/src/domains/context/contract.ts

@@ -9,10 +9,16 @@ export interface ProjectPromptContext {
 	text: string;
 	clioMd: ParsedClioMd | null;
 	warnings: string[];
+	/**
+	 * Absolute paths of the effective project handbooks, ancestor to nearest.
+	 * An override resets the inherited chain, so this names exactly the files
+	 * whose content is in `text`; empty when no selected handbook parsed.
+	 */
+	handbookFiles: string[];
 }
 
 /**
- * Structured CLIO-CODER.md fields for bounded injection into worker prompts.
+ * Structured fields from the effective layered project handbook for bounded injection into worker prompts.
  * Deliberately excludes the identity paragraph and any raw handbook prose.
  */
 export interface ProjectStructuredContext {
@@ -20,7 +26,7 @@ export interface ProjectStructuredContext {
 	conventions: ReadonlyArray<string>;
 	invariants: ReadonlyArray<string>;
 	/**
-	 * Body of the CLIO-CODER.md H2 section titled exactly "Verification expectations"
+	 * Bodies of effective handbook H2 sections titled exactly "Verification expectations"
 	 * (case-insensitive). The only custom section ever projected to workers;
 	 * dispatch includes it for verification-class runs only. Absent when the
 	 * handbook has no such section.
@@ -37,7 +43,7 @@ export interface ContextContract extends DomainContract {
 	runBootstrap(input?: RunBootstrapInput): Promise<RunBootstrapResult>;
 	runContextClear(input?: RunContextClearInput): Promise<RunContextClearResult>;
 	/**
-	 * Rebuild the codewiki index and `.clio-coder` state without touching CLIO-CODER.md.
+	 * Rebuild the codewiki index and `.clio-coder` state without touching project handbooks.
 	 * Markdown wiki updates require the explicit refresh input flag.
 	 * Backs `/context refresh` and `clio-coder context refresh`.
 	 */
@@ -45,8 +51,8 @@ export interface ContextContract extends DomainContract {
 	runWikiGenerate(input?: RunWikiGenerateInput): Promise<RunWikiGenerateResult>;
 	renderPromptContext(cwd: string): ProjectPromptContext;
 	/**
-	 * Parsed CLIO-CODER.md structured fields (project name, conventions, invariants)
-	 * or null when CLIO-CODER.md is absent or malformed. Never returns raw handbook
+	 * Effective project handbook fields (project name, conventions, invariants)
+	 * or null when no selected handbook parses. Never returns raw handbook
 	 * text; used by dispatch to give workers bounded project context.
 	 */
 	projectStructuredContext(cwd?: string): ProjectStructuredContext | null;