@clinebot/core 0.0.21 → 0.0.23

package/src/input/file-indexer.ts

@@ -0,0 +1,327 @@
+import { spawn } from "node:child_process";
+import { readdir } from "node:fs/promises";
+import path from "node:path";
+import { isMainThread, parentPort, Worker } from "node:worker_threads";
+
+const DEFAULT_INDEX_TTL_MS = 15_000;
+const STALE_CACHE_EVICTION_MS = 10 * 60_000;
+const WORKER_INDEX_REQUEST_TIMEOUT_MS = 1_000;
+const DEFAULT_EXCLUDE_DIRS = new Set([
+	".git",
+	"node_modules",
+	"dist",
+	"build",
+	".next",
+	"coverage",
+	".turbo",
+	".cache",
+	"target",
+	"out",
+]);
+
+interface CacheEntry {
+	files: Set<string>;
+	lastBuiltAt: number;
+	lastAccessedAt: number;
+	pending: Promise<Set<string>> | null;
+}
+
+export interface FastFileIndexOptions {
+	ttlMs?: number;
+}
+
+interface IndexRequestMessage {
+	type: "index";
+	requestId: number;
+	cwd: string;
+}
+
+interface IndexResponseMessage {
+	type: "indexResult";
+	requestId: number;
+	files?: string[];
+	error?: string;
+}
+
+const CACHE = new Map<string, CacheEntry>();
+
+function pruneStaleCacheEntries(now: number): void {
+	if (CACHE.size <= 1) {
+		return;
+	}
+	for (const [cwd, entry] of CACHE.entries()) {
+		if (entry.pending) {
+			continue;
+		}
+		if (now - entry.lastAccessedAt > STALE_CACHE_EVICTION_MS) {
+			CACHE.delete(cwd);
+		}
+	}
+}
+
+function toPosixRelative(cwd: string, absolutePath: string): string {
+	return path.relative(cwd, absolutePath).split(path.sep).join("/");
+}
+
+async function listFilesWithRg(cwd: string): Promise<Set<string>> {
+	const output = await new Promise<string>((resolve, reject) => {
+		const child = spawn("rg", ["--files", "--hidden", "-g", "!.git"], {
+			cwd,
+			stdio: ["ignore", "pipe", "pipe"],
+		});
+
+		let stdout = "";
+		let stderr = "";
+
+		child.stdout.on("data", (chunk: Buffer | string) => {
+			stdout += chunk.toString();
+		});
+		child.stderr.on("data", (chunk: Buffer | string) => {
+			stderr += chunk.toString();
+		});
+		child.on("error", reject);
+		child.on("close", (code: number | null) => {
+			if (code === 0) {
+				resolve(stdout);
+				return;
+			}
+			reject(new Error(stderr || `rg exited with code ${code}`));
+		});
+	});
+
+	const files = output
+		.split(/\r?\n/)
+		.map((line) => line.trim())
+		.filter((line) => line.length > 0)
+		.map((line) => line.replace(/\\/g, "/"));
+
+	return new Set(files);
+}
+
+async function walkDir(
+	cwd: string,
+	dir: string,
+	files: Set<string>,
+): Promise<void> {
+	const entries = await readdir(dir, { withFileTypes: true });
+	for (const entry of entries) {
+		const absolutePath = path.join(dir, entry.name);
+		if (entry.isDirectory()) {
+			if (DEFAULT_EXCLUDE_DIRS.has(entry.name)) {
+				continue;
+			}
+			await walkDir(cwd, absolutePath, files);
+			continue;
+		}
+		if (entry.isFile()) {
+			files.add(toPosixRelative(cwd, absolutePath));
+		}
+	}
+}
+
+async function listFilesFallback(cwd: string): Promise<Set<string>> {
+	const files = new Set<string>();
+	await walkDir(cwd, cwd, files);
+	return files;
+}
+
+async function buildIndex(cwd: string): Promise<Set<string>> {
+	try {
+		return await listFilesWithRg(cwd);
+	} catch {
+		return listFilesFallback(cwd);
+	}
+}
+
+function startWorkerServer(): void {
+	if (isMainThread || !parentPort) {
+		return;
+	}
+	const port = parentPort;
+
+	port.on("message", (message: IndexRequestMessage) => {
+		if (message.type !== "index") {
+			return;
+		}
+
+		void buildIndex(message.cwd)
+			.then((files) => {
+				const response: IndexResponseMessage = {
+					type: "indexResult",
+					requestId: message.requestId,
+					files: Array.from(files),
+				};
+				port.postMessage(response);
+			})
+			.catch((error: unknown) => {
+				const response: IndexResponseMessage = {
+					type: "indexResult",
+					requestId: message.requestId,
+					error:
+						error instanceof Error
+							? error.message
+							: "Failed to build file index",
+				};
+				port.postMessage(response);
+			});
+	});
+}
+
+class FileIndexWorkerClient {
+	private readonly worker = new Worker(new URL(import.meta.url));
+	private nextRequestId = 0;
+	private pending = new Map<
+		number,
+		{
+			resolve: (files: string[]) => void;
+			reject: (reason: Error) => void;
+		}
+	>();
+
+	constructor() {
+		// Keep indexing opportunistic: this worker should never block process exit.
+		this.worker.unref();
+		this.worker.on("message", (message: IndexResponseMessage) => {
+			if (message.type !== "indexResult") {
+				return;
+			}
+			const request = this.pending.get(message.requestId);
+			if (!request) {
+				return;
+			}
+			this.pending.delete(message.requestId);
+			if (message.error) {
+				request.reject(new Error(message.error));
+				return;
+			}
+			request.resolve(message.files ?? []);
+		});
+
+		this.worker.on("error", (error: Error) => {
+			this.flushPending(error);
+		});
+
+		this.worker.on("exit", (code) => {
+			if (code !== 0) {
+				this.flushPending(
+					new Error(`File index worker exited with code ${code}`),
+				);
+			}
+		});
+	}
+
+	requestIndex(cwd: string): Promise<string[]> {
+		const requestId = ++this.nextRequestId;
+		const result = new Promise<string[]>((resolve, reject) => {
+			const timeout = setTimeout(() => {
+				this.pending.delete(requestId);
+				reject(new Error("Timed out waiting for file index worker response"));
+			}, WORKER_INDEX_REQUEST_TIMEOUT_MS);
+			timeout.unref();
+			this.pending.set(requestId, {
+				resolve: (files) => {
+					clearTimeout(timeout);
+					resolve(files);
+				},
+				reject: (reason) => {
+					clearTimeout(timeout);
+					reject(reason);
+				},
+			});
+		});
+
+		const message: IndexRequestMessage = {
+			type: "index",
+			requestId,
+			cwd,
+		};
+		this.worker.postMessage(message);
+		return result;
+	}
+
+	private flushPending(error: Error): void {
+		for (const [requestId, request] of this.pending.entries()) {
+			request.reject(error);
+			this.pending.delete(requestId);
+		}
+	}
+}
+
+startWorkerServer();
+
+let workerClient: FileIndexWorkerClient | null | undefined;
+
+function getWorkerClient(): FileIndexWorkerClient | null {
+	if (!isMainThread) {
+		return null;
+	}
+	if (workerClient === undefined) {
+		workerClient = new FileIndexWorkerClient();
+	}
+	return workerClient;
+}
+
+async function buildIndexInBackground(cwd: string): Promise<Set<string>> {
+	const workerClient = getWorkerClient();
+	if (!workerClient) {
+		return buildIndex(cwd);
+	}
+
+	try {
+		const files = await workerClient.requestIndex(cwd);
+		return new Set(files);
+	} catch {
+		return buildIndex(cwd);
+	}
+}
+
+export async function getFileIndex(
+	cwd: string,
+	options: FastFileIndexOptions = {},
+): Promise<Set<string>> {
+	const ttlMs = options.ttlMs ?? DEFAULT_INDEX_TTL_MS;
+	const now = Date.now();
+	pruneStaleCacheEntries(now);
+	const existing = CACHE.get(cwd);
+
+	if (
+		existing &&
+		ttlMs > 0 &&
+		now - existing.lastBuiltAt <= ttlMs &&
+		existing.files.size > 0
+	) {
+		existing.lastAccessedAt = now;
+		return existing.files;
+	}
+
+	if (existing?.pending) {
+		existing.lastAccessedAt = now;
+		return existing.pending;
+	}
+
+	const pending = buildIndexInBackground(cwd).then((files) => {
+		CACHE.set(cwd, {
+			files,
+			lastBuiltAt: Date.now(),
+			lastAccessedAt: Date.now(),
+			pending: null,
+		});
+		return files;
+	});
+
+	CACHE.set(cwd, {
+		files: existing?.files ?? new Set<string>(),
+		lastBuiltAt: existing?.lastBuiltAt ?? 0,
+		lastAccessedAt: now,
+		pending,
+	});
+
+	return pending;
+}
+
+export async function prewarmFileIndex(
+	cwd: string,
+	options: FastFileIndexOptions = {},
+): Promise<void> {
+	await getFileIndex(cwd, { ...options, ttlMs: 0 });
+}

package/src/input/index.ts

@@ -0,0 +1,7 @@
+export type { FastFileIndexOptions } from "./file-indexer";
+export { getFileIndex, prewarmFileIndex } from "./file-indexer";
+export type {
+	MentionEnricherOptions,
+	MentionEnrichmentResult,
+} from "./mention-enricher";
+export { enrichPromptWithMentions } from "./mention-enricher";

package/src/input/mention-enricher.test.ts

@@ -0,0 +1,85 @@
+import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { describe, expect, it, vi } from "vitest";
+import { enrichPromptWithMentions } from "./mention-enricher";
+
+vi.mock("node:worker_threads", async () => {
+	const actual = await vi.importActual<typeof import("node:worker_threads")>(
+		"node:worker_threads",
+	);
+	return {
+		...actual,
+		isMainThread: false,
+		parentPort: null,
+	};
+});
+
+async function createTempWorkspace(): Promise<string> {
+	return mkdtemp(path.join(os.tmpdir(), "core-mentions-"));
+}
+
+describe("enrichPromptWithMentions", () => {
+	it("returns matched files for matching @path mentions", async () => {
+		const cwd = await createTempWorkspace();
+		try {
+			const sourcePath = path.join(cwd, "src", "index.ts");
+			await mkdir(path.dirname(sourcePath), { recursive: true });
+			await writeFile(sourcePath, "export const answer = 42\n", "utf8");
+
+			const result = await enrichPromptWithMentions(
+				"Review @src/index.ts",
+				cwd,
+			);
+
+			expect(result.mentions).toEqual(["src/index.ts"]);
+			expect(result.matchedFiles).toEqual(["src/index.ts"]);
+			expect(result.ignoredMentions).toEqual([]);
+			expect(result.prompt).toBe("Review @src/index.ts");
+		} finally {
+			await rm(cwd, { recursive: true, force: true });
+		}
+	});
+
+	it("ignores emails and unmatched mentions", async () => {
+		const cwd = await createTempWorkspace();
+		try {
+			await writeFile(path.join(cwd, "README.md"), "# Demo\n", "utf8");
+
+			const result = await enrichPromptWithMentions(
+				"Ping me at test@example.com and check @missing/file.ts.",
+				cwd,
+			);
+
+			expect(result.mentions).toEqual(["missing/file.ts"]);
+			expect(result.matchedFiles).toEqual([]);
+			expect(result.ignoredMentions).toEqual(["missing/file.ts"]);
+			expect(result.prompt).toBe(
+				"Ping me at test@example.com and check @missing/file.ts.",
+			);
+		} finally {
+			await rm(cwd, { recursive: true, force: true });
+		}
+	});
+
+	it("respects maxTotalBytes while keeping prompt unchanged", async () => {
+		const cwd = await createTempWorkspace();
+		try {
+			await writeFile(path.join(cwd, "a.ts"), "123", "utf8");
+			await writeFile(path.join(cwd, "b.ts"), "const b = 2\n", "utf8");
+
+			const result = await enrichPromptWithMentions(
+				"Use @a.ts and @b.ts",
+				cwd,
+				{ maxTotalBytes: 5, maxFiles: 2, maxFileBytes: 5 },
+			);
+
+			expect(result.mentions).toEqual(["a.ts", "b.ts"]);
+			expect(result.matchedFiles).toEqual(["a.ts"]);
+			expect(result.ignoredMentions).toEqual(["b.ts"]);
+			expect(result.prompt).toBe("Use @a.ts and @b.ts");
+		} finally {
+			await rm(cwd, { recursive: true, force: true });
+		}
+	});
+});

package/src/input/mention-enricher.ts

@@ -0,0 +1,122 @@
+import { stat } from "node:fs/promises";
+import path from "node:path";
+import { type FastFileIndexOptions, getFileIndex } from "./file-indexer";
+
+const TRAILING_PUNCTUATION = /[),.:;!?`'"]+$/;
+const LEADING_WRAPPERS = /^[(`'"]+/;
+
+export interface MentionEnricherOptions extends FastFileIndexOptions {
+	maxFiles?: number;
+	maxFileBytes?: number;
+	maxTotalBytes?: number;
+}
+
+export interface MentionEnrichmentResult {
+	prompt: string;
+	mentions: string[];
+	matchedFiles: string[];
+	ignoredMentions: string[];
+}
+
+function extractMentionTokens(input: string): string[] {
+	const matches = input.matchAll(/(^|[\s])@([^\s]+)/g);
+	const mentions: string[] = [];
+	for (const match of matches) {
+		const token = (match[2] ?? "").trim();
+		if (token.length === 0) {
+			continue;
+		}
+		const normalized = token
+			.replace(LEADING_WRAPPERS, "")
+			.replace(TRAILING_PUNCTUATION, "");
+		if (normalized.length === 0 || normalized.includes("@")) {
+			continue;
+		}
+		mentions.push(normalized);
+	}
+	return Array.from(new Set(mentions));
+}
+
+function normalizeMentionPath(
+	mention: string,
+	cwd: string,
+): string | undefined {
+	const candidate = mention.replace(/\\/g, "/");
+	const maybeAbsolute = path.isAbsolute(candidate)
+		? path.resolve(candidate)
+		: path.resolve(cwd, candidate);
+	const relative = path.relative(cwd, maybeAbsolute);
+	if (relative.startsWith("..") || path.isAbsolute(relative)) {
+		return undefined;
+	}
+	return relative.split(path.sep).join("/");
+}
+
+export async function enrichPromptWithMentions(
+	input: string,
+	cwd: string,
+	options: MentionEnricherOptions = {},
+): Promise<MentionEnrichmentResult> {
+	const mentions = extractMentionTokens(input);
+	if (mentions.length === 0) {
+		return {
+			prompt: input,
+			mentions: [],
+			matchedFiles: [],
+			ignoredMentions: [],
+		};
+	}
+
+	const maxFiles = options.maxFiles;
+	const maxFileBytes = options.maxFileBytes;
+	const maxTotalBytes = options.maxTotalBytes;
+	const fileList = await getFileIndex(cwd, { ttlMs: options.ttlMs });
+	const matched: string[] = [];
+	const ignored: string[] = [];
+	const attachments: Array<{ path: string; content: string }> = [];
+	let totalBytes = 0;
+
+	for (const mention of mentions) {
+		if (maxFiles && attachments.length >= maxFiles) {
+			ignored.push(mention);
+			continue;
+		}
+
+		const relativePath = normalizeMentionPath(mention, cwd);
+		if (!relativePath || !fileList.has(relativePath)) {
+			ignored.push(mention);
+			continue;
+		}
+
+		if (!maxFileBytes || !maxTotalBytes) {
+			matched.push(relativePath);
+			continue;
+		}
+
+		const absolutePath = path.join(cwd, relativePath);
+		try {
+			const fileStat = await stat(absolutePath);
+			if (!fileStat.isFile()) {
+				ignored.push(mention);
+				continue;
+			}
+			const nextBytes = totalBytes + maxFileBytes;
+			if (nextBytes > maxTotalBytes) {
+				ignored.push(mention);
+				continue;
+			}
+
+			totalBytes += nextBytes;
+			matched.push(relativePath);
+		} catch {
+			ignored.push(mention);
+		}
+	}
+
+	return {
+		prompt: input,
+		mentions,
+		matchedFiles: matched,
+		ignoredMentions: ignored,
+	};
+}