@clinebot/core 0.0.20 → 0.0.21

package/src/input/file-indexer.ts

@@ -1,327 +0,0 @@
-import { spawn } from "node:child_process";
-import { readdir } from "node:fs/promises";
-import path from "node:path";
-import { isMainThread, parentPort, Worker } from "node:worker_threads";
-
-const DEFAULT_INDEX_TTL_MS = 15_000;
-const STALE_CACHE_EVICTION_MS = 10 * 60_000;
-const WORKER_INDEX_REQUEST_TIMEOUT_MS = 1_000;
-const DEFAULT_EXCLUDE_DIRS = new Set([
-	".git",
-	"node_modules",
-	"dist",
-	"build",
-	".next",
-	"coverage",
-	".turbo",
-	".cache",
-	"target",
-	"out",
-]);
-
-interface CacheEntry {
-	files: Set<string>;
-	lastBuiltAt: number;
-	lastAccessedAt: number;
-	pending: Promise<Set<string>> | null;
-}
-
-export interface FastFileIndexOptions {
-	ttlMs?: number;
-}
-
-interface IndexRequestMessage {
-	type: "index";
-	requestId: number;
-	cwd: string;
-}
-
-interface IndexResponseMessage {
-	type: "indexResult";
-	requestId: number;
-	files?: string[];
-	error?: string;
-}
-
-const CACHE = new Map<string, CacheEntry>();
-
-function pruneStaleCacheEntries(now: number): void {
-	if (CACHE.size <= 1) {
-		return;
-	}
-	for (const [cwd, entry] of CACHE.entries()) {
-		if (entry.pending) {
-			continue;
-		}
-		if (now - entry.lastAccessedAt > STALE_CACHE_EVICTION_MS) {
-			CACHE.delete(cwd);
-		}
-	}
-}
-
-function toPosixRelative(cwd: string, absolutePath: string): string {
-	return path.relative(cwd, absolutePath).split(path.sep).join("/");
-}
-
-async function listFilesWithRg(cwd: string): Promise<Set<string>> {
-	const output = await new Promise<string>((resolve, reject) => {
-		const child = spawn("rg", ["--files", "--hidden", "-g", "!.git"], {
-			cwd,
-			stdio: ["ignore", "pipe", "pipe"],
-		});
-
-		let stdout = "";
-		let stderr = "";
-
-		child.stdout.on("data", (chunk: Buffer | string) => {
-			stdout += chunk.toString();
-		});
-		child.stderr.on("data", (chunk: Buffer | string) => {
-			stderr += chunk.toString();
-		});
-		child.on("error", reject);
-		child.on("close", (code: number | null) => {
-			if (code === 0) {
-				resolve(stdout);
-				return;
-			}
-			reject(new Error(stderr || `rg exited with code ${code}`));
-		});
-	});
-
-	const files = output
-		.split(/\r?\n/)
-		.map((line) => line.trim())
-		.filter((line) => line.length > 0)
-		.map((line) => line.replace(/\\/g, "/"));
-
-	return new Set(files);
-}
-
-async function walkDir(
-	cwd: string,
-	dir: string,
-	files: Set<string>,
-): Promise<void> {
-	const entries = await readdir(dir, { withFileTypes: true });
-	for (const entry of entries) {
-		const absolutePath = path.join(dir, entry.name);
-		if (entry.isDirectory()) {
-			if (DEFAULT_EXCLUDE_DIRS.has(entry.name)) {
-				continue;
-			}
-			await walkDir(cwd, absolutePath, files);
-			continue;
-		}
-		if (entry.isFile()) {
-			files.add(toPosixRelative(cwd, absolutePath));
-		}
-	}
-}
-
-async function listFilesFallback(cwd: string): Promise<Set<string>> {
-	const files = new Set<string>();
-	await walkDir(cwd, cwd, files);
-	return files;
-}
-
-async function buildIndex(cwd: string): Promise<Set<string>> {
-	try {
-		return await listFilesWithRg(cwd);
-	} catch {
-		return listFilesFallback(cwd);
-	}
-}
-
-function startWorkerServer(): void {
-	if (isMainThread || !parentPort) {
-		return;
-	}
-	const port = parentPort;
-
-	port.on("message", (message: IndexRequestMessage) => {
-		if (message.type !== "index") {
-			return;
-		}
-
-		void buildIndex(message.cwd)
-			.then((files) => {
-				const response: IndexResponseMessage = {
-					type: "indexResult",
-					requestId: message.requestId,
-					files: Array.from(files),
-				};
-				port.postMessage(response);
-			})
-			.catch((error: unknown) => {
-				const response: IndexResponseMessage = {
-					type: "indexResult",
-					requestId: message.requestId,
-					error:
-						error instanceof Error
-							? error.message
-							: "Failed to build file index",
-				};
-				port.postMessage(response);
-			});
-	});
-}
-
-class FileIndexWorkerClient {
-	private readonly worker = new Worker(new URL(import.meta.url));
-	private nextRequestId = 0;
-	private pending = new Map<
-		number,
-		{
-			resolve: (files: string[]) => void;
-			reject: (reason: Error) => void;
-		}
-	>();
-
-	constructor() {
-		// Keep indexing opportunistic: this worker should never block process exit.
-		this.worker.unref();
-		this.worker.on("message", (message: IndexResponseMessage) => {
-			if (message.type !== "indexResult") {
-				return;
-			}
-			const request = this.pending.get(message.requestId);
-			if (!request) {
-				return;
-			}
-			this.pending.delete(message.requestId);
-			if (message.error) {
-				request.reject(new Error(message.error));
-				return;
-			}
-			request.resolve(message.files ?? []);
-		});
-
-		this.worker.on("error", (error: Error) => {
-			this.flushPending(error);
-		});
-
-		this.worker.on("exit", (code) => {
-			if (code !== 0) {
-				this.flushPending(
-					new Error(`File index worker exited with code ${code}`),
-				);
-			}
-		});
-	}
-
-	requestIndex(cwd: string): Promise<string[]> {
-		const requestId = ++this.nextRequestId;
-		const result = new Promise<string[]>((resolve, reject) => {
-			const timeout = setTimeout(() => {
-				this.pending.delete(requestId);
-				reject(new Error("Timed out waiting for file index worker response"));
-			}, WORKER_INDEX_REQUEST_TIMEOUT_MS);
-			timeout.unref();
-			this.pending.set(requestId, {
-				resolve: (files) => {
-					clearTimeout(timeout);
-					resolve(files);
-				},
-				reject: (reason) => {
-					clearTimeout(timeout);
-					reject(reason);
-				},
-			});
-		});
-
-		const message: IndexRequestMessage = {
-			type: "index",
-			requestId,
-			cwd,
-		};
-		this.worker.postMessage(message);
-		return result;
-	}
-
-	private flushPending(error: Error): void {
-		for (const [requestId, request] of this.pending.entries()) {
-			request.reject(error);
-			this.pending.delete(requestId);
-		}
-	}
-}
-
-startWorkerServer();
-
-let workerClient: FileIndexWorkerClient | null | undefined;
-
-function getWorkerClient(): FileIndexWorkerClient | null {
-	if (!isMainThread) {
-		return null;
-	}
-	if (workerClient === undefined) {
-		workerClient = new FileIndexWorkerClient();
-	}
-	return workerClient;
-}
-
-async function buildIndexInBackground(cwd: string): Promise<Set<string>> {
-	const workerClient = getWorkerClient();
-	if (!workerClient) {
-		return buildIndex(cwd);
-	}
-
-	try {
-		const files = await workerClient.requestIndex(cwd);
-		return new Set(files);
-	} catch {
-		return buildIndex(cwd);
-	}
-}
-
-export async function getFileIndex(
-	cwd: string,
-	options: FastFileIndexOptions = {},
-): Promise<Set<string>> {
-	const ttlMs = options.ttlMs ?? DEFAULT_INDEX_TTL_MS;
-	const now = Date.now();
-	pruneStaleCacheEntries(now);
-	const existing = CACHE.get(cwd);
-
-	if (
-		existing &&
-		ttlMs > 0 &&
-		now - existing.lastBuiltAt <= ttlMs &&
-		existing.files.size > 0
-	) {
-		existing.lastAccessedAt = now;
-		return existing.files;
-	}
-
-	if (existing?.pending) {
-		existing.lastAccessedAt = now;
-		return existing.pending;
-	}
-
-	const pending = buildIndexInBackground(cwd).then((files) => {
-		CACHE.set(cwd, {
-			files,
-			lastBuiltAt: Date.now(),
-			lastAccessedAt: Date.now(),
-			pending: null,
-		});
-		return files;
-	});
-
-	CACHE.set(cwd, {
-		files: existing?.files ?? new Set<string>(),
-		lastBuiltAt: existing?.lastBuiltAt ?? 0,
-		lastAccessedAt: now,
-		pending,
-	});
-
-	return pending;
-}
-
-export async function prewarmFileIndex(
-	cwd: string,
-	options: FastFileIndexOptions = {},
-): Promise<void> {
-	await getFileIndex(cwd, { ...options, ttlMs: 0 });
-}

package/src/input/index.ts

@@ -1,7 +0,0 @@
-export type { FastFileIndexOptions } from "./file-indexer";
-export { getFileIndex, prewarmFileIndex } from "./file-indexer";
-export type {
-	MentionEnricherOptions,
-	MentionEnrichmentResult,
-} from "./mention-enricher";
-export { enrichPromptWithMentions } from "./mention-enricher";

package/src/input/mention-enricher.test.ts

@@ -1,85 +0,0 @@
-import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
-import os from "node:os";
-import path from "node:path";
-import { describe, expect, it, vi } from "vitest";
-import { enrichPromptWithMentions } from "./mention-enricher";
-
-vi.mock("node:worker_threads", async () => {
-	const actual = await vi.importActual<typeof import("node:worker_threads")>(
-		"node:worker_threads",
-	);
-	return {
-		...actual,
-		isMainThread: false,
-		parentPort: null,
-	};
-});
-
-async function createTempWorkspace(): Promise<string> {
-	return mkdtemp(path.join(os.tmpdir(), "core-mentions-"));
-}
-
-describe("enrichPromptWithMentions", () => {
-	it("returns matched files for matching @path mentions", async () => {
-		const cwd = await createTempWorkspace();
-		try {
-			const sourcePath = path.join(cwd, "src", "index.ts");
-			await mkdir(path.dirname(sourcePath), { recursive: true });
-			await writeFile(sourcePath, "export const answer = 42\n", "utf8");
-
-			const result = await enrichPromptWithMentions(
-				"Review @src/index.ts",
-				cwd,
-			);
-
-			expect(result.mentions).toEqual(["src/index.ts"]);
-			expect(result.matchedFiles).toEqual(["src/index.ts"]);
-			expect(result.ignoredMentions).toEqual([]);
-			expect(result.prompt).toBe("Review @src/index.ts");
-		} finally {
-			await rm(cwd, { recursive: true, force: true });
-		}
-	});
-
-	it("ignores emails and unmatched mentions", async () => {
-		const cwd = await createTempWorkspace();
-		try {
-			await writeFile(path.join(cwd, "README.md"), "# Demo\n", "utf8");
-
-			const result = await enrichPromptWithMentions(
-				"Ping me at test@example.com and check @missing/file.ts.",
-				cwd,
-			);
-
-			expect(result.mentions).toEqual(["missing/file.ts"]);
-			expect(result.matchedFiles).toEqual([]);
-			expect(result.ignoredMentions).toEqual(["missing/file.ts"]);
-			expect(result.prompt).toBe(
-				"Ping me at test@example.com and check @missing/file.ts.",
-			);
-		} finally {
-			await rm(cwd, { recursive: true, force: true });
-		}
-	});
-
-	it("respects maxTotalBytes while keeping prompt unchanged", async () => {
-		const cwd = await createTempWorkspace();
-		try {
-			await writeFile(path.join(cwd, "a.ts"), "123", "utf8");
-			await writeFile(path.join(cwd, "b.ts"), "const b = 2\n", "utf8");
-
-			const result = await enrichPromptWithMentions(
-				"Use @a.ts and @b.ts",
-				cwd,
-				{ maxTotalBytes: 5, maxFiles: 2, maxFileBytes: 5 },
-			);
-
-			expect(result.mentions).toEqual(["a.ts", "b.ts"]);
-			expect(result.matchedFiles).toEqual(["a.ts"]);
-			expect(result.ignoredMentions).toEqual(["b.ts"]);
-			expect(result.prompt).toBe("Use @a.ts and @b.ts");
-		} finally {
-			await rm(cwd, { recursive: true, force: true });
-		}
-	});
-});

package/src/input/mention-enricher.ts

@@ -1,122 +0,0 @@
-import { stat } from "node:fs/promises";
-import path from "node:path";
-import { type FastFileIndexOptions, getFileIndex } from "./file-indexer";
-
-const TRAILING_PUNCTUATION = /[),.:;!?`'"]+$/;
-const LEADING_WRAPPERS = /^[(`'"]+/;
-
-export interface MentionEnricherOptions extends FastFileIndexOptions {
-	maxFiles?: number;
-	maxFileBytes?: number;
-	maxTotalBytes?: number;
-}
-
-export interface MentionEnrichmentResult {
-	prompt: string;
-	mentions: string[];
-	matchedFiles: string[];
-	ignoredMentions: string[];
-}
-
-function extractMentionTokens(input: string): string[] {
-	const matches = input.matchAll(/(^|[\s])@([^\s]+)/g);
-	const mentions: string[] = [];
-	for (const match of matches) {
-		const token = (match[2] ?? "").trim();
-		if (token.length === 0) {
-			continue;
-		}
-		const normalized = token
-			.replace(LEADING_WRAPPERS, "")
-			.replace(TRAILING_PUNCTUATION, "");
-		if (normalized.length === 0 || normalized.includes("@")) {
-			continue;
-		}
-		mentions.push(normalized);
-	}
-	return Array.from(new Set(mentions));
-}
-
-function normalizeMentionPath(
-	mention: string,
-	cwd: string,
-): string | undefined {
-	const candidate = mention.replace(/\\/g, "/");
-	const maybeAbsolute = path.isAbsolute(candidate)
-		? path.resolve(candidate)
-		: path.resolve(cwd, candidate);
-	const relative = path.relative(cwd, maybeAbsolute);
-	if (relative.startsWith("..") || path.isAbsolute(relative)) {
-		return undefined;
-	}
-	return relative.split(path.sep).join("/");
-}
-
-export async function enrichPromptWithMentions(
-	input: string,
-	cwd: string,
-	options: MentionEnricherOptions = {},
-): Promise<MentionEnrichmentResult> {
-	const mentions = extractMentionTokens(input);
-	if (mentions.length === 0) {
-		return {
-			prompt: input,
-			mentions: [],
-			matchedFiles: [],
-			ignoredMentions: [],
-		};
-	}
-
-	const maxFiles = options.maxFiles;
-	const maxFileBytes = options.maxFileBytes;
-	const maxTotalBytes = options.maxTotalBytes;
-	const fileList = await getFileIndex(cwd, { ttlMs: options.ttlMs });
-	const matched: string[] = [];
-	const ignored: string[] = [];
-	const attachments: Array<{ path: string; content: string }> = [];
-	let totalBytes = 0;
-
-	for (const mention of mentions) {
-		if (maxFiles && attachments.length >= maxFiles) {
-			ignored.push(mention);
-			continue;
-		}
-
-		const relativePath = normalizeMentionPath(mention, cwd);
-		if (!relativePath || !fileList.has(relativePath)) {
-			ignored.push(mention);
-			continue;
-		}
-
-		if (!maxFileBytes || !maxTotalBytes) {
-			matched.push(relativePath);
-			continue;
-		}
-
-		const absolutePath = path.join(cwd, relativePath);
-		try {
-			const fileStat = await stat(absolutePath);
-			if (!fileStat.isFile()) {
-				ignored.push(mention);
-				continue;
-			}
-			const nextBytes = totalBytes + maxFileBytes;
-			if (nextBytes > maxTotalBytes) {
-				ignored.push(mention);
-				continue;
-			}
-
-			totalBytes += nextBytes;
-			matched.push(relativePath);
-		} catch {
-			ignored.push(mention);
-		}
-	}
-
-	return {
-		prompt: input,
-		mentions,
-		matchedFiles: matched,
-		ignoredMentions: ignored,
-	};
-}