@nghyane/arcane 0.1.0

package/src/utils/file-mentions.ts

@@ -0,0 +1,384 @@
+/**
+ * Auto-read file mentions from user prompts.
+ *
+ * When users reference files with @path syntax (e.g., "@src/foo.ts"),
+ * we automatically inject the file contents as a FileMentionMessage
+ * so the agent doesn't need to read them manually.
+ */
+import * as fs from "node:fs/promises";
+import path from "node:path";
+import type { AgentMessage } from "@nghyane/arcane-agent";
+import { glob } from "@nghyane/arcane-natives";
+import { formatHashLines } from "../patch/hashline";
+import type { FileMentionMessage } from "../session/messages";
+import {
+	DEFAULT_MAX_BYTES,
+	formatBytes,
+	truncateHead,
+	truncateStringToBytesFromStart,
+} from "../session/streaming-output";
+import { resolveReadPath } from "../tools/path-utils";
+import { formatAge } from "../tools/render-utils";
+import { fuzzyMatch } from "./fuzzy";
+import { formatDimensionNote, resizeImage } from "./image-resize";
+import { detectSupportedImageMimeTypeFromFile } from "./mime";
+
+/** Regex to match @filepath patterns in text */
+const FILE_MENTION_REGEX = /@([^\s@]+)/g;
+const LEADING_PUNCTUATION_REGEX = /^[`"'([{<]+/;
+const TRAILING_PUNCTUATION_REGEX = /[)\]}>.,;:!?"'`]+$/;
+const MENTION_BOUNDARY_REGEX = /[\s([{<"'`]/;
+const DEFAULT_DIR_LIMIT = 500;
+const MIN_FUZZY_QUERY_LENGTH = 5;
+const MAX_RESOLUTION_CANDIDATES = 20_000;
+const PATH_SEPARATOR_REGEX = /[/._\-\s]+/g;
+
+type MentionDiscoveryProfile = {
+	hidden: boolean;
+	gitignore: boolean;
+	includeNodeModules: boolean;
+	maxResults: number;
+	cache: boolean;
+};
+
+function getMentionCandidateDiscoveryProfile(): MentionDiscoveryProfile {
+	return {
+		hidden: true,
+		gitignore: true,
+		cache: true,
+		includeNodeModules: true,
+		maxResults: MAX_RESOLUTION_CANDIDATES,
+	};
+}
+
+// Avoid OOM when users @mention very large files. Above these limits we skip
+// auto-reading and only include the path in the message.
+const MAX_AUTO_READ_TEXT_BYTES = 5 * 1024 * 1024; // 5MB
+const MAX_AUTO_READ_IMAGE_BYTES = 25 * 1024 * 1024; // 25MB
+
+function isMentionBoundary(text: string, index: number): boolean {
+	if (index === 0) return true;
+	return MENTION_BOUNDARY_REGEX.test(text[index - 1]);
+}
+
+function sanitizeMentionPath(rawPath: string): string | null {
+	let cleaned = rawPath.trim();
+	cleaned = cleaned.replace(LEADING_PUNCTUATION_REGEX, "");
+	cleaned = cleaned.replace(TRAILING_PUNCTUATION_REGEX, "");
+	cleaned = cleaned.trim();
+	return cleaned.length > 0 ? cleaned : null;
+}
+
+type MentionCandidate = {
+	path: string;
+	pathLower: string;
+	normalizedPath: string;
+};
+
+function normalizeMentionQuery(query: string): string {
+	return query.toLowerCase().replace(PATH_SEPARATOR_REGEX, "");
+}
+
+async function pathExists(filePath: string): Promise<boolean> {
+	try {
+		await Bun.file(filePath).stat();
+		return true;
+	} catch {
+		return false;
+	}
+}
+
+async function listMentionCandidates(cwd: string): Promise<MentionCandidate[]> {
+	let entries: string[];
+	try {
+		const discoveryProfile = getMentionCandidateDiscoveryProfile();
+		const result = await glob({
+			pattern: "**/*",
+			path: cwd,
+			...discoveryProfile,
+		});
+		entries = result.matches.map(match => match.path);
+	} catch {
+		return [];
+	}
+
+	entries.sort((a, b) => a.toLowerCase().localeCompare(b.toLowerCase()));
+	const candidates: MentionCandidate[] = [];
+	for (const entry of entries) {
+		const pathLower = entry.toLowerCase();
+		const normalizedPath = normalizeMentionQuery(entry);
+		if (normalizedPath.length === 0) {
+			continue;
+		}
+		candidates.push({ path: entry, pathLower, normalizedPath });
+	}
+	return candidates;
+}
+
+async function resolveMentionPath(
+	filePath: string,
+	cwd: string,
+	getMentionCandidates: () => Promise<MentionCandidate[]>,
+): Promise<string | null> {
+	const absolutePath = resolveReadPath(filePath, cwd);
+	if (await pathExists(absolutePath)) {
+		return filePath;
+	}
+
+	const queryLower = filePath.toLowerCase();
+	const candidates = await getMentionCandidates();
+	const prefixMatches = candidates.filter(candidate => candidate.pathLower.startsWith(queryLower));
+	if (prefixMatches.length === 1) {
+		return prefixMatches[0]?.path ?? null;
+	}
+	if (prefixMatches.length > 1) {
+		return null;
+	}
+
+	const normalizedQuery = normalizeMentionQuery(filePath);
+	if (normalizedQuery.length < MIN_FUZZY_QUERY_LENGTH) {
+		return null;
+	}
+
+	const scored = candidates
+		.map(candidate => ({ candidate, match: fuzzyMatch(normalizedQuery, candidate.normalizedPath) }))
+		.filter(entry => entry.match.matches)
+		.sort((a, b) => {
+			if (a.match.score !== b.match.score) {
+				return a.match.score - b.match.score;
+			}
+			return a.candidate.path.localeCompare(b.candidate.path);
+		});
+
+	if (scored.length === 0) {
+		return null;
+	}
+
+	const best = scored[0];
+
+	return best?.candidate.path ?? null;
+}
+
+function buildTextOutput(textContent: string): { output: string; lineCount: number } {
+	const allLines = textContent.split("\n");
+	const totalFileLines = allLines.length;
+	const truncation = truncateHead(textContent);
+
+	if (truncation.firstLineExceedsLimit) {
+		const firstLine = allLines[0] ?? "";
+		const firstLineBytes = Buffer.byteLength(firstLine, "utf-8");
+		const snippet = truncateStringToBytesFromStart(firstLine, DEFAULT_MAX_BYTES);
+		let outputText = snippet.text;
+
+		if (outputText.length > 0) {
+			outputText += `\n\n[Line 1 is ${formatBytes(firstLineBytes)}, exceeds ${formatBytes(
+				DEFAULT_MAX_BYTES,
+			)} limit. Showing first ${formatBytes(snippet.bytes)} of the line.]`;
+		} else {
+			outputText = `[Line 1 is ${formatBytes(firstLineBytes)}, exceeds ${formatBytes(
+				DEFAULT_MAX_BYTES,
+			)} limit. Unable to display a valid UTF-8 snippet.]`;
+		}
+
+		return { output: outputText, lineCount: totalFileLines };
+	}
+
+	let outputText = truncation.content;
+
+	if (truncation.truncated) {
+		const endLineDisplay = truncation.outputLines;
+		const nextOffset = endLineDisplay + 1;
+
+		if (truncation.truncatedBy === "lines") {
+			outputText += `\n\n[Showing lines 1-${endLineDisplay} of ${totalFileLines}. Use offset=${nextOffset} to continue]`;
+		} else {
+			outputText += `\n\n[Showing lines 1-${endLineDisplay} of ${totalFileLines} (${formatBytes(
+				DEFAULT_MAX_BYTES,
+			)} limit). Use offset=${nextOffset} to continue]`;
+		}
+	}
+
+	return { output: outputText, lineCount: totalFileLines };
+}
+
+async function buildDirectoryListing(absolutePath: string): Promise<{ output: string; lineCount: number }> {
+	let entries: string[];
+	try {
+		entries = await Array.fromAsync(new Bun.Glob("*").scan({ cwd: absolutePath, dot: true, onlyFiles: false }));
+	} catch {
+		return { output: "(empty directory)", lineCount: 1 };
+	}
+
+	entries.sort((a, b) => a.toLowerCase().localeCompare(b.toLowerCase()));
+
+	const results: string[] = [];
+	let entryLimitReached = false;
+
+	for (const entry of entries) {
+		if (results.length >= DEFAULT_DIR_LIMIT) {
+			entryLimitReached = true;
+			break;
+		}
+
+		const fullPath = path.join(absolutePath, entry);
+		let suffix = "";
+		let age = "";
+
+		try {
+			const stat = await Bun.file(fullPath).stat();
+			if (stat.isDirectory()) {
+				suffix = "/";
+			}
+			const ageSeconds = Math.floor((Date.now() - stat.mtimeMs) / 1000);
+			age = formatAge(ageSeconds);
+		} catch {
+			continue;
+		}
+
+		const line = age ? `${entry}${suffix} (${age})` : `${entry}${suffix}`;
+		results.push(line);
+	}
+
+	if (results.length === 0) {
+		return { output: "(empty directory)", lineCount: 1 };
+	}
+
+	const rawOutput = results.join("\n");
+	const truncation = truncateHead(rawOutput, { maxLines: Number.MAX_SAFE_INTEGER });
+	let output = truncation.content;
+
+	const notices: string[] = [];
+	if (entryLimitReached) {
+		notices.push(`${DEFAULT_DIR_LIMIT} entries limit reached. Use limit=${DEFAULT_DIR_LIMIT * 2} for more`);
+	}
+	if (truncation.truncated) {
+		notices.push(`${formatBytes(DEFAULT_MAX_BYTES)} limit reached`);
+	}
+	if (notices.length > 0) {
+		output += `\n\n[${notices.join(". ")}]`;
+	}
+
+	return { output, lineCount: output.split("\n").length };
+}
+
+/** Extract all @filepath mentions from text */
+export function extractFileMentions(text: string): string[] {
+	const matches = [...text.matchAll(FILE_MENTION_REGEX)];
+	const mentions: string[] = [];
+
+	for (const match of matches) {
+		const index = match.index ?? 0;
+		if (!isMentionBoundary(text, index)) continue;
+
+		const cleaned = sanitizeMentionPath(match[1]);
+		if (!cleaned) continue;
+
+		mentions.push(cleaned);
+	}
+
+	return [...new Set(mentions)];
+}
+
+/**
+ * Generate a FileMentionMessage containing the contents of mentioned files.
+ * Returns empty array if no files could be read.
+ */
+export async function generateFileMentionMessages(
+	filePaths: string[],
+	cwd: string,
+	options?: { autoResizeImages?: boolean; useHashLines?: boolean },
+): Promise<AgentMessage[]> {
+	if (filePaths.length === 0) return [];
+
+	const autoResizeImages = options?.autoResizeImages ?? true;
+
+	const files: FileMentionMessage["files"] = [];
+	let mentionCandidatesPromise: Promise<MentionCandidate[]> | null = null;
+	const getMentionCandidates = (): Promise<MentionCandidate[]> => {
+		mentionCandidatesPromise ??= listMentionCandidates(cwd);
+		return mentionCandidatesPromise;
+	};
+
+	for (const filePath of filePaths) {
+		const resolvedPath = await resolveMentionPath(filePath, cwd, getMentionCandidates);
+		if (!resolvedPath) {
+			continue;
+		}
+		const absolutePath = resolveReadPath(resolvedPath, cwd);
+		try {
+			const stat = await Bun.file(absolutePath).stat();
+			if (stat.isDirectory()) {
+				const { output, lineCount } = await buildDirectoryListing(absolutePath);
+				files.push({ path: resolvedPath, content: output, lineCount });
+				continue;
+			}
+
+			const mimeType = await detectSupportedImageMimeTypeFromFile(absolutePath);
+			if (mimeType) {
+				if (stat.size > MAX_AUTO_READ_IMAGE_BYTES) {
+					files.push({
+						path: resolvedPath,
+						content: `(skipped auto-read: too large, ${formatBytes(stat.size)})`,
+						byteSize: stat.size,
+						skippedReason: "tooLarge",
+					});
+					continue;
+				}
+				const buffer = await fs.readFile(absolutePath);
+				if (buffer.length === 0) {
+					continue;
+				}
+
+				const base64Content = buffer.toBase64();
+				let image = { type: "image" as const, mimeType, data: base64Content };
+				let dimensionNote: string | undefined;
+
+				if (autoResizeImages) {
+					try {
+						const resized = await resizeImage({ type: "image", data: base64Content, mimeType });
+						dimensionNote = formatDimensionNote(resized);
+						image = {
+							type: "image" as const,
+							mimeType: resized.mimeType,
+							data: resized.data,
+						};
+					} catch {
+						image = { type: "image" as const, mimeType, data: base64Content };
+					}
+				}
+
+				files.push({ path: resolvedPath, content: dimensionNote ?? "", image });
+				continue;
+			}
+
+			if (stat.size > MAX_AUTO_READ_TEXT_BYTES) {
+				files.push({
+					path: resolvedPath,
+					content: `(skipped auto-read: too large, ${formatBytes(stat.size)})`,
+					byteSize: stat.size,
+					skippedReason: "tooLarge",
+				});
+				continue;
+			}
+
+			const content = await Bun.file(absolutePath).text();
+			let { output, lineCount } = buildTextOutput(content);
+			if (options?.useHashLines) {
+				output = formatHashLines(output);
+			}
+			files.push({ path: resolvedPath, content: output, lineCount });
+		} catch {
+			// File doesn't exist or isn't readable - skip silently
+		}
+	}
+
+	if (files.length === 0) return [];
+
+	const message: FileMentionMessage = {
+		role: "fileMention",
+		files,
+		timestamp: Date.now(),
+	};
+
+	return [message];
+}

package/src/utils/frontmatter.ts

@@ -0,0 +1,101 @@
+import { logger } from "@nghyane/arcane-utils";
+import { YAML } from "bun";
+
+function stripHtmlComments(content: string): string {
+	return content.replace(/<!--[\s\S]*?-->/g, "");
+}
+
+function toError(value: unknown): Error {
+	return value instanceof Error ? value : new Error(String(`YAML: ${value}`));
+}
+
+function truncate(content: string, maxLength: number): string {
+	return content.length > maxLength ? `${content.slice(0, maxLength)}…` : content;
+}
+
+export class FrontmatterError extends Error {
+	constructor(
+		error: Error,
+		readonly source?: unknown,
+	) {
+		super(`Failed to parse YAML frontmatter (${source}): ${error.message}`, { cause: error });
+		this.name = "FrontmatterError";
+	}
+
+	toString(): string {
+		// Format the error with stack and detail, including the error message, stack, and source if present
+		const details: string[] = [this.message];
+		if (this.source !== undefined) {
+			details.push(`Source: ${JSON.stringify(this.source)}`);
+		}
+		if (this.cause && typeof this.cause === "object" && "stack" in this.cause && this.cause.stack) {
+			details.push(`Stack:\n${this.cause.stack}`);
+		} else if (this.stack) {
+			details.push(`Stack:\n${this.stack}`);
+		}
+		return details.join("\n\n");
+	}
+}
+
+export interface FrontmatterOptions {
+	/** Source of the content (alias: source) */
+	location?: unknown;
+	/** Source of the content (alias for location) */
+	source?: unknown;
+	/** Fallback frontmatter values */
+	fallback?: Record<string, unknown>;
+	/** Normalize the content */
+	normalize?: boolean;
+	/** Level of error handling */
+	level?: "off" | "warn" | "fatal";
+}
+
+/**
+ * Parse YAML frontmatter from markdown content
+ * Returns { frontmatter, body } where body has frontmatter stripped
+ */
+export function parseFrontmatter(
+	content: string,
+	options?: FrontmatterOptions,
+): { frontmatter: Record<string, unknown>; body: string } {
+	const { location, source, fallback, normalize = true, level = "warn" } = options ?? {};
+	const loc = location ?? source;
+	const frontmatter: Record<string, unknown> = Object.assign({}, fallback);
+
+	const normalized = normalize ? stripHtmlComments(content.replace(/\r\n/g, "\n").replace(/\r/g, "\n")) : content;
+	if (!normalized.startsWith("---")) {
+		return { frontmatter, body: normalized };
+	}
+
+	const endIndex = normalized.indexOf("\n---", 3);
+	if (endIndex === -1) {
+		return { frontmatter, body: normalized };
+	}
+
+	const metadata = normalized.slice(4, endIndex);
+	const body = normalized.slice(endIndex + 4).trim();
+
+	try {
+		// Replace tabs with spaces for YAML compatibility, use failsafe mode for robustness
+		const loaded = YAML.parse(metadata.replaceAll("\t", "  ")) as Record<string, unknown> | null;
+		return { frontmatter: Object.assign(frontmatter, loaded), body: body };
+	} catch (error) {
+		const err = new FrontmatterError(toError(error), loc ?? `Inline '${truncate(content, 64)}'`);
+		if (level === "warn" || level === "fatal") {
+			logger.warn("Failed to parse YAML frontmatter", { err: err.toString() });
+		}
+		if (level === "fatal") {
+			throw err;
+		}
+
+		// Simple YAML parsing - just key: value pairs
+		for (const line of metadata.split("\n")) {
+			const match = line.match(/^(\w+):\s*(.*)$/);
+			if (match) {
+				frontmatter[match[1]] = match[2].trim();
+			}
+		}
+
+		return { frontmatter, body: body };
+	}
+}

package/src/utils/fuzzy.ts

@@ -0,0 +1,108 @@
+// Fuzzy search. Matches if all query characters appear in order (not necessarily consecutive).
+// Lower score = better match.
+
+export interface FuzzyMatch {
+	matches: boolean;
+	score: number;
+}
+
+export function fuzzyMatch(query: string, text: string): FuzzyMatch {
+	const queryLower = query.toLowerCase();
+	const textLower = text.toLowerCase();
+
+	if (queryLower.length === 0) {
+		return { matches: true, score: 0 };
+	}
+
+	if (queryLower.length > textLower.length) {
+		return { matches: false, score: 0 };
+	}
+
+	let queryIndex = 0;
+	let score = 0;
+	let lastMatchIndex = -1;
+	let consecutiveMatches = 0;
+
+	for (let i = 0; i < textLower.length && queryIndex < queryLower.length; i++) {
+		if (textLower[i] === queryLower[queryIndex]) {
+			const isWordBoundary = i === 0 || /[\s\-_./]/.test(textLower[i - 1]!);
+
+			// Reward consecutive character matches (e.g., typing "foo" matches "foobar" better than "f_o_o")
+			if (lastMatchIndex === i - 1) {
+				consecutiveMatches++;
+				score -= consecutiveMatches * 5;
+			} else {
+				consecutiveMatches = 0;
+				// Penalize gaps between matched characters
+				if (lastMatchIndex >= 0) {
+					score += (i - lastMatchIndex - 1) * 2;
+				}
+			}
+
+			// Reward matches at word boundaries (start of words are more likely intentional targets)
+			if (isWordBoundary) {
+				score -= 10;
+			}
+
+			// Slight penalty for matches later in the string (prefer earlier matches)
+			score += i * 0.1;
+
+			lastMatchIndex = i;
+			queryIndex++;
+		}
+	}
+
+	// Not all query characters were found in order
+	if (queryIndex < queryLower.length) {
+		return { matches: false, score: 0 };
+	}
+
+	return { matches: true, score };
+}
+
+// Filter and sort items by fuzzy match quality (best matches first)
+// Supports space-separated tokens: all tokens must match, sorted by match count then score
+export function fuzzyFilter<T>(items: T[], query: string, getText: (item: T) => string): T[] {
+	if (!query.trim()) {
+		return items;
+	}
+
+	// Split query into tokens
+	const tokens = query
+		.trim()
+		.split(/\s+/)
+		.filter(t => t.length > 0);
+
+	if (tokens.length === 0) {
+		return items;
+	}
+
+	const results: { item: T; totalScore: number }[] = [];
+
+	for (const item of items) {
+		const text = getText(item);
+		let totalScore = 0;
+		let allMatch = true;
+
+		// Check each token against the text - ALL must match
+		for (const token of tokens) {
+			const match = fuzzyMatch(token, text);
+			if (match.matches) {
+				totalScore += match.score;
+			} else {
+				allMatch = false;
+				break;
+			}
+		}
+
+		// Only include if all tokens match
+		if (allMatch) {
+			results.push({ item, totalScore });
+		}
+	}
+
+	// Sort by score (asc, lower is better)
+	results.sort((a, b) => a.totalScore - b.totalScore);
+
+	return results.map(r => r.item);
+}

package/src/utils/ignore-files.ts

@@ -0,0 +1,119 @@
+/**
+ * Ignore file handling for .gitignore/.ignore/.fdignore support when scanning directories.
+ */
+import * as path from "node:path";
+import ignore from "ignore";
+
+export const IGNORE_FILE_NAMES = [".gitignore", ".ignore", ".fdignore"] as const;
+
+export type IgnoreMatcher = ignore.Ignore;
+
+/**
+ * Convert a path to POSIX format (forward slashes).
+ */
+export function toPosixPath(p: string): string {
+	return p.split(path.sep).join("/");
+}
+
+/**
+ * Prefix an ignore pattern to make it relative to a subdirectory.
+ * Returns null for comments and empty lines.
+ */
+export function prefixIgnorePattern(line: string, prefix: string): string | null {
+	const trimmed = line.trim();
+	if (!trimmed) return null;
+	if (trimmed.startsWith("#") && !trimmed.startsWith("\\#")) return null;
+
+	let pattern = line;
+	let negated = false;
+
+	if (pattern.startsWith("!")) {
+		negated = true;
+		pattern = pattern.slice(1);
+	} else if (pattern.startsWith("\\!")) {
+		pattern = pattern.slice(1);
+	}
+
+	if (pattern.startsWith("/")) {
+		pattern = pattern.slice(1);
+	}
+
+	const prefixed = prefix ? `${prefix}${pattern}` : pattern;
+	return negated ? `!${prefixed}` : prefixed;
+}
+
+/**
+ * Read and add ignore rules from a directory to the matcher.
+ */
+export async function addIgnoreRules(
+	ig: IgnoreMatcher,
+	dir: string,
+	rootDir: string,
+	readFile: (path: string) => Promise<string | null>,
+): Promise<void> {
+	const relativeDir = path.relative(rootDir, dir);
+	const prefix = relativeDir ? `${toPosixPath(relativeDir)}/` : "";
+
+	for (const filename of IGNORE_FILE_NAMES) {
+		const ignorePath = path.join(dir, filename);
+		const content = await readFile(ignorePath);
+		if (!content) continue;
+
+		const patterns = content
+			.split(/\r?\n/)
+			.map(line => prefixIgnorePattern(line, prefix))
+			.filter((line): line is string => Boolean(line));
+
+		if (patterns.length > 0) {
+			ig.add(patterns);
+		}
+	}
+}
+
+/**
+ * Read and add ignore rules from a directory to the matcher (synchronous version).
+ */
+export function addIgnoreRulesSync(
+	ig: IgnoreMatcher,
+	dir: string,
+	rootDir: string,
+	readFileSync: (path: string) => string | null,
+): void {
+	const relativeDir = path.relative(rootDir, dir);
+	const prefix = relativeDir ? `${toPosixPath(relativeDir)}/` : "";
+
+	for (const filename of IGNORE_FILE_NAMES) {
+		const ignorePath = path.join(dir, filename);
+		const content = readFileSync(ignorePath);
+		if (!content) continue;
+
+		const patterns = content
+			.split(/\r?\n/)
+			.map(line => prefixIgnorePattern(line, prefix))
+			.filter((line): line is string => Boolean(line));
+
+		if (patterns.length > 0) {
+			ig.add(patterns);
+		}
+	}
+}
+
+/**
+ * Create a fresh ignore matcher.
+ */
+export function createIgnoreMatcher(): IgnoreMatcher {
+	return ignore();
+}
+
+/**
+ * Check if a path should be ignored.
+ * @param ig - The ignore matcher
+ * @param root - The root directory for relative path calculation
+ * @param fullPath - The full path to check
+ * @param isDir - Whether the path is a directory
+ */
+export function shouldIgnore(ig: IgnoreMatcher, root: string, fullPath: string, isDir: boolean): boolean {
+	const relPath = toPosixPath(path.relative(root, fullPath));
+	const ignorePath = isDir ? `${relPath}/` : relPath;
+	return ig.ignores(ignorePath);
+}