headlesscode 1.0.2

package/src/codesearch/airunner-embedder.ts

@@ -0,0 +1,185 @@
+/**
+ * AIRunner embedder for the codebase index.
+ *
+ * A third backend alongside the OpenRouter (src/codesearch/embedder.ts) and
+ * Ollama (src/codesearch/ollama-embedder.ts) embedders. AIRunner (the local
+ * FastAPI/Python server in the sibling `airunner` repo) exposes its native
+ * in-process embedding stack (intfloat/e5-large) through a plain HTTP
+ * endpoint — POST /api/v1/embed/text — so headlesscode can embed chunks
+ * against a locally running AIRunner server without going through the
+ * Ollama-shaped shim or spending cloud money.
+ *
+ * Opt-in via `HEADLESSCODE_EMBEDDING_BACKEND=airunner` or
+ * `--embedding-backend airunner` on `headlesscode index`; the default stays
+ * OpenRouter.
+ *
+ * ─── AIRunner embeddings — endpoint contract ────────────────────────────
+ *
+ * 1. ENDPOINT + BATCHING: POST {AIRUNNER_EMBED_URL}/api/v1/embed/text
+ *    accepts `{"texts": string[]}` and returns
+ *    `{"model": "intfloat/e5-large", "embeddings": number[][]}` — one HTTP
+ *    call per whole batch, vectors in input order (mirrors the Ollama
+ *    /api/embed contract). e5-large produces 1024-dim vectors.
+ * 2. RESPONSE SHAPE: embeddings are plain JSON number arrays. There is no
+ *    usage/token accounting for a fully local call — promptTokens is
+ *    approximate (vector length) and only used for display, exactly like the
+ *    Ollama backend.
+ * 3. DIMENSION: e5-large = 1024 dims, NOT interchangeable with Ollama's
+ *    qwen3-embedding:8b (4096) or OpenRouter's qwen/qwen3-embedding-4b
+ *    (2560). The index build records {backend, model} metadata and the
+ *    `codebase_search` tool refuses a backend mismatch (see
+ *    src/codesearch/index.ts + the executor handler), same as the other
+ *    local backend.
+ *
+ * The embedder interface (Embedder) is shared with the other paths; only the
+ * HTTP client differs. No batching/hashing logic is duplicated here — the
+ * caller (src/codesearch/index.ts buildIndex) slices batches exactly as it
+ * does for the other backends.
+ */
+
+import type { EmbedResult } from "./embedder.js"
+
+/** Default AIRunner server base URL (AIRunner's standard local port). */
+export const DEFAULT_AIRUNNER_URL = "http://localhost:8080"
+
+/** Default local embedding model (the one AIRunner serves). */
+export const DEFAULT_AIRUNNER_EMBEDDING_MODEL = "intfloat/e5-large"
+
+/** Env var that overrides the AIRunner server URL. */
+export const AIRUNNER_URL_ENV = "HEADLESSCODE_AIRUNNER_EMBED_URL"
+
+/** Env var that overrides the AIRunner embedding model name. */
+export const AIRUNNER_EMBEDDING_MODEL_ENV = "HEADLESSCODE_AIRUNNER_EMBED_MODEL"
+
+/** Resolve the AIRunner URL: env override → default. */
+export function resolveAirunnerUrl(env: NodeJS.ProcessEnv = process.env): string {
+	return env[AIRUNNER_URL_ENV]?.trim() || DEFAULT_AIRUNNER_URL
+}
+
+/** Resolve the AIRunner embedding model: env override → default. */
+export function resolveAirunnerEmbeddingModel(env: NodeJS.ProcessEnv = process.env): string {
+	return env[AIRUNNER_EMBEDDING_MODEL_ENV]?.trim() || DEFAULT_AIRUNNER_EMBEDDING_MODEL
+}
+
+/** Typed error for AIRunner failures, carrying a pre-built actionable message. */
+export class AirunnerEmbedError extends Error {
+	/** HTTP status when AIRunner answered (undefined for network-level failures). */
+	readonly status?: number
+	/** The raw error body AIRunner returned, when one exists. */
+	readonly body?: string
+
+	constructor(message: string, status?: number, body?: string) {
+		super(message)
+		this.name = "AirunnerEmbedError"
+		this.status = status
+		this.body = body
+	}
+}
+
+/**
+ * AIRunner-backed embedder. One HTTP call per batch (POST /api/v1/embed/text),
+ * matching the other backends' batching. Failures are wrapped in actionable
+ * errors — never a silent fallback to another backend (that would surprise
+ * the user with unexpected cloud spend).
+ */
+export class AirunnerEmbedder {
+	readonly backend = "airunner" as const
+	readonly model: string
+	private readonly baseUrl: string
+
+	constructor(model: string = resolveAirunnerEmbeddingModel(), baseUrl: string = resolveAirunnerUrl()) {
+		this.model = model
+		this.baseUrl = baseUrl.replace(/\/+$/, "")
+	}
+
+	async embedBatch(texts: string[]): Promise<EmbedResult> {
+		if (texts.length === 0) {
+			return { embeddings: [], model: this.model, promptTokens: 0, totalTokens: 0 }
+		}
+
+		// Same guard as the other embedders: empty inputs are rejected
+		// upstream and produce no useful vector anyway.
+		const emptyIndex = texts.findIndex((t) => t.trim() === "")
+		if (emptyIndex !== -1) {
+			throw new AirunnerEmbedError(
+				`cannot embed empty/whitespace-only input at index ${emptyIndex} — providers reject it`,
+			)
+		}
+
+		const url = `${this.baseUrl}/api/v1/embed/text`
+		let response: Response
+		try {
+			response = await fetch(url, {
+				method: "POST",
+				headers: { "Content-Type": "application/json" },
+				body: JSON.stringify({ texts }),
+			})
+		} catch (err) {
+			// Network-level failure (connection refused, DNS, ...) — AIRunner
+			// is either not running or unreachable. Actionable, not generic.
+			throw new AirunnerEmbedError(
+				`AIRunner backend selected but ${this.baseUrl} is unreachable — is the AIRunner server running? ` +
+					`(or set HEADLESSCODE_AIRUNNER_EMBED_URL if AIRunner listens elsewhere): ` +
+					`${err instanceof Error ? err.message : String(err)}`,
+			)
+		}
+
+		const rawBody = await response.text().catch(() => "")
+		if (!response.ok) {
+			throw new AirunnerEmbedError(
+				`AIRunner embeddings returned HTTP ${response.status}: ${excerpt(rawBody || "(empty body)")}`,
+				response.status,
+				excerpt(rawBody),
+			)
+		}
+
+		let data:
+			| {
+					model?: unknown
+					embeddings?: unknown
+					error?: { message?: string }
+			  }
+			| undefined
+		try {
+			data = JSON.parse(rawBody)
+		} catch {
+			throw new AirunnerEmbedError(
+				`AIRunner embeddings returned HTTP 200 with a non-JSON/unparseable body: ${excerpt(rawBody || "(empty)")}`,
+			)
+		}
+
+		if (!Array.isArray(data?.embeddings) || data.embeddings.length !== texts.length) {
+			const n = Array.isArray(data?.embeddings) ? data.embeddings.length : 0
+			throw new AirunnerEmbedError(
+				`AIRunner embeddings response contained ${n} embeddings for ${texts.length} inputs. ` +
+					`Raw body: ${excerpt(rawBody || "(empty)")}`,
+			)
+		}
+
+		// Validate every entry is a numeric array — garbage here would silently
+		// poison the index (dimension is whatever the model returns, so the
+		// per-vector length must at least be self-consistent).
+		const embeddings: number[][] = []
+		let promptTokens = 0
+		for (const item of data.embeddings) {
+			if (!Array.isArray(item) || item.some((v) => typeof v !== "number")) {
+				throw new AirunnerEmbedError(
+					`AIRunner embeddings response contained a non-numeric embedding vector. Raw body: ${excerpt(rawBody)}`,
+				)
+			}
+			embeddings.push(item as number[])
+			promptTokens += (item as number[]).length
+		}
+
+		// No usage/cost to report for a fully local call — promptTokens is
+		// approximate (vector length, not true token count) and only used for
+		// display; totalTokens mirrors it so accounting call sites never see
+		// NaN/undefined.
+		return { embeddings, model: this.model, promptTokens, totalTokens: promptTokens }
+	}
+}
+
+/** Truncate a raw body to a bounded excerpt for error messages. */
+function excerpt(body: string): string {
+	return body.length > 500 ? `${body.slice(0, 500)}…` : body
+}

package/src/codesearch/chunk.ts

@@ -0,0 +1,339 @@
+/**
+ * Source-file chunking for the codebase index.
+ *
+ * Splits a source file into chunks that (a) are small enough to embed usefully
+ * and (b) map back to a `file:startLine-endLine` reference so a search result
+ * can be cited to the model.
+ *
+ * Strategy (deliberately heuristic, NOT a full per-language parser — the spec
+ * says good-enough regex/indentation beats over-engineering language-aware
+ * parsing, and the reference upstream implementation uses tree-sitter which is
+ * out of scope here):
+ *
+ *   1. Files whose language uses brace-based block structure (the C family:
+ *      ts/js/tsx/jsx/rs/go/java/kt/cs/c/cpp/h/hpp/php/swift/scala/dart/…)
+ *      are split at TOP-LEVEL function/method/class boundaries. A brace-
+ *      counting scanner finds the column-0 (or nearly top-level) brace ranges
+ *      that begin with a function/class signature, and each such range becomes
+ *      a chunk. The scanner is indentation-aware: a `{` at the top level of a
+ *      block-structure file (column 0, or preceded only by whitespace at a
+ *      low indentation that matches a declaration) opens a boundary.
+ *
+ *   2. Python (significant-whitespace): top-level `def`/`class` lines open
+ *      boundaries; the chunk extends to the next top-level declaration.
+ *
+ *   3. Everything else (markdown, json, yaml, sh, txt, files with no clear
+ *      boundaries, files too big for the boundary scanner) falls back to
+ *      fixed-size line windows with a small overlap.
+ *
+ * Limitations (documented, accepted):
+ *   - Regex/indentation heuristics can mis-split unusual code (a `{` on its
+ *     own line at column 0 that is actually a continuation, C++ templates,
+ *     nested namespaces, decorators, multiline signatures). A mis-split
+ *     produces a chunk that is still a valid file:line citation, just with
+ *     slightly less ideal boundaries — the semantic search stays usable.
+ *   - Very large single files (e.g. a 5k-line bundle) are chunked by line
+ *     windows, not by function — acceptable for this project's scale.
+ */
+
+import * as fs from "node:fs"
+import * as path from "node:path"
+
+import { MAX_CHUNK_CHARS } from "./embedder.js"
+
+/** A chunk of a file with its line citation. */
+export interface Chunk {
+	/** Workspace-relative POSIX path. */
+	file: string
+	/** 1-based inclusive start line. */
+	startLine: number
+	/** 1-based inclusive end line. */
+	endLine: number
+	/** Chunk text (the exact lines startLine..endLine). */
+	content: string
+}
+
+/** Max lines per fallback window. */
+export const FALLBACK_WINDOW_LINES = 60
+/** Overlap lines between fallback windows (keeps boundaries fuzzy-joined). */
+export const FALLBACK_OVERLAP_LINES = 10
+/** Files larger than this (lines) use fallback chunking even if they have boundaries. */
+export const MAX_BOUNDARY_SCAN_LINES = 3000
+
+/** `{`-using languages whose top-level declarations we can find with brace counting. */
+const BRACE_LANGS = new Set([
+	".ts",
+	".tsx",
+	".js",
+	".jsx",
+	".mjs",
+	".cjs",
+	".rs",
+	".go",
+	".java",
+	".kt",
+	".kts",
+	".cs",
+	".c",
+	".h",
+	".cpp",
+	".hpp",
+	".php",
+	".swift",
+	".scala",
+	".dart",
+	".css",
+])
+
+/** Python / significant-whitespace languages chunked by top-level def/class. */
+const INDENT_LANGS = new Set([".py", ".rb", ".ex", ".exs", ".el", ".sh", ".bash", ".zsh"])
+
+/** Lines that look like a C-family declaration (function/method/class). */
+const DECL_RE =
+	/^(export\s+)?(abstract\s+|async\s+|static\s+|public\s+|private\s+|protected\s+|internal\s+|final\s+|override\s+|sealed\s+)*\s*(function|def|class|interface|struct|enum|trait|impl|namespace|module|type|fn)\b/
+
+/** A `{`-using file's top-level brace ranges, in line order. */
+function findTopLevelBraceRanges(lines: string[]): Array<{ start: number; end: number }> {
+	const ranges: Array<{ start: number; end: number }> = []
+	let depth = 0
+	let rangeStart: number | undefined
+
+	for (let i = 0; i < lines.length; i++) {
+		const line = lines[i]
+		for (const ch of line) {
+			if (ch === "{") {
+				if (depth === 0) {
+					rangeStart = i
+				}
+				depth++
+			} else if (ch === "}") {
+				depth--
+				if (depth === 0 && rangeStart !== undefined) {
+					ranges.push({ start: rangeStart, end: i })
+					rangeStart = undefined
+				}
+			}
+		}
+	}
+	return ranges
+}
+
+/** True when the line starts a top-level declaration (column-0-ish keyword). */
+function isTopLevelDeclaration(line: string): boolean {
+	const trimmed = line.trim()
+	if (trimmed === "" || trimmed.startsWith("//") || trimmed.startsWith("/*") || trimmed.startsWith("*")) {
+		return false
+	}
+	// Column 0 check: the declaration must start at the very left edge (or a
+	// leading tab that's effectively top level). This avoids matching
+	// declarations nested inside methods.
+	const indent = line.match(/^(\s*)/)?.[1] ?? ""
+	if (indent.length > 0 && !indent.startsWith("\t")) {
+		// Indented with spaces → nested (Python-style or inside a block) → not top level.
+		return false
+	}
+	return DECL_RE.test(trimmed)
+}
+
+/** Chunk a brace-based file by top-level declaration ranges. */
+function chunkBraceFile(lines: string[], file: string): Chunk[] {
+	const ranges = findTopLevelBraceRanges(lines)
+	if (ranges.length === 0) {
+		return []
+	}
+	const chunks: Chunk[] = []
+	let cursor = 0
+	for (const range of ranges) {
+		const start = range.start
+		const end = range.end
+		// Lines between the previous range and this one (head imports,
+		// module-level constants, standalone statements like `const helper = …`
+		// that carry no braces) must not vanish from the index — emit them as
+		// their own chunk when they contain anything non-blank. Blank-only
+		// gaps are not worth a citation.
+		if (cursor < start) {
+			const gap = lines.slice(cursor, start)
+			if (gap.some((l) => l.trim() !== "")) {
+				chunks.push({
+					file,
+					startLine: cursor + 1,
+					endLine: start,
+					content: gap.join("\n"),
+				})
+			}
+		}
+		// The brace range itself (function/class/struct body). A tiny stub
+		// (e.g. a one-line `{ }`) is left to the next gap slice instead of
+		// getting its own chunk.
+		if (end - start >= 2) {
+			chunks.push({
+				file,
+				startLine: start + 1,
+				endLine: end + 1,
+				content: lines.slice(start, end + 1).join("\n"),
+			})
+			cursor = end + 1
+		}
+	}
+	// Any leftover lines after the last range (file tail) become a small chunk
+	// (skipped when it would be an empty/whitespace-only stub — the trailing
+	// blank line after the final `}` is not worth a citation).
+	if (cursor < lines.length) {
+		const tail = lines.slice(cursor)
+		if (tail.some((l) => l.trim() !== "")) {
+			chunks.push({
+				file,
+				startLine: cursor + 1,
+				endLine: lines.length,
+				content: tail.join("\n"),
+			})
+		}
+	}
+	return chunks
+}
+
+/** Chunk a Python-style file by top-level def/class/block boundaries. */
+function chunkIndentFile(lines: string[], file: string): Chunk[] {
+	const chunks: Chunk[] = []
+	let start = 0
+	for (let i = 0; i < lines.length; i++) {
+		const trimmed = lines[i].trim()
+		// A top-level (column 0) def/class/import/if/for/while opens a new chunk.
+		if (/^(def|class|async\s+def|import|from|if|for|while|with|try|except|@)\b/.test(trimmed) && !lines[i].startsWith(" ")) {
+			if (i > start) {
+				chunks.push({ file, startLine: start + 1, endLine: i, content: lines.slice(start, i).join("\n") })
+			}
+			start = i
+		}
+	}
+	if (start < lines.length) {
+		chunks.push({ file, startLine: start + 1, endLine: lines.length, content: lines.slice(start).join("\n") })
+	}
+	return chunks
+}
+
+/** Fallback: fixed-size line windows with overlap. */
+function chunkFallback(lines: string[], file: string): Chunk[] {
+	const chunks: Chunk[] = []
+	const windowSize = FALLBACK_WINDOW_LINES
+	const overlap = FALLBACK_OVERLAP_LINES
+	if (lines.length <= windowSize) {
+		chunks.push({ file, startLine: 1, endLine: lines.length, content: lines.join("\n") })
+		return chunks
+	}
+	let start = 0
+	while (start < lines.length) {
+		const end = Math.min(lines.length, start + windowSize)
+		chunks.push({ file, startLine: start + 1, endLine: end, content: lines.slice(start, end).join("\n") })
+		if (end >= lines.length) {
+			break
+		}
+		start = end - overlap
+	}
+	return chunks
+}
+
+/**
+ * Split one oversized chunk into pieces of at most MAX_CHUNK_CHARS (the cap
+ * lives in embedder.ts — the OpenRouter embeddings endpoint rejects any single
+ * input over the model's limit, seen live as an HTTP 422 on an 8.8M-char
+ * minified/data chunk and an HTTP 400 at ~100k chars on qwen/qwen3-embedding-4b
+ * whose context is 40960 tokens). Line-based windowing alone cannot guarantee
+ * the cap (a file can be a single giant line), so every chunk is split down to
+ * at most MAX_CHUNK_CHARS, preferring line boundaries and hard-slicing
+ * mid-line only for unbroken giant lines.
+ */
+function splitOversizedChunk(chunk: Chunk): Chunk[] {
+	if (chunk.content.length <= MAX_CHUNK_CHARS) {
+		return [chunk]
+	}
+	const out: Chunk[] = []
+	let cursor = 0
+	let startLine = chunk.startLine
+	while (cursor < chunk.content.length) {
+		let end = Math.min(cursor + MAX_CHUNK_CHARS, chunk.content.length)
+		// Back off to the previous newline (consuming it) unless this is the
+		// final slice or the window is one unbroken line.
+		if (end < chunk.content.length) {
+			const nl = chunk.content.lastIndexOf("\n", end - 1)
+			if (nl > cursor) {
+				end = nl + 1
+			}
+		}
+		const piece = chunk.content.slice(cursor, end)
+		const newlineCount = (piece.match(/\n/g) ?? []).length
+		// Line bookkeeping: a trailing \n terminates the final line of the
+		// piece, so it covers newlineCount lines; otherwise it covers
+		// newlineCount + 1. A piece with no \n at all is a mid-line hard slice
+		// of one giant line, so it cites the same line and the next slice
+		// starts on the same line number.
+		const coversLines = newlineCount + (piece.endsWith("\n") ? 0 : 1)
+		out.push({
+			file: chunk.file,
+			startLine,
+			endLine: startLine + coversLines - 1,
+			content: piece,
+		})
+		if (newlineCount > 0) {
+			startLine += coversLines
+		}
+		cursor = end
+	}
+	return out
+}
+
+/**
+ * Split a source file into line-cited chunks.
+ *
+ * @param absPath  absolute path of the file to chunk
+ * @param relPath  workspace-relative POSIX path (used for citations)
+ */
+function chunkFileUncapped(absPath: string, relPath: string): Chunk[] {
+	let content: string
+	try {
+		content = fs.readFileSync(absPath, "utf-8")
+	} catch {
+		return []
+	}
+	const lines = content.split(/\r?\n/)
+	if (lines.length === 0 || (lines.length === 1 && lines[0].trim() === "")) {
+		return []
+	}
+	const ext = path.extname(relPath).toLowerCase()
+
+	// Very large files fall back to windows — the boundary scanner is O(n)
+	// brace counting but a 10k-line minified bundle isn't a boundary case worth
+	// scanning, and windowing is what the reference does for huge files too.
+	if (lines.length > MAX_BOUNDARY_SCAN_LINES) {
+		return chunkFallback(lines, relPath)
+	}
+
+	if (BRACE_LANGS.has(ext)) {
+		const chunks = chunkBraceFile(lines, relPath)
+		if (chunks.length > 0) {
+			return chunks
+		}
+		return chunkFallback(lines, relPath)
+	}
+
+	if (INDENT_LANGS.has(ext)) {
+		const chunks = chunkIndentFile(lines, relPath)
+		if (chunks.length > 0) {
+			return chunks
+		}
+		return chunkFallback(lines, relPath)
+	}
+
+	return chunkFallback(lines, relPath)
+}
+
+/**
+	* Split a source file into line-cited chunks, each capped at MAX_CHUNK_CHARS
+	* characters so no chunk can exceed the embedding endpoint's input limit.
+	*
+	* @param absPath  absolute path of the file to chunk
+	* @param relPath  workspace-relative POSIX path (used for citations)
+	*/
+export function chunkFile(absPath: string, relPath: string): Chunk[] {
+	return chunkFileUncapped(absPath, relPath).flatMap(splitOversizedChunk)
+}