membot 0.0.1 → 0.1.0

package/src/ingest/ingest.ts

@@ -0,0 +1,444 @@
+import type { AppContext } from "../context.ts";
+import { upsertBlob } from "../db/blobs.ts";
+import { insertChunksForVersion, rebuildFts } from "../db/chunks.ts";
+import { type FetcherKind, getCurrent, insertVersion, millisIso, type SourceType } from "../db/files.ts";
+import { asHelpful, HelpfulError } from "../errors.ts";
+import { chunkDeterministic } from "./chunker.ts";
+import { convert } from "./converter/index.ts";
+import { describe } from "./describer.ts";
+import { embed } from "./embedder.ts";
+import { fetchRemote } from "./fetcher.ts";
+import { readLocalFile, sha256Hex } from "./local-reader.ts";
+import { buildSearchText } from "./search-text.ts";
+import { type ResolvedLocalEntry, type ResolvedSource, resolveSource } from "./source-resolver.ts";
+
+export interface IngestInput {
+	source: string;
+	logical_path?: string;
+	include?: string;
+	exclude?: string;
+	follow_symlinks?: boolean;
+	refresh_frequency?: string;
+	fetcher_hint?: string;
+	change_note?: string;
+}
+
+export interface IngestEntryResult {
+	source_path: string;
+	logical_path: string;
+	version_id: string | null;
+	status: "ok" | "failed";
+	error?: string;
+	mime_type: string | null;
+	size_bytes: number;
+	fetcher: FetcherKind;
+	source_sha256: string;
+}
+
+export interface IngestResult {
+	ingested: IngestEntryResult[];
+	total: number;
+	ok: number;
+	failed: number;
+}
+
+/**
+ * Top-level ingest orchestrator. Resolves the source arg, dispatches to the
+ * right reader (local / remote / inline), runs the pipeline (convert →
+ * describe → chunk → embed → write), and returns one entry per matched
+ * file. Partial failures are reported per-entry; the entire call doesn't
+ * abort because one URL or PDF is bad.
+ */
+export async function ingest(input: IngestInput, ctx: AppContext): Promise<IngestResult> {
+	const resolved = await resolveSource(input.source, {
+		include: input.include,
+		exclude: input.exclude,
+		followSymlinks: input.follow_symlinks ?? true,
+	});
+
+	const refreshSec = parseDuration(input.refresh_frequency);
+
+	if (resolved.kind === "inline") {
+		return ingestInline(resolved.text, input, ctx, refreshSec);
+	}
+	if (resolved.kind === "url") {
+		return ingestUrl(resolved.url, input, ctx, refreshSec);
+	}
+	return ingestLocalFiles(resolved, input, ctx, refreshSec);
+}
+
+/** Ingest a single inline blob (source_type='inline'). */
+async function ingestInline(
+	text: string,
+	input: IngestInput,
+	ctx: AppContext,
+	refreshSec: number | null,
+): Promise<IngestResult> {
+	const logicalPath = input.logical_path ?? defaultInlinePath();
+	const bytes = new TextEncoder().encode(text);
+	const sha = sha256Hex(bytes);
+	const result: IngestEntryResult = {
+		source_path: "inline:",
+		logical_path: logicalPath,
+		version_id: null,
+		status: "ok",
+		mime_type: "text/markdown",
+		size_bytes: bytes.byteLength,
+		fetcher: "inline",
+		source_sha256: sha,
+	};
+	try {
+		const versionId = await persistVersion(ctx, {
+			logicalPath,
+			sourceType: "inline",
+			sourcePath: null,
+			sourceMtimeMs: null,
+			sourceSha: sha,
+			blobSha: null,
+			mime: "text/markdown",
+			bytes: null,
+			markdown: text,
+			fetcher: "inline",
+			fetcherServer: null,
+			fetcherTool: null,
+			fetcherArgs: null,
+			refreshSec,
+			changeNote: input.change_note ?? null,
+		});
+		result.version_id = versionId;
+	} catch (err) {
+		result.status = "failed";
+		result.error = errorMessage(err);
+	}
+	return summarize([result]);
+}
+
+/** Ingest one URL (source_type='remote'). */
+async function ingestUrl(
+	url: string,
+	input: IngestInput,
+	ctx: AppContext,
+	refreshSec: number | null,
+): Promise<IngestResult> {
+	const mcpxAdapter = ctx.mcpx
+		? {
+				async listTools() {
+					const tools = await ctx.mcpx!.listTools();
+					return tools;
+				},
+				async exec(server: string, tool: string, args: Record<string, unknown>) {
+					return ctx.mcpx!.exec(server, tool, args);
+				},
+			}
+		: null;
+
+	const logicalPath = input.logical_path ?? defaultLogicalForUrl(url);
+	const result: IngestEntryResult = {
+		source_path: url,
+		logical_path: logicalPath,
+		version_id: null,
+		status: "ok",
+		mime_type: null,
+		size_bytes: 0,
+		fetcher: "http",
+		source_sha256: "",
+	};
+
+	try {
+		const fetched = await fetchRemote(url, { hint: input.fetcher_hint, mcpx: mcpxAdapter });
+		result.mime_type = fetched.mimeType;
+		result.size_bytes = fetched.bytes.byteLength;
+		result.fetcher = fetched.fetcher;
+		result.source_sha256 = fetched.sha256;
+
+		const versionId = await pipelineForBytes(ctx, {
+			logicalPath,
+			bytes: fetched.bytes,
+			mime: fetched.mimeType,
+			source: url,
+			sourceType: "remote",
+			sourcePath: url,
+			sourceMtimeMs: null,
+			sourceSha: fetched.sha256,
+			fetcher: fetched.fetcher,
+			fetcherServer: fetched.fetcherServer,
+			fetcherTool: fetched.fetcherTool,
+			fetcherArgs: fetched.fetcherArgs,
+			refreshSec,
+			changeNote: input.change_note ?? null,
+		});
+		result.version_id = versionId;
+	} catch (err) {
+		result.status = "failed";
+		result.error = errorMessage(err);
+	}
+	return summarize([result]);
+}
+
+/** Ingest a list of local files (source_type='local'). One transaction per entry. */
+async function ingestLocalFiles(
+	resolved: Extract<ResolvedSource, { kind: "local-files" }>,
+	input: IngestInput,
+	ctx: AppContext,
+	refreshSec: number | null,
+): Promise<IngestResult> {
+	if (resolved.entries.length === 0) {
+		throw new HelpfulError({
+			kind: "input_error",
+			message: `Glob/path matched 0 files`,
+			hint: `Try a broader pattern (e.g. ./**/*.md) or relax --exclude.`,
+		});
+	}
+
+	const results: IngestEntryResult[] = [];
+	ctx.progress.start(resolved.entries.length, "ingest");
+	const isMulti = resolved.entries.length > 1;
+
+	for (const entry of resolved.entries) {
+		ctx.progress.tick(entry.relPath);
+		const logicalPath = pickLogicalPath(input.logical_path, entry, isMulti);
+		const result: IngestEntryResult = {
+			source_path: entry.absPath,
+			logical_path: logicalPath,
+			version_id: null,
+			status: "ok",
+			mime_type: null,
+			size_bytes: 0,
+			fetcher: "local",
+			source_sha256: "",
+		};
+		try {
+			const local = await readLocalFile(entry.absPath);
+			result.mime_type = local.mimeType;
+			result.size_bytes = local.sizeBytes;
+			result.source_sha256 = local.sha256;
+
+			const versionId = await pipelineForBytes(ctx, {
+				logicalPath,
+				bytes: local.bytes,
+				mime: local.mimeType,
+				source: entry.absPath,
+				sourceType: "local",
+				sourcePath: entry.absPath,
+				sourceMtimeMs: local.mtimeMs,
+				sourceSha: local.sha256,
+				fetcher: "local",
+				fetcherServer: null,
+				fetcherTool: null,
+				fetcherArgs: null,
+				refreshSec,
+				changeNote: input.change_note ?? null,
+			});
+			result.version_id = versionId;
+		} catch (err) {
+			result.status = "failed";
+			result.error = errorMessage(err);
+		}
+		results.push(result);
+	}
+	ctx.progress.done(`ingested ${results.filter((r) => r.status === "ok").length}/${results.length}`);
+
+	return summarize(results);
+}
+
+interface PipelineParams {
+	logicalPath: string;
+	bytes: Uint8Array;
+	mime: string;
+	source: string;
+	sourceType: SourceType;
+	sourcePath: string | null;
+	sourceMtimeMs: number | null;
+	sourceSha: string;
+	fetcher: FetcherKind;
+	fetcherServer: string | null;
+	fetcherTool: string | null;
+	fetcherArgs: Record<string, unknown> | null;
+	refreshSec: number | null;
+	changeNote: string | null;
+}
+
+/**
+ * Run the bytes-in / version-out pipeline: store the blob, convert to
+ * markdown, describe, chunk, embed, and write a new files row + chunks
+ * rows under a fresh version_id. Returns the version_id so callers can
+ * report it back.
+ */
+async function pipelineForBytes(ctx: AppContext, p: PipelineParams): Promise<string> {
+	await upsertBlob(ctx.db, {
+		sha256: p.sourceSha,
+		mime_type: p.mime,
+		size_bytes: p.bytes.byteLength,
+		bytes: p.bytes,
+	});
+
+	const conversion = await convert(p.bytes, p.mime, p.source, ctx.config.llm);
+	const markdown = conversion.markdown;
+	const contentSha = sha256Hex(new TextEncoder().encode(markdown));
+
+	return persistVersion(ctx, {
+		logicalPath: p.logicalPath,
+		sourceType: p.sourceType,
+		sourcePath: p.sourcePath,
+		sourceMtimeMs: p.sourceMtimeMs,
+		sourceSha: p.sourceSha,
+		blobSha: p.sourceSha,
+		mime: p.mime,
+		bytes: p.bytes,
+		markdown,
+		contentSha,
+		fetcher: p.fetcher,
+		fetcherServer: p.fetcherServer,
+		fetcherTool: p.fetcherTool,
+		fetcherArgs: p.fetcherArgs,
+		refreshSec: p.refreshSec,
+		changeNote: p.changeNote,
+	});
+}
+
+interface PersistParams {
+	logicalPath: string;
+	sourceType: SourceType;
+	sourcePath: string | null;
+	sourceMtimeMs: number | null;
+	sourceSha: string;
+	blobSha: string | null;
+	mime: string;
+	bytes: Uint8Array | null;
+	markdown: string;
+	contentSha?: string;
+	fetcher: FetcherKind;
+	fetcherServer: string | null;
+	fetcherTool: string | null;
+	fetcherArgs: Record<string, unknown> | null;
+	refreshSec: number | null;
+	changeNote: string | null;
+}
+
+/**
+ * Insert a new (logical_path, version_id) row plus its chunks. Description
+ * is generated on every ingest (LLM with deterministic fallback). The
+ * embedded text per chunk is `<path>\n<description>\n\n<body>`, stored
+ * verbatim as `chunks.search_text` and later FTS-indexed.
+ */
+async function persistVersion(ctx: AppContext, p: PersistParams): Promise<string> {
+	const description = await describe(p.logicalPath, p.mime, p.markdown, ctx.config.llm);
+	const chunks = chunkDeterministic(p.markdown, ctx.config.chunker);
+	const searchTexts = chunks.map((c) => buildSearchText(p.logicalPath, description, c.content));
+	let embeddings: number[][];
+	try {
+		embeddings = await embed(searchTexts, ctx.config.embedding_model);
+	} catch (err) {
+		throw asHelpful(
+			err,
+			`while embedding chunks for ${p.logicalPath}`,
+			"Run `bun run prebuild` to apply the transformers WASM patch, or set a different config.embedding_model.",
+		);
+	}
+
+	const versionId = millisIso(Date.now());
+	const contentSha = p.contentSha ?? sha256Hex(new TextEncoder().encode(p.markdown));
+	await insertVersion(ctx.db, {
+		logical_path: p.logicalPath,
+		version_id: versionId,
+		source_type: p.sourceType,
+		source_path: p.sourcePath,
+		source_mtime_ms: p.sourceMtimeMs,
+		source_sha256: p.sourceSha,
+		blob_sha256: p.blobSha,
+		content_sha256: contentSha,
+		content: p.markdown,
+		description,
+		mime_type: p.mime,
+		size_bytes: p.bytes?.byteLength ?? new TextEncoder().encode(p.markdown).byteLength,
+		fetcher: p.fetcher,
+		fetcher_server: p.fetcherServer,
+		fetcher_tool: p.fetcherTool,
+		fetcher_args: p.fetcherArgs,
+		refresh_frequency_sec: p.refreshSec,
+		refreshed_at: new Date().toISOString(),
+		last_refresh_status: "ok",
+		change_note: p.changeNote,
+	});
+
+	await insertChunksForVersion(
+		ctx.db,
+		p.logicalPath,
+		versionId,
+		chunks.map((c, i) => ({
+			chunk_index: c.index,
+			chunk_content: c.content,
+			search_text: searchTexts[i] ?? buildSearchText(p.logicalPath, description, c.content),
+			embedding: embeddings[i] ?? new Array(embeddings[0]?.length ?? 0).fill(0),
+		})),
+	);
+	await rebuildFts(ctx.db);
+	return versionId;
+}
+
+/**
+ * Pick the logical path for a single matched entry. For a single-file
+ * ingest with explicit `logical_path`, use it as-is. For multi-entry
+ * ingests with `logical_path` set, treat it as a *prefix* under which
+ * each entry's relative path is placed.
+ */
+function pickLogicalPath(explicit: string | undefined, entry: ResolvedLocalEntry, isMulti: boolean): string {
+	if (!explicit) return entry.relPath.replaceAll("\\", "/");
+	if (!isMulti) return explicit;
+	const prefix = explicit.endsWith("/") ? explicit.slice(0, -1) : explicit;
+	return `${prefix}/${entry.relPath.replaceAll("\\", "/")}`;
+}
+
+/** Default logical path for an ingested URL — host + path, sanitized. */
+function defaultLogicalForUrl(url: string): string {
+	try {
+		const u = new URL(url);
+		const tail = u.pathname.replace(/^\/+/, "").replaceAll("/", "_") || "root";
+		return `urls/${u.hostname}/${tail || "root"}`;
+	} catch {
+		return `urls/${url.replace(/[^a-z0-9.-]/gi, "_")}`;
+	}
+}
+
+function defaultInlinePath(): string {
+	return `inline/${new Date().toISOString().replace(/[:.]/g, "-")}.md`;
+}
+
+/**
+ * Convert a duration string (`5m`, `1h`, `24h`, `7d`) to seconds. Returns
+ * null when the input is undefined / blank, throws a HelpfulError on
+ * malformed input.
+ */
+export function parseDuration(input: string | null | undefined): number | null {
+	if (!input?.trim()) return null;
+	const m = input.trim().match(/^(\d+)([smhd])$/i);
+	if (!m) {
+		throw new HelpfulError({
+			kind: "input_error",
+			message: `invalid duration: ${input}`,
+			hint: `Use forms like 5m, 1h, 24h, 7d.`,
+		});
+	}
+	const n = Number(m[1]);
+	const unit = m[2]?.toLowerCase() ?? "s";
+	const multiplier = unit === "s" ? 1 : unit === "m" ? 60 : unit === "h" ? 3600 : 86400;
+	return n * multiplier;
+}
+
+/** Roll a list of per-entry results into the top-level summary shape. */
+function summarize(entries: IngestEntryResult[]): IngestResult {
+	let ok = 0;
+	let failed = 0;
+	for (const e of entries) {
+		if (e.status === "ok") ok += 1;
+		else failed += 1;
+	}
+	return { ingested: entries, total: entries.length, ok, failed };
+}
+
+function errorMessage(err: unknown): string {
+	if (err instanceof Error) return err.message;
+	return String(err);
+}
+
+export { getCurrent };

package/src/ingest/local-reader.ts

@@ -0,0 +1,64 @@
+import { createHash } from "node:crypto";
+import { stat } from "node:fs/promises";
+import { asHelpful } from "../errors.ts";
+
+export interface LocalRead {
+	bytes: Uint8Array;
+	sha256: string;
+	mtimeMs: number;
+	sizeBytes: number;
+	mimeType: string;
+}
+
+/**
+ * Best-effort filename → MIME mapping using Bun's built-in resolver. Strips
+ * the `;charset=...` suffix Bun adds for text types so the value is safe to
+ * compare with `===` against fixed MIME strings in the converter dispatch.
+ * Lowercases the path's basename so `.PNG` and `.png` resolve identically.
+ */
+export function mimeFromPath(path: string): string {
+	const lastSlash = Math.max(path.lastIndexOf("/"), path.lastIndexOf("\\"));
+	const base =
+		lastSlash >= 0 ? path.slice(0, lastSlash + 1) + path.slice(lastSlash + 1).toLowerCase() : path.toLowerCase();
+	const raw = Bun.file(base).type;
+	const mime = raw.split(";")[0]?.trim();
+	if (!mime || mime === "application/octet-stream") return "application/octet-stream";
+	return mime;
+}
+
+/**
+ * Read a local file: bytes, sha256, last-mtime, size, and an inferred MIME
+ * type. Used by the ingest pipeline as the universal entry point for
+ * source_type='local'.
+ */
+export async function readLocalFile(path: string): Promise<LocalRead> {
+	let stats: Awaited<ReturnType<typeof stat>>;
+	try {
+		stats = await stat(path);
+	} catch (err) {
+		throw asHelpful(
+			err,
+			`while stat'ing ${path}`,
+			`Check that the path exists and you have read access. \`ls -la ${path}\`.`,
+			"not_found",
+		);
+	}
+	const file = Bun.file(path);
+	const ab = await file.arrayBuffer();
+	const bytes = new Uint8Array(ab);
+	const sha256 = sha256Hex(bytes);
+	return {
+		bytes,
+		sha256,
+		mtimeMs: stats.mtimeMs,
+		sizeBytes: stats.size,
+		mimeType: mimeFromPath(path),
+	};
+}
+
+/** Compute a hex SHA-256 over the provided bytes. */
+export function sha256Hex(bytes: Uint8Array): string {
+	const hash = createHash("sha256");
+	hash.update(bytes);
+	return hash.digest("hex");
+}

package/src/ingest/search-text.ts

@@ -0,0 +1,18 @@
+/**
+ * Build the exact string that gets embedded AND FTS-indexed for a chunk.
+ * Format:
+ *   <logical_path>
+ *   <description>
+ *   <blank line>
+ *   <chunk_content>
+ *
+ * The path + description prefix lifts recall on filename-only or
+ * description-only queries (e.g. "the OAuth diagram" finds an empty PNG
+ * row whose body is only its caption). Stored verbatim as
+ * `chunks.search_text` so retrieval consumers can return clean snippets
+ * by reading `chunks.chunk_content` separately.
+ */
+export function buildSearchText(logicalPath: string, description: string | null, chunkContent: string): string {
+	const desc = (description ?? "").trim();
+	return `${logicalPath}\n${desc}\n\n${chunkContent}`;
+}

package/src/ingest/source-resolver.ts

@@ -0,0 +1,186 @@
+import { realpath, stat } from "node:fs/promises";
+import { isAbsolute, join, relative, resolve, sep } from "node:path";
+import picomatch from "picomatch";
+import { asHelpful, HelpfulError } from "../errors.ts";
+
+export type ResolvedSource =
+	| { kind: "inline"; text: string; logicalHint: string | null }
+	| { kind: "url"; url: string; logicalHint: string | null }
+	| { kind: "local-files"; entries: ResolvedLocalEntry[]; basePath: string };
+
+export interface ResolvedLocalEntry {
+	absPath: string;
+	/** Path relative to the base; used to derive a default logical_path. */
+	relPath: string;
+}
+
+export interface ResolveOptions {
+	include?: string;
+	exclude?: string;
+	followSymlinks?: boolean;
+}
+
+const DEFAULT_EXCLUDES = ["**/node_modules/**", "**/.git/**", "**/.DS_Store", "**/dist/**", "**/.cache/**"];
+
+/**
+ * Polymorphic source-arg expander. Accepts:
+ *   - "inline:<text>"             → inline literal
+ *   - "http://..." | "https://..." → URL (fetched downstream by fetcher)
+ *   - existing file               → single-file local
+ *   - existing directory          → recursive walk (symlinks via realpath cache)
+ *   - glob pattern                → picomatch-filtered walk
+ *
+ * Symlink loops are broken via a realpath cache. Include / exclude are
+ * applied to the entry's path *relative to the base* so users don't need
+ * absolute-path globs.
+ */
+export async function resolveSource(source: string, options: ResolveOptions = {}): Promise<ResolvedSource> {
+	if (source.startsWith("inline:")) {
+		return { kind: "inline", text: source.slice("inline:".length), logicalHint: null };
+	}
+	if (source.startsWith("http://") || source.startsWith("https://")) {
+		return { kind: "url", url: source, logicalHint: null };
+	}
+
+	const followSymlinks = options.followSymlinks !== false;
+	const includeMatchers = (options.include ?? "**/*")
+		.split(",")
+		.map((g) => g.trim())
+		.filter(Boolean);
+	const excludeMatchers = [
+		...DEFAULT_EXCLUDES,
+		...(options.exclude ?? "")
+			.split(",")
+			.map((g) => g.trim())
+			.filter(Boolean),
+	];
+
+	if (isGlob(source)) {
+		const base = globBase(source);
+		try {
+			const realBase = await realpath(base);
+			return walk(realBase, [source, ...includeMatchers], excludeMatchers, followSymlinks);
+		} catch (err) {
+			throw asHelpful(
+				err,
+				`while resolving glob base ${base}`,
+				`Check that the directory ${base} exists.`,
+				"not_found",
+			);
+		}
+	}
+
+	const abs = resolve(source);
+	let st: Awaited<ReturnType<typeof stat>>;
+	try {
+		st = await stat(abs);
+	} catch (err) {
+		throw asHelpful(
+			err,
+			`while stat'ing ${source}`,
+			`Check that the path exists. Run \`ls ${source}\`. For URLs, prefix with http:// or https://.`,
+			"not_found",
+		);
+	}
+
+	if (st.isFile()) {
+		return {
+			kind: "local-files",
+			basePath: abs,
+			entries: [{ absPath: abs, relPath: source.split(sep).pop() ?? source }],
+		};
+	}
+
+	if (st.isDirectory()) {
+		const realBase = await realpath(abs);
+		return walk(realBase, includeMatchers, excludeMatchers, followSymlinks);
+	}
+
+	throw new HelpfulError({
+		kind: "input_error",
+		message: `${source} is neither a file, directory, nor URL`,
+		hint: `Pass a file path, directory, glob (e.g. "docs/**/*.md"), URL, or "inline:<text>".`,
+	});
+}
+
+/** Crude glob detector — matches what picomatch treats as a pattern. */
+export function isGlob(s: string): boolean {
+	return /[*?[\]{}!]/.test(s);
+}
+
+/** Take the static directory prefix of a glob (everything before the first wildcard). */
+export function globBase(glob: string): string {
+	const parts = glob.split(sep);
+	const out: string[] = [];
+	for (const p of parts) {
+		if (/[*?[\]{}!]/.test(p)) break;
+		out.push(p);
+	}
+	const base = out.join(sep);
+	return base.length === 0 || !isAbsolute(base) ? resolve(base || ".") : base;
+}
+
+/**
+ * Recursively walk `base`, returning files matched by `includes` and not
+ * matched by `excludes`. Both globsets match against the entry's path
+ * relative to `base`. Symlinks are followed when `followSymlinks` is true,
+ * with cycles detected via a realpath cache.
+ */
+async function walk(
+	base: string,
+	includes: string[],
+	excludes: string[],
+	followSymlinks: boolean,
+): Promise<ResolvedSource> {
+	const seen = new Set<string>();
+	const entries: ResolvedLocalEntry[] = [];
+
+	const isInclude = picomatch(includes, { dot: false, nocase: false });
+	const isExclude = excludes.length ? picomatch(excludes, { dot: false }) : null;
+
+	const queue: string[] = [base];
+	while (queue.length > 0) {
+		const cur = queue.shift();
+		if (cur === undefined) break;
+		let real: string;
+		try {
+			real = await realpath(cur);
+		} catch {
+			continue;
+		}
+		if (seen.has(real)) continue;
+		seen.add(real);
+		let st: Awaited<ReturnType<typeof stat>>;
+		try {
+			st = await stat(real);
+		} catch {
+			continue;
+		}
+		if (st.isSymbolicLink() && !followSymlinks) continue;
+		if (st.isDirectory()) {
+			let names: string[];
+			try {
+				names = await readdir(real);
+			} catch {
+				continue;
+			}
+			for (const name of names) {
+				queue.push(join(real, name));
+			}
+			continue;
+		}
+		if (!st.isFile()) continue;
+		const rel = relative(base, real);
+		const relForMatch = rel.length === 0 ? (cur.split(sep).pop() ?? cur) : rel;
+		if (isExclude?.(relForMatch)) continue;
+		if (!isInclude(relForMatch)) continue;
+		entries.push({ absPath: real, relPath: relForMatch });
+	}
+
+	return { kind: "local-files", basePath: base, entries };
+}
+
+async function readdir(path: string): Promise<string[]> {
+	const fs = await import("node:fs/promises");
+	return fs.readdir(path);
+}