@lucas-bur/pix 0.9.1 → 0.11.0

package/dist/index.mjs

@@ -2,16 +2,18 @@
 import { createRequire } from "node:module";
 import { Args, CliConfig, Command, Options } from "@effect/cli";
 import { NodeContext, NodeRuntime } from "@effect/platform-node";
-import { Clock, Context, Data, Effect, Layer, Option, Ref } from "effect";
+import { Clock, Context, Data, Effect, Exit, Layer, Option, Ref, Stream } from "effect";
+import * as Chunk from "effect/Chunk";
 import { styleText } from "node:util";
 import * as clack from "@clack/prompts";
-import crypto from "node:crypto";
 import { FileSystem } from "@effect/platform";
+import crypto from "node:crypto";
 import { env } from "@huggingface/transformers";
 import ignore from "ignore";
 //#region src/domain/ports.ts
 var ConfigStore = class extends Context.Tag("ConfigStore")() {};
 var Scanner = class extends Context.Tag("Scanner")() {};
+var ContentExtractor = class extends Context.Tag("ContentExtractor")() {};
 var Chunker = class extends Context.Tag("Chunker")() {};
 var Embedder = class extends Context.Tag("Embedder")() {};
 var VectorStore = class extends Context.Tag("VectorStore")() {};
@@ -73,6 +75,7 @@ const withInteractive = (activeRef, acquire, setActive, release, effect) => Ref.
 /** Display implementation using @clack/prompts for interactive terminal output */
 const ClackDisplay = { layer: Layer.effect(Display, Effect.gen(function* () {
 	const activeRef = yield* Ref.make(null);
+	const lastSpinnerMsg = yield* Ref.make("");
 	return {
 		intro: (title) => Effect.sync(() => clack.intro(styleText("inverse", ` ${title} `))),
 		outro: (message) => Effect.sync(() => clack.outro(message)),
@@ -83,11 +86,17 @@ const ClackDisplay = { layer: Layer.effect(Display, Effect.gen(function* () {
 			const s = clack.spinner();
 			s.start(message);
 			return s;
-		}), (s) => ({
+		}).pipe(Effect.tap(() => Ref.set(lastSpinnerMsg, message))), (s) => ({
 			type: "spinner",
 			handle: s
-		}), (s, exit) => Effect.sync(() => s.stop(exit._tag === "Success" ? message : `${message} (failed)`)), effect),
-		progress: (opts, effect) => withInteractive(activeRef, Effect.sync(() => {
+		}), (s, exit) => lastSpinnerMsg.pipe(Effect.flatMap((lastMsg) => Effect.sync(() => s.stop(exit._tag === "Success" && lastMsg ? lastMsg : `${message} (failed)`)))), effect),
+		progress: (opts, effect) => Effect.gen(function* () {
+			const current = yield* Ref.get(activeRef);
+			if (current && current.type === "spinner") {
+				const msg = yield* Ref.get(lastSpinnerMsg);
+				current.handle.stop(msg || opts.message);
+				yield* Ref.set(activeRef, null);
+			}
 			const bar = clack.progress({
 				max: opts.max,
 				style: opts.style ?? "heavy",
@@ -95,16 +104,27 @@ const ClackDisplay = { layer: Layer.effect(Display, Effect.gen(function* () {
 				indicator: opts.indicator ?? "dots"
 			});
 			bar.start(opts.message);
-			return bar;
-		}), (bar) => ({
-			type: "progress",
-			handle: bar,
-			value: 0,
-			max: opts.max
-		}), (bar, exit) => Effect.sync(() => exit._tag === "Success" ? bar.stop(opts.message) : bar.error(opts.message)), effect),
+			yield* Ref.set(activeRef, {
+				type: "progress",
+				handle: bar,
+				value: 0,
+				max: opts.max
+			});
+			const exit = yield* Effect.exit(effect);
+			yield* Ref.set(activeRef, null);
+			if (Exit.isSuccess(exit)) {
+				bar.stop(opts.message);
+				return exit.value;
+			}
+			bar.error(opts.message);
+			return yield* Effect.failCause(exit.cause);
+		}),
 		updateInteractive: (payload) => Ref.get(activeRef).pipe(Effect.flatMap((active) => {
 			if (!active) return Effect.void;
-			if (active.type === "spinner") return Effect.sync(() => active.handle.message(payloadText(payload)));
+			if (active.type === "spinner") {
+				const msg = payloadText(payload);
+				return Effect.sync(() => active.handle.message(msg)).pipe(Effect.andThen(Ref.set(lastSpinnerMsg, msg)));
+			}
 			const delta = computeDelta(payload, {
 				value: active.value,
 				max: active.max
@@ -140,19 +160,198 @@ const DEFAULT_CONFIG = {
 	chunkLines: 60,
 	overlapLines: 10,
 	chunkConcurrency: 8,
-	files: {},
+	skipExtensions: [],
+	ignoredPaths: [
+		".pix",
+		"node_modules",
+		".git",
+		"dist",
+		"build",
+		".next",
+		".vscode",
+		"coverage",
+		"*-lock.yaml",
+		"*-lock.json",
+		"*.lock",
+		".vite-hooks",
+		".fallow"
+	],
 	embedder: {
 		model: "Xenova/all-MiniLM-L6-v2",
 		device: "auto",
-		dtype: "fp32"
+		dtype: "fp32",
+		batchSize: 16
 	}
 };
 //#endregion
-//#region src/application/index-project.ts
+//#region src/domain/errors.ts
+/** Config file or directory does not exist. Run pix init first. */
+var ConfigNotFoundError = class extends Data.TaggedError("ConfigNotFoundError") {};
+/** Config file exists but contains invalid JSON. */
+var ConfigMalformedError = class extends Data.TaggedError("ConfigMalformedError") {};
+/** Index files (chunks.jsonl, vectors.bin) do not exist. Run pix index first. */
+var NoIndexError = class extends Data.TaggedError("NoIndexError") {};
+/** Disk is full — write operation could not complete. */
+var DiskFullError = class extends Data.TaggedError("DiskFullError") {};
+/** Generic index store I/O failure (read, write, delete). */
+var StoreError = class extends Data.TaggedError("StoreError") {};
+/** Source file could not be read during chunking (binary, permissions, encoding). */
+var ChunkerError = class extends Data.TaggedError("ChunkerError") {};
+/** Embedding model could not be downloaded or loaded. */
+var ModelLoadError = class extends Data.TaggedError("ModelLoadError") {};
+/** Embedding model failed during inference. */
+var InferenceError = class extends Data.TaggedError("InferenceError") {};
 /**
-* Use case: index project files. Pipeline: scan → chunk → embed → store. Depends on ConfigStore,
-* Scanner, Chunker, Embedder, VectorStore, Display via Effect tags.
+* Fatal scan failure — gitignore loading failed entirely. Non-fatal per-entry skips are reported
+* via ScanResult.skipped.
 */
+var ScanFailed = class extends Data.TaggedError("ScanFailed") {};
+/** File type is unsupported for text extraction. */
+var UnsupportedFormat = class extends Data.TaggedError("UnsupportedFormat") {};
+/** Text extraction failed for a supported file type. */
+var ExtractionFailed = class extends Data.TaggedError("ExtractionFailed") {};
+//#endregion
+//#region src/services/processors/identity.ts
+const identityProcessor = (file) => FileSystem.FileSystem.pipe(Effect.flatMap((fs) => fs.readFileString(file)), Effect.mapError((cause) => new ExtractionFailed({
+	message: `Failed to read file for extraction: ${file}`,
+	file,
+	cause
+})));
+//#endregion
+//#region src/services/processors/skip.ts
+const skipProcessor = (extension) => {
+	const error = new UnsupportedFormat({
+		message: `Unsupported file type: ${extension}`,
+		extension
+	});
+	return (_file) => Effect.fail(error);
+};
+//#endregion
+//#region src/services/processors/index.ts
+const DEFAULT_PROCESSOR_MAP = {
+	".ts": identityProcessor,
+	".tsx": identityProcessor,
+	".js": identityProcessor,
+	".jsx": identityProcessor,
+	".py": identityProcessor,
+	".rs": identityProcessor,
+	".go": identityProcessor,
+	".java": identityProcessor,
+	".c": identityProcessor,
+	".cpp": identityProcessor,
+	".h": identityProcessor,
+	".hpp": identityProcessor,
+	".json": identityProcessor,
+	".yaml": identityProcessor,
+	".yml": identityProcessor,
+	".toml": identityProcessor,
+	".xml": identityProcessor,
+	".csv": identityProcessor,
+	".md": identityProcessor,
+	".mdx": identityProcessor,
+	".txt": identityProcessor,
+	".rst": identityProcessor,
+	".html": identityProcessor,
+	".css": identityProcessor,
+	".scss": identityProcessor,
+	".less": identityProcessor,
+	".sql": identityProcessor,
+	".graphql": identityProcessor,
+	".sh": identityProcessor,
+	".bash": identityProcessor,
+	".ps1": identityProcessor,
+	".bat": identityProcessor,
+	".cmake": identityProcessor,
+	".dockerfile": identityProcessor,
+	dockerfile: identityProcessor,
+	makefile: identityProcessor,
+	gemfile: identityProcessor,
+	".pdf": skipProcessor(".pdf"),
+	".png": skipProcessor(".png"),
+	".jpg": skipProcessor(".jpg"),
+	".jpeg": skipProcessor(".jpeg"),
+	".gif": skipProcessor(".gif"),
+	".svg": identityProcessor,
+	".ico": skipProcessor(".ico"),
+	".webp": skipProcessor(".webp"),
+	".mp3": skipProcessor(".mp3"),
+	".mp4": skipProcessor(".mp4"),
+	".wav": skipProcessor(".wav"),
+	".avi": skipProcessor(".avi"),
+	".mov": skipProcessor(".mov"),
+	".mkv": skipProcessor(".mkv"),
+	".exe": skipProcessor(".exe"),
+	".dll": skipProcessor(".dll"),
+	".so": skipProcessor(".so"),
+	".zip": skipProcessor(".zip"),
+	".tar": skipProcessor(".tar"),
+	".gz": skipProcessor(".gz"),
+	".7z": skipProcessor(".7z"),
+	".rar": skipProcessor(".rar"),
+	".ttf": skipProcessor(".ttf"),
+	".woff": skipProcessor(".woff"),
+	".woff2": skipProcessor(".woff2"),
+	".eot": skipProcessor(".eot"),
+	".otf": skipProcessor(".otf"),
+	".lock": identityProcessor,
+	lock: identityProcessor
+};
+/**
+* Builds the processor map by merging domain defaults with user-specified skip extensions. Skip
+* extensions override any existing mapping with a skip processor. Unknown extensions remain absent
+* from the map — callers decide how to handle them.
+*/
+function buildProcessorMap(skipExtensions) {
+	const mapped = { ...DEFAULT_PROCESSOR_MAP };
+	for (const ext of skipExtensions) mapped[ext] = skipProcessor(ext);
+	return mapped;
+}
+//#endregion
+//#region src/application/index-project.ts
+const deriveEffectiveConfig = (opts, config) => ({
+	batchSize: opts.batchSize ?? config.embedder.batchSize ?? 16,
+	concurrency: Math.max(1, opts.chunkConcurrency ?? config.chunkConcurrency ?? 8),
+	skipExtensions: opts.skipExtensions ? [...config.skipExtensions, ...opts.skipExtensions] : config.skipExtensions,
+	ignoredPaths: opts.ignorePaths ? [...config.ignoredPaths ?? DEFAULT_CONFIG.ignoredPaths, ...opts.ignorePaths] : config.ignoredPaths ?? DEFAULT_CONFIG.ignoredPaths,
+	ignoreGitignore: opts.ignoreGitignore ?? config.ignoreGitignore ?? false
+});
+function getExtension(file) {
+	const lastSlash = file.lastIndexOf("/");
+	const name = lastSlash >= 0 ? file.slice(lastSlash + 1) : file;
+	const dotIndex = name.lastIndexOf(".");
+	if (dotIndex === -1) return name.toLowerCase();
+	return name.slice(dotIndex).toLowerCase();
+}
+const classifyFiles = (files, processorMap) => {
+	const knownFiles = [];
+	const skippedFiles = [];
+	const unknownExtensions = /* @__PURE__ */ new Set();
+	for (const file of files) {
+		const ext = getExtension(file);
+		if (!processorMap[ext]) {
+			unknownExtensions.add(ext);
+			skippedFiles.push(file);
+		} else knownFiles.push(file);
+	}
+	return {
+		knownFiles,
+		skippedFiles,
+		unknownExtensions
+	};
+};
+const classifyAndCollectChunks = (knownFiles, extractor, chunker, concurrency, skipped) => Stream.fromIterable(knownFiles).pipe(Stream.mapEffect((file) => extractor.extract(file).pipe(Effect.flatMap((text) => Effect.succeed(Option.some({
+	file,
+	text
+}))), Effect.catchAll((err) => Ref.update(skipped, (prev) => [...prev, {
+	path: file,
+	reason: err.message
+}]).pipe(Effect.flatMap(() => Effect.succeed(Option.none()))))), { concurrency }), Stream.filterMap((opt) => opt), Stream.mapEffect(({ file, text }) => chunker.chunkText(text, file), { concurrency }), Stream.flatMap((chunks) => Stream.fromIterable(chunks)), Stream.runCollect, Effect.map((allChunks) => {
+	const chunks = Chunk.toArray(allChunks);
+	return {
+		chunks,
+		totalChunks: chunks.length
+	};
+}));
 var IndexProject = class extends Effect.Service()("IndexProject", {
 	accessors: true,
 	effect: Effect.gen(function* () {
@@ -162,49 +361,124 @@ var IndexProject = class extends Effect.Service()("IndexProject", {
 		const embedder = yield* Embedder;
 		const vectorStore = yield* VectorStore;
 		const d = yield* Display;
-		const index = () => Effect.gen(function* () {
+		const extractor = yield* ContentExtractor;
+		const index = (opts = {}) => Effect.gen(function* () {
+			const start = Date.now();
 			if (!(yield* configStore.configExists())) yield* configStore.writeConfig(DEFAULT_CONFIG);
-			const config = yield* configStore.readConfig();
-			const extensions = Object.keys(config.files).length > 0 ? Object.keys(config.files) : [
-				".ts",
-				".tsx",
-				".js",
-				".jsx"
-			];
+			const eff = deriveEffectiveConfig(opts, yield* configStore.readConfig());
+			const processorMap = buildProcessorMap(eff.skipExtensions);
 			yield* d.updateInteractive("Scanning source files...");
-			const scanResult = yield* scanner.scanFiles(extensions);
-			yield* d.updateInteractive(`Chunking ${scanResult.files.length} files...`);
-			const allChunks = (yield* Effect.forEach(scanResult.files, (file) => chunker.chunkFile(file), { concurrency: Math.max(1, config.chunkConcurrency ?? 8) })).flat();
-			const totalChunks = allChunks.length;
-			const totalFiles = new Set(allChunks.map((c) => c.file)).size;
-			const totalLines = allChunks.reduce((sum, c) => sum + (c.endLine - c.startLine + 1), 0);
-			if (totalChunks === 0) return {
-				success: true,
-				status: {
-					chunks: 0,
-					files: 0,
-					totalLines: 0,
-					byteSize: 0
-				}
-			};
-			yield* d.updateInteractive(`Embedding ${totalChunks} chunks...`);
-			const texts = allChunks.map((c) => c.text);
-			const embeddings = yield* embedder.batch(texts);
-			yield* vectorStore.store(allChunks, embeddings);
-			const dims = embeddings[0]?.dims ?? 384;
+			const scanResult = yield* scanner.scanFiles(eff.ignoredPaths, eff.ignoreGitignore);
+			const { knownFiles, skippedFiles, unknownExtensions } = classifyFiles(scanResult.files, processorMap);
+			const skipped = yield* Ref.make(scanResult.skipped.filter((s) => !s.reason.startsWith("Ignored by config pattern")).map((s) => ({
+				path: s.path,
+				reason: s.reason
+			})));
+			if (unknownExtensions.size > 0) yield* Ref.update(skipped, (prev) => [...prev, ...skippedFiles.map((f) => ({
+				path: f,
+				reason: "unknown extension"
+			}))]);
+			if (knownFiles.length === 0) {
+				yield* displaySkippedNote(d, yield* Ref.get(skipped));
+				return {
+					success: true,
+					status: {
+						chunks: 0,
+						files: 0,
+						totalLines: 0,
+						byteSize: 0
+					},
+					durationMs: Date.now() - start
+				};
+			}
+			yield* d.updateInteractive(`Processing ${knownFiles.length} files...`);
+			const { chunks, totalChunks } = yield* classifyAndCollectChunks(knownFiles, extractor, chunker, eff.concurrency, skipped);
+			if (totalChunks === 0) {
+				yield* displaySkippedNote(d, yield* Ref.get(skipped));
+				return {
+					success: true,
+					status: {
+						chunks: 0,
+						files: 0,
+						totalLines: 0,
+						byteSize: 0
+					},
+					durationMs: Date.now() - start
+				};
+			}
+			yield* vectorStore.storeBegin();
+			const embeddedRef = yield* Ref.make(0);
+			const stats = yield* d.progress({
+				message: `Embedding ${totalChunks} chunks...`,
+				max: totalChunks
+			}, Stream.fromIterable(chunks).pipe(Stream.grouped(eff.batchSize), Stream.mapEffect((batchChunk) => Effect.gen(function* () {
+				const batch = Chunk.toArray(batchChunk);
+				const texts = batch.map((c) => c.text);
+				const embeddings = yield* embedder.batch(texts);
+				yield* vectorStore.storeBatch(batch, embeddings);
+				const count = yield* Ref.updateAndGet(embeddedRef, (n) => n + batch.length);
+				yield* d.updateInteractive({
+					message: `Embedding ${count} of ${totalChunks} chunks`,
+					setTo: count
+				});
+			})), Stream.runDrain, Effect.matchEffect({
+				onSuccess: () => vectorStore.storeCommit(),
+				onFailure: (err) => vectorStore.storeAbort().pipe(Effect.flatMap(() => Effect.fail(err)))
+			})));
+			yield* displaySkippedNote(d, yield* Ref.get(skipped));
+			const durationSec = ((Date.now() - start) / 1e3).toFixed(1);
+			yield* d.log(`Indexed ${stats.chunks} chunks from ${stats.files} files in ${durationSec}s`, "success");
+			const fallbackInfo = yield* embedder.getFallbackInfo();
 			return {
 				success: true,
 				status: {
-					chunks: totalChunks,
-					files: totalFiles,
-					totalLines,
-					byteSize: embeddings.length * dims * 4
-				}
+					chunks: stats.chunks,
+					files: stats.files,
+					totalLines: stats.totalLines,
+					byteSize: stats.byteSize
+				},
+				durationMs: Date.now() - start,
+				embedderFallback: fallbackInfo
 			};
 		});
 		return { index };
 	})
 }) {};
+const getFilename = (path) => path.split("/").pop() ?? path;
+const getFileExtension = (filename) => {
+	const dotIndex = filename.lastIndexOf(".");
+	return dotIndex >= 0 ? filename.slice(dotIndex) : "(no extension)";
+};
+const groupByExtension = (entries) => {
+	const byExt = /* @__PURE__ */ new Map();
+	for (const s of entries) {
+		const name = getFilename(s.path);
+		const ext = getFileExtension(name);
+		if (!byExt.has(ext)) byExt.set(ext, []);
+		byExt.get(ext).push(name);
+	}
+	return byExt;
+};
+const formatFileList = (files, maxDisplay = 5) => files.length > maxDisplay ? `${files.slice(0, maxDisplay).join(", ")} +${files.length - maxDisplay} more` : files.join(", ");
+const buildSkippedLines = (extFailures, extractErrors) => {
+	const lines = [];
+	if (extFailures.length > 0) {
+		lines.push(`Unknown extensions (${extFailures.length})`);
+		for (const [ext, files] of groupByExtension(extFailures)) lines.push(`  ${ext} (${files.length}): ${formatFileList(files)}`);
+	}
+	if (extractErrors.length > 0) {
+		if (lines.length > 0) lines.push("");
+		lines.push(`Extraction errors (${extractErrors.length})`);
+		for (const s of extractErrors) lines.push(`  ${getFilename(s.path)}: ${s.reason}`);
+	}
+	return lines;
+};
+const displaySkippedNote = (d, skipped) => {
+	if (skipped.length === 0) return Effect.void;
+	const extFailures = skipped.filter((s) => s.reason === "unknown extension");
+	const extractErrors = skipped.filter((s) => s.reason !== "unknown extension");
+	return d.note(buildSkippedLines(extFailures, extractErrors).join("\n"), `Skipped ${skipped.length} files`);
+};
 //#endregion
 //#region src/application/init-project.ts
 /**
@@ -293,22 +567,60 @@ const reportError = (error) => Effect.gen(function* () {
 });
 //#endregion
 //#region src/commands/index-cmd.ts
-/** CLI command: pix index [--force] [--verbose] [--json] */
+const batchSizeOption = Options.integer("batch-size").pipe(Options.withAlias("b"), Options.optional);
+const chunkConcurrencyOption = Options.integer("chunk-concurrency").pipe(Options.withAlias("c"), Options.optional);
+const skipExtensionsOption = Options.text("skip-extensions").pipe(Options.withAlias("s"), Options.repeated);
+const ignorePathOption = Options.text("ignore-path").pipe(Options.repeated);
+const ignorePathsOption = Options.text("ignore-paths").pipe(Options.repeated);
+const ignoreGitignoreOption = Options.boolean("ignore-gitignore").pipe(Options.withDefault(false));
+const splitCsv = (values) => values.flatMap((v) => v.split(",").map((s) => s.trim()).filter((s) => s.length > 0));
+const buildIndexOptions = (args) => {
+	if (Option.isSome(args.batchSize) && args.batchSize.value <= 0) throw new Error(`--batch-size must be positive, got ${args.batchSize.value}`);
+	if (Option.isSome(args.chunkConcurrency) && args.chunkConcurrency.value <= 0) throw new Error(`--chunk-concurrency must be positive, got ${args.chunkConcurrency.value}`);
+	const cliSkipExtensions = splitCsv(args.skipExtensions);
+	const cliIgnorePaths = [...args.ignorePath.map((s) => s.trim()).filter((s) => s.length > 0), ...splitCsv(args.ignorePaths)];
+	return {
+		batchSize: Option.getOrUndefined(args.batchSize),
+		chunkConcurrency: Option.getOrUndefined(args.chunkConcurrency),
+		skipExtensions: cliSkipExtensions.length > 0 ? cliSkipExtensions : void 0,
+		ignorePaths: cliIgnorePaths.length > 0 ? cliIgnorePaths : void 0,
+		ignoreGitignore: args.ignoreGitignore || void 0
+	};
+};
+const emitIndexResult = (d, result) => Effect.gen(function* () {
+	yield* d.json({
+		chunks: result.status.chunks,
+		files: result.status.files,
+		totalLines: result.status.totalLines,
+		byteSize: result.status.byteSize,
+		durationMs: result.durationMs,
+		...result.embedderFallback && { embedderFallback: result.embedderFallback }
+	});
+	if (result.status.chunks === 0) yield* d.log("No chunks to index.", "warn");
+});
 const indexCommand = Command.make("index", {
 	force: Options.boolean("force").pipe(Options.withDefault(false)),
 	verbose: Options.boolean("verbose").pipe(Options.withDefault(false)),
-	json: Options.boolean("json").pipe(Options.withDefault(false))
-}, ({ force, verbose }) => Effect.gen(function* () {
+	json: Options.boolean("json").pipe(Options.withDefault(false)),
+	batchSize: batchSizeOption,
+	chunkConcurrency: chunkConcurrencyOption,
+	skipExtensions: skipExtensionsOption,
+	ignorePath: ignorePathOption,
+	ignorePaths: ignorePathsOption,
+	ignoreGitignore: ignoreGitignoreOption
+}, ({ force, verbose, batchSize, chunkConcurrency, skipExtensions, ignorePath, ignorePaths, ignoreGitignore }) => Effect.gen(function* () {
 	const d = yield* Display;
 	if (force) yield* d.log("--force is currently not implemented and only a placeholder.", "warn");
 	if (verbose) yield* d.log("--verbose is currently not implemented and only a placeholder.", "warn");
-	const result = yield* d.spinner("Indexing project...", IndexProject.index());
-	yield* d.json({
-		chunks: result.status.chunks,
-		files: result.status.files
+	const options = buildIndexOptions({
+		batchSize,
+		chunkConcurrency,
+		skipExtensions,
+		ignorePath,
+		ignorePaths,
+		ignoreGitignore
 	});
-	if (result.status.chunks === 0) yield* d.log("No chunks to index.", "warn");
-	else yield* d.log(`Indexed ${result.status.chunks} chunks from ${result.status.files} files.`, "success");
+	yield* emitIndexResult(d, yield* d.spinner("Indexing project...", IndexProject.index(options)));
 }).pipe(Effect.catchAll(reportError)));
 //#endregion
 //#region src/commands/init.ts
@@ -493,29 +805,6 @@ const setupTerminalCleanup = () => {
 	process.on("exit", makeTerminalCleanupHandler(process.stdin, process.stdout));
 };
 //#endregion
-//#region src/domain/errors.ts
-/** Config file or directory does not exist. Run pix init first. */
-var ConfigNotFoundError = class extends Data.TaggedError("ConfigNotFoundError") {};
-/** Config file exists but contains invalid JSON. */
-var ConfigMalformedError = class extends Data.TaggedError("ConfigMalformedError") {};
-/** Index files (chunks.jsonl, vectors.bin) do not exist. Run pix index first. */
-var NoIndexError = class extends Data.TaggedError("NoIndexError") {};
-/** Disk is full — write operation could not complete. */
-var DiskFullError = class extends Data.TaggedError("DiskFullError") {};
-/** Generic index store I/O failure (read, write, delete). */
-var StoreError = class extends Data.TaggedError("StoreError") {};
-/** Source file could not be read during chunking (binary, permissions, encoding). */
-var ChunkerError = class extends Data.TaggedError("ChunkerError") {};
-/** Embedding model could not be downloaded or loaded. */
-var ModelLoadError = class extends Data.TaggedError("ModelLoadError") {};
-/** Embedding model failed during inference. */
-var InferenceError = class extends Data.TaggedError("InferenceError") {};
-/**
-* Fatal scan failure — gitignore loading failed entirely. Non-fatal per-entry skips are reported
-* via ScanResult.skipped.
-*/
-var ScanFailed = class extends Data.TaggedError("ScanFailed") {};
-//#endregion
 //#region src/services/chunker.ts
 const MIN_CHUNK_CHARS = 20;
 const readFileContent = (fs, file) => fs.readFileString(file).pipe(Effect.mapError((cause) => new ChunkerError({
@@ -547,17 +836,24 @@ const buildChunks = (file, content, config) => {
 	}
 	return chunks;
 };
-const make$4 = Effect.gen(function* () {
+const make$5 = Effect.gen(function* () {
 	const fs = yield* FileSystem.FileSystem;
 	const config = yield* (yield* ConfigStore).readConfig().pipe(Effect.catchAll(() => Effect.succeed(DEFAULT_CONFIG)));
+	const chunkText = (text, file) => Effect.sync(() => {
+		if (text === "") return [];
+		return buildChunks(file, text, config);
+	});
 	const chunkFile = (file) => Effect.gen(function* () {
 		const content = yield* readFileContent(fs, file);
 		if (content === "") return [];
 		return buildChunks(file, content, config);
 	});
-	return { chunkFile };
+	return {
+		chunkFile,
+		chunkText
+	};
 });
-const ChunkerLive = Layer.effect(Chunker, make$4);
+const ChunkerLive = Layer.effect(Chunker, make$5);
 //#endregion
 //#region src/services/config-store.ts
 const CONFIG_DIR = ".pix";
@@ -574,7 +870,7 @@ const mapConfigWriteError = (cause, path, action) => {
 		cause
 	});
 };
-const make$3 = Effect.gen(function* () {
+const make$4 = Effect.gen(function* () {
 	const fs = yield* FileSystem.FileSystem;
 	const writeConfig = (config) => Effect.gen(function* () {
 		const configJson = JSON.stringify(config, null, 2);
@@ -611,7 +907,28 @@ const make$3 = Effect.gen(function* () {
 		configExists
 	};
 });
-const ConfigStoreLive = Layer.effect(ConfigStore, make$3);
+const ConfigStoreLive = Layer.effect(ConfigStore, make$4);
+//#endregion
+//#region src/services/content-extractor.ts
+const make$3 = Effect.gen(function* () {
+	const fs = yield* FileSystem.FileSystem;
+	const processorMap = buildProcessorMap([]);
+	const extract = (file) => {
+		const lastSlash = file.lastIndexOf("/");
+		const name = lastSlash >= 0 ? file.slice(lastSlash + 1) : file;
+		const dotIndex = name.lastIndexOf(".");
+		const ext = dotIndex === -1 ? name.toLowerCase() : name.slice(dotIndex).toLowerCase();
+		const processor = processorMap[ext];
+		if (!processor) return Effect.fail({
+			_tag: "UnsupportedFormat",
+			message: `No processor for extension: ${ext}`,
+			extension: ext
+		});
+		return processor(file).pipe(Effect.provideService(FileSystem.FileSystem, fs));
+	};
+	return { extract };
+});
+const ContentExtractorLive = Layer.effect(ContentExtractor, make$3);
 //#endregion
 //#region src/domain/models.ts
 /** Registry of supported embedding models. */
@@ -641,9 +958,7 @@ const MODEL_REGISTRY = {
 };
 //#endregion
 //#region src/services/embedder.ts
-const CACHE_DIR = ".pix/cache";
-const BATCH_SIZE = 16;
-env.cacheDir = CACHE_DIR;
+env.cacheDir = ".pix/cache";
 const normalize = (arr) => {
 	let norm = 0;
 	for (let i = 0; i < arr.length; i++) norm += arr[i] * arr[i];
@@ -685,16 +1000,14 @@ const createExtractor = (opts) => Effect.tryPromise(async () => {
 	model: opts.model,
 	cause
 })));
-const createExtractorWithFallback = (opts) => {
+const createExtractorWithFallback = (opts, fallbackRef) => {
 	if (opts.device === "cpu") return createExtractor(opts);
 	return createExtractor(opts).pipe(Effect.catchAll((originalError) => Effect.gen(function* () {
-		const d = yield* Display;
-		yield* d.log(`GPU (${opts.device}) failed, falling back to CPU...`, "warn");
-		yield* d.json({
-			event: "embedder_fallback",
+		yield* (yield* Display).log(`GPU (${opts.device}) failed, falling back to CPU...`, "warn");
+		yield* Ref.set(fallbackRef, Option.some({
 			originalDevice: opts.device,
 			reason: originalError.message
-		});
+		}));
 		return yield* createExtractor({
 			...opts,
 			device: "cpu"
@@ -705,7 +1018,8 @@ const make$2 = Effect.gen(function* () {
 	const configStore = yield* ConfigStore;
 	const d = yield* Display;
 	const cfg = yield* resolveEmbedderConfig(configStore);
-	const getExtractor = yield* Effect.cached(createExtractorWithFallback(cfg));
+	const fallbackRef = yield* Ref.make(Option.none());
+	const getExtractor = yield* Effect.cached(createExtractorWithFallback(cfg, fallbackRef));
 	const embed = (text) => Effect.gen(function* () {
 		const extractor = yield* getExtractor;
 		const data = (yield* Effect.tryPromise(() => extractor(text, {
@@ -722,44 +1036,35 @@ const make$2 = Effect.gen(function* () {
 	}).pipe(Effect.provideService(Display, d));
 	const batch = (texts) => Effect.gen(function* () {
 		const extractor = yield* getExtractor;
+		const tensor = yield* Effect.tryPromise(() => extractor([...texts], {
+			pooling: "mean",
+			normalize: false
+		})).pipe(Effect.mapError((cause) => new InferenceError({
+			message: "Batch embedding inference failed",
+			cause
+		})));
+		const data = tensor.data;
+		const n = tensor.dims[0];
 		const results = [];
-		for (let i = 0; i < texts.length; i += BATCH_SIZE) {
-			const slice = texts.slice(i, i + BATCH_SIZE);
-			const tensor = yield* Effect.tryPromise(() => extractor(slice, {
-				pooling: "mean",
-				normalize: false
-			})).pipe(Effect.mapError((cause) => new InferenceError({
-				message: "Batch embedding inference failed",
-				cause
-			})));
-			const data = tensor.data;
-			const n = tensor.dims[0];
-			for (let j = 0; j < n; j++) {
-				const offset = j * cfg.dims;
-				results.push(normalize(data.slice(offset, offset + cfg.dims)));
-			}
+		for (let j = 0; j < n; j++) {
+			const offset = j * cfg.dims;
+			results.push({
+				vector: normalize(data.slice(offset, offset + cfg.dims)),
+				dims: cfg.dims
+			});
 		}
-		return results.map((vector) => ({
-			vector,
-			dims: cfg.dims
-		}));
+		return results;
 	}).pipe(Effect.provideService(Display, d));
+	const getFallbackInfo = () => Ref.get(fallbackRef).pipe(Effect.map(Option.getOrElse(() => void 0)));
 	return {
 		embed,
-		batch
+		batch,
+		getFallbackInfo
 	};
 });
 const OnnxEmbedderLive = Layer.provideMerge(Layer.effect(Embedder, make$2), ConfigStoreLive);
 //#endregion
 //#region src/services/scanner.ts
-const ALWAYS_IGNORE = new Set([
-	".pix",
-	"node_modules",
-	".git",
-	"dist",
-	"build",
-	".next"
-]);
 const make$1 = Effect.gen(function* () {
 	const fs = yield* FileSystem.FileSystem;
 	const readFileWithSkip = (path, mkReason) => fs.readFileString(path).pipe(Effect.map((content) => ({
@@ -792,47 +1097,93 @@ const make$1 = Effect.gen(function* () {
 			reason: `Could not stat: ${String(error)}`
 		}
 	})));
-	const loadGitignoreRules = Effect.gen(function* () {
+	const computeRelative = (fullPath, cwd) => fullPath.startsWith(cwd) ? fullPath.slice(cwd.length + 1) : fullPath;
+	const loadIgnoreFile = (filePath, ig, skipped) => Effect.gen(function* () {
+		const result = yield* readFileWithSkip(filePath, (error) => `Could not read ignore file: ${String(error)}`);
+		if (result.skipped) skipped.push(result.skipped);
+		if (result.content.trim()) ig.add(result.content.split("\n"));
+	});
+	const loadGitignoreRules = (ignoredPaths) => {
 		const ig = ignore();
-		const cwd = process.cwd();
 		const skipped = [];
-		const rootContent = yield* readFileWithSkip(`${cwd}/.gitignore`, (error) => `Could not read gitignore: ${String(error)}`);
-		if (rootContent.skipped) skipped.push(rootContent.skipped);
-		if (rootContent.content.trim()) ig.add(rootContent.content.split("\n"));
+		if (ignoredPaths.length > 0) ig.add(ignoredPaths);
+		return Effect.succeed({
+			ig,
+			skipped
+		});
+	};
+	const loadGitignoreRulesWithFiles = (ignoredPaths, cwd) => Effect.gen(function* () {
+		const ig = ignore();
+		const skipped = [];
+		if (ignoredPaths.length > 0) ig.add(ignoredPaths);
+		const gitignorePath = `${cwd}/.gitignore`;
+		if (yield* fs.exists(gitignorePath).pipe(Effect.catchAll(() => Effect.succeed(false)))) yield* loadIgnoreFile(gitignorePath, ig, skipped);
 		const excludePath = `${cwd}/.git/info/exclude`;
-		if (yield* fs.exists(excludePath)) {
-			const excludeContent = yield* readFileWithSkip(excludePath, (error) => `Could not read exclude file: ${String(error)}`);
-			if (excludeContent.skipped) skipped.push(excludeContent.skipped);
-			if (excludeContent.content.trim()) ig.add(excludeContent.content.split("\n"));
-		}
+		if (yield* fs.exists(excludePath).pipe(Effect.catchAll(() => Effect.succeed(false)))) yield* loadIgnoreFile(excludePath, ig, skipped);
 		return {
 			ig,
 			skipped
 		};
 	});
-	const walk = (dir, extensions) => Effect.gen(function* () {
+	const processEntry = (entry, dir, ig, cwd) => Effect.gen(function* () {
+		const fullPath = `${dir}/${entry}`;
+		const statResult = yield* statWithSkip(fullPath);
+		if (statResult.skipped) return {
+			files: [],
+			skipped: [statResult.skipped]
+		};
+		if (!statResult.info) return {
+			files: [],
+			skipped: []
+		};
+		const info = statResult.info;
+		if (info.type === "Directory") {
+			const relativeDir = computeRelative(fullPath, cwd);
+			if (ig.ignores(relativeDir)) return {
+				files: [],
+				skipped: [{
+					path: fullPath,
+					reason: `Ignored by config pattern: ${relativeDir}`
+				}]
+			};
+			return {
+				files: [],
+				skipped: [],
+				recurse: true
+			};
+		}
+		if (info.type === "File") {
+			const relativePath = computeRelative(fullPath, cwd);
+			if (ig.ignores(relativePath)) return {
+				files: [],
+				skipped: [{
+					path: fullPath,
+					reason: `Ignored by config pattern: ${relativePath}`
+				}]
+			};
+			return {
+				files: [fullPath],
+				skipped: []
+			};
+		}
+		return {
+			files: [],
+			skipped: []
+		};
+	});
+	const walk = (dir, ig, cwd) => Effect.gen(function* () {
 		const result = yield* readDirectoryWithSkip(dir);
 		let files = [];
 		const skipped = [];
 		if (result.skipped) skipped.push(result.skipped);
 		for (const entry of result.entries) {
-			if (ALWAYS_IGNORE.has(entry)) continue;
-			const fullPath = `${dir}/${entry}`;
-			const info = yield* statWithSkip(fullPath);
-			if (info.skipped) {
-				skipped.push(info.skipped);
-				continue;
-			}
-			if (!info.info) continue;
-			if (info.info.type === "Directory") {
-				const sub = yield* walk(fullPath, extensions);
+			const entryResult = yield* processEntry(entry, dir, ig, cwd);
+			files.push(...entryResult.files);
+			skipped.push(...entryResult.skipped);
+			if ("recurse" in entryResult) {
+				const sub = yield* walk(`${dir}/${entry}`, ig, cwd);
 				files.push(...sub.files);
 				skipped.push(...sub.skipped);
-			} else if (info.info.type === "File") {
-				const dotIndex = entry.lastIndexOf(".");
-				if (dotIndex === -1) continue;
-				const ext = entry.slice(dotIndex);
-				if (extensions.has(ext)) files.push(fullPath);
 			}
 		}
 		return {
@@ -840,16 +1191,15 @@ const make$1 = Effect.gen(function* () {
 			skipped
 		};
 	});
-	const scanFiles = (extensions) => Effect.gen(function* () {
-		const { ig, skipped: ignoreSkipped } = yield* loadGitignoreRules.pipe(Effect.mapError((cause) => new ScanFailed({
-			message: `Failed to load gitignore rules: ${String(cause)}`,
+	const scanFiles = (ignoredPaths, ignoreGitignore) => Effect.gen(function* () {
+		const cwd = process.cwd();
+		const { ig, skipped: ignoreSkipped } = yield* (ignoreGitignore ? loadGitignoreRules(ignoredPaths) : loadGitignoreRulesWithFiles(ignoredPaths, cwd)).pipe(Effect.mapError((cause) => new ScanFailed({
+			message: `Failed to load ignore rules: ${String(cause)}`,
 			cause
 		})));
-		const cwd = process.cwd();
-		const { files: paths, skipped: walkSkipped } = yield* walk(cwd, new Set(extensions));
-		const relativePaths = paths.map((p) => p.startsWith(cwd) ? p.slice(cwd.length + 1) : p);
+		const { files, skipped: walkSkipped } = yield* walk(cwd, ig, cwd);
 		return {
-			files: ig.filter(relativePaths).map((p) => `${cwd}/${p}`),
+			files,
 			skipped: [...ignoreSkipped, ...walkSkipped]
 		};
 	});
@@ -868,6 +1218,22 @@ const isPlatformReason = (cause, reason) => typeof cause === "object" && cause !
 */
 const make = Effect.gen(function* () {
 	const fs = yield* FileSystem.FileSystem;
+	const chunksTemp = `${CHUNKS_FILE}.tmp`;
+	const vectorsTemp = `${VECTORS_FILE}.tmp`;
+	const seenFiles = yield* Ref.make(/* @__PURE__ */ new Set());
+	const statsAccumulator = yield* Ref.make({
+		chunks: 0,
+		files: 0,
+		totalLines: 0,
+		byteSize: 0
+	});
+	const serializeVectors = (embeddings) => {
+		const dims = embeddings[0]?.dims ?? 384;
+		const totalFloats = embeddings.length * dims;
+		const vectorsArray = new Float32Array(totalFloats);
+		for (let i = 0; i < embeddings.length; i++) vectorsArray.set(embeddings[i].vector, i * dims);
+		return Buffer.from(vectorsArray.buffer);
+	};
 	/**
 	* Count total lines across all chunks in chunks.jsonl. Each line is a JSON object; the 'text'
 	* field contains the source code.
@@ -930,6 +1296,60 @@ const make = Effect.gen(function* () {
 			deleted: true
 		};
 	});
+	const storeBegin = () => Effect.gen(function* () {
+		yield* ensureDirExists(STORE_DIR, ".pix directory");
+		yield* Ref.set(seenFiles, /* @__PURE__ */ new Set());
+		yield* Ref.set(statsAccumulator, {
+			chunks: 0,
+			files: 0,
+			totalLines: 0,
+			byteSize: 0
+		});
+		if (yield* withStoreError(fs.exists(chunksTemp), "check chunks temp")) yield* withStoreError(fs.remove(chunksTemp), "clean stale chunks temp", chunksTemp);
+		if (yield* withStoreError(fs.exists(vectorsTemp), "check vectors temp")) yield* withStoreError(fs.remove(vectorsTemp), "clean stale vectors temp", vectorsTemp);
+	});
+	const storeBatch = (chunks, embeddings) => Effect.gen(function* () {
+		const content = chunks.map((c) => JSON.stringify({
+			id: c.id,
+			idx: c.idx,
+			file: c.file,
+			startLine: c.startLine,
+			endLine: c.endLine,
+			text: c.text
+		})).join("\n") + "\n";
+		yield* withStoreError(fs.writeFile(chunksTemp, Buffer.from(content), { flag: "a" }), "append chunks", chunksTemp);
+		const buffer = serializeVectors(embeddings);
+		yield* withStoreError(fs.writeFile(vectorsTemp, buffer, { flag: "a" }), "append vectors", vectorsTemp);
+		const dims = embeddings[0]?.dims ?? 384;
+		const batchLines = chunks.reduce((sum, c) => sum + (c.endLine - c.startLine + 1), 0);
+		const batchBytes = embeddings.length * dims * 4;
+		yield* Ref.update(seenFiles, (prev) => {
+			for (const c of chunks) prev.add(c.file);
+			return prev;
+		});
+		yield* Ref.update(statsAccumulator, (prev) => ({
+			chunks: prev.chunks + chunks.length,
+			files: 0,
+			totalLines: prev.totalLines + batchLines,
+			byteSize: prev.byteSize + batchBytes
+		}));
+	});
+	const storeCommit = () => Effect.gen(function* () {
+		yield* withStoreError(fs.rename(chunksTemp, CHUNKS_FILE), "commit chunks", CHUNKS_FILE);
+		yield* withStoreError(fs.rename(vectorsTemp, VECTORS_FILE), "commit vectors", VECTORS_FILE);
+		const stats = yield* Ref.get(statsAccumulator);
+		const files = yield* Ref.get(seenFiles);
+		yield* Ref.set(seenFiles, /* @__PURE__ */ new Set());
+		return {
+			...stats,
+			files: files.size
+		};
+	});
+	const storeAbort = () => Effect.gen(function* () {
+		yield* Ref.set(seenFiles, /* @__PURE__ */ new Set());
+		if (yield* withReadError(fs.exists(chunksTemp), "check chunks temp")) yield* withReadError(fs.remove(chunksTemp), "abort chunks temp", chunksTemp);
+		if (yield* withReadError(fs.exists(vectorsTemp), "check vectors temp")) yield* withReadError(fs.remove(vectorsTemp), "abort vectors temp", vectorsTemp);
+	});
 	const store = (chunks, embeddings) => Effect.gen(function* () {
 		yield* ensureDirExists(STORE_DIR, ".pix directory");
 		const chunksTemp = `${CHUNKS_FILE}.tmp`;
@@ -944,11 +1364,7 @@ const make = Effect.gen(function* () {
 		yield* withStoreError(fs.writeFileString(chunksTemp, chunksLines.join("\n")), "write chunks", chunksTemp);
 		yield* withStoreError(fs.rename(chunksTemp, CHUNKS_FILE), "commit chunks", CHUNKS_FILE);
 		const vectorsTemp = `${VECTORS_FILE}.tmp`;
-		const dims = embeddings[0]?.dims ?? 384;
-		const totalFloats = embeddings.length * dims;
-		const vectorsArray = new Float32Array(totalFloats);
-		for (let i = 0; i < embeddings.length; i++) vectorsArray.set(embeddings[i].vector, i * dims);
-		const buffer = Buffer.from(vectorsArray.buffer);
+		const buffer = serializeVectors(embeddings);
 		yield* withStoreError(fs.writeFile(vectorsTemp, buffer), "write vectors", vectorsTemp);
 		yield* withStoreError(fs.rename(vectorsTemp, VECTORS_FILE), "commit vectors", VECTORS_FILE);
 	});
@@ -1017,6 +1433,10 @@ const make = Effect.gen(function* () {
 	});
 	return {
 		store,
+		storeBegin,
+		storeBatch,
+		storeCommit,
+		storeAbort,
 		search,
 		getStatus,
 		reset
@@ -1025,7 +1445,7 @@ const make = Effect.gen(function* () {
 const VectorStoreLive = Layer.effect(VectorStore, make);
 //#endregion
 //#region src/index.ts
-const ServicesLayer = Layer.mergeAll(ConfigStoreLive, ScannerLive, OnnxEmbedderLive, VectorStoreLive);
+const ServicesLayer = Layer.mergeAll(ConfigStoreLive, ScannerLive, OnnxEmbedderLive, VectorStoreLive, ContentExtractorLive);
 const ChunkerLayer = ChunkerLive.pipe(Layer.provide(ServicesLayer));
 const InfraLayer = Layer.mergeAll(ServicesLayer, ChunkerLayer).pipe(Layer.provide(NodeContext.layer));
 const UseCaseLayer = Layer.mergeAll(InitProject.Default, GetStatus.Default, QueryProject.Default, IndexProject.Default, ResetIndex.Default);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lucas-bur/pix",
-  "version": "0.9.1",
+  "version": "0.11.0",
   "description": "Lightweight local semantic project indexer",
   "keywords": [
     "cli",