@lucas-bur/pix 0.10.0 → 0.11.0

package/dist/index.mjs

@@ -2,7 +2,8 @@
 import { createRequire } from "node:module";
 import { Args, CliConfig, Command, Options } from "@effect/cli";
 import { NodeContext, NodeRuntime } from "@effect/platform-node";
-import { Clock, Context, Data, Effect, Layer, Option, Ref } from "effect";
+import { Clock, Context, Data, Effect, Exit, Layer, Option, Ref, Stream } from "effect";
+import * as Chunk from "effect/Chunk";
 import { styleText } from "node:util";
 import * as clack from "@clack/prompts";
 import { FileSystem } from "@effect/platform";
@@ -74,6 +75,7 @@ const withInteractive = (activeRef, acquire, setActive, release, effect) => Ref.
 /** Display implementation using @clack/prompts for interactive terminal output */
 const ClackDisplay = { layer: Layer.effect(Display, Effect.gen(function* () {
 	const activeRef = yield* Ref.make(null);
+	const lastSpinnerMsg = yield* Ref.make("");
 	return {
 		intro: (title) => Effect.sync(() => clack.intro(styleText("inverse", ` ${title} `))),
 		outro: (message) => Effect.sync(() => clack.outro(message)),
@@ -84,11 +86,17 @@ const ClackDisplay = { layer: Layer.effect(Display, Effect.gen(function* () {
 			const s = clack.spinner();
 			s.start(message);
 			return s;
-		}), (s) => ({
+		}).pipe(Effect.tap(() => Ref.set(lastSpinnerMsg, message))), (s) => ({
 			type: "spinner",
 			handle: s
-		}), (s, exit) => Effect.sync(() => s.stop(exit._tag === "Success" ? message : `${message} (failed)`)), effect),
-		progress: (opts, effect) => withInteractive(activeRef, Effect.sync(() => {
+		}), (s, exit) => lastSpinnerMsg.pipe(Effect.flatMap((lastMsg) => Effect.sync(() => s.stop(exit._tag === "Success" && lastMsg ? lastMsg : `${message} (failed)`)))), effect),
+		progress: (opts, effect) => Effect.gen(function* () {
+			const current = yield* Ref.get(activeRef);
+			if (current && current.type === "spinner") {
+				const msg = yield* Ref.get(lastSpinnerMsg);
+				current.handle.stop(msg || opts.message);
+				yield* Ref.set(activeRef, null);
+			}
 			const bar = clack.progress({
 				max: opts.max,
 				style: opts.style ?? "heavy",
@@ -96,16 +104,27 @@ const ClackDisplay = { layer: Layer.effect(Display, Effect.gen(function* () {
 				indicator: opts.indicator ?? "dots"
 			});
 			bar.start(opts.message);
-			return bar;
-		}), (bar) => ({
-			type: "progress",
-			handle: bar,
-			value: 0,
-			max: opts.max
-		}), (bar, exit) => Effect.sync(() => exit._tag === "Success" ? bar.stop(opts.message) : bar.error(opts.message)), effect),
+			yield* Ref.set(activeRef, {
+				type: "progress",
+				handle: bar,
+				value: 0,
+				max: opts.max
+			});
+			const exit = yield* Effect.exit(effect);
+			yield* Ref.set(activeRef, null);
+			if (Exit.isSuccess(exit)) {
+				bar.stop(opts.message);
+				return exit.value;
+			}
+			bar.error(opts.message);
+			return yield* Effect.failCause(exit.cause);
+		}),
 		updateInteractive: (payload) => Ref.get(activeRef).pipe(Effect.flatMap((active) => {
 			if (!active) return Effect.void;
-			if (active.type === "spinner") return Effect.sync(() => active.handle.message(payloadText(payload)));
+			if (active.type === "spinner") {
+				const msg = payloadText(payload);
+				return Effect.sync(() => active.handle.message(msg)).pipe(Effect.andThen(Ref.set(lastSpinnerMsg, msg)));
+			}
 			const delta = computeDelta(payload, {
 				value: active.value,
 				max: active.max
@@ -149,19 +168,19 @@ const DEFAULT_CONFIG = {
 		"dist",
 		"build",
 		".next",
-		".agents",
-		".claude",
 		".vscode",
-		".github",
 		"coverage",
 		"*-lock.yaml",
 		"*-lock.json",
-		"*.lock"
+		"*.lock",
+		".vite-hooks",
+		".fallow"
 	],
 	embedder: {
 		model: "Xenova/all-MiniLM-L6-v2",
 		device: "auto",
-		dtype: "fp32"
+		dtype: "fp32",
+		batchSize: 16
 	}
 };
 //#endregion
@@ -289,6 +308,13 @@ function buildProcessorMap(skipExtensions) {
 }
 //#endregion
 //#region src/application/index-project.ts
+const deriveEffectiveConfig = (opts, config) => ({
+	batchSize: opts.batchSize ?? config.embedder.batchSize ?? 16,
+	concurrency: Math.max(1, opts.chunkConcurrency ?? config.chunkConcurrency ?? 8),
+	skipExtensions: opts.skipExtensions ? [...config.skipExtensions, ...opts.skipExtensions] : config.skipExtensions,
+	ignoredPaths: opts.ignorePaths ? [...config.ignoredPaths ?? DEFAULT_CONFIG.ignoredPaths, ...opts.ignorePaths] : config.ignoredPaths ?? DEFAULT_CONFIG.ignoredPaths,
+	ignoreGitignore: opts.ignoreGitignore ?? config.ignoreGitignore ?? false
+});
 function getExtension(file) {
 	const lastSlash = file.lastIndexOf("/");
 	const name = lastSlash >= 0 ? file.slice(lastSlash + 1) : file;
@@ -313,11 +339,19 @@ const classifyFiles = (files, processorMap) => {
 		unknownExtensions
 	};
 };
-/**
-* Use case: index project files. Pipeline: scan → ContentExtractor → chunk → embed → store. Depends
-* on ConfigStore, Scanner, Chunker, Embedder, VectorStore, Display, ContentExtractor via Effect
-* tags.
-*/
+const classifyAndCollectChunks = (knownFiles, extractor, chunker, concurrency, skipped) => Stream.fromIterable(knownFiles).pipe(Stream.mapEffect((file) => extractor.extract(file).pipe(Effect.flatMap((text) => Effect.succeed(Option.some({
+	file,
+	text
+}))), Effect.catchAll((err) => Ref.update(skipped, (prev) => [...prev, {
+	path: file,
+	reason: err.message
+}]).pipe(Effect.flatMap(() => Effect.succeed(Option.none()))))), { concurrency }), Stream.filterMap((opt) => opt), Stream.mapEffect(({ file, text }) => chunker.chunkText(text, file), { concurrency }), Stream.flatMap((chunks) => Stream.fromIterable(chunks)), Stream.runCollect, Effect.map((allChunks) => {
+	const chunks = Chunk.toArray(allChunks);
+	return {
+		chunks,
+		totalChunks: chunks.length
+	};
+}));
 var IndexProject = class extends Effect.Service()("IndexProject", {
 	accessors: true,
 	effect: Effect.gen(function* () {
@@ -328,65 +362,123 @@ var IndexProject = class extends Effect.Service()("IndexProject", {
 		const vectorStore = yield* VectorStore;
 		const d = yield* Display;
 		const extractor = yield* ContentExtractor;
-		const index = () => Effect.gen(function* () {
+		const index = (opts = {}) => Effect.gen(function* () {
+			const start = Date.now();
 			if (!(yield* configStore.configExists())) yield* configStore.writeConfig(DEFAULT_CONFIG);
-			const config = yield* configStore.readConfig();
-			const processorMap = buildProcessorMap(config.skipExtensions);
+			const eff = deriveEffectiveConfig(opts, yield* configStore.readConfig());
+			const processorMap = buildProcessorMap(eff.skipExtensions);
 			yield* d.updateInteractive("Scanning source files...");
-			const ignoredPaths = config.ignoredPaths ?? DEFAULT_CONFIG.ignoredPaths;
-			const { knownFiles, skippedFiles, unknownExtensions } = classifyFiles((yield* scanner.scanFiles(ignoredPaths)).files, processorMap);
-			if (unknownExtensions.size > 0) yield* d.log(`Skipped ${skippedFiles.length} files with unknown extensions: ${[...unknownExtensions].join(", ")}`, "warn");
-			if (knownFiles.length === 0) return {
-				success: true,
-				status: {
-					chunks: 0,
-					files: 0,
-					totalLines: 0,
-					byteSize: 0
-				}
-			};
+			const scanResult = yield* scanner.scanFiles(eff.ignoredPaths, eff.ignoreGitignore);
+			const { knownFiles, skippedFiles, unknownExtensions } = classifyFiles(scanResult.files, processorMap);
+			const skipped = yield* Ref.make(scanResult.skipped.filter((s) => !s.reason.startsWith("Ignored by config pattern")).map((s) => ({
+				path: s.path,
+				reason: s.reason
+			})));
+			if (unknownExtensions.size > 0) yield* Ref.update(skipped, (prev) => [...prev, ...skippedFiles.map((f) => ({
+				path: f,
+				reason: "unknown extension"
+			}))]);
+			if (knownFiles.length === 0) {
+				yield* displaySkippedNote(d, yield* Ref.get(skipped));
+				return {
+					success: true,
+					status: {
+						chunks: 0,
+						files: 0,
+						totalLines: 0,
+						byteSize: 0
+					},
+					durationMs: Date.now() - start
+				};
+			}
 			yield* d.updateInteractive(`Processing ${knownFiles.length} files...`);
-			const allChunks = (yield* Effect.forEach(knownFiles, (file) => Effect.gen(function* () {
-				const result = yield* Effect.either(extractor.extract(file));
-				if (result._tag === "Left") {
-					if (result.left._tag === "UnsupportedFormat") {
-						yield* d.log(`Skipping ${file}: ${result.left.message}`, "warn");
-						return [];
-					}
-					return yield* Effect.fail(result.left);
-				}
-				return yield* chunker.chunkText(result.right, file);
-			}), { concurrency: Math.max(1, config.chunkConcurrency ?? 8) })).flat();
-			const totalChunks = allChunks.length;
-			const totalFiles = new Set(allChunks.map((c) => c.file)).size;
-			const totalLines = allChunks.reduce((sum, c) => sum + (c.endLine - c.startLine + 1), 0);
-			if (totalChunks === 0) return {
-				success: true,
-				status: {
-					chunks: 0,
-					files: 0,
-					totalLines: 0,
-					byteSize: 0
-				}
-			};
-			yield* d.updateInteractive(`Embedding ${totalChunks} chunks...`);
-			const texts = allChunks.map((c) => c.text);
-			const embeddings = yield* embedder.batch(texts);
-			yield* vectorStore.store(allChunks, embeddings);
-			const dims = embeddings[0]?.dims ?? 384;
+			const { chunks, totalChunks } = yield* classifyAndCollectChunks(knownFiles, extractor, chunker, eff.concurrency, skipped);
+			if (totalChunks === 0) {
+				yield* displaySkippedNote(d, yield* Ref.get(skipped));
+				return {
+					success: true,
+					status: {
+						chunks: 0,
+						files: 0,
+						totalLines: 0,
+						byteSize: 0
+					},
+					durationMs: Date.now() - start
+				};
+			}
+			yield* vectorStore.storeBegin();
+			const embeddedRef = yield* Ref.make(0);
+			const stats = yield* d.progress({
+				message: `Embedding ${totalChunks} chunks...`,
+				max: totalChunks
+			}, Stream.fromIterable(chunks).pipe(Stream.grouped(eff.batchSize), Stream.mapEffect((batchChunk) => Effect.gen(function* () {
+				const batch = Chunk.toArray(batchChunk);
+				const texts = batch.map((c) => c.text);
+				const embeddings = yield* embedder.batch(texts);
+				yield* vectorStore.storeBatch(batch, embeddings);
+				const count = yield* Ref.updateAndGet(embeddedRef, (n) => n + batch.length);
+				yield* d.updateInteractive({
+					message: `Embedding ${count} of ${totalChunks} chunks`,
+					setTo: count
+				});
+			})), Stream.runDrain, Effect.matchEffect({
+				onSuccess: () => vectorStore.storeCommit(),
+				onFailure: (err) => vectorStore.storeAbort().pipe(Effect.flatMap(() => Effect.fail(err)))
+			})));
+			yield* displaySkippedNote(d, yield* Ref.get(skipped));
+			const durationSec = ((Date.now() - start) / 1e3).toFixed(1);
+			yield* d.log(`Indexed ${stats.chunks} chunks from ${stats.files} files in ${durationSec}s`, "success");
+			const fallbackInfo = yield* embedder.getFallbackInfo();
 			return {
 				success: true,
 				status: {
-					chunks: totalChunks,
-					files: totalFiles,
-					totalLines,
-					byteSize: embeddings.length * dims * 4
-				}
+					chunks: stats.chunks,
+					files: stats.files,
+					totalLines: stats.totalLines,
+					byteSize: stats.byteSize
+				},
+				durationMs: Date.now() - start,
+				embedderFallback: fallbackInfo
 			};
 		});
 		return { index };
 	})
 }) {};
+const getFilename = (path) => path.split("/").pop() ?? path;
+const getFileExtension = (filename) => {
+	const dotIndex = filename.lastIndexOf(".");
+	return dotIndex >= 0 ? filename.slice(dotIndex) : "(no extension)";
+};
+const groupByExtension = (entries) => {
+	const byExt = /* @__PURE__ */ new Map();
+	for (const s of entries) {
+		const name = getFilename(s.path);
+		const ext = getFileExtension(name);
+		if (!byExt.has(ext)) byExt.set(ext, []);
+		byExt.get(ext).push(name);
+	}
+	return byExt;
+};
+const formatFileList = (files, maxDisplay = 5) => files.length > maxDisplay ? `${files.slice(0, maxDisplay).join(", ")} +${files.length - maxDisplay} more` : files.join(", ");
+const buildSkippedLines = (extFailures, extractErrors) => {
+	const lines = [];
+	if (extFailures.length > 0) {
+		lines.push(`Unknown extensions (${extFailures.length})`);
+		for (const [ext, files] of groupByExtension(extFailures)) lines.push(`  ${ext} (${files.length}): ${formatFileList(files)}`);
+	}
+	if (extractErrors.length > 0) {
+		if (lines.length > 0) lines.push("");
+		lines.push(`Extraction errors (${extractErrors.length})`);
+		for (const s of extractErrors) lines.push(`  ${getFilename(s.path)}: ${s.reason}`);
+	}
+	return lines;
+};
+const displaySkippedNote = (d, skipped) => {
+	if (skipped.length === 0) return Effect.void;
+	const extFailures = skipped.filter((s) => s.reason === "unknown extension");
+	const extractErrors = skipped.filter((s) => s.reason !== "unknown extension");
+	return d.note(buildSkippedLines(extFailures, extractErrors).join("\n"), `Skipped ${skipped.length} files`);
+};
 //#endregion
 //#region src/application/init-project.ts
 /**
@@ -475,22 +567,60 @@ const reportError = (error) => Effect.gen(function* () {
 });
 //#endregion
 //#region src/commands/index-cmd.ts
-/** CLI command: pix index [--force] [--verbose] [--json] */
+const batchSizeOption = Options.integer("batch-size").pipe(Options.withAlias("b"), Options.optional);
+const chunkConcurrencyOption = Options.integer("chunk-concurrency").pipe(Options.withAlias("c"), Options.optional);
+const skipExtensionsOption = Options.text("skip-extensions").pipe(Options.withAlias("s"), Options.repeated);
+const ignorePathOption = Options.text("ignore-path").pipe(Options.repeated);
+const ignorePathsOption = Options.text("ignore-paths").pipe(Options.repeated);
+const ignoreGitignoreOption = Options.boolean("ignore-gitignore").pipe(Options.withDefault(false));
+const splitCsv = (values) => values.flatMap((v) => v.split(",").map((s) => s.trim()).filter((s) => s.length > 0));
+const buildIndexOptions = (args) => {
+	if (Option.isSome(args.batchSize) && args.batchSize.value <= 0) throw new Error(`--batch-size must be positive, got ${args.batchSize.value}`);
+	if (Option.isSome(args.chunkConcurrency) && args.chunkConcurrency.value <= 0) throw new Error(`--chunk-concurrency must be positive, got ${args.chunkConcurrency.value}`);
+	const cliSkipExtensions = splitCsv(args.skipExtensions);
+	const cliIgnorePaths = [...args.ignorePath.map((s) => s.trim()).filter((s) => s.length > 0), ...splitCsv(args.ignorePaths)];
+	return {
+		batchSize: Option.getOrUndefined(args.batchSize),
+		chunkConcurrency: Option.getOrUndefined(args.chunkConcurrency),
+		skipExtensions: cliSkipExtensions.length > 0 ? cliSkipExtensions : void 0,
+		ignorePaths: cliIgnorePaths.length > 0 ? cliIgnorePaths : void 0,
+		ignoreGitignore: args.ignoreGitignore || void 0
+	};
+};
+const emitIndexResult = (d, result) => Effect.gen(function* () {
+	yield* d.json({
+		chunks: result.status.chunks,
+		files: result.status.files,
+		totalLines: result.status.totalLines,
+		byteSize: result.status.byteSize,
+		durationMs: result.durationMs,
+		...result.embedderFallback && { embedderFallback: result.embedderFallback }
+	});
+	if (result.status.chunks === 0) yield* d.log("No chunks to index.", "warn");
+});
 const indexCommand = Command.make("index", {
 	force: Options.boolean("force").pipe(Options.withDefault(false)),
 	verbose: Options.boolean("verbose").pipe(Options.withDefault(false)),
-	json: Options.boolean("json").pipe(Options.withDefault(false))
-}, ({ force, verbose }) => Effect.gen(function* () {
+	json: Options.boolean("json").pipe(Options.withDefault(false)),
+	batchSize: batchSizeOption,
+	chunkConcurrency: chunkConcurrencyOption,
+	skipExtensions: skipExtensionsOption,
+	ignorePath: ignorePathOption,
+	ignorePaths: ignorePathsOption,
+	ignoreGitignore: ignoreGitignoreOption
+}, ({ force, verbose, batchSize, chunkConcurrency, skipExtensions, ignorePath, ignorePaths, ignoreGitignore }) => Effect.gen(function* () {
 	const d = yield* Display;
 	if (force) yield* d.log("--force is currently not implemented and only a placeholder.", "warn");
 	if (verbose) yield* d.log("--verbose is currently not implemented and only a placeholder.", "warn");
-	const result = yield* d.spinner("Indexing project...", IndexProject.index());
-	yield* d.json({
-		chunks: result.status.chunks,
-		files: result.status.files
+	const options = buildIndexOptions({
+		batchSize,
+		chunkConcurrency,
+		skipExtensions,
+		ignorePath,
+		ignorePaths,
+		ignoreGitignore
 	});
-	if (result.status.chunks === 0) yield* d.log("No chunks to index.", "warn");
-	else yield* d.log(`Indexed ${result.status.chunks} chunks from ${result.status.files} files.`, "success");
+	yield* emitIndexResult(d, yield* d.spinner("Indexing project...", IndexProject.index(options)));
 }).pipe(Effect.catchAll(reportError)));
 //#endregion
 //#region src/commands/init.ts
@@ -828,9 +958,7 @@ const MODEL_REGISTRY = {
 };
 //#endregion
 //#region src/services/embedder.ts
-const CACHE_DIR = ".pix/cache";
-const BATCH_SIZE = 16;
-env.cacheDir = CACHE_DIR;
+env.cacheDir = ".pix/cache";
 const normalize = (arr) => {
 	let norm = 0;
 	for (let i = 0; i < arr.length; i++) norm += arr[i] * arr[i];
@@ -872,16 +1000,14 @@ const createExtractor = (opts) => Effect.tryPromise(async () => {
 	model: opts.model,
 	cause
 })));
-const createExtractorWithFallback = (opts) => {
+const createExtractorWithFallback = (opts, fallbackRef) => {
 	if (opts.device === "cpu") return createExtractor(opts);
 	return createExtractor(opts).pipe(Effect.catchAll((originalError) => Effect.gen(function* () {
-		const d = yield* Display;
-		yield* d.log(`GPU (${opts.device}) failed, falling back to CPU...`, "warn");
-		yield* d.json({
-			event: "embedder_fallback",
+		yield* (yield* Display).log(`GPU (${opts.device}) failed, falling back to CPU...`, "warn");
+		yield* Ref.set(fallbackRef, Option.some({
 			originalDevice: opts.device,
 			reason: originalError.message
-		});
+		}));
 		return yield* createExtractor({
 			...opts,
 			device: "cpu"
@@ -892,7 +1018,8 @@ const make$2 = Effect.gen(function* () {
 	const configStore = yield* ConfigStore;
 	const d = yield* Display;
 	const cfg = yield* resolveEmbedderConfig(configStore);
-	const getExtractor = yield* Effect.cached(createExtractorWithFallback(cfg));
+	const fallbackRef = yield* Ref.make(Option.none());
+	const getExtractor = yield* Effect.cached(createExtractorWithFallback(cfg, fallbackRef));
 	const embed = (text) => Effect.gen(function* () {
 		const extractor = yield* getExtractor;
 		const data = (yield* Effect.tryPromise(() => extractor(text, {
@@ -909,31 +1036,30 @@ const make$2 = Effect.gen(function* () {
 	}).pipe(Effect.provideService(Display, d));
 	const batch = (texts) => Effect.gen(function* () {
 		const extractor = yield* getExtractor;
+		const tensor = yield* Effect.tryPromise(() => extractor([...texts], {
+			pooling: "mean",
+			normalize: false
+		})).pipe(Effect.mapError((cause) => new InferenceError({
+			message: "Batch embedding inference failed",
+			cause
+		})));
+		const data = tensor.data;
+		const n = tensor.dims[0];
 		const results = [];
-		for (let i = 0; i < texts.length; i += BATCH_SIZE) {
-			const slice = texts.slice(i, i + BATCH_SIZE);
-			const tensor = yield* Effect.tryPromise(() => extractor(slice, {
-				pooling: "mean",
-				normalize: false
-			})).pipe(Effect.mapError((cause) => new InferenceError({
-				message: "Batch embedding inference failed",
-				cause
-			})));
-			const data = tensor.data;
-			const n = tensor.dims[0];
-			for (let j = 0; j < n; j++) {
-				const offset = j * cfg.dims;
-				results.push(normalize(data.slice(offset, offset + cfg.dims)));
-			}
+		for (let j = 0; j < n; j++) {
+			const offset = j * cfg.dims;
+			results.push({
+				vector: normalize(data.slice(offset, offset + cfg.dims)),
+				dims: cfg.dims
+			});
 		}
-		return results.map((vector) => ({
-			vector,
-			dims: cfg.dims
-		}));
+		return results;
 	}).pipe(Effect.provideService(Display, d));
+	const getFallbackInfo = () => Ref.get(fallbackRef).pipe(Effect.map(Option.getOrElse(() => void 0)));
 	return {
 		embed,
-		batch
+		batch,
+		getFallbackInfo
 	};
 });
 const OnnxEmbedderLive = Layer.provideMerge(Layer.effect(Embedder, make$2), ConfigStoreLive);
@@ -977,7 +1103,16 @@ const make$1 = Effect.gen(function* () {
 		if (result.skipped) skipped.push(result.skipped);
 		if (result.content.trim()) ig.add(result.content.split("\n"));
 	});
-	const loadGitignoreRules = (ignoredPaths, cwd) => Effect.gen(function* () {
+	const loadGitignoreRules = (ignoredPaths) => {
+		const ig = ignore();
+		const skipped = [];
+		if (ignoredPaths.length > 0) ig.add(ignoredPaths);
+		return Effect.succeed({
+			ig,
+			skipped
+		});
+	};
+	const loadGitignoreRulesWithFiles = (ignoredPaths, cwd) => Effect.gen(function* () {
 		const ig = ignore();
 		const skipped = [];
 		if (ignoredPaths.length > 0) ig.add(ignoredPaths);
@@ -1056,10 +1191,10 @@ const make$1 = Effect.gen(function* () {
 			skipped
 		};
 	});
-	const scanFiles = (ignoredPaths) => Effect.gen(function* () {
+	const scanFiles = (ignoredPaths, ignoreGitignore) => Effect.gen(function* () {
 		const cwd = process.cwd();
-		const { ig, skipped: ignoreSkipped } = yield* loadGitignoreRules(ignoredPaths, cwd).pipe(Effect.mapError((cause) => new ScanFailed({
-			message: `Failed to load gitignore rules: ${String(cause)}`,
+		const { ig, skipped: ignoreSkipped } = yield* (ignoreGitignore ? loadGitignoreRules(ignoredPaths) : loadGitignoreRulesWithFiles(ignoredPaths, cwd)).pipe(Effect.mapError((cause) => new ScanFailed({
+			message: `Failed to load ignore rules: ${String(cause)}`,
 			cause
 		})));
 		const { files, skipped: walkSkipped } = yield* walk(cwd, ig, cwd);
@@ -1083,6 +1218,22 @@ const isPlatformReason = (cause, reason) => typeof cause === "object" && cause !
 */
 const make = Effect.gen(function* () {
 	const fs = yield* FileSystem.FileSystem;
+	const chunksTemp = `${CHUNKS_FILE}.tmp`;
+	const vectorsTemp = `${VECTORS_FILE}.tmp`;
+	const seenFiles = yield* Ref.make(/* @__PURE__ */ new Set());
+	const statsAccumulator = yield* Ref.make({
+		chunks: 0,
+		files: 0,
+		totalLines: 0,
+		byteSize: 0
+	});
+	const serializeVectors = (embeddings) => {
+		const dims = embeddings[0]?.dims ?? 384;
+		const totalFloats = embeddings.length * dims;
+		const vectorsArray = new Float32Array(totalFloats);
+		for (let i = 0; i < embeddings.length; i++) vectorsArray.set(embeddings[i].vector, i * dims);
+		return Buffer.from(vectorsArray.buffer);
+	};
 	/**
 	* Count total lines across all chunks in chunks.jsonl. Each line is a JSON object; the 'text'
 	* field contains the source code.
@@ -1145,6 +1296,60 @@ const make = Effect.gen(function* () {
 			deleted: true
 		};
 	});
+	const storeBegin = () => Effect.gen(function* () {
+		yield* ensureDirExists(STORE_DIR, ".pix directory");
+		yield* Ref.set(seenFiles, /* @__PURE__ */ new Set());
+		yield* Ref.set(statsAccumulator, {
+			chunks: 0,
+			files: 0,
+			totalLines: 0,
+			byteSize: 0
+		});
+		if (yield* withStoreError(fs.exists(chunksTemp), "check chunks temp")) yield* withStoreError(fs.remove(chunksTemp), "clean stale chunks temp", chunksTemp);
+		if (yield* withStoreError(fs.exists(vectorsTemp), "check vectors temp")) yield* withStoreError(fs.remove(vectorsTemp), "clean stale vectors temp", vectorsTemp);
+	});
+	const storeBatch = (chunks, embeddings) => Effect.gen(function* () {
+		const content = chunks.map((c) => JSON.stringify({
+			id: c.id,
+			idx: c.idx,
+			file: c.file,
+			startLine: c.startLine,
+			endLine: c.endLine,
+			text: c.text
+		})).join("\n") + "\n";
+		yield* withStoreError(fs.writeFile(chunksTemp, Buffer.from(content), { flag: "a" }), "append chunks", chunksTemp);
+		const buffer = serializeVectors(embeddings);
+		yield* withStoreError(fs.writeFile(vectorsTemp, buffer, { flag: "a" }), "append vectors", vectorsTemp);
+		const dims = embeddings[0]?.dims ?? 384;
+		const batchLines = chunks.reduce((sum, c) => sum + (c.endLine - c.startLine + 1), 0);
+		const batchBytes = embeddings.length * dims * 4;
+		yield* Ref.update(seenFiles, (prev) => {
+			for (const c of chunks) prev.add(c.file);
+			return prev;
+		});
+		yield* Ref.update(statsAccumulator, (prev) => ({
+			chunks: prev.chunks + chunks.length,
+			files: 0,
+			totalLines: prev.totalLines + batchLines,
+			byteSize: prev.byteSize + batchBytes
+		}));
+	});
+	const storeCommit = () => Effect.gen(function* () {
+		yield* withStoreError(fs.rename(chunksTemp, CHUNKS_FILE), "commit chunks", CHUNKS_FILE);
+		yield* withStoreError(fs.rename(vectorsTemp, VECTORS_FILE), "commit vectors", VECTORS_FILE);
+		const stats = yield* Ref.get(statsAccumulator);
+		const files = yield* Ref.get(seenFiles);
+		yield* Ref.set(seenFiles, /* @__PURE__ */ new Set());
+		return {
+			...stats,
+			files: files.size
+		};
+	});
+	const storeAbort = () => Effect.gen(function* () {
+		yield* Ref.set(seenFiles, /* @__PURE__ */ new Set());
+		if (yield* withReadError(fs.exists(chunksTemp), "check chunks temp")) yield* withReadError(fs.remove(chunksTemp), "abort chunks temp", chunksTemp);
+		if (yield* withReadError(fs.exists(vectorsTemp), "check vectors temp")) yield* withReadError(fs.remove(vectorsTemp), "abort vectors temp", vectorsTemp);
+	});
 	const store = (chunks, embeddings) => Effect.gen(function* () {
 		yield* ensureDirExists(STORE_DIR, ".pix directory");
 		const chunksTemp = `${CHUNKS_FILE}.tmp`;
@@ -1159,11 +1364,7 @@ const make = Effect.gen(function* () {
 		yield* withStoreError(fs.writeFileString(chunksTemp, chunksLines.join("\n")), "write chunks", chunksTemp);
 		yield* withStoreError(fs.rename(chunksTemp, CHUNKS_FILE), "commit chunks", CHUNKS_FILE);
 		const vectorsTemp = `${VECTORS_FILE}.tmp`;
-		const dims = embeddings[0]?.dims ?? 384;
-		const totalFloats = embeddings.length * dims;
-		const vectorsArray = new Float32Array(totalFloats);
-		for (let i = 0; i < embeddings.length; i++) vectorsArray.set(embeddings[i].vector, i * dims);
-		const buffer = Buffer.from(vectorsArray.buffer);
+		const buffer = serializeVectors(embeddings);
 		yield* withStoreError(fs.writeFile(vectorsTemp, buffer), "write vectors", vectorsTemp);
 		yield* withStoreError(fs.rename(vectorsTemp, VECTORS_FILE), "commit vectors", VECTORS_FILE);
 	});
@@ -1232,6 +1433,10 @@ const make = Effect.gen(function* () {
 	});
 	return {
 		store,
+		storeBegin,
+		storeBatch,
+		storeCommit,
+		storeAbort,
 		search,
 		getStatus,
 		reset

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lucas-bur/pix",
-  "version": "0.10.0",
+  "version": "0.11.0",
   "description": "Lightweight local semantic project indexer",
   "keywords": [
     "cli",