@lucas-bur/pix 0.12.0 → 0.13.0

package/dist/index.mjs

@@ -2,12 +2,13 @@
 import { createRequire } from "node:module";
 import { Args, CliConfig, Command, Options } from "@effect/cli";
 import { NodeContext, NodeRuntime } from "@effect/platform-node";
-import { Clock, Context, Data, Effect, Exit, Layer, Option, Ref, Stream } from "effect";
+import { Clock, Context, Data, Effect, Exit, Layer, Option, Ref, Schema, Stream } from "effect";
 import * as Chunk from "effect/Chunk";
 import { styleText } from "node:util";
 import * as clack from "@clack/prompts";
 import { FileSystem } from "@effect/platform";
 import crypto from "node:crypto";
+import * as ParseResult from "effect/ParseResult";
 import { env } from "@huggingface/transformers";
 import ignore from "ignore";
 //#region src/domain/ports.ts
@@ -154,7 +155,26 @@ const JsonDisplay = { layer: Layer.succeed(Display, {
 }) };
 //#endregion
 //#region src/domain/config.ts
-var ConfigError = class extends Data.TaggedError("ConfigError") {};
+const EmbedderConfigSchema = Schema.Struct({
+	model: Schema.String,
+	device: Schema.Literal("auto", "cpu", "cuda", "dml", "coreml"),
+	dtype: Schema.Literal("fp32", "fp16", "q8"),
+	batchSize: Schema.Number
+});
+/**
+* Runtime schema for persisted project configuration. Defines the structure and validation rules
+* for `.pix/config.json`.
+*/
+const ConfigSchema = Schema.Struct({
+	schema: Schema.Literal("1"),
+	chunkLines: Schema.Number,
+	overlapLines: Schema.Number,
+	chunkConcurrency: Schema.optionalWith(Schema.Number, { exact: true }),
+	skipExtensions: Schema.Array(Schema.String),
+	ignoredPaths: Schema.Array(Schema.String),
+	ignoreGitignore: Schema.optionalWith(Schema.Boolean, { exact: true }),
+	embedder: EmbedderConfigSchema
+});
 const DEFAULT_CONFIG = {
 	schema: "1",
 	chunkLines: 60,
@@ -211,10 +231,14 @@ const getFileExtension = (filename) => {
 };
 //#endregion
 //#region src/domain/errors.ts
+/** Generic config I/O failure (read, write, encode). */
+var ConfigError = class extends Data.TaggedError("ConfigError") {};
 /** Config file or directory does not exist. Run pix init first. */
 var ConfigNotFoundError = class extends Data.TaggedError("ConfigNotFoundError") {};
 /** Config file exists but contains invalid JSON. */
 var ConfigMalformedError = class extends Data.TaggedError("ConfigMalformedError") {};
+/** Config failed schema validation — missing/invalid fields. */
+var ConfigValidationError = class extends Data.TaggedError("ConfigValidationError") {};
 /** Index files (chunks.jsonl, vectors.bin) do not exist. Run pix index first. */
 var NoIndexError = class extends Data.TaggedError("NoIndexError") {};
 /** Disk is full — write operation could not complete. */
@@ -232,6 +256,8 @@ var InferenceError = class extends Data.TaggedError("InferenceError") {};
 * via ScanResult.skipped.
 */
 var ScanFailed = class extends Data.TaggedError("ScanFailed") {};
+/** A chunk line in chunks.jsonl failed schema validation. */
+var ChunkValidationError = class extends Data.TaggedError("ChunkValidationError") {};
 /** File type is unsupported for text extraction. */
 var UnsupportedFormat = class extends Data.TaggedError("UnsupportedFormat") {};
 /** Text extraction failed for a supported file type. */
@@ -405,7 +431,8 @@ var IndexProject = class extends Effect.Service()("IndexProject", {
 						chunks: 0,
 						files: 0,
 						totalLines: 0,
-						byteSize: 0
+						byteSize: 0,
+						validationErrors: []
 					},
 					durationMs: Date.now() - start
 				};
@@ -420,7 +447,8 @@ var IndexProject = class extends Effect.Service()("IndexProject", {
 						chunks: 0,
 						files: 0,
 						totalLines: 0,
-						byteSize: 0
+						byteSize: 0,
+						validationErrors: []
 					},
 					durationMs: Date.now() - start
 				};
@@ -454,7 +482,8 @@ var IndexProject = class extends Effect.Service()("IndexProject", {
 					chunks: stats.chunks,
 					files: stats.files,
 					totalLines: stats.totalLines,
-					byteSize: stats.byteSize
+					byteSize: stats.byteSize,
+					validationErrors: []
 				},
 				durationMs: Date.now() - start,
 				embedderFallback: fallbackInfo
@@ -543,6 +572,8 @@ const errorCodes = {
 	ConfigError: "CONFIG_ERROR",
 	ConfigNotFoundError: "CONFIG_NOT_FOUND",
 	ConfigMalformedError: "CONFIG_MALFORMED",
+	ConfigValidationError: "CONFIG_VALIDATION_ERROR",
+	ChunkValidationError: "CHUNK_VALIDATION_ERROR",
 	NoIndexError: "NO_INDEX",
 	DiskFullError: "DISK_FULL",
 	StoreError: "STORE_ERROR",
@@ -754,8 +785,12 @@ const buildSearchOptions = (top, ignorePath, onlyPath) => {
 	};
 };
 /** Render search results via Display — JSON + human-readable text. */
-const renderResults = (d, results, ctxLines, noContent) => Effect.gen(function* () {
-	yield* d.json(toJsonOutput(results, ctxLines, noContent));
+const renderResults = (d, response, ctxLines, noContent) => Effect.gen(function* () {
+	const { results, validationErrors } = response;
+	yield* d.json({
+		results: toJsonOutput(results, ctxLines, noContent),
+		...validationErrors.length > 0 && { validationErrors }
+	});
 	if (results.length === 0) yield* d.log("No results found", "warn");
 	else for (const result of results) yield* d.text(noContent ? formatLocation(result) : formatResult(result));
 });
@@ -777,8 +812,12 @@ const queryCommand = Command.make("query", {
 	const ctxLines = Option.getOrElse(contextLines, () => DEFAULT_CONTEXT_LINES);
 	const { options: searchOptions, clamped, rawValue } = buildSearchOptions(top, ignorePath, onlyPath);
 	if (clamped) yield* d.log(`topK clamped from ${rawValue} to ${searchOptions.topK}`, "warn");
-	const results = yield* d.spinner("Searching...", QueryProject.queryProject(queryText, searchOptions));
-	yield* renderResults(d, noContent ? results : applyCharBudget(results, Option.getOrUndefined(maxCharacters)).results, ctxLines, noContent);
+	const searchResponse = yield* d.spinner("Searching...", QueryProject.queryProject(queryText, searchOptions));
+	const finalResults = noContent ? searchResponse.results : applyCharBudget(searchResponse.results, Option.getOrUndefined(maxCharacters)).results;
+	yield* renderResults(d, {
+		...searchResponse,
+		results: finalResults
+	}, ctxLines, noContent);
 }).pipe(Effect.catchTags({
 	ModelLoadError: reportError,
 	InferenceError: reportError,
@@ -825,6 +864,7 @@ const statusCommand = Command.make("status", { json: Options.boolean("json").pip
 	yield* d.log(`Total lines: ${result.totalLines.toLocaleString()}`, "info");
 	yield* d.log(`Index size: ${result.byteSize.toLocaleString()} bytes`, "info");
 	yield* d.log(`Last indexed: ${lastIndexStr}`, "info");
+	if (result.validationErrors.length > 0) yield* d.log(`Warnings: ${result.validationErrors[0].message}`, "warn");
 }).pipe(Effect.catchTags({ StoreError: reportError })));
 //#endregion
 //#region src/cli.ts
@@ -947,6 +987,46 @@ const ChunkerLive = Layer.effect(Chunker, make$5);
 */
 const isPlatformReason = (cause, reason) => typeof cause === "object" && cause !== null && "reason" in cause && String(cause.reason) === reason;
 //#endregion
+//#region src/lib/validation.ts
+const mergeMessages = (messages) => {
+	if (messages.length === 1) return messages[0];
+	const uniq = [...new Set(messages)];
+	if (uniq.every((m) => m.startsWith("Expected"))) {
+		const actualMatch = uniq[0].match(/actual (.+)$/);
+		const actual = actualMatch ? actualMatch[1] : "";
+		return `Expected ${uniq.map((m) => m.replace(/^Expected /, "").replace(/, actual .+$/, "")).join(" | ")}, actual ${actual}`;
+	}
+	return uniq.join("\n");
+};
+const formatSchemaErrors = (error) => {
+	const issues = ParseResult.ArrayFormatter.formatErrorSync(error);
+	const byPath = /* @__PURE__ */ new Map();
+	for (const issue of issues) {
+		const path = issue.path.join(".");
+		if (!byPath.has(path)) byPath.set(path, []);
+		byPath.get(path).push(issue.message);
+	}
+	return Array.from(byPath.entries()).map(([path, messages]) => ({
+		path,
+		message: mergeMessages(messages)
+	}));
+};
+const formatSchemaMessage = (error) => ParseResult.TreeFormatter.formatErrorSync(error);
+const isJsonSyntaxError = (error) => error.issue._tag === "Transformation" && error.issue.kind === "Transformation";
+const decodeJsonWithErrors = (schema, json) => Schema.decodeUnknown(Schema.parseJson(schema))(json).pipe(Effect.mapError((error) => {
+	const base = {
+		message: formatSchemaMessage(error),
+		errors: formatSchemaErrors(error)
+	};
+	return isJsonSyntaxError(error) ? {
+		...base,
+		_tag: "JsonSyntaxError"
+	} : {
+		...base,
+		_tag: "SchemaValidationError"
+	};
+}));
+//#endregion
 //#region src/services/config-store.ts
 const CONFIG_DIR = ".pix";
 const CONFIG_PATH = `${CONFIG_DIR}/config.json`;
@@ -964,12 +1044,16 @@ const mapConfigWriteError = (cause, path, action) => {
 const make$4 = Effect.gen(function* () {
 	const fs = yield* FileSystem.FileSystem;
 	const writeConfig = (config) => Effect.gen(function* () {
-		const configJson = JSON.stringify(config, null, 2);
+		const encodeJson = Schema.parseJson(ConfigSchema, { space: 2 });
+		const configJson = yield* Schema.encode(encodeJson)(config).pipe(Effect.mapError((e) => new ConfigError({
+			message: "Failed to encode config",
+			cause: e
+		})));
 		yield* fs.makeDirectory(CONFIG_DIR, { recursive: true }).pipe(Effect.mapError((cause) => mapConfigWriteError(cause, CONFIG_DIR, "create .pix directory")));
 		yield* fs.writeFileString(CONFIG_PATH, configJson).pipe(Effect.mapError((cause) => mapConfigWriteError(cause, CONFIG_PATH, "write config.json")));
 	});
 	const readConfig = () => Effect.gen(function* () {
-		const content = yield* fs.readFileString(CONFIG_PATH).pipe(Effect.mapError((cause) => {
+		return yield* decodeJsonWithErrors(ConfigSchema, yield* fs.readFileString(CONFIG_PATH).pipe(Effect.mapError((cause) => {
 			if (isPlatformReason(cause, "NotFound")) return new ConfigNotFoundError({
 				message: "Config file not found. Run pix init first.",
 				path: CONFIG_PATH,
@@ -979,15 +1063,17 @@ const make$4 = Effect.gen(function* () {
 				message: "Failed to read config.json",
 				cause
 			});
-		}));
-		return yield* Effect.try({
-			try: () => JSON.parse(content),
-			catch: (error) => new ConfigMalformedError({
+		}))).pipe(Effect.mapError((err) => {
+			if (err._tag === "JsonSyntaxError") return new ConfigMalformedError({
 				message: "Invalid JSON in config.json",
 				path: CONFIG_PATH,
-				cause: error
-			})
-		});
+				cause: err
+			});
+			return new ConfigValidationError({
+				message: err.message,
+				errors: err.errors
+			});
+		}));
 	});
 	const configExists = () => Effect.gen(function* () {
 		return yield* fs.exists(CONFIG_PATH);
@@ -1295,21 +1381,26 @@ const make$1 = Effect.gen(function* () {
 });
 const ScannerLive = Layer.effect(Scanner, make$1);
 //#endregion
+//#region src/domain/chunk.ts
+/** Runtime schema for persisted/searchable chunk entries. */
+const ChunkSchema = Schema.Struct({
+	id: Schema.String,
+	idx: Schema.Number,
+	file: Schema.String,
+	startLine: Schema.Number,
+	endLine: Schema.Number,
+	text: Schema.String,
+	contextBefore: Schema.Union(Schema.String, Schema.Null),
+	contextAfter: Schema.Union(Schema.String, Schema.Null)
+});
+//#endregion
 //#region src/services/vector-store.ts
-/**
-* Parse a single JSON line from chunks.jsonl and normalize context fields (old indexes may lack
-* them).
-*/
 const parseChunkLine = (line) => {
-	const raw = JSON.parse(line);
-	return {
-		file: typeof raw.file === "string" ? raw.file : "",
-		startLine: typeof raw.startLine === "number" ? raw.startLine : 0,
-		endLine: typeof raw.endLine === "number" ? raw.endLine : 0,
-		text: typeof raw.text === "string" ? raw.text : "",
-		contextBefore: typeof raw.contextBefore === "string" ? raw.contextBefore : null,
-		contextAfter: typeof raw.contextAfter === "string" ? raw.contextAfter : null
-	};
+	try {
+		return Option.some(Schema.decodeUnknownSync(Schema.parseJson(ChunkSchema))(line));
+	} catch {
+		return Option.none();
+	}
 };
 /** Compute dot-product similarity between a chunk vector and the query embedding. */
 const computeDotProduct = (chunkVector, query) => {
@@ -1320,20 +1411,16 @@ const computeDotProduct = (chunkVector, query) => {
 const STORE_DIR = ".pix";
 const CHUNKS_FILE = `${STORE_DIR}/chunks.jsonl`;
 const VECTORS_FILE = `${STORE_DIR}/vectors.bin`;
-/**
-* Serialize a Chunk to a JSON object for storage in chunks.jsonl. Always includes context fields
-* for schema consistency.
-*/
-const serializeChunk = (c) => ({
-	id: c.id,
-	idx: c.idx,
-	file: c.file,
-	startLine: c.startLine,
-	endLine: c.endLine,
-	text: c.text,
-	contextBefore: c.contextBefore,
-	contextAfter: c.contextAfter
-});
+/** Pre-built Schema instance for chunk encode/decode. */
+const parseJsonChunk = Schema.parseJson(ChunkSchema);
+/** Build ChunkValidationError array from malformed line count, or [] if none. */
+const buildChunkValidationErrors = (malformedLines) => malformedLines > 0 ? [new ChunkValidationError({
+	message: `Skipped ${malformedLines} malformed chunk line(s) in chunks.jsonl`,
+	errors: [{
+		path: "chunks.jsonl",
+		message: `${malformedLines} line(s) failed schema validation`
+	}]
+})] : [];
 /**
 * FileSystem adapter for VectorStore port. Reads from chunks.jsonl and vectors.bin to provide index
 * statistics.
@@ -1360,22 +1447,30 @@ const make = Effect.gen(function* () {
 	* Count total lines across all chunks in chunks.jsonl. Each line is a JSON object; the 'text'
 	* field contains the source code.
 	*/
-	const countTotalLines = (lines) => lines.reduce((sum, line) => {
-		try {
-			return sum + JSON.parse(line).text.split("\n").length;
-		} catch {
-			return sum;
-		}
-	}, 0);
-	/** Count unique files across all chunks in chunks.jsonl. */
-	const countUniqueFiles = (lines) => {
+	/** Count files, total lines, and malformed lines in a single pass. */
+	const countChunkStats = (lines) => {
 		const files = /* @__PURE__ */ new Set();
-		for (const line of lines) try {
-			const chunk = JSON.parse(line);
-			files.add(chunk.file);
-		} catch {}
-		return files;
+		let totalLines = 0;
+		let malformedLines = 0;
+		for (const line of lines) {
+			const chunk = parseChunkLine(line);
+			if (Option.isSome(chunk)) {
+				files.add(chunk.value.file);
+				totalLines += chunk.value.text.split("\n").length;
+			} else malformedLines++;
+		}
+		return {
+			files,
+			totalLines,
+			malformedLines
+		};
 	};
+	/** Check that index files exist; fail with NoIndexError if either is missing. */
+	const requireIndex = () => Effect.gen(function* () {
+		const chunksExists = yield* withReadError(fs.exists(CHUNKS_FILE), "check chunks file");
+		const vectorsExists = yield* withReadError(fs.exists(VECTORS_FILE), "check vectors file");
+		if (!chunksExists || !vectorsExists) return yield* new NoIndexError({ message: "No index found. Run pix index first." });
+	});
 	const toStoreError = (operation, path) => (cause) => {
 		if (isPlatformReason(cause, "BadResource")) return new DiskFullError({
 			message: `Disk full during ${operation}`,
@@ -1431,7 +1526,10 @@ const make = Effect.gen(function* () {
 		if (yield* withStoreError(fs.exists(vectorsTemp), "check vectors temp")) yield* withStoreError(fs.remove(vectorsTemp), "clean stale vectors temp", vectorsTemp);
 	});
 	const storeBatch = (chunks, embeddings) => Effect.gen(function* () {
-		const content = chunks.map((c) => JSON.stringify(serializeChunk(c))).join("\n") + "\n";
+		const content = (yield* Effect.forEach(chunks, (c) => Schema.encode(parseJsonChunk)(c).pipe(Effect.mapError((e) => new StoreError({
+			message: "Failed to encode chunk",
+			cause: e
+		}))))).join("\n") + "\n";
 		yield* withStoreError(fs.writeFile(chunksTemp, Buffer.from(content), { flag: "a" }), "append chunks", chunksTemp);
 		const buffer = serializeVectors(embeddings);
 		yield* withStoreError(fs.writeFile(vectorsTemp, buffer, { flag: "a" }), "append vectors", vectorsTemp);
@@ -1468,7 +1566,10 @@ const make = Effect.gen(function* () {
 	const store = (chunks, embeddings) => Effect.gen(function* () {
 		yield* ensureDirExists(STORE_DIR, ".pix directory");
 		const chunksTemp = `${CHUNKS_FILE}.tmp`;
-		const chunksJson = chunks.map((c) => JSON.stringify(serializeChunk(c))).join("\n");
+		const chunksJson = (yield* Effect.forEach(chunks, (c) => Schema.encode(parseJsonChunk)(c).pipe(Effect.mapError((e) => new StoreError({
+			message: "Failed to encode chunk",
+			cause: e
+		}))))).join("\n");
 		yield* withStoreError(fs.writeFileString(chunksTemp, chunksJson), "write chunks", chunksTemp);
 		yield* withStoreError(fs.rename(chunksTemp, CHUNKS_FILE), "commit chunks", CHUNKS_FILE);
 		const vectorsTemp = `${VECTORS_FILE}.tmp`;
@@ -1477,17 +1578,21 @@ const make = Effect.gen(function* () {
 		yield* withStoreError(fs.rename(vectorsTemp, VECTORS_FILE), "commit vectors", VECTORS_FILE);
 	});
 	const search = (query, options) => Effect.gen(function* () {
-		const chunksExists = yield* withReadError(fs.exists(CHUNKS_FILE), "check chunks file");
-		const vectorsExists = yield* withReadError(fs.exists(VECTORS_FILE), "check vectors file");
-		if (!chunksExists || !vectorsExists) return yield* new NoIndexError({ message: "No index found. Run pix index first." });
+		yield* requireIndex();
 		const chunkLines = (yield* withReadError(fs.readFileString(CHUNKS_FILE), "read chunks", CHUNKS_FILE)).split("\n").filter((l) => l.trim().length > 0);
 		const vectorsBuffer = yield* withReadError(fs.readFile(VECTORS_FILE), "read vectors", VECTORS_FILE);
 		const vectors = new Float32Array(vectorsBuffer.buffer, vectorsBuffer.byteOffset, vectorsBuffer.byteLength / Float32Array.BYTES_PER_ELEMENT);
 		const ignoreIg = options?.ignorePaths?.length ? ignore().add([...options.ignorePaths]) : null;
 		const onlyIg = options?.onlyPaths?.length ? ignore().add([...options.onlyPaths]) : null;
 		const results = [];
-		for (let i = 0; i < chunkLines.length; i++) try {
-			const chunk = parseChunkLine(chunkLines[i]);
+		let malformedLines = 0;
+		for (let i = 0; i < chunkLines.length; i++) {
+			const parsed = parseChunkLine(chunkLines[i]);
+			if (Option.isNone(parsed)) {
+				malformedLines++;
+				continue;
+			}
+			const chunk = parsed.value;
 			if (ignoreIg && ignoreIg.ignores(chunk.file)) continue;
 			if (onlyIg && !onlyIg.ignores(chunk.file)) continue;
 			const startIdx = i * query.dims;
@@ -1501,12 +1606,19 @@ const make = Effect.gen(function* () {
 				contextBefore: chunk.contextBefore,
 				contextAfter: chunk.contextAfter
 			});
-		} catch {}
+		}
+		const validationErrors = buildChunkValidationErrors(malformedLines);
 		results.sort((a, b) => b.score - a.score);
 		const topK = options?.topK;
-		if (topK == null) return results;
+		if (topK == null) return {
+			results,
+			validationErrors
+		};
 		const clamped = Math.max(0, Math.min(Math.floor(topK), results.length));
-		return results.slice(0, clamped);
+		return {
+			results: results.slice(0, clamped),
+			validationErrors
+		};
 	});
 	const getStatus = () => Effect.gen(function* () {
 		const chunksExists = yield* withReadError(fs.exists(CHUNKS_FILE), "check chunks file");
@@ -1517,13 +1629,15 @@ const make = Effect.gen(function* () {
 			model: "",
 			lastIndex: 0,
 			totalLines: 0,
-			byteSize: 0
+			byteSize: 0,
+			validationErrors: []
 		};
 		const lines = (yield* withReadError(fs.readFileString(CHUNKS_FILE), "read chunks", CHUNKS_FILE)).split("\n").filter((l) => l.trim().length > 0);
-		const chunks = lines.length;
-		const files = countUniqueFiles(lines).size;
+		const { files: uniqueFiles, totalLines, malformedLines } = countChunkStats(lines);
+		const chunks = lines.length - malformedLines;
+		const files = uniqueFiles.size;
 		const model = "";
-		const totalLines = countTotalLines(lines);
+		const validationErrors = buildChunkValidationErrors(malformedLines);
 		const vectorsStat = yield* withReadError(fs.stat(VECTORS_FILE), "stat vectors", VECTORS_FILE);
 		const byteSize = "size" in vectorsStat ? Number(vectorsStat.size) : 0;
 		return {
@@ -1532,7 +1646,8 @@ const make = Effect.gen(function* () {
 			model,
 			lastIndex: Option.map(vectorsStat?.mtime ?? Option.none(), (d) => d instanceof Date ? d.getTime() : 0).pipe(Option.getOrElse(() => 0)),
 			totalLines,
-			byteSize
+			byteSize,
+			validationErrors
 		};
 	});
 	const reset = () => Effect.gen(function* () {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lucas-bur/pix",
-  "version": "0.12.0",
+  "version": "0.13.0",
   "description": "Lightweight local semantic project indexer",
   "keywords": [
     "cli",