@lucas-bur/pix 0.9.1 → 0.10.0

package/dist/index.mjs

@@ -5,13 +5,14 @@ import { NodeContext, NodeRuntime } from "@effect/platform-node";
 import { Clock, Context, Data, Effect, Layer, Option, Ref } from "effect";
 import { styleText } from "node:util";
 import * as clack from "@clack/prompts";
-import crypto from "node:crypto";
 import { FileSystem } from "@effect/platform";
+import crypto from "node:crypto";
 import { env } from "@huggingface/transformers";
 import ignore from "ignore";
 //#region src/domain/ports.ts
 var ConfigStore = class extends Context.Tag("ConfigStore")() {};
 var Scanner = class extends Context.Tag("Scanner")() {};
+var ContentExtractor = class extends Context.Tag("ContentExtractor")() {};
 var Chunker = class extends Context.Tag("Chunker")() {};
 var Embedder = class extends Context.Tag("Embedder")() {};
 var VectorStore = class extends Context.Tag("VectorStore")() {};
@@ -140,7 +141,23 @@ const DEFAULT_CONFIG = {
 	chunkLines: 60,
 	overlapLines: 10,
 	chunkConcurrency: 8,
-	files: {},
+	skipExtensions: [],
+	ignoredPaths: [
+		".pix",
+		"node_modules",
+		".git",
+		"dist",
+		"build",
+		".next",
+		".agents",
+		".claude",
+		".vscode",
+		".github",
+		"coverage",
+		"*-lock.yaml",
+		"*-lock.json",
+		"*.lock"
+	],
 	embedder: {
 		model: "Xenova/all-MiniLM-L6-v2",
 		device: "auto",
@@ -148,10 +165,158 @@ const DEFAULT_CONFIG = {
 	}
 };
 //#endregion
+//#region src/domain/errors.ts
+/** Config file or directory does not exist. Run pix init first. */
+var ConfigNotFoundError = class extends Data.TaggedError("ConfigNotFoundError") {};
+/** Config file exists but contains invalid JSON. */
+var ConfigMalformedError = class extends Data.TaggedError("ConfigMalformedError") {};
+/** Index files (chunks.jsonl, vectors.bin) do not exist. Run pix index first. */
+var NoIndexError = class extends Data.TaggedError("NoIndexError") {};
+/** Disk is full — write operation could not complete. */
+var DiskFullError = class extends Data.TaggedError("DiskFullError") {};
+/** Generic index store I/O failure (read, write, delete). */
+var StoreError = class extends Data.TaggedError("StoreError") {};
+/** Source file could not be read during chunking (binary, permissions, encoding). */
+var ChunkerError = class extends Data.TaggedError("ChunkerError") {};
+/** Embedding model could not be downloaded or loaded. */
+var ModelLoadError = class extends Data.TaggedError("ModelLoadError") {};
+/** Embedding model failed during inference. */
+var InferenceError = class extends Data.TaggedError("InferenceError") {};
+/**
+* Fatal scan failure — gitignore loading failed entirely. Non-fatal per-entry skips are reported
+* via ScanResult.skipped.
+*/
+var ScanFailed = class extends Data.TaggedError("ScanFailed") {};
+/** File type is unsupported for text extraction. */
+var UnsupportedFormat = class extends Data.TaggedError("UnsupportedFormat") {};
+/** Text extraction failed for a supported file type. */
+var ExtractionFailed = class extends Data.TaggedError("ExtractionFailed") {};
+//#endregion
+//#region src/services/processors/identity.ts
+const identityProcessor = (file) => FileSystem.FileSystem.pipe(Effect.flatMap((fs) => fs.readFileString(file)), Effect.mapError((cause) => new ExtractionFailed({
+	message: `Failed to read file for extraction: ${file}`,
+	file,
+	cause
+})));
+//#endregion
+//#region src/services/processors/skip.ts
+const skipProcessor = (extension) => {
+	const error = new UnsupportedFormat({
+		message: `Unsupported file type: ${extension}`,
+		extension
+	});
+	return (_file) => Effect.fail(error);
+};
+//#endregion
+//#region src/services/processors/index.ts
+const DEFAULT_PROCESSOR_MAP = {
+	".ts": identityProcessor,
+	".tsx": identityProcessor,
+	".js": identityProcessor,
+	".jsx": identityProcessor,
+	".py": identityProcessor,
+	".rs": identityProcessor,
+	".go": identityProcessor,
+	".java": identityProcessor,
+	".c": identityProcessor,
+	".cpp": identityProcessor,
+	".h": identityProcessor,
+	".hpp": identityProcessor,
+	".json": identityProcessor,
+	".yaml": identityProcessor,
+	".yml": identityProcessor,
+	".toml": identityProcessor,
+	".xml": identityProcessor,
+	".csv": identityProcessor,
+	".md": identityProcessor,
+	".mdx": identityProcessor,
+	".txt": identityProcessor,
+	".rst": identityProcessor,
+	".html": identityProcessor,
+	".css": identityProcessor,
+	".scss": identityProcessor,
+	".less": identityProcessor,
+	".sql": identityProcessor,
+	".graphql": identityProcessor,
+	".sh": identityProcessor,
+	".bash": identityProcessor,
+	".ps1": identityProcessor,
+	".bat": identityProcessor,
+	".cmake": identityProcessor,
+	".dockerfile": identityProcessor,
+	dockerfile: identityProcessor,
+	makefile: identityProcessor,
+	gemfile: identityProcessor,
+	".pdf": skipProcessor(".pdf"),
+	".png": skipProcessor(".png"),
+	".jpg": skipProcessor(".jpg"),
+	".jpeg": skipProcessor(".jpeg"),
+	".gif": skipProcessor(".gif"),
+	".svg": identityProcessor,
+	".ico": skipProcessor(".ico"),
+	".webp": skipProcessor(".webp"),
+	".mp3": skipProcessor(".mp3"),
+	".mp4": skipProcessor(".mp4"),
+	".wav": skipProcessor(".wav"),
+	".avi": skipProcessor(".avi"),
+	".mov": skipProcessor(".mov"),
+	".mkv": skipProcessor(".mkv"),
+	".exe": skipProcessor(".exe"),
+	".dll": skipProcessor(".dll"),
+	".so": skipProcessor(".so"),
+	".zip": skipProcessor(".zip"),
+	".tar": skipProcessor(".tar"),
+	".gz": skipProcessor(".gz"),
+	".7z": skipProcessor(".7z"),
+	".rar": skipProcessor(".rar"),
+	".ttf": skipProcessor(".ttf"),
+	".woff": skipProcessor(".woff"),
+	".woff2": skipProcessor(".woff2"),
+	".eot": skipProcessor(".eot"),
+	".otf": skipProcessor(".otf"),
+	".lock": identityProcessor,
+	lock: identityProcessor
+};
+/**
+* Builds the processor map by merging domain defaults with user-specified skip extensions. Skip
+* extensions override any existing mapping with a skip processor. Unknown extensions remain absent
+* from the map — callers decide how to handle them.
+*/
+function buildProcessorMap(skipExtensions) {
+	const mapped = { ...DEFAULT_PROCESSOR_MAP };
+	for (const ext of skipExtensions) mapped[ext] = skipProcessor(ext);
+	return mapped;
+}
+//#endregion
 //#region src/application/index-project.ts
+function getExtension(file) {
+	const lastSlash = file.lastIndexOf("/");
+	const name = lastSlash >= 0 ? file.slice(lastSlash + 1) : file;
+	const dotIndex = name.lastIndexOf(".");
+	if (dotIndex === -1) return name.toLowerCase();
+	return name.slice(dotIndex).toLowerCase();
+}
+const classifyFiles = (files, processorMap) => {
+	const knownFiles = [];
+	const skippedFiles = [];
+	const unknownExtensions = /* @__PURE__ */ new Set();
+	for (const file of files) {
+		const ext = getExtension(file);
+		if (!processorMap[ext]) {
+			unknownExtensions.add(ext);
+			skippedFiles.push(file);
+		} else knownFiles.push(file);
+	}
+	return {
+		knownFiles,
+		skippedFiles,
+		unknownExtensions
+	};
+};
 /**
-* Use case: index project files. Pipeline: scan → chunk → embed → store. Depends on ConfigStore,
-* Scanner, Chunker, Embedder, VectorStore, Display via Effect tags.
+* Use case: index project files. Pipeline: scan → ContentExtractor → chunk → embed → store. Depends
+* on ConfigStore, Scanner, Chunker, Embedder, VectorStore, Display, ContentExtractor via Effect
+* tags.
 */
 var IndexProject = class extends Effect.Service()("IndexProject", {
 	accessors: true,
@@ -162,19 +327,36 @@ var IndexProject = class extends Effect.Service()("IndexProject", {
 		const embedder = yield* Embedder;
 		const vectorStore = yield* VectorStore;
 		const d = yield* Display;
+		const extractor = yield* ContentExtractor;
 		const index = () => Effect.gen(function* () {
 			if (!(yield* configStore.configExists())) yield* configStore.writeConfig(DEFAULT_CONFIG);
 			const config = yield* configStore.readConfig();
-			const extensions = Object.keys(config.files).length > 0 ? Object.keys(config.files) : [
-				".ts",
-				".tsx",
-				".js",
-				".jsx"
-			];
+			const processorMap = buildProcessorMap(config.skipExtensions);
 			yield* d.updateInteractive("Scanning source files...");
-			const scanResult = yield* scanner.scanFiles(extensions);
-			yield* d.updateInteractive(`Chunking ${scanResult.files.length} files...`);
-			const allChunks = (yield* Effect.forEach(scanResult.files, (file) => chunker.chunkFile(file), { concurrency: Math.max(1, config.chunkConcurrency ?? 8) })).flat();
+			const ignoredPaths = config.ignoredPaths ?? DEFAULT_CONFIG.ignoredPaths;
+			const { knownFiles, skippedFiles, unknownExtensions } = classifyFiles((yield* scanner.scanFiles(ignoredPaths)).files, processorMap);
+			if (unknownExtensions.size > 0) yield* d.log(`Skipped ${skippedFiles.length} files with unknown extensions: ${[...unknownExtensions].join(", ")}`, "warn");
+			if (knownFiles.length === 0) return {
+				success: true,
+				status: {
+					chunks: 0,
+					files: 0,
+					totalLines: 0,
+					byteSize: 0
+				}
+			};
+			yield* d.updateInteractive(`Processing ${knownFiles.length} files...`);
+			const allChunks = (yield* Effect.forEach(knownFiles, (file) => Effect.gen(function* () {
+				const result = yield* Effect.either(extractor.extract(file));
+				if (result._tag === "Left") {
+					if (result.left._tag === "UnsupportedFormat") {
+						yield* d.log(`Skipping ${file}: ${result.left.message}`, "warn");
+						return [];
+					}
+					return yield* Effect.fail(result.left);
+				}
+				return yield* chunker.chunkText(result.right, file);
+			}), { concurrency: Math.max(1, config.chunkConcurrency ?? 8) })).flat();
 			const totalChunks = allChunks.length;
 			const totalFiles = new Set(allChunks.map((c) => c.file)).size;
 			const totalLines = allChunks.reduce((sum, c) => sum + (c.endLine - c.startLine + 1), 0);
@@ -493,29 +675,6 @@ const setupTerminalCleanup = () => {
 	process.on("exit", makeTerminalCleanupHandler(process.stdin, process.stdout));
 };
 //#endregion
-//#region src/domain/errors.ts
-/** Config file or directory does not exist. Run pix init first. */
-var ConfigNotFoundError = class extends Data.TaggedError("ConfigNotFoundError") {};
-/** Config file exists but contains invalid JSON. */
-var ConfigMalformedError = class extends Data.TaggedError("ConfigMalformedError") {};
-/** Index files (chunks.jsonl, vectors.bin) do not exist. Run pix index first. */
-var NoIndexError = class extends Data.TaggedError("NoIndexError") {};
-/** Disk is full — write operation could not complete. */
-var DiskFullError = class extends Data.TaggedError("DiskFullError") {};
-/** Generic index store I/O failure (read, write, delete). */
-var StoreError = class extends Data.TaggedError("StoreError") {};
-/** Source file could not be read during chunking (binary, permissions, encoding). */
-var ChunkerError = class extends Data.TaggedError("ChunkerError") {};
-/** Embedding model could not be downloaded or loaded. */
-var ModelLoadError = class extends Data.TaggedError("ModelLoadError") {};
-/** Embedding model failed during inference. */
-var InferenceError = class extends Data.TaggedError("InferenceError") {};
-/**
-* Fatal scan failure — gitignore loading failed entirely. Non-fatal per-entry skips are reported
-* via ScanResult.skipped.
-*/
-var ScanFailed = class extends Data.TaggedError("ScanFailed") {};
-//#endregion
 //#region src/services/chunker.ts
 const MIN_CHUNK_CHARS = 20;
 const readFileContent = (fs, file) => fs.readFileString(file).pipe(Effect.mapError((cause) => new ChunkerError({
@@ -547,17 +706,24 @@ const buildChunks = (file, content, config) => {
 	}
 	return chunks;
 };
-const make$4 = Effect.gen(function* () {
+const make$5 = Effect.gen(function* () {
 	const fs = yield* FileSystem.FileSystem;
 	const config = yield* (yield* ConfigStore).readConfig().pipe(Effect.catchAll(() => Effect.succeed(DEFAULT_CONFIG)));
+	const chunkText = (text, file) => Effect.sync(() => {
+		if (text === "") return [];
+		return buildChunks(file, text, config);
+	});
 	const chunkFile = (file) => Effect.gen(function* () {
 		const content = yield* readFileContent(fs, file);
 		if (content === "") return [];
 		return buildChunks(file, content, config);
 	});
-	return { chunkFile };
+	return {
+		chunkFile,
+		chunkText
+	};
 });
-const ChunkerLive = Layer.effect(Chunker, make$4);
+const ChunkerLive = Layer.effect(Chunker, make$5);
 //#endregion
 //#region src/services/config-store.ts
 const CONFIG_DIR = ".pix";
@@ -574,7 +740,7 @@ const mapConfigWriteError = (cause, path, action) => {
 		cause
 	});
 };
-const make$3 = Effect.gen(function* () {
+const make$4 = Effect.gen(function* () {
 	const fs = yield* FileSystem.FileSystem;
 	const writeConfig = (config) => Effect.gen(function* () {
 		const configJson = JSON.stringify(config, null, 2);
@@ -611,7 +777,28 @@ const make$3 = Effect.gen(function* () {
 		configExists
 	};
 });
-const ConfigStoreLive = Layer.effect(ConfigStore, make$3);
+const ConfigStoreLive = Layer.effect(ConfigStore, make$4);
+//#endregion
+//#region src/services/content-extractor.ts
+const make$3 = Effect.gen(function* () {
+	const fs = yield* FileSystem.FileSystem;
+	const processorMap = buildProcessorMap([]);
+	const extract = (file) => {
+		const lastSlash = file.lastIndexOf("/");
+		const name = lastSlash >= 0 ? file.slice(lastSlash + 1) : file;
+		const dotIndex = name.lastIndexOf(".");
+		const ext = dotIndex === -1 ? name.toLowerCase() : name.slice(dotIndex).toLowerCase();
+		const processor = processorMap[ext];
+		if (!processor) return Effect.fail({
+			_tag: "UnsupportedFormat",
+			message: `No processor for extension: ${ext}`,
+			extension: ext
+		});
+		return processor(file).pipe(Effect.provideService(FileSystem.FileSystem, fs));
+	};
+	return { extract };
+});
+const ContentExtractorLive = Layer.effect(ContentExtractor, make$3);
 //#endregion
 //#region src/domain/models.ts
 /** Registry of supported embedding models. */
@@ -752,14 +939,6 @@ const make$2 = Effect.gen(function* () {
 const OnnxEmbedderLive = Layer.provideMerge(Layer.effect(Embedder, make$2), ConfigStoreLive);
 //#endregion
 //#region src/services/scanner.ts
-const ALWAYS_IGNORE = new Set([
-	".pix",
-	"node_modules",
-	".git",
-	"dist",
-	"build",
-	".next"
-]);
 const make$1 = Effect.gen(function* () {
 	const fs = yield* FileSystem.FileSystem;
 	const readFileWithSkip = (path, mkReason) => fs.readFileString(path).pipe(Effect.map((content) => ({
@@ -792,47 +971,84 @@ const make$1 = Effect.gen(function* () {
 			reason: `Could not stat: ${String(error)}`
 		}
 	})));
-	const loadGitignoreRules = Effect.gen(function* () {
+	const computeRelative = (fullPath, cwd) => fullPath.startsWith(cwd) ? fullPath.slice(cwd.length + 1) : fullPath;
+	const loadIgnoreFile = (filePath, ig, skipped) => Effect.gen(function* () {
+		const result = yield* readFileWithSkip(filePath, (error) => `Could not read ignore file: ${String(error)}`);
+		if (result.skipped) skipped.push(result.skipped);
+		if (result.content.trim()) ig.add(result.content.split("\n"));
+	});
+	const loadGitignoreRules = (ignoredPaths, cwd) => Effect.gen(function* () {
 		const ig = ignore();
-		const cwd = process.cwd();
 		const skipped = [];
-		const rootContent = yield* readFileWithSkip(`${cwd}/.gitignore`, (error) => `Could not read gitignore: ${String(error)}`);
-		if (rootContent.skipped) skipped.push(rootContent.skipped);
-		if (rootContent.content.trim()) ig.add(rootContent.content.split("\n"));
+		if (ignoredPaths.length > 0) ig.add(ignoredPaths);
+		const gitignorePath = `${cwd}/.gitignore`;
+		if (yield* fs.exists(gitignorePath).pipe(Effect.catchAll(() => Effect.succeed(false)))) yield* loadIgnoreFile(gitignorePath, ig, skipped);
 		const excludePath = `${cwd}/.git/info/exclude`;
-		if (yield* fs.exists(excludePath)) {
-			const excludeContent = yield* readFileWithSkip(excludePath, (error) => `Could not read exclude file: ${String(error)}`);
-			if (excludeContent.skipped) skipped.push(excludeContent.skipped);
-			if (excludeContent.content.trim()) ig.add(excludeContent.content.split("\n"));
-		}
+		if (yield* fs.exists(excludePath).pipe(Effect.catchAll(() => Effect.succeed(false)))) yield* loadIgnoreFile(excludePath, ig, skipped);
 		return {
 			ig,
 			skipped
 		};
 	});
-	const walk = (dir, extensions) => Effect.gen(function* () {
+	const processEntry = (entry, dir, ig, cwd) => Effect.gen(function* () {
+		const fullPath = `${dir}/${entry}`;
+		const statResult = yield* statWithSkip(fullPath);
+		if (statResult.skipped) return {
+			files: [],
+			skipped: [statResult.skipped]
+		};
+		if (!statResult.info) return {
+			files: [],
+			skipped: []
+		};
+		const info = statResult.info;
+		if (info.type === "Directory") {
+			const relativeDir = computeRelative(fullPath, cwd);
+			if (ig.ignores(relativeDir)) return {
+				files: [],
+				skipped: [{
+					path: fullPath,
+					reason: `Ignored by config pattern: ${relativeDir}`
+				}]
+			};
+			return {
+				files: [],
+				skipped: [],
+				recurse: true
+			};
+		}
+		if (info.type === "File") {
+			const relativePath = computeRelative(fullPath, cwd);
+			if (ig.ignores(relativePath)) return {
+				files: [],
+				skipped: [{
+					path: fullPath,
+					reason: `Ignored by config pattern: ${relativePath}`
+				}]
+			};
+			return {
+				files: [fullPath],
+				skipped: []
+			};
+		}
+		return {
+			files: [],
+			skipped: []
+		};
+	});
+	const walk = (dir, ig, cwd) => Effect.gen(function* () {
 		const result = yield* readDirectoryWithSkip(dir);
 		let files = [];
 		const skipped = [];
 		if (result.skipped) skipped.push(result.skipped);
 		for (const entry of result.entries) {
-			if (ALWAYS_IGNORE.has(entry)) continue;
-			const fullPath = `${dir}/${entry}`;
-			const info = yield* statWithSkip(fullPath);
-			if (info.skipped) {
-				skipped.push(info.skipped);
-				continue;
-			}
-			if (!info.info) continue;
-			if (info.info.type === "Directory") {
-				const sub = yield* walk(fullPath, extensions);
+			const entryResult = yield* processEntry(entry, dir, ig, cwd);
+			files.push(...entryResult.files);
+			skipped.push(...entryResult.skipped);
+			if ("recurse" in entryResult) {
+				const sub = yield* walk(`${dir}/${entry}`, ig, cwd);
 				files.push(...sub.files);
 				skipped.push(...sub.skipped);
-			} else if (info.info.type === "File") {
-				const dotIndex = entry.lastIndexOf(".");
-				if (dotIndex === -1) continue;
-				const ext = entry.slice(dotIndex);
-				if (extensions.has(ext)) files.push(fullPath);
 			}
 		}
 		return {
@@ -840,16 +1056,15 @@ const make$1 = Effect.gen(function* () {
 			skipped
 		};
 	});
-	const scanFiles = (extensions) => Effect.gen(function* () {
-		const { ig, skipped: ignoreSkipped } = yield* loadGitignoreRules.pipe(Effect.mapError((cause) => new ScanFailed({
+	const scanFiles = (ignoredPaths) => Effect.gen(function* () {
+		const cwd = process.cwd();
+		const { ig, skipped: ignoreSkipped } = yield* loadGitignoreRules(ignoredPaths, cwd).pipe(Effect.mapError((cause) => new ScanFailed({
 			message: `Failed to load gitignore rules: ${String(cause)}`,
 			cause
 		})));
-		const cwd = process.cwd();
-		const { files: paths, skipped: walkSkipped } = yield* walk(cwd, new Set(extensions));
-		const relativePaths = paths.map((p) => p.startsWith(cwd) ? p.slice(cwd.length + 1) : p);
+		const { files, skipped: walkSkipped } = yield* walk(cwd, ig, cwd);
 		return {
-			files: ig.filter(relativePaths).map((p) => `${cwd}/${p}`),
+			files,
 			skipped: [...ignoreSkipped, ...walkSkipped]
 		};
 	});
@@ -1025,7 +1240,7 @@ const make = Effect.gen(function* () {
 const VectorStoreLive = Layer.effect(VectorStore, make);
 //#endregion
 //#region src/index.ts
-const ServicesLayer = Layer.mergeAll(ConfigStoreLive, ScannerLive, OnnxEmbedderLive, VectorStoreLive);
+const ServicesLayer = Layer.mergeAll(ConfigStoreLive, ScannerLive, OnnxEmbedderLive, VectorStoreLive, ContentExtractorLive);
 const ChunkerLayer = ChunkerLive.pipe(Layer.provide(ServicesLayer));
 const InfraLayer = Layer.mergeAll(ServicesLayer, ChunkerLayer).pipe(Layer.provide(NodeContext.layer));
 const UseCaseLayer = Layer.mergeAll(InitProject.Default, GetStatus.Default, QueryProject.Default, IndexProject.Default, ResetIndex.Default);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lucas-bur/pix",
-  "version": "0.9.1",
+  "version": "0.10.0",
   "description": "Lightweight local semantic project indexer",
   "keywords": [
     "cli",