log-search 1.0.0

package/dist/core/indexer/IndexBuilder.js

@@ -0,0 +1,274 @@
+"use strict";
+/**
+ * IndexBuilder.ts
+ * Core engine for building the inverted index from a log file.
+ * Splits the file into chunks and processes them in parallel using WorkerPool.
+ * Then merges all chunk results into a single InvertedIndex and serializes it.
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.IndexBuilder = void 0;
+const fs_1 = require("fs");
+const promises_1 = require("fs/promises");
+const path = __importStar(require("path"));
+const os = __importStar(require("os"));
+const crypto = __importStar(require("crypto"));
+const IndexSerializer_1 = require("./IndexSerializer");
+const WorkerPool_1 = require("../workers/WorkerPool");
+const DEFAULT_CHUNK_SIZE = 64 * 1024 * 1024; // 64 MB
+const INDEX_VERSION = '1.0.0';
+class IndexBuilder {
+    filePath;
+    config;
+    workerPool;
+    serializer;
+    constructor(filePath, config = {}) {
+        this.filePath = path.resolve(filePath);
+        this.config = {
+            chunkSize: config.chunkSize ?? DEFAULT_CHUNK_SIZE,
+            maxWorkers: config.maxWorkers ?? Math.max(1, os.cpus().length - 1),
+            compression: config.compression ?? 'gzip',
+            tokenMinLength: config.tokenMinLength ?? 2,
+            includeLineNumbers: config.includeLineNumbers ?? true,
+            includeTimestamps: config.includeTimestamps ?? true,
+            stopWords: config.stopWords ?? [],
+        };
+        this.workerPool = new WorkerPool_1.WorkerPool(this.config.maxWorkers);
+        this.serializer = new IndexSerializer_1.IndexSerializer();
+    }
+    /**
+     * Build the full inverted index for this.filePath.
+     * Calls onProgress with a percentage 0–100 as work proceeds.
+     */
+    async build(onProgress) {
+        const startTime = Date.now();
+        onProgress?.(0);
+        // ── Step 1: File metadata ────────────────────────────────────────────────
+        const fileStats = await (0, promises_1.stat)(this.filePath);
+        const fileSize = fileStats.size;
+        const fileHash = await this.computeFileHash(fileSize);
+        onProgress?.(3);
+        // ── Step 2: Calculate chunk boundaries (aligned to newlines) ────────────
+        const chunks = await this.calculateChunkBoundaries(fileSize);
+        onProgress?.(5);
+        // ── Step 3: Parallel chunk processing ───────────────────────────────────
+        let completedChunks = 0;
+        const chunkResults = await Promise.all(chunks.map((chunk) => this.workerPool
+            .run({
+            task: 'processChunk',
+            filePath: this.filePath,
+            startByte: chunk.start,
+            endByte: chunk.end,
+            chunkId: chunk.chunkId,
+            config: this.config,
+        })
+            .then((result) => {
+            completedChunks++;
+            const progress = 5 + (completedChunks / chunks.length) * 75;
+            onProgress?.(Math.round(progress));
+            return result;
+        })));
+        onProgress?.(80);
+        // ── Step 4: Merge chunk indexes ──────────────────────────────────────────
+        const mergedIndex = this.mergeChunkResults(chunkResults, fileSize, fileHash);
+        onProgress?.(90);
+        // ── Step 5: Serialize and save ───────────────────────────────────────────
+        const indexPath = await this.resolveIndexPath();
+        await this.serializer.save(mergedIndex, indexPath, this.config.compression);
+        // Update compression ratio
+        const indexSize = await this.serializer.getSize(indexPath);
+        mergedIndex.metadata.compressionRatio =
+            fileSize > 0 ? Math.round((1 - indexSize / fileSize) * 100) / 100 : 0;
+        onProgress?.(100);
+        await this.workerPool.terminate();
+        const elapsed = Date.now() - startTime;
+        mergedIndex.metadata.buildTimeMs = elapsed;
+        return {
+            indexPath,
+            fileSize,
+            totalLines: mergedIndex.metadata.totalLines,
+            uniqueTerms: mergedIndex.metadata.uniqueTerms,
+            buildTimeMs: elapsed,
+            compressionRatio: mergedIndex.metadata.compressionRatio,
+        };
+    }
+    // ─── Private Helpers ────────────────────────────────────────────────────────
+    /**
+     * Divide the file into N chunks, each starting and ending exactly at a newline.
+     */
+    async calculateChunkBoundaries(fileSize) {
+        if (fileSize === 0)
+            return [{ start: 0, end: 0, chunkId: 0 }];
+        const numChunks = Math.max(1, Math.ceil(fileSize / this.config.chunkSize));
+        const boundaries = [];
+        let position = 0;
+        for (let i = 0; i < numChunks; i++) {
+            const targetEnd = Math.min(position + this.config.chunkSize, fileSize - 1);
+            const actualEnd = i === numChunks - 1 ? fileSize - 1 : await this.findNextNewline(targetEnd);
+            boundaries.push({ start: position, end: actualEnd, chunkId: i });
+            position = actualEnd + 1;
+            if (position >= fileSize)
+                break;
+        }
+        return boundaries;
+    }
+    /**
+     * Scan forward from `position` until we find a '\n' character.
+     * Returns the offset of that '\n'.
+     */
+    findNextNewline(position) {
+        return new Promise((resolve) => {
+            const stream = (0, fs_1.createReadStream)(this.filePath, {
+                start: position,
+                end: position + 1024,
+            });
+            let offset = position;
+            let found = false;
+            stream.on('data', (chunk) => {
+                const newlinePos = chunk.indexOf(0x0a);
+                if (newlinePos !== -1 && !found) {
+                    found = true;
+                    resolve(offset + newlinePos);
+                    stream.destroy();
+                }
+                else {
+                    offset += chunk.length;
+                }
+            });
+            stream.on('end', () => {
+                if (!found)
+                    resolve(position);
+            });
+            stream.on('error', () => resolve(position));
+            stream.on('close', () => {
+                if (!found)
+                    resolve(position);
+            });
+        });
+    }
+    /**
+     * Merge all chunk InvertedIndexes into a single master index.
+     */
+    mergeChunkResults(chunks, fileSize, fileHash) {
+        // Sort chunks by their chunk ID to ensure correct line ordering
+        const sorted = [...chunks].sort((a, b) => a.chunkId - b.chunkId);
+        const masterIndex = new Map();
+        const allLineOffsets = [];
+        let totalLines = 0;
+        for (const chunk of sorted) {
+            totalLines += chunk.lineCount;
+            // Merge line offsets
+            for (const offset of chunk.lineOffsets) {
+                allLineOffsets.push(offset);
+            }
+            // Merge inverted index
+            for (const [term, offsets] of Object.entries(chunk.termOffsets)) {
+                const existing = masterIndex.get(term);
+                if (existing) {
+                    for (const o of offsets)
+                        existing.push(o);
+                }
+                else {
+                    masterIndex.set(term, [...offsets]);
+                }
+            }
+        }
+        // Sort line offsets
+        allLineOffsets.sort((a, b) => a - b);
+        return {
+            invertedIndex: Object.fromEntries(masterIndex),
+            lineOffsets: allLineOffsets,
+            metadata: {
+                totalLines,
+                uniqueTerms: masterIndex.size,
+                fileSize,
+                filePath: this.filePath,
+                fileHash,
+                builtAt: new Date().toISOString(),
+                buildTimeMs: 0, // filled in after
+                compressionRatio: 0, // filled in after
+                version: INDEX_VERSION,
+            },
+        };
+    }
+    /**
+     * Compute a quick hash of the file for staleness detection.
+     * Only reads the first and last 64KB + file size for speed.
+     */
+    async computeFileHash(fileSize) {
+        return new Promise((resolve) => {
+            const hash = crypto.createHash('md5');
+            hash.update(String(fileSize));
+            const readChunk = (start, end, cb) => {
+                const s = (0, fs_1.createReadStream)(this.filePath, { start, end });
+                s.on('data', (d) => hash.update(d));
+                s.on('end', cb);
+                s.on('error', cb);
+            };
+            const tail = Math.max(0, fileSize - 65536);
+            readChunk(0, Math.min(65535, fileSize - 1), () => {
+                if (tail > 65536) {
+                    readChunk(tail, fileSize - 1, () => resolve(hash.digest('hex').slice(0, 16)));
+                }
+                else {
+                    resolve(hash.digest('hex').slice(0, 16));
+                }
+            });
+        });
+    }
+    /**
+     * Resolve the path where the index will be stored.
+     * Format: ~/.log-search/indexes/<filename>-<hash>.lsi
+     */
+    async resolveIndexPath() {
+        const hash = crypto
+            .createHash('md5')
+            .update(this.filePath)
+            .digest('hex')
+            .slice(0, 8);
+        const cacheDir = path.join(os.homedir(), '.log-search', 'indexes');
+        await (0, promises_1.mkdir)(cacheDir, { recursive: true });
+        const basename = path.basename(this.filePath);
+        return path.join(cacheDir, `${basename}-${hash}.lsi`);
+    }
+    /**
+     * Get the index path for this file (without building it).
+     */
+    async getIndexPath() {
+        return this.resolveIndexPath();
+    }
+}
+exports.IndexBuilder = IndexBuilder;
+//# sourceMappingURL=IndexBuilder.js.map

package/dist/core/indexer/IndexBuilder.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"IndexBuilder.js","sourceRoot":"","sources":["../../../src/core/indexer/IndexBuilder.ts"],"names":[],"mappings":";AAAA;;;;;GAKG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEH,2BAAsC;AACtC,0CAA0C;AAC1C,2CAA6B;AAC7B,uCAAyB;AACzB,+CAAiC;AACjC,uDAAoD;AACpD,sDAAmD;AASnD,MAAM,kBAAkB,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,QAAQ;AACrD,MAAM,aAAa,GAAG,OAAO,CAAC;AAE9B,MAAa,YAAY;IACf,QAAQ,CAAS;IACjB,MAAM,CAAwB;IAC9B,UAAU,CAAa;IACvB,UAAU,CAAkB;IAEpC,YAAY,QAAgB,EAAE,SAAsB,EAAE;QACpD,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QACvC,IAAI,CAAC,MAAM,GAAG;YACZ,SAAS,EAAE,MAAM,CAAC,SAAS,IAAI,kBAAkB;YACjD,UAAU,EAAE,MAAM,CAAC,UAAU,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC;YAClE,WAAW,EAAE,MAAM,CAAC,WAAW,IAAI,MAAM;YACzC,cAAc,EAAE,MAAM,CAAC,cAAc,IAAI,CAAC;YAC1C,kBAAkB,EAAE,MAAM,CAAC,kBAAkB,IAAI,IAAI;YACrD,iBAAiB,EAAE,MAAM,CAAC,iBAAiB,IAAI,IAAI;YACnD,SAAS,EAAE,MAAM,CAAC,SAAS,IAAI,EAAE;SAClC,CAAC;QACF,IAAI,CAAC,UAAU,GAAG,IAAI,uBAAU,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QACzD,IAAI,CAAC,UAAU,GAAG,IAAI,iCAAe,EAAE,CAAC;IAC1C,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,KAAK,CAAC,UAAsC;QAChD,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,UAAU,EAAE,CAAC,CAAC,CAAC,CAAC;QAEhB,4EAA4E;QAC5E,MAAM,SAAS,GAAG,MAAM,IAAA,eAAI,EAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC5C,MAAM,QAAQ,GAAG,SAAS,CAAC,IAAI,CAAC;QAChC,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC,CAAC;QAEtD,UAAU,EAAE,CAAC,CAAC,CAAC,CAAC;QAEhB,2EAA2E;QAC3E,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,wBAAwB,CAAC,QAAQ,CAAC,CAAC;QAC7D,UAAU,EAAE,CAAC,CAAC,CAAC,CAAC;QAEhB,2EAA2E;QAC3E,IAAI,eAAe,GAAG,CAAC,CAAC;QACxB,MAAM,YAAY,GAAkB,MAAM,OAAO,CAAC,GAAG,CACnD,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CACnB,IAAI,CAAC,UAAU;aACZ,GAAG,CAAC;YACH,IAAI,EAAE,cAAc;YACpB,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,SAAS,EAAE,KAAK,CAAC,KAAK;YACtB,OAAO,EAAE,KAAK,CAAC,GAAG;YAClB,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,MAAM,EAAE,IAAI,CAAC,MAAM;SACpB,CAAC;aACD,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE;YACf,eAAe,EAAE,CAAC;YAClB,MAAM,QAAQ,GAAG,CAAC,GAAG,CAAC,eAAe,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC;YAC5D,UAAU,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC;YACnC,OAAO,MAAM,CAAC;QAChB,CAAC,CAAC,CACL,CACF,CAAC;QAEF,UAAU,EAAE,CAAC,EAAE,CAAC,CAAC;QAEjB,4EAA4E;QAC5E,MAAM,WAAW,GAAG,IAAI,CAAC,iBAAiB,CAAC,YAAY,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC;QAC7E,UAAU,EAAE,CAAC,EAAE,CAAC,CAAC;QAEjB,4EAA4E;QAC5E,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAChD,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,WAAW,EAAE,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;QAE5E,2BAA2B;QAC3B,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QAC3D,WAAW,CAAC,QAAQ,CAAC,gBAAgB;YACnC,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,SAAS,GAAG,QAAQ,CAAC,GAAG,GAAG,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAExE,UAAU,EAAE,CAAC,GAAG,CAAC,CAAC;QAElB,MAAM,IAAI,CAAC,UAAU,CAAC,SAAS,EAAE,CAAC;QAElC,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;QACvC,WAAW,CAAC,QAAQ,CAAC,WAAW,GAAG,OAAO,CAAC;QAE3C,OAAO;YACL,SAAS;YACT,QAAQ;YACR,UAAU,EAAE,WAAW,CAAC,QAAQ,CAAC,UAAU;YAC3C,WAAW,EAAE,WAAW,CAAC,QAAQ,CAAC,WAAW;YAC7C,WAAW,EAAE,OAAO;YACpB,gBAAgB,EAAE,WAAW,CAAC,QAAQ,CAAC,gBAAgB;SACxD,CAAC;IACJ,CAAC;IAED,+EAA+E;IAE/E;;OAEG;IACK,KAAK,CAAC,wBAAwB,CAAC,QAAgB;QACrD,IAAI,QAAQ,KAAK,CAAC;YAAE,OAAO,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC;QAE9D,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC;QAC3E,MAAM,UAAU,GAAoB,EAAE,CAAC;QACvC,IAAI,QAAQ,GAAG,CAAC,CAAC;QAEjB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,EAAE,CAAC,EAAE,EAAE,CAAC;YACnC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,QAAQ,GAAG,CAAC,CAAC,CAAC;YAC3E,MAAM,SAAS,GACb,CAAC,KAAK,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,eAAe,CAAC,SAAS,CAAC,CAAC;YAE7E,UAAU,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,QAAQ,EAAE,GAAG,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC;YACjE,QAAQ,GAAG,SAAS,GAAG,CAAC,CAAC;YAEzB,IAAI,QAAQ,IAAI,QAAQ;gBAAE,MAAM;QAClC,CAAC;QAED,OAAO,UAAU,CAAC;IACpB,CAAC;IAED;;;OAGG;IACK,eAAe,CAAC,QAAgB;QACtC,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;YAC7B,MAAM,MAAM,GAAG,IAAA,qBAAgB,EAAC,IAAI,CAAC,QAAQ,EAAE;gBAC7C,KAAK,EAAE,QAAQ;gBACf,GAAG,EAAE,QAAQ,GAAG,IAAI;aACrB,CAAC,CAAC;YAEH,IAAI,MAAM,GAAG,QAAQ,CAAC;YACtB,IAAI,KAAK,GAAG,KAAK,CAAC;YAElB,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAU,EAAE,EAAE;gBAC/B,MAAM,UAAU,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;gBACvC,IAAI,UAAU,KAAK,CAAC,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;oBAChC,KAAK,GAAG,IAAI,CAAC;oBACb,OAAO,CAAC,MAAM,GAAG,UAAU,CAAC,CAAC;oBAC7B,MAAM,CAAC,OAAO,EAAE,CAAC;gBACnB,CAAC;qBAAM,CAAC;oBACN,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC;gBACzB,CAAC;YACH,CAAC,CAAC,CAAC;YAEH,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;gBACpB,IAAI,CAAC,KAAK;oBAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;YAChC,CAAC,CAAC,CAAC;YAEH,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;YAC5C,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;gBACtB,IAAI,CAAC,KAAK;oBAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;YAChC,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACK,iBAAiB,CACvB,MAAqB,EACrB,QAAgB,EAChB,QAAgB;QAEhB,gEAAgE;QAChE,MAAM,MAAM,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC;QAEjE,MAAM,WAAW,GAA0B,IAAI,GAAG,EAAE,CAAC;QACrD,MAAM,cAAc,GAAa,EAAE,CAAC;QACpC,IAAI,UAAU,GAAG,CAAC,CAAC;QAEnB,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,UAAU,IAAI,KAAK,CAAC,SAAS,CAAC;YAE9B,qBAAqB;YACrB,KAAK,MAAM,MAAM,IAAI,KAAK,CAAC,WAAW,EAAE,CAAC;gBACvC,cAAc,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAC9B,CAAC;YAED,uBAAuB;YACvB,KAAK,MAAM,CAAC,IAAI,EAAE,OAAO,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,EAAE,CAAC;gBAChE,MAAM,QAAQ,GAAG,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;gBACvC,IAAI,QAAQ,EAAE,CAAC;oBACb,KAAK,MAAM,CAAC,IAAI,OAAO;wBAAE,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;gBAC5C,CAAC;qBAAM,CAAC;oBACN,WAAW,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;gBACtC,CAAC;YACH,CAAC;QACH,CAAC;QAED,oBAAoB;QACpB,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAErC,OAAO;YACL,aAAa,EAAE,MAAM,CAAC,WAAW,CAAC,WAAW,CAAC;YAC9C,WAAW,EAAE,cAAc;YAC3B,QAAQ,EAAE;gBACR,UAAU;gBACV,WAAW,EAAE,WAAW,CAAC,IAAI;gBAC7B,QAAQ;gBACR,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,QAAQ;gBACR,OAAO,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBACjC,WAAW,EAAE,CAAC,EAAE,kBAAkB;gBAClC,gBAAgB,EAAE,CAAC,EAAE,kBAAkB;gBACvC,OAAO,EAAE,aAAa;aACvB;SACF,CAAC;IACJ,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,eAAe,CAAC,QAAgB;QAC5C,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;YAC7B,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;YACtC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC;YAE9B,MAAM,SAAS,GAAG,CAAC,KAAa,EAAE,GAAW,EAAE,EAAc,EAAE,EAAE;gBAC/D,MAAM,CAAC,GAAG,IAAA,qBAAgB,EAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;gBAC1D,CAAC,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;gBACpC,CAAC,CAAC,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;gBAChB,CAAC,CAAC,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;YACpB,CAAC,CAAC;YAEF,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,GAAG,KAAK,CAAC,CAAC;YAC3C,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,QAAQ,GAAG,CAAC,CAAC,EAAE,GAAG,EAAE;gBAC/C,IAAI,IAAI,GAAG,KAAK,EAAE,CAAC;oBACjB,SAAS,CAAC,IAAI,EAAE,QAAQ,GAAG,CAAC,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;gBAChF,CAAC;qBAAM,CAAC;oBACN,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;gBAC3C,CAAC;YACH,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,gBAAgB;QAC5B,MAAM,IAAI,GAAG,MAAM;aAChB,UAAU,CAAC,KAAK,CAAC;aACjB,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC;aACrB,MAAM,CAAC,KAAK,CAAC;aACb,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAEf,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,EAAE,aAAa,EAAE,SAAS,CAAC,CAAC;QACnE,MAAM,IAAA,gBAAK,EAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAE3C,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC9C,OAAO,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,GAAG,QAAQ,IAAI,IAAI,MAAM,CAAC,CAAC;IACxD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,YAAY;QAChB,OAAO,IAAI,CAAC,gBAAgB,EAAE,CAAC;IACjC,CAAC;CACF;AArQD,oCAqQC"}

package/dist/core/indexer/IndexSerializer.d.ts

@@ -0,0 +1,30 @@
+/**
+ * IndexSerializer.ts
+ * Handles saving and loading the inverted index to/from disk.
+ * Uses msgpackr for binary serialization + zlib gzip for compression.
+ * Format: [4-byte magic] [1-byte version] [4-byte flags] [msgpack+gzip payload]
+ */
+import type { InvertedIndex, CompressionAlgorithm } from '../../types/IndexTypes';
+export declare class IndexSerializer {
+    /**
+     * Serialize and save an InvertedIndex to disk.
+     * @param index         The full index object
+     * @param outputPath    Destination file path (.lsi)
+     * @param compression   Compression algorithm to use
+     */
+    save(index: InvertedIndex, outputPath: string, compression?: CompressionAlgorithm): Promise<void>;
+    /**
+     * Load and deserialize an InvertedIndex from disk.
+     * @param indexPath  Path to the .lsi file
+     */
+    load(indexPath: string): Promise<InvertedIndex>;
+    /**
+     * Get the size of an index file on disk (bytes).
+     */
+    getSize(indexPath: string): Promise<number>;
+    /**
+     * Check if an index file exists and is readable.
+     */
+    exists(indexPath: string): Promise<boolean>;
+}
+//# sourceMappingURL=IndexSerializer.d.ts.map

package/dist/core/indexer/IndexSerializer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"IndexSerializer.d.ts","sourceRoot":"","sources":["../../../src/core/indexer/IndexSerializer.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAOH,OAAO,KAAK,EAAE,aAAa,EAAE,oBAAoB,EAAE,MAAM,wBAAwB,CAAC;AASlF,qBAAa,eAAe;IAC1B;;;;;OAKG;IACG,IAAI,CACR,KAAK,EAAE,aAAa,EACpB,UAAU,EAAE,MAAM,EAClB,WAAW,GAAE,oBAA6B,GACzC,OAAO,CAAC,IAAI,CAAC;IAiChB;;;OAGG;IACG,IAAI,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC;IA8BrD;;OAEG;IACG,OAAO,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IASjD;;OAEG;IACG,MAAM,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;CAQlD"}

package/dist/core/indexer/IndexSerializer.js

@@ -0,0 +1,142 @@
+"use strict";
+/**
+ * IndexSerializer.ts
+ * Handles saving and loading the inverted index to/from disk.
+ * Uses msgpackr for binary serialization + zlib gzip for compression.
+ * Format: [4-byte magic] [1-byte version] [4-byte flags] [msgpack+gzip payload]
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.IndexSerializer = void 0;
+const promises_1 = require("fs/promises");
+const zlib = __importStar(require("zlib"));
+const util_1 = require("util");
+const msgpackr_1 = require("msgpackr");
+const gzip = (0, util_1.promisify)(zlib.gzip);
+const gunzip = (0, util_1.promisify)(zlib.gunzip);
+// Magic bytes: "LSI\x01" = Log Search Index v1
+const MAGIC = Buffer.from([0x4c, 0x53, 0x49, 0x01]);
+const FORMAT_VERSION = 1;
+class IndexSerializer {
+    /**
+     * Serialize and save an InvertedIndex to disk.
+     * @param index         The full index object
+     * @param outputPath    Destination file path (.lsi)
+     * @param compression   Compression algorithm to use
+     */
+    async save(index, outputPath, compression = 'gzip') {
+        // 1. Serialize with msgpackr (compact binary format)
+        const packed = (0, msgpackr_1.pack)(index);
+        // 2. Compress
+        let payload;
+        let compressionFlag = 0;
+        if (compression === 'gzip') {
+            payload = await gzip(packed, { level: zlib.constants.Z_BEST_SPEED });
+            compressionFlag = 1;
+        }
+        else {
+            payload = Buffer.from(packed);
+            compressionFlag = 0;
+        }
+        // 3. Write with header
+        const header = Buffer.alloc(10);
+        MAGIC.copy(header, 0);
+        header.writeUInt8(FORMAT_VERSION, 4);
+        header.writeUInt8(compressionFlag, 5);
+        header.writeUInt32BE(payload.length, 6);
+        const output = Buffer.concat([header, payload]);
+        await (0, promises_1.writeFile)(outputPath, output);
+        // 4. Update compression ratio in metadata (estimate)
+        const originalSize = Buffer.byteLength((0, msgpackr_1.pack)({ ...index, metadata: {} }));
+        const compressed = output.length;
+        index.metadata.compressionRatio =
+            originalSize > 0 ? Math.round((1 - compressed / originalSize) * 100) / 100 : 0;
+    }
+    /**
+     * Load and deserialize an InvertedIndex from disk.
+     * @param indexPath  Path to the .lsi file
+     */
+    async load(indexPath) {
+        const raw = await (0, promises_1.readFile)(indexPath);
+        // Validate magic bytes
+        if (raw.slice(0, 4).toString('hex') !== MAGIC.toString('hex')) {
+            throw new Error(`Invalid index file format: ${indexPath}`);
+        }
+        const version = raw.readUInt8(4);
+        if (version !== FORMAT_VERSION) {
+            throw new Error(`Unsupported index version: ${version}. Expected ${FORMAT_VERSION}.`);
+        }
+        const compressionFlag = raw.readUInt8(5);
+        const payloadLength = raw.readUInt32BE(6);
+        const payload = raw.slice(10, 10 + payloadLength);
+        // Decompress
+        let decompressed;
+        if (compressionFlag === 1) {
+            decompressed = await gunzip(payload);
+        }
+        else {
+            decompressed = payload;
+        }
+        // Deserialize
+        const index = (0, msgpackr_1.unpack)(decompressed);
+        return index;
+    }
+    /**
+     * Get the size of an index file on disk (bytes).
+     */
+    async getSize(indexPath) {
+        try {
+            const s = await (0, promises_1.stat)(indexPath);
+            return s.size;
+        }
+        catch {
+            return 0;
+        }
+    }
+    /**
+     * Check if an index file exists and is readable.
+     */
+    async exists(indexPath) {
+        try {
+            await (0, promises_1.stat)(indexPath);
+            return true;
+        }
+        catch {
+            return false;
+        }
+    }
+}
+exports.IndexSerializer = IndexSerializer;
+//# sourceMappingURL=IndexSerializer.js.map

package/dist/core/indexer/IndexSerializer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"IndexSerializer.js","sourceRoot":"","sources":["../../../src/core/indexer/IndexSerializer.ts"],"names":[],"mappings":";AAAA;;;;;GAKG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAGH,0CAAwD;AACxD,2CAA6B;AAC7B,+BAAiC;AACjC,uCAAwC;AAGxC,MAAM,IAAI,GAAG,IAAA,gBAAS,EAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAClC,MAAM,MAAM,GAAG,IAAA,gBAAS,EAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AAEtC,+CAA+C;AAC/C,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;AACpD,MAAM,cAAc,GAAG,CAAC,CAAC;AAEzB,MAAa,eAAe;IAC1B;;;;;OAKG;IACH,KAAK,CAAC,IAAI,CACR,KAAoB,EACpB,UAAkB,EAClB,cAAoC,MAAM;QAE1C,qDAAqD;QACrD,MAAM,MAAM,GAAG,IAAA,eAAI,EAAC,KAAK,CAAC,CAAC;QAE3B,cAAc;QACd,IAAI,OAAe,CAAC;QACpB,IAAI,eAAe,GAAG,CAAC,CAAC;QAExB,IAAI,WAAW,KAAK,MAAM,EAAE,CAAC;YAC3B,OAAO,GAAG,MAAM,IAAI,CAAC,MAAM,EAAE,EAAE,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,YAAY,EAAE,CAAC,CAAC;YACrE,eAAe,GAAG,CAAC,CAAC;QACtB,CAAC;aAAM,CAAC;YACN,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAC9B,eAAe,GAAG,CAAC,CAAC;QACtB,CAAC;QAED,uBAAuB;QACvB,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAChC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QACtB,MAAM,CAAC,UAAU,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC;QACrC,MAAM,CAAC,UAAU,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC;QACtC,MAAM,CAAC,aAAa,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QAExC,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;QAChD,MAAM,IAAA,oBAAS,EAAC,UAAU,EAAE,MAAM,CAAC,CAAC;QAEpC,qDAAqD;QACrD,MAAM,YAAY,GAAG,MAAM,CAAC,UAAU,CAAC,IAAA,eAAI,EAAC,EAAE,GAAG,KAAK,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC;QACzE,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,CAAC;QACjC,KAAK,CAAC,QAAQ,CAAC,gBAAgB;YAC7B,YAAY,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,UAAU,GAAG,YAAY,CAAC,GAAG,GAAG,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IACnF,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,IAAI,CAAC,SAAiB;QAC1B,MAAM,GAAG,GAAG,MAAM,IAAA,mBAAQ,EAAC,SAAS,CAAC,CAAC;QAEtC,uBAAuB;QACvB,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;YAC9D,MAAM,IAAI,KAAK,CAAC,8BAA8B,SAAS,EAAE,CAAC,CAAC;QAC7D,CAAC;QAED,MAAM,OAAO,GAAG,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;QACjC,IAAI,OAAO,KAAK,cAAc,EAAE,CAAC;YAC/B,MAAM,IAAI,KAAK,CAAC,8BAA8B,OAAO,cAAc,cAAc,GAAG,CAAC,CAAC;QACxF,CAAC;QAED,MAAM,eAAe,GAAG,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;QACzC,MAAM,aAAa,GAAG,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAC1C,MAAM,OAAO,GAAG,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,GAAG,aAAa,CAAC,CAAC;QAElD,aAAa;QACb,IAAI,YAAoB,CAAC;QACzB,IAAI,eAAe,KAAK,CAAC,EAAE,CAAC;YAC1B,YAAY,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,CAAC;QACvC,CAAC;aAAM,CAAC;YACN,YAAY,GAAG,OAAO,CAAC;QACzB,CAAC;QAED,cAAc;QACd,MAAM,KAAK,GAAG,IAAA,iBAAM,EAAC,YAAY,CAAkB,CAAC;QACpD,OAAO,KAAK,CAAC;IACf,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,OAAO,CAAC,SAAiB;QAC7B,IAAI,CAAC;YACH,MAAM,CAAC,GAAG,MAAM,IAAA,eAAI,EAAC,SAAS,CAAC,CAAC;YAChC,OAAO,CAAC,CAAC,IAAI,CAAC;QAChB,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,CAAC,CAAC;QACX,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,MAAM,CAAC,SAAiB;QAC5B,IAAI,CAAC;YACH,MAAM,IAAA,eAAI,EAAC,SAAS,CAAC,CAAC;YACtB,OAAO,IAAI,CAAC;QACd,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;CACF;AArGD,0CAqGC"}

package/dist/core/indexer/OffsetMapper.d.ts

@@ -0,0 +1,55 @@
+/**
+ * OffsetMapper.ts
+ * Maintains the mapping between line numbers and their byte offsets in the file.
+ * This is the core structure that enables O(1) line retrieval by position.
+ */
+export declare class OffsetMapper {
+    /** Sorted array of byte offsets, index = lineNumber - 1 */
+    private lineOffsets;
+    constructor(initialOffsets?: number[]);
+    /**
+     * Record the byte offset of a new line.
+     */
+    addOffset(byteOffset: number): void;
+    /**
+     * Bulk add offsets (more efficient than individual adds).
+     */
+    addOffsets(offsets: number[]): void;
+    /**
+     * Get the byte offset of a specific line (1-based).
+     */
+    getOffset(lineNumber: number): number | undefined;
+    /**
+     * Find the 1-based line number for a given byte offset using binary search.
+     * Returns the line number, or -1 if not found.
+     */
+    findLineNumber(byteOffset: number): number;
+    /**
+     * Get the total number of lines tracked.
+     */
+    get totalLines(): number;
+    /**
+     * Get the raw offsets array (for serialization).
+     */
+    toArray(): number[];
+    /**
+     * Merge another OffsetMapper into this one (for combining chunk results).
+     * The other mapper's offsets are appended (assumed to come after current ones).
+     */
+    merge(other: OffsetMapper): void;
+    /**
+     * Sort offsets (needed after merging multiple chunks).
+     */
+    sort(): void;
+    /**
+     * Get a slice of offsets around a given byte offset (for context lines).
+     * @param byteOffset  The central line's byte offset
+     * @param before      Number of lines to include before
+     * @param after       Number of lines to include after
+     */
+    getContext(byteOffset: number, before: number, after: number): {
+        beforeOffsets: number[];
+        afterOffsets: number[];
+    };
+}
+//# sourceMappingURL=OffsetMapper.d.ts.map

package/dist/core/indexer/OffsetMapper.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"OffsetMapper.d.ts","sourceRoot":"","sources":["../../../src/core/indexer/OffsetMapper.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,qBAAa,YAAY;IACvB,2DAA2D;IAC3D,OAAO,CAAC,WAAW,CAAgB;gBAEvB,cAAc,GAAE,MAAM,EAAO;IAIzC;;OAEG;IACH,SAAS,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI;IAInC;;OAEG;IACH,UAAU,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,IAAI;IAMnC;;OAEG;IACH,SAAS,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS;IAIjD;;;OAGG;IACH,cAAc,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM;IAgB1C;;OAEG;IACH,IAAI,UAAU,IAAI,MAAM,CAEvB;IAED;;OAEG;IACH,OAAO,IAAI,MAAM,EAAE;IAInB;;;OAGG;IACH,KAAK,CAAC,KAAK,EAAE,YAAY,GAAG,IAAI;IAIhC;;OAEG;IACH,IAAI,IAAI,IAAI;IAIZ;;;;;OAKG;IACH,UAAU,CACR,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,MAAM,GACZ;QAAE,aAAa,EAAE,MAAM,EAAE,CAAC;QAAC,YAAY,EAAE,MAAM,EAAE,CAAA;KAAE;CAYvD"}

package/dist/core/indexer/OffsetMapper.js

@@ -0,0 +1,94 @@
+"use strict";
+/**
+ * OffsetMapper.ts
+ * Maintains the mapping between line numbers and their byte offsets in the file.
+ * This is the core structure that enables O(1) line retrieval by position.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.OffsetMapper = void 0;
+class OffsetMapper {
+    /** Sorted array of byte offsets, index = lineNumber - 1 */
+    lineOffsets = [];
+    constructor(initialOffsets = []) {
+        this.lineOffsets = [...initialOffsets];
+    }
+    /**
+     * Record the byte offset of a new line.
+     */
+    addOffset(byteOffset) {
+        this.lineOffsets.push(byteOffset);
+    }
+    /**
+     * Bulk add offsets (more efficient than individual adds).
+     */
+    addOffsets(offsets) {
+        for (const o of offsets) {
+            this.lineOffsets.push(o);
+        }
+    }
+    /**
+     * Get the byte offset of a specific line (1-based).
+     */
+    getOffset(lineNumber) {
+        return this.lineOffsets[lineNumber - 1];
+    }
+    /**
+     * Find the 1-based line number for a given byte offset using binary search.
+     * Returns the line number, or -1 if not found.
+     */
+    findLineNumber(byteOffset) {
+        const arr = this.lineOffsets;
+        let lo = 0;
+        let hi = arr.length - 1;
+        while (lo <= hi) {
+            const mid = (lo + hi) >>> 1;
+            if (arr[mid] === byteOffset)
+                return mid + 1;
+            if (arr[mid] < byteOffset)
+                lo = mid + 1;
+            else
+                hi = mid - 1;
+        }
+        // Return the closest line (lo is the insertion point)
+        return Math.min(lo + 1, arr.length);
+    }
+    /**
+     * Get the total number of lines tracked.
+     */
+    get totalLines() {
+        return this.lineOffsets.length;
+    }
+    /**
+     * Get the raw offsets array (for serialization).
+     */
+    toArray() {
+        return [...this.lineOffsets];
+    }
+    /**
+     * Merge another OffsetMapper into this one (for combining chunk results).
+     * The other mapper's offsets are appended (assumed to come after current ones).
+     */
+    merge(other) {
+        this.addOffsets(other.toArray());
+    }
+    /**
+     * Sort offsets (needed after merging multiple chunks).
+     */
+    sort() {
+        this.lineOffsets.sort((a, b) => a - b);
+    }
+    /**
+     * Get a slice of offsets around a given byte offset (for context lines).
+     * @param byteOffset  The central line's byte offset
+     * @param before      Number of lines to include before
+     * @param after       Number of lines to include after
+     */
+    getContext(byteOffset, before, after) {
+        const lineIdx = this.findLineNumber(byteOffset) - 1; // 0-based index
+        const beforeOffsets = this.lineOffsets.slice(Math.max(0, lineIdx - before), lineIdx);
+        const afterOffsets = this.lineOffsets.slice(lineIdx + 1, lineIdx + 1 + after);
+        return { beforeOffsets, afterOffsets };
+    }
+}
+exports.OffsetMapper = OffsetMapper;
+//# sourceMappingURL=OffsetMapper.js.map

package/dist/core/indexer/OffsetMapper.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"OffsetMapper.js","sourceRoot":"","sources":["../../../src/core/indexer/OffsetMapper.ts"],"names":[],"mappings":";AAAA;;;;GAIG;;;AAEH,MAAa,YAAY;IACvB,2DAA2D;IACnD,WAAW,GAAa,EAAE,CAAC;IAEnC,YAAY,iBAA2B,EAAE;QACvC,IAAI,CAAC,WAAW,GAAG,CAAC,GAAG,cAAc,CAAC,CAAC;IACzC,CAAC;IAED;;OAEG;IACH,SAAS,CAAC,UAAkB;QAC1B,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IACpC,CAAC;IAED;;OAEG;IACH,UAAU,CAAC,OAAiB;QAC1B,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;YACxB,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC3B,CAAC;IACH,CAAC;IAED;;OAEG;IACH,SAAS,CAAC,UAAkB;QAC1B,OAAO,IAAI,CAAC,WAAW,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC;IAC1C,CAAC;IAED;;;OAGG;IACH,cAAc,CAAC,UAAkB;QAC/B,MAAM,GAAG,GAAG,IAAI,CAAC,WAAW,CAAC;QAC7B,IAAI,EAAE,GAAG,CAAC,CAAC;QACX,IAAI,EAAE,GAAG,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC;QAExB,OAAO,EAAE,IAAI,EAAE,EAAE,CAAC;YAChB,MAAM,GAAG,GAAG,CAAC,EAAE,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC;YAC5B,IAAI,GAAG,CAAC,GAAG,CAAC,KAAK,UAAU;gBAAE,OAAO,GAAG,GAAG,CAAC,CAAC;YAC5C,IAAI,GAAG,CAAC,GAAG,CAAC,GAAG,UAAU;gBAAE,EAAE,GAAG,GAAG,GAAG,CAAC,CAAC;;gBACnC,EAAE,GAAG,GAAG,GAAG,CAAC,CAAC;QACpB,CAAC;QAED,sDAAsD;QACtD,OAAO,IAAI,CAAC,GAAG,CAAC,EAAE,GAAG,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IACtC,CAAC;IAED;;OAEG;IACH,IAAI,UAAU;QACZ,OAAO,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC;IACjC,CAAC;IAED;;OAEG;IACH,OAAO;QACL,OAAO,CAAC,GAAG,IAAI,CAAC,WAAW,CAAC,CAAC;IAC/B,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,KAAmB;QACvB,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;IACnC,CAAC;IAED;;OAEG;IACH,IAAI;QACF,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACzC,CAAC;IAED;;;;;OAKG;IACH,UAAU,CACR,UAAkB,EAClB,MAAc,EACd,KAAa;QAEb,MAAM,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,gBAAgB;QACrE,MAAM,aAAa,GAAG,IAAI,CAAC,WAAW,CAAC,KAAK,CAC1C,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,GAAG,MAAM,CAAC,EAC7B,OAAO,CACR,CAAC;QACF,MAAM,YAAY,GAAG,IAAI,CAAC,WAAW,CAAC,KAAK,CACzC,OAAO,GAAG,CAAC,EACX,OAAO,GAAG,CAAC,GAAG,KAAK,CACpB,CAAC;QACF,OAAO,EAAE,aAAa,EAAE,YAAY,EAAE,CAAC;IACzC,CAAC;CACF;AAtGD,oCAsGC"}

package/dist/core/indexer/TokenExtractor.d.ts

@@ -0,0 +1,27 @@
+/**
+ * TokenExtractor.ts
+ * Extracts normalized tokens (terms) from a log line for indexing.
+ * Handles deduplication, normalization, and stop-word filtering.
+ */
+export declare class TokenExtractor {
+    private minLength;
+    private stopWords;
+    private includeTimestamps;
+    constructor(options?: {
+        minLength?: number;
+        stopWords?: string[];
+        includeTimestamps?: boolean;
+    });
+    /**
+     * Extract all unique tokens from a single line.
+     * Returns a Set of normalized lowercase tokens.
+     */
+    extract(line: string): Set<string>;
+    /**
+     * Break compound tokens into sub-tokens for better searchability.
+     * e.g. "192.168.1.1" → "192", "168"
+     *      "/api/v2/users" → "api", "v2", "users"
+     */
+    private addSubTokens;
+}
+//# sourceMappingURL=TokenExtractor.d.ts.map

package/dist/core/indexer/TokenExtractor.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"TokenExtractor.d.ts","sourceRoot":"","sources":["../../../src/core/indexer/TokenExtractor.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAYH,qBAAa,cAAc;IACzB,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,SAAS,CAAc;IAC/B,OAAO,CAAC,iBAAiB,CAAU;gBAEvB,OAAO,GAAE;QACnB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;QACrB,iBAAiB,CAAC,EAAE,OAAO,CAAC;KACxB;IASN;;;OAGG;IACH,OAAO,CAAC,IAAI,EAAE,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC;IA4BlC;;;;OAIG;IACH,OAAO,CAAC,YAAY;CAgCrB"}