indexer-cli 0.2.1

package/dist/embedding/ollama.js

@@ -0,0 +1,264 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.OllamaEmbeddingProvider = void 0;
+const node_child_process_1 = require("node:child_process");
+const logger_js_1 = require("../core/logger.js");
+const logger = new logger_js_1.SystemLogger("embeddings-ollama");
+class OllamaEmbeddingProvider {
+    id = "ollama";
+    dimension = 768;
+    baseUrl;
+    model;
+    batchSize;
+    concurrency;
+    numCtx;
+    reconnectInFlight = null;
+    requestTimeoutMs = 15_000;
+    constructor(baseUrl, model = "jina-8k", batchSize = 1, concurrency = 1, numCtx = 512) {
+        this.baseUrl = baseUrl.replace(/\/$/, "");
+        this.model = model;
+        this.batchSize = batchSize;
+        this.concurrency = concurrency;
+        this.numCtx = numCtx;
+    }
+    async initialize() {
+        await this.ensureOllamaAvailable("initialize");
+    }
+    async close() { }
+    getDimension() {
+        return this.dimension;
+    }
+    async embed(texts) {
+        const startTime = Date.now();
+        const batches = this.createBatches(texts, this.batchSize);
+        const results = new Array(texts.length);
+        const batchesWithIndices = batches.map((batch, index) => ({
+            batch,
+            startIdx: index * this.batchSize,
+        }));
+        const queue = [...batchesWithIndices];
+        const workers = Array(Math.min(this.concurrency, batches.length))
+            .fill(null)
+            .map(async () => {
+            while (queue.length > 0) {
+                const current = queue.shift();
+                if (!current)
+                    break;
+                const { batch, startIdx } = current;
+                const embeddings = await this.embedBatch(batch);
+                embeddings.forEach((embedding, i) => {
+                    results[startIdx + i] = embedding;
+                });
+            }
+        });
+        await Promise.all(workers);
+        void startTime;
+        return results;
+    }
+    createBatches(items, batchSize) {
+        const batches = [];
+        for (let i = 0; i < items.length; i += batchSize) {
+            batches.push(items.slice(i, i + batchSize));
+        }
+        return batches;
+    }
+    async embedBatch(texts) {
+        try {
+            return await this.performEmbedRequest(texts);
+        }
+        catch (error) {
+            if (this.isConnectionError(error)) {
+                logger.warn("Ollama connection failed. Waiting for service readiness...");
+                await this.ensureOllamaAvailable("embedBatch");
+                return await this.performEmbedRequest(texts);
+            }
+            if (this.isNotFoundError(error)) {
+                logger.info(`Model ${this.model} not found. Pulling...`);
+                await this.pullModel();
+                return await this.performEmbedRequest(texts);
+            }
+            const message = error instanceof Error ? error.message : String(error);
+            throw new Error(`Ollama embedding failed: ${message}`);
+        }
+    }
+    isConnectionError(error) {
+        if (!error || typeof error !== "object") {
+            return false;
+        }
+        // Native fetch wraps network errors in TypeError with a cause
+        const cause = error.cause;
+        const code = error.code ?? cause?.code;
+        const messageValue = error.message;
+        const message = typeof messageValue === "string" ? messageValue : "";
+        if (code &&
+            [
+                "ECONNREFUSED",
+                "ETIMEDOUT",
+                "ECONNRESET",
+                "EHOSTUNREACH",
+                "ECONNABORTED",
+            ].includes(code)) {
+            return true;
+        }
+        return (message.includes("ECONNREFUSED") ||
+            message.includes("ETIMEDOUT") ||
+            message.includes("ECONNRESET") ||
+            message.includes("EHOSTUNREACH") ||
+            message.toLowerCase().includes("timeout"));
+    }
+    isNotFoundError(error) {
+        if (error instanceof Response) {
+            return error.status === 404;
+        }
+        if (!error || typeof error !== "object") {
+            return false;
+        }
+        const status = error.status;
+        return status === 404;
+    }
+    async performEmbedRequest(texts) {
+        try {
+            const response = await this.fetchWithTimeout(`${this.baseUrl}/api/embed`, {
+                method: "POST",
+                headers: { "Content-Type": "application/json" },
+                body: JSON.stringify({
+                    model: this.model,
+                    input: texts,
+                    options: {
+                        num_ctx: this.numCtx,
+                    },
+                }),
+            });
+            if (response.status === 404) {
+                return await this.fallbackSequentialEmbed(texts);
+            }
+            if (!response.ok) {
+                throw new Error(`Ollama responded with status ${response.status}`);
+            }
+            const data = await response.json();
+            if (data?.embeddings) {
+                return data.embeddings;
+            }
+        }
+        catch (error) {
+            if (this.isNotFoundError(error)) {
+                return await this.fallbackSequentialEmbed(texts);
+            }
+            throw error;
+        }
+        throw new Error("Invalid response from Ollama");
+    }
+    async fallbackSequentialEmbed(texts) {
+        const embeddings = [];
+        for (const text of texts) {
+            const response = await this.fetchWithTimeout(`${this.baseUrl}/api/embeddings`, {
+                method: "POST",
+                headers: { "Content-Type": "application/json" },
+                body: JSON.stringify({
+                    model: this.model,
+                    prompt: text,
+                    options: {
+                        num_ctx: this.numCtx,
+                    },
+                }),
+            });
+            if (!response.ok) {
+                throw new Error(`Ollama responded with status ${response.status} (fallback)`);
+            }
+            const data = await response.json();
+            if (data?.embedding) {
+                embeddings.push(data.embedding);
+            }
+            else {
+                throw new Error("Invalid response from Ollama (fallback)");
+            }
+        }
+        return embeddings;
+    }
+    async pullModel() {
+        try {
+            await this.fetchWithTimeout(`${this.baseUrl}/api/pull`, {
+                method: "POST",
+                headers: { "Content-Type": "application/json" },
+                body: JSON.stringify({
+                    name: this.model,
+                    stream: false,
+                }),
+            });
+            logger.info(`Model ${this.model} pulled successfully.`);
+        }
+        catch (error) {
+            const message = error instanceof Error ? error.message : String(error);
+            throw new Error(`Failed to pull model ${this.model}: ${message}`);
+        }
+    }
+    async checkOllamaRunning() {
+        try {
+            const controller = new AbortController();
+            const timeoutId = setTimeout(() => controller.abort(), 2000);
+            try {
+                const response = await fetch(`${this.baseUrl}/api/version`, {
+                    signal: controller.signal,
+                });
+                return response.status === 200;
+            }
+            finally {
+                clearTimeout(timeoutId);
+            }
+        }
+        catch {
+            return false;
+        }
+    }
+    async waitForOllama(maxAttempts = 60, delayMs = 500) {
+        for (let i = 0; i < maxAttempts; i++) {
+            if (await this.checkOllamaRunning()) {
+                return;
+            }
+            await new Promise((resolve) => setTimeout(resolve, delayMs));
+        }
+        throw new Error(`Ollama did not become ready at ${this.baseUrl} within ${maxAttempts * delayMs}ms`);
+    }
+    startOllama() {
+        try {
+            (0, node_child_process_1.execSync)("ollama serve > /dev/null 2>&1 &", {
+                stdio: "pipe",
+                timeout: 3000,
+            });
+            logger.info("Started Ollama daemon in background.");
+        }
+        catch {
+            // already running or not installed — waitForOllama handles both
+        }
+    }
+    async ensureOllamaAvailable(context) {
+        if (await this.checkOllamaRunning()) {
+            return;
+        }
+        if (!this.reconnectInFlight) {
+            this.reconnectInFlight = (async () => {
+                logger.warn(`Ollama is unavailable during ${context}. Starting and waiting for readiness...`);
+                this.startOllama();
+                await this.waitForOllama();
+            })();
+        }
+        try {
+            await this.reconnectInFlight;
+        }
+        finally {
+            this.reconnectInFlight = null;
+        }
+    }
+    async fetchWithTimeout(url, init, timeoutMs = this.requestTimeoutMs) {
+        const controller = new AbortController();
+        const timeoutId = setTimeout(() => controller.abort(), timeoutMs);
+        try {
+            return await fetch(url, { ...init, signal: controller.signal });
+        }
+        finally {
+            clearTimeout(timeoutId);
+        }
+    }
+}
+exports.OllamaEmbeddingProvider = OllamaEmbeddingProvider;
+//# sourceMappingURL=ollama.js.map

package/dist/embedding/ollama.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ollama.js","sourceRoot":"","sources":["../../src/embedding/ollama.ts"],"names":[],"mappings":";;;AAAA,2DAA8C;AAE9C,iDAAiD;AAEjD,MAAM,MAAM,GAAG,IAAI,wBAAY,CAAC,mBAAmB,CAAC,CAAC;AAErD,MAAa,uBAAuB;IACnB,EAAE,GAAG,QAAQ,CAAC;IACd,SAAS,GAAG,GAAG,CAAC;IAExB,OAAO,CAAS;IAChB,KAAK,CAAS;IACd,SAAS,CAAS;IAClB,WAAW,CAAS;IACpB,MAAM,CAAS;IACf,iBAAiB,GAAyB,IAAI,CAAC;IACtC,gBAAgB,GAAG,MAAM,CAAC;IAE3C,YACC,OAAe,EACf,KAAK,GAAG,SAAS,EACjB,SAAS,GAAG,CAAC,EACb,WAAW,GAAG,CAAC,EACf,MAAM,GAAG,GAAG;QAEZ,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QAC1C,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACnB,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAC/B,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACtB,CAAC;IAED,KAAK,CAAC,UAAU;QACf,MAAM,IAAI,CAAC,qBAAqB,CAAC,YAAY,CAAC,CAAC;IAChD,CAAC;IAED,KAAK,CAAC,KAAK,KAAmB,CAAC;IAE/B,YAAY;QACX,OAAO,IAAI,CAAC,SAAS,CAAC;IACvB,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,KAAe;QAC1B,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,MAAM,OAAO,GAAG,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;QAC1D,MAAM,OAAO,GAAe,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAEpD,MAAM,kBAAkB,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC;YACzD,KAAK;YACL,QAAQ,EAAE,KAAK,GAAG,IAAI,CAAC,SAAS;SAChC,CAAC,CAAC,CAAC;QAEJ,MAAM,KAAK,GAAG,CAAC,GAAG,kBAAkB,CAAC,CAAC;QACtC,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;aAC/D,IAAI,CAAC,IAAI,CAAC;aACV,GAAG,CAAC,KAAK,IAAI,EAAE;YACf,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACzB,MAAM,OAAO,GAAG,KAAK,CAAC,KAAK,EAAE,CAAC;gBAC9B,IAAI,CAAC,OAAO;oBAAE,MAAM;gBAEpB,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC;gBAEpC,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;gBAChD,UAAU,CAAC,OAAO,CAAC,CAAC,SAAS,EAAE,CAAC,EAAE,EAAE;oBACnC,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC,GAAG,SAAS,CAAC;gBACnC,CAAC,CAAC,CAAC;YACJ,CAAC;QACF,CAAC,CAAC,CAAC;QAEJ,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAE3B,KAAK,SAAS,CAAC;QACf,OAAO,OAAO,CAAC;IAChB,CAAC;IAEO,aAAa,CAAI,KAAU,EAAE,SAAiB;QACrD,MAAM,OAAO,GAAU,EAAE,CAAC;QAC1B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC;YAClD,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC;QAC7C,CAAC;QACD,OAAO,OAAO,CAAC;IAChB,CAAC;IAEO,KAAK,CAAC,UAAU,CAAC,KAAe;QACvC,IAAI,CAAC;YACJ,OAAO,MAAM,IAAI,CAAC,mBAAmB,CAAC,KAAK,CAAC,CAAC;QAC9C,CAAC;QAAC,OAAO,KAAc,EAAE,CAAC;YACzB,IAAI,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC,EAAE,CAAC;gBACnC,MAAM,CAAC,IAAI,CACV,4DAA4D,CAC5D,CAAC;gBACF,MAAM,IAAI,CAAC,qBAAqB,CAAC,YAAY,CAAC,CAAC;gBAC/C,OAAO,MAAM,IAAI,CAAC,mBAAmB,CAAC,KAAK,CAAC,CAAC;YAC9C,CAAC;YAED,IAAI,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,EAAE,CAAC;gBACjC,MAAM,CAAC,IAAI,CAAC,SAAS,IAAI,CAAC,KAAK,wBAAwB,CAAC,CAAC;gBACzD,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;gBACvB,OAAO,MAAM,IAAI,CAAC,mBAAmB,CAAC,KAAK,CAAC,CAAC;YAC9C,CAAC;YAED,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACvE,MAAM,IAAI,KAAK,CAAC,4BAA4B,OAAO,EAAE,CAAC,CAAC;QACxD,CAAC;IACF,CAAC;IAEO,iBAAiB,CAAC,KAAc;QACvC,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;YACzC,OAAO,KAAK,CAAC;QACd,CAAC;QAED,8DAA8D;QAC9D,MAAM,KAAK,GAAI,KAAuC,CAAC,KAAK,CAAC;QAC7D,MAAM,IAAI,GAAI,KAA2B,CAAC,IAAI,IAAI,KAAK,EAAE,IAAI,CAAC;QAC9D,MAAM,YAAY,GAAI,KAA+B,CAAC,OAAO,CAAC;QAC9D,MAAM,OAAO,GAAG,OAAO,YAAY,KAAK,QAAQ,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,CAAC;QAErE,IACC,IAAI;YACJ;gBACC,cAAc;gBACd,WAAW;gBACX,YAAY;gBACZ,cAAc;gBACd,cAAc;aACd,CAAC,QAAQ,CAAC,IAAI,CAAC,EACf,CAAC;YACF,OAAO,IAAI,CAAC;QACb,CAAC;QAED,OAAO,CACN,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC;YAChC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC;YAC7B,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC;YAC9B,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC;YAChC,OAAO,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,CACzC,CAAC;IACH,CAAC;IAEO,eAAe,CAAC,KAAc;QACrC,IAAI,KAAK,YAAY,QAAQ,EAAE,CAAC;YAC/B,OAAO,KAAK,CAAC,MAAM,KAAK,GAAG,CAAC;QAC7B,CAAC;QACD,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;YACzC,OAAO,KAAK,CAAC;QACd,CAAC;QACD,MAAM,MAAM,GAAI,KAA6B,CAAC,MAAM,CAAC;QACrD,OAAO,MAAM,KAAK,GAAG,CAAC;IACvB,CAAC;IAEO,KAAK,CAAC,mBAAmB,CAAC,KAAe;QAChD,IAAI,CAAC;YACJ,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAC3C,GAAG,IAAI,CAAC,OAAO,YAAY,EAC3B;gBACC,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE;gBAC/C,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;oBACpB,KAAK,EAAE,IAAI,CAAC,KAAK;oBACjB,KAAK,EAAE,KAAK;oBACZ,OAAO,EAAE;wBACR,OAAO,EAAE,IAAI,CAAC,MAAM;qBACpB;iBACD,CAAC;aACF,CACD,CAAC;YAEF,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;gBAC7B,OAAO,MAAM,IAAI,CAAC,uBAAuB,CAAC,KAAK,CAAC,CAAC;YAClD,CAAC;YAED,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBAClB,MAAM,IAAI,KAAK,CAAC,gCAAgC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;YACpE,CAAC;YAED,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YACnC,IAAI,IAAI,EAAE,UAAU,EAAE,CAAC;gBACtB,OAAO,IAAI,CAAC,UAAU,CAAC;YACxB,CAAC;QACF,CAAC;QAAC,OAAO,KAAc,EAAE,CAAC;YACzB,IAAI,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,EAAE,CAAC;gBACjC,OAAO,MAAM,IAAI,CAAC,uBAAuB,CAAC,KAAK,CAAC,CAAC;YAClD,CAAC;YACD,MAAM,KAAK,CAAC;QACb,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC;IACjD,CAAC;IAEO,KAAK,CAAC,uBAAuB,CAAC,KAAe;QACpD,MAAM,UAAU,GAAe,EAAE,CAAC;QAElC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YAC1B,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAC3C,GAAG,IAAI,CAAC,OAAO,iBAAiB,EAChC;gBACC,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE;gBAC/C,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;oBACpB,KAAK,EAAE,IAAI,CAAC,KAAK;oBACjB,MAAM,EAAE,IAAI;oBACZ,OAAO,EAAE;wBACR,OAAO,EAAE,IAAI,CAAC,MAAM;qBACpB;iBACD,CAAC;aACF,CACD,CAAC;YAEF,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBAClB,MAAM,IAAI,KAAK,CACd,gCAAgC,QAAQ,CAAC,MAAM,aAAa,CAC5D,CAAC;YACH,CAAC;YAED,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YACnC,IAAI,IAAI,EAAE,SAAS,EAAE,CAAC;gBACrB,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YACjC,CAAC;iBAAM,CAAC;gBACP,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;YAC5D,CAAC;QACF,CAAC;QAED,OAAO,UAAU,CAAC;IACnB,CAAC;IAEO,KAAK,CAAC,SAAS;QACtB,IAAI,CAAC;YACJ,MAAM,IAAI,CAAC,gBAAgB,CAAC,GAAG,IAAI,CAAC,OAAO,WAAW,EAAE;gBACvD,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE;gBAC/C,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;oBACpB,IAAI,EAAE,IAAI,CAAC,KAAK;oBAChB,MAAM,EAAE,KAAK;iBACb,CAAC;aACF,CAAC,CAAC;YACH,MAAM,CAAC,IAAI,CAAC,SAAS,IAAI,CAAC,KAAK,uBAAuB,CAAC,CAAC;QACzD,CAAC;QAAC,OAAO,KAAc,EAAE,CAAC;YACzB,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACvE,MAAM,IAAI,KAAK,CAAC,wBAAwB,IAAI,CAAC,KAAK,KAAK,OAAO,EAAE,CAAC,CAAC;QACnE,CAAC;IACF,CAAC;IAEO,KAAK,CAAC,kBAAkB;QAC/B,IAAI,CAAC;YACJ,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;YACzC,MAAM,SAAS,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,IAAI,CAAC,CAAC;YAC7D,IAAI,CAAC;gBACJ,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,IAAI,CAAC,OAAO,cAAc,EAAE;oBAC3D,MAAM,EAAE,UAAU,CAAC,MAAM;iBACzB,CAAC,CAAC;gBACH,OAAO,QAAQ,CAAC,MAAM,KAAK,GAAG,CAAC;YAChC,CAAC;oBAAS,CAAC;gBACV,YAAY,CAAC,SAAS,CAAC,CAAC;YACzB,CAAC;QACF,CAAC;QAAC,MAAM,CAAC;YACR,OAAO,KAAK,CAAC;QACd,CAAC;IACF,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,WAAW,GAAG,EAAE,EAAE,OAAO,GAAG,GAAG;QAC1D,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,EAAE,CAAC,EAAE,EAAE,CAAC;YACtC,IAAI,MAAM,IAAI,CAAC,kBAAkB,EAAE,EAAE,CAAC;gBACrC,OAAO;YACR,CAAC;YACD,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC;QAC9D,CAAC;QAED,MAAM,IAAI,KAAK,CACd,kCAAkC,IAAI,CAAC,OAAO,WAAW,WAAW,GAAG,OAAO,IAAI,CAClF,CAAC;IACH,CAAC;IAEO,WAAW;QAClB,IAAI,CAAC;YACJ,IAAA,6BAAQ,EAAC,iCAAiC,EAAE;gBAC3C,KAAK,EAAE,MAAM;gBACb,OAAO,EAAE,IAAI;aACb,CAAC,CAAC;YACH,MAAM,CAAC,IAAI,CAAC,sCAAsC,CAAC,CAAC;QACrD,CAAC;QAAC,MAAM,CAAC;YACR,gEAAgE;QACjE,CAAC;IACF,CAAC;IAEO,KAAK,CAAC,qBAAqB,CAAC,OAAe;QAClD,IAAI,MAAM,IAAI,CAAC,kBAAkB,EAAE,EAAE,CAAC;YACrC,OAAO;QACR,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,iBAAiB,EAAE,CAAC;YAC7B,IAAI,CAAC,iBAAiB,GAAG,CAAC,KAAK,IAAI,EAAE;gBACpC,MAAM,CAAC,IAAI,CACV,gCAAgC,OAAO,yCAAyC,CAChF,CAAC;gBACF,IAAI,CAAC,WAAW,EAAE,CAAC;gBACnB,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;YAC5B,CAAC,CAAC,EAAE,CAAC;QACN,CAAC;QAED,IAAI,CAAC;YACJ,MAAM,IAAI,CAAC,iBAAiB,CAAC;QAC9B,CAAC;gBAAS,CAAC;YACV,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC;QAC/B,CAAC;IACF,CAAC;IAEO,KAAK,CAAC,gBAAgB,CAC7B,GAAW,EACX,IAAiB,EACjB,SAAS,GAAG,IAAI,CAAC,gBAAgB;QAEjC,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,MAAM,SAAS,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,SAAS,CAAC,CAAC;QAClE,IAAI,CAAC;YACJ,OAAO,MAAM,KAAK,CAAC,GAAG,EAAE,EAAE,GAAG,IAAI,EAAE,MAAM,EAAE,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC;QACjE,CAAC;gBAAS,CAAC;YACV,YAAY,CAAC,SAAS,CAAC,CAAC;QACzB,CAAC;IACF,CAAC;CACD;AAzTD,0DAyTC"}

package/dist/engine/architecture.d.ts

@@ -0,0 +1,55 @@
+import type { MetadataStore, ProjectId, SnapshotId } from "../core/types.js";
+import type { LanguagePlugin } from "../languages/plugin.js";
+export interface DirectoryNode {
+    name: string;
+    path: string;
+    type: "directory";
+    children: (DirectoryNode | FileNode)[];
+}
+export interface FileNode {
+    name: string;
+    path: string;
+    type: "file";
+    size: number;
+    language: string;
+}
+export interface ArchitectureDependencyMap {
+    internal: Record<string, string[]>;
+    external: Record<string, string[]>;
+    builtin: Record<string, string[]>;
+    unresolved: Record<string, string[]>;
+}
+export interface ArchitectureFileSummary {
+    path: string;
+    language: string;
+}
+export interface ArchitectureSnapshot {
+    structure: DirectoryNode;
+    entrypoints: string[];
+    dependencies: Record<string, number>;
+    dependency_map: ArchitectureDependencyMap;
+    file_stats: Record<string, number>;
+    files?: ArchitectureFileSummary[];
+    module_files?: Record<string, string[]>;
+}
+export declare function matchesPathPatterns(filePath: string, patterns: string[]): boolean;
+export declare function filterArchitectureSnapshot(snapshot: ArchitectureSnapshot, excludePathPatterns: string[]): ArchitectureSnapshot;
+export declare class ArchitectureGenerator {
+    private metadataStore;
+    private languagePlugins;
+    constructor(metadataStore: MetadataStore, languagePlugins?: LanguagePlugin[]);
+    generate(projectId: ProjectId, snapshotId: SnapshotId): Promise<void>;
+    private buildModuleFiles;
+    private buildDirectoryTree;
+    private calculateFileStats;
+    private findEntrypoints;
+    private dedupePreserveOrder;
+    private buildDependencies;
+    private addDependencyToBuckets;
+    private summarizeDependencies;
+    private mapToSortedRecord;
+    private normalizePath;
+    private canonicalizePath;
+    private getModuleKey;
+    private isTestPath;
+}

package/dist/engine/architecture.js

@@ -0,0 +1,359 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ArchitectureGenerator = void 0;
+exports.matchesPathPatterns = matchesPathPatterns;
+exports.filterArchitectureSnapshot = filterArchitectureSnapshot;
+const node_path_1 = __importDefault(require("node:path"));
+function normalizePath(value) {
+    return value.replace(/\\/g, "/");
+}
+function escapeRegExp(value) {
+    return value.replace(/[|\\{}()[\]^$+?.]/g, "\\$&");
+}
+function pathPatternToRegExp(pattern) {
+    const normalized = normalizePath(pattern.trim()).replace(/^\.\//, "");
+    const doubleWildcardToken = "__INDEXER_DOUBLE_WILDCARD__";
+    const regexSource = escapeRegExp(normalized)
+        .replace(/\*\*/g, doubleWildcardToken)
+        .replace(/\*/g, "[^/]*")
+        .replace(/\?/g, "[^/]")
+        .replace(new RegExp(doubleWildcardToken, "g"), ".*");
+    return new RegExp(`^${regexSource}$`);
+}
+function matchesPathPatterns(filePath, patterns) {
+    const normalizedPath = normalizePath(filePath);
+    for (const pattern of patterns) {
+        const trimmedPattern = pattern.trim();
+        if (!trimmedPattern) {
+            continue;
+        }
+        const regex = pathPatternToRegExp(trimmedPattern);
+        if (regex.test(normalizedPath)) {
+            return true;
+        }
+    }
+    return false;
+}
+function calculateFileStatsFromSummary(files) {
+    const stats = {};
+    for (const file of files) {
+        const language = file.language || "unknown";
+        stats[language] = (stats[language] || 0) + 1;
+    }
+    return stats;
+}
+function summarizeDependencies(detail) {
+    const summary = new Map();
+    const bump = (name) => summary.set(name, (summary.get(name) || 0) + 1);
+    for (const bucket of Object.values(detail)) {
+        for (const deps of Object.values(bucket)) {
+            for (const dep of new Set(deps)) {
+                bump(dep);
+            }
+        }
+    }
+    return Object.fromEntries(Array.from(summary.entries()).sort((a, b) => a[0].localeCompare(b[0])));
+}
+function filterStructure(node, includedFilePaths) {
+    const children = [];
+    for (const child of node.children) {
+        if (child.type === "file") {
+            if (includedFilePaths.has(normalizePath(child.path))) {
+                children.push(child);
+            }
+            continue;
+        }
+        const filteredChild = filterStructure(child, includedFilePaths);
+        if (filteredChild) {
+            children.push(filteredChild);
+        }
+    }
+    if (node.path !== "." && children.length === 0) {
+        return null;
+    }
+    return {
+        ...node,
+        children,
+    };
+}
+function filterArchitectureSnapshot(snapshot, excludePathPatterns) {
+    if (excludePathPatterns.length === 0) {
+        return snapshot;
+    }
+    const files = snapshot.files ?? [];
+    const includedFiles = files.filter((file) => !matchesPathPatterns(file.path, excludePathPatterns));
+    const includedFilePaths = new Set(includedFiles.map((file) => normalizePath(file.path)));
+    const filteredModuleFiles = Object.fromEntries(Object.entries(snapshot.module_files ?? {})
+        .map(([moduleKey, filePaths]) => [
+        moduleKey,
+        filePaths.filter((filePath) => includedFilePaths.has(normalizePath(filePath))),
+    ])
+        .filter(([, filePaths]) => filePaths.length > 0));
+    const includedModules = new Set(Object.keys(filteredModuleFiles));
+    const dependencyMap = {
+        internal: Object.fromEntries(Object.entries(snapshot.dependency_map.internal ?? {})
+            .filter(([fromModule, toModules]) => includedModules.has(fromModule) &&
+            toModules.some((toModule) => includedModules.has(toModule)))
+            .map(([fromModule, toModules]) => [
+            fromModule,
+            toModules.filter((toModule) => includedModules.has(toModule)),
+        ])),
+        external: Object.fromEntries(Object.entries(snapshot.dependency_map.external ?? {}).filter(([fromModule]) => includedModules.has(fromModule))),
+        builtin: Object.fromEntries(Object.entries(snapshot.dependency_map.builtin ?? {}).filter(([fromModule]) => includedModules.has(fromModule))),
+        unresolved: Object.fromEntries(Object.entries(snapshot.dependency_map.unresolved ?? {}).filter(([fromModule]) => includedModules.has(fromModule))),
+    };
+    return {
+        ...snapshot,
+        structure: filterStructure(snapshot.structure, includedFilePaths) ??
+            snapshot.structure,
+        entrypoints: (snapshot.entrypoints ?? []).filter((entrypoint) => includedFilePaths.has(normalizePath(entrypoint))),
+        dependencies: summarizeDependencies(dependencyMap),
+        dependency_map: dependencyMap,
+        file_stats: calculateFileStatsFromSummary(includedFiles),
+        files: includedFiles,
+        module_files: filteredModuleFiles,
+    };
+}
+class ArchitectureGenerator {
+    metadataStore;
+    languagePlugins;
+    constructor(metadataStore, languagePlugins = []) {
+        this.metadataStore = metadataStore;
+        this.languagePlugins = languagePlugins;
+    }
+    async generate(projectId, snapshotId) {
+        const files = await this.metadataStore.listFiles(projectId, snapshotId);
+        const dependencies = await this.buildDependencies(projectId, snapshotId, files);
+        const structure = this.buildDirectoryTree(files);
+        const file_stats = this.calculateFileStats(files);
+        const entrypoints = this.findEntrypoints(files);
+        const filesSummary = files.map((file) => ({
+            path: file.path,
+            language: file.languageId || "unknown",
+        }));
+        const module_files = this.buildModuleFiles(files);
+        await this.metadataStore.upsertArtifact(projectId, {
+            snapshotId,
+            projectId,
+            artifactType: "architecture_snapshot",
+            scope: "project",
+            dataJson: JSON.stringify({
+                structure,
+                entrypoints,
+                dependencies: dependencies.summary,
+                dependency_map: dependencies.detail,
+                file_stats,
+                files: filesSummary,
+                module_files,
+            }),
+        });
+    }
+    buildModuleFiles(files) {
+        const moduleFiles = new Map();
+        for (const file of files) {
+            const moduleKey = this.getModuleKey(file.path);
+            if (!moduleFiles.has(moduleKey)) {
+                moduleFiles.set(moduleKey, new Set());
+            }
+            moduleFiles.get(moduleKey)?.add(file.path);
+        }
+        return Object.fromEntries(Array.from(moduleFiles.entries())
+            .sort((a, b) => a[0].localeCompare(b[0]))
+            .map(([moduleKey, filePaths]) => [
+            moduleKey,
+            Array.from(filePaths).sort(),
+        ]));
+    }
+    buildDirectoryTree(files) {
+        const root = {
+            name: "root",
+            path: ".",
+            type: "directory",
+            children: [],
+        };
+        const map = new Map([[".", root]]);
+        for (const file of files) {
+            const parts = file.path.split("/");
+            let currentPath = ".";
+            for (let index = 0; index < parts.length - 1; index += 1) {
+                const part = parts[index];
+                const parentPath = currentPath;
+                currentPath = currentPath === "." ? part : `${currentPath}/${part}`;
+                if (!map.has(currentPath)) {
+                    const node = {
+                        name: part,
+                        path: currentPath,
+                        type: "directory",
+                        children: [],
+                    };
+                    map.set(currentPath, node);
+                    map.get(parentPath)?.children.push(node);
+                }
+            }
+            const fileName = parts[parts.length - 1];
+            const parentPath = parts.length > 1 ? parts.slice(0, -1).join("/") : ".";
+            map.get(parentPath)?.children.push({
+                name: fileName,
+                path: file.path,
+                type: "file",
+                size: file.size,
+                language: file.languageId,
+            });
+        }
+        return root;
+    }
+    calculateFileStats(files) {
+        const stats = {};
+        for (const file of files) {
+            const language = file.languageId || "unknown";
+            stats[language] = (stats[language] || 0) + 1;
+        }
+        return stats;
+    }
+    findEntrypoints(files) {
+        const entrypoints = [];
+        for (const plugin of this.languagePlugins) {
+            if (!plugin.getEntrypoints) {
+                continue;
+            }
+            const extensions = new Set(plugin.fileExtensions.map((ext) => ext.toLowerCase()));
+            const pluginFiles = files
+                .filter((file) => !this.isTestPath(file.path))
+                .filter((file) => extensions.has(node_path_1.default.extname(file.path).toLowerCase()))
+                .map((file) => file.path);
+            if (pluginFiles.length === 0) {
+                continue;
+            }
+            entrypoints.push(...plugin.getEntrypoints(pluginFiles));
+        }
+        return this.dedupePreserveOrder(entrypoints);
+    }
+    dedupePreserveOrder(paths) {
+        const seen = new Set();
+        const result = [];
+        for (const value of paths) {
+            if (seen.has(value)) {
+                continue;
+            }
+            seen.add(value);
+            result.push(value);
+        }
+        return result;
+    }
+    async buildDependencies(projectId, snapshotId, files) {
+        const dependencies = await this.metadataStore.listDependencies(projectId, snapshotId);
+        const fileSet = new Set(files.map((file) => this.normalizePath(file.path)));
+        const buckets = {
+            internal: new Map(),
+            external: new Map(),
+            builtin: new Map(),
+            unresolved: new Map(),
+        };
+        const add = (bucket, from, to) => {
+            if (!to)
+                return;
+            if (!buckets[bucket].has(from)) {
+                buckets[bucket].set(from, new Set());
+            }
+            buckets[bucket].get(from)?.add(to);
+        };
+        for (const dependency of dependencies) {
+            this.addDependencyToBuckets(dependency, fileSet, add);
+        }
+        const detail = {
+            internal: this.mapToSortedRecord(buckets.internal),
+            external: this.mapToSortedRecord(buckets.external),
+            builtin: this.mapToSortedRecord(buckets.builtin),
+            unresolved: this.mapToSortedRecord(buckets.unresolved),
+        };
+        return {
+            summary: this.summarizeDependencies(detail),
+            detail,
+        };
+    }
+    addDependencyToBuckets(dependency, fileSet, add) {
+        const fromPath = this.normalizePath(dependency.fromPath);
+        const fromModule = this.getModuleKey(fromPath);
+        const dependencyType = dependency.dependencyType ?? "unresolved";
+        if (dependencyType === "internal" && dependency.toPath) {
+            const toPath = this.normalizePath(dependency.toPath);
+            const toModule = this.getModuleKey(this.canonicalizePath(toPath, fileSet));
+            if (toModule && toModule !== fromModule) {
+                add("internal", fromModule, toModule);
+            }
+            return;
+        }
+        const target = dependency.toSpecifier || dependency.toPath;
+        if (!target) {
+            return;
+        }
+        if (dependencyType === "external") {
+            add("external", fromModule, target);
+        }
+        else if (dependencyType === "builtin") {
+            add("builtin", fromModule, target);
+        }
+        else {
+            add("unresolved", fromModule, target);
+        }
+    }
+    summarizeDependencies(detail) {
+        return summarizeDependencies(detail);
+    }
+    mapToSortedRecord(map) {
+        const entries = Array.from(map.entries()).sort((a, b) => a[0].localeCompare(b[0]));
+        return Object.fromEntries(entries.map(([key, values]) => [key, Array.from(values).sort()]));
+    }
+    normalizePath(value) {
+        return normalizePath(value);
+    }
+    canonicalizePath(value, fileSet) {
+        const normalized = this.normalizePath(value);
+        if (!normalized.includes("/dist/"))
+            return normalized;
+        if (!normalized.endsWith(".js") && !normalized.endsWith(".jsx"))
+            return normalized;
+        const tsCandidate = normalized
+            .replace("/dist/", "/src/")
+            .replace(/\.jsx?$/, ".ts");
+        if (fileSet.has(tsCandidate))
+            return tsCandidate;
+        const tsxCandidate = tsCandidate.replace(/\.ts$/, ".tsx");
+        if (fileSet.has(tsxCandidate))
+            return tsxCandidate;
+        return normalized;
+    }
+    getModuleKey(filePath) {
+        const parts = this.normalizePath(filePath).split("/");
+        if (parts.length === 1)
+            return "root";
+        const rootGroup = parts[0];
+        if (["packages", "services", "apps", "libs", "modules"].includes(rootGroup) &&
+            parts[1]) {
+            return `${rootGroup}/${parts[1]}`;
+        }
+        if (parts.length >= 2) {
+            return `${parts[0]}/${parts[1]}`;
+        }
+        return rootGroup;
+    }
+    isTestPath(filePath) {
+        const normalized = this.normalizePath(filePath).toLowerCase();
+        if (normalized.includes("/__tests__/"))
+            return true;
+        if (normalized.includes("/__test__/"))
+            return true;
+        if (normalized.includes("/tests/"))
+            return true;
+        if (normalized.includes("/test/"))
+            return true;
+        if (normalized.includes("/fixtures/"))
+            return true;
+        return /(\.|\/)(spec|test)\.[^/]+$/.test(normalized);
+    }
+}
+exports.ArchitectureGenerator = ArchitectureGenerator;
+//# sourceMappingURL=architecture.js.map