@ninebix/nmt-system 1.0.0

package/dist/services/efficient-rag.js

@@ -0,0 +1,311 @@
+/**
+ * Efficient RAG Pipeline
+ * 비용 최적화된 RAG 파이프라인
+ *
+ * @module services/efficient-rag
+ */
+import { ContextCompressor } from './context-compressor.js';
+import { QueryCache } from './query-cache.js';
+import { LLMRouter } from './llm-router.js';
+/**
+ * Efficient RAG Pipeline 클래스
+ */
+export class EfficientRAGPipeline {
+    queryService;
+    compressor;
+    cache;
+    router;
+    config;
+    // 임베딩 함수 (외부 주입)
+    embedFn = null;
+    // 통계
+    stats = {
+        totalRequests: 0,
+        cacheHits: 0,
+        totalCost: 0,
+        totalSaved: 0,
+        totalSearchTime: 0,
+        totalLLMTime: 0
+    };
+    constructor(queryService, config = {}) {
+        this.queryService = queryService;
+        this.compressor = new ContextCompressor();
+        this.cache = new QueryCache();
+        this.router = new LLMRouter();
+        this.config = {
+            enableCache: config.enableCache ?? true,
+            enableCompression: config.enableCompression ?? true,
+            enableSmartRouting: config.enableSmartRouting ?? true,
+            defaultK: config.defaultK ?? 10,
+            defaultMaxTokens: config.defaultMaxTokens ?? 2000,
+            apiKeys: config.apiKeys ?? {}
+        };
+        // API 키 설정
+        for (const [provider, key] of Object.entries(this.config.apiKeys)) {
+            if (key) {
+                this.router.setApiKey(provider, key);
+            }
+        }
+    }
+    /**
+     * 임베딩 함수 설정
+     */
+    setEmbeddingFunction(fn) {
+        this.embedFn = fn;
+    }
+    /**
+     * API 키 설정
+     */
+    setApiKey(provider, apiKey) {
+        this.config.apiKeys[provider] = apiKey;
+        this.router.setApiKey(provider, apiKey);
+    }
+    /**
+     * 메인 RAG 실행
+     */
+    async query(request) {
+        const startTime = Date.now();
+        this.stats.totalRequests++;
+        const { query, options = {} } = request;
+        const { k = this.config.defaultK, maxTokens = this.config.defaultMaxTokens, forceComplexity, skipCache = false, includeProof = false, systemPrompt, tags } = options;
+        // 쿼리 임베딩 생성 (캐시/시맨틱 매칭용)
+        let queryEmbedding;
+        if (this.embedFn) {
+            queryEmbedding = await this.embedFn(query);
+        }
+        // 1. 캐시 확인
+        let cacheResult = { hit: false };
+        if (this.config.enableCache && !skipCache) {
+            cacheResult = this.cache.get(query, queryEmbedding);
+            if (cacheResult.hit && cacheResult.entry) {
+                this.stats.cacheHits++;
+                return {
+                    answer: cacheResult.entry.response,
+                    sources: cacheResult.entry.contexts.map((c, i) => ({
+                        id: `cached-${i}`,
+                        content: c,
+                        score: 1.0
+                    })),
+                    metrics: {
+                        searchTimeMs: 0,
+                        compressionTimeMs: 0,
+                        llmTimeMs: 0,
+                        totalTimeMs: Date.now() - startTime,
+                        cacheHit: true,
+                        cacheType: cacheResult.type,
+                        complexity: 'simple',
+                        model: cacheResult.entry.llmModel,
+                        tokensUsed: 0,
+                        estimatedCost: 0,
+                        savedCost: ((cacheResult.entry.tokenUsage?.prompt ?? 0) + (cacheResult.entry.tokenUsage?.completion ?? 0)) * 0.00003,
+                        compressionRatio: 0
+                    }
+                };
+            }
+        }
+        // 2. 시맨틱 검색
+        const searchStart = Date.now();
+        const searchOptions = {
+            k,
+            includeContent: true,
+            includeProof,
+            tags
+        };
+        const searchResults = await this.queryService.search(query, searchOptions);
+        const searchTime = Date.now() - searchStart;
+        this.stats.totalSearchTime += searchTime;
+        // 3. 컨텍스트 압축
+        const compressionStart = Date.now();
+        let compressedContext;
+        if (this.config.enableCompression) {
+            const chunks = searchResults.map(r => ({
+                id: r.neuron.id,
+                content: r.content || '',
+                score: r.score,
+                embedding: r.neuron.embedding
+            }));
+            compressedContext = this.compressor.compress(chunks, queryEmbedding, {
+                maxTokens,
+                maxChunks: Math.min(k, 7),
+                strategy: 'hybrid'
+            });
+        }
+        else {
+            // 압축 없이 상위 결과만
+            compressedContext = {
+                chunks: searchResults.slice(0, 5).map(r => ({
+                    id: r.neuron.id,
+                    content: r.content || '',
+                    score: r.score
+                })),
+                originalCount: searchResults.length,
+                compressedCount: Math.min(5, searchResults.length),
+                totalTokens: 0,
+                savedTokens: 0,
+                compressionRatio: 0
+            };
+        }
+        const compressionTime = Date.now() - compressionStart;
+        // 4. LLM 라우팅 & 응답 생성
+        const llmStart = Date.now();
+        const contexts = compressedContext.chunks.map(c => c.content);
+        let routingResult;
+        if (this.config.enableSmartRouting) {
+            routingResult = await this.router.route(query, contexts, {
+                systemPrompt,
+                forceModel: forceComplexity,
+                maxTokens: 1024
+            });
+        }
+        else {
+            // 스마트 라우팅 비활성화 시 기본 모델 사용
+            routingResult = await this.router.route(query, contexts, {
+                systemPrompt,
+                forceModel: 'medium',
+                maxTokens: 1024
+            });
+        }
+        const llmTime = Date.now() - llmStart;
+        this.stats.totalLLMTime += llmTime;
+        this.stats.totalCost += routingResult.actualCost;
+        this.stats.totalSaved += routingResult.savedCost;
+        // 5. 캐시 저장
+        if (this.config.enableCache && !skipCache) {
+            this.cache.set(query, routingResult.response.content, contexts, {
+                queryEmbedding,
+                llmModel: routingResult.selectedModel.model,
+                processingTime: llmTime,
+                tokenUsage: routingResult.response.usage ? {
+                    prompt: routingResult.response.usage.promptTokens,
+                    completion: routingResult.response.usage.completionTokens
+                } : undefined
+            });
+        }
+        // 6. 결과 구성
+        const sources = compressedContext.chunks.map(c => {
+            const original = searchResults.find(r => r.neuron.id === c.id);
+            return {
+                id: c.id,
+                content: c.content,
+                score: c.score,
+                proof: original?.proof ? JSON.stringify(original.proof) : undefined
+            };
+        });
+        return {
+            answer: routingResult.response.content,
+            sources,
+            metrics: {
+                searchTimeMs: searchTime,
+                compressionTimeMs: compressionTime,
+                llmTimeMs: llmTime,
+                totalTimeMs: Date.now() - startTime,
+                cacheHit: false,
+                complexity: routingResult.complexity.level,
+                model: routingResult.selectedModel.model,
+                tokensUsed: routingResult.response.usage?.totalTokens ?? routingResult.complexity.estimatedTokens,
+                estimatedCost: routingResult.actualCost,
+                savedCost: routingResult.savedCost + (compressedContext.savedTokens * 0.00003),
+                compressionRatio: compressedContext.compressionRatio
+            }
+        };
+    }
+    /**
+     * 빠른 쿼리 (캐시 우선, 압축 최대)
+     */
+    async quickQuery(query) {
+        return this.query({
+            query,
+            options: {
+                k: 5,
+                maxTokens: 1000,
+                forceComplexity: 'simple'
+            }
+        });
+    }
+    /**
+     * 정밀 쿼리 (캐시 스킵, 압축 최소)
+     */
+    async preciseQuery(query, tags) {
+        return this.query({
+            query,
+            options: {
+                k: 15,
+                maxTokens: 4000,
+                skipCache: true,
+                includeProof: true,
+                tags
+            }
+        });
+    }
+    /**
+     * 통계 조회
+     */
+    getStats() {
+        const cacheStats = this.cache.getStats();
+        const routerStats = this.router.getStats();
+        return {
+            pipeline: {
+                totalRequests: this.stats.totalRequests,
+                cacheHitRate: this.stats.totalRequests > 0
+                    ? this.stats.cacheHits / this.stats.totalRequests
+                    : 0,
+                avgSearchTimeMs: this.stats.totalRequests > 0
+                    ? this.stats.totalSearchTime / this.stats.totalRequests
+                    : 0,
+                avgLLMTimeMs: this.stats.totalRequests > 0
+                    ? this.stats.totalLLMTime / this.stats.totalRequests
+                    : 0,
+                totalCostUSD: Math.round(this.stats.totalCost * 10000) / 10000,
+                totalSavedUSD: Math.round(this.stats.totalSaved * 10000) / 10000,
+                savingsRate: this.stats.totalCost > 0
+                    ? this.stats.totalSaved / (this.stats.totalCost + this.stats.totalSaved)
+                    : 0
+            },
+            cache: cacheStats,
+            router: routerStats
+        };
+    }
+    /**
+     * 캐시 워밍
+     */
+    async warmupCache(queries) {
+        let warmedUp = 0;
+        for (const query of queries) {
+            try {
+                await this.query({ query, options: { skipCache: false } });
+                warmedUp++;
+            }
+            catch (error) {
+                console.error(`Failed to warm up query: ${query}`);
+            }
+        }
+        return warmedUp;
+    }
+    /**
+     * 캐시 클리어
+     */
+    clearCache() {
+        this.cache.clear();
+    }
+    /**
+     * 통계 초기화
+     */
+    resetStats() {
+        this.stats = {
+            totalRequests: 0,
+            cacheHits: 0,
+            totalCost: 0,
+            totalSaved: 0,
+            totalSearchTime: 0,
+            totalLLMTime: 0
+        };
+        this.router.resetStats();
+    }
+}
+/**
+ * 파이프라인 팩토리 함수
+ */
+export function createEfficientRAGPipeline(queryService, config) {
+    return new EfficientRAGPipeline(queryService, config);
+}
+//# sourceMappingURL=efficient-rag.js.map

package/dist/services/efficient-rag.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"efficient-rag.js","sourceRoot":"","sources":["../../src/services/efficient-rag.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAGH,OAAO,EAAE,iBAAiB,EAAwC,MAAM,yBAAyB,CAAC;AAClG,OAAO,EAAE,UAAU,EAAkB,MAAM,kBAAkB,CAAC;AAC9D,OAAO,EAAE,SAAS,EAAkC,MAAM,iBAAiB,CAAC;AA2D5E;;GAEG;AACH,MAAM,OAAO,oBAAoB;IACvB,YAAY,CAAe;IAC3B,UAAU,CAAoB;IAC9B,KAAK,CAAa;IAClB,MAAM,CAAY;IAClB,MAAM,CAA2B;IAEzC,iBAAiB;IACT,OAAO,GAAqD,IAAI,CAAC;IAEzE,KAAK;IACG,KAAK,GAAG;QACd,aAAa,EAAE,CAAC;QAChB,SAAS,EAAE,CAAC;QACZ,SAAS,EAAE,CAAC;QACZ,UAAU,EAAE,CAAC;QACb,eAAe,EAAE,CAAC;QAClB,YAAY,EAAE,CAAC;KAChB,CAAC;IAEF,YACE,YAA0B,EAC1B,SAAyB,EAAE;QAE3B,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;QACjC,IAAI,CAAC,UAAU,GAAG,IAAI,iBAAiB,EAAE,CAAC;QAC1C,IAAI,CAAC,KAAK,GAAG,IAAI,UAAU,EAAE,CAAC;QAC9B,IAAI,CAAC,MAAM,GAAG,IAAI,SAAS,EAAE,CAAC;QAE9B,IAAI,CAAC,MAAM,GAAG;YACZ,WAAW,EAAE,MAAM,CAAC,WAAW,IAAI,IAAI;YACvC,iBAAiB,EAAE,MAAM,CAAC,iBAAiB,IAAI,IAAI;YACnD,kBAAkB,EAAE,MAAM,CAAC,kBAAkB,IAAI,IAAI;YACrD,QAAQ,EAAE,MAAM,CAAC,QAAQ,IAAI,EAAE;YAC/B,gBAAgB,EAAE,MAAM,CAAC,gBAAgB,IAAI,IAAI;YACjD,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,EAAE;SAC9B,CAAC;QAEF,WAAW;QACX,KAAK,MAAM,CAAC,QAAQ,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC;YAClE,IAAI,GAAG,EAAE,CAAC;gBACR,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,QAAuB,EAAE,GAAG,CAAC,CAAC;YACtD,CAAC;QACH,CAAC;IACH,CAAC;IAED;;OAEG;IACH,oBAAoB,CAAC,EAA2C;QAC9D,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC;IACpB,CAAC;IAED;;OAEG;IACH,SAAS,CAAC,QAAqB,EAAE,MAAc;QAC7C,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,MAAM,CAAC;QACvC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IAC1C,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK,CAAC,OAA4B;QACtC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,IAAI,CAAC,KAAK,CAAC,aAAa,EAAE,CAAC;QAE3B,MAAM,EACJ,KAAK,EACL,OAAO,GAAG,EAAE,EACb,GAAG,OAAO,CAAC;QAEZ,MAAM,EACJ,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,EACxB,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,gBAAgB,EACxC,eAAe,EACf,SAAS,GAAG,KAAK,EACjB,YAAY,GAAG,KAAK,EACpB,YAAY,EACZ,IAAI,EACL,GAAG,OAAO,CAAC;QAEZ,yBAAyB;QACzB,IAAI,cAAwC,CAAC;QAC7C,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,cAAc,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QAC7C,CAAC;QAED,WAAW;QACX,IAAI,WAAW,GAAmB,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC;QACjD,IAAI,IAAI,CAAC,MAAM,CAAC,WAAW,IAAI,CAAC,SAAS,EAAE,CAAC;YAC1C,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,cAAc,CAAC,CAAC;YAEpD,IAAI,WAAW,CAAC,GAAG,IAAI,WAAW,CAAC,KAAK,EAAE,CAAC;gBACzC,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC;gBAEvB,OAAO;oBACL,MAAM,EAAE,WAAW,CAAC,KAAK,CAAC,QAAQ;oBAClC,OAAO,EAAE,WAAW,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;wBACjD,EAAE,EAAE,UAAU,CAAC,EAAE;wBACjB,OAAO,EAAE,CAAC;wBACV,KAAK,EAAE,GAAG;qBACX,CAAC,CAAC;oBACH,OAAO,EAAE;wBACP,YAAY,EAAE,CAAC;wBACf,iBAAiB,EAAE,CAAC;wBACpB,SAAS,EAAE,CAAC;wBACZ,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;wBACnC,QAAQ,EAAE,IAAI;wBACd,SAAS,EAAE,WAAW,CAAC,IAAI;wBAC3B,UAAU,EAAE,QAAQ;wBACpB,KAAK,EAAE,WAAW,CAAC,KAAK,CAAC,QAAQ;wBACjC,UAAU,EAAE,CAAC;wBACb,aAAa,EAAE,CAAC;wBAChB,SAAS,EAAE,CAAC,CAAC,WAAW,CAAC,KAAK,CAAC,UAAU,EAAE,MAAM,IAAI,CAAC,CAAC,GAAG,CAAC,WAAW,CAAC,KAAK,CAAC,UAAU,EAAE,UAAU,IAAI,CAAC,CAAC,CAAC,GAAG,OAAO;wBACpH,gBAAgB,EAAE,CAAC;qBACpB;iBACF,CAAC;YACJ,CAAC;QACH,CAAC;QAED,YAAY;QACZ,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC/B,MAAM,aAAa,GAAkB;YACnC,CAAC;YACD,cAAc,EAAE,IAAI;YACpB,YAAY;YACZ,IAAI;SACL,CAAC;QAEF,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,KAAK,EAAE,aAAa,CAAC,CAAC;QAC3E,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,WAAW,CAAC;QAC5C,IAAI,CAAC,KAAK,CAAC,eAAe,IAAI,UAAU,CAAC;QAEzC,aAAa;QACb,MAAM,gBAAgB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACpC,IAAI,iBAAoC,CAAC;QAEzC,IAAI,IAAI,CAAC,MAAM,CAAC,iBAAiB,EAAE,CAAC;YAClC,MAAM,MAAM,GAAwB,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;gBAC1D,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE;gBACf,OAAO,EAAE,CAAC,CAAC,OAAO,IAAI,EAAE;gBACxB,KAAK,EAAE,CAAC,CAAC,KAAK;gBACd,SAAS,EAAE,CAAC,CAAC,MAAM,CAAC,SAAS;aAC9B,CAAC,CAAC,CAAC;YAEJ,iBAAiB,GAAG,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,MAAM,EAAE,cAAc,EAAE;gBACnE,SAAS;gBACT,SAAS,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC;gBACzB,QAAQ,EAAE,QAAQ;aACnB,CAAC,CAAC;QACL,CAAC;aAAM,CAAC;YACN,eAAe;YACf,iBAAiB,GAAG;gBAClB,MAAM,EAAE,aAAa,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;oBAC1C,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE;oBACf,OAAO,EAAE,CAAC,CAAC,OAAO,IAAI,EAAE;oBACxB,KAAK,EAAE,CAAC,CAAC,KAAK;iBACf,CAAC,CAAC;gBACH,aAAa,EAAE,aAAa,CAAC,MAAM;gBACnC,eAAe,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,aAAa,CAAC,MAAM,CAAC;gBAClD,WAAW,EAAE,CAAC;gBACd,WAAW,EAAE,CAAC;gBACd,gBAAgB,EAAE,CAAC;aACpB,CAAC;QACJ,CAAC;QACD,MAAM,eAAe,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,gBAAgB,CAAC;QAEtD,qBAAqB;QACrB,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC5B,MAAM,QAAQ,GAAG,iBAAiB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;QAE9D,IAAI,aAA4B,CAAC;QAEjC,IAAI,IAAI,CAAC,MAAM,CAAC,kBAAkB,EAAE,CAAC;YACnC,aAAa,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,EAAE,QAAQ,EAAE;gBACvD,YAAY;gBACZ,UAAU,EAAE,eAAe;gBAC3B,SAAS,EAAE,IAAI;aAChB,CAAC,CAAC;QACL,CAAC;aAAM,CAAC;YACN,0BAA0B;YAC1B,aAAa,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,EAAE,QAAQ,EAAE;gBACvD,YAAY;gBACZ,UAAU,EAAE,QAAQ;gBACpB,SAAS,EAAE,IAAI;aAChB,CAAC,CAAC;QACL,CAAC;QAED,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,QAAQ,CAAC;QACtC,IAAI,CAAC,KAAK,CAAC,YAAY,IAAI,OAAO,CAAC;QACnC,IAAI,CAAC,KAAK,CAAC,SAAS,IAAI,aAAa,CAAC,UAAU,CAAC;QACjD,IAAI,CAAC,KAAK,CAAC,UAAU,IAAI,aAAa,CAAC,SAAS,CAAC;QAEjD,WAAW;QACX,IAAI,IAAI,CAAC,MAAM,CAAC,WAAW,IAAI,CAAC,SAAS,EAAE,CAAC;YAC1C,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,aAAa,CAAC,QAAQ,CAAC,OAAO,EAAE,QAAQ,EAAE;gBAC9D,cAAc;gBACd,QAAQ,EAAE,aAAa,CAAC,aAAa,CAAC,KAAK;gBAC3C,cAAc,EAAE,OAAO;gBACvB,UAAU,EAAE,aAAa,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC;oBACzC,MAAM,EAAE,aAAa,CAAC,QAAQ,CAAC,KAAK,CAAC,YAAY;oBACjD,UAAU,EAAE,aAAa,CAAC,QAAQ,CAAC,KAAK,CAAC,gBAAgB;iBAC1D,CAAC,CAAC,CAAC,SAAS;aACd,CAAC,CAAC;QACL,CAAC;QAED,WAAW;QACX,MAAM,OAAO,GAAG,iBAAiB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE;YAC/C,MAAM,QAAQ,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;YAC/D,OAAO;gBACL,EAAE,EAAE,CAAC,CAAC,EAAE;gBACR,OAAO,EAAE,CAAC,CAAC,OAAO;gBAClB,KAAK,EAAE,CAAC,CAAC,KAAK;gBACd,KAAK,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS;aACpE,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,OAAO;YACL,MAAM,EAAE,aAAa,CAAC,QAAQ,CAAC,OAAO;YACtC,OAAO;YACP,OAAO,EAAE;gBACP,YAAY,EAAE,UAAU;gBACxB,iBAAiB,EAAE,eAAe;gBAClC,SAAS,EAAE,OAAO;gBAClB,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;gBACnC,QAAQ,EAAE,KAAK;gBACf,UAAU,EAAE,aAAa,CAAC,UAAU,CAAC,KAAK;gBAC1C,KAAK,EAAE,aAAa,CAAC,aAAa,CAAC,KAAK;gBACxC,UAAU,EAAE,aAAa,CAAC,QAAQ,CAAC,KAAK,EAAE,WAAW,IAAI,aAAa,CAAC,UAAU,CAAC,eAAe;gBACjG,aAAa,EAAE,aAAa,CAAC,UAAU;gBACvC,SAAS,EAAE,aAAa,CAAC,SAAS,GAAG,CAAC,iBAAiB,CAAC,WAAW,GAAG,OAAO,CAAC;gBAC9E,gBAAgB,EAAE,iBAAiB,CAAC,gBAAgB;aACrD;SACF,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU,CAAC,KAAa;QAC5B,OAAO,IAAI,CAAC,KAAK,CAAC;YAChB,KAAK;YACL,OAAO,EAAE;gBACP,CAAC,EAAE,CAAC;gBACJ,SAAS,EAAE,IAAI;gBACf,eAAe,EAAE,QAAQ;aAC1B;SACF,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,YAAY,CAAC,KAAa,EAAE,IAAe;QAC/C,OAAO,IAAI,CAAC,KAAK,CAAC;YAChB,KAAK;YACL,OAAO,EAAE;gBACP,CAAC,EAAE,EAAE;gBACL,SAAS,EAAE,IAAI;gBACf,SAAS,EAAE,IAAI;gBACf,YAAY,EAAE,IAAI;gBAClB,IAAI;aACL;SACF,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACH,QAAQ;QACN,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC;QACzC,MAAM,WAAW,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC;QAE3C,OAAO;YACL,QAAQ,EAAE;gBACR,aAAa,EAAE,IAAI,CAAC,KAAK,CAAC,aAAa;gBACvC,YAAY,EAAE,IAAI,CAAC,KAAK,CAAC,aAAa,GAAG,CAAC;oBACxC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,aAAa;oBACjD,CAAC,CAAC,CAAC;gBACL,eAAe,EAAE,IAAI,CAAC,KAAK,CAAC,aAAa,GAAG,CAAC;oBAC3C,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,eAAe,GAAG,IAAI,CAAC,KAAK,CAAC,aAAa;oBACvD,CAAC,CAAC,CAAC;gBACL,YAAY,EAAE,IAAI,CAAC,KAAK,CAAC,aAAa,GAAG,CAAC;oBACxC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,aAAa;oBACpD,CAAC,CAAC,CAAC;gBACL,YAAY,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,KAAK,CAAC,GAAG,KAAK;gBAC9D,aAAa,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,GAAG,KAAK,CAAC,GAAG,KAAK;gBAChE,WAAW,EAAE,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,CAAC;oBACnC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC;oBACxE,CAAC,CAAC,CAAC;aACN;YACD,KAAK,EAAE,UAAU;YACjB,MAAM,EAAE,WAAW;SACpB,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,WAAW,CAAC,OAAiB;QACjC,IAAI,QAAQ,GAAG,CAAC,CAAC;QAEjB,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC;gBAC3D,QAAQ,EAAE,CAAC;YACb,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,CAAC,KAAK,CAAC,4BAA4B,KAAK,EAAE,CAAC,CAAC;YACrD,CAAC;QACH,CAAC;QAED,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED;;OAEG;IACH,UAAU;QACR,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;IACrB,CAAC;IAED;;OAEG;IACH,UAAU;QACR,IAAI,CAAC,KAAK,GAAG;YACX,aAAa,EAAE,CAAC;YAChB,SAAS,EAAE,CAAC;YACZ,SAAS,EAAE,CAAC;YACZ,UAAU,EAAE,CAAC;YACb,eAAe,EAAE,CAAC;YAClB,YAAY,EAAE,CAAC;SAChB,CAAC;QACF,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC;IAC3B,CAAC;CACF;AAED;;GAEG;AACH,MAAM,UAAU,0BAA0B,CACxC,YAA0B,EAC1B,MAAuB;IAEvB,OAAO,IAAI,oBAAoB,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;AACxD,CAAC"}

package/dist/services/embedding-provider.d.ts

@@ -0,0 +1,72 @@
+/**
+ * Embedding Provider - Xenova Transformers integration
+ *
+ * Provides semantic embeddings using Xenova/transformers.js
+ * Model: all-MiniLM-L6-v2 (384 dimensions)
+ *
+ * @module services/embedding-provider
+ */
+import type { Embedding384 } from '../types/index.js';
+/**
+ * Embedding provider interface
+ */
+export interface IEmbeddingProvider {
+    embed(text: string): Promise<Embedding384>;
+    embedBatch(texts: string[]): Promise<Embedding384[]>;
+    isReady(): boolean;
+}
+/**
+ * Xenova Embedding Provider
+ *
+ * Uses Xenova/transformers.js for high-quality semantic embeddings
+ */
+export declare class XenovaEmbeddingProvider implements IEmbeddingProvider {
+    private ready;
+    private initPromise;
+    /**
+     * Initialize the provider (lazy)
+     */
+    init(): Promise<void>;
+    /**
+     * Check if provider is ready
+     */
+    isReady(): boolean;
+    /**
+     * Generate embedding for a single text
+     */
+    embed(text: string): Promise<Embedding384>;
+    /**
+     * Generate embeddings for multiple texts (batched)
+     */
+    embedBatch(texts: string[]): Promise<Embedding384[]>;
+}
+/**
+ * Deterministic Embedding Provider (fallback)
+ *
+ * Creates pseudo-random but deterministic embeddings from text hash.
+ * Used when Xenova is not available or for testing.
+ */
+export declare class DeterministicEmbeddingProvider implements IEmbeddingProvider {
+    isReady(): boolean;
+    embed(text: string): Promise<Embedding384>;
+    embedBatch(texts: string[]): Promise<Embedding384[]>;
+}
+/**
+ * Get the Xenova embedding provider (singleton)
+ */
+export declare function getXenovaProvider(): XenovaEmbeddingProvider;
+/**
+ * Get the deterministic embedding provider (singleton)
+ */
+export declare function getDeterministicProvider(): DeterministicEmbeddingProvider;
+/**
+ * Get the best available embedding provider
+ *
+ * Tries Xenova first, falls back to deterministic if unavailable
+ */
+export declare function getEmbeddingProvider(): Promise<IEmbeddingProvider>;
+/**
+ * Quick embedding function using the best available provider
+ */
+export declare function quickEmbed(text: string): Promise<Embedding384>;
+//# sourceMappingURL=embedding-provider.d.ts.map

package/dist/services/embedding-provider.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"embedding-provider.d.ts","sourceRoot":"","sources":["../../src/services/embedding-provider.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAEtD;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC;IAC3C,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC,CAAC;IACrD,OAAO,IAAI,OAAO,CAAC;CACpB;AAyBD;;;;GAIG;AACH,qBAAa,uBAAwB,YAAW,kBAAkB;IAChE,OAAO,CAAC,KAAK,CAAkB;IAC/B,OAAO,CAAC,WAAW,CAA8B;IAEjD;;OAEG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAY3B;;OAEG;IACH,OAAO,IAAI,OAAO;IAIlB;;OAEG;IACG,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC;IAehD;;OAEG;IACG,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;CAyB3D;AAED;;;;;GAKG;AACH,qBAAa,8BAA+B,YAAW,kBAAkB;IACvE,OAAO,IAAI,OAAO;IAIZ,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC;IA6B1C,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;CAG3D;AAMD;;GAEG;AACH,wBAAgB,iBAAiB,IAAI,uBAAuB,CAK3D;AAED;;GAEG;AACH,wBAAgB,wBAAwB,IAAI,8BAA8B,CAKzE;AAED;;;;GAIG;AACH,wBAAsB,oBAAoB,IAAI,OAAO,CAAC,kBAAkB,CAAC,CASxE;AAED;;GAEG;AACH,wBAAsB,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC,CAGpE"}

package/dist/services/embedding-provider.js

@@ -0,0 +1,176 @@
+/**
+ * Embedding Provider - Xenova Transformers integration
+ *
+ * Provides semantic embeddings using Xenova/transformers.js
+ * Model: all-MiniLM-L6-v2 (384 dimensions)
+ *
+ * @module services/embedding-provider
+ */
+// Lazy-loaded pipeline
+let _pipeline = null;
+let _pipelinePromise = null;
+/**
+ * Initialize the embedding pipeline
+ */
+async function getPipeline() {
+    if (_pipeline)
+        return _pipeline;
+    if (_pipelinePromise)
+        return _pipelinePromise;
+    _pipelinePromise = (async () => {
+        const { pipeline } = await import('@xenova/transformers');
+        _pipeline = await pipeline('feature-extraction', 'Xenova/all-MiniLM-L6-v2', {
+            quantized: true,
+        });
+        return _pipeline;
+    })();
+    return _pipelinePromise;
+}
+/**
+ * Xenova Embedding Provider
+ *
+ * Uses Xenova/transformers.js for high-quality semantic embeddings
+ */
+export class XenovaEmbeddingProvider {
+    ready = false;
+    initPromise = null;
+    /**
+     * Initialize the provider (lazy)
+     */
+    async init() {
+        if (this.ready)
+            return;
+        if (this.initPromise)
+            return this.initPromise;
+        this.initPromise = (async () => {
+            await getPipeline();
+            this.ready = true;
+        })();
+        return this.initPromise;
+    }
+    /**
+     * Check if provider is ready
+     */
+    isReady() {
+        return this.ready;
+    }
+    /**
+     * Generate embedding for a single text
+     */
+    async embed(text) {
+        const pipe = await getPipeline();
+        const output = await pipe(text, { pooling: 'mean', normalize: true });
+        // Convert to Float32Array
+        const embedding = new Float32Array(384);
+        const data = output.data;
+        for (let i = 0; i < 384; i++) {
+            embedding[i] = data[i];
+        }
+        return embedding;
+    }
+    /**
+     * Generate embeddings for multiple texts (batched)
+     */
+    async embedBatch(texts) {
+        const pipe = await getPipeline();
+        // Process in batches of 32 for memory efficiency
+        const batchSize = 32;
+        const results = [];
+        for (let i = 0; i < texts.length; i += batchSize) {
+            const batch = texts.slice(i, i + batchSize);
+            const outputs = await pipe(batch, { pooling: 'mean', normalize: true });
+            for (let j = 0; j < batch.length; j++) {
+                const embedding = new Float32Array(384);
+                const startIdx = j * 384;
+                for (let k = 0; k < 384; k++) {
+                    embedding[k] = outputs.data[startIdx + k];
+                }
+                results.push(embedding);
+            }
+        }
+        return results;
+    }
+}
+/**
+ * Deterministic Embedding Provider (fallback)
+ *
+ * Creates pseudo-random but deterministic embeddings from text hash.
+ * Used when Xenova is not available or for testing.
+ */
+export class DeterministicEmbeddingProvider {
+    isReady() {
+        return true;
+    }
+    async embed(text) {
+        const embedding = new Float32Array(384);
+        // Generate deterministic hash-based embedding
+        let hash = 0;
+        for (let i = 0; i < text.length; i++) {
+            hash = ((hash << 5) - hash) + text.charCodeAt(i);
+            hash |= 0;
+        }
+        for (let i = 0; i < 384; i++) {
+            hash = ((hash << 5) - hash) + i;
+            hash |= 0;
+            embedding[i] = (hash % 1000) / 1000 - 0.5;
+        }
+        // Normalize to unit vector
+        let norm = 0;
+        for (let i = 0; i < 384; i++) {
+            norm += embedding[i] * embedding[i];
+        }
+        norm = Math.sqrt(norm);
+        for (let i = 0; i < 384; i++) {
+            embedding[i] /= norm;
+        }
+        return embedding;
+    }
+    async embedBatch(texts) {
+        return Promise.all(texts.map(t => this.embed(t)));
+    }
+}
+// Singleton instances
+let _xenovaProvider = null;
+let _deterministicProvider = null;
+/**
+ * Get the Xenova embedding provider (singleton)
+ */
+export function getXenovaProvider() {
+    if (!_xenovaProvider) {
+        _xenovaProvider = new XenovaEmbeddingProvider();
+    }
+    return _xenovaProvider;
+}
+/**
+ * Get the deterministic embedding provider (singleton)
+ */
+export function getDeterministicProvider() {
+    if (!_deterministicProvider) {
+        _deterministicProvider = new DeterministicEmbeddingProvider();
+    }
+    return _deterministicProvider;
+}
+/**
+ * Get the best available embedding provider
+ *
+ * Tries Xenova first, falls back to deterministic if unavailable
+ */
+export async function getEmbeddingProvider() {
+    try {
+        const xenova = getXenovaProvider();
+        await xenova.init();
+        return xenova;
+    }
+    catch {
+        console.warn('Xenova not available, using deterministic embeddings');
+        return getDeterministicProvider();
+    }
+}
+/**
+ * Quick embedding function using the best available provider
+ */
+export async function quickEmbed(text) {
+    const provider = await getEmbeddingProvider();
+    return provider.embed(text);
+}
+//# sourceMappingURL=embedding-provider.js.map

package/dist/services/embedding-provider.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"embedding-provider.js","sourceRoot":"","sources":["../../src/services/embedding-provider.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAaH,uBAAuB;AACvB,IAAI,SAAS,GAAQ,IAAI,CAAC;AAC1B,IAAI,gBAAgB,GAAwB,IAAI,CAAC;AAEjD;;GAEG;AACH,KAAK,UAAU,WAAW;IACxB,IAAI,SAAS;QAAE,OAAO,SAAS,CAAC;IAEhC,IAAI,gBAAgB;QAAE,OAAO,gBAAgB,CAAC;IAE9C,gBAAgB,GAAG,CAAC,KAAK,IAAI,EAAE;QAC7B,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,MAAM,CAAC,sBAAsB,CAAC,CAAC;QAC1D,SAAS,GAAG,MAAM,QAAQ,CAAC,oBAAoB,EAAE,yBAAyB,EAAE;YAC1E,SAAS,EAAE,IAAI;SAChB,CAAC,CAAC;QACH,OAAO,SAAS,CAAC;IACnB,CAAC,CAAC,EAAE,CAAC;IAEL,OAAO,gBAAgB,CAAC;AAC1B,CAAC;AAED;;;;GAIG;AACH,MAAM,OAAO,uBAAuB;IAC1B,KAAK,GAAY,KAAK,CAAC;IACvB,WAAW,GAAyB,IAAI,CAAC;IAEjD;;OAEG;IACH,KAAK,CAAC,IAAI;QACR,IAAI,IAAI,CAAC,KAAK;YAAE,OAAO;QACvB,IAAI,IAAI,CAAC,WAAW;YAAE,OAAO,IAAI,CAAC,WAAW,CAAC;QAE9C,IAAI,CAAC,WAAW,GAAG,CAAC,KAAK,IAAI,EAAE;YAC7B,MAAM,WAAW,EAAE,CAAC;YACpB,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;QACpB,CAAC,CAAC,EAAE,CAAC;QAEL,OAAO,IAAI,CAAC,WAAW,CAAC;IAC1B,CAAC;IAED;;OAEG;IACH,OAAO;QACL,OAAO,IAAI,CAAC,KAAK,CAAC;IACpB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK,CAAC,IAAY;QACtB,MAAM,IAAI,GAAG,MAAM,WAAW,EAAE,CAAC;QACjC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAEtE,0BAA0B;QAC1B,MAAM,SAAS,GAAG,IAAI,YAAY,CAAC,GAAG,CAAC,CAAC;QACxC,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC;QAEzB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;YAC7B,SAAS,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;QACzB,CAAC;QAED,OAAO,SAAS,CAAC;IACnB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU,CAAC,KAAe;QAC9B,MAAM,IAAI,GAAG,MAAM,WAAW,EAAE,CAAC;QAEjC,iDAAiD;QACjD,MAAM,SAAS,GAAG,EAAE,CAAC;QACrB,MAAM,OAAO,GAAmB,EAAE,CAAC;QAEnC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC;YACjD,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC;YAC5C,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,KAAK,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YAExE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACtC,MAAM,SAAS,GAAG,IAAI,YAAY,CAAC,GAAG,CAAC,CAAC;gBACxC,MAAM,QAAQ,GAAG,CAAC,GAAG,GAAG,CAAC;gBAEzB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;oBAC7B,SAAS,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC;gBAC5C,CAAC;gBAED,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAC1B,CAAC;QACH,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;CACF;AAED;;;;;GAKG;AACH,MAAM,OAAO,8BAA8B;IACzC,OAAO;QACL,OAAO,IAAI,CAAC;IACd,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,IAAY;QACtB,MAAM,SAAS,GAAG,IAAI,YAAY,CAAC,GAAG,CAAC,CAAC;QAExC,8CAA8C;QAC9C,IAAI,IAAI,GAAG,CAAC,CAAC;QACb,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACrC,IAAI,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;YACjD,IAAI,IAAI,CAAC,CAAC;QACZ,CAAC;QAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;YAC7B,IAAI,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC;YAChC,IAAI,IAAI,CAAC,CAAC;YACV,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC,GAAG,IAAI,GAAG,GAAG,CAAC;QAC5C,CAAC;QAED,2BAA2B;QAC3B,IAAI,IAAI,GAAG,CAAC,CAAC;QACb,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;YAC7B,IAAI,IAAI,SAAS,CAAC,CAAC,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;QACtC,CAAC;QACD,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACvB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;YAC7B,SAAS,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC;QACvB,CAAC;QAED,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,KAAe;QAC9B,OAAO,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACpD,CAAC;CACF;AAED,sBAAsB;AACtB,IAAI,eAAe,GAAmC,IAAI,CAAC;AAC3D,IAAI,sBAAsB,GAA0C,IAAI,CAAC;AAEzE;;GAEG;AACH,MAAM,UAAU,iBAAiB;IAC/B,IAAI,CAAC,eAAe,EAAE,CAAC;QACrB,eAAe,GAAG,IAAI,uBAAuB,EAAE,CAAC;IAClD,CAAC;IACD,OAAO,eAAe,CAAC;AACzB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,wBAAwB;IACtC,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAC5B,sBAAsB,GAAG,IAAI,8BAA8B,EAAE,CAAC;IAChE,CAAC;IACD,OAAO,sBAAsB,CAAC;AAChC,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,oBAAoB;IACxC,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,iBAAiB,EAAE,CAAC;QACnC,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;QACpB,OAAO,MAAM,CAAC;IAChB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,CAAC,IAAI,CAAC,sDAAsD,CAAC,CAAC;QACrE,OAAO,wBAAwB,EAAE,CAAC;IACpC,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,IAAY;IAC3C,MAAM,QAAQ,GAAG,MAAM,oBAAoB,EAAE,CAAC;IAC9C,OAAO,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;AAC9B,CAAC"}

package/dist/services/file-ingestion.d.ts

@@ -0,0 +1,72 @@
+/**
+ * File Ingestion Service
+ * Handles Excel, CSV file parsing and neuron creation
+ */
+export interface ParsedRow {
+    rowIndex: number;
+    data: Record<string, any>;
+    text: string;
+}
+export interface ParsedFile {
+    fileName: string;
+    fileType: 'xlsx' | 'csv';
+    sheetName?: string;
+    headers: string[];
+    rows: ParsedRow[];
+    totalRows: number;
+}
+export interface FileIngestionResult {
+    fileName: string;
+    totalRows: number;
+    neuronsCreated: number;
+    neurons: Array<{
+        id: string;
+        rowIndex: number;
+        preview: string;
+    }>;
+    errors: string[];
+}
+export interface FileIngestionOptions {
+    headerRow?: number;
+    startRow?: number;
+    sheetName?: string;
+    sheetIndex?: number;
+    columns?: string[];
+    excludeColumns?: string[];
+    textTemplate?: string;
+    sourceType?: string;
+    tags?: string[];
+    maxRows?: number;
+}
+export declare class FileIngestionService {
+    /**
+     * Parse Excel file buffer
+     */
+    parseExcel(buffer: Buffer, options?: FileIngestionOptions): ParsedFile;
+    /**
+     * Parse CSV file buffer
+     */
+    parseCSV(buffer: Buffer, options?: FileIngestionOptions): ParsedFile;
+    /**
+     * Generate text from row data for embedding
+     */
+    private generateText;
+    /**
+     * Detect file type from extension or content
+     */
+    detectFileType(fileName: string, buffer?: Buffer): 'xlsx' | 'csv' | 'unknown';
+    /**
+     * Parse file automatically detecting type
+     */
+    parseFile(buffer: Buffer, fileName: string, options?: FileIngestionOptions): ParsedFile;
+    /**
+     * Get sheet names from Excel file
+     */
+    getSheetNames(buffer: Buffer): string[];
+    /**
+     * Preview file (first N rows)
+     */
+    previewFile(buffer: Buffer, fileName: string, maxRows?: number): ParsedFile;
+}
+export declare const fileIngestion: FileIngestionService;
+//# sourceMappingURL=file-ingestion.d.ts.map

package/dist/services/file-ingestion.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"file-ingestion.d.ts","sourceRoot":"","sources":["../../src/services/file-ingestion.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAKH,MAAM,WAAW,SAAS;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC1B,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,UAAU;IACzB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,GAAG,KAAK,CAAC;IACzB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,IAAI,EAAE,SAAS,EAAE,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,mBAAmB;IAClC,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,cAAc,EAAE,MAAM,CAAC;IACvB,OAAO,EAAE,KAAK,CAAC;QACb,EAAE,EAAE,MAAM,CAAC;QACX,QAAQ,EAAE,MAAM,CAAC;QACjB,OAAO,EAAE,MAAM,CAAC;KACjB,CAAC,CAAC;IACH,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB;AAED,MAAM,WAAW,oBAAoB;IACnC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,qBAAa,oBAAoB;IAC/B;;OAEG;IACH,UAAU,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,GAAE,oBAAyB,GAAG,UAAU;IA6F1E;;OAEG;IACH,QAAQ,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,GAAE,oBAAyB,GAAG,UAAU;IAgFxE;;OAEG;IACH,OAAO,CAAC,YAAY;IAuBpB;;OAEG;IACH,cAAc,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,KAAK,GAAG,SAAS;IA0B7E;;OAEG;IACH,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,GAAE,oBAAyB,GAAG,UAAU;IAiB3F;;OAEG;IACH,aAAa,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE;IAKvC;;OAEG;IACH,WAAW,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,GAAE,MAAU,GAAG,UAAU;CAG/E;AAGD,eAAO,MAAM,aAAa,sBAA6B,CAAC"}