@pleaseai/context-please-core 0.1.0

package/dist/embedding/voyageai-embedding.js

@@ -0,0 +1,227 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.VoyageAIEmbedding = void 0;
+const voyageai_1 = require("voyageai");
+const base_embedding_1 = require("./base-embedding");
+class VoyageAIEmbedding extends base_embedding_1.Embedding {
+    constructor(config) {
+        super();
+        this.dimension = 1024; // Default dimension for voyage-code-3
+        this.inputType = 'document';
+        this.maxTokens = 32000; // Default max tokens
+        this.config = config;
+        this.client = new voyageai_1.VoyageAIClient({
+            apiKey: config.apiKey,
+        });
+        // Set dimension and context length based on different models
+        this.updateModelSettings(config.model || 'voyage-code-3');
+    }
+    updateModelSettings(model) {
+        const supportedModels = VoyageAIEmbedding.getSupportedModels();
+        const modelInfo = supportedModels[model];
+        if (modelInfo) {
+            // If dimension is a string (indicating variable dimension), use default value 1024
+            if (typeof modelInfo.dimension === 'string') {
+                this.dimension = 1024; // Default dimension
+            }
+            else {
+                this.dimension = modelInfo.dimension;
+            }
+            // Set max tokens based on model's context length
+            this.maxTokens = modelInfo.contextLength;
+        }
+        else {
+            // Use default dimension and context length for unknown models
+            this.dimension = 1024;
+            this.maxTokens = 32000;
+        }
+    }
+    updateDimensionForModel(model) {
+        const supportedModels = VoyageAIEmbedding.getSupportedModels();
+        const modelInfo = supportedModels[model];
+        if (modelInfo) {
+            // If dimension is a string (indicating variable dimension), use default value 1024
+            if (typeof modelInfo.dimension === 'string') {
+                this.dimension = 1024; // Default dimension
+            }
+            else {
+                this.dimension = modelInfo.dimension;
+            }
+        }
+        else {
+            // Use default dimension for unknown models
+            this.dimension = 1024;
+        }
+    }
+    async detectDimension() {
+        // VoyageAI doesn't need dynamic detection, return configured dimension
+        return this.dimension;
+    }
+    async embed(text) {
+        const processedText = this.preprocessText(text);
+        const model = this.config.model || 'voyage-code-3';
+        const response = await this.client.embed({
+            input: processedText,
+            model: model,
+            inputType: this.inputType,
+        });
+        if (!response.data || !response.data[0] || !response.data[0].embedding) {
+            throw new Error('VoyageAI API returned invalid response');
+        }
+        return {
+            vector: response.data[0].embedding,
+            dimension: this.dimension
+        };
+    }
+    async embedBatch(texts) {
+        const processedTexts = this.preprocessTexts(texts);
+        const model = this.config.model || 'voyage-code-3';
+        const response = await this.client.embed({
+            input: processedTexts,
+            model: model,
+            inputType: this.inputType,
+        });
+        if (!response.data) {
+            throw new Error('VoyageAI API returned invalid response');
+        }
+        return response.data.map((item) => {
+            if (!item.embedding) {
+                throw new Error('VoyageAI API returned invalid embedding data');
+            }
+            return {
+                vector: item.embedding,
+                dimension: this.dimension
+            };
+        });
+    }
+    getDimension() {
+        return this.dimension;
+    }
+    getProvider() {
+        return 'VoyageAI';
+    }
+    /**
+     * Set model type
+     * @param model Model name
+     */
+    setModel(model) {
+        this.config.model = model;
+        this.updateModelSettings(model);
+    }
+    /**
+     * Set input type (VoyageAI specific feature)
+     * @param inputType Input type: 'document' | 'query'
+     */
+    setInputType(inputType) {
+        this.inputType = inputType;
+    }
+    /**
+     * Get client instance (for advanced usage)
+     */
+    getClient() {
+        return this.client;
+    }
+    /**
+     * Get list of supported models
+     */
+    static getSupportedModels() {
+        return {
+            // Latest recommended models
+            'voyage-3-large': {
+                dimension: '1024 (default), 256, 512, 2048',
+                contextLength: 32000,
+                description: 'The best general-purpose and multilingual retrieval quality'
+            },
+            'voyage-3.5': {
+                dimension: '1024 (default), 256, 512, 2048',
+                contextLength: 32000,
+                description: 'Optimized for general-purpose and multilingual retrieval quality'
+            },
+            'voyage-3.5-lite': {
+                dimension: '1024 (default), 256, 512, 2048',
+                contextLength: 32000,
+                description: 'Optimized for latency and cost'
+            },
+            'voyage-code-3': {
+                dimension: '1024 (default), 256, 512, 2048',
+                contextLength: 32000,
+                description: 'Optimized for code retrieval (recommended for code)'
+            },
+            // Professional domain models
+            'voyage-finance-2': {
+                dimension: 1024,
+                contextLength: 32000,
+                description: 'Optimized for finance retrieval and RAG'
+            },
+            'voyage-law-2': {
+                dimension: 1024,
+                contextLength: 16000,
+                description: 'Optimized for legal retrieval and RAG'
+            },
+            'voyage-multilingual-2': {
+                dimension: 1024,
+                contextLength: 32000,
+                description: 'Legacy: Use voyage-3.5 for multilingual tasks'
+            },
+            'voyage-large-2-instruct': {
+                dimension: 1024,
+                contextLength: 16000,
+                description: 'Legacy: Use voyage-3.5 instead'
+            },
+            // Legacy models
+            'voyage-large-2': {
+                dimension: 1536,
+                contextLength: 16000,
+                description: 'Legacy: Use voyage-3.5 instead'
+            },
+            'voyage-code-2': {
+                dimension: 1536,
+                contextLength: 16000,
+                description: 'Previous generation of code embeddings'
+            },
+            'voyage-3': {
+                dimension: 1024,
+                contextLength: 32000,
+                description: 'Legacy: Use voyage-3.5 instead'
+            },
+            'voyage-3-lite': {
+                dimension: 512,
+                contextLength: 32000,
+                description: 'Legacy: Use voyage-3.5-lite instead'
+            },
+            'voyage-2': {
+                dimension: 1024,
+                contextLength: 4000,
+                description: 'Legacy: Use voyage-3.5-lite instead'
+            },
+            // Other legacy models
+            'voyage-02': {
+                dimension: 1024,
+                contextLength: 4000,
+                description: 'Legacy model'
+            },
+            'voyage-01': {
+                dimension: 1024,
+                contextLength: 4000,
+                description: 'Legacy model'
+            },
+            'voyage-lite-01': {
+                dimension: 1024,
+                contextLength: 4000,
+                description: 'Legacy model'
+            },
+            'voyage-lite-01-instruct': {
+                dimension: 1024,
+                contextLength: 4000,
+                description: 'Legacy model'
+            },
+            'voyage-lite-02-instruct': {
+                dimension: 1024,
+                contextLength: 4000,
+                description: 'Legacy model'
+            }
+        };
+    }
+}
+exports.VoyageAIEmbedding = VoyageAIEmbedding;
+//# sourceMappingURL=voyageai-embedding.js.map

package/dist/embedding/voyageai-embedding.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"voyageai-embedding.js","sourceRoot":"","sources":["../../src/embedding/voyageai-embedding.ts"],"names":[],"mappings":";;;AAAA,uCAA0C;AAC1C,qDAA8D;AAO9D,MAAa,iBAAkB,SAAQ,0BAAS;IAO5C,YAAY,MAA+B;QACvC,KAAK,EAAE,CAAC;QALJ,cAAS,GAAW,IAAI,CAAC,CAAC,sCAAsC;QAChE,cAAS,GAAyB,UAAU,CAAC;QAC3C,cAAS,GAAW,KAAK,CAAC,CAAC,qBAAqB;QAItD,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,MAAM,GAAG,IAAI,yBAAc,CAAC;YAC7B,MAAM,EAAE,MAAM,CAAC,MAAM;SACxB,CAAC,CAAC;QAEH,6DAA6D;QAC7D,IAAI,CAAC,mBAAmB,CAAC,MAAM,CAAC,KAAK,IAAI,eAAe,CAAC,CAAC;IAC9D,CAAC;IAEO,mBAAmB,CAAC,KAAa;QACrC,MAAM,eAAe,GAAG,iBAAiB,CAAC,kBAAkB,EAAE,CAAC;QAC/D,MAAM,SAAS,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;QAEzC,IAAI,SAAS,EAAE,CAAC;YACZ,mFAAmF;YACnF,IAAI,OAAO,SAAS,CAAC,SAAS,KAAK,QAAQ,EAAE,CAAC;gBAC1C,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,CAAC,oBAAoB;YAC/C,CAAC;iBAAM,CAAC;gBACJ,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC,SAAS,CAAC;YACzC,CAAC;YACD,iDAAiD;YACjD,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC,aAAa,CAAC;QAC7C,CAAC;aAAM,CAAC;YACJ,8DAA8D;YAC9D,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;YACtB,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;QAC3B,CAAC;IACL,CAAC;IAEO,uBAAuB,CAAC,KAAa;QACzC,MAAM,eAAe,GAAG,iBAAiB,CAAC,kBAAkB,EAAE,CAAC;QAC/D,MAAM,SAAS,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;QAEzC,IAAI,SAAS,EAAE,CAAC;YACZ,mFAAmF;YACnF,IAAI,OAAO,SAAS,CAAC,SAAS,KAAK,QAAQ,EAAE,CAAC;gBAC1C,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,CAAC,oBAAoB;YAC/C,CAAC;iBAAM,CAAC;gBACJ,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC,SAAS,CAAC;YACzC,CAAC;QACL,CAAC;aAAM,CAAC;YACJ,2CAA2C;YAC3C,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QAC1B,CAAC;IACL,CAAC;IAED,KAAK,CAAC,eAAe;QACjB,uEAAuE;QACvE,OAAO,IAAI,CAAC,SAAS,CAAC;IAC1B,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,IAAY;QACpB,MAAM,aAAa,GAAG,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;QAChD,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,IAAI,eAAe,CAAC;QAEnD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC;YACrC,KAAK,EAAE,aAAa;YACpB,KAAK,EAAE,KAAK;YACZ,SAAS,EAAE,IAAI,CAAC,SAAS;SAC5B,CAAC,CAAC;QAEH,IAAI,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC;YACrE,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;QAC9D,CAAC;QAED,OAAO;YACH,MAAM,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS;YAClC,SAAS,EAAE,IAAI,CAAC,SAAS;SAC5B,CAAC;IACN,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,KAAe;QAC5B,MAAM,cAAc,GAAG,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;QACnD,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,IAAI,eAAe,CAAC;QAEnD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC;YACrC,KAAK,EAAE,cAAc;YACrB,KAAK,EAAE,KAAK;YACZ,SAAS,EAAE,IAAI,CAAC,SAAS;SAC5B,CAAC,CAAC;QAEH,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;QAC9D,CAAC;QAED,OAAO,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;YAC9B,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;gBAClB,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;YACpE,CAAC;YACD,OAAO;gBACH,MAAM,EAAE,IAAI,CAAC,SAAS;gBACtB,SAAS,EAAE,IAAI,CAAC,SAAS;aAC5B,CAAC;QACN,CAAC,CAAC,CAAC;IACP,CAAC;IAED,YAAY;QACR,OAAO,IAAI,CAAC,SAAS,CAAC;IAC1B,CAAC;IAED,WAAW;QACP,OAAO,UAAU,CAAC;IACtB,CAAC;IAED;;;OAGG;IACH,QAAQ,CAAC,KAAa;QAClB,IAAI,CAAC,MAAM,CAAC,KAAK,GAAG,KAAK,CAAC;QAC1B,IAAI,CAAC,mBAAmB,CAAC,KAAK,CAAC,CAAC;IACpC,CAAC;IAED;;;OAGG;IACH,YAAY,CAAC,SAA+B;QACxC,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;IAC/B,CAAC;IAED;;OAEG;IACH,SAAS;QACL,OAAO,IAAI,CAAC,MAAM,CAAC;IACvB,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,kBAAkB;QACrB,OAAO;YACH,4BAA4B;YAC5B,gBAAgB,EAAE;gBACd,SAAS,EAAE,gCAAgC;gBAC3C,aAAa,EAAE,KAAK;gBACpB,WAAW,EAAE,6DAA6D;aAC7E;YACD,YAAY,EAAE;gBACV,SAAS,EAAE,gCAAgC;gBAC3C,aAAa,EAAE,KAAK;gBACpB,WAAW,EAAE,kEAAkE;aAClF;YACD,iBAAiB,EAAE;gBACf,SAAS,EAAE,gCAAgC;gBAC3C,aAAa,EAAE,KAAK;gBACpB,WAAW,EAAE,gCAAgC;aAChD;YACD,eAAe,EAAE;gBACb,SAAS,EAAE,gCAAgC;gBAC3C,aAAa,EAAE,KAAK;gBACpB,WAAW,EAAE,qDAAqD;aACrE;YACD,6BAA6B;YAC7B,kBAAkB,EAAE;gBAChB,SAAS,EAAE,IAAI;gBACf,aAAa,EAAE,KAAK;gBACpB,WAAW,EAAE,yCAAyC;aACzD;YACD,cAAc,EAAE;gBACZ,SAAS,EAAE,IAAI;gBACf,aAAa,EAAE,KAAK;gBACpB,WAAW,EAAE,uCAAuC;aACvD;YACD,uBAAuB,EAAE;gBACrB,SAAS,EAAE,IAAI;gBACf,aAAa,EAAE,KAAK;gBACpB,WAAW,EAAE,+CAA+C;aAC/D;YACD,yBAAyB,EAAE;gBACvB,SAAS,EAAE,IAAI;gBACf,aAAa,EAAE,KAAK;gBACpB,WAAW,EAAE,gCAAgC;aAChD;YACD,gBAAgB;YAChB,gBAAgB,EAAE;gBACd,SAAS,EAAE,IAAI;gBACf,aAAa,EAAE,KAAK;gBACpB,WAAW,EAAE,gCAAgC;aAChD;YACD,eAAe,EAAE;gBACb,SAAS,EAAE,IAAI;gBACf,aAAa,EAAE,KAAK;gBACpB,WAAW,EAAE,wCAAwC;aACxD;YACD,UAAU,EAAE;gBACR,SAAS,EAAE,IAAI;gBACf,aAAa,EAAE,KAAK;gBACpB,WAAW,EAAE,gCAAgC;aAChD;YACD,eAAe,EAAE;gBACb,SAAS,EAAE,GAAG;gBACd,aAAa,EAAE,KAAK;gBACpB,WAAW,EAAE,qCAAqC;aACrD;YACD,UAAU,EAAE;gBACR,SAAS,EAAE,IAAI;gBACf,aAAa,EAAE,IAAI;gBACnB,WAAW,EAAE,qCAAqC;aACrD;YACD,sBAAsB;YACtB,WAAW,EAAE;gBACT,SAAS,EAAE,IAAI;gBACf,aAAa,EAAE,IAAI;gBACnB,WAAW,EAAE,cAAc;aAC9B;YACD,WAAW,EAAE;gBACT,SAAS,EAAE,IAAI;gBACf,aAAa,EAAE,IAAI;gBACnB,WAAW,EAAE,cAAc;aAC9B;YACD,gBAAgB,EAAE;gBACd,SAAS,EAAE,IAAI;gBACf,aAAa,EAAE,IAAI;gBACnB,WAAW,EAAE,cAAc;aAC9B;YACD,yBAAyB,EAAE;gBACvB,SAAS,EAAE,IAAI;gBACf,aAAa,EAAE,IAAI;gBACnB,WAAW,EAAE,cAAc;aAC9B;YACD,yBAAyB,EAAE;gBACvB,SAAS,EAAE,IAAI;gBACf,aAAa,EAAE,IAAI;gBACnB,WAAW,EAAE,cAAc;aAC9B;SACJ,CAAC;IACN,CAAC;CACJ;AA9OD,8CA8OC"}

package/dist/index.d.ts

@@ -0,0 +1,8 @@
+export * from './splitter';
+export * from './embedding';
+export * from './vectordb';
+export * from './types';
+export * from './context';
+export * from './sync/synchronizer';
+export * from './utils';
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,YAAY,CAAC;AAC3B,cAAc,aAAa,CAAC;AAC5B,cAAc,YAAY,CAAC;AAC3B,cAAc,SAAS,CAAC;AACxB,cAAc,WAAW,CAAC;AAC1B,cAAc,qBAAqB,CAAC;AACpC,cAAc,SAAS,CAAC"}

package/dist/index.js

@@ -0,0 +1,24 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+__exportStar(require("./splitter"), exports);
+__exportStar(require("./embedding"), exports);
+__exportStar(require("./vectordb"), exports);
+__exportStar(require("./types"), exports);
+__exportStar(require("./context"), exports);
+__exportStar(require("./sync/synchronizer"), exports);
+__exportStar(require("./utils"), exports);
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,6CAA2B;AAC3B,8CAA4B;AAC5B,6CAA2B;AAC3B,0CAAwB;AACxB,4CAA0B;AAC1B,sDAAoC;AACpC,0CAAwB"}

package/dist/splitter/ast-splitter.d.ts

@@ -0,0 +1,22 @@
+import { Splitter, CodeChunk } from './index';
+export declare class AstCodeSplitter implements Splitter {
+    private chunkSize;
+    private chunkOverlap;
+    private parser;
+    private langchainFallback;
+    constructor(chunkSize?: number, chunkOverlap?: number);
+    split(code: string, language: string, filePath?: string): Promise<CodeChunk[]>;
+    setChunkSize(chunkSize: number): void;
+    setChunkOverlap(chunkOverlap: number): void;
+    private getLanguageConfig;
+    private extractChunks;
+    private refineChunks;
+    private splitLargeChunk;
+    private addOverlap;
+    private getLineCount;
+    /**
+     * Check if AST splitting is supported for the given language
+     */
+    static isLanguageSupported(language: string): boolean;
+}
+//# sourceMappingURL=ast-splitter.d.ts.map

package/dist/splitter/ast-splitter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ast-splitter.d.ts","sourceRoot":"","sources":["../../src/splitter/ast-splitter.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AA0B9C,qBAAa,eAAgB,YAAW,QAAQ;IAC5C,OAAO,CAAC,SAAS,CAAgB;IACjC,OAAO,CAAC,YAAY,CAAe;IACnC,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,iBAAiB,CAAM;gBAEnB,SAAS,CAAC,EAAE,MAAM,EAAE,YAAY,CAAC,EAAE,MAAM;IAU/C,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC;IAgCpF,YAAY,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI;IAKrC,eAAe,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI;IAK3C,OAAO,CAAC,iBAAiB;IAuBzB,OAAO,CAAC,aAAa;YAuDP,YAAY;IAgB1B,OAAO,CAAC,eAAe;IAgDvB,OAAO,CAAC,UAAU;IA4BlB,OAAO,CAAC,YAAY;IAIpB;;OAEG;IACH,MAAM,CAAC,mBAAmB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO;CAOxD"}

package/dist/splitter/ast-splitter.js

@@ -0,0 +1,234 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.AstCodeSplitter = void 0;
+const tree_sitter_1 = __importDefault(require("tree-sitter"));
+// Language parsers
+const JavaScript = require('tree-sitter-javascript');
+const TypeScript = require('tree-sitter-typescript').typescript;
+const Python = require('tree-sitter-python');
+const Java = require('tree-sitter-java');
+const Cpp = require('tree-sitter-cpp');
+const Go = require('tree-sitter-go');
+const Rust = require('tree-sitter-rust');
+const CSharp = require('tree-sitter-c-sharp');
+const Scala = require('tree-sitter-scala');
+// Node types that represent logical code units
+const SPLITTABLE_NODE_TYPES = {
+    javascript: ['function_declaration', 'arrow_function', 'class_declaration', 'method_definition', 'export_statement'],
+    typescript: ['function_declaration', 'arrow_function', 'class_declaration', 'method_definition', 'export_statement', 'interface_declaration', 'type_alias_declaration'],
+    python: ['function_definition', 'class_definition', 'decorated_definition', 'async_function_definition'],
+    java: ['method_declaration', 'class_declaration', 'interface_declaration', 'constructor_declaration'],
+    cpp: ['function_definition', 'class_specifier', 'namespace_definition', 'declaration'],
+    go: ['function_declaration', 'method_declaration', 'type_declaration', 'var_declaration', 'const_declaration'],
+    rust: ['function_item', 'impl_item', 'struct_item', 'enum_item', 'trait_item', 'mod_item'],
+    csharp: ['method_declaration', 'class_declaration', 'interface_declaration', 'struct_declaration', 'enum_declaration'],
+    scala: ['method_declaration', 'class_declaration', 'interface_declaration', 'constructor_declaration']
+};
+class AstCodeSplitter {
+    constructor(chunkSize, chunkOverlap) {
+        this.chunkSize = 2500;
+        this.chunkOverlap = 300;
+        if (chunkSize)
+            this.chunkSize = chunkSize;
+        if (chunkOverlap)
+            this.chunkOverlap = chunkOverlap;
+        this.parser = new tree_sitter_1.default();
+        // Initialize fallback splitter
+        const { LangChainCodeSplitter } = require('./langchain-splitter');
+        this.langchainFallback = new LangChainCodeSplitter(chunkSize, chunkOverlap);
+    }
+    async split(code, language, filePath) {
+        // Check if language is supported by AST splitter
+        const langConfig = this.getLanguageConfig(language);
+        if (!langConfig) {
+            console.log(`📝 Language ${language} not supported by AST, using LangChain splitter for: ${filePath || 'unknown'}`);
+            return await this.langchainFallback.split(code, language, filePath);
+        }
+        try {
+            console.log(`🌳 Using AST splitter for ${language} file: ${filePath || 'unknown'}`);
+            this.parser.setLanguage(langConfig.parser);
+            const tree = this.parser.parse(code);
+            if (!tree.rootNode) {
+                console.warn(`[ASTSplitter] ⚠️  Failed to parse AST for ${language}, falling back to LangChain: ${filePath || 'unknown'}`);
+                return await this.langchainFallback.split(code, language, filePath);
+            }
+            // Extract chunks based on AST nodes
+            const chunks = this.extractChunks(tree.rootNode, code, langConfig.nodeTypes, language, filePath);
+            // If chunks are too large, split them further
+            const refinedChunks = await this.refineChunks(chunks, code);
+            return refinedChunks;
+        }
+        catch (error) {
+            console.warn(`[ASTSplitter] ⚠️  AST splitter failed for ${language}, falling back to LangChain: ${error}`);
+            return await this.langchainFallback.split(code, language, filePath);
+        }
+    }
+    setChunkSize(chunkSize) {
+        this.chunkSize = chunkSize;
+        this.langchainFallback.setChunkSize(chunkSize);
+    }
+    setChunkOverlap(chunkOverlap) {
+        this.chunkOverlap = chunkOverlap;
+        this.langchainFallback.setChunkOverlap(chunkOverlap);
+    }
+    getLanguageConfig(language) {
+        const langMap = {
+            'javascript': { parser: JavaScript, nodeTypes: SPLITTABLE_NODE_TYPES.javascript },
+            'js': { parser: JavaScript, nodeTypes: SPLITTABLE_NODE_TYPES.javascript },
+            'typescript': { parser: TypeScript, nodeTypes: SPLITTABLE_NODE_TYPES.typescript },
+            'ts': { parser: TypeScript, nodeTypes: SPLITTABLE_NODE_TYPES.typescript },
+            'python': { parser: Python, nodeTypes: SPLITTABLE_NODE_TYPES.python },
+            'py': { parser: Python, nodeTypes: SPLITTABLE_NODE_TYPES.python },
+            'java': { parser: Java, nodeTypes: SPLITTABLE_NODE_TYPES.java },
+            'cpp': { parser: Cpp, nodeTypes: SPLITTABLE_NODE_TYPES.cpp },
+            'c++': { parser: Cpp, nodeTypes: SPLITTABLE_NODE_TYPES.cpp },
+            'c': { parser: Cpp, nodeTypes: SPLITTABLE_NODE_TYPES.cpp },
+            'go': { parser: Go, nodeTypes: SPLITTABLE_NODE_TYPES.go },
+            'rust': { parser: Rust, nodeTypes: SPLITTABLE_NODE_TYPES.rust },
+            'rs': { parser: Rust, nodeTypes: SPLITTABLE_NODE_TYPES.rust },
+            'cs': { parser: CSharp, nodeTypes: SPLITTABLE_NODE_TYPES.csharp },
+            'csharp': { parser: CSharp, nodeTypes: SPLITTABLE_NODE_TYPES.csharp },
+            'scala': { parser: Scala, nodeTypes: SPLITTABLE_NODE_TYPES.scala }
+        };
+        return langMap[language.toLowerCase()] || null;
+    }
+    extractChunks(node, code, splittableTypes, language, filePath) {
+        const chunks = [];
+        const codeLines = code.split('\n');
+        const traverse = (currentNode) => {
+            // Check if this node type should be split into a chunk
+            if (splittableTypes.includes(currentNode.type)) {
+                const startLine = currentNode.startPosition.row + 1;
+                const endLine = currentNode.endPosition.row + 1;
+                const nodeText = code.slice(currentNode.startIndex, currentNode.endIndex);
+                // Only create chunk if it has meaningful content
+                if (nodeText.trim().length > 0) {
+                    chunks.push({
+                        content: nodeText,
+                        metadata: {
+                            startLine,
+                            endLine,
+                            language,
+                            filePath,
+                        }
+                    });
+                }
+            }
+            // Continue traversing child nodes
+            for (const child of currentNode.children) {
+                traverse(child);
+            }
+        };
+        traverse(node);
+        // If no meaningful chunks found, create a single chunk with the entire code
+        if (chunks.length === 0) {
+            chunks.push({
+                content: code,
+                metadata: {
+                    startLine: 1,
+                    endLine: codeLines.length,
+                    language,
+                    filePath,
+                }
+            });
+        }
+        return chunks;
+    }
+    async refineChunks(chunks, originalCode) {
+        const refinedChunks = [];
+        for (const chunk of chunks) {
+            if (chunk.content.length <= this.chunkSize) {
+                refinedChunks.push(chunk);
+            }
+            else {
+                // Split large chunks using character-based splitting
+                const subChunks = this.splitLargeChunk(chunk, originalCode);
+                refinedChunks.push(...subChunks);
+            }
+        }
+        return this.addOverlap(refinedChunks);
+    }
+    splitLargeChunk(chunk, originalCode) {
+        const lines = chunk.content.split('\n');
+        const subChunks = [];
+        let currentChunk = '';
+        let currentStartLine = chunk.metadata.startLine;
+        let currentLineCount = 0;
+        for (let i = 0; i < lines.length; i++) {
+            const line = lines[i];
+            const lineWithNewline = i === lines.length - 1 ? line : line + '\n';
+            if (currentChunk.length + lineWithNewline.length > this.chunkSize && currentChunk.length > 0) {
+                // Create a sub-chunk
+                subChunks.push({
+                    content: currentChunk.trim(),
+                    metadata: {
+                        startLine: currentStartLine,
+                        endLine: currentStartLine + currentLineCount - 1,
+                        language: chunk.metadata.language,
+                        filePath: chunk.metadata.filePath,
+                    }
+                });
+                currentChunk = lineWithNewline;
+                currentStartLine = chunk.metadata.startLine + i;
+                currentLineCount = 1;
+            }
+            else {
+                currentChunk += lineWithNewline;
+                currentLineCount++;
+            }
+        }
+        // Add the last sub-chunk
+        if (currentChunk.trim().length > 0) {
+            subChunks.push({
+                content: currentChunk.trim(),
+                metadata: {
+                    startLine: currentStartLine,
+                    endLine: currentStartLine + currentLineCount - 1,
+                    language: chunk.metadata.language,
+                    filePath: chunk.metadata.filePath,
+                }
+            });
+        }
+        return subChunks;
+    }
+    addOverlap(chunks) {
+        if (chunks.length <= 1 || this.chunkOverlap <= 0) {
+            return chunks;
+        }
+        const overlappedChunks = [];
+        for (let i = 0; i < chunks.length; i++) {
+            let content = chunks[i].content;
+            const metadata = { ...chunks[i].metadata };
+            // Add overlap from previous chunk
+            if (i > 0 && this.chunkOverlap > 0) {
+                const prevChunk = chunks[i - 1];
+                const overlapText = prevChunk.content.slice(-this.chunkOverlap);
+                content = overlapText + '\n' + content;
+                metadata.startLine = Math.max(1, metadata.startLine - this.getLineCount(overlapText));
+            }
+            overlappedChunks.push({
+                content,
+                metadata
+            });
+        }
+        return overlappedChunks;
+    }
+    getLineCount(text) {
+        return text.split('\n').length;
+    }
+    /**
+     * Check if AST splitting is supported for the given language
+     */
+    static isLanguageSupported(language) {
+        const supportedLanguages = [
+            'javascript', 'js', 'typescript', 'ts', 'python', 'py',
+            'java', 'cpp', 'c++', 'c', 'go', 'rust', 'rs', 'cs', 'csharp', 'scala'
+        ];
+        return supportedLanguages.includes(language.toLowerCase());
+    }
+}
+exports.AstCodeSplitter = AstCodeSplitter;
+//# sourceMappingURL=ast-splitter.js.map

package/dist/splitter/ast-splitter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ast-splitter.js","sourceRoot":"","sources":["../../src/splitter/ast-splitter.ts"],"names":[],"mappings":";;;;;;AAAA,8DAAiC;AAGjC,mBAAmB;AACnB,MAAM,UAAU,GAAG,OAAO,CAAC,wBAAwB,CAAC,CAAC;AACrD,MAAM,UAAU,GAAG,OAAO,CAAC,wBAAwB,CAAC,CAAC,UAAU,CAAC;AAChE,MAAM,MAAM,GAAG,OAAO,CAAC,oBAAoB,CAAC,CAAC;AAC7C,MAAM,IAAI,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAAC;AACzC,MAAM,GAAG,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAAC;AACvC,MAAM,EAAE,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAAC;AACrC,MAAM,IAAI,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAAC;AACzC,MAAM,MAAM,GAAG,OAAO,CAAC,qBAAqB,CAAC,CAAC;AAC9C,MAAM,KAAK,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAAC;AAE3C,+CAA+C;AAC/C,MAAM,qBAAqB,GAAG;IAC1B,UAAU,EAAE,CAAC,sBAAsB,EAAE,gBAAgB,EAAE,mBAAmB,EAAE,mBAAmB,EAAE,kBAAkB,CAAC;IACpH,UAAU,EAAE,CAAC,sBAAsB,EAAE,gBAAgB,EAAE,mBAAmB,EAAE,mBAAmB,EAAE,kBAAkB,EAAE,uBAAuB,EAAE,wBAAwB,CAAC;IACvK,MAAM,EAAE,CAAC,qBAAqB,EAAE,kBAAkB,EAAE,sBAAsB,EAAE,2BAA2B,CAAC;IACxG,IAAI,EAAE,CAAC,oBAAoB,EAAE,mBAAmB,EAAE,uBAAuB,EAAE,yBAAyB,CAAC;IACrG,GAAG,EAAE,CAAC,qBAAqB,EAAE,iBAAiB,EAAE,sBAAsB,EAAE,aAAa,CAAC;IACtF,EAAE,EAAE,CAAC,sBAAsB,EAAE,oBAAoB,EAAE,kBAAkB,EAAE,iBAAiB,EAAE,mBAAmB,CAAC;IAC9G,IAAI,EAAE,CAAC,eAAe,EAAE,WAAW,EAAE,aAAa,EAAE,WAAW,EAAE,YAAY,EAAE,UAAU,CAAC;IAC1F,MAAM,EAAE,CAAC,oBAAoB,EAAE,mBAAmB,EAAE,uBAAuB,EAAE,oBAAoB,EAAE,kBAAkB,CAAC;IACtH,KAAK,EAAE,CAAC,oBAAoB,EAAE,mBAAmB,EAAE,uBAAuB,EAAE,yBAAyB,CAAC;CACzG,CAAC;AAEF,MAAa,eAAe;IAMxB,YAAY,SAAkB,EAAE,YAAqB;QAL7C,cAAS,GAAW,IAAI,CAAC;QACzB,iBAAY,GAAW,GAAG,CAAC;QAK/B,IAAI,SAAS;YAAE,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC1C,IAAI,YAAY;YAAE,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;QACnD,IAAI,CAAC,MAAM,GAAG,IAAI,qBAAM,EAAE,CAAC;QAE3B,+BAA+B;QAC/B,MAAM,EAAE,qBAAqB,EAAE,GAAG,OAAO,CAAC,sBAAsB,CAAC,CAAC;QAClE,IAAI,CAAC,iBAAiB,GAAG,IAAI,qBAAqB,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC;IAChF,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,IAAY,EAAE,QAAgB,EAAE,QAAiB;QACzD,iDAAiD;QACjD,MAAM,UAAU,GAAG,IAAI,CAAC,iBAAiB,CAAC,QAAQ,CAAC,CAAC;QACpD,IAAI,CAAC,UAAU,EAAE,CAAC;YACd,OAAO,CAAC,GAAG,CAAC,eAAe,QAAQ,wDAAwD,QAAQ,IAAI,SAAS,EAAE,CAAC,CAAC;YACpH,OAAO,MAAM,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC,IAAI,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC;QACxE,CAAC;QAED,IAAI,CAAC;YACD,OAAO,CAAC,GAAG,CAAC,6BAA6B,QAAQ,UAAU,QAAQ,IAAI,SAAS,EAAE,CAAC,CAAC;YAEpF,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;YAC3C,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAErC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACjB,OAAO,CAAC,IAAI,CAAC,6CAA6C,QAAQ,gCAAgC,QAAQ,IAAI,SAAS,EAAE,CAAC,CAAC;gBAC3H,OAAO,MAAM,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC,IAAI,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC;YACxE,CAAC;YAED,oCAAoC;YACpC,MAAM,MAAM,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,EAAE,UAAU,CAAC,SAAS,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC;YAEjG,8CAA8C;YAC9C,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;YAE5D,OAAO,aAAa,CAAC;QACzB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,OAAO,CAAC,IAAI,CAAC,6CAA6C,QAAQ,gCAAgC,KAAK,EAAE,CAAC,CAAC;YAC3G,OAAO,MAAM,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC,IAAI,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC;QACxE,CAAC;IACL,CAAC;IAED,YAAY,CAAC,SAAiB;QAC1B,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,iBAAiB,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC;IACnD,CAAC;IAED,eAAe,CAAC,YAAoB;QAChC,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;QACjC,IAAI,CAAC,iBAAiB,CAAC,eAAe,CAAC,YAAY,CAAC,CAAC;IACzD,CAAC;IAEO,iBAAiB,CAAC,QAAgB;QACtC,MAAM,OAAO,GAAyD;YAClE,YAAY,EAAE,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,qBAAqB,CAAC,UAAU,EAAE;YACjF,IAAI,EAAE,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,qBAAqB,CAAC,UAAU,EAAE;YACzE,YAAY,EAAE,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,qBAAqB,CAAC,UAAU,EAAE;YACjF,IAAI,EAAE,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,qBAAqB,CAAC,UAAU,EAAE;YACzE,QAAQ,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,qBAAqB,CAAC,MAAM,EAAE;YACrE,IAAI,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,qBAAqB,CAAC,MAAM,EAAE;YACjE,MAAM,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,qBAAqB,CAAC,IAAI,EAAE;YAC/D,KAAK,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,SAAS,EAAE,qBAAqB,CAAC,GAAG,EAAE;YAC5D,KAAK,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,SAAS,EAAE,qBAAqB,CAAC,GAAG,EAAE;YAC5D,GAAG,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,SAAS,EAAE,qBAAqB,CAAC,GAAG,EAAE;YAC1D,IAAI,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,SAAS,EAAE,qBAAqB,CAAC,EAAE,EAAE;YACzD,MAAM,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,qBAAqB,CAAC,IAAI,EAAE;YAC/D,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,qBAAqB,CAAC,IAAI,EAAE;YAC7D,IAAI,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,qBAAqB,CAAC,MAAM,EAAE;YACjE,QAAQ,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,qBAAqB,CAAC,MAAM,EAAE;YACrE,OAAO,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,qBAAqB,CAAC,KAAK,EAAE;SACrE,CAAC;QAEF,OAAO,OAAO,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC,IAAI,IAAI,CAAC;IACnD,CAAC;IAEO,aAAa,CACjB,IAAuB,EACvB,IAAY,EACZ,eAAyB,EACzB,QAAgB,EAChB,QAAiB;QAEjB,MAAM,MAAM,GAAgB,EAAE,CAAC;QAC/B,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAEnC,MAAM,QAAQ,GAAG,CAAC,WAA8B,EAAE,EAAE;YAChD,uDAAuD;YACvD,IAAI,eAAe,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC7C,MAAM,SAAS,GAAG,WAAW,CAAC,aAAa,CAAC,GAAG,GAAG,CAAC,CAAC;gBACpD,MAAM,OAAO,GAAG,WAAW,CAAC,WAAW,CAAC,GAAG,GAAG,CAAC,CAAC;gBAChD,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,UAAU,EAAE,WAAW,CAAC,QAAQ,CAAC,CAAC;gBAE1E,iDAAiD;gBACjD,IAAI,QAAQ,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAC7B,MAAM,CAAC,IAAI,CAAC;wBACR,OAAO,EAAE,QAAQ;wBACjB,QAAQ,EAAE;4BACN,SAAS;4BACT,OAAO;4BACP,QAAQ;4BACR,QAAQ;yBACX;qBACJ,CAAC,CAAC;gBACP,CAAC;YACL,CAAC;YAED,kCAAkC;YAClC,KAAK,MAAM,KAAK,IAAI,WAAW,CAAC,QAAQ,EAAE,CAAC;gBACvC,QAAQ,CAAC,KAAK,CAAC,CAAC;YACpB,CAAC;QACL,CAAC,CAAC;QAEF,QAAQ,CAAC,IAAI,CAAC,CAAC;QAEf,4EAA4E;QAC5E,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACtB,MAAM,CAAC,IAAI,CAAC;gBACR,OAAO,EAAE,IAAI;gBACb,QAAQ,EAAE;oBACN,SAAS,EAAE,CAAC;oBACZ,OAAO,EAAE,SAAS,CAAC,MAAM;oBACzB,QAAQ;oBACR,QAAQ;iBACX;aACJ,CAAC,CAAC;QACP,CAAC;QAED,OAAO,MAAM,CAAC;IAClB,CAAC;IAEO,KAAK,CAAC,YAAY,CAAC,MAAmB,EAAE,YAAoB;QAChE,MAAM,aAAa,GAAgB,EAAE,CAAC;QAEtC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YACzB,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;gBACzC,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC9B,CAAC;iBAAM,CAAC;gBACJ,qDAAqD;gBACrD,MAAM,SAAS,GAAG,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,YAAY,CAAC,CAAC;gBAC5D,aAAa,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,CAAC;YACrC,CAAC;QACL,CAAC;QAED,OAAO,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC;IAC1C,CAAC;IAEO,eAAe,CAAC,KAAgB,EAAE,YAAoB;QAC1D,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACxC,MAAM,SAAS,GAAgB,EAAE,CAAC;QAClC,IAAI,YAAY,GAAG,EAAE,CAAC;QACtB,IAAI,gBAAgB,GAAG,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC;QAChD,IAAI,gBAAgB,GAAG,CAAC,CAAC;QAEzB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACpC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YACtB,MAAM,eAAe,GAAG,CAAC,KAAK,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,GAAG,IAAI,CAAC;YAEpE,IAAI,YAAY,CAAC,MAAM,GAAG,eAAe,CAAC,MAAM,GAAG,IAAI,CAAC,SAAS,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC3F,qBAAqB;gBACrB,SAAS,CAAC,IAAI,CAAC;oBACX,OAAO,EAAE,YAAY,CAAC,IAAI,EAAE;oBAC5B,QAAQ,EAAE;wBACN,SAAS,EAAE,gBAAgB;wBAC3B,OAAO,EAAE,gBAAgB,GAAG,gBAAgB,GAAG,CAAC;wBAChD,QAAQ,EAAE,KAAK,CAAC,QAAQ,CAAC,QAAQ;wBACjC,QAAQ,EAAE,KAAK,CAAC,QAAQ,CAAC,QAAQ;qBACpC;iBACJ,CAAC,CAAC;gBAEH,YAAY,GAAG,eAAe,CAAC;gBAC/B,gBAAgB,GAAG,KAAK,CAAC,QAAQ,CAAC,SAAS,GAAG,CAAC,CAAC;gBAChD,gBAAgB,GAAG,CAAC,CAAC;YACzB,CAAC;iBAAM,CAAC;gBACJ,YAAY,IAAI,eAAe,CAAC;gBAChC,gBAAgB,EAAE,CAAC;YACvB,CAAC;QACL,CAAC;QAED,yBAAyB;QACzB,IAAI,YAAY,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACjC,SAAS,CAAC,IAAI,CAAC;gBACX,OAAO,EAAE,YAAY,CAAC,IAAI,EAAE;gBAC5B,QAAQ,EAAE;oBACN,SAAS,EAAE,gBAAgB;oBAC3B,OAAO,EAAE,gBAAgB,GAAG,gBAAgB,GAAG,CAAC;oBAChD,QAAQ,EAAE,KAAK,CAAC,QAAQ,CAAC,QAAQ;oBACjC,QAAQ,EAAE,KAAK,CAAC,QAAQ,CAAC,QAAQ;iBACpC;aACJ,CAAC,CAAC;QACP,CAAC;QAED,OAAO,SAAS,CAAC;IACrB,CAAC;IAEO,UAAU,CAAC,MAAmB;QAClC,IAAI,MAAM,CAAC,MAAM,IAAI,CAAC,IAAI,IAAI,CAAC,YAAY,IAAI,CAAC,EAAE,CAAC;YAC/C,OAAO,MAAM,CAAC;QAClB,CAAC;QAED,MAAM,gBAAgB,GAAgB,EAAE,CAAC;QAEzC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACrC,IAAI,OAAO,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;YAChC,MAAM,QAAQ,GAAG,EAAE,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC;YAE3C,kCAAkC;YAClC,IAAI,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,YAAY,GAAG,CAAC,EAAE,CAAC;gBACjC,MAAM,SAAS,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;gBAChC,MAAM,WAAW,GAAG,SAAS,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;gBAChE,OAAO,GAAG,WAAW,GAAG,IAAI,GAAG,OAAO,CAAC;gBACvC,QAAQ,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,CAAC,SAAS,GAAG,IAAI,CAAC,YAAY,CAAC,WAAW,CAAC,CAAC,CAAC;YAC1F,CAAC;YAED,gBAAgB,CAAC,IAAI,CAAC;gBAClB,OAAO;gBACP,QAAQ;aACX,CAAC,CAAC;QACP,CAAC;QAED,OAAO,gBAAgB,CAAC;IAC5B,CAAC;IAEO,YAAY,CAAC,IAAY;QAC7B,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;IACnC,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,mBAAmB,CAAC,QAAgB;QACvC,MAAM,kBAAkB,GAAG;YACvB,YAAY,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI;YACtD,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO;SACzE,CAAC;QACF,OAAO,kBAAkB,CAAC,QAAQ,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC;IAC/D,CAAC;CACJ;AAlPD,0CAkPC"}

package/dist/splitter/index.d.ts

@@ -0,0 +1,41 @@
+export interface CodeChunk {
+    content: string;
+    metadata: {
+        startLine: number;
+        endLine: number;
+        language?: string;
+        filePath?: string;
+    };
+}
+export declare enum SplitterType {
+    LANGCHAIN = "langchain",
+    AST = "ast"
+}
+export interface SplitterConfig {
+    type?: SplitterType;
+    chunkSize?: number;
+    chunkOverlap?: number;
+}
+export interface Splitter {
+    /**
+     * Split code into code chunks
+     * @param code Code content
+     * @param language Programming language
+     * @param filePath File path
+     * @returns Array of code chunks
+     */
+    split(code: string, language: string, filePath?: string): Promise<CodeChunk[]>;
+    /**
+     * Set chunk size
+     * @param chunkSize Chunk size
+     */
+    setChunkSize(chunkSize: number): void;
+    /**
+     * Set chunk overlap size
+     * @param chunkOverlap Chunk overlap size
+     */
+    setChunkOverlap(chunkOverlap: number): void;
+}
+export * from './langchain-splitter';
+export * from './ast-splitter';
+//# sourceMappingURL=index.d.ts.map

package/dist/splitter/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/splitter/index.ts"],"names":[],"mappings":"AACA,MAAM,WAAW,SAAS;IACtB,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE;QACN,SAAS,EAAE,MAAM,CAAC;QAClB,OAAO,EAAE,MAAM,CAAC;QAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;KACrB,CAAC;CACL;AAGD,oBAAY,YAAY;IACpB,SAAS,cAAc;IACvB,GAAG,QAAQ;CACd;AAGD,MAAM,WAAW,cAAc;IAC3B,IAAI,CAAC,EAAE,YAAY,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,QAAQ;IACrB;;;;;;OAMG;IACH,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC;IAE/E;;;OAGG;IACH,YAAY,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IAEtC;;;OAGG;IACH,eAAe,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;CAC/C;AAGD,cAAc,sBAAsB,CAAC;AACrC,cAAc,gBAAgB,CAAC"}

package/dist/splitter/index.js

@@ -0,0 +1,27 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SplitterType = void 0;
+// Splitter type enumeration
+var SplitterType;
+(function (SplitterType) {
+    SplitterType["LANGCHAIN"] = "langchain";
+    SplitterType["AST"] = "ast";
+})(SplitterType || (exports.SplitterType = SplitterType = {}));
+// Implementation class exports
+__exportStar(require("./langchain-splitter"), exports);
+__exportStar(require("./ast-splitter"), exports);
+//# sourceMappingURL=index.js.map

package/dist/splitter/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/splitter/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;AAWA,4BAA4B;AAC5B,IAAY,YAGX;AAHD,WAAY,YAAY;IACpB,uCAAuB,CAAA;IACvB,2BAAW,CAAA;AACf,CAAC,EAHW,YAAY,4BAAZ,YAAY,QAGvB;AAgCD,+BAA+B;AAC/B,uDAAqC;AACrC,iDAA+B"}

package/dist/splitter/langchain-splitter.d.ts

@@ -0,0 +1,13 @@
+import { Splitter, CodeChunk } from './index';
+export declare class LangChainCodeSplitter implements Splitter {
+    private chunkSize;
+    private chunkOverlap;
+    constructor(chunkSize?: number, chunkOverlap?: number);
+    split(code: string, language: string, filePath?: string): Promise<CodeChunk[]>;
+    setChunkSize(chunkSize: number): void;
+    setChunkOverlap(chunkOverlap: number): void;
+    private mapLanguage;
+    private fallbackSplit;
+    private estimateLines;
+}
+//# sourceMappingURL=langchain-splitter.d.ts.map

package/dist/splitter/langchain-splitter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"langchain-splitter.d.ts","sourceRoot":"","sources":["../../src/splitter/langchain-splitter.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAK9C,qBAAa,qBAAsB,YAAW,QAAQ;IAClD,OAAO,CAAC,SAAS,CAAgB;IACjC,OAAO,CAAC,YAAY,CAAe;gBAEvB,SAAS,CAAC,EAAE,MAAM,EAAE,YAAY,CAAC,EAAE,MAAM;IAK/C,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC;IAwCpF,YAAY,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI;IAIrC,eAAe,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI;IAI3C,OAAO,CAAC,WAAW;YA4BL,aAAa;IAuB3B,OAAO,CAAC,aAAa;CAiBxB"}