codevault 1.6.0 → 1.7.0

package/dist/mcp/tools/use-context-pack.js

@@ -0,0 +1,91 @@
+import { z } from 'zod';
+import { loadContextPack } from '../../context/packs.js';
+export const useContextPackInputSchema = z.object({
+    name: z.string().min(1, 'Context pack name is required'),
+    path: z.string().optional()
+});
+export const useContextPackResultSchema = z.object({
+    success: z.boolean(),
+    message: z.string(),
+    pack: z.object({
+        key: z.string(),
+        name: z.string(),
+        description: z.string().nullable(),
+        scope: z.record(z.string(), z.any())
+    }).optional()
+});
+export function createUseContextPackHandler(options) {
+    const { getWorkingPath, setSessionPack, clearSessionPack, errorLogger } = options;
+    return async ({ name, path: explicitPath }) => {
+        const basePath = explicitPath && explicitPath.trim().length > 0
+            ? explicitPath.trim()
+            : (typeof getWorkingPath === 'function' ? getWorkingPath() : '.');
+        if (name === 'default' || name === 'none' || name === 'clear') {
+            if (typeof clearSessionPack === 'function') {
+                clearSessionPack();
+            }
+            if (errorLogger && typeof errorLogger.debugLog === 'function') {
+                errorLogger.debugLog('Cleared MCP session context pack', { basePath, name });
+            }
+            return {
+                success: true,
+                message: 'Cleared active context pack for this session'
+            };
+        }
+        try {
+            const pack = loadContextPack(name, basePath);
+            const sessionPack = {
+                ...pack,
+                basePath
+            };
+            if (typeof setSessionPack === 'function') {
+                setSessionPack(sessionPack);
+            }
+            if (errorLogger && typeof errorLogger.debugLog === 'function') {
+                errorLogger.debugLog('Activated MCP session context pack', {
+                    pack: pack.key,
+                    basePath
+                });
+            }
+            return {
+                success: true,
+                message: `Context pack "${pack.key}" activated for session`,
+                pack: {
+                    key: pack.key,
+                    name: pack.name,
+                    description: pack.description || null,
+                    scope: pack.scope
+                }
+            };
+        }
+        catch (error) {
+            if (errorLogger && typeof errorLogger.log === 'function') {
+                errorLogger.log(error, {
+                    operation: 'use_context_pack',
+                    name,
+                    basePath
+                });
+            }
+            throw error;
+        }
+    };
+}
+export function registerUseContextPackTool(server, options) {
+    const handler = createUseContextPackHandler(options);
+    server.tool('use_context_pack', {
+        name: z.string().min(1).describe('Context pack name (e.g., "test-pack", "stripe-backend") or "clear" to reset'),
+        path: z.string().optional().describe('PROJECT ROOT directory path (defaults to ".")')
+    }, async (params) => {
+        const result = await handler(params);
+        return {
+            content: [
+                {
+                    type: 'text',
+                    text: result.message
+                }
+            ]
+        };
+    });
+    return handler;
+}
+//# sourceMappingURL=use-context-pack.js.map

package/dist/mcp/tools/use-context-pack.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"use-context-pack.js","sourceRoot":"","sources":["../../../src/mcp/tools/use-context-pack.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AAEzD,MAAM,CAAC,MAAM,yBAAyB,GAAG,CAAC,CAAC,MAAM,CAAC;IAChD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,+BAA+B,CAAC;IACxD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;CAC5B,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,0BAA0B,GAAG,CAAC,CAAC,MAAM,CAAC;IACjD,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE;IACpB,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE;IACnB,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC;QACb,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE;QACf,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;QAChB,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;QAClC,KAAK,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC;KACrC,CAAC,CAAC,QAAQ,EAAE;CACd,CAAC,CAAC;AASH,MAAM,UAAU,2BAA2B,CAAC,OAA6B;IACvE,MAAM,EAAE,cAAc,EAAE,cAAc,EAAE,gBAAgB,EAAE,WAAW,EAAE,GAAG,OAAO,CAAC;IAElF,OAAO,KAAK,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,YAAY,EAAmC,EAAE,EAAE;QAC7E,MAAM,QAAQ,GAAG,YAAY,IAAI,YAAY,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC;YAC7D,CAAC,CAAC,YAAY,CAAC,IAAI,EAAE;YACrB,CAAC,CAAC,CAAC,OAAO,cAAc,KAAK,UAAU,CAAC,CAAC,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAEpE,IAAI,IAAI,KAAK,SAAS,IAAI,IAAI,KAAK,MAAM,IAAI,IAAI,KAAK,OAAO,EAAE,CAAC;YAC9D,IAAI,OAAO,gBAAgB,KAAK,UAAU,EAAE,CAAC;gBAC3C,gBAAgB,EAAE,CAAC;YACrB,CAAC;YACD,IAAI,WAAW,IAAI,OAAO,WAAW,CAAC,QAAQ,KAAK,UAAU,EAAE,CAAC;gBAC9D,WAAW,CAAC,QAAQ,CAAC,kCAAkC,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;YAC/E,CAAC;YACD,OAAO;gBACL,OAAO,EAAE,IAAI;gBACb,OAAO,EAAE,8CAA8C;aACxD,CAAC;QACJ,CAAC;QAED,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,eAAe,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;YAC7C,MAAM,WAAW,GAAG;gBAClB,GAAG,IAAI;gBACP,QAAQ;aACT,CAAC;YAEF,IAAI,OAAO,cAAc,KAAK,UAAU,EAAE,CAAC;gBACzC,cAAc,CAAC,WAAW,CAAC,CAAC;YAC9B,CAAC;YAED,IAAI,WAAW,IAAI,OAAO,WAAW,CAAC,QAAQ,KAAK,UAAU,EAAE,CAAC;gBAC9D,WAAW,CAAC,QAAQ,CAAC,oCAAoC,EAAE;oBACzD,IAAI,EAAE,IAAI,CAAC,GAAG;oBACd,QAAQ;iBACT,CAAC,CAAC;YACL,CAAC;YAED,OAAO;gBACL,OAAO,EAAE,IAAI;gBACb,OAAO,EAAE,iBAAiB,IAAI,CAAC,GAAG,yBAAyB;gBAC3D,IAAI,EAAE;oBACJ,GAAG,EAAE,IAAI,CAAC,GAAG;oBACb,IAAI,EAAE,IAAI,CAAC,IAAI;oBACf,WAAW,EAAE,IAAI,CAAC,WAAW,IAAI,IAAI;oBACrC,KAAK,EAAE,IAAI,CAAC,KAAK;iBAClB;aACF,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,WAAW,IAAI,OAAO,WAAW,CAAC,GAAG,KAAK,UAAU,EAAE,CAAC;gBACzD,WAAW,CAAC,GAAG,CAAC,KAAK,EAAE;oBACrB,SAAS,EAAE,kBAAkB;oBAC7B,IAAI;oBACJ,QAAQ;iBACT,CAAC,CAAC;YACL,CAAC;YACD,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,0BAA0B,CAAC,MAAW,EAAE,OAA6B;IACnF,MAAM,OAAO,GAAG,2BAA2B,CAAC,OAAO,CAAC,CAAC;IAErD,MAAM,CAAC,IAAI,CACT,kBAAkB,EAClB;QACE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,6EAA6E,CAAC;QAC/G,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,+CAA+C,CAAC;KACtF,EACD,KAAK,EAAE,MAAW,EAAE,EAAE;QACpB,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,MAAM,CAAC,CAAC;QACrC,OAAO;YACL,OAAO,EAAE;gBACP;oBACE,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,MAAM,CAAC,OAAO;iBACrB;aACF;SACF,CAAC;IACJ,CAAC,CACF,CAAC;IAEF,OAAO,OAAO,CAAC;AACjB,CAAC"}

package/dist/mcp-server.d.ts

@@ -0,0 +1,13 @@
+#!/usr/bin/env node
+import 'dotenv/config';
+export declare class McpServer {
+    private server;
+    private sessionContextPack;
+    private cacheCleanupTimer;
+    constructor();
+    private setupHandlers;
+    start(): Promise<void>;
+    private scheduleCacheCleanup;
+    private setupShutdownHandlers;
+}
+//# sourceMappingURL=mcp-server.d.ts.map

package/dist/mcp-server.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"mcp-server.d.ts","sourceRoot":"","sources":["../src/mcp-server.ts"],"names":[],"mappings":";AAEA,OAAO,eAAe,CAAC;AAgCvB,qBAAa,SAAS;IACpB,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,kBAAkB,CAAa;IACvC,OAAO,CAAC,iBAAiB,CAA+B;;IAkBxD,OAAO,CAAC,aAAa;IAmMR,KAAK;IAUlB,OAAO,CAAC,oBAAoB;IAiB5B,OAAO,CAAC,qBAAqB;CAsB9B"}

package/dist/mcp-server.js

@@ -0,0 +1,263 @@
+#!/usr/bin/env node
+import 'dotenv/config';
+import { Server } from '@modelcontextprotocol/sdk/server/index.js';
+import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
+import { CallToolRequestSchema, ListToolsRequestSchema } from '@modelcontextprotocol/sdk/types.js';
+import fs from 'fs';
+import path from 'path';
+import { fileURLToPath } from 'url';
+import * as handlers from './mcp/handlers/index.js';
+import { SearchCodeArgsSchema, SearchCodeWithChunksArgsSchema, GetCodeChunkArgsSchema, IndexProjectArgsSchema, UpdateProjectArgsSchema, GetProjectStatsArgsSchema, UseContextPackArgsSchema, AskCodebaseArgsSchema } from './mcp/schemas.js';
+import { CACHE_CONSTANTS } from './config/constants.js';
+import { clearSearchCaches } from './core/search.js';
+import { clearTokenCache } from './chunking/token-counter.js';
+import { logger } from './utils/logger.js';
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+const packageJson = JSON.parse(fs.readFileSync(path.join(__dirname, '..', 'package.json'), 'utf8'));
+export class McpServer {
+    server;
+    sessionContextPack = null;
+    cacheCleanupTimer = null;
+    constructor() {
+        this.server = new Server({
+            name: 'codevault-code-memory',
+            version: packageJson.version,
+        }, {
+            capabilities: {
+                tools: {},
+            },
+        });
+        this.setupHandlers();
+    }
+    setupHandlers() {
+        this.server.setRequestHandler(ListToolsRequestSchema, async () => {
+            const tools = [
+                {
+                    name: 'search_code',
+                    description: 'Search code semantically using embeddings',
+                    inputSchema: {
+                        type: 'object',
+                        properties: {
+                            query: { type: 'string', description: 'Search query' },
+                            limit: { type: 'number', description: 'Max results (default: 50, max: 200)', default: 50 },
+                            provider: { type: 'string', description: 'Embedding provider (auto|openai)', default: 'auto' },
+                            path: { type: 'string', description: 'Project root directory', default: '.' },
+                            path_glob: { type: ['string', 'array'], description: 'File patterns to filter' },
+                            tags: { type: ['string', 'array'], description: 'Tags to filter' },
+                            lang: { type: ['string', 'array'], description: 'Languages to filter' },
+                            reranker: { type: 'string', enum: ['off', 'api'], default: 'off' },
+                            hybrid: { type: 'string', enum: ['on', 'off'], default: 'on' },
+                            bm25: { type: 'string', enum: ['on', 'off'], default: 'on' },
+                            symbol_boost: { type: 'string', enum: ['on', 'off'], default: 'on' },
+                        },
+                        required: ['query'],
+                    },
+                },
+                {
+                    name: 'search_code_with_chunks',
+                    description: 'Search code and return full code chunks',
+                    inputSchema: {
+                        type: 'object',
+                        properties: {
+                            query: { type: 'string', description: 'Search query' },
+                            limit: { type: 'number', description: 'Max results', default: 10 },
+                            provider: { type: 'string', default: 'auto' },
+                            path: { type: 'string', default: '.' },
+                            path_glob: { type: ['string', 'array'] },
+                            tags: { type: ['string', 'array'] },
+                            lang: { type: ['string', 'array'] },
+                            reranker: { type: 'string', enum: ['off', 'api'], default: 'off' },
+                            hybrid: { type: 'string', enum: ['on', 'off'], default: 'on' },
+                            bm25: { type: 'string', enum: ['on', 'off'], default: 'on' },
+                            symbol_boost: { type: 'string', enum: ['on', 'off'], default: 'on' },
+                        },
+                        required: ['query'],
+                    },
+                },
+                {
+                    name: 'get_code_chunk',
+                    description: 'Get code chunk by SHA',
+                    inputSchema: {
+                        type: 'object',
+                        properties: {
+                            sha: { type: 'string', description: 'SHA of code chunk' },
+                            path: { type: 'string', description: 'Project root directory', default: '.' },
+                        },
+                        required: ['sha'],
+                    },
+                },
+                {
+                    name: 'index_project',
+                    description: 'Index a project for semantic search',
+                    inputSchema: {
+                        type: 'object',
+                        properties: {
+                            path: { type: 'string', description: 'Project root directory', default: '.' },
+                            provider: { type: 'string', description: 'Embedding provider', default: 'auto' },
+                        },
+                    },
+                },
+                {
+                    name: 'update_project',
+                    description: 'Update project index',
+                    inputSchema: {
+                        type: 'object',
+                        properties: {
+                            path: { type: 'string', description: 'Project root directory', default: '.' },
+                            provider: { type: 'string', default: 'auto' },
+                        },
+                    },
+                },
+                {
+                    name: 'get_project_stats',
+                    description: 'Get project statistics',
+                    inputSchema: {
+                        type: 'object',
+                        properties: {
+                            path: { type: 'string', description: 'Project root directory', default: '.' },
+                        },
+                    },
+                },
+                {
+                    name: 'use_context_pack',
+                    description: 'Activate a context pack for scoped search',
+                    inputSchema: {
+                        type: 'object',
+                        properties: {
+                            name: { type: 'string', description: 'Context pack name or "clear"' },
+                            path: { type: 'string', description: 'Project root directory', default: '.' },
+                        },
+                        required: ['name'],
+                    },
+                },
+                {
+                    name: 'ask_codebase',
+                    description: 'Ask a question and get LLM-synthesized answer with code citations',
+                    inputSchema: {
+                        type: 'object',
+                        properties: {
+                            question: { type: 'string', description: 'Natural language question about the codebase' },
+                            provider: { type: 'string', description: 'Embedding provider (auto|openai)', default: 'auto' },
+                            chat_provider: { type: 'string', description: 'Chat LLM provider (auto|openai)', default: 'auto' },
+                            path: { type: 'string', description: 'Project root directory', default: '.' },
+                            max_chunks: { type: 'number', description: 'Max code chunks to analyze', default: 10 },
+                            path_glob: { type: ['string', 'array'], description: 'File patterns to filter' },
+                            tags: { type: ['string', 'array'], description: 'Tags to filter' },
+                            lang: { type: ['string', 'array'], description: 'Languages to filter' },
+                            reranker: { type: 'string', enum: ['on', 'off'], default: 'on', description: 'Use API reranking' },
+                            multi_query: { type: 'boolean', default: false, description: 'Break complex questions into sub-queries' },
+                            temperature: { type: 'number', minimum: 0, maximum: 2, default: 0.7, description: 'LLM temperature' },
+                        },
+                        required: ['question'],
+                    },
+                },
+            ];
+            return { tools };
+        });
+        this.server.setRequestHandler(CallToolRequestSchema, async (request) => {
+            const { name, arguments: args } = request.params;
+            const rawArgs = args || {};
+            try {
+                switch (name) {
+                    case 'search_code': {
+                        const validArgs = SearchCodeArgsSchema.parse(rawArgs);
+                        return await handlers.handleSearchCode(validArgs, this.sessionContextPack);
+                    }
+                    case 'search_code_with_chunks': {
+                        const validArgs = SearchCodeWithChunksArgsSchema.parse(rawArgs);
+                        return await handlers.handleSearchCodeWithChunks(validArgs, this.sessionContextPack);
+                    }
+                    case 'get_code_chunk': {
+                        const validArgs = GetCodeChunkArgsSchema.parse(rawArgs);
+                        return await handlers.handleGetCodeChunk(validArgs);
+                    }
+                    case 'index_project': {
+                        const validArgs = IndexProjectArgsSchema.parse(rawArgs);
+                        return await handlers.handleIndexProject(validArgs);
+                    }
+                    case 'update_project': {
+                        const validArgs = UpdateProjectArgsSchema.parse(rawArgs);
+                        return await handlers.handleUpdateProject(validArgs);
+                    }
+                    case 'get_project_stats': {
+                        const validArgs = GetProjectStatsArgsSchema.parse(rawArgs);
+                        return await handlers.handleGetProjectStats(validArgs);
+                    }
+                    case 'use_context_pack': {
+                        const validArgs = UseContextPackArgsSchema.parse(rawArgs);
+                        return await handlers.handleUseContextPack(validArgs, (pack) => {
+                            this.sessionContextPack = pack;
+                        });
+                    }
+                    case 'ask_codebase': {
+                        const validArgs = AskCodebaseArgsSchema.parse(rawArgs);
+                        return await handlers.handleAskCodebase(validArgs, this.sessionContextPack);
+                    }
+                    default:
+                        throw new Error(`Unknown tool: ${name}`);
+                }
+            }
+            catch (error) {
+                if (error && typeof error === 'object' && 'issues' in error && Array.isArray(error.issues)) {
+                    const validationError = `Validation Error: ${error.issues.map((i) => `${i.path.join('.')}: ${i.message}`).join(', ')}`;
+                    return {
+                        content: [{ type: 'text', text: validationError }],
+                        isError: true,
+                    };
+                }
+                return {
+                    content: [{ type: 'text', text: `ERROR: ${error.message}` }],
+                    isError: true,
+                };
+            }
+        });
+    }
+    async start() {
+        const transport = new StdioServerTransport();
+        await this.server.connect(transport);
+        logger.info('CodeVault MCP Server started', { version: packageJson.version });
+        this.scheduleCacheCleanup();
+        this.setupShutdownHandlers();
+    }
+    scheduleCacheCleanup() {
+        if (this.cacheCleanupTimer) {
+            clearInterval(this.cacheCleanupTimer);
+        }
+        this.cacheCleanupTimer = setInterval(() => {
+            try {
+                clearSearchCaches();
+                clearTokenCache();
+                logger.debug('Cache cleared periodically');
+            }
+            catch (error) {
+                // Ignore errors during cleanup
+            }
+        }, CACHE_CONSTANTS.CACHE_CLEAR_INTERVAL_MS);
+    }
+    setupShutdownHandlers() {
+        const cleanup = async () => {
+            if (this.cacheCleanupTimer) {
+                clearInterval(this.cacheCleanupTimer);
+                this.cacheCleanupTimer = null;
+            }
+            this.sessionContextPack = null;
+            clearSearchCaches();
+            clearTokenCache();
+        };
+        process.on('SIGINT', async () => {
+            await cleanup();
+            process.exit(0);
+        });
+        process.on('SIGTERM', async () => {
+            await cleanup();
+            process.exit(0);
+        });
+    }
+}
+const server = new McpServer();
+server.start().catch((error) => {
+    logger.error('Fatal error', error);
+    process.exit(1);
+});
+//# sourceMappingURL=mcp-server.js.map

package/dist/mcp-server.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"mcp-server.js","sourceRoot":"","sources":["../src/mcp-server.ts"],"names":[],"mappings":";AAEA,OAAO,eAAe,CAAC;AACvB,OAAO,EAAE,MAAM,EAAE,MAAM,2CAA2C,CAAC;AACnE,OAAO,EAAE,oBAAoB,EAAE,MAAM,2CAA2C,CAAC;AACjF,OAAO,EACL,qBAAqB,EACrB,sBAAsB,EAEvB,MAAM,oCAAoC,CAAC;AAC5C,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,aAAa,EAAE,MAAM,KAAK,CAAC;AACpC,OAAO,KAAK,QAAQ,MAAM,yBAAyB,CAAC;AACpD,OAAO,EACL,oBAAoB,EACpB,8BAA8B,EAC9B,sBAAsB,EACtB,sBAAsB,EACtB,uBAAuB,EACvB,yBAAyB,EACzB,wBAAwB,EACxB,qBAAqB,EACtB,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EAAE,eAAe,EAAE,MAAM,6BAA6B,CAAC;AAC9D,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAE3C,MAAM,UAAU,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAClD,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;AAE3C,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,EAAE,cAAc,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC;AAEpG,MAAM,OAAO,SAAS;IACZ,MAAM,CAAS;IACf,kBAAkB,GAAQ,IAAI,CAAC;IAC/B,iBAAiB,GAA0B,IAAI,CAAC;IAExD;QACE,IAAI,CAAC,MAAM,GAAG,IAAI,MAAM,CACtB;YACE,IAAI,EAAE,uBAAuB;YAC7B,OAAO,EAAE,WAAW,CAAC,OAAO;SAC7B,EACD;YACE,YAAY,EAAE;gBACZ,KAAK,EAAE,EAAE;aACV;SACF,CACF,CAAC;QAEF,IAAI,CAAC,aAAa,EAAE,CAAC;IACvB,CAAC;IAEO,aAAa;QACnB,IAAI,CAAC,MAAM,CAAC,iBAAiB,CAAC,sBAAsB,EAAE,KAAK,IAAI,EAAE;YAC/D,MAAM,KAAK,GAAW;gBACpB;oBACE,IAAI,EAAE,aAAa;oBACnB,WAAW,EAAE,2CAA2C;oBACxD,WAAW,EAAE;wBACX,IAAI,EAAE,QAAQ;wBACd,UAAU,EAAE;4BACV,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,cAAc,EAAE;4BACtD,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,qCAAqC,EAAE,OAAO,EAAE,EAAE,EAAE;4BAC1F,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,kCAAkC,EAAE,OAAO,EAAE,MAAM,EAAE;4BAC9F,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,wBAAwB,EAAE,OAAO,EAAE,GAAG,EAAE;4BAC7E,SAAS,EAAE,EAAE,IAAI,EAAE,CAAC,QAAQ,EAAE,OAAO,CAAC,EAAE,WAAW,EAAE,yBAAyB,EAAE;4BAChF,IAAI,EAAE,EAAE,IAAI,EAAE,CAAC,QAAQ,EAAE,OAAO,CAAC,EAAE,WAAW,EAAE,gBAAgB,EAAE;4BAClE,IAAI,EAAE,EAAE,IAAI,EAAE,CAAC,QAAQ,EAAE,OAAO,CAAC,EAAE,WAAW,EAAE,qBAAqB,EAAE;4BACvE,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,KAAK,EAAE,KAAK,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE;4BAClE,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE;4BAC9D,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE;4BAC5D,YAAY,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE;yBACrE;wBACD,QAAQ,EAAE,CAAC,OAAO,CAAC;qBACpB;iBACF;gBACD;oBACE,IAAI,EAAE,yBAAyB;oBAC/B,WAAW,EAAE,yCAAyC;oBACtD,WAAW,EAAE;wBACX,IAAI,EAAE,QAAQ;wBACd,UAAU,EAAE;4BACV,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,cAAc,EAAE;4BACtD,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,aAAa,EAAE,OAAO,EAAE,EAAE,EAAE;4BAClE,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE;4BAC7C,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,GAAG,EAAE;4BACtC,SAAS,EAAE,EAAE,IAAI,EAAE,CAAC,QAAQ,EAAE,OAAO,CAAC,EAAE;4BACxC,IAAI,EAAE,EAAE,IAAI,EAAE,CAAC,QAAQ,EAAE,OAAO,CAAC,EAAE;4BACnC,IAAI,EAAE,EAAE,IAAI,EAAE,CAAC,QAAQ,EAAE,OAAO,CAAC,EAAE;4BACnC,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,KAAK,EAAE,KAAK,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE;4BAClE,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE;4BAC9D,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE;4BAC5D,YAAY,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE;yBACrE;wBACD,QAAQ,EAAE,CAAC,OAAO,CAAC;qBACpB;iBACF;gBACD;oBACE,IAAI,EAAE,gBAAgB;oBACtB,WAAW,EAAE,uBAAuB;oBACpC,WAAW,EAAE;wBACX,IAAI,EAAE,QAAQ;wBACd,UAAU,EAAE;4BACV,GAAG,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,mBAAmB,EAAE;4BACzD,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,wBAAwB,EAAE,OAAO,EAAE,GAAG,EAAE;yBAC9E;wBACD,QAAQ,EAAE,CAAC,KAAK,CAAC;qBAClB;iBACF;gBACD;oBACE,IAAI,EAAE,eAAe;oBACrB,WAAW,EAAE,qCAAqC;oBAClD,WAAW,EAAE;wBACX,IAAI,EAAE,QAAQ;wBACd,UAAU,EAAE;4BACV,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,wBAAwB,EAAE,OAAO,EAAE,GAAG,EAAE;4BAC7E,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,oBAAoB,EAAE,OAAO,EAAE,MAAM,EAAE;yBACjF;qBACF;iBACF;gBACD;oBACE,IAAI,EAAE,gBAAgB;oBACtB,WAAW,EAAE,sBAAsB;oBACnC,WAAW,EAAE;wBACX,IAAI,EAAE,QAAQ;wBACd,UAAU,EAAE;4BACV,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,wBAAwB,EAAE,OAAO,EAAE,GAAG,EAAE;4BAC7E,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE;yBAC9C;qBACF;iBACF;gBACD;oBACE,IAAI,EAAE,mBAAmB;oBACzB,WAAW,EAAE,wBAAwB;oBACrC,WAAW,EAAE;wBACX,IAAI,EAAE,QAAQ;wBACd,UAAU,EAAE;4BACV,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,wBAAwB,EAAE,OAAO,EAAE,GAAG,EAAE;yBAC9E;qBACF;iBACF;gBACD;oBACE,IAAI,EAAE,kBAAkB;oBACxB,WAAW,EAAE,2CAA2C;oBACxD,WAAW,EAAE;wBACX,IAAI,EAAE,QAAQ;wBACd,UAAU,EAAE;4BACV,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,8BAA8B,EAAE;4BACrE,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,wBAAwB,EAAE,OAAO,EAAE,GAAG,EAAE;yBAC9E;wBACD,QAAQ,EAAE,CAAC,MAAM,CAAC;qBACnB;iBACF;gBACD;oBACE,IAAI,EAAE,cAAc;oBACpB,WAAW,EAAE,mEAAmE;oBAChF,WAAW,EAAE;wBACX,IAAI,EAAE,QAAQ;wBACd,UAAU,EAAE;4BACV,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,8CAA8C,EAAE;4BACzF,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,kCAAkC,EAAE,OAAO,EAAE,MAAM,EAAE;4BAC9F,aAAa,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,iCAAiC,EAAE,OAAO,EAAE,MAAM,EAAE;4BAClG,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,wBAAwB,EAAE,OAAO,EAAE,GAAG,EAAE;4BAC7E,UAAU,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,4BAA4B,EAAE,OAAO,EAAE,EAAE,EAAE;4BACtF,SAAS,EAAE,EAAE,IAAI,EAAE,CAAC,QAAQ,EAAE,OAAO,CAAC,EAAE,WAAW,EAAE,yBAAyB,EAAE;4BAChF,IAAI,EAAE,EAAE,IAAI,EAAE,CAAC,QAAQ,EAAE,OAAO,CAAC,EAAE,WAAW,EAAE,gBAAgB,EAAE;4BAClE,IAAI,EAAE,EAAE,IAAI,EAAE,CAAC,QAAQ,EAAE,OAAO,CAAC,EAAE,WAAW,EAAE,qBAAqB,EAAE;4BACvE,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,mBAAmB,EAAE;4BAClG,WAAW,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,0CAA0C,EAAE;4BACzG,WAAW,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,GAAG,EAAE,WAAW,EAAE,iBAAiB,EAAE;yBACtG;wBACD,QAAQ,EAAE,CAAC,UAAU,CAAC;qBACvB;iBACF;aACF,CAAC;YAEF,OAAO,EAAE,KAAK,EAAE,CAAC;QACnB,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,MAAM,CAAC,iBAAiB,CAAC,qBAAqB,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;YACrE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC;YACjD,MAAM,OAAO,GAAG,IAAI,IAAI,EAAE,CAAC;YAE3B,IAAI,CAAC;gBACH,QAAQ,IAAI,EAAE,CAAC;oBACb,KAAK,aAAa,CAAC,CAAC,CAAC;wBACnB,MAAM,SAAS,GAAG,oBAAoB,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;wBACtD,OAAO,MAAM,QAAQ,CAAC,gBAAgB,CAAC,SAAS,EAAE,IAAI,CAAC,kBAAkB,CAAC,CAAC;oBAC7E,CAAC;oBAED,KAAK,yBAAyB,CAAC,CAAC,CAAC;wBAC/B,MAAM,SAAS,GAAG,8BAA8B,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;wBAChE,OAAO,MAAM,QAAQ,CAAC,0BAA0B,CAAC,SAAS,EAAE,IAAI,CAAC,kBAAkB,CAAC,CAAC;oBACvF,CAAC;oBAED,KAAK,gBAAgB,CAAC,CAAC,CAAC;wBACtB,MAAM,SAAS,GAAG,sBAAsB,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;wBACxD,OAAO,MAAM,QAAQ,CAAC,kBAAkB,CAAC,SAAS,CAAC,CAAC;oBACtD,CAAC;oBAED,KAAK,eAAe,CAAC,CAAC,CAAC;wBACrB,MAAM,SAAS,GAAG,sBAAsB,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;wBACxD,OAAO,MAAM,QAAQ,CAAC,kBAAkB,CAAC,SAAS,CAAC,CAAC;oBACtD,CAAC;oBAED,KAAK,gBAAgB,CAAC,CAAC,CAAC;wBACtB,MAAM,SAAS,GAAG,uBAAuB,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;wBACzD,OAAO,MAAM,QAAQ,CAAC,mBAAmB,CAAC,SAAS,CAAC,CAAC;oBACvD,CAAC;oBAED,KAAK,mBAAmB,CAAC,CAAC,CAAC;wBACzB,MAAM,SAAS,GAAG,yBAAyB,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;wBAC3D,OAAO,MAAM,QAAQ,CAAC,qBAAqB,CAAC,SAAS,CAAC,CAAC;oBACzD,CAAC;oBAED,KAAK,kBAAkB,CAAC,CAAC,CAAC;wBACxB,MAAM,SAAS,GAAG,wBAAwB,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;wBAC1D,OAAO,MAAM,QAAQ,CAAC,oBAAoB,CAAC,SAAS,EAAE,CAAC,IAAI,EAAE,EAAE;4BAC7D,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC;wBACjC,CAAC,CAAC,CAAC;oBACL,CAAC;oBAED,KAAK,cAAc,CAAC,CAAC,CAAC;wBACpB,MAAM,SAAS,GAAG,qBAAqB,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;wBACvD,OAAO,MAAM,QAAQ,CAAC,iBAAiB,CAAC,SAAS,EAAE,IAAI,CAAC,kBAAkB,CAAC,CAAC;oBAC9E,CAAC;oBAED;wBACE,MAAM,IAAI,KAAK,CAAC,iBAAiB,IAAI,EAAE,CAAC,CAAC;gBAC7C,CAAC;YACH,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,QAAQ,IAAI,KAAK,IAAI,KAAK,CAAC,OAAO,CAAE,KAAa,CAAC,MAAM,CAAC,EAAE,CAAC;oBACpG,MAAM,eAAe,GAAG,qBAAsB,KAAa,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;oBACrI,OAAO;wBACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,eAAe,EAAE,CAAC;wBAClD,OAAO,EAAE,IAAI;qBACd,CAAC;gBACJ,CAAC;gBAED,OAAO;oBACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,UAAW,KAAe,CAAC,OAAO,EAAE,EAAE,CAAC;oBACvE,OAAO,EAAE,IAAI;iBACd,CAAC;YACJ,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IAEM,KAAK,CAAC,KAAK;QAChB,MAAM,SAAS,GAAG,IAAI,oBAAoB,EAAE,CAAC;QAC7C,MAAM,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QAErC,MAAM,CAAC,IAAI,CAAC,8BAA8B,EAAE,EAAE,OAAO,EAAE,WAAW,CAAC,OAAO,EAAE,CAAC,CAAC;QAE9E,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAC5B,IAAI,CAAC,qBAAqB,EAAE,CAAC;IAC/B,CAAC;IAEO,oBAAoB;QAC1B,IAAI,IAAI,CAAC,iBAAiB,EAAE,CAAC;YAC3B,aAAa,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;QACxC,CAAC;QAED,IAAI,CAAC,iBAAiB,GAAG,WAAW,CAAC,GAAG,EAAE;YACxC,IAAI,CAAC;gBACH,iBAAiB,EAAE,CAAC;gBACpB,eAAe,EAAE,CAAC;gBAElB,MAAM,CAAC,KAAK,CAAC,4BAA4B,CAAC,CAAC;YAC7C,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,+BAA+B;YACjC,CAAC;QACH,CAAC,EAAE,eAAe,CAAC,uBAAuB,CAAC,CAAC;IAC9C,CAAC;IAEO,qBAAqB;QAC3B,MAAM,OAAO,GAAG,KAAK,IAAI,EAAE;YACzB,IAAI,IAAI,CAAC,iBAAiB,EAAE,CAAC;gBAC3B,aAAa,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;gBACtC,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC;YAChC,CAAC;YAED,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC;YAC/B,iBAAiB,EAAE,CAAC;YACpB,eAAe,EAAE,CAAC;QACpB,CAAC,CAAC;QAEF,OAAO,CAAC,EAAE,CAAC,QAAQ,EAAE,KAAK,IAAI,EAAE;YAC9B,MAAM,OAAO,EAAE,CAAC;YAChB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC,CAAC,CAAC;QAEH,OAAO,CAAC,EAAE,CAAC,SAAS,EAAE,KAAK,IAAI,EAAE;YAC/B,MAAM,OAAO,EAAE,CAAC;YAChB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC,CAAC,CAAC;IACL,CAAC;CACF;AAED,MAAM,MAAM,GAAG,IAAI,SAAS,EAAE,CAAC;AAC/B,MAAM,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;IAC7B,MAAM,CAAC,KAAK,CAAC,aAAa,EAAE,KAAK,CAAC,CAAC;IACnC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC,CAAC,CAAC"}

package/dist/providers/base.d.ts

@@ -0,0 +1,39 @@
+export interface ModelProfile {
+    maxTokens: number;
+    optimalTokens: number;
+    minChunkTokens: number;
+    maxChunkTokens: number;
+    overlapTokens: number;
+    optimalChars: number;
+    minChunkChars: number;
+    maxChunkChars: number;
+    overlapChars: number;
+    dimensions: number;
+    useTokens: boolean;
+    tokenizerType: string;
+    encoding?: string;
+    tokenCounter?: (text: string) => number | Promise<number>;
+}
+export declare abstract class EmbeddingProvider {
+    abstract generateEmbedding(text: string): Promise<number[]>;
+    abstract getDimensions(): number;
+    abstract getName(): string;
+    abstract getModelName?(): string;
+    abstract init?(): Promise<void>;
+    generateEmbeddings(texts: string[]): Promise<number[][]>;
+    rateLimiter?: any;
+}
+export declare const BATCH_SIZE = 50;
+export declare const MAX_BATCH_TOKENS = 100000;
+export declare const MAX_ITEM_TOKENS = 8191;
+export declare function estimateTokens(text: string): number;
+export declare const MODEL_PROFILES: Record<string, Omit<ModelProfile, 'tokenCounter'>>;
+export declare function getModelProfile(providerName: string, modelName: string | null): Promise<ModelProfile>;
+export declare function getSizeLimits(profile: ModelProfile): {
+    optimal: number;
+    min: number;
+    max: number;
+    overlap: number;
+    unit: string;
+};
+//# sourceMappingURL=base.d.ts.map

package/dist/providers/base.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"base.d.ts","sourceRoot":"","sources":["../../src/providers/base.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,YAAY;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,aAAa,EAAE,MAAM,CAAC;IACtB,cAAc,EAAE,MAAM,CAAC;IACvB,cAAc,EAAE,MAAM,CAAC;IACvB,aAAa,EAAE,MAAM,CAAC;IACtB,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,OAAO,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,YAAY,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;CAC3D;AAED,8BAAsB,iBAAiB;IACrC,QAAQ,CAAC,iBAAiB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAC3D,QAAQ,CAAC,aAAa,IAAI,MAAM;IAChC,QAAQ,CAAC,OAAO,IAAI,MAAM;IAC1B,QAAQ,CAAC,YAAY,CAAC,IAAI,MAAM;IAChC,QAAQ,CAAC,IAAI,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC;IAGzB,kBAAkB,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;IAU9D,WAAW,CAAC,EAAE,GAAG,CAAC;CACnB;AAGD,eAAO,MAAM,UAAU,KAAK,CAAC;AAC7B,eAAO,MAAM,gBAAgB,SAAS,CAAC;AACvC,eAAO,MAAM,eAAe,OAAO,CAAC;AAGpC,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAEnD;AAED,eAAO,MAAM,cAAc,EAAE,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,YAAY,EAAE,cAAc,CAAC,CAyG7E,CAAC;AAEF,wBAAsB,eAAe,CAAC,YAAY,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,IAAI,GAAG,OAAO,CAAC,YAAY,CAAC,CA2D3G;AAED,wBAAgB,aAAa,CAAC,OAAO,EAAE,YAAY,GAAG;IACpD,OAAO,EAAE,MAAM,CAAC;IAChB,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;CACd,CAiBA"}

package/dist/providers/base.js

@@ -0,0 +1,198 @@
+export class EmbeddingProvider {
+    // Batch processing support (optional - providers can override)
+    async generateEmbeddings(texts) {
+        // Default implementation: process one by one (backward compatible)
+        const embeddings = [];
+        for (const text of texts) {
+            const embedding = await this.generateEmbedding(text);
+            embeddings.push(embedding);
+        }
+        return embeddings;
+    }
+    rateLimiter;
+}
+// Batching constants
+export const BATCH_SIZE = 50; // Number of chunks per batch
+export const MAX_BATCH_TOKENS = 100000; // OpenAI limit
+export const MAX_ITEM_TOKENS = 8191; // Per-item limit
+// Estimate tokens from text (4 chars ≈ 1 token)
+export function estimateTokens(text) {
+    return Math.ceil(text.length / 4);
+}
+export const MODEL_PROFILES = {
+    'text-embedding-3-large': {
+        maxTokens: 8191,
+        optimalTokens: 4000,
+        minChunkTokens: 400,
+        maxChunkTokens: 6000,
+        overlapTokens: 100,
+        optimalChars: 16000,
+        minChunkChars: 1600,
+        maxChunkChars: 24000,
+        overlapChars: 400,
+        dimensions: 3072,
+        useTokens: true,
+        tokenizerType: 'tiktoken',
+        encoding: 'cl100k_base'
+    },
+    'text-embedding-3-small': {
+        maxTokens: 8191,
+        optimalTokens: 4000,
+        minChunkTokens: 400,
+        maxChunkTokens: 6000,
+        overlapTokens: 100,
+        optimalChars: 16000,
+        minChunkChars: 1600,
+        maxChunkChars: 24000,
+        overlapChars: 400,
+        dimensions: 1536,
+        useTokens: true,
+        tokenizerType: 'tiktoken',
+        encoding: 'cl100k_base'
+    },
+    'text-embedding-ada-002': {
+        maxTokens: 8191,
+        optimalTokens: 4000,
+        minChunkTokens: 400,
+        maxChunkTokens: 6000,
+        overlapTokens: 100,
+        optimalChars: 16000,
+        minChunkChars: 1600,
+        maxChunkChars: 24000,
+        overlapChars: 400,
+        dimensions: 1536,
+        useTokens: true,
+        tokenizerType: 'tiktoken',
+        encoding: 'cl100k_base'
+    },
+    'nomic-embed-text': {
+        maxTokens: 8192,
+        optimalTokens: 4000,
+        minChunkTokens: 400,
+        maxChunkTokens: 6000,
+        overlapTokens: 100,
+        optimalChars: 16000,
+        minChunkChars: 1600,
+        maxChunkChars: 24000,
+        overlapChars: 400,
+        dimensions: 768,
+        useTokens: true,
+        tokenizerType: 'tiktoken',
+        encoding: 'cl100k_base'
+    },
+    'Qwen/Qwen3-Embedding-8B': {
+        maxTokens: 32000,
+        optimalTokens: 16000,
+        minChunkTokens: 1000,
+        maxChunkTokens: 28000,
+        overlapTokens: 500,
+        optimalChars: 64000,
+        minChunkChars: 4000,
+        maxChunkChars: 112000,
+        overlapChars: 2000,
+        dimensions: 4096,
+        useTokens: true,
+        tokenizerType: 'tiktoken',
+        encoding: 'cl100k_base'
+    },
+    'qwen3-embedding:0.6b': {
+        maxTokens: 32000,
+        optimalTokens: 26000,
+        minChunkTokens: 1000,
+        maxChunkTokens: 30000,
+        overlapTokens: 500,
+        optimalChars: 128000,
+        minChunkChars: 4000,
+        maxChunkChars: 120000,
+        overlapChars: 2000,
+        dimensions: 1024,
+        useTokens: true,
+        tokenizerType: 'tiktoken',
+        encoding: 'cl100k_base'
+    },
+    'default': {
+        maxTokens: 512,
+        optimalTokens: 400,
+        minChunkTokens: 50,
+        maxChunkTokens: 480,
+        overlapTokens: 30,
+        optimalChars: 1600,
+        minChunkChars: 200,
+        maxChunkChars: 1920,
+        overlapChars: 120,
+        dimensions: 384,
+        useTokens: false,
+        tokenizerType: 'estimate'
+    }
+};
+export async function getModelProfile(providerName, modelName) {
+    let baseProfile = modelName ? MODEL_PROFILES[modelName] : undefined;
+    if (!baseProfile) {
+        const providerDefaults = {
+            'OpenAI': MODEL_PROFILES['text-embedding-3-large']
+        };
+        baseProfile = providerDefaults[providerName] || MODEL_PROFILES['default'];
+    }
+    const profile = { ...baseProfile };
+    if (process.env.CODEVAULT_EMBEDDING_MAX_TOKENS || process.env.CODEVAULT_MAX_TOKENS) {
+        const maxTokens = parseInt(process.env.CODEVAULT_EMBEDDING_MAX_TOKENS || process.env.CODEVAULT_MAX_TOKENS || '0', 10);
+        if (!isNaN(maxTokens) && maxTokens > 0) {
+            const originalMaxTokens = profile.maxTokens;
+            const scalingRatio = maxTokens / originalMaxTokens;
+            profile.maxTokens = maxTokens;
+            profile.optimalTokens = Math.floor(maxTokens * 0.82);
+            profile.minChunkTokens = Math.max(Math.floor(profile.minChunkTokens * scalingRatio), 50);
+            profile.maxChunkTokens = Math.floor(maxTokens * 0.95);
+            profile.overlapTokens = Math.floor(profile.overlapTokens * scalingRatio);
+            if (!process.env.CODEVAULT_QUIET) {
+                console.log(`Using custom max tokens: ${maxTokens}`);
+                console.log(`Auto-scaled optimal tokens: ${profile.optimalTokens} (82% of max)`);
+                console.log(`Auto-scaled min tokens: ${profile.minChunkTokens}`);
+                console.log(`Auto-scaled max chunk tokens: ${profile.maxChunkTokens} (95% of max)`);
+            }
+        }
+    }
+    if (process.env.CODEVAULT_EMBEDDING_DIMENSIONS || process.env.CODEVAULT_DIMENSIONS) {
+        const dimensions = parseInt(process.env.CODEVAULT_EMBEDDING_DIMENSIONS || process.env.CODEVAULT_DIMENSIONS || '0', 10);
+        if (!isNaN(dimensions) && dimensions > 0) {
+            profile.dimensions = dimensions;
+            if (!process.env.CODEVAULT_QUIET) {
+                console.log(`Using custom dimensions: ${dimensions}`);
+            }
+        }
+    }
+    if (profile.useTokens) {
+        const { getTokenCounter } = await import('./token-counter.js');
+        const counter = await getTokenCounter(modelName || providerName);
+        if (!counter) {
+            if (!process.env.CODEVAULT_QUIET) {
+                console.warn(`Token counter unavailable for ${modelName}, using character estimation`);
+            }
+            profile.useTokens = false;
+            profile.tokenCounter = undefined;
+        }
+        else {
+            profile.tokenCounter = counter;
+        }
+    }
+    return profile;
+}
+export function getSizeLimits(profile) {
+    if (profile.useTokens && profile.tokenCounter) {
+        return {
+            optimal: profile.optimalTokens,
+            min: profile.minChunkTokens,
+            max: profile.maxChunkTokens,
+            overlap: profile.overlapTokens,
+            unit: 'tokens'
+        };
+    }
+    return {
+        optimal: profile.optimalChars,
+        min: profile.minChunkChars,
+        max: profile.maxChunkChars,
+        overlap: profile.overlapChars,
+        unit: 'characters'
+    };
+}
+//# sourceMappingURL=base.js.map

package/dist/providers/base.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"base.js","sourceRoot":"","sources":["../../src/providers/base.ts"],"names":[],"mappings":"AAiBA,MAAM,OAAgB,iBAAiB;IAOrC,+DAA+D;IAC/D,KAAK,CAAC,kBAAkB,CAAC,KAAe;QACtC,mEAAmE;QACnE,MAAM,UAAU,GAAe,EAAE,CAAC;QAClC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC;YACrD,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAC7B,CAAC;QACD,OAAO,UAAU,CAAC;IACpB,CAAC;IAED,WAAW,CAAO;CACnB;AAED,qBAAqB;AACrB,MAAM,CAAC,MAAM,UAAU,GAAG,EAAE,CAAC,CAAC,6BAA6B;AAC3D,MAAM,CAAC,MAAM,gBAAgB,GAAG,MAAM,CAAC,CAAC,eAAe;AACvD,MAAM,CAAC,MAAM,eAAe,GAAG,IAAI,CAAC,CAAC,iBAAiB;AAEtD,gDAAgD;AAChD,MAAM,UAAU,cAAc,CAAC,IAAY;IACzC,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AACpC,CAAC;AAED,MAAM,CAAC,MAAM,cAAc,GAAuD;IAChF,wBAAwB,EAAE;QACxB,SAAS,EAAE,IAAI;QACf,aAAa,EAAE,IAAI;QACnB,cAAc,EAAE,GAAG;QACnB,cAAc,EAAE,IAAI;QACpB,aAAa,EAAE,GAAG;QAClB,YAAY,EAAE,KAAK;QACnB,aAAa,EAAE,IAAI;QACnB,aAAa,EAAE,KAAK;QACpB,YAAY,EAAE,GAAG;QACjB,UAAU,EAAE,IAAI;QAChB,SAAS,EAAE,IAAI;QACf,aAAa,EAAE,UAAU;QACzB,QAAQ,EAAE,aAAa;KACxB;IACD,wBAAwB,EAAE;QACxB,SAAS,EAAE,IAAI;QACf,aAAa,EAAE,IAAI;QACnB,cAAc,EAAE,GAAG;QACnB,cAAc,EAAE,IAAI;QACpB,aAAa,EAAE,GAAG;QAClB,YAAY,EAAE,KAAK;QACnB,aAAa,EAAE,IAAI;QACnB,aAAa,EAAE,KAAK;QACpB,YAAY,EAAE,GAAG;QACjB,UAAU,EAAE,IAAI;QAChB,SAAS,EAAE,IAAI;QACf,aAAa,EAAE,UAAU;QACzB,QAAQ,EAAE,aAAa;KACxB;IACD,wBAAwB,EAAE;QACxB,SAAS,EAAE,IAAI;QACf,aAAa,EAAE,IAAI;QACnB,cAAc,EAAE,GAAG;QACnB,cAAc,EAAE,IAAI;QACpB,aAAa,EAAE,GAAG;QAClB,YAAY,EAAE,KAAK;QACnB,aAAa,EAAE,IAAI;QACnB,aAAa,EAAE,KAAK;QACpB,YAAY,EAAE,GAAG;QACjB,UAAU,EAAE,IAAI;QAChB,SAAS,EAAE,IAAI;QACf,aAAa,EAAE,UAAU;QACzB,QAAQ,EAAE,aAAa;KACxB;IACD,kBAAkB,EAAE;QAClB,SAAS,EAAE,IAAI;QACf,aAAa,EAAE,IAAI;QACnB,cAAc,EAAE,GAAG;QACnB,cAAc,EAAE,IAAI;QACpB,aAAa,EAAE,GAAG;QAClB,YAAY,EAAE,KAAK;QACnB,aAAa,EAAE,IAAI;QACnB,aAAa,EAAE,KAAK;QACpB,YAAY,EAAE,GAAG;QACjB,UAAU,EAAE,GAAG;QACf,SAAS,EAAE,IAAI;QACf,aAAa,EAAE,UAAU;QACzB,QAAQ,EAAE,aAAa;KACxB;IACD,yBAAyB,EAAE;QACzB,SAAS,EAAE,KAAK;QAChB,aAAa,EAAE,KAAK;QACpB,cAAc,EAAE,IAAI;QACpB,cAAc,EAAE,KAAK;QACrB,aAAa,EAAE,GAAG;QAClB,YAAY,EAAE,KAAK;QACnB,aAAa,EAAE,IAAI;QACnB,aAAa,EAAE,MAAM;QACrB,YAAY,EAAE,IAAI;QAClB,UAAU,EAAE,IAAI;QAChB,SAAS,EAAE,IAAI;QACf,aAAa,EAAE,UAAU;QACzB,QAAQ,EAAE,aAAa;KACxB;IACD,sBAAsB,EAAE;QACtB,SAAS,EAAE,KAAK;QAChB,aAAa,EAAE,KAAK;QACpB,cAAc,EAAE,IAAI;QACpB,cAAc,EAAE,KAAK;QACrB,aAAa,EAAE,GAAG;QAClB,YAAY,EAAE,MAAM;QACpB,aAAa,EAAE,IAAI;QACnB,aAAa,EAAE,MAAM;QACrB,YAAY,EAAE,IAAI;QAClB,UAAU,EAAE,IAAI;QAChB,SAAS,EAAE,IAAI;QACf,aAAa,EAAE,UAAU;QACzB,QAAQ,EAAE,aAAa;KACxB;IACD,SAAS,EAAE;QACT,SAAS,EAAE,GAAG;QACd,aAAa,EAAE,GAAG;QAClB,cAAc,EAAE,EAAE;QAClB,cAAc,EAAE,GAAG;QACnB,aAAa,EAAE,EAAE;QACjB,YAAY,EAAE,IAAI;QAClB,aAAa,EAAE,GAAG;QAClB,aAAa,EAAE,IAAI;QACnB,YAAY,EAAE,GAAG;QACjB,UAAU,EAAE,GAAG;QACf,SAAS,EAAE,KAAK;QAChB,aAAa,EAAE,UAAU;KAC1B;CACF,CAAC;AAEF,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,YAAoB,EAAE,SAAwB;IAClF,IAAI,WAAW,GAAG,SAAS,CAAC,CAAC,CAAC,cAAc,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAEpE,IAAI,CAAC,WAAW,EAAE,CAAC;QACjB,MAAM,gBAAgB,GAAuD;YAC3E,QAAQ,EAAE,cAAc,CAAC,wBAAwB,CAAC;SACnD,CAAC;QACF,WAAW,GAAG,gBAAgB,CAAC,YAAY,CAAC,IAAI,cAAc,CAAC,SAAS,CAAC,CAAC;IAC5E,CAAC;IAED,MAAM,OAAO,GAAiB,EAAE,GAAG,WAAW,EAAkB,CAAC;IAEjE,IAAI,OAAO,CAAC,GAAG,CAAC,8BAA8B,IAAI,OAAO,CAAC,GAAG,CAAC,oBAAoB,EAAE,CAAC;QACnF,MAAM,SAAS,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,8BAA8B,IAAI,OAAO,CAAC,GAAG,CAAC,oBAAoB,IAAI,GAAG,EAAE,EAAE,CAAC,CAAC;QACtH,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;YACvC,MAAM,iBAAiB,GAAG,OAAO,CAAC,SAAS,CAAC;YAC5C,MAAM,YAAY,GAAG,SAAS,GAAG,iBAAiB,CAAC;YAEnD,OAAO,CAAC,SAAS,GAAG,SAAS,CAAC;YAC9B,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC,CAAC;YACrD,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,cAAc,GAAG,YAAY,CAAC,EAAE,EAAE,CAAC,CAAC;YACzF,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC,CAAC;YACtD,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,aAAa,GAAG,YAAY,CAAC,CAAC;YAEzE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,EAAE,CAAC;gBACjC,OAAO,CAAC,GAAG,CAAC,4BAA4B,SAAS,EAAE,CAAC,CAAC;gBACrD,OAAO,CAAC,GAAG,CAAC,+BAA+B,OAAO,CAAC,aAAa,eAAe,CAAC,CAAC;gBACjF,OAAO,CAAC,GAAG,CAAC,2BAA2B,OAAO,CAAC,cAAc,EAAE,CAAC,CAAC;gBACjE,OAAO,CAAC,GAAG,CAAC,iCAAiC,OAAO,CAAC,cAAc,eAAe,CAAC,CAAC;YACtF,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,OAAO,CAAC,GAAG,CAAC,8BAA8B,IAAI,OAAO,CAAC,GAAG,CAAC,oBAAoB,EAAE,CAAC;QACnF,MAAM,UAAU,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,8BAA8B,IAAI,OAAO,CAAC,GAAG,CAAC,oBAAoB,IAAI,GAAG,EAAE,EAAE,CAAC,CAAC;QACvH,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,UAAU,GAAG,CAAC,EAAE,CAAC;YACzC,OAAO,CAAC,UAAU,GAAG,UAAU,CAAC;YAChC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,EAAE,CAAC;gBACjC,OAAO,CAAC,GAAG,CAAC,4BAA4B,UAAU,EAAE,CAAC,CAAC;YACxD,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;QACtB,MAAM,EAAE,eAAe,EAAE,GAAG,MAAM,MAAM,CAAC,oBAAoB,CAAC,CAAC;QAC/D,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC,SAAS,IAAI,YAAY,CAAC,CAAC;QAEjE,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,EAAE,CAAC;gBACjC,OAAO,CAAC,IAAI,CAAC,iCAAiC,SAAS,8BAA8B,CAAC,CAAC;YACzF,CAAC;YACD,OAAO,CAAC,SAAS,GAAG,KAAK,CAAC;YAC1B,OAAO,CAAC,YAAY,GAAG,SAAS,CAAC;QACnC,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,YAAY,GAAG,OAAO,CAAC;QACjC,CAAC;IACH,CAAC;IAED,OAAO,OAAuB,CAAC;AACjC,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,OAAqB;IAOjD,IAAI,OAAO,CAAC,SAAS,IAAI,OAAO,CAAC,YAAY,EAAE,CAAC;QAC9C,OAAO;YACL,OAAO,EAAE,OAAO,CAAC,aAAa;YAC9B,GAAG,EAAE,OAAO,CAAC,cAAc;YAC3B,GAAG,EAAE,OAAO,CAAC,cAAc;YAC3B,OAAO,EAAE,OAAO,CAAC,aAAa;YAC9B,IAAI,EAAE,QAAQ;SACf,CAAC;IACJ,CAAC;IACD,OAAO;QACL,OAAO,EAAE,OAAO,CAAC,YAAY;QAC7B,GAAG,EAAE,OAAO,CAAC,aAAa;QAC1B,GAAG,EAAE,OAAO,CAAC,aAAa;QAC1B,OAAO,EAAE,OAAO,CAAC,YAAY;QAC7B,IAAI,EAAE,YAAY;KACnB,CAAC;AACJ,CAAC"}