@noyrax/5d-database-plugin 0.1.14-beta.10 → 0.1.14-beta.12

package/out/cli/debug-semantic-search.js

@@ -0,0 +1,337 @@
+#!/usr/bin/env node
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+const path = __importStar(require("path"));
+const fs = __importStar(require("fs"));
+const dotenv_1 = require("dotenv");
+const multi_db_manager_1 = require("../core/multi-db-manager");
+const semantic_search_api_1 = require("../api/semantic-search-api");
+const embedding_generator_1 = require("../embedding/embedding-generator");
+const adr_api_1 = require("../api/adr-api");
+const query_rewriter_1 = require("../services/query-rewriter");
+const docs_path_resolver_1 = require("../core/docs-path-resolver");
+const embedding_pipeline_1 = require("../embedding/embedding-pipeline");
+/**
+ * Loads .env file from workspace root or parent directories.
+ */
+function loadEnvFile(workspaceRoot) {
+    let currentPath = workspaceRoot;
+    const maxDepth = 5;
+    for (let depth = 0; depth < maxDepth; depth++) {
+        const envPath = path.join(currentPath, '.env');
+        if (fs.existsSync(envPath)) {
+            (0, dotenv_1.config)({ path: envPath });
+            return;
+        }
+        const parentPath = path.dirname(currentPath);
+        if (parentPath === currentPath) {
+            break;
+        }
+        currentPath = parentPath;
+    }
+}
+/**
+ * Validates that docs/ directory exists.
+ */
+function validateDocsDirectory(workspaceRoot) {
+    const docsPath = docs_path_resolver_1.DocsPathResolver.findDocsDirectoryFromPath(workspaceRoot);
+    if (!docsPath) {
+        console.error('ERROR: docs/ directory not found in workspace or parent directories.');
+        console.error('Please run Documentation System Plugin (Noyrax) first to generate docs/ directory.');
+        return false;
+    }
+    return true;
+}
+/**
+ * Gets embedding content for an ADR.
+ */
+async function getAdrEmbeddingContent(adrNumber, pluginId, dbManager, embeddingPipeline) {
+    const adrApi = new adr_api_1.AdrApi(dbManager);
+    const adr = await adrApi.getAdrByNumber(adrNumber, pluginId);
+    if (!adr) {
+        return null;
+    }
+    // Extract content using the same logic as EmbeddingPipeline
+    const adrDb = await dbManager.getDatabase('W');
+    const { AdrRepository } = await Promise.resolve().then(() => __importStar(require('../repositories/adr-repository')));
+    const adrRepo = new AdrRepository(adrDb);
+    const fileMappings = await adrRepo.getAdrFileMappings(adr.id);
+    const filesStr = fileMappings.length > 0
+        ? `\n\nLinked files: ${fileMappings.map(m => m.file_path).join(', ')}`
+        : '';
+    return `${adr.title}\n${adr.content_markdown}${filesStr}`;
+}
+/**
+ * Command: content - Shows embedding content for an entity.
+ */
+async function commandContent(args, workspaceRoot, dbManager, pluginId) {
+    if (args.length < 2) {
+        console.error('Usage: debug-semantic-search <workspace-root> content <dimension> <entity-id>');
+        console.error('Example: debug-semantic-search . content W 0016');
+        process.exit(1);
+    }
+    const dimension = args[0].toUpperCase();
+    const entityId = args[1];
+    if (dimension !== 'W') {
+        console.error('Currently only W (ADRs) dimension is supported for content command');
+        process.exit(1);
+    }
+    const embeddingGenerator = new embedding_generator_1.EmbeddingGenerator();
+    const embeddingPipeline = new embedding_pipeline_1.EmbeddingPipeline(dbManager, embeddingGenerator);
+    const content = await getAdrEmbeddingContent(entityId, pluginId, dbManager, embeddingPipeline);
+    if (!content) {
+        console.error(`ADR-${entityId} not found`);
+        process.exit(1);
+    }
+    console.log(JSON.stringify({
+        dimension,
+        entityId,
+        content,
+        contentLength: content.length,
+        estimatedTokens: Math.ceil(content.length / 4) // Rough estimate
+    }, null, 2));
+}
+/**
+ * Command: scores - Analyzes semantic search scores.
+ */
+async function commandScores(args, workspaceRoot, dbManager, pluginId) {
+    if (args.length < 1) {
+        console.error('Usage: debug-semantic-search <workspace-root> scores <query> [--dimension W] [--limit 20]');
+        process.exit(1);
+    }
+    let query = args[0];
+    let dimension = undefined;
+    let limit = 20;
+    // Parse optional arguments
+    for (let i = 1; i < args.length; i++) {
+        if (args[i] === '--dimension' && i + 1 < args.length) {
+            const dim = args[i + 1].toUpperCase();
+            if (['X', 'Y', 'Z', 'W', 'T'].includes(dim)) {
+                dimension = [dim];
+            }
+            i++;
+        }
+        else if (args[i] === '--limit' && i + 1 < args.length) {
+            limit = parseInt(args[i + 1], 10);
+            if (isNaN(limit)) {
+                console.error('Error: --limit must be a number');
+                process.exit(1);
+            }
+            i++;
+        }
+    }
+    const embeddingGenerator = new embedding_generator_1.EmbeddingGenerator();
+    const semanticSearchApi = new semantic_search_api_1.SemanticSearchApi(dbManager, embeddingGenerator);
+    const results = await semanticSearchApi.search(query, pluginId, {
+        dimensions: dimension,
+        limit
+    });
+    console.log(JSON.stringify({
+        query,
+        dimension: dimension || ['X', 'Y', 'Z', 'W', 'T'],
+        limit,
+        resultCount: results.length,
+        results: results.map(r => ({
+            dimension: r.dimension,
+            externalId: r.externalId,
+            score: r.score,
+            vectorScore: r.vectorScore,
+            importanceScore: r.importanceScore
+        }))
+    }, null, 2));
+}
+/**
+ * Command: test - Tests multiple query variations for an ADR.
+ */
+async function commandTest(args, workspaceRoot, dbManager, pluginId) {
+    if (args.length < 1) {
+        console.error('Usage: debug-semantic-search <workspace-root> test <adr-number>');
+        console.error('Example: debug-semantic-search . test 0016');
+        process.exit(1);
+    }
+    const adrNumber = args[0];
+    // Generate test queries
+    const testQueries = [
+        `ADR-${adrNumber}`,
+        `ADR ${adrNumber}`,
+        adrNumber,
+        `Why was ADR-${adrNumber} created?`,
+        `What is ADR-${adrNumber} about?`,
+        `ADR-${adrNumber} migration`,
+        `TypeScript-only migration`,
+        `migration TypeScript`
+    ];
+    const embeddingGenerator = new embedding_generator_1.EmbeddingGenerator();
+    const semanticSearchApi = new semantic_search_api_1.SemanticSearchApi(dbManager, embeddingGenerator);
+    const results = [];
+    for (const query of testQueries) {
+        const searchResults = await semanticSearchApi.search(query, pluginId, {
+            dimensions: ['W'],
+            limit: 20
+        });
+        const foundIndex = searchResults.findIndex(r => r.externalId === adrNumber || r.externalId === adrNumber.padStart(3, '0') || r.externalId === adrNumber.replace(/^0+/, ''));
+        const found = foundIndex >= 0;
+        results.push({
+            query,
+            found,
+            rank: found ? foundIndex + 1 : undefined,
+            score: found ? searchResults[foundIndex].score : undefined
+        });
+    }
+    console.log(JSON.stringify({
+        adrNumber,
+        testQueries: results
+    }, null, 2));
+}
+/**
+ * Command: rewrite - Tests query rewriting.
+ */
+async function commandRewrite(args, workspaceRoot, dbManager, pluginId) {
+    if (args.length < 1) {
+        console.error('Usage: debug-semantic-search <workspace-root> rewrite <query>');
+        console.error('Example: debug-semantic-search . rewrite "Why was Python removed?"');
+        process.exit(1);
+    }
+    const query = args.join(' ');
+    const queryRewriter = new query_rewriter_1.QueryRewriter();
+    const rewritten = queryRewriter.rewriteQuery(query);
+    const adrNumbers = queryRewriter.extractAdrNumbers(query);
+    console.log(JSON.stringify({
+        originalQuery: query,
+        rewrittenQuery: rewritten,
+        extractedAdrNumbers: adrNumbers,
+        changed: query !== rewritten
+    }, null, 2));
+}
+/**
+ * CLI tool for debugging semantic search and discovery.
+ * Usage: debug-semantic-search <workspace-root> <command> <args...>
+ *
+ * Commands:
+ *   content <dimension> <entity-id>  - Show embedding content for an entity
+ *   scores <query> [--dimension W] [--limit 20] - Analyze semantic search scores
+ *   test <adr-number>                - Test multiple query variations for an ADR
+ *   rewrite <query>                  - Test query rewriting
+ */
+async function main() {
+    const args = process.argv.slice(2);
+    if (args.length < 2) {
+        console.error('Usage: debug-semantic-search <workspace-root> <command> <args...>');
+        console.error('');
+        console.error('Commands:');
+        console.error('  content <dimension> <entity-id>  - Show embedding content for an entity');
+        console.error('  scores <query> [--dimension W] [--limit 20] - Analyze semantic search scores');
+        console.error('  test <adr-number>                - Test multiple query variations for an ADR');
+        console.error('  rewrite <query>                  - Test query rewriting');
+        process.exit(1);
+    }
+    // Auto-detect workspace root if not provided (first arg is command)
+    let workspaceRoot;
+    let command;
+    let commandArgs;
+    // Check if first arg is a command (no path separators) or a workspace root
+    if (args.length >= 2 && !args[0].includes('/') && !args[0].includes('\\') && !path.isAbsolute(args[0]) &&
+        ['content', 'scores', 'test', 'rewrite'].includes(args[0])) {
+        // First arg is command, use current directory as workspace root
+        workspaceRoot = process.cwd();
+        command = args[0];
+        commandArgs = args.slice(1);
+    }
+    else if (args.length >= 3) {
+        // First arg is workspace root, second is command
+        workspaceRoot = path.resolve(args[0]);
+        command = args[1];
+        commandArgs = args.slice(2);
+    }
+    else {
+        console.error('Usage: debug-semantic-search <workspace-root> <command> <args...>');
+        process.exit(1);
+    }
+    // Validate workspace root exists
+    if (!fs.existsSync(workspaceRoot)) {
+        console.error(`ERROR: Workspace root does not exist: ${workspaceRoot}`);
+        process.exit(1);
+    }
+    // Load .env file
+    loadEnvFile(workspaceRoot);
+    // Check for OpenAI API key (required for some commands)
+    if (!process.env.OPENAI_API_KEY && (command === 'scores' || command === 'test')) {
+        console.error('ERROR: OPENAI_API_KEY not found in environment or .env file');
+        console.error('Semantic search requires OpenAI API key for embedding generation.');
+        process.exit(1);
+    }
+    // Validate docs/ directory
+    if (!validateDocsDirectory(workspaceRoot)) {
+        process.exit(1);
+    }
+    const dbManager = new multi_db_manager_1.MultiDbManager(workspaceRoot);
+    const pluginId = dbManager.getPluginId();
+    try {
+        switch (command) {
+            case 'content':
+                await commandContent(commandArgs, workspaceRoot, dbManager, pluginId);
+                break;
+            case 'scores':
+                await commandScores(commandArgs, workspaceRoot, dbManager, pluginId);
+                break;
+            case 'test':
+                await commandTest(commandArgs, workspaceRoot, dbManager, pluginId);
+                break;
+            case 'rewrite':
+                await commandRewrite(commandArgs, workspaceRoot, dbManager, pluginId);
+                break;
+            default:
+                console.error(`Unknown command: ${command}`);
+                console.error('Use "content", "scores", "test", or "rewrite"');
+                process.exit(1);
+        }
+    }
+    catch (error) {
+        const errorMsg = error instanceof Error ? error.message : String(error);
+        console.error(`Error: ${errorMsg}`);
+        if (error instanceof Error && error.stack) {
+            console.error(`Stack trace: ${error.stack}`);
+        }
+        process.exit(1);
+    }
+}
+if (require.main === module) {
+    main().catch((error) => {
+        console.error('Fatal error:', error);
+        process.exit(1);
+    });
+}
+//# sourceMappingURL=debug-semantic-search.js.map

package/out/cli/debug-semantic-search.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"debug-semantic-search.js","sourceRoot":"","sources":["../../src/cli/debug-semantic-search.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEA,2CAA6B;AAC7B,uCAAyB;AACzB,mCAAgC;AAChC,+DAA0D;AAC1D,oEAA+D;AAC/D,0EAAsE;AAEtE,4CAAwC;AACxC,+DAA2D;AAC3D,mEAA8D;AAC9D,wEAAoE;AAEpE;;GAEG;AACH,SAAS,WAAW,CAAC,aAAqB;IACtC,IAAI,WAAW,GAAG,aAAa,CAAC;IAChC,MAAM,QAAQ,GAAG,CAAC,CAAC;IAEnB,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,QAAQ,EAAE,KAAK,EAAE,EAAE,CAAC;QAC5C,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;QAC/C,IAAI,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;YACzB,IAAA,eAAM,EAAC,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC;YAC1B,OAAO;QACX,CAAC;QAED,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;QAC7C,IAAI,UAAU,KAAK,WAAW,EAAE,CAAC;YAC7B,MAAM;QACV,CAAC;QACD,WAAW,GAAG,UAAU,CAAC;IAC7B,CAAC;AACL,CAAC;AAED;;GAEG;AACH,SAAS,qBAAqB,CAAC,aAAqB;IAChD,MAAM,QAAQ,GAAG,qCAAgB,CAAC,yBAAyB,CAAC,aAAa,CAAC,CAAC;IAC3E,IAAI,CAAC,QAAQ,EAAE,CAAC;QACZ,OAAO,CAAC,KAAK,CAAC,sEAAsE,CAAC,CAAC;QACtF,OAAO,CAAC,KAAK,CAAC,oFAAoF,CAAC,CAAC;QACpG,OAAO,KAAK,CAAC;IACjB,CAAC;IACD,OAAO,IAAI,CAAC;AAChB,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,sBAAsB,CAAC,SAAiB,EAAE,QAAgB,EAAE,SAAyB,EAAE,iBAAoC;IACtI,MAAM,MAAM,GAAG,IAAI,gBAAM,CAAC,SAAS,CAAC,CAAC;IACrC,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,cAAc,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;IAE7D,IAAI,CAAC,GAAG,EAAE,CAAC;QACP,OAAO,IAAI,CAAC;IAChB,CAAC;IAED,4DAA4D;IAC5D,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;IAC/C,MAAM,EAAE,aAAa,EAAE,GAAG,wDAAa,gCAAgC,GAAC,CAAC;IACzE,MAAM,OAAO,GAAG,IAAI,aAAa,CAAC,KAAK,CAAC,CAAC;IACzC,MAAM,YAAY,GAAG,MAAM,OAAO,CAAC,kBAAkB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAE9D,MAAM,QAAQ,GAAG,YAAY,CAAC,MAAM,GAAG,CAAC;QACpC,CAAC,CAAC,qBAAqB,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;QACtE,CAAC,CAAC,EAAE,CAAC;IAET,OAAO,GAAG,GAAG,CAAC,KAAK,KAAK,GAAG,CAAC,gBAAgB,GAAG,QAAQ,EAAE,CAAC;AAC9D,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,cAAc,CAAC,IAAc,EAAE,aAAqB,EAAE,SAAyB,EAAE,QAAgB;IAC5G,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAClB,OAAO,CAAC,KAAK,CAAC,+EAA+E,CAAC,CAAC;QAC/F,OAAO,CAAC,KAAK,CAAC,iDAAiD,CAAC,CAAC;QACjE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACpB,CAAC;IAED,MAAM,SAAS,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;IACxC,MAAM,QAAQ,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;IAEzB,IAAI,SAAS,KAAK,GAAG,EAAE,CAAC;QACpB,OAAO,CAAC,KAAK,CAAC,oEAAoE,CAAC,CAAC;QACpF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACpB,CAAC;IAED,MAAM,kBAAkB,GAAG,IAAI,wCAAkB,EAAE,CAAC;IACpD,MAAM,iBAAiB,GAAG,IAAI,sCAAiB,CAAC,SAAS,EAAE,kBAAkB,CAAC,CAAC;IAE/E,MAAM,OAAO,GAAG,MAAM,sBAAsB,CAAC,QAAQ,EAAE,QAAQ,EAAE,SAAS,EAAE,iBAAiB,CAAC,CAAC;IAE/F,IAAI,CAAC,OAAO,EAAE,CAAC;QACX,OAAO,CAAC,KAAK,CAAC,OAAO,QAAQ,YAAY,CAAC,CAAC;QAC3C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACpB,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC;QACvB,SAAS;QACT,QAAQ;QACR,OAAO;QACP,aAAa,EAAE,OAAO,CAAC,MAAM;QAC7B,eAAe,EAAE,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,iBAAiB;KACnE,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;AACjB,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,aAAa,CAAC,IAAc,EAAE,aAAqB,EAAE,SAAyB,EAAE,QAAgB;IAC3G,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAClB,OAAO,CAAC,KAAK,CAAC,2FAA2F,CAAC,CAAC;QAC3G,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACpB,CAAC;IAED,IAAI,KAAK,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;IACpB,IAAI,SAAS,GAAgD,SAAS,CAAC;IACvE,IAAI,KAAK,GAAG,EAAE,CAAC;IAEf,2BAA2B;IAC3B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACnC,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,aAAa,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;YACnD,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;YACtC,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC1C,SAAS,GAAG,CAAC,GAAkC,CAAC,CAAC;YACrD,CAAC;YACD,CAAC,EAAE,CAAC;QACR,CAAC;aAAM,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,SAAS,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;YACtD,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAClC,IAAI,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;gBACf,OAAO,CAAC,KAAK,CAAC,iCAAiC,CAAC,CAAC;gBACjD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACpB,CAAC;YACD,CAAC,EAAE,CAAC;QACR,CAAC;IACL,CAAC;IAED,MAAM,kBAAkB,GAAG,IAAI,wCAAkB,EAAE,CAAC;IACpD,MAAM,iBAAiB,GAAG,IAAI,uCAAiB,CAAC,SAAS,EAAE,kBAAkB,CAAC,CAAC;IAE/E,MAAM,OAAO,GAAG,MAAM,iBAAiB,CAAC,MAAM,CAAC,KAAK,EAAE,QAAQ,EAAE;QAC5D,UAAU,EAAE,SAAS;QACrB,KAAK;KACR,CAAC,CAAC;IAEH,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC;QACvB,KAAK;QACL,SAAS,EAAE,SAAS,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC;QACjD,KAAK;QACL,WAAW,EAAE,OAAO,CAAC,MAAM;QAC3B,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YACvB,SAAS,EAAE,CAAC,CAAC,SAAS;YACtB,UAAU,EAAE,CAAC,CAAC,UAAU;YACxB,KAAK,EAAE,CAAC,CAAC,KAAK;YACd,WAAW,EAAE,CAAC,CAAC,WAAW;YAC1B,eAAe,EAAE,CAAC,CAAC,eAAe;SACrC,CAAC,CAAC;KACN,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;AACjB,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,WAAW,CAAC,IAAc,EAAE,aAAqB,EAAE,SAAyB,EAAE,QAAgB;IACzG,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAClB,OAAO,CAAC,KAAK,CAAC,iEAAiE,CAAC,CAAC;QACjF,OAAO,CAAC,KAAK,CAAC,4CAA4C,CAAC,CAAC;QAC5D,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACpB,CAAC;IAED,MAAM,SAAS,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;IAE1B,wBAAwB;IACxB,MAAM,WAAW,GAAG;QAChB,OAAO,SAAS,EAAE;QAClB,OAAO,SAAS,EAAE;QAClB,SAAS;QACT,eAAe,SAAS,WAAW;QACnC,eAAe,SAAS,SAAS;QACjC,OAAO,SAAS,YAAY;QAC5B,2BAA2B;QAC3B,sBAAsB;KACzB,CAAC;IAEF,MAAM,kBAAkB,GAAG,IAAI,wCAAkB,EAAE,CAAC;IACpD,MAAM,iBAAiB,GAAG,IAAI,uCAAiB,CAAC,SAAS,EAAE,kBAAkB,CAAC,CAAC;IAE/E,MAAM,OAAO,GAKR,EAAE,CAAC;IAER,KAAK,MAAM,KAAK,IAAI,WAAW,EAAE,CAAC;QAC9B,MAAM,aAAa,GAAG,MAAM,iBAAiB,CAAC,MAAM,CAAC,KAAK,EAAE,QAAQ,EAAE;YAClE,UAAU,EAAE,CAAC,GAAG,CAAC;YACjB,KAAK,EAAE,EAAE;SACZ,CAAC,CAAC;QAEH,MAAM,UAAU,GAAG,aAAa,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,UAAU,KAAK,SAAS,IAAI,CAAC,CAAC,UAAU,KAAK,SAAS,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC,UAAU,KAAK,SAAS,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC;QAC5K,MAAM,KAAK,GAAG,UAAU,IAAI,CAAC,CAAC;QAE9B,OAAO,CAAC,IAAI,CAAC;YACT,KAAK;YACL,KAAK;YACL,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS;YACxC,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS;SAC7D,CAAC,CAAC;IACP,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC;QACvB,SAAS;QACT,WAAW,EAAE,OAAO;KACvB,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;AACjB,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,cAAc,CAAC,IAAc,EAAE,aAAqB,EAAE,SAAyB,EAAE,QAAgB;IAC5G,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAClB,OAAO,CAAC,KAAK,CAAC,+DAA+D,CAAC,CAAC;QAC/E,OAAO,CAAC,KAAK,CAAC,oEAAoE,CAAC,CAAC;QACpF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACpB,CAAC;IAED,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC7B,MAAM,aAAa,GAAG,IAAI,8BAAa,EAAE,CAAC;IAE1C,MAAM,SAAS,GAAG,aAAa,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC;IACpD,MAAM,UAAU,GAAG,aAAa,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC;IAE1D,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC;QACvB,aAAa,EAAE,KAAK;QACpB,cAAc,EAAE,SAAS;QACzB,mBAAmB,EAAE,UAAU;QAC/B,OAAO,EAAE,KAAK,KAAK,SAAS;KAC/B,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;AACjB,CAAC;AAED;;;;;;;;;GASG;AACH,KAAK,UAAU,IAAI;IACf,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAEnC,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAClB,OAAO,CAAC,KAAK,CAAC,mEAAmE,CAAC,CAAC;QACnF,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAClB,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;QAC3B,OAAO,CAAC,KAAK,CAAC,2EAA2E,CAAC,CAAC;QAC3F,OAAO,CAAC,KAAK,CAAC,gFAAgF,CAAC,CAAC;QAChG,OAAO,CAAC,KAAK,CAAC,gFAAgF,CAAC,CAAC;QAChG,OAAO,CAAC,KAAK,CAAC,2DAA2D,CAAC,CAAC;QAC3E,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACpB,CAAC;IAED,oEAAoE;IACpE,IAAI,aAAqB,CAAC;IAC1B,IAAI,OAAe,CAAC;IACpB,IAAI,WAAqB,CAAC;IAE1B,2EAA2E;IAC3E,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClG,CAAC,SAAS,EAAE,QAAQ,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC7D,gEAAgE;QAChE,aAAa,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;QAC9B,OAAO,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAChC,CAAC;SAAM,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;QAC1B,iDAAiD;QACjD,aAAa,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;QACtC,OAAO,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAChC,CAAC;SAAM,CAAC;QACJ,OAAO,CAAC,KAAK,CAAC,mEAAmE,CAAC,CAAC;QACnF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACpB,CAAC;IAED,iCAAiC;IACjC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,aAAa,CAAC,EAAE,CAAC;QAChC,OAAO,CAAC,KAAK,CAAC,yCAAyC,aAAa,EAAE,CAAC,CAAC;QACxE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACpB,CAAC;IAED,iBAAiB;IACjB,WAAW,CAAC,aAAa,CAAC,CAAC;IAE3B,wDAAwD;IACxD,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,IAAI,CAAC,OAAO,KAAK,QAAQ,IAAI,OAAO,KAAK,MAAM,CAAC,EAAE,CAAC;QAC9E,OAAO,CAAC,KAAK,CAAC,6DAA6D,CAAC,CAAC;QAC7E,OAAO,CAAC,KAAK,CAAC,mEAAmE,CAAC,CAAC;QACnF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACpB,CAAC;IAED,2BAA2B;IAC3B,IAAI,CAAC,qBAAqB,CAAC,aAAa,CAAC,EAAE,CAAC;QACxC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACpB,CAAC;IAED,MAAM,SAAS,GAAG,IAAI,iCAAc,CAAC,aAAa,CAAC,CAAC;IACpD,MAAM,QAAQ,GAAG,SAAS,CAAC,WAAW,EAAE,CAAC;IAEzC,IAAI,CAAC;QACD,QAAQ,OAAO,EAAE,CAAC;YACd,KAAK,SAAS;gBACV,MAAM,cAAc,CAAC,WAAW,EAAE,aAAa,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC;gBACtE,MAAM;YACV,KAAK,QAAQ;gBACT,MAAM,aAAa,CAAC,WAAW,EAAE,aAAa,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC;gBACrE,MAAM;YACV,KAAK,MAAM;gBACP,MAAM,WAAW,CAAC,WAAW,EAAE,aAAa,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC;gBACnE,MAAM;YACV,KAAK,SAAS;gBACV,MAAM,cAAc,CAAC,WAAW,EAAE,aAAa,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC;gBACtE,MAAM;YACV;gBACI,OAAO,CAAC,KAAK,CAAC,oBAAoB,OAAO,EAAE,CAAC,CAAC;gBAC7C,OAAO,CAAC,KAAK,CAAC,+CAA+C,CAAC,CAAC;gBAC/D,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACxB,CAAC;IACL,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACb,MAAM,QAAQ,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACxE,OAAO,CAAC,KAAK,CAAC,UAAU,QAAQ,EAAE,CAAC,CAAC;QACpC,IAAI,KAAK,YAAY,KAAK,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;YACxC,OAAO,CAAC,KAAK,CAAC,gBAAgB,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC;QACjD,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACpB,CAAC;AACL,CAAC;AAED,IAAI,OAAO,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;IAC1B,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;QACnB,OAAO,CAAC,KAAK,CAAC,cAAc,EAAE,KAAK,CAAC,CAAC;QACrC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACpB,CAAC,CAAC,CAAC;AACP,CAAC"}

package/out/embedding/embedding-pipeline.d.ts

@@ -39,9 +39,18 @@ export declare class EmbeddingPipeline {
      * Computes content hash for change detection.
      */
     private computeContentHash;
+    /**
+     * Estimates token count from content length (Markdown-optimized).
+     * Markdown has different token densities:
+     * - Code blocks: ~3 chars/token (more tokens per char due to formatting)
+     * - Normal text: ~4 chars/token (standard estimate)
+     * - Tables: ~2.5 chars/token (many separators increase token count)
+     */
+    private estimateTokens;
     /**
      * Optimizes module documentation for embedding while preserving semantic meaning.
      * Strategy: Keep structure and important information, remove detailed tables.
+     * Aggressively truncates code blocks to signatures only, limits table rows.
      *
      * @param content Full markdown content
      * @param maxTokens Maximum tokens allowed
@@ -51,6 +60,8 @@ export declare class EmbeddingPipeline {
     /**
      * Extracts only the structure of a module (header, interface/method names, signatures).
      * Used for hierarchical embeddings where details are in Y-Dimension (Symbols).
+     * Only keeps: Headers, interface/method/class/variable names, first line of code blocks (signatures).
+     * Removes: All tables, code block bodies, normal text.
      *
      * @param content Full markdown content
      * @returns Module structure (header + names + signatures, no details)

package/out/embedding/embedding-pipeline.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"embedding-pipeline.d.ts","sourceRoot":"","sources":["../../src/embedding/embedding-pipeline.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAC;AAC1D,OAAO,EAAE,kBAAkB,EAAE,MAAM,uBAAuB,CAAC;AAgB3D;;;GAGG;AACH,qBAAa,iBAAiB;IAC1B,OAAO,CAAC,SAAS,CAAiB;IAClC,OAAO,CAAC,kBAAkB,CAAqB;IAC/C,OAAO,CAAC,SAAS,CAAY;IAC7B,OAAO,CAAC,SAAS,CAAY;IAC7B,OAAO,CAAC,aAAa,CAAgB;IACrC,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,SAAS,CAAY;IAC7B,OAAO,CAAC,gBAAgB,CAAmB;gBAE/B,SAAS,EAAE,cAAc,EAAE,kBAAkB,EAAE,kBAAkB;IAW7E;;;OAGG;IACG,cAAc,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAoBrD;;OAEG;IACG,aAAa,CACf,SAAS,EAAE,GAAG,GAAG,GAAG,GAAG,GAAG,GAAG,GAAG,GAAG,GAAG,EACtC,QAAQ,EAAE,MAAM,EAChB,KAAK,EAAE,MAAM,GACd,OAAO,CAAC,IAAI,CAAC;IAkJhB;;OAEG;YACW,uBAAuB;IAoBrC;;OAEG;YACW,0BAA0B;IAwExC;;OAEG;IACH,OAAO,CAAC,aAAa;IAgBrB;;OAEG;IACH,OAAO,CAAC,kBAAkB;IAI1B;;;;;;;OAOG;IACH,OAAO,CAAC,iCAAiC;IAkGzC;;;;;;OAMG;IACH,OAAO,CAAC,sBAAsB;CA8DjC"}
+{"version":3,"file":"embedding-pipeline.d.ts","sourceRoot":"","sources":["../../src/embedding/embedding-pipeline.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAC;AAC1D,OAAO,EAAE,kBAAkB,EAAE,MAAM,uBAAuB,CAAC;AAgB3D;;;GAGG;AACH,qBAAa,iBAAiB;IAC1B,OAAO,CAAC,SAAS,CAAiB;IAClC,OAAO,CAAC,kBAAkB,CAAqB;IAC/C,OAAO,CAAC,SAAS,CAAY;IAC7B,OAAO,CAAC,SAAS,CAAY;IAC7B,OAAO,CAAC,aAAa,CAAgB;IACrC,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,SAAS,CAAY;IAC7B,OAAO,CAAC,gBAAgB,CAAmB;gBAE/B,SAAS,EAAE,cAAc,EAAE,kBAAkB,EAAE,kBAAkB;IAW7E;;;OAGG;IACG,cAAc,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAoBrD;;OAEG;IACG,aAAa,CACf,SAAS,EAAE,GAAG,GAAG,GAAG,GAAG,GAAG,GAAG,GAAG,GAAG,GAAG,EACtC,QAAQ,EAAE,MAAM,EAChB,KAAK,EAAE,MAAM,GACd,OAAO,CAAC,IAAI,CAAC;IAyJhB;;OAEG;YACW,uBAAuB;IAoBrC;;OAEG;YACW,0BAA0B;IA6FxC;;OAEG;IACH,OAAO,CAAC,aAAa;IAgBrB;;OAEG;IACH,OAAO,CAAC,kBAAkB;IAI1B;;;;;;OAMG;IACH,OAAO,CAAC,cAAc;IAoBtB;;;;;;;;OAQG;IACH,OAAO,CAAC,iCAAiC;IAkJzC;;;;;;;;OAQG;IACH,OAAO,CAAC,sBAAsB;CAwEjC"}