@softerist/heuristic-mcp 2.0.0

package/lib/ignore-patterns.js

@@ -0,0 +1,314 @@
+// Comprehensive ignore patterns based on industry best practices
+// Researched from gitignore templates and development community standards
+
+export const IGNORE_PATTERNS = {
+  // JavaScript/Node.js
+  javascript: [
+    '**/node_modules/**',
+    '**/.next/**',
+    '**/dist/**',
+    '**/build/**',
+    '**/.nuxt/**',
+    '**/.output/**',
+    '**/.vercel/**',
+    '**/.netlify/**',
+    '**/out/**',
+    '**/coverage/**',
+    '**/.nyc_output/**',
+    '**/npm-debug.log*',
+    '**/yarn-debug.log*',
+    '**/yarn-error.log*',
+    '**/.pnpm-store/**',
+    '**/.turbo/**'
+  ],
+
+  // Python
+  python: [
+    '**/__pycache__/**',
+    '**/*.pyc',
+    '**/*.pyd',
+    '**/.Python',
+    '**/build/**',
+    '**/develop-eggs/**',
+    '**/dist/**',
+    '**/downloads/**',
+    '**/eggs/**',
+    '**/.eggs/**',
+    '**/lib/**',
+    '**/lib64/**',
+    '**/parts/**',
+    '**/sdist/**',
+    '**/var/**',
+    '**/*.egg-info/**',
+    '**/.installed.cfg',
+    '**/*.egg',
+    '**/.venv/**',
+    '**/venv/**',
+    '**/env/**',
+    '**/ENV/**',
+    '**/.pytest_cache/**',
+    '**/htmlcov/**',
+    '**/.tox/**',
+    '**/.coverage',
+    '**/.hypothesis/**',
+    '**/.mypy_cache/**',
+    '**/.ruff_cache/**'
+  ],
+
+  // Java/Maven
+  java: [
+    '**/target/**',
+    '**/.gradle/**',
+    '**/build/**',
+    '**/.idea/**',
+    '**/*.iml',
+    '**/out/**',
+    '**/gen/**',
+    '**/classes/**',
+    '**/.classpath',
+    '**/.project',
+    '**/.settings/**',
+    '**/.m2/**',
+    '**/*.class',
+    '**/*.jar',
+    '**/*.war',
+    '**/*.ear'
+  ],
+
+  // Android
+  android: [
+    '**/.gradle/**',
+    '**/build/**',
+    '**/.idea/**',
+    '**/*.iml',
+    '**/local.properties',
+    '**/captures/**',
+    '**/.externalNativeBuild/**',
+    '**/.cxx/**',
+    '**/*.apk',
+    '**/*.aar',
+    '**/*.ap_',
+    '**/*.dex',
+    '**/google-services.json',
+    '**/gradle-app.setting',
+    '**/.navigation/**'
+  ],
+
+  // iOS/Swift
+  ios: [
+    '**/Pods/**',
+    '**/DerivedData/**',
+    '**/xcuserdata/**',
+    '**/*.xcarchive',
+    '**/build/**',
+    '**/.build/**',
+    '**/Packages/**',
+    '**/.swiftpm/**',
+    '**/Carthage/Build/**',
+    '**/fastlane/report.xml',
+    '**/fastlane/Preview.html',
+    '**/fastlane/screenshots/**',
+    '**/fastlane/test_output/**',
+    '**/*.moved-aside',
+    '**/*.xcuserstate',
+    '**/*.hmap',
+    '**/*.ipa'
+  ],
+
+  // Go
+  go: [
+    '**/vendor/**',
+    '**/bin/**',
+    '**/pkg/**',
+    '**/*.exe',
+    '**/*.test',
+    '**/*.prof'
+  ],
+
+  // PHP
+  php: [
+    '**/vendor/**',
+    '**/composer.phar',
+    '**/composer.lock',
+    '**/.phpunit.result.cache'
+  ],
+
+  // Rust
+  rust: [
+    '**/target/**',
+    '**/Cargo.lock',
+    '**/*.rs.bk'
+  ],
+
+  // Ruby
+  ruby: [
+    '**/vendor/bundle/**',
+    '**/.bundle/**',
+    '**/Gemfile.lock',
+    '**/.byebug_history'
+  ],
+
+  // .NET/C#
+  dotnet: [
+    '**/bin/**',
+    '**/obj/**',
+    '**/packages/**',
+    '**/*.user',
+    '**/*.suo',
+    '**/.vs/**',
+    '**/node_modules/**'
+  ],
+
+  // Common (IDE, OS, Build tools)
+  common: [
+    // Version control
+    '**/.git/**',
+    '**/.svn/**',
+    '**/.hg/**',
+    '**/.bzr/**',
+    
+    // OS files
+    '**/.DS_Store',
+    '**/Thumbs.db',
+    '**/desktop.ini',
+    '**/$RECYCLE.BIN/**',
+    
+    // Backup files
+    '**/*.bak',
+    '**/*.backup',
+    '**/*~',
+    '**/*.swp',
+    '**/*.swo',
+    '**/*.swn',
+    '**/#*#',
+    '**/.#*',
+    
+    // Lock files (editor/runtime, not package managers)
+    '**/*.lock',
+    '**/.~lock*',
+    
+    // Logs
+    '**/*.log',
+    '**/logs/**',
+    '**/*.log.*',
+    
+    // IDEs and Editors
+    '**/.vscode/**',
+    '**/.idea/**',
+    '**/.sublime-project',
+    '**/.sublime-workspace',
+    '**/nbproject/**',
+    '**/.settings/**',
+    '**/.metadata/**',
+    '**/.classpath',
+    '**/.project',
+    '**/.c9/**',
+    '**/*.launch',
+    '**/*.tmproj',
+    '**/*.tmproject',
+    '**/tmtags',
+    
+    // Vim
+    '**/*~',
+    '**/*.swp',
+    '**/*.swo',
+    '**/.*.sw?',
+    '**/Session.vim',
+    
+    // Emacs
+    '**/*~',
+    '**/#*#',
+    '**/.#*',
+    
+    // Environment files (secrets)
+    '**/.env',
+    '**/.env.local',
+    '**/.env.*.local',
+    '**/.env.production',
+    '**/.env.development',
+    '**/.env.test',
+    '**/secrets.json',
+    '**/secrets.yaml',
+    '**/secrets.yml',
+    '**/*.key',
+    '**/*.pem',
+    '**/*.crt',
+    '**/*.cer',
+    '**/*.p12',
+    '**/*.pfx',
+    
+    // Temporary files
+    '**/tmp/**',
+    '**/temp/**',
+    '**/*.tmp',
+    '**/*.temp',
+    '**/.cache/**',
+    
+    // Session & runtime
+    '**/.sass-cache/**',
+    '**/connect.lock',
+    '**/*.pid',
+    '**/*.seed',
+    '**/*.pid.lock',
+    
+    // Coverage & test output
+    '**/coverage/**',
+    '**/.nyc_output/**',
+    '**/test-results/**',
+    '**/*.cover',
+    '**/*.coverage',
+    '**/htmlcov/**',
+    
+    // Documentation builds
+    '**/docs/_build/**',
+    '**/site/**',
+    
+    // Misc
+    '**/*.orig',
+    '**/core',
+    '**/*.core'
+  ]
+};
+
+// Map marker files to project types
+export const FILE_TYPE_MAP = {
+  // JavaScript/Node
+  'package.json': 'javascript',
+  'package-lock.json': 'javascript',
+  'yarn.lock': 'javascript',
+  'pnpm-lock.yaml': 'javascript',
+  
+  // Python
+  'requirements.txt': 'python',
+  'Pipfile': 'python',
+  'pyproject.toml': 'python',
+  'setup.py': 'python',
+  
+  // Android
+  'build.gradle': 'android',
+  'build.gradle.kts': 'android',
+  'settings.gradle': 'android',
+  
+  // Java
+  'pom.xml': 'java',
+  
+  // iOS
+  'Podfile': 'ios',
+  'Package.swift': 'ios',
+  
+  // Go
+  'go.mod': 'go',
+  
+  // PHP
+  'composer.json': 'php',
+  
+  // Rust
+  'Cargo.toml': 'rust',
+  
+  // Ruby
+  'Gemfile': 'ruby',
+  
+  // .NET
+  '*.csproj': 'dotnet',
+  '*.sln': 'dotnet'
+};

package/lib/project-detector.js

@@ -0,0 +1,75 @@
+import fs from "fs/promises";
+import path from "path";
+import { FILE_TYPE_MAP, IGNORE_PATTERNS } from "./ignore-patterns.js";
+
+export class ProjectDetector {
+  constructor(searchDirectory) {
+    this.searchDirectory = searchDirectory;
+    this.detectedTypes = new Set();
+  }
+
+  async detectProjectTypes() {
+    const markerFiles = Object.keys(FILE_TYPE_MAP);
+    
+    for (const marker of markerFiles) {
+      // Handle wildcard patterns like *.csproj
+      if (marker.includes('*')) {
+        await this.detectWithWildcard(marker);
+      } else {
+        await this.detectExactFile(marker);
+      }
+    }
+
+    return Array.from(this.detectedTypes);
+  }
+
+  async detectExactFile(markerFile) {
+    const markerPath = path.join(this.searchDirectory, markerFile);
+    try {
+      await fs.access(markerPath);
+      const projectType = FILE_TYPE_MAP[markerFile];
+      this.detectedTypes.add(projectType);
+      console.error(`[Detector] Detected ${projectType} project (${markerFile})`);
+    } catch {
+      // File doesn't exist, continue
+    }
+  }
+
+  async detectWithWildcard(pattern) {
+    try {
+      const files = await fs.readdir(this.searchDirectory);
+      const regex = new RegExp('^' + pattern.replace('*', '.*') + '$');
+      
+      for (const file of files) {
+        if (regex.test(file)) {
+          const projectType = FILE_TYPE_MAP[pattern];
+          this.detectedTypes.add(projectType);
+          console.error(`[Detector] Detected ${projectType} project (${file})`);
+          break;
+        }
+      }
+    } catch {
+      // Directory read failed, continue
+    }
+  }
+
+  getSmartIgnorePatterns() {
+    const patterns = [...IGNORE_PATTERNS.common];
+    
+    for (const type of this.detectedTypes) {
+      if (IGNORE_PATTERNS[type]) {
+        patterns.push(...IGNORE_PATTERNS[type]);
+      }
+    }
+
+    // Remove duplicates
+    return [...new Set(patterns)];
+  }
+
+  getSummary() {
+    return {
+      detectedTypes: Array.from(this.detectedTypes),
+      patternCount: this.getSmartIgnorePatterns().length
+    };
+  }
+}

package/lib/tokenizer.js

@@ -0,0 +1,142 @@
+/**
+ * Token estimation and limits for embedding models
+ * 
+ * This module provides token counting utilities and model-specific limits
+ * to ensure text chunks don't exceed the model's maximum sequence length.
+ */
+
+/**
+ * Token limits for supported embedding models
+ * Each model has its own maximum sequence length
+ */
+export const MODEL_TOKEN_LIMITS = {
+  // Sentence Transformers / MiniLM family
+  "Xenova/all-MiniLM-L6-v2": 256,
+  "Xenova/all-MiniLM-L12-v2": 256,
+  "Xenova/paraphrase-MiniLM-L6-v2": 128,
+  "Xenova/paraphrase-MiniLM-L3-v2": 128,
+  
+  // MPNet models
+  "Xenova/all-mpnet-base-v2": 384,
+  "Xenova/paraphrase-mpnet-base-v2": 384,
+  
+  // Multilingual models
+  "Xenova/paraphrase-multilingual-MiniLM-L12-v2": 128,
+  "Xenova/paraphrase-multilingual-mpnet-base-v2": 256,
+  
+  // Code-specific models
+  "Xenova/codebert-base": 512,
+  "Xenova/graphcodebert-base": 512,
+  
+  // E5 models
+  "Xenova/e5-small-v2": 512,
+  "Xenova/e5-base-v2": 512,
+  "Xenova/e5-large-v2": 512,
+  
+  // BGE models
+  "Xenova/bge-small-en-v1.5": 512,
+  "Xenova/bge-base-en-v1.5": 512,
+  "Xenova/bge-large-en-v1.5": 512,
+  
+  // Default fallback
+  "default": 256
+};
+
+/**
+ * Get the maximum token limit for a given model
+ * Case-insensitive lookup for robustness
+ * @param {string} modelName - The model name (e.g., "Xenova/all-MiniLM-L6-v2")
+ * @returns {number} Maximum tokens supported by the model
+ */
+export function getModelTokenLimit(modelName) {
+  if (!modelName) return MODEL_TOKEN_LIMITS["default"];
+  
+  // Direct match first (fastest)
+  if (MODEL_TOKEN_LIMITS[modelName] !== undefined) {
+    return MODEL_TOKEN_LIMITS[modelName];
+  }
+  
+  // Case-insensitive search
+  const normalizedName = modelName.toLowerCase();
+  for (const [key, value] of Object.entries(MODEL_TOKEN_LIMITS)) {
+    if (key.toLowerCase() === normalizedName) {
+      return value;
+    }
+  }
+  
+  return MODEL_TOKEN_LIMITS["default"];
+}
+
+/**
+ * Get chunking parameters for a model
+ * Returns target and overlap tokens based on the model's limit
+ * @param {string} modelName - The model name
+ * @returns {{ maxTokens: number, targetTokens: number, overlapTokens: number }}
+ */
+export function getChunkingParams(modelName) {
+  const maxTokens = getModelTokenLimit(modelName);
+  
+  // Target: 85% of max to leave safety buffer
+  const targetTokens = Math.floor(maxTokens * 0.85);
+  
+  // Overlap: 15-20% of target for context continuity
+  const overlapTokens = Math.floor(targetTokens * 0.18);
+  
+  return {
+    maxTokens,
+    targetTokens,
+    overlapTokens
+  };
+}
+
+/**
+ * Estimate token count for text (conservative estimate for code)
+ * Uses a simple heuristic: counts words, special characters, and estimates subwords
+ * 
+ * This is conservative - actual tokenizers may produce fewer tokens.
+ * For most accurate results, use the actual tokenizer, but this is much faster.
+ * 
+ * @param {string} text - The text to estimate tokens for
+ * @returns {number} Estimated token count
+ */
+export function estimateTokens(text) {
+  if (!text || text.length === 0) return 0;
+  
+  // Count words (split by whitespace)
+  const words = text.split(/\s+/).filter(w => w.length > 0);
+  
+  // Count special characters/punctuation that often become separate tokens
+  const specialChars = (text.match(/[{}()\[\];:,.<>!=+\-*\/%&|^~@#$"'`\\]/g) || []).length;
+  
+  // Estimate: words + special chars + 2 (for [CLS] and [SEP] special tokens)
+  // For long words, add extra tokens due to subword tokenization
+  let tokenCount = 2; // [CLS] and [SEP]
+  
+  for (const word of words) {
+    if (word.length <= 4) {
+      tokenCount += 1;
+    } else if (word.length <= 10) {
+      tokenCount += 2;
+    } else {
+      // Long words get split into ~4-char subwords
+      tokenCount += Math.ceil(word.length / 4);
+    }
+  }
+  
+  // Many special chars merge with adjacent tokens, so count ~50%
+  tokenCount += Math.floor(specialChars * 0.5);
+  
+  return tokenCount;
+}
+
+/**
+ * Check if text exceeds the token limit for a model
+ * @param {string} text - The text to check
+ * @param {string} modelName - The model name
+ * @returns {boolean} True if the text exceeds the limit
+ */
+export function exceedsTokenLimit(text, modelName) {
+  const limit = getModelTokenLimit(modelName);
+  const tokens = estimateTokens(text);
+  return tokens > limit;
+}