claude-brain 0.15.2 → 0.16.0

package/src/retrieval/bm25/tokenizer.ts

@@ -1,184 +1,184 @@
-/**
- * Text Tokenizer for BM25
- * Handles text preprocessing for sparse search
- */
-
-// Common English stopwords to filter
-const STOPWORDS = new Set([
-  'a', 'an', 'the', 'and', 'or', 'but', 'in', 'on', 'at', 'to', 'for',
-  'of', 'with', 'by', 'from', 'as', 'is', 'was', 'are', 'were', 'been',
-  'be', 'have', 'has', 'had', 'do', 'does', 'did', 'will', 'would', 'could',
-  'should', 'may', 'might', 'must', 'shall', 'can', 'need', 'dare', 'ought',
-  'used', 'it', 'its', 'this', 'that', 'these', 'those', 'i', 'me', 'my',
-  'myself', 'we', 'our', 'ours', 'ourselves', 'you', 'your', 'yours',
-  'yourself', 'yourselves', 'he', 'him', 'his', 'himself', 'she', 'her',
-  'hers', 'herself', 'they', 'them', 'their', 'theirs', 'themselves',
-  'what', 'which', 'who', 'whom', 'when', 'where', 'why', 'how', 'all',
-  'each', 'every', 'both', 'few', 'more', 'most', 'other', 'some', 'such',
-  'no', 'nor', 'not', 'only', 'own', 'same', 'so', 'than', 'too', 'very',
-  's', 't', 'just', 'don', 'now', 'then', 'here', 'there', 'also'
-])
-
-// Common programming terms to keep
-const KEEP_TERMS = new Set([
-  'api', 'sql', 'css', 'html', 'json', 'xml', 'http', 'https', 'url', 'uri',
-  'jwt', 'oauth', 'rest', 'graphql', 'grpc', 'tcp', 'udp', 'ip', 'dns',
-  'aws', 'gcp', 'azure', 'docker', 'kubernetes', 'k8s', 'npm', 'yarn', 'pnpm',
-  'git', 'github', 'gitlab', 'ci', 'cd', 'devops', 'mlops', 'db', 'orm',
-  'ui', 'ux', 'cli', 'gui', 'ide', 'sdk', 'mcp', 'llm', 'ai', 'ml'
-])
-
-export interface TokenizerOptions {
-  /** Minimum token length to keep */
-  minLength?: number
-  /** Maximum token length */
-  maxLength?: number
-  /** Remove stopwords */
-  removeStopwords?: boolean
-  /** Convert to lowercase */
-  lowercase?: boolean
-  /** Apply stemming (basic) */
-  stemming?: boolean
-  /** Split on camelCase */
-  splitCamelCase?: boolean
-}
-
-const DEFAULT_OPTIONS: TokenizerOptions = {
-  minLength: 2,
-  maxLength: 50,
-  removeStopwords: true,
-  lowercase: true,
-  stemming: true,
-  splitCamelCase: true
-}
-
-/**
- * Tokenize text for BM25 indexing and search
- */
-export function tokenize(text: string, options: TokenizerOptions = {}): string[] {
-  const opts = { ...DEFAULT_OPTIONS, ...options }
-
-  if (!text || typeof text !== 'string') {
-    return []
-  }
-
-  // Split on camelCase if enabled
-  let processedText = text
-  if (opts.splitCamelCase) {
-    processedText = text.replace(/([a-z])([A-Z])/g, '$1 $2')
-  }
-
-  // Lowercase if enabled
-  if (opts.lowercase) {
-    processedText = processedText.toLowerCase()
-  }
-
-  // Split into tokens
-  // Match words, numbers, and hyphenated compounds
-  const tokenRegex = /[\w]+(?:[-_][\w]+)*/g
-  const rawTokens = processedText.match(tokenRegex) || []
-
-  // Process tokens
-  let tokens = rawTokens
-    .map(token => {
-      // Keep programming terms intact
-      if (KEEP_TERMS.has(token.toLowerCase())) {
-        return token
-      }
-
-      // Apply basic stemming if enabled
-      if (opts.stemming) {
-        return basicStem(token)
-      }
-
-      return token
-    })
-    .filter(token => {
-      // Length filter
-      if (token.length < opts.minLength! || token.length > opts.maxLength!) {
-        return false
-      }
-
-      // Stopword filter
-      if (opts.removeStopwords && STOPWORDS.has(token.toLowerCase())) {
-        return false
-      }
-
-      // Filter pure numbers (but keep alphanumeric)
-      if (/^\d+$/.test(token)) {
-        return false
-      }
-
-      return true
-    })
-
-  return tokens
-}
-
-/**
- * Basic Porter-like stemming
- * Simplified for performance - handles common English suffixes
- */
-function basicStem(word: string): string {
-  if (word.length < 4) return word
-
-  // Common suffix replacements
-  const suffixRules: [RegExp, string][] = [
-    [/ies$/, 'y'],
-    [/ied$/, 'y'],
-    [/es$/, ''],
-    [/s$/, ''],
-    [/ing$/, ''],
-    [/ed$/, ''],
-    [/tion$/, 't'],
-    [/ness$/, ''],
-    [/ment$/, ''],
-    [/able$/, ''],
-    [/ible$/, ''],
-    [/ful$/, ''],
-    [/less$/, ''],
-    [/ly$/, '']
-  ]
-
-  let stemmed = word
-  for (const [pattern, replacement] of suffixRules) {
-    if (pattern.test(word)) {
-      const candidate = word.replace(pattern, replacement)
-      // Only apply if result is at least 3 chars
-      if (candidate.length >= 3) {
-        stemmed = candidate
-        break
-      }
-    }
-  }
-
-  return stemmed
-}
-
-/**
- * Get n-grams from tokens
- */
-export function getNGrams(tokens: string[], n: number = 2): string[] {
-  if (tokens.length < n) return []
-
-  const ngrams: string[] = []
-  for (let i = 0; i <= tokens.length - n; i++) {
-    ngrams.push(tokens.slice(i, i + n).join(' '))
-  }
-
-  return ngrams
-}
-
-/**
- * Combined tokenizer for search that includes unigrams and bigrams
- */
-export function tokenizeForSearch(
-  text: string,
-  options: TokenizerOptions = {}
-): string[] {
-  const unigrams = tokenize(text, options)
-  const bigrams = getNGrams(unigrams, 2)
-
-  // Return unique tokens (unigrams + bigrams)
-  return [...new Set([...unigrams, ...bigrams])]
-}
+/**
+ * Text Tokenizer for BM25
+ * Handles text preprocessing for sparse search
+ */
+
+// Common English stopwords to filter
+const STOPWORDS = new Set([
+  'a', 'an', 'the', 'and', 'or', 'but', 'in', 'on', 'at', 'to', 'for',
+  'of', 'with', 'by', 'from', 'as', 'is', 'was', 'are', 'were', 'been',
+  'be', 'have', 'has', 'had', 'do', 'does', 'did', 'will', 'would', 'could',
+  'should', 'may', 'might', 'must', 'shall', 'can', 'need', 'dare', 'ought',
+  'used', 'it', 'its', 'this', 'that', 'these', 'those', 'i', 'me', 'my',
+  'myself', 'we', 'our', 'ours', 'ourselves', 'you', 'your', 'yours',
+  'yourself', 'yourselves', 'he', 'him', 'his', 'himself', 'she', 'her',
+  'hers', 'herself', 'they', 'them', 'their', 'theirs', 'themselves',
+  'what', 'which', 'who', 'whom', 'when', 'where', 'why', 'how', 'all',
+  'each', 'every', 'both', 'few', 'more', 'most', 'other', 'some', 'such',
+  'no', 'nor', 'not', 'only', 'own', 'same', 'so', 'than', 'too', 'very',
+  's', 't', 'just', 'don', 'now', 'then', 'here', 'there', 'also'
+])
+
+// Common programming terms to keep
+const KEEP_TERMS = new Set([
+  'api', 'sql', 'css', 'html', 'json', 'xml', 'http', 'https', 'url', 'uri',
+  'jwt', 'oauth', 'rest', 'graphql', 'grpc', 'tcp', 'udp', 'ip', 'dns',
+  'aws', 'gcp', 'azure', 'docker', 'kubernetes', 'k8s', 'npm', 'yarn', 'pnpm',
+  'git', 'github', 'gitlab', 'ci', 'cd', 'devops', 'mlops', 'db', 'orm',
+  'ui', 'ux', 'cli', 'gui', 'ide', 'sdk', 'mcp', 'llm', 'ai', 'ml'
+])
+
+export interface TokenizerOptions {
+  /** Minimum token length to keep */
+  minLength?: number
+  /** Maximum token length */
+  maxLength?: number
+  /** Remove stopwords */
+  removeStopwords?: boolean
+  /** Convert to lowercase */
+  lowercase?: boolean
+  /** Apply stemming (basic) */
+  stemming?: boolean
+  /** Split on camelCase */
+  splitCamelCase?: boolean
+}
+
+const DEFAULT_OPTIONS: TokenizerOptions = {
+  minLength: 2,
+  maxLength: 50,
+  removeStopwords: true,
+  lowercase: true,
+  stemming: true,
+  splitCamelCase: true
+}
+
+/**
+ * Tokenize text for BM25 indexing and search
+ */
+export function tokenize(text: string, options: TokenizerOptions = {}): string[] {
+  const opts = { ...DEFAULT_OPTIONS, ...options }
+
+  if (!text || typeof text !== 'string') {
+    return []
+  }
+
+  // Split on camelCase if enabled
+  let processedText = text
+  if (opts.splitCamelCase) {
+    processedText = text.replace(/([a-z])([A-Z])/g, '$1 $2')
+  }
+
+  // Lowercase if enabled
+  if (opts.lowercase) {
+    processedText = processedText.toLowerCase()
+  }
+
+  // Split into tokens
+  // Match words, numbers, and hyphenated compounds
+  const tokenRegex = /[\w]+(?:[-_][\w]+)*/g
+  const rawTokens = processedText.match(tokenRegex) || []
+
+  // Process tokens
+  let tokens = rawTokens
+    .map(token => {
+      // Keep programming terms intact
+      if (KEEP_TERMS.has(token.toLowerCase())) {
+        return token
+      }
+
+      // Apply basic stemming if enabled
+      if (opts.stemming) {
+        return basicStem(token)
+      }
+
+      return token
+    })
+    .filter(token => {
+      // Length filter
+      if (token.length < opts.minLength! || token.length > opts.maxLength!) {
+        return false
+      }
+
+      // Stopword filter
+      if (opts.removeStopwords && STOPWORDS.has(token.toLowerCase())) {
+        return false
+      }
+
+      // Filter pure numbers (but keep alphanumeric)
+      if (/^\d+$/.test(token)) {
+        return false
+      }
+
+      return true
+    })
+
+  return tokens
+}
+
+/**
+ * Basic Porter-like stemming
+ * Simplified for performance - handles common English suffixes
+ */
+function basicStem(word: string): string {
+  if (word.length < 4) return word
+
+  // Common suffix replacements
+  const suffixRules: [RegExp, string][] = [
+    [/ies$/, 'y'],
+    [/ied$/, 'y'],
+    [/es$/, ''],
+    [/s$/, ''],
+    [/ing$/, ''],
+    [/ed$/, ''],
+    [/tion$/, 't'],
+    [/ness$/, ''],
+    [/ment$/, ''],
+    [/able$/, ''],
+    [/ible$/, ''],
+    [/ful$/, ''],
+    [/less$/, ''],
+    [/ly$/, '']
+  ]
+
+  let stemmed = word
+  for (const [pattern, replacement] of suffixRules) {
+    if (pattern.test(word)) {
+      const candidate = word.replace(pattern, replacement)
+      // Only apply if result is at least 3 chars
+      if (candidate.length >= 3) {
+        stemmed = candidate
+        break
+      }
+    }
+  }
+
+  return stemmed
+}
+
+/**
+ * Get n-grams from tokens
+ */
+export function getNGrams(tokens: string[], n: number = 2): string[] {
+  if (tokens.length < n) return []
+
+  const ngrams: string[] = []
+  for (let i = 0; i <= tokens.length - n; i++) {
+    ngrams.push(tokens.slice(i, i + n).join(' '))
+  }
+
+  return ngrams
+}
+
+/**
+ * Combined tokenizer for search that includes unigrams and bigrams
+ */
+export function tokenizeForSearch(
+  text: string,
+  options: TokenizerOptions = {}
+): string[] {
+  const unigrams = tokenize(text, options)
+  const bigrams = getNGrams(unigrams, 2)
+
+  // Return unique tokens (unigrams + bigrams)
+  return [...new Set([...unigrams, ...bigrams])]
+}