@getmikk/core 2.0.14 → 2.0.16

package/src/search/embedding-provider.ts

@@ -0,0 +1,249 @@
+import * as path from 'node:path';
+
+/**
+ * Common interface for embedding providers
+ */
+export interface EmbeddingProvider {
+  embed(text: string): Promise<number[]>;
+  embedBatch(texts: string[]): Promise<number[][]>;
+  getDimensions(): number;
+  isAvailable(): Promise<boolean>;
+}
+
+const VOCABULARY = [
+  'function', 'class', 'method', 'async', 'await', 'return', 'const', 'let', 'var',
+  'import', 'export', 'from', 'type', 'interface', 'extends', 'implements',
+  'constructor', 'prototype', 'static', 'private', 'public', 'protected',
+  'if', 'else', 'for', 'while', 'do', 'switch', 'case', 'break', 'continue',
+  'try', 'catch', 'finally', 'throw', 'error', 'exception',
+  'parse', 'format', 'validate', 'create', 'update', 'delete', 'remove',
+  'get', 'set', 'find', 'search', 'filter', 'map', 'reduce', 'transform',
+  'init', 'setup', 'config', 'options', 'settings', 'defaults',
+  'data', 'object', 'array', 'string', 'number', 'boolean', 'null', 'undefined',
+  'request', 'response', 'http', 'https', 'api', 'endpoint', 'route',
+  'auth', 'token', 'jwt', 'session', 'cookie', 'header',
+  'database', 'query', 'sql', 'transaction', 'connection', 'pool',
+  'file', 'path', 'directory', 'read', 'write', 'stream', 'buffer',
+  'event', 'listener', 'handler', 'callback', 'promise', 'observer',
+  'log', 'debug', 'info', 'warn', 'error', 'trace',
+  'test', 'mock', 'stub', 'assert', 'expect',
+  'cache', 'store', 'memory', 'session', 'local',
+  'user', 'account', 'profile', 'permission', 'role',
+  'create', 'register', 'login', 'logout', 'verify',
+  'send', 'receive', 'push', 'pull', 'fetch', 'upload', 'download',
+  'process', 'worker', 'thread', 'task', 'job', 'queue',
+  'client', 'server', 'service', 'endpoint', 'middleware',
+];
+
+function tokenize(text: string): string[] {
+  return text
+    .toLowerCase()
+    .replace(/[^a-z0-9\s]/g, ' ')
+    .split(/\s+/)
+    .filter(t => t.length > 1)
+    .filter(t => !['the', 'and', 'for', 'with', 'from', 'this', 'that', 'have', 'has'].includes(t));
+}
+
+function computeTF(tokens: string[]): Map<string, number> {
+  const tf = new Map<string, number>();
+  for (const token of tokens) {
+    tf.set(token, (tf.get(token) || 0) + 1);
+  }
+  const max = Math.max(...tf.values(), 1);
+  for (const [key, value] of tf) {
+    tf.set(key, value / max);
+  }
+  return tf;
+}
+
+/**
+ * Fast vocabulary-based embeddings for when ML models aren't available.
+ * Uses TF-IDF with a programming-focused vocabulary.
+ */
+export class VocabularyEmbedder implements EmbeddingProvider {
+  readonly dimensions: number;
+  private vocabMap: Map<string, number>;
+  private defaultIDF: number;
+
+  constructor(vocab: string[] = VOCABULARY, dimensions = 128) {
+    this.dimensions = dimensions;
+    this.vocabMap = new Map();
+    
+    for (let i = 0; i < Math.min(vocab.length, dimensions); i++) {
+      this.vocabMap.set(vocab[i], i);
+    }
+    
+    this.defaultIDF = Math.log(vocab.length + 1) + 1;
+  }
+
+  async embed(text: string): Promise<number[]> {
+    const tokens = tokenize(text);
+    const tf = computeTF(tokens);
+    
+    const vector = new Array(this.dimensions).fill(0);
+    const magnitude = Math.sqrt(tokens.length || 1);
+    
+    for (const [token, tfScore] of tf) {
+      const idx = this.vocabMap.get(token);
+      if (idx !== undefined && idx < this.dimensions) {
+        vector[idx] = tfScore * this.defaultIDF / magnitude;
+      }
+    }
+    
+    const norm = Math.sqrt(vector.reduce((sum, v) => sum + v * v, 0)) || 1;
+    return vector.map(v => v / norm);
+  }
+
+  async embedBatch(texts: string[]): Promise<number[][]> {
+    return Promise.all(texts.map(t => this.embed(t)));
+  }
+
+  getDimensions(): number {
+    return this.dimensions;
+  }
+
+  async isAvailable(): Promise<boolean> {
+    return true;
+  }
+}
+
+/**
+ * Local embeddings using ONNX runtime (via @xenova/transformers)
+ */
+export class LocalONNXEmbedder implements EmbeddingProvider {
+  private pipeline: unknown = null;
+  readonly dimensions = 384;
+  
+  readonly MODEL_NAME = 'Xenova/all-MiniLM-L6-v2';
+
+  async isAvailable(): Promise<boolean> {
+    try {
+      await import('@xenova/transformers');
+      return true;
+    } catch {
+      return false;
+    }
+  }
+
+  private async ensurePipeline() {
+    if (this.pipeline) return;
+    const { pipeline } = await import('@xenova/transformers');
+    this.pipeline = await pipeline('feature-extraction', this.MODEL_NAME);
+  }
+
+  async embed(text: string): Promise<number[]> {
+    await this.ensurePipeline();
+    const p = this.pipeline as (texts: string[], options: unknown) => Promise<Array<{ data: Float32Array }>>;
+    const output = await p([text], { pooling: 'mean', normalize: true });
+    return Array.from(output[0].data);
+  }
+
+  async embedBatch(texts: string[]): Promise<number[][]> {
+    await this.ensurePipeline();
+    const p = this.pipeline as (texts: string[], options: unknown) => Promise<Array<{ data: Float32Array }>>;
+    const output = await p(texts, { pooling: 'mean', normalize: true });
+    return output.map(o => Array.from(o.data));
+  }
+
+  getDimensions(): number {
+    return this.dimensions;
+  }
+}
+
+const GEMINI_MODEL_NAME = 'gemini-embedding-001';
+const GEMINI_DIMENSIONS = 3072;
+
+/**
+ * Gemini-backed embedding provider
+ */
+export class GeminiEmbedder implements EmbeddingProvider {
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  private model: any;
+  readonly dimensions = GEMINI_DIMENSIONS;
+
+  constructor(apiKey: string) {
+    this.initialize(apiKey);
+  }
+
+  private initialize(apiKey: string) {
+    const { GoogleGenerativeAI } = require('@google/generative-ai');
+    const genAI = new GoogleGenerativeAI(apiKey);
+    this.model = genAI.getGenerativeModel({ model: GEMINI_MODEL_NAME });
+  }
+
+  async embed(text: string): Promise<number[]> {
+    const result = await this.model.embedContent(text);
+    return result.embedding.values;
+  }
+
+  async embedBatch(texts: string[]): Promise<number[][]> {
+    const result = await this.model.batchEmbedContents({
+      requests: texts.map((t) => ({ content: { role: 'user', parts: [{ text: t }] } })),
+    });
+    return result.embeddings.map((e: { values: number[] }) => e.values);
+  }
+
+  getDimensions(): number {
+    return this.dimensions;
+  }
+
+  async isAvailable(): Promise<boolean> {
+    return !!process.env.GEMINI_API_KEY;
+  }
+}
+
+let cachedProvider: EmbeddingProvider | null = null;
+let providerInitPromise: Promise<EmbeddingProvider> | null = null;
+
+/**
+ * Factory to create the best available provider.
+ * Caches the provider for subsequent calls.
+ */
+export async function createEmbeddingProvider(): Promise<EmbeddingProvider> {
+  if (cachedProvider) {
+    return cachedProvider;
+  }
+  
+  if (providerInitPromise) {
+    return providerInitPromise;
+  }
+  
+  providerInitPromise = (async () => {
+    const localONNX = new LocalONNXEmbedder();
+    
+    if (await localONNX.isAvailable()) {
+      cachedProvider = localONNX;
+      return localONNX;
+    }
+    
+    const apiKey = process.env.GEMINI_API_KEY;
+    if (apiKey) {
+      try {
+        cachedProvider = new GeminiEmbedder(apiKey);
+        return cachedProvider;
+      } catch {
+        // Fall through to vocabulary embedder
+      }
+    }
+    
+    cachedProvider = new VocabularyEmbedder();
+    return cachedProvider;
+  })();
+  
+  return providerInitPromise;
+}
+
+/**
+ * Get the cached provider synchronously (may be null if not yet initialized)
+ */
+export function getCachedProvider(): EmbeddingProvider | null {
+  return cachedProvider;
+}
+
+/**
+ * Clear the cached provider
+ */
+export function clearProviderCache(): void {
+  cachedProvider = null;
+  providerInitPromise = null;
+}

package/src/search/index.ts

@@ -1,3 +1,15 @@
 // @getmikk/core search module
 export { BM25Index, reciprocalRankFusion, tokenize, buildFunctionTokens } from './bm25.js'
 export type { BM25Result } from './bm25.js'
+export { DirectSearchEngine, createDirectSearch, extractSignatures, extractNames, extractSignaturesMap, summarizeFunction, formatFunctionList } from './direct-search.js'
+export type { DirectQuery, DirectContext } from './direct-search.js'
+
+export { 
+  VocabularyEmbedder, 
+  LocalONNXEmbedder, 
+  GeminiEmbedder,
+  createEmbeddingProvider,
+  getCachedProvider,
+  clearProviderCache
+} from './embedding-provider.js'
+export type { EmbeddingProvider } from './embedding-provider.js'

package/src/search/semantic-search.ts

@@ -0,0 +1,435 @@
+/**
+ * Semantic Code Search — code embeddings for semantic similarity search
+ * Provides natural language code search and code-to-code similarity
+ */
+
+import type { MikkLock, MikkLockFunction } from '../contract/schema.js'
+
+// ---------------------------------------------------------------------------
+// Types
+// ---------------------------------------------------------------------------
+
+export interface CodeEmbedding {
+  id: string
+  vector: number[]
+  metadata: {
+    name: string
+    file: string
+    moduleId: string
+    purpose?: string
+    params?: string
+    returnType?: string
+  }
+}
+
+export interface SemanticSearchResult {
+  functionId: string
+  name: string
+  file: string
+  moduleId: string
+  score: number
+  purpose?: string
+  snippet?: string
+}
+
+export interface SemanticSearchOptions {
+  limit?: number
+  minScore?: number
+  filterModule?: string
+  filterFile?: string
+}
+
+interface EmbeddingIndex {
+  functions: Map<string, CodeEmbedding>
+  dimensions: number
+  indexedAt: number
+}
+
+// ---------------------------------------------------------------------------
+// Simple embedding model using TF-IDF-like approach
+// For production, would use transformer-based embeddings
+// ---------------------------------------------------------------------------
+
+export class SemanticCodeSearch {
+  private lock: MikkLock
+  private index: EmbeddingIndex | null = null
+  private readonly DIMENSIONS = 128
+
+  constructor(lock: MikkLock) {
+    this.lock = lock
+  }
+
+  /**
+   * Build semantic index from lock file
+   */
+  async buildIndex(): Promise<void> {
+    const functions = Object.values(this.lock.functions)
+    const embeddings = new Map<string, CodeEmbedding>()
+
+    for (const fn of functions) {
+      const vector = this.computeEmbedding(fn)
+      embeddings.set(fn.id, {
+        id: fn.id,
+        vector,
+        metadata: {
+          name: fn.name,
+          file: fn.file,
+          moduleId: fn.moduleId,
+          purpose: fn.purpose,
+          params: fn.params?.map(p => p.name).join(', '),
+          returnType: fn.returnType,
+        },
+      })
+    }
+
+    this.index = {
+      functions: embeddings,
+      dimensions: this.DIMENSIONS,
+      indexedAt: Date.now(),
+    }
+  }
+
+  /**
+   * Search code using natural language query
+   */
+  async search(query: string, options: SemanticSearchOptions = {}): Promise<SemanticSearchResult[]> {
+    if (!this.index) {
+      await this.buildIndex()
+    }
+
+    const queryVector = this.computeQueryEmbedding(query)
+    const results: Array<{ fn: MikkLockFunction; score: number }> = []
+
+    const functions = Object.values(this.lock.functions)
+    for (const fn of functions) {
+      const embedding = this.index!.functions.get(fn.id)
+      if (!embedding) continue
+
+      // Filter by module if specified
+      if (options.filterModule && fn.moduleId !== options.filterModule) continue
+
+      // Filter by file if specified
+      if (options.filterFile && !fn.file.includes(options.filterFile)) continue
+
+      const score = this.cosineSimilarity(queryVector, embedding.vector)
+
+      if (score >= (options.minScore ?? 0)) {
+        results.push({ fn, score })
+      }
+    }
+
+    // Sort by score descending
+    results.sort((a, b) => b.score - a.score)
+
+    const limit = options.limit ?? 20
+    return results.slice(0, limit).map(({ fn, score }) => ({
+      functionId: fn.id,
+      name: fn.name,
+      file: fn.file,
+      moduleId: fn.moduleId,
+      score,
+      purpose: fn.purpose,
+    }))
+  }
+
+  /**
+   * Find similar code to given code snippet
+   */
+  async findSimilarCode(code: string, options: SemanticSearchOptions = {}): Promise<SemanticSearchResult[]> {
+    if (!this.index) {
+      await this.buildIndex()
+    }
+
+    const codeVector = this.computeCodeEmbedding(code)
+    const results: Array<{ fn: MikkLockFunction; score: number }> = []
+
+    const functions = Object.values(this.lock.functions)
+    for (const fn of functions) {
+      const embedding = this.index!.functions.get(fn.id)
+      if (!embedding) continue
+
+      const score = this.cosineSimilarity(codeVector, embedding.vector)
+
+      if (score >= (options.minScore ?? 0.3)) {
+        results.push({ fn, score })
+      }
+    }
+
+    results.sort((a, b) => b.score - a.score)
+
+    const limit = options.limit ?? 10
+    return results.slice(0, limit).map(({ fn, score }) => ({
+      functionId: fn.id,
+      name: fn.name,
+      file: fn.file,
+      moduleId: fn.moduleId,
+      score,
+      purpose: fn.purpose,
+    }))
+  }
+
+  /**
+   * Compute embedding for a function using keyword + structural features
+   */
+  private computeEmbedding(fn: MikkLockFunction): number[] {
+    const vector = new Array(this.DIMENSIONS).fill(0)
+
+    // Feature 1: Function name tokens (first 32 dims)
+    const nameTokens = this.tokenize(fn.name)
+    for (let i = 0; i < Math.min(nameTokens.length, 32); i++) {
+      vector[i] = this.hashToken(nameTokens[i], i)
+    }
+
+    // Feature 2: Purpose keywords (next 32 dims)
+    if (fn.purpose) {
+      const purposeTokens = this.tokenize(fn.purpose)
+      for (let i = 0; i < Math.min(purposeTokens.length, 32); i++) {
+        vector[32 + i] = this.hashToken(purposeTokens[i], 32 + i)
+      }
+    }
+
+    // Feature 3: Module context (next 32 dims)
+    if (fn.moduleId) {
+      const moduleTokens = this.tokenize(fn.moduleId)
+      for (let i = 0; i < Math.min(moduleTokens.length, 32); i++) {
+        vector[64 + i] = this.hashToken(moduleTokens[i], 64 + i)
+      }
+    }
+
+    // Feature 4: Structural features (last 32 dims)
+    vector[96] = fn.isAsync ? 1 : 0
+    vector[97] = fn.isExported ? 1 : 0
+    vector[98] = fn.params?.length ?? 0
+    vector[99] = (fn.endLine - fn.startLine) / 100 // normalized function size
+    vector[100] = fn.calls?.length ?? 0 // number of calls
+    vector[101] = fn.calledBy?.length ?? 0 // number of callers
+
+    // Hash additional features
+    if (fn.returnType) {
+      vector[102] = this.hashToken(fn.returnType, 102) % 1
+    }
+
+    // Normalize vector
+    return this.normalizeVector(vector)
+  }
+
+  /**
+   * Compute query embedding
+   */
+  private computeQueryEmbedding(query: string): number[] {
+    const vector = new Array(this.DIMENSIONS).fill(0)
+    const tokens = this.tokenize(query)
+
+    // Weight recent tokens more heavily
+    for (let i = 0; i < tokens.length; i++) {
+      const weight = 1 - (i / tokens.length) * 0.5 // decreasing weight
+      const hash = this.hashToken(tokens[i], i % this.DIMENSIONS)
+      vector[i % this.DIMENSIONS] += hash * weight
+    }
+
+    return this.normalizeVector(vector)
+  }
+
+  /**
+   * Compute embedding for arbitrary code snippet
+   */
+  private computeCodeEmbedding(code: string): number[] {
+    const vector = new Array(this.DIMENSIONS).fill(0)
+    const tokens = this.tokenize(code)
+
+    for (let i = 0; i < Math.min(tokens.length, this.DIMENSIONS); i++) {
+      vector[i] = this.hashToken(tokens[i], i)
+    }
+
+    return this.normalizeVector(vector)
+  }
+
+  /**
+   * Tokenize text into words
+   */
+  private tokenize(text: string): string[] {
+    return text
+      .toLowerCase()
+      .replace(/([a-z])([A-Z])/g, '$1 $2')
+      .split(/[\s_./\\{}()[]"']+/)
+      .filter(Boolean)
+      .filter(w => w.length > 1)
+  }
+
+  /**
+   * Hash token to 0-1 range for embedding
+   */
+  private hashToken(token: string, seed: number): number {
+    let hash = 0
+    for (let i = 0; i < token.length; i++) {
+      hash = ((hash << 5) - hash + token.charCodeAt(i) + seed) >>> 0
+    }
+    return (hash % 1000) / 1000
+  }
+
+  /**
+   * Normalize vector to unit length
+   */
+  private normalizeVector(vector: number[]): number[] {
+    const magnitude = Math.sqrt(vector.reduce((sum, v) => sum + v * v, 0))
+    if (magnitude === 0) return vector
+    return vector.map(v => v / magnitude)
+  }
+
+  /**
+   * Compute cosine similarity between two vectors
+   */
+  private cosineSimilarity(a: number[], b: number[]): number {
+    let dotProduct = 0
+    let normA = 0
+    let normB = 0
+
+    for (let i = 0; i < a.length; i++) {
+      dotProduct += a[i] * b[i]
+      normA += a[i] * a[i]
+      normB += b[i] * b[i]
+    }
+
+    const denominator = Math.sqrt(normA) * Math.sqrt(normB)
+    if (denominator === 0) return 0
+
+    return dotProduct / denominator
+  }
+
+  /**
+   * Get index statistics
+   */
+  getIndexStats(): { functionCount: number; dimensions: number; indexedAt: number } | null {
+    if (!this.index) return null
+
+    return {
+      functionCount: this.index.functions.size,
+      dimensions: this.index.dimensions,
+      indexedAt: this.index.indexedAt,
+    }
+  }
+}
+
+/**
+ * Hybrid search combining BM25 and semantic search
+ */
+export class HybridSearchEngine {
+  private lock: MikkLock
+  private semanticSearch: SemanticCodeSearch
+  private readonly SEMANTIC_WEIGHT = 0.6
+  private readonly BM25_WEIGHT = 0.4
+
+  constructor(lock: MikkLock) {
+    this.lock = lock
+    this.semanticSearch = new SemanticCodeSearch(lock)
+  }
+
+  /**
+   * Search using both BM25 and semantic search with reranking
+   */
+  async search(
+    query: string,
+    options: SemanticSearchOptions & { useHybrid?: boolean } = {}
+  ): Promise<SemanticSearchResult[]> {
+    const { useHybrid = true, limit = 20, ...filterOptions } = options
+
+    if (!useHybrid) {
+      return this.semanticSearch.search(query, { ...filterOptions, limit })
+    }
+
+    // Run both searches in parallel
+    const [semanticResults, bm25Results] = await Promise.all([
+      this.semanticSearch.search(query, { ...filterOptions, limit: limit * 2 }),
+      this.bm25Search(query, { ...filterOptions, limit: limit * 2 }),
+    ])
+
+    // Combine scores using weighted RRF
+    const combinedScores = new Map<string, { fn: MikkLockFunction; score: number }>()
+
+    // Add semantic scores
+    for (const result of semanticResults) {
+      combinedScores.set(result.functionId, {
+        fn: this.lock.functions[result.functionId],
+        score: result.score * this.SEMANTIC_WEIGHT,
+      })
+    }
+
+    // Add BM25 scores
+    for (const result of bm25Results) {
+      const existing = combinedScores.get(result.functionId)
+      const bm25Score = result.score * this.BM25_WEIGHT
+
+      if (existing) {
+        existing.score += bm25Score
+      } else {
+        combinedScores.set(result.functionId, {
+          fn: this.lock.functions[result.functionId],
+          score: bm25Score,
+        })
+      }
+    }
+
+    // Sort by combined score
+    const results = Array.from(combinedScores.values())
+      .sort((a, b) => b.score - a.score)
+      .slice(0, limit)
+      .map(({ fn, score }) => ({
+        functionId: fn.id,
+        name: fn.name,
+        file: fn.file,
+        moduleId: fn.moduleId,
+        score,
+        purpose: fn.purpose,
+      }))
+
+    return results
+  }
+
+  /**
+   * Simple BM25 search for hybrid results
+   */
+  private async bm25Search(
+    query: string,
+    options: SemanticSearchOptions
+  ): Promise<Array<{ functionId: string; score: number }>> {
+    const tokens = this.tokenize(query)
+    const functions = Object.values(this.lock.functions)
+
+    const scores: Array<{ fn: MikkLockFunction; score: number }> = []
+
+    for (const fn of functions) {
+      let score = 0
+      const fnText = `${fn.name} ${fn.purpose || ''}`.toLowerCase()
+
+      for (const token of tokens) {
+        if (fnText.includes(token)) {
+          score += 1
+        }
+      }
+
+      if (score > 0) {
+        scores.push({ fn, score: score / tokens.length })
+      }
+    }
+
+    scores.sort((a, b) => b.score - a.score)
+
+    return scores.slice(0, options.limit ?? 20).map(({ fn, score }) => ({
+      functionId: fn.id,
+      score,
+    }))
+  }
+
+  private tokenize(text: string): string[] {
+    return text
+      .toLowerCase()
+      .split(/[\s]+/)
+      .filter(Boolean)
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Re-export for compatibility
+// ---------------------------------------------------------------------------
+
+// Exported as SemanticCodeSearch above

package/src/utils/artifact-transaction.ts

@@ -1,3 +1,4 @@
+/* eslint-disable @typescript-eslint/no-explicit-any */
 import * as fs from 'node:fs/promises'
 import * as path from 'node:path'
 import { randomUUID } from 'node:crypto'

package/src/utils/atomic-write.ts

@@ -1,3 +1,4 @@
+/* eslint-disable @typescript-eslint/no-explicit-any */
 import * as fs from 'node:fs/promises'
 import * as path from 'node:path'
 import { randomUUID } from 'node:crypto'