@vinaes/succ 1.3.19

package/dist/lib/benchmark.js

@@ -0,0 +1,778 @@
+/**
+ * Benchmark Metrics Library
+ *
+ * Provides standard IR (Information Retrieval) metrics for evaluating
+ * succ's search and memory recall quality:
+ * - Recall@K: What fraction of relevant items were retrieved in top K
+ * - MRR (Mean Reciprocal Rank): Average of reciprocal ranks of first relevant result
+ * - NDCG (Normalized Discounted Cumulative Gain): Measures ranking quality
+ * - Latency metrics: Embedding time, search time, full pipeline time
+ */
+// ============================================================================
+// Accuracy Metrics
+// ============================================================================
+/**
+ * Calculate Recall@K
+ *
+ * What fraction of relevant items appear in the top K results?
+ * Recall@K = |relevant ∩ retrieved@K| / |relevant|
+ *
+ * @param results - Ranked search results (ordered by score desc)
+ * @param relevantIds - Set of relevant item IDs (ground truth)
+ * @param k - Number of top results to consider
+ */
+export function recallAtK(results, relevantIds, k) {
+    if (relevantIds.size === 0)
+        return 0;
+    const topK = results.slice(0, k);
+    const retrievedRelevant = topK.filter((r) => relevantIds.has(r.id)).length;
+    return retrievedRelevant / relevantIds.size;
+}
+/**
+ * Calculate Precision@K
+ *
+ * What fraction of top K results are relevant?
+ * Precision@K = |relevant ∩ retrieved@K| / K
+ *
+ * @param results - Ranked search results (ordered by score desc)
+ * @param relevantIds - Set of relevant item IDs (ground truth)
+ * @param k - Number of top results to consider
+ */
+export function precisionAtK(results, relevantIds, k) {
+    if (k === 0)
+        return 0;
+    const topK = results.slice(0, k);
+    const retrievedRelevant = topK.filter((r) => relevantIds.has(r.id)).length;
+    return retrievedRelevant / Math.min(k, results.length || 1);
+}
+/**
+ * Calculate F1@K
+ *
+ * Harmonic mean of Precision@K and Recall@K
+ * F1@K = 2 * (Precision@K * Recall@K) / (Precision@K + Recall@K)
+ *
+ * @param results - Ranked search results (ordered by score desc)
+ * @param relevantIds - Set of relevant item IDs (ground truth)
+ * @param k - Number of top results to consider
+ */
+export function f1AtK(results, relevantIds, k) {
+    const precision = precisionAtK(results, relevantIds, k);
+    const recall = recallAtK(results, relevantIds, k);
+    if (precision + recall === 0)
+        return 0;
+    return (2 * precision * recall) / (precision + recall);
+}
+/**
+ * Calculate Reciprocal Rank
+ *
+ * RR = 1 / rank of first relevant result
+ * Returns 0 if no relevant result found
+ *
+ * @param results - Ranked search results
+ * @param relevantIds - Set of relevant item IDs
+ */
+export function reciprocalRank(results, relevantIds) {
+    for (let i = 0; i < results.length; i++) {
+        if (relevantIds.has(results[i].id)) {
+            return 1 / (i + 1);
+        }
+    }
+    return 0;
+}
+/**
+ * Calculate Mean Reciprocal Rank across multiple queries
+ *
+ * MRR = (1/|Q|) * Σ RR(q) for each query q
+ *
+ * @param queries - Array of benchmark queries with results
+ */
+export function meanReciprocalRank(queries) {
+    if (queries.length === 0)
+        return 0;
+    const sumRR = queries.reduce((sum, q) => sum + reciprocalRank(q.results, q.relevantIds), 0);
+    return sumRR / queries.length;
+}
+/**
+ * Calculate DCG (Discounted Cumulative Gain)
+ *
+ * DCG@K = Σ (2^rel(i) - 1) / log2(i + 1) for i = 1 to K
+ *
+ * @param results - Ranked search results
+ * @param relevanceScores - Map of item ID to relevance score (default 1.0 for relevant)
+ * @param k - Number of results to consider
+ */
+export function dcg(results, relevanceScores, k) {
+    let dcgScore = 0;
+    for (let i = 0; i < Math.min(k, results.length); i++) {
+        const rel = relevanceScores.get(results[i].id) ?? 0;
+        // Using the standard DCG formula: (2^rel - 1) / log2(rank + 1)
+        dcgScore += (Math.pow(2, rel) - 1) / Math.log2(i + 2);
+    }
+    return dcgScore;
+}
+/**
+ * Calculate NDCG (Normalized DCG)
+ *
+ * NDCG@K = DCG@K / IDCG@K
+ * where IDCG is the ideal DCG (perfect ranking)
+ *
+ * @param results - Ranked search results
+ * @param relevanceScores - Map of item ID to relevance score
+ * @param k - Number of results to consider
+ */
+export function ndcg(results, relevanceScores, k) {
+    // Calculate actual DCG
+    const actualDcg = dcg(results, relevanceScores, k);
+    // Calculate ideal DCG (perfect ranking)
+    // Sort all relevance scores descending to get ideal order
+    const idealRanking = [...relevanceScores.entries()]
+        .sort((a, b) => b[1] - a[1])
+        .slice(0, k)
+        .map(([id, score]) => ({ id, score }));
+    const idealDcg = dcg(idealRanking, relevanceScores, k);
+    if (idealDcg === 0)
+        return 0;
+    return actualDcg / idealDcg;
+}
+/**
+ * Calculate all accuracy metrics for a set of queries
+ */
+export function calculateAccuracyMetrics(queries, k = 5) {
+    if (queries.length === 0) {
+        return {
+            recallAtK: 0,
+            precisionAtK: 0,
+            f1AtK: 0,
+            k,
+            mrr: 0,
+            ndcg: 0,
+            queryCount: 0,
+            queriesWithHits: 0,
+        };
+    }
+    let totalRecall = 0;
+    let totalPrecision = 0;
+    let totalF1 = 0;
+    let totalNdcg = 0;
+    let queriesWithHits = 0;
+    for (const q of queries) {
+        // For relevance scores, use provided or default to 1.0 for all relevant items
+        const relScores = q.relevanceScores ?? new Map([...q.relevantIds].map((id) => [id, 1.0]));
+        totalRecall += recallAtK(q.results, q.relevantIds, k);
+        totalPrecision += precisionAtK(q.results, q.relevantIds, k);
+        totalF1 += f1AtK(q.results, q.relevantIds, k);
+        totalNdcg += ndcg(q.results, relScores, k);
+        // Check if any relevant item was found
+        if (q.results.some((r) => q.relevantIds.has(r.id))) {
+            queriesWithHits++;
+        }
+    }
+    return {
+        recallAtK: totalRecall / queries.length,
+        precisionAtK: totalPrecision / queries.length,
+        f1AtK: totalF1 / queries.length,
+        k,
+        mrr: meanReciprocalRank(queries),
+        ndcg: totalNdcg / queries.length,
+        queryCount: queries.length,
+        queriesWithHits,
+    };
+}
+// ============================================================================
+// Latency Metrics
+// ============================================================================
+/**
+ * Calculate latency statistics from an array of measurements
+ */
+export function calculateLatencyStats(measurements) {
+    if (measurements.length === 0) {
+        return { min: 0, max: 0, avg: 0, p50: 0, p95: 0, p99: 0, samples: 0 };
+    }
+    const sorted = [...measurements].sort((a, b) => a - b);
+    const sum = sorted.reduce((a, b) => a + b, 0);
+    return {
+        min: sorted[0],
+        max: sorted[sorted.length - 1],
+        avg: sum / sorted.length,
+        p50: percentile(sorted, 50),
+        p95: percentile(sorted, 95),
+        p99: percentile(sorted, 99),
+        samples: sorted.length,
+    };
+}
+/**
+ * Calculate percentile from sorted array
+ */
+function percentile(sorted, p) {
+    if (sorted.length === 0)
+        return 0;
+    const index = (p / 100) * (sorted.length - 1);
+    const lower = Math.floor(index);
+    const upper = Math.ceil(index);
+    if (lower === upper)
+        return sorted[lower];
+    return sorted[lower] * (upper - index) + sorted[upper] * (index - lower);
+}
+// ============================================================================
+// Formatting
+// ============================================================================
+/**
+ * Format accuracy metrics as human-readable string
+ */
+export function formatAccuracyMetrics(metrics) {
+    const lines = [
+        '┌─────────────────────────────────────────┐',
+        '│         Accuracy Metrics                │',
+        '├─────────────────────────────────────────┤',
+        `│ Recall@${metrics.k}:     ${(metrics.recallAtK * 100).toFixed(1).padStart(6)}%             │`,
+        `│ Precision@${metrics.k}: ${(metrics.precisionAtK * 100).toFixed(1).padStart(6)}%             │`,
+        `│ F1@${metrics.k}:        ${(metrics.f1AtK * 100).toFixed(1).padStart(6)}%             │`,
+        `│ MRR:         ${(metrics.mrr * 100).toFixed(1).padStart(6)}%             │`,
+        `│ NDCG@${metrics.k}:      ${(metrics.ndcg * 100).toFixed(1).padStart(6)}%             │`,
+        '├─────────────────────────────────────────┤',
+        `│ Queries:     ${metrics.queryCount.toString().padStart(6)}              │`,
+        `│ With hits:   ${metrics.queriesWithHits.toString().padStart(6)} (${((metrics.queriesWithHits / Math.max(metrics.queryCount, 1)) * 100).toFixed(0)}%)         │`,
+        '└─────────────────────────────────────────┘',
+    ];
+    return lines.join('\n');
+}
+/**
+ * Format latency stats as human-readable string
+ */
+export function formatLatencyStats(stats, name) {
+    return `${name.padEnd(12)} │ ${stats.avg.toFixed(1).padStart(7)}ms │ ${stats.p50.toFixed(1).padStart(7)}ms │ ${stats.p95.toFixed(1).padStart(7)}ms │ ${stats.p99.toFixed(1).padStart(7)}ms │`;
+}
+/**
+ * Format latency metrics as human-readable string
+ */
+export function formatLatencyMetrics(metrics) {
+    const lines = [
+        '┌─────────────────────────────────────────────────────────────────┐',
+        '│                     Latency Metrics                            │',
+        '├──────────────┬─────────┬─────────┬─────────┬─────────┤',
+        '│ Operation    │   Avg   │   P50   │   P95   │   P99   │',
+        '├──────────────┼─────────┼─────────┼─────────┼─────────┤',
+        `│ ${formatLatencyStats(metrics.embedding, 'Embedding')}`,
+        `│ ${formatLatencyStats(metrics.search, 'Search')}`,
+        `│ ${formatLatencyStats(metrics.pipeline, 'Pipeline')}`,
+        '└──────────────┴─────────┴─────────┴─────────┴─────────┘',
+    ];
+    return lines.join('\n');
+}
+/**
+ * Format full benchmark results
+ */
+export function formatBenchmarkResults(results) {
+    const lines = [
+        '═══════════════════════════════════════════════════════════════════',
+        '                      SUCC BENCHMARK RESULTS                       ',
+        '═══════════════════════════════════════════════════════════════════',
+        '',
+        `Timestamp: ${results.timestamp}`,
+        `Model: ${results.config.embeddingModel} (${results.config.embeddingMode})`,
+        `Memories: ${results.config.totalMemories}`,
+        `Test queries: ${results.config.testQueries}`,
+        '',
+        formatAccuracyMetrics(results.accuracy),
+        '',
+        formatLatencyMetrics(results.latency),
+        '',
+        '═══════════════════════════════════════════════════════════════════',
+    ];
+    return lines.join('\n');
+}
+// Extended categories for larger benchmarks
+const EXTENDED_CATEGORIES = {
+    typescript: [
+        {
+            content: 'TypeScript is a strongly typed programming language that builds on JavaScript',
+            tags: ['typescript', 'programming', 'language'],
+        },
+        {
+            content: 'TypeScript interfaces define the shape of objects with type checking',
+            tags: ['typescript', 'interfaces', 'types'],
+        },
+        {
+            content: 'TypeScript generics allow creating reusable components with type safety',
+            tags: ['typescript', 'generics', 'reusable'],
+        },
+        {
+            content: 'TypeScript decorators are a stage 2 proposal for JavaScript',
+            tags: ['typescript', 'decorators', 'metadata'],
+        },
+        {
+            content: 'TypeScript enums provide a way to define named constants',
+            tags: ['typescript', 'enums', 'constants'],
+        },
+        {
+            content: 'TypeScript utility types like Partial, Required, Pick transform types',
+            tags: ['typescript', 'utility', 'types'],
+        },
+        {
+            content: 'TypeScript namespaces organize code and prevent naming collisions',
+            tags: ['typescript', 'namespaces', 'modules'],
+        },
+        {
+            content: 'TypeScript strict mode enables stricter type checking options',
+            tags: ['typescript', 'strict', 'config'],
+        },
+    ],
+    react: [
+        {
+            content: 'React is a JavaScript library for building user interfaces',
+            tags: ['react', 'frontend', 'ui'],
+        },
+        {
+            content: 'React hooks like useState and useEffect manage component state and side effects',
+            tags: ['react', 'hooks', 'state'],
+        },
+        {
+            content: 'React components can be functional or class-based',
+            tags: ['react', 'components', 'functional'],
+        },
+        {
+            content: 'React context provides a way to pass data through the component tree',
+            tags: ['react', 'context', 'state'],
+        },
+        {
+            content: 'React Server Components render on the server for better performance',
+            tags: ['react', 'server', 'performance'],
+        },
+        {
+            content: 'React Suspense handles async operations with loading states',
+            tags: ['react', 'suspense', 'async'],
+        },
+        {
+            content: 'React memo prevents unnecessary re-renders of components',
+            tags: ['react', 'memo', 'optimization'],
+        },
+        {
+            content: 'React portals render children into different DOM nodes',
+            tags: ['react', 'portals', 'dom'],
+        },
+    ],
+    database: [
+        {
+            content: 'PostgreSQL is a powerful open source relational database',
+            tags: ['database', 'postgresql', 'sql'],
+        },
+        {
+            content: 'SQLite is a self-contained, serverless SQL database engine',
+            tags: ['database', 'sqlite', 'embedded'],
+        },
+        {
+            content: 'Database indexes improve query performance by creating sorted references',
+            tags: ['database', 'indexes', 'performance'],
+        },
+        {
+            content: 'Database transactions ensure ACID properties for data integrity',
+            tags: ['database', 'transactions', 'acid'],
+        },
+        {
+            content: 'MongoDB is a document-oriented NoSQL database',
+            tags: ['database', 'mongodb', 'nosql'],
+        },
+        {
+            content: 'Redis is an in-memory data structure store for caching',
+            tags: ['database', 'redis', 'cache'],
+        },
+        {
+            content: 'Database migrations track schema changes over time',
+            tags: ['database', 'migrations', 'schema'],
+        },
+        {
+            content: 'Connection pooling reuses database connections for efficiency',
+            tags: ['database', 'pooling', 'connections'],
+        },
+    ],
+    devops: [
+        {
+            content: 'Docker containers package software into standardized units',
+            tags: ['docker', 'containers', 'devops'],
+        },
+        {
+            content: 'Kubernetes orchestrates containerized applications at scale',
+            tags: ['kubernetes', 'orchestration', 'devops'],
+        },
+        {
+            content: 'CI/CD pipelines automate building, testing, and deploying code',
+            tags: ['cicd', 'automation', 'devops'],
+        },
+        {
+            content: 'Infrastructure as Code defines infrastructure using configuration files',
+            tags: ['iac', 'terraform', 'devops'],
+        },
+        {
+            content: 'GitHub Actions automates workflows directly in repositories',
+            tags: ['github', 'actions', 'devops'],
+        },
+        {
+            content: 'Prometheus monitors systems and collects metrics',
+            tags: ['prometheus', 'monitoring', 'devops'],
+        },
+        {
+            content: 'Helm manages Kubernetes applications with charts',
+            tags: ['helm', 'kubernetes', 'devops'],
+        },
+        {
+            content: 'Ansible automates configuration management across servers',
+            tags: ['ansible', 'automation', 'devops'],
+        },
+    ],
+    architecture: [
+        {
+            content: 'Microservices architecture splits applications into small, independent services',
+            tags: ['microservices', 'architecture', 'distributed'],
+        },
+        {
+            content: 'Event-driven architecture uses events to trigger and communicate between services',
+            tags: ['events', 'architecture', 'async'],
+        },
+        {
+            content: 'Domain-Driven Design focuses on the core domain and domain logic',
+            tags: ['ddd', 'architecture', 'domain'],
+        },
+        {
+            content: 'Clean Architecture separates concerns into layers with dependency rules',
+            tags: ['clean', 'architecture', 'layers'],
+        },
+        {
+            content: 'CQRS separates read and write operations for scalability',
+            tags: ['cqrs', 'architecture', 'patterns'],
+        },
+        {
+            content: 'Event Sourcing stores state as a sequence of events',
+            tags: ['eventsourcing', 'architecture', 'patterns'],
+        },
+        {
+            content: 'Hexagonal Architecture organizes code around ports and adapters',
+            tags: ['hexagonal', 'architecture', 'ports'],
+        },
+        {
+            content: 'Saga pattern manages distributed transactions across services',
+            tags: ['saga', 'architecture', 'distributed'],
+        },
+    ],
+    security: [
+        {
+            content: 'JWT tokens provide stateless authentication for APIs',
+            tags: ['jwt', 'security', 'auth'],
+        },
+        {
+            content: 'OAuth 2.0 enables secure delegated access to resources',
+            tags: ['oauth', 'security', 'auth'],
+        },
+        {
+            content: 'CORS controls which origins can access your API',
+            tags: ['cors', 'security', 'web'],
+        },
+        {
+            content: 'SQL injection attacks exploit poorly sanitized user input',
+            tags: ['sql', 'security', 'vulnerabilities'],
+        },
+        {
+            content: 'XSS attacks inject malicious scripts into web pages',
+            tags: ['xss', 'security', 'vulnerabilities'],
+        },
+        {
+            content: 'HTTPS encrypts data in transit using TLS',
+            tags: ['https', 'security', 'encryption'],
+        },
+        {
+            content: 'API rate limiting prevents abuse and denial of service',
+            tags: ['ratelimit', 'security', 'api'],
+        },
+        {
+            content: 'Password hashing with bcrypt protects stored credentials',
+            tags: ['bcrypt', 'security', 'passwords'],
+        },
+    ],
+    testing: [
+        {
+            content: 'Unit tests verify individual functions in isolation',
+            tags: ['unit', 'testing', 'isolation'],
+        },
+        {
+            content: 'Integration tests verify components work together correctly',
+            tags: ['integration', 'testing', 'components'],
+        },
+        {
+            content: 'End-to-end tests simulate real user scenarios',
+            tags: ['e2e', 'testing', 'scenarios'],
+        },
+        {
+            content: 'Test-driven development writes tests before implementation',
+            tags: ['tdd', 'testing', 'methodology'],
+        },
+        {
+            content: 'Mocking replaces dependencies with controlled implementations',
+            tags: ['mocking', 'testing', 'dependencies'],
+        },
+        {
+            content: 'Code coverage measures how much code is executed by tests',
+            tags: ['coverage', 'testing', 'metrics'],
+        },
+        {
+            content: 'Snapshot testing captures component output for comparison',
+            tags: ['snapshot', 'testing', 'ui'],
+        },
+        {
+            content: 'Property-based testing generates random inputs to find edge cases',
+            tags: ['property', 'testing', 'random'],
+        },
+    ],
+    performance: [
+        {
+            content: 'Caching stores frequently accessed data for faster retrieval',
+            tags: ['caching', 'performance', 'speed'],
+        },
+        {
+            content: 'Lazy loading defers resource loading until needed',
+            tags: ['lazy', 'performance', 'loading'],
+        },
+        {
+            content: 'Code splitting breaks bundles into smaller chunks',
+            tags: ['splitting', 'performance', 'bundles'],
+        },
+        {
+            content: 'CDN distributes content closer to users geographically',
+            tags: ['cdn', 'performance', 'distribution'],
+        },
+        {
+            content: 'Database query optimization reduces execution time',
+            tags: ['optimization', 'performance', 'database'],
+        },
+        {
+            content: 'Memory profiling identifies memory leaks and inefficiencies',
+            tags: ['memory', 'performance', 'profiling'],
+        },
+        {
+            content: 'Load balancing distributes traffic across multiple servers',
+            tags: ['loadbalancing', 'performance', 'scaling'],
+        },
+        {
+            content: 'Compression reduces data size for faster transmission',
+            tags: ['compression', 'performance', 'network'],
+        },
+    ],
+};
+// Extended queries for larger benchmarks
+const EXTENDED_QUERIES = [
+    // TypeScript queries
+    { query: 'strongly typed javascript superset language', categories: ['typescript'] },
+    { query: 'type checking and interfaces in code', categories: ['typescript'] },
+    { query: 'generic type parameters reusable components', categories: ['typescript'] },
+    { query: 'decorators metadata annotations', categories: ['typescript'] },
+    // React queries
+    { query: 'UI component library for frontend', categories: ['react'] },
+    { query: 'managing state in functional components', categories: ['react'] },
+    { query: 'server side rendering performance', categories: ['react'] },
+    { query: 'prevent unnecessary component renders', categories: ['react', 'performance'] },
+    // Database queries
+    { query: 'SQL relational database open source', categories: ['database'] },
+    { query: 'query performance optimization indexes', categories: ['database', 'performance'] },
+    { query: 'document store nosql database', categories: ['database'] },
+    { query: 'in-memory caching data structure', categories: ['database', 'performance'] },
+    // DevOps queries
+    { query: 'container orchestration deployment', categories: ['devops'] },
+    { query: 'automate build test deploy pipeline', categories: ['devops'] },
+    { query: 'infrastructure configuration management', categories: ['devops'] },
+    { query: 'monitoring metrics collection', categories: ['devops'] },
+    // Architecture queries
+    { query: 'distributed services communication patterns', categories: ['architecture'] },
+    { query: 'domain logic separation layers', categories: ['architecture'] },
+    { query: 'event sourcing state management', categories: ['architecture'] },
+    { query: 'read write separation scalability', categories: ['architecture'] },
+    // Security queries
+    { query: 'token based authentication API', categories: ['security'] },
+    { query: 'prevent injection attacks input validation', categories: ['security'] },
+    { query: 'encrypt data transmission HTTPS', categories: ['security'] },
+    { query: 'rate limiting prevent abuse', categories: ['security'] },
+    // Testing queries
+    { query: 'isolated function verification unit tests', categories: ['testing'] },
+    { query: 'component integration verification', categories: ['testing'] },
+    { query: 'test driven development methodology', categories: ['testing'] },
+    { query: 'code coverage test metrics', categories: ['testing'] },
+    // Performance queries
+    { query: 'speed up data retrieval caching', categories: ['performance'] },
+    { query: 'reduce bundle size code splitting', categories: ['performance'] },
+    { query: 'distribute load multiple servers', categories: ['performance'] },
+    { query: 'identify memory leaks profiling', categories: ['performance'] },
+    // Cross-category queries
+    {
+        query: 'building scalable web applications',
+        categories: ['react', 'architecture', 'performance'],
+    },
+    { query: 'secure API development best practices', categories: ['security', 'architecture'] },
+    { query: 'automated testing CI/CD integration', categories: ['testing', 'devops'] },
+    {
+        query: 'database performance optimization techniques',
+        categories: ['database', 'performance'],
+    },
+];
+/**
+ * Generate a synthetic test dataset for benchmarking.
+ * Uses diverse topics with known relationships for ground truth.
+ *
+ * @param size - 'small' (20 memories, 12 queries), 'medium' (64 memories, 36 queries), 'large' (all)
+ */
+export function generateTestDataset(size = 'small') {
+    // Select categories based on size
+    const categoryNames = size === 'small'
+        ? ['typescript', 'react', 'database', 'devops', 'architecture']
+        : Object.keys(EXTENDED_CATEGORIES);
+    const itemsPerCategory = size === 'small' ? 4 : 8;
+    // Build categories object
+    const categories = {};
+    for (const name of categoryNames) {
+        categories[name] = EXTENDED_CATEGORIES[name].slice(0, itemsPerCategory);
+    }
+    // Flatten memories with category info
+    const memories = [];
+    let idCounter = 1;
+    const categoryIds = {};
+    for (const [category, items] of Object.entries(categories)) {
+        categoryIds[category] = [];
+        for (const item of items) {
+            memories.push({ ...item, category });
+            categoryIds[category].push(idCounter);
+            idCounter++;
+        }
+    }
+    // Generate queries based on size
+    const queries = [];
+    if (size === 'small') {
+        // Original small dataset queries
+        queries.push({ query: 'strongly typed javascript superset language', relevantIds: categoryIds.typescript }, { query: 'type checking and interfaces in code', relevantIds: categoryIds.typescript }, { query: 'UI component library for frontend', relevantIds: categoryIds.react }, { query: 'managing state in functional components', relevantIds: categoryIds.react }, { query: 'SQL relational database open source', relevantIds: categoryIds.database }, { query: 'query performance optimization indexes', relevantIds: categoryIds.database }, { query: 'container orchestration deployment', relevantIds: [...categoryIds.devops] }, { query: 'automate build test deploy pipeline', relevantIds: categoryIds.devops }, {
+            query: 'distributed services communication patterns',
+            relevantIds: categoryIds.architecture,
+        }, { query: 'domain logic separation layers', relevantIds: categoryIds.architecture }, {
+            query: 'building scalable web applications',
+            relevantIds: [...categoryIds.react, ...categoryIds.architecture],
+        }, { query: 'data storage and retrieval systems', relevantIds: categoryIds.database });
+    }
+    else {
+        // Extended queries for medium/large
+        for (const q of EXTENDED_QUERIES) {
+            // Only include queries where all required categories exist
+            const relevantCategories = q.categories.filter((c) => categoryIds[c]);
+            if (relevantCategories.length > 0) {
+                const relevantIds = relevantCategories.flatMap((c) => categoryIds[c] || []);
+                queries.push({ query: q.query, relevantIds });
+            }
+        }
+    }
+    return { memories, queries };
+}
+/**
+ * Generate a unique benchmark ID based on timestamp and random suffix
+ */
+export function generateBenchmarkId() {
+    const timestamp = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19);
+    const suffix = Math.random().toString(36).slice(2, 6);
+    return `bench-${timestamp}-${suffix}`;
+}
+/**
+ * Compare two benchmark results and return regression/improvement info
+ */
+export function compareBenchmarks(current, previous) {
+    const recallDelta = current.recallAtK - previous.recallAtK;
+    const precisionDelta = current.precisionAtK - previous.precisionAtK;
+    const f1Delta = current.f1AtK - previous.f1AtK;
+    const mrrDelta = current.mrr - previous.mrr;
+    const ndcgDelta = current.ndcg - previous.ndcg;
+    // Consider it a regression if F1 or MRR dropped by more than 5%
+    const isRegression = f1Delta < -0.05 || mrrDelta < -0.05;
+    // Consider it an improvement if F1 or MRR improved by more than 5%
+    const isImprovement = f1Delta > 0.05 || mrrDelta > 0.05;
+    return {
+        recallDelta,
+        precisionDelta,
+        f1Delta,
+        mrrDelta,
+        ndcgDelta,
+        isRegression,
+        isImprovement,
+    };
+}
+/**
+ * Format benchmark comparison as human-readable string
+ */
+export function formatBenchmarkComparison(current, previous, previousTimestamp) {
+    const comparison = compareBenchmarks(current, previous);
+    const formatDelta = (d) => {
+        const sign = d >= 0 ? '+' : '';
+        return `${sign}${(d * 100).toFixed(1)}%`;
+    };
+    const status = comparison.isRegression
+        ? '⚠️  REGRESSION DETECTED'
+        : comparison.isImprovement
+            ? '✅ IMPROVEMENT'
+            : '➡️  NO SIGNIFICANT CHANGE';
+    const lines = [
+        '┌─────────────────────────────────────────────────┐',
+        '│         Benchmark Comparison                    │',
+        '├─────────────────────────────────────────────────┤',
+        `│ ${status.padEnd(47)} │`,
+        `│ Previous: ${previousTimestamp.padEnd(36)} │`,
+        '├─────────────────────────────────────────────────┤',
+        `│ Recall@K:    ${formatDelta(comparison.recallDelta).padStart(8)}  (${(previous.recallAtK * 100).toFixed(1)}% → ${(current.recallAtK * 100).toFixed(1)}%) │`,
+        `│ Precision@K: ${formatDelta(comparison.precisionDelta).padStart(8)}  (${(previous.precisionAtK * 100).toFixed(1)}% → ${(current.precisionAtK * 100).toFixed(1)}%) │`,
+        `│ F1@K:        ${formatDelta(comparison.f1Delta).padStart(8)}  (${(previous.f1AtK * 100).toFixed(1)}% → ${(current.f1AtK * 100).toFixed(1)}%) │`,
+        `│ MRR:         ${formatDelta(comparison.mrrDelta).padStart(8)}  (${(previous.mrr * 100).toFixed(1)}% → ${(current.mrr * 100).toFixed(1)}%) │`,
+        `│ NDCG@K:      ${formatDelta(comparison.ndcgDelta).padStart(8)}  (${(previous.ndcg * 100).toFixed(1)}% → ${(current.ndcg * 100).toFixed(1)}%) │`,
+        '└─────────────────────────────────────────────────┘',
+    ];
+    return lines.join('\n');
+}
+/**
+ * Compare hybrid search modes and determine best performer
+ */
+export function compareHybridModes(semantic, bm25, hybrid) {
+    // Use F1 as the primary comparison metric
+    const semanticF1 = semantic.f1AtK;
+    const bm25F1 = bm25.f1AtK;
+    const hybridF1 = hybrid.f1AtK;
+    let bestMode = 'hybrid';
+    if (semanticF1 >= hybridF1 && semanticF1 >= bm25F1) {
+        bestMode = 'semantic';
+    }
+    else if (bm25F1 >= hybridF1 && bm25F1 >= semanticF1) {
+        bestMode = 'bm25';
+    }
+    return {
+        semantic,
+        bm25,
+        hybrid,
+        bestMode,
+        hybridVsSemantic: hybridF1 - semanticF1,
+        hybridVsBm25: hybridF1 - bm25F1,
+    };
+}
+/**
+ * Format hybrid search comparison as human-readable string
+ */
+export function formatHybridComparison(metrics) {
+    const formatPct = (n) => `${(n * 100).toFixed(1)}%`;
+    const formatDelta = (d) => {
+        const sign = d >= 0 ? '+' : '';
+        return `${sign}${(d * 100).toFixed(1)}%`;
+    };
+    const bestIndicator = (mode) => (mode === metrics.bestMode ? ' ⭐' : '');
+    const lines = [
+        '┌───────────────────────────────────────────────────────────────┐',
+        '│              Hybrid Search Comparison                         │',
+        '├───────────┬──────────┬──────────┬──────────┬──────────┬──────┤',
+        '│ Mode      │ Recall@K │ Prec@K   │ F1@K     │ MRR      │ Best │',
+        '├───────────┼──────────┼──────────┼──────────┼──────────┼──────┤',
+        `│ Semantic  │ ${formatPct(metrics.semantic.recallAtK).padStart(8)} │ ${formatPct(metrics.semantic.precisionAtK).padStart(8)} │ ${formatPct(metrics.semantic.f1AtK).padStart(8)} │ ${formatPct(metrics.semantic.mrr).padStart(8)} │${bestIndicator('semantic').padStart(5)} │`,
+        `│ BM25      │ ${formatPct(metrics.bm25.recallAtK).padStart(8)} │ ${formatPct(metrics.bm25.precisionAtK).padStart(8)} │ ${formatPct(metrics.bm25.f1AtK).padStart(8)} │ ${formatPct(metrics.bm25.mrr).padStart(8)} │${bestIndicator('bm25').padStart(5)} │`,
+        `│ Hybrid    │ ${formatPct(metrics.hybrid.recallAtK).padStart(8)} │ ${formatPct(metrics.hybrid.precisionAtK).padStart(8)} │ ${formatPct(metrics.hybrid.f1AtK).padStart(8)} │ ${formatPct(metrics.hybrid.mrr).padStart(8)} │${bestIndicator('hybrid').padStart(5)} │`,
+        '├───────────┴──────────┴──────────┴──────────┴──────────┴──────┤',
+        `│ Hybrid vs Semantic: ${formatDelta(metrics.hybridVsSemantic).padStart(7)} F1                            │`,
+        `│ Hybrid vs BM25:     ${formatDelta(metrics.hybridVsBm25).padStart(7)} F1                            │`,
+        '└───────────────────────────────────────────────────────────────┘',
+    ];
+    return lines.join('\n');
+}
+//# sourceMappingURL=benchmark.js.map