memento-mcp-server 2.0.3 → 2.0.5

package/dist/node_modules/@memento/core/dist/domains/monitoring/services/quality-assurance/search-metrics-collector.js

@@ -1,5 +1,6 @@
 import Database from 'better-sqlite3';
 import { logger } from '../../../../shared/utils/logger.js';
+import { getErrorMessage } from '../../../../shared/services/llm-client-initializer/shared-helpers.js';
 import { calculatePrecisionAtK, calculateRecallAtK, calculateNDCGAtK, } from './search-quality-metrics.js';
 import { calculateKendallTau, loadGroundTruth, generateVectorOnlySearchResults, generateConsolidationSearchResults, } from './vector-search-quality-metrics.js';
 import { loadBenchmarkManifest, assertStrictBenchmark, buildBenchmarkQueryLookup, loadBenchmarkCorpus, loadBenchmarkQueries, } from './search-quality-benchmark-fixtures.js';
@@ -30,303 +31,354 @@ export function calculateMRR(queryResults, groundTruths) {
     }
     return validQueries > 0 ? sumReciprocalRank / groundTruths.length : 0;
 }
+/** Precision@K·Recall@K·NDCG@K (K=5,10) 의 쿼리 평균. 결과가 없는 쿼리는 빼고 센다. */
+function computeRankingMetricsAtK(groundTruths, queryResults) {
+    const metrics = {};
+    const kValues = [5, 10];
+    for (const k of kValues) {
+        let sumPrecision = 0;
+        let sumRecall = 0;
+        let sumNDCG = 0;
+        let validQueries = 0;
+        for (const groundTruth of groundTruths) {
+            const results = queryResults.get(groundTruth.queryId);
+            if (!results || results.length === 0)
+                continue;
+            const precision = calculatePrecisionAtK(results, groundTruth.relevantIds, k);
+            const recall = calculateRecallAtK(results, groundTruth.relevantIds, k);
+            const ndcg = calculateNDCGAtK(results, groundTruth.relevantIds, k);
+            sumPrecision += precision;
+            sumRecall += recall;
+            sumNDCG += ndcg;
+            validQueries++;
+        }
+        if (validQueries > 0) {
+            metrics[`precision_at_${k}`] = sumPrecision / validQueries;
+            metrics[`recall_at_${k}`] = sumRecall / validQueries;
+            metrics[`ndcg_at_${k}`] = sumNDCG / validQueries;
+        }
+        else {
+            metrics[`precision_at_${k}`] = 0;
+            metrics[`recall_at_${k}`] = 0;
+            metrics[`ndcg_at_${k}`] = 0;
+        }
+    }
+    return metrics;
+}
+/** 벡터 단독 대비 consolidation 결과의 Kendall's Tau·Top5/Top10 유지율 평균. */
+function computeOrderPreservationMetrics(searchResultPairs) {
+    const metrics = {};
+    let sumKendallTau = 0;
+    let sumTop5Retention = 0;
+    let sumTop10Retention = 0;
+    let validPairs = 0;
+    for (const pair of searchResultPairs) {
+        const vectorIds = pair.vectorOnly.map(r => r.id);
+        const consolidationIds = pair.withConsolidation.map(r => r.id);
+        const kendallTau = calculateKendallTau(vectorIds, consolidationIds);
+        sumKendallTau += kendallTau;
+        // Top-K 유지율 계산
+        const top5Vector = new Set(vectorIds.slice(0, 5));
+        const top10Vector = new Set(vectorIds.slice(0, 10));
+        const top5Consolidation = new Set(consolidationIds.slice(0, 5));
+        const top10Consolidation = new Set(consolidationIds.slice(0, 10));
+        const top5Retention = Array.from(top5Vector).filter(id => top5Consolidation.has(id)).length / 5;
+        const top10Retention = Array.from(top10Vector).filter(id => top10Consolidation.has(id)).length / 10;
+        sumTop5Retention += top5Retention;
+        sumTop10Retention += top10Retention;
+        validPairs++;
+    }
+    if (validPairs > 0) {
+        metrics.kendalls_tau = sumKendallTau / validPairs;
+        metrics.top_5_retention = sumTop5Retention / validPairs;
+        metrics.top_10_retention = sumTop10Retention / validPairs;
+    }
+    else {
+        metrics.kendalls_tau = 0;
+        metrics.top_5_retention = 0;
+        metrics.top_10_retention = 0;
+    }
+    return metrics;
+}
+/** Ground Truth 가 없을 때 돌려주는 지표 */
+const EMPTY_SEARCH_METRICS = {
+    precision_at_5: 0,
+    precision_at_10: 0,
+    recall_at_5: 0,
+    recall_at_10: 0,
+    ndcg_at_5: 0,
+    ndcg_at_10: 0,
+    mrr: 0,
+    kendalls_tau: 0,
+    top_5_retention: 0,
+    top_10_retention: 0
+};
+const EMPTY_ORDER_PRESERVATION_METRICS = {
+    kendalls_tau: 0,
+    top_5_retention: 0,
+    top_10_retention: 0
+};
+const SEARCH_LIMIT = 20;
+const NO_PAIRS_REASON = '검색 결과가 부족하거나 vectorScore/finalScore가 없을 수 있습니다';
+function hasItems(values) {
+    return values != null && values.length > 0;
+}
+/** manifest → (strict 면 검증) → ground truth 정규화 → corpus id 매핑 → query id→text 매핑 */
+function loadBenchmarkInputs(benchmarkDir, strict) {
+    const manifest = loadBenchmarkManifest(benchmarkDir);
+    if (strict) {
+        assertStrictBenchmark(manifest);
+        const verification = verifyReviewableBenchmark(benchmarkDir, { requireReviewed: true });
+        if (!verification.ok) {
+            throw new Error(`Strict benchmark verification failed: ${verification.errors.join('; ')}`);
+        }
+    }
+    const groundTruths = normalizeBenchmarkGroundTruths(benchmarkDir);
+    let memoryIdToBenchmarkId;
+    try {
+        const corpus = loadBenchmarkCorpus(benchmarkDir);
+        memoryIdToBenchmarkId = new Map(corpus.map((e) => [e.source_memory_id, e.benchmark_id]));
+    }
+    catch (error) {
+        throw new Error(`Benchmark corpus load failed: ${getErrorMessage(error)}`);
+    }
+    const queries = loadBenchmarkQueries(benchmarkDir);
+    const queryIdToQueryText = new Map([...buildBenchmarkQueryLookup(queries).byId.entries()].map(([queryId, query]) => [queryId, query.query]));
+    return { groundTruths, memoryIdToBenchmarkId, queryIdToQueryText };
+}
+/** 옵션에도 benchmarkDir 에도 없을 때 파일에서 Ground Truth 를 읽는다. 없거나 실패하면 undefined. */
+function autoLoadGroundTruths(context) {
+    try {
+        const loaded = loadGroundTruth();
+        if (!hasItems(loaded))
+            return undefined;
+        logger.info('Ground Truth 자동 로드 완료', {
+            context,
+            count: loaded.length
+        });
+        return loaded;
+    }
+    catch (error) {
+        logger.warn('Ground Truth 파일 로드 실패', {
+            context,
+            error: getErrorMessage(error)
+        });
+        return undefined;
+    }
+}
+/** 주어진 queryResults 의 query id 를 쿼리 본문으로, memory id 를 benchmark id 로 바꾼다. */
+function remapQueryResults(queryResults, benchmark) {
+    return new Map([...queryResults.entries()].map(([queryId, results]) => [
+        benchmark?.queryIdToQueryText.get(queryId) ?? queryId,
+        results.map((result) => ({
+            ...result,
+            id: benchmark?.memoryIdToBenchmarkId.get(result.id) ?? result.id,
+        })),
+    ]));
+}
+/** 검색 결과로 vector-only / consolidation 쌍을 만든다. 둘 중 하나라도 2건 미만이면 null. */
+function toSearchResultPair(items, context, query) {
+    const vectorOnlyResults = generateVectorOnlySearchResults(items, SEARCH_LIMIT);
+    const consolidationResults = generateConsolidationSearchResults(items, SEARCH_LIMIT);
+    const added = vectorOnlyResults.length >= 2 && consolidationResults.length >= 2;
+    if (!added) {
+        logger.warn('검색 결과 쌍 생성 실패: 결과 부족', {
+            context,
+            query,
+            searchResultCount: items.length,
+            vectorOnlyCount: vectorOnlyResults.length,
+            consolidationCount: consolidationResults.length,
+            reason: vectorOnlyResults.length < 2 ? 'vectorOnlyResults 부족' : 'consolidationResults 부족'
+        });
+    }
+    logger.debug('검색 결과 쌍 생성 완료', {
+        context,
+        query,
+        searchResultCount: items.length,
+        vectorOnlyCount: vectorOnlyResults.length,
+        consolidationCount: consolidationResults.length,
+        added
+    });
+    return added ? { vectorOnly: vectorOnlyResults, withConsolidation: consolidationResults } : null;
+}
+function logPairSummary(context, pairCount, groundTruthCount) {
+    if (pairCount === 0) {
+        logger.warn('검색 결과 쌍이 생성되지 않았습니다', {
+            context,
+            groundTruthCount,
+            reason: NO_PAIRS_REASON
+        });
+    }
+    else {
+        logger.info('검색 결과 쌍 자동 생성 완료', {
+            context,
+            pairCount,
+            groundTruthCount
+        });
+    }
+}
+/** has_ground_truth: 옵션으로 제공되었거나 benchmarkDir 에서 로드된 경우 true */
+function groundTruthMetadata(options, groundTruths, queryResults) {
+    const loaded = hasItems(groundTruths);
+    const hasExplicitGroundTruth = hasItems(options?.groundTruths) || (options?.benchmarkDir !== undefined && loaded);
+    const hasAutoLoadedGroundTruth = !options?.groundTruths && !options?.benchmarkDir && loaded;
+    return {
+        has_ground_truth: hasExplicitGroundTruth,
+        ground_truth_count: hasExplicitGroundTruth ? (groundTruths?.length ?? 0) : 0,
+        auto_loaded: hasAutoLoadedGroundTruth,
+        auto_searched: !options?.queryResults && queryResults !== undefined
+    };
+}
 export class SearchMetricsCollector {
     db;
-    constructor(db) {
+    repositories;
+    constructor(db, repositories) {
         this.db = db;
+        this.repositories = repositories;
     }
     async collect(context = 'default', options) {
-        const metrics = {};
-        let groundTruths = options?.groundTruths;
-        let memoryIdToBenchmarkId;
-        let queryIdToQueryText;
         if (options?.strictBenchmark && !options.benchmarkDir) {
             throw new Error('Strict benchmark mode requires benchmarkDir');
         }
-        if (options?.benchmarkDir) {
-            const manifest = loadBenchmarkManifest(options.benchmarkDir);
-            if (options.strictBenchmark) {
-                assertStrictBenchmark(manifest);
-                const verification = verifyReviewableBenchmark(options.benchmarkDir, { requireReviewed: true });
-                if (!verification.ok) {
-                    throw new Error(`Strict benchmark verification failed: ${verification.errors.join('; ')}`);
-                }
-            }
-            groundTruths = normalizeBenchmarkGroundTruths(options.benchmarkDir);
-            try {
-                const corpus = loadBenchmarkCorpus(options.benchmarkDir);
-                memoryIdToBenchmarkId = new Map(corpus.map((e) => [e.source_memory_id, e.benchmark_id]));
-            }
-            catch (error) {
-                throw new Error(`Benchmark corpus load failed: ${error instanceof Error ? error.message : String(error)}`);
-            }
-            const queries = loadBenchmarkQueries(options.benchmarkDir);
-            queryIdToQueryText = new Map([...buildBenchmarkQueryLookup(queries).byId.entries()].map(([queryId, query]) => [queryId, query.query]));
-        }
-        if (options?.strictBenchmark && (!groundTruths || groundTruths.length === 0)) {
+        const benchmark = options?.benchmarkDir
+            ? loadBenchmarkInputs(options.benchmarkDir, options.strictBenchmark === true)
+            : undefined;
+        let groundTruths = benchmark ? benchmark.groundTruths : options?.groundTruths;
+        if (options?.strictBenchmark && !hasItems(groundTruths)) {
             throw new Error('Strict benchmark mode requires human-labeled ground truth');
         }
-        // Ground Truth 자동 로드 (옵션에 없고 benchmarkDir도 없으면 파일에서 로드 시도)
-        if (!groundTruths || groundTruths.length === 0) {
-            try {
-                const loaded = loadGroundTruth();
-                if (loaded && loaded.length > 0) {
-                    groundTruths = loaded;
-                    logger.info('Ground Truth 자동 로드 완료', {
-                        context,
-                        count: groundTruths.length
-                    });
-                }
-            }
-            catch (error) {
-                logger.warn('Ground Truth 파일 로드 실패', {
-                    context,
-                    error: error instanceof Error ? error.message : String(error)
-                });
-            }
+        if (!hasItems(groundTruths)) {
+            groundTruths = autoLoadGroundTruths(context) ?? groundTruths;
         }
-        // 검색 결과 자동 생성 (옵션에 없고 Ground Truth가 있으면 실제 검색 수행, 병렬로 한 번만)
-        let queryResults = options?.queryResults;
         const fullResultsByQuery = new Map();
-        if (queryResults) {
-            queryResults = new Map([...queryResults.entries()].map(([queryId, results]) => [
-                queryIdToQueryText?.get(queryId) ?? queryId,
-                results.map((result) => ({
-                    ...result,
-                    id: memoryIdToBenchmarkId?.get(result.id) ?? result.id,
-                })),
-            ]));
+        let queryResults;
+        if (options?.queryResults) {
+            queryResults = remapQueryResults(options.queryResults, benchmark);
         }
-        if (!queryResults && groundTruths && groundTruths.length > 0) {
-            try {
-                queryResults = new Map();
-                const searchEngine = HybridSearchFactory.createDefaultEngine(this.db);
-                const searchPromises = groundTruths.map(async (groundTruth) => {
-                    try {
-                        const searchResult = await searchEngine.search(this.db, { query: groundTruth.queryId, limit: 20 });
-                        return { queryId: groundTruth.queryId, searchResult };
-                    }
-                    catch (error) {
-                        logger.warn('검색 수행 실패', { context, query: groundTruth.queryId, error: error instanceof Error ? error.message : String(error) });
-                        return { queryId: groundTruth.queryId, searchResult: { items: [] } };
-                    }
-                });
-                const resolved = await Promise.all(searchPromises);
-                for (const { queryId, searchResult } of resolved) {
-                    const mapped = searchResult.items.map((item) => {
-                        const id = memoryIdToBenchmarkId?.get(item.id) ?? item.id;
-                        return { id, score: item.finalScore };
-                    });
-                    queryResults.set(queryId, mapped);
-                    fullResultsByQuery.set(queryId, searchResult);
-                }
-                logger.info('검색 결과 자동 생성 완료', { context, queryCount: queryResults.size });
-            }
-            catch (error) {
-                logger.warn('검색 엔진 초기화 또는 검색 수행 실패', {
-                    context,
-                    error: error instanceof Error ? error.message : String(error)
-                });
-                queryResults = new Map();
-            }
+        else if (hasItems(groundTruths)) {
+            queryResults = await this.searchGroundTruths(groundTruths, benchmark, fullResultsByQuery, context);
         }
-        // Ground Truth와 검색 결과가 제공된 경우 실제 측정 수행
-        if (groundTruths && queryResults && groundTruths.length > 0) {
-            // groundTruths와 queryResults는 위에서 로드/생성됨
-            // Precision@K, Recall@K, NDCG@K 계산
-            const kValues = [5, 10];
-            for (const k of kValues) {
-                let sumPrecision = 0;
-                let sumRecall = 0;
-                let sumNDCG = 0;
-                let validQueries = 0;
-                for (const groundTruth of groundTruths) {
-                    const results = queryResults.get(groundTruth.queryId);
-                    if (!results || results.length === 0)
-                        continue;
-                    const precision = calculatePrecisionAtK(results, groundTruth.relevantIds, k);
-                    const recall = calculateRecallAtK(results, groundTruth.relevantIds, k);
-                    const ndcg = calculateNDCGAtK(results, groundTruth.relevantIds, k);
-                    sumPrecision += precision;
-                    sumRecall += recall;
-                    sumNDCG += ndcg;
-                    validQueries++;
-                }
-                if (validQueries > 0) {
-                    metrics[`precision_at_${k}`] = sumPrecision / validQueries;
-                    metrics[`recall_at_${k}`] = sumRecall / validQueries;
-                    metrics[`ndcg_at_${k}`] = sumNDCG / validQueries;
-                }
-                else {
-                    metrics[`precision_at_${k}`] = 0;
-                    metrics[`recall_at_${k}`] = 0;
-                    metrics[`ndcg_at_${k}`] = 0;
-                }
-            }
-            // MRR 계산
-            metrics.mrr = calculateMRR(queryResults, groundTruths);
-            // searchResultPairs 자동 생성 (제공되지 않은 경우, fullResultsByQuery 재사용)
-            let searchResultPairs = options?.searchResultPairs;
-            if (!searchResultPairs || searchResultPairs.length === 0) {
+        const metrics = hasItems(groundTruths) && queryResults
+            ? await this.measure(groundTruths, queryResults, fullResultsByQuery, options?.searchResultPairs, context)
+            : this.emptyMetrics(context);
+        return {
+            namespace: 'search',
+            context,
+            measured_at: new Date().toISOString(),
+            metrics,
+            metadata: groundTruthMetadata(options, groundTruths, queryResults)
+        };
+    }
+    emptyMetrics(context) {
+        logger.info('검색 품질 지표 수집 완료 (기본값)', {
+            context,
+            note: 'Ground Truth 데이터가 없어 기본값을 반환했습니다. 실제 측정을 위해서는 Ground Truth 데이터가 필요합니다.'
+        });
+        return { ...EMPTY_SEARCH_METRICS };
+    }
+    /**
+     * Ground Truth 쿼리를 엔진 하나로 병렬 검색한다. 원본 결과는 fullResultsByQuery 에 남긴다.
+     * 엔진 초기화가 실패하면 빈 Map. 쿼리 하나의 실패는 그 쿼리만 빈 결과로 둔다.
+     */
+    async searchGroundTruths(groundTruths, benchmark, fullResultsByQuery, context) {
+        try {
+            const queryResults = new Map();
+            const searchEngine = HybridSearchFactory.createDefaultEngine(this.repositories);
+            const resolved = await Promise.all(groundTruths.map(async (groundTruth) => {
                 try {
-                    searchResultPairs = [];
-                    for (const groundTruth of groundTruths) {
-                        let items = fullResultsByQuery.get(groundTruth.queryId)?.items ?? [];
-                        if (items.length === 0) {
-                            const searchEngine = HybridSearchFactory.createDefaultEngine(this.db);
-                            const sr = await searchEngine.search(this.db, { query: groundTruth.queryId, limit: 20 });
-                            items = sr.items;
-                        }
-                        try {
-                            const vectorOnlyResults = generateVectorOnlySearchResults(items, 20);
-                            const consolidationResults = generateConsolidationSearchResults(items, 20);
-                            if (vectorOnlyResults.length >= 2 && consolidationResults.length >= 2) {
-                                searchResultPairs.push({
-                                    vectorOnly: vectorOnlyResults,
-                                    withConsolidation: consolidationResults
-                                });
-                            }
-                            else {
-                                logger.warn('검색 결과 쌍 생성 실패: 결과 부족', {
-                                    context,
-                                    query: groundTruth.queryId,
-                                    searchResultCount: items.length,
-                                    vectorOnlyCount: vectorOnlyResults.length,
-                                    consolidationCount: consolidationResults.length,
-                                    reason: vectorOnlyResults.length < 2
-                                        ? 'vectorOnlyResults 부족'
-                                        : consolidationResults.length < 2
-                                            ? 'consolidationResults 부족'
-                                            : '알 수 없음'
-                                });
-                            }
-                            logger.debug('검색 결과 쌍 생성 완료', {
-                                context,
-                                query: groundTruth.queryId,
-                                searchResultCount: items.length,
-                                vectorOnlyCount: vectorOnlyResults.length,
-                                consolidationCount: consolidationResults.length,
-                                added: vectorOnlyResults.length >= 2 && consolidationResults.length >= 2
-                            });
-                        }
-                        catch (error) {
-                            logger.warn('검색 결과 쌍 생성 실패', {
-                                context,
-                                query: groundTruth.queryId,
-                                error: error instanceof Error ? error.message : String(error)
-                            });
-                        }
-                    }
-                    if (searchResultPairs.length === 0) {
-                        logger.warn('검색 결과 쌍이 생성되지 않았습니다', {
-                            context,
-                            groundTruthCount: groundTruths.length,
-                            reason: '검색 결과가 부족하거나 vectorScore/finalScore가 없을 수 있습니다'
-                        });
-                    }
-                    else {
-                        logger.info('검색 결과 쌍 자동 생성 완료', {
-                            context,
-                            pairCount: searchResultPairs.length,
-                            groundTruthCount: groundTruths.length
-                        });
-                    }
+                    const searchResult = await searchEngine.search(this.db, { query: groundTruth.queryId, limit: SEARCH_LIMIT });
+                    return { queryId: groundTruth.queryId, searchResult };
                 }
                 catch (error) {
-                    logger.warn('검색 결과 쌍 자동 생성 실패', {
-                        context,
-                        error: error instanceof Error ? error.message : String(error)
-                    });
-                    searchResultPairs = [];
+                    logger.warn('검색 수행 실패', { context, query: groundTruth.queryId, error: getErrorMessage(error) });
+                    return { queryId: groundTruth.queryId, searchResult: { items: [] } };
                 }
+            }));
+            for (const { queryId, searchResult } of resolved) {
+                queryResults.set(queryId, searchResult.items.map((item) => ({
+                    id: benchmark?.memoryIdToBenchmarkId.get(item.id) ?? item.id,
+                    score: item.finalScore
+                })));
+                fullResultsByQuery.set(queryId, searchResult);
             }
-            // Kendall's Tau 및 순서 보존 지표 계산
-            if (searchResultPairs && searchResultPairs.length > 0) {
-                let sumKendallTau = 0;
-                let sumTop5Retention = 0;
-                let sumTop10Retention = 0;
-                let validPairs = 0;
-                for (const pair of searchResultPairs) {
-                    const vectorIds = pair.vectorOnly.map(r => r.id);
-                    const consolidationIds = pair.withConsolidation.map(r => r.id);
-                    const kendallTau = calculateKendallTau(vectorIds, consolidationIds);
-                    sumKendallTau += kendallTau;
-                    // Top-K 유지율 계산
-                    const top5Vector = new Set(vectorIds.slice(0, 5));
-                    const top10Vector = new Set(vectorIds.slice(0, 10));
-                    const top5Consolidation = new Set(consolidationIds.slice(0, 5));
-                    const top10Consolidation = new Set(consolidationIds.slice(0, 10));
-                    const top5Retention = Array.from(top5Vector).filter(id => top5Consolidation.has(id)).length / 5;
-                    const top10Retention = Array.from(top10Vector).filter(id => top10Consolidation.has(id)).length / 10;
-                    sumTop5Retention += top5Retention;
-                    sumTop10Retention += top10Retention;
-                    validPairs++;
-                }
-                if (validPairs > 0) {
-                    metrics.kendalls_tau = sumKendallTau / validPairs;
-                    metrics.top_5_retention = sumTop5Retention / validPairs;
-                    metrics.top_10_retention = sumTop10Retention / validPairs;
-                }
-                else {
-                    metrics.kendalls_tau = 0;
-                    metrics.top_5_retention = 0;
-                    metrics.top_10_retention = 0;
-                }
-            }
-            else {
-                metrics.kendalls_tau = 0;
-                metrics.top_5_retention = 0;
-                metrics.top_10_retention = 0;
-                logger.warn('Kendall\'s Tau 계산 불가: searchResultPairs가 비어있습니다', {
-                    context,
-                    groundTruthCount: groundTruths.length,
-                    reason: '검색 결과가 부족하거나 vectorScore/finalScore가 없을 수 있습니다'
-                });
-            }
-            logger.info('검색 품질 지표 수집 완료', {
+            logger.info('검색 결과 자동 생성 완료', { context, queryCount: queryResults.size });
+            return queryResults;
+        }
+        catch (error) {
+            logger.warn('검색 엔진 초기화 또는 검색 수행 실패', {
                 context,
-                metrics_count: Object.keys(metrics).length,
-                ground_truth_count: groundTruths.length,
-                kendalls_tau: metrics.kendalls_tau,
-                searchResultPairsCount: searchResultPairs?.length || 0
+                error: getErrorMessage(error)
             });
+            return new Map();
+        }
+    }
+    /** 랭킹 지표·MRR·순서 보존 지표 */
+    async measure(groundTruths, queryResults, fullResultsByQuery, givenPairs, context) {
+        const metrics = {
+            ...computeRankingMetricsAtK(groundTruths, queryResults),
+            mrr: calculateMRR(queryResults, groundTruths)
+        };
+        const searchResultPairs = hasItems(givenPairs)
+            ? givenPairs
+            : await this.buildSearchResultPairs(groundTruths, fullResultsByQuery, context);
+        if (searchResultPairs.length > 0) {
+            Object.assign(metrics, computeOrderPreservationMetrics(searchResultPairs));
         }
         else {
-            // Ground Truth가 없으면 기본값 반환
-            metrics.precision_at_5 = 0;
-            metrics.precision_at_10 = 0;
-            metrics.recall_at_5 = 0;
-            metrics.recall_at_10 = 0;
-            metrics.ndcg_at_5 = 0;
-            metrics.ndcg_at_10 = 0;
-            metrics.mrr = 0;
-            metrics.kendalls_tau = 0;
-            metrics.top_5_retention = 0;
-            metrics.top_10_retention = 0;
-            logger.info('검색 품질 지표 수집 완료 (기본값)', {
+            Object.assign(metrics, EMPTY_ORDER_PRESERVATION_METRICS);
+            logger.warn('Kendall\'s Tau 계산 불가: searchResultPairs가 비어있습니다', {
                 context,
-                note: 'Ground Truth 데이터가 없어 기본값을 반환했습니다. 실제 측정을 위해서는 Ground Truth 데이터가 필요합니다.'
+                groundTruthCount: groundTruths.length,
+                reason: NO_PAIRS_REASON
             });
         }
-        // has_ground_truth: 옵션으로 제공되었거나 benchmarkDir에서 로드된 경우 true
-        const hasExplicitGroundTruth = (options?.groundTruths !== undefined && options.groundTruths.length > 0) ||
-            (options?.benchmarkDir !== undefined && groundTruths !== undefined && groundTruths.length > 0);
-        const hasAutoLoadedGroundTruth = !options?.groundTruths &&
-            !options?.benchmarkDir &&
-            groundTruths !== undefined &&
-            groundTruths.length > 0;
-        return {
-            namespace: 'search',
+        logger.info('검색 품질 지표 수집 완료', {
             context,
-            measured_at: new Date().toISOString(),
-            metrics,
-            metadata: {
-                has_ground_truth: hasExplicitGroundTruth,
-                ground_truth_count: hasExplicitGroundTruth ? (groundTruths?.length ?? 0) : 0,
-                auto_loaded: hasAutoLoadedGroundTruth,
-                auto_searched: !options?.queryResults && queryResults !== undefined
+            metrics_count: Object.keys(metrics).length,
+            ground_truth_count: groundTruths.length,
+            kendalls_tau: metrics.kendalls_tau,
+            searchResultPairsCount: searchResultPairs.length
+        });
+        return metrics;
+    }
+    /**
+     * fullResultsByQuery 를 재사용해 쌍을 만든다. 결과가 없는 쿼리(queryResults 를 옵션으로 받았거나
+     * 0건이던 쿼리)는 다시 검색한다. 엔진은 처음 필요할 때 한 번만 만든다.
+     * 검색·엔진 오류가 나면 전체를 빈 배열로 둔다.
+     */
+    async buildSearchResultPairs(groundTruths, fullResultsByQuery, context) {
+        try {
+            const pairs = [];
+            let searchEngine;
+            for (const groundTruth of groundTruths) {
+                let items = fullResultsByQuery.get(groundTruth.queryId)?.items ?? [];
+                if (items.length === 0) {
+                    searchEngine ??= HybridSearchFactory.createDefaultEngine(this.repositories);
+                    items = (await searchEngine.search(this.db, { query: groundTruth.queryId, limit: SEARCH_LIMIT })).items;
+                }
+                try {
+                    const pair = toSearchResultPair(items, context, groundTruth.queryId);
+                    if (pair)
+                        pairs.push(pair);
+                }
+                catch (error) {
+                    logger.warn('검색 결과 쌍 생성 실패', {
+                        context,
+                        query: groundTruth.queryId,
+                        error: getErrorMessage(error)
+                    });
+                }
             }
-        };
+            logPairSummary(context, pairs.length, groundTruths.length);
+            return pairs;
+        }
+        catch (error) {
+            logger.warn('검색 결과 쌍 자동 생성 실패', {
+                context,
+                error: getErrorMessage(error)
+            });
+            return [];
+        }
     }
 }
 //# sourceMappingURL=search-metrics-collector.js.map