memento-mcp-server 1.15.0 → 1.16.0-b
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/domains/relation/tools/visualize-relations-tool.d.ts +2 -2
- package/dist/infrastructure/database/database/migration/migrations/009-quality-assurance-schema.d.ts +60 -0
- package/dist/infrastructure/database/database/migration/migrations/009-quality-assurance-schema.d.ts.map +1 -0
- package/dist/infrastructure/database/database/migration/migrations/009-quality-assurance-schema.js +276 -0
- package/dist/infrastructure/database/database/migration/migrations/009-quality-assurance-schema.js.map +1 -0
- package/dist/infrastructure/database/database/migration/migrations/009-quality-assurance-schema.sql +128 -0
- package/dist/infrastructure/scheduler/batch-scheduler.d.ts +17 -0
- package/dist/infrastructure/scheduler/batch-scheduler.d.ts.map +1 -1
- package/dist/infrastructure/scheduler/batch-scheduler.js +130 -0
- package/dist/infrastructure/scheduler/batch-scheduler.js.map +1 -1
- package/dist/infrastructure/scheduler/jobs/quality-measurement-batch-job.d.ts +108 -0
- package/dist/infrastructure/scheduler/jobs/quality-measurement-batch-job.d.ts.map +1 -0
- package/dist/infrastructure/scheduler/jobs/quality-measurement-batch-job.js +184 -0
- package/dist/infrastructure/scheduler/jobs/quality-measurement-batch-job.js.map +1 -0
- package/dist/server/http-server.d.ts.map +1 -1
- package/dist/server/http-server.js +3 -0
- package/dist/server/http-server.js.map +1 -1
- package/dist/server/routes/quality.routes.d.ts +14 -0
- package/dist/server/routes/quality.routes.d.ts.map +1 -0
- package/dist/server/routes/quality.routes.js +460 -0
- package/dist/server/routes/quality.routes.js.map +1 -0
- package/dist/services/quality-assurance/quality-assurance-service.d.ts +207 -0
- package/dist/services/quality-assurance/quality-assurance-service.d.ts.map +1 -0
- package/dist/services/quality-assurance/quality-assurance-service.js +247 -0
- package/dist/services/quality-assurance/quality-assurance-service.js.map +1 -0
- package/dist/services/quality-assurance/quality-evaluator.d.ts +163 -0
- package/dist/services/quality-assurance/quality-evaluator.d.ts.map +1 -0
- package/dist/services/quality-assurance/quality-evaluator.js +256 -0
- package/dist/services/quality-assurance/quality-evaluator.js.map +1 -0
- package/dist/services/quality-assurance/quality-metrics-collector.d.ts +221 -0
- package/dist/services/quality-assurance/quality-metrics-collector.d.ts.map +1 -0
- package/dist/services/quality-assurance/quality-metrics-collector.js +796 -0
- package/dist/services/quality-assurance/quality-metrics-collector.js.map +1 -0
- package/dist/services/quality-assurance/quality-recorder.d.ts +108 -0
- package/dist/services/quality-assurance/quality-recorder.d.ts.map +1 -0
- package/dist/services/quality-assurance/quality-recorder.js +281 -0
- package/dist/services/quality-assurance/quality-recorder.js.map +1 -0
- package/dist/services/quality-assurance/quality-reporter.d.ts +189 -0
- package/dist/services/quality-assurance/quality-reporter.d.ts.map +1 -0
- package/dist/services/quality-assurance/quality-reporter.js +558 -0
- package/dist/services/quality-assurance/quality-reporter.js.map +1 -0
- package/dist/services/quality-assurance/quality-threshold-manager.d.ts +102 -0
- package/dist/services/quality-assurance/quality-threshold-manager.d.ts.map +1 -0
- package/dist/services/quality-assurance/quality-threshold-manager.js +252 -0
- package/dist/services/quality-assurance/quality-threshold-manager.js.map +1 -0
- package/dist/shared/utils/database.d.ts +7 -0
- package/dist/shared/utils/database.d.ts.map +1 -1
- package/dist/shared/utils/database.js +24 -0
- package/dist/shared/utils/database.js.map +1 -1
- package/dist/test/helpers/search-quality-metrics.d.ts +96 -0
- package/dist/test/helpers/search-quality-metrics.d.ts.map +1 -0
- package/dist/test/helpers/search-quality-metrics.js +185 -0
- package/dist/test/helpers/search-quality-metrics.js.map +1 -0
- package/dist/test/helpers/vector-search-quality-metrics.d.ts +1287 -0
- package/dist/test/helpers/vector-search-quality-metrics.d.ts.map +1 -0
- package/dist/test/helpers/vector-search-quality-metrics.js +2214 -0
- package/dist/test/helpers/vector-search-quality-metrics.js.map +1 -0
- package/package.json +5 -1
- package/scripts/generate-ground-truth.ts +353 -0
- package/scripts/quality-report.ts +166 -0
- package/scripts/quality-thresholds.ts +279 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"quality-evaluator.js","sourceRoot":"","sources":["../../../src/services/quality-assurance/quality-evaluator.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,QAAQ,MAAM,gBAAgB,CAAC;AACtC,OAAO,UAAU,MAAM,aAAa,CAAC;AACrC,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,uBAAuB,EAAyB,MAAM,gCAAgC,CAAC;AAEhG,OAAO,EAAE,MAAM,EAAE,MAAM,8BAA8B,CAAC;AAmGtD;;;;GAIG;AACH,MAAM,OAAO,gBAAgB;IAGP;IAFZ,gBAAgB,CAA0B;IAElD,YAAoB,EAAqB;QAArB,OAAE,GAAF,EAAE,CAAmB;QACvC,IAAI,CAAC,EAAE,EAAE,CAAC;YACR,MAAM,IAAI,KAAK,CAAC,+BAA+B,CAAC,CAAC;QACnD,CAAC;QACD,IAAI,CAAC,gBAAgB,GAAG,IAAI,uBAAuB,CAAC,EAAE,CAAC,CAAC;IAC1D,CAAC;IAED;;;;;;;;OAQG;IACH,cAAc,CACZ,SAAiB,EACjB,GAAW,EACX,KAAa,EACb,UAAkB,SAAS;QAE3B,MAAM,UAAU,GAAG,IAAI,CAAC,gBAAgB,CAAC,iBAAiB,CAAC,SAAS,EAAE,GAAG,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;QAE3F,IAAI,UAAU,GAAkB,IAAI,CAAC;QACrC,IAAI,UAAU,CAAC,SAAS,EAAE,CAAC;YACzB,IAAI,UAAU,CAAC,SAAS,CAAC,cAAc,KAAK,KAAK,EAAE,CAAC;gBAClD,mDAAmD;gBACnD,UAAU,GAAG,KAAK,GAAG,UAAU,CAAC,SAAS,CAAC,eAAe,CAAC;YAC5D,CAAC;iBAAM,IAAI,UAAU,CAAC,SAAS,CAAC,cAAc,KAAK,KAAK,EAAE,CAAC;gBACzD,mDAAmD;gBACnD,UAAU,GAAG,UAAU,CAAC,SAAS,CAAC,eAAe,GAAG,KAAK,CAAC;YAC5D,CAAC;QACH,CAAC;QAED,OAAO;YACL,SAAS;YACT,GAAG;YACH,KAAK;YACL,SAAS,EAAE,UAAU,CAAC,SAAS;YAC/B,MAAM,EAAE,UAAU,CAAC,MAAM;YACzB,UAAU;YACV,OAAO,EAAE,UAAU,CAAC,OAAO;SAC5B,CAAC;IACJ,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,eAAe,CACnB,OAAyB,EACzB,UAAkB,SAAS;QAE3B,MAAM,aAAa,GAA6B,EAAE,CAAC;QACnD,MAAM,QAAQ,GAAwC,EAAE,CAAC;QAEzD,WAAW;QACX,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;YAC3D,MAAM,MAAM,GAAG,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,SAAS,EAAE,GAAG,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;YAC3E,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAE3B,iBAAiB;YACjB,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;gBACvC,QAAQ,CAAC,IAAI,CAAC;oBACZ,UAAU,EAAE,GAAG;oBACf,KAAK;oBACL,eAAe,EAAE,MAAM,CAAC,SAAS,CAAC,eAAe;oBACjD,UAAU,EAAE,MAAM,CAAC,UAAU,IAAI,CAAC;oBAClC,OAAO,EAAE,MAAM,CAAC,OAAO;iBACxB,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,QAAQ;QACR,MAAM,WAAW,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;QAC/D,MAAM,WAAW,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;QAChE,MAAM,UAAU,GAAG,aAAa,CAAC,MAAM,CAAC;QAExC,YAAY;QACZ,oBAAoB;QACpB,wDAAwD;QACxD,wCAAwC;QACxC,IAAI,MAAM,GAAgC,MAAM,CAAC;QAEjD,IAAI,WAAW,GAAG,CAAC,EAAE,CAAC;YACpB,gCAAgC;YAChC,MAAM,mBAAmB,GAAG,aAAa,CAAC,MAAM,CAC9C,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,SAAS,KAAK,IAAI,CACvC,CAAC,MAAM,CAAC;YAET,IAAI,mBAAmB,GAAG,CAAC,EAAE,CAAC;gBAC5B,MAAM,GAAG,MAAM,CAAC;YAClB,CAAC;iBAAM,CAAC;gBACN,mCAAmC;gBACnC,MAAM,GAAG,SAAS,CAAC;YACrB,CAAC;QACH,CAAC;QAED,WAAW;QACX,MAAM,gBAAgB,GAA4B;YAChD,SAAS,EAAE,OAAO,CAAC,SAAS;YAC5B,OAAO;YACP,MAAM;YACN,aAAa;YACb,WAAW;YACX,WAAW;YACX,UAAU;YACV,QAAQ;YACR,YAAY,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;SACvC,CAAC;QAEF,uBAAuB;QACvB,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACxB,MAAM,CAAC,IAAI,CAAC,aAAa,OAAO,CAAC,SAAS,KAAK,OAAO,GAAG,EAAE;gBACzD,SAAS,EAAE,OAAO,CAAC,SAAS;gBAC5B,OAAO;gBACP,QAAQ,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;oBAC3B,MAAM,EAAE,CAAC,CAAC,UAAU;oBACpB,KAAK,EAAE,CAAC,CAAC,KAAK;oBACd,SAAS,EAAE,CAAC,CAAC,eAAe;oBAC5B,UAAU,EAAE,CAAC,CAAC,UAAU;iBACzB,CAAC,CAAC;aACJ,CAAC,CAAC;YAEH,iCAAiC;YACjC,MAAM,IAAI,CAAC,gBAAgB,CAAC,gBAAgB,CAAC,CAAC;QAChD,CAAC;QAED,OAAO,gBAAgB,CAAC;IAC1B,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,kBAAkB,CACtB,WAA+B,EAC/B,UAAkB,SAAS;QAE3B,OAAO,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC;IACzF,CAAC;IAED;;;;;;;OAOG;IACH,sBAAsB,CAAC,iBAA4C;QACjE,IAAI,iBAAiB,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACnC,OAAO,MAAM,CAAC;QAChB,CAAC;QAED,yBAAyB;QACzB,IAAI,iBAAiB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,EAAE,CAAC;YACrD,OAAO,MAAM,CAAC;QAChB,CAAC;QAED,+BAA+B;QAC/B,IAAI,iBAAiB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,SAAS,CAAC,EAAE,CAAC;YACxD,OAAO,SAAS,CAAC;QACnB,CAAC;QAED,mBAAmB;QACnB,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;;;;;;OAOG;IACH,mBAAmB,CAAC,gBAAyC;QAC3D,IAAI,gBAAgB,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3C,OAAO,IAAI,CAAC,SAAS,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;QAC1C,CAAC;QAED,MAAM,WAAW,GAAG;YAClB,SAAS,EAAE,gBAAgB,CAAC,SAAS;YACrC,OAAO,EAAE,gBAAgB,CAAC,OAAO;YACjC,MAAM,EAAE,gBAAgB,CAAC,MAAM;YAC/B,YAAY,EAAE,gBAAgB,CAAC,YAAY;YAC3C,QAAQ,EAAE,gBAAgB,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;gBAC5C,UAAU,EAAE,CAAC,CAAC,UAAU;gBACxB,KAAK,EAAE,CAAC,CAAC,KAAK;gBACd,eAAe,EAAE,CAAC,CAAC,eAAe;gBAClC,cAAc,EAAE,gBAAgB,CAAC,aAAa,CAAC,IAAI,CACjD,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,UAAU,CAC5B,EAAE,SAAS,EAAE,cAAc,IAAI,SAAS;gBACzC,UAAU,EAAE,CAAC,CAAC,UAAU;gBACxB,OAAO,EAAE,CAAC,CAAC,OAAO;aACnB,CAAC,CAAC;SACJ,CAAC;QAEF,OAAO,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;IAC9C,CAAC;IAED;;;;;;;OAOG;IACK,KAAK,CAAC,gBAAgB,CAAC,gBAAyC;QACtE,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,MAAM,CAAC,CAAC;YAEhD,aAAa;YACb,IAAI,CAAC;gBACH,MAAM,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YAClC,CAAC;YAAC,MAAM,CAAC;gBACP,MAAM,UAAU,CAAC,KAAK,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YACtD,CAAC;YAED,gCAAgC;YAChC,MAAM,KAAK,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;YACrD,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,oBAAoB,KAAK,MAAM,CAAC,CAAC;YAEvE,sBAAsB;YACtB,MAAM,QAAQ,GAAG;gBACf,SAAS,EAAE,gBAAgB,CAAC,YAAY;gBACxC,SAAS,EAAE,gBAAgB,CAAC,SAAS;gBACrC,OAAO,EAAE,gBAAgB,CAAC,OAAO;gBACjC,MAAM,EAAE,gBAAgB,CAAC,MAAM;gBAC/B,YAAY,EAAE,gBAAgB,CAAC,WAAW;gBAC1C,YAAY,EAAE,gBAAgB,CAAC,WAAW;gBAC1C,WAAW,EAAE,gBAAgB,CAAC,UAAU;gBACxC,QAAQ,EAAE,gBAAgB,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;oBAC5C,UAAU,EAAE,CAAC,CAAC,UAAU;oBACxB,KAAK,EAAE,CAAC,CAAC,KAAK;oBACd,eAAe,EAAE,CAAC,CAAC,eAAe;oBAClC,cAAc,EAAE,gBAAgB,CAAC,aAAa,CAAC,IAAI,CACjD,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,UAAU,CAC5B,EAAE,SAAS,EAAE,cAAc,IAAI,SAAS;oBACzC,UAAU,EAAE,CAAC,CAAC,UAAU;oBACxB,OAAO,EAAE,CAAC,CAAC,OAAO;iBACnB,CAAC,CAAC;aACJ,CAAC;YAEF,yBAAyB;YACzB,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,GAAG,IAAI,CAAC;YAChD,MAAM,UAAU,CAAC,UAAU,CAAC,WAAW,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;QAC7D,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,sBAAsB;YACtB,MAAM,CAAC,KAAK,CAAC,mBAAmB,EAAE;gBAChC,SAAS,EAAE,gBAAgB,CAAC,SAAS;gBACrC,OAAO,EAAE,gBAAgB,CAAC,OAAO;gBACjC,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;aAC9D,CAAC,CAAC;QACL,CAAC;IACH,CAAC;CACF"}
|
|
@@ -0,0 +1,221 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Quality Metrics Collector
|
|
3
|
+
*
|
|
4
|
+
* 품질 지표 수집 서비스
|
|
5
|
+
*
|
|
6
|
+
* 주요 기능:
|
|
7
|
+
* - 기존 품질 검증 시스템들을 호출하여 지표 수집
|
|
8
|
+
* - namespace 단위 수집 메서드 제공 (search, relation, consolidation, storage)
|
|
9
|
+
* - 수집된 지표를 구조화된 형태로 반환
|
|
10
|
+
*
|
|
11
|
+
* PRD FR-1.1: Collector 역할 - 품질 지표 수집 (기존 검증 시스템 호출)
|
|
12
|
+
*/
|
|
13
|
+
import Database from 'better-sqlite3';
|
|
14
|
+
import { type SearchResult, type GroundTruth } from '../../test/helpers/search-quality-metrics.js';
|
|
15
|
+
import { type SearchResultPair } from '../../test/helpers/vector-search-quality-metrics.js';
|
|
16
|
+
import { type ExpectedRelation, type ExtractedRelation } from '../../domains/relation/services/relation-quality-validator.js';
|
|
17
|
+
/**
|
|
18
|
+
* 품질 지표 수집 결과
|
|
19
|
+
*/
|
|
20
|
+
export interface CollectedMetrics {
|
|
21
|
+
/**
|
|
22
|
+
* 네임스페이스 (예: 'search', 'relation', 'consolidation', 'storage')
|
|
23
|
+
*/
|
|
24
|
+
namespace: string;
|
|
25
|
+
/**
|
|
26
|
+
* 컨텍스트 (예: 'default', 'ci', 'nightly')
|
|
27
|
+
*/
|
|
28
|
+
context: string;
|
|
29
|
+
/**
|
|
30
|
+
* 측정 시간
|
|
31
|
+
*/
|
|
32
|
+
measured_at: string;
|
|
33
|
+
/**
|
|
34
|
+
* 지표 데이터 (키-값 쌍)
|
|
35
|
+
* 예: { 'precision_at_5': 0.85, 'recall_at_5': 0.72, ... }
|
|
36
|
+
*/
|
|
37
|
+
metrics: Record<string, number>;
|
|
38
|
+
/**
|
|
39
|
+
* 메타데이터 (선택적)
|
|
40
|
+
*/
|
|
41
|
+
metadata?: Record<string, any>;
|
|
42
|
+
}
|
|
43
|
+
/**
|
|
44
|
+
* 검색 품질 지표
|
|
45
|
+
*/
|
|
46
|
+
export interface SearchQualityMetrics {
|
|
47
|
+
precision_at_5?: number;
|
|
48
|
+
precision_at_10?: number;
|
|
49
|
+
recall_at_5?: number;
|
|
50
|
+
recall_at_10?: number;
|
|
51
|
+
ndcg_at_5?: number;
|
|
52
|
+
ndcg_at_10?: number;
|
|
53
|
+
mrr?: number;
|
|
54
|
+
kendalls_tau?: number;
|
|
55
|
+
top_5_retention?: number;
|
|
56
|
+
top_10_retention?: number;
|
|
57
|
+
vector_similarity_mean?: number;
|
|
58
|
+
vector_similarity_std?: number;
|
|
59
|
+
consolidation_score_mean?: number;
|
|
60
|
+
consolidation_score_std?: number;
|
|
61
|
+
}
|
|
62
|
+
/**
|
|
63
|
+
* 관계 추출 품질 지표
|
|
64
|
+
*/
|
|
65
|
+
export interface RelationQualityMetrics {
|
|
66
|
+
precision?: number;
|
|
67
|
+
recall?: number;
|
|
68
|
+
f1_score?: number;
|
|
69
|
+
true_positives?: number;
|
|
70
|
+
false_positives?: number;
|
|
71
|
+
false_negatives?: number;
|
|
72
|
+
confidence_compliance_rate?: number;
|
|
73
|
+
type_precision?: Record<string, number>;
|
|
74
|
+
type_recall?: Record<string, number>;
|
|
75
|
+
type_f1_score?: Record<string, number>;
|
|
76
|
+
}
|
|
77
|
+
/**
|
|
78
|
+
* Consolidation 점수 품질 지표
|
|
79
|
+
*/
|
|
80
|
+
export interface ConsolidationQualityMetrics {
|
|
81
|
+
score_stability?: number;
|
|
82
|
+
order_preservation?: number;
|
|
83
|
+
score_mean?: number;
|
|
84
|
+
score_std?: number;
|
|
85
|
+
kendalls_tau?: number;
|
|
86
|
+
}
|
|
87
|
+
/**
|
|
88
|
+
* 저장 품질 지표
|
|
89
|
+
*/
|
|
90
|
+
export interface StorageQualityMetrics {
|
|
91
|
+
duplication_rate?: number;
|
|
92
|
+
data_integrity?: number;
|
|
93
|
+
schema_compliance?: number;
|
|
94
|
+
data_loss_rate?: number;
|
|
95
|
+
}
|
|
96
|
+
/**
|
|
97
|
+
* Quality Metrics Collector
|
|
98
|
+
*
|
|
99
|
+
* PRD FR-1.1: Collector 역할 - 품질 지표 수집 (기존 검증 시스템 호출)
|
|
100
|
+
*/
|
|
101
|
+
export declare class QualityMetricsCollector {
|
|
102
|
+
private db;
|
|
103
|
+
constructor(db: Database.Database);
|
|
104
|
+
/**
|
|
105
|
+
* MRR (Mean Reciprocal Rank) 계산
|
|
106
|
+
* 첫 번째 관련 결과의 역순위의 평균
|
|
107
|
+
*
|
|
108
|
+
* @param queryResults 쿼리별 검색 결과
|
|
109
|
+
* @param groundTruths 쿼리별 Ground Truth
|
|
110
|
+
* @returns MRR (0-1)
|
|
111
|
+
*/
|
|
112
|
+
private calculateMRR;
|
|
113
|
+
/**
|
|
114
|
+
* 검색 품질 지표 수집
|
|
115
|
+
*
|
|
116
|
+
* PRD FR-2.1: 검색 품질 지표 정의
|
|
117
|
+
* - Precision@K, Recall@K, NDCG@K, MRR, Kendall's Tau 등
|
|
118
|
+
*
|
|
119
|
+
* @param context - 컨텍스트 (기본값: 'default')
|
|
120
|
+
* @param options - 옵션 (Ground Truth 데이터, 검색 결과 등)
|
|
121
|
+
* @returns 검색 품질 지표
|
|
122
|
+
*
|
|
123
|
+
* Note: Ground Truth 데이터가 제공되면 실제 측정을 수행하고,
|
|
124
|
+
* 없으면 기본값(0)을 반환합니다.
|
|
125
|
+
*/
|
|
126
|
+
collectSearchMetrics(context?: string, options?: {
|
|
127
|
+
/**
|
|
128
|
+
* Ground Truth 데이터 (선택적)
|
|
129
|
+
* 제공되면 실제 측정을 수행합니다.
|
|
130
|
+
* 제공되지 않으면 파일에서 자동 로드 시도
|
|
131
|
+
*/
|
|
132
|
+
groundTruths?: GroundTruth[];
|
|
133
|
+
/**
|
|
134
|
+
* 검색 결과 (선택적)
|
|
135
|
+
* 쿼리 ID를 키로 하는 Map
|
|
136
|
+
* 제공되지 않으면 Ground Truth를 기반으로 실제 검색 수행
|
|
137
|
+
*/
|
|
138
|
+
queryResults?: Map<string, SearchResult[]>;
|
|
139
|
+
/**
|
|
140
|
+
* 순서 보존 검증용 검색 결과 쌍 (선택적)
|
|
141
|
+
* 벡터-only와 Consolidation 반영 후 결과 비교
|
|
142
|
+
*/
|
|
143
|
+
searchResultPairs?: SearchResultPair[];
|
|
144
|
+
}): Promise<CollectedMetrics>;
|
|
145
|
+
/**
|
|
146
|
+
* 관계 추출 품질 지표 수집
|
|
147
|
+
*
|
|
148
|
+
* PRD FR-2.4: 관계 추출 품질 지표 정의
|
|
149
|
+
* - Precision, Recall, F1-Score, 관계 유형별 정확도
|
|
150
|
+
*
|
|
151
|
+
* @param context - 컨텍스트 (기본값: 'default')
|
|
152
|
+
* @param options - 옵션 (예상 관계, 추출된 관계 등)
|
|
153
|
+
* @returns 관계 추출 품질 지표
|
|
154
|
+
*
|
|
155
|
+
* Note: 예상 관계와 추출된 관계가 제공되면 실제 측정을 수행하고,
|
|
156
|
+
* 없으면 기본값(0)을 반환합니다.
|
|
157
|
+
*/
|
|
158
|
+
collectRelationMetrics(context?: string, options?: {
|
|
159
|
+
/**
|
|
160
|
+
* 예상 관계 목록 (Ground Truth, 선택적)
|
|
161
|
+
* 제공되면 실제 측정을 수행합니다.
|
|
162
|
+
*/
|
|
163
|
+
expectedRelations?: ExpectedRelation[];
|
|
164
|
+
/**
|
|
165
|
+
* 추출된 관계 목록 (선택적)
|
|
166
|
+
* 제공되면 실제 측정을 수행합니다.
|
|
167
|
+
*/
|
|
168
|
+
extractedRelations?: ExtractedRelation[];
|
|
169
|
+
}): Promise<CollectedMetrics>;
|
|
170
|
+
/**
|
|
171
|
+
* Consolidation 점수 품질 지표 수집
|
|
172
|
+
*
|
|
173
|
+
* PRD FR-2.5: Consolidation 점수 안정성 지표 정의
|
|
174
|
+
* - 점수 분포, 순서 보존 검증
|
|
175
|
+
*
|
|
176
|
+
* @param context - 컨텍스트 (기본값: 'default')
|
|
177
|
+
* @param options - 옵션 (검색 결과 쌍, 점수 샘플 등)
|
|
178
|
+
* @returns Consolidation 점수 품질 지표
|
|
179
|
+
*
|
|
180
|
+
* Note: 검색 결과 쌍이나 점수 샘플이 제공되면 실제 측정을 수행하고,
|
|
181
|
+
* 없으면 기본값(0)을 반환합니다.
|
|
182
|
+
*/
|
|
183
|
+
collectConsolidationMetrics(context?: string, options?: {
|
|
184
|
+
/**
|
|
185
|
+
* 순서 보존 검증용 검색 결과 쌍 (선택적)
|
|
186
|
+
* 벡터-only와 Consolidation 반영 후 결과 비교
|
|
187
|
+
*/
|
|
188
|
+
searchResultPairs?: SearchResultPair[];
|
|
189
|
+
/**
|
|
190
|
+
* Consolidation 점수 샘플 (선택적)
|
|
191
|
+
* 점수 분포 분석을 위한 샘플 데이터
|
|
192
|
+
*/
|
|
193
|
+
consolidationScores?: number[];
|
|
194
|
+
}): Promise<CollectedMetrics>;
|
|
195
|
+
/**
|
|
196
|
+
* 저장 품질 지표 수집
|
|
197
|
+
*
|
|
198
|
+
* PRD FR-2.5: 기억 저장 품질 지표 정의
|
|
199
|
+
* - 중복 비율, 데이터 무결성, 스키마 준수율, 데이터 손실률
|
|
200
|
+
*
|
|
201
|
+
* @param context - 컨텍스트 (기본값: 'default')
|
|
202
|
+
* @returns 저장 품질 지표
|
|
203
|
+
*/
|
|
204
|
+
collectStorageMetrics(context?: string): Promise<CollectedMetrics>;
|
|
205
|
+
/**
|
|
206
|
+
* 모든 네임스페이스의 품질 지표 수집
|
|
207
|
+
*
|
|
208
|
+
* @param context - 컨텍스트 (기본값: 'default')
|
|
209
|
+
* @returns 모든 네임스페이스의 품질 지표
|
|
210
|
+
*/
|
|
211
|
+
collectAllMetrics(context?: string): Promise<CollectedMetrics[]>;
|
|
212
|
+
/**
|
|
213
|
+
* 특정 네임스페이스의 품질 지표 수집
|
|
214
|
+
*
|
|
215
|
+
* @param namespace - 네임스페이스 ('search', 'relation', 'consolidation', 'storage')
|
|
216
|
+
* @param context - 컨텍스트 (기본값: 'default')
|
|
217
|
+
* @returns 품질 지표
|
|
218
|
+
*/
|
|
219
|
+
collectMetricsByNamespace(namespace: string, context?: string): Promise<CollectedMetrics>;
|
|
220
|
+
}
|
|
221
|
+
//# sourceMappingURL=quality-metrics-collector.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"quality-metrics-collector.d.ts","sourceRoot":"","sources":["../../../src/services/quality-assurance/quality-metrics-collector.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,QAAQ,MAAM,gBAAgB,CAAC;AAEtC,OAAO,EAIL,KAAK,YAAY,EACjB,KAAK,WAAW,EACjB,MAAM,8CAA8C,CAAC;AACtD,OAAO,EAIL,KAAK,gBAAgB,EAEtB,MAAM,qDAAqD,CAAC;AAC7D,OAAO,EAEL,KAAK,gBAAgB,EACrB,KAAK,iBAAiB,EACvB,MAAM,+DAA+D,CAAC;AAIvE;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B;;OAEG;IACH,SAAS,EAAE,MAAM,CAAC;IAElB;;OAEG;IACH,OAAO,EAAE,MAAM,CAAC;IAEhB;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC;IAEpB;;;OAGG;IACH,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAEhC;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;CAChC;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IAEnC,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,eAAe,CAAC,EAAE,MAAM,CAAC;IAGzB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IAGtB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IAGpB,GAAG,CAAC,EAAE,MAAM,CAAC;IAGb,YAAY,CAAC,EAAE,MAAM,CAAC;IAGtB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAG1B,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAChC,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAG/B,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAClC,uBAAuB,CAAC,EAAE,MAAM,CAAC;CAClC;AAED;;GAEG;AACH,MAAM,WAAW,sBAAsB;IAErC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAGlB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,eAAe,CAAC,EAAE,MAAM,CAAC;IAGzB,0BAA0B,CAAC,EAAE,MAAM,CAAC;IAGpC,cAAc,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACxC,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACrC,aAAa,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CACxC;AAED;;GAEG;AACH,MAAM,WAAW,2BAA2B;IAE1C,eAAe,CAAC,EAAE,MAAM,CAAC;IAGzB,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAG5B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IAGnB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED;;GAEG;AACH,MAAM,WAAW,qBAAqB;IAEpC,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAG1B,cAAc,CAAC,EAAE,MAAM,CAAC;IAGxB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAG3B,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED;;;;GAIG;AACH,qBAAa,uBAAuB;IACtB,OAAO,CAAC,EAAE;gBAAF,EAAE,EAAE,QAAQ,CAAC,QAAQ;IAMzC;;;;;;;OAOG;IACH,OAAO,CAAC,YAAY;IAiCpB;;;;;;;;;;;;OAYG;IACG,oBAAoB,CACxB,OAAO,GAAE,MAAkB,EAC3B,OAAO,CAAC,EAAE;QACR;;;;WAIG;QACH,YAAY,CAAC,EAAE,WAAW,EAAE,CAAC;QAE7B;;;;WAIG;QACH,YAAY,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,YAAY,EAAE,CAAC,CAAC;QAE3C;;;WAGG;QACH,iBAAiB,CAAC,EAAE,gBAAgB,EAAE,CAAC;KACxC,GACA,OAAO,CAAC,gBAAgB,CAAC;IAqM5B;;;;;;;;;;;;OAYG;IACG,sBAAsB,CAC1B,OAAO,GAAE,MAAkB,EAC3B,OAAO,CAAC,EAAE;QACR;;;WAGG;QACH,iBAAiB,CAAC,EAAE,gBAAgB,EAAE,CAAC;QAEvC;;;WAGG;QACH,kBAAkB,CAAC,EAAE,iBAAiB,EAAE,CAAC;KAC1C,GACA,OAAO,CAAC,gBAAgB,CAAC;IAwF5B;;;;;;;;;;;;OAYG;IACG,2BAA2B,CAC/B,OAAO,GAAE,MAAkB,EAC3B,OAAO,CAAC,EAAE;QACR;;;WAGG;QACH,iBAAiB,CAAC,EAAE,gBAAgB,EAAE,CAAC;QAEvC;;;WAGG;QACH,mBAAmB,CAAC,EAAE,MAAM,EAAE,CAAC;KAChC,GACA,OAAO,CAAC,gBAAgB,CAAC;IA2I5B;;;;;;;;OAQG;IACG,qBAAqB,CAAC,OAAO,GAAE,MAAkB,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAyQnF;;;;;OAKG;IACG,iBAAiB,CAAC,OAAO,GAAE,MAAkB,GAAG,OAAO,CAAC,gBAAgB,EAAE,CAAC;IAWjF;;;;;;OAMG;IACG,yBAAyB,CAC7B,SAAS,EAAE,MAAM,EACjB,OAAO,GAAE,MAAkB,GAC1B,OAAO,CAAC,gBAAgB,CAAC;CAc7B"}
|