gravito-eval 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +137 -0
- package/dist/cli/index.d.ts +14 -0
- package/dist/cli/index.d.ts.map +1 -0
- package/dist/cli/index.js +276 -0
- package/dist/cli/index.js.map +1 -0
- package/dist/src/adjudication/index.d.ts +36 -0
- package/dist/src/adjudication/index.d.ts.map +1 -0
- package/dist/src/adjudication/index.js +149 -0
- package/dist/src/adjudication/index.js.map +1 -0
- package/dist/src/calibration/index.d.ts +38 -0
- package/dist/src/calibration/index.d.ts.map +1 -0
- package/dist/src/calibration/index.js +104 -0
- package/dist/src/calibration/index.js.map +1 -0
- package/dist/src/confidence/index.d.ts +27 -0
- package/dist/src/confidence/index.d.ts.map +1 -0
- package/dist/src/confidence/index.js +168 -0
- package/dist/src/confidence/index.js.map +1 -0
- package/dist/src/index.d.ts +26 -0
- package/dist/src/index.d.ts.map +1 -0
- package/dist/src/index.js +47 -0
- package/dist/src/index.js.map +1 -0
- package/dist/src/matching/index.d.ts +37 -0
- package/dist/src/matching/index.d.ts.map +1 -0
- package/dist/src/matching/index.js +292 -0
- package/dist/src/matching/index.js.map +1 -0
- package/dist/src/metrics/index.d.ts +15 -0
- package/dist/src/metrics/index.d.ts.map +1 -0
- package/dist/src/metrics/index.js +177 -0
- package/dist/src/metrics/index.js.map +1 -0
- package/dist/src/telemetry/index.d.ts +10 -0
- package/dist/src/telemetry/index.d.ts.map +1 -0
- package/dist/src/telemetry/index.js +106 -0
- package/dist/src/telemetry/index.js.map +1 -0
- package/dist/src/types.d.ts +131 -0
- package/dist/src/types.d.ts.map +1 -0
- package/dist/src/types.js +28 -0
- package/dist/src/types.js.map +1 -0
- package/examples/basic/input.json +76 -0
- package/examples/basic/run.ts +33 -0
- package/package.json +50 -0
|
@@ -0,0 +1,292 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Gravito Eval — Multi-Pass Semantic Matcher
|
|
4
|
+
*
|
|
5
|
+
* Matches AI findings against human findings using three passes:
|
|
6
|
+
* 1. Strict: same category + high keyword similarity (>0.75)
|
|
7
|
+
* 2. Cross-category: high similarity (>0.80) + category equivalence
|
|
8
|
+
* 3. Conceptual merge: cluster related findings, match cluster ↔ single issue
|
|
9
|
+
*
|
|
10
|
+
* Each pass uses greedy one-to-one matching — no double-counting.
|
|
11
|
+
*/
|
|
12
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
13
|
+
exports.keywordSimilarity = keywordSimilarity;
|
|
14
|
+
exports.multiPassMatch = multiPassMatch;
|
|
15
|
+
exports.toFlatMatchResult = toFlatMatchResult;
|
|
16
|
+
// ─── Category Equivalence Map ─────────────────────────────────────────────
|
|
17
|
+
const CATEGORY_EQUIVALENCE = {
|
|
18
|
+
trust: ["content", "conversion"],
|
|
19
|
+
content: ["trust", "navigation"],
|
|
20
|
+
navigation: ["conversion"],
|
|
21
|
+
conversion: ["navigation", "trust"],
|
|
22
|
+
visual_hierarchy: ["conversion", "content"],
|
|
23
|
+
compliance: ["trust"],
|
|
24
|
+
performance: ["conversion"],
|
|
25
|
+
};
|
|
26
|
+
function areCategoriesEquivalent(a, b) {
|
|
27
|
+
if (a === b)
|
|
28
|
+
return true;
|
|
29
|
+
return CATEGORY_EQUIVALENCE[a]?.includes(b) || CATEGORY_EQUIVALENCE[b]?.includes(a) || false;
|
|
30
|
+
}
|
|
31
|
+
// ─── Keyword Similarity ───────────────────────────────────────────────────
|
|
32
|
+
function tokenize(text) {
|
|
33
|
+
return new Set(text
|
|
34
|
+
.toLowerCase()
|
|
35
|
+
.replace(/[^a-z0-9\s]/g, " ")
|
|
36
|
+
.split(/\s+/)
|
|
37
|
+
.filter((w) => w.length > 2));
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* Compute keyword similarity between two findings.
|
|
41
|
+
* Uses a hybrid approach:
|
|
42
|
+
* 1. Jaccard on full text (description + keywords)
|
|
43
|
+
* 2. Keyword-specific overlap (weighted higher)
|
|
44
|
+
* 3. Location similarity bonus
|
|
45
|
+
* Final score = max(jaccard, keywordOverlap) with location bonus
|
|
46
|
+
*/
|
|
47
|
+
function keywordSimilarity(a, b) {
|
|
48
|
+
// Full-text Jaccard
|
|
49
|
+
const textA = [a.description, ...(a.keywords || [])].join(" ");
|
|
50
|
+
const textB = [b.description, ...(b.keywords || [])].join(" ");
|
|
51
|
+
const tokensA = tokenize(textA);
|
|
52
|
+
const tokensB = tokenize(textB);
|
|
53
|
+
if (tokensA.size === 0 && tokensB.size === 0)
|
|
54
|
+
return 1;
|
|
55
|
+
if (tokensA.size === 0 || tokensB.size === 0)
|
|
56
|
+
return 0;
|
|
57
|
+
let textIntersection = 0;
|
|
58
|
+
for (const t of tokensA) {
|
|
59
|
+
if (tokensB.has(t))
|
|
60
|
+
textIntersection++;
|
|
61
|
+
}
|
|
62
|
+
const textUnion = tokensA.size + tokensB.size - textIntersection;
|
|
63
|
+
const jaccardScore = textUnion === 0 ? 0 : textIntersection / textUnion;
|
|
64
|
+
// Keyword-specific overlap (Dice coefficient — more generous for small sets)
|
|
65
|
+
let keywordScore = 0;
|
|
66
|
+
const kwA = new Set((a.keywords || []).map((k) => k.toLowerCase()));
|
|
67
|
+
const kwB = new Set((b.keywords || []).map((k) => k.toLowerCase()));
|
|
68
|
+
if (kwA.size > 0 && kwB.size > 0) {
|
|
69
|
+
let kwOverlap = 0;
|
|
70
|
+
for (const k of kwA) {
|
|
71
|
+
for (const kb of kwB) {
|
|
72
|
+
// Exact match or substring containment
|
|
73
|
+
if (k === kb || k.includes(kb) || kb.includes(k)) {
|
|
74
|
+
kwOverlap++;
|
|
75
|
+
break;
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
keywordScore = (2 * kwOverlap) / (kwA.size + kwB.size);
|
|
80
|
+
}
|
|
81
|
+
// Location bonus
|
|
82
|
+
let locationBonus = 0;
|
|
83
|
+
if (a.location && b.location) {
|
|
84
|
+
const locA = tokenize(a.location);
|
|
85
|
+
const locB = tokenize(b.location);
|
|
86
|
+
let locOverlap = 0;
|
|
87
|
+
for (const t of locA) {
|
|
88
|
+
if (locB.has(t))
|
|
89
|
+
locOverlap++;
|
|
90
|
+
}
|
|
91
|
+
const locUnion = locA.size + locB.size - locOverlap;
|
|
92
|
+
if (locUnion > 0 && locOverlap / locUnion > 0.3) {
|
|
93
|
+
locationBonus = 0.1;
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
// Hybrid: take the best of Jaccard and keyword overlap, add location bonus
|
|
97
|
+
const baseScore = Math.max(jaccardScore, keywordScore);
|
|
98
|
+
return Math.min(1, baseScore + locationBonus);
|
|
99
|
+
}
|
|
100
|
+
function greedyMatch(candidates, usedAi, usedHuman) {
|
|
101
|
+
// Sort by similarity descending
|
|
102
|
+
const sorted = [...candidates].sort((a, b) => b.similarity - a.similarity);
|
|
103
|
+
const matched = [];
|
|
104
|
+
for (const pair of sorted) {
|
|
105
|
+
if (usedAi.has(pair.aiIdx) || usedHuman.has(pair.humanIdx))
|
|
106
|
+
continue;
|
|
107
|
+
matched.push(pair);
|
|
108
|
+
usedAi.add(pair.aiIdx);
|
|
109
|
+
usedHuman.add(pair.humanIdx);
|
|
110
|
+
}
|
|
111
|
+
return matched;
|
|
112
|
+
}
|
|
113
|
+
// ─── Pass 1: Strict Match ─────────────────────────────────────────────────
|
|
114
|
+
function strictPass(aiFindings, humanFindings, usedAi, usedHuman, threshold = 0.55) {
|
|
115
|
+
const candidates = [];
|
|
116
|
+
for (let i = 0; i < aiFindings.length; i++) {
|
|
117
|
+
if (usedAi.has(i))
|
|
118
|
+
continue;
|
|
119
|
+
for (let j = 0; j < humanFindings.length; j++) {
|
|
120
|
+
if (usedHuman.has(j))
|
|
121
|
+
continue;
|
|
122
|
+
// Same category required
|
|
123
|
+
if (aiFindings[i].category !== humanFindings[j].category)
|
|
124
|
+
continue;
|
|
125
|
+
const sim = keywordSimilarity(aiFindings[i], humanFindings[j]);
|
|
126
|
+
if (sim >= threshold) {
|
|
127
|
+
candidates.push({ aiIdx: i, humanIdx: j, similarity: sim });
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
const matched = greedyMatch(candidates, usedAi, usedHuman);
|
|
132
|
+
return matched.map((m) => ({
|
|
133
|
+
aiIssue: aiFindings[m.aiIdx],
|
|
134
|
+
humanIssue: humanFindings[m.humanIdx],
|
|
135
|
+
similarity: m.similarity,
|
|
136
|
+
matchType: "strict",
|
|
137
|
+
}));
|
|
138
|
+
}
|
|
139
|
+
// ─── Pass 2: Cross-Category Match ─────────────────────────────────────────
|
|
140
|
+
function crossCategoryPass(aiFindings, humanFindings, usedAi, usedHuman, threshold = 0.50) {
|
|
141
|
+
const candidates = [];
|
|
142
|
+
for (let i = 0; i < aiFindings.length; i++) {
|
|
143
|
+
if (usedAi.has(i))
|
|
144
|
+
continue;
|
|
145
|
+
for (let j = 0; j < humanFindings.length; j++) {
|
|
146
|
+
if (usedHuman.has(j))
|
|
147
|
+
continue;
|
|
148
|
+
// Must be equivalent categories (not same — that was pass 1)
|
|
149
|
+
if (aiFindings[i].category === humanFindings[j].category)
|
|
150
|
+
continue;
|
|
151
|
+
if (!areCategoriesEquivalent(aiFindings[i].category, humanFindings[j].category))
|
|
152
|
+
continue;
|
|
153
|
+
const sim = keywordSimilarity(aiFindings[i], humanFindings[j]);
|
|
154
|
+
if (sim >= threshold) {
|
|
155
|
+
candidates.push({ aiIdx: i, humanIdx: j, similarity: sim });
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
const matched = greedyMatch(candidates, usedAi, usedHuman);
|
|
160
|
+
return matched.map((m) => ({
|
|
161
|
+
aiIssue: aiFindings[m.aiIdx],
|
|
162
|
+
humanIssue: humanFindings[m.humanIdx],
|
|
163
|
+
similarity: m.similarity,
|
|
164
|
+
matchType: "cross_category",
|
|
165
|
+
}));
|
|
166
|
+
}
|
|
167
|
+
function clusterFindings(findings, usedIndices, similarityThreshold = 0.40) {
|
|
168
|
+
const clusters = [];
|
|
169
|
+
const assigned = new Set();
|
|
170
|
+
for (let i = 0; i < findings.length; i++) {
|
|
171
|
+
if (usedIndices.has(i) || assigned.has(i))
|
|
172
|
+
continue;
|
|
173
|
+
const cluster = {
|
|
174
|
+
findings: [{ finding: findings[i], originalIdx: i }],
|
|
175
|
+
centroidTokens: tokenize(findings[i].description),
|
|
176
|
+
};
|
|
177
|
+
assigned.add(i);
|
|
178
|
+
for (let j = i + 1; j < findings.length; j++) {
|
|
179
|
+
if (usedIndices.has(j) || assigned.has(j))
|
|
180
|
+
continue;
|
|
181
|
+
const sim = keywordSimilarity(findings[i], findings[j]);
|
|
182
|
+
if (sim >= similarityThreshold) {
|
|
183
|
+
cluster.findings.push({ finding: findings[j], originalIdx: j });
|
|
184
|
+
// Expand centroid
|
|
185
|
+
for (const t of tokenize(findings[j].description)) {
|
|
186
|
+
cluster.centroidTokens.add(t);
|
|
187
|
+
}
|
|
188
|
+
assigned.add(j);
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
if (cluster.findings.length >= 2) {
|
|
192
|
+
clusters.push(cluster);
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
return clusters;
|
|
196
|
+
}
|
|
197
|
+
function conceptualMergePass(aiFindings, humanFindings, usedAi, usedHuman, clusterThreshold = 0.40, matchThreshold = 0.35) {
|
|
198
|
+
const matches = [];
|
|
199
|
+
// Cluster unmatched AI findings
|
|
200
|
+
const aiClusters = clusterFindings(aiFindings, usedAi, clusterThreshold);
|
|
201
|
+
// Try to match each cluster against unmatched human findings
|
|
202
|
+
for (const cluster of aiClusters) {
|
|
203
|
+
let bestHumanIdx = -1;
|
|
204
|
+
let bestSim = 0;
|
|
205
|
+
for (let j = 0; j < humanFindings.length; j++) {
|
|
206
|
+
if (usedHuman.has(j))
|
|
207
|
+
continue;
|
|
208
|
+
const humanTokens = tokenize(humanFindings[j].description);
|
|
209
|
+
// Compute overlap between cluster centroid and human finding
|
|
210
|
+
let intersection = 0;
|
|
211
|
+
for (const t of humanTokens) {
|
|
212
|
+
if (cluster.centroidTokens.has(t))
|
|
213
|
+
intersection++;
|
|
214
|
+
}
|
|
215
|
+
const union = cluster.centroidTokens.size + humanTokens.size - intersection;
|
|
216
|
+
const sim = union > 0 ? intersection / union : 0;
|
|
217
|
+
if (sim > bestSim && sim >= matchThreshold) {
|
|
218
|
+
bestSim = sim;
|
|
219
|
+
bestHumanIdx = j;
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
if (bestHumanIdx >= 0) {
|
|
223
|
+
// Match the highest-severity finding from the cluster
|
|
224
|
+
const bestAi = cluster.findings.reduce((best, curr) => {
|
|
225
|
+
const sevOrder = { low: 1, medium: 2, high: 3, critical: 4 };
|
|
226
|
+
return sevOrder[curr.finding.severity] > sevOrder[best.finding.severity] ? curr : best;
|
|
227
|
+
});
|
|
228
|
+
matches.push({
|
|
229
|
+
aiIssue: bestAi.finding,
|
|
230
|
+
humanIssue: humanFindings[bestHumanIdx],
|
|
231
|
+
similarity: bestSim,
|
|
232
|
+
matchType: "conceptual",
|
|
233
|
+
});
|
|
234
|
+
// Mark all cluster members as used
|
|
235
|
+
for (const member of cluster.findings) {
|
|
236
|
+
usedAi.add(member.originalIdx);
|
|
237
|
+
}
|
|
238
|
+
usedHuman.add(bestHumanIdx);
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
return matches;
|
|
242
|
+
}
|
|
243
|
+
/**
|
|
244
|
+
* Run multi-pass semantic matching between AI and human findings.
|
|
245
|
+
*
|
|
246
|
+
* Returns detailed results with match breakdown by pass type.
|
|
247
|
+
*/
|
|
248
|
+
function multiPassMatch(aiFindings, humanFindings, options) {
|
|
249
|
+
const usedAi = new Set();
|
|
250
|
+
const usedHuman = new Set();
|
|
251
|
+
// Pass 1: Strict
|
|
252
|
+
const strict = strictPass(aiFindings, humanFindings, usedAi, usedHuman, options?.strictThreshold ?? 0.55);
|
|
253
|
+
// Pass 2: Cross-category
|
|
254
|
+
const crossCategory = crossCategoryPass(aiFindings, humanFindings, usedAi, usedHuman, options?.crossCategoryThreshold ?? 0.50);
|
|
255
|
+
// Pass 3: Conceptual merge
|
|
256
|
+
const conceptual = conceptualMergePass(aiFindings, humanFindings, usedAi, usedHuman, options?.clusterThreshold ?? 0.40, options?.conceptualMatchThreshold ?? 0.35);
|
|
257
|
+
// Collect unmatched
|
|
258
|
+
const aiOnly = aiFindings.filter((_, i) => !usedAi.has(i));
|
|
259
|
+
const humanOnly = humanFindings.filter((_, i) => !usedHuman.has(i));
|
|
260
|
+
return {
|
|
261
|
+
strictMatches: strict,
|
|
262
|
+
crossCategoryMatches: crossCategory,
|
|
263
|
+
conceptualMatches: conceptual,
|
|
264
|
+
aiOnly,
|
|
265
|
+
humanOnly,
|
|
266
|
+
summary: {
|
|
267
|
+
total_ai: aiFindings.length,
|
|
268
|
+
total_human: humanFindings.length,
|
|
269
|
+
strict_matched: strict.length,
|
|
270
|
+
cross_category_matched: crossCategory.length,
|
|
271
|
+
conceptual_matched: conceptual.length,
|
|
272
|
+
total_matched: strict.length + crossCategory.length + conceptual.length,
|
|
273
|
+
ai_only: aiOnly.length,
|
|
274
|
+
human_only: humanOnly.length,
|
|
275
|
+
},
|
|
276
|
+
};
|
|
277
|
+
}
|
|
278
|
+
/**
|
|
279
|
+
* Convert MultiPassMatchResult to a flat MatchResult for simpler consumers.
|
|
280
|
+
*/
|
|
281
|
+
function toFlatMatchResult(result) {
|
|
282
|
+
return {
|
|
283
|
+
matched: [
|
|
284
|
+
...result.strictMatches,
|
|
285
|
+
...result.crossCategoryMatches,
|
|
286
|
+
...result.conceptualMatches,
|
|
287
|
+
],
|
|
288
|
+
aiOnly: result.aiOnly,
|
|
289
|
+
humanOnly: result.humanOnly,
|
|
290
|
+
};
|
|
291
|
+
}
|
|
292
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/matching/index.ts"],"names":[],"mappings":";AAAA;;;;;;;;;GASG;;AA+CH,8CAqDC;AAwND,wCAyDC;AAKD,8CAUC;AA1XD,6EAA6E;AAE7E,MAAM,oBAAoB,GAA2C;IACnE,KAAK,EAAE,CAAC,SAAS,EAAE,YAAY,CAAC;IAChC,OAAO,EAAE,CAAC,OAAO,EAAE,YAAY,CAAC;IAChC,UAAU,EAAE,CAAC,YAAY,CAAC;IAC1B,UAAU,EAAE,CAAC,YAAY,EAAE,OAAO,CAAC;IACnC,gBAAgB,EAAE,CAAC,YAAY,EAAE,SAAS,CAAC;IAC3C,UAAU,EAAE,CAAC,OAAO,CAAC;IACrB,WAAW,EAAE,CAAC,YAAY,CAAC;CAC5B,CAAC;AAEF,SAAS,uBAAuB,CAAC,CAAgB,EAAE,CAAgB;IACjE,IAAI,CAAC,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACzB,OAAO,oBAAoB,CAAC,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC,CAAC,IAAI,oBAAoB,CAAC,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC;AAC/F,CAAC;AAED,6EAA6E;AAE7E,SAAS,QAAQ,CAAC,IAAY;IAC5B,OAAO,IAAI,GAAG,CACZ,IAAI;SACD,WAAW,EAAE;SACb,OAAO,CAAC,cAAc,EAAE,GAAG,CAAC;SAC5B,KAAK,CAAC,KAAK,CAAC;SACZ,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAC/B,CAAC;AACJ,CAAC;AAED;;;;;;;GAOG;AACH,SAAgB,iBAAiB,CAAC,CAAU,EAAE,CAAU;IACtD,oBAAoB;IACpB,MAAM,KAAK,GAAG,CAAC,CAAC,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC/D,MAAM,KAAK,GAAG,CAAC,CAAC,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC/D,MAAM,OAAO,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC;IAChC,MAAM,OAAO,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC;IAEhC,IAAI,OAAO,CAAC,IAAI,KAAK,CAAC,IAAI,OAAO,CAAC,IAAI,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IACvD,IAAI,OAAO,CAAC,IAAI,KAAK,CAAC,IAAI,OAAO,CAAC,IAAI,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAEvD,IAAI,gBAAgB,GAAG,CAAC,CAAC;IACzB,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,IAAI,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC;YAAE,gBAAgB,EAAE,CAAC;IACzC,CAAC;IACD,MAAM,SAAS,GAAG,OAAO,CAAC,IAAI,GAAG,OAAO,CAAC,IAAI,GAAG,gBAAgB,CAAC;IACjE,MAAM,YAAY,GAAG,SAAS,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,gBAAgB,GAAG,SAAS,CAAC;IAExE,6EAA6E;IAC7E,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;IACpE,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;IACpE,IAAI,GAAG,CAAC,IAAI,GAAG,CAAC,IAAI,GAAG,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;QACjC,IAAI,SAAS,GAAG,CAAC,CAAC;QAClB,KAAK,MAAM,CAAC,IAAI,GAAG,EAAE,CAAC;YACpB,KAAK,MAAM,EAAE,IAAI,GAAG,EAAE,CAAC;gBACrB,uCAAuC;gBACvC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC;oBACjD,SAAS,EAAE,CAAC;oBACZ,MAAM;gBACR,CAAC;YACH,CAAC;QACH,CAAC;QACD,YAAY,GAAG,CAAC,CAAC,GAAG,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC;IACzD,CAAC;IAED,iBAAiB;IACjB,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,IAAI,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC;QAC7B,MAAM,IAAI,GAAG,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;QAClC,MAAM,IAAI,GAAG,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;QAClC,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;YACrB,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;gBAAE,UAAU,EAAE,CAAC;QAChC,CAAC;QACD,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,GAAG,UAAU,CAAC;QACpD,IAAI,QAAQ,GAAG,CAAC,IAAI,UAAU,GAAG,QAAQ,GAAG,GAAG,EAAE,CAAC;YAChD,aAAa,GAAG,GAAG,CAAC;QACtB,CAAC;IACH,CAAC;IAED,2EAA2E;IAC3E,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,YAAY,EAAE,YAAY,CAAC,CAAC;IACvD,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,SAAS,GAAG,aAAa,CAAC,CAAC;AAChD,CAAC;AAUD,SAAS,WAAW,CAClB,UAA2B,EAC3B,MAAmB,EACnB,SAAsB;IAEtB,gCAAgC;IAChC,MAAM,MAAM,GAAG,CAAC,GAAG,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC;IAC3E,MAAM,OAAO,GAAoB,EAAE,CAAC;IAEpC,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;QAC1B,IAAI,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC;YAAE,SAAS;QACrE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACnB,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACvB,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAC/B,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,6EAA6E;AAE7E,SAAS,UAAU,CACjB,UAAqB,EACrB,aAAwB,EACxB,MAAmB,EACnB,SAAsB,EACtB,YAAoB,IAAI;IAExB,MAAM,UAAU,GAAoB,EAAE,CAAC;IAEvC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC3C,IAAI,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;YAAE,SAAS;QAC5B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,aAAa,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC9C,IAAI,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC;gBAAE,SAAS;YAC/B,yBAAyB;YACzB,IAAI,UAAU,CAAC,CAAC,CAAC,CAAC,QAAQ,KAAK,aAAa,CAAC,CAAC,CAAC,CAAC,QAAQ;gBAAE,SAAS;YACnE,MAAM,GAAG,GAAG,iBAAiB,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC;YAC/D,IAAI,GAAG,IAAI,SAAS,EAAE,CAAC;gBACrB,UAAU,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,UAAU,EAAE,GAAG,EAAE,CAAC,CAAC;YAC9D,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,OAAO,GAAG,WAAW,CAAC,UAAU,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC;IAC3D,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACzB,OAAO,EAAE,UAAU,CAAC,CAAC,CAAC,KAAK,CAAC;QAC5B,UAAU,EAAE,aAAa,CAAC,CAAC,CAAC,QAAQ,CAAC;QACrC,UAAU,EAAE,CAAC,CAAC,UAAU;QACxB,SAAS,EAAE,QAAiB;KAC7B,CAAC,CAAC,CAAC;AACN,CAAC;AAED,6EAA6E;AAE7E,SAAS,iBAAiB,CACxB,UAAqB,EACrB,aAAwB,EACxB,MAAmB,EACnB,SAAsB,EACtB,YAAoB,IAAI;IAExB,MAAM,UAAU,GAAoB,EAAE,CAAC;IAEvC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC3C,IAAI,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;YAAE,SAAS;QAC5B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,aAAa,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC9C,IAAI,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC;gBAAE,SAAS;YAC/B,6DAA6D;YAC7D,IAAI,UAAU,CAAC,CAAC,CAAC,CAAC,QAAQ,KAAK,aAAa,CAAC,CAAC,CAAC,CAAC,QAAQ;gBAAE,SAAS;YACnE,IAAI,CAAC,uBAAuB,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;gBAAE,SAAS;YAC1F,MAAM,GAAG,GAAG,iBAAiB,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC;YAC/D,IAAI,GAAG,IAAI,SAAS,EAAE,CAAC;gBACrB,UAAU,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,UAAU,EAAE,GAAG,EAAE,CAAC,CAAC;YAC9D,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,OAAO,GAAG,WAAW,CAAC,UAAU,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC;IAC3D,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACzB,OAAO,EAAE,UAAU,CAAC,CAAC,CAAC,KAAK,CAAC;QAC5B,UAAU,EAAE,aAAa,CAAC,CAAC,CAAC,QAAQ,CAAC;QACrC,UAAU,EAAE,CAAC,CAAC,UAAU;QACxB,SAAS,EAAE,gBAAyB;KACrC,CAAC,CAAC,CAAC;AACN,CAAC;AASD,SAAS,eAAe,CACtB,QAAmB,EACnB,WAAwB,EACxB,sBAA8B,IAAI;IAElC,MAAM,QAAQ,GAAc,EAAE,CAAC;IAC/B,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAC;IAEnC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACzC,IAAI,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;YAAE,SAAS;QAEpD,MAAM,OAAO,GAAY;YACvB,QAAQ,EAAE,CAAC,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,CAAC;YACpD,cAAc,EAAE,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC;SAClD,CAAC;QACF,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QAEhB,KAAK,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC7C,IAAI,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;gBAAE,SAAS;YACpD,MAAM,GAAG,GAAG,iBAAiB,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;YACxD,IAAI,GAAG,IAAI,mBAAmB,EAAE,CAAC;gBAC/B,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,CAAC,CAAC;gBAChE,kBAAkB;gBAClB,KAAK,MAAM,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,EAAE,CAAC;oBAClD,OAAO,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;gBAChC,CAAC;gBACD,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YAClB,CAAC;QACH,CAAC;QAED,IAAI,OAAO,CAAC,QAAQ,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YACjC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACzB,CAAC;IACH,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,SAAS,mBAAmB,CAC1B,UAAqB,EACrB,aAAwB,EACxB,MAAmB,EACnB,SAAsB,EACtB,mBAA2B,IAAI,EAC/B,iBAAyB,IAAI;IAE7B,MAAM,OAAO,GAAgB,EAAE,CAAC;IAEhC,gCAAgC;IAChC,MAAM,UAAU,GAAG,eAAe,CAAC,UAAU,EAAE,MAAM,EAAE,gBAAgB,CAAC,CAAC;IAEzE,6DAA6D;IAC7D,KAAK,MAAM,OAAO,IAAI,UAAU,EAAE,CAAC;QACjC,IAAI,YAAY,GAAG,CAAC,CAAC,CAAC;QACtB,IAAI,OAAO,GAAG,CAAC,CAAC;QAEhB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,aAAa,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC9C,IAAI,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC;gBAAE,SAAS;YAE/B,MAAM,WAAW,GAAG,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC;YAC3D,6DAA6D;YAC7D,IAAI,YAAY,GAAG,CAAC,CAAC;YACrB,KAAK,MAAM,CAAC,IAAI,WAAW,EAAE,CAAC;gBAC5B,IAAI,OAAO,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC;oBAAE,YAAY,EAAE,CAAC;YACpD,CAAC;YACD,MAAM,KAAK,GAAG,OAAO,CAAC,cAAc,CAAC,IAAI,GAAG,WAAW,CAAC,IAAI,GAAG,YAAY,CAAC;YAC5E,MAAM,GAAG,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;YAEjD,IAAI,GAAG,GAAG,OAAO,IAAI,GAAG,IAAI,cAAc,EAAE,CAAC;gBAC3C,OAAO,GAAG,GAAG,CAAC;gBACd,YAAY,GAAG,CAAC,CAAC;YACnB,CAAC;QACH,CAAC;QAED,IAAI,YAAY,IAAI,CAAC,EAAE,CAAC;YACtB,sDAAsD;YACtD,MAAM,MAAM,GAAG,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,EAAE;gBACpD,MAAM,QAAQ,GAAG,EAAE,GAAG,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC;gBAC7D,OAAO,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;YACzF,CAAC,CAAC,CAAC;YAEH,OAAO,CAAC,IAAI,CAAC;gBACX,OAAO,EAAE,MAAM,CAAC,OAAO;gBACvB,UAAU,EAAE,aAAa,CAAC,YAAY,CAAC;gBACvC,UAAU,EAAE,OAAO;gBACnB,SAAS,EAAE,YAAY;aACxB,CAAC,CAAC;YAEH,mCAAmC;YACnC,KAAK,MAAM,MAAM,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;gBACtC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;YACjC,CAAC;YACD,SAAS,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;QAC9B,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAWD;;;;GAIG;AACH,SAAgB,cAAc,CAC5B,UAAqB,EACrB,aAAwB,EACxB,OAA0B;IAE1B,MAAM,MAAM,GAAG,IAAI,GAAG,EAAU,CAAC;IACjC,MAAM,SAAS,GAAG,IAAI,GAAG,EAAU,CAAC;IAEpC,iBAAiB;IACjB,MAAM,MAAM,GAAG,UAAU,CACvB,UAAU,EACV,aAAa,EACb,MAAM,EACN,SAAS,EACT,OAAO,EAAE,eAAe,IAAI,IAAI,CACjC,CAAC;IAEF,yBAAyB;IACzB,MAAM,aAAa,GAAG,iBAAiB,CACrC,UAAU,EACV,aAAa,EACb,MAAM,EACN,SAAS,EACT,OAAO,EAAE,sBAAsB,IAAI,IAAI,CACxC,CAAC;IAEF,2BAA2B;IAC3B,MAAM,UAAU,GAAG,mBAAmB,CACpC,UAAU,EACV,aAAa,EACb,MAAM,EACN,SAAS,EACT,OAAO,EAAE,gBAAgB,IAAI,IAAI,EACjC,OAAO,EAAE,wBAAwB,IAAI,IAAI,CAC1C,CAAC;IAEF,oBAAoB;IACpB,MAAM,MAAM,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IAC3D,MAAM,SAAS,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IAEpE,OAAO;QACL,aAAa,EAAE,MAAM;QACrB,oBAAoB,EAAE,aAAa;QACnC,iBAAiB,EAAE,UAAU;QAC7B,MAAM;QACN,SAAS;QACT,OAAO,EAAE;YACP,QAAQ,EAAE,UAAU,CAAC,MAAM;YAC3B,WAAW,EAAE,aAAa,CAAC,MAAM;YACjC,cAAc,EAAE,MAAM,CAAC,MAAM;YAC7B,sBAAsB,EAAE,aAAa,CAAC,MAAM;YAC5C,kBAAkB,EAAE,UAAU,CAAC,MAAM;YACrC,aAAa,EAAE,MAAM,CAAC,MAAM,GAAG,aAAa,CAAC,MAAM,GAAG,UAAU,CAAC,MAAM;YACvE,OAAO,EAAE,MAAM,CAAC,MAAM;YACtB,UAAU,EAAE,SAAS,CAAC,MAAM;SAC7B;KACF,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,SAAgB,iBAAiB,CAAC,MAA4B;IAC5D,OAAO;QACL,OAAO,EAAE;YACP,GAAG,MAAM,CAAC,aAAa;YACvB,GAAG,MAAM,CAAC,oBAAoB;YAC9B,GAAG,MAAM,CAAC,iBAAiB;SAC5B;QACD,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,SAAS,EAAE,MAAM,CAAC,SAAS;KAC5B,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Gravito Eval — Metrics Engine
|
|
3
|
+
*
|
|
4
|
+
* Computes detection, ranking, and severity agreement metrics
|
|
5
|
+
* from matched AI vs human findings.
|
|
6
|
+
*/
|
|
7
|
+
import type { Finding, MatchResult, MultiPassMatchResult, DetectionMetrics, RankingMetrics, SeverityMetrics, ConfidenceInterval } from "../types";
|
|
8
|
+
export declare function computeDetectionMetrics(matchResult: MatchResult | MultiPassMatchResult, totalAI: number, totalHuman: number): DetectionMetrics;
|
|
9
|
+
export declare function computeRankingMetrics(aiFindings: Finding[], humanFindings: Finding[], matchResult: MatchResult | MultiPassMatchResult): RankingMetrics;
|
|
10
|
+
export declare function computeSeverityMetrics(matchResult: MatchResult | MultiPassMatchResult): SeverityMetrics;
|
|
11
|
+
/**
|
|
12
|
+
* Wilson score confidence interval for proportions.
|
|
13
|
+
*/
|
|
14
|
+
export declare function wilsonInterval(successes: number, total: number, z?: number): ConfidenceInterval;
|
|
15
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/metrics/index.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EACV,OAAO,EACP,WAAW,EACX,oBAAoB,EACpB,gBAAgB,EAChB,cAAc,EACd,eAAe,EAEf,kBAAkB,EACnB,MAAM,UAAU,CAAC;AAKlB,wBAAgB,uBAAuB,CACrC,WAAW,EAAE,WAAW,GAAG,oBAAoB,EAC/C,OAAO,EAAE,MAAM,EACf,UAAU,EAAE,MAAM,GACjB,gBAAgB,CASlB;AAiBD,wBAAgB,qBAAqB,CACnC,UAAU,EAAE,OAAO,EAAE,EACrB,aAAa,EAAE,OAAO,EAAE,EACxB,WAAW,EAAE,WAAW,GAAG,oBAAoB,GAC9C,cAAc,CAkDhB;AAiCD,wBAAgB,sBAAsB,CACpC,WAAW,EAAE,WAAW,GAAG,oBAAoB,GAC9C,eAAe,CAgCjB;AAiDD;;GAEG;AACH,wBAAgB,cAAc,CAC5B,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,MAAM,EACb,CAAC,GAAE,MAAa,GACf,kBAAkB,CAcpB"}
|
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Gravito Eval — Metrics Engine
|
|
4
|
+
*
|
|
5
|
+
* Computes detection, ranking, and severity agreement metrics
|
|
6
|
+
* from matched AI vs human findings.
|
|
7
|
+
*/
|
|
8
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
9
|
+
exports.computeDetectionMetrics = computeDetectionMetrics;
|
|
10
|
+
exports.computeRankingMetrics = computeRankingMetrics;
|
|
11
|
+
exports.computeSeverityMetrics = computeSeverityMetrics;
|
|
12
|
+
exports.wilsonInterval = wilsonInterval;
|
|
13
|
+
const matching_1 = require("../matching");
|
|
14
|
+
// ─── Detection Metrics ────────────────────────────────────────────────────
|
|
15
|
+
function computeDetectionMetrics(matchResult, totalAI, totalHuman) {
|
|
16
|
+
const flat = "matched" in matchResult ? matchResult : (0, matching_1.toFlatMatchResult)(matchResult);
|
|
17
|
+
const matchedCount = flat.matched.length;
|
|
18
|
+
const recall = totalHuman > 0 ? matchedCount / totalHuman : 0;
|
|
19
|
+
const precision = totalAI > 0 ? matchedCount / totalAI : 0;
|
|
20
|
+
const f1 = recall + precision > 0 ? (2 * recall * precision) / (recall + precision) : 0;
|
|
21
|
+
return { recall, precision, f1, matchedCount, totalAI, totalHuman };
|
|
22
|
+
}
|
|
23
|
+
// ─── Ranking Metrics ──────────────────────────────────────────────────────
|
|
24
|
+
const SEVERITY_ORDER = {
|
|
25
|
+
critical: 4,
|
|
26
|
+
high: 3,
|
|
27
|
+
medium: 2,
|
|
28
|
+
low: 1,
|
|
29
|
+
};
|
|
30
|
+
function sortBySeverity(findings) {
|
|
31
|
+
return [...findings].sort((a, b) => SEVERITY_ORDER[b.severity] - SEVERITY_ORDER[a.severity]);
|
|
32
|
+
}
|
|
33
|
+
function computeRankingMetrics(aiFindings, humanFindings, matchResult) {
|
|
34
|
+
const flat = "matched" in matchResult ? matchResult : (0, matching_1.toFlatMatchResult)(matchResult);
|
|
35
|
+
const sortedAI = sortBySeverity(aiFindings);
|
|
36
|
+
const sortedHuman = sortBySeverity(humanFindings);
|
|
37
|
+
const humanTop3Ids = new Set(sortedHuman.slice(0, 3).map((f) => f.id));
|
|
38
|
+
const humanTop5Ids = new Set(sortedHuman.slice(0, 5).map((f) => f.id));
|
|
39
|
+
// Build mapping: AI finding ID → matched human finding ID
|
|
40
|
+
const aiToHuman = new Map();
|
|
41
|
+
for (const pair of flat.matched) {
|
|
42
|
+
aiToHuman.set(pair.aiIssue.id, pair.humanIssue.id);
|
|
43
|
+
}
|
|
44
|
+
// Top-3 overlap: how many of AI's top 3 match human's top 3
|
|
45
|
+
const aiTop3 = sortedAI.slice(0, 3);
|
|
46
|
+
let top3Overlap = 0;
|
|
47
|
+
for (const ai of aiTop3) {
|
|
48
|
+
const matchedHumanId = aiToHuman.get(ai.id);
|
|
49
|
+
if (matchedHumanId && humanTop3Ids.has(matchedHumanId)) {
|
|
50
|
+
top3Overlap++;
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
// Top-5 overlap
|
|
54
|
+
const aiTop5 = sortedAI.slice(0, 5);
|
|
55
|
+
let top5Overlap = 0;
|
|
56
|
+
for (const ai of aiTop5) {
|
|
57
|
+
const matchedHumanId = aiToHuman.get(ai.id);
|
|
58
|
+
if (matchedHumanId && humanTop5Ids.has(matchedHumanId)) {
|
|
59
|
+
top5Overlap++;
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
const top3Rate = Math.min(sortedHuman.length, 3) > 0
|
|
63
|
+
? top3Overlap / Math.min(sortedHuman.length, 3)
|
|
64
|
+
: 0;
|
|
65
|
+
const top5Rate = Math.min(sortedHuman.length, 5) > 0
|
|
66
|
+
? top5Overlap / Math.min(sortedHuman.length, 5)
|
|
67
|
+
: 0;
|
|
68
|
+
// Spearman correlation on matched pairs
|
|
69
|
+
const spearman = computeSpearmanOnMatched(sortedAI, sortedHuman, flat.matched);
|
|
70
|
+
return {
|
|
71
|
+
top3Overlap: top3Rate,
|
|
72
|
+
top5Overlap: top5Rate,
|
|
73
|
+
spearmanCorrelation: spearman,
|
|
74
|
+
};
|
|
75
|
+
}
|
|
76
|
+
function computeSpearmanOnMatched(sortedAI, sortedHuman, matched) {
|
|
77
|
+
if (matched.length < 2)
|
|
78
|
+
return 0;
|
|
79
|
+
const aiRank = new Map();
|
|
80
|
+
const humanRank = new Map();
|
|
81
|
+
sortedAI.forEach((f, i) => aiRank.set(f.id, i + 1));
|
|
82
|
+
sortedHuman.forEach((f, i) => humanRank.set(f.id, i + 1));
|
|
83
|
+
let sumD2 = 0;
|
|
84
|
+
let n = 0;
|
|
85
|
+
for (const pair of matched) {
|
|
86
|
+
const rAi = aiRank.get(pair.aiIssue.id);
|
|
87
|
+
const rHuman = humanRank.get(pair.humanIssue.id);
|
|
88
|
+
if (rAi !== undefined && rHuman !== undefined) {
|
|
89
|
+
const d = rAi - rHuman;
|
|
90
|
+
sumD2 += d * d;
|
|
91
|
+
n++;
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
if (n < 2)
|
|
95
|
+
return 0;
|
|
96
|
+
return 1 - (6 * sumD2) / (n * (n * n - 1));
|
|
97
|
+
}
|
|
98
|
+
// ─── Severity Agreement ───────────────────────────────────────────────────
|
|
99
|
+
function computeSeverityMetrics(matchResult) {
|
|
100
|
+
const flat = "matched" in matchResult ? matchResult : (0, matching_1.toFlatMatchResult)(matchResult);
|
|
101
|
+
const levels = ["low", "medium", "high", "critical"];
|
|
102
|
+
// Build confusion matrix
|
|
103
|
+
const matrix = {};
|
|
104
|
+
for (const l of levels) {
|
|
105
|
+
matrix[l] = {};
|
|
106
|
+
for (const l2 of levels) {
|
|
107
|
+
matrix[l][l2] = 0;
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
let totalAbsError = 0;
|
|
111
|
+
for (const pair of flat.matched) {
|
|
112
|
+
matrix[pair.aiIssue.severity][pair.humanIssue.severity]++;
|
|
113
|
+
totalAbsError += Math.abs(SEVERITY_ORDER[pair.aiIssue.severity] - SEVERITY_ORDER[pair.humanIssue.severity]);
|
|
114
|
+
}
|
|
115
|
+
const n = flat.matched.length;
|
|
116
|
+
const mae = n > 0 ? totalAbsError / n : 0;
|
|
117
|
+
// Weighted Cohen's Kappa
|
|
118
|
+
const kappa = computeWeightedKappa(flat.matched, levels);
|
|
119
|
+
return {
|
|
120
|
+
weightedKappa: kappa,
|
|
121
|
+
meanAbsoluteError: mae,
|
|
122
|
+
confusionMatrix: matrix,
|
|
123
|
+
};
|
|
124
|
+
}
|
|
125
|
+
function computeWeightedKappa(matched, levels) {
|
|
126
|
+
const n = matched.length;
|
|
127
|
+
if (n === 0)
|
|
128
|
+
return 0;
|
|
129
|
+
const k = levels.length;
|
|
130
|
+
const observed = Array.from({ length: k }, () => Array(k).fill(0));
|
|
131
|
+
const weights = Array.from({ length: k }, (_, i) => Array.from({ length: k }, (_, j) => {
|
|
132
|
+
const maxDist = k - 1;
|
|
133
|
+
return maxDist > 0 ? Math.pow(Math.abs(i - j) / maxDist, 2) : 0;
|
|
134
|
+
}));
|
|
135
|
+
const levelIdx = new Map(levels.map((l, i) => [l, i]));
|
|
136
|
+
for (const pair of matched) {
|
|
137
|
+
const ai = levelIdx.get(pair.aiIssue.severity) ?? 0;
|
|
138
|
+
const hu = levelIdx.get(pair.humanIssue.severity) ?? 0;
|
|
139
|
+
observed[ai][hu]++;
|
|
140
|
+
}
|
|
141
|
+
// Marginals
|
|
142
|
+
const rowSums = observed.map((row) => row.reduce((a, b) => a + b, 0));
|
|
143
|
+
const colSums = Array(k).fill(0);
|
|
144
|
+
for (let i = 0; i < k; i++) {
|
|
145
|
+
for (let j = 0; j < k; j++) {
|
|
146
|
+
colSums[j] += observed[i][j];
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
let po = 0;
|
|
150
|
+
let pe = 0;
|
|
151
|
+
for (let i = 0; i < k; i++) {
|
|
152
|
+
for (let j = 0; j < k; j++) {
|
|
153
|
+
po += weights[i][j] * (observed[i][j] / n);
|
|
154
|
+
pe += weights[i][j] * ((rowSums[i] / n) * (colSums[j] / n));
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
return pe === 0 ? 1 : 1 - po / pe;
|
|
158
|
+
}
|
|
159
|
+
// ─── Confidence Intervals ─────────────────────────────────────────────────
|
|
160
|
+
/**
|
|
161
|
+
* Wilson score confidence interval for proportions.
|
|
162
|
+
*/
|
|
163
|
+
function wilsonInterval(successes, total, z = 1.96) {
|
|
164
|
+
if (total === 0)
|
|
165
|
+
return { mean: 0, lowerBound: 0, upperBound: 0 };
|
|
166
|
+
const p = successes / total;
|
|
167
|
+
const z2 = z * z;
|
|
168
|
+
const denominator = 1 + z2 / total;
|
|
169
|
+
const center = p + z2 / (2 * total);
|
|
170
|
+
const margin = z * Math.sqrt((p * (1 - p) + z2 / (4 * total)) / total);
|
|
171
|
+
return {
|
|
172
|
+
mean: p,
|
|
173
|
+
lowerBound: Math.max(0, (center - margin) / denominator),
|
|
174
|
+
upperBound: Math.min(1, (center + margin) / denominator),
|
|
175
|
+
};
|
|
176
|
+
}
|
|
177
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/metrics/index.ts"],"names":[],"mappings":";AAAA;;;;;GAKG;;AAgBH,0DAaC;AAiBD,sDAsDC;AAiCD,wDAkCC;AAoDD,wCAkBC;AAjOD,0CAAgD;AAEhD,6EAA6E;AAE7E,SAAgB,uBAAuB,CACrC,WAA+C,EAC/C,OAAe,EACf,UAAkB;IAElB,MAAM,IAAI,GAAG,SAAS,IAAI,WAAW,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,IAAA,4BAAiB,EAAC,WAAW,CAAC,CAAC;IACrF,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;IAEzC,MAAM,MAAM,GAAG,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC;IAC9D,MAAM,SAAS,GAAG,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;IAC3D,MAAM,EAAE,GAAG,MAAM,GAAG,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAExF,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,EAAE,EAAE,YAAY,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC;AACtE,CAAC;AAED,6EAA6E;AAE7E,MAAM,cAAc,GAAkC;IACpD,QAAQ,EAAE,CAAC;IACX,IAAI,EAAE,CAAC;IACP,MAAM,EAAE,CAAC;IACT,GAAG,EAAE,CAAC;CACP,CAAC;AAEF,SAAS,cAAc,CAAC,QAAmB;IACzC,OAAO,CAAC,GAAG,QAAQ,CAAC,CAAC,IAAI,CACvB,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,cAAc,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,cAAc,CAAC,CAAC,CAAC,QAAQ,CAAC,CAClE,CAAC;AACJ,CAAC;AAED,SAAgB,qBAAqB,CACnC,UAAqB,EACrB,aAAwB,EACxB,WAA+C;IAE/C,MAAM,IAAI,GAAG,SAAS,IAAI,WAAW,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,IAAA,4BAAiB,EAAC,WAAW,CAAC,CAAC;IAErF,MAAM,QAAQ,GAAG,cAAc,CAAC,UAAU,CAAC,CAAC;IAC5C,MAAM,WAAW,GAAG,cAAc,CAAC,aAAa,CAAC,CAAC;IAElD,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACvE,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAEvE,0DAA0D;IAC1D,MAAM,SAAS,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC5C,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;QAChC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,EAAE,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;IACrD,CAAC;IAED,4DAA4D;IAC5D,MAAM,MAAM,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACpC,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,KAAK,MAAM,EAAE,IAAI,MAAM,EAAE,CAAC;QACxB,MAAM,cAAc,GAAG,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;QAC5C,IAAI,cAAc,IAAI,YAAY,CAAC,GAAG,CAAC,cAAc,CAAC,EAAE,CAAC;YACvD,WAAW,EAAE,CAAC;QAChB,CAAC;IACH,CAAC;IAED,gBAAgB;IAChB,MAAM,MAAM,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACpC,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,KAAK,MAAM,EAAE,IAAI,MAAM,EAAE,CAAC;QACxB,MAAM,cAAc,GAAG,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;QAC5C,IAAI,cAAc,IAAI,YAAY,CAAC,GAAG,CAAC,cAAc,CAAC,EAAE,CAAC;YACvD,WAAW,EAAE,CAAC;QAChB,CAAC;IACH,CAAC;IAED,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC;QAClD,CAAC,CAAC,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC,CAAC;QAC/C,CAAC,CAAC,CAAC,CAAC;IACN,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC;QAClD,CAAC,CAAC,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC,CAAC;QAC/C,CAAC,CAAC,CAAC,CAAC;IAEN,wCAAwC;IACxC,MAAM,QAAQ,GAAG,wBAAwB,CAAC,QAAQ,EAAE,WAAW,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;IAE/E,OAAO;QACL,WAAW,EAAE,QAAQ;QACrB,WAAW,EAAE,QAAQ;QACrB,mBAAmB,EAAE,QAAQ;KAC9B,CAAC;AACJ,CAAC;AAED,SAAS,wBAAwB,CAC/B,QAAmB,EACnB,WAAsB,EACtB,OAAoD;IAEpD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,CAAC,CAAC;IAEjC,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;IACzC,MAAM,SAAS,GAAG,IAAI,GAAG,EAAkB,CAAC;IAE5C,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IACpD,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IAE1D,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE,CAAC;QAC3B,MAAM,GAAG,GAAG,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QACxC,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACjD,IAAI,GAAG,KAAK,SAAS,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;YAC9C,MAAM,CAAC,GAAG,GAAG,GAAG,MAAM,CAAC;YACvB,KAAK,IAAI,CAAC,GAAG,CAAC,CAAC;YACf,CAAC,EAAE,CAAC;QACN,CAAC;IACH,CAAC;IAED,IAAI,CAAC,GAAG,CAAC;QAAE,OAAO,CAAC,CAAC;IACpB,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;AAC7C,CAAC;AAED,6EAA6E;AAE7E,SAAgB,sBAAsB,CACpC,WAA+C;IAE/C,MAAM,IAAI,GAAG,SAAS,IAAI,WAAW,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,IAAA,4BAAiB,EAAC,WAAW,CAAC,CAAC;IACrF,MAAM,MAAM,GAAoB,CAAC,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,UAAU,CAAC,CAAC;IAEtE,yBAAyB;IACzB,MAAM,MAAM,GAA2C,EAAE,CAAC;IAC1D,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;QACvB,MAAM,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC;QACf,KAAK,MAAM,EAAE,IAAI,MAAM,EAAE,CAAC;YACxB,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC;QACpB,CAAC;IACH,CAAC;IAED,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;QAChC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC1D,aAAa,IAAI,IAAI,CAAC,GAAG,CACvB,cAAc,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,cAAc,CAAC,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,CACjF,CAAC;IACJ,CAAC;IAED,MAAM,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;IAC9B,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,aAAa,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAE1C,yBAAyB;IACzB,MAAM,KAAK,GAAG,oBAAoB,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IAEzD,OAAO;QACL,aAAa,EAAE,KAAK;QACpB,iBAAiB,EAAE,GAAG;QACtB,eAAe,EAAE,MAAM;KACxB,CAAC;AACJ,CAAC;AAED,SAAS,oBAAoB,CAC3B,OAAoD,EACpD,MAAuB;IAEvB,MAAM,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC;IACzB,IAAI,CAAC,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAEtB,MAAM,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;IACxB,MAAM,QAAQ,GAAe,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,EAAE,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAC/E,MAAM,OAAO,GAAe,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAC7D,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACjC,MAAM,OAAO,GAAG,CAAC,GAAG,CAAC,CAAC;QACtB,OAAO,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAClE,CAAC,CAAC,CACH,CAAC;IAEF,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAEvD,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE,CAAC;QAC3B,MAAM,EAAE,GAAG,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACpD,MAAM,EAAE,GAAG,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACvD,QAAQ,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC;IACrB,CAAC;IAED,YAAY;IACZ,MAAM,OAAO,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IACtE,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACjC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC3B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3B,OAAO,CAAC,CAAC,CAAC,IAAI,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC/B,CAAC;IACH,CAAC;IAED,IAAI,EAAE,GAAG,CAAC,CAAC;IACX,IAAI,EAAE,GAAG,CAAC,CAAC;IACX,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC3B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3B,EAAE,IAAI,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YAC3C,EAAE,IAAI,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QAC9D,CAAC;IACH,CAAC;IAED,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC;AACpC,CAAC;AAED,6EAA6E;AAE7E;;GAEG;AACH,SAAgB,cAAc,CAC5B,SAAiB,EACjB,KAAa,EACb,IAAY,IAAI;IAEhB,IAAI,KAAK,KAAK,CAAC;QAAE,OAAO,EAAE,IAAI,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,CAAC;IAElE,MAAM,CAAC,GAAG,SAAS,GAAG,KAAK,CAAC;IAC5B,MAAM,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC;IACjB,MAAM,WAAW,GAAG,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC;IACnC,MAAM,MAAM,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC;IACpC,MAAM,MAAM,GAAG,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC;IAEvE,OAAO;QACL,IAAI,EAAE,CAAC;QACP,UAAU,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,MAAM,GAAG,MAAM,CAAC,GAAG,WAAW,CAAC;QACxD,UAAU,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,MAAM,GAAG,MAAM,CAAC,GAAG,WAAW,CAAC;KACzD,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Optional anonymous telemetry for gravito-eval.
|
|
3
|
+
*
|
|
4
|
+
* Tracks: timestamp, package version, command name.
|
|
5
|
+
* No PII. No findings data. No IP logging.
|
|
6
|
+
*
|
|
7
|
+
* Disable with: GRAVITO_TELEMETRY=0 or --no-telemetry flag.
|
|
8
|
+
*/
|
|
9
|
+
export declare function trackRun(command: string): void;
|
|
10
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/telemetry/index.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAkCH,wBAAgB,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAwC9C"}
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Optional anonymous telemetry for gravito-eval.
|
|
4
|
+
*
|
|
5
|
+
* Tracks: timestamp, package version, command name.
|
|
6
|
+
* No PII. No findings data. No IP logging.
|
|
7
|
+
*
|
|
8
|
+
* Disable with: GRAVITO_TELEMETRY=0 or --no-telemetry flag.
|
|
9
|
+
*/
|
|
10
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
11
|
+
if (k2 === undefined) k2 = k;
|
|
12
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
13
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
14
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
15
|
+
}
|
|
16
|
+
Object.defineProperty(o, k2, desc);
|
|
17
|
+
}) : (function(o, m, k, k2) {
|
|
18
|
+
if (k2 === undefined) k2 = k;
|
|
19
|
+
o[k2] = m[k];
|
|
20
|
+
}));
|
|
21
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
22
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
23
|
+
}) : function(o, v) {
|
|
24
|
+
o["default"] = v;
|
|
25
|
+
});
|
|
26
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
27
|
+
var ownKeys = function(o) {
|
|
28
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
29
|
+
var ar = [];
|
|
30
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
31
|
+
return ar;
|
|
32
|
+
};
|
|
33
|
+
return ownKeys(o);
|
|
34
|
+
};
|
|
35
|
+
return function (mod) {
|
|
36
|
+
if (mod && mod.__esModule) return mod;
|
|
37
|
+
var result = {};
|
|
38
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
39
|
+
__setModuleDefault(result, mod);
|
|
40
|
+
return result;
|
|
41
|
+
};
|
|
42
|
+
})();
|
|
43
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
44
|
+
exports.trackRun = trackRun;
|
|
45
|
+
const https = __importStar(require("https"));
|
|
46
|
+
const fs = __importStar(require("fs"));
|
|
47
|
+
const path = __importStar(require("path"));
|
|
48
|
+
const TELEMETRY_ENDPOINT = "https://gravito.ai/api/telemetry/eval";
|
|
49
|
+
function isDisabled() {
|
|
50
|
+
return (process.env.GRAVITO_TELEMETRY === "0" ||
|
|
51
|
+
process.env.GRAVITO_TELEMETRY === "false" ||
|
|
52
|
+
process.env.DO_NOT_TRACK === "1" ||
|
|
53
|
+
process.argv.includes("--no-telemetry"));
|
|
54
|
+
}
|
|
55
|
+
function getVersion() {
|
|
56
|
+
try {
|
|
57
|
+
let dir = __dirname;
|
|
58
|
+
while (!fs.existsSync(path.join(dir, "package.json"))) {
|
|
59
|
+
const parent = path.dirname(dir);
|
|
60
|
+
if (parent === dir)
|
|
61
|
+
return "unknown";
|
|
62
|
+
dir = parent;
|
|
63
|
+
}
|
|
64
|
+
const pkg = JSON.parse(fs.readFileSync(path.join(dir, "package.json"), "utf-8"));
|
|
65
|
+
return pkg.version || "unknown";
|
|
66
|
+
}
|
|
67
|
+
catch {
|
|
68
|
+
return "unknown";
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
function trackRun(command) {
|
|
72
|
+
if (isDisabled())
|
|
73
|
+
return;
|
|
74
|
+
const payload = JSON.stringify({
|
|
75
|
+
timestamp: new Date().toISOString(),
|
|
76
|
+
version: getVersion(),
|
|
77
|
+
command,
|
|
78
|
+
});
|
|
79
|
+
try {
|
|
80
|
+
const url = new URL(TELEMETRY_ENDPOINT);
|
|
81
|
+
const req = https.request({
|
|
82
|
+
hostname: url.hostname,
|
|
83
|
+
port: 443,
|
|
84
|
+
path: url.pathname,
|
|
85
|
+
method: "POST",
|
|
86
|
+
headers: {
|
|
87
|
+
"Content-Type": "application/json",
|
|
88
|
+
"Content-Length": Buffer.byteLength(payload),
|
|
89
|
+
},
|
|
90
|
+
timeout: 2000,
|
|
91
|
+
}, () => {
|
|
92
|
+
// Intentionally ignore response
|
|
93
|
+
});
|
|
94
|
+
req.on("error", () => {
|
|
95
|
+
// Silently fail — telemetry must never block the CLI
|
|
96
|
+
});
|
|
97
|
+
req.write(payload);
|
|
98
|
+
req.end();
|
|
99
|
+
// Unref so the process can exit without waiting
|
|
100
|
+
req.socket?.unref?.();
|
|
101
|
+
}
|
|
102
|
+
catch {
|
|
103
|
+
// Silently fail
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/telemetry/index.ts"],"names":[],"mappings":";AAAA;;;;;;;GAOG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAkCH,4BAwCC;AAxED,6CAA+B;AAC/B,uCAAyB;AACzB,2CAA6B;AAE7B,MAAM,kBAAkB,GAAG,uCAAuC,CAAC;AAEnE,SAAS,UAAU;IACjB,OAAO,CACL,OAAO,CAAC,GAAG,CAAC,iBAAiB,KAAK,GAAG;QACrC,OAAO,CAAC,GAAG,CAAC,iBAAiB,KAAK,OAAO;QACzC,OAAO,CAAC,GAAG,CAAC,YAAY,KAAK,GAAG;QAChC,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CACxC,CAAC;AACJ,CAAC;AAED,SAAS,UAAU;IACjB,IAAI,CAAC;QACH,IAAI,GAAG,GAAG,SAAS,CAAC;QACpB,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,cAAc,CAAC,CAAC,EAAE,CAAC;YACtD,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;YACjC,IAAI,MAAM,KAAK,GAAG;gBAAE,OAAO,SAAS,CAAC;YACrC,GAAG,GAAG,MAAM,CAAC;QACf,CAAC;QACD,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CACpB,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,cAAc,CAAC,EAAE,OAAO,CAAC,CACzD,CAAC;QACF,OAAO,GAAG,CAAC,OAAO,IAAI,SAAS,CAAC;IAClC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC;AAED,SAAgB,QAAQ,CAAC,OAAe;IACtC,IAAI,UAAU,EAAE;QAAE,OAAO;IAEzB,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC;QAC7B,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,OAAO,EAAE,UAAU,EAAE;QACrB,OAAO;KACR,CAAC,CAAC;IAEH,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,kBAAkB,CAAC,CAAC;QACxC,MAAM,GAAG,GAAG,KAAK,CAAC,OAAO,CACvB;YACE,QAAQ,EAAE,GAAG,CAAC,QAAQ;YACtB,IAAI,EAAE,GAAG;YACT,IAAI,EAAE,GAAG,CAAC,QAAQ;YAClB,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACP,cAAc,EAAE,kBAAkB;gBAClC,gBAAgB,EAAE,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC;aAC7C;YACD,OAAO,EAAE,IAAI;SACd,EACD,GAAG,EAAE;YACH,gCAAgC;QAClC,CAAC,CACF,CAAC;QAEF,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;YACnB,qDAAqD;QACvD,CAAC,CAAC,CAAC;QAEH,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QACnB,GAAG,CAAC,GAAG,EAAE,CAAC;QAEV,gDAAgD;QAChD,GAAG,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC;IACxB,CAAC;IAAC,MAAM,CAAC;QACP,gBAAgB;IAClB,CAAC;AACH,CAAC"}
|