@aldus-runtime/regression 0.2.0-next.18 → 0.2.0-next.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/corpus.d.ts +3 -1
- package/dist/corpus.d.ts.map +1 -1
- package/dist/corpus.js +20 -1
- package/dist/corpus.js.map +1 -1
- package/dist/metrics.d.ts +21 -0
- package/dist/metrics.d.ts.map +1 -1
- package/dist/metrics.js +6 -0
- package/dist/metrics.js.map +1 -1
- package/package.json +3 -3
- package/src/corpus.ts +20 -1
- package/src/metrics.ts +28 -0
package/dist/corpus.d.ts
CHANGED
|
@@ -140,6 +140,7 @@ export declare const evaluatorOutcomeSchema: z.ZodObject<{
|
|
|
140
140
|
severity: z.ZodOptional<z.ZodString>;
|
|
141
141
|
confidence: z.ZodOptional<z.ZodNumber>;
|
|
142
142
|
}, z.core.$strip>>;
|
|
143
|
+
siteMetricsMeasurable: z.ZodOptional<z.ZodBoolean>;
|
|
143
144
|
}, z.core.$strip>;
|
|
144
145
|
/** @see evaluatorOutcomeSchema */
|
|
145
146
|
export type EvaluatorOutcome = z.infer<typeof evaluatorOutcomeSchema>;
|
|
@@ -162,13 +163,14 @@ export declare const evaluatorRunSchema: z.ZodObject<{
|
|
|
162
163
|
severity: z.ZodOptional<z.ZodString>;
|
|
163
164
|
confidence: z.ZodOptional<z.ZodNumber>;
|
|
164
165
|
}, z.core.$strip>>;
|
|
166
|
+
siteMetricsMeasurable: z.ZodOptional<z.ZodBoolean>;
|
|
165
167
|
}, z.core.$strip>>;
|
|
166
168
|
executedAt: z.ZodISODateTime;
|
|
167
169
|
}, z.core.$strip>;
|
|
168
170
|
/** @see evaluatorRunSchema */
|
|
169
171
|
export type EvaluatorRun = z.infer<typeof evaluatorRunSchema>;
|
|
170
172
|
/** Schema version this package stamps on records it constructs. */
|
|
171
|
-
export declare const REGRESSION_SCHEMA_VERSION = "1.
|
|
173
|
+
export declare const REGRESSION_SCHEMA_VERSION = "1.8";
|
|
172
174
|
/**
|
|
173
175
|
* Validate a corpus and check the uniqueness constraint Zod cannot express.
|
|
174
176
|
*
|
package/dist/corpus.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"corpus.d.ts","sourceRoot":"","sources":["../src/corpus.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAGH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAOxB;;;;;;GAMG;AACH,eAAO,MAAM,eAAe,aAA6B,CAAC;AAE1D;;;;;;GAMG;AACH,eAAO,MAAM,aAAa,aAA6B,CAAC;AAExD;;;;;;GAMG;AACH,eAAO,MAAM,eAAe,uCAAmE,CAAC;AAEhG,2BAA2B;AAC3B,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AAE9D,wDAAwD;AACxD,eAAO,MAAM,kBAAkB;;;;iBASuB,CAAC;AAEvD,8BAA8B;AAC9B,MAAM,MAAM,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,kBAAkB,CAAC,CAAC;AAE9D;;;;;;;GAOG;AACH,eAAO,MAAM,gBAAgB;;;;;;;;;;;;;;;;;;;;;;;;;iBA2DzB,CAAC;AAEL,4BAA4B;AAC5B,MAAM,MAAM,UAAU,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,gBAAgB,CAAC,CAAC;AAE1D,gDAAgD;AAChD,eAAO,MAAM,kBAAkB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAkB3B,CAAC;AAEL,8BAA8B;AAC9B,MAAM,MAAM,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,kBAAkB,CAAC,CAAC;AAE9D,yCAAyC;AACzC,eAAO,MAAM,sBAAsB;;;;iBAS2B,CAAC;AAE/D,kCAAkC;AAClC,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,sBAAsB,CAAC,CAAC;AAEtE,6CAA6C;AAC7C,eAAO,MAAM,sBAAsB
|
|
1
|
+
{"version":3,"file":"corpus.d.ts","sourceRoot":"","sources":["../src/corpus.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAGH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAOxB;;;;;;GAMG;AACH,eAAO,MAAM,eAAe,aAA6B,CAAC;AAE1D;;;;;;GAMG;AACH,eAAO,MAAM,aAAa,aAA6B,CAAC;AAExD;;;;;;GAMG;AACH,eAAO,MAAM,eAAe,uCAAmE,CAAC;AAEhG,2BAA2B;AAC3B,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AAE9D,wDAAwD;AACxD,eAAO,MAAM,kBAAkB;;;;iBASuB,CAAC;AAEvD,8BAA8B;AAC9B,MAAM,MAAM,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,kBAAkB,CAAC,CAAC;AAE9D;;;;;;;GAOG;AACH,eAAO,MAAM,gBAAgB;;;;;;;;;;;;;;;;;;;;;;;;;iBA2DzB,CAAC;AAEL,4BAA4B;AAC5B,MAAM,MAAM,UAAU,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,gBAAgB,CAAC,CAAC;AAE1D,gDAAgD;AAChD,eAAO,MAAM,kBAAkB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAkB3B,CAAC;AAEL,8BAA8B;AAC9B,MAAM,MAAM,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,kBAAkB,CAAC,CAAC;AAE9D,yCAAyC;AACzC,eAAO,MAAM,sBAAsB;;;;iBAS2B,CAAC;AAE/D,kCAAkC;AAClC,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,sBAAsB,CAAC,CAAC;AAEtE,6CAA6C;AAC7C,eAAO,MAAM,sBAAsB;;;;;;;;;iBA4B2B,CAAC;AAE/D,kCAAkC;AAClC,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,sBAAsB,CAAC,CAAC;AAEtE;;;;;GAKG;AACH,eAAO,MAAM,kBAAkB;;;;;;;;;;;;;;;;iBAsB3B,CAAC;AAEL,8BAA8B;AAC9B,MAAM,MAAM,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,kBAAkB,CAAC,CAAC;AAE9D,mEAAmE;AACnE,eAAO,MAAM,yBAAyB,QAAiB,CAAC;AAExD;;;;GAIG;AACH,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,OAAO,GAAG,YAAY,CAgC9D;AAED;;;;GAIG;AACH,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,OAAO,GAAG,YAAY,CA8B9D"}
|
package/dist/corpus.js
CHANGED
|
@@ -161,8 +161,27 @@ export const evaluatorOutcomeSchema = z
|
|
|
161
161
|
caseId: identifier,
|
|
162
162
|
/** Whether the evaluator flagged the subject. */
|
|
163
163
|
flagged: z.boolean(),
|
|
164
|
-
/**
|
|
164
|
+
/**
|
|
165
|
+
* The defect occurrences it enumerated. Empty when not flagged, **and also empty when the
|
|
166
|
+
* evaluator flagged without enumerating** (#140).
|
|
167
|
+
*
|
|
168
|
+
* An empty list under `flagged: true` is therefore not a contradiction and must not be
|
|
169
|
+
* repaired by inventing an entry. A wrapped legacy evaluator reports that it had something to
|
|
170
|
+
* say and not how much; fabricating one finding would make its report count as one defect,
|
|
171
|
+
* which is the statistic this distinction exists to protect.
|
|
172
|
+
*/
|
|
165
173
|
findings: z.array(evaluatorFindingSchema).max(256),
|
|
174
|
+
/**
|
|
175
|
+
* Whether site-level metrics are computable for this case (#140).
|
|
176
|
+
*
|
|
177
|
+
* `false` when the evaluator flagged without enumerating, or when a report could not be mapped
|
|
178
|
+
* to this case's subject scope at all. Site-level precision and recall are then **unmeasurable
|
|
179
|
+
* rather than zero**, and a metric that silently treated an unenumerated flag as zero findings
|
|
180
|
+
* would report perfect precision for an evaluator nobody measured.
|
|
181
|
+
*
|
|
182
|
+
* Absent means measurable, which is what every record written before this field meant.
|
|
183
|
+
*/
|
|
184
|
+
siteMetricsMeasurable: z.boolean().optional(),
|
|
166
185
|
})
|
|
167
186
|
.meta({ id: "EvaluatorOutcome", title: "EvaluatorOutcome" });
|
|
168
187
|
/**
|
package/dist/corpus.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"corpus.js","sourceRoot":"","sources":["../src/corpus.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,OAAO,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AACrE,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,oBAAoB,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAEpE,iCAAiC;AACjC,MAAM,UAAU,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;AAE9C;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;AAE1D;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;AAExD;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;AAKhG,wDAAwD;AACxD,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC;KAChC,MAAM,CAAC;IACN,6DAA6D;IAC7D,QAAQ,EAAE,eAAe;IACzB,2EAA2E;IAC3E,QAAQ,EAAE,aAAa;IACvB,kDAAkD;IAClD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE;CACtC,CAAC;KACD,IAAI,CAAC,EAAE,EAAE,EAAE,cAAc,EAAE,KAAK,EAAE,cAAc,EAAE,CAAC,CAAC;AAKvD;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC;KAC9B,MAAM,CAAC;IACN,+CAA+C;IAC/C,MAAM,EAAE,UAAU;IAClB,+FAA+F;IAC/F,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC;IACvC,mDAAmD;IACnD,KAAK,EAAE,eAAe;IACtB;;;;;;;OAOG;IACH,SAAS,EAAE,CAAC,CAAC,OAAO,EAAE;IACtB,gEAAgE;IAChE,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC;IAC9C;;;;OAIG;IACH,QAAQ,EAAE,aAAa,CAAC,QAAQ,EAAE;IAClC;;;;;;;;OAQG;IACH,gBAAgB,EAAE,UAAU,CAAC,QAAQ,EAAE;IACvC,6FAA6F;IAC7F,UAAU,EAAE,cAAc;IAC1B,mCAAmC;IACnC,UAAU,EAAE,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;CAC7C,CAAC;KACD,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,SAAS,KAAK,CAAC,KAAK,CAAC,QAAQ,KAAK,SAAS,CAAC,EAAE;IACrE,OAAO,EACL,oIAAoI;IACtI,IAAI,EAAE,CAAC,UAAU,CAAC;CACnB,CAAC;KACD,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,SAAS,IAAI,KAAK,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE;IACjE,OAAO,EAAE,8CAA8C;IACvD,IAAI,EAAE,CAAC,UAAU,CAAC;CACnB,CAAC;KACD,IAAI,CAAC;IACJ,EAAE,EAAE,YAAY;IAChB,KAAK,EAAE,YAAY;IACnB,WAAW,EACT,2FAA2F;QAC3F,4FAA4F;QAC5F,sFAAsF;QACtF,0FAA0F;QAC1F,wFAAwF;QACxF,8BAA8B;CACjC,CAAC,CAAC;AAKL,gDAAgD;AAChD,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC;KAChC,MAAM,CAAC;IACN,gDAAgD;IAChD,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,8BAA8B,CAAC;IAC/D,+BAA+B;IAC/B,QAAQ,EAAE,UAAU;IACpB,8BAA8B;IAC9B,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE;IAC5C,0BAA0B;IAC1B,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC;CAC9C,CAAC;KACD,IAAI,CAAC;IACJ,EAAE,EAAE,cAAc;IAClB,KAAK,EAAE,cAAc;IACrB,WAAW,EACT,yFAAyF;QACzF,uFAAuF;QACvF,mCAAmC;CACtC,CAAC,CAAC;AAKL,yCAAyC;AACzC,MAAM,CAAC,MAAM,sBAAsB,GAAG,CAAC;KACpC,MAAM,CAAC;IACN,6FAA6F;IAC7F,QAAQ,EAAE,eAAe;IACzB,6DAA6D;IAC7D,QAAQ,EAAE,aAAa,CAAC,QAAQ,EAAE;IAClC,8DAA8D;IAC9D,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;CAChD,CAAC;KACD,IAAI,CAAC,EAAE,EAAE,EAAE,kBAAkB,EAAE,KAAK,EAAE,kBAAkB,EAAE,CAAC,CAAC;AAK/D,6CAA6C;AAC7C,MAAM,CAAC,MAAM,sBAAsB,GAAG,CAAC;KACpC,MAAM,CAAC;IACN,oCAAoC;IACpC,MAAM,EAAE,UAAU;IAClB,iDAAiD;IACjD,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE;IACpB
|
|
1
|
+
{"version":3,"file":"corpus.js","sourceRoot":"","sources":["../src/corpus.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,OAAO,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AACrE,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,oBAAoB,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAEpE,iCAAiC;AACjC,MAAM,UAAU,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;AAE9C;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;AAE1D;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;AAExD;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;AAKhG,wDAAwD;AACxD,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC;KAChC,MAAM,CAAC;IACN,6DAA6D;IAC7D,QAAQ,EAAE,eAAe;IACzB,2EAA2E;IAC3E,QAAQ,EAAE,aAAa;IACvB,kDAAkD;IAClD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE;CACtC,CAAC;KACD,IAAI,CAAC,EAAE,EAAE,EAAE,cAAc,EAAE,KAAK,EAAE,cAAc,EAAE,CAAC,CAAC;AAKvD;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC;KAC9B,MAAM,CAAC;IACN,+CAA+C;IAC/C,MAAM,EAAE,UAAU;IAClB,+FAA+F;IAC/F,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC;IACvC,mDAAmD;IACnD,KAAK,EAAE,eAAe;IACtB;;;;;;;OAOG;IACH,SAAS,EAAE,CAAC,CAAC,OAAO,EAAE;IACtB,gEAAgE;IAChE,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC;IAC9C;;;;OAIG;IACH,QAAQ,EAAE,aAAa,CAAC,QAAQ,EAAE;IAClC;;;;;;;;OAQG;IACH,gBAAgB,EAAE,UAAU,CAAC,QAAQ,EAAE;IACvC,6FAA6F;IAC7F,UAAU,EAAE,cAAc;IAC1B,mCAAmC;IACnC,UAAU,EAAE,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;CAC7C,CAAC;KACD,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,SAAS,KAAK,CAAC,KAAK,CAAC,QAAQ,KAAK,SAAS,CAAC,EAAE;IACrE,OAAO,EACL,oIAAoI;IACtI,IAAI,EAAE,CAAC,UAAU,CAAC;CACnB,CAAC;KACD,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,SAAS,IAAI,KAAK,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE;IACjE,OAAO,EAAE,8CAA8C;IACvD,IAAI,EAAE,CAAC,UAAU,CAAC;CACnB,CAAC;KACD,IAAI,CAAC;IACJ,EAAE,EAAE,YAAY;IAChB,KAAK,EAAE,YAAY;IACnB,WAAW,EACT,2FAA2F;QAC3F,4FAA4F;QAC5F,sFAAsF;QACtF,0FAA0F;QAC1F,wFAAwF;QACxF,8BAA8B;CACjC,CAAC,CAAC;AAKL,gDAAgD;AAChD,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC;KAChC,MAAM,CAAC;IACN,gDAAgD;IAChD,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,8BAA8B,CAAC;IAC/D,+BAA+B;IAC/B,QAAQ,EAAE,UAAU;IACpB,8BAA8B;IAC9B,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE;IAC5C,0BAA0B;IAC1B,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC;CAC9C,CAAC;KACD,IAAI,CAAC;IACJ,EAAE,EAAE,cAAc;IAClB,KAAK,EAAE,cAAc;IACrB,WAAW,EACT,yFAAyF;QACzF,uFAAuF;QACvF,mCAAmC;CACtC,CAAC,CAAC;AAKL,yCAAyC;AACzC,MAAM,CAAC,MAAM,sBAAsB,GAAG,CAAC;KACpC,MAAM,CAAC;IACN,6FAA6F;IAC7F,QAAQ,EAAE,eAAe;IACzB,6DAA6D;IAC7D,QAAQ,EAAE,aAAa,CAAC,QAAQ,EAAE;IAClC,8DAA8D;IAC9D,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;CAChD,CAAC;KACD,IAAI,CAAC,EAAE,EAAE,EAAE,kBAAkB,EAAE,KAAK,EAAE,kBAAkB,EAAE,CAAC,CAAC;AAK/D,6CAA6C;AAC7C,MAAM,CAAC,MAAM,sBAAsB,GAAG,CAAC;KACpC,MAAM,CAAC;IACN,oCAAoC;IACpC,MAAM,EAAE,UAAU;IAClB,iDAAiD;IACjD,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE;IACpB;;;;;;;;OAQG;IACH,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC;IAClD;;;;;;;;;OASG;IACH,qBAAqB,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;CAC9C,CAAC;KACD,IAAI,CAAC,EAAE,EAAE,EAAE,kBAAkB,EAAE,KAAK,EAAE,kBAAkB,EAAE,CAAC,CAAC;AAK/D;;;;;GAKG;AACH,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC;KAChC,MAAM,CAAC;IACN,gDAAgD;IAChD,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,8BAA8B,CAAC;IAC/D,+CAA+C;IAC/C,WAAW,EAAE,UAAU;IACvB,oFAAoF;IACpF,gBAAgB,EAAE,UAAU;IAC5B,uCAAuC;IACvC,QAAQ,EAAE,UAAU;IACpB,oBAAoB;IACpB,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC;IACtD,iCAAiC;IACjC,UAAU,EAAE,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;CAC7C,CAAC;KACD,IAAI,CAAC;IACJ,EAAE,EAAE,cAAc;IAClB,KAAK,EAAE,cAAc;IACrB,WAAW,EACT,0FAA0F;QAC1F,mFAAmF;QACnF,wEAAwE;CAC3E,CAAC,CAAC;AAKL,mEAAmE;AACnE,MAAM,CAAC,MAAM,yBAAyB,GAAG,cAAc,CAAC;AAExD;;;;GAIG;AACH,MAAM,UAAU,iBAAiB,CAAC,KAAc;IAC9C,MAAM,MAAM,GAAG,kBAAkB,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;IACnD,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,oFAAoF;QACpF,MAAM,eAAe,CACnB,oBAAoB,CAAC,gBAAgB,EACrC,sCAAsC,EACtC;YACE,QAAQ,EAAE,YAAY;YACtB,OAAO,EAAE;gBACP,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;oBAC1C,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC;oBAC1B,IAAI,EAAE,KAAK,CAAC,IAAI;iBACjB,CAAC,CAAC;aACJ;SACF,CACF,CAAC;IACJ,CAAC;IAED,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;QACtC,IAAI,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC;YAC3B,MAAM,eAAe,CACnB,oBAAoB,CAAC,qBAAqB,EAC1C,WAAW,MAAM,CAAC,IAAI,CAAC,QAAQ,0CAA0C,KAAK,CAAC,MAAM,KAAK;gBACxF,uFAAuF,EACzF,EAAE,QAAQ,EAAE,YAAY,EAAE,OAAO,EAAE,EAAE,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,CACxE,CAAC;QACJ,CAAC;QACD,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IACzB,CAAC;IACD,OAAO,MAAM,CAAC,IAAI,CAAC;AACrB,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,iBAAiB,CAAC,KAAc;IAC9C,MAAM,MAAM,GAAG,kBAAkB,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;IACnD,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,MAAM,eAAe,CACnB,oBAAoB,CAAC,gBAAgB,EACrC,sCAAsC,EACtC;YACE,QAAQ,EAAE,YAAY;YACtB,OAAO,EAAE;gBACP,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;oBAC1C,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC;oBAC1B,IAAI,EAAE,KAAK,CAAC,IAAI;iBACjB,CAAC,CAAC;aACJ;SACF,CACF,CAAC;IACJ,CAAC;IAED,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,KAAK,MAAM,OAAO,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;QAC3C,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;YAC7B,MAAM,eAAe,CACnB,oBAAoB,CAAC,iBAAiB,EACtC,sBAAsB,MAAM,CAAC,IAAI,CAAC,WAAW,4BAA4B,OAAO,CAAC,MAAM,IAAI,EAC3F,EAAE,QAAQ,EAAE,YAAY,EAAE,OAAO,EAAE,EAAE,WAAW,EAAE,MAAM,CAAC,IAAI,CAAC,WAAW,EAAE,EAAE,CAC9E,CAAC;QACJ,CAAC;QACD,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IAC3B,CAAC;IACD,OAAO,MAAM,CAAC,IAAI,CAAC;AACrB,CAAC"}
|
package/dist/metrics.d.ts
CHANGED
|
@@ -44,6 +44,19 @@ export interface CaseComparison {
|
|
|
44
44
|
* negative — it did catch the case — but a promotion reader should see it.
|
|
45
45
|
*/
|
|
46
46
|
categoryMismatch: boolean;
|
|
47
|
+
/**
|
|
48
|
+
* Whether site-level comparison means anything for this case (#140).
|
|
49
|
+
*
|
|
50
|
+
* `false` when the evaluator flagged without enumerating what it found — a wrapped legacy
|
|
51
|
+
* evaluator reporting that it had something to say, and nothing about how much or where.
|
|
52
|
+
*
|
|
53
|
+
* The case-level verdict is still sound: a flag is a flag, and §12's precision and recall over
|
|
54
|
+
* *cases* are computable. What is not computable is anything per site, and the previous code
|
|
55
|
+
* quietly answered it anyway — with no evaluator categories to compare, `categoryMismatch` came
|
|
56
|
+
* out `false`, which reads as *the categories agreed*. An evaluator nobody measured would have
|
|
57
|
+
* scored a clean sheet.
|
|
58
|
+
*/
|
|
59
|
+
siteMetricsMeasurable: boolean;
|
|
47
60
|
/** Harm a spurious flag would cause here; zero unless this is a false positive. */
|
|
48
61
|
unnecessaryCorrectionHarm: number;
|
|
49
62
|
}
|
|
@@ -68,6 +81,14 @@ export interface SliceMetrics {
|
|
|
68
81
|
trueNegatives: number;
|
|
69
82
|
/** True positives found under the wrong category (contract §12.3). */
|
|
70
83
|
categoryMismatches: number;
|
|
84
|
+
/**
|
|
85
|
+
* Cases whose site-level metrics are unmeasurable (#140).
|
|
86
|
+
*
|
|
87
|
+
* Reported rather than folded in, because a slice where this is large is a slice whose
|
|
88
|
+
* category agreement means much less than the other numbers suggest. Zero is the honest value
|
|
89
|
+
* only when it is true.
|
|
90
|
+
*/
|
|
91
|
+
unmeasurableSiteMetrics: number;
|
|
71
92
|
/** `truePositives / defectiveCases`; `undefined` when there are no defective cases. */
|
|
72
93
|
recall: number | undefined;
|
|
73
94
|
/** `falsePositives / cleanCases`; `undefined` when there are no clean cases. */
|
package/dist/metrics.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"metrics.d.ts","sourceRoot":"","sources":["../src/metrics.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,KAAK,EAAc,YAAY,EAAoB,YAAY,EAAE,MAAM,aAAa,CAAC;AAE5F,OAAO,EAAkC,KAAK,eAAe,EAAE,MAAM,aAAa,CAAC;AACnF,OAAO,EAKL,KAAK,aAAa,EACnB,MAAM,YAAY,CAAC;AAEpB,wDAAwD;AACxD,MAAM,MAAM,WAAW;AACrB,kDAAkD;AAChD,cAAc;AAChB,+DAA+D;GAC7D,eAAe;AACjB,+DAA+D;GAC7D,eAAe;AACjB,mDAAmD;GACjD,cAAc,CAAC;AAEnB,6DAA6D;AAC7D,MAAM,WAAW,cAAc;IAC7B,gBAAgB;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,yBAAyB;IACzB,KAAK,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IACxC,uBAAuB;IACvB,OAAO,EAAE,WAAW,CAAC;IACrB,gEAAgE;IAChE,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7B,wCAAwC;IACxC,cAAc,EAAE,MAAM,CAAC;IACvB;;;;;;OAMG;IACH,gBAAgB,EAAE,OAAO,CAAC;IAC1B,mFAAmF;IACnF,yBAAyB,EAAE,MAAM,CAAC;CACnC;AAED,4CAA4C;AAC5C,MAAM,WAAW,YAAY;IAC3B,mBAAmB;IACnB,QAAQ,EAAE,aAAa,CAAC;IACxB,gCAAgC;IAChC,GAAG,EAAE,MAAM,CAAC;IACZ,gCAAgC;IAChC,KAAK,EAAE,MAAM,CAAC;IACd,wCAAwC;IACxC,cAAc,EAAE,MAAM,CAAC;IACvB,oCAAoC;IACpC,UAAU,EAAE,MAAM,CAAC;IACnB,4CAA4C;IAC5C,SAAS,EAAE,MAAM,CAAC;IAClB,wBAAwB;IACxB,aAAa,EAAE,MAAM,CAAC;IACtB,cAAc,EAAE,MAAM,CAAC;IACvB,cAAc,EAAE,MAAM,CAAC;IACvB,aAAa,EAAE,MAAM,CAAC;IACtB,sEAAsE;IACtE,kBAAkB,EAAE,MAAM,CAAC;IAC3B,uFAAuF;IACvF,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3B,gFAAgF;IAChF,iBAAiB,EAAE,MAAM,GAAG,SAAS,CAAC;IACtC,6FAA6F;IAC7F,sBAAsB,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3C,gEAAgE;IAChE,8BAA8B,EAAE,MAAM,CAAC;IACvC,8DAA8D;IAC9D,yBAAyB,EAAE,MAAM,CAAC;IAClC,2EAA2E;IAC3E,6BAA6B,EAAE,MAAM,GAAG,SAAS,CAAC;IAClD;;;;;OAKG;IACH,wBAAwB,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7C,gEAAgE;IAChE,OAAO,EAAE,SAAS,MAAM,EAAE,CAAC;CAC5B;AAED,qFAAqF;AACrF,MAAM,WAAW,gBAAgB;IAC/B,0BAA0B;IAC1B,WAAW,EAAE,MAAM,CAAC;IACpB,4EAA4E;IAC5E,gBAAgB,EAAE,MAAM,CAAC;IACzB,+BAA+B;IAC/B,QAAQ,EAAE,MAAM,CAAC;IACjB;;;;;;OAMG;IACH,WAAW,EAAE,YAAY,CAAC;IAC1B,2EAA2E;IAC3E,MAAM,EAAE,SAAS,YAAY,EAAE,CAAC;IAChC,+DAA+D;IAC/D,WAAW,EAAE,SAAS,cAAc,EAAE,CAAC;IACvC,+DAA+D;IAC/D,kBAAkB,EAAE,SAAS,MAAM,EAAE,CAAC;CACvC;
|
|
1
|
+
{"version":3,"file":"metrics.d.ts","sourceRoot":"","sources":["../src/metrics.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,KAAK,EAAc,YAAY,EAAoB,YAAY,EAAE,MAAM,aAAa,CAAC;AAE5F,OAAO,EAAkC,KAAK,eAAe,EAAE,MAAM,aAAa,CAAC;AACnF,OAAO,EAKL,KAAK,aAAa,EACnB,MAAM,YAAY,CAAC;AAEpB,wDAAwD;AACxD,MAAM,MAAM,WAAW;AACrB,kDAAkD;AAChD,cAAc;AAChB,+DAA+D;GAC7D,eAAe;AACjB,+DAA+D;GAC7D,eAAe;AACjB,mDAAmD;GACjD,cAAc,CAAC;AAEnB,6DAA6D;AAC7D,MAAM,WAAW,cAAc;IAC7B,gBAAgB;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,yBAAyB;IACzB,KAAK,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IACxC,uBAAuB;IACvB,OAAO,EAAE,WAAW,CAAC;IACrB,gEAAgE;IAChE,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7B,wCAAwC;IACxC,cAAc,EAAE,MAAM,CAAC;IACvB;;;;;;OAMG;IACH,gBAAgB,EAAE,OAAO,CAAC;IAC1B;;;;;;;;;;;OAWG;IACH,qBAAqB,EAAE,OAAO,CAAC;IAC/B,mFAAmF;IACnF,yBAAyB,EAAE,MAAM,CAAC;CACnC;AAED,4CAA4C;AAC5C,MAAM,WAAW,YAAY;IAC3B,mBAAmB;IACnB,QAAQ,EAAE,aAAa,CAAC;IACxB,gCAAgC;IAChC,GAAG,EAAE,MAAM,CAAC;IACZ,gCAAgC;IAChC,KAAK,EAAE,MAAM,CAAC;IACd,wCAAwC;IACxC,cAAc,EAAE,MAAM,CAAC;IACvB,oCAAoC;IACpC,UAAU,EAAE,MAAM,CAAC;IACnB,4CAA4C;IAC5C,SAAS,EAAE,MAAM,CAAC;IAClB,wBAAwB;IACxB,aAAa,EAAE,MAAM,CAAC;IACtB,cAAc,EAAE,MAAM,CAAC;IACvB,cAAc,EAAE,MAAM,CAAC;IACvB,aAAa,EAAE,MAAM,CAAC;IACtB,sEAAsE;IACtE,kBAAkB,EAAE,MAAM,CAAC;IAC3B;;;;;;OAMG;IACH,uBAAuB,EAAE,MAAM,CAAC;IAChC,uFAAuF;IACvF,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3B,gFAAgF;IAChF,iBAAiB,EAAE,MAAM,GAAG,SAAS,CAAC;IACtC,6FAA6F;IAC7F,sBAAsB,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3C,gEAAgE;IAChE,8BAA8B,EAAE,MAAM,CAAC;IACvC,8DAA8D;IAC9D,yBAAyB,EAAE,MAAM,CAAC;IAClC,2EAA2E;IAC3E,6BAA6B,EAAE,MAAM,GAAG,SAAS,CAAC;IAClD;;;;;OAKG;IACH,wBAAwB,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7C,gEAAgE;IAChE,OAAO,EAAE,SAAS,MAAM,EAAE,CAAC;CAC5B;AAED,qFAAqF;AACrF,MAAM,WAAW,gBAAgB;IAC/B,0BAA0B;IAC1B,WAAW,EAAE,MAAM,CAAC;IACpB,4EAA4E;IAC5E,gBAAgB,EAAE,MAAM,CAAC;IACzB,+BAA+B;IAC/B,QAAQ,EAAE,MAAM,CAAC;IACjB;;;;;;OAMG;IACH,WAAW,EAAE,YAAY,CAAC;IAC1B,2EAA2E;IAC3E,MAAM,EAAE,SAAS,YAAY,EAAE,CAAC;IAChC,+DAA+D;IAC/D,WAAW,EAAE,SAAS,cAAc,EAAE,CAAC;IACvC,+DAA+D;IAC/D,kBAAkB,EAAE,SAAS,MAAM,EAAE,CAAC;CACvC;AAoGD,sCAAsC;AACtC,MAAM,WAAW,cAAc;IAC7B;;;;OAIG;IACH,cAAc,CAAC,EAAE,SAAS,CAAC,SAAS,MAAM,EAAE,CAAC,EAAE,CAAC;CACjD;AAED;;;;;;;GAOG;AACH,wBAAgB,UAAU,CACxB,MAAM,EAAE,YAAY,EACpB,GAAG,EAAE,YAAY,EACjB,MAAM,EAAE,eAAe,EACvB,OAAO,GAAE,cAAmB,GAC3B,gBAAgB,CAkDlB"}
|
package/dist/metrics.js
CHANGED
|
@@ -39,6 +39,10 @@ function compareCase(entry, outcome, policy) {
|
|
|
39
39
|
severity: entry.severity,
|
|
40
40
|
severityWeight: weight,
|
|
41
41
|
categoryMismatch,
|
|
42
|
+
// A flag with nothing enumerated. Not a contradiction to be repaired by inventing a finding —
|
|
43
|
+
// that would make one report count as one defect, which is exactly the statistic #140 exists
|
|
44
|
+
// to protect.
|
|
45
|
+
siteMetricsMeasurable: outcome?.siteMetricsMeasurable ?? !(flagged && (outcome?.findings ?? []).length === 0),
|
|
42
46
|
unnecessaryCorrectionHarm: harm,
|
|
43
47
|
};
|
|
44
48
|
}
|
|
@@ -72,6 +76,8 @@ function summarise(selector, comparisons, labellers) {
|
|
|
72
76
|
falsePositives,
|
|
73
77
|
trueNegatives,
|
|
74
78
|
categoryMismatches: comparisons.filter((comparison) => comparison.categoryMismatch).length,
|
|
79
|
+
unmeasurableSiteMetrics: comparisons.filter((comparison) => !comparison.siteMetricsMeasurable)
|
|
80
|
+
.length,
|
|
75
81
|
recall: defectiveCases === 0 ? undefined : truePositives / defectiveCases,
|
|
76
82
|
falsePositiveRate: cleanCases === 0 ? undefined : falsePositives / cleanCases,
|
|
77
83
|
severityWeightedRecall: defectiveWeight === 0 ? undefined : caughtWeight / defectiveWeight,
|
package/dist/metrics.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"metrics.js","sourceRoot":"","sources":["../src/metrics.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAGH,OAAO,EAAE,oBAAoB,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AACpE,OAAO,EAAE,cAAc,EAAE,cAAc,EAAwB,MAAM,aAAa,CAAC;AACnF,OAAO,EACL,oBAAoB,EACpB,QAAQ,EACR,YAAY,EACZ,kBAAkB,GAEnB,MAAM,YAAY,CAAC;
|
|
1
|
+
{"version":3,"file":"metrics.js","sourceRoot":"","sources":["../src/metrics.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAGH,OAAO,EAAE,oBAAoB,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AACpE,OAAO,EAAE,cAAc,EAAE,cAAc,EAAwB,MAAM,aAAa,CAAC;AACnF,OAAO,EACL,oBAAoB,EACpB,QAAQ,EACR,YAAY,EACZ,kBAAkB,GAEnB,MAAM,YAAY,CAAC;AA8HpB,uCAAuC;AACvC,SAAS,WAAW,CAClB,KAAiB,EACjB,OAAqC,EACrC,MAAuB;IAEvB,MAAM,OAAO,GAAG,OAAO,EAAE,OAAO,IAAI,KAAK,CAAC;IAC1C,MAAM,MAAM,GAAG,KAAK,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,MAAM,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;IAEzF,IAAI,OAAoB,CAAC;IACzB,IAAI,KAAK,CAAC,SAAS;QAAE,OAAO,GAAG,OAAO,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,eAAe,CAAC;;QACrE,OAAO,GAAG,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,cAAc,CAAC;IAE1D,MAAM,eAAe,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;IACnF,MAAM,mBAAmB,GAAG,IAAI,GAAG,CAAC,CAAC,OAAO,EAAE,QAAQ,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;IAClG,MAAM,gBAAgB,GACpB,OAAO,KAAK,cAAc;QAC1B,eAAe,CAAC,IAAI,GAAG,CAAC;QACxB,mBAAmB,CAAC,IAAI,GAAG,CAAC;QAC5B,CAAC,CAAC,GAAG,mBAAmB,CAAC,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,eAAe,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC;IAE9E,MAAM,IAAI,GACR,OAAO,KAAK,eAAe;QACzB,CAAC,CAAC,cAAc,CAAC,MAAM,EAAE,KAAK,CAAC,gBAAgB,IAAI,MAAM,CAAC,sBAAsB,CAAC;QACjF,CAAC,CAAC,CAAC,CAAC;IAER,OAAO;QACL,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,OAAO;QACP,QAAQ,EAAE,KAAK,CAAC,QAAQ;QACxB,cAAc,EAAE,MAAM;QACtB,gBAAgB;QAChB,8FAA8F;QAC9F,6FAA6F;QAC7F,cAAc;QACd,qBAAqB,EACnB,OAAO,EAAE,qBAAqB,IAAI,CAAC,CAAC,OAAO,IAAI,CAAC,OAAO,EAAE,QAAQ,IAAI,EAAE,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC;QACxF,yBAAyB,EAAE,IAAI;KAChC,CAAC;AACJ,CAAC;AAED,wDAAwD;AACxD,SAAS,SAAS,CAChB,QAAuB,EACvB,WAAsC,EACtC,SAA8B;IAE9B,MAAM,KAAK,GAAG,CAAC,OAAoB,EAAU,EAAE,CAC7C,WAAW,CAAC,MAAM,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,KAAK,OAAO,CAAC,CAAC,MAAM,CAAC;IAE5E,MAAM,aAAa,GAAG,KAAK,CAAC,cAAc,CAAC,CAAC;IAC5C,MAAM,cAAc,GAAG,KAAK,CAAC,eAAe,CAAC,CAAC;IAC9C,MAAM,cAAc,GAAG,KAAK,CAAC,eAAe,CAAC,CAAC;IAC9C,MAAM,aAAa,GAAG,KAAK,CAAC,cAAc,CAAC,CAAC;IAE5C,MAAM,cAAc,GAAG,aAAa,GAAG,cAAc,CAAC;IACtD,MAAM,UAAU,GAAG,cAAc,GAAG,aAAa,CAAC;IAClD,MAAM,KAAK,GAAG,WAAW,CAAC,MAAM,CAAC;IAEjC,MAAM,YAAY,GAAG,WAAW;SAC7B,MAAM,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,KAAK,cAAc,CAAC;SAC7D,MAAM,CAAC,CAAC,KAAK,EAAE,UAAU,EAAE,EAAE,CAAC,KAAK,GAAG,UAAU,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC;IACvE,MAAM,YAAY,GAAG,WAAW;SAC7B,MAAM,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,KAAK,eAAe,CAAC;SAC9D,MAAM,CAAC,CAAC,KAAK,EAAE,UAAU,EAAE,EAAE,CAAC,KAAK,GAAG,UAAU,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC;IACvE,MAAM,eAAe,GAAG,YAAY,GAAG,YAAY,CAAC;IAEpD,MAAM,IAAI,GAAG,WAAW,CAAC,MAAM,CAC7B,CAAC,KAAK,EAAE,UAAU,EAAE,EAAE,CAAC,KAAK,GAAG,UAAU,CAAC,yBAAyB,EACnE,CAAC,CACF,CAAC;IAEF,OAAO;QACL,QAAQ;QACR,GAAG,EAAE,QAAQ,CAAC,QAAQ,CAAC;QACvB,KAAK;QACL,cAAc;QACd,UAAU;QACV,SAAS,EAAE,SAAS,CAAC,IAAI;QACzB,aAAa;QACb,cAAc;QACd,cAAc;QACd,aAAa;QACb,kBAAkB,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,UAAU,CAAC,gBAAgB,CAAC,CAAC,MAAM;QAC1F,uBAAuB,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,qBAAqB,CAAC;aAC3F,MAAM;QACT,MAAM,EAAE,cAAc,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,aAAa,GAAG,cAAc;QACzE,iBAAiB,EAAE,UAAU,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,cAAc,GAAG,UAAU;QAC7E,sBAAsB,EAAE,eAAe,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,YAAY,GAAG,eAAe;QAC1F,8BAA8B,EAAE,YAAY;QAC5C,yBAAyB,EAAE,IAAI;QAC/B,6BAA6B,EAAE,UAAU,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,GAAG,UAAU;QAC/E,wBAAwB,EAAE,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,aAAa,GAAG,aAAa,CAAC,GAAG,KAAK;QAC3F,OAAO,EAAE,WAAW,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC;KAC5D,CAAC;AACJ,CAAC;AAYD;;;;;;;GAOG;AACH,MAAM,UAAU,UAAU,CACxB,MAAoB,EACpB,GAAiB,EACjB,MAAuB,EACvB,OAAO,GAAmB,EAAE;IAE5B,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC;IAC7E,KAAK,MAAM,OAAO,IAAI,GAAG,CAAC,QAAQ,EAAE,CAAC;QACnC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;YAClC,MAAM,eAAe,CACnB,oBAAoB,CAAC,oBAAoB,EACzC,sCAAsC,OAAO,CAAC,MAAM,kBAAkB;gBACpE,IAAI,MAAM,CAAC,QAAQ,qBAAqB,EAC1C;gBACE,QAAQ,EAAE,YAAY;gBACtB,OAAO,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE;aAC/D,CACF,CAAC;QACJ,CAAC;IACH,CAAC;IAED,MAAM,eAAe,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC;IAC1F,MAAM,WAAW,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAC7C,WAAW,CAAC,KAAK,EAAE,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,CAC9D,CAAC;IAEF,MAAM,YAAY,GAAG,CAAC,MAA6B,EAAe,EAAE,CAClE,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC,CAAC;IAEtD,MAAM,SAAS,GAAG,oBAAoB,CACpC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,EACxC,OAAO,CAAC,cAAc,CACvB,CAAC;IAEF,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE;QACxC,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,YAAY,CAAC,KAAK,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAC;QACrF,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;QACnE,OAAO,SAAS,CACd,QAAQ,EACR,WAAW,CAAC,MAAM,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,WAAW,CAAC,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,EACtE,YAAY,CAAC,QAAQ,CAAC,CACvB,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,OAAO;QACL,WAAW,EAAE,GAAG,CAAC,WAAW;QAC5B,gBAAgB,EAAE,GAAG,CAAC,gBAAgB;QACtC,QAAQ,EAAE,MAAM,CAAC,QAAQ;QACzB,WAAW,EAAE,SAAS,CAAC,kBAAkB,EAAE,WAAW,EAAE,YAAY,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACnF,MAAM;QACN,WAAW;QACX,kBAAkB,EAAE,MAAM,CAAC,KAAK;aAC7B,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;aACrD,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC;KAChC,CAAC;AACJ,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@aldus-runtime/regression",
|
|
3
|
-
"version": "0.2.0-next.
|
|
3
|
+
"version": "0.2.0-next.19",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "Aldus Regression Harness \u2014 defect corpora, scope-aware evaluator metrics, blind-spot registry, and evaluator promotion evidence.",
|
|
6
6
|
"license": "Apache-2.0",
|
|
@@ -33,11 +33,11 @@
|
|
|
33
33
|
"typecheck:test": "tsc -p tsconfig.test.json"
|
|
34
34
|
},
|
|
35
35
|
"dependencies": {
|
|
36
|
-
"@aldus-runtime/core": "0.2.0-next.
|
|
36
|
+
"@aldus-runtime/core": "0.2.0-next.19",
|
|
37
37
|
"zod": "^4.4.3"
|
|
38
38
|
},
|
|
39
39
|
"devDependencies": {
|
|
40
|
-
"@aldus-runtime/testkit": "0.2.0-next.
|
|
40
|
+
"@aldus-runtime/testkit": "0.2.0-next.19",
|
|
41
41
|
"@types/node": "^26.2.0",
|
|
42
42
|
"typescript": "^7.0.2",
|
|
43
43
|
"vitest": "^4.1.10"
|
package/src/corpus.ts
CHANGED
|
@@ -190,8 +190,27 @@ export const evaluatorOutcomeSchema = z
|
|
|
190
190
|
caseId: identifier,
|
|
191
191
|
/** Whether the evaluator flagged the subject. */
|
|
192
192
|
flagged: z.boolean(),
|
|
193
|
-
/**
|
|
193
|
+
/**
|
|
194
|
+
* The defect occurrences it enumerated. Empty when not flagged, **and also empty when the
|
|
195
|
+
* evaluator flagged without enumerating** (#140).
|
|
196
|
+
*
|
|
197
|
+
* An empty list under `flagged: true` is therefore not a contradiction and must not be
|
|
198
|
+
* repaired by inventing an entry. A wrapped legacy evaluator reports that it had something to
|
|
199
|
+
* say and not how much; fabricating one finding would make its report count as one defect,
|
|
200
|
+
* which is the statistic this distinction exists to protect.
|
|
201
|
+
*/
|
|
194
202
|
findings: z.array(evaluatorFindingSchema).max(256),
|
|
203
|
+
/**
|
|
204
|
+
* Whether site-level metrics are computable for this case (#140).
|
|
205
|
+
*
|
|
206
|
+
* `false` when the evaluator flagged without enumerating, or when a report could not be mapped
|
|
207
|
+
* to this case's subject scope at all. Site-level precision and recall are then **unmeasurable
|
|
208
|
+
* rather than zero**, and a metric that silently treated an unenumerated flag as zero findings
|
|
209
|
+
* would report perfect precision for an evaluator nobody measured.
|
|
210
|
+
*
|
|
211
|
+
* Absent means measurable, which is what every record written before this field meant.
|
|
212
|
+
*/
|
|
213
|
+
siteMetricsMeasurable: z.boolean().optional(),
|
|
195
214
|
})
|
|
196
215
|
.meta({ id: "EvaluatorOutcome", title: "EvaluatorOutcome" });
|
|
197
216
|
|
package/src/metrics.ts
CHANGED
|
@@ -54,6 +54,19 @@ export interface CaseComparison {
|
|
|
54
54
|
* negative — it did catch the case — but a promotion reader should see it.
|
|
55
55
|
*/
|
|
56
56
|
categoryMismatch: boolean;
|
|
57
|
+
/**
|
|
58
|
+
* Whether site-level comparison means anything for this case (#140).
|
|
59
|
+
*
|
|
60
|
+
* `false` when the evaluator flagged without enumerating what it found — a wrapped legacy
|
|
61
|
+
* evaluator reporting that it had something to say, and nothing about how much or where.
|
|
62
|
+
*
|
|
63
|
+
* The case-level verdict is still sound: a flag is a flag, and §12's precision and recall over
|
|
64
|
+
* *cases* are computable. What is not computable is anything per site, and the previous code
|
|
65
|
+
* quietly answered it anyway — with no evaluator categories to compare, `categoryMismatch` came
|
|
66
|
+
* out `false`, which reads as *the categories agreed*. An evaluator nobody measured would have
|
|
67
|
+
* scored a clean sheet.
|
|
68
|
+
*/
|
|
69
|
+
siteMetricsMeasurable: boolean;
|
|
57
70
|
/** Harm a spurious flag would cause here; zero unless this is a false positive. */
|
|
58
71
|
unnecessaryCorrectionHarm: number;
|
|
59
72
|
}
|
|
@@ -79,6 +92,14 @@ export interface SliceMetrics {
|
|
|
79
92
|
trueNegatives: number;
|
|
80
93
|
/** True positives found under the wrong category (contract §12.3). */
|
|
81
94
|
categoryMismatches: number;
|
|
95
|
+
/**
|
|
96
|
+
* Cases whose site-level metrics are unmeasurable (#140).
|
|
97
|
+
*
|
|
98
|
+
* Reported rather than folded in, because a slice where this is large is a slice whose
|
|
99
|
+
* category agreement means much less than the other numbers suggest. Zero is the honest value
|
|
100
|
+
* only when it is true.
|
|
101
|
+
*/
|
|
102
|
+
unmeasurableSiteMetrics: number;
|
|
82
103
|
/** `truePositives / defectiveCases`; `undefined` when there are no defective cases. */
|
|
83
104
|
recall: number | undefined;
|
|
84
105
|
/** `falsePositives / cleanCases`; `undefined` when there are no clean cases. */
|
|
@@ -159,6 +180,11 @@ function compareCase(
|
|
|
159
180
|
severity: entry.severity,
|
|
160
181
|
severityWeight: weight,
|
|
161
182
|
categoryMismatch,
|
|
183
|
+
// A flag with nothing enumerated. Not a contradiction to be repaired by inventing a finding —
|
|
184
|
+
// that would make one report count as one defect, which is exactly the statistic #140 exists
|
|
185
|
+
// to protect.
|
|
186
|
+
siteMetricsMeasurable:
|
|
187
|
+
outcome?.siteMetricsMeasurable ?? !(flagged && (outcome?.findings ?? []).length === 0),
|
|
162
188
|
unnecessaryCorrectionHarm: harm,
|
|
163
189
|
};
|
|
164
190
|
}
|
|
@@ -206,6 +232,8 @@ function summarise(
|
|
|
206
232
|
falsePositives,
|
|
207
233
|
trueNegatives,
|
|
208
234
|
categoryMismatches: comparisons.filter((comparison) => comparison.categoryMismatch).length,
|
|
235
|
+
unmeasurableSiteMetrics: comparisons.filter((comparison) => !comparison.siteMetricsMeasurable)
|
|
236
|
+
.length,
|
|
209
237
|
recall: defectiveCases === 0 ? undefined : truePositives / defectiveCases,
|
|
210
238
|
falsePositiveRate: cleanCases === 0 ? undefined : falsePositives / cleanCases,
|
|
211
239
|
severityWeightedRecall: defectiveWeight === 0 ? undefined : caughtWeight / defectiveWeight,
|