@aldus-runtime/regression 0.2.0-next.6 → 0.2.0-next.60
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/corpus.d.ts +27 -4
- package/dist/corpus.d.ts.map +1 -1
- package/dist/corpus.js +145 -6
- package/dist/corpus.js.map +1 -1
- package/dist/errors.d.ts +10 -0
- package/dist/errors.d.ts.map +1 -1
- package/dist/errors.js +10 -0
- package/dist/errors.js.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/metrics.d.ts +21 -0
- package/dist/metrics.d.ts.map +1 -1
- package/dist/metrics.js +6 -0
- package/dist/metrics.js.map +1 -1
- package/package.json +3 -3
- package/src/corpus.ts +157 -6
- package/src/errors.ts +10 -0
- package/src/index.ts +2 -0
- package/src/metrics.ts +28 -0
package/dist/corpus.d.ts
CHANGED
|
@@ -19,6 +19,27 @@
|
|
|
19
19
|
* Zod is the single source of truth; TypeScript types are inferred (ADR-0002).
|
|
20
20
|
*/
|
|
21
21
|
import { z } from "zod";
|
|
22
|
+
/** Schema version this package stamps on records it constructs. */
|
|
23
|
+
export declare const REGRESSION_SCHEMA_VERSION = "1.15";
|
|
24
|
+
/** How a record's declared schema version relates to the one this runtime implements. */
|
|
25
|
+
export type SchemaVersionRelation = "older" | "same" | "newer";
|
|
26
|
+
/**
|
|
27
|
+
* Compare a record's declared schema version against {@link REGRESSION_SCHEMA_VERSION}.
|
|
28
|
+
*
|
|
29
|
+
* Exported because the two directions are not the same kind of question. **Newer** is soundness
|
|
30
|
+
* and belongs here: this runtime cannot know what a later version's fields mean, so it refuses.
|
|
31
|
+
* **Older** is policy and belongs to the caller — whether a run from an earlier version is
|
|
32
|
+
* comparable to today's depends on what changed between them, which this package cannot know
|
|
33
|
+
* either. What it can do is stop every caller reimplementing the comparison, which is the part
|
|
34
|
+
* that would drift.
|
|
35
|
+
*
|
|
36
|
+
* A record must be readable in order to be upgraded, so this is deliberately a question a caller
|
|
37
|
+
* can ask *before* parsing, and an older record still parses.
|
|
38
|
+
*
|
|
39
|
+
* @param recordVersion - The `schemaVersion` field of a record, `MAJOR.MINOR` (ADR-0003).
|
|
40
|
+
* @throws {AldusError} `ALDUS_CORPUS_MALFORMED` if `recordVersion` is not `MAJOR.MINOR`.
|
|
41
|
+
*/
|
|
42
|
+
export declare function compareSchemaVersion(recordVersion: string): SchemaVersionRelation;
|
|
22
43
|
/**
|
|
23
44
|
* A finding category, structured per contract §12.3.
|
|
24
45
|
*
|
|
@@ -120,7 +141,8 @@ export declare const defectCorpusSchema: z.ZodObject<{
|
|
|
120
141
|
}, z.core.$strip>;
|
|
121
142
|
labelledAt: z.ZodISODateTime;
|
|
122
143
|
}, z.core.$strip>>;
|
|
123
|
-
|
|
144
|
+
metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
145
|
+
}, z.core.$strict>;
|
|
124
146
|
/** @see defectCorpusSchema */
|
|
125
147
|
export type DefectCorpus = z.infer<typeof defectCorpusSchema>;
|
|
126
148
|
/** One finding an evaluator reported. */
|
|
@@ -140,6 +162,7 @@ export declare const evaluatorOutcomeSchema: z.ZodObject<{
|
|
|
140
162
|
severity: z.ZodOptional<z.ZodString>;
|
|
141
163
|
confidence: z.ZodOptional<z.ZodNumber>;
|
|
142
164
|
}, z.core.$strip>>;
|
|
165
|
+
siteMetricsMeasurable: z.ZodOptional<z.ZodBoolean>;
|
|
143
166
|
}, z.core.$strip>;
|
|
144
167
|
/** @see evaluatorOutcomeSchema */
|
|
145
168
|
export type EvaluatorOutcome = z.infer<typeof evaluatorOutcomeSchema>;
|
|
@@ -162,13 +185,13 @@ export declare const evaluatorRunSchema: z.ZodObject<{
|
|
|
162
185
|
severity: z.ZodOptional<z.ZodString>;
|
|
163
186
|
confidence: z.ZodOptional<z.ZodNumber>;
|
|
164
187
|
}, z.core.$strip>>;
|
|
188
|
+
siteMetricsMeasurable: z.ZodOptional<z.ZodBoolean>;
|
|
165
189
|
}, z.core.$strip>>;
|
|
166
190
|
executedAt: z.ZodISODateTime;
|
|
167
|
-
|
|
191
|
+
metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
192
|
+
}, z.core.$strict>;
|
|
168
193
|
/** @see evaluatorRunSchema */
|
|
169
194
|
export type EvaluatorRun = z.infer<typeof evaluatorRunSchema>;
|
|
170
|
-
/** Schema version this package stamps on records it constructs. */
|
|
171
|
-
export declare const REGRESSION_SCHEMA_VERSION = "1.3";
|
|
172
195
|
/**
|
|
173
196
|
* Validate a corpus and check the uniqueness constraint Zod cannot express.
|
|
174
197
|
*
|
package/dist/corpus.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"corpus.d.ts","sourceRoot":"","sources":["../src/corpus.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAGH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;
|
|
1
|
+
{"version":3,"file":"corpus.d.ts","sourceRoot":"","sources":["../src/corpus.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAGH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AASxB,mEAAmE;AACnE,eAAO,MAAM,yBAAyB,SAAiB,CAAC;AAExD,yFAAyF;AACzF,MAAM,MAAM,qBAAqB,GAAG,OAAO,GAAG,MAAM,GAAG,OAAO,CAAC;AAE/D;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,oBAAoB,CAAC,aAAa,EAAE,MAAM,GAAG,qBAAqB,CAqBjF;AAgCD;;;;;;GAMG;AACH,eAAO,MAAM,eAAe,aAA6B,CAAC;AAE1D;;;;;;GAMG;AACH,eAAO,MAAM,aAAa,aAA6B,CAAC;AAExD;;;;;;GAMG;AACH,eAAO,MAAM,eAAe,uCAAmE,CAAC;AAEhG,2BAA2B;AAC3B,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AAE9D,wDAAwD;AACxD,eAAO,MAAM,kBAAkB;;;;iBASuB,CAAC;AAEvD,8BAA8B;AAC9B,MAAM,MAAM,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,kBAAkB,CAAC,CAAC;AAE9D;;;;;;;GAOG;AACH,eAAO,MAAM,gBAAgB;;;;;;;;;;;;;;;;;;;;;;;;;iBA2DzB,CAAC;AAEL,4BAA4B;AAC5B,MAAM,MAAM,UAAU,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,gBAAgB,CAAC,CAAC;AAE1D,gDAAgD;AAChD,eAAO,MAAM,kBAAkB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBA0C3B,CAAC;AAEL,8BAA8B;AAC9B,MAAM,MAAM,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,kBAAkB,CAAC,CAAC;AAE9D,yCAAyC;AACzC,eAAO,MAAM,sBAAsB;;;;iBAS2B,CAAC;AAE/D,kCAAkC;AAClC,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,sBAAsB,CAAC,CAAC;AAEtE,6CAA6C;AAC7C,eAAO,MAAM,sBAAsB;;;;;;;;;iBA4B2B,CAAC;AAE/D,kCAAkC;AAClC,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,sBAAsB,CAAC,CAAC;AAEtE;;;;;GAKG;AACH,eAAO,MAAM,kBAAkB;;;;;;;;;;;;;;;;;kBAyB3B,CAAC;AAEL,8BAA8B;AAC9B,MAAM,MAAM,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,kBAAkB,CAAC,CAAC;AA+B9D;;;;GAIG;AACH,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,OAAO,GAAG,YAAY,CAiC9D;AAED;;;;GAIG;AACH,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,OAAO,GAAG,YAAY,CA+B9D"}
|
package/dist/corpus.js
CHANGED
|
@@ -18,11 +18,78 @@
|
|
|
18
18
|
*
|
|
19
19
|
* Zod is the single source of truth; TypeScript types are inferred (ADR-0002).
|
|
20
20
|
*/
|
|
21
|
-
import { actorRefSchema, SCHEMA_VERSION } from "@aldus-runtime/core";
|
|
21
|
+
import { actorRefSchema, schemaVersionString, SCHEMA_VERSION } from "@aldus-runtime/core";
|
|
22
22
|
import { z } from "zod";
|
|
23
23
|
import { RegressionErrorCodes, regressionError } from "./errors.js";
|
|
24
24
|
/** A short opaque identifier. */
|
|
25
25
|
const identifier = z.string().min(1).max(200);
|
|
26
|
+
// --- Schema version (ADR-0003, ADR-0053) -----------------------------------------------------
|
|
27
|
+
/** Schema version this package stamps on records it constructs. */
|
|
28
|
+
export const REGRESSION_SCHEMA_VERSION = SCHEMA_VERSION;
|
|
29
|
+
/**
|
|
30
|
+
* Compare a record's declared schema version against {@link REGRESSION_SCHEMA_VERSION}.
|
|
31
|
+
*
|
|
32
|
+
* Exported because the two directions are not the same kind of question. **Newer** is soundness
|
|
33
|
+
* and belongs here: this runtime cannot know what a later version's fields mean, so it refuses.
|
|
34
|
+
* **Older** is policy and belongs to the caller — whether a run from an earlier version is
|
|
35
|
+
* comparable to today's depends on what changed between them, which this package cannot know
|
|
36
|
+
* either. What it can do is stop every caller reimplementing the comparison, which is the part
|
|
37
|
+
* that would drift.
|
|
38
|
+
*
|
|
39
|
+
* A record must be readable in order to be upgraded, so this is deliberately a question a caller
|
|
40
|
+
* can ask *before* parsing, and an older record still parses.
|
|
41
|
+
*
|
|
42
|
+
* @param recordVersion - The `schemaVersion` field of a record, `MAJOR.MINOR` (ADR-0003).
|
|
43
|
+
* @throws {AldusError} `ALDUS_CORPUS_MALFORMED` if `recordVersion` is not `MAJOR.MINOR`.
|
|
44
|
+
*/
|
|
45
|
+
export function compareSchemaVersion(recordVersion) {
|
|
46
|
+
const parse = (value) => {
|
|
47
|
+
const match = /^(0|[1-9]\d*)\.(0|[1-9]\d*)$/.exec(value);
|
|
48
|
+
if (match === null) {
|
|
49
|
+
// Path and code only — never the received value (contract §19.2).
|
|
50
|
+
throw regressionError(RegressionErrorCodes.CORPUS_MALFORMED, "A schema version must be MAJOR.MINOR.", {
|
|
51
|
+
category: "validation",
|
|
52
|
+
details: { issues: [{ path: "schemaVersion", code: "invalid_format" }] },
|
|
53
|
+
});
|
|
54
|
+
}
|
|
55
|
+
return [Number(match[1]), Number(match[2])];
|
|
56
|
+
};
|
|
57
|
+
const [recordMajor, recordMinor] = parse(recordVersion);
|
|
58
|
+
const [runtimeMajor, runtimeMinor] = parse(REGRESSION_SCHEMA_VERSION);
|
|
59
|
+
if (recordMajor !== runtimeMajor)
|
|
60
|
+
return recordMajor > runtimeMajor ? "newer" : "older";
|
|
61
|
+
if (recordMinor !== runtimeMinor)
|
|
62
|
+
return recordMinor > runtimeMinor ? "newer" : "older";
|
|
63
|
+
return "same";
|
|
64
|
+
}
|
|
65
|
+
/**
|
|
66
|
+
* True when `value` is a well-formed version this runtime does not implement.
|
|
67
|
+
*
|
|
68
|
+
* A malformed value answers `false` rather than throwing, so the format check that owns that
|
|
69
|
+
* failure reports it under its own code and the version rule fires only on a version.
|
|
70
|
+
*/
|
|
71
|
+
function declaresNewerVersion(value) {
|
|
72
|
+
return /^(0|[1-9]\d*)\.(0|[1-9]\d*)$/.test(value) && compareSchemaVersion(value) === "newer";
|
|
73
|
+
}
|
|
74
|
+
/**
|
|
75
|
+
* The `schemaVersion` field of a record this package reads.
|
|
76
|
+
*
|
|
77
|
+
* **The exported schema carries the rule** (ADR-0053). `parseDefectCorpus` and `parseEvaluatorRun`
|
|
78
|
+
* refused a newer record while `defectCorpusSchema.safeParse` and `evaluatorRunSchema.safeParse`
|
|
79
|
+
* accepted the same bytes — a rule enforced at one entry point and not the other is a rule plus a
|
|
80
|
+
* bypass, and the exported object is the obvious door (#199). So the refusal lives on the field, and
|
|
81
|
+
* the parse functions check it first only so that a newer record fails with one clear
|
|
82
|
+
* `ALDUS_SCHEMA_VERSION_UNSUPPORTED` rather than a field issue about the version.
|
|
83
|
+
*
|
|
84
|
+
* This is **this package's** rule, not Core's: any newer version is refused, minor included, because
|
|
85
|
+
* these schemas are strict and a newer minor would fail on its added keys anyway. Refusing on the
|
|
86
|
+
* version names the reason. Core's records accept a newer minor and report it as a `forward` read
|
|
87
|
+
* (ADR-0003, ADR-0053).
|
|
88
|
+
*/
|
|
89
|
+
const recordSchemaVersion = schemaVersionString.refine((value) => !declaresNewerVersion(value), {
|
|
90
|
+
// Names this package's own constant, never the received value (contract §19.2).
|
|
91
|
+
message: `Schema version must not be newer than ${REGRESSION_SCHEMA_VERSION}, which is what this runtime implements.`,
|
|
92
|
+
});
|
|
26
93
|
/**
|
|
27
94
|
* A finding category, structured per contract §12.3.
|
|
28
95
|
*
|
|
@@ -128,14 +195,38 @@ export const defectCaseSchema = z
|
|
|
128
195
|
export const defectCorpusSchema = z
|
|
129
196
|
.object({
|
|
130
197
|
/** Schema version of this record (ADR-0003). */
|
|
131
|
-
schemaVersion:
|
|
198
|
+
schemaVersion: recordSchemaVersion,
|
|
132
199
|
/** Identity of this corpus. */
|
|
133
200
|
corpusId: identifier,
|
|
134
201
|
/** Human-readable purpose. */
|
|
135
202
|
description: z.string().max(2000).optional(),
|
|
136
203
|
/** The labelled cases. */
|
|
137
204
|
cases: z.array(defectCaseSchema).max(100_000),
|
|
205
|
+
/**
|
|
206
|
+
* Adopter-owned data carried with this record.
|
|
207
|
+
*
|
|
208
|
+
* A declared extension point, because strictness without one is a rule with no exit. Refusing
|
|
209
|
+
* an undeclared key is right — a record from a later runtime whose added fields vanish hands
|
|
210
|
+
* the caller an object that looks complete and is not. But that argument is about *the
|
|
211
|
+
* runtime's* future fields, and it was applied to an adopter's own data, which is a different
|
|
212
|
+
* category and had a real pattern behind it: the provenance of a label belongs beside the
|
|
213
|
+
* label, in the same file, versioned together, because a label whose source lives in another
|
|
214
|
+
* file is one refactor away from a label with no source.
|
|
215
|
+
*
|
|
216
|
+
* So the two cases are separated rather than collapsed. An unknown key is refused; a declared
|
|
217
|
+
* one is **preserved through the parse**, which the previous sibling-key arrangement could not
|
|
218
|
+
* do — it survived only because readers went around the parser to the raw JSON.
|
|
219
|
+
*
|
|
220
|
+
* Core never interprets this. Same shape as `WorkflowEvent.details` and
|
|
221
|
+
* `SpendReservationTransition.detail` (§4.2: adopter-shaped concepts are opaque here).
|
|
222
|
+
*/
|
|
223
|
+
metadata: z.record(z.string(), z.unknown()).optional(),
|
|
138
224
|
})
|
|
225
|
+
// Strict: an unknown key is refused, not stripped. Without this a record from a later runtime
|
|
226
|
+
// parses and its added fields vanish, handing the caller an object that looks complete and is
|
|
227
|
+
// not. See SCHEMA_VERSION_UNSUPPORTED — the version check and this are the same guarantee, one
|
|
228
|
+
// for a declared newer version and one for an undeclared newer shape.
|
|
229
|
+
.strict()
|
|
139
230
|
.meta({
|
|
140
231
|
id: "DefectCorpus",
|
|
141
232
|
title: "DefectCorpus",
|
|
@@ -161,8 +252,27 @@ export const evaluatorOutcomeSchema = z
|
|
|
161
252
|
caseId: identifier,
|
|
162
253
|
/** Whether the evaluator flagged the subject. */
|
|
163
254
|
flagged: z.boolean(),
|
|
164
|
-
/**
|
|
255
|
+
/**
|
|
256
|
+
* The defect occurrences it enumerated. Empty when not flagged, **and also empty when the
|
|
257
|
+
* evaluator flagged without enumerating** (#140).
|
|
258
|
+
*
|
|
259
|
+
* An empty list under `flagged: true` is therefore not a contradiction and must not be
|
|
260
|
+
* repaired by inventing an entry. A wrapped legacy evaluator reports that it had something to
|
|
261
|
+
* say and not how much; fabricating one finding would make its report count as one defect,
|
|
262
|
+
* which is the statistic this distinction exists to protect.
|
|
263
|
+
*/
|
|
165
264
|
findings: z.array(evaluatorFindingSchema).max(256),
|
|
265
|
+
/**
|
|
266
|
+
* Whether site-level metrics are computable for this case (#140).
|
|
267
|
+
*
|
|
268
|
+
* `false` when the evaluator flagged without enumerating, or when a report could not be mapped
|
|
269
|
+
* to this case's subject scope at all. Site-level precision and recall are then **unmeasurable
|
|
270
|
+
* rather than zero**, and a metric that silently treated an unenumerated flag as zero findings
|
|
271
|
+
* would report perfect precision for an evaluator nobody measured.
|
|
272
|
+
*
|
|
273
|
+
* Absent means measurable, which is what every record written before this field meant.
|
|
274
|
+
*/
|
|
275
|
+
siteMetricsMeasurable: z.boolean().optional(),
|
|
166
276
|
})
|
|
167
277
|
.meta({ id: "EvaluatorOutcome", title: "EvaluatorOutcome" });
|
|
168
278
|
/**
|
|
@@ -174,7 +284,7 @@ export const evaluatorOutcomeSchema = z
|
|
|
174
284
|
export const evaluatorRunSchema = z
|
|
175
285
|
.object({
|
|
176
286
|
/** Schema version of this record (ADR-0003). */
|
|
177
|
-
schemaVersion:
|
|
287
|
+
schemaVersion: recordSchemaVersion,
|
|
178
288
|
/** Which evaluator produced these outcomes. */
|
|
179
289
|
evaluatorId: identifier,
|
|
180
290
|
/** Version of the evaluator. Metrics from different versions are not comparable. */
|
|
@@ -185,7 +295,10 @@ export const evaluatorRunSchema = z
|
|
|
185
295
|
outcomes: z.array(evaluatorOutcomeSchema).max(100_000),
|
|
186
296
|
/** When the run was executed. */
|
|
187
297
|
executedAt: z.iso.datetime({ offset: true }),
|
|
298
|
+
/** @see defectCorpusSchema.metadata */
|
|
299
|
+
metadata: z.record(z.string(), z.unknown()).optional(),
|
|
188
300
|
})
|
|
301
|
+
.strict()
|
|
189
302
|
.meta({
|
|
190
303
|
id: "EvaluatorRun",
|
|
191
304
|
title: "EvaluatorRun",
|
|
@@ -193,14 +306,39 @@ export const evaluatorRunSchema = z
|
|
|
193
306
|
"package consumes runs; it does not execute evaluators. ADDITIONAL CONSTRAINT NOT " +
|
|
194
307
|
"EXPRESSIBLE IN JSON SCHEMA: `caseId` must be unique within `outcomes`.",
|
|
195
308
|
});
|
|
196
|
-
/**
|
|
197
|
-
|
|
309
|
+
/**
|
|
310
|
+
* Refuse a record declaring a version this runtime does not implement, **before** its shape is
|
|
311
|
+
* read.
|
|
312
|
+
*
|
|
313
|
+
* Only `newer` is refused. An older record parses, and {@link compareSchemaVersion} is how a
|
|
314
|
+
* caller decides whether to trust it.
|
|
315
|
+
*
|
|
316
|
+
* Version before fields, for the same reason Core's `validateRecord` orders them so: a newer record
|
|
317
|
+
* fails with one `ALDUS_SCHEMA_VERSION_UNSUPPORTED` that names the cause, rather than a field issue
|
|
318
|
+
* that describes the symptom. A missing or malformed version is not this check's to report — it
|
|
319
|
+
* falls through, and the schema reports it precisely under `ALDUS_CORPUS_MALFORMED`.
|
|
320
|
+
*/
|
|
321
|
+
function assertSchemaVersionSupported(input, record) {
|
|
322
|
+
if (typeof input !== "object" || input === null)
|
|
323
|
+
return;
|
|
324
|
+
const declared = input.schemaVersion;
|
|
325
|
+
if (typeof declared !== "string" || !declaresNewerVersion(declared))
|
|
326
|
+
return;
|
|
327
|
+
throw regressionError(RegressionErrorCodes.SCHEMA_VERSION_UNSUPPORTED, `A ${record} declares a schema version newer than this runtime implements. Its fields cannot ` +
|
|
328
|
+
"be interpreted here, so accepting it would return a record that looks complete and is not.", {
|
|
329
|
+
category: "validation",
|
|
330
|
+
// The runtime's own version is this package's constant, not received input, so naming it is
|
|
331
|
+
// not a value leak. The record's version is not named (contract §19.2).
|
|
332
|
+
details: { runtimeSchemaVersion: REGRESSION_SCHEMA_VERSION, relation: "newer" },
|
|
333
|
+
});
|
|
334
|
+
}
|
|
198
335
|
/**
|
|
199
336
|
* Validate a corpus and check the uniqueness constraint Zod cannot express.
|
|
200
337
|
*
|
|
201
338
|
* @throws {AldusError} `ALDUS_CORPUS_MALFORMED` or `ALDUS_CORPUS_DUPLICATE_CASE`.
|
|
202
339
|
*/
|
|
203
340
|
export function parseDefectCorpus(input) {
|
|
341
|
+
assertSchemaVersionSupported(input, "corpus");
|
|
204
342
|
const result = defectCorpusSchema.safeParse(input);
|
|
205
343
|
if (!result.success) {
|
|
206
344
|
// Paths and issue codes only — never the received value (contract §19.2, ADR-0002).
|
|
@@ -230,6 +368,7 @@ export function parseDefectCorpus(input) {
|
|
|
230
368
|
* @throws {AldusError} `ALDUS_CORPUS_MALFORMED` or `ALDUS_OUTCOME_DUPLICATE`.
|
|
231
369
|
*/
|
|
232
370
|
export function parseEvaluatorRun(input) {
|
|
371
|
+
assertSchemaVersionSupported(input, "run");
|
|
233
372
|
const result = evaluatorRunSchema.safeParse(input);
|
|
234
373
|
if (!result.success) {
|
|
235
374
|
throw regressionError(RegressionErrorCodes.CORPUS_MALFORMED, "The evaluator run does not validate.", {
|
package/dist/corpus.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"corpus.js","sourceRoot":"","sources":["../src/corpus.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,OAAO,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;
|
|
1
|
+
{"version":3,"file":"corpus.js","sourceRoot":"","sources":["../src/corpus.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,OAAO,EAAE,cAAc,EAAE,mBAAmB,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAC1F,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,oBAAoB,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAEpE,iCAAiC;AACjC,MAAM,UAAU,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;AAE9C,gGAAgG;AAEhG,mEAAmE;AACnE,MAAM,CAAC,MAAM,yBAAyB,GAAG,cAAc,CAAC;AAKxD;;;;;;;;;;;;;;;GAeG;AACH,MAAM,UAAU,oBAAoB,CAAC,aAAqB;IACxD,MAAM,KAAK,GAAG,CAAC,KAAa,EAA6B,EAAE;QACzD,MAAM,KAAK,GAAG,8BAA8B,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACzD,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;YACnB,kEAAkE;YAClE,MAAM,eAAe,CACnB,oBAAoB,CAAC,gBAAgB,EACrC,uCAAuC,EACvC;gBACE,QAAQ,EAAE,YAAY;gBACtB,OAAO,EAAE,EAAE,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,eAAe,EAAE,IAAI,EAAE,gBAAgB,EAAE,CAAC,EAAE;aACzE,CACF,CAAC;QACJ,CAAC;QACD,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAU,CAAC;IACvD,CAAC,CAAC;IACF,MAAM,CAAC,WAAW,EAAE,WAAW,CAAC,GAAG,KAAK,CAAC,aAAa,CAAC,CAAC;IACxD,MAAM,CAAC,YAAY,EAAE,YAAY,CAAC,GAAG,KAAK,CAAC,yBAAyB,CAAC,CAAC;IACtE,IAAI,WAAW,KAAK,YAAY;QAAE,OAAO,WAAW,GAAG,YAAY,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC;IACxF,IAAI,WAAW,KAAK,YAAY;QAAE,OAAO,WAAW,GAAG,YAAY,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC;IACxF,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;;GAKG;AACH,SAAS,oBAAoB,CAAC,KAAa;IACzC,OAAO,8BAA8B,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,oBAAoB,CAAC,KAAK,CAAC,KAAK,OAAO,CAAC;AAC/F,CAAC;AAED;;;;;;;;;;;;;;GAcG;AACH,MAAM,mBAAmB,GAAG,mBAAmB,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,oBAAoB,CAAC,KAAK,CAAC,EAAE;IAC9F,gFAAgF;IAChF,OAAO,EAAE,yCAAyC,yBAAyB,0CAA0C;CACtH,CAAC,CAAC;AAEH;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;AAE1D;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;AAExD;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;AAKhG,wDAAwD;AACxD,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC;KAChC,MAAM,CAAC;IACN,6DAA6D;IAC7D,QAAQ,EAAE,eAAe;IACzB,2EAA2E;IAC3E,QAAQ,EAAE,aAAa;IACvB,kDAAkD;IAClD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE;CACtC,CAAC;KACD,IAAI,CAAC,EAAE,EAAE,EAAE,cAAc,EAAE,KAAK,EAAE,cAAc,EAAE,CAAC,CAAC;AAKvD;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC;KAC9B,MAAM,CAAC;IACN,+CAA+C;IAC/C,MAAM,EAAE,UAAU;IAClB,+FAA+F;IAC/F,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC;IACvC,mDAAmD;IACnD,KAAK,EAAE,eAAe;IACtB;;;;;;;OAOG;IACH,SAAS,EAAE,CAAC,CAAC,OAAO,EAAE;IACtB,gEAAgE;IAChE,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC;IAC9C;;;;OAIG;IACH,QAAQ,EAAE,aAAa,CAAC,QAAQ,EAAE;IAClC;;;;;;;;OAQG;IACH,gBAAgB,EAAE,UAAU,CAAC,QAAQ,EAAE;IACvC,6FAA6F;IAC7F,UAAU,EAAE,cAAc;IAC1B,mCAAmC;IACnC,UAAU,EAAE,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;CAC7C,CAAC;KACD,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,SAAS,KAAK,CAAC,KAAK,CAAC,QAAQ,KAAK,SAAS,CAAC,EAAE;IACrE,OAAO,EACL,oIAAoI;IACtI,IAAI,EAAE,CAAC,UAAU,CAAC;CACnB,CAAC;KACD,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,SAAS,IAAI,KAAK,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE;IACjE,OAAO,EAAE,8CAA8C;IACvD,IAAI,EAAE,CAAC,UAAU,CAAC;CACnB,CAAC;KACD,IAAI,CAAC;IACJ,EAAE,EAAE,YAAY;IAChB,KAAK,EAAE,YAAY;IACnB,WAAW,EACT,2FAA2F;QAC3F,4FAA4F;QAC5F,sFAAsF;QACtF,0FAA0F;QAC1F,wFAAwF;QACxF,8BAA8B;CACjC,CAAC,CAAC;AAKL,gDAAgD;AAChD,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC;KAChC,MAAM,CAAC;IACN,gDAAgD;IAChD,aAAa,EAAE,mBAAmB;IAClC,+BAA+B;IAC/B,QAAQ,EAAE,UAAU;IACpB,8BAA8B;IAC9B,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE;IAC5C,0BAA0B;IAC1B,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC;IAC7C;;;;;;;;;;;;;;;;;OAiBG;IACH,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,EAAE;CACvD,CAAC;IACF,8FAA8F;IAC9F,8FAA8F;IAC9F,+FAA+F;IAC/F,sEAAsE;KACrE,MAAM,EAAE;KACR,IAAI,CAAC;IACJ,EAAE,EAAE,cAAc;IAClB,KAAK,EAAE,cAAc;IACrB,WAAW,EACT,yFAAyF;QACzF,uFAAuF;QACvF,mCAAmC;CACtC,CAAC,CAAC;AAKL,yCAAyC;AACzC,MAAM,CAAC,MAAM,sBAAsB,GAAG,CAAC;KACpC,MAAM,CAAC;IACN,6FAA6F;IAC7F,QAAQ,EAAE,eAAe;IACzB,6DAA6D;IAC7D,QAAQ,EAAE,aAAa,CAAC,QAAQ,EAAE;IAClC,8DAA8D;IAC9D,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;CAChD,CAAC;KACD,IAAI,CAAC,EAAE,EAAE,EAAE,kBAAkB,EAAE,KAAK,EAAE,kBAAkB,EAAE,CAAC,CAAC;AAK/D,6CAA6C;AAC7C,MAAM,CAAC,MAAM,sBAAsB,GAAG,CAAC;KACpC,MAAM,CAAC;IACN,oCAAoC;IACpC,MAAM,EAAE,UAAU;IAClB,iDAAiD;IACjD,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE;IACpB;;;;;;;;OAQG;IACH,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC;IAClD;;;;;;;;;OASG;IACH,qBAAqB,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;CAC9C,CAAC;KACD,IAAI,CAAC,EAAE,EAAE,EAAE,kBAAkB,EAAE,KAAK,EAAE,kBAAkB,EAAE,CAAC,CAAC;AAK/D;;;;;GAKG;AACH,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC;KAChC,MAAM,CAAC;IACN,gDAAgD;IAChD,aAAa,EAAE,mBAAmB;IAClC,+CAA+C;IAC/C,WAAW,EAAE,UAAU;IACvB,oFAAoF;IACpF,gBAAgB,EAAE,UAAU;IAC5B,uCAAuC;IACvC,QAAQ,EAAE,UAAU;IACpB,oBAAoB;IACpB,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC;IACtD,iCAAiC;IACjC,UAAU,EAAE,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;IAC5C,uCAAuC;IACvC,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,EAAE;CACvD,CAAC;KACD,MAAM,EAAE;KACR,IAAI,CAAC;IACJ,EAAE,EAAE,cAAc;IAClB,KAAK,EAAE,cAAc;IACrB,WAAW,EACT,0FAA0F;QAC1F,mFAAmF;QACnF,wEAAwE;CAC3E,CAAC,CAAC;AAKL;;;;;;;;;;;GAWG;AACH,SAAS,4BAA4B,CAAC,KAAc,EAAE,MAAwB;IAC5E,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI;QAAE,OAAO;IACxD,MAAM,QAAQ,GAAI,KAAqC,CAAC,aAAa,CAAC;IACtE,IAAI,OAAO,QAAQ,KAAK,QAAQ,IAAI,CAAC,oBAAoB,CAAC,QAAQ,CAAC;QAAE,OAAO;IAC5E,MAAM,eAAe,CACnB,oBAAoB,CAAC,0BAA0B,EAC/C,KAAK,MAAM,mFAAmF;QAC5F,4FAA4F,EAC9F;QACE,QAAQ,EAAE,YAAY;QACtB,4FAA4F;QAC5F,wEAAwE;QACxE,OAAO,EAAE,EAAE,oBAAoB,EAAE,yBAAyB,EAAE,QAAQ,EAAE,OAAO,EAAE;KAChF,CACF,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,iBAAiB,CAAC,KAAc;IAC9C,4BAA4B,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;IAC9C,MAAM,MAAM,GAAG,kBAAkB,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;IACnD,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,oFAAoF;QACpF,MAAM,eAAe,CACnB,oBAAoB,CAAC,gBAAgB,EACrC,sCAAsC,EACtC;YACE,QAAQ,EAAE,YAAY;YACtB,OAAO,EAAE;gBACP,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;oBAC1C,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC;oBAC1B,IAAI,EAAE,KAAK,CAAC,IAAI;iBACjB,CAAC,CAAC;aACJ;SACF,CACF,CAAC;IACJ,CAAC;IAED,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;QACtC,IAAI,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC;YAC3B,MAAM,eAAe,CACnB,oBAAoB,CAAC,qBAAqB,EAC1C,WAAW,MAAM,CAAC,IAAI,CAAC,QAAQ,0CAA0C,KAAK,CAAC,MAAM,KAAK;gBACxF,uFAAuF,EACzF,EAAE,QAAQ,EAAE,YAAY,EAAE,OAAO,EAAE,EAAE,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,CACxE,CAAC;QACJ,CAAC;QACD,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IACzB,CAAC;IACD,OAAO,MAAM,CAAC,IAAI,CAAC;AACrB,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,iBAAiB,CAAC,KAAc;IAC9C,4BAA4B,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;IAC3C,MAAM,MAAM,GAAG,kBAAkB,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;IACnD,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,MAAM,eAAe,CACnB,oBAAoB,CAAC,gBAAgB,EACrC,sCAAsC,EACtC;YACE,QAAQ,EAAE,YAAY;YACtB,OAAO,EAAE;gBACP,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;oBAC1C,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC;oBAC1B,IAAI,EAAE,KAAK,CAAC,IAAI;iBACjB,CAAC,CAAC;aACJ;SACF,CACF,CAAC;IACJ,CAAC;IAED,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,KAAK,MAAM,OAAO,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;QAC3C,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;YAC7B,MAAM,eAAe,CACnB,oBAAoB,CAAC,iBAAiB,EACtC,sBAAsB,MAAM,CAAC,IAAI,CAAC,WAAW,4BAA4B,OAAO,CAAC,MAAM,IAAI,EAC3F,EAAE,QAAQ,EAAE,YAAY,EAAE,OAAO,EAAE,EAAE,WAAW,EAAE,MAAM,CAAC,IAAI,CAAC,WAAW,EAAE,EAAE,CAC9E,CAAC;QACJ,CAAC;QACD,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IAC3B,CAAC;IACD,OAAO,MAAM,CAAC,IAAI,CAAC;AACrB,CAAC"}
|
package/dist/errors.d.ts
CHANGED
|
@@ -32,6 +32,16 @@ export declare const RegressionErrorCodes: {
|
|
|
32
32
|
readonly BLIND_SPOT_DUPLICATE: "ALDUS_BLIND_SPOT_DUPLICATE";
|
|
33
33
|
/** A promotion policy is internally inconsistent — a threshold outside its valid range. */
|
|
34
34
|
readonly POLICY_INVALID: "ALDUS_POLICY_INVALID";
|
|
35
|
+
/**
|
|
36
|
+
* A record declares a schema version newer than this runtime implements.
|
|
37
|
+
*
|
|
38
|
+
* Refused rather than accepted, because this runtime cannot know what a later version's fields
|
|
39
|
+
* mean and the schemas discard what they do not declare. Accepting one returns an object that
|
|
40
|
+
* looks complete and is not — a read path answering wrongly with no signal, which is worse than
|
|
41
|
+
* refusing. Whether an *older* record is comparable is the caller's policy, not this package's:
|
|
42
|
+
* see `compareSchemaVersion`.
|
|
43
|
+
*/
|
|
44
|
+
readonly SCHEMA_VERSION_UNSUPPORTED: "ALDUS_SCHEMA_VERSION_UNSUPPORTED";
|
|
35
45
|
};
|
|
36
46
|
/** @see RegressionErrorCodes */
|
|
37
47
|
export type RegressionErrorCode = (typeof RegressionErrorCodes)[keyof typeof RegressionErrorCodes];
|
package/dist/errors.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"errors.d.ts","sourceRoot":"","sources":["../src/errors.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,UAAU,EAAE,KAAK,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAErE,oDAAoD;AACpD,eAAO,MAAM,oBAAoB;IAC/B,+DAA+D;aAC/D,gBAAgB,EAAE,wBAAwB;IAC1C,gDAAgD;aAChD,qBAAqB,EAAE,6BAA6B;IACpD,8EAA8E;aAC9E,oBAAoB,EAAE,4BAA4B;IAClD,6DAA6D;aAC7D,iBAAiB,EAAE,yBAAyB;IAC5C;;;;;;OAMG;aACH,mBAAmB,EAAE,2BAA2B;IAChD,mFAAmF;aACnF,2BAA2B,EAAE,mCAAmC;IAChE,6CAA6C;aAC7C,oBAAoB,EAAE,4BAA4B;IAClD,6CAA6C;aAC7C,oBAAoB,EAAE,4BAA4B;IAClD,2FAA2F;aAC3F,cAAc,EAAE,sBAAsB;
|
|
1
|
+
{"version":3,"file":"errors.d.ts","sourceRoot":"","sources":["../src/errors.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,UAAU,EAAE,KAAK,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAErE,oDAAoD;AACpD,eAAO,MAAM,oBAAoB;IAC/B,+DAA+D;aAC/D,gBAAgB,EAAE,wBAAwB;IAC1C,gDAAgD;aAChD,qBAAqB,EAAE,6BAA6B;IACpD,8EAA8E;aAC9E,oBAAoB,EAAE,4BAA4B;IAClD,6DAA6D;aAC7D,iBAAiB,EAAE,yBAAyB;IAC5C;;;;;;OAMG;aACH,mBAAmB,EAAE,2BAA2B;IAChD,mFAAmF;aACnF,2BAA2B,EAAE,mCAAmC;IAChE,6CAA6C;aAC7C,oBAAoB,EAAE,4BAA4B;IAClD,6CAA6C;aAC7C,oBAAoB,EAAE,4BAA4B;IAClD,2FAA2F;aAC3F,cAAc,EAAE,sBAAsB;IACtC;;;;;;;;OAQG;aACH,0BAA0B,EAAE,kCAAkC;CACtD,CAAC;AAEX,gCAAgC;AAChC,MAAM,MAAM,mBAAmB,GAAG,CAAC,OAAO,oBAAoB,CAAC,CAAC,MAAM,OAAO,oBAAoB,CAAC,CAAC;AAEnG,sEAAsE;AACtE,wBAAgB,eAAe,CAC7B,IAAI,EAAE,mBAAmB,EACzB,OAAO,EAAE,MAAM,EACf,OAAO,EAAE;IAAE,QAAQ,EAAE,aAAa,CAAC;IAAC,SAAS,CAAC,EAAE,OAAO,CAAC;IAAC,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CAAE,GAC3F,UAAU,CAEZ"}
|
package/dist/errors.js
CHANGED
|
@@ -32,6 +32,16 @@ export const RegressionErrorCodes = {
|
|
|
32
32
|
BLIND_SPOT_DUPLICATE: "ALDUS_BLIND_SPOT_DUPLICATE",
|
|
33
33
|
/** A promotion policy is internally inconsistent — a threshold outside its valid range. */
|
|
34
34
|
POLICY_INVALID: "ALDUS_POLICY_INVALID",
|
|
35
|
+
/**
|
|
36
|
+
* A record declares a schema version newer than this runtime implements.
|
|
37
|
+
*
|
|
38
|
+
* Refused rather than accepted, because this runtime cannot know what a later version's fields
|
|
39
|
+
* mean and the schemas discard what they do not declare. Accepting one returns an object that
|
|
40
|
+
* looks complete and is not — a read path answering wrongly with no signal, which is worse than
|
|
41
|
+
* refusing. Whether an *older* record is comparable is the caller's policy, not this package's:
|
|
42
|
+
* see `compareSchemaVersion`.
|
|
43
|
+
*/
|
|
44
|
+
SCHEMA_VERSION_UNSUPPORTED: "ALDUS_SCHEMA_VERSION_UNSUPPORTED",
|
|
35
45
|
};
|
|
36
46
|
/** Construct an {@link AldusError} with a regression-harness code. */
|
|
37
47
|
export function regressionError(code, message, options) {
|
package/dist/errors.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"errors.js","sourceRoot":"","sources":["../src/errors.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,UAAU,EAAsB,MAAM,qBAAqB,CAAC;AAErE,oDAAoD;AACpD,MAAM,CAAC,MAAM,oBAAoB,GAAG;IAClC,+DAA+D;IAC/D,gBAAgB,EAAE,wBAAwB;IAC1C,gDAAgD;IAChD,qBAAqB,EAAE,6BAA6B;IACpD,8EAA8E;IAC9E,oBAAoB,EAAE,4BAA4B;IAClD,6DAA6D;IAC7D,iBAAiB,EAAE,yBAAyB;IAC5C;;;;;;OAMG;IACH,mBAAmB,EAAE,2BAA2B;IAChD,mFAAmF;IACnF,2BAA2B,EAAE,mCAAmC;IAChE,6CAA6C;IAC7C,oBAAoB,EAAE,4BAA4B;IAClD,6CAA6C;IAC7C,oBAAoB,EAAE,4BAA4B;IAClD,2FAA2F;IAC3F,cAAc,EAAE,sBAAsB;
|
|
1
|
+
{"version":3,"file":"errors.js","sourceRoot":"","sources":["../src/errors.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,UAAU,EAAsB,MAAM,qBAAqB,CAAC;AAErE,oDAAoD;AACpD,MAAM,CAAC,MAAM,oBAAoB,GAAG;IAClC,+DAA+D;IAC/D,gBAAgB,EAAE,wBAAwB;IAC1C,gDAAgD;IAChD,qBAAqB,EAAE,6BAA6B;IACpD,8EAA8E;IAC9E,oBAAoB,EAAE,4BAA4B;IAClD,6DAA6D;IAC7D,iBAAiB,EAAE,yBAAyB;IAC5C;;;;;;OAMG;IACH,mBAAmB,EAAE,2BAA2B;IAChD,mFAAmF;IACnF,2BAA2B,EAAE,mCAAmC;IAChE,6CAA6C;IAC7C,oBAAoB,EAAE,4BAA4B;IAClD,6CAA6C;IAC7C,oBAAoB,EAAE,4BAA4B;IAClD,2FAA2F;IAC3F,cAAc,EAAE,sBAAsB;IACtC;;;;;;;;OAQG;IACH,0BAA0B,EAAE,kCAAkC;CACtD,CAAC;AAKX,sEAAsE;AACtE,MAAM,UAAU,eAAe,CAC7B,IAAyB,EACzB,OAAe,EACf,OAA4F;IAE5F,OAAO,IAAI,UAAU,CAAC,IAAI,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;AAChD,CAAC"}
|
package/dist/index.d.ts
CHANGED
|
@@ -17,7 +17,7 @@
|
|
|
17
17
|
*
|
|
18
18
|
* @packageDocumentation
|
|
19
19
|
*/
|
|
20
|
-
export { REGRESSION_SCHEMA_VERSION, defectCaseSchema, defectCorpusSchema, evaluatorFindingSchema, evaluatorOutcomeSchema, evaluatorRunSchema, findingCategory, humanFindingSchema, parseDefectCorpus, parseEvaluatorRun, scopeDimensions, severityLevel, type DefectCase, type DefectCorpus, type EvaluatorFinding, type EvaluatorOutcome, type EvaluatorRun, type HumanFinding, type ScopeDimensions, } from "./corpus.js";
|
|
20
|
+
export { REGRESSION_SCHEMA_VERSION, defectCaseSchema, compareSchemaVersion, defectCorpusSchema, evaluatorFindingSchema, evaluatorOutcomeSchema, evaluatorRunSchema, findingCategory, humanFindingSchema, parseDefectCorpus, parseEvaluatorRun, scopeDimensions, severityLevel, type DefectCase, type DefectCorpus, type EvaluatorFinding, type EvaluatorOutcome, type EvaluatorRun, type HumanFinding, type ScopeDimensions, type SchemaVersionRelation, } from "./corpus.js";
|
|
21
21
|
export { WHOLE_CORPUS_SLICE, deriveScopeSelectors, observedDimensions, scopeKey, scopeLabel, scopeMatches, type ScopeSelector, } from "./scope.js";
|
|
22
22
|
export { DEFAULT_CORRECTION_HARM_WEIGHTS, DEFAULT_PROMOTION_THRESHOLDS, assertPolicyValid, correctionHarm, defaultPromotionPolicy, severityWeight, type CorrectionHarmWeights, type PolicyOrigin, type PromotionPolicy, type PromotionThresholds, type SeverityWeights, } from "./policy.js";
|
|
23
23
|
export { compareRun, type CaseComparison, type CaseVerdict, type CompareOptions, type ComparisonReport, type SliceMetrics, } from "./metrics.js";
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAGH,OAAO,EACL,yBAAyB,EACzB,gBAAgB,EAChB,kBAAkB,EAClB,sBAAsB,EACtB,sBAAsB,EACtB,kBAAkB,EAClB,eAAe,EACf,kBAAkB,EAClB,iBAAiB,EACjB,iBAAiB,EACjB,eAAe,EACf,aAAa,EACb,KAAK,UAAU,EACf,KAAK,YAAY,EACjB,KAAK,gBAAgB,EACrB,KAAK,gBAAgB,EACrB,KAAK,YAAY,EACjB,KAAK,YAAY,EACjB,KAAK,eAAe,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAGH,OAAO,EACL,yBAAyB,EACzB,gBAAgB,EAChB,oBAAoB,EACpB,kBAAkB,EAClB,sBAAsB,EACtB,sBAAsB,EACtB,kBAAkB,EAClB,eAAe,EACf,kBAAkB,EAClB,iBAAiB,EACjB,iBAAiB,EACjB,eAAe,EACf,aAAa,EACb,KAAK,UAAU,EACf,KAAK,YAAY,EACjB,KAAK,gBAAgB,EACrB,KAAK,gBAAgB,EACrB,KAAK,YAAY,EACjB,KAAK,YAAY,EACjB,KAAK,eAAe,EACpB,KAAK,qBAAqB,GAC3B,MAAM,aAAa,CAAC;AAGrB,OAAO,EACL,kBAAkB,EAClB,oBAAoB,EACpB,kBAAkB,EAClB,QAAQ,EACR,UAAU,EACV,YAAY,EACZ,KAAK,aAAa,GACnB,MAAM,YAAY,CAAC;AAGpB,OAAO,EACL,+BAA+B,EAC/B,4BAA4B,EAC5B,iBAAiB,EACjB,cAAc,EACd,sBAAsB,EACtB,cAAc,EACd,KAAK,qBAAqB,EAC1B,KAAK,YAAY,EACjB,KAAK,eAAe,EACpB,KAAK,mBAAmB,EACxB,KAAK,eAAe,GACrB,MAAM,aAAa,CAAC;AAGrB,OAAO,EACL,UAAU,EACV,KAAK,cAAc,EACnB,KAAK,WAAW,EAChB,KAAK,cAAc,EACnB,KAAK,gBAAgB,EACrB,KAAK,YAAY,GAClB,MAAM,cAAc,CAAC;AAGtB,OAAO,EACL,mBAAmB,EACnB,iBAAiB,EACjB,mBAAmB,EACnB,eAAe,EACf,KAAK,SAAS,EACd,KAAK,eAAe,GACrB,MAAM,iBAAiB,CAAC;AAGzB,OAAO,EACL,eAAe,EACf,8BAA8B,EAC9B,KAAK,kBAAkB,EACvB,KAAK,sBAAsB,EAC3B,KAAK,gBAAgB,EACrB,KAAK,YAAY,GAClB,MAAM,gBAAgB,CAAC;AAGxB,OAAO,EAAE,aAAa,EAAE,mBAAmB,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;AAGxF,OAAO,EAAE,oBAAoB,EAAE,eAAe,EAAE,KAAK,mBAAmB,EAAE,MAAM,aAAa,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -18,7 +18,7 @@
|
|
|
18
18
|
* @packageDocumentation
|
|
19
19
|
*/
|
|
20
20
|
// --- Defect corpus (§12.1, §12.3, §24) --------------------------------------------------------
|
|
21
|
-
export { REGRESSION_SCHEMA_VERSION, defectCaseSchema, defectCorpusSchema, evaluatorFindingSchema, evaluatorOutcomeSchema, evaluatorRunSchema, findingCategory, humanFindingSchema, parseDefectCorpus, parseEvaluatorRun, scopeDimensions, severityLevel, } from "./corpus.js";
|
|
21
|
+
export { REGRESSION_SCHEMA_VERSION, defectCaseSchema, compareSchemaVersion, defectCorpusSchema, evaluatorFindingSchema, evaluatorOutcomeSchema, evaluatorRunSchema, findingCategory, humanFindingSchema, parseDefectCorpus, parseEvaluatorRun, scopeDimensions, severityLevel, } from "./corpus.js";
|
|
22
22
|
// --- Scope slicing (§12.1) --------------------------------------------------------------------
|
|
23
23
|
export { WHOLE_CORPUS_SLICE, deriveScopeSelectors, observedDimensions, scopeKey, scopeLabel, scopeMatches, } from "./scope.js";
|
|
24
24
|
// --- Promotion policy (§12.1, §25.9; ADR-0010) ------------------------------------------------
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAEH,iGAAiG;AACjG,OAAO,EACL,yBAAyB,EACzB,gBAAgB,EAChB,kBAAkB,EAClB,sBAAsB,EACtB,sBAAsB,EACtB,kBAAkB,EAClB,eAAe,EACf,kBAAkB,EAClB,iBAAiB,EACjB,iBAAiB,EACjB,eAAe,EACf,aAAa,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAEH,iGAAiG;AACjG,OAAO,EACL,yBAAyB,EACzB,gBAAgB,EAChB,oBAAoB,EACpB,kBAAkB,EAClB,sBAAsB,EACtB,sBAAsB,EACtB,kBAAkB,EAClB,eAAe,EACf,kBAAkB,EAClB,iBAAiB,EACjB,iBAAiB,EACjB,eAAe,EACf,aAAa,GASd,MAAM,aAAa,CAAC;AAErB,iGAAiG;AACjG,OAAO,EACL,kBAAkB,EAClB,oBAAoB,EACpB,kBAAkB,EAClB,QAAQ,EACR,UAAU,EACV,YAAY,GAEb,MAAM,YAAY,CAAC;AAEpB,iGAAiG;AACjG,OAAO,EACL,+BAA+B,EAC/B,4BAA4B,EAC5B,iBAAiB,EACjB,cAAc,EACd,sBAAsB,EACtB,cAAc,GAMf,MAAM,aAAa,CAAC;AAErB,iGAAiG;AACjG,OAAO,EACL,UAAU,GAMX,MAAM,cAAc,CAAC;AAEtB,iGAAiG;AACjG,OAAO,EACL,mBAAmB,EACnB,iBAAiB,EACjB,mBAAmB,EACnB,eAAe,GAGhB,MAAM,iBAAiB,CAAC;AAEzB,iGAAiG;AACjG,OAAO,EACL,eAAe,EACf,8BAA8B,GAK/B,MAAM,gBAAgB,CAAC;AAExB,iGAAiG;AACjG,OAAO,EAAE,aAAa,EAAE,mBAAmB,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;AAExF,iGAAiG;AACjG,OAAO,EAAE,oBAAoB,EAAE,eAAe,EAA4B,MAAM,aAAa,CAAC"}
|
package/dist/metrics.d.ts
CHANGED
|
@@ -44,6 +44,19 @@ export interface CaseComparison {
|
|
|
44
44
|
* negative — it did catch the case — but a promotion reader should see it.
|
|
45
45
|
*/
|
|
46
46
|
categoryMismatch: boolean;
|
|
47
|
+
/**
|
|
48
|
+
* Whether site-level comparison means anything for this case (#140).
|
|
49
|
+
*
|
|
50
|
+
* `false` when the evaluator flagged without enumerating what it found — a wrapped legacy
|
|
51
|
+
* evaluator reporting that it had something to say, and nothing about how much or where.
|
|
52
|
+
*
|
|
53
|
+
* The case-level verdict is still sound: a flag is a flag, and §12's precision and recall over
|
|
54
|
+
* *cases* are computable. What is not computable is anything per site, and the previous code
|
|
55
|
+
* quietly answered it anyway — with no evaluator categories to compare, `categoryMismatch` came
|
|
56
|
+
* out `false`, which reads as *the categories agreed*. An evaluator nobody measured would have
|
|
57
|
+
* scored a clean sheet.
|
|
58
|
+
*/
|
|
59
|
+
siteMetricsMeasurable: boolean;
|
|
47
60
|
/** Harm a spurious flag would cause here; zero unless this is a false positive. */
|
|
48
61
|
unnecessaryCorrectionHarm: number;
|
|
49
62
|
}
|
|
@@ -68,6 +81,14 @@ export interface SliceMetrics {
|
|
|
68
81
|
trueNegatives: number;
|
|
69
82
|
/** True positives found under the wrong category (contract §12.3). */
|
|
70
83
|
categoryMismatches: number;
|
|
84
|
+
/**
|
|
85
|
+
* Cases whose site-level metrics are unmeasurable (#140).
|
|
86
|
+
*
|
|
87
|
+
* Reported rather than folded in, because a slice where this is large is a slice whose
|
|
88
|
+
* category agreement means much less than the other numbers suggest. Zero is the honest value
|
|
89
|
+
* only when it is true.
|
|
90
|
+
*/
|
|
91
|
+
unmeasurableSiteMetrics: number;
|
|
71
92
|
/** `truePositives / defectiveCases`; `undefined` when there are no defective cases. */
|
|
72
93
|
recall: number | undefined;
|
|
73
94
|
/** `falsePositives / cleanCases`; `undefined` when there are no clean cases. */
|
package/dist/metrics.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"metrics.d.ts","sourceRoot":"","sources":["../src/metrics.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,KAAK,EAAc,YAAY,EAAoB,YAAY,EAAE,MAAM,aAAa,CAAC;AAE5F,OAAO,EAAkC,KAAK,eAAe,EAAE,MAAM,aAAa,CAAC;AACnF,OAAO,EAKL,KAAK,aAAa,EACnB,MAAM,YAAY,CAAC;AAEpB,wDAAwD;AACxD,MAAM,MAAM,WAAW;AACrB,kDAAkD;AAChD,cAAc;AAChB,+DAA+D;GAC7D,eAAe;AACjB,+DAA+D;GAC7D,eAAe;AACjB,mDAAmD;GACjD,cAAc,CAAC;AAEnB,6DAA6D;AAC7D,MAAM,WAAW,cAAc;IAC7B,gBAAgB;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,yBAAyB;IACzB,KAAK,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IACxC,uBAAuB;IACvB,OAAO,EAAE,WAAW,CAAC;IACrB,gEAAgE;IAChE,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7B,wCAAwC;IACxC,cAAc,EAAE,MAAM,CAAC;IACvB;;;;;;OAMG;IACH,gBAAgB,EAAE,OAAO,CAAC;IAC1B,mFAAmF;IACnF,yBAAyB,EAAE,MAAM,CAAC;CACnC;AAED,4CAA4C;AAC5C,MAAM,WAAW,YAAY;IAC3B,mBAAmB;IACnB,QAAQ,EAAE,aAAa,CAAC;IACxB,gCAAgC;IAChC,GAAG,EAAE,MAAM,CAAC;IACZ,gCAAgC;IAChC,KAAK,EAAE,MAAM,CAAC;IACd,wCAAwC;IACxC,cAAc,EAAE,MAAM,CAAC;IACvB,oCAAoC;IACpC,UAAU,EAAE,MAAM,CAAC;IACnB,4CAA4C;IAC5C,SAAS,EAAE,MAAM,CAAC;IAClB,wBAAwB;IACxB,aAAa,EAAE,MAAM,CAAC;IACtB,cAAc,EAAE,MAAM,CAAC;IACvB,cAAc,EAAE,MAAM,CAAC;IACvB,aAAa,EAAE,MAAM,CAAC;IACtB,sEAAsE;IACtE,kBAAkB,EAAE,MAAM,CAAC;IAC3B,uFAAuF;IACvF,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3B,gFAAgF;IAChF,iBAAiB,EAAE,MAAM,GAAG,SAAS,CAAC;IACtC,6FAA6F;IAC7F,sBAAsB,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3C,gEAAgE;IAChE,8BAA8B,EAAE,MAAM,CAAC;IACvC,8DAA8D;IAC9D,yBAAyB,EAAE,MAAM,CAAC;IAClC,2EAA2E;IAC3E,6BAA6B,EAAE,MAAM,GAAG,SAAS,CAAC;IAClD;;;;;OAKG;IACH,wBAAwB,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7C,gEAAgE;IAChE,OAAO,EAAE,SAAS,MAAM,EAAE,CAAC;CAC5B;AAED,qFAAqF;AACrF,MAAM,WAAW,gBAAgB;IAC/B,0BAA0B;IAC1B,WAAW,EAAE,MAAM,CAAC;IACpB,4EAA4E;IAC5E,gBAAgB,EAAE,MAAM,CAAC;IACzB,+BAA+B;IAC/B,QAAQ,EAAE,MAAM,CAAC;IACjB;;;;;;OAMG;IACH,WAAW,EAAE,YAAY,CAAC;IAC1B,2EAA2E;IAC3E,MAAM,EAAE,SAAS,YAAY,EAAE,CAAC;IAChC,+DAA+D;IAC/D,WAAW,EAAE,SAAS,cAAc,EAAE,CAAC;IACvC,+DAA+D;IAC/D,kBAAkB,EAAE,SAAS,MAAM,EAAE,CAAC;CACvC;
|
|
1
|
+
{"version":3,"file":"metrics.d.ts","sourceRoot":"","sources":["../src/metrics.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,KAAK,EAAc,YAAY,EAAoB,YAAY,EAAE,MAAM,aAAa,CAAC;AAE5F,OAAO,EAAkC,KAAK,eAAe,EAAE,MAAM,aAAa,CAAC;AACnF,OAAO,EAKL,KAAK,aAAa,EACnB,MAAM,YAAY,CAAC;AAEpB,wDAAwD;AACxD,MAAM,MAAM,WAAW;AACrB,kDAAkD;AAChD,cAAc;AAChB,+DAA+D;GAC7D,eAAe;AACjB,+DAA+D;GAC7D,eAAe;AACjB,mDAAmD;GACjD,cAAc,CAAC;AAEnB,6DAA6D;AAC7D,MAAM,WAAW,cAAc;IAC7B,gBAAgB;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,yBAAyB;IACzB,KAAK,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IACxC,uBAAuB;IACvB,OAAO,EAAE,WAAW,CAAC;IACrB,gEAAgE;IAChE,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7B,wCAAwC;IACxC,cAAc,EAAE,MAAM,CAAC;IACvB;;;;;;OAMG;IACH,gBAAgB,EAAE,OAAO,CAAC;IAC1B;;;;;;;;;;;OAWG;IACH,qBAAqB,EAAE,OAAO,CAAC;IAC/B,mFAAmF;IACnF,yBAAyB,EAAE,MAAM,CAAC;CACnC;AAED,4CAA4C;AAC5C,MAAM,WAAW,YAAY;IAC3B,mBAAmB;IACnB,QAAQ,EAAE,aAAa,CAAC;IACxB,gCAAgC;IAChC,GAAG,EAAE,MAAM,CAAC;IACZ,gCAAgC;IAChC,KAAK,EAAE,MAAM,CAAC;IACd,wCAAwC;IACxC,cAAc,EAAE,MAAM,CAAC;IACvB,oCAAoC;IACpC,UAAU,EAAE,MAAM,CAAC;IACnB,4CAA4C;IAC5C,SAAS,EAAE,MAAM,CAAC;IAClB,wBAAwB;IACxB,aAAa,EAAE,MAAM,CAAC;IACtB,cAAc,EAAE,MAAM,CAAC;IACvB,cAAc,EAAE,MAAM,CAAC;IACvB,aAAa,EAAE,MAAM,CAAC;IACtB,sEAAsE;IACtE,kBAAkB,EAAE,MAAM,CAAC;IAC3B;;;;;;OAMG;IACH,uBAAuB,EAAE,MAAM,CAAC;IAChC,uFAAuF;IACvF,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3B,gFAAgF;IAChF,iBAAiB,EAAE,MAAM,GAAG,SAAS,CAAC;IACtC,6FAA6F;IAC7F,sBAAsB,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3C,gEAAgE;IAChE,8BAA8B,EAAE,MAAM,CAAC;IACvC,8DAA8D;IAC9D,yBAAyB,EAAE,MAAM,CAAC;IAClC,2EAA2E;IAC3E,6BAA6B,EAAE,MAAM,GAAG,SAAS,CAAC;IAClD;;;;;OAKG;IACH,wBAAwB,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7C,gEAAgE;IAChE,OAAO,EAAE,SAAS,MAAM,EAAE,CAAC;CAC5B;AAED,qFAAqF;AACrF,MAAM,WAAW,gBAAgB;IAC/B,0BAA0B;IAC1B,WAAW,EAAE,MAAM,CAAC;IACpB,4EAA4E;IAC5E,gBAAgB,EAAE,MAAM,CAAC;IACzB,+BAA+B;IAC/B,QAAQ,EAAE,MAAM,CAAC;IACjB;;;;;;OAMG;IACH,WAAW,EAAE,YAAY,CAAC;IAC1B,2EAA2E;IAC3E,MAAM,EAAE,SAAS,YAAY,EAAE,CAAC;IAChC,+DAA+D;IAC/D,WAAW,EAAE,SAAS,cAAc,EAAE,CAAC;IACvC,+DAA+D;IAC/D,kBAAkB,EAAE,SAAS,MAAM,EAAE,CAAC;CACvC;AAoGD,sCAAsC;AACtC,MAAM,WAAW,cAAc;IAC7B;;;;OAIG;IACH,cAAc,CAAC,EAAE,SAAS,CAAC,SAAS,MAAM,EAAE,CAAC,EAAE,CAAC;CACjD;AAED;;;;;;;GAOG;AACH,wBAAgB,UAAU,CACxB,MAAM,EAAE,YAAY,EACpB,GAAG,EAAE,YAAY,EACjB,MAAM,EAAE,eAAe,EACvB,OAAO,GAAE,cAAmB,GAC3B,gBAAgB,CAkDlB"}
|
package/dist/metrics.js
CHANGED
|
@@ -39,6 +39,10 @@ function compareCase(entry, outcome, policy) {
|
|
|
39
39
|
severity: entry.severity,
|
|
40
40
|
severityWeight: weight,
|
|
41
41
|
categoryMismatch,
|
|
42
|
+
// A flag with nothing enumerated. Not a contradiction to be repaired by inventing a finding —
|
|
43
|
+
// that would make one report count as one defect, which is exactly the statistic #140 exists
|
|
44
|
+
// to protect.
|
|
45
|
+
siteMetricsMeasurable: outcome?.siteMetricsMeasurable ?? !(flagged && (outcome?.findings ?? []).length === 0),
|
|
42
46
|
unnecessaryCorrectionHarm: harm,
|
|
43
47
|
};
|
|
44
48
|
}
|
|
@@ -72,6 +76,8 @@ function summarise(selector, comparisons, labellers) {
|
|
|
72
76
|
falsePositives,
|
|
73
77
|
trueNegatives,
|
|
74
78
|
categoryMismatches: comparisons.filter((comparison) => comparison.categoryMismatch).length,
|
|
79
|
+
unmeasurableSiteMetrics: comparisons.filter((comparison) => !comparison.siteMetricsMeasurable)
|
|
80
|
+
.length,
|
|
75
81
|
recall: defectiveCases === 0 ? undefined : truePositives / defectiveCases,
|
|
76
82
|
falsePositiveRate: cleanCases === 0 ? undefined : falsePositives / cleanCases,
|
|
77
83
|
severityWeightedRecall: defectiveWeight === 0 ? undefined : caughtWeight / defectiveWeight,
|
package/dist/metrics.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"metrics.js","sourceRoot":"","sources":["../src/metrics.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAGH,OAAO,EAAE,oBAAoB,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AACpE,OAAO,EAAE,cAAc,EAAE,cAAc,EAAwB,MAAM,aAAa,CAAC;AACnF,OAAO,EACL,oBAAoB,EACpB,QAAQ,EACR,YAAY,EACZ,kBAAkB,GAEnB,MAAM,YAAY,CAAC;
|
|
1
|
+
{"version":3,"file":"metrics.js","sourceRoot":"","sources":["../src/metrics.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAGH,OAAO,EAAE,oBAAoB,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AACpE,OAAO,EAAE,cAAc,EAAE,cAAc,EAAwB,MAAM,aAAa,CAAC;AACnF,OAAO,EACL,oBAAoB,EACpB,QAAQ,EACR,YAAY,EACZ,kBAAkB,GAEnB,MAAM,YAAY,CAAC;AA8HpB,uCAAuC;AACvC,SAAS,WAAW,CAClB,KAAiB,EACjB,OAAqC,EACrC,MAAuB;IAEvB,MAAM,OAAO,GAAG,OAAO,EAAE,OAAO,IAAI,KAAK,CAAC;IAC1C,MAAM,MAAM,GAAG,KAAK,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,MAAM,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;IAEzF,IAAI,OAAoB,CAAC;IACzB,IAAI,KAAK,CAAC,SAAS;QAAE,OAAO,GAAG,OAAO,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,eAAe,CAAC;;QACrE,OAAO,GAAG,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,cAAc,CAAC;IAE1D,MAAM,eAAe,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;IACnF,MAAM,mBAAmB,GAAG,IAAI,GAAG,CAAC,CAAC,OAAO,EAAE,QAAQ,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;IAClG,MAAM,gBAAgB,GACpB,OAAO,KAAK,cAAc;QAC1B,eAAe,CAAC,IAAI,GAAG,CAAC;QACxB,mBAAmB,CAAC,IAAI,GAAG,CAAC;QAC5B,CAAC,CAAC,GAAG,mBAAmB,CAAC,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,eAAe,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC;IAE9E,MAAM,IAAI,GACR,OAAO,KAAK,eAAe;QACzB,CAAC,CAAC,cAAc,CAAC,MAAM,EAAE,KAAK,CAAC,gBAAgB,IAAI,MAAM,CAAC,sBAAsB,CAAC;QACjF,CAAC,CAAC,CAAC,CAAC;IAER,OAAO;QACL,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,OAAO;QACP,QAAQ,EAAE,KAAK,CAAC,QAAQ;QACxB,cAAc,EAAE,MAAM;QACtB,gBAAgB;QAChB,8FAA8F;QAC9F,6FAA6F;QAC7F,cAAc;QACd,qBAAqB,EACnB,OAAO,EAAE,qBAAqB,IAAI,CAAC,CAAC,OAAO,IAAI,CAAC,OAAO,EAAE,QAAQ,IAAI,EAAE,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC;QACxF,yBAAyB,EAAE,IAAI;KAChC,CAAC;AACJ,CAAC;AAED,wDAAwD;AACxD,SAAS,SAAS,CAChB,QAAuB,EACvB,WAAsC,EACtC,SAA8B;IAE9B,MAAM,KAAK,GAAG,CAAC,OAAoB,EAAU,EAAE,CAC7C,WAAW,CAAC,MAAM,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,KAAK,OAAO,CAAC,CAAC,MAAM,CAAC;IAE5E,MAAM,aAAa,GAAG,KAAK,CAAC,cAAc,CAAC,CAAC;IAC5C,MAAM,cAAc,GAAG,KAAK,CAAC,eAAe,CAAC,CAAC;IAC9C,MAAM,cAAc,GAAG,KAAK,CAAC,eAAe,CAAC,CAAC;IAC9C,MAAM,aAAa,GAAG,KAAK,CAAC,cAAc,CAAC,CAAC;IAE5C,MAAM,cAAc,GAAG,aAAa,GAAG,cAAc,CAAC;IACtD,MAAM,UAAU,GAAG,cAAc,GAAG,aAAa,CAAC;IAClD,MAAM,KAAK,GAAG,WAAW,CAAC,MAAM,CAAC;IAEjC,MAAM,YAAY,GAAG,WAAW;SAC7B,MAAM,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,KAAK,cAAc,CAAC;SAC7D,MAAM,CAAC,CAAC,KAAK,EAAE,UAAU,EAAE,EAAE,CAAC,KAAK,GAAG,UAAU,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC;IACvE,MAAM,YAAY,GAAG,WAAW;SAC7B,MAAM,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,KAAK,eAAe,CAAC;SAC9D,MAAM,CAAC,CAAC,KAAK,EAAE,UAAU,EAAE,EAAE,CAAC,KAAK,GAAG,UAAU,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC;IACvE,MAAM,eAAe,GAAG,YAAY,GAAG,YAAY,CAAC;IAEpD,MAAM,IAAI,GAAG,WAAW,CAAC,MAAM,CAC7B,CAAC,KAAK,EAAE,UAAU,EAAE,EAAE,CAAC,KAAK,GAAG,UAAU,CAAC,yBAAyB,EACnE,CAAC,CACF,CAAC;IAEF,OAAO;QACL,QAAQ;QACR,GAAG,EAAE,QAAQ,CAAC,QAAQ,CAAC;QACvB,KAAK;QACL,cAAc;QACd,UAAU;QACV,SAAS,EAAE,SAAS,CAAC,IAAI;QACzB,aAAa;QACb,cAAc;QACd,cAAc;QACd,aAAa;QACb,kBAAkB,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,UAAU,CAAC,gBAAgB,CAAC,CAAC,MAAM;QAC1F,uBAAuB,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,qBAAqB,CAAC;aAC3F,MAAM;QACT,MAAM,EAAE,cAAc,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,aAAa,GAAG,cAAc;QACzE,iBAAiB,EAAE,UAAU,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,cAAc,GAAG,UAAU;QAC7E,sBAAsB,EAAE,eAAe,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,YAAY,GAAG,eAAe;QAC1F,8BAA8B,EAAE,YAAY;QAC5C,yBAAyB,EAAE,IAAI;QAC/B,6BAA6B,EAAE,UAAU,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,GAAG,UAAU;QAC/E,wBAAwB,EAAE,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,aAAa,GAAG,aAAa,CAAC,GAAG,KAAK;QAC3F,OAAO,EAAE,WAAW,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC;KAC5D,CAAC;AACJ,CAAC;AAYD;;;;;;;GAOG;AACH,MAAM,UAAU,UAAU,CACxB,MAAoB,EACpB,GAAiB,EACjB,MAAuB,EACvB,OAAO,GAAmB,EAAE;IAE5B,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC;IAC7E,KAAK,MAAM,OAAO,IAAI,GAAG,CAAC,QAAQ,EAAE,CAAC;QACnC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;YAClC,MAAM,eAAe,CACnB,oBAAoB,CAAC,oBAAoB,EACzC,sCAAsC,OAAO,CAAC,MAAM,kBAAkB;gBACpE,IAAI,MAAM,CAAC,QAAQ,qBAAqB,EAC1C;gBACE,QAAQ,EAAE,YAAY;gBACtB,OAAO,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE;aAC/D,CACF,CAAC;QACJ,CAAC;IACH,CAAC;IAED,MAAM,eAAe,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC;IAC1F,MAAM,WAAW,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAC7C,WAAW,CAAC,KAAK,EAAE,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,CAC9D,CAAC;IAEF,MAAM,YAAY,GAAG,CAAC,MAA6B,EAAe,EAAE,CAClE,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC,CAAC;IAEtD,MAAM,SAAS,GAAG,oBAAoB,CACpC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,EACxC,OAAO,CAAC,cAAc,CACvB,CAAC;IAEF,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE;QACxC,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,YAAY,CAAC,KAAK,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAC;QACrF,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;QACnE,OAAO,SAAS,CACd,QAAQ,EACR,WAAW,CAAC,MAAM,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,WAAW,CAAC,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,EACtE,YAAY,CAAC,QAAQ,CAAC,CACvB,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,OAAO;QACL,WAAW,EAAE,GAAG,CAAC,WAAW;QAC5B,gBAAgB,EAAE,GAAG,CAAC,gBAAgB;QACtC,QAAQ,EAAE,MAAM,CAAC,QAAQ;QACzB,WAAW,EAAE,SAAS,CAAC,kBAAkB,EAAE,WAAW,EAAE,YAAY,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACnF,MAAM;QACN,WAAW;QACX,kBAAkB,EAAE,MAAM,CAAC,KAAK;aAC7B,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;aACrD,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC;KAChC,CAAC;AACJ,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@aldus-runtime/regression",
|
|
3
|
-
"version": "0.2.0-next.
|
|
3
|
+
"version": "0.2.0-next.60",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "Aldus Regression Harness — defect corpora, scope-aware evaluator metrics, blind-spot registry, and evaluator promotion evidence.",
|
|
6
6
|
"license": "Apache-2.0",
|
|
@@ -33,11 +33,11 @@
|
|
|
33
33
|
"typecheck:test": "tsc -p tsconfig.test.json"
|
|
34
34
|
},
|
|
35
35
|
"dependencies": {
|
|
36
|
-
"@aldus-runtime/core": "0.2.0-next.
|
|
36
|
+
"@aldus-runtime/core": "0.2.0-next.60",
|
|
37
37
|
"zod": "^4.4.3"
|
|
38
38
|
},
|
|
39
39
|
"devDependencies": {
|
|
40
|
-
"@aldus-runtime/testkit": "0.2.0-next.
|
|
40
|
+
"@aldus-runtime/testkit": "0.2.0-next.60",
|
|
41
41
|
"@types/node": "^26.2.0",
|
|
42
42
|
"typescript": "^7.0.2",
|
|
43
43
|
"vitest": "^4.1.10"
|
package/src/corpus.ts
CHANGED
|
@@ -19,7 +19,7 @@
|
|
|
19
19
|
* Zod is the single source of truth; TypeScript types are inferred (ADR-0002).
|
|
20
20
|
*/
|
|
21
21
|
|
|
22
|
-
import { actorRefSchema, SCHEMA_VERSION } from "@aldus-runtime/core";
|
|
22
|
+
import { actorRefSchema, schemaVersionString, SCHEMA_VERSION } from "@aldus-runtime/core";
|
|
23
23
|
import { z } from "zod";
|
|
24
24
|
|
|
25
25
|
import { RegressionErrorCodes, regressionError } from "./errors.js";
|
|
@@ -27,6 +27,83 @@ import { RegressionErrorCodes, regressionError } from "./errors.js";
|
|
|
27
27
|
/** A short opaque identifier. */
|
|
28
28
|
const identifier = z.string().min(1).max(200);
|
|
29
29
|
|
|
30
|
+
// --- Schema version (ADR-0003, ADR-0053) -----------------------------------------------------
|
|
31
|
+
|
|
32
|
+
/** Schema version this package stamps on records it constructs. */
|
|
33
|
+
export const REGRESSION_SCHEMA_VERSION = SCHEMA_VERSION;
|
|
34
|
+
|
|
35
|
+
/** How a record's declared schema version relates to the one this runtime implements. */
|
|
36
|
+
export type SchemaVersionRelation = "older" | "same" | "newer";
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* Compare a record's declared schema version against {@link REGRESSION_SCHEMA_VERSION}.
|
|
40
|
+
*
|
|
41
|
+
* Exported because the two directions are not the same kind of question. **Newer** is soundness
|
|
42
|
+
* and belongs here: this runtime cannot know what a later version's fields mean, so it refuses.
|
|
43
|
+
* **Older** is policy and belongs to the caller — whether a run from an earlier version is
|
|
44
|
+
* comparable to today's depends on what changed between them, which this package cannot know
|
|
45
|
+
* either. What it can do is stop every caller reimplementing the comparison, which is the part
|
|
46
|
+
* that would drift.
|
|
47
|
+
*
|
|
48
|
+
* A record must be readable in order to be upgraded, so this is deliberately a question a caller
|
|
49
|
+
* can ask *before* parsing, and an older record still parses.
|
|
50
|
+
*
|
|
51
|
+
* @param recordVersion - The `schemaVersion` field of a record, `MAJOR.MINOR` (ADR-0003).
|
|
52
|
+
* @throws {AldusError} `ALDUS_CORPUS_MALFORMED` if `recordVersion` is not `MAJOR.MINOR`.
|
|
53
|
+
*/
|
|
54
|
+
export function compareSchemaVersion(recordVersion: string): SchemaVersionRelation {
|
|
55
|
+
const parse = (value: string): readonly [number, number] => {
|
|
56
|
+
const match = /^(0|[1-9]\d*)\.(0|[1-9]\d*)$/.exec(value);
|
|
57
|
+
if (match === null) {
|
|
58
|
+
// Path and code only — never the received value (contract §19.2).
|
|
59
|
+
throw regressionError(
|
|
60
|
+
RegressionErrorCodes.CORPUS_MALFORMED,
|
|
61
|
+
"A schema version must be MAJOR.MINOR.",
|
|
62
|
+
{
|
|
63
|
+
category: "validation",
|
|
64
|
+
details: { issues: [{ path: "schemaVersion", code: "invalid_format" }] },
|
|
65
|
+
},
|
|
66
|
+
);
|
|
67
|
+
}
|
|
68
|
+
return [Number(match[1]), Number(match[2])] as const;
|
|
69
|
+
};
|
|
70
|
+
const [recordMajor, recordMinor] = parse(recordVersion);
|
|
71
|
+
const [runtimeMajor, runtimeMinor] = parse(REGRESSION_SCHEMA_VERSION);
|
|
72
|
+
if (recordMajor !== runtimeMajor) return recordMajor > runtimeMajor ? "newer" : "older";
|
|
73
|
+
if (recordMinor !== runtimeMinor) return recordMinor > runtimeMinor ? "newer" : "older";
|
|
74
|
+
return "same";
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* True when `value` is a well-formed version this runtime does not implement.
|
|
79
|
+
*
|
|
80
|
+
* A malformed value answers `false` rather than throwing, so the format check that owns that
|
|
81
|
+
* failure reports it under its own code and the version rule fires only on a version.
|
|
82
|
+
*/
|
|
83
|
+
function declaresNewerVersion(value: string): boolean {
|
|
84
|
+
return /^(0|[1-9]\d*)\.(0|[1-9]\d*)$/.test(value) && compareSchemaVersion(value) === "newer";
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/**
|
|
88
|
+
* The `schemaVersion` field of a record this package reads.
|
|
89
|
+
*
|
|
90
|
+
* **The exported schema carries the rule** (ADR-0053). `parseDefectCorpus` and `parseEvaluatorRun`
|
|
91
|
+
* refused a newer record while `defectCorpusSchema.safeParse` and `evaluatorRunSchema.safeParse`
|
|
92
|
+
* accepted the same bytes — a rule enforced at one entry point and not the other is a rule plus a
|
|
93
|
+
* bypass, and the exported object is the obvious door (#199). So the refusal lives on the field, and
|
|
94
|
+
* the parse functions check it first only so that a newer record fails with one clear
|
|
95
|
+
* `ALDUS_SCHEMA_VERSION_UNSUPPORTED` rather than a field issue about the version.
|
|
96
|
+
*
|
|
97
|
+
* This is **this package's** rule, not Core's: any newer version is refused, minor included, because
|
|
98
|
+
* these schemas are strict and a newer minor would fail on its added keys anyway. Refusing on the
|
|
99
|
+
* version names the reason. Core's records accept a newer minor and report it as a `forward` read
|
|
100
|
+
* (ADR-0003, ADR-0053).
|
|
101
|
+
*/
|
|
102
|
+
const recordSchemaVersion = schemaVersionString.refine((value) => !declaresNewerVersion(value), {
|
|
103
|
+
// Names this package's own constant, never the received value (contract §19.2).
|
|
104
|
+
message: `Schema version must not be newer than ${REGRESSION_SCHEMA_VERSION}, which is what this runtime implements.`,
|
|
105
|
+
});
|
|
106
|
+
|
|
30
107
|
/**
|
|
31
108
|
* A finding category, structured per contract §12.3.
|
|
32
109
|
*
|
|
@@ -148,14 +225,38 @@ export type DefectCase = z.infer<typeof defectCaseSchema>;
|
|
|
148
225
|
export const defectCorpusSchema = z
|
|
149
226
|
.object({
|
|
150
227
|
/** Schema version of this record (ADR-0003). */
|
|
151
|
-
schemaVersion:
|
|
228
|
+
schemaVersion: recordSchemaVersion,
|
|
152
229
|
/** Identity of this corpus. */
|
|
153
230
|
corpusId: identifier,
|
|
154
231
|
/** Human-readable purpose. */
|
|
155
232
|
description: z.string().max(2000).optional(),
|
|
156
233
|
/** The labelled cases. */
|
|
157
234
|
cases: z.array(defectCaseSchema).max(100_000),
|
|
235
|
+
/**
|
|
236
|
+
* Adopter-owned data carried with this record.
|
|
237
|
+
*
|
|
238
|
+
* A declared extension point, because strictness without one is a rule with no exit. Refusing
|
|
239
|
+
* an undeclared key is right — a record from a later runtime whose added fields vanish hands
|
|
240
|
+
* the caller an object that looks complete and is not. But that argument is about *the
|
|
241
|
+
* runtime's* future fields, and it was applied to an adopter's own data, which is a different
|
|
242
|
+
* category and had a real pattern behind it: the provenance of a label belongs beside the
|
|
243
|
+
* label, in the same file, versioned together, because a label whose source lives in another
|
|
244
|
+
* file is one refactor away from a label with no source.
|
|
245
|
+
*
|
|
246
|
+
* So the two cases are separated rather than collapsed. An unknown key is refused; a declared
|
|
247
|
+
* one is **preserved through the parse**, which the previous sibling-key arrangement could not
|
|
248
|
+
* do — it survived only because readers went around the parser to the raw JSON.
|
|
249
|
+
*
|
|
250
|
+
* Core never interprets this. Same shape as `WorkflowEvent.details` and
|
|
251
|
+
* `SpendReservationTransition.detail` (§4.2: adopter-shaped concepts are opaque here).
|
|
252
|
+
*/
|
|
253
|
+
metadata: z.record(z.string(), z.unknown()).optional(),
|
|
158
254
|
})
|
|
255
|
+
// Strict: an unknown key is refused, not stripped. Without this a record from a later runtime
|
|
256
|
+
// parses and its added fields vanish, handing the caller an object that looks complete and is
|
|
257
|
+
// not. See SCHEMA_VERSION_UNSUPPORTED — the version check and this are the same guarantee, one
|
|
258
|
+
// for a declared newer version and one for an undeclared newer shape.
|
|
259
|
+
.strict()
|
|
159
260
|
.meta({
|
|
160
261
|
id: "DefectCorpus",
|
|
161
262
|
title: "DefectCorpus",
|
|
@@ -190,8 +291,27 @@ export const evaluatorOutcomeSchema = z
|
|
|
190
291
|
caseId: identifier,
|
|
191
292
|
/** Whether the evaluator flagged the subject. */
|
|
192
293
|
flagged: z.boolean(),
|
|
193
|
-
/**
|
|
294
|
+
/**
|
|
295
|
+
* The defect occurrences it enumerated. Empty when not flagged, **and also empty when the
|
|
296
|
+
* evaluator flagged without enumerating** (#140).
|
|
297
|
+
*
|
|
298
|
+
* An empty list under `flagged: true` is therefore not a contradiction and must not be
|
|
299
|
+
* repaired by inventing an entry. A wrapped legacy evaluator reports that it had something to
|
|
300
|
+
* say and not how much; fabricating one finding would make its report count as one defect,
|
|
301
|
+
* which is the statistic this distinction exists to protect.
|
|
302
|
+
*/
|
|
194
303
|
findings: z.array(evaluatorFindingSchema).max(256),
|
|
304
|
+
/**
|
|
305
|
+
* Whether site-level metrics are computable for this case (#140).
|
|
306
|
+
*
|
|
307
|
+
* `false` when the evaluator flagged without enumerating, or when a report could not be mapped
|
|
308
|
+
* to this case's subject scope at all. Site-level precision and recall are then **unmeasurable
|
|
309
|
+
* rather than zero**, and a metric that silently treated an unenumerated flag as zero findings
|
|
310
|
+
* would report perfect precision for an evaluator nobody measured.
|
|
311
|
+
*
|
|
312
|
+
* Absent means measurable, which is what every record written before this field meant.
|
|
313
|
+
*/
|
|
314
|
+
siteMetricsMeasurable: z.boolean().optional(),
|
|
195
315
|
})
|
|
196
316
|
.meta({ id: "EvaluatorOutcome", title: "EvaluatorOutcome" });
|
|
197
317
|
|
|
@@ -207,7 +327,7 @@ export type EvaluatorOutcome = z.infer<typeof evaluatorOutcomeSchema>;
|
|
|
207
327
|
export const evaluatorRunSchema = z
|
|
208
328
|
.object({
|
|
209
329
|
/** Schema version of this record (ADR-0003). */
|
|
210
|
-
schemaVersion:
|
|
330
|
+
schemaVersion: recordSchemaVersion,
|
|
211
331
|
/** Which evaluator produced these outcomes. */
|
|
212
332
|
evaluatorId: identifier,
|
|
213
333
|
/** Version of the evaluator. Metrics from different versions are not comparable. */
|
|
@@ -218,7 +338,10 @@ export const evaluatorRunSchema = z
|
|
|
218
338
|
outcomes: z.array(evaluatorOutcomeSchema).max(100_000),
|
|
219
339
|
/** When the run was executed. */
|
|
220
340
|
executedAt: z.iso.datetime({ offset: true }),
|
|
341
|
+
/** @see defectCorpusSchema.metadata */
|
|
342
|
+
metadata: z.record(z.string(), z.unknown()).optional(),
|
|
221
343
|
})
|
|
344
|
+
.strict()
|
|
222
345
|
.meta({
|
|
223
346
|
id: "EvaluatorRun",
|
|
224
347
|
title: "EvaluatorRun",
|
|
@@ -231,8 +354,34 @@ export const evaluatorRunSchema = z
|
|
|
231
354
|
/** @see evaluatorRunSchema */
|
|
232
355
|
export type EvaluatorRun = z.infer<typeof evaluatorRunSchema>;
|
|
233
356
|
|
|
234
|
-
/**
|
|
235
|
-
|
|
357
|
+
/**
|
|
358
|
+
* Refuse a record declaring a version this runtime does not implement, **before** its shape is
|
|
359
|
+
* read.
|
|
360
|
+
*
|
|
361
|
+
* Only `newer` is refused. An older record parses, and {@link compareSchemaVersion} is how a
|
|
362
|
+
* caller decides whether to trust it.
|
|
363
|
+
*
|
|
364
|
+
* Version before fields, for the same reason Core's `validateRecord` orders them so: a newer record
|
|
365
|
+
* fails with one `ALDUS_SCHEMA_VERSION_UNSUPPORTED` that names the cause, rather than a field issue
|
|
366
|
+
* that describes the symptom. A missing or malformed version is not this check's to report — it
|
|
367
|
+
* falls through, and the schema reports it precisely under `ALDUS_CORPUS_MALFORMED`.
|
|
368
|
+
*/
|
|
369
|
+
function assertSchemaVersionSupported(input: unknown, record: "corpus" | "run"): void {
|
|
370
|
+
if (typeof input !== "object" || input === null) return;
|
|
371
|
+
const declared = (input as { schemaVersion?: unknown }).schemaVersion;
|
|
372
|
+
if (typeof declared !== "string" || !declaresNewerVersion(declared)) return;
|
|
373
|
+
throw regressionError(
|
|
374
|
+
RegressionErrorCodes.SCHEMA_VERSION_UNSUPPORTED,
|
|
375
|
+
`A ${record} declares a schema version newer than this runtime implements. Its fields cannot ` +
|
|
376
|
+
"be interpreted here, so accepting it would return a record that looks complete and is not.",
|
|
377
|
+
{
|
|
378
|
+
category: "validation",
|
|
379
|
+
// The runtime's own version is this package's constant, not received input, so naming it is
|
|
380
|
+
// not a value leak. The record's version is not named (contract §19.2).
|
|
381
|
+
details: { runtimeSchemaVersion: REGRESSION_SCHEMA_VERSION, relation: "newer" },
|
|
382
|
+
},
|
|
383
|
+
);
|
|
384
|
+
}
|
|
236
385
|
|
|
237
386
|
/**
|
|
238
387
|
* Validate a corpus and check the uniqueness constraint Zod cannot express.
|
|
@@ -240,6 +389,7 @@ export const REGRESSION_SCHEMA_VERSION = SCHEMA_VERSION;
|
|
|
240
389
|
* @throws {AldusError} `ALDUS_CORPUS_MALFORMED` or `ALDUS_CORPUS_DUPLICATE_CASE`.
|
|
241
390
|
*/
|
|
242
391
|
export function parseDefectCorpus(input: unknown): DefectCorpus {
|
|
392
|
+
assertSchemaVersionSupported(input, "corpus");
|
|
243
393
|
const result = defectCorpusSchema.safeParse(input);
|
|
244
394
|
if (!result.success) {
|
|
245
395
|
// Paths and issue codes only — never the received value (contract §19.2, ADR-0002).
|
|
@@ -279,6 +429,7 @@ export function parseDefectCorpus(input: unknown): DefectCorpus {
|
|
|
279
429
|
* @throws {AldusError} `ALDUS_CORPUS_MALFORMED` or `ALDUS_OUTCOME_DUPLICATE`.
|
|
280
430
|
*/
|
|
281
431
|
export function parseEvaluatorRun(input: unknown): EvaluatorRun {
|
|
432
|
+
assertSchemaVersionSupported(input, "run");
|
|
282
433
|
const result = evaluatorRunSchema.safeParse(input);
|
|
283
434
|
if (!result.success) {
|
|
284
435
|
throw regressionError(
|
package/src/errors.ts
CHANGED
|
@@ -34,6 +34,16 @@ export const RegressionErrorCodes = {
|
|
|
34
34
|
BLIND_SPOT_DUPLICATE: "ALDUS_BLIND_SPOT_DUPLICATE",
|
|
35
35
|
/** A promotion policy is internally inconsistent — a threshold outside its valid range. */
|
|
36
36
|
POLICY_INVALID: "ALDUS_POLICY_INVALID",
|
|
37
|
+
/**
|
|
38
|
+
* A record declares a schema version newer than this runtime implements.
|
|
39
|
+
*
|
|
40
|
+
* Refused rather than accepted, because this runtime cannot know what a later version's fields
|
|
41
|
+
* mean and the schemas discard what they do not declare. Accepting one returns an object that
|
|
42
|
+
* looks complete and is not — a read path answering wrongly with no signal, which is worse than
|
|
43
|
+
* refusing. Whether an *older* record is comparable is the caller's policy, not this package's:
|
|
44
|
+
* see `compareSchemaVersion`.
|
|
45
|
+
*/
|
|
46
|
+
SCHEMA_VERSION_UNSUPPORTED: "ALDUS_SCHEMA_VERSION_UNSUPPORTED",
|
|
37
47
|
} as const;
|
|
38
48
|
|
|
39
49
|
/** @see RegressionErrorCodes */
|
package/src/index.ts
CHANGED
|
@@ -22,6 +22,7 @@
|
|
|
22
22
|
export {
|
|
23
23
|
REGRESSION_SCHEMA_VERSION,
|
|
24
24
|
defectCaseSchema,
|
|
25
|
+
compareSchemaVersion,
|
|
25
26
|
defectCorpusSchema,
|
|
26
27
|
evaluatorFindingSchema,
|
|
27
28
|
evaluatorOutcomeSchema,
|
|
@@ -39,6 +40,7 @@ export {
|
|
|
39
40
|
type EvaluatorRun,
|
|
40
41
|
type HumanFinding,
|
|
41
42
|
type ScopeDimensions,
|
|
43
|
+
type SchemaVersionRelation,
|
|
42
44
|
} from "./corpus.js";
|
|
43
45
|
|
|
44
46
|
// --- Scope slicing (§12.1) --------------------------------------------------------------------
|
package/src/metrics.ts
CHANGED
|
@@ -54,6 +54,19 @@ export interface CaseComparison {
|
|
|
54
54
|
* negative — it did catch the case — but a promotion reader should see it.
|
|
55
55
|
*/
|
|
56
56
|
categoryMismatch: boolean;
|
|
57
|
+
/**
|
|
58
|
+
* Whether site-level comparison means anything for this case (#140).
|
|
59
|
+
*
|
|
60
|
+
* `false` when the evaluator flagged without enumerating what it found — a wrapped legacy
|
|
61
|
+
* evaluator reporting that it had something to say, and nothing about how much or where.
|
|
62
|
+
*
|
|
63
|
+
* The case-level verdict is still sound: a flag is a flag, and §12's precision and recall over
|
|
64
|
+
* *cases* are computable. What is not computable is anything per site, and the previous code
|
|
65
|
+
* quietly answered it anyway — with no evaluator categories to compare, `categoryMismatch` came
|
|
66
|
+
* out `false`, which reads as *the categories agreed*. An evaluator nobody measured would have
|
|
67
|
+
* scored a clean sheet.
|
|
68
|
+
*/
|
|
69
|
+
siteMetricsMeasurable: boolean;
|
|
57
70
|
/** Harm a spurious flag would cause here; zero unless this is a false positive. */
|
|
58
71
|
unnecessaryCorrectionHarm: number;
|
|
59
72
|
}
|
|
@@ -79,6 +92,14 @@ export interface SliceMetrics {
|
|
|
79
92
|
trueNegatives: number;
|
|
80
93
|
/** True positives found under the wrong category (contract §12.3). */
|
|
81
94
|
categoryMismatches: number;
|
|
95
|
+
/**
|
|
96
|
+
* Cases whose site-level metrics are unmeasurable (#140).
|
|
97
|
+
*
|
|
98
|
+
* Reported rather than folded in, because a slice where this is large is a slice whose
|
|
99
|
+
* category agreement means much less than the other numbers suggest. Zero is the honest value
|
|
100
|
+
* only when it is true.
|
|
101
|
+
*/
|
|
102
|
+
unmeasurableSiteMetrics: number;
|
|
82
103
|
/** `truePositives / defectiveCases`; `undefined` when there are no defective cases. */
|
|
83
104
|
recall: number | undefined;
|
|
84
105
|
/** `falsePositives / cleanCases`; `undefined` when there are no clean cases. */
|
|
@@ -159,6 +180,11 @@ function compareCase(
|
|
|
159
180
|
severity: entry.severity,
|
|
160
181
|
severityWeight: weight,
|
|
161
182
|
categoryMismatch,
|
|
183
|
+
// A flag with nothing enumerated. Not a contradiction to be repaired by inventing a finding —
|
|
184
|
+
// that would make one report count as one defect, which is exactly the statistic #140 exists
|
|
185
|
+
// to protect.
|
|
186
|
+
siteMetricsMeasurable:
|
|
187
|
+
outcome?.siteMetricsMeasurable ?? !(flagged && (outcome?.findings ?? []).length === 0),
|
|
162
188
|
unnecessaryCorrectionHarm: harm,
|
|
163
189
|
};
|
|
164
190
|
}
|
|
@@ -206,6 +232,8 @@ function summarise(
|
|
|
206
232
|
falsePositives,
|
|
207
233
|
trueNegatives,
|
|
208
234
|
categoryMismatches: comparisons.filter((comparison) => comparison.categoryMismatch).length,
|
|
235
|
+
unmeasurableSiteMetrics: comparisons.filter((comparison) => !comparison.siteMetricsMeasurable)
|
|
236
|
+
.length,
|
|
209
237
|
recall: defectiveCases === 0 ? undefined : truePositives / defectiveCases,
|
|
210
238
|
falsePositiveRate: cleanCases === 0 ? undefined : falsePositives / cleanCases,
|
|
211
239
|
severityWeightedRecall: defectiveWeight === 0 ? undefined : caughtWeight / defectiveWeight,
|