@aldus-runtime/regression 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -0
- package/NOTICE +21 -0
- package/dist/blindspots.d.ts +107 -0
- package/dist/blindspots.d.ts.map +1 -0
- package/dist/blindspots.js +169 -0
- package/dist/blindspots.js.map +1 -0
- package/dist/corpus.d.ts +184 -0
- package/dist/corpus.d.ts.map +1 -0
- package/dist/corpus.js +254 -0
- package/dist/corpus.js.map +1 -0
- package/dist/errors.d.ts +44 -0
- package/dist/errors.d.ts.map +1 -0
- package/dist/errors.js +40 -0
- package/dist/errors.js.map +1 -0
- package/dist/index.d.ts +28 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +36 -0
- package/dist/index.js.map +1 -0
- package/dist/metrics.d.ts +134 -0
- package/dist/metrics.d.ts.map +1 -0
- package/dist/metrics.js +125 -0
- package/dist/metrics.js.map +1 -0
- package/dist/policy.d.ts +140 -0
- package/dist/policy.d.ts.map +1 -0
- package/dist/policy.js +148 -0
- package/dist/policy.js.map +1 -0
- package/dist/promotion.d.ts +108 -0
- package/dist/promotion.d.ts.map +1 -0
- package/dist/promotion.js +130 -0
- package/dist/promotion.js.map +1 -0
- package/dist/report.d.ts +42 -0
- package/dist/report.d.ts.map +1 -0
- package/dist/report.js +116 -0
- package/dist/report.js.map +1 -0
- package/dist/scope.d.ts +57 -0
- package/dist/scope.d.ts.map +1 -0
- package/dist/scope.js +81 -0
- package/dist/scope.js.map +1 -0
- package/package.json +49 -0
- package/src/blindspots.ts +199 -0
- package/src/corpus.ts +311 -0
- package/src/errors.ts +49 -0
- package/src/index.ts +104 -0
- package/src/metrics.ts +293 -0
- package/src/policy.ts +267 -0
- package/src/promotion.ts +336 -0
- package/src/report.ts +154 -0
- package/src/scope.ts +100 -0
package/dist/corpus.d.ts
ADDED
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The defect corpus (architecture contract §22 WP-10, §12.1, §12.3, §24).
|
|
3
|
+
*
|
|
4
|
+
* §12.1 permits an evaluator to become blocking "only after it is calibrated against
|
|
5
|
+
* human-labeled examples", and §24 requires that "a representative defect corpus is executed
|
|
6
|
+
* during regression testing". This module defines what such a corpus is.
|
|
7
|
+
*
|
|
8
|
+
* Two shapes here are deliberately open rather than enumerated:
|
|
9
|
+
*
|
|
10
|
+
* - **Finding categories.** §12.3 presents its taxonomy as "for example", and §4.2 keeps
|
|
11
|
+
* adopter concepts out of the runtime. A category is an opaque caller-supplied string.
|
|
12
|
+
* - **Severity levels.** Likewise named by the caller. The *weights* live in the promotion
|
|
13
|
+
* policy, so an adopter can say a missed pronunciation error costs less than a missed
|
|
14
|
+
* unsupported claim without Core deciding that for them.
|
|
15
|
+
*
|
|
16
|
+
* The human label is the oracle (§12 level 4). Nothing in this module treats an evaluator's
|
|
17
|
+
* output as ground truth.
|
|
18
|
+
*
|
|
19
|
+
* Zod is the single source of truth; TypeScript types are inferred (ADR-0002).
|
|
20
|
+
*/
|
|
21
|
+
import { z } from "zod";
|
|
22
|
+
/**
|
|
23
|
+
* A finding category, structured per contract §12.3.
|
|
24
|
+
*
|
|
25
|
+
* An OPEN string, never a Core-defined enum. §12.3 introduces its taxonomy with "for example",
|
|
26
|
+
* and §4.2 forbids Core from owning an adopter's diagnosis vocabulary. Do not narrow this to a
|
|
27
|
+
* union.
|
|
28
|
+
*/
|
|
29
|
+
export declare const findingCategory: z.ZodString;
|
|
30
|
+
/**
|
|
31
|
+
* A severity level, named by the caller.
|
|
32
|
+
*
|
|
33
|
+
* An OPEN string for the same reason as {@link findingCategory}. Ordering and weighting are
|
|
34
|
+
* supplied by the promotion policy, not inferred from the name — a package that decided
|
|
35
|
+
* `"critical"` outranks `"major"` would be guessing at an adopter's scale.
|
|
36
|
+
*/
|
|
37
|
+
export declare const severityLevel: z.ZodString;
|
|
38
|
+
/**
|
|
39
|
+
* Scope dimensions a case belongs to (contract §12.1 "show, host, voice, model, and
|
|
40
|
+
* script-form scope").
|
|
41
|
+
*
|
|
42
|
+
* `Record<string, string>`, consistent with Knowledge Pack scope (§9.2, ADR-0006). §12.1's list
|
|
43
|
+
* is illustrative and §4.2 forbids naming a provider, so dimensions stay caller-supplied.
|
|
44
|
+
*/
|
|
45
|
+
export declare const scopeDimensions: z.ZodRecord<z.ZodString, z.ZodString>;
|
|
46
|
+
/** @see scopeDimensions */
|
|
47
|
+
export type ScopeDimensions = z.infer<typeof scopeDimensions>;
|
|
48
|
+
/** One defect a human labeller identified in a case. */
|
|
49
|
+
export declare const humanFindingSchema: z.ZodObject<{
|
|
50
|
+
category: z.ZodString;
|
|
51
|
+
severity: z.ZodString;
|
|
52
|
+
note: z.ZodOptional<z.ZodString>;
|
|
53
|
+
}, z.core.$strip>;
|
|
54
|
+
/** @see humanFindingSchema */
|
|
55
|
+
export type HumanFinding = z.infer<typeof humanFindingSchema>;
|
|
56
|
+
/**
|
|
57
|
+
* One labelled case: an input, and what a human said about it.
|
|
58
|
+
*
|
|
59
|
+
* The case does not carry the input itself. §8.1 makes artifacts addressable by ID and hash, and
|
|
60
|
+
* a corpus that embedded audio or scripts would be unreviewable and would risk carrying private
|
|
61
|
+
* source material into a test fixture (§19.2). `subjectRef` points at the subject; what it
|
|
62
|
+
* points into is the caller's business.
|
|
63
|
+
*/
|
|
64
|
+
export declare const defectCaseSchema: z.ZodObject<{
|
|
65
|
+
caseId: z.ZodString;
|
|
66
|
+
subjectRef: z.ZodString;
|
|
67
|
+
scope: z.ZodRecord<z.ZodString, z.ZodString>;
|
|
68
|
+
defective: z.ZodBoolean;
|
|
69
|
+
findings: z.ZodArray<z.ZodObject<{
|
|
70
|
+
category: z.ZodString;
|
|
71
|
+
severity: z.ZodString;
|
|
72
|
+
note: z.ZodOptional<z.ZodString>;
|
|
73
|
+
}, z.core.$strip>>;
|
|
74
|
+
severity: z.ZodOptional<z.ZodString>;
|
|
75
|
+
correctionOnFlag: z.ZodOptional<z.ZodString>;
|
|
76
|
+
labelledBy: z.ZodObject<{
|
|
77
|
+
kind: z.ZodEnum<{
|
|
78
|
+
agent: "agent";
|
|
79
|
+
human: "human";
|
|
80
|
+
system: "system";
|
|
81
|
+
worker: "worker";
|
|
82
|
+
}>;
|
|
83
|
+
id: z.ZodString;
|
|
84
|
+
displayName: z.ZodOptional<z.ZodString>;
|
|
85
|
+
backendId: z.ZodOptional<z.ZodString>;
|
|
86
|
+
sessionRef: z.ZodOptional<z.ZodString>;
|
|
87
|
+
}, z.core.$strip>;
|
|
88
|
+
labelledAt: z.ZodISODateTime;
|
|
89
|
+
}, z.core.$strip>;
|
|
90
|
+
/** @see defectCaseSchema */
|
|
91
|
+
export type DefectCase = z.infer<typeof defectCaseSchema>;
|
|
92
|
+
/** A named, versioned set of labelled cases. */
|
|
93
|
+
export declare const defectCorpusSchema: z.ZodObject<{
|
|
94
|
+
schemaVersion: z.ZodString;
|
|
95
|
+
corpusId: z.ZodString;
|
|
96
|
+
description: z.ZodOptional<z.ZodString>;
|
|
97
|
+
cases: z.ZodArray<z.ZodObject<{
|
|
98
|
+
caseId: z.ZodString;
|
|
99
|
+
subjectRef: z.ZodString;
|
|
100
|
+
scope: z.ZodRecord<z.ZodString, z.ZodString>;
|
|
101
|
+
defective: z.ZodBoolean;
|
|
102
|
+
findings: z.ZodArray<z.ZodObject<{
|
|
103
|
+
category: z.ZodString;
|
|
104
|
+
severity: z.ZodString;
|
|
105
|
+
note: z.ZodOptional<z.ZodString>;
|
|
106
|
+
}, z.core.$strip>>;
|
|
107
|
+
severity: z.ZodOptional<z.ZodString>;
|
|
108
|
+
correctionOnFlag: z.ZodOptional<z.ZodString>;
|
|
109
|
+
labelledBy: z.ZodObject<{
|
|
110
|
+
kind: z.ZodEnum<{
|
|
111
|
+
agent: "agent";
|
|
112
|
+
human: "human";
|
|
113
|
+
system: "system";
|
|
114
|
+
worker: "worker";
|
|
115
|
+
}>;
|
|
116
|
+
id: z.ZodString;
|
|
117
|
+
displayName: z.ZodOptional<z.ZodString>;
|
|
118
|
+
backendId: z.ZodOptional<z.ZodString>;
|
|
119
|
+
sessionRef: z.ZodOptional<z.ZodString>;
|
|
120
|
+
}, z.core.$strip>;
|
|
121
|
+
labelledAt: z.ZodISODateTime;
|
|
122
|
+
}, z.core.$strip>>;
|
|
123
|
+
}, z.core.$strip>;
|
|
124
|
+
/** @see defectCorpusSchema */
|
|
125
|
+
export type DefectCorpus = z.infer<typeof defectCorpusSchema>;
|
|
126
|
+
/** One finding an evaluator reported. */
|
|
127
|
+
export declare const evaluatorFindingSchema: z.ZodObject<{
|
|
128
|
+
category: z.ZodString;
|
|
129
|
+
severity: z.ZodOptional<z.ZodString>;
|
|
130
|
+
confidence: z.ZodOptional<z.ZodNumber>;
|
|
131
|
+
}, z.core.$strip>;
|
|
132
|
+
/** @see evaluatorFindingSchema */
|
|
133
|
+
export type EvaluatorFinding = z.infer<typeof evaluatorFindingSchema>;
|
|
134
|
+
/** What an evaluator said about one case. */
|
|
135
|
+
export declare const evaluatorOutcomeSchema: z.ZodObject<{
|
|
136
|
+
caseId: z.ZodString;
|
|
137
|
+
flagged: z.ZodBoolean;
|
|
138
|
+
findings: z.ZodArray<z.ZodObject<{
|
|
139
|
+
category: z.ZodString;
|
|
140
|
+
severity: z.ZodOptional<z.ZodString>;
|
|
141
|
+
confidence: z.ZodOptional<z.ZodNumber>;
|
|
142
|
+
}, z.core.$strip>>;
|
|
143
|
+
}, z.core.$strip>;
|
|
144
|
+
/** @see evaluatorOutcomeSchema */
|
|
145
|
+
export type EvaluatorOutcome = z.infer<typeof evaluatorOutcomeSchema>;
|
|
146
|
+
/**
|
|
147
|
+
* One evaluator's outcomes over one corpus.
|
|
148
|
+
*
|
|
149
|
+
* This package does **not** run evaluators (contract §22 WP-10 scope). A run is produced
|
|
150
|
+
* elsewhere and handed here for comparison.
|
|
151
|
+
*/
|
|
152
|
+
export declare const evaluatorRunSchema: z.ZodObject<{
|
|
153
|
+
schemaVersion: z.ZodString;
|
|
154
|
+
evaluatorId: z.ZodString;
|
|
155
|
+
evaluatorVersion: z.ZodString;
|
|
156
|
+
corpusId: z.ZodString;
|
|
157
|
+
outcomes: z.ZodArray<z.ZodObject<{
|
|
158
|
+
caseId: z.ZodString;
|
|
159
|
+
flagged: z.ZodBoolean;
|
|
160
|
+
findings: z.ZodArray<z.ZodObject<{
|
|
161
|
+
category: z.ZodString;
|
|
162
|
+
severity: z.ZodOptional<z.ZodString>;
|
|
163
|
+
confidence: z.ZodOptional<z.ZodNumber>;
|
|
164
|
+
}, z.core.$strip>>;
|
|
165
|
+
}, z.core.$strip>>;
|
|
166
|
+
executedAt: z.ZodISODateTime;
|
|
167
|
+
}, z.core.$strip>;
|
|
168
|
+
/** @see evaluatorRunSchema */
|
|
169
|
+
export type EvaluatorRun = z.infer<typeof evaluatorRunSchema>;
|
|
170
|
+
/** Schema version this package stamps on records it constructs. */
|
|
171
|
+
export declare const REGRESSION_SCHEMA_VERSION = "1.2";
|
|
172
|
+
/**
|
|
173
|
+
* Validate a corpus and check the uniqueness constraint Zod cannot express.
|
|
174
|
+
*
|
|
175
|
+
* @throws {AldusError} `ALDUS_CORPUS_MALFORMED` or `ALDUS_CORPUS_DUPLICATE_CASE`.
|
|
176
|
+
*/
|
|
177
|
+
export declare function parseDefectCorpus(input: unknown): DefectCorpus;
|
|
178
|
+
/**
|
|
179
|
+
* Validate an evaluator run and check uniqueness.
|
|
180
|
+
*
|
|
181
|
+
* @throws {AldusError} `ALDUS_CORPUS_MALFORMED` or `ALDUS_OUTCOME_DUPLICATE`.
|
|
182
|
+
*/
|
|
183
|
+
export declare function parseEvaluatorRun(input: unknown): EvaluatorRun;
|
|
184
|
+
//# sourceMappingURL=corpus.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"corpus.d.ts","sourceRoot":"","sources":["../src/corpus.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAGH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAOxB;;;;;;GAMG;AACH,eAAO,MAAM,eAAe,aAA6B,CAAC;AAE1D;;;;;;GAMG;AACH,eAAO,MAAM,aAAa,aAA6B,CAAC;AAExD;;;;;;GAMG;AACH,eAAO,MAAM,eAAe,uCAAmE,CAAC;AAEhG,2BAA2B;AAC3B,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AAE9D,wDAAwD;AACxD,eAAO,MAAM,kBAAkB;;;;iBASuB,CAAC;AAEvD,8BAA8B;AAC9B,MAAM,MAAM,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,kBAAkB,CAAC,CAAC;AAE9D;;;;;;;GAOG;AACH,eAAO,MAAM,gBAAgB;;;;;;;;;;;;;;;;;;;;;;;;;iBA2DzB,CAAC;AAEL,4BAA4B;AAC5B,MAAM,MAAM,UAAU,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,gBAAgB,CAAC,CAAC;AAE1D,gDAAgD;AAChD,eAAO,MAAM,kBAAkB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAkB3B,CAAC;AAEL,8BAA8B;AAC9B,MAAM,MAAM,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,kBAAkB,CAAC,CAAC;AAE9D,yCAAyC;AACzC,eAAO,MAAM,sBAAsB;;;;iBAS2B,CAAC;AAE/D,kCAAkC;AAClC,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,sBAAsB,CAAC,CAAC;AAEtE,6CAA6C;AAC7C,eAAO,MAAM,sBAAsB;;;;;;;;iBAS2B,CAAC;AAE/D,kCAAkC;AAClC,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,sBAAsB,CAAC,CAAC;AAEtE;;;;;GAKG;AACH,eAAO,MAAM,kBAAkB;;;;;;;;;;;;;;;iBAsB3B,CAAC;AAEL,8BAA8B;AAC9B,MAAM,MAAM,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,kBAAkB,CAAC,CAAC;AAE9D,mEAAmE;AACnE,eAAO,MAAM,yBAAyB,QAAiB,CAAC;AAExD;;;;GAIG;AACH,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,OAAO,GAAG,YAAY,CAgC9D;AAED;;;;GAIG;AACH,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,OAAO,GAAG,YAAY,CA8B9D"}
|
package/dist/corpus.js
ADDED
|
@@ -0,0 +1,254 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The defect corpus (architecture contract §22 WP-10, §12.1, §12.3, §24).
|
|
3
|
+
*
|
|
4
|
+
* §12.1 permits an evaluator to become blocking "only after it is calibrated against
|
|
5
|
+
* human-labeled examples", and §24 requires that "a representative defect corpus is executed
|
|
6
|
+
* during regression testing". This module defines what such a corpus is.
|
|
7
|
+
*
|
|
8
|
+
* Two shapes here are deliberately open rather than enumerated:
|
|
9
|
+
*
|
|
10
|
+
* - **Finding categories.** §12.3 presents its taxonomy as "for example", and §4.2 keeps
|
|
11
|
+
* adopter concepts out of the runtime. A category is an opaque caller-supplied string.
|
|
12
|
+
* - **Severity levels.** Likewise named by the caller. The *weights* live in the promotion
|
|
13
|
+
* policy, so an adopter can say a missed pronunciation error costs less than a missed
|
|
14
|
+
* unsupported claim without Core deciding that for them.
|
|
15
|
+
*
|
|
16
|
+
* The human label is the oracle (§12 level 4). Nothing in this module treats an evaluator's
|
|
17
|
+
* output as ground truth.
|
|
18
|
+
*
|
|
19
|
+
* Zod is the single source of truth; TypeScript types are inferred (ADR-0002).
|
|
20
|
+
*/
|
|
21
|
+
import { actorRefSchema, SCHEMA_VERSION } from "@aldus-runtime/core";
|
|
22
|
+
import { z } from "zod";
|
|
23
|
+
import { RegressionErrorCodes, regressionError } from "./errors.js";
|
|
24
|
+
/** A short opaque identifier. */
|
|
25
|
+
const identifier = z.string().min(1).max(200);
|
|
26
|
+
/**
|
|
27
|
+
* A finding category, structured per contract §12.3.
|
|
28
|
+
*
|
|
29
|
+
* An OPEN string, never a Core-defined enum. §12.3 introduces its taxonomy with "for example",
|
|
30
|
+
* and §4.2 forbids Core from owning an adopter's diagnosis vocabulary. Do not narrow this to a
|
|
31
|
+
* union.
|
|
32
|
+
*/
|
|
33
|
+
export const findingCategory = z.string().min(1).max(200);
|
|
34
|
+
/**
|
|
35
|
+
* A severity level, named by the caller.
|
|
36
|
+
*
|
|
37
|
+
* An OPEN string for the same reason as {@link findingCategory}. Ordering and weighting are
|
|
38
|
+
* supplied by the promotion policy, not inferred from the name — a package that decided
|
|
39
|
+
* `"critical"` outranks `"major"` would be guessing at an adopter's scale.
|
|
40
|
+
*/
|
|
41
|
+
export const severityLevel = z.string().min(1).max(100);
|
|
42
|
+
/**
|
|
43
|
+
* Scope dimensions a case belongs to (contract §12.1 "show, host, voice, model, and
|
|
44
|
+
* script-form scope").
|
|
45
|
+
*
|
|
46
|
+
* `Record<string, string>`, consistent with Knowledge Pack scope (§9.2, ADR-0006). §12.1's list
|
|
47
|
+
* is illustrative and §4.2 forbids naming a provider, so dimensions stay caller-supplied.
|
|
48
|
+
*/
|
|
49
|
+
export const scopeDimensions = z.record(z.string().min(1).max(100), z.string().min(1).max(200));
|
|
50
|
+
/** One defect a human labeller identified in a case. */
|
|
51
|
+
export const humanFindingSchema = z
|
|
52
|
+
.object({
|
|
53
|
+
/** Category from the adopter's taxonomy (contract §12.3). */
|
|
54
|
+
category: findingCategory,
|
|
55
|
+
/** Severity from the adopter's scale; weighted by the promotion policy. */
|
|
56
|
+
severity: severityLevel,
|
|
57
|
+
/** What the labeller saw. Prose, never parsed. */
|
|
58
|
+
note: z.string().max(4000).optional(),
|
|
59
|
+
})
|
|
60
|
+
.meta({ id: "HumanFinding", title: "HumanFinding" });
|
|
61
|
+
/**
|
|
62
|
+
* One labelled case: an input, and what a human said about it.
|
|
63
|
+
*
|
|
64
|
+
* The case does not carry the input itself. §8.1 makes artifacts addressable by ID and hash, and
|
|
65
|
+
* a corpus that embedded audio or scripts would be unreviewable and would risk carrying private
|
|
66
|
+
* source material into a test fixture (§19.2). `subjectRef` points at the subject; what it
|
|
67
|
+
* points into is the caller's business.
|
|
68
|
+
*/
|
|
69
|
+
export const defectCaseSchema = z
|
|
70
|
+
.object({
|
|
71
|
+
/** Identity of this case within its corpus. */
|
|
72
|
+
caseId: identifier,
|
|
73
|
+
/** Reference to the material under test — an artifact ID, hash, or adopter-defined locator. */
|
|
74
|
+
subjectRef: z.string().min(1).max(1024),
|
|
75
|
+
/** Scope this case belongs to (contract §12.1). */
|
|
76
|
+
scope: scopeDimensions,
|
|
77
|
+
/**
|
|
78
|
+
* Whether a human judged the subject defective. **This is the oracle** (§12 level 4).
|
|
79
|
+
*
|
|
80
|
+
* Stored separately from `findings` so a case can be labelled clean explicitly. An empty
|
|
81
|
+
* findings array on a case nobody reviewed is not the same claim as "a human looked and
|
|
82
|
+
* found nothing", and conflating them would inflate the true-negative count with unreviewed
|
|
83
|
+
* material.
|
|
84
|
+
*/
|
|
85
|
+
defective: z.boolean(),
|
|
86
|
+
/** What the labeller found. Empty when `defective` is false. */
|
|
87
|
+
findings: z.array(humanFindingSchema).max(256),
|
|
88
|
+
/**
|
|
89
|
+
* Overall severity of the case, used for severity-weighted metrics (contract §12.1).
|
|
90
|
+
*
|
|
91
|
+
* Required when `defective`, absent otherwise — enforced by refinement below.
|
|
92
|
+
*/
|
|
93
|
+
severity: severityLevel.optional(),
|
|
94
|
+
/**
|
|
95
|
+
* Correction class this case would trigger if a blocking evaluator flagged it.
|
|
96
|
+
*
|
|
97
|
+
* §12.1 names "asymmetric harm caused by unnecessary automatic correction" as its own
|
|
98
|
+
* consideration, and §12.4 makes the repair layer explicit — regenerating one TTS segment
|
|
99
|
+
* and revising narration with cascading approval invalidation are not the same act. This
|
|
100
|
+
* names which one a flag would trigger here; the policy assigns the harm weight. Absent
|
|
101
|
+
* means the policy's default class applies.
|
|
102
|
+
*/
|
|
103
|
+
correctionOnFlag: identifier.optional(),
|
|
104
|
+
/** Who labelled the case (contract §19.2: a decision without an actor is not a decision). */
|
|
105
|
+
labelledBy: actorRefSchema,
|
|
106
|
+
/** When the label was recorded. */
|
|
107
|
+
labelledAt: z.iso.datetime({ offset: true }),
|
|
108
|
+
})
|
|
109
|
+
.refine((value) => value.defective === (value.severity !== undefined), {
|
|
110
|
+
message: "a defective case must carry a severity, and a clean case must not (architecture contract §12.1 requires severity-weighted metrics)",
|
|
111
|
+
path: ["severity"],
|
|
112
|
+
})
|
|
113
|
+
.refine((value) => value.defective || value.findings.length === 0, {
|
|
114
|
+
message: "a case labelled clean must carry no findings",
|
|
115
|
+
path: ["findings"],
|
|
116
|
+
})
|
|
117
|
+
.meta({
|
|
118
|
+
id: "DefectCase",
|
|
119
|
+
title: "DefectCase",
|
|
120
|
+
description: "One human-labelled case in a defect corpus (architecture contract §12.1, §24). The human " +
|
|
121
|
+
"label is the oracle; nothing treats an evaluator's output as ground truth. `category` and " +
|
|
122
|
+
"`severity` are open strings because §12.3's taxonomy is illustrative and §4.2 keeps " +
|
|
123
|
+
"adopter vocabularies out of the runtime. ADDITIONAL CONSTRAINTS NOT EXPRESSIBLE IN JSON " +
|
|
124
|
+
"SCHEMA: a defective case must carry a severity and a clean case must not, and a clean " +
|
|
125
|
+
"case must carry no findings.",
|
|
126
|
+
});
|
|
127
|
+
/** A named, versioned set of labelled cases. */
|
|
128
|
+
export const defectCorpusSchema = z
|
|
129
|
+
.object({
|
|
130
|
+
/** Schema version of this record (ADR-0003). */
|
|
131
|
+
schemaVersion: z.string().regex(/^(0|[1-9]\d*)\.(0|[1-9]\d*)$/),
|
|
132
|
+
/** Identity of this corpus. */
|
|
133
|
+
corpusId: identifier,
|
|
134
|
+
/** Human-readable purpose. */
|
|
135
|
+
description: z.string().max(2000).optional(),
|
|
136
|
+
/** The labelled cases. */
|
|
137
|
+
cases: z.array(defectCaseSchema).max(100_000),
|
|
138
|
+
})
|
|
139
|
+
.meta({
|
|
140
|
+
id: "DefectCorpus",
|
|
141
|
+
title: "DefectCorpus",
|
|
142
|
+
description: "A defect corpus: human-labelled cases an evaluator is calibrated against (architecture " +
|
|
143
|
+
"contract §12.1, §24). ADDITIONAL CONSTRAINT NOT EXPRESSIBLE IN JSON SCHEMA: `caseId` " +
|
|
144
|
+
"must be unique within the corpus.",
|
|
145
|
+
});
|
|
146
|
+
/** One finding an evaluator reported. */
|
|
147
|
+
export const evaluatorFindingSchema = z
|
|
148
|
+
.object({
|
|
149
|
+
/** Category the evaluator assigned, from the same taxonomy as {@link humanFindingSchema}. */
|
|
150
|
+
category: findingCategory,
|
|
151
|
+
/** Severity the evaluator assigned, where it assigns one. */
|
|
152
|
+
severity: severityLevel.optional(),
|
|
153
|
+
/** Evaluator confidence in `[0, 1]`, where it reports one. */
|
|
154
|
+
confidence: z.number().min(0).max(1).optional(),
|
|
155
|
+
})
|
|
156
|
+
.meta({ id: "EvaluatorFinding", title: "EvaluatorFinding" });
|
|
157
|
+
/** What an evaluator said about one case. */
|
|
158
|
+
export const evaluatorOutcomeSchema = z
|
|
159
|
+
.object({
|
|
160
|
+
/** Case this outcome reports on. */
|
|
161
|
+
caseId: identifier,
|
|
162
|
+
/** Whether the evaluator flagged the subject. */
|
|
163
|
+
flagged: z.boolean(),
|
|
164
|
+
/** What it reported. Empty when not flagged. */
|
|
165
|
+
findings: z.array(evaluatorFindingSchema).max(256),
|
|
166
|
+
})
|
|
167
|
+
.meta({ id: "EvaluatorOutcome", title: "EvaluatorOutcome" });
|
|
168
|
+
/**
|
|
169
|
+
* One evaluator's outcomes over one corpus.
|
|
170
|
+
*
|
|
171
|
+
* This package does **not** run evaluators (contract §22 WP-10 scope). A run is produced
|
|
172
|
+
* elsewhere and handed here for comparison.
|
|
173
|
+
*/
|
|
174
|
+
export const evaluatorRunSchema = z
|
|
175
|
+
.object({
|
|
176
|
+
/** Schema version of this record (ADR-0003). */
|
|
177
|
+
schemaVersion: z.string().regex(/^(0|[1-9]\d*)\.(0|[1-9]\d*)$/),
|
|
178
|
+
/** Which evaluator produced these outcomes. */
|
|
179
|
+
evaluatorId: identifier,
|
|
180
|
+
/** Version of the evaluator. Metrics from different versions are not comparable. */
|
|
181
|
+
evaluatorVersion: identifier,
|
|
182
|
+
/** Corpus the outcomes are against. */
|
|
183
|
+
corpusId: identifier,
|
|
184
|
+
/** The outcomes. */
|
|
185
|
+
outcomes: z.array(evaluatorOutcomeSchema).max(100_000),
|
|
186
|
+
/** When the run was executed. */
|
|
187
|
+
executedAt: z.iso.datetime({ offset: true }),
|
|
188
|
+
})
|
|
189
|
+
.meta({
|
|
190
|
+
id: "EvaluatorRun",
|
|
191
|
+
title: "EvaluatorRun",
|
|
192
|
+
description: "One evaluator's outcomes over one defect corpus (architecture contract §22 WP-10). This " +
|
|
193
|
+
"package consumes runs; it does not execute evaluators. ADDITIONAL CONSTRAINT NOT " +
|
|
194
|
+
"EXPRESSIBLE IN JSON SCHEMA: `caseId` must be unique within `outcomes`.",
|
|
195
|
+
});
|
|
196
|
+
/** Schema version this package stamps on records it constructs. */
|
|
197
|
+
export const REGRESSION_SCHEMA_VERSION = SCHEMA_VERSION;
|
|
198
|
+
/**
|
|
199
|
+
* Validate a corpus and check the uniqueness constraint Zod cannot express.
|
|
200
|
+
*
|
|
201
|
+
* @throws {AldusError} `ALDUS_CORPUS_MALFORMED` or `ALDUS_CORPUS_DUPLICATE_CASE`.
|
|
202
|
+
*/
|
|
203
|
+
export function parseDefectCorpus(input) {
|
|
204
|
+
const result = defectCorpusSchema.safeParse(input);
|
|
205
|
+
if (!result.success) {
|
|
206
|
+
// Paths and issue codes only — never the received value (contract §19.2, ADR-0002).
|
|
207
|
+
throw regressionError(RegressionErrorCodes.CORPUS_MALFORMED, "The defect corpus does not validate.", {
|
|
208
|
+
category: "validation",
|
|
209
|
+
details: {
|
|
210
|
+
issues: result.error.issues.map((issue) => ({
|
|
211
|
+
path: issue.path.join("."),
|
|
212
|
+
code: issue.code,
|
|
213
|
+
})),
|
|
214
|
+
},
|
|
215
|
+
});
|
|
216
|
+
}
|
|
217
|
+
const seen = new Set();
|
|
218
|
+
for (const entry of result.data.cases) {
|
|
219
|
+
if (seen.has(entry.caseId)) {
|
|
220
|
+
throw regressionError(RegressionErrorCodes.CORPUS_DUPLICATE_CASE, `Corpus "${result.data.corpusId}" contains more than one case with id "${entry.caseId}". ` +
|
|
221
|
+
"Duplicate ids would let one case be counted twice and silently reweight every metric.", { category: "validation", details: { corpusId: result.data.corpusId } });
|
|
222
|
+
}
|
|
223
|
+
seen.add(entry.caseId);
|
|
224
|
+
}
|
|
225
|
+
return result.data;
|
|
226
|
+
}
|
|
227
|
+
/**
|
|
228
|
+
* Validate an evaluator run and check uniqueness.
|
|
229
|
+
*
|
|
230
|
+
* @throws {AldusError} `ALDUS_CORPUS_MALFORMED` or `ALDUS_OUTCOME_DUPLICATE`.
|
|
231
|
+
*/
|
|
232
|
+
export function parseEvaluatorRun(input) {
|
|
233
|
+
const result = evaluatorRunSchema.safeParse(input);
|
|
234
|
+
if (!result.success) {
|
|
235
|
+
throw regressionError(RegressionErrorCodes.CORPUS_MALFORMED, "The evaluator run does not validate.", {
|
|
236
|
+
category: "validation",
|
|
237
|
+
details: {
|
|
238
|
+
issues: result.error.issues.map((issue) => ({
|
|
239
|
+
path: issue.path.join("."),
|
|
240
|
+
code: issue.code,
|
|
241
|
+
})),
|
|
242
|
+
},
|
|
243
|
+
});
|
|
244
|
+
}
|
|
245
|
+
const seen = new Set();
|
|
246
|
+
for (const outcome of result.data.outcomes) {
|
|
247
|
+
if (seen.has(outcome.caseId)) {
|
|
248
|
+
throw regressionError(RegressionErrorCodes.OUTCOME_DUPLICATE, `Evaluator run for "${result.data.evaluatorId}" reports twice on case "${outcome.caseId}".`, { category: "validation", details: { evaluatorId: result.data.evaluatorId } });
|
|
249
|
+
}
|
|
250
|
+
seen.add(outcome.caseId);
|
|
251
|
+
}
|
|
252
|
+
return result.data;
|
|
253
|
+
}
|
|
254
|
+
//# sourceMappingURL=corpus.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"corpus.js","sourceRoot":"","sources":["../src/corpus.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,OAAO,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AACrE,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,oBAAoB,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAEpE,iCAAiC;AACjC,MAAM,UAAU,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;AAE9C;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;AAE1D;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;AAExD;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;AAKhG,wDAAwD;AACxD,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC;KAChC,MAAM,CAAC;IACN,6DAA6D;IAC7D,QAAQ,EAAE,eAAe;IACzB,2EAA2E;IAC3E,QAAQ,EAAE,aAAa;IACvB,kDAAkD;IAClD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE;CACtC,CAAC;KACD,IAAI,CAAC,EAAE,EAAE,EAAE,cAAc,EAAE,KAAK,EAAE,cAAc,EAAE,CAAC,CAAC;AAKvD;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC;KAC9B,MAAM,CAAC;IACN,+CAA+C;IAC/C,MAAM,EAAE,UAAU;IAClB,+FAA+F;IAC/F,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC;IACvC,mDAAmD;IACnD,KAAK,EAAE,eAAe;IACtB;;;;;;;OAOG;IACH,SAAS,EAAE,CAAC,CAAC,OAAO,EAAE;IACtB,gEAAgE;IAChE,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC;IAC9C;;;;OAIG;IACH,QAAQ,EAAE,aAAa,CAAC,QAAQ,EAAE;IAClC;;;;;;;;OAQG;IACH,gBAAgB,EAAE,UAAU,CAAC,QAAQ,EAAE;IACvC,6FAA6F;IAC7F,UAAU,EAAE,cAAc;IAC1B,mCAAmC;IACnC,UAAU,EAAE,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;CAC7C,CAAC;KACD,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,SAAS,KAAK,CAAC,KAAK,CAAC,QAAQ,KAAK,SAAS,CAAC,EAAE;IACrE,OAAO,EACL,oIAAoI;IACtI,IAAI,EAAE,CAAC,UAAU,CAAC;CACnB,CAAC;KACD,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,SAAS,IAAI,KAAK,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE;IACjE,OAAO,EAAE,8CAA8C;IACvD,IAAI,EAAE,CAAC,UAAU,CAAC;CACnB,CAAC;KACD,IAAI,CAAC;IACJ,EAAE,EAAE,YAAY;IAChB,KAAK,EAAE,YAAY;IACnB,WAAW,EACT,2FAA2F;QAC3F,4FAA4F;QAC5F,sFAAsF;QACtF,0FAA0F;QAC1F,wFAAwF;QACxF,8BAA8B;CACjC,CAAC,CAAC;AAKL,gDAAgD;AAChD,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC;KAChC,MAAM,CAAC;IACN,gDAAgD;IAChD,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,8BAA8B,CAAC;IAC/D,+BAA+B;IAC/B,QAAQ,EAAE,UAAU;IACpB,8BAA8B;IAC9B,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE;IAC5C,0BAA0B;IAC1B,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC;CAC9C,CAAC;KACD,IAAI,CAAC;IACJ,EAAE,EAAE,cAAc;IAClB,KAAK,EAAE,cAAc;IACrB,WAAW,EACT,yFAAyF;QACzF,uFAAuF;QACvF,mCAAmC;CACtC,CAAC,CAAC;AAKL,yCAAyC;AACzC,MAAM,CAAC,MAAM,sBAAsB,GAAG,CAAC;KACpC,MAAM,CAAC;IACN,6FAA6F;IAC7F,QAAQ,EAAE,eAAe;IACzB,6DAA6D;IAC7D,QAAQ,EAAE,aAAa,CAAC,QAAQ,EAAE;IAClC,8DAA8D;IAC9D,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;CAChD,CAAC;KACD,IAAI,CAAC,EAAE,EAAE,EAAE,kBAAkB,EAAE,KAAK,EAAE,kBAAkB,EAAE,CAAC,CAAC;AAK/D,6CAA6C;AAC7C,MAAM,CAAC,MAAM,sBAAsB,GAAG,CAAC;KACpC,MAAM,CAAC;IACN,oCAAoC;IACpC,MAAM,EAAE,UAAU;IAClB,iDAAiD;IACjD,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE;IACpB,gDAAgD;IAChD,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC;CACnD,CAAC;KACD,IAAI,CAAC,EAAE,EAAE,EAAE,kBAAkB,EAAE,KAAK,EAAE,kBAAkB,EAAE,CAAC,CAAC;AAK/D;;;;;GAKG;AACH,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC;KAChC,MAAM,CAAC;IACN,gDAAgD;IAChD,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,8BAA8B,CAAC;IAC/D,+CAA+C;IAC/C,WAAW,EAAE,UAAU;IACvB,oFAAoF;IACpF,gBAAgB,EAAE,UAAU;IAC5B,uCAAuC;IACvC,QAAQ,EAAE,UAAU;IACpB,oBAAoB;IACpB,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC;IACtD,iCAAiC;IACjC,UAAU,EAAE,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;CAC7C,CAAC;KACD,IAAI,CAAC;IACJ,EAAE,EAAE,cAAc;IAClB,KAAK,EAAE,cAAc;IACrB,WAAW,EACT,0FAA0F;QAC1F,mFAAmF;QACnF,wEAAwE;CAC3E,CAAC,CAAC;AAKL,mEAAmE;AACnE,MAAM,CAAC,MAAM,yBAAyB,GAAG,cAAc,CAAC;AAExD;;;;GAIG;AACH,MAAM,UAAU,iBAAiB,CAAC,KAAc;IAC9C,MAAM,MAAM,GAAG,kBAAkB,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;IACnD,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,oFAAoF;QACpF,MAAM,eAAe,CACnB,oBAAoB,CAAC,gBAAgB,EACrC,sCAAsC,EACtC;YACE,QAAQ,EAAE,YAAY;YACtB,OAAO,EAAE;gBACP,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;oBAC1C,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC;oBAC1B,IAAI,EAAE,KAAK,CAAC,IAAI;iBACjB,CAAC,CAAC;aACJ;SACF,CACF,CAAC;IACJ,CAAC;IAED,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;QACtC,IAAI,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC;YAC3B,MAAM,eAAe,CACnB,oBAAoB,CAAC,qBAAqB,EAC1C,WAAW,MAAM,CAAC,IAAI,CAAC,QAAQ,0CAA0C,KAAK,CAAC,MAAM,KAAK;gBACxF,uFAAuF,EACzF,EAAE,QAAQ,EAAE,YAAY,EAAE,OAAO,EAAE,EAAE,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,CACxE,CAAC;QACJ,CAAC;QACD,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IACzB,CAAC;IACD,OAAO,MAAM,CAAC,IAAI,CAAC;AACrB,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,iBAAiB,CAAC,KAAc;IAC9C,MAAM,MAAM,GAAG,kBAAkB,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;IACnD,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,MAAM,eAAe,CACnB,oBAAoB,CAAC,gBAAgB,EACrC,sCAAsC,EACtC;YACE,QAAQ,EAAE,YAAY;YACtB,OAAO,EAAE;gBACP,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;oBAC1C,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC;oBAC1B,IAAI,EAAE,KAAK,CAAC,IAAI;iBACjB,CAAC,CAAC;aACJ;SACF,CACF,CAAC;IACJ,CAAC;IAED,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,KAAK,MAAM,OAAO,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;QAC3C,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;YAC7B,MAAM,eAAe,CACnB,oBAAoB,CAAC,iBAAiB,EACtC,sBAAsB,MAAM,CAAC,IAAI,CAAC,WAAW,4BAA4B,OAAO,CAAC,MAAM,IAAI,EAC3F,EAAE,QAAQ,EAAE,YAAY,EAAE,OAAO,EAAE,EAAE,WAAW,EAAE,MAAM,CAAC,IAAI,CAAC,WAAW,EAAE,EAAE,CAC9E,CAAC;QACJ,CAAC;QACD,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IAC3B,CAAC;IACD,OAAO,MAAM,CAAC,IAAI,CAAC;AACrB,CAAC"}
|
package/dist/errors.d.ts
ADDED
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Failures specific to the regression harness.
|
|
3
|
+
*
|
|
4
|
+
* Aldus Core deliberately keeps no central error-code registry, so a package can name a new
|
|
5
|
+
* failure without forking Core. These codes carry the same `ALDUS_` prefix and
|
|
6
|
+
* `SCREAMING_SNAKE_CASE` shape so production trace (contract §20) stays uniform across packages.
|
|
7
|
+
*/
|
|
8
|
+
import { AldusError, type ErrorCategory } from "@aldus-runtime/core";
|
|
9
|
+
/** Error codes raised by the regression harness. */
|
|
10
|
+
export declare const RegressionErrorCodes: {
|
|
11
|
+
/** A defect corpus contained a case that does not validate. */
|
|
12
|
+
readonly CORPUS_MALFORMED: "ALDUS_CORPUS_MALFORMED";
|
|
13
|
+
/** Two cases in one corpus share a `caseId`. */
|
|
14
|
+
readonly CORPUS_DUPLICATE_CASE: "ALDUS_CORPUS_DUPLICATE_CASE";
|
|
15
|
+
/** An evaluator outcome referenced a `caseId` the corpus does not contain. */
|
|
16
|
+
readonly OUTCOME_UNKNOWN_CASE: "ALDUS_OUTCOME_UNKNOWN_CASE";
|
|
17
|
+
/** Two outcomes in one run reported on the same `caseId`. */
|
|
18
|
+
readonly OUTCOME_DUPLICATE: "ALDUS_OUTCOME_DUPLICATE";
|
|
19
|
+
/**
|
|
20
|
+
* A case carries a severity the policy assigns no weight to.
|
|
21
|
+
*
|
|
22
|
+
* Refused rather than defaulted: silently weighting an unknown severity as zero would drop a
|
|
23
|
+
* missed defect out of the severity-weighted recall that contract §12.1 requires be
|
|
24
|
+
* considered, and the metric would still report a number.
|
|
25
|
+
*/
|
|
26
|
+
readonly SEVERITY_UNWEIGHTED: "ALDUS_SEVERITY_UNWEIGHTED";
|
|
27
|
+
/** A correction class was referenced that the policy assigns no harm weight to. */
|
|
28
|
+
readonly CORRECTION_CLASS_UNWEIGHTED: "ALDUS_CORRECTION_CLASS_UNWEIGHTED";
|
|
29
|
+
/** A blind-spot record does not validate. */
|
|
30
|
+
readonly BLIND_SPOT_MALFORMED: "ALDUS_BLIND_SPOT_MALFORMED";
|
|
31
|
+
/** Two blind spots share a `blindSpotId`. */
|
|
32
|
+
readonly BLIND_SPOT_DUPLICATE: "ALDUS_BLIND_SPOT_DUPLICATE";
|
|
33
|
+
/** A promotion policy is internally inconsistent — a threshold outside its valid range. */
|
|
34
|
+
readonly POLICY_INVALID: "ALDUS_POLICY_INVALID";
|
|
35
|
+
};
|
|
36
|
+
/** @see RegressionErrorCodes */
|
|
37
|
+
export type RegressionErrorCode = (typeof RegressionErrorCodes)[keyof typeof RegressionErrorCodes];
|
|
38
|
+
/** Construct an {@link AldusError} with a regression-harness code. */
|
|
39
|
+
export declare function regressionError(code: RegressionErrorCode, message: string, options: {
|
|
40
|
+
category: ErrorCategory;
|
|
41
|
+
retryable?: boolean;
|
|
42
|
+
details?: Record<string, unknown>;
|
|
43
|
+
}): AldusError;
|
|
44
|
+
//# sourceMappingURL=errors.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"errors.d.ts","sourceRoot":"","sources":["../src/errors.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,UAAU,EAAE,KAAK,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAErE,oDAAoD;AACpD,eAAO,MAAM,oBAAoB;IAC/B,+DAA+D;aAC/D,gBAAgB,EAAE,wBAAwB;IAC1C,gDAAgD;aAChD,qBAAqB,EAAE,6BAA6B;IACpD,8EAA8E;aAC9E,oBAAoB,EAAE,4BAA4B;IAClD,6DAA6D;aAC7D,iBAAiB,EAAE,yBAAyB;IAC5C;;;;;;OAMG;aACH,mBAAmB,EAAE,2BAA2B;IAChD,mFAAmF;aACnF,2BAA2B,EAAE,mCAAmC;IAChE,6CAA6C;aAC7C,oBAAoB,EAAE,4BAA4B;IAClD,6CAA6C;aAC7C,oBAAoB,EAAE,4BAA4B;IAClD,2FAA2F;aAC3F,cAAc,EAAE,sBAAsB;CAC9B,CAAC;AAEX,gCAAgC;AAChC,MAAM,MAAM,mBAAmB,GAAG,CAAC,OAAO,oBAAoB,CAAC,CAAC,MAAM,OAAO,oBAAoB,CAAC,CAAC;AAEnG,sEAAsE;AACtE,wBAAgB,eAAe,CAC7B,IAAI,EAAE,mBAAmB,EACzB,OAAO,EAAE,MAAM,EACf,OAAO,EAAE;IAAE,QAAQ,EAAE,aAAa,CAAC;IAAC,SAAS,CAAC,EAAE,OAAO,CAAC;IAAC,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CAAE,GAC3F,UAAU,CAEZ"}
|
package/dist/errors.js
ADDED
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Failures specific to the regression harness.
|
|
3
|
+
*
|
|
4
|
+
* Aldus Core deliberately keeps no central error-code registry, so a package can name a new
|
|
5
|
+
* failure without forking Core. These codes carry the same `ALDUS_` prefix and
|
|
6
|
+
* `SCREAMING_SNAKE_CASE` shape so production trace (contract §20) stays uniform across packages.
|
|
7
|
+
*/
|
|
8
|
+
import { AldusError } from "@aldus-runtime/core";
|
|
9
|
+
/** Error codes raised by the regression harness. */
|
|
10
|
+
export const RegressionErrorCodes = {
|
|
11
|
+
/** A defect corpus contained a case that does not validate. */
|
|
12
|
+
CORPUS_MALFORMED: "ALDUS_CORPUS_MALFORMED",
|
|
13
|
+
/** Two cases in one corpus share a `caseId`. */
|
|
14
|
+
CORPUS_DUPLICATE_CASE: "ALDUS_CORPUS_DUPLICATE_CASE",
|
|
15
|
+
/** An evaluator outcome referenced a `caseId` the corpus does not contain. */
|
|
16
|
+
OUTCOME_UNKNOWN_CASE: "ALDUS_OUTCOME_UNKNOWN_CASE",
|
|
17
|
+
/** Two outcomes in one run reported on the same `caseId`. */
|
|
18
|
+
OUTCOME_DUPLICATE: "ALDUS_OUTCOME_DUPLICATE",
|
|
19
|
+
/**
|
|
20
|
+
* A case carries a severity the policy assigns no weight to.
|
|
21
|
+
*
|
|
22
|
+
* Refused rather than defaulted: silently weighting an unknown severity as zero would drop a
|
|
23
|
+
* missed defect out of the severity-weighted recall that contract §12.1 requires be
|
|
24
|
+
* considered, and the metric would still report a number.
|
|
25
|
+
*/
|
|
26
|
+
SEVERITY_UNWEIGHTED: "ALDUS_SEVERITY_UNWEIGHTED",
|
|
27
|
+
/** A correction class was referenced that the policy assigns no harm weight to. */
|
|
28
|
+
CORRECTION_CLASS_UNWEIGHTED: "ALDUS_CORRECTION_CLASS_UNWEIGHTED",
|
|
29
|
+
/** A blind-spot record does not validate. */
|
|
30
|
+
BLIND_SPOT_MALFORMED: "ALDUS_BLIND_SPOT_MALFORMED",
|
|
31
|
+
/** Two blind spots share a `blindSpotId`. */
|
|
32
|
+
BLIND_SPOT_DUPLICATE: "ALDUS_BLIND_SPOT_DUPLICATE",
|
|
33
|
+
/** A promotion policy is internally inconsistent — a threshold outside its valid range. */
|
|
34
|
+
POLICY_INVALID: "ALDUS_POLICY_INVALID",
|
|
35
|
+
};
|
|
36
|
+
/** Construct an {@link AldusError} with a regression-harness code. */
|
|
37
|
+
export function regressionError(code, message, options) {
|
|
38
|
+
return new AldusError(code, message, options);
|
|
39
|
+
}
|
|
40
|
+
//# sourceMappingURL=errors.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"errors.js","sourceRoot":"","sources":["../src/errors.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,UAAU,EAAsB,MAAM,qBAAqB,CAAC;AAErE,oDAAoD;AACpD,MAAM,CAAC,MAAM,oBAAoB,GAAG;IAClC,+DAA+D;IAC/D,gBAAgB,EAAE,wBAAwB;IAC1C,gDAAgD;IAChD,qBAAqB,EAAE,6BAA6B;IACpD,8EAA8E;IAC9E,oBAAoB,EAAE,4BAA4B;IAClD,6DAA6D;IAC7D,iBAAiB,EAAE,yBAAyB;IAC5C;;;;;;OAMG;IACH,mBAAmB,EAAE,2BAA2B;IAChD,mFAAmF;IACnF,2BAA2B,EAAE,mCAAmC;IAChE,6CAA6C;IAC7C,oBAAoB,EAAE,4BAA4B;IAClD,6CAA6C;IAC7C,oBAAoB,EAAE,4BAA4B;IAClD,2FAA2F;IAC3F,cAAc,EAAE,sBAAsB;CAC9B,CAAC;AAKX,sEAAsE;AACtE,MAAM,UAAU,eAAe,CAC7B,IAAyB,EACzB,OAAe,EACf,OAA4F;IAE5F,OAAO,IAAI,UAAU,CAAC,IAAI,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;AAChD,CAAC"}
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `@aldus-runtime/regression` — the regression harness (architecture contract §22 WP-10).
|
|
3
|
+
*
|
|
4
|
+
* §12.1 permits an evaluator to become blocking "only after it is calibrated against
|
|
5
|
+
* human-labeled examples". This package is what makes that an evidence-based decision instead of
|
|
6
|
+
* a judgement call: it holds the defect corpus, compares an evaluator's outcomes against human
|
|
7
|
+
* labels, computes the metrics §12.1 names, and reports whether the bar is met — **per scope**.
|
|
8
|
+
*
|
|
9
|
+
* It does **not** run evaluators. Outcomes are produced elsewhere and handed here.
|
|
10
|
+
*
|
|
11
|
+
* Two things are load-bearing and easy to lose in a refactor:
|
|
12
|
+
*
|
|
13
|
+
* - **Promotion is scoped.** No API here returns "this evaluator is promotable". §12.1 lists
|
|
14
|
+
* show, host, voice, model, and script-form scope because calibration does not generalise.
|
|
15
|
+
* - **The aggregate never decides.** The whole-corpus figure is descriptive; no threshold is
|
|
16
|
+
* applied to it (ADR-0010).
|
|
17
|
+
*
|
|
18
|
+
* @packageDocumentation
|
|
19
|
+
*/
|
|
20
|
+
export { REGRESSION_SCHEMA_VERSION, defectCaseSchema, defectCorpusSchema, evaluatorFindingSchema, evaluatorOutcomeSchema, evaluatorRunSchema, findingCategory, humanFindingSchema, parseDefectCorpus, parseEvaluatorRun, scopeDimensions, severityLevel, type DefectCase, type DefectCorpus, type EvaluatorFinding, type EvaluatorOutcome, type EvaluatorRun, type HumanFinding, type ScopeDimensions, } from "./corpus.js";
|
|
21
|
+
export { WHOLE_CORPUS_SLICE, deriveScopeSelectors, observedDimensions, scopeKey, scopeLabel, scopeMatches, type ScopeSelector, } from "./scope.js";
|
|
22
|
+
export { DEFAULT_CORRECTION_HARM_WEIGHTS, DEFAULT_PROMOTION_THRESHOLDS, assertPolicyValid, correctionHarm, defaultPromotionPolicy, severityWeight, type CorrectionHarmWeights, type PolicyOrigin, type PromotionPolicy, type PromotionThresholds, type SeverityWeights, } from "./policy.js";
|
|
23
|
+
export { compareRun, type CaseComparison, type CaseVerdict, type CompareOptions, type ComparisonReport, type SliceMetrics, } from "./metrics.js";
|
|
24
|
+
export { BLIND_SPOT_STATUSES, BlindSpotRegistry, blindSpotCoversCase, blindSpotSchema, type BlindSpot, type BlindSpotStatus, } from "./blindspots.js";
|
|
25
|
+
export { assessPromotion, isPromotableEverywhereMeasured, type PromotionShortfall, type PromotionShortfallCode, type PromotionVerdict, type SliceVerdict, } from "./promotion.js";
|
|
26
|
+
export { REPORT_CAVEAT, forbiddenClaimWords, renderPromotionReport } from "./report.js";
|
|
27
|
+
export { RegressionErrorCodes, regressionError, type RegressionErrorCode } from "./errors.js";
|
|
28
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAGH,OAAO,EACL,yBAAyB,EACzB,gBAAgB,EAChB,kBAAkB,EAClB,sBAAsB,EACtB,sBAAsB,EACtB,kBAAkB,EAClB,eAAe,EACf,kBAAkB,EAClB,iBAAiB,EACjB,iBAAiB,EACjB,eAAe,EACf,aAAa,EACb,KAAK,UAAU,EACf,KAAK,YAAY,EACjB,KAAK,gBAAgB,EACrB,KAAK,gBAAgB,EACrB,KAAK,YAAY,EACjB,KAAK,YAAY,EACjB,KAAK,eAAe,GACrB,MAAM,aAAa,CAAC;AAGrB,OAAO,EACL,kBAAkB,EAClB,oBAAoB,EACpB,kBAAkB,EAClB,QAAQ,EACR,UAAU,EACV,YAAY,EACZ,KAAK,aAAa,GACnB,MAAM,YAAY,CAAC;AAGpB,OAAO,EACL,+BAA+B,EAC/B,4BAA4B,EAC5B,iBAAiB,EACjB,cAAc,EACd,sBAAsB,EACtB,cAAc,EACd,KAAK,qBAAqB,EAC1B,KAAK,YAAY,EACjB,KAAK,eAAe,EACpB,KAAK,mBAAmB,EACxB,KAAK,eAAe,GACrB,MAAM,aAAa,CAAC;AAGrB,OAAO,EACL,UAAU,EACV,KAAK,cAAc,EACnB,KAAK,WAAW,EAChB,KAAK,cAAc,EACnB,KAAK,gBAAgB,EACrB,KAAK,YAAY,GAClB,MAAM,cAAc,CAAC;AAGtB,OAAO,EACL,mBAAmB,EACnB,iBAAiB,EACjB,mBAAmB,EACnB,eAAe,EACf,KAAK,SAAS,EACd,KAAK,eAAe,GACrB,MAAM,iBAAiB,CAAC;AAGzB,OAAO,EACL,eAAe,EACf,8BAA8B,EAC9B,KAAK,kBAAkB,EACvB,KAAK,sBAAsB,EAC3B,KAAK,gBAAgB,EACrB,KAAK,YAAY,GAClB,MAAM,gBAAgB,CAAC;AAGxB,OAAO,EAAE,aAAa,EAAE,mBAAmB,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;AAGxF,OAAO,EAAE,oBAAoB,EAAE,eAAe,EAAE,KAAK,mBAAmB,EAAE,MAAM,aAAa,CAAC"}
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `@aldus-runtime/regression` — the regression harness (architecture contract §22 WP-10).
|
|
3
|
+
*
|
|
4
|
+
* §12.1 permits an evaluator to become blocking "only after it is calibrated against
|
|
5
|
+
* human-labeled examples". This package is what makes that an evidence-based decision instead of
|
|
6
|
+
* a judgement call: it holds the defect corpus, compares an evaluator's outcomes against human
|
|
7
|
+
* labels, computes the metrics §12.1 names, and reports whether the bar is met — **per scope**.
|
|
8
|
+
*
|
|
9
|
+
* It does **not** run evaluators. Outcomes are produced elsewhere and handed here.
|
|
10
|
+
*
|
|
11
|
+
* Two things are load-bearing and easy to lose in a refactor:
|
|
12
|
+
*
|
|
13
|
+
* - **Promotion is scoped.** No API here returns "this evaluator is promotable". §12.1 lists
|
|
14
|
+
* show, host, voice, model, and script-form scope because calibration does not generalise.
|
|
15
|
+
* - **The aggregate never decides.** The whole-corpus figure is descriptive; no threshold is
|
|
16
|
+
* applied to it (ADR-0010).
|
|
17
|
+
*
|
|
18
|
+
* @packageDocumentation
|
|
19
|
+
*/
|
|
20
|
+
// --- Defect corpus (§12.1, §12.3, §24) --------------------------------------------------------
|
|
21
|
+
export { REGRESSION_SCHEMA_VERSION, defectCaseSchema, defectCorpusSchema, evaluatorFindingSchema, evaluatorOutcomeSchema, evaluatorRunSchema, findingCategory, humanFindingSchema, parseDefectCorpus, parseEvaluatorRun, scopeDimensions, severityLevel, } from "./corpus.js";
|
|
22
|
+
// --- Scope slicing (§12.1) --------------------------------------------------------------------
|
|
23
|
+
export { WHOLE_CORPUS_SLICE, deriveScopeSelectors, observedDimensions, scopeKey, scopeLabel, scopeMatches, } from "./scope.js";
|
|
24
|
+
// --- Promotion policy (§12.1, §25.9; ADR-0010) ------------------------------------------------
|
|
25
|
+
export { DEFAULT_CORRECTION_HARM_WEIGHTS, DEFAULT_PROMOTION_THRESHOLDS, assertPolicyValid, correctionHarm, defaultPromotionPolicy, severityWeight, } from "./policy.js";
|
|
26
|
+
// --- Comparison and metrics (§12.1) -----------------------------------------------------------
|
|
27
|
+
export { compareRun, } from "./metrics.js";
|
|
28
|
+
// --- Known blind spots (§12.1, §9.3) ----------------------------------------------------------
|
|
29
|
+
export { BLIND_SPOT_STATUSES, BlindSpotRegistry, blindSpotCoversCase, blindSpotSchema, } from "./blindspots.js";
|
|
30
|
+
// --- Promotion verdict (§12.1; ADR-0010) ------------------------------------------------------
|
|
31
|
+
export { assessPromotion, isPromotableEverywhereMeasured, } from "./promotion.js";
|
|
32
|
+
// --- Report rendering (§12) -------------------------------------------------------------------
|
|
33
|
+
export { REPORT_CAVEAT, forbiddenClaimWords, renderPromotionReport } from "./report.js";
|
|
34
|
+
// --- Errors -----------------------------------------------------------------------------------
|
|
35
|
+
export { RegressionErrorCodes, regressionError } from "./errors.js";
|
|
36
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAEH,iGAAiG;AACjG,OAAO,EACL,yBAAyB,EACzB,gBAAgB,EAChB,kBAAkB,EAClB,sBAAsB,EACtB,sBAAsB,EACtB,kBAAkB,EAClB,eAAe,EACf,kBAAkB,EAClB,iBAAiB,EACjB,iBAAiB,EACjB,eAAe,EACf,aAAa,GAQd,MAAM,aAAa,CAAC;AAErB,iGAAiG;AACjG,OAAO,EACL,kBAAkB,EAClB,oBAAoB,EACpB,kBAAkB,EAClB,QAAQ,EACR,UAAU,EACV,YAAY,GAEb,MAAM,YAAY,CAAC;AAEpB,iGAAiG;AACjG,OAAO,EACL,+BAA+B,EAC/B,4BAA4B,EAC5B,iBAAiB,EACjB,cAAc,EACd,sBAAsB,EACtB,cAAc,GAMf,MAAM,aAAa,CAAC;AAErB,iGAAiG;AACjG,OAAO,EACL,UAAU,GAMX,MAAM,cAAc,CAAC;AAEtB,iGAAiG;AACjG,OAAO,EACL,mBAAmB,EACnB,iBAAiB,EACjB,mBAAmB,EACnB,eAAe,GAGhB,MAAM,iBAAiB,CAAC;AAEzB,iGAAiG;AACjG,OAAO,EACL,eAAe,EACf,8BAA8B,GAK/B,MAAM,gBAAgB,CAAC;AAExB,iGAAiG;AACjG,OAAO,EAAE,aAAa,EAAE,mBAAmB,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;AAExF,iGAAiG;AACjG,OAAO,EAAE,oBAAoB,EAAE,eAAe,EAA4B,MAAM,aAAa,CAAC"}
|