@aldus-runtime/regression 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. package/LICENSE +201 -0
  2. package/NOTICE +21 -0
  3. package/dist/blindspots.d.ts +107 -0
  4. package/dist/blindspots.d.ts.map +1 -0
  5. package/dist/blindspots.js +169 -0
  6. package/dist/blindspots.js.map +1 -0
  7. package/dist/corpus.d.ts +184 -0
  8. package/dist/corpus.d.ts.map +1 -0
  9. package/dist/corpus.js +254 -0
  10. package/dist/corpus.js.map +1 -0
  11. package/dist/errors.d.ts +44 -0
  12. package/dist/errors.d.ts.map +1 -0
  13. package/dist/errors.js +40 -0
  14. package/dist/errors.js.map +1 -0
  15. package/dist/index.d.ts +28 -0
  16. package/dist/index.d.ts.map +1 -0
  17. package/dist/index.js +36 -0
  18. package/dist/index.js.map +1 -0
  19. package/dist/metrics.d.ts +134 -0
  20. package/dist/metrics.d.ts.map +1 -0
  21. package/dist/metrics.js +125 -0
  22. package/dist/metrics.js.map +1 -0
  23. package/dist/policy.d.ts +140 -0
  24. package/dist/policy.d.ts.map +1 -0
  25. package/dist/policy.js +148 -0
  26. package/dist/policy.js.map +1 -0
  27. package/dist/promotion.d.ts +108 -0
  28. package/dist/promotion.d.ts.map +1 -0
  29. package/dist/promotion.js +130 -0
  30. package/dist/promotion.js.map +1 -0
  31. package/dist/report.d.ts +42 -0
  32. package/dist/report.d.ts.map +1 -0
  33. package/dist/report.js +116 -0
  34. package/dist/report.js.map +1 -0
  35. package/dist/scope.d.ts +57 -0
  36. package/dist/scope.d.ts.map +1 -0
  37. package/dist/scope.js +81 -0
  38. package/dist/scope.js.map +1 -0
  39. package/package.json +49 -0
  40. package/src/blindspots.ts +199 -0
  41. package/src/corpus.ts +311 -0
  42. package/src/errors.ts +49 -0
  43. package/src/index.ts +104 -0
  44. package/src/metrics.ts +293 -0
  45. package/src/policy.ts +267 -0
  46. package/src/promotion.ts +336 -0
  47. package/src/report.ts +154 -0
  48. package/src/scope.ts +100 -0
@@ -0,0 +1,199 @@
1
+ /**
2
+ * The known-blind-spot registry (architecture contract §12.1, §9.3).
3
+ *
4
+ * §12.1 lists "known blind spots" among what evaluator promotion must consider, and §9.3 makes
5
+ * negative knowledge first-class: "Known failed approaches, unsafe transformations, evaluator
6
+ * blind spots, and provider limitations SHOULD be first-class pack content. Learning does not
7
+ * mean storing only successful examples."
8
+ *
9
+ * A blind spot is a failure mode an evaluator is *known* not to catch. It matters to promotion
10
+ * precisely because the corpus does not contain it — so metrics computed over the corpus are not
11
+ * evidence against a blind spot, they are evidence that the corpus did not look. An evaluator
12
+ * with an open blind spot in a slice should never read as promotable in that slice, however good
13
+ * its numbers are.
14
+ *
15
+ * ### Relationship to Knowledge Packs
16
+ *
17
+ * `KnowledgePackManifest.negativeKnowledge` (WP-09) holds *resource paths* — Core indexes them
18
+ * and never parses what is behind them (§1.2, §9.1). This registry holds the structured,
19
+ * machine-checkable records that gate promotion. They are complementary: a pack points at the
20
+ * prose describing a blind spot, and this registry carries the part a policy can evaluate.
21
+ */
22
+
23
+ import { actorRefSchema } from "@aldus-runtime/core";
24
+ import { z } from "zod";
25
+
26
+ import { findingCategory, scopeDimensions, severityLevel } from "./corpus.js";
27
+ import { RegressionErrorCodes, regressionError } from "./errors.js";
28
+ import { scopeMatches, type ScopeSelector } from "./scope.js";
29
+
30
+ /** How a blind spot has been dealt with. */
31
+ export const BLIND_SPOT_STATUSES = [
32
+ /** Demonstrated and unaddressed. Disqualifies promotion in scope. */
33
+ "open",
34
+ /** Addressed, with the mitigation recorded. */
35
+ "mitigated",
36
+ /**
37
+ * Known, unmitigated, and deliberately tolerated.
38
+ *
39
+ * Distinct from `mitigated` because it is a decision, not a fix — and it must stay visible in
40
+ * the promotion report so tolerating it is a standing choice rather than a forgotten one.
41
+ */
42
+ "accepted",
43
+ ] as const;
44
+
45
+ /** @see BLIND_SPOT_STATUSES */
46
+ export type BlindSpotStatus = (typeof BLIND_SPOT_STATUSES)[number];
47
+
48
+ /** A failure mode an evaluator is known not to catch. */
49
+ export const blindSpotSchema = z
50
+ .object({
51
+ /** Identity of this record. */
52
+ blindSpotId: z.string().min(1).max(200),
53
+ /** Evaluator the blind spot belongs to. */
54
+ evaluatorId: z.string().min(1).max(200),
55
+ /** What the evaluator fails to catch. Prose, for a human reading the promotion report. */
56
+ description: z.string().min(1).max(4000),
57
+ /**
58
+ * Scope the blind spot applies to.
59
+ *
60
+ * An empty scope means it applies everywhere. A scoped blind spot disqualifies only the
61
+ * slices it covers — an evaluator blind to one voice's artefacts is not thereby disqualified
62
+ * for every other voice.
63
+ */
64
+ scope: scopeDimensions,
65
+ /** Category from the adopter's taxonomy, where the blind spot maps to one (§12.3). */
66
+ category: findingCategory.optional(),
67
+ /** Severity of what goes undetected. */
68
+ severity: severityLevel.optional(),
69
+ /** Current status. */
70
+ status: z.enum(BLIND_SPOT_STATUSES),
71
+ /**
72
+ * Cases that demonstrate it, where the corpus contains any.
73
+ *
74
+ * Often empty, and that is the point: a blind spot the corpus can demonstrate is a
75
+ * measurable false negative, while one it cannot is invisible to every metric here.
76
+ */
77
+ evidenceCaseIds: z.array(z.string().min(1).max(200)).max(1024),
78
+ /** What was done about it. Required in spirit when `mitigated`; prose either way. */
79
+ mitigation: z.string().max(4000).optional(),
80
+ /** Who recorded it (contract §19.2). */
81
+ recordedBy: actorRefSchema,
82
+ /** When it was recorded. */
83
+ recordedAt: z.iso.datetime({ offset: true }),
84
+ })
85
+ .meta({
86
+ id: "BlindSpot",
87
+ title: "BlindSpot",
88
+ description:
89
+ "A failure mode an evaluator is known not to catch (architecture contract §12.1, §9.3). " +
90
+ "Corpus metrics are not evidence against a blind spot — a blind spot is by definition " +
91
+ "something the corpus did not sample. An `open` blind spot in scope disqualifies promotion " +
92
+ "there regardless of metrics.",
93
+ });
94
+
95
+ /** @see blindSpotSchema */
96
+ export type BlindSpot = z.infer<typeof blindSpotSchema>;
97
+
98
+ /**
99
+ * An in-memory, serialisable set of blind-spot records.
100
+ *
101
+ * Deliberately not file-backed. Persistence is the caller's choice — these records belong
102
+ * alongside an adopter's Knowledge Packs (§9.3), and binding the registry to one storage
103
+ * implementation would put a storage dependency in a package that only needs to read a list.
104
+ */
105
+ export class BlindSpotRegistry {
106
+ readonly #records = new Map<string, BlindSpot>();
107
+
108
+ /** Build a registry from records, validating each. */
109
+ static from(records: readonly unknown[]): BlindSpotRegistry {
110
+ const registry = new BlindSpotRegistry();
111
+ for (const record of records) registry.record(record);
112
+ return registry;
113
+ }
114
+
115
+ /**
116
+ * Validate and add one record.
117
+ *
118
+ * @throws {AldusError} `ALDUS_BLIND_SPOT_MALFORMED` or `ALDUS_BLIND_SPOT_DUPLICATE`.
119
+ */
120
+ record(input: unknown): BlindSpot {
121
+ const result = blindSpotSchema.safeParse(input);
122
+ if (!result.success) {
123
+ // Paths and issue codes only — never the received value (contract §19.2).
124
+ throw regressionError(
125
+ RegressionErrorCodes.BLIND_SPOT_MALFORMED,
126
+ "The blind-spot record does not validate.",
127
+ {
128
+ category: "validation",
129
+ details: {
130
+ issues: result.error.issues.map((issue) => ({
131
+ path: issue.path.join("."),
132
+ code: issue.code,
133
+ })),
134
+ },
135
+ },
136
+ );
137
+ }
138
+ if (this.#records.has(result.data.blindSpotId)) {
139
+ throw regressionError(
140
+ RegressionErrorCodes.BLIND_SPOT_DUPLICATE,
141
+ `A blind spot with id "${result.data.blindSpotId}" is already registered.`,
142
+ { category: "conflict", details: { blindSpotId: result.data.blindSpotId } },
143
+ );
144
+ }
145
+ this.#records.set(result.data.blindSpotId, result.data);
146
+ return result.data;
147
+ }
148
+
149
+ /** Every record, in insertion order. */
150
+ list(): BlindSpot[] {
151
+ return [...this.#records.values()];
152
+ }
153
+
154
+ /** Every record for one evaluator. */
155
+ forEvaluator(evaluatorId: string): BlindSpot[] {
156
+ return this.list().filter((record) => record.evaluatorId === evaluatorId);
157
+ }
158
+
159
+ /**
160
+ * Open blind spots for an evaluator that apply within a scope slice.
161
+ *
162
+ * A blind spot applies to a slice when everything it scopes itself to is either held at the
163
+ * same value by the slice, or not held by the slice at all. The second half matters: a blind
164
+ * spot scoped to one voice applies to the whole-corpus slice, because the whole corpus
165
+ * includes that voice.
166
+ */
167
+ openFor(evaluatorId: string, selector: ScopeSelector): BlindSpot[] {
168
+ return this.forEvaluator(evaluatorId).filter((record) => {
169
+ if (record.status !== "open") return false;
170
+ return Object.entries(record.scope).every(([dimension, value]) => {
171
+ const held = selector.values[dimension];
172
+ return held === undefined || held === value;
173
+ });
174
+ });
175
+ }
176
+
177
+ /** Blind spots demonstrated by at least one case in a corpus subset. */
178
+ demonstratedBy(evaluatorId: string, caseIds: ReadonlySet<string>): BlindSpot[] {
179
+ return this.forEvaluator(evaluatorId).filter((record) =>
180
+ record.evidenceCaseIds.some((caseId) => caseIds.has(caseId)),
181
+ );
182
+ }
183
+
184
+ /** Serialisable form. */
185
+ toJSON(): BlindSpot[] {
186
+ return this.list();
187
+ }
188
+ }
189
+
190
+ /** True when a blind spot's own scope is satisfied by a case's scope. */
191
+ export function blindSpotCoversCase(
192
+ record: BlindSpot,
193
+ scope: Readonly<Record<string, string>>,
194
+ ): boolean {
195
+ return scopeMatches(scope, {
196
+ dimensions: Object.keys(record.scope),
197
+ values: record.scope,
198
+ });
199
+ }
package/src/corpus.ts ADDED
@@ -0,0 +1,311 @@
1
+ /**
2
+ * The defect corpus (architecture contract §22 WP-10, §12.1, §12.3, §24).
3
+ *
4
+ * §12.1 permits an evaluator to become blocking "only after it is calibrated against
5
+ * human-labeled examples", and §24 requires that "a representative defect corpus is executed
6
+ * during regression testing". This module defines what such a corpus is.
7
+ *
8
+ * Two shapes here are deliberately open rather than enumerated:
9
+ *
10
+ * - **Finding categories.** §12.3 presents its taxonomy as "for example", and §4.2 keeps
11
+ * adopter concepts out of the runtime. A category is an opaque caller-supplied string.
12
+ * - **Severity levels.** Likewise named by the caller. The *weights* live in the promotion
13
+ * policy, so an adopter can say a missed pronunciation error costs less than a missed
14
+ * unsupported claim without Core deciding that for them.
15
+ *
16
+ * The human label is the oracle (§12 level 4). Nothing in this module treats an evaluator's
17
+ * output as ground truth.
18
+ *
19
+ * Zod is the single source of truth; TypeScript types are inferred (ADR-0002).
20
+ */
21
+
22
+ import { actorRefSchema, SCHEMA_VERSION } from "@aldus-runtime/core";
23
+ import { z } from "zod";
24
+
25
+ import { RegressionErrorCodes, regressionError } from "./errors.js";
26
+
27
+ /** A short opaque identifier. */
28
+ const identifier = z.string().min(1).max(200);
29
+
30
+ /**
31
+ * A finding category, structured per contract §12.3.
32
+ *
33
+ * An OPEN string, never a Core-defined enum. §12.3 introduces its taxonomy with "for example",
34
+ * and §4.2 forbids Core from owning an adopter's diagnosis vocabulary. Do not narrow this to a
35
+ * union.
36
+ */
37
+ export const findingCategory = z.string().min(1).max(200);
38
+
39
+ /**
40
+ * A severity level, named by the caller.
41
+ *
42
+ * An OPEN string for the same reason as {@link findingCategory}. Ordering and weighting are
43
+ * supplied by the promotion policy, not inferred from the name — a package that decided
44
+ * `"critical"` outranks `"major"` would be guessing at an adopter's scale.
45
+ */
46
+ export const severityLevel = z.string().min(1).max(100);
47
+
48
+ /**
49
+ * Scope dimensions a case belongs to (contract §12.1 "show, host, voice, model, and
50
+ * script-form scope").
51
+ *
52
+ * `Record<string, string>`, consistent with Knowledge Pack scope (§9.2, ADR-0006). §12.1's list
53
+ * is illustrative and §4.2 forbids naming a provider, so dimensions stay caller-supplied.
54
+ */
55
+ export const scopeDimensions = z.record(z.string().min(1).max(100), z.string().min(1).max(200));
56
+
57
+ /** @see scopeDimensions */
58
+ export type ScopeDimensions = z.infer<typeof scopeDimensions>;
59
+
60
+ /** One defect a human labeller identified in a case. */
61
+ export const humanFindingSchema = z
62
+ .object({
63
+ /** Category from the adopter's taxonomy (contract §12.3). */
64
+ category: findingCategory,
65
+ /** Severity from the adopter's scale; weighted by the promotion policy. */
66
+ severity: severityLevel,
67
+ /** What the labeller saw. Prose, never parsed. */
68
+ note: z.string().max(4000).optional(),
69
+ })
70
+ .meta({ id: "HumanFinding", title: "HumanFinding" });
71
+
72
+ /** @see humanFindingSchema */
73
+ export type HumanFinding = z.infer<typeof humanFindingSchema>;
74
+
75
+ /**
76
+ * One labelled case: an input, and what a human said about it.
77
+ *
78
+ * The case does not carry the input itself. §8.1 makes artifacts addressable by ID and hash, and
79
+ * a corpus that embedded audio or scripts would be unreviewable and would risk carrying private
80
+ * source material into a test fixture (§19.2). `subjectRef` points at the subject; what it
81
+ * points into is the caller's business.
82
+ */
83
+ export const defectCaseSchema = z
84
+ .object({
85
+ /** Identity of this case within its corpus. */
86
+ caseId: identifier,
87
+ /** Reference to the material under test — an artifact ID, hash, or adopter-defined locator. */
88
+ subjectRef: z.string().min(1).max(1024),
89
+ /** Scope this case belongs to (contract §12.1). */
90
+ scope: scopeDimensions,
91
+ /**
92
+ * Whether a human judged the subject defective. **This is the oracle** (§12 level 4).
93
+ *
94
+ * Stored separately from `findings` so a case can be labelled clean explicitly. An empty
95
+ * findings array on a case nobody reviewed is not the same claim as "a human looked and
96
+ * found nothing", and conflating them would inflate the true-negative count with unreviewed
97
+ * material.
98
+ */
99
+ defective: z.boolean(),
100
+ /** What the labeller found. Empty when `defective` is false. */
101
+ findings: z.array(humanFindingSchema).max(256),
102
+ /**
103
+ * Overall severity of the case, used for severity-weighted metrics (contract §12.1).
104
+ *
105
+ * Required when `defective`, absent otherwise — enforced by refinement below.
106
+ */
107
+ severity: severityLevel.optional(),
108
+ /**
109
+ * Correction class this case would trigger if a blocking evaluator flagged it.
110
+ *
111
+ * §12.1 names "asymmetric harm caused by unnecessary automatic correction" as its own
112
+ * consideration, and §12.4 makes the repair layer explicit — regenerating one TTS segment
113
+ * and revising narration with cascading approval invalidation are not the same act. This
114
+ * names which one a flag would trigger here; the policy assigns the harm weight. Absent
115
+ * means the policy's default class applies.
116
+ */
117
+ correctionOnFlag: identifier.optional(),
118
+ /** Who labelled the case (contract §19.2: a decision without an actor is not a decision). */
119
+ labelledBy: actorRefSchema,
120
+ /** When the label was recorded. */
121
+ labelledAt: z.iso.datetime({ offset: true }),
122
+ })
123
+ .refine((value) => value.defective === (value.severity !== undefined), {
124
+ message:
125
+ "a defective case must carry a severity, and a clean case must not (architecture contract §12.1 requires severity-weighted metrics)",
126
+ path: ["severity"],
127
+ })
128
+ .refine((value) => value.defective || value.findings.length === 0, {
129
+ message: "a case labelled clean must carry no findings",
130
+ path: ["findings"],
131
+ })
132
+ .meta({
133
+ id: "DefectCase",
134
+ title: "DefectCase",
135
+ description:
136
+ "One human-labelled case in a defect corpus (architecture contract §12.1, §24). The human " +
137
+ "label is the oracle; nothing treats an evaluator's output as ground truth. `category` and " +
138
+ "`severity` are open strings because §12.3's taxonomy is illustrative and §4.2 keeps " +
139
+ "adopter vocabularies out of the runtime. ADDITIONAL CONSTRAINTS NOT EXPRESSIBLE IN JSON " +
140
+ "SCHEMA: a defective case must carry a severity and a clean case must not, and a clean " +
141
+ "case must carry no findings.",
142
+ });
143
+
144
+ /** @see defectCaseSchema */
145
+ export type DefectCase = z.infer<typeof defectCaseSchema>;
146
+
147
+ /** A named, versioned set of labelled cases. */
148
+ export const defectCorpusSchema = z
149
+ .object({
150
+ /** Schema version of this record (ADR-0003). */
151
+ schemaVersion: z.string().regex(/^(0|[1-9]\d*)\.(0|[1-9]\d*)$/),
152
+ /** Identity of this corpus. */
153
+ corpusId: identifier,
154
+ /** Human-readable purpose. */
155
+ description: z.string().max(2000).optional(),
156
+ /** The labelled cases. */
157
+ cases: z.array(defectCaseSchema).max(100_000),
158
+ })
159
+ .meta({
160
+ id: "DefectCorpus",
161
+ title: "DefectCorpus",
162
+ description:
163
+ "A defect corpus: human-labelled cases an evaluator is calibrated against (architecture " +
164
+ "contract §12.1, §24). ADDITIONAL CONSTRAINT NOT EXPRESSIBLE IN JSON SCHEMA: `caseId` " +
165
+ "must be unique within the corpus.",
166
+ });
167
+
168
+ /** @see defectCorpusSchema */
169
+ export type DefectCorpus = z.infer<typeof defectCorpusSchema>;
170
+
171
+ /** One finding an evaluator reported. */
172
+ export const evaluatorFindingSchema = z
173
+ .object({
174
+ /** Category the evaluator assigned, from the same taxonomy as {@link humanFindingSchema}. */
175
+ category: findingCategory,
176
+ /** Severity the evaluator assigned, where it assigns one. */
177
+ severity: severityLevel.optional(),
178
+ /** Evaluator confidence in `[0, 1]`, where it reports one. */
179
+ confidence: z.number().min(0).max(1).optional(),
180
+ })
181
+ .meta({ id: "EvaluatorFinding", title: "EvaluatorFinding" });
182
+
183
+ /** @see evaluatorFindingSchema */
184
+ export type EvaluatorFinding = z.infer<typeof evaluatorFindingSchema>;
185
+
186
+ /** What an evaluator said about one case. */
187
+ export const evaluatorOutcomeSchema = z
188
+ .object({
189
+ /** Case this outcome reports on. */
190
+ caseId: identifier,
191
+ /** Whether the evaluator flagged the subject. */
192
+ flagged: z.boolean(),
193
+ /** What it reported. Empty when not flagged. */
194
+ findings: z.array(evaluatorFindingSchema).max(256),
195
+ })
196
+ .meta({ id: "EvaluatorOutcome", title: "EvaluatorOutcome" });
197
+
198
+ /** @see evaluatorOutcomeSchema */
199
+ export type EvaluatorOutcome = z.infer<typeof evaluatorOutcomeSchema>;
200
+
201
+ /**
202
+ * One evaluator's outcomes over one corpus.
203
+ *
204
+ * This package does **not** run evaluators (contract §22 WP-10 scope). A run is produced
205
+ * elsewhere and handed here for comparison.
206
+ */
207
+ export const evaluatorRunSchema = z
208
+ .object({
209
+ /** Schema version of this record (ADR-0003). */
210
+ schemaVersion: z.string().regex(/^(0|[1-9]\d*)\.(0|[1-9]\d*)$/),
211
+ /** Which evaluator produced these outcomes. */
212
+ evaluatorId: identifier,
213
+ /** Version of the evaluator. Metrics from different versions are not comparable. */
214
+ evaluatorVersion: identifier,
215
+ /** Corpus the outcomes are against. */
216
+ corpusId: identifier,
217
+ /** The outcomes. */
218
+ outcomes: z.array(evaluatorOutcomeSchema).max(100_000),
219
+ /** When the run was executed. */
220
+ executedAt: z.iso.datetime({ offset: true }),
221
+ })
222
+ .meta({
223
+ id: "EvaluatorRun",
224
+ title: "EvaluatorRun",
225
+ description:
226
+ "One evaluator's outcomes over one defect corpus (architecture contract §22 WP-10). This " +
227
+ "package consumes runs; it does not execute evaluators. ADDITIONAL CONSTRAINT NOT " +
228
+ "EXPRESSIBLE IN JSON SCHEMA: `caseId` must be unique within `outcomes`.",
229
+ });
230
+
231
+ /** @see evaluatorRunSchema */
232
+ export type EvaluatorRun = z.infer<typeof evaluatorRunSchema>;
233
+
234
+ /** Schema version this package stamps on records it constructs. */
235
+ export const REGRESSION_SCHEMA_VERSION = SCHEMA_VERSION;
236
+
237
+ /**
238
+ * Validate a corpus and check the uniqueness constraint Zod cannot express.
239
+ *
240
+ * @throws {AldusError} `ALDUS_CORPUS_MALFORMED` or `ALDUS_CORPUS_DUPLICATE_CASE`.
241
+ */
242
+ export function parseDefectCorpus(input: unknown): DefectCorpus {
243
+ const result = defectCorpusSchema.safeParse(input);
244
+ if (!result.success) {
245
+ // Paths and issue codes only — never the received value (contract §19.2, ADR-0002).
246
+ throw regressionError(
247
+ RegressionErrorCodes.CORPUS_MALFORMED,
248
+ "The defect corpus does not validate.",
249
+ {
250
+ category: "validation",
251
+ details: {
252
+ issues: result.error.issues.map((issue) => ({
253
+ path: issue.path.join("."),
254
+ code: issue.code,
255
+ })),
256
+ },
257
+ },
258
+ );
259
+ }
260
+
261
+ const seen = new Set<string>();
262
+ for (const entry of result.data.cases) {
263
+ if (seen.has(entry.caseId)) {
264
+ throw regressionError(
265
+ RegressionErrorCodes.CORPUS_DUPLICATE_CASE,
266
+ `Corpus "${result.data.corpusId}" contains more than one case with id "${entry.caseId}". ` +
267
+ "Duplicate ids would let one case be counted twice and silently reweight every metric.",
268
+ { category: "validation", details: { corpusId: result.data.corpusId } },
269
+ );
270
+ }
271
+ seen.add(entry.caseId);
272
+ }
273
+ return result.data;
274
+ }
275
+
276
+ /**
277
+ * Validate an evaluator run and check uniqueness.
278
+ *
279
+ * @throws {AldusError} `ALDUS_CORPUS_MALFORMED` or `ALDUS_OUTCOME_DUPLICATE`.
280
+ */
281
+ export function parseEvaluatorRun(input: unknown): EvaluatorRun {
282
+ const result = evaluatorRunSchema.safeParse(input);
283
+ if (!result.success) {
284
+ throw regressionError(
285
+ RegressionErrorCodes.CORPUS_MALFORMED,
286
+ "The evaluator run does not validate.",
287
+ {
288
+ category: "validation",
289
+ details: {
290
+ issues: result.error.issues.map((issue) => ({
291
+ path: issue.path.join("."),
292
+ code: issue.code,
293
+ })),
294
+ },
295
+ },
296
+ );
297
+ }
298
+
299
+ const seen = new Set<string>();
300
+ for (const outcome of result.data.outcomes) {
301
+ if (seen.has(outcome.caseId)) {
302
+ throw regressionError(
303
+ RegressionErrorCodes.OUTCOME_DUPLICATE,
304
+ `Evaluator run for "${result.data.evaluatorId}" reports twice on case "${outcome.caseId}".`,
305
+ { category: "validation", details: { evaluatorId: result.data.evaluatorId } },
306
+ );
307
+ }
308
+ seen.add(outcome.caseId);
309
+ }
310
+ return result.data;
311
+ }
package/src/errors.ts ADDED
@@ -0,0 +1,49 @@
1
+ /**
2
+ * Failures specific to the regression harness.
3
+ *
4
+ * Aldus Core deliberately keeps no central error-code registry, so a package can name a new
5
+ * failure without forking Core. These codes carry the same `ALDUS_` prefix and
6
+ * `SCREAMING_SNAKE_CASE` shape so production trace (contract §20) stays uniform across packages.
7
+ */
8
+
9
+ import { AldusError, type ErrorCategory } from "@aldus-runtime/core";
10
+
11
+ /** Error codes raised by the regression harness. */
12
+ export const RegressionErrorCodes = {
13
+ /** A defect corpus contained a case that does not validate. */
14
+ CORPUS_MALFORMED: "ALDUS_CORPUS_MALFORMED",
15
+ /** Two cases in one corpus share a `caseId`. */
16
+ CORPUS_DUPLICATE_CASE: "ALDUS_CORPUS_DUPLICATE_CASE",
17
+ /** An evaluator outcome referenced a `caseId` the corpus does not contain. */
18
+ OUTCOME_UNKNOWN_CASE: "ALDUS_OUTCOME_UNKNOWN_CASE",
19
+ /** Two outcomes in one run reported on the same `caseId`. */
20
+ OUTCOME_DUPLICATE: "ALDUS_OUTCOME_DUPLICATE",
21
+ /**
22
+ * A case carries a severity the policy assigns no weight to.
23
+ *
24
+ * Refused rather than defaulted: silently weighting an unknown severity as zero would drop a
25
+ * missed defect out of the severity-weighted recall that contract §12.1 requires be
26
+ * considered, and the metric would still report a number.
27
+ */
28
+ SEVERITY_UNWEIGHTED: "ALDUS_SEVERITY_UNWEIGHTED",
29
+ /** A correction class was referenced that the policy assigns no harm weight to. */
30
+ CORRECTION_CLASS_UNWEIGHTED: "ALDUS_CORRECTION_CLASS_UNWEIGHTED",
31
+ /** A blind-spot record does not validate. */
32
+ BLIND_SPOT_MALFORMED: "ALDUS_BLIND_SPOT_MALFORMED",
33
+ /** Two blind spots share a `blindSpotId`. */
34
+ BLIND_SPOT_DUPLICATE: "ALDUS_BLIND_SPOT_DUPLICATE",
35
+ /** A promotion policy is internally inconsistent — a threshold outside its valid range. */
36
+ POLICY_INVALID: "ALDUS_POLICY_INVALID",
37
+ } as const;
38
+
39
+ /** @see RegressionErrorCodes */
40
+ export type RegressionErrorCode = (typeof RegressionErrorCodes)[keyof typeof RegressionErrorCodes];
41
+
42
+ /** Construct an {@link AldusError} with a regression-harness code. */
43
+ export function regressionError(
44
+ code: RegressionErrorCode,
45
+ message: string,
46
+ options: { category: ErrorCategory; retryable?: boolean; details?: Record<string, unknown> },
47
+ ): AldusError {
48
+ return new AldusError(code, message, options);
49
+ }
package/src/index.ts ADDED
@@ -0,0 +1,104 @@
1
+ /**
2
+ * `@aldus-runtime/regression` — the regression harness (architecture contract §22 WP-10).
3
+ *
4
+ * §12.1 permits an evaluator to become blocking "only after it is calibrated against
5
+ * human-labeled examples". This package is what makes that an evidence-based decision instead of
6
+ * a judgement call: it holds the defect corpus, compares an evaluator's outcomes against human
7
+ * labels, computes the metrics §12.1 names, and reports whether the bar is met — **per scope**.
8
+ *
9
+ * It does **not** run evaluators. Outcomes are produced elsewhere and handed here.
10
+ *
11
+ * Two things are load-bearing and easy to lose in a refactor:
12
+ *
13
+ * - **Promotion is scoped.** No API here returns "this evaluator is promotable". §12.1 lists
14
+ * show, host, voice, model, and script-form scope because calibration does not generalise.
15
+ * - **The aggregate never decides.** The whole-corpus figure is descriptive; no threshold is
16
+ * applied to it (ADR-0010).
17
+ *
18
+ * @packageDocumentation
19
+ */
20
+
21
+ // --- Defect corpus (§12.1, §12.3, §24) --------------------------------------------------------
22
+ export {
23
+ REGRESSION_SCHEMA_VERSION,
24
+ defectCaseSchema,
25
+ defectCorpusSchema,
26
+ evaluatorFindingSchema,
27
+ evaluatorOutcomeSchema,
28
+ evaluatorRunSchema,
29
+ findingCategory,
30
+ humanFindingSchema,
31
+ parseDefectCorpus,
32
+ parseEvaluatorRun,
33
+ scopeDimensions,
34
+ severityLevel,
35
+ type DefectCase,
36
+ type DefectCorpus,
37
+ type EvaluatorFinding,
38
+ type EvaluatorOutcome,
39
+ type EvaluatorRun,
40
+ type HumanFinding,
41
+ type ScopeDimensions,
42
+ } from "./corpus.js";
43
+
44
+ // --- Scope slicing (§12.1) --------------------------------------------------------------------
45
+ export {
46
+ WHOLE_CORPUS_SLICE,
47
+ deriveScopeSelectors,
48
+ observedDimensions,
49
+ scopeKey,
50
+ scopeLabel,
51
+ scopeMatches,
52
+ type ScopeSelector,
53
+ } from "./scope.js";
54
+
55
+ // --- Promotion policy (§12.1, §25.9; ADR-0010) ------------------------------------------------
56
+ export {
57
+ DEFAULT_CORRECTION_HARM_WEIGHTS,
58
+ DEFAULT_PROMOTION_THRESHOLDS,
59
+ assertPolicyValid,
60
+ correctionHarm,
61
+ defaultPromotionPolicy,
62
+ severityWeight,
63
+ type CorrectionHarmWeights,
64
+ type PolicyOrigin,
65
+ type PromotionPolicy,
66
+ type PromotionThresholds,
67
+ type SeverityWeights,
68
+ } from "./policy.js";
69
+
70
+ // --- Comparison and metrics (§12.1) -----------------------------------------------------------
71
+ export {
72
+ compareRun,
73
+ type CaseComparison,
74
+ type CaseVerdict,
75
+ type CompareOptions,
76
+ type ComparisonReport,
77
+ type SliceMetrics,
78
+ } from "./metrics.js";
79
+
80
+ // --- Known blind spots (§12.1, §9.3) ----------------------------------------------------------
81
+ export {
82
+ BLIND_SPOT_STATUSES,
83
+ BlindSpotRegistry,
84
+ blindSpotCoversCase,
85
+ blindSpotSchema,
86
+ type BlindSpot,
87
+ type BlindSpotStatus,
88
+ } from "./blindspots.js";
89
+
90
+ // --- Promotion verdict (§12.1; ADR-0010) ------------------------------------------------------
91
+ export {
92
+ assessPromotion,
93
+ isPromotableEverywhereMeasured,
94
+ type PromotionShortfall,
95
+ type PromotionShortfallCode,
96
+ type PromotionVerdict,
97
+ type SliceVerdict,
98
+ } from "./promotion.js";
99
+
100
+ // --- Report rendering (§12) -------------------------------------------------------------------
101
+ export { REPORT_CAVEAT, forbiddenClaimWords, renderPromotionReport } from "./report.js";
102
+
103
+ // --- Errors -----------------------------------------------------------------------------------
104
+ export { RegressionErrorCodes, regressionError, type RegressionErrorCode } from "./errors.js";