@descryy/core 0.6.3 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/contracts/index.d.ts +3 -1
- package/dist/contracts/index.d.ts.map +1 -1
- package/dist/contracts/index.js +2 -1
- package/dist/contracts/index.js.map +1 -1
- package/dist/contracts/paths.d.ts +7 -0
- package/dist/contracts/paths.d.ts.map +1 -1
- package/dist/contracts/paths.js +7 -4
- package/dist/contracts/paths.js.map +1 -1
- package/dist/contracts/shapes.d.ts +12 -1
- package/dist/contracts/shapes.d.ts.map +1 -1
- package/dist/contracts/shapes.js +68 -0
- package/dist/contracts/shapes.js.map +1 -1
- package/dist/contracts/workspace-join.d.ts +104 -0
- package/dist/contracts/workspace-join.d.ts.map +1 -0
- package/dist/contracts/workspace-join.js +208 -0
- package/dist/contracts/workspace-join.js.map +1 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +4 -0
- package/dist/index.js.map +1 -1
- package/dist/query/index.d.ts +7 -3
- package/dist/query/index.d.ts.map +1 -1
- package/dist/query/index.js +4 -2
- package/dist/query/index.js.map +1 -1
- package/dist/query/model-table.d.ts +7 -0
- package/dist/query/model-table.d.ts.map +1 -1
- package/dist/query/model-table.js +7 -4
- package/dist/query/model-table.js.map +1 -1
- package/dist/query/persistence-reach.d.ts +150 -0
- package/dist/query/persistence-reach.d.ts.map +1 -0
- package/dist/query/persistence-reach.js +393 -0
- package/dist/query/persistence-reach.js.map +1 -0
- package/dist/query/queries.d.ts +5 -0
- package/dist/query/queries.d.ts.map +1 -1
- package/dist/query/queries.js +58 -0
- package/dist/query/queries.js.map +1 -1
- package/dist/query/user-chain.d.ts +89 -0
- package/dist/query/user-chain.d.ts.map +1 -0
- package/dist/query/user-chain.js +203 -0
- package/dist/query/user-chain.js.map +1 -0
- package/dist/sources/git/history.d.ts +12 -0
- package/dist/sources/git/history.d.ts.map +1 -1
- package/dist/sources/git/history.js +27 -0
- package/dist/sources/git/history.js.map +1 -1
- package/dist/sources/git/index.d.ts +1 -1
- package/dist/sources/git/index.d.ts.map +1 -1
- package/dist/sources/git/index.js +1 -1
- package/dist/sources/git/index.js.map +1 -1
- package/dist/store/incremental-cache.d.ts +120 -0
- package/dist/store/incremental-cache.d.ts.map +1 -0
- package/dist/store/incremental-cache.js +161 -0
- package/dist/store/incremental-cache.js.map +1 -0
- package/dist/store/index.d.ts +2 -0
- package/dist/store/index.d.ts.map +1 -1
- package/dist/store/index.js +1 -0
- package/dist/store/index.js.map +1 -1
- package/dist/store/reader.d.ts.map +1 -1
- package/dist/store/reader.js +6 -1
- package/dist/store/reader.js.map +1 -1
- package/dist/store/schema.d.ts +4 -4
- package/dist/store/schema.d.ts.map +1 -1
- package/dist/store/schema.js +46 -1
- package/dist/store/schema.js.map +1 -1
- package/dist/store/writer.d.ts.map +1 -1
- package/dist/store/writer.js +4 -3
- package/dist/store/writer.js.map +1 -1
- package/dist/tiers/certify.d.ts +174 -0
- package/dist/tiers/certify.d.ts.map +1 -1
- package/dist/tiers/certify.js +309 -5
- package/dist/tiers/certify.js.map +1 -1
- package/dist/tiers/index.d.ts +1 -1
- package/dist/tiers/index.d.ts.map +1 -1
- package/dist/tiers/index.js.map +1 -1
- package/dist/workspace/graph-layout.d.ts +85 -0
- package/dist/workspace/graph-layout.d.ts.map +1 -0
- package/dist/workspace/graph-layout.js +90 -0
- package/dist/workspace/graph-layout.js.map +1 -0
- package/dist/workspace/index.d.ts +3 -0
- package/dist/workspace/index.d.ts.map +1 -0
- package/dist/workspace/index.js +2 -0
- package/dist/workspace/index.js.map +1 -0
- package/package.json +2 -2
package/dist/tiers/certify.d.ts
CHANGED
|
@@ -28,6 +28,17 @@ export interface PrecisionDraw {
|
|
|
28
28
|
/** The negative control over the same adjudicator. Must be ~0. */
|
|
29
29
|
readonly controlDrawn: number;
|
|
30
30
|
readonly controlCorrect: number;
|
|
31
|
+
/** Control rows drawn but never scored, because the swap could not be made WRONG — every
|
|
32
|
+
* candidate the pool offered was itself a real edge, so the row tested nothing. Excluded from
|
|
33
|
+
* `controlDrawn` (zero out of zero is undefined, and a swap nothing tested must not be scored
|
|
34
|
+
* like one that was tested and rejected) and recorded here instead of vanishing.
|
|
35
|
+
*
|
|
36
|
+
* `undefined` is not zero: a grader with no such bucket decided every swap it drew, and is
|
|
37
|
+
* charged nothing — same reason an absent `decided` is not charged COVERAGE_FLOOR.
|
|
38
|
+
*
|
|
39
|
+
* It IS a weakening of the control, and `CONTROL_CONSTRUCTION_FLOOR` is what stops it being an
|
|
40
|
+
* unlimited one — a denominator the grader may shrink is one the grader can shrink to one row. */
|
|
41
|
+
readonly controlUncontrollable?: number;
|
|
31
42
|
/** Did the adapter's author pick this repo? DEC-075 §6.2 requires two that didn't. Declared, not inferred. */
|
|
32
43
|
readonly authorChose: boolean;
|
|
33
44
|
/** Who recorded this, so an attested claim has a name against it. */
|
|
@@ -49,6 +60,105 @@ export interface ConformanceEvidence {
|
|
|
49
60
|
* corpus's fault) — conflating them failed every adapter at Tier C on a low-resolution sweep. */
|
|
50
61
|
readonly manifestClean: boolean;
|
|
51
62
|
}
|
|
63
|
+
/**
|
|
64
|
+
* Which denominator a recall figure is over. **The headline is `wholeCodebase`,
|
|
65
|
+
* ruled and not inferred.**
|
|
66
|
+
*
|
|
67
|
+
* `inCodebase` divides by the references whose TARGET is declared inside the
|
|
68
|
+
* repository, so every call into a library is absent from its denominator — the
|
|
69
|
+
* axis excluded precisely the part of the codebase the graph could not
|
|
70
|
+
* represent, which made it the most flattering figure available and the least
|
|
71
|
+
* answerable one. External-symbol nodes made those references representable and
|
|
72
|
+
* measured what it was worth: go/gin 24.5% → 59.5%, python/dispatch 11.8% →
|
|
73
|
+
* 37.3%, both at zero false positives. Once a call into a library can be a
|
|
74
|
+
* node, the axis that counts it is the one that answers "how much of my code
|
|
75
|
+
* did you analyse".
|
|
76
|
+
*
|
|
77
|
+
* `narrow` is `found / (found + missing)` — what every `bench/<lang>-recall.mjs`
|
|
78
|
+
* prints and what every earlier recall number in this repository means. It is
|
|
79
|
+
* the narrowest of the three and is kept only for continuity.
|
|
80
|
+
*
|
|
81
|
+
* Both narrower axes are reportable. Neither is the headline, and a draw that
|
|
82
|
+
* offers one as its headline is refused with that reason — see
|
|
83
|
+
* {@link ruleRecallDraw}.
|
|
84
|
+
*/
|
|
85
|
+
export type RecallAxis = "wholeCodebase" | "inCodebase" | "narrow";
|
|
86
|
+
/**
|
|
87
|
+
* A recall measurement, as `bench/recall-baseline.json` records one.
|
|
88
|
+
*
|
|
89
|
+
* Deliberately NOT shaped like {@link PrecisionDraw}: it carries no control and
|
|
90
|
+
* no `authorChose`, because it gates nothing and therefore needs neither. What
|
|
91
|
+
* it does carry is the pair that makes a recall figure readable at all — the
|
|
92
|
+
* denominator's axis, and the draw size that denominator came out of.
|
|
93
|
+
*/
|
|
94
|
+
export interface RecallDraw {
|
|
95
|
+
readonly repository: string;
|
|
96
|
+
/** Which denominator {@link denominator} is. Required: a rate with no stated
|
|
97
|
+
* axis is three different claims wearing one number. */
|
|
98
|
+
readonly axis: RecallAxis;
|
|
99
|
+
/** Every site the instrument drew, before any exclusion. Coverage's denominator. */
|
|
100
|
+
readonly drawn: number;
|
|
101
|
+
/** The figure's own denominator: `drawn` minus whatever this axis excludes. */
|
|
102
|
+
readonly denominator: number;
|
|
103
|
+
/** How many of `denominator` reached the graph. */
|
|
104
|
+
readonly found: number;
|
|
105
|
+
/** The second axis, disclosed beside the headline so a reader has both.
|
|
106
|
+
* Optional; its absence is printed as absence, never as agreement. */
|
|
107
|
+
readonly secondAxis?: {
|
|
108
|
+
readonly axis: RecallAxis;
|
|
109
|
+
readonly found: number;
|
|
110
|
+
readonly denominator: number;
|
|
111
|
+
};
|
|
112
|
+
/** What the generated-code exclusion removed from this draw, if it was applied.
|
|
113
|
+
* `undefined` means no exclusion was recorded — printed as such, because an
|
|
114
|
+
* undisclosed exclusion and no exclusion must not read the same. */
|
|
115
|
+
readonly generatedExcluded?: {
|
|
116
|
+
readonly files: number;
|
|
117
|
+
readonly sites: number;
|
|
118
|
+
readonly rules: readonly string[];
|
|
119
|
+
};
|
|
120
|
+
/** Who recorded it. Same rule as everywhere else here: unattributable is not admissible. */
|
|
121
|
+
readonly recordedBy: string;
|
|
122
|
+
}
|
|
123
|
+
/** One {@link RecallDraw}, ruled. `admissible` is about the FIGURE, never about the adapter. */
|
|
124
|
+
export interface RecallRuling {
|
|
125
|
+
readonly repository: string;
|
|
126
|
+
readonly axis: RecallAxis;
|
|
127
|
+
readonly found: number;
|
|
128
|
+
readonly denominator: number;
|
|
129
|
+
readonly drawn: number;
|
|
130
|
+
/** `found / denominator`, or `null` where the denominator is zero — which is
|
|
131
|
+
* undefined, not 0% and not 100%. */
|
|
132
|
+
readonly rate: number | null;
|
|
133
|
+
/** `denominator / drawn`: how much of its own draw the instrument was willing
|
|
134
|
+
* to be graded on. `null` where there is no draw to divide by. */
|
|
135
|
+
readonly coverage: number | null;
|
|
136
|
+
/** Does this figure say anything at all? False does NOT mean the adapter fell
|
|
137
|
+
* short — it means the figure is a claim about what the instrument judged, or
|
|
138
|
+
* is over the wrong denominator. */
|
|
139
|
+
readonly admissible: boolean;
|
|
140
|
+
readonly detail: string;
|
|
141
|
+
readonly secondAxis?: RecallDraw["secondAxis"];
|
|
142
|
+
readonly generatedExcluded?: RecallDraw["generatedExcluded"];
|
|
143
|
+
readonly recordedBy: string;
|
|
144
|
+
}
|
|
145
|
+
/**
|
|
146
|
+
* What the certificate says about recall.
|
|
147
|
+
*
|
|
148
|
+
* `not-measured` is its own state and is printed. A certificate that simply
|
|
149
|
+
* omitted recall when none was recorded would let "we never measured how much
|
|
150
|
+
* of your codebase we reached" and "we reached it" render identically, which is
|
|
151
|
+
* the silent degradation rule 7 forbids.
|
|
152
|
+
*/
|
|
153
|
+
export type RecallDisclosure = {
|
|
154
|
+
readonly state: "not-measured";
|
|
155
|
+
readonly detail: string;
|
|
156
|
+
readonly rows: readonly [];
|
|
157
|
+
} | {
|
|
158
|
+
readonly state: "disclosed";
|
|
159
|
+
readonly detail: string;
|
|
160
|
+
readonly rows: readonly RecallRuling[];
|
|
161
|
+
};
|
|
52
162
|
export interface GateEvidence {
|
|
53
163
|
readonly adapter: string;
|
|
54
164
|
readonly language: string;
|
|
@@ -65,9 +175,16 @@ export interface GateEvidence {
|
|
|
65
175
|
* that fails on absence; JS has none). Distinct from "not built yet" (§20.2) — doesn't block the
|
|
66
176
|
* tier and doesn't vanish, it's printed on the certificate. `declaredBy` required; unattributed = ignored. */
|
|
67
177
|
readonly structuralLimits?: readonly StructuralLimit[];
|
|
178
|
+
/** Requirements the admitted reference corpus cannot supply evidence for (see `CorpusCeiling`).
|
|
179
|
+
* Waives nothing and moves no tier — changes "not done yet" into "not reachable from this corpus". */
|
|
180
|
+
readonly corpusCeilings?: readonly CorpusCeiling[];
|
|
68
181
|
/** Capabilities claimed below what evidence supports, pending an open policy (see `PolicyCap`).
|
|
69
182
|
* Doesn't waive a requirement or block a tier — stops a deliberate cap being published as an absence. */
|
|
70
183
|
readonly policyCaps?: readonly PolicyCap[];
|
|
184
|
+
/** Recall measurements. **Disclosed on the certificate and gating nothing** —
|
|
185
|
+
* see `RECALL_COVERAGE_FLOOR` for why, and for what would have to be true
|
|
186
|
+
* before this could become a requirement. Absent is printed as absent. */
|
|
187
|
+
readonly recallDraws?: readonly RecallDraw[];
|
|
71
188
|
}
|
|
72
189
|
/** A capability this language cannot have. Declared, never inferred. */
|
|
73
190
|
export interface StructuralLimit {
|
|
@@ -78,6 +195,40 @@ export interface StructuralLimit {
|
|
|
78
195
|
/** Who declared it. An anonymous structural limit is not admissible. */
|
|
79
196
|
readonly declaredBy: string;
|
|
80
197
|
}
|
|
198
|
+
/**
|
|
199
|
+
* A requirement the ADMITTED REFERENCE CORPUS cannot supply evidence for, published as a
|
|
200
|
+
* ceiling rather than left standing as unmet work.
|
|
201
|
+
*
|
|
202
|
+
* Three neighbouring things, and conflating any two of them publishes a falsehood:
|
|
203
|
+
*
|
|
204
|
+
* `StructuralLimit` the LANGUAGE cannot have this, ever. Waives the requirement.
|
|
205
|
+
* `unreachableHere` this repository's population is a measured zero. Derived, waives
|
|
206
|
+
* nothing, and says the remedy is other repositories.
|
|
207
|
+
* `CorpusCeiling` the admitted repositories contain almost none of the edge type the
|
|
208
|
+
* requirement is scored on, AND the rules forbid admitting a better
|
|
209
|
+
* one for that reason. Declared, waives nothing.
|
|
210
|
+
*
|
|
211
|
+
* The difference between the second and the third is who is allowed to fix it. A measured
|
|
212
|
+
* zero invites other repositories; a corpus ceiling is the case where reaching for one is
|
|
213
|
+
* the prohibited move — the outcome-blind admission rules exist precisely to stop a
|
|
214
|
+
* repository being admitted because it would move a tier. So the row is not pending work,
|
|
215
|
+
* and printing it as unmet tells a reader to go and do something the rules forbid.
|
|
216
|
+
*
|
|
217
|
+
* **It waives nothing, deliberately.** A ceiling that waived its own requirement would be
|
|
218
|
+
* the cheapest route to a tier available: declare the corpus inadequate, collect the badge.
|
|
219
|
+
* The tier stays exactly where the evidence puts it; only the sentence beside it changes.
|
|
220
|
+
*
|
|
221
|
+
* `declaredBy` is required for the same reason `StructuralLimit` requires it — an anonymous
|
|
222
|
+
* "the corpus is inadequate" is a sentence anyone can add to stop a row reading as work.
|
|
223
|
+
*/
|
|
224
|
+
export interface CorpusCeiling {
|
|
225
|
+
/** A requirement id, or the name of a check. */
|
|
226
|
+
readonly id: string;
|
|
227
|
+
/** Why the admitted reference set cannot supply this, in terms of the corpus. */
|
|
228
|
+
readonly reason: string;
|
|
229
|
+
/** Who declared it. An anonymous ceiling is not admissible. */
|
|
230
|
+
readonly declaredBy: string;
|
|
231
|
+
}
|
|
81
232
|
/** A capability claimable higher but deliberately claimed lower, pending an unwritten policy —
|
|
82
233
|
* distinct from absent (unmeasured) and StructuralLimit (impossible). E.g. an R4-introspected
|
|
83
234
|
* route table claimed at R3 pending R4's expiry policy (R4 is unconditionally class A, DEC-115). */
|
|
@@ -99,6 +250,17 @@ export interface EmptyPopulation {
|
|
|
99
250
|
readonly repository: string;
|
|
100
251
|
readonly edgeType: EdgeType | "mixed";
|
|
101
252
|
}
|
|
253
|
+
/** A draw whose negative control excluded rows it could not make wrong. Derived from the draw,
|
|
254
|
+
* never declared — the grader counted them, nobody attests to them. Printed because a control
|
|
255
|
+
* that excluded rows and a control that excluded none read identically otherwise. */
|
|
256
|
+
export interface WeakenedControl {
|
|
257
|
+
readonly repository: string;
|
|
258
|
+
readonly edgeType: EdgeType | "mixed";
|
|
259
|
+
/** Swaps that were actually made wrong and scored — the control's real denominator. */
|
|
260
|
+
readonly constructed: number;
|
|
261
|
+
/** Swaps drawn where every candidate in the pool was itself a real edge. */
|
|
262
|
+
readonly uncontrollable: number;
|
|
263
|
+
}
|
|
102
264
|
export interface UnmetRequirement {
|
|
103
265
|
readonly requirement: Requirement;
|
|
104
266
|
/** Why it is unmet, in terms of what was found rather than what was wanted. */
|
|
@@ -130,8 +292,20 @@ export interface TierCertificate {
|
|
|
130
292
|
/** Draws whose population was zero — nothing here to measure, not "the adapter fell short."
|
|
131
293
|
* Printed beside structural limitations; the remedy is other repositories. */
|
|
132
294
|
readonly unreachableHere: readonly EmptyPopulation[];
|
|
295
|
+
/** Requirements published as unreachable from the admitted corpus. Declared, attributed, and
|
|
296
|
+
* waiving nothing — never omit these when printing, or a ceiling reads as pending work. */
|
|
297
|
+
readonly corpusCeilings: readonly CorpusCeiling[];
|
|
298
|
+
/** Draws whose control excluded rows it could not make wrong. Empty for every draw that
|
|
299
|
+
* records no such bucket. Never omit when printing — an undisclosed exclusion is the thing
|
|
300
|
+
* the bucket exists to stop. */
|
|
301
|
+
readonly weakenedControls: readonly WeakenedControl[];
|
|
133
302
|
/** DEC-075 §6.3: does `capabilities()` match what was emitted? */
|
|
134
303
|
readonly capabilitiesHonest: boolean | "unknown";
|
|
304
|
+
/** How much of the codebase was reached, and on which denominator. Never omit
|
|
305
|
+
* when printing: a tier says how RIGHT the edges are, and says nothing at all
|
|
306
|
+
* about how MANY of them there are. Those are different questions and a
|
|
307
|
+
* reader handed only the first will answer the second with it. */
|
|
308
|
+
readonly recall: RecallDisclosure;
|
|
135
309
|
}
|
|
136
310
|
export declare function certify(evidence: GateEvidence): TierCertificate;
|
|
137
311
|
/** Tier first, then immediately why it isn't higher. Printing "B" alone teaches a reader B is
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"certify.d.ts","sourceRoot":"","sources":["../../src/tiers/certify.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EAAE,gBAAgB,EAAE,QAAQ,EAAE,QAAQ,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAEzF,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AAC5D,OAAO,EAAU,KAAK,WAAW,EAAE,KAAK,IAAI,EAA0B,MAAM,aAAa,CAAC;AAE1F;yGACyG;AACzG,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAAC;IACtC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB;;mGAE+F;IAC/F,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAC1B;;4GAEwG;IACxG,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;IAC7B;;iGAE6F;IAC7F,QAAQ,CAAC,QAAQ,CAAC,EAAE,aAAa,CAAC;IAClC,kEAAkE;IAClE,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,8GAA8G;IAC9G,QAAQ,CAAC,WAAW,EAAE,OAAO,CAAC;IAC9B,qEAAqE;IACrE,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B;;2GAEuG;IACvG,QAAQ,CAAC,WAAW,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE;QAAE,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC,CAAC;CACzG;AAED,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,UAAU,EAAE,eAAe,CAAC;IACrC,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC;IACzB;;sGAEkG;IAClG,QAAQ,CAAC,aAAa,EAAE,OAAO,CAAC;CACjC;AAED,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,YAAY,EAAE,gBAAgB,CAAC;IACxC,2EAA2E;IAC3E,QAAQ,CAAC,gBAAgB,CAAC,EAAE,SAAS,QAAQ,EAAE,CAAC;IAChD,QAAQ,CAAC,gBAAgB,CAAC,EAAE,SAAS,QAAQ,EAAE,CAAC;IAChD,QAAQ,CAAC,WAAW,EAAE,SAAS,mBAAmB,EAAE,CAAC;IACrD,QAAQ,CAAC,cAAc,EAAE,SAAS,aAAa,EAAE,CAAC;IAClD;qGACiG;IACjG,QAAQ,CAAC,kBAAkB,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAChD;;mHAE+G;IAC/G,QAAQ,CAAC,gBAAgB,CAAC,EAAE,SAAS,eAAe,EAAE,CAAC;IACvD;8GAC0G;IAC1G,QAAQ,CAAC,UAAU,CAAC,EAAE,SAAS,SAAS,EAAE,CAAC;
|
|
1
|
+
{"version":3,"file":"certify.d.ts","sourceRoot":"","sources":["../../src/tiers/certify.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EAAE,gBAAgB,EAAE,QAAQ,EAAE,QAAQ,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAEzF,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AAC5D,OAAO,EAAU,KAAK,WAAW,EAAE,KAAK,IAAI,EAA0B,MAAM,aAAa,CAAC;AAE1F;yGACyG;AACzG,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAAC;IACtC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB;;mGAE+F;IAC/F,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAC1B;;4GAEwG;IACxG,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;IAC7B;;iGAE6F;IAC7F,QAAQ,CAAC,QAAQ,CAAC,EAAE,aAAa,CAAC;IAClC,kEAAkE;IAClE,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC;;;;;;;;;uGASmG;IACnG,QAAQ,CAAC,qBAAqB,CAAC,EAAE,MAAM,CAAC;IACxC,8GAA8G;IAC9G,QAAQ,CAAC,WAAW,EAAE,OAAO,CAAC;IAC9B,qEAAqE;IACrE,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B;;2GAEuG;IACvG,QAAQ,CAAC,WAAW,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE;QAAE,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC,CAAC;CACzG;AAED,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,UAAU,EAAE,eAAe,CAAC;IACrC,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC;IACzB;;sGAEkG;IAClG,QAAQ,CAAC,aAAa,EAAE,OAAO,CAAC;CACjC;AAED;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,MAAM,MAAM,UAAU,GAAG,eAAe,GAAG,YAAY,GAAG,QAAQ,CAAC;AAEnE;;;;;;;GAOG;AACH,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B;6DACyD;IACzD,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC;IAC1B,oFAAoF;IACpF,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,+EAA+E;IAC/E,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,mDAAmD;IACnD,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB;2EACuE;IACvE,QAAQ,CAAC,UAAU,CAAC,EAAE;QAAE,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC;QAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAA;KAAE,CAAC;IAC1G;;yEAEqE;IACrE,QAAQ,CAAC,iBAAiB,CAAC,EAAE;QAC3B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;QACvB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;QACvB,QAAQ,CAAC,KAAK,EAAE,SAAS,MAAM,EAAE,CAAC;KACnC,CAAC;IACF,4FAA4F;IAC5F,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;CAC7B;AAED,gGAAgG;AAChG,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC;IAC1B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB;0CACsC;IACtC,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B;uEACmE;IACnE,QAAQ,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC;;yCAEqC;IACrC,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAC;IAC7B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,UAAU,CAAC,EAAE,UAAU,CAAC,YAAY,CAAC,CAAC;IAC/C,QAAQ,CAAC,iBAAiB,CAAC,EAAE,UAAU,CAAC,mBAAmB,CAAC,CAAC;IAC7D,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;CAC7B;AAED;;;;;;;GAOG;AACH,MAAM,MAAM,gBAAgB,GACxB;IAAE,QAAQ,CAAC,KAAK,EAAE,cAAc,CAAC;IAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,IAAI,EAAE,SAAS,EAAE,CAAA;CAAE,GACvF;IAAE,QAAQ,CAAC,KAAK,EAAE,WAAW,CAAC;IAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,IAAI,EAAE,SAAS,YAAY,EAAE,CAAA;CAAE,CAAC;AAErG,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,YAAY,EAAE,gBAAgB,CAAC;IACxC,2EAA2E;IAC3E,QAAQ,CAAC,gBAAgB,CAAC,EAAE,SAAS,QAAQ,EAAE,CAAC;IAChD,QAAQ,CAAC,gBAAgB,CAAC,EAAE,SAAS,QAAQ,EAAE,CAAC;IAChD,QAAQ,CAAC,WAAW,EAAE,SAAS,mBAAmB,EAAE,CAAC;IACrD,QAAQ,CAAC,cAAc,EAAE,SAAS,aAAa,EAAE,CAAC;IAClD;qGACiG;IACjG,QAAQ,CAAC,kBAAkB,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAChD;;mHAE+G;IAC/G,QAAQ,CAAC,gBAAgB,CAAC,EAAE,SAAS,eAAe,EAAE,CAAC;IACvD;2GACuG;IACvG,QAAQ,CAAC,cAAc,CAAC,EAAE,SAAS,aAAa,EAAE,CAAC;IACnD;8GAC0G;IAC1G,QAAQ,CAAC,UAAU,CAAC,EAAE,SAAS,SAAS,EAAE,CAAC;IAC3C;;+EAE2E;IAC3E,QAAQ,CAAC,WAAW,CAAC,EAAE,SAAS,UAAU,EAAE,CAAC;CAC9C;AAED,wEAAwE;AACxE,MAAM,WAAW,eAAe;IAC9B,gDAAgD;IAChD,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,0EAA0E;IAC1E,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,wEAAwE;IACxE,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;CAC7B;AAED;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,MAAM,WAAW,aAAa;IAC5B,gDAAgD;IAChD,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,iFAAiF;IACjF,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,+DAA+D;IAC/D,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;CAC7B;AAED;;qGAEqG;AACrG,MAAM,WAAW,SAAS;IACxB,gDAAgD;IAChD,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,gDAAgD;IAChD,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,gCAAgC;IAChC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB;gGAC4F;IAC5F,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,yFAAyF;IACzF,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;CAC7B;AAED,iFAAiF;AACjF,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAAC;CACvC;AAED;;sFAEsF;AACtF,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAAC;IACtC,uFAAuF;IACvF,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,4EAA4E;IAC5E,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;CACjC;AAED,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,WAAW,EAAE,WAAW,CAAC;IAClC,+EAA+E;IAC/E,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;CAC1B;AAED,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,8EAA8E;IAC9E,QAAQ,CAAC,IAAI,EAAE,IAAI,GAAG,IAAI,CAAC;IAC3B,yEAAyE;IACzE,QAAQ,CAAC,QAAQ,EAAE,IAAI,GAAG,IAAI,CAAC;IAC/B,QAAQ,CAAC,YAAY,EAAE,SAAS,gBAAgB,EAAE,CAAC;IACnD,0EAA0E;IAC1E,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,IAAI,EAAE,SAAS,gBAAgB,EAAE,CAAC,CAAC,CAAC;IACrE;mCAC+B;IAC/B,QAAQ,CAAC,UAAU,EAAE,SAAS,MAAM,EAAE,CAAC;IACvC;;2FAEuF;IACvF,QAAQ,CAAC,iBAAiB,EAAE,SAAS,CAAC,QAAQ,GAAG,OAAO,CAAC,EAAE,CAAC;IAC5D;qGACiG;IACjG,QAAQ,CAAC,WAAW,EAAE,SAAS,eAAe,EAAE,CAAC;IACjD;+CAC2C;IAC3C,QAAQ,CAAC,UAAU,EAAE,SAAS,SAAS,EAAE,CAAC;IAC1C;mFAC+E;IAC/E,QAAQ,CAAC,eAAe,EAAE,SAAS,eAAe,EAAE,CAAC;IACrD;gGAC4F;IAC5F,QAAQ,CAAC,cAAc,EAAE,SAAS,aAAa,EAAE,CAAC;IAClD;;qCAEiC;IACjC,QAAQ,CAAC,gBAAgB,EAAE,SAAS,eAAe,EAAE,CAAC;IACtD,kEAAkE;IAClE,QAAQ,CAAC,kBAAkB,EAAE,OAAO,GAAG,SAAS,CAAC;IACjD;;;uEAGmE;IACnE,QAAQ,CAAC,MAAM,EAAE,gBAAgB,CAAC;CACnC;AAieD,wBAAgB,OAAO,CAAC,QAAQ,EAAE,YAAY,GAAG,eAAe,CAuE/D;AAED;iFACiF;AACjF,wBAAgB,iBAAiB,CAAC,WAAW,EAAE,eAAe,GAAG,MAAM,CA+GtE"}
|
package/dist/tiers/certify.js
CHANGED
|
@@ -11,7 +11,59 @@ const MAX_CONTROL = 0.02;
|
|
|
11
11
|
* bench/ can't be imported from packages/core. User-ruled: a capability that moves a tier needs
|
|
12
12
|
* this many decided rows of its OWN edge type, not just an aggregate pass. */
|
|
13
13
|
const TYPE_FLOOR = 20;
|
|
14
|
+
/** The other half of the same floor, and it was missing. `bench/grader-outcome-adjudicate.mjs`'s
|
|
15
|
+
* `scoreLines` has always marked a type `<- below floor` at `n >= TYPE_FLOOR && ok / n < 0.95`;
|
|
16
|
+
* this file counted the rows and never looked at how many were right, so a draw that FAILED a
|
|
17
|
+
* capability was the evidence that granted it — csharp/eShopOnWeb's USES_API at 19/25 = 76% moved
|
|
18
|
+
* csharp from B to A. Same value as the bench layer's, for the same reason TYPE_FLOOR is. */
|
|
19
|
+
const TYPE_RATE_FLOOR = 0.95;
|
|
14
20
|
const REPOSITORIES_REQUIRED = 2;
|
|
21
|
+
/** Mirrors `bench/kotlin-precision.mjs`'s `COVERAGE_FLOOR` — the other half of scoring over
|
|
22
|
+
* `decided`. Precision says how well the grader did on what it judged; coverage says how much
|
|
23
|
+
* it was willing to judge. Neither alone is a claim, and only one of the two was ever gated
|
|
24
|
+
* here. Restated rather than imported for the same reason `TYPE_FLOOR` is: a bench instrument
|
|
25
|
+
* is not a dependency of the shipped gate. If you change one, change both. */
|
|
26
|
+
const COVERAGE_FLOOR = 0.6;
|
|
27
|
+
/** The same floor again, one layer down: of the control swaps a grader ATTEMPTED, how many it
|
|
28
|
+
* managed to make wrong at all. Mirrors `bench/grader-outcome-adjudicate.mjs`'s
|
|
29
|
+
* `CONTROL_COVERAGE_FLOOR`, which guards the neighbouring shrink (rows the adjudicator refused
|
|
30
|
+
* to judge) at the same 0.6 — restated rather than imported, on the same terms as TYPE_FLOOR and
|
|
31
|
+
* COVERAGE_FLOOR. If you change one, change both. */
|
|
32
|
+
const CONTROL_CONSTRUCTION_FLOOR = 0.6;
|
|
33
|
+
/**
|
|
34
|
+
* The headline recall denominator. Mirrors `HEADLINE_AXIS` in
|
|
35
|
+
* `bench/lib/recall-baseline.mjs`, restated on the same terms as `TYPE_FLOOR`
|
|
36
|
+
* and `COVERAGE_FLOOR`: a bench instrument is not a dependency of the shipped
|
|
37
|
+
* gate. **If you change one, change both.** `packages/core/test/tiers.test.ts`
|
|
38
|
+
* and `bench/test/recall-verdict.test.mjs` both pin it.
|
|
39
|
+
*/
|
|
40
|
+
const HEADLINE_RECALL_AXIS = "wholeCodebase";
|
|
41
|
+
/**
|
|
42
|
+
* `COVERAGE_FLOOR` again, for recall.
|
|
43
|
+
*
|
|
44
|
+
* Precision says how well the grader did on what it judged; coverage says how
|
|
45
|
+
* much it was willing to judge. Recall has the identical hole: an instrument
|
|
46
|
+
* that draws 2000 sites, decides 118 of them and reports 118/118 = 100% has
|
|
47
|
+
* measured its own willingness to be graded. That is `ruby/fluentd`, verbatim.
|
|
48
|
+
*
|
|
49
|
+
* **This floor DISQUALIFIES a figure and does not fail an adapter**, and unlike
|
|
50
|
+
* every other constant in this file it is attached to nothing that moves a
|
|
51
|
+
* tier. Two measured reasons, both of which would have to change first:
|
|
52
|
+
*
|
|
53
|
+
* 1. `bench/gate-evidence.json` carries no recall rows for any adapter. This
|
|
54
|
+
* file's doctrine is that absent evidence fails, so a recall REQUIREMENT
|
|
55
|
+
* today would drop every adapter at once — on the evidence's absence, not
|
|
56
|
+
* on any adapter's performance.
|
|
57
|
+
* 2. Nothing in the corpus is close. On `bench/recall-baseline.json` the
|
|
58
|
+
* headline axis reads 6.7%–45.3% across eighteen rows, best
|
|
59
|
+
* `java/shopizer` at 45.3%. A requirement at any rate a reader would accept
|
|
60
|
+
* is red for every row, and a gate red for every row is a gate people stop
|
|
61
|
+
* reading.
|
|
62
|
+
*
|
|
63
|
+
* Mirrors `RECALL_COVERAGE_FLOOR` in `bench/recall-verdict.mjs`, which mirrors
|
|
64
|
+
* `COVERAGE_FLOOR` above. Three places, one value; change one, change all.
|
|
65
|
+
*/
|
|
66
|
+
const RECALL_COVERAGE_FLOOR = 0.6;
|
|
15
67
|
function missing(wanted, have) {
|
|
16
68
|
if (wanted === undefined)
|
|
17
69
|
return [];
|
|
@@ -22,10 +74,16 @@ function missing(wanted, have) {
|
|
|
22
74
|
* Fixes a defect where a dedicated single-type draw read as silence — Python's SERVES_API drew
|
|
23
75
|
* 200/200 and 48/48 (re-verified) but printed "0/20 decided". A dedicated zero-row draw counts as 0. */
|
|
24
76
|
function decidedRowsOfType(draw, type) {
|
|
25
|
-
|
|
26
|
-
|
|
77
|
+
return rowsOfType(draw, type)?.decided;
|
|
78
|
+
}
|
|
79
|
+
/** Decided AND correct for one edge type. Both, because the floor is about both — see `TYPE_RATE_FLOOR`. */
|
|
80
|
+
function rowsOfType(draw, type) {
|
|
81
|
+
if (draw.perEdgeType !== undefined) {
|
|
82
|
+
const bucket = draw.perEdgeType[type];
|
|
83
|
+
return bucket === undefined ? { decided: 0, correct: 0 } : { decided: bucket.decided, correct: bucket.correct };
|
|
84
|
+
}
|
|
27
85
|
if (draw.edgeType === type)
|
|
28
|
-
return draw.decided ?? draw.drawn;
|
|
86
|
+
return { decided: draw.decided ?? draw.drawn, correct: draw.correct };
|
|
29
87
|
return undefined;
|
|
30
88
|
}
|
|
31
89
|
function checkCapability(requirement, evidence) {
|
|
@@ -55,7 +113,23 @@ function checkCapability(requirement, evidence) {
|
|
|
55
113
|
continue; // already named in edgeGap above
|
|
56
114
|
if (!isPromotionRequirement)
|
|
57
115
|
continue;
|
|
58
|
-
|
|
116
|
+
// **Only draws whose own negative control held.** The control is the sole
|
|
117
|
+
// detector for an adjudicator that says yes to everything, so a draw whose
|
|
118
|
+
// control exceeded the ceiling carries an UNGUARDED precision figure — and
|
|
119
|
+
// routing its per-type rows into a capability check would be the control
|
|
120
|
+
// gate in `checkPrecisionDraws` bypassed by a second code path. Measured:
|
|
121
|
+
// kotlin/ktor's USES_API draw reads 54/54 with a control of 4/54 = 7.4%, and
|
|
122
|
+
// its own artefact records `passed: false`; recording it moved kotlin from
|
|
123
|
+
// Tier B to Tier A until this line existed.
|
|
124
|
+
const guarded = evidence.precisionDraws.filter((d) => d.controlDrawn > 0 && d.controlCorrect / d.controlDrawn <= MAX_CONTROL);
|
|
125
|
+
const unguarded = evidence.precisionDraws.filter((d) => !guarded.includes(d) && (decidedRowsOfType(d, type) ?? 0) > 0);
|
|
126
|
+
for (const d of unguarded) {
|
|
127
|
+
gaps.push(`${type} evidence from ${d.repository} is not counted: its negative control is ` +
|
|
128
|
+
`${d.controlCorrect}/${d.controlDrawn}` +
|
|
129
|
+
(d.controlDrawn === 0 ? " — no control was run at all" : `, above ${MAX_CONTROL * 100}%`) +
|
|
130
|
+
`, so that draw's precision figure is unguarded and cannot earn a capability`);
|
|
131
|
+
}
|
|
132
|
+
const contributions = guarded
|
|
59
133
|
.map((d) => decidedRowsOfType(d, type))
|
|
60
134
|
.filter((n) => n !== undefined);
|
|
61
135
|
if (contributions.length === 0) {
|
|
@@ -64,6 +138,21 @@ function checkCapability(requirement, evidence) {
|
|
|
64
138
|
continue;
|
|
65
139
|
}
|
|
66
140
|
const decided = contributions.reduce((sum, n) => sum + n, 0);
|
|
141
|
+
// **The rate, checked at the floor and never above it.** Below TYPE_FLOOR the
|
|
142
|
+
// sample is too small to say anything, and the count check below is what
|
|
143
|
+
// rejects it; at or above the floor a type that is measurably wrong must not
|
|
144
|
+
// be the evidence that grants the requirement it fails.
|
|
145
|
+
const measured = guarded
|
|
146
|
+
.map((d) => rowsOfType(d, type))
|
|
147
|
+
.filter((r) => r !== undefined)
|
|
148
|
+
.reduce((sum, r) => ({ decided: sum.decided + r.decided, correct: sum.correct + r.correct }), { decided: 0, correct: 0 });
|
|
149
|
+
if (measured.decided >= TYPE_FLOOR && measured.correct / measured.decided < TYPE_RATE_FLOOR) {
|
|
150
|
+
gaps.push(`${type} ${measured.correct}/${measured.decided} = ` +
|
|
151
|
+
`${Math.round((100 * measured.correct) / measured.decided)}% across the recorded precision draws, ` +
|
|
152
|
+
`below the per-type floor of ${TYPE_RATE_FLOOR * 100}% — the draws that measure this capability ` +
|
|
153
|
+
`measure it FAILING, and a failing measurement is not evidence for the requirement it fails`);
|
|
154
|
+
continue;
|
|
155
|
+
}
|
|
67
156
|
if (decided < TYPE_FLOOR) {
|
|
68
157
|
gaps.push(`${type} ${decided}/${TYPE_FLOOR} decided rows across the recorded precision draws — declared, ` +
|
|
69
158
|
`but not validated by the draw at the floor every edge type is otherwise held to`);
|
|
@@ -98,6 +187,32 @@ function checkPrecisionDraws(requirement, evidence) {
|
|
|
98
187
|
}
|
|
99
188
|
const reasons = [];
|
|
100
189
|
const qualifying = draws.filter((d) => {
|
|
190
|
+
// **Checked before anything else, because every check after it is a `<`.**
|
|
191
|
+
// `undefined / 32` is NaN, and every comparison with NaN is false — so a row
|
|
192
|
+
// carrying no usable counts does not fail these gates, it passes all of
|
|
193
|
+
// them in turn. That is not hypothetical: a recorder wrote four rows with
|
|
194
|
+
// `correct: undefined` and this function awarded Tier A on them, one of
|
|
195
|
+
// which had really measured 19/25 = 76%.
|
|
196
|
+
//
|
|
197
|
+
// A numeric guard rather than a `=== undefined` guard, because NaN and
|
|
198
|
+
// Infinity arrive the same way (a rate multiplied back into a count, a
|
|
199
|
+
// division by a zero denominator) and read the same way downstream.
|
|
200
|
+
const counts = [
|
|
201
|
+
["drawn", d.drawn],
|
|
202
|
+
["correct", d.correct],
|
|
203
|
+
["controlDrawn", d.controlDrawn],
|
|
204
|
+
["controlCorrect", d.controlCorrect],
|
|
205
|
+
["decided", d.decided ?? 0],
|
|
206
|
+
["population", d.population ?? 0],
|
|
207
|
+
["controlUncontrollable", d.controlUncontrollable ?? 0],
|
|
208
|
+
];
|
|
209
|
+
const unusable = counts.filter(([, value]) => typeof value !== "number" || !Number.isFinite(value));
|
|
210
|
+
if (unusable.length > 0) {
|
|
211
|
+
reasons.push(`${d.repository}: no usable counts — ${unusable.map(([name]) => name).join(", ")} ` +
|
|
212
|
+
`${unusable.length === 1 ? "is" : "are"} not a number. A draw that cannot be compared against a ` +
|
|
213
|
+
`threshold must not pass one: absent evidence is a failure, never a pass`);
|
|
214
|
+
return false;
|
|
215
|
+
}
|
|
101
216
|
if (d.authorChose) {
|
|
102
217
|
reasons.push(`${d.repository}: the author chose this repository, so it does not count toward the two`);
|
|
103
218
|
return false;
|
|
@@ -129,8 +244,24 @@ function checkPrecisionDraws(requirement, evidence) {
|
|
|
129
244
|
return false;
|
|
130
245
|
}
|
|
131
246
|
// Precision = correct/DECIDED, never /drawn — folding an honest refusal-to-judge into the
|
|
132
|
-
// denominator punishes it exactly like a wrong answer.
|
|
247
|
+
// denominator punishes it exactly like a wrong answer.
|
|
133
248
|
const decided = d.decided ?? d.drawn;
|
|
249
|
+
// ...but a denominator the grader chooses is a denominator the grader can shrink. Gated
|
|
250
|
+
// together with precision, never apart: `bench/kotlin-precision.mjs` reasoned this out
|
|
251
|
+
// first ("refusal with no floor is a way to buy any precision you like by refusing
|
|
252
|
+
// whatever you would have got wrong") and enforced it inside itself, where it protected
|
|
253
|
+
// Kotlin's draws and no others. Moving it here is what makes the refusal design honest
|
|
254
|
+
// for every grader that records `decided`, not just the one that invented the field.
|
|
255
|
+
//
|
|
256
|
+
// A draw with no `decided` is NOT charged this. Its grader has no third outcome, so it
|
|
257
|
+
// decided every row it drew; there is no shrinking to guard against, and charging it
|
|
258
|
+
// would fail every row recorded before the distinction existed.
|
|
259
|
+
if (d.decided !== undefined && decided / d.drawn < COVERAGE_FLOOR) {
|
|
260
|
+
reasons.push(`${d.repository}: decided ${decided}/${d.drawn}, coverage ` +
|
|
261
|
+
`${Math.round((decided / d.drawn) * 100)}% below ${COVERAGE_FLOOR * 100}% — precision over a ` +
|
|
262
|
+
`denominator this small is a claim about what the grader chose to judge, not about the adapter`);
|
|
263
|
+
return false;
|
|
264
|
+
}
|
|
134
265
|
if (d.correct / decided < MIN_PRECISION) {
|
|
135
266
|
reasons.push(`${d.repository}: precision ${d.correct}/${decided}` +
|
|
136
267
|
(decided === d.drawn ? "" : ` (of ${d.drawn} drawn, ${decided} decided)`) +
|
|
@@ -145,6 +276,17 @@ function checkPrecisionDraws(requirement, evidence) {
|
|
|
145
276
|
reasons.push(`${d.repository}: no negative control was run, so the precision figure is unguarded`);
|
|
146
277
|
return false;
|
|
147
278
|
}
|
|
279
|
+
// ...and the same shrink again, one layer down. A row whose swap could not be made wrong
|
|
280
|
+
// tested nothing and is rightly out of the denominator; a grader free to declare most of its
|
|
281
|
+
// rows that way can report 0/10 and print the same clean sentence as a control that rejected
|
|
282
|
+
// 200. Charged only where the grader records the bucket at all.
|
|
283
|
+
const attempted = d.controlDrawn + (d.controlUncontrollable ?? 0);
|
|
284
|
+
if (d.controlUncontrollable !== undefined && d.controlDrawn / attempted < CONTROL_CONSTRUCTION_FLOOR) {
|
|
285
|
+
reasons.push(`${d.repository}: ${d.controlDrawn} of ${attempted} control swaps could be made wrong at all, ` +
|
|
286
|
+
`below ${CONTROL_CONSTRUCTION_FLOOR * 100}% — a control this much of which tested nothing is not ` +
|
|
287
|
+
`a guard on the precision figure beside it`);
|
|
288
|
+
return false;
|
|
289
|
+
}
|
|
148
290
|
return true;
|
|
149
291
|
});
|
|
150
292
|
const repositories = new Set(qualifying.map((d) => d.repository));
|
|
@@ -204,6 +346,95 @@ function capabilitiesHonest(evidence) {
|
|
|
204
346
|
const overEdges = missing(evidence.capabilities.edgeTypes, evidence.emittedEdgeTypes);
|
|
205
347
|
return overNodes.length === 0 && overEdges.length === 0;
|
|
206
348
|
}
|
|
349
|
+
/**
|
|
350
|
+
* Rule one recall draw. Pure, and it decides nothing about a tier.
|
|
351
|
+
*
|
|
352
|
+
* Three ways a figure says nothing, and they are different statements:
|
|
353
|
+
* - it is over a NARROWER axis than the headline, so it answers a different
|
|
354
|
+
* question from the one a reader will read it as answering;
|
|
355
|
+
* - it has no draw size, so its coverage has no denominator;
|
|
356
|
+
* - it was graded on too little of its own draw.
|
|
357
|
+
*
|
|
358
|
+
* None of them is "the adapter did badly", and the certificate says so.
|
|
359
|
+
*/
|
|
360
|
+
function ruleRecallDraw(draw) {
|
|
361
|
+
const rate = draw.denominator === 0 ? null : draw.found / draw.denominator;
|
|
362
|
+
const coverage = draw.drawn > 0 ? draw.denominator / draw.drawn : null;
|
|
363
|
+
const base = {
|
|
364
|
+
repository: draw.repository,
|
|
365
|
+
axis: draw.axis,
|
|
366
|
+
found: draw.found,
|
|
367
|
+
denominator: draw.denominator,
|
|
368
|
+
drawn: draw.drawn,
|
|
369
|
+
rate,
|
|
370
|
+
coverage,
|
|
371
|
+
...(draw.secondAxis === undefined ? {} : { secondAxis: draw.secondAxis }),
|
|
372
|
+
...(draw.generatedExcluded === undefined ? {} : { generatedExcluded: draw.generatedExcluded }),
|
|
373
|
+
recordedBy: draw.recordedBy,
|
|
374
|
+
};
|
|
375
|
+
if (draw.axis !== HEADLINE_RECALL_AXIS) {
|
|
376
|
+
return {
|
|
377
|
+
...base,
|
|
378
|
+
admissible: false,
|
|
379
|
+
detail: `recorded on the '${draw.axis}' axis, not the headline '${HEADLINE_RECALL_AXIS}'. ` +
|
|
380
|
+
(draw.axis === "inCodebase"
|
|
381
|
+
? "`inCodebase` divides by the references whose target is declared inside the repository, so every " +
|
|
382
|
+
"call into a library is absent from its denominator — it excludes precisely what the graph could " +
|
|
383
|
+
"not represent. Reportable beside the headline; never as it"
|
|
384
|
+
: "`narrow` divides by found + missing and drops everything the instrument declined to attempt — " +
|
|
385
|
+
"on this corpus that is 78% of drawn sites. Reportable beside the headline; never as it"),
|
|
386
|
+
};
|
|
387
|
+
}
|
|
388
|
+
if (coverage === null) {
|
|
389
|
+
return { ...base, admissible: false, detail: "the draw records no draw size, so coverage has no denominator" };
|
|
390
|
+
}
|
|
391
|
+
if (draw.denominator === 0) {
|
|
392
|
+
return {
|
|
393
|
+
...base,
|
|
394
|
+
admissible: false,
|
|
395
|
+
detail: `all ${draw.drawn} drawn sites were excluded; nothing was graded, which is undefined and not 0%`,
|
|
396
|
+
};
|
|
397
|
+
}
|
|
398
|
+
if (coverage < RECALL_COVERAGE_FLOOR) {
|
|
399
|
+
return {
|
|
400
|
+
...base,
|
|
401
|
+
admissible: false,
|
|
402
|
+
detail: `${draw.found}/${draw.denominator} over ${draw.drawn} drawn — coverage ` +
|
|
403
|
+
`${Math.round(coverage * 100)}% is below ${RECALL_COVERAGE_FLOOR * 100}%, so this figure is a claim ` +
|
|
404
|
+
`about what the instrument was willing to grade, not about the adapter`,
|
|
405
|
+
};
|
|
406
|
+
}
|
|
407
|
+
return {
|
|
408
|
+
...base,
|
|
409
|
+
admissible: true,
|
|
410
|
+
detail: `${draw.found}/${draw.denominator} on the ${HEADLINE_RECALL_AXIS} axis, over ${draw.drawn} drawn`,
|
|
411
|
+
};
|
|
412
|
+
}
|
|
413
|
+
/**
|
|
414
|
+
* The whole recall disclosure, including the case where there is none.
|
|
415
|
+
*
|
|
416
|
+
* **It never contributes to `awarded`.** `packages/core/test/tiers.test.ts`
|
|
417
|
+
* asserts that the worst possible recall draw leaves the tier where it was; if
|
|
418
|
+
* that assertion ever fails, recall has quietly become a gate and every tier on
|
|
419
|
+
* every certificate is now a claim about a measurement no adapter records.
|
|
420
|
+
*/
|
|
421
|
+
function discloseRecall(evidence) {
|
|
422
|
+
const draws = evidence.recallDraws ?? [];
|
|
423
|
+
if (draws.length === 0) {
|
|
424
|
+
return {
|
|
425
|
+
state: "not-measured",
|
|
426
|
+
detail: "no recall draw is recorded for this adapter, so this certificate says nothing about how much of a " +
|
|
427
|
+
"codebase reaches the graph. The tier above is about how RIGHT the edges are, not how many there are",
|
|
428
|
+
rows: [],
|
|
429
|
+
};
|
|
430
|
+
}
|
|
431
|
+
return {
|
|
432
|
+
state: "disclosed",
|
|
433
|
+
detail: `recall is disclosed and gates nothing: the headline axis is '${HEADLINE_RECALL_AXIS}', and a figure ` +
|
|
434
|
+
`below the ${RECALL_COVERAGE_FLOOR * 100}% coverage floor is disqualified rather than charged to the adapter`,
|
|
435
|
+
rows: draws.map(ruleRecallDraw),
|
|
436
|
+
};
|
|
437
|
+
}
|
|
207
438
|
export function certify(evidence) {
|
|
208
439
|
const byTier = { S: [], A: [], B: [], C: [] };
|
|
209
440
|
const limits = admissibleLimits(evidence);
|
|
@@ -245,9 +476,26 @@ export function certify(evidence) {
|
|
|
245
476
|
unreachableHere: evidence.precisionDraws
|
|
246
477
|
.filter((d) => drawableFrom(d) === 0)
|
|
247
478
|
.map((d) => ({ repository: d.repository, edgeType: d.edgeType })),
|
|
479
|
+
// Not folded into `limitedIds` either, and for a sharper version of the same reason: a
|
|
480
|
+
// ceiling that waived would let any language buy a tier by declaring its own corpus poor.
|
|
481
|
+
corpusCeilings: (evidence.corpusCeilings ?? []).filter((c) => c.declaredBy.trim() !== ""),
|
|
482
|
+
// Derived like `unreachableHere`, and disclosed whether or not the draw qualified: a draw
|
|
483
|
+
// rejected by the construction floor still says something about its grader, and a draw that
|
|
484
|
+
// passed with rows excluded has a weaker control than its clean number suggests.
|
|
485
|
+
weakenedControls: evidence.precisionDraws
|
|
486
|
+
.filter((d) => (d.controlUncontrollable ?? 0) > 0)
|
|
487
|
+
.map((d) => ({
|
|
488
|
+
repository: d.repository,
|
|
489
|
+
edgeType: d.edgeType,
|
|
490
|
+
constructed: d.controlDrawn,
|
|
491
|
+
uncontrollable: d.controlUncontrollable ?? 0,
|
|
492
|
+
})),
|
|
248
493
|
// Not folded into `limitedIds` — a cap waives nothing, and if it could skip a requirement that'd be the cheapest route to a tier.
|
|
249
494
|
policyCaps: admissibleCaps(evidence),
|
|
250
495
|
capabilitiesHonest: capabilitiesHonest(evidence),
|
|
496
|
+
// Disclosed, never gating. See `RECALL_COVERAGE_FLOOR` for the two measured
|
|
497
|
+
// reasons, and `discloseRecall` for the test that keeps it that way.
|
|
498
|
+
recall: discloseRecall(evidence),
|
|
251
499
|
};
|
|
252
500
|
}
|
|
253
501
|
/** Tier first, then immediately why it isn't higher. Printing "B" alone teaches a reader B is
|
|
@@ -273,6 +521,26 @@ export function formatCertificate(certificate) {
|
|
|
273
521
|
lines.push(` NOT REACHABLE IN THIS REFERENCE SET — ${empty.edgeType} on ${empty.repository}: population 0`);
|
|
274
522
|
lines.push(" Nothing here to sample. The remedy is other repositories, not more measurement");
|
|
275
523
|
}
|
|
524
|
+
// "Not done yet" and "not reachable from the repositories we are allowed to use" are
|
|
525
|
+
// different sentences, and the second is the one eight languages are actually in. Printed
|
|
526
|
+
// beside the unmet rows it explains, never instead of them — the requirement is still unmet.
|
|
527
|
+
for (const ceiling of certificate.corpusCeilings) {
|
|
528
|
+
lines.push(` CEILING ACCEPTED — ${ceiling.id}: ${ceiling.reason}`);
|
|
529
|
+
lines.push(` declared by ${ceiling.declaredBy}; the admitted corpus cannot supply this, and admitting ` +
|
|
530
|
+
`another repository BECAUSE it would move a tier is the move the selection rules forbid. ` +
|
|
531
|
+
`Published as a ceiling, not as work outstanding. Waives nothing`);
|
|
532
|
+
}
|
|
533
|
+
// The control's own disclosure. A swap that could not be made wrong tested nothing, which is
|
|
534
|
+
// why it left the denominator; printing the count is what keeps "0 of 196, 4 untestable" from
|
|
535
|
+
// reading as "0 of 196".
|
|
536
|
+
for (const weak of certificate.weakenedControls) {
|
|
537
|
+
const attempted = weak.constructed + weak.uncontrollable;
|
|
538
|
+
lines.push(` CONTROL WEAKENED — ${weak.repository} (${weak.edgeType}): ${weak.uncontrollable} of ${attempted} ` +
|
|
539
|
+
`control swaps could not be made wrong at all`);
|
|
540
|
+
lines.push(` Every candidate the pool offered for those rows was itself a real edge, so the swap tested ` +
|
|
541
|
+
`nothing and left the denominator. Scored over ${weak.constructed}. A genuine over-acceptance on ` +
|
|
542
|
+
`one of those rows would not have been counted`);
|
|
543
|
+
}
|
|
276
544
|
for (const cap of certificate.policyCaps) {
|
|
277
545
|
lines.push(` CAPPED BY POLICY — ${cap.id}: claims ${cap.claimed}, evidence supports ${cap.claimable}`);
|
|
278
546
|
lines.push(` pending ${cap.pendingOn}; declared by ${cap.declaredBy}. Not a limit — a deliberate under-claim`);
|
|
@@ -283,6 +551,42 @@ export function formatCertificate(certificate) {
|
|
|
283
551
|
else if (certificate.capabilitiesHonest === "unknown") {
|
|
284
552
|
lines.push(" capabilities() not compared against a run — DEC-075 §6.3 unverified here");
|
|
285
553
|
}
|
|
554
|
+
// COVERAGE, NOT CORRECTNESS, and it is printed on every certificate including
|
|
555
|
+
// the ones that have nothing to say. A tier is a statement about how right the
|
|
556
|
+
// edges are; a reader handed only that will answer "how much of my codebase did
|
|
557
|
+
// you analyse" with it, and on this corpus the honest answer to that question is
|
|
558
|
+
// between 6.7% and 45.3%. Printed BEFORE the early return below, so a Tier S
|
|
559
|
+
// certificate does not skip it.
|
|
560
|
+
if (certificate.recall.state === "not-measured") {
|
|
561
|
+
lines.push(" RECALL — not measured");
|
|
562
|
+
lines.push(` ${certificate.recall.detail}`);
|
|
563
|
+
}
|
|
564
|
+
else {
|
|
565
|
+
for (const row of certificate.recall.rows) {
|
|
566
|
+
const rate = row.rate === null ? "undefined (nothing graded)" : `${(100 * row.rate).toFixed(1)}%`;
|
|
567
|
+
const coverage = row.coverage === null ? "—" : `${(100 * row.coverage).toFixed(1)}%`;
|
|
568
|
+
lines.push(` RECALL — ${row.repository}: ${rate} on the ${row.axis} axis ` +
|
|
569
|
+
`(${row.found}/${row.denominator}, ${row.drawn} drawn, coverage ${coverage})` +
|
|
570
|
+
(row.admissible ? "" : " <- DISQUALIFIED AS A FIGURE, not charged to the adapter"));
|
|
571
|
+
lines.push(` ${row.detail}; recorded by ${row.recordedBy}`);
|
|
572
|
+
if (row.secondAxis !== undefined) {
|
|
573
|
+
const second = row.secondAxis;
|
|
574
|
+
const secondRate = second.denominator === 0 ? "undefined" : `${((100 * second.found) / second.denominator).toFixed(1)}%`;
|
|
575
|
+
lines.push(` second axis ${second.axis}: ${second.found}/${second.denominator} = ${secondRate} ` +
|
|
576
|
+
`— reported beside the headline, never as it`);
|
|
577
|
+
}
|
|
578
|
+
if (row.generatedExcluded === undefined) {
|
|
579
|
+
// An undisclosed exclusion and no exclusion must not read the same.
|
|
580
|
+
lines.push(" no generated-code exclusion was recorded for this draw");
|
|
581
|
+
}
|
|
582
|
+
else {
|
|
583
|
+
const excluded = row.generatedExcluded;
|
|
584
|
+
lines.push(` generated code excluded: ${excluded.files} file(s), ${excluded.sites} drawn site(s), by ` +
|
|
585
|
+
`${excluded.rules.join(", ")} — the figure above is over a denominator that much smaller`);
|
|
586
|
+
}
|
|
587
|
+
}
|
|
588
|
+
lines.push(` ${certificate.recall.detail}`);
|
|
589
|
+
}
|
|
286
590
|
if (certificate.nextTier === null) {
|
|
287
591
|
lines.push(" nothing above this tier");
|
|
288
592
|
return lines.join("\n");
|