eval-quality 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +142 -67
- package/corpus/dev/README.md +17 -11
- package/corpus/dev/compile-seal-example/brief.json +1 -1
- package/corpus/dev/compile-seal-example/contract.json +1 -1
- package/corpus/dev/contracts/absent-collection-locations.json +1 -1
- package/corpus/dev/contracts/absent-sibling-groups.json +1 -1
- package/corpus/dev/contracts/absent-success-indicator.json +1 -1
- package/corpus/dev/contracts/empty-channel-roles.json +1 -1
- package/corpus/dev/contracts/empty-collection-locations.json +1 -1
- package/corpus/dev/contracts/empty-request-shapes.json +1 -1
- package/corpus/dev/contracts/empty-sibling-groups.json +1 -1
- package/corpus/dev/contracts/no-collection-quantifier.json +1 -1
- package/corpus/dev/contracts/no-operation-inventory.json +1 -1
- package/corpus/dev/contracts/no-read-back-relation.json +1 -1
- package/corpus/dev/contracts/no-state-change-marker.json +1 -1
- package/corpus/dev/contracts/no-type-violating-step.json +1 -1
- package/corpus/dev/contracts/per-key-split-oracles.json +1 -1
- package/corpus/dev/contracts/satisfied-declarations.json +1 -1
- package/corpus/dev/contracts/single-required-response-key.json +1 -1
- package/corpus/dev/contracts/split-indicator-oracle.json +1 -1
- package/corpus/dev/contracts/unaddressed-parameter-sibling.json +1 -1
- package/corpus/dev/contracts/unnamed-reference-set.json +1 -1
- package/corpus/dev/contracts/wrong-cardinality-form.json +1 -1
- package/corpus/dev/index.json +1 -1
- package/dist/application/index.d.ts +2 -0
- package/dist/application/index.js +1 -0
- package/dist/application/score.d.ts +48 -0
- package/dist/application/score.js +186 -0
- package/dist/cli/arguments.d.ts +7 -3
- package/dist/cli/arguments.js +73 -5
- package/dist/cli/exit-codes.d.ts +6 -2
- package/dist/cli/exit-codes.js +8 -9
- package/dist/cli/main.js +9 -0
- package/dist/cli/render.d.ts +1 -1
- package/dist/cli/render.js +6 -4
- package/dist/cli/run.d.ts +13 -3
- package/dist/cli/run.js +136 -16
- package/dist/core/canonical/canonicalize.d.ts +12 -0
- package/dist/core/canonical/canonicalize.js +18 -4
- package/dist/core/compile/bindings.d.ts +70 -0
- package/dist/core/compile/bindings.js +317 -0
- package/dist/core/compile/compile.d.ts +5 -0
- package/dist/core/compile/compile.js +9 -0
- package/dist/core/compile/expression-legality.d.ts +49 -0
- package/dist/core/compile/expression-legality.js +105 -31
- package/dist/core/compile/forbidden-inputs.d.ts +8 -1
- package/dist/core/compile/forbidden-inputs.js +16 -3
- package/dist/core/compile/interface-inventory.d.ts +23 -1
- package/dist/core/compile/interface-inventory.js +32 -3
- package/dist/core/compile/reachability.d.ts +18 -1
- package/dist/core/compile/reachability.js +46 -1
- package/dist/core/compile/sensitivity-witness.d.ts +1 -1
- package/dist/core/compile/sensitivity-witness.js +1 -1
- package/dist/core/emit/emit.d.ts +9 -0
- package/dist/core/emit/emit.js +214 -0
- package/dist/core/emit/private-artifact-digest.d.ts +11 -0
- package/dist/core/emit/private-artifact-digest.js +31 -0
- package/dist/core/evaluate/evidence-resolution.d.ts +11 -1
- package/dist/core/evaluate/evidence-resolution.js +4 -1
- package/dist/core/failure-codes.d.ts +2 -2
- package/dist/core/failure-codes.js +4 -2
- package/dist/core/ingest/conditions.d.ts +201 -0
- package/dist/core/ingest/conditions.js +48 -0
- package/dist/core/ingest/index.d.ts +13 -0
- package/dist/core/ingest/index.js +11 -0
- package/dist/core/ingest/ingest.d.ts +68 -0
- package/dist/core/ingest/ingest.js +310 -0
- package/dist/core/lineage/stage-table.d.ts +21 -4
- package/dist/core/lineage/stage-table.js +35 -3
- package/dist/core/preflight/witness-evidence.js +10 -0
- package/dist/core/schemas/artifact.d.ts +385 -14
- package/dist/core/schemas/constraint-ledger.js +40 -0
- package/dist/core/schemas/defect-signature.d.ts +273 -0
- package/dist/core/schemas/defect-signature.js +127 -0
- package/dist/core/schemas/eval-contract.d.ts +33 -0
- package/dist/core/schemas/eval-contract.js +14 -0
- package/dist/core/schemas/evidence-artifact.d.ts +99 -19
- package/dist/core/schemas/evidence-artifact.js +64 -8
- package/dist/core/schemas/plan.d.ts +112 -0
- package/dist/core/schemas/plan.js +67 -2
- package/dist/core/schemas/probe-qualification.d.ts +168 -0
- package/dist/core/schemas/probe-qualification.js +106 -0
- package/dist/core/schemas/probe.d.ts +312 -0
- package/dist/core/schemas/probe.js +6 -2
- package/dist/core/schemas/scoring-policy.d.ts +1 -0
- package/dist/core/schemas/scoring-policy.js +6 -1
- package/dist/core/schemas/sealed-evaluator-brief.d.ts +1 -0
- package/dist/core/schemas/sealed-evaluator-brief.js +4 -1
- package/dist/core/schemas/sealed-run-record.d.ts +30 -3
- package/dist/core/schemas/sealed-run-record.js +37 -5
- package/dist/core/score/binding-order.d.ts +31 -0
- package/dist/core/score/binding-order.js +88 -0
- package/dist/core/score/bindings.d.ts +93 -0
- package/dist/core/score/bindings.js +297 -0
- package/dist/core/score/ladder-table.d.ts +23 -0
- package/dist/core/score/ladder-table.js +109 -0
- package/dist/core/score/ladder.d.ts +229 -0
- package/dist/core/score/ladder.js +464 -0
- package/dist/core/score/mode-agreement.d.ts +30 -0
- package/dist/core/score/mode-agreement.js +16 -0
- package/dist/core/score/outcome-table.d.ts +17 -0
- package/dist/core/score/outcome-table.js +172 -0
- package/dist/core/score/outcome.d.ts +466 -0
- package/dist/core/score/outcome.js +562 -0
- package/dist/core/score/qualification.d.ts +83 -0
- package/dist/core/score/qualification.js +615 -0
- package/dist/core/score/quotation.d.ts +79 -0
- package/dist/core/score/quotation.js +201 -0
- package/dist/core/score/reduce-trials.d.ts +80 -0
- package/dist/core/score/reduce-trials.js +90 -0
- package/dist/core/score/score.d.ts +72 -0
- package/dist/core/score/score.js +581 -0
- package/dist/core/score/selection.d.ts +80 -0
- package/dist/core/score/selection.js +98 -0
- package/dist/core/score/strength.d.ts +49 -0
- package/dist/core/score/strength.js +193 -0
- package/dist/core/score/witness.d.ts +117 -0
- package/dist/core/score/witness.js +315 -0
- package/dist/core/seal/derived-reference.d.ts +1 -1
- package/dist/core/seal/derived-reference.js +236 -28
- package/dist/core/seal/seal.js +8 -3
- package/dist/core/stage-contracts.d.ts +62 -2
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/testing/index.d.ts +2 -0
- package/dist/testing/index.js +1 -0
- package/package.json +12 -3
- package/schemas/eval-contract.schema.json +100 -2
- package/schemas/evidence-artifact.schema.json +96 -9
- package/schemas/probe.schema.json +508 -3
- package/schemas/scoring-policy.schema.json +8 -1
- package/schemas/sealed-evaluator-brief.schema.json +11 -1
- package/schemas/sealed-run-record.schema.json +19 -2
|
@@ -0,0 +1,562 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The four buckets `mapFindings` sorts a defect finding into, each carrying a
|
|
3
|
+
* distinct AD consequence. Keyed on the cited finding's own `probeId`, which
|
|
4
|
+
* is unrelated to whatever probe an outcome row is about.
|
|
5
|
+
*/
|
|
6
|
+
export const FINDING_BUCKETS = [
|
|
7
|
+
'mapped',
|
|
8
|
+
'unmapped',
|
|
9
|
+
'dangling',
|
|
10
|
+
'signatureless',
|
|
11
|
+
];
|
|
12
|
+
/**
|
|
13
|
+
* AD-6 qualifies `not-applicable` by an unexpired waiver satisfying AD-5.
|
|
14
|
+
* Completeness is settled at compile time by `checkWaiverCompleteness`, and
|
|
15
|
+
* `core/score` reads no clock, so both expiry and whether the waiver's opaque
|
|
16
|
+
* context condition was met arrive already decided.
|
|
17
|
+
*/
|
|
18
|
+
export const WAIVER_STATES = [
|
|
19
|
+
'none',
|
|
20
|
+
'applied-condition-met',
|
|
21
|
+
'applied-condition-unmet',
|
|
22
|
+
'expired',
|
|
23
|
+
];
|
|
24
|
+
/**
|
|
25
|
+
* AD-17 names no field for a malformed judge response and `JudgeResult` is
|
|
26
|
+
* keyed by rubric criterion, so conduct arrives per oracle. `absent` is the
|
|
27
|
+
* ordinary value: a contract with no rubric produces no judge call.
|
|
28
|
+
*/
|
|
29
|
+
export const JUDGE_CONDUCT_STATES = [
|
|
30
|
+
'absent',
|
|
31
|
+
'conforming',
|
|
32
|
+
'malformed',
|
|
33
|
+
];
|
|
34
|
+
const witnessResultOf = (inputs) => inputs.witness === null ? null : inputs.witness.result;
|
|
35
|
+
const witnessIs = (inputs, result) => witnessResultOf(inputs) === result;
|
|
36
|
+
/** whether a defect finding cites this oracle. */
|
|
37
|
+
const citesDefect = (inputs) => inputs.citedFinding !== null;
|
|
38
|
+
const bucketOf = (inputs) => inputs.citedFinding === null ? null : inputs.citedFinding.bucket;
|
|
39
|
+
const probeUnqualified = (inputs) => inputs.probeClass !== null && inputs.probeQualified === false;
|
|
40
|
+
const someSelectionResolved = (inputs) => inputs.selections.some((selection) => selection.result !== 'none');
|
|
41
|
+
/** `true` under `expects-hold`, `false` under `expects-violation`. AD-4 makes `insufficient-evidence` terminal, and a `null` resolution never ran, so neither value satisfies and neither fails. */
|
|
42
|
+
const checkSatisfied = (inputs) => inputs.checkResolution ===
|
|
43
|
+
(inputs.polarity === 'expects-hold' ? 'true' : 'false');
|
|
44
|
+
const checkExaminedNothing = (inputs) => inputs.checkResolution === 'insufficient-evidence';
|
|
45
|
+
const dispositionUnsupported = (inputs) => {
|
|
46
|
+
const disposition = inputs.disposition;
|
|
47
|
+
return (disposition !== null &&
|
|
48
|
+
disposition.observationIds.length === 0 &&
|
|
49
|
+
(disposition.disposition === 'held' ||
|
|
50
|
+
disposition.disposition === 'violated'));
|
|
51
|
+
};
|
|
52
|
+
/**
|
|
53
|
+
* `violated` with no defect finding, `held` with one, or `not-attempted` with
|
|
54
|
+
* one. A `not-attempted` citing observations is well supported: those are the
|
|
55
|
+
* observations showing why the evaluator could not proceed. Its contradiction
|
|
56
|
+
* is a defect finding filed against the same oracle.
|
|
57
|
+
*/
|
|
58
|
+
const dispositionContradictsEvidence = (inputs) => {
|
|
59
|
+
const disposition = inputs.disposition;
|
|
60
|
+
if (disposition === null)
|
|
61
|
+
return false;
|
|
62
|
+
const cited = citesDefect(inputs);
|
|
63
|
+
if (disposition.disposition === 'violated')
|
|
64
|
+
return !cited;
|
|
65
|
+
return cited;
|
|
66
|
+
};
|
|
67
|
+
const citationDeclined = (inputs) => {
|
|
68
|
+
const bucket = bucketOf(inputs);
|
|
69
|
+
return bucket === 'unmapped' || bucket === 'signatureless';
|
|
70
|
+
};
|
|
71
|
+
/**
|
|
72
|
+
* A canary carries no signature, so it has no witness and step reachability is
|
|
73
|
+
* the only proxy for whether the evaluator took the path. AD-33 names
|
|
74
|
+
* punishing a contract for the evaluator's path choice as an anti-goal, so the
|
|
75
|
+
* condition and its ladder row share this guard.
|
|
76
|
+
*/
|
|
77
|
+
const canaryUndetected = (inputs) => inputs.probeClass === 'canary' &&
|
|
78
|
+
someSelectionResolved(inputs) &&
|
|
79
|
+
!citesDefect(inputs);
|
|
80
|
+
/**
|
|
81
|
+
* Stage A. Ten predicates evaluated independently of the state ladder, so a
|
|
82
|
+
* condition is never masked by another that fired first. Every one that holds
|
|
83
|
+
* is returned.
|
|
84
|
+
*
|
|
85
|
+
* Nine reach AD-21's Invalid rung through an AD-6 invalidating state or, for
|
|
86
|
+
* `disposition-missing`, by name. `unsupported-disposition` produces no state
|
|
87
|
+
* and is not in that enumeration; the half enforced here is the corroboration,
|
|
88
|
+
* which is AD-33's own requirement that an unsupported disposition invalidates
|
|
89
|
+
* cross-artifact agreement.
|
|
90
|
+
*/
|
|
91
|
+
export const INVALIDATING_CONDITIONS = [
|
|
92
|
+
{
|
|
93
|
+
id: 'evaluation-fault',
|
|
94
|
+
guard: 'an AD-26 evaluation fault was recorded',
|
|
95
|
+
holds: (inputs) => inputs.evaluationFault,
|
|
96
|
+
},
|
|
97
|
+
{
|
|
98
|
+
id: 'judge-malformed',
|
|
99
|
+
guard: 'judge conduct `malformed`',
|
|
100
|
+
holds: (inputs) => inputs.judgeConduct === 'malformed',
|
|
101
|
+
},
|
|
102
|
+
{
|
|
103
|
+
id: 'unqualified-probe-in-sealed-set',
|
|
104
|
+
guard: 'a probe is present and its qualification failed',
|
|
105
|
+
holds: probeUnqualified,
|
|
106
|
+
},
|
|
107
|
+
{
|
|
108
|
+
id: 'dangling-probe-citation',
|
|
109
|
+
guard: "the cited finding's bucket is `dangling`",
|
|
110
|
+
holds: (inputs) => bucketOf(inputs) === 'dangling',
|
|
111
|
+
},
|
|
112
|
+
{
|
|
113
|
+
id: 'unwitnessed-detection-claim',
|
|
114
|
+
guard: 'witness result `unwitnessed-claim`',
|
|
115
|
+
holds: (inputs) => witnessIs(inputs, 'unwitnessed-claim'),
|
|
116
|
+
},
|
|
117
|
+
{
|
|
118
|
+
id: 'vacuous-signature',
|
|
119
|
+
guard: 'witness result `vacuous`',
|
|
120
|
+
holds: (inputs) => witnessIs(inputs, 'vacuous'),
|
|
121
|
+
},
|
|
122
|
+
{
|
|
123
|
+
id: 'selector-ambiguity',
|
|
124
|
+
guard: 'a step matched several observations under a single-valued cardinality',
|
|
125
|
+
holds: (inputs) => inputs.selectorAmbiguity,
|
|
126
|
+
},
|
|
127
|
+
{
|
|
128
|
+
id: 'canary-non-detection',
|
|
129
|
+
guard: 'class `canary`, some selection resolved other than `none`, and no defect finding cites the oracle',
|
|
130
|
+
holds: canaryUndetected,
|
|
131
|
+
},
|
|
132
|
+
{
|
|
133
|
+
id: 'unsupported-disposition',
|
|
134
|
+
guard: 'the disposition is `held` or `violated` with empty `observationIds`',
|
|
135
|
+
holds: dispositionUnsupported,
|
|
136
|
+
},
|
|
137
|
+
{
|
|
138
|
+
id: 'disposition-missing',
|
|
139
|
+
guard: 'the oracle is required and its disposition is `null`',
|
|
140
|
+
holds: (inputs) => inputs.required && inputs.disposition === null,
|
|
141
|
+
},
|
|
142
|
+
];
|
|
143
|
+
/**
|
|
144
|
+
* Row 9. The witness conjunct keeps an unmatched selection from deleting a
|
|
145
|
+
* witnessed detection, and admits `not-triggered`, the one witness value
|
|
146
|
+
* carrying no detection to protect. The length conjunct keeps an oracle that
|
|
147
|
+
* declared no step out, since AD-6 scopes `unreached` to declared steps and
|
|
148
|
+
* `every` is vacuous over an empty array.
|
|
149
|
+
*/
|
|
150
|
+
const stepsUnreached = (inputs) => (inputs.witness === null || witnessIs(inputs, 'not-triggered')) &&
|
|
151
|
+
inputs.selections.length > 0 &&
|
|
152
|
+
inputs.selections.every((selection) => selection.result === 'none');
|
|
153
|
+
/** AD-33's first fixed cell. A `zero-action` probe on the seeding branch seeds a defect whose correct behaviour is refusal, so "satisfied" is the signature manifesting and a finding witnessing it, which is `matched`. */
|
|
154
|
+
const zeroActionDetected = (inputs) => inputs.probeClass === 'zero-action' &&
|
|
155
|
+
inputs.expectedClean === false &&
|
|
156
|
+
witnessIs(inputs, 'matched');
|
|
157
|
+
const cleanControlFalsePositive = (inputs) => inputs.expectedClean === true && citesDefect(inputs);
|
|
158
|
+
/**
|
|
159
|
+
* A clean control whose check examined nothing falls past this row to the
|
|
160
|
+
* abstention row: AD-4 makes `insufficient-evidence` terminal and AD-6 lands
|
|
161
|
+
* it on `abstained`, so a build cannot pass green on an oracle that examined
|
|
162
|
+
* an empty collection. The false-positive row stays above both; a filed
|
|
163
|
+
* finding survives a check that examined nothing.
|
|
164
|
+
*/
|
|
165
|
+
const cleanControlPassed = (inputs) => inputs.expectedClean === true && !checkExaminedNothing(inputs);
|
|
166
|
+
const canaryDetected = (inputs) => inputs.probeClass === 'canary' && citesDefect(inputs);
|
|
167
|
+
/**
|
|
168
|
+
* The three witness rows above the clean-control pair each carry
|
|
169
|
+
* `expectedClean`, so AD-9's legal states for a clean control hold over the
|
|
170
|
+
* whole input type, beyond the tuples the qualification gate can produce. The
|
|
171
|
+
* unexercised row carries the guard for its own reason: AD-6 legalises
|
|
172
|
+
* `not-applicable` for a probe AD-40 records as unexercised, and a clean
|
|
173
|
+
* control carries no signature, so AD-40 records nothing about it.
|
|
174
|
+
*/
|
|
175
|
+
const witnessUnexercised = (inputs) => witnessIs(inputs, 'unexercised') && inputs.expectedClean !== true;
|
|
176
|
+
const witnessMatched = (inputs) => witnessIs(inputs, 'matched') && inputs.expectedClean !== true;
|
|
177
|
+
const witnessManifestedUnclaimed = (inputs) => witnessIs(inputs, 'manifested-unclaimed') && inputs.expectedClean !== true;
|
|
178
|
+
/**
|
|
179
|
+
* On a witness-free oracle, `mapped` is the one bucket that resolves `caught`.
|
|
180
|
+
* AD-40 calls an unmapped finding an unexpected real defect and keeps it out
|
|
181
|
+
* of every catch.
|
|
182
|
+
*/
|
|
183
|
+
const oracleCitedDefect = (inputs) => inputs.witness === null && bucketOf(inputs) === 'mapped';
|
|
184
|
+
/** Row 20's guard, written out as the negation of every guard above it. */
|
|
185
|
+
const outcomeClear = (inputs) => !inputs.evaluationFault &&
|
|
186
|
+
inputs.judgeConduct !== 'malformed' &&
|
|
187
|
+
!probeUnqualified(inputs) &&
|
|
188
|
+
bucketOf(inputs) !== 'dangling' &&
|
|
189
|
+
!witnessIs(inputs, 'unwitnessed-claim') &&
|
|
190
|
+
!witnessIs(inputs, 'vacuous') &&
|
|
191
|
+
!inputs.selectorAmbiguity &&
|
|
192
|
+
!witnessUnexercised(inputs) &&
|
|
193
|
+
!stepsUnreached(inputs) &&
|
|
194
|
+
!zeroActionDetected(inputs) &&
|
|
195
|
+
!cleanControlFalsePositive(inputs) &&
|
|
196
|
+
!cleanControlPassed(inputs) &&
|
|
197
|
+
!canaryDetected(inputs) &&
|
|
198
|
+
!canaryUndetected(inputs) &&
|
|
199
|
+
!witnessMatched(inputs) &&
|
|
200
|
+
!witnessManifestedUnclaimed(inputs) &&
|
|
201
|
+
!oracleCitedDefect(inputs) &&
|
|
202
|
+
!checkExaminedNothing(inputs) &&
|
|
203
|
+
!witnessIs(inputs, 'not-triggered');
|
|
204
|
+
/**
|
|
205
|
+
* Stage B. First match wins; the identifier of the row that fired is
|
|
206
|
+
* returned.
|
|
207
|
+
*
|
|
208
|
+
* Rows 5, 6, 8, 15, 16, and 19 consume all six witness results, so anything
|
|
209
|
+
* reaching row 20 carries no witness. Rows 9 and 18 each take a subset of an
|
|
210
|
+
* earlier row's domain without widening it, which is why the witness rows are
|
|
211
|
+
* not contiguous. Rows 13 and 14 are deliberately not total over `canary`: an
|
|
212
|
+
* undetected canary that matched no selection falls past both and lands on row
|
|
213
|
+
* 18 or row 20 unless row 9 took it.
|
|
214
|
+
*
|
|
215
|
+
* Rows 10 and 15 overlap on a `zero-action` probe with a `matched` witness.
|
|
216
|
+
* Row 10 is kept so AD-33's first fixed cell has a line of its own in the
|
|
217
|
+
* emitted table.
|
|
218
|
+
*/
|
|
219
|
+
export const OUTCOME_RULES = [
|
|
220
|
+
{
|
|
221
|
+
id: 'evaluation-fault',
|
|
222
|
+
guard: 'an AD-26 evaluation fault was recorded',
|
|
223
|
+
state: 'oracle-error',
|
|
224
|
+
resolvesFromCitation: false,
|
|
225
|
+
holds: (inputs) => inputs.evaluationFault,
|
|
226
|
+
},
|
|
227
|
+
{
|
|
228
|
+
id: 'judge-malformed',
|
|
229
|
+
guard: 'judge conduct `malformed`',
|
|
230
|
+
state: 'judge-error',
|
|
231
|
+
resolvesFromCitation: false,
|
|
232
|
+
holds: (inputs) => inputs.judgeConduct === 'malformed',
|
|
233
|
+
},
|
|
234
|
+
{
|
|
235
|
+
id: 'probe-unqualified',
|
|
236
|
+
guard: 'a probe is present and its qualification failed',
|
|
237
|
+
state: 'infrastructure-error',
|
|
238
|
+
resolvesFromCitation: false,
|
|
239
|
+
holds: probeUnqualified,
|
|
240
|
+
},
|
|
241
|
+
{
|
|
242
|
+
id: 'finding-dangling-probe',
|
|
243
|
+
guard: "the cited finding's bucket is `dangling`",
|
|
244
|
+
state: 'infrastructure-error',
|
|
245
|
+
resolvesFromCitation: true,
|
|
246
|
+
holds: (inputs) => bucketOf(inputs) === 'dangling',
|
|
247
|
+
},
|
|
248
|
+
{
|
|
249
|
+
id: 'witness-unwitnessed-claim',
|
|
250
|
+
guard: 'witness result `unwitnessed-claim`',
|
|
251
|
+
state: 'infrastructure-error',
|
|
252
|
+
resolvesFromCitation: false,
|
|
253
|
+
holds: (inputs) => witnessIs(inputs, 'unwitnessed-claim'),
|
|
254
|
+
},
|
|
255
|
+
{
|
|
256
|
+
id: 'witness-vacuous',
|
|
257
|
+
guard: 'witness result `vacuous`',
|
|
258
|
+
state: 'infrastructure-error',
|
|
259
|
+
resolvesFromCitation: false,
|
|
260
|
+
holds: (inputs) => witnessIs(inputs, 'vacuous'),
|
|
261
|
+
},
|
|
262
|
+
{
|
|
263
|
+
id: 'selector-ambiguous',
|
|
264
|
+
guard: 'a step matched several observations under a single-valued cardinality',
|
|
265
|
+
state: 'infrastructure-error',
|
|
266
|
+
resolvesFromCitation: false,
|
|
267
|
+
holds: (inputs) => inputs.selectorAmbiguity,
|
|
268
|
+
},
|
|
269
|
+
{
|
|
270
|
+
id: 'witness-unexercised',
|
|
271
|
+
guard: 'witness result `unexercised` on a probe outside the `expectedClean` branch',
|
|
272
|
+
state: 'not-applicable',
|
|
273
|
+
resolvesFromCitation: false,
|
|
274
|
+
holds: witnessUnexercised,
|
|
275
|
+
},
|
|
276
|
+
{
|
|
277
|
+
id: 'steps-unreached',
|
|
278
|
+
guard: 'no witness or witness result `not-triggered`; `selections` non-empty; every member resolved `none`',
|
|
279
|
+
state: 'unreached',
|
|
280
|
+
resolvesFromCitation: false,
|
|
281
|
+
holds: stepsUnreached,
|
|
282
|
+
},
|
|
283
|
+
{
|
|
284
|
+
id: 'zero-action-detected',
|
|
285
|
+
guard: 'class `zero-action` on the seeding branch with witness result `matched`',
|
|
286
|
+
state: 'caught',
|
|
287
|
+
resolvesFromCitation: false,
|
|
288
|
+
holds: zeroActionDetected,
|
|
289
|
+
},
|
|
290
|
+
{
|
|
291
|
+
id: 'clean-control-false-positive',
|
|
292
|
+
guard: '`expectedClean` and a defect finding cites the oracle',
|
|
293
|
+
state: 'false-positive',
|
|
294
|
+
resolvesFromCitation: true,
|
|
295
|
+
holds: cleanControlFalsePositive,
|
|
296
|
+
},
|
|
297
|
+
{
|
|
298
|
+
id: 'clean-control-passed',
|
|
299
|
+
guard: '`expectedClean` and the check root did not resolve `insufficient-evidence`',
|
|
300
|
+
state: 'passed-clean-control',
|
|
301
|
+
resolvesFromCitation: false,
|
|
302
|
+
holds: cleanControlPassed,
|
|
303
|
+
},
|
|
304
|
+
{
|
|
305
|
+
id: 'canary-detected',
|
|
306
|
+
guard: 'class `canary` and a defect finding cites the oracle',
|
|
307
|
+
state: 'caught',
|
|
308
|
+
resolvesFromCitation: true,
|
|
309
|
+
holds: canaryDetected,
|
|
310
|
+
},
|
|
311
|
+
{
|
|
312
|
+
id: 'canary-undetected',
|
|
313
|
+
guard: 'class `canary`, some selection resolved other than `none`, and no defect finding cites the oracle',
|
|
314
|
+
state: 'infrastructure-error',
|
|
315
|
+
resolvesFromCitation: false,
|
|
316
|
+
holds: canaryUndetected,
|
|
317
|
+
},
|
|
318
|
+
{
|
|
319
|
+
id: 'witness-matched',
|
|
320
|
+
guard: 'witness result `matched` on a probe outside the `expectedClean` branch',
|
|
321
|
+
state: 'caught',
|
|
322
|
+
resolvesFromCitation: false,
|
|
323
|
+
holds: witnessMatched,
|
|
324
|
+
},
|
|
325
|
+
{
|
|
326
|
+
id: 'witness-manifested-unclaimed',
|
|
327
|
+
guard: 'witness result `manifested-unclaimed` on a probe outside the `expectedClean` branch',
|
|
328
|
+
state: 'missed',
|
|
329
|
+
resolvesFromCitation: false,
|
|
330
|
+
holds: witnessManifestedUnclaimed,
|
|
331
|
+
},
|
|
332
|
+
{
|
|
333
|
+
id: 'oracle-cited-defect',
|
|
334
|
+
guard: "no witness and the cited finding's bucket is `mapped`",
|
|
335
|
+
state: 'caught',
|
|
336
|
+
resolvesFromCitation: true,
|
|
337
|
+
holds: oracleCitedDefect,
|
|
338
|
+
},
|
|
339
|
+
{
|
|
340
|
+
id: 'check-insufficient-evidence',
|
|
341
|
+
guard: 'the check root resolved `insufficient-evidence`',
|
|
342
|
+
state: 'abstained',
|
|
343
|
+
resolvesFromCitation: false,
|
|
344
|
+
holds: checkExaminedNothing,
|
|
345
|
+
},
|
|
346
|
+
{
|
|
347
|
+
id: 'witness-not-triggered',
|
|
348
|
+
guard: 'witness result `not-triggered`',
|
|
349
|
+
state: 'confirmed',
|
|
350
|
+
resolvesFromCitation: false,
|
|
351
|
+
holds: (inputs) => witnessIs(inputs, 'not-triggered'),
|
|
352
|
+
},
|
|
353
|
+
{
|
|
354
|
+
id: 'outcome-clear',
|
|
355
|
+
guard: 'the stated negation of every guard above',
|
|
356
|
+
state: 'confirmed',
|
|
357
|
+
resolvesFromCitation: false,
|
|
358
|
+
holds: outcomeClear,
|
|
359
|
+
},
|
|
360
|
+
];
|
|
361
|
+
/**
|
|
362
|
+
* A waiver excuses a known gap, and `missed` is the only gap here.
|
|
363
|
+
* `false-positive` is a clean control's own calibration; `abstained` is an
|
|
364
|
+
* unknown. AD-7 counts every exercised probe in its denominator, so a waiver
|
|
365
|
+
* honoured over a `matched` witness would depress the catch rate.
|
|
366
|
+
*/
|
|
367
|
+
export const WAIVABLE_FAILURES = [
|
|
368
|
+
'missed',
|
|
369
|
+
];
|
|
370
|
+
const isWaivable = (state) => WAIVABLE_FAILURES.some((waivable) => waivable === state);
|
|
371
|
+
/**
|
|
372
|
+
* Neither rule fires on `none` or `expired`, which is AD-21's expired waiver
|
|
373
|
+
* reinstating its gap. `bypassed` is a gap excused without earning the excuse,
|
|
374
|
+
* which is the group AD-6 puts it in and the only firing condition the AD
|
|
375
|
+
* gives it anywhere.
|
|
376
|
+
*/
|
|
377
|
+
export const WAIVER_RULES = [
|
|
378
|
+
{
|
|
379
|
+
id: 'waiver-honoured',
|
|
380
|
+
guard: 'waiver `applied-condition-met` over a waivable failure',
|
|
381
|
+
state: 'not-applicable',
|
|
382
|
+
holds: (inputs, provisional) => inputs.waiver === 'applied-condition-met' && isWaivable(provisional),
|
|
383
|
+
},
|
|
384
|
+
{
|
|
385
|
+
id: 'waiver-bypassed',
|
|
386
|
+
guard: 'waiver `applied-condition-unmet` over a waivable failure',
|
|
387
|
+
state: 'bypassed',
|
|
388
|
+
holds: (inputs, provisional) => inputs.waiver === 'applied-condition-unmet' && isWaivable(provisional),
|
|
389
|
+
},
|
|
390
|
+
];
|
|
391
|
+
/**
|
|
392
|
+
* The corroboration table, decided after the final state. First match wins and
|
|
393
|
+
* no row is an `otherwise`.
|
|
394
|
+
*
|
|
395
|
+
* The order carries three obligations. Rows 1 to 3 sit above both
|
|
396
|
+
* check-derived rows so a disposition and a declined citation are not
|
|
397
|
+
* believed. Row 4 sits above row 5 so a check that resolved
|
|
398
|
+
* `insufficient-evidence` never records `not-evaluable`, which AD-33 forbids
|
|
399
|
+
* and which row 9 of the ladder makes reachable. Rows 6, 7, and 8 partition
|
|
400
|
+
* satisfaction against citation, so the table is total.
|
|
401
|
+
*
|
|
402
|
+
* A `disagrees` is diagnostic and moves nothing on its own, and it carries the
|
|
403
|
+
* whole cost of the tail collapsing to `confirmed`. Row 3 is where an
|
|
404
|
+
* `unmapped` or `signatureless` citation records that the evaluator and the
|
|
405
|
+
* oracle are pointing at different defects.
|
|
406
|
+
*
|
|
407
|
+
* Row 5 widens AD-33's `not-evaluable` from unreached steps to any `null` root
|
|
408
|
+
* resolution, which is the same condition reached by a second route: an oracle
|
|
409
|
+
* whose `check` is `null`, half of what `oracle-missing-channel` fires on.
|
|
410
|
+
*/
|
|
411
|
+
export const CORROBORATION_RULES = [
|
|
412
|
+
{
|
|
413
|
+
id: 'disposition-unsupported',
|
|
414
|
+
guard: 'the disposition is `held` or `violated` with empty `observationIds`',
|
|
415
|
+
value: '`disagrees`',
|
|
416
|
+
holds: dispositionUnsupported,
|
|
417
|
+
corroboration: () => 'disagrees',
|
|
418
|
+
},
|
|
419
|
+
{
|
|
420
|
+
id: 'disposition-contradicts-evidence',
|
|
421
|
+
guard: '`violated` with no defect finding, `held` with one, or `not-attempted` with one',
|
|
422
|
+
value: '`disagrees`',
|
|
423
|
+
holds: dispositionContradictsEvidence,
|
|
424
|
+
corroboration: () => 'disagrees',
|
|
425
|
+
},
|
|
426
|
+
{
|
|
427
|
+
id: 'citation-declined',
|
|
428
|
+
guard: "the cited finding's bucket is `unmapped` or `signatureless`",
|
|
429
|
+
value: '`disagrees`',
|
|
430
|
+
holds: citationDeclined,
|
|
431
|
+
corroboration: () => 'disagrees',
|
|
432
|
+
},
|
|
433
|
+
{
|
|
434
|
+
id: 'examined-nothing',
|
|
435
|
+
guard: 'the check root resolved `insufficient-evidence`',
|
|
436
|
+
value: '`disagrees` where a defect finding cited the oracle, `agrees` where none did',
|
|
437
|
+
holds: checkExaminedNothing,
|
|
438
|
+
corroboration: (inputs) => (citesDefect(inputs) ? 'disagrees' : 'agrees'),
|
|
439
|
+
},
|
|
440
|
+
{
|
|
441
|
+
id: 'never-ran',
|
|
442
|
+
guard: 'the final state is `unreached`, or the check root resolution is `null`',
|
|
443
|
+
value: '`not-evaluable`',
|
|
444
|
+
holds: (inputs, state) => state === 'unreached' || inputs.checkResolution === null,
|
|
445
|
+
corroboration: () => 'not-evaluable',
|
|
446
|
+
},
|
|
447
|
+
{
|
|
448
|
+
id: 'check-confirms-silence',
|
|
449
|
+
guard: 'the check satisfies and no defect finding cited the oracle',
|
|
450
|
+
value: '`agrees`',
|
|
451
|
+
holds: (inputs) => checkSatisfied(inputs) && !citesDefect(inputs),
|
|
452
|
+
corroboration: () => 'agrees',
|
|
453
|
+
},
|
|
454
|
+
{
|
|
455
|
+
id: 'check-confirms-finding',
|
|
456
|
+
guard: 'the check does not satisfy and a defect finding cited the oracle',
|
|
457
|
+
value: '`agrees`',
|
|
458
|
+
holds: (inputs) => !checkSatisfied(inputs) && citesDefect(inputs),
|
|
459
|
+
corroboration: () => 'agrees',
|
|
460
|
+
},
|
|
461
|
+
{
|
|
462
|
+
id: 'check-and-findings-diverge',
|
|
463
|
+
guard: 'the check satisfies with a finding cited, or does not satisfy with none',
|
|
464
|
+
value: '`disagrees`',
|
|
465
|
+
holds: (inputs) => checkSatisfied(inputs) === citesDefect(inputs),
|
|
466
|
+
corroboration: () => 'disagrees',
|
|
467
|
+
},
|
|
468
|
+
];
|
|
469
|
+
/**
|
|
470
|
+
* The identifiers the outcome records: each selection's matches in the array
|
|
471
|
+
* order of `selections`, then the witness's candidates, deduplicated and
|
|
472
|
+
* keeping first appearance.
|
|
473
|
+
*
|
|
474
|
+
* A step selects on its own operation and the witness on the signature's home
|
|
475
|
+
* operation, so neither list contains the other. Each is already ascending
|
|
476
|
+
* under the same total comparator, which makes the concatenation deterministic
|
|
477
|
+
* and stable under a permutation of the record's observations, as owed item 2
|
|
478
|
+
* requires. One ascending order over both would need the observations
|
|
479
|
+
* themselves among the inputs.
|
|
480
|
+
*/
|
|
481
|
+
const selectedObservationIdsOf = (inputs) => {
|
|
482
|
+
const selected = [];
|
|
483
|
+
const seen = new Set();
|
|
484
|
+
const take = (ids) => {
|
|
485
|
+
for (const id of ids) {
|
|
486
|
+
if (seen.has(id))
|
|
487
|
+
continue;
|
|
488
|
+
seen.add(id);
|
|
489
|
+
selected.push(id);
|
|
490
|
+
}
|
|
491
|
+
};
|
|
492
|
+
for (const selection of inputs.selections)
|
|
493
|
+
take(selection.matchedObservationIds);
|
|
494
|
+
if (inputs.witness !== null)
|
|
495
|
+
take(inputs.witness.observationIds);
|
|
496
|
+
return selected;
|
|
497
|
+
};
|
|
498
|
+
const byIdentifier = (left, right) => left < right ? -1 : left > right ? 1 : 0;
|
|
499
|
+
/**
|
|
500
|
+
* AD-33's total reference decision procedure. One AD-6 state, one
|
|
501
|
+
* corroboration value, the rules that produced them, and the evidence the
|
|
502
|
+
* resolution read.
|
|
503
|
+
*/
|
|
504
|
+
export function resolveOutcome(inputs) {
|
|
505
|
+
const invalidatingConditions = INVALIDATING_CONDITIONS.filter((condition) => condition.holds(inputs))
|
|
506
|
+
.map((condition) => condition.id)
|
|
507
|
+
.sort(byIdentifier);
|
|
508
|
+
// Total: row 20's guard is the negation of every guard above it.
|
|
509
|
+
const rule = OUTCOME_RULES.find((candidate) => candidate.holds(inputs));
|
|
510
|
+
const waiverRule = WAIVER_RULES.find((candidate) => candidate.holds(inputs, rule.state)) ??
|
|
511
|
+
null;
|
|
512
|
+
const state = waiverRule === null ? rule.state : waiverRule.state;
|
|
513
|
+
// Total: rows 6, 7, and 8 partition satisfaction against citation.
|
|
514
|
+
const corroborationRule = CORROBORATION_RULES.find((candidate) => candidate.holds(inputs, state));
|
|
515
|
+
const cited = inputs.citedFinding;
|
|
516
|
+
const resolvedFrom = cited !== null && rule.resolvesFromCitation ? cited.findingId : null;
|
|
517
|
+
const declined = cited !== null && !rule.resolvesFromCitation && citationDeclined(inputs);
|
|
518
|
+
return {
|
|
519
|
+
rule: rule.id,
|
|
520
|
+
waiverRule: waiverRule === null ? null : waiverRule.id,
|
|
521
|
+
corroborationRule: corroborationRule.id,
|
|
522
|
+
state,
|
|
523
|
+
corroboration: corroborationRule.corroboration(inputs),
|
|
524
|
+
resolvedFrom,
|
|
525
|
+
selectedObservationIds: selectedObservationIdsOf(inputs),
|
|
526
|
+
declinedFindingIds: declined ? [cited.findingId] : [],
|
|
527
|
+
invalidatingConditions,
|
|
528
|
+
};
|
|
529
|
+
}
|
|
530
|
+
/**
|
|
531
|
+
* The findings citing no oracle. AD-33 keeps them: discarding one would hide
|
|
532
|
+
* the evaluator-chosen detection AD-23 exists to preserve. Named apart from
|
|
533
|
+
* the artifact's own `uncitedFindings` field so the two do not collide, and
|
|
534
|
+
* covering every finding type, since AD-23's carve-out is about the oracle
|
|
535
|
+
* citation.
|
|
536
|
+
*/
|
|
537
|
+
export function uncitedFindingIds(record) {
|
|
538
|
+
return record.findings
|
|
539
|
+
.filter((finding) => finding.oracleId === null)
|
|
540
|
+
.map((finding) => finding.findingId)
|
|
541
|
+
.sort(byIdentifier);
|
|
542
|
+
}
|
|
543
|
+
/**
|
|
544
|
+
* Owed item 5: every uncited `defect` finding, as `UncitedFindingGap`'s
|
|
545
|
+
* persisted shape. Deliberately not a rename or extension of
|
|
546
|
+
* `uncitedFindingIds` above: that function is broader (every finding type)
|
|
547
|
+
* and thinner (an identifier only), and already backs the artifact's own bare
|
|
548
|
+
* `uncitedFindings` field; this one is narrower (`defect` only, since only
|
|
549
|
+
* that branch carries `quotedEvidence`) and richer (the full gap record), and
|
|
550
|
+
* the two coexist rather than one replacing the other.
|
|
551
|
+
*/
|
|
552
|
+
export function uncitedDefectFindingGaps(record) {
|
|
553
|
+
return record.findings
|
|
554
|
+
.filter((finding) => finding.findingType === 'defect' && finding.oracleId === null)
|
|
555
|
+
.map((finding) => ({
|
|
556
|
+
findingId: finding.findingId,
|
|
557
|
+
observationIds: finding.observationIds,
|
|
558
|
+
quotedEvidence: finding.quotedEvidence,
|
|
559
|
+
severity: finding.severity,
|
|
560
|
+
}))
|
|
561
|
+
.sort((a, b) => byIdentifier(a.findingId, b.findingId));
|
|
562
|
+
}
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
import { type DefectSignature } from '../schemas/defect-signature.ts';
|
|
2
|
+
import type { Operation, PermittedInterface } from '../schemas/interface.ts';
|
|
3
|
+
import type { Probe } from '../schemas/probe.ts';
|
|
4
|
+
/**
|
|
5
|
+
* The closed reason set. Nothing outside it can come back from `qualifyProbe`,
|
|
6
|
+
* so a caller routing a rejection to a rung has a finite table to write.
|
|
7
|
+
*/
|
|
8
|
+
export declare const QUALIFICATION_FAILURES: readonly ['qualification-route-incompatible', 'qualification-defect-sources-mixed', 'qualification-evidence-unverified', 'signature-absent', 'signature-present-on-canary', 'signature-interface-kind-unsupported', 'signature-observable-channel-not-response-side', 'condition-channels-underspecified', 'condition-disjunct-without-response-channel', 'condition-selector-key-undeclared', 'condition-pointer-not-observation-rooted', 'condition-pointer-unwritable', 'condition-text-channel-on-api', 'condition-reference-set-operand', 'condition-operand-illegal', 'condition-regex-illegal', 'condition-quantifier-nesting', 'condition-quantifier-over-non-collection', 'condition-bound-element-outside-quantifier'];
|
|
9
|
+
export type QualificationFailureCode = (typeof QUALIFICATION_FAILURES)[number];
|
|
10
|
+
export type QualificationFailure = {
|
|
11
|
+
readonly code: QualificationFailureCode;
|
|
12
|
+
/** the probe-rooted path, in the same spelling a structural failure uses. */
|
|
13
|
+
readonly artifactPath: string;
|
|
14
|
+
readonly detail: string;
|
|
15
|
+
};
|
|
16
|
+
export type QualificationResult = {
|
|
17
|
+
readonly qualified: boolean;
|
|
18
|
+
readonly failures: readonly QualificationFailure[];
|
|
19
|
+
/**
|
|
20
|
+
* Whether the declaration-dependent checks ran. Three of them read the home
|
|
21
|
+
* operation's declared request and response shapes, which a corpus holds
|
|
22
|
+
* nowhere: the quantifier-over-non-collection rule, evidence reachability,
|
|
23
|
+
* and the selector's own keys. The first two catch a predicate that was
|
|
24
|
+
* never writable; the third catches a selector that matches nothing, which
|
|
25
|
+
* is a different failure with the same silent outcome. A caller qualifying
|
|
26
|
+
* against no inventory is told which three did not run rather than being
|
|
27
|
+
* handed a pass that hid them.
|
|
28
|
+
*/
|
|
29
|
+
readonly declarationChecksRan: boolean;
|
|
30
|
+
};
|
|
31
|
+
/**
|
|
32
|
+
* Resolves a signature's home operation against a contract's operation
|
|
33
|
+
* inventory, comparing method plus path template with parameter names erased
|
|
34
|
+
* first, so a corpus signature on `/notes/{id}` binds a contract declaring
|
|
35
|
+
* `/notes/{noteId}`. A post-erasure collision inside one contract has already
|
|
36
|
+
* failed compilation under `duplicate-operation-signature`, so the first match
|
|
37
|
+
* is the only match for any contract that compiled.
|
|
38
|
+
*/
|
|
39
|
+
export declare function resolveHomeOperation(signature: DefectSignature, interfaces: readonly PermittedInterface[]): Operation | null;
|
|
40
|
+
/**
|
|
41
|
+
* AD-9's gate over one probe. `homeOperation` is the operation the signature
|
|
42
|
+
* resolves to in whatever inventory the caller is qualifying against, or `null`
|
|
43
|
+
* when there is none to qualify against; the three declaration-dependent checks
|
|
44
|
+
* are skipped in that case and `declarationChecksRan` says so.
|
|
45
|
+
*/
|
|
46
|
+
export declare function qualifyProbe(probe: Probe, homeOperation: Operation | null): QualificationResult;
|
|
47
|
+
export type QualifiedProbe = {
|
|
48
|
+
readonly probe: Probe;
|
|
49
|
+
readonly result: QualificationResult;
|
|
50
|
+
};
|
|
51
|
+
export type SealedProbeSet = {
|
|
52
|
+
/**
|
|
53
|
+
* Each admitted probe with the result that admitted it, not a bare probe.
|
|
54
|
+
* `declarationChecksRan` is the reason: an admission granted without an
|
|
55
|
+
* operation inventory skipped the three checks that read declared shapes,
|
|
56
|
+
* and a bare `Probe[]` would drop that fact at the seal. A sealed set whose
|
|
57
|
+
* admissions were only half-checked is a different artifact from one whose
|
|
58
|
+
* admissions were fully checked, and whoever scores it has to be able to
|
|
59
|
+
* tell them apart.
|
|
60
|
+
*/
|
|
61
|
+
readonly admitted: readonly QualifiedProbe[];
|
|
62
|
+
readonly rejected: readonly QualifiedProbe[];
|
|
63
|
+
};
|
|
64
|
+
/**
|
|
65
|
+
* AD-9's "an unqualified probe cannot enter a sealed set", as a
|
|
66
|
+
* construction-time filter that reports its exclusions.
|
|
67
|
+
*
|
|
68
|
+
* Construction-time, and score never re-filters. Silently dropping an
|
|
69
|
+
* unqualified probe at score time would shrink AD-7's denominator and
|
|
70
|
+
* desynchronise the AD-8 corpus digest from the probes actually scored, while
|
|
71
|
+
* AD-7 makes comparability the corpus digest restricted to the probes both
|
|
72
|
+
* results cover. A sealed set that nonetheless contains an unqualified probe is
|
|
73
|
+
* an invalidating condition for whoever scores it, never something the witness
|
|
74
|
+
* match quietly repairs.
|
|
75
|
+
*
|
|
76
|
+
* `homeOperationOf` is required rather than defaulted. A default resolving
|
|
77
|
+
* nothing would make the unchecked path the one a caller reaches by writing
|
|
78
|
+
* less, and the three checks it skips are the ones that catch a signature that
|
|
79
|
+
* was never writable and a selector that matches nothing. A caller who holds
|
|
80
|
+
* no inventory writes `() => null` at the call site, where the choice shows up
|
|
81
|
+
* in a diff.
|
|
82
|
+
*/
|
|
83
|
+
export declare function sealProbeSet(probes: readonly Probe[], homeOperationOf: (probe: Probe) => Operation | null): SealedProbeSet;
|