eval-quality 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +142 -67
- package/corpus/dev/README.md +17 -11
- package/corpus/dev/compile-seal-example/brief.json +1 -1
- package/corpus/dev/compile-seal-example/contract.json +1 -1
- package/corpus/dev/contracts/absent-collection-locations.json +1 -1
- package/corpus/dev/contracts/absent-sibling-groups.json +1 -1
- package/corpus/dev/contracts/absent-success-indicator.json +1 -1
- package/corpus/dev/contracts/empty-channel-roles.json +1 -1
- package/corpus/dev/contracts/empty-collection-locations.json +1 -1
- package/corpus/dev/contracts/empty-request-shapes.json +1 -1
- package/corpus/dev/contracts/empty-sibling-groups.json +1 -1
- package/corpus/dev/contracts/no-collection-quantifier.json +1 -1
- package/corpus/dev/contracts/no-operation-inventory.json +1 -1
- package/corpus/dev/contracts/no-read-back-relation.json +1 -1
- package/corpus/dev/contracts/no-state-change-marker.json +1 -1
- package/corpus/dev/contracts/no-type-violating-step.json +1 -1
- package/corpus/dev/contracts/per-key-split-oracles.json +1 -1
- package/corpus/dev/contracts/satisfied-declarations.json +1 -1
- package/corpus/dev/contracts/single-required-response-key.json +1 -1
- package/corpus/dev/contracts/split-indicator-oracle.json +1 -1
- package/corpus/dev/contracts/unaddressed-parameter-sibling.json +1 -1
- package/corpus/dev/contracts/unnamed-reference-set.json +1 -1
- package/corpus/dev/contracts/wrong-cardinality-form.json +1 -1
- package/corpus/dev/index.json +1 -1
- package/dist/application/index.d.ts +2 -0
- package/dist/application/index.js +1 -0
- package/dist/application/score.d.ts +48 -0
- package/dist/application/score.js +186 -0
- package/dist/cli/arguments.d.ts +7 -3
- package/dist/cli/arguments.js +73 -5
- package/dist/cli/exit-codes.d.ts +6 -2
- package/dist/cli/exit-codes.js +8 -9
- package/dist/cli/main.js +9 -0
- package/dist/cli/render.d.ts +1 -1
- package/dist/cli/render.js +6 -4
- package/dist/cli/run.d.ts +13 -3
- package/dist/cli/run.js +136 -16
- package/dist/core/canonical/canonicalize.d.ts +12 -0
- package/dist/core/canonical/canonicalize.js +18 -4
- package/dist/core/compile/bindings.d.ts +70 -0
- package/dist/core/compile/bindings.js +317 -0
- package/dist/core/compile/compile.d.ts +5 -0
- package/dist/core/compile/compile.js +9 -0
- package/dist/core/compile/expression-legality.d.ts +49 -0
- package/dist/core/compile/expression-legality.js +105 -31
- package/dist/core/compile/forbidden-inputs.d.ts +8 -1
- package/dist/core/compile/forbidden-inputs.js +16 -3
- package/dist/core/compile/interface-inventory.d.ts +23 -1
- package/dist/core/compile/interface-inventory.js +32 -3
- package/dist/core/compile/reachability.d.ts +18 -1
- package/dist/core/compile/reachability.js +46 -1
- package/dist/core/compile/sensitivity-witness.d.ts +1 -1
- package/dist/core/compile/sensitivity-witness.js +1 -1
- package/dist/core/emit/emit.d.ts +9 -0
- package/dist/core/emit/emit.js +214 -0
- package/dist/core/emit/private-artifact-digest.d.ts +11 -0
- package/dist/core/emit/private-artifact-digest.js +31 -0
- package/dist/core/evaluate/evidence-resolution.d.ts +11 -1
- package/dist/core/evaluate/evidence-resolution.js +4 -1
- package/dist/core/failure-codes.d.ts +2 -2
- package/dist/core/failure-codes.js +4 -2
- package/dist/core/ingest/conditions.d.ts +201 -0
- package/dist/core/ingest/conditions.js +48 -0
- package/dist/core/ingest/index.d.ts +13 -0
- package/dist/core/ingest/index.js +11 -0
- package/dist/core/ingest/ingest.d.ts +68 -0
- package/dist/core/ingest/ingest.js +310 -0
- package/dist/core/lineage/stage-table.d.ts +21 -4
- package/dist/core/lineage/stage-table.js +35 -3
- package/dist/core/preflight/witness-evidence.js +10 -0
- package/dist/core/schemas/artifact.d.ts +385 -14
- package/dist/core/schemas/constraint-ledger.js +40 -0
- package/dist/core/schemas/defect-signature.d.ts +273 -0
- package/dist/core/schemas/defect-signature.js +127 -0
- package/dist/core/schemas/eval-contract.d.ts +33 -0
- package/dist/core/schemas/eval-contract.js +14 -0
- package/dist/core/schemas/evidence-artifact.d.ts +99 -19
- package/dist/core/schemas/evidence-artifact.js +64 -8
- package/dist/core/schemas/plan.d.ts +112 -0
- package/dist/core/schemas/plan.js +67 -2
- package/dist/core/schemas/probe-qualification.d.ts +168 -0
- package/dist/core/schemas/probe-qualification.js +106 -0
- package/dist/core/schemas/probe.d.ts +312 -0
- package/dist/core/schemas/probe.js +6 -2
- package/dist/core/schemas/scoring-policy.d.ts +1 -0
- package/dist/core/schemas/scoring-policy.js +6 -1
- package/dist/core/schemas/sealed-evaluator-brief.d.ts +1 -0
- package/dist/core/schemas/sealed-evaluator-brief.js +4 -1
- package/dist/core/schemas/sealed-run-record.d.ts +30 -3
- package/dist/core/schemas/sealed-run-record.js +37 -5
- package/dist/core/score/binding-order.d.ts +31 -0
- package/dist/core/score/binding-order.js +88 -0
- package/dist/core/score/bindings.d.ts +93 -0
- package/dist/core/score/bindings.js +297 -0
- package/dist/core/score/ladder-table.d.ts +23 -0
- package/dist/core/score/ladder-table.js +109 -0
- package/dist/core/score/ladder.d.ts +229 -0
- package/dist/core/score/ladder.js +464 -0
- package/dist/core/score/mode-agreement.d.ts +30 -0
- package/dist/core/score/mode-agreement.js +16 -0
- package/dist/core/score/outcome-table.d.ts +17 -0
- package/dist/core/score/outcome-table.js +172 -0
- package/dist/core/score/outcome.d.ts +466 -0
- package/dist/core/score/outcome.js +562 -0
- package/dist/core/score/qualification.d.ts +83 -0
- package/dist/core/score/qualification.js +615 -0
- package/dist/core/score/quotation.d.ts +79 -0
- package/dist/core/score/quotation.js +201 -0
- package/dist/core/score/reduce-trials.d.ts +80 -0
- package/dist/core/score/reduce-trials.js +90 -0
- package/dist/core/score/score.d.ts +72 -0
- package/dist/core/score/score.js +581 -0
- package/dist/core/score/selection.d.ts +80 -0
- package/dist/core/score/selection.js +98 -0
- package/dist/core/score/strength.d.ts +49 -0
- package/dist/core/score/strength.js +193 -0
- package/dist/core/score/witness.d.ts +117 -0
- package/dist/core/score/witness.js +315 -0
- package/dist/core/seal/derived-reference.d.ts +1 -1
- package/dist/core/seal/derived-reference.js +236 -28
- package/dist/core/seal/seal.js +8 -3
- package/dist/core/stage-contracts.d.ts +62 -2
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/testing/index.d.ts +2 -0
- package/dist/testing/index.js +1 -0
- package/package.json +12 -3
- package/schemas/eval-contract.schema.json +100 -2
- package/schemas/evidence-artifact.schema.json +96 -9
- package/schemas/probe.schema.json +508 -3
- package/schemas/scoring-policy.schema.json +8 -1
- package/schemas/sealed-evaluator-brief.schema.json +11 -1
- package/schemas/sealed-run-record.schema.json +19 -2
|
@@ -22,6 +22,11 @@ export const ScoringPolicy = z
|
|
|
22
22
|
.min(0)
|
|
23
23
|
.max(1)
|
|
24
24
|
.describe('The same closed unit interval a finding\'s `confidence` uses. AD-21\'s "a finding whose confidence falls below the policy threshold" compares the two, and two different scales would make that comparison meaningless.'),
|
|
25
|
+
catchThreshold: z
|
|
26
|
+
.number()
|
|
27
|
+
.min(0)
|
|
28
|
+
.max(1)
|
|
29
|
+
.describe('AD-7\'s trial-set reducer: a probe counts as caught only when its caught-trial count is strictly greater than this fraction of its valid-trial count, so an exact tie never counts as caught. The published default artifact carries 0.5, the pre-registered "at least two catches in three valid repetitions" read at a valid count of three. No `.default()`, on the same reasoning `confidenceThreshold` gives. Required, not optional, which makes this the scoring policy\'s `schemaVersion` 1 -> 2 BREAKING bump under AD-11, whose rule is that "adding an optional field is a `schemaVersion` bump recorded in the field\'s own description; removing or retyping is breaking". With no default to fill it in, every version-1 policy document fails to parse.'),
|
|
25
30
|
minimumTrialCount: z
|
|
26
31
|
.int()
|
|
27
32
|
.min(1)
|
|
@@ -47,5 +52,5 @@ export const ScoringPolicy = z
|
|
|
47
52
|
})
|
|
48
53
|
.meta({
|
|
49
54
|
id: 'ScoringPolicy',
|
|
50
|
-
description: 'The scoring policy, with no prior art. It carries the severity floor, confidence threshold, minimum trial count, re-execution cap, remediation cap, and a regex match-step budget that AD-6, AD-12, AD-4, and AD-21 read, and its digest is one of the
|
|
55
|
+
description: 'The scoring policy, with no prior art. It carries the severity floor, confidence threshold, catch threshold, minimum trial count, re-execution cap, remediation cap, and a regex match-step budget that AD-6, AD-7, AD-12, AD-4, and AD-21 read, and its digest is one of the six named inputs to AD-11\'s scoring version. It is a published artifact rather than a set of constants so that "the default" has an identity a result can name by digest.',
|
|
51
56
|
});
|
|
@@ -45,6 +45,9 @@ export const SealedEvaluatorBrief = z
|
|
|
45
45
|
scopedResources: z
|
|
46
46
|
.array(ScopedResource)
|
|
47
47
|
.describe("AD-16's scoped resource references. Not nullable here, unlike on the contract: absent-versus-explicitly-empty is an AD-31 grading of a CALLER declaration, and this artifact is generated by `seal`, which always knows the answer. No scoped reference may resolve to a forbidden input; that resolution is `scoped-reference-resolves-forbidden` at compile time and is not a schema check."),
|
|
48
|
+
principals: z
|
|
49
|
+
.array(Identifier)
|
|
50
|
+
.describe("The names declared in the contract's `testData.principals`, sorted. Not nullable, on `scopedResources`'s reasoning above: `seal` always knows the answer, and empty is the legal spelling for a contract declaring none. Names only, no `kind`: `kind` is the author's own opaque vocabulary with no value space any AD declares, so shipping it would put uninterpretable text on the one artifact AD-16 keeps minimal. A name is an opaque label (AD-18), so it discloses no account identifier or credential, and it carries no step identifier (AD-16). Required, not optional, which makes this the sealed brief's `schemaVersion` 1 -> 2 BREAKING bump under AD-11, whose rule is that \"adding an optional field is a `schemaVersion` bump recorded in the field's own description; removing or retyping is breaking\". `[]` is the empty spelling and the key is mandatory, so no version-1 brief parses."),
|
|
48
51
|
budgets: Budgets.describe("The contract's declared ceilings, carried because AD-16 says the caller who executes needs the ceilings the isolation manifest reports against. The manifest's own `resourceCeilings` is a different shape and a different concern: this is a declaration, that is a record of what the harness enforced."),
|
|
49
52
|
safetyLimits: z.array(z.string()),
|
|
50
53
|
probeStepBound: z
|
|
@@ -55,5 +58,5 @@ export const SealedEvaluatorBrief = z
|
|
|
55
58
|
})
|
|
56
59
|
.meta({
|
|
57
60
|
id: 'SealedEvaluatorBrief',
|
|
58
|
-
description: "The sealed evaluator brief, with no prior art. AD-16 fixes what it carries: the contract's behaviours, the generated evaluator-facing directions of AD-3, permitted interfaces,
|
|
61
|
+
description: "The sealed evaluator brief, with no prior art. AD-16 fixes what it carries: the contract's behaviours, the generated evaluator-facing directions of AD-3, permitted interfaces, scoped resource references, and the declared test-data principal names, plus budgets and safety limits. It also fixes what the brief never carries: author commentary, the interaction plan, or the plan's step identifiers, and never a prescribed action sequence. Those exclusions are structural rather than conventional: this object is strict, so a brief carrying `interactionPlan`, `commentary`, or a `stepId` fails with `unrecognized_keys`, and each of the three has its own reject fixture rather than resting on strictness alone. It carries no identity field of its own, because every artifact that refers to it does so by digest.",
|
|
59
62
|
});
|
|
@@ -132,6 +132,7 @@ export declare const OracleDisposition: z.ZodObject<{
|
|
|
132
132
|
observationIds: z.ZodArray<z.ZodString>;
|
|
133
133
|
note: z.ZodNullable<z.ZodString>;
|
|
134
134
|
}, z.core.$strict>;
|
|
135
|
+
export type OracleDisposition = z.infer<typeof OracleDisposition>;
|
|
135
136
|
/**
|
|
136
137
|
* A flat map would break pointer addressing: AD-26 keys `call-inputs` by
|
|
137
138
|
* transport channel, so a pointer like
|
|
@@ -147,16 +148,19 @@ export declare const ObservedCallInputs: z.ZodObject<{
|
|
|
147
148
|
header: z.ZodNullable<z.ZodType<import("./primitives.ts").JsonObject, unknown, z.core.$ZodTypeInternals<import("./primitives.ts").JsonObject, unknown>>>;
|
|
148
149
|
body: z.ZodNullable<z.ZodType<import("./primitives.ts").JsonObject, unknown, z.core.$ZodTypeInternals<import("./primitives.ts").JsonObject, unknown>>>;
|
|
149
150
|
}, z.core.$strict>;
|
|
151
|
+
/** the constraint identifier the ledger carries for the check below. */
|
|
152
|
+
export declare const OBSERVATION_SEQUENCE_UNIQUE = "observation-sequence-unique";
|
|
150
153
|
/**
|
|
151
154
|
* One ingested observation, carrying AD-26's closed channel set so every
|
|
152
155
|
* pointer in the addressing grammar has something to resolve against.
|
|
153
156
|
*
|
|
154
|
-
*
|
|
155
|
-
*
|
|
156
|
-
*
|
|
157
|
+
* `sequence` closes owed item 2: ADR-006 forbids using array position as
|
|
158
|
+
* ordering. A required, per-record-unique `sequence` is the total order a
|
|
159
|
+
* selector reads.
|
|
157
160
|
*/
|
|
158
161
|
export declare const Observation: z.ZodObject<{
|
|
159
162
|
observationId: z.ZodString;
|
|
163
|
+
sequence: z.ZodInt;
|
|
160
164
|
operationId: z.ZodString;
|
|
161
165
|
provenance: z.ZodEnum<{
|
|
162
166
|
baseline: "baseline";
|
|
@@ -205,12 +209,34 @@ export declare const EvidenceDisclosure: z.ZodObject<{
|
|
|
205
209
|
truncationBound: z.ZodNullable<z.ZodInt>;
|
|
206
210
|
reportedIncomplete: z.ZodBoolean;
|
|
207
211
|
}, z.core.$strict>;
|
|
212
|
+
/** exported so the ladder names this shape without importing Zod. */
|
|
213
|
+
export type EvidenceDisclosure = z.infer<typeof EvidenceDisclosure>;
|
|
214
|
+
/**
|
|
215
|
+
* AD-21's two modes, closed. In `production` the subject is the system under
|
|
216
|
+
* test; in `contract-scoring` the subject is the contract, the probe is
|
|
217
|
+
* knowingly defective, and a `caught` outcome is the contract succeeding.
|
|
218
|
+
*
|
|
219
|
+
* Declared on the record rather than on the evidence artifact because this is
|
|
220
|
+
* where mode is now fixed: AD-21 requires mode "fixed before ingest", and an
|
|
221
|
+
* evidence artifact is `emit`'s output, four stages past the only place a
|
|
222
|
+
* caller can supply one.
|
|
223
|
+
*/
|
|
224
|
+
export declare const RUN_MODES: readonly ['production', 'contract-scoring'];
|
|
225
|
+
export type RunModeValue = (typeof RUN_MODES)[number];
|
|
226
|
+
export declare const RunMode: z.ZodEnum<{
|
|
227
|
+
"contract-scoring": "contract-scoring";
|
|
228
|
+
production: "production";
|
|
229
|
+
}>;
|
|
208
230
|
export declare const SealedRunRecord: z.ZodObject<{
|
|
209
231
|
schemaVersion: z.ZodInt;
|
|
210
232
|
parentDigest: z.ZodNullable<z.ZodString>;
|
|
211
233
|
revisionCount: z.ZodInt;
|
|
212
234
|
runId: z.ZodString;
|
|
213
235
|
conditionArm: z.ZodString;
|
|
236
|
+
mode: z.ZodEnum<{
|
|
237
|
+
"contract-scoring": "contract-scoring";
|
|
238
|
+
production: "production";
|
|
239
|
+
}>;
|
|
214
240
|
trialIndex: z.ZodInt;
|
|
215
241
|
contractDigest: z.ZodString;
|
|
216
242
|
sealedBriefDigest: z.ZodString;
|
|
@@ -320,6 +346,7 @@ export declare const SealedRunRecord: z.ZodObject<{
|
|
|
320
346
|
}, z.core.$strict>], "findingType">>;
|
|
321
347
|
observations: z.ZodArray<z.ZodObject<{
|
|
322
348
|
observationId: z.ZodString;
|
|
349
|
+
sequence: z.ZodInt;
|
|
323
350
|
operationId: z.ZodString;
|
|
324
351
|
provenance: z.ZodEnum<{
|
|
325
352
|
baseline: "baseline";
|
|
@@ -109,16 +109,22 @@ export const ObservedCallInputs = z.strictObject({
|
|
|
109
109
|
header: JsonObjectValue.nullable(),
|
|
110
110
|
body: JsonObjectValue.nullable(),
|
|
111
111
|
});
|
|
112
|
+
/** the constraint identifier the ledger carries for the check below. */
|
|
113
|
+
export const OBSERVATION_SEQUENCE_UNIQUE = 'observation-sequence-unique';
|
|
112
114
|
/**
|
|
113
115
|
* One ingested observation, carrying AD-26's closed channel set so every
|
|
114
116
|
* pointer in the addressing grammar has something to resolve against.
|
|
115
117
|
*
|
|
116
|
-
*
|
|
117
|
-
*
|
|
118
|
-
*
|
|
118
|
+
* `sequence` closes owed item 2: ADR-006 forbids using array position as
|
|
119
|
+
* ordering. A required, per-record-unique `sequence` is the total order a
|
|
120
|
+
* selector reads.
|
|
119
121
|
*/
|
|
120
122
|
export const Observation = z.strictObject({
|
|
121
123
|
observationId: Identifier,
|
|
124
|
+
sequence: z
|
|
125
|
+
.int()
|
|
126
|
+
.positive()
|
|
127
|
+
.describe('The total order ADR-006 forbids reading off array position (owed item 2). A positive integer, unique across every observation in the same record: uniqueness is what a sorted-by-`sequence` read needs to be strictly increasing, so no separate monotonicity check is required. Neither starting at 1 nor contiguous is required across a record: only positivity and per-record uniqueness are enforced, so a record whose sequences are e.g. [5, 12, 40] is equally valid. Required rather than optional, which makes this a BREAKING `schemaVersion` bump under AD-11: adding an optional field is additive, and this field is not optional. A record with an absent or duplicated `sequence` fails to parse.'),
|
|
122
128
|
operationId: Identifier.describe('The operation this observation exercised. `Operation.operationId` is scoped to a `PermittedInterface`, so two interfaces may declare the same one; that collision is a cross-artifact rule with no AD-5 code, since `duplicate-operation-signature` covers method plus path template only, and it is left to ingest.'),
|
|
123
129
|
provenance: z
|
|
124
130
|
.enum(['baseline', 'evaluator-chosen'])
|
|
@@ -178,6 +184,18 @@ export const EvidenceDisclosure = z.strictObject({
|
|
|
178
184
|
.boolean()
|
|
179
185
|
.describe('AD-17: "a case that cannot be bounded without discarding disconfirming material is reported incomplete". This is the operand AD-21\'s incomplete condition reads. The remaining two conditions read no field here: unavailable is an `ArtifactReference` that does not resolve through the corpus port, and internally inconsistent is AD-32\'s cross-artifact agreement check. AD-17\'s "must retain evidence contradicting the leading verdict" is decidable by none of them and is recorded as unenforced in v0.'),
|
|
180
186
|
});
|
|
187
|
+
/**
|
|
188
|
+
* AD-21's two modes, closed. In `production` the subject is the system under
|
|
189
|
+
* test; in `contract-scoring` the subject is the contract, the probe is
|
|
190
|
+
* knowingly defective, and a `caught` outcome is the contract succeeding.
|
|
191
|
+
*
|
|
192
|
+
* Declared on the record rather than on the evidence artifact because this is
|
|
193
|
+
* where mode is now fixed: AD-21 requires mode "fixed before ingest", and an
|
|
194
|
+
* evidence artifact is `emit`'s output, four stages past the only place a
|
|
195
|
+
* caller can supply one.
|
|
196
|
+
*/
|
|
197
|
+
export const RUN_MODES = ['production', 'contract-scoring'];
|
|
198
|
+
export const RunMode = z.enum(RUN_MODES);
|
|
181
199
|
export const SealedRunRecord = z
|
|
182
200
|
.strictObject({
|
|
183
201
|
...lineageFields,
|
|
@@ -189,6 +207,7 @@ export const SealedRunRecord = z
|
|
|
189
207
|
.string()
|
|
190
208
|
.min(1)
|
|
191
209
|
.describe("An opaque caller label with no product semantics, per AD-24. The prior art's five-member enum does not survive, and its own extension history is the reason: an enum a local amendment had to widen once for `self-review` will be widened again."),
|
|
210
|
+
mode: RunMode.describe('AD-21\'s run mode, supplied by the caller on the record and never derived, recomputed, or defaulted afterwards. AD-21 requires mode to be "fixed before ingest", and owed item 4 records what its absence costs: the same sealed run could be relabelled after ingest and scored under the same scoring version. Required rather than optional, which makes this a BREAKING `schemaVersion` bump under AD-11, whose rule is that "adding an optional field is a `schemaVersion` bump recorded in the field\'s own description; removing or retyping is breaking". A version-1 record carries no mode, and no default may repair one into a version-2 record, because a defaulted mode is the relabelling this field exists to stop. A record presenting no mode fails to parse, which AD-28 makes a `schema-parse-failure` fault rather than an AD-21 verdict or an AD-5 code, the same routing `evaluatorRecommendation` already records for an unrecognised value. This field is where mode is read from; the evidence artifact restates it and is never the source.'),
|
|
192
211
|
trialIndex: z
|
|
193
212
|
.int()
|
|
194
213
|
.min(1)
|
|
@@ -199,7 +218,20 @@ export const SealedRunRecord = z
|
|
|
199
218
|
evaluatorRecommendation: EvaluatorRecommendation,
|
|
200
219
|
oracleDispositions: z.array(OracleDisposition),
|
|
201
220
|
findings: z.array(Finding),
|
|
202
|
-
observations: z
|
|
221
|
+
observations: z
|
|
222
|
+
.array(Observation)
|
|
223
|
+
.refine((observations) => {
|
|
224
|
+
const seen = new Set();
|
|
225
|
+
for (const observation of observations) {
|
|
226
|
+
if (seen.has(observation.sequence))
|
|
227
|
+
return false;
|
|
228
|
+
seen.add(observation.sequence);
|
|
229
|
+
}
|
|
230
|
+
return true;
|
|
231
|
+
}, {
|
|
232
|
+
error: "every observation's `sequence` is unique within the record (ADR-006, owed item 2); a duplicate leaves the total order the fix exists to supply ambiguous",
|
|
233
|
+
})
|
|
234
|
+
.describe('Per-observation `sequence` uniqueness is enforced here. The published JSON Schema dialect has no keyword for uniqueness of a nested field across array items, so this constraint is Zod-only: a non-Zod consumer must reimplement it, exactly as with any other cross-item invariant this dialect cannot state.'),
|
|
203
235
|
judgeResults: z
|
|
204
236
|
.array(JudgeResult)
|
|
205
237
|
.describe('Empty is legal: a contract with no rubric produces no judge call.'),
|
|
@@ -211,5 +243,5 @@ export const SealedRunRecord = z
|
|
|
211
243
|
})
|
|
212
244
|
.meta({
|
|
213
245
|
id: 'SealedRunRecord',
|
|
214
|
-
description: "One sealed evaluator trial, as the caller presents it. Succeeds the prior-art `h0-run-result` schema per AD-24, keeping its run identifier, condition arm, findings, action-log reference, resource use, invalidation reason, evaluator recommendation as a closed enum, and per-finding confidence on a declared scale. Divergences: `condition` is demoted to the opaque `conditionArm`, `verdict` becomes `evaluatorRecommendation` without `NOT_APPLICABLE`, money is a decimal string, and `taskId`, `note`, and per-finding `actionIds` do not survive: the contract is pinned by `contractDigest`, an unstructured orchestrator annotation is the free-prose channel the Conventions close everywhere else, and two citation vocabularies on one finding is the ambiguity ADR-009 removed.
|
|
246
|
+
description: "One sealed evaluator trial, as the caller presents it. Succeeds the prior-art `h0-run-result` schema per AD-24, keeping its run identifier, condition arm, findings, action-log reference, resource use, invalidation reason, evaluator recommendation as a closed enum, and per-finding confidence on a declared scale. Divergences: `condition` is demoted to the opaque `conditionArm`, `verdict` becomes `evaluatorRecommendation` without `NOT_APPLICABLE`, money is a decimal string, and `taskId`, `note`, and per-finding `actionIds` do not survive: the contract is pinned by `contractDigest`, an unstructured orchestrator annotation is the free-prose channel the Conventions close everywhere else, and two citation vocabularies on one finding is the ambiguity ADR-009 removed. The run MODE landed here as a required field under a BREAKING `schemaVersion` bump, which is where AD-21's \"fixed before ingest\" puts it; owed item 4 is now closed: mode enters AD-11's identity inputs as `ScoringVersionInputs`'s sixth field, and `core/score/ladder.ts` carries `ProductionAssessment`/`ContractAssessment` as the two assessment input types with their own total ladders. Observation ORDERING landed here too, under its own BREAKING `schemaVersion` bump: `sequence` is required and unique per record, closing owed item 2's ADR-006 gap, since array position was never a legal ordering.",
|
|
215
247
|
});
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import type { InteractionStep } from '../schemas/plan.ts';
|
|
2
|
+
export type BindingOrder = {
|
|
3
|
+
/** Ascending tiers; within each, steps in declaration order. Every declared id appears exactly once. */
|
|
4
|
+
readonly tiers: readonly (readonly string[])[];
|
|
5
|
+
/**
|
|
6
|
+
* Ids Kahn's algorithm left unplaced, in declaration order: the members of an
|
|
7
|
+
* ordering cycle and every step downstream of one, since a step whose
|
|
8
|
+
* dependency never gets a tier never gets one either.
|
|
9
|
+
*/
|
|
10
|
+
readonly cyclic: readonly string[];
|
|
11
|
+
};
|
|
12
|
+
/**
|
|
13
|
+
* Cyclic steps come back as data instead of an exception, matching
|
|
14
|
+
* `selectObservations`'s own policy of reporting ambiguity rather than deciding
|
|
15
|
+
* it. A compiled contract whose step ids are unique never reaches that list:
|
|
16
|
+
* `binding-cycle` rejects every cycle carrying a capture edge and
|
|
17
|
+
* `nested-temporal-clause` every cycle made of `after` edges alone, so the
|
|
18
|
+
* union graph is acyclic by the time any record is scored.
|
|
19
|
+
*
|
|
20
|
+
* The uniqueness clause is load-bearing. `checkNestedTemporalClause` resolves
|
|
21
|
+
* `after` through a plan index built with `duplicateIds: 'unresolved'`, which
|
|
22
|
+
* deletes a duplicated id and reads any clause naming it as dangling, while
|
|
23
|
+
* this function and `checkBindingCycle` both test against the raw declared-id
|
|
24
|
+
* set, where it is present. So a plan declaring one id twice can carry an
|
|
25
|
+
* `after` cycle those two disagree about, and its steps land here. Nothing
|
|
26
|
+
* downstream sees it: `seal` builds its own index with the default throwing
|
|
27
|
+
* option and rejects a duplicate id outright. Changing how
|
|
28
|
+
* `checkNestedTemporalClause` reads a duplicate is deliberate shipped
|
|
29
|
+
* behaviour and belongs to whoever owns that check.
|
|
30
|
+
*/
|
|
31
|
+
export declare function bindingOrder(interactionPlan: readonly InteractionStep[]): BindingOrder;
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The order a plan's steps must be resolved in when one step's selection
|
|
3
|
+
* depends on another's. Kahn tiers over the union of the capture edges and
|
|
4
|
+
* AD-39's `after` edges, the same graph `checkBindingCycle` builds: every step
|
|
5
|
+
* in tier n depends only on steps in tiers below n, so a caller walking the
|
|
6
|
+
* tiers in order always has what it needs resolved by the time it reads it.
|
|
7
|
+
*
|
|
8
|
+
* The `after` edge is in the graph because `selectWithBindings` resolves a
|
|
9
|
+
* temporal anchor through the same binding filter as everything else, so a step
|
|
10
|
+
* can read an anchor whose own capture sites must already be written. Tiering
|
|
11
|
+
* captures alone made the answer depend on declaration order within a tier: two
|
|
12
|
+
* same-tier steps swapped flipped a verdict between `none` and one match.
|
|
13
|
+
*
|
|
14
|
+
* `{ literal }`, `{ matcher }`, and `{ principal }` bindings declare no
|
|
15
|
+
* dependency, so a step carrying only those and no clause is tier zero. So is a
|
|
16
|
+
* step whose captured pointer or `after` clause names an id the plan does not
|
|
17
|
+
* declare: a dangling capture is `unreachable-check-evidence`'s at compile time,
|
|
18
|
+
* AD-39 makes a dangling `after` permissive, and nothing here should invent an
|
|
19
|
+
* ordering for either.
|
|
20
|
+
*
|
|
21
|
+
* Within a tier the order is the order the steps are declared in
|
|
22
|
+
* `interactionPlan`, which is what "sequence order within a tier" can mean at
|
|
23
|
+
* compile time, where no observation `sequence` exists yet. ADR-006 bans
|
|
24
|
+
* reading order off array position in a run record, where the array is an
|
|
25
|
+
* ingest artifact nobody authored; a contract's `interactionPlan` is the
|
|
26
|
+
* author's own declaration, and permuting it yields a different contract with a
|
|
27
|
+
* different digest, so there is no permutation invariance to preserve.
|
|
28
|
+
*/
|
|
29
|
+
import { capturedBindings } from '../compile/bindings.js';
|
|
30
|
+
/**
|
|
31
|
+
* Cyclic steps come back as data instead of an exception, matching
|
|
32
|
+
* `selectObservations`'s own policy of reporting ambiguity rather than deciding
|
|
33
|
+
* it. A compiled contract whose step ids are unique never reaches that list:
|
|
34
|
+
* `binding-cycle` rejects every cycle carrying a capture edge and
|
|
35
|
+
* `nested-temporal-clause` every cycle made of `after` edges alone, so the
|
|
36
|
+
* union graph is acyclic by the time any record is scored.
|
|
37
|
+
*
|
|
38
|
+
* The uniqueness clause is load-bearing. `checkNestedTemporalClause` resolves
|
|
39
|
+
* `after` through a plan index built with `duplicateIds: 'unresolved'`, which
|
|
40
|
+
* deletes a duplicated id and reads any clause naming it as dangling, while
|
|
41
|
+
* this function and `checkBindingCycle` both test against the raw declared-id
|
|
42
|
+
* set, where it is present. So a plan declaring one id twice can carry an
|
|
43
|
+
* `after` cycle those two disagree about, and its steps land here. Nothing
|
|
44
|
+
* downstream sees it: `seal` builds its own index with the default throwing
|
|
45
|
+
* option and rejects a duplicate id outright. Changing how
|
|
46
|
+
* `checkNestedTemporalClause` reads a duplicate is deliberate shipped
|
|
47
|
+
* behaviour and belongs to whoever owns that check.
|
|
48
|
+
*/
|
|
49
|
+
export function bindingOrder(interactionPlan) {
|
|
50
|
+
// A duplicated id collapses to one node: the plan declares one name, and
|
|
51
|
+
// which of two same-named steps a capture meant is undecidable here.
|
|
52
|
+
const declared = [];
|
|
53
|
+
const dependencies = new Map();
|
|
54
|
+
for (const step of interactionPlan) {
|
|
55
|
+
if (dependencies.has(step.stepId))
|
|
56
|
+
continue;
|
|
57
|
+
declared.push(step.stepId);
|
|
58
|
+
dependencies.set(step.stepId, new Set());
|
|
59
|
+
}
|
|
60
|
+
for (const step of interactionPlan) {
|
|
61
|
+
const own = dependencies.get(step.stepId);
|
|
62
|
+
if (own === undefined)
|
|
63
|
+
continue;
|
|
64
|
+
for (const capture of capturedBindings(step)) {
|
|
65
|
+
const { stepId } = capture.target;
|
|
66
|
+
// A self-reference is a cycle of one, so it stays in the graph and lands
|
|
67
|
+
// in `cyclic` rather than being filtered into tier zero.
|
|
68
|
+
if (dependencies.has(stepId))
|
|
69
|
+
own.add(stepId);
|
|
70
|
+
}
|
|
71
|
+
if (step.after !== null && dependencies.has(step.after)) {
|
|
72
|
+
own.add(step.after);
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
const tiers = [];
|
|
76
|
+
const placed = new Set();
|
|
77
|
+
while (placed.size < declared.length) {
|
|
78
|
+
const tier = declared.filter((stepId) => !placed.has(stepId) &&
|
|
79
|
+
[...(dependencies.get(stepId) ?? [])].every((id) => placed.has(id)));
|
|
80
|
+
// Every remaining step depends on another remaining step, which is a cycle.
|
|
81
|
+
if (tier.length === 0)
|
|
82
|
+
break;
|
|
83
|
+
tiers.push(tier);
|
|
84
|
+
for (const stepId of tier)
|
|
85
|
+
placed.add(stepId);
|
|
86
|
+
}
|
|
87
|
+
return { tiers, cyclic: declared.filter((stepId) => !placed.has(stepId)) };
|
|
88
|
+
}
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
import { type ResolvedValue } from '../evaluate/resolved-value.ts';
|
|
2
|
+
import type { InteractionStep } from '../schemas/plan.ts';
|
|
3
|
+
import { type TransportChannelName } from '../schemas/pointer.ts';
|
|
4
|
+
import type { JsonValue } from '../schemas/primitives.ts';
|
|
5
|
+
import type { Observation } from '../schemas/sealed-run-record.ts';
|
|
6
|
+
import { type PlanIndex } from '../seal/plan-index.ts';
|
|
7
|
+
import { type StepSelection } from './selection.ts';
|
|
8
|
+
/**
|
|
9
|
+
* One captured pointer's resolution. `ResolvedValue` alone cannot carry this:
|
|
10
|
+
* it is `JsonValue | ABSENT` with nowhere to put the observation the value came
|
|
11
|
+
* from, its `sequence`, or the referenced step's named ambiguity. `sequence` is
|
|
12
|
+
* what gives "earlier" a runtime meaning, since a capture graph that is acyclic
|
|
13
|
+
* still says nothing about which observation came first.
|
|
14
|
+
*/
|
|
15
|
+
export type CapturedResolution = {
|
|
16
|
+
readonly status: 'resolved';
|
|
17
|
+
readonly value: ResolvedValue;
|
|
18
|
+
readonly observationId: Observation['observationId'];
|
|
19
|
+
readonly sequence: number;
|
|
20
|
+
} | {
|
|
21
|
+
readonly status: 'absent';
|
|
22
|
+
} | {
|
|
23
|
+
readonly status: 'ambiguous';
|
|
24
|
+
readonly matchedObservationIds: readonly Observation['observationId'][];
|
|
25
|
+
};
|
|
26
|
+
/**
|
|
27
|
+
* The key one captured binding occupies in the resolution map. Keyed by
|
|
28
|
+
* binding site rather than by pointer: one step can carry several captured
|
|
29
|
+
* bindings, and two of them may name the same pointer in different channels.
|
|
30
|
+
* JSON-encoded rather than delimiter-joined, since a parameter key is arbitrary
|
|
31
|
+
* caller-supplied text and could contain any separator.
|
|
32
|
+
*/
|
|
33
|
+
export declare function bindingSiteKey(stepId: string, transportChannel: TransportChannelName, key: string): string;
|
|
34
|
+
/**
|
|
35
|
+
* Resolves one captured pointer against the record.
|
|
36
|
+
*
|
|
37
|
+
* The referenced step is selected through `selectWithBindings`, so its own
|
|
38
|
+
* selection predicate applies before a value is read off it. Selecting it with
|
|
39
|
+
* `selectObservations` alone would match on `operationId` and leave a step that
|
|
40
|
+
* only its own bindings separate reporting the named ambiguity, which would
|
|
41
|
+
* make a capture from either half of a literal-bound collision pair resolve
|
|
42
|
+
* nothing at all. `resolved` carries the referenced step's own captured values,
|
|
43
|
+
* already filled by the tier below it, which is what makes the tiering
|
|
44
|
+
* load-bearing.
|
|
45
|
+
*
|
|
46
|
+
* The three dispositions already settled hold unchanged over the filtered
|
|
47
|
+
* result: one match binds, several under a declared `any` binds the lowest
|
|
48
|
+
* `sequence`, and several under `exactly-one`/`at-most-one` is the named
|
|
49
|
+
* ambiguity, returned as data with no value resolved.
|
|
50
|
+
*
|
|
51
|
+
* A pointer whose step the plan does not declare, or whose referenced step
|
|
52
|
+
* matched nothing, resolves `absent`.
|
|
53
|
+
*
|
|
54
|
+
* `resolved` as a status means an observation was selected, which is a
|
|
55
|
+
* different fact from the pointer finding a value in it: a tail that walks off
|
|
56
|
+
* the observed body comes back `resolved` carrying `ABSENT`, with the source
|
|
57
|
+
* observation and its `sequence` intact. That is why `CapturedResolution.value`
|
|
58
|
+
* is `ResolvedValue`. AD-26 makes absent an observation rather than an error,
|
|
59
|
+
* and `deepEquals` is false against it, so the referencing step still selects
|
|
60
|
+
* `none`; keeping the source observation means the ordering floor stays
|
|
61
|
+
* computable either way.
|
|
62
|
+
*/
|
|
63
|
+
export declare function resolveCapturedValue(pointer: string, index: PlanIndex, observations: readonly Observation[], resolved: ReadonlyMap<string, CapturedResolution>): CapturedResolution;
|
|
64
|
+
/**
|
|
65
|
+
* Every captured binding in the plan, resolved in `bindingOrder`'s tiers and
|
|
66
|
+
* keyed by binding site. Walking the tiers in order is what lets a step's
|
|
67
|
+
* captured values be resolved against a referenced step whose own captures are
|
|
68
|
+
* already in the map. Steps `bindingOrder` reports cyclic are left out, so
|
|
69
|
+
* their bindings resolve as unlisted and filter every candidate away.
|
|
70
|
+
*/
|
|
71
|
+
export declare function resolveCapturedBindings(interactionPlan: readonly InteractionStep[], index: PlanIndex, observations: readonly Observation[]): ReadonlyMap<string, CapturedResolution>;
|
|
72
|
+
export declare function deepEquals(a: ResolvedValue, b: ResolvedValue): boolean;
|
|
73
|
+
export declare function jsonTypeOf(value: JsonValue): string;
|
|
74
|
+
/**
|
|
75
|
+
* `selectObservations`, then the step's own selection predicate as a filter
|
|
76
|
+
* over the matches, then the cardinality verdict over what survived.
|
|
77
|
+
*
|
|
78
|
+
* The order is temporal clause, then capture ordering, then the binding
|
|
79
|
+
* filters. Ordering is enforced at score time as well as at compile time: the
|
|
80
|
+
* persistence read-back this exists for is exactly a claim about order, and a
|
|
81
|
+
* record whose `GET` sits at `sequence` 2 and whose `POST` sits at `sequence` 9
|
|
82
|
+
* would otherwise satisfy the binding and pass an oracle proving the opposite.
|
|
83
|
+
*
|
|
84
|
+
* `resolved` is the map `resolveCapturedBindings` fills from `bindingOrder`'s
|
|
85
|
+
* tiers, keyed by binding site. An unlisted or unresolved site filters every
|
|
86
|
+
* candidate out, so a caller that skipped a tier gets `none` rather than a
|
|
87
|
+
* silently wrong match.
|
|
88
|
+
*
|
|
89
|
+
* A filter over zero bindings and no clause separates nothing, so two steps
|
|
90
|
+
* sharing an operation and declaring neither both still return `several`: the
|
|
91
|
+
* declared structure does not distinguish them.
|
|
92
|
+
*/
|
|
93
|
+
export declare function selectWithBindings(step: InteractionStep, observations: readonly Observation[], index: PlanIndex, resolved: ReadonlyMap<string, CapturedResolution>): StepSelection;
|