eval-quality 2.0.0 → 3.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -1
- package/dist/application/index.d.ts +4 -0
- package/dist/application/index.js +2 -0
- package/dist/core/compile/compile.js +6 -1
- package/dist/core/compile/schema-version.d.ts +15 -2
- package/dist/core/compile/schema-version.js +11 -3
- package/dist/core/emit/emit.js +4 -2
- package/dist/core/preflight/plan.js +15 -0
- package/dist/core/preflight/reduce.d.ts +1 -1
- package/dist/core/preflight/reduce.js +5 -2
- package/dist/core/schemas/evaluator-configuration.d.ts +9 -0
- package/dist/core/schemas/evaluator-configuration.js +9 -0
- package/dist/core/schemas/evidence-artifact.d.ts +9 -0
- package/dist/core/schemas/evidence-artifact.js +9 -0
- package/dist/core/schemas/isolation-manifest.d.ts +18 -0
- package/dist/core/schemas/isolation-manifest.js +18 -0
- package/dist/core/schemas/preflight-verdict.d.ts +9 -0
- package/dist/core/schemas/preflight-verdict.js +9 -0
- package/dist/core/schemas/private-artifact-manifest.d.ts +10 -0
- package/dist/core/schemas/private-artifact-manifest.js +10 -0
- package/dist/core/schemas/probe.d.ts +41 -0
- package/dist/core/schemas/probe.js +43 -0
- package/dist/core/schemas/scoring-policy.d.ts +11 -0
- package/dist/core/schemas/scoring-policy.js +11 -0
- package/dist/core/schemas/sealed-evaluator-brief.d.ts +12 -0
- package/dist/core/schemas/sealed-evaluator-brief.js +12 -0
- package/dist/core/schemas/sealed-run-record.d.ts +11 -0
- package/dist/core/schemas/sealed-run-record.js +11 -0
- package/dist/core/score/score.d.ts +1 -1
- package/dist/core/score/score.js +23 -0
- package/dist/core/seal/seal.js +4 -5
- package/dist/gates/audit-lockfile-age.mjs +295 -0
- package/dist/gates/check-dependency-direction.js +303 -0
- package/dist/gates/check-licenses.mjs +305 -0
- package/dist/gates/dependency-direction.js +555 -0
- package/dist/gates/discover-source-files.js +44 -0
- package/dist/gates/gate-config.js +251 -0
- package/dist/gates/gates-cli.js +410 -0
- package/dist/gates/lineage-ownership.js +364 -0
- package/dist/gates/package-boundary.js +388 -0
- package/dist/gates/token-scan.js +203 -0
- package/dist/index.d.ts +11 -1
- package/dist/index.js +20 -1
- package/dist/testing/probe-conformance.d.ts +23 -18
- package/package.json +20 -8
package/README.md
CHANGED
|
@@ -179,7 +179,9 @@ import spec from 'eval-quality/schemas/eval-contract.schema.json' with { type: '
|
|
|
179
179
|
|
|
180
180
|
The import attribute is required: ESM on Node 22 and 24 both throw `ERR_IMPORT_ATTRIBUTE_MISSING` without it. The development corpus ships the same way, at `eval-quality/corpus/dev/`, so you can read twenty-four real contracts and one compiled-and-sealed pair without cloning this repository.
|
|
181
181
|
|
|
182
|
-
|
|
182
|
+
**The repository gates** are a second binary, `eval-quality-gates`, which holds your own trees to rules you declare in one JSON file at your repository root. `lockfile-age` audits every entry of every lockfile you name against its real publication timestamp on the npm registry. `licences` holds every locked entry's licence expression against an allowlist of identifiers you declare. The file carries only the gates you have adopted, and a gate you invoke with no section for it refuses by name with no fallback to this package's own values. [Run the gates on your repository](https://bmad-code-org.github.io/bmad-eval-quality/how-to/run-the-gates-on-your-repository/) is the page for it.
|
|
183
|
+
|
|
184
|
+
The published surface is stable: a breaking change to a command, an export, or a schema is a major version bump. `compile` refuses a contract whose `schemaVersion` differs from the one this build reads, and `preflight` and `score` refuse a probe the same way, so check the stamp on anything you did not author against this version. The library exports both numbers, `EVAL_CONTRACT_SCHEMA_VERSION` and `PROBE_SCHEMA_VERSION`, so the version to check against comes from the package. `CHANGELOG.md` records what each release breaks.
|
|
183
185
|
|
|
184
186
|
## Relationship with BMad and TEA
|
|
185
187
|
|
|
@@ -238,6 +240,7 @@ Several files are generated from the code and guarded byte for byte, so a hand e
|
|
|
238
240
|
| `docs/ad31-coverage-predicates.generated.md`, the coverage predicates | `npm run generate:ad31-table` | `npm run check:ad31-table` |
|
|
239
241
|
| `docs/ad33-outcome-decision.generated.md`, the outcome decision procedure | `npm run generate:ad33-table` | `npm run check:ad33-table` |
|
|
240
242
|
| the three committed worked chains | `npm run generate:worked-example` | `npm run check:worked-example` |
|
|
243
|
+
| `VERSION` in `src/index.ts`, from `package.json` | `npm run generate:version` | `npm run check:version` |
|
|
241
244
|
| `_bmad-output/shareable/`, this README, CONTRIBUTING, and the planning artifacts as standalone HTML | `npm run build:shareable` | `npm run check:shareable` |
|
|
242
245
|
|
|
243
246
|
Every artifact the library hands back is deep-frozen. A revision is a new artifact carrying its parent's digest and a revision count one greater, and `npm run check:lineage` fails the build when a lineage field is written outside the modules that own it. `npm run check:boundary` fails it when anything the tarball carries references the planning system that produced it.
|
|
@@ -11,12 +11,16 @@ export { FAILURE_CODES, StructuralFailure } from '../core/failure-codes.ts';
|
|
|
11
11
|
export type { LineageChainReport, LineageFinding, } from '../core/lineage/chain.ts';
|
|
12
12
|
export { validateLineageChain } from '../core/lineage/chain.ts';
|
|
13
13
|
export { INTERCHANGE_ARTIFACT_KEYS } from '../core/schemas/artifact.ts';
|
|
14
|
+
export type { Severity } from '../core/schemas/eval-contract.ts';
|
|
15
|
+
export { SEVERITY_LEVELS } from '../core/schemas/eval-contract.ts';
|
|
14
16
|
export type { RuntimeFaultCode } from '../core/schemas/faults.ts';
|
|
15
17
|
export { RUNTIME_FAULT_CODES, RuntimeFault } from '../core/schemas/faults.ts';
|
|
16
18
|
export type { EvaluatorRecommendation, Verdict, } from '../core/schemas/verdict.ts';
|
|
17
19
|
export { EVALUATOR_RECOMMENDATIONS, VERDICTS, } from '../core/schemas/verdict.ts';
|
|
18
20
|
export type { QualificationFailure, QualificationFailureCode, QualificationResult, } from '../core/score/qualification.ts';
|
|
19
21
|
export { QUALIFICATION_FAILURES } from '../core/score/qualification.ts';
|
|
22
|
+
export type { ComparableResult, DominanceRelationValue, } from '../core/score/strength.ts';
|
|
23
|
+
export { compareDominance, DOMINANCE_RELATIONS, } from '../core/score/strength.ts';
|
|
20
24
|
export { compile } from './compile.ts';
|
|
21
25
|
export type { Diagnostic, DiagnosticSink } from './diagnostics.ts';
|
|
22
26
|
export type { PreflightFromObservationsOptions, RunPreflightOptions, } from './preflight.ts';
|
|
@@ -9,9 +9,11 @@ export { digestArtifact, digestBytes, digestComposite, } from '../core/canonical
|
|
|
9
9
|
export { FAILURE_CODES, StructuralFailure } from '../core/failure-codes.js';
|
|
10
10
|
export { validateLineageChain } from '../core/lineage/chain.js';
|
|
11
11
|
export { INTERCHANGE_ARTIFACT_KEYS } from '../core/schemas/artifact.js';
|
|
12
|
+
export { SEVERITY_LEVELS } from '../core/schemas/eval-contract.js';
|
|
12
13
|
export { RUNTIME_FAULT_CODES, RuntimeFault } from '../core/schemas/faults.js';
|
|
13
14
|
export { EVALUATOR_RECOMMENDATIONS, VERDICTS, } from '../core/schemas/verdict.js';
|
|
14
15
|
export { QUALIFICATION_FAILURES } from '../core/score/qualification.js';
|
|
16
|
+
export { compareDominance, DOMINANCE_RELATIONS, } from '../core/score/strength.js';
|
|
15
17
|
export { compile } from './compile.js';
|
|
16
18
|
export { preflightFromObservations, runPreflight } from './preflight.js';
|
|
17
19
|
export { runScore } from './score.js';
|
|
@@ -55,7 +55,12 @@ export function compile(contract, options) {
|
|
|
55
55
|
// First, and before any check reads a declaration. AD-11 makes an unequal
|
|
56
56
|
// stamp a rejection rather than a degraded read, and every check below is
|
|
57
57
|
// written against this version's field shapes.
|
|
58
|
-
checkSchemaVersion(
|
|
58
|
+
checkSchemaVersion({
|
|
59
|
+
stamped: contract.schemaVersion,
|
|
60
|
+
accepted: EVAL_CONTRACT_SCHEMA_VERSION,
|
|
61
|
+
artifactPath: 'EvalContract.schemaVersion',
|
|
62
|
+
consequence: 'since its stale version would travel into the scoring version',
|
|
63
|
+
});
|
|
59
64
|
checkRequirementLinkage(contract);
|
|
60
65
|
checkObservableSuccessCriterion(contract);
|
|
61
66
|
// Ahead of reachability, because a pointer naming an artifact nothing
|
|
@@ -1,2 +1,15 @@
|
|
|
1
|
-
/**
|
|
2
|
-
|
|
1
|
+
/**
|
|
2
|
+
* `schema-version-mismatch`: the stamp is not the version this build reads.
|
|
3
|
+
*
|
|
4
|
+
* An options object rather than four positional arguments. `artifactPath` and
|
|
5
|
+
* `consequence` are both free-form strings, and transposed they typecheck: the
|
|
6
|
+
* fault's `artifactPath`, which callers read programmatically, would then hold
|
|
7
|
+
* a sentence.
|
|
8
|
+
*/
|
|
9
|
+
export declare function checkSchemaVersion(options: {
|
|
10
|
+
readonly stamped: number;
|
|
11
|
+
readonly accepted: number;
|
|
12
|
+
readonly artifactPath: string;
|
|
13
|
+
/** Why an unequal stamp is a rejection for this artifact, as a clause. */
|
|
14
|
+
readonly consequence: string;
|
|
15
|
+
}): void;
|
|
@@ -17,9 +17,17 @@
|
|
|
17
17
|
* happened to notice first.
|
|
18
18
|
*/
|
|
19
19
|
import { RuntimeFault } from '../schemas/faults.js';
|
|
20
|
-
/**
|
|
21
|
-
|
|
20
|
+
/**
|
|
21
|
+
* `schema-version-mismatch`: the stamp is not the version this build reads.
|
|
22
|
+
*
|
|
23
|
+
* An options object rather than four positional arguments. `artifactPath` and
|
|
24
|
+
* `consequence` are both free-form strings, and transposed they typecheck: the
|
|
25
|
+
* fault's `artifactPath`, which callers read programmatically, would then hold
|
|
26
|
+
* a sentence.
|
|
27
|
+
*/
|
|
28
|
+
export function checkSchemaVersion(options) {
|
|
29
|
+
const { stamped, accepted, artifactPath, consequence } = options;
|
|
22
30
|
if (stamped === accepted)
|
|
23
31
|
return;
|
|
24
|
-
throw new RuntimeFault('schema-version-mismatch', artifactPath, `carries "schemaVersion" ${stamped} where this build reads ${accepted};
|
|
32
|
+
throw new RuntimeFault('schema-version-mismatch', artifactPath, `carries "schemaVersion" ${stamped} where this build reads ${accepted}; an artifact written for another version is not read leniently, ${consequence} (AD-11)`);
|
|
25
33
|
}
|
package/dist/core/emit/emit.js
CHANGED
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
*/
|
|
12
12
|
import { digestArtifact } from '../canonical/digest.js';
|
|
13
13
|
import { freezeArtifact } from '../lineage/freeze.js';
|
|
14
|
-
import { EvidenceArtifact, } from '../schemas/evidence-artifact.js';
|
|
14
|
+
import { EVIDENCE_ARTIFACT_SCHEMA_VERSION, EvidenceArtifact, } from '../schemas/evidence-artifact.js';
|
|
15
15
|
import { checkModeAgreement } from '../score/mode-agreement.js';
|
|
16
16
|
import { buildStrengthVector } from '../score/strength.js';
|
|
17
17
|
const SCORING_POLICY_ARTIFACT_PATH = 'ScoringPolicy';
|
|
@@ -74,7 +74,9 @@ export const emit = (scored, corpusDigest, fixtureDigest, evaluatorConfiguration
|
|
|
74
74
|
// re-checked a second time.
|
|
75
75
|
const verdict = scored.ladder.verdict;
|
|
76
76
|
const commonFields = {
|
|
77
|
-
|
|
77
|
+
// Read from the evidence schema's own constant, so a bump is one edit
|
|
78
|
+
// beside the shape that moved.
|
|
79
|
+
schemaVersion: EVIDENCE_ARTIFACT_SCHEMA_VERSION,
|
|
78
80
|
parentDigest: null,
|
|
79
81
|
// v0 mints no revision path for an evidence artifact: every `emit`
|
|
80
82
|
// call is a lineage root, matching `seal.ts`'s own root-artifact
|
|
@@ -9,11 +9,13 @@
|
|
|
9
9
|
* identifier up in at reduce time.
|
|
10
10
|
*/
|
|
11
11
|
import { isSupportedInterfaceKind, SUPPORTED_KINDS_CLAUSE, } from '../compile/interface-inventory.js';
|
|
12
|
+
import { checkSchemaVersion } from '../compile/schema-version.js';
|
|
12
13
|
import { checkInputsAgainstShape, isApiWitnessInputs, isMcpWitnessInputs, } from '../compile/sensitivity-witness.js';
|
|
13
14
|
import { declaresNoRequiredKeys, isCommandOperation, isMcpOperation, } from '../declared-inputs.js';
|
|
14
15
|
import { referenceSetKeysOf } from '../evaluate/evidence-resolution.js';
|
|
15
16
|
import { StructuralFailure } from '../failure-codes.js';
|
|
16
17
|
import { operationsOf } from '../schemas/interface.js';
|
|
18
|
+
import { PROBE_SCHEMA_VERSION, PROBE_SCHEMA_VERSION_CONSEQUENCE, } from '../schemas/probe.js';
|
|
17
19
|
import { referenceSetMembers } from './witness-evidence.js';
|
|
18
20
|
const requestOf = (legId, interfaceId, operation, inputs) => {
|
|
19
21
|
// NFR9's correlation by identifier: the port echoes these back, and they
|
|
@@ -202,6 +204,19 @@ const selectControl = (contract, permittedInterfaces) => {
|
|
|
202
204
|
*/
|
|
203
205
|
export const planPreflight = (input) => {
|
|
204
206
|
const { contract, probes, runId } = input;
|
|
207
|
+
// First, and before any check reads a probe. AD-11 makes an unequal stamp a
|
|
208
|
+
// rejection rather than a degraded read, and every rule below is written
|
|
209
|
+
// against this version's field shapes. `compile` performs the same check for
|
|
210
|
+
// the contract; this is the probe's reader, since a probe reaches no other
|
|
211
|
+
// core stage on the pre-flight path.
|
|
212
|
+
for (const probe of probes) {
|
|
213
|
+
checkSchemaVersion({
|
|
214
|
+
stamped: probe.schemaVersion,
|
|
215
|
+
accepted: PROBE_SCHEMA_VERSION,
|
|
216
|
+
artifactPath: `Probe[probeId=${probe.probeId}].schemaVersion`,
|
|
217
|
+
consequence: PROBE_SCHEMA_VERSION_CONSEQUENCE,
|
|
218
|
+
});
|
|
219
|
+
}
|
|
205
220
|
for (const iface of contract.permittedInterfaces) {
|
|
206
221
|
// Already thrown at compile; asserted again because the plan is
|
|
207
222
|
// reachable from a caller who assembled a contract by hand. Reads the
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { ProbeObservation } from '../schemas/port-messages.ts';
|
|
2
|
-
import type
|
|
2
|
+
import { type PreflightVerdict } from '../schemas/preflight-verdict.ts';
|
|
3
3
|
import type { ReduceStage } from '../stage-contracts.ts';
|
|
4
4
|
import type { PreflightPlan } from './plan.ts';
|
|
5
5
|
export type PreflightObservations = {
|
|
@@ -11,6 +11,7 @@ import { digestArtifact } from '../canonical/digest.js';
|
|
|
11
11
|
import { declaresNoRequestKeys } from '../declared-inputs.js';
|
|
12
12
|
import { freezeArtifact } from '../lineage/freeze.js';
|
|
13
13
|
import { RuntimeFault } from '../schemas/faults.js';
|
|
14
|
+
import { PREFLIGHT_VERDICT_SCHEMA_VERSION, } from '../schemas/preflight-verdict.js';
|
|
14
15
|
import { fixtureDigest, PREFLIGHT_ARTIFACT_PATH, projectObservation, } from './projection.js';
|
|
15
16
|
import { evidenceOf, resolveWitnessRelation } from './witness-evidence.js';
|
|
16
17
|
/**
|
|
@@ -297,8 +298,10 @@ export const reducePreflight = (plan, { observations }) => {
|
|
|
297
298
|
const projections = [...states.values()].map((state) => state.projected);
|
|
298
299
|
return freezeArtifact({
|
|
299
300
|
// A pre-flight verdict is an origin artifact, so AD-29's lineage fields
|
|
300
|
-
// carry their origin values.
|
|
301
|
-
|
|
301
|
+
// carry their origin values. The version is read from the verdict
|
|
302
|
+
// schema's own constant, so a bump is one edit beside the shape that
|
|
303
|
+
// moved.
|
|
304
|
+
schemaVersion: PREFLIGHT_VERDICT_SCHEMA_VERSION,
|
|
302
305
|
parentDigest: null,
|
|
303
306
|
revisionCount: 0,
|
|
304
307
|
runId: plan.runId,
|
|
@@ -12,6 +12,15 @@ export declare const JudgeConfiguration: z.ZodObject<{
|
|
|
12
12
|
modelSnapshot: z.ZodString;
|
|
13
13
|
systemPromptDigest: z.ZodNullable<z.ZodString>;
|
|
14
14
|
}, z.core.$strict>;
|
|
15
|
+
/**
|
|
16
|
+
* The configuration version this build accepts. No stage writes one, so a
|
|
17
|
+
* caller assembling a configuration had no value to read and transcribed the
|
|
18
|
+
* number.
|
|
19
|
+
*
|
|
20
|
+
* `1`: this artifact has never moved, so it has no predecessor shape and a
|
|
21
|
+
* parse-behaviour case over one would be vacuous.
|
|
22
|
+
*/
|
|
23
|
+
export declare const EVALUATOR_CONFIGURATION_SCHEMA_VERSION = 1;
|
|
15
24
|
export declare const EvaluatorConfiguration: z.ZodObject<{
|
|
16
25
|
schemaVersion: z.ZodInt;
|
|
17
26
|
parentDigest: z.ZodNullable<z.ZodString>;
|
|
@@ -15,6 +15,15 @@ export const JudgeConfiguration = z.strictObject({
|
|
|
15
15
|
modelSnapshot: z.string().min(1),
|
|
16
16
|
systemPromptDigest: Digest.nullable().describe('`null` for a judge configured with no system prompt of its own. This is the only nullable member: the absent-judge state belongs to the field above, not to this shape.'),
|
|
17
17
|
});
|
|
18
|
+
/**
|
|
19
|
+
* The configuration version this build accepts. No stage writes one, so a
|
|
20
|
+
* caller assembling a configuration had no value to read and transcribed the
|
|
21
|
+
* number.
|
|
22
|
+
*
|
|
23
|
+
* `1`: this artifact has never moved, so it has no predecessor shape and a
|
|
24
|
+
* parse-behaviour case over one would be vacuous.
|
|
25
|
+
*/
|
|
26
|
+
export const EVALUATOR_CONFIGURATION_SCHEMA_VERSION = 1;
|
|
18
27
|
export const EvaluatorConfiguration = z
|
|
19
28
|
.strictObject({
|
|
20
29
|
...lineageFields,
|
|
@@ -270,6 +270,15 @@ export declare const Remediation: z.ZodObject<{
|
|
|
270
270
|
noGap: z.ZodBoolean;
|
|
271
271
|
}, z.core.$strict>;
|
|
272
272
|
}, z.core.$strict>;
|
|
273
|
+
/**
|
|
274
|
+
* The evidence version `emit` stamps, and the single place that number is
|
|
275
|
+
* written. It was a literal inside `emit`'s own assembly, so the value a
|
|
276
|
+
* consumer needed sat in a stage rather than beside the shape it names.
|
|
277
|
+
*
|
|
278
|
+
* `3` on two recorded bumps: `mode` made it 2 and `uncitedFindingGaps` made
|
|
279
|
+
* it 3, each in the field's own description above.
|
|
280
|
+
*/
|
|
281
|
+
export declare const EVIDENCE_ARTIFACT_SCHEMA_VERSION = 3;
|
|
273
282
|
/**
|
|
274
283
|
* A discriminated union, because AD-21 requires that no shape hold a
|
|
275
284
|
* production verdict and a contract verdict at once, with neither mode able
|
|
@@ -253,6 +253,15 @@ const evidenceCommonFields = {
|
|
|
253
253
|
strength: Strength,
|
|
254
254
|
remediation: Remediation,
|
|
255
255
|
};
|
|
256
|
+
/**
|
|
257
|
+
* The evidence version `emit` stamps, and the single place that number is
|
|
258
|
+
* written. It was a literal inside `emit`'s own assembly, so the value a
|
|
259
|
+
* consumer needed sat in a stage rather than beside the shape it names.
|
|
260
|
+
*
|
|
261
|
+
* `3` on two recorded bumps: `mode` made it 2 and `uncitedFindingGaps` made
|
|
262
|
+
* it 3, each in the field's own description above.
|
|
263
|
+
*/
|
|
264
|
+
export const EVIDENCE_ARTIFACT_SCHEMA_VERSION = 3;
|
|
256
265
|
/**
|
|
257
266
|
* A discriminated union, because AD-21 requires that no shape hold a
|
|
258
267
|
* production verdict and a contract verdict at once, with neither mode able
|
|
@@ -69,6 +69,24 @@ export declare const ForbiddenInputAccountingMap: z.ZodObject<{
|
|
|
69
69
|
note: z.ZodNullable<z.ZodString>;
|
|
70
70
|
}, z.core.$strict>;
|
|
71
71
|
}, z.core.$strict>;
|
|
72
|
+
/**
|
|
73
|
+
* The manifest version this build accepts. No stage writes one, so a caller
|
|
74
|
+
* assembling a manifest had no value to read and transcribed the number.
|
|
75
|
+
*
|
|
76
|
+
* `1`, on one shape move the number never marked. `cb1cae8` retyped the six
|
|
77
|
+
* label arrays below from `z.array(z.string())` to `z.array(NonEmptyLabel)`
|
|
78
|
+
* and gave `violation` a `.min(1)`, which AD-11 counts as breaking, and the
|
|
79
|
+
* same commit took the eval contract to 4, the sealed run record to 4 and the
|
|
80
|
+
* probe to 3 while leaving this at 1. A manifest carrying
|
|
81
|
+
* `allowedMounts: ['']` parsed before that commit and fails after it, under
|
|
82
|
+
* the same version, in every release since.
|
|
83
|
+
*
|
|
84
|
+
* So the parse-behaviour method has nothing to compare here. It builds one
|
|
85
|
+
* fixture at the constant and one at the constant minus one, and 1 has no
|
|
86
|
+
* predecessor to build: the one break this shape has taken is inside the
|
|
87
|
+
* version it is still stamped with.
|
|
88
|
+
*/
|
|
89
|
+
export declare const ISOLATION_MANIFEST_SCHEMA_VERSION = 1;
|
|
72
90
|
export declare const IsolationManifest: z.ZodObject<{
|
|
73
91
|
schemaVersion: z.ZodInt;
|
|
74
92
|
parentDigest: z.ZodNullable<z.ZodString>;
|
|
@@ -62,6 +62,24 @@ export const ForbiddenInputAccountingMap = z.strictObject(accountingShape);
|
|
|
62
62
|
* nothing after its colon.
|
|
63
63
|
*/
|
|
64
64
|
const NonEmptyLabel = z.string().min(1);
|
|
65
|
+
/**
|
|
66
|
+
* The manifest version this build accepts. No stage writes one, so a caller
|
|
67
|
+
* assembling a manifest had no value to read and transcribed the number.
|
|
68
|
+
*
|
|
69
|
+
* `1`, on one shape move the number never marked. `cb1cae8` retyped the six
|
|
70
|
+
* label arrays below from `z.array(z.string())` to `z.array(NonEmptyLabel)`
|
|
71
|
+
* and gave `violation` a `.min(1)`, which AD-11 counts as breaking, and the
|
|
72
|
+
* same commit took the eval contract to 4, the sealed run record to 4 and the
|
|
73
|
+
* probe to 3 while leaving this at 1. A manifest carrying
|
|
74
|
+
* `allowedMounts: ['']` parsed before that commit and fails after it, under
|
|
75
|
+
* the same version, in every release since.
|
|
76
|
+
*
|
|
77
|
+
* So the parse-behaviour method has nothing to compare here. It builds one
|
|
78
|
+
* fixture at the constant and one at the constant minus one, and 1 has no
|
|
79
|
+
* predecessor to build: the one break this shape has taken is inside the
|
|
80
|
+
* version it is still stamped with.
|
|
81
|
+
*/
|
|
82
|
+
export const ISOLATION_MANIFEST_SCHEMA_VERSION = 1;
|
|
65
83
|
export const IsolationManifest = z
|
|
66
84
|
.strictObject({
|
|
67
85
|
...lineageFields,
|
|
@@ -32,6 +32,15 @@ export declare const PreflightCheck: z.ZodObject<{
|
|
|
32
32
|
note: z.ZodNullable<z.ZodString>;
|
|
33
33
|
}, z.core.$strict>;
|
|
34
34
|
export type PreflightCheck = z.infer<typeof PreflightCheck>;
|
|
35
|
+
/**
|
|
36
|
+
* The verdict version `preflight`'s reducer stamps, and the single place that
|
|
37
|
+
* number is written. It was a literal inside `reduce.ts`, so the value a
|
|
38
|
+
* consumer needed sat in a stage rather than beside the shape it names.
|
|
39
|
+
*
|
|
40
|
+
* `1`: this artifact has never moved, so it has no predecessor shape and a
|
|
41
|
+
* parse-behaviour case over one would be vacuous.
|
|
42
|
+
*/
|
|
43
|
+
export declare const PREFLIGHT_VERDICT_SCHEMA_VERSION = 1;
|
|
35
44
|
export declare const PreflightVerdict: z.ZodObject<{
|
|
36
45
|
schemaVersion: z.ZodInt;
|
|
37
46
|
parentDigest: z.ZodNullable<z.ZodString>;
|
|
@@ -24,6 +24,15 @@ export const PreflightCheck = z.strictObject({
|
|
|
24
24
|
.describe('`exempt` exists because AD-10 says "an operation declaring no inputs in any channel is exempt and records the exemption", and an exemption with no spelling is an exemption nobody records. That a failed pre-flight invalidates the run, and that a sensitivity witness resolving `insufficient-evidence` fails rather than passes, are AD-10 semantics for `core/preflight`; the schema carries the outcome and refines nothing.'),
|
|
25
25
|
note: z.string().nullable(),
|
|
26
26
|
});
|
|
27
|
+
/**
|
|
28
|
+
* The verdict version `preflight`'s reducer stamps, and the single place that
|
|
29
|
+
* number is written. It was a literal inside `reduce.ts`, so the value a
|
|
30
|
+
* consumer needed sat in a stage rather than beside the shape it names.
|
|
31
|
+
*
|
|
32
|
+
* `1`: this artifact has never moved, so it has no predecessor shape and a
|
|
33
|
+
* parse-behaviour case over one would be vacuous.
|
|
34
|
+
*/
|
|
35
|
+
export const PREFLIGHT_VERDICT_SCHEMA_VERSION = 1;
|
|
27
36
|
export const PreflightVerdict = z
|
|
28
37
|
.strictObject({
|
|
29
38
|
...lineageFields,
|
|
@@ -32,6 +32,16 @@ export declare const PrivateArtifactEntry: z.ZodObject<{
|
|
|
32
32
|
publicSafeRunId: z.ZodNullable<z.ZodString>;
|
|
33
33
|
sanitizationPolicy: z.ZodNullable<z.ZodString>;
|
|
34
34
|
}, z.core.$strict>;
|
|
35
|
+
/**
|
|
36
|
+
* The manifest version this build accepts. No stage writes one, so a caller
|
|
37
|
+
* publishing a manifest had no value to read and transcribed the number.
|
|
38
|
+
*
|
|
39
|
+
* `1`: this artifact has never moved, so it has no predecessor shape and a
|
|
40
|
+
* parse-behaviour case over one would be vacuous. The prior art's
|
|
41
|
+
* `manifestVersion: const 1` became AD-11's plain integer, so the number is
|
|
42
|
+
* not readable off the published document either.
|
|
43
|
+
*/
|
|
44
|
+
export declare const PRIVATE_ARTIFACT_MANIFEST_SCHEMA_VERSION = 1;
|
|
35
45
|
export declare const PrivateArtifactManifest: z.ZodObject<{
|
|
36
46
|
schemaVersion: z.ZodInt;
|
|
37
47
|
parentDigest: z.ZodNullable<z.ZodString>;
|
|
@@ -35,6 +35,16 @@ export const PrivateArtifactEntry = z.strictObject({
|
|
|
35
35
|
.nullable()
|
|
36
36
|
.describe('AD-8 carries "the sanitization policy applied to it", and the nearest antecedent is the entry rather than the manifest: a manifest holding a raw trace beside a human label gives the two different treatment, so per-entry is the only reading that survives. Opaque rather than an enum for the reason `ScopedResource.kind` is opaque: no AD supplies a value space, and inventing one is the unshaped-declaration defect in reverse. `null` spells "none applied", which must stay representable.'),
|
|
37
37
|
});
|
|
38
|
+
/**
|
|
39
|
+
* The manifest version this build accepts. No stage writes one, so a caller
|
|
40
|
+
* publishing a manifest had no value to read and transcribed the number.
|
|
41
|
+
*
|
|
42
|
+
* `1`: this artifact has never moved, so it has no predecessor shape and a
|
|
43
|
+
* parse-behaviour case over one would be vacuous. The prior art's
|
|
44
|
+
* `manifestVersion: const 1` became AD-11's plain integer, so the number is
|
|
45
|
+
* not readable off the published document either.
|
|
46
|
+
*/
|
|
47
|
+
export const PRIVATE_ARTIFACT_MANIFEST_SCHEMA_VERSION = 1;
|
|
38
48
|
export const PrivateArtifactManifest = z
|
|
39
49
|
.strictObject({
|
|
40
50
|
...lineageFields,
|
|
@@ -71,6 +71,47 @@ export declare const Defect: z.ZodObject<{
|
|
|
71
71
|
relation: z.ZodType<import("./expression.ts").Expression, unknown, z.core.$ZodTypeInternals<import("./expression.ts").Expression, unknown>>;
|
|
72
72
|
}, z.core.$strict>>;
|
|
73
73
|
}, z.core.$strict>;
|
|
74
|
+
/**
|
|
75
|
+
* The probe's current schema version. `EVAL_CONTRACT_SCHEMA_VERSION` is the
|
|
76
|
+
* same thing for the eval contract.
|
|
77
|
+
*
|
|
78
|
+
* It exists for the reason that one does: `lineage.ts` keeps the field a plain
|
|
79
|
+
* integer so a stale artifact fails as AD-28's `schema-version-mismatch` rather
|
|
80
|
+
* than as an anonymous parse error, which puts the comparison on the reader.
|
|
81
|
+
* `compile` is that reader for a contract. A probe has two in this pipeline and
|
|
82
|
+
* both compare against this constant: `planPreflight` before it plans a leg,
|
|
83
|
+
* and `score` before it seals the probe. An unequal stamp leaves by the fault
|
|
84
|
+
* path rather than being read leniently.
|
|
85
|
+
*
|
|
86
|
+
* A third reader exists for a caller outside this package.
|
|
87
|
+
* `validateLineageChain` takes an `acceptedSchemaVersion` and raises the same
|
|
88
|
+
* code over a presented chain, and `Probe` carries lineage, so a caller who
|
|
89
|
+
* presents one gets the comparison there. It words the fault its own way, which
|
|
90
|
+
* is why the two spellings of `schema-version-mismatch` in this tree are not a
|
|
91
|
+
* drift.
|
|
92
|
+
*
|
|
93
|
+
* It is also the single place the number is written. The committed
|
|
94
|
+
* worked-example chains build their probes from it and `check:doc-claims` reads
|
|
95
|
+
* it to hold the published sentence that names it, where the number was a
|
|
96
|
+
* literal in three places that could disagree in silence.
|
|
97
|
+
*/
|
|
98
|
+
export declare const PROBE_SCHEMA_VERSION = 5;
|
|
99
|
+
/**
|
|
100
|
+
* Why a stale probe stamp is a rejection, in the words the fault carries.
|
|
101
|
+
*
|
|
102
|
+
* One string for both readers. Two copies of a sentence this long disagree in
|
|
103
|
+
* silence exactly as the number did before `PROBE_SCHEMA_VERSION` existed, and
|
|
104
|
+
* it lives here because this is where a version bump is one edit.
|
|
105
|
+
*
|
|
106
|
+
* It splits the shapes by branch, because a clean control carries neither a
|
|
107
|
+
* `defectSignature` nor a manifestation witness: the signature is declared on
|
|
108
|
+
* the seeded branch below and a witness hangs off a `Defect`, which that branch
|
|
109
|
+
* bounds at zero. What every probe carries is the qualification record, whose
|
|
110
|
+
* arrival on both branches is what made the 1 to 2 bump breaking. A clean
|
|
111
|
+
* control is the probe a reader is most likely to meet this message with, so
|
|
112
|
+
* naming a field it does not have would be the wrong half to lead with.
|
|
113
|
+
*/
|
|
114
|
+
export declare const PROBE_SCHEMA_VERSION_CONSEQUENCE: string;
|
|
74
115
|
/**
|
|
75
116
|
* The prior art's `expectedClean` conditional, re-expressed as a discriminated
|
|
76
117
|
* union per AD-13 (a boolean literal discriminator parses on this pin,
|
|
@@ -48,6 +48,49 @@ const probeCommonFields = {
|
|
|
48
48
|
rationale: z.string().min(1),
|
|
49
49
|
qualification: ProbeQualification.describe('AD-9\'s qualification record: which of the five routes earned this probe its ground truth, and the evidence that route demands. Required on every branch and on every class, canaries included, because AD-9 closes with "an unqualified probe cannot enter a sealed set" and spells a route for all five kinds. That the route is compatible with this probe\'s class and `expectedClean` flag is a cross-field rule the export cannot carry; the corpus qualification gate enforces it and returns a reason code. Required, not optional, which with `defectSignature` below makes the probe\'s `schemaVersion` 1 -> 2 BREAKING bump under AD-11, whose rule is that "adding an optional field is a `schemaVersion` bump recorded in the field\'s own description; removing or retyping is breaking". This field is on both branches, so it alone is what stops every version-1 probe from parsing.'),
|
|
50
50
|
};
|
|
51
|
+
/**
|
|
52
|
+
* The probe's current schema version. `EVAL_CONTRACT_SCHEMA_VERSION` is the
|
|
53
|
+
* same thing for the eval contract.
|
|
54
|
+
*
|
|
55
|
+
* It exists for the reason that one does: `lineage.ts` keeps the field a plain
|
|
56
|
+
* integer so a stale artifact fails as AD-28's `schema-version-mismatch` rather
|
|
57
|
+
* than as an anonymous parse error, which puts the comparison on the reader.
|
|
58
|
+
* `compile` is that reader for a contract. A probe has two in this pipeline and
|
|
59
|
+
* both compare against this constant: `planPreflight` before it plans a leg,
|
|
60
|
+
* and `score` before it seals the probe. An unequal stamp leaves by the fault
|
|
61
|
+
* path rather than being read leniently.
|
|
62
|
+
*
|
|
63
|
+
* A third reader exists for a caller outside this package.
|
|
64
|
+
* `validateLineageChain` takes an `acceptedSchemaVersion` and raises the same
|
|
65
|
+
* code over a presented chain, and `Probe` carries lineage, so a caller who
|
|
66
|
+
* presents one gets the comparison there. It words the fault its own way, which
|
|
67
|
+
* is why the two spellings of `schema-version-mismatch` in this tree are not a
|
|
68
|
+
* drift.
|
|
69
|
+
*
|
|
70
|
+
* It is also the single place the number is written. The committed
|
|
71
|
+
* worked-example chains build their probes from it and `check:doc-claims` reads
|
|
72
|
+
* it to hold the published sentence that names it, where the number was a
|
|
73
|
+
* literal in three places that could disagree in silence.
|
|
74
|
+
*/
|
|
75
|
+
export const PROBE_SCHEMA_VERSION = 5;
|
|
76
|
+
/**
|
|
77
|
+
* Why a stale probe stamp is a rejection, in the words the fault carries.
|
|
78
|
+
*
|
|
79
|
+
* One string for both readers. Two copies of a sentence this long disagree in
|
|
80
|
+
* silence exactly as the number did before `PROBE_SCHEMA_VERSION` existed, and
|
|
81
|
+
* it lives here because this is where a version bump is one edit.
|
|
82
|
+
*
|
|
83
|
+
* It splits the shapes by branch, because a clean control carries neither a
|
|
84
|
+
* `defectSignature` nor a manifestation witness: the signature is declared on
|
|
85
|
+
* the seeded branch below and a witness hangs off a `Defect`, which that branch
|
|
86
|
+
* bounds at zero. What every probe carries is the qualification record, whose
|
|
87
|
+
* arrival on both branches is what made the 1 to 2 bump breaking. A clean
|
|
88
|
+
* control is the probe a reader is most likely to meet this message with, so
|
|
89
|
+
* naming a field it does not have would be the wrong half to lead with.
|
|
90
|
+
*/
|
|
91
|
+
export const PROBE_SCHEMA_VERSION_CONSEQUENCE = 'since the stamp says which shapes the probe was authored against: the ' +
|
|
92
|
+
'qualification record on every probe, and the witness legs and the defect ' +
|
|
93
|
+
'signature grammar on a seeded one';
|
|
51
94
|
/**
|
|
52
95
|
* The prior art's `expectedClean` conditional, re-expressed as a discriminated
|
|
53
96
|
* union per AD-13 (a boolean literal discriminator parses on this pin,
|
|
@@ -1,5 +1,16 @@
|
|
|
1
1
|
/** the published policy artifact the scorer reads its thresholds from. */
|
|
2
2
|
import { z } from 'zod';
|
|
3
|
+
/**
|
|
4
|
+
* The policy version this build accepts. No stage writes one, so a caller
|
|
5
|
+
* publishing a policy document had no value to read and transcribed the
|
|
6
|
+
* number.
|
|
7
|
+
*
|
|
8
|
+
* `2` because `catchThreshold` became required with no default, which the
|
|
9
|
+
* field's own description below records; a version-1 document omits it and
|
|
10
|
+
* fails to parse, which is the predecessor shape the parse-behaviour case is
|
|
11
|
+
* built on.
|
|
12
|
+
*/
|
|
13
|
+
export declare const SCORING_POLICY_SCHEMA_VERSION = 2;
|
|
3
14
|
/**
|
|
4
15
|
* A published artifact rather than constants, per the Consistency
|
|
5
16
|
* Conventions, so "the default" has an identity a no-op edit cannot move.
|
|
@@ -3,6 +3,17 @@ import { z } from 'zod';
|
|
|
3
3
|
import { Severity } from './eval-contract.js';
|
|
4
4
|
import { lineageFields } from './lineage.js';
|
|
5
5
|
import { Identifier } from './primitives.js';
|
|
6
|
+
/**
|
|
7
|
+
* The policy version this build accepts. No stage writes one, so a caller
|
|
8
|
+
* publishing a policy document had no value to read and transcribed the
|
|
9
|
+
* number.
|
|
10
|
+
*
|
|
11
|
+
* `2` because `catchThreshold` became required with no default, which the
|
|
12
|
+
* field's own description below records; a version-1 document omits it and
|
|
13
|
+
* fails to parse, which is the predecessor shape the parse-behaviour case is
|
|
14
|
+
* built on.
|
|
15
|
+
*/
|
|
16
|
+
export const SCORING_POLICY_SCHEMA_VERSION = 2;
|
|
6
17
|
/**
|
|
7
18
|
* A published artifact rather than constants, per the Consistency
|
|
8
19
|
* Conventions, so "the default" has an identity a no-op edit cannot move.
|
|
@@ -28,6 +28,18 @@ export declare const BriefInterface: z.ZodObject<{
|
|
|
28
28
|
web: "web";
|
|
29
29
|
}>;
|
|
30
30
|
}, z.core.$strict>;
|
|
31
|
+
/**
|
|
32
|
+
* The brief version `seal` stamps, and the single place that number is
|
|
33
|
+
* written. `seal` read it as a literal in its own module, so a bump meant
|
|
34
|
+
* editing a number in a file that declares no schema, and the value a
|
|
35
|
+
* consumer needed lived in a stage rather than beside the shape it names.
|
|
36
|
+
*
|
|
37
|
+
* `2` because `principals` is required, which AD-11 counts as breaking; the
|
|
38
|
+
* field's own description below records the bump. No `z.literal` on
|
|
39
|
+
* `schemaVersion` itself: `lineage.ts` states why, and this constant is the
|
|
40
|
+
* reader-side half of that decision seen from the caller.
|
|
41
|
+
*/
|
|
42
|
+
export declare const SEALED_EVALUATOR_BRIEF_SCHEMA_VERSION = 2;
|
|
31
43
|
export declare const SealedEvaluatorBrief: z.ZodObject<{
|
|
32
44
|
schemaVersion: z.ZodInt;
|
|
33
45
|
parentDigest: z.ZodNullable<z.ZodString>;
|
|
@@ -29,6 +29,18 @@ export const BriefInterface = z.strictObject({
|
|
|
29
29
|
logicalId: Identifier.describe('AD-35 binds the brief as it binds the contract: a logical interface identifier only, never a URL, host, or port. The brief is the artifact that actually reaches the executing caller, so a URL leaking onto it defeats the mapping AD-35 keeps outside the package.'),
|
|
30
30
|
kind: InterfaceKind,
|
|
31
31
|
});
|
|
32
|
+
/**
|
|
33
|
+
* The brief version `seal` stamps, and the single place that number is
|
|
34
|
+
* written. `seal` read it as a literal in its own module, so a bump meant
|
|
35
|
+
* editing a number in a file that declares no schema, and the value a
|
|
36
|
+
* consumer needed lived in a stage rather than beside the shape it names.
|
|
37
|
+
*
|
|
38
|
+
* `2` because `principals` is required, which AD-11 counts as breaking; the
|
|
39
|
+
* field's own description below records the bump. No `z.literal` on
|
|
40
|
+
* `schemaVersion` itself: `lineage.ts` states why, and this constant is the
|
|
41
|
+
* reader-side half of that decision seen from the caller.
|
|
42
|
+
*/
|
|
43
|
+
export const SEALED_EVALUATOR_BRIEF_SCHEMA_VERSION = 2;
|
|
32
44
|
export const SealedEvaluatorBrief = z
|
|
33
45
|
.strictObject({
|
|
34
46
|
...lineageFields,
|
|
@@ -320,6 +320,17 @@ export declare const RunMode: z.ZodEnum<{
|
|
|
320
320
|
"contract-scoring": "contract-scoring";
|
|
321
321
|
production: "production";
|
|
322
322
|
}>;
|
|
323
|
+
/**
|
|
324
|
+
* The record version this build accepts. No stage writes a sealed run record,
|
|
325
|
+
* so before this constant the number existed in `src/` only as prose in the
|
|
326
|
+
* description below, and a caller assembling a record transcribed it. One
|
|
327
|
+
* consumer transcribed it as 3 and emitted records no stage could read.
|
|
328
|
+
*
|
|
329
|
+
* `6` on five recorded bumps, the last of which drops `invalidReason`: a
|
|
330
|
+
* version-5 record carrying that key fails `strictObject`, which is the
|
|
331
|
+
* predecessor shape the parse-behaviour case is built on.
|
|
332
|
+
*/
|
|
333
|
+
export declare const SEALED_RUN_RECORD_SCHEMA_VERSION = 6;
|
|
323
334
|
export declare const SealedRunRecord: z.ZodObject<{
|
|
324
335
|
schemaVersion: z.ZodInt;
|
|
325
336
|
parentDigest: z.ZodNullable<z.ZodString>;
|
|
@@ -246,6 +246,17 @@ export const EvidenceDisclosure = z.strictObject({
|
|
|
246
246
|
*/
|
|
247
247
|
export const RUN_MODES = ['production', 'contract-scoring'];
|
|
248
248
|
export const RunMode = z.enum(RUN_MODES);
|
|
249
|
+
/**
|
|
250
|
+
* The record version this build accepts. No stage writes a sealed run record,
|
|
251
|
+
* so before this constant the number existed in `src/` only as prose in the
|
|
252
|
+
* description below, and a caller assembling a record transcribed it. One
|
|
253
|
+
* consumer transcribed it as 3 and emitted records no stage could read.
|
|
254
|
+
*
|
|
255
|
+
* `6` on five recorded bumps, the last of which drops `invalidReason`: a
|
|
256
|
+
* version-5 record carrying that key fails `strictObject`, which is the
|
|
257
|
+
* predecessor shape the parse-behaviour case is built on.
|
|
258
|
+
*/
|
|
259
|
+
export const SEALED_RUN_RECORD_SCHEMA_VERSION = 6;
|
|
249
260
|
export const SealedRunRecord = z
|
|
250
261
|
.strictObject({
|
|
251
262
|
// A record carries lineage fields and nothing here ever puts one in a
|
|
@@ -31,7 +31,7 @@
|
|
|
31
31
|
import type { ValidatedObservations } from '../ingest/ingest.ts';
|
|
32
32
|
import { type EvalContract } from '../schemas/eval-contract.ts';
|
|
33
33
|
import type { Outcome } from '../schemas/evidence-artifact.ts';
|
|
34
|
-
import type
|
|
34
|
+
import { type Probe } from '../schemas/probe.ts';
|
|
35
35
|
import type { ScoringPolicy } from '../schemas/scoring-policy.ts';
|
|
36
36
|
import type { ScoreStage } from '../stage-contracts.ts';
|
|
37
37
|
import type { ContractAssessment, LadderResolution, ProductionAssessment } from './ladder.ts';
|