eval-quality 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +190 -0
- package/README.md +355 -0
- package/corpus/dev/README.md +36 -0
- package/corpus/dev/compile-seal-example/brief.json +1 -0
- package/corpus/dev/compile-seal-example/contract.json +1 -0
- package/corpus/dev/contracts/absent-collection-locations.json +1 -0
- package/corpus/dev/contracts/absent-sibling-groups.json +1 -0
- package/corpus/dev/contracts/absent-success-indicator.json +1 -0
- package/corpus/dev/contracts/empty-channel-roles.json +1 -0
- package/corpus/dev/contracts/empty-collection-locations.json +1 -0
- package/corpus/dev/contracts/empty-request-shapes.json +1 -0
- package/corpus/dev/contracts/empty-sibling-groups.json +1 -0
- package/corpus/dev/contracts/no-collection-quantifier.json +1 -0
- package/corpus/dev/contracts/no-operation-inventory.json +1 -0
- package/corpus/dev/contracts/no-read-back-relation.json +1 -0
- package/corpus/dev/contracts/no-state-change-marker.json +1 -0
- package/corpus/dev/contracts/no-type-violating-step.json +1 -0
- package/corpus/dev/contracts/per-key-split-oracles.json +1 -0
- package/corpus/dev/contracts/satisfied-declarations.json +1 -0
- package/corpus/dev/contracts/single-required-response-key.json +1 -0
- package/corpus/dev/contracts/split-indicator-oracle.json +1 -0
- package/corpus/dev/contracts/unaddressed-parameter-sibling.json +1 -0
- package/corpus/dev/contracts/unnamed-reference-set.json +1 -0
- package/corpus/dev/contracts/wrong-cardinality-form.json +1 -0
- package/corpus/dev/index.json +1 -0
- package/dist/adapters/index.d.ts +12 -0
- package/dist/adapters/index.js +3 -0
- package/dist/adapters/local-corpus-adapter.d.ts +7 -0
- package/dist/adapters/local-corpus-adapter.js +78 -0
- package/dist/adapters/node-file-system-adapter.d.ts +7 -0
- package/dist/adapters/node-file-system-adapter.js +39 -0
- package/dist/adapters/port-boundary.d.ts +18 -0
- package/dist/adapters/port-boundary.js +83 -0
- package/dist/adapters/system-clock-adapter.d.ts +10 -0
- package/dist/adapters/system-clock-adapter.js +18 -0
- package/dist/application/compile.d.ts +4 -0
- package/dist/application/compile.js +24 -0
- package/dist/application/diagnostics.d.ts +22 -0
- package/dist/application/diagnostics.js +18 -0
- package/dist/application/index.d.ts +23 -0
- package/dist/application/index.js +17 -0
- package/dist/application/invoke-port.d.ts +2 -0
- package/dist/application/invoke-port.js +45 -0
- package/dist/application/preflight.d.ts +27 -0
- package/dist/application/preflight.js +131 -0
- package/dist/application/seal.d.ts +4 -0
- package/dist/application/seal.js +28 -0
- package/dist/application/serialize.d.ts +6 -0
- package/dist/application/serialize.js +9 -0
- package/dist/cli/arguments.d.ts +28 -0
- package/dist/cli/arguments.js +167 -0
- package/dist/cli/exit-codes.d.ts +36 -0
- package/dist/cli/exit-codes.js +39 -0
- package/dist/cli/main.d.ts +2 -0
- package/dist/cli/main.js +143 -0
- package/dist/cli/render.d.ts +26 -0
- package/dist/cli/render.js +53 -0
- package/dist/cli/run.d.ts +41 -0
- package/dist/cli/run.js +203 -0
- package/dist/core/canonical/canonicalize.d.ts +1 -0
- package/dist/core/canonical/canonicalize.js +113 -0
- package/dist/core/canonical/digest.d.ts +7 -0
- package/dist/core/canonical/digest.js +81 -0
- package/dist/core/canonical/scan-json.d.ts +1 -0
- package/dist/core/canonical/scan-json.js +298 -0
- package/dist/core/canonical/value-domain.d.ts +4 -0
- package/dist/core/canonical/value-domain.js +113 -0
- package/dist/core/compile/compile.d.ts +24 -0
- package/dist/core/compile/compile.js +67 -0
- package/dist/core/compile/declarations.d.ts +5 -0
- package/dist/core/compile/declarations.js +22 -0
- package/dist/core/compile/expression-legality.d.ts +14 -0
- package/dist/core/compile/expression-legality.js +346 -0
- package/dist/core/compile/forbidden-inputs.d.ts +5 -0
- package/dist/core/compile/forbidden-inputs.js +19 -0
- package/dist/core/compile/interface-inventory.d.ts +10 -0
- package/dist/core/compile/interface-inventory.js +55 -0
- package/dist/core/compile/oracle-alignment.d.ts +14 -0
- package/dist/core/compile/oracle-alignment.js +93 -0
- package/dist/core/compile/reachability.d.ts +17 -0
- package/dist/core/compile/reachability.js +177 -0
- package/dist/core/compile/rubrics.d.ts +55 -0
- package/dist/core/compile/rubrics.js +198 -0
- package/dist/core/compile/scripting-bound.d.ts +5 -0
- package/dist/core/compile/scripting-bound.js +151 -0
- package/dist/core/compile/sensitivity-witness.d.ts +73 -0
- package/dist/core/compile/sensitivity-witness.js +232 -0
- package/dist/core/compile/waivers.d.ts +3 -0
- package/dist/core/compile/waivers.js +21 -0
- package/dist/core/coverage/coverage.d.ts +18 -0
- package/dist/core/coverage/coverage.js +47 -0
- package/dist/core/coverage/relevance.d.ts +75 -0
- package/dist/core/coverage/relevance.js +186 -0
- package/dist/core/coverage/rules.d.ts +7 -0
- package/dist/core/coverage/rules.js +18 -0
- package/dist/core/coverage/satisfaction.d.ts +107 -0
- package/dist/core/coverage/satisfaction.js +415 -0
- package/dist/core/coverage/table.d.ts +26 -0
- package/dist/core/coverage/table.js +222 -0
- package/dist/core/declared-inputs.d.ts +16 -0
- package/dist/core/declared-inputs.js +18 -0
- package/dist/core/evaluate/evidence-resolution.d.ts +54 -0
- package/dist/core/evaluate/evidence-resolution.js +153 -0
- package/dist/core/evaluate/operators.d.ts +87 -0
- package/dist/core/evaluate/operators.js +350 -0
- package/dist/core/evaluate/resolution.d.ts +40 -0
- package/dist/core/evaluate/resolution.js +336 -0
- package/dist/core/evaluate/resolved-value.d.ts +9 -0
- package/dist/core/evaluate/resolved-value.js +6 -0
- package/dist/core/failure-codes.d.ts +18 -0
- package/dist/core/failure-codes.js +50 -0
- package/dist/core/lineage/chain.d.ts +49 -0
- package/dist/core/lineage/chain.js +210 -0
- package/dist/core/lineage/freeze.d.ts +12 -0
- package/dist/core/lineage/freeze.js +35 -0
- package/dist/core/lineage/stage-table.d.ts +43 -0
- package/dist/core/lineage/stage-table.js +106 -0
- package/dist/core/preflight/plan.d.ts +67 -0
- package/dist/core/preflight/plan.js +313 -0
- package/dist/core/preflight/projection.d.ts +40 -0
- package/dist/core/preflight/projection.js +96 -0
- package/dist/core/preflight/reduce.d.ts +8 -0
- package/dist/core/preflight/reduce.js +180 -0
- package/dist/core/preflight/witness-evidence.d.ts +42 -0
- package/dist/core/preflight/witness-evidence.js +88 -0
- package/dist/core/probe/target-policy.d.ts +63 -0
- package/dist/core/probe/target-policy.js +328 -0
- package/dist/core/schemas/artifact-reference.d.ts +19 -0
- package/dist/core/schemas/artifact-reference.js +30 -0
- package/dist/core/schemas/artifact.d.ts +1153 -0
- package/dist/core/schemas/artifact.js +84 -0
- package/dist/core/schemas/constraint-ledger.d.ts +57 -0
- package/dist/core/schemas/constraint-ledger.js +143 -0
- package/dist/core/schemas/eval-contract.d.ts +402 -0
- package/dist/core/schemas/eval-contract.js +147 -0
- package/dist/core/schemas/evaluator-configuration.d.ts +37 -0
- package/dist/core/schemas/evaluator-configuration.js +43 -0
- package/dist/core/schemas/evidence-artifact.d.ts +471 -0
- package/dist/core/schemas/evidence-artifact.js +232 -0
- package/dist/core/schemas/expression.d.ts +197 -0
- package/dist/core/schemas/expression.js +284 -0
- package/dist/core/schemas/faults.d.ts +9 -0
- package/dist/core/schemas/faults.js +32 -0
- package/dist/core/schemas/interface.d.ts +417 -0
- package/dist/core/schemas/interface.js +115 -0
- package/dist/core/schemas/isolation-manifest.d.ts +136 -0
- package/dist/core/schemas/isolation-manifest.js +92 -0
- package/dist/core/schemas/lineage.d.ts +21 -0
- package/dist/core/schemas/lineage.js +28 -0
- package/dist/core/schemas/oracle.d.ts +72 -0
- package/dist/core/schemas/oracle.js +35 -0
- package/dist/core/schemas/plan.d.ts +115 -0
- package/dist/core/schemas/plan.js +58 -0
- package/dist/core/schemas/pointer.d.ts +63 -0
- package/dist/core/schemas/pointer.js +94 -0
- package/dist/core/schemas/port-messages.d.ts +103 -0
- package/dist/core/schemas/port-messages.js +90 -0
- package/dist/core/schemas/preflight-verdict.d.ts +60 -0
- package/dist/core/schemas/preflight-verdict.js +38 -0
- package/dist/core/schemas/primitives.d.ts +78 -0
- package/dist/core/schemas/primitives.js +121 -0
- package/dist/core/schemas/private-artifact-manifest.d.ts +56 -0
- package/dist/core/schemas/private-artifact-manifest.js +46 -0
- package/dist/core/schemas/probe-body.d.ts +25 -0
- package/dist/core/schemas/probe-body.js +23 -0
- package/dist/core/schemas/probe-policy.d.ts +76 -0
- package/dist/core/schemas/probe-policy.js +33 -0
- package/dist/core/schemas/probe.d.ts +189 -0
- package/dist/core/schemas/probe.js +82 -0
- package/dist/core/schemas/publish.d.ts +30 -0
- package/dist/core/schemas/publish.js +135 -0
- package/dist/core/schemas/reference-set.d.ts +18 -0
- package/dist/core/schemas/reference-set.js +27 -0
- package/dist/core/schemas/rubric.d.ts +62 -0
- package/dist/core/schemas/rubric.js +68 -0
- package/dist/core/schemas/scoring-policy.d.ts +28 -0
- package/dist/core/schemas/scoring-policy.js +51 -0
- package/dist/core/schemas/sealed-evaluator-brief.d.ts +80 -0
- package/dist/core/schemas/sealed-evaluator-brief.js +59 -0
- package/dist/core/schemas/sealed-run-record.d.ts +382 -0
- package/dist/core/schemas/sealed-run-record.js +215 -0
- package/dist/core/schemas/sensitivity-witness.d.ts +138 -0
- package/dist/core/schemas/sensitivity-witness.js +86 -0
- package/dist/core/schemas/verdict.d.ts +25 -0
- package/dist/core/schemas/verdict.js +20 -0
- package/dist/core/schemas/waiver.d.ts +17 -0
- package/dist/core/schemas/waiver.js +24 -0
- package/dist/core/seal/derived-reference.d.ts +36 -0
- package/dist/core/seal/derived-reference.js +383 -0
- package/dist/core/seal/direction-prose.d.ts +13 -0
- package/dist/core/seal/direction-prose.js +120 -0
- package/dist/core/seal/plan-index.d.ts +63 -0
- package/dist/core/seal/plan-index.js +155 -0
- package/dist/core/seal/scripting-audit.d.ts +15 -0
- package/dist/core/seal/scripting-audit.js +51 -0
- package/dist/core/seal/seal.d.ts +18 -0
- package/dist/core/seal/seal.js +155 -0
- package/dist/core/stage-contracts.d.ts +30 -0
- package/dist/core/stage-contracts.js +1 -0
- package/dist/index.d.ts +15 -0
- package/dist/index.js +22 -0
- package/dist/ports/clock-port.d.ts +13 -0
- package/dist/ports/clock-port.js +7 -0
- package/dist/ports/corpus-port.d.ts +16 -0
- package/dist/ports/corpus-port.js +7 -0
- package/dist/ports/environment-probe-port.d.ts +76 -0
- package/dist/ports/environment-probe-port.js +32 -0
- package/dist/ports/file-system-port.d.ts +27 -0
- package/dist/ports/file-system-port.js +11 -0
- package/dist/ports/port.d.ts +33 -0
- package/dist/ports/port.js +14 -0
- package/dist/testing/conformance.d.ts +88 -0
- package/dist/testing/conformance.js +281 -0
- package/dist/testing/index.d.ts +26 -0
- package/dist/testing/index.js +18 -0
- package/dist/testing/probe-conformance.d.ts +45 -0
- package/dist/testing/probe-conformance.js +186 -0
- package/package.json +130 -0
- package/schemas/artifact-reference.schema.json +64 -0
- package/schemas/eval-contract.schema.json +2145 -0
- package/schemas/evaluator-configuration.schema.json +199 -0
- package/schemas/evidence-artifact.schema.json +1229 -0
- package/schemas/isolation-manifest.schema.json +394 -0
- package/schemas/preflight-verdict.schema.json +114 -0
- package/schemas/private-artifact-manifest.schema.json +103 -0
- package/schemas/probe.schema.json +1151 -0
- package/schemas/rubric.schema.json +148 -0
- package/schemas/scoring-policy.schema.json +90 -0
- package/schemas/sealed-evaluator-brief.schema.json +268 -0
- package/schemas/sealed-run-record.schema.json +875 -0
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
/** the rubric body a contract embeds, and the published Rubric artifact. */
|
|
2
|
+
import { z } from 'zod';
|
|
3
|
+
import { lineageFields } from './lineage.js';
|
|
4
|
+
import { InteractionPointer } from './pointer.js';
|
|
5
|
+
import { RubricCriterionId, RubricId } from './primitives.js';
|
|
6
|
+
// AD-22 requires anchored scale levels but supplies no shape. Minted here as
|
|
7
|
+
// an ordinal plus its anchor condition, nothing more; see the field
|
|
8
|
+
// descriptions below for what stays unenforced in v0 and why.
|
|
9
|
+
export const ScaleLevel = z.strictObject({
|
|
10
|
+
level: z
|
|
11
|
+
.int()
|
|
12
|
+
.describe('The ordinal this level sits at. A repeated ordinal is the one half the compiler closes, failing under `rubric-unanchored`, since two levels at one ordinal make the ordinal address two anchors. Magnitude, sign, ordering, and contiguity stay free everywhere, because a scale running -2 to 2 or 1, 3, 5 is an ordinary authoring choice.'),
|
|
13
|
+
anchor: z
|
|
14
|
+
.string()
|
|
15
|
+
.describe('The observable condition that anchors this level. AD-22 calls an unanchored scale a compile-time failure under `rubric-unanchored`, so the anchor is a declared field rather than a convention.'),
|
|
16
|
+
});
|
|
17
|
+
export const FailureModePenalty = z.strictObject({
|
|
18
|
+
name: z.string().min(1),
|
|
19
|
+
description: z
|
|
20
|
+
.string()
|
|
21
|
+
.describe("AD-22 requires named failure-mode penalties and states no magnitude. None is invented here: a penalty weight would be a scoring semantic no module below `score` has authority to mint, and AD-7 keeps weighted composites out of the reported result. AD-22's compile checks ship without one, so a penalty carries a name and a description and nothing else."),
|
|
22
|
+
});
|
|
23
|
+
export const RubricCriterion = z.strictObject({
|
|
24
|
+
id: RubricCriterionId,
|
|
25
|
+
text: z
|
|
26
|
+
.string()
|
|
27
|
+
.describe("A criterion whose wording matches the compiler's closed stated-reasoning vocabulary fails compilation under `rubric-scores-reasoning-prose`, so such a criterion must stay representable here. A blank text parses for the same reason and fails under `rubric-unanchored`."),
|
|
28
|
+
evidence: InteractionPointer.describe('Where the criterion is answerable from. A pointer that resolves nowhere is `rubric-evidence-unreachable`, a compile-time code, and is deliberately not a schema rejection.'),
|
|
29
|
+
});
|
|
30
|
+
/**
|
|
31
|
+
* The embeddable rubric body: the published `Rubric` artifact below adds
|
|
32
|
+
* `schemaVersion`, lineage, and a prior-art declaration on top of this.
|
|
33
|
+
*/
|
|
34
|
+
export const RubricBody = z
|
|
35
|
+
.strictObject({
|
|
36
|
+
id: RubricId,
|
|
37
|
+
scaleLevels: z
|
|
38
|
+
.array(ScaleLevel)
|
|
39
|
+
.nullable()
|
|
40
|
+
.describe('`null` and `[]` are both the unanchored shape `rubric-unanchored` fires on, and both must parse.'),
|
|
41
|
+
failureModePenalties: z
|
|
42
|
+
.array(FailureModePenalty)
|
|
43
|
+
.nullable()
|
|
44
|
+
.describe('Missing named penalties is `rubric-unanchored`, not a parse failure.'),
|
|
45
|
+
maxLength: z
|
|
46
|
+
.int()
|
|
47
|
+
.min(1)
|
|
48
|
+
.nullable()
|
|
49
|
+
.describe('An unbounded length is `null`, which is one of the shapes `rubric-unanchored` fires on.'),
|
|
50
|
+
criteria: z.array(RubricCriterion),
|
|
51
|
+
})
|
|
52
|
+
.meta({
|
|
53
|
+
id: 'RubricBody',
|
|
54
|
+
description: 'The embeddable rubric body. Named so the shared body has a stable `$defs` key distinct from the published `Rubric` artifact, which is this body plus `schemaVersion` and AD-29 lineage; without the name the two collide in the published-schema drift check under a generated positional name.',
|
|
55
|
+
});
|
|
56
|
+
/**
|
|
57
|
+
* The published Rubric artifact: `RubricBody` above, spread flat, plus
|
|
58
|
+
* AD-11's version and AD-29's lineage.
|
|
59
|
+
*/
|
|
60
|
+
export const Rubric = z
|
|
61
|
+
.strictObject({
|
|
62
|
+
...RubricBody.shape,
|
|
63
|
+
...lineageFields,
|
|
64
|
+
})
|
|
65
|
+
.meta({
|
|
66
|
+
id: 'Rubric',
|
|
67
|
+
description: "A published rubric, with no prior art: the experiments' evaluator-result and trace-label schemas belong to the deferred semantic layer and their names are not reused. AD-22's compile checks cover anchored scale levels, named failure-mode penalties, a bounded length, criteria that state a question, addressable rubric and criterion identifiers, and wording that matches the compiler's closed stated-reasoning vocabulary. All of them fire under `rubric-unanchored`, `rubric-evidence-unreachable`, and `rubric-scores-reasoning-prose` at compile time, so every shape those codes fire on parses here.",
|
|
68
|
+
});
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
/** the published policy artifact the scorer reads its thresholds from. */
|
|
2
|
+
import { z } from 'zod';
|
|
3
|
+
/**
|
|
4
|
+
* A published artifact rather than constants, per the Consistency
|
|
5
|
+
* Conventions, so "the default" has an identity a no-op edit cannot move.
|
|
6
|
+
* No `.default()` anywhere: it would diverge the input- and output-mode
|
|
7
|
+
* exports by dropping the key from `required` only in input mode. `strictMode`
|
|
8
|
+
* and `corpusLocation` are deliberately absent: AD-14 fixes strict mode as a
|
|
9
|
+
* CLI flag and AD-8 puts the corpus behind `CorpusPort`, so either would give
|
|
10
|
+
* one concern two homes.
|
|
11
|
+
*/
|
|
12
|
+
export declare const ScoringPolicy: z.ZodObject<{
|
|
13
|
+
schemaVersion: z.ZodInt;
|
|
14
|
+
parentDigest: z.ZodNullable<z.ZodString>;
|
|
15
|
+
revisionCount: z.ZodInt;
|
|
16
|
+
policyId: z.ZodString;
|
|
17
|
+
severityFloor: z.ZodEnum<{
|
|
18
|
+
critical: "critical";
|
|
19
|
+
low: "low";
|
|
20
|
+
material: "material";
|
|
21
|
+
}>;
|
|
22
|
+
confidenceThreshold: z.ZodNumber;
|
|
23
|
+
minimumTrialCount: z.ZodInt;
|
|
24
|
+
reExecutionCap: z.ZodInt;
|
|
25
|
+
remediationCap: z.ZodInt;
|
|
26
|
+
regexMatchStepBudget: z.ZodInt;
|
|
27
|
+
}, z.core.$strict>;
|
|
28
|
+
export type ScoringPolicy = z.infer<typeof ScoringPolicy>;
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
/** the published policy artifact the scorer reads its thresholds from. */
|
|
2
|
+
import { z } from 'zod';
|
|
3
|
+
import { Severity } from './eval-contract.js';
|
|
4
|
+
import { lineageFields } from './lineage.js';
|
|
5
|
+
import { Identifier } from './primitives.js';
|
|
6
|
+
/**
|
|
7
|
+
* A published artifact rather than constants, per the Consistency
|
|
8
|
+
* Conventions, so "the default" has an identity a no-op edit cannot move.
|
|
9
|
+
* No `.default()` anywhere: it would diverge the input- and output-mode
|
|
10
|
+
* exports by dropping the key from `required` only in input mode. `strictMode`
|
|
11
|
+
* and `corpusLocation` are deliberately absent: AD-14 fixes strict mode as a
|
|
12
|
+
* CLI flag and AD-8 puts the corpus behind `CorpusPort`, so either would give
|
|
13
|
+
* one concern two homes.
|
|
14
|
+
*/
|
|
15
|
+
export const ScoringPolicy = z
|
|
16
|
+
.strictObject({
|
|
17
|
+
...lineageFields,
|
|
18
|
+
policyId: Identifier,
|
|
19
|
+
severityFloor: Severity.describe('AD-21 routes on this: a behavioural failure at or above the floor is FAIL and one below it is CONCERNS, and AD-7 makes it the one severity that constrains the dominance relation.'),
|
|
20
|
+
confidenceThreshold: z
|
|
21
|
+
.number()
|
|
22
|
+
.min(0)
|
|
23
|
+
.max(1)
|
|
24
|
+
.describe('The same closed unit interval a finding\'s `confidence` uses. AD-21\'s "a finding whose confidence falls below the policy threshold" compares the two, and two different scales would make that comparison meaningless.'),
|
|
25
|
+
minimumTrialCount: z
|
|
26
|
+
.int()
|
|
27
|
+
.min(1)
|
|
28
|
+
.describe('AD-6: repeated runs are trials, never retries. The published default artifact carries 3, "because that is what the instrument behind the measured effect used"; that number is recorded here and is deliberately not a schema default.'),
|
|
29
|
+
reExecutionCap: z
|
|
30
|
+
.int()
|
|
31
|
+
.min(0)
|
|
32
|
+
.describe('AD-6: exceeding the cap resolves under AD-21 as a statement that the harness is unfit, never that the contract is weak. The published default artifact carries 2.'),
|
|
33
|
+
remediationCap: z
|
|
34
|
+
.int()
|
|
35
|
+
.min(0)
|
|
36
|
+
.describe("AD-12's remediation cap. The published default artifact carries 3. AD-12 states the package validates rather than enforces it, which is why the Evidence Artifact records the cap's source as caller-attested."),
|
|
37
|
+
regexMatchStepBudget: z
|
|
38
|
+
.int()
|
|
39
|
+
.min(1)
|
|
40
|
+
.describe('AD-4: "a match-step budget from the scoring policy whose breach is a fault, not an outcome ' +
|
|
41
|
+
'state." The budget bounds a pure, static, pre-execution complexity estimate over the ' +
|
|
42
|
+
'declared pattern and the candidate string length, never a literal engine-internal step ' +
|
|
43
|
+
'count, and that length is whatever the system under test returned, which the contract ' +
|
|
44
|
+
'author does not control. The structural nested-quantifier rejection stays unconditional ' +
|
|
45
|
+
'and independent of this value, so a generous ceiling here does not weaken that real ' +
|
|
46
|
+
'backstop. The published default artifact carries 1000000.'),
|
|
47
|
+
})
|
|
48
|
+
.meta({
|
|
49
|
+
id: 'ScoringPolicy',
|
|
50
|
+
description: 'The scoring policy, with no prior art. It carries the severity floor, confidence threshold, minimum trial count, re-execution cap, remediation cap, and a regex match-step budget that AD-6, AD-12, AD-4, and AD-21 read, and its digest is one of the five named inputs to AD-11\'s scoring version. It is a published artifact rather than a set of constants so that "the default" has an identity a result can name by digest.',
|
|
51
|
+
});
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
/** the only artifact that reaches the executing caller. */
|
|
2
|
+
import { z } from 'zod';
|
|
3
|
+
/**
|
|
4
|
+
* AD-3's generated evaluator-facing direction, keyed by the oracle it belongs
|
|
5
|
+
* to. The evaluator sees oracle identifiers but never step identifiers
|
|
6
|
+
* (AD-23 requires citing the oracle; AD-16 bars step identifiers), and
|
|
7
|
+
* carries no `behaviorId`: `Behavior.oracles` is an array, so two behaviours
|
|
8
|
+
* can cite one oracle and a single id here would not be truthful.
|
|
9
|
+
*/
|
|
10
|
+
export declare const BriefDirection: z.ZodObject<{
|
|
11
|
+
oracleId: z.ZodString;
|
|
12
|
+
text: z.ZodString;
|
|
13
|
+
}, z.core.$strict>;
|
|
14
|
+
export type BriefDirection = z.infer<typeof BriefDirection>;
|
|
15
|
+
/**
|
|
16
|
+
* Interface identity only, no operation inventory: shipping one would hand
|
|
17
|
+
* the evaluator the action list AD-39 withholds, one artifact over from
|
|
18
|
+
* where AD-5's scripting-shape predicate looks. Withholding it costs nothing
|
|
19
|
+
* in practice: an isolation-manifest arm found a search parameter by trying
|
|
20
|
+
* `?search=`, `?query=`, and `?q=` in turn, and still caught the defect.
|
|
21
|
+
*/
|
|
22
|
+
export declare const BriefInterface: z.ZodObject<{
|
|
23
|
+
logicalId: z.ZodString;
|
|
24
|
+
kind: z.ZodEnum<{
|
|
25
|
+
api: "api";
|
|
26
|
+
cli: "cli";
|
|
27
|
+
mcp: "mcp";
|
|
28
|
+
web: "web";
|
|
29
|
+
}>;
|
|
30
|
+
}, z.core.$strict>;
|
|
31
|
+
export declare const SealedEvaluatorBrief: z.ZodObject<{
|
|
32
|
+
schemaVersion: z.ZodInt;
|
|
33
|
+
parentDigest: z.ZodNullable<z.ZodString>;
|
|
34
|
+
revisionCount: z.ZodInt;
|
|
35
|
+
contractDigest: z.ZodString;
|
|
36
|
+
behaviors: z.ZodArray<z.ZodObject<{
|
|
37
|
+
id: z.ZodString;
|
|
38
|
+
description: z.ZodString;
|
|
39
|
+
severity: z.ZodEnum<{
|
|
40
|
+
critical: "critical";
|
|
41
|
+
low: "low";
|
|
42
|
+
material: "material";
|
|
43
|
+
}>;
|
|
44
|
+
observableSuccessCriterion: z.ZodNullable<z.ZodString>;
|
|
45
|
+
requirementLinks: z.ZodArray<z.ZodObject<{
|
|
46
|
+
scheme: z.ZodString;
|
|
47
|
+
id: z.ZodString;
|
|
48
|
+
}, z.core.$strict>>;
|
|
49
|
+
riskLinks: z.ZodArray<z.ZodObject<{
|
|
50
|
+
scheme: z.ZodString;
|
|
51
|
+
id: z.ZodString;
|
|
52
|
+
}, z.core.$strict>>;
|
|
53
|
+
oracles: z.ZodArray<z.ZodString>;
|
|
54
|
+
}, z.core.$strict>>;
|
|
55
|
+
directions: z.ZodArray<z.ZodObject<{
|
|
56
|
+
oracleId: z.ZodString;
|
|
57
|
+
text: z.ZodString;
|
|
58
|
+
}, z.core.$strict>>;
|
|
59
|
+
permittedInterfaces: z.ZodArray<z.ZodObject<{
|
|
60
|
+
logicalId: z.ZodString;
|
|
61
|
+
kind: z.ZodEnum<{
|
|
62
|
+
api: "api";
|
|
63
|
+
cli: "cli";
|
|
64
|
+
mcp: "mcp";
|
|
65
|
+
web: "web";
|
|
66
|
+
}>;
|
|
67
|
+
}, z.core.$strict>>;
|
|
68
|
+
scopedResources: z.ZodArray<z.ZodObject<{
|
|
69
|
+
reference: z.ZodString;
|
|
70
|
+
kind: z.ZodString;
|
|
71
|
+
}, z.core.$strict>>;
|
|
72
|
+
budgets: z.ZodObject<{
|
|
73
|
+
maxToolCalls: z.ZodInt;
|
|
74
|
+
maxWallClockMinutes: z.ZodInt;
|
|
75
|
+
maxCostUsd: z.ZodString;
|
|
76
|
+
}, z.core.$strict>;
|
|
77
|
+
safetyLimits: z.ZodArray<z.ZodString>;
|
|
78
|
+
probeStepBound: z.ZodNullable<z.ZodInt>;
|
|
79
|
+
}, z.core.$strict>;
|
|
80
|
+
export type SealedEvaluatorBrief = z.infer<typeof SealedEvaluatorBrief>;
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
/** the only artifact that reaches the executing caller. */
|
|
2
|
+
import { z } from 'zod';
|
|
3
|
+
import { Behavior, Budgets, ScopedResource } from './eval-contract.js';
|
|
4
|
+
import { InterfaceKind } from './interface.js';
|
|
5
|
+
import { lineageFields } from './lineage.js';
|
|
6
|
+
import { Digest, Identifier, OracleId } from './primitives.js';
|
|
7
|
+
/**
|
|
8
|
+
* AD-3's generated evaluator-facing direction, keyed by the oracle it belongs
|
|
9
|
+
* to. The evaluator sees oracle identifiers but never step identifiers
|
|
10
|
+
* (AD-23 requires citing the oracle; AD-16 bars step identifiers), and
|
|
11
|
+
* carries no `behaviorId`: `Behavior.oracles` is an array, so two behaviours
|
|
12
|
+
* can cite one oracle and a single id here would not be truthful.
|
|
13
|
+
*/
|
|
14
|
+
export const BriefDirection = z.strictObject({
|
|
15
|
+
oracleId: OracleId,
|
|
16
|
+
text: z
|
|
17
|
+
.string()
|
|
18
|
+
.min(1)
|
|
19
|
+
.describe('The generated prose. `seal` renders each evidence target as a descriptive reference derived from the step\'s operation and selection predicate, such as "the response you obtained when you sent an invalid identifier", never as the step identifier itself; the mapping from step to phrase is one-way. It never carries a prescribed action sequence.'),
|
|
20
|
+
});
|
|
21
|
+
/**
|
|
22
|
+
* Interface identity only, no operation inventory: shipping one would hand
|
|
23
|
+
* the evaluator the action list AD-39 withholds, one artifact over from
|
|
24
|
+
* where AD-5's scripting-shape predicate looks. Withholding it costs nothing
|
|
25
|
+
* in practice: an isolation-manifest arm found a search parameter by trying
|
|
26
|
+
* `?search=`, `?query=`, and `?q=` in turn, and still caught the defect.
|
|
27
|
+
*/
|
|
28
|
+
export const BriefInterface = z.strictObject({
|
|
29
|
+
logicalId: Identifier.describe('AD-35 binds the brief as it binds the contract: a logical interface identifier only, never a URL, host, or port. The brief is the artifact that actually reaches the executing caller, so a URL leaking onto it defeats the mapping AD-35 keeps outside the package.'),
|
|
30
|
+
kind: InterfaceKind,
|
|
31
|
+
});
|
|
32
|
+
export const SealedEvaluatorBrief = z
|
|
33
|
+
.strictObject({
|
|
34
|
+
...lineageFields,
|
|
35
|
+
contractDigest: Digest.describe('Binds this brief to the contract it seals. The brief carries no contract identifier of its own and no identity field at all: every artifact that refers to a brief does so by `sealedBriefDigest`, so an identifier would be a second name nothing reads.'),
|
|
36
|
+
behaviors: z
|
|
37
|
+
.array(Behavior)
|
|
38
|
+
.min(1)
|
|
39
|
+
.describe("The contract's behaviours, carried through unchanged. At least one, on the contract's own reasoning: a brief with no behaviour directs nothing."),
|
|
40
|
+
directions: z
|
|
41
|
+
.array(BriefDirection)
|
|
42
|
+
.min(1)
|
|
43
|
+
.describe("AD-3's generated evaluator-facing directions, one per discharged oracle. At least one, on the same reasoning as `behaviors` above: a brief with no direction verifies nothing."),
|
|
44
|
+
permittedInterfaces: z.array(BriefInterface),
|
|
45
|
+
scopedResources: z
|
|
46
|
+
.array(ScopedResource)
|
|
47
|
+
.describe("AD-16's scoped resource references. Not nullable here, unlike on the contract: absent-versus-explicitly-empty is an AD-31 grading of a CALLER declaration, and this artifact is generated by `seal`, which always knows the answer. No scoped reference may resolve to a forbidden input; that resolution is `scoped-reference-resolves-forbidden` at compile time and is not a schema check."),
|
|
48
|
+
budgets: Budgets.describe("The contract's declared ceilings, carried because AD-16 says the caller who executes needs the ceilings the isolation manifest reports against. The manifest's own `resourceCeilings` is a different shape and a different concern: this is a declaration, that is a record of what the harness enforced."),
|
|
49
|
+
safetyLimits: z.array(z.string()),
|
|
50
|
+
probeStepBound: z
|
|
51
|
+
.int()
|
|
52
|
+
.min(0)
|
|
53
|
+
.nullable()
|
|
54
|
+
.describe("AD-16's declared bound on enumerated probe steps, carried through from the contract. The brief-side scripting audit reads it after generation and fires `brief-exceeds-scripting-bound` on any single direction whose rendered text carries more markers than the bound; counts are never summed across directions."),
|
|
55
|
+
})
|
|
56
|
+
.meta({
|
|
57
|
+
id: 'SealedEvaluatorBrief',
|
|
58
|
+
description: "The sealed evaluator brief, with no prior art. AD-16 fixes what it carries: the contract's behaviours, the generated evaluator-facing directions of AD-3, permitted interfaces, and scoped resource references, plus budgets and safety limits. It also fixes what the brief never carries: author commentary, the interaction plan, or the plan's step identifiers, and never a prescribed action sequence. Those exclusions are structural rather than conventional: this object is strict, so a brief carrying `interactionPlan`, `commentary`, or a `stepId` fails with `unrecognized_keys`, and each of the three has its own reject fixture rather than resting on strictness alone. It carries no identity field of its own, because every artifact that refers to it does so by digest.",
|
|
59
|
+
});
|
|
@@ -0,0 +1,382 @@
|
|
|
1
|
+
/** the caller's inbound record of one sealed evaluator trial. */
|
|
2
|
+
import { z } from 'zod';
|
|
3
|
+
import { JsonValue } from './primitives.ts';
|
|
4
|
+
/**
|
|
5
|
+
* AD-23's verbatim quotation, paired with the channel it came from: a
|
|
6
|
+
* quotation with no channel cannot be audited against its source observation
|
|
7
|
+
* (ADR-009 Decision 2).
|
|
8
|
+
*/
|
|
9
|
+
export declare const QuotedEvidence: z.ZodObject<{
|
|
10
|
+
quote: z.ZodString;
|
|
11
|
+
channel: z.ZodEnum<{
|
|
12
|
+
"call-inputs": "call-inputs";
|
|
13
|
+
"exit-code": "exit-code";
|
|
14
|
+
"response-body": "response-body";
|
|
15
|
+
"response-headers": "response-headers";
|
|
16
|
+
"response-status": "response-status";
|
|
17
|
+
stderr: "stderr";
|
|
18
|
+
stdout: "stdout";
|
|
19
|
+
}>;
|
|
20
|
+
}, z.core.$strict>;
|
|
21
|
+
/**
|
|
22
|
+
* `defect` findings additionally require observation identifiers and quoted
|
|
23
|
+
* evidence (AD-23). A nullable field can't express "required on this branch
|
|
24
|
+
* only", so this is a discriminated union, the house treatment for a
|
|
25
|
+
* conditional (AD-13).
|
|
26
|
+
*/
|
|
27
|
+
export declare const Finding: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
28
|
+
findingId: z.ZodString;
|
|
29
|
+
oracleId: z.ZodNullable<z.ZodString>;
|
|
30
|
+
probeId: z.ZodString;
|
|
31
|
+
behaviorId: z.ZodNullable<z.ZodString>;
|
|
32
|
+
severity: z.ZodEnum<{
|
|
33
|
+
critical: "critical";
|
|
34
|
+
low: "low";
|
|
35
|
+
material: "material";
|
|
36
|
+
}>;
|
|
37
|
+
summary: z.ZodString;
|
|
38
|
+
confidence: z.ZodNumber;
|
|
39
|
+
evidenceArtifacts: z.ZodArray<z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
40
|
+
storage: z.ZodLiteral<"public">;
|
|
41
|
+
path: z.ZodString;
|
|
42
|
+
privateRef: z.ZodNull;
|
|
43
|
+
digest: z.ZodString;
|
|
44
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
45
|
+
storage: z.ZodLiteral<"private">;
|
|
46
|
+
path: z.ZodNull;
|
|
47
|
+
privateRef: z.ZodString;
|
|
48
|
+
digest: z.ZodString;
|
|
49
|
+
}, z.core.$strict>], "storage">>;
|
|
50
|
+
findingType: z.ZodLiteral<"defect">;
|
|
51
|
+
observationIds: z.ZodArray<z.ZodString>;
|
|
52
|
+
quotedEvidence: z.ZodArray<z.ZodObject<{
|
|
53
|
+
quote: z.ZodString;
|
|
54
|
+
channel: z.ZodEnum<{
|
|
55
|
+
"call-inputs": "call-inputs";
|
|
56
|
+
"exit-code": "exit-code";
|
|
57
|
+
"response-body": "response-body";
|
|
58
|
+
"response-headers": "response-headers";
|
|
59
|
+
"response-status": "response-status";
|
|
60
|
+
stderr: "stderr";
|
|
61
|
+
stdout: "stdout";
|
|
62
|
+
}>;
|
|
63
|
+
}, z.core.$strict>>;
|
|
64
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
65
|
+
findingId: z.ZodString;
|
|
66
|
+
oracleId: z.ZodNullable<z.ZodString>;
|
|
67
|
+
probeId: z.ZodString;
|
|
68
|
+
behaviorId: z.ZodNullable<z.ZodString>;
|
|
69
|
+
severity: z.ZodEnum<{
|
|
70
|
+
critical: "critical";
|
|
71
|
+
low: "low";
|
|
72
|
+
material: "material";
|
|
73
|
+
}>;
|
|
74
|
+
summary: z.ZodString;
|
|
75
|
+
confidence: z.ZodNumber;
|
|
76
|
+
observationIds: z.ZodArray<z.ZodString>;
|
|
77
|
+
evidenceArtifacts: z.ZodArray<z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
78
|
+
storage: z.ZodLiteral<"public">;
|
|
79
|
+
path: z.ZodString;
|
|
80
|
+
privateRef: z.ZodNull;
|
|
81
|
+
digest: z.ZodString;
|
|
82
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
83
|
+
storage: z.ZodLiteral<"private">;
|
|
84
|
+
path: z.ZodNull;
|
|
85
|
+
privateRef: z.ZodString;
|
|
86
|
+
digest: z.ZodString;
|
|
87
|
+
}, z.core.$strict>], "storage">>;
|
|
88
|
+
findingType: z.ZodLiteral<"observation">;
|
|
89
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
90
|
+
findingId: z.ZodString;
|
|
91
|
+
oracleId: z.ZodNullable<z.ZodString>;
|
|
92
|
+
probeId: z.ZodString;
|
|
93
|
+
behaviorId: z.ZodNullable<z.ZodString>;
|
|
94
|
+
severity: z.ZodEnum<{
|
|
95
|
+
critical: "critical";
|
|
96
|
+
low: "low";
|
|
97
|
+
material: "material";
|
|
98
|
+
}>;
|
|
99
|
+
summary: z.ZodString;
|
|
100
|
+
confidence: z.ZodNumber;
|
|
101
|
+
observationIds: z.ZodArray<z.ZodString>;
|
|
102
|
+
evidenceArtifacts: z.ZodArray<z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
103
|
+
storage: z.ZodLiteral<"public">;
|
|
104
|
+
path: z.ZodString;
|
|
105
|
+
privateRef: z.ZodNull;
|
|
106
|
+
digest: z.ZodString;
|
|
107
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
108
|
+
storage: z.ZodLiteral<"private">;
|
|
109
|
+
path: z.ZodNull;
|
|
110
|
+
privateRef: z.ZodString;
|
|
111
|
+
digest: z.ZodString;
|
|
112
|
+
}, z.core.$strict>], "storage">>;
|
|
113
|
+
findingType: z.ZodLiteral<"confirmation">;
|
|
114
|
+
}, z.core.$strict>], "findingType">;
|
|
115
|
+
/**
|
|
116
|
+
* AD-23 requires one disposition per required oracle: held, violated, or
|
|
117
|
+
* not-attempted. The three words are AD-23's own vocabulary.
|
|
118
|
+
*/
|
|
119
|
+
export declare const ORACLE_DISPOSITIONS: readonly ['held', 'violated', 'not-attempted'];
|
|
120
|
+
export declare const OracleDispositionValue: z.ZodEnum<{
|
|
121
|
+
held: "held";
|
|
122
|
+
"not-attempted": "not-attempted";
|
|
123
|
+
violated: "violated";
|
|
124
|
+
}>;
|
|
125
|
+
export declare const OracleDisposition: z.ZodObject<{
|
|
126
|
+
oracleId: z.ZodString;
|
|
127
|
+
disposition: z.ZodEnum<{
|
|
128
|
+
held: "held";
|
|
129
|
+
"not-attempted": "not-attempted";
|
|
130
|
+
violated: "violated";
|
|
131
|
+
}>;
|
|
132
|
+
observationIds: z.ZodArray<z.ZodString>;
|
|
133
|
+
note: z.ZodNullable<z.ZodString>;
|
|
134
|
+
}, z.core.$strict>;
|
|
135
|
+
/**
|
|
136
|
+
* A flat map would break pointer addressing: AD-26 keys `call-inputs` by
|
|
137
|
+
* transport channel, so a pointer like
|
|
138
|
+
* `/interactions/write/call-inputs/body/title` needs that segment to resolve
|
|
139
|
+
* against. A four-key strict object rather than a record over the transport
|
|
140
|
+
* enum, for the same reason as `RequestShape` and `InputBinding`: a record
|
|
141
|
+
* demands every enum member at parse time, but a real observation binds only
|
|
142
|
+
* a subset.
|
|
143
|
+
*/
|
|
144
|
+
export declare const ObservedCallInputs: z.ZodObject<{
|
|
145
|
+
path: z.ZodNullable<z.ZodType<import("./primitives.ts").JsonObject, unknown, z.core.$ZodTypeInternals<import("./primitives.ts").JsonObject, unknown>>>;
|
|
146
|
+
query: z.ZodNullable<z.ZodType<import("./primitives.ts").JsonObject, unknown, z.core.$ZodTypeInternals<import("./primitives.ts").JsonObject, unknown>>>;
|
|
147
|
+
header: z.ZodNullable<z.ZodType<import("./primitives.ts").JsonObject, unknown, z.core.$ZodTypeInternals<import("./primitives.ts").JsonObject, unknown>>>;
|
|
148
|
+
body: z.ZodNullable<z.ZodType<import("./primitives.ts").JsonObject, unknown, z.core.$ZodTypeInternals<import("./primitives.ts").JsonObject, unknown>>>;
|
|
149
|
+
}, z.core.$strict>;
|
|
150
|
+
/**
|
|
151
|
+
* One ingested observation, carrying AD-26's closed channel set so every
|
|
152
|
+
* pointer in the addressing grammar has something to resolve against.
|
|
153
|
+
*
|
|
154
|
+
* Observation ordering is deliberately absent (Owed item 2): ADR-006 forbids
|
|
155
|
+
* using array position as ordering, and the fix, a monotonic sequence, arrives
|
|
156
|
+
* as an additive `schemaVersion` bump under AD-11.
|
|
157
|
+
*/
|
|
158
|
+
export declare const Observation: z.ZodObject<{
|
|
159
|
+
observationId: z.ZodString;
|
|
160
|
+
operationId: z.ZodString;
|
|
161
|
+
provenance: z.ZodEnum<{
|
|
162
|
+
baseline: "baseline";
|
|
163
|
+
"evaluator-chosen": "evaluator-chosen";
|
|
164
|
+
}>;
|
|
165
|
+
callInputs: z.ZodObject<{
|
|
166
|
+
path: z.ZodNullable<z.ZodType<import("./primitives.ts").JsonObject, unknown, z.core.$ZodTypeInternals<import("./primitives.ts").JsonObject, unknown>>>;
|
|
167
|
+
query: z.ZodNullable<z.ZodType<import("./primitives.ts").JsonObject, unknown, z.core.$ZodTypeInternals<import("./primitives.ts").JsonObject, unknown>>>;
|
|
168
|
+
header: z.ZodNullable<z.ZodType<import("./primitives.ts").JsonObject, unknown, z.core.$ZodTypeInternals<import("./primitives.ts").JsonObject, unknown>>>;
|
|
169
|
+
body: z.ZodNullable<z.ZodType<import("./primitives.ts").JsonObject, unknown, z.core.$ZodTypeInternals<import("./primitives.ts").JsonObject, unknown>>>;
|
|
170
|
+
}, z.core.$strict>;
|
|
171
|
+
responseBody: z.ZodNullable<z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>;
|
|
172
|
+
responseHeaders: z.ZodNullable<z.ZodType<import("./primitives.ts").JsonObject, unknown, z.core.$ZodTypeInternals<import("./primitives.ts").JsonObject, unknown>>>;
|
|
173
|
+
responseStatus: z.ZodNullable<z.ZodInt>;
|
|
174
|
+
stdout: z.ZodNullable<z.ZodString>;
|
|
175
|
+
stderr: z.ZodNullable<z.ZodString>;
|
|
176
|
+
exitCode: z.ZodNullable<z.ZodInt>;
|
|
177
|
+
}, z.core.$strict>;
|
|
178
|
+
export type ObservedCallInputs = z.infer<typeof ObservedCallInputs>;
|
|
179
|
+
export type Observation = z.infer<typeof Observation>;
|
|
180
|
+
/**
|
|
181
|
+
* AD-17: judge results arrive inside the sealed run record and the package
|
|
182
|
+
* never calls a judge.
|
|
183
|
+
*/
|
|
184
|
+
export declare const JudgeResult: z.ZodObject<{
|
|
185
|
+
rubricId: z.ZodString;
|
|
186
|
+
criterionId: z.ZodString;
|
|
187
|
+
score: z.ZodNullable<z.ZodInt>;
|
|
188
|
+
note: z.ZodNullable<z.ZodString>;
|
|
189
|
+
}, z.core.$strict>;
|
|
190
|
+
/** the prior art's five members, with money as a string per AD-36. */
|
|
191
|
+
export declare const ResourceUse: z.ZodObject<{
|
|
192
|
+
toolCalls: z.ZodInt;
|
|
193
|
+
inputTokens: z.ZodInt;
|
|
194
|
+
outputTokens: z.ZodInt;
|
|
195
|
+
wallClockSeconds: z.ZodNumber;
|
|
196
|
+
costUsd: z.ZodString;
|
|
197
|
+
}, z.core.$strict>;
|
|
198
|
+
/**
|
|
199
|
+
* AD-21's FAIL rung reads on evidence that is incomplete, over-truncated,
|
|
200
|
+
* unavailable, or internally inconsistent (AD-17), but no artifact declares
|
|
201
|
+
* those fields. Two of the four are caller statements and land here; the other
|
|
202
|
+
* two are derived and declared nowhere.
|
|
203
|
+
*/
|
|
204
|
+
export declare const EvidenceDisclosure: z.ZodObject<{
|
|
205
|
+
truncationBound: z.ZodNullable<z.ZodInt>;
|
|
206
|
+
reportedIncomplete: z.ZodBoolean;
|
|
207
|
+
}, z.core.$strict>;
|
|
208
|
+
export declare const SealedRunRecord: z.ZodObject<{
|
|
209
|
+
schemaVersion: z.ZodInt;
|
|
210
|
+
parentDigest: z.ZodNullable<z.ZodString>;
|
|
211
|
+
revisionCount: z.ZodInt;
|
|
212
|
+
runId: z.ZodString;
|
|
213
|
+
conditionArm: z.ZodString;
|
|
214
|
+
trialIndex: z.ZodInt;
|
|
215
|
+
contractDigest: z.ZodString;
|
|
216
|
+
sealedBriefDigest: z.ZodString;
|
|
217
|
+
evaluatorConfigurationDigest: z.ZodString;
|
|
218
|
+
evaluatorRecommendation: z.ZodEnum<{
|
|
219
|
+
CONCERNS: "CONCERNS";
|
|
220
|
+
FAIL: "FAIL";
|
|
221
|
+
PASS: "PASS";
|
|
222
|
+
}>;
|
|
223
|
+
oracleDispositions: z.ZodArray<z.ZodObject<{
|
|
224
|
+
oracleId: z.ZodString;
|
|
225
|
+
disposition: z.ZodEnum<{
|
|
226
|
+
held: "held";
|
|
227
|
+
"not-attempted": "not-attempted";
|
|
228
|
+
violated: "violated";
|
|
229
|
+
}>;
|
|
230
|
+
observationIds: z.ZodArray<z.ZodString>;
|
|
231
|
+
note: z.ZodNullable<z.ZodString>;
|
|
232
|
+
}, z.core.$strict>>;
|
|
233
|
+
findings: z.ZodArray<z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
234
|
+
findingId: z.ZodString;
|
|
235
|
+
oracleId: z.ZodNullable<z.ZodString>;
|
|
236
|
+
probeId: z.ZodString;
|
|
237
|
+
behaviorId: z.ZodNullable<z.ZodString>;
|
|
238
|
+
severity: z.ZodEnum<{
|
|
239
|
+
critical: "critical";
|
|
240
|
+
low: "low";
|
|
241
|
+
material: "material";
|
|
242
|
+
}>;
|
|
243
|
+
summary: z.ZodString;
|
|
244
|
+
confidence: z.ZodNumber;
|
|
245
|
+
evidenceArtifacts: z.ZodArray<z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
246
|
+
storage: z.ZodLiteral<"public">;
|
|
247
|
+
path: z.ZodString;
|
|
248
|
+
privateRef: z.ZodNull;
|
|
249
|
+
digest: z.ZodString;
|
|
250
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
251
|
+
storage: z.ZodLiteral<"private">;
|
|
252
|
+
path: z.ZodNull;
|
|
253
|
+
privateRef: z.ZodString;
|
|
254
|
+
digest: z.ZodString;
|
|
255
|
+
}, z.core.$strict>], "storage">>;
|
|
256
|
+
findingType: z.ZodLiteral<"defect">;
|
|
257
|
+
observationIds: z.ZodArray<z.ZodString>;
|
|
258
|
+
quotedEvidence: z.ZodArray<z.ZodObject<{
|
|
259
|
+
quote: z.ZodString;
|
|
260
|
+
channel: z.ZodEnum<{
|
|
261
|
+
"call-inputs": "call-inputs";
|
|
262
|
+
"exit-code": "exit-code";
|
|
263
|
+
"response-body": "response-body";
|
|
264
|
+
"response-headers": "response-headers";
|
|
265
|
+
"response-status": "response-status";
|
|
266
|
+
stderr: "stderr";
|
|
267
|
+
stdout: "stdout";
|
|
268
|
+
}>;
|
|
269
|
+
}, z.core.$strict>>;
|
|
270
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
271
|
+
findingId: z.ZodString;
|
|
272
|
+
oracleId: z.ZodNullable<z.ZodString>;
|
|
273
|
+
probeId: z.ZodString;
|
|
274
|
+
behaviorId: z.ZodNullable<z.ZodString>;
|
|
275
|
+
severity: z.ZodEnum<{
|
|
276
|
+
critical: "critical";
|
|
277
|
+
low: "low";
|
|
278
|
+
material: "material";
|
|
279
|
+
}>;
|
|
280
|
+
summary: z.ZodString;
|
|
281
|
+
confidence: z.ZodNumber;
|
|
282
|
+
observationIds: z.ZodArray<z.ZodString>;
|
|
283
|
+
evidenceArtifacts: z.ZodArray<z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
284
|
+
storage: z.ZodLiteral<"public">;
|
|
285
|
+
path: z.ZodString;
|
|
286
|
+
privateRef: z.ZodNull;
|
|
287
|
+
digest: z.ZodString;
|
|
288
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
289
|
+
storage: z.ZodLiteral<"private">;
|
|
290
|
+
path: z.ZodNull;
|
|
291
|
+
privateRef: z.ZodString;
|
|
292
|
+
digest: z.ZodString;
|
|
293
|
+
}, z.core.$strict>], "storage">>;
|
|
294
|
+
findingType: z.ZodLiteral<"observation">;
|
|
295
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
296
|
+
findingId: z.ZodString;
|
|
297
|
+
oracleId: z.ZodNullable<z.ZodString>;
|
|
298
|
+
probeId: z.ZodString;
|
|
299
|
+
behaviorId: z.ZodNullable<z.ZodString>;
|
|
300
|
+
severity: z.ZodEnum<{
|
|
301
|
+
critical: "critical";
|
|
302
|
+
low: "low";
|
|
303
|
+
material: "material";
|
|
304
|
+
}>;
|
|
305
|
+
summary: z.ZodString;
|
|
306
|
+
confidence: z.ZodNumber;
|
|
307
|
+
observationIds: z.ZodArray<z.ZodString>;
|
|
308
|
+
evidenceArtifacts: z.ZodArray<z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
309
|
+
storage: z.ZodLiteral<"public">;
|
|
310
|
+
path: z.ZodString;
|
|
311
|
+
privateRef: z.ZodNull;
|
|
312
|
+
digest: z.ZodString;
|
|
313
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
314
|
+
storage: z.ZodLiteral<"private">;
|
|
315
|
+
path: z.ZodNull;
|
|
316
|
+
privateRef: z.ZodString;
|
|
317
|
+
digest: z.ZodString;
|
|
318
|
+
}, z.core.$strict>], "storage">>;
|
|
319
|
+
findingType: z.ZodLiteral<"confirmation">;
|
|
320
|
+
}, z.core.$strict>], "findingType">>;
|
|
321
|
+
observations: z.ZodArray<z.ZodObject<{
|
|
322
|
+
observationId: z.ZodString;
|
|
323
|
+
operationId: z.ZodString;
|
|
324
|
+
provenance: z.ZodEnum<{
|
|
325
|
+
baseline: "baseline";
|
|
326
|
+
"evaluator-chosen": "evaluator-chosen";
|
|
327
|
+
}>;
|
|
328
|
+
callInputs: z.ZodObject<{
|
|
329
|
+
path: z.ZodNullable<z.ZodType<import("./primitives.ts").JsonObject, unknown, z.core.$ZodTypeInternals<import("./primitives.ts").JsonObject, unknown>>>;
|
|
330
|
+
query: z.ZodNullable<z.ZodType<import("./primitives.ts").JsonObject, unknown, z.core.$ZodTypeInternals<import("./primitives.ts").JsonObject, unknown>>>;
|
|
331
|
+
header: z.ZodNullable<z.ZodType<import("./primitives.ts").JsonObject, unknown, z.core.$ZodTypeInternals<import("./primitives.ts").JsonObject, unknown>>>;
|
|
332
|
+
body: z.ZodNullable<z.ZodType<import("./primitives.ts").JsonObject, unknown, z.core.$ZodTypeInternals<import("./primitives.ts").JsonObject, unknown>>>;
|
|
333
|
+
}, z.core.$strict>;
|
|
334
|
+
responseBody: z.ZodNullable<z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>;
|
|
335
|
+
responseHeaders: z.ZodNullable<z.ZodType<import("./primitives.ts").JsonObject, unknown, z.core.$ZodTypeInternals<import("./primitives.ts").JsonObject, unknown>>>;
|
|
336
|
+
responseStatus: z.ZodNullable<z.ZodInt>;
|
|
337
|
+
stdout: z.ZodNullable<z.ZodString>;
|
|
338
|
+
stderr: z.ZodNullable<z.ZodString>;
|
|
339
|
+
exitCode: z.ZodNullable<z.ZodInt>;
|
|
340
|
+
}, z.core.$strict>>;
|
|
341
|
+
judgeResults: z.ZodArray<z.ZodObject<{
|
|
342
|
+
rubricId: z.ZodString;
|
|
343
|
+
criterionId: z.ZodString;
|
|
344
|
+
score: z.ZodNullable<z.ZodInt>;
|
|
345
|
+
note: z.ZodNullable<z.ZodString>;
|
|
346
|
+
}, z.core.$strict>>;
|
|
347
|
+
actionsArtifact: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
348
|
+
storage: z.ZodLiteral<"public">;
|
|
349
|
+
path: z.ZodString;
|
|
350
|
+
privateRef: z.ZodNull;
|
|
351
|
+
digest: z.ZodString;
|
|
352
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
353
|
+
storage: z.ZodLiteral<"private">;
|
|
354
|
+
path: z.ZodNull;
|
|
355
|
+
privateRef: z.ZodString;
|
|
356
|
+
digest: z.ZodString;
|
|
357
|
+
}, z.core.$strict>], "storage">;
|
|
358
|
+
isolationManifestArtifact: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
359
|
+
storage: z.ZodLiteral<"public">;
|
|
360
|
+
path: z.ZodString;
|
|
361
|
+
privateRef: z.ZodNull;
|
|
362
|
+
digest: z.ZodString;
|
|
363
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
364
|
+
storage: z.ZodLiteral<"private">;
|
|
365
|
+
path: z.ZodNull;
|
|
366
|
+
privateRef: z.ZodString;
|
|
367
|
+
digest: z.ZodString;
|
|
368
|
+
}, z.core.$strict>], "storage">;
|
|
369
|
+
resourceUse: z.ZodObject<{
|
|
370
|
+
toolCalls: z.ZodInt;
|
|
371
|
+
inputTokens: z.ZodInt;
|
|
372
|
+
outputTokens: z.ZodInt;
|
|
373
|
+
wallClockSeconds: z.ZodNumber;
|
|
374
|
+
costUsd: z.ZodString;
|
|
375
|
+
}, z.core.$strict>;
|
|
376
|
+
evidenceDisclosure: z.ZodObject<{
|
|
377
|
+
truncationBound: z.ZodNullable<z.ZodInt>;
|
|
378
|
+
reportedIncomplete: z.ZodBoolean;
|
|
379
|
+
}, z.core.$strict>;
|
|
380
|
+
invalidReason: z.ZodNullable<z.ZodString>;
|
|
381
|
+
}, z.core.$strict>;
|
|
382
|
+
export type SealedRunRecord = z.infer<typeof SealedRunRecord>;
|