eval-quality 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +142 -67
- package/corpus/dev/README.md +17 -11
- package/corpus/dev/compile-seal-example/brief.json +1 -1
- package/corpus/dev/compile-seal-example/contract.json +1 -1
- package/corpus/dev/contracts/absent-collection-locations.json +1 -1
- package/corpus/dev/contracts/absent-sibling-groups.json +1 -1
- package/corpus/dev/contracts/absent-success-indicator.json +1 -1
- package/corpus/dev/contracts/empty-channel-roles.json +1 -1
- package/corpus/dev/contracts/empty-collection-locations.json +1 -1
- package/corpus/dev/contracts/empty-request-shapes.json +1 -1
- package/corpus/dev/contracts/empty-sibling-groups.json +1 -1
- package/corpus/dev/contracts/no-collection-quantifier.json +1 -1
- package/corpus/dev/contracts/no-operation-inventory.json +1 -1
- package/corpus/dev/contracts/no-read-back-relation.json +1 -1
- package/corpus/dev/contracts/no-state-change-marker.json +1 -1
- package/corpus/dev/contracts/no-type-violating-step.json +1 -1
- package/corpus/dev/contracts/per-key-split-oracles.json +1 -1
- package/corpus/dev/contracts/satisfied-declarations.json +1 -1
- package/corpus/dev/contracts/single-required-response-key.json +1 -1
- package/corpus/dev/contracts/split-indicator-oracle.json +1 -1
- package/corpus/dev/contracts/unaddressed-parameter-sibling.json +1 -1
- package/corpus/dev/contracts/unnamed-reference-set.json +1 -1
- package/corpus/dev/contracts/wrong-cardinality-form.json +1 -1
- package/corpus/dev/index.json +1 -1
- package/dist/application/index.d.ts +2 -0
- package/dist/application/index.js +1 -0
- package/dist/application/score.d.ts +48 -0
- package/dist/application/score.js +186 -0
- package/dist/cli/arguments.d.ts +7 -3
- package/dist/cli/arguments.js +73 -5
- package/dist/cli/exit-codes.d.ts +6 -2
- package/dist/cli/exit-codes.js +8 -9
- package/dist/cli/main.js +9 -0
- package/dist/cli/render.d.ts +1 -1
- package/dist/cli/render.js +6 -4
- package/dist/cli/run.d.ts +13 -3
- package/dist/cli/run.js +136 -16
- package/dist/core/canonical/canonicalize.d.ts +12 -0
- package/dist/core/canonical/canonicalize.js +18 -4
- package/dist/core/compile/bindings.d.ts +70 -0
- package/dist/core/compile/bindings.js +317 -0
- package/dist/core/compile/compile.d.ts +5 -0
- package/dist/core/compile/compile.js +9 -0
- package/dist/core/compile/expression-legality.d.ts +49 -0
- package/dist/core/compile/expression-legality.js +105 -31
- package/dist/core/compile/forbidden-inputs.d.ts +8 -1
- package/dist/core/compile/forbidden-inputs.js +16 -3
- package/dist/core/compile/interface-inventory.d.ts +23 -1
- package/dist/core/compile/interface-inventory.js +32 -3
- package/dist/core/compile/reachability.d.ts +18 -1
- package/dist/core/compile/reachability.js +46 -1
- package/dist/core/compile/sensitivity-witness.d.ts +1 -1
- package/dist/core/compile/sensitivity-witness.js +1 -1
- package/dist/core/emit/emit.d.ts +9 -0
- package/dist/core/emit/emit.js +214 -0
- package/dist/core/emit/private-artifact-digest.d.ts +11 -0
- package/dist/core/emit/private-artifact-digest.js +31 -0
- package/dist/core/evaluate/evidence-resolution.d.ts +11 -1
- package/dist/core/evaluate/evidence-resolution.js +4 -1
- package/dist/core/failure-codes.d.ts +2 -2
- package/dist/core/failure-codes.js +4 -2
- package/dist/core/ingest/conditions.d.ts +201 -0
- package/dist/core/ingest/conditions.js +48 -0
- package/dist/core/ingest/index.d.ts +13 -0
- package/dist/core/ingest/index.js +11 -0
- package/dist/core/ingest/ingest.d.ts +68 -0
- package/dist/core/ingest/ingest.js +310 -0
- package/dist/core/lineage/stage-table.d.ts +21 -4
- package/dist/core/lineage/stage-table.js +35 -3
- package/dist/core/preflight/witness-evidence.js +10 -0
- package/dist/core/schemas/artifact.d.ts +385 -14
- package/dist/core/schemas/constraint-ledger.js +40 -0
- package/dist/core/schemas/defect-signature.d.ts +273 -0
- package/dist/core/schemas/defect-signature.js +127 -0
- package/dist/core/schemas/eval-contract.d.ts +33 -0
- package/dist/core/schemas/eval-contract.js +14 -0
- package/dist/core/schemas/evidence-artifact.d.ts +99 -19
- package/dist/core/schemas/evidence-artifact.js +64 -8
- package/dist/core/schemas/plan.d.ts +112 -0
- package/dist/core/schemas/plan.js +67 -2
- package/dist/core/schemas/probe-qualification.d.ts +168 -0
- package/dist/core/schemas/probe-qualification.js +106 -0
- package/dist/core/schemas/probe.d.ts +312 -0
- package/dist/core/schemas/probe.js +6 -2
- package/dist/core/schemas/scoring-policy.d.ts +1 -0
- package/dist/core/schemas/scoring-policy.js +6 -1
- package/dist/core/schemas/sealed-evaluator-brief.d.ts +1 -0
- package/dist/core/schemas/sealed-evaluator-brief.js +4 -1
- package/dist/core/schemas/sealed-run-record.d.ts +30 -3
- package/dist/core/schemas/sealed-run-record.js +37 -5
- package/dist/core/score/binding-order.d.ts +31 -0
- package/dist/core/score/binding-order.js +88 -0
- package/dist/core/score/bindings.d.ts +93 -0
- package/dist/core/score/bindings.js +297 -0
- package/dist/core/score/ladder-table.d.ts +23 -0
- package/dist/core/score/ladder-table.js +109 -0
- package/dist/core/score/ladder.d.ts +229 -0
- package/dist/core/score/ladder.js +464 -0
- package/dist/core/score/mode-agreement.d.ts +30 -0
- package/dist/core/score/mode-agreement.js +16 -0
- package/dist/core/score/outcome-table.d.ts +17 -0
- package/dist/core/score/outcome-table.js +172 -0
- package/dist/core/score/outcome.d.ts +466 -0
- package/dist/core/score/outcome.js +562 -0
- package/dist/core/score/qualification.d.ts +83 -0
- package/dist/core/score/qualification.js +615 -0
- package/dist/core/score/quotation.d.ts +79 -0
- package/dist/core/score/quotation.js +201 -0
- package/dist/core/score/reduce-trials.d.ts +80 -0
- package/dist/core/score/reduce-trials.js +90 -0
- package/dist/core/score/score.d.ts +72 -0
- package/dist/core/score/score.js +581 -0
- package/dist/core/score/selection.d.ts +80 -0
- package/dist/core/score/selection.js +98 -0
- package/dist/core/score/strength.d.ts +49 -0
- package/dist/core/score/strength.js +193 -0
- package/dist/core/score/witness.d.ts +117 -0
- package/dist/core/score/witness.js +315 -0
- package/dist/core/seal/derived-reference.d.ts +1 -1
- package/dist/core/seal/derived-reference.js +236 -28
- package/dist/core/seal/seal.js +8 -3
- package/dist/core/stage-contracts.d.ts +62 -2
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/testing/index.d.ts +2 -0
- package/dist/testing/index.js +1 -0
- package/package.json +12 -3
- package/schemas/eval-contract.schema.json +100 -2
- package/schemas/evidence-artifact.schema.json +96 -9
- package/schemas/probe.schema.json +508 -3
- package/schemas/scoring-policy.schema.json +8 -1
- package/schemas/sealed-evaluator-brief.schema.json +11 -1
- package/schemas/sealed-run-record.schema.json +19 -2
package/dist/cli/arguments.js
CHANGED
|
@@ -4,24 +4,68 @@
|
|
|
4
4
|
* because every command is non-interactive. Paths are carried through exactly
|
|
5
5
|
* as given and resolved by the caller.
|
|
6
6
|
*/
|
|
7
|
-
export const COMMANDS = [
|
|
7
|
+
export const COMMANDS = [
|
|
8
|
+
'compile',
|
|
9
|
+
'seal',
|
|
10
|
+
'preflight',
|
|
11
|
+
'score',
|
|
12
|
+
];
|
|
8
13
|
/** The input flags each command takes, and which of them it requires. */
|
|
9
14
|
const INPUT_KEYS = {
|
|
10
15
|
compile: ['in'],
|
|
11
16
|
seal: ['in'],
|
|
12
17
|
preflight: ['contract', 'probes', 'observations'],
|
|
18
|
+
score: [
|
|
19
|
+
'record',
|
|
20
|
+
'isolation-manifest',
|
|
21
|
+
'evaluator-configuration',
|
|
22
|
+
'contract',
|
|
23
|
+
'probe',
|
|
24
|
+
'preflight-verdict',
|
|
25
|
+
'policy',
|
|
26
|
+
'private-manifest',
|
|
27
|
+
],
|
|
13
28
|
};
|
|
14
|
-
/**
|
|
29
|
+
/**
|
|
30
|
+
* Keys whose absence is legal even though the command declares them: `in`
|
|
31
|
+
* falls back to stdin, and `score`'s three name artifacts that are genuinely
|
|
32
|
+
* optional at the domain level (AD-16's absent isolation manifest, an absent
|
|
33
|
+
* evaluator configuration, and an optional private-manifest no stage row
|
|
34
|
+
* requires).
|
|
35
|
+
*/
|
|
36
|
+
const OPTIONAL_INPUT_KEYS = new Set([
|
|
37
|
+
'in',
|
|
38
|
+
'isolation-manifest',
|
|
39
|
+
'evaluator-configuration',
|
|
40
|
+
'private-manifest',
|
|
41
|
+
]);
|
|
42
|
+
/** `--strict-inputs` is AD-4's compile mode; `preflight` and `score` have no compile step. */
|
|
15
43
|
const TAKES_STRICT_INPUTS = {
|
|
16
44
|
compile: true,
|
|
17
45
|
seal: true,
|
|
18
46
|
preflight: false,
|
|
47
|
+
score: false,
|
|
19
48
|
};
|
|
20
|
-
/** `--run-id` names the run a verdict is minted for, which only `preflight` does. */
|
|
49
|
+
/** `--run-id` names the run a verdict is minted for, which only `preflight` does: `score` reads its run identifier off `--record` instead. */
|
|
21
50
|
const TAKES_RUN_ID = {
|
|
22
51
|
compile: false,
|
|
23
52
|
seal: false,
|
|
24
53
|
preflight: true,
|
|
54
|
+
score: false,
|
|
55
|
+
};
|
|
56
|
+
/** `--corpus-digest` supplies `score`'s one caller-attested scoring-version input with no artifact source; required whenever it applies, the same posture `--run-id` takes for `preflight`. */
|
|
57
|
+
const TAKES_CORPUS_DIGEST = {
|
|
58
|
+
compile: false,
|
|
59
|
+
seal: false,
|
|
60
|
+
preflight: false,
|
|
61
|
+
score: true,
|
|
62
|
+
};
|
|
63
|
+
/** `--corpus-root` names the directory `score`'s corpus-port adapter resolves a private reference under; always optional at this layer, since whether one is actually needed depends on the artifacts' own content, not the argument grammar. */
|
|
64
|
+
const TAKES_CORPUS_ROOT = {
|
|
65
|
+
compile: false,
|
|
66
|
+
seal: false,
|
|
67
|
+
preflight: false,
|
|
68
|
+
score: true,
|
|
25
69
|
};
|
|
26
70
|
const STDIN = '-';
|
|
27
71
|
const isCommand = (token) => COMMANDS.includes(token);
|
|
@@ -63,10 +107,17 @@ function parseCommand(command, rest) {
|
|
|
63
107
|
valueFlags.set('--out', 'out');
|
|
64
108
|
if (TAKES_RUN_ID[command])
|
|
65
109
|
valueFlags.set('--run-id', 'run-id');
|
|
110
|
+
if (TAKES_CORPUS_DIGEST[command]) {
|
|
111
|
+
valueFlags.set('--corpus-digest', 'corpus-digest');
|
|
112
|
+
}
|
|
113
|
+
if (TAKES_CORPUS_ROOT[command])
|
|
114
|
+
valueFlags.set('--corpus-root', 'corpus-root');
|
|
66
115
|
const inputs = {};
|
|
67
116
|
const seen = new Map();
|
|
68
117
|
let out = null;
|
|
69
118
|
let runId = null;
|
|
119
|
+
let corpusDigest = null;
|
|
120
|
+
let corpusRoot = null;
|
|
70
121
|
let strictInputs = true;
|
|
71
122
|
let strict = false;
|
|
72
123
|
const takeValue = (flag, inline, index) => {
|
|
@@ -114,6 +165,10 @@ function parseCommand(command, rest) {
|
|
|
114
165
|
out = taken.value;
|
|
115
166
|
else if (target === 'run-id')
|
|
116
167
|
runId = taken.value;
|
|
168
|
+
else if (target === 'corpus-digest')
|
|
169
|
+
corpusDigest = taken.value;
|
|
170
|
+
else if (target === 'corpus-root')
|
|
171
|
+
corpusRoot = taken.value;
|
|
117
172
|
else
|
|
118
173
|
inputs[target] = taken.value;
|
|
119
174
|
index = taken.next;
|
|
@@ -140,10 +195,13 @@ function parseCommand(command, rest) {
|
|
|
140
195
|
return usageError(`unknown flag "${token}" for ${command}`);
|
|
141
196
|
}
|
|
142
197
|
const missing = INPUT_KEYS[command]
|
|
143
|
-
.filter((key) => key
|
|
198
|
+
.filter((key) => !OPTIONAL_INPUT_KEYS.has(key) && inputs[key] === undefined)
|
|
144
199
|
.map((key) => `--${key}`);
|
|
145
200
|
if (TAKES_RUN_ID[command] && runId === null)
|
|
146
201
|
missing.push('--run-id');
|
|
202
|
+
if (TAKES_CORPUS_DIGEST[command] && corpusDigest === null) {
|
|
203
|
+
missing.push('--corpus-digest');
|
|
204
|
+
}
|
|
147
205
|
// `PreflightVerdict.runId` is `z.string().min(1)`, which a run of spaces
|
|
148
206
|
// satisfies. It then renders as blank in every diagnostic line and names
|
|
149
207
|
// nothing a reader can correlate, so the argument surface refuses it here
|
|
@@ -163,5 +221,15 @@ function parseCommand(command, rest) {
|
|
|
163
221
|
: `${flags.join(', ')} all name`;
|
|
164
222
|
return usageError(`only one input may read stdin, but ${named} "-"`);
|
|
165
223
|
}
|
|
166
|
-
return {
|
|
224
|
+
return {
|
|
225
|
+
kind: 'run',
|
|
226
|
+
command,
|
|
227
|
+
inputs,
|
|
228
|
+
out,
|
|
229
|
+
runId,
|
|
230
|
+
corpusDigest,
|
|
231
|
+
corpusRoot,
|
|
232
|
+
strictInputs,
|
|
233
|
+
strict,
|
|
234
|
+
};
|
|
167
235
|
}
|
package/dist/cli/exit-codes.d.ts
CHANGED
|
@@ -22,8 +22,12 @@ export type CommandOutcome = {
|
|
|
22
22
|
readonly passed: boolean;
|
|
23
23
|
} | {
|
|
24
24
|
readonly kind: 'verdict';
|
|
25
|
-
|
|
26
|
-
readonly
|
|
25
|
+
/** `null` is AD-21's Invalid rung: `LadderResolution.verdict` on that rung, never a `Verdict`. */
|
|
26
|
+
readonly verdict: Verdict | null;
|
|
27
|
+
/** `LadderResolution.exitCode`, read directly rather than recomputed from `verdict`. */
|
|
28
|
+
readonly exitCode: number;
|
|
29
|
+
/** `LadderResolution.strictPromotable`, carried through unchanged: never inverted into a locally-invented `evidenceConditionsOnly`. */
|
|
30
|
+
readonly strictPromotable: boolean;
|
|
27
31
|
} | {
|
|
28
32
|
readonly kind: 'structural-failure';
|
|
29
33
|
} | {
|
package/dist/cli/exit-codes.js
CHANGED
|
@@ -7,17 +7,16 @@ export const EXIT_FAULT = 5;
|
|
|
7
7
|
/** sysexits.h EX_USAGE. Outside the verdict range and outside AD-21's codes. */
|
|
8
8
|
export const EXIT_USAGE = 64;
|
|
9
9
|
function verdictExit(outcome, strict) {
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
//
|
|
13
|
-
//
|
|
14
|
-
//
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
!outcome.evidenceConditionsOnly) {
|
|
10
|
+
// `--strict` promotes a CONCERNS to one, "except a CONCERNS whose only
|
|
11
|
+
// firing conditions are evidence conditions, which `--strict` never
|
|
12
|
+
// promotes" (AD-21). `LADDER_EXIT_CODES.CONCERNS` is already zero, so this
|
|
13
|
+
// promotion cannot be read out of `outcome.exitCode` itself: it is applied
|
|
14
|
+
// on top of it. Every other rung -- PASS, WAIVED, FAIL, and Invalid's
|
|
15
|
+
// `null` -- returns its own `exitCode` unchanged.
|
|
16
|
+
if (outcome.verdict === 'CONCERNS' && strict && outcome.strictPromotable) {
|
|
18
17
|
return EXIT_CONCERNS_PROMOTED;
|
|
19
18
|
}
|
|
20
|
-
return
|
|
19
|
+
return outcome.exitCode;
|
|
21
20
|
}
|
|
22
21
|
export function exitCodeFor(outcome, options) {
|
|
23
22
|
switch (outcome.kind) {
|
package/dist/cli/main.js
CHANGED
|
@@ -11,6 +11,7 @@
|
|
|
11
11
|
import { mkdir, readFile, stat, writeFile } from 'node:fs/promises';
|
|
12
12
|
import { dirname, join, resolve } from 'node:path';
|
|
13
13
|
import process from 'node:process';
|
|
14
|
+
import { createLocalCorpusAdapter } from '../adapters/local-corpus-adapter.js';
|
|
14
15
|
import { parseArguments } from './arguments.js';
|
|
15
16
|
import { exitCodeFor } from './exit-codes.js';
|
|
16
17
|
import { renderError, renderUsage } from './render.js';
|
|
@@ -88,6 +89,14 @@ function environmentOf(version) {
|
|
|
88
89
|
return false;
|
|
89
90
|
}
|
|
90
91
|
},
|
|
92
|
+
// `score`'s corpus-port wiring: the one reference adapter, over a root
|
|
93
|
+
// known only once `--corpus-root` is parsed, which is why this is a
|
|
94
|
+
// factory rather than a constructed port.
|
|
95
|
+
corpusPort: (root) => createLocalCorpusAdapter({ root }),
|
|
96
|
+
// One controller per process invocation, never aborted here: no flag
|
|
97
|
+
// this binary parses requests cancellation, so the signal only ever
|
|
98
|
+
// answers "was this already aborted before the call" as false.
|
|
99
|
+
signal: new AbortController().signal,
|
|
91
100
|
version,
|
|
92
101
|
};
|
|
93
102
|
}
|
package/dist/cli/render.d.ts
CHANGED
|
@@ -21,6 +21,6 @@ export declare function renderError(error: unknown): string;
|
|
|
21
21
|
* AD-21's seven exit codes, one line each. The `--help` output and the README
|
|
22
22
|
* table are this text, so the two cannot drift.
|
|
23
23
|
*/
|
|
24
|
-
export declare const EXIT_CODE_TABLE = "Exit codes (AD-21):\n 0 success, and every verdict other than FAIL or a promoted CONCERNS\n 1 CONCERNS promoted by --strict\n 2 FAIL\n 3 invalid: a pre-flight
|
|
24
|
+
export declare const EXIT_CODE_TABLE = "Exit codes (AD-21):\n 0 success, and every verdict other than FAIL or a promoted CONCERNS\n 1 CONCERNS promoted by --strict\n 2 FAIL\n 3 invalid: a failed pre-flight, or any other AD-21 invalidating condition\n 4 structural failure\n 5 runtime fault\n 64 usage error\n\n --strict never promotes a CONCERNS whose firing conditions are all evidence\n conditions: those conditions report that the measurement fell short of the\n policy. 1 and 2 report a verdict the score command's ladder resolved; every\n other invalidating condition behind 3 is reachable there too, alongside the\n failed pre-flight the preflight command itself reports.";
|
|
25
25
|
/** `eval-quality: usage: <message>` */
|
|
26
26
|
export declare function renderUsage(message: string): string;
|
package/dist/cli/render.js
CHANGED
|
@@ -39,14 +39,16 @@ export const EXIT_CODE_TABLE = `Exit codes (AD-21):
|
|
|
39
39
|
0 success, and every verdict other than FAIL or a promoted CONCERNS
|
|
40
40
|
1 CONCERNS promoted by --strict
|
|
41
41
|
2 FAIL
|
|
42
|
-
3 invalid: a pre-flight
|
|
42
|
+
3 invalid: a failed pre-flight, or any other AD-21 invalidating condition
|
|
43
43
|
4 structural failure
|
|
44
44
|
5 runtime fault
|
|
45
45
|
64 usage error
|
|
46
46
|
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
47
|
+
--strict never promotes a CONCERNS whose firing conditions are all evidence
|
|
48
|
+
conditions: those conditions report that the measurement fell short of the
|
|
49
|
+
policy. 1 and 2 report a verdict the score command's ladder resolved; every
|
|
50
|
+
other invalidating condition behind 3 is reachable there too, alongside the
|
|
51
|
+
failed pre-flight the preflight command itself reports.`;
|
|
50
52
|
/** `eval-quality: usage: <message>` */
|
|
51
53
|
export function renderUsage(message) {
|
|
52
54
|
return `${PREFIX}: usage: ${message}`;
|
package/dist/cli/run.d.ts
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* The
|
|
2
|
+
* The four commands. Each one reads its inputs, makes exactly one call into
|
|
3
3
|
* `application/`, serializes what came back, and returns the outcome the exit
|
|
4
4
|
* code is derived from. Every effect is a member of `RunEnvironment`, so this
|
|
5
5
|
* module touches no stream, no `process`, and no Node builtin, and its tests
|
|
6
6
|
* run in memory.
|
|
7
7
|
*/
|
|
8
|
-
import { compile, preflightFromObservations, seal } from '../application/index.ts';
|
|
8
|
+
import { compile, preflightFromObservations, type RunScoreOptions, runScore, seal } from '../application/index.ts';
|
|
9
9
|
import type { Command, ParsedInvocation } from './arguments.ts';
|
|
10
10
|
import type { CommandOutcome } from './exit-codes.ts';
|
|
11
11
|
export type RunEnvironment = {
|
|
@@ -21,10 +21,19 @@ export type RunEnvironment = {
|
|
|
21
21
|
* spellings that no amount of folding brings together.
|
|
22
22
|
*/
|
|
23
23
|
readonly sameFile: (left: string, right: string) => Promise<boolean>;
|
|
24
|
+
/**
|
|
25
|
+
* `score`'s corpus-port factory over a caller-named root directory: a
|
|
26
|
+
* constructor, not a constructed value, since the root is only known once
|
|
27
|
+
* `--corpus-root` is parsed. `cli/` may not import `ports/` directly, so
|
|
28
|
+
* the port's own type is read off `RunScoreOptions` instead.
|
|
29
|
+
*/
|
|
30
|
+
readonly corpusPort: (root: string) => NonNullable<RunScoreOptions['port']>;
|
|
31
|
+
/** One per process invocation, the same shape `RunPreflightOptions.signal` already declares. */
|
|
32
|
+
readonly signal: AbortSignal;
|
|
24
33
|
readonly version: string;
|
|
25
34
|
};
|
|
26
35
|
/**
|
|
27
|
-
* The
|
|
36
|
+
* The four orchestration calls, behind one object so a test can count them.
|
|
28
37
|
* An input/output count says nothing about how many calls into `application/`
|
|
29
38
|
* happened, which is the property AD-14 constrains.
|
|
30
39
|
*/
|
|
@@ -32,6 +41,7 @@ export type ApplicationFacade = {
|
|
|
32
41
|
readonly compile: typeof compile;
|
|
33
42
|
readonly seal: typeof seal;
|
|
34
43
|
readonly preflightFromObservations: typeof preflightFromObservations;
|
|
44
|
+
readonly runScore: typeof runScore;
|
|
35
45
|
};
|
|
36
46
|
export declare const APPLICATION: ApplicationFacade;
|
|
37
47
|
export type RunResult = {
|
package/dist/cli/run.js
CHANGED
|
@@ -1,16 +1,17 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* The
|
|
2
|
+
* The four commands. Each one reads its inputs, makes exactly one call into
|
|
3
3
|
* `application/`, serializes what came back, and returns the outcome the exit
|
|
4
4
|
* code is derived from. Every effect is a member of `RunEnvironment`, so this
|
|
5
5
|
* module touches no stream, no `process`, and no Node builtin, and its tests
|
|
6
6
|
* run in memory.
|
|
7
7
|
*/
|
|
8
|
-
import { compile, preflightFromObservations, RuntimeFault, StructuralFailure, seal, } from '../application/index.js';
|
|
8
|
+
import { compile, preflightFromObservations, RuntimeFault, runScore, StructuralFailure, seal, } from '../application/index.js';
|
|
9
9
|
import { EXIT_CODE_TABLE, renderArtifact, renderDiagnostic, renderError, renderUsage, } from './render.js';
|
|
10
10
|
export const APPLICATION = {
|
|
11
11
|
compile,
|
|
12
12
|
seal,
|
|
13
13
|
preflightFromObservations,
|
|
14
|
+
runScore,
|
|
14
15
|
};
|
|
15
16
|
/** The artifact each command emits: its schema name and its file name. */
|
|
16
17
|
const EMITTED = {
|
|
@@ -20,6 +21,7 @@ const EMITTED = {
|
|
|
20
21
|
kind: 'sealed-evaluator-brief',
|
|
21
22
|
},
|
|
22
23
|
preflight: { artifactPath: 'PreflightVerdict', kind: 'preflight-verdict' },
|
|
24
|
+
score: { artifactPath: 'EvidenceArtifact', kind: 'evidence-artifact' },
|
|
23
25
|
};
|
|
24
26
|
/** The schema an input key deserializes into, for the parse fault's path. */
|
|
25
27
|
const INPUT_ARTIFACT_PATH = {
|
|
@@ -27,6 +29,13 @@ const INPUT_ARTIFACT_PATH = {
|
|
|
27
29
|
contract: 'EvalContract',
|
|
28
30
|
probes: 'Probe',
|
|
29
31
|
observations: 'ProbeObservation',
|
|
32
|
+
record: 'SealedRunRecord',
|
|
33
|
+
'isolation-manifest': 'IsolationManifest',
|
|
34
|
+
'evaluator-configuration': 'EvaluatorConfiguration',
|
|
35
|
+
probe: 'Probe',
|
|
36
|
+
'preflight-verdict': 'PreflightVerdict',
|
|
37
|
+
policy: 'ScoringPolicy',
|
|
38
|
+
'private-manifest': 'PrivateArtifactManifest',
|
|
30
39
|
};
|
|
31
40
|
const USAGE = `Usage:
|
|
32
41
|
eval-quality compile [--in <path>] [--out <target>]
|
|
@@ -35,6 +44,12 @@ const USAGE = `Usage:
|
|
|
35
44
|
[--strict-inputs | --no-strict-inputs] [--strict]
|
|
36
45
|
eval-quality preflight --contract <path> --probes <path> --observations <path>
|
|
37
46
|
--run-id <id> [--out <target>] [--strict]
|
|
47
|
+
eval-quality score --record <path> --contract <path> --probe <path>
|
|
48
|
+
--preflight-verdict <path> --policy <path>
|
|
49
|
+
--corpus-digest <digest>
|
|
50
|
+
[--isolation-manifest <path>] [--evaluator-configuration <path>]
|
|
51
|
+
[--private-manifest <path>] [--corpus-root <dir>]
|
|
52
|
+
[--out <target>] [--strict]
|
|
38
53
|
eval-quality --help | -h | help [<command>]
|
|
39
54
|
eval-quality --version | -V`;
|
|
40
55
|
const COMMAND_USAGE = {
|
|
@@ -66,10 +81,36 @@ const COMMAND_USAGE = {
|
|
|
66
81
|
--run-id <id> the run identifier the verdict is minted for
|
|
67
82
|
--out <target> a .json file path, or a directory taking preflight-verdict.json
|
|
68
83
|
--strict promote CONCERNS to exit 1`,
|
|
84
|
+
score: `Usage:
|
|
85
|
+
eval-quality score --record <path> --contract <path> --probe <path>
|
|
86
|
+
--preflight-verdict <path> --policy <path>
|
|
87
|
+
--corpus-digest <digest>
|
|
88
|
+
[--isolation-manifest <path>] [--evaluator-configuration <path>]
|
|
89
|
+
[--private-manifest <path>] [--corpus-root <dir>]
|
|
90
|
+
[--out <target>] [--strict]
|
|
91
|
+
|
|
92
|
+
--record <path> the sealed run record to ingest
|
|
93
|
+
--contract <path> the compiled contract to score against
|
|
94
|
+
--probe <path> the probe the record was run against
|
|
95
|
+
--preflight-verdict <path> the pre-flight verdict, also the source of the AD-11 fixture digest
|
|
96
|
+
--policy <path> the scoring policy
|
|
97
|
+
--corpus-digest <digest> AD-11's caller-attested corpus digest; no artifact carries it
|
|
98
|
+
--isolation-manifest <path> the isolation manifest; absent invalidates the run under AD-16
|
|
99
|
+
--evaluator-configuration <path> the evaluator configuration; absent invalidates the run
|
|
100
|
+
--private-manifest <path> each entry's digest is checked against its resolved bytes
|
|
101
|
+
--corpus-root <dir> the directory a private reference resolves under; required only
|
|
102
|
+
when --private-manifest or a private-storage isolation-manifest
|
|
103
|
+
reference is present
|
|
104
|
+
--out <target> a .json file path, or a directory taking evidence-artifact.json
|
|
105
|
+
--strict promote CONCERNS to exit 1`,
|
|
69
106
|
};
|
|
70
107
|
const IO_RULES = `Inputs and outputs:
|
|
71
|
-
|
|
72
|
-
|
|
108
|
+
--in is the only input that falls back to stdin: compile and seal read it
|
|
109
|
+
when --in is left out. "-" names stdin explicitly on any input, and at most
|
|
110
|
+
one input may be "-" per invocation. compile and seal each take one input;
|
|
111
|
+
preflight takes three, all required; score takes eight, three of them
|
|
112
|
+
optional (--isolation-manifest, --evaluator-configuration, and
|
|
113
|
+
--private-manifest). Without --out the artifact goes to stdout. An --out
|
|
73
114
|
ending in .json is a file path; anything else is a directory taking
|
|
74
115
|
<target>/<kind>.json. Diagnostics and errors go to stderr.`;
|
|
75
116
|
export function helpText(command) {
|
|
@@ -155,19 +196,32 @@ async function runCommand(invocation, environment, application) {
|
|
|
155
196
|
}
|
|
156
197
|
}
|
|
157
198
|
try {
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
199
|
+
// Exhaustive over `Command`, not a fallthrough: a fifth command with no
|
|
200
|
+
// case here is a compile error ("not all code paths return a value"),
|
|
201
|
+
// never a silent seal. `compile` and `seal` share one case body, and the
|
|
202
|
+
// ternary inside it is itself exhaustive over the narrowed two-member
|
|
203
|
+
// union the shared case leaves `command` as.
|
|
204
|
+
switch (command) {
|
|
205
|
+
case 'preflight': {
|
|
206
|
+
const verdict = await runPreflightCommand(invocation, environment, application, target);
|
|
207
|
+
return { outcome: { kind: 'preflight', passed: verdict.passed } };
|
|
208
|
+
}
|
|
209
|
+
case 'score':
|
|
210
|
+
return await runScoreCommand(invocation, environment, application, target);
|
|
211
|
+
case 'compile':
|
|
212
|
+
case 'seal': {
|
|
213
|
+
const input = await readJson(environment, 'in', inputs.in);
|
|
214
|
+
const options = { strict: strictInputs };
|
|
215
|
+
const artifact = command === 'compile'
|
|
216
|
+
? application.compile(input, options)
|
|
217
|
+
: application.seal(input, options);
|
|
218
|
+
// No diagnostic on success: neither command carries a run
|
|
219
|
+
// identifier, so any line either wrote would name no run and no
|
|
220
|
+
// stage.
|
|
221
|
+
await emitArtifact(environment, artifact, command, target);
|
|
222
|
+
return { outcome: { kind: 'artifact' } };
|
|
223
|
+
}
|
|
161
224
|
}
|
|
162
|
-
const input = await readJson(environment, 'in', inputs.in);
|
|
163
|
-
const options = { strict: strictInputs };
|
|
164
|
-
const artifact = command === 'compile'
|
|
165
|
-
? application.compile(input, options)
|
|
166
|
-
: application.seal(input, options);
|
|
167
|
-
// No diagnostic on success: neither command carries a run identifier,
|
|
168
|
-
// so any line either wrote would name no run and no stage.
|
|
169
|
-
await emitArtifact(environment, artifact, command, target);
|
|
170
|
-
return { outcome: { kind: 'artifact' } };
|
|
171
225
|
}
|
|
172
226
|
catch (error) {
|
|
173
227
|
if (error instanceof StructuralFailure) {
|
|
@@ -201,3 +255,69 @@ async function runPreflightCommand(invocation, environment, application, target)
|
|
|
201
255
|
await emitArtifact(environment, verdict, 'preflight', target);
|
|
202
256
|
return verdict;
|
|
203
257
|
}
|
|
258
|
+
/** `undefined` means the optional flag was not given; unlike `'in'`, none of `score`'s three optional inputs falls back to stdin. */
|
|
259
|
+
async function readOptionalJson(environment, key, value) {
|
|
260
|
+
return value === undefined ? null : await readJson(environment, key, value);
|
|
261
|
+
}
|
|
262
|
+
/**
|
|
263
|
+
* Whether `score` needs a `CorpusPort` at all: a `--private-manifest` with at
|
|
264
|
+
* least one entry, or a private-storage `isolationManifestArtifact`. Read off
|
|
265
|
+
* unvalidated JSON with optional chaining throughout -- a malformed shape
|
|
266
|
+
* here simply reads `undefined` and falls through to `application.runScore`'s
|
|
267
|
+
* own real parse, which raises the accurate `schema-parse-failure` rather
|
|
268
|
+
* than this pre-check inventing one.
|
|
269
|
+
*/
|
|
270
|
+
function needsCorpusPort(record, privateManifest) {
|
|
271
|
+
const manifest = privateManifest;
|
|
272
|
+
if (manifest !== null && (manifest.entries?.length ?? 0) > 0)
|
|
273
|
+
return true;
|
|
274
|
+
const typedRecord = record;
|
|
275
|
+
return typedRecord?.isolationManifestArtifact?.storage === 'private';
|
|
276
|
+
}
|
|
277
|
+
/** `CommandOutcome`'s `'verdict'` kind, read straight off `LadderResolution`: no inversion, no recomputation. */
|
|
278
|
+
function scoreOutcomeOf(ladder) {
|
|
279
|
+
return {
|
|
280
|
+
kind: 'verdict',
|
|
281
|
+
verdict: ladder.verdict,
|
|
282
|
+
exitCode: ladder.exitCode,
|
|
283
|
+
strictPromotable: ladder.strictPromotable,
|
|
284
|
+
};
|
|
285
|
+
}
|
|
286
|
+
async function runScoreCommand(invocation, environment, application, target) {
|
|
287
|
+
const { inputs, corpusDigest, corpusRoot } = invocation;
|
|
288
|
+
const record = (await readJson(environment, 'record', inputs.record));
|
|
289
|
+
const manifest = (await readOptionalJson(environment, 'isolation-manifest', inputs['isolation-manifest']));
|
|
290
|
+
const configuration = (await readOptionalJson(environment, 'evaluator-configuration', inputs['evaluator-configuration']));
|
|
291
|
+
const contract = (await readJson(environment, 'contract', inputs.contract));
|
|
292
|
+
const probe = (await readJson(environment, 'probe', inputs.probe));
|
|
293
|
+
const preflightVerdict = (await readJson(environment, 'preflight-verdict', inputs['preflight-verdict']));
|
|
294
|
+
const policy = (await readJson(environment, 'policy', inputs.policy));
|
|
295
|
+
const privateManifest = (await readOptionalJson(environment, 'private-manifest', inputs['private-manifest']));
|
|
296
|
+
// A private reference with no `--corpus-root` to resolve it under is a
|
|
297
|
+
// usage error naming the missing flag, not a silently skipped check.
|
|
298
|
+
// `application.runScore` never returns `usage-error` itself (that
|
|
299
|
+
// vocabulary is `cli/`'s alone), so this is checked here, before the call.
|
|
300
|
+
if (corpusRoot === null && needsCorpusPort(record, privateManifest)) {
|
|
301
|
+
environment.writeDiagnostic(renderUsage('--corpus-root is required to resolve a --private-manifest entry or a private-storage isolationManifestArtifact reference'));
|
|
302
|
+
return { outcome: { kind: 'usage-error' } };
|
|
303
|
+
}
|
|
304
|
+
const result = await application.runScore({
|
|
305
|
+
record,
|
|
306
|
+
manifest,
|
|
307
|
+
configuration,
|
|
308
|
+
contract,
|
|
309
|
+
probe,
|
|
310
|
+
preflightVerdict,
|
|
311
|
+
policy,
|
|
312
|
+
privateManifest,
|
|
313
|
+
// The parser requires `--corpus-digest` on this command, so it is
|
|
314
|
+
// never null.
|
|
315
|
+
corpusDigest: corpusDigest ?? '',
|
|
316
|
+
port: corpusRoot === null ? undefined : environment.corpusPort(corpusRoot),
|
|
317
|
+
signal: environment.signal,
|
|
318
|
+
});
|
|
319
|
+
if (result.artifact !== null) {
|
|
320
|
+
await emitArtifact(environment, result.artifact, 'score', target);
|
|
321
|
+
}
|
|
322
|
+
return { outcome: scoreOutcomeOf(result.ladder) };
|
|
323
|
+
}
|
|
@@ -1 +1,13 @@
|
|
|
1
1
|
export declare function canonicalize(value: unknown, artifactPath: string): Uint8Array;
|
|
2
|
+
/**
|
|
3
|
+
* The RFC 8785 string form, exported because AD-40's quotation audit projects a
|
|
4
|
+
* structured evidence channel to text and asks for the quoted evidence as an
|
|
5
|
+
* exact substring of it. `canonicalize` returns a `Uint8Array` and `core/` may
|
|
6
|
+
* import no Node builtin, so decoding those bytes back is not available; the
|
|
7
|
+
* string is what already exists one call in.
|
|
8
|
+
*
|
|
9
|
+
* Two parameters, the same two `canonicalize` takes. The traversal state is
|
|
10
|
+
* seeded here and reachable nowhere else, so no caller can hand in a primed
|
|
11
|
+
* ancestor set or a depth that starts the AD-36 nesting guard part-spent.
|
|
12
|
+
*/
|
|
13
|
+
export declare function serialize(value: unknown, artifactPath: string): string;
|
|
@@ -11,12 +11,26 @@ import { assertDomainNumber, assertDomainString, MAX_NESTING_DEPTH, } from './va
|
|
|
11
11
|
// lying Proxy could answer the emit-time read differently than the
|
|
12
12
|
// validation read.
|
|
13
13
|
export function canonicalize(value, artifactPath) {
|
|
14
|
-
return new TextEncoder().encode(serialize(value, artifactPath
|
|
14
|
+
return new TextEncoder().encode(serialize(value, artifactPath));
|
|
15
15
|
}
|
|
16
16
|
const fault = (artifactPath, detail) => {
|
|
17
17
|
throw new RuntimeFault('non-canonicalizable-value', artifactPath, detail);
|
|
18
18
|
};
|
|
19
|
-
|
|
19
|
+
/**
|
|
20
|
+
* The RFC 8785 string form, exported because AD-40's quotation audit projects a
|
|
21
|
+
* structured evidence channel to text and asks for the quoted evidence as an
|
|
22
|
+
* exact substring of it. `canonicalize` returns a `Uint8Array` and `core/` may
|
|
23
|
+
* import no Node builtin, so decoding those bytes back is not available; the
|
|
24
|
+
* string is what already exists one call in.
|
|
25
|
+
*
|
|
26
|
+
* Two parameters, the same two `canonicalize` takes. The traversal state is
|
|
27
|
+
* seeded here and reachable nowhere else, so no caller can hand in a primed
|
|
28
|
+
* ancestor set or a depth that starts the AD-36 nesting guard part-spent.
|
|
29
|
+
*/
|
|
30
|
+
export function serialize(value, artifactPath) {
|
|
31
|
+
return serializeNode(value, artifactPath, '$', new Set(), 0);
|
|
32
|
+
}
|
|
33
|
+
function serializeNode(value, artifactPath, location, ancestors, depth) {
|
|
20
34
|
if (value === null)
|
|
21
35
|
return 'null';
|
|
22
36
|
switch (typeof value) {
|
|
@@ -75,7 +89,7 @@ function serialize(value, artifactPath, location, ancestors, depth) {
|
|
|
75
89
|
fault(artifactPath, `array with holes or non-index properties at ${location}`);
|
|
76
90
|
}
|
|
77
91
|
rendered = `[${elements
|
|
78
|
-
.map((element, index) =>
|
|
92
|
+
.map((element, index) => serializeNode(element, artifactPath, `${location}[${index}]`, ancestors, depth + 1))
|
|
79
93
|
.join(',')}]`;
|
|
80
94
|
}
|
|
81
95
|
else {
|
|
@@ -105,7 +119,7 @@ function serialize(value, artifactPath, location, ancestors, depth) {
|
|
|
105
119
|
// never localeCompare, never Intl, never .normalize().
|
|
106
120
|
properties.sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0));
|
|
107
121
|
rendered = `{${properties
|
|
108
|
-
.map(([key, entry]) => `${JSON.stringify(key)}:${
|
|
122
|
+
.map(([key, entry]) => `${JSON.stringify(key)}:${serializeNode(entry, artifactPath, `${location}.${key}`, ancestors, depth + 1)}`)
|
|
109
123
|
.join(',')}}`;
|
|
110
124
|
}
|
|
111
125
|
ancestors.delete(object);
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
import type { EvalContract } from '../schemas/eval-contract.ts';
|
|
2
|
+
import type { InteractionStep } from '../schemas/plan.ts';
|
|
3
|
+
import { type TransportChannelName } from '../schemas/pointer.ts';
|
|
4
|
+
import { type EvidenceTarget } from '../seal/plan-index.ts';
|
|
5
|
+
/** One `{ captured }` binding, resolved to the pointer target it addresses. */
|
|
6
|
+
export type CapturedBinding = {
|
|
7
|
+
readonly transportChannel: TransportChannelName;
|
|
8
|
+
readonly key: string;
|
|
9
|
+
readonly pointer: string;
|
|
10
|
+
readonly target: EvidenceTarget;
|
|
11
|
+
};
|
|
12
|
+
/**
|
|
13
|
+
* Every captured binding one step declares, in fixed transport-channel order
|
|
14
|
+
* then by key name, so which binding a check reports never depends on a
|
|
15
|
+
* caller-keyed map's insertion order. Exported because `score/binding-order.ts`
|
|
16
|
+
* and `score/bindings.ts` need the same reading of which bindings are captures.
|
|
17
|
+
*/
|
|
18
|
+
export declare function capturedBindings(step: InteractionStep): readonly CapturedBinding[];
|
|
19
|
+
/**
|
|
20
|
+
* `binding-cycle`: a cycle over the union of the capture edges and AD-39's
|
|
21
|
+
* `after` edges that contains at least one capture edge.
|
|
22
|
+
*
|
|
23
|
+
* Both edge kinds assert that this step's observation comes after that one, so
|
|
24
|
+
* a capture B to A together with `A.after = B` is an unsatisfiable ordering
|
|
25
|
+
* while each graph alone stays acyclic. A cycle made only of `after` edges is
|
|
26
|
+
* left to `checkNestedTemporalClause`, which already catches every one of them,
|
|
27
|
+
* so requiring a capture edge in the cycle keeps the two codes disjoint.
|
|
28
|
+
*
|
|
29
|
+
* Decided by strongly connected components rather than by a depth-first walk
|
|
30
|
+
* that inspects each back edge. A cycle lies entirely inside one component, and
|
|
31
|
+
* a capture edge whose endpoints share a component is closed by a path back
|
|
32
|
+
* through that component, so "some cycle contains a capture edge" is exactly
|
|
33
|
+
* "some capture edge has both endpoints in one component". A self-capture is
|
|
34
|
+
* the singleton case and needs no rule of its own. The depth-first form was
|
|
35
|
+
* written first and rejected: settling a node after rejecting a pure-`after`
|
|
36
|
+
* back edge can hide a mixed cycle a later root would reach, and whether it
|
|
37
|
+
* does depends on the order edges happen to be enumerated in, which is an
|
|
38
|
+
* invariant nothing states.
|
|
39
|
+
*
|
|
40
|
+
* Nodes are the plan's declared step ids; an edge whose target no step declares
|
|
41
|
+
* is dropped, since a dangling reference is not a cycle and AD-39 makes a
|
|
42
|
+
* dangling `after` permissive besides. The binding reported is the first, in
|
|
43
|
+
* the plan's own declaration order, whose endpoints share a component.
|
|
44
|
+
*/
|
|
45
|
+
export declare function checkBindingCycle(contract: EvalContract): void;
|
|
46
|
+
/**
|
|
47
|
+
* `captured-channel-undeclared`: a captured pointer naming any AD-26 channel
|
|
48
|
+
* but `response-body`. `call-inputs` addresses a step's own request,
|
|
49
|
+
* `stdout`, `stderr`, and `exit-code` are process channels no operation
|
|
50
|
+
* surviving `unsupported-interface-kind` produces, and `response-headers` and
|
|
51
|
+
* `response-status` have no declared structure to give a captured value a
|
|
52
|
+
* type.
|
|
53
|
+
*/
|
|
54
|
+
export declare function checkCapturedChannel(contract: EvalContract): void;
|
|
55
|
+
/**
|
|
56
|
+
* `unreachable-check-evidence` over the captured-pointer residue: an
|
|
57
|
+
* unresolvable step or operation, an undeclared body key, a tail that is not
|
|
58
|
+
* exactly one segment, an array index, an absent, indeterminate, or non-scalar
|
|
59
|
+
* declared type, and a declared type that does not equal the bound
|
|
60
|
+
* parameter's.
|
|
61
|
+
*
|
|
62
|
+
* A capture naming another channel is reported here only for the two conditions
|
|
63
|
+
* that are decidable without a response descriptor: an unresolvable step and an
|
|
64
|
+
* unresolvable operation. Everything else about such a pointer is
|
|
65
|
+
* `checkCapturedChannel`'s. That split is what lets this check run at the
|
|
66
|
+
* registry's third rung without shadowing the fifteenth, and it keeps the
|
|
67
|
+
* higher-ranked code first on a pointer that is unresolvable and off-body at
|
|
68
|
+
* once.
|
|
69
|
+
*/
|
|
70
|
+
export declare function checkCapturedReachability(contract: EvalContract): void;
|