@testmuai/evidence-cli 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -0
- package/README.md +124 -0
- package/dist/cli.d.ts +2 -0
- package/dist/cli.js +55 -0
- package/dist/cli.js.map +1 -0
- package/dist/config.d.ts +6 -0
- package/dist/config.js +71 -0
- package/dist/config.js.map +1 -0
- package/dist/contract.d.ts +66 -0
- package/dist/contract.js +50 -0
- package/dist/contract.js.map +1 -0
- package/dist/diagnostics.d.ts +3 -0
- package/dist/diagnostics.js +11 -0
- package/dist/diagnostics.js.map +1 -0
- package/dist/finalize/index.d.ts +21 -0
- package/dist/finalize/index.js +196 -0
- package/dist/finalize/index.js.map +1 -0
- package/dist/index.d.ts +12 -0
- package/dist/index.js +37 -0
- package/dist/index.js.map +1 -0
- package/dist/pack/byte-source.d.ts +33 -0
- package/dist/pack/byte-source.js +50 -0
- package/dist/pack/byte-source.js.map +1 -0
- package/dist/pack/container.d.ts +58 -0
- package/dist/pack/container.js +215 -0
- package/dist/pack/container.js.map +1 -0
- package/dist/pack/remote.d.ts +49 -0
- package/dist/pack/remote.js +258 -0
- package/dist/pack/remote.js.map +1 -0
- package/dist/report/reporter.d.ts +22 -0
- package/dist/report/reporter.js +66 -0
- package/dist/report/reporter.js.map +1 -0
- package/dist/schemas/0.1/L0/result.schema.json +150 -0
- package/dist/schemas/0.1/L0/run.schema.json +114 -0
- package/dist/schemas/0.1/L1/logs-meta.schema.json +39 -0
- package/dist/schemas/0.1/L1/video.schema.json +16 -0
- package/dist/schemas/compile.d.ts +11 -0
- package/dist/schemas/compile.js +45 -0
- package/dist/schemas/compile.js.map +1 -0
- package/dist/schemas/registry.d.ts +11 -0
- package/dist/schemas/registry.js +40 -0
- package/dist/schemas/registry.js.map +1 -0
- package/dist/validate/checks.d.ts +15 -0
- package/dist/validate/checks.js +152 -0
- package/dist/validate/checks.js.map +1 -0
- package/dist/validate/index.d.ts +5 -0
- package/dist/validate/index.js +99 -0
- package/dist/validate/index.js.map +1 -0
- package/dist/validate/l1.d.ts +12 -0
- package/dist/validate/l1.js +117 -0
- package/dist/validate/l1.js.map +1 -0
- package/dist/yaml.d.ts +4 -0
- package/dist/yaml.js +16 -0
- package/dist/yaml.js.map +1 -0
- package/package.json +70 -0
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
3
|
+
"$id": "https://evidence-cli.dev/schemas/0.1/L0/run.schema.json",
|
|
4
|
+
"title": "run.yaml (L0)",
|
|
5
|
+
"description": "Run-level manifest for an .evidence pack. Its presence at the top of a pack is what identifies the pack (the manifest anchor). One pack = one run. evidence-cli is framework-agnostic: nothing here is specific to any test framework.",
|
|
6
|
+
"type": "object",
|
|
7
|
+
"required": ["evidence", "run_id", "status", "started", "title"],
|
|
8
|
+
"properties": {
|
|
9
|
+
"evidence": {
|
|
10
|
+
"description": "Evidence CONTRACT version this pack conforms to. One contract version spans all profiles (L0–L3); adding a profile is additive and does NOT change it — only a breaking change to an existing meaning bumps the version. A validator is pinned to one contract version, so this is a const: a 0.1 validator rejects other versions rather than silently mis-reading them. See decision 0027.",
|
|
11
|
+
"const": "0.1"
|
|
12
|
+
},
|
|
13
|
+
"run_id": {
|
|
14
|
+
"description": "Stable identifier for this run.",
|
|
15
|
+
"type": "string",
|
|
16
|
+
"minLength": 1
|
|
17
|
+
},
|
|
18
|
+
"status": {
|
|
19
|
+
"description": "Lifecycle of the run, NOT a test verdict. While `running`, totals are not yet authoritative; `finalize` seals the pack to `finalized`. `aborted` is a run that ended without sealing.",
|
|
20
|
+
"enum": ["running", "finalized", "aborted"]
|
|
21
|
+
},
|
|
22
|
+
"title": {
|
|
23
|
+
"description": "Human-readable run title.",
|
|
24
|
+
"type": "string",
|
|
25
|
+
"minLength": 1
|
|
26
|
+
},
|
|
27
|
+
"started": {
|
|
28
|
+
"description": "When the run started (RFC3339 / ISO-8601).",
|
|
29
|
+
"type": "string",
|
|
30
|
+
"format": "date-time"
|
|
31
|
+
},
|
|
32
|
+
"ended": {
|
|
33
|
+
"description": "When the run ended (RFC3339 / ISO-8601). Required once status is `finalized`.",
|
|
34
|
+
"type": "string",
|
|
35
|
+
"format": "date-time"
|
|
36
|
+
},
|
|
37
|
+
"totals": {
|
|
38
|
+
"description": "Verdict roll-up over all tests. DERIVED by `evidence finalize` from every tests/<id>/result.yaml — not hand-authored. Required once status is `finalized`.",
|
|
39
|
+
"type": "object",
|
|
40
|
+
"required": ["tests", "passed", "failed", "broken", "skipped"],
|
|
41
|
+
"additionalProperties": false,
|
|
42
|
+
"properties": {
|
|
43
|
+
"tests": { "type": "integer", "minimum": 0 },
|
|
44
|
+
"passed": { "type": "integer", "minimum": 0 },
|
|
45
|
+
"failed": { "type": "integer", "minimum": 0 },
|
|
46
|
+
"broken": { "type": "integer", "minimum": 0 },
|
|
47
|
+
"skipped": { "type": "integer", "minimum": 0 }
|
|
48
|
+
}
|
|
49
|
+
},
|
|
50
|
+
"metrics": {
|
|
51
|
+
"description": "Free-form, framework-defined metrics. Each metric is a typed FACT, never a verdict. The set of metric names and their `type` strings is entirely up to the producing framework.",
|
|
52
|
+
"type": "object",
|
|
53
|
+
"additionalProperties": {
|
|
54
|
+
"type": "object",
|
|
55
|
+
"required": ["value", "type"],
|
|
56
|
+
"properties": {
|
|
57
|
+
"value": {
|
|
58
|
+
"description": "The measured value. A metric is a FACT, never a verdict, so boolean is intentionally excluded — a bare true/false reads as a pass/fail. Emit a flag as a typed string instead. See decision 0012.",
|
|
59
|
+
"type": ["number", "integer", "string"]
|
|
60
|
+
},
|
|
61
|
+
"type": {
|
|
62
|
+
"description": "Producer-defined type of the metric (open string, e.g. percentage, ratio, count, ms, currency). evidence-cli validates presence only, not vocabulary.",
|
|
63
|
+
"type": "string",
|
|
64
|
+
"minLength": 1
|
|
65
|
+
},
|
|
66
|
+
"unit": {
|
|
67
|
+
"description": "Optional display unit.",
|
|
68
|
+
"type": "string"
|
|
69
|
+
},
|
|
70
|
+
"label": {
|
|
71
|
+
"description": "Optional human label.",
|
|
72
|
+
"type": "string"
|
|
73
|
+
}
|
|
74
|
+
},
|
|
75
|
+
"additionalProperties": true
|
|
76
|
+
}
|
|
77
|
+
},
|
|
78
|
+
"environment": {
|
|
79
|
+
"description": "Optional provenance/context block — an OPEN map of key:value pairs. The keys below are conventional, none is required, and any additional key:value pairs are allowed. A non-browser framework (unit, API) may legitimately omit model/surfaces — or the whole block — entirely. See decision 0013.",
|
|
80
|
+
"type": "object",
|
|
81
|
+
"properties": {
|
|
82
|
+
"producer": {
|
|
83
|
+
"description": "The tool that produced this pack.",
|
|
84
|
+
"type": "object",
|
|
85
|
+
"properties": {
|
|
86
|
+
"name": { "type": "string" },
|
|
87
|
+
"version": { "type": "string" }
|
|
88
|
+
},
|
|
89
|
+
"additionalProperties": true
|
|
90
|
+
},
|
|
91
|
+
"model": { "type": "string" },
|
|
92
|
+
"surfaces": { "type": "array", "items": { "type": "string" } },
|
|
93
|
+
"ci": {
|
|
94
|
+
"type": "object",
|
|
95
|
+
"additionalProperties": true
|
|
96
|
+
}
|
|
97
|
+
},
|
|
98
|
+
"additionalProperties": true
|
|
99
|
+
}
|
|
100
|
+
},
|
|
101
|
+
"allOf": [
|
|
102
|
+
{
|
|
103
|
+
"description": "Finalized packs are sealed: ended timestamp and the derived totals must be present.",
|
|
104
|
+
"if": {
|
|
105
|
+
"properties": { "status": { "const": "finalized" } },
|
|
106
|
+
"required": ["status"]
|
|
107
|
+
},
|
|
108
|
+
"then": {
|
|
109
|
+
"required": ["ended", "totals"]
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
],
|
|
113
|
+
"additionalProperties": true
|
|
114
|
+
}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
3
|
+
"$id": "https://evidence-cli.dev/schemas/0.1/L1/logs-meta.schema.json",
|
|
4
|
+
"title": "logs/meta.yaml (L1)",
|
|
5
|
+
"description": "Declares the execution logs kept for one test, at tests/<id>/logs/meta.yaml. The log files themselves are OPAQUE to evidence-cli — referenced by `file` and checked for existence and declared `format`, never parsed. L1 requires at least one log. See decision 0040.",
|
|
6
|
+
"type": "object",
|
|
7
|
+
"required": ["logs"],
|
|
8
|
+
"properties": {
|
|
9
|
+
"logs": {
|
|
10
|
+
"description": "The logs this test kept. L1 requires at least one. Both the log `name` and its `format` are open vocabulary — the framework chooses what to keep and how to label it; evidence-cli records both and parses neither.",
|
|
11
|
+
"type": "array",
|
|
12
|
+
"minItems": 1,
|
|
13
|
+
"items": {
|
|
14
|
+
"type": "object",
|
|
15
|
+
"required": ["name", "file", "format"],
|
|
16
|
+
"properties": {
|
|
17
|
+
"name": {
|
|
18
|
+
"description": "Human-facing log name (e.g. console, network). Open vocabulary.",
|
|
19
|
+
"type": "string",
|
|
20
|
+
"minLength": 1
|
|
21
|
+
},
|
|
22
|
+
"file": {
|
|
23
|
+
"description": "Path to the log file, relative to and CONTAINED WITHIN this test's logs/ directory. No leading '/', no '..' segment, no URL scheme — same containment rule as definition.path (decision 0029). The file MUST exist (a validator cross-check).",
|
|
24
|
+
"type": "string",
|
|
25
|
+
"minLength": 1,
|
|
26
|
+
"pattern": "^(?!/)(?!.*(^|/)\\.\\.(/|$))(?![a-zA-Z][a-zA-Z0-9+.-]*:)[^\\\\]+$"
|
|
27
|
+
},
|
|
28
|
+
"format": {
|
|
29
|
+
"description": "Producer-declared log format (open string, e.g. ndjson, har, log, csv, jsonl, txt). Since evidence-cli never parses the log content, it validates presence and non-emptiness only, never the vocabulary — same stance as step `kind` (decision 0009) and metric `type` (decision 0012).",
|
|
30
|
+
"type": "string",
|
|
31
|
+
"minLength": 1
|
|
32
|
+
}
|
|
33
|
+
},
|
|
34
|
+
"additionalProperties": true
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
},
|
|
38
|
+
"additionalProperties": true
|
|
39
|
+
}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
3
|
+
"$id": "https://evidence-cli.dev/schemas/0.1/L1/video.schema.json",
|
|
4
|
+
"title": "video.yaml (L1)",
|
|
5
|
+
"description": "Optional pointer to a test's video when it is not stored inline. Present only when a test uses the external form instead of an inline video.<ext> file. Validated only when present (video is optional at L1). See decision 0040.",
|
|
6
|
+
"type": "object",
|
|
7
|
+
"required": ["url"],
|
|
8
|
+
"properties": {
|
|
9
|
+
"url": {
|
|
10
|
+
"description": "Location of the video (e.g. an object-store or CDN URL).",
|
|
11
|
+
"type": "string",
|
|
12
|
+
"minLength": 1
|
|
13
|
+
}
|
|
14
|
+
},
|
|
15
|
+
"additionalProperties": true
|
|
16
|
+
}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import type { ValidateFunction } from "ajv";
|
|
2
|
+
export interface CompiledSchemas {
|
|
3
|
+
run: ValidateFunction;
|
|
4
|
+
result: ValidateFunction;
|
|
5
|
+
}
|
|
6
|
+
export interface CompiledL1Schemas {
|
|
7
|
+
logsMeta: ValidateFunction;
|
|
8
|
+
video: ValidateFunction;
|
|
9
|
+
}
|
|
10
|
+
export declare function loadSchemas(version: string, profile: string): CompiledSchemas;
|
|
11
|
+
export declare function loadL1Schemas(version: string): CompiledL1Schemas;
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.loadSchemas = loadSchemas;
|
|
7
|
+
exports.loadL1Schemas = loadL1Schemas;
|
|
8
|
+
const _2020_1 = __importDefault(require("ajv/dist/2020"));
|
|
9
|
+
const ajv_formats_1 = __importDefault(require("ajv-formats"));
|
|
10
|
+
const registry_1 = require("./registry");
|
|
11
|
+
const cache = new Map();
|
|
12
|
+
const l1Cache = new Map();
|
|
13
|
+
function newAjv() {
|
|
14
|
+
const ajv = new _2020_1.default({ allErrors: true, strict: false });
|
|
15
|
+
(0, ajv_formats_1.default)(ajv);
|
|
16
|
+
return ajv;
|
|
17
|
+
}
|
|
18
|
+
function loadSchemas(version, profile) {
|
|
19
|
+
const key = `${version}/${profile}`;
|
|
20
|
+
const hit = cache.get(key);
|
|
21
|
+
if (hit)
|
|
22
|
+
return hit;
|
|
23
|
+
const { run, result } = (0, registry_1.getSchemas)(version, profile);
|
|
24
|
+
const ajv = newAjv();
|
|
25
|
+
const compiled = {
|
|
26
|
+
run: ajv.compile(run),
|
|
27
|
+
result: ajv.compile(result),
|
|
28
|
+
};
|
|
29
|
+
cache.set(key, compiled);
|
|
30
|
+
return compiled;
|
|
31
|
+
}
|
|
32
|
+
function loadL1Schemas(version) {
|
|
33
|
+
const hit = l1Cache.get(version);
|
|
34
|
+
if (hit)
|
|
35
|
+
return hit;
|
|
36
|
+
const { logsMeta, video } = (0, registry_1.getL1Schemas)(version);
|
|
37
|
+
const ajv = newAjv();
|
|
38
|
+
const compiled = {
|
|
39
|
+
logsMeta: ajv.compile(logsMeta),
|
|
40
|
+
video: ajv.compile(video),
|
|
41
|
+
};
|
|
42
|
+
l1Cache.set(version, compiled);
|
|
43
|
+
return compiled;
|
|
44
|
+
}
|
|
45
|
+
//# sourceMappingURL=compile.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"compile.js","sourceRoot":"","sources":["../../src/schemas/compile.ts"],"names":[],"mappings":";;;;;AAwBA,kCAaC;AAED,sCAYC;AAnDD,0DAAoC;AACpC,8DAAqC;AAErC,yCAAsD;AAYtD,MAAM,KAAK,GAAG,IAAI,GAAG,EAA2B,CAAC;AACjD,MAAM,OAAO,GAAG,IAAI,GAAG,EAA6B,CAAC;AAErD,SAAS,MAAM;IACb,MAAM,GAAG,GAAG,IAAI,eAAO,CAAC,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;IAC5D,IAAA,qBAAU,EAAC,GAAG,CAAC,CAAC;IAChB,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAgB,WAAW,CAAC,OAAe,EAAE,OAAe;IAC1D,MAAM,GAAG,GAAG,GAAG,OAAO,IAAI,OAAO,EAAE,CAAC;IACpC,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAC3B,IAAI,GAAG;QAAE,OAAO,GAAG,CAAC;IAEpB,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,IAAA,qBAAU,EAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IACrD,MAAM,GAAG,GAAG,MAAM,EAAE,CAAC;IACrB,MAAM,QAAQ,GAAoB;QAChC,GAAG,EAAE,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC;QACrB,MAAM,EAAE,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC;KAC5B,CAAC;IACF,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IACzB,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,SAAgB,aAAa,CAAC,OAAe;IAC3C,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IACjC,IAAI,GAAG;QAAE,OAAO,GAAG,CAAC;IAEpB,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,GAAG,IAAA,uBAAY,EAAC,OAAO,CAAC,CAAC;IAClD,MAAM,GAAG,GAAG,MAAM,EAAE,CAAC;IACrB,MAAM,QAAQ,GAAsB;QAClC,QAAQ,EAAE,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC;QAC/B,KAAK,EAAE,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC;KAC1B,CAAC;IACF,OAAO,CAAC,GAAG,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;IAC/B,OAAO,QAAQ,CAAC;AAClB,CAAC"}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
export interface SchemaPair {
|
|
2
|
+
run: object;
|
|
3
|
+
result: object;
|
|
4
|
+
}
|
|
5
|
+
/** The L1 declaration schemas evidence-cli parses (artifacts stay opaque). */
|
|
6
|
+
export interface L1Schemas {
|
|
7
|
+
logsMeta: object;
|
|
8
|
+
video: object;
|
|
9
|
+
}
|
|
10
|
+
export declare function getSchemas(version: string, profile: string): SchemaPair;
|
|
11
|
+
export declare function getL1Schemas(version: string): L1Schemas;
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.getSchemas = getSchemas;
|
|
7
|
+
exports.getL1Schemas = getL1Schemas;
|
|
8
|
+
const run_schema_json_1 = __importDefault(require("./0.1/L0/run.schema.json"));
|
|
9
|
+
const result_schema_json_1 = __importDefault(require("./0.1/L0/result.schema.json"));
|
|
10
|
+
const logs_meta_schema_json_1 = __importDefault(require("./0.1/L1/logs-meta.schema.json"));
|
|
11
|
+
const video_schema_json_1 = __importDefault(require("./0.1/L1/video.schema.json"));
|
|
12
|
+
// Version-first (decision 0038): one contract version contains its profile
|
|
13
|
+
// ladder. A future breaking 0.2 is a sibling subtree, leaving 0.1 untouched.
|
|
14
|
+
const REGISTRY = {
|
|
15
|
+
"0.1": {
|
|
16
|
+
L0: { run: run_schema_json_1.default, result: result_schema_json_1.default },
|
|
17
|
+
},
|
|
18
|
+
};
|
|
19
|
+
const L1_REGISTRY = {
|
|
20
|
+
"0.1": { logsMeta: logs_meta_schema_json_1.default, video: video_schema_json_1.default },
|
|
21
|
+
};
|
|
22
|
+
function getSchemas(version, profile) {
|
|
23
|
+
const byProfile = REGISTRY[version];
|
|
24
|
+
if (!byProfile) {
|
|
25
|
+
throw new Error(`no schemas for contract version ${version}`);
|
|
26
|
+
}
|
|
27
|
+
const pair = byProfile[profile];
|
|
28
|
+
if (!pair) {
|
|
29
|
+
throw new Error(`no schemas for profile ${profile} at version ${version}`);
|
|
30
|
+
}
|
|
31
|
+
return pair;
|
|
32
|
+
}
|
|
33
|
+
function getL1Schemas(version) {
|
|
34
|
+
const s = L1_REGISTRY[version];
|
|
35
|
+
if (!s) {
|
|
36
|
+
throw new Error(`no L1 schemas for contract version ${version}`);
|
|
37
|
+
}
|
|
38
|
+
return s;
|
|
39
|
+
}
|
|
40
|
+
//# sourceMappingURL=registry.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"registry.js","sourceRoot":"","sources":["../../src/schemas/registry.ts"],"names":[],"mappings":";;;;;AA4BA,gCAUC;AAED,oCAMC;AA9CD,+EAA6C;AAC7C,qFAAmD;AACnD,2FAAwD;AACxD,mFAAiD;AAajD,2EAA2E;AAC3E,6EAA6E;AAC7E,MAAM,QAAQ,GAA+C;IAC3D,KAAK,EAAE;QACL,EAAE,EAAE,EAAE,GAAG,EAAE,yBAAe,EAAE,MAAM,EAAE,4BAAkB,EAAE;KACzD;CACF,CAAC;AAEF,MAAM,WAAW,GAA8B;IAC7C,KAAK,EAAE,EAAE,QAAQ,EAAE,+BAAoB,EAAE,KAAK,EAAE,2BAAiB,EAAE;CACpE,CAAC;AAEF,SAAgB,UAAU,CAAC,OAAe,EAAE,OAAe;IACzD,MAAM,SAAS,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAC;IACpC,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CAAC,mCAAmC,OAAO,EAAE,CAAC,CAAC;IAChE,CAAC;IACD,MAAM,IAAI,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC;IAChC,IAAI,CAAC,IAAI,EAAE,CAAC;QACV,MAAM,IAAI,KAAK,CAAC,0BAA0B,OAAO,eAAe,OAAO,EAAE,CAAC,CAAC;IAC7E,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAgB,YAAY,CAAC,OAAe;IAC1C,MAAM,CAAC,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;IAC/B,IAAI,CAAC,CAAC,EAAE,CAAC;QACP,MAAM,IAAI,KAAK,CAAC,sCAAsC,OAAO,EAAE,CAAC,CAAC;IACnE,CAAC;IACD,OAAO,CAAC,CAAC;AACX,CAAC"}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import type { Diagnostic } from "../contract";
|
|
2
|
+
import type { PackContainer } from "../pack/container";
|
|
3
|
+
interface TestEntry {
|
|
4
|
+
testId: string;
|
|
5
|
+
result: any;
|
|
6
|
+
}
|
|
7
|
+
export interface CrossCheckInput {
|
|
8
|
+
run: any;
|
|
9
|
+
tests: TestEntry[];
|
|
10
|
+
container: PackContainer;
|
|
11
|
+
}
|
|
12
|
+
export declare function runCrossChecks(input: CrossCheckInput): Promise<Diagnostic[]>;
|
|
13
|
+
/** A path is contained iff it is relative, scheme-less, and never climbs out. */
|
|
14
|
+
export declare function isContained(relPath: string): boolean;
|
|
15
|
+
export {};
|
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
35
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
+
exports.runCrossChecks = runCrossChecks;
|
|
37
|
+
exports.isContained = isContained;
|
|
38
|
+
const path = __importStar(require("node:path"));
|
|
39
|
+
const node_crypto_1 = require("node:crypto");
|
|
40
|
+
const contract_1 = require("../contract");
|
|
41
|
+
const diagnostics_1 = require("../diagnostics");
|
|
42
|
+
async function runCrossChecks(input) {
|
|
43
|
+
const diags = [];
|
|
44
|
+
const finalized = input.run?.status === "finalized";
|
|
45
|
+
// --- Structure-only (every run status) ---
|
|
46
|
+
for (const { testId, result } of input.tests) {
|
|
47
|
+
const loc = `tests/${testId}/result.yaml`;
|
|
48
|
+
// (a) result.test equals the directory name
|
|
49
|
+
if (result?.test !== undefined && result.test !== testId) {
|
|
50
|
+
diags.push((0, diagnostics_1.err)(contract_1.Codes.TEST_ID_MISMATCH, loc, `result.test "${result.test}" does not match directory name "${testId}"`));
|
|
51
|
+
}
|
|
52
|
+
// (b)+(c) definition.path declared, contained, and the file exists
|
|
53
|
+
const defPath = result?.definition?.path;
|
|
54
|
+
if (typeof defPath === "string" && defPath.length > 0) {
|
|
55
|
+
if (!isContained(defPath)) {
|
|
56
|
+
diags.push((0, diagnostics_1.err)(contract_1.Codes.DEFINITION_PATH_ESCAPE, `${loc}#/definition/path`, `definition.path "${defPath}" escapes the test directory`));
|
|
57
|
+
}
|
|
58
|
+
else if (!(await input.container.fileExists(testId, defPath))) {
|
|
59
|
+
diags.push((0, diagnostics_1.err)(contract_1.Codes.DEFINITION_MISSING, `${loc}#/definition/path`, `definition file "${defPath}" does not exist`));
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
// (d) ordinals unique and strictly increasing in array order (gaps allowed)
|
|
63
|
+
const steps = Array.isArray(result?.steps) ? result.steps : [];
|
|
64
|
+
let prev = -Infinity;
|
|
65
|
+
const seen = new Set();
|
|
66
|
+
steps.forEach((step, i) => {
|
|
67
|
+
const ord = step?.ordinal;
|
|
68
|
+
if (typeof ord !== "number")
|
|
69
|
+
return; // schema catches missing/typed ordinals
|
|
70
|
+
if (seen.has(ord)) {
|
|
71
|
+
diags.push((0, diagnostics_1.err)(contract_1.Codes.ORDINAL_COLLISION, `${loc}#/steps/${i}/ordinal`, `duplicate step ordinal ${ord}`));
|
|
72
|
+
}
|
|
73
|
+
seen.add(ord);
|
|
74
|
+
if (ord <= prev) {
|
|
75
|
+
diags.push((0, diagnostics_1.err)(contract_1.Codes.ORDINAL_NOT_INCREASING, `${loc}#/steps/${i}/ordinal`, `step ordinal ${ord} is not strictly increasing (previous ${prev})`));
|
|
76
|
+
}
|
|
77
|
+
prev = ord;
|
|
78
|
+
});
|
|
79
|
+
// Advisory only: a "passed" test with a non-passing step (verdict is authored — 0016)
|
|
80
|
+
const bad = steps.find((s) => s?.status === "failed" || s?.status === "broken");
|
|
81
|
+
if (result?.status === "passed" && bad) {
|
|
82
|
+
diags.push((0, diagnostics_1.warn)(contract_1.Codes.STATUS_DISAGREES, loc, `test status "passed" but step "${bad.id}" is "${bad.status}"`));
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
if (!finalized)
|
|
86
|
+
return diags;
|
|
87
|
+
// --- Full seal (status: finalized) ---
|
|
88
|
+
// (e) ended >= started
|
|
89
|
+
const started = input.run?.started;
|
|
90
|
+
const ended = input.run?.ended;
|
|
91
|
+
if (typeof started === "string" && typeof ended === "string") {
|
|
92
|
+
if (Date.parse(ended) < Date.parse(started)) {
|
|
93
|
+
diags.push((0, diagnostics_1.err)(contract_1.Codes.ENDED_BEFORE_STARTED, "run.yaml#/ended", `ended (${ended}) is before started (${started})`));
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
// (f) totals equals the rolled-up verdicts AND tests == sum of the four buckets
|
|
97
|
+
const rolled = rollup(input.tests);
|
|
98
|
+
const totals = input.run?.totals;
|
|
99
|
+
if (totals && typeof totals === "object") {
|
|
100
|
+
["tests", "passed", "failed", "broken", "skipped"].forEach((k) => {
|
|
101
|
+
if (totals[k] !== rolled[k]) {
|
|
102
|
+
diags.push((0, diagnostics_1.err)(contract_1.Codes.TOTALS_MISMATCH, `run.yaml#/totals/${k}`, `totals.${k} is ${totals[k]} but the rolled-up value is ${rolled[k]}`));
|
|
103
|
+
}
|
|
104
|
+
});
|
|
105
|
+
const bucketSum = (totals.passed ?? 0) + (totals.failed ?? 0) + (totals.broken ?? 0) + (totals.skipped ?? 0);
|
|
106
|
+
if (totals.tests !== bucketSum) {
|
|
107
|
+
diags.push((0, diagnostics_1.err)(contract_1.Codes.TOTALS_MISMATCH, "run.yaml#/totals/tests", `totals.tests (${totals.tests}) does not equal the sum of the four buckets (${bucketSum})`));
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
// (g) every definition.sha256 present and hash-matches its file
|
|
111
|
+
for (const { testId, result } of input.tests) {
|
|
112
|
+
const loc = `tests/${testId}/result.yaml`;
|
|
113
|
+
const defPath = result?.definition?.path;
|
|
114
|
+
if (typeof defPath !== "string" || !isContained(defPath))
|
|
115
|
+
continue; // already flagged
|
|
116
|
+
const declared = result?.definition?.sha256;
|
|
117
|
+
if (typeof declared !== "string" || declared.length === 0) {
|
|
118
|
+
diags.push((0, diagnostics_1.err)(contract_1.Codes.HASH_MISSING, `${loc}#/definition/sha256`, "definition.sha256 is required on a finalized pack"));
|
|
119
|
+
continue;
|
|
120
|
+
}
|
|
121
|
+
const bytes = await input.container.readBytes(testId, defPath);
|
|
122
|
+
if (!bytes)
|
|
123
|
+
continue; // missing file already flagged structure-only
|
|
124
|
+
const actual = `sha256:${(0, node_crypto_1.createHash)("sha256").update(bytes).digest("hex")}`;
|
|
125
|
+
if (actual !== declared) {
|
|
126
|
+
diags.push((0, diagnostics_1.err)(contract_1.Codes.HASH_MISMATCH, `${loc}#/definition/sha256`, `definition.sha256 ${declared} does not match the file hash ${actual}`));
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
return diags;
|
|
130
|
+
}
|
|
131
|
+
/** A path is contained iff it is relative, scheme-less, and never climbs out. */
|
|
132
|
+
function isContained(relPath) {
|
|
133
|
+
if (relPath.startsWith("/"))
|
|
134
|
+
return false;
|
|
135
|
+
if (/^[a-zA-Z][a-zA-Z0-9+.-]*:/.test(relPath))
|
|
136
|
+
return false; // URL scheme
|
|
137
|
+
const norm = path.posix.normalize(relPath);
|
|
138
|
+
if (norm.startsWith("/") || norm === ".." || norm.startsWith("../"))
|
|
139
|
+
return false;
|
|
140
|
+
return !norm.split("/").includes("..");
|
|
141
|
+
}
|
|
142
|
+
function rollup(tests) {
|
|
143
|
+
const t = { tests: 0, passed: 0, failed: 0, broken: 0, skipped: 0 };
|
|
144
|
+
for (const { result } of tests) {
|
|
145
|
+
t.tests++;
|
|
146
|
+
const s = result?.status;
|
|
147
|
+
if (s === "passed" || s === "failed" || s === "broken" || s === "skipped")
|
|
148
|
+
t[s]++;
|
|
149
|
+
}
|
|
150
|
+
return t;
|
|
151
|
+
}
|
|
152
|
+
//# sourceMappingURL=checks.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"checks.js","sourceRoot":"","sources":["../../src/validate/checks.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAkBA,wCA+FC;AAGD,kCAMC;AA1HD,gDAAkC;AAClC,6CAAyC;AACzC,0CAAoC;AAEpC,gDAA2C;AAcpC,KAAK,UAAU,cAAc,CAAC,KAAsB;IACzD,MAAM,KAAK,GAAiB,EAAE,CAAC;IAC/B,MAAM,SAAS,GAAG,KAAK,CAAC,GAAG,EAAE,MAAM,KAAK,WAAW,CAAC;IAEpD,4CAA4C;IAC5C,KAAK,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;QAC7C,MAAM,GAAG,GAAG,SAAS,MAAM,cAAc,CAAC;QAE1C,4CAA4C;QAC5C,IAAI,MAAM,EAAE,IAAI,KAAK,SAAS,IAAI,MAAM,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YACzD,KAAK,CAAC,IAAI,CAAC,IAAA,iBAAG,EAAC,gBAAK,CAAC,gBAAgB,EAAE,GAAG,EAAE,gBAAgB,MAAM,CAAC,IAAI,oCAAoC,MAAM,GAAG,CAAC,CAAC,CAAC;QACzH,CAAC;QAED,mEAAmE;QACnE,MAAM,OAAO,GAAG,MAAM,EAAE,UAAU,EAAE,IAAI,CAAC;QACzC,IAAI,OAAO,OAAO,KAAK,QAAQ,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtD,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,EAAE,CAAC;gBAC1B,KAAK,CAAC,IAAI,CAAC,IAAA,iBAAG,EAAC,gBAAK,CAAC,sBAAsB,EAAE,GAAG,GAAG,mBAAmB,EAAE,oBAAoB,OAAO,8BAA8B,CAAC,CAAC,CAAC;YACtI,CAAC;iBAAM,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,SAAS,CAAC,UAAU,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,EAAE,CAAC;gBAChE,KAAK,CAAC,IAAI,CAAC,IAAA,iBAAG,EAAC,gBAAK,CAAC,kBAAkB,EAAE,GAAG,GAAG,mBAAmB,EAAE,oBAAoB,OAAO,kBAAkB,CAAC,CAAC,CAAC;YACtH,CAAC;QACH,CAAC;QAED,4EAA4E;QAC5E,MAAM,KAAK,GAAU,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;QACtE,IAAI,IAAI,GAAG,CAAC,QAAQ,CAAC;QACrB,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;QAC/B,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE;YACxB,MAAM,GAAG,GAAG,IAAI,EAAE,OAAO,CAAC;YAC1B,IAAI,OAAO,GAAG,KAAK,QAAQ;gBAAE,OAAO,CAAC,wCAAwC;YAC7E,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;gBAClB,KAAK,CAAC,IAAI,CAAC,IAAA,iBAAG,EAAC,gBAAK,CAAC,iBAAiB,EAAE,GAAG,GAAG,WAAW,CAAC,UAAU,EAAE,0BAA0B,GAAG,EAAE,CAAC,CAAC,CAAC;YAC1G,CAAC;YACD,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACd,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;gBAChB,KAAK,CAAC,IAAI,CAAC,IAAA,iBAAG,EAAC,gBAAK,CAAC,sBAAsB,EAAE,GAAG,GAAG,WAAW,CAAC,UAAU,EAAE,gBAAgB,GAAG,yCAAyC,IAAI,GAAG,CAAC,CAAC,CAAC;YACnJ,CAAC;YACD,IAAI,GAAG,GAAG,CAAC;QACb,CAAC,CAAC,CAAC;QAEH,sFAAsF;QACtF,MAAM,GAAG,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,MAAM,KAAK,QAAQ,IAAI,CAAC,EAAE,MAAM,KAAK,QAAQ,CAAC,CAAC;QAChF,IAAI,MAAM,EAAE,MAAM,KAAK,QAAQ,IAAI,GAAG,EAAE,CAAC;YACvC,KAAK,CAAC,IAAI,CAAC,IAAA,kBAAI,EAAC,gBAAK,CAAC,gBAAgB,EAAE,GAAG,EAAE,kCAAkC,GAAG,CAAC,EAAE,SAAS,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAChH,CAAC;IACH,CAAC;IAED,IAAI,CAAC,SAAS;QAAE,OAAO,KAAK,CAAC;IAE7B,wCAAwC;IAExC,uBAAuB;IACvB,MAAM,OAAO,GAAG,KAAK,CAAC,GAAG,EAAE,OAAO,CAAC;IACnC,MAAM,KAAK,GAAG,KAAK,CAAC,GAAG,EAAE,KAAK,CAAC;IAC/B,IAAI,OAAO,OAAO,KAAK,QAAQ,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC7D,IAAI,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;YAC5C,KAAK,CAAC,IAAI,CAAC,IAAA,iBAAG,EAAC,gBAAK,CAAC,oBAAoB,EAAE,iBAAiB,EAAE,UAAU,KAAK,wBAAwB,OAAO,GAAG,CAAC,CAAC,CAAC;QACpH,CAAC;IACH,CAAC;IAED,gFAAgF;IAChF,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACnC,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,EAAE,MAAM,CAAC;IACjC,IAAI,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;QACxC,CAAC,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,SAAS,CAAW,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE;YAC1E,IAAI,MAAM,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC5B,KAAK,CAAC,IAAI,CAAC,IAAA,iBAAG,EAAC,gBAAK,CAAC,eAAe,EAAE,oBAAoB,CAAC,EAAE,EAAE,UAAU,CAAC,OAAO,MAAM,CAAC,CAAC,CAAC,+BAA+B,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YACzI,CAAC;QACH,CAAC,CAAC,CAAC;QACH,MAAM,SAAS,GACb,CAAC,MAAM,CAAC,MAAM,IAAI,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,IAAI,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,IAAI,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,IAAI,CAAC,CAAC,CAAC;QAC7F,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAC/B,KAAK,CAAC,IAAI,CAAC,IAAA,iBAAG,EAAC,gBAAK,CAAC,eAAe,EAAE,wBAAwB,EAAE,iBAAiB,MAAM,CAAC,KAAK,iDAAiD,SAAS,GAAG,CAAC,CAAC,CAAC;QAC/J,CAAC;IACH,CAAC;IAED,gEAAgE;IAChE,KAAK,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;QAC7C,MAAM,GAAG,GAAG,SAAS,MAAM,cAAc,CAAC;QAC1C,MAAM,OAAO,GAAG,MAAM,EAAE,UAAU,EAAE,IAAI,CAAC;QACzC,IAAI,OAAO,OAAO,KAAK,QAAQ,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC;YAAE,SAAS,CAAC,kBAAkB;QACtF,MAAM,QAAQ,GAAG,MAAM,EAAE,UAAU,EAAE,MAAM,CAAC;QAC5C,IAAI,OAAO,QAAQ,KAAK,QAAQ,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC1D,KAAK,CAAC,IAAI,CAAC,IAAA,iBAAG,EAAC,gBAAK,CAAC,YAAY,EAAE,GAAG,GAAG,qBAAqB,EAAE,mDAAmD,CAAC,CAAC,CAAC;YACtH,SAAS;QACX,CAAC;QACD,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,SAAS,CAAC,SAAS,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QAC/D,IAAI,CAAC,KAAK;YAAE,SAAS,CAAC,8CAA8C;QACpE,MAAM,MAAM,GAAG,UAAU,IAAA,wBAAU,EAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;QAC5E,IAAI,MAAM,KAAK,QAAQ,EAAE,CAAC;YACxB,KAAK,CAAC,IAAI,CAAC,IAAA,iBAAG,EAAC,gBAAK,CAAC,aAAa,EAAE,GAAG,GAAG,qBAAqB,EAAE,qBAAqB,QAAQ,iCAAiC,MAAM,EAAE,CAAC,CAAC,CAAC;QAC5I,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED,iFAAiF;AACjF,SAAgB,WAAW,CAAC,OAAe;IACzC,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC;QAAE,OAAO,KAAK,CAAC;IAC1C,IAAI,2BAA2B,CAAC,IAAI,CAAC,OAAO,CAAC;QAAE,OAAO,KAAK,CAAC,CAAC,aAAa;IAC1E,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;IAC3C,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAClF,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;AACzC,CAAC;AAED,SAAS,MAAM,CAAC,KAAkB;IAChC,MAAM,CAAC,GAAW,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;IAC5E,KAAK,MAAM,EAAE,MAAM,EAAE,IAAI,KAAK,EAAE,CAAC;QAC/B,CAAC,CAAC,KAAK,EAAE,CAAC;QACV,MAAM,CAAC,GAAG,MAAM,EAAE,MAA6B,CAAC;QAChD,IAAI,CAAC,KAAK,QAAQ,IAAI,CAAC,KAAK,QAAQ,IAAI,CAAC,KAAK,QAAQ,IAAI,CAAC,KAAK,SAAS;YAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IACpF,CAAC;IACD,OAAO,CAAC,CAAC;AACX,CAAC"}
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.validate = validate;
|
|
4
|
+
const contract_1 = require("../contract");
|
|
5
|
+
const diagnostics_1 = require("../diagnostics");
|
|
6
|
+
const container_1 = require("../pack/container");
|
|
7
|
+
const compile_1 = require("../schemas/compile");
|
|
8
|
+
const yaml_1 = require("../yaml");
|
|
9
|
+
const checks_1 = require("./checks");
|
|
10
|
+
const l1_1 = require("./l1");
|
|
11
|
+
async function validate(target, opts = {}) {
|
|
12
|
+
const profile = opts.profile ?? contract_1.DEFAULT_PROFILE;
|
|
13
|
+
const chain = (0, contract_1.profileChain)(profile);
|
|
14
|
+
if (!chain) {
|
|
15
|
+
const e = new Error(`unknown profile "${profile}"`);
|
|
16
|
+
e.code = "USAGE";
|
|
17
|
+
throw e;
|
|
18
|
+
}
|
|
19
|
+
const diagnostics = [];
|
|
20
|
+
let status = null;
|
|
21
|
+
const finish = () => ({
|
|
22
|
+
valid: !diagnostics.some((d) => d.severity === "error"),
|
|
23
|
+
profile,
|
|
24
|
+
version: contract_1.CONTRACT_VERSION,
|
|
25
|
+
status,
|
|
26
|
+
diagnostics,
|
|
27
|
+
});
|
|
28
|
+
const container = await (0, container_1.openContainer)(target);
|
|
29
|
+
const manifestRaw = await container.readManifest();
|
|
30
|
+
if (manifestRaw == null) {
|
|
31
|
+
diagnostics.push((0, diagnostics_1.err)(contract_1.Codes.MANIFEST_MISSING, "run.yaml", "no top-level run.yaml — not a valid evidence pack"));
|
|
32
|
+
return finish();
|
|
33
|
+
}
|
|
34
|
+
let run;
|
|
35
|
+
try {
|
|
36
|
+
run = (0, yaml_1.parseYaml)(manifestRaw);
|
|
37
|
+
}
|
|
38
|
+
catch (e) {
|
|
39
|
+
diagnostics.push((0, diagnostics_1.err)(contract_1.Codes.MANIFEST_PARSE, "run.yaml", `run.yaml is not valid YAML: ${e?.message ?? e}`));
|
|
40
|
+
return finish();
|
|
41
|
+
}
|
|
42
|
+
status = typeof run?.status === "string" ? run.status : null;
|
|
43
|
+
// Version gate (decision 0027): reject rather than mis-read; halt on mismatch.
|
|
44
|
+
const vg = versionGate(run, "run.yaml");
|
|
45
|
+
if (vg) {
|
|
46
|
+
diagnostics.push(vg);
|
|
47
|
+
return finish();
|
|
48
|
+
}
|
|
49
|
+
// The L0 layer always validates against the L0 schemas — higher profiles are
|
|
50
|
+
// additive and add checks, never a different run/result shape (decisions 0027,
|
|
51
|
+
// 0040). The requested profile only selects which extra layers run.
|
|
52
|
+
const schemas = (0, compile_1.loadSchemas)(contract_1.CONTRACT_VERSION, "L0");
|
|
53
|
+
diagnostics.push(...schemaDiagnostics(schemas.run, run, "run.yaml"));
|
|
54
|
+
const testIds = await container.listTestIds();
|
|
55
|
+
const tests = [];
|
|
56
|
+
for (const testId of testIds) {
|
|
57
|
+
const loc = `tests/${testId}/result.yaml`;
|
|
58
|
+
const raw = await container.readResult(testId);
|
|
59
|
+
if (raw == null) {
|
|
60
|
+
diagnostics.push((0, diagnostics_1.err)(contract_1.Codes.RESULT_MISSING, loc, `missing result.yaml for test "${testId}"`));
|
|
61
|
+
continue;
|
|
62
|
+
}
|
|
63
|
+
let result;
|
|
64
|
+
try {
|
|
65
|
+
result = (0, yaml_1.parseYaml)(raw);
|
|
66
|
+
}
|
|
67
|
+
catch (e) {
|
|
68
|
+
diagnostics.push((0, diagnostics_1.err)(contract_1.Codes.RESULT_PARSE, loc, `result.yaml is not valid YAML: ${e?.message ?? e}`));
|
|
69
|
+
continue;
|
|
70
|
+
}
|
|
71
|
+
const rvg = versionGate(result, loc);
|
|
72
|
+
if (rvg) {
|
|
73
|
+
diagnostics.push(rvg);
|
|
74
|
+
continue; // skip this result's further checks
|
|
75
|
+
}
|
|
76
|
+
diagnostics.push(...schemaDiagnostics(schemas.result, result, loc));
|
|
77
|
+
tests.push({ testId, result });
|
|
78
|
+
}
|
|
79
|
+
diagnostics.push(...(await (0, checks_1.runCrossChecks)({ run, tests, container })));
|
|
80
|
+
if (chain.includes("L1")) {
|
|
81
|
+
diagnostics.push(...(await (0, l1_1.runL1Checks)({ container, status, tests, version: contract_1.CONTRACT_VERSION })));
|
|
82
|
+
}
|
|
83
|
+
return finish();
|
|
84
|
+
}
|
|
85
|
+
function versionGate(obj, location) {
|
|
86
|
+
const v = obj?.evidence;
|
|
87
|
+
if (v === undefined || v === null)
|
|
88
|
+
return null; // schema reports the required field
|
|
89
|
+
if (v !== contract_1.CONTRACT_VERSION) {
|
|
90
|
+
return (0, diagnostics_1.err)(contract_1.Codes.VERSION_UNSUPPORTED, location, `pack declares evidence ${JSON.stringify(v)}; this validator implements ${contract_1.CONTRACT_VERSION} — cannot validate`);
|
|
91
|
+
}
|
|
92
|
+
return null;
|
|
93
|
+
}
|
|
94
|
+
function schemaDiagnostics(validateFn, data, location) {
|
|
95
|
+
if (validateFn(data))
|
|
96
|
+
return [];
|
|
97
|
+
return (validateFn.errors ?? []).map((e) => (0, diagnostics_1.err)(contract_1.Codes.SCHEMA, `${location}${e.instancePath || ""}`, `${e.instancePath || "/"} ${e.message ?? "schema violation"}`));
|
|
98
|
+
}
|
|
99
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/validate/index.ts"],"names":[],"mappings":";;AAcA,4BAuFC;AArGD,0CAAqF;AAErF,gDAAqC;AACrC,iDAAkD;AAClD,gDAAiD;AACjD,kCAAoC;AACpC,qCAA0C;AAC1C,6BAAmC;AAO5B,KAAK,UAAU,QAAQ,CAC5B,MAAc,EACd,OAAwB,EAAE;IAE1B,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,IAAI,0BAAe,CAAC;IAChD,MAAM,KAAK,GAAG,IAAA,uBAAY,EAAC,OAAO,CAAC,CAAC;IACpC,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,MAAM,CAAC,GAAG,IAAI,KAAK,CAAC,oBAAoB,OAAO,GAAG,CAA8B,CAAC;QACjF,CAAC,CAAC,IAAI,GAAG,OAAO,CAAC;QACjB,MAAM,CAAC,CAAC;IACV,CAAC;IACD,MAAM,WAAW,GAAiB,EAAE,CAAC;IACrC,IAAI,MAAM,GAAkB,IAAI,CAAC;IACjC,MAAM,MAAM,GAAG,GAAqB,EAAE,CAAC,CAAC;QACtC,KAAK,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,OAAO,CAAC;QACvD,OAAO;QACP,OAAO,EAAE,2BAAgB;QACzB,MAAM;QACN,WAAW;KACZ,CAAC,CAAC;IAEH,MAAM,SAAS,GAAG,MAAM,IAAA,yBAAa,EAAC,MAAM,CAAC,CAAC;IAE9C,MAAM,WAAW,GAAG,MAAM,SAAS,CAAC,YAAY,EAAE,CAAC;IACnD,IAAI,WAAW,IAAI,IAAI,EAAE,CAAC;QACxB,WAAW,CAAC,IAAI,CACd,IAAA,iBAAG,EAAC,gBAAK,CAAC,gBAAgB,EAAE,UAAU,EAAE,mDAAmD,CAAC,CAC7F,CAAC;QACF,OAAO,MAAM,EAAE,CAAC;IAClB,CAAC;IAED,IAAI,GAAQ,CAAC;IACb,IAAI,CAAC;QACH,GAAG,GAAG,IAAA,gBAAS,EAAC,WAAW,CAAC,CAAC;IAC/B,CAAC;IAAC,OAAO,CAAM,EAAE,CAAC;QAChB,WAAW,CAAC,IAAI,CAAC,IAAA,iBAAG,EAAC,gBAAK,CAAC,cAAc,EAAE,UAAU,EAAE,+BAA+B,CAAC,EAAE,OAAO,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;QAC1G,OAAO,MAAM,EAAE,CAAC;IAClB,CAAC;IACD,MAAM,GAAG,OAAO,GAAG,EAAE,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC;IAE7D,+EAA+E;IAC/E,MAAM,EAAE,GAAG,WAAW,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;IACxC,IAAI,EAAE,EAAE,CAAC;QACP,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACrB,OAAO,MAAM,EAAE,CAAC;IAClB,CAAC;IAED,6EAA6E;IAC7E,+EAA+E;IAC/E,oEAAoE;IACpE,MAAM,OAAO,GAAG,IAAA,qBAAW,EAAC,2BAAgB,EAAE,IAAI,CAAC,CAAC;IACpD,WAAW,CAAC,IAAI,CAAC,GAAG,iBAAiB,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,EAAE,UAAU,CAAC,CAAC,CAAC;IAErE,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,WAAW,EAAE,CAAC;IAC9C,MAAM,KAAK,GAAsC,EAAE,CAAC;IACpD,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,MAAM,GAAG,GAAG,SAAS,MAAM,cAAc,CAAC;QAC1C,MAAM,GAAG,GAAG,MAAM,SAAS,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;QAC/C,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;YAChB,WAAW,CAAC,IAAI,CAAC,IAAA,iBAAG,EAAC,gBAAK,CAAC,cAAc,EAAE,GAAG,EAAE,iCAAiC,MAAM,GAAG,CAAC,CAAC,CAAC;YAC7F,SAAS;QACX,CAAC;QACD,IAAI,MAAW,CAAC;QAChB,IAAI,CAAC;YACH,MAAM,GAAG,IAAA,gBAAS,EAAC,GAAG,CAAC,CAAC;QAC1B,CAAC;QAAC,OAAO,CAAM,EAAE,CAAC;YAChB,WAAW,CAAC,IAAI,CAAC,IAAA,iBAAG,EAAC,gBAAK,CAAC,YAAY,EAAE,GAAG,EAAE,kCAAkC,CAAC,EAAE,OAAO,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;YACpG,SAAS;QACX,CAAC;QACD,MAAM,GAAG,GAAG,WAAW,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QACrC,IAAI,GAAG,EAAE,CAAC;YACR,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACtB,SAAS,CAAC,oCAAoC;QAChD,CAAC;QACD,WAAW,CAAC,IAAI,CAAC,GAAG,iBAAiB,CAAC,OAAO,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC;QACpE,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;IACjC,CAAC;IAED,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,IAAA,uBAAc,EAAC,EAAE,GAAG,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,CAAC;IAEvE,IAAI,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;QACzB,WAAW,CAAC,IAAI,CACd,GAAG,CAAC,MAAM,IAAA,gBAAW,EAAC,EAAE,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,2BAAgB,EAAE,CAAC,CAAC,CAChF,CAAC;IACJ,CAAC;IAED,OAAO,MAAM,EAAE,CAAC;AAClB,CAAC;AAED,SAAS,WAAW,CAAC,GAAQ,EAAE,QAAgB;IAC7C,MAAM,CAAC,GAAG,GAAG,EAAE,QAAQ,CAAC;IACxB,IAAI,CAAC,KAAK,SAAS,IAAI,CAAC,KAAK,IAAI;QAAE,OAAO,IAAI,CAAC,CAAC,oCAAoC;IACpF,IAAI,CAAC,KAAK,2BAAgB,EAAE,CAAC;QAC3B,OAAO,IAAA,iBAAG,EACR,gBAAK,CAAC,mBAAmB,EACzB,QAAQ,EACR,0BAA0B,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,+BAA+B,2BAAgB,oBAAoB,CAC/G,CAAC;IACJ,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,iBAAiB,CACxB,UAA4B,EAC5B,IAAa,EACb,QAAgB;IAEhB,IAAI,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO,EAAE,CAAC;IAChC,OAAO,CAAC,UAAU,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CACzC,IAAA,iBAAG,EACD,gBAAK,CAAC,MAAM,EACZ,GAAG,QAAQ,GAAG,CAAC,CAAC,YAAY,IAAI,EAAE,EAAE,EACpC,GAAG,CAAC,CAAC,YAAY,IAAI,GAAG,IAAI,CAAC,CAAC,OAAO,IAAI,kBAAkB,EAAE,CAC9D,CACF,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import type { Diagnostic } from "../contract";
|
|
2
|
+
import type { PackContainer } from "../pack/container";
|
|
3
|
+
export interface L1Input {
|
|
4
|
+
container: PackContainer;
|
|
5
|
+
status: string | null;
|
|
6
|
+
tests: {
|
|
7
|
+
testId: string;
|
|
8
|
+
result: any;
|
|
9
|
+
}[];
|
|
10
|
+
version: string;
|
|
11
|
+
}
|
|
12
|
+
export declare function runL1Checks(input: L1Input): Promise<Diagnostic[]>;
|