@rotifer/playground 0.25.1 → 0.26.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +25 -0
- package/README.md +1 -1
- package/README.zh.md +1 -1
- package/dist/commands/arena-submit.d.ts.map +1 -1
- package/dist/commands/arena-submit.js +2 -57
- package/dist/commands/arena-submit.js.map +1 -1
- package/dist/commands/compile.d.ts.map +1 -1
- package/dist/commands/compile.js +22 -0
- package/dist/commands/compile.js.map +1 -1
- package/dist/commands/install.d.ts.map +1 -1
- package/dist/commands/install.js +18 -0
- package/dist/commands/install.js.map +1 -1
- package/dist/commands/publish.d.ts.map +1 -1
- package/dist/commands/publish.js +49 -0
- package/dist/commands/publish.js.map +1 -1
- package/dist/commands/test.d.ts.map +1 -1
- package/dist/commands/test.js +164 -0
- package/dist/commands/test.js.map +1 -1
- package/dist/scanner/rules.d.ts.map +1 -1
- package/dist/scanner/rules.js +14 -4
- package/dist/scanner/rules.js.map +1 -1
- package/dist/testsuite/gate.d.ts +82 -0
- package/dist/testsuite/gate.d.ts.map +1 -0
- package/dist/testsuite/gate.js +179 -0
- package/dist/testsuite/gate.js.map +1 -0
- package/dist/testsuite/generate.d.ts +12 -0
- package/dist/testsuite/generate.d.ts.map +1 -0
- package/dist/testsuite/generate.js +70 -0
- package/dist/testsuite/generate.js.map +1 -0
- package/dist/testsuite/load.d.ts +23 -0
- package/dist/testsuite/load.d.ts.map +1 -0
- package/dist/testsuite/load.js +121 -0
- package/dist/testsuite/load.js.map +1 -0
- package/dist/testsuite/run.d.ts +109 -0
- package/dist/testsuite/run.d.ts.map +1 -0
- package/dist/testsuite/run.js +298 -0
- package/dist/testsuite/run.js.map +1 -0
- package/dist/testsuite/scaffold.d.ts +20 -0
- package/dist/testsuite/scaffold.d.ts.map +1 -0
- package/dist/testsuite/scaffold.js +146 -0
- package/dist/testsuite/scaffold.js.map +1 -0
- package/dist/testsuite/types.d.ts +67 -0
- package/dist/testsuite/types.d.ts.map +1 -0
- package/dist/testsuite/types.js +19 -0
- package/dist/testsuite/types.js.map +1 -0
- package/dist/utils/artifact-consistency.d.ts +37 -0
- package/dist/utils/artifact-consistency.d.ts.map +1 -0
- package/dist/utils/artifact-consistency.js +43 -0
- package/dist/utils/artifact-consistency.js.map +1 -0
- package/genes/genesis-code-format/.cloud-manifest.json +4 -4
- package/genes/genesis-code-format/index.ts +24 -6
- package/genes/genesis-code-format/phenotype.json +36 -7
- package/genes/genesis-code-format/testsuite.json +25 -0
- package/genes/genesis-file-read/.cloud-manifest.json +4 -4
- package/genes/genesis-file-read/index.ts +21 -2
- package/genes/genesis-file-read/phenotype.json +32 -8
- package/genes/genesis-file-read/testsuite.json +41 -0
- package/genes/genesis-web-search/.cloud-manifest.json +4 -4
- package/genes/genesis-web-search/index.ts +23 -4
- package/genes/genesis-web-search/phenotype.json +36 -9
- package/genes/genesis-web-search/testsuite.json +82 -0
- package/genes/genesis-web-search-lite/.cloud-manifest.json +4 -4
- package/genes/genesis-web-search-lite/index.ts +14 -2
- package/genes/genesis-web-search-lite/phenotype.json +21 -5
- package/genes/genesis-web-search-lite/testsuite.json +24 -0
- package/package.json +12 -7
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Seeded input generation, shared by `arena submit` and the §47.5 T1 property
|
|
4
|
+
* test.
|
|
5
|
+
*
|
|
6
|
+
* Moved verbatim out of arena-submit.ts rather than reimplemented: S_r has to
|
|
7
|
+
* stay reproducible across releases, and a second generator that drifted from
|
|
8
|
+
* this one would make `rotifer test` and Arena admission disagree about what a
|
|
9
|
+
* Gene was fed. Same seed, same input, both callers.
|
|
10
|
+
*/
|
|
11
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
|
+
exports.mulberry32 = mulberry32;
|
|
13
|
+
exports.generateTestInput = generateTestInput;
|
|
14
|
+
function mulberry32(seed) {
|
|
15
|
+
let s = seed | 0;
|
|
16
|
+
return () => {
|
|
17
|
+
s = (s + 0x6d2b79f5) | 0;
|
|
18
|
+
let t = Math.imul(s ^ (s >>> 15), 1 | s);
|
|
19
|
+
t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t;
|
|
20
|
+
return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
|
|
21
|
+
};
|
|
22
|
+
}
|
|
23
|
+
function generateTestInput(schema, seed) {
|
|
24
|
+
if (!schema || !schema.properties)
|
|
25
|
+
return { data: `test-input-${seed}` };
|
|
26
|
+
const rng = mulberry32(seed * 2654435761);
|
|
27
|
+
const input = {};
|
|
28
|
+
for (const [key, prop] of Object.entries(schema.properties)) {
|
|
29
|
+
if (prop.default !== undefined) {
|
|
30
|
+
input[key] = prop.default;
|
|
31
|
+
}
|
|
32
|
+
else if (prop.enum && prop.enum.length > 0) {
|
|
33
|
+
input[key] = prop.enum[seed % prop.enum.length];
|
|
34
|
+
}
|
|
35
|
+
else if (prop.type === "array" && prop.items) {
|
|
36
|
+
const count = 50 + seed * 10;
|
|
37
|
+
const arr = [];
|
|
38
|
+
if (prop.items.type === "number") {
|
|
39
|
+
let price = 100;
|
|
40
|
+
for (let i = 0; i < count; i++) {
|
|
41
|
+
price *= 1 + (rng() - 0.48) * 0.04;
|
|
42
|
+
arr.push(Math.round(price * 100) / 100);
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
else if (prop.items.type === "string") {
|
|
46
|
+
for (let i = 0; i < Math.min(count, 5); i++)
|
|
47
|
+
arr.push(`item-${i}`);
|
|
48
|
+
}
|
|
49
|
+
else if (prop.items.type === "object") {
|
|
50
|
+
for (let i = 0; i < Math.min(count, 3); i++) {
|
|
51
|
+
arr.push(generateTestInput({ properties: prop.items.properties }, seed + i));
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
input[key] = arr;
|
|
55
|
+
}
|
|
56
|
+
else if (prop.type === "number") {
|
|
57
|
+
const min = prop.minimum ?? 1;
|
|
58
|
+
const max = prop.maximum ?? 100;
|
|
59
|
+
input[key] = Math.min(min + seed, max);
|
|
60
|
+
}
|
|
61
|
+
else if (prop.type === "string") {
|
|
62
|
+
input[key] = `test-${seed}`;
|
|
63
|
+
}
|
|
64
|
+
else if (prop.type === "boolean") {
|
|
65
|
+
input[key] = seed % 2 === 0;
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
return input;
|
|
69
|
+
}
|
|
70
|
+
//# sourceMappingURL=generate.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"generate.js","sourceRoot":"","sources":["../../src/testsuite/generate.ts"],"names":[],"mappings":";AAAA;;;;;;;;GAQG;;AAEH,gCAQC;AAED,8CAsCC;AAhDD,SAAgB,UAAU,CAAC,IAAY;IACrC,IAAI,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC;IACjB,OAAO,GAAG,EAAE;QACV,CAAC,GAAG,CAAC,CAAC,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC;QACzB,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;QACzC,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAC/C,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,UAAU,CAAC;IAC/C,CAAC,CAAC;AACJ,CAAC;AAED,SAAgB,iBAAiB,CAAC,MAAW,EAAE,IAAY;IACzD,IAAI,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,UAAU;QAAE,OAAO,EAAE,IAAI,EAAE,cAAc,IAAI,EAAE,EAAE,CAAC;IAEzE,MAAM,GAAG,GAAG,UAAU,CAAC,IAAI,GAAG,UAAU,CAAC,CAAC;IAC1C,MAAM,KAAK,GAAwB,EAAE,CAAC;IACtC,KAAK,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,CAAoB,EAAE,CAAC;QAC/E,IAAI,IAAI,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;YAC/B,KAAK,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC;QAC5B,CAAC;aAAM,IAAI,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC7C,KAAK,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAClD,CAAC;aAAM,IAAI,IAAI,CAAC,IAAI,KAAK,OAAO,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YAC/C,MAAM,KAAK,GAAG,EAAE,GAAG,IAAI,GAAG,EAAE,CAAC;YAC7B,MAAM,GAAG,GAAU,EAAE,CAAC;YACtB,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;gBACjC,IAAI,KAAK,GAAG,GAAG,CAAC;gBAChB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC;oBAC/B,KAAK,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC;oBACnC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,GAAG,CAAC,GAAG,GAAG,CAAC,CAAC;gBAC1C,CAAC;YACH,CAAC;iBAAM,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;gBACxC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE;oBAAE,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;YACrE,CAAC;iBAAM,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;gBACxC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;oBAC5C,GAAG,CAAC,IAAI,CAAC,iBAAiB,CAAC,EAAE,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,UAAU,EAAE,EAAE,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC;gBAC/E,CAAC;YACH,CAAC;YACD,KAAK,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC;QACnB,CAAC;aAAM,IAAI,IAAI,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YAClC,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,IAAI,CAAC,CAAC;YAC9B,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,IAAI,GAAG,CAAC;YAChC,KAAK,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG,IAAI,EAAE,GAAG,CAAC,CAAC;QACzC,CAAC;aAAM,IAAI,IAAI,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YAClC,KAAK,CAAC,GAAG,CAAC,GAAG,QAAQ,IAAI,EAAE,CAAC;QAC9B,CAAC;aAAM,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;YACnC,KAAK,CAAC,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC;QAC9B,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC"}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import type { TestSuite } from "./types.js";
|
|
2
|
+
export declare const TESTSUITE_FILENAME = "testsuite.json";
|
|
3
|
+
export interface LoadOk {
|
|
4
|
+
status: "ok";
|
|
5
|
+
suite: TestSuite;
|
|
6
|
+
path: string;
|
|
7
|
+
}
|
|
8
|
+
/** The file is absent. Stage 1 sets no gate, so this is not an error — the
|
|
9
|
+
* caller decides. §47.5's gate (stage 2) is the thing that turns it into one. */
|
|
10
|
+
export interface LoadAbsent {
|
|
11
|
+
status: "absent";
|
|
12
|
+
path: string;
|
|
13
|
+
}
|
|
14
|
+
export interface LoadInvalid {
|
|
15
|
+
status: "invalid";
|
|
16
|
+
path: string;
|
|
17
|
+
/** Field-level, e.g. `testCases[2].timeout: expected a number, got string`. */
|
|
18
|
+
errors: string[];
|
|
19
|
+
}
|
|
20
|
+
export type LoadResult = LoadOk | LoadAbsent | LoadInvalid;
|
|
21
|
+
export declare function parseTestSuite(text: string, path: string): LoadOk | LoadInvalid;
|
|
22
|
+
export declare function loadTestSuite(geneDir: string): LoadResult;
|
|
23
|
+
//# sourceMappingURL=load.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"load.d.ts","sourceRoot":"","sources":["../../src/testsuite/load.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,SAAS,EAAwB,MAAM,YAAY,CAAC;AAElE,eAAO,MAAM,kBAAkB,mBAAmB,CAAC;AAEnD,MAAM,WAAW,MAAM;IACrB,MAAM,EAAE,IAAI,CAAC;IACb,KAAK,EAAE,SAAS,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;CACd;AAED;iFACiF;AACjF,MAAM,WAAW,UAAU;IACzB,MAAM,EAAE,QAAQ,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,WAAW;IAC1B,MAAM,EAAE,SAAS,CAAC;IAClB,IAAI,EAAE,MAAM,CAAC;IACb,+EAA+E;IAC/E,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB;AAED,MAAM,MAAM,UAAU,GAAG,MAAM,GAAG,UAAU,GAAG,WAAW,CAAC;AAuE3D,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,WAAW,CA2C/E;AAED,wBAAgB,aAAa,CAAC,OAAO,EAAE,MAAM,GAAG,UAAU,CAIzD"}
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.TESTSUITE_FILENAME = void 0;
|
|
4
|
+
exports.parseTestSuite = parseTestSuite;
|
|
5
|
+
exports.loadTestSuite = loadTestSuite;
|
|
6
|
+
const node_fs_1 = require("node:fs");
|
|
7
|
+
const node_path_1 = require("node:path");
|
|
8
|
+
exports.TESTSUITE_FILENAME = "testsuite.json";
|
|
9
|
+
function isPlainObject(v) {
|
|
10
|
+
return typeof v === "object" && v !== null && !Array.isArray(v);
|
|
11
|
+
}
|
|
12
|
+
/**
|
|
13
|
+
* Validation is hand-written rather than delegated to ajv on purpose: the
|
|
14
|
+
* struct is six fields, and the value of this file is the error message. A
|
|
15
|
+
* schema validator would answer "does not match schema" where the author
|
|
16
|
+
* needs "testCases[2].timeout: expected a number, got string".
|
|
17
|
+
*
|
|
18
|
+
* Unknown fields are rejected rather than ignored. A typo'd `expectedOutputs`
|
|
19
|
+
* that is silently dropped means a case that never asserts anything and still
|
|
20
|
+
* reports as passing — the exact failure mode this whole plan exists to close.
|
|
21
|
+
*/
|
|
22
|
+
const CASE_FIELDS = new Set([
|
|
23
|
+
"input",
|
|
24
|
+
"expectedOutput",
|
|
25
|
+
"expectedSchema",
|
|
26
|
+
"timeout",
|
|
27
|
+
]);
|
|
28
|
+
const CONFIG_FIELDS = new Set(["fuzzIterations", "seed"]);
|
|
29
|
+
const SUITE_FIELDS = new Set(["testCases", "config"]);
|
|
30
|
+
function validateCase(raw, at, errors) {
|
|
31
|
+
if (!isPlainObject(raw)) {
|
|
32
|
+
errors.push(`${at}: expected an object, got ${Array.isArray(raw) ? "array" : typeof raw}`);
|
|
33
|
+
return;
|
|
34
|
+
}
|
|
35
|
+
for (const key of Object.keys(raw)) {
|
|
36
|
+
if (!CASE_FIELDS.has(key)) {
|
|
37
|
+
errors.push(`${at}.${key}: unknown field (spec §4.2 TestCase has input, expectedOutput, expectedSchema, timeout)`);
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
if (!("input" in raw)) {
|
|
41
|
+
errors.push(`${at}.input: required (spec §4.2 TestCase)`);
|
|
42
|
+
}
|
|
43
|
+
if ("expectedSchema" in raw && !isPlainObject(raw.expectedSchema)) {
|
|
44
|
+
errors.push(`${at}.expectedSchema: expected a JSON Schema object, got ${typeof raw.expectedSchema}`);
|
|
45
|
+
}
|
|
46
|
+
if ("timeout" in raw) {
|
|
47
|
+
const t = raw.timeout;
|
|
48
|
+
if (typeof t !== "number" || !Number.isFinite(t) || t <= 0) {
|
|
49
|
+
errors.push(`${at}.timeout: expected a positive number of milliseconds, got ${JSON.stringify(t)}`);
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
function validateConfig(raw, errors) {
|
|
54
|
+
if (!isPlainObject(raw)) {
|
|
55
|
+
errors.push(`config: expected an object, got ${typeof raw}`);
|
|
56
|
+
return;
|
|
57
|
+
}
|
|
58
|
+
for (const key of Object.keys(raw)) {
|
|
59
|
+
if (!CONFIG_FIELDS.has(key)) {
|
|
60
|
+
errors.push(`config.${key}: unknown field (spec §4.2 TestConfig has fuzzIterations, seed)`);
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
for (const key of ["fuzzIterations", "seed"]) {
|
|
64
|
+
if (key in raw) {
|
|
65
|
+
const v = raw[key];
|
|
66
|
+
if (typeof v !== "number" || !Number.isInteger(v) || v < 0) {
|
|
67
|
+
errors.push(`config.${key}: expected a non-negative integer, got ${JSON.stringify(v)}`);
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
function parseTestSuite(text, path) {
|
|
73
|
+
let raw;
|
|
74
|
+
try {
|
|
75
|
+
raw = JSON.parse(text);
|
|
76
|
+
}
|
|
77
|
+
catch (e) {
|
|
78
|
+
return { status: "invalid", path, errors: [`not valid JSON — ${e.message}`] };
|
|
79
|
+
}
|
|
80
|
+
const errors = [];
|
|
81
|
+
if (!isPlainObject(raw)) {
|
|
82
|
+
return {
|
|
83
|
+
status: "invalid",
|
|
84
|
+
path,
|
|
85
|
+
errors: [`expected an object at the top level, got ${Array.isArray(raw) ? "array" : typeof raw}`],
|
|
86
|
+
};
|
|
87
|
+
}
|
|
88
|
+
for (const key of Object.keys(raw)) {
|
|
89
|
+
if (!SUITE_FIELDS.has(key)) {
|
|
90
|
+
errors.push(`${key}: unknown field (spec §4.2 TestSuite has testCases, config)`);
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
if (!Array.isArray(raw.testCases)) {
|
|
94
|
+
errors.push(`testCases: required, expected an array (spec §4.2 TestSuite)`);
|
|
95
|
+
}
|
|
96
|
+
else if (raw.testCases.length === 0) {
|
|
97
|
+
errors.push(`testCases: must not be empty — a suite with no cases asserts nothing`);
|
|
98
|
+
}
|
|
99
|
+
else {
|
|
100
|
+
raw.testCases.forEach((c, i) => validateCase(c, `testCases[${i}]`, errors));
|
|
101
|
+
}
|
|
102
|
+
if ("config" in raw)
|
|
103
|
+
validateConfig(raw.config, errors);
|
|
104
|
+
if (errors.length > 0)
|
|
105
|
+
return { status: "invalid", path, errors };
|
|
106
|
+
return {
|
|
107
|
+
status: "ok",
|
|
108
|
+
path,
|
|
109
|
+
suite: {
|
|
110
|
+
testCases: raw.testCases,
|
|
111
|
+
...(isPlainObject(raw.config) ? { config: raw.config } : {}),
|
|
112
|
+
},
|
|
113
|
+
};
|
|
114
|
+
}
|
|
115
|
+
function loadTestSuite(geneDir) {
|
|
116
|
+
const path = (0, node_path_1.join)(geneDir, exports.TESTSUITE_FILENAME);
|
|
117
|
+
if (!(0, node_fs_1.existsSync)(path))
|
|
118
|
+
return { status: "absent", path };
|
|
119
|
+
return parseTestSuite((0, node_fs_1.readFileSync)(path, "utf-8"), path);
|
|
120
|
+
}
|
|
121
|
+
//# sourceMappingURL=load.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"load.js","sourceRoot":"","sources":["../../src/testsuite/load.ts"],"names":[],"mappings":";;;AAiGA,wCA2CC;AAED,sCAIC;AAlJD,qCAAmD;AACnD,yCAAiC;AAGpB,QAAA,kBAAkB,GAAG,gBAAgB,CAAC;AAwBnD,SAAS,aAAa,CAAC,CAAU;IAC/B,OAAO,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;AAClE,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC;IAC1B,OAAO;IACP,gBAAgB;IAChB,gBAAgB;IAChB,SAAS;CACV,CAAC,CAAC;AACH,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC,CAAC,gBAAgB,EAAE,MAAM,CAAC,CAAC,CAAC;AAC1D,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC,CAAC;AAEtD,SAAS,YAAY,CAAC,GAAY,EAAE,EAAU,EAAE,MAAgB;IAC9D,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,EAAE,CAAC;QACxB,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,6BAA6B,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,GAAG,EAAE,CAAC,CAAC;QAC3F,OAAO;IACT,CAAC;IACD,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;QACnC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YAC1B,MAAM,CAAC,IAAI,CACT,GAAG,EAAE,IAAI,GAAG,yFAAyF,CACtG,CAAC;QACJ,CAAC;IACH,CAAC;IACD,IAAI,CAAC,CAAC,OAAO,IAAI,GAAG,CAAC,EAAE,CAAC;QACtB,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,uCAAuC,CAAC,CAAC;IAC5D,CAAC;IACD,IAAI,gBAAgB,IAAI,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,cAAc,CAAC,EAAE,CAAC;QAClE,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,uDAAuD,OAAO,GAAG,CAAC,cAAc,EAAE,CAAC,CAAC;IACvG,CAAC;IACD,IAAI,SAAS,IAAI,GAAG,EAAE,CAAC;QACrB,MAAM,CAAC,GAAG,GAAG,CAAC,OAAO,CAAC;QACtB,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAC3D,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,6DAA6D,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QACrG,CAAC;IACH,CAAC;AACH,CAAC;AAED,SAAS,cAAc,CAAC,GAAY,EAAE,MAAgB;IACpD,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,EAAE,CAAC;QACxB,MAAM,CAAC,IAAI,CAAC,mCAAmC,OAAO,GAAG,EAAE,CAAC,CAAC;QAC7D,OAAO;IACT,CAAC;IACD,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;QACnC,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YAC5B,MAAM,CAAC,IAAI,CAAC,UAAU,GAAG,iEAAiE,CAAC,CAAC;QAC9F,CAAC;IACH,CAAC;IACD,KAAK,MAAM,GAAG,IAAI,CAAC,gBAAgB,EAAE,MAAM,CAAU,EAAE,CAAC;QACtD,IAAI,GAAG,IAAI,GAAG,EAAE,CAAC;YACf,MAAM,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC;YACnB,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC3D,MAAM,CAAC,IAAI,CAAC,UAAU,GAAG,0CAA0C,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YAC1F,CAAC;QACH,CAAC;IACH,CAAC;AACH,CAAC;AAED,SAAgB,cAAc,CAAC,IAAY,EAAE,IAAY;IACvD,IAAI,GAAY,CAAC;IACjB,IAAI,CAAC;QACH,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACzB,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,oBAAqB,CAAW,CAAC,OAAO,EAAE,CAAC,EAAE,CAAC;IAC3F,CAAC;IAED,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,EAAE,CAAC;QACxB,OAAO;YACL,MAAM,EAAE,SAAS;YACjB,IAAI;YACJ,MAAM,EAAE,CAAC,4CAA4C,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,GAAG,EAAE,CAAC;SAClG,CAAC;IACJ,CAAC;IAED,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;QACnC,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YAC3B,MAAM,CAAC,IAAI,CAAC,GAAG,GAAG,6DAA6D,CAAC,CAAC;QACnF,CAAC;IACH,CAAC;IAED,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;QAClC,MAAM,CAAC,IAAI,CAAC,8DAA8D,CAAC,CAAC;IAC9E,CAAC;SAAM,IAAI,GAAG,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACtC,MAAM,CAAC,IAAI,CAAC,sEAAsE,CAAC,CAAC;IACtF,CAAC;SAAM,CAAC;QACN,GAAG,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,YAAY,CAAC,CAAC,EAAE,aAAa,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC,CAAC;IAC9E,CAAC;IAED,IAAI,QAAQ,IAAI,GAAG;QAAE,cAAc,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAExD,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;IAElE,OAAO;QACL,MAAM,EAAE,IAAI;QACZ,IAAI;QACJ,KAAK,EAAE;YACL,SAAS,EAAE,GAAG,CAAC,SAAuB;YACtC,GAAG,CAAC,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,GAAG,CAAC,MAAoB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAC3E;KACF,CAAC;AACJ,CAAC;AAED,SAAgB,aAAa,CAAC,OAAe;IAC3C,MAAM,IAAI,GAAG,IAAA,gBAAI,EAAC,OAAO,EAAE,0BAAkB,CAAC,CAAC;IAC/C,IAAI,CAAC,IAAA,oBAAU,EAAC,IAAI,CAAC;QAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;IACzD,OAAO,cAAc,CAAC,IAAA,sBAAY,EAAC,IAAI,EAAE,OAAO,CAAC,EAAE,IAAI,CAAC,CAAC;AAC3D,CAAC"}
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
import Ajv from "ajv";
|
|
2
|
+
import type { ValidateFunction } from "ajv";
|
|
3
|
+
import type { ClassifiedCase, TestCase, TestResult, TestSuite } from "./types.js";
|
|
4
|
+
/** §47.5 fixes the publishing gate's property test at 10 random inputs. */
|
|
5
|
+
export declare const DEFAULT_FUZZ_ITERATIONS = 10;
|
|
6
|
+
/**
|
|
7
|
+
* What running one input against the Gene produced. The runner is injected
|
|
8
|
+
* rather than imported so this module stays free of the native binding: the
|
|
9
|
+
* caller already owns a compiled Gene and a fuel policy, and the T1 rules are
|
|
10
|
+
* about interpreting outcomes, not about producing them.
|
|
11
|
+
*/
|
|
12
|
+
export interface RunOutcome {
|
|
13
|
+
/** The Gene ran to completion and returned an output. */
|
|
14
|
+
success: boolean;
|
|
15
|
+
output?: unknown;
|
|
16
|
+
/** Present when success is false. A *controlled* rejection also lands here. */
|
|
17
|
+
errorMessage?: string;
|
|
18
|
+
/** True when the Gene did not reject but died — trap, fuel exhaustion,
|
|
19
|
+
* timeout. §47.5's negative case is about handling illegal input
|
|
20
|
+
* "correctly", and dying is not handling it. */
|
|
21
|
+
crashed?: boolean;
|
|
22
|
+
durationMs?: number;
|
|
23
|
+
}
|
|
24
|
+
export type GeneRunner = (input: unknown, timeoutMs?: number) => RunOutcome;
|
|
25
|
+
/**
|
|
26
|
+
* Classify a case as positive or negative from the input alone.
|
|
27
|
+
*
|
|
28
|
+
* spec §4.2's TestCase has no field saying which it is, and §47.5 words the
|
|
29
|
+
* negative case as "gene can correctly handle illegal input" — legality being
|
|
30
|
+
* decided by the Gene's own inputSchema. So the classification is derived:
|
|
31
|
+
* an input the schema accepts is a positive case, one it rejects is negative.
|
|
32
|
+
*
|
|
33
|
+
* The consequence worth stating: a positive case whose input is accidentally
|
|
34
|
+
* malformed silently becomes a negative case. That is why the reason string
|
|
35
|
+
* carries the validator's complaint — an author who meant it to be positive
|
|
36
|
+
* sees exactly which field disqualified it.
|
|
37
|
+
*/
|
|
38
|
+
export declare function classify(testCases: TestCase[], inputValidator: ValidateFunction | null): ClassifiedCase[];
|
|
39
|
+
/**
|
|
40
|
+
* A positive case passes when the Gene runs and its output satisfies whatever
|
|
41
|
+
* the case asserts: `expectedOutput` exactly, `expectedSchema` structurally,
|
|
42
|
+
* or — when the case asserts neither — the Gene's declared outputSchema.
|
|
43
|
+
*
|
|
44
|
+
* The fallback matters. Without it a case carrying only an input would pass on
|
|
45
|
+
* any output at all, including one that violates the Gene's own contract, and
|
|
46
|
+
* the suite would look populated while asserting nothing.
|
|
47
|
+
*/
|
|
48
|
+
export declare function runPositive(c: ClassifiedCase, run: GeneRunner, outputValidator: ValidateFunction | null, ajv: Ajv): TestResult;
|
|
49
|
+
/**
|
|
50
|
+
* A negative case: the Gene is fed input its own inputSchema forbids.
|
|
51
|
+
*
|
|
52
|
+
* §47.5 asks that the Gene "correctly handle illegal input" without saying
|
|
53
|
+
* what handling looks like, and measuring four real Genes shows why it cannot
|
|
54
|
+
* be decided from the outcome alone:
|
|
55
|
+
*
|
|
56
|
+
* grammar-checker throws — `not a function`, a TypeError from inside
|
|
57
|
+
* source-linker returns — echoes the illegal value straight back
|
|
58
|
+
* url-extractor returns — an empty result
|
|
59
|
+
* json-validator returns — `{ valid: false }`, which IS correct handling
|
|
60
|
+
*
|
|
61
|
+
* "Returned output" therefore cannot mean acceptance: for json-validator it is
|
|
62
|
+
* the right answer, and for source-linker it is the bug. The difference is not
|
|
63
|
+
* in the outcome, it is in what the author intended — so the author has to say,
|
|
64
|
+
* using the fields spec §4.2 already provides.
|
|
65
|
+
*
|
|
66
|
+
* The rules, in order:
|
|
67
|
+
*
|
|
68
|
+
* 1. Crashed — fail. Dying is not handling, whatever was intended.
|
|
69
|
+
* 2. Errored without crashing — pass. The Gene refused, deliberately.
|
|
70
|
+
* 3. Returned, and the case declares expectedOutput/expectedSchema that
|
|
71
|
+
* matches — pass. This is the author saying "rejection looks like this".
|
|
72
|
+
* 4. Returned, and the declaration does not match — fail.
|
|
73
|
+
* 5. Returned, and the case declares nothing — fail, and say why. Silent
|
|
74
|
+
* acceptance and correct handling are indistinguishable here; refusing to
|
|
75
|
+
* guess is the whole point, since guessing "pass" is what lets a
|
|
76
|
+
* source-linker through.
|
|
77
|
+
*/
|
|
78
|
+
export declare function runNegative(c: ClassifiedCase, run: GeneRunner, ajv: Ajv): TestResult;
|
|
79
|
+
/**
|
|
80
|
+
* §47.4 Schema Legality, run as §47.5's third T1 requirement: generated inputs
|
|
81
|
+
* must always produce output conforming to outputSchema.
|
|
82
|
+
*
|
|
83
|
+
* The seed is what makes a failure actionable — the same seed replays the same
|
|
84
|
+
* inputs, so a property failure can be reproduced rather than merely observed.
|
|
85
|
+
* When the suite gives no seed one is derived from nothing but a constant, so
|
|
86
|
+
* two runs of an unseeded suite still agree; randomness across runs would make
|
|
87
|
+
* the gate flaky and the flake would be blamed on the Gene.
|
|
88
|
+
*/
|
|
89
|
+
export declare function runProperty(suite: TestSuite, inputSchema: Record<string, unknown> | null, run: GeneRunner, outputValidator: ValidateFunction | null): TestResult;
|
|
90
|
+
export interface T1Report {
|
|
91
|
+
results: TestResult[];
|
|
92
|
+
classified: ClassifiedCase[];
|
|
93
|
+
/** The three §47.5 T1 requirements, each satisfied or not. */
|
|
94
|
+
requirements: {
|
|
95
|
+
positive: boolean;
|
|
96
|
+
negative: boolean;
|
|
97
|
+
property: boolean;
|
|
98
|
+
};
|
|
99
|
+
/** True only when all three hold. This is what stage 2's publish gate reads. */
|
|
100
|
+
gatePassed: boolean;
|
|
101
|
+
}
|
|
102
|
+
export declare function runT1(suite: TestSuite, opts: {
|
|
103
|
+
inputSchema: Record<string, unknown> | null;
|
|
104
|
+
outputSchema: Record<string, unknown> | null;
|
|
105
|
+
run: GeneRunner;
|
|
106
|
+
}): T1Report;
|
|
107
|
+
/** Stable id for a report, for callers that persist results. */
|
|
108
|
+
export declare function newTestRunId(): string;
|
|
109
|
+
//# sourceMappingURL=run.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../src/testsuite/run.ts"],"names":[],"mappings":"AAAA,OAAO,GAAG,MAAM,KAAK,CAAC;AACtB,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,KAAK,CAAC;AAE5C,OAAO,KAAK,EACV,cAAc,EACd,QAAQ,EACR,UAAU,EACV,SAAS,EACV,MAAM,YAAY,CAAC;AAGpB,2EAA2E;AAC3E,eAAO,MAAM,uBAAuB,KAAK,CAAC;AAE1C;;;;;GAKG;AACH,MAAM,WAAW,UAAU;IACzB,yDAAyD;IACzD,OAAO,EAAE,OAAO,CAAC;IACjB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,+EAA+E;IAC/E,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB;;oDAEgD;IAChD,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,MAAM,UAAU,GAAG,CAAC,KAAK,EAAE,OAAO,EAAE,SAAS,CAAC,EAAE,MAAM,KAAK,UAAU,CAAC;AAE5E;;;;;;;;;;;;GAYG;AACH,wBAAgB,QAAQ,CACtB,SAAS,EAAE,QAAQ,EAAE,EACrB,cAAc,EAAE,gBAAgB,GAAG,IAAI,GACtC,cAAc,EAAE,CA2BlB;AA4CD;;;;;;;;GAQG;AACH,wBAAgB,WAAW,CACzB,CAAC,EAAE,cAAc,EACjB,GAAG,EAAE,UAAU,EACf,eAAe,EAAE,gBAAgB,GAAG,IAAI,EACxC,GAAG,EAAE,GAAG,GACP,UAAU,CA2CZ;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,wBAAgB,WAAW,CACzB,CAAC,EAAE,cAAc,EACjB,GAAG,EAAE,UAAU,EACf,GAAG,EAAE,GAAG,GACP,UAAU,CAmDZ;AAED;;;;;;;;;GASG;AACH,wBAAgB,WAAW,CACzB,KAAK,EAAE,SAAS,EAChB,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,EAC3C,GAAG,EAAE,UAAU,EACf,eAAe,EAAE,gBAAgB,GAAG,IAAI,GACvC,UAAU,CAkCZ;AAED,MAAM,WAAW,QAAQ;IACvB,OAAO,EAAE,UAAU,EAAE,CAAC;IACtB,UAAU,EAAE,cAAc,EAAE,CAAC;IAC7B,8DAA8D;IAC9D,YAAY,EAAE;QACZ,QAAQ,EAAE,OAAO,CAAC;QAClB,QAAQ,EAAE,OAAO,CAAC;QAClB,QAAQ,EAAE,OAAO,CAAC;KACnB,CAAC;IACF,gFAAgF;IAChF,UAAU,EAAE,OAAO,CAAC;CACrB;AAED,wBAAgB,KAAK,CACnB,KAAK,EAAE,SAAS,EAChB,IAAI,EAAE;IACJ,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;IAC5C,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;IAC7C,GAAG,EAAE,UAAU,CAAC;CACjB,GACA,QAAQ,CA2DV;AAED,gEAAgE;AAChE,wBAAgB,YAAY,IAAI,MAAM,CAErC"}
|
|
@@ -0,0 +1,298 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.DEFAULT_FUZZ_ITERATIONS = void 0;
|
|
7
|
+
exports.classify = classify;
|
|
8
|
+
exports.runPositive = runPositive;
|
|
9
|
+
exports.runNegative = runNegative;
|
|
10
|
+
exports.runProperty = runProperty;
|
|
11
|
+
exports.runT1 = runT1;
|
|
12
|
+
exports.newTestRunId = newTestRunId;
|
|
13
|
+
const ajv_1 = __importDefault(require("ajv"));
|
|
14
|
+
const node_crypto_1 = require("node:crypto");
|
|
15
|
+
const generate_js_1 = require("./generate.js");
|
|
16
|
+
/** §47.5 fixes the publishing gate's property test at 10 random inputs. */
|
|
17
|
+
exports.DEFAULT_FUZZ_ITERATIONS = 10;
|
|
18
|
+
/**
|
|
19
|
+
* Classify a case as positive or negative from the input alone.
|
|
20
|
+
*
|
|
21
|
+
* spec §4.2's TestCase has no field saying which it is, and §47.5 words the
|
|
22
|
+
* negative case as "gene can correctly handle illegal input" — legality being
|
|
23
|
+
* decided by the Gene's own inputSchema. So the classification is derived:
|
|
24
|
+
* an input the schema accepts is a positive case, one it rejects is negative.
|
|
25
|
+
*
|
|
26
|
+
* The consequence worth stating: a positive case whose input is accidentally
|
|
27
|
+
* malformed silently becomes a negative case. That is why the reason string
|
|
28
|
+
* carries the validator's complaint — an author who meant it to be positive
|
|
29
|
+
* sees exactly which field disqualified it.
|
|
30
|
+
*/
|
|
31
|
+
function classify(testCases, inputValidator) {
|
|
32
|
+
return testCases.map((testCase, index) => {
|
|
33
|
+
if (!inputValidator) {
|
|
34
|
+
// No inputSchema to judge against. Everything is a positive case, and
|
|
35
|
+
// the §47.5 gate will report the missing negative rather than inventing
|
|
36
|
+
// one — see requirementsMet().
|
|
37
|
+
return {
|
|
38
|
+
index,
|
|
39
|
+
kind: "positive",
|
|
40
|
+
testCase,
|
|
41
|
+
reason: "gene declares no inputSchema, so no input can be illegal",
|
|
42
|
+
};
|
|
43
|
+
}
|
|
44
|
+
const isLegalInput = inputValidator(testCase.input);
|
|
45
|
+
if (isLegalInput) {
|
|
46
|
+
return { index, kind: "positive", testCase, reason: "input conforms to inputSchema" };
|
|
47
|
+
}
|
|
48
|
+
const complaint = (inputValidator.errors ?? [])
|
|
49
|
+
.map((e) => `${e.instancePath || "(root)"} ${e.message}`)
|
|
50
|
+
.join("; ");
|
|
51
|
+
return {
|
|
52
|
+
index,
|
|
53
|
+
kind: "negative",
|
|
54
|
+
testCase,
|
|
55
|
+
reason: `input violates inputSchema — ${complaint || "no detail"}`,
|
|
56
|
+
};
|
|
57
|
+
});
|
|
58
|
+
}
|
|
59
|
+
/**
|
|
60
|
+
* Serialize for comparison with object keys in a stable order.
|
|
61
|
+
*
|
|
62
|
+
* `expectedOutput` was compared with plain JSON.stringify, which is key-order
|
|
63
|
+
* sensitive: a case declaring `{formatted, changed, language}` failed against a
|
|
64
|
+
* Gene returning `{changed, formatted, language}` — same object by every
|
|
65
|
+
* meaning JavaScript gives the word, unequal as text. That is a property of the
|
|
66
|
+
* serializer, not of the Gene, and every author writing an expectedOutput by
|
|
67
|
+
* hand would hit it. Found by hitting it.
|
|
68
|
+
*
|
|
69
|
+
* Array order is preserved: in an array it carries meaning.
|
|
70
|
+
*/
|
|
71
|
+
function canonical(value) {
|
|
72
|
+
const walk = (v) => {
|
|
73
|
+
if (Array.isArray(v))
|
|
74
|
+
return v.map(walk);
|
|
75
|
+
if (v && typeof v === "object") {
|
|
76
|
+
const out = {};
|
|
77
|
+
for (const k of Object.keys(v).sort()) {
|
|
78
|
+
out[k] = walk(v[k]);
|
|
79
|
+
}
|
|
80
|
+
return out;
|
|
81
|
+
}
|
|
82
|
+
return v;
|
|
83
|
+
};
|
|
84
|
+
return JSON.stringify(walk(value));
|
|
85
|
+
}
|
|
86
|
+
function fail(testId, started, reason, details) {
|
|
87
|
+
return {
|
|
88
|
+
testId,
|
|
89
|
+
level: "T1",
|
|
90
|
+
passed: false,
|
|
91
|
+
duration: Date.now() - started,
|
|
92
|
+
failureReason: reason,
|
|
93
|
+
...(details ? { details } : {}),
|
|
94
|
+
};
|
|
95
|
+
}
|
|
96
|
+
function pass(testId, started, details) {
|
|
97
|
+
return { testId, level: "T1", passed: true, duration: Date.now() - started, details };
|
|
98
|
+
}
|
|
99
|
+
/**
|
|
100
|
+
* A positive case passes when the Gene runs and its output satisfies whatever
|
|
101
|
+
* the case asserts: `expectedOutput` exactly, `expectedSchema` structurally,
|
|
102
|
+
* or — when the case asserts neither — the Gene's declared outputSchema.
|
|
103
|
+
*
|
|
104
|
+
* The fallback matters. Without it a case carrying only an input would pass on
|
|
105
|
+
* any output at all, including one that violates the Gene's own contract, and
|
|
106
|
+
* the suite would look populated while asserting nothing.
|
|
107
|
+
*/
|
|
108
|
+
function runPositive(c, run, outputValidator, ajv) {
|
|
109
|
+
const started = Date.now();
|
|
110
|
+
const id = `t1-positive-${c.index}`;
|
|
111
|
+
const outcome = run(c.testCase.input, c.testCase.timeout);
|
|
112
|
+
if (!outcome.success) {
|
|
113
|
+
return fail(id, started, `Gene rejected a legal input: ${outcome.errorMessage ?? "no message"}`, c.reason);
|
|
114
|
+
}
|
|
115
|
+
if ("expectedOutput" in c.testCase) {
|
|
116
|
+
const got = canonical(outcome.output);
|
|
117
|
+
const want = canonical(c.testCase.expectedOutput);
|
|
118
|
+
if (got !== want) {
|
|
119
|
+
return fail(id, started, "output does not equal expectedOutput", `expected ${want}, got ${got}`);
|
|
120
|
+
}
|
|
121
|
+
return pass(id, started, "output matches expectedOutput exactly");
|
|
122
|
+
}
|
|
123
|
+
if (c.testCase.expectedSchema) {
|
|
124
|
+
let validate;
|
|
125
|
+
try {
|
|
126
|
+
validate = ajv.compile(c.testCase.expectedSchema);
|
|
127
|
+
}
|
|
128
|
+
catch (e) {
|
|
129
|
+
return fail(id, started, `expectedSchema is not a valid JSON Schema: ${e.message}`);
|
|
130
|
+
}
|
|
131
|
+
if (!validate(outcome.output)) {
|
|
132
|
+
return fail(id, started, "output does not conform to expectedSchema", JSON.stringify(validate.errors));
|
|
133
|
+
}
|
|
134
|
+
return pass(id, started, "output conforms to expectedSchema");
|
|
135
|
+
}
|
|
136
|
+
if (outputValidator) {
|
|
137
|
+
if (!outputValidator(outcome.output)) {
|
|
138
|
+
return fail(id, started, "output does not conform to the Gene's outputSchema", JSON.stringify(outputValidator.errors));
|
|
139
|
+
}
|
|
140
|
+
return pass(id, started, "output conforms to the Gene's outputSchema");
|
|
141
|
+
}
|
|
142
|
+
return fail(id, started, "case asserts nothing: no expectedOutput, no expectedSchema, and the Gene declares no outputSchema");
|
|
143
|
+
}
|
|
144
|
+
/**
|
|
145
|
+
* A negative case: the Gene is fed input its own inputSchema forbids.
|
|
146
|
+
*
|
|
147
|
+
* §47.5 asks that the Gene "correctly handle illegal input" without saying
|
|
148
|
+
* what handling looks like, and measuring four real Genes shows why it cannot
|
|
149
|
+
* be decided from the outcome alone:
|
|
150
|
+
*
|
|
151
|
+
* grammar-checker throws — `not a function`, a TypeError from inside
|
|
152
|
+
* source-linker returns — echoes the illegal value straight back
|
|
153
|
+
* url-extractor returns — an empty result
|
|
154
|
+
* json-validator returns — `{ valid: false }`, which IS correct handling
|
|
155
|
+
*
|
|
156
|
+
* "Returned output" therefore cannot mean acceptance: for json-validator it is
|
|
157
|
+
* the right answer, and for source-linker it is the bug. The difference is not
|
|
158
|
+
* in the outcome, it is in what the author intended — so the author has to say,
|
|
159
|
+
* using the fields spec §4.2 already provides.
|
|
160
|
+
*
|
|
161
|
+
* The rules, in order:
|
|
162
|
+
*
|
|
163
|
+
* 1. Crashed — fail. Dying is not handling, whatever was intended.
|
|
164
|
+
* 2. Errored without crashing — pass. The Gene refused, deliberately.
|
|
165
|
+
* 3. Returned, and the case declares expectedOutput/expectedSchema that
|
|
166
|
+
* matches — pass. This is the author saying "rejection looks like this".
|
|
167
|
+
* 4. Returned, and the declaration does not match — fail.
|
|
168
|
+
* 5. Returned, and the case declares nothing — fail, and say why. Silent
|
|
169
|
+
* acceptance and correct handling are indistinguishable here; refusing to
|
|
170
|
+
* guess is the whole point, since guessing "pass" is what lets a
|
|
171
|
+
* source-linker through.
|
|
172
|
+
*/
|
|
173
|
+
function runNegative(c, run, ajv) {
|
|
174
|
+
const started = Date.now();
|
|
175
|
+
const id = `t1-negative-${c.index}`;
|
|
176
|
+
const outcome = run(c.testCase.input, c.testCase.timeout);
|
|
177
|
+
if (outcome.crashed) {
|
|
178
|
+
// ADR-333: a crash fails not because it is necessarily a bug, but because
|
|
179
|
+
// it is necessarily unusable. A trap reaching the caller carries no
|
|
180
|
+
// information — the caller cannot tell deliberate refusal from defect, so
|
|
181
|
+
// nothing downstream can act on it correctly. §47.5 asks for "correctly
|
|
182
|
+
// handle", and a signal the caller cannot handle is not handling completed.
|
|
183
|
+
return fail(id, started, `Gene crashed on illegal input instead of handling it: ${outcome.errorMessage ?? "no message"}`, c.reason);
|
|
184
|
+
}
|
|
185
|
+
if (!outcome.success) {
|
|
186
|
+
return pass(id, started, `Gene refused illegal input: ${outcome.errorMessage ?? "no message"}`);
|
|
187
|
+
}
|
|
188
|
+
if ("expectedOutput" in c.testCase) {
|
|
189
|
+
const got = canonical(outcome.output);
|
|
190
|
+
const want = canonical(c.testCase.expectedOutput);
|
|
191
|
+
return got === want
|
|
192
|
+
? pass(id, started, "Gene handled illegal input exactly as the case declares")
|
|
193
|
+
: fail(id, started, "output does not equal the declared expectedOutput", `expected ${want}, got ${got}`);
|
|
194
|
+
}
|
|
195
|
+
if (c.testCase.expectedSchema) {
|
|
196
|
+
let validate;
|
|
197
|
+
try {
|
|
198
|
+
validate = ajv.compile(c.testCase.expectedSchema);
|
|
199
|
+
}
|
|
200
|
+
catch (e) {
|
|
201
|
+
return fail(id, started, `expectedSchema is not a valid JSON Schema: ${e.message}`);
|
|
202
|
+
}
|
|
203
|
+
return validate(outcome.output)
|
|
204
|
+
? pass(id, started, "Gene handled illegal input in the declared shape")
|
|
205
|
+
: fail(id, started, "output does not conform to the declared expectedSchema", JSON.stringify(validate.errors));
|
|
206
|
+
}
|
|
207
|
+
return fail(id, started, "Gene returned output for illegal input and the case declares no expected shape", `${c.reason}; output was ${JSON.stringify(outcome.output).slice(0, 200)}. ` +
|
|
208
|
+
`If returning this is correct handling, declare it with expectedOutput or expectedSchema; ` +
|
|
209
|
+
`otherwise the Gene is silently accepting input its own schema forbids.`);
|
|
210
|
+
}
|
|
211
|
+
/**
|
|
212
|
+
* §47.4 Schema Legality, run as §47.5's third T1 requirement: generated inputs
|
|
213
|
+
* must always produce output conforming to outputSchema.
|
|
214
|
+
*
|
|
215
|
+
* The seed is what makes a failure actionable — the same seed replays the same
|
|
216
|
+
* inputs, so a property failure can be reproduced rather than merely observed.
|
|
217
|
+
* When the suite gives no seed one is derived from nothing but a constant, so
|
|
218
|
+
* two runs of an unseeded suite still agree; randomness across runs would make
|
|
219
|
+
* the gate flaky and the flake would be blamed on the Gene.
|
|
220
|
+
*/
|
|
221
|
+
function runProperty(suite, inputSchema, run, outputValidator) {
|
|
222
|
+
const started = Date.now();
|
|
223
|
+
const id = "t1-property-schema-legality";
|
|
224
|
+
const iterations = suite.config?.fuzzIterations ?? exports.DEFAULT_FUZZ_ITERATIONS;
|
|
225
|
+
const seed = suite.config?.seed ?? 1;
|
|
226
|
+
if (iterations === 0) {
|
|
227
|
+
return fail(id, started, "config.fuzzIterations is 0 — the property test asserts nothing");
|
|
228
|
+
}
|
|
229
|
+
if (!outputValidator) {
|
|
230
|
+
return fail(id, started, "Gene declares no outputSchema, so schema legality cannot be asserted");
|
|
231
|
+
}
|
|
232
|
+
for (let i = 0; i < iterations; i++) {
|
|
233
|
+
const input = (0, generate_js_1.generateTestInput)(inputSchema, seed + i);
|
|
234
|
+
const outcome = run(input);
|
|
235
|
+
if (!outcome.success) {
|
|
236
|
+
return fail(id, started, `iteration ${i + 1}/${iterations} did not produce output: ${outcome.errorMessage ?? "no message"}`, `reproduce with seed ${seed}, input ${JSON.stringify(input).slice(0, 200)}`);
|
|
237
|
+
}
|
|
238
|
+
if (!outputValidator(outcome.output)) {
|
|
239
|
+
return fail(id, started, `iteration ${i + 1}/${iterations} produced output violating outputSchema`, `reproduce with seed ${seed}, input ${JSON.stringify(input).slice(0, 200)}`);
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
return pass(id, started, `${iterations} generated inputs all produced schema-legal output (seed ${seed})`);
|
|
243
|
+
}
|
|
244
|
+
function runT1(suite, opts) {
|
|
245
|
+
const ajv = new ajv_1.default({ allErrors: true, strict: false });
|
|
246
|
+
let inputValidator = null;
|
|
247
|
+
let outputValidator = null;
|
|
248
|
+
try {
|
|
249
|
+
if (opts.inputSchema)
|
|
250
|
+
inputValidator = ajv.compile(opts.inputSchema);
|
|
251
|
+
}
|
|
252
|
+
catch {
|
|
253
|
+
inputValidator = null;
|
|
254
|
+
}
|
|
255
|
+
try {
|
|
256
|
+
if (opts.outputSchema)
|
|
257
|
+
outputValidator = ajv.compile(opts.outputSchema);
|
|
258
|
+
}
|
|
259
|
+
catch {
|
|
260
|
+
outputValidator = null;
|
|
261
|
+
}
|
|
262
|
+
const classified = classify(suite.testCases, inputValidator);
|
|
263
|
+
const results = [];
|
|
264
|
+
for (const c of classified) {
|
|
265
|
+
results.push(c.kind === "positive"
|
|
266
|
+
? runPositive(c, opts.run, outputValidator, ajv)
|
|
267
|
+
: runNegative(c, opts.run, ajv));
|
|
268
|
+
}
|
|
269
|
+
const property = runProperty(suite, opts.inputSchema, opts.run, outputValidator);
|
|
270
|
+
results.push(property);
|
|
271
|
+
const hasPassingPositive = classified.some((c, i) => c.kind === "positive" && results[i].passed);
|
|
272
|
+
const hasPassingNegative = classified.some((c, i) => c.kind === "negative" && results[i].passed);
|
|
273
|
+
const requirements = {
|
|
274
|
+
positive: hasPassingPositive,
|
|
275
|
+
negative: hasPassingNegative,
|
|
276
|
+
property: property.passed,
|
|
277
|
+
};
|
|
278
|
+
// §47.5 sets a floor on coverage — "at least 1 positive", "at least 1
|
|
279
|
+
// negative" — not permission to ship failing assertions. A suite with three
|
|
280
|
+
// negatives where two fail still satisfies the letter of the requirement, and
|
|
281
|
+
// that is how a case an author wrote, and a Gene does not honour, could reach
|
|
282
|
+
// the registry. Every declared case has to pass as well.
|
|
283
|
+
const hasFailingCase = results.some((r) => !r.passed);
|
|
284
|
+
return {
|
|
285
|
+
results,
|
|
286
|
+
classified,
|
|
287
|
+
requirements,
|
|
288
|
+
gatePassed: requirements.positive &&
|
|
289
|
+
requirements.negative &&
|
|
290
|
+
requirements.property &&
|
|
291
|
+
!hasFailingCase,
|
|
292
|
+
};
|
|
293
|
+
}
|
|
294
|
+
/** Stable id for a report, for callers that persist results. */
|
|
295
|
+
function newTestRunId() {
|
|
296
|
+
return (0, node_crypto_1.randomUUID)();
|
|
297
|
+
}
|
|
298
|
+
//# sourceMappingURL=run.js.map
|