@arjunkhera/atlas 0.3.41 → 0.3.42
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/plugin.json +1 -1
- package/door/cli.mjs +3 -3
- package/door/kit-releases.json +4 -0
- package/door/lib/tests.mjs +3 -0
- package/package.json +1 -1
- package/skills/tests/SKILL.md +25 -0
- package/tests/bridge.mjs +91 -0
- package/tests/contract.py +893 -0
- package/tests/drivers/index.mjs +3 -0
- package/tests/link-check.mjs +221 -9
- package/tests/tests-yaml.mjs +13 -4
- package/tests/verdict.mjs +8 -4
package/door/cli.mjs
CHANGED
|
@@ -103,7 +103,7 @@ const HELP = `atlas — the door into a repo's Atlas files
|
|
|
103
103
|
--evidence <dir> where the evidence files go; default <area>/test/evidence
|
|
104
104
|
--scenario <id> run one scenario only
|
|
105
105
|
|
|
106
|
-
atlas tests rerun --root <area> --run <id> list the scenario test files whose first attempt did not end well
|
|
106
|
+
atlas tests rerun --root <area> --run <id> [--language python] list the scenario test files whose first attempt did not end well
|
|
107
107
|
--tests <dir> the test folder of the area; default test. The test files are in <dir>/scenarios
|
|
108
108
|
--evidence <dir> where the evidence files are; default <area>/test/evidence
|
|
109
109
|
|
|
@@ -146,7 +146,7 @@ export const FLAGS = Object.freeze({
|
|
|
146
146
|
ste: { share: 'switch', terms: 'value' },
|
|
147
147
|
design: { tracker: 'value', out: 'value', draft: 'switch', root: 'optional' },
|
|
148
148
|
view: { product: 'value', name: 'value', out: 'value' },
|
|
149
|
-
tests: { root: 'optional', halves: 'value', evidence: 'value', 'guards-from': 'value', tests: 'value', 'dry-run': 'switch', run: 'value', own: 'value', base: 'value', covers: 'switch', findings: 'value', repo: 'value', max: 'value', bot: 'value', since: 'value', out: 'value', text: 'switch', json: 'switch', scenario: 'value' },
|
|
149
|
+
tests: { root: 'optional', halves: 'value', evidence: 'value', 'guards-from': 'value', tests: 'value', 'dry-run': 'switch', run: 'value', own: 'value', base: 'value', covers: 'switch', findings: 'value', repo: 'value', max: 'value', bot: 'value', since: 'value', out: 'value', text: 'switch', json: 'switch', scenario: 'value', language: 'value' },
|
|
150
150
|
upkeep: { root: 'optional', event: 'value', json: 'switch' },
|
|
151
151
|
judge: { tasks: 'value', verdict: 'value', seal: 'value', runs: 'value', out: 'value', repo: 'value', budget: 'value', model: 'value' },
|
|
152
152
|
install: { local: 'switch', from: 'value', 'skip-global': 'switch', yes: 'switch' },
|
|
@@ -347,7 +347,7 @@ async function testsCommand(chosen) {
|
|
|
347
347
|
if (what === 'sweep') return testsSweep({ evidence: chosen.evidence, since: chosen.since, base: chosen.base, out: chosen.out, text: Boolean(chosen.text) }, root, line);
|
|
348
348
|
if (what === 'covers') return testsCovers({ itemId: chosen._[2], evidence: chosen.evidence, json: Boolean(chosen.json) }, root, line);
|
|
349
349
|
if (what === 'kept') return (await runKept({ root, evidenceDir: chosen.evidence, only: chosen.scenario ?? null, write: (text) => process.stdout.write(text) })).exit;
|
|
350
|
-
if (what === 'rerun') return testsRerun({ run: chosen.run, evidence: chosen.evidence, tests: chosen.tests }, root, line);
|
|
350
|
+
if (what === 'rerun') return testsRerun({ run: chosen.run, evidence: chosen.evidence, tests: chosen.tests, language: chosen.language ?? 'javascript' }, root, line);
|
|
351
351
|
if (what === 'issues') return testsIssues({ findings: chosen.findings, repo: chosen.repo, max: chosen.max, bot: chosen.bot, dryRun: Boolean(chosen['dry-run']), args: [...chosen._, ...Object.values(chosen).filter((v) => typeof v === 'string')] }, line);
|
|
352
352
|
throw new Error(`there is no tests verb "${what ?? ''}". Use check, write, proof, verdict, named, sweep, covers, kept, issues or rerun.`);
|
|
353
353
|
}
|
package/door/kit-releases.json
CHANGED
|
@@ -44,6 +44,10 @@
|
|
|
44
44
|
{
|
|
45
45
|
"kit_sha256": "ea6dc64f5ae4b5caca925a9613c829f5738d16b21d2664434dada208027b60b6",
|
|
46
46
|
"first_package": "0.3.38"
|
|
47
|
+
},
|
|
48
|
+
{
|
|
49
|
+
"kit_sha256": "8d6ae3c4dede74bafe292a992de3ef650e5c5566fb282081a9de23b8d30d9025",
|
|
50
|
+
"first_package": "0.3.39"
|
|
47
51
|
}
|
|
48
52
|
]
|
|
49
53
|
}
|
package/door/lib/tests.mjs
CHANGED
|
@@ -31,9 +31,12 @@ export class KitRefusal extends Error {
|
|
|
31
31
|
constructor(message) { super(message); this.name = 'KitRefusal'; }
|
|
32
32
|
}
|
|
33
33
|
|
|
34
|
+
// Python bytecode (__pycache__) is never part of the kit: a dev tree that ran a
|
|
35
|
+
// Python test may hold it, and it must not reach a repo.
|
|
34
36
|
function filesUnder(dir, prefix = '') {
|
|
35
37
|
const out = [];
|
|
36
38
|
for (const name of readdirSync(dir).sort()) {
|
|
39
|
+
if (name === '__pycache__') continue;
|
|
37
40
|
const full = join(dir, name);
|
|
38
41
|
if (statSync(full).isDirectory()) out.push(...filesUnder(full, `${prefix}${name}/`));
|
|
39
42
|
else out.push(`${prefix}${name}`);
|
package/package.json
CHANGED
package/skills/tests/SKILL.md
CHANGED
|
@@ -358,6 +358,31 @@ not counted, and the test passes.
|
|
|
358
358
|
After the product is ready, a lost connection, a server exit or a time-out is
|
|
359
359
|
`fail`. `blocked` is for a guard refusal, a start-up fault and the first-call check.
|
|
360
360
|
|
|
361
|
+
### A test in Python
|
|
362
|
+
|
|
363
|
+
Use the Python library when the runner of the area is Python. Its header,
|
|
364
|
+
`test/atlas/contract.py`, holds a worked example.
|
|
365
|
+
|
|
366
|
+
1. Name the file `test/scenarios/<name>_test.py`. List a Python runner in
|
|
367
|
+
`runner` of `tests.yaml`. `runner` may be a list, one runner for each
|
|
368
|
+
language.
|
|
369
|
+
2. Before the import, set `sys.dont_write_bytecode = True`. Put
|
|
370
|
+
`test/atlas` on `sys.path`, and import `scenario` from `contract`.
|
|
371
|
+
Run Python with `-B`: the check refuses bytecode in the test folder.
|
|
372
|
+
Never run it with `-O`, which removes every assert.
|
|
373
|
+
3. Decorate the body: `@scenario("../../scenarios/<name>.md", adapter=adapter)`.
|
|
374
|
+
4. Write each check as a `with` block that holds an assert statement:
|
|
375
|
+
`with run.check("<scenario>/e1#<fingerprint>"):`. Use `run.gate` for a gate.
|
|
376
|
+
To keep a short proof, set `proof.value` in `with ... as proof:`.
|
|
377
|
+
5. Use `run.not_exercised` and `run.wait_for`, in snake case.
|
|
378
|
+
6. Give the driver of each way in through the adapter (`define_adapter`).
|
|
379
|
+
Build its child environment with `child_env`.
|
|
380
|
+
|
|
381
|
+
The library reads the scenario through node, so the runner needs node and
|
|
382
|
+
python3. It does not yet start an environment, give run secrets, repeat a
|
|
383
|
+
run, ask a judge or ship drivers. Each of these ends `blocked`. Write such a
|
|
384
|
+
scenario with `contract.mjs`.
|
|
385
|
+
|
|
361
386
|
### A model in the test
|
|
362
387
|
|
|
363
388
|
A model makes a test vary and costs money. Three parts of the kit handle it.
|
package/tests/bridge.mjs
ADDED
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// The bridge for a contract library in another language (design section 8.1).
|
|
3
|
+
// A library in Python, say, must read a scenario exactly as the link check
|
|
4
|
+
// reads it: the same parser, the same fingerprints, the same tests.yaml check
|
|
5
|
+
// and the same guard comparison. So it does not parse the words itself. It
|
|
6
|
+
// runs this file and reads one JSON object:
|
|
7
|
+
//
|
|
8
|
+
// node bridge.mjs load <scenario.md>
|
|
9
|
+
// -> { root, file, environment, evidenceDir, guardRefusal, tests, envSpec, shippedDrivers, wayGuardParts,
|
|
10
|
+
// scenario: { id, title, file, sha256, method, fields, through, ways, repeat,
|
|
11
|
+
// assertions: [{ id, fullId, kind, gate, ways, fp, names, line }],
|
|
12
|
+
// tables: { T1: { name, kind, header, rows, fingerprint, cases } } } }
|
|
13
|
+
//
|
|
14
|
+
// `cases` is every case of a combination table (every combination, or a set
|
|
15
|
+
// that holds every pair), so the other language need not make them again.
|
|
16
|
+
// A fault in the scenario or in tests.yaml prints { error } and exits 1.
|
|
17
|
+
// The environment and the folders come from the same variables as
|
|
18
|
+
// contract.mjs: ATLAS_ENV, CI, ATLAS_EVIDENCE_DIR and ATLAS_GUARDS_FROM.
|
|
19
|
+
import { existsSync, readFileSync } from 'node:fs';
|
|
20
|
+
import { dirname, join, resolve } from 'node:path';
|
|
21
|
+
import { fileURLToPath } from 'node:url';
|
|
22
|
+
import { readTestsYaml, checkTestsYaml, compareGuardParts, resolveEnvironments, GUARD_PARTS } from './tests-yaml.mjs';
|
|
23
|
+
import { SHIPPED_DRIVERS } from './drivers/index.mjs';
|
|
24
|
+
import { readScenario, addFingerprints, combinations, pairs } from './scenario.mjs';
|
|
25
|
+
|
|
26
|
+
function findRoot(file) {
|
|
27
|
+
let dir = dirname(file);
|
|
28
|
+
for (;;) {
|
|
29
|
+
if (existsSync(join(dir, 'atlas', 'tests.yaml'))) return dir;
|
|
30
|
+
const up = dirname(dir);
|
|
31
|
+
if (up === dir) throw new Error(`no atlas/tests.yaml above ${file}`);
|
|
32
|
+
dir = up;
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
export function loadForLibrary(path, env = process.env) {
|
|
37
|
+
const file = resolve(String(path));
|
|
38
|
+
const root = findRoot(file);
|
|
39
|
+
const yaml = readTestsYaml(join(root, 'atlas', 'tests.yaml'));
|
|
40
|
+
const findings = checkTestsYaml(yaml);
|
|
41
|
+
if (findings.length) throw new Error(`atlas/tests.yaml: ${findings.map((f) => `line ${f.line}: ${f.message}`).join('; ')}`);
|
|
42
|
+
const sc = readScenario(file);
|
|
43
|
+
if (sc.problems.length) throw new Error(`${path}: ${sc.problems.map((p) => `line ${p.line}: ${p.message}`).join('; ')}`);
|
|
44
|
+
const tests = yaml.value;
|
|
45
|
+
addFingerprints(sc, tests);
|
|
46
|
+
// The same rule as contract.mjs: a branch whose guard parts differ from the main branch copy is refused.
|
|
47
|
+
let guardRefusal = null;
|
|
48
|
+
if (env.ATLAS_GUARDS_FROM) {
|
|
49
|
+
try {
|
|
50
|
+
const changes = compareGuardParts(readFileSync(env.ATLAS_GUARDS_FROM, 'utf8'), yaml.value);
|
|
51
|
+
if (changes.length) guardRefusal = `the guard parts of atlas/tests.yaml differ from the main branch copy (${changes.map((c) => c.part).join(', ')}); the run is refused. A change to a guard part comes to the owner`;
|
|
52
|
+
} catch (error) {
|
|
53
|
+
guardRefusal = `ATLAS_GUARDS_FROM names a file that cannot be read as tests.yaml (${error.code ?? error.message}); the run is refused`;
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
const environment = env.ATLAS_ENV ?? (env.CI ? 'ci' : 'local');
|
|
57
|
+
const evidenceDir = env.ATLAS_EVIDENCE_DIR ?? join(root, 'test', 'evidence');
|
|
58
|
+
const tables = {};
|
|
59
|
+
for (const [name, t] of sc.tables) {
|
|
60
|
+
const cases = t.kind === 'combination' ? (t.cover === 'pair' ? pairs(t.dimensions) : combinations(t.dimensions)) : null;
|
|
61
|
+
tables[name] = { name, kind: t.kind, header: t.header ?? null, rows: t.rows ?? [], fingerprint: t.fingerprint, cases };
|
|
62
|
+
}
|
|
63
|
+
return {
|
|
64
|
+
root, file, environment, evidenceDir, guardRefusal, tests,
|
|
65
|
+
// What a library in another language must refuse until it has its own form of the guard.
|
|
66
|
+
shippedDrivers: SHIPPED_DRIVERS,
|
|
67
|
+
wayGuardParts: GUARD_PARTS.filter((p) => p.startsWith('ways-in.*.')).map((p) => p.slice('ways-in.*.'.length)),
|
|
68
|
+
envSpec: resolveEnvironments(tests.environments ?? {})[environment] ?? null,
|
|
69
|
+
scenario: {
|
|
70
|
+
id: sc.id, title: sc.title, file: sc.file, sha256: sc.sha256, method: sc.method,
|
|
71
|
+
names: [...new Set([...sc.start, ...sc.steps, ...sc.assertions].flatMap((x) => x.names ?? []))],
|
|
72
|
+
fields: sc.fields, through: sc.through, ways: sc.ways ?? sc.through, repeat: sc.repeat ?? null,
|
|
73
|
+
assertions: sc.assertions.map((a) => ({ id: a.id, fullId: a.fullId, kind: a.kind, gate: Boolean(a.gate), ways: a.ways ?? sc.through, fp: a.fp, names: a.names ?? [], line: a.line })),
|
|
74
|
+
tables,
|
|
75
|
+
},
|
|
76
|
+
};
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
export function main(argv, write = (text) => process.stdout.write(text)) {
|
|
80
|
+
const [verb, path] = argv;
|
|
81
|
+
if (verb !== 'load' || !path) { process.stderr.write('bridge: use "bridge.mjs load <scenario.md>"\n'); return 2; }
|
|
82
|
+
try {
|
|
83
|
+
write(`${JSON.stringify(loadForLibrary(path))}\n`);
|
|
84
|
+
return 0;
|
|
85
|
+
} catch (error) {
|
|
86
|
+
write(`${JSON.stringify({ error: error.message })}\n`);
|
|
87
|
+
return 1;
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) process.exitCode = main(process.argv.slice(2));
|