@asrunjs/asrun 0.6.1 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/settings.json +5 -5
- package/.claude/skills/exploring-cases/references/assertion-targets.md +103 -103
- package/.claude/skills/exploring-cases/references/handoff-card.js +128 -128
- package/.claude/skills/making-cases/references/gaps.md +36 -36
- package/.claude/skills/making-cases/references/splitting.md +28 -28
- package/AGENTS.md +3 -0
- package/README.md +16 -2
- package/bin/asrun.mjs +2 -1
- package/dist/scripts/cli.js +14 -6
- package/dist/scripts/go.mjs +36 -6
- package/dist/scripts/heal-evidence.mjs +149 -143
- package/dist/scripts/init.d.ts +7 -0
- package/dist/scripts/init.js +57 -15
- package/dist/scripts/reset.d.ts +1 -0
- package/dist/scripts/reset.js +233 -0
- package/dist/scripts/run.js +23 -23
- package/docs/agents/framework-dev.md +29 -1
- package/docs/commands.md +25 -8
- package/docs/install.md +47 -10
- package/docs/troubleshooting.md +31 -10
- package/package.json +34 -30
- package/templates/.claude/settings.json +5 -5
- package/templates/CLAUDE.md +1 -1
- package/templates/gitignore +13 -13
- package/templates/package.json +33 -32
- package/templates/playwright.config.ts +3 -3
- package/templates/pnpm-workspace.yaml +8 -5
- package/templates/seed.spec.ts +9 -9
- package/templates/tsconfig.json +14 -14
- package/ui/dist/favicon.svg +11 -11
- package/ui/dist/logos/claude.svg +1 -1
- package/ui/dist/logos/opencode.svg +1 -1
|
@@ -1,28 +1,28 @@
|
|
|
1
|
-
# Splitting notes into cases
|
|
2
|
-
|
|
3
|
-
## Principle
|
|
4
|
-
One case = one real user path. Split only when something real differs:
|
|
5
|
-
1. **Starting state** differs (a landing page you simply open, versus a flow that needs data it creates itself).
|
|
6
|
-
2. **Write level** differs (read-only versus `writes: [<env>, ...]`). Keep read-only checks out of write cases: they can
|
|
7
|
-
run anywhere, without confirmation, in the smoke suite.
|
|
8
|
-
3. **Independence** differs (the checks can pass or fail without the other part, and a failure in one should not hide
|
|
9
|
-
the other).
|
|
10
|
-
|
|
11
|
-
Do not split because a case is long, or because the notes list things under separate headings.
|
|
12
|
-
|
|
13
|
-
## Typical result
|
|
14
|
-
- A read-only landing or overview check (what is on the page): small, fast, `suites: [smoke, regression]`, no `writes` (read-only).
|
|
15
|
-
- A user flow that creates data and cleans it up: `writes: [staging]`, one case, steps in the real order (even if the
|
|
16
|
-
flow is not linear), ending with the deletion.
|
|
17
|
-
- A logo or visual check of data that already exists, kept read-only: its own case when it does not need the flow.
|
|
18
|
-
|
|
19
|
-
## Existing cases
|
|
20
|
-
Look at `cases/` before writing. If a new case overlaps an existing one, do not overwrite it: propose keep, merge or
|
|
21
|
-
replace, with the reason, and wait for the user. Never rename or move an existing case yourself; the Cases page of the web UI does that and keeps the spec in step.
|
|
22
|
-
|
|
23
|
-
## Naming and folders
|
|
24
|
-
Cases of one area live in a folder: `cases/documents/landing-page.md` has the id `documents/landing-page`. The name is
|
|
25
|
-
`<what>` in kebab-case (`hub-landing`, `draft-to-publish`), the folder is the area (`documents`). One folder level only.
|
|
26
|
-
Look at the folders that already exist in `cases/` and reuse them. A case that fits no area stays directly in
|
|
27
|
-
`cases/`. The `title` says what the case proves. Two ids that differ only in `-` versus `/` (`documents-x` and
|
|
28
|
-
`documents/x`) are not allowed: one group level only, so ids stay unambiguous.
|
|
1
|
+
# Splitting notes into cases
|
|
2
|
+
|
|
3
|
+
## Principle
|
|
4
|
+
One case = one real user path. Split only when something real differs:
|
|
5
|
+
1. **Starting state** differs (a landing page you simply open, versus a flow that needs data it creates itself).
|
|
6
|
+
2. **Write level** differs (read-only versus `writes: [<env>, ...]`). Keep read-only checks out of write cases: they can
|
|
7
|
+
run anywhere, without confirmation, in the smoke suite.
|
|
8
|
+
3. **Independence** differs (the checks can pass or fail without the other part, and a failure in one should not hide
|
|
9
|
+
the other).
|
|
10
|
+
|
|
11
|
+
Do not split because a case is long, or because the notes list things under separate headings.
|
|
12
|
+
|
|
13
|
+
## Typical result
|
|
14
|
+
- A read-only landing or overview check (what is on the page): small, fast, `suites: [smoke, regression]`, no `writes` (read-only).
|
|
15
|
+
- A user flow that creates data and cleans it up: `writes: [staging]`, one case, steps in the real order (even if the
|
|
16
|
+
flow is not linear), ending with the deletion.
|
|
17
|
+
- A logo or visual check of data that already exists, kept read-only: its own case when it does not need the flow.
|
|
18
|
+
|
|
19
|
+
## Existing cases
|
|
20
|
+
Look at `cases/` before writing. If a new case overlaps an existing one, do not overwrite it: propose keep, merge or
|
|
21
|
+
replace, with the reason, and wait for the user. Never rename or move an existing case yourself; the Cases page of the web UI does that and keeps the spec in step.
|
|
22
|
+
|
|
23
|
+
## Naming and folders
|
|
24
|
+
Cases of one area live in a folder: `cases/documents/landing-page.md` has the id `documents/landing-page`. The name is
|
|
25
|
+
`<what>` in kebab-case (`hub-landing`, `draft-to-publish`), the folder is the area (`documents`). One folder level only.
|
|
26
|
+
Look at the folders that already exist in `cases/` and reuse them. A case that fits no area stays directly in
|
|
27
|
+
`cases/`. The `title` says what the case proves. Two ids that differ only in `-` versus `/` (`documents-x` and
|
|
28
|
+
`documents/x`) are not allowed: one group level only, so ids stay unambiguous.
|
package/AGENTS.md
CHANGED
|
@@ -151,6 +151,9 @@ Three steps, each learns one thing: the intent, the facts and inputs, the implem
|
|
|
151
151
|
writes data and never creates baselines. A tagged step with no `- Recipe:` line (an old `## Inputs` line counts
|
|
152
152
|
until it is moved) stops with `needs-input`: run `/explore-case` first. Running specs is a separate step
|
|
153
153
|
(`pnpm asrun`, or the Run page).
|
|
154
|
+
Workspace maintenance is separate from the AI steps: `pnpm asrun init [--update] [--force]` writes or refreshes the
|
|
155
|
+
workspace files, and `pnpm asrun reset` removes `node_modules` and the files init wrote (edited files, cases, specs,
|
|
156
|
+
`profiles.yaml` and `package.json` stay), so `pnpm dlx @asrunjs/asrun init --install` can scaffold again.
|
|
154
157
|
Do not hand-write specs unless asked. Do not edit `lib/` to make a single spec pass.
|
|
155
158
|
Every case step is wrapped in `test.step('<n>. <case step text>', ...)` so reports and traces show the
|
|
156
159
|
case's own steps (see the skill's `references/conventions.md`).
|
package/README.md
CHANGED
|
@@ -56,7 +56,19 @@ pnpm asrun doctor
|
|
|
56
56
|
```
|
|
57
57
|
|
|
58
58
|
`init --update` refreshes only the generated files; it never overwrites `AGENTS.md`, `CLAUDE.md` or your skill
|
|
59
|
-
overrides. `doctor` checks profiles, generated files, the installed version and the project rules.
|
|
59
|
+
overrides. `doctor` checks profiles, generated files, the installed version and the project rules. `init --force`
|
|
60
|
+
rewrites every file init writes (rules still need `--reset-rules`; `profiles.yaml` and `package.json` stay yours).
|
|
61
|
+
|
|
62
|
+
To start over with fresh dependencies and a fresh scaffold:
|
|
63
|
+
|
|
64
|
+
```bash
|
|
65
|
+
pnpm asrun reset # or: pnpm asrun reset --yes --state
|
|
66
|
+
pnpm dlx @asrunjs/asrun init --install
|
|
67
|
+
pnpm go
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
`reset` removes `node_modules` and the files init wrote; cases, specs, `profiles.yaml`, `package.json` and files you
|
|
71
|
+
edited are kept.
|
|
60
72
|
|
|
61
73
|
<details>
|
|
62
74
|
<summary>Manual install or a pinned version</summary>
|
|
@@ -68,7 +80,8 @@ pnpm go
|
|
|
68
80
|
```
|
|
69
81
|
|
|
70
82
|
On pnpm 12, run `init` before adding the dependency: it writes the `allowBuilds` entry that `esbuild` and
|
|
71
|
-
`ffmpeg-static` need, or `pnpm add` stops with `ERR_PNPM_IGNORED_BUILDS`.
|
|
83
|
+
`ffmpeg-static` need, or `pnpm add` stops with `ERR_PNPM_IGNORED_BUILDS`. The same file exempts `@asrunjs/asrun` from
|
|
84
|
+
pnpm's 24-hour release age, so a fresh release installs right away. Details and flags:
|
|
72
85
|
[install.md](./docs/install.md).
|
|
73
86
|
|
|
74
87
|
</details>
|
|
@@ -135,6 +148,7 @@ Details: [ai-workflow.md](./docs/ai-workflow.md#customizing-the-prompts).
|
|
|
135
148
|
| I want to... | Command |
|
|
136
149
|
|---|---|
|
|
137
150
|
| Check the workspace | `pnpm asrun doctor` |
|
|
151
|
+
| Start over with fresh dependencies and scaffold | `pnpm asrun reset [--yes] [--state] [--lockfile]` |
|
|
138
152
|
| See which profiles are logged in | `pnpm status [--env <env>]` |
|
|
139
153
|
| Log in a profile (a person, with 2FA) | `pnpm signin <profile-id>` |
|
|
140
154
|
| Run everything in one environment | `pnpm asrun --env staging` |
|
package/bin/asrun.mjs
CHANGED
|
@@ -13,6 +13,7 @@ const args = process.argv.slice(2);
|
|
|
13
13
|
const SCRIPTS = {
|
|
14
14
|
init: 'cli.js',
|
|
15
15
|
doctor: 'cli.js',
|
|
16
|
+
reset: 'cli.js',
|
|
16
17
|
go: 'go.mjs',
|
|
17
18
|
ui: 'ui.js',
|
|
18
19
|
'ui:dev': 'ui-dev.js',
|
|
@@ -39,7 +40,7 @@ if (!fs.existsSync(script)) {
|
|
|
39
40
|
process.exit(1);
|
|
40
41
|
}
|
|
41
42
|
|
|
42
|
-
// `init` and `
|
|
43
|
+
// `init`, `doctor` and `reset` are subcommands of cli.js; it reads them from its own arguments.
|
|
43
44
|
const cliArgs = scriptName === 'cli.js' && subcommand ? [subcommand, ...args] : args;
|
|
44
45
|
const child = spawn(process.execPath, [script, ...cliArgs], {cwd: process.cwd(), stdio: 'inherit'});
|
|
45
46
|
child.on('error', (error) => {
|
package/dist/scripts/cli.js
CHANGED
|
@@ -12,17 +12,25 @@ async function main() {
|
|
|
12
12
|
process.exitCode = await doctor(rest);
|
|
13
13
|
return;
|
|
14
14
|
}
|
|
15
|
+
if (command === 'reset') {
|
|
16
|
+
const { main: reset } = await import('./reset.js');
|
|
17
|
+
process.exitCode = await reset(rest);
|
|
18
|
+
return;
|
|
19
|
+
}
|
|
15
20
|
if (command === 'help' || command === '--help-init') {
|
|
16
21
|
console.log(USAGE);
|
|
17
22
|
return;
|
|
18
23
|
}
|
|
19
24
|
await import('./run.js');
|
|
20
25
|
}
|
|
21
|
-
const USAGE = `Usage: pnpm asrun [init|doctor] [options]
|
|
22
|
-
|
|
23
|
-
(no subcommand) Run the tests of one environment. See "pnpm asrun --help".
|
|
24
|
-
init Create the files a workspace needs (never overwrites; --update refreshes generated files,
|
|
25
|
-
--root <dir> targets another folder, --force
|
|
26
|
-
|
|
26
|
+
const USAGE = `Usage: pnpm asrun [init|doctor|reset] [options]
|
|
27
|
+
|
|
28
|
+
(no subcommand) Run the tests of one environment. See "pnpm asrun --help".
|
|
29
|
+
init Create the files a workspace needs (never overwrites; --update refreshes generated files,
|
|
30
|
+
--root <dir> targets another folder, --force overwrites every file init writes,
|
|
31
|
+
--force <path> overwrites one file).
|
|
32
|
+
doctor Check the workspace: profiles, tsconfig alias, generated files, skill overrides.
|
|
33
|
+
reset Remove node_modules and the files init writes, for a clean reinstall (keeps cases, specs,
|
|
34
|
+
profiles.yaml, package.json and edited scaffold files; see "pnpm asrun reset --help").`;
|
|
27
35
|
await main();
|
|
28
36
|
export {};
|
package/dist/scripts/go.mjs
CHANGED
|
@@ -4,16 +4,25 @@
|
|
|
4
4
|
// pnpm add -D @asrunjs/asrun (or github:wizcas/asrun#<tag>)
|
|
5
5
|
// pnpm exec asrun init
|
|
6
6
|
// pnpm go
|
|
7
|
+
//
|
|
8
|
+
// `--no-open` does not open a browser. `--no-ui` stops after step 3 (the workspace is prepared, no server starts).
|
|
9
|
+
// `--dry-run` only prints what the command resolved (workspace, framework, the asrun command) and exits.
|
|
7
10
|
import {spawn, spawnSync} from 'node:child_process';
|
|
8
11
|
import fs from 'node:fs';
|
|
9
12
|
import path from 'node:path';
|
|
10
13
|
|
|
11
|
-
|
|
14
|
+
// The package root: `scripts/go.mjs` in a development checkout, `dist/scripts/go.mjs` in an installed package, so the
|
|
15
|
+
// same rule as lib/paths.ts. `bin/`, `examples/` and the framework's `.env` hang off this folder, never off `dist/`.
|
|
16
|
+
const FRAMEWORK = path.basename(path.dirname(import.meta.dirname)) === 'dist'
|
|
17
|
+
? path.resolve(import.meta.dirname, '..', '..')
|
|
18
|
+
: path.resolve(import.meta.dirname, '..');
|
|
12
19
|
const PACKAGE = '@asrunjs/asrun';
|
|
13
20
|
const MARKERS = ['.asrun.json', 'profiles.yaml'];
|
|
14
21
|
const port = Number(process.env.ASRUN_UI_PORT ?? 4173);
|
|
15
22
|
const url = `http://127.0.0.1:${port}`;
|
|
16
23
|
const openBrowser = !process.argv.includes('--no-open');
|
|
24
|
+
const noUi = process.argv.includes('--no-ui');
|
|
25
|
+
const dryRun = process.argv.includes('--dry-run');
|
|
17
26
|
const shell = process.platform === 'win32';
|
|
18
27
|
|
|
19
28
|
// `.env` next to the command, or the framework's own `.env`; a real environment variable always wins.
|
|
@@ -33,8 +42,9 @@ function loadRootEnv(file) {
|
|
|
33
42
|
loadRootEnv(path.join(process.cwd(), '.env'));
|
|
34
43
|
if (process.env.ASRUN_ROOT === undefined) loadRootEnv(path.join(FRAMEWORK, '.env'));
|
|
35
44
|
|
|
36
|
-
// The workspace: the project that owns profiles.yaml, cases/ and tests/. Inside the framework repository
|
|
37
|
-
//
|
|
45
|
+
// The workspace: the project that owns profiles.yaml, cases/ and tests/. Inside the framework repository (which has
|
|
46
|
+
// none of those) the demo workspace is used, so the framework itself can be developed this way. An installed package
|
|
47
|
+
// that ships no `examples/` and runs outside a project has neither: it says so instead of preparing a missing folder.
|
|
38
48
|
function findWorkspace(from) {
|
|
39
49
|
for (let dir = path.resolve(from); ; dir = path.dirname(dir)) {
|
|
40
50
|
if (MARKERS.some((marker) => fs.existsSync(path.join(dir, marker)))) return dir;
|
|
@@ -42,11 +52,16 @@ function findWorkspace(from) {
|
|
|
42
52
|
}
|
|
43
53
|
}
|
|
44
54
|
const demo = path.join(FRAMEWORK, 'examples', 'demo');
|
|
45
|
-
const
|
|
46
|
-
|
|
55
|
+
const chosen = process.env.ASRUN_ROOT ? path.resolve(process.env.ASRUN_ROOT) : findWorkspace(process.cwd());
|
|
56
|
+
const ROOT = chosen ?? (fs.existsSync(demo) ? demo : null);
|
|
57
|
+
if (ROOT === null) {
|
|
47
58
|
console.error('No workspace found: run this command in the project that uses asrun, or set ASRUN_ROOT.');
|
|
48
59
|
process.exit(1);
|
|
49
60
|
}
|
|
61
|
+
if (!fs.existsSync(ROOT)) {
|
|
62
|
+
console.error(`${ROOT} does not exist: check ASRUN_ROOT (or the .env that sets it).`);
|
|
63
|
+
process.exit(1);
|
|
64
|
+
}
|
|
50
65
|
|
|
51
66
|
const step = (n, text) => console.log(`\n\x1b[1;36m[${n}/4] ${text}\x1b[0m`);
|
|
52
67
|
const warn = (text) => console.log(`\x1b[33m! ${text}\x1b[0m`);
|
|
@@ -72,7 +87,10 @@ function packageBin(name, command = name) {
|
|
|
72
87
|
}
|
|
73
88
|
|
|
74
89
|
/** The installed command of this workspace, or the framework's own bin while developing it here. */
|
|
75
|
-
const bin = packageBin(PACKAGE) ?? path.join(FRAMEWORK, 'bin', 'asrun.mjs');
|
|
90
|
+
const bin = packageBin(PACKAGE, 'asrun') ?? path.join(FRAMEWORK, 'bin', 'asrun.mjs');
|
|
91
|
+
if (!fs.existsSync(bin)) {
|
|
92
|
+
fail(`The asrun command is missing: ${bin}\nReinstall the package: "pnpm add -D @asrunjs/asrun".`);
|
|
93
|
+
}
|
|
76
94
|
/** The workspace's copy of the package, if any. */
|
|
77
95
|
const installed = fs.existsSync(path.join(ROOT, 'node_modules', PACKAGE, 'package.json'));
|
|
78
96
|
const frameworkRelative = installed
|
|
@@ -82,6 +100,13 @@ const frameworkRelative = installed
|
|
|
82
100
|
return relative === '' ? '.' : relative.startsWith('.') ? relative : `./${relative}`;
|
|
83
101
|
})();
|
|
84
102
|
|
|
103
|
+
if (dryRun) {
|
|
104
|
+
console.log(`workspace: ${ROOT}`);
|
|
105
|
+
console.log(`framework: ${FRAMEWORK}${installed ? ' (installed)' : ' (development checkout)'}`);
|
|
106
|
+
console.log(`command: ${bin}`);
|
|
107
|
+
process.exit(0);
|
|
108
|
+
}
|
|
109
|
+
|
|
85
110
|
/** pnpm only installs dependencies normally; its build scripts need an explicit approval in pnpm-workspace.yaml. */
|
|
86
111
|
function ensurePnpmWorkspace() {
|
|
87
112
|
if (frameworkRelative.startsWith('..')) return 'skipped';
|
|
@@ -164,6 +189,11 @@ if (!run(process.execPath, [bin, 'init', '--root', ROOT], {cwd: process.cwd()}))
|
|
|
164
189
|
}
|
|
165
190
|
run(process.execPath, [bin, 'doctor']);
|
|
166
191
|
|
|
192
|
+
if (noUi) {
|
|
193
|
+
console.log(`\n\x1b[1;32mWorkspace ready: ${ROOT}\x1b[0m (not starting the web UI)`);
|
|
194
|
+
process.exit(0);
|
|
195
|
+
}
|
|
196
|
+
|
|
167
197
|
step(4, `Starting the web UI at ${url}`);
|
|
168
198
|
if (await uiIsUp()) {
|
|
169
199
|
console.log('The UI is already running.');
|
|
@@ -1,143 +1,149 @@
|
|
|
1
|
-
// Reads the existing run reports (reports/<runId>/summary.json) for one case and prints what they already say,
|
|
2
|
-
// so that healing starts from the evidence instead of from a new run. It never opens a browser and never writes.
|
|
3
|
-
// Plain node (no tsx): it must work inside an agent's sandbox.
|
|
4
|
-
import fs from 'node:fs';
|
|
5
|
-
import path from 'node:path';
|
|
6
|
-
|
|
7
|
-
// The
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
const
|
|
44
|
-
const
|
|
45
|
-
const
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
const
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
);
|
|
77
|
-
if (
|
|
78
|
-
|
|
79
|
-
if (
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
}
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
}
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
}
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
console.log(
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
}
|
|
131
|
-
for (const
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
(
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
1
|
+
// Reads the existing run reports (reports/<runId>/summary.json) for one case and prints what they already say,
|
|
2
|
+
// so that healing starts from the evidence instead of from a new run. It never opens a browser and never writes.
|
|
3
|
+
// Plain node (no tsx): it must work inside an agent's sandbox.
|
|
4
|
+
import fs from 'node:fs';
|
|
5
|
+
import path from 'node:path';
|
|
6
|
+
|
|
7
|
+
// The package root: `scripts/heal-evidence.mjs` in a development checkout, `dist/scripts/heal-evidence.mjs` installed.
|
|
8
|
+
// The same rule as lib/paths.ts: a compiled file sits one folder deeper.
|
|
9
|
+
const FRAMEWORK = path.basename(path.dirname(import.meta.dirname)) === 'dist'
|
|
10
|
+
? path.resolve(import.meta.dirname, '..', '..')
|
|
11
|
+
: path.resolve(import.meta.dirname, '..');
|
|
12
|
+
|
|
13
|
+
// The workspace root: the project that uses the framework. Same rule as lib/paths.ts, in plain Node.
|
|
14
|
+
const findWorkspace = (from) => {
|
|
15
|
+
for (let dir = path.resolve(from); ; dir = path.dirname(dir)) {
|
|
16
|
+
if (fs.existsSync(path.join(dir, 'profiles.yaml')) || fs.existsSync(path.join(dir, '.asrun.json'))) return dir;
|
|
17
|
+
if (path.dirname(dir) === dir) return null;
|
|
18
|
+
}
|
|
19
|
+
};
|
|
20
|
+
// `.env` next to the command, or the framework's own `.env`; a real environment variable always wins.
|
|
21
|
+
const loadRootEnv = (file) => {
|
|
22
|
+
try {
|
|
23
|
+
for (const raw of fs.readFileSync(file, 'utf8').split(/\r?\n/)) {
|
|
24
|
+
if (process.env.ASRUN_ROOT !== undefined) return;
|
|
25
|
+
const match = /^(?:export\s+)?ASRUN_ROOT\s*=\s*(.*)$/.exec(raw.trim());
|
|
26
|
+
if (match) {
|
|
27
|
+
process.env.ASRUN_ROOT = path.resolve(path.dirname(file), match[1].trim().replace(/^(['"])(.*)\1$/, '$2'));
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
} catch {
|
|
31
|
+
// no .env: nothing to do
|
|
32
|
+
}
|
|
33
|
+
};
|
|
34
|
+
loadRootEnv(path.join(process.cwd(), '.env'));
|
|
35
|
+
if (process.env.ASRUN_ROOT === undefined) loadRootEnv(path.join(FRAMEWORK, '.env'));
|
|
36
|
+
const ROOT = process.env.ASRUN_ROOT
|
|
37
|
+
? path.resolve(process.env.ASRUN_ROOT)
|
|
38
|
+
: (findWorkspace(process.cwd()) ?? FRAMEWORK);
|
|
39
|
+
const REPORTS = path.join(ROOT, 'reports');
|
|
40
|
+
const RUN_ID = /^\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2}(-[a-z0-9]+(-[a-z0-9]+)*)?$/;
|
|
41
|
+
const NOT_LOGGED_IN = /is not logged in to/;
|
|
42
|
+
|
|
43
|
+
const args = process.argv.slice(2);
|
|
44
|
+
const option = (name) => {
|
|
45
|
+
const index = args.indexOf(name);
|
|
46
|
+
return index >= 0 ? args[index + 1] : undefined;
|
|
47
|
+
};
|
|
48
|
+
const caseId = option('--case');
|
|
49
|
+
const env = option('--env');
|
|
50
|
+
const profileFilter = (option('--profile') ?? '').split(',').filter(Boolean);
|
|
51
|
+
const keep = Number(option('--max-reports') ?? 1);
|
|
52
|
+
|
|
53
|
+
if (args.includes('--help') || !caseId || !env) {
|
|
54
|
+
console.log('Usage: pnpm heal:evidence --case <case-id> --env <env> [--profile a,b] [--max-reports 1]');
|
|
55
|
+
process.exit(args.includes('--help') ? 0 : 1);
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
const readSummary = (id) => {
|
|
59
|
+
try {
|
|
60
|
+
return JSON.parse(fs.readFileSync(path.join(REPORTS, id, 'summary.json'), 'utf8'));
|
|
61
|
+
} catch {
|
|
62
|
+
return null;
|
|
63
|
+
}
|
|
64
|
+
};
|
|
65
|
+
const mtime = (file) => (fs.existsSync(file) ? fs.statSync(file).mtimeMs : 0);
|
|
66
|
+
const iso = (ms) => new Date(ms).toISOString().replace(/\.\d+Z$/, 'Z');
|
|
67
|
+
const firstLines = (text, count) => text.split('\n').slice(0, count).join('\n');
|
|
68
|
+
const indent = (text, prefix) => text.split('\n').map((line) => prefix + line).join('\n');
|
|
69
|
+
|
|
70
|
+
const runIds = fs.existsSync(REPORTS)
|
|
71
|
+
? fs.readdirSync(REPORTS).filter((name) => RUN_ID.test(name)).sort().reverse()
|
|
72
|
+
: [];
|
|
73
|
+
|
|
74
|
+
const matches = [];
|
|
75
|
+
for (const id of runIds) {
|
|
76
|
+
const summary = readSummary(id);
|
|
77
|
+
if (!summary) continue;
|
|
78
|
+
const runEnv = summary.env ?? /^\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2}-(.+)$/.exec(id)?.[1] ?? 'production';
|
|
79
|
+
if (runEnv !== env) continue;
|
|
80
|
+
const results = summary.results.filter(
|
|
81
|
+
(result) => result.caseId === caseId && (profileFilter.length === 0 || profileFilter.includes(result.profile))
|
|
82
|
+
);
|
|
83
|
+
if (results.length === 0) continue;
|
|
84
|
+
matches.push({id, summary, results});
|
|
85
|
+
if (matches.length >= keep) break;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
console.log(`Heal evidence for case "${caseId}" in ${env}`);
|
|
89
|
+
|
|
90
|
+
if (matches.length === 0) {
|
|
91
|
+
console.log(`\nNO REPORT: no run in ${env} has results for this case${profileFilter.length ? ` and these profiles (${profileFilter})` : ''}.`);
|
|
92
|
+
console.log('EVIDENCE: none -> run the case once: pnpm asrun --env ' + env + ' --logged-in --grep ' + caseId);
|
|
93
|
+
process.exit(0);
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
const watched = [
|
|
97
|
+
['spec', path.join(ROOT, 'tests', `${caseId}.spec.ts`)],
|
|
98
|
+
['case', path.join(ROOT, 'cases', `${caseId}.md`)],
|
|
99
|
+
['profiles.yaml', path.join(ROOT, 'profiles.yaml')],
|
|
100
|
+
];
|
|
101
|
+
|
|
102
|
+
let newest = null;
|
|
103
|
+
for (const {id, summary, results} of matches) {
|
|
104
|
+
const endedAt = Date.parse(summary.startedAt) + summary.durationMs;
|
|
105
|
+
const changed = watched
|
|
106
|
+
.map(([label, file]) => ({label, at: mtime(file)}))
|
|
107
|
+
.filter((item) => item.at > endedAt);
|
|
108
|
+
newest ??= {changed};
|
|
109
|
+
|
|
110
|
+
console.log(`\n=== Report ${id} (${summary.startedAt}) ===`);
|
|
111
|
+
console.log(`report: reports/${id}/summary.json html: reports/${id}/html`);
|
|
112
|
+
if (changed.length > 0) {
|
|
113
|
+
console.log(`STALE: changed after this run: ${changed.map((item) => `${item.label} (${iso(item.at)})`).join(', ')}`);
|
|
114
|
+
} else {
|
|
115
|
+
console.log('FRESH: the spec, the case and profiles.yaml have not changed since this run.');
|
|
116
|
+
}
|
|
117
|
+
if (summary.errors?.length) {
|
|
118
|
+
console.log('Run-level errors (not tied to a test):');
|
|
119
|
+
for (const error of summary.errors) console.log(indent(error.headline, ' - '));
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
for (const result of results) {
|
|
123
|
+
const loginProblem = result.errors.some((error) => NOT_LOGGED_IN.test(error.headline));
|
|
124
|
+
console.log(`\n[${result.profile}] ${result.status.toUpperCase()} (${result.attempts} attempt${result.attempts === 1 ? '' : 's'}, ${result.durationMs} ms)`);
|
|
125
|
+
if (result.status === 'passed' || result.status === 'skipped') continue;
|
|
126
|
+
if (loginProblem) {
|
|
127
|
+
console.log(' Not a test failure: the profile was not logged in during this run. Not evidence about the spec.');
|
|
128
|
+
continue;
|
|
129
|
+
}
|
|
130
|
+
if (result.failedStep) console.log(` failed step: ${result.failedStep}`);
|
|
131
|
+
for (const error of result.errors) {
|
|
132
|
+
console.log(indent(error.headline, ' error: '));
|
|
133
|
+
if (error.context) console.log(indent(firstLines(error.context, 8), ' '));
|
|
134
|
+
if (error.callLog) console.log(indent(firstLines(error.callLog, 12), ' log: '));
|
|
135
|
+
if (error.snippet) console.log(indent(firstLines(error.snippet, 8), ' code: '));
|
|
136
|
+
}
|
|
137
|
+
for (const file of result.screenshots) console.log(` screenshot: reports/${id}/html/${file}`);
|
|
138
|
+
for (const file of result.videos) console.log(` video: reports/${id}/html/${file}`);
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
const usable = matches[0].results.filter(
|
|
143
|
+
(result) =>
|
|
144
|
+
(result.status === 'failed' || result.status === 'flaky') &&
|
|
145
|
+
!result.errors.some((error) => NOT_LOGGED_IN.test(error.headline))
|
|
146
|
+
);
|
|
147
|
+
const state = newest.changed.length > 0 ? 'stale' : usable.length === 0 ? 'no-failure' : 'fresh';
|
|
148
|
+
console.log(`\nEVIDENCE: ${state}`);
|
|
149
|
+
if (usable.length > 0) console.log(`failing profiles in the newest report: ${usable.map((result) => result.profile).join(',')}`);
|
package/dist/scripts/init.d.ts
CHANGED
|
@@ -3,6 +3,13 @@ export declare const GENERATED: {
|
|
|
3
3
|
file: string;
|
|
4
4
|
template: string;
|
|
5
5
|
}[];
|
|
6
|
+
/** Files `init` creates once; after that the project owns them and may edit them freely. */
|
|
7
|
+
export declare const ONCE: {
|
|
8
|
+
file: string;
|
|
9
|
+
template: string;
|
|
10
|
+
}[];
|
|
11
|
+
export declare const MARKER = ".asrun.json";
|
|
12
|
+
export declare const DOTENV_HEADER = "# asrun: the workspace these commands run against (relative to this file).";
|
|
6
13
|
export declare const PACKAGE = "@asrunjs/asrun";
|
|
7
14
|
/** How the workspace reaches the framework: `./node_modules/@asrunjs/asrun`, or a relative folder in the framework repo. */
|
|
8
15
|
export declare function frameworkPath(root: string): string;
|