@yemi33/minions 0.1.2070 → 0.1.2072
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dashboard/js/qa.js +358 -0
- package/dashboard/js/state.js +2 -1
- package/dashboard/pages/qa.html +72 -0
- package/dashboard/styles.css +102 -0
- package/dashboard.js +410 -6
- package/docs/qa-runbook-lifecycle.md +232 -0
- package/engine/cleanup.js +4 -1
- package/engine/comment-classifier.js +8 -1
- package/engine/cooldown.js +6 -2
- package/engine/gh-comment.js +74 -3
- package/engine/gh-token.js +7 -9
- package/engine/lifecycle.js +100 -0
- package/engine/pipeline.js +9 -1
- package/engine/playbook.js +39 -0
- package/engine/qa-runners/maestro.js +152 -0
- package/engine/qa-runners/playwright.js +149 -0
- package/engine/qa-runners.js +323 -0
- package/engine/qa-sessions.js +1008 -0
- package/engine/shared.js +71 -12
- package/engine.js +140 -0
- package/package.json +1 -1
- package/playbooks/qa-session-draft.md +158 -0
- package/playbooks/qa-session-execute.md +165 -0
- package/playbooks/qa-session-setup.md +154 -0
- package/prompts/cc-system.md +43 -0
- package/routing.md +3 -0
package/engine/playbook.js
CHANGED
|
@@ -301,6 +301,30 @@ const PLAYBOOK_OPTIONAL_VARS = new Set([
|
|
|
301
301
|
// PR-context vars on non-PR tasks (implement/explore/etc.)
|
|
302
302
|
'pr_id', 'pr_number', 'pr_title', 'pr_branch', 'pr_author', 'pr_url',
|
|
303
303
|
'reviewer',
|
|
304
|
+
// P-e6b3c2d8 — QA Session template vars. session_id / target_kind /
|
|
305
|
+
// flows_raw / managed_spawn_name are required (declared in
|
|
306
|
+
// PLAYBOOK_REQUIRED_VARS['qa-session-setup']); these target_* sub-fields
|
|
307
|
+
// are conditional on target.kind so only one is populated per session.
|
|
308
|
+
// runner_hint / capture / target_json / session_mode / session_phase are
|
|
309
|
+
// surfaced for completeness but legitimately empty when not specified.
|
|
310
|
+
'target_pr_id', // populated only for target.kind === 'pr'
|
|
311
|
+
'target_branch', // populated only for target.kind === 'branch'
|
|
312
|
+
'target_sha', // populated only for target.kind === 'commit'
|
|
313
|
+
'target_worktree', // populated only for target.kind === 'current'
|
|
314
|
+
'target_json', // JSON-encoded target spec (always present when session_id is)
|
|
315
|
+
'runner_hint', // explicit runner name; empty = auto-detect
|
|
316
|
+
'capture', // 'video,screenshots,logs' summary
|
|
317
|
+
'session_mode', // 'confirm' | 'auto'
|
|
318
|
+
'session_phase', // 'setup' | 'draft' | 'execute'
|
|
319
|
+
// P-f9a2e1b4 — Runner adapter briefs. engine.js calls the registered
|
|
320
|
+
// runner's generateBrief()/executeBrief() at render time and surfaces the
|
|
321
|
+
// resulting markdown strings here. Required for the DRAFT/EXECUTE
|
|
322
|
+
// playbooks (declared in PLAYBOOK_REQUIRED_VARS below); empty in the
|
|
323
|
+
// SETUP phase, which is why they live in the optional set — we don't
|
|
324
|
+
// want SETUP renders to emit unresolved-var warnings.
|
|
325
|
+
'runner_brief', // generateBrief() output — used by qa-session-draft
|
|
326
|
+
'runner_execute_brief', // executeBrief() output — used by qa-session-execute
|
|
327
|
+
'test_file', // session.testFile — set after DRAFT, used by EXECUTE
|
|
304
328
|
]);
|
|
305
329
|
|
|
306
330
|
const PLAYBOOK_REQUIRED_VARS = {
|
|
@@ -319,6 +343,21 @@ const PLAYBOOK_REQUIRED_VARS = {
|
|
|
319
343
|
'docs': ['item_id', 'item_name'],
|
|
320
344
|
'setup': ['item_id', 'item_name', 'project_path'],
|
|
321
345
|
'qa-validate': ['item_id', 'item_name', 'qa_run_id'],
|
|
346
|
+
// P-e6b3c2d8 — QA Session SETUP phase. Required vars are session
|
|
347
|
+
// identification + target kind + flows + the deterministic managed-spawn
|
|
348
|
+
// name the engine joins back to the session on. Conditional target_*
|
|
349
|
+
// sub-fields (target_pr_id / target_branch / target_sha / target_worktree)
|
|
350
|
+
// are optional — only one is populated per target.kind.
|
|
351
|
+
'qa-session-setup': ['session_id', 'target_kind', 'flows_raw', 'managed_spawn_name'],
|
|
352
|
+
// P-f9a2e1b4 — QA Session DRAFT phase. Required vars are the session id +
|
|
353
|
+
// flows + the live managed-spawn name + the runner adapter's
|
|
354
|
+
// generateBrief() output. The brief is computed at render time by
|
|
355
|
+
// engine.js (lazy-required qa-sessions + qa-runners + managed-spawn).
|
|
356
|
+
'qa-session-draft': ['session_id', 'flows_raw', 'managed_spawn_name', 'runner_brief'],
|
|
357
|
+
// P-f9a2e1b4 — QA Session EXECUTE phase. Required vars are session id +
|
|
358
|
+
// live managed-spawn name + executeBrief() output + the qa-runs record id
|
|
359
|
+
// (the agent stamps it on the result sidecar so lifecycle.js can ingest it).
|
|
360
|
+
'qa-session-execute': ['session_id', 'managed_spawn_name', 'runner_execute_brief', 'qa_run_id'],
|
|
322
361
|
'work-item': ['item_id', 'item_name'],
|
|
323
362
|
'meeting-investigate': ['meeting_title', 'agenda'],
|
|
324
363
|
'meeting-debate': ['meeting_title', 'agenda'],
|
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* engine/qa-runners/maestro.js — P-c4a9e7f3
|
|
3
|
+
*
|
|
4
|
+
* Maestro built-in runner adapter for QA Sessions. Registered at boot by
|
|
5
|
+
* engine/qa-runners.js (_registerBuiltins). Priority 80 — higher than the
|
|
6
|
+
* Playwright safe default so a project with `.maestro/` wins on tie.
|
|
7
|
+
*
|
|
8
|
+
* Detection: project.localPath (or target.worktree for kind='current') has
|
|
9
|
+
* a `.maestro/` directory. Explicit `runner: 'maestro'` is honoured directly
|
|
10
|
+
* by qa-runners.js#detectRunner before detect() runs, so it does not need
|
|
11
|
+
* a branch here.
|
|
12
|
+
*
|
|
13
|
+
* Hooks: see engine/qa-runners.js header for the contract. We return
|
|
14
|
+
* markdown strings from generateBrief / executeBrief — those flow into the
|
|
15
|
+
* qa-session-draft.md / qa-session-execute.md playbooks as {{runner_brief}}
|
|
16
|
+
* and {{runner_execute_brief}} (wired in P-f9a2e1b4).
|
|
17
|
+
*/
|
|
18
|
+
|
|
19
|
+
const fs = require('fs');
|
|
20
|
+
const path = require('path');
|
|
21
|
+
|
|
22
|
+
const NAME = 'maestro';
|
|
23
|
+
const PRIORITY = 80;
|
|
24
|
+
const INSTALL_HINT =
|
|
25
|
+
'Maestro CLI is required. Install via Homebrew (`brew tap mobile-dev-inc/tap && brew install maestro`) ' +
|
|
26
|
+
'on macOS/Linux, or follow https://maestro.mobile.dev/getting-started/installing-maestro for Windows / manual install. ' +
|
|
27
|
+
'Confirm with `maestro --version` before re-running the session.';
|
|
28
|
+
|
|
29
|
+
function _resolveProbeRoot(target, project) {
|
|
30
|
+
if (target && target.kind === 'current' && typeof target.worktree === 'string' && target.worktree) {
|
|
31
|
+
return target.worktree;
|
|
32
|
+
}
|
|
33
|
+
if (project && typeof project.localPath === 'string' && project.localPath) {
|
|
34
|
+
return project.localPath;
|
|
35
|
+
}
|
|
36
|
+
return null;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
function detect(target, project) {
|
|
40
|
+
const root = _resolveProbeRoot(target, project);
|
|
41
|
+
if (!root) return false;
|
|
42
|
+
try {
|
|
43
|
+
const stat = fs.statSync(path.join(root, '.maestro'));
|
|
44
|
+
return stat.isDirectory();
|
|
45
|
+
} catch {
|
|
46
|
+
return false;
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
function _spawnUrl(spawnInfo) {
|
|
51
|
+
if (!spawnInfo) return '';
|
|
52
|
+
if (spawnInfo.attrs && typeof spawnInfo.attrs.base_url === 'string') return spawnInfo.attrs.base_url;
|
|
53
|
+
if (Array.isArray(spawnInfo.ports) && spawnInfo.ports.length > 0) {
|
|
54
|
+
return `http://localhost:${spawnInfo.ports[0]}`;
|
|
55
|
+
}
|
|
56
|
+
return '';
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
function generateBrief(opts) {
|
|
60
|
+
opts = opts || {};
|
|
61
|
+
const baseUrl = _spawnUrl(opts.spawnInfo);
|
|
62
|
+
const flows = opts.flowsRaw || '(no flows provided)';
|
|
63
|
+
const sessionId = (opts.session && opts.session.id) || (opts.sessionId) || '<session-id>';
|
|
64
|
+
const capture = opts.capture || {};
|
|
65
|
+
const captureSummary = Object.entries(capture).filter(([, v]) => !!v).map(([k]) => k).join(', ') || '(none requested)';
|
|
66
|
+
|
|
67
|
+
return [
|
|
68
|
+
'## Maestro test draft',
|
|
69
|
+
'',
|
|
70
|
+
`Write a single Maestro flow file at \`engine/qa-tests/${sessionId}/flow.yaml\` that exercises:`,
|
|
71
|
+
'',
|
|
72
|
+
flows.trim(),
|
|
73
|
+
'',
|
|
74
|
+
'### Required structure',
|
|
75
|
+
'',
|
|
76
|
+
'- Top-level `appId:` set to the package id Maestro should drive (Android package or iOS bundle).',
|
|
77
|
+
'- A sequence of Maestro commands (`launchApp`, `tapOn`, `inputText`, `assertVisible`, …) that maps 1:1 to the flow above.',
|
|
78
|
+
baseUrl
|
|
79
|
+
? `- For any HTTP step, use the live target base URL \`${baseUrl}\` (resolved from the managed-spawn).`
|
|
80
|
+
: '- The managed-spawn target did not advertise a base URL; rely on the app id alone.',
|
|
81
|
+
'- Keep the flow under 50 lines — split into named sub-flows (`runFlow:`) when the user described independent journeys.',
|
|
82
|
+
'',
|
|
83
|
+
'### Capture configuration',
|
|
84
|
+
'',
|
|
85
|
+
`Requested artifacts: ${captureSummary}. Surface them via the standard Maestro recording flags ` +
|
|
86
|
+
'(`maestro record` or `maestro test --output …`) in the EXECUTE phase; do NOT enable them inside the flow YAML itself.',
|
|
87
|
+
'',
|
|
88
|
+
'See https://maestro.mobile.dev/api-reference/commands for the command vocabulary.',
|
|
89
|
+
].join('\n');
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
function executeBrief(opts) {
|
|
93
|
+
opts = opts || {};
|
|
94
|
+
const sessionId = (opts.session && opts.session.id) || (opts.sessionId) || '<session-id>';
|
|
95
|
+
const testFile = opts.testFile || `engine/qa-tests/${sessionId}/flow.yaml`;
|
|
96
|
+
const capture = opts.capture || {};
|
|
97
|
+
const outputDir = `engine/qa-artifacts/${sessionId}`;
|
|
98
|
+
|
|
99
|
+
const cmdFlags = [`--output ${outputDir}`];
|
|
100
|
+
if (capture.logs) cmdFlags.push(`--debug-output ${outputDir}/maestro-debug`);
|
|
101
|
+
// `maestro test` does not support video; surface the limitation in the
|
|
102
|
+
// brief body rather than silently dropping the capture request.
|
|
103
|
+
|
|
104
|
+
return [
|
|
105
|
+
'## Maestro execute',
|
|
106
|
+
'',
|
|
107
|
+
'Run the drafted flow against the live managed-spawn:',
|
|
108
|
+
'',
|
|
109
|
+
'```bash',
|
|
110
|
+
`maestro test ${testFile} ${cmdFlags.join(' ')}`,
|
|
111
|
+
'```',
|
|
112
|
+
'',
|
|
113
|
+
`Write the result sidecar to \`agents/<your-id>/qa-run-result.json\` referencing artifacts under \`${outputDir}/\`.`,
|
|
114
|
+
capture.video
|
|
115
|
+
? '\nVideo capture was requested — note in the result summary that `maestro test` does not record video and the human will need a separate `maestro record` pass if they want it.'
|
|
116
|
+
: '',
|
|
117
|
+
capture.screenshots
|
|
118
|
+
? '\nScreenshot capture was requested — add `takeScreenshot:` steps at the key flow boundaries; Maestro writes them under the --output dir.'
|
|
119
|
+
: '',
|
|
120
|
+
].filter(Boolean).join('\n');
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
function validateOutputDir(dir) {
|
|
124
|
+
const errors = [];
|
|
125
|
+
if (typeof dir !== 'string' || dir === '') {
|
|
126
|
+
return { ok: false, errors: ['dir must be a non-empty string'] };
|
|
127
|
+
}
|
|
128
|
+
let entries;
|
|
129
|
+
try { entries = fs.readdirSync(dir); }
|
|
130
|
+
catch (err) {
|
|
131
|
+
return { ok: false, errors: [`qa-tests dir ${dir} unreadable: ${err.message}`] };
|
|
132
|
+
}
|
|
133
|
+
const yamlFiles = entries.filter(f => f.endsWith('.yaml') || f.endsWith('.yml'));
|
|
134
|
+
if (yamlFiles.length === 0) {
|
|
135
|
+
errors.push(`no Maestro flow file (*.yaml) found in ${dir}`);
|
|
136
|
+
}
|
|
137
|
+
return { ok: errors.length === 0, errors };
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
module.exports = {
|
|
141
|
+
name: NAME,
|
|
142
|
+
spec: {
|
|
143
|
+
priority: PRIORITY,
|
|
144
|
+
label: 'Maestro',
|
|
145
|
+
description: 'Mobile-first NL flows. Detects when the target project has a .maestro/ directory.',
|
|
146
|
+
installHint: INSTALL_HINT,
|
|
147
|
+
detect,
|
|
148
|
+
generateBrief,
|
|
149
|
+
executeBrief,
|
|
150
|
+
validateOutputDir,
|
|
151
|
+
},
|
|
152
|
+
};
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* engine/qa-runners/playwright.js — P-c4a9e7f3
|
|
3
|
+
*
|
|
4
|
+
* Playwright built-in runner adapter for QA Sessions. Registered at boot by
|
|
5
|
+
* engine/qa-runners.js (_registerBuiltins). Priority 50 — the safe default
|
|
6
|
+
* for any HTTP target. Maestro (priority 80) wins on detect tie when the
|
|
7
|
+
* project has `.maestro/`.
|
|
8
|
+
*
|
|
9
|
+
* Detection: always true. Playwright is the safe-default web runner, and
|
|
10
|
+
* QA-session targets are overwhelmingly web apps with an HTTP spawn (PR
|
|
11
|
+
* preview, branch dev server, current worktree dev server, etc.). The
|
|
12
|
+
* priority gate above lets Maestro win when present; explicit
|
|
13
|
+
* `runner: '...'` overrides win before detect() runs at all.
|
|
14
|
+
*
|
|
15
|
+
* Hooks: see engine/qa-runners.js header for the contract.
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
const fs = require('fs');
|
|
19
|
+
|
|
20
|
+
const NAME = 'playwright';
|
|
21
|
+
const PRIORITY = 50;
|
|
22
|
+
const INSTALL_HINT =
|
|
23
|
+
'Playwright is required. From the target project run `npm i -D @playwright/test` and ' +
|
|
24
|
+
'`npx playwright install` (the second command installs the browser binaries). ' +
|
|
25
|
+
'Confirm with `npx playwright --version`.';
|
|
26
|
+
|
|
27
|
+
function detect(/* target, project */) {
|
|
28
|
+
return true;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
function _spawnUrl(spawnInfo) {
|
|
32
|
+
if (!spawnInfo) return '';
|
|
33
|
+
if (spawnInfo.attrs && typeof spawnInfo.attrs.base_url === 'string') return spawnInfo.attrs.base_url;
|
|
34
|
+
if (Array.isArray(spawnInfo.ports) && spawnInfo.ports.length > 0) {
|
|
35
|
+
return `http://localhost:${spawnInfo.ports[0]}`;
|
|
36
|
+
}
|
|
37
|
+
return '';
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
function _portsLabel(spawnInfo) {
|
|
41
|
+
if (!spawnInfo || !Array.isArray(spawnInfo.ports) || spawnInfo.ports.length === 0) return '(no ports)';
|
|
42
|
+
return spawnInfo.ports.join(', ');
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
function generateBrief(opts) {
|
|
46
|
+
opts = opts || {};
|
|
47
|
+
const baseUrl = _spawnUrl(opts.spawnInfo);
|
|
48
|
+
const ports = _portsLabel(opts.spawnInfo);
|
|
49
|
+
const flows = opts.flowsRaw || '(no flows provided)';
|
|
50
|
+
const sessionId = (opts.session && opts.session.id) || (opts.sessionId) || '<session-id>';
|
|
51
|
+
const capture = opts.capture || {};
|
|
52
|
+
const captureSummary = Object.entries(capture).filter(([, v]) => !!v).map(([k]) => k).join(', ') || '(none requested)';
|
|
53
|
+
|
|
54
|
+
return [
|
|
55
|
+
'## Playwright test draft',
|
|
56
|
+
'',
|
|
57
|
+
`Write a single Playwright test at \`engine/qa-tests/${sessionId}/test.spec.js\` that exercises:`,
|
|
58
|
+
'',
|
|
59
|
+
flows.trim(),
|
|
60
|
+
'',
|
|
61
|
+
'### Required structure',
|
|
62
|
+
'',
|
|
63
|
+
'```js',
|
|
64
|
+
"const { test, expect } = require('@playwright/test');",
|
|
65
|
+
'',
|
|
66
|
+
"test('qa-session flow', async ({ page }) => {",
|
|
67
|
+
baseUrl
|
|
68
|
+
? ` await page.goto('${baseUrl}');`
|
|
69
|
+
: ' // Managed-spawn did not advertise base_url; build the URL from the spawn port.',
|
|
70
|
+
' // … translate each flow step into Playwright actions (locator + assertion) …',
|
|
71
|
+
'});',
|
|
72
|
+
'```',
|
|
73
|
+
'',
|
|
74
|
+
baseUrl
|
|
75
|
+
? `- Live target: \`${baseUrl}\` (ports: ${ports}). Anchor every \`page.goto\` to this URL.`
|
|
76
|
+
: `- Live target ports: ${ports}. Build the URL from the port the dev server bound to.`,
|
|
77
|
+
'- Keep selectors resilient — prefer `getByRole`, `getByText`, `getByTestId` over CSS where possible.',
|
|
78
|
+
'- Each step the user described becomes one or more Playwright actions + a matching `expect(...).toBeVisible()` / `.toHaveURL()` / `.toHaveText()`.',
|
|
79
|
+
'',
|
|
80
|
+
'### Capture configuration',
|
|
81
|
+
'',
|
|
82
|
+
`Requested artifacts: ${captureSummary}. The EXECUTE phase wires \`--video\`, \`--screenshot\`, and trace flags ` +
|
|
83
|
+
'into the `npx playwright test` invocation. The test file itself should NOT hard-code recording (let the CLI flags drive it).',
|
|
84
|
+
].join('\n');
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
function executeBrief(opts) {
|
|
88
|
+
opts = opts || {};
|
|
89
|
+
const sessionId = (opts.session && opts.session.id) || (opts.sessionId) || '<session-id>';
|
|
90
|
+
const testFile = opts.testFile || `engine/qa-tests/${sessionId}/test.spec.js`;
|
|
91
|
+
const capture = opts.capture || {};
|
|
92
|
+
const outputDir = `engine/qa-artifacts/${sessionId}`;
|
|
93
|
+
|
|
94
|
+
const flags = [`--output ${outputDir}`, '--reporter=json'];
|
|
95
|
+
if (capture.video) flags.push('--video=on');
|
|
96
|
+
if (capture.screenshots) flags.push('--screenshot=on');
|
|
97
|
+
if (capture.logs) flags.push('--trace=on');
|
|
98
|
+
|
|
99
|
+
const artifactTypes = [];
|
|
100
|
+
if (capture.video) artifactTypes.push('`video` (webm)');
|
|
101
|
+
if (capture.screenshots) artifactTypes.push('`screenshot` (png)');
|
|
102
|
+
if (capture.logs) artifactTypes.push('`log` (trace.zip + stdout)');
|
|
103
|
+
if (artifactTypes.length === 0) artifactTypes.push('`log` (test stdout/stderr only — no capture flags requested)');
|
|
104
|
+
|
|
105
|
+
return [
|
|
106
|
+
'## Playwright execute',
|
|
107
|
+
'',
|
|
108
|
+
'Run the drafted test against the live managed-spawn:',
|
|
109
|
+
'',
|
|
110
|
+
'```bash',
|
|
111
|
+
`npx playwright test ${testFile} ${flags.join(' ')}`,
|
|
112
|
+
'```',
|
|
113
|
+
'',
|
|
114
|
+
`Write the result sidecar to \`agents/<your-id>/qa-run-result.json\` referencing artifacts under \`${outputDir}/\`.`,
|
|
115
|
+
'',
|
|
116
|
+
`Artifact types: ${artifactTypes.join(', ')}`,
|
|
117
|
+
].join('\n');
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
function validateOutputDir(dir) {
|
|
121
|
+
const errors = [];
|
|
122
|
+
if (typeof dir !== 'string' || dir === '') {
|
|
123
|
+
return { ok: false, errors: ['dir must be a non-empty string'] };
|
|
124
|
+
}
|
|
125
|
+
let entries;
|
|
126
|
+
try { entries = fs.readdirSync(dir); }
|
|
127
|
+
catch (err) {
|
|
128
|
+
return { ok: false, errors: [`qa-tests dir ${dir} unreadable: ${err.message}`] };
|
|
129
|
+
}
|
|
130
|
+
const specFiles = entries.filter(f => /\.spec\.(js|ts|mjs|cjs)$/.test(f));
|
|
131
|
+
if (specFiles.length === 0) {
|
|
132
|
+
errors.push(`no Playwright test file (*.spec.{js,ts,mjs,cjs}) found in ${dir}`);
|
|
133
|
+
}
|
|
134
|
+
return { ok: errors.length === 0, errors };
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
module.exports = {
|
|
138
|
+
name: NAME,
|
|
139
|
+
spec: {
|
|
140
|
+
priority: PRIORITY,
|
|
141
|
+
label: 'Playwright',
|
|
142
|
+
description: 'Safe-default web runner. Wins for any HTTP target unless a higher-priority runner detects.',
|
|
143
|
+
installHint: INSTALL_HINT,
|
|
144
|
+
detect,
|
|
145
|
+
generateBrief,
|
|
146
|
+
executeBrief,
|
|
147
|
+
validateOutputDir,
|
|
148
|
+
},
|
|
149
|
+
};
|
|
@@ -0,0 +1,323 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* engine/qa-runners.js — P-b8e1d4a6
|
|
3
|
+
*
|
|
4
|
+
* Pluggable test-runner registry for QA Sessions. Mirrors the
|
|
5
|
+
* engine/watches.js target-type registry pattern (registerTargetType /
|
|
6
|
+
* getTargetType / listTargetTypes / plugin folder discovery).
|
|
7
|
+
*
|
|
8
|
+
* A runner adapter teaches QA Sessions how to translate a natural-language
|
|
9
|
+
* QA flow into a runner-native test file, execute it against a managed-spawn
|
|
10
|
+
* target, and surface artifacts. Each runner is registered with five hooks:
|
|
11
|
+
*
|
|
12
|
+
* detect(target, project) → boolean. Inspect the target/project
|
|
13
|
+
* and return true when this runner is
|
|
14
|
+
* the right choice (e.g. a Playwright
|
|
15
|
+
* adapter returns true when the project
|
|
16
|
+
* has playwright.config.* in its repo).
|
|
17
|
+
* generateBrief(opts) → string|object. Produces the
|
|
18
|
+
* instructions handed to the DRAFT
|
|
19
|
+
* agent so it emits a runner-native
|
|
20
|
+
* test file under engine/qa-tests/<id>/.
|
|
21
|
+
* executeBrief(opts) → result. Produces the instructions /
|
|
22
|
+
* command the EXECUTE agent runs to
|
|
23
|
+
* invoke the drafted test against the
|
|
24
|
+
* live managed-spawn target.
|
|
25
|
+
* validateOutputDir(dir) → { ok: boolean, errors: string[] }.
|
|
26
|
+
* Called after DRAFT lands a test file
|
|
27
|
+
* to confirm the runner-native artifact
|
|
28
|
+
* is present before allowing EXECUTE
|
|
29
|
+
* to schedule.
|
|
30
|
+
* installHint → string. Human-readable instructions
|
|
31
|
+
* shown to the user when detect()
|
|
32
|
+
* returns true but the runner CLI is
|
|
33
|
+
* missing (e.g. "Run `npm i -D
|
|
34
|
+
* @playwright/test` and `npx playwright
|
|
35
|
+
* install` to enable this runner.").
|
|
36
|
+
*
|
|
37
|
+
* Resolution order (detectRunner):
|
|
38
|
+
* 1. Explicit override: when the caller passes a registered runner name,
|
|
39
|
+
* return that runner immediately — no detect() call. This is what the
|
|
40
|
+
* `qa-session` POST payload's `runner` field flows into.
|
|
41
|
+
* 2. Priority-desc iteration: registered runners are scanned in descending
|
|
42
|
+
* `priority` order (ties broken by name asc for determinism). The first
|
|
43
|
+
* runner whose `detect(target, project)` returns truthy wins.
|
|
44
|
+
* 3. No match: return null. Callers surface this as a "no QA runner
|
|
45
|
+
* detected — pass runner=… explicitly or install one" error.
|
|
46
|
+
*
|
|
47
|
+
* Plugin folder discovery:
|
|
48
|
+
* At module load, every `*.js` file under `<MINIONS_DIR>/qa-runners.d/`
|
|
49
|
+
* is required inside its own try/catch. Each file exports either:
|
|
50
|
+
*
|
|
51
|
+
* module.exports = { name: 'playwright', spec: { ... } };
|
|
52
|
+
*
|
|
53
|
+
* or an array of those objects for multi-runner plugins. Bad plugin files
|
|
54
|
+
* log WARN with the file path and do NOT block sibling plugins or built-in
|
|
55
|
+
* runners from registering. A missing qa-runners.d/ directory is a silent
|
|
56
|
+
* no-op (matches watches.d/ behaviour).
|
|
57
|
+
*
|
|
58
|
+
* Security: plugins are user-trusted JS (same trust level as `playbooks/`
|
|
59
|
+
* and `watches.d/`); no sandboxing. Hot-reload is not supported at module
|
|
60
|
+
* level — restart the engine to pick up new plugin files, or POST
|
|
61
|
+
* /api/qa/runners/reload (implemented in P-d2f5a8c9) which re-invokes
|
|
62
|
+
* loadRunnerPlugins().
|
|
63
|
+
*
|
|
64
|
+
* Built-in runner adapters (Maestro + Playwright) ship in P-c4a9e7f3.
|
|
65
|
+
*/
|
|
66
|
+
|
|
67
|
+
const fs = require('fs');
|
|
68
|
+
const path = require('path');
|
|
69
|
+
const shared = require('./shared');
|
|
70
|
+
const { log } = shared;
|
|
71
|
+
|
|
72
|
+
const _KEBAB_RE = /^[a-z0-9]+(?:-[a-z0-9]+)*$/;
|
|
73
|
+
const RUNNER_NAME_MAX = 64;
|
|
74
|
+
const REQUIRED_HOOK_FNS = ['detect', 'generateBrief', 'executeBrief', 'validateOutputDir'];
|
|
75
|
+
|
|
76
|
+
// Registry. Keyed by runner name (kebab-case). The whole object is the
|
|
77
|
+
// allowlist — listRunners / detectRunner / `POST /api/qa/runners/reload`
|
|
78
|
+
// derive their answers from it.
|
|
79
|
+
const RUNNERS = {};
|
|
80
|
+
|
|
81
|
+
function _isNonEmptyString(v) {
|
|
82
|
+
return typeof v === 'string' && v.length > 0;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
/**
|
|
86
|
+
* Validate a runner spec without registering it. Returns { ok, errors }.
|
|
87
|
+
* Never throws — `registerQaRunner` is the throwing wrapper.
|
|
88
|
+
*/
|
|
89
|
+
function validateRunnerSpec(name, spec) {
|
|
90
|
+
const errors = [];
|
|
91
|
+
|
|
92
|
+
if (!_isNonEmptyString(name)) {
|
|
93
|
+
errors.push('name is required (non-empty string)');
|
|
94
|
+
} else {
|
|
95
|
+
if (name.length > RUNNER_NAME_MAX) {
|
|
96
|
+
errors.push(`name exceeds ${RUNNER_NAME_MAX} chars`);
|
|
97
|
+
}
|
|
98
|
+
if (!_KEBAB_RE.test(name)) {
|
|
99
|
+
errors.push('name must be kebab-case (a-z, 0-9, hyphens; no leading/trailing hyphen, no double hyphen)');
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
if (!spec || typeof spec !== 'object' || Array.isArray(spec)) {
|
|
104
|
+
return { ok: false, errors: errors.concat(['spec must be a plain object']) };
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
if (typeof spec.priority !== 'number' || !Number.isFinite(spec.priority)) {
|
|
108
|
+
errors.push('spec.priority is required (finite number; higher = preferred)');
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
for (const hook of REQUIRED_HOOK_FNS) {
|
|
112
|
+
if (typeof spec[hook] !== 'function') {
|
|
113
|
+
errors.push(`spec.${hook} is required (function)`);
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
if (!_isNonEmptyString(spec.installHint)) {
|
|
118
|
+
errors.push('spec.installHint is required (non-empty string)');
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
return { ok: errors.length === 0, errors };
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
/**
|
|
125
|
+
* Register a QA runner adapter. Throws on validation failure so a bad
|
|
126
|
+
* registration is loud — the engine prints the file path + reason at boot
|
|
127
|
+
* via the per-plugin try/catch in _loadRunnerPlugins.
|
|
128
|
+
*
|
|
129
|
+
* Last-write-wins: re-registering an existing name replaces the previous
|
|
130
|
+
* spec. This matches watches.js so plugins can override built-ins.
|
|
131
|
+
*/
|
|
132
|
+
function registerQaRunner(name, spec) {
|
|
133
|
+
const v = validateRunnerSpec(name, spec);
|
|
134
|
+
if (!v.ok) {
|
|
135
|
+
const err = new Error(`registerQaRunner(${name}): ${v.errors.join('; ')}`);
|
|
136
|
+
err.validationErrors = v.errors;
|
|
137
|
+
throw err;
|
|
138
|
+
}
|
|
139
|
+
// Freeze the registry entry so callers can't mutate hooks after register-time.
|
|
140
|
+
// installHint stays as a string, priority stays as a number.
|
|
141
|
+
RUNNERS[name] = Object.freeze({
|
|
142
|
+
name,
|
|
143
|
+
priority: spec.priority,
|
|
144
|
+
detect: spec.detect,
|
|
145
|
+
generateBrief: spec.generateBrief,
|
|
146
|
+
executeBrief: spec.executeBrief,
|
|
147
|
+
validateOutputDir: spec.validateOutputDir,
|
|
148
|
+
installHint: spec.installHint,
|
|
149
|
+
// Optional human-readable label/description for dashboard pickers.
|
|
150
|
+
label: _isNonEmptyString(spec.label) ? spec.label : name,
|
|
151
|
+
description: _isNonEmptyString(spec.description) ? spec.description : '',
|
|
152
|
+
});
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
/** Returns the registered spec for a runner name, or null. */
|
|
156
|
+
function getRunner(name) {
|
|
157
|
+
if (!_isNonEmptyString(name)) return null;
|
|
158
|
+
return RUNNERS[name] || null;
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
/**
|
|
162
|
+
* Returns the registered runners as a serializable array, sorted priority
|
|
163
|
+
* desc, then name asc for determinism. Hooks are NOT included because they
|
|
164
|
+
* are functions — only metadata (name, priority, label, description,
|
|
165
|
+
* installHint) is returned, suitable for `GET /api/qa/runners` and
|
|
166
|
+
* dashboard pickers.
|
|
167
|
+
*/
|
|
168
|
+
function listRunners() {
|
|
169
|
+
return Object.keys(RUNNERS)
|
|
170
|
+
.sort()
|
|
171
|
+
.map(k => RUNNERS[k])
|
|
172
|
+
.sort((a, b) => (b.priority - a.priority) || a.name.localeCompare(b.name))
|
|
173
|
+
.map(r => ({
|
|
174
|
+
name: r.name,
|
|
175
|
+
priority: r.priority,
|
|
176
|
+
label: r.label,
|
|
177
|
+
description: r.description,
|
|
178
|
+
installHint: r.installHint,
|
|
179
|
+
}));
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
/**
|
|
183
|
+
* Resolve which runner should handle a target. Resolution order:
|
|
184
|
+
*
|
|
185
|
+
* 1. If `explicitRunner` names a registered runner, return it (no detect
|
|
186
|
+
* call). Unknown explicit names return null — the caller decides
|
|
187
|
+
* whether to fall through to auto-detect or surface an error.
|
|
188
|
+
* 2. Iterate registered runners by priority desc (ties broken by name
|
|
189
|
+
* asc) and return the first whose `detect(target, project)` returns
|
|
190
|
+
* truthy. Per-runner detect() errors are caught + logged at WARN so
|
|
191
|
+
* one broken adapter can't poison the iteration.
|
|
192
|
+
* 3. No match: return null.
|
|
193
|
+
*
|
|
194
|
+
* @param {object} target - target object from the session spec
|
|
195
|
+
* (e.g. { kind: 'pr', prId: 1234 }).
|
|
196
|
+
* @param {object|null} project - optional project config (or null when
|
|
197
|
+
* the session is central / project-less).
|
|
198
|
+
* @param {string|null} [explicitRunner] - explicit runner name from the
|
|
199
|
+
* session spec; takes precedence.
|
|
200
|
+
* @returns {object|null} - the frozen registry entry, or null.
|
|
201
|
+
*/
|
|
202
|
+
function detectRunner(target, project, explicitRunner) {
|
|
203
|
+
// Explicit-runner-wins: when the caller specifies a runner by name and it
|
|
204
|
+
// exists, use it immediately. detect() is intentionally NOT consulted —
|
|
205
|
+
// the user said this runner, so honour it. Unknown explicit names fall
|
|
206
|
+
// through to null (auto-detect is suppressed to avoid surprising the user).
|
|
207
|
+
if (explicitRunner !== undefined && explicitRunner !== null && explicitRunner !== '') {
|
|
208
|
+
if (!_isNonEmptyString(explicitRunner)) return null;
|
|
209
|
+
return RUNNERS[explicitRunner] || null;
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
const ordered = Object.keys(RUNNERS)
|
|
213
|
+
.sort()
|
|
214
|
+
.map(k => RUNNERS[k])
|
|
215
|
+
.sort((a, b) => (b.priority - a.priority) || a.name.localeCompare(b.name));
|
|
216
|
+
|
|
217
|
+
for (const r of ordered) {
|
|
218
|
+
let hit = false;
|
|
219
|
+
try {
|
|
220
|
+
hit = !!r.detect(target, project);
|
|
221
|
+
} catch (err) {
|
|
222
|
+
log('warn', `qa-runners: detect() threw for runner '${r.name}': ${err.message}`);
|
|
223
|
+
continue;
|
|
224
|
+
}
|
|
225
|
+
if (hit) return r;
|
|
226
|
+
}
|
|
227
|
+
return null;
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
// ── Plugin folder discovery (qa-runners.d/) ─────────────────────────────────
|
|
231
|
+
// Mirrors engine/watches.js:_loadPluginTargetTypes (W-mp7hg58e000b5212).
|
|
232
|
+
// Auto-loaded at module load time below. Tests bust the require cache via
|
|
233
|
+
// test/_helpers.js ISOLATED_MODULES so MINIONS_TEST_DIR/qa-runners.d/ takes
|
|
234
|
+
// effect on every createTestMinionsDir() boundary.
|
|
235
|
+
function _loadRunnerPlugins() {
|
|
236
|
+
const dir = path.join(shared.MINIONS_DIR, 'qa-runners.d');
|
|
237
|
+
let entries;
|
|
238
|
+
try {
|
|
239
|
+
entries = fs.readdirSync(dir);
|
|
240
|
+
} catch (err) {
|
|
241
|
+
if (err && err.code === 'ENOENT') return; // silent no-op
|
|
242
|
+
log('warn', `qa-runners.d/ readdir failed: ${err.message}`);
|
|
243
|
+
return;
|
|
244
|
+
}
|
|
245
|
+
for (const fname of entries.sort()) {
|
|
246
|
+
if (!fname.endsWith('.js')) continue;
|
|
247
|
+
const fp = path.join(dir, fname);
|
|
248
|
+
try {
|
|
249
|
+
// Bust cache so re-loads (tests, /api/qa/runners/reload) pick up edits.
|
|
250
|
+
try { delete require.cache[require.resolve(fp)]; } catch {}
|
|
251
|
+
const mod = require(fp);
|
|
252
|
+
const list = Array.isArray(mod) ? mod : [mod];
|
|
253
|
+
for (const entry of list) {
|
|
254
|
+
if (!entry || !entry.name || !entry.spec) {
|
|
255
|
+
log('warn', `qa-runners.d/${fname}: export missing { name, spec } — skipping entry`);
|
|
256
|
+
continue;
|
|
257
|
+
}
|
|
258
|
+
try {
|
|
259
|
+
registerQaRunner(entry.name, entry.spec);
|
|
260
|
+
log('info', `Loaded QA runner '${entry.name}' from qa-runners.d/${fname}`);
|
|
261
|
+
} catch (regErr) {
|
|
262
|
+
log('warn', `qa-runners.d/${fname}: registerQaRunner('${entry.name}') failed: ${regErr.message}`);
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
} catch (err) {
|
|
266
|
+
log('warn', `qa-runners.d/${fname}: load failed: ${err.message}`);
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
// Clear the in-process registry. Exposed for tests + `POST
|
|
272
|
+
// /api/qa/runners/reload` (P-d2f5a8c9), which calls _clearRunners() →
|
|
273
|
+
// _registerBuiltins() → _loadRunnerPlugins() to re-scan disk without
|
|
274
|
+
// restarting the engine.
|
|
275
|
+
function _clearRunners() {
|
|
276
|
+
for (const k of Object.keys(RUNNERS)) delete RUNNERS[k];
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
// Register the built-in adapters that ship with the engine. Plugins under
|
|
280
|
+
// <MINIONS_DIR>/qa-runners.d/ run AFTER this and may override built-ins by
|
|
281
|
+
// name (last-write-wins, mirrors watches.js). Surface registration
|
|
282
|
+
// failures as WARN so a broken built-in does not block other adapters or
|
|
283
|
+
// disk plugins. Built-ins are required relative to this file (not
|
|
284
|
+
// MINIONS_DIR) so test isolation can't strand them on a stale module path.
|
|
285
|
+
function _registerBuiltins() {
|
|
286
|
+
const BUILTIN_FILES = ['./qa-runners/maestro', './qa-runners/playwright'];
|
|
287
|
+
for (const rel of BUILTIN_FILES) {
|
|
288
|
+
try {
|
|
289
|
+
const resolved = require.resolve(rel);
|
|
290
|
+
delete require.cache[resolved];
|
|
291
|
+
const mod = require(rel);
|
|
292
|
+
if (!mod || !mod.name || !mod.spec) {
|
|
293
|
+
log('warn', `qa-runners built-in ${rel}: export missing { name, spec } — skipping`);
|
|
294
|
+
continue;
|
|
295
|
+
}
|
|
296
|
+
try {
|
|
297
|
+
registerQaRunner(mod.name, mod.spec);
|
|
298
|
+
} catch (regErr) {
|
|
299
|
+
log('warn', `qa-runners built-in ${rel}: registerQaRunner('${mod.name}') failed: ${regErr.message}`);
|
|
300
|
+
}
|
|
301
|
+
} catch (err) {
|
|
302
|
+
log('warn', `qa-runners built-in ${rel}: load failed: ${err.message}`);
|
|
303
|
+
}
|
|
304
|
+
}
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
_registerBuiltins();
|
|
308
|
+
_loadRunnerPlugins();
|
|
309
|
+
|
|
310
|
+
module.exports = {
|
|
311
|
+
RUNNER_NAME_MAX,
|
|
312
|
+
REQUIRED_HOOK_FNS,
|
|
313
|
+
validateRunnerSpec,
|
|
314
|
+
registerQaRunner,
|
|
315
|
+
getRunner,
|
|
316
|
+
listRunners,
|
|
317
|
+
detectRunner,
|
|
318
|
+
// Exposed for tests + the reload endpoint (P-d2f5a8c9).
|
|
319
|
+
_loadRunnerPlugins,
|
|
320
|
+
_registerBuiltins,
|
|
321
|
+
_clearRunners,
|
|
322
|
+
_RUNNERS: RUNNERS,
|
|
323
|
+
};
|