comfy-qa 1.0.0 → 1.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +73 -6
- package/package.json +18 -3
- package/src/agent/browser-agent.ts +298 -0
- package/src/agent/demo-editor.ts +450 -0
- package/src/agent/demo-research.ts +725 -0
- package/src/agent/orchestrator.ts +268 -0
- package/src/agent/qa-research.ts +813 -0
- package/src/agent/research.ts +221 -0
- package/src/browser/hud.ts +136 -0
- package/src/browser/recorder.ts +131 -0
- package/src/cli.ts +69 -28
- package/src/commands/full.ts +40 -0
- package/src/commands/issue.ts +23 -0
- package/src/commands/pr.ts +23 -0
- package/src/commands/setup.ts +46 -0
- package/src/recorder/narration.ts +176 -0
- package/src/recorder/post-mix.ts +81 -0
- package/src/report/e2e-test.ts +132 -0
- package/src/report/generate.ts +271 -0
- package/src/utils/comfyui.ts +349 -0
- package/src/utils/github.ts +87 -0
- package/src/utils/parse-url.ts +11 -0
- package/src/utils/qa-skill.ts +376 -0
|
@@ -0,0 +1,221 @@
|
|
|
1
|
+
import { $ } from "bun";
|
|
2
|
+
import type { PRInfo, IssueInfo } from "../utils/github";
|
|
3
|
+
|
|
4
|
+
/** Call Claude via CLI (already authenticated) or SDK if ANTHROPIC_API_KEY is set */
|
|
5
|
+
async function callClaude(prompt: string): Promise<string> {
|
|
6
|
+
const apiKey = process.env.ANTHROPIC_API_KEY_QA ?? process.env.ANTHROPIC_API_KEY;
|
|
7
|
+
if (apiKey) {
|
|
8
|
+
const Anthropic = (await import("@anthropic-ai/sdk")).default;
|
|
9
|
+
const client = new Anthropic({ apiKey });
|
|
10
|
+
const response = await client.messages.create({
|
|
11
|
+
model: "claude-opus-4-6",
|
|
12
|
+
max_tokens: 4096,
|
|
13
|
+
messages: [{ role: "user", content: prompt }],
|
|
14
|
+
});
|
|
15
|
+
return response.content[0].type === "text" ? response.content[0].text : "";
|
|
16
|
+
}
|
|
17
|
+
// Fallback: pipe through claude CLI via stdin
|
|
18
|
+
const proc = Bun.spawn(["claude", "--print", "--model", "claude-opus-4-6"], {
|
|
19
|
+
stdin: new TextEncoder().encode(prompt),
|
|
20
|
+
stdout: "pipe",
|
|
21
|
+
stderr: "pipe",
|
|
22
|
+
});
|
|
23
|
+
const output = await new Response(proc.stdout).text();
|
|
24
|
+
await proc.exited;
|
|
25
|
+
return output;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/** Extract JSON from Claude response, handling code blocks and markdown wrapping */
|
|
29
|
+
function parseResearchJSON(text: string): ResearchResult {
|
|
30
|
+
// Try extracting from ```json ... ``` code block first
|
|
31
|
+
const codeBlockMatch = text.match(/```(?:json)?\s*\n?([\s\S]*?)```/);
|
|
32
|
+
const candidate = codeBlockMatch ? codeBlockMatch[1].trim() : text;
|
|
33
|
+
|
|
34
|
+
// Find the outermost JSON object
|
|
35
|
+
const jsonMatch = candidate.match(/\{[\s\S]*\}/);
|
|
36
|
+
if (!jsonMatch) throw new Error("Research agent returned no JSON");
|
|
37
|
+
|
|
38
|
+
try {
|
|
39
|
+
return JSON.parse(jsonMatch[0]) as ResearchResult;
|
|
40
|
+
} catch (e) {
|
|
41
|
+
// Try cleaning common issues: trailing commas, comments
|
|
42
|
+
const cleaned = jsonMatch[0]
|
|
43
|
+
.replace(/,\s*([\]}])/g, "$1") // trailing commas
|
|
44
|
+
.replace(/\/\/.*$/gm, ""); // line comments
|
|
45
|
+
return JSON.parse(cleaned) as ResearchResult;
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
export interface ResearchResult {
|
|
50
|
+
summary: string;
|
|
51
|
+
bugType: "bug" | "feature" | "regression" | "performance" | "unknown";
|
|
52
|
+
severity: "critical" | "high" | "medium" | "low";
|
|
53
|
+
affectedArea: string;
|
|
54
|
+
reproductionSteps: string[];
|
|
55
|
+
expectedBehavior: string;
|
|
56
|
+
actualBehavior: string;
|
|
57
|
+
rootCauseSummary: string;
|
|
58
|
+
qaChecklist: QAChecklistItem[];
|
|
59
|
+
testScenarios: TestScenario[];
|
|
60
|
+
risks: string[];
|
|
61
|
+
relatedFiles: string[];
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
export interface QAChecklistItem {
|
|
65
|
+
id: string;
|
|
66
|
+
category: "functional" | "visual" | "regression" | "edge_case" | "performance";
|
|
67
|
+
description: string;
|
|
68
|
+
steps: string[];
|
|
69
|
+
expectedResult: string;
|
|
70
|
+
priority: "P0" | "P1" | "P2";
|
|
71
|
+
status: "pending" | "pass" | "fail" | "blocked";
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
export interface TestScenario {
|
|
75
|
+
name: string;
|
|
76
|
+
description: string;
|
|
77
|
+
preconditions: string[];
|
|
78
|
+
steps: string[];
|
|
79
|
+
expectedOutcome: string;
|
|
80
|
+
playwrightHint: string;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
export async function researchPR(pr: PRInfo): Promise<ResearchResult> {
|
|
84
|
+
console.log(` [research] Analyzing PR #${pr.number}: ${pr.title}`);
|
|
85
|
+
|
|
86
|
+
const filesSummary = pr.files
|
|
87
|
+
.slice(0, 30)
|
|
88
|
+
.map((f) => ` ${f.path} (+${f.additions}/-${f.deletions})`)
|
|
89
|
+
.join("\n");
|
|
90
|
+
|
|
91
|
+
const commentsSummary = pr.comments
|
|
92
|
+
.slice(0, 10)
|
|
93
|
+
.map((c) => ` [${c.author}]: ${c.body.slice(0, 300)}`)
|
|
94
|
+
.join("\n---\n");
|
|
95
|
+
|
|
96
|
+
const prompt = `You are a senior QA engineer analyzing a GitHub PR for ComfyUI (a node-based AI image generation UI).
|
|
97
|
+
|
|
98
|
+
## PR #${pr.number}: ${pr.title}
|
|
99
|
+
**URL**: ${pr.url}
|
|
100
|
+
**Author**: ${pr.author}
|
|
101
|
+
**Branch**: ${pr.headRefName} → ${pr.baseRefName}
|
|
102
|
+
**Labels**: ${pr.labels.join(", ") || "none"}
|
|
103
|
+
|
|
104
|
+
## PR Description
|
|
105
|
+
${pr.body.slice(0, 3000)}
|
|
106
|
+
|
|
107
|
+
## Changed Files (${pr.files.length} total)
|
|
108
|
+
${filesSummary}
|
|
109
|
+
|
|
110
|
+
## Key Comments
|
|
111
|
+
${commentsSummary || "(none)"}
|
|
112
|
+
|
|
113
|
+
---
|
|
114
|
+
|
|
115
|
+
Produce a comprehensive QA analysis as JSON with this exact schema:
|
|
116
|
+
{
|
|
117
|
+
"summary": "2-3 sentence executive summary",
|
|
118
|
+
"bugType": "bug|feature|regression|performance|unknown",
|
|
119
|
+
"severity": "critical|high|medium|low",
|
|
120
|
+
"affectedArea": "e.g. canvas, workflow tabs, sidebar, node editor",
|
|
121
|
+
"reproductionSteps": ["step 1", "step 2", ...],
|
|
122
|
+
"expectedBehavior": "what should happen",
|
|
123
|
+
"actualBehavior": "what actually happens (the bug)",
|
|
124
|
+
"rootCauseSummary": "technical root cause in 1-2 sentences",
|
|
125
|
+
"qaChecklist": [
|
|
126
|
+
{
|
|
127
|
+
"id": "QA-001",
|
|
128
|
+
"category": "functional|visual|regression|edge_case|performance",
|
|
129
|
+
"description": "test description",
|
|
130
|
+
"steps": ["step 1", ...],
|
|
131
|
+
"expectedResult": "expected outcome",
|
|
132
|
+
"priority": "P0|P1|P2",
|
|
133
|
+
"status": "pending"
|
|
134
|
+
}
|
|
135
|
+
],
|
|
136
|
+
"testScenarios": [
|
|
137
|
+
{
|
|
138
|
+
"name": "scenario name",
|
|
139
|
+
"description": "what this tests",
|
|
140
|
+
"preconditions": ["precondition 1", ...],
|
|
141
|
+
"steps": ["step 1", ...],
|
|
142
|
+
"expectedOutcome": "what should happen",
|
|
143
|
+
"playwrightHint": "which playwright selectors/actions to use"
|
|
144
|
+
}
|
|
145
|
+
],
|
|
146
|
+
"risks": ["risk 1", "risk 2"],
|
|
147
|
+
"relatedFiles": ["path/to/file.ts"]
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
Create 5-8 QA checklist items covering: happy path, regression, edge cases, visual checks.
|
|
151
|
+
Create 3-5 test scenarios focused on the core bug/feature.
|
|
152
|
+
Be specific and actionable. Use ComfyUI-specific terminology.`;
|
|
153
|
+
|
|
154
|
+
const text = await callClaude(prompt);
|
|
155
|
+
return parseResearchJSON(text);
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
export async function researchIssue(issue: IssueInfo): Promise<ResearchResult> {
|
|
159
|
+
console.log(` [research] Analyzing issue #${issue.number}: ${issue.title}`);
|
|
160
|
+
|
|
161
|
+
const commentsSummary = issue.comments
|
|
162
|
+
.slice(0, 8)
|
|
163
|
+
.map((c) => ` [${c.author}]: ${c.body.slice(0, 500)}`)
|
|
164
|
+
.join("\n---\n");
|
|
165
|
+
|
|
166
|
+
const prompt = `You are a senior QA engineer analyzing a GitHub issue for ComfyUI (a node-based AI image generation UI).
|
|
167
|
+
|
|
168
|
+
## Issue #${issue.number}: ${issue.title}
|
|
169
|
+
**URL**: ${issue.url}
|
|
170
|
+
**Author**: ${issue.author}
|
|
171
|
+
**Labels**: ${issue.labels.join(", ") || "none"}
|
|
172
|
+
**State**: ${issue.state}
|
|
173
|
+
|
|
174
|
+
## Issue Description
|
|
175
|
+
${issue.body.slice(0, 3000)}
|
|
176
|
+
|
|
177
|
+
## Comments (${issue.comments.length} total)
|
|
178
|
+
${commentsSummary || "(none)"}
|
|
179
|
+
|
|
180
|
+
---
|
|
181
|
+
|
|
182
|
+
Produce a comprehensive QA analysis as JSON with this exact schema:
|
|
183
|
+
{
|
|
184
|
+
"summary": "2-3 sentence executive summary",
|
|
185
|
+
"bugType": "bug|feature|regression|performance|unknown",
|
|
186
|
+
"severity": "critical|high|medium|low",
|
|
187
|
+
"affectedArea": "e.g. canvas, workflow tabs, sidebar, node editor",
|
|
188
|
+
"reproductionSteps": ["step 1", "step 2", ...],
|
|
189
|
+
"expectedBehavior": "what should happen",
|
|
190
|
+
"actualBehavior": "what actually happens",
|
|
191
|
+
"rootCauseSummary": "technical root cause hypothesis",
|
|
192
|
+
"qaChecklist": [
|
|
193
|
+
{
|
|
194
|
+
"id": "QA-001",
|
|
195
|
+
"category": "functional|visual|regression|edge_case|performance",
|
|
196
|
+
"description": "test description",
|
|
197
|
+
"steps": ["step 1", ...],
|
|
198
|
+
"expectedResult": "expected outcome",
|
|
199
|
+
"priority": "P0|P1|P2",
|
|
200
|
+
"status": "pending"
|
|
201
|
+
}
|
|
202
|
+
],
|
|
203
|
+
"testScenarios": [
|
|
204
|
+
{
|
|
205
|
+
"name": "scenario name",
|
|
206
|
+
"description": "what this tests",
|
|
207
|
+
"preconditions": ["precondition 1", ...],
|
|
208
|
+
"steps": ["step 1", ...],
|
|
209
|
+
"expectedOutcome": "what should happen",
|
|
210
|
+
"playwrightHint": "playwright selectors/actions hint"
|
|
211
|
+
}
|
|
212
|
+
],
|
|
213
|
+
"risks": ["risk 1", "risk 2"],
|
|
214
|
+
"relatedFiles": []
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
Create 5-8 QA checklist items. Create 3-5 test scenarios. Be specific and actionable.`;
|
|
218
|
+
|
|
219
|
+
const text = await callClaude(prompt);
|
|
220
|
+
return parseResearchJSON(text);
|
|
221
|
+
}
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
import type { Page } from "playwright";
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Inject the comfy-qa info panel (step/plan/status/annotation).
|
|
5
|
+
* Cursor + keystroke display is handled by qa-hud.
|
|
6
|
+
*/
|
|
7
|
+
export async function injectHUD(page: Page, title: string): Promise<void> {
|
|
8
|
+
await page.addStyleTag({
|
|
9
|
+
content: `
|
|
10
|
+
#comfy-qa-hud {
|
|
11
|
+
position: fixed;
|
|
12
|
+
top: 12px;
|
|
13
|
+
left: 12px;
|
|
14
|
+
z-index: 2147483640;
|
|
15
|
+
background: rgba(0,0,0,0.82);
|
|
16
|
+
color: #00ff88;
|
|
17
|
+
font-family: 'JetBrains Mono', 'Fira Code', monospace;
|
|
18
|
+
font-size: 13px;
|
|
19
|
+
padding: 10px 14px;
|
|
20
|
+
border-radius: 8px;
|
|
21
|
+
border: 1px solid #00ff88;
|
|
22
|
+
min-width: 340px;
|
|
23
|
+
pointer-events: none;
|
|
24
|
+
line-height: 1.6;
|
|
25
|
+
box-shadow: 0 0 20px rgba(0,255,136,0.3);
|
|
26
|
+
}
|
|
27
|
+
#comfy-qa-hud .hud-title { color: #fff; font-weight: bold; margin-bottom: 4px; font-size: 11px; letter-spacing: 1px; text-transform: uppercase; }
|
|
28
|
+
#comfy-qa-hud .hud-step { color: #ffeb3b; }
|
|
29
|
+
#comfy-qa-hud .hud-plan { color: #80deea; font-size: 11px; margin-top: 4px; }
|
|
30
|
+
#comfy-qa-hud .hud-status { color: #ff9800; }
|
|
31
|
+
#comfy-qa-hud .hud-ts { color: #666; font-size: 10px; }
|
|
32
|
+
#comfy-qa-annotation {
|
|
33
|
+
position: fixed;
|
|
34
|
+
z-index: 2147483639;
|
|
35
|
+
background: rgba(255,235,59,0.9);
|
|
36
|
+
color: #000;
|
|
37
|
+
padding: 6px 10px;
|
|
38
|
+
border-radius: 4px;
|
|
39
|
+
font-family: monospace;
|
|
40
|
+
font-size: 12px;
|
|
41
|
+
font-weight: bold;
|
|
42
|
+
pointer-events: none;
|
|
43
|
+
display: none;
|
|
44
|
+
max-width: 320px;
|
|
45
|
+
box-shadow: 0 2px 8px rgba(0,0,0,0.5);
|
|
46
|
+
}
|
|
47
|
+
`,
|
|
48
|
+
});
|
|
49
|
+
|
|
50
|
+
await page.evaluate((t: string) => {
|
|
51
|
+
// Info panel
|
|
52
|
+
const hud = document.createElement("div");
|
|
53
|
+
hud.id = "comfy-qa-hud";
|
|
54
|
+
hud.innerHTML = `
|
|
55
|
+
<div class="hud-title">Comfy-QA Agent</div>
|
|
56
|
+
<div class="hud-step" id="hud-step">Initializing…</div>
|
|
57
|
+
<div class="hud-plan" id="hud-plan"></div>
|
|
58
|
+
<div class="hud-status" id="hud-status"></div>
|
|
59
|
+
<div class="hud-ts" id="hud-ts"></div>
|
|
60
|
+
`;
|
|
61
|
+
document.body.appendChild(hud);
|
|
62
|
+
|
|
63
|
+
// Annotation bubble
|
|
64
|
+
const ann = document.createElement("div");
|
|
65
|
+
ann.id = "comfy-qa-annotation";
|
|
66
|
+
document.body.appendChild(ann);
|
|
67
|
+
|
|
68
|
+
// Expose update APIs
|
|
69
|
+
(window as any).__comfyQA = {
|
|
70
|
+
setStep: (text: string) => {
|
|
71
|
+
const el = document.getElementById("hud-step");
|
|
72
|
+
if (el) el.textContent = "▶ " + text;
|
|
73
|
+
const ts = document.getElementById("hud-ts");
|
|
74
|
+
if (ts) ts.textContent = new Date().toISOString();
|
|
75
|
+
},
|
|
76
|
+
setPlan: (text: string) => {
|
|
77
|
+
const el = document.getElementById("hud-plan");
|
|
78
|
+
if (el) el.textContent = "📋 " + text;
|
|
79
|
+
},
|
|
80
|
+
setStatus: (text: string) => {
|
|
81
|
+
const el = document.getElementById("hud-status");
|
|
82
|
+
if (el) el.textContent = "⚡ " + text;
|
|
83
|
+
},
|
|
84
|
+
annotate: (x: number, y: number, text: string, durationMs = 2500) => {
|
|
85
|
+
const a = document.getElementById("comfy-qa-annotation");
|
|
86
|
+
if (!a) return;
|
|
87
|
+
a.textContent = text;
|
|
88
|
+
a.style.left = Math.min(x + 20, window.innerWidth - 340) + "px";
|
|
89
|
+
a.style.top = Math.max(y - 40, 10) + "px";
|
|
90
|
+
a.style.display = "block";
|
|
91
|
+
setTimeout(() => { a.style.display = "none"; }, durationMs);
|
|
92
|
+
},
|
|
93
|
+
};
|
|
94
|
+
|
|
95
|
+
(window as any).__comfyQA.setStep("Starting QA session: " + t);
|
|
96
|
+
}, title);
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/** Update HUD step text */
|
|
100
|
+
export async function hudStep(page: Page, step: string): Promise<void> {
|
|
101
|
+
try {
|
|
102
|
+
await page.evaluate((s: string) => (window as any).__comfyQA?.setStep(s), step);
|
|
103
|
+
} catch {}
|
|
104
|
+
console.log(` [hud] ${step}`);
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/** Update HUD plan text */
|
|
108
|
+
export async function hudPlan(page: Page, plan: string): Promise<void> {
|
|
109
|
+
try {
|
|
110
|
+
await page.evaluate((s: string) => (window as any).__comfyQA?.setPlan(s), plan);
|
|
111
|
+
} catch {}
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/** Update HUD status */
|
|
115
|
+
export async function hudStatus(page: Page, status: string): Promise<void> {
|
|
116
|
+
try {
|
|
117
|
+
await page.evaluate((s: string) => (window as any).__comfyQA?.setStatus(s), status);
|
|
118
|
+
} catch {}
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
/** Show annotation bubble near coordinates */
|
|
122
|
+
export async function hudAnnotate(
|
|
123
|
+
page: Page,
|
|
124
|
+
x: number,
|
|
125
|
+
y: number,
|
|
126
|
+
text: string,
|
|
127
|
+
durationMs = 3000
|
|
128
|
+
): Promise<void> {
|
|
129
|
+
try {
|
|
130
|
+
await page.evaluate(
|
|
131
|
+
({ x, y, text, durationMs }: { x: number; y: number; text: string; durationMs: number }) =>
|
|
132
|
+
(window as any).__comfyQA?.annotate(x, y, text, durationMs),
|
|
133
|
+
{ x, y, text, durationMs }
|
|
134
|
+
);
|
|
135
|
+
} catch {}
|
|
136
|
+
}
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
import { chromium, type Browser, type BrowserContext, type Page } from "playwright";
|
|
2
|
+
import * as path from "path";
|
|
3
|
+
import * as fs from "fs";
|
|
4
|
+
import { applyHud } from "../../lib/demowright/dist/setup.mjs";
|
|
5
|
+
import { injectHUD, hudStep, hudPlan, hudStatus, hudAnnotate } from "./hud";
|
|
6
|
+
|
|
7
|
+
export interface RecorderSession {
|
|
8
|
+
page: Page;
|
|
9
|
+
context: BrowserContext;
|
|
10
|
+
browser: Browser;
|
|
11
|
+
screenshotDir: string;
|
|
12
|
+
screenshots: string[];
|
|
13
|
+
step: (text: string) => Promise<void>;
|
|
14
|
+
plan: (text: string) => Promise<void>;
|
|
15
|
+
status: (text: string) => Promise<void>;
|
|
16
|
+
annotate: (x: number, y: number, text: string, ms?: number) => Promise<void>;
|
|
17
|
+
screenshot: (name: string) => Promise<string>;
|
|
18
|
+
/** Set narration durations and the demo start timestamp */
|
|
19
|
+
attachNarration: (durations: Map<string, number>) => void;
|
|
20
|
+
/** Run a narrated step: updates HUD and waits for the segment's audio duration */
|
|
21
|
+
narrate: (id: string, hudText: string) => Promise<void>;
|
|
22
|
+
/** Get demo start time (set when first narrate() is called or attachNarration) */
|
|
23
|
+
getDemoStartMs: () => number;
|
|
24
|
+
stop: () => Promise<void>;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export async function startRecorder(
|
|
28
|
+
outputDir: string,
|
|
29
|
+
videoName = "qa-session",
|
|
30
|
+
viewportWidth = 1280,
|
|
31
|
+
viewportHeight = 800
|
|
32
|
+
): Promise<RecorderSession> {
|
|
33
|
+
fs.mkdirSync(outputDir, { recursive: true });
|
|
34
|
+
const screenshotDir = path.join(outputDir, "screenshots");
|
|
35
|
+
fs.mkdirSync(screenshotDir, { recursive: true });
|
|
36
|
+
|
|
37
|
+
const browser = await chromium.launch({ headless: true });
|
|
38
|
+
const context = await browser.newContext({
|
|
39
|
+
viewport: { width: viewportWidth, height: viewportHeight },
|
|
40
|
+
recordVideo: {
|
|
41
|
+
dir: outputDir,
|
|
42
|
+
size: { width: viewportWidth, height: viewportHeight },
|
|
43
|
+
},
|
|
44
|
+
});
|
|
45
|
+
|
|
46
|
+
// Apply qa-hud: cursor overlay, keystroke display, action slowdown
|
|
47
|
+
await applyHud(context, {
|
|
48
|
+
cursor: true,
|
|
49
|
+
keyboard: true,
|
|
50
|
+
cursorStyle: "default",
|
|
51
|
+
actionDelay: 150,
|
|
52
|
+
});
|
|
53
|
+
|
|
54
|
+
const page = await context.newPage();
|
|
55
|
+
const screenshots: string[] = [];
|
|
56
|
+
|
|
57
|
+
page.on("console", (msg) => {
|
|
58
|
+
if (msg.type() === "error") {
|
|
59
|
+
console.error(` [browser:error] ${msg.text()}`);
|
|
60
|
+
}
|
|
61
|
+
});
|
|
62
|
+
|
|
63
|
+
const step = async (text: string) => {
|
|
64
|
+
await hudStep(page, text);
|
|
65
|
+
await page.waitForTimeout(300);
|
|
66
|
+
};
|
|
67
|
+
const plan = async (text: string) => hudPlan(page, text);
|
|
68
|
+
const status = async (text: string) => hudStatus(page, text);
|
|
69
|
+
const annotate = async (x: number, y: number, text: string, ms = 3000) =>
|
|
70
|
+
hudAnnotate(page, x, y, text, ms);
|
|
71
|
+
|
|
72
|
+
const screenshot = async (name: string): Promise<string> => {
|
|
73
|
+
const file = path.join(screenshotDir, `${name}.png`);
|
|
74
|
+
await page.screenshot({ path: file, fullPage: false });
|
|
75
|
+
screenshots.push(file);
|
|
76
|
+
console.log(` [screenshot] ${file}`);
|
|
77
|
+
return file;
|
|
78
|
+
};
|
|
79
|
+
|
|
80
|
+
let narrationDurations: Map<string, number> | null = null;
|
|
81
|
+
let demoStartMs = 0;
|
|
82
|
+
const recordingStartMs = Date.now();
|
|
83
|
+
|
|
84
|
+
const attachNarration = (durations: Map<string, number>) => {
|
|
85
|
+
narrationDurations = durations;
|
|
86
|
+
demoStartMs = Date.now();
|
|
87
|
+
};
|
|
88
|
+
|
|
89
|
+
const getDemoStartMs = () => demoStartMs || recordingStartMs;
|
|
90
|
+
|
|
91
|
+
const narrate = async (id: string, hudText: string) => {
|
|
92
|
+
if (!demoStartMs) demoStartMs = Date.now();
|
|
93
|
+
await hudStep(page, hudText);
|
|
94
|
+
const dur = narrationDurations?.get(id) ?? 1500;
|
|
95
|
+
const segStart = Date.now();
|
|
96
|
+
// Caller can do work between this and we'll sleep the remainder
|
|
97
|
+
// For now just sleep the full duration since we don't yield mid-call
|
|
98
|
+
const elapsed = Date.now() - segStart;
|
|
99
|
+
const remaining = dur - elapsed;
|
|
100
|
+
if (remaining > 50) await page.waitForTimeout(remaining);
|
|
101
|
+
};
|
|
102
|
+
|
|
103
|
+
const stop = async () => {
|
|
104
|
+
await page.waitForTimeout(1500);
|
|
105
|
+
const video = await page.video()?.path();
|
|
106
|
+
await context.close();
|
|
107
|
+
await browser.close();
|
|
108
|
+
if (video) {
|
|
109
|
+
const dest = path.join(outputDir, `${videoName}.webm`);
|
|
110
|
+
fs.renameSync(video, dest);
|
|
111
|
+
console.log(` [video] Saved → ${dest}`);
|
|
112
|
+
}
|
|
113
|
+
};
|
|
114
|
+
|
|
115
|
+
return { page, context, browser, screenshotDir, screenshots, step, plan, status, annotate, screenshot, attachNarration, narrate, getDemoStartMs, stop };
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
/** Navigate to URL and inject comfy-qa info panel */
|
|
119
|
+
export async function navigateWithHUD(
|
|
120
|
+
session: RecorderSession,
|
|
121
|
+
url: string,
|
|
122
|
+
title: string
|
|
123
|
+
): Promise<void> {
|
|
124
|
+
await session.page.goto(url, { waitUntil: "domcontentloaded", timeout: 30000 });
|
|
125
|
+
try {
|
|
126
|
+
await injectHUD(session.page, title);
|
|
127
|
+
} catch {
|
|
128
|
+
// HUD injection fails on some pages (cross-origin) — that's OK
|
|
129
|
+
}
|
|
130
|
+
await session.page.waitForTimeout(500);
|
|
131
|
+
}
|
package/src/cli.ts
CHANGED
|
@@ -1,37 +1,78 @@
|
|
|
1
1
|
#!/usr/bin/env bun
|
|
2
2
|
|
|
3
|
-
import {
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
3
|
+
import { commandPR } from "./commands/pr";
|
|
4
|
+
import { commandIssue } from "./commands/issue";
|
|
5
|
+
import { commandFull } from "./commands/full";
|
|
6
|
+
import { commandSetup } from "./commands/setup";
|
|
7
|
+
import { parseGitHubUrl } from "./utils/parse-url";
|
|
8
|
+
|
|
9
|
+
const args = Bun.argv.slice(2);
|
|
10
|
+
const cmd = args[0];
|
|
11
|
+
const rest = args.slice(1);
|
|
12
|
+
|
|
13
|
+
const HELP = `cmqa — E2E QA automation for frontend repos
|
|
14
|
+
|
|
15
|
+
USAGE
|
|
16
|
+
cmqa setup Emit setup prompt for your agent
|
|
17
|
+
cmqa <github-url> Auto-detect PR or issue from URL
|
|
18
|
+
cmqa pr <github-url | owner/repo#N> Research & QA a pull request
|
|
19
|
+
cmqa issue <github-url | owner/repo#N> Research & QA an issue / bug report
|
|
20
|
+
cmqa full <owner/repo> Batch QA recent open issues
|
|
19
21
|
|
|
20
|
-
|
|
21
|
-
|
|
22
|
+
SETUP (one-shot)
|
|
23
|
+
Tell your agent: "run npx cmqa setup"
|
|
24
|
+
The agent reads the emitted prompt and sets up a complete QA workflow
|
|
25
|
+
for the current repo — Playwright config, E2E tests, skill files, etc.
|
|
22
26
|
|
|
23
|
-
|
|
24
|
-
|
|
27
|
+
OPTIONS
|
|
28
|
+
--no-record Disable video recording (recording is ON by default)
|
|
29
|
+
--add-comment Post QA report as a comment on the GitHub issue/PR
|
|
30
|
+
--comfy-url <url> Point to a running dev server (default: auto-detect)
|
|
31
|
+
--limit N (full only) Number of issues to process [default: 5]
|
|
32
|
+
-h, --help Show this help
|
|
33
|
+
-v, --version Show version
|
|
25
34
|
|
|
26
|
-
|
|
27
|
-
|
|
35
|
+
EXAMPLES
|
|
36
|
+
cmqa setup
|
|
37
|
+
cmqa https://github.com/org/repo/pull/123
|
|
38
|
+
cmqa https://github.com/org/repo/issues/456
|
|
39
|
+
cmqa pr org/repo#123
|
|
40
|
+
cmqa issue org/repo#456 --no-record
|
|
41
|
+
cmqa full org/repo --limit 3
|
|
42
|
+
`;
|
|
28
43
|
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
-v, --version Show version
|
|
32
|
-
`);
|
|
44
|
+
if (!cmd || cmd === "-h" || cmd === "--help") {
|
|
45
|
+
console.log(HELP);
|
|
33
46
|
process.exit(0);
|
|
34
47
|
}
|
|
35
48
|
|
|
36
|
-
|
|
37
|
-
|
|
49
|
+
if (cmd === "-v" || cmd === "--version") {
|
|
50
|
+
const pkg = await import("../package.json");
|
|
51
|
+
console.log(pkg.default.version);
|
|
52
|
+
process.exit(0);
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
// Try GitHub URL as first arg (no subcommand needed)
|
|
56
|
+
const urlParsed = parseGitHubUrl(cmd);
|
|
57
|
+
if (urlParsed) {
|
|
58
|
+
const handler = urlParsed.type === "pr" ? commandPR : commandIssue;
|
|
59
|
+
await handler([urlParsed.ref, ...rest]);
|
|
60
|
+
} else {
|
|
61
|
+
switch (cmd) {
|
|
62
|
+
case "setup":
|
|
63
|
+
await commandSetup(rest);
|
|
64
|
+
break;
|
|
65
|
+
case "pr":
|
|
66
|
+
await commandPR(rest);
|
|
67
|
+
break;
|
|
68
|
+
case "issue":
|
|
69
|
+
await commandIssue(rest);
|
|
70
|
+
break;
|
|
71
|
+
case "full":
|
|
72
|
+
await commandFull(rest);
|
|
73
|
+
break;
|
|
74
|
+
default:
|
|
75
|
+
console.error(`Unknown command: ${cmd}\nRun 'cmqa --help' for usage.`);
|
|
76
|
+
process.exit(1);
|
|
77
|
+
}
|
|
78
|
+
}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
import { runQA } from "../agent/orchestrator";
|
|
2
|
+
import { fetchRecentIssues, parseRef } from "../utils/github";
|
|
3
|
+
|
|
4
|
+
export async function commandFull(args: string[]): Promise<void> {
|
|
5
|
+
const repoRef = args.find((a) => !a.startsWith("--"));
|
|
6
|
+
const record = !args.includes("--no-record");
|
|
7
|
+
const limitIdx = args.indexOf("--limit");
|
|
8
|
+
const limit = limitIdx >= 0 ? parseInt(args[limitIdx + 1]) : 5;
|
|
9
|
+
const comfyUrlIdx = args.indexOf("--comfy-url");
|
|
10
|
+
const comfyUrl = comfyUrlIdx >= 0 ? args[comfyUrlIdx + 1] : undefined;
|
|
11
|
+
|
|
12
|
+
if (!repoRef) {
|
|
13
|
+
console.error("Usage: cmqa full <owner/repo> [--limit N] [--no-record]");
|
|
14
|
+
process.exit(1);
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
const { owner, repo } = parseRef(repoRef);
|
|
18
|
+
console.log(`\n[full] Fetching top ${limit} open issues from ${owner}/${repo}…`);
|
|
19
|
+
|
|
20
|
+
const issues = await fetchRecentIssues(owner, repo, limit);
|
|
21
|
+
console.log(` Found ${issues.length} issues\n`);
|
|
22
|
+
|
|
23
|
+
for (const issue of issues) {
|
|
24
|
+
console.log(`\n${"─".repeat(60)}`);
|
|
25
|
+
console.log(` Processing issue #${issue.number}: ${issue.title}`);
|
|
26
|
+
try {
|
|
27
|
+
await runQA({
|
|
28
|
+
ref: `${owner}/${repo}#${issue.number}`,
|
|
29
|
+
type: "issue",
|
|
30
|
+
record,
|
|
31
|
+
outputBase: ".comfy-qa",
|
|
32
|
+
comfyUrl,
|
|
33
|
+
});
|
|
34
|
+
} catch (err) {
|
|
35
|
+
console.error(` ✗ Failed: ${err}`);
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
console.log(`\n[full] Done. Reports in .comfy-qa/`);
|
|
40
|
+
}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import { runQA } from "../agent/orchestrator";
|
|
2
|
+
import { parseGitHubUrl } from "../utils/parse-url";
|
|
3
|
+
|
|
4
|
+
export async function commandIssue(args: string[]): Promise<void> {
|
|
5
|
+
let ref = args.find((a) => !a.startsWith("--"));
|
|
6
|
+
const record = !args.includes("--no-record");
|
|
7
|
+
const comfyUrlIdx = args.indexOf("--comfy-url");
|
|
8
|
+
const comfyUrl = comfyUrlIdx >= 0 ? args[comfyUrlIdx + 1] : undefined;
|
|
9
|
+
|
|
10
|
+
// Accept GitHub URL: cmqa issue https://github.com/org/repo/issues/456
|
|
11
|
+
if (ref) {
|
|
12
|
+
const parsed = parseGitHubUrl(ref);
|
|
13
|
+
if (parsed) ref = parsed.ref;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
if (!ref) {
|
|
17
|
+
console.error("Usage: cmqa issue <github-url | owner/repo#number>");
|
|
18
|
+
console.error(" cmqa issue https://github.com/org/repo/issues/456");
|
|
19
|
+
process.exit(1);
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
await runQA({ ref, type: "issue", record, outputBase: ".comfy-qa", comfyUrl });
|
|
23
|
+
}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import { runQA } from "../agent/orchestrator";
|
|
2
|
+
import { parseGitHubUrl } from "../utils/parse-url";
|
|
3
|
+
|
|
4
|
+
export async function commandPR(args: string[]): Promise<void> {
|
|
5
|
+
let ref = args.find((a) => !a.startsWith("--"));
|
|
6
|
+
const record = !args.includes("--no-record");
|
|
7
|
+
const comfyUrlIdx = args.indexOf("--comfy-url");
|
|
8
|
+
const comfyUrl = comfyUrlIdx >= 0 ? args[comfyUrlIdx + 1] : undefined;
|
|
9
|
+
|
|
10
|
+
// Accept GitHub URL: cmqa pr https://github.com/org/repo/pull/123
|
|
11
|
+
if (ref) {
|
|
12
|
+
const parsed = parseGitHubUrl(ref);
|
|
13
|
+
if (parsed) ref = parsed.ref;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
if (!ref) {
|
|
17
|
+
console.error("Usage: cmqa pr <github-url | owner/repo#number>");
|
|
18
|
+
console.error(" cmqa pr https://github.com/org/repo/pull/123");
|
|
19
|
+
process.exit(1);
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
await runQA({ ref, type: "pr", record, outputBase: ".comfy-qa", comfyUrl });
|
|
23
|
+
}
|