@vibecheckai/cli 3.2.2 → 3.2.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/.generated +25 -25
- package/bin/dev/run-v2-torture.js +30 -30
- package/bin/runners/ENHANCEMENT_GUIDE.md +121 -121
- package/bin/runners/lib/__tests__/entitlements-v2.test.js +295 -295
- package/bin/runners/lib/agent-firewall/ai/false-positive-analyzer.js +474 -0
- package/bin/runners/lib/agent-firewall/claims/extractor.js +117 -28
- package/bin/runners/lib/agent-firewall/evidence/env-evidence.js +23 -14
- package/bin/runners/lib/agent-firewall/evidence/route-evidence.js +72 -1
- package/bin/runners/lib/agent-firewall/interceptor/base.js +2 -2
- package/bin/runners/lib/agent-firewall/policy/default-policy.json +6 -0
- package/bin/runners/lib/agent-firewall/policy/engine.js +34 -3
- package/bin/runners/lib/agent-firewall/policy/rules/fake-success.js +29 -4
- package/bin/runners/lib/agent-firewall/policy/rules/ghost-route.js +12 -0
- package/bin/runners/lib/agent-firewall/truthpack/loader.js +21 -0
- package/bin/runners/lib/agent-firewall/utils/ignore-checker.js +118 -0
- package/bin/runners/lib/analyzers.js +606 -325
- package/bin/runners/lib/auth-truth.js +193 -193
- package/bin/runners/lib/backup.js +62 -62
- package/bin/runners/lib/billing.js +107 -107
- package/bin/runners/lib/claims.js +118 -118
- package/bin/runners/lib/cli-ui.js +540 -540
- package/bin/runners/lib/contracts/auth-contract.js +202 -202
- package/bin/runners/lib/contracts/env-contract.js +181 -181
- package/bin/runners/lib/contracts/external-contract.js +206 -206
- package/bin/runners/lib/contracts/guard.js +168 -168
- package/bin/runners/lib/contracts/index.js +89 -89
- package/bin/runners/lib/contracts/plan-validator.js +311 -311
- package/bin/runners/lib/contracts/route-contract.js +199 -199
- package/bin/runners/lib/contracts.js +804 -804
- package/bin/runners/lib/detect.js +89 -89
- package/bin/runners/lib/doctor/autofix.js +254 -254
- package/bin/runners/lib/doctor/index.js +37 -37
- package/bin/runners/lib/doctor/modules/dependencies.js +325 -325
- package/bin/runners/lib/doctor/modules/index.js +46 -46
- package/bin/runners/lib/doctor/modules/network.js +250 -250
- package/bin/runners/lib/doctor/modules/project.js +312 -312
- package/bin/runners/lib/doctor/modules/runtime.js +224 -224
- package/bin/runners/lib/doctor/modules/security.js +348 -348
- package/bin/runners/lib/doctor/modules/system.js +213 -213
- package/bin/runners/lib/doctor/modules/vibecheck.js +394 -394
- package/bin/runners/lib/doctor/reporter.js +262 -262
- package/bin/runners/lib/doctor/service.js +262 -262
- package/bin/runners/lib/doctor/types.js +113 -113
- package/bin/runners/lib/doctor/ui.js +263 -263
- package/bin/runners/lib/doctor-v2.js +608 -608
- package/bin/runners/lib/drift.js +425 -425
- package/bin/runners/lib/enforcement.js +72 -72
- package/bin/runners/lib/engines/accessibility-engine.js +190 -0
- package/bin/runners/lib/engines/api-consistency-engine.js +162 -0
- package/bin/runners/lib/engines/ast-cache.js +99 -0
- package/bin/runners/lib/engines/code-quality-engine.js +255 -0
- package/bin/runners/lib/engines/console-logs-engine.js +115 -0
- package/bin/runners/lib/engines/cross-file-analysis-engine.js +268 -0
- package/bin/runners/lib/engines/dead-code-engine.js +198 -0
- package/bin/runners/lib/engines/deprecated-api-engine.js +226 -0
- package/bin/runners/lib/engines/empty-catch-engine.js +150 -0
- package/bin/runners/lib/engines/file-filter.js +131 -0
- package/bin/runners/lib/engines/hardcoded-secrets-engine.js +251 -0
- package/bin/runners/lib/engines/mock-data-engine.js +272 -0
- package/bin/runners/lib/engines/parallel-processor.js +71 -0
- package/bin/runners/lib/engines/performance-issues-engine.js +265 -0
- package/bin/runners/lib/engines/security-vulnerabilities-engine.js +243 -0
- package/bin/runners/lib/engines/todo-fixme-engine.js +115 -0
- package/bin/runners/lib/engines/type-aware-engine.js +152 -0
- package/bin/runners/lib/engines/unsafe-regex-engine.js +225 -0
- package/bin/runners/lib/engines/vibecheck-engines/README.md +53 -0
- package/bin/runners/lib/engines/vibecheck-engines/index.js +15 -0
- package/bin/runners/lib/engines/vibecheck-engines/lib/ast-cache.js +164 -0
- package/bin/runners/lib/engines/vibecheck-engines/lib/code-quality-engine.js +291 -0
- package/bin/runners/lib/engines/vibecheck-engines/lib/console-logs-engine.js +83 -0
- package/bin/runners/lib/engines/vibecheck-engines/lib/dead-code-engine.js +198 -0
- package/bin/runners/lib/engines/vibecheck-engines/lib/deprecated-api-engine.js +275 -0
- package/bin/runners/lib/engines/vibecheck-engines/lib/empty-catch-engine.js +167 -0
- package/bin/runners/lib/engines/vibecheck-engines/lib/file-filter.js +217 -0
- package/bin/runners/lib/engines/vibecheck-engines/lib/hardcoded-secrets-engine.js +139 -0
- package/bin/runners/lib/engines/vibecheck-engines/lib/mock-data-engine.js +140 -0
- package/bin/runners/lib/engines/vibecheck-engines/lib/parallel-processor.js +164 -0
- package/bin/runners/lib/engines/vibecheck-engines/lib/performance-issues-engine.js +234 -0
- package/bin/runners/lib/engines/vibecheck-engines/lib/type-aware-engine.js +217 -0
- package/bin/runners/lib/engines/vibecheck-engines/lib/unsafe-regex-engine.js +78 -0
- package/bin/runners/lib/engines/vibecheck-engines/package.json +13 -0
- package/bin/runners/lib/enterprise-detect.js +603 -603
- package/bin/runners/lib/enterprise-init.js +942 -942
- package/bin/runners/lib/env-resolver.js +417 -417
- package/bin/runners/lib/env-template.js +66 -66
- package/bin/runners/lib/env.js +189 -189
- package/bin/runners/lib/extractors/client-calls.js +990 -990
- package/bin/runners/lib/extractors/fastify-route-dump.js +573 -573
- package/bin/runners/lib/extractors/fastify-routes.js +426 -426
- package/bin/runners/lib/extractors/index.js +363 -363
- package/bin/runners/lib/extractors/next-routes.js +524 -524
- package/bin/runners/lib/extractors/proof-graph.js +431 -431
- package/bin/runners/lib/extractors/route-matcher.js +451 -451
- package/bin/runners/lib/extractors/truthpack-v2.js +377 -377
- package/bin/runners/lib/extractors/ui-bindings.js +547 -547
- package/bin/runners/lib/findings-schema.js +281 -281
- package/bin/runners/lib/firewall-prompt.js +50 -50
- package/bin/runners/lib/global-flags.js +213 -213
- package/bin/runners/lib/graph/graph-builder.js +265 -265
- package/bin/runners/lib/graph/html-renderer.js +413 -413
- package/bin/runners/lib/graph/index.js +32 -32
- package/bin/runners/lib/graph/runtime-collector.js +215 -215
- package/bin/runners/lib/graph/static-extractor.js +518 -518
- package/bin/runners/lib/html-report.js +650 -650
- package/bin/runners/lib/interactive-menu.js +1496 -1496
- package/bin/runners/lib/llm.js +75 -75
- package/bin/runners/lib/meter.js +61 -61
- package/bin/runners/lib/missions/evidence.js +126 -126
- package/bin/runners/lib/patch.js +40 -40
- package/bin/runners/lib/permissions/auth-model.js +213 -213
- package/bin/runners/lib/permissions/idor-prover.js +205 -205
- package/bin/runners/lib/permissions/index.js +45 -45
- package/bin/runners/lib/permissions/matrix-builder.js +198 -198
- package/bin/runners/lib/pkgjson.js +28 -28
- package/bin/runners/lib/policy.js +295 -295
- package/bin/runners/lib/preflight.js +142 -142
- package/bin/runners/lib/reality/correlation-detectors.js +359 -359
- package/bin/runners/lib/reality/index.js +318 -318
- package/bin/runners/lib/reality/request-hashing.js +416 -416
- package/bin/runners/lib/reality/request-mapper.js +453 -453
- package/bin/runners/lib/reality/safety-rails.js +463 -463
- package/bin/runners/lib/reality/semantic-snapshot.js +408 -408
- package/bin/runners/lib/reality/toast-detector.js +393 -393
- package/bin/runners/lib/reality-findings.js +84 -84
- package/bin/runners/lib/receipts.js +179 -179
- package/bin/runners/lib/redact.js +29 -29
- package/bin/runners/lib/replay/capsule-manager.js +154 -154
- package/bin/runners/lib/replay/index.js +263 -263
- package/bin/runners/lib/replay/player.js +348 -348
- package/bin/runners/lib/replay/recorder.js +331 -331
- package/bin/runners/lib/report-output.js +187 -187
- package/bin/runners/lib/report.js +135 -135
- package/bin/runners/lib/route-detection.js +1140 -1140
- package/bin/runners/lib/sandbox/index.js +59 -59
- package/bin/runners/lib/sandbox/proof-chain.js +399 -399
- package/bin/runners/lib/sandbox/sandbox-runner.js +205 -205
- package/bin/runners/lib/sandbox/worktree.js +174 -174
- package/bin/runners/lib/scan-output.js +525 -190
- package/bin/runners/lib/schema-validator.js +350 -350
- package/bin/runners/lib/schemas/contracts.schema.json +160 -160
- package/bin/runners/lib/schemas/finding.schema.json +100 -100
- package/bin/runners/lib/schemas/mission-pack.schema.json +206 -206
- package/bin/runners/lib/schemas/proof-graph.schema.json +176 -176
- package/bin/runners/lib/schemas/reality-report.schema.json +162 -162
- package/bin/runners/lib/schemas/share-pack.schema.json +180 -180
- package/bin/runners/lib/schemas/ship-report.schema.json +117 -117
- package/bin/runners/lib/schemas/truthpack-v2.schema.json +303 -303
- package/bin/runners/lib/schemas/validator.js +438 -438
- package/bin/runners/lib/score-history.js +282 -282
- package/bin/runners/lib/share-pack.js +239 -239
- package/bin/runners/lib/snippets.js +67 -67
- package/bin/runners/lib/status-output.js +253 -253
- package/bin/runners/lib/terminal-ui.js +351 -271
- package/bin/runners/lib/upsell.js +510 -510
- package/bin/runners/lib/usage.js +153 -153
- package/bin/runners/lib/validate-patch.js +156 -156
- package/bin/runners/lib/verdict-engine.js +628 -628
- package/bin/runners/reality/engine.js +917 -917
- package/bin/runners/reality/flows.js +122 -122
- package/bin/runners/reality/report.js +378 -378
- package/bin/runners/reality/session.js +193 -193
- package/bin/runners/runGuard.js +168 -168
- package/bin/runners/runProof.zip +0 -0
- package/bin/runners/runProve.js +8 -0
- package/bin/runners/runReality.js +14 -0
- package/bin/runners/runScan.js +17 -1
- package/bin/runners/runTruth.js +15 -3
- package/mcp-server/tier-auth.js +4 -4
- package/mcp-server/tools/index.js +72 -72
- package/package.json +1 -1
|
@@ -1,917 +1,917 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Reality Mode Execution Engine
|
|
3
|
-
* Step execution, danger classification, assertions, surface discovery
|
|
4
|
-
*/
|
|
5
|
-
|
|
6
|
-
const fs = require("fs");
|
|
7
|
-
const path = require("path");
|
|
8
|
-
const crypto = require("crypto");
|
|
9
|
-
|
|
10
|
-
// ============================================================================
|
|
11
|
-
// Utilities
|
|
12
|
-
// ============================================================================
|
|
13
|
-
|
|
14
|
-
function uuid() {
|
|
15
|
-
try {
|
|
16
|
-
return crypto.randomUUID();
|
|
17
|
-
} catch {
|
|
18
|
-
return crypto.randomBytes(16).toString("hex");
|
|
19
|
-
}
|
|
20
|
-
}
|
|
21
|
-
|
|
22
|
-
function randomEmail() {
|
|
23
|
-
return `reality+${Date.now()}_${Math.floor(Math.random() * 1e6)}@example.com`;
|
|
24
|
-
}
|
|
25
|
-
|
|
26
|
-
function template(str, vars) {
|
|
27
|
-
if (typeof str !== "string") return str;
|
|
28
|
-
return str.replace(/\{\{([^}]+)\}\}/g, (_, key) => {
|
|
29
|
-
const k = String(key || "").trim();
|
|
30
|
-
if (k === "$timestamp") return String(Date.now());
|
|
31
|
-
if (k === "$uuid") return uuid();
|
|
32
|
-
if (k === "$randomEmail") return randomEmail();
|
|
33
|
-
if (k === "$isoDate") return new Date().toISOString();
|
|
34
|
-
if (k.startsWith("env.")) return process.env[k.slice(4)] || "";
|
|
35
|
-
if (k.startsWith("stored.")) return vars?.[k] ?? "";
|
|
36
|
-
return vars?.[k] != null ? String(vars[k]) : "";
|
|
37
|
-
});
|
|
38
|
-
}
|
|
39
|
-
|
|
40
|
-
// ============================================================================
|
|
41
|
-
// Danger Classification System
|
|
42
|
-
// ============================================================================
|
|
43
|
-
|
|
44
|
-
const DANGER_TEXT_PATTERNS = [
|
|
45
|
-
/\bdelete\b/i,
|
|
46
|
-
/\bremove\b/i,
|
|
47
|
-
/\bdestroy\b/i,
|
|
48
|
-
/\bdeactivate\b/i,
|
|
49
|
-
/\bterminate\b/i,
|
|
50
|
-
/\bclose\s*account\b/i,
|
|
51
|
-
/\bcancel\s*subscription\b/i,
|
|
52
|
-
/\breset\s*all\b/i,
|
|
53
|
-
/\bwipe\b/i,
|
|
54
|
-
/\berase\b/i,
|
|
55
|
-
/\birreversible\b/i,
|
|
56
|
-
/\bpermanently\b/i,
|
|
57
|
-
/\bcannot\s*be\s*undone\b/i,
|
|
58
|
-
/\brevoke\b/i,
|
|
59
|
-
/\bdisconnect\b/i,
|
|
60
|
-
/\bunlink\b/i,
|
|
61
|
-
];
|
|
62
|
-
|
|
63
|
-
const DANGER_CLASS_PATTERNS = [
|
|
64
|
-
/\bbtn-danger\b/,
|
|
65
|
-
/\bdestructive\b/,
|
|
66
|
-
/\bdelete-/,
|
|
67
|
-
/\bdanger-/,
|
|
68
|
-
/\bremove-/,
|
|
69
|
-
];
|
|
70
|
-
|
|
71
|
-
function classifyDanger({ text, className, ariaLabel, role, url, method }) {
|
|
72
|
-
const signals = [];
|
|
73
|
-
let score = 0;
|
|
74
|
-
|
|
75
|
-
const t = `${text || ""}`.toLowerCase();
|
|
76
|
-
const cls = `${className || ""}`.toLowerCase();
|
|
77
|
-
const aria = `${ariaLabel || ""}`.toLowerCase();
|
|
78
|
-
const r = `${role || ""}`.toLowerCase();
|
|
79
|
-
const u = `${url || ""}`.toLowerCase();
|
|
80
|
-
const m = `${method || ""}`.toUpperCase();
|
|
81
|
-
|
|
82
|
-
// Text patterns
|
|
83
|
-
for (const pattern of DANGER_TEXT_PATTERNS) {
|
|
84
|
-
if (pattern.test(t) || pattern.test(aria)) {
|
|
85
|
-
signals.push({ type: "text", value: pattern.source, weight: 0.3 });
|
|
86
|
-
score += 0.3;
|
|
87
|
-
}
|
|
88
|
-
}
|
|
89
|
-
|
|
90
|
-
// Class patterns
|
|
91
|
-
for (const pattern of DANGER_CLASS_PATTERNS) {
|
|
92
|
-
if (pattern.test(cls)) {
|
|
93
|
-
signals.push({ type: "class", value: pattern.source, weight: 0.25 });
|
|
94
|
-
score += 0.25;
|
|
95
|
-
}
|
|
96
|
-
}
|
|
97
|
-
|
|
98
|
-
// ARIA patterns
|
|
99
|
-
if (r === "alertdialog") {
|
|
100
|
-
signals.push({ type: "aria", value: "role=alertdialog", weight: 0.2 });
|
|
101
|
-
score += 0.2;
|
|
102
|
-
}
|
|
103
|
-
|
|
104
|
-
// Network patterns (DELETE method or destructive endpoints)
|
|
105
|
-
if (m === "DELETE") {
|
|
106
|
-
signals.push({ type: "network", value: "DELETE method", weight: 0.4 });
|
|
107
|
-
score += 0.4;
|
|
108
|
-
}
|
|
109
|
-
if (/\/delete\b|\/destroy\b|\/deactivate\b|\/remove\b/.test(u)) {
|
|
110
|
-
signals.push({ type: "network", value: "destructive endpoint", weight: 0.35 });
|
|
111
|
-
score += 0.35;
|
|
112
|
-
}
|
|
113
|
-
|
|
114
|
-
const dangerous = score >= 0.25;
|
|
115
|
-
const reason = signals.length > 0 ? signals[0].type : "none";
|
|
116
|
-
|
|
117
|
-
return {
|
|
118
|
-
dangerous,
|
|
119
|
-
score: Math.min(1, score),
|
|
120
|
-
signals,
|
|
121
|
-
reason,
|
|
122
|
-
};
|
|
123
|
-
}
|
|
124
|
-
|
|
125
|
-
// ============================================================================
|
|
126
|
-
// Screenshots & DOM Snapshots
|
|
127
|
-
// ============================================================================
|
|
128
|
-
|
|
129
|
-
async function screenshot(page, outPath) {
|
|
130
|
-
try {
|
|
131
|
-
await page.screenshot({ path: outPath, fullPage: true });
|
|
132
|
-
} catch {
|
|
133
|
-
// Non-fatal
|
|
134
|
-
}
|
|
135
|
-
}
|
|
136
|
-
|
|
137
|
-
async function domSnapshot(page) {
|
|
138
|
-
try {
|
|
139
|
-
return await page.evaluate(() => {
|
|
140
|
-
const el = document.activeElement;
|
|
141
|
-
return {
|
|
142
|
-
url: location.href,
|
|
143
|
-
title: document.title,
|
|
144
|
-
active: el ? { tag: el.tagName, id: el.id, name: el.getAttribute("name") } : null,
|
|
145
|
-
bodyTextSample: document.body?.innerText?.slice(0, 500) || "",
|
|
146
|
-
};
|
|
147
|
-
});
|
|
148
|
-
} catch {
|
|
149
|
-
return null;
|
|
150
|
-
}
|
|
151
|
-
}
|
|
152
|
-
|
|
153
|
-
// ============================================================================
|
|
154
|
-
// Navigation Helpers
|
|
155
|
-
// ============================================================================
|
|
156
|
-
|
|
157
|
-
async function safeGoto(page, url, timeout) {
|
|
158
|
-
const started = Date.now();
|
|
159
|
-
try {
|
|
160
|
-
const res = await page.goto(url, { waitUntil: "domcontentloaded", timeout });
|
|
161
|
-
await page.waitForLoadState("networkidle", { timeout: Math.min(timeout, 5000) }).catch(() => {});
|
|
162
|
-
const status = res ? res.status() : 0;
|
|
163
|
-
return { ok: status >= 200 && status < 400, status, ms: Date.now() - started };
|
|
164
|
-
} catch (e) {
|
|
165
|
-
return { ok: false, status: 0, ms: Date.now() - started, error: String(e?.message || e).slice(0, 180) };
|
|
166
|
-
}
|
|
167
|
-
}
|
|
168
|
-
|
|
169
|
-
async function findLocator(page, target) {
|
|
170
|
-
return page.locator(target).first();
|
|
171
|
-
}
|
|
172
|
-
|
|
173
|
-
// ============================================================================
|
|
174
|
-
// Surface Discovery
|
|
175
|
-
// ============================================================================
|
|
176
|
-
|
|
177
|
-
async function discoverSurface({ page, baseUrl, timeout, maxPages }) {
|
|
178
|
-
const discovered = {
|
|
179
|
-
routes: [],
|
|
180
|
-
elements: [],
|
|
181
|
-
forms: [],
|
|
182
|
-
};
|
|
183
|
-
|
|
184
|
-
// Visit base
|
|
185
|
-
const home = await safeGoto(page, baseUrl, timeout);
|
|
186
|
-
if (home.status) {
|
|
187
|
-
discovered.routes.push({ path: "/", status: home.ok ? "success" : "error", httpStatus: home.status });
|
|
188
|
-
}
|
|
189
|
-
|
|
190
|
-
// Extract routes
|
|
191
|
-
const routes = await page.evaluate(() => {
|
|
192
|
-
const out = new Set();
|
|
193
|
-
document.querySelectorAll("a[href]").forEach(a => {
|
|
194
|
-
const href = a.getAttribute("href") || "";
|
|
195
|
-
if (!href || href.startsWith("http") || href.startsWith("//") || href.includes("#")) return;
|
|
196
|
-
if (href.startsWith("/")) out.add(href);
|
|
197
|
-
});
|
|
198
|
-
// Common SPA routes
|
|
199
|
-
["/dashboard", "/app", "/account", "/settings", "/profile", "/projects", "/analytics", "/billing"].forEach(r => out.add(r));
|
|
200
|
-
return Array.from(out);
|
|
201
|
-
}).catch(() => []);
|
|
202
|
-
|
|
203
|
-
const queue = routes.slice(0, maxPages * 2);
|
|
204
|
-
const seen = new Set(["/", baseUrl]);
|
|
205
|
-
|
|
206
|
-
for (const route of queue.slice(0, maxPages)) {
|
|
207
|
-
if (seen.has(route)) continue;
|
|
208
|
-
seen.add(route);
|
|
209
|
-
|
|
210
|
-
const res = await safeGoto(page, baseUrl + route, timeout);
|
|
211
|
-
discovered.routes.push({
|
|
212
|
-
path: route,
|
|
213
|
-
status: res.ok ? "success" : "error",
|
|
214
|
-
httpStatus: res.status || 0,
|
|
215
|
-
error: res.error,
|
|
216
|
-
});
|
|
217
|
-
|
|
218
|
-
// Discover elements on this page
|
|
219
|
-
const pageEls = await page.evaluate(() => {
|
|
220
|
-
const pickText = (el) => (el.textContent || el.getAttribute("aria-label") || "").trim().slice(0, 80);
|
|
221
|
-
const els = Array.from(document.querySelectorAll('button, [role="button"], input[type="submit"], a[href], [data-testid]'));
|
|
222
|
-
return els.slice(0, 100).map((el, idx) => ({
|
|
223
|
-
tag: el.tagName.toLowerCase(),
|
|
224
|
-
text: pickText(el),
|
|
225
|
-
id: el.id || "",
|
|
226
|
-
testid: el.getAttribute("data-testid") || "",
|
|
227
|
-
className: el.className || "",
|
|
228
|
-
ariaLabel: el.getAttribute("aria-label") || "",
|
|
229
|
-
href: el.getAttribute("href") || "",
|
|
230
|
-
index: idx,
|
|
231
|
-
}));
|
|
232
|
-
}).catch(() => []);
|
|
233
|
-
|
|
234
|
-
for (const e of pageEls) {
|
|
235
|
-
if (!e.text && !e.testid && !e.id) continue;
|
|
236
|
-
discovered.elements.push({ ...e, page: route });
|
|
237
|
-
}
|
|
238
|
-
|
|
239
|
-
// Discover forms
|
|
240
|
-
const forms = await page.evaluate(() => {
|
|
241
|
-
const forms = Array.from(document.querySelectorAll("form"));
|
|
242
|
-
return forms.slice(0, 20).map((form, idx) => {
|
|
243
|
-
const fields = Array.from(form.querySelectorAll("input, textarea, select")).slice(0, 30).map((f) => ({
|
|
244
|
-
name: f.getAttribute("name") || f.id || "",
|
|
245
|
-
type: f.getAttribute("type") || f.tagName.toLowerCase(),
|
|
246
|
-
required: !!f.required,
|
|
247
|
-
}));
|
|
248
|
-
return {
|
|
249
|
-
selector: form.id ? `#${form.id}` : form.getAttribute("data-testid") ? `[data-testid="${form.getAttribute("data-testid")}"]` : `form:nth-of-type(${idx + 1})`,
|
|
250
|
-
fields,
|
|
251
|
-
};
|
|
252
|
-
});
|
|
253
|
-
}).catch(() => []);
|
|
254
|
-
|
|
255
|
-
for (const f of forms) discovered.forms.push({ ...f, page: route });
|
|
256
|
-
}
|
|
257
|
-
|
|
258
|
-
// Dedupe
|
|
259
|
-
discovered.routes = dedupe(discovered.routes, r => r.path);
|
|
260
|
-
discovered.elements = dedupe(discovered.elements, e => `${e.page}|${e.tag}|${e.testid}|${e.id}|${e.text}`);
|
|
261
|
-
discovered.forms = dedupe(discovered.forms, f => `${f.page}|${f.selector}`);
|
|
262
|
-
|
|
263
|
-
return discovered;
|
|
264
|
-
}
|
|
265
|
-
|
|
266
|
-
function dedupe(arr, keyFn) {
|
|
267
|
-
const seen = new Set();
|
|
268
|
-
const out = [];
|
|
269
|
-
for (const x of arr) {
|
|
270
|
-
const k = keyFn(x);
|
|
271
|
-
if (seen.has(k)) continue;
|
|
272
|
-
seen.add(k);
|
|
273
|
-
out.push(x);
|
|
274
|
-
}
|
|
275
|
-
return out;
|
|
276
|
-
}
|
|
277
|
-
|
|
278
|
-
// ============================================================================
|
|
279
|
-
// Flow Plan Execution
|
|
280
|
-
// ============================================================================
|
|
281
|
-
|
|
282
|
-
async function runFlowPlan({ plan, page, context, telemetry }) {
|
|
283
|
-
const results = {
|
|
284
|
-
meta: {
|
|
285
|
-
baseUrl: plan.baseUrl,
|
|
286
|
-
startedAt: new Date().toISOString(),
|
|
287
|
-
danger: plan.danger,
|
|
288
|
-
maxPages: plan.maxPages,
|
|
289
|
-
timeout: plan.timeout,
|
|
290
|
-
},
|
|
291
|
-
discovery: null,
|
|
292
|
-
coverage: {
|
|
293
|
-
routesDiscovered: 0,
|
|
294
|
-
routesWorking: 0,
|
|
295
|
-
elementsDiscovered: 0,
|
|
296
|
-
elementsWorking: 0,
|
|
297
|
-
formsDiscovered: 0,
|
|
298
|
-
formsWorking: 0,
|
|
299
|
-
},
|
|
300
|
-
flows: [],
|
|
301
|
-
routes: [],
|
|
302
|
-
elements: [],
|
|
303
|
-
forms: [],
|
|
304
|
-
errors: [],
|
|
305
|
-
network: [],
|
|
306
|
-
console: [],
|
|
307
|
-
timeline: [],
|
|
308
|
-
score: 0,
|
|
309
|
-
duration: 0,
|
|
310
|
-
};
|
|
311
|
-
|
|
312
|
-
// Discovery pass
|
|
313
|
-
const discovery = await discoverSurface({
|
|
314
|
-
page,
|
|
315
|
-
baseUrl: plan.baseUrl,
|
|
316
|
-
timeout: plan.timeout,
|
|
317
|
-
maxPages: plan.maxPages,
|
|
318
|
-
});
|
|
319
|
-
|
|
320
|
-
results.discovery = {
|
|
321
|
-
routes: discovery.routes.length,
|
|
322
|
-
elements: discovery.elements.length,
|
|
323
|
-
forms: discovery.forms.length,
|
|
324
|
-
};
|
|
325
|
-
|
|
326
|
-
results.routes = discovery.routes;
|
|
327
|
-
results.elements = discovery.elements;
|
|
328
|
-
results.forms = discovery.forms;
|
|
329
|
-
|
|
330
|
-
results.coverage.routesDiscovered = discovery.routes.length;
|
|
331
|
-
results.coverage.routesWorking = discovery.routes.filter(r => r.status === "success").length;
|
|
332
|
-
results.coverage.elementsDiscovered = discovery.elements.length;
|
|
333
|
-
results.coverage.formsDiscovered = discovery.forms.length;
|
|
334
|
-
|
|
335
|
-
// Collect telemetry
|
|
336
|
-
results.console = telemetry.console.slice(-500);
|
|
337
|
-
results.errors = [
|
|
338
|
-
...telemetry.pageErrors.map(e => ({ type: "uncaught", message: e.message, url: e.url, ts: e.ts })),
|
|
339
|
-
...telemetry.console.filter(m => m.type === "error").map(m => ({ type: "console", message: m.text, url: m.url, ts: m.ts })),
|
|
340
|
-
];
|
|
341
|
-
results.network = telemetry.responses.slice(-500);
|
|
342
|
-
|
|
343
|
-
// Run Flow Packs
|
|
344
|
-
const flowVarsBase = makeBaseVars(plan);
|
|
345
|
-
const flowsToRun = Array.isArray(plan.flows) ? plan.flows : [];
|
|
346
|
-
|
|
347
|
-
for (const flow of flowsToRun) {
|
|
348
|
-
const flowResult = await runSingleFlow({
|
|
349
|
-
flow,
|
|
350
|
-
plan,
|
|
351
|
-
page,
|
|
352
|
-
context,
|
|
353
|
-
telemetry,
|
|
354
|
-
varsBase: flowVarsBase,
|
|
355
|
-
results,
|
|
356
|
-
});
|
|
357
|
-
results.flows.push(flowResult);
|
|
358
|
-
}
|
|
359
|
-
|
|
360
|
-
// Count working elements from successful flow steps
|
|
361
|
-
let elementWorking = 0;
|
|
362
|
-
for (const f of results.flows) {
|
|
363
|
-
for (const s of (f.steps || [])) {
|
|
364
|
-
if ((s.action === "click" || s.action === "fill") && s.status === "success") elementWorking++;
|
|
365
|
-
}
|
|
366
|
-
}
|
|
367
|
-
results.coverage.elementsWorking = Math.min(elementWorking, results.coverage.elementsDiscovered);
|
|
368
|
-
|
|
369
|
-
// Calculate score
|
|
370
|
-
results.score = calculateScore(results);
|
|
371
|
-
|
|
372
|
-
return results;
|
|
373
|
-
}
|
|
374
|
-
|
|
375
|
-
function makeBaseVars(plan) {
|
|
376
|
-
const vars = {
|
|
377
|
-
baseUrl: plan.baseUrl,
|
|
378
|
-
timestamp: String(Date.now()),
|
|
379
|
-
$timestamp: String(Date.now()),
|
|
380
|
-
$uuid: uuid(),
|
|
381
|
-
$randomEmail: randomEmail(),
|
|
382
|
-
$isoDate: new Date().toISOString(),
|
|
383
|
-
};
|
|
384
|
-
|
|
385
|
-
if (plan.auth && typeof plan.auth === "string" && plan.auth.includes(":")) {
|
|
386
|
-
const [email, ...rest] = plan.auth.split(":");
|
|
387
|
-
vars.email = email;
|
|
388
|
-
vars.password = rest.join(":");
|
|
389
|
-
}
|
|
390
|
-
|
|
391
|
-
return vars;
|
|
392
|
-
}
|
|
393
|
-
|
|
394
|
-
// ============================================================================
|
|
395
|
-
// Single Flow Execution
|
|
396
|
-
// ============================================================================
|
|
397
|
-
|
|
398
|
-
async function runSingleFlow({ flow, plan, page, telemetry, varsBase, results }) {
|
|
399
|
-
const flowStart = Date.now();
|
|
400
|
-
const flowVars = { ...varsBase, ...(flow.vars || {}) };
|
|
401
|
-
|
|
402
|
-
// Template all vars
|
|
403
|
-
for (const k of Object.keys(flowVars)) {
|
|
404
|
-
flowVars[k] = template(flowVars[k], flowVars);
|
|
405
|
-
}
|
|
406
|
-
|
|
407
|
-
const flowOut = {
|
|
408
|
-
id: flow.id,
|
|
409
|
-
name: flow.name,
|
|
410
|
-
source: flow.__source,
|
|
411
|
-
required: !!flow.required,
|
|
412
|
-
status: "success",
|
|
413
|
-
startedAt: new Date(flowStart).toISOString(),
|
|
414
|
-
durationMs: 0,
|
|
415
|
-
steps: [],
|
|
416
|
-
assertions: [],
|
|
417
|
-
errors: [],
|
|
418
|
-
};
|
|
419
|
-
|
|
420
|
-
// Run steps
|
|
421
|
-
let stepIndex = 0;
|
|
422
|
-
for (const step of flow.steps || []) {
|
|
423
|
-
stepIndex++;
|
|
424
|
-
const stepId = `${flow.id}#${stepIndex}`;
|
|
425
|
-
const stepName = step.name || `${step.action || "step"}-${stepIndex}`;
|
|
426
|
-
const stepStart = Date.now();
|
|
427
|
-
|
|
428
|
-
const stepRec = {
|
|
429
|
-
id: stepId,
|
|
430
|
-
name: stepName,
|
|
431
|
-
action: step.action,
|
|
432
|
-
target: step.target || null,
|
|
433
|
-
status: "success",
|
|
434
|
-
error: null,
|
|
435
|
-
danger: null,
|
|
436
|
-
artifacts: {},
|
|
437
|
-
startedAt: new Date(stepStart).toISOString(),
|
|
438
|
-
durationMs: 0,
|
|
439
|
-
};
|
|
440
|
-
|
|
441
|
-
// Trace marker
|
|
442
|
-
try {
|
|
443
|
-
await page.evaluate((label) => console.log(`[reality:trace] ${label}`), `STEP ${stepId}`);
|
|
444
|
-
} catch {}
|
|
445
|
-
|
|
446
|
-
// Before screenshot
|
|
447
|
-
const beforeShot = path.join(plan.artifacts.screenshotsDir, `${flow.id}__${stepIndex}__before.png`);
|
|
448
|
-
await screenshot(page, beforeShot);
|
|
449
|
-
stepRec.artifacts.beforeScreenshot = path.relative(plan.outputDir, beforeShot);
|
|
450
|
-
stepRec.artifacts.beforeDom = await domSnapshot(page);
|
|
451
|
-
|
|
452
|
-
// Execute action
|
|
453
|
-
try {
|
|
454
|
-
await executeStep({ step, stepRec, plan, page, flowVars, telemetry });
|
|
455
|
-
} catch (e) {
|
|
456
|
-
stepRec.status = "error";
|
|
457
|
-
stepRec.error = String(e?.message || e).slice(0, 220);
|
|
458
|
-
flowOut.errors.push({ step: stepIndex, message: stepRec.error });
|
|
459
|
-
}
|
|
460
|
-
|
|
461
|
-
// After screenshot
|
|
462
|
-
const afterShot = path.join(plan.artifacts.screenshotsDir, `${flow.id}__${stepIndex}__after.png`);
|
|
463
|
-
await screenshot(page, afterShot);
|
|
464
|
-
stepRec.artifacts.afterScreenshot = path.relative(plan.outputDir, afterShot);
|
|
465
|
-
stepRec.artifacts.afterDom = await domSnapshot(page);
|
|
466
|
-
|
|
467
|
-
stepRec.durationMs = Date.now() - stepStart;
|
|
468
|
-
|
|
469
|
-
// Add to timeline
|
|
470
|
-
results.timeline.push({
|
|
471
|
-
flowId: flow.id,
|
|
472
|
-
step: stepIndex,
|
|
473
|
-
name: stepRec.name,
|
|
474
|
-
action: stepRec.action,
|
|
475
|
-
status: stepRec.status,
|
|
476
|
-
at: Date.now(),
|
|
477
|
-
url: page.url(),
|
|
478
|
-
before: stepRec.artifacts.beforeScreenshot,
|
|
479
|
-
after: stepRec.artifacts.afterScreenshot,
|
|
480
|
-
});
|
|
481
|
-
|
|
482
|
-
flowOut.steps.push(stepRec);
|
|
483
|
-
}
|
|
484
|
-
|
|
485
|
-
// Run assertions
|
|
486
|
-
for (const a of flow.assertions || []) {
|
|
487
|
-
const ar = await runAssertion({ assertion: a, plan, page, telemetry, vars: flowVars });
|
|
488
|
-
flowOut.assertions.push(ar);
|
|
489
|
-
if (ar.status === "fail" && a.critical) {
|
|
490
|
-
flowOut.status = "error";
|
|
491
|
-
}
|
|
492
|
-
}
|
|
493
|
-
|
|
494
|
-
// Cleanup (best-effort)
|
|
495
|
-
for (const step of flow.cleanup || []) {
|
|
496
|
-
try {
|
|
497
|
-
await executeStep({ step, stepRec: {}, plan, page, flowVars, telemetry });
|
|
498
|
-
} catch {}
|
|
499
|
-
}
|
|
500
|
-
|
|
501
|
-
flowOut.durationMs = Date.now() - flowStart;
|
|
502
|
-
if (flowOut.errors.length > 0) flowOut.status = "error";
|
|
503
|
-
|
|
504
|
-
return flowOut;
|
|
505
|
-
}
|
|
506
|
-
|
|
507
|
-
// ============================================================================
|
|
508
|
-
// Step Execution
|
|
509
|
-
// ============================================================================
|
|
510
|
-
|
|
511
|
-
async function executeStep({ step, stepRec, plan, page, flowVars, telemetry }) {
|
|
512
|
-
const action = String(step.action || "").toLowerCase();
|
|
513
|
-
|
|
514
|
-
if (action === "navigate") {
|
|
515
|
-
const target = template(step.target || "/", flowVars);
|
|
516
|
-
const url = target.startsWith("http") ? target : (plan.baseUrl + (target.startsWith("/") ? target : `/${target}`));
|
|
517
|
-
const res = await safeGoto(page, url, plan.timeout);
|
|
518
|
-
if (!res.ok) throw new Error(res.error || `Navigation failed: ${url}`);
|
|
519
|
-
}
|
|
520
|
-
|
|
521
|
-
else if (action === "wait") {
|
|
522
|
-
if (step.for === "selector" && step.target) {
|
|
523
|
-
const target = template(step.target, flowVars);
|
|
524
|
-
const state = step.state || "visible";
|
|
525
|
-
const loc = await findLocator(page, target);
|
|
526
|
-
await loc.waitFor({ state, timeout: Number(step.timeout || plan.timeout) });
|
|
527
|
-
} else if (step.for === "navigation" && step.match) {
|
|
528
|
-
const match = new RegExp(template(step.match, flowVars));
|
|
529
|
-
const timeout = Number(step.timeout || plan.timeout);
|
|
530
|
-
const end = Date.now() + timeout;
|
|
531
|
-
while (Date.now() < end) {
|
|
532
|
-
if (match.test(page.url())) break;
|
|
533
|
-
await page.waitForTimeout(200);
|
|
534
|
-
}
|
|
535
|
-
if (!match.test(page.url())) throw new Error(`URL did not match ${step.match}`);
|
|
536
|
-
} else if (step.for === "network" && step.match) {
|
|
537
|
-
const match = template(step.match, flowVars);
|
|
538
|
-
const timeout = Number(step.timeout || plan.timeout);
|
|
539
|
-
const since = Date.now();
|
|
540
|
-
const end = since + timeout;
|
|
541
|
-
let found = false;
|
|
542
|
-
while (Date.now() < end) {
|
|
543
|
-
const recent = telemetry.responses.filter(r => r.ts >= since);
|
|
544
|
-
if (recent.some(r => r.url.includes(match))) { found = true; break; }
|
|
545
|
-
await page.waitForTimeout(200);
|
|
546
|
-
}
|
|
547
|
-
if (!found) throw new Error(`No network call matching ${match}`);
|
|
548
|
-
} else {
|
|
549
|
-
const ms = Number(step.timeout || step.ms || 1000);
|
|
550
|
-
await page.waitForTimeout(ms);
|
|
551
|
-
}
|
|
552
|
-
}
|
|
553
|
-
|
|
554
|
-
else if (action === "fill") {
|
|
555
|
-
const target = template(step.target, flowVars);
|
|
556
|
-
const value = template(step.value, flowVars);
|
|
557
|
-
const loc = await findLocator(page, target);
|
|
558
|
-
await loc.waitFor({ state: "visible", timeout: plan.timeout }).catch(() => {});
|
|
559
|
-
if (step.clear !== false) await loc.clear().catch(() => {});
|
|
560
|
-
await loc.fill(String(value ?? ""), { timeout: plan.timeout });
|
|
561
|
-
}
|
|
562
|
-
|
|
563
|
-
else if (action === "click") {
|
|
564
|
-
const target = template(step.target, flowVars);
|
|
565
|
-
const loc = await findLocator(page, target);
|
|
566
|
-
|
|
567
|
-
// Danger detection
|
|
568
|
-
let meta = {};
|
|
569
|
-
try {
|
|
570
|
-
meta = await loc.evaluate((el) => ({
|
|
571
|
-
text: (el.textContent || "").trim().slice(0, 120),
|
|
572
|
-
className: el.className || "",
|
|
573
|
-
ariaLabel: el.getAttribute("aria-label") || "",
|
|
574
|
-
role: el.getAttribute("role") || "",
|
|
575
|
-
}));
|
|
576
|
-
} catch {}
|
|
577
|
-
|
|
578
|
-
const danger = classifyDanger({ ...meta, url: page.url() });
|
|
579
|
-
stepRec.danger = danger;
|
|
580
|
-
|
|
581
|
-
const policy = String(step.dangerPolicy || "skip").toLowerCase();
|
|
582
|
-
if (danger.dangerous && !plan.danger) {
|
|
583
|
-
if (policy === "block") {
|
|
584
|
-
stepRec.status = "blocked";
|
|
585
|
-
stepRec.error = `Blocked destructive action (${danger.reason}). Use --danger to allow.`;
|
|
586
|
-
return;
|
|
587
|
-
}
|
|
588
|
-
stepRec.status = "skipped";
|
|
589
|
-
stepRec.error = `Skipped destructive action (${danger.reason}). Use --danger to allow.`;
|
|
590
|
-
return;
|
|
591
|
-
}
|
|
592
|
-
|
|
593
|
-
await loc.waitFor({ state: "visible", timeout: plan.timeout }).catch(() => {});
|
|
594
|
-
try {
|
|
595
|
-
await loc.click({ timeout: plan.timeout });
|
|
596
|
-
} catch {
|
|
597
|
-
await loc.click({ timeout: plan.timeout, force: true });
|
|
598
|
-
}
|
|
599
|
-
await page.waitForTimeout(300);
|
|
600
|
-
}
|
|
601
|
-
|
|
602
|
-
else if (action === "check") {
|
|
603
|
-
const target = template(step.target, flowVars);
|
|
604
|
-
const loc = await findLocator(page, target);
|
|
605
|
-
await loc.check({ timeout: plan.timeout });
|
|
606
|
-
}
|
|
607
|
-
|
|
608
|
-
else if (action === "uncheck") {
|
|
609
|
-
const target = template(step.target, flowVars);
|
|
610
|
-
const loc = await findLocator(page, target);
|
|
611
|
-
await loc.uncheck({ timeout: plan.timeout });
|
|
612
|
-
}
|
|
613
|
-
|
|
614
|
-
else if (action === "select") {
|
|
615
|
-
const target = template(step.target, flowVars);
|
|
616
|
-
const loc = await findLocator(page, target);
|
|
617
|
-
if (step.value) {
|
|
618
|
-
await loc.selectOption({ value: template(step.value, flowVars) });
|
|
619
|
-
} else if (step.label) {
|
|
620
|
-
await loc.selectOption({ label: template(step.label, flowVars) });
|
|
621
|
-
} else if (step.index != null) {
|
|
622
|
-
await loc.selectOption({ index: Number(step.index) });
|
|
623
|
-
}
|
|
624
|
-
}
|
|
625
|
-
|
|
626
|
-
else if (action === "hover") {
|
|
627
|
-
const target = template(step.target, flowVars);
|
|
628
|
-
const loc = await findLocator(page, target);
|
|
629
|
-
await loc.hover({ timeout: plan.timeout });
|
|
630
|
-
}
|
|
631
|
-
|
|
632
|
-
else if (action === "press") {
|
|
633
|
-
const key = template(step.key, flowVars);
|
|
634
|
-
await page.keyboard.press(key);
|
|
635
|
-
}
|
|
636
|
-
|
|
637
|
-
else if (action === "screenshot") {
|
|
638
|
-
const name = template(step.name || `manual-${Date.now()}`, flowVars);
|
|
639
|
-
const outPath = path.join(plan.artifacts.screenshotsDir, `${name}.png`);
|
|
640
|
-
await screenshot(page, outPath);
|
|
641
|
-
stepRec.artifacts.manualScreenshot = path.relative(plan.outputDir, outPath);
|
|
642
|
-
}
|
|
643
|
-
|
|
644
|
-
else if (action === "scroll") {
|
|
645
|
-
if (step.to === "bottom") {
|
|
646
|
-
await page.evaluate(() => window.scrollTo(0, document.body.scrollHeight));
|
|
647
|
-
} else if (step.to === "top") {
|
|
648
|
-
await page.evaluate(() => window.scrollTo(0, 0));
|
|
649
|
-
} else if (step.target) {
|
|
650
|
-
const target = template(step.target, flowVars);
|
|
651
|
-
const loc = await findLocator(page, target);
|
|
652
|
-
await loc.scrollIntoViewIfNeeded();
|
|
653
|
-
}
|
|
654
|
-
}
|
|
655
|
-
|
|
656
|
-
else if (action === "execute") {
|
|
657
|
-
if (step.script) {
|
|
658
|
-
await page.evaluate(step.script);
|
|
659
|
-
} else if (step.scriptFile) {
|
|
660
|
-
const scriptPath = path.resolve(step.scriptFile);
|
|
661
|
-
const script = fs.readFileSync(scriptPath, "utf8");
|
|
662
|
-
await page.evaluate(script);
|
|
663
|
-
}
|
|
664
|
-
}
|
|
665
|
-
|
|
666
|
-
else {
|
|
667
|
-
stepRec.status = "unknown";
|
|
668
|
-
stepRec.error = `Unknown action: ${step.action}`;
|
|
669
|
-
}
|
|
670
|
-
}
|
|
671
|
-
|
|
672
|
-
// ============================================================================
|
|
673
|
-
// Assertions
|
|
674
|
-
// ============================================================================
|
|
675
|
-
|
|
676
|
-
async function runAssertion({ assertion, plan, page, telemetry, vars }) {
|
|
677
|
-
const type = String(assertion.type || "").toLowerCase().replace(/-/g, "");
|
|
678
|
-
const critical = !!assertion.critical;
|
|
679
|
-
|
|
680
|
-
const out = {
|
|
681
|
-
type: assertion.type,
|
|
682
|
-
critical,
|
|
683
|
-
status: "pass",
|
|
684
|
-
message: "",
|
|
685
|
-
};
|
|
686
|
-
|
|
687
|
-
try {
|
|
688
|
-
if (type === "urlcontains") {
|
|
689
|
-
const v = template(assertion.value, vars);
|
|
690
|
-
if (!page.url().toLowerCase().includes(v.toLowerCase())) {
|
|
691
|
-
throw new Error(`URL does not contain: ${v} (got ${page.url()})`);
|
|
692
|
-
}
|
|
693
|
-
}
|
|
694
|
-
|
|
695
|
-
else if (type === "urlmatches") {
|
|
696
|
-
const v = template(assertion.pattern || assertion.value, vars);
|
|
697
|
-
const re = new RegExp(v);
|
|
698
|
-
if (!re.test(page.url())) throw new Error(`URL does not match: ${v}`);
|
|
699
|
-
}
|
|
700
|
-
|
|
701
|
-
else if (type === "urlnotcontains") {
|
|
702
|
-
const v = template(assertion.value, vars);
|
|
703
|
-
if (page.url().toLowerCase().includes(v.toLowerCase())) {
|
|
704
|
-
throw new Error(`URL should not contain: ${v}`);
|
|
705
|
-
}
|
|
706
|
-
}
|
|
707
|
-
|
|
708
|
-
else if (type === "routechanged") {
|
|
709
|
-
const within = Number(assertion.within || 3000);
|
|
710
|
-
const match = assertion.match ? new RegExp(template(assertion.match, vars)) : null;
|
|
711
|
-
const before = page.url();
|
|
712
|
-
await page.waitForTimeout(Math.min(within, 500));
|
|
713
|
-
const after = page.url();
|
|
714
|
-
if (after === before) throw new Error(`Route did not change`);
|
|
715
|
-
if (match && !match.test(after)) throw new Error(`Route changed but does not match: ${assertion.match}`);
|
|
716
|
-
}
|
|
717
|
-
|
|
718
|
-
else if (type === "elementvisible") {
|
|
719
|
-
const target = template(assertion.target, vars);
|
|
720
|
-
const loc = await findLocator(page, target);
|
|
721
|
-
await loc.waitFor({ state: "visible", timeout: Number(assertion.within || plan.timeout) });
|
|
722
|
-
}
|
|
723
|
-
|
|
724
|
-
else if (type === "elementhidden") {
|
|
725
|
-
const target = template(assertion.target, vars);
|
|
726
|
-
const loc = await findLocator(page, target);
|
|
727
|
-
await loc.waitFor({ state: "hidden", timeout: Number(assertion.within || plan.timeout) });
|
|
728
|
-
}
|
|
729
|
-
|
|
730
|
-
else if (type === "elementtext") {
|
|
731
|
-
const target = template(assertion.target, vars);
|
|
732
|
-
const loc = await findLocator(page, target);
|
|
733
|
-
const text = await loc.textContent({ timeout: plan.timeout });
|
|
734
|
-
if (assertion.contains) {
|
|
735
|
-
const expected = template(assertion.contains, vars);
|
|
736
|
-
if (!String(text || "").includes(expected)) {
|
|
737
|
-
throw new Error(`Element text does not contain "${expected}"`);
|
|
738
|
-
}
|
|
739
|
-
}
|
|
740
|
-
if (assertion.matches) {
|
|
741
|
-
const re = new RegExp(template(assertion.matches, vars));
|
|
742
|
-
if (!re.test(text || "")) {
|
|
743
|
-
throw new Error(`Element text does not match ${assertion.matches}`);
|
|
744
|
-
}
|
|
745
|
-
}
|
|
746
|
-
}
|
|
747
|
-
|
|
748
|
-
else if (type === "elementcount") {
|
|
749
|
-
const target = template(assertion.target, vars);
|
|
750
|
-
const count = await page.locator(target).count();
|
|
751
|
-
if (assertion.min != null && count < assertion.min) {
|
|
752
|
-
throw new Error(`Element count ${count} < min ${assertion.min}`);
|
|
753
|
-
}
|
|
754
|
-
if (assertion.max != null && count > assertion.max) {
|
|
755
|
-
throw new Error(`Element count ${count} > max ${assertion.max}`);
|
|
756
|
-
}
|
|
757
|
-
if (assertion.equals != null && count !== assertion.equals) {
|
|
758
|
-
throw new Error(`Element count ${count} !== ${assertion.equals}`);
|
|
759
|
-
}
|
|
760
|
-
}
|
|
761
|
-
|
|
762
|
-
else if (type === "toastcontains" || type === "notificationvisible") {
|
|
763
|
-
const v = template(assertion.text || assertion.contains || assertion.value, vars);
|
|
764
|
-
const within = Number(assertion.within || 5000);
|
|
765
|
-
const toastSel = assertion.target || '[role="status"], [role="alert"], .toast, .sonner-toast, [data-sonner-toast], .notification';
|
|
766
|
-
const loc = page.locator(toastSel).filter({ hasText: new RegExp(v, "i") }).first();
|
|
767
|
-
await loc.waitFor({ state: "visible", timeout: within });
|
|
768
|
-
}
|
|
769
|
-
|
|
770
|
-
else if (type === "toastnotcontains") {
|
|
771
|
-
const v = template(assertion.text || assertion.value, vars);
|
|
772
|
-
const within = Number(assertion.within || 3000);
|
|
773
|
-
const toastSel = assertion.target || '[role="status"], [role="alert"], .toast, .sonner-toast, [data-sonner-toast], .notification';
|
|
774
|
-
await page.waitForTimeout(Math.min(within, 1500));
|
|
775
|
-
const count = await page.locator(toastSel).filter({ hasText: new RegExp(v, "i") }).count();
|
|
776
|
-
if (count > 0) throw new Error(`Toast contains forbidden text: ${v}`);
|
|
777
|
-
}
|
|
778
|
-
|
|
779
|
-
else if (type === "networkcalled") {
|
|
780
|
-
const v = template(assertion.match || assertion.value, vars);
|
|
781
|
-
const within = Number(assertion.within || 10000);
|
|
782
|
-
const minTimes = assertion.times ?? assertion.minTimes ?? 1;
|
|
783
|
-
const since = Date.now() - within;
|
|
784
|
-
const matches = telemetry.responses.filter(r => r.ts >= since && r.url.includes(v));
|
|
785
|
-
if (matches.length < minTimes) {
|
|
786
|
-
throw new Error(`Expected ${minTimes} calls matching ${v}, got ${matches.length}`);
|
|
787
|
-
}
|
|
788
|
-
}
|
|
789
|
-
|
|
790
|
-
else if (type === "networkstatus") {
|
|
791
|
-
const v = template(assertion.match || assertion.value, vars);
|
|
792
|
-
const expectStatus = Number(assertion.status || 200);
|
|
793
|
-
const within = Number(assertion.within || 10000);
|
|
794
|
-
const since = Date.now() - within;
|
|
795
|
-
const match = telemetry.responses.find(r => r.ts >= since && r.url.includes(v));
|
|
796
|
-
if (!match) throw new Error(`No response matching: ${v}`);
|
|
797
|
-
if (match.status !== expectStatus) {
|
|
798
|
-
throw new Error(`Expected ${expectStatus}, got ${match.status} for ${match.url}`);
|
|
799
|
-
}
|
|
800
|
-
}
|
|
801
|
-
|
|
802
|
-
else if (type === "networktiming") {
|
|
803
|
-
const v = template(assertion.match || assertion.value, vars);
|
|
804
|
-
const maxDuration = Number(assertion.maxDuration || 5000);
|
|
805
|
-
// Check request/response pairs - simplified version
|
|
806
|
-
const match = telemetry.responses.find(r => r.url.includes(v));
|
|
807
|
-
if (!match) throw new Error(`No response matching: ${v}`);
|
|
808
|
-
// We don't have timing data in this simplified version
|
|
809
|
-
out.message = `Network timing check (simplified): ${v}`;
|
|
810
|
-
}
|
|
811
|
-
|
|
812
|
-
else if (type === "noconsoleerrors") {
|
|
813
|
-
const severity = String(assertion.severity || "error").toLowerCase();
|
|
814
|
-
const windowMs = Number(assertion.within || 60000);
|
|
815
|
-
const since = Date.now() - windowMs;
|
|
816
|
-
|
|
817
|
-
const ignorePatterns = (assertion.ignore || []).map(p => new RegExp(p));
|
|
818
|
-
|
|
819
|
-
const bad = telemetry.console
|
|
820
|
-
.filter(m => m.ts >= since)
|
|
821
|
-
.filter(m => {
|
|
822
|
-
if (severity === "error") return m.type === "error";
|
|
823
|
-
if (severity === "warning" || severity === "warn") return m.type === "error" || m.type === "warning";
|
|
824
|
-
return m.type === "error";
|
|
825
|
-
})
|
|
826
|
-
.filter(m => !ignorePatterns.some(re => re.test(m.text)));
|
|
827
|
-
|
|
828
|
-
if (bad.length > 0) {
|
|
829
|
-
throw new Error(`Console ${bad[0].type}: ${String(bad[0].text || "").slice(0, 140)}`);
|
|
830
|
-
}
|
|
831
|
-
}
|
|
832
|
-
|
|
833
|
-
else if (type === "consolecontains") {
|
|
834
|
-
const match = template(assertion.match || assertion.value, vars);
|
|
835
|
-
const severity = String(assertion.severity || "log").toLowerCase();
|
|
836
|
-
const found = telemetry.console.some(m => {
|
|
837
|
-
if (severity !== "all" && m.type !== severity) return false;
|
|
838
|
-
return m.text.includes(match);
|
|
839
|
-
});
|
|
840
|
-
if (!found) throw new Error(`Console does not contain: ${match}`);
|
|
841
|
-
}
|
|
842
|
-
|
|
843
|
-
else if (type === "localstorage") {
|
|
844
|
-
const key = template(assertion.key, vars);
|
|
845
|
-
const value = await page.evaluate((k) => localStorage.getItem(k), key);
|
|
846
|
-
if (assertion.exists === true && value === null) {
|
|
847
|
-
throw new Error(`localStorage key "${key}" does not exist`);
|
|
848
|
-
}
|
|
849
|
-
if (assertion.exists === false && value !== null) {
|
|
850
|
-
throw new Error(`localStorage key "${key}" should not exist`);
|
|
851
|
-
}
|
|
852
|
-
if (assertion.equals != null && value !== assertion.equals) {
|
|
853
|
-
throw new Error(`localStorage "${key}" !== "${assertion.equals}"`);
|
|
854
|
-
}
|
|
855
|
-
if (assertion.matches) {
|
|
856
|
-
const re = new RegExp(assertion.matches);
|
|
857
|
-
if (!re.test(value || "")) throw new Error(`localStorage "${key}" does not match ${assertion.matches}`);
|
|
858
|
-
}
|
|
859
|
-
}
|
|
860
|
-
|
|
861
|
-
else if (type === "sessionstorage") {
|
|
862
|
-
const key = template(assertion.key, vars);
|
|
863
|
-
const value = await page.evaluate((k) => sessionStorage.getItem(k), key);
|
|
864
|
-
if (assertion.exists === true && value === null) {
|
|
865
|
-
throw new Error(`sessionStorage key "${key}" does not exist`);
|
|
866
|
-
}
|
|
867
|
-
if (assertion.matches) {
|
|
868
|
-
const re = new RegExp(assertion.matches);
|
|
869
|
-
if (!re.test(value || "")) throw new Error(`sessionStorage "${key}" does not match ${assertion.matches}`);
|
|
870
|
-
}
|
|
871
|
-
}
|
|
872
|
-
|
|
873
|
-
else {
|
|
874
|
-
out.status = "skip";
|
|
875
|
-
out.message = `Assertion type not implemented: ${assertion.type}`;
|
|
876
|
-
}
|
|
877
|
-
} catch (e) {
|
|
878
|
-
out.status = "fail";
|
|
879
|
-
out.message = String(e?.message || e).slice(0, 220);
|
|
880
|
-
}
|
|
881
|
-
|
|
882
|
-
return out;
|
|
883
|
-
}
|
|
884
|
-
|
|
885
|
-
// ============================================================================
|
|
886
|
-
// Score Calculation
|
|
887
|
-
// ============================================================================
|
|
888
|
-
|
|
889
|
-
function calculateScore(results) {
|
|
890
|
-
const routesDiscovered = results.coverage?.routesDiscovered || 0;
|
|
891
|
-
const routesWorking = results.coverage?.routesWorking || 0;
|
|
892
|
-
const routePct = routesDiscovered ? (routesWorking / routesDiscovered) : 1;
|
|
893
|
-
|
|
894
|
-
const flows = results.flows || [];
|
|
895
|
-
const flowTotal = flows.length || 1;
|
|
896
|
-
const flowOk = flows.filter(f => f.status === "success").length;
|
|
897
|
-
|
|
898
|
-
let assertsTotal = 0;
|
|
899
|
-
let assertsPass = 0;
|
|
900
|
-
for (const f of flows) {
|
|
901
|
-
for (const a of (f.assertions || [])) {
|
|
902
|
-
if (a.status === "skip") continue;
|
|
903
|
-
assertsTotal++;
|
|
904
|
-
if (a.status === "pass") assertsPass++;
|
|
905
|
-
}
|
|
906
|
-
}
|
|
907
|
-
const assertPct = assertsTotal ? (assertsPass / assertsTotal) : 1;
|
|
908
|
-
|
|
909
|
-
const errors = results.errors?.length || 0;
|
|
910
|
-
const penalty = Math.min(errors * 3, 20);
|
|
911
|
-
|
|
912
|
-
const score = (routePct * 30) + ((flowOk / flowTotal) * 40) + (assertPct * 30) - penalty;
|
|
913
|
-
|
|
914
|
-
return Math.max(0, Math.min(100, Math.round(score)));
|
|
915
|
-
}
|
|
916
|
-
|
|
917
|
-
module.exports = { runFlowPlan, classifyDanger, discoverSurface };
|
|
1
|
+
/**
|
|
2
|
+
* Reality Mode Execution Engine
|
|
3
|
+
* Step execution, danger classification, assertions, surface discovery
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
const fs = require("fs");
|
|
7
|
+
const path = require("path");
|
|
8
|
+
const crypto = require("crypto");
|
|
9
|
+
|
|
10
|
+
// ============================================================================
|
|
11
|
+
// Utilities
|
|
12
|
+
// ============================================================================
|
|
13
|
+
|
|
14
|
+
function uuid() {
|
|
15
|
+
try {
|
|
16
|
+
return crypto.randomUUID();
|
|
17
|
+
} catch {
|
|
18
|
+
return crypto.randomBytes(16).toString("hex");
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
function randomEmail() {
|
|
23
|
+
return `reality+${Date.now()}_${Math.floor(Math.random() * 1e6)}@example.com`;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
function template(str, vars) {
|
|
27
|
+
if (typeof str !== "string") return str;
|
|
28
|
+
return str.replace(/\{\{([^}]+)\}\}/g, (_, key) => {
|
|
29
|
+
const k = String(key || "").trim();
|
|
30
|
+
if (k === "$timestamp") return String(Date.now());
|
|
31
|
+
if (k === "$uuid") return uuid();
|
|
32
|
+
if (k === "$randomEmail") return randomEmail();
|
|
33
|
+
if (k === "$isoDate") return new Date().toISOString();
|
|
34
|
+
if (k.startsWith("env.")) return process.env[k.slice(4)] || "";
|
|
35
|
+
if (k.startsWith("stored.")) return vars?.[k] ?? "";
|
|
36
|
+
return vars?.[k] != null ? String(vars[k]) : "";
|
|
37
|
+
});
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
// ============================================================================
|
|
41
|
+
// Danger Classification System
|
|
42
|
+
// ============================================================================
|
|
43
|
+
|
|
44
|
+
const DANGER_TEXT_PATTERNS = [
|
|
45
|
+
/\bdelete\b/i,
|
|
46
|
+
/\bremove\b/i,
|
|
47
|
+
/\bdestroy\b/i,
|
|
48
|
+
/\bdeactivate\b/i,
|
|
49
|
+
/\bterminate\b/i,
|
|
50
|
+
/\bclose\s*account\b/i,
|
|
51
|
+
/\bcancel\s*subscription\b/i,
|
|
52
|
+
/\breset\s*all\b/i,
|
|
53
|
+
/\bwipe\b/i,
|
|
54
|
+
/\berase\b/i,
|
|
55
|
+
/\birreversible\b/i,
|
|
56
|
+
/\bpermanently\b/i,
|
|
57
|
+
/\bcannot\s*be\s*undone\b/i,
|
|
58
|
+
/\brevoke\b/i,
|
|
59
|
+
/\bdisconnect\b/i,
|
|
60
|
+
/\bunlink\b/i,
|
|
61
|
+
];
|
|
62
|
+
|
|
63
|
+
const DANGER_CLASS_PATTERNS = [
|
|
64
|
+
/\bbtn-danger\b/,
|
|
65
|
+
/\bdestructive\b/,
|
|
66
|
+
/\bdelete-/,
|
|
67
|
+
/\bdanger-/,
|
|
68
|
+
/\bremove-/,
|
|
69
|
+
];
|
|
70
|
+
|
|
71
|
+
function classifyDanger({ text, className, ariaLabel, role, url, method }) {
|
|
72
|
+
const signals = [];
|
|
73
|
+
let score = 0;
|
|
74
|
+
|
|
75
|
+
const t = `${text || ""}`.toLowerCase();
|
|
76
|
+
const cls = `${className || ""}`.toLowerCase();
|
|
77
|
+
const aria = `${ariaLabel || ""}`.toLowerCase();
|
|
78
|
+
const r = `${role || ""}`.toLowerCase();
|
|
79
|
+
const u = `${url || ""}`.toLowerCase();
|
|
80
|
+
const m = `${method || ""}`.toUpperCase();
|
|
81
|
+
|
|
82
|
+
// Text patterns
|
|
83
|
+
for (const pattern of DANGER_TEXT_PATTERNS) {
|
|
84
|
+
if (pattern.test(t) || pattern.test(aria)) {
|
|
85
|
+
signals.push({ type: "text", value: pattern.source, weight: 0.3 });
|
|
86
|
+
score += 0.3;
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
// Class patterns
|
|
91
|
+
for (const pattern of DANGER_CLASS_PATTERNS) {
|
|
92
|
+
if (pattern.test(cls)) {
|
|
93
|
+
signals.push({ type: "class", value: pattern.source, weight: 0.25 });
|
|
94
|
+
score += 0.25;
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
// ARIA patterns
|
|
99
|
+
if (r === "alertdialog") {
|
|
100
|
+
signals.push({ type: "aria", value: "role=alertdialog", weight: 0.2 });
|
|
101
|
+
score += 0.2;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
// Network patterns (DELETE method or destructive endpoints)
|
|
105
|
+
if (m === "DELETE") {
|
|
106
|
+
signals.push({ type: "network", value: "DELETE method", weight: 0.4 });
|
|
107
|
+
score += 0.4;
|
|
108
|
+
}
|
|
109
|
+
if (/\/delete\b|\/destroy\b|\/deactivate\b|\/remove\b/.test(u)) {
|
|
110
|
+
signals.push({ type: "network", value: "destructive endpoint", weight: 0.35 });
|
|
111
|
+
score += 0.35;
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
const dangerous = score >= 0.25;
|
|
115
|
+
const reason = signals.length > 0 ? signals[0].type : "none";
|
|
116
|
+
|
|
117
|
+
return {
|
|
118
|
+
dangerous,
|
|
119
|
+
score: Math.min(1, score),
|
|
120
|
+
signals,
|
|
121
|
+
reason,
|
|
122
|
+
};
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
// ============================================================================
|
|
126
|
+
// Screenshots & DOM Snapshots
|
|
127
|
+
// ============================================================================
|
|
128
|
+
|
|
129
|
+
async function screenshot(page, outPath) {
|
|
130
|
+
try {
|
|
131
|
+
await page.screenshot({ path: outPath, fullPage: true });
|
|
132
|
+
} catch {
|
|
133
|
+
// Non-fatal
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
async function domSnapshot(page) {
|
|
138
|
+
try {
|
|
139
|
+
return await page.evaluate(() => {
|
|
140
|
+
const el = document.activeElement;
|
|
141
|
+
return {
|
|
142
|
+
url: location.href,
|
|
143
|
+
title: document.title,
|
|
144
|
+
active: el ? { tag: el.tagName, id: el.id, name: el.getAttribute("name") } : null,
|
|
145
|
+
bodyTextSample: document.body?.innerText?.slice(0, 500) || "",
|
|
146
|
+
};
|
|
147
|
+
});
|
|
148
|
+
} catch {
|
|
149
|
+
return null;
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
// ============================================================================
|
|
154
|
+
// Navigation Helpers
|
|
155
|
+
// ============================================================================
|
|
156
|
+
|
|
157
|
+
async function safeGoto(page, url, timeout) {
|
|
158
|
+
const started = Date.now();
|
|
159
|
+
try {
|
|
160
|
+
const res = await page.goto(url, { waitUntil: "domcontentloaded", timeout });
|
|
161
|
+
await page.waitForLoadState("networkidle", { timeout: Math.min(timeout, 5000) }).catch(() => {});
|
|
162
|
+
const status = res ? res.status() : 0;
|
|
163
|
+
return { ok: status >= 200 && status < 400, status, ms: Date.now() - started };
|
|
164
|
+
} catch (e) {
|
|
165
|
+
return { ok: false, status: 0, ms: Date.now() - started, error: String(e?.message || e).slice(0, 180) };
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
async function findLocator(page, target) {
|
|
170
|
+
return page.locator(target).first();
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
// ============================================================================
|
|
174
|
+
// Surface Discovery
|
|
175
|
+
// ============================================================================
|
|
176
|
+
|
|
177
|
+
async function discoverSurface({ page, baseUrl, timeout, maxPages }) {
|
|
178
|
+
const discovered = {
|
|
179
|
+
routes: [],
|
|
180
|
+
elements: [],
|
|
181
|
+
forms: [],
|
|
182
|
+
};
|
|
183
|
+
|
|
184
|
+
// Visit base
|
|
185
|
+
const home = await safeGoto(page, baseUrl, timeout);
|
|
186
|
+
if (home.status) {
|
|
187
|
+
discovered.routes.push({ path: "/", status: home.ok ? "success" : "error", httpStatus: home.status });
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
// Extract routes
|
|
191
|
+
const routes = await page.evaluate(() => {
|
|
192
|
+
const out = new Set();
|
|
193
|
+
document.querySelectorAll("a[href]").forEach(a => {
|
|
194
|
+
const href = a.getAttribute("href") || "";
|
|
195
|
+
if (!href || href.startsWith("http") || href.startsWith("//") || href.includes("#")) return;
|
|
196
|
+
if (href.startsWith("/")) out.add(href);
|
|
197
|
+
});
|
|
198
|
+
// Common SPA routes
|
|
199
|
+
["/dashboard", "/app", "/account", "/settings", "/profile", "/projects", "/analytics", "/billing"].forEach(r => out.add(r));
|
|
200
|
+
return Array.from(out);
|
|
201
|
+
}).catch(() => []);
|
|
202
|
+
|
|
203
|
+
const queue = routes.slice(0, maxPages * 2);
|
|
204
|
+
const seen = new Set(["/", baseUrl]);
|
|
205
|
+
|
|
206
|
+
for (const route of queue.slice(0, maxPages)) {
|
|
207
|
+
if (seen.has(route)) continue;
|
|
208
|
+
seen.add(route);
|
|
209
|
+
|
|
210
|
+
const res = await safeGoto(page, baseUrl + route, timeout);
|
|
211
|
+
discovered.routes.push({
|
|
212
|
+
path: route,
|
|
213
|
+
status: res.ok ? "success" : "error",
|
|
214
|
+
httpStatus: res.status || 0,
|
|
215
|
+
error: res.error,
|
|
216
|
+
});
|
|
217
|
+
|
|
218
|
+
// Discover elements on this page
|
|
219
|
+
const pageEls = await page.evaluate(() => {
|
|
220
|
+
const pickText = (el) => (el.textContent || el.getAttribute("aria-label") || "").trim().slice(0, 80);
|
|
221
|
+
const els = Array.from(document.querySelectorAll('button, [role="button"], input[type="submit"], a[href], [data-testid]'));
|
|
222
|
+
return els.slice(0, 100).map((el, idx) => ({
|
|
223
|
+
tag: el.tagName.toLowerCase(),
|
|
224
|
+
text: pickText(el),
|
|
225
|
+
id: el.id || "",
|
|
226
|
+
testid: el.getAttribute("data-testid") || "",
|
|
227
|
+
className: el.className || "",
|
|
228
|
+
ariaLabel: el.getAttribute("aria-label") || "",
|
|
229
|
+
href: el.getAttribute("href") || "",
|
|
230
|
+
index: idx,
|
|
231
|
+
}));
|
|
232
|
+
}).catch(() => []);
|
|
233
|
+
|
|
234
|
+
for (const e of pageEls) {
|
|
235
|
+
if (!e.text && !e.testid && !e.id) continue;
|
|
236
|
+
discovered.elements.push({ ...e, page: route });
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
// Discover forms
|
|
240
|
+
const forms = await page.evaluate(() => {
|
|
241
|
+
const forms = Array.from(document.querySelectorAll("form"));
|
|
242
|
+
return forms.slice(0, 20).map((form, idx) => {
|
|
243
|
+
const fields = Array.from(form.querySelectorAll("input, textarea, select")).slice(0, 30).map((f) => ({
|
|
244
|
+
name: f.getAttribute("name") || f.id || "",
|
|
245
|
+
type: f.getAttribute("type") || f.tagName.toLowerCase(),
|
|
246
|
+
required: !!f.required,
|
|
247
|
+
}));
|
|
248
|
+
return {
|
|
249
|
+
selector: form.id ? `#${form.id}` : form.getAttribute("data-testid") ? `[data-testid="${form.getAttribute("data-testid")}"]` : `form:nth-of-type(${idx + 1})`,
|
|
250
|
+
fields,
|
|
251
|
+
};
|
|
252
|
+
});
|
|
253
|
+
}).catch(() => []);
|
|
254
|
+
|
|
255
|
+
for (const f of forms) discovered.forms.push({ ...f, page: route });
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
// Dedupe
|
|
259
|
+
discovered.routes = dedupe(discovered.routes, r => r.path);
|
|
260
|
+
discovered.elements = dedupe(discovered.elements, e => `${e.page}|${e.tag}|${e.testid}|${e.id}|${e.text}`);
|
|
261
|
+
discovered.forms = dedupe(discovered.forms, f => `${f.page}|${f.selector}`);
|
|
262
|
+
|
|
263
|
+
return discovered;
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
function dedupe(arr, keyFn) {
|
|
267
|
+
const seen = new Set();
|
|
268
|
+
const out = [];
|
|
269
|
+
for (const x of arr) {
|
|
270
|
+
const k = keyFn(x);
|
|
271
|
+
if (seen.has(k)) continue;
|
|
272
|
+
seen.add(k);
|
|
273
|
+
out.push(x);
|
|
274
|
+
}
|
|
275
|
+
return out;
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
// ============================================================================
|
|
279
|
+
// Flow Plan Execution
|
|
280
|
+
// ============================================================================
|
|
281
|
+
|
|
282
|
+
async function runFlowPlan({ plan, page, context, telemetry }) {
|
|
283
|
+
const results = {
|
|
284
|
+
meta: {
|
|
285
|
+
baseUrl: plan.baseUrl,
|
|
286
|
+
startedAt: new Date().toISOString(),
|
|
287
|
+
danger: plan.danger,
|
|
288
|
+
maxPages: plan.maxPages,
|
|
289
|
+
timeout: plan.timeout,
|
|
290
|
+
},
|
|
291
|
+
discovery: null,
|
|
292
|
+
coverage: {
|
|
293
|
+
routesDiscovered: 0,
|
|
294
|
+
routesWorking: 0,
|
|
295
|
+
elementsDiscovered: 0,
|
|
296
|
+
elementsWorking: 0,
|
|
297
|
+
formsDiscovered: 0,
|
|
298
|
+
formsWorking: 0,
|
|
299
|
+
},
|
|
300
|
+
flows: [],
|
|
301
|
+
routes: [],
|
|
302
|
+
elements: [],
|
|
303
|
+
forms: [],
|
|
304
|
+
errors: [],
|
|
305
|
+
network: [],
|
|
306
|
+
console: [],
|
|
307
|
+
timeline: [],
|
|
308
|
+
score: 0,
|
|
309
|
+
duration: 0,
|
|
310
|
+
};
|
|
311
|
+
|
|
312
|
+
// Discovery pass
|
|
313
|
+
const discovery = await discoverSurface({
|
|
314
|
+
page,
|
|
315
|
+
baseUrl: plan.baseUrl,
|
|
316
|
+
timeout: plan.timeout,
|
|
317
|
+
maxPages: plan.maxPages,
|
|
318
|
+
});
|
|
319
|
+
|
|
320
|
+
results.discovery = {
|
|
321
|
+
routes: discovery.routes.length,
|
|
322
|
+
elements: discovery.elements.length,
|
|
323
|
+
forms: discovery.forms.length,
|
|
324
|
+
};
|
|
325
|
+
|
|
326
|
+
results.routes = discovery.routes;
|
|
327
|
+
results.elements = discovery.elements;
|
|
328
|
+
results.forms = discovery.forms;
|
|
329
|
+
|
|
330
|
+
results.coverage.routesDiscovered = discovery.routes.length;
|
|
331
|
+
results.coverage.routesWorking = discovery.routes.filter(r => r.status === "success").length;
|
|
332
|
+
results.coverage.elementsDiscovered = discovery.elements.length;
|
|
333
|
+
results.coverage.formsDiscovered = discovery.forms.length;
|
|
334
|
+
|
|
335
|
+
// Collect telemetry
|
|
336
|
+
results.console = telemetry.console.slice(-500);
|
|
337
|
+
results.errors = [
|
|
338
|
+
...telemetry.pageErrors.map(e => ({ type: "uncaught", message: e.message, url: e.url, ts: e.ts })),
|
|
339
|
+
...telemetry.console.filter(m => m.type === "error").map(m => ({ type: "console", message: m.text, url: m.url, ts: m.ts })),
|
|
340
|
+
];
|
|
341
|
+
results.network = telemetry.responses.slice(-500);
|
|
342
|
+
|
|
343
|
+
// Run Flow Packs
|
|
344
|
+
const flowVarsBase = makeBaseVars(plan);
|
|
345
|
+
const flowsToRun = Array.isArray(plan.flows) ? plan.flows : [];
|
|
346
|
+
|
|
347
|
+
for (const flow of flowsToRun) {
|
|
348
|
+
const flowResult = await runSingleFlow({
|
|
349
|
+
flow,
|
|
350
|
+
plan,
|
|
351
|
+
page,
|
|
352
|
+
context,
|
|
353
|
+
telemetry,
|
|
354
|
+
varsBase: flowVarsBase,
|
|
355
|
+
results,
|
|
356
|
+
});
|
|
357
|
+
results.flows.push(flowResult);
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
// Count working elements from successful flow steps
|
|
361
|
+
let elementWorking = 0;
|
|
362
|
+
for (const f of results.flows) {
|
|
363
|
+
for (const s of (f.steps || [])) {
|
|
364
|
+
if ((s.action === "click" || s.action === "fill") && s.status === "success") elementWorking++;
|
|
365
|
+
}
|
|
366
|
+
}
|
|
367
|
+
results.coverage.elementsWorking = Math.min(elementWorking, results.coverage.elementsDiscovered);
|
|
368
|
+
|
|
369
|
+
// Calculate score
|
|
370
|
+
results.score = calculateScore(results);
|
|
371
|
+
|
|
372
|
+
return results;
|
|
373
|
+
}
|
|
374
|
+
|
|
375
|
+
function makeBaseVars(plan) {
|
|
376
|
+
const vars = {
|
|
377
|
+
baseUrl: plan.baseUrl,
|
|
378
|
+
timestamp: String(Date.now()),
|
|
379
|
+
$timestamp: String(Date.now()),
|
|
380
|
+
$uuid: uuid(),
|
|
381
|
+
$randomEmail: randomEmail(),
|
|
382
|
+
$isoDate: new Date().toISOString(),
|
|
383
|
+
};
|
|
384
|
+
|
|
385
|
+
if (plan.auth && typeof plan.auth === "string" && plan.auth.includes(":")) {
|
|
386
|
+
const [email, ...rest] = plan.auth.split(":");
|
|
387
|
+
vars.email = email;
|
|
388
|
+
vars.password = rest.join(":");
|
|
389
|
+
}
|
|
390
|
+
|
|
391
|
+
return vars;
|
|
392
|
+
}
|
|
393
|
+
|
|
394
|
+
// ============================================================================
|
|
395
|
+
// Single Flow Execution
|
|
396
|
+
// ============================================================================
|
|
397
|
+
|
|
398
|
+
async function runSingleFlow({ flow, plan, page, telemetry, varsBase, results }) {
|
|
399
|
+
const flowStart = Date.now();
|
|
400
|
+
const flowVars = { ...varsBase, ...(flow.vars || {}) };
|
|
401
|
+
|
|
402
|
+
// Template all vars
|
|
403
|
+
for (const k of Object.keys(flowVars)) {
|
|
404
|
+
flowVars[k] = template(flowVars[k], flowVars);
|
|
405
|
+
}
|
|
406
|
+
|
|
407
|
+
const flowOut = {
|
|
408
|
+
id: flow.id,
|
|
409
|
+
name: flow.name,
|
|
410
|
+
source: flow.__source,
|
|
411
|
+
required: !!flow.required,
|
|
412
|
+
status: "success",
|
|
413
|
+
startedAt: new Date(flowStart).toISOString(),
|
|
414
|
+
durationMs: 0,
|
|
415
|
+
steps: [],
|
|
416
|
+
assertions: [],
|
|
417
|
+
errors: [],
|
|
418
|
+
};
|
|
419
|
+
|
|
420
|
+
// Run steps
|
|
421
|
+
let stepIndex = 0;
|
|
422
|
+
for (const step of flow.steps || []) {
|
|
423
|
+
stepIndex++;
|
|
424
|
+
const stepId = `${flow.id}#${stepIndex}`;
|
|
425
|
+
const stepName = step.name || `${step.action || "step"}-${stepIndex}`;
|
|
426
|
+
const stepStart = Date.now();
|
|
427
|
+
|
|
428
|
+
const stepRec = {
|
|
429
|
+
id: stepId,
|
|
430
|
+
name: stepName,
|
|
431
|
+
action: step.action,
|
|
432
|
+
target: step.target || null,
|
|
433
|
+
status: "success",
|
|
434
|
+
error: null,
|
|
435
|
+
danger: null,
|
|
436
|
+
artifacts: {},
|
|
437
|
+
startedAt: new Date(stepStart).toISOString(),
|
|
438
|
+
durationMs: 0,
|
|
439
|
+
};
|
|
440
|
+
|
|
441
|
+
// Trace marker
|
|
442
|
+
try {
|
|
443
|
+
await page.evaluate((label) => console.log(`[reality:trace] ${label}`), `STEP ${stepId}`);
|
|
444
|
+
} catch {}
|
|
445
|
+
|
|
446
|
+
// Before screenshot
|
|
447
|
+
const beforeShot = path.join(plan.artifacts.screenshotsDir, `${flow.id}__${stepIndex}__before.png`);
|
|
448
|
+
await screenshot(page, beforeShot);
|
|
449
|
+
stepRec.artifacts.beforeScreenshot = path.relative(plan.outputDir, beforeShot);
|
|
450
|
+
stepRec.artifacts.beforeDom = await domSnapshot(page);
|
|
451
|
+
|
|
452
|
+
// Execute action
|
|
453
|
+
try {
|
|
454
|
+
await executeStep({ step, stepRec, plan, page, flowVars, telemetry });
|
|
455
|
+
} catch (e) {
|
|
456
|
+
stepRec.status = "error";
|
|
457
|
+
stepRec.error = String(e?.message || e).slice(0, 220);
|
|
458
|
+
flowOut.errors.push({ step: stepIndex, message: stepRec.error });
|
|
459
|
+
}
|
|
460
|
+
|
|
461
|
+
// After screenshot
|
|
462
|
+
const afterShot = path.join(plan.artifacts.screenshotsDir, `${flow.id}__${stepIndex}__after.png`);
|
|
463
|
+
await screenshot(page, afterShot);
|
|
464
|
+
stepRec.artifacts.afterScreenshot = path.relative(plan.outputDir, afterShot);
|
|
465
|
+
stepRec.artifacts.afterDom = await domSnapshot(page);
|
|
466
|
+
|
|
467
|
+
stepRec.durationMs = Date.now() - stepStart;
|
|
468
|
+
|
|
469
|
+
// Add to timeline
|
|
470
|
+
results.timeline.push({
|
|
471
|
+
flowId: flow.id,
|
|
472
|
+
step: stepIndex,
|
|
473
|
+
name: stepRec.name,
|
|
474
|
+
action: stepRec.action,
|
|
475
|
+
status: stepRec.status,
|
|
476
|
+
at: Date.now(),
|
|
477
|
+
url: page.url(),
|
|
478
|
+
before: stepRec.artifacts.beforeScreenshot,
|
|
479
|
+
after: stepRec.artifacts.afterScreenshot,
|
|
480
|
+
});
|
|
481
|
+
|
|
482
|
+
flowOut.steps.push(stepRec);
|
|
483
|
+
}
|
|
484
|
+
|
|
485
|
+
// Run assertions
|
|
486
|
+
for (const a of flow.assertions || []) {
|
|
487
|
+
const ar = await runAssertion({ assertion: a, plan, page, telemetry, vars: flowVars });
|
|
488
|
+
flowOut.assertions.push(ar);
|
|
489
|
+
if (ar.status === "fail" && a.critical) {
|
|
490
|
+
flowOut.status = "error";
|
|
491
|
+
}
|
|
492
|
+
}
|
|
493
|
+
|
|
494
|
+
// Cleanup (best-effort)
|
|
495
|
+
for (const step of flow.cleanup || []) {
|
|
496
|
+
try {
|
|
497
|
+
await executeStep({ step, stepRec: {}, plan, page, flowVars, telemetry });
|
|
498
|
+
} catch {}
|
|
499
|
+
}
|
|
500
|
+
|
|
501
|
+
flowOut.durationMs = Date.now() - flowStart;
|
|
502
|
+
if (flowOut.errors.length > 0) flowOut.status = "error";
|
|
503
|
+
|
|
504
|
+
return flowOut;
|
|
505
|
+
}
|
|
506
|
+
|
|
507
|
+
// ============================================================================
|
|
508
|
+
// Step Execution
|
|
509
|
+
// ============================================================================
|
|
510
|
+
|
|
511
|
+
async function executeStep({ step, stepRec, plan, page, flowVars, telemetry }) {
|
|
512
|
+
const action = String(step.action || "").toLowerCase();
|
|
513
|
+
|
|
514
|
+
if (action === "navigate") {
|
|
515
|
+
const target = template(step.target || "/", flowVars);
|
|
516
|
+
const url = target.startsWith("http") ? target : (plan.baseUrl + (target.startsWith("/") ? target : `/${target}`));
|
|
517
|
+
const res = await safeGoto(page, url, plan.timeout);
|
|
518
|
+
if (!res.ok) throw new Error(res.error || `Navigation failed: ${url}`);
|
|
519
|
+
}
|
|
520
|
+
|
|
521
|
+
else if (action === "wait") {
|
|
522
|
+
if (step.for === "selector" && step.target) {
|
|
523
|
+
const target = template(step.target, flowVars);
|
|
524
|
+
const state = step.state || "visible";
|
|
525
|
+
const loc = await findLocator(page, target);
|
|
526
|
+
await loc.waitFor({ state, timeout: Number(step.timeout || plan.timeout) });
|
|
527
|
+
} else if (step.for === "navigation" && step.match) {
|
|
528
|
+
const match = new RegExp(template(step.match, flowVars));
|
|
529
|
+
const timeout = Number(step.timeout || plan.timeout);
|
|
530
|
+
const end = Date.now() + timeout;
|
|
531
|
+
while (Date.now() < end) {
|
|
532
|
+
if (match.test(page.url())) break;
|
|
533
|
+
await page.waitForTimeout(200);
|
|
534
|
+
}
|
|
535
|
+
if (!match.test(page.url())) throw new Error(`URL did not match ${step.match}`);
|
|
536
|
+
} else if (step.for === "network" && step.match) {
|
|
537
|
+
const match = template(step.match, flowVars);
|
|
538
|
+
const timeout = Number(step.timeout || plan.timeout);
|
|
539
|
+
const since = Date.now();
|
|
540
|
+
const end = since + timeout;
|
|
541
|
+
let found = false;
|
|
542
|
+
while (Date.now() < end) {
|
|
543
|
+
const recent = telemetry.responses.filter(r => r.ts >= since);
|
|
544
|
+
if (recent.some(r => r.url.includes(match))) { found = true; break; }
|
|
545
|
+
await page.waitForTimeout(200);
|
|
546
|
+
}
|
|
547
|
+
if (!found) throw new Error(`No network call matching ${match}`);
|
|
548
|
+
} else {
|
|
549
|
+
const ms = Number(step.timeout || step.ms || 1000);
|
|
550
|
+
await page.waitForTimeout(ms);
|
|
551
|
+
}
|
|
552
|
+
}
|
|
553
|
+
|
|
554
|
+
else if (action === "fill") {
|
|
555
|
+
const target = template(step.target, flowVars);
|
|
556
|
+
const value = template(step.value, flowVars);
|
|
557
|
+
const loc = await findLocator(page, target);
|
|
558
|
+
await loc.waitFor({ state: "visible", timeout: plan.timeout }).catch(() => {});
|
|
559
|
+
if (step.clear !== false) await loc.clear().catch(() => {});
|
|
560
|
+
await loc.fill(String(value ?? ""), { timeout: plan.timeout });
|
|
561
|
+
}
|
|
562
|
+
|
|
563
|
+
else if (action === "click") {
|
|
564
|
+
const target = template(step.target, flowVars);
|
|
565
|
+
const loc = await findLocator(page, target);
|
|
566
|
+
|
|
567
|
+
// Danger detection
|
|
568
|
+
let meta = {};
|
|
569
|
+
try {
|
|
570
|
+
meta = await loc.evaluate((el) => ({
|
|
571
|
+
text: (el.textContent || "").trim().slice(0, 120),
|
|
572
|
+
className: el.className || "",
|
|
573
|
+
ariaLabel: el.getAttribute("aria-label") || "",
|
|
574
|
+
role: el.getAttribute("role") || "",
|
|
575
|
+
}));
|
|
576
|
+
} catch {}
|
|
577
|
+
|
|
578
|
+
const danger = classifyDanger({ ...meta, url: page.url() });
|
|
579
|
+
stepRec.danger = danger;
|
|
580
|
+
|
|
581
|
+
const policy = String(step.dangerPolicy || "skip").toLowerCase();
|
|
582
|
+
if (danger.dangerous && !plan.danger) {
|
|
583
|
+
if (policy === "block") {
|
|
584
|
+
stepRec.status = "blocked";
|
|
585
|
+
stepRec.error = `Blocked destructive action (${danger.reason}). Use --danger to allow.`;
|
|
586
|
+
return;
|
|
587
|
+
}
|
|
588
|
+
stepRec.status = "skipped";
|
|
589
|
+
stepRec.error = `Skipped destructive action (${danger.reason}). Use --danger to allow.`;
|
|
590
|
+
return;
|
|
591
|
+
}
|
|
592
|
+
|
|
593
|
+
await loc.waitFor({ state: "visible", timeout: plan.timeout }).catch(() => {});
|
|
594
|
+
try {
|
|
595
|
+
await loc.click({ timeout: plan.timeout });
|
|
596
|
+
} catch {
|
|
597
|
+
await loc.click({ timeout: plan.timeout, force: true });
|
|
598
|
+
}
|
|
599
|
+
await page.waitForTimeout(300);
|
|
600
|
+
}
|
|
601
|
+
|
|
602
|
+
else if (action === "check") {
|
|
603
|
+
const target = template(step.target, flowVars);
|
|
604
|
+
const loc = await findLocator(page, target);
|
|
605
|
+
await loc.check({ timeout: plan.timeout });
|
|
606
|
+
}
|
|
607
|
+
|
|
608
|
+
else if (action === "uncheck") {
|
|
609
|
+
const target = template(step.target, flowVars);
|
|
610
|
+
const loc = await findLocator(page, target);
|
|
611
|
+
await loc.uncheck({ timeout: plan.timeout });
|
|
612
|
+
}
|
|
613
|
+
|
|
614
|
+
else if (action === "select") {
|
|
615
|
+
const target = template(step.target, flowVars);
|
|
616
|
+
const loc = await findLocator(page, target);
|
|
617
|
+
if (step.value) {
|
|
618
|
+
await loc.selectOption({ value: template(step.value, flowVars) });
|
|
619
|
+
} else if (step.label) {
|
|
620
|
+
await loc.selectOption({ label: template(step.label, flowVars) });
|
|
621
|
+
} else if (step.index != null) {
|
|
622
|
+
await loc.selectOption({ index: Number(step.index) });
|
|
623
|
+
}
|
|
624
|
+
}
|
|
625
|
+
|
|
626
|
+
else if (action === "hover") {
|
|
627
|
+
const target = template(step.target, flowVars);
|
|
628
|
+
const loc = await findLocator(page, target);
|
|
629
|
+
await loc.hover({ timeout: plan.timeout });
|
|
630
|
+
}
|
|
631
|
+
|
|
632
|
+
else if (action === "press") {
|
|
633
|
+
const key = template(step.key, flowVars);
|
|
634
|
+
await page.keyboard.press(key);
|
|
635
|
+
}
|
|
636
|
+
|
|
637
|
+
else if (action === "screenshot") {
|
|
638
|
+
const name = template(step.name || `manual-${Date.now()}`, flowVars);
|
|
639
|
+
const outPath = path.join(plan.artifacts.screenshotsDir, `${name}.png`);
|
|
640
|
+
await screenshot(page, outPath);
|
|
641
|
+
stepRec.artifacts.manualScreenshot = path.relative(plan.outputDir, outPath);
|
|
642
|
+
}
|
|
643
|
+
|
|
644
|
+
else if (action === "scroll") {
|
|
645
|
+
if (step.to === "bottom") {
|
|
646
|
+
await page.evaluate(() => window.scrollTo(0, document.body.scrollHeight));
|
|
647
|
+
} else if (step.to === "top") {
|
|
648
|
+
await page.evaluate(() => window.scrollTo(0, 0));
|
|
649
|
+
} else if (step.target) {
|
|
650
|
+
const target = template(step.target, flowVars);
|
|
651
|
+
const loc = await findLocator(page, target);
|
|
652
|
+
await loc.scrollIntoViewIfNeeded();
|
|
653
|
+
}
|
|
654
|
+
}
|
|
655
|
+
|
|
656
|
+
else if (action === "execute") {
|
|
657
|
+
if (step.script) {
|
|
658
|
+
await page.evaluate(step.script);
|
|
659
|
+
} else if (step.scriptFile) {
|
|
660
|
+
const scriptPath = path.resolve(step.scriptFile);
|
|
661
|
+
const script = fs.readFileSync(scriptPath, "utf8");
|
|
662
|
+
await page.evaluate(script);
|
|
663
|
+
}
|
|
664
|
+
}
|
|
665
|
+
|
|
666
|
+
else {
|
|
667
|
+
stepRec.status = "unknown";
|
|
668
|
+
stepRec.error = `Unknown action: ${step.action}`;
|
|
669
|
+
}
|
|
670
|
+
}
|
|
671
|
+
|
|
672
|
+
// ============================================================================
|
|
673
|
+
// Assertions
|
|
674
|
+
// ============================================================================
|
|
675
|
+
|
|
676
|
+
async function runAssertion({ assertion, plan, page, telemetry, vars }) {
|
|
677
|
+
const type = String(assertion.type || "").toLowerCase().replace(/-/g, "");
|
|
678
|
+
const critical = !!assertion.critical;
|
|
679
|
+
|
|
680
|
+
const out = {
|
|
681
|
+
type: assertion.type,
|
|
682
|
+
critical,
|
|
683
|
+
status: "pass",
|
|
684
|
+
message: "",
|
|
685
|
+
};
|
|
686
|
+
|
|
687
|
+
try {
|
|
688
|
+
if (type === "urlcontains") {
|
|
689
|
+
const v = template(assertion.value, vars);
|
|
690
|
+
if (!page.url().toLowerCase().includes(v.toLowerCase())) {
|
|
691
|
+
throw new Error(`URL does not contain: ${v} (got ${page.url()})`);
|
|
692
|
+
}
|
|
693
|
+
}
|
|
694
|
+
|
|
695
|
+
else if (type === "urlmatches") {
|
|
696
|
+
const v = template(assertion.pattern || assertion.value, vars);
|
|
697
|
+
const re = new RegExp(v);
|
|
698
|
+
if (!re.test(page.url())) throw new Error(`URL does not match: ${v}`);
|
|
699
|
+
}
|
|
700
|
+
|
|
701
|
+
else if (type === "urlnotcontains") {
|
|
702
|
+
const v = template(assertion.value, vars);
|
|
703
|
+
if (page.url().toLowerCase().includes(v.toLowerCase())) {
|
|
704
|
+
throw new Error(`URL should not contain: ${v}`);
|
|
705
|
+
}
|
|
706
|
+
}
|
|
707
|
+
|
|
708
|
+
else if (type === "routechanged") {
|
|
709
|
+
const within = Number(assertion.within || 3000);
|
|
710
|
+
const match = assertion.match ? new RegExp(template(assertion.match, vars)) : null;
|
|
711
|
+
const before = page.url();
|
|
712
|
+
await page.waitForTimeout(Math.min(within, 500));
|
|
713
|
+
const after = page.url();
|
|
714
|
+
if (after === before) throw new Error(`Route did not change`);
|
|
715
|
+
if (match && !match.test(after)) throw new Error(`Route changed but does not match: ${assertion.match}`);
|
|
716
|
+
}
|
|
717
|
+
|
|
718
|
+
else if (type === "elementvisible") {
|
|
719
|
+
const target = template(assertion.target, vars);
|
|
720
|
+
const loc = await findLocator(page, target);
|
|
721
|
+
await loc.waitFor({ state: "visible", timeout: Number(assertion.within || plan.timeout) });
|
|
722
|
+
}
|
|
723
|
+
|
|
724
|
+
else if (type === "elementhidden") {
|
|
725
|
+
const target = template(assertion.target, vars);
|
|
726
|
+
const loc = await findLocator(page, target);
|
|
727
|
+
await loc.waitFor({ state: "hidden", timeout: Number(assertion.within || plan.timeout) });
|
|
728
|
+
}
|
|
729
|
+
|
|
730
|
+
else if (type === "elementtext") {
|
|
731
|
+
const target = template(assertion.target, vars);
|
|
732
|
+
const loc = await findLocator(page, target);
|
|
733
|
+
const text = await loc.textContent({ timeout: plan.timeout });
|
|
734
|
+
if (assertion.contains) {
|
|
735
|
+
const expected = template(assertion.contains, vars);
|
|
736
|
+
if (!String(text || "").includes(expected)) {
|
|
737
|
+
throw new Error(`Element text does not contain "${expected}"`);
|
|
738
|
+
}
|
|
739
|
+
}
|
|
740
|
+
if (assertion.matches) {
|
|
741
|
+
const re = new RegExp(template(assertion.matches, vars));
|
|
742
|
+
if (!re.test(text || "")) {
|
|
743
|
+
throw new Error(`Element text does not match ${assertion.matches}`);
|
|
744
|
+
}
|
|
745
|
+
}
|
|
746
|
+
}
|
|
747
|
+
|
|
748
|
+
else if (type === "elementcount") {
|
|
749
|
+
const target = template(assertion.target, vars);
|
|
750
|
+
const count = await page.locator(target).count();
|
|
751
|
+
if (assertion.min != null && count < assertion.min) {
|
|
752
|
+
throw new Error(`Element count ${count} < min ${assertion.min}`);
|
|
753
|
+
}
|
|
754
|
+
if (assertion.max != null && count > assertion.max) {
|
|
755
|
+
throw new Error(`Element count ${count} > max ${assertion.max}`);
|
|
756
|
+
}
|
|
757
|
+
if (assertion.equals != null && count !== assertion.equals) {
|
|
758
|
+
throw new Error(`Element count ${count} !== ${assertion.equals}`);
|
|
759
|
+
}
|
|
760
|
+
}
|
|
761
|
+
|
|
762
|
+
else if (type === "toastcontains" || type === "notificationvisible") {
|
|
763
|
+
const v = template(assertion.text || assertion.contains || assertion.value, vars);
|
|
764
|
+
const within = Number(assertion.within || 5000);
|
|
765
|
+
const toastSel = assertion.target || '[role="status"], [role="alert"], .toast, .sonner-toast, [data-sonner-toast], .notification';
|
|
766
|
+
const loc = page.locator(toastSel).filter({ hasText: new RegExp(v, "i") }).first();
|
|
767
|
+
await loc.waitFor({ state: "visible", timeout: within });
|
|
768
|
+
}
|
|
769
|
+
|
|
770
|
+
else if (type === "toastnotcontains") {
|
|
771
|
+
const v = template(assertion.text || assertion.value, vars);
|
|
772
|
+
const within = Number(assertion.within || 3000);
|
|
773
|
+
const toastSel = assertion.target || '[role="status"], [role="alert"], .toast, .sonner-toast, [data-sonner-toast], .notification';
|
|
774
|
+
await page.waitForTimeout(Math.min(within, 1500));
|
|
775
|
+
const count = await page.locator(toastSel).filter({ hasText: new RegExp(v, "i") }).count();
|
|
776
|
+
if (count > 0) throw new Error(`Toast contains forbidden text: ${v}`);
|
|
777
|
+
}
|
|
778
|
+
|
|
779
|
+
else if (type === "networkcalled") {
|
|
780
|
+
const v = template(assertion.match || assertion.value, vars);
|
|
781
|
+
const within = Number(assertion.within || 10000);
|
|
782
|
+
const minTimes = assertion.times ?? assertion.minTimes ?? 1;
|
|
783
|
+
const since = Date.now() - within;
|
|
784
|
+
const matches = telemetry.responses.filter(r => r.ts >= since && r.url.includes(v));
|
|
785
|
+
if (matches.length < minTimes) {
|
|
786
|
+
throw new Error(`Expected ${minTimes} calls matching ${v}, got ${matches.length}`);
|
|
787
|
+
}
|
|
788
|
+
}
|
|
789
|
+
|
|
790
|
+
else if (type === "networkstatus") {
|
|
791
|
+
const v = template(assertion.match || assertion.value, vars);
|
|
792
|
+
const expectStatus = Number(assertion.status || 200);
|
|
793
|
+
const within = Number(assertion.within || 10000);
|
|
794
|
+
const since = Date.now() - within;
|
|
795
|
+
const match = telemetry.responses.find(r => r.ts >= since && r.url.includes(v));
|
|
796
|
+
if (!match) throw new Error(`No response matching: ${v}`);
|
|
797
|
+
if (match.status !== expectStatus) {
|
|
798
|
+
throw new Error(`Expected ${expectStatus}, got ${match.status} for ${match.url}`);
|
|
799
|
+
}
|
|
800
|
+
}
|
|
801
|
+
|
|
802
|
+
else if (type === "networktiming") {
|
|
803
|
+
const v = template(assertion.match || assertion.value, vars);
|
|
804
|
+
const maxDuration = Number(assertion.maxDuration || 5000);
|
|
805
|
+
// Check request/response pairs - simplified version
|
|
806
|
+
const match = telemetry.responses.find(r => r.url.includes(v));
|
|
807
|
+
if (!match) throw new Error(`No response matching: ${v}`);
|
|
808
|
+
// We don't have timing data in this simplified version
|
|
809
|
+
out.message = `Network timing check (simplified): ${v}`;
|
|
810
|
+
}
|
|
811
|
+
|
|
812
|
+
else if (type === "noconsoleerrors") {
|
|
813
|
+
const severity = String(assertion.severity || "error").toLowerCase();
|
|
814
|
+
const windowMs = Number(assertion.within || 60000);
|
|
815
|
+
const since = Date.now() - windowMs;
|
|
816
|
+
|
|
817
|
+
const ignorePatterns = (assertion.ignore || []).map(p => new RegExp(p));
|
|
818
|
+
|
|
819
|
+
const bad = telemetry.console
|
|
820
|
+
.filter(m => m.ts >= since)
|
|
821
|
+
.filter(m => {
|
|
822
|
+
if (severity === "error") return m.type === "error";
|
|
823
|
+
if (severity === "warning" || severity === "warn") return m.type === "error" || m.type === "warning";
|
|
824
|
+
return m.type === "error";
|
|
825
|
+
})
|
|
826
|
+
.filter(m => !ignorePatterns.some(re => re.test(m.text)));
|
|
827
|
+
|
|
828
|
+
if (bad.length > 0) {
|
|
829
|
+
throw new Error(`Console ${bad[0].type}: ${String(bad[0].text || "").slice(0, 140)}`);
|
|
830
|
+
}
|
|
831
|
+
}
|
|
832
|
+
|
|
833
|
+
else if (type === "consolecontains") {
|
|
834
|
+
const match = template(assertion.match || assertion.value, vars);
|
|
835
|
+
const severity = String(assertion.severity || "log").toLowerCase();
|
|
836
|
+
const found = telemetry.console.some(m => {
|
|
837
|
+
if (severity !== "all" && m.type !== severity) return false;
|
|
838
|
+
return m.text.includes(match);
|
|
839
|
+
});
|
|
840
|
+
if (!found) throw new Error(`Console does not contain: ${match}`);
|
|
841
|
+
}
|
|
842
|
+
|
|
843
|
+
else if (type === "localstorage") {
|
|
844
|
+
const key = template(assertion.key, vars);
|
|
845
|
+
const value = await page.evaluate((k) => localStorage.getItem(k), key);
|
|
846
|
+
if (assertion.exists === true && value === null) {
|
|
847
|
+
throw new Error(`localStorage key "${key}" does not exist`);
|
|
848
|
+
}
|
|
849
|
+
if (assertion.exists === false && value !== null) {
|
|
850
|
+
throw new Error(`localStorage key "${key}" should not exist`);
|
|
851
|
+
}
|
|
852
|
+
if (assertion.equals != null && value !== assertion.equals) {
|
|
853
|
+
throw new Error(`localStorage "${key}" !== "${assertion.equals}"`);
|
|
854
|
+
}
|
|
855
|
+
if (assertion.matches) {
|
|
856
|
+
const re = new RegExp(assertion.matches);
|
|
857
|
+
if (!re.test(value || "")) throw new Error(`localStorage "${key}" does not match ${assertion.matches}`);
|
|
858
|
+
}
|
|
859
|
+
}
|
|
860
|
+
|
|
861
|
+
else if (type === "sessionstorage") {
|
|
862
|
+
const key = template(assertion.key, vars);
|
|
863
|
+
const value = await page.evaluate((k) => sessionStorage.getItem(k), key);
|
|
864
|
+
if (assertion.exists === true && value === null) {
|
|
865
|
+
throw new Error(`sessionStorage key "${key}" does not exist`);
|
|
866
|
+
}
|
|
867
|
+
if (assertion.matches) {
|
|
868
|
+
const re = new RegExp(assertion.matches);
|
|
869
|
+
if (!re.test(value || "")) throw new Error(`sessionStorage "${key}" does not match ${assertion.matches}`);
|
|
870
|
+
}
|
|
871
|
+
}
|
|
872
|
+
|
|
873
|
+
else {
|
|
874
|
+
out.status = "skip";
|
|
875
|
+
out.message = `Assertion type not implemented: ${assertion.type}`;
|
|
876
|
+
}
|
|
877
|
+
} catch (e) {
|
|
878
|
+
out.status = "fail";
|
|
879
|
+
out.message = String(e?.message || e).slice(0, 220);
|
|
880
|
+
}
|
|
881
|
+
|
|
882
|
+
return out;
|
|
883
|
+
}
|
|
884
|
+
|
|
885
|
+
// ============================================================================
|
|
886
|
+
// Score Calculation
|
|
887
|
+
// ============================================================================
|
|
888
|
+
|
|
889
|
+
function calculateScore(results) {
|
|
890
|
+
const routesDiscovered = results.coverage?.routesDiscovered || 0;
|
|
891
|
+
const routesWorking = results.coverage?.routesWorking || 0;
|
|
892
|
+
const routePct = routesDiscovered ? (routesWorking / routesDiscovered) : 1;
|
|
893
|
+
|
|
894
|
+
const flows = results.flows || [];
|
|
895
|
+
const flowTotal = flows.length || 1;
|
|
896
|
+
const flowOk = flows.filter(f => f.status === "success").length;
|
|
897
|
+
|
|
898
|
+
let assertsTotal = 0;
|
|
899
|
+
let assertsPass = 0;
|
|
900
|
+
for (const f of flows) {
|
|
901
|
+
for (const a of (f.assertions || [])) {
|
|
902
|
+
if (a.status === "skip") continue;
|
|
903
|
+
assertsTotal++;
|
|
904
|
+
if (a.status === "pass") assertsPass++;
|
|
905
|
+
}
|
|
906
|
+
}
|
|
907
|
+
const assertPct = assertsTotal ? (assertsPass / assertsTotal) : 1;
|
|
908
|
+
|
|
909
|
+
const errors = results.errors?.length || 0;
|
|
910
|
+
const penalty = Math.min(errors * 3, 20);
|
|
911
|
+
|
|
912
|
+
const score = (routePct * 30) + ((flowOk / flowTotal) * 40) + (assertPct * 30) - penalty;
|
|
913
|
+
|
|
914
|
+
return Math.max(0, Math.min(100, Math.round(score)));
|
|
915
|
+
}
|
|
916
|
+
|
|
917
|
+
module.exports = { runFlowPlan, classifyDanger, discoverSurface };
|