@mcoda/integrations 0.1.8 → 0.1.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +3 -0
- package/README.md +3 -1
- package/dist/docdex/DocdexClient.d.ts +19 -4
- package/dist/docdex/DocdexClient.d.ts.map +1 -1
- package/dist/docdex/DocdexClient.js +359 -133
- package/dist/docdex/DocdexRuntime.d.ts +59 -0
- package/dist/docdex/DocdexRuntime.d.ts.map +1 -0
- package/dist/docdex/DocdexRuntime.js +219 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -0
- package/dist/qa/ChromiumQaAdapter.d.ts +59 -1
- package/dist/qa/ChromiumQaAdapter.d.ts.map +1 -1
- package/dist/qa/ChromiumQaAdapter.js +1634 -42
- package/dist/qa/CliQaAdapter.d.ts +2 -0
- package/dist/qa/CliQaAdapter.d.ts.map +1 -1
- package/dist/qa/CliQaAdapter.js +88 -33
- package/dist/qa/QaTypes.d.ts +4 -0
- package/dist/qa/QaTypes.d.ts.map +1 -1
- package/dist/telemetry/TelemetryClient.d.ts +13 -0
- package/dist/telemetry/TelemetryClient.d.ts.map +1 -1
- package/dist/vcs/VcsClient.d.ts +20 -1
- package/dist/vcs/VcsClient.d.ts.map +1 -1
- package/dist/vcs/VcsClient.js +123 -10
- package/package.json +3 -2
|
@@ -1,36 +1,1396 @@
|
|
|
1
1
|
import path from 'node:path';
|
|
2
|
-
import
|
|
3
|
-
import
|
|
2
|
+
import os from 'node:os';
|
|
3
|
+
import net from 'node:net';
|
|
4
|
+
import { spawn } from 'node:child_process';
|
|
5
|
+
import { setTimeout as delay } from 'node:timers/promises';
|
|
4
6
|
import fs from 'node:fs/promises';
|
|
5
|
-
const exec = promisify(execCb);
|
|
6
7
|
const shouldSkipInstall = (ctx) => process.env.MCODA_QA_SKIP_INSTALL === '1' || ctx.env?.MCODA_QA_SKIP_INSTALL === '1';
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
8
|
+
const DOCDEX_CHROMIUM_MISSING_MESSAGE = 'Docdex Chromium not available. Install via docdex or set MCODA_QA_CHROMIUM_PATH.';
|
|
9
|
+
const DEFAULT_BROWSER_TIMEOUT_MS = 15000;
|
|
10
|
+
const DEFAULT_BROWSER_URL = 'about:blank';
|
|
11
|
+
const DOCDEX_STATE_ENV = 'DOCDEX_STATE_DIR';
|
|
12
|
+
const DOCDEX_CONFIG_PATH_ENV = 'DOCDEX_CONFIG_PATH';
|
|
13
|
+
const CHROMIUM_PATH_ENV = 'MCODA_QA_CHROMIUM_PATH';
|
|
14
|
+
const CHROMIUM_HEADLESS_ENV = 'MCODA_QA_CHROMIUM_HEADLESS';
|
|
15
|
+
const CHROMIUM_TIMEOUT_ENV = 'MCODA_QA_CHROMIUM_TIMEOUT_MS';
|
|
16
|
+
const CHROMIUM_USER_AGENT_ENV = 'MCODA_QA_BROWSER_USER_AGENT';
|
|
17
|
+
const CHROMIUM_PROFILE_ENV = 'MCODA_QA_BROWSER_USER_DATA_DIR';
|
|
18
|
+
const CHROMIUM_PROFILE_ENV_ALIAS = 'MCODA_QA_CHROMIUM_USER_DATA_DIR';
|
|
19
|
+
const DOCDEX_WEB_BROWSER_ENV = 'DOCDEX_WEB_BROWSER';
|
|
20
|
+
const DOCDEX_CHROME_PATH_ENV = 'DOCDEX_CHROME_PATH';
|
|
21
|
+
const DOCDEX_BROWSER_PROFILE_ENV = 'DOCDEX_BROWSER_USER_DATA_DIR';
|
|
22
|
+
const DOCDEX_USER_AGENT_ENV = 'DOCDEX_WEB_USER_AGENT';
|
|
23
|
+
const DOCDEX_BROWSER_CONCURRENCY_ENV = 'DOCDEX_WEB_MAX_CONCURRENT_BROWSER_FETCHES';
|
|
24
|
+
const DEFAULT_USER_AGENT = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36';
|
|
25
|
+
const CHROME_WINDOW_SIZE = '1920,1080';
|
|
26
|
+
const CHROME_HEALTH_CHECK_TIMEOUT_MS = 800;
|
|
27
|
+
const CHROME_STARTUP_TIMEOUT_MS = 10000;
|
|
28
|
+
const CHROME_COOKIE_DISMISS_TIMEOUT_MS = 1500;
|
|
29
|
+
const CDP_CONNECT_TIMEOUT_MS = 5000;
|
|
30
|
+
const CDP_CALL_TIMEOUT_MS = 8000;
|
|
31
|
+
const CHROME_THINK_DELAY_MIN_MS = 150;
|
|
32
|
+
const CHROME_THINK_DELAY_MAX_MS = 650;
|
|
33
|
+
const MIN_TEXT_LEN = 80;
|
|
34
|
+
const COOKIE_DISMISS_SCRIPT = `(function () {
|
|
35
|
+
const acceptWords = ["accept", "agree", "allow", "ok", "okay", "got it", "yes"];
|
|
36
|
+
const cookieWords = ["cookie", "cookies", "consent", "gdpr", "privacy", "tracking"];
|
|
37
|
+
const nodes = Array.from(
|
|
38
|
+
document.querySelectorAll("button, a, input[type='button'], input[type='submit']")
|
|
39
|
+
);
|
|
40
|
+
for (const node of nodes) {
|
|
41
|
+
const raw = (node.innerText || node.value || "").trim().toLowerCase();
|
|
42
|
+
if (!raw) continue;
|
|
43
|
+
const hasAccept = acceptWords.some((word) => raw.includes(word));
|
|
44
|
+
const hasCookie = cookieWords.some((word) => raw.includes(word));
|
|
45
|
+
if (hasAccept && (hasCookie || raw.length <= 16)) {
|
|
46
|
+
node.click();
|
|
47
|
+
return true;
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
const selectors = [
|
|
51
|
+
"[id*='cookie']",
|
|
52
|
+
"[class*='cookie']",
|
|
53
|
+
"[id*='consent']",
|
|
54
|
+
"[class*='consent']",
|
|
55
|
+
"[aria-label*='cookie']",
|
|
56
|
+
"[aria-label*='consent']",
|
|
57
|
+
"[data-testid*='cookie']",
|
|
58
|
+
"[data-testid*='consent']",
|
|
59
|
+
];
|
|
60
|
+
let removed = false;
|
|
61
|
+
for (const selector of selectors) {
|
|
62
|
+
document.querySelectorAll(selector).forEach((el) => {
|
|
63
|
+
el.remove();
|
|
64
|
+
removed = true;
|
|
65
|
+
});
|
|
66
|
+
}
|
|
67
|
+
return removed;
|
|
68
|
+
})()`;
|
|
69
|
+
const WEBDRIVER_OVERRIDE_SCRIPT = "Object.defineProperty(navigator, 'webdriver', { get: () => undefined });";
|
|
70
|
+
const readJsonFile = async (filePath) => {
|
|
71
|
+
try {
|
|
72
|
+
const raw = await fs.readFile(filePath, 'utf8');
|
|
73
|
+
return JSON.parse(raw);
|
|
74
|
+
}
|
|
75
|
+
catch {
|
|
76
|
+
return undefined;
|
|
77
|
+
}
|
|
78
|
+
};
|
|
79
|
+
const readTextFile = async (filePath) => {
|
|
80
|
+
try {
|
|
81
|
+
return await fs.readFile(filePath, 'utf8');
|
|
82
|
+
}
|
|
83
|
+
catch {
|
|
84
|
+
return undefined;
|
|
85
|
+
}
|
|
86
|
+
};
|
|
87
|
+
const fileExists = async (filePath) => {
|
|
88
|
+
try {
|
|
89
|
+
const stat = await fs.stat(filePath);
|
|
90
|
+
return stat.isFile();
|
|
91
|
+
}
|
|
92
|
+
catch {
|
|
93
|
+
return false;
|
|
94
|
+
}
|
|
95
|
+
};
|
|
96
|
+
const resolveDocdexStateDir = () => {
|
|
97
|
+
const envDir = process.env[DOCDEX_STATE_ENV];
|
|
98
|
+
if (envDir && envDir.trim())
|
|
99
|
+
return envDir.trim();
|
|
100
|
+
return path.join(os.homedir(), '.docdex', 'state');
|
|
101
|
+
};
|
|
102
|
+
const resolveDocdexConfigPath = () => {
|
|
103
|
+
const envPath = process.env[DOCDEX_CONFIG_PATH_ENV];
|
|
104
|
+
if (envPath && envPath.trim())
|
|
105
|
+
return envPath.trim();
|
|
106
|
+
const stateDir = resolveDocdexStateDir();
|
|
107
|
+
const baseDir = path.dirname(stateDir);
|
|
108
|
+
return path.join(baseDir, 'config.toml');
|
|
109
|
+
};
|
|
110
|
+
const stripTomlInlineComment = (value) => {
|
|
111
|
+
let inSingle = false;
|
|
112
|
+
let inDouble = false;
|
|
113
|
+
for (let index = 0; index < value.length; index += 1) {
|
|
114
|
+
const char = value[index];
|
|
115
|
+
if (char === "'" && !inDouble) {
|
|
116
|
+
inSingle = !inSingle;
|
|
117
|
+
continue;
|
|
118
|
+
}
|
|
119
|
+
if (char === '"' && !inSingle) {
|
|
120
|
+
const prev = value[index - 1];
|
|
121
|
+
if (prev !== '\\') {
|
|
122
|
+
inDouble = !inDouble;
|
|
123
|
+
}
|
|
124
|
+
continue;
|
|
125
|
+
}
|
|
126
|
+
if (char === '#' && !inSingle && !inDouble) {
|
|
127
|
+
return value.slice(0, index).trim();
|
|
13
128
|
}
|
|
14
|
-
return ctx.workspaceRoot;
|
|
15
129
|
}
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
130
|
+
return value.trim();
|
|
131
|
+
};
|
|
132
|
+
const parseTomlString = (value) => {
|
|
133
|
+
const trimmed = value.trim();
|
|
134
|
+
if (!trimmed)
|
|
135
|
+
return undefined;
|
|
136
|
+
if (trimmed.startsWith('"') && trimmed.endsWith('"')) {
|
|
137
|
+
try {
|
|
138
|
+
return JSON.parse(trimmed);
|
|
139
|
+
}
|
|
140
|
+
catch {
|
|
141
|
+
return trimmed.slice(1, -1);
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
if (trimmed.startsWith("'") && trimmed.endsWith("'")) {
|
|
145
|
+
return trimmed.slice(1, -1);
|
|
146
|
+
}
|
|
147
|
+
return undefined;
|
|
148
|
+
};
|
|
149
|
+
const parseTomlSection = (raw, section) => {
|
|
150
|
+
const lines = raw.split(/\r?\n/);
|
|
151
|
+
let inSection = false;
|
|
152
|
+
const result = {};
|
|
153
|
+
for (const line of lines) {
|
|
154
|
+
const trimmed = line.trim();
|
|
155
|
+
if (!trimmed || trimmed.startsWith('#'))
|
|
156
|
+
continue;
|
|
157
|
+
const sectionMatch = /^\[([^\]]+)\]$/.exec(trimmed);
|
|
158
|
+
if (sectionMatch) {
|
|
159
|
+
inSection = sectionMatch[1].trim() === section;
|
|
160
|
+
continue;
|
|
161
|
+
}
|
|
162
|
+
if (!inSection)
|
|
163
|
+
continue;
|
|
164
|
+
const match = /^([A-Za-z0-9_.-]+)\s*=\s*(.+)$/.exec(trimmed);
|
|
165
|
+
if (!match)
|
|
166
|
+
continue;
|
|
167
|
+
const key = match[1];
|
|
168
|
+
const rawValue = stripTomlInlineComment(match[2]);
|
|
169
|
+
const parsed = parseTomlString(rawValue);
|
|
170
|
+
if (parsed !== undefined)
|
|
171
|
+
result[key] = parsed;
|
|
172
|
+
}
|
|
173
|
+
return result;
|
|
174
|
+
};
|
|
175
|
+
const readDocdexScraperConfig = async () => {
|
|
176
|
+
const configPath = resolveDocdexConfigPath();
|
|
177
|
+
if (!configPath)
|
|
178
|
+
return {};
|
|
179
|
+
const raw = await readTextFile(configPath);
|
|
180
|
+
if (!raw)
|
|
181
|
+
return {};
|
|
182
|
+
const section = parseTomlSection(raw, 'web.scraper');
|
|
183
|
+
return {
|
|
184
|
+
chromeBinaryPath: section.chrome_binary_path,
|
|
185
|
+
userDataDir: section.user_data_dir,
|
|
186
|
+
};
|
|
187
|
+
};
|
|
188
|
+
const resolveDocdexBrowserProfileDir = () => path.join(resolveDocdexStateDir(), 'browser_profiles', 'chrome');
|
|
189
|
+
const resolveBinaryFromPath = async (command) => {
|
|
190
|
+
const pathValue = process.env.PATH;
|
|
191
|
+
if (!pathValue)
|
|
192
|
+
return undefined;
|
|
193
|
+
const extensions = process.platform === 'win32'
|
|
194
|
+
? [''].concat((process.env.PATHEXT ?? '.EXE;.CMD;.BAT;.COM')
|
|
195
|
+
.split(';')
|
|
196
|
+
.filter(Boolean)
|
|
197
|
+
.map((ext) => {
|
|
198
|
+
const normalized = ext.startsWith('.') ? ext : `.${ext}`;
|
|
199
|
+
return normalized.toLowerCase();
|
|
200
|
+
}))
|
|
201
|
+
: [''];
|
|
202
|
+
for (const base of pathValue.split(path.delimiter)) {
|
|
203
|
+
if (!base)
|
|
204
|
+
continue;
|
|
205
|
+
for (const ext of extensions) {
|
|
206
|
+
const candidate = path.join(base, `${command}${ext}`);
|
|
207
|
+
if (await fileExists(candidate))
|
|
208
|
+
return candidate;
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
return undefined;
|
|
212
|
+
};
|
|
213
|
+
const resolveBinaryHint = async (hint) => {
|
|
214
|
+
if (!hint)
|
|
215
|
+
return undefined;
|
|
216
|
+
const trimmed = hint.trim();
|
|
217
|
+
if (!trimmed)
|
|
218
|
+
return undefined;
|
|
219
|
+
if (await fileExists(trimmed))
|
|
220
|
+
return trimmed;
|
|
221
|
+
if (trimmed.includes('/') || trimmed.includes('\\'))
|
|
222
|
+
return undefined;
|
|
223
|
+
return await resolveBinaryFromPath(trimmed);
|
|
224
|
+
};
|
|
225
|
+
const resolveKnownChromiumPath = async () => {
|
|
226
|
+
if (process.platform === 'darwin') {
|
|
227
|
+
const candidate = '/Applications/Chromium.app/Contents/MacOS/Chromium';
|
|
228
|
+
return (await fileExists(candidate)) ? candidate : undefined;
|
|
229
|
+
}
|
|
230
|
+
if (process.platform === 'win32') {
|
|
231
|
+
const bases = [
|
|
232
|
+
process.env.PROGRAMFILES,
|
|
233
|
+
process.env['PROGRAMFILES(X86)'],
|
|
234
|
+
process.env.LOCALAPPDATA,
|
|
235
|
+
].filter(Boolean);
|
|
236
|
+
for (const base of bases) {
|
|
237
|
+
const candidate = path.join(base, 'Chromium', 'Application', 'chrome.exe');
|
|
238
|
+
if (await fileExists(candidate))
|
|
239
|
+
return candidate;
|
|
240
|
+
}
|
|
241
|
+
return undefined;
|
|
242
|
+
}
|
|
243
|
+
const linuxCandidates = ['/usr/bin/chromium', '/usr/bin/chromium-browser', '/snap/bin/chromium'];
|
|
244
|
+
for (const candidate of linuxCandidates) {
|
|
245
|
+
if (await fileExists(candidate))
|
|
246
|
+
return candidate;
|
|
247
|
+
}
|
|
248
|
+
return undefined;
|
|
249
|
+
};
|
|
250
|
+
const resolveDocdexChromiumBinary = async () => {
|
|
251
|
+
const overrides = [
|
|
252
|
+
process.env[CHROMIUM_PATH_ENV],
|
|
253
|
+
process.env[DOCDEX_WEB_BROWSER_ENV],
|
|
254
|
+
process.env[DOCDEX_CHROME_PATH_ENV],
|
|
255
|
+
process.env.CHROME_PATH,
|
|
256
|
+
];
|
|
257
|
+
for (const override of overrides) {
|
|
258
|
+
const resolved = await resolveBinaryHint(override);
|
|
259
|
+
if (resolved)
|
|
260
|
+
return resolved;
|
|
261
|
+
}
|
|
262
|
+
const config = await readDocdexScraperConfig();
|
|
263
|
+
if (config.chromeBinaryPath && (await fileExists(config.chromeBinaryPath))) {
|
|
264
|
+
return config.chromeBinaryPath;
|
|
265
|
+
}
|
|
266
|
+
const manifestPath = path.join(resolveDocdexStateDir(), 'bin', 'chromium', 'manifest.json');
|
|
267
|
+
const manifest = await readJsonFile(manifestPath);
|
|
268
|
+
if (manifest?.path && (await fileExists(manifest.path))) {
|
|
269
|
+
return manifest.path;
|
|
270
|
+
}
|
|
271
|
+
const whichCandidate = (await resolveBinaryFromPath('chromium')) ?? (await resolveBinaryFromPath('chromium-browser'));
|
|
272
|
+
if (whichCandidate)
|
|
273
|
+
return whichCandidate;
|
|
274
|
+
return await resolveKnownChromiumPath();
|
|
275
|
+
};
|
|
276
|
+
export const resolveChromiumBinary = async () => resolveDocdexChromiumBinary();
|
|
277
|
+
const isUrl = (value) => {
|
|
278
|
+
if (!value)
|
|
279
|
+
return false;
|
|
280
|
+
try {
|
|
281
|
+
const parsed = new URL(value);
|
|
282
|
+
return parsed.protocol === 'http:' || parsed.protocol === 'https:';
|
|
283
|
+
}
|
|
284
|
+
catch {
|
|
285
|
+
return false;
|
|
286
|
+
}
|
|
287
|
+
};
|
|
288
|
+
const parseBoolean = (value) => {
|
|
289
|
+
if (!value)
|
|
290
|
+
return undefined;
|
|
291
|
+
const normalized = value.trim().toLowerCase();
|
|
292
|
+
if (['1', 'true', 'yes', 'y', 'on'].includes(normalized))
|
|
293
|
+
return true;
|
|
294
|
+
if (['0', 'false', 'no', 'n', 'off'].includes(normalized))
|
|
295
|
+
return false;
|
|
296
|
+
return undefined;
|
|
297
|
+
};
|
|
298
|
+
const resolveHeadless = (ctx) => {
|
|
299
|
+
const envValue = ctx.env?.[CHROMIUM_HEADLESS_ENV] ?? process.env[CHROMIUM_HEADLESS_ENV];
|
|
300
|
+
const parsed = parseBoolean(envValue);
|
|
301
|
+
return parsed ?? true;
|
|
302
|
+
};
|
|
303
|
+
const resolveTimeoutMs = (ctx) => {
|
|
304
|
+
const raw = ctx.env?.[CHROMIUM_TIMEOUT_ENV] ?? process.env[CHROMIUM_TIMEOUT_ENV];
|
|
305
|
+
const parsed = raw ? Number(raw) : NaN;
|
|
306
|
+
return Number.isFinite(parsed) ? parsed : DEFAULT_BROWSER_TIMEOUT_MS;
|
|
307
|
+
};
|
|
308
|
+
const resolveUserAgent = (ctx) => {
|
|
309
|
+
const envAgent = ctx.env?.[CHROMIUM_USER_AGENT_ENV] ??
|
|
310
|
+
process.env[CHROMIUM_USER_AGENT_ENV] ??
|
|
311
|
+
process.env[DOCDEX_USER_AGENT_ENV];
|
|
312
|
+
return envAgent?.trim() || DEFAULT_USER_AGENT;
|
|
313
|
+
};
|
|
314
|
+
const resolveUserDataDir = async (ctx) => {
|
|
315
|
+
const envPath = ctx.env?.[CHROMIUM_PROFILE_ENV] ??
|
|
316
|
+
ctx.env?.[CHROMIUM_PROFILE_ENV_ALIAS] ??
|
|
317
|
+
process.env[CHROMIUM_PROFILE_ENV] ??
|
|
318
|
+
process.env[CHROMIUM_PROFILE_ENV_ALIAS] ??
|
|
319
|
+
process.env[DOCDEX_BROWSER_PROFILE_ENV];
|
|
320
|
+
if (envPath && envPath.trim()) {
|
|
321
|
+
const resolved = envPath.trim();
|
|
322
|
+
await fs.mkdir(resolved, { recursive: true });
|
|
323
|
+
return { path: resolved, cleanup: null };
|
|
324
|
+
}
|
|
325
|
+
const config = await readDocdexScraperConfig();
|
|
326
|
+
if (config.userDataDir && config.userDataDir.trim()) {
|
|
327
|
+
const resolved = config.userDataDir.trim();
|
|
328
|
+
await fs.mkdir(resolved, { recursive: true });
|
|
329
|
+
return { path: resolved, cleanup: null };
|
|
330
|
+
}
|
|
331
|
+
const defaultPath = resolveDocdexBrowserProfileDir();
|
|
332
|
+
await fs.mkdir(defaultPath, { recursive: true });
|
|
333
|
+
return { path: defaultPath, cleanup: null };
|
|
334
|
+
};
|
|
335
|
+
const createTempUserDataDir = async () => {
|
|
336
|
+
const tempBase = path.join(os.tmpdir(), 'mcoda-qa-chrome-');
|
|
337
|
+
const tempDir = await fs.mkdtemp(tempBase);
|
|
338
|
+
return {
|
|
339
|
+
path: tempDir,
|
|
340
|
+
cleanup: async () => {
|
|
341
|
+
await fs.rm(tempDir, { recursive: true, force: true });
|
|
342
|
+
},
|
|
343
|
+
};
|
|
344
|
+
};
|
|
345
|
+
const resolveBrowserTarget = (profile, ctx) => {
|
|
346
|
+
const override = ctx.browserBaseUrl ?? ctx.testCommandOverride ?? profile.test_command;
|
|
347
|
+
if (override && isUrl(override))
|
|
348
|
+
return override;
|
|
349
|
+
return undefined;
|
|
350
|
+
};
|
|
351
|
+
const randomDelayMs = (minMs, maxMs) => {
|
|
352
|
+
if (maxMs <= minMs)
|
|
353
|
+
return minMs;
|
|
354
|
+
const span = maxMs - minMs;
|
|
355
|
+
return minMs + Math.floor(Math.random() * (span + 1));
|
|
356
|
+
};
|
|
357
|
+
const resolveChromeFetchConcurrency = () => {
|
|
358
|
+
const raw = process.env[DOCDEX_BROWSER_CONCURRENCY_ENV];
|
|
359
|
+
const parsed = raw ? Number(raw) : NaN;
|
|
360
|
+
if (Number.isFinite(parsed) && parsed > 0)
|
|
361
|
+
return Math.floor(parsed);
|
|
362
|
+
return 1;
|
|
363
|
+
};
|
|
364
|
+
class Semaphore {
|
|
365
|
+
constructor(capacity) {
|
|
366
|
+
this.queue = [];
|
|
367
|
+
this.available = Math.max(1, Math.floor(capacity));
|
|
368
|
+
}
|
|
369
|
+
async acquire() {
|
|
370
|
+
if (this.available > 0) {
|
|
371
|
+
this.available -= 1;
|
|
372
|
+
return () => this.release();
|
|
373
|
+
}
|
|
374
|
+
return await new Promise((resolve) => {
|
|
375
|
+
this.queue.push(() => resolve(() => this.release()));
|
|
376
|
+
});
|
|
377
|
+
}
|
|
378
|
+
release() {
|
|
379
|
+
this.available += 1;
|
|
380
|
+
const next = this.queue.shift();
|
|
381
|
+
if (next) {
|
|
382
|
+
this.available -= 1;
|
|
383
|
+
next();
|
|
384
|
+
}
|
|
385
|
+
}
|
|
386
|
+
}
|
|
387
|
+
const chromeFetchSemaphore = new Semaphore(resolveChromeFetchConcurrency());
|
|
388
|
+
class ChromeInstance {
|
|
389
|
+
constructor(child, debugPort, config, ownsProcess) {
|
|
390
|
+
this.child = child;
|
|
391
|
+
this.debugPort = debugPort;
|
|
392
|
+
this.config = config;
|
|
393
|
+
this.ownsProcess = ownsProcess;
|
|
394
|
+
}
|
|
395
|
+
static async spawn(config, env) {
|
|
396
|
+
const debugPort = await pickFreePort();
|
|
397
|
+
await clearDevtoolsPort(config.userDataDir.path);
|
|
398
|
+
const launchContext = {
|
|
399
|
+
chromeBinary: config.chromeBinary,
|
|
400
|
+
headless: config.headless,
|
|
401
|
+
userAgent: config.userAgent,
|
|
402
|
+
userDataDir: config.userDataDir.path,
|
|
403
|
+
debugPort,
|
|
404
|
+
};
|
|
405
|
+
const args = chromeCommonArgs(launchContext);
|
|
406
|
+
args.push('about:blank');
|
|
407
|
+
const child = spawn(config.chromeBinary, args, {
|
|
408
|
+
detached: process.platform !== 'win32',
|
|
409
|
+
stdio: 'ignore',
|
|
410
|
+
env: { ...process.env, ...env },
|
|
411
|
+
});
|
|
412
|
+
child.unref();
|
|
413
|
+
try {
|
|
414
|
+
await waitForCdpReady(debugPort, CHROME_STARTUP_TIMEOUT_MS);
|
|
415
|
+
}
|
|
416
|
+
catch (err) {
|
|
417
|
+
await terminateProcessTree(child);
|
|
418
|
+
throw err;
|
|
419
|
+
}
|
|
420
|
+
return new ChromeInstance(child, debugPort, config, true);
|
|
421
|
+
}
|
|
422
|
+
static attachExisting(config, debugPort) {
|
|
423
|
+
return new ChromeInstance(null, debugPort, config, false);
|
|
424
|
+
}
|
|
425
|
+
matches(config) {
|
|
426
|
+
return (this.config.chromeBinary === config.chromeBinary &&
|
|
427
|
+
this.config.headless === config.headless &&
|
|
428
|
+
this.config.userAgent === config.userAgent &&
|
|
429
|
+
this.config.userDataDir.path === config.userDataDir.path);
|
|
430
|
+
}
|
|
431
|
+
async isHealthy() {
|
|
432
|
+
if (this.child && this.child.exitCode !== null)
|
|
433
|
+
return false;
|
|
434
|
+
return await probeCdp(this.debugPort);
|
|
435
|
+
}
|
|
436
|
+
getDebugPort() {
|
|
437
|
+
return this.debugPort;
|
|
438
|
+
}
|
|
439
|
+
async fetchDom(url, timeoutMs) {
|
|
440
|
+
const target = await createCdpTarget(this.debugPort, CHROME_STARTUP_TIMEOUT_MS);
|
|
441
|
+
try {
|
|
442
|
+
return await fetchDomViaCdp(target.wsUrl, url, timeoutMs);
|
|
443
|
+
}
|
|
444
|
+
finally {
|
|
445
|
+
if (target.targetId) {
|
|
446
|
+
try {
|
|
447
|
+
await closeCdpTarget(this.debugPort, target.targetId);
|
|
448
|
+
}
|
|
449
|
+
catch {
|
|
450
|
+
// ignore
|
|
451
|
+
}
|
|
452
|
+
}
|
|
453
|
+
}
|
|
454
|
+
}
|
|
455
|
+
async shutdown() {
|
|
456
|
+
if (this.ownsProcess && this.child) {
|
|
457
|
+
await terminateProcessTree(this.child);
|
|
458
|
+
}
|
|
459
|
+
if (this.ownsProcess && this.config.userDataDir.cleanup) {
|
|
460
|
+
try {
|
|
461
|
+
await this.config.userDataDir.cleanup();
|
|
462
|
+
}
|
|
463
|
+
catch {
|
|
464
|
+
// ignore
|
|
465
|
+
}
|
|
466
|
+
}
|
|
467
|
+
}
|
|
468
|
+
}
|
|
469
|
+
class ChromeManager {
|
|
470
|
+
async getOrLaunch(config, env) {
|
|
471
|
+
const effectiveConfig = this.fallbackUserDataDir
|
|
472
|
+
? { ...config, userDataDir: this.fallbackUserDataDir }
|
|
473
|
+
: config;
|
|
474
|
+
if (this.instance) {
|
|
475
|
+
if (this.instance.matches(effectiveConfig) && (await this.instance.isHealthy())) {
|
|
476
|
+
return this.instance;
|
|
477
|
+
}
|
|
478
|
+
await this.resetIfCurrent(this.instance);
|
|
479
|
+
}
|
|
480
|
+
if (this.launching) {
|
|
481
|
+
const instance = await this.launching;
|
|
482
|
+
if (instance.matches(effectiveConfig) && (await instance.isHealthy())) {
|
|
483
|
+
this.instance = instance;
|
|
484
|
+
return instance;
|
|
485
|
+
}
|
|
486
|
+
}
|
|
487
|
+
const existingPort = await readDevtoolsPort(effectiveConfig.userDataDir.path);
|
|
488
|
+
if (existingPort && (await probeCdp(existingPort))) {
|
|
489
|
+
const existing = ChromeInstance.attachExisting(effectiveConfig, existingPort);
|
|
490
|
+
this.instance = existing;
|
|
491
|
+
return existing;
|
|
492
|
+
}
|
|
493
|
+
const spawnInstance = async (targetConfig) => {
|
|
494
|
+
this.launching = ChromeInstance.spawn(targetConfig, env);
|
|
495
|
+
try {
|
|
496
|
+
const instance = await this.launching;
|
|
497
|
+
this.instance = instance;
|
|
498
|
+
return instance;
|
|
499
|
+
}
|
|
500
|
+
finally {
|
|
501
|
+
this.launching = undefined;
|
|
502
|
+
}
|
|
503
|
+
};
|
|
504
|
+
try {
|
|
505
|
+
return await spawnInstance(effectiveConfig);
|
|
506
|
+
}
|
|
507
|
+
catch (err) {
|
|
508
|
+
if (!this.fallbackUserDataDir && !effectiveConfig.userDataDir.cleanup) {
|
|
509
|
+
const tempUserDataDir = await createTempUserDataDir();
|
|
510
|
+
this.fallbackUserDataDir = tempUserDataDir;
|
|
511
|
+
return await spawnInstance({ ...config, userDataDir: tempUserDataDir });
|
|
512
|
+
}
|
|
513
|
+
throw err;
|
|
514
|
+
}
|
|
515
|
+
}
|
|
516
|
+
async resetIfCurrent(instance) {
|
|
517
|
+
if (this.instance !== instance)
|
|
518
|
+
return false;
|
|
519
|
+
this.instance = undefined;
|
|
520
|
+
await instance.shutdown();
|
|
521
|
+
return true;
|
|
522
|
+
}
|
|
523
|
+
async resetIfUnhealthy(instance) {
|
|
524
|
+
if (await instance.isHealthy())
|
|
525
|
+
return false;
|
|
526
|
+
return await this.resetIfCurrent(instance);
|
|
527
|
+
}
|
|
528
|
+
}
|
|
529
|
+
const chromeManager = new ChromeManager();
|
|
530
|
+
const chromeCommonArgs = (ctx) => {
|
|
531
|
+
const args = [];
|
|
532
|
+
if (ctx.headless)
|
|
533
|
+
args.push('--headless=new');
|
|
534
|
+
args.push('--disable-gpu');
|
|
535
|
+
args.push('--disable-extensions');
|
|
536
|
+
args.push('--disable-dev-shm-usage');
|
|
537
|
+
args.push('--disable-blink-features=AutomationControlled');
|
|
538
|
+
args.push('--no-sandbox');
|
|
539
|
+
args.push('--no-first-run');
|
|
540
|
+
args.push('--no-default-browser-check');
|
|
541
|
+
args.push('--remote-allow-origins=*');
|
|
542
|
+
args.push(`--window-size=${CHROME_WINDOW_SIZE}`);
|
|
543
|
+
args.push(`--user-data-dir=${ctx.userDataDir}`);
|
|
544
|
+
args.push('--disable-background-timer-throttling');
|
|
545
|
+
args.push('--disable-backgrounding-occluded-windows');
|
|
546
|
+
args.push('--disable-renderer-backgrounding');
|
|
547
|
+
args.push('--run-all-compositor-stages-before-draw');
|
|
548
|
+
args.push(`--user-agent=${ctx.userAgent}`);
|
|
549
|
+
args.push('--remote-debugging-address=127.0.0.1');
|
|
550
|
+
args.push(`--remote-debugging-port=${ctx.debugPort}`);
|
|
551
|
+
return args;
|
|
552
|
+
};
|
|
553
|
+
const clearDevtoolsPort = async (userDataDir) => {
|
|
554
|
+
const portFile = path.join(userDataDir, 'DevToolsActivePort');
|
|
555
|
+
try {
|
|
556
|
+
await fs.rm(portFile, { force: true });
|
|
557
|
+
}
|
|
558
|
+
catch {
|
|
559
|
+
// ignore
|
|
560
|
+
}
|
|
561
|
+
};
|
|
562
|
+
const readDevtoolsPort = async (userDataDir) => {
|
|
563
|
+
const portFile = path.join(userDataDir, 'DevToolsActivePort');
|
|
564
|
+
try {
|
|
565
|
+
const raw = await fs.readFile(portFile, 'utf8');
|
|
566
|
+
const [portLine] = raw.trim().split(/\s+/);
|
|
567
|
+
const port = Number(portLine);
|
|
568
|
+
if (Number.isFinite(port) && port > 0)
|
|
569
|
+
return port;
|
|
570
|
+
}
|
|
571
|
+
catch {
|
|
572
|
+
// ignore
|
|
573
|
+
}
|
|
574
|
+
return undefined;
|
|
575
|
+
};
|
|
576
|
+
const pickFreePort = async () => await new Promise((resolve, reject) => {
|
|
577
|
+
const server = net.createServer();
|
|
578
|
+
server.unref();
|
|
579
|
+
server.on('error', reject);
|
|
580
|
+
server.listen(0, '127.0.0.1', () => {
|
|
581
|
+
const address = server.address();
|
|
582
|
+
if (typeof address === 'object' && address?.port) {
|
|
583
|
+
const port = address.port;
|
|
584
|
+
server.close(() => resolve(port));
|
|
585
|
+
}
|
|
586
|
+
else {
|
|
587
|
+
server.close(() => reject(new Error('Failed to acquire free port')));
|
|
588
|
+
}
|
|
589
|
+
});
|
|
590
|
+
});
|
|
591
|
+
const fetchWithTimeout = async (url, timeoutMs, init) => {
|
|
592
|
+
const controller = new AbortController();
|
|
593
|
+
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
|
594
|
+
try {
|
|
595
|
+
return await fetch(url, { signal: controller.signal, ...init });
|
|
596
|
+
}
|
|
597
|
+
finally {
|
|
598
|
+
clearTimeout(timer);
|
|
599
|
+
}
|
|
600
|
+
};
|
|
601
|
+
const probeCdp = async (port) => {
|
|
602
|
+
try {
|
|
603
|
+
const resp = await fetchWithTimeout(`http://127.0.0.1:${port}/json/version`, CHROME_HEALTH_CHECK_TIMEOUT_MS);
|
|
604
|
+
return resp.ok;
|
|
605
|
+
}
|
|
606
|
+
catch {
|
|
607
|
+
return false;
|
|
608
|
+
}
|
|
609
|
+
};
|
|
610
|
+
const waitForCdpReady = async (port, timeoutMs) => {
|
|
611
|
+
const start = Date.now();
|
|
612
|
+
while (Date.now() - start < timeoutMs) {
|
|
613
|
+
if (await probeCdp(port))
|
|
614
|
+
return;
|
|
615
|
+
await delay(100);
|
|
616
|
+
}
|
|
617
|
+
throw new Error(`devtools endpoint not available within ${timeoutMs}ms`);
|
|
618
|
+
};
|
|
619
|
+
const extractCdpTarget = (value) => {
|
|
620
|
+
if (value && typeof value === 'object' && value.webSocketDebuggerUrl) {
|
|
621
|
+
return { wsUrl: String(value.webSocketDebuggerUrl), targetId: value.id ? String(value.id) : undefined };
|
|
622
|
+
}
|
|
623
|
+
if (Array.isArray(value)) {
|
|
624
|
+
for (const item of value) {
|
|
625
|
+
if (item && item.webSocketDebuggerUrl) {
|
|
626
|
+
return { wsUrl: String(item.webSocketDebuggerUrl), targetId: item.id ? String(item.id) : undefined };
|
|
627
|
+
}
|
|
628
|
+
}
|
|
629
|
+
}
|
|
630
|
+
return undefined;
|
|
631
|
+
};
|
|
632
|
+
const fetchDevtoolsTarget = async (endpoint, method = 'GET') => {
|
|
633
|
+
const resp = await fetchWithTimeout(endpoint, CHROME_HEALTH_CHECK_TIMEOUT_MS, { method });
|
|
634
|
+
if (!resp.ok) {
|
|
635
|
+
throw new Error(`devtools endpoint ${endpoint} failed with status ${resp.status}`);
|
|
636
|
+
}
|
|
637
|
+
const body = await resp.text();
|
|
638
|
+
try {
|
|
639
|
+
const value = JSON.parse(body);
|
|
640
|
+
return extractCdpTarget(value);
|
|
641
|
+
}
|
|
642
|
+
catch {
|
|
643
|
+
return undefined;
|
|
644
|
+
}
|
|
645
|
+
};
|
|
646
|
+
const createCdpTarget = async (port, timeoutMs) => {
|
|
647
|
+
const endpointNew = `http://127.0.0.1:${port}/json/new`;
|
|
648
|
+
const endpointList = `http://127.0.0.1:${port}/json/list`;
|
|
649
|
+
const start = Date.now();
|
|
650
|
+
let lastError;
|
|
651
|
+
while (Date.now() - start < timeoutMs) {
|
|
652
|
+
try {
|
|
653
|
+
const target = await fetchDevtoolsTarget(endpointNew, 'PUT');
|
|
654
|
+
if (target)
|
|
655
|
+
return target;
|
|
656
|
+
}
|
|
657
|
+
catch (err) {
|
|
658
|
+
lastError = err;
|
|
659
|
+
}
|
|
660
|
+
try {
|
|
661
|
+
const target = await fetchDevtoolsTarget(endpointList);
|
|
662
|
+
if (target)
|
|
663
|
+
return target;
|
|
664
|
+
}
|
|
665
|
+
catch (err) {
|
|
666
|
+
lastError = err;
|
|
667
|
+
}
|
|
668
|
+
await delay(100);
|
|
669
|
+
}
|
|
670
|
+
if (lastError)
|
|
671
|
+
throw lastError;
|
|
672
|
+
throw new Error(`devtools websocket not available within ${timeoutMs}ms`);
|
|
673
|
+
};
|
|
674
|
+
const closeCdpTarget = async (port, targetId) => {
|
|
675
|
+
const endpoint = `http://127.0.0.1:${port}/json/close/${targetId}`;
|
|
676
|
+
const resp = await fetchWithTimeout(endpoint, CHROME_HEALTH_CHECK_TIMEOUT_MS);
|
|
677
|
+
if (!resp.ok) {
|
|
678
|
+
throw new Error(`devtools close target ${targetId} failed with status ${resp.status}`);
|
|
679
|
+
}
|
|
680
|
+
};
|
|
681
|
+
class MessageQueue {
|
|
682
|
+
constructor() {
|
|
683
|
+
this.queue = [];
|
|
684
|
+
this.resolvers = [];
|
|
685
|
+
}
|
|
686
|
+
push(value) {
|
|
687
|
+
const resolver = this.resolvers.shift();
|
|
688
|
+
if (resolver) {
|
|
689
|
+
resolver(value);
|
|
690
|
+
return;
|
|
691
|
+
}
|
|
692
|
+
this.queue.push(value);
|
|
693
|
+
}
|
|
694
|
+
async next(timeoutMs) {
|
|
695
|
+
if (this.queue.length)
|
|
696
|
+
return this.queue.shift();
|
|
697
|
+
return await new Promise((resolve) => {
|
|
698
|
+
const timer = typeof timeoutMs === 'number'
|
|
699
|
+
? setTimeout(() => {
|
|
700
|
+
this.resolvers = this.resolvers.filter((r) => r !== resolve);
|
|
701
|
+
resolve(undefined);
|
|
702
|
+
}, timeoutMs)
|
|
703
|
+
: undefined;
|
|
704
|
+
this.resolvers.push((value) => {
|
|
705
|
+
if (timer)
|
|
706
|
+
clearTimeout(timer);
|
|
707
|
+
resolve(value);
|
|
708
|
+
});
|
|
709
|
+
});
|
|
710
|
+
}
|
|
711
|
+
}
|
|
712
|
+
class NetworkIdleTracker {
|
|
713
|
+
constructor() {
|
|
714
|
+
this.inflight = 0;
|
|
715
|
+
this.lastActivity = Date.now();
|
|
716
|
+
this.sawLoad = false;
|
|
717
|
+
}
|
|
718
|
+
handle(method, params) {
|
|
719
|
+
switch (method) {
|
|
720
|
+
case 'Network.requestWillBeSent':
|
|
721
|
+
this.inflight += 1;
|
|
722
|
+
this.lastActivity = Date.now();
|
|
723
|
+
break;
|
|
724
|
+
case 'Network.loadingFinished':
|
|
725
|
+
case 'Network.loadingFailed':
|
|
726
|
+
this.inflight = Math.max(0, this.inflight - 1);
|
|
727
|
+
this.lastActivity = Date.now();
|
|
728
|
+
break;
|
|
729
|
+
case 'Network.responseReceived': {
|
|
730
|
+
const resourceType = params?.type;
|
|
731
|
+
if (resourceType === 'Document') {
|
|
732
|
+
const status = params?.response?.status;
|
|
733
|
+
const url = params?.response?.url;
|
|
734
|
+
if (typeof status === 'number')
|
|
735
|
+
this.documentStatus = status;
|
|
736
|
+
if (typeof url === 'string')
|
|
737
|
+
this.documentUrl = url;
|
|
738
|
+
}
|
|
739
|
+
this.lastActivity = Date.now();
|
|
740
|
+
break;
|
|
741
|
+
}
|
|
742
|
+
case 'Page.loadEventFired':
|
|
743
|
+
this.sawLoad = true;
|
|
744
|
+
this.lastActivity = Date.now();
|
|
745
|
+
break;
|
|
746
|
+
default:
|
|
747
|
+
break;
|
|
748
|
+
}
|
|
749
|
+
}
|
|
750
|
+
}
|
|
751
|
+
class CdpClient {
|
|
752
|
+
static async connect(wsUrl) {
|
|
753
|
+
const WebSocketImpl = globalThis.WebSocket;
|
|
754
|
+
if (!WebSocketImpl) {
|
|
755
|
+
throw new Error('WebSocket is not available in this Node runtime.');
|
|
756
|
+
}
|
|
757
|
+
const ws = new WebSocketImpl(wsUrl);
|
|
758
|
+
await new Promise((resolve, reject) => {
|
|
759
|
+
const timer = setTimeout(() => {
|
|
760
|
+
reject(new Error('devtools websocket connect timeout'));
|
|
761
|
+
}, CDP_CONNECT_TIMEOUT_MS);
|
|
762
|
+
ws.addEventListener('open', () => {
|
|
763
|
+
clearTimeout(timer);
|
|
764
|
+
resolve();
|
|
765
|
+
});
|
|
766
|
+
ws.addEventListener('error', (event) => {
|
|
767
|
+
clearTimeout(timer);
|
|
768
|
+
reject(new Error(event?.message || 'devtools websocket error'));
|
|
769
|
+
});
|
|
770
|
+
ws.addEventListener('close', () => {
|
|
771
|
+
clearTimeout(timer);
|
|
772
|
+
});
|
|
773
|
+
});
|
|
774
|
+
return new CdpClient(ws);
|
|
775
|
+
}
|
|
776
|
+
constructor(ws) {
|
|
777
|
+
this.queue = new MessageQueue();
|
|
778
|
+
this.nextId = 1;
|
|
779
|
+
this.closed = false;
|
|
780
|
+
this.ws = ws;
|
|
781
|
+
ws.addEventListener('message', (event) => {
|
|
782
|
+
const data = event?.data ?? event;
|
|
783
|
+
let text = '';
|
|
784
|
+
if (typeof data === 'string')
|
|
785
|
+
text = data;
|
|
786
|
+
else if (data instanceof Buffer)
|
|
787
|
+
text = data.toString('utf8');
|
|
788
|
+
else if (data instanceof ArrayBuffer)
|
|
789
|
+
text = Buffer.from(data).toString('utf8');
|
|
790
|
+
else if (ArrayBuffer.isView(data))
|
|
791
|
+
text = Buffer.from(data.buffer).toString('utf8');
|
|
792
|
+
else
|
|
793
|
+
text = String(data ?? '');
|
|
794
|
+
if (!text.trim())
|
|
795
|
+
return;
|
|
796
|
+
try {
|
|
797
|
+
const value = JSON.parse(text);
|
|
798
|
+
this.queue.push(value);
|
|
799
|
+
}
|
|
800
|
+
catch {
|
|
801
|
+
// ignore malformed
|
|
802
|
+
}
|
|
803
|
+
});
|
|
804
|
+
ws.addEventListener('close', () => {
|
|
805
|
+
this.closed = true;
|
|
806
|
+
this.queue.push({ __closed: true });
|
|
807
|
+
});
|
|
808
|
+
ws.addEventListener('error', (event) => {
|
|
809
|
+
this.queue.push({ __error: event?.message || 'devtools websocket error' });
|
|
810
|
+
});
|
|
811
|
+
}
|
|
812
|
+
async call(method, params, tracker, timeoutMs = CDP_CALL_TIMEOUT_MS) {
|
|
813
|
+
if (this.closed)
|
|
814
|
+
throw new Error('devtools websocket closed');
|
|
815
|
+
const id = this.nextId++;
|
|
816
|
+
this.ws.send(JSON.stringify({ id, method, params }));
|
|
817
|
+
const startedAt = Date.now();
|
|
818
|
+
while (true) {
|
|
819
|
+
const remaining = timeoutMs - (Date.now() - startedAt);
|
|
820
|
+
if (Number.isFinite(timeoutMs) && remaining <= 0) {
|
|
821
|
+
throw new Error(`devtools timeout waiting for ${method}`);
|
|
822
|
+
}
|
|
823
|
+
const message = await this.queue.next(Number.isFinite(timeoutMs) ? Math.max(0, remaining) : undefined);
|
|
824
|
+
if (!message && Number.isFinite(timeoutMs)) {
|
|
825
|
+
throw new Error(`devtools timeout waiting for ${method}`);
|
|
826
|
+
}
|
|
827
|
+
if (!message)
|
|
828
|
+
continue;
|
|
829
|
+
if (message.__closed)
|
|
830
|
+
throw new Error('devtools websocket closed');
|
|
831
|
+
if (message.__error)
|
|
832
|
+
throw new Error(String(message.__error));
|
|
833
|
+
if (message.id === id) {
|
|
834
|
+
if (message.error) {
|
|
835
|
+
throw new Error(`devtools error for ${method}: ${JSON.stringify(message.error)}`);
|
|
836
|
+
}
|
|
837
|
+
return message.result ?? null;
|
|
838
|
+
}
|
|
839
|
+
if (message.method && tracker) {
|
|
840
|
+
tracker.handle(message.method, message.params);
|
|
841
|
+
}
|
|
842
|
+
}
|
|
843
|
+
}
|
|
844
|
+
async waitForNetworkIdle(tracker, timeoutMs) {
|
|
845
|
+
const idleDelay = 800;
|
|
846
|
+
const start = Date.now();
|
|
847
|
+
while (Date.now() - start < timeoutMs) {
|
|
848
|
+
const elapsed = Date.now() - start;
|
|
849
|
+
const idleReady = tracker.inflight === 0 &&
|
|
850
|
+
Date.now() - tracker.lastActivity >= idleDelay &&
|
|
851
|
+
(tracker.sawLoad || elapsed >= idleDelay);
|
|
852
|
+
if (idleReady)
|
|
853
|
+
return true;
|
|
854
|
+
const remaining = timeoutMs - elapsed;
|
|
855
|
+
const waitFor = tracker.inflight === 0
|
|
856
|
+
? Math.min(idleDelay - (Date.now() - tracker.lastActivity), remaining)
|
|
857
|
+
: Math.min(100, remaining);
|
|
858
|
+
const message = await this.queue.next(Math.max(0, waitFor));
|
|
859
|
+
if (message?.method) {
|
|
860
|
+
tracker.handle(message.method, message.params);
|
|
861
|
+
}
|
|
862
|
+
}
|
|
863
|
+
return false;
|
|
864
|
+
}
|
|
865
|
+
close() {
|
|
20
866
|
try {
|
|
21
|
-
|
|
22
|
-
|
|
867
|
+
this.ws.close();
|
|
868
|
+
}
|
|
869
|
+
catch {
|
|
870
|
+
// ignore
|
|
23
871
|
}
|
|
24
|
-
|
|
25
|
-
|
|
872
|
+
}
|
|
873
|
+
}
|
|
874
|
+
const injectWebdriverOverride = async (client) => {
|
|
875
|
+
await client.call('Page.addScriptToEvaluateOnNewDocument', { source: WEBDRIVER_OVERRIDE_SCRIPT });
|
|
876
|
+
};
|
|
877
|
+
const dismissCookieBanners = async (client) => {
|
|
878
|
+
const result = await client.call('Runtime.evaluate', {
|
|
879
|
+
expression: COOKIE_DISMISS_SCRIPT,
|
|
880
|
+
returnByValue: true,
|
|
881
|
+
});
|
|
882
|
+
return Boolean(result?.result?.value ?? result?.value ?? false);
|
|
883
|
+
};
|
|
884
|
+
const evalString = async (client, expression) => {
|
|
885
|
+
const result = await client.call('Runtime.evaluate', {
|
|
886
|
+
expression,
|
|
887
|
+
returnByValue: true,
|
|
888
|
+
});
|
|
889
|
+
const value = result?.result?.value ?? result?.value;
|
|
890
|
+
return typeof value === 'string' ? value : '';
|
|
891
|
+
};
|
|
892
|
+
const evalNumber = async (client, expression) => {
|
|
893
|
+
const result = await client.call('Runtime.evaluate', {
|
|
894
|
+
expression,
|
|
895
|
+
returnByValue: true,
|
|
896
|
+
});
|
|
897
|
+
const value = result?.result?.value ?? result?.value;
|
|
898
|
+
return typeof value === 'number' ? value : Number(value) || 0;
|
|
899
|
+
};
|
|
900
|
+
const captureDomText = async (client, timeoutMs, pollIntervalMs, useInnerText) => {
|
|
901
|
+
const start = Date.now();
|
|
902
|
+
const expression = useInnerText
|
|
903
|
+
? 'document.body ? document.body.innerText : ""'
|
|
904
|
+
: 'document.body ? document.body.textContent : ""';
|
|
905
|
+
let lastValue = '';
|
|
906
|
+
while (Date.now() - start < timeoutMs) {
|
|
907
|
+
const value = await evalString(client, expression).catch(() => '');
|
|
908
|
+
if (value.trim())
|
|
909
|
+
return value.trim();
|
|
910
|
+
lastValue = value;
|
|
911
|
+
await delay(pollIntervalMs);
|
|
912
|
+
}
|
|
913
|
+
return lastValue.trim();
|
|
914
|
+
};
|
|
915
|
+
const evalJson = async (client, expression) => {
|
|
916
|
+
const result = await client.call('Runtime.evaluate', {
|
|
917
|
+
expression,
|
|
918
|
+
returnByValue: true,
|
|
919
|
+
});
|
|
920
|
+
return result?.result?.value ?? result?.value;
|
|
921
|
+
};
|
|
922
|
+
const resolveActionTimeoutMs = (action, fallbackMs) => {
|
|
923
|
+
if ('timeout_ms' in action && typeof action.timeout_ms === 'number') {
|
|
924
|
+
const value = Math.round(action.timeout_ms);
|
|
925
|
+
if (Number.isFinite(value) && value > 0)
|
|
926
|
+
return value;
|
|
927
|
+
}
|
|
928
|
+
return fallbackMs;
|
|
929
|
+
};
|
|
930
|
+
const resolveActionUrl = (actionUrl, baseUrl) => {
|
|
931
|
+
if (actionUrl && isUrl(actionUrl))
|
|
932
|
+
return actionUrl;
|
|
933
|
+
if (!baseUrl || !isUrl(baseUrl))
|
|
934
|
+
return undefined;
|
|
935
|
+
if (!actionUrl)
|
|
936
|
+
return baseUrl;
|
|
937
|
+
try {
|
|
938
|
+
return new URL(actionUrl, baseUrl).toString();
|
|
939
|
+
}
|
|
940
|
+
catch {
|
|
941
|
+
return baseUrl;
|
|
942
|
+
}
|
|
943
|
+
};
|
|
944
|
+
const waitForDocumentReady = async (client, timeoutMs) => {
|
|
945
|
+
const start = Date.now();
|
|
946
|
+
while (Date.now() - start < timeoutMs) {
|
|
947
|
+
const readyState = await evalString(client, 'document.readyState');
|
|
948
|
+
if (readyState === 'complete' || readyState === 'interactive')
|
|
949
|
+
return true;
|
|
950
|
+
await delay(200);
|
|
951
|
+
}
|
|
952
|
+
return false;
|
|
953
|
+
};
|
|
954
|
+
const waitForSelector = async (client, selector, timeoutMs) => {
|
|
955
|
+
const start = Date.now();
|
|
956
|
+
while (Date.now() - start < timeoutMs) {
|
|
957
|
+
const exists = await evalJson(client, `Boolean(document.querySelector(${JSON.stringify(selector)}))`);
|
|
958
|
+
if (exists)
|
|
959
|
+
return true;
|
|
960
|
+
await delay(200);
|
|
961
|
+
}
|
|
962
|
+
return false;
|
|
963
|
+
};
|
|
964
|
+
const clickSelector = async (client, selector, text) => {
|
|
965
|
+
const expression = `(function () {
|
|
966
|
+
const selector = ${JSON.stringify(selector)};
|
|
967
|
+
const text = ${text ? JSON.stringify(text) : 'null'};
|
|
968
|
+
const nodes = Array.from(document.querySelectorAll(selector));
|
|
969
|
+
if (!nodes.length) return { ok: false, message: 'element not found' };
|
|
970
|
+
let target = nodes[0];
|
|
971
|
+
if (text) {
|
|
972
|
+
const normalized = text.toLowerCase();
|
|
973
|
+
target =
|
|
974
|
+
nodes.find((node) =>
|
|
975
|
+
((node.innerText || node.textContent || '') + '').toLowerCase().includes(normalized),
|
|
976
|
+
) || null;
|
|
977
|
+
}
|
|
978
|
+
if (!target) return { ok: false, message: 'element with text not found' };
|
|
979
|
+
if (target.scrollIntoView) target.scrollIntoView({ block: 'center', inline: 'center' });
|
|
980
|
+
if (target.click) target.click();
|
|
981
|
+
return { ok: true };
|
|
982
|
+
})()`;
|
|
983
|
+
const result = await evalJson(client, expression);
|
|
984
|
+
if (result?.ok)
|
|
985
|
+
return { ok: true };
|
|
986
|
+
return { ok: false, message: result?.message || 'click failed' };
|
|
987
|
+
};
|
|
988
|
+
const typeSelector = async (client, selector, text, clear) => {
|
|
989
|
+
const expression = `(function () {
|
|
990
|
+
const selector = ${JSON.stringify(selector)};
|
|
991
|
+
const text = ${JSON.stringify(text)};
|
|
992
|
+
const clear = ${clear ? 'true' : 'false'};
|
|
993
|
+
const el = document.querySelector(selector);
|
|
994
|
+
if (!el) return { ok: false, message: 'element not found' };
|
|
995
|
+
if (el.focus) el.focus();
|
|
996
|
+
if (clear && 'value' in el) {
|
|
997
|
+
el.value = '';
|
|
998
|
+
}
|
|
999
|
+
if ('value' in el) {
|
|
1000
|
+
el.value = String(el.value ?? '') + text;
|
|
1001
|
+
} else {
|
|
1002
|
+
el.textContent = String(el.textContent ?? '') + text;
|
|
1003
|
+
}
|
|
1004
|
+
el.dispatchEvent(new Event('input', { bubbles: true }));
|
|
1005
|
+
el.dispatchEvent(new Event('change', { bubbles: true }));
|
|
1006
|
+
return { ok: true };
|
|
1007
|
+
})()`;
|
|
1008
|
+
const result = await evalJson(client, expression);
|
|
1009
|
+
if (result?.ok)
|
|
1010
|
+
return { ok: true };
|
|
1011
|
+
return { ok: false, message: result?.message || 'type failed' };
|
|
1012
|
+
};
|
|
1013
|
+
const assertText = async (client, selector, text, contains) => {
|
|
1014
|
+
const expression = selector
|
|
1015
|
+
? `(() => {
|
|
1016
|
+
const el = document.querySelector(${JSON.stringify(selector)});
|
|
1017
|
+
return el ? (el.innerText || el.textContent || '') : '';
|
|
1018
|
+
})()`
|
|
1019
|
+
: `document.body ? (document.body.innerText || document.body.textContent || '') : ''`;
|
|
1020
|
+
const actual = String((await evalJson(client, expression)) ?? '');
|
|
1021
|
+
const matches = contains ? actual.includes(text) : actual.trim() === text;
|
|
1022
|
+
if (matches)
|
|
1023
|
+
return { ok: true };
|
|
1024
|
+
return {
|
|
1025
|
+
ok: false,
|
|
1026
|
+
message: `assert_text failed (expected ${contains ? 'contains' : 'equals'} "${text}")`,
|
|
1027
|
+
};
|
|
1028
|
+
};
|
|
1029
|
+
const captureSnapshot = async (client, index, name) => {
|
|
1030
|
+
const html = await evalString(client, 'document.documentElement.outerHTML');
|
|
1031
|
+
const innerText = await evalString(client, 'document.body ? document.body.innerText : ""');
|
|
1032
|
+
const textContent = await evalString(client, 'document.body ? document.body.textContent : ""');
|
|
1033
|
+
const url = await evalString(client, 'document.location.href');
|
|
1034
|
+
return {
|
|
1035
|
+
index,
|
|
1036
|
+
name,
|
|
1037
|
+
html: html || '',
|
|
1038
|
+
innerText: innerText || undefined,
|
|
1039
|
+
textContent: textContent || undefined,
|
|
1040
|
+
url: url || undefined,
|
|
1041
|
+
};
|
|
1042
|
+
};
|
|
1043
|
+
const captureFinalDom = async (client) => {
|
|
1044
|
+
const html = await evalString(client, 'document.documentElement.outerHTML');
|
|
1045
|
+
const innerText = await evalString(client, 'document.body ? document.body.innerText : ""');
|
|
1046
|
+
const textContent = await evalString(client, 'document.body ? document.body.textContent : ""');
|
|
1047
|
+
const url = await evalString(client, 'document.location.href');
|
|
1048
|
+
return {
|
|
1049
|
+
html: html || '',
|
|
1050
|
+
innerText: innerText || undefined,
|
|
1051
|
+
textContent: textContent || undefined,
|
|
1052
|
+
status: undefined,
|
|
1053
|
+
finalUrl: url || undefined,
|
|
1054
|
+
};
|
|
1055
|
+
};
|
|
1056
|
+
const fetchDomViaCdp = async (wsUrl, url, timeoutMs) => {
|
|
1057
|
+
const deadline = Date.now() + timeoutMs;
|
|
1058
|
+
const targetUrl = url ?? DEFAULT_BROWSER_URL;
|
|
1059
|
+
const allowBlank = targetUrl === DEFAULT_BROWSER_URL;
|
|
1060
|
+
const client = await CdpClient.connect(wsUrl);
|
|
1061
|
+
try {
|
|
1062
|
+
await client.call('Network.enable', {});
|
|
1063
|
+
await client.call('Page.enable', {});
|
|
1064
|
+
await client.call('Runtime.enable', {});
|
|
1065
|
+
await injectWebdriverOverride(client);
|
|
1066
|
+
const thinkDelay = randomDelayMs(CHROME_THINK_DELAY_MIN_MS, CHROME_THINK_DELAY_MAX_MS);
|
|
1067
|
+
if (thinkDelay > 0)
|
|
1068
|
+
await delay(thinkDelay);
|
|
1069
|
+
const navResult = await client.call('Page.navigate', { url: targetUrl });
|
|
1070
|
+
if (navResult?.errorText) {
|
|
1071
|
+
throw new Error(`navigation failed: ${navResult.errorText}`);
|
|
1072
|
+
}
|
|
1073
|
+
const dismissed = await dismissCookieBanners(client).catch(() => false);
|
|
1074
|
+
if (dismissed) {
|
|
1075
|
+
const followUp = Math.min(Math.max(0, deadline - Date.now()), CHROME_COOKIE_DISMISS_TIMEOUT_MS);
|
|
1076
|
+
if (followUp > 0)
|
|
1077
|
+
await delay(followUp);
|
|
1078
|
+
}
|
|
1079
|
+
let html = '';
|
|
1080
|
+
let finalUrl;
|
|
1081
|
+
const pollInterval = 200;
|
|
1082
|
+
while (Date.now() < deadline) {
|
|
1083
|
+
const href = await evalString(client, 'document.location.href');
|
|
1084
|
+
if (!finalUrl && href.trim())
|
|
1085
|
+
finalUrl = href.trim();
|
|
1086
|
+
const readyState = await evalString(client, 'document.readyState');
|
|
1087
|
+
const textLen = await evalNumber(client, 'document.body ? document.body.innerText.length : 0');
|
|
1088
|
+
const htmlValue = await evalString(client, 'document.documentElement.outerHTML');
|
|
1089
|
+
if (htmlValue.trim())
|
|
1090
|
+
html = htmlValue;
|
|
1091
|
+
const hasText = textLen >= MIN_TEXT_LEN;
|
|
1092
|
+
const readyComplete = readyState === 'complete' && (allowBlank || href !== 'about:blank');
|
|
1093
|
+
if (hasText || readyComplete)
|
|
1094
|
+
break;
|
|
1095
|
+
await delay(pollInterval);
|
|
1096
|
+
}
|
|
1097
|
+
if (!html.trim())
|
|
1098
|
+
throw new Error('devtools returned empty HTML');
|
|
1099
|
+
const remaining = Math.max(0, deadline - Date.now());
|
|
1100
|
+
const innerText = await captureDomText(client, remaining, pollInterval, true);
|
|
1101
|
+
const textContent = await captureDomText(client, remaining, pollInterval, false);
|
|
1102
|
+
return {
|
|
1103
|
+
html,
|
|
1104
|
+
innerText: innerText || undefined,
|
|
1105
|
+
textContent: textContent || undefined,
|
|
1106
|
+
status: undefined,
|
|
1107
|
+
finalUrl,
|
|
1108
|
+
};
|
|
1109
|
+
}
|
|
1110
|
+
finally {
|
|
1111
|
+
client.close();
|
|
1112
|
+
}
|
|
1113
|
+
};
|
|
1114
|
+
const runBrowserActionsWithClient = async (client, actions, baseUrl, timeoutMs) => {
|
|
1115
|
+
await client.call('Network.enable', {});
|
|
1116
|
+
await client.call('Page.enable', {});
|
|
1117
|
+
await client.call('Runtime.enable', {});
|
|
1118
|
+
await injectWebdriverOverride(client);
|
|
1119
|
+
const tracker = new NetworkIdleTracker();
|
|
1120
|
+
const results = [];
|
|
1121
|
+
const snapshots = [];
|
|
1122
|
+
for (let index = 0; index < actions.length; index += 1) {
|
|
1123
|
+
const action = actions[index];
|
|
1124
|
+
const actionType = action.type;
|
|
1125
|
+
const actionIndex = index + 1;
|
|
1126
|
+
const startedAt = Date.now();
|
|
1127
|
+
const actionTimeoutMs = resolveActionTimeoutMs(action, timeoutMs);
|
|
1128
|
+
try {
|
|
1129
|
+
if (action.type === 'navigate') {
|
|
1130
|
+
const targetUrl = resolveActionUrl(action.url, baseUrl);
|
|
1131
|
+
if (!targetUrl) {
|
|
1132
|
+
throw new Error('navigate requires base_url or absolute url');
|
|
1133
|
+
}
|
|
1134
|
+
const navResult = await client.call('Page.navigate', { url: targetUrl }, tracker, actionTimeoutMs);
|
|
1135
|
+
if (navResult?.errorText) {
|
|
1136
|
+
throw new Error(`navigation failed: ${navResult.errorText}`);
|
|
1137
|
+
}
|
|
1138
|
+
if (action.wait_for === 'idle') {
|
|
1139
|
+
await client.waitForNetworkIdle(tracker, actionTimeoutMs);
|
|
1140
|
+
}
|
|
1141
|
+
else {
|
|
1142
|
+
await waitForDocumentReady(client, actionTimeoutMs);
|
|
1143
|
+
}
|
|
1144
|
+
await dismissCookieBanners(client).catch(() => false);
|
|
1145
|
+
results.push({
|
|
1146
|
+
index: actionIndex,
|
|
1147
|
+
type: actionType,
|
|
1148
|
+
ok: true,
|
|
1149
|
+
url: targetUrl,
|
|
1150
|
+
durationMs: Date.now() - startedAt,
|
|
1151
|
+
});
|
|
1152
|
+
continue;
|
|
1153
|
+
}
|
|
1154
|
+
if (action.type === 'click') {
|
|
1155
|
+
const clickResult = await clickSelector(client, action.selector, action.text);
|
|
1156
|
+
if (!clickResult.ok)
|
|
1157
|
+
throw new Error(clickResult.message ?? 'click failed');
|
|
1158
|
+
results.push({
|
|
1159
|
+
index: actionIndex,
|
|
1160
|
+
type: actionType,
|
|
1161
|
+
ok: true,
|
|
1162
|
+
durationMs: Date.now() - startedAt,
|
|
1163
|
+
});
|
|
1164
|
+
continue;
|
|
1165
|
+
}
|
|
1166
|
+
if (action.type === 'type') {
|
|
1167
|
+
const typeResult = await typeSelector(client, action.selector, action.text, action.clear ?? true);
|
|
1168
|
+
if (!typeResult.ok)
|
|
1169
|
+
throw new Error(typeResult.message ?? 'type failed');
|
|
1170
|
+
results.push({
|
|
1171
|
+
index: actionIndex,
|
|
1172
|
+
type: actionType,
|
|
1173
|
+
ok: true,
|
|
1174
|
+
durationMs: Date.now() - startedAt,
|
|
1175
|
+
});
|
|
1176
|
+
continue;
|
|
1177
|
+
}
|
|
1178
|
+
if (action.type === 'wait_for') {
|
|
1179
|
+
if (action.selector) {
|
|
1180
|
+
const ok = await waitForSelector(client, action.selector, actionTimeoutMs);
|
|
1181
|
+
if (!ok) {
|
|
1182
|
+
throw new Error(`wait_for timeout for selector ${action.selector}`);
|
|
1183
|
+
}
|
|
1184
|
+
}
|
|
1185
|
+
else {
|
|
1186
|
+
await delay(actionTimeoutMs);
|
|
1187
|
+
}
|
|
1188
|
+
results.push({
|
|
1189
|
+
index: actionIndex,
|
|
1190
|
+
type: actionType,
|
|
1191
|
+
ok: true,
|
|
1192
|
+
durationMs: Date.now() - startedAt,
|
|
1193
|
+
});
|
|
1194
|
+
continue;
|
|
1195
|
+
}
|
|
1196
|
+
if (action.type === 'assert_text') {
|
|
1197
|
+
const assertResult = await assertText(client, action.selector, action.text, action.contains ?? true);
|
|
1198
|
+
if (!assertResult.ok)
|
|
1199
|
+
throw new Error(assertResult.message ?? 'assert_text failed');
|
|
1200
|
+
results.push({
|
|
1201
|
+
index: actionIndex,
|
|
1202
|
+
type: actionType,
|
|
1203
|
+
ok: true,
|
|
1204
|
+
durationMs: Date.now() - startedAt,
|
|
1205
|
+
});
|
|
1206
|
+
continue;
|
|
1207
|
+
}
|
|
1208
|
+
if (action.type === 'snapshot') {
|
|
1209
|
+
const name = action.name?.trim() || `snapshot-${actionIndex}`;
|
|
1210
|
+
const snapshot = await captureSnapshot(client, actionIndex, name);
|
|
1211
|
+
snapshots.push(snapshot);
|
|
1212
|
+
results.push({
|
|
1213
|
+
index: actionIndex,
|
|
1214
|
+
type: actionType,
|
|
1215
|
+
ok: true,
|
|
1216
|
+
durationMs: Date.now() - startedAt,
|
|
1217
|
+
});
|
|
1218
|
+
continue;
|
|
1219
|
+
}
|
|
1220
|
+
if (action.type === 'script') {
|
|
1221
|
+
const value = await evalJson(client, action.expression);
|
|
1222
|
+
if (action.expect !== undefined) {
|
|
1223
|
+
const actual = value === undefined || value === null ? '' : String(value);
|
|
1224
|
+
if (!actual.includes(action.expect)) {
|
|
1225
|
+
throw new Error(`script expect failed (expected "${action.expect}")`);
|
|
1226
|
+
}
|
|
1227
|
+
}
|
|
1228
|
+
results.push({
|
|
1229
|
+
index: actionIndex,
|
|
1230
|
+
type: actionType,
|
|
1231
|
+
ok: true,
|
|
1232
|
+
durationMs: Date.now() - startedAt,
|
|
1233
|
+
});
|
|
1234
|
+
continue;
|
|
1235
|
+
}
|
|
1236
|
+
results.push({
|
|
1237
|
+
index: actionIndex,
|
|
1238
|
+
type: actionType,
|
|
1239
|
+
ok: false,
|
|
1240
|
+
message: 'unsupported action',
|
|
1241
|
+
durationMs: Date.now() - startedAt,
|
|
1242
|
+
});
|
|
1243
|
+
return {
|
|
1244
|
+
outcome: 'fail',
|
|
1245
|
+
errorMessage: 'unsupported action',
|
|
1246
|
+
results,
|
|
1247
|
+
snapshots,
|
|
1248
|
+
finalDom: await captureFinalDom(client).catch(() => undefined),
|
|
1249
|
+
};
|
|
1250
|
+
}
|
|
1251
|
+
catch (err) {
|
|
1252
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
1253
|
+
results.push({
|
|
1254
|
+
index: actionIndex,
|
|
1255
|
+
type: actionType,
|
|
1256
|
+
ok: false,
|
|
1257
|
+
message,
|
|
1258
|
+
durationMs: Date.now() - startedAt,
|
|
1259
|
+
});
|
|
1260
|
+
return {
|
|
1261
|
+
outcome: 'fail',
|
|
1262
|
+
errorMessage: message,
|
|
1263
|
+
results,
|
|
1264
|
+
snapshots,
|
|
1265
|
+
finalDom: await captureFinalDom(client).catch(() => undefined),
|
|
1266
|
+
};
|
|
1267
|
+
}
|
|
1268
|
+
}
|
|
1269
|
+
return {
|
|
1270
|
+
outcome: 'pass',
|
|
1271
|
+
results,
|
|
1272
|
+
snapshots,
|
|
1273
|
+
finalDom: await captureFinalDom(client).catch(() => undefined),
|
|
1274
|
+
};
|
|
1275
|
+
};
|
|
1276
|
+
const runDumpDom = async (chromeBinary, url, headless, userAgent, userDataDir, timeoutMs) => {
|
|
1277
|
+
const targetUrl = url ?? DEFAULT_BROWSER_URL;
|
|
1278
|
+
const args = chromeCommonArgs({
|
|
1279
|
+
chromeBinary,
|
|
1280
|
+
headless,
|
|
1281
|
+
userAgent,
|
|
1282
|
+
userDataDir,
|
|
1283
|
+
debugPort: 0,
|
|
1284
|
+
}).filter((arg) => !arg.startsWith('--remote-debugging-'));
|
|
1285
|
+
args.push('--virtual-time-budget=15000');
|
|
1286
|
+
args.push('--dump-dom');
|
|
1287
|
+
args.push(targetUrl);
|
|
1288
|
+
const child = spawn(chromeBinary, args, { stdio: ['ignore', 'pipe', 'pipe'] });
|
|
1289
|
+
let stdout = '';
|
|
1290
|
+
let stderr = '';
|
|
1291
|
+
child.stdout?.on('data', (chunk) => {
|
|
1292
|
+
stdout += chunk.toString();
|
|
1293
|
+
});
|
|
1294
|
+
child.stderr?.on('data', (chunk) => {
|
|
1295
|
+
stderr += chunk.toString();
|
|
1296
|
+
});
|
|
1297
|
+
const result = await new Promise((resolve) => {
|
|
1298
|
+
let settled = false;
|
|
1299
|
+
const timer = setTimeout(() => {
|
|
1300
|
+
if (settled)
|
|
1301
|
+
return;
|
|
1302
|
+
settled = true;
|
|
1303
|
+
child.kill('SIGTERM');
|
|
1304
|
+
setTimeout(() => child.kill('SIGKILL'), 2000).unref();
|
|
1305
|
+
resolve({ exitCode: null, timedOut: true });
|
|
1306
|
+
}, timeoutMs);
|
|
1307
|
+
child.on('close', (code) => {
|
|
1308
|
+
if (settled)
|
|
1309
|
+
return;
|
|
1310
|
+
settled = true;
|
|
1311
|
+
clearTimeout(timer);
|
|
1312
|
+
resolve({ exitCode: code ?? null, timedOut: false });
|
|
1313
|
+
});
|
|
1314
|
+
child.on('error', () => {
|
|
1315
|
+
if (settled)
|
|
1316
|
+
return;
|
|
1317
|
+
settled = true;
|
|
1318
|
+
clearTimeout(timer);
|
|
1319
|
+
resolve({ exitCode: null, timedOut: false });
|
|
1320
|
+
});
|
|
1321
|
+
});
|
|
1322
|
+
return { html: stdout.trim(), stdout, stderr, exitCode: result.exitCode, timedOut: result.timedOut };
|
|
1323
|
+
};
|
|
1324
|
+
const terminateProcessTree = async (child) => {
|
|
1325
|
+
if (!child.pid || child.exitCode !== null)
|
|
1326
|
+
return;
|
|
1327
|
+
try {
|
|
1328
|
+
if (process.platform !== 'win32') {
|
|
26
1329
|
try {
|
|
27
|
-
|
|
28
|
-
|
|
1330
|
+
process.kill(-child.pid, 'SIGTERM');
|
|
1331
|
+
}
|
|
1332
|
+
catch {
|
|
1333
|
+
child.kill('SIGTERM');
|
|
1334
|
+
}
|
|
1335
|
+
}
|
|
1336
|
+
else {
|
|
1337
|
+
child.kill('SIGTERM');
|
|
1338
|
+
}
|
|
1339
|
+
}
|
|
1340
|
+
catch {
|
|
1341
|
+
// ignore
|
|
1342
|
+
}
|
|
1343
|
+
await delay(2000);
|
|
1344
|
+
if (child.exitCode === null) {
|
|
1345
|
+
try {
|
|
1346
|
+
if (process.platform !== 'win32') {
|
|
1347
|
+
try {
|
|
1348
|
+
process.kill(-child.pid, 'SIGKILL');
|
|
1349
|
+
}
|
|
1350
|
+
catch {
|
|
1351
|
+
child.kill('SIGKILL');
|
|
1352
|
+
}
|
|
29
1353
|
}
|
|
30
|
-
|
|
31
|
-
|
|
1354
|
+
else {
|
|
1355
|
+
child.kill('SIGKILL');
|
|
32
1356
|
}
|
|
33
1357
|
}
|
|
1358
|
+
catch {
|
|
1359
|
+
// ignore
|
|
1360
|
+
}
|
|
1361
|
+
}
|
|
1362
|
+
};
|
|
1363
|
+
const truncateText = (value, maxLength) => {
|
|
1364
|
+
if (!value)
|
|
1365
|
+
return '';
|
|
1366
|
+
if (value.length <= maxLength)
|
|
1367
|
+
return value;
|
|
1368
|
+
return `${value.slice(0, maxLength)}\n[truncated ${value.length - maxLength} chars]`;
|
|
1369
|
+
};
|
|
1370
|
+
const formatBrowserStdout = (result, url) => {
|
|
1371
|
+
const displayUrl = result.finalUrl ?? url ?? DEFAULT_BROWSER_URL;
|
|
1372
|
+
const lines = [
|
|
1373
|
+
'MCODA_BROWSER_QA_RESULT',
|
|
1374
|
+
`status: ${result.status ?? 'unknown'}`,
|
|
1375
|
+
`final_url: ${displayUrl}`,
|
|
1376
|
+
];
|
|
1377
|
+
const text = result.innerText ?? result.textContent ?? '';
|
|
1378
|
+
if (text.trim()) {
|
|
1379
|
+
lines.push('');
|
|
1380
|
+
lines.push('inner_text:');
|
|
1381
|
+
lines.push(truncateText(text.trim(), 8000));
|
|
1382
|
+
}
|
|
1383
|
+
return lines.join('\n');
|
|
1384
|
+
};
|
|
1385
|
+
export class ChromiumQaAdapter {
|
|
1386
|
+
async ensureInstalled(profile, ctx) {
|
|
1387
|
+
if (shouldSkipInstall(ctx))
|
|
1388
|
+
return { ok: true, details: { skipped: true } };
|
|
1389
|
+
const chromiumPath = await resolveDocdexChromiumBinary();
|
|
1390
|
+
if (!chromiumPath) {
|
|
1391
|
+
return { ok: false, message: DOCDEX_CHROMIUM_MISSING_MESSAGE };
|
|
1392
|
+
}
|
|
1393
|
+
return { ok: true, details: { chromiumPath } };
|
|
34
1394
|
}
|
|
35
1395
|
async persistLogs(ctx, stdout, stderr) {
|
|
36
1396
|
const artifacts = [];
|
|
@@ -44,42 +1404,274 @@ export class ChromiumQaAdapter {
|
|
|
44
1404
|
artifacts.push(path.relative(ctx.workspaceRoot, outPath), path.relative(ctx.workspaceRoot, errPath));
|
|
45
1405
|
return artifacts;
|
|
46
1406
|
}
|
|
1407
|
+
async persistBrowserArtifacts(ctx, result) {
|
|
1408
|
+
const artifacts = [];
|
|
1409
|
+
if (!ctx.artifactDir)
|
|
1410
|
+
return artifacts;
|
|
1411
|
+
await fs.mkdir(ctx.artifactDir, { recursive: true });
|
|
1412
|
+
const htmlPath = path.join(ctx.artifactDir, 'browser.html');
|
|
1413
|
+
await fs.writeFile(htmlPath, result.html ?? '', 'utf8');
|
|
1414
|
+
artifacts.push(path.relative(ctx.workspaceRoot, htmlPath));
|
|
1415
|
+
if (result.innerText) {
|
|
1416
|
+
const innerPath = path.join(ctx.artifactDir, 'browser.inner_text.txt');
|
|
1417
|
+
await fs.writeFile(innerPath, result.innerText, 'utf8');
|
|
1418
|
+
artifacts.push(path.relative(ctx.workspaceRoot, innerPath));
|
|
1419
|
+
}
|
|
1420
|
+
if (result.textContent) {
|
|
1421
|
+
const textPath = path.join(ctx.artifactDir, 'browser.text_content.txt');
|
|
1422
|
+
await fs.writeFile(textPath, result.textContent, 'utf8');
|
|
1423
|
+
artifacts.push(path.relative(ctx.workspaceRoot, textPath));
|
|
1424
|
+
}
|
|
1425
|
+
const metaPath = path.join(ctx.artifactDir, 'browser.json');
|
|
1426
|
+
await fs.writeFile(metaPath, JSON.stringify({
|
|
1427
|
+
status: result.status ?? null,
|
|
1428
|
+
final_url: result.finalUrl ?? null,
|
|
1429
|
+
}, null, 2), 'utf8');
|
|
1430
|
+
artifacts.push(path.relative(ctx.workspaceRoot, metaPath));
|
|
1431
|
+
return artifacts;
|
|
1432
|
+
}
|
|
47
1433
|
async invoke(profile, ctx) {
|
|
48
|
-
const command = ctx.testCommandOverride ?? profile.test_command ?? 'npx playwright test --reporter=list';
|
|
49
1434
|
const startedAt = new Date().toISOString();
|
|
50
|
-
const
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
1435
|
+
const actions = (ctx.browserActions ?? []).filter(Boolean);
|
|
1436
|
+
if (actions.length) {
|
|
1437
|
+
return await this.runBrowserActions(actions, ctx, startedAt);
|
|
1438
|
+
}
|
|
1439
|
+
const url = resolveBrowserTarget(profile, ctx);
|
|
1440
|
+
return await this.runBrowser(url, ctx, startedAt);
|
|
1441
|
+
}
|
|
1442
|
+
async runBrowserActions(actions, ctx, startedAt) {
|
|
1443
|
+
const chromiumPath = await resolveDocdexChromiumBinary();
|
|
1444
|
+
if (!chromiumPath) {
|
|
56
1445
|
const finishedAt = new Date().toISOString();
|
|
57
|
-
const artifacts = await this.persistLogs(ctx, stdout, stderr);
|
|
58
1446
|
return {
|
|
59
|
-
outcome: '
|
|
60
|
-
exitCode:
|
|
61
|
-
stdout,
|
|
62
|
-
stderr,
|
|
1447
|
+
outcome: 'infra_issue',
|
|
1448
|
+
exitCode: null,
|
|
1449
|
+
stdout: '',
|
|
1450
|
+
stderr: DOCDEX_CHROMIUM_MISSING_MESSAGE,
|
|
1451
|
+
artifacts: [],
|
|
1452
|
+
startedAt,
|
|
1453
|
+
finishedAt,
|
|
1454
|
+
};
|
|
1455
|
+
}
|
|
1456
|
+
const headless = resolveHeadless(ctx);
|
|
1457
|
+
const timeoutMs = resolveTimeoutMs(ctx);
|
|
1458
|
+
const userAgent = resolveUserAgent(ctx);
|
|
1459
|
+
const userDataDir = await resolveUserDataDir(ctx);
|
|
1460
|
+
const sessionConfig = {
|
|
1461
|
+
chromeBinary: chromiumPath,
|
|
1462
|
+
headless,
|
|
1463
|
+
userAgent,
|
|
1464
|
+
userDataDir,
|
|
1465
|
+
};
|
|
1466
|
+
let actionOutcome;
|
|
1467
|
+
let actionError;
|
|
1468
|
+
let infraError;
|
|
1469
|
+
let actionResults = [];
|
|
1470
|
+
let snapshots = [];
|
|
1471
|
+
let finalDom;
|
|
1472
|
+
const executeOnce = async () => {
|
|
1473
|
+
const instance = await chromeManager.getOrLaunch(sessionConfig, ctx.env);
|
|
1474
|
+
const target = await createCdpTarget(instance.getDebugPort(), CHROME_STARTUP_TIMEOUT_MS);
|
|
1475
|
+
const client = await CdpClient.connect(target.wsUrl);
|
|
1476
|
+
try {
|
|
1477
|
+
return await runBrowserActionsWithClient(client, actions, ctx.browserBaseUrl, timeoutMs);
|
|
1478
|
+
}
|
|
1479
|
+
finally {
|
|
1480
|
+
client.close();
|
|
1481
|
+
if (target.targetId) {
|
|
1482
|
+
try {
|
|
1483
|
+
await closeCdpTarget(instance.getDebugPort(), target.targetId);
|
|
1484
|
+
}
|
|
1485
|
+
catch {
|
|
1486
|
+
// ignore
|
|
1487
|
+
}
|
|
1488
|
+
}
|
|
1489
|
+
}
|
|
1490
|
+
};
|
|
1491
|
+
try {
|
|
1492
|
+
const release = await chromeFetchSemaphore.acquire();
|
|
1493
|
+
try {
|
|
1494
|
+
const result = await executeOnce();
|
|
1495
|
+
actionOutcome = result.outcome;
|
|
1496
|
+
actionError = result.errorMessage;
|
|
1497
|
+
actionResults = result.results;
|
|
1498
|
+
snapshots = result.snapshots;
|
|
1499
|
+
finalDom = result.finalDom;
|
|
1500
|
+
}
|
|
1501
|
+
finally {
|
|
1502
|
+
release();
|
|
1503
|
+
}
|
|
1504
|
+
}
|
|
1505
|
+
catch (err) {
|
|
1506
|
+
infraError = err instanceof Error ? err.message : String(err);
|
|
1507
|
+
}
|
|
1508
|
+
const finishedAt = new Date().toISOString();
|
|
1509
|
+
if (infraError) {
|
|
1510
|
+
const artifacts = await this.persistLogs(ctx, '', infraError);
|
|
1511
|
+
return {
|
|
1512
|
+
outcome: 'infra_issue',
|
|
1513
|
+
exitCode: null,
|
|
1514
|
+
stdout: '',
|
|
1515
|
+
stderr: infraError,
|
|
63
1516
|
artifacts,
|
|
64
1517
|
startedAt,
|
|
65
1518
|
finishedAt,
|
|
66
1519
|
};
|
|
67
1520
|
}
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
const
|
|
71
|
-
|
|
1521
|
+
const lines = ['MCODA_BROWSER_QA_ACTIONS'];
|
|
1522
|
+
for (const result of actionResults) {
|
|
1523
|
+
const status = result.ok ? 'ok' : 'fail';
|
|
1524
|
+
lines.push(`${result.index}. ${result.type} ${status}${result.message ? ` - ${result.message}` : ''}`);
|
|
1525
|
+
}
|
|
1526
|
+
const stdout = lines.join('\n');
|
|
1527
|
+
const stderr = actionOutcome === 'fail' ? actionError ?? 'Browser action failed' : '';
|
|
1528
|
+
const artifacts = await this.persistLogs(ctx, stdout, stderr);
|
|
1529
|
+
if (finalDom) {
|
|
1530
|
+
const browserArtifacts = await this.persistBrowserArtifacts(ctx, finalDom);
|
|
1531
|
+
artifacts.push(...browserArtifacts);
|
|
1532
|
+
}
|
|
1533
|
+
if (ctx.artifactDir) {
|
|
1534
|
+
await fs.mkdir(ctx.artifactDir, { recursive: true });
|
|
1535
|
+
const actionsPath = path.join(ctx.artifactDir, 'browser-actions.json');
|
|
1536
|
+
await fs.writeFile(actionsPath, JSON.stringify({ actions: actionResults, snapshots }, null, 2), 'utf8');
|
|
1537
|
+
artifacts.push(path.relative(ctx.workspaceRoot, actionsPath));
|
|
1538
|
+
if (snapshots.length) {
|
|
1539
|
+
const snapDir = path.join(ctx.artifactDir, 'browser-snapshots');
|
|
1540
|
+
await fs.mkdir(snapDir, { recursive: true });
|
|
1541
|
+
for (const snapshot of snapshots) {
|
|
1542
|
+
const safeName = snapshot.name.replace(/[^a-zA-Z0-9_-]+/g, '-');
|
|
1543
|
+
const baseName = `${snapshot.index}-${safeName}`;
|
|
1544
|
+
const htmlPath = path.join(snapDir, `${baseName}.html`);
|
|
1545
|
+
await fs.writeFile(htmlPath, snapshot.html ?? '', 'utf8');
|
|
1546
|
+
artifacts.push(path.relative(ctx.workspaceRoot, htmlPath));
|
|
1547
|
+
if (snapshot.innerText) {
|
|
1548
|
+
const innerPath = path.join(snapDir, `${baseName}.inner_text.txt`);
|
|
1549
|
+
await fs.writeFile(innerPath, snapshot.innerText, 'utf8');
|
|
1550
|
+
artifacts.push(path.relative(ctx.workspaceRoot, innerPath));
|
|
1551
|
+
}
|
|
1552
|
+
if (snapshot.textContent) {
|
|
1553
|
+
const textPath = path.join(snapDir, `${baseName}.text_content.txt`);
|
|
1554
|
+
await fs.writeFile(textPath, snapshot.textContent, 'utf8');
|
|
1555
|
+
artifacts.push(path.relative(ctx.workspaceRoot, textPath));
|
|
1556
|
+
}
|
|
1557
|
+
}
|
|
1558
|
+
}
|
|
1559
|
+
}
|
|
1560
|
+
return {
|
|
1561
|
+
outcome: actionOutcome === 'fail' ? 'fail' : 'pass',
|
|
1562
|
+
exitCode: actionOutcome === 'fail' ? 1 : 0,
|
|
1563
|
+
stdout,
|
|
1564
|
+
stderr: stderr || '',
|
|
1565
|
+
artifacts,
|
|
1566
|
+
startedAt,
|
|
1567
|
+
finishedAt,
|
|
1568
|
+
};
|
|
1569
|
+
}
|
|
1570
|
+
async runBrowser(url, ctx, startedAt) {
|
|
1571
|
+
const chromiumPath = await resolveDocdexChromiumBinary();
|
|
1572
|
+
if (!chromiumPath) {
|
|
72
1573
|
const finishedAt = new Date().toISOString();
|
|
73
|
-
const artifacts = await this.persistLogs(ctx, stdout, stderr);
|
|
74
1574
|
return {
|
|
75
|
-
outcome:
|
|
76
|
-
exitCode,
|
|
77
|
-
stdout,
|
|
1575
|
+
outcome: 'infra_issue',
|
|
1576
|
+
exitCode: null,
|
|
1577
|
+
stdout: '',
|
|
1578
|
+
stderr: DOCDEX_CHROMIUM_MISSING_MESSAGE,
|
|
1579
|
+
artifacts: [],
|
|
1580
|
+
startedAt,
|
|
1581
|
+
finishedAt,
|
|
1582
|
+
};
|
|
1583
|
+
}
|
|
1584
|
+
const headless = resolveHeadless(ctx);
|
|
1585
|
+
const timeoutMs = resolveTimeoutMs(ctx);
|
|
1586
|
+
const userAgent = resolveUserAgent(ctx);
|
|
1587
|
+
const userDataDir = await resolveUserDataDir(ctx);
|
|
1588
|
+
const sessionConfig = {
|
|
1589
|
+
chromeBinary: chromiumPath,
|
|
1590
|
+
headless,
|
|
1591
|
+
userAgent,
|
|
1592
|
+
userDataDir,
|
|
1593
|
+
};
|
|
1594
|
+
let browserResult;
|
|
1595
|
+
let cdpError;
|
|
1596
|
+
try {
|
|
1597
|
+
const release = await chromeFetchSemaphore.acquire();
|
|
1598
|
+
try {
|
|
1599
|
+
const instance = await chromeManager.getOrLaunch(sessionConfig, ctx.env);
|
|
1600
|
+
try {
|
|
1601
|
+
browserResult = await instance.fetchDom(url, timeoutMs);
|
|
1602
|
+
}
|
|
1603
|
+
catch (err) {
|
|
1604
|
+
cdpError = err instanceof Error ? err.message : String(err);
|
|
1605
|
+
if (await chromeManager.resetIfUnhealthy(instance)) {
|
|
1606
|
+
try {
|
|
1607
|
+
const nextInstance = await chromeManager.getOrLaunch(sessionConfig, ctx.env);
|
|
1608
|
+
browserResult = await nextInstance.fetchDom(url, timeoutMs);
|
|
1609
|
+
cdpError = undefined;
|
|
1610
|
+
}
|
|
1611
|
+
catch (err2) {
|
|
1612
|
+
cdpError = err2 instanceof Error ? err2.message : String(err2);
|
|
1613
|
+
}
|
|
1614
|
+
}
|
|
1615
|
+
}
|
|
1616
|
+
}
|
|
1617
|
+
finally {
|
|
1618
|
+
release();
|
|
1619
|
+
}
|
|
1620
|
+
}
|
|
1621
|
+
catch (err) {
|
|
1622
|
+
cdpError = err instanceof Error ? err.message : String(err);
|
|
1623
|
+
}
|
|
1624
|
+
if (!browserResult) {
|
|
1625
|
+
try {
|
|
1626
|
+
const dump = await runDumpDom(chromiumPath, url, headless, userAgent, userDataDir.path, timeoutMs);
|
|
1627
|
+
if (dump.html.trim()) {
|
|
1628
|
+
browserResult = { html: dump.html, finalUrl: url ?? DEFAULT_BROWSER_URL };
|
|
1629
|
+
if (!cdpError && dump.stderr)
|
|
1630
|
+
cdpError = dump.stderr;
|
|
1631
|
+
}
|
|
1632
|
+
else if (!cdpError) {
|
|
1633
|
+
cdpError = dump.timedOut
|
|
1634
|
+
? `Timed out after ${timeoutMs}ms while loading ${url ?? DEFAULT_BROWSER_URL}.`
|
|
1635
|
+
: 'chrome dump-dom returned empty HTML';
|
|
1636
|
+
}
|
|
1637
|
+
}
|
|
1638
|
+
catch (err) {
|
|
1639
|
+
if (!cdpError)
|
|
1640
|
+
cdpError = err instanceof Error ? err.message : String(err);
|
|
1641
|
+
}
|
|
1642
|
+
}
|
|
1643
|
+
const finishedAt = new Date().toISOString();
|
|
1644
|
+
if (!browserResult) {
|
|
1645
|
+
const stderr = cdpError || 'Chromium QA failed to load page.';
|
|
1646
|
+
const artifacts = await this.persistLogs(ctx, '', stderr);
|
|
1647
|
+
return {
|
|
1648
|
+
outcome: 'infra_issue',
|
|
1649
|
+
exitCode: null,
|
|
1650
|
+
stdout: '',
|
|
78
1651
|
stderr,
|
|
79
1652
|
artifacts,
|
|
80
1653
|
startedAt,
|
|
81
1654
|
finishedAt,
|
|
82
1655
|
};
|
|
83
1656
|
}
|
|
1657
|
+
const stdout = formatBrowserStdout(browserResult, url);
|
|
1658
|
+
const stderr = cdpError ?? '';
|
|
1659
|
+
const artifacts = await this.persistLogs(ctx, stdout, stderr || '');
|
|
1660
|
+
const browserArtifacts = await this.persistBrowserArtifacts(ctx, browserResult);
|
|
1661
|
+
artifacts.push(...browserArtifacts);
|
|
1662
|
+
return {
|
|
1663
|
+
outcome: 'pass',
|
|
1664
|
+
exitCode: 0,
|
|
1665
|
+
stdout,
|
|
1666
|
+
stderr: stderr || '',
|
|
1667
|
+
artifacts,
|
|
1668
|
+
startedAt,
|
|
1669
|
+
finishedAt,
|
|
1670
|
+
};
|
|
84
1671
|
}
|
|
85
1672
|
}
|
|
1673
|
+
export const __testing = {
|
|
1674
|
+
resolveActionUrl,
|
|
1675
|
+
resolveActionTimeoutMs,
|
|
1676
|
+
runBrowserActionsWithClient,
|
|
1677
|
+
};
|