iframer-toolkit 0.7.1 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -1
- package/dist/cli.js +276 -86
- package/dist/mcp-server.cjs +520 -414
- package/package.json +6 -3
package/dist/cli.js
CHANGED
|
@@ -2936,6 +2936,23 @@ function tryParseJson(text) {
|
|
|
2936
2936
|
return;
|
|
2937
2937
|
}
|
|
2938
2938
|
}
|
|
2939
|
+
function sanitizeString(s) {
|
|
2940
|
+
return s.replace(/[\uD800-\uDFFF]/g, "�");
|
|
2941
|
+
}
|
|
2942
|
+
function sanitizeDeep(val) {
|
|
2943
|
+
if (typeof val === "string")
|
|
2944
|
+
return sanitizeString(val);
|
|
2945
|
+
if (Array.isArray(val))
|
|
2946
|
+
return val.map(sanitizeDeep);
|
|
2947
|
+
if (val && typeof val === "object") {
|
|
2948
|
+
const out = {};
|
|
2949
|
+
for (const [k, v] of Object.entries(val)) {
|
|
2950
|
+
out[sanitizeString(k)] = sanitizeDeep(v);
|
|
2951
|
+
}
|
|
2952
|
+
return out;
|
|
2953
|
+
}
|
|
2954
|
+
return val;
|
|
2955
|
+
}
|
|
2939
2956
|
function hasGraphQLShape(body) {
|
|
2940
2957
|
if (!body || typeof body !== "object")
|
|
2941
2958
|
return false;
|
|
@@ -3083,7 +3100,8 @@ function buildCurl(method, url, headers, auth, body) {
|
|
|
3083
3100
|
}
|
|
3084
3101
|
if (body !== undefined) {
|
|
3085
3102
|
const bodyStr = typeof body === "string" ? body : JSON.stringify(body);
|
|
3086
|
-
|
|
3103
|
+
const safeBody = bodyStr.length > 1e4 ? bodyStr.slice(0, 1e4) + "...[truncated]" : bodyStr;
|
|
3104
|
+
parts.push(` -d '${safeBody.replace(/'/g, "'\\''")}'`);
|
|
3087
3105
|
}
|
|
3088
3106
|
parts.push(` '${url}'`);
|
|
3089
3107
|
return parts.join(" \\\n");
|
|
@@ -3123,16 +3141,28 @@ class ApiCapture {
|
|
|
3123
3141
|
const allHeaders = req.headers();
|
|
3124
3142
|
let requestBody = undefined;
|
|
3125
3143
|
try {
|
|
3144
|
+
const ct = (allHeaders["content-type"] || allHeaders["Content-Type"] || "").toLowerCase();
|
|
3126
3145
|
const postData = req.postData();
|
|
3127
3146
|
if (postData) {
|
|
3128
|
-
|
|
3147
|
+
if (ct.includes("multipart/form-data")) {
|
|
3148
|
+
const fields = [...postData.matchAll(/name="([^"]+)"/g)].map((m) => m[1]);
|
|
3149
|
+
requestBody = { _type: "multipart/form-data", fields: [...new Set(fields)] };
|
|
3150
|
+
} else if (ct.includes("application/octet-stream") || ct.startsWith("video/") || ct.startsWith("image/") || ct.startsWith("audio/")) {
|
|
3151
|
+
requestBody = { _type: ct, _size: postData.length };
|
|
3152
|
+
} else if (postData.length < 500000) {
|
|
3153
|
+
requestBody = sanitizeDeep(tryParseJson(postData) ?? postData);
|
|
3154
|
+
} else {
|
|
3155
|
+
requestBody = `[body truncated — ${postData.length} bytes, content-type: ${ct}]`;
|
|
3156
|
+
}
|
|
3129
3157
|
}
|
|
3130
3158
|
} catch {}
|
|
3131
3159
|
let responseBody = undefined;
|
|
3132
3160
|
try {
|
|
3133
3161
|
const resText = await res.text();
|
|
3134
|
-
if (resText && resText.length <
|
|
3135
|
-
responseBody = tryParseJson(resText) ?? resText;
|
|
3162
|
+
if (resText && resText.length < 500000) {
|
|
3163
|
+
responseBody = sanitizeDeep(tryParseJson(resText) ?? resText);
|
|
3164
|
+
} else if (resText) {
|
|
3165
|
+
responseBody = `[response truncated — ${resText.length} bytes]`;
|
|
3136
3166
|
}
|
|
3137
3167
|
} catch {}
|
|
3138
3168
|
this.requests.push({
|
|
@@ -3163,6 +3193,13 @@ class ApiCapture {
|
|
|
3163
3193
|
this.page.off("request", this.requestHandler);
|
|
3164
3194
|
this.page.off("response", this.responseHandler);
|
|
3165
3195
|
}
|
|
3196
|
+
async drain(ms = 3000, pendingTimeoutMs = 5000) {
|
|
3197
|
+
await new Promise((r) => setTimeout(r, ms));
|
|
3198
|
+
const deadline = Date.now() + pendingTimeoutMs;
|
|
3199
|
+
while (this.pendingRequests.size > 0 && Date.now() < deadline) {
|
|
3200
|
+
await new Promise((r) => setTimeout(r, 100));
|
|
3201
|
+
}
|
|
3202
|
+
}
|
|
3166
3203
|
extractAuth(requests) {
|
|
3167
3204
|
const auth = { cookies: {}, tokens: {} };
|
|
3168
3205
|
for (const req of requests) {
|
|
@@ -3376,7 +3413,7 @@ class PipelineRunner {
|
|
|
3376
3413
|
const capture = opts.captureApi ? new ApiCapture(page) : null;
|
|
3377
3414
|
if (capture)
|
|
3378
3415
|
capture.start();
|
|
3379
|
-
const finishCapture = () => {
|
|
3416
|
+
const finishCapture = async () => {
|
|
3380
3417
|
if (!capture)
|
|
3381
3418
|
return;
|
|
3382
3419
|
capture.stop();
|
|
@@ -3423,7 +3460,7 @@ class PipelineRunner {
|
|
|
3423
3460
|
retryable: isRetryable(errorType)
|
|
3424
3461
|
},
|
|
3425
3462
|
durationMs: Date.now() - startTime,
|
|
3426
|
-
capturedApi: finishCapture()
|
|
3463
|
+
capturedApi: await finishCapture()
|
|
3427
3464
|
};
|
|
3428
3465
|
}
|
|
3429
3466
|
if (screenshotAfterEach && stepResult.ok) {
|
|
@@ -3453,7 +3490,7 @@ class PipelineRunner {
|
|
|
3453
3490
|
retryable: isRetryable(errorType)
|
|
3454
3491
|
},
|
|
3455
3492
|
durationMs: Date.now() - startTime,
|
|
3456
|
-
capturedApi: finishCapture()
|
|
3493
|
+
capturedApi: await finishCapture()
|
|
3457
3494
|
};
|
|
3458
3495
|
}
|
|
3459
3496
|
if (step.type === "navigate" && continueOnObstacle) {
|
|
@@ -3487,12 +3524,13 @@ class PipelineRunner {
|
|
|
3487
3524
|
retryable: true
|
|
3488
3525
|
},
|
|
3489
3526
|
durationMs: Date.now() - startTime,
|
|
3490
|
-
capturedApi: finishCapture()
|
|
3527
|
+
capturedApi: await finishCapture()
|
|
3491
3528
|
};
|
|
3492
3529
|
}
|
|
3493
3530
|
}
|
|
3494
3531
|
}
|
|
3495
3532
|
}
|
|
3533
|
+
const capturedApi = await finishCapture();
|
|
3496
3534
|
const finalState = await getPageState(page, this.ctx, true);
|
|
3497
3535
|
return {
|
|
3498
3536
|
ok: true,
|
|
@@ -3502,7 +3540,7 @@ class PipelineRunner {
|
|
|
3502
3540
|
obstacles,
|
|
3503
3541
|
finalState,
|
|
3504
3542
|
durationMs: Date.now() - startTime,
|
|
3505
|
-
capturedApi
|
|
3543
|
+
capturedApi
|
|
3506
3544
|
};
|
|
3507
3545
|
}
|
|
3508
3546
|
}
|
|
@@ -3524,14 +3562,32 @@ function findChromeExecutable() {
|
|
|
3524
3562
|
return;
|
|
3525
3563
|
}
|
|
3526
3564
|
async function getBrowser(_name = "chromium") {
|
|
3527
|
-
if (cachedBrowser
|
|
3528
|
-
|
|
3565
|
+
if (cachedBrowser) {
|
|
3566
|
+
if (cachedBrowser.isConnected())
|
|
3567
|
+
return cachedBrowser;
|
|
3568
|
+
try {
|
|
3569
|
+
await cachedBrowser.close();
|
|
3570
|
+
} catch (e) {
|
|
3571
|
+
log5.warn(`stale browser close failed: ${e}`);
|
|
3572
|
+
}
|
|
3573
|
+
cachedBrowser = null;
|
|
3574
|
+
}
|
|
3529
3575
|
cachedBrowser = await import_patchright.chromium.launch({
|
|
3530
3576
|
headless: true,
|
|
3531
3577
|
args: STEALTH_ARGS
|
|
3532
3578
|
});
|
|
3533
3579
|
return cachedBrowser;
|
|
3534
3580
|
}
|
|
3581
|
+
async function closeBrowser() {
|
|
3582
|
+
if (!cachedBrowser)
|
|
3583
|
+
return;
|
|
3584
|
+
try {
|
|
3585
|
+
await cachedBrowser.close();
|
|
3586
|
+
} catch (e) {
|
|
3587
|
+
log5.warn(`closeBrowser failed: ${e}`);
|
|
3588
|
+
}
|
|
3589
|
+
cachedBrowser = null;
|
|
3590
|
+
}
|
|
3535
3591
|
async function getBrowserWithFallback(_preferred) {
|
|
3536
3592
|
return { browser: await getBrowser(), name: "chromium" };
|
|
3537
3593
|
}
|
|
@@ -4003,14 +4059,70 @@ var init_chrome_downloader = __esm(() => {
|
|
|
4003
4059
|
DEFAULT_INSTALL_DIR = import_path5.default.join(import_os2.default.homedir(), ".iframer", "chrome");
|
|
4004
4060
|
});
|
|
4005
4061
|
|
|
4062
|
+
// src/lib/browser/cloak-browser.ts
|
|
4063
|
+
async function tryImport() {
|
|
4064
|
+
try {
|
|
4065
|
+
return await import("cloakbrowser");
|
|
4066
|
+
} catch {
|
|
4067
|
+
return null;
|
|
4068
|
+
}
|
|
4069
|
+
}
|
|
4070
|
+
async function ensureBinary() {
|
|
4071
|
+
const cloak = await tryImport();
|
|
4072
|
+
if (!cloak)
|
|
4073
|
+
return false;
|
|
4074
|
+
try {
|
|
4075
|
+
const info = cloak.binaryInfo();
|
|
4076
|
+
if (!info.installed) {
|
|
4077
|
+
log8.info("Downloading CloakBrowser binary...");
|
|
4078
|
+
await cloak.ensureBinary();
|
|
4079
|
+
log8.info("CloakBrowser ready");
|
|
4080
|
+
}
|
|
4081
|
+
_available = true;
|
|
4082
|
+
return true;
|
|
4083
|
+
} catch (err) {
|
|
4084
|
+
log8.warn(`CloakBrowser setup failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
4085
|
+
_available = false;
|
|
4086
|
+
return false;
|
|
4087
|
+
}
|
|
4088
|
+
}
|
|
4089
|
+
async function launchCloakBrowser(options) {
|
|
4090
|
+
const cloak = await tryImport();
|
|
4091
|
+
if (!cloak)
|
|
4092
|
+
return null;
|
|
4093
|
+
try {
|
|
4094
|
+
const ok = await ensureBinary();
|
|
4095
|
+
if (!ok)
|
|
4096
|
+
return null;
|
|
4097
|
+
const browser = await cloak.launch({
|
|
4098
|
+
headless: options.headless,
|
|
4099
|
+
args: options.args
|
|
4100
|
+
});
|
|
4101
|
+
log8.info(`CloakBrowser launched (headless=${options.headless})`);
|
|
4102
|
+
return browser;
|
|
4103
|
+
} catch (err) {
|
|
4104
|
+
log8.warn(`CloakBrowser launch failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
4105
|
+
return null;
|
|
4106
|
+
}
|
|
4107
|
+
}
|
|
4108
|
+
var log8, _available = null;
|
|
4109
|
+
var init_cloak_browser = __esm(() => {
|
|
4110
|
+
init_logger();
|
|
4111
|
+
log8 = createLogger("cloak");
|
|
4112
|
+
});
|
|
4113
|
+
|
|
4006
4114
|
// src/lib/browser/daemon.ts
|
|
4115
|
+
function keyOf(mode, instanceId) {
|
|
4116
|
+
return `${mode}::${instanceId}`;
|
|
4117
|
+
}
|
|
4118
|
+
|
|
4007
4119
|
class BrowserDaemon {
|
|
4008
4120
|
instances = new Map;
|
|
4009
4121
|
idleTimers = new Map;
|
|
4010
4122
|
idleTimeout;
|
|
4011
4123
|
constructor(idleTimeout = DEFAULT_IDLE_TIMEOUT) {
|
|
4012
4124
|
this.idleTimeout = idleTimeout;
|
|
4013
|
-
const cleanup = () => this.stopAll().catch((err) =>
|
|
4125
|
+
const cleanup = () => this.stopAll().catch((err) => log9.warn(`cleanup failed: ${err}`));
|
|
4014
4126
|
process.on("exit", cleanup);
|
|
4015
4127
|
process.on("SIGINT", () => {
|
|
4016
4128
|
cleanup();
|
|
@@ -4021,11 +4133,12 @@ class BrowserDaemon {
|
|
|
4021
4133
|
process.exit(0);
|
|
4022
4134
|
});
|
|
4023
4135
|
}
|
|
4024
|
-
async ensure(mode) {
|
|
4136
|
+
async ensure(mode, instanceId = DEFAULT_INSTANCE) {
|
|
4025
4137
|
if (mode === "docker-headful") {
|
|
4026
4138
|
throw new Error("Docker mode doesn't use the daemon. Use the Docker API.");
|
|
4027
4139
|
}
|
|
4028
|
-
|
|
4140
|
+
const key = keyOf(mode, instanceId);
|
|
4141
|
+
let instance = this.instances.get(key);
|
|
4029
4142
|
if (instance) {
|
|
4030
4143
|
try {
|
|
4031
4144
|
if (instance.browser.isConnected()) {
|
|
@@ -4034,7 +4147,7 @@ class BrowserDaemon {
|
|
|
4034
4147
|
try {
|
|
4035
4148
|
await page2.evaluate("1");
|
|
4036
4149
|
} catch {
|
|
4037
|
-
|
|
4150
|
+
log9.info(`Page for ${mode} is dead, creating fresh context`);
|
|
4038
4151
|
try {
|
|
4039
4152
|
await context2.close();
|
|
4040
4153
|
} catch {}
|
|
@@ -4043,26 +4156,34 @@ class BrowserDaemon {
|
|
|
4043
4156
|
instance.context = context2;
|
|
4044
4157
|
instance.page = page2;
|
|
4045
4158
|
}
|
|
4046
|
-
this.resetIdleTimer(
|
|
4159
|
+
this.resetIdleTimer(key);
|
|
4047
4160
|
return { browser: instance.browser, context: context2, page: page2 };
|
|
4048
4161
|
}
|
|
4049
4162
|
} catch {}
|
|
4050
|
-
|
|
4051
|
-
|
|
4052
|
-
|
|
4053
|
-
|
|
4054
|
-
const
|
|
4055
|
-
|
|
4056
|
-
|
|
4057
|
-
|
|
4058
|
-
|
|
4059
|
-
|
|
4060
|
-
|
|
4061
|
-
|
|
4062
|
-
|
|
4063
|
-
|
|
4064
|
-
|
|
4065
|
-
|
|
4163
|
+
log9.info(`Browser for ${key} disconnected (window closed?), relaunching...`);
|
|
4164
|
+
await this.stopMode(mode, instanceId);
|
|
4165
|
+
}
|
|
4166
|
+
let browser;
|
|
4167
|
+
const cloakBrowser = await launchCloakBrowser({ headless: mode === "headless" });
|
|
4168
|
+
if (cloakBrowser) {
|
|
4169
|
+
log9.info(`CloakBrowser ${mode} ready`);
|
|
4170
|
+
browser = cloakBrowser;
|
|
4171
|
+
} else {
|
|
4172
|
+
const executablePath = await ensureChrome();
|
|
4173
|
+
log9.info(`Falling back to Chrome for Testing in ${mode} mode: ${executablePath}`);
|
|
4174
|
+
const userDataDir = import_path6.default.join(import_os3.default.homedir(), ".iframer", "chrome-profile", mode);
|
|
4175
|
+
import_fs7.default.mkdirSync(userDataDir, { recursive: true });
|
|
4176
|
+
browser = await import_patchright2.chromium.launch({
|
|
4177
|
+
executablePath,
|
|
4178
|
+
headless: mode === "headless",
|
|
4179
|
+
args: [
|
|
4180
|
+
"--disable-blink-features=AutomationControlled",
|
|
4181
|
+
"--no-first-run",
|
|
4182
|
+
"--no-default-browser-check",
|
|
4183
|
+
"--disable-infobars"
|
|
4184
|
+
]
|
|
4185
|
+
});
|
|
4186
|
+
}
|
|
4066
4187
|
const context = await browser.newContext();
|
|
4067
4188
|
const page = await context.newPage();
|
|
4068
4189
|
instance = {
|
|
@@ -4070,15 +4191,16 @@ class BrowserDaemon {
|
|
|
4070
4191
|
context,
|
|
4071
4192
|
page,
|
|
4072
4193
|
mode,
|
|
4194
|
+
instanceId,
|
|
4073
4195
|
createdAt: new Date
|
|
4074
4196
|
};
|
|
4075
|
-
this.instances.set(
|
|
4076
|
-
this.resetIdleTimer(
|
|
4077
|
-
|
|
4197
|
+
this.instances.set(key, instance);
|
|
4198
|
+
this.resetIdleTimer(key);
|
|
4199
|
+
log9.info(`Chrome ${key} ready`);
|
|
4078
4200
|
return { browser, context, page };
|
|
4079
4201
|
}
|
|
4080
|
-
isRunning(mode) {
|
|
4081
|
-
const instance = this.instances.get(mode);
|
|
4202
|
+
isRunning(mode, instanceId = DEFAULT_INSTANCE) {
|
|
4203
|
+
const instance = this.instances.get(keyOf(mode, instanceId));
|
|
4082
4204
|
if (!instance)
|
|
4083
4205
|
return false;
|
|
4084
4206
|
try {
|
|
@@ -4087,6 +4209,9 @@ class BrowserDaemon {
|
|
|
4087
4209
|
return false;
|
|
4088
4210
|
}
|
|
4089
4211
|
}
|
|
4212
|
+
runningModes() {
|
|
4213
|
+
return [...new Set(this.liveInstances().map((i) => i.mode))];
|
|
4214
|
+
}
|
|
4090
4215
|
liveInstances() {
|
|
4091
4216
|
return [...this.instances.values()].filter((inst) => {
|
|
4092
4217
|
try {
|
|
@@ -4096,46 +4221,50 @@ class BrowserDaemon {
|
|
|
4096
4221
|
}
|
|
4097
4222
|
});
|
|
4098
4223
|
}
|
|
4099
|
-
async stopMode(mode) {
|
|
4100
|
-
|
|
4224
|
+
async stopMode(mode, instanceId = DEFAULT_INSTANCE) {
|
|
4225
|
+
await this.stopKey(keyOf(mode, instanceId));
|
|
4226
|
+
}
|
|
4227
|
+
async stopKey(key) {
|
|
4228
|
+
const instance = this.instances.get(key);
|
|
4101
4229
|
if (!instance)
|
|
4102
4230
|
return;
|
|
4103
|
-
const timer = this.idleTimers.get(
|
|
4231
|
+
const timer = this.idleTimers.get(key);
|
|
4104
4232
|
if (timer)
|
|
4105
4233
|
clearTimeout(timer);
|
|
4106
|
-
this.idleTimers.delete(
|
|
4107
|
-
|
|
4234
|
+
this.idleTimers.delete(key);
|
|
4235
|
+
log9.info(`Stopping Chrome ${key}...`);
|
|
4108
4236
|
try {
|
|
4109
4237
|
await instance.context.close();
|
|
4110
4238
|
} catch {}
|
|
4111
4239
|
try {
|
|
4112
4240
|
await instance.browser.close();
|
|
4113
4241
|
} catch {}
|
|
4114
|
-
this.instances.delete(
|
|
4242
|
+
this.instances.delete(key);
|
|
4115
4243
|
}
|
|
4116
4244
|
async stopAll() {
|
|
4117
|
-
const
|
|
4118
|
-
await Promise.all(
|
|
4245
|
+
const keys = [...this.instances.keys()];
|
|
4246
|
+
await Promise.all(keys.map((k) => this.stopKey(k)));
|
|
4119
4247
|
}
|
|
4120
|
-
resetIdleTimer(
|
|
4121
|
-
const existing = this.idleTimers.get(
|
|
4248
|
+
resetIdleTimer(key) {
|
|
4249
|
+
const existing = this.idleTimers.get(key);
|
|
4122
4250
|
if (existing)
|
|
4123
4251
|
clearTimeout(existing);
|
|
4124
|
-
this.idleTimers.set(
|
|
4125
|
-
|
|
4126
|
-
this.
|
|
4252
|
+
this.idleTimers.set(key, setTimeout(() => {
|
|
4253
|
+
log9.info(`Idle timeout for ${key}, stopping...`);
|
|
4254
|
+
this.stopKey(key);
|
|
4127
4255
|
}, this.idleTimeout));
|
|
4128
4256
|
}
|
|
4129
4257
|
}
|
|
4130
|
-
var import_patchright2, import_os3, import_path6, import_fs7,
|
|
4258
|
+
var import_patchright2, import_os3, import_path6, import_fs7, log9, DEFAULT_IDLE_TIMEOUT, DEFAULT_INSTANCE = "default";
|
|
4131
4259
|
var init_daemon = __esm(() => {
|
|
4132
4260
|
init_chrome_downloader();
|
|
4261
|
+
init_cloak_browser();
|
|
4133
4262
|
init_logger();
|
|
4134
4263
|
import_patchright2 = require("patchright");
|
|
4135
4264
|
import_os3 = __toESM(require("os"));
|
|
4136
4265
|
import_path6 = __toESM(require("path"));
|
|
4137
4266
|
import_fs7 = __toESM(require("fs"));
|
|
4138
|
-
|
|
4267
|
+
log9 = createLogger("daemon");
|
|
4139
4268
|
DEFAULT_IDLE_TIMEOUT = 5 * 60 * 1000;
|
|
4140
4269
|
});
|
|
4141
4270
|
|
|
@@ -4232,17 +4361,17 @@ class DomainModeStore {
|
|
|
4232
4361
|
import_fs8.default.mkdirSync(import_path7.default.dirname(this.filePath), { recursive: true });
|
|
4233
4362
|
import_fs8.default.writeFileSync(this.filePath, JSON.stringify(this.data, null, 2));
|
|
4234
4363
|
} catch (err) {
|
|
4235
|
-
|
|
4364
|
+
log10.error("Failed to save:", err);
|
|
4236
4365
|
}
|
|
4237
4366
|
}
|
|
4238
4367
|
}
|
|
4239
|
-
var import_fs8, import_path7,
|
|
4368
|
+
var import_fs8, import_path7, log10, TTL_DAYS = 14, ESCALATION_LADDER;
|
|
4240
4369
|
var init_domain_modes = __esm(() => {
|
|
4241
4370
|
init_logger();
|
|
4242
4371
|
init_paths();
|
|
4243
4372
|
import_fs8 = __toESM(require("fs"));
|
|
4244
4373
|
import_path7 = __toESM(require("path"));
|
|
4245
|
-
|
|
4374
|
+
log10 = createLogger("domain-modes");
|
|
4246
4375
|
ESCALATION_LADDER = ["headless", "docker-headful", "binary-headful"];
|
|
4247
4376
|
});
|
|
4248
4377
|
|
|
@@ -4266,7 +4395,7 @@ async function detectBlock(page) {
|
|
|
4266
4395
|
const hasCfChallenge = await page.evaluate(() => {
|
|
4267
4396
|
return !!document.querySelector('iframe[src*="challenges.cloudflare.com"]');
|
|
4268
4397
|
}).catch((err) => {
|
|
4269
|
-
|
|
4398
|
+
log11.warn(`CF challenge check failed, assuming blocked: ${err}`);
|
|
4270
4399
|
return true;
|
|
4271
4400
|
});
|
|
4272
4401
|
if (hasCfChallenge) {
|
|
@@ -4285,7 +4414,7 @@ async function detectBlock(page) {
|
|
|
4285
4414
|
const hasCaptchaIframe = await page.evaluate(() => {
|
|
4286
4415
|
return !!(document.querySelector('iframe[src*="recaptcha"]') || document.querySelector('iframe[src*="hcaptcha"]'));
|
|
4287
4416
|
}).catch((err) => {
|
|
4288
|
-
|
|
4417
|
+
log11.warn(`captcha iframe check failed, assuming blocked: ${err}`);
|
|
4289
4418
|
return true;
|
|
4290
4419
|
});
|
|
4291
4420
|
if (hasCaptchaIframe) {
|
|
@@ -4298,15 +4427,15 @@ async function detectBlock(page) {
|
|
|
4298
4427
|
if (!url.includes("about:blank") && bodyText.trim().length < 20 && title.length < 5) {}
|
|
4299
4428
|
return { blocked: false };
|
|
4300
4429
|
} catch (err) {
|
|
4301
|
-
|
|
4430
|
+
log11.warn(`page evaluation failed, assuming blocked: ${err}`);
|
|
4302
4431
|
return { blocked: true, reason: "evaluation-failed" };
|
|
4303
4432
|
}
|
|
4304
4433
|
}
|
|
4305
|
-
var
|
|
4434
|
+
var log11;
|
|
4306
4435
|
var init_block_detection = __esm(() => {
|
|
4307
4436
|
init_constants();
|
|
4308
4437
|
init_logger();
|
|
4309
|
-
|
|
4438
|
+
log11 = createLogger("block-detection");
|
|
4310
4439
|
});
|
|
4311
4440
|
|
|
4312
4441
|
// src/lib/browser/cdp-launcher.ts
|
|
@@ -4321,11 +4450,11 @@ function checkModeAvailability() {
|
|
|
4321
4450
|
binaryHeadful: hasDisplay()
|
|
4322
4451
|
};
|
|
4323
4452
|
}
|
|
4324
|
-
var
|
|
4453
|
+
var log12;
|
|
4325
4454
|
var init_cdp_launcher = __esm(() => {
|
|
4326
4455
|
init_chrome_downloader();
|
|
4327
4456
|
init_logger();
|
|
4328
|
-
|
|
4457
|
+
log12 = createLogger("cdp-launcher");
|
|
4329
4458
|
});
|
|
4330
4459
|
|
|
4331
4460
|
// src/lib/iframer.ts
|
|
@@ -4346,6 +4475,7 @@ class Iframer {
|
|
|
4346
4475
|
daemon;
|
|
4347
4476
|
domainModes;
|
|
4348
4477
|
operatingMode;
|
|
4478
|
+
persistentCaptures = new Map;
|
|
4349
4479
|
constructor(config = {}) {
|
|
4350
4480
|
this.screenshotDir = config.screenshotDir || DEFAULT_SCREENSHOT_DIR;
|
|
4351
4481
|
this.publicUrl = config.publicUrl || DEFAULT_PUBLIC_URL;
|
|
@@ -4498,6 +4628,9 @@ class Iframer {
|
|
|
4498
4628
|
getSession(userId) {
|
|
4499
4629
|
return getSession(userId);
|
|
4500
4630
|
}
|
|
4631
|
+
sessionKey(userId, instanceId = DEFAULT_INSTANCE) {
|
|
4632
|
+
return instanceId === DEFAULT_INSTANCE ? userId : `${userId}::${instanceId}`;
|
|
4633
|
+
}
|
|
4501
4634
|
async stopSession(userId, token) {
|
|
4502
4635
|
let sessionSaved = false;
|
|
4503
4636
|
if (token) {
|
|
@@ -4507,12 +4640,11 @@ class Iframer {
|
|
|
4507
4640
|
const data = await extractSession(inst.context, inst.page);
|
|
4508
4641
|
if (data) {
|
|
4509
4642
|
const encrypted = encrypt(JSON.stringify(data), encryptionKey);
|
|
4510
|
-
await this.store.setSession(userId, encrypted);
|
|
4643
|
+
await this.store.setSession(this.sessionKey(userId, inst.instanceId), encrypted);
|
|
4511
4644
|
sessionSaved = true;
|
|
4512
|
-
break;
|
|
4513
4645
|
}
|
|
4514
4646
|
} catch (err) {
|
|
4515
|
-
|
|
4647
|
+
log13.warn(`stopSession: failed to extract daemon state for ${inst.mode}::${inst.instanceId}: ${getErrorMessage2(err)}`);
|
|
4516
4648
|
}
|
|
4517
4649
|
}
|
|
4518
4650
|
}
|
|
@@ -4539,6 +4671,7 @@ class Iframer {
|
|
|
4539
4671
|
const opts = pipeline.options || {};
|
|
4540
4672
|
const forcedMode = opts.mode;
|
|
4541
4673
|
const autoEscalate = opts.autoEscalate !== false;
|
|
4674
|
+
const instanceId = opts.instanceId || DEFAULT_INSTANCE;
|
|
4542
4675
|
const firstNav = pipeline.steps.find((s) => s.type === "navigate");
|
|
4543
4676
|
const domain = firstNav ? new URL(firstNav.url).hostname : null;
|
|
4544
4677
|
const availableModes = this.getAvailableModes();
|
|
@@ -4550,18 +4683,18 @@ class Iframer {
|
|
|
4550
4683
|
} else {
|
|
4551
4684
|
mode = availableModes[0] || "headless";
|
|
4552
4685
|
}
|
|
4553
|
-
let result = await this.executeWithMode(userId, token, pipeline, mode);
|
|
4686
|
+
let result = await this.executeWithMode(userId, token, pipeline, mode, instanceId);
|
|
4554
4687
|
if (!result.ok && autoEscalate && domain && result.error?.errorType === "bot-blocked") {
|
|
4555
4688
|
const failedMode = mode;
|
|
4556
4689
|
if (domain)
|
|
4557
4690
|
this.domainModes.recordFailure(domain, failedMode, result.error?.message || "blocked");
|
|
4558
4691
|
const nextMode = this.domainModes.getNextMode(failedMode, availableModes);
|
|
4559
4692
|
if (nextMode) {
|
|
4560
|
-
|
|
4693
|
+
log13.info(`Auto-escalating from ${failedMode} to ${nextMode} for ${domain}`);
|
|
4561
4694
|
if (failedMode !== "docker-headful") {
|
|
4562
|
-
await this.daemon.stopMode(failedMode);
|
|
4695
|
+
await this.daemon.stopMode(failedMode, instanceId);
|
|
4563
4696
|
}
|
|
4564
|
-
result = await this.executeWithMode(userId, token, pipeline, nextMode);
|
|
4697
|
+
result = await this.executeWithMode(userId, token, pipeline, nextMode, instanceId);
|
|
4565
4698
|
result.modeEscalated = true;
|
|
4566
4699
|
result.modeUsed = nextMode;
|
|
4567
4700
|
if (result.ok && domain) {
|
|
@@ -4570,11 +4703,11 @@ class Iframer {
|
|
|
4570
4703
|
this.domainModes.recordFailure(domain, nextMode, result.error?.message || "blocked");
|
|
4571
4704
|
const thirdMode = this.domainModes.getNextMode(nextMode, availableModes);
|
|
4572
4705
|
if (thirdMode) {
|
|
4573
|
-
|
|
4706
|
+
log13.info(`Auto-escalating from ${nextMode} to ${thirdMode} for ${domain}`);
|
|
4574
4707
|
if (nextMode !== "docker-headful") {
|
|
4575
|
-
await this.daemon.stopMode(nextMode);
|
|
4708
|
+
await this.daemon.stopMode(nextMode, instanceId);
|
|
4576
4709
|
}
|
|
4577
|
-
result = await this.executeWithMode(userId, token, pipeline, thirdMode);
|
|
4710
|
+
result = await this.executeWithMode(userId, token, pipeline, thirdMode, instanceId);
|
|
4578
4711
|
result.modeEscalated = true;
|
|
4579
4712
|
result.modeUsed = thirdMode;
|
|
4580
4713
|
if (result.ok && domain) {
|
|
@@ -4588,11 +4721,11 @@ class Iframer {
|
|
|
4588
4721
|
}
|
|
4589
4722
|
return result;
|
|
4590
4723
|
}
|
|
4591
|
-
async executeWithMode(userId, token, pipeline, mode) {
|
|
4724
|
+
async executeWithMode(userId, token, pipeline, mode, instanceId = DEFAULT_INSTANCE) {
|
|
4592
4725
|
if (mode === "docker-headful") {
|
|
4593
4726
|
return this.executeDocker(userId, token, pipeline);
|
|
4594
4727
|
}
|
|
4595
|
-
return this.executeLocal(userId, token, pipeline, mode);
|
|
4728
|
+
return this.executeLocal(userId, token, pipeline, mode, instanceId);
|
|
4596
4729
|
}
|
|
4597
4730
|
async executeDocker(userId, token, pipeline) {
|
|
4598
4731
|
let session = getSession(userId);
|
|
@@ -4631,12 +4764,13 @@ class Iframer {
|
|
|
4631
4764
|
result.modeUsed = "docker-headful";
|
|
4632
4765
|
return result;
|
|
4633
4766
|
}
|
|
4634
|
-
async executeLocal(userId, token, pipeline, mode) {
|
|
4767
|
+
async executeLocal(userId, token, pipeline, mode, instanceId = DEFAULT_INSTANCE) {
|
|
4635
4768
|
const startTime = Date.now();
|
|
4636
4769
|
try {
|
|
4637
|
-
const { page } = await this.daemon.ensure(mode);
|
|
4770
|
+
const { page } = await this.daemon.ensure(mode, instanceId);
|
|
4771
|
+
const storeKey = this.sessionKey(userId, instanceId);
|
|
4638
4772
|
const encryptionKey = await deriveKey(token);
|
|
4639
|
-
const blob = await this.store.getSession(
|
|
4773
|
+
const blob = await this.store.getSession(storeKey);
|
|
4640
4774
|
let sessionData = null;
|
|
4641
4775
|
if (blob && blob.length > 0) {
|
|
4642
4776
|
try {
|
|
@@ -4676,14 +4810,14 @@ class Iframer {
|
|
|
4676
4810
|
try {
|
|
4677
4811
|
updatedSession = await extractSession(page.context(), page);
|
|
4678
4812
|
const encrypted = encrypt(JSON.stringify(updatedSession), encryptionKey);
|
|
4679
|
-
await this.store.setSession(
|
|
4813
|
+
await this.store.setSession(storeKey, encrypted);
|
|
4680
4814
|
} catch {}
|
|
4681
4815
|
}
|
|
4682
4816
|
if (result.ok) {
|
|
4683
4817
|
try {
|
|
4684
4818
|
this.updateKnowledgeFromRun(pipeline, result, updatedSession, mode);
|
|
4685
4819
|
} catch (err) {
|
|
4686
|
-
|
|
4820
|
+
log13.warn(`knowledge update failed: ${getErrorMessage2(err)}`);
|
|
4687
4821
|
}
|
|
4688
4822
|
}
|
|
4689
4823
|
const refs = this.userRefs.get(userId);
|
|
@@ -4814,12 +4948,66 @@ class Iframer {
|
|
|
4814
4948
|
return { url: "", title: "" };
|
|
4815
4949
|
}
|
|
4816
4950
|
}
|
|
4951
|
+
async startCapture(mode = "binary-headful", instanceId = DEFAULT_INSTANCE) {
|
|
4952
|
+
const key = `${mode}::${instanceId}`;
|
|
4953
|
+
if (this.persistentCaptures.has(key)) {
|
|
4954
|
+
return { ok: true, message: `Capture already running on ${key}. Call capture-stop to flush.` };
|
|
4955
|
+
}
|
|
4956
|
+
const { page } = await this.daemon.ensure(mode, instanceId);
|
|
4957
|
+
const capture = new ApiCapture(page);
|
|
4958
|
+
capture.start();
|
|
4959
|
+
this.persistentCaptures.set(key, capture);
|
|
4960
|
+
return { ok: true, message: `Capture started on ${key}. Use 'session capture-stop' when ready to collect results.` };
|
|
4961
|
+
}
|
|
4962
|
+
async stopCapture(mode = "binary-headful", instanceId = DEFAULT_INSTANCE) {
|
|
4963
|
+
const key = `${mode}::${instanceId}`;
|
|
4964
|
+
const capture = this.persistentCaptures.get(key);
|
|
4965
|
+
if (!capture) {
|
|
4966
|
+
return { ok: false, capturedApi: undefined, message: `No active capture on ${key}. Start one with 'session capture-start'.` };
|
|
4967
|
+
}
|
|
4968
|
+
capture.stop();
|
|
4969
|
+
this.persistentCaptures.delete(key);
|
|
4970
|
+
const capturedApi = capture.getResults();
|
|
4971
|
+
const total = capturedApi.reduce((n, a) => n + a.endpoints.length, 0);
|
|
4972
|
+
return { ok: true, capturedApi, message: `Capture stopped. ${total} endpoints across ${capturedApi.length} domain(s).` };
|
|
4973
|
+
}
|
|
4974
|
+
async getCookies(mode = "binary-headful", urls, instanceId = DEFAULT_INSTANCE) {
|
|
4975
|
+
const { context } = await this.daemon.ensure(mode, instanceId);
|
|
4976
|
+
const cookies = urls && urls.length > 0 ? await context.cookies(urls) : await context.cookies();
|
|
4977
|
+
return { ok: true, cookies, message: `${cookies.length} cookies extracted via CDP.` };
|
|
4978
|
+
}
|
|
4979
|
+
async getFullAuth(mode = "binary-headful", urls, instanceId = DEFAULT_INSTANCE) {
|
|
4980
|
+
const { context, page } = await this.daemon.ensure(mode, instanceId);
|
|
4981
|
+
const cookies = urls && urls.length > 0 ? await context.cookies(urls) : await context.cookies();
|
|
4982
|
+
const localStorage = {};
|
|
4983
|
+
const sessionStorage = {};
|
|
4984
|
+
try {
|
|
4985
|
+
const stores = await page.evaluate(() => {
|
|
4986
|
+
const ls = {};
|
|
4987
|
+
const ss = {};
|
|
4988
|
+
for (let i = 0;i < window.localStorage.length; i++) {
|
|
4989
|
+
const k = window.localStorage.key(i);
|
|
4990
|
+
ls[k] = window.localStorage.getItem(k) ?? "";
|
|
4991
|
+
}
|
|
4992
|
+
for (let i = 0;i < window.sessionStorage.length; i++) {
|
|
4993
|
+
const k = window.sessionStorage.key(i);
|
|
4994
|
+
ss[k] = window.sessionStorage.getItem(k) ?? "";
|
|
4995
|
+
}
|
|
4996
|
+
return { origin: window.location.origin, ls, ss };
|
|
4997
|
+
});
|
|
4998
|
+
localStorage[stores.origin] = stores.ls;
|
|
4999
|
+
sessionStorage[stores.origin] = stores.ss;
|
|
5000
|
+
} catch {}
|
|
5001
|
+
return {
|
|
5002
|
+
ok: true,
|
|
5003
|
+
cookies,
|
|
5004
|
+
localStorage,
|
|
5005
|
+
sessionStorage,
|
|
5006
|
+
message: `${cookies.length} cookies, ${Object.values(localStorage).reduce((n, s) => n + Object.keys(s).length, 0)} localStorage keys, ${Object.values(sessionStorage).reduce((n, s) => n + Object.keys(s).length, 0)} sessionStorage keys.`
|
|
5007
|
+
};
|
|
5008
|
+
}
|
|
4817
5009
|
browserHealth() {
|
|
4818
|
-
const modes =
|
|
4819
|
-
for (const m of ["headless", "binary-headful"]) {
|
|
4820
|
-
if (this.daemon.isRunning(m))
|
|
4821
|
-
modes.push(m);
|
|
4822
|
-
}
|
|
5010
|
+
const modes = this.daemon.runningModes();
|
|
4823
5011
|
return { alive: modes.length > 0, modes };
|
|
4824
5012
|
}
|
|
4825
5013
|
async restartBrowser() {
|
|
@@ -4924,13 +5112,14 @@ class Iframer {
|
|
|
4924
5112
|
}
|
|
4925
5113
|
async shutdown() {
|
|
4926
5114
|
await this.daemon.stopAll();
|
|
5115
|
+
await closeBrowser();
|
|
4927
5116
|
await cleanupAllSessions();
|
|
4928
5117
|
if ("close" in this.store && typeof this.store.close === "function") {
|
|
4929
5118
|
this.store.close();
|
|
4930
5119
|
}
|
|
4931
5120
|
}
|
|
4932
5121
|
}
|
|
4933
|
-
var import_path8,
|
|
5122
|
+
var import_path8, log13, DEFAULT_SCREENSHOT_DIR, DEFAULT_PUBLIC_URL, DEFAULT_STALE_TIMEOUT_MS3 = 20000;
|
|
4934
5123
|
var init_iframer = __esm(() => {
|
|
4935
5124
|
init_pipeline();
|
|
4936
5125
|
init_session_manager();
|
|
@@ -4942,13 +5131,14 @@ var init_iframer = __esm(() => {
|
|
|
4942
5131
|
init_screenshot();
|
|
4943
5132
|
init_storage();
|
|
4944
5133
|
init_daemon();
|
|
5134
|
+
init_api_capture();
|
|
4945
5135
|
init_domain_modes();
|
|
4946
5136
|
init_block_detection();
|
|
4947
5137
|
init_cdp_launcher();
|
|
4948
5138
|
init_constants();
|
|
4949
5139
|
init_logger();
|
|
4950
5140
|
import_path8 = __toESM(require("path"));
|
|
4951
|
-
|
|
5141
|
+
log13 = createLogger("iframer");
|
|
4952
5142
|
DEFAULT_SCREENSHOT_DIR = import_path8.default.join("/Users/eduardoverona/tools/iframer-toolkit/src/lib", "../../.screenshots");
|
|
4953
5143
|
DEFAULT_PUBLIC_URL = process.env.PUBLIC_URL || `http://localhost:${process.env.PORT || 3021}`;
|
|
4954
5144
|
});
|