devez-vibe 1.10.21 → 1.11.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -21
- package/README.md +73 -73
- package/bin/dvz.exe +0 -0
- package/bin/dvz.js +35 -35
- package/bridge/claude-agent-sdk-bridge.mjs +364 -35
- package/install-skills.mjs +11 -11
- package/package.json +50 -50
- package/prepare-search-deps.mjs +42 -42
- package/skills/insane-search/SKILL.md +36 -36
- package/skills/insane-search/engine/__main__.py +44 -44
- package/skills/insane-search/engine/executor.py +246 -246
- package/skills/insane-search/engine/fetch_chain.py +443 -443
- package/skills/insane-search/engine/network.py +108 -108
- package/skills/insane-search/engine/phase0.py +380 -380
- package/skills/insane-search/engine/session_input.py +120 -120
- package/skills/insane-search/engine/templates/invisible_playwright_fetch.py +57 -57
- package/skills/insane-search/engine/templates/nodriver_fetch.py +9 -9
- package/skills/insane-search/engine/templates/patchright_fetch.py +9 -9
- package/skills/insane-search/engine/templates/playwright_mobile_chrome.js +9 -9
- package/skills/insane-search/engine/templates/playwright_real_chrome.js +7 -7
- package/skills/insane-search/engine/templates/scrapling_fetch.py +46 -46
- package/skills/insane-search/engine/tests/test_smoke.py +8 -8
- package/skills/insane-search/engine/tests/test_t10_artifact_routes.py +126 -126
- package/skills/insane-search/engine/tests/test_t11_session_handoff.py +157 -157
- package/skills/insane-search/engine/tests/test_t5_pdfplumber.py +11 -11
- package/skills/insane-search/engine/tests/test_t7_browser_gate.py +3 -3
- package/skills/insane-search/engine/tests/test_t8_network_routes.py +207 -207
- package/skills/insane-search/engine/tests/test_t9_evidence_bundle.py +89 -89
- package/skills/insane-search/engine/tests/test_u5.py +9 -9
- package/skills/insane-search/engine/transport.py +94 -94
- package/skills/insane-search/engine/waf_profiles.yaml +38 -38
- package/skills/insane-search/references/cache-archive.md +13 -13
- package/skills/insane-search/references/media.md +14 -14
- package/skills/insane-search/references/playwright.md +27 -27
- package/skills/insane-search/references/research-evidence.md +36 -36
- package/skills/insane-search/references/session-handoff.md +26 -26
- package/skills/insane-search/tests/test_x_search.py +3 -3
- package/skills/luna-loop/agents/openai.yaml +7 -7
|
@@ -33,14 +33,14 @@ from .waf_detector import load_profile
|
|
|
33
33
|
from .fetch_chain import Attempt
|
|
34
34
|
|
|
35
35
|
|
|
36
|
-
TEMPLATES_DIR = os.path.join(os.path.dirname(__file__), "templates")
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
def _redact_diagnostics(text: Optional[str], proxy: Optional[str],
|
|
40
|
-
cookies: Optional[list[dict]]) -> Optional[str]:
|
|
41
|
-
from .network import redact_proxy
|
|
42
|
-
from .session_input import redact_cookie_values
|
|
43
|
-
return redact_cookie_values(redact_proxy(text, proxy), cookies)
|
|
36
|
+
TEMPLATES_DIR = os.path.join(os.path.dirname(__file__), "templates")
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _redact_diagnostics(text: Optional[str], proxy: Optional[str],
|
|
40
|
+
cookies: Optional[list[dict]]) -> Optional[str]:
|
|
41
|
+
from .network import redact_proxy
|
|
42
|
+
from .session_input import redact_cookie_values
|
|
43
|
+
return redact_cookie_values(redact_proxy(text, proxy), cookies)
|
|
44
44
|
|
|
45
45
|
|
|
46
46
|
def _profile_dir_for(url: str, choice: str) -> str:
|
|
@@ -150,26 +150,26 @@ def _module_available(name: str) -> bool:
|
|
|
150
150
|
return False
|
|
151
151
|
|
|
152
152
|
|
|
153
|
-
def _auto_install(pkg: str, module: Optional[str] = None) -> bool:
|
|
154
|
-
if os.environ.get("INSANE_AUTO_INSTALL", "").strip().lower() not in ("1", "true", "yes"):
|
|
155
|
-
return False
|
|
153
|
+
def _auto_install(pkg: str, module: Optional[str] = None) -> bool:
|
|
154
|
+
if os.environ.get("INSANE_AUTO_INSTALL", "").strip().lower() not in ("1", "true", "yes"):
|
|
155
|
+
return False
|
|
156
156
|
try:
|
|
157
157
|
subprocess.run([sys.executable, "-m", "pip", "install", pkg, "-q"],
|
|
158
158
|
capture_output=True, timeout=180, check=False)
|
|
159
159
|
except Exception:
|
|
160
160
|
return False
|
|
161
161
|
importlib.invalidate_caches()
|
|
162
|
-
return _module_available(module or pkg)
|
|
162
|
+
return _module_available(module or pkg)
|
|
163
163
|
|
|
164
164
|
|
|
165
|
-
def _run_python_template(template: str, args: dict, timeout: int = 90,
|
|
166
|
-
python_executable: Optional[str] = None) -> tuple[int, str, str]:
|
|
165
|
+
def _run_python_template(template: str, args: dict, timeout: int = 90,
|
|
166
|
+
python_executable: Optional[str] = None) -> tuple[int, str, str]:
|
|
167
167
|
path = os.path.join(TEMPLATES_DIR, template)
|
|
168
168
|
if not os.path.isfile(path):
|
|
169
169
|
return 127, "", f"template not found: {path}"
|
|
170
170
|
try:
|
|
171
171
|
proc = subprocess.run(
|
|
172
|
-
[python_executable or sys.executable, path], input=json.dumps(args), cwd=TEMPLATES_DIR,
|
|
172
|
+
[python_executable or sys.executable, path], input=json.dumps(args), cwd=TEMPLATES_DIR,
|
|
173
173
|
capture_output=True, text=True, timeout=timeout,
|
|
174
174
|
)
|
|
175
175
|
return proc.returncode, proc.stdout, proc.stderr
|
|
@@ -179,10 +179,10 @@ def _run_python_template(template: str, args: dict, timeout: int = 90,
|
|
|
179
179
|
return 1, "", f"{type(e).__name__}:{e}"
|
|
180
180
|
|
|
181
181
|
|
|
182
|
-
def _run_protocol_stealth(
|
|
183
|
-
att: Attempt, url: str, *, success_selectors: Optional[list[str]], timeout: int, t0: float,
|
|
184
|
-
proxy: Optional[str] = None, cookies: Optional[list[dict]] = None,
|
|
185
|
-
) -> tuple[Attempt, str]:
|
|
182
|
+
def _run_protocol_stealth(
|
|
183
|
+
att: Attempt, url: str, *, success_selectors: Optional[list[str]], timeout: int, t0: float,
|
|
184
|
+
proxy: Optional[str] = None, cookies: Optional[list[dict]] = None,
|
|
185
|
+
) -> tuple[Attempt, str]:
|
|
186
186
|
"""nodriver (raw CDP, no Playwright shim) first, patchright channel=chrome next.
|
|
187
187
|
|
|
188
188
|
For gates that fingerprint the automation protocol (Runtime.enable), every
|
|
@@ -190,25 +190,25 @@ def _run_protocol_stealth(
|
|
|
190
190
|
strongest free option, patchright the license-safe next. Missing drivers are
|
|
191
191
|
reported so the caller continues down its fallback list.
|
|
192
192
|
"""
|
|
193
|
-
args: dict = {"url": url, "timeout": timeout * 1000, "headless": True}
|
|
194
|
-
if proxy:
|
|
195
|
-
from .network import proxy_for_browser
|
|
196
|
-
args["proxy"] = proxy_for_browser(proxy)
|
|
197
|
-
if cookies:
|
|
198
|
-
args["cookies"] = cookies
|
|
193
|
+
args: dict = {"url": url, "timeout": timeout * 1000, "headless": True}
|
|
194
|
+
if proxy:
|
|
195
|
+
from .network import proxy_for_browser
|
|
196
|
+
args["proxy"] = proxy_for_browser(proxy)
|
|
197
|
+
if cookies:
|
|
198
|
+
args["cookies"] = cookies
|
|
199
199
|
if success_selectors:
|
|
200
200
|
args["waitSelector"] = success_selectors[0]
|
|
201
|
-
drivers = (("patchright", "patchright_fetch.py"),) if cookies else (
|
|
202
|
-
("nodriver", "nodriver_fetch.py"), ("patchright", "patchright_fetch.py"))
|
|
203
|
-
for pkg, template in drivers:
|
|
204
|
-
if not _module_available(pkg) and not _auto_install(pkg):
|
|
205
|
-
continue
|
|
206
|
-
att._executed = True
|
|
207
|
-
rc, stdout, stderr = _run_python_template(template, args, timeout=timeout + 30)
|
|
201
|
+
drivers = (("patchright", "patchright_fetch.py"),) if cookies else (
|
|
202
|
+
("nodriver", "nodriver_fetch.py"), ("patchright", "patchright_fetch.py"))
|
|
203
|
+
for pkg, template in drivers:
|
|
204
|
+
if not _module_available(pkg) and not _auto_install(pkg):
|
|
205
|
+
continue
|
|
206
|
+
att._executed = True
|
|
207
|
+
rc, stdout, stderr = _run_python_template(template, args, timeout=timeout + 30)
|
|
208
208
|
att.executor = f"protocol_stealth_chrome:{pkg}"
|
|
209
209
|
att.elapsed_s = round(time.time() - t0, 3)
|
|
210
|
-
if rc != 0 or not stdout:
|
|
211
|
-
att.error = f"{pkg}: {(_redact_diagnostics(stderr, proxy, cookies) or 'no stdout')[:200]}"
|
|
210
|
+
if rc != 0 or not stdout:
|
|
211
|
+
att.error = f"{pkg}: {(_redact_diagnostics(stderr, proxy, cookies) or 'no stdout')[:200]}"
|
|
212
212
|
continue
|
|
213
213
|
resp = _FakeResp(stdout)
|
|
214
214
|
vr = validate(resp, success_selectors=success_selectors)
|
|
@@ -221,165 +221,165 @@ def _run_protocol_stealth(
|
|
|
221
221
|
if not att.error:
|
|
222
222
|
att.error = "nodriver/patchright not installed (pip install nodriver, or INSANE_AUTO_INSTALL=1)"
|
|
223
223
|
att.verdict = Verdict.UNKNOWN.value
|
|
224
|
-
return att, ""
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
_SCRAPLING_PYTHON_CACHE: Optional[str] = None
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
def _scrapling_python() -> Optional[str]:
|
|
231
|
-
"""Return an isolated Scrapling runtime, creating it on first real use."""
|
|
232
|
-
global _SCRAPLING_PYTHON_CACHE
|
|
233
|
-
if _module_available("scrapling"):
|
|
234
|
-
return sys.executable
|
|
235
|
-
if _SCRAPLING_PYTHON_CACHE is not None:
|
|
236
|
-
return _SCRAPLING_PYTHON_CACHE or None
|
|
237
|
-
if os.environ.get("INSANE_AUTO_INSTALL", "").strip().lower() in ("0", "false", "no"):
|
|
238
|
-
_SCRAPLING_PYTHON_CACHE = ""
|
|
239
|
-
return None
|
|
240
|
-
|
|
241
|
-
root = os.path.expanduser("~/.insane-search/scrapling-venv")
|
|
242
|
-
python = os.path.join(root, "Scripts", "python.exe") if os.name == "nt" else os.path.join(root, "bin", "python")
|
|
243
|
-
try:
|
|
244
|
-
if not os.path.isfile(python):
|
|
245
|
-
subprocess.run(
|
|
246
|
-
[sys.executable, "-m", "venv", root], capture_output=True,
|
|
247
|
-
text=True, timeout=180, check=False)
|
|
248
|
-
probe = subprocess.run(
|
|
249
|
-
[python, "-c", "import scrapling"], capture_output=True,
|
|
250
|
-
text=True, timeout=30, check=False)
|
|
251
|
-
if probe.returncode != 0:
|
|
252
|
-
install = subprocess.run(
|
|
253
|
-
[python, "-m", "pip", "install", "scrapling[fetchers]>=0.4.8", "-q"],
|
|
254
|
-
capture_output=True, text=True, timeout=600, check=False)
|
|
255
|
-
if install.returncode != 0:
|
|
256
|
-
_SCRAPLING_PYTHON_CACHE = ""
|
|
257
|
-
return None
|
|
258
|
-
_SCRAPLING_PYTHON_CACHE = python
|
|
259
|
-
return python
|
|
260
|
-
except Exception:
|
|
261
|
-
_SCRAPLING_PYTHON_CACHE = ""
|
|
262
|
-
return None
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
def _run_scrapling(
|
|
266
|
-
att: Attempt, url: str, *, profile_id: str,
|
|
267
|
-
success_selectors: Optional[list[str]], timeout: int, t0: float,
|
|
268
|
-
proxy: Optional[str], cookies: Optional[list[dict]] = None,
|
|
269
|
-
) -> tuple[Attempt, str]:
|
|
270
|
-
"""Run Scrapling as the primary hidden browser fallback."""
|
|
271
|
-
python = _scrapling_python()
|
|
272
|
-
if python is None:
|
|
273
|
-
att.executor = "scrapling:stealthy_fetcher"
|
|
274
|
-
att.error = "Scrapling installation failed or is unavailable"
|
|
275
|
-
att.verdict = Verdict.UNKNOWN.value
|
|
276
|
-
att.elapsed_s = round(time.time() - t0, 3)
|
|
277
|
-
return att, ""
|
|
278
|
-
|
|
279
|
-
args: dict = {
|
|
280
|
-
"url": url,
|
|
281
|
-
"timeout": max(timeout, 60) * 1000,
|
|
282
|
-
"headless": True,
|
|
283
|
-
"realChrome": True,
|
|
284
|
-
"solveCloudflare": profile_id in ("cloudflare_turnstile", "unknown_challenge"),
|
|
285
|
-
"blockAds": True,
|
|
286
|
-
}
|
|
287
|
-
if success_selectors:
|
|
288
|
-
args["waitSelector"] = success_selectors[0]
|
|
289
|
-
if proxy:
|
|
290
|
-
from .network import proxy_for_browser
|
|
291
|
-
args["proxy"] = proxy_for_browser(proxy)
|
|
292
|
-
if cookies:
|
|
293
|
-
args["cookies"] = cookies
|
|
294
|
-
|
|
295
|
-
att._executed = True
|
|
296
|
-
if python == sys.executable:
|
|
297
|
-
rc, stdout, stderr = _run_python_template(
|
|
298
|
-
"scrapling_fetch.py", args, timeout=max(timeout, 60) + 30)
|
|
299
|
-
else:
|
|
300
|
-
rc, stdout, stderr = _run_python_template(
|
|
301
|
-
"scrapling_fetch.py", args, timeout=max(timeout, 60) + 30,
|
|
302
|
-
python_executable=python)
|
|
303
|
-
att.executor = "scrapling:stealthy_fetcher"
|
|
304
|
-
att.elapsed_s = round(time.time() - t0, 3)
|
|
305
|
-
if rc != 0 or not stdout:
|
|
306
|
-
att.error = (_redact_diagnostics(stderr, proxy, cookies) or "no stdout")[:300]
|
|
307
|
-
att.verdict = Verdict.UNKNOWN.value
|
|
308
|
-
return att, ""
|
|
309
|
-
|
|
310
|
-
html, final_url, status, cookies, user_agent, automation, inner_text = _parse_envelope(stdout, url)
|
|
311
|
-
resp = _FakeResp(html, status=status, final_url=final_url)
|
|
312
|
-
vr = validate(resp, success_selectors=success_selectors)
|
|
313
|
-
att.status = status
|
|
314
|
-
att.body_size = len(html)
|
|
315
|
-
att.verdict = vr.verdict.value
|
|
316
|
-
att.reasons = list(vr.reasons) + ([f"automation:{automation}"] if automation else [])
|
|
317
|
-
att.url = final_url or url
|
|
318
|
-
if inner_text:
|
|
319
|
-
att._inner_text = inner_text
|
|
320
|
-
return att, html
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
def _run_stealth_firefox(
|
|
324
|
-
att: Attempt, url: str, *, success_selectors: Optional[list[str]], timeout: int,
|
|
325
|
-
t0: float, profile_dir: Optional[str], proxy: Optional[str],
|
|
326
|
-
cookies: Optional[list[dict]] = None,
|
|
327
|
-
) -> tuple[Attempt, str]:
|
|
328
|
-
"""Run the optional Python-only stealth Firefox without a sidecar server."""
|
|
329
|
-
module = "invisible_playwright"
|
|
330
|
-
if not _module_available(module) and not _auto_install("invisible-playwright", module):
|
|
331
|
-
att.executor = "stealth_firefox:invisible_playwright"
|
|
332
|
-
att.error = (
|
|
333
|
-
"invisible-playwright not installed "
|
|
334
|
-
"(pip install invisible-playwright, or INSANE_AUTO_INSTALL=1)"
|
|
335
|
-
)
|
|
336
|
-
att.verdict = Verdict.UNKNOWN.value
|
|
337
|
-
att.elapsed_s = round(time.time() - t0, 3)
|
|
338
|
-
return att, ""
|
|
339
|
-
|
|
340
|
-
ephemeral_profile = None
|
|
341
|
-
if cookies and profile_dir is None:
|
|
342
|
-
ephemeral_profile = tempfile.mkdtemp(prefix="insane-firefox-session-")
|
|
343
|
-
args: dict = {
|
|
344
|
-
"url": url,
|
|
345
|
-
"timeout": timeout * 1000,
|
|
346
|
-
"profileDir": profile_dir or ephemeral_profile or _profile_dir_for(url, "stealth_firefox"),
|
|
347
|
-
"headless": True,
|
|
348
|
-
}
|
|
349
|
-
if success_selectors:
|
|
350
|
-
args["waitSelector"] = success_selectors[0]
|
|
351
|
-
if proxy:
|
|
352
|
-
from .network import proxy_for_browser
|
|
353
|
-
args["proxy"] = proxy_for_browser(proxy)
|
|
354
|
-
if cookies:
|
|
355
|
-
args["cookies"] = cookies
|
|
356
|
-
|
|
357
|
-
rc, stdout, stderr = _run_python_template(
|
|
358
|
-
"invisible_playwright_fetch.py", args, timeout=timeout + 30)
|
|
359
|
-
if ephemeral_profile:
|
|
360
|
-
shutil.rmtree(ephemeral_profile, ignore_errors=True)
|
|
361
|
-
att._executed = True
|
|
362
|
-
att.executor = "stealth_firefox:invisible_playwright"
|
|
363
|
-
att.elapsed_s = round(time.time() - t0, 3)
|
|
364
|
-
if rc != 0 or not stdout:
|
|
365
|
-
att.error = (_redact_diagnostics(stderr, proxy, cookies) or "no stdout")[:300]
|
|
366
|
-
att.verdict = Verdict.UNKNOWN.value
|
|
367
|
-
return att, ""
|
|
368
|
-
|
|
369
|
-
html, final_url, status, cookies, user_agent, automation, inner_text = _parse_envelope(stdout, url)
|
|
370
|
-
resp = _FakeResp(html, status=status, final_url=final_url)
|
|
371
|
-
vr = validate(resp, success_selectors=success_selectors)
|
|
372
|
-
att.status = status
|
|
373
|
-
att.body_size = len(html)
|
|
374
|
-
att.verdict = vr.verdict.value
|
|
375
|
-
att.reasons = list(vr.reasons) + ([f"automation:{automation}"] if automation else [])
|
|
376
|
-
att.url = final_url or url
|
|
377
|
-
if vr.verdict in (Verdict.STRONG_OK, Verdict.WEAK_OK) and cookies:
|
|
378
|
-
_bridge_cookies_to_pool(
|
|
379
|
-
url, cookies, user_agent, proxy=proxy, impersonate="firefox")
|
|
380
|
-
if inner_text:
|
|
381
|
-
att._inner_text = inner_text
|
|
382
|
-
return att, html
|
|
224
|
+
return att, ""
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
_SCRAPLING_PYTHON_CACHE: Optional[str] = None
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
def _scrapling_python() -> Optional[str]:
|
|
231
|
+
"""Return an isolated Scrapling runtime, creating it on first real use."""
|
|
232
|
+
global _SCRAPLING_PYTHON_CACHE
|
|
233
|
+
if _module_available("scrapling"):
|
|
234
|
+
return sys.executable
|
|
235
|
+
if _SCRAPLING_PYTHON_CACHE is not None:
|
|
236
|
+
return _SCRAPLING_PYTHON_CACHE or None
|
|
237
|
+
if os.environ.get("INSANE_AUTO_INSTALL", "").strip().lower() in ("0", "false", "no"):
|
|
238
|
+
_SCRAPLING_PYTHON_CACHE = ""
|
|
239
|
+
return None
|
|
240
|
+
|
|
241
|
+
root = os.path.expanduser("~/.insane-search/scrapling-venv")
|
|
242
|
+
python = os.path.join(root, "Scripts", "python.exe") if os.name == "nt" else os.path.join(root, "bin", "python")
|
|
243
|
+
try:
|
|
244
|
+
if not os.path.isfile(python):
|
|
245
|
+
subprocess.run(
|
|
246
|
+
[sys.executable, "-m", "venv", root], capture_output=True,
|
|
247
|
+
text=True, timeout=180, check=False)
|
|
248
|
+
probe = subprocess.run(
|
|
249
|
+
[python, "-c", "import scrapling"], capture_output=True,
|
|
250
|
+
text=True, timeout=30, check=False)
|
|
251
|
+
if probe.returncode != 0:
|
|
252
|
+
install = subprocess.run(
|
|
253
|
+
[python, "-m", "pip", "install", "scrapling[fetchers]>=0.4.8", "-q"],
|
|
254
|
+
capture_output=True, text=True, timeout=600, check=False)
|
|
255
|
+
if install.returncode != 0:
|
|
256
|
+
_SCRAPLING_PYTHON_CACHE = ""
|
|
257
|
+
return None
|
|
258
|
+
_SCRAPLING_PYTHON_CACHE = python
|
|
259
|
+
return python
|
|
260
|
+
except Exception:
|
|
261
|
+
_SCRAPLING_PYTHON_CACHE = ""
|
|
262
|
+
return None
|
|
263
|
+
|
|
264
|
+
|
|
265
|
+
def _run_scrapling(
|
|
266
|
+
att: Attempt, url: str, *, profile_id: str,
|
|
267
|
+
success_selectors: Optional[list[str]], timeout: int, t0: float,
|
|
268
|
+
proxy: Optional[str], cookies: Optional[list[dict]] = None,
|
|
269
|
+
) -> tuple[Attempt, str]:
|
|
270
|
+
"""Run Scrapling as the primary hidden browser fallback."""
|
|
271
|
+
python = _scrapling_python()
|
|
272
|
+
if python is None:
|
|
273
|
+
att.executor = "scrapling:stealthy_fetcher"
|
|
274
|
+
att.error = "Scrapling installation failed or is unavailable"
|
|
275
|
+
att.verdict = Verdict.UNKNOWN.value
|
|
276
|
+
att.elapsed_s = round(time.time() - t0, 3)
|
|
277
|
+
return att, ""
|
|
278
|
+
|
|
279
|
+
args: dict = {
|
|
280
|
+
"url": url,
|
|
281
|
+
"timeout": max(timeout, 60) * 1000,
|
|
282
|
+
"headless": True,
|
|
283
|
+
"realChrome": True,
|
|
284
|
+
"solveCloudflare": profile_id in ("cloudflare_turnstile", "unknown_challenge"),
|
|
285
|
+
"blockAds": True,
|
|
286
|
+
}
|
|
287
|
+
if success_selectors:
|
|
288
|
+
args["waitSelector"] = success_selectors[0]
|
|
289
|
+
if proxy:
|
|
290
|
+
from .network import proxy_for_browser
|
|
291
|
+
args["proxy"] = proxy_for_browser(proxy)
|
|
292
|
+
if cookies:
|
|
293
|
+
args["cookies"] = cookies
|
|
294
|
+
|
|
295
|
+
att._executed = True
|
|
296
|
+
if python == sys.executable:
|
|
297
|
+
rc, stdout, stderr = _run_python_template(
|
|
298
|
+
"scrapling_fetch.py", args, timeout=max(timeout, 60) + 30)
|
|
299
|
+
else:
|
|
300
|
+
rc, stdout, stderr = _run_python_template(
|
|
301
|
+
"scrapling_fetch.py", args, timeout=max(timeout, 60) + 30,
|
|
302
|
+
python_executable=python)
|
|
303
|
+
att.executor = "scrapling:stealthy_fetcher"
|
|
304
|
+
att.elapsed_s = round(time.time() - t0, 3)
|
|
305
|
+
if rc != 0 or not stdout:
|
|
306
|
+
att.error = (_redact_diagnostics(stderr, proxy, cookies) or "no stdout")[:300]
|
|
307
|
+
att.verdict = Verdict.UNKNOWN.value
|
|
308
|
+
return att, ""
|
|
309
|
+
|
|
310
|
+
html, final_url, status, cookies, user_agent, automation, inner_text = _parse_envelope(stdout, url)
|
|
311
|
+
resp = _FakeResp(html, status=status, final_url=final_url)
|
|
312
|
+
vr = validate(resp, success_selectors=success_selectors)
|
|
313
|
+
att.status = status
|
|
314
|
+
att.body_size = len(html)
|
|
315
|
+
att.verdict = vr.verdict.value
|
|
316
|
+
att.reasons = list(vr.reasons) + ([f"automation:{automation}"] if automation else [])
|
|
317
|
+
att.url = final_url or url
|
|
318
|
+
if inner_text:
|
|
319
|
+
att._inner_text = inner_text
|
|
320
|
+
return att, html
|
|
321
|
+
|
|
322
|
+
|
|
323
|
+
def _run_stealth_firefox(
|
|
324
|
+
att: Attempt, url: str, *, success_selectors: Optional[list[str]], timeout: int,
|
|
325
|
+
t0: float, profile_dir: Optional[str], proxy: Optional[str],
|
|
326
|
+
cookies: Optional[list[dict]] = None,
|
|
327
|
+
) -> tuple[Attempt, str]:
|
|
328
|
+
"""Run the optional Python-only stealth Firefox without a sidecar server."""
|
|
329
|
+
module = "invisible_playwright"
|
|
330
|
+
if not _module_available(module) and not _auto_install("invisible-playwright", module):
|
|
331
|
+
att.executor = "stealth_firefox:invisible_playwright"
|
|
332
|
+
att.error = (
|
|
333
|
+
"invisible-playwright not installed "
|
|
334
|
+
"(pip install invisible-playwright, or INSANE_AUTO_INSTALL=1)"
|
|
335
|
+
)
|
|
336
|
+
att.verdict = Verdict.UNKNOWN.value
|
|
337
|
+
att.elapsed_s = round(time.time() - t0, 3)
|
|
338
|
+
return att, ""
|
|
339
|
+
|
|
340
|
+
ephemeral_profile = None
|
|
341
|
+
if cookies and profile_dir is None:
|
|
342
|
+
ephemeral_profile = tempfile.mkdtemp(prefix="insane-firefox-session-")
|
|
343
|
+
args: dict = {
|
|
344
|
+
"url": url,
|
|
345
|
+
"timeout": timeout * 1000,
|
|
346
|
+
"profileDir": profile_dir or ephemeral_profile or _profile_dir_for(url, "stealth_firefox"),
|
|
347
|
+
"headless": True,
|
|
348
|
+
}
|
|
349
|
+
if success_selectors:
|
|
350
|
+
args["waitSelector"] = success_selectors[0]
|
|
351
|
+
if proxy:
|
|
352
|
+
from .network import proxy_for_browser
|
|
353
|
+
args["proxy"] = proxy_for_browser(proxy)
|
|
354
|
+
if cookies:
|
|
355
|
+
args["cookies"] = cookies
|
|
356
|
+
|
|
357
|
+
rc, stdout, stderr = _run_python_template(
|
|
358
|
+
"invisible_playwright_fetch.py", args, timeout=timeout + 30)
|
|
359
|
+
if ephemeral_profile:
|
|
360
|
+
shutil.rmtree(ephemeral_profile, ignore_errors=True)
|
|
361
|
+
att._executed = True
|
|
362
|
+
att.executor = "stealth_firefox:invisible_playwright"
|
|
363
|
+
att.elapsed_s = round(time.time() - t0, 3)
|
|
364
|
+
if rc != 0 or not stdout:
|
|
365
|
+
att.error = (_redact_diagnostics(stderr, proxy, cookies) or "no stdout")[:300]
|
|
366
|
+
att.verdict = Verdict.UNKNOWN.value
|
|
367
|
+
return att, ""
|
|
368
|
+
|
|
369
|
+
html, final_url, status, cookies, user_agent, automation, inner_text = _parse_envelope(stdout, url)
|
|
370
|
+
resp = _FakeResp(html, status=status, final_url=final_url)
|
|
371
|
+
vr = validate(resp, success_selectors=success_selectors)
|
|
372
|
+
att.status = status
|
|
373
|
+
att.body_size = len(html)
|
|
374
|
+
att.verdict = vr.verdict.value
|
|
375
|
+
att.reasons = list(vr.reasons) + ([f"automation:{automation}"] if automation else [])
|
|
376
|
+
att.url = final_url or url
|
|
377
|
+
if vr.verdict in (Verdict.STRONG_OK, Verdict.WEAK_OK) and cookies:
|
|
378
|
+
_bridge_cookies_to_pool(
|
|
379
|
+
url, cookies, user_agent, proxy=proxy, impersonate="firefox")
|
|
380
|
+
if inner_text:
|
|
381
|
+
att._inner_text = inner_text
|
|
382
|
+
return att, html
|
|
383
383
|
|
|
384
384
|
|
|
385
385
|
def _run_node_template(template: str, args: dict, timeout: int = 90,
|
|
@@ -437,7 +437,7 @@ class _FakeCookies:
|
|
|
437
437
|
return iter([])
|
|
438
438
|
|
|
439
439
|
|
|
440
|
-
def run_playwright_fallback(
|
|
440
|
+
def run_playwright_fallback(
|
|
441
441
|
url: str,
|
|
442
442
|
*,
|
|
443
443
|
profile_id: str,
|
|
@@ -445,16 +445,16 @@ def run_playwright_fallback(
|
|
|
445
445
|
device_class: str = "auto",
|
|
446
446
|
timeout: int = 90,
|
|
447
447
|
profile_dir: Optional[str] = None,
|
|
448
|
-
force_executor: Optional[str] = None,
|
|
449
|
-
proxy: Optional[str] = None,
|
|
450
|
-
cookies: Optional[list[dict]] = None,
|
|
451
|
-
) -> tuple[Attempt, str]:
|
|
448
|
+
force_executor: Optional[str] = None,
|
|
449
|
+
proxy: Optional[str] = None,
|
|
450
|
+
cookies: Optional[list[dict]] = None,
|
|
451
|
+
) -> tuple[Attempt, str]:
|
|
452
452
|
"""Invoke the appropriate Playwright executor.
|
|
453
453
|
|
|
454
454
|
force_executor: caller-specified executor name (from a profile's
|
|
455
455
|
`fallback_when_challenge` list). When set, it overrides capability-based
|
|
456
456
|
inference. Recognized values: "playwright_real_chrome",
|
|
457
|
-
"playwright_mobile_chrome", "playwright_mcp", "scrapling", "stealth_firefox".
|
|
457
|
+
"playwright_mobile_chrome", "playwright_mcp", "scrapling", "stealth_firefox".
|
|
458
458
|
|
|
459
459
|
Returns (Attempt, html_content). Attempt.verdict reflects validation.
|
|
460
460
|
"""
|
|
@@ -463,30 +463,30 @@ def run_playwright_fallback(
|
|
|
463
463
|
choice = force_executor or _pick_executor(capabilities, device_class)
|
|
464
464
|
|
|
465
465
|
t0 = time.time()
|
|
466
|
-
att = Attempt(
|
|
466
|
+
att = Attempt(
|
|
467
467
|
phase="fallback",
|
|
468
468
|
executor=choice,
|
|
469
469
|
url=url,
|
|
470
470
|
url_transform="original",
|
|
471
471
|
impersonate=None,
|
|
472
|
-
referer="",
|
|
473
|
-
)
|
|
474
|
-
att._executed = False
|
|
475
|
-
|
|
476
|
-
if choice == "scrapling":
|
|
477
|
-
return _run_scrapling(
|
|
478
|
-
att, url, profile_id=profile_id, success_selectors=success_selectors,
|
|
479
|
-
timeout=timeout, t0=t0, proxy=proxy, cookies=cookies)
|
|
480
|
-
|
|
481
|
-
if choice == "protocol_stealth_chrome":
|
|
482
|
-
return _run_protocol_stealth(
|
|
483
|
-
att, url, success_selectors=success_selectors, timeout=timeout,
|
|
484
|
-
t0=t0, proxy=proxy, cookies=cookies)
|
|
485
|
-
|
|
486
|
-
if choice == "stealth_firefox":
|
|
487
|
-
return _run_stealth_firefox(
|
|
488
|
-
att, url, success_selectors=success_selectors, timeout=timeout,
|
|
489
|
-
t0=t0, profile_dir=profile_dir, proxy=proxy, cookies=cookies)
|
|
472
|
+
referer="",
|
|
473
|
+
)
|
|
474
|
+
att._executed = False
|
|
475
|
+
|
|
476
|
+
if choice == "scrapling":
|
|
477
|
+
return _run_scrapling(
|
|
478
|
+
att, url, profile_id=profile_id, success_selectors=success_selectors,
|
|
479
|
+
timeout=timeout, t0=t0, proxy=proxy, cookies=cookies)
|
|
480
|
+
|
|
481
|
+
if choice == "protocol_stealth_chrome":
|
|
482
|
+
return _run_protocol_stealth(
|
|
483
|
+
att, url, success_selectors=success_selectors, timeout=timeout,
|
|
484
|
+
t0=t0, proxy=proxy, cookies=cookies)
|
|
485
|
+
|
|
486
|
+
if choice == "stealth_firefox":
|
|
487
|
+
return _run_stealth_firefox(
|
|
488
|
+
att, url, success_selectors=success_selectors, timeout=timeout,
|
|
489
|
+
t0=t0, profile_dir=profile_dir, proxy=proxy, cookies=cookies)
|
|
490
490
|
|
|
491
491
|
if choice.startswith("playwright_mcp"):
|
|
492
492
|
att.error = (
|
|
@@ -518,39 +518,39 @@ def run_playwright_fallback(
|
|
|
518
518
|
att.elapsed_s = round(time.time() - t0, 3)
|
|
519
519
|
return att, ""
|
|
520
520
|
|
|
521
|
-
ephemeral_profile = None
|
|
522
|
-
if cookies and profile_dir is None:
|
|
523
|
-
ephemeral_profile = tempfile.mkdtemp(prefix="insane-browser-session-")
|
|
524
|
-
args: dict = {
|
|
525
|
-
"url": url,
|
|
521
|
+
ephemeral_profile = None
|
|
522
|
+
if cookies and profile_dir is None:
|
|
523
|
+
ephemeral_profile = tempfile.mkdtemp(prefix="insane-browser-session-")
|
|
524
|
+
args: dict = {
|
|
525
|
+
"url": url,
|
|
526
526
|
# Per-host + per-device profile isolation. A single shared profile dir
|
|
527
527
|
# (the old default) leaked cookies/storage across hosts and caused
|
|
528
528
|
# profile-lock collisions when two fallbacks ran concurrently. Hashing
|
|
529
529
|
# the host (not storing it) keeps the No-Site-Name Rule intact while
|
|
530
530
|
# letting a host reuse its own warm storageState across calls.
|
|
531
|
-
"profileDir": profile_dir or ephemeral_profile or _profile_dir_for(url, choice),
|
|
532
|
-
"timeout": timeout * 1000,
|
|
533
|
-
"headless": True,
|
|
531
|
+
"profileDir": profile_dir or ephemeral_profile or _profile_dir_for(url, choice),
|
|
532
|
+
"timeout": timeout * 1000,
|
|
533
|
+
"headless": True,
|
|
534
534
|
}
|
|
535
535
|
if choice == "playwright_mobile_chrome":
|
|
536
536
|
args["device"] = "iPhone 13 Pro"
|
|
537
|
-
if success_selectors:
|
|
538
|
-
args["waitSelector"] = success_selectors[0]
|
|
539
|
-
if proxy:
|
|
540
|
-
from .network import proxy_for_browser
|
|
541
|
-
args["proxy"] = proxy_for_browser(proxy)
|
|
542
|
-
if cookies:
|
|
543
|
-
args["cookies"] = cookies
|
|
544
|
-
|
|
545
|
-
att._executed = True
|
|
546
|
-
rc, stdout, stderr = _run_node_template(template, args, timeout=timeout + 10,
|
|
547
|
-
deps_root=deps_root)
|
|
548
|
-
if ephemeral_profile:
|
|
549
|
-
shutil.rmtree(ephemeral_profile, ignore_errors=True)
|
|
537
|
+
if success_selectors:
|
|
538
|
+
args["waitSelector"] = success_selectors[0]
|
|
539
|
+
if proxy:
|
|
540
|
+
from .network import proxy_for_browser
|
|
541
|
+
args["proxy"] = proxy_for_browser(proxy)
|
|
542
|
+
if cookies:
|
|
543
|
+
args["cookies"] = cookies
|
|
544
|
+
|
|
545
|
+
att._executed = True
|
|
546
|
+
rc, stdout, stderr = _run_node_template(template, args, timeout=timeout + 10,
|
|
547
|
+
deps_root=deps_root)
|
|
548
|
+
if ephemeral_profile:
|
|
549
|
+
shutil.rmtree(ephemeral_profile, ignore_errors=True)
|
|
550
550
|
att.elapsed_s = round(time.time() - t0, 3)
|
|
551
551
|
|
|
552
552
|
if rc != 0 or not stdout:
|
|
553
|
-
att.error = (_redact_diagnostics(stderr, proxy, cookies) or "no stdout")[:300]
|
|
553
|
+
att.error = (_redact_diagnostics(stderr, proxy, cookies) or "no stdout")[:300]
|
|
554
554
|
att.verdict = Verdict.UNKNOWN.value
|
|
555
555
|
return att, ""
|
|
556
556
|
|
|
@@ -571,7 +571,7 @@ def run_playwright_fallback(
|
|
|
571
571
|
# cookies + UA a plain HTTP client needs. Seed the curl_cffi pool so
|
|
572
572
|
# subsequent same-host pages are collected cheaply (FlareSolverr pattern).
|
|
573
573
|
if vr.verdict in (Verdict.STRONG_OK, Verdict.WEAK_OK) and cookies:
|
|
574
|
-
_bridge_cookies_to_pool(url, cookies, user_agent, proxy=proxy)
|
|
574
|
+
_bridge_cookies_to_pool(url, cookies, user_agent, proxy=proxy)
|
|
575
575
|
|
|
576
576
|
# Stash the rendered innerText for the render-merge step: many SPAs expose
|
|
577
577
|
# visible text only via innerText; the rescue gate in fetch_chain compares
|
|
@@ -610,14 +610,14 @@ def _parse_envelope(stdout: str, url: str):
|
|
|
610
610
|
return stdout, url, 200, [], None, None, ""
|
|
611
611
|
|
|
612
612
|
|
|
613
|
-
def _bridge_cookies_to_pool(url: str, cookies: list, user_agent: Optional[str],
|
|
614
|
-
proxy: Optional[str] = None,
|
|
615
|
-
impersonate: str = "chrome") -> None:
|
|
613
|
+
def _bridge_cookies_to_pool(url: str, cookies: list, user_agent: Optional[str],
|
|
614
|
+
proxy: Optional[str] = None,
|
|
615
|
+
impersonate: str = "chrome") -> None:
|
|
616
616
|
try:
|
|
617
617
|
from .transport import POOL, pool_enabled, _host_of
|
|
618
618
|
if not pool_enabled():
|
|
619
619
|
return
|
|
620
|
-
POOL.inject_cookies(_host_of(url), impersonate, cookies,
|
|
621
|
-
user_agent=user_agent, proxy=proxy)
|
|
620
|
+
POOL.inject_cookies(_host_of(url), impersonate, cookies,
|
|
621
|
+
user_agent=user_agent, proxy=proxy)
|
|
622
622
|
except Exception:
|
|
623
623
|
pass
|