devez-vibe 1.10.21 → 1.11.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -21
- package/README.md +73 -73
- package/bin/dvz.exe +0 -0
- package/bin/dvz.js +35 -35
- package/bridge/claude-agent-sdk-bridge.mjs +364 -35
- package/install-skills.mjs +11 -11
- package/package.json +50 -50
- package/prepare-search-deps.mjs +42 -42
- package/skills/insane-search/SKILL.md +36 -36
- package/skills/insane-search/engine/__main__.py +44 -44
- package/skills/insane-search/engine/executor.py +246 -246
- package/skills/insane-search/engine/fetch_chain.py +443 -443
- package/skills/insane-search/engine/network.py +108 -108
- package/skills/insane-search/engine/phase0.py +380 -380
- package/skills/insane-search/engine/session_input.py +120 -120
- package/skills/insane-search/engine/templates/invisible_playwright_fetch.py +57 -57
- package/skills/insane-search/engine/templates/nodriver_fetch.py +9 -9
- package/skills/insane-search/engine/templates/patchright_fetch.py +9 -9
- package/skills/insane-search/engine/templates/playwright_mobile_chrome.js +9 -9
- package/skills/insane-search/engine/templates/playwright_real_chrome.js +7 -7
- package/skills/insane-search/engine/templates/scrapling_fetch.py +46 -46
- package/skills/insane-search/engine/tests/test_smoke.py +8 -8
- package/skills/insane-search/engine/tests/test_t10_artifact_routes.py +126 -126
- package/skills/insane-search/engine/tests/test_t11_session_handoff.py +157 -157
- package/skills/insane-search/engine/tests/test_t5_pdfplumber.py +11 -11
- package/skills/insane-search/engine/tests/test_t7_browser_gate.py +3 -3
- package/skills/insane-search/engine/tests/test_t8_network_routes.py +207 -207
- package/skills/insane-search/engine/tests/test_t9_evidence_bundle.py +89 -89
- package/skills/insane-search/engine/tests/test_u5.py +9 -9
- package/skills/insane-search/engine/transport.py +94 -94
- package/skills/insane-search/engine/waf_profiles.yaml +38 -38
- package/skills/insane-search/references/cache-archive.md +13 -13
- package/skills/insane-search/references/media.md +14 -14
- package/skills/insane-search/references/playwright.md +27 -27
- package/skills/insane-search/references/research-evidence.md +36 -36
- package/skills/insane-search/references/session-handoff.md +26 -26
- package/skills/insane-search/tests/test_x_search.py +3 -3
- package/skills/luna-loop/agents/openai.yaml +7 -7
|
@@ -1,120 +1,120 @@
|
|
|
1
|
-
"""Load an explicitly supplied browser cookie export without persisting secrets."""
|
|
2
|
-
from __future__ import annotations
|
|
3
|
-
|
|
4
|
-
import json
|
|
5
|
-
from pathlib import Path
|
|
6
|
-
from typing import Any
|
|
7
|
-
from urllib.parse import urlsplit
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
_MAX_COOKIE_FILE_BYTES = 1024 * 1024
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
def _has_control(value: str) -> bool:
|
|
14
|
-
return any(ord(char) < 0x20 or ord(char) == 0x7F for char in value)
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
def _domain_matches(host: str, domain: str) -> bool:
|
|
18
|
-
clean = domain.lstrip(".").lower().rstrip(".")
|
|
19
|
-
return bool(clean) and (host == clean or host.endswith("." + clean))
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
def _normalize_cookie(item: dict[str, Any], host: str) -> dict[str, Any] | None:
|
|
23
|
-
name = str(item.get("name") or "")
|
|
24
|
-
value = str(item.get("value") or "")
|
|
25
|
-
domain = str(item.get("domain") or host).lower()
|
|
26
|
-
path = str(item.get("path") or "/")
|
|
27
|
-
if not name or _has_control(name) or _has_control(value):
|
|
28
|
-
raise ValueError("cookie name or value contains control characters")
|
|
29
|
-
if not _domain_matches(host, domain):
|
|
30
|
-
return None
|
|
31
|
-
if not path.startswith("/") or _has_control(path) or _has_control(domain):
|
|
32
|
-
raise ValueError("cookie path or domain is invalid")
|
|
33
|
-
result: dict[str, Any] = {
|
|
34
|
-
"name": name,
|
|
35
|
-
"value": value,
|
|
36
|
-
"domain": domain,
|
|
37
|
-
"path": path,
|
|
38
|
-
"secure": bool(item.get("secure", False)),
|
|
39
|
-
}
|
|
40
|
-
if bool(item.get("httpOnly", False)):
|
|
41
|
-
result["httpOnly"] = True
|
|
42
|
-
same_site = str(item.get("sameSite") or "").capitalize()
|
|
43
|
-
if same_site in ("Strict", "Lax", "None"):
|
|
44
|
-
result["sameSite"] = same_site
|
|
45
|
-
expires = item.get("expires", item.get("expirationDate"))
|
|
46
|
-
if expires not in (None, "", 0, "0"):
|
|
47
|
-
try:
|
|
48
|
-
result["expires"] = int(float(expires))
|
|
49
|
-
except (TypeError, ValueError) as exc:
|
|
50
|
-
raise ValueError("cookie expiry is invalid") from exc
|
|
51
|
-
return result
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
def _parse_netscape(text: str) -> list[dict[str, Any]]:
|
|
55
|
-
cookies: list[dict[str, Any]] = []
|
|
56
|
-
for line in text.splitlines():
|
|
57
|
-
if not line or (line.startswith("#") and not line.startswith("#HttpOnly_")):
|
|
58
|
-
continue
|
|
59
|
-
http_only = line.startswith("#HttpOnly_")
|
|
60
|
-
if http_only:
|
|
61
|
-
line = line[len("#HttpOnly_"):]
|
|
62
|
-
parts = line.split("\t")
|
|
63
|
-
if len(parts) != 7:
|
|
64
|
-
continue
|
|
65
|
-
domain, _include_subdomains, path, secure, expires, name, value = parts
|
|
66
|
-
cookies.append({
|
|
67
|
-
"domain": domain,
|
|
68
|
-
"path": path,
|
|
69
|
-
"secure": secure.upper() == "TRUE",
|
|
70
|
-
"expires": expires,
|
|
71
|
-
"name": name,
|
|
72
|
-
"value": value,
|
|
73
|
-
"httpOnly": http_only,
|
|
74
|
-
})
|
|
75
|
-
return cookies
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
def load_cookie_file(path: str, target_url: str) -> list[dict[str, Any]]:
|
|
79
|
-
"""Read JSON or Netscape cookies and keep only cookies valid for target_url."""
|
|
80
|
-
target_host = (urlsplit(target_url).hostname or "").lower()
|
|
81
|
-
if not target_host:
|
|
82
|
-
raise ValueError("target URL must include a host")
|
|
83
|
-
cookie_path = Path(path).expanduser()
|
|
84
|
-
if not cookie_path.is_file():
|
|
85
|
-
raise ValueError("cookie file does not exist")
|
|
86
|
-
if cookie_path.stat().st_size > _MAX_COOKIE_FILE_BYTES:
|
|
87
|
-
raise ValueError("cookie file is too large")
|
|
88
|
-
text = cookie_path.read_text(encoding="utf-8", errors="strict")
|
|
89
|
-
stripped = text.lstrip()
|
|
90
|
-
if stripped.startswith(("[", "{")):
|
|
91
|
-
parsed = json.loads(text)
|
|
92
|
-
raw = parsed.get("cookies") if isinstance(parsed, dict) else parsed
|
|
93
|
-
if not isinstance(raw, list):
|
|
94
|
-
raise ValueError("JSON cookie file must contain a list")
|
|
95
|
-
else:
|
|
96
|
-
raw = _parse_netscape(text)
|
|
97
|
-
if len(raw) > 5000:
|
|
98
|
-
raise ValueError("cookie file contains too many entries")
|
|
99
|
-
cookies = []
|
|
100
|
-
for item in raw:
|
|
101
|
-
if not isinstance(item, dict):
|
|
102
|
-
continue
|
|
103
|
-
normalized = _normalize_cookie(item, target_host)
|
|
104
|
-
if normalized is not None:
|
|
105
|
-
cookies.append(normalized)
|
|
106
|
-
if not cookies:
|
|
107
|
-
raise ValueError("cookie file has no cookies for the target host")
|
|
108
|
-
return cookies
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
def redact_cookie_values(text: str | None, cookies: list[dict] | None) -> str | None:
|
|
112
|
-
"""Remove supplied cookie names and values from subprocess diagnostics."""
|
|
113
|
-
if text is None or not cookies:
|
|
114
|
-
return text
|
|
115
|
-
redacted = text
|
|
116
|
-
for cookie in cookies:
|
|
117
|
-
for value in (str(cookie.get("name") or ""), str(cookie.get("value") or "")):
|
|
118
|
-
if value:
|
|
119
|
-
redacted = redacted.replace(value, "***")
|
|
120
|
-
return redacted
|
|
1
|
+
"""Load an explicitly supplied browser cookie export without persisting secrets."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import json
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from typing import Any
|
|
7
|
+
from urllib.parse import urlsplit
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
_MAX_COOKIE_FILE_BYTES = 1024 * 1024
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def _has_control(value: str) -> bool:
|
|
14
|
+
return any(ord(char) < 0x20 or ord(char) == 0x7F for char in value)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def _domain_matches(host: str, domain: str) -> bool:
|
|
18
|
+
clean = domain.lstrip(".").lower().rstrip(".")
|
|
19
|
+
return bool(clean) and (host == clean or host.endswith("." + clean))
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _normalize_cookie(item: dict[str, Any], host: str) -> dict[str, Any] | None:
|
|
23
|
+
name = str(item.get("name") or "")
|
|
24
|
+
value = str(item.get("value") or "")
|
|
25
|
+
domain = str(item.get("domain") or host).lower()
|
|
26
|
+
path = str(item.get("path") or "/")
|
|
27
|
+
if not name or _has_control(name) or _has_control(value):
|
|
28
|
+
raise ValueError("cookie name or value contains control characters")
|
|
29
|
+
if not _domain_matches(host, domain):
|
|
30
|
+
return None
|
|
31
|
+
if not path.startswith("/") or _has_control(path) or _has_control(domain):
|
|
32
|
+
raise ValueError("cookie path or domain is invalid")
|
|
33
|
+
result: dict[str, Any] = {
|
|
34
|
+
"name": name,
|
|
35
|
+
"value": value,
|
|
36
|
+
"domain": domain,
|
|
37
|
+
"path": path,
|
|
38
|
+
"secure": bool(item.get("secure", False)),
|
|
39
|
+
}
|
|
40
|
+
if bool(item.get("httpOnly", False)):
|
|
41
|
+
result["httpOnly"] = True
|
|
42
|
+
same_site = str(item.get("sameSite") or "").capitalize()
|
|
43
|
+
if same_site in ("Strict", "Lax", "None"):
|
|
44
|
+
result["sameSite"] = same_site
|
|
45
|
+
expires = item.get("expires", item.get("expirationDate"))
|
|
46
|
+
if expires not in (None, "", 0, "0"):
|
|
47
|
+
try:
|
|
48
|
+
result["expires"] = int(float(expires))
|
|
49
|
+
except (TypeError, ValueError) as exc:
|
|
50
|
+
raise ValueError("cookie expiry is invalid") from exc
|
|
51
|
+
return result
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _parse_netscape(text: str) -> list[dict[str, Any]]:
|
|
55
|
+
cookies: list[dict[str, Any]] = []
|
|
56
|
+
for line in text.splitlines():
|
|
57
|
+
if not line or (line.startswith("#") and not line.startswith("#HttpOnly_")):
|
|
58
|
+
continue
|
|
59
|
+
http_only = line.startswith("#HttpOnly_")
|
|
60
|
+
if http_only:
|
|
61
|
+
line = line[len("#HttpOnly_"):]
|
|
62
|
+
parts = line.split("\t")
|
|
63
|
+
if len(parts) != 7:
|
|
64
|
+
continue
|
|
65
|
+
domain, _include_subdomains, path, secure, expires, name, value = parts
|
|
66
|
+
cookies.append({
|
|
67
|
+
"domain": domain,
|
|
68
|
+
"path": path,
|
|
69
|
+
"secure": secure.upper() == "TRUE",
|
|
70
|
+
"expires": expires,
|
|
71
|
+
"name": name,
|
|
72
|
+
"value": value,
|
|
73
|
+
"httpOnly": http_only,
|
|
74
|
+
})
|
|
75
|
+
return cookies
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def load_cookie_file(path: str, target_url: str) -> list[dict[str, Any]]:
|
|
79
|
+
"""Read JSON or Netscape cookies and keep only cookies valid for target_url."""
|
|
80
|
+
target_host = (urlsplit(target_url).hostname or "").lower()
|
|
81
|
+
if not target_host:
|
|
82
|
+
raise ValueError("target URL must include a host")
|
|
83
|
+
cookie_path = Path(path).expanduser()
|
|
84
|
+
if not cookie_path.is_file():
|
|
85
|
+
raise ValueError("cookie file does not exist")
|
|
86
|
+
if cookie_path.stat().st_size > _MAX_COOKIE_FILE_BYTES:
|
|
87
|
+
raise ValueError("cookie file is too large")
|
|
88
|
+
text = cookie_path.read_text(encoding="utf-8", errors="strict")
|
|
89
|
+
stripped = text.lstrip()
|
|
90
|
+
if stripped.startswith(("[", "{")):
|
|
91
|
+
parsed = json.loads(text)
|
|
92
|
+
raw = parsed.get("cookies") if isinstance(parsed, dict) else parsed
|
|
93
|
+
if not isinstance(raw, list):
|
|
94
|
+
raise ValueError("JSON cookie file must contain a list")
|
|
95
|
+
else:
|
|
96
|
+
raw = _parse_netscape(text)
|
|
97
|
+
if len(raw) > 5000:
|
|
98
|
+
raise ValueError("cookie file contains too many entries")
|
|
99
|
+
cookies = []
|
|
100
|
+
for item in raw:
|
|
101
|
+
if not isinstance(item, dict):
|
|
102
|
+
continue
|
|
103
|
+
normalized = _normalize_cookie(item, target_host)
|
|
104
|
+
if normalized is not None:
|
|
105
|
+
cookies.append(normalized)
|
|
106
|
+
if not cookies:
|
|
107
|
+
raise ValueError("cookie file has no cookies for the target host")
|
|
108
|
+
return cookies
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def redact_cookie_values(text: str | None, cookies: list[dict] | None) -> str | None:
|
|
112
|
+
"""Remove supplied cookie names and values from subprocess diagnostics."""
|
|
113
|
+
if text is None or not cookies:
|
|
114
|
+
return text
|
|
115
|
+
redacted = text
|
|
116
|
+
for cookie in cookies:
|
|
117
|
+
for value in (str(cookie.get("name") or ""), str(cookie.get("value") or "")):
|
|
118
|
+
if value:
|
|
119
|
+
redacted = redacted.replace(value, "***")
|
|
120
|
+
return redacted
|
|
@@ -1,57 +1,57 @@
|
|
|
1
|
-
"""Python-only stealth Firefox fallback using invisible-playwright."""
|
|
2
|
-
from __future__ import annotations
|
|
3
|
-
|
|
4
|
-
import json
|
|
5
|
-
import sys
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
def main() -> int:
|
|
9
|
-
args = json.load(sys.stdin)
|
|
10
|
-
from invisible_playwright import InvisiblePlaywright
|
|
11
|
-
|
|
12
|
-
launch = {
|
|
13
|
-
"headless": bool(args.get("headless", False)),
|
|
14
|
-
"profile_dir": args.get("profileDir"),
|
|
15
|
-
}
|
|
16
|
-
if args.get("proxy"):
|
|
17
|
-
launch["proxy"] = args["proxy"]
|
|
18
|
-
|
|
19
|
-
timeout_ms = int(args.get("timeout", 60000))
|
|
20
|
-
with InvisiblePlaywright(**launch) as browser:
|
|
21
|
-
page = browser.new_page()
|
|
22
|
-
if args.get("cookies"):
|
|
23
|
-
page.context.add_cookies(args["cookies"])
|
|
24
|
-
page.set_default_timeout(timeout_ms)
|
|
25
|
-
response = page.goto(args["url"], wait_until="domcontentloaded", timeout=timeout_ms)
|
|
26
|
-
page.wait_for_timeout(3500)
|
|
27
|
-
selector = args.get("waitSelector")
|
|
28
|
-
if selector:
|
|
29
|
-
try:
|
|
30
|
-
page.wait_for_selector(selector, timeout=min(timeout_ms, 15000))
|
|
31
|
-
except Exception as exc:
|
|
32
|
-
print(f"best-effort waitSelector failed: {exc}", file=sys.stderr)
|
|
33
|
-
|
|
34
|
-
context = page.context
|
|
35
|
-
cookies = [
|
|
36
|
-
{"name": item["name"], "value": item["value"], "domain": item.get("domain", "")}
|
|
37
|
-
for item in context.cookies()
|
|
38
|
-
]
|
|
39
|
-
payload = {
|
|
40
|
-
"html": page.content() or "",
|
|
41
|
-
"finalUrl": page.url,
|
|
42
|
-
"status": response.status if response else 200,
|
|
43
|
-
"cookies": cookies,
|
|
44
|
-
"userAgent": page.evaluate("() => navigator.userAgent"),
|
|
45
|
-
"automation": "invisible-playwright",
|
|
46
|
-
"innerText": page.evaluate("() => document.body && document.body.innerText || ''"),
|
|
47
|
-
}
|
|
48
|
-
sys.stdout.write(json.dumps(payload, ensure_ascii=False))
|
|
49
|
-
return 0
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
if __name__ == "__main__":
|
|
53
|
-
try:
|
|
54
|
-
sys.exit(main())
|
|
55
|
-
except Exception as exc:
|
|
56
|
-
print(f"{type(exc).__name__}: {exc}", file=sys.stderr)
|
|
57
|
-
sys.exit(1)
|
|
1
|
+
"""Python-only stealth Firefox fallback using invisible-playwright."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import json
|
|
5
|
+
import sys
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def main() -> int:
|
|
9
|
+
args = json.load(sys.stdin)
|
|
10
|
+
from invisible_playwright import InvisiblePlaywright
|
|
11
|
+
|
|
12
|
+
launch = {
|
|
13
|
+
"headless": bool(args.get("headless", False)),
|
|
14
|
+
"profile_dir": args.get("profileDir"),
|
|
15
|
+
}
|
|
16
|
+
if args.get("proxy"):
|
|
17
|
+
launch["proxy"] = args["proxy"]
|
|
18
|
+
|
|
19
|
+
timeout_ms = int(args.get("timeout", 60000))
|
|
20
|
+
with InvisiblePlaywright(**launch) as browser:
|
|
21
|
+
page = browser.new_page()
|
|
22
|
+
if args.get("cookies"):
|
|
23
|
+
page.context.add_cookies(args["cookies"])
|
|
24
|
+
page.set_default_timeout(timeout_ms)
|
|
25
|
+
response = page.goto(args["url"], wait_until="domcontentloaded", timeout=timeout_ms)
|
|
26
|
+
page.wait_for_timeout(3500)
|
|
27
|
+
selector = args.get("waitSelector")
|
|
28
|
+
if selector:
|
|
29
|
+
try:
|
|
30
|
+
page.wait_for_selector(selector, timeout=min(timeout_ms, 15000))
|
|
31
|
+
except Exception as exc:
|
|
32
|
+
print(f"best-effort waitSelector failed: {exc}", file=sys.stderr)
|
|
33
|
+
|
|
34
|
+
context = page.context
|
|
35
|
+
cookies = [
|
|
36
|
+
{"name": item["name"], "value": item["value"], "domain": item.get("domain", "")}
|
|
37
|
+
for item in context.cookies()
|
|
38
|
+
]
|
|
39
|
+
payload = {
|
|
40
|
+
"html": page.content() or "",
|
|
41
|
+
"finalUrl": page.url,
|
|
42
|
+
"status": response.status if response else 200,
|
|
43
|
+
"cookies": cookies,
|
|
44
|
+
"userAgent": page.evaluate("() => navigator.userAgent"),
|
|
45
|
+
"automation": "invisible-playwright",
|
|
46
|
+
"innerText": page.evaluate("() => document.body && document.body.innerText || ''"),
|
|
47
|
+
}
|
|
48
|
+
sys.stdout.write(json.dumps(payload, ensure_ascii=False))
|
|
49
|
+
return 0
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
if __name__ == "__main__":
|
|
53
|
+
try:
|
|
54
|
+
sys.exit(main())
|
|
55
|
+
except Exception as exc:
|
|
56
|
+
print(f"{type(exc).__name__}: {exc}", file=sys.stderr)
|
|
57
|
+
sys.exit(1)
|
|
@@ -19,15 +19,15 @@ async def main() -> int:
|
|
|
19
19
|
|
|
20
20
|
# Headful by default. Cloudflare/DataDome score headless Chrome as a bot on
|
|
21
21
|
# signal alone, so a headless CDP lane loses challenges a headful one clears
|
|
22
|
-
# (matching the headless:false default of the Playwright templates).
|
|
23
|
-
headless = bool(args.get("headless", False))
|
|
24
|
-
browser_args = []
|
|
25
|
-
proxy = args.get("proxy") or {}
|
|
26
|
-
if proxy:
|
|
27
|
-
if proxy.get("username") or proxy.get("password"):
|
|
28
|
-
raise ValueError("nodriver does not support authenticated proxy URLs")
|
|
29
|
-
browser_args.append(f"--proxy-server={proxy['server']}")
|
|
30
|
-
browser = await uc.start(headless=headless, browser_args=browser_args)
|
|
22
|
+
# (matching the headless:false default of the Playwright templates).
|
|
23
|
+
headless = bool(args.get("headless", False))
|
|
24
|
+
browser_args = []
|
|
25
|
+
proxy = args.get("proxy") or {}
|
|
26
|
+
if proxy:
|
|
27
|
+
if proxy.get("username") or proxy.get("password"):
|
|
28
|
+
raise ValueError("nodriver does not support authenticated proxy URLs")
|
|
29
|
+
browser_args.append(f"--proxy-server={proxy['server']}")
|
|
30
|
+
browser = await uc.start(headless=headless, browser_args=browser_args)
|
|
31
31
|
try:
|
|
32
32
|
tab = await browser.get(args["url"])
|
|
33
33
|
await tab.sleep(6) # JS challenge solve window
|
|
@@ -21,15 +21,15 @@ def main() -> int:
|
|
|
21
21
|
with sync_playwright() as p:
|
|
22
22
|
# Headful by default — headless Chrome is scored as a bot before the
|
|
23
23
|
# protocol fingerprint is even examined.
|
|
24
|
-
launch = {"channel": "chrome", "headless": bool(args.get("headless", False))}
|
|
25
|
-
if args.get("proxy"):
|
|
26
|
-
launch["proxy"] = args["proxy"]
|
|
27
|
-
browser = p.chromium.launch(**launch)
|
|
28
|
-
try:
|
|
29
|
-
page = browser.new_page()
|
|
30
|
-
if args.get("cookies"):
|
|
31
|
-
page.context.add_cookies(args["cookies"])
|
|
32
|
-
page.goto(args["url"], timeout=timeout_ms, wait_until="domcontentloaded")
|
|
24
|
+
launch = {"channel": "chrome", "headless": bool(args.get("headless", False))}
|
|
25
|
+
if args.get("proxy"):
|
|
26
|
+
launch["proxy"] = args["proxy"]
|
|
27
|
+
browser = p.chromium.launch(**launch)
|
|
28
|
+
try:
|
|
29
|
+
page = browser.new_page()
|
|
30
|
+
if args.get("cookies"):
|
|
31
|
+
page.context.add_cookies(args["cookies"])
|
|
32
|
+
page.goto(args["url"], timeout=timeout_ms, wait_until="domcontentloaded")
|
|
33
33
|
page.wait_for_timeout(5000) # JS challenge solve window
|
|
34
34
|
sel = args.get("waitSelector")
|
|
35
35
|
if sel:
|
|
@@ -82,15 +82,15 @@ async function main() {
|
|
|
82
82
|
|
|
83
83
|
let ctx;
|
|
84
84
|
try {
|
|
85
|
-
const ctxOpts = {
|
|
86
|
-
channel: 'chrome',
|
|
87
|
-
headless,
|
|
88
|
-
...dev,
|
|
89
|
-
};
|
|
90
|
-
if (args.proxy) ctxOpts.proxy = args.proxy;
|
|
91
|
-
ctx = await chromium.launchPersistentContext(profileDir, ctxOpts);
|
|
92
|
-
if (Array.isArray(args.cookies) && args.cookies.length) await ctx.addCookies(args.cookies);
|
|
93
|
-
const page = await ctx.newPage();
|
|
85
|
+
const ctxOpts = {
|
|
86
|
+
channel: 'chrome',
|
|
87
|
+
headless,
|
|
88
|
+
...dev,
|
|
89
|
+
};
|
|
90
|
+
if (args.proxy) ctxOpts.proxy = args.proxy;
|
|
91
|
+
ctx = await chromium.launchPersistentContext(profileDir, ctxOpts);
|
|
92
|
+
if (Array.isArray(args.cookies) && args.cookies.length) await ctx.addCookies(args.cookies);
|
|
93
|
+
const page = await ctx.newPage();
|
|
94
94
|
const deadline = Date.now() + timeoutMs;
|
|
95
95
|
const rem = (cap) => Math.max(1000, Math.min(cap || timeoutMs, deadline - Date.now()));
|
|
96
96
|
const mainResp = await page.goto(url, { waitUntil: 'domcontentloaded', timeout: rem(90000) });
|
|
@@ -96,15 +96,15 @@ async function main() {
|
|
|
96
96
|
// headers/UA/flags. We only override viewport for patchright; plain
|
|
97
97
|
// playwright keeps the fixed viewport it has always used.
|
|
98
98
|
const ctxOpts = { channel: 'chrome', headless };
|
|
99
|
-
if (automation === 'patchright') {
|
|
99
|
+
if (automation === 'patchright') {
|
|
100
100
|
ctxOpts.viewport = null; // == no_viewport=True (use real window size)
|
|
101
101
|
} else {
|
|
102
|
-
ctxOpts.viewport = viewport;
|
|
103
|
-
}
|
|
104
|
-
if (args.proxy) ctxOpts.proxy = args.proxy;
|
|
105
|
-
ctx = await chromium.launchPersistentContext(profileDir, ctxOpts);
|
|
106
|
-
if (Array.isArray(args.cookies) && args.cookies.length) await ctx.addCookies(args.cookies);
|
|
107
|
-
const page = await ctx.newPage();
|
|
102
|
+
ctxOpts.viewport = viewport;
|
|
103
|
+
}
|
|
104
|
+
if (args.proxy) ctxOpts.proxy = args.proxy;
|
|
105
|
+
ctx = await chromium.launchPersistentContext(profileDir, ctxOpts);
|
|
106
|
+
if (Array.isArray(args.cookies) && args.cookies.length) await ctx.addCookies(args.cookies);
|
|
107
|
+
const page = await ctx.newPage();
|
|
108
108
|
// Single shared deadline across warmup + main + reload navigations so the
|
|
109
109
|
// first nav can't eat the whole budget and starve the rest.
|
|
110
110
|
const deadline = Date.now() + timeoutMs;
|
|
@@ -1,46 +1,46 @@
|
|
|
1
|
-
"""One-shot hidden browser fallback using Scrapling's StealthyFetcher."""
|
|
2
|
-
from __future__ import annotations
|
|
3
|
-
|
|
4
|
-
import json
|
|
5
|
-
import sys
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
def main() -> int:
|
|
9
|
-
args = json.load(sys.stdin)
|
|
10
|
-
from scrapling.fetchers import StealthyFetcher
|
|
11
|
-
|
|
12
|
-
options = {
|
|
13
|
-
"headless": bool(args.get("headless", True)),
|
|
14
|
-
"real_chrome": bool(args.get("realChrome", True)),
|
|
15
|
-
"solve_cloudflare": bool(args.get("solveCloudflare", False)),
|
|
16
|
-
"block_ads": bool(args.get("blockAds", True)),
|
|
17
|
-
"timeout": int(args.get("timeout", 90000)),
|
|
18
|
-
}
|
|
19
|
-
if args.get("waitSelector"):
|
|
20
|
-
options["wait_selector"] = args["waitSelector"]
|
|
21
|
-
if args.get("proxy"):
|
|
22
|
-
options["proxy"] = args["proxy"]
|
|
23
|
-
if args.get("cookies"):
|
|
24
|
-
options["cookies"] = args["cookies"]
|
|
25
|
-
|
|
26
|
-
page = StealthyFetcher.fetch(args["url"], **options)
|
|
27
|
-
html = getattr(page, "html_content", "") or ""
|
|
28
|
-
payload = {
|
|
29
|
-
"html": html,
|
|
30
|
-
"finalUrl": str(getattr(page, "url", "") or args["url"]),
|
|
31
|
-
"status": int(getattr(page, "status", 0) or 200),
|
|
32
|
-
"cookies": [],
|
|
33
|
-
"userAgent": "",
|
|
34
|
-
"automation": "scrapling-stealthy-fetcher",
|
|
35
|
-
"innerText": "",
|
|
36
|
-
}
|
|
37
|
-
sys.stdout.write(json.dumps(payload, ensure_ascii=False))
|
|
38
|
-
return 0
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
if __name__ == "__main__":
|
|
42
|
-
try:
|
|
43
|
-
sys.exit(main())
|
|
44
|
-
except Exception as exc:
|
|
45
|
-
print(f"{type(exc).__name__}: {exc}", file=sys.stderr)
|
|
46
|
-
sys.exit(1)
|
|
1
|
+
"""One-shot hidden browser fallback using Scrapling's StealthyFetcher."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import json
|
|
5
|
+
import sys
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def main() -> int:
|
|
9
|
+
args = json.load(sys.stdin)
|
|
10
|
+
from scrapling.fetchers import StealthyFetcher
|
|
11
|
+
|
|
12
|
+
options = {
|
|
13
|
+
"headless": bool(args.get("headless", True)),
|
|
14
|
+
"real_chrome": bool(args.get("realChrome", True)),
|
|
15
|
+
"solve_cloudflare": bool(args.get("solveCloudflare", False)),
|
|
16
|
+
"block_ads": bool(args.get("blockAds", True)),
|
|
17
|
+
"timeout": int(args.get("timeout", 90000)),
|
|
18
|
+
}
|
|
19
|
+
if args.get("waitSelector"):
|
|
20
|
+
options["wait_selector"] = args["waitSelector"]
|
|
21
|
+
if args.get("proxy"):
|
|
22
|
+
options["proxy"] = args["proxy"]
|
|
23
|
+
if args.get("cookies"):
|
|
24
|
+
options["cookies"] = args["cookies"]
|
|
25
|
+
|
|
26
|
+
page = StealthyFetcher.fetch(args["url"], **options)
|
|
27
|
+
html = getattr(page, "html_content", "") or ""
|
|
28
|
+
payload = {
|
|
29
|
+
"html": html,
|
|
30
|
+
"finalUrl": str(getattr(page, "url", "") or args["url"]),
|
|
31
|
+
"status": int(getattr(page, "status", 0) or 200),
|
|
32
|
+
"cookies": [],
|
|
33
|
+
"userAgent": "",
|
|
34
|
+
"automation": "scrapling-stealthy-fetcher",
|
|
35
|
+
"innerText": "",
|
|
36
|
+
}
|
|
37
|
+
sys.stdout.write(json.dumps(payload, ensure_ascii=False))
|
|
38
|
+
return 0
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
if __name__ == "__main__":
|
|
42
|
+
try:
|
|
43
|
+
sys.exit(main())
|
|
44
|
+
except Exception as exc:
|
|
45
|
+
print(f"{type(exc).__name__}: {exc}", file=sys.stderr)
|
|
46
|
+
sys.exit(1)
|
|
@@ -97,18 +97,18 @@ def t_online_benign_site():
|
|
|
97
97
|
"""A simple, usually-open site should pass probe directly when selectors provided."""
|
|
98
98
|
# example.com serves ~1.2KB content — below tiny_body threshold — but with
|
|
99
99
|
# success_selectors we trust caller's "content exists" definition.
|
|
100
|
-
r = fetch(
|
|
100
|
+
r = fetch(
|
|
101
101
|
"https://example.com/",
|
|
102
102
|
success_selectors=["h1", "p"],
|
|
103
103
|
timeout=15,
|
|
104
104
|
max_attempts=3,
|
|
105
|
-
enable_playwright=False,
|
|
106
|
-
)
|
|
107
|
-
if (not r.ok and r.network_diagnosis.get("category")
|
|
108
|
-
== "certificate_verification"):
|
|
109
|
-
print(" - skipped: host certificate trust blocks the curl-only lane")
|
|
110
|
-
return
|
|
111
|
-
assert r.ok, f"{r.summary} | trace: {[a.verdict for a in r.trace]}"
|
|
105
|
+
enable_playwright=False,
|
|
106
|
+
)
|
|
107
|
+
if (not r.ok and r.network_diagnosis.get("category")
|
|
108
|
+
== "certificate_verification"):
|
|
109
|
+
print(" - skipped: host certificate trust blocks the curl-only lane")
|
|
110
|
+
return
|
|
111
|
+
assert r.ok, f"{r.summary} | trace: {[a.verdict for a in r.trace]}"
|
|
112
112
|
assert r.verdict in ("strong_ok", "weak_ok"), r.verdict
|
|
113
113
|
print(f" ✓ benign site → verdict={r.verdict} size={len(r.content)}")
|
|
114
114
|
|