devez-vibe 1.10.21 → 1.11.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -21
- package/README.md +73 -73
- package/bin/dvz.exe +0 -0
- package/bin/dvz.js +35 -35
- package/bridge/claude-agent-sdk-bridge.mjs +364 -35
- package/install-skills.mjs +11 -11
- package/package.json +50 -50
- package/prepare-search-deps.mjs +42 -42
- package/skills/insane-search/SKILL.md +36 -36
- package/skills/insane-search/engine/__main__.py +44 -44
- package/skills/insane-search/engine/executor.py +246 -246
- package/skills/insane-search/engine/fetch_chain.py +443 -443
- package/skills/insane-search/engine/network.py +108 -108
- package/skills/insane-search/engine/phase0.py +380 -380
- package/skills/insane-search/engine/session_input.py +120 -120
- package/skills/insane-search/engine/templates/invisible_playwright_fetch.py +57 -57
- package/skills/insane-search/engine/templates/nodriver_fetch.py +9 -9
- package/skills/insane-search/engine/templates/patchright_fetch.py +9 -9
- package/skills/insane-search/engine/templates/playwright_mobile_chrome.js +9 -9
- package/skills/insane-search/engine/templates/playwright_real_chrome.js +7 -7
- package/skills/insane-search/engine/templates/scrapling_fetch.py +46 -46
- package/skills/insane-search/engine/tests/test_smoke.py +8 -8
- package/skills/insane-search/engine/tests/test_t10_artifact_routes.py +126 -126
- package/skills/insane-search/engine/tests/test_t11_session_handoff.py +157 -157
- package/skills/insane-search/engine/tests/test_t5_pdfplumber.py +11 -11
- package/skills/insane-search/engine/tests/test_t7_browser_gate.py +3 -3
- package/skills/insane-search/engine/tests/test_t8_network_routes.py +207 -207
- package/skills/insane-search/engine/tests/test_t9_evidence_bundle.py +89 -89
- package/skills/insane-search/engine/tests/test_u5.py +9 -9
- package/skills/insane-search/engine/transport.py +94 -94
- package/skills/insane-search/engine/waf_profiles.yaml +38 -38
- package/skills/insane-search/references/cache-archive.md +13 -13
- package/skills/insane-search/references/media.md +14 -14
- package/skills/insane-search/references/playwright.md +27 -27
- package/skills/insane-search/references/research-evidence.md +36 -36
- package/skills/insane-search/references/session-handoff.md +26 -26
- package/skills/insane-search/tests/test_x_search.py +3 -3
- package/skills/luna-loop/agents/openai.yaml +7 -7
|
@@ -1,126 +1,126 @@
|
|
|
1
|
-
#!/usr/bin/env python3
|
|
2
|
-
"""Document, media-transcript, and archive fallback regressions."""
|
|
3
|
-
from __future__ import annotations
|
|
4
|
-
|
|
5
|
-
import os
|
|
6
|
-
import sys
|
|
7
|
-
from unittest import mock
|
|
8
|
-
|
|
9
|
-
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
10
|
-
ROOT = os.path.abspath(os.path.join(HERE, "..", ".."))
|
|
11
|
-
sys.path.insert(0, ROOT)
|
|
12
|
-
|
|
13
|
-
import engine.fetch_chain as fc # noqa: E402
|
|
14
|
-
import engine.phase0 as phase0 # noqa: E402
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
class _Response:
|
|
18
|
-
def __init__(self, status: int, text: str, payload=None, url=""):
|
|
19
|
-
self.status_code = status
|
|
20
|
-
self.text = text
|
|
21
|
-
self.url = url
|
|
22
|
-
self._payload = payload
|
|
23
|
-
|
|
24
|
-
def json(self):
|
|
25
|
-
return self._payload
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
def t_pdf_table_is_preserved_as_markdown() -> None:
|
|
29
|
-
table = [["Name", "Value|Unit"], ["Alpha", "3\nkg"], [None, "4"]]
|
|
30
|
-
markdown = fc._pdf_table_to_markdown(table)
|
|
31
|
-
assert "| Name | Value\\|Unit |" in markdown
|
|
32
|
-
assert "| Alpha | 3 kg |" in markdown
|
|
33
|
-
assert "| | 4 |" in markdown
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
def t_scanned_pdf_ocr_degrades_when_tools_are_missing() -> None:
|
|
37
|
-
with mock.patch.object(fc._shutil, "which", return_value=None):
|
|
38
|
-
text, error = fc._extract_pdf_ocr(b"%PDF-1.4\n")
|
|
39
|
-
assert text == ""
|
|
40
|
-
assert error == "pdf_ocr_tools_missing"
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
def t_vtt_cleanup_removes_timing_and_duplicate_caption_lines() -> None:
|
|
44
|
-
vtt = """WEBVTT
|
|
45
|
-
|
|
46
|
-
00:00:00.000 --> 00:00:02.000
|
|
47
|
-
<c>Hello world</c>
|
|
48
|
-
|
|
49
|
-
00:00:01.500 --> 00:00:03.000
|
|
50
|
-
Hello world
|
|
51
|
-
Next line
|
|
52
|
-
"""
|
|
53
|
-
assert phase0._vtt_to_text(vtt) == "Hello world\nNext line"
|
|
54
|
-
segments = phase0._vtt_segments(vtt)
|
|
55
|
-
assert segments[0] == {
|
|
56
|
-
"start": "00:00:00.000", "end": "00:00:02.000", "text": "Hello world"}
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
def t_subtitle_language_prefers_creator_korean_then_english() -> None:
|
|
60
|
-
metadata = {
|
|
61
|
-
"subtitles": {"en": [{"ext": "vtt"}], "ko": [{"ext": "vtt"}]},
|
|
62
|
-
"automatic_captions": {"ko-orig": [{"ext": "vtt"}]},
|
|
63
|
-
}
|
|
64
|
-
assert phase0._choose_subtitle_language(metadata, ("ko", "en")) == ("ko", "creator")
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
def t_archive_route_returns_only_a_valid_wayback_snapshot() -> None:
|
|
68
|
-
api = _Response(200, "{}", {
|
|
69
|
-
"archived_snapshots": {"closest": {
|
|
70
|
-
"available": True,
|
|
71
|
-
"status": "200",
|
|
72
|
-
"timestamp": "20240102030405",
|
|
73
|
-
"url": "https://web.archive.org/web/20240102030405/https://example.com/a",
|
|
74
|
-
}}
|
|
75
|
-
})
|
|
76
|
-
page = _Response(200, "archived body " * 100,
|
|
77
|
-
url="https://web.archive.org/web/20240102030405/https://example.com/a")
|
|
78
|
-
with mock.patch.object(phase0, "_archive_get", side_effect=[api, page]):
|
|
79
|
-
result = phase0.route_archive("https://example.com/a", timeout=2)
|
|
80
|
-
assert result["ok"] is True
|
|
81
|
-
assert result["timestamp"] == "20240102030405"
|
|
82
|
-
assert result["content"].startswith("archived body")
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
def t_archive_fallback_is_labeled_historical_in_evidence() -> None:
|
|
86
|
-
base = fc.FetchResult(
|
|
87
|
-
ok=False, final_url="https://example.com/a", verdict="not_found",
|
|
88
|
-
trace=[fc.Attempt(
|
|
89
|
-
phase="probe", executor="curl_cffi", url="https://example.com/a",
|
|
90
|
-
url_transform="original", impersonate="safari", referer="self_root",
|
|
91
|
-
verdict="not_found")],
|
|
92
|
-
)
|
|
93
|
-
archived = {
|
|
94
|
-
"ok": True,
|
|
95
|
-
"content": "<html><body>" + ("historical text " * 100) + "</body></html>",
|
|
96
|
-
"final_url": "https://web.archive.org/web/20240102030405/https://example.com/a",
|
|
97
|
-
"timestamp": "20240102030405",
|
|
98
|
-
"attempts": [{
|
|
99
|
-
"route": "wayback-snapshot", "ok": True, "status": 200,
|
|
100
|
-
"bytes": 1600, "note": "historical_snapshot",
|
|
101
|
-
}],
|
|
102
|
-
}
|
|
103
|
-
with mock.patch.object(phase0, "route_archive", return_value=archived):
|
|
104
|
-
result = fc._archive_fallback(
|
|
105
|
-
base, "https://example.com/a", timeout=2, proxy=None,
|
|
106
|
-
enable_extraction=True, enable_markdown=True, enable_maincontent=False)
|
|
107
|
-
assert result.ok is True
|
|
108
|
-
assert result.extraction_source.startswith("archive+")
|
|
109
|
-
assert result.evidence_record()["currentness"] == "historical_snapshot"
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
def main() -> int:
|
|
113
|
-
tests = [v for k, v in sorted(globals().items()) if k.startswith("t_") and callable(v)]
|
|
114
|
-
failed = 0
|
|
115
|
-
for test in tests:
|
|
116
|
-
try:
|
|
117
|
-
test()
|
|
118
|
-
except AssertionError as exc:
|
|
119
|
-
print(f" x {test.__name__}: {exc}")
|
|
120
|
-
failed += 1
|
|
121
|
-
print(f"\n{'OK' if failed == 0 else 'FAIL'}: {len(tests) - failed}/{len(tests)} passed")
|
|
122
|
-
return 1 if failed else 0
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
if __name__ == "__main__":
|
|
126
|
-
sys.exit(main())
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Document, media-transcript, and archive fallback regressions."""
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import os
|
|
6
|
+
import sys
|
|
7
|
+
from unittest import mock
|
|
8
|
+
|
|
9
|
+
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
10
|
+
ROOT = os.path.abspath(os.path.join(HERE, "..", ".."))
|
|
11
|
+
sys.path.insert(0, ROOT)
|
|
12
|
+
|
|
13
|
+
import engine.fetch_chain as fc # noqa: E402
|
|
14
|
+
import engine.phase0 as phase0 # noqa: E402
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class _Response:
|
|
18
|
+
def __init__(self, status: int, text: str, payload=None, url=""):
|
|
19
|
+
self.status_code = status
|
|
20
|
+
self.text = text
|
|
21
|
+
self.url = url
|
|
22
|
+
self._payload = payload
|
|
23
|
+
|
|
24
|
+
def json(self):
|
|
25
|
+
return self._payload
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def t_pdf_table_is_preserved_as_markdown() -> None:
|
|
29
|
+
table = [["Name", "Value|Unit"], ["Alpha", "3\nkg"], [None, "4"]]
|
|
30
|
+
markdown = fc._pdf_table_to_markdown(table)
|
|
31
|
+
assert "| Name | Value\\|Unit |" in markdown
|
|
32
|
+
assert "| Alpha | 3 kg |" in markdown
|
|
33
|
+
assert "| | 4 |" in markdown
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def t_scanned_pdf_ocr_degrades_when_tools_are_missing() -> None:
|
|
37
|
+
with mock.patch.object(fc._shutil, "which", return_value=None):
|
|
38
|
+
text, error = fc._extract_pdf_ocr(b"%PDF-1.4\n")
|
|
39
|
+
assert text == ""
|
|
40
|
+
assert error == "pdf_ocr_tools_missing"
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def t_vtt_cleanup_removes_timing_and_duplicate_caption_lines() -> None:
|
|
44
|
+
vtt = """WEBVTT
|
|
45
|
+
|
|
46
|
+
00:00:00.000 --> 00:00:02.000
|
|
47
|
+
<c>Hello world</c>
|
|
48
|
+
|
|
49
|
+
00:00:01.500 --> 00:00:03.000
|
|
50
|
+
Hello world
|
|
51
|
+
Next line
|
|
52
|
+
"""
|
|
53
|
+
assert phase0._vtt_to_text(vtt) == "Hello world\nNext line"
|
|
54
|
+
segments = phase0._vtt_segments(vtt)
|
|
55
|
+
assert segments[0] == {
|
|
56
|
+
"start": "00:00:00.000", "end": "00:00:02.000", "text": "Hello world"}
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def t_subtitle_language_prefers_creator_korean_then_english() -> None:
|
|
60
|
+
metadata = {
|
|
61
|
+
"subtitles": {"en": [{"ext": "vtt"}], "ko": [{"ext": "vtt"}]},
|
|
62
|
+
"automatic_captions": {"ko-orig": [{"ext": "vtt"}]},
|
|
63
|
+
}
|
|
64
|
+
assert phase0._choose_subtitle_language(metadata, ("ko", "en")) == ("ko", "creator")
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def t_archive_route_returns_only_a_valid_wayback_snapshot() -> None:
|
|
68
|
+
api = _Response(200, "{}", {
|
|
69
|
+
"archived_snapshots": {"closest": {
|
|
70
|
+
"available": True,
|
|
71
|
+
"status": "200",
|
|
72
|
+
"timestamp": "20240102030405",
|
|
73
|
+
"url": "https://web.archive.org/web/20240102030405/https://example.com/a",
|
|
74
|
+
}}
|
|
75
|
+
})
|
|
76
|
+
page = _Response(200, "archived body " * 100,
|
|
77
|
+
url="https://web.archive.org/web/20240102030405/https://example.com/a")
|
|
78
|
+
with mock.patch.object(phase0, "_archive_get", side_effect=[api, page]):
|
|
79
|
+
result = phase0.route_archive("https://example.com/a", timeout=2)
|
|
80
|
+
assert result["ok"] is True
|
|
81
|
+
assert result["timestamp"] == "20240102030405"
|
|
82
|
+
assert result["content"].startswith("archived body")
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def t_archive_fallback_is_labeled_historical_in_evidence() -> None:
|
|
86
|
+
base = fc.FetchResult(
|
|
87
|
+
ok=False, final_url="https://example.com/a", verdict="not_found",
|
|
88
|
+
trace=[fc.Attempt(
|
|
89
|
+
phase="probe", executor="curl_cffi", url="https://example.com/a",
|
|
90
|
+
url_transform="original", impersonate="safari", referer="self_root",
|
|
91
|
+
verdict="not_found")],
|
|
92
|
+
)
|
|
93
|
+
archived = {
|
|
94
|
+
"ok": True,
|
|
95
|
+
"content": "<html><body>" + ("historical text " * 100) + "</body></html>",
|
|
96
|
+
"final_url": "https://web.archive.org/web/20240102030405/https://example.com/a",
|
|
97
|
+
"timestamp": "20240102030405",
|
|
98
|
+
"attempts": [{
|
|
99
|
+
"route": "wayback-snapshot", "ok": True, "status": 200,
|
|
100
|
+
"bytes": 1600, "note": "historical_snapshot",
|
|
101
|
+
}],
|
|
102
|
+
}
|
|
103
|
+
with mock.patch.object(phase0, "route_archive", return_value=archived):
|
|
104
|
+
result = fc._archive_fallback(
|
|
105
|
+
base, "https://example.com/a", timeout=2, proxy=None,
|
|
106
|
+
enable_extraction=True, enable_markdown=True, enable_maincontent=False)
|
|
107
|
+
assert result.ok is True
|
|
108
|
+
assert result.extraction_source.startswith("archive+")
|
|
109
|
+
assert result.evidence_record()["currentness"] == "historical_snapshot"
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def main() -> int:
|
|
113
|
+
tests = [v for k, v in sorted(globals().items()) if k.startswith("t_") and callable(v)]
|
|
114
|
+
failed = 0
|
|
115
|
+
for test in tests:
|
|
116
|
+
try:
|
|
117
|
+
test()
|
|
118
|
+
except AssertionError as exc:
|
|
119
|
+
print(f" x {test.__name__}: {exc}")
|
|
120
|
+
failed += 1
|
|
121
|
+
print(f"\n{'OK' if failed == 0 else 'FAIL'}: {len(tests) - failed}/{len(tests)} passed")
|
|
122
|
+
return 1 if failed else 0
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
if __name__ == "__main__":
|
|
126
|
+
sys.exit(main())
|
|
@@ -1,157 +1,157 @@
|
|
|
1
|
-
#!/usr/bin/env python3
|
|
2
|
-
"""Explicit cookie-session handoff regressions."""
|
|
3
|
-
from __future__ import annotations
|
|
4
|
-
|
|
5
|
-
import json
|
|
6
|
-
import os
|
|
7
|
-
import sys
|
|
8
|
-
import tempfile
|
|
9
|
-
import types
|
|
10
|
-
from unittest import mock
|
|
11
|
-
|
|
12
|
-
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
13
|
-
ROOT = os.path.abspath(os.path.join(HERE, "..", ".."))
|
|
14
|
-
sys.path.insert(0, ROOT)
|
|
15
|
-
|
|
16
|
-
from engine.session_input import load_cookie_file, redact_cookie_values # noqa: E402
|
|
17
|
-
from engine.transport import SessionPool # noqa: E402
|
|
18
|
-
import engine.executor as executor # noqa: E402
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
def _write(text: str) -> str:
|
|
22
|
-
handle = tempfile.NamedTemporaryFile("w", encoding="utf-8", delete=False)
|
|
23
|
-
try:
|
|
24
|
-
handle.write(text)
|
|
25
|
-
return handle.name
|
|
26
|
-
finally:
|
|
27
|
-
handle.close()
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
def t_json_cookie_export_is_scoped_to_target_host() -> None:
|
|
31
|
-
path = _write(json.dumps({"cookies": [
|
|
32
|
-
{"name": "sid", "value": "one", "domain": ".example.com", "path": "/"},
|
|
33
|
-
{"name": "other", "value": "two", "domain": "unrelated.test", "path": "/"},
|
|
34
|
-
]}))
|
|
35
|
-
try:
|
|
36
|
-
cookies = load_cookie_file(path, "https://news.example.com/article")
|
|
37
|
-
finally:
|
|
38
|
-
os.unlink(path)
|
|
39
|
-
assert cookies == [{
|
|
40
|
-
"name": "sid", "value": "one", "domain": ".example.com",
|
|
41
|
-
"path": "/", "secure": False,
|
|
42
|
-
}]
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
def t_netscape_cookie_file_is_supported() -> None:
|
|
46
|
-
path = _write(
|
|
47
|
-
"# Netscape HTTP Cookie File\n"
|
|
48
|
-
".example.com\tTRUE\t/\tTRUE\t2147483647\tsid\tsecret\n")
|
|
49
|
-
try:
|
|
50
|
-
cookies = load_cookie_file(path, "https://example.com/")
|
|
51
|
-
finally:
|
|
52
|
-
os.unlink(path)
|
|
53
|
-
assert cookies[0]["name"] == "sid"
|
|
54
|
-
assert cookies[0]["secure"] is True
|
|
55
|
-
assert cookies[0]["expires"] == 2147483647
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
def t_unrelated_only_cookie_file_is_rejected() -> None:
|
|
59
|
-
path = _write(json.dumps([
|
|
60
|
-
{"name": "sid", "value": "secret", "domain": "unrelated.test"},
|
|
61
|
-
]))
|
|
62
|
-
try:
|
|
63
|
-
try:
|
|
64
|
-
load_cookie_file(path, "https://example.com/")
|
|
65
|
-
except ValueError as exc:
|
|
66
|
-
assert "target host" in str(exc)
|
|
67
|
-
else:
|
|
68
|
-
raise AssertionError("unrelated cookies must not be accepted")
|
|
69
|
-
finally:
|
|
70
|
-
os.unlink(path)
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
def t_cookie_value_with_line_break_is_rejected() -> None:
|
|
74
|
-
path = _write(json.dumps([
|
|
75
|
-
{"name": "sid", "value": "secret\nleak", "domain": "example.com"},
|
|
76
|
-
]))
|
|
77
|
-
try:
|
|
78
|
-
try:
|
|
79
|
-
load_cookie_file(path, "https://example.com/")
|
|
80
|
-
except ValueError as exc:
|
|
81
|
-
assert "control" in str(exc)
|
|
82
|
-
else:
|
|
83
|
-
raise AssertionError("control characters must be rejected")
|
|
84
|
-
finally:
|
|
85
|
-
os.unlink(path)
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
def t_cookie_seed_applies_to_future_tls_identities_without_disk_persistence() -> None:
|
|
89
|
-
recorded = []
|
|
90
|
-
|
|
91
|
-
class _Cookies:
|
|
92
|
-
def set(self, name, value, **kwargs):
|
|
93
|
-
recorded.append((name, value, kwargs))
|
|
94
|
-
|
|
95
|
-
class _Session:
|
|
96
|
-
def __init__(self, **_kwargs):
|
|
97
|
-
self.cookies = _Cookies()
|
|
98
|
-
|
|
99
|
-
fake = types.SimpleNamespace(requests=types.SimpleNamespace(Session=_Session))
|
|
100
|
-
pool = SessionPool()
|
|
101
|
-
pool.seed_cookies("example.com", [{
|
|
102
|
-
"name": "sid", "value": "secret", "domain": ".example.com", "path": "/",
|
|
103
|
-
}])
|
|
104
|
-
with mock.patch.dict(sys.modules, {"curl_cffi": fake}):
|
|
105
|
-
assert pool.get("example.com", "safari") is not None
|
|
106
|
-
assert recorded == [("sid", "secret", {"domain": ".example.com", "path": "/"})]
|
|
107
|
-
assert pool.stats()["sessions"] == 1
|
|
108
|
-
pool.reset()
|
|
109
|
-
assert pool._seeds == {}
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
def t_browser_cookie_handoff_uses_an_ephemeral_profile() -> None:
|
|
113
|
-
captured = {}
|
|
114
|
-
cookies = [{
|
|
115
|
-
"name": "sid", "value": "secret", "domain": "example.com", "path": "/",
|
|
116
|
-
"secure": True,
|
|
117
|
-
}]
|
|
118
|
-
|
|
119
|
-
def fake_run(_template, args, **_kwargs):
|
|
120
|
-
captured.update(args)
|
|
121
|
-
html = "<html><body>" + ("real article text " * 400) + "</body></html>"
|
|
122
|
-
return 0, json.dumps({
|
|
123
|
-
"html": html, "finalUrl": "https://example.com/", "status": 200,
|
|
124
|
-
"cookies": [], "userAgent": "", "automation": "test", "innerText": "",
|
|
125
|
-
}), ""
|
|
126
|
-
|
|
127
|
-
with mock.patch.object(executor, "_resolve_node_deps", return_value=ROOT), \
|
|
128
|
-
mock.patch.object(executor, "_run_node_template", side_effect=fake_run):
|
|
129
|
-
attempt, content = executor.run_playwright_fallback(
|
|
130
|
-
"https://example.com/", profile_id="unknown_challenge",
|
|
131
|
-
force_executor="playwright_real_chrome", cookies=cookies)
|
|
132
|
-
assert attempt.status == 200 and content
|
|
133
|
-
assert captured["cookies"] == cookies
|
|
134
|
-
assert not os.path.exists(captured["profileDir"])
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
def t_cookie_secrets_are_removed_from_diagnostics() -> None:
|
|
138
|
-
text = redact_cookie_values(
|
|
139
|
-
"failed with sid=super-secret", [{"name": "sid", "value": "super-secret"}])
|
|
140
|
-
assert text == "failed with ***=***"
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
def main() -> int:
|
|
144
|
-
tests = [v for k, v in sorted(globals().items()) if k.startswith("t_") and callable(v)]
|
|
145
|
-
failed = 0
|
|
146
|
-
for test in tests:
|
|
147
|
-
try:
|
|
148
|
-
test()
|
|
149
|
-
except AssertionError as exc:
|
|
150
|
-
print(f" x {test.__name__}: {exc}")
|
|
151
|
-
failed += 1
|
|
152
|
-
print(f"\n{'OK' if failed == 0 else 'FAIL'}: {len(tests) - failed}/{len(tests)} passed")
|
|
153
|
-
return 1 if failed else 0
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
if __name__ == "__main__":
|
|
157
|
-
sys.exit(main())
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Explicit cookie-session handoff regressions."""
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import os
|
|
7
|
+
import sys
|
|
8
|
+
import tempfile
|
|
9
|
+
import types
|
|
10
|
+
from unittest import mock
|
|
11
|
+
|
|
12
|
+
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
13
|
+
ROOT = os.path.abspath(os.path.join(HERE, "..", ".."))
|
|
14
|
+
sys.path.insert(0, ROOT)
|
|
15
|
+
|
|
16
|
+
from engine.session_input import load_cookie_file, redact_cookie_values # noqa: E402
|
|
17
|
+
from engine.transport import SessionPool # noqa: E402
|
|
18
|
+
import engine.executor as executor # noqa: E402
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _write(text: str) -> str:
|
|
22
|
+
handle = tempfile.NamedTemporaryFile("w", encoding="utf-8", delete=False)
|
|
23
|
+
try:
|
|
24
|
+
handle.write(text)
|
|
25
|
+
return handle.name
|
|
26
|
+
finally:
|
|
27
|
+
handle.close()
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def t_json_cookie_export_is_scoped_to_target_host() -> None:
|
|
31
|
+
path = _write(json.dumps({"cookies": [
|
|
32
|
+
{"name": "sid", "value": "one", "domain": ".example.com", "path": "/"},
|
|
33
|
+
{"name": "other", "value": "two", "domain": "unrelated.test", "path": "/"},
|
|
34
|
+
]}))
|
|
35
|
+
try:
|
|
36
|
+
cookies = load_cookie_file(path, "https://news.example.com/article")
|
|
37
|
+
finally:
|
|
38
|
+
os.unlink(path)
|
|
39
|
+
assert cookies == [{
|
|
40
|
+
"name": "sid", "value": "one", "domain": ".example.com",
|
|
41
|
+
"path": "/", "secure": False,
|
|
42
|
+
}]
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def t_netscape_cookie_file_is_supported() -> None:
|
|
46
|
+
path = _write(
|
|
47
|
+
"# Netscape HTTP Cookie File\n"
|
|
48
|
+
".example.com\tTRUE\t/\tTRUE\t2147483647\tsid\tsecret\n")
|
|
49
|
+
try:
|
|
50
|
+
cookies = load_cookie_file(path, "https://example.com/")
|
|
51
|
+
finally:
|
|
52
|
+
os.unlink(path)
|
|
53
|
+
assert cookies[0]["name"] == "sid"
|
|
54
|
+
assert cookies[0]["secure"] is True
|
|
55
|
+
assert cookies[0]["expires"] == 2147483647
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def t_unrelated_only_cookie_file_is_rejected() -> None:
|
|
59
|
+
path = _write(json.dumps([
|
|
60
|
+
{"name": "sid", "value": "secret", "domain": "unrelated.test"},
|
|
61
|
+
]))
|
|
62
|
+
try:
|
|
63
|
+
try:
|
|
64
|
+
load_cookie_file(path, "https://example.com/")
|
|
65
|
+
except ValueError as exc:
|
|
66
|
+
assert "target host" in str(exc)
|
|
67
|
+
else:
|
|
68
|
+
raise AssertionError("unrelated cookies must not be accepted")
|
|
69
|
+
finally:
|
|
70
|
+
os.unlink(path)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def t_cookie_value_with_line_break_is_rejected() -> None:
|
|
74
|
+
path = _write(json.dumps([
|
|
75
|
+
{"name": "sid", "value": "secret\nleak", "domain": "example.com"},
|
|
76
|
+
]))
|
|
77
|
+
try:
|
|
78
|
+
try:
|
|
79
|
+
load_cookie_file(path, "https://example.com/")
|
|
80
|
+
except ValueError as exc:
|
|
81
|
+
assert "control" in str(exc)
|
|
82
|
+
else:
|
|
83
|
+
raise AssertionError("control characters must be rejected")
|
|
84
|
+
finally:
|
|
85
|
+
os.unlink(path)
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def t_cookie_seed_applies_to_future_tls_identities_without_disk_persistence() -> None:
|
|
89
|
+
recorded = []
|
|
90
|
+
|
|
91
|
+
class _Cookies:
|
|
92
|
+
def set(self, name, value, **kwargs):
|
|
93
|
+
recorded.append((name, value, kwargs))
|
|
94
|
+
|
|
95
|
+
class _Session:
|
|
96
|
+
def __init__(self, **_kwargs):
|
|
97
|
+
self.cookies = _Cookies()
|
|
98
|
+
|
|
99
|
+
fake = types.SimpleNamespace(requests=types.SimpleNamespace(Session=_Session))
|
|
100
|
+
pool = SessionPool()
|
|
101
|
+
pool.seed_cookies("example.com", [{
|
|
102
|
+
"name": "sid", "value": "secret", "domain": ".example.com", "path": "/",
|
|
103
|
+
}])
|
|
104
|
+
with mock.patch.dict(sys.modules, {"curl_cffi": fake}):
|
|
105
|
+
assert pool.get("example.com", "safari") is not None
|
|
106
|
+
assert recorded == [("sid", "secret", {"domain": ".example.com", "path": "/"})]
|
|
107
|
+
assert pool.stats()["sessions"] == 1
|
|
108
|
+
pool.reset()
|
|
109
|
+
assert pool._seeds == {}
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def t_browser_cookie_handoff_uses_an_ephemeral_profile() -> None:
|
|
113
|
+
captured = {}
|
|
114
|
+
cookies = [{
|
|
115
|
+
"name": "sid", "value": "secret", "domain": "example.com", "path": "/",
|
|
116
|
+
"secure": True,
|
|
117
|
+
}]
|
|
118
|
+
|
|
119
|
+
def fake_run(_template, args, **_kwargs):
|
|
120
|
+
captured.update(args)
|
|
121
|
+
html = "<html><body>" + ("real article text " * 400) + "</body></html>"
|
|
122
|
+
return 0, json.dumps({
|
|
123
|
+
"html": html, "finalUrl": "https://example.com/", "status": 200,
|
|
124
|
+
"cookies": [], "userAgent": "", "automation": "test", "innerText": "",
|
|
125
|
+
}), ""
|
|
126
|
+
|
|
127
|
+
with mock.patch.object(executor, "_resolve_node_deps", return_value=ROOT), \
|
|
128
|
+
mock.patch.object(executor, "_run_node_template", side_effect=fake_run):
|
|
129
|
+
attempt, content = executor.run_playwright_fallback(
|
|
130
|
+
"https://example.com/", profile_id="unknown_challenge",
|
|
131
|
+
force_executor="playwright_real_chrome", cookies=cookies)
|
|
132
|
+
assert attempt.status == 200 and content
|
|
133
|
+
assert captured["cookies"] == cookies
|
|
134
|
+
assert not os.path.exists(captured["profileDir"])
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def t_cookie_secrets_are_removed_from_diagnostics() -> None:
|
|
138
|
+
text = redact_cookie_values(
|
|
139
|
+
"failed with sid=super-secret", [{"name": "sid", "value": "super-secret"}])
|
|
140
|
+
assert text == "failed with ***=***"
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def main() -> int:
|
|
144
|
+
tests = [v for k, v in sorted(globals().items()) if k.startswith("t_") and callable(v)]
|
|
145
|
+
failed = 0
|
|
146
|
+
for test in tests:
|
|
147
|
+
try:
|
|
148
|
+
test()
|
|
149
|
+
except AssertionError as exc:
|
|
150
|
+
print(f" x {test.__name__}: {exc}")
|
|
151
|
+
failed += 1
|
|
152
|
+
print(f"\n{'OK' if failed == 0 else 'FAIL'}: {len(tests) - failed}/{len(tests)} passed")
|
|
153
|
+
return 1 if failed else 0
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
if __name__ == "__main__":
|
|
157
|
+
sys.exit(main())
|
|
@@ -113,17 +113,17 @@ def t_no_extractor_graceful() -> None:
|
|
|
113
113
|
print(f" ✓ neither extractor -> graceful ({err}), no crash")
|
|
114
114
|
|
|
115
115
|
|
|
116
|
-
def t_no_agpl_import() -> None:
|
|
117
|
-
# AGPL guard: no actual import of pymupdf / fitz (the pymupdf import name).
|
|
118
|
-
# A prohibition mention in a comment is fine — only real imports are banned.
|
|
119
|
-
import pathlib
|
|
120
|
-
import re
|
|
121
|
-
hits = []
|
|
122
|
-
pattern = re.compile(r"^\s*(import|from)\s+(pymupdf|fitz)\b", re.MULTILINE)
|
|
123
|
-
for path in (pathlib.Path(ROOT) / "engine").rglob("*.py"):
|
|
124
|
-
if pattern.search(path.read_text(encoding="utf-8", errors="ignore")):
|
|
125
|
-
hits.append(str(path))
|
|
126
|
-
assert not hits, f"AGPL pymupdf/fitz import found: {hits}"
|
|
116
|
+
def t_no_agpl_import() -> None:
|
|
117
|
+
# AGPL guard: no actual import of pymupdf / fitz (the pymupdf import name).
|
|
118
|
+
# A prohibition mention in a comment is fine — only real imports are banned.
|
|
119
|
+
import pathlib
|
|
120
|
+
import re
|
|
121
|
+
hits = []
|
|
122
|
+
pattern = re.compile(r"^\s*(import|from)\s+(pymupdf|fitz)\b", re.MULTILINE)
|
|
123
|
+
for path in (pathlib.Path(ROOT) / "engine").rglob("*.py"):
|
|
124
|
+
if pattern.search(path.read_text(encoding="utf-8", errors="ignore")):
|
|
125
|
+
hits.append(str(path))
|
|
126
|
+
assert not hits, f"AGPL pymupdf/fitz import found: {hits}"
|
|
127
127
|
print(" ✓ no AGPL pymupdf/fitz import in engine/ (comment mention allowed)")
|
|
128
128
|
|
|
129
129
|
|
|
@@ -87,9 +87,9 @@ def _install_common_patches(*, probe_verdict, grid_verdict, fb_order):
|
|
|
87
87
|
def fake_load_profile(pid, profiles=None):
|
|
88
88
|
return {"fallback_when_challenge": list(fb_order)}
|
|
89
89
|
|
|
90
|
-
def fake_run_pw(url, *, profile_id, success_selectors=None, device_class="auto",
|
|
91
|
-
timeout=90, profile_dir=None, force_executor=None, proxy=None,
|
|
92
|
-
cookies=None):
|
|
90
|
+
def fake_run_pw(url, *, profile_id, success_selectors=None, device_class="auto",
|
|
91
|
+
timeout=90, profile_dir=None, force_executor=None, proxy=None,
|
|
92
|
+
cookies=None):
|
|
93
93
|
browser_calls.append(force_executor)
|
|
94
94
|
att = Attempt(phase="fallback", executor=force_executor, url=url,
|
|
95
95
|
url_transform="original", impersonate=None, referer="")
|