@sayknow-cli/coding-agent 0.5.2 → 0.5.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +66 -16
- package/dist/types/cli/update-cli.d.ts +8 -1
- package/dist/types/config/atomic-yaml-patch.d.ts +20 -1
- package/dist/types/config/keybindings.d.ts +10 -0
- package/dist/types/config/model-registry.d.ts +1 -1
- package/dist/types/config/models-config-schema.d.ts +4 -0
- package/dist/types/config/settings-schema.d.ts +73 -0
- package/dist/types/dap/client.d.ts +1 -0
- package/dist/types/lsp/client.d.ts +5 -0
- package/dist/types/modes/components/welcome.d.ts +1 -0
- package/dist/types/modes/controllers/selector-controller.d.ts +3 -1
- package/dist/types/modes/interactive-mode.d.ts +13 -1
- package/dist/types/modes/types.d.ts +7 -1
- package/dist/types/runtime-mcp/transports/stdio.d.ts +2 -0
- package/dist/types/session/agent-session.d.ts +8 -0
- package/dist/types/session/internal/managed-session-scope.d.ts +3 -1
- package/dist/types/session/internal/managed-session-storage.d.ts +2 -0
- package/dist/types/session/session-manager.d.ts +9 -0
- package/dist/types/session/session-storage.d.ts +17 -0
- package/dist/types/session-import/claude.d.ts +26 -0
- package/dist/types/session-import/codex.d.ts +22 -0
- package/dist/types/session-import/command.d.ts +20 -0
- package/dist/types/session-import/detect.d.ts +22 -0
- package/dist/types/session-import/index.d.ts +7 -0
- package/dist/types/session-import/redact.d.ts +23 -0
- package/dist/types/session-import/service.d.ts +28 -0
- package/dist/types/session-import/types.d.ts +125 -0
- package/dist/types/skc-runtime/launch-worktree.d.ts +17 -1
- package/dist/types/slash-commands/builtin-registry.d.ts +4 -0
- package/dist/types/slash-commands/types.d.ts +5 -0
- package/dist/types/system-prompt.d.ts +5 -0
- package/dist/types/tools/ask.d.ts +5 -0
- package/dist/types/utils/mupdf-wasm.d.ts +1 -0
- package/package.json +13 -7
- package/scripts/compile-args.ts +8 -4
- package/scripts/generate-sdk-operation-inventory.ts +5 -6
- package/src/cli/config-cli.ts +23 -8
- package/src/cli/update-cli.ts +265 -21
- package/src/config/atomic-yaml-patch.ts +167 -30
- package/src/config/keybindings.ts +10 -0
- package/src/config/model-profiles.ts +84 -16
- package/src/config/model-registry.ts +60 -38
- package/src/config/models-config-schema.ts +1 -1
- package/src/config/settings-schema.ts +79 -0
- package/src/config/settings.ts +13 -2
- package/src/dap/client.ts +78 -30
- package/src/internal-urls/docs-index.generated.ts +5 -4
- package/src/lsp/client.ts +17 -29
- package/src/modes/action-registry.ts +2 -0
- package/src/modes/components/welcome.ts +5 -0
- package/src/modes/controllers/goal-mode-controller.ts +7 -2
- package/src/modes/controllers/input-controller.ts +71 -6
- package/src/modes/controllers/plan-mode-controller.ts +8 -1
- package/src/modes/controllers/runtime-mcp-command-controller.ts +14 -0
- package/src/modes/controllers/selector-controller.ts +47 -19
- package/src/modes/interactive-mode.ts +54 -4
- package/src/modes/types.ts +5 -1
- package/src/prompts/system/system-prompt.md +8 -0
- package/src/runtime-mcp/transports/stdio.ts +96 -33
- package/src/sdk/broker/lifecycle.ts +6 -5
- package/src/sdk/protocol/operation-inventory.generated.json +33 -0
- package/src/sdk/session.ts +27 -18
- package/src/session/agent-session.ts +155 -15
- package/src/session/internal/managed-session-scope.ts +41 -6
- package/src/session/internal/managed-session-storage.ts +14 -0
- package/src/session/session-manager.ts +113 -11
- package/src/session/session-storage.ts +70 -0
- package/src/session-import/claude.ts +382 -0
- package/src/session-import/codex.ts +458 -0
- package/src/session-import/command.ts +61 -0
- package/src/session-import/detect.ts +133 -0
- package/src/session-import/index.ts +27 -0
- package/src/session-import/redact.ts +125 -0
- package/src/session-import/service.ts +749 -0
- package/src/session-import/types.ts +163 -0
- package/src/skc-runtime/launch-worktree.ts +277 -4
- package/src/slash-commands/acp-builtins.ts +5 -1
- package/src/slash-commands/builtin-registry.ts +100 -1
- package/src/slash-commands/types.ts +5 -0
- package/src/system-prompt.ts +7 -0
- package/src/tools/ask.ts +5 -0
- package/src/tools/fetch.ts +7 -2
- package/src/utils/markit.ts +24 -2
- package/src/utils/mupdf-wasm.ts +26 -0
- package/src/export/html/template.css +0 -1060
- package/src/export/html/template.html +0 -47
- package/src/export/html/template.js +0 -2348
- package/vendor/insane-search/engine/tests/test_hardening.py +0 -57
- package/vendor/insane-search/engine/tests/test_smoke.py +0 -152
- package/vendor/insane-search/engine/tests/test_u1.py +0 -200
- package/vendor/insane-search/engine/tests/test_u4.py +0 -131
- package/vendor/insane-search/engine/tests/test_u5.py +0 -163
- package/vendor/insane-search/engine/tests/test_u7.py +0 -124
- package/vendor/insane-search/engine/tests/test_u8.py +0 -216
|
@@ -1,57 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env python3
|
|
2
|
-
from __future__ import annotations
|
|
3
|
-
|
|
4
|
-
import os
|
|
5
|
-
import subprocess
|
|
6
|
-
import sys
|
|
7
|
-
import unittest
|
|
8
|
-
|
|
9
|
-
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
10
|
-
ROOT = os.path.abspath(os.path.join(HERE, "..", ".."))
|
|
11
|
-
sys.path.insert(0, ROOT)
|
|
12
|
-
|
|
13
|
-
from engine.fetch_chain import FetchResult # noqa: E402
|
|
14
|
-
from engine.transport import SessionPool # noqa: E402
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
class FetchResultJsonContractTest(unittest.TestCase):
|
|
18
|
-
def test_to_dict_omits_content_by_default_but_reports_length(self):
|
|
19
|
-
payload = FetchResult(ok=True, content="recovered").to_dict()
|
|
20
|
-
self.assertNotIn("content", payload)
|
|
21
|
-
self.assertEqual(payload["content_length"], len("recovered"))
|
|
22
|
-
self.assertFalse(payload["content_truncated"])
|
|
23
|
-
|
|
24
|
-
def test_to_dict_includes_bounded_content_for_cli_json(self):
|
|
25
|
-
payload = FetchResult(ok=True, content="abcdef").to_dict(include_content=True, content_limit=3)
|
|
26
|
-
self.assertEqual(payload["content"], "abc")
|
|
27
|
-
self.assertEqual(payload["content_length"], 6)
|
|
28
|
-
self.assertTrue(payload["content_truncated"])
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
class RedirectSafetyTest(unittest.TestCase):
|
|
32
|
-
def test_transport_redirect_to_private_target_is_blocked(self):
|
|
33
|
-
class Resp:
|
|
34
|
-
status_code = 302
|
|
35
|
-
headers = {"Location": "http://127.0.0.1/private"}
|
|
36
|
-
text = ""
|
|
37
|
-
url = "https://public.example/redirect"
|
|
38
|
-
|
|
39
|
-
resp, err = SessionPool._fetch_following(lambda _url: Resp(), "https://public.example/redirect", False, 10, None)
|
|
40
|
-
self.assertIsNone(resp)
|
|
41
|
-
self.assertTrue(err.startswith("ssrf_redirect_blocked:"), err)
|
|
42
|
-
|
|
43
|
-
def test_playwright_templates_reject_private_initial_url_before_browser_launch(self):
|
|
44
|
-
template = os.path.join(ROOT, "engine", "templates", "playwright_real_chrome.js")
|
|
45
|
-
proc = subprocess.run(
|
|
46
|
-
["node", template],
|
|
47
|
-
input='{"url":"http://127.0.0.1/private"}',
|
|
48
|
-
capture_output=True,
|
|
49
|
-
text=True,
|
|
50
|
-
timeout=10,
|
|
51
|
-
)
|
|
52
|
-
self.assertNotEqual(proc.returncode, 0)
|
|
53
|
-
self.assertIn("unsafe_url", proc.stderr)
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
if __name__ == "__main__":
|
|
57
|
-
unittest.main()
|
|
@@ -1,152 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env python3
|
|
2
|
-
"""Smoke / regression test for the generic fetch chain.
|
|
3
|
-
|
|
4
|
-
These tests hit real endpoints — mark as online / integration. They verify
|
|
5
|
-
behaviour patterns, not content. No assertions on specific site brands.
|
|
6
|
-
|
|
7
|
-
Run manually:
|
|
8
|
-
python3 engine/tests/test_smoke.py
|
|
9
|
-
"""
|
|
10
|
-
from __future__ import annotations
|
|
11
|
-
|
|
12
|
-
import json
|
|
13
|
-
import os
|
|
14
|
-
import sys
|
|
15
|
-
import time
|
|
16
|
-
|
|
17
|
-
# Allow running from anywhere.
|
|
18
|
-
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
19
|
-
ROOT = os.path.abspath(os.path.join(HERE, "..", ".."))
|
|
20
|
-
sys.path.insert(0, ROOT)
|
|
21
|
-
|
|
22
|
-
from engine import fetch # noqa: E402
|
|
23
|
-
from engine.validators import validate, Verdict # noqa: E402
|
|
24
|
-
from engine.waf_detector import detect, _load_profiles # noqa: E402
|
|
25
|
-
from engine.url_transforms import iter_transformed # noqa: E402
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
# --- unit-level -------------------------------------------------------------
|
|
29
|
-
def t_validator_tiny_body_is_challenge():
|
|
30
|
-
class R:
|
|
31
|
-
status_code = 200
|
|
32
|
-
text = "<html>short</html>"
|
|
33
|
-
headers = {}
|
|
34
|
-
cookies = type("C", (), {"jar": iter(())})()
|
|
35
|
-
vr = validate(R())
|
|
36
|
-
assert vr.verdict == Verdict.CHALLENGE, vr.verdict
|
|
37
|
-
assert any("tiny_body" in r for r in vr.reasons)
|
|
38
|
-
print(" ✓ tiny body → challenge")
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
def t_validator_marker_is_challenge():
|
|
42
|
-
class R:
|
|
43
|
-
status_code = 200
|
|
44
|
-
text = "<html>" + ("x" * 5000) + " sec-if-cpt-container found </html>"
|
|
45
|
-
headers = {}
|
|
46
|
-
cookies = type("C", (), {"jar": iter(())})()
|
|
47
|
-
vr = validate(R())
|
|
48
|
-
assert vr.verdict == Verdict.CHALLENGE, vr.verdict
|
|
49
|
-
print(" ✓ challenge marker → challenge")
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
def t_validator_weak_ok_without_selectors():
|
|
53
|
-
class R:
|
|
54
|
-
status_code = 200
|
|
55
|
-
text = "<html>" + ("x" * 5000) + "</html>"
|
|
56
|
-
headers = {}
|
|
57
|
-
cookies = type("C", (), {"jar": iter(())})()
|
|
58
|
-
vr = validate(R())
|
|
59
|
-
assert vr.verdict == Verdict.WEAK_OK, vr.verdict
|
|
60
|
-
print(" ✓ clean body w/o selectors → weak_ok")
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
def t_validator_strong_ok_with_selectors():
|
|
64
|
-
class R:
|
|
65
|
-
status_code = 200
|
|
66
|
-
text = "<html><body>" + ("x" * 5000) + "<article>hello</article></body></html>"
|
|
67
|
-
headers = {}
|
|
68
|
-
cookies = type("C", (), {"jar": iter(())})()
|
|
69
|
-
vr = validate(R(), success_selectors=["article"])
|
|
70
|
-
assert vr.verdict == Verdict.STRONG_OK, vr.verdict
|
|
71
|
-
assert "article" in vr.matched_selectors
|
|
72
|
-
print(" ✓ selectors matched → strong_ok")
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
def t_profiles_load():
|
|
76
|
-
p = _load_profiles()
|
|
77
|
-
for required in ("akamai_bot_manager", "cloudflare_turnstile", "unknown_challenge"):
|
|
78
|
-
assert required in p, f"missing profile: {required}"
|
|
79
|
-
print(f" ✓ profiles loaded ({len(p)} keys)")
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
def t_url_transforms():
|
|
83
|
-
# www → m
|
|
84
|
-
out = iter_transformed("https://www.example.com/a", ["original", "mobile_subdomain"])
|
|
85
|
-
urls = [u for _, u in out]
|
|
86
|
-
assert "https://www.example.com/a" in urls
|
|
87
|
-
assert "https://m.example.com/a" in urls, urls
|
|
88
|
-
# apex with am_prefix
|
|
89
|
-
out2 = iter_transformed("https://example.com/", ["original", "am_prefix"])
|
|
90
|
-
urls2 = [u for _, u in out2]
|
|
91
|
-
assert "https://m.example.com/" in urls2, urls2
|
|
92
|
-
print(f" ✓ url_transforms produce expected forms")
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
# --- online (network) -------------------------------------------------------
|
|
96
|
-
def t_online_benign_site():
|
|
97
|
-
"""A simple, usually-open site should pass probe directly when selectors provided."""
|
|
98
|
-
# example.com serves ~1.2KB content — below tiny_body threshold — but with
|
|
99
|
-
# success_selectors we trust caller's "content exists" definition.
|
|
100
|
-
r = fetch(
|
|
101
|
-
"https://example.com/",
|
|
102
|
-
success_selectors=["h1", "p"],
|
|
103
|
-
timeout=15,
|
|
104
|
-
max_attempts=3,
|
|
105
|
-
enable_playwright=False,
|
|
106
|
-
)
|
|
107
|
-
assert r.ok, f"{r.summary} | trace: {[a.verdict for a in r.trace]}"
|
|
108
|
-
assert r.verdict in ("strong_ok", "weak_ok"), r.verdict
|
|
109
|
-
print(f" ✓ benign site → verdict={r.verdict} size={len(r.content)}")
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
def t_online_trace_shape():
|
|
113
|
-
"""Even on failure, trace should be populated and well-formed."""
|
|
114
|
-
r = fetch("https://httpbin.org/status/403", timeout=10, max_attempts=3, enable_playwright=False)
|
|
115
|
-
assert isinstance(r.trace, list) and len(r.trace) >= 1
|
|
116
|
-
for att in r.trace:
|
|
117
|
-
d = att.to_dict()
|
|
118
|
-
assert "phase" in d and "executor" in d and "verdict" in d
|
|
119
|
-
print(f" ✓ httpbin 403 → trace_len={len(r.trace)} final={r.verdict}")
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
ALL_TESTS = [
|
|
123
|
-
("validator_tiny_body_is_challenge", t_validator_tiny_body_is_challenge),
|
|
124
|
-
("validator_marker_is_challenge", t_validator_marker_is_challenge),
|
|
125
|
-
("validator_weak_ok_without_selectors", t_validator_weak_ok_without_selectors),
|
|
126
|
-
("validator_strong_ok_with_selectors", t_validator_strong_ok_with_selectors),
|
|
127
|
-
("profiles_load", t_profiles_load),
|
|
128
|
-
("url_transforms", t_url_transforms),
|
|
129
|
-
("online_benign_site", t_online_benign_site),
|
|
130
|
-
("online_trace_shape", t_online_trace_shape),
|
|
131
|
-
]
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
def main() -> int:
|
|
135
|
-
passed, failed = 0, 0
|
|
136
|
-
for name, fn in ALL_TESTS:
|
|
137
|
-
try:
|
|
138
|
-
print(f"[{name}]")
|
|
139
|
-
fn()
|
|
140
|
-
passed += 1
|
|
141
|
-
except AssertionError as e:
|
|
142
|
-
failed += 1
|
|
143
|
-
print(f" ✗ FAIL: {e}")
|
|
144
|
-
except Exception as e:
|
|
145
|
-
failed += 1
|
|
146
|
-
print(f" ✗ ERROR: {type(e).__name__}: {e}")
|
|
147
|
-
print(f"\n{passed} passed, {failed} failed")
|
|
148
|
-
return 0 if failed == 0 else 1
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
if __name__ == "__main__":
|
|
152
|
-
sys.exit(main())
|
|
@@ -1,200 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env python3
|
|
2
|
-
"""U1 regression tests — validator v2 + diversity scheduler.
|
|
3
|
-
|
|
4
|
-
Deterministic, network-free. Locks in the multi-AI-review fixes:
|
|
5
|
-
* grid diversity under a small cap (all TLS families + both transforms)
|
|
6
|
-
* avoid targets deprioritized, NOT deleted
|
|
7
|
-
* validator: small JSON ok, _abck-unresolved non-terminal, soft-marker
|
|
8
|
-
overridden by selector, status semantics.
|
|
9
|
-
|
|
10
|
-
Run: python3 engine/tests/test_u1.py
|
|
11
|
-
"""
|
|
12
|
-
from __future__ import annotations
|
|
13
|
-
|
|
14
|
-
import os
|
|
15
|
-
import sys
|
|
16
|
-
|
|
17
|
-
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
18
|
-
ROOT = os.path.abspath(os.path.join(HERE, "..", ".."))
|
|
19
|
-
sys.path.insert(0, ROOT)
|
|
20
|
-
|
|
21
|
-
from engine.validators import validate, Verdict # noqa: E402
|
|
22
|
-
from engine.waf_detector import _load_profiles # noqa: E402
|
|
23
|
-
from engine.fetch_chain import _build_plan, _family # noqa: E402
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
class _Ck:
|
|
27
|
-
def __init__(self, name, value):
|
|
28
|
-
self.name, self.value = name, value
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
class _Jar:
|
|
32
|
-
def __init__(self, d):
|
|
33
|
-
self.jar = [_Ck(k, v) for k, v in d.items()]
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
class _Resp:
|
|
37
|
-
def __init__(self, status=200, text="", headers=None, cookies=None):
|
|
38
|
-
self.status_code = status
|
|
39
|
-
self.text = text
|
|
40
|
-
self.headers = headers or {}
|
|
41
|
-
self.cookies = _Jar(cookies or {})
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
class _Hit:
|
|
45
|
-
def __init__(self, pid):
|
|
46
|
-
self.profile_id = pid
|
|
47
|
-
self.confidence = 0.9
|
|
48
|
-
self.signals = []
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
# ---------- scheduler ----------
|
|
52
|
-
def t_scheduler_diversity_under_cap():
|
|
53
|
-
profiles = _load_profiles()
|
|
54
|
-
plan = _build_plan("https://www.example.com/p", [_Hit("akamai_bot_manager")],
|
|
55
|
-
profiles, "auto", "safari", "self_root")
|
|
56
|
-
budget = 11 # max_attempts 12 - probe
|
|
57
|
-
head = plan[:budget]
|
|
58
|
-
fams = set(_family(c.impersonate) for c in head)
|
|
59
|
-
transforms = set(c.transform for c in head)
|
|
60
|
-
assert fams == {"safari", "safari_ios", "chrome", "chrome_android", "edge"}, fams
|
|
61
|
-
assert transforms == {"original", "mobile_subdomain"}, transforms
|
|
62
|
-
print(f" ✓ first {budget} cover all families {sorted(fams)} + transforms {sorted(transforms)}")
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
def t_scheduler_avoid_deprioritized_not_deleted():
|
|
66
|
-
profiles = _load_profiles()
|
|
67
|
-
plan = _build_plan("https://www.example.com/p", [_Hit("akamai_bot_manager")],
|
|
68
|
-
profiles, "auto", "safari", "self_root")
|
|
69
|
-
imps = [c.impersonate for c in plan]
|
|
70
|
-
# chrome145/146 are in avoid; must still be present (exhaustive) but late.
|
|
71
|
-
assert "chrome145" in imps and "chrome146" in imps, "avoid targets were deleted!"
|
|
72
|
-
pos145 = min(i for i, x in enumerate(imps) if x == "chrome145")
|
|
73
|
-
early = imps[: len(imps) // 2]
|
|
74
|
-
assert "chrome145" not in early, "avoid target not deprioritized"
|
|
75
|
-
print(f" ✓ avoid targets retained but late (chrome145 idx={pos145}/{len(imps)})")
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
def t_scheduler_desktop_drops_mobile_transform():
|
|
79
|
-
profiles = _load_profiles()
|
|
80
|
-
plan = _build_plan("https://www.example.com/p", [_Hit("akamai_bot_manager")],
|
|
81
|
-
profiles, "desktop", "safari", "self_root")
|
|
82
|
-
transforms = set(c.transform for c in plan)
|
|
83
|
-
fams = set(_family(c.impersonate) for c in plan)
|
|
84
|
-
assert "mobile_subdomain" not in transforms, transforms
|
|
85
|
-
assert "safari_ios" not in fams and "chrome_android" not in fams, fams
|
|
86
|
-
print(f" ✓ desktop drops mobile transform & mobile TLS (transforms={sorted(transforms)})")
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
# ---------- validator v2 ----------
|
|
90
|
-
def t_validator_small_json_ok():
|
|
91
|
-
r = _Resp(200, '{"items":[{"id":1}],"total":1}', headers={"Content-Type": "application/json"})
|
|
92
|
-
v = validate(r)
|
|
93
|
-
assert v.verdict == Verdict.WEAK_OK, v.verdict
|
|
94
|
-
assert not (v.verdict == Verdict.CHALLENGE)
|
|
95
|
-
print(f" ✓ small JSON → {v.verdict.value} (was challenge)")
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
def t_validator_abck_unresolved_is_non_terminal():
|
|
99
|
-
r = _Resp(200, "<html>" + "x" * 5000 + "</html>", cookies={"_abck": "AA~-1~bb"})
|
|
100
|
-
v = validate(r)
|
|
101
|
-
assert v.verdict == Verdict.SUSPECT_OK, v.verdict
|
|
102
|
-
assert v.ok is False, "SUSPECT_OK must not count as terminal success"
|
|
103
|
-
print(f" ✓ _abck unresolved → {v.verdict.value}, ok={v.ok} (was weak_ok/ok=True)")
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
def t_validator_soft_marker_overridden_by_selector():
|
|
107
|
-
html = "<html><script>var s='captcha';</script><body>" + "x" * 5000 + "<main id='c'>real</main></body></html>"
|
|
108
|
-
v = validate(_Resp(200, html), success_selectors=["#c"])
|
|
109
|
-
assert v.verdict == Verdict.STRONG_OK, v.verdict
|
|
110
|
-
print(f" ✓ 'captcha' word + matching selector → {v.verdict.value} (was challenge)")
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
def t_validator_hard_marker_still_challenge():
|
|
114
|
-
v = validate(_Resp(200, "<html>" + "x" * 5000 + " sec-if-cpt-container </html>"))
|
|
115
|
-
assert v.verdict == Verdict.CHALLENGE, v.verdict
|
|
116
|
-
print(f" ✓ hard marker still → {v.verdict.value}")
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
def t_validator_status_semantics():
|
|
120
|
-
assert validate(_Resp(429, "slow down")).verdict == Verdict.RATE_LIMITED
|
|
121
|
-
assert validate(_Resp(401, "nope")).verdict == Verdict.AUTH_REQUIRED
|
|
122
|
-
assert validate(_Resp(404, "gone")).verdict == Verdict.NOT_FOUND
|
|
123
|
-
assert validate(_Resp(503, "later")).verdict == Verdict.BLOCKED
|
|
124
|
-
print(" ✓ status semantics 429/401/404/503 differentiated")
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
def t_validator_byte_size_not_char_count():
|
|
128
|
-
# 1500 Korean chars = 1500 chars but 4500 bytes (>threshold) → not tiny.
|
|
129
|
-
body = "가" * 1500
|
|
130
|
-
v = validate(_Resp(200, body, headers={"Content-Type": "text/html"}))
|
|
131
|
-
# 4500 bytes ≥ 3000 → not tiny_body; no markers/selectors → weak_ok
|
|
132
|
-
assert v.body_size >= 3000, v.body_size
|
|
133
|
-
assert v.verdict == Verdict.WEAK_OK, (v.verdict, v.body_size)
|
|
134
|
-
print(f" ✓ byte size counts UTF-8 bytes ({v.body_size}B from 1500 chars) → {v.verdict.value}")
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
def t_validator_small_complete_page_is_weak_ok():
|
|
138
|
-
# example.com is a complete ~600B HTML document with real text — a small but
|
|
139
|
-
# genuine page must NOT be mislabelled a challenge stub (regression guard).
|
|
140
|
-
body = ('<!doctype html><html lang="en"><head><title>Example Domain</title>'
|
|
141
|
-
'</head><body><div><h1>Example Domain</h1><p>This domain is for use in '
|
|
142
|
-
'documentation examples without needing permission.</p>'
|
|
143
|
-
'<p><a href="https://iana.org/domains/example">Learn more</a></p>'
|
|
144
|
-
'</div></body></html>')
|
|
145
|
-
v = validate(_Resp(200, body, headers={"Content-Type": "text/html"}))
|
|
146
|
-
assert v.body_size < 3000, v.body_size
|
|
147
|
-
assert v.verdict == Verdict.WEAK_OK, (v.verdict, v.reasons)
|
|
148
|
-
print(f" ✓ small complete page → {v.verdict.value} ({v.reasons})")
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
def t_validator_small_script_stub_still_challenge():
|
|
152
|
-
# Script-only tiny body (no visible text) is still a suspicious stub.
|
|
153
|
-
body = '<html><head></head><body><script src="/cdn-cgi/challenge.js"></script></body></html>'
|
|
154
|
-
v = validate(_Resp(200, body, headers={"Content-Type": "text/html"}))
|
|
155
|
-
assert v.verdict == Verdict.CHALLENGE, (v.verdict, v.reasons)
|
|
156
|
-
print(f" ✓ script-only tiny body → {v.verdict.value}")
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
def t_validator_small_fragment_still_challenge():
|
|
160
|
-
# Incomplete fragment (no closing </html>/</body>) stays suspicious.
|
|
161
|
-
v = validate(_Resp(200, "<div>loading", headers={"Content-Type": "text/html"}))
|
|
162
|
-
assert v.verdict == Verdict.CHALLENGE, (v.verdict, v.reasons)
|
|
163
|
-
print(f" ✓ incomplete fragment → {v.verdict.value}")
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
ALL = [
|
|
167
|
-
("scheduler_diversity_under_cap", t_scheduler_diversity_under_cap),
|
|
168
|
-
("scheduler_avoid_deprioritized_not_deleted", t_scheduler_avoid_deprioritized_not_deleted),
|
|
169
|
-
("scheduler_desktop_drops_mobile_transform", t_scheduler_desktop_drops_mobile_transform),
|
|
170
|
-
("validator_small_json_ok", t_validator_small_json_ok),
|
|
171
|
-
("validator_abck_unresolved_is_non_terminal", t_validator_abck_unresolved_is_non_terminal),
|
|
172
|
-
("validator_soft_marker_overridden_by_selector", t_validator_soft_marker_overridden_by_selector),
|
|
173
|
-
("validator_hard_marker_still_challenge", t_validator_hard_marker_still_challenge),
|
|
174
|
-
("validator_status_semantics", t_validator_status_semantics),
|
|
175
|
-
("validator_byte_size_not_char_count", t_validator_byte_size_not_char_count),
|
|
176
|
-
("validator_small_complete_page_is_weak_ok", t_validator_small_complete_page_is_weak_ok),
|
|
177
|
-
("validator_small_script_stub_still_challenge", t_validator_small_script_stub_still_challenge),
|
|
178
|
-
("validator_small_fragment_still_challenge", t_validator_small_fragment_still_challenge),
|
|
179
|
-
]
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
def main() -> int:
|
|
183
|
-
p = f = 0
|
|
184
|
-
for name, fn in ALL:
|
|
185
|
-
try:
|
|
186
|
-
print(f"[{name}]")
|
|
187
|
-
fn()
|
|
188
|
-
p += 1
|
|
189
|
-
except AssertionError as e:
|
|
190
|
-
f += 1
|
|
191
|
-
print(f" ✗ FAIL: {e}")
|
|
192
|
-
except Exception as e:
|
|
193
|
-
f += 1
|
|
194
|
-
print(f" ✗ ERROR: {type(e).__name__}: {e}")
|
|
195
|
-
print(f"\n{p} passed, {f} failed")
|
|
196
|
-
return 0 if f == 0 else 1
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
if __name__ == "__main__":
|
|
200
|
-
sys.exit(main())
|
|
@@ -1,131 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env python3
|
|
2
|
-
"""U4 tests — SessionPool, root warmup, browser→curl cookie bridge.
|
|
3
|
-
|
|
4
|
-
Offline unit tests + a couple of benign online checks (example.com). Run:
|
|
5
|
-
python3 engine/tests/test_u4.py
|
|
6
|
-
"""
|
|
7
|
-
from __future__ import annotations
|
|
8
|
-
|
|
9
|
-
import os
|
|
10
|
-
import sys
|
|
11
|
-
|
|
12
|
-
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
13
|
-
ROOT = os.path.abspath(os.path.join(HERE, "..", ""))
|
|
14
|
-
sys.path.insert(0, os.path.abspath(os.path.join(HERE, "..", "..")))
|
|
15
|
-
|
|
16
|
-
from engine.transport import SessionPool, _host_of, _root_of # noqa: E402
|
|
17
|
-
from engine.executor import _parse_envelope # noqa: E402
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
def t_host_and_root_helpers():
|
|
21
|
-
assert _host_of("https://www.x.com/a/b?q=1") == "www.x.com"
|
|
22
|
-
assert _root_of("https://www.x.com/a/b?q=1") == "https://www.x.com/"
|
|
23
|
-
print(" ✓ host/root helpers")
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
def t_session_reuse_same_key():
|
|
27
|
-
p = SessionPool()
|
|
28
|
-
e1 = p.get("www.x.com", "safari")
|
|
29
|
-
e2 = p.get("www.x.com", "safari")
|
|
30
|
-
e3 = p.get("www.x.com", "chrome")
|
|
31
|
-
if e1 is None:
|
|
32
|
-
print(" ⚠ curl_cffi unavailable — skipped reuse check")
|
|
33
|
-
return
|
|
34
|
-
assert e1 is e2, "same (host,impersonate) must reuse entry"
|
|
35
|
-
assert e1 is not e3, "different impersonate must be separate session"
|
|
36
|
-
assert p.stats()["sessions"] == 2, p.stats()
|
|
37
|
-
print(f" ✓ session reuse (same key→same, diff impersonate→new): {p.stats()}")
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
def t_inject_cookies_then_present():
|
|
41
|
-
p = SessionPool()
|
|
42
|
-
ok = p.inject_cookies("www.x.com", "chrome",
|
|
43
|
-
[{"name": "cf_clearance", "value": "abc", "domain": "www.x.com"}],
|
|
44
|
-
user_agent="UA/1.0")
|
|
45
|
-
ent = p.get("www.x.com", "chrome")
|
|
46
|
-
if ent is None:
|
|
47
|
-
print(" ⚠ curl_cffi unavailable — skipped cookie inject check")
|
|
48
|
-
return
|
|
49
|
-
assert ok, "inject should report success"
|
|
50
|
-
assert ent.injected_ua == "UA/1.0"
|
|
51
|
-
names = {c.name for c in ent.session.cookies.jar}
|
|
52
|
-
assert "cf_clearance" in names, names
|
|
53
|
-
print(f" ✓ injected cookies present on session: {sorted(names)}")
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
def t_parse_envelope_json():
|
|
57
|
-
env = '{"html":"<h1>hi</h1>","finalUrl":"https://x/p","status":200,' \
|
|
58
|
-
'"cookies":[{"name":"a","value":"b"}],"userAgent":"UA"}'
|
|
59
|
-
html, final, status, cookies, ua, automation = _parse_envelope(env, "https://x/q")
|
|
60
|
-
assert html == "<h1>hi</h1>" and final == "https://x/p" and status == 200
|
|
61
|
-
assert cookies and cookies[0]["name"] == "a" and ua == "UA"
|
|
62
|
-
print(" ✓ envelope JSON parsed")
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
def t_parse_envelope_raw_html_fallback():
|
|
66
|
-
html, final, status, cookies, ua, automation = _parse_envelope("<html>raw</html>", "https://x/q")
|
|
67
|
-
assert html == "<html>raw</html>" and final == "https://x/q" and status == 200
|
|
68
|
-
assert cookies == [] and ua is None
|
|
69
|
-
print(" ✓ raw-HTML fallback (non-JSON stdout)")
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
def t_warmup_once_guard_online():
|
|
73
|
-
p = SessionPool()
|
|
74
|
-
first = p.warmup("example.com", "safari", "https://example.com/", timeout=15)
|
|
75
|
-
second = p.warmup("example.com", "safari", "https://example.com/", timeout=15)
|
|
76
|
-
ent = p.get("example.com", "safari")
|
|
77
|
-
if ent is None:
|
|
78
|
-
print(" ⚠ curl_cffi unavailable — skipped warmup check")
|
|
79
|
-
return
|
|
80
|
-
# first may be True (network) or False (offline); second must be False (guard).
|
|
81
|
-
assert second is False, "warmup must be idempotent"
|
|
82
|
-
assert ent.warmed is True
|
|
83
|
-
print(f" ✓ warmup once-guard (first={first}, second={second})")
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
def t_fetch_many_reuses_pool_online():
|
|
87
|
-
from engine import transport
|
|
88
|
-
from engine.fetch_chain import fetch_many
|
|
89
|
-
transport.POOL.reset()
|
|
90
|
-
results = fetch_many(
|
|
91
|
-
["https://example.com/", "https://example.com/index.html"],
|
|
92
|
-
success_selectors=["h1", "p"], timeout=15, max_attempts=2, enable_playwright=False,
|
|
93
|
-
)
|
|
94
|
-
st = transport.POOL.stats()
|
|
95
|
-
assert len(results) == 2
|
|
96
|
-
# Same host → should not spawn a separate session per URL per identity.
|
|
97
|
-
assert st["sessions"] <= 2, st
|
|
98
|
-
oks = sum(1 for r in results if r.ok)
|
|
99
|
-
print(f" ✓ fetch_many reused pool: stats={st}, ok={oks}/2")
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
ALL = [
|
|
103
|
-
("host_and_root_helpers", t_host_and_root_helpers),
|
|
104
|
-
("session_reuse_same_key", t_session_reuse_same_key),
|
|
105
|
-
("inject_cookies_then_present", t_inject_cookies_then_present),
|
|
106
|
-
("parse_envelope_json", t_parse_envelope_json),
|
|
107
|
-
("parse_envelope_raw_html_fallback", t_parse_envelope_raw_html_fallback),
|
|
108
|
-
("warmup_once_guard_online", t_warmup_once_guard_online),
|
|
109
|
-
("fetch_many_reuses_pool_online", t_fetch_many_reuses_pool_online),
|
|
110
|
-
]
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
def main() -> int:
|
|
114
|
-
p = f = 0
|
|
115
|
-
for name, fn in ALL:
|
|
116
|
-
try:
|
|
117
|
-
print(f"[{name}]")
|
|
118
|
-
fn()
|
|
119
|
-
p += 1
|
|
120
|
-
except AssertionError as e:
|
|
121
|
-
f += 1
|
|
122
|
-
print(f" ✗ FAIL: {e}")
|
|
123
|
-
except Exception as e:
|
|
124
|
-
f += 1
|
|
125
|
-
print(f" ✗ ERROR: {type(e).__name__}: {e}")
|
|
126
|
-
print(f"\n{p} passed, {f} failed")
|
|
127
|
-
return 0 if f == 0 else 1
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
if __name__ == "__main__":
|
|
131
|
-
sys.exit(main())
|