captchakraken 2.2.0 → 2.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +105 -0
- package/README.md +3 -1
- package/dist/errors.d.ts +45 -0
- package/dist/errors.js +74 -0
- package/dist/index.d.ts +23 -0
- package/dist/index.js +24 -1
- package/dist/playwright-types.d.ts +2 -0
- package/dist/solver.d.ts +67 -0
- package/dist/solver.js +389 -15
- package/dist/types.d.ts +103 -0
- package/package.json +4 -3
- package/python/LICENSE +105 -0
- package/python/README.md +3 -1
- package/python/pyproject.toml +6 -2
- package/python/src/captchakraken/__init__.py +13 -1
- package/python/src/captchakraken/cli.py +75 -0
- package/python/src/captchakraken/config.py +156 -6
- package/python/src/captchakraken/errors.py +195 -0
- package/python/src/captchakraken/page_solver.py +1579 -0
- package/python/src/captchakraken/pinned_model.json +28 -0
- package/python/src/captchakraken/planner.py +324 -32
- package/python/src/captchakraken/server_manager.py +3 -2
- package/python/src/captchakraken/trajectory.py +172 -0
- package/python/src/captchakraken/updater.py +168 -0
- package/scripts/setup-python.js +16 -2
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
{
|
|
2
|
+
"_comment": [
|
|
3
|
+
"The model this release of CaptchaKraken is PINNED to, plus the prompts it",
|
|
4
|
+
"was trained to answer. CI reads this file; see tests/test_pinned_model.py.",
|
|
5
|
+
"",
|
|
6
|
+
"Why a file and not just env defaults: the adapter and the serving prompts",
|
|
7
|
+
"are a matched pair. On 2026-07-18 the LoRA was retrained on the content",
|
|
8
|
+
"schema (action: drag / drags[] / lowercase) while planner.py still asked",
|
|
9
|
+
"for the legacy output/PascalCase schema. The model answered in a hybrid,",
|
|
10
|
+
"the parser dropped it, and EVERY drag puzzle failed as 'unsupported' with",
|
|
11
|
+
"no test going red. Recording the prompt hashes here means any edit to a",
|
|
12
|
+
"serving prompt fails CI until someone consciously re-pins — which forces",
|
|
13
|
+
"the question 'does the pinned adapter still expect this prompt?'.",
|
|
14
|
+
"",
|
|
15
|
+
"To ship a new adapter: update lora_adapter/lora_revision, re-run the Tier 2",
|
|
16
|
+
"static-image gate against it, and update prompt_sha256 if the prompts moved."
|
|
17
|
+
],
|
|
18
|
+
|
|
19
|
+
"base_model": "RedHatAI/Qwen3.5-9B-FP8-dynamic",
|
|
20
|
+
"lora_adapter": "CaptchaKraken/CaptchaKraken_v1.1",
|
|
21
|
+
"lora_revision": "main",
|
|
22
|
+
"lora_name": "captcha",
|
|
23
|
+
|
|
24
|
+
"prompt_sha256": {
|
|
25
|
+
"PIXEL_ACTION_PROMPT": "15d4833135330aa57ec64eb9132d15739cbf4a10bcda2e6d85d2d0ad92e0216a",
|
|
26
|
+
"SELECT_GRID_PROMPT": "f014dbc1cb8c773cc9501ff97aea92ff0281e5687b3d637d1c626ded9212fd3c"
|
|
27
|
+
}
|
|
28
|
+
}
|
|
@@ -10,18 +10,155 @@ planner with detect/segment/drag-refine; that code is preserved on the
|
|
|
10
10
|
import base64
|
|
11
11
|
import json
|
|
12
12
|
import os
|
|
13
|
+
import re
|
|
13
14
|
import sys
|
|
14
15
|
from mimetypes import guess_type
|
|
15
16
|
from typing import Any, Dict, List, Optional
|
|
16
17
|
|
|
17
18
|
import requests
|
|
18
19
|
|
|
19
|
-
from . import config
|
|
20
|
+
from . import config, errors
|
|
20
21
|
from .server_manager import ensure_server
|
|
21
22
|
from .timing import timed
|
|
22
23
|
|
|
23
24
|
DEBUG = os.getenv("CAPTCHA_DEBUG", "0") == "1"
|
|
24
25
|
|
|
26
|
+
# Header the fleet's haproxy front (on the reverse-proxy EC2) routes on. When a
|
|
27
|
+
# caller sets CAPTCHA_REQUEST_PRIORITY to a positive int, every request carries
|
|
28
|
+
# `X-JH-Priority: <n>`; haproxy sends anything above its threshold (5) straight
|
|
29
|
+
# to the backup GPUs so it never competes with production traffic on the main
|
|
30
|
+
# 5090. The tier-2 CI gate sets this — its traffic is throwaway and must stay
|
|
31
|
+
# off the primary.
|
|
32
|
+
#
|
|
33
|
+
# Deliberately a HEADER, not vLLM's request-body `priority` field: that field is
|
|
34
|
+
# lower-is-higher and already means captcha=0 / apply=100 / label=200 on the
|
|
35
|
+
# priority-scheduled primary, so a value like 10 would MISORDER there (it would
|
|
36
|
+
# outrank apply/label). Routing is a fleet concern, kept out of the model's
|
|
37
|
+
# scheduler.
|
|
38
|
+
_PRIORITY_HEADER = "X-JH-Priority"
|
|
39
|
+
_PRIORITY_ENV = "CAPTCHA_REQUEST_PRIORITY"
|
|
40
|
+
|
|
41
|
+
# Hosted-API metadata. Both are OPTIONAL and absent unless deliberately set, so
|
|
42
|
+
# self-hosted users never send them and are unaffected.
|
|
43
|
+
#
|
|
44
|
+
# X-CK-Client — which integration issued this solve (e.g. "camoufox/0.4.11").
|
|
45
|
+
# camoufox sets CAPTCHA_KRAKEN_CLIENT in the env it spawns the solver with, so
|
|
46
|
+
# the gateway can account camoufox-attributed usage separately. Attribution
|
|
47
|
+
# ONLY: it is caller-supplied and therefore never priced on (the gateway
|
|
48
|
+
# derives billable puzzle class from the request body instead).
|
|
49
|
+
# X-CK-Session — groups the 1..N inference rounds of ONE captcha into a single
|
|
50
|
+
# billable attempt. The JS driver mints a UUID per solve() and reuses it for
|
|
51
|
+
# every CLI invocation in that solve, which is what lets the gateway cap an
|
|
52
|
+
# attempt's billable rounds instead of charging per round without limit.
|
|
53
|
+
_CLIENT_HEADER = "X-CK-Client"
|
|
54
|
+
_CLIENT_ENV = "CAPTCHA_KRAKEN_CLIENT"
|
|
55
|
+
_SESSION_HEADER = "X-CK-Session"
|
|
56
|
+
_SESSION_ENV = "CAPTCHA_KRAKEN_SESSION"
|
|
57
|
+
|
|
58
|
+
# Arbitrary extra headers, as "Name: value" pairs separated by newlines or
|
|
59
|
+
# commas. Empty and absent by default.
|
|
60
|
+
#
|
|
61
|
+
# WHY THIS EXISTS. Some endpoints sit behind a gate that is not the API key —
|
|
62
|
+
# our own dev gateway (api.dev.captchakraken.com) wants X-CK-Dev-Auth, and a
|
|
63
|
+
# corporate egress proxy may want its own token. Without this the published
|
|
64
|
+
# client simply cannot reach such an endpoint, which meant the dev environment
|
|
65
|
+
# could not exercise the real client path at all: the thing dev exists to
|
|
66
|
+
# rehearse was the one thing it could not do.
|
|
67
|
+
#
|
|
68
|
+
# IT CANNOT OVERWRITE Authorization, Content-Type, OR THE X-CK-* HEADERS. Those
|
|
69
|
+
# carry the credential and the billing attribution. Allowing an env var to
|
|
70
|
+
# rewrite Authorization would turn a stray export into "your key is not the key
|
|
71
|
+
# being charged", and allowing it to rewrite X-CK-Session would let a caller
|
|
72
|
+
# escape the per-attempt billing cap by pinning one session id forever.
|
|
73
|
+
_EXTRA_HEADERS_ENV = "CAPTCHA_KRAKEN_EXTRA_HEADERS"
|
|
74
|
+
_PROTECTED_HEADERS = frozenset(
|
|
75
|
+
{"authorization", "content-type", _CLIENT_HEADER.lower(), _SESSION_HEADER.lower()}
|
|
76
|
+
)
|
|
77
|
+
|
|
78
|
+
# These values reach the wire verbatim from the environment, so they are
|
|
79
|
+
# sanitized rather than trusted: a CR/LF would otherwise splice arbitrary
|
|
80
|
+
# headers into the upstream request.
|
|
81
|
+
_HEADER_VALUE_MAX = 128
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
# RFC 7230 token characters. Anything else is not a header name.
|
|
85
|
+
_VALID_HEADER_NAME = re.compile(r"^[!#$%&'*+\-.^_`|~0-9A-Za-z]+$")
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def _clean_header_value(raw: str) -> str:
|
|
89
|
+
"""Printable-ASCII, length-capped header value ("" when nothing survives)."""
|
|
90
|
+
return "".join(c for c in raw.strip() if 0x20 <= ord(c) < 0x7F)[:_HEADER_VALUE_MAX]
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def _extra_headers(raw: str) -> Dict[str, str]:
|
|
94
|
+
"""Parse CAPTCHA_KRAKEN_EXTRA_HEADERS into a header dict.
|
|
95
|
+
|
|
96
|
+
Accepts "Name: value" pairs separated by newlines or commas:
|
|
97
|
+
|
|
98
|
+
CAPTCHA_KRAKEN_EXTRA_HEADERS='X-CK-Dev-Auth: hunter2'
|
|
99
|
+
CAPTCHA_KRAKEN_EXTRA_HEADERS=$'X-One: a\nX-Two: b'
|
|
100
|
+
|
|
101
|
+
Malformed entries are DROPPED, never guessed at. A pair with no colon is
|
|
102
|
+
not a header, and inventing one from it would put an attacker-shaped string
|
|
103
|
+
on the wire. Names are restricted to the RFC 7230 token characters for the
|
|
104
|
+
same reason the values are sanitized: a CR/LF in either would splice extra
|
|
105
|
+
headers into the request.
|
|
106
|
+
|
|
107
|
+
Protected headers are refused, so this cannot be used to replace the
|
|
108
|
+
credential or the billing attribution.
|
|
109
|
+
"""
|
|
110
|
+
out: Dict[str, str] = {}
|
|
111
|
+
if not raw.strip():
|
|
112
|
+
return out
|
|
113
|
+
|
|
114
|
+
for entry in raw.replace(",", "\n").splitlines():
|
|
115
|
+
entry = entry.strip()
|
|
116
|
+
if not entry or ":" not in entry:
|
|
117
|
+
continue
|
|
118
|
+
name, _, value = entry.partition(":")
|
|
119
|
+
name = name.strip()
|
|
120
|
+
value = _clean_header_value(value)
|
|
121
|
+
if not name or not value:
|
|
122
|
+
continue
|
|
123
|
+
if not _VALID_HEADER_NAME.match(name):
|
|
124
|
+
continue
|
|
125
|
+
if name.lower() in _PROTECTED_HEADERS:
|
|
126
|
+
continue
|
|
127
|
+
out[name] = value
|
|
128
|
+
return out
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def routing_headers(env=None) -> Dict[str, str]:
|
|
132
|
+
"""Fleet-routing + hosted-API headers derived from the environment.
|
|
133
|
+
|
|
134
|
+
Empty by default. Split out and env-injectable so it can be unit-tested
|
|
135
|
+
without a live server. A missing or non-integer CAPTCHA_REQUEST_PRIORITY
|
|
136
|
+
yields no header at all — an unset/garbage value must never silently tag
|
|
137
|
+
traffic for the backups.
|
|
138
|
+
|
|
139
|
+
Each header is derived independently: a malformed priority must not suppress
|
|
140
|
+
the attribution headers, or one typo would silently make a camoufox solve
|
|
141
|
+
look like direct traffic and understate the partner's revenue share.
|
|
142
|
+
"""
|
|
143
|
+
env = os.environ if env is None else env
|
|
144
|
+
headers: Dict[str, str] = {}
|
|
145
|
+
|
|
146
|
+
raw = (env.get(_PRIORITY_ENV) or "").strip()
|
|
147
|
+
if raw:
|
|
148
|
+
try:
|
|
149
|
+
headers[_PRIORITY_HEADER] = str(int(raw))
|
|
150
|
+
except ValueError:
|
|
151
|
+
pass
|
|
152
|
+
|
|
153
|
+
for header, var in ((_CLIENT_HEADER, _CLIENT_ENV), (_SESSION_HEADER, _SESSION_ENV)):
|
|
154
|
+
value = _clean_header_value(env.get(var) or "")
|
|
155
|
+
if value:
|
|
156
|
+
headers[header] = value
|
|
157
|
+
|
|
158
|
+
headers.update(_extra_headers(env.get(_EXTRA_HEADERS_ENV) or ""))
|
|
159
|
+
|
|
160
|
+
return headers
|
|
161
|
+
|
|
25
162
|
|
|
26
163
|
# Matches the training-distribution grid prompts in
|
|
27
164
|
# cleanSamples/test/test_solutions.json -> grade.synthesize_instruction.
|
|
@@ -45,21 +182,21 @@ PIXEL_ACTION_PROMPT = (
|
|
|
45
182
|
"Your task is to solve the captcha. Read the instruction at the top of the image carefully.\n\n"
|
|
46
183
|
"Look at the puzzle and decide what action solves it. All coordinates you return must be on a "
|
|
47
184
|
"normalized 0–1000 image scale (top-left = (0, 0), bottom-right = (1000, 1000)).\n\n"
|
|
185
|
+
"Name WHAT each object is with a short 1–2 word label, then give its position. "
|
|
48
186
|
"Choose ONE response:\n\n"
|
|
49
187
|
"FOR CLICK PUZZLES:\n"
|
|
50
|
-
"
|
|
51
|
-
" → \"action\":
|
|
188
|
+
" Label each thing you click and give its point — subjects[i] names points[i]:\n"
|
|
189
|
+
" → \"action\": \"click\", \"subjects\": [\"<label>\", ...], "
|
|
190
|
+
"\"points\": [[x1, y1], [x2, y2], ...]\n\n"
|
|
52
191
|
"FOR DRAG PUZZLES:\n"
|
|
53
|
-
" Drag ONE item at a time.
|
|
54
|
-
"
|
|
55
|
-
"
|
|
56
|
-
"
|
|
57
|
-
"\"SourceDescription\": \"...\", \"SourcePosition\": { \"x\": 1-1000, \"y\": 1-1000 }, "
|
|
58
|
-
"\"DestinationDescription\": \"...\", \"EstimatedPosition\": { \"x\": 1-1000, \"y\": 1-1000 } }]\n\n"
|
|
192
|
+
" Drag ONE item at a time. Label the source (the piece you pick up) and the destination "
|
|
193
|
+
"(where it belongs), each with a short 1–2 word label, and give both points:\n"
|
|
194
|
+
" → \"action\": \"drag\", \"drags\": [{ \"source\": \"<label>\", \"from\": [x, y], "
|
|
195
|
+
"\"destination\": \"<label>\", \"to\": [x, y] }, ...]\n\n"
|
|
59
196
|
"Respond ONLY with JSON:\n"
|
|
60
197
|
"{\n"
|
|
61
|
-
" \"action\":
|
|
62
|
-
" // OR \"
|
|
198
|
+
" \"action\": \"click\", \"subjects\": [ ... ], \"points\": [ ... ]\n"
|
|
199
|
+
" // OR \"action\": \"drag\", \"drags\": [ ... ]\n"
|
|
63
200
|
"}"
|
|
64
201
|
)
|
|
65
202
|
|
|
@@ -128,6 +265,10 @@ class ActionPlanner:
|
|
|
128
265
|
headers = {
|
|
129
266
|
"Authorization": f"Bearer {self.api_key}",
|
|
130
267
|
"Content-Type": "application/json",
|
|
268
|
+
# Fleet routing: absent unless CAPTCHA_REQUEST_PRIORITY is set (see
|
|
269
|
+
# routing_headers). Low-priority batch traffic is steered to the
|
|
270
|
+
# backup GPUs by the haproxy front on this header.
|
|
271
|
+
**routing_headers(),
|
|
131
272
|
}
|
|
132
273
|
|
|
133
274
|
# Hands-off server: before the very first request, make sure something
|
|
@@ -147,21 +288,18 @@ class ActionPlanner:
|
|
|
147
288
|
with timed("planner.chat"):
|
|
148
289
|
resp = requests.post(url, headers=headers, json=payload, timeout=120)
|
|
149
290
|
|
|
150
|
-
# Surface auth / server errors
|
|
151
|
-
# resp.json() blow up with a cryptic "Expecting
|
|
152
|
-
# on a non-JSON response
|
|
153
|
-
#
|
|
154
|
-
#
|
|
291
|
+
# Surface auth / billing / server errors as something the reader can act
|
|
292
|
+
# on, instead of letting resp.json() blow up with a cryptic "Expecting
|
|
293
|
+
# value: line 1 column 1" on a non-JSON response.
|
|
294
|
+
#
|
|
295
|
+
# `errors.from_response` decides which of the two worlds we are in by
|
|
296
|
+
# looking for the gateway's `error.code`: present means the hosted API
|
|
297
|
+
# refused us and there is a real explanation to give (out of credits,
|
|
298
|
+
# rate limited, attempt abandoned); absent means a local vLLM or a proxy,
|
|
299
|
+
# and the old message — including the bearer-token hint on 401/403 — is
|
|
300
|
+
# reproduced unchanged.
|
|
155
301
|
if not resp.ok:
|
|
156
|
-
|
|
157
|
-
hint = ""
|
|
158
|
-
if resp.status_code in (401, 403):
|
|
159
|
-
hint = (
|
|
160
|
-
" — check CAPTCHA_KRAKEN_API_KEY is set and forwarded to the CLI"
|
|
161
|
-
)
|
|
162
|
-
raise RuntimeError(
|
|
163
|
-
f"vLLM {resp.status_code} {resp.reason} at {url}{hint}. Body: {body}"
|
|
164
|
-
)
|
|
302
|
+
raise errors.from_response(resp, url)
|
|
165
303
|
|
|
166
304
|
try:
|
|
167
305
|
data = resp.json()
|
|
@@ -198,8 +336,13 @@ class ActionPlanner:
|
|
|
198
336
|
else:
|
|
199
337
|
return None
|
|
200
338
|
|
|
339
|
+
# strict=False tolerates unescaped control chars INSIDE strings — the
|
|
340
|
+
# model routinely emits coordinates as pretty-printed strings with a
|
|
341
|
+
# literal newline, e.g. "click": ["277,\n 728", ...], which strict JSON
|
|
342
|
+
# rejects. Without this the whole response fails to parse and a solvable
|
|
343
|
+
# click puzzle is dropped as "unsupported".
|
|
201
344
|
try:
|
|
202
|
-
return json.loads(text[start:end])
|
|
345
|
+
return json.loads(text[start:end], strict=False)
|
|
203
346
|
except json.JSONDecodeError:
|
|
204
347
|
# The model sometimes truncates (hit max_tokens) or over-nests its
|
|
205
348
|
# pretty-printed JSON, leaving brackets unclosed. Repair by balancing
|
|
@@ -207,7 +350,7 @@ class ActionPlanner:
|
|
|
207
350
|
repaired = ActionPlanner._balance_json(text[start:])
|
|
208
351
|
if repaired is not None:
|
|
209
352
|
try:
|
|
210
|
-
return json.loads(repaired)
|
|
353
|
+
return json.loads(repaired, strict=False)
|
|
211
354
|
except json.JSONDecodeError:
|
|
212
355
|
return None
|
|
213
356
|
return None
|
|
@@ -284,12 +427,31 @@ class ActionPlanner:
|
|
|
284
427
|
"including any matches you may have overlooked."
|
|
285
428
|
)
|
|
286
429
|
raw = self._chat_with_image(prompt, image_path, max_tokens=128)
|
|
287
|
-
|
|
430
|
+
out = self._normalize_grid(self._parse_json(raw), total)
|
|
431
|
+
self._log(f"grid selection -> {out}")
|
|
432
|
+
return out
|
|
433
|
+
|
|
434
|
+
@staticmethod
|
|
435
|
+
def _normalize_grid(data: Any, total: int) -> List[int]:
|
|
436
|
+
"""Map the model's grid JSON to a list of 1-indexed cell numbers.
|
|
437
|
+
|
|
438
|
+
Accepts the trained shape (a bare JSON array) plus the wrapped forms the
|
|
439
|
+
model drifts into: {"target_ids": [...]} and {"action": {"target_ids":
|
|
440
|
+
[...]}}. Out-of-range and non-numeric entries are dropped rather than
|
|
441
|
+
raising — a single junk element must not cost the whole selection.
|
|
288
442
|
|
|
443
|
+
Split out of get_grid_selection so the parse is testable without a
|
|
444
|
+
model, mirroring _normalize_pixel.
|
|
445
|
+
"""
|
|
289
446
|
if isinstance(data, list):
|
|
290
447
|
ids = data
|
|
291
448
|
elif isinstance(data, dict):
|
|
292
|
-
|
|
449
|
+
# `action` is a dict on the grid path but a bare string ("drag") on
|
|
450
|
+
# the pixel path — guard so a mis-routed response returns [] rather
|
|
451
|
+
# than raising AttributeError.
|
|
452
|
+
nested = data.get("action")
|
|
453
|
+
nested_ids = nested.get("target_ids") if isinstance(nested, dict) else None
|
|
454
|
+
ids = data.get("target_ids") or nested_ids or []
|
|
293
455
|
else:
|
|
294
456
|
ids = []
|
|
295
457
|
|
|
@@ -297,11 +459,10 @@ class ActionPlanner:
|
|
|
297
459
|
for v in ids:
|
|
298
460
|
try:
|
|
299
461
|
iv = int(v)
|
|
300
|
-
if 1 <= iv <= total:
|
|
301
|
-
out.append(iv)
|
|
302
462
|
except (TypeError, ValueError):
|
|
303
463
|
continue
|
|
304
|
-
|
|
464
|
+
if 1 <= iv <= total:
|
|
465
|
+
out.append(iv)
|
|
305
466
|
return out
|
|
306
467
|
|
|
307
468
|
def get_pixel_actions(self, image_path: str) -> List[Dict[str, Any]]:
|
|
@@ -347,10 +508,73 @@ class ActionPlanner:
|
|
|
347
508
|
fx, fy = min(max(fx, 0.0), 1.0), min(max(fy, 0.0), 1.0)
|
|
348
509
|
return (fx, fy)
|
|
349
510
|
|
|
511
|
+
def flat_numbers(v: Any) -> List[float]:
|
|
512
|
+
"""Every number anywhere inside v, in order. Handles nested lists,
|
|
513
|
+
{x,y} dicts, bare numbers, and numeric strings like "277, 728" (the
|
|
514
|
+
model sometimes emits coordinates as strings, occasionally even split
|
|
515
|
+
across separate array elements)."""
|
|
516
|
+
nums: List[float] = []
|
|
517
|
+
if isinstance(v, bool):
|
|
518
|
+
return nums
|
|
519
|
+
if isinstance(v, (list, tuple)):
|
|
520
|
+
for e in v:
|
|
521
|
+
nums.extend(flat_numbers(e))
|
|
522
|
+
elif isinstance(v, dict):
|
|
523
|
+
for k in ("x", "y", "X", "Y"):
|
|
524
|
+
if k in v:
|
|
525
|
+
nums.extend(flat_numbers(v[k]))
|
|
526
|
+
elif isinstance(v, (int, float)):
|
|
527
|
+
nums.append(float(v))
|
|
528
|
+
elif isinstance(v, str):
|
|
529
|
+
nums.extend(float(m) for m in re.findall(r"-?\d+(?:\.\d+)?", v))
|
|
530
|
+
return nums
|
|
531
|
+
|
|
532
|
+
def coordish(v: Any) -> bool:
|
|
533
|
+
"""True when v is a non-empty list of COORDINATES (numbers, [x,y]
|
|
534
|
+
pairs, {x,y} dicts, or numeric strings) — not text labels like
|
|
535
|
+
"dog". Used to decide whether a "click"/"coordinates" array carries
|
|
536
|
+
points we should salvage."""
|
|
537
|
+
if not isinstance(v, (list, tuple)) or not v:
|
|
538
|
+
return False
|
|
539
|
+
for e in v:
|
|
540
|
+
if isinstance(e, bool):
|
|
541
|
+
return False
|
|
542
|
+
if isinstance(e, (int, float, list, tuple, dict)):
|
|
543
|
+
continue
|
|
544
|
+
if isinstance(e, str) and re.search(r"\d", e):
|
|
545
|
+
continue
|
|
546
|
+
return False
|
|
547
|
+
return True
|
|
548
|
+
|
|
350
549
|
out: List[Dict[str, Any]] = []
|
|
351
550
|
if not isinstance(data, dict):
|
|
352
551
|
return out
|
|
353
552
|
|
|
553
|
+
# ---- drag (CANONICAL — the schema PIXEL_ACTION_PROMPT asks for and the
|
|
554
|
+
# LoRA is trained on, see instructions.py::ACTION_INSTRUCTION):
|
|
555
|
+
# {"action":"drag","drags":[{"source","from":[x,y],"destination","to":[x,y]}]}
|
|
556
|
+
# Note "action" here is the STRING "drag", not a dict, so the click path
|
|
557
|
+
# below never sees it. The legacy/simulate_drag branches that follow stay
|
|
558
|
+
# as fallbacks for older adapters.
|
|
559
|
+
content_drags = data.get("drags")
|
|
560
|
+
if content_drags is None and isinstance(data.get("action"), dict):
|
|
561
|
+
content_drags = data["action"].get("drags")
|
|
562
|
+
if isinstance(content_drags, dict):
|
|
563
|
+
content_drags = [content_drags]
|
|
564
|
+
if isinstance(content_drags, list) and content_drags:
|
|
565
|
+
for d in content_drags:
|
|
566
|
+
if not isinstance(d, dict):
|
|
567
|
+
continue
|
|
568
|
+
snums = flat_numbers(d.get("from"))
|
|
569
|
+
dnums = flat_numbers(d.get("to"))
|
|
570
|
+
if len(snums) >= 2 and len(dnums) >= 2:
|
|
571
|
+
src = norm_xy(snums[0], snums[1])
|
|
572
|
+
dst = norm_xy(dnums[0], dnums[1])
|
|
573
|
+
if src and dst:
|
|
574
|
+
out.append({"kind": "drag", "src": src, "dst": dst})
|
|
575
|
+
if out:
|
|
576
|
+
return out
|
|
577
|
+
|
|
354
578
|
# ---- drag: {"output": [ {simulate_drag ...}, ... ]} ----
|
|
355
579
|
drags = data.get("output")
|
|
356
580
|
if isinstance(drags, list) and drags:
|
|
@@ -366,6 +590,54 @@ class ActionPlanner:
|
|
|
366
590
|
if out:
|
|
367
591
|
return out
|
|
368
592
|
|
|
593
|
+
# ---- drag: {"action": {"simulate_drag": [{source_position,
|
|
594
|
+
# destination_position}]}} ----
|
|
595
|
+
# The full-puzzle LoRA actually emits drags in this snake_case shape
|
|
596
|
+
# (not the prompt's {"output":[{"Action":"simulate_drag",...}]}), and
|
|
597
|
+
# packs each coordinate as {"x": [x, y]} — flat_numbers() pulls the pair
|
|
598
|
+
# out of that. Without this branch a correctly-solved drag (e.g. hCaptcha
|
|
599
|
+
# "drag ONE character to the matching character") is dropped as
|
|
600
|
+
# "unsupported".
|
|
601
|
+
sd = data.get("simulate_drag")
|
|
602
|
+
if sd is None and isinstance(action := data.get("action"), dict):
|
|
603
|
+
sd = action.get("simulate_drag")
|
|
604
|
+
# The model emits simulate_drag as EITHER a list of drags OR a single
|
|
605
|
+
# drag object; normalize the single-object form to a one-element list so
|
|
606
|
+
# both parse (the object form was silently dropped as "unsupported").
|
|
607
|
+
if isinstance(sd, dict):
|
|
608
|
+
sd = [sd]
|
|
609
|
+
if isinstance(sd, list) and sd:
|
|
610
|
+
def _drag_coords(obj: Any, *roles: str) -> List[float]:
|
|
611
|
+
# Pull the source/destination coordinate pair regardless of key
|
|
612
|
+
# casing or separators — the LoRA freely varies between
|
|
613
|
+
# source_position / sourcePosition / SourcePosition and
|
|
614
|
+
# destination_position / destinationPosition / EstimatedPosition.
|
|
615
|
+
# The coord value may be {"x": [x, y]}, [x, y], {x, y}, or a
|
|
616
|
+
# string; flat_numbers() handles all of those.
|
|
617
|
+
if not isinstance(obj, dict):
|
|
618
|
+
return []
|
|
619
|
+
for k, v in obj.items():
|
|
620
|
+
kn = str(k).lower().replace("_", "").replace("-", "")
|
|
621
|
+
if "pos" not in kn:
|
|
622
|
+
continue
|
|
623
|
+
if any(r in kn for r in roles):
|
|
624
|
+
nums = flat_numbers(v)
|
|
625
|
+
if len(nums) >= 2:
|
|
626
|
+
return nums
|
|
627
|
+
return []
|
|
628
|
+
for d in sd:
|
|
629
|
+
if not isinstance(d, dict):
|
|
630
|
+
continue
|
|
631
|
+
snums = _drag_coords(d, "source", "src")
|
|
632
|
+
dnums = _drag_coords(d, "destination", "dest", "estimated", "target")
|
|
633
|
+
if len(snums) >= 2 and len(dnums) >= 2:
|
|
634
|
+
src = norm_xy(snums[0], snums[1])
|
|
635
|
+
dst = norm_xy(dnums[0], dnums[1])
|
|
636
|
+
if src and dst:
|
|
637
|
+
out.append({"kind": "drag", "src": src, "dst": dst})
|
|
638
|
+
if out:
|
|
639
|
+
return out
|
|
640
|
+
|
|
369
641
|
# ---- click: {"action": {"action":"click","points":[...]}} ----
|
|
370
642
|
action = data.get("action")
|
|
371
643
|
# The model sometimes over-wraps: {"action": {"action": {"action":
|
|
@@ -392,6 +664,26 @@ class ActionPlanner:
|
|
|
392
664
|
return [{"kind": "drag", "src": src, "dst": dst}]
|
|
393
665
|
if points is None:
|
|
394
666
|
points = data.get("points")
|
|
667
|
+
# Salvage: some responses carry the click coordinates under "click" (or
|
|
668
|
+
# "coordinates") instead of "points", as [x,y] pairs, {x,y} dicts, or
|
|
669
|
+
# "x, y" strings (sometimes split across elements), with no "points" key
|
|
670
|
+
# at all. Pull every number out of a coordinate-like value and pair them
|
|
671
|
+
# into points so a well-intentioned answer isn't dropped as "unsupported".
|
|
672
|
+
# Skipped when the value is text labels ("dog", "duck") rather than coords.
|
|
673
|
+
if not (isinstance(points, list) and points):
|
|
674
|
+
for container in (action if isinstance(action, dict) else None, data):
|
|
675
|
+
if not isinstance(container, dict):
|
|
676
|
+
continue
|
|
677
|
+
cand = container.get("click")
|
|
678
|
+
if cand is None:
|
|
679
|
+
cand = container.get("coordinates")
|
|
680
|
+
if coordish(cand):
|
|
681
|
+
nums = flat_numbers(cand)
|
|
682
|
+
if len(nums) >= 2:
|
|
683
|
+
points = [
|
|
684
|
+
[nums[i], nums[i + 1]] for i in range(0, len(nums) - 1, 2)
|
|
685
|
+
]
|
|
686
|
+
break
|
|
395
687
|
if isinstance(points, list) and points:
|
|
396
688
|
pts = []
|
|
397
689
|
for p in points:
|
|
@@ -29,8 +29,9 @@ import requests
|
|
|
29
29
|
|
|
30
30
|
from . import config
|
|
31
31
|
|
|
32
|
-
# One shared state dir for the pidfile, lockfile, and
|
|
33
|
-
|
|
32
|
+
# One shared state dir for the pidfile, lockfile, server log, and the hosted-API
|
|
33
|
+
# credentials file. Defined in `config` so the path has a single definition.
|
|
34
|
+
STATE_DIR = config.state_dir()
|
|
34
35
|
LOCK_FILE = STATE_DIR / "vllm.lock"
|
|
35
36
|
PID_FILE = STATE_DIR / "vllm.pid"
|
|
36
37
|
LOG_FILE = STATE_DIR / "vllm.log"
|
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Human-like mouse trajectories for the page driver.
|
|
3
|
+
|
|
4
|
+
MIRRORS, DOES NOT PORT, `cursory-ts`. The TypeScript driver calls
|
|
5
|
+
`generate_trajectory()` from cursory-ts, which selects and warps a real recorded
|
|
6
|
+
human trace out of a bundled ~MB corpus (`data/trajectories.json.gz`). That
|
|
7
|
+
corpus is the package's own asset, so the Python port synthesises comparable
|
|
8
|
+
motion instead of shipping a second copy of someone else's data.
|
|
9
|
+
|
|
10
|
+
The CONTRACT is identical, which is what matters for keeping the two drivers in
|
|
11
|
+
step: given a start point, an end point and a sampling frequency, return
|
|
12
|
+
`(points, timings)` where `timings[i]` is the cumulative milliseconds from the
|
|
13
|
+
start of the gesture at which `points[i]` should be delivered. The caller
|
|
14
|
+
(`page_solver`) sleeps against those cumulative timings exactly as the TS driver
|
|
15
|
+
does, so movement pacing is driver-independent.
|
|
16
|
+
|
|
17
|
+
What makes the output human rather than a lerp:
|
|
18
|
+
- a Bezier arc, not a straight line (hand motion bows)
|
|
19
|
+
- Fitts's-law duration: distance and precision set the time, not a constant
|
|
20
|
+
- an ease-in-out velocity profile, so the cursor accelerates then brakes
|
|
21
|
+
- sub-pixel jitter that scales with speed (fast motion is sloppier)
|
|
22
|
+
- a small overshoot-and-correct on longer moves, which is the single most
|
|
23
|
+
recognisable human tell and the one a linear interpolation never produces
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
from __future__ import annotations
|
|
27
|
+
|
|
28
|
+
import math
|
|
29
|
+
import random
|
|
30
|
+
from typing import List, Sequence, Tuple
|
|
31
|
+
|
|
32
|
+
Point = Tuple[float, float]
|
|
33
|
+
|
|
34
|
+
# Fitts's law: MT = a + b * log2(distance / width + 1). The constants are the
|
|
35
|
+
# usual empirical range for a mouse; `_WIDTH` stands in for target precision.
|
|
36
|
+
_FITTS_A_MS = 90.0
|
|
37
|
+
_FITTS_B_MS = 105.0
|
|
38
|
+
_WIDTH_PX = 28.0
|
|
39
|
+
|
|
40
|
+
# Below this, a move is a nudge inside one element: no overshoot, no arc.
|
|
41
|
+
_SHORT_MOVE_PX = 24.0
|
|
42
|
+
# Overshooting a 30px hop looks like a twitch, not a human.
|
|
43
|
+
_OVERSHOOT_MIN_DISTANCE_PX = 220.0
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _ease_in_out(t: float) -> float:
|
|
47
|
+
"""Cubic ease. Real pointer motion is roughly bell-shaped in velocity."""
|
|
48
|
+
if t < 0.5:
|
|
49
|
+
return 4.0 * t * t * t
|
|
50
|
+
return 1.0 - ((-2.0 * t + 2.0) ** 3) / 2.0
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def _cubic_bezier(p0: Point, p1: Point, p2: Point, p3: Point, t: float) -> Point:
|
|
54
|
+
u = 1.0 - t
|
|
55
|
+
x = u * u * u * p0[0] + 3 * u * u * t * p1[0] + 3 * u * t * t * p2[0] + t * t * t * p3[0]
|
|
56
|
+
y = u * u * u * p0[1] + 3 * u * u * t * p1[1] + 3 * u * t * t * p2[1] + t * t * t * p3[1]
|
|
57
|
+
return (x, y)
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def _control_points(start: Point, end: Point, distance: float) -> Tuple[Point, Point]:
|
|
61
|
+
"""
|
|
62
|
+
Two control points offset PERPENDICULAR to the travel direction, so the path
|
|
63
|
+
bows to one side. The sign is random per gesture: always bowing the same way
|
|
64
|
+
would itself be a fingerprint.
|
|
65
|
+
"""
|
|
66
|
+
dx, dy = end[0] - start[0], end[1] - start[1]
|
|
67
|
+
if distance == 0:
|
|
68
|
+
return start, end
|
|
69
|
+
# Unit normal to the direction of travel.
|
|
70
|
+
nx, ny = -dy / distance, dx / distance
|
|
71
|
+
# Bow proportional to distance, but capped — long drags don't arc absurdly.
|
|
72
|
+
bow = min(distance * random.uniform(0.06, 0.16), 140.0) * random.choice((-1.0, 1.0))
|
|
73
|
+
# Control points at roughly 1/3 and 2/3 along, each pushed off-axis.
|
|
74
|
+
c1 = (
|
|
75
|
+
start[0] + dx * random.uniform(0.20, 0.40) + nx * bow,
|
|
76
|
+
start[1] + dy * random.uniform(0.20, 0.40) + ny * bow,
|
|
77
|
+
)
|
|
78
|
+
c2 = (
|
|
79
|
+
start[0] + dx * random.uniform(0.60, 0.80) + nx * bow * random.uniform(0.4, 0.9),
|
|
80
|
+
start[1] + dy * random.uniform(0.60, 0.80) + ny * bow * random.uniform(0.4, 0.9),
|
|
81
|
+
)
|
|
82
|
+
return c1, c2
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def generate_trajectory(
|
|
86
|
+
target_start: Sequence[float],
|
|
87
|
+
target_end: Sequence[float],
|
|
88
|
+
frequency: int = 60,
|
|
89
|
+
frequency_randomizer: float = 0.12,
|
|
90
|
+
) -> Tuple[List[Point], List[float]]:
|
|
91
|
+
"""
|
|
92
|
+
Mirror of cursory-ts `generate_trajectory`.
|
|
93
|
+
|
|
94
|
+
Returns `(points, timings)`. `timings` is CUMULATIVE milliseconds from the
|
|
95
|
+
gesture start — not per-step deltas — because that is what the TS driver's
|
|
96
|
+
`tracePath` sleeps against, and the two must pace identically.
|
|
97
|
+
|
|
98
|
+
`frequency` is samples per second (the TS driver passes 60).
|
|
99
|
+
`frequency_randomizer` jitters each frame's timestamp so the sample interval
|
|
100
|
+
is not a metronome; a perfectly periodic mousemove stream is trivially
|
|
101
|
+
detectable.
|
|
102
|
+
"""
|
|
103
|
+
start: Point = (float(target_start[0]), float(target_start[1]))
|
|
104
|
+
end: Point = (float(target_end[0]), float(target_end[1]))
|
|
105
|
+
|
|
106
|
+
dx, dy = end[0] - start[0], end[1] - start[1]
|
|
107
|
+
distance = math.hypot(dx, dy)
|
|
108
|
+
|
|
109
|
+
if distance < 1e-6:
|
|
110
|
+
return [end], [0.0]
|
|
111
|
+
|
|
112
|
+
duration_ms = _FITTS_A_MS + _FITTS_B_MS * math.log2(distance / _WIDTH_PX + 1.0)
|
|
113
|
+
duration_ms *= random.uniform(0.85, 1.20)
|
|
114
|
+
|
|
115
|
+
steps = max(2, int(round(duration_ms / 1000.0 * max(1, frequency))))
|
|
116
|
+
|
|
117
|
+
# A short hop is a straight ease with jitter — bowing a 15px move looks worse
|
|
118
|
+
# than not bowing it.
|
|
119
|
+
if distance < _SHORT_MOVE_PX:
|
|
120
|
+
c1, c2 = start, end
|
|
121
|
+
else:
|
|
122
|
+
c1, c2 = _control_points(start, end, distance)
|
|
123
|
+
|
|
124
|
+
# Overshoot: on a long move the hand arrives past the target and corrects.
|
|
125
|
+
# Modelled as a longer primary gesture to a point beyond the target, then a
|
|
126
|
+
# short settle back — the settle is appended as extra samples below.
|
|
127
|
+
overshoot = distance >= _OVERSHOOT_MIN_DISTANCE_PX and random.random() < 0.55
|
|
128
|
+
aim: Point = end
|
|
129
|
+
if overshoot:
|
|
130
|
+
over = random.uniform(0.01, 0.035) * distance
|
|
131
|
+
aim = (end[0] + dx / distance * over, end[1] + dy / distance * over)
|
|
132
|
+
|
|
133
|
+
points: List[Point] = []
|
|
134
|
+
timings: List[float] = []
|
|
135
|
+
elapsed = 0.0
|
|
136
|
+
per_step = duration_ms / steps
|
|
137
|
+
|
|
138
|
+
for i in range(steps + 1):
|
|
139
|
+
t = _ease_in_out(i / steps)
|
|
140
|
+
px, py = _cubic_bezier(start, c1, c2, aim, t)
|
|
141
|
+
|
|
142
|
+
# Jitter scales with instantaneous speed: the faster the cursor is
|
|
143
|
+
# moving, the less precisely a human tracks the intended path.
|
|
144
|
+
speed = abs(_ease_in_out(min(1.0, (i + 1) / steps)) - _ease_in_out(i / steps))
|
|
145
|
+
jitter = min(1.6, 0.25 + speed * steps * 0.5)
|
|
146
|
+
px += random.gauss(0.0, jitter * 0.5)
|
|
147
|
+
py += random.gauss(0.0, jitter * 0.5)
|
|
148
|
+
|
|
149
|
+
points.append((px, py))
|
|
150
|
+
timings.append(elapsed)
|
|
151
|
+
elapsed += per_step * random.uniform(1.0 - frequency_randomizer, 1.0 + frequency_randomizer)
|
|
152
|
+
|
|
153
|
+
if overshoot:
|
|
154
|
+
# Correction: a couple of small samples back onto the true target, at the
|
|
155
|
+
# slower pace of a deliberate fine adjustment.
|
|
156
|
+
settle_steps = random.randint(2, 4)
|
|
157
|
+
origin = points[-1]
|
|
158
|
+
for i in range(1, settle_steps + 1):
|
|
159
|
+
t = i / settle_steps
|
|
160
|
+
points.append(
|
|
161
|
+
(
|
|
162
|
+
origin[0] + (end[0] - origin[0]) * t + random.gauss(0.0, 0.35),
|
|
163
|
+
origin[1] + (end[1] - origin[1]) * t + random.gauss(0.0, 0.35),
|
|
164
|
+
)
|
|
165
|
+
)
|
|
166
|
+
timings.append(elapsed)
|
|
167
|
+
elapsed += random.uniform(14.0, 30.0)
|
|
168
|
+
|
|
169
|
+
# Land exactly on the requested pixel. Everything above is texture; the final
|
|
170
|
+
# sample must be the point the caller asked for or clicks drift off-target.
|
|
171
|
+
points[-1] = end
|
|
172
|
+
return points, timings
|