captchakraken 2.9.0 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +112 -0
- package/NOTICE +36 -0
- package/dist/cli-invocation.d.ts +7 -36
- package/dist/cli-invocation.js +8 -49
- package/dist/errors.d.ts +7 -27
- package/dist/errors.js +6 -25
- package/dist/humanize.d.ts +56 -240
- package/dist/humanize.js +159 -415
- package/dist/index.d.ts +5 -129
- package/dist/index.js +4 -98
- package/dist/kinds.d.ts +135 -0
- package/dist/kinds.js +70 -0
- package/dist/limits.d.ts +0 -48
- package/dist/limits.js +2 -48
- package/dist/model-name.d.ts +4 -19
- package/dist/model-name.js +32 -40
- package/dist/playwright-types.d.ts +25 -95
- package/dist/playwright-types.js +0 -23
- package/dist/puppeteer-adapter.d.ts +4 -43
- package/dist/puppeteer-adapter.js +22 -58
- package/dist/python-command.d.ts +0 -41
- package/dist/python-command.js +0 -37
- package/dist/selectors.d.ts +54 -0
- package/dist/selectors.js +117 -0
- package/dist/slide-geometry.d.ts +2 -29
- package/dist/slide-geometry.js +18 -44
- package/dist/solver.d.ts +94 -712
- package/dist/solver.js +923 -3362
- package/dist/timing.d.ts +7 -40
- package/dist/timing.js +4 -36
- package/dist/token-usage.d.ts +1 -7
- package/dist/token-usage.js +10 -95
- package/dist/types.d.ts +75 -493
- package/dist/watcher.d.ts +5 -78
- package/dist/watcher.js +0 -56
- package/package.json +11 -4
- package/python/AGENTS.md +122 -0
- package/python/NOTICE +36 -0
- package/python/README.md +1 -2
- package/python/examples/README.md +10 -0
- package/python/examples/_harness.py +4 -73
- package/python/examples/demoHcaptcha.py +0 -13
- package/python/examples/demoRecaptcha.py +0 -10
- package/python/examples/watch_playwright.py +1 -26
- package/python/examples/with_local_gguf.py +28 -0
- package/python/examples/with_playwright.py +0 -19
- package/python/pyproject.toml +68 -19
- package/python/src/captchakraken/__init__.py +37 -45
- package/python/src/captchakraken/action_types.py +14 -36
- package/python/src/captchakraken/cli.py +262 -883
- package/python/src/captchakraken/config.py +60 -146
- package/python/src/captchakraken/errors.py +17 -95
- package/python/src/captchakraken/humanize.py +174 -597
- package/python/src/captchakraken/image_processor.py +16 -235
- package/python/src/captchakraken/keyframes.py +46 -484
- package/python/src/captchakraken/kinds.py +152 -0
- package/python/src/captchakraken/models.json +36 -4
- package/python/src/captchakraken/overlay.py +65 -498
- package/python/src/captchakraken/page_solver.py +1052 -3098
- package/python/src/captchakraken/planner.py +50 -487
- package/python/src/captchakraken/prompts.py +45 -261
- package/python/src/captchakraken/selectors.py +184 -0
- package/python/src/captchakraken/server_manager.py +7 -58
- package/python/src/captchakraken/solver.py +128 -522
- package/python/src/captchakraken/timing.py +20 -74
- package/python/src/captchakraken/tool_calls/board_painted.py +47 -0
- package/python/src/captchakraken/tool_calls/find_checkbox.py +20 -30
- package/python/src/captchakraken/tool_calls/find_grid.py +186 -1339
- package/python/src/captchakraken/tool_calls/track_piece.py +81 -57
- package/python/src/captchakraken/updater.py +7 -76
- package/python/src/captchakraken/watcher.py +12 -101
- package/scripts/copy-python.mjs +10 -10
- package/scripts/setup-python.js +1 -21
- package/dist/trajectory.d.ts +0 -58
- package/dist/trajectory.js +0 -266
- package/python/src/captchakraken/tool_calls/move_indicator.py +0 -431
- package/python/src/captchakraken/trajectory.py +0 -284
|
@@ -1,12 +1,3 @@
|
|
|
1
|
-
"""
|
|
2
|
-
ActionPlanner (v2) — talks to local vLLM serving the `captcha` LoRA over the
|
|
3
|
-
OpenAI-compatible /v1/chat/completions endpoint.
|
|
4
|
-
|
|
5
|
-
v1 supported transformers/vllm-local/gemini/openrouter and a whole tool-using
|
|
6
|
-
planner with detect/segment/drag-refine; that code is preserved on the
|
|
7
|
-
`v1-old-architecture` branch.
|
|
8
|
-
"""
|
|
9
|
-
|
|
10
1
|
import base64
|
|
11
2
|
import io
|
|
12
3
|
import json
|
|
@@ -21,46 +12,19 @@ import requests
|
|
|
21
12
|
from PIL import Image
|
|
22
13
|
|
|
23
14
|
from . import config, errors, prompts
|
|
15
|
+
from .kinds import ActionKind, PixelAnswerKind, PromptFamily, RetryMode
|
|
24
16
|
from .server_manager import ensure_server
|
|
25
|
-
from .timing import timed
|
|
26
17
|
|
|
27
18
|
DEBUG = os.getenv("CAPTCHA_DEBUG", "0") == "1"
|
|
28
19
|
|
|
29
|
-
#
|
|
30
|
-
#
|
|
31
|
-
#
|
|
32
|
-
# Training exports `MIN_PIXELS=200704` (448², in the training repo's
|
|
33
|
-
# the training pipeline), so anything smaller is enlarged before the vision
|
|
34
|
-
# encoder sees it. Serving did none of that: vLLM runs with no
|
|
35
|
-
# `--mm-processor-kwargs` and this client used to re-encode the file unchanged,
|
|
36
|
-
# so a small captcha arrived at a geometry the model was never tuned on. It
|
|
37
|
-
# fails as plausible-but-wrong coordinates, never as an error — measured on real
|
|
38
|
-
# a GeeTest v3 slider captures (277x285), predictions landed 80-105 px from the
|
|
39
|
-
# hand label at native size and 1-4 px away once upscaled.
|
|
40
|
-
#
|
|
41
|
-
# A FLOOR, NOT A RESIZE. Images already above it are passed through byte-for-byte;
|
|
42
|
-
# re-encoding every screenshot would spend time and fidelity on the types that
|
|
43
|
-
# were never affected.
|
|
44
|
-
#
|
|
45
|
-
# Unconditional as a DEFAULT, because the deployed v1.1 adapter improves under
|
|
46
|
-
# it too (mean error ~40 px native vs ~4 px upscaled). But it is no longer a
|
|
47
|
-
# constant: `models.json` may declare a `pixel_budget` per model, because the
|
|
48
|
-
# band an adapter TRAINED under is a property of that adapter, exactly like its
|
|
49
|
-
# prompt generation. See prompts.pixel_budget.
|
|
20
|
+
# A floor, not a resize, and the AREA is clamped rather than the sides: predictions on a 277x285 GeeTest
|
|
21
|
+
# board landed 80-105px from the hand label at native size and 1-4px once upscaled, and squashing the
|
|
22
|
+
# aspect moves every tile centre.
|
|
50
23
|
MIN_PIXELS = 448 * 448
|
|
51
24
|
|
|
52
25
|
|
|
53
26
|
def _encode_image(path: str,
|
|
54
27
|
budget: "Optional[prompts.PixelBudget]" = None) -> tuple:
|
|
55
|
-
"""`(mime, base64)` for one image, fitted into `budget`'s pixel band.
|
|
56
|
-
|
|
57
|
-
Area is clamped, never dimensions: aspect ratio is the geometry of a grid
|
|
58
|
-
puzzle, and squashing it moves every tile centre — fatal when the answer is
|
|
59
|
-
a coordinate on a normalized 0-1000 scale.
|
|
60
|
-
|
|
61
|
-
Anything Pillow cannot open is passed through untouched — this is a
|
|
62
|
-
preprocessing nicety, and it must never be the reason a solve fails.
|
|
63
|
-
"""
|
|
64
28
|
budget = budget or prompts.pixel_budget(None)
|
|
65
29
|
with open(path, "rb") as f:
|
|
66
30
|
raw = f.read()
|
|
@@ -68,12 +32,10 @@ def _encode_image(path: str,
|
|
|
68
32
|
try:
|
|
69
33
|
im = Image.open(io.BytesIO(raw))
|
|
70
34
|
width, height = im.size
|
|
71
|
-
except Exception:
|
|
35
|
+
except Exception:
|
|
72
36
|
return mime, base64.b64encode(raw).decode()
|
|
73
37
|
cap = budget.maximum
|
|
74
38
|
if cap and width * height > cap:
|
|
75
|
-
# floor, not ceil: rounding up here can land back OVER the cap, which
|
|
76
|
-
# is the mirror of the bug the floor branch below guards against.
|
|
77
39
|
scale = math.sqrt(cap / (width * height))
|
|
78
40
|
im = im.convert("RGB").resize(
|
|
79
41
|
(max(1, int(width * scale)), max(1, int(height * scale))),
|
|
@@ -83,8 +45,6 @@ def _encode_image(path: str,
|
|
|
83
45
|
return "image/png", base64.b64encode(buf.getvalue()).decode()
|
|
84
46
|
if width * height >= budget.minimum:
|
|
85
47
|
return mime, base64.b64encode(raw).decode()
|
|
86
|
-
# ceil, not round: rounding both sides down lands just under the floor
|
|
87
|
-
# (442x454 = 200,668 for a 277x285 capture) and defeats the whole point.
|
|
88
48
|
scale = math.sqrt(budget.minimum / (width * height))
|
|
89
49
|
im = im.convert("RGB").resize(
|
|
90
50
|
(math.ceil(width * scale), math.ceil(height * scale)), Image.BICUBIC)
|
|
@@ -92,98 +52,39 @@ def _encode_image(path: str,
|
|
|
92
52
|
im.save(buf, "PNG")
|
|
93
53
|
return "image/png", base64.b64encode(buf.getvalue()).decode()
|
|
94
54
|
|
|
95
|
-
#
|
|
96
|
-
#
|
|
97
|
-
#
|
|
98
|
-
# to the backup GPUs so it never competes with production traffic on the main
|
|
99
|
-
# 5090. The tier-2 CI gate sets this — its traffic is throwaway and must stay
|
|
100
|
-
# off the primary.
|
|
101
|
-
#
|
|
102
|
-
# Deliberately a HEADER, not vLLM's request-body `priority` field: that field is
|
|
103
|
-
# lower-is-higher and already means captcha=0 / apply=100 / label=200 on the
|
|
104
|
-
# priority-scheduled primary, so a value like 10 would MISORDER there (it would
|
|
105
|
-
# outrank apply/label). Routing is a fleet concern, kept out of the model's
|
|
106
|
-
# scheduler.
|
|
55
|
+
# A header, deliberately not vLLM's body `priority`: that field is lower-is-higher and already means
|
|
56
|
+
# captcha=0 / apply=100 / label=200 on the primary, so a 10 there would misorder. The gateway routes >5 to
|
|
57
|
+
# the backup GPUs; tier-2 CI sets it.
|
|
107
58
|
_PRIORITY_HEADER = "X-JH-Priority"
|
|
108
59
|
_PRIORITY_ENV = "CAPTCHA_REQUEST_PRIORITY"
|
|
109
60
|
|
|
110
|
-
# Hosted-API metadata. Both are OPTIONAL and absent unless deliberately set, so
|
|
111
|
-
# self-hosted users never send them and are unaffected.
|
|
112
|
-
#
|
|
113
|
-
# X-CK-Client — which integration issued this solve (e.g. "camoufox/0.4.11").
|
|
114
|
-
# camoufox sets CAPTCHA_KRAKEN_CLIENT in the env it spawns the solver with, so
|
|
115
|
-
# the gateway can account camoufox-attributed usage separately. Attribution
|
|
116
|
-
# ONLY: it is caller-supplied and therefore never priced on (the gateway
|
|
117
|
-
# derives billable puzzle class from the request body instead).
|
|
118
|
-
# X-CK-Session — groups the 1..N inference rounds of ONE captcha into a single
|
|
119
|
-
# billable attempt. The JS driver mints a UUID per solve() and reuses it for
|
|
120
|
-
# every CLI invocation in that solve, which is what lets the gateway cap an
|
|
121
|
-
# attempt's billable rounds instead of charging per round without limit.
|
|
122
61
|
_CLIENT_HEADER = "X-CK-Client"
|
|
123
62
|
_CLIENT_ENV = "CAPTCHA_KRAKEN_CLIENT"
|
|
63
|
+
# One id per solve lets the gateway cap an attempt's billable rounds.
|
|
124
64
|
_SESSION_HEADER = "X-CK-Session"
|
|
125
65
|
_SESSION_ENV = "CAPTCHA_KRAKEN_SESSION"
|
|
126
66
|
|
|
127
|
-
#
|
|
128
|
-
#
|
|
129
|
-
# ON by default because the report is what makes a failed solve improvable, and
|
|
130
|
-
# because it is the only way a hosted account's failures can be found at all.
|
|
131
|
-
# Opting out here stops the CLIENT sending it; `captureOptOut` on the account is
|
|
132
|
-
# the server-side half, and either one alone is enough to keep nothing.
|
|
67
|
+
# On by default; `captureOptOut` is the server half. Tier 3 sets it to 0 for ~100 solves a run.
|
|
133
68
|
_REPORT_OUTCOME_ENV = "CAPTCHA_REPORT_OUTCOME"
|
|
134
69
|
|
|
135
|
-
#
|
|
136
|
-
# commas. Empty and absent by default.
|
|
137
|
-
#
|
|
138
|
-
# WHY THIS EXISTS. Some endpoints sit behind a gate that is not the API key —
|
|
139
|
-
# our own dev gateway (api.dev.captchakraken.com) wants X-CK-Dev-Auth, and a
|
|
140
|
-
# corporate egress proxy may want its own token. Without this the published
|
|
141
|
-
# client simply cannot reach such an endpoint, which meant the dev environment
|
|
142
|
-
# could not exercise the real client path at all: the thing dev exists to
|
|
143
|
-
# rehearse was the one thing it could not do.
|
|
144
|
-
#
|
|
145
|
-
# IT CANNOT OVERWRITE Authorization, Content-Type, OR THE X-CK-* HEADERS. Those
|
|
146
|
-
# carry the credential and the billing attribution. Allowing an env var to
|
|
147
|
-
# rewrite Authorization would turn a stray export into "your key is not the key
|
|
148
|
-
# being charged", and allowing it to rewrite X-CK-Session would let a caller
|
|
149
|
-
# escape the per-attempt billing cap by pinning one session id forever.
|
|
70
|
+
# Extra headers may not rewrite these: pinning one X-CK-Session forever would escape the per-attempt billing cap.
|
|
150
71
|
_EXTRA_HEADERS_ENV = "CAPTCHA_KRAKEN_EXTRA_HEADERS"
|
|
151
72
|
_PROTECTED_HEADERS = frozenset(
|
|
152
73
|
{"authorization", "content-type", _CLIENT_HEADER.lower(), _SESSION_HEADER.lower()}
|
|
153
74
|
)
|
|
154
75
|
|
|
155
|
-
# These values reach the wire verbatim from the environment, so they are
|
|
156
|
-
# sanitized rather than trusted: a CR/LF would otherwise splice arbitrary
|
|
157
|
-
# headers into the upstream request.
|
|
158
76
|
_HEADER_VALUE_MAX = 128
|
|
159
77
|
|
|
160
78
|
|
|
161
|
-
# RFC 7230 token characters. Anything else is not a header name.
|
|
162
79
|
_VALID_HEADER_NAME = re.compile(r"^[!#$%&'*+\-.^_`|~0-9A-Za-z]+$")
|
|
163
80
|
|
|
164
81
|
|
|
165
82
|
def _clean_header_value(raw: str) -> str:
|
|
166
|
-
|
|
83
|
+
# CR/LF would splice arbitrary headers into the request.
|
|
167
84
|
return "".join(c for c in raw.strip() if 0x20 <= ord(c) < 0x7F)[:_HEADER_VALUE_MAX]
|
|
168
85
|
|
|
169
86
|
|
|
170
87
|
def _extra_headers(raw: str) -> Dict[str, str]:
|
|
171
|
-
"""Parse CAPTCHA_KRAKEN_EXTRA_HEADERS into a header dict.
|
|
172
|
-
|
|
173
|
-
Accepts "Name: value" pairs separated by newlines or commas:
|
|
174
|
-
|
|
175
|
-
CAPTCHA_KRAKEN_EXTRA_HEADERS='X-CK-Dev-Auth: hunter2'
|
|
176
|
-
CAPTCHA_KRAKEN_EXTRA_HEADERS=$'X-One: a\nX-Two: b'
|
|
177
|
-
|
|
178
|
-
Malformed entries are DROPPED, never guessed at. A pair with no colon is
|
|
179
|
-
not a header, and inventing one from it would put an attacker-shaped string
|
|
180
|
-
on the wire. Names are restricted to the RFC 7230 token characters for the
|
|
181
|
-
same reason the values are sanitized: a CR/LF in either would splice extra
|
|
182
|
-
headers into the request.
|
|
183
|
-
|
|
184
|
-
Protected headers are refused, so this cannot be used to replace the
|
|
185
|
-
credential or the billing attribution.
|
|
186
|
-
"""
|
|
187
88
|
out: Dict[str, str] = {}
|
|
188
89
|
if not raw.strip():
|
|
189
90
|
return out
|
|
@@ -206,17 +107,7 @@ def _extra_headers(raw: str) -> Dict[str, str]:
|
|
|
206
107
|
|
|
207
108
|
|
|
208
109
|
def routing_headers(env=None) -> Dict[str, str]:
|
|
209
|
-
"""
|
|
210
|
-
|
|
211
|
-
Empty by default. Split out and env-injectable so it can be unit-tested
|
|
212
|
-
without a live server. A missing or non-integer CAPTCHA_REQUEST_PRIORITY
|
|
213
|
-
yields no header at all — an unset/garbage value must never silently tag
|
|
214
|
-
traffic for the backups.
|
|
215
|
-
|
|
216
|
-
Each header is derived independently: a malformed priority must not suppress
|
|
217
|
-
the attribution headers, or one typo would silently make a camoufox solve
|
|
218
|
-
look like direct traffic and understate the partner's revenue share.
|
|
219
|
-
"""
|
|
110
|
+
"""Each header is derived on its own: a malformed priority must not drop the attribution headers and understate a partner's revenue share."""
|
|
220
111
|
env = os.environ if env is None else env
|
|
221
112
|
headers: Dict[str, str] = {}
|
|
222
113
|
|
|
@@ -237,88 +128,23 @@ def routing_headers(env=None) -> Dict[str, str]:
|
|
|
237
128
|
return headers
|
|
238
129
|
|
|
239
130
|
|
|
240
|
-
# ── Prompt constants ────────────────────────────────────────────────────────
|
|
241
|
-
#
|
|
242
|
-
# These are ALIASES for the newest generation's built-ins, kept so that code and
|
|
243
|
-
# tests written against the old module-level constants keep working. They are
|
|
244
|
-
# NOT what a solve sends. The prompt a model gets is resolved per model
|
|
245
|
-
# (`prompts.resolve`), because a model answers in whatever schema its prompt
|
|
246
|
-
# asks for — sending a version-1 model the version-2 prompt does not error, it
|
|
247
|
-
# silently degrades every puzzle. See prompts.py and models.json.
|
|
248
|
-
#
|
|
249
|
-
# The text itself lives in prompts.py and nowhere else. Two copies of a prompt
|
|
250
|
-
# is the drift this module already shipped once (2026-07-18).
|
|
251
131
|
_LATEST = prompts.builtin(prompts.LATEST_PROMPT_VERSION)
|
|
252
132
|
|
|
253
133
|
SELECT_GRID_PROMPT = _LATEST.grid_template
|
|
254
134
|
|
|
255
135
|
|
|
256
|
-
# Non-grid click/drag puzzles, newest generation. Alias — see the note above.
|
|
257
136
|
PIXEL_ACTION_PROMPT = _LATEST.action_prompt
|
|
258
137
|
|
|
259
138
|
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
#
|
|
263
|
-
#
|
|
264
|
-
#
|
|
265
|
-
#
|
|
266
|
-
# Alias for the newest generation — see the note above. Generation 1 has NO
|
|
267
|
-
# animated prompt (the family did not exist), which is why resolution is per
|
|
268
|
-
# model rather than per client.
|
|
269
|
-
VIDEO_ACTION_PROMPT_TEMPLATE = _LATEST.video_template
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
def video_action_prompt(n_keyframes: int) -> str:
|
|
273
|
-
"""The prompt for a challenge served as `n_keyframes` stills.
|
|
274
|
-
|
|
275
|
-
The count is in the text because the model has no other way to know how many
|
|
276
|
-
images arrived — frame identities live in the prompt, not in the image payload.
|
|
277
|
-
Mirrors `instructions.video_instruction` in the training repo exactly.
|
|
278
|
-
"""
|
|
279
|
-
n = int(n_keyframes)
|
|
280
|
-
if n < 1:
|
|
281
|
-
raise ValueError(f"a keyframe request needs at least one frame, got {n_keyframes}")
|
|
282
|
-
listing = ", ".join(f"frame {i}" for i in range(1, n + 1))
|
|
283
|
-
return VIDEO_ACTION_PROMPT_TEMPLATE.format(n=n, listing=listing)
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
#: Sampling temperature per RE-ASK of a board the vendor refused, first entry
|
|
287
|
-
#: first. THE ONE COPY: the JS port reaches this through `CAPTCHA_RESAMPLE_LEVEL`
|
|
288
|
-
#: rather than carrying its own numbers, because a schedule written down twice is
|
|
289
|
-
#: a schedule the two ports can disagree about — and the driver gate drives both
|
|
290
|
-
#: and averages them.
|
|
291
|
-
#:
|
|
292
|
-
#: GREEDY ONLY, AND THAT IS A MEASUREMENT, NOT AN OVERSIGHT. Heating the re-ask
|
|
293
|
-
#: was tried: `(0.0, 0.35, 0.7)`, escalating each time a board came back with the
|
|
294
|
-
#: answer it had already given. On five puzzle types that a greedy driver passes,
|
|
295
|
-
#: the escalating one scored 0/5 against 4/5, and the median FAILED attempt cost
|
|
296
|
-
#: 24.0s against 13.9s. Two things went wrong and they compound:
|
|
297
|
-
#:
|
|
298
|
-
#: 1. A hotter sample is a WORSE answer. The greedy read is the best single
|
|
299
|
-
#: guess this model has; the schedule spends rounds walking away from it.
|
|
300
|
-
#: 2. It defeats the give-up rule. `_no_progress_rounds` counts IDENTICAL
|
|
301
|
-
#: answers, so making each re-ask differ resets the counter — the attempt
|
|
302
|
-
#: stops bailing at round 3 and runs to the round ceiling instead. Slower
|
|
303
|
-
#: failures, not fewer.
|
|
304
|
-
#:
|
|
305
|
-
#: The plumbing stays because the LEVEL is the right thing to travel; what was
|
|
306
|
-
#: wrong was the trigger. A re-ask should be driven by the vendor actually
|
|
307
|
-
#: refusing the answer — read off the network response — and not by the model
|
|
308
|
-
#: agreeing with itself, which is only ever evidence that decoding is
|
|
309
|
-
#: deterministic. Add temperatures here when that signal exists to gate them.
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
# Greedy only, by measurement: (0.0, 0.35, 0.7) scored 0/5 against 4/5 and made the median failed attempt
|
|
142
|
+
# 24.0s against 13.9s, because a differing re-ask resets the no-progress counter. The level plumbing stays for
|
|
143
|
+
# a network-refusal trigger. Lives here once so the two ports cannot drift (TRIBAL_KNOWLEDGE.md).
|
|
310
144
|
RESAMPLE_TEMPERATURES = (0.0,)
|
|
311
145
|
|
|
312
146
|
|
|
313
147
|
def sampling_for_level(level: int) -> Dict[str, Any]:
|
|
314
|
-
"""Decode overrides for the `level`-th re-ask of one board.
|
|
315
|
-
|
|
316
|
-
Level 0 is greedy and returns {}, so a first look is byte-identical to what
|
|
317
|
-
this client has always sent. Past that it carries a temperature AND a seed:
|
|
318
|
-
the temperature is what makes a second sample possible at all, and the seed
|
|
319
|
-
is what makes two re-asks at one temperature two samples rather than one
|
|
320
|
-
sample twice.
|
|
321
|
-
"""
|
|
322
148
|
temps = RESAMPLE_TEMPERATURES or (0.0,)
|
|
323
149
|
temp = temps[min(max(level, 0), len(temps) - 1)]
|
|
324
150
|
if not temp:
|
|
@@ -327,13 +153,6 @@ def sampling_for_level(level: int) -> Dict[str, Any]:
|
|
|
327
153
|
|
|
328
154
|
|
|
329
155
|
def _sampling_from_env() -> Dict[str, Any]:
|
|
330
|
-
"""The JS port's re-ask level, which arrives as an env var.
|
|
331
|
-
|
|
332
|
-
That port drives its own round loop and spawns this CLI once per round, so
|
|
333
|
-
a fresh process has no memory of how many times the board has already been
|
|
334
|
-
read. Passing the LEVEL rather than the temperature keeps the schedule in
|
|
335
|
-
one place — see `RESAMPLE_TEMPERATURES`.
|
|
336
|
-
"""
|
|
337
156
|
raw = os.environ.get("CAPTCHA_RESAMPLE_LEVEL", "").strip()
|
|
338
157
|
if not raw:
|
|
339
158
|
return {}
|
|
@@ -344,119 +163,44 @@ def _sampling_from_env() -> Dict[str, Any]:
|
|
|
344
163
|
|
|
345
164
|
|
|
346
165
|
class ActionPlanner:
|
|
347
|
-
"""Thin client for the vLLM `captcha` LoRA."""
|
|
348
166
|
|
|
349
167
|
def __init__(
|
|
350
168
|
self,
|
|
351
169
|
model: Optional[str] = None,
|
|
352
|
-
debug_callback: Optional[Any] = None,
|
|
353
170
|
base_url: Optional[str] = None,
|
|
354
171
|
api_key: Optional[str] = None,
|
|
355
|
-
expert: Optional[
|
|
172
|
+
expert: Optional[PromptFamily] = None,
|
|
356
173
|
**_: Any,
|
|
357
174
|
):
|
|
358
|
-
self.debug_callback = debug_callback
|
|
359
175
|
self.token_usage: List[Dict[str, Any]] = []
|
|
360
176
|
|
|
361
|
-
# All model/endpoint defaults come from `config` (env-overridable), so
|
|
362
|
-
# the planner itself is model-agnostic — swap models via env, not code.
|
|
363
177
|
self.model = model or config.lora_name()
|
|
364
178
|
self.base_url = base_url or config.base_url()
|
|
365
179
|
self.api_key = api_key or config.api_key()
|
|
366
|
-
#: Decode overrides merged into every request payload, or empty.
|
|
367
|
-
#:
|
|
368
|
-
#: The page driver sets this while RE-ASKING a board the vendor already
|
|
369
|
-
#: refused, so that the second look is a second SAMPLE rather than the
|
|
370
|
-
#: same arithmetic run twice — see
|
|
371
|
-
#: `PageSolverConfig.resample_temperatures`. Empty by default and
|
|
372
|
-
#: cleared the moment a fresh board arrives, because greedy is the best
|
|
373
|
-
#: single guess and every solve that works today works on it.
|
|
374
180
|
self.sampling: Dict[str, Any] = {}
|
|
375
|
-
# Prompts follow the MODEL, not this client's release. Resolve by the
|
|
376
|
-
# served name first (a hosted endpoint serves `captcha`, not a repo id,
|
|
377
|
-
# and models.json maps the alias back), falling back to the adapter repo
|
|
378
|
-
# id for a self-hosted deploy that serves under its own name.
|
|
379
|
-
#
|
|
380
|
-
# Doing this ONCE per planner rather than per request: resolution can
|
|
381
|
-
# touch the Hub, and a per-request lookup would put a network call in
|
|
382
|
-
# front of every round of every solve.
|
|
383
181
|
_prompt_key = (self.model if prompts.canonical_model_id(self.model)
|
|
384
182
|
else config.lora_adapter())
|
|
385
183
|
self.prompts = prompts.resolve(_prompt_key)
|
|
386
|
-
# Resolution follows the MODEL for the same reason prompts do, and off
|
|
387
|
-
# the same key: an adapter reads a puzzle at the pixel band it trained
|
|
388
|
-
# under. Resolved once per planner — see the note above.
|
|
389
184
|
self.pixel_budget = prompts.pixel_budget(_prompt_key)
|
|
390
|
-
# WHICH ADAPTER answers each prompt family. Empty for every model that
|
|
391
|
-
# is not routed, which is every model published so far — `_model_for`
|
|
392
|
-
# then returns `self.model` for every request and the wire is unchanged.
|
|
393
|
-
#
|
|
394
|
-
# Resolved once, like the two above, and for a stronger reason: the pin
|
|
395
|
-
# is validated here, so `--expert grid` against a single-adapter model
|
|
396
|
-
# fails when the solver is BUILT rather than on the first grid it meets.
|
|
397
185
|
self.expert = (expert if expert is not None
|
|
398
186
|
else prompts.expert_pin())
|
|
399
187
|
self.experts = prompts.experts(_prompt_key)
|
|
400
188
|
if self.expert:
|
|
401
|
-
# Raises on an unknown family, and on a pin against an unrouted
|
|
402
|
-
# model. Both are configuration errors, and a benchmark that
|
|
403
|
-
# quietly measured the generalist while reporting an expert is the
|
|
404
|
-
# exact class of silent mispairing models.json exists to prevent.
|
|
405
189
|
prompts.route(_prompt_key, None, pin=self.expert)
|
|
406
|
-
# Auto-start a local vLLM server on the first request if one isn't up
|
|
407
|
-
# (no-op for a healthy or remote endpoint). Guarded so we only try once.
|
|
408
190
|
self._server_ensured = False
|
|
409
|
-
# ONE connection, reused for every round of every solve this planner
|
|
410
|
-
# serves. `requests.post` opens a fresh TCP connection each call — and
|
|
411
|
-
# to a hosted endpoint a TLS handshake with it. Measured against the
|
|
412
|
-
# gate's own endpoint: 258ms per request connectionless vs 144ms
|
|
413
|
-
# pooled, so every inference was paying ~110ms to re-dial a server it
|
|
414
|
-
# had just finished talking to. A multi-round grid pays it per round.
|
|
415
|
-
#
|
|
416
|
-
# The planner is per-PageSolver and a PageSolver is reusable across
|
|
417
|
-
# solves, so a caller who keeps one keeps the connection warm too.
|
|
418
191
|
self._http = requests.Session()
|
|
419
|
-
# Latched off by the first 404 — see `report_outcome`.
|
|
420
192
|
self._outcome_supported = True
|
|
421
193
|
|
|
422
|
-
def _model_for(self, family: Optional[
|
|
423
|
-
"""The `model` string for a request in `family`.
|
|
424
|
-
|
|
425
|
-
`self.model` whenever the model is not routed, so a caller may pass a
|
|
426
|
-
family unconditionally. A routed model with no expert for this family
|
|
427
|
-
degrades to the generalist — never to another expert, never to an
|
|
428
|
-
error (docs/MOE_LORA_DESIGN.md §11).
|
|
429
|
-
"""
|
|
194
|
+
def _model_for(self, family: Optional[PromptFamily]) -> str:
|
|
430
195
|
if not self.experts:
|
|
431
196
|
return self.model
|
|
432
197
|
return prompts.route(self.model, family, pin=self.expert) or self.model
|
|
433
198
|
|
|
434
|
-
# ── the solve's own verdict, sent back ─────────────────────────────────
|
|
435
|
-
#
|
|
436
|
-
# THE ONE THING THE SERVER CANNOT SEE. A wrong answer reaches the gateway as
|
|
437
|
-
# a 200 with well-formed JSON in it; only this driver watched the widget.
|
|
438
|
-
# Without this report the hosted API cannot tell a solved captcha from a
|
|
439
|
-
# failed one, and the failures — the exact boards the model is worst at, on
|
|
440
|
-
# real sites — are the one dataset that cannot be collected any other way.
|
|
441
|
-
#
|
|
442
|
-
# BEST-EFFORT, ALWAYS. It runs after the solve is over and its result is
|
|
443
|
-
# already decided, so nothing it does can change the answer the caller gets.
|
|
444
|
-
# Every failure is swallowed: a self-hosted vLLM has no such route, a
|
|
445
|
-
# network blip is not the caller's problem, and an exception here would turn
|
|
446
|
-
# a successful solve into a raised one.
|
|
447
|
-
#
|
|
448
|
-
# ONE 404 IS ENOUGH. A self-hosted endpoint answers 404 forever, so the
|
|
449
|
-
# first one latches this off for the life of the planner rather than paying
|
|
450
|
-
# a round trip per solve to be told the same thing again.
|
|
451
199
|
_OUTCOME_PATH = "/solve-outcome"
|
|
452
200
|
_OUTCOME_TIMEOUT_S = 3.0
|
|
453
201
|
|
|
454
202
|
def report_outcome(self, session_id: Optional[str], solved: bool) -> bool:
|
|
455
|
-
"""
|
|
456
|
-
|
|
457
|
-
Never raises. The return value is for tests and for `--debug`; no caller
|
|
458
|
-
should branch on it.
|
|
459
|
-
"""
|
|
203
|
+
"""A 404 disables it for the planner's lifetime: a self-hosted vLLM has no /solve-outcome."""
|
|
460
204
|
if not session_id or not self._outcome_supported:
|
|
461
205
|
return False
|
|
462
206
|
if os.getenv(_REPORT_OUTCOME_ENV, "1") == "0":
|
|
@@ -472,11 +216,10 @@ class ActionPlanner:
|
|
|
472
216
|
json={"session": session_id, "solved": bool(solved)},
|
|
473
217
|
timeout=self._OUTCOME_TIMEOUT_S,
|
|
474
218
|
)
|
|
475
|
-
except Exception as exc:
|
|
219
|
+
except Exception as exc:
|
|
476
220
|
self._log(f"outcome report failed: {exc}")
|
|
477
221
|
return False
|
|
478
222
|
if resp.status_code == 404:
|
|
479
|
-
# Not an error: this endpoint does not serve the route. Stop asking.
|
|
480
223
|
self._outcome_supported = False
|
|
481
224
|
self._log("outcome reporting: endpoint has no /solve-outcome; disabled")
|
|
482
225
|
return False
|
|
@@ -487,28 +230,17 @@ class ActionPlanner:
|
|
|
487
230
|
def _log(self, message: str) -> None:
|
|
488
231
|
if DEBUG:
|
|
489
232
|
print(f"[Planner] {message}", file=sys.stderr)
|
|
490
|
-
if self.debug_callback:
|
|
491
|
-
self.debug_callback(f"[Planner] {message}")
|
|
492
233
|
|
|
493
234
|
def _chat_with_image(self, prompt: str, image_path: str, max_tokens: int = 512,
|
|
494
|
-
family: Optional[
|
|
235
|
+
family: Optional[PromptFamily] = None) -> str:
|
|
495
236
|
return self._chat_with_images(prompt, [image_path], max_tokens=max_tokens,
|
|
496
237
|
family=family)
|
|
497
238
|
|
|
498
239
|
def _chat_with_images(
|
|
499
240
|
self, prompt: str, image_paths: List[str], max_tokens: int = 512,
|
|
500
|
-
family: Optional[
|
|
241
|
+
family: Optional[PromptFamily] = None,
|
|
501
242
|
) -> str:
|
|
502
|
-
"""One request
|
|
503
|
-
|
|
504
|
-
The images go in ONE message, before the text, in the order given. That
|
|
505
|
-
ordering is what makes an animated challenge answerable: the model is told
|
|
506
|
-
"you are given N keyframes, in order: frame 1, frame 2, …" and matches those
|
|
507
|
-
names to the images positionally. Splitting them across requests, or putting
|
|
508
|
-
the text first, would break the correspondence between the numbers in the
|
|
509
|
-
prompt and the pictures — and the frame number is the part the page driver
|
|
510
|
-
acts on.
|
|
511
|
-
"""
|
|
243
|
+
"""One request per image set, images before the text in one message: the model reads frames positionally, and per-frame queries would cost N billable rounds."""
|
|
512
244
|
if not image_paths:
|
|
513
245
|
raise ValueError("no images to send")
|
|
514
246
|
|
|
@@ -530,23 +262,15 @@ class ActionPlanner:
|
|
|
530
262
|
]
|
|
531
263
|
|
|
532
264
|
model = self._model_for(family)
|
|
533
|
-
# `sampling` is set by the page driver when it is RE-ASKING a board the
|
|
534
|
-
# vendor already refused — see `PageSolverConfig.resample_temperatures`.
|
|
535
|
-
# Absent (the normal case, and every first look) it is greedy exactly as
|
|
536
|
-
# before, so nothing about a solve that works today changes.
|
|
537
265
|
payload = {
|
|
538
266
|
"model": model,
|
|
539
267
|
"messages": messages,
|
|
540
268
|
"temperature": 0,
|
|
541
269
|
"max_tokens": max_tokens,
|
|
542
|
-
#
|
|
543
|
-
# in the prompt alone is unreliable; disabling at the chat-template
|
|
544
|
-
# level is the documented way.
|
|
270
|
+
# Both spellings: Ollama ignores chat_template_kwargs and defaults thinking on; vLLM reads reasoning_effort only when the kwargs are unset.
|
|
545
271
|
"chat_template_kwargs": {"enable_thinking": False},
|
|
272
|
+
"reasoning_effort": "none",
|
|
546
273
|
}
|
|
547
|
-
# In-process the driver sets `sampling` directly; the JS port drives
|
|
548
|
-
# its own loop and spawns this as a CLI, so its level arrives in the
|
|
549
|
-
# environment. Explicit beats inherited when both are present.
|
|
550
274
|
sampling = self.sampling or _sampling_from_env()
|
|
551
275
|
if sampling:
|
|
552
276
|
payload.update(sampling)
|
|
@@ -554,20 +278,13 @@ class ActionPlanner:
|
|
|
554
278
|
headers = {
|
|
555
279
|
"Authorization": f"Bearer {self.api_key}",
|
|
556
280
|
"Content-Type": "application/json",
|
|
557
|
-
# Fleet routing: absent unless CAPTCHA_REQUEST_PRIORITY is set (see
|
|
558
|
-
# routing_headers). Low-priority batch traffic is steered to the
|
|
559
|
-
# backup GPUs by the gateway front on this header.
|
|
560
281
|
**routing_headers(),
|
|
561
282
|
}
|
|
562
283
|
|
|
563
|
-
# Hands-off server: before the very first request, make sure something
|
|
564
|
-
# is listening (auto-start a local vLLM if needed; no-op if it's already
|
|
565
|
-
# up or the endpoint is remote). Best-effort — a failure here surfaces as
|
|
566
|
-
# a normal connection error on the request below.
|
|
567
284
|
if not self._server_ensured:
|
|
568
285
|
try:
|
|
569
286
|
ensure_server(self.base_url)
|
|
570
|
-
except Exception as e:
|
|
287
|
+
except Exception as e:
|
|
571
288
|
self._log(f"ensure_server: {e}")
|
|
572
289
|
self._server_ensured = True
|
|
573
290
|
|
|
@@ -575,19 +292,8 @@ class ActionPlanner:
|
|
|
575
292
|
self._log(f"POST {url} model={model} max_tokens={max_tokens} "
|
|
576
293
|
f"images={len(parts)}")
|
|
577
294
|
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
# Surface auth / billing / server errors as something the reader can act
|
|
582
|
-
# on, instead of letting resp.json() blow up with a cryptic "Expecting
|
|
583
|
-
# value: line 1 column 1" on a non-JSON response.
|
|
584
|
-
#
|
|
585
|
-
# `errors.from_response` decides which of the two worlds we are in by
|
|
586
|
-
# looking for the gateway's `error.code`: present means the hosted API
|
|
587
|
-
# refused us and there is a real explanation to give (out of credits,
|
|
588
|
-
# rate limited, attempt abandoned); absent means a local vLLM or a proxy,
|
|
589
|
-
# and the old message — including the bearer-token hint on 401/403 — is
|
|
590
|
-
# reproduced unchanged.
|
|
295
|
+
resp = self._http.post(url, headers=headers, json=payload, timeout=120)
|
|
296
|
+
|
|
591
297
|
if not resp.ok:
|
|
592
298
|
raise errors.from_response(resp, url)
|
|
593
299
|
|
|
@@ -626,17 +332,9 @@ class ActionPlanner:
|
|
|
626
332
|
else:
|
|
627
333
|
return None
|
|
628
334
|
|
|
629
|
-
# strict=False tolerates unescaped control chars INSIDE strings — the
|
|
630
|
-
# model routinely emits coordinates as pretty-printed strings with a
|
|
631
|
-
# literal newline, e.g. "click": ["277,\n 728", ...], which strict JSON
|
|
632
|
-
# rejects. Without this the whole response fails to parse and a solvable
|
|
633
|
-
# click puzzle is dropped as "unsupported".
|
|
634
335
|
try:
|
|
635
336
|
return json.loads(text[start:end], strict=False)
|
|
636
337
|
except json.JSONDecodeError:
|
|
637
|
-
# The model sometimes truncates (hit max_tokens) or over-nests its
|
|
638
|
-
# pretty-printed JSON, leaving brackets unclosed. Repair by balancing
|
|
639
|
-
# from the first opener to the end of the raw text.
|
|
640
338
|
repaired = ActionPlanner._balance_json(text[start:])
|
|
641
339
|
if repaired is not None:
|
|
642
340
|
try:
|
|
@@ -647,9 +345,6 @@ class ActionPlanner:
|
|
|
647
345
|
|
|
648
346
|
@staticmethod
|
|
649
347
|
def _balance_json(text: str) -> Optional[str]:
|
|
650
|
-
"""Close any brackets/braces the model left open (string-aware), so a
|
|
651
|
-
truncated ``{"action": {"points": [[1,2],`` still parses. Returns None if
|
|
652
|
-
there was nothing to balance."""
|
|
653
348
|
stack: List[str] = []
|
|
654
349
|
in_str = False
|
|
655
350
|
esc = False
|
|
@@ -677,7 +372,6 @@ class ActionPlanner:
|
|
|
677
372
|
out = text
|
|
678
373
|
if in_str:
|
|
679
374
|
out += '"'
|
|
680
|
-
# Drop a dangling comma / partial token before the closers.
|
|
681
375
|
out = out.rstrip().rstrip(",")
|
|
682
376
|
for opener in reversed(stack):
|
|
683
377
|
out += "}" if opener == "{" else "]"
|
|
@@ -688,16 +382,8 @@ class ActionPlanner:
|
|
|
688
382
|
image_path: str,
|
|
689
383
|
rows: int,
|
|
690
384
|
cols: int,
|
|
691
|
-
retry_mode: Optional[
|
|
385
|
+
retry_mode: Optional[RetryMode] = None,
|
|
692
386
|
) -> List[int]:
|
|
693
|
-
"""Return the list of 1-indexed cells the model wants to click.
|
|
694
|
-
|
|
695
|
-
retry_mode == "missed-tiles": the previous submission was rejected
|
|
696
|
-
by the captcha vendor with an under-selection error. Append an
|
|
697
|
-
explicit recovery instruction that tells the model the FULL grid
|
|
698
|
-
contains at least one matching tile it didn't pick last time. This
|
|
699
|
-
nudges it off the "I already covered everything" attractor.
|
|
700
|
-
"""
|
|
701
387
|
total = rows * cols
|
|
702
388
|
if rows == 4 and cols == 4:
|
|
703
389
|
grid_hint = "Hint: Single large image split into tiles. Select ALL parts."
|
|
@@ -707,7 +393,7 @@ class ActionPlanner:
|
|
|
707
393
|
prompt = self.prompts.grid_prompt(
|
|
708
394
|
rows=rows, cols=cols, grid_hint=grid_hint
|
|
709
395
|
)
|
|
710
|
-
if retry_mode ==
|
|
396
|
+
if retry_mode == RetryMode.MISSED_TILES:
|
|
711
397
|
prompt = (
|
|
712
398
|
prompt
|
|
713
399
|
+ "\n\nIMPORTANT: A previous submission was rejected because not all "
|
|
@@ -717,29 +403,16 @@ class ActionPlanner:
|
|
|
717
403
|
"including any matches you may have overlooked."
|
|
718
404
|
)
|
|
719
405
|
raw = self._chat_with_image(prompt, image_path, max_tokens=128,
|
|
720
|
-
family=
|
|
406
|
+
family=PromptFamily.GRID)
|
|
721
407
|
out = self._normalize_grid(self._parse_json(raw), total)
|
|
722
408
|
self._log(f"grid selection -> {out}")
|
|
723
409
|
return out
|
|
724
410
|
|
|
725
411
|
@staticmethod
|
|
726
412
|
def _normalize_grid(data: Any, total: int) -> List[int]:
|
|
727
|
-
"""Map the model's grid JSON to a list of 1-indexed cell numbers.
|
|
728
|
-
|
|
729
|
-
Accepts the trained shape (a bare JSON array) plus the wrapped forms the
|
|
730
|
-
model drifts into: {"target_ids": [...]} and {"action": {"target_ids":
|
|
731
|
-
[...]}}. Out-of-range and non-numeric entries are dropped rather than
|
|
732
|
-
raising — a single junk element must not cost the whole selection.
|
|
733
|
-
|
|
734
|
-
Split out of get_grid_selection so the parse is testable without a
|
|
735
|
-
model, mirroring _normalize_pixel.
|
|
736
|
-
"""
|
|
737
413
|
if isinstance(data, list):
|
|
738
414
|
ids = data
|
|
739
415
|
elif isinstance(data, dict):
|
|
740
|
-
# `action` is a dict on the grid path but a bare string ("drag") on
|
|
741
|
-
# the pixel path — guard so a mis-routed response returns [] rather
|
|
742
|
-
# than raising AttributeError.
|
|
743
416
|
nested = data.get("action")
|
|
744
417
|
nested_ids = nested.get("target_ids") if isinstance(nested, dict) else None
|
|
745
418
|
ids = data.get("target_ids") or nested_ids or []
|
|
@@ -757,56 +430,20 @@ class ActionPlanner:
|
|
|
757
430
|
return out
|
|
758
431
|
|
|
759
432
|
def get_pixel_actions(self, image_path: str, text_mode: bool = False) -> List[Dict[str, Any]]:
|
|
760
|
-
"""Solve a non-grid click/drag/slide/text puzzle.
|
|
761
|
-
|
|
762
|
-
Sends the trained action prompt + image, parses the model's JSON, and
|
|
763
|
-
returns a list of normalized actions with all coordinates on a 0–1
|
|
764
|
-
scale:
|
|
765
|
-
|
|
766
|
-
{"kind": "click", "points": [(x, y), ...]}
|
|
767
|
-
{"kind": "drag", "src": (x, y), "dst": (x, y)}
|
|
768
|
-
{"kind": "slide", "dst": (x, y)} — puzzle-piece slider
|
|
769
|
-
{"kind": "type", "text": "<the code>"} — distorted text
|
|
770
|
-
|
|
771
|
-
`text_mode` swaps in the distorted-text prompt. It is chosen by the
|
|
772
|
-
DRIVER, from the widget's DOM (a visible text box in the challenge),
|
|
773
|
-
because nothing about the picture reliably says "this one is typed" —
|
|
774
|
-
BotDetect's warped letters and hCaptcha's "click the matching letter"
|
|
775
|
-
look alike to a pixel classifier, and the two want opposite answers.
|
|
776
|
-
|
|
777
|
-
Returns [] if the model produced nothing usable. The solver turns these
|
|
778
|
-
into ClickAction / DragAction / TypeAction.
|
|
779
|
-
"""
|
|
780
433
|
prompt = self.prompts.text_prompt() if text_mode else self.prompts.action_prompt
|
|
781
434
|
raw = self._chat_with_image(prompt, image_path, max_tokens=512,
|
|
782
|
-
family=
|
|
435
|
+
family=PromptFamily.TEXT if text_mode else PromptFamily.PIXEL)
|
|
783
436
|
data = self._parse_json(raw)
|
|
784
437
|
actions = self._normalize_pixel(data)
|
|
785
438
|
self._log(f"pixel actions -> {actions}")
|
|
786
439
|
return actions
|
|
787
440
|
|
|
788
441
|
def get_keyframe_actions(self, keyframe_paths: List[str]) -> List[Dict[str, Any]]:
|
|
789
|
-
"""Solve an ANIMATED challenge from its keyframes.
|
|
790
|
-
|
|
791
|
-
Same normalized actions as `get_pixel_actions`, each additionally carrying
|
|
792
|
-
`"frame"`: the 1-based keyframe the model chose to act on, or None if it did
|
|
793
|
-
not name a usable one.
|
|
794
|
-
|
|
795
|
-
The frame is not decoration. On these puzzles the target is only there part
|
|
796
|
-
of the time, so the coordinates are only correct while the widget looks like
|
|
797
|
-
the frame they were read off. The page driver holds the mouse until it does.
|
|
798
|
-
An action with `frame=None` is still returned — the caller decides whether to
|
|
799
|
-
click blind or give up, and dropping it here would look identical to "the
|
|
800
|
-
model had nothing to say".
|
|
801
|
-
"""
|
|
802
442
|
if not keyframe_paths:
|
|
803
443
|
return []
|
|
804
|
-
# Per model, not per client: generation 1 has no animated prompt at all,
|
|
805
|
-
# so this raises a clear error there instead of sending a v1 model a
|
|
806
|
-
# keyframe request it was never trained to read.
|
|
807
444
|
prompt = self.prompts.video_prompt(len(keyframe_paths))
|
|
808
445
|
raw = self._chat_with_images(prompt, list(keyframe_paths), max_tokens=512,
|
|
809
|
-
family=
|
|
446
|
+
family=PromptFamily.VIDEO)
|
|
810
447
|
data = self._parse_json(raw)
|
|
811
448
|
actions = self._normalize_pixel(data)
|
|
812
449
|
frame = self._normalize_frame(data, len(keyframe_paths))
|
|
@@ -817,14 +454,7 @@ class ActionPlanner:
|
|
|
817
454
|
|
|
818
455
|
@staticmethod
|
|
819
456
|
def _normalize_frame(data: Any, n_keyframes: int) -> Optional[int]:
|
|
820
|
-
"""
|
|
821
|
-
|
|
822
|
-
Out-of-range is treated as "did not choose" rather than clamped. Clamping a
|
|
823
|
-
7 to 6 would invent an intent the model never had, and the driver would then
|
|
824
|
-
wait for a picture the answer was not about — a slow failure that looks like
|
|
825
|
-
a flaky page. Tolerates the nested shape the model drifts into
|
|
826
|
-
({"action": {"frame": n}}), the same way the click/drag parsing does.
|
|
827
|
-
"""
|
|
457
|
+
"""Out of range is None, not clamped: clamping a 7 to 6 would invent an intent."""
|
|
828
458
|
if not isinstance(data, dict):
|
|
829
459
|
return None
|
|
830
460
|
raw = data.get("frame")
|
|
@@ -838,22 +468,13 @@ class ActionPlanner:
|
|
|
838
468
|
|
|
839
469
|
@staticmethod
|
|
840
470
|
def _normalize_pixel(data: Any) -> List[Dict[str, Any]]:
|
|
841
|
-
"""
|
|
842
|
-
|
|
843
|
-
Tolerant of the two trained shapes plus a few near-misses:
|
|
844
|
-
click: {"action": {"action": "click", "points": [[x, y], ...]}}
|
|
845
|
-
or top-level {"points": [...]} / {"action": {"points": [...]}}
|
|
846
|
-
drag: {"output": [{"Action": "simulate_drag",
|
|
847
|
-
"SourcePosition": {x, y},
|
|
848
|
-
"EstimatedPosition": {x, y}}, ...]}
|
|
849
|
-
"""
|
|
471
|
+
"""Every answer shape the model has been seen to produce, oldest last; `kind` is a PixelAnswerKind."""
|
|
850
472
|
def norm_xy(x: Any, y: Any) -> Optional[tuple]:
|
|
851
473
|
try:
|
|
852
474
|
fx, fy = float(x) / 1000.0, float(y) / 1000.0
|
|
853
475
|
except (TypeError, ValueError):
|
|
854
476
|
return None
|
|
855
477
|
if not (0.0 <= fx <= 1.0 and 0.0 <= fy <= 1.0):
|
|
856
|
-
# Some outputs use 0–1 already; accept those too.
|
|
857
478
|
if 0.0 <= float(x) <= 1.0 and 0.0 <= float(y) <= 1.0:
|
|
858
479
|
fx, fy = float(x), float(y)
|
|
859
480
|
else:
|
|
@@ -861,10 +482,6 @@ class ActionPlanner:
|
|
|
861
482
|
return (fx, fy)
|
|
862
483
|
|
|
863
484
|
def flat_numbers(v: Any) -> List[float]:
|
|
864
|
-
"""Every number anywhere inside v, in order. Handles nested lists,
|
|
865
|
-
{x,y} dicts, bare numbers, and numeric strings like "277, 728" (the
|
|
866
|
-
model sometimes emits coordinates as strings, occasionally even split
|
|
867
|
-
across separate array elements)."""
|
|
868
485
|
nums: List[float] = []
|
|
869
486
|
if isinstance(v, bool):
|
|
870
487
|
return nums
|
|
@@ -882,10 +499,6 @@ class ActionPlanner:
|
|
|
882
499
|
return nums
|
|
883
500
|
|
|
884
501
|
def coordish(v: Any) -> bool:
|
|
885
|
-
"""True when v is a non-empty list of COORDINATES (numbers, [x,y]
|
|
886
|
-
pairs, {x,y} dicts, or numeric strings) — not text labels like
|
|
887
|
-
"dog". Used to decide whether a "click"/"coordinates" array carries
|
|
888
|
-
points we should salvage."""
|
|
889
502
|
if not isinstance(v, (list, tuple)) or not v:
|
|
890
503
|
return False
|
|
891
504
|
for e in v:
|
|
@@ -902,29 +515,15 @@ class ActionPlanner:
|
|
|
902
515
|
if not isinstance(data, dict):
|
|
903
516
|
return out
|
|
904
517
|
|
|
905
|
-
# ---- type (CANONICAL — the schema TEXT_INSTRUCTION asks for on the
|
|
906
|
-
# distorted-text captchas: {"action": "type", "text": "<the code>"}).
|
|
907
|
-
# First, because it is the one answer family with NO coordinate in it:
|
|
908
|
-
# every branch below keys off a number, so a typed answer that fell
|
|
909
|
-
# through reached the end and normalized to nothing.
|
|
910
|
-
# The code is passed through VERBATIM — case and spacing are part of what
|
|
911
|
-
# the model read off the image, and "tidying" them submits a different
|
|
912
|
-
# answer than the one it gave.
|
|
913
518
|
act = data.get("action")
|
|
914
|
-
if isinstance(act, dict) and act.get("action") ==
|
|
915
|
-
act = act.get("action")
|
|
916
|
-
if act ==
|
|
519
|
+
if isinstance(act, dict) and act.get("action") == ActionKind.TYPE:
|
|
520
|
+
act = act.get("action")
|
|
521
|
+
if act == ActionKind.TYPE or (isinstance(data.get("text"), str) and "drags" not in data):
|
|
917
522
|
container = data if isinstance(data.get("text"), str) else data.get("action")
|
|
918
523
|
text = container.get("text") if isinstance(container, dict) else None
|
|
919
524
|
if isinstance(text, str) and text:
|
|
920
|
-
return [{"kind":
|
|
921
|
-
|
|
922
|
-
# ---- drag (CANONICAL — the schema PIXEL_ACTION_PROMPT asks for and the
|
|
923
|
-
# LoRA is trained on, see instructions.py::ACTION_INSTRUCTION):
|
|
924
|
-
# {"action":"drag","drags":[{"source","from":[x,y],"destination","to":[x,y]}]}
|
|
925
|
-
# Note "action" here is the STRING "drag", not a dict, so the click path
|
|
926
|
-
# below never sees it. The legacy/simulate_drag branches that follow stay
|
|
927
|
-
# as fallbacks for older adapters.
|
|
525
|
+
return [{"kind": PixelAnswerKind.TYPE, "text": text}]
|
|
526
|
+
|
|
928
527
|
content_drags = data.get("drags")
|
|
929
528
|
if content_drags is None and isinstance(data.get("action"), dict):
|
|
930
529
|
content_drags = data["action"].get("drags")
|
|
@@ -937,27 +536,19 @@ class ActionPlanner:
|
|
|
937
536
|
snums = flat_numbers(d.get("from"))
|
|
938
537
|
dnums = flat_numbers(d.get("to"))
|
|
939
538
|
if len(dnums) < 2:
|
|
940
|
-
continue
|
|
539
|
+
continue
|
|
941
540
|
dst = norm_xy(dnums[0], dnums[1])
|
|
942
541
|
if not dst:
|
|
943
542
|
continue
|
|
944
543
|
if len(snums) >= 2:
|
|
945
544
|
src = norm_xy(snums[0], snums[1])
|
|
946
545
|
if src:
|
|
947
|
-
out.append({"kind":
|
|
546
|
+
out.append({"kind": PixelAnswerKind.DRAG, "src": src, "dst": dst})
|
|
948
547
|
else:
|
|
949
|
-
|
|
950
|
-
# PUZZLE PIECE SLIDER PUZZLES" clause tells the model to
|
|
951
|
-
# leave the source empty precisely because the piece is not
|
|
952
|
-
# what you pick up: the handle is, somewhere else entirely,
|
|
953
|
-
# and how far it has to travel is not knowable from the
|
|
954
|
-
# picture. Requiring both ends here dropped the one answer
|
|
955
|
-
# shape the prompt asks for on every slide puzzle.
|
|
956
|
-
out.append({"kind": "slide", "dst": dst})
|
|
548
|
+
out.append({"kind": PixelAnswerKind.SLIDE, "dst": dst})
|
|
957
549
|
if out:
|
|
958
550
|
return out
|
|
959
551
|
|
|
960
|
-
# ---- drag: {"output": [ {simulate_drag ...}, ... ]} ----
|
|
961
552
|
drags = data.get("output")
|
|
962
553
|
if isinstance(drags, list) and drags:
|
|
963
554
|
for d in drags:
|
|
@@ -968,34 +559,17 @@ class ActionPlanner:
|
|
|
968
559
|
src = norm_xy(sp.get("x"), sp.get("y")) if isinstance(sp, dict) else None
|
|
969
560
|
dst = norm_xy(ep.get("x"), ep.get("y")) if isinstance(ep, dict) else None
|
|
970
561
|
if src and dst:
|
|
971
|
-
out.append({"kind":
|
|
562
|
+
out.append({"kind": PixelAnswerKind.DRAG, "src": src, "dst": dst})
|
|
972
563
|
if out:
|
|
973
564
|
return out
|
|
974
565
|
|
|
975
|
-
# ---- drag: {"action": {"simulate_drag": [{source_position,
|
|
976
|
-
# destination_position}]}} ----
|
|
977
|
-
# The full-puzzle LoRA actually emits drags in this snake_case shape
|
|
978
|
-
# (not the prompt's {"output":[{"Action":"simulate_drag",...}]}), and
|
|
979
|
-
# packs each coordinate as {"x": [x, y]} — flat_numbers() pulls the pair
|
|
980
|
-
# out of that. Without this branch a correctly-solved drag (e.g. hCaptcha
|
|
981
|
-
# "drag ONE character to the matching character") is dropped as
|
|
982
|
-
# "unsupported".
|
|
983
566
|
sd = data.get("simulate_drag")
|
|
984
567
|
if sd is None and isinstance(action := data.get("action"), dict):
|
|
985
568
|
sd = action.get("simulate_drag")
|
|
986
|
-
# The model emits simulate_drag as EITHER a list of drags OR a single
|
|
987
|
-
# drag object; normalize the single-object form to a one-element list so
|
|
988
|
-
# both parse (the object form was silently dropped as "unsupported").
|
|
989
569
|
if isinstance(sd, dict):
|
|
990
570
|
sd = [sd]
|
|
991
571
|
if isinstance(sd, list) and sd:
|
|
992
572
|
def _drag_coords(obj: Any, *roles: str) -> List[float]:
|
|
993
|
-
# Pull the source/destination coordinate pair regardless of key
|
|
994
|
-
# casing or separators — the LoRA freely varies between
|
|
995
|
-
# source_position / sourcePosition / SourcePosition and
|
|
996
|
-
# destination_position / destinationPosition / EstimatedPosition.
|
|
997
|
-
# The coord value may be {"x": [x, y]}, [x, y], {x, y}, or a
|
|
998
|
-
# string; flat_numbers() handles all of those.
|
|
999
573
|
if not isinstance(obj, dict):
|
|
1000
574
|
return []
|
|
1001
575
|
for k, v in obj.items():
|
|
@@ -1016,15 +590,11 @@ class ActionPlanner:
|
|
|
1016
590
|
src = norm_xy(snums[0], snums[1])
|
|
1017
591
|
dst = norm_xy(dnums[0], dnums[1])
|
|
1018
592
|
if src and dst:
|
|
1019
|
-
out.append({"kind":
|
|
593
|
+
out.append({"kind": PixelAnswerKind.DRAG, "src": src, "dst": dst})
|
|
1020
594
|
if out:
|
|
1021
595
|
return out
|
|
1022
596
|
|
|
1023
|
-
# ---- click: {"action": {"action":"click","points":[...]}} ----
|
|
1024
597
|
action = data.get("action")
|
|
1025
|
-
# The model sometimes over-wraps: {"action": {"action": {"action":
|
|
1026
|
-
# "click", "points":[...]}}}. Peel nested "action" dicts until we reach
|
|
1027
|
-
# the one carrying points/coords (bounded so we never loop forever).
|
|
1028
598
|
for _ in range(4):
|
|
1029
599
|
if (
|
|
1030
600
|
isinstance(action, dict)
|
|
@@ -1038,20 +608,13 @@ class ActionPlanner:
|
|
|
1038
608
|
points = None
|
|
1039
609
|
if isinstance(action, dict):
|
|
1040
610
|
points = action.get("points")
|
|
1041
|
-
|
|
1042
|
-
if points is None and action.get("action") == "drag":
|
|
611
|
+
if points is None and action.get("action") == ActionKind.DRAG:
|
|
1043
612
|
src = norm_xy(*(action.get("source") or (None, None)))
|
|
1044
613
|
dst = norm_xy(*(action.get("target") or (None, None)))
|
|
1045
614
|
if src and dst:
|
|
1046
|
-
return [{"kind":
|
|
615
|
+
return [{"kind": PixelAnswerKind.DRAG, "src": src, "dst": dst}]
|
|
1047
616
|
if points is None:
|
|
1048
617
|
points = data.get("points")
|
|
1049
|
-
# Salvage: some responses carry the click coordinates under "click" (or
|
|
1050
|
-
# "coordinates") instead of "points", as [x,y] pairs, {x,y} dicts, or
|
|
1051
|
-
# "x, y" strings (sometimes split across elements), with no "points" key
|
|
1052
|
-
# at all. Pull every number out of a coordinate-like value and pair them
|
|
1053
|
-
# into points so a well-intentioned answer isn't dropped as "unsupported".
|
|
1054
|
-
# Skipped when the value is text labels ("dog", "duck") rather than coords.
|
|
1055
618
|
if not (isinstance(points, list) and points):
|
|
1056
619
|
for container in (action if isinstance(action, dict) else None, data):
|
|
1057
620
|
if not isinstance(container, dict):
|
|
@@ -1078,5 +641,5 @@ class ActionPlanner:
|
|
|
1078
641
|
if xy:
|
|
1079
642
|
pts.append(xy)
|
|
1080
643
|
if pts:
|
|
1081
|
-
out.append({"kind":
|
|
644
|
+
out.append({"kind": PixelAnswerKind.CLICK, "points": pts})
|
|
1082
645
|
return out
|