captchakraken 2.9.0 → 3.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (77) hide show
  1. package/AGENTS.md +112 -0
  2. package/NOTICE +36 -0
  3. package/dist/cli-invocation.d.ts +7 -36
  4. package/dist/cli-invocation.js +8 -49
  5. package/dist/errors.d.ts +7 -27
  6. package/dist/errors.js +6 -25
  7. package/dist/humanize.d.ts +56 -240
  8. package/dist/humanize.js +159 -415
  9. package/dist/index.d.ts +5 -129
  10. package/dist/index.js +4 -98
  11. package/dist/kinds.d.ts +135 -0
  12. package/dist/kinds.js +70 -0
  13. package/dist/limits.d.ts +0 -48
  14. package/dist/limits.js +2 -48
  15. package/dist/model-name.d.ts +4 -19
  16. package/dist/model-name.js +32 -40
  17. package/dist/playwright-types.d.ts +25 -95
  18. package/dist/playwright-types.js +0 -23
  19. package/dist/puppeteer-adapter.d.ts +4 -43
  20. package/dist/puppeteer-adapter.js +22 -58
  21. package/dist/python-command.d.ts +0 -41
  22. package/dist/python-command.js +0 -37
  23. package/dist/selectors.d.ts +54 -0
  24. package/dist/selectors.js +117 -0
  25. package/dist/slide-geometry.d.ts +2 -29
  26. package/dist/slide-geometry.js +18 -44
  27. package/dist/solver.d.ts +131 -708
  28. package/dist/solver.js +1319 -3382
  29. package/dist/timing.d.ts +7 -40
  30. package/dist/timing.js +4 -36
  31. package/dist/token-usage.d.ts +1 -7
  32. package/dist/token-usage.js +10 -95
  33. package/dist/types.d.ts +77 -493
  34. package/dist/watcher.d.ts +5 -78
  35. package/dist/watcher.js +0 -56
  36. package/package.json +11 -4
  37. package/python/AGENTS.md +122 -0
  38. package/python/NOTICE +36 -0
  39. package/python/README.md +1 -2
  40. package/python/examples/README.md +10 -0
  41. package/python/examples/_harness.py +4 -73
  42. package/python/examples/demoHcaptcha.py +0 -13
  43. package/python/examples/demoRecaptcha.py +0 -10
  44. package/python/examples/watch_playwright.py +1 -26
  45. package/python/examples/with_local_gguf.py +28 -0
  46. package/python/examples/with_playwright.py +0 -19
  47. package/python/pyproject.toml +68 -19
  48. package/python/src/captchakraken/__init__.py +37 -45
  49. package/python/src/captchakraken/action_types.py +14 -36
  50. package/python/src/captchakraken/cli.py +262 -883
  51. package/python/src/captchakraken/config.py +60 -146
  52. package/python/src/captchakraken/errors.py +17 -95
  53. package/python/src/captchakraken/humanize.py +174 -597
  54. package/python/src/captchakraken/image_processor.py +16 -235
  55. package/python/src/captchakraken/keyframes.py +46 -484
  56. package/python/src/captchakraken/kinds.py +152 -0
  57. package/python/src/captchakraken/models.json +36 -4
  58. package/python/src/captchakraken/overlay.py +65 -498
  59. package/python/src/captchakraken/page_solver.py +1282 -3101
  60. package/python/src/captchakraken/planner.py +65 -487
  61. package/python/src/captchakraken/prompts.py +45 -261
  62. package/python/src/captchakraken/selectors.py +184 -0
  63. package/python/src/captchakraken/server_manager.py +7 -58
  64. package/python/src/captchakraken/solver.py +128 -522
  65. package/python/src/captchakraken/timing.py +20 -74
  66. package/python/src/captchakraken/tool_calls/board_painted.py +47 -0
  67. package/python/src/captchakraken/tool_calls/find_checkbox.py +20 -30
  68. package/python/src/captchakraken/tool_calls/find_grid.py +186 -1339
  69. package/python/src/captchakraken/tool_calls/track_piece.py +81 -57
  70. package/python/src/captchakraken/updater.py +7 -76
  71. package/python/src/captchakraken/watcher.py +12 -101
  72. package/scripts/copy-python.mjs +10 -10
  73. package/scripts/setup-python.js +1 -21
  74. package/dist/trajectory.d.ts +0 -58
  75. package/dist/trajectory.js +0 -266
  76. package/python/src/captchakraken/tool_calls/move_indicator.py +0 -431
  77. package/python/src/captchakraken/trajectory.py +0 -284
@@ -1,407 +1,173 @@
1
- """
2
- CaptchaSolver (v2) — vLLM-backed.
3
-
4
- Flow:
5
- 1. find_grid → if a grid is detected, draw the numbered overlay, ask the
6
- `captcha` LoRA which cells to click, return ClickActions with
7
- per-tile bounding boxes.
8
- 2. find_checkbox on small images → if a lone checkbox is detected, return a
9
- ClickAction targeting it directly.
10
- 3. Otherwise (click / drag / other still-image puzzle) → route the image to
11
- the full-puzzle pixel/action path (planner.get_pixel_actions), which the
12
- LoRA is trained on. Only raise UnsupportedCaptchaError if the model returns
13
- nothing usable.
14
-
15
- ANIMATED challenges take a separate entry point, `solve_keyframes`. They used to be
16
- detected and skipped: the settle monitor called them "never settles" and the solve
17
- was abandoned. Now the driver records the widget, `keyframes.py` slices the
18
- recording into the few stills that carry the answer, and those go to the model as
19
- one multi-image request. The answer comes back with a `frame`, and the driver waits
20
- for the page to look like that frame before it clicks — because on these puzzles
21
- the coordinates are only correct while the target is actually on screen.
22
-
23
- v1 had a SAM3-backed tool-using planner with detect/segment/drag-refine; it
24
- lives on the `v1-old-architecture` branch.
25
- """
1
+ """Image in, actions out: grid detection plus the model, for one still or one keyframe set."""
26
2
 
3
+ import math
27
4
  import os
28
- import shutil
29
5
  import sys
30
6
  import tempfile
31
- from datetime import datetime
32
7
  from pathlib import Path
33
8
  from typing import Any, List, Optional, Sequence, Tuple, Union
34
9
 
35
10
  from PIL import Image
36
11
 
37
- from .action_types import (
38
- CaptchaAction,
39
- ClickAction,
40
- DoneAction,
41
- DragAction,
42
- TypeAction,
43
- WaitAction,
44
- )
45
- from .image_processor import ImageProcessor
12
+ from .action_types import CaptchaAction, ClickAction, DoneAction, DragAction, TypeAction, WaitAction
13
+ from .kinds import ActionKind, PixelAnswerKind, PromptFamily, RetryMode, Vendor
46
14
  from .planner import ActionPlanner
47
- from .timing import timed
48
15
  from .tool_calls.find_checkbox import find_checkbox
49
- from .tool_calls.find_grid import (
50
- detect_selected_cells, find_grid, get_numbered_grid_overlay,
51
- )
16
+ from .tool_calls.find_grid import detect_selected_cells, find_grid, get_numbered_grid_overlay
52
17
 
53
18
  DEBUG = os.getenv("CAPTCHA_DEBUG", "0") == "1"
54
-
55
- # Half-size (0–1 fraction) of the click/drag target box built around each point
56
- # the model returns. The TS solver clicks the box center, so this only sets how
57
- # much positional slack executeClick has; ~1.2% ≈ ±6px on a 512px challenge.
19
+ # ~1.2% ≈ ±6px on a 512px challenge.
58
20
  _PIXEL_BOX_HALF = float(os.getenv("CAPTCHA_PIXEL_BOX_HALF", "0.012"))
59
21
 
60
-
61
- class UnsupportedCaptchaError(Exception):
62
- """Raised when the captcha is neither a supported grid nor a checkbox."""
22
+ # The only cell counts a trained grid puzzle has, and which of them each named vendor ships. UNKNOWN stays
23
+ # permissive in both directions: the driver reports it for GeeTest and Prosopo, and the offline grader defaults
24
+ # to it, so narrowing it would silently stop scoring reCAPTCHA 4x4 in evaluation.
25
+ GRID_CELL_SHAPES = {9: (3, 3), 16: (4, 4)}
26
+ VENDOR_GRID_CELLS: dict[Vendor, frozenset[int]] = {Vendor.HCAPTCHA: frozenset({9}), Vendor.RECAPTCHA: frozenset({9, 16})}
27
+ # Every true grid measures 0.000 cells out of true; the nearest false lattice measured 0.128.
28
+ _GRID_REGULARITY_TOL = 0.02
63
29
 
64
30
 
65
- class DebugManager:
66
- """Writes per-run artifacts under `latestDebugRun/` when CAPTCHA_DEBUG=1."""
31
+ def _debug(message: str) -> None:
32
+ if DEBUG:
33
+ print(f"[Solver] {message}", file=sys.stderr)
67
34
 
68
- def __init__(self, debug_enabled: bool):
69
- self.enabled = debug_enabled
70
- self.base_dir = Path("latestDebugRun").resolve()
71
- self.log_file = self.base_dir / "log.txt"
72
- if self.enabled:
73
- self._setup_dir()
74
35
 
75
- def _setup_dir(self):
76
- if self.base_dir.exists():
77
- try:
78
- shutil.rmtree(self.base_dir)
79
- except Exception as e:
80
- print(f"[DebugManager] Warning: Could not clear debug dir: {e}", file=sys.stderr)
81
- try:
82
- self.base_dir.mkdir(parents=True, exist_ok=True)
83
- with open(self.log_file, "w") as f:
84
- f.write(f"Debug Run Started: {datetime.now()}\n")
85
- except Exception as e:
86
- print(f"[DebugManager] Error creating debug dir: {e}", file=sys.stderr)
36
+ class UnsupportedCaptchaError(Exception):
37
+ pass
87
38
 
88
- def log(self, message: str):
89
- if self.enabled:
90
- print(f"[DEBUG] {message}", file=sys.stderr)
91
- try:
92
- with open(self.log_file, "a") as f:
93
- f.write(f"[{datetime.now().strftime('%H:%M:%S')}] {message}\n")
94
- except Exception:
95
- pass
96
- elif DEBUG:
97
- print(f"[Solver] {message}", file=sys.stderr)
98
-
99
- def save_image(self, image_path: str, name: str) -> Optional[str]:
100
- if not self.enabled:
101
- return None
102
- if not self.base_dir.exists():
103
- self._setup_dir()
104
- target = self.base_dir / name
105
- try:
106
- shutil.copy2(image_path, target)
107
- self.log(f"Saved image: {name}")
108
- return str(target)
109
- except Exception as e:
110
- self.log(f"Failed to save image {name}: {e}")
111
- return None
112
39
 
40
+ def _lattice_irregularity(grid_boxes) -> Optional[float]:
41
+ """How far the cells stray from a regular lattice, in cells; None when not a square count."""
42
+ boxes = [tuple(b) for b in grid_boxes]
43
+ n = len(boxes)
44
+ k = int(round(math.sqrt(n))) if n else 0
45
+ if k < 2 or k * k != n:
46
+ return None
47
+ widths = [b[2] - b[0] for b in boxes]
48
+ heights = [b[3] - b[1] for b in boxes]
49
+ mid_w, mid_h = sorted(widths)[n // 2], sorted(heights)[n // 2]
50
+ if mid_w <= 0 or mid_h <= 0:
51
+ return None
52
+ rows = [boxes[i * k:(i + 1) * k] for i in range(k)]
53
+ cols = [[boxes[r * k + c] for r in range(k)] for c in range(k)]
54
+ row_skew = max(max(b[1] for b in r) - min(b[1] for b in r) for r in rows) / mid_h
55
+ col_skew = max(max(b[0] for b in c) - min(b[0] for b in c) for c in cols) / mid_w
56
+ size_spread = max((max(widths) - min(widths)) / mid_w, (max(heights) - min(heights)) / mid_h)
57
+ return max(row_skew, col_skew, size_spread)
113
58
 
114
- #: The only cell counts a registered grid puzzle has. `find_grid` is a lattice
115
- #: detector, not a catalogue — it allows rectangles and any dimension from
116
- #: MIN_GRID_DIM up — so it will happily report 12 or 20 cells on a click puzzle's
117
- #: header/footer bands. Anything not in here is a false positive by construction,
118
- #: and it is also a board the model has never seen: the training pipeline's own
119
- #: `grid_overlay.detect_cells` refuses every count but these two.
120
- GRID_CELL_SHAPES = {9: (3, 3), 16: (4, 4)}
121
59
 
122
- #: Cell counts each vendor actually ships, for the vendors the driver can name.
123
- #: Only reCAPTCHA has a 4x4; hCaptcha's single grid puzzle is 3x3, so 16 cells on
124
- #: an hCaptcha board is a contradiction rather than a close call. Measured on real
125
- #: captures, that one line removes four of the five non-grid boards that clear
126
- #: both find_grid and _is_real_grid today.
127
- #:
128
- #: A vendor NOT in this table — `unknown`, or anything new — gets every shape.
129
- #: `unknown` is what the driver reports for GeeTest and Prosopo, both of which
130
- #: ship real 3x3 grids, and it is what the one-shot image API defaults to, which
131
- #: is the path the offline grader uses to score reCAPTCHA 4x4. Narrowing it would
132
- #: stop scoring a whole puzzle type while looking like a tightening.
133
- VENDOR_GRID_CELLS = {
134
- "hcaptcha": frozenset({9}),
135
- "recaptcha": frozenset({9, 16}),
136
- }
137
-
138
-
139
- def _grid_dims(n_cells, puzzle_source="unknown"):
140
- """(rows, cols) if this detection could be a grid we ship, else None.
141
-
142
- None means "do not solve this as a grid" — the caller falls through to the
143
- click/drag path, which is what the board actually is. It is deliberately not
144
- an exception: a false positive here is an ordinary event, not an error.
145
- """
60
+ def _grid_dims(n_cells: int, puzzle_source: Vendor = Vendor.UNKNOWN):
146
61
  shape = GRID_CELL_SHAPES.get(n_cells)
147
- if shape is None:
148
- return None
149
- allowed = VENDOR_GRID_CELLS.get(puzzle_source)
150
- if allowed is not None and n_cells not in allowed:
151
- return None
152
- return shape
62
+ allowed = VENDOR_GRID_CELLS.get(Vendor(puzzle_source))
63
+ return None if shape is None or (allowed is not None and n_cells not in allowed) else shape
153
64
 
154
65
 
155
66
  class CaptchaSolver:
156
- """v2 solver: OpenCV grid detection + vLLM `captcha` LoRA."""
157
-
158
- def __init__(
159
- self,
160
- model: Optional[str] = None,
161
- provider: str = "captchaKrakenApi",
162
- api_key: Optional[str] = None,
163
- expert: Optional[str] = None,
164
- ):
165
- self.debug = DebugManager(DEBUG)
166
- # `provider` kept for argv compatibility with the v1 CLI signature.
167
- if provider not in {"captchaKrakenApi"}:
168
- self.debug.log(f"Provider {provider!r} ignored; v2 only supports captchaKrakenApi.")
169
- self.planner = ActionPlanner(
170
- model=model, api_key=api_key, debug_callback=self.debug.log,
171
- expert=expert,
172
- )
173
- self.image_processor = ImageProcessor(None, self.planner, self.debug)
67
+ def __init__(self, model: Optional[str] = None, provider: str = "captchaKrakenApi",
68
+ api_key: Optional[str] = None, expert: Optional[PromptFamily] = None):
69
+ self.planner = ActionPlanner(model=model, api_key=api_key, expert=expert)
174
70
  self._image_size: Optional[Tuple[int, int]] = None
175
71
  self._temp_files: List[str] = []
176
72
 
177
73
  def __del__(self):
178
74
  for f in self._temp_files:
179
- if os.path.exists(f):
180
- try:
181
- os.unlink(f)
182
- except Exception:
183
- pass
184
-
185
- # ------------------------------------------------------------------
186
- # Public entry point
187
- # ------------------------------------------------------------------
188
- def solve(
189
- self,
190
- media_path: str,
191
- instruction: str = "",
192
- puzzle_source: str = "unknown",
193
- retry_mode: Optional[str] = None,
194
- text_mode: bool = False,
195
- ) -> Union[CaptchaAction, List[CaptchaAction]]:
75
+ try:
76
+ os.unlink(f)
77
+ except OSError:
78
+ pass
79
+
80
+ def solve(self, media_path: str, instruction: str = "", puzzle_source: Vendor = Vendor.UNKNOWN,
81
+ retry_mode: Optional[RetryMode] = None, text_mode: bool = False) -> Union[CaptchaAction, List[CaptchaAction]]:
82
+ # String defaults stay: the public signature is pinned by contract.json. A bad value raises here.
83
+ puzzle_source, retry_mode = Vendor(puzzle_source), None if retry_mode is None else RetryMode(retry_mode)
196
84
  media_path = str(Path(media_path).resolve())
197
85
  if not os.path.exists(media_path):
198
86
  raise FileNotFoundError(f"Media not found: {media_path}")
199
-
200
- cv_image_path = self._materialize_image(media_path)
201
- self.debug.save_image(cv_image_path, "00_base_image.png")
202
- assert self._image_size is not None
87
+ image_path = self._materialize_image(media_path)
203
88
  img_w, img_h = self._image_size
204
89
 
205
90
  if text_mode:
206
- # The driver saw a text box in the widget, so the answer is a string
207
- # and there is nothing on the picture to find. Grid and checkbox
208
- # detection are skipped rather than merely ignored: BotDetect's
209
- # boxed, evenly-spaced glyphs are exactly the lattice find_grid
210
- # looks for, and a false grid here would answer a typing puzzle
211
- # with a list of cell numbers.
91
+ # A typed captcha's boxed glyphs are exactly the lattice find_grid looks for, so skip it.
212
92
  try:
213
- actions = self._solve_pixel(cv_image_path, text_mode=True)
93
+ actions = self._solve_pixel(image_path, text_mode=True)
214
94
  except ValueError as exc:
215
- # The served model predates the distorted-text family, so there
216
- # is no prompt to send it. Surfaced as UNSUPPORTED rather than
217
- # crashing: it is the same class of outcome as "this LoRA only
218
- # does grids", the driver already knows how to report it, and
219
- # the message names the real fix (a generation-2 model) instead
220
- # of a stack trace in prompts.py.
221
95
  raise UnsupportedCaptchaError(str(exc)) from exc
222
96
  if actions:
223
97
  return actions
224
98
  raise UnsupportedCaptchaError("Could not read the text captcha")
225
99
 
226
- with timed("solver.find_grid"):
227
- grid_boxes = find_grid(cv_image_path)
228
- # The SHAPE gate runs before the content gate: a lattice no vendor ships
229
- # is not a grid whatever its cells look like, and saying so here keeps
230
- # `_solve_grid` from ever being handed dimensions it would have to invent.
100
+ # The animated path never runs find_grid: it false-positives on the header and footer bands of hCaptcha's click puzzles.
101
+ grid_boxes = find_grid(image_path)
231
102
  dims = _grid_dims(len(grid_boxes), puzzle_source) if grid_boxes else None
232
- if grid_boxes and dims and self._is_real_grid(cv_image_path, grid_boxes):
233
- self.debug.log(f"Detected grid with {len(grid_boxes)} cells")
234
- return self._solve_grid(cv_image_path, grid_boxes, dims[0], dims[1],
235
- retry_mode=retry_mode)
236
- elif grid_boxes and not dims:
237
- # e.g. a 16-cell lattice on an hCaptcha board (it ships no 4x4), or a
238
- # 12-cell one on anything. Fall through to the click path, which is
239
- # what this puzzle is.
240
- self.debug.log(
241
- f"find_grid returned {len(grid_boxes)} cells, which is not a shape "
242
- f"{puzzle_source} ships — not solving as a grid."
243
- )
244
- elif grid_boxes:
245
- # find_grid latched onto e.g. an hCaptcha click-puzzle's
246
- # header/footer bands. Reject — only true grids are supported.
247
- self.debug.log(
248
- f"find_grid returned {len(grid_boxes)} cells but failed the "
249
- "real-grid sanity check."
250
- )
103
+ if grid_boxes and dims and self._is_real_grid(image_path, grid_boxes):
104
+ return self._solve_grid(image_path, grid_boxes, dims[0], dims[1], retry_mode=retry_mode)
105
+ if grid_boxes:
106
+ _debug(f"find_grid returned {len(grid_boxes)} cells; not solving as a grid")
251
107
 
252
108
  if img_h < 400:
253
- with timed("solver.find_checkbox"):
254
- checkbox = find_checkbox(cv_image_path)
109
+ checkbox = find_checkbox(image_path)
255
110
  if checkbox:
256
- self.debug.log(f"Detected checkbox at {checkbox}")
257
111
  x, y, w, h = checkbox
258
- return ClickAction(
259
- action="click",
260
- target_bounding_boxes=[
261
- [x / img_w, y / img_h, (x + w) / img_w, (y + h) / img_h]
262
- ],
263
- )
264
-
265
- # Not a grid or checkbox → a click/drag/pixel puzzle. The full-puzzle
266
- # LoRA is trained on all of these, so route the image to the pixel
267
- # action path rather than bailing. (Animated challenges come in through
268
- # `solve_keyframes` instead, so anything here is a still worth attempting.)
269
- actions = self._solve_pixel(cv_image_path)
112
+ return ClickAction(action=ActionKind.CLICK,
113
+ target_bounding_boxes=[[x / img_w, y / img_h, (x + w) / img_w, (y + h) / img_h]])
114
+
115
+ actions = self._solve_pixel(image_path)
270
116
  if actions:
271
117
  return actions
272
-
273
- # The model returned nothing usable (blank/unrendered frame or a truly
274
- # un-actionable image). Surface as unsupported so the caller fails fast
275
- # instead of clicking nothing.
276
118
  raise UnsupportedCaptchaError("Cannot solve this kind of captcha")
277
119
 
278
- def solve_keyframes(
279
- self, keyframe_paths: Sequence[str]
280
- ) -> List[Union[ClickAction, DragAction]]:
281
- """Solve an ANIMATED challenge from the keyframes of a recording.
282
-
283
- `keyframe_paths` must be in model order (frame 1 first) — the numbers the
284
- answer refers to are positional, so a reordered list silently remaps the
285
- answer onto the wrong picture. `keyframes.materialize_keyframes` and
286
- `read_keyframe_paths` both return them correctly ordered; do not sort them
287
- yourself with a plain string sort past nine frames.
288
-
289
- Grid detection is deliberately NOT attempted. An animated challenge is never
290
- a tile grid (vendors do not animate those), and `find_grid` false-positives
291
- on the header/footer bands of hCaptcha's click puzzles — which is what these
292
- are. Every returned action carries `await_keyframe`, the still the driver
293
- must see on screen before it acts.
294
- """
120
+ def solve_keyframes(self, keyframe_paths: Sequence[str]) -> List[Union[ClickAction, DragAction]]:
121
+ """Solve an animated challenge from its keyframes, in model order (frame 1 first)."""
295
122
  paths = [str(p) for p in keyframe_paths]
296
123
  if not paths:
297
124
  raise UnsupportedCaptchaError("no keyframes to solve from")
298
125
  with Image.open(paths[0]) as im:
299
126
  self._image_size = im.size
300
- self.debug.save_image(paths[0], "00_keyframe_01.png")
301
- self.debug.log(f"solving animated challenge from {len(paths)} keyframes")
302
-
303
- actions = self._to_actions(
304
- self.planner.get_keyframe_actions(paths), keyframe_paths=paths
305
- )
127
+ actions = self._to_actions(self.planner.get_keyframe_actions(paths), keyframe_paths=paths)
306
128
  if actions:
307
129
  return actions
308
130
  raise UnsupportedCaptchaError("Cannot solve this animated captcha")
309
131
 
310
- def _solve_pixel(
311
- self, image_path: str, text_mode: bool = False
312
- ) -> List[Union[ClickAction, DragAction]]:
313
- """Turn the model's normalized 0–1 click/drag actions into ClickAction /
314
- DragAction bboxes. Each point becomes a small box centered on it (the TS
315
- solver clicks the box center)."""
132
+ def _solve_pixel(self, image_path: str, text_mode: bool = False) -> List[Union[ClickAction, DragAction]]:
316
133
  return self._to_actions(self.planner.get_pixel_actions(image_path, text_mode=text_mode))
317
134
 
318
- def _to_actions(
319
- self,
320
- raw_actions: List[dict],
321
- keyframe_paths: Optional[Sequence[str]] = None,
322
- ) -> List[Union[ClickAction, DragAction]]:
323
- """Shared conversion from the planner's normalized 0–1 actions to typed
324
- Click/Drag actions with small boxes around each point.
325
-
326
- When `keyframe_paths` is given, the chosen keyframe is attached so the
327
- driver knows which page state to wait for. A model answer with no usable
328
- frame is still converted — it just carries no keyframe, and the driver
329
- treats that as "act on the current frame". That is the honest fallback:
330
- refusing the action outright would turn a good coordinate into a failed
331
- solve over a missing integer.
332
- """
333
- R = _PIXEL_BOX_HALF
334
- clamp = lambda v: min(max(v, 0.0), 1.0)
135
+ def _to_actions(self, raw_actions: List[dict],
136
+ keyframe_paths: Optional[Sequence[str]] = None) -> List[Union[ClickAction, DragAction]]:
137
+ """Normalised 0-1 points become small boxes; a slide is a drag with no source."""
138
+ r = _PIXEL_BOX_HALF
335
139
 
336
140
  def box(cx: float, cy: float) -> List[float]:
337
- return [clamp(cx - R), clamp(cy - R), clamp(cx + R), clamp(cy + R)]
141
+ return [min(max(v, 0.0), 1.0) for v in (cx - r, cy - r, cx + r, cy + r)]
338
142
 
339
143
  def wait_for(a: dict) -> dict:
340
- if not keyframe_paths:
341
- return {}
144
+ # An out-of-range frame is dropped, not clamped: clamping a 7 to 6 would invent an intent.
342
145
  frame = a.get("frame")
343
- if not isinstance(frame, int) or not 1 <= frame <= len(keyframe_paths):
344
- self.debug.log(
345
- "animated answer named no usable keyframe; acting on the "
346
- "current frame without waiting"
347
- )
146
+ if not keyframe_paths or not isinstance(frame, int) or not 1 <= frame <= len(keyframe_paths):
348
147
  return {}
349
148
  return {"frame": frame, "await_keyframe": str(keyframe_paths[frame - 1])}
350
149
 
351
150
  out: List[Union[ClickAction, DragAction]] = []
352
151
  for a in raw_actions:
353
- if a.get("kind") == "click":
152
+ kind = PixelAnswerKind(a["kind"])
153
+ if kind == PixelAnswerKind.CLICK:
354
154
  boxes = [box(x, y) for (x, y) in a.get("points", [])]
355
155
  if boxes:
356
- out.append(
357
- ClickAction(action="click", target_bounding_boxes=boxes,
358
- **wait_for(a))
359
- )
360
- elif a.get("kind") == "drag":
361
- sx, sy = a["src"]
362
- dx, dy = a["dst"]
363
- out.append(
364
- DragAction(
365
- action="drag",
366
- source_bounding_box=box(sx, sy),
367
- target_bounding_box=box(dx, dy),
368
- **wait_for(a),
369
- )
370
- )
371
- elif a.get("kind") == "slide":
372
- # A puzzle-piece slider, carried as a drag with NO source. That
373
- # is not a lossy encoding of a normal drag — it is the whole
374
- # shape of the answer: the thing you grab (the handle) is not
375
- # the thing that has to arrive (the piece), and only the page
376
- # knows where the handle is. The driver reads a null source as
377
- # "find the slider and close the loop on the piece".
378
- dx, dy = a["dst"]
379
- out.append(
380
- DragAction(
381
- action="drag",
382
- source_bounding_box=None,
383
- target_bounding_box=box(dx, dy),
384
- **wait_for(a),
385
- )
386
- )
387
- elif a.get("kind") == "type":
388
- out.append(TypeAction(action="type", text=a["text"]))
156
+ out.append(ClickAction(action=ActionKind.CLICK, target_bounding_boxes=boxes, **wait_for(a)))
157
+ elif kind in (PixelAnswerKind.DRAG, PixelAnswerKind.SLIDE):
158
+ src = box(*a["src"]) if kind == PixelAnswerKind.DRAG else None
159
+ out.append(DragAction(action=ActionKind.DRAG, source_bounding_box=src,
160
+ target_bounding_box=box(*a["dst"]), **wait_for(a)))
161
+ elif kind == PixelAnswerKind.TYPE:
162
+ out.append(TypeAction(action=ActionKind.TYPE, text=a["text"]))
389
163
  return out
390
164
 
391
- # Back-compat alias. Kept pointing at `solve` (a single still), NOT at
392
- # `solve_keyframes`: callers of this name pass one media path and expect one
393
- # answer, and quietly reinterpreting that as "record and slice" would change
394
- # what an old integration does. New code calls `solve_keyframes` explicitly.
395
165
  def solveVideo(self, *args, **kwargs):
396
166
  return self.solve(*args, **kwargs)
397
167
 
398
- # ------------------------------------------------------------------
399
- # Internals
400
- # ------------------------------------------------------------------
401
168
  def _materialize_image(self, media_path: str) -> str:
402
- """Return a path to a static PNG (extracting first frame for videos)."""
403
- is_video = any(media_path.lower().endswith(ext) for ext in [".mp4", ".gif", ".avi", ".webm"])
404
- if is_video:
169
+ """A PNG path for `media_path`; a video contributes its first frame."""
170
+ if any(media_path.lower().endswith(ext) for ext in (".mp4", ".gif", ".avi", ".webm")):
405
171
  import cv2
406
172
 
407
173
  cap = cv2.VideoCapture(media_path)
@@ -414,232 +180,72 @@ class CaptchaSolver:
414
180
  cv2.imwrite(tf.name, frame)
415
181
  self._temp_files.append(tf.name)
416
182
  return tf.name
417
-
418
183
  with Image.open(media_path) as img:
419
184
  self._image_size = img.size
420
185
  return media_path
421
186
 
422
- def _is_real_grid(
423
- self,
424
- image_path: str,
425
- grid_boxes: List[Tuple[int, int, int, int]],
426
- ) -> bool:
427
- """Reject candidates that look like click-puzzle false positives.
428
-
429
- A real captcha grid has 9 or 16 photographic tiles. Each tile is
430
- independent imagery so the per-cell color stddev is *all* high.
431
-
432
- hCaptcha click puzzles look like a 3-row stack (header band /
433
- center image / footer band). find_grid sometimes interprets the
434
- band edges as 2 horizontal grid lines × 2 spurious vertical lines,
435
- producing 9 "cells" where the top/bottom rows are mostly a single
436
- flat color (the band). Filter on that.
437
- """
187
+ def _is_real_grid(self, image_path: str, grid_boxes: List[Tuple[int, int, int, int]]) -> bool:
188
+ """Reject a lattice that is sheared, a sprite board on one background, or a flat image."""
189
+ irregularity = _lattice_irregularity(grid_boxes)
190
+ if irregularity is not None and irregularity > _GRID_REGULARITY_TOL:
191
+ _debug(f"_is_real_grid: cells are {irregularity:.3f} of a cell out of true → reject")
192
+ return False
438
193
  try:
439
194
  import cv2
440
195
  import numpy as np
441
196
 
442
197
  img = cv2.imread(image_path)
443
198
  if img is None:
444
- return True # Be permissive on read failure.
445
-
446
- # Per-cell mean color standard deviation. Photo tiles ~ 30-80.
447
- # Flat color band rows ~ 0-12.
448
- stds: List[float] = []
449
- for (x1, y1, x2, y2) in grid_boxes:
450
- if x2 <= x1 or y2 <= y1:
451
- continue
452
- roi = img[y1:y2, x1:x2]
453
- if roi.size == 0:
454
- continue
455
- # mean of per-channel stddev — robust to monochrome cells.
456
- stds.append(float(np.mean(np.std(roi.reshape(-1, 3), axis=0))))
457
-
458
- if not stds:
199
+ return True
200
+ rois = [img[y1:y2, x1:x2].reshape(-1, 3) for (x1, y1, x2, y2) in grid_boxes
201
+ if x2 > x1 and y2 > y1 and img[y1:y2, x1:x2].size]
202
+ if not rois:
459
203
  return False
460
-
461
- self.debug.log(
462
- f"_is_real_grid: per-cell stddev = {[round(s, 1) for s in stds]}"
463
- )
464
-
465
- # A SPRITE BOARD is not a tile grid, however grid-shaped it is.
466
- #
467
- # GeeTest's iconcrush and gobang are lattices of sprites drawn on
468
- # ONE flat board, so find_grid finds them and every cell is
469
- # colourful enough (mean stddev ~52) to clear every check below.
470
- # iconcrush was routed to `_solve_grid` on every capture, which
471
- # asks the model to "choose the cell numbers that match the
472
- # description" — the wrong question for a board you answer by
473
- # tapping two tiles to swap them. Nothing errors: the reply comes
474
- # back as `[3, 6, 9]`, is re-encoded as clicks on those cell
475
- # centres, and is graded against a two-click gold. The type scored
476
- # 0.089 that way and read as the model's weakest.
477
- #
478
- # The discriminator is the BACKGROUND, not the content. A real
479
- # captcha grid is independent photographs, so each tile's median
480
- # colour is its own; a sprite board shares one board colour behind
481
- # every cell, so the medians collapse onto a single value.
482
- # Measured over the real captures: iconcrush and gobang 0.0-0.8,
483
- # against 14.9 (a reCAPTCHA 4x4 grid) up to 93.0 (a GeeTest v4 icon board)
484
- # for the five true grid families. The threshold sits in an 18x
485
- # gap, which is why it is a fixed number rather than a tuned one.
486
- medians = []
487
- for (x1, y1, x2, y2) in grid_boxes:
488
- roi = img[y1:y2, x1:x2]
489
- if roi.size:
490
- medians.append(np.median(roi.reshape(-1, 3), axis=0))
491
- if len(medians) >= 4:
492
- spread = float(np.mean(np.std(np.array(medians), axis=0)))
493
- self.debug.log(f"_is_real_grid: median-colour spread = {spread:.1f}")
494
- if spread < 6.0:
495
- self.debug.log(
496
- f"_is_real_grid: every cell shares one background colour "
497
- f"({spread:.1f} < 6.0) → reject (sprite board, not tiles)."
498
- )
499
- return False
500
-
501
- # If the *whole* image is mostly flat (e.g. screenshotting the
502
- # tiny hCaptcha anchor iframe with just the "I'm not a robot"
503
- # text), find_grid hallucinates a 9-cell grid. Reject early.
504
- import statistics
505
- if statistics.mean(stds) < 25.0:
506
- self.debug.log(
507
- f"_is_real_grid: overall mean stddev "
508
- f"{statistics.mean(stds):.1f} too low → reject."
509
- )
204
+ # Real tiles are independent photographs; a sprite board shares one colour behind every cell.
205
+ if len(rois) >= 4 and float(np.mean(np.std(np.array([np.median(r, axis=0) for r in rois]), axis=0))) < 6.0:
206
+ _debug("_is_real_grid: every cell shares one background colour → reject")
207
+ return False
208
+ if float(np.mean([np.mean(np.std(r, axis=0)) for r in rois])) < 25.0:
209
+ _debug("_is_real_grid: overall mean stddev too low → reject")
510
210
  return False
511
-
512
- # THE FLAT-CELL CONTIGUITY HEURISTIC IS GONE, and it was the only
513
- # thing rejecting real grids.
514
- #
515
- # It assumed flat tiles in a genuine photo grid always form ONE
516
- # contiguous blob (a top-left sky stack, a bottom road row), because
517
- # the board is a single photo cut up; a click-puzzle false grid puts
518
- # its flat cells in the left and right margins of the middle row,
519
- # which is two components. True of the boards it was written for,
520
- # and false of plenty of photographs — a subject centred against
521
- # plain sky leaves two flat corners and nothing between them.
522
- #
523
- # MEASURED over every real capture we hold, restricted to lattices
524
- # `_grid_dims` actually accepts (9 or 16 cells — a 25-cell gobang
525
- # board never reaches this function as a grid decision):
526
- #
527
- # rule grid false neg board false pos
528
- # spread + mean-stddev 2/1748 0.11% 0/269 0.00%
529
- # ...plus contiguity 40/1748 2.29% 0/269 0.00%
530
- #
531
- # It cost 38 real grids and caught nothing the two checks above did
532
- # not already catch — 19 of those were a reCAPTCHA 4x4 grid alone,
533
- # 5.5% of that family, every one of them sent to the click path to
534
- # be asked for coordinates on a board answered with cell numbers.
535
- # Nothing errors when that happens; the reply is simply graded
536
- # against a gold it cannot match.
537
- #
538
- # The median-colour spread check above supersedes it and is a much
539
- # cleaner separation (an 18x gap rather than a topological guess),
540
- # which is why this is a deletion and not a retuning.
541
211
  return True
542
212
  except Exception as e:
543
- self.debug.log(f"_is_real_grid check errored ({e}); accepting grid.")
213
+ _debug(f"_is_real_grid check errored ({e}); accepting grid.")
544
214
  return True
545
215
 
546
- def _solve_grid(
547
- self,
548
- image_path: str,
549
- grid_boxes: List[Tuple[int, int, int, int]],
550
- rows: int,
551
- cols: int,
552
- retry_mode: Optional[str] = None,
553
- ) -> Union[ClickAction, DoneAction, WaitAction]:
554
- # The shape is PASSED IN, already cleared by `_grid_dims`. It used to be
555
- # derived here, and the derivation had an `else` branch that turned any
556
- # unrecognised count into `cols = int(sqrt(n)); rows = ceil(n / cols)` —
557
- # so a 12-cell false positive became a 4x3 board, got stamped with twelve
558
- # numbers, and was answered with cell ids. No such board can exist in the
559
- # training corpus, because the generator's own detector refuses every
560
- # count but 9 and 16.
561
- self.debug.log(f"grid {rows}x{cols} ({len(grid_boxes)} cells)")
562
-
563
- cv_selected: List[int] = []
564
- cv_loading: List[int] = []
216
+ def _solve_grid(self, image_path: str, grid_boxes: List[Tuple[int, int, int, int]], rows: int, cols: int,
217
+ retry_mode: Optional[RetryMode] = None) -> Union[ClickAction, DoneAction, WaitAction]:
565
218
  try:
566
- cv_selected, cv_loading = detect_selected_cells(image_path, grid_boxes, self.debug)
567
- if cv_selected:
568
- self.debug.log(f"CV: cells already selected -> {cv_selected}")
569
- if cv_loading:
570
- self.debug.log(f"CV: cells loading -> {cv_loading}")
219
+ cv_selected, cv_loading = detect_selected_cells(image_path, grid_boxes)
571
220
  except Exception as e:
572
- self.debug.log(f"detect_selected_cells failed: {e}")
573
-
574
- # SINGLE canonical overlay: get_numbered_grid_overlay (RED labels,
575
- # top-right, ALL cells 1..N) — byte-for-byte the same overlay used to
576
- # generate the training data (scripts/build_grid_overlays.py) and the
577
- # offline grader. The model was trained ONLY on this style; any other
578
- # overlay (e.g. green, or skipping cells) is out-of-distribution and
579
- # tanks the live solve rate. We number EVERY cell so the model sees the
580
- # exact grid it trained on; already-selected/loading cells are filtered
581
- # AFTER the model responds (below), never by renumbering the grid.
582
- ext = os.path.splitext(image_path)[1] or ".png"
583
- with tempfile.NamedTemporaryFile(suffix=ext, delete=False) as tf:
221
+ _debug(f"detect_selected_cells failed: {e}")
222
+ cv_selected, cv_loading = [], []
223
+
224
+ # Every cell is numbered, exactly as the training overlays were; CV state is filtered after.
225
+ with tempfile.NamedTemporaryFile(suffix=os.path.splitext(image_path)[1] or ".png", delete=False) as tf:
584
226
  overlay_path = tf.name
585
227
  self._temp_files.append(overlay_path)
586
228
  get_numbered_grid_overlay(image_path, grid_boxes, output_path=overlay_path)
587
- self.debug.save_image(overlay_path, "01_grid_overlay.png")
588
-
589
- with timed("planner.grid"):
590
- selected = self.planner.get_grid_selection(
591
- overlay_path, rows=rows, cols=cols, retry_mode=retry_mode,
592
- )
229
+ selected = self.planner.get_grid_selection(overlay_path, rows=rows, cols=cols, retry_mode=retry_mode)
593
230
 
594
- # Map the model's tile IDs to clicks. Skip cells the CV layer already
595
- # flagged as selected (avoid re-toggling) or still loading.
596
- final: List[int] = []
231
+ final = []
597
232
  for n in selected:
598
233
  try:
599
234
  v = int(n)
600
235
  except (TypeError, ValueError):
601
236
  continue
602
- if v < 1 or v > len(grid_boxes):
603
- continue # hallucinated index
604
- if v in cv_selected or v in cv_loading:
605
- continue # already selected / not ready — don't re-click
606
- final.append(v)
607
-
237
+ if 1 <= v <= len(grid_boxes) and v not in cv_selected and v not in cv_loading:
238
+ final.append(v)
608
239
  if not final:
609
- # Nothing new to click: wait if tiles are still loading, else done.
610
- if cv_loading:
611
- return WaitAction(action="wait", duration_ms=1000)
612
- return DoneAction(action="done")
613
-
614
- img_w, img_h = self._image_size # type: ignore[misc]
615
- bboxes: List[List[float]] = []
616
- for v in final:
617
- x1, y1, x2, y2 = grid_boxes[v - 1]
618
- bboxes.append([x1 / img_w, y1 / img_h, x2 / img_w, y2 / img_h])
619
- return ClickAction(action="click", target_bounding_boxes=bboxes)
620
-
621
-
622
- def solve_captcha(
623
- media_path: str,
624
- instruction: str = "",
625
- *,
626
- puzzle_source: str = "unknown",
627
- retry_mode: Optional[str] = None,
628
- text_mode: bool = False,
629
- **kwargs: Any,
630
- ) -> Any:
631
- """Solve one captcha image. `kwargs` configure the solver; the arguments
632
- named above are forwarded to `CaptchaSolver.solve`.
633
-
634
- They are spelled out rather than left in `**kwargs` because everything in
635
- `kwargs` goes to the CONSTRUCTOR. A caller passing `text_mode=True` — the
636
- only way this one-shot API can say "the answer is a typed string, not a
637
- place on the picture" — got `TypeError: __init__() got an unexpected keyword
638
- argument`, so a static distorted-text image was unsolvable through the
639
- public entry point. Only the `PageSolver` path could reach the text prompt,
640
- and it decides `text_mode` from the DOM, which a bare image does not have.
641
- """
642
- return CaptchaSolver(**kwargs).solve(
643
- media_path, instruction,
644
- puzzle_source=puzzle_source, retry_mode=retry_mode, text_mode=text_mode,
645
- )
240
+ return WaitAction(action=ActionKind.WAIT, duration_ms=1000) if cv_loading else DoneAction(action=ActionKind.DONE)
241
+ img_w, img_h = self._image_size
242
+ return ClickAction(action=ActionKind.CLICK, target_bounding_boxes=[
243
+ [x1 / img_w, y1 / img_h, x2 / img_w, y2 / img_h] for (x1, y1, x2, y2) in (grid_boxes[v - 1] for v in final)])
244
+
245
+
246
+ def solve_captcha(media_path: str, instruction: str = "", *, puzzle_source: Vendor = Vendor.UNKNOWN,
247
+ retry_mode: Optional[RetryMode] = None, text_mode: bool = False, **kwargs: Any) -> Any:
248
+ # Solve arguments are spelled out because `**kwargs` go to the constructor: `text_mode=True` once raised
249
+ # TypeError there, which made a static distorted-text image unsolvable through the public entry point.
250
+ return CaptchaSolver(**kwargs).solve(media_path, instruction, puzzle_source=puzzle_source,
251
+ retry_mode=retry_mode, text_mode=text_mode)