captchakraken 2.4.0 → 2.6.1

package/python/src/captchakraken/solver.py

@@ -10,7 +10,15 @@ Flow:
   3. Otherwise (click / drag / other still-image puzzle) → route the image to
      the full-puzzle pixel/action path (planner.get_pixel_actions), which the
      LoRA is trained on. Only raise UnsupportedCaptchaError if the model returns
-     nothing usable. Video challenges are filtered out upstream by the caller.
+     nothing usable.
+
+ANIMATED challenges take a separate entry point, `solve_keyframes`. They used to be
+detected and skipped: the settle monitor called them "never settles" and the solve
+was abandoned. Now the driver records the widget, `keyframes.py` slices the
+recording into the few stills that carry the answer, and those go to the model as
+one multi-image request. The answer comes back with a `frame`, and the driver waits
+for the page to look like that frame before it clicks — because on these puzzles
+the coordinates are only correct while the target is actually on screen.
 
 v1 had a SAM3-backed tool-using planner with detect/segment/drag-refine; it
 lives on the `v1-old-architecture` branch.
@@ -23,7 +31,7 @@ import sys
 import tempfile
 from datetime import datetime
 from pathlib import Path
-from typing import Any, List, Optional, Tuple, Union
+from typing import Any, List, Optional, Sequence, Tuple, Union
 
 from PIL import Image
 
@@ -32,6 +40,7 @@ from .action_types import (
     ClickAction,
     DoneAction,
     DragAction,
+    TypeAction,
     WaitAction,
 )
 from .image_processor import ImageProcessor
@@ -140,6 +149,7 @@ class CaptchaSolver:
         instruction: str = "",
         puzzle_source: str = "unknown",
         retry_mode: Optional[str] = None,
+        text_mode: bool = False,
     ) -> Union[CaptchaAction, List[CaptchaAction]]:
         media_path = str(Path(media_path).resolve())
         if not os.path.exists(media_path):
@@ -150,6 +160,27 @@ class CaptchaSolver:
         assert self._image_size is not None
         img_w, img_h = self._image_size
 
+        if text_mode:
+            # The driver saw a text box in the widget, so the answer is a string
+            # and there is nothing on the picture to find. Grid and checkbox
+            # detection are skipped rather than merely ignored: BotDetect's
+            # boxed, evenly-spaced glyphs are exactly the lattice find_grid
+            # looks for, and a false grid here would answer a typing puzzle
+            # with a list of cell numbers.
+            try:
+                actions = self._solve_pixel(cv_image_path, text_mode=True)
+            except ValueError as exc:
+                # The served model predates the distorted-text family, so there
+                # is no prompt to send it. Surfaced as UNSUPPORTED rather than
+                # crashing: it is the same class of outcome as "this LoRA only
+                # does grids", the driver already knows how to report it, and
+                # the message names the real fix (a generation-2 model) instead
+                # of a stack trace in prompts.py.
+                raise UnsupportedCaptchaError(str(exc)) from exc
+            if actions:
+                return actions
+            raise UnsupportedCaptchaError("Could not read the text captcha")
+
         with timed("solver.find_grid"):
             grid_boxes = find_grid(cv_image_path)
         if grid_boxes and self._is_real_grid(cv_image_path, grid_boxes):
@@ -178,9 +209,8 @@ class CaptchaSolver:
 
         # Not a grid or checkbox → a click/drag/pixel puzzle. The full-puzzle
         # LoRA is trained on all of these, so route the image to the pixel
-        # action path rather than bailing. (Video challenges are skipped
-        # upstream by the caller before they reach the solver, so anything here
-        # is a still-image puzzle worth attempting.)
+        # action path rather than bailing. (Animated challenges come in through
+        # `solve_keyframes` instead, so anything here is a still worth attempting.)
         actions = self._solve_pixel(cv_image_path)
         if actions:
             return actions
@@ -190,26 +220,87 @@ class CaptchaSolver:
         # instead of clicking nothing.
         raise UnsupportedCaptchaError("Cannot solve this kind of captcha")
 
+    def solve_keyframes(
+        self, keyframe_paths: Sequence[str]
+    ) -> List[Union[ClickAction, DragAction]]:
+        """Solve an ANIMATED challenge from the keyframes of a recording.
+
+        `keyframe_paths` must be in model order (frame 1 first) — the numbers the
+        answer refers to are positional, so a reordered list silently remaps the
+        answer onto the wrong picture. `keyframes.materialize_keyframes` and
+        `read_keyframe_paths` both return them correctly ordered; do not sort them
+        yourself with a plain string sort past nine frames.
+
+        Grid detection is deliberately NOT attempted. An animated challenge is never
+        a tile grid (vendors do not animate those), and `find_grid` false-positives
+        on the header/footer bands of hCaptcha's click puzzles — which is what these
+        are. Every returned action carries `await_keyframe`, the still the driver
+        must see on screen before it acts.
+        """
+        paths = [str(p) for p in keyframe_paths]
+        if not paths:
+            raise UnsupportedCaptchaError("no keyframes to solve from")
+        with Image.open(paths[0]) as im:
+            self._image_size = im.size
+        self.debug.save_image(paths[0], "00_keyframe_01.png")
+        self.debug.log(f"solving animated challenge from {len(paths)} keyframes")
+
+        actions = self._to_actions(
+            self.planner.get_keyframe_actions(paths), keyframe_paths=paths
+        )
+        if actions:
+            return actions
+        raise UnsupportedCaptchaError("Cannot solve this animated captcha")
+
     def _solve_pixel(
-        self, image_path: str
+        self, image_path: str, text_mode: bool = False
     ) -> List[Union[ClickAction, DragAction]]:
         """Turn the model's normalized 0–1 click/drag actions into ClickAction /
         DragAction bboxes. Each point becomes a small box centered on it (the TS
         solver clicks the box center)."""
-        raw_actions = self.planner.get_pixel_actions(image_path)
+        return self._to_actions(self.planner.get_pixel_actions(image_path, text_mode=text_mode))
+
+    def _to_actions(
+        self,
+        raw_actions: List[dict],
+        keyframe_paths: Optional[Sequence[str]] = None,
+    ) -> List[Union[ClickAction, DragAction]]:
+        """Shared conversion from the planner's normalized 0–1 actions to typed
+        Click/Drag actions with small boxes around each point.
+
+        When `keyframe_paths` is given, the chosen keyframe is attached so the
+        driver knows which page state to wait for. A model answer with no usable
+        frame is still converted — it just carries no keyframe, and the driver
+        treats that as "act on the current frame". That is the honest fallback:
+        refusing the action outright would turn a good coordinate into a failed
+        solve over a missing integer.
+        """
         R = _PIXEL_BOX_HALF
         clamp = lambda v: min(max(v, 0.0), 1.0)
 
         def box(cx: float, cy: float) -> List[float]:
             return [clamp(cx - R), clamp(cy - R), clamp(cx + R), clamp(cy + R)]
 
+        def wait_for(a: dict) -> dict:
+            if not keyframe_paths:
+                return {}
+            frame = a.get("frame")
+            if not isinstance(frame, int) or not 1 <= frame <= len(keyframe_paths):
+                self.debug.log(
+                    "animated answer named no usable keyframe; acting on the "
+                    "current frame without waiting"
+                )
+                return {}
+            return {"frame": frame, "await_keyframe": str(keyframe_paths[frame - 1])}
+
         out: List[Union[ClickAction, DragAction]] = []
         for a in raw_actions:
             if a.get("kind") == "click":
                 boxes = [box(x, y) for (x, y) in a.get("points", [])]
                 if boxes:
                     out.append(
-                        ClickAction(action="click", target_bounding_boxes=boxes)
+                        ClickAction(action="click", target_bounding_boxes=boxes,
+                                    **wait_for(a))
                     )
             elif a.get("kind") == "drag":
                 sx, sy = a["src"]
@@ -219,11 +310,33 @@ class CaptchaSolver:
                         action="drag",
                         source_bounding_box=box(sx, sy),
                         target_bounding_box=box(dx, dy),
+                        **wait_for(a),
+                    )
+                )
+            elif a.get("kind") == "slide":
+                # A puzzle-piece slider, carried as a drag with NO source. That
+                # is not a lossy encoding of a normal drag — it is the whole
+                # shape of the answer: the thing you grab (the handle) is not
+                # the thing that has to arrive (the piece), and only the page
+                # knows where the handle is. The driver reads a null source as
+                # "find the slider and close the loop on the piece".
+                dx, dy = a["dst"]
+                out.append(
+                    DragAction(
+                        action="drag",
+                        source_bounding_box=None,
+                        target_bounding_box=box(dx, dy),
+                        **wait_for(a),
                     )
                 )
+            elif a.get("kind") == "type":
+                out.append(TypeAction(action="type", text=a["text"]))
         return out
 
-    # Back-compat alias.
+    # Back-compat alias. Kept pointing at `solve` (a single still), NOT at
+    # `solve_keyframes`: callers of this name pass one media path and expect one
+    # answer, and quietly reinterpreting that as "record and slice" would change
+    # what an old integration does. New code calls `solve_keyframes` explicitly.
     def solveVideo(self, *args, **kwargs):
         return self.solve(*args, **kwargs)
 
@@ -299,6 +412,42 @@ class CaptchaSolver:
                 f"_is_real_grid: per-cell stddev = {[round(s, 1) for s in stds]}"
             )
 
+            # A SPRITE BOARD is not a tile grid, however grid-shaped it is.
+            #
+            # GeeTest's iconcrush and gobang are lattices of sprites drawn on
+            # ONE flat board, so find_grid finds them and every cell is
+            # colourful enough (mean stddev ~52) to clear every check below.
+            # iconcrush was routed to `_solve_grid` on every capture, which
+            # asks the model to "choose the cell numbers that match the
+            # description" — the wrong question for a board you answer by
+            # tapping two tiles to swap them. Nothing errors: the reply comes
+            # back as `[3, 6, 9]`, is re-encoded as clicks on those cell
+            # centres, and is graded against a two-click gold. The type scored
+            # 0.089 that way and read as the model's weakest.
+            #
+            # The discriminator is the BACKGROUND, not the content. A real
+            # captcha grid is independent photographs, so each tile's median
+            # colour is its own; a sprite board shares one board colour behind
+            # every cell, so the medians collapse onto a single value.
+            # Measured over the real captures: iconcrush and gobang 0.0-0.8,
+            # against 14.9 (recaptcha_grid_4x4) up to 93.0 (geetest_v4_nine)
+            # for the five true grid families. The threshold sits in an 18x
+            # gap, which is why it is a fixed number rather than a tuned one.
+            medians = []
+            for (x1, y1, x2, y2) in grid_boxes:
+                roi = img[y1:y2, x1:x2]
+                if roi.size:
+                    medians.append(np.median(roi.reshape(-1, 3), axis=0))
+            if len(medians) >= 4:
+                spread = float(np.mean(np.std(np.array(medians), axis=0)))
+                self.debug.log(f"_is_real_grid: median-colour spread = {spread:.1f}")
+                if spread < 6.0:
+                    self.debug.log(
+                        f"_is_real_grid: every cell shares one background colour "
+                        f"({spread:.1f} < 6.0) → reject (sprite board, not tiles)."
+                    )
+                    return False
+
             # If the *whole* image is mostly flat (e.g. screenshotting the
             # tiny hCaptcha anchor iframe with just the "I'm not a robot"
             # text), find_grid hallucinates a 9-cell grid. Reject early.
@@ -430,5 +579,27 @@ class CaptchaSolver:
         return ClickAction(action="click", target_bounding_boxes=bboxes)
 
 
-def solve_captcha(media_path: str, instruction: str = "", **kwargs) -> Any:
-    return CaptchaSolver(**kwargs).solve(media_path, instruction)
+def solve_captcha(
+    media_path: str,
+    instruction: str = "",
+    *,
+    puzzle_source: str = "unknown",
+    retry_mode: Optional[str] = None,
+    text_mode: bool = False,
+    **kwargs: Any,
+) -> Any:
+    """Solve one captcha image. `kwargs` configure the solver; the arguments
+    named above are forwarded to `CaptchaSolver.solve`.
+
+    They are spelled out rather than left in `**kwargs` because everything in
+    `kwargs` goes to the CONSTRUCTOR. A caller passing `text_mode=True` — the
+    only way this one-shot API can say "the answer is a typed string, not a
+    place on the picture" — got `TypeError: __init__() got an unexpected keyword
+    argument`, so a static distorted-text image was unsolvable through the
+    public entry point. Only the `PageSolver` path could reach the text prompt,
+    and it decides `text_mode` from the DOM, which a bare image does not have.
+    """
+    return CaptchaSolver(**kwargs).solve(
+        media_path, instruction,
+        puzzle_source=puzzle_source, retry_mode=retry_mode, text_mode=text_mode,
+    )

package/python/src/captchakraken/timing.py

@@ -40,3 +40,73 @@ def timed(label: str, extra: Optional[str] = None) -> Iterator[None]:
         print(f"[TIMING] {label}: {dt_ms:.2f}ms{suffix}", file=sys.stderr)
 
 
+#: What a phase's time COUNTS AS, when asking how much of a solve was useful.
+#:
+#: Only two kinds of second are worth spending: one the model is thinking in,
+#: and one the mouse is travelling in (which has to look human, so it cannot be
+#: rushed). Everything else is the driver waiting on a clock, and every such
+#: wait is a candidate for deletion or for overlapping with something real.
+PRODUCTIVE = {"inference", "mouse"}
+
+
+class PhaseBudget:
+    """Where one solve's wall-clock went, by phase.
+
+    Exists because "the solve took 77s" is not actionable and "the settle
+    monitor spent 31s of it" is. Always on — it is a few dict updates per phase
+    against multi-second waits — but only PRINTED under CAPTCHA_TIMINGS=1.
+
+    Phases may nest (a burst contains its screenshots); only the outermost of a
+    given name accumulates, so the totals stay a partition of the solve rather
+    than double-counting.
+    """
+
+    def __init__(self) -> None:
+        self.totals: dict = {}
+        self.counts: dict = {}
+        self._open: list = []
+        self._t0 = time.perf_counter()
+
+    @contextmanager
+    def phase(self, name: str) -> Iterator[None]:
+        if name in self._open:          # nested re-entry: attribute it to the outer one
+            yield
+            return
+        self._open.append(name)
+        t0 = time.perf_counter()
+        try:
+            yield
+        finally:
+            self._open.remove(name)
+            dt = (time.perf_counter() - t0) * 1000.0
+            self.totals[name] = self.totals.get(name, 0.0) + dt
+            self.counts[name] = self.counts.get(name, 0) + 1
+
+    def add(self, name: str, ms: float) -> None:
+        """Record a measured span directly.
+
+        For blocks that would need re-indenting to sit under `phase()` — the
+        accounting is not worth reshaping working control flow over.
+        """
+        self.totals[name] = self.totals.get(name, 0.0) + ms
+        self.counts[name] = self.counts.get(name, 0) + 1
+
+    def elapsed_ms(self) -> float:
+        return (time.perf_counter() - self._t0) * 1000.0
+
+    def report(self) -> str:
+        total = self.elapsed_ms()
+        useful = sum(v for k, v in self.totals.items() if k in PRODUCTIVE)
+        rows = sorted(self.totals.items(), key=lambda kv: -kv[1])
+        lines = [f"[BUDGET] solve {total / 1000:.1f}s — "
+                 f"{useful / 1000:.1f}s useful ({100 * useful / total if total else 0:.0f}%), "
+                 f"{(total - useful) / 1000:.1f}s waiting"]
+        for name, ms in rows:
+            tag = "*" if name in PRODUCTIVE else " "
+            lines.append(f"[BUDGET] {tag} {name:22s} {ms / 1000:6.2f}s  x{self.counts[name]}")
+        unattributed = total - sum(self.totals.values())
+        if unattributed > 50:
+            lines.append(f"[BUDGET]   {'(unattributed)':22s} {unattributed / 1000:6.2f}s")
+        return "\n".join(lines)
+
+