captchakraken 3.2.1 → 4.0.0

package/python/src/captchakraken/solver.py

@@ -1,15 +1,28 @@
-"""Image in, actions out: grid detection plus the model, for one still or one keyframe set."""
+"""Image in, actions out: the element detector plus the model, for one still or one recorded animation."""
 
-import math
 import os
 import sys
 import tempfile
+from dataclasses import dataclass
 from pathlib import Path
-from typing import Any, List, Optional, Sequence, Tuple, Union
+from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple, Union
 
+import numpy as np
 from PIL import Image
 
-from .action_types import CaptchaAction, ClickAction, DoneAction, DragAction, TypeAction, WaitAction
+from .action_types import (
+    CaptchaAction,
+    ClickAction,
+    DoneAction,
+    DragAction,
+    HoldAction,
+    RefreshAction,
+    SlideAction,
+    TypeAction,
+    WaitAction,
+)
+from .clipvideo import clip_video, time_to_index
+from .keyframes import KeyframeSet, extract_keyframes, number_for_source_index, write_frame, write_keyframes
 from .kinds import ActionKind, PixelAnswerKind, PromptFamily, RetryMode, Vendor
 from .planner import ActionPlanner
 from .tool_calls.find_checkbox import find_checkbox
@@ -19,13 +32,12 @@ DEBUG = os.getenv("CAPTCHA_DEBUG", "0") == "1"
 # ~1.2% ≈ ±6px on a 512px challenge.
 _PIXEL_BOX_HALF = float(os.getenv("CAPTCHA_PIXEL_BOX_HALF", "0.012"))
 
-# The only cell counts a trained grid puzzle has, and which of them each named vendor ships. UNKNOWN stays
-# permissive in both directions: the driver reports it for GeeTest and Prosopo, and the offline grader defaults
-# to it, so narrowing it would silently stop scoring reCAPTCHA 4x4 in evaluation.
+# The grid expert was trained on reCAPTCHA's 3x3 and 4x4 and hCaptcha's 3x3 and nothing else, so only those
+# boards are asked as grids; every other vendor's grid goes to the pixel expert, which was trained on them. A
+# caller that names no vendor (the CLI, a script) gets the expert for any 3x3 or 4x4, which is what those boards
+# most often are.
 GRID_CELL_SHAPES = {9: (3, 3), 16: (4, 4)}
 VENDOR_GRID_CELLS: dict[Vendor, frozenset[int]] = {Vendor.HCAPTCHA: frozenset({9}), Vendor.RECAPTCHA: frozenset({9, 16})}
-# Every true grid measures 0.000 cells out of true; the nearest false lattice measured 0.128.
-_GRID_REGULARITY_TOL = 0.02
 
 
 def _debug(message: str) -> None:
@@ -37,30 +49,46 @@ class UnsupportedCaptchaError(Exception):
     pass
 
 
-def _lattice_irregularity(grid_boxes) -> Optional[float]:
-    """How far the cells stray from a regular lattice, in cells; None when not a square count."""
-    boxes = [tuple(b) for b in grid_boxes]
-    n = len(boxes)
-    k = int(round(math.sqrt(n))) if n else 0
-    if k < 2 or k * k != n:
-        return None
-    widths = [b[2] - b[0] for b in boxes]
-    heights = [b[3] - b[1] for b in boxes]
-    mid_w, mid_h = sorted(widths)[n // 2], sorted(heights)[n // 2]
-    if mid_w <= 0 or mid_h <= 0:
-        return None
-    rows = [boxes[i * k:(i + 1) * k] for i in range(k)]
-    cols = [[boxes[r * k + c] for r in range(k)] for c in range(k)]
-    row_skew = max(max(b[1] for b in r) - min(b[1] for b in r) for r in rows) / mid_h
-    col_skew = max(max(b[0] for b in c) - min(b[0] for b in c) for c in cols) / mid_w
-    size_spread = max((max(widths) - min(widths)) / mid_w, (max(heights) - min(heights)) / mid_h)
-    return max(row_skew, col_skew, size_spread)
+@dataclass
+class AnimatedRecording:
+    """One recorded burst of an animated challenge, cut for the driver and for the model.
+
+    The keyframes belong to the driver whichever way the model is asked: they say how many steady screens the
+    board has, and they are the sibling screens the wait gate compares an answer against. The model is shown
+    either the keyframes (prompt generation 2) or the frames themselves as one video (generation 3 on).
+    """
+
+    frames: List[np.ndarray]
+    fps: float
+    keyframes: KeyframeSet
+    keyframe_paths: List[str]
+
+
+def cut_recording(frames: Sequence[np.ndarray], fps: float, out_dir: str) -> AnimatedRecording:
+    """Slice `frames` into keyframes written to `out_dir`, keeping the frames for a video ask."""
+    kfset = extract_keyframes(frames, fps=float(fps))
+    paths = [str(p) for p in write_keyframes(kfset, out_dir, stem="challenge")]
+    return AnimatedRecording(frames=list(frames), fps=float(fps), keyframes=kfset, keyframe_paths=paths)
+
+
+def _first_frame(path: str) -> Image.Image:
+    """The first picture of a GIF or video file, in RGB."""
+    import av
+
+    try:
+        if path.lower().endswith(".gif"):
+            with Image.open(path) as img:
+                return img.convert("RGB")
+        with av.open(path) as container:
+            return next(container.decode(video=0)).to_image()
+    except (OSError, av.FFmpegError, StopIteration) as exc:
+        raise ValueError(f"Could not read video frame from {path}") from exc
 
 
 def _grid_dims(n_cells: int, puzzle_source: Vendor = Vendor.UNKNOWN):
-    shape = GRID_CELL_SHAPES.get(n_cells)
-    allowed = VENDOR_GRID_CELLS.get(Vendor(puzzle_source))
-    return None if shape is None or (allowed is not None and n_cells not in allowed) else shape
+    vendor = Vendor(puzzle_source)
+    allowed = frozenset(GRID_CELL_SHAPES) if vendor == Vendor.UNKNOWN else VENDOR_GRID_CELLS.get(vendor, frozenset())
+    return GRID_CELL_SHAPES[n_cells] if n_cells in allowed else None
 
 
 class CaptchaSolver:
@@ -97,15 +125,16 @@ class CaptchaSolver:
                 return actions
             raise UnsupportedCaptchaError("Could not read the text captcha")
 
-        # The animated path never runs find_grid: it false-positives on the header and footer bands of hCaptcha's click puzzles.
-        grid_boxes = find_grid(image_path)
+        # Generation 4 never runs find_grid: the pointer server's element detector numbers a grid's cells along with
+        # everything else on the board, and its answer is points (and element numbers) like any other board's.
+        grid_boxes = [] if self.planner.prompts.pointer else find_grid(image_path)
         dims = _grid_dims(len(grid_boxes), puzzle_source) if grid_boxes else None
-        if grid_boxes and dims and self._is_real_grid(image_path, grid_boxes):
+        if dims:
             return self._solve_grid(image_path, grid_boxes, dims[0], dims[1], retry_mode=retry_mode)
         if grid_boxes:
             _debug(f"find_grid returned {len(grid_boxes)} cells; not solving as a grid")
 
-        if img_h < 400:
+        if not self.planner.prompts.pointer:
             checkbox = find_checkbox(image_path)
             if checkbox:
                 x, y, w, h = checkbox
@@ -117,49 +146,117 @@ class CaptchaSolver:
             return actions
         raise UnsupportedCaptchaError("Cannot solve this kind of captcha")
 
-    def solve_keyframes(self, keyframe_paths: Sequence[str]) -> List[Union[ClickAction, DragAction]]:
+    def solve_keyframes(self, keyframe_paths: Sequence[str]) -> List[CaptchaAction]:
         """Solve an animated challenge from its keyframes, in model order (frame 1 first)."""
         paths = [str(p) for p in keyframe_paths]
         if not paths:
             raise UnsupportedCaptchaError("no keyframes to solve from")
         with Image.open(paths[0]) as im:
             self._image_size = im.size
-        actions = self._to_actions(self.planner.get_keyframe_actions(paths), keyframe_paths=paths)
+
+        def wait_for(a: dict) -> dict:
+            # An out-of-range frame is dropped, not clamped: clamping a 7 to 6 would invent an intent.
+            frame = a.get("frame")
+            if not isinstance(frame, int) or not 1 <= frame <= len(paths):
+                return {}
+            return {"frame": frame, "await_keyframe": paths[frame - 1]}
+
+        actions = self._to_actions(self._ask_planner(lambda: self.planner.get_keyframe_actions(paths)),
+                                   wait_for=wait_for)
         if actions:
             return actions
         raise UnsupportedCaptchaError("Cannot solve this animated captcha")
 
-    def _solve_pixel(self, image_path: str, text_mode: bool = False) -> List[Union[ClickAction, DragAction]]:
+    def solve_animated(self, recording: AnimatedRecording) -> List[CaptchaAction]:
+        """Solve a recorded animation the way the model's prompt generation was trained to be asked."""
+        if not self.planner.prompts.sends_video:
+            return self.solve_keyframes(recording.keyframe_paths)
+        if not recording.frames:
+            raise UnsupportedCaptchaError("no frames to solve from")
+        height, width = recording.frames[0].shape[:2]
+        self._image_size = (width, height)
+        clip = clip_video(recording.frames, recording.fps, max_pixels=self.planner.video_max_pixels())
+        written: Dict[int, str] = {}
+
+        def wait_for(a: dict) -> dict:
+            t = a.get("t")
+            if t is None:
+                return {}
+            source = time_to_index(clip, t)
+            if source not in written:
+                written[source] = self._await_frame(recording, source)
+            return {"t": t, "await_keyframe": written[source]}
+
+        actions = self._to_actions(self._ask_planner(lambda: self.planner.get_video_actions(clip)), wait_for=wait_for)
+        if actions:
+            return actions
+        raise UnsupportedCaptchaError("Cannot solve this animated captcha")
+
+    @staticmethod
+    def _ask_planner(ask: Callable[[], List[dict]]) -> List[dict]:
+        """A prompt generation that cannot be asked this way is an unsupported puzzle, not a crash."""
+        try:
+            return ask()
+        except ValueError as exc:
+            raise UnsupportedCaptchaError(str(exc)) from exc
+
+    @staticmethod
+    def _await_frame(recording: AnimatedRecording, source: int) -> str:
+        """The wait gate's reference for a video answer: the recorded frame `t` names, in its screen's file.
+
+        It replaces the keyframe of the screen it belongs to, so the gate's sibling check still compares the
+        answer against the OTHER screens only, while the page is matched against the exact moment chosen.
+        """
+        by_number = {kf.number: path for kf, path in zip(recording.keyframes.keyframes, recording.keyframe_paths)}
+        path = by_number[number_for_source_index(recording.keyframes, source)]
+        write_frame(path, recording.frames[source])
+        return path
+
+    def _solve_pixel(self, image_path: str, text_mode: bool = False) -> List[CaptchaAction]:
         return self._to_actions(self.planner.get_pixel_actions(image_path, text_mode=text_mode))
 
+    def pointer_fallback_actions(self) -> List[List[CaptchaAction]]:
+        """The complete plans after the first, and a refresh; [] for a non-pointer model."""
+        return [self._to_actions(plan) for plan in self.planner.pointer_fallback_actions()]
+
     def _to_actions(self, raw_actions: List[dict],
-                    keyframe_paths: Optional[Sequence[str]] = None) -> List[Union[ClickAction, DragAction]]:
-        """Normalised 0-1 points become small boxes; a slide is a drag with no source."""
+                    wait_for: Optional[Callable[[dict], dict]] = None) -> List[CaptchaAction]:
+        """Normalised 0-1 points become small boxes. A slide with a handle is generation 3's; one without is
+        generation 2's sourceless drag.
+
+        `wait_for` maps an animated answer to the fields that tell the driver which moment to wait for.
+        """
         r = _PIXEL_BOX_HALF
+        wait_for = wait_for or (lambda a: {})
 
         def box(cx: float, cy: float) -> List[float]:
             return [min(max(v, 0.0), 1.0) for v in (cx - r, cy - r, cx + r, cy + r)]
 
-        def wait_for(a: dict) -> dict:
-            # An out-of-range frame is dropped, not clamped: clamping a 7 to 6 would invent an intent.
-            frame = a.get("frame")
-            if not keyframe_paths or not isinstance(frame, int) or not 1 <= frame <= len(keyframe_paths):
-                return {}
-            return {"frame": frame, "await_keyframe": str(keyframe_paths[frame - 1])}
-
-        out: List[Union[ClickAction, DragAction]] = []
+        out: List[CaptchaAction] = []
         for a in raw_actions:
             kind = PixelAnswerKind(a["kind"])
             if kind == PixelAnswerKind.CLICK:
                 boxes = [box(x, y) for (x, y) in a.get("points", [])]
-                if boxes:
-                    out.append(ClickAction(action=ActionKind.CLICK, target_bounding_boxes=boxes, **wait_for(a)))
+                # Only generation 3 sends an empty click, and it means nothing on the board matches: submit as is.
+                out.append(ClickAction(action=ActionKind.CLICK, target_bounding_boxes=boxes, **wait_for(a)) if boxes
+                           else DoneAction(action=ActionKind.DONE))
+            elif kind == PixelAnswerKind.SLIDE and "handle" in a:
+                out.append(SlideAction(action=ActionKind.SLIDE, handle_bounding_box=box(*a["handle"]),
+                                       source_bounding_box=box(*a["src"]), target_bounding_box=box(*a["dst"])))
             elif kind in (PixelAnswerKind.DRAG, PixelAnswerKind.SLIDE):
                 src = box(*a["src"]) if kind == PixelAnswerKind.DRAG else None
                 out.append(DragAction(action=ActionKind.DRAG, source_bounding_box=src,
                                       target_bounding_box=box(*a["dst"]), **wait_for(a)))
             elif kind == PixelAnswerKind.TYPE:
                 out.append(TypeAction(action=ActionKind.TYPE, text=a["text"]))
+            elif kind == PixelAnswerKind.HOLD and a.get("points"):
+                fill = ({"fill_from_bounding_box": box(*a["src"]), "fill_to_bounding_box": box(*a["dst"])}
+                        if a.get("src") and a.get("dst") else {})
+                out.append(HoldAction(action=ActionKind.HOLD, target_bounding_box=box(*a["points"][0]), **fill))
+            elif kind == PixelAnswerKind.REFRESH:
+                points = a.get("points")
+                out.append(RefreshAction(action=ActionKind.REFRESH,
+                                         target_bounding_box=box(*points[0]) if points else None))
         return out
 
     def solveVideo(self, *args, **kwargs):
@@ -168,53 +265,29 @@ class CaptchaSolver:
     def _materialize_image(self, media_path: str) -> str:
         """A PNG path for `media_path`; a video contributes its first frame."""
         if any(media_path.lower().endswith(ext) for ext in (".mp4", ".gif", ".avi", ".webm")):
-            import cv2
-
-            cap = cv2.VideoCapture(media_path)
-            ok, frame = cap.read()
-            cap.release()
-            if not ok:
-                raise ValueError(f"Could not read video frame from {media_path}")
-            self._image_size = (frame.shape[1], frame.shape[0])
+            frame = _first_frame(media_path)
+            self._image_size = frame.size
             with tempfile.NamedTemporaryFile(suffix=".png", delete=False) as tf:
-                cv2.imwrite(tf.name, frame)
+                frame.save(tf, "PNG")
                 self._temp_files.append(tf.name)
                 return tf.name
         with Image.open(media_path) as img:
             self._image_size = img.size
         return media_path
 
-    def _is_real_grid(self, image_path: str, grid_boxes: List[Tuple[int, int, int, int]]) -> bool:
-        """Reject a lattice that is sheared, a sprite board on one background, or a flat image."""
-        irregularity = _lattice_irregularity(grid_boxes)
-        if irregularity is not None and irregularity > _GRID_REGULARITY_TOL:
-            _debug(f"_is_real_grid: cells are {irregularity:.3f} of a cell out of true → reject")
-            return False
-        try:
-            import cv2
-            import numpy as np
-
-            img = cv2.imread(image_path)
-            if img is None:
-                return True
-            rois = [img[y1:y2, x1:x2].reshape(-1, 3) for (x1, y1, x2, y2) in grid_boxes
-                    if x2 > x1 and y2 > y1 and img[y1:y2, x1:x2].size]
-            if not rois:
-                return False
-            # Real tiles are independent photographs; a sprite board shares one colour behind every cell.
-            if len(rois) >= 4 and float(np.mean(np.std(np.array([np.median(r, axis=0) for r in rois]), axis=0))) < 6.0:
-                _debug("_is_real_grid: every cell shares one background colour → reject")
-                return False
-            if float(np.mean([np.mean(np.std(r, axis=0)) for r in rois])) < 25.0:
-                _debug("_is_real_grid: overall mean stddev too low → reject")
-                return False
-            return True
-        except Exception as e:
-            _debug(f"_is_real_grid check errored ({e}); accepting grid.")
-            return True
+    def _cells_of(self, click: dict, grid_boxes: List[Tuple[int, int, int, int]]) -> List[int]:
+        """The 1-based cells a generation-3 click names: its ids when any is a cell, else where its points land."""
+        n = len(grid_boxes)
+        ids = [i for i in click.get("ids", []) if 1 <= i <= n]
+        if not ids:
+            img_w, img_h = self._image_size
+            ids = [c for c in (next((k + 1 for k, (x1, y1, x2, y2) in enumerate(grid_boxes)
+                                     if x1 <= x * img_w <= x2 and y1 <= y * img_h <= y2), None)
+                               for (x, y) in click.get("points", [])) if c is not None]
+        return list(dict.fromkeys(ids))
 
     def _solve_grid(self, image_path: str, grid_boxes: List[Tuple[int, int, int, int]], rows: int, cols: int,
-                    retry_mode: Optional[RetryMode] = None) -> Union[ClickAction, DoneAction, WaitAction]:
+                    retry_mode: Optional[RetryMode] = None) -> Union[CaptchaAction, List[CaptchaAction]]:
         try:
             cv_selected, cv_loading = detect_selected_cells(image_path, grid_boxes)
         except Exception as e:
@@ -226,7 +299,15 @@ class CaptchaSolver:
             overlay_path = tf.name
         self._temp_files.append(overlay_path)
         get_numbered_grid_overlay(image_path, grid_boxes, output_path=overlay_path)
-        selected = self.planner.get_grid_selection(overlay_path, rows=rows, cols=cols, retry_mode=retry_mode)
+        if self.planner.prompts.general is not None:
+            raw = self.planner.get_general_actions(overlay_path)
+            if not raw:
+                raise UnsupportedCaptchaError("the model's answer to the grid could not be read")
+            if raw[0]["kind"] != PixelAnswerKind.CLICK:
+                return self._to_actions(raw)
+            selected = self._cells_of(raw[0], grid_boxes)
+        else:
+            selected = self.planner.get_grid_selection(overlay_path, rows=rows, cols=cols, retry_mode=retry_mode)
 
         final = []
         for n in selected:

package/python/src/captchakraken/tool_calls/board_painted.py

@@ -1,7 +1,10 @@
 """Has the widget painted its puzzle yet, or are we photographing the rebuild hole?
 
-Measured on gt4.geetest.com's slide demo (2026-09-12): 6 of 52 requests carried a board with under 5% ink and
-every one came back dead centre. The settle gate cannot catch it because a blank panel is perfectly still.
+First seen on GeeTest's slide demo (2026-09-12): 6 of 52 requests carried a board with under 5% ink. The settle gate
+cannot catch it because a blank panel is perfectly still. Measured 2026-11-02 over 1,557 real captures of every
+vendor: no painted board read as blank, and no board with its centre blanked read as painted. Asking the element
+detector instead was tried and lost: it missed every text captcha and saw items at the edges of a blank centre, at
+0.2-0.4 s a call against this check's ~1 ms.
 Structure is measured, not ink, because geetest_v4_svg is line art on white. A caller must treat False as
 "wait a moment longer", never "give up": a gate that can refuse to ever take a picture is worse than the blank one.
 """
@@ -10,7 +13,7 @@ from __future__ import annotations
 
 from typing import Optional, Sequence
 
-# Trimmed in from every edge: the caption and icon row are chrome that paints early, on a blank board too.
+# Trimmed in from every edge: a caption, an icon row or a slider track is chrome that paints early, on a blank board too.
 CENTRE_BOX = (0.15, 0.25, 0.85, 0.75)
 
 # Below this is JPEG noise and anti-aliasing on a flat fill.
@@ -21,12 +24,14 @@ TEXTURE_FLOOR = 0.015
 
 
 def centre_texture(path: str, box: Optional[Sequence[float]] = None) -> Optional[float]:
-    import cv2
     import numpy as np
 
-    img = cv2.imread(path, cv2.IMREAD_GRAYSCALE)
-    if img is None:
+    from ..keyframes import read_frame, to_gray
+
+    frame = read_frame(path)
+    if frame is None:
         return None
+    img = to_gray(frame)
     h, w = img.shape[:2]
     b = tuple(box) if box is not None else CENTRE_BOX
     x1, y1 = int(b[0] * w), int(b[1] * h)

package/python/src/captchakraken/tool_calls/fill_tracker.py

@@ -0,0 +1,152 @@
+"""Follow a press-and-hold widget's fill, and say when to let go.
+
+The model names three points: what to press, where the fill starts and where it should be released. Holding makes
+something grow from `start` toward `end` (a bar filling, a ring closing along its chord, a pill colouring in), so
+the shot at the press and a later shot differ along that line up to the fill's leading edge, and nowhere past it.
+
+`fill_step` reads one look: the leading edge in pixels along start->end, and whether to release now. Nothing about
+the widget is read from its DOM, and nothing is kept between calls: the caller passes back what earlier looks
+returned, so the CLI worker can serve the JS port the same way.
+
+All coordinates are SHOT pixels; the caller converts to CSS with its own device scale.
+"""
+
+from __future__ import annotations
+
+import math
+from typing import Any, Dict, Optional, Sequence, Tuple, Union
+
+import numpy as np
+from PIL import Image as PILImage
+from scipy import ndimage
+
+Image = Union[str, np.ndarray]
+Point = Tuple[float, float]
+
+# A pixel the fill has reached differs from the press in at least one channel by this much; a re-composited page
+# shot jitters by a few levels at edges, and the 3x3 blur takes most of that out.
+_CHANGED = 24.0
+# Half-width of the band read across the line: the model's points are a few pixels off the bar's axis.
+_BAND_PX = 2
+# The edge is the furthest changed sample whose last _RUN_PX samples are mostly changed: a lone flicker past the
+# fill (a cursor, a badge) is not the fill.
+_RUN_PX = 6
+_RUN_SHARE = 0.6
+# Read this far past `end` too, so a fill that overshot between looks is seen as past, not as stalled at it.
+_PAST = 0.15
+
+
+# A timed release is aimed this long after the predicted arrival. Early and late do not cost the same: when `end` is
+# where the fill stops (HUMAN's pill, a ring closing) the widget completes on its own and a release a few ms early
+# starts it over, while a late one is free; on a mark mid-bar this is a few pixels of the mark's tolerance.
+_LATE_S = 0.06
+# The pace is fitted over this many looks: two alone turn a pixel of edge noise on a slow fill into a release a
+# whole tolerance off, and a widget that eases changes pace over more than a few.
+_PACE_LOOKS = 4
+
+
+class FillError(ValueError):
+    """The line cannot be read: a degenerate segment or shots of different sizes."""
+
+
+def _rgb(img: Image) -> np.ndarray:
+    """The shot's three colour channels, each softened by a 3x3 binomial blur; their order does not matter here."""
+    if isinstance(img, str):
+        try:
+            with PILImage.open(img) as f:
+                a = np.asarray(f.convert("RGB"))
+        except OSError as exc:
+            raise FillError(f"cannot read {img!r}") from exc
+    else:
+        a = np.asarray(img)
+    if a.ndim == 2:
+        a = np.repeat(a[..., None], 3, axis=2)
+    out = a[..., :3].astype(np.float32)
+    for axis in (0, 1):
+        out = ndimage.correlate1d(out, [0.25, 0.5, 0.25], axis=axis, mode="mirror")
+    return out
+
+
+def profile(first: Image, current: Image, start: Point, end: Point) -> Tuple[np.ndarray, float]:
+    """How much each pixel step along start->end (and _PAST beyond) changed since the press, and the line's length."""
+    a, b = _rgb(first), _rgb(current)
+    if a.shape != b.shape:
+        raise FillError(f"shots differ in size: {a.shape[:2]} vs {b.shape[:2]}")
+    h, w = a.shape[:2]
+    (x0, y0), (x1, y1) = (float(v) for v in start), (float(v) for v in end)
+    length = math.hypot(x1 - x0, y1 - y0)
+    if length < 2.0:
+        raise FillError("the fill's start and end are the same point")
+    ux, uy = (x1 - x0) / length, (y1 - y0) / length
+    steps = np.arange(int(math.ceil(length * (1 + _PAST))) + 1, dtype=np.float32)
+    diff = np.abs(a - b).max(axis=2)
+    across = []
+    for o in range(-_BAND_PX, _BAND_PX + 1):
+        xs = np.clip(np.round(x0 + steps * ux - o * uy), 0, w - 1).astype(int)
+        ys = np.clip(np.round(y0 + steps * uy + o * ux), 0, h - 1).astype(int)
+        across.append(diff[ys, xs])
+    # The median across the band: the bar's edge running along one side of the line is not the fill.
+    return np.median(np.stack(across), axis=0), length
+
+
+def _solid(changes: np.ndarray) -> np.ndarray:
+    """The changed steps whose last _RUN_PX steps are mostly changed too."""
+    on = changes > _CHANGED
+    run = np.convolve(on.astype(np.float32), np.ones(_RUN_PX, np.float32), mode="full")[:len(on)]
+    count = np.minimum(np.arange(1, len(on) + 1), _RUN_PX)
+    return on & (run / count >= _RUN_SHARE)
+
+
+def leading_edge(changes: np.ndarray) -> Optional[float]:
+    """The furthest step the fill has reached, or None when nothing along the line has changed."""
+    idx = np.flatnonzero(_solid(changes))
+    return float(idx[-1]) if len(idx) else None
+
+
+def trailing_edge(changes: np.ndarray) -> Optional[float]:
+    """Where the fill was when the reference was shot: the changed run starts there, because what had already filled
+    by then looks the same in both shots."""
+    idx = np.flatnonzero(_solid(changes[::-1]))
+    return float(len(changes) - 1 - idx[-1]) if len(idx) else None
+
+
+def _pace(looks: Sequence[Tuple[float, float]]) -> Optional[float]:
+    """Pixels per second, least squares over (t, edge) looks; None from fewer than two."""
+    if len(looks) < 2:
+        return None
+    ts, es = np.array(looks, dtype=np.float64).T
+    if np.ptp(ts) <= 0:
+        return None
+    return float(np.polyfit(ts, es, 1)[0])
+
+
+def fill_step(first: Image, current: Image, *, start: Point, end: Point, t: float,
+              history: Sequence[Dict[str, Any]] = (), poll_s: float = 0.25, tol: float = 2.0,
+              first_t: Optional[float] = None) -> Dict[str, Any]:
+    """One look at a held widget: how far its fill has come, and whether to release it now.
+
+    `t` is the caller's clock (seconds) when this shot was taken and `history` the `observation` of each earlier
+    look, oldest first; `first_t` is when `first` was shot, which gives the first look a pace of its own (see
+    `trailing_edge`) so a mark close to the start is not passed while two looks are gathered. `release` is true once the edge is within `tol` of `end` or past it, or when at its measured
+    pace the fill reaches `end` before the next look, `poll_s` away. `release_in` is then how long after `t` to let
+    go, a little after the arrival (`_LATE_S`): releasing on the look instead lands up to a whole look's travel past
+    the mark, and a fast fill covers more than a mark's tolerance in one.
+    """
+    changes, length = profile(first, current, start, end)
+    edge = leading_edge(changes)
+    obs = {"t": float(t), "edge": edge}
+    out: Dict[str, Any] = {"edge": edge, "length": length, "observation": obs, "release": False, "release_in": None,
+                           "speed": None}
+    if edge is None:
+        return out
+    short = length - edge
+    seen = [(float(o["t"]), float(o["edge"])) for o in history if o.get("edge") is not None and float(o["t"]) < t]
+    back = trailing_edge(changes) if first_t is not None and float(first_t) < t else None
+    if back is not None and back < edge and all(float(first_t) < ts for ts, _ in seen):
+        seen = [(float(first_t), back)] + seen
+    out["speed"] = _pace(seen[-(_PACE_LOOKS - 1):] + [(t, edge)])
+    if short <= tol:
+        out["release"], out["release_in"] = True, 0.0
+    elif out["speed"] and out["speed"] > 0 and short / out["speed"] <= poll_s:
+        out["release"], out["release_in"] = True, short / out["speed"] + _LATE_S
+    return out

package/python/src/captchakraken/tool_calls/find_checkbox.py

@@ -1,66 +1,21 @@
-"""Where the driver clicks first, before any model runs. Neither a false positive nor a false negative raises."""
+"""Where the driver clicks first, before any model runs: the tick box of a vendor's
+checkbox widget, as the element detector's `checkbox` class finds it. Neither a
+false positive nor a false negative raises."""
 
-import cv2
-import numpy as np
 from typing import Optional, Tuple
 
-def find_checkbox(image_path: str) -> Optional[Tuple[int, int, int, int]]:
-    img = cv2.imread(image_path)
-    if img is None:
-        return None
-
-    gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
-
-    _, binary = cv2.threshold(gray, 200, 255, cv2.THRESH_BINARY_INV)
-
-    contours, _ = cv2.findContours(binary, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
-
-    height, width = img.shape[:2]
-    image_area = width * height
-
-    candidates = []
-
-    for cnt in contours:
-        x, y, w, h = cv2.boundingRect(cnt)
-        area = w * h
-        aspect_ratio = float(w) / h
-        
-
-        if not (0.8 < aspect_ratio < 1.2):
-            continue
+#: Measured on widgets in a language the detector never trained on: every one of
+#: 96 tick boxes scored >= 0.997, a ticked or passed widget 0, and the highest
+#: `checkbox` score on 600 real puzzle boards was 0.145. 0.6 sits far from both.
+MIN_SCORE = 0.6
 
-        # Below 20px a glyph is indistinguishable from a box.
-        if w < 20 or h < 20:
-            continue
 
-        # Above 5% of the image it is a panel or the challenge frame, not the checkbox inside one.
-        if not (0.001 * image_area < area < 0.05 * image_area):
-            continue
-
-        # Every checkbox widget puts its box at the left; a square on the right is a logo or a badge.
-        if x + w / 2 > width / 2:
-            continue
-
-        contour_area = cv2.contourArea(cnt)
-        extent = contour_area / area
-        if extent < 0.8:
-            continue
-
-        # A checkbox is empty inside; without this the first tile of a 3x3 grid gets clicked as the widget.
-        roi = gray[y+5:y+h-5, x+5:x+w-5]
-        if roi.size > 0:
-            std_dev = np.std(roi)
-            if std_dev > 35.0:
-                continue
-
-        candidates.append((x, y, w, h))
-
-    # Many candidates means a grid, not a checkbox page.
-    if len(candidates) > 2:
-        return None
+def find_checkbox(image_path: str) -> Optional[Tuple[int, int, int, int]]:
+    """(x, y, w, h) of the most confident tick box in pixels, or None."""
+    from ..pointer.detector import detect_file
 
-    if not candidates:
+    boxes = [d for d in detect_file(image_path) if d["class"] == "checkbox" and d["score"] >= MIN_SCORE]
+    if not boxes:
         return None
-
-    best_candidate = max(candidates, key=lambda c: c[2] * c[3])
-    return best_candidate
+    x0, y0, x1, y1 = max(boxes, key=lambda d: d["score"])["box"]
+    return int(round(x0)), int(round(y0)), int(round(x1 - x0)), int(round(y1 - y0))