captchakraken 3.2.0 → 4.0.0

package/python/src/captchakraken/pointer/detector.py

@@ -0,0 +1,333 @@
+"""The element detector: a board in, every element on it out, each with a class,
+a box, a score and a mask.
+
+torchvision's Mask R-CNN, in two sizes (`WEIGHTS`): `standard`, a ResNet-18 FPN,
+and `lite`, a MobileNetV3-Large FPN ~6x faster on one CPU. Two runtimes, one network:
+
+  Detector      the torch checkpoint, for serving next to a GPU model (needs the
+                `serve` extra: torch, torchvision)
+  OnnxDetector  the same weights exported to ONNX, on onnxruntime's CPU kernels.
+                This is what the client runs to find a grid or a checkbox before
+                any model is asked; `local()` loads the released weights.
+
+Both return the same detections, so code above them never knows which ran.
+"""
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import tempfile
+from functools import lru_cache, partial
+from pathlib import Path
+from typing import Dict, List, Optional, Sequence, Union
+
+import numpy as np
+
+#: h/w: text inputs are ~9:1 wide, which the stock (0.5, 1, 2) cannot propose.
+ASPECTS = (0.1, 0.25, 0.5, 1.0, 2.0)
+#: Detections this weak are dropped here; `elements.number` applies its own bar.
+MIN_SCORE = 0.3
+
+#: The released ONNX weights, pinned by commit and content: `standard` (ResNet-18
+#: FPN) and `lite` (MobileNetV3-Large FPN, ~6x faster on one CPU).
+WEIGHTS = {
+    "standard": {
+        "repo": "CaptchaKraken/CaptchaKraken-v2-Detector",
+        "revision": "2c47ab53e0158e35299832484889672703058dc7",
+        "file": "detector.onnx",
+        "sha256": "725f18914ad0f8621ed332752b6fde4949bbbe2b589bbac406f47e07fe2a6a71",
+    },
+    "lite": {
+        "repo": "CaptchaKraken/CaptchaKraken-v2-Detector",
+        "revision": "184987101f0b2ad3f2d5e1dfc4697108c4f2d967",
+        "file": "detector-lite.onnx",
+        "sha256": "b654e6b02028e29ccdc4c4ca0f52ba7110c9f064f97d507f03ac308745eb41ad",
+    },
+}
+#: What `auto` may choose from, and the names a checkpoint's `arch` has had.
+MODELS = ("auto", *WEIGHTS)
+_ARCH_NAMES = {"standard": "standard", "lite": "lite", "normal": "standard", "light": "lite"}
+#: `auto` runs `lite` at or below this many usable CPUs without measuring anything.
+LITE_AT_CPUS = 2
+#: Above that, `auto` times `lite` once and runs `standard` only when it is predicted
+#: to finish a board inside this budget. `standard` takes this many times as long as
+#: `lite` on the same CPUs. Measured on 520x570 grid boards, one x86 core per thread:
+#: 5.3x at 1 thread, 5.1x at 2, 4.6x at 4, 3.9x at 8. The ratio falls as threads are
+#: added, so the worst of them, rounded up, never predicts `standard` too fast.
+AUTO_BUDGET_S = 0.2
+STANDARD_OVER_LITE = 5.5
+
+
+def build(num_classes: int, pretrained: bool = False, min_size: Sequence[int] = (512,),
+          max_size: int = 1024, arch: str = "standard"):
+    """`num_classes` excludes background. `pretrained` loads ImageNet weights into
+    the backbone only; the heads always start fresh. `arch` is the checkpoint's
+    (one without it is `standard`); the Finetune repo's `src/detector.py` builds the same."""
+    import torchvision.models as tm
+    from torchvision.models.detection import MaskRCNN
+    from torchvision.models.detection.anchor_utils import AnchorGenerator
+    anchors = AnchorGenerator(((32,), (64,), (128,), (256,), (512,)), (ASPECTS,) * 5)
+    if arch not in _ARCH_NAMES:
+        raise ValueError(f"unknown detector arch {arch!r}: one of {tuple(WEIGHTS)}")
+    arch = _ARCH_NAMES[arch]
+    if arch == "standard":
+        from torchvision.models.detection.backbone_utils import resnet_fpn_backbone
+        backbone = resnet_fpn_backbone(backbone_name="resnet18",
+                                       weights=tm.ResNet18_Weights.DEFAULT if pretrained else None,
+                                       trainable_layers=5)
+        return MaskRCNN(backbone, num_classes=num_classes + 1, rpn_anchor_generator=anchors,
+                        min_size=tuple(min_size), max_size=max_size)
+    import torch.nn as nn
+    from torchvision.models.detection.backbone_utils import BackboneWithFPN
+    from torchvision.models.detection.faster_rcnn import FastRCNNPredictor, TwoMLPHead
+    from torchvision.models.detection.mask_rcnn import MaskRCNNHeads, MaskRCNNPredictor
+    from torchvision.ops.feature_pyramid_network import LastLevelMaxPool
+    from torchvision.ops.misc import FrozenBatchNorm2d
+    features = tm.mobilenet_v3_large(
+        weights=tm.MobileNet_V3_Large_Weights.DEFAULT if pretrained else None,
+        norm_layer=FrozenBatchNorm2d if pretrained else nn.BatchNorm2d).features
+    # The last block at each of strides 4, 8, 16 and 32: small items need stride 4.
+    taps, channels = {"3": "0", "6": "1", "12": "2", "16": "3"}, [24, 40, 112, 960]
+    # A 1024-wide box head like `standard`'s: at 256 the GeeTest bar (a box filling the whole shot)
+    # scored 0.25-0.70 as a checkbox where `standard` scores 0.999, for 12 ms less on one CPU.
+    ch, rep = 128, 1024
+    # GroupNorm: MobileNetV3's stride-4 and stride-8 outputs are unnormalised, 11x and 5x ResNet's scale.
+    backbone = BackboneWithFPN(features, taps, channels, ch, extra_blocks=LastLevelMaxPool(),
+                               norm_layer=partial(nn.GroupNorm, 32))
+    return MaskRCNN(backbone, num_classes=None, rpn_anchor_generator=anchors,
+                    min_size=tuple(min_size), max_size=max_size,
+                    box_head=TwoMLPHead(ch * 7 * 7, rep), box_predictor=FastRCNNPredictor(rep, num_classes + 1),
+                    mask_head=MaskRCNNHeads(ch, [ch, ch], 1),
+                    mask_predictor=MaskRCNNPredictor(ch, ch, num_classes + 1),
+                    # 1000 before NMS: GeeTest's five-in-a-row balls sit on the stride-4 level, the most
+                    # crowded, and at 300 two in three were never proposed. 300 after keeps it at ~100 ms.
+                    rpn_pre_nms_top_n_test=1000, rpn_post_nms_top_n_test=300, box_detections_per_img=64)
+
+
+def _rgb(image):
+    from PIL import Image
+    if isinstance(image, (str, Path)):
+        with Image.open(image) as im:
+            return im.convert("RGB")
+    return image.convert("RGB")
+
+
+def _detections(boxes, labels, scores, masks, classes: Sequence[str], min_score: float) -> List[Dict]:
+    """Raw network output -> [{"class", "box": [x0, y0, x1, y1] px, "score",
+    "mask": ((x, y), bool array)}], best first, the mask cropped to its pixels."""
+    out = []
+    for b, c, s, m in zip(np.asarray(boxes).tolist(), np.asarray(labels).tolist(),
+                          np.asarray(scores).tolist(), np.asarray(masks)[:, 0] > 0.5):
+        if s < min_score:
+            continue
+        ys, xs = np.nonzero(m)
+        mask = (((int(xs.min()), int(ys.min())), m[ys.min():ys.max() + 1, xs.min():xs.max() + 1])
+                if len(xs) else ((int(b[0]), int(b[1])), np.zeros((0, 0), bool)))
+        out.append({"class": classes[c - 1], "box": [round(v, 1) for v in b],
+                    "score": round(s, 4), "mask": mask})
+    return out
+
+
+class Detector:
+    """A checkpoint the Finetune repo's `scripts/train_detector.py` wrote
+    ({"classes", "min_size", "max_size", "state_dict"}); `detect` one image."""
+
+    def __init__(self, checkpoint: Union[str, Path], device: Optional[str] = None):
+        import torch
+        self.torch = torch
+        self.device = device or ("cuda" if torch.cuda.is_available() else "cpu")
+        ck = torch.load(str(checkpoint), map_location=self.device, weights_only=False)
+        self.classes = list(ck["classes"])
+        self.model = build(len(self.classes), min_size=ck["min_size"], max_size=ck["max_size"],
+                           arch=ck.get("arch", "standard"))
+        self.model.load_state_dict(ck["state_dict"])
+        self.model.to(self.device).eval()
+
+    def detect(self, image, min_score: float = MIN_SCORE) -> List[Dict]:
+        """PIL image or path -> detections, best first (see `_detections`)."""
+        from torchvision.transforms.functional import pil_to_tensor
+        x = pil_to_tensor(_rgb(image)).float().div(255).to(self.device)
+        with self.torch.inference_mode():
+            o = self.model([x])[0]
+        return _detections(o["boxes"].cpu(), o["labels"].cpu(), o["scores"].cpu(), o["masks"].cpu(),
+                           self.classes, min_score)
+
+
+class OnnxDetector:
+    """An ONNX export of the same network (image (3, H, W) float 0-1 in; boxes,
+    labels, scores, masks out; the class names in the model's `classes`
+    metadata). CPU only: a board is one image, and the client has no GPU to assume."""
+
+    def __init__(self, model: Union[str, Path]):
+        import onnxruntime as ort
+        so = ort.SessionOptions()
+        # One thread per CPU the process may use, not per core it can see: a container held to
+        # one vCPU would otherwise run a thread for every host core on it.
+        cpus = usable_cpus()
+        if cpus < (os.cpu_count() or 1):
+            so.intra_op_num_threads = max(1, int(cpus))
+        self.session = ort.InferenceSession(str(model), so, providers=["CPUExecutionProvider"])
+        meta = self.session.get_modelmeta().custom_metadata_map
+        if "classes" not in meta:
+            raise ValueError(f"{model} carries no class names: it was not exported for this client")
+        self.classes = json.loads(meta["classes"])
+
+    def detect(self, image, min_score: float = MIN_SCORE) -> List[Dict]:
+        """PIL image or path -> detections, best first (see `_detections`)."""
+        x = np.ascontiguousarray(np.asarray(_rgb(image), np.float32).transpose(2, 0, 1) / 255.0)
+        boxes, labels, scores, masks = self.session.run(None, {"image": x})
+        return _detections(boxes, labels, scores, masks, self.classes, min_score)
+
+
+def usable_cpus() -> float:
+    """The CPUs this process may run on: its affinity, capped by a cgroup quota. A
+    one-vCPU container usually sees every core of its host and is held to one by
+    `cpu.max`, so the core count alone would pick the heavy model there."""
+    try:
+        n = float(len(os.sched_getaffinity(0)))
+    except (AttributeError, OSError):
+        n = float(os.cpu_count() or 1)
+    try:
+        quota, period = Path("/sys/fs/cgroup/cpu.max").read_text().split()[:2]
+        if quota != "max":
+            n = min(n, int(quota) / int(period))
+    except (OSError, ValueError):
+        pass
+    return n
+
+
+def model_name(choice: Optional[str] = None) -> str:
+    """The released model to run. `choice` is the config's `detector` (`auto`,
+    `standard` or `lite`); `CAPTCHA_KRAKEN_DETECTOR_MODEL` overrides it."""
+    name = (os.getenv("CAPTCHA_KRAKEN_DETECTOR_MODEL") or choice or "auto").strip().lower()
+    if name not in MODELS:
+        raise ValueError(f"detector model {name!r}: one of {', '.join(MODELS)}")
+    return _auto() if name == "auto" else name
+
+
+def _auto() -> str:
+    """`lite` on LITE_AT_CPUS usable CPUs or fewer. Otherwise `lite` is timed on a board
+    once, and `standard` runs when STANDARD_OVER_LITE times that fits AUTO_BUDGET_S. The
+    verdict is kept beside the weights, keyed on what it measured, so it runs once a machine."""
+    cpus = usable_cpus()
+    if cpus <= LITE_AT_CPUS:
+        return "lite"
+    import platform
+    from ..config import state_dir
+    key = f"{platform.machine()}|{platform.processor()}|{os.cpu_count()}|{cpus:g}|{WEIGHTS['lite']['sha256'][:16]}"
+    cache = state_dir() / "detector" / "auto.json"
+    try:
+        seen = json.loads(cache.read_text())
+        if seen.get("key") == key and seen.get("model") in WEIGHTS:
+            return seen["model"]
+    except (OSError, ValueError):
+        pass
+    seconds = _time_one_board(OnnxDetector(weights_path("lite")))
+    model = "standard" if seconds * STANDARD_OVER_LITE <= AUTO_BUDGET_S else "lite"
+    try:
+        cache.parent.mkdir(parents=True, exist_ok=True)
+        cache.write_text(json.dumps({"key": key, "model": model, "lite_s": round(seconds, 4)}))
+    except OSError:
+        pass
+    return model
+
+
+def _time_one_board(det: "OnnxDetector") -> float:
+    """The best of three detections on a drawn 3x3 board (one warm-up first)."""
+    import time
+    from PIL import Image, ImageDraw
+    im = Image.new("RGB", (400, 520), (255, 255, 255))
+    d = ImageDraw.Draw(im)
+    d.rectangle((0, 0, 399, 96), fill=(74, 144, 226))
+    for r in range(3):
+        for c in range(3):
+            x, y = 6 + c * 130, 110 + r * 130
+            d.rectangle((x, y, x + 124, y + 124), fill=(40 + 60 * r, 90 + 50 * c, 160 - 40 * r))
+    det.detect(im)
+    best = float("inf")
+    for _ in range(3):
+        t = time.perf_counter()
+        det.detect(im)
+        best = min(best, time.perf_counter() - t)
+    return best
+
+
+def weights_path(model: Optional[str] = None) -> Path:
+    """The ONNX file to run: `CAPTCHA_KRAKEN_DETECTOR` when set, else the pinned
+    release of `model` (`standard` or `lite`; default `model_name()`), downloaded
+    once into the state directory and checked against its hash."""
+    # A local .onnx to run instead of the released one (self-hosting, CI, offline).
+    local = os.getenv("CAPTCHA_KRAKEN_DETECTOR")
+    if local:
+        if not Path(local).is_file():
+            raise FileNotFoundError(f"CAPTCHA_KRAKEN_DETECTOR={local} is not a file")
+        return Path(local)
+    pin = WEIGHTS[model or model_name()]
+    from ..config import state_dir
+    path = state_dir() / "detector" / f"{pin['sha256']}.onnx"
+    if path.is_file():
+        return path
+    import requests
+    url = f"https://huggingface.co/{pin['repo']}/resolve/{pin['revision']}/{pin['file']}"
+    path.parent.mkdir(parents=True, exist_ok=True)
+    digest = hashlib.sha256()
+    # Hugging Face's own variable: a private copy of the weights (CI, a self-hosted mirror) needs it.
+    token = os.getenv("HF_TOKEN")
+    headers = {"Authorization": f"Bearer {token}"} if token else {}
+    with requests.get(url, stream=True, timeout=60, headers=headers) as r:
+        r.raise_for_status()
+        with tempfile.NamedTemporaryFile(dir=path.parent, delete=False) as tf:
+            for chunk in r.iter_content(1 << 20):
+                digest.update(chunk)
+                tf.write(chunk)
+    if digest.hexdigest() != pin["sha256"]:
+        os.unlink(tf.name)
+        raise ValueError(f"{url} does not match the pinned sha256: refusing to run unverified weights")
+    os.replace(tf.name, path)
+    return path
+
+
+_CHOICE: Optional[str] = None
+
+
+def configure(choice: Optional[str]) -> None:
+    """Set the config's `detector` choice (`auto`, `standard`, `lite`) for this process; the next
+    detection loads it. A process that never calls this runs the shipped config file's choice."""
+    global _CHOICE
+    if choice is not None and choice.strip().lower() not in MODELS:
+        raise ValueError(f"detector {choice!r}: one of {', '.join(MODELS)}")
+    if choice != _CHOICE:
+        _CHOICE = choice
+        local.cache_clear()
+        _detect_file.cache_clear()
+
+
+@lru_cache(maxsize=1)
+def local() -> OnnxDetector:
+    """The client's detector, loaded once per process."""
+    if os.getenv("CAPTCHA_KRAKEN_DETECTOR"):
+        return OnnxDetector(weights_path())
+    if _CHOICE is None:
+        # Nothing configured this process (the CLI, a script): the shipped config file's choice.
+        from ..selectors import DEFAULT_CONFIG
+        return OnnxDetector(weights_path(model_name(DEFAULT_CONFIG.detector)))
+    return OnnxDetector(weights_path(model_name(_CHOICE)))
+
+
+@lru_cache(maxsize=8)
+def _detect_file(path: str, mtime_ns: int, size: int) -> List[Dict]:
+    return local().detect(path)
+
+
+def detect_file(path: Union[str, Path]) -> List[Dict]:
+    """`local().detect` on an image file, once per version of the file: the grid
+    and the checkbox are both asked of the same shot. Treat the result as read-only.
+    A file that is missing or is not an image shows no elements: a capture that
+    failed is a board with nothing on it to click, not a crash."""
+    from PIL import UnidentifiedImageError
+    try:
+        st = os.stat(path)
+        return _detect_file(str(path), st.st_mtime_ns, st.st_size)
+    except (FileNotFoundError, UnidentifiedImageError):
+        return []

package/python/src/captchakraken/pointer/elements.py

@@ -0,0 +1,273 @@
+"""Detector output -> the numbered elements a board is shown with.
+
+An element is {"class", "box": [x0, y0, x1, y1], "score", "mask": ((x, y), bool
+array)} in board pixels; the mask is cropped to its own extent and placed at
+(x, y), as `detector.Detector.detect` returns it. Numbers are 1-based in reading
+order (rows of ~12 px, then left to right), so a grid's cells number row-major.
+
+`element_at` is the inverse question training asks: which numbered element does
+a gold point land on? That decides the id a slot is trained to give.
+"""
+from __future__ import annotations
+
+from typing import Dict, List, Optional, Sequence, Tuple
+
+import numpy as np
+
+from .layout import MAX_ELEMENTS
+
+#: Detector classes that are numbered. `grid` is acted on through its cells, and
+#: `click_area` is the well a free click lands in: it holds every point on the
+#: board and so names none of them. The detector's `submit` is a button. A
+#: `fill` is what a press-and-hold widget fills: the bar or ring a hold's
+#: `from`/`to` lie on.
+NUMBERED = ("cell", "item", "draggable", "drop_target", "slider", "button", "text_input", "fill")
+RENAME = {"submit": "button"}
+#: A point this close to an element's mask (px) still lands on it: anti-aliased edges.
+EDGE_PX = 3
+#: Detections below this score are not shown. A spare outline costs the model one
+#: more option to reject; a missing one cannot be chosen at all.
+MIN_SCORE = 0.5
+
+
+#: A cell whose box overlaps a piece's or slot's this much (IoU) IS that piece or
+#: slot: a drag grid labels every position as both.
+SAME_PLACE_IOU = 0.6
+
+
+def number(dets: Sequence[Dict], min_score: float = MIN_SCORE,
+           limit: Optional[int] = MAX_ELEMENTS) -> List[Dict]:
+    """Detector output -> the numbered elements, in reading order, at most `limit`
+    (None: all, to count a board that has too many). A cell that a piece, slot
+    or item fills is numbered once, as that element: it is one place to act
+    on, and the specific class says what is there."""
+    els = []
+    for d in dets:
+        cls = RENAME.get(d["class"], d["class"])
+        if cls in NUMBERED and d["score"] >= min_score:
+            els.append({**d, "class": cls})
+    filled = [e["box"] for e in els if e["class"] in ("item", "draggable", "drop_target")]
+    els = [e for e in els if e["class"] != "cell" or all(_iou(e["box"], b) < SAME_PLACE_IOU for b in filled)]
+    els.sort(key=lambda d: (round(((d["box"][1] + d["box"][3]) / 2) / 12),
+                            (d["box"][0] + d["box"][2]) / 2))
+    return els[:limit]
+
+
+#: Two detections on neighbouring frames are one element when their boxes overlap
+#: this much (IoU) and their class agrees. A sprite moving under ~1/3 of its width
+#: a frame stays linked; a clip's frames are 0.1-0.5 s apart.
+TRACK_IOU = 0.3
+
+
+def _iou(a: Sequence[float], b: Sequence[float]) -> float:
+    ix = max(0.0, min(a[2], b[2]) - max(a[0], b[0]))
+    iy = max(0.0, min(a[3], b[3]) - max(a[1], b[1]))
+    inter = ix * iy
+    union = (a[2] - a[0]) * (a[3] - a[1]) + (b[2] - b[0]) * (b[3] - b[1]) - inter
+    return inter / union if union > 0 else 0.0
+
+
+def track(per_frame: Sequence[Sequence[Dict]]) -> List[Dict]:
+    """A video's per-frame numbered elements -> its elements across the clip.
+
+    Each track is {"class", "frames": [element | None, ...]} (one entry per frame),
+    linked greedily frame to frame by box IoU >= TRACK_IOU within a class; a track
+    that misses a frame can be picked up again on the next (a fading sprite dips
+    under the score cut for a frame). Tracks are numbered in reading order of
+    where each first appears, then by that frame, so an element keeps ONE number
+    for the whole clip and the static parts of a board number as a still would."""
+    n = len(per_frame)
+    tracks: List[Dict] = []
+    for f, els in enumerate(per_frame):
+        free = list(range(len(els)))
+        pairs = []
+        for t, tr in enumerate(tracks):
+            last = next((tr["frames"][g] for g in range(f - 1, max(-1, f - 3), -1)
+                         if tr["frames"][g] is not None), None)
+            if last is None:
+                continue
+            for i in free:
+                if els[i]["class"] == tr["class"]:
+                    v = _iou(last["box"], els[i]["box"])
+                    if v >= TRACK_IOU:
+                        pairs.append((v, t, i))
+        used_t, used_i = set(), set()
+        for v, t, i in sorted(pairs, reverse=True):
+            if t in used_t or i in used_i:
+                continue
+            tracks[t]["frames"][f] = els[i]
+            used_t.add(t)
+            used_i.add(i)
+        for i in free:
+            if i not in used_i:
+                tr = {"class": els[i]["class"], "frames": [None] * n}
+                tr["frames"][f] = els[i]
+                tracks.append(tr)
+
+    def first(tr):
+        f = next(g for g, e in enumerate(tr["frames"]) if e is not None)
+        b = tr["frames"][f]["box"]
+        return (round(((b[1] + b[3]) / 2) / 12), (b[0] + b[2]) / 2, f)
+
+    return sorted(tracks, key=first)[:MAX_ELEMENTS]
+
+
+def moment_frame(moment: int, n_frames: int) -> int:
+    """The frame a video moment's elements are read on: the first of its pair
+    (the processor merges frames 2m and 2m+1 into moment m)."""
+    return min(2 * moment, n_frames - 1)
+
+
+def on_frame(tracks: Sequence[Dict], f: int) -> List[Optional[Dict]]:
+    """Frame `f`'s elements, indexed by track (None where a track is absent), so
+    `element_at` over it answers with the TRACK's index."""
+    return [tr["frames"][f] for tr in tracks]
+
+
+def scale(els: Sequence[Dict], fx: float, fy: float) -> List[Dict]:
+    """Elements detected on one size of a board, moved onto another (fx, fy = new/old)."""
+    out = []
+    for e in els:
+        (x, y), m = e["mask"]
+        x0, y0 = int(round(x * fx)), int(round(y * fy))
+        w, h = max(1, int(round(m.shape[1] * fx))), max(1, int(round(m.shape[0] * fy)))
+        mm = _nearest(m.astype(bool), w, h) if m.size else np.zeros((0, 0), bool)
+        b = e["box"]
+        out.append({**e, "box": [b[0] * fx, b[1] * fy, b[2] * fx, b[3] * fy], "mask": ((x0, y0), mm)})
+    return out
+
+
+def _solid(mask) -> Tuple[Tuple[int, int], np.ndarray]:
+    """The mask with its enclosed holes filled (a ring's centre is ON the ring) and
+    grown by EDGE_PX. An open gap (a chain's break) is not enclosed, so it stays out."""
+    (x, y), m = mask
+    m = _grow(fill_holes(np.pad(m.astype(bool), EDGE_PX + 1)), EDGE_PX)
+    return (x - EDGE_PX - 1, y - EDGE_PX - 1), m
+
+
+def _nearest(m: np.ndarray, w: int, h: int) -> np.ndarray:
+    """`m` resized to (w, h) by nearest neighbour, sampling the pixels OpenCV's INTER_NEAREST does."""
+    def index(src: int, dst: int) -> np.ndarray:
+        return np.minimum(np.floor(np.arange(dst) * (1.0 / (dst / src))).astype(np.int64), src - 1)
+    return m[index(m.shape[0], h)][:, index(m.shape[1], w)]
+
+
+def _grow(m: np.ndarray, r: int) -> np.ndarray:
+    """`m` dilated by a (2r+1)-pixel square."""
+    win = np.lib.stride_tricks.sliding_window_view
+    p = win(np.pad(m, r), 2 * r + 1, axis=0).any(-1)
+    return win(p, 2 * r + 1, axis=1).any(-1)
+
+
+def _spread_along_rows(reached: np.ndarray, free: np.ndarray) -> np.ndarray:
+    """`reached` extended over every run of `free` pixels in a row that it touches."""
+    starts = free & ~np.pad(free, ((0, 0), (1, 0)))[:, :-1]
+    run = np.cumsum(starts).reshape(free.shape) * free
+    hit = np.zeros(int(run.max()) + 1, bool)
+    hit[run[reached & free]] = True
+    hit[0] = False
+    return hit[run]
+
+
+def fill_holes(m: np.ndarray) -> np.ndarray:
+    """`m` with its holes filled: every background pixel not 4-connected to the array's edge.
+
+    Spreads along whole rows and columns of background at a time, so it converges in as many passes as the
+    background's path to the edge has turns, not pixels.
+    """
+    free = ~m
+    reached = np.zeros_like(free)
+    reached[[0, -1], :] = free[[0, -1], :]
+    reached[:, [0, -1]] |= free[:, [0, -1]]
+    while True:
+        grown = _spread_along_rows(_spread_along_rows(reached, free).T, free.T).T
+        if np.array_equal(grown, reached):
+            return ~reached
+        reached = grown
+
+
+def element_at(pt: Sequence[float], size: Tuple[int, int], els: Sequence[Dict]) -> Optional[int]:
+    """0-based index of the smallest element (by mask area) whose mask holds the
+    0-1000 point `pt` on a board of `size`, or None. A None in `els` (a track
+    absent from this frame, `on_frame`) holds nothing but keeps its index."""
+    x = int(round(pt[0] * size[0] / 1000.0))
+    y = int(round(pt[1] * size[1] / 1000.0))
+    best, area = None, None
+    for i, e in enumerate(els):
+        if e is None or not e["mask"][1].size:
+            continue
+        (mx, my), m = _solid(e["mask"])
+        if 0 <= y - my < m.shape[0] and 0 <= x - mx < m.shape[1] and m[y - my, x - mx]:
+            a = int(e["mask"][1].sum())
+            if area is None or a < area:
+                best, area = i, a
+    return best
+
+
+def _on_mask(pt: Sequence[float], size: Tuple[int, int],
+             els: Sequence[Optional[Dict]]) -> List[int]:
+    point = [int(round(v)) for v in pt]
+    if element_at(point, size, els) is not None:
+        return point
+    x = int(round(point[0] * size[0] / 1000.0))
+    y = int(round(point[1] * size[1] / 1000.0))
+    best = None
+    for i, e in enumerate(els):
+        if e is None or not e["mask"][1].size:
+            continue
+        x0, y0, x1, y1 = e["box"]
+        if not x0 <= x < x1 or not y0 <= y < y1:
+            continue
+        (mx, my), mask = e["mask"]
+        yy, xx = np.nonzero(mask)
+        if not len(xx):
+            continue
+        distances = (xx + mx - x) ** 2 + (yy + my - y) ** 2
+        k = int(np.argmin(distances))
+        candidate = (int(distances[k]), int(mask.sum()), i,
+                     [round((xx[k] + mx) * 1000 / size[0]), round((yy[k] + my) * 1000 / size[1])])
+        if best is None or candidate[:3] < best[:3]:
+            best = candidate
+    return best[3] if best is not None else point
+
+
+def snap_to_elements(answer: Dict, size: Tuple[int, int],
+                     els: Sequence[Optional[Dict]]) -> Dict:
+    """Move a point that misses its element's mask to the nearest mask pixel.
+
+    Only a point inside an element's box can move, so intentional empty-space
+    points stay put. This keeps clicks on open shapes such as letters off their holes.
+    """
+    out = dict(answer)
+    if out.get("points") is not None:
+        out["points"] = [_on_mask(p, size, els) for p in out["points"]]
+    if out.get("drags") is not None:
+        out["drags"] = [{**d, "from": _on_mask(d["from"], size, els),
+                         "to": _on_mask(d["to"], size, els)} for d in out["drags"]]
+    for key in ("handle", "from", "to", "press"):
+        if out.get(key) is not None:
+            out[key] = _on_mask(out[key], size, els)
+    return out
+
+
+def ids_for(answer: Dict, size: Tuple[int, int], els: Sequence[Optional[Dict]]) -> Dict:
+    """`answer` (generation 3's shape, 0-1000 points) with the 1-based number of
+    the element under each point beside it: `ids` for clicks, `from_id`/`to_id`
+    for drags, `handle_id` for a slide, `press_id` for a hold. A point on no element gets None, and a
+    board with no elements gets no ids at all. `els` is what the board was shown
+    with (a video's: `on_frame` of the answer's moment)."""
+    if not any(e is not None for e in els):
+        return answer
+    def n(p):
+        i = element_at(p, size, els)
+        return None if i is None else i + 1
+    out = snap_to_elements(answer, size, els)
+    if out.get("points") is not None:
+        out["ids"] = [n(p) for p in out["points"]]
+    if out.get("drags") is not None:
+        out["drags"] = [{**d, "from_id": n(d["from"]), "to_id": n(d["to"])} for d in out["drags"]]
+    if out.get("handle") is not None:
+        out["handle_id"] = n(out["handle"])
+    if out.get("press") is not None:
+        out["press_id"] = n(out["press"])
+    return out