captchakraken 3.2.1 → 4.0.0

package/python/src/captchakraken/pointer/heads.py

@@ -0,0 +1,163 @@
+"""The pointer heads: hidden states of one prefill in, decisions out.
+
+Shared by training (the Finetune repo's `src/pointer/model.py`, on transformers
+hidden states) and serving (`vllm_plugin`, inside vLLM's pooler), so the two
+read the same positions with the same arithmetic.
+
+Option vectors, all from the last hidden layer:
+    token    each vision token: where a point can land
+    moment   each `<|vision_end|>`: one per frame pair of a video (one for a still)
+and one query vector per question, at its `<|quad_end|>`. The element a point
+names is the one under the point (`elements.ids_for`). Each slot ALSO reads the
+number drawn on its element, 1..MAX_ELEMENTS or 0 for none, from its query alone:
+a second answer to "which element", decided apart from the point, so the two can
+check each other. On a plain board there are no numbers and it answers 0.
+
+Each point also carries its runner-up tokens and each typed character its
+runner-up, with their probabilities (`layout.variants`): a retry after a refused
+answer takes the next-likeliest one instead of asking the same question again.
+
+A point is a softmax over the chosen moment's tokens plus an offset inside the
+winning token, so a coordinate is never decoded digit by digit. The offset head
+normalises its input and answers linearly, clamped to the token only when
+deciding: the hidden states it reads are large, and behind a final tanh a single
+optimiser step could saturate it for good, leaving every point at its token's
+centre. A typed answer
+reads each `textK` marker through the language model's own head, one token per
+marker, in parallel.
+"""
+from __future__ import annotations
+
+import math
+from typing import Dict, List, Optional
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+from . import layout as L
+
+
+class Positions:
+    """Where each kind of state sits in one prompt, from its token ids."""
+
+    def __init__(self, ids: torch.Tensor, token_ids: Dict[str, int]):
+        ids = ids.flatten()
+        self.q = (ids == token_ids[L.Q_END]).nonzero().flatten()
+        vis = (ids == token_ids[L.IMAGE_PAD]) | (ids == token_ids[L.VIDEO_PAD])
+        self.vis = vis.nonzero().flatten()
+        # A moment ends where its frame pair's tokens do. The chat template wraps a
+        # whole video in one more vision_start/end pair, whose end follows another
+        # end, not a vision token, and so is not a moment.
+        after_vis = torch.zeros_like(vis)
+        after_vis[1:] = vis[:-1]
+        self.moments = ((ids == token_ids[L.VISION_END]) & after_vis).nonzero().flatten()
+        # Earlier screens (`layout.chat`'s history) are images before the board:
+        # context the questions see, never a place a point lands. Only the last
+        # image is the board.
+        self.history = 0
+        if (ids == token_ids[L.IMAGE_PAD]).any() and self.moments.numel() > 1:
+            if (ids == token_ids[L.VIDEO_PAD]).any():
+                raise ValueError("a video board carries no earlier screens")
+            self.history = int(self.moments.numel()) - 1
+            self.vis = self.vis[self.vis > self.moments[-2]]
+            self.moments = self.moments[-1:]
+        if self.q.numel() != len(L.QUESTIONS):
+            raise ValueError(f"{self.q.numel()} question markers, expected {len(L.QUESTIONS)}")
+        n_m = max(int(self.moments.numel()), 1)
+        if self.vis.numel() == 0 or self.vis.numel() % n_m:
+            raise ValueError(f"{self.vis.numel()} vision tokens do not split into {n_m} moments")
+        self.per_moment = self.vis.numel() // n_m
+        self.n_moments = n_m
+
+
+def token_ids(tokenizer) -> Dict[str, int]:
+    return {t: tokenizer.convert_tokens_to_ids(t)
+            for t in (L.Q_END, L.IMAGE_PAD, L.VIDEO_PAD, L.VISION_END)}
+
+
+class Heads(nn.Module):
+    def __init__(self, d: int, dp: int = 256, text_rank: int = 256):
+        super().__init__()
+        self.action = nn.Linear(d, len(L.ACTIONS))
+        self.count = nn.Linear(d, L.N_COUNTS)
+        self.drag_count = nn.Linear(d, L.N_DRAG_COUNTS)
+        self.pq = nn.ModuleDict({k: nn.Linear(d, dp) for k in L.KINDS})
+        self.pk = nn.Linear(d, dp)
+        self.mq, self.mk = nn.Linear(d, dp), nn.Linear(d, dp)
+        self.offset = nn.Sequential(nn.LayerNorm(2 * d), nn.Linear(2 * d, 512), nn.GELU(), nn.Linear(512, 2))
+        self.elem = nn.ModuleDict({k: nn.Sequential(nn.LayerNorm(d), nn.Linear(d, L.MAX_ELEMENTS + 1))
+                                   for k in L.KINDS})
+        # A low-rank nudge before the language model's head, zero at the start, so
+        # a `textK` marker begins as the model's own next-token guess.
+        self.text_down = nn.Linear(d, text_rank, bias=False)
+        self.text_up = nn.Linear(text_rank, d, bias=False)
+        nn.init.zeros_(self.text_up.weight)
+        self.scale = 1 / math.sqrt(dp)
+
+    # ── scores (training reads these; `decide` takes their argmax) ───────────
+    def point(self, kind: str, hq, htok):                 # [d], [P,d] -> [P]
+        return (self.pk(htok) @ self.pq[kind](hq)) * self.scale
+
+    def moment(self, hq, hm):                             # [d], [M,d] -> [M]
+        return (self.mk(hm) @ self.mq(hq)) * self.scale
+
+    def off(self, hq, htok):                              # [d], [d] -> [2], in tokens, unclamped
+        return self.offset(torch.cat([hq, htok], -1))
+
+    def element(self, kind: str, hq):                     # [d] -> [MAX_ELEMENTS + 1]; 0 is none
+        return self.elem[kind](hq)
+
+    def text(self, hq, lm_weight: torch.Tensor):          # [T,d], [V,d] -> [T,V]
+        z = hq.float() + self.text_up(self.text_down(hq.float()))
+        return z.to(lm_weight.dtype) @ lm_weight.T
+
+    # ── reading one prompt ───────────────────────────────────────────────────
+    def views(self, h: torch.Tensor, pos: Positions) -> Dict:
+        """h [N,d] (float32) -> the option and query vectors of one prompt."""
+        hm = h[pos.moments] if pos.moments.numel() else h[pos.vis].mean(0, keepdim=True)
+        return {"q": dict(zip(L.QUESTIONS, h[pos.q])),
+                "tok": h[pos.vis].reshape(pos.n_moments, pos.per_moment, -1), "moment": hm}
+
+    @torch.no_grad()
+    def decide(self, h: torch.Tensor, pos: Positions, lm_weight: Optional[torch.Tensor]) -> torch.Tensor:
+        """h [N,d] -> the packed decision vector (`layout.unpack` reads it).
+        `lm_weight` is the language model's output matrix [V,d], for typed text."""
+        v = self.views(h.float(), pos)
+        q = v["q"]
+        parts: List[torch.Tensor] = [
+            F.softmax(self.action(q["action"]), -1),
+            F.softmax(self.count(q["count"]), -1),
+            F.softmax(self.drag_count(q["count"]), -1),
+        ]
+        pm = F.softmax(self.moment(q["moment"], v["moment"]), -1)
+        m = int(pm.argmax())
+        tok = v["tok"][m]
+        alts: List[torch.Tensor] = []
+        for name, kind in L.SLOTS:
+            pp = F.softmax(self.point(kind, q[name], tok), -1)
+            top = pp.topk(min(1 + L.ALTS, pp.numel())).indices.tolist()
+            t = top[0]
+            dx, dy = self.off(q[name], tok[t]).clamp(-0.5, 0.5)
+            pe = F.softmax(self.element(kind, q[name]), -1)
+            e = int(pe.argmax())
+            parts.append(torch.stack([torch.tensor(float(t), device=h.device), dx, dy, pp[t],
+                                      torch.tensor(float(e), device=h.device), pe[e]]))
+            for k in range(1, 1 + L.ALTS):
+                if k < len(top):
+                    a = top[k]
+                    adx, ady = self.off(q[name], tok[a]).clamp(-0.5, 0.5)
+                    alts.append(torch.stack([torch.tensor(float(a), device=h.device), adx, ady, pp[a]]))
+                else:      # a board of one token has no runner-up: p 0 is never chosen
+                    alts.append(torch.tensor([float(t), 0.0, 0.0, 0.0], device=h.device))
+        if lm_weight is not None:
+            hq = torch.stack([q[n] for n in L.TEXT])
+            pt, it = F.softmax(self.text(hq, lm_weight).float(), -1).topk(2, -1)
+            ids, text_alts = it[:, 0].float(), [pt[:, 0], it[:, 1].float(), pt[:, 1]]
+        else:
+            ids = torch.full((L.MAX_TEXT,), -1.0, device=h.device)
+            text_alts = [torch.zeros(L.MAX_TEXT, device=h.device), ids, torch.zeros(L.MAX_TEXT, device=h.device)]
+        parts += [ids, torch.tensor([float(m), float(pm[m])], device=h.device), *alts, *text_alts]
+        out = torch.cat([p.float().flatten() for p in parts])
+        assert out.numel() == L.PACKED, (out.numel(), L.PACKED)
+        return out

package/python/src/captchakraken/pointer/layout.py

@@ -0,0 +1,319 @@
+"""What the pointer model is asked, and how its answer comes back.
+
+The prompt is the SAME text for every captcha, with the board between its two
+halves:
+
+    <|im_start|>user
+    Solve the captcha. ...                      PREFIX: identical on every request
+    <|vision_start|>[image | video]<|vision_end|>
+    action<|quad_end|> count<|quad_end|> click1<|quad_end|> ... moment<|quad_end|>
+
+Everything up to the board is one cached prefix. The question markers have to
+come AFTER the board: a causal model's state at a marker can only see what is
+before it. They are fixed text too, so the board is the only thing a request
+changes. Nothing about a board's elements is written in the prompt; a marked
+board carries them as drawn outlines and numbers (`marks`), and the element a
+point names is the one under it (`elements.ids_for`).
+
+The heads read the hidden state at every `<|quad_end|>` (one per question), at
+every vision token (the places a point can land) and at every `<|vision_end|>`
+(one per moment of a video). All of those are found from the token ids alone, so
+the same code reads them in training (transformers) and in serving (vLLM's
+pooler).
+
+The heads' decisions travel as one flat float vector (`pack` / `unpack`): a
+pooling endpoint returns a tensor, not JSON. A point comes back as a vision
+token index plus an offset inside it; `to_point` turns it into 0-1000 with the
+board's token grid, which only the caller knows.
+"""
+from __future__ import annotations
+
+import copy
+import math
+from typing import Dict, List, Optional, Sequence, Tuple
+
+from ..clipvideo import frame_size
+
+ACTIONS = ("click", "drag", "slide", "hold", "type", "refresh")
+MAX_CLICKS = 16
+MAX_DRAGS = 4
+MAX_TEXT = 16
+MAX_ELEMENTS = 48
+#: Earlier screens of the same puzzle a request may carry before the board
+#: (`chat`'s `history`): FunCaptcha's pager shows 7 options and a solve may page
+#: back, so this covers a solve that looks at each option once and returns.
+MAX_HISTORY = 8
+#: A still is resized to this many pixels at most, on the same 64 px lattice a
+#: video frame is (`clipvideo.frame_size`), so neither processor resizes it again
+#: and a token is always the same 32 px square of the board.
+MAX_PIXELS = 720 * 720
+#: Pixels per vision token: Qwen3.5's 16 px patch, merged 2x2.
+TOKEN_PX = 32
+
+Q_END = "<|quad_end|>"
+IMAGE_PAD = "<|image_pad|>"
+VIDEO_PAD = "<|video_pad|>"
+VISION_END = "<|vision_end|>"
+#: What vLLM's video processor searches the prompt for. It replaces this whole
+#: span with the per-pair expansion, so a request that carries only `VIDEO_PAD`
+#: gives it nothing to find (`Failed to apply prompt replacement`). Training's
+#: transformers processor instead wraps `chat`'s bare pad in the outer pair
+#: itself, which is why `chat` stays bare and only the serving request is wrapped.
+VLLM_VIDEO = "<|vision_start|><|video_pad|><|vision_end|>"
+
+#: (question, kind). A kind has its own query projections: what a click names
+#: differs from what a drag picks up (`pick`), where it lets go (`drop`), and what
+#: a slider presses. A slide's `from` is picked up and its `to` dropped on; a
+#: hold presses like a handle, and its fill runs from `from` to where it is let go.
+SLOTS: List[Tuple[str, str]] = (
+    [(f"click{k}", "click") for k in range(1, MAX_CLICKS + 1)]
+    + [(f"drag{k} {end}", kind) for k in range(1, MAX_DRAGS + 1)
+       for end, kind in (("from", "pick"), ("to", "drop"))]
+    + [("slide handle", "handle"), ("slide from", "pick"), ("slide to", "drop")]
+    + [("hold press", "handle"), ("hold from", "pick"), ("hold to", "drop")]
+)
+KINDS = ("click", "pick", "drop", "handle")
+TEXT = [f"text{k}" for k in range(1, MAX_TEXT + 1)]
+QUESTIONS: List[str] = ["action", "count"] + [s for s, _ in SLOTS] + TEXT + ["moment"]
+
+INSTRUCTION = (
+    "Solve the captcha. Read its instruction and decide what a person would do. "
+    "Thin numbered outlines, where drawn, are ours and not part of the puzzle."
+)
+#: The fixed text after the board: one marker per question.
+QUESTION_TEXT = " ".join(f"{q}{Q_END}" for q in QUESTIONS)
+
+# ── the packed decision vector ──────────────────────────────────────────────
+#: Per slot: vision token index (within the chosen moment), dx, dy (offset from
+#: the token's centre, in tokens), p(token), the element number read off the
+#: marks (0 for none), p(element).
+SLOT_WIDTH = 6
+N_COUNTS = MAX_CLICKS + 1          # clicks 0..16
+N_DRAG_COUNTS = MAX_DRAGS          # drags 1..4
+#: Runner-up places per slot, after the decisions: each slot's next-likeliest
+#: tokens (index, dx, dy, p), and each `textK` marker's p(token) with its
+#: runner-up token and that one's p. A retry reads them (`variants`) when the
+#: widget refused the first answer, so it never has to ask the same thing twice.
+ALTS = 2
+ALT_WIDTH = 4
+PACKED = (len(ACTIONS) + N_COUNTS + N_DRAG_COUNTS + SLOT_WIDTH * len(SLOTS)
+          + MAX_TEXT + 2 + ALTS * ALT_WIDTH * len(SLOTS) + 3 * MAX_TEXT)
+
+
+def board_size(width: int, height: int) -> Tuple[int, int]:
+    """(W, H) a still is shown at."""
+    h, w = frame_size(height, width, MAX_PIXELS)
+    return w, h
+
+
+def token_grid(size: Tuple[int, int]) -> Tuple[int, int]:
+    """(columns, rows) of vision tokens on a board of `size` = board_size(...)."""
+    return size[0] // TOKEN_PX, size[1] // TOKEN_PX
+
+
+def to_norm(x: float, y: float, size: Tuple[int, int]) -> List[int]:
+    return [int(round(x * 1000.0 / size[0])), int(round(y * 1000.0 / size[1]))]
+
+
+def chat(processor, video: bool, history: int = 0) -> str:
+    """The whole prompt string, as training builds it and serving sends it: the
+    same for every board (see the module docstring).
+
+    `history`: how many earlier screens of the same puzzle come before the board,
+    oldest first, each an image exactly as it was sent then. A solve's next
+    request is then this one up to its board plus one more screen, so the cached
+    prefix covers every screen already seen; the questions after the last image
+    see them all, and points land on the last only (`heads.Positions`). A video
+    is one look and has none.
+
+    This is the TRAINING form: transformers' processor wraps the bare pad in the
+    outer `<|vision_start|>...<|vision_end|>` pair itself. A vLLM request must
+    instead carry `VLLM_VIDEO`, because vLLM searches the prompt for that whole
+    span and replaces it with its per-pair expansion."""
+    if video and history:
+        raise ValueError("a video board carries no earlier screens")
+    if not 0 <= history <= MAX_HISTORY:
+        raise ValueError(f"{history} earlier screens; at most {MAX_HISTORY}")
+    media = {"type": "video"} if video else {"type": "image"}
+    msgs = [{"role": "user", "content": [{"type": "text", "text": INSTRUCTION},
+                                         *({"type": "image"} for _ in range(history)), media,
+                                         {"type": "text", "text": QUESTION_TEXT}]}]
+    text = processor.apply_chat_template(msgs, tokenize=False, add_generation_prompt=False)
+    return text.replace("<|vision_start|><|video_pad|><|vision_end|>", VIDEO_PAD)
+
+
+def unpack(v: Sequence[float]) -> Dict:
+    """The heads' flat vector -> {"action_p", "count_p", "drag_count_p", "slots":
+    {name: {...}}, "text_ids", "moment", "moment_p"}."""
+    v = [float(x) for x in v]
+    if len(v) != PACKED:
+        raise ValueError(f"decision vector has {len(v)} values, expected {PACKED}")
+    i = 0
+
+    def take(n):
+        nonlocal i
+        out = v[i:i + n]
+        i += n
+        return out
+    out = {"action_p": take(len(ACTIONS)), "count_p": take(N_COUNTS),
+           "drag_count_p": take(N_DRAG_COUNTS), "slots": {}}
+    for name, _ in SLOTS:
+        tok, dx, dy, pt, el, pe = take(SLOT_WIDTH)
+        out["slots"][name] = {"token": int(tok), "dx": dx, "dy": dy, "p_point": pt,
+                              "element": int(el), "p_element": pe}
+    out["text_ids"] = [int(t) for t in take(MAX_TEXT)]
+    out["moment"], out["moment_p"] = take(2)
+    out["moment"] = int(out["moment"])
+    for name, _ in SLOTS:
+        alts = []
+        for _k in range(ALTS):
+            tok, dx, dy, pt = take(ALT_WIDTH)
+            alts.append({"token": int(tok), "dx": dx, "dy": dy, "p_point": pt})
+        out["slots"][name]["alts"] = alts
+    out["text_p"] = take(MAX_TEXT)
+    out["text_alt_ids"] = [int(t) for t in take(MAX_TEXT)]
+    out["text_alt_p"] = take(MAX_TEXT)
+    return out
+
+
+def to_point(slot: Dict, grid: Tuple[int, int]) -> List[int]:
+    """A slot's token + offset -> [x, y] on 0-1000, for a token grid (cols, rows)."""
+    cols, rows = grid
+    r, c = divmod(slot["token"], cols)
+    x = (c + 0.5 + slot["dx"]) / cols * 1000
+    y = (r + 0.5 + slot["dy"]) / rows * 1000
+    return [int(round(min(max(x, 0), 1000))), int(round(min(max(y, 0), 1000)))]
+
+
+def answer(decision: Dict, grid: Tuple[int, int], *, text: Optional[str] = None,
+           timestamps: Optional[Sequence[float]] = None) -> Dict:
+    """Unpacked decisions -> the generation-3 answer shape, points only
+    (`elements.ids_for` adds the element numbers). `text` is the decoded typed
+    string (the caller owns the tokenizer); `timestamps` are a video's pair times.
+
+    `checks` says how sure each decision was: the action's, the count's and the
+    moment's probability, and per slot used, in answer order, p(point) and the
+    element number read off the marks with its probability (None: no number)."""
+    def p(name):
+        return to_point(decision["slots"][name], grid)
+    a = decision.get("action", max(range(len(ACTIONS)), key=lambda k: decision["action_p"][k]))
+    act = ACTIONS[a]
+    out: Dict = {}
+    checks: Dict = {"action_p": round(decision["action_p"][a], 4)}
+    if timestamps:
+        out["t"] = float(timestamps[min(decision["moment"], len(timestamps) - 1)])
+        checks["moment_p"] = round(decision["moment_p"], 4)
+    out["action"] = act
+    names: List[str] = []
+    if act in ("click", "refresh"):
+        n = decision.get("count", max(range(N_COUNTS), key=lambda k: decision["count_p"][k]))
+        checks["count_p"] = round(decision["count_p"][n], 4)
+        if act == "refresh":
+            n = min(n, 1)
+        names = [f"click{k}" for k in range(1, n + 1)]
+        out["points"] = [p(s) for s in names]
+    elif act == "drag":
+        k = decision.get("drag_count", max(range(N_DRAG_COUNTS), key=lambda k: decision["drag_count_p"][k]))
+        checks["count_p"] = round(decision["drag_count_p"][k], 4)
+        drags = []
+        for j in range(1, k + 2):
+            drags.append({"from": p(f"drag{j} from"), "to": p(f"drag{j} to")})
+            names += [f"drag{j} from", f"drag{j} to"]
+        out["drags"] = drags
+    elif act == "slide":
+        names = ["slide handle", "slide from", "slide to"]
+        out.update({"handle": p("slide handle"), "from": p("slide from"), "to": p("slide to")})
+    elif act == "hold":
+        names = ["hold press", "hold from", "hold to"]
+        out.update({"press": p("hold press"), "from": p("hold from"), "to": p("hold to")})
+    else:
+        out["text"] = text or ""
+    checks["slots"] = [{"slot": s, "p_point": round(decision["slots"][s]["p_point"], 4),
+                        "element": decision["slots"][s]["element"] or None,
+                        "p_element": round(decision["slots"][s]["p_element"], 4)} for s in names]
+    if decision.get("retry"):
+        checks["retry"] = decision["retry"]
+    out["checks"] = checks
+    return out
+
+
+def _argmax(p: Sequence[float]) -> int:
+    return max(range(len(p)), key=lambda k: p[k])
+
+
+def _chosen(decision: Dict) -> Tuple[str, List[str]]:
+    """(the action `answer` takes, the slots its answer reads), as `answer` picks them."""
+    act = ACTIONS[decision.get("action", _argmax(decision["action_p"]))]
+    if act in ("click", "refresh"):
+        n = decision.get("count", _argmax(decision["count_p"]))
+        return act, [f"click{k}" for k in range(1, (min(n, 1) if act == "refresh" else n) + 1)]
+    if act == "drag":
+        k = decision.get("drag_count", _argmax(decision["drag_count_p"]))
+        return act, [f"drag{j} {end}" for j in range(1, k + 2) for end in ("from", "to")]
+    if act in ("slide", "hold"):
+        return act, [n for n, _ in SLOTS if n.startswith(act)]
+    return act, []
+
+
+def variants(decision: Dict, stop_id: Optional[int] = None) -> List[Dict]:
+    """`decision`, then each one-place change to it, likeliest first: another action,
+    another count, one slot's point moved to its runner-up token, or one typed
+    character swapped for its runner-up. A change costs log(p_chosen / p_runner_up),
+    so the first is the place the model was least sure of.
+
+    The heads' scores are a distribution, so a refused answer is a fact about the
+    board: the next try is the next-likeliest answer, not the same one asked again.
+    Each variant names its change in `retry`. `stop_id` ends the typed text: markers
+    after it are not read, but the stop itself can be swapped (a longer answer)."""
+    def cost(p_top, p_alt):
+        return math.log(max(p_top, 1e-9)) - math.log(max(p_alt, 1e-9))
+
+    act, names = _chosen(decision)
+    changes = []
+    a = _argmax(decision["action_p"])
+    for k, p in enumerate(decision["action_p"]):
+        if k != a:
+            changes.append((cost(decision["action_p"][a], p), {"action": k}, f"action {ACTIONS[k]}"))
+    for key, probs in (("count", "count_p"), ("drag_count", "drag_count_p")):
+        if (key == "count") != (act == "click") or (key == "drag_count") != (act == "drag"):
+            continue
+        n = _argmax(decision[probs])
+        for k, p in enumerate(decision[probs]):
+            if k != n:
+                changes.append((cost(decision[probs][n], p), {key: k}, f"{key} {k + (key == 'drag_count')}"))
+    for name in names:
+        slot = decision["slots"][name]
+        for j, alt in enumerate(slot.get("alts") or [], 2):
+            changes.append((cost(slot["p_point"], alt["p_point"]), {"slot": (name, alt)}, f"{name} place {j}"))
+    if act == "type" and decision.get("text_alt_ids"):
+        ids = decision["text_ids"]
+        end = ids.index(stop_id) + 1 if stop_id is not None and stop_id in ids else len(ids)
+        for i in range(end):
+            changes.append((cost(decision["text_p"][i], decision["text_alt_p"][i]), {"text": i}, f"text{i + 1} place 2"))
+    out = [decision]
+    for _c, change, what in sorted(changes, key=lambda c: c[0]):
+        v = copy.deepcopy(decision)
+        if "slot" in change:
+            name, alt = change["slot"]
+            v["slots"][name].update({k: alt[k] for k in ("token", "dx", "dy", "p_point")})
+        elif "text" in change:
+            i = change["text"]
+            v["text_ids"][i] = v["text_alt_ids"][i]
+        else:
+            v.update(change)
+        v["retry"] = what
+        out.append(v)
+    return out
+
+
+def slot_point(ans: Dict, slot: str) -> Optional[List[int]]:
+    """Where `answer`'s `slot` put its point (after any later move of it), or None."""
+    if slot.startswith("click"):
+        pts = ans.get("points") or []
+        k = int(slot[5:]) - 1
+        return pts[k] if k < len(pts) else None
+    if slot.startswith("drag"):
+        k, end = slot[4:].split()
+        drags = ans.get("drags") or []
+        return drags[int(k) - 1][end] if int(k) <= len(drags) else None
+    return ans.get(slot.split()[1])

package/python/src/captchakraken/pointer/marks.py

@@ -0,0 +1,184 @@
+"""The outlines and numbers drawn over a board's elements.
+
+Each numbered element gets a 1 px outline traced from its MASK, not its box: a
+box around a long twisty shape (a chain, a line, a snake of tiles) covers half
+the board and names nothing. Its number sits in a small tag of the outline's
+colour just outside the object's top edge, so the tag hides the object's
+background rather than the object.
+
+The digits are a 5x7 pixel font defined here, not a system font: what the model
+was trained on must be what it is shown, on every machine, to the pixel. Each
+font pixel is drawn `SCALE` x `SCALE`, so a digit is 15x21 px with 3 px strokes
+and spans more than one 16 px vision patch; at 1x a digit was a fifth of a
+patch, too small to tell 3 from 8 from 9. The look is frozen. Changing anything
+here changes the model's input and needs a retrain.
+"""
+from __future__ import annotations
+
+from typing import Dict, List, Sequence, Set, Tuple
+
+import numpy as np
+from PIL import Image
+
+from .elements import fill_holes
+
+_GLYPHS = {
+    "0": ("01110", "10001", "10011", "10101", "11001", "10001", "01110"),
+    "1": ("00100", "01100", "00100", "00100", "00100", "00100", "01110"),
+    "2": ("01110", "10001", "00001", "00010", "00100", "01000", "11111"),
+    "3": ("11110", "00001", "00001", "01110", "00001", "00001", "11110"),
+    "4": ("00010", "00110", "01010", "10010", "11111", "00010", "00010"),
+    "5": ("11111", "10000", "11110", "00001", "00001", "10001", "01110"),
+    "6": ("00110", "01000", "10000", "11110", "10001", "10001", "01110"),
+    "7": ("11111", "00001", "00010", "00100", "01000", "01000", "01000"),
+    "8": ("01110", "10001", "10001", "01110", "10001", "10001", "01110"),
+    "9": ("01110", "10001", "10001", "01111", "00001", "00010", "01100"),
+}
+GLYPH_W, GLYPH_H = 5, 7
+#: Board pixels per font pixel.
+SCALE = 3
+#: Tag padding around the digits, px.
+PAD = 2
+#: Outline and tag colours, by element number: saturated, mutually distinct, and
+#: dark enough that white digits read on every one of them.
+PALETTE = [
+    (230, 25, 75), (0, 130, 200), (60, 160, 40), (145, 30, 180), (245, 110, 0),
+    (0, 140, 140), (200, 30, 160), (110, 80, 20), (70, 70, 220), (160, 120, 0),
+]
+
+
+def colour(n: int) -> Tuple[int, int, int]:
+    """The colour of element number `n` (1-based)."""
+    return PALETTE[(n - 1) % len(PALETTE)]
+
+
+def _digits(text: str) -> np.ndarray:
+    cols = []
+    for k, ch in enumerate(text):
+        if k:
+            cols.append(np.zeros((GLYPH_H, 1), bool))
+        cols.append(np.array([[c == "1" for c in row] for row in _GLYPHS[ch]], bool))
+    return np.kron(np.hstack(cols), np.ones((SCALE, SCALE), bool))
+
+
+def tag_size(n: int) -> Tuple[int, int]:
+    d = _digits(str(n))
+    return d.shape[1] + 2 * PAD, d.shape[0] + 2 * PAD
+
+
+# Chain-code directions (dx, dy), counter-clockwise from east, as OpenCV's border follower numbers them.
+_STEPS = ((1, 0), (1, -1), (0, -1), (-1, -1), (-1, 0), (-1, 1), (0, 1), (1, 1))
+
+
+def _trace(m: np.ndarray, x0: int, y0: int) -> List[Tuple[int, int]]:
+    """The outer border that starts at (x0, y0), its component's first pixel in raster order.
+
+    Suzuki-Abe border following exactly as OpenCV's findContours walks it (CHAIN_APPROX_NONE): the drawn line
+    fills each diagonal step on one side, so the walk's direction and its revisits are part of the picture.
+    """
+    def lit(s: int, x: int, y: int) -> bool:
+        return bool(m[y + _STEPS[s][1], x + _STEPS[s][0]])
+
+    s = 4
+    while True:
+        s = (s - 1) & 7
+        if lit(s, x0, y0) or s == 4:
+            break
+    if not lit(s, x0, y0):
+        return [(x0, y0)]
+    second = (x0 + _STEPS[s][0], y0 + _STEPS[s][1])
+    pts: List[Tuple[int, int]] = []
+    x, y = x0, y0
+    while True:
+        s += 1
+        while not lit(s & 7, x, y):
+            s += 1
+        s &= 7
+        pts.append((x, y))
+        nxt = (x + _STEPS[s][0], y + _STEPS[s][1])
+        if nxt == (x0, y0) and (x, y) == second:
+            return pts
+        (x, y), s = nxt, (s + 4) & 7
+
+
+def outline(mask: Tuple[Tuple[int, int], np.ndarray]) -> List[np.ndarray]:
+    """The mask's outer contours, as (N, 2) int arrays of board pixels.
+
+    A hole's edge is not drawn, nor is anything inside a hole, so the holes are filled before tracing.
+    """
+    (x, y), m = mask
+    if not m.size or not m.any():
+        return []
+    p = fill_holes(np.pad(m.astype(bool), 1))
+    seen: Set[Tuple[int, int]] = set()
+    cs = []
+    for row, col in np.argwhere(p[:, 1:] & ~p[:, :-1]):
+        start = (int(col) + 1, int(row))
+        if start not in seen:
+            c = _trace(p, *start)
+            seen.update(c)
+            cs.append(np.array(c) + np.array([x - 1, y - 1]))
+    return cs
+
+
+def _anchors(cs: Sequence[np.ndarray], tw: int, th: int) -> List[Tuple[int, int]]:
+    """Tag positions (top-left) to try, best first: above the topmost point, left of
+    the leftmost, below the bottommost, right of the rightmost."""
+    pts = np.vstack(cs)
+    top = pts[np.lexsort((pts[:, 0], pts[:, 1]))[0]]
+    left = pts[np.lexsort((pts[:, 1], pts[:, 0]))[0]]
+    bot = pts[np.lexsort((pts[:, 0], -pts[:, 1]))[0]]
+    right = pts[np.lexsort((pts[:, 1], -pts[:, 0]))[0]]
+    return [(int(top[0]) - tw // 2, int(top[1]) - th), (int(left[0]) - tw, int(left[1]) - th // 2),
+            (int(bot[0]) - tw // 2, int(bot[1]) + 1), (int(right[0]) + 1, int(right[1]) - th // 2),
+            (int(top[0]) - tw // 2, int(top[1]) + 1)]
+
+
+def _overlap(a, b) -> int:
+    return max(0, min(a[2], b[2]) - max(a[0], b[0])) * max(0, min(a[3], b[3]) - max(a[1], b[1]))
+
+
+def draw(image: Image.Image, elements: Sequence[Dict]) -> Image.Image:
+    """The board with every element outlined and numbered (1-based, in list order).
+    A None keeps its number but draws nothing: a video frame is drawn with
+    `elements.on_frame`, so an element carries its track's number on every frame."""
+    out = np.array(image.convert("RGB"))
+    H, W = out.shape[:2]
+    plans = []
+    for n, e in enumerate(elements, 1):
+        if e is None:                 # a video track absent from this frame
+            continue
+        cs = outline(e["mask"])
+        if not cs:
+            x0, y0, x1, y1 = (int(round(v)) for v in e["box"])
+            cs = [np.array([[x0, y0], [x1, y0], [x1, y1], [x0, y1]])]
+        plans.append((n, cs))
+    for n, cs in plans:
+        c = colour(n)
+        for poly in cs:
+            p = np.clip(poly, [0, 0], [W - 1, H - 1])
+            out[p[:, 1], p[:, 0]] = c
+            # Contours are 8-connected; fill the diagonal steps so the line never
+            # reads as dotted at 1 px.
+            q = np.roll(p, -1, axis=0)
+            diag = (p[:, 0] != q[:, 0]) & (p[:, 1] != q[:, 1])
+            out[p[diag, 1], q[diag, 0]] = c
+    placed: List[Tuple[int, int, int, int]] = []
+    for n, cs in plans:
+        tw, th = tag_size(n)
+        best = None
+        for ax, ay in _anchors(cs, tw, th):
+            ax = min(max(ax, 0), W - tw)
+            ay = min(max(ay, 0), H - th)
+            r = (ax, ay, ax + tw, ay + th)
+            cost = sum(_overlap(r, o) for o in placed)
+            if best is None or cost < best[0]:
+                best = (cost, r)
+            if cost == 0:
+                break
+        x0, y0, x1, y1 = best[1]
+        out[y0:y1, x0:x1] = colour(n)
+        d = _digits(str(n))
+        out[y0 + PAD:y0 + PAD + d.shape[0], x0 + PAD:x0 + PAD + d.shape[1]][d] = (255, 255, 255)
+        placed.append(best[1])
+    return Image.fromarray(out)