captchakraken 2.4.0 → 2.6.0

package/python/src/captchakraken/pinned_model.json

@@ -9,20 +9,18 @@
     "for the legacy output/PascalCase schema. The model answered in a hybrid,",
     "the parser dropped it, and EVERY drag puzzle failed as 'unsupported' with",
     "no test going red. Recording the prompt hashes here means any edit to a",
-    "serving prompt fails CI until someone consciously re-pins — which forces",
+    "serving prompt fails CI until someone consciously re-pins \u2014 which forces",
     "the question 'does the pinned adapter still expect this prompt?'.",
     "",
     "To ship a new adapter: update lora_adapter/lora_revision, re-run the Tier 2",
     "static-image gate against it, and update prompt_sha256 if the prompts moved."
   ],
-
   "base_model": "RedHatAI/Qwen3.5-9B-FP8-dynamic",
-  "lora_adapter": "CaptchaKraken/CaptchaKraken_v1.1",
+  "lora_adapter": "CaptchaKraken/CaptchaKraken-Lora-v1.2",
   "lora_revision": "main",
-  "lora_name": "captcha",
-
+  "lora_name": "captcha-v12",
   "prompt_sha256": {
-    "PIXEL_ACTION_PROMPT": "15d4833135330aa57ec64eb9132d15739cbf4a10bcda2e6d85d2d0ad92e0216a",
-    "SELECT_GRID_PROMPT": "f014dbc1cb8c773cc9501ff97aea92ff0281e5687b3d637d1c626ded9212fd3c"
+    "PIXEL_ACTION_PROMPT": "21afc1def6faaa706ba4a8afc2e243f949fc6bb8936e624ecf8be0b82baeaae1",
+    "SELECT_GRID_PROMPT": "b259d39ac78adc92c4fe4e78aee248d073342aa4a797e9708d37dcea7532367d"
   }
 }

package/python/src/captchakraken/planner.py

@@ -8,7 +8,9 @@ planner with detect/segment/drag-refine; that code is preserved on the
 """
 
 import base64
+import io
 import json
+import math
 import os
 import re
 import sys
@@ -16,13 +18,80 @@ from mimetypes import guess_type
 from typing import Any, Dict, List, Optional
 
 import requests
+from PIL import Image
 
-from . import config, errors
+from . import config, errors, prompts
 from .server_manager import ensure_server
 from .timing import timed
 
 DEBUG = os.getenv("CAPTCHA_DEBUG", "0") == "1"
 
+# The pixel floor the adapters are TRAINED at, and therefore the floor an image
+# has to clear before it is worth sending.
+#
+# Training exports `MIN_PIXELS=200704` (448², in the finetune repo's
+# scripts/train_unified.sh), so anything smaller is enlarged before the vision
+# encoder sees it. Serving did none of that: vLLM runs with no
+# `--mm-processor-kwargs` and this client used to re-encode the file unchanged,
+# so a small captcha arrived at a geometry the model was never tuned on. It
+# fails as plausible-but-wrong coordinates, never as an error — measured on real
+# geetest_v3_slide captures (277x285), predictions landed 80-105 px from the
+# hand label at native size and 1-4 px away once upscaled.
+#
+# A FLOOR, NOT A RESIZE. Images already above it are passed through byte-for-byte;
+# re-encoding every screenshot would spend time and fidelity on the types that
+# were never affected.
+#
+# Unconditional as a DEFAULT, because the deployed v1.1 adapter improves under
+# it too (mean error ~40 px native vs ~4 px upscaled). But it is no longer a
+# constant: `models.json` may declare a `pixel_budget` per model, because the
+# band an adapter TRAINED under is a property of that adapter, exactly like its
+# prompt generation. See prompts.pixel_budget.
+MIN_PIXELS = 448 * 448
+
+
+def _encode_image(path: str,
+                  budget: "Optional[prompts.PixelBudget]" = None) -> tuple:
+    """`(mime, base64)` for one image, fitted into `budget`'s pixel band.
+
+    Area is clamped, never dimensions: aspect ratio is the geometry of a grid
+    puzzle, and squashing it moves every tile centre — fatal when the answer is
+    a coordinate on a normalized 0-1000 scale.
+
+    Anything Pillow cannot open is passed through untouched — this is a
+    preprocessing nicety, and it must never be the reason a solve fails.
+    """
+    budget = budget or prompts.pixel_budget(None)
+    with open(path, "rb") as f:
+        raw = f.read()
+    mime = guess_type(path)[0] or "image/png"
+    try:
+        im = Image.open(io.BytesIO(raw))
+        width, height = im.size
+    except Exception:  # noqa: BLE001 — unreadable/animated: send it as-is
+        return mime, base64.b64encode(raw).decode()
+    cap = budget.maximum
+    if cap and width * height > cap:
+        # floor, not ceil: rounding up here can land back OVER the cap, which
+        # is the mirror of the bug the floor branch below guards against.
+        scale = math.sqrt(cap / (width * height))
+        im = im.convert("RGB").resize(
+            (max(1, int(width * scale)), max(1, int(height * scale))),
+            Image.BICUBIC)
+        buf = io.BytesIO()
+        im.save(buf, "PNG")
+        return "image/png", base64.b64encode(buf.getvalue()).decode()
+    if width * height >= budget.minimum:
+        return mime, base64.b64encode(raw).decode()
+    # ceil, not round: rounding both sides down lands just under the floor
+    # (442x454 = 200,668 for a 277x285 capture) and defeats the whole point.
+    scale = math.sqrt(budget.minimum / (width * height))
+    im = im.convert("RGB").resize(
+        (math.ceil(width * scale), math.ceil(height * scale)), Image.BICUBIC)
+    buf = io.BytesIO()
+    im.save(buf, "PNG")
+    return "image/png", base64.b64encode(buf.getvalue()).decode()
+
 # Header the fleet's haproxy front (on the reverse-proxy EC2) routes on. When a
 # caller sets CAPTCHA_REQUEST_PRIORITY to a positive int, every request carries
 # `X-JH-Priority: <n>`; haproxy sends anything above its threshold (5) straight
@@ -160,45 +229,50 @@ def routing_headers(env=None) -> Dict[str, str]:
     return headers
 
 
-# Matches the training-distribution grid prompts in
-# cleanSamples/test/test_solutions.json -> grade.synthesize_instruction.
-# Drifting from these costs measurable accuracy.
-SELECT_GRID_PROMPT = """Solve the captcha grid by choosing the cell numbers that match the description from the captcha image prompt.
-
-Grid: {rows}x{cols} ({total} cells)
-{grid_hint}
-
-If no tiles match the description (e.g., they have all been cleared or none were present), return an empty list for target_ids: [].
-
-Return JSON Array: [list of cell numbers (1-{total})]"""
-
-
-# Non-grid click/drag puzzles. MUST stay byte-identical to
-# src/synthetic/reasoning/instructions.py::ACTION_INSTRUCTION in the finetune
-# repo — that is the exact prompt the LoRA was trained (and graded) on. Drift
-# here silently degrades every click/drag puzzle. Coordinates come back on a
-# 0–1000 scale; the solver converts them to 0–1 bboxes.
-PIXEL_ACTION_PROMPT = (
-    "Your task is to solve the captcha. Read the instruction at the top of the image carefully.\n\n"
-    "Look at the puzzle and decide what action solves it. All coordinates you return must be on a "
-    "normalized 0–1000 image scale (top-left = (0, 0), bottom-right = (1000, 1000)).\n\n"
-    "Name WHAT each object is with a short 1–2 word label, then give its position. "
-    "Choose ONE response:\n\n"
-    "FOR CLICK PUZZLES:\n"
-    "  Label each thing you click and give its point — subjects[i] names points[i]:\n"
-    "  → \"action\": \"click\", \"subjects\": [\"<label>\", ...], "
-    "\"points\": [[x1, y1], [x2, y2], ...]\n\n"
-    "FOR DRAG PUZZLES:\n"
-    "  Drag ONE item at a time. Label the source (the piece you pick up) and the destination "
-    "(where it belongs), each with a short 1–2 word label, and give both points:\n"
-    "  → \"action\": \"drag\", \"drags\": [{ \"source\": \"<label>\", \"from\": [x, y], "
-    "\"destination\": \"<label>\", \"to\": [x, y] }, ...]\n\n"
-    "Respond ONLY with JSON:\n"
-    "{\n"
-    "  \"action\": \"click\", \"subjects\": [ ... ], \"points\": [ ... ]\n"
-    "  // OR \"action\": \"drag\", \"drags\": [ ... ]\n"
-    "}"
-)
+# ── Prompt constants ────────────────────────────────────────────────────────
+#
+# These are ALIASES for the newest generation's built-ins, kept so that code and
+# tests written against the old module-level constants keep working. They are
+# NOT what a solve sends. The prompt a model gets is resolved per model
+# (`prompts.resolve`), because a model answers in whatever schema its prompt
+# asks for — sending a version-1 model the version-2 prompt does not error, it
+# silently degrades every puzzle. See prompts.py and models.json.
+#
+# The text itself lives in prompts.py and nowhere else. Two copies of a prompt
+# is the drift this module already shipped once (2026-07-18).
+_LATEST = prompts.builtin(prompts.LATEST_PROMPT_VERSION)
+
+SELECT_GRID_PROMPT = _LATEST.grid_template
+
+
+# Non-grid click/drag puzzles, newest generation. Alias — see the note above.
+PIXEL_ACTION_PROMPT = _LATEST.action_prompt
+
+
+# Animated puzzles: the challenge is recorded, sliced into keyframes
+# (`keyframes.py`) and sent as an ORDINARY MULTI-IMAGE request — one image per
+# keyframe, all in one context so the model can compare them. The answer carries
+# a `frame` naming which keyframe it acted on, and the page driver waits for the
+# live widget to look like that frame before pressing the mouse down.
+#
+# Alias for the newest generation — see the note above. Generation 1 has NO
+# animated prompt (the family did not exist), which is why resolution is per
+# model rather than per client.
+VIDEO_ACTION_PROMPT_TEMPLATE = _LATEST.video_template
+
+
+def video_action_prompt(n_keyframes: int) -> str:
+    """The prompt for a challenge served as `n_keyframes` stills.
+
+    The count is in the text because the model has no other way to know how many
+    images arrived — frame identities live in the prompt, not in the image payload.
+    Mirrors `instructions.video_instruction` in the finetune repo exactly.
+    """
+    n = int(n_keyframes)
+    if n < 1:
+        raise ValueError(f"a keyframe request needs at least one frame, got {n_keyframes}")
+    listing = ", ".join(f"frame {i}" for i in range(1, n + 1))
+    return VIDEO_ACTION_PROMPT_TEMPLATE.format(n=n, listing=listing)
 
 
 class ActionPlanner:
@@ -220,6 +294,21 @@ class ActionPlanner:
         self.model = model or config.lora_name()
         self.base_url = base_url or config.base_url()
         self.api_key = api_key or config.api_key()
+        # Prompts follow the MODEL, not this client's release. Resolve by the
+        # served name first (a hosted endpoint serves `captcha`, not a repo id,
+        # and models.json maps the alias back), falling back to the adapter repo
+        # id for a self-hosted deploy that serves under its own name.
+        #
+        # Doing this ONCE per planner rather than per request: resolution can
+        # touch the Hub, and a per-request lookup would put a network call in
+        # front of every round of every solve.
+        _prompt_key = (self.model if prompts.canonical_model_id(self.model)
+                       else config.lora_adapter())
+        self.prompts = prompts.resolve(_prompt_key)
+        # Resolution follows the MODEL for the same reason prompts do, and off
+        # the same key: an adapter reads a puzzle at the pixel band it trained
+        # under. Resolved once per planner — see the note above.
+        self.pixel_budget = prompts.pixel_budget(_prompt_key)
         # Auto-start a local vLLM server on the first request if one isn't up
         # (no-op for a healthy or remote endpoint). Guarded so we only try once.
         self._server_ensured = False
@@ -231,11 +320,29 @@ class ActionPlanner:
             self.debug_callback(f"[Planner] {message}")
 
     def _chat_with_image(self, prompt: str, image_path: str, max_tokens: int = 512) -> str:
-        with open(image_path, "rb") as f:
-            b64 = base64.b64encode(f.read()).decode()
-        mime, _ = guess_type(image_path)
-        if mime is None:
-            mime = "image/png"
+        return self._chat_with_images(prompt, [image_path], max_tokens=max_tokens)
+
+    def _chat_with_images(
+        self, prompt: str, image_paths: List[str], max_tokens: int = 512
+    ) -> str:
+        """One request carrying N images followed by the prompt.
+
+        The images go in ONE message, before the text, in the order given. That
+        ordering is what makes an animated challenge answerable: the model is told
+        "you are given N keyframes, in order: frame 1, frame 2, …" and matches those
+        names to the images positionally. Splitting them across requests, or putting
+        the text first, would break the correspondence between the numbers in the
+        prompt and the pictures — and the frame number is the part the page driver
+        acts on.
+        """
+        if not image_paths:
+            raise ValueError("no images to send")
+
+        parts: List[Dict[str, Any]] = []
+        for p in image_paths:
+            mime, b64 = _encode_image(p, self.pixel_budget)
+            parts.append({"type": "image_url",
+                          "image_url": {"url": f"data:{mime};base64,{b64}"}})
 
         messages = [
             {
@@ -244,10 +351,7 @@ class ActionPlanner:
             },
             {
                 "role": "user",
-                "content": [
-                    {"type": "image_url", "image_url": {"url": f"data:{mime};base64,{b64}"}},
-                    {"type": "text", "text": prompt},
-                ],
+                "content": [*parts, {"type": "text", "text": prompt}],
             },
         ]
 
@@ -283,7 +387,8 @@ class ActionPlanner:
             self._server_ensured = True
 
         url = f"{self.base_url}/chat/completions"
-        self._log(f"POST {url} model={self.model} max_tokens={max_tokens}")
+        self._log(f"POST {url} model={self.model} max_tokens={max_tokens} "
+                  f"images={len(parts)}")
 
         with timed("planner.chat"):
             resp = requests.post(url, headers=headers, json=payload, timeout=120)
@@ -414,8 +519,8 @@ class ActionPlanner:
         else:
             grid_hint = "Hint: Separate images. Select only clear matches."
 
-        prompt = SELECT_GRID_PROMPT.format(
-            rows=rows, cols=cols, total=total, grid_hint=grid_hint
+        prompt = self.prompts.grid_prompt(
+            rows=rows, cols=cols, grid_hint=grid_hint
         )
         if retry_mode == "missed-tiles":
             prompt = (
@@ -465,8 +570,8 @@ class ActionPlanner:
                 out.append(iv)
         return out
 
-    def get_pixel_actions(self, image_path: str) -> List[Dict[str, Any]]:
-        """Solve a non-grid click/drag puzzle.
+    def get_pixel_actions(self, image_path: str, text_mode: bool = False) -> List[Dict[str, Any]]:
+        """Solve a non-grid click/drag/slide/text puzzle.
 
         Sends the trained action prompt + image, parses the model's JSON, and
         returns a list of normalized actions with all coordinates on a 0–1
@@ -474,16 +579,75 @@ class ActionPlanner:
 
           {"kind": "click", "points": [(x, y), ...]}
           {"kind": "drag",  "src": (x, y), "dst": (x, y)}
+          {"kind": "slide", "dst": (x, y)}          — puzzle-piece slider
+          {"kind": "type",  "text": "<the code>"}   — distorted text
+
+        `text_mode` swaps in the distorted-text prompt. It is chosen by the
+        DRIVER, from the widget's DOM (a visible text box in the challenge),
+        because nothing about the picture reliably says "this one is typed" —
+        BotDetect's warped letters and hCaptcha's "click the matching letter"
+        look alike to a pixel classifier, and the two want opposite answers.
 
         Returns [] if the model produced nothing usable. The solver turns these
-        into ClickAction / DragAction bboxes.
+        into ClickAction / DragAction / TypeAction.
         """
-        raw = self._chat_with_image(PIXEL_ACTION_PROMPT, image_path, max_tokens=512)
+        prompt = self.prompts.text_prompt() if text_mode else self.prompts.action_prompt
+        raw = self._chat_with_image(prompt, image_path, max_tokens=512)
         data = self._parse_json(raw)
         actions = self._normalize_pixel(data)
         self._log(f"pixel actions -> {actions}")
         return actions
 
+    def get_keyframe_actions(self, keyframe_paths: List[str]) -> List[Dict[str, Any]]:
+        """Solve an ANIMATED challenge from its keyframes.
+
+        Same normalized actions as `get_pixel_actions`, each additionally carrying
+        `"frame"`: the 1-based keyframe the model chose to act on, or None if it did
+        not name a usable one.
+
+        The frame is not decoration. On these puzzles the target is only there part
+        of the time, so the coordinates are only correct while the widget looks like
+        the frame they were read off. The page driver holds the mouse until it does.
+        An action with `frame=None` is still returned — the caller decides whether to
+        click blind or give up, and dropping it here would look identical to "the
+        model had nothing to say".
+        """
+        if not keyframe_paths:
+            return []
+        # Per model, not per client: generation 1 has no animated prompt at all,
+        # so this raises a clear error there instead of sending a v1 model a
+        # keyframe request it was never trained to read.
+        prompt = self.prompts.video_prompt(len(keyframe_paths))
+        raw = self._chat_with_images(prompt, list(keyframe_paths), max_tokens=512)
+        data = self._parse_json(raw)
+        actions = self._normalize_pixel(data)
+        frame = self._normalize_frame(data, len(keyframe_paths))
+        for a in actions:
+            a["frame"] = frame
+        self._log(f"keyframe actions (frame={frame}) -> {actions}")
+        return actions
+
+    @staticmethod
+    def _normalize_frame(data: Any, n_keyframes: int) -> Optional[int]:
+        """The keyframe number the model chose, or None.
+
+        Out-of-range is treated as "did not choose" rather than clamped. Clamping a
+        7 to 6 would invent an intent the model never had, and the driver would then
+        wait for a picture the answer was not about — a slow failure that looks like
+        a flaky page. Tolerates the nested shape the model drifts into
+        ({"action": {"frame": n}}), the same way the click/drag parsing does.
+        """
+        if not isinstance(data, dict):
+            return None
+        raw = data.get("frame")
+        if raw is None and isinstance(data.get("action"), dict):
+            raw = data["action"].get("frame")
+        try:
+            n = int(raw)
+        except (TypeError, ValueError):
+            return None
+        return n if 1 <= n <= n_keyframes else None
+
     @staticmethod
     def _normalize_pixel(data: Any) -> List[Dict[str, Any]]:
         """Map the model's 0–1000 click/drag JSON to 0–1 normalized actions.
@@ -550,6 +714,23 @@ class ActionPlanner:
         if not isinstance(data, dict):
             return out
 
+        # ---- type (CANONICAL — the schema TEXT_INSTRUCTION asks for on the
+        #      distorted-text captchas: {"action": "type", "text": "<the code>"}).
+        # First, because it is the one answer family with NO coordinate in it:
+        # every branch below keys off a number, so a typed answer that fell
+        # through reached the end and normalized to nothing.
+        # The code is passed through VERBATIM — case and spacing are part of what
+        # the model read off the image, and "tidying" them submits a different
+        # answer than the one it gave.
+        act = data.get("action")
+        if isinstance(act, dict) and act.get("action") == "type":
+            act = act.get("action")  # unwrap {"action": {"action": "type", ...}}
+        if act == "type" or (isinstance(data.get("text"), str) and "drags" not in data):
+            container = data if isinstance(data.get("text"), str) else data.get("action")
+            text = container.get("text") if isinstance(container, dict) else None
+            if isinstance(text, str) and text:
+                return [{"kind": "type", "text": text}]
+
         # ---- drag (CANONICAL — the schema PIXEL_ACTION_PROMPT asks for and the
         #      LoRA is trained on, see instructions.py::ACTION_INSTRUCTION):
         #      {"action":"drag","drags":[{"source","from":[x,y],"destination","to":[x,y]}]}
@@ -567,11 +748,24 @@ class ActionPlanner:
                     continue
                 snums = flat_numbers(d.get("from"))
                 dnums = flat_numbers(d.get("to"))
-                if len(snums) >= 2 and len(dnums) >= 2:
+                if len(dnums) < 2:
+                    continue  # a piece with nowhere to go is not actionable
+                dst = norm_xy(dnums[0], dnums[1])
+                if not dst:
+                    continue
+                if len(snums) >= 2:
                     src = norm_xy(snums[0], snums[1])
-                    dst = norm_xy(dnums[0], dnums[1])
-                    if src and dst:
+                    if src:
                         out.append({"kind": "drag", "src": src, "dst": dst})
+                else:
+                    # SOURCELESS — a puzzle-piece slider. The prompt's "FOR
+                    # PUZZLE PIECE SLIDER PUZZLES" clause tells the model to
+                    # leave the source empty precisely because the piece is not
+                    # what you pick up: the handle is, somewhere else entirely,
+                    # and how far it has to travel is not knowable from the
+                    # picture. Requiring both ends here dropped the one answer
+                    # shape the prompt asks for on every slide puzzle.
+                    out.append({"kind": "slide", "dst": dst})
             if out:
                 return out