captchakraken 2.4.0 → 2.6.0

package/python/pyproject.toml

@@ -4,13 +4,14 @@ build-backend = "hatchling.build"
 
 [project]
 name = "captchakraken"
-version = "2.4.0"
+version = "2.6.0"
 description = "Self-hosted captcha solver: OpenCV grid detection + a fine-tuned Qwen3.5-9B vision LoRA served on vLLM."
 readme = "README.md"
 requires-python = ">=3.10"
 # Source-available, NOT GPL. The CaptchaKraken Source-Available License permits
-# commercial use as an internal component but prohibits selling the solve itself
-# or thin-wrapper resale — see LICENSE. (GPL-3.0 would wrongly permit reselling.)
+# commercial use as an internal component but prohibits selling the solve itself,
+# thin-wrapper resale, and bundling it into a stealth browser someone else ships
+# — see LICENSE. (GPL-3.0 would wrongly permit all three.)
 license = { file = "LICENSE" }
 authors = [{ name = "Jake Writer" }]
 keywords = ["captcha", "recaptcha", "hcaptcha", "vllm", "qwen", "computer-vision", "automation"]

package/python/src/captchakraken/__init__.py

@@ -9,8 +9,7 @@ Usage:
 
 Model/endpoint defaults live in `captchakraken.config` and are fully
 env-overridable (VLLM_BASE_URL, CAPTCHA_LORA_ADAPTER, …); the solver itself is
-model-agnostic. The legacy v1 stack (SAM3 grounding, multi-provider planner,
-detect/segment/drag-refine) lives on the `v1-old-architecture` branch.
+model-agnostic.
 """
 
 from pathlib import Path
@@ -46,11 +45,13 @@ try:  # pragma: no cover - exercised only when the serving stack is installed
     # caller supplies the Playwright-compatible page (see page_solver's module
     # docstring on why we duck-type rather than import one).
     from .page_solver import PageSolver, SolveResult, solve_captcha_on_page
+    from .watcher import CaptchaWatcher
 except ModuleNotFoundError:
     ActionPlanner = None  # type: ignore[assignment,misc]
     CaptchaSolver = None  # type: ignore[assignment,misc]
     solve_captcha = None  # type: ignore[assignment]
     PageSolver = None  # type: ignore[assignment,misc]
+    CaptchaWatcher = None  # type: ignore[assignment,misc]
     SolveResult = None  # type: ignore[assignment,misc]
     solve_captcha_on_page = None  # type: ignore[assignment]
 
@@ -58,6 +59,7 @@ __all__ = [
     "CaptchaSolver",
     "solve_captcha",
     "PageSolver",
+    "CaptchaWatcher",
     "SolveResult",
     "solve_captcha_on_page",
     "ActionPlanner",
@@ -70,4 +72,4 @@ __all__ = [
     "add_overlays_to_image",
 ]
 
-__version__ = "2.3.0"
+__version__ = "2.6.0"

package/python/src/captchakraken/action_types.py

@@ -22,6 +22,22 @@ class BoundingBox(RootModel):
 
 class Action(BaseModel):
     action: str
+    # ANIMATED CHALLENGES ONLY. Path to the keyframe the model chose to act on, and
+    # its 1-based number in the set that was sent.
+    #
+    # Why an action carries a picture: on an animated puzzle the target is only
+    # present part of the time, so these coordinates are correct only while the
+    # widget looks the way it did in that keyframe. The driver therefore holds the
+    # mouse until the live neighbourhood around the click point matches the same
+    # neighbourhood of this file, then clicks. Without it the click fires on
+    # whatever happens to be on screen, which for a fading sprite is usually
+    # background.
+    #
+    # Absent on every still puzzle, where there is no moment to wait for. Both are
+    # set together or not at all — a number with no image cannot be waited on, and
+    # an image with no number cannot be reported.
+    await_keyframe: Optional[str] = None
+    frame: Optional[int] = None
 
 
 class ClickAction(Action):
@@ -31,15 +47,20 @@ class ClickAction(Action):
 
 class DragAction(Action):
     action: Literal["drag"]
-    source_bounding_box: BoundingBox = None  # [x1, y1, x2, y2] in percentages
-    target_bounding_box: BoundingBox = None  # [x1, y1, x2, y2] in percentages
+    # Optional, not merely defaulted: a PUZZLE-PIECE SLIDER is carried as a drag
+    # with source_bounding_box explicitly None, and pydantic validates a value
+    # that is passed even when it equals the default. Declaring these as bare
+    # `BoundingBox = None` made constructing a slide raise ValidationError —
+    # every slide puzzle, at the moment of acting on a correct answer.
+    source_bounding_box: Optional[BoundingBox] = None  # [x1, y1, x2, y2] in percentages
+    target_bounding_box: Optional[BoundingBox] = None  # [x1, y1, x2, y2] in percentages
 
 
 class TypeAction(Action):
     """Type text into an input."""
     action: Literal["type"]
     text: str
-    target_bounding_box: BoundingBox = None  # [x1, y1, x2, y2] in percentages
+    target_bounding_box: Optional[BoundingBox] = None  # [x1, y1, x2, y2] in percentages
 
 
 class WaitAction(Action):

package/python/src/captchakraken/cli.py

@@ -3,12 +3,25 @@ CaptchaKraken CLI (v2).
 
 Modes:
   captchakraken image.png [model_name] [api_provider] [api_key]
-        Solve a captcha image / video. `api_provider` is kept for v1
+        Solve a captcha from ONE still image. `api_provider` is kept for v1
         compat; only `captchaKrakenApi` is supported and is the default.
 
+  captchakraken solve-animated --frames-dir DIR [--fps N] [--model M] [--api-key K]
+        Solve an ANIMATED challenge from a recorded burst. DIR holds one
+        zero-padded PNG per frame; the burst is sliced into keyframes (written to
+        DIR/keyframes/) and sent to the model as one multi-image request. Prints
+        {"actions": [...], "keyframes": [...], "keyframe_mode": ...}. Every action
+        carries `await_keyframe` — the still the driver must see on screen before
+        acting — and `frame`, its number. Poll for that state with `match-region`.
+
+  captchakraken match-region ref.png live.png cx cy [tolerance]
+        Does `live` look like `ref` around the 0-1 point (cx, cy)? The wait gate
+        behind an animated click -> {"match": bool, "diff": float}. Also a `serve`
+        cmd (`match-region`), which is what a driver polling every ~120ms wants.
+
   captchakraken check-movement   img1.png img2.png [threshold]
   captchakraken check-movement-batch threshold img1 img2 [img3 ...]
-        Frame-diff helpers used by the Playwright lib's video flow.
+        Frame-diff helpers used by the Playwright lib's settle monitor.
 
   captchakraken find-grid       image.png
   captchakraken detect-selected image.png
@@ -55,6 +68,7 @@ import json
 import os
 import subprocess
 import sys
+from typing import Optional
 
 from .errors import CaptchaKrakenAPIError
 from .solver import CaptchaSolver, UnsupportedCaptchaError
@@ -381,6 +395,8 @@ def _handle_serve() -> bool:
       grid-cell-states        {a, b}                        -> states | {grid: null}
       grid-cell-states-fixed  {a, b, grid_boxes}            -> states
       check-movement          {a, b, threshold?}            -> {has_movement: bool}
+      match-region            {ref, live, cx, cy, ...}      -> {match, diff, ...}
+      track-piece             {before, after, exclude?}     -> {bbox, moved}
 
     Unknown cmd / malformed line -> an {ok:false} response (the process keeps
     running). EOF on stdin ends the loop cleanly. All heavy detection delegates
@@ -414,6 +430,18 @@ def _handle_serve() -> bool:
             threshold = float(req.get("threshold", 0.005))
             moved = ImageProcessor.detect_movement(req["a"], req["b"], threshold)
             return {"has_movement": bool(moved)}
+        if cmd == "match-region":
+            # Animated wait gate: is the widget back in the state the model
+            # answered about? Polled every ~120ms while a click is held, so it
+            # belongs on the worker rather than paying a process spawn per poll.
+            return _match_region(req["ref"], req["live"],
+                                 float(req["cx"]), float(req["cy"]),
+                                 req.get("tolerance"))
+        if cmd == "track-piece":
+            # Slider closed loop: where has the piece got to? Called several
+            # times per drag WITH THE MOUSE HELD DOWN, so a process spawn per
+            # reading would stretch the drag into something no human hand does.
+            return _track_piece(req["before"], req["after"], req.get("exclude"))
         raise ValueError(f"unknown cmd: {cmd!r}")
 
     # Signal readiness so the JS side knows imports are done before it polls.
@@ -436,6 +464,172 @@ def _handle_serve() -> bool:
     return True
 
 
+def _frames_in(frames_dir: str) -> list:
+    """The recorded burst's frames in capture order.
+
+    Sorted by NAME, which is why the recorder must zero-pad: `frame_9.png` sorts
+    after `frame_10.png`, and a shuffled burst destroys the temporal structure the
+    cycle detector reads. `_handle_solve_animated` rejects an unsorted-looking set
+    rather than silently slicing noise.
+    """
+    import glob
+
+    return sorted(
+        p for ext in ("png", "jpg", "jpeg")
+        for p in glob.glob(os.path.join(frames_dir, f"*.{ext}"))
+    )
+
+
+def _handle_solve_animated() -> bool:
+    """`captchakraken solve-animated --frames-dir DIR [--fps N] [...]`
+
+    The animated counterpart of the default solve command, for the TypeScript
+    driver: TS owns the browser, so IT records the burst (one zero-padded PNG per
+    frame into a directory it owns) and this slices the burst into keyframes, sends
+    them to the model as one multi-image request, and prints the actions.
+
+    Each action carries `await_keyframe`, an absolute path to the still the driver
+    must see on screen before it acts. Those files are written into
+    `DIR/keyframes/`, i.e. inside the caller's own directory — the CLI never leaves
+    temp files behind for the caller to guess at, and the caller cleans up when it
+    cleans up the burst. Poll for the state with `match-region`.
+    """
+    if len(sys.argv) <= 1 or sys.argv[1] != "solve-animated":
+        return False
+
+    parser = argparse.ArgumentParser(prog="captchakraken solve-animated")
+    parser.add_argument("--frames-dir", required=True,
+                        help="Directory of the recorded burst (zero-padded PNGs).")
+    parser.add_argument("--fps", type=float, default=10.0,
+                        help="Frame rate the burst was recorded at (default 10, the "
+                             "rate the training data was collected at).")
+    parser.add_argument("--model", default=None)
+    parser.add_argument("--api-key", default=None)
+    args = parser.parse_args(sys.argv[2:])
+
+    frames = _frames_in(args.frames_dir)
+    if not frames:
+        print(json.dumps({"error": f"no frames in {args.frames_dir}"}), file=sys.stderr)
+        sys.exit(1)
+
+    try:
+        import cv2
+
+        from .keyframes import extract_keyframes, write_keyframes
+
+        with timed("cli.total"):
+            imgs = [im for im in (cv2.imread(p) for p in frames) if im is not None]
+            if not imgs:
+                print(json.dumps({"error": "no frame decoded"}), file=sys.stderr)
+                sys.exit(1)
+            kfset = extract_keyframes(imgs, fps=float(args.fps))
+            kf_dir = os.path.join(args.frames_dir, "keyframes")
+            paths = [str(os.path.abspath(p))
+                     for p in write_keyframes(kfset, kf_dir, stem="challenge")]
+
+            solver = CaptchaSolver(model=args.model, api_key=args.api_key)
+            result = solver.solve_keyframes(paths)
+
+        action_data = [a.model_dump() for a in result]
+        print(json.dumps({
+            "actions": action_data,
+            "token_usage": solver.planner.token_usage,
+            # Reported so the driver can log what it actually sent, and so a
+            # surprising slicing (6 frames collapsing to 1) is visible in the
+            # solve log rather than only inferable from the answer.
+            "keyframes": paths,
+            "keyframe_mode": kfset.mode,
+            "source_frames": len(imgs),
+        }))
+    except UnsupportedCaptchaError as e:
+        print(json.dumps({"error": str(e), "unsupported": True}), file=sys.stderr)
+        sys.exit(2)
+    except CaptchaKrakenAPIError as e:
+        print(json.dumps(e.to_payload()), file=sys.stderr)
+        sys.exit(3)
+    except Exception as e:  # noqa: BLE001
+        import traceback
+
+        traceback.print_exc()
+        print(json.dumps({"error": str(e)}), file=sys.stderr)
+        sys.exit(1)
+    return True
+
+
+def _match_region(ref: str, live: str, cx: float, cy: float,
+                  tolerance: Optional[float] = None) -> dict:
+    """Does `live` look like `ref` around the 0–1 point (cx, cy)?
+
+    The wait-for-state gate behind an animated click. Compares only the
+    neighbourhood of the action point, with the same box and metric the training
+    label's frame was chosen with (`keyframes.region_box` / `region_diff_ratio`) —
+    local rather than whole-frame because everything else in these puzzles is also
+    moving, so a whole-frame match would need every unrelated sprite to align too
+    and would essentially never open.
+    """
+    import cv2
+
+    from .keyframes import MATCH_REGION_TOLERANCE, region_box, region_diff_ratio
+
+    tol = MATCH_REGION_TOLERANCE if tolerance is None else float(tolerance)
+    a, b = cv2.imread(ref), cv2.imread(live)
+    if a is None or b is None:
+        return {"match": False, "diff": 1.0, "error": "unreadable image"}
+    box = region_box(a.shape[1::-1], (cx, cy))
+    d = region_diff_ratio(a, b, box)
+    return {"match": bool(d <= tol), "diff": float(d), "tolerance": tol}
+
+
+def _track_piece(before: str, after: str, exclude=None) -> dict:
+    """Where did the puzzle piece get to? See tool_calls/track_piece.py."""
+    from .tool_calls.track_piece import changed_bbox
+
+    bbox = changed_bbox(before, after, exclude)
+    return {"bbox": bbox, "moved": bbox is not None}
+
+
+def _handle_track_piece() -> bool:
+    """`captchakraken track-piece before.png after.png [exclude_json]`
+
+    `exclude_json` is `[x1, y1, x2, y2]` in pixels — the slider handle, which is
+    moving too and would otherwise be measured instead of the piece.
+
+    Also available over the persistent worker as cmd `track-piece`, which is
+    what the driver should use: this runs mid-drag, several times, with the
+    mouse button held.
+    """
+    if len(sys.argv) <= 1 or sys.argv[1] != "track-piece":
+        return False
+    if len(sys.argv) < 4:
+        print(json.dumps({
+            "error": "Usage: captchakraken track-piece before.png after.png [exclude_json]"
+        }), file=sys.stderr)
+        sys.exit(1)
+    exclude = json.loads(sys.argv[4]) if len(sys.argv) > 4 else None
+    print(json.dumps(_track_piece(sys.argv[2], sys.argv[3], exclude)))
+    return True
+
+
+def _handle_match_region() -> bool:
+    """`captchakraken match-region ref.png live.png cx cy [tolerance]`
+
+    cx/cy are 0–1 fractions of the reference image (the centre of the action).
+    Also available over the persistent worker as cmd `match-region`, which is what
+    the driver should use — this is polled every ~120 ms while waiting.
+    """
+    if len(sys.argv) <= 1 or sys.argv[1] != "match-region":
+        return False
+    if len(sys.argv) < 6:
+        print(json.dumps({
+            "error": "Usage: captchakraken match-region ref.png live.png cx cy [tolerance]"
+        }), file=sys.stderr)
+        sys.exit(1)
+    tol = float(sys.argv[6]) if len(sys.argv) > 6 else None
+    print(json.dumps(_match_region(sys.argv[2], sys.argv[3],
+                                   float(sys.argv[4]), float(sys.argv[5]), tol)))
+    return True
+
+
 def _handle_tool_commands() -> bool:
     if len(sys.argv) <= 1:
         return False
@@ -641,6 +835,12 @@ def main():
         return
     if _handle_tool_commands():
         return
+    if _handle_solve_animated():
+        return
+    if _handle_match_region():
+        return
+    if _handle_track_piece():
+        return
 
     parser = argparse.ArgumentParser(description="CaptchaKraken v2 (vLLM)")
     parser.add_argument("image_path", help="Path to the captcha image or video")
@@ -679,6 +879,15 @@ def main():
         "images'). Switches the grid prompt to a more aggressive variant that "
         "instructs the LoRA to look at the full grid for tiles it missed.",
     )
+    parser.add_argument(
+        "--text-mode",
+        action="store_true",
+        help="The widget has a text box, so the answer is a string to type rather "
+        "than a place to click. Sends the distorted-text prompt and skips grid "
+        "detection. Set by the Playwright wrapper from the DOM — the picture "
+        "alone does not distinguish a BotDetect code from an hCaptcha "
+        "'click the matching letter'.",
+    )
 
     args = parser.parse_args()
 
@@ -693,6 +902,7 @@ def main():
                 args.image_path,
                 puzzle_source=args.puzzle_source,
                 retry_mode=args.retry_mode,
+                text_mode=args.text_mode,
             )
 
         if isinstance(result, list):

package/python/src/captchakraken/config.py

@@ -27,7 +27,7 @@ import json
 import os
 from functools import lru_cache
 from pathlib import Path
-from typing import Any, Dict
+from typing import Any, Dict, Optional
 
 _PINNED_MODEL_PATH = Path(__file__).with_name("pinned_model.json")
 
@@ -176,24 +176,58 @@ def api_key() -> str:
 
 
 # ── Model identity (all overridable — the CLI itself is model-agnostic) ──────
+#
+# Precedence, for every value below:
+#
+#   1. The environment. An explicit override always wins — this is how you PIN
+#      to a specific model, and pinning is opt-in by design.
+#   2. models.json's `latest` entry. The default moves forward with the
+#      registry, so "the default download" and "the default prompt" advance
+#      together and cannot get out of step: prompts.resolve() reads the SAME
+#      entry to decide which prompt generation to send.
+#   3. pinned_model.json. The record of what this release was validated
+#      against, kept as the fallback so an older checkout or a hand-edited
+#      manifest still resolves.
+#
+# Order 2-before-3 is the behaviour change: `latest` leads. Today both name
+# CaptchaKraken_v1.1, so nothing moves until a newer model is registered — at
+# which point that IS the intent.
+def _registry_default(field: str) -> Optional[str]:
+    """`field` from the registry's `latest` model entry, or None."""
+    try:
+        from . import prompts  # local: prompts imports nothing from config
+
+        entry = prompts.registered_models().get(prompts.latest_model() or "")
+        value = (entry or {}).get(field)
+        return value if isinstance(value, str) else None
+    except Exception:  # noqa: BLE001 — a broken registry falls back, never raises
+        return None
+
+
 def base_model() -> str:
-    return os.getenv("CAPTCHA_BASE_MODEL", pinned()["base_model"])
+    return os.getenv("CAPTCHA_BASE_MODEL") or _registry_default("base_model") \
+        or pinned()["base_model"]
 
 
 def lora_adapter() -> str:
     """HF repo id or local path of the captcha adapter served on top of the base."""
-    return os.getenv("CAPTCHA_LORA_ADAPTER", pinned()["lora_adapter"])
+    from . import prompts
+
+    return (os.getenv("CAPTCHA_LORA_ADAPTER") or prompts.latest_model()
+            or pinned()["lora_adapter"])
 
 
 def lora_revision() -> str:
     """Git revision of the adapter repo to serve — pinning this is what makes
     'the adapter this release was tested against' a reproducible statement."""
-    return os.getenv("CAPTCHA_LORA_REVISION", pinned()["lora_revision"])
+    return os.getenv("CAPTCHA_LORA_REVISION") or _registry_default("lora_revision") \
+        or pinned()["lora_revision"]
 
 
 def lora_name() -> str:
     """The served LoRA name the client sends as the `model` field."""
-    return os.getenv("CAPTCHA_LORA_NAME", pinned()["lora_name"])
+    return os.getenv("CAPTCHA_LORA_NAME") or _registry_default("lora_name") \
+        or pinned()["lora_name"]
 
 
 # ── Local vLLM server knobs ─────────────────────────────────────────────────