@mengruo/dsh-vision-toolkit 0.0.1

package/vendor/agent-vision-toolkit/bin/glance

@@ -0,0 +1,93 @@
+#!/usr/bin/env python3
+"""glance: standalone image description, Q&A, and OCR CLI."""
+
+import argparse
+from pathlib import Path
+import sys
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
+from vision_client import (  # noqa: E402
+    VisionError,
+    describe_image,
+    image_path_to_data_url,
+    load_default_env,
+)
+
+
+def region_data_url(path, region):
+    try:
+        from PIL import Image
+    except ImportError:
+        raise VisionError("--region requires Pillow; install the optional dependency pillow first")
+    try:
+        x1, y1, x2, y2 = (int(v) for v in region.split(","))
+    except ValueError:
+        raise VisionError("--region expects four integers: X1,Y1,X2,Y2 (pixels)")
+    import base64
+    import io
+    try:
+        with Image.open(Path(path).expanduser()) as image:
+            width, height = image.size
+            box = (max(0, min(x1, x2)), max(0, min(y1, y2)),
+                   min(width, max(x1, x2)), min(height, max(y1, y2)))
+            if box[2] <= box[0] or box[3] <= box[1]:
+                raise VisionError(f"--region {region} is empty after clamping to {width}x{height}")
+            buffer = io.BytesIO()
+            image.crop(box).save(buffer, format="PNG")
+    except (OSError, ValueError) as exc:
+        raise VisionError(f"Cannot read image: {path}") from exc
+    return "data:image/png;base64," + base64.b64encode(buffer.getvalue()).decode()
+
+
+def build_prompt(args, count):
+    if args.ocr is not None:
+        extra = f" Additional requirements: {args.ocr}" if args.ocr else ""
+        scope = "these images" if count > 1 else "this image"
+        return (
+            f"Transcribe every piece of visible text in {scope} verbatim (titles, body text, labels, watermarks, etc.), "
+            "line by line, without omitting any characters. Do not rewrite, summarize, or translate the text, "
+            "and do not add any preamble, explanation, or extra content."
+            + (" Label each image's text with its ordinal (Image 1, Image 2, ...)." if count > 1 else "")
+            + extra
+        )
+    if args.query:
+        return args.query
+    if count > 1:
+        return ("Describe each image in detail (label them Image 1, Image 2, ...), "
+                "then point out the notable differences between them.")
+    return None
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        prog="glance",
+        description="Describe, answer questions about, or OCR an image with the configured vision model",
+    )
+    parser.add_argument("images", nargs="+", metavar="image",
+                        help="path(s) to the image(s); pass several to compare them in one call")
+    group = parser.add_mutually_exclusive_group()
+    group.add_argument("-q", "--query", help="ask a question about the image(s)")
+    group.add_argument("--ocr", nargs="?", const="", metavar="EXTRA", help="transcribe all visible text verbatim")
+    parser.add_argument("--region", metavar="X1,Y1,X2,Y2",
+                        help="crop to this pixel box (e.g. from ground) and send only the crop")
+    args = parser.parse_args()
+    load_default_env()
+    try:
+        if args.region and len(args.images) > 1:
+            raise VisionError("--region works with exactly one image")
+        urls = ([region_data_url(args.images[0], args.region)] if args.region
+                else [image_path_to_data_url(path) for path in args.images])
+        answer = describe_image(
+            urls,
+            build_prompt(args, len(urls)),
+            max_tokens=None,  # no output cap; the vision model decides
+            apply_lang=args.ocr is None,
+        )
+    except VisionError as exc:
+        parser.exit(1, f"glance: {exc}\n")
+    print(answer)
+
+
+if __name__ == "__main__":
+    main()

package/vendor/agent-vision-toolkit/bin/ground

@@ -0,0 +1,13 @@
+#!/usr/bin/env python3
+"""ground: locate objects in an image and print pixel bounding boxes."""
+
+from pathlib import Path
+import sys
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
+from ground import main  # noqa: E402
+
+
+if __name__ == "__main__":
+    main()

package/vendor/agent-vision-toolkit/bin/trace

@@ -0,0 +1,129 @@
+#!/usr/bin/env python3
+"""trace: deterministic image-to-SVG tracing for exact shape geometry.
+
+Runs fully local (no vision API, no credentials): coordinates come from the
+actual pixels, so use this instead of asking a vision model for shapes,
+sizes, or outlines. Flat, high-contrast graphics only; text becomes curves.
+"""
+
+import argparse
+from pathlib import Path
+import re
+import sys
+import tempfile
+
+WHITE_FILLS = {"#ffffff", "#fff", "white"}
+
+# Speckle filtering drops components under a fixed pixel area, so a 30px icon
+# binarizes to nothing at 1x while the same icon enlarged traces cleanly.
+# Enlarge anything smaller than this; full-size screenshots already clear it.
+TARGET_MIN_SIDE = 256
+
+
+def strip_background(svg):
+    """Drop the leading full-canvas white path vtracer emits for the background."""
+    match = re.search(r"<path [^>]*/>", svg)
+    if match:
+        fill = re.search(r'fill="([^"]+)"', match.group(0))
+        if fill and fill.group(1).strip().lower() in WHITE_FILLS:
+            return svg.replace(match.group(0), "", 1)
+    return svg
+
+
+def truncate_decimals(svg, places=2):
+    return re.sub(r"-?\d+\.\d{3,}", lambda m: f"{float(m.group()):.{places}f}", svg)
+
+
+def write_svg(path, svg):
+    """Write the exact UTF-8 payload whose byte count is reported to callers."""
+    payload = svg.encode("utf-8")
+    return Path(path).expanduser().write_bytes(payload)
+
+
+def prepare_input(path, region, scale):
+    """Crop/upscale with Pillow when asked; return the path vtracer reads and the scale used.
+
+    `scale` may be None, meaning "pick one from the size": small shapes are
+    enlarged until they survive speckle filtering. Without Pillow we cannot
+    measure the image, so that choice falls back to no upscale.
+    """
+    try:
+        from PIL import Image
+    except ImportError:
+        if region or (scale or 1) != 1:
+            sys.exit("trace: --region/--scale require Pillow; install the optional dependency pillow first")
+        return str(path), 1
+    with Image.open(path) as image:
+        if region:
+            try:
+                x1, y1, x2, y2 = (int(v) for v in region.split(","))
+            except ValueError:
+                sys.exit("trace: --region expects four integers: X1,Y1,X2,Y2 (pixels)")
+            box = (max(0, min(x1, x2)), max(0, min(y1, y2)),
+                   min(image.width, max(x1, x2)), min(image.height, max(y1, y2)))
+            if box[2] <= box[0] or box[3] <= box[1]:
+                sys.exit(f"trace: --region {region} is empty after clamping to {image.width}x{image.height}")
+            image = image.crop(box)
+        if scale is None:
+            shortest = max(min(image.width, image.height), 1)
+            scale = max(2 if region else 1, -(-TARGET_MIN_SIDE // shortest))
+        if not region and scale == 1:
+            return str(path), 1
+        if scale != 1:
+            image = image.resize((image.width * scale, image.height * scale), Image.LANCZOS)
+        handle = tempfile.NamedTemporaryFile(suffix=".png", delete=False)
+        image.save(handle.name)
+        return handle.name, scale
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        prog="trace",
+        description="Trace an image into SVG with exact pixel-derived geometry (local, deterministic)",
+    )
+    parser.add_argument("image", help="path to the image")
+    parser.add_argument("--region", metavar="X1,Y1,X2,Y2",
+                        help="crop to this pixel box first (e.g. a box from ground)")
+    parser.add_argument("--scale", type=int, metavar="N",
+                        help="upscale N times before tracing (default: enough to clear "
+                             f"{TARGET_MIN_SIDE}px on the shorter side, so small icons survive)")
+    parser.add_argument("--polygon", action="store_true",
+                        help="polygon mode for boxy diagrams/wireframes (default: spline for curved shapes)")
+    parser.add_argument("--color", action="store_true",
+                        help="keep colors (default: black-and-white, which is far more compact)")
+    parser.add_argument("-o", "--output", help="write the SVG here instead of stdout")
+    args = parser.parse_args()
+    try:
+        import vtracer
+    except ImportError:
+        parser.exit(1, "trace: requires vtracer; install the optional dependency vtracer first\n")
+    source = Path(args.image).expanduser()
+    if not source.is_file():
+        parser.exit(1, f"trace: Image not found: {source}\n")
+    work, scale = prepare_input(source, args.region, args.scale)
+    with tempfile.NamedTemporaryFile(suffix=".svg", delete=False) as out:
+        vtracer.convert_image_to_svg_py(
+            work, out.name,
+            colormode="color" if args.color else "binary",
+            filter_speckle=8, corner_threshold=40,
+            mode="polygon" if args.polygon else "spline",
+        )
+        svg = Path(out.name).read_text()
+    svg = truncate_decimals(strip_background(svg))
+    paths = svg.count("<path")
+    if not paths:
+        # This is where an agent decides the tool cannot see the shape and goes
+        # back to guessing. Name the recoveries instead, cheapest first.
+        print("trace: 0 paths - nothing survived binarization. Try a larger --scale, "
+              "--region to crop closer to the shape, or pre-invert a light-on-dark image. "
+              "--color is a last resort: on an anti-aliased image it splits every gray "
+              "level into its own path.", file=sys.stderr)
+    if args.output:
+        bytes_written = write_svg(args.output, svg)
+        print(f"wrote {args.output} ({bytes_written} bytes, {paths} paths, traced at {scale}x)")
+    else:
+        print(svg)
+
+
+if __name__ == "__main__":
+    main()

package/vendor/agent-vision-toolkit/detect.py

@@ -0,0 +1,60 @@
+from __future__ import annotations
+
+import argparse
+from pathlib import Path
+
+try:
+    from PIL import Image
+except ImportError:
+    Image = None
+
+from ground import GroundError, _position, locate
+from vision_client import VisionError
+
+DEFAULT_CATEGORY = ("UI element (buttons, links, inputs, icons, labels, "
+                    "headings, images, badges)")
+
+
+def build_target(category: str | None) -> str:
+    target = (category or DEFAULT_CATEGORY).strip()
+    if not target.lower().startswith("every distinct "):
+        target = f"every distinct {target}"
+    if "exact visible text" not in target.lower():
+        target += " — include the exact visible text in each label"
+    return target
+
+
+def format_inventory(matches, width: int, height: int) -> list[str]:
+    lines = []
+    for index, match in enumerate(matches, 1):
+        x1, y1, x2, y2 = match.bbox
+        position = _position(match.bbox, width, height)
+        lines.append(f"{index}. {position} {match.label} x1: {x1}, y1: {y1}, x2: {x2}, y2: {y2}")
+    return lines or ["no elements detected"]
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        prog="detect",
+        description="Inventory the elements in an image (or a region) with pixel bounding boxes",
+    )
+    parser.add_argument("image", type=Path, help="path to the image")
+    parser.add_argument("category", nargs="?",
+                        help='restrict to a category, e.g. "buttons" or "icons" (default: all UI elements)')
+    parser.add_argument("--region", metavar="X1,Y1,X2,Y2",
+                        help="inventory only this pixel box; output stays in original-image coordinates")
+    args = parser.parse_args()
+    try:
+        matches = locate(args.image.expanduser(), build_target(args.category), region=args.region)
+        if Image is None:
+            raise GroundError("detect requires Pillow; install the optional dependency pillow first")
+        with Image.open(args.image.expanduser()) as image:
+            width, height = image.size
+    except (GroundError, VisionError) as exc:
+        parser.exit(1, f"detect: {exc}\n")
+    for line in format_inventory(matches, width, height):
+        print(line)
+
+
+if __name__ == "__main__":
+    main()

package/vendor/agent-vision-toolkit/ground.py

@@ -0,0 +1,267 @@
+from __future__ import annotations
+
+import argparse
+import base64
+import io
+import json
+import os
+import re
+import sys
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+try:
+    from PIL import Image
+except ImportError:
+    Image = None
+
+from vision_client import VisionError, describe_image, image_path_to_data_url, load_default_env
+
+
+@dataclass(frozen=True)
+class Match:
+    label: str
+    bbox: tuple[int, int, int, int]
+
+
+class GroundError(Exception):
+    pass
+
+
+def coordinate_order() -> str:
+    configured = os.environ.get("VISION_BOX_ORDER", "").strip().lower()
+    if configured:
+        if configured not in {"xyxy", "yxyx"}:
+            raise GroundError("VISION_BOX_ORDER must be either xyxy or yxyx")
+        return configured
+    model = os.environ.get("VISION_MODEL", "").lower()
+    return "xyxy" if "qwen" in model else "yxyx"
+
+
+def build_prompt(target: str, box_order: str = "yxyx") -> str:
+    coordinates = "[x0, y0, x1, y1]" if box_order == "xyxy" else "[y0, x0, y1, x1]"
+    return (
+        "Locate every visible object or region matching this target:\n"
+        f"{target}\n\n"
+        'Return only a JSON array. Each item must contain "box_2d" as '
+        f'{coordinates} on a 0-1000 grid and "label" as a short description. '
+        "Use tight boxes in the original image. Return [] when nothing matches."
+    )
+
+
+def _json_text(text: str) -> str:
+    cleaned = str(text or "").strip()
+    fenced = re.findall(r"```(?:json)?\s*(.*?)\s*```", cleaned, re.DOTALL | re.IGNORECASE)
+    return (fenced[-1] if fenced else cleaned).strip()
+
+
+def _fallback_items(text: str) -> list[dict[str, Any]]:
+    items = []
+    object_pattern = re.compile(r"\{[^{}]*['\"](?:box_2d|bbox_2d|box2d|bbox|box)['\"]\s*:\s*\[[^\]]+\][^{}]*\}", re.DOTALL)
+    box_pattern = re.compile(r"['\"](?:box_2d|bbox_2d|box2d|bbox|box)['\"]\s*:\s*\[([^\]]+)\]", re.DOTALL)
+    label_pattern = re.compile(r"['\"](?:label|caption|description)['\"]\s*:\s*['\"]([^'\"]+)['\"]", re.DOTALL)
+    for match in object_pattern.finditer(text):
+        block = match.group(0)
+        box_match = box_pattern.search(block)
+        if not box_match:
+            continue
+        numbers = re.findall(r"-?\d+(?:\.\d+)?", box_match.group(1))
+        if len(numbers) < 4:
+            continue
+        item: dict[str, Any] = {"box_2d": [float(value) for value in numbers[:4]]}
+        label_match = label_pattern.search(block)
+        if label_match:
+            item["label"] = label_match.group(1).strip()
+        items.append(item)
+    return items
+
+
+def _items(text: str) -> list[Any]:
+    cleaned = _json_text(text)
+    try:
+        payload = json.loads(cleaned)
+    except json.JSONDecodeError:
+        if cleaned.count("```") % 2 or _has_unclosed_json(cleaned):
+            raise GroundError("Vision API bounding-box JSON was truncated or incomplete") from None
+        fallback = _fallback_items(cleaned)
+        if fallback:
+            return fallback
+        raise GroundError("Vision API did not return parseable bounding-box JSON")
+    if isinstance(payload, list):
+        return payload
+    if isinstance(payload, dict):
+        for key in ("boxes", "bounding_boxes", "bboxes", "objects", "items", "results"):
+            if isinstance(payload.get(key), list):
+                return payload[key]
+    raise GroundError("Vision API returned an incompatible bounding-box JSON structure")
+
+
+def _has_unclosed_json(text: str) -> bool:
+    start_positions = [position for position in (text.find("["), text.find("{")) if position >= 0]
+    if not start_positions:
+        return False
+    stack = []
+    in_string = False
+    escaped = False
+    pairs = {"]": "[", "}": "{"}
+    for character in text[min(start_positions):]:
+        if in_string:
+            if escaped:
+                escaped = False
+            elif character == "\\":
+                escaped = True
+            elif character == '"':
+                in_string = False
+            continue
+        if character == '"':
+            in_string = True
+        elif character in "[{":
+            stack.append(character)
+        elif character in "]}":
+            if not stack or stack[-1] != pairs[character]:
+                return False
+            stack.pop()
+    return bool(stack)
+
+
+def _normalize_box(
+    item: dict[str, Any], width: int, height: int, box_order: str = "yxyx",
+) -> tuple[int, int, int, int] | None:
+    raw = item.get("box_2d")
+    if not isinstance(raw, list):
+        for key in ("bbox_2d", "box2d", "bbox", "box"):
+            if isinstance(item.get(key), list):
+                raw = item[key]
+                break
+    if not isinstance(raw, list) or len(raw) != 4:
+        return None
+    try:
+        values = tuple(float(value) for value in raw)
+    except (TypeError, ValueError):
+        return None
+    if box_order == "xyxy":
+        x0, y0, x1, y1 = values
+    else:
+        y0, x0, y1, x1 = values
+    if x0 > x1:
+        x0, x1 = x1, x0
+    if y0 > y1:
+        y0, y1 = y1, y0
+    box = (
+        max(0, min(width, round(x0 / 1000 * width))),
+        max(0, min(height, round(y0 / 1000 * height))),
+        max(0, min(width, round(x1 / 1000 * width))),
+        max(0, min(height, round(y1 / 1000 * height))),
+    )
+    return box if box[2] > box[0] and box[3] > box[1] else None
+
+
+def parse_matches(
+    text: str, width: int, height: int, target: str, box_order: str = "yxyx",
+) -> list[Match]:
+    matches = []
+    for item in _items(text):
+        if not isinstance(item, dict):
+            continue
+        box = _normalize_box(item, width, height, box_order)
+        if box is None:
+            continue
+        label = str(item.get("label") or item.get("caption") or item.get("description") or target).strip()
+        matches.append(Match(label or target, box))
+    return matches
+
+
+def _parse_region(region: str, width: int, height: int) -> tuple[int, int, int, int]:
+    try:
+        x1, y1, x2, y2 = (int(value) for value in region.split(","))
+    except ValueError:
+        raise GroundError("--region expects four integers: X1,Y1,X2,Y2 (pixels)") from None
+    box = (max(0, min(x1, x2)), max(0, min(y1, y2)),
+           min(width, max(x1, x2)), min(height, max(y1, y2)))
+    if box[2] <= box[0] or box[3] <= box[1]:
+        raise GroundError(f"--region {region} is empty after clamping to {width}x{height}")
+    return box
+
+
+def locate(image_path: Path, target: str, region: str | None = None) -> list[Match]:
+    if Image is None:
+        raise GroundError("ground requires Pillow; install the optional dependency pillow first")
+    load_default_env()
+    box = None
+    try:
+        with Image.open(image_path) as image:
+            width, height = image.size
+            if region:
+                box = _parse_region(region, width, height)
+                buffer = io.BytesIO()
+                image.crop(box).save(buffer, format="PNG")
+                url = "data:image/png;base64," + base64.b64encode(buffer.getvalue()).decode()
+    except (OSError, ValueError) as exc:
+        raise GroundError(f"Cannot read image: {image_path}") from exc
+    if box is None:
+        url = image_path_to_data_url(image_path)
+        width_used, height_used = width, height
+    else:
+        width_used, height_used = box[2] - box[0], box[3] - box[1]
+    # 8192 leaves room for exhaustive targets ("every UI element"): a dense
+    # screen can emit dozens of boxes and 2048 truncated the JSON mid-array.
+    box_order = coordinate_order()
+    response = describe_image(url, build_prompt(target, box_order), max_tokens=8192)
+    matches = parse_matches(response, width_used, height_used, target, box_order)
+    if box is None:
+        return matches
+    # Matches were parsed in crop coordinates; report them in the original image.
+    return [Match(m.label, (m.bbox[0] + box[0], m.bbox[1] + box[1],
+                            m.bbox[2] + box[0], m.bbox[3] + box[1])) for m in matches]
+
+
+def _position(box: tuple[int, int, int, int], width: int, height: int) -> str:
+    x1, y1, x2, y2 = box
+    x = (x1 + x2) / 2
+    y = (y1 + y2) / 2
+    horizontal = "left" if x < width / 3 else ("right" if x > width * 2 / 3 else "center")
+    vertical = "top" if y < height / 3 else ("bottom" if y > height * 2 / 3 else "center")
+    return {
+        ("left", "top"): "top-left", ("center", "top"): "top", ("right", "top"): "top-right",
+        ("left", "center"): "left", ("center", "center"): "center", ("right", "center"): "right",
+        ("left", "bottom"): "bottom-left", ("center", "bottom"): "bottom", ("right", "bottom"): "bottom-right",
+    }[(horizontal, vertical)]
+
+
+def format_matches(matches: list[Match], width: int, height: int) -> list[str]:
+    if len(matches) == 1:
+        x1, y1, x2, y2 = matches[0].bbox
+        return [f"x1: {x1}, y1: {y1}, x2: {x2}, y2: {y2}"]
+    lines = []
+    for index, match in enumerate(matches, 1):
+        x1, y1, x2, y2 = match.bbox
+        position = _position(match.bbox, width, height)
+        lines.append(f"{index}. {position} {match.label} x1: {x1}, y1: {y1}, x2: {x2}, y2: {y2}")
+    return lines
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        prog="ground",
+        description="Locate targets in an image with natural language and output pixel coordinates",
+    )
+    parser.add_argument("image", type=Path, help="path to the image")
+    parser.add_argument("target", help="target object or region to locate")
+    parser.add_argument("--region", metavar="X1,Y1,X2,Y2",
+                        help="search only this pixel box; output stays in original-image coordinates")
+    args = parser.parse_args()
+    try:
+        matches = locate(args.image.expanduser(), args.target, region=args.region)
+        if Image is None:
+            raise GroundError("ground requires Pillow; install the optional dependency pillow first")
+        with Image.open(args.image.expanduser()) as image:
+            width, height = image.size
+    except (GroundError, VisionError) as exc:
+        parser.exit(1, f"ground: {exc}\n")
+    for line in format_matches(matches, width, height):
+        print(line)
+
+
+if __name__ == "__main__":
+    main()