@mengruo/dsh-vision-toolkit 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.i18n.yaml +6 -0
- package/README.md +356 -0
- package/README.zh.md +354 -0
- package/assets/community-group-qr.png +0 -0
- package/assets/dsh-conversation-artifact.png +0 -0
- package/assets/dsh-conversation-image-qa-top.png +0 -0
- package/assets/dsh-conversation-image-qa.png +0 -0
- package/assets/dsh-conversation-pixel-diff.png +0 -0
- package/assets/dsh-conversation-screenshot-debugging-top.png +0 -0
- package/assets/dsh-conversation-screenshot-debugging.png +0 -0
- package/assets/dsh-conversation-tool-call.png +0 -0
- package/assets/dsh-conversation-vision-trace.png +0 -0
- package/assets/dsh-view-example.png +0 -0
- package/assets/hero-v2.png +0 -0
- package/assets/logo_aihubmix.png +0 -0
- package/assets/python-bootstrap.json +48 -0
- package/assets/skill/SKILL.md +329 -0
- package/assets/skill/UPSTREAM.json +71 -0
- package/assets/skill/references/gui.md +88 -0
- package/assets/skill/references/long-screenshot-ocr.md +77 -0
- package/assets/skill/references/restore-graphic.md +84 -0
- package/assets/skill/references/restore-structure.md +45 -0
- package/assets/skill/references/restore-ui.md +202 -0
- package/assets/social-preview.png +0 -0
- package/assets/upstream/README.md +18 -0
- package/assets/upstream/focus-hint-comparison-1.webp +0 -0
- package/assets/upstream/focus-hint-comparison-2.webp +0 -0
- package/assets/upstream/infographic-reference.webp +0 -0
- package/assets/upstream/infographic-result.webp +0 -0
- package/assets/upstream/ui-fast-restore-reference.webp +0 -0
- package/assets/upstream/ui-fast-restore-result.webp +0 -0
- package/assets/upstream/ui-result.webp +0 -0
- package/assets/upstream/ui-sketch.webp +0 -0
- package/assets/vision-model-test.png +0 -0
- package/assets/vision-settings.png +0 -0
- package/assets/wechat-reward.png +0 -0
- package/cordis.patch.yml +6 -0
- package/docs/aihubmix-gemini-vision.i18n.yaml +6 -0
- package/docs/aihubmix-gemini-vision.md +138 -0
- package/docs/aihubmix-gemini-vision.zh.md +138 -0
- package/docs/assets/aihubmix-api-keys.png +0 -0
- package/docs/assets/aihubmix-create-key.png +0 -0
- package/docs/assets/aihubmix-free-vision-model.png +0 -0
- package/docs/assets/aihubmix-home.png +0 -0
- package/docs/assets/aihubmix-sign-up.png +0 -0
- package/docs/assets/groq-console-home.png +0 -0
- package/docs/assets/groq-console-keys-login.png +0 -0
- package/docs/assets/groq-docs-quickstart.png +0 -0
- package/docs/assets/groq-docs-qwen3.6.png +0 -0
- package/docs/assets/groq-docs-vision.png +0 -0
- package/docs/assets/vision-settings.png +0 -0
- package/docs/dsh-desktop-install.md +67 -0
- package/docs/dsh-desktop-install.zh.md +67 -0
- package/docs/groq-qwen3.6-vision.i18n.yaml +6 -0
- package/docs/groq-qwen3.6-vision.md +236 -0
- package/docs/groq-qwen3.6-vision.zh.md +236 -0
- package/docs/python-runtime.i18n.yaml +6 -0
- package/docs/python-runtime.md +89 -0
- package/docs/python-runtime.zh.md +89 -0
- package/docs/requirements-traceability/README.i18n.yaml +6 -0
- package/docs/requirements-traceability/README.md +75 -0
- package/docs/requirements-traceability/README.zh.md +75 -0
- package/examples/ui-restoration/README.i18n.yaml +6 -0
- package/examples/ui-restoration/README.md +70 -0
- package/examples/ui-restoration/README.zh.md +70 -0
- package/examples/ui-restoration/assets/final-heatmap.png +0 -0
- package/examples/ui-restoration/assets/final-report.json +83 -0
- package/examples/ui-restoration/assets/implementation.png +0 -0
- package/examples/ui-restoration/assets/initial-heatmap.png +0 -0
- package/examples/ui-restoration/assets/initial-report.json +83 -0
- package/examples/ui-restoration/assets/initial.png +0 -0
- package/examples/ui-restoration/assets/metrics.json +12 -0
- package/examples/ui-restoration/assets/reference.png +0 -0
- package/examples/ui-restoration/implementation.html +94 -0
- package/examples/ui-restoration/initial.html +57 -0
- package/lib/artifact-access.js +369 -0
- package/lib/artifact-access.js.map +1 -0
- package/lib/artifacts.js +56 -0
- package/lib/artifacts.js.map +1 -0
- package/lib/client.js +2099 -0
- package/lib/client.js.map +1 -0
- package/lib/config.js +172 -0
- package/lib/config.js.map +1 -0
- package/lib/defaults.js +6 -0
- package/lib/defaults.js.map +1 -0
- package/lib/errors.js +56 -0
- package/lib/errors.js.map +1 -0
- package/lib/evidence-cache.js +350 -0
- package/lib/evidence-cache.js.map +1 -0
- package/lib/exposure.js +247 -0
- package/lib/exposure.js.map +1 -0
- package/lib/image-input-variants.js +882 -0
- package/lib/image-input-variants.js.map +1 -0
- package/lib/index.js +105 -0
- package/lib/index.js.map +1 -0
- package/lib/paste-images.js +217 -0
- package/lib/paste-images.js.map +1 -0
- package/lib/paths.js +354 -0
- package/lib/paths.js.map +1 -0
- package/lib/plugin-update.js +1003 -0
- package/lib/plugin-update.js.map +1 -0
- package/lib/runtime-install.js +964 -0
- package/lib/runtime-install.js.map +1 -0
- package/lib/runtime-manager.js +133 -0
- package/lib/runtime-manager.js.map +1 -0
- package/lib/runtime.js +1678 -0
- package/lib/runtime.js.map +1 -0
- package/lib/skill.js +25 -0
- package/lib/skill.js.map +1 -0
- package/lib/tools.js +549 -0
- package/lib/tools.js.map +1 -0
- package/lib/types/artifact-access.d.ts +61 -0
- package/lib/types/artifact-access.d.ts.map +1 -0
- package/lib/types/artifacts.d.ts +42 -0
- package/lib/types/artifacts.d.ts.map +1 -0
- package/lib/types/client/display-config.d.ts +24 -0
- package/lib/types/client/display-config.d.ts.map +1 -0
- package/lib/types/client/index.d.ts +350 -0
- package/lib/types/client/index.d.ts.map +1 -0
- package/lib/types/client/model-variants-hider.d.ts +40 -0
- package/lib/types/client/model-variants-hider.d.ts.map +1 -0
- package/lib/types/client/paste-images.d.ts +126 -0
- package/lib/types/client/paste-images.d.ts.map +1 -0
- package/lib/types/config.d.ts +123 -0
- package/lib/types/config.d.ts.map +1 -0
- package/lib/types/defaults.d.ts +6 -0
- package/lib/types/defaults.d.ts.map +1 -0
- package/lib/types/errors.d.ts +35 -0
- package/lib/types/errors.d.ts.map +1 -0
- package/lib/types/evidence-cache.d.ts +91 -0
- package/lib/types/evidence-cache.d.ts.map +1 -0
- package/lib/types/exposure.d.ts +50 -0
- package/lib/types/exposure.d.ts.map +1 -0
- package/lib/types/image-input-variants.d.ts +144 -0
- package/lib/types/image-input-variants.d.ts.map +1 -0
- package/lib/types/index.d.ts +19 -0
- package/lib/types/index.d.ts.map +1 -0
- package/lib/types/paste-images.d.ts +73 -0
- package/lib/types/paste-images.d.ts.map +1 -0
- package/lib/types/paths.d.ts +117 -0
- package/lib/types/paths.d.ts.map +1 -0
- package/lib/types/plugin-update.d.ts +119 -0
- package/lib/types/plugin-update.d.ts.map +1 -0
- package/lib/types/runtime-install.d.ts +92 -0
- package/lib/types/runtime-install.d.ts.map +1 -0
- package/lib/types/runtime-manager.d.ts +60 -0
- package/lib/types/runtime-manager.d.ts.map +1 -0
- package/lib/types/runtime.d.ts +412 -0
- package/lib/types/runtime.d.ts.map +1 -0
- package/lib/types/skill.d.ts +14 -0
- package/lib/types/skill.d.ts.map +1 -0
- package/lib/types/tools.d.ts +35 -0
- package/lib/types/tools.d.ts.map +1 -0
- package/lib/types/upstream.d.ts +233 -0
- package/lib/types/upstream.d.ts.map +1 -0
- package/lib/types/version.d.ts +15 -0
- package/lib/types/version.d.ts.map +1 -0
- package/lib/types/web-request.d.ts +11 -0
- package/lib/types/web-request.d.ts.map +1 -0
- package/lib/types/web.d.ts +118 -0
- package/lib/types/web.d.ts.map +1 -0
- package/lib/upstream.js +860 -0
- package/lib/upstream.js.map +1 -0
- package/lib/version.js +18 -0
- package/lib/version.js.map +1 -0
- package/lib/web-request.js +29 -0
- package/lib/web-request.js.map +1 -0
- package/lib/web.js +462 -0
- package/lib/web.js.map +1 -0
- package/package.json +149 -0
- package/patches/vision-tools-dsh.patch +970 -0
- package/runtime/requirements.lock +3 -0
- package/src/artifact-access.ts +386 -0
- package/src/artifacts.ts +85 -0
- package/src/client/display-config.ts +62 -0
- package/src/client/index.tsx +1580 -0
- package/src/client/model-variants-hider.ts +159 -0
- package/src/client/paste-images.tsx +747 -0
- package/src/config.ts +290 -0
- package/src/defaults.ts +5 -0
- package/src/errors.ts +62 -0
- package/src/evidence-cache.ts +418 -0
- package/src/exposure.ts +265 -0
- package/src/image-input-variants.ts +996 -0
- package/src/index.ts +141 -0
- package/src/paste-images.ts +286 -0
- package/src/paths.ts +383 -0
- package/src/plugin-update.ts +1143 -0
- package/src/runtime-install.ts +1161 -0
- package/src/runtime-manager.ts +174 -0
- package/src/runtime.ts +2189 -0
- package/src/skill.ts +35 -0
- package/src/tools.ts +691 -0
- package/src/upstream.ts +1078 -0
- package/src/version.ts +37 -0
- package/src/web-request.ts +27 -0
- package/src/web.ts +594 -0
- package/vendor/agent-vision-toolkit/CHANGELOG.md +30 -0
- package/vendor/agent-vision-toolkit/LICENSE +21 -0
- package/vendor/agent-vision-toolkit/README.md +405 -0
- package/vendor/agent-vision-toolkit/UPSTREAM_MANIFEST.json +94 -0
- package/vendor/agent-vision-toolkit/bin/crop +90 -0
- package/vendor/agent-vision-toolkit/bin/detect +13 -0
- package/vendor/agent-vision-toolkit/bin/glance +93 -0
- package/vendor/agent-vision-toolkit/bin/ground +13 -0
- package/vendor/agent-vision-toolkit/bin/trace +129 -0
- package/vendor/agent-vision-toolkit/detect.py +60 -0
- package/vendor/agent-vision-toolkit/ground.py +267 -0
- package/vendor/agent-vision-toolkit/skills/vision-tools/scripts/dominant_colors.py +224 -0
- package/vendor/agent-vision-toolkit/skills/vision-tools/scripts/extract_fg.py +278 -0
- package/vendor/agent-vision-toolkit/skills/vision-tools/scripts/html_shot.py +420 -0
- package/vendor/agent-vision-toolkit/skills/vision-tools/scripts/long_screenshot_ocr.py +1245 -0
- package/vendor/agent-vision-toolkit/skills/vision-tools/scripts/pixel_diff.py +88 -0
- package/vendor/agent-vision-toolkit/tests/test_vision_client.py +450 -0
- package/vendor/agent-vision-toolkit/vision_client.py +310 -0
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""pixel_diff: compare a rebuilt image against the original and rank the worst regions.
|
|
3
|
+
|
|
4
|
+
Verification is the step agents skip or fake, because comparing two
|
|
5
|
+
descriptions feels like comparing two images. It is not. This runs the real
|
|
6
|
+
comparison and answers the only question that matters next: which region is
|
|
7
|
+
most wrong, so you know where to look and what to fix first.
|
|
8
|
+
|
|
9
|
+
Boxes are printed in the same `x1: .., y1: ..` form as ground/detect, so a
|
|
10
|
+
bad region can be pasted straight into `glance --region` or `detect --region`.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import argparse
|
|
16
|
+
from pathlib import Path
|
|
17
|
+
|
|
18
|
+
try:
|
|
19
|
+
from PIL import Image, ImageChops, ImageStat
|
|
20
|
+
except ImportError:
|
|
21
|
+
Image = None
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def load(path: Path, size: tuple[int, int] | None = None) -> "Image.Image":
|
|
25
|
+
# Traced SVGs and screenshots of transparent UI render with an alpha
|
|
26
|
+
# channel; unflattened transparency reads as black and shows up as a huge
|
|
27
|
+
# phantom diff. Compositing on white is what a viewer would show.
|
|
28
|
+
image = Image.open(path)
|
|
29
|
+
if image.mode in ("RGBA", "LA", "P"):
|
|
30
|
+
image = image.convert("RGBA")
|
|
31
|
+
canvas = Image.new("RGB", image.size, "white")
|
|
32
|
+
canvas.paste(image, mask=image.split()[-1])
|
|
33
|
+
image = canvas
|
|
34
|
+
else:
|
|
35
|
+
image = image.convert("RGB")
|
|
36
|
+
return image.resize(size, Image.LANCZOS) if size and image.size != size else image
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def cell_scores(diff: "Image.Image", grid: int) -> list[tuple[float, tuple[int, int, int, int]]]:
|
|
40
|
+
width, height = diff.size
|
|
41
|
+
grey = diff.convert("L")
|
|
42
|
+
scores = []
|
|
43
|
+
for row in range(grid):
|
|
44
|
+
for column in range(grid):
|
|
45
|
+
box = (round(column * width / grid), round(row * height / grid),
|
|
46
|
+
round((column + 1) * width / grid), round((row + 1) * height / grid))
|
|
47
|
+
if box[2] > box[0] and box[3] > box[1]:
|
|
48
|
+
mean = ImageStat.Stat(grey.crop(box)).mean[0]
|
|
49
|
+
scores.append((mean / 255 * 100, box))
|
|
50
|
+
return sorted(scores, reverse=True)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def main() -> None:
|
|
54
|
+
parser = argparse.ArgumentParser(
|
|
55
|
+
prog="pixel_diff",
|
|
56
|
+
description="Pixel-diff a rebuilt image against the original and rank the worst regions",
|
|
57
|
+
)
|
|
58
|
+
parser.add_argument("original", type=Path, help="the reference image")
|
|
59
|
+
parser.add_argument("rebuilt", type=Path, help="your rendered reproduction")
|
|
60
|
+
parser.add_argument("--grid", type=int, default=6, help="split into GRID x GRID cells (default: 6)")
|
|
61
|
+
parser.add_argument("--top", type=int, default=5, help="how many worst regions to print (default: 5)")
|
|
62
|
+
parser.add_argument("-o", "--output", type=Path, help="write a diff heatmap image here")
|
|
63
|
+
args = parser.parse_args()
|
|
64
|
+
if Image is None:
|
|
65
|
+
parser.exit(1, "pixel_diff: requires Pillow; install the optional dependency pillow first\n")
|
|
66
|
+
for path in (args.original, args.rebuilt):
|
|
67
|
+
if not path.expanduser().is_file():
|
|
68
|
+
parser.exit(1, f"pixel_diff: image not found: {path}\n")
|
|
69
|
+
original = load(args.original.expanduser())
|
|
70
|
+
with Image.open(args.rebuilt.expanduser()) as probe:
|
|
71
|
+
raw_size = probe.size
|
|
72
|
+
rebuilt = load(args.rebuilt.expanduser(), size=original.size)
|
|
73
|
+
if raw_size != original.size:
|
|
74
|
+
# Worth saying out loud: a size mismatch is itself a finding, and every
|
|
75
|
+
# box printed below is in the original's coordinates, not the rebuild's.
|
|
76
|
+
print(f"note: rebuilt was {raw_size[0]}x{raw_size[1]}, scaled to {original.size[0]}x{original.size[1]}")
|
|
77
|
+
diff = ImageChops.difference(original, rebuilt)
|
|
78
|
+
overall = ImageStat.Stat(diff.convert("L")).mean[0] / 255 * 100
|
|
79
|
+
print(f"overall difference: {overall:.2f}%")
|
|
80
|
+
if args.output:
|
|
81
|
+
diff.save(args.output.expanduser())
|
|
82
|
+
print(f"heatmap: {args.output}")
|
|
83
|
+
for index, (score, box) in enumerate(cell_scores(diff, args.grid)[:args.top], 1):
|
|
84
|
+
print(f"{index}. {score:.2f}% x1: {box[0]}, y1: {box[1]}, x2: {box[2]}, y2: {box[3]}")
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
if __name__ == "__main__":
|
|
88
|
+
main()
|
|
@@ -0,0 +1,450 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Core retry/error test for the shared vision client and glance CLI."""
|
|
3
|
+
|
|
4
|
+
from http.server import BaseHTTPRequestHandler, HTTPServer
|
|
5
|
+
import json
|
|
6
|
+
import os
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
import ssl
|
|
9
|
+
import subprocess
|
|
10
|
+
import sys
|
|
11
|
+
import tempfile
|
|
12
|
+
import threading
|
|
13
|
+
import urllib.error
|
|
14
|
+
|
|
15
|
+
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
|
16
|
+
|
|
17
|
+
import vision_client
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class Handler(BaseHTTPRequestHandler):
|
|
21
|
+
statuses = []
|
|
22
|
+
bodies = []
|
|
23
|
+
response_headers = []
|
|
24
|
+
calls = 0
|
|
25
|
+
last_body = b""
|
|
26
|
+
last_headers = {}
|
|
27
|
+
last_path = ""
|
|
28
|
+
|
|
29
|
+
def do_POST(self):
|
|
30
|
+
Handler.calls += 1
|
|
31
|
+
Handler.last_headers = dict(self.headers)
|
|
32
|
+
Handler.last_path = self.path
|
|
33
|
+
length = int(self.headers.get("Content-Length", 0))
|
|
34
|
+
Handler.last_body = self.rfile.read(length)
|
|
35
|
+
status = Handler.statuses.pop(0)
|
|
36
|
+
if Handler.bodies:
|
|
37
|
+
body = Handler.bodies.pop(0)
|
|
38
|
+
elif status == 200:
|
|
39
|
+
body = json.dumps({"choices": [{"message": {"content": "fixture answer"}}]}).encode()
|
|
40
|
+
else:
|
|
41
|
+
body = b'{"error":{"message":"fixture error"}}'
|
|
42
|
+
self.send_response(status)
|
|
43
|
+
if Handler.response_headers:
|
|
44
|
+
for name, value in Handler.response_headers.pop(0).items():
|
|
45
|
+
self.send_header(name, value)
|
|
46
|
+
self.send_header("Content-Length", str(len(body)))
|
|
47
|
+
self.end_headers()
|
|
48
|
+
self.wfile.write(body)
|
|
49
|
+
|
|
50
|
+
def log_message(self, *_args):
|
|
51
|
+
pass
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def main():
|
|
55
|
+
with tempfile.TemporaryDirectory() as raw:
|
|
56
|
+
root = Path(raw)
|
|
57
|
+
explicit_env = root / "explicit.env"
|
|
58
|
+
explicit_env.write_text("ENV_PRIORITY_PROBE=explicit\n")
|
|
59
|
+
windows_env = root / "local-app-data" / "agent-vision-toolkit" / "env"
|
|
60
|
+
windows_env.parent.mkdir(parents=True)
|
|
61
|
+
windows_env.write_text("ENV_PRIORITY_PROBE=local-app-data\n")
|
|
62
|
+
cwd = root / "cwd"
|
|
63
|
+
cwd.mkdir()
|
|
64
|
+
(cwd / ".env").write_text("ENV_PRIORITY_PROBE=cwd\n")
|
|
65
|
+
previous_cwd = Path.cwd()
|
|
66
|
+
previous_home = os.environ.get("HOME")
|
|
67
|
+
previous_local_appdata = os.environ.get("LOCALAPPDATA")
|
|
68
|
+
previous_explicit = os.environ.get("VISION_ENV_FILE")
|
|
69
|
+
previous_probe = os.environ.get("ENV_PRIORITY_PROBE")
|
|
70
|
+
os.environ["HOME"] = raw
|
|
71
|
+
os.environ["LOCALAPPDATA"] = str(root / "local-app-data")
|
|
72
|
+
os.environ["VISION_ENV_FILE"] = str(explicit_env)
|
|
73
|
+
os.environ.pop("ENV_PRIORITY_PROBE", None)
|
|
74
|
+
os.chdir(cwd)
|
|
75
|
+
try:
|
|
76
|
+
vision_client.load_default_env()
|
|
77
|
+
assert os.environ.get("ENV_PRIORITY_PROBE") == "explicit"
|
|
78
|
+
finally:
|
|
79
|
+
os.chdir(previous_cwd)
|
|
80
|
+
for name, value in (
|
|
81
|
+
("HOME", previous_home),
|
|
82
|
+
("LOCALAPPDATA", previous_local_appdata),
|
|
83
|
+
("VISION_ENV_FILE", previous_explicit),
|
|
84
|
+
("ENV_PRIORITY_PROBE", previous_probe),
|
|
85
|
+
):
|
|
86
|
+
if value is None:
|
|
87
|
+
os.environ.pop(name, None)
|
|
88
|
+
else:
|
|
89
|
+
os.environ[name] = value
|
|
90
|
+
|
|
91
|
+
with tempfile.TemporaryDirectory() as raw:
|
|
92
|
+
windows_env = Path(raw) / "agent-vision-toolkit" / "env"
|
|
93
|
+
windows_env.parent.mkdir()
|
|
94
|
+
windows_env.write_text("WINDOWS_ENV_PROBE=loaded\n")
|
|
95
|
+
previous_local_appdata = os.environ.get("LOCALAPPDATA")
|
|
96
|
+
previous_explicit = os.environ.get("VISION_ENV_FILE")
|
|
97
|
+
os.environ["LOCALAPPDATA"] = raw
|
|
98
|
+
os.environ.pop("VISION_ENV_FILE", None)
|
|
99
|
+
os.environ.pop("WINDOWS_ENV_PROBE", None)
|
|
100
|
+
try:
|
|
101
|
+
vision_client.load_default_env()
|
|
102
|
+
assert os.environ.get("WINDOWS_ENV_PROBE") == "loaded"
|
|
103
|
+
finally:
|
|
104
|
+
os.environ.pop("WINDOWS_ENV_PROBE", None)
|
|
105
|
+
if previous_explicit is None:
|
|
106
|
+
os.environ.pop("VISION_ENV_FILE", None)
|
|
107
|
+
else:
|
|
108
|
+
os.environ["VISION_ENV_FILE"] = previous_explicit
|
|
109
|
+
if previous_local_appdata is None:
|
|
110
|
+
os.environ.pop("LOCALAPPDATA", None)
|
|
111
|
+
else:
|
|
112
|
+
os.environ["LOCALAPPDATA"] = previous_local_appdata
|
|
113
|
+
|
|
114
|
+
server = HTTPServer(("127.0.0.1", 0), Handler)
|
|
115
|
+
threading.Thread(target=server.serve_forever, daemon=True).start()
|
|
116
|
+
environment = dict(os.environ, VISION_API_KEY="test-key",
|
|
117
|
+
VISION_BASE_URL=f"http://127.0.0.1:{server.server_port}/v1",
|
|
118
|
+
VISION_MODEL="fixture-model")
|
|
119
|
+
environment.pop("VISION_API_PROTOCOL", None)
|
|
120
|
+
environment.pop("VISION_REASONING_EFFORT", None)
|
|
121
|
+
environment.pop("VISION_ANTHROPIC_THINKING", None)
|
|
122
|
+
environment.pop("VISION_SSL_VERIFY", None)
|
|
123
|
+
environment.pop("VISION_USER_AGENT", None)
|
|
124
|
+
environment.pop("VISION_ENV_FILE", None)
|
|
125
|
+
saved = dict(os.environ)
|
|
126
|
+
os.environ.pop("VISION_API_PROTOCOL", None)
|
|
127
|
+
os.environ.pop("VISION_REASONING_EFFORT", None)
|
|
128
|
+
os.environ.pop("VISION_ANTHROPIC_THINKING", None)
|
|
129
|
+
os.environ.pop("VISION_SSL_VERIFY", None)
|
|
130
|
+
os.environ.pop("VISION_USER_AGENT", None)
|
|
131
|
+
os.environ.pop("VISION_ENV_FILE", None)
|
|
132
|
+
os.environ.update(environment)
|
|
133
|
+
try:
|
|
134
|
+
assert vision_client._ssl_context() is None
|
|
135
|
+
for disabled_value in ("0", "false", "off", "no", "none", "disabled", " FALSE "):
|
|
136
|
+
os.environ["VISION_SSL_VERIFY"] = disabled_value
|
|
137
|
+
context = vision_client._ssl_context()
|
|
138
|
+
assert isinstance(context, ssl.SSLContext)
|
|
139
|
+
assert context.check_hostname is False
|
|
140
|
+
assert context.verify_mode == ssl.CERT_NONE
|
|
141
|
+
for enabled_value in ("", "1", "true", "on", "yes"):
|
|
142
|
+
os.environ["VISION_SSL_VERIFY"] = enabled_value
|
|
143
|
+
assert vision_client._ssl_context() is None
|
|
144
|
+
os.environ.pop("VISION_SSL_VERIFY", None)
|
|
145
|
+
|
|
146
|
+
Handler.calls, Handler.statuses, Handler.bodies = 0, [200, 200], []
|
|
147
|
+
original_urlopen = vision_client.urllib.request.urlopen
|
|
148
|
+
contexts = []
|
|
149
|
+
|
|
150
|
+
def capture_context(*args, **kwargs):
|
|
151
|
+
contexts.append(kwargs.get("context"))
|
|
152
|
+
return original_urlopen(*args, **kwargs)
|
|
153
|
+
|
|
154
|
+
vision_client.urllib.request.urlopen = capture_context
|
|
155
|
+
try:
|
|
156
|
+
os.environ["VISION_SSL_VERIFY"] = "off"
|
|
157
|
+
assert vision_client.describe_image("data:image/png;base64,AAAA") == "fixture answer"
|
|
158
|
+
os.environ["VISION_SSL_VERIFY"] = "true"
|
|
159
|
+
assert vision_client.describe_image("data:image/png;base64,AAAA") == "fixture answer"
|
|
160
|
+
finally:
|
|
161
|
+
vision_client.urllib.request.urlopen = original_urlopen
|
|
162
|
+
os.environ.pop("VISION_SSL_VERIFY", None)
|
|
163
|
+
assert contexts[0].check_hostname is False
|
|
164
|
+
assert contexts[0].verify_mode == ssl.CERT_NONE
|
|
165
|
+
assert contexts[1] is None
|
|
166
|
+
|
|
167
|
+
Handler.calls, Handler.statuses, Handler.bodies, Handler.response_headers = (
|
|
168
|
+
0, [429, 200], [], [{"Retry-After": "17"}, {}]
|
|
169
|
+
)
|
|
170
|
+
original_sleep = vision_client.time.sleep
|
|
171
|
+
delays = []
|
|
172
|
+
vision_client.time.sleep = delays.append
|
|
173
|
+
try:
|
|
174
|
+
assert vision_client.describe_image("data:image/png;base64,AAAA") == "fixture answer"
|
|
175
|
+
finally:
|
|
176
|
+
vision_client.time.sleep = original_sleep
|
|
177
|
+
assert Handler.calls == 2
|
|
178
|
+
assert delays == [17.0]
|
|
179
|
+
assert Handler.last_headers.get("User-Agent") == vision_client.DEFAULT_USER_AGENT
|
|
180
|
+
assert not Handler.last_headers["User-Agent"].startswith("Python-urllib/")
|
|
181
|
+
|
|
182
|
+
Handler.calls, Handler.statuses, Handler.bodies, Handler.response_headers = (
|
|
183
|
+
0, [429], [b'{"error":{"code":"daily_rate_limit_exceeded","message":"Daily limit reached"}}'],
|
|
184
|
+
[{"Retry-After": "3600"}]
|
|
185
|
+
)
|
|
186
|
+
delays = []
|
|
187
|
+
original_sleep = vision_client.time.sleep
|
|
188
|
+
vision_client.time.sleep = delays.append
|
|
189
|
+
try:
|
|
190
|
+
vision_client.describe_image("data:image/png;base64,AAAA")
|
|
191
|
+
except vision_client.VisionError as exc:
|
|
192
|
+
assert "daily_rate_limit_exceeded" in str(exc)
|
|
193
|
+
else:
|
|
194
|
+
raise AssertionError("daily quota exhaustion must fail immediately")
|
|
195
|
+
finally:
|
|
196
|
+
vision_client.time.sleep = original_sleep
|
|
197
|
+
assert Handler.calls == 1, "daily quota exhaustion must not be retried"
|
|
198
|
+
assert delays == []
|
|
199
|
+
|
|
200
|
+
Handler.calls, Handler.statuses, Handler.bodies = 0, [200], []
|
|
201
|
+
os.environ["VISION_USER_AGENT"] = "custom-vision-client/2.0"
|
|
202
|
+
try:
|
|
203
|
+
assert vision_client.describe_image("data:image/png;base64,AAAA") == "fixture answer"
|
|
204
|
+
finally:
|
|
205
|
+
os.environ.pop("VISION_USER_AGENT", None)
|
|
206
|
+
assert Handler.last_headers.get("User-Agent") == "custom-vision-client/2.0"
|
|
207
|
+
assert Handler.calls == 1
|
|
208
|
+
|
|
209
|
+
Handler.calls, Handler.statuses, Handler.bodies = 0, [401], []
|
|
210
|
+
try:
|
|
211
|
+
vision_client.describe_image("data:image/png;base64,AAAA")
|
|
212
|
+
except vision_client.VisionError:
|
|
213
|
+
pass
|
|
214
|
+
else:
|
|
215
|
+
raise AssertionError("401 must fail cleanly")
|
|
216
|
+
assert Handler.calls == 1, "401 must not be retried"
|
|
217
|
+
|
|
218
|
+
Handler.calls, Handler.statuses, Handler.bodies = (
|
|
219
|
+
0, [403], [b'{"error":"Cloudflare 1010 rejected test-key"}']
|
|
220
|
+
)
|
|
221
|
+
try:
|
|
222
|
+
vision_client.describe_image("data:image/png;base64,AAAA")
|
|
223
|
+
except vision_client.VisionError as exc:
|
|
224
|
+
assert "test-key" not in str(exc)
|
|
225
|
+
assert "<redacted>" in str(exc)
|
|
226
|
+
else:
|
|
227
|
+
raise AssertionError("HTTP errors must fail cleanly")
|
|
228
|
+
assert Handler.calls == 1, "403 must not be retried"
|
|
229
|
+
|
|
230
|
+
original_urlopen = vision_client.urllib.request.urlopen
|
|
231
|
+
original_sleep = vision_client.time.sleep
|
|
232
|
+
|
|
233
|
+
def fail_with_secret(*_args, **_kwargs):
|
|
234
|
+
raise urllib.error.URLError("connection failed for test-key")
|
|
235
|
+
|
|
236
|
+
vision_client.urllib.request.urlopen = fail_with_secret
|
|
237
|
+
vision_client.time.sleep = lambda _seconds: None
|
|
238
|
+
try:
|
|
239
|
+
try:
|
|
240
|
+
vision_client.describe_image("data:image/png;base64,AAAA")
|
|
241
|
+
except vision_client.VisionError as exc:
|
|
242
|
+
assert "test-key" not in str(exc)
|
|
243
|
+
assert "<redacted>" in str(exc)
|
|
244
|
+
else:
|
|
245
|
+
raise AssertionError("network errors must fail with redacted details")
|
|
246
|
+
finally:
|
|
247
|
+
vision_client.urllib.request.urlopen = original_urlopen
|
|
248
|
+
vision_client.time.sleep = original_sleep
|
|
249
|
+
|
|
250
|
+
Handler.calls, Handler.statuses, Handler.bodies = 0, [200], []
|
|
251
|
+
os.environ["LANG"] = "en"
|
|
252
|
+
try:
|
|
253
|
+
vision_client.describe_image("data:image/png;base64,AAAA")
|
|
254
|
+
finally:
|
|
255
|
+
os.environ.pop("LANG", None)
|
|
256
|
+
parts = json.loads(Handler.last_body)["messages"][0]["content"]
|
|
257
|
+
text = next(part["text"] for part in parts if part.get("type") == "text")
|
|
258
|
+
assert text.startswith("Please respond in English.")
|
|
259
|
+
assert Handler.calls == 1
|
|
260
|
+
|
|
261
|
+
Handler.calls, Handler.statuses, Handler.bodies = 0, [200], []
|
|
262
|
+
os.environ["LANG"] = "zh"
|
|
263
|
+
try:
|
|
264
|
+
vision_client.describe_image("data:image/png;base64,AAAA")
|
|
265
|
+
finally:
|
|
266
|
+
os.environ.pop("LANG", None)
|
|
267
|
+
parts = json.loads(Handler.last_body)["messages"][0]["content"]
|
|
268
|
+
text = next(part["text"] for part in parts if part.get("type") == "text")
|
|
269
|
+
assert text.startswith("请使用简体中文回答。")
|
|
270
|
+
assert Handler.calls == 1
|
|
271
|
+
|
|
272
|
+
Handler.calls, Handler.statuses, Handler.bodies = 0, [200], []
|
|
273
|
+
os.environ.pop("LANG", None)
|
|
274
|
+
vision_client.describe_image("data:image/png;base64,AAAA")
|
|
275
|
+
parts = json.loads(Handler.last_body)["messages"][0]["content"]
|
|
276
|
+
text = next(part["text"] for part in parts if part.get("type") == "text")
|
|
277
|
+
assert "Please respond in English." not in text
|
|
278
|
+
assert "请使用简体中文回答。" not in text
|
|
279
|
+
assert Handler.calls == 1
|
|
280
|
+
|
|
281
|
+
Handler.calls, Handler.statuses, Handler.bodies = 0, [200], []
|
|
282
|
+
vision_client.describe_image(["data:image/png;base64,AAAA", "data:image/png;base64,BBBB"])
|
|
283
|
+
content = json.loads(Handler.last_body)["messages"][0]["content"]
|
|
284
|
+
assert content[0].get("type") == "image_url", \
|
|
285
|
+
"vision payloads must put image parts before text for OpenCode Go MiMo compatibility"
|
|
286
|
+
assert Handler.last_headers.get("User-Agent") == vision_client.DEFAULT_USER_AGENT
|
|
287
|
+
image_parts = [part for part in content if part.get("type") == "image_url"]
|
|
288
|
+
assert len(image_parts) == 2, "a list of URLs must become one request with all images"
|
|
289
|
+
assert Handler.calls == 1
|
|
290
|
+
|
|
291
|
+
Handler.calls, Handler.statuses, Handler.bodies, Handler.response_headers = 0, [200], [json.dumps({
|
|
292
|
+
"object": "response",
|
|
293
|
+
"output": [{
|
|
294
|
+
"type": "message",
|
|
295
|
+
"content": [{"type": "output_text", "text": "responses fixture answer"}],
|
|
296
|
+
}],
|
|
297
|
+
}).encode()], []
|
|
298
|
+
os.environ["VISION_API_PROTOCOL"] = "responses"
|
|
299
|
+
os.environ["VISION_REASONING_EFFORT"] = "medium"
|
|
300
|
+
try:
|
|
301
|
+
assert vision_client.describe_image(
|
|
302
|
+
["data:image/png;base64,AAAA", "data:image/png;base64,BBBB"],
|
|
303
|
+
prompt="read both images",
|
|
304
|
+
max_tokens=123,
|
|
305
|
+
) == "responses fixture answer"
|
|
306
|
+
finally:
|
|
307
|
+
os.environ.pop("VISION_API_PROTOCOL", None)
|
|
308
|
+
os.environ.pop("VISION_REASONING_EFFORT", None)
|
|
309
|
+
assert Handler.last_path == "/v1/responses"
|
|
310
|
+
payload = json.loads(Handler.last_body)
|
|
311
|
+
content = payload["input"][0]["content"]
|
|
312
|
+
assert [part["type"] for part in content] == ["input_image", "input_image", "input_text"]
|
|
313
|
+
assert payload["store"] is False
|
|
314
|
+
assert payload["max_output_tokens"] == 123
|
|
315
|
+
assert payload["reasoning"] == {"effort": "medium"}
|
|
316
|
+
assert Handler.calls == 1
|
|
317
|
+
|
|
318
|
+
Handler.calls, Handler.statuses, Handler.bodies = 0, [200], [json.dumps({
|
|
319
|
+
"content": [
|
|
320
|
+
{"type": "thinking", "thinking": "internal reasoning"},
|
|
321
|
+
{"type": "text", "text": "anthropic fixture answer"},
|
|
322
|
+
{"type": "text", "text": "second text block"},
|
|
323
|
+
],
|
|
324
|
+
"usage": {"input_tokens": 42, "output_tokens": 7},
|
|
325
|
+
}).encode()]
|
|
326
|
+
os.environ["VISION_API_PROTOCOL"] = "anthropic"
|
|
327
|
+
try:
|
|
328
|
+
assert vision_client.describe_image(
|
|
329
|
+
["data:image/png;base64,AAAA", "https://example.com/remote.webp"],
|
|
330
|
+
prompt="read both images",
|
|
331
|
+
max_tokens=123,
|
|
332
|
+
) == "anthropic fixture answer\nsecond text block"
|
|
333
|
+
finally:
|
|
334
|
+
os.environ.pop("VISION_API_PROTOCOL", None)
|
|
335
|
+
assert Handler.last_path == "/v1/messages"
|
|
336
|
+
assert next(v for k, v in Handler.last_headers.items() if k.lower() == "x-api-key") == "test-key"
|
|
337
|
+
assert next(v for k, v in Handler.last_headers.items() if k.lower() == "anthropic-version") == "2023-06-01"
|
|
338
|
+
assert not any(k.lower() == "authorization" for k in Handler.last_headers)
|
|
339
|
+
payload = json.loads(Handler.last_body)
|
|
340
|
+
assert payload["max_tokens"] == 123
|
|
341
|
+
assert "thinking" not in payload
|
|
342
|
+
content = payload["messages"][0]["content"]
|
|
343
|
+
assert [part["type"] for part in content] == ["image", "image", "text"]
|
|
344
|
+
assert content[0]["source"] == {
|
|
345
|
+
"type": "base64", "media_type": "image/png", "data": "AAAA"
|
|
346
|
+
}
|
|
347
|
+
assert content[1]["source"] == {"type": "url", "url": "https://example.com/remote.webp"}
|
|
348
|
+
assert Handler.calls == 1
|
|
349
|
+
|
|
350
|
+
Handler.calls, Handler.statuses, Handler.bodies, Handler.response_headers = (
|
|
351
|
+
0, [529, 200], [b'{"error":{"type":"overloaded_error"}}', json.dumps({
|
|
352
|
+
"content": [{"type": "text", "text": "recovered"}],
|
|
353
|
+
}).encode()], [{"Retry-After": "3"}, {}]
|
|
354
|
+
)
|
|
355
|
+
delays = []
|
|
356
|
+
original_sleep = vision_client.time.sleep
|
|
357
|
+
vision_client.time.sleep = delays.append
|
|
358
|
+
os.environ["VISION_API_PROTOCOL"] = "anthropic"
|
|
359
|
+
os.environ["VISION_ANTHROPIC_THINKING"] = "disabled"
|
|
360
|
+
try:
|
|
361
|
+
assert vision_client.describe_image("data:image/png;base64,AAAA") == "recovered"
|
|
362
|
+
finally:
|
|
363
|
+
os.environ.pop("VISION_API_PROTOCOL", None)
|
|
364
|
+
os.environ.pop("VISION_ANTHROPIC_THINKING", None)
|
|
365
|
+
vision_client.time.sleep = original_sleep
|
|
366
|
+
assert json.loads(Handler.last_body)["thinking"] == {"type": "disabled"}
|
|
367
|
+
assert delays == [3.0]
|
|
368
|
+
assert Handler.calls == 2
|
|
369
|
+
|
|
370
|
+
Handler.calls, Handler.statuses, Handler.bodies = 0, [200], [json.dumps({
|
|
371
|
+
"content": [{"type": "text", "text": "adaptive answer"}],
|
|
372
|
+
}).encode()]
|
|
373
|
+
os.environ["VISION_API_PROTOCOL"] = "anthropic"
|
|
374
|
+
os.environ["VISION_ANTHROPIC_THINKING"] = "adaptive"
|
|
375
|
+
try:
|
|
376
|
+
assert vision_client.describe_image("data:image/png;base64,AAAA") == "adaptive answer"
|
|
377
|
+
finally:
|
|
378
|
+
os.environ.pop("VISION_API_PROTOCOL", None)
|
|
379
|
+
os.environ.pop("VISION_ANTHROPIC_THINKING", None)
|
|
380
|
+
assert json.loads(Handler.last_body)["thinking"] == {"type": "adaptive"}
|
|
381
|
+
assert Handler.calls == 1
|
|
382
|
+
|
|
383
|
+
Handler.calls, Handler.statuses, Handler.bodies, Handler.response_headers = 0, [], [], []
|
|
384
|
+
os.environ["VISION_API_PROTOCOL"] = "anthropic"
|
|
385
|
+
os.environ["VISION_ANTHROPIC_THINKING"] = "unsupported"
|
|
386
|
+
try:
|
|
387
|
+
try:
|
|
388
|
+
vision_client.describe_image("data:image/png;base64,AAAA")
|
|
389
|
+
except vision_client.VisionError as exc:
|
|
390
|
+
assert "Unsupported VISION_ANTHROPIC_THINKING" in str(exc)
|
|
391
|
+
else:
|
|
392
|
+
raise AssertionError("an unsupported thinking mode must fail before making a request")
|
|
393
|
+
finally:
|
|
394
|
+
os.environ.pop("VISION_API_PROTOCOL", None)
|
|
395
|
+
os.environ.pop("VISION_ANTHROPIC_THINKING", None)
|
|
396
|
+
assert Handler.calls == 0
|
|
397
|
+
|
|
398
|
+
Handler.calls, Handler.statuses, Handler.bodies = 0, [], []
|
|
399
|
+
os.environ["VISION_API_PROTOCOL"] = "unsupported"
|
|
400
|
+
try:
|
|
401
|
+
try:
|
|
402
|
+
vision_client.describe_image("data:image/png;base64,AAAA")
|
|
403
|
+
except vision_client.VisionError as exc:
|
|
404
|
+
assert "Unsupported VISION_API_PROTOCOL" in str(exc)
|
|
405
|
+
else:
|
|
406
|
+
raise AssertionError("an unsupported protocol must fail before making a request")
|
|
407
|
+
finally:
|
|
408
|
+
os.environ.pop("VISION_API_PROTOCOL", None)
|
|
409
|
+
assert Handler.calls == 0
|
|
410
|
+
|
|
411
|
+
Handler.calls, Handler.statuses, Handler.bodies = 0, [200], []
|
|
412
|
+
with tempfile.TemporaryDirectory() as raw:
|
|
413
|
+
image = Path(raw) / "fixture.png"
|
|
414
|
+
image.write_bytes(b"\x89PNG\r\n\x1a\nfixture")
|
|
415
|
+
# Pin the subprocess to one explicit fixture file so caller and
|
|
416
|
+
# checkout env files cannot redirect requests away from the server.
|
|
417
|
+
fixture_env = Path(raw) / "vision.env"
|
|
418
|
+
fixture_env.write_text(
|
|
419
|
+
"VISION_API_KEY=test-key\n"
|
|
420
|
+
f"VISION_BASE_URL=http://127.0.0.1:{server.server_port}/v1\n"
|
|
421
|
+
"VISION_MODEL=fixture-model\n"
|
|
422
|
+
"VISION_API_PROTOCOL=chat_completions\n"
|
|
423
|
+
)
|
|
424
|
+
isolated_env = dict(environment, HOME=raw, VISION_ENV_FILE=str(fixture_env))
|
|
425
|
+
glance = Path(__file__).resolve().parent.parent / "bin/glance"
|
|
426
|
+
glance_cmd = [sys.executable, str(glance)] if os.name == "nt" else [str(glance)]
|
|
427
|
+
result = subprocess.run(
|
|
428
|
+
[*glance_cmd, str(image), "-q", "图里有什么?"],
|
|
429
|
+
env=isolated_env, cwd=raw, text=True, capture_output=True, check=True,
|
|
430
|
+
)
|
|
431
|
+
assert result.stdout.strip() == "fixture answer"
|
|
432
|
+
|
|
433
|
+
Handler.calls, Handler.statuses, Handler.bodies = 0, [200], []
|
|
434
|
+
result = subprocess.run(
|
|
435
|
+
[*glance_cmd, str(image), str(image), "-q", "differences?"],
|
|
436
|
+
env=isolated_env, cwd=raw, text=True, capture_output=True, check=True,
|
|
437
|
+
)
|
|
438
|
+
assert result.stdout.strip() == "fixture answer"
|
|
439
|
+
content = json.loads(Handler.last_body)["messages"][0]["content"]
|
|
440
|
+
assert sum(part.get("type") == "image_url" for part in content) == 2, \
|
|
441
|
+
"glance with two paths must send both images in one call"
|
|
442
|
+
finally:
|
|
443
|
+
server.shutdown()
|
|
444
|
+
os.environ.clear()
|
|
445
|
+
os.environ.update(saved)
|
|
446
|
+
print("VISION CLIENT TEST PASS")
|
|
447
|
+
|
|
448
|
+
|
|
449
|
+
if __name__ == "__main__":
|
|
450
|
+
main()
|