@mengruo/dsh-vision-toolkit 0.1.4 → 0.1.5

package/src/web.ts

@@ -75,6 +75,12 @@ export interface VisionToolkitSettingsSnapshot {
     source?: string
     writable: boolean
   }>
+  /** Object-storage credential state; `ref` is empty when object storage is unset. */
+  objectStorageCredential: {
+    ref: string
+    configured: boolean
+    writable: boolean
+  }
   runtime: RuntimeManagerStatus
   release: {
     pluginVersion: string
@@ -121,7 +127,11 @@ interface ApplyUpdateRequest {
   expectedVersion: string
 }
 
-type SettingsRequest = SaveRequest | HealthRequest | CredentialRequest | DeleteCredentialRequest | CheckUpdateRequest | ApplyUpdateRequest
+interface StorageTestRequest {
+  action: 'test-storage'
+}
+
+type SettingsRequest = SaveRequest | HealthRequest | CredentialRequest | DeleteCredentialRequest | CheckUpdateRequest | ApplyUpdateRequest | StorageTestRequest
 
 interface JsonError {
   ok: false
@@ -261,6 +271,7 @@ function parseRequest(value: unknown): SettingsRequest {
     }
     return { action: 'apply-update', expectedVersion: value.expectedVersion.trim() }
   }
+  if (value.action === 'test-storage') return { action: 'test-storage' }
   throw new TypeError(`unsupported action: ${value.action}`)
 }
 
@@ -315,6 +326,14 @@ export class VisionToolkitWebBackend {
         writable: info.writable,
       }
     }))
+    const objectStorageRef = resolved.objectStorage.credential === undefined ? '' : String(resolved.objectStorage.credential)
+    const objectStorageCredential = objectStorageRef === ''
+      ? { ref: '', configured: false, writable: this.ctx.settings.writable }
+      : await this.ctx.credentials.describe(credentialRef(objectStorageRef)).then(info => ({
+        ref: objectStorageRef,
+        configured: info.configured,
+        writable: info.writable,
+      }))
     const update = await this.updater.capability()
     return {
       schemaVersion: 1,
@@ -333,6 +352,7 @@ export class VisionToolkitWebBackend {
         writable: credential.writable,
       },
       credentials,
+      objectStorageCredential,
       runtime: this.manager.status(),
       release: {
         pluginVersion: PLUGIN_VERSION,
@@ -380,10 +400,18 @@ export class VisionToolkitWebBackend {
       )
     }
     const resolved = resolveConfig(descriptor.value as VisionToolkitConfig)
-    const provider = resolved.providers.find(entry => String(entry.credential) === String(request.ref))
+    const ref = String(request.ref)
+    if (resolved.objectStorage.credential !== undefined && String(resolved.objectStorage.credential) === ref) {
+      if (!request.value.includes(':')) {
+        throw new Error('object storage credential must be "accessKeyId:secretAccessKey"')
+      }
+      await this.ctx.credentials.set(request.ref, request.value)
+      return this.snapshot()
+    }
+    const provider = resolved.providers.find(entry => String(entry.credential) === ref)
     if (provider === undefined) {
       throw new CredentialReferenceConflictError(
-        `credential reference "${String(request.ref)}" does not match any configured vision provider; reload Settings and try again`,
+        `credential reference "${ref}" does not match any configured vision provider; reload Settings and try again`,
       )
     }
     if (isBuiltInFreeVisionProvider(provider)) {
@@ -426,6 +454,11 @@ export class VisionToolkitWebBackend {
     }
   }
 
+  private async testStorage(): Promise<{ detail: string }> {
+    if (!this.manager.ready) throw new Error('runtime is not ready; fix Settings and save a valid configuration first')
+    return this.manager.current().testObjectStorage()
+  }
+
   /** Handle the exact Settings route. */
   async handle(req: IncomingMessage, res: ServerResponse): Promise<void> {
     if (req.method === 'GET') {
@@ -458,6 +491,9 @@ export class VisionToolkitWebBackend {
         case 'health':
           responseJson(res, 200, { ok: true, value: await this.health(parsed, req) })
           break
+        case 'test-storage':
+          responseJson(res, 200, { ok: true, value: await this.testStorage() })
+          break
         case 'save':
           responseJson(res, 200, { ok: true, value: await this.save(parsed) })
           break
@@ -486,14 +522,16 @@ export class VisionToolkitWebBackend {
             ? error.code
           : parsed.action === 'health'
             ? 'health-failed'
-            : parsed.action === 'credential'
-              ? 'credential-rejected'
-              : 'settings-rejected'
+            : parsed.action === 'test-storage'
+              ? 'storage-test-failed'
+              : parsed.action === 'credential'
+                ? 'credential-rejected'
+                : 'settings-rejected'
       const updateConflict = updateError && ['update-in-progress', 'update-stale', 'update-unavailable', 'already-current'].includes(error.code)
       const updateGateway = updateError && error.code === 'update-check-failed'
       const status = settingsConflict || credentialConflict || updateConflict
         ? 409
-        : parsed.action === 'health'
+        : parsed.action === 'health' || parsed.action === 'test-storage'
           ? 503
           : updateGateway
             ? 502

package/vendor/agent-vision-toolkit/UPSTREAM_MANIFEST.json

@@ -3,7 +3,7 @@
   "repository": "https://github.com/Anionex/agent-vision-toolkit",
   "version": "v0.1.0+snapshot.bc9803d",
   "commit": "bc9803d7d6300c864d17460ecbb33540b26638e0",
-  "contentSha256": "0eaa22a0d1d0dd6d6523a0ddd3fa8197246005042bc164a703fb05102b287a38",
+  "contentSha256": "09bedf007e1469a28981227def5aa33e03667ea21fe2972937f0359898e2928e",
   "files": [
     {
       "path": "CHANGELOG.md",
@@ -32,8 +32,8 @@
     },
     {
       "path": "bin/glance",
-      "bytes": 3742,
-      "sha256": "fa4ba52e8e180475b948daec817151d893d957f973f7d9df1b8bbf201051cefc"
+      "bytes": 3941,
+      "sha256": "7552bef0ba396162f64e8fc4c4a53404dfe2d3149ca3d1c5c8bd717facf82af6"
     },
     {
       "path": "bin/ground",
@@ -47,13 +47,13 @@
     },
     {
       "path": "detect.py",
-      "bytes": 2218,
-      "sha256": "48a7070084f5b23b1477fa9a690ef1e679da8a03228e64a1ac583de633040bfc"
+      "bytes": 2532,
+      "sha256": "cac2135f9d835215f3ba90bfd70fd84dd02b0e88299f2b60e109e006b4f3a4d9"
     },
     {
       "path": "ground.py",
-      "bytes": 10117,
-      "sha256": "845e56dbdf92f2c79495170f5d215de49985b3099012b4d398231d3c78bd0090"
+      "bytes": 11289,
+      "sha256": "6d3175d3f5a7d561d55b2a967c5ca3d49940e80014de658ed9a3e5d45a233a2f"
     },
     {
       "path": "skills/vision-tools/scripts/dominant_colors.py",
@@ -82,13 +82,13 @@
     },
     {
       "path": "tests/test_vision_client.py",
-      "bytes": 20481,
-      "sha256": "b019f0da28b7567226292044f791059ee34d7f0088bd9ceb328cfa1bab250a48"
+      "bytes": 24882,
+      "sha256": "87e39429766b38caed6a4ccb1ac8b193094c69858a0f7541940a005f66421e45"
     },
     {
       "path": "vision_client.py",
-      "bytes": 11838,
-      "sha256": "c6a48048c864e99513faf90ba17c4ac0d9462bb718683682ed8eda12c0e2cfe0"
+      "bytes": 15053,
+      "sha256": "6800843b99df500a476b1dd6ff5a4c05aa8122ed990f6f91b16a11ee07bdebe6"
     }
   ]
 }

package/vendor/agent-vision-toolkit/bin/glance

@@ -40,6 +40,13 @@ def region_data_url(path, region):
     return "data:image/png;base64," + base64.b64encode(buffer.getvalue()).decode()
 
 
+def image_url(path):
+    """Pass http(s) URLs through unchanged; encode local files as data URLs."""
+    if path.startswith(("http://", "https://")):
+        return path
+    return image_path_to_data_url(path)
+
+
 def build_prompt(args, count):
     if args.ocr is not None:
         extra = f" Additional requirements: {args.ocr}" if args.ocr else ""
@@ -77,7 +84,7 @@ def main():
         if args.region and len(args.images) > 1:
             raise VisionError("--region works with exactly one image")
         urls = ([region_data_url(args.images[0], args.region)] if args.region
-                else [image_path_to_data_url(path) for path in args.images])
+                else [image_url(path) for path in args.images])
         answer = describe_image(
             urls,
             build_prompt(args, len(urls)),

package/vendor/agent-vision-toolkit/detect.py

@@ -8,7 +8,7 @@ try:
 except ImportError:
     Image = None
 
-from ground import GroundError, _position, locate
+from ground import GroundError, _parse_size, _position, locate
 from vision_client import VisionError
 
 DEFAULT_CATEGORY = ("UI element (buttons, links, inputs, icons, labels, "
@@ -38,18 +38,24 @@ def main() -> None:
         prog="detect",
         description="Inventory the elements in an image (or a region) with pixel bounding boxes",
     )
-    parser.add_argument("image", type=Path, help="path to the image")
+    parser.add_argument("image", help="path or http(s) URL to the image")
     parser.add_argument("category", nargs="?",
                         help='restrict to a category, e.g. "buttons" or "icons" (default: all UI elements)')
     parser.add_argument("--region", metavar="X1,Y1,X2,Y2",
                         help="inventory only this pixel box; output stays in original-image coordinates")
+    parser.add_argument("--size", metavar="WxH",
+                        help="analyzed image dimensions when the image is an http(s) URL")
     args = parser.parse_args()
     try:
-        matches = locate(args.image.expanduser(), build_target(args.category), region=args.region)
-        if Image is None:
-            raise GroundError("detect requires Pillow; install the optional dependency pillow first")
-        with Image.open(args.image.expanduser()) as image:
-            width, height = image.size
+        size = _parse_size(args.size) if args.size else None
+        matches = locate(args.image, build_target(args.category), region=args.region, size=size)
+        if size is not None:
+            width, height = size
+        else:
+            if Image is None:
+                raise GroundError("detect requires Pillow; install the optional dependency pillow first")
+            with Image.open(Path(args.image).expanduser()) as image:
+                width, height = image.size
     except (GroundError, VisionError) as exc:
         parser.exit(1, f"detect: {exc}\n")
     for line in format_inventory(matches, width, height):

package/vendor/agent-vision-toolkit/ground.py

@@ -184,23 +184,32 @@ def _parse_region(region: str, width: int, height: int) -> tuple[int, int, int,
     return box
 
 
-def locate(image_path: Path, target: str, region: str | None = None) -> list[Match]:
+def locate(image_path: str, target: str, region: str | None = None,
+           size: tuple[int, int] | None = None) -> list[Match]:
     if Image is None:
         raise GroundError("ground requires Pillow; install the optional dependency pillow first")
     load_default_env()
     box = None
-    try:
-        with Image.open(image_path) as image:
-            width, height = image.size
-            if region:
-                box = _parse_region(region, width, height)
-                buffer = io.BytesIO()
-                image.crop(box).save(buffer, format="PNG")
-                url = "data:image/png;base64," + base64.b64encode(buffer.getvalue()).decode()
-    except (OSError, ValueError) as exc:
-        raise GroundError(f"Cannot read image: {image_path}") from exc
+    if size is not None:
+        # URL transfer: the image was uploaded to object storage and this caller
+        # supplies the analyzed dimensions, so no local file access is needed.
+        width, height = size
+        url = image_path if image_path.startswith(("http://", "https://")) else image_path_to_data_url(image_path)
+    else:
+        local = Path(image_path).expanduser()
+        try:
+            with Image.open(local) as image:
+                width, height = image.size
+                if region:
+                    box = _parse_region(region, width, height)
+                    buffer = io.BytesIO()
+                    image.crop(box).save(buffer, format="PNG")
+                    url = "data:image/png;base64," + base64.b64encode(buffer.getvalue()).decode()
+        except (OSError, ValueError) as exc:
+            raise GroundError(f"Cannot read image: {image_path}") from exc
+        if box is None:
+            url = image_path_to_data_url(local)
     if box is None:
-        url = image_path_to_data_url(image_path)
         width_used, height_used = width, height
     else:
         width_used, height_used = box[2] - box[0], box[3] - box[1]
@@ -216,6 +225,16 @@ def locate(image_path: Path, target: str, region: str | None = None) -> list[Mat
                             m.bbox[2] + box[0], m.bbox[3] + box[1])) for m in matches]
 
 
+def _parse_size(value: str) -> tuple[int, int]:
+    try:
+        width, height = (int(part) for part in value.lower().split("x"))
+    except (ValueError, AttributeError):
+        raise GroundError("--size expects WIDTHxHEIGHT (e.g. 1024x768)") from None
+    if width <= 0 or height <= 0:
+        raise GroundError("--size expects positive WIDTHxHEIGHT")
+    return width, height
+
+
 def _position(box: tuple[int, int, int, int], width: int, height: int) -> str:
     x1, y1, x2, y2 = box
     x = (x1 + x2) / 2
@@ -246,17 +265,23 @@ def main() -> None:
         prog="ground",
         description="Locate targets in an image with natural language and output pixel coordinates",
     )
-    parser.add_argument("image", type=Path, help="path to the image")
+    parser.add_argument("image", help="path or http(s) URL to the image")
     parser.add_argument("target", help="target object or region to locate")
     parser.add_argument("--region", metavar="X1,Y1,X2,Y2",
                         help="search only this pixel box; output stays in original-image coordinates")
+    parser.add_argument("--size", metavar="WxH",
+                        help="analyzed image dimensions when the image is an http(s) URL")
     args = parser.parse_args()
     try:
-        matches = locate(args.image.expanduser(), args.target, region=args.region)
-        if Image is None:
-            raise GroundError("ground requires Pillow; install the optional dependency pillow first")
-        with Image.open(args.image.expanduser()) as image:
-            width, height = image.size
+        size = _parse_size(args.size) if args.size else None
+        matches = locate(args.image, args.target, region=args.region, size=size)
+        if size is not None:
+            width, height = size
+        else:
+            if Image is None:
+                raise GroundError("ground requires Pillow; install the optional dependency pillow first")
+            with Image.open(Path(args.image).expanduser()) as image:
+                width, height = image.size
     except (GroundError, VisionError) as exc:
         parser.exit(1, f"ground: {exc}\n")
     for line in format_matches(matches, width, height):

package/vendor/agent-vision-toolkit/tests/test_vision_client.py

@@ -408,6 +408,94 @@ def main():
             os.environ.pop("VISION_API_PROTOCOL", None)
         assert Handler.calls == 0
 
+        # VISION_STREAM truthiness mirrors VISION_SSL_VERIFY: off by default.
+        for enabled_value in ("1", "true", "yes", "on", " TRUE "):
+            os.environ["VISION_STREAM"] = enabled_value
+            assert vision_client._stream_enabled() is True
+        for disabled_value in ("", "0", "false", "no", "off", "disabled"):
+            os.environ["VISION_STREAM"] = disabled_value
+            assert vision_client._stream_enabled() is False
+        os.environ.pop("VISION_STREAM", None)
+
+        # Streamed chat_completions: `stream: true` is sent and deltas accumulate.
+        Handler.calls, Handler.statuses, Handler.bodies, Handler.response_headers = 0, [200], [
+            (
+                'data: {"choices":[{"delta":{"content":"Hello "}}]}\n\n'
+                'data: {"choices":[{"delta":{"content":"streamed"}}]}\n\n'
+                'data: {"choices":[{"delta":{},"finish_reason":"stop"}]}\n\n'
+                'data: [DONE]\n\n'
+            ).encode(),
+        ], []
+        os.environ["VISION_STREAM"] = "1"
+        try:
+            assert vision_client.describe_image("data:image/png;base64,AAAA") == "Hello streamed"
+        finally:
+            os.environ.pop("VISION_STREAM", None)
+        assert json.loads(Handler.last_body)["stream"] is True
+        assert Handler.calls == 1
+
+        # Default is non-streaming: no `stream` key is sent.
+        Handler.calls, Handler.statuses, Handler.bodies = 0, [200], []
+        vision_client.describe_image("data:image/png;base64,AAAA")
+        assert "stream" not in json.loads(Handler.last_body)
+        assert Handler.calls == 1
+
+        # Anthropic streaming: text_delta deltas accumulate across blocks.
+        Handler.calls, Handler.statuses, Handler.bodies, Handler.response_headers = 0, [200], [
+            (
+                'data: {"type":"message_start","message":{"id":"m1"}}\n\n'
+                'data: {"type":"content_block_start","index":0,"content_block":{"type":"text"}}\n\n'
+                'data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"anthropic "}}\n\n'
+                'data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"streamed"}}\n\n'
+                'data: {"type":"content_block_stop","index":0}\n\n'
+                'data: {"type":"message_stop"}\n\n'
+            ).encode(),
+        ], []
+        os.environ["VISION_API_PROTOCOL"] = "anthropic"
+        os.environ["VISION_STREAM"] = "1"
+        try:
+            assert vision_client.describe_image("data:image/png;base64,AAAA") == "anthropic streamed"
+        finally:
+            os.environ.pop("VISION_API_PROTOCOL", None)
+            os.environ.pop("VISION_STREAM", None)
+        assert json.loads(Handler.last_body)["stream"] is True
+        assert Handler.calls == 1
+
+        # Responses streaming: output_text deltas accumulate.
+        Handler.calls, Handler.statuses, Handler.bodies, Handler.response_headers = 0, [200], [
+            (
+                'data: {"type":"response.output_text.delta","delta":"responses "}\n\n'
+                'data: {"type":"response.output_text.delta","delta":"streamed"}\n\n'
+                'data: {"type":"response.completed"}\n\n'
+            ).encode(),
+        ], []
+        os.environ["VISION_API_PROTOCOL"] = "responses"
+        os.environ["VISION_STREAM"] = "1"
+        try:
+            assert vision_client.describe_image("data:image/png;base64,AAAA") == "responses streamed"
+        finally:
+            os.environ.pop("VISION_API_PROTOCOL", None)
+            os.environ.pop("VISION_STREAM", None)
+        assert json.loads(Handler.last_body)["stream"] is True
+        assert Handler.calls == 1
+
+        # A streamed error event surfaces as a redacted VisionError.
+        Handler.calls, Handler.statuses, Handler.bodies = 0, [200], [
+            b'data: {"error":{"message":"overloaded for test-key","code":"overloaded"}}\n\n'
+        ]
+        os.environ["VISION_STREAM"] = "1"
+        try:
+            try:
+                vision_client.describe_image("data:image/png;base64,AAAA")
+            except vision_client.VisionError as exc:
+                assert "test-key" not in str(exc)
+                assert "<redacted>" in str(exc)
+            else:
+                raise AssertionError("streamed errors must fail cleanly")
+        finally:
+            os.environ.pop("VISION_STREAM", None)
+        assert Handler.calls == 1
+
         Handler.calls, Handler.statuses, Handler.bodies = 0, [200], []
         with tempfile.TemporaryDirectory() as raw:
             image = Path(raw) / "fixture.png"

package/vendor/agent-vision-toolkit/vision_client.py

@@ -196,6 +196,77 @@ def _retryable_http_error(status: int, body: bytes) -> bool:
     }
 
 
+def _stream_enabled() -> bool:
+    """Whether VISION_STREAM asks for a streamed completion (default off)."""
+    value = os.environ.get("VISION_STREAM", "").strip().lower()
+    return value in {"1", "true", "yes", "on"}
+
+
+def _stream_error(payload: dict) -> str | None:
+    """Return a human-readable error detail from one SSE event, or None."""
+    error = payload.get("error")
+    if isinstance(error, dict):
+        code = error.get("code")
+        message = error.get("message")
+        if code is not None or message is not None:
+            return str(message) if message is not None else str(code)
+        return json.dumps(error)
+    if payload.get("type") == "error":
+        error = payload.get("error")
+        if isinstance(error, dict) and error.get("message") is not None:
+            return str(error["message"])
+    return None
+
+
+def _stream_text(stream, protocol: str, api_key: str) -> str:
+    """Accumulate the text deltas of a server-sent-events completion stream."""
+    parts: list[str] = []
+    error_detail: str | None = None
+    for raw_line in stream:
+        if isinstance(raw_line, (bytes, bytearray)):
+            line = raw_line.decode("utf-8", errors="replace")
+        else:
+            line = str(raw_line)
+        line = line.strip()
+        if not line or line.startswith(":"):
+            continue
+        data = line[5:].strip() if line.startswith("data:") else line
+        if data == "[DONE]":
+            break
+        try:
+            payload = json.loads(data)
+        except (json.JSONDecodeError, ValueError):
+            continue
+        if not isinstance(payload, dict):
+            continue
+        error_detail = _stream_error(payload)
+        if error_detail is not None:
+            break
+        if protocol == "anthropic":
+            if payload.get("type") == "content_block_delta":
+                delta = payload.get("delta")
+                if isinstance(delta, dict) and delta.get("type") == "text_delta":
+                    text = delta.get("text")
+                    if isinstance(text, str):
+                        parts.append(text)
+        elif protocol == "responses":
+            if payload.get("type") == "response.output_text.delta":
+                delta = payload.get("delta")
+                if isinstance(delta, str):
+                    parts.append(delta)
+        else:  # chat_completions
+            choices = payload.get("choices")
+            if isinstance(choices, list) and choices:
+                choice = choices[0]
+                delta = choice.get("delta") if isinstance(choice, dict) else None
+                content = delta.get("content") if isinstance(delta, dict) else None
+                if isinstance(content, str):
+                    parts.append(content)
+    if error_detail is not None:
+        raise VisionError(f"Vision API stream error: {_redact(error_detail, api_key)}")
+    return "".join(parts)
+
+
 def describe_image(image_url: str | list[str], prompt: str | None = None, max_tokens: int = 4096,
                    apply_lang: bool = True) -> str:
     """Describe one data/http image URL (str) or several (list) in a single call."""
@@ -216,6 +287,7 @@ def describe_image(image_url: str | list[str], prompt: str | None = None, max_to
             text = f"{instruction}\n\n{text}"
     model = _required("VISION_MODEL")
     protocol = os.environ.get("VISION_API_PROTOCOL", "").strip().lower() or "chat_completions"
+    stream = _stream_enabled()
     if protocol == "responses":
         payload = {
             "model": model,
@@ -263,6 +335,8 @@ def describe_image(image_url: str | list[str], prompt: str | None = None, max_to
         raise VisionError(
             "Unsupported VISION_API_PROTOCOL; use chat_completions, responses, or anthropic"
         )
+    if stream:
+        payload["stream"] = True
     headers = {
         "Content-Type": "application/json",
         "User-Agent": user_agent,
@@ -280,12 +354,16 @@ def describe_image(image_url: str | list[str], prompt: str | None = None, max_to
     timeout = 180
     for attempt in range(retries + 1):
         try:
-            with urllib.request.urlopen(request, timeout=timeout, context=context) as response:
-                data = json.load(response)
-            try:
-                text = extract_text(data)
-            except (KeyError, IndexError, TypeError) as exc:
-                raise VisionError("Vision API returned an incompatible response structure") from exc
+            if stream:
+                with urllib.request.urlopen(request, timeout=timeout, context=context) as response:
+                    text = _stream_text(response, protocol, api_key)
+            else:
+                with urllib.request.urlopen(request, timeout=timeout, context=context) as response:
+                    data = json.load(response)
+                try:
+                    text = extract_text(data)
+                except (KeyError, IndexError, TypeError) as exc:
+                    raise VisionError("Vision API returned an incompatible response structure") from exc
             if not text:
                 raise VisionError("Vision API returned an empty description")
             return text