@qingflow-tech/qingflow-cli 1.1.60 → 1.1.61-beta.1

package/src/qingflow_mcp/tools/data_files.py

@@ -0,0 +1,174 @@
+from __future__ import annotations
+
+import mimetypes
+import os
+import re
+import time
+from pathlib import Path
+from typing import Any, Callable
+from urllib.parse import unquote, urlsplit
+from uuid import uuid4
+
+from ..backend_client import BackendRequestContext
+from ..config import get_mcp_home
+from ..errors import QingflowApiError
+from ..json_types import JSONObject
+from .storage_download import download_storage_content, requires_storage_cookie, storage_auth_type
+
+
+DATA_FILE_MAX_FILES = 50
+DATA_FILE_MAX_BYTES = 50 * 1024 * 1024
+DATA_FILE_MAX_TOTAL_BYTES = 200 * 1024 * 1024
+DATA_FILE_TIME_BUDGET_SECONDS = 55.0
+
+
+def attachment_candidates(record: JSONObject, schema: JSONObject) -> list[JSONObject]:
+    candidates: list[JSONObject] = []
+
+    def walk(values: Any, fields: Any, prefix: str) -> None:
+        if not isinstance(values, dict) or not isinstance(fields, list):
+            return
+        for field in fields:
+            if not isinstance(field, dict) or field.get("field_id") is None:
+                continue
+            key = str(field["field_id"])
+            value = values.get(key)
+            path = f"{prefix}.{key}"
+            if field.get("data_type") == "table" and isinstance(value, list):
+                for index, row in enumerate(value):
+                    walk(row, field.get("children"), f"{path}[{index}]")
+            elif field.get("data_type") == "attachment":
+                for index, item in enumerate(value if isinstance(value, list) else [value]):
+                    url = item.get("value") or item.get("url") if isinstance(item, dict) else item
+                    if not isinstance(url, str) or not url.strip():
+                        continue
+                    name = (item.get("other_info") or item.get("otherInfo") or item.get("name")) if isinstance(item, dict) else None
+                    if not isinstance(name, str) or not name.strip():
+                        try:
+                            name = Path(unquote(urlsplit(url).path)).name
+                        except ValueError:
+                            name = None
+                        name = name or f"attachment_{len(candidates) + 1}"
+                    candidates.append({
+                        "field_id": field["field_id"], "file_name": name,
+                        "source_path": f"{path}[{index}]", "source_url": url.strip(),
+                    })
+
+    walk(record.get("fields"), schema.get("fields"), "fields")
+    return candidates
+
+
+def _download_url(candidate: JSONObject) -> str:
+    url = str(candidate["source_url"])
+    parsed = urlsplit(url)
+    if parsed.scheme not in {"http", "https"} or not parsed.netloc or parsed.username or parsed.password:
+        raise QingflowApiError.config_error("附件没有可直接下载的 HTTP 地址")
+    return url
+
+
+def _matching_attachment(
+    candidate: JSONObject, refreshed: list[JSONObject],
+) -> JSONObject | None:
+    # shortcut: 沿用旧链路，同一字段内同名附件取首个；需要区分时再引入稳定文件 ID。
+    return next((item for item in refreshed if (
+        str(item["field_id"]) == str(candidate["field_id"])
+        and str(item["file_name"]).strip() == str(candidate["file_name"]).strip()
+    )), None)
+
+
+def download_data_attachments(
+    *, backend: Any, context: BackendRequestContext, record: JSONObject, schema: JSONObject,
+    refresh_record: Callable[[], JSONObject],
+) -> JSONObject:
+    candidates = attachment_candidates(record, schema)
+    if not candidates:
+        return {"status": "none", "local_dir": None, "items": [], "warnings": []}
+    custom_home = os.environ.get("QINGFLOW_MCP_DATA_FILES_HOME")
+    local_dir = (Path(custom_home).expanduser() if custom_home else get_mcp_home() / "data-files") / uuid4().hex
+    items: list[JSONObject] = []
+    warnings: list[JSONObject] = []
+    prefix_cache: dict[str, str] = {}
+    fresh_candidates: list[JSONObject] | None = None
+    deadline = time.monotonic() + DATA_FILE_TIME_BUDGET_SECONDS
+    total_bytes = 0
+
+    for index, original in enumerate(candidates):
+        item: JSONObject = {**original, "local_path": None}
+        items.append(item)
+        if index >= DATA_FILE_MAX_FILES or time.monotonic() >= deadline:
+            item["access_status"] = "skipped_limit"
+            warnings.append({"code": "FILE_ASSET_LIMIT_EXCEEDED", "message": "附件数量或下载时间超过本次预算"})
+            continue
+        try:
+            if fresh_candidates is not None:
+                replacement = _matching_attachment(item, fresh_candidates)
+                if replacement is None:
+                    raise QingflowApiError(category="http", message="刷新后附件不在当前可见范围内", http_status=403)
+                item["source_url"] = replacement["source_url"]
+            url = _download_url(item)
+
+            def download(source_url: str) -> tuple[bytes, JSONObject]:
+                return download_storage_content(
+                    backend=backend, context=context, source_url=source_url, warnings=warnings,
+                    environment_prefix_cache=prefix_cache, operation="data_get",
+                    max_bytes=min(DATA_FILE_MAX_BYTES, DATA_FILE_MAX_TOTAL_BYTES - total_bytes),
+                    requested_strategy="qingflow_storage_cookie_redirect" if requires_storage_cookie(
+                        source_url, storage_auth_type(source_url)
+                    ) else "referer_acl",
+                )
+
+            try:
+                content, metadata = download(url)
+            except QingflowApiError as exc:
+                if exc.http_status not in {401, 403}:
+                    raise
+                if fresh_candidates is None:
+                    fresh_candidates = []
+                    fresh = refresh_record()
+                    if fresh.get("ok") is False:
+                        raise QingflowApiError(
+                            category="backend", message=str(fresh.get("message") or "刷新附件访问范围失败"),
+                            backend_code=fresh.get("backend_code"),
+                        ) from exc
+                    if not isinstance(fresh.get("record"), dict) or not isinstance(fresh.get("schema"), dict):
+                        raise QingflowApiError(category="backend", message="刷新附件需要数据面返回 record 和 schema") from exc
+                    fresh_candidates = attachment_candidates(fresh["record"], fresh["schema"])
+                replacement = _matching_attachment(item, fresh_candidates)
+                if replacement is None or replacement["source_url"] == url:
+                    raise
+                url = _download_url(replacement)
+                content, metadata = download(url)
+                item["source_url"] = url
+                warnings.append({
+                    "code": "FILE_ASSET_STORAGE_URL_REFRESHED", "source_path": item["source_path"],
+                    "message": "已通过数据面重新读取同一记录并刷新附件链接",
+                })
+            item.update(metadata)
+            item["size_bytes"] = len(content)
+            if len(content) > DATA_FILE_MAX_BYTES or total_bytes + len(content) > DATA_FILE_MAX_TOTAL_BYTES:
+                item["access_status"] = "too_large"
+                warnings.append({"code": "FILE_ASSET_SIZE_LIMIT_EXCEEDED", "message": "附件超过单文件或总大小预算"})
+                continue
+            suffix = Path(str(item["file_name"])).suffix
+            suffix = suffix if re.fullmatch(r"\.[A-Za-z0-9]{1,10}", suffix) else ""
+            local_dir.mkdir(parents=True, exist_ok=True)
+            local_path = local_dir / f"file_{index + 1:04d}{suffix}"
+            local_path.write_bytes(content)
+            total_bytes += len(content)
+            item.update({
+                "local_path": str(local_path), "access_status": "downloaded",
+                "mime_type": mimetypes.guess_type(str(item["file_name"]))[0],
+            })
+        except (QingflowApiError, OSError, ValueError) as exc:
+            blocked = isinstance(exc, QingflowApiError) and exc.http_status in {401, 403}
+            too_large = isinstance(exc, QingflowApiError) and exc.category == "file_size_limit"
+            item["access_status"] = "too_large" if too_large else "blocked_private_url" if blocked else "download_failed"
+            warnings.append({
+                "code": "FILE_ASSET_SIZE_LIMIT_EXCEEDED" if too_large else "FILE_ASSET_DOWNLOAD_FAILED", "source_path": item["source_path"],
+                "message": exc.message if isinstance(exc, QingflowApiError) else str(exc),
+                "http_status": exc.http_status if isinstance(exc, QingflowApiError) else None,
+            })
+    return {
+        "status": "ok" if all(item["access_status"] == "downloaded" for item in items) else "partial",
+        "local_dir": str(local_dir) if local_dir.exists() else None, "items": items, "warnings": warnings,
+    }

package/src/qingflow_mcp/tools/data_tools.py

@@ -2,9 +2,10 @@ from __future__ import annotations
 
 from datetime import datetime
 from pathlib import Path
-from typing import Any, Literal, TypedDict
+from typing import Any, Literal
 
 from mcp.server.fastmcp import FastMCP
+from typing_extensions import TypedDict
 
 from ..backend_client import BackendRequestContext
 from ..config import DEFAULT_PROFILE
@@ -12,6 +13,7 @@ from ..errors import QingflowApiError, raise_tool_error
 from ..json_types import JSONObject, JSONValue
 from .base import ToolBase
 from .csv_export import CsvPageFailure, csv_cell, export_csv_pages
+from .data_files import download_data_attachments
 
 
 DATA_PAGE_SIZE = 1000
@@ -64,7 +66,11 @@ class DataTools(ToolBase):
             all_pages: bool = False,
             output: str = "json",
         ) -> JSONObject:
-            """按字段 ID 查询应用数据；先读取 field_capabilities 再构造 where 和 sort。"""
+            """按字段 ID 查询应用数据；先读取 field_capabilities 再构造 where 和 sort。
+
+            查询不下载附件；需要下载时对目标 app_key、record_id 调用
+            data_get(download_attachments=true)，使用 file_assets.items[].local_path。
+            """
             return self.data_query(
                 profile=profile, app_key=app_key, columns=columns or [], where=where or [], sort=sort or [],
                 search=search, page=page, page_size=page_size, include_schema=include_schema,
@@ -73,9 +79,18 @@ class DataTools(ToolBase):
 
         @mcp.tool()
         def data_get(
-            profile: str = DEFAULT_PROFILE, app_key: str = "", record_id: str = "", include_schema: bool = True
+            profile: str = DEFAULT_PROFILE, app_key: str = "", record_id: str = "", include_schema: bool = True,
+            download_attachments: bool = False,
         ) -> JSONObject:
-            return self.data_get(profile=profile, app_key=app_key, record_id=record_id, include_schema=include_schema)
+            """读取单条应用数据；需要下载附件或图片时设置 download_attachments=true，
+            保留 include_schema=true，并使用 file_assets.items[].local_path。
+            下载复用当前登录身份，通过数据面刷新链接；不需要 view_id，也不调用 record_get。
+            默认只读取字段；file_assets.status=partial 表示部分附件下载失败，请读取 warnings。
+            """
+            return self.data_get(
+                profile=profile, app_key=app_key, record_id=record_id, include_schema=include_schema,
+                download_attachments=download_attachments,
+            )
 
         @mcp.tool()
         def data_insert(
@@ -128,19 +143,33 @@ class DataTools(ToolBase):
 
         return self._run(profile, runner, tool_name="数据面查询")
 
-    def data_get(self, *, profile: str, app_key: str, record_id: str | int, include_schema: bool = True) -> JSONObject:
+    def data_get(
+        self, *, profile: str, app_key: str, record_id: str | int, include_schema: bool = True,
+        download_attachments: bool = False,
+    ) -> JSONObject:
         self._require_app_key(app_key)
         self._require_record_id(record_id)
-        return self._run(
-            profile,
-            lambda session_profile, context: self._attach_context(
-                self._request_data_plane(
+        if download_attachments and not include_schema:
+            raise_tool_error(QingflowApiError.config_error("下载附件需要 schema，请保留 include_schema=true（不要使用 --no-schema）"))
+
+        def runner(session_profile, context):
+            def read_record():
+                return self._request_data_plane(
                     context, "GET", f"{self._base_path(app_key)}/{record_id}",
                     params={"include_schema": include_schema},
-                ), profile, session_profile.selected_ws_id,
-            ),
-            tool_name="数据面单条读取",
-        )
+                )
+
+            result = read_record()
+            if download_attachments and result.get("ok") is not False:
+                if not isinstance(result.get("record"), dict) or not isinstance(result.get("schema"), dict):
+                    raise QingflowApiError(category="backend", message="下载附件需要数据面返回 record 和 schema")
+                result["file_assets"] = download_data_attachments(
+                    backend=self.backend, context=context, record=result.get("record", {}),
+                    schema=result.get("schema", {}), refresh_record=read_record,
+                )
+            return self._attach_context(result, profile, session_profile.selected_ws_id)
+
+        return self._run(profile, runner, tool_name="数据面单条读取")
 
     def data_insert(
         self, *, profile: str, app_key: str, items: list[DataInsertItem], include_schema: bool = True

package/src/qingflow_mcp/tools/record_tools.py

@@ -34,6 +34,10 @@ from ..list_type_labels import (
 from .base import ToolBase
 from .directory_tools import _directory_has_more, _directory_items
 from .csv_export import CsvShardWriter
+from .storage_download import (
+    download_storage_content, environment_cookie_prefix, is_storage_url,
+    requires_storage_cookie, storage_auth_type,
+)
 
 
 DEFAULT_QUERY_PAGE_SIZE = 50
@@ -17237,40 +17241,10 @@ def _record_detail_download_resolved_media_content(
     environment_prefix_cache: dict[str, str],
     requested_strategy: str,
 ) -> tuple[bytes, JSONObject]:
-    storage_auth_type = _record_detail_storage_auth_type(source_url)
-    if _record_detail_should_use_storage_cookie(source_url, storage_auth_type):
-        storage_cookie_prefix = _record_detail_environment_prefix(
-            backend,
-            context,
-            warnings=warnings,
-            environment_prefix_cache=environment_prefix_cache,
-        )
-        cookie_name = f"{storage_cookie_prefix}token"
-        headers = _record_detail_media_download_headers(context)
-        if hasattr(backend, "download_binary_with_cookie"):
-            response = backend.download_binary_with_cookie(context, source_url, cookie_name=cookie_name, headers=headers)
-            return response.content, {
-                "download_strategy": requested_strategy,
-                "storage_auth_type": storage_auth_type,
-                "storage_cookie_prefix": storage_cookie_prefix,
-                "redirected": bool(getattr(response, "redirected", False)),
-            }
-        cookie_headers = dict(headers)
-        cookie_headers["Cookie"] = f"{cookie_name}={context.token}"
-        content = backend.download_binary(source_url, headers=cookie_headers)
-        return content, {
-            "download_strategy": requested_strategy,
-            "storage_auth_type": storage_auth_type,
-            "storage_cookie_prefix": storage_cookie_prefix,
-            "redirected": False,
-        }
-    content = backend.download_binary(source_url, headers=_record_detail_media_download_headers(context))
-    return content, {
-        "download_strategy": requested_strategy,
-        "storage_auth_type": storage_auth_type,
-        "storage_cookie_prefix": None,
-        "redirected": False,
-    }
+    return download_storage_content(
+        backend=backend, context=context, source_url=source_url, warnings=warnings,
+        environment_prefix_cache=environment_prefix_cache, requested_strategy=requested_strategy,
+    )
 
 
 def _record_detail_media_download_strategy(source_url: str) -> str:
@@ -17282,28 +17256,15 @@ def _record_detail_media_download_strategy(source_url: str) -> str:
 
 
 def _record_detail_should_use_storage_cookie(source_url: str, storage_auth_type: str | None) -> bool:
-    if not _record_detail_is_qingflow_storage_url(source_url):
-        return False
-    if not storage_auth_type:
-        return True
-    return storage_auth_type.upper() != "ANONYMOUS"
+    return requires_storage_cookie(source_url, storage_auth_type)
 
 
 def _record_detail_is_qingflow_storage_url(source_url: str) -> bool:
-    parsed = urlsplit(source_url)
-    path = parsed.path.lower()
-    if "/storage/file/" in path:
-        return True
-    query = parse_qs(parsed.query)
-    return any(key.lower() == "qingflow-auth_type" for key in query)
+    return is_storage_url(source_url)
 
 
 def _record_detail_storage_auth_type(source_url: str) -> str | None:
-    query = parse_qs(urlsplit(source_url).query)
-    for key, values in query.items():
-        if key.lower() == "qingflow-auth_type" and values:
-            return _normalize_optional_text(values[0])
-    return None
+    return storage_auth_type(source_url)
 
 
 def _record_detail_is_download_file_url(source_url: str) -> bool:
@@ -17353,28 +17314,9 @@ def _record_detail_environment_prefix(
     warnings: list[JSONObject],
     environment_prefix_cache: dict[str, str],
 ) -> str:
-    if "value" in environment_prefix_cache:
-        return environment_prefix_cache["value"]
-    try:
-        payload = backend.request("GET", context, "/environment/properties")
-        environment = payload.get("environmentProperty") if isinstance(payload, dict) else None
-        if not isinstance(environment, dict):
-            environment = payload.get("environment_property") if isinstance(payload, dict) else None
-        prefix = _normalize_optional_text(environment.get("environmentPrefix")) if isinstance(environment, dict) else None
-        if not prefix and isinstance(environment, dict):
-            prefix = _normalize_optional_text(environment.get("environment_prefix"))
-    except QingflowApiError as exc:
-        prefix = None
-        warnings.append(
-            {
-                "code": "MEDIA_ASSET_ENVIRONMENT_PREFIX_UNAVAILABLE",
-                "message": f"record_get could not read /environment/properties; falling back to qf_ token cookie prefix: {exc.message}",
-                "http_status": exc.http_status,
-            }
-        )
-    prefix = prefix if prefix is not None else "qf_"
-    environment_prefix_cache["value"] = prefix
-    return prefix
+    return environment_cookie_prefix(
+        backend, context, warnings=warnings, environment_prefix_cache=environment_prefix_cache,
+    )
 
 
 def _record_detail_context_origin(context: BackendRequestContext) -> str:

package/src/qingflow_mcp/tools/storage_download.py

@@ -0,0 +1,89 @@
+from __future__ import annotations
+
+from typing import Any
+from urllib.parse import parse_qs, urlsplit
+
+from ..backend_client import BackendRequestContext
+from ..config import DEFAULT_USER_AGENT
+from ..errors import QingflowApiError
+from ..json_types import JSONObject
+
+
+def storage_auth_type(source_url: str) -> str | None:
+    for key, values in parse_qs(urlsplit(source_url).query).items():
+        if key.lower() == "qingflow-auth_type" and values:
+            return values[0].strip() or None
+    return None
+
+
+def is_storage_url(source_url: str) -> bool:
+    parsed = urlsplit(source_url)
+    return "/storage/file/" in parsed.path.lower() or any(
+        key.lower() == "qingflow-auth_type" for key in parse_qs(parsed.query)
+    )
+
+
+def requires_storage_cookie(source_url: str, auth_type: str | None) -> bool:
+    return is_storage_url(source_url) and (not auth_type or auth_type.upper() != "ANONYMOUS")
+
+
+def environment_cookie_prefix(
+    backend: Any, context: BackendRequestContext, *, warnings: list[JSONObject],
+    environment_prefix_cache: dict[str, str], operation: str = "record_get",
+) -> str:
+    if "value" in environment_prefix_cache:
+        return environment_prefix_cache["value"]
+    try:
+        payload = backend.request("GET", context, "/environment/properties")
+        environment = payload.get("environmentProperty") if isinstance(payload, dict) else None
+        if not isinstance(environment, dict):
+            environment = payload.get("environment_property") if isinstance(payload, dict) else None
+        prefix = environment.get("environmentPrefix") if isinstance(environment, dict) else None
+        if not prefix and isinstance(environment, dict):
+            prefix = environment.get("environment_prefix")
+        if prefix is not None:
+            prefix = str(prefix).strip() or None
+    except QingflowApiError as exc:
+        prefix = None
+        warnings.append({
+            "code": "MEDIA_ASSET_ENVIRONMENT_PREFIX_UNAVAILABLE",
+            "message": f"{operation} could not read /environment/properties; falling back to qf_ token cookie prefix: {exc.message}",
+            "http_status": exc.http_status,
+        })
+    prefix = prefix if prefix is not None else "qf_"
+    environment_prefix_cache["value"] = prefix
+    return prefix
+
+
+def download_storage_content(
+    *, backend: Any, context: BackendRequestContext, source_url: str,
+    warnings: list[JSONObject], environment_prefix_cache: dict[str, str],
+    requested_strategy: str, operation: str = "record_get",
+    max_bytes: int | None = None,
+) -> tuple[bytes, JSONObject]:
+    parsed = urlsplit(getattr(context, "base_url", None) or "https://qingflow.com")
+    origin = f"{parsed.scheme}://{parsed.netloc}" if parsed.scheme and parsed.netloc else "https://qingflow.com"
+    headers = {"User-Agent": DEFAULT_USER_AGENT, "Referer": f"{origin}/", "Origin": origin}
+    auth_type = storage_auth_type(source_url)
+    prefix = None
+    redirected = False
+    limits = {"max_bytes": max_bytes} if max_bytes is not None else {}
+    if requires_storage_cookie(source_url, auth_type):
+        prefix = environment_cookie_prefix(
+            backend, context, warnings=warnings, environment_prefix_cache=environment_prefix_cache,
+            operation=operation,
+        )
+        cookie_name = f"{prefix}token"
+        if hasattr(backend, "download_binary_with_cookie"):
+            response = backend.download_binary_with_cookie(context, source_url, cookie_name=cookie_name, headers=headers, **limits)
+            content = response.content
+            redirected = bool(getattr(response, "redirected", False))
+        else:
+            headers["Cookie"] = f"{cookie_name}={context.token}"
+            content = backend.download_binary(source_url, headers=headers, **limits)
+    else:
+        content = backend.download_binary(source_url, headers=headers, **limits)
+    return content, {
+        "download_strategy": requested_strategy, "storage_auth_type": auth_type,
+        "storage_cookie_prefix": prefix, "redirected": redirected,
+    }