@qingflow-tech/qingflow-cli 1.1.60 → 1.1.61-beta.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -3
- package/package.json +1 -1
- package/pyproject.toml +3 -1
- package/skills/qingflow-cli/SKILL.md +120 -8
- package/skills/qingflow-cli/reference/QINGFLOW_CLI_DATA_RETRIEVAL_WORKFLOW.md +1 -1
- package/skills/qingflow-cli/reference/QINGFLOW_CLI_EXPLORATION_REPORT.md +1 -1
- package/skills/qingflow-cli/reference/builder/20-build-complete-system.md +2 -2
- package/skills/qingflow-cli/reference/builder/30-schema-fields.md +9 -5
- package/skills/qingflow-cli/reference/builder/reference/app-delivery-sop.md +3 -3
- package/skills/qingflow-cli/reference/core/QINGFLOW_CLI_DATA_RETRIEVAL_WORKFLOW.md +4 -4
- package/skills/qingflow-cli/reference/core/QINGFLOW_CLI_EXPLORATION_REPORT.md +1 -1
- package/skills/qingflow-cli/reference/data/README.md +20 -0
- package/skills/qingflow-cli/reference/record/QINGFLOW_CLI_RECORD_SPECIALIZED_WORKFLOW.md +7 -1
- package/skills/qingflow-common/SKILL.md +5 -2
- package/src/qingflow_mcp/app_form_compat.py +3 -0
- package/src/qingflow_mcp/app_form_validator.py +69 -17
- package/src/qingflow_mcp/backend_client.py +33 -8
- package/src/qingflow_mcp/builder_facade/app_form.py +497 -100
- package/src/qingflow_mcp/builder_facade/schemas/app_form/manifest.json +1 -0
- package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/complete.json +64 -30
- package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/fields/address.json +4 -8
- package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/fields/amount.json +4 -8
- package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/fields/attachment.json +6 -10
- package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/fields/code_block.json +6 -10
- package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/fields/data_relation.json +11 -18
- package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/fields/date.json +4 -8
- package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/fields/datetime.json +4 -8
- package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/fields/department.json +6 -10
- package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/fields/email.json +4 -8
- package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/fields/link.json +4 -8
- package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/fields/long_text.json +4 -8
- package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/fields/member.json +6 -10
- package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/fields/multi_select.json +4 -8
- package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/fields/number.json +4 -8
- package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/fields/phone.json +4 -8
- package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/fields/q_linker.json +6 -10
- package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/fields/relation.json +46 -15
- package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/fields/rich_text.json +146 -0
- package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/fields/single_choice.json +4 -8
- package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/fields/single_select.json +4 -8
- package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/fields/subtable.json +61 -27
- package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/fields/text.json +4 -8
- package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/formula.json +43 -0
- package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/summary.json +7 -12
- package/src/qingflow_mcp/builder_facade/service.py +39 -1
- package/src/qingflow_mcp/cli/commands/data.py +13 -3
- package/src/qingflow_mcp/cli/formatters.py +7 -0
- package/src/qingflow_mcp/cli/main.py +34 -1
- package/src/qingflow_mcp/cloud_context.py +16 -1
- package/src/qingflow_mcp/public_surface.py +67 -8
- package/src/qingflow_mcp/response_trim.py +2 -1
- package/src/qingflow_mcp/server.py +13 -7
- package/src/qingflow_mcp/server_app_builder.py +9 -5
- package/src/qingflow_mcp/server_app_user.py +10 -6
- package/src/qingflow_mcp/server_cloud.py +43 -7
- package/src/qingflow_mcp/tool_catalog.py +387 -0
- package/src/qingflow_mcp/tool_catalog_i18n.py +185 -0
- package/src/qingflow_mcp/tool_policy.py +197 -0
- package/src/qingflow_mcp/tools/data_files.py +174 -0
- package/src/qingflow_mcp/tools/data_tools.py +42 -13
- package/src/qingflow_mcp/tools/record_tools.py +14 -72
- package/src/qingflow_mcp/tools/storage_download.py +89 -0
|
@@ -0,0 +1,197 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import os
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
from typing import Any, Literal, Mapping, Sequence
|
|
7
|
+
from urllib.parse import parse_qs
|
|
8
|
+
|
|
9
|
+
from mcp.server.fastmcp import FastMCP
|
|
10
|
+
from mcp.server.fastmcp.exceptions import ToolError
|
|
11
|
+
|
|
12
|
+
from .cloud_context import current_http_tool_policy, current_tool_policy_header
|
|
13
|
+
from .tool_catalog import TOOL_ID_PATTERN, require_registered_tool_id, tool_is_exempt
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
TOOL_POLICY_ENV = "QINGFLOW_TOOL_POLICY"
|
|
17
|
+
TOOL_POLICY_HEADER = "X-Qingflow-Tool-Policy"
|
|
18
|
+
TOOL_POLICY_INVALID = "TOOL_POLICY_INVALID"
|
|
19
|
+
TOOL_POLICY_DENIED = "TOOL_POLICY_DENIED"
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
@dataclass(frozen=True, slots=True)
|
|
23
|
+
class ToolPolicySnapshot:
|
|
24
|
+
mode: Literal["unrestricted", "restricted", "invalid"]
|
|
25
|
+
allowed_tools: tuple[str, ...] = ()
|
|
26
|
+
error: str | None = None
|
|
27
|
+
|
|
28
|
+
def violation_for(self, stable_id: str) -> "ToolPolicyViolation | None":
|
|
29
|
+
# 基础恢复能力在非法策略下也必须可用。
|
|
30
|
+
if tool_is_exempt(stable_id):
|
|
31
|
+
return None
|
|
32
|
+
if self.mode == "unrestricted":
|
|
33
|
+
return None
|
|
34
|
+
if self.mode == "invalid":
|
|
35
|
+
return ToolPolicyViolation(TOOL_POLICY_INVALID, stable_id, self.error or "工具策略无效")
|
|
36
|
+
if stable_id in self.allowed_tools:
|
|
37
|
+
return None
|
|
38
|
+
return ToolPolicyViolation(TOOL_POLICY_DENIED, stable_id, "当前策略未允许该工具")
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class ToolPolicyViolation(Exception):
|
|
42
|
+
def __init__(self, code: str, tool_id_value: str, message: str) -> None:
|
|
43
|
+
super().__init__(message)
|
|
44
|
+
self.code = code
|
|
45
|
+
self.tool_id = tool_id_value
|
|
46
|
+
self.message = message
|
|
47
|
+
|
|
48
|
+
def payload(self, *, camel_case: bool = False) -> dict[str, Any]:
|
|
49
|
+
if camel_case:
|
|
50
|
+
return {
|
|
51
|
+
"status": "failed",
|
|
52
|
+
"errorCode": self.code,
|
|
53
|
+
"message": self.message,
|
|
54
|
+
"toolId": self.tool_id,
|
|
55
|
+
}
|
|
56
|
+
return {
|
|
57
|
+
"status": "failed",
|
|
58
|
+
"error_code": self.code,
|
|
59
|
+
"message": self.message,
|
|
60
|
+
"toolId": self.tool_id,
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def parse_tool_policy(value: str | Mapping[str, Any] | None) -> ToolPolicySnapshot:
|
|
65
|
+
if value is None:
|
|
66
|
+
return ToolPolicySnapshot("unrestricted")
|
|
67
|
+
payload: Any = value
|
|
68
|
+
if isinstance(value, str):
|
|
69
|
+
if value == "":
|
|
70
|
+
return ToolPolicySnapshot("unrestricted")
|
|
71
|
+
try:
|
|
72
|
+
payload = json.loads(value)
|
|
73
|
+
except json.JSONDecodeError:
|
|
74
|
+
return ToolPolicySnapshot("invalid", error="工具策略不是合法 JSON")
|
|
75
|
+
if not isinstance(payload, Mapping):
|
|
76
|
+
return ToolPolicySnapshot("invalid", error="工具策略必须是对象")
|
|
77
|
+
if set(payload) != {"version", "allowed_tools"}:
|
|
78
|
+
return ToolPolicySnapshot("invalid", error="工具策略字段必须为 version 和 allowed_tools")
|
|
79
|
+
if type(payload.get("version")) is not int or payload.get("version") != 1: # noqa: E721
|
|
80
|
+
return ToolPolicySnapshot("invalid", error="工具策略 version 必须为 1")
|
|
81
|
+
allowed = payload.get("allowed_tools")
|
|
82
|
+
if not isinstance(allowed, list):
|
|
83
|
+
return ToolPolicySnapshot("invalid", error="allowed_tools 必须是数组")
|
|
84
|
+
normalized: set[str] = set()
|
|
85
|
+
for stable_id in allowed:
|
|
86
|
+
if not isinstance(stable_id, str) or TOOL_ID_PATTERN.fullmatch(stable_id) is None:
|
|
87
|
+
return ToolPolicySnapshot("invalid", error="allowed_tools 包含非法工具 ID")
|
|
88
|
+
normalized.add(stable_id)
|
|
89
|
+
return ToolPolicySnapshot("restricted", tuple(sorted(normalized)))
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def parse_http_tool_policy(header: str | None, query_string: bytes) -> ToolPolicySnapshot:
|
|
93
|
+
header_policy = parse_tool_policy(header)
|
|
94
|
+
# 保留空值并在 URL 解码后检查重复,tools= 与缺省不能合并为同一种状态。
|
|
95
|
+
values = parse_qs(query_string.decode("latin1"), keep_blank_values=True).get("tools", [])
|
|
96
|
+
if len(values) > 1:
|
|
97
|
+
return ToolPolicySnapshot("invalid", error="tools 查询参数不能重复")
|
|
98
|
+
url_policy = parse_tool_policy(
|
|
99
|
+
{"version": 1, "allowed_tools": values[0].split(",") if values[0] else []}
|
|
100
|
+
if values else None
|
|
101
|
+
)
|
|
102
|
+
if header_policy.mode == "invalid":
|
|
103
|
+
return header_policy
|
|
104
|
+
if url_policy.mode == "invalid":
|
|
105
|
+
return ToolPolicySnapshot("invalid", error=f"tools 查询参数无效: {url_policy.error}")
|
|
106
|
+
if header_policy.mode == "unrestricted":
|
|
107
|
+
return url_policy
|
|
108
|
+
if url_policy.mode == "unrestricted":
|
|
109
|
+
return header_policy
|
|
110
|
+
return ToolPolicySnapshot(
|
|
111
|
+
"restricted", tuple(sorted(set(header_policy.allowed_tools).intersection(url_policy.allowed_tools)))
|
|
112
|
+
)
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
class PolicyAwareFastMCP(FastMCP):
|
|
116
|
+
"""让 MCP 列表与调用在 FastMCP handler 边界共用策略快照。"""
|
|
117
|
+
|
|
118
|
+
def __init__(self, *args: Any, **kwargs: Any) -> None:
|
|
119
|
+
self._tool_policy_bindings: dict[str, str] | None = None
|
|
120
|
+
self._tool_policy_source: Literal["environment", "request"] = "environment"
|
|
121
|
+
self._tool_policy_environment_snapshot = ToolPolicySnapshot("unrestricted")
|
|
122
|
+
super().__init__(*args, **kwargs)
|
|
123
|
+
|
|
124
|
+
def configure_tool_policy(
|
|
125
|
+
self,
|
|
126
|
+
bindings: Mapping[str, str],
|
|
127
|
+
*,
|
|
128
|
+
source: Literal["environment", "request"],
|
|
129
|
+
) -> None:
|
|
130
|
+
registered = set(self._tool_manager._tools)
|
|
131
|
+
bound = set(bindings)
|
|
132
|
+
if registered != bound:
|
|
133
|
+
missing = sorted(registered - bound)
|
|
134
|
+
extra = sorted(bound - registered)
|
|
135
|
+
raise ValueError(f"工具策略绑定不完整: missing={missing}, extra={extra}")
|
|
136
|
+
for stable_id in bindings.values():
|
|
137
|
+
# 登记校验只依赖提供方稳定 ID,不依赖 Wingent-X 的产品分类。
|
|
138
|
+
require_registered_tool_id(stable_id)
|
|
139
|
+
self._tool_policy_bindings = dict(bindings)
|
|
140
|
+
self._tool_policy_source = source
|
|
141
|
+
if source == "environment":
|
|
142
|
+
self._tool_policy_environment_snapshot = parse_tool_policy(os.getenv(TOOL_POLICY_ENV))
|
|
143
|
+
|
|
144
|
+
@property
|
|
145
|
+
def tool_policy_bindings(self) -> dict[str, str]:
|
|
146
|
+
return dict(self._tool_policy_bindings or {})
|
|
147
|
+
|
|
148
|
+
async def list_tools(self): # type: ignore[no-untyped-def]
|
|
149
|
+
tools = await super().list_tools()
|
|
150
|
+
bindings = self._require_bindings()
|
|
151
|
+
snapshot = self._current_snapshot()
|
|
152
|
+
return [
|
|
153
|
+
tool
|
|
154
|
+
for tool in tools
|
|
155
|
+
if snapshot.violation_for(bindings[tool.name]) is None
|
|
156
|
+
]
|
|
157
|
+
|
|
158
|
+
async def call_tool(self, name: str, arguments: dict[str, Any]) -> Sequence[Any] | dict[str, Any]:
|
|
159
|
+
bindings = self._require_bindings()
|
|
160
|
+
stable_id = bindings.get(name)
|
|
161
|
+
if stable_id is None:
|
|
162
|
+
raise ToolError(json.dumps(
|
|
163
|
+
{
|
|
164
|
+
"status": "failed",
|
|
165
|
+
"errorCode": TOOL_POLICY_INVALID,
|
|
166
|
+
"message": "工具未登记稳定 ID",
|
|
167
|
+
},
|
|
168
|
+
ensure_ascii=False,
|
|
169
|
+
))
|
|
170
|
+
violation = self._current_snapshot().violation_for(stable_id)
|
|
171
|
+
if violation is not None:
|
|
172
|
+
raise ToolError(json.dumps(violation.payload(camel_case=True), ensure_ascii=False))
|
|
173
|
+
return await super().call_tool(name, arguments)
|
|
174
|
+
|
|
175
|
+
def _require_bindings(self) -> dict[str, str]:
|
|
176
|
+
if self._tool_policy_bindings is None:
|
|
177
|
+
raise RuntimeError("MCP 工具策略绑定尚未配置")
|
|
178
|
+
return self._tool_policy_bindings
|
|
179
|
+
|
|
180
|
+
def _current_snapshot(self) -> ToolPolicySnapshot:
|
|
181
|
+
if self._tool_policy_source == "request":
|
|
182
|
+
snapshot = current_http_tool_policy.get()
|
|
183
|
+
if snapshot is not None:
|
|
184
|
+
return snapshot
|
|
185
|
+
# 保留未经过 HTTP 入口的请求头上下文用法,绝不回退共享 env。
|
|
186
|
+
return parse_tool_policy(current_tool_policy_header.get())
|
|
187
|
+
return self._tool_policy_environment_snapshot
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
def configure_tool_policy(
|
|
191
|
+
server: PolicyAwareFastMCP,
|
|
192
|
+
bindings: Mapping[str, str],
|
|
193
|
+
*,
|
|
194
|
+
source: Literal["environment", "request"],
|
|
195
|
+
) -> PolicyAwareFastMCP:
|
|
196
|
+
server.configure_tool_policy(bindings, source=source)
|
|
197
|
+
return server
|
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import mimetypes
|
|
4
|
+
import os
|
|
5
|
+
import re
|
|
6
|
+
import time
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from typing import Any, Callable
|
|
9
|
+
from urllib.parse import unquote, urlsplit
|
|
10
|
+
from uuid import uuid4
|
|
11
|
+
|
|
12
|
+
from ..backend_client import BackendRequestContext
|
|
13
|
+
from ..config import get_mcp_home
|
|
14
|
+
from ..errors import QingflowApiError
|
|
15
|
+
from ..json_types import JSONObject
|
|
16
|
+
from .storage_download import download_storage_content, requires_storage_cookie, storage_auth_type
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
DATA_FILE_MAX_FILES = 50
|
|
20
|
+
DATA_FILE_MAX_BYTES = 50 * 1024 * 1024
|
|
21
|
+
DATA_FILE_MAX_TOTAL_BYTES = 200 * 1024 * 1024
|
|
22
|
+
DATA_FILE_TIME_BUDGET_SECONDS = 55.0
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def attachment_candidates(record: JSONObject, schema: JSONObject) -> list[JSONObject]:
|
|
26
|
+
candidates: list[JSONObject] = []
|
|
27
|
+
|
|
28
|
+
def walk(values: Any, fields: Any, prefix: str) -> None:
|
|
29
|
+
if not isinstance(values, dict) or not isinstance(fields, list):
|
|
30
|
+
return
|
|
31
|
+
for field in fields:
|
|
32
|
+
if not isinstance(field, dict) or field.get("field_id") is None:
|
|
33
|
+
continue
|
|
34
|
+
key = str(field["field_id"])
|
|
35
|
+
value = values.get(key)
|
|
36
|
+
path = f"{prefix}.{key}"
|
|
37
|
+
if field.get("data_type") == "table" and isinstance(value, list):
|
|
38
|
+
for index, row in enumerate(value):
|
|
39
|
+
walk(row, field.get("children"), f"{path}[{index}]")
|
|
40
|
+
elif field.get("data_type") == "attachment":
|
|
41
|
+
for index, item in enumerate(value if isinstance(value, list) else [value]):
|
|
42
|
+
url = item.get("value") or item.get("url") if isinstance(item, dict) else item
|
|
43
|
+
if not isinstance(url, str) or not url.strip():
|
|
44
|
+
continue
|
|
45
|
+
name = (item.get("other_info") or item.get("otherInfo") or item.get("name")) if isinstance(item, dict) else None
|
|
46
|
+
if not isinstance(name, str) or not name.strip():
|
|
47
|
+
try:
|
|
48
|
+
name = Path(unquote(urlsplit(url).path)).name
|
|
49
|
+
except ValueError:
|
|
50
|
+
name = None
|
|
51
|
+
name = name or f"attachment_{len(candidates) + 1}"
|
|
52
|
+
candidates.append({
|
|
53
|
+
"field_id": field["field_id"], "file_name": name,
|
|
54
|
+
"source_path": f"{path}[{index}]", "source_url": url.strip(),
|
|
55
|
+
})
|
|
56
|
+
|
|
57
|
+
walk(record.get("fields"), schema.get("fields"), "fields")
|
|
58
|
+
return candidates
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def _download_url(candidate: JSONObject) -> str:
|
|
62
|
+
url = str(candidate["source_url"])
|
|
63
|
+
parsed = urlsplit(url)
|
|
64
|
+
if parsed.scheme not in {"http", "https"} or not parsed.netloc or parsed.username or parsed.password:
|
|
65
|
+
raise QingflowApiError.config_error("附件没有可直接下载的 HTTP 地址")
|
|
66
|
+
return url
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _matching_attachment(
|
|
70
|
+
candidate: JSONObject, refreshed: list[JSONObject],
|
|
71
|
+
) -> JSONObject | None:
|
|
72
|
+
# shortcut: 沿用旧链路,同一字段内同名附件取首个;需要区分时再引入稳定文件 ID。
|
|
73
|
+
return next((item for item in refreshed if (
|
|
74
|
+
str(item["field_id"]) == str(candidate["field_id"])
|
|
75
|
+
and str(item["file_name"]).strip() == str(candidate["file_name"]).strip()
|
|
76
|
+
)), None)
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def download_data_attachments(
|
|
80
|
+
*, backend: Any, context: BackendRequestContext, record: JSONObject, schema: JSONObject,
|
|
81
|
+
refresh_record: Callable[[], JSONObject],
|
|
82
|
+
) -> JSONObject:
|
|
83
|
+
candidates = attachment_candidates(record, schema)
|
|
84
|
+
if not candidates:
|
|
85
|
+
return {"status": "none", "local_dir": None, "items": [], "warnings": []}
|
|
86
|
+
custom_home = os.environ.get("QINGFLOW_MCP_DATA_FILES_HOME")
|
|
87
|
+
local_dir = (Path(custom_home).expanduser() if custom_home else get_mcp_home() / "data-files") / uuid4().hex
|
|
88
|
+
items: list[JSONObject] = []
|
|
89
|
+
warnings: list[JSONObject] = []
|
|
90
|
+
prefix_cache: dict[str, str] = {}
|
|
91
|
+
fresh_candidates: list[JSONObject] | None = None
|
|
92
|
+
deadline = time.monotonic() + DATA_FILE_TIME_BUDGET_SECONDS
|
|
93
|
+
total_bytes = 0
|
|
94
|
+
|
|
95
|
+
for index, original in enumerate(candidates):
|
|
96
|
+
item: JSONObject = {**original, "local_path": None}
|
|
97
|
+
items.append(item)
|
|
98
|
+
if index >= DATA_FILE_MAX_FILES or time.monotonic() >= deadline:
|
|
99
|
+
item["access_status"] = "skipped_limit"
|
|
100
|
+
warnings.append({"code": "FILE_ASSET_LIMIT_EXCEEDED", "message": "附件数量或下载时间超过本次预算"})
|
|
101
|
+
continue
|
|
102
|
+
try:
|
|
103
|
+
if fresh_candidates is not None:
|
|
104
|
+
replacement = _matching_attachment(item, fresh_candidates)
|
|
105
|
+
if replacement is None:
|
|
106
|
+
raise QingflowApiError(category="http", message="刷新后附件不在当前可见范围内", http_status=403)
|
|
107
|
+
item["source_url"] = replacement["source_url"]
|
|
108
|
+
url = _download_url(item)
|
|
109
|
+
|
|
110
|
+
def download(source_url: str) -> tuple[bytes, JSONObject]:
|
|
111
|
+
return download_storage_content(
|
|
112
|
+
backend=backend, context=context, source_url=source_url, warnings=warnings,
|
|
113
|
+
environment_prefix_cache=prefix_cache, operation="data_get",
|
|
114
|
+
max_bytes=min(DATA_FILE_MAX_BYTES, DATA_FILE_MAX_TOTAL_BYTES - total_bytes),
|
|
115
|
+
requested_strategy="qingflow_storage_cookie_redirect" if requires_storage_cookie(
|
|
116
|
+
source_url, storage_auth_type(source_url)
|
|
117
|
+
) else "referer_acl",
|
|
118
|
+
)
|
|
119
|
+
|
|
120
|
+
try:
|
|
121
|
+
content, metadata = download(url)
|
|
122
|
+
except QingflowApiError as exc:
|
|
123
|
+
if exc.http_status not in {401, 403}:
|
|
124
|
+
raise
|
|
125
|
+
if fresh_candidates is None:
|
|
126
|
+
fresh_candidates = []
|
|
127
|
+
fresh = refresh_record()
|
|
128
|
+
if fresh.get("ok") is False:
|
|
129
|
+
raise QingflowApiError(
|
|
130
|
+
category="backend", message=str(fresh.get("message") or "刷新附件访问范围失败"),
|
|
131
|
+
backend_code=fresh.get("backend_code"),
|
|
132
|
+
) from exc
|
|
133
|
+
if not isinstance(fresh.get("record"), dict) or not isinstance(fresh.get("schema"), dict):
|
|
134
|
+
raise QingflowApiError(category="backend", message="刷新附件需要数据面返回 record 和 schema") from exc
|
|
135
|
+
fresh_candidates = attachment_candidates(fresh["record"], fresh["schema"])
|
|
136
|
+
replacement = _matching_attachment(item, fresh_candidates)
|
|
137
|
+
if replacement is None or replacement["source_url"] == url:
|
|
138
|
+
raise
|
|
139
|
+
url = _download_url(replacement)
|
|
140
|
+
content, metadata = download(url)
|
|
141
|
+
item["source_url"] = url
|
|
142
|
+
warnings.append({
|
|
143
|
+
"code": "FILE_ASSET_STORAGE_URL_REFRESHED", "source_path": item["source_path"],
|
|
144
|
+
"message": "已通过数据面重新读取同一记录并刷新附件链接",
|
|
145
|
+
})
|
|
146
|
+
item.update(metadata)
|
|
147
|
+
item["size_bytes"] = len(content)
|
|
148
|
+
if len(content) > DATA_FILE_MAX_BYTES or total_bytes + len(content) > DATA_FILE_MAX_TOTAL_BYTES:
|
|
149
|
+
item["access_status"] = "too_large"
|
|
150
|
+
warnings.append({"code": "FILE_ASSET_SIZE_LIMIT_EXCEEDED", "message": "附件超过单文件或总大小预算"})
|
|
151
|
+
continue
|
|
152
|
+
suffix = Path(str(item["file_name"])).suffix
|
|
153
|
+
suffix = suffix if re.fullmatch(r"\.[A-Za-z0-9]{1,10}", suffix) else ""
|
|
154
|
+
local_dir.mkdir(parents=True, exist_ok=True)
|
|
155
|
+
local_path = local_dir / f"file_{index + 1:04d}{suffix}"
|
|
156
|
+
local_path.write_bytes(content)
|
|
157
|
+
total_bytes += len(content)
|
|
158
|
+
item.update({
|
|
159
|
+
"local_path": str(local_path), "access_status": "downloaded",
|
|
160
|
+
"mime_type": mimetypes.guess_type(str(item["file_name"]))[0],
|
|
161
|
+
})
|
|
162
|
+
except (QingflowApiError, OSError, ValueError) as exc:
|
|
163
|
+
blocked = isinstance(exc, QingflowApiError) and exc.http_status in {401, 403}
|
|
164
|
+
too_large = isinstance(exc, QingflowApiError) and exc.category == "file_size_limit"
|
|
165
|
+
item["access_status"] = "too_large" if too_large else "blocked_private_url" if blocked else "download_failed"
|
|
166
|
+
warnings.append({
|
|
167
|
+
"code": "FILE_ASSET_SIZE_LIMIT_EXCEEDED" if too_large else "FILE_ASSET_DOWNLOAD_FAILED", "source_path": item["source_path"],
|
|
168
|
+
"message": exc.message if isinstance(exc, QingflowApiError) else str(exc),
|
|
169
|
+
"http_status": exc.http_status if isinstance(exc, QingflowApiError) else None,
|
|
170
|
+
})
|
|
171
|
+
return {
|
|
172
|
+
"status": "ok" if all(item["access_status"] == "downloaded" for item in items) else "partial",
|
|
173
|
+
"local_dir": str(local_dir) if local_dir.exists() else None, "items": items, "warnings": warnings,
|
|
174
|
+
}
|
|
@@ -2,9 +2,10 @@ from __future__ import annotations
|
|
|
2
2
|
|
|
3
3
|
from datetime import datetime
|
|
4
4
|
from pathlib import Path
|
|
5
|
-
from typing import Any, Literal
|
|
5
|
+
from typing import Any, Literal
|
|
6
6
|
|
|
7
7
|
from mcp.server.fastmcp import FastMCP
|
|
8
|
+
from typing_extensions import TypedDict
|
|
8
9
|
|
|
9
10
|
from ..backend_client import BackendRequestContext
|
|
10
11
|
from ..config import DEFAULT_PROFILE
|
|
@@ -12,6 +13,7 @@ from ..errors import QingflowApiError, raise_tool_error
|
|
|
12
13
|
from ..json_types import JSONObject, JSONValue
|
|
13
14
|
from .base import ToolBase
|
|
14
15
|
from .csv_export import CsvPageFailure, csv_cell, export_csv_pages
|
|
16
|
+
from .data_files import download_data_attachments
|
|
15
17
|
|
|
16
18
|
|
|
17
19
|
DATA_PAGE_SIZE = 1000
|
|
@@ -64,7 +66,11 @@ class DataTools(ToolBase):
|
|
|
64
66
|
all_pages: bool = False,
|
|
65
67
|
output: str = "json",
|
|
66
68
|
) -> JSONObject:
|
|
67
|
-
"""按字段 ID 查询应用数据;先读取 field_capabilities 再构造 where 和 sort。
|
|
69
|
+
"""按字段 ID 查询应用数据;先读取 field_capabilities 再构造 where 和 sort。
|
|
70
|
+
|
|
71
|
+
查询不下载附件;需要下载时对目标 app_key、record_id 调用
|
|
72
|
+
data_get(download_attachments=true),使用 file_assets.items[].local_path。
|
|
73
|
+
"""
|
|
68
74
|
return self.data_query(
|
|
69
75
|
profile=profile, app_key=app_key, columns=columns or [], where=where or [], sort=sort or [],
|
|
70
76
|
search=search, page=page, page_size=page_size, include_schema=include_schema,
|
|
@@ -73,9 +79,18 @@ class DataTools(ToolBase):
|
|
|
73
79
|
|
|
74
80
|
@mcp.tool()
|
|
75
81
|
def data_get(
|
|
76
|
-
profile: str = DEFAULT_PROFILE, app_key: str = "", record_id: str = "", include_schema: bool = True
|
|
82
|
+
profile: str = DEFAULT_PROFILE, app_key: str = "", record_id: str = "", include_schema: bool = True,
|
|
83
|
+
download_attachments: bool = False,
|
|
77
84
|
) -> JSONObject:
|
|
78
|
-
|
|
85
|
+
"""读取单条应用数据;需要下载附件或图片时设置 download_attachments=true,
|
|
86
|
+
保留 include_schema=true,并使用 file_assets.items[].local_path。
|
|
87
|
+
下载复用当前登录身份,通过数据面刷新链接;不需要 view_id,也不调用 record_get。
|
|
88
|
+
默认只读取字段;file_assets.status=partial 表示部分附件下载失败,请读取 warnings。
|
|
89
|
+
"""
|
|
90
|
+
return self.data_get(
|
|
91
|
+
profile=profile, app_key=app_key, record_id=record_id, include_schema=include_schema,
|
|
92
|
+
download_attachments=download_attachments,
|
|
93
|
+
)
|
|
79
94
|
|
|
80
95
|
@mcp.tool()
|
|
81
96
|
def data_insert(
|
|
@@ -128,19 +143,33 @@ class DataTools(ToolBase):
|
|
|
128
143
|
|
|
129
144
|
return self._run(profile, runner, tool_name="数据面查询")
|
|
130
145
|
|
|
131
|
-
def data_get(
|
|
146
|
+
def data_get(
|
|
147
|
+
self, *, profile: str, app_key: str, record_id: str | int, include_schema: bool = True,
|
|
148
|
+
download_attachments: bool = False,
|
|
149
|
+
) -> JSONObject:
|
|
132
150
|
self._require_app_key(app_key)
|
|
133
151
|
self._require_record_id(record_id)
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
152
|
+
if download_attachments and not include_schema:
|
|
153
|
+
raise_tool_error(QingflowApiError.config_error("下载附件需要 schema,请保留 include_schema=true(不要使用 --no-schema)"))
|
|
154
|
+
|
|
155
|
+
def runner(session_profile, context):
|
|
156
|
+
def read_record():
|
|
157
|
+
return self._request_data_plane(
|
|
138
158
|
context, "GET", f"{self._base_path(app_key)}/{record_id}",
|
|
139
159
|
params={"include_schema": include_schema},
|
|
140
|
-
)
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
160
|
+
)
|
|
161
|
+
|
|
162
|
+
result = read_record()
|
|
163
|
+
if download_attachments and result.get("ok") is not False:
|
|
164
|
+
if not isinstance(result.get("record"), dict) or not isinstance(result.get("schema"), dict):
|
|
165
|
+
raise QingflowApiError(category="backend", message="下载附件需要数据面返回 record 和 schema")
|
|
166
|
+
result["file_assets"] = download_data_attachments(
|
|
167
|
+
backend=self.backend, context=context, record=result.get("record", {}),
|
|
168
|
+
schema=result.get("schema", {}), refresh_record=read_record,
|
|
169
|
+
)
|
|
170
|
+
return self._attach_context(result, profile, session_profile.selected_ws_id)
|
|
171
|
+
|
|
172
|
+
return self._run(profile, runner, tool_name="数据面单条读取")
|
|
144
173
|
|
|
145
174
|
def data_insert(
|
|
146
175
|
self, *, profile: str, app_key: str, items: list[DataInsertItem], include_schema: bool = True
|
|
@@ -34,6 +34,10 @@ from ..list_type_labels import (
|
|
|
34
34
|
from .base import ToolBase
|
|
35
35
|
from .directory_tools import _directory_has_more, _directory_items
|
|
36
36
|
from .csv_export import CsvShardWriter
|
|
37
|
+
from .storage_download import (
|
|
38
|
+
download_storage_content, environment_cookie_prefix, is_storage_url,
|
|
39
|
+
requires_storage_cookie, storage_auth_type,
|
|
40
|
+
)
|
|
37
41
|
|
|
38
42
|
|
|
39
43
|
DEFAULT_QUERY_PAGE_SIZE = 50
|
|
@@ -17237,40 +17241,10 @@ def _record_detail_download_resolved_media_content(
|
|
|
17237
17241
|
environment_prefix_cache: dict[str, str],
|
|
17238
17242
|
requested_strategy: str,
|
|
17239
17243
|
) -> tuple[bytes, JSONObject]:
|
|
17240
|
-
|
|
17241
|
-
|
|
17242
|
-
|
|
17243
|
-
|
|
17244
|
-
context,
|
|
17245
|
-
warnings=warnings,
|
|
17246
|
-
environment_prefix_cache=environment_prefix_cache,
|
|
17247
|
-
)
|
|
17248
|
-
cookie_name = f"{storage_cookie_prefix}token"
|
|
17249
|
-
headers = _record_detail_media_download_headers(context)
|
|
17250
|
-
if hasattr(backend, "download_binary_with_cookie"):
|
|
17251
|
-
response = backend.download_binary_with_cookie(context, source_url, cookie_name=cookie_name, headers=headers)
|
|
17252
|
-
return response.content, {
|
|
17253
|
-
"download_strategy": requested_strategy,
|
|
17254
|
-
"storage_auth_type": storage_auth_type,
|
|
17255
|
-
"storage_cookie_prefix": storage_cookie_prefix,
|
|
17256
|
-
"redirected": bool(getattr(response, "redirected", False)),
|
|
17257
|
-
}
|
|
17258
|
-
cookie_headers = dict(headers)
|
|
17259
|
-
cookie_headers["Cookie"] = f"{cookie_name}={context.token}"
|
|
17260
|
-
content = backend.download_binary(source_url, headers=cookie_headers)
|
|
17261
|
-
return content, {
|
|
17262
|
-
"download_strategy": requested_strategy,
|
|
17263
|
-
"storage_auth_type": storage_auth_type,
|
|
17264
|
-
"storage_cookie_prefix": storage_cookie_prefix,
|
|
17265
|
-
"redirected": False,
|
|
17266
|
-
}
|
|
17267
|
-
content = backend.download_binary(source_url, headers=_record_detail_media_download_headers(context))
|
|
17268
|
-
return content, {
|
|
17269
|
-
"download_strategy": requested_strategy,
|
|
17270
|
-
"storage_auth_type": storage_auth_type,
|
|
17271
|
-
"storage_cookie_prefix": None,
|
|
17272
|
-
"redirected": False,
|
|
17273
|
-
}
|
|
17244
|
+
return download_storage_content(
|
|
17245
|
+
backend=backend, context=context, source_url=source_url, warnings=warnings,
|
|
17246
|
+
environment_prefix_cache=environment_prefix_cache, requested_strategy=requested_strategy,
|
|
17247
|
+
)
|
|
17274
17248
|
|
|
17275
17249
|
|
|
17276
17250
|
def _record_detail_media_download_strategy(source_url: str) -> str:
|
|
@@ -17282,28 +17256,15 @@ def _record_detail_media_download_strategy(source_url: str) -> str:
|
|
|
17282
17256
|
|
|
17283
17257
|
|
|
17284
17258
|
def _record_detail_should_use_storage_cookie(source_url: str, storage_auth_type: str | None) -> bool:
|
|
17285
|
-
|
|
17286
|
-
return False
|
|
17287
|
-
if not storage_auth_type:
|
|
17288
|
-
return True
|
|
17289
|
-
return storage_auth_type.upper() != "ANONYMOUS"
|
|
17259
|
+
return requires_storage_cookie(source_url, storage_auth_type)
|
|
17290
17260
|
|
|
17291
17261
|
|
|
17292
17262
|
def _record_detail_is_qingflow_storage_url(source_url: str) -> bool:
|
|
17293
|
-
|
|
17294
|
-
path = parsed.path.lower()
|
|
17295
|
-
if "/storage/file/" in path:
|
|
17296
|
-
return True
|
|
17297
|
-
query = parse_qs(parsed.query)
|
|
17298
|
-
return any(key.lower() == "qingflow-auth_type" for key in query)
|
|
17263
|
+
return is_storage_url(source_url)
|
|
17299
17264
|
|
|
17300
17265
|
|
|
17301
17266
|
def _record_detail_storage_auth_type(source_url: str) -> str | None:
|
|
17302
|
-
|
|
17303
|
-
for key, values in query.items():
|
|
17304
|
-
if key.lower() == "qingflow-auth_type" and values:
|
|
17305
|
-
return _normalize_optional_text(values[0])
|
|
17306
|
-
return None
|
|
17267
|
+
return storage_auth_type(source_url)
|
|
17307
17268
|
|
|
17308
17269
|
|
|
17309
17270
|
def _record_detail_is_download_file_url(source_url: str) -> bool:
|
|
@@ -17353,28 +17314,9 @@ def _record_detail_environment_prefix(
|
|
|
17353
17314
|
warnings: list[JSONObject],
|
|
17354
17315
|
environment_prefix_cache: dict[str, str],
|
|
17355
17316
|
) -> str:
|
|
17356
|
-
|
|
17357
|
-
|
|
17358
|
-
|
|
17359
|
-
payload = backend.request("GET", context, "/environment/properties")
|
|
17360
|
-
environment = payload.get("environmentProperty") if isinstance(payload, dict) else None
|
|
17361
|
-
if not isinstance(environment, dict):
|
|
17362
|
-
environment = payload.get("environment_property") if isinstance(payload, dict) else None
|
|
17363
|
-
prefix = _normalize_optional_text(environment.get("environmentPrefix")) if isinstance(environment, dict) else None
|
|
17364
|
-
if not prefix and isinstance(environment, dict):
|
|
17365
|
-
prefix = _normalize_optional_text(environment.get("environment_prefix"))
|
|
17366
|
-
except QingflowApiError as exc:
|
|
17367
|
-
prefix = None
|
|
17368
|
-
warnings.append(
|
|
17369
|
-
{
|
|
17370
|
-
"code": "MEDIA_ASSET_ENVIRONMENT_PREFIX_UNAVAILABLE",
|
|
17371
|
-
"message": f"record_get could not read /environment/properties; falling back to qf_ token cookie prefix: {exc.message}",
|
|
17372
|
-
"http_status": exc.http_status,
|
|
17373
|
-
}
|
|
17374
|
-
)
|
|
17375
|
-
prefix = prefix if prefix is not None else "qf_"
|
|
17376
|
-
environment_prefix_cache["value"] = prefix
|
|
17377
|
-
return prefix
|
|
17317
|
+
return environment_cookie_prefix(
|
|
17318
|
+
backend, context, warnings=warnings, environment_prefix_cache=environment_prefix_cache,
|
|
17319
|
+
)
|
|
17378
17320
|
|
|
17379
17321
|
|
|
17380
17322
|
def _record_detail_context_origin(context: BackendRequestContext) -> str:
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import Any
|
|
4
|
+
from urllib.parse import parse_qs, urlsplit
|
|
5
|
+
|
|
6
|
+
from ..backend_client import BackendRequestContext
|
|
7
|
+
from ..config import DEFAULT_USER_AGENT
|
|
8
|
+
from ..errors import QingflowApiError
|
|
9
|
+
from ..json_types import JSONObject
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def storage_auth_type(source_url: str) -> str | None:
|
|
13
|
+
for key, values in parse_qs(urlsplit(source_url).query).items():
|
|
14
|
+
if key.lower() == "qingflow-auth_type" and values:
|
|
15
|
+
return values[0].strip() or None
|
|
16
|
+
return None
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def is_storage_url(source_url: str) -> bool:
|
|
20
|
+
parsed = urlsplit(source_url)
|
|
21
|
+
return "/storage/file/" in parsed.path.lower() or any(
|
|
22
|
+
key.lower() == "qingflow-auth_type" for key in parse_qs(parsed.query)
|
|
23
|
+
)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def requires_storage_cookie(source_url: str, auth_type: str | None) -> bool:
|
|
27
|
+
return is_storage_url(source_url) and (not auth_type or auth_type.upper() != "ANONYMOUS")
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def environment_cookie_prefix(
|
|
31
|
+
backend: Any, context: BackendRequestContext, *, warnings: list[JSONObject],
|
|
32
|
+
environment_prefix_cache: dict[str, str], operation: str = "record_get",
|
|
33
|
+
) -> str:
|
|
34
|
+
if "value" in environment_prefix_cache:
|
|
35
|
+
return environment_prefix_cache["value"]
|
|
36
|
+
try:
|
|
37
|
+
payload = backend.request("GET", context, "/environment/properties")
|
|
38
|
+
environment = payload.get("environmentProperty") if isinstance(payload, dict) else None
|
|
39
|
+
if not isinstance(environment, dict):
|
|
40
|
+
environment = payload.get("environment_property") if isinstance(payload, dict) else None
|
|
41
|
+
prefix = environment.get("environmentPrefix") if isinstance(environment, dict) else None
|
|
42
|
+
if not prefix and isinstance(environment, dict):
|
|
43
|
+
prefix = environment.get("environment_prefix")
|
|
44
|
+
if prefix is not None:
|
|
45
|
+
prefix = str(prefix).strip() or None
|
|
46
|
+
except QingflowApiError as exc:
|
|
47
|
+
prefix = None
|
|
48
|
+
warnings.append({
|
|
49
|
+
"code": "MEDIA_ASSET_ENVIRONMENT_PREFIX_UNAVAILABLE",
|
|
50
|
+
"message": f"{operation} could not read /environment/properties; falling back to qf_ token cookie prefix: {exc.message}",
|
|
51
|
+
"http_status": exc.http_status,
|
|
52
|
+
})
|
|
53
|
+
prefix = prefix if prefix is not None else "qf_"
|
|
54
|
+
environment_prefix_cache["value"] = prefix
|
|
55
|
+
return prefix
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def download_storage_content(
|
|
59
|
+
*, backend: Any, context: BackendRequestContext, source_url: str,
|
|
60
|
+
warnings: list[JSONObject], environment_prefix_cache: dict[str, str],
|
|
61
|
+
requested_strategy: str, operation: str = "record_get",
|
|
62
|
+
max_bytes: int | None = None,
|
|
63
|
+
) -> tuple[bytes, JSONObject]:
|
|
64
|
+
parsed = urlsplit(getattr(context, "base_url", None) or "https://qingflow.com")
|
|
65
|
+
origin = f"{parsed.scheme}://{parsed.netloc}" if parsed.scheme and parsed.netloc else "https://qingflow.com"
|
|
66
|
+
headers = {"User-Agent": DEFAULT_USER_AGENT, "Referer": f"{origin}/", "Origin": origin}
|
|
67
|
+
auth_type = storage_auth_type(source_url)
|
|
68
|
+
prefix = None
|
|
69
|
+
redirected = False
|
|
70
|
+
limits = {"max_bytes": max_bytes} if max_bytes is not None else {}
|
|
71
|
+
if requires_storage_cookie(source_url, auth_type):
|
|
72
|
+
prefix = environment_cookie_prefix(
|
|
73
|
+
backend, context, warnings=warnings, environment_prefix_cache=environment_prefix_cache,
|
|
74
|
+
operation=operation,
|
|
75
|
+
)
|
|
76
|
+
cookie_name = f"{prefix}token"
|
|
77
|
+
if hasattr(backend, "download_binary_with_cookie"):
|
|
78
|
+
response = backend.download_binary_with_cookie(context, source_url, cookie_name=cookie_name, headers=headers, **limits)
|
|
79
|
+
content = response.content
|
|
80
|
+
redirected = bool(getattr(response, "redirected", False))
|
|
81
|
+
else:
|
|
82
|
+
headers["Cookie"] = f"{cookie_name}={context.token}"
|
|
83
|
+
content = backend.download_binary(source_url, headers=headers, **limits)
|
|
84
|
+
else:
|
|
85
|
+
content = backend.download_binary(source_url, headers=headers, **limits)
|
|
86
|
+
return content, {
|
|
87
|
+
"download_strategy": requested_strategy, "storage_auth_type": auth_type,
|
|
88
|
+
"storage_cookie_prefix": prefix, "redirected": redirected,
|
|
89
|
+
}
|