@qingflow-tech/qingflow-cli 1.1.48-beta.5 → 1.1.49
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -4
- package/npm/lib/runtime.mjs +13 -29
- package/package.json +1 -1
- package/pyproject.toml +1 -1
- package/skills/qingflow-cli/SKILL.md +410 -91
- package/skills/qingflow-cli/manifest.yaml +1 -1
- package/skills/qingflow-cli/reference/00-INDEX.md +4 -4
- package/skills/qingflow-cli/reference/QINGFLOW_CLI_ADMIN_CHEATSHEET.md +1 -1
- package/skills/qingflow-cli/reference/QINGFLOW_CLI_DATA_RETRIEVAL_WORKFLOW.md +137 -3
- package/skills/qingflow-cli/reference/QINGFLOW_CLI_EXPLORATION_REPORT.md +4 -6
- package/skills/qingflow-cli/reference/QINGFLOW_CLI_FIELD_DATA_TYPES.md +14 -14
- package/skills/qingflow-cli/reference/QINGFLOW_CLI_MEMBER_CHEATSHEET.md +19 -20
- package/skills/qingflow-cli/reference/QINGFLOW_CLI_ONE_SHOT_CHEATSHEET.md +14 -18
- package/skills/qingflow-cli/reference/QINGFLOW_CLI_RECORD_CREATE_WORKFLOW.md +12 -7
- package/skills/qingflow-cli/reference/QINGFLOW_CLI_RECORD_UPDATE_WORKFLOW.md +1 -2
- package/skills/qingflow-cli/reference/core/QINGFLOW_CLI_ADMIN_CHEATSHEET.md +1 -1
- package/skills/qingflow-cli/reference/core/QINGFLOW_CLI_DATA_RETRIEVAL_WORKFLOW.md +95 -76
- package/skills/qingflow-cli/reference/core/QINGFLOW_CLI_EXPLORATION_REPORT.md +4 -6
- package/skills/qingflow-cli/reference/core/QINGFLOW_CLI_FIELD_DATA_TYPES.md +13 -13
- package/skills/qingflow-cli/reference/core/QINGFLOW_CLI_MEMBER_CHEATSHEET.md +19 -20
- package/skills/qingflow-cli/reference/core/QINGFLOW_CLI_ONE_SHOT_CHEATSHEET.md +14 -18
- package/skills/qingflow-cli/reference/record/QINGFLOW_CLI_RECORD_UPDATE_WORKFLOW.md +1 -2
- package/skills/qingflow-cli/reference/record/analysis/README.md +113 -21
- package/skills/qingflow-cli/reference/record/analysis/analysis-gotchas.md +10 -10
- package/skills/qingflow-cli/reference/record/analysis/analysis-patterns.md +9 -9
- package/skills/qingflow-cli/reference/record/analysis/business-context.md +1 -1
- package/skills/qingflow-cli/reference/record/analysis/confidence-reporting.md +10 -10
- package/skills/qingflow-cli/reference/record/analysis/data-access-playbook.md +80 -52
- package/skills/qingflow-cli/reference/record/analysis/pandas-recipes.md +2 -2
- package/skills/qingflow-common/SKILL.md +17 -10
- package/src/qingflow_mcp/builder_facade/service.py +1 -119
- package/src/qingflow_mcp/cli/commands/__init__.py +1 -2
- package/src/qingflow_mcp/cli/commands/auth.py +1 -1
- package/src/qingflow_mcp/cli/commands/record.py +1 -1
- package/src/qingflow_mcp/cli/context.py +1 -4
- package/src/qingflow_mcp/cli/formatters.py +4 -159
- package/src/qingflow_mcp/cli/main.py +1 -4
- package/src/qingflow_mcp/public_surface.py +0 -6
- package/src/qingflow_mcp/response_trim.py +3 -3
- package/src/qingflow_mcp/server.py +0 -2
- package/src/qingflow_mcp/server_app_builder.py +0 -16
- package/src/qingflow_mcp/server_app_user.py +3 -10
- package/src/qingflow_mcp/tools/ai_builder_tools.py +39 -208
- package/src/qingflow_mcp/tools/app_tools.py +13 -52
- package/src/qingflow_mcp/tools/login_link_tools.py +6 -34
- package/src/qingflow_mcp/tools/record_tools.py +46 -11
- package/src/qingflow_mcp/tools/resource_read_tools.py +0 -13
- package/skills/qingflow-cli/reference/data/README.md +0 -32
- package/skills/qingflow-cli/reference/record/QINGFLOW_CLI_LEGACY_RECORD_RETRIEVAL_WORKFLOW.md +0 -50
- package/src/qingflow_mcp/cli/commands/data.py +0 -134
- package/src/qingflow_mcp/resource_uri.py +0 -241
- package/src/qingflow_mcp/tools/csv_export.py +0 -168
- package/src/qingflow_mcp/tools/data_tools.py +0 -271
|
@@ -1,241 +0,0 @@
|
|
|
1
|
-
from __future__ import annotations
|
|
2
|
-
|
|
3
|
-
import re
|
|
4
|
-
from collections.abc import Iterator
|
|
5
|
-
from contextlib import contextmanager
|
|
6
|
-
from contextvars import ContextVar
|
|
7
|
-
from dataclasses import dataclass
|
|
8
|
-
from typing import Any
|
|
9
|
-
from urllib.parse import quote, unquote, urlsplit, urlunsplit
|
|
10
|
-
|
|
11
|
-
from .config import DEFAULT_WEB_BASE_URL, get_web_base_url
|
|
12
|
-
from .errors import QingflowApiError
|
|
13
|
-
from .json_types import JSONObject
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
SYSTEM_VIEW_NAMES = frozenset({"all", "initiated", "todo", "done", "cc"})
|
|
17
|
-
_ENCODED_OCTET = re.compile(r"%[0-9A-Fa-f]{2}")
|
|
18
|
-
_CLI_RESOURCE_BASE_URL: ContextVar[str | None] = ContextVar(
|
|
19
|
-
"qingflow_cli_resource_base_url",
|
|
20
|
-
default=None,
|
|
21
|
-
)
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
@dataclass(frozen=True, slots=True)
|
|
25
|
-
class ResourcePath:
|
|
26
|
-
resource_type: str
|
|
27
|
-
ws_id: int
|
|
28
|
-
package_id: int | None = None
|
|
29
|
-
app_key: str | None = None
|
|
30
|
-
view_kind: str | None = None
|
|
31
|
-
view_key: str | None = None
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
@contextmanager
|
|
35
|
-
def cli_resource_base_url(value: str) -> Iterator[None]:
|
|
36
|
-
"""让一次 CLI 调用的资源 URI 使用 QINGFLOW_BASE_URL 对应的网页环境。"""
|
|
37
|
-
token = _CLI_RESOURCE_BASE_URL.set(value)
|
|
38
|
-
try:
|
|
39
|
-
yield
|
|
40
|
-
finally:
|
|
41
|
-
_CLI_RESOURCE_BASE_URL.reset(token)
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
def validate_resource_identity(value: object, field: str = "resource_id") -> str:
|
|
45
|
-
"""按资源 URI 单路径段合同校验并返回规范身份,不读取网页域名配置。"""
|
|
46
|
-
return _validate_identity(value, field)
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
def validate_positive_resource_id(value: object, field: str = "resource_id") -> int:
|
|
50
|
-
"""按资源 URI 数字路径段合同校验正整数,不接受布尔值或浮点截断。"""
|
|
51
|
-
return int(_positive_int_segment(value, field))
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
def build_resource_uri(
|
|
55
|
-
*,
|
|
56
|
-
ws_id: object,
|
|
57
|
-
resource_type: str,
|
|
58
|
-
package_id: object = None,
|
|
59
|
-
app_key: object = None,
|
|
60
|
-
view_kind: str | None = None,
|
|
61
|
-
view_key: object = None,
|
|
62
|
-
web_origin: str | None = None,
|
|
63
|
-
) -> str:
|
|
64
|
-
"""按固定公共合同构造资源 URI,不接受通用目标路由。"""
|
|
65
|
-
workspace = _positive_int_segment(ws_id, "ws_id")
|
|
66
|
-
if resource_type == "package":
|
|
67
|
-
path = f"/resources/ws/{workspace}/package/{_positive_int_segment(package_id, 'package_id')}"
|
|
68
|
-
elif resource_type == "app":
|
|
69
|
-
path = f"/resources/ws/{workspace}/app/{_identity_segment(app_key, 'app_key')}"
|
|
70
|
-
elif resource_type == "view":
|
|
71
|
-
app_segment = _identity_segment(app_key, "app_key")
|
|
72
|
-
if view_kind == "custom":
|
|
73
|
-
view_segment = _identity_segment(view_key, "view_key")
|
|
74
|
-
elif view_kind == "system":
|
|
75
|
-
name = str(view_key or "").strip()
|
|
76
|
-
if name not in SYSTEM_VIEW_NAMES:
|
|
77
|
-
raise ValueError("system view name is not supported")
|
|
78
|
-
view_segment = name
|
|
79
|
-
else:
|
|
80
|
-
raise ValueError("view_kind must be custom or system")
|
|
81
|
-
path = f"/resources/ws/{workspace}/app/{app_segment}/view/{view_kind}/{view_segment}"
|
|
82
|
-
else:
|
|
83
|
-
raise ValueError("resource_type is not supported")
|
|
84
|
-
return f"{resolve_resource_web_origin(web_origin)}{path}"
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
def parse_resource_path(value: str) -> ResourcePath:
|
|
88
|
-
"""严格解析资源 pathname,供免登录签发前校验工作区。"""
|
|
89
|
-
try:
|
|
90
|
-
parsed = urlsplit(value)
|
|
91
|
-
if parsed.scheme or parsed.netloc or parsed.query or parsed.fragment or parsed.path != value:
|
|
92
|
-
raise ValueError
|
|
93
|
-
parts = parsed.path.split("/")
|
|
94
|
-
if len(parts) < 6 or parts[:3] != ["", "resources", "ws"]:
|
|
95
|
-
raise ValueError
|
|
96
|
-
ws_id = int(_decode_segment(parts[3], "ws_id"))
|
|
97
|
-
if ws_id <= 0:
|
|
98
|
-
raise ValueError
|
|
99
|
-
if len(parts) == 6 and parts[4] == "package":
|
|
100
|
-
package_id = int(_decode_segment(parts[5], "package_id"))
|
|
101
|
-
if package_id <= 0:
|
|
102
|
-
raise ValueError
|
|
103
|
-
return ResourcePath("package", ws_id, package_id=package_id)
|
|
104
|
-
if len(parts) == 6 and parts[4] == "app":
|
|
105
|
-
return ResourcePath("app", ws_id, app_key=_decode_segment(parts[5], "app_key"))
|
|
106
|
-
if len(parts) == 9 and parts[4] == "app" and parts[6] == "view":
|
|
107
|
-
app_key = _decode_segment(parts[5], "app_key")
|
|
108
|
-
view_kind = parts[7]
|
|
109
|
-
view_key = _decode_segment(parts[8], "view_key")
|
|
110
|
-
if view_kind == "system" and view_key not in SYSTEM_VIEW_NAMES:
|
|
111
|
-
raise ValueError
|
|
112
|
-
if view_kind not in {"custom", "system"}:
|
|
113
|
-
raise ValueError
|
|
114
|
-
return ResourcePath("view", ws_id, app_key=app_key, view_kind=view_kind, view_key=view_key)
|
|
115
|
-
except (TypeError, ValueError, UnicodeError):
|
|
116
|
-
pass
|
|
117
|
-
raise ValueError("resource path is invalid")
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
def try_build_resource_uri(**kwargs: Any) -> tuple[str | None, JSONObject | None]:
|
|
121
|
-
try:
|
|
122
|
-
return build_resource_uri(**kwargs), None
|
|
123
|
-
except (QingflowApiError, ValueError, UnicodeError) as error:
|
|
124
|
-
return None, {
|
|
125
|
-
"code": "RESOURCE_URI_UNAVAILABLE",
|
|
126
|
-
"message": f"resource URI was omitted because its identity or web origin is invalid: {error}",
|
|
127
|
-
}
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
def _positive_int_segment(value: object, field: str) -> str:
|
|
131
|
-
if isinstance(value, bool):
|
|
132
|
-
raise ValueError(f"{field} must be a positive integer")
|
|
133
|
-
try:
|
|
134
|
-
parsed = int(value)
|
|
135
|
-
except (TypeError, ValueError):
|
|
136
|
-
raise ValueError(f"{field} must be a positive integer") from None
|
|
137
|
-
if parsed <= 0 or str(value).strip() != str(parsed):
|
|
138
|
-
raise ValueError(f"{field} must be a positive integer")
|
|
139
|
-
return str(parsed)
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
def _normalize_web_origin(value: str) -> str:
|
|
143
|
-
try:
|
|
144
|
-
parsed = urlsplit(value.strip())
|
|
145
|
-
if (
|
|
146
|
-
parsed.scheme not in {"http", "https"}
|
|
147
|
-
or not parsed.hostname
|
|
148
|
-
or parsed.username is not None
|
|
149
|
-
or parsed.password is not None
|
|
150
|
-
or parsed.path not in {"", "/"}
|
|
151
|
-
or parsed.query
|
|
152
|
-
or parsed.fragment
|
|
153
|
-
or any(character.isspace() or ord(character) < 32 for character in value)
|
|
154
|
-
):
|
|
155
|
-
raise ValueError
|
|
156
|
-
_ = parsed.port
|
|
157
|
-
except (AttributeError, ValueError):
|
|
158
|
-
raise ValueError("web_origin must be an absolute HTTP(S) origin") from None
|
|
159
|
-
return urlunsplit((parsed.scheme, parsed.netloc, "", "", ""))
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
def resolve_resource_web_origin(web_origin: str | None = None) -> str:
|
|
163
|
-
"""解析当前调用面的网页根地址;CLI 与 MCP 分别沿用各自的环境配置。"""
|
|
164
|
-
if web_origin is not None:
|
|
165
|
-
return _normalize_web_origin(web_origin)
|
|
166
|
-
cli_base_url = _CLI_RESOURCE_BASE_URL.get()
|
|
167
|
-
if cli_base_url is None:
|
|
168
|
-
return get_web_base_url()
|
|
169
|
-
return _web_origin_from_api_base_url(cli_base_url)
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
def _web_origin_from_api_base_url(value: str) -> str:
|
|
173
|
-
normalized = value.strip()
|
|
174
|
-
if not normalized:
|
|
175
|
-
return DEFAULT_WEB_BASE_URL
|
|
176
|
-
try:
|
|
177
|
-
parsed = urlsplit(normalized)
|
|
178
|
-
host = parsed.hostname or ""
|
|
179
|
-
valid_host = bool(host) and (":" in host or all(
|
|
180
|
-
re.fullmatch(r"[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?", label)
|
|
181
|
-
for label in host.rstrip(".").split(".")
|
|
182
|
-
))
|
|
183
|
-
if (
|
|
184
|
-
parsed.scheme not in {"http", "https"}
|
|
185
|
-
or not valid_host
|
|
186
|
-
or host.lower() in {"null", "undefined"}
|
|
187
|
-
or parsed.username is not None
|
|
188
|
-
or parsed.password is not None
|
|
189
|
-
or parsed.query
|
|
190
|
-
or parsed.fragment
|
|
191
|
-
or any(character.isspace() or ord(character) < 32 for character in normalized)
|
|
192
|
-
):
|
|
193
|
-
raise ValueError
|
|
194
|
-
_ = parsed.port
|
|
195
|
-
return _normalize_web_origin(urlunsplit((parsed.scheme, parsed.netloc, "", "", "")))
|
|
196
|
-
except ValueError:
|
|
197
|
-
raise QingflowApiError.config_error(
|
|
198
|
-
"QINGFLOW_BASE_URL 必须是有效的 HTTP(S) API 地址,不包含查询参数、片段或用户凭证"
|
|
199
|
-
) from None
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
def _identity_segment(value: object, field: str) -> str:
|
|
203
|
-
raw = _validate_identity(value, field)
|
|
204
|
-
return quote(raw, safe="")
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
def _decode_segment(value: str, field: str) -> str:
|
|
208
|
-
decoded = _validate_identity(unquote(value, errors="strict"), field)
|
|
209
|
-
if quote(decoded, safe="") != value:
|
|
210
|
-
raise ValueError(f"{field} must use canonical path encoding")
|
|
211
|
-
return decoded
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
def _validate_identity(value: object, field: str) -> str:
|
|
215
|
-
source = str(value or "")
|
|
216
|
-
if any(ord(character) < 32 or ord(character) == 127 for character in source):
|
|
217
|
-
raise ValueError(f"{field} must be one safe path segment")
|
|
218
|
-
raw = source.strip()
|
|
219
|
-
if not raw:
|
|
220
|
-
raise ValueError(f"{field} is required")
|
|
221
|
-
current = raw
|
|
222
|
-
for _ in range(8):
|
|
223
|
-
if (
|
|
224
|
-
current in {".", ".."}
|
|
225
|
-
or any(character in current for character in "/\\?#")
|
|
226
|
-
or any(ord(character) < 32 or ord(character) == 127 for character in current)
|
|
227
|
-
):
|
|
228
|
-
raise ValueError(f"{field} must be one safe path segment")
|
|
229
|
-
decoded = unquote(current, errors="strict")
|
|
230
|
-
if decoded == current:
|
|
231
|
-
return raw
|
|
232
|
-
current = decoded
|
|
233
|
-
if (
|
|
234
|
-
current in {".", ".."}
|
|
235
|
-
or any(character in current for character in "/\\?#")
|
|
236
|
-
or any(ord(character) < 32 or ord(character) == 127 for character in current)
|
|
237
|
-
):
|
|
238
|
-
raise ValueError(f"{field} must be one safe path segment")
|
|
239
|
-
if _ENCODED_OCTET.search(current):
|
|
240
|
-
raise ValueError(f"{field} has excessive encoding")
|
|
241
|
-
return raw
|
|
@@ -1,168 +0,0 @@
|
|
|
1
|
-
from __future__ import annotations
|
|
2
|
-
|
|
3
|
-
import csv
|
|
4
|
-
import json
|
|
5
|
-
import time
|
|
6
|
-
from pathlib import Path
|
|
7
|
-
from typing import Any, Callable
|
|
8
|
-
|
|
9
|
-
from ..json_types import JSONObject
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
PageFetcher = Callable[[int, int], JSONObject]
|
|
13
|
-
RowProjector = Callable[[JSONObject], list[str]]
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
class CsvPageFailure(RuntimeError):
|
|
17
|
-
"""后续分页返回结构化业务错误;已写文件保留为不完整结果。"""
|
|
18
|
-
|
|
19
|
-
def __init__(self, payload: JSONObject) -> None:
|
|
20
|
-
super().__init__(str(payload.get("message") or "分页请求失败"))
|
|
21
|
-
self.payload = payload
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
class CsvShardWriter:
|
|
25
|
-
"""视图无关的 UTF-8 BOM CSV 分片写入器。"""
|
|
26
|
-
|
|
27
|
-
def __init__(self, *, output_dir: Path, file_prefix: str, header: list[str], shard_rows: int) -> None:
|
|
28
|
-
self.output_dir = output_dir
|
|
29
|
-
self.file_prefix = file_prefix
|
|
30
|
-
self.header = header
|
|
31
|
-
self.shard_rows = shard_rows
|
|
32
|
-
self.files: list[JSONObject] = []
|
|
33
|
-
self.row_count = 0
|
|
34
|
-
self._part = 0
|
|
35
|
-
self._current_rows = 0
|
|
36
|
-
self._file: Any | None = None
|
|
37
|
-
self._writer: Any | None = None
|
|
38
|
-
output_dir.mkdir(parents=True, exist_ok=True)
|
|
39
|
-
|
|
40
|
-
def write(self, values: list[str]) -> None:
|
|
41
|
-
if self._current_rows >= self.shard_rows:
|
|
42
|
-
self.close_shard()
|
|
43
|
-
self._ensure_writer().writerow(values)
|
|
44
|
-
self._current_rows += 1
|
|
45
|
-
self.row_count += 1
|
|
46
|
-
|
|
47
|
-
def close_shard(self) -> None:
|
|
48
|
-
if self._file is None:
|
|
49
|
-
return
|
|
50
|
-
path = Path(self._file.name)
|
|
51
|
-
self._file.close()
|
|
52
|
-
self.files.append({"local_path": str(path), "row_count": self._current_rows, "part": len(self.files) + 1})
|
|
53
|
-
self._file = None
|
|
54
|
-
self._writer = None
|
|
55
|
-
self._current_rows = 0
|
|
56
|
-
|
|
57
|
-
def _ensure_writer(self) -> Any:
|
|
58
|
-
if self._writer is not None:
|
|
59
|
-
return self._writer
|
|
60
|
-
self._part += 1
|
|
61
|
-
path = self.output_dir / f"{self.file_prefix}-{self._part:04d}.csv"
|
|
62
|
-
self._file = path.open("w", newline="", encoding="utf-8-sig")
|
|
63
|
-
self._writer = csv.writer(self._file)
|
|
64
|
-
self._writer.writerow(self.header)
|
|
65
|
-
return self._writer
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
def export_csv_pages(
|
|
69
|
-
*,
|
|
70
|
-
fetch_page: PageFetcher,
|
|
71
|
-
header: list[str],
|
|
72
|
-
project_row: RowProjector,
|
|
73
|
-
output_dir: Path,
|
|
74
|
-
file_prefix: str,
|
|
75
|
-
page_size: int,
|
|
76
|
-
shard_rows: int,
|
|
77
|
-
time_budget_seconds: float,
|
|
78
|
-
) -> JSONObject:
|
|
79
|
-
"""执行与业务权限无关的分页、CSV 分片和时间预算控制。"""
|
|
80
|
-
output_dir.mkdir(parents=True, exist_ok=True)
|
|
81
|
-
deadline = time.monotonic() + time_budget_seconds
|
|
82
|
-
files: list[JSONObject] = []
|
|
83
|
-
row_count = 0
|
|
84
|
-
fetched_pages = 0
|
|
85
|
-
current_page = 1
|
|
86
|
-
file_handle: Any | None = None
|
|
87
|
-
writer: Any | None = None
|
|
88
|
-
current_rows = 0
|
|
89
|
-
part = 0
|
|
90
|
-
has_more = False
|
|
91
|
-
stopped_reason: str | None = None
|
|
92
|
-
page_error: JSONObject | None = None
|
|
93
|
-
|
|
94
|
-
def close_file() -> None:
|
|
95
|
-
nonlocal file_handle, writer, current_rows
|
|
96
|
-
if file_handle is None:
|
|
97
|
-
return
|
|
98
|
-
path = Path(file_handle.name)
|
|
99
|
-
file_handle.close()
|
|
100
|
-
files.append({"local_path": str(path), "row_count": current_rows, "part": len(files) + 1})
|
|
101
|
-
file_handle = None
|
|
102
|
-
writer = None
|
|
103
|
-
current_rows = 0
|
|
104
|
-
|
|
105
|
-
def get_writer() -> Any:
|
|
106
|
-
nonlocal file_handle, writer, part
|
|
107
|
-
if writer is not None:
|
|
108
|
-
return writer
|
|
109
|
-
part += 1
|
|
110
|
-
path = output_dir / f"{file_prefix}-{part:04d}.csv"
|
|
111
|
-
file_handle = path.open("w", newline="", encoding="utf-8-sig")
|
|
112
|
-
writer = csv.writer(file_handle)
|
|
113
|
-
writer.writerow(header)
|
|
114
|
-
return writer
|
|
115
|
-
|
|
116
|
-
try:
|
|
117
|
-
while True:
|
|
118
|
-
page = fetch_page(current_page, page_size)
|
|
119
|
-
records = page.get("records") if isinstance(page, dict) else None
|
|
120
|
-
container = records if isinstance(records, dict) else page
|
|
121
|
-
raw_items = container.get("items", container.get("list", [])) if isinstance(container, dict) else []
|
|
122
|
-
items = raw_items if isinstance(raw_items, list) else []
|
|
123
|
-
has_more = bool(container.get("has_more")) if isinstance(container, dict) else False
|
|
124
|
-
for item in items:
|
|
125
|
-
if not isinstance(item, dict):
|
|
126
|
-
continue
|
|
127
|
-
if current_rows >= shard_rows:
|
|
128
|
-
close_file()
|
|
129
|
-
get_writer().writerow(project_row(item))
|
|
130
|
-
current_rows += 1
|
|
131
|
-
row_count += 1
|
|
132
|
-
fetched_pages += 1
|
|
133
|
-
if not has_more:
|
|
134
|
-
break
|
|
135
|
-
if time.monotonic() >= deadline:
|
|
136
|
-
stopped_reason = "TIME_BUDGET_EXCEEDED"
|
|
137
|
-
break
|
|
138
|
-
current_page += 1
|
|
139
|
-
except CsvPageFailure as exception:
|
|
140
|
-
stopped_reason = "PAGE_FETCH_FAILED"
|
|
141
|
-
page_error = exception.payload
|
|
142
|
-
except Exception:
|
|
143
|
-
# 已落盘分片仍是有效的部分结果;调用方据此禁止形成完整性结论。
|
|
144
|
-
stopped_reason = "PAGE_FETCH_FAILED"
|
|
145
|
-
finally:
|
|
146
|
-
close_file()
|
|
147
|
-
|
|
148
|
-
complete = stopped_reason is None and not has_more
|
|
149
|
-
result: JSONObject = {
|
|
150
|
-
"files": files,
|
|
151
|
-
"row_count": row_count,
|
|
152
|
-
"complete": complete,
|
|
153
|
-
"truncated": not complete,
|
|
154
|
-
"safe_for_final_conclusion": complete,
|
|
155
|
-
"fetched_pages": fetched_pages,
|
|
156
|
-
"stopped_reason": stopped_reason,
|
|
157
|
-
}
|
|
158
|
-
if page_error is not None:
|
|
159
|
-
result["page_error"] = page_error
|
|
160
|
-
return result
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
def csv_cell(value: Any) -> str:
|
|
164
|
-
if value is None:
|
|
165
|
-
return ""
|
|
166
|
-
if isinstance(value, (dict, list)):
|
|
167
|
-
return json.dumps(value, ensure_ascii=False, separators=(",", ":"))
|
|
168
|
-
return str(value)
|
|
@@ -1,271 +0,0 @@
|
|
|
1
|
-
from __future__ import annotations
|
|
2
|
-
|
|
3
|
-
from datetime import datetime
|
|
4
|
-
from pathlib import Path
|
|
5
|
-
from typing import Any, Literal, TypedDict
|
|
6
|
-
|
|
7
|
-
from mcp.server.fastmcp import FastMCP
|
|
8
|
-
|
|
9
|
-
from ..backend_client import BackendRequestContext
|
|
10
|
-
from ..config import DEFAULT_PROFILE
|
|
11
|
-
from ..errors import QingflowApiError, raise_tool_error
|
|
12
|
-
from ..json_types import JSONObject, JSONValue
|
|
13
|
-
from .base import ToolBase
|
|
14
|
-
from .csv_export import CsvPageFailure, csv_cell, export_csv_pages
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
DATA_PAGE_SIZE = 1000
|
|
18
|
-
DATA_CSV_SHARD_ROWS = 20_000
|
|
19
|
-
DATA_UNBOUNDED_ROW_THRESHOLD = 50_000
|
|
20
|
-
DATA_CSV_TIME_BUDGET_SECONDS = 55.0
|
|
21
|
-
|
|
22
|
-
DataFilterOperator = Literal[
|
|
23
|
-
"eq", "ne", "contains", "not_contains", "gt", "gte", "lt", "lte", "in", "not_in"
|
|
24
|
-
]
|
|
25
|
-
DataSortDirection = Literal["asc", "desc"]
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
class DataWhereItem(TypedDict):
|
|
29
|
-
"""数据面筛选项;in/not_in 使用数组,其余操作符使用单值。"""
|
|
30
|
-
|
|
31
|
-
field_id: int
|
|
32
|
-
op: DataFilterOperator
|
|
33
|
-
value: JSONValue
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
class DataSortItem(TypedDict):
|
|
37
|
-
"""数据面排序项。"""
|
|
38
|
-
|
|
39
|
-
field_id: int
|
|
40
|
-
direction: DataSortDirection
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
class DataInsertItem(TypedDict):
|
|
44
|
-
"""数据面新增项,fields 的键为字符串形式 field_id。"""
|
|
45
|
-
|
|
46
|
-
fields: dict[str, JSONValue]
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
class DataTools(ToolBase):
|
|
50
|
-
"""视图无关的应用数据面工具。"""
|
|
51
|
-
|
|
52
|
-
def register(self, mcp: FastMCP) -> None:
|
|
53
|
-
@mcp.tool()
|
|
54
|
-
def data_query(
|
|
55
|
-
profile: str = DEFAULT_PROFILE,
|
|
56
|
-
app_key: str = "",
|
|
57
|
-
columns: list[int] | None = None,
|
|
58
|
-
where: list[DataWhereItem] | None = None,
|
|
59
|
-
sort: list[DataSortItem] | None = None,
|
|
60
|
-
search: str = "",
|
|
61
|
-
page: int = 1,
|
|
62
|
-
page_size: int = 100,
|
|
63
|
-
include_schema: bool = True,
|
|
64
|
-
all_pages: bool = False,
|
|
65
|
-
output: str = "json",
|
|
66
|
-
) -> JSONObject:
|
|
67
|
-
"""按字段 ID 查询应用数据;先读取 field_capabilities 再构造 where 和 sort。"""
|
|
68
|
-
return self.data_query(
|
|
69
|
-
profile=profile, app_key=app_key, columns=columns or [], where=where or [], sort=sort or [],
|
|
70
|
-
search=search, page=page, page_size=page_size, include_schema=include_schema,
|
|
71
|
-
all_pages=all_pages, output=output,
|
|
72
|
-
)
|
|
73
|
-
|
|
74
|
-
@mcp.tool()
|
|
75
|
-
def data_get(
|
|
76
|
-
profile: str = DEFAULT_PROFILE, app_key: str = "", record_id: str = "", include_schema: bool = True
|
|
77
|
-
) -> JSONObject:
|
|
78
|
-
return self.data_get(profile=profile, app_key=app_key, record_id=record_id, include_schema=include_schema)
|
|
79
|
-
|
|
80
|
-
@mcp.tool()
|
|
81
|
-
def data_insert(
|
|
82
|
-
profile: str = DEFAULT_PROFILE, app_key: str = "", items: list[DataInsertItem] | None = None,
|
|
83
|
-
include_schema: bool = True,
|
|
84
|
-
) -> JSONObject:
|
|
85
|
-
"""新增应用数据;每项形如 {"fields": {"字段ID": 值}}。"""
|
|
86
|
-
return self.data_insert(profile=profile, app_key=app_key, items=items or [], include_schema=include_schema)
|
|
87
|
-
|
|
88
|
-
@mcp.tool()
|
|
89
|
-
def data_update(
|
|
90
|
-
profile: str = DEFAULT_PROFILE, app_key: str = "", record_id: str = "",
|
|
91
|
-
fields: dict | None = None, include_schema: bool = True,
|
|
92
|
-
) -> JSONObject:
|
|
93
|
-
return self.data_update(
|
|
94
|
-
profile=profile, app_key=app_key, record_id=record_id, fields=fields or {},
|
|
95
|
-
include_schema=include_schema,
|
|
96
|
-
)
|
|
97
|
-
|
|
98
|
-
@mcp.tool()
|
|
99
|
-
def data_delete(
|
|
100
|
-
profile: str = DEFAULT_PROFILE, app_key: str = "", record_ids: list[str | int] | None = None,
|
|
101
|
-
) -> JSONObject:
|
|
102
|
-
return self.data_delete(profile=profile, app_key=app_key, record_ids=record_ids or [])
|
|
103
|
-
|
|
104
|
-
def data_query(
|
|
105
|
-
self, *, profile: str, app_key: str, columns: list[int], where: list[DataWhereItem],
|
|
106
|
-
sort: list[DataSortItem],
|
|
107
|
-
search: str = "", page: int = 1, page_size: int = 100, include_schema: bool = True,
|
|
108
|
-
all_pages: bool = False, output: str = "json",
|
|
109
|
-
) -> JSONObject:
|
|
110
|
-
self._require_app_key(app_key)
|
|
111
|
-
if all_pages and output != "csv":
|
|
112
|
-
raise_tool_error(QingflowApiError.config_error("all_pages 仅支持 output=csv"))
|
|
113
|
-
|
|
114
|
-
def runner(session_profile, context):
|
|
115
|
-
body: JSONObject = {
|
|
116
|
-
"page": page, "page_size": page_size, "columns": columns, "where": where, "sort": sort,
|
|
117
|
-
"search": search, "include_schema": include_schema,
|
|
118
|
-
}
|
|
119
|
-
if not all_pages:
|
|
120
|
-
return self._attach_context(
|
|
121
|
-
self._request_data_plane(context, "POST", self._base_path(app_key) + "/query", json_body=body),
|
|
122
|
-
profile, session_profile.selected_ws_id,
|
|
123
|
-
)
|
|
124
|
-
return self._export_all(
|
|
125
|
-
context=context, profile=profile, ws_id=session_profile.selected_ws_id, app_key=app_key,
|
|
126
|
-
body=body, has_scope=bool(where or search),
|
|
127
|
-
)
|
|
128
|
-
|
|
129
|
-
return self._run(profile, runner, tool_name="数据面查询")
|
|
130
|
-
|
|
131
|
-
def data_get(self, *, profile: str, app_key: str, record_id: str | int, include_schema: bool = True) -> JSONObject:
|
|
132
|
-
self._require_app_key(app_key)
|
|
133
|
-
self._require_record_id(record_id)
|
|
134
|
-
return self._run(
|
|
135
|
-
profile,
|
|
136
|
-
lambda session_profile, context: self._attach_context(
|
|
137
|
-
self._request_data_plane(
|
|
138
|
-
context, "GET", f"{self._base_path(app_key)}/{record_id}",
|
|
139
|
-
params={"include_schema": include_schema},
|
|
140
|
-
), profile, session_profile.selected_ws_id,
|
|
141
|
-
),
|
|
142
|
-
tool_name="数据面单条读取",
|
|
143
|
-
)
|
|
144
|
-
|
|
145
|
-
def data_insert(
|
|
146
|
-
self, *, profile: str, app_key: str, items: list[DataInsertItem], include_schema: bool = True
|
|
147
|
-
) -> JSONObject:
|
|
148
|
-
self._require_app_key(app_key)
|
|
149
|
-
return self._write(profile, app_key, "POST", self._base_path(app_key), {"items": items, "include_schema": include_schema}, "数据面新增")
|
|
150
|
-
|
|
151
|
-
def data_update(
|
|
152
|
-
self, *, profile: str, app_key: str, record_id: str | int, fields: dict, include_schema: bool = True
|
|
153
|
-
) -> JSONObject:
|
|
154
|
-
self._require_app_key(app_key)
|
|
155
|
-
self._require_record_id(record_id)
|
|
156
|
-
return self._write(
|
|
157
|
-
profile, app_key, "POST", f"{self._base_path(app_key)}/{record_id}/update",
|
|
158
|
-
{"fields": fields, "include_schema": include_schema}, "数据面更新",
|
|
159
|
-
)
|
|
160
|
-
|
|
161
|
-
def data_delete(self, *, profile: str, app_key: str, record_ids: list[str | int]) -> JSONObject:
|
|
162
|
-
self._require_app_key(app_key)
|
|
163
|
-
return self._write(profile, app_key, "POST", self._base_path(app_key) + "/delete", {"record_ids": record_ids}, "数据面删除")
|
|
164
|
-
|
|
165
|
-
def _write(self, profile: str, app_key: str, method: str, path: str, body: JSONObject, tool_name: str) -> JSONObject:
|
|
166
|
-
return self._run(
|
|
167
|
-
profile,
|
|
168
|
-
lambda session_profile, context: self._attach_context(
|
|
169
|
-
self._request_data_plane(context, method, path, json_body=body), profile, session_profile.selected_ws_id,
|
|
170
|
-
),
|
|
171
|
-
tool_name=tool_name,
|
|
172
|
-
)
|
|
173
|
-
|
|
174
|
-
def _request_data_plane(
|
|
175
|
-
self, context: BackendRequestContext, method: str, path: str, *, params: JSONObject | None = None,
|
|
176
|
-
json_body: JSONValue = None,
|
|
177
|
-
) -> JSONObject:
|
|
178
|
-
raw = self.backend.request(method, context, path, params=params, json_body=json_body, unwrap=False)
|
|
179
|
-
if not isinstance(raw, dict):
|
|
180
|
-
raise QingflowApiError(category="backend", message="数据面返回了非对象响应")
|
|
181
|
-
code = raw.get("code")
|
|
182
|
-
payload = raw.get("data")
|
|
183
|
-
result: JSONObject = dict(payload) if isinstance(payload, dict) else {}
|
|
184
|
-
success = code in (None, 0, 200, 20000) or str(code).lower() in {"success", "ok"}
|
|
185
|
-
result.setdefault("ok", success)
|
|
186
|
-
result.setdefault("status", "success" if success else "failed")
|
|
187
|
-
result.setdefault("hints", [])
|
|
188
|
-
if not success:
|
|
189
|
-
result.setdefault("message", str(raw.get("message") or raw.get("msg") or "数据面请求失败"))
|
|
190
|
-
result.setdefault("backend_code", code)
|
|
191
|
-
return result
|
|
192
|
-
|
|
193
|
-
def _export_all(
|
|
194
|
-
self, *, context: BackendRequestContext, profile: str, ws_id: int | None, app_key: str,
|
|
195
|
-
body: JSONObject, has_scope: bool,
|
|
196
|
-
) -> JSONObject:
|
|
197
|
-
probe_body = dict(body)
|
|
198
|
-
probe_body.update({"page": 1, "page_size": 1, "include_schema": True})
|
|
199
|
-
probe = self._request_data_plane(context, "POST", self._base_path(app_key) + "/query", json_body=probe_body)
|
|
200
|
-
if probe.get("ok") is False:
|
|
201
|
-
# 探测请求没有产生文件,保留后端结构化错误,不把失败误报为空数据成功。
|
|
202
|
-
return self._attach_context(probe, profile, ws_id)
|
|
203
|
-
records = probe.get("records") if isinstance(probe.get("records"), dict) else {}
|
|
204
|
-
total = records.get("total") if isinstance(records, dict) else None
|
|
205
|
-
if isinstance(total, int) and total > DATA_UNBOUNDED_ROW_THRESHOLD and not has_scope:
|
|
206
|
-
return self._attach_context({
|
|
207
|
-
"ok": True, "status": "needs_scope", "app_key": app_key, "format": "csv", "row_count": 0,
|
|
208
|
-
"complete": False, "truncated": True, "safe_for_final_conclusion": False, "files": [],
|
|
209
|
-
"hints": probe.get("hints", []),
|
|
210
|
-
"scope": {"reported_total": total, "row_threshold": DATA_UNBOUNDED_ROW_THRESHOLD},
|
|
211
|
-
}, profile, ws_id)
|
|
212
|
-
schema = probe.get("schema") if isinstance(probe.get("schema"), dict) else {}
|
|
213
|
-
fields = schema.get("fields") if isinstance(schema, dict) else None
|
|
214
|
-
field_list = fields if isinstance(fields, list) else []
|
|
215
|
-
business_fields = [field for field in field_list if isinstance(field, dict) and field.get("field_id") is not None]
|
|
216
|
-
columns = [(str(field.get("field_id")), str(field.get("title") or field.get("field_id"))) for field in business_fields]
|
|
217
|
-
header = ["record_id"] + [f"{title}__field_{field_id}" for field_id, title in columns]
|
|
218
|
-
run_dir = Path.home() / ".qingflow" / "data-exports" / datetime.now().strftime("%Y%m%d-%H%M%S-%f")
|
|
219
|
-
|
|
220
|
-
def fetch_page(page: int, page_size: int) -> JSONObject:
|
|
221
|
-
request_body = dict(body)
|
|
222
|
-
request_body.update({"page": page, "page_size": page_size, "include_schema": page == 1})
|
|
223
|
-
response = self._request_data_plane(
|
|
224
|
-
context, "POST", self._base_path(app_key) + "/query", json_body=request_body
|
|
225
|
-
)
|
|
226
|
-
if response.get("ok") is False:
|
|
227
|
-
raise CsvPageFailure(response)
|
|
228
|
-
return response
|
|
229
|
-
|
|
230
|
-
def project_row(record: JSONObject) -> list[str]:
|
|
231
|
-
values = record.get("fields") if isinstance(record.get("fields"), dict) else record
|
|
232
|
-
return [str(record.get("record_id") or "")] + [csv_cell(values.get(field_id)) for field_id, _ in columns]
|
|
233
|
-
|
|
234
|
-
exported = export_csv_pages(
|
|
235
|
-
fetch_page=fetch_page, header=header, project_row=project_row, output_dir=run_dir,
|
|
236
|
-
file_prefix="records", page_size=DATA_PAGE_SIZE, shard_rows=DATA_CSV_SHARD_ROWS,
|
|
237
|
-
time_budget_seconds=DATA_CSV_TIME_BUDGET_SECONDS,
|
|
238
|
-
)
|
|
239
|
-
page_error = exported.pop("page_error", None)
|
|
240
|
-
result: JSONObject = {
|
|
241
|
-
"ok": True, "status": "success" if exported["complete"] else "partial", "app_key": app_key,
|
|
242
|
-
"format": "csv", "local_dir": str(run_dir), "fields": business_fields, "hints": probe.get("hints", []),
|
|
243
|
-
**exported,
|
|
244
|
-
}
|
|
245
|
-
if isinstance(page_error, dict):
|
|
246
|
-
# 后续页失败时保留已落盘分片,同时把后端稳定错误字段原样带回。
|
|
247
|
-
result["ok"] = False
|
|
248
|
-
for key in ("error_code", "message", "backend_code", "write_executed", "retry_safe", "field_errors"):
|
|
249
|
-
if key in page_error:
|
|
250
|
-
result[key] = page_error[key]
|
|
251
|
-
if "hints" in page_error:
|
|
252
|
-
result["hints"] = page_error["hints"]
|
|
253
|
-
return self._attach_context(result, profile, ws_id)
|
|
254
|
-
|
|
255
|
-
@staticmethod
|
|
256
|
-
def _base_path(app_key: str) -> str:
|
|
257
|
-
return f"/app/{app_key}/data/v1/records"
|
|
258
|
-
|
|
259
|
-
@staticmethod
|
|
260
|
-
def _attach_context(result: JSONObject, profile: str, ws_id: int | None) -> JSONObject:
|
|
261
|
-
return {"profile": profile, "ws_id": ws_id, **result}
|
|
262
|
-
|
|
263
|
-
@staticmethod
|
|
264
|
-
def _require_app_key(app_key: str) -> None:
|
|
265
|
-
if not str(app_key or "").strip():
|
|
266
|
-
raise_tool_error(QingflowApiError.config_error("app_key is required"))
|
|
267
|
-
|
|
268
|
-
@staticmethod
|
|
269
|
-
def _require_record_id(record_id: str | int) -> None:
|
|
270
|
-
if not str(record_id or "").strip():
|
|
271
|
-
raise_tool_error(QingflowApiError.config_error("record_id is required"))
|