@qingflow-tech/qingflow-cli 1.1.59 → 1.1.61-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -4
- package/npm/lib/runtime.mjs +29 -13
- package/package.json +1 -1
- package/pyproject.toml +3 -1
- package/skills/qingflow-cli/SKILL.md +92 -318
- package/skills/qingflow-cli/manifest.yaml +1 -1
- package/skills/qingflow-cli/reference/00-INDEX.md +4 -4
- package/skills/qingflow-cli/reference/QINGFLOW_CLI_ADMIN_CHEATSHEET.md +1 -1
- package/skills/qingflow-cli/reference/QINGFLOW_CLI_DATA_RETRIEVAL_WORKFLOW.md +3 -137
- package/skills/qingflow-cli/reference/QINGFLOW_CLI_EXPLORATION_REPORT.md +6 -4
- package/skills/qingflow-cli/reference/QINGFLOW_CLI_FIELD_DATA_TYPES.md +14 -14
- package/skills/qingflow-cli/reference/QINGFLOW_CLI_MEMBER_CHEATSHEET.md +20 -19
- package/skills/qingflow-cli/reference/QINGFLOW_CLI_ONE_SHOT_CHEATSHEET.md +18 -14
- package/skills/qingflow-cli/reference/QINGFLOW_CLI_RECORD_CREATE_WORKFLOW.md +7 -12
- package/skills/qingflow-cli/reference/QINGFLOW_CLI_RECORD_UPDATE_WORKFLOW.md +2 -1
- package/skills/qingflow-cli/reference/builder/20-build-complete-system.md +2 -2
- package/skills/qingflow-cli/reference/builder/30-schema-fields.md +9 -5
- package/skills/qingflow-cli/reference/builder/reference/app-delivery-sop.md +3 -3
- package/skills/qingflow-cli/reference/core/QINGFLOW_CLI_ADMIN_CHEATSHEET.md +1 -1
- package/skills/qingflow-cli/reference/core/QINGFLOW_CLI_DATA_RETRIEVAL_WORKFLOW.md +76 -95
- package/skills/qingflow-cli/reference/core/QINGFLOW_CLI_EXPLORATION_REPORT.md +6 -4
- package/skills/qingflow-cli/reference/core/QINGFLOW_CLI_FIELD_DATA_TYPES.md +13 -13
- package/skills/qingflow-cli/reference/core/QINGFLOW_CLI_MEMBER_CHEATSHEET.md +20 -19
- package/skills/qingflow-cli/reference/core/QINGFLOW_CLI_ONE_SHOT_CHEATSHEET.md +18 -14
- package/skills/qingflow-cli/reference/data/README.md +52 -0
- package/skills/qingflow-cli/reference/record/QINGFLOW_CLI_RECORD_SPECIALIZED_WORKFLOW.md +56 -0
- package/skills/qingflow-cli/reference/record/QINGFLOW_CLI_RECORD_UPDATE_WORKFLOW.md +2 -1
- package/skills/qingflow-cli/reference/record/analysis/README.md +21 -113
- package/skills/qingflow-cli/reference/record/analysis/analysis-gotchas.md +10 -10
- package/skills/qingflow-cli/reference/record/analysis/analysis-patterns.md +9 -9
- package/skills/qingflow-cli/reference/record/analysis/business-context.md +1 -1
- package/skills/qingflow-cli/reference/record/analysis/confidence-reporting.md +10 -10
- package/skills/qingflow-cli/reference/record/analysis/data-access-playbook.md +52 -80
- package/skills/qingflow-cli/reference/record/analysis/pandas-recipes.md +2 -2
- package/skills/qingflow-common/SKILL.md +10 -4
- package/src/qingflow_mcp/app_form_compat.py +3 -0
- package/src/qingflow_mcp/app_form_validator.py +69 -17
- package/src/qingflow_mcp/backend_client.py +33 -8
- package/src/qingflow_mcp/builder_facade/app_form.py +497 -100
- package/src/qingflow_mcp/builder_facade/schemas/app_form/manifest.json +1 -0
- package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/complete.json +64 -30
- package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/fields/address.json +4 -8
- package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/fields/amount.json +4 -8
- package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/fields/attachment.json +6 -10
- package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/fields/code_block.json +6 -10
- package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/fields/data_relation.json +11 -18
- package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/fields/date.json +4 -8
- package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/fields/datetime.json +4 -8
- package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/fields/department.json +6 -10
- package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/fields/email.json +4 -8
- package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/fields/link.json +4 -8
- package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/fields/long_text.json +4 -8
- package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/fields/member.json +6 -10
- package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/fields/multi_select.json +4 -8
- package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/fields/number.json +4 -8
- package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/fields/phone.json +4 -8
- package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/fields/q_linker.json +6 -10
- package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/fields/relation.json +46 -15
- package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/fields/rich_text.json +146 -0
- package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/fields/single_choice.json +4 -8
- package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/fields/single_select.json +4 -8
- package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/fields/subtable.json +61 -27
- package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/fields/text.json +4 -8
- package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/formula.json +43 -0
- package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/summary.json +7 -12
- package/src/qingflow_mcp/builder_facade/service.py +39 -1
- package/src/qingflow_mcp/cli/commands/__init__.py +2 -1
- package/src/qingflow_mcp/cli/commands/data.py +144 -0
- package/src/qingflow_mcp/cli/commands/record.py +1 -1
- package/src/qingflow_mcp/cli/context.py +3 -0
- package/src/qingflow_mcp/cli/formatters.py +104 -1
- package/src/qingflow_mcp/public_surface.py +5 -0
- package/src/qingflow_mcp/response_trim.py +2 -1
- package/src/qingflow_mcp/server.py +3 -1
- package/src/qingflow_mcp/server_app_user.py +5 -1
- package/src/qingflow_mcp/tools/csv_export.py +168 -0
- package/src/qingflow_mcp/tools/data_files.py +174 -0
- package/src/qingflow_mcp/tools/data_tools.py +313 -0
- package/src/qingflow_mcp/tools/record_tools.py +25 -118
- package/src/qingflow_mcp/tools/storage_download.py +89 -0
|
@@ -0,0 +1,313 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from datetime import datetime
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
from typing import Any, Literal
|
|
6
|
+
|
|
7
|
+
from mcp.server.fastmcp import FastMCP
|
|
8
|
+
from typing_extensions import TypedDict
|
|
9
|
+
|
|
10
|
+
from ..backend_client import BackendRequestContext
|
|
11
|
+
from ..config import DEFAULT_PROFILE
|
|
12
|
+
from ..errors import QingflowApiError, raise_tool_error
|
|
13
|
+
from ..json_types import JSONObject, JSONValue
|
|
14
|
+
from .base import ToolBase
|
|
15
|
+
from .csv_export import CsvPageFailure, csv_cell, export_csv_pages
|
|
16
|
+
from .data_files import download_data_attachments
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
DATA_PAGE_SIZE = 1000
|
|
20
|
+
DATA_CSV_SHARD_ROWS = 20_000
|
|
21
|
+
DATA_UNBOUNDED_ROW_THRESHOLD = 50_000
|
|
22
|
+
DATA_CSV_TIME_BUDGET_SECONDS = 55.0
|
|
23
|
+
|
|
24
|
+
DataFilterOperator = Literal[
|
|
25
|
+
"eq", "ne", "contains", "not_contains", "gt", "gte", "lt", "lte", "in", "not_in"
|
|
26
|
+
]
|
|
27
|
+
DataSortDirection = Literal["asc", "desc"]
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class DataWhereItem(TypedDict):
|
|
31
|
+
"""数据面筛选项;in/not_in 使用数组,其余操作符使用单值。"""
|
|
32
|
+
|
|
33
|
+
field_id: int
|
|
34
|
+
op: DataFilterOperator
|
|
35
|
+
value: JSONValue
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class DataSortItem(TypedDict):
|
|
39
|
+
"""数据面排序项。"""
|
|
40
|
+
|
|
41
|
+
field_id: int
|
|
42
|
+
direction: DataSortDirection
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
class DataInsertItem(TypedDict):
|
|
46
|
+
"""数据面新增项,fields 的键为字符串形式 field_id。"""
|
|
47
|
+
|
|
48
|
+
fields: dict[str, JSONValue]
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
class DataTools(ToolBase):
|
|
52
|
+
"""视图无关的应用数据面工具。"""
|
|
53
|
+
|
|
54
|
+
def register(self, mcp: FastMCP) -> None:
|
|
55
|
+
@mcp.tool()
|
|
56
|
+
def data_query(
|
|
57
|
+
profile: str = DEFAULT_PROFILE,
|
|
58
|
+
app_key: str = "",
|
|
59
|
+
columns: list[int] | None = None,
|
|
60
|
+
where: list[DataWhereItem] | None = None,
|
|
61
|
+
sort: list[DataSortItem] | None = None,
|
|
62
|
+
search: str = "",
|
|
63
|
+
page: int = 1,
|
|
64
|
+
page_size: int = 100,
|
|
65
|
+
include_schema: bool = True,
|
|
66
|
+
all_pages: bool = False,
|
|
67
|
+
output: str = "json",
|
|
68
|
+
) -> JSONObject:
|
|
69
|
+
"""按字段 ID 查询应用数据;先读取 field_capabilities 再构造 where 和 sort。
|
|
70
|
+
|
|
71
|
+
查询不下载附件;需要下载时对目标 app_key、record_id 调用
|
|
72
|
+
data_get(download_attachments=true),使用 file_assets.items[].local_path。
|
|
73
|
+
"""
|
|
74
|
+
return self.data_query(
|
|
75
|
+
profile=profile, app_key=app_key, columns=columns or [], where=where or [], sort=sort or [],
|
|
76
|
+
search=search, page=page, page_size=page_size, include_schema=include_schema,
|
|
77
|
+
all_pages=all_pages, output=output,
|
|
78
|
+
)
|
|
79
|
+
|
|
80
|
+
@mcp.tool()
|
|
81
|
+
def data_get(
|
|
82
|
+
profile: str = DEFAULT_PROFILE, app_key: str = "", record_id: str = "", include_schema: bool = True,
|
|
83
|
+
download_attachments: bool = False,
|
|
84
|
+
) -> JSONObject:
|
|
85
|
+
"""读取单条应用数据;需要下载附件或图片时设置 download_attachments=true,
|
|
86
|
+
保留 include_schema=true,并使用 file_assets.items[].local_path。
|
|
87
|
+
下载复用当前登录身份,通过数据面刷新链接;不需要 view_id,也不调用 record_get。
|
|
88
|
+
默认只读取字段;file_assets.status=partial 表示部分附件下载失败,请读取 warnings。
|
|
89
|
+
"""
|
|
90
|
+
return self.data_get(
|
|
91
|
+
profile=profile, app_key=app_key, record_id=record_id, include_schema=include_schema,
|
|
92
|
+
download_attachments=download_attachments,
|
|
93
|
+
)
|
|
94
|
+
|
|
95
|
+
@mcp.tool()
|
|
96
|
+
def data_insert(
|
|
97
|
+
profile: str = DEFAULT_PROFILE, app_key: str = "", items: list[DataInsertItem] | None = None,
|
|
98
|
+
include_schema: bool = True,
|
|
99
|
+
) -> JSONObject:
|
|
100
|
+
"""新增应用数据;每项形如 {"fields": {"字段ID": 值}}。"""
|
|
101
|
+
return self.data_insert(profile=profile, app_key=app_key, items=items or [], include_schema=include_schema)
|
|
102
|
+
|
|
103
|
+
@mcp.tool()
|
|
104
|
+
def data_update(
|
|
105
|
+
profile: str = DEFAULT_PROFILE, app_key: str = "", record_id: str = "",
|
|
106
|
+
fields: dict | None = None, include_schema: bool = True,
|
|
107
|
+
) -> JSONObject:
|
|
108
|
+
return self.data_update(
|
|
109
|
+
profile=profile, app_key=app_key, record_id=record_id, fields=fields or {},
|
|
110
|
+
include_schema=include_schema,
|
|
111
|
+
)
|
|
112
|
+
|
|
113
|
+
@mcp.tool()
|
|
114
|
+
def data_delete(
|
|
115
|
+
profile: str = DEFAULT_PROFILE, app_key: str = "", record_ids: list[str | int] | None = None,
|
|
116
|
+
) -> JSONObject:
|
|
117
|
+
return self.data_delete(profile=profile, app_key=app_key, record_ids=record_ids or [])
|
|
118
|
+
|
|
119
|
+
def data_query(
|
|
120
|
+
self, *, profile: str, app_key: str, columns: list[int], where: list[DataWhereItem],
|
|
121
|
+
sort: list[DataSortItem],
|
|
122
|
+
search: str = "", page: int = 1, page_size: int = 100, include_schema: bool = True,
|
|
123
|
+
all_pages: bool = False, output: str = "json",
|
|
124
|
+
) -> JSONObject:
|
|
125
|
+
self._require_app_key(app_key)
|
|
126
|
+
if all_pages and output != "csv":
|
|
127
|
+
raise_tool_error(QingflowApiError.config_error("all_pages 仅支持 output=csv"))
|
|
128
|
+
|
|
129
|
+
def runner(session_profile, context):
|
|
130
|
+
body: JSONObject = {
|
|
131
|
+
"page": page, "page_size": page_size, "columns": columns, "where": where, "sort": sort,
|
|
132
|
+
"search": search, "include_schema": include_schema,
|
|
133
|
+
}
|
|
134
|
+
if not all_pages:
|
|
135
|
+
return self._attach_context(
|
|
136
|
+
self._request_data_plane(context, "POST", self._base_path(app_key) + "/query", json_body=body),
|
|
137
|
+
profile, session_profile.selected_ws_id,
|
|
138
|
+
)
|
|
139
|
+
return self._export_all(
|
|
140
|
+
context=context, profile=profile, ws_id=session_profile.selected_ws_id, app_key=app_key,
|
|
141
|
+
body=body, has_scope=bool(where or search),
|
|
142
|
+
)
|
|
143
|
+
|
|
144
|
+
return self._run(profile, runner, tool_name="数据面查询")
|
|
145
|
+
|
|
146
|
+
def data_get(
|
|
147
|
+
self, *, profile: str, app_key: str, record_id: str | int, include_schema: bool = True,
|
|
148
|
+
download_attachments: bool = False,
|
|
149
|
+
) -> JSONObject:
|
|
150
|
+
self._require_app_key(app_key)
|
|
151
|
+
self._require_record_id(record_id)
|
|
152
|
+
if download_attachments and not include_schema:
|
|
153
|
+
raise_tool_error(QingflowApiError.config_error("下载附件需要 schema,请保留 include_schema=true(不要使用 --no-schema)"))
|
|
154
|
+
|
|
155
|
+
def runner(session_profile, context):
|
|
156
|
+
def read_record():
|
|
157
|
+
return self._request_data_plane(
|
|
158
|
+
context, "GET", f"{self._base_path(app_key)}/{record_id}",
|
|
159
|
+
params={"include_schema": include_schema},
|
|
160
|
+
)
|
|
161
|
+
|
|
162
|
+
result = read_record()
|
|
163
|
+
if download_attachments and result.get("ok") is not False:
|
|
164
|
+
if not isinstance(result.get("record"), dict) or not isinstance(result.get("schema"), dict):
|
|
165
|
+
raise QingflowApiError(category="backend", message="下载附件需要数据面返回 record 和 schema")
|
|
166
|
+
result["file_assets"] = download_data_attachments(
|
|
167
|
+
backend=self.backend, context=context, record=result.get("record", {}),
|
|
168
|
+
schema=result.get("schema", {}), refresh_record=read_record,
|
|
169
|
+
)
|
|
170
|
+
return self._attach_context(result, profile, session_profile.selected_ws_id)
|
|
171
|
+
|
|
172
|
+
return self._run(profile, runner, tool_name="数据面单条读取")
|
|
173
|
+
|
|
174
|
+
def data_insert(
|
|
175
|
+
self, *, profile: str, app_key: str, items: list[DataInsertItem], include_schema: bool = True
|
|
176
|
+
) -> JSONObject:
|
|
177
|
+
self._require_app_key(app_key)
|
|
178
|
+
return self._write(profile, app_key, "POST", self._base_path(app_key), {"items": items, "include_schema": include_schema}, "数据面新增")
|
|
179
|
+
|
|
180
|
+
def data_update(
|
|
181
|
+
self, *, profile: str, app_key: str, record_id: str | int, fields: dict, include_schema: bool = True
|
|
182
|
+
) -> JSONObject:
|
|
183
|
+
self._require_app_key(app_key)
|
|
184
|
+
self._require_record_id(record_id)
|
|
185
|
+
return self._write(
|
|
186
|
+
profile, app_key, "POST", f"{self._base_path(app_key)}/{record_id}/update",
|
|
187
|
+
{"fields": fields, "include_schema": include_schema}, "数据面更新",
|
|
188
|
+
)
|
|
189
|
+
|
|
190
|
+
def data_delete(self, *, profile: str, app_key: str, record_ids: list[str | int]) -> JSONObject:
|
|
191
|
+
self._require_app_key(app_key)
|
|
192
|
+
return self._write(profile, app_key, "POST", self._base_path(app_key) + "/delete", {"record_ids": record_ids}, "数据面删除")
|
|
193
|
+
|
|
194
|
+
def _write(self, profile: str, app_key: str, method: str, path: str, body: JSONObject, tool_name: str) -> JSONObject:
|
|
195
|
+
return self._run(
|
|
196
|
+
profile,
|
|
197
|
+
lambda session_profile, context: self._attach_context(
|
|
198
|
+
self._request_data_plane(context, method, path, json_body=body), profile, session_profile.selected_ws_id,
|
|
199
|
+
),
|
|
200
|
+
tool_name=tool_name,
|
|
201
|
+
)
|
|
202
|
+
|
|
203
|
+
def _request_data_plane(
|
|
204
|
+
self, context: BackendRequestContext, method: str, path: str, *, params: JSONObject | None = None,
|
|
205
|
+
json_body: JSONValue = None,
|
|
206
|
+
) -> JSONObject:
|
|
207
|
+
raw = self.backend.request(method, context, path, params=params, json_body=json_body, unwrap=False)
|
|
208
|
+
if not isinstance(raw, dict):
|
|
209
|
+
raise QingflowApiError(category="backend", message="数据面返回了非对象响应")
|
|
210
|
+
code = raw.get("code")
|
|
211
|
+
payload = raw.get("data")
|
|
212
|
+
result: JSONObject = dict(payload) if isinstance(payload, dict) else {}
|
|
213
|
+
success = code in (None, 0, 200, 20000) or str(code).lower() in {"success", "ok"}
|
|
214
|
+
result.setdefault("ok", success)
|
|
215
|
+
result.setdefault("status", "success" if success else "failed")
|
|
216
|
+
result.setdefault("hints", [])
|
|
217
|
+
if not success:
|
|
218
|
+
result.setdefault("message", str(raw.get("message") or raw.get("msg") or "数据面请求失败"))
|
|
219
|
+
result.setdefault("backend_code", code)
|
|
220
|
+
return result
|
|
221
|
+
|
|
222
|
+
def _export_all(
|
|
223
|
+
self, *, context: BackendRequestContext, profile: str, ws_id: int | None, app_key: str,
|
|
224
|
+
body: JSONObject, has_scope: bool,
|
|
225
|
+
) -> JSONObject:
|
|
226
|
+
probe_body = dict(body)
|
|
227
|
+
probe_body.update({"page": 1, "page_size": 1, "include_schema": True})
|
|
228
|
+
probe = self._request_data_plane(context, "POST", self._base_path(app_key) + "/query", json_body=probe_body)
|
|
229
|
+
if probe.get("ok") is False:
|
|
230
|
+
# 探测请求没有产生文件,保留后端结构化错误,不把失败误报为空数据成功。
|
|
231
|
+
return self._attach_context(probe, profile, ws_id)
|
|
232
|
+
records = probe.get("records") if isinstance(probe.get("records"), dict) else {}
|
|
233
|
+
total = records.get("total") if isinstance(records, dict) else None
|
|
234
|
+
if isinstance(total, int) and total > DATA_UNBOUNDED_ROW_THRESHOLD and not has_scope:
|
|
235
|
+
return self._attach_context({
|
|
236
|
+
"ok": True, "status": "needs_scope", "app_key": app_key, "format": "csv", "row_count": 0,
|
|
237
|
+
"complete": False, "truncated": True, "safe_for_final_conclusion": False, "files": [],
|
|
238
|
+
"hints": probe.get("hints", []),
|
|
239
|
+
"schema": probe.get("schema", {}),
|
|
240
|
+
"field_capabilities": probe.get("field_capabilities", []),
|
|
241
|
+
"scope": {"reported_total": total, "row_threshold": DATA_UNBOUNDED_ROW_THRESHOLD},
|
|
242
|
+
}, profile, ws_id)
|
|
243
|
+
schema = probe.get("schema") if isinstance(probe.get("schema"), dict) else {}
|
|
244
|
+
fields = schema.get("fields") if isinstance(schema, dict) else None
|
|
245
|
+
field_list = fields if isinstance(fields, list) else []
|
|
246
|
+
business_fields = [field for field in field_list if isinstance(field, dict) and field.get("field_id") is not None]
|
|
247
|
+
columns = [(str(field.get("field_id")), str(field.get("title") or field.get("field_id"))) for field in business_fields]
|
|
248
|
+
header = ["record_id"] + [f"{title}__field_{field_id}" for field_id, title in columns]
|
|
249
|
+
run_dir = Path.home() / ".qingflow" / "data-exports" / datetime.now().strftime("%Y%m%d-%H%M%S-%f")
|
|
250
|
+
latest_hints = probe.get("hints", [])
|
|
251
|
+
latest_schema = schema
|
|
252
|
+
latest_capabilities = probe.get("field_capabilities", [])
|
|
253
|
+
|
|
254
|
+
def fetch_page(page: int, page_size: int) -> JSONObject:
|
|
255
|
+
nonlocal latest_hints, latest_schema, latest_capabilities
|
|
256
|
+
request_body = dict(body)
|
|
257
|
+
# 每页都请求 schema,使最后一页的后端原始 hints 同时准确表达完整性和字段能力。
|
|
258
|
+
request_body.update({"page": page, "page_size": page_size, "include_schema": True})
|
|
259
|
+
response = self._request_data_plane(
|
|
260
|
+
context, "POST", self._base_path(app_key) + "/query", json_body=request_body
|
|
261
|
+
)
|
|
262
|
+
if response.get("ok") is False:
|
|
263
|
+
raise CsvPageFailure(response)
|
|
264
|
+
latest_hints = response.get("hints", [])
|
|
265
|
+
if isinstance(response.get("schema"), dict):
|
|
266
|
+
latest_schema = response["schema"]
|
|
267
|
+
if isinstance(response.get("field_capabilities"), list):
|
|
268
|
+
latest_capabilities = response["field_capabilities"]
|
|
269
|
+
return response
|
|
270
|
+
|
|
271
|
+
def project_row(record: JSONObject) -> list[str]:
|
|
272
|
+
values = record.get("fields") if isinstance(record.get("fields"), dict) else record
|
|
273
|
+
return [str(record.get("record_id") or "")] + [csv_cell(values.get(field_id)) for field_id, _ in columns]
|
|
274
|
+
|
|
275
|
+
exported = export_csv_pages(
|
|
276
|
+
fetch_page=fetch_page, header=header, project_row=project_row, output_dir=run_dir,
|
|
277
|
+
file_prefix="records", page_size=DATA_PAGE_SIZE, shard_rows=DATA_CSV_SHARD_ROWS,
|
|
278
|
+
time_budget_seconds=DATA_CSV_TIME_BUDGET_SECONDS,
|
|
279
|
+
)
|
|
280
|
+
page_error = exported.pop("page_error", None)
|
|
281
|
+
result: JSONObject = {
|
|
282
|
+
"ok": True, "status": "success" if exported["complete"] else "partial", "app_key": app_key,
|
|
283
|
+
"format": "csv", "local_dir": str(run_dir), "fields": business_fields,
|
|
284
|
+
"schema": latest_schema, "field_capabilities": latest_capabilities, "hints": latest_hints,
|
|
285
|
+
**exported,
|
|
286
|
+
}
|
|
287
|
+
if isinstance(page_error, dict):
|
|
288
|
+
# 后续页失败时保留已落盘分片,同时把后端稳定错误字段原样带回。
|
|
289
|
+
result["ok"] = False
|
|
290
|
+
for key in ("error_code", "message", "backend_code", "write_executed", "retry_safe", "field_errors"):
|
|
291
|
+
if key in page_error:
|
|
292
|
+
result[key] = page_error[key]
|
|
293
|
+
if "hints" in page_error:
|
|
294
|
+
result["hints"] = page_error["hints"]
|
|
295
|
+
return self._attach_context(result, profile, ws_id)
|
|
296
|
+
|
|
297
|
+
@staticmethod
|
|
298
|
+
def _base_path(app_key: str) -> str:
|
|
299
|
+
return f"/app/{app_key}/data/v1/records"
|
|
300
|
+
|
|
301
|
+
@staticmethod
|
|
302
|
+
def _attach_context(result: JSONObject, profile: str, ws_id: int | None) -> JSONObject:
|
|
303
|
+
return {"profile": profile, "ws_id": ws_id, **result}
|
|
304
|
+
|
|
305
|
+
@staticmethod
|
|
306
|
+
def _require_app_key(app_key: str) -> None:
|
|
307
|
+
if not str(app_key or "").strip():
|
|
308
|
+
raise_tool_error(QingflowApiError.config_error("app_key is required"))
|
|
309
|
+
|
|
310
|
+
@staticmethod
|
|
311
|
+
def _require_record_id(record_id: str | int) -> None:
|
|
312
|
+
if not str(record_id or "").strip():
|
|
313
|
+
raise_tool_error(QingflowApiError.config_error("record_id is required"))
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
|
-
import csv
|
|
4
3
|
import html
|
|
5
4
|
import mimetypes
|
|
6
5
|
import json
|
|
@@ -34,6 +33,11 @@ from ..list_type_labels import (
|
|
|
34
33
|
)
|
|
35
34
|
from .base import ToolBase
|
|
36
35
|
from .directory_tools import _directory_has_more, _directory_items
|
|
36
|
+
from .csv_export import CsvShardWriter
|
|
37
|
+
from .storage_download import (
|
|
38
|
+
download_storage_content, environment_cookie_prefix, is_storage_url,
|
|
39
|
+
requires_storage_cookie, storage_auth_type,
|
|
40
|
+
)
|
|
37
41
|
|
|
38
42
|
|
|
39
43
|
DEFAULT_QUERY_PAGE_SIZE = 50
|
|
@@ -2284,57 +2288,22 @@ class RecordTools(ToolBase):
|
|
|
2284
2288
|
}
|
|
2285
2289
|
|
|
2286
2290
|
run_dir = _record_access_run_dir()
|
|
2287
|
-
run_dir.mkdir(parents=True, exist_ok=True)
|
|
2288
2291
|
header = ["record_id"] + [str(field["column_name"]) for field in fields_payload]
|
|
2289
|
-
|
|
2290
|
-
|
|
2291
|
-
|
|
2292
|
-
|
|
2293
|
-
|
|
2294
|
-
|
|
2295
|
-
|
|
2296
|
-
def close_shard() -> None:
|
|
2297
|
-
nonlocal file_handle, writer, shard_rows
|
|
2298
|
-
if file_handle is None:
|
|
2299
|
-
return
|
|
2300
|
-
path = Path(file_handle.name)
|
|
2301
|
-
file_handle.close()
|
|
2302
|
-
files.append(
|
|
2303
|
-
{
|
|
2304
|
-
"local_path": str(path),
|
|
2305
|
-
"row_count": shard_rows,
|
|
2306
|
-
"part": len(files) + 1,
|
|
2307
|
-
}
|
|
2308
|
-
)
|
|
2309
|
-
file_handle = None
|
|
2310
|
-
writer = None
|
|
2311
|
-
shard_rows = 0
|
|
2312
|
-
|
|
2313
|
-
def ensure_writer() -> Any:
|
|
2314
|
-
nonlocal file_handle, writer, shard_part
|
|
2315
|
-
if writer is not None:
|
|
2316
|
-
return writer
|
|
2317
|
-
shard_part += 1
|
|
2318
|
-
path = run_dir / f"records-{shard_part:04d}.csv"
|
|
2319
|
-
file_handle = path.open("w", newline="", encoding="utf-8-sig")
|
|
2320
|
-
writer = csv.writer(file_handle)
|
|
2321
|
-
writer.writerow(header)
|
|
2322
|
-
return writer
|
|
2292
|
+
shard_writer = CsvShardWriter(
|
|
2293
|
+
output_dir=run_dir,
|
|
2294
|
+
file_prefix="records",
|
|
2295
|
+
header=header,
|
|
2296
|
+
shard_rows=DEFAULT_RECORD_ACCESS_SHARD_ROWS,
|
|
2297
|
+
)
|
|
2323
2298
|
|
|
2324
2299
|
def write_record(apply_id: int | None, answer_list: list[JSONValue]) -> None:
|
|
2325
|
-
nonlocal row_count, shard_rows
|
|
2326
2300
|
if apply_id is None:
|
|
2327
2301
|
return
|
|
2328
|
-
if shard_rows >= DEFAULT_RECORD_ACCESS_SHARD_ROWS:
|
|
2329
|
-
close_shard()
|
|
2330
|
-
csv_writer = ensure_writer()
|
|
2331
2302
|
values: list[str] = [_public_record_id_text(apply_id) or ""]
|
|
2332
2303
|
for field in selected_fields:
|
|
2333
2304
|
answer = _find_answer_for_field(answer_list, field)
|
|
2334
2305
|
values.append(_record_access_csv_cell(_normalize_answer_field_value_for_output(answer, field)))
|
|
2335
|
-
|
|
2336
|
-
row_count += 1
|
|
2337
|
-
shard_rows += 1
|
|
2306
|
+
shard_writer.write(values)
|
|
2338
2307
|
|
|
2339
2308
|
current_page = 1
|
|
2340
2309
|
has_more = False
|
|
@@ -2374,7 +2343,7 @@ class RecordTools(ToolBase):
|
|
|
2374
2343
|
break
|
|
2375
2344
|
page = fetch_page(current_page, BACKEND_RECORD_ACCESS_PAGE_SIZE)
|
|
2376
2345
|
finally:
|
|
2377
|
-
close_shard()
|
|
2346
|
+
shard_writer.close_shard()
|
|
2378
2347
|
|
|
2379
2348
|
if stopped_reason == "TIME_BUDGET_EXCEEDED":
|
|
2380
2349
|
warnings.append(_record_access_time_budget_warning(reported_total=reported_total, fetched_pages=fetched_pages))
|
|
@@ -2399,11 +2368,11 @@ class RecordTools(ToolBase):
|
|
|
2399
2368
|
"view_id": view_route.view_id,
|
|
2400
2369
|
"format": "csv",
|
|
2401
2370
|
"local_dir": str(run_dir),
|
|
2402
|
-
"row_count": row_count,
|
|
2371
|
+
"row_count": shard_writer.row_count,
|
|
2403
2372
|
"complete": complete,
|
|
2404
2373
|
"truncated": not complete,
|
|
2405
2374
|
"safe_for_final_conclusion": safe_for_final_conclusion,
|
|
2406
|
-
"files": files,
|
|
2375
|
+
"files": shard_writer.files,
|
|
2407
2376
|
"fields": fields_payload,
|
|
2408
2377
|
"warnings": warnings,
|
|
2409
2378
|
"verification": verification_payload,
|
|
@@ -17272,40 +17241,10 @@ def _record_detail_download_resolved_media_content(
|
|
|
17272
17241
|
environment_prefix_cache: dict[str, str],
|
|
17273
17242
|
requested_strategy: str,
|
|
17274
17243
|
) -> tuple[bytes, JSONObject]:
|
|
17275
|
-
|
|
17276
|
-
|
|
17277
|
-
|
|
17278
|
-
|
|
17279
|
-
context,
|
|
17280
|
-
warnings=warnings,
|
|
17281
|
-
environment_prefix_cache=environment_prefix_cache,
|
|
17282
|
-
)
|
|
17283
|
-
cookie_name = f"{storage_cookie_prefix}token"
|
|
17284
|
-
headers = _record_detail_media_download_headers(context)
|
|
17285
|
-
if hasattr(backend, "download_binary_with_cookie"):
|
|
17286
|
-
response = backend.download_binary_with_cookie(context, source_url, cookie_name=cookie_name, headers=headers)
|
|
17287
|
-
return response.content, {
|
|
17288
|
-
"download_strategy": requested_strategy,
|
|
17289
|
-
"storage_auth_type": storage_auth_type,
|
|
17290
|
-
"storage_cookie_prefix": storage_cookie_prefix,
|
|
17291
|
-
"redirected": bool(getattr(response, "redirected", False)),
|
|
17292
|
-
}
|
|
17293
|
-
cookie_headers = dict(headers)
|
|
17294
|
-
cookie_headers["Cookie"] = f"{cookie_name}={context.token}"
|
|
17295
|
-
content = backend.download_binary(source_url, headers=cookie_headers)
|
|
17296
|
-
return content, {
|
|
17297
|
-
"download_strategy": requested_strategy,
|
|
17298
|
-
"storage_auth_type": storage_auth_type,
|
|
17299
|
-
"storage_cookie_prefix": storage_cookie_prefix,
|
|
17300
|
-
"redirected": False,
|
|
17301
|
-
}
|
|
17302
|
-
content = backend.download_binary(source_url, headers=_record_detail_media_download_headers(context))
|
|
17303
|
-
return content, {
|
|
17304
|
-
"download_strategy": requested_strategy,
|
|
17305
|
-
"storage_auth_type": storage_auth_type,
|
|
17306
|
-
"storage_cookie_prefix": None,
|
|
17307
|
-
"redirected": False,
|
|
17308
|
-
}
|
|
17244
|
+
return download_storage_content(
|
|
17245
|
+
backend=backend, context=context, source_url=source_url, warnings=warnings,
|
|
17246
|
+
environment_prefix_cache=environment_prefix_cache, requested_strategy=requested_strategy,
|
|
17247
|
+
)
|
|
17309
17248
|
|
|
17310
17249
|
|
|
17311
17250
|
def _record_detail_media_download_strategy(source_url: str) -> str:
|
|
@@ -17317,28 +17256,15 @@ def _record_detail_media_download_strategy(source_url: str) -> str:
|
|
|
17317
17256
|
|
|
17318
17257
|
|
|
17319
17258
|
def _record_detail_should_use_storage_cookie(source_url: str, storage_auth_type: str | None) -> bool:
|
|
17320
|
-
|
|
17321
|
-
return False
|
|
17322
|
-
if not storage_auth_type:
|
|
17323
|
-
return True
|
|
17324
|
-
return storage_auth_type.upper() != "ANONYMOUS"
|
|
17259
|
+
return requires_storage_cookie(source_url, storage_auth_type)
|
|
17325
17260
|
|
|
17326
17261
|
|
|
17327
17262
|
def _record_detail_is_qingflow_storage_url(source_url: str) -> bool:
|
|
17328
|
-
|
|
17329
|
-
path = parsed.path.lower()
|
|
17330
|
-
if "/storage/file/" in path:
|
|
17331
|
-
return True
|
|
17332
|
-
query = parse_qs(parsed.query)
|
|
17333
|
-
return any(key.lower() == "qingflow-auth_type" for key in query)
|
|
17263
|
+
return is_storage_url(source_url)
|
|
17334
17264
|
|
|
17335
17265
|
|
|
17336
17266
|
def _record_detail_storage_auth_type(source_url: str) -> str | None:
|
|
17337
|
-
|
|
17338
|
-
for key, values in query.items():
|
|
17339
|
-
if key.lower() == "qingflow-auth_type" and values:
|
|
17340
|
-
return _normalize_optional_text(values[0])
|
|
17341
|
-
return None
|
|
17267
|
+
return storage_auth_type(source_url)
|
|
17342
17268
|
|
|
17343
17269
|
|
|
17344
17270
|
def _record_detail_is_download_file_url(source_url: str) -> bool:
|
|
@@ -17388,28 +17314,9 @@ def _record_detail_environment_prefix(
|
|
|
17388
17314
|
warnings: list[JSONObject],
|
|
17389
17315
|
environment_prefix_cache: dict[str, str],
|
|
17390
17316
|
) -> str:
|
|
17391
|
-
|
|
17392
|
-
|
|
17393
|
-
|
|
17394
|
-
payload = backend.request("GET", context, "/environment/properties")
|
|
17395
|
-
environment = payload.get("environmentProperty") if isinstance(payload, dict) else None
|
|
17396
|
-
if not isinstance(environment, dict):
|
|
17397
|
-
environment = payload.get("environment_property") if isinstance(payload, dict) else None
|
|
17398
|
-
prefix = _normalize_optional_text(environment.get("environmentPrefix")) if isinstance(environment, dict) else None
|
|
17399
|
-
if not prefix and isinstance(environment, dict):
|
|
17400
|
-
prefix = _normalize_optional_text(environment.get("environment_prefix"))
|
|
17401
|
-
except QingflowApiError as exc:
|
|
17402
|
-
prefix = None
|
|
17403
|
-
warnings.append(
|
|
17404
|
-
{
|
|
17405
|
-
"code": "MEDIA_ASSET_ENVIRONMENT_PREFIX_UNAVAILABLE",
|
|
17406
|
-
"message": f"record_get could not read /environment/properties; falling back to qf_ token cookie prefix: {exc.message}",
|
|
17407
|
-
"http_status": exc.http_status,
|
|
17408
|
-
}
|
|
17409
|
-
)
|
|
17410
|
-
prefix = prefix if prefix is not None else "qf_"
|
|
17411
|
-
environment_prefix_cache["value"] = prefix
|
|
17412
|
-
return prefix
|
|
17317
|
+
return environment_cookie_prefix(
|
|
17318
|
+
backend, context, warnings=warnings, environment_prefix_cache=environment_prefix_cache,
|
|
17319
|
+
)
|
|
17413
17320
|
|
|
17414
17321
|
|
|
17415
17322
|
def _record_detail_context_origin(context: BackendRequestContext) -> str:
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import Any
|
|
4
|
+
from urllib.parse import parse_qs, urlsplit
|
|
5
|
+
|
|
6
|
+
from ..backend_client import BackendRequestContext
|
|
7
|
+
from ..config import DEFAULT_USER_AGENT
|
|
8
|
+
from ..errors import QingflowApiError
|
|
9
|
+
from ..json_types import JSONObject
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def storage_auth_type(source_url: str) -> str | None:
|
|
13
|
+
for key, values in parse_qs(urlsplit(source_url).query).items():
|
|
14
|
+
if key.lower() == "qingflow-auth_type" and values:
|
|
15
|
+
return values[0].strip() or None
|
|
16
|
+
return None
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def is_storage_url(source_url: str) -> bool:
|
|
20
|
+
parsed = urlsplit(source_url)
|
|
21
|
+
return "/storage/file/" in parsed.path.lower() or any(
|
|
22
|
+
key.lower() == "qingflow-auth_type" for key in parse_qs(parsed.query)
|
|
23
|
+
)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def requires_storage_cookie(source_url: str, auth_type: str | None) -> bool:
|
|
27
|
+
return is_storage_url(source_url) and (not auth_type or auth_type.upper() != "ANONYMOUS")
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def environment_cookie_prefix(
|
|
31
|
+
backend: Any, context: BackendRequestContext, *, warnings: list[JSONObject],
|
|
32
|
+
environment_prefix_cache: dict[str, str], operation: str = "record_get",
|
|
33
|
+
) -> str:
|
|
34
|
+
if "value" in environment_prefix_cache:
|
|
35
|
+
return environment_prefix_cache["value"]
|
|
36
|
+
try:
|
|
37
|
+
payload = backend.request("GET", context, "/environment/properties")
|
|
38
|
+
environment = payload.get("environmentProperty") if isinstance(payload, dict) else None
|
|
39
|
+
if not isinstance(environment, dict):
|
|
40
|
+
environment = payload.get("environment_property") if isinstance(payload, dict) else None
|
|
41
|
+
prefix = environment.get("environmentPrefix") if isinstance(environment, dict) else None
|
|
42
|
+
if not prefix and isinstance(environment, dict):
|
|
43
|
+
prefix = environment.get("environment_prefix")
|
|
44
|
+
if prefix is not None:
|
|
45
|
+
prefix = str(prefix).strip() or None
|
|
46
|
+
except QingflowApiError as exc:
|
|
47
|
+
prefix = None
|
|
48
|
+
warnings.append({
|
|
49
|
+
"code": "MEDIA_ASSET_ENVIRONMENT_PREFIX_UNAVAILABLE",
|
|
50
|
+
"message": f"{operation} could not read /environment/properties; falling back to qf_ token cookie prefix: {exc.message}",
|
|
51
|
+
"http_status": exc.http_status,
|
|
52
|
+
})
|
|
53
|
+
prefix = prefix if prefix is not None else "qf_"
|
|
54
|
+
environment_prefix_cache["value"] = prefix
|
|
55
|
+
return prefix
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def download_storage_content(
|
|
59
|
+
*, backend: Any, context: BackendRequestContext, source_url: str,
|
|
60
|
+
warnings: list[JSONObject], environment_prefix_cache: dict[str, str],
|
|
61
|
+
requested_strategy: str, operation: str = "record_get",
|
|
62
|
+
max_bytes: int | None = None,
|
|
63
|
+
) -> tuple[bytes, JSONObject]:
|
|
64
|
+
parsed = urlsplit(getattr(context, "base_url", None) or "https://qingflow.com")
|
|
65
|
+
origin = f"{parsed.scheme}://{parsed.netloc}" if parsed.scheme and parsed.netloc else "https://qingflow.com"
|
|
66
|
+
headers = {"User-Agent": DEFAULT_USER_AGENT, "Referer": f"{origin}/", "Origin": origin}
|
|
67
|
+
auth_type = storage_auth_type(source_url)
|
|
68
|
+
prefix = None
|
|
69
|
+
redirected = False
|
|
70
|
+
limits = {"max_bytes": max_bytes} if max_bytes is not None else {}
|
|
71
|
+
if requires_storage_cookie(source_url, auth_type):
|
|
72
|
+
prefix = environment_cookie_prefix(
|
|
73
|
+
backend, context, warnings=warnings, environment_prefix_cache=environment_prefix_cache,
|
|
74
|
+
operation=operation,
|
|
75
|
+
)
|
|
76
|
+
cookie_name = f"{prefix}token"
|
|
77
|
+
if hasattr(backend, "download_binary_with_cookie"):
|
|
78
|
+
response = backend.download_binary_with_cookie(context, source_url, cookie_name=cookie_name, headers=headers, **limits)
|
|
79
|
+
content = response.content
|
|
80
|
+
redirected = bool(getattr(response, "redirected", False))
|
|
81
|
+
else:
|
|
82
|
+
headers["Cookie"] = f"{cookie_name}={context.token}"
|
|
83
|
+
content = backend.download_binary(source_url, headers=headers, **limits)
|
|
84
|
+
else:
|
|
85
|
+
content = backend.download_binary(source_url, headers=headers, **limits)
|
|
86
|
+
return content, {
|
|
87
|
+
"download_strategy": requested_strategy, "storage_auth_type": auth_type,
|
|
88
|
+
"storage_cookie_prefix": prefix, "redirected": redirected,
|
|
89
|
+
}
|