@qingflow-tech/qingflow-cli 1.1.59 → 1.1.61-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -4
- package/npm/lib/runtime.mjs +29 -13
- package/package.json +1 -1
- package/pyproject.toml +3 -1
- package/skills/qingflow-cli/SKILL.md +92 -318
- package/skills/qingflow-cli/manifest.yaml +1 -1
- package/skills/qingflow-cli/reference/00-INDEX.md +4 -4
- package/skills/qingflow-cli/reference/QINGFLOW_CLI_ADMIN_CHEATSHEET.md +1 -1
- package/skills/qingflow-cli/reference/QINGFLOW_CLI_DATA_RETRIEVAL_WORKFLOW.md +3 -137
- package/skills/qingflow-cli/reference/QINGFLOW_CLI_EXPLORATION_REPORT.md +6 -4
- package/skills/qingflow-cli/reference/QINGFLOW_CLI_FIELD_DATA_TYPES.md +14 -14
- package/skills/qingflow-cli/reference/QINGFLOW_CLI_MEMBER_CHEATSHEET.md +20 -19
- package/skills/qingflow-cli/reference/QINGFLOW_CLI_ONE_SHOT_CHEATSHEET.md +18 -14
- package/skills/qingflow-cli/reference/QINGFLOW_CLI_RECORD_CREATE_WORKFLOW.md +7 -12
- package/skills/qingflow-cli/reference/QINGFLOW_CLI_RECORD_UPDATE_WORKFLOW.md +2 -1
- package/skills/qingflow-cli/reference/builder/20-build-complete-system.md +2 -2
- package/skills/qingflow-cli/reference/builder/30-schema-fields.md +9 -5
- package/skills/qingflow-cli/reference/builder/reference/app-delivery-sop.md +3 -3
- package/skills/qingflow-cli/reference/core/QINGFLOW_CLI_ADMIN_CHEATSHEET.md +1 -1
- package/skills/qingflow-cli/reference/core/QINGFLOW_CLI_DATA_RETRIEVAL_WORKFLOW.md +76 -95
- package/skills/qingflow-cli/reference/core/QINGFLOW_CLI_EXPLORATION_REPORT.md +6 -4
- package/skills/qingflow-cli/reference/core/QINGFLOW_CLI_FIELD_DATA_TYPES.md +13 -13
- package/skills/qingflow-cli/reference/core/QINGFLOW_CLI_MEMBER_CHEATSHEET.md +20 -19
- package/skills/qingflow-cli/reference/core/QINGFLOW_CLI_ONE_SHOT_CHEATSHEET.md +18 -14
- package/skills/qingflow-cli/reference/data/README.md +52 -0
- package/skills/qingflow-cli/reference/record/QINGFLOW_CLI_RECORD_SPECIALIZED_WORKFLOW.md +56 -0
- package/skills/qingflow-cli/reference/record/QINGFLOW_CLI_RECORD_UPDATE_WORKFLOW.md +2 -1
- package/skills/qingflow-cli/reference/record/analysis/README.md +21 -113
- package/skills/qingflow-cli/reference/record/analysis/analysis-gotchas.md +10 -10
- package/skills/qingflow-cli/reference/record/analysis/analysis-patterns.md +9 -9
- package/skills/qingflow-cli/reference/record/analysis/business-context.md +1 -1
- package/skills/qingflow-cli/reference/record/analysis/confidence-reporting.md +10 -10
- package/skills/qingflow-cli/reference/record/analysis/data-access-playbook.md +52 -80
- package/skills/qingflow-cli/reference/record/analysis/pandas-recipes.md +2 -2
- package/skills/qingflow-common/SKILL.md +10 -4
- package/src/qingflow_mcp/app_form_compat.py +3 -0
- package/src/qingflow_mcp/app_form_validator.py +69 -17
- package/src/qingflow_mcp/backend_client.py +33 -8
- package/src/qingflow_mcp/builder_facade/app_form.py +497 -100
- package/src/qingflow_mcp/builder_facade/schemas/app_form/manifest.json +1 -0
- package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/complete.json +64 -30
- package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/fields/address.json +4 -8
- package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/fields/amount.json +4 -8
- package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/fields/attachment.json +6 -10
- package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/fields/code_block.json +6 -10
- package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/fields/data_relation.json +11 -18
- package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/fields/date.json +4 -8
- package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/fields/datetime.json +4 -8
- package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/fields/department.json +6 -10
- package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/fields/email.json +4 -8
- package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/fields/link.json +4 -8
- package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/fields/long_text.json +4 -8
- package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/fields/member.json +6 -10
- package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/fields/multi_select.json +4 -8
- package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/fields/number.json +4 -8
- package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/fields/phone.json +4 -8
- package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/fields/q_linker.json +6 -10
- package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/fields/relation.json +46 -15
- package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/fields/rich_text.json +146 -0
- package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/fields/single_choice.json +4 -8
- package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/fields/single_select.json +4 -8
- package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/fields/subtable.json +61 -27
- package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/fields/text.json +4 -8
- package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/formula.json +43 -0
- package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/summary.json +7 -12
- package/src/qingflow_mcp/builder_facade/service.py +39 -1
- package/src/qingflow_mcp/cli/commands/__init__.py +2 -1
- package/src/qingflow_mcp/cli/commands/data.py +144 -0
- package/src/qingflow_mcp/cli/commands/record.py +1 -1
- package/src/qingflow_mcp/cli/context.py +3 -0
- package/src/qingflow_mcp/cli/formatters.py +104 -1
- package/src/qingflow_mcp/public_surface.py +5 -0
- package/src/qingflow_mcp/response_trim.py +2 -1
- package/src/qingflow_mcp/server.py +3 -1
- package/src/qingflow_mcp/server_app_user.py +5 -1
- package/src/qingflow_mcp/tools/csv_export.py +168 -0
- package/src/qingflow_mcp/tools/data_files.py +174 -0
- package/src/qingflow_mcp/tools/data_tools.py +313 -0
- package/src/qingflow_mcp/tools/record_tools.py +25 -118
- package/src/qingflow_mcp/tools/storage_download.py +89 -0
|
@@ -8,7 +8,10 @@ from typing import Any, TextIO
|
|
|
8
8
|
def emit_text_result(result: dict[str, Any], *, hint: str, stream: TextIO) -> None:
|
|
9
9
|
text = _format_cancelled_result(result)
|
|
10
10
|
if text is None:
|
|
11
|
-
if
|
|
11
|
+
if hint in _DATA_FORMAT_HINTS:
|
|
12
|
+
# 数据面失败也必须保留 field_errors 与列表型 hints,不能先走通用失败裁剪。
|
|
13
|
+
text = _format_data_result(result)
|
|
14
|
+
elif _is_failed_result(result) and hint != "task_action_execute":
|
|
12
15
|
text = _format_failed_result(result)
|
|
13
16
|
else:
|
|
14
17
|
formatter = _FORMATTERS.get(hint, _format_generic)
|
|
@@ -90,6 +93,99 @@ def _format_generic(result: dict[str, Any]) -> str:
|
|
|
90
93
|
return "\n".join(lines) + "\n"
|
|
91
94
|
|
|
92
95
|
|
|
96
|
+
def _format_data_result(result: dict[str, Any]) -> str:
|
|
97
|
+
lines = [f"Status: {result.get('status') or ('success' if result.get('ok') is not False else 'failed')}"]
|
|
98
|
+
if result.get("format") == "csv":
|
|
99
|
+
files = result.get("files") if isinstance(result.get("files"), list) else []
|
|
100
|
+
lines.extend(
|
|
101
|
+
[
|
|
102
|
+
f"Local dir: {result.get('local_dir') or '-'}",
|
|
103
|
+
f"Files: {len(files)}",
|
|
104
|
+
]
|
|
105
|
+
)
|
|
106
|
+
for item in files:
|
|
107
|
+
if isinstance(item, dict):
|
|
108
|
+
lines.append(
|
|
109
|
+
f"- part {item.get('part')}: {item.get('local_path')} ({item.get('row_count')} rows)"
|
|
110
|
+
)
|
|
111
|
+
lines.extend(
|
|
112
|
+
[
|
|
113
|
+
f"Rows: {result.get('row_count')}",
|
|
114
|
+
f"Complete: {result.get('complete')}",
|
|
115
|
+
f"Truncated: {result.get('truncated')}",
|
|
116
|
+
f"Safe for final conclusion: {result.get('safe_for_final_conclusion')}",
|
|
117
|
+
f"Stopped reason: {result.get('stopped_reason') or '-'}",
|
|
118
|
+
]
|
|
119
|
+
)
|
|
120
|
+
access = result.get("access") if isinstance(result.get("access"), dict) else {}
|
|
121
|
+
if access:
|
|
122
|
+
sources = access.get("scope_sources") if isinstance(access.get("scope_sources"), list) else []
|
|
123
|
+
lines.append(
|
|
124
|
+
f"Access: level={access.get('level') or '-'} / can_write={access.get('can_write')}"
|
|
125
|
+
f" / scope_sources={','.join(str(item) for item in sources) or '-'}"
|
|
126
|
+
)
|
|
127
|
+
|
|
128
|
+
records = result.get("records") if isinstance(result.get("records"), dict) else {}
|
|
129
|
+
if records:
|
|
130
|
+
items = records.get("items") if isinstance(records.get("items"), list) else []
|
|
131
|
+
lines.append(
|
|
132
|
+
"Pagination: "
|
|
133
|
+
f"page={records.get('page')} / requested_page_size={records.get('requested_page_size')} / "
|
|
134
|
+
f"page_size={records.get('page_size')} / total={records.get('total')} / "
|
|
135
|
+
f"has_more={records.get('has_more')} / returned={len(items)}"
|
|
136
|
+
)
|
|
137
|
+
record = result.get("record") if isinstance(result.get("record"), dict) else {}
|
|
138
|
+
if record:
|
|
139
|
+
lines.append(f"Record ID: {record.get('record_id') or '-'}")
|
|
140
|
+
file_assets = result.get("file_assets")
|
|
141
|
+
if isinstance(file_assets, dict):
|
|
142
|
+
lines.append(f"Attachments: {file_assets.get('status') or '-'}")
|
|
143
|
+
for asset in file_assets.get("items", []):
|
|
144
|
+
if isinstance(asset, dict):
|
|
145
|
+
lines.append(f"- {asset.get('file_name')}: {asset.get('access_status')} / {asset.get('local_path') or '-'}")
|
|
146
|
+
_append_warnings(lines, file_assets.get("warnings"))
|
|
147
|
+
items = result.get("items") if isinstance(result.get("items"), list) else []
|
|
148
|
+
if items:
|
|
149
|
+
lines.append(f"Items: {len(items)}")
|
|
150
|
+
|
|
151
|
+
for key, label in (
|
|
152
|
+
("error_code", "Error Code"),
|
|
153
|
+
("message", "Message"),
|
|
154
|
+
("write_executed", "Write Executed"),
|
|
155
|
+
("retry_safe", "Retry Safe"),
|
|
156
|
+
):
|
|
157
|
+
if key in result:
|
|
158
|
+
lines.append(f"{label}: {result.get(key)}")
|
|
159
|
+
|
|
160
|
+
field_errors = result.get("field_errors") if isinstance(result.get("field_errors"), list) else []
|
|
161
|
+
if field_errors:
|
|
162
|
+
lines.append("Field Errors:")
|
|
163
|
+
_append_data_field_errors(lines, field_errors)
|
|
164
|
+
for index, item in enumerate(items):
|
|
165
|
+
if not isinstance(item, dict) or not isinstance(item.get("field_errors"), list):
|
|
166
|
+
continue
|
|
167
|
+
lines.append(f"Item {index} Field Errors:")
|
|
168
|
+
_append_data_field_errors(lines, item["field_errors"])
|
|
169
|
+
|
|
170
|
+
hints = result.get("hints") if isinstance(result.get("hints"), list) else []
|
|
171
|
+
lines.append(f"Hints: {len(hints)}")
|
|
172
|
+
for hint in hints:
|
|
173
|
+
if isinstance(hint, dict):
|
|
174
|
+
lines.append(f"- {hint.get('code') or '-'}: {hint.get('message') or '-'}")
|
|
175
|
+
else:
|
|
176
|
+
lines.append(f"- {hint}")
|
|
177
|
+
return "\n".join(lines) + "\n"
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def _append_data_field_errors(lines: list[str], field_errors: list[Any]) -> None:
|
|
181
|
+
for error in field_errors:
|
|
182
|
+
if isinstance(error, dict):
|
|
183
|
+
lines.append(
|
|
184
|
+
f"- {error.get('path') or '-'}: {error.get('message') or '-'}"
|
|
185
|
+
f" / fix_hint={error.get('fix_hint') or '-'}"
|
|
186
|
+
)
|
|
187
|
+
|
|
188
|
+
|
|
93
189
|
def _format_whoami(result: dict[str, Any]) -> str:
|
|
94
190
|
lines = [
|
|
95
191
|
f"User: {result.get('nick_name') or '-'} ({result.get('email') or '-'})",
|
|
@@ -1669,4 +1765,11 @@ _FORMATTERS = {
|
|
|
1669
1765
|
"builder_summary": _format_builder_summary,
|
|
1670
1766
|
"file_upload_local": _format_file_upload_local,
|
|
1671
1767
|
"feedback_submit": _format_feedback_submit,
|
|
1768
|
+
"data_query": _format_data_result,
|
|
1769
|
+
"data_get": _format_data_result,
|
|
1770
|
+
"data_insert": _format_data_result,
|
|
1771
|
+
"data_update": _format_data_result,
|
|
1772
|
+
"data_delete": _format_data_result,
|
|
1672
1773
|
}
|
|
1774
|
+
|
|
1775
|
+
_DATA_FORMAT_HINTS = {"data_query", "data_get", "data_insert", "data_update", "data_delete"}
|
|
@@ -40,6 +40,11 @@ USER_PUBLIC_TOOL_SPECS: tuple[PublicToolSpec, ...] = (
|
|
|
40
40
|
PublicToolSpec(USER_DOMAIN, "workspace_select", ("workspace_select",), ("workspace", "select")),
|
|
41
41
|
PublicToolSpec(USER_DOMAIN, "app_list", ("app_list",), ("app", "list"), cli_show_effective_context=True),
|
|
42
42
|
PublicToolSpec(USER_DOMAIN, "app_get", ("app_get",), ("app", "get"), cli_show_effective_context=True),
|
|
43
|
+
PublicToolSpec(USER_DOMAIN, "data_query", ("data_query",), ("data", "query"), cli_show_effective_context=True),
|
|
44
|
+
PublicToolSpec(USER_DOMAIN, "data_get", ("data_get",), ("data", "get"), cli_show_effective_context=True),
|
|
45
|
+
PublicToolSpec(USER_DOMAIN, "data_insert", ("data_insert",), ("data", "insert"), cli_show_effective_context=True, cli_context_write=True),
|
|
46
|
+
PublicToolSpec(USER_DOMAIN, "data_update", ("data_update",), ("data", "update"), cli_show_effective_context=True, cli_context_write=True),
|
|
47
|
+
PublicToolSpec(USER_DOMAIN, "data_delete", ("data_delete",), ("data", "delete"), cli_show_effective_context=True, cli_context_write=True),
|
|
43
48
|
PublicToolSpec(USER_DOMAIN, "portal_list", ("portal_list",), ("portal", "list"), cli_show_effective_context=True),
|
|
44
49
|
PublicToolSpec(USER_DOMAIN, "portal_get", ("portal_get",), ("portal", "get"), cli_show_effective_context=True),
|
|
45
50
|
PublicToolSpec(USER_DOMAIN, "view_get", ("view_get",), ("view", "get"), cli_show_effective_context=True),
|
|
@@ -51,8 +51,9 @@ COMMON_ERROR_DROP_TOP = {
|
|
|
51
51
|
|
|
52
52
|
SUCCESS_POLICY_BY_TOOL: dict[str, TransformFn] = {}
|
|
53
53
|
|
|
54
|
-
#:
|
|
54
|
+
#: 这些校验命令的产出契约明确包含 ok 字段(供阶段 2 SKILL.md 引用),
|
|
55
55
|
#: 需在 success 裁剪时保留 ok(默认 COMMON_SUCCESS_DROP_TOP 会丢弃 ok)。
|
|
56
|
+
#: `builder:app_views_validate`(views 声明式校验)同属校验类工具,ok 同样进产出契约。
|
|
56
57
|
_KEEP_OK_TOOLS = {
|
|
57
58
|
"builder:app_form_validate_formula",
|
|
58
59
|
"builder:app_form_validate_code_block",
|
|
@@ -7,6 +7,7 @@ from mcp.server.fastmcp import FastMCP
|
|
|
7
7
|
from .backend_client import BackendClient
|
|
8
8
|
from .session_store import SessionStore
|
|
9
9
|
from .tools.app_tools import AppTools
|
|
10
|
+
from .tools.data_tools import DataTools
|
|
10
11
|
from .tools.auth_tools import AuthTools
|
|
11
12
|
from .tools.code_block_tools import CodeBlockTools
|
|
12
13
|
from .tools.feedback_tools import FeedbackTools
|
|
@@ -148,7 +149,7 @@ Analysis answers must include concrete numbers. When applicable, include percent
|
|
|
148
149
|
- `record_delete` deletes by `record_id` or `record_ids`, and requires an accessible system `view_id`; use custom views only to locate records, not as the delete route.
|
|
149
150
|
- `record_get` is the single-record frontend detail context tool. It returns detail-page visible fields, one-level relation targets, first-page data/workflow logs, associated views/reports, local readable image assets, local downloadable file assets, unavailable context, and `semantic_context`.
|
|
150
151
|
- Use `record_logs_get` only when the user needs the full visible data/workflow log history for a specific record. It writes JSONL files locally and returns file paths plus completeness metadata; do not expect full log arrays in the response.
|
|
151
|
-
-
|
|
152
|
+
- Download record attachments with `data_get(download_attachments=true, include_schema=true)` for the same app and record; use `file_assets.items[].local_path` only when `access_status=downloaded`. This includes original images and videos. Inspect `file_assets.warnings` for partial failures; do not request remote private URLs or fall back to record/detail tools after a data-plane denial. For specialized detail-page context, read `record_get` rich-text images from `media_assets` and its files from `file_assets` local paths or `extraction.text_path` when present.
|
|
152
153
|
- `record_get.columns` are focus hints only; they do not project the detail fields. Read facts from top-level `fields[]`.
|
|
153
154
|
- When readback shape matters after insert or update, prefer `record_get` for human/detail-page context, or `record_list(..., output_profile="normalized")` for batch-shaped normalized rows.
|
|
154
155
|
|
|
@@ -249,6 +250,7 @@ If the current MCP capability is unsupported, the workflow is awkward, or the us
|
|
|
249
250
|
TaskContextTools(sessions, backend).register(server)
|
|
250
251
|
RoleTools(sessions, backend).register(server)
|
|
251
252
|
AppTools(sessions, backend).register(server)
|
|
253
|
+
DataTools(sessions, backend).register(server)
|
|
252
254
|
QingbiReportTools(sessions, backend).register(server)
|
|
253
255
|
PackageTools(sessions, backend).register(server)
|
|
254
256
|
NavigationTools(sessions, backend).register(server)
|
|
@@ -10,6 +10,7 @@ from .config import DEFAULT_PROFILE
|
|
|
10
10
|
from .response_trim import USER_SERVER_METHOD_MAP, trim_error_response, trim_public_response, wrap_trimmed_methods
|
|
11
11
|
from .session_store import SessionStore
|
|
12
12
|
from .tools.app_tools import AppTools
|
|
13
|
+
from .tools.data_tools import DataTools
|
|
13
14
|
from .tools.auth_tools import AuthTools
|
|
14
15
|
from .tools.code_block_tools import CodeBlockTools
|
|
15
16
|
from .tools.directory_tools import DirectoryTools
|
|
@@ -143,7 +144,7 @@ Analysis answers must include concrete numbers. When applicable, include percent
|
|
|
143
144
|
- `record_delete` deletes by `record_id` or `record_ids`, and requires an accessible system `view_id`; use custom views only to locate records, not as the delete route.
|
|
144
145
|
- `record_get` is the single-record frontend detail context tool. It returns detail-page visible fields, one-level relation targets, first-page data/workflow logs, associated views/reports, local readable image assets, local downloadable file assets, unavailable context, and `semantic_context`.
|
|
145
146
|
- Use `record_logs_get` only when the user needs the full visible data/workflow log history for a specific record. It writes JSONL files locally and returns file paths plus completeness metadata; do not expect full log arrays in the response.
|
|
146
|
-
-
|
|
147
|
+
- Download record attachments with `data_get(download_attachments=true, include_schema=true)` for the same app and record; use `file_assets.items[].local_path` only when `access_status=downloaded`. This includes original images and videos. Inspect `file_assets.warnings` for partial failures; do not request remote private URLs or fall back to record/detail tools after a data-plane denial. For specialized detail-page context, read `record_get` rich-text images from `media_assets` and its files from `file_assets` local paths or `extraction.text_path` when present.
|
|
147
148
|
- `record_get.columns` are focus hints only; they do not project the detail fields. Read facts from top-level `fields[]`.
|
|
148
149
|
- When readback shape matters after insert or update, prefer `record_get` for human/detail-page context, or `record_list(..., output_profile="normalized")` for batch-shaped normalized rows.
|
|
149
150
|
|
|
@@ -242,6 +243,7 @@ If the current MCP capability is unsupported, the workflow is awkward, or the us
|
|
|
242
243
|
backend = BackendClient()
|
|
243
244
|
auth = wrap_trimmed_methods(AuthTools(sessions, backend), USER_SERVER_METHOD_MAP)
|
|
244
245
|
apps = wrap_trimmed_methods(AppTools(sessions, backend), USER_SERVER_METHOD_MAP)
|
|
246
|
+
data_tools = wrap_trimmed_methods(DataTools(sessions, backend), USER_SERVER_METHOD_MAP)
|
|
245
247
|
workspace = wrap_trimmed_methods(WorkspaceTools(sessions, backend), USER_SERVER_METHOD_MAP)
|
|
246
248
|
file_tools = wrap_trimmed_methods(FileTools(sessions, backend), USER_SERVER_METHOD_MAP)
|
|
247
249
|
imports = wrap_trimmed_methods(ImportTools(sessions, backend), USER_SERVER_METHOD_MAP)
|
|
@@ -395,6 +397,8 @@ If the current MCP capability is unsupported, the workflow is awkward, or the us
|
|
|
395
397
|
def app_get(profile: str = DEFAULT_PROFILE, app_key: str = "") -> dict:
|
|
396
398
|
return apps.app_get(profile=profile, app_key=app_key)
|
|
397
399
|
|
|
400
|
+
data_tools.register(server)
|
|
401
|
+
|
|
398
402
|
@server.tool()
|
|
399
403
|
def portal_list(profile: str = DEFAULT_PROFILE) -> dict:
|
|
400
404
|
return resources.portal_list(profile=profile)
|
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import csv
|
|
4
|
+
import json
|
|
5
|
+
import time
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import Any, Callable
|
|
8
|
+
|
|
9
|
+
from ..json_types import JSONObject
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
PageFetcher = Callable[[int, int], JSONObject]
|
|
13
|
+
RowProjector = Callable[[JSONObject], list[str]]
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class CsvPageFailure(RuntimeError):
|
|
17
|
+
"""后续分页返回结构化业务错误;已写文件保留为不完整结果。"""
|
|
18
|
+
|
|
19
|
+
def __init__(self, payload: JSONObject) -> None:
|
|
20
|
+
super().__init__(str(payload.get("message") or "分页请求失败"))
|
|
21
|
+
self.payload = payload
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class CsvShardWriter:
|
|
25
|
+
"""视图无关的 UTF-8 BOM CSV 分片写入器。"""
|
|
26
|
+
|
|
27
|
+
def __init__(self, *, output_dir: Path, file_prefix: str, header: list[str], shard_rows: int) -> None:
|
|
28
|
+
self.output_dir = output_dir
|
|
29
|
+
self.file_prefix = file_prefix
|
|
30
|
+
self.header = header
|
|
31
|
+
self.shard_rows = shard_rows
|
|
32
|
+
self.files: list[JSONObject] = []
|
|
33
|
+
self.row_count = 0
|
|
34
|
+
self._part = 0
|
|
35
|
+
self._current_rows = 0
|
|
36
|
+
self._file: Any | None = None
|
|
37
|
+
self._writer: Any | None = None
|
|
38
|
+
output_dir.mkdir(parents=True, exist_ok=True)
|
|
39
|
+
|
|
40
|
+
def write(self, values: list[str]) -> None:
|
|
41
|
+
if self._current_rows >= self.shard_rows:
|
|
42
|
+
self.close_shard()
|
|
43
|
+
self._ensure_writer().writerow(values)
|
|
44
|
+
self._current_rows += 1
|
|
45
|
+
self.row_count += 1
|
|
46
|
+
|
|
47
|
+
def close_shard(self) -> None:
|
|
48
|
+
if self._file is None:
|
|
49
|
+
return
|
|
50
|
+
path = Path(self._file.name)
|
|
51
|
+
self._file.close()
|
|
52
|
+
self.files.append({"local_path": str(path), "row_count": self._current_rows, "part": len(self.files) + 1})
|
|
53
|
+
self._file = None
|
|
54
|
+
self._writer = None
|
|
55
|
+
self._current_rows = 0
|
|
56
|
+
|
|
57
|
+
def _ensure_writer(self) -> Any:
|
|
58
|
+
if self._writer is not None:
|
|
59
|
+
return self._writer
|
|
60
|
+
self._part += 1
|
|
61
|
+
path = self.output_dir / f"{self.file_prefix}-{self._part:04d}.csv"
|
|
62
|
+
self._file = path.open("w", newline="", encoding="utf-8-sig")
|
|
63
|
+
self._writer = csv.writer(self._file)
|
|
64
|
+
self._writer.writerow(self.header)
|
|
65
|
+
return self._writer
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def export_csv_pages(
|
|
69
|
+
*,
|
|
70
|
+
fetch_page: PageFetcher,
|
|
71
|
+
header: list[str],
|
|
72
|
+
project_row: RowProjector,
|
|
73
|
+
output_dir: Path,
|
|
74
|
+
file_prefix: str,
|
|
75
|
+
page_size: int,
|
|
76
|
+
shard_rows: int,
|
|
77
|
+
time_budget_seconds: float,
|
|
78
|
+
) -> JSONObject:
|
|
79
|
+
"""执行与业务权限无关的分页、CSV 分片和时间预算控制。"""
|
|
80
|
+
output_dir.mkdir(parents=True, exist_ok=True)
|
|
81
|
+
deadline = time.monotonic() + time_budget_seconds
|
|
82
|
+
files: list[JSONObject] = []
|
|
83
|
+
row_count = 0
|
|
84
|
+
fetched_pages = 0
|
|
85
|
+
current_page = 1
|
|
86
|
+
file_handle: Any | None = None
|
|
87
|
+
writer: Any | None = None
|
|
88
|
+
current_rows = 0
|
|
89
|
+
part = 0
|
|
90
|
+
has_more = False
|
|
91
|
+
stopped_reason: str | None = None
|
|
92
|
+
page_error: JSONObject | None = None
|
|
93
|
+
|
|
94
|
+
def close_file() -> None:
|
|
95
|
+
nonlocal file_handle, writer, current_rows
|
|
96
|
+
if file_handle is None:
|
|
97
|
+
return
|
|
98
|
+
path = Path(file_handle.name)
|
|
99
|
+
file_handle.close()
|
|
100
|
+
files.append({"local_path": str(path), "row_count": current_rows, "part": len(files) + 1})
|
|
101
|
+
file_handle = None
|
|
102
|
+
writer = None
|
|
103
|
+
current_rows = 0
|
|
104
|
+
|
|
105
|
+
def get_writer() -> Any:
|
|
106
|
+
nonlocal file_handle, writer, part
|
|
107
|
+
if writer is not None:
|
|
108
|
+
return writer
|
|
109
|
+
part += 1
|
|
110
|
+
path = output_dir / f"{file_prefix}-{part:04d}.csv"
|
|
111
|
+
file_handle = path.open("w", newline="", encoding="utf-8-sig")
|
|
112
|
+
writer = csv.writer(file_handle)
|
|
113
|
+
writer.writerow(header)
|
|
114
|
+
return writer
|
|
115
|
+
|
|
116
|
+
try:
|
|
117
|
+
while True:
|
|
118
|
+
page = fetch_page(current_page, page_size)
|
|
119
|
+
records = page.get("records") if isinstance(page, dict) else None
|
|
120
|
+
container = records if isinstance(records, dict) else page
|
|
121
|
+
raw_items = container.get("items", container.get("list", [])) if isinstance(container, dict) else []
|
|
122
|
+
items = raw_items if isinstance(raw_items, list) else []
|
|
123
|
+
has_more = bool(container.get("has_more")) if isinstance(container, dict) else False
|
|
124
|
+
for item in items:
|
|
125
|
+
if not isinstance(item, dict):
|
|
126
|
+
continue
|
|
127
|
+
if current_rows >= shard_rows:
|
|
128
|
+
close_file()
|
|
129
|
+
get_writer().writerow(project_row(item))
|
|
130
|
+
current_rows += 1
|
|
131
|
+
row_count += 1
|
|
132
|
+
fetched_pages += 1
|
|
133
|
+
if not has_more:
|
|
134
|
+
break
|
|
135
|
+
if time.monotonic() >= deadline:
|
|
136
|
+
stopped_reason = "TIME_BUDGET_EXCEEDED"
|
|
137
|
+
break
|
|
138
|
+
current_page += 1
|
|
139
|
+
except CsvPageFailure as exception:
|
|
140
|
+
stopped_reason = "PAGE_FETCH_FAILED"
|
|
141
|
+
page_error = exception.payload
|
|
142
|
+
except Exception:
|
|
143
|
+
# 已落盘分片仍是有效的部分结果;调用方据此禁止形成完整性结论。
|
|
144
|
+
stopped_reason = "PAGE_FETCH_FAILED"
|
|
145
|
+
finally:
|
|
146
|
+
close_file()
|
|
147
|
+
|
|
148
|
+
complete = stopped_reason is None and not has_more
|
|
149
|
+
result: JSONObject = {
|
|
150
|
+
"files": files,
|
|
151
|
+
"row_count": row_count,
|
|
152
|
+
"complete": complete,
|
|
153
|
+
"truncated": not complete,
|
|
154
|
+
"safe_for_final_conclusion": complete,
|
|
155
|
+
"fetched_pages": fetched_pages,
|
|
156
|
+
"stopped_reason": stopped_reason,
|
|
157
|
+
}
|
|
158
|
+
if page_error is not None:
|
|
159
|
+
result["page_error"] = page_error
|
|
160
|
+
return result
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def csv_cell(value: Any) -> str:
|
|
164
|
+
if value is None:
|
|
165
|
+
return ""
|
|
166
|
+
if isinstance(value, (dict, list)):
|
|
167
|
+
return json.dumps(value, ensure_ascii=False, separators=(",", ":"))
|
|
168
|
+
return str(value)
|
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import mimetypes
|
|
4
|
+
import os
|
|
5
|
+
import re
|
|
6
|
+
import time
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from typing import Any, Callable
|
|
9
|
+
from urllib.parse import unquote, urlsplit
|
|
10
|
+
from uuid import uuid4
|
|
11
|
+
|
|
12
|
+
from ..backend_client import BackendRequestContext
|
|
13
|
+
from ..config import get_mcp_home
|
|
14
|
+
from ..errors import QingflowApiError
|
|
15
|
+
from ..json_types import JSONObject
|
|
16
|
+
from .storage_download import download_storage_content, requires_storage_cookie, storage_auth_type
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
DATA_FILE_MAX_FILES = 50
|
|
20
|
+
DATA_FILE_MAX_BYTES = 50 * 1024 * 1024
|
|
21
|
+
DATA_FILE_MAX_TOTAL_BYTES = 200 * 1024 * 1024
|
|
22
|
+
DATA_FILE_TIME_BUDGET_SECONDS = 55.0
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def attachment_candidates(record: JSONObject, schema: JSONObject) -> list[JSONObject]:
|
|
26
|
+
candidates: list[JSONObject] = []
|
|
27
|
+
|
|
28
|
+
def walk(values: Any, fields: Any, prefix: str) -> None:
|
|
29
|
+
if not isinstance(values, dict) or not isinstance(fields, list):
|
|
30
|
+
return
|
|
31
|
+
for field in fields:
|
|
32
|
+
if not isinstance(field, dict) or field.get("field_id") is None:
|
|
33
|
+
continue
|
|
34
|
+
key = str(field["field_id"])
|
|
35
|
+
value = values.get(key)
|
|
36
|
+
path = f"{prefix}.{key}"
|
|
37
|
+
if field.get("data_type") == "table" and isinstance(value, list):
|
|
38
|
+
for index, row in enumerate(value):
|
|
39
|
+
walk(row, field.get("children"), f"{path}[{index}]")
|
|
40
|
+
elif field.get("data_type") == "attachment":
|
|
41
|
+
for index, item in enumerate(value if isinstance(value, list) else [value]):
|
|
42
|
+
url = item.get("value") or item.get("url") if isinstance(item, dict) else item
|
|
43
|
+
if not isinstance(url, str) or not url.strip():
|
|
44
|
+
continue
|
|
45
|
+
name = (item.get("other_info") or item.get("otherInfo") or item.get("name")) if isinstance(item, dict) else None
|
|
46
|
+
if not isinstance(name, str) or not name.strip():
|
|
47
|
+
try:
|
|
48
|
+
name = Path(unquote(urlsplit(url).path)).name
|
|
49
|
+
except ValueError:
|
|
50
|
+
name = None
|
|
51
|
+
name = name or f"attachment_{len(candidates) + 1}"
|
|
52
|
+
candidates.append({
|
|
53
|
+
"field_id": field["field_id"], "file_name": name,
|
|
54
|
+
"source_path": f"{path}[{index}]", "source_url": url.strip(),
|
|
55
|
+
})
|
|
56
|
+
|
|
57
|
+
walk(record.get("fields"), schema.get("fields"), "fields")
|
|
58
|
+
return candidates
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def _download_url(candidate: JSONObject) -> str:
|
|
62
|
+
url = str(candidate["source_url"])
|
|
63
|
+
parsed = urlsplit(url)
|
|
64
|
+
if parsed.scheme not in {"http", "https"} or not parsed.netloc or parsed.username or parsed.password:
|
|
65
|
+
raise QingflowApiError.config_error("附件没有可直接下载的 HTTP 地址")
|
|
66
|
+
return url
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _matching_attachment(
|
|
70
|
+
candidate: JSONObject, refreshed: list[JSONObject],
|
|
71
|
+
) -> JSONObject | None:
|
|
72
|
+
# shortcut: 沿用旧链路,同一字段内同名附件取首个;需要区分时再引入稳定文件 ID。
|
|
73
|
+
return next((item for item in refreshed if (
|
|
74
|
+
str(item["field_id"]) == str(candidate["field_id"])
|
|
75
|
+
and str(item["file_name"]).strip() == str(candidate["file_name"]).strip()
|
|
76
|
+
)), None)
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def download_data_attachments(
|
|
80
|
+
*, backend: Any, context: BackendRequestContext, record: JSONObject, schema: JSONObject,
|
|
81
|
+
refresh_record: Callable[[], JSONObject],
|
|
82
|
+
) -> JSONObject:
|
|
83
|
+
candidates = attachment_candidates(record, schema)
|
|
84
|
+
if not candidates:
|
|
85
|
+
return {"status": "none", "local_dir": None, "items": [], "warnings": []}
|
|
86
|
+
custom_home = os.environ.get("QINGFLOW_MCP_DATA_FILES_HOME")
|
|
87
|
+
local_dir = (Path(custom_home).expanduser() if custom_home else get_mcp_home() / "data-files") / uuid4().hex
|
|
88
|
+
items: list[JSONObject] = []
|
|
89
|
+
warnings: list[JSONObject] = []
|
|
90
|
+
prefix_cache: dict[str, str] = {}
|
|
91
|
+
fresh_candidates: list[JSONObject] | None = None
|
|
92
|
+
deadline = time.monotonic() + DATA_FILE_TIME_BUDGET_SECONDS
|
|
93
|
+
total_bytes = 0
|
|
94
|
+
|
|
95
|
+
for index, original in enumerate(candidates):
|
|
96
|
+
item: JSONObject = {**original, "local_path": None}
|
|
97
|
+
items.append(item)
|
|
98
|
+
if index >= DATA_FILE_MAX_FILES or time.monotonic() >= deadline:
|
|
99
|
+
item["access_status"] = "skipped_limit"
|
|
100
|
+
warnings.append({"code": "FILE_ASSET_LIMIT_EXCEEDED", "message": "附件数量或下载时间超过本次预算"})
|
|
101
|
+
continue
|
|
102
|
+
try:
|
|
103
|
+
if fresh_candidates is not None:
|
|
104
|
+
replacement = _matching_attachment(item, fresh_candidates)
|
|
105
|
+
if replacement is None:
|
|
106
|
+
raise QingflowApiError(category="http", message="刷新后附件不在当前可见范围内", http_status=403)
|
|
107
|
+
item["source_url"] = replacement["source_url"]
|
|
108
|
+
url = _download_url(item)
|
|
109
|
+
|
|
110
|
+
def download(source_url: str) -> tuple[bytes, JSONObject]:
|
|
111
|
+
return download_storage_content(
|
|
112
|
+
backend=backend, context=context, source_url=source_url, warnings=warnings,
|
|
113
|
+
environment_prefix_cache=prefix_cache, operation="data_get",
|
|
114
|
+
max_bytes=min(DATA_FILE_MAX_BYTES, DATA_FILE_MAX_TOTAL_BYTES - total_bytes),
|
|
115
|
+
requested_strategy="qingflow_storage_cookie_redirect" if requires_storage_cookie(
|
|
116
|
+
source_url, storage_auth_type(source_url)
|
|
117
|
+
) else "referer_acl",
|
|
118
|
+
)
|
|
119
|
+
|
|
120
|
+
try:
|
|
121
|
+
content, metadata = download(url)
|
|
122
|
+
except QingflowApiError as exc:
|
|
123
|
+
if exc.http_status not in {401, 403}:
|
|
124
|
+
raise
|
|
125
|
+
if fresh_candidates is None:
|
|
126
|
+
fresh_candidates = []
|
|
127
|
+
fresh = refresh_record()
|
|
128
|
+
if fresh.get("ok") is False:
|
|
129
|
+
raise QingflowApiError(
|
|
130
|
+
category="backend", message=str(fresh.get("message") or "刷新附件访问范围失败"),
|
|
131
|
+
backend_code=fresh.get("backend_code"),
|
|
132
|
+
) from exc
|
|
133
|
+
if not isinstance(fresh.get("record"), dict) or not isinstance(fresh.get("schema"), dict):
|
|
134
|
+
raise QingflowApiError(category="backend", message="刷新附件需要数据面返回 record 和 schema") from exc
|
|
135
|
+
fresh_candidates = attachment_candidates(fresh["record"], fresh["schema"])
|
|
136
|
+
replacement = _matching_attachment(item, fresh_candidates)
|
|
137
|
+
if replacement is None or replacement["source_url"] == url:
|
|
138
|
+
raise
|
|
139
|
+
url = _download_url(replacement)
|
|
140
|
+
content, metadata = download(url)
|
|
141
|
+
item["source_url"] = url
|
|
142
|
+
warnings.append({
|
|
143
|
+
"code": "FILE_ASSET_STORAGE_URL_REFRESHED", "source_path": item["source_path"],
|
|
144
|
+
"message": "已通过数据面重新读取同一记录并刷新附件链接",
|
|
145
|
+
})
|
|
146
|
+
item.update(metadata)
|
|
147
|
+
item["size_bytes"] = len(content)
|
|
148
|
+
if len(content) > DATA_FILE_MAX_BYTES or total_bytes + len(content) > DATA_FILE_MAX_TOTAL_BYTES:
|
|
149
|
+
item["access_status"] = "too_large"
|
|
150
|
+
warnings.append({"code": "FILE_ASSET_SIZE_LIMIT_EXCEEDED", "message": "附件超过单文件或总大小预算"})
|
|
151
|
+
continue
|
|
152
|
+
suffix = Path(str(item["file_name"])).suffix
|
|
153
|
+
suffix = suffix if re.fullmatch(r"\.[A-Za-z0-9]{1,10}", suffix) else ""
|
|
154
|
+
local_dir.mkdir(parents=True, exist_ok=True)
|
|
155
|
+
local_path = local_dir / f"file_{index + 1:04d}{suffix}"
|
|
156
|
+
local_path.write_bytes(content)
|
|
157
|
+
total_bytes += len(content)
|
|
158
|
+
item.update({
|
|
159
|
+
"local_path": str(local_path), "access_status": "downloaded",
|
|
160
|
+
"mime_type": mimetypes.guess_type(str(item["file_name"]))[0],
|
|
161
|
+
})
|
|
162
|
+
except (QingflowApiError, OSError, ValueError) as exc:
|
|
163
|
+
blocked = isinstance(exc, QingflowApiError) and exc.http_status in {401, 403}
|
|
164
|
+
too_large = isinstance(exc, QingflowApiError) and exc.category == "file_size_limit"
|
|
165
|
+
item["access_status"] = "too_large" if too_large else "blocked_private_url" if blocked else "download_failed"
|
|
166
|
+
warnings.append({
|
|
167
|
+
"code": "FILE_ASSET_SIZE_LIMIT_EXCEEDED" if too_large else "FILE_ASSET_DOWNLOAD_FAILED", "source_path": item["source_path"],
|
|
168
|
+
"message": exc.message if isinstance(exc, QingflowApiError) else str(exc),
|
|
169
|
+
"http_status": exc.http_status if isinstance(exc, QingflowApiError) else None,
|
|
170
|
+
})
|
|
171
|
+
return {
|
|
172
|
+
"status": "ok" if all(item["access_status"] == "downloaded" for item in items) else "partial",
|
|
173
|
+
"local_dir": str(local_dir) if local_dir.exists() else None, "items": items, "warnings": warnings,
|
|
174
|
+
}
|