@qingflow-tech/qingflow-cli 1.1.59 → 1.1.61-beta.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. package/README.md +4 -4
  2. package/npm/lib/runtime.mjs +29 -13
  3. package/package.json +1 -1
  4. package/pyproject.toml +3 -1
  5. package/skills/qingflow-cli/SKILL.md +92 -318
  6. package/skills/qingflow-cli/manifest.yaml +1 -1
  7. package/skills/qingflow-cli/reference/00-INDEX.md +4 -4
  8. package/skills/qingflow-cli/reference/QINGFLOW_CLI_ADMIN_CHEATSHEET.md +1 -1
  9. package/skills/qingflow-cli/reference/QINGFLOW_CLI_DATA_RETRIEVAL_WORKFLOW.md +3 -137
  10. package/skills/qingflow-cli/reference/QINGFLOW_CLI_EXPLORATION_REPORT.md +6 -4
  11. package/skills/qingflow-cli/reference/QINGFLOW_CLI_FIELD_DATA_TYPES.md +14 -14
  12. package/skills/qingflow-cli/reference/QINGFLOW_CLI_MEMBER_CHEATSHEET.md +20 -19
  13. package/skills/qingflow-cli/reference/QINGFLOW_CLI_ONE_SHOT_CHEATSHEET.md +18 -14
  14. package/skills/qingflow-cli/reference/QINGFLOW_CLI_RECORD_CREATE_WORKFLOW.md +7 -12
  15. package/skills/qingflow-cli/reference/QINGFLOW_CLI_RECORD_UPDATE_WORKFLOW.md +2 -1
  16. package/skills/qingflow-cli/reference/builder/20-build-complete-system.md +2 -2
  17. package/skills/qingflow-cli/reference/builder/30-schema-fields.md +9 -5
  18. package/skills/qingflow-cli/reference/builder/reference/app-delivery-sop.md +3 -3
  19. package/skills/qingflow-cli/reference/core/QINGFLOW_CLI_ADMIN_CHEATSHEET.md +1 -1
  20. package/skills/qingflow-cli/reference/core/QINGFLOW_CLI_DATA_RETRIEVAL_WORKFLOW.md +76 -95
  21. package/skills/qingflow-cli/reference/core/QINGFLOW_CLI_EXPLORATION_REPORT.md +6 -4
  22. package/skills/qingflow-cli/reference/core/QINGFLOW_CLI_FIELD_DATA_TYPES.md +13 -13
  23. package/skills/qingflow-cli/reference/core/QINGFLOW_CLI_MEMBER_CHEATSHEET.md +20 -19
  24. package/skills/qingflow-cli/reference/core/QINGFLOW_CLI_ONE_SHOT_CHEATSHEET.md +18 -14
  25. package/skills/qingflow-cli/reference/data/README.md +52 -0
  26. package/skills/qingflow-cli/reference/record/QINGFLOW_CLI_RECORD_SPECIALIZED_WORKFLOW.md +56 -0
  27. package/skills/qingflow-cli/reference/record/QINGFLOW_CLI_RECORD_UPDATE_WORKFLOW.md +2 -1
  28. package/skills/qingflow-cli/reference/record/analysis/README.md +21 -113
  29. package/skills/qingflow-cli/reference/record/analysis/analysis-gotchas.md +10 -10
  30. package/skills/qingflow-cli/reference/record/analysis/analysis-patterns.md +9 -9
  31. package/skills/qingflow-cli/reference/record/analysis/business-context.md +1 -1
  32. package/skills/qingflow-cli/reference/record/analysis/confidence-reporting.md +10 -10
  33. package/skills/qingflow-cli/reference/record/analysis/data-access-playbook.md +52 -80
  34. package/skills/qingflow-cli/reference/record/analysis/pandas-recipes.md +2 -2
  35. package/skills/qingflow-common/SKILL.md +10 -4
  36. package/src/qingflow_mcp/app_form_compat.py +3 -0
  37. package/src/qingflow_mcp/app_form_validator.py +69 -17
  38. package/src/qingflow_mcp/backend_client.py +33 -8
  39. package/src/qingflow_mcp/builder_facade/app_form.py +497 -100
  40. package/src/qingflow_mcp/builder_facade/schemas/app_form/manifest.json +1 -0
  41. package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/complete.json +64 -30
  42. package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/fields/address.json +4 -8
  43. package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/fields/amount.json +4 -8
  44. package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/fields/attachment.json +6 -10
  45. package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/fields/code_block.json +6 -10
  46. package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/fields/data_relation.json +11 -18
  47. package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/fields/date.json +4 -8
  48. package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/fields/datetime.json +4 -8
  49. package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/fields/department.json +6 -10
  50. package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/fields/email.json +4 -8
  51. package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/fields/link.json +4 -8
  52. package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/fields/long_text.json +4 -8
  53. package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/fields/member.json +6 -10
  54. package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/fields/multi_select.json +4 -8
  55. package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/fields/number.json +4 -8
  56. package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/fields/phone.json +4 -8
  57. package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/fields/q_linker.json +6 -10
  58. package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/fields/relation.json +46 -15
  59. package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/fields/rich_text.json +146 -0
  60. package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/fields/single_choice.json +4 -8
  61. package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/fields/single_select.json +4 -8
  62. package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/fields/subtable.json +61 -27
  63. package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/fields/text.json +4 -8
  64. package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/formula.json +43 -0
  65. package/src/qingflow_mcp/builder_facade/schemas/app_form/v1alpha1/summary.json +7 -12
  66. package/src/qingflow_mcp/builder_facade/service.py +39 -1
  67. package/src/qingflow_mcp/cli/commands/__init__.py +2 -1
  68. package/src/qingflow_mcp/cli/commands/data.py +144 -0
  69. package/src/qingflow_mcp/cli/commands/record.py +1 -1
  70. package/src/qingflow_mcp/cli/context.py +3 -0
  71. package/src/qingflow_mcp/cli/formatters.py +104 -1
  72. package/src/qingflow_mcp/public_surface.py +5 -0
  73. package/src/qingflow_mcp/response_trim.py +2 -1
  74. package/src/qingflow_mcp/server.py +3 -1
  75. package/src/qingflow_mcp/server_app_user.py +5 -1
  76. package/src/qingflow_mcp/tools/csv_export.py +168 -0
  77. package/src/qingflow_mcp/tools/data_files.py +174 -0
  78. package/src/qingflow_mcp/tools/data_tools.py +313 -0
  79. package/src/qingflow_mcp/tools/record_tools.py +25 -118
  80. package/src/qingflow_mcp/tools/storage_download.py +89 -0
@@ -8,7 +8,10 @@ from typing import Any, TextIO
8
8
  def emit_text_result(result: dict[str, Any], *, hint: str, stream: TextIO) -> None:
9
9
  text = _format_cancelled_result(result)
10
10
  if text is None:
11
- if _is_failed_result(result) and hint != "task_action_execute":
11
+ if hint in _DATA_FORMAT_HINTS:
12
+ # 数据面失败也必须保留 field_errors 与列表型 hints,不能先走通用失败裁剪。
13
+ text = _format_data_result(result)
14
+ elif _is_failed_result(result) and hint != "task_action_execute":
12
15
  text = _format_failed_result(result)
13
16
  else:
14
17
  formatter = _FORMATTERS.get(hint, _format_generic)
@@ -90,6 +93,99 @@ def _format_generic(result: dict[str, Any]) -> str:
90
93
  return "\n".join(lines) + "\n"
91
94
 
92
95
 
96
+ def _format_data_result(result: dict[str, Any]) -> str:
97
+ lines = [f"Status: {result.get('status') or ('success' if result.get('ok') is not False else 'failed')}"]
98
+ if result.get("format") == "csv":
99
+ files = result.get("files") if isinstance(result.get("files"), list) else []
100
+ lines.extend(
101
+ [
102
+ f"Local dir: {result.get('local_dir') or '-'}",
103
+ f"Files: {len(files)}",
104
+ ]
105
+ )
106
+ for item in files:
107
+ if isinstance(item, dict):
108
+ lines.append(
109
+ f"- part {item.get('part')}: {item.get('local_path')} ({item.get('row_count')} rows)"
110
+ )
111
+ lines.extend(
112
+ [
113
+ f"Rows: {result.get('row_count')}",
114
+ f"Complete: {result.get('complete')}",
115
+ f"Truncated: {result.get('truncated')}",
116
+ f"Safe for final conclusion: {result.get('safe_for_final_conclusion')}",
117
+ f"Stopped reason: {result.get('stopped_reason') or '-'}",
118
+ ]
119
+ )
120
+ access = result.get("access") if isinstance(result.get("access"), dict) else {}
121
+ if access:
122
+ sources = access.get("scope_sources") if isinstance(access.get("scope_sources"), list) else []
123
+ lines.append(
124
+ f"Access: level={access.get('level') or '-'} / can_write={access.get('can_write')}"
125
+ f" / scope_sources={','.join(str(item) for item in sources) or '-'}"
126
+ )
127
+
128
+ records = result.get("records") if isinstance(result.get("records"), dict) else {}
129
+ if records:
130
+ items = records.get("items") if isinstance(records.get("items"), list) else []
131
+ lines.append(
132
+ "Pagination: "
133
+ f"page={records.get('page')} / requested_page_size={records.get('requested_page_size')} / "
134
+ f"page_size={records.get('page_size')} / total={records.get('total')} / "
135
+ f"has_more={records.get('has_more')} / returned={len(items)}"
136
+ )
137
+ record = result.get("record") if isinstance(result.get("record"), dict) else {}
138
+ if record:
139
+ lines.append(f"Record ID: {record.get('record_id') or '-'}")
140
+ file_assets = result.get("file_assets")
141
+ if isinstance(file_assets, dict):
142
+ lines.append(f"Attachments: {file_assets.get('status') or '-'}")
143
+ for asset in file_assets.get("items", []):
144
+ if isinstance(asset, dict):
145
+ lines.append(f"- {asset.get('file_name')}: {asset.get('access_status')} / {asset.get('local_path') or '-'}")
146
+ _append_warnings(lines, file_assets.get("warnings"))
147
+ items = result.get("items") if isinstance(result.get("items"), list) else []
148
+ if items:
149
+ lines.append(f"Items: {len(items)}")
150
+
151
+ for key, label in (
152
+ ("error_code", "Error Code"),
153
+ ("message", "Message"),
154
+ ("write_executed", "Write Executed"),
155
+ ("retry_safe", "Retry Safe"),
156
+ ):
157
+ if key in result:
158
+ lines.append(f"{label}: {result.get(key)}")
159
+
160
+ field_errors = result.get("field_errors") if isinstance(result.get("field_errors"), list) else []
161
+ if field_errors:
162
+ lines.append("Field Errors:")
163
+ _append_data_field_errors(lines, field_errors)
164
+ for index, item in enumerate(items):
165
+ if not isinstance(item, dict) or not isinstance(item.get("field_errors"), list):
166
+ continue
167
+ lines.append(f"Item {index} Field Errors:")
168
+ _append_data_field_errors(lines, item["field_errors"])
169
+
170
+ hints = result.get("hints") if isinstance(result.get("hints"), list) else []
171
+ lines.append(f"Hints: {len(hints)}")
172
+ for hint in hints:
173
+ if isinstance(hint, dict):
174
+ lines.append(f"- {hint.get('code') or '-'}: {hint.get('message') or '-'}")
175
+ else:
176
+ lines.append(f"- {hint}")
177
+ return "\n".join(lines) + "\n"
178
+
179
+
180
+ def _append_data_field_errors(lines: list[str], field_errors: list[Any]) -> None:
181
+ for error in field_errors:
182
+ if isinstance(error, dict):
183
+ lines.append(
184
+ f"- {error.get('path') or '-'}: {error.get('message') or '-'}"
185
+ f" / fix_hint={error.get('fix_hint') or '-'}"
186
+ )
187
+
188
+
93
189
  def _format_whoami(result: dict[str, Any]) -> str:
94
190
  lines = [
95
191
  f"User: {result.get('nick_name') or '-'} ({result.get('email') or '-'})",
@@ -1669,4 +1765,11 @@ _FORMATTERS = {
1669
1765
  "builder_summary": _format_builder_summary,
1670
1766
  "file_upload_local": _format_file_upload_local,
1671
1767
  "feedback_submit": _format_feedback_submit,
1768
+ "data_query": _format_data_result,
1769
+ "data_get": _format_data_result,
1770
+ "data_insert": _format_data_result,
1771
+ "data_update": _format_data_result,
1772
+ "data_delete": _format_data_result,
1672
1773
  }
1774
+
1775
+ _DATA_FORMAT_HINTS = {"data_query", "data_get", "data_insert", "data_update", "data_delete"}
@@ -40,6 +40,11 @@ USER_PUBLIC_TOOL_SPECS: tuple[PublicToolSpec, ...] = (
40
40
  PublicToolSpec(USER_DOMAIN, "workspace_select", ("workspace_select",), ("workspace", "select")),
41
41
  PublicToolSpec(USER_DOMAIN, "app_list", ("app_list",), ("app", "list"), cli_show_effective_context=True),
42
42
  PublicToolSpec(USER_DOMAIN, "app_get", ("app_get",), ("app", "get"), cli_show_effective_context=True),
43
+ PublicToolSpec(USER_DOMAIN, "data_query", ("data_query",), ("data", "query"), cli_show_effective_context=True),
44
+ PublicToolSpec(USER_DOMAIN, "data_get", ("data_get",), ("data", "get"), cli_show_effective_context=True),
45
+ PublicToolSpec(USER_DOMAIN, "data_insert", ("data_insert",), ("data", "insert"), cli_show_effective_context=True, cli_context_write=True),
46
+ PublicToolSpec(USER_DOMAIN, "data_update", ("data_update",), ("data", "update"), cli_show_effective_context=True, cli_context_write=True),
47
+ PublicToolSpec(USER_DOMAIN, "data_delete", ("data_delete",), ("data", "delete"), cli_show_effective_context=True, cli_context_write=True),
43
48
  PublicToolSpec(USER_DOMAIN, "portal_list", ("portal_list",), ("portal", "list"), cli_show_effective_context=True),
44
49
  PublicToolSpec(USER_DOMAIN, "portal_get", ("portal_get",), ("portal", "get"), cli_show_effective_context=True),
45
50
  PublicToolSpec(USER_DOMAIN, "view_get", ("view_get",), ("view", "get"), cli_show_effective_context=True),
@@ -51,8 +51,9 @@ COMMON_ERROR_DROP_TOP = {
51
51
 
52
52
  SUCCESS_POLICY_BY_TOOL: dict[str, TransformFn] = {}
53
53
 
54
- #: 这两个校验命令的产出契约明确包含 ok 字段(供阶段 2 SKILL.md 引用),
54
+ #: 这些校验命令的产出契约明确包含 ok 字段(供阶段 2 SKILL.md 引用),
55
55
  #: 需在 success 裁剪时保留 ok(默认 COMMON_SUCCESS_DROP_TOP 会丢弃 ok)。
56
+ #: `builder:app_views_validate`(views 声明式校验)同属校验类工具,ok 同样进产出契约。
56
57
  _KEEP_OK_TOOLS = {
57
58
  "builder:app_form_validate_formula",
58
59
  "builder:app_form_validate_code_block",
@@ -7,6 +7,7 @@ from mcp.server.fastmcp import FastMCP
7
7
  from .backend_client import BackendClient
8
8
  from .session_store import SessionStore
9
9
  from .tools.app_tools import AppTools
10
+ from .tools.data_tools import DataTools
10
11
  from .tools.auth_tools import AuthTools
11
12
  from .tools.code_block_tools import CodeBlockTools
12
13
  from .tools.feedback_tools import FeedbackTools
@@ -148,7 +149,7 @@ Analysis answers must include concrete numbers. When applicable, include percent
148
149
  - `record_delete` deletes by `record_id` or `record_ids`, and requires an accessible system `view_id`; use custom views only to locate records, not as the delete route.
149
150
  - `record_get` is the single-record frontend detail context tool. It returns detail-page visible fields, one-level relation targets, first-page data/workflow logs, associated views/reports, local readable image assets, local downloadable file assets, unavailable context, and `semantic_context`.
150
151
  - Use `record_logs_get` only when the user needs the full visible data/workflow log history for a specific record. It writes JSONL files locally and returns file paths plus completeness metadata; do not expect full log arrays in the response.
151
- - Read record images from `record_get.media_assets.items[].local_path` when `readable_by_agent=true`; read attachments/documents/tables from `record_get.file_assets.items[].local_path` and `extraction.text_path` when present. `record_get` follows the frontend storage cookie redirect path for Qingflow attachments, and remote file URLs should not be treated as directly readable.
152
+ - Download record attachments with `data_get(download_attachments=true, include_schema=true)` for the same app and record; use `file_assets.items[].local_path` only when `access_status=downloaded`. This includes original images and videos. Inspect `file_assets.warnings` for partial failures; do not request remote private URLs or fall back to record/detail tools after a data-plane denial. For specialized detail-page context, read `record_get` rich-text images from `media_assets` and its files from `file_assets` local paths or `extraction.text_path` when present.
152
153
  - `record_get.columns` are focus hints only; they do not project the detail fields. Read facts from top-level `fields[]`.
153
154
  - When readback shape matters after insert or update, prefer `record_get` for human/detail-page context, or `record_list(..., output_profile="normalized")` for batch-shaped normalized rows.
154
155
 
@@ -249,6 +250,7 @@ If the current MCP capability is unsupported, the workflow is awkward, or the us
249
250
  TaskContextTools(sessions, backend).register(server)
250
251
  RoleTools(sessions, backend).register(server)
251
252
  AppTools(sessions, backend).register(server)
253
+ DataTools(sessions, backend).register(server)
252
254
  QingbiReportTools(sessions, backend).register(server)
253
255
  PackageTools(sessions, backend).register(server)
254
256
  NavigationTools(sessions, backend).register(server)
@@ -10,6 +10,7 @@ from .config import DEFAULT_PROFILE
10
10
  from .response_trim import USER_SERVER_METHOD_MAP, trim_error_response, trim_public_response, wrap_trimmed_methods
11
11
  from .session_store import SessionStore
12
12
  from .tools.app_tools import AppTools
13
+ from .tools.data_tools import DataTools
13
14
  from .tools.auth_tools import AuthTools
14
15
  from .tools.code_block_tools import CodeBlockTools
15
16
  from .tools.directory_tools import DirectoryTools
@@ -143,7 +144,7 @@ Analysis answers must include concrete numbers. When applicable, include percent
143
144
  - `record_delete` deletes by `record_id` or `record_ids`, and requires an accessible system `view_id`; use custom views only to locate records, not as the delete route.
144
145
  - `record_get` is the single-record frontend detail context tool. It returns detail-page visible fields, one-level relation targets, first-page data/workflow logs, associated views/reports, local readable image assets, local downloadable file assets, unavailable context, and `semantic_context`.
145
146
  - Use `record_logs_get` only when the user needs the full visible data/workflow log history for a specific record. It writes JSONL files locally and returns file paths plus completeness metadata; do not expect full log arrays in the response.
146
- - Read record images from `record_get.media_assets.items[].local_path` when `readable_by_agent=true`; read attachments/documents/tables from `record_get.file_assets.items[].local_path` and `extraction.text_path` when present. `record_get` follows the frontend storage cookie redirect path for Qingflow attachments, and remote file URLs should not be treated as directly readable.
147
+ - Download record attachments with `data_get(download_attachments=true, include_schema=true)` for the same app and record; use `file_assets.items[].local_path` only when `access_status=downloaded`. This includes original images and videos. Inspect `file_assets.warnings` for partial failures; do not request remote private URLs or fall back to record/detail tools after a data-plane denial. For specialized detail-page context, read `record_get` rich-text images from `media_assets` and its files from `file_assets` local paths or `extraction.text_path` when present.
147
148
  - `record_get.columns` are focus hints only; they do not project the detail fields. Read facts from top-level `fields[]`.
148
149
  - When readback shape matters after insert or update, prefer `record_get` for human/detail-page context, or `record_list(..., output_profile="normalized")` for batch-shaped normalized rows.
149
150
 
@@ -242,6 +243,7 @@ If the current MCP capability is unsupported, the workflow is awkward, or the us
242
243
  backend = BackendClient()
243
244
  auth = wrap_trimmed_methods(AuthTools(sessions, backend), USER_SERVER_METHOD_MAP)
244
245
  apps = wrap_trimmed_methods(AppTools(sessions, backend), USER_SERVER_METHOD_MAP)
246
+ data_tools = wrap_trimmed_methods(DataTools(sessions, backend), USER_SERVER_METHOD_MAP)
245
247
  workspace = wrap_trimmed_methods(WorkspaceTools(sessions, backend), USER_SERVER_METHOD_MAP)
246
248
  file_tools = wrap_trimmed_methods(FileTools(sessions, backend), USER_SERVER_METHOD_MAP)
247
249
  imports = wrap_trimmed_methods(ImportTools(sessions, backend), USER_SERVER_METHOD_MAP)
@@ -395,6 +397,8 @@ If the current MCP capability is unsupported, the workflow is awkward, or the us
395
397
  def app_get(profile: str = DEFAULT_PROFILE, app_key: str = "") -> dict:
396
398
  return apps.app_get(profile=profile, app_key=app_key)
397
399
 
400
+ data_tools.register(server)
401
+
398
402
  @server.tool()
399
403
  def portal_list(profile: str = DEFAULT_PROFILE) -> dict:
400
404
  return resources.portal_list(profile=profile)
@@ -0,0 +1,168 @@
1
+ from __future__ import annotations
2
+
3
+ import csv
4
+ import json
5
+ import time
6
+ from pathlib import Path
7
+ from typing import Any, Callable
8
+
9
+ from ..json_types import JSONObject
10
+
11
+
12
+ PageFetcher = Callable[[int, int], JSONObject]
13
+ RowProjector = Callable[[JSONObject], list[str]]
14
+
15
+
16
+ class CsvPageFailure(RuntimeError):
17
+ """后续分页返回结构化业务错误;已写文件保留为不完整结果。"""
18
+
19
+ def __init__(self, payload: JSONObject) -> None:
20
+ super().__init__(str(payload.get("message") or "分页请求失败"))
21
+ self.payload = payload
22
+
23
+
24
+ class CsvShardWriter:
25
+ """视图无关的 UTF-8 BOM CSV 分片写入器。"""
26
+
27
+ def __init__(self, *, output_dir: Path, file_prefix: str, header: list[str], shard_rows: int) -> None:
28
+ self.output_dir = output_dir
29
+ self.file_prefix = file_prefix
30
+ self.header = header
31
+ self.shard_rows = shard_rows
32
+ self.files: list[JSONObject] = []
33
+ self.row_count = 0
34
+ self._part = 0
35
+ self._current_rows = 0
36
+ self._file: Any | None = None
37
+ self._writer: Any | None = None
38
+ output_dir.mkdir(parents=True, exist_ok=True)
39
+
40
+ def write(self, values: list[str]) -> None:
41
+ if self._current_rows >= self.shard_rows:
42
+ self.close_shard()
43
+ self._ensure_writer().writerow(values)
44
+ self._current_rows += 1
45
+ self.row_count += 1
46
+
47
+ def close_shard(self) -> None:
48
+ if self._file is None:
49
+ return
50
+ path = Path(self._file.name)
51
+ self._file.close()
52
+ self.files.append({"local_path": str(path), "row_count": self._current_rows, "part": len(self.files) + 1})
53
+ self._file = None
54
+ self._writer = None
55
+ self._current_rows = 0
56
+
57
+ def _ensure_writer(self) -> Any:
58
+ if self._writer is not None:
59
+ return self._writer
60
+ self._part += 1
61
+ path = self.output_dir / f"{self.file_prefix}-{self._part:04d}.csv"
62
+ self._file = path.open("w", newline="", encoding="utf-8-sig")
63
+ self._writer = csv.writer(self._file)
64
+ self._writer.writerow(self.header)
65
+ return self._writer
66
+
67
+
68
+ def export_csv_pages(
69
+ *,
70
+ fetch_page: PageFetcher,
71
+ header: list[str],
72
+ project_row: RowProjector,
73
+ output_dir: Path,
74
+ file_prefix: str,
75
+ page_size: int,
76
+ shard_rows: int,
77
+ time_budget_seconds: float,
78
+ ) -> JSONObject:
79
+ """执行与业务权限无关的分页、CSV 分片和时间预算控制。"""
80
+ output_dir.mkdir(parents=True, exist_ok=True)
81
+ deadline = time.monotonic() + time_budget_seconds
82
+ files: list[JSONObject] = []
83
+ row_count = 0
84
+ fetched_pages = 0
85
+ current_page = 1
86
+ file_handle: Any | None = None
87
+ writer: Any | None = None
88
+ current_rows = 0
89
+ part = 0
90
+ has_more = False
91
+ stopped_reason: str | None = None
92
+ page_error: JSONObject | None = None
93
+
94
+ def close_file() -> None:
95
+ nonlocal file_handle, writer, current_rows
96
+ if file_handle is None:
97
+ return
98
+ path = Path(file_handle.name)
99
+ file_handle.close()
100
+ files.append({"local_path": str(path), "row_count": current_rows, "part": len(files) + 1})
101
+ file_handle = None
102
+ writer = None
103
+ current_rows = 0
104
+
105
+ def get_writer() -> Any:
106
+ nonlocal file_handle, writer, part
107
+ if writer is not None:
108
+ return writer
109
+ part += 1
110
+ path = output_dir / f"{file_prefix}-{part:04d}.csv"
111
+ file_handle = path.open("w", newline="", encoding="utf-8-sig")
112
+ writer = csv.writer(file_handle)
113
+ writer.writerow(header)
114
+ return writer
115
+
116
+ try:
117
+ while True:
118
+ page = fetch_page(current_page, page_size)
119
+ records = page.get("records") if isinstance(page, dict) else None
120
+ container = records if isinstance(records, dict) else page
121
+ raw_items = container.get("items", container.get("list", [])) if isinstance(container, dict) else []
122
+ items = raw_items if isinstance(raw_items, list) else []
123
+ has_more = bool(container.get("has_more")) if isinstance(container, dict) else False
124
+ for item in items:
125
+ if not isinstance(item, dict):
126
+ continue
127
+ if current_rows >= shard_rows:
128
+ close_file()
129
+ get_writer().writerow(project_row(item))
130
+ current_rows += 1
131
+ row_count += 1
132
+ fetched_pages += 1
133
+ if not has_more:
134
+ break
135
+ if time.monotonic() >= deadline:
136
+ stopped_reason = "TIME_BUDGET_EXCEEDED"
137
+ break
138
+ current_page += 1
139
+ except CsvPageFailure as exception:
140
+ stopped_reason = "PAGE_FETCH_FAILED"
141
+ page_error = exception.payload
142
+ except Exception:
143
+ # 已落盘分片仍是有效的部分结果;调用方据此禁止形成完整性结论。
144
+ stopped_reason = "PAGE_FETCH_FAILED"
145
+ finally:
146
+ close_file()
147
+
148
+ complete = stopped_reason is None and not has_more
149
+ result: JSONObject = {
150
+ "files": files,
151
+ "row_count": row_count,
152
+ "complete": complete,
153
+ "truncated": not complete,
154
+ "safe_for_final_conclusion": complete,
155
+ "fetched_pages": fetched_pages,
156
+ "stopped_reason": stopped_reason,
157
+ }
158
+ if page_error is not None:
159
+ result["page_error"] = page_error
160
+ return result
161
+
162
+
163
+ def csv_cell(value: Any) -> str:
164
+ if value is None:
165
+ return ""
166
+ if isinstance(value, (dict, list)):
167
+ return json.dumps(value, ensure_ascii=False, separators=(",", ":"))
168
+ return str(value)
@@ -0,0 +1,174 @@
1
+ from __future__ import annotations
2
+
3
+ import mimetypes
4
+ import os
5
+ import re
6
+ import time
7
+ from pathlib import Path
8
+ from typing import Any, Callable
9
+ from urllib.parse import unquote, urlsplit
10
+ from uuid import uuid4
11
+
12
+ from ..backend_client import BackendRequestContext
13
+ from ..config import get_mcp_home
14
+ from ..errors import QingflowApiError
15
+ from ..json_types import JSONObject
16
+ from .storage_download import download_storage_content, requires_storage_cookie, storage_auth_type
17
+
18
+
19
+ DATA_FILE_MAX_FILES = 50
20
+ DATA_FILE_MAX_BYTES = 50 * 1024 * 1024
21
+ DATA_FILE_MAX_TOTAL_BYTES = 200 * 1024 * 1024
22
+ DATA_FILE_TIME_BUDGET_SECONDS = 55.0
23
+
24
+
25
+ def attachment_candidates(record: JSONObject, schema: JSONObject) -> list[JSONObject]:
26
+ candidates: list[JSONObject] = []
27
+
28
+ def walk(values: Any, fields: Any, prefix: str) -> None:
29
+ if not isinstance(values, dict) or not isinstance(fields, list):
30
+ return
31
+ for field in fields:
32
+ if not isinstance(field, dict) or field.get("field_id") is None:
33
+ continue
34
+ key = str(field["field_id"])
35
+ value = values.get(key)
36
+ path = f"{prefix}.{key}"
37
+ if field.get("data_type") == "table" and isinstance(value, list):
38
+ for index, row in enumerate(value):
39
+ walk(row, field.get("children"), f"{path}[{index}]")
40
+ elif field.get("data_type") == "attachment":
41
+ for index, item in enumerate(value if isinstance(value, list) else [value]):
42
+ url = item.get("value") or item.get("url") if isinstance(item, dict) else item
43
+ if not isinstance(url, str) or not url.strip():
44
+ continue
45
+ name = (item.get("other_info") or item.get("otherInfo") or item.get("name")) if isinstance(item, dict) else None
46
+ if not isinstance(name, str) or not name.strip():
47
+ try:
48
+ name = Path(unquote(urlsplit(url).path)).name
49
+ except ValueError:
50
+ name = None
51
+ name = name or f"attachment_{len(candidates) + 1}"
52
+ candidates.append({
53
+ "field_id": field["field_id"], "file_name": name,
54
+ "source_path": f"{path}[{index}]", "source_url": url.strip(),
55
+ })
56
+
57
+ walk(record.get("fields"), schema.get("fields"), "fields")
58
+ return candidates
59
+
60
+
61
+ def _download_url(candidate: JSONObject) -> str:
62
+ url = str(candidate["source_url"])
63
+ parsed = urlsplit(url)
64
+ if parsed.scheme not in {"http", "https"} or not parsed.netloc or parsed.username or parsed.password:
65
+ raise QingflowApiError.config_error("附件没有可直接下载的 HTTP 地址")
66
+ return url
67
+
68
+
69
+ def _matching_attachment(
70
+ candidate: JSONObject, refreshed: list[JSONObject],
71
+ ) -> JSONObject | None:
72
+ # shortcut: 沿用旧链路,同一字段内同名附件取首个;需要区分时再引入稳定文件 ID。
73
+ return next((item for item in refreshed if (
74
+ str(item["field_id"]) == str(candidate["field_id"])
75
+ and str(item["file_name"]).strip() == str(candidate["file_name"]).strip()
76
+ )), None)
77
+
78
+
79
+ def download_data_attachments(
80
+ *, backend: Any, context: BackendRequestContext, record: JSONObject, schema: JSONObject,
81
+ refresh_record: Callable[[], JSONObject],
82
+ ) -> JSONObject:
83
+ candidates = attachment_candidates(record, schema)
84
+ if not candidates:
85
+ return {"status": "none", "local_dir": None, "items": [], "warnings": []}
86
+ custom_home = os.environ.get("QINGFLOW_MCP_DATA_FILES_HOME")
87
+ local_dir = (Path(custom_home).expanduser() if custom_home else get_mcp_home() / "data-files") / uuid4().hex
88
+ items: list[JSONObject] = []
89
+ warnings: list[JSONObject] = []
90
+ prefix_cache: dict[str, str] = {}
91
+ fresh_candidates: list[JSONObject] | None = None
92
+ deadline = time.monotonic() + DATA_FILE_TIME_BUDGET_SECONDS
93
+ total_bytes = 0
94
+
95
+ for index, original in enumerate(candidates):
96
+ item: JSONObject = {**original, "local_path": None}
97
+ items.append(item)
98
+ if index >= DATA_FILE_MAX_FILES or time.monotonic() >= deadline:
99
+ item["access_status"] = "skipped_limit"
100
+ warnings.append({"code": "FILE_ASSET_LIMIT_EXCEEDED", "message": "附件数量或下载时间超过本次预算"})
101
+ continue
102
+ try:
103
+ if fresh_candidates is not None:
104
+ replacement = _matching_attachment(item, fresh_candidates)
105
+ if replacement is None:
106
+ raise QingflowApiError(category="http", message="刷新后附件不在当前可见范围内", http_status=403)
107
+ item["source_url"] = replacement["source_url"]
108
+ url = _download_url(item)
109
+
110
+ def download(source_url: str) -> tuple[bytes, JSONObject]:
111
+ return download_storage_content(
112
+ backend=backend, context=context, source_url=source_url, warnings=warnings,
113
+ environment_prefix_cache=prefix_cache, operation="data_get",
114
+ max_bytes=min(DATA_FILE_MAX_BYTES, DATA_FILE_MAX_TOTAL_BYTES - total_bytes),
115
+ requested_strategy="qingflow_storage_cookie_redirect" if requires_storage_cookie(
116
+ source_url, storage_auth_type(source_url)
117
+ ) else "referer_acl",
118
+ )
119
+
120
+ try:
121
+ content, metadata = download(url)
122
+ except QingflowApiError as exc:
123
+ if exc.http_status not in {401, 403}:
124
+ raise
125
+ if fresh_candidates is None:
126
+ fresh_candidates = []
127
+ fresh = refresh_record()
128
+ if fresh.get("ok") is False:
129
+ raise QingflowApiError(
130
+ category="backend", message=str(fresh.get("message") or "刷新附件访问范围失败"),
131
+ backend_code=fresh.get("backend_code"),
132
+ ) from exc
133
+ if not isinstance(fresh.get("record"), dict) or not isinstance(fresh.get("schema"), dict):
134
+ raise QingflowApiError(category="backend", message="刷新附件需要数据面返回 record 和 schema") from exc
135
+ fresh_candidates = attachment_candidates(fresh["record"], fresh["schema"])
136
+ replacement = _matching_attachment(item, fresh_candidates)
137
+ if replacement is None or replacement["source_url"] == url:
138
+ raise
139
+ url = _download_url(replacement)
140
+ content, metadata = download(url)
141
+ item["source_url"] = url
142
+ warnings.append({
143
+ "code": "FILE_ASSET_STORAGE_URL_REFRESHED", "source_path": item["source_path"],
144
+ "message": "已通过数据面重新读取同一记录并刷新附件链接",
145
+ })
146
+ item.update(metadata)
147
+ item["size_bytes"] = len(content)
148
+ if len(content) > DATA_FILE_MAX_BYTES or total_bytes + len(content) > DATA_FILE_MAX_TOTAL_BYTES:
149
+ item["access_status"] = "too_large"
150
+ warnings.append({"code": "FILE_ASSET_SIZE_LIMIT_EXCEEDED", "message": "附件超过单文件或总大小预算"})
151
+ continue
152
+ suffix = Path(str(item["file_name"])).suffix
153
+ suffix = suffix if re.fullmatch(r"\.[A-Za-z0-9]{1,10}", suffix) else ""
154
+ local_dir.mkdir(parents=True, exist_ok=True)
155
+ local_path = local_dir / f"file_{index + 1:04d}{suffix}"
156
+ local_path.write_bytes(content)
157
+ total_bytes += len(content)
158
+ item.update({
159
+ "local_path": str(local_path), "access_status": "downloaded",
160
+ "mime_type": mimetypes.guess_type(str(item["file_name"]))[0],
161
+ })
162
+ except (QingflowApiError, OSError, ValueError) as exc:
163
+ blocked = isinstance(exc, QingflowApiError) and exc.http_status in {401, 403}
164
+ too_large = isinstance(exc, QingflowApiError) and exc.category == "file_size_limit"
165
+ item["access_status"] = "too_large" if too_large else "blocked_private_url" if blocked else "download_failed"
166
+ warnings.append({
167
+ "code": "FILE_ASSET_SIZE_LIMIT_EXCEEDED" if too_large else "FILE_ASSET_DOWNLOAD_FAILED", "source_path": item["source_path"],
168
+ "message": exc.message if isinstance(exc, QingflowApiError) else str(exc),
169
+ "http_status": exc.http_status if isinstance(exc, QingflowApiError) else None,
170
+ })
171
+ return {
172
+ "status": "ok" if all(item["access_status"] == "downloaded" for item in items) else "partial",
173
+ "local_dir": str(local_dir) if local_dir.exists() else None, "items": items, "warnings": warnings,
174
+ }