@josephyan/qingflow-app-builder-mcp 0.2.0-beta.2 → 0.2.0-beta.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -2
- package/npm/lib/runtime.mjs +37 -0
- package/npm/scripts/postinstall.mjs +5 -1
- package/package.json +3 -2
- package/pyproject.toml +1 -1
- package/skills/qingflow-app-builder/SKILL.md +225 -0
- package/skills/qingflow-app-builder/agents/openai.yaml +4 -0
- package/skills/qingflow-app-builder/references/create-app.md +148 -0
- package/skills/qingflow-app-builder/references/environments.md +63 -0
- package/skills/qingflow-app-builder/references/flow-actors-and-permissions.md +124 -0
- package/skills/qingflow-app-builder/references/gotchas.md +64 -0
- package/skills/qingflow-app-builder/references/solution-playbooks.md +58 -0
- package/skills/qingflow-app-builder/references/tool-selection.md +96 -0
- package/skills/qingflow-app-builder/references/update-flow.md +233 -0
- package/skills/qingflow-app-builder/references/update-layout.md +91 -0
- package/skills/qingflow-app-builder/references/update-schema.md +90 -0
- package/skills/qingflow-app-builder/references/update-views.md +184 -0
- package/src/qingflow_mcp/__init__.py +1 -1
- package/src/qingflow_mcp/builder_facade/models.py +294 -1
- package/src/qingflow_mcp/builder_facade/service.py +2727 -235
- package/src/qingflow_mcp/server.py +2 -0
- package/src/qingflow_mcp/server_app_builder.py +80 -4
- package/src/qingflow_mcp/server_app_user.py +3 -2
- package/src/qingflow_mcp/solution/compiler/form_compiler.py +1 -1
- package/src/qingflow_mcp/solution/compiler/workflow_compiler.py +21 -2
- package/src/qingflow_mcp/solution/executor.py +34 -7
- package/src/qingflow_mcp/tools/ai_builder_tools.py +1038 -30
- package/src/qingflow_mcp/tools/app_tools.py +1 -2
- package/src/qingflow_mcp/tools/record_tools.py +1249 -767
- package/src/qingflow_mcp/tools/workflow_tools.py +78 -4
|
@@ -2,6 +2,7 @@ from __future__ import annotations
|
|
|
2
2
|
|
|
3
3
|
import json
|
|
4
4
|
import re
|
|
5
|
+
import time
|
|
5
6
|
from dataclasses import dataclass
|
|
6
7
|
from datetime import UTC, datetime
|
|
7
8
|
from typing import cast
|
|
@@ -16,14 +17,17 @@ from .base import ToolBase
|
|
|
16
17
|
|
|
17
18
|
|
|
18
19
|
DEFAULT_QUERY_PAGE_SIZE = 50
|
|
20
|
+
DEFAULT_LIST_PAGE_SIZE = 200
|
|
21
|
+
DEFAULT_ANALYSIS_PAGE_SIZE = 1000
|
|
19
22
|
DEFAULT_SCAN_MAX_PAGES = 10
|
|
23
|
+
DEFAULT_ANALYSIS_SCAN_MAX_PAGES = 100
|
|
24
|
+
DEFAULT_ANALYSIS_AUTO_EXPAND_PAGE_CAP = 100
|
|
20
25
|
DEFAULT_ROW_LIMIT = 200
|
|
21
26
|
DEFAULT_OUTPUT_PROFILE = "compact"
|
|
22
27
|
MAX_LIST_COLUMN_LIMIT = 20
|
|
23
28
|
MAX_RECORD_COLUMN_LIMIT = 20
|
|
24
29
|
MAX_SUMMARY_PREVIEW_COLUMN_LIMIT = 6
|
|
25
30
|
BACKEND_LIST_SEARCH_FIELD_LIMIT = 10
|
|
26
|
-
SUPPORTED_QUERY_TOOLS = {"record_query", "record_aggregate", "record_get"}
|
|
27
31
|
MEMBER_QUE_TYPES = {5}
|
|
28
32
|
DEPARTMENT_QUE_TYPES = {22}
|
|
29
33
|
DATE_QUE_TYPES = {4}
|
|
@@ -136,36 +140,55 @@ class RecordTools(ToolBase):
|
|
|
136
140
|
|
|
137
141
|
def register(self, mcp: FastMCP) -> None:
|
|
138
142
|
@mcp.tool()
|
|
139
|
-
def
|
|
143
|
+
def record_schema_get(
|
|
140
144
|
profile: str = DEFAULT_PROFILE,
|
|
141
145
|
app_key: str = "",
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
fuzzy: bool = True,
|
|
146
|
+
view_key: str | None = None,
|
|
147
|
+
view_name: str | None = None,
|
|
148
|
+
output_profile: str = "normal",
|
|
146
149
|
) -> JSONObject:
|
|
147
|
-
return self.
|
|
150
|
+
return self.record_schema_get(
|
|
148
151
|
profile=profile,
|
|
149
152
|
app_key=app_key,
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
fuzzy=fuzzy,
|
|
153
|
+
view_key=view_key,
|
|
154
|
+
view_name=view_name,
|
|
155
|
+
output_profile=output_profile,
|
|
154
156
|
)
|
|
155
157
|
|
|
156
158
|
@mcp.tool(
|
|
157
159
|
description=(
|
|
158
|
-
"
|
|
159
|
-
"
|
|
160
|
+
"Run schema-first analytics on a Qingflow app using a restricted DSL. "
|
|
161
|
+
"Use record_schema_get first, then let the model build a DSL with field_id references only. "
|
|
162
|
+
"dimensions=[] means whole-table summary; dimensions!=[] means grouped analysis. "
|
|
163
|
+
"This route hides paging and scan-budget controls from callers."
|
|
160
164
|
)
|
|
161
165
|
)
|
|
162
|
-
def
|
|
166
|
+
def record_analyze(
|
|
163
167
|
profile: str = DEFAULT_PROFILE,
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
168
|
+
app_key: str = "",
|
|
169
|
+
dimensions: list[JSONObject] | None = None,
|
|
170
|
+
metrics: list[JSONObject] | None = None,
|
|
171
|
+
filters: list[JSONObject] | None = None,
|
|
172
|
+
sort: list[JSONObject] | None = None,
|
|
173
|
+
limit: int = 50,
|
|
174
|
+
strict_full: bool = True,
|
|
175
|
+
view_key: str | None = None,
|
|
176
|
+
view_name: str | None = None,
|
|
177
|
+
output_profile: str = "normal",
|
|
167
178
|
) -> JSONObject:
|
|
168
|
-
return self.
|
|
179
|
+
return self.record_analyze(
|
|
180
|
+
profile=profile,
|
|
181
|
+
app_key=app_key,
|
|
182
|
+
dimensions=dimensions or [],
|
|
183
|
+
metrics=metrics or [],
|
|
184
|
+
filters=filters or [],
|
|
185
|
+
sort=sort or [],
|
|
186
|
+
limit=limit,
|
|
187
|
+
strict_full=strict_full,
|
|
188
|
+
view_key=view_key,
|
|
189
|
+
view_name=view_name,
|
|
190
|
+
output_profile=output_profile,
|
|
191
|
+
)
|
|
169
192
|
|
|
170
193
|
@mcp.tool(
|
|
171
194
|
description=(
|
|
@@ -194,9 +217,10 @@ class RecordTools(ToolBase):
|
|
|
194
217
|
|
|
195
218
|
@mcp.tool(
|
|
196
219
|
description=(
|
|
197
|
-
"Unified read entry for record list / record detail
|
|
198
|
-
"
|
|
199
|
-
"
|
|
220
|
+
"Unified read entry for record list / record detail. "
|
|
221
|
+
"Use query_mode=auto to route: apply_id -> record, otherwise -> list. "
|
|
222
|
+
"query_mode only supports auto, list, or record. "
|
|
223
|
+
"For statistical analysis use record_schema_get and record_analyze."
|
|
200
224
|
)
|
|
201
225
|
)
|
|
202
226
|
def record_query(
|
|
@@ -205,94 +229,38 @@ class RecordTools(ToolBase):
|
|
|
205
229
|
app_key: str = "",
|
|
206
230
|
apply_id: int | None = None,
|
|
207
231
|
page_num: int = 1,
|
|
208
|
-
page_size: int = DEFAULT_QUERY_PAGE_SIZE,
|
|
209
|
-
requested_pages: int = 1,
|
|
210
|
-
scan_max_pages: int = DEFAULT_SCAN_MAX_PAGES,
|
|
211
232
|
query_key: str | None = None,
|
|
212
233
|
filters: list[JSONObject] | None = None,
|
|
213
234
|
sorts: list[JSONObject] | None = None,
|
|
214
235
|
max_rows: int = DEFAULT_ROW_LIMIT,
|
|
215
236
|
max_columns: int | None = None,
|
|
216
237
|
select_columns: list[str | int] | None = None,
|
|
217
|
-
amount_column: str | int | None = None,
|
|
218
|
-
time_range: JSONObject | None = None,
|
|
219
|
-
stat_policy: JSONObject | None = None,
|
|
220
|
-
strict_full: bool = False,
|
|
221
238
|
output_profile: str = DEFAULT_OUTPUT_PROFILE,
|
|
222
239
|
list_type: int = DEFAULT_RECORD_LIST_TYPE,
|
|
223
240
|
view_key: str | None = None,
|
|
224
241
|
view_name: str | None = None,
|
|
225
242
|
) -> JSONObject:
|
|
243
|
+
paging = _fixed_list_scan_policy()
|
|
226
244
|
return self.record_query(
|
|
227
245
|
profile=profile,
|
|
228
246
|
query_mode=query_mode,
|
|
229
247
|
app_key=app_key,
|
|
230
248
|
apply_id=apply_id,
|
|
231
249
|
page_num=page_num,
|
|
232
|
-
page_size=page_size,
|
|
233
|
-
requested_pages=requested_pages,
|
|
234
|
-
scan_max_pages=scan_max_pages,
|
|
250
|
+
page_size=int(paging["page_size"]),
|
|
251
|
+
requested_pages=int(paging["requested_pages"]),
|
|
252
|
+
scan_max_pages=int(paging["scan_max_pages"]),
|
|
253
|
+
auto_expand_pages=bool(paging["auto_expand_pages"]),
|
|
235
254
|
query_key=query_key,
|
|
236
255
|
filters=filters or [],
|
|
237
256
|
sorts=sorts or [],
|
|
238
257
|
max_rows=max_rows,
|
|
239
258
|
max_columns=max_columns,
|
|
240
259
|
select_columns=select_columns or [],
|
|
241
|
-
amount_column=
|
|
242
|
-
time_range=
|
|
243
|
-
stat_policy=
|
|
244
|
-
strict_full=
|
|
245
|
-
output_profile=output_profile,
|
|
246
|
-
list_type=list_type,
|
|
247
|
-
view_key=view_key,
|
|
248
|
-
view_name=view_name,
|
|
249
|
-
)
|
|
250
|
-
|
|
251
|
-
@mcp.tool(
|
|
252
|
-
description=(
|
|
253
|
-
"Grouped record analysis endpoint. Aggregates record counts and numeric metrics by selected dimensions. "
|
|
254
|
-
"Use strict_full=true when the result will be used as a final conclusion."
|
|
255
|
-
)
|
|
256
|
-
)
|
|
257
|
-
def record_aggregate(
|
|
258
|
-
profile: str = DEFAULT_PROFILE,
|
|
259
|
-
app_key: str = "",
|
|
260
|
-
group_by: list[str | int] | None = None,
|
|
261
|
-
amount_column: str | int | None = None,
|
|
262
|
-
metrics: list[str] | None = None,
|
|
263
|
-
page_num: int = 1,
|
|
264
|
-
page_size: int = DEFAULT_QUERY_PAGE_SIZE,
|
|
265
|
-
requested_pages: int = 1,
|
|
266
|
-
scan_max_pages: int = DEFAULT_SCAN_MAX_PAGES,
|
|
267
|
-
query_key: str | None = None,
|
|
268
|
-
filters: list[JSONObject] | None = None,
|
|
269
|
-
sorts: list[JSONObject] | None = None,
|
|
270
|
-
time_range: JSONObject | None = None,
|
|
271
|
-
time_bucket: str | None = None,
|
|
272
|
-
max_groups: int = 200,
|
|
273
|
-
strict_full: bool = False,
|
|
274
|
-
output_profile: str = DEFAULT_OUTPUT_PROFILE,
|
|
275
|
-
list_type: int = DEFAULT_RECORD_LIST_TYPE,
|
|
276
|
-
view_key: str | None = None,
|
|
277
|
-
view_name: str | None = None,
|
|
278
|
-
) -> JSONObject:
|
|
279
|
-
return self.record_aggregate(
|
|
280
|
-
profile=profile,
|
|
281
|
-
app_key=app_key,
|
|
282
|
-
group_by=group_by or [],
|
|
283
|
-
amount_column=amount_column,
|
|
284
|
-
metrics=metrics or [],
|
|
285
|
-
page_num=page_num,
|
|
286
|
-
page_size=page_size,
|
|
287
|
-
requested_pages=requested_pages,
|
|
288
|
-
scan_max_pages=scan_max_pages,
|
|
289
|
-
query_key=query_key,
|
|
290
|
-
filters=filters or [],
|
|
291
|
-
sorts=sorts or [],
|
|
292
|
-
time_range=time_range or {},
|
|
293
|
-
time_bucket=time_bucket,
|
|
294
|
-
max_groups=max_groups,
|
|
295
|
-
strict_full=strict_full,
|
|
260
|
+
amount_column=None,
|
|
261
|
+
time_range={},
|
|
262
|
+
stat_policy={},
|
|
263
|
+
strict_full=False,
|
|
296
264
|
output_profile=output_profile,
|
|
297
265
|
list_type=list_type,
|
|
298
266
|
view_key=view_key,
|
|
@@ -373,95 +341,911 @@ class RecordTools(ToolBase):
|
|
|
373
341
|
) -> JSONObject:
|
|
374
342
|
return self.record_delete(profile=profile, app_key=app_key, apply_id=apply_id, list_type=list_type)
|
|
375
343
|
|
|
376
|
-
def
|
|
344
|
+
def record_schema_get(
|
|
377
345
|
self,
|
|
378
346
|
*,
|
|
379
347
|
profile: str,
|
|
380
348
|
app_key: str,
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
fuzzy: bool,
|
|
349
|
+
view_key: str | None,
|
|
350
|
+
view_name: str | None,
|
|
351
|
+
output_profile: str,
|
|
385
352
|
) -> JSONObject:
|
|
386
353
|
if not app_key:
|
|
387
354
|
raise_tool_error(QingflowApiError.config_error("app_key is required"))
|
|
388
|
-
requested = [item for item in (queries or []) if item is not None]
|
|
389
|
-
if query is not None:
|
|
390
|
-
requested = [query]
|
|
391
|
-
if not requested:
|
|
392
|
-
raise_tool_error(QingflowApiError.config_error("query or queries is required"))
|
|
393
355
|
|
|
394
356
|
def runner(session_profile, context):
|
|
395
357
|
index = self._get_field_index(profile, context, app_key, force_refresh=False)
|
|
396
|
-
|
|
397
|
-
for
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
358
|
+
view_selection = self._resolve_view_selection(profile, context, app_key, view_key=view_key, view_name=view_name)
|
|
359
|
+
fields = [self._schema_field_payload(field) for field in index.by_id.values()]
|
|
360
|
+
suggested_dimensions = [
|
|
361
|
+
{"field_id": item["field_id"], "title": item["title"]}
|
|
362
|
+
for item in fields
|
|
363
|
+
if bool(cast(JSONObject, item["role_hints"]).get("dimension_candidate"))
|
|
364
|
+
]
|
|
365
|
+
suggested_metrics = [
|
|
366
|
+
{"field_id": item["field_id"], "title": item["title"]}
|
|
367
|
+
for item in fields
|
|
368
|
+
if bool(cast(JSONObject, item["role_hints"]).get("metric_candidate"))
|
|
369
|
+
]
|
|
370
|
+
suggested_time_fields = [
|
|
371
|
+
{"field_id": item["field_id"], "title": item["title"]}
|
|
372
|
+
for item in fields
|
|
373
|
+
if bool(cast(JSONObject, item["role_hints"]).get("time_candidate"))
|
|
374
|
+
]
|
|
375
|
+
response: JSONObject = {
|
|
403
376
|
"profile": profile,
|
|
404
377
|
"ws_id": session_profile.selected_ws_id,
|
|
405
378
|
"ok": True,
|
|
379
|
+
"status": "success",
|
|
406
380
|
"request_route": self._request_route_payload(context),
|
|
407
381
|
"data": {
|
|
408
382
|
"app_key": app_key,
|
|
409
|
-
"
|
|
410
|
-
"
|
|
383
|
+
"view_resolution": _view_selection_payload(view_selection),
|
|
384
|
+
"fields": fields,
|
|
385
|
+
"suggested_dimensions": suggested_dimensions,
|
|
386
|
+
"suggested_metrics": suggested_metrics,
|
|
387
|
+
"suggested_time_fields": suggested_time_fields,
|
|
411
388
|
},
|
|
412
389
|
}
|
|
390
|
+
if output_profile == "verbose":
|
|
391
|
+
response["data"]["field_count"] = len(fields)
|
|
392
|
+
return response
|
|
413
393
|
|
|
414
394
|
return self._run_record_tool(profile, runner)
|
|
415
395
|
|
|
416
|
-
def
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
396
|
+
def record_analyze(
|
|
397
|
+
self,
|
|
398
|
+
*,
|
|
399
|
+
profile: str,
|
|
400
|
+
app_key: str,
|
|
401
|
+
dimensions: list[JSONObject],
|
|
402
|
+
metrics: list[JSONObject],
|
|
403
|
+
filters: list[JSONObject],
|
|
404
|
+
sort: list[JSONObject],
|
|
405
|
+
limit: int,
|
|
406
|
+
strict_full: bool,
|
|
407
|
+
view_key: str | None,
|
|
408
|
+
view_name: str | None,
|
|
409
|
+
output_profile: str,
|
|
410
|
+
) -> JSONObject:
|
|
411
|
+
if not app_key:
|
|
412
|
+
raise_tool_error(QingflowApiError.config_error("app_key is required"))
|
|
413
|
+
if limit <= 0:
|
|
414
|
+
raise_tool_error(QingflowApiError.config_error("limit must be positive"))
|
|
421
415
|
|
|
422
416
|
def runner(session_profile, context):
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
view_key=_normalize_optional_text(normalized.get("view_key")),
|
|
434
|
-
view_name=_normalize_optional_text(normalized.get("view_name")),
|
|
417
|
+
index = self._get_field_index(profile, context, app_key, force_refresh=False)
|
|
418
|
+
view_selection = self._resolve_view_selection(profile, context, app_key, view_key=view_key, view_name=view_name)
|
|
419
|
+
compiled_dimensions = self._compile_analyze_dimensions(index, dimensions)
|
|
420
|
+
compiled_metrics = self._compile_analyze_metrics(index, metrics)
|
|
421
|
+
duplicate_aliases = {str(item["alias"]) for item in compiled_dimensions} & {str(item["alias"]) for item in compiled_metrics}
|
|
422
|
+
if duplicate_aliases:
|
|
423
|
+
raise RecordInputError(
|
|
424
|
+
message=f"dimensions and metrics cannot share aliases: {sorted(duplicate_aliases)}",
|
|
425
|
+
error_code="DUPLICATE_ANALYZE_ALIAS",
|
|
426
|
+
fix_hint="Use distinct aliases across dimensions and metrics.",
|
|
435
427
|
)
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
428
|
+
compiled_filters = self._compile_analyze_filters(index, filters)
|
|
429
|
+
compiled_sort = self._compile_analyze_sort(sort, compiled_dimensions, compiled_metrics)
|
|
430
|
+
return self._execute_record_analyze(
|
|
431
|
+
profile=profile,
|
|
432
|
+
session_profile=session_profile,
|
|
433
|
+
context=context,
|
|
434
|
+
app_key=app_key,
|
|
435
|
+
index=index,
|
|
436
|
+
view_selection=view_selection,
|
|
437
|
+
dimensions=compiled_dimensions,
|
|
438
|
+
metrics=compiled_metrics,
|
|
439
|
+
filters=compiled_filters,
|
|
440
|
+
sort=compiled_sort,
|
|
441
|
+
limit=limit,
|
|
442
|
+
strict_full=strict_full,
|
|
443
|
+
output_profile=output_profile,
|
|
444
|
+
)
|
|
445
|
+
|
|
446
|
+
return self._run_record_tool(profile, runner)
|
|
447
|
+
|
|
448
|
+
def _schema_field_payload(self, field: FormField) -> JSONObject:
|
|
449
|
+
return {
|
|
450
|
+
"field_id": field.que_id,
|
|
451
|
+
"title": field.que_title,
|
|
452
|
+
"que_type": field.que_type,
|
|
453
|
+
"system": field.system,
|
|
454
|
+
"readonly": field.readonly,
|
|
455
|
+
"options": field.options,
|
|
456
|
+
"aliases": field.aliases,
|
|
457
|
+
"role_hints": self._schema_role_hints(field),
|
|
458
|
+
}
|
|
459
|
+
|
|
460
|
+
def _schema_role_hints(self, field: FormField) -> JSONObject:
|
|
461
|
+
field_family = self._schema_field_family(field)
|
|
462
|
+
time_candidate = field.que_type in DATE_QUE_TYPES
|
|
463
|
+
identifier_like = self._schema_is_identifier_like(field, field_family=field_family)
|
|
464
|
+
metric_candidate = bool(field.que_type == 8 and not field.system and not field.readonly and not identifier_like)
|
|
465
|
+
dimension_candidate = bool(
|
|
466
|
+
field.que_type not in ATTACHMENT_QUE_TYPES | RELATION_QUE_TYPES | SUBTABLE_QUE_TYPES | VERIFY_UNSUPPORTED_WRITE_QUE_TYPES
|
|
467
|
+
and not field.system
|
|
468
|
+
)
|
|
469
|
+
return {
|
|
470
|
+
"dimension_candidate": dimension_candidate,
|
|
471
|
+
"metric_candidate": metric_candidate,
|
|
472
|
+
"time_candidate": time_candidate,
|
|
473
|
+
"field_family": field_family,
|
|
474
|
+
"supported_metric_ops": self._schema_supported_metric_ops(field, field_family=field_family),
|
|
475
|
+
"semantic_hints": self._schema_semantic_hint(field, field_family=field_family),
|
|
476
|
+
}
|
|
477
|
+
|
|
478
|
+
def _schema_field_family(self, field: FormField) -> str:
|
|
479
|
+
if self._schema_is_identifier_like(field):
|
|
480
|
+
return "text"
|
|
481
|
+
que_type = field.que_type
|
|
482
|
+
if que_type == 8:
|
|
483
|
+
return "number"
|
|
484
|
+
if que_type in DATE_QUE_TYPES:
|
|
485
|
+
return "date"
|
|
486
|
+
if que_type in SINGLE_SELECT_QUE_TYPES | MULTI_SELECT_QUE_TYPES:
|
|
487
|
+
return "category"
|
|
488
|
+
if que_type in MEMBER_QUE_TYPES:
|
|
489
|
+
return "member"
|
|
490
|
+
if que_type in DEPARTMENT_QUE_TYPES:
|
|
491
|
+
return "department"
|
|
492
|
+
if que_type in BOOLEAN_QUE_TYPES:
|
|
493
|
+
return "boolean"
|
|
494
|
+
if que_type in ATTACHMENT_QUE_TYPES | RELATION_QUE_TYPES | SUBTABLE_QUE_TYPES:
|
|
495
|
+
return "unknown"
|
|
496
|
+
if que_type is None:
|
|
497
|
+
return "unknown"
|
|
498
|
+
return "text"
|
|
499
|
+
|
|
500
|
+
def _schema_is_identifier_like(self, field: FormField, *, field_family: str | None = None) -> bool:
|
|
501
|
+
normalized_title = _normalize_field_lookup_key(field.que_title)
|
|
502
|
+
if field.que_id == 0:
|
|
503
|
+
return True
|
|
504
|
+
if any(
|
|
505
|
+
token in normalized_title for token in ("编号", "单号", "流水号", "编码", "序号", "uid", "id", "code")
|
|
506
|
+
):
|
|
507
|
+
return True
|
|
508
|
+
return False
|
|
509
|
+
|
|
510
|
+
def _schema_supported_metric_ops(self, field: FormField, *, field_family: str) -> list[str]:
|
|
511
|
+
if field.que_type in ATTACHMENT_QUE_TYPES | RELATION_QUE_TYPES | SUBTABLE_QUE_TYPES:
|
|
512
|
+
return []
|
|
513
|
+
if self._schema_is_identifier_like(field, field_family=field_family):
|
|
514
|
+
return ["distinct_count"]
|
|
515
|
+
if field_family == "number":
|
|
516
|
+
return ["sum", "avg", "min", "max", "distinct_count"]
|
|
517
|
+
if field_family in {"date", "category", "member", "department", "text", "boolean", "unknown"}:
|
|
518
|
+
return ["distinct_count"]
|
|
519
|
+
return []
|
|
520
|
+
|
|
521
|
+
def _schema_semantic_hint(self, field: FormField, *, field_family: str) -> str:
|
|
522
|
+
if self._schema_is_identifier_like(field, field_family=field_family):
|
|
523
|
+
return "unknown"
|
|
524
|
+
if field_family != "number":
|
|
525
|
+
return "unknown"
|
|
526
|
+
normalized_title = _normalize_field_lookup_key(field.que_title)
|
|
527
|
+
if any(token in normalized_title for token in ("比例", "比率", "占比", "转化率", "渗透率", "毛利率", "折扣率", "百分比")):
|
|
528
|
+
return "ratio_like"
|
|
529
|
+
if any(token in normalized_title for token in ("金额", "arr", "mrr", "收入", "成本", "价格", "单价", "费用", "预算", "报价", "应收", "实收", "总额", "价税")):
|
|
530
|
+
return "money_like"
|
|
531
|
+
if any(token in normalized_title for token in ("数量", "人数", "单量", "个数", "次数", "件数", "台数", "天数", "笔数", "数")):
|
|
532
|
+
return "quantity_like"
|
|
533
|
+
return "unknown"
|
|
534
|
+
|
|
535
|
+
def _resolve_field_by_id(self, field_id: int | None, index: FieldIndex, *, location: str) -> FormField:
|
|
536
|
+
if field_id is None:
|
|
537
|
+
raise RecordInputError(
|
|
538
|
+
message=f"{location} requires field_id",
|
|
539
|
+
error_code="MISSING_FIELD_ID",
|
|
540
|
+
fix_hint="Use record_schema_get and pass a valid field_id from the schema response.",
|
|
541
|
+
)
|
|
542
|
+
field = index.by_id.get(str(field_id))
|
|
543
|
+
if field is None:
|
|
544
|
+
raise RecordInputError(
|
|
545
|
+
message=f"{location} references unknown field_id '{field_id}'",
|
|
546
|
+
error_code="FIELD_NOT_FOUND",
|
|
547
|
+
fix_hint="Use record_schema_get to confirm the exact field_id before calling record_analyze.",
|
|
548
|
+
details={"location": location, "field_id": field_id},
|
|
549
|
+
)
|
|
550
|
+
return field
|
|
551
|
+
|
|
552
|
+
def _ensure_allowed_analyze_keys(
|
|
553
|
+
self,
|
|
554
|
+
item: JSONObject,
|
|
555
|
+
*,
|
|
556
|
+
location: str,
|
|
557
|
+
allowed_keys: set[str],
|
|
558
|
+
example: str,
|
|
559
|
+
) -> None:
|
|
560
|
+
unexpected_keys = sorted(str(key) for key in item.keys() if str(key) not in allowed_keys)
|
|
561
|
+
if unexpected_keys:
|
|
562
|
+
raise RecordInputError(
|
|
563
|
+
message=f"{location} contains unsupported keys: {unexpected_keys}",
|
|
564
|
+
error_code="UNSUPPORTED_ANALYZE_DSL_KEY",
|
|
565
|
+
fix_hint=f"Use {location} in the documented DSL shape only, for example {example}.",
|
|
566
|
+
details={
|
|
567
|
+
"location": location,
|
|
568
|
+
"unexpected_keys": unexpected_keys,
|
|
569
|
+
"allowed_keys": sorted(allowed_keys),
|
|
461
570
|
},
|
|
571
|
+
)
|
|
572
|
+
|
|
573
|
+
def _validate_analyze_filter_value(
|
|
574
|
+
self,
|
|
575
|
+
*,
|
|
576
|
+
field: FormField,
|
|
577
|
+
op: str,
|
|
578
|
+
value: JSONValue,
|
|
579
|
+
location: str,
|
|
580
|
+
) -> JSONValue:
|
|
581
|
+
if op in {"is_null", "not_null"}:
|
|
582
|
+
if value is not None:
|
|
583
|
+
raise RecordInputError(
|
|
584
|
+
message=f"{location} with op '{op}' must omit value",
|
|
585
|
+
error_code="INVALID_ANALYZE_FILTER_VALUE",
|
|
586
|
+
fix_hint="Remove value when using is_null or not_null.",
|
|
587
|
+
details={"location": location, "op": op, "field": _field_ref_payload(field)},
|
|
588
|
+
)
|
|
589
|
+
return None
|
|
590
|
+
|
|
591
|
+
if op in {"in", "not_in"}:
|
|
592
|
+
if not isinstance(value, list) or not value:
|
|
593
|
+
raise RecordInputError(
|
|
594
|
+
message=f"{location} with op '{op}' requires a non-empty array value",
|
|
595
|
+
error_code="INVALID_ANALYZE_FILTER_VALUE",
|
|
596
|
+
fix_hint="Use an array for in/not_in, for example ['A', 'B'].",
|
|
597
|
+
details={"location": location, "op": op, "field": _field_ref_payload(field), "received_value": value},
|
|
598
|
+
)
|
|
599
|
+
if field.que_type in DATE_QUE_TYPES:
|
|
600
|
+
for idx, item in enumerate(value):
|
|
601
|
+
self._validate_strict_date_filter_value(item, location=f"{location}.value[{idx}]")
|
|
602
|
+
return value
|
|
603
|
+
|
|
604
|
+
if op == "between":
|
|
605
|
+
lower, upper = _coerce_filter_range(value)
|
|
606
|
+
if lower is None and upper is None:
|
|
607
|
+
raise RecordInputError(
|
|
608
|
+
message=f"{location} with op 'between' requires a two-bound range",
|
|
609
|
+
error_code="INVALID_ANALYZE_FILTER_VALUE",
|
|
610
|
+
fix_hint="Use value like ['2026-03-01', '2026-03-31'] or [100, 200].",
|
|
611
|
+
details={"location": location, "op": op, "field": _field_ref_payload(field), "received_value": value},
|
|
612
|
+
)
|
|
613
|
+
if field.que_type == 8:
|
|
614
|
+
lower_amount = _coerce_amount(lower) if lower is not None else None
|
|
615
|
+
upper_amount = _coerce_amount(upper) if upper is not None else None
|
|
616
|
+
if lower is not None and lower_amount is None:
|
|
617
|
+
raise RecordInputError(
|
|
618
|
+
message=f"{location} lower bound is not a valid numeric value",
|
|
619
|
+
error_code="INVALID_ANALYZE_FILTER_VALUE",
|
|
620
|
+
fix_hint="Use numeric bounds when filtering a numeric field.",
|
|
621
|
+
details={"location": location, "bound": "lower", "field": _field_ref_payload(field), "received_value": lower},
|
|
622
|
+
)
|
|
623
|
+
if upper is not None and upper_amount is None:
|
|
624
|
+
raise RecordInputError(
|
|
625
|
+
message=f"{location} upper bound is not a valid numeric value",
|
|
626
|
+
error_code="INVALID_ANALYZE_FILTER_VALUE",
|
|
627
|
+
fix_hint="Use numeric bounds when filtering a numeric field.",
|
|
628
|
+
details={"location": location, "bound": "upper", "field": _field_ref_payload(field), "received_value": upper},
|
|
629
|
+
)
|
|
630
|
+
if lower_amount is not None and upper_amount is not None and lower_amount > upper_amount:
|
|
631
|
+
raise RecordInputError(
|
|
632
|
+
message=f"{location} lower bound cannot be greater than upper bound",
|
|
633
|
+
error_code="INVALID_ANALYZE_FILTER_RANGE",
|
|
634
|
+
fix_hint="Swap the between bounds so the lower value comes first.",
|
|
635
|
+
details={"location": location, "field": _field_ref_payload(field), "received_value": value},
|
|
636
|
+
)
|
|
637
|
+
return [lower, upper]
|
|
638
|
+
if field.que_type in DATE_QUE_TYPES:
|
|
639
|
+
lower_dt = self._validate_strict_date_filter_value(lower, location=f"{location}.value[0]") if lower is not None else None
|
|
640
|
+
upper_dt = self._validate_strict_date_filter_value(upper, location=f"{location}.value[1]") if upper is not None else None
|
|
641
|
+
if lower_dt is not None and upper_dt is not None and lower_dt > upper_dt:
|
|
642
|
+
raise RecordInputError(
|
|
643
|
+
message=f"{location} lower bound cannot be later than upper bound",
|
|
644
|
+
error_code="INVALID_ANALYZE_FILTER_RANGE",
|
|
645
|
+
fix_hint="Swap the date bounds so the earlier date comes first.",
|
|
646
|
+
details={"location": location, "field": _field_ref_payload(field), "received_value": value},
|
|
647
|
+
)
|
|
648
|
+
return [lower, upper]
|
|
649
|
+
raise RecordInputError(
|
|
650
|
+
message=f"{location} with op 'between' requires a numeric or date/time field",
|
|
651
|
+
error_code="INVALID_ANALYZE_FILTER_FIELD_TYPE",
|
|
652
|
+
fix_hint="Use between only on numeric or date/time fields.",
|
|
653
|
+
details={"location": location, "field": _field_ref_payload(field), "op": op},
|
|
654
|
+
)
|
|
655
|
+
|
|
656
|
+
if op in {"gt", "gte", "lt", "lte"}:
|
|
657
|
+
if value is None or isinstance(value, (list, dict)):
|
|
658
|
+
raise RecordInputError(
|
|
659
|
+
message=f"{location} with op '{op}' requires a single scalar value",
|
|
660
|
+
error_code="INVALID_ANALYZE_FILTER_VALUE",
|
|
661
|
+
fix_hint="Use a single number or date string for comparison filters.",
|
|
662
|
+
details={"location": location, "op": op, "field": _field_ref_payload(field), "received_value": value},
|
|
663
|
+
)
|
|
664
|
+
if field.que_type == 8:
|
|
665
|
+
if _coerce_amount(value) is None:
|
|
666
|
+
raise RecordInputError(
|
|
667
|
+
message=f"{location} requires a numeric comparison value",
|
|
668
|
+
error_code="INVALID_ANALYZE_FILTER_VALUE",
|
|
669
|
+
fix_hint="Use numeric values with gt/gte/lt/lte on numeric fields.",
|
|
670
|
+
details={"location": location, "field": _field_ref_payload(field), "received_value": value},
|
|
671
|
+
)
|
|
672
|
+
return value
|
|
673
|
+
if field.que_type in DATE_QUE_TYPES:
|
|
674
|
+
self._validate_strict_date_filter_value(value, location=f"{location}.value")
|
|
675
|
+
return value
|
|
676
|
+
raise RecordInputError(
|
|
677
|
+
message=f"{location} with op '{op}' requires a numeric or date/time field",
|
|
678
|
+
error_code="INVALID_ANALYZE_FILTER_FIELD_TYPE",
|
|
679
|
+
fix_hint="Use gt/gte/lt/lte only on numeric or date/time fields.",
|
|
680
|
+
details={"location": location, "field": _field_ref_payload(field), "op": op},
|
|
681
|
+
)
|
|
682
|
+
|
|
683
|
+
if value is None:
|
|
684
|
+
raise RecordInputError(
|
|
685
|
+
message=f"{location} with op '{op}' requires value",
|
|
686
|
+
error_code="INVALID_ANALYZE_FILTER_VALUE",
|
|
687
|
+
fix_hint="Provide value for this filter, or use is_null / not_null.",
|
|
688
|
+
details={"location": location, "op": op, "field": _field_ref_payload(field)},
|
|
689
|
+
)
|
|
690
|
+
|
|
691
|
+
if op == "contains" and isinstance(value, (list, dict)):
|
|
692
|
+
raise RecordInputError(
|
|
693
|
+
message=f"{location} with op 'contains' requires a single text value",
|
|
694
|
+
error_code="INVALID_ANALYZE_FILTER_VALUE",
|
|
695
|
+
fix_hint="Use a single string for contains filters.",
|
|
696
|
+
details={"location": location, "field": _field_ref_payload(field), "received_value": value},
|
|
697
|
+
)
|
|
698
|
+
|
|
699
|
+
if field.que_type in DATE_QUE_TYPES and op in {"eq", "neq"}:
|
|
700
|
+
if isinstance(value, list):
|
|
701
|
+
raise RecordInputError(
|
|
702
|
+
message=f"{location} with op '{op}' requires a single date value",
|
|
703
|
+
error_code="INVALID_ANALYZE_FILTER_VALUE",
|
|
704
|
+
fix_hint="Use a single date string for eq/neq on date fields.",
|
|
705
|
+
details={"location": location, "field": _field_ref_payload(field), "received_value": value},
|
|
706
|
+
)
|
|
707
|
+
self._validate_strict_date_filter_value(value, location=f"{location}.value")
|
|
708
|
+
return value
|
|
709
|
+
|
|
710
|
+
def _validate_strict_date_filter_value(self, value: JSONValue, *, location: str) -> datetime:
|
|
711
|
+
text = _normalize_optional_text(value)
|
|
712
|
+
if text is None:
|
|
713
|
+
raise RecordInputError(
|
|
714
|
+
message=f"{location} requires a concrete date or datetime string",
|
|
715
|
+
error_code="INVALID_DATE_FILTER_VALUE",
|
|
716
|
+
fix_hint="Use a valid ISO-like date such as 2026-03-01 or 2026-03-01 00:00:00.",
|
|
717
|
+
details={"location": location, "received_value": value},
|
|
718
|
+
)
|
|
719
|
+
parsed = _parse_datetime_like(text)
|
|
720
|
+
if parsed is None:
|
|
721
|
+
raise RecordInputError(
|
|
722
|
+
message=f"{location} uses an invalid date value '{text}'",
|
|
723
|
+
error_code="INVALID_DATE_FILTER_VALUE",
|
|
724
|
+
fix_hint="Normalize relative time phrases into a real date range, and avoid impossible dates like 2026-02-29.",
|
|
725
|
+
details={"location": location, "received_value": value},
|
|
726
|
+
)
|
|
727
|
+
return parsed
|
|
728
|
+
|
|
729
|
+
def _compile_analyze_dimensions(self, index: FieldIndex, dimensions: list[JSONObject]) -> list[JSONObject]:
|
|
730
|
+
supported_buckets = {None, "day", "week", "month", "quarter", "year"}
|
|
731
|
+
compiled: list[JSONObject] = []
|
|
732
|
+
used_aliases: set[str] = set()
|
|
733
|
+
for idx, item in enumerate(dimensions):
|
|
734
|
+
if not isinstance(item, dict):
|
|
735
|
+
raise RecordInputError(
|
|
736
|
+
message=f"dimensions[{idx}] must be an object",
|
|
737
|
+
error_code="INVALID_ANALYZE_DIMENSION",
|
|
738
|
+
fix_hint="Pass dimensions like {'field_id': 2, 'alias': '状态'}",
|
|
739
|
+
)
|
|
740
|
+
self._ensure_allowed_analyze_keys(
|
|
741
|
+
item,
|
|
742
|
+
location=f"dimensions[{idx}]",
|
|
743
|
+
allowed_keys={"field_id", "fieldId", "alias", "bucket"},
|
|
744
|
+
example="{'field_id': 2, 'alias': '状态', 'bucket': 'month'}",
|
|
745
|
+
)
|
|
746
|
+
field_id = _coerce_count(item.get("field_id", item.get("fieldId")))
|
|
747
|
+
field = self._resolve_field_by_id(field_id, index, location=f"dimensions[{idx}]")
|
|
748
|
+
bucket = _normalize_optional_text(item.get("bucket"))
|
|
749
|
+
if bucket not in supported_buckets:
|
|
750
|
+
raise RecordInputError(
|
|
751
|
+
message=f"dimensions[{idx}] uses unsupported bucket '{bucket}'",
|
|
752
|
+
error_code="UNSUPPORTED_TIME_BUCKET",
|
|
753
|
+
fix_hint="Use one of day/week/month/quarter/year, or omit bucket.",
|
|
754
|
+
)
|
|
755
|
+
if bucket is not None and field.que_type not in DATE_QUE_TYPES:
|
|
756
|
+
raise RecordInputError(
|
|
757
|
+
message=f"dimensions[{idx}] bucket requires a date/time field",
|
|
758
|
+
error_code="INVALID_TIME_BUCKET_FIELD",
|
|
759
|
+
fix_hint="Use bucket only on fields returned in suggested_time_fields by record_schema_get.",
|
|
760
|
+
details={"field": _field_ref_payload(field), "bucket": bucket},
|
|
761
|
+
)
|
|
762
|
+
alias = _normalize_optional_text(item.get("alias")) or field.que_title
|
|
763
|
+
if alias in used_aliases:
|
|
764
|
+
raise RecordInputError(
|
|
765
|
+
message=f"dimensions[{idx}] alias '{alias}' is duplicated",
|
|
766
|
+
error_code="DUPLICATE_ANALYZE_ALIAS",
|
|
767
|
+
fix_hint="Use unique aliases across dimensions and metrics.",
|
|
768
|
+
)
|
|
769
|
+
used_aliases.add(alias)
|
|
770
|
+
compiled.append({"field": field, "alias": alias, "bucket": bucket})
|
|
771
|
+
return compiled
|
|
772
|
+
|
|
773
|
+
def _compile_analyze_metrics(self, index: FieldIndex, metrics: list[JSONObject]) -> list[JSONObject]:
|
|
774
|
+
requested_metrics = metrics or [{"op": "count", "alias": "记录数"}]
|
|
775
|
+
supported_ops = {"count", "sum", "avg", "min", "max", "distinct_count"}
|
|
776
|
+
compiled: list[JSONObject] = []
|
|
777
|
+
used_aliases: set[str] = set()
|
|
778
|
+
for idx, item in enumerate(requested_metrics):
|
|
779
|
+
if not isinstance(item, dict):
|
|
780
|
+
raise RecordInputError(
|
|
781
|
+
message=f"metrics[{idx}] must be an object",
|
|
782
|
+
error_code="INVALID_ANALYZE_METRIC",
|
|
783
|
+
fix_hint="Pass metrics like {'op': 'count', 'alias': '记录数'}",
|
|
784
|
+
)
|
|
785
|
+
self._ensure_allowed_analyze_keys(
|
|
786
|
+
item,
|
|
787
|
+
location=f"metrics[{idx}]",
|
|
788
|
+
allowed_keys={"op", "field_id", "fieldId", "alias"},
|
|
789
|
+
example="{'op': 'sum', 'field_id': 7, 'alias': '总金额'}",
|
|
790
|
+
)
|
|
791
|
+
op = _normalize_optional_text(item.get("op"))
|
|
792
|
+
if op not in supported_ops:
|
|
793
|
+
raise RecordInputError(
|
|
794
|
+
message=f"metrics[{idx}] uses unsupported op '{op}'",
|
|
795
|
+
error_code="UNSUPPORTED_ANALYZE_METRIC",
|
|
796
|
+
fix_hint="Use one of count/sum/avg/min/max/distinct_count.",
|
|
797
|
+
)
|
|
798
|
+
field: FormField | None = None
|
|
799
|
+
if op != "count":
|
|
800
|
+
field_id = _coerce_count(item.get("field_id", item.get("fieldId")))
|
|
801
|
+
field = self._resolve_field_by_id(field_id, index, location=f"metrics[{idx}]")
|
|
802
|
+
if op in {"sum", "avg", "min", "max"} and field.que_type != 8:
|
|
803
|
+
raise RecordInputError(
|
|
804
|
+
message=f"metrics[{idx}] with op '{op}' requires a numeric field",
|
|
805
|
+
error_code="INVALID_METRIC_FIELD_TYPE",
|
|
806
|
+
fix_hint="Use sum/avg/min/max only on numeric fields returned by record_schema_get.",
|
|
807
|
+
details={"location": f"metrics[{idx}]", "field": _field_ref_payload(field), "op": op},
|
|
808
|
+
)
|
|
809
|
+
elif item.get("field_id", item.get("fieldId")) is not None:
|
|
810
|
+
raise RecordInputError(
|
|
811
|
+
message=f"metrics[{idx}] with op 'count' must not include field_id",
|
|
812
|
+
error_code="INVALID_ANALYZE_METRIC",
|
|
813
|
+
fix_hint="Remove field_id from count metrics.",
|
|
814
|
+
details={"location": f"metrics[{idx}]", "op": op},
|
|
815
|
+
)
|
|
816
|
+
alias = _normalize_optional_text(item.get("alias"))
|
|
817
|
+
if alias is None:
|
|
818
|
+
if op == "count":
|
|
819
|
+
alias = "count"
|
|
820
|
+
elif field is not None:
|
|
821
|
+
alias = f"{field.que_title}_{op}"
|
|
822
|
+
else:
|
|
823
|
+
alias = op
|
|
824
|
+
if alias in used_aliases:
|
|
825
|
+
raise RecordInputError(
|
|
826
|
+
message=f"metrics[{idx}] alias '{alias}' is duplicated",
|
|
827
|
+
error_code="DUPLICATE_ANALYZE_ALIAS",
|
|
828
|
+
fix_hint="Use unique aliases across dimensions and metrics.",
|
|
829
|
+
)
|
|
830
|
+
used_aliases.add(alias)
|
|
831
|
+
compiled.append({"op": op, "field": field, "alias": alias})
|
|
832
|
+
return compiled
|
|
833
|
+
|
|
834
|
+
def _compile_analyze_filters(self, index: FieldIndex, filters: list[JSONObject]) -> list[JSONObject]:
|
|
835
|
+
supported_ops = {"eq", "neq", "in", "not_in", "gt", "gte", "lt", "lte", "between", "contains", "is_null", "not_null"}
|
|
836
|
+
compiled: list[JSONObject] = []
|
|
837
|
+
for idx, item in enumerate(filters):
|
|
838
|
+
if not isinstance(item, dict):
|
|
839
|
+
raise RecordInputError(
|
|
840
|
+
message=f"filters[{idx}] must be an object",
|
|
841
|
+
error_code="INVALID_ANALYZE_FILTER",
|
|
842
|
+
fix_hint="Pass filters like {'field_id': 2, 'op': 'eq', 'value': '进行中'}.",
|
|
843
|
+
)
|
|
844
|
+
self._ensure_allowed_analyze_keys(
|
|
845
|
+
item,
|
|
846
|
+
location=f"filters[{idx}]",
|
|
847
|
+
allowed_keys={"field_id", "fieldId", "op", "operator", "value", "values"},
|
|
848
|
+
example="{'field_id': 2, 'op': 'eq', 'value': '进行中'}",
|
|
849
|
+
)
|
|
850
|
+
field_id = _coerce_count(item.get("field_id", item.get("fieldId")))
|
|
851
|
+
op = _normalize_optional_text(item.get("op", item.get("operator"))) or "eq"
|
|
852
|
+
if op not in supported_ops:
|
|
853
|
+
raise RecordInputError(
|
|
854
|
+
message=f"filters[{idx}] uses unsupported op '{op}'",
|
|
855
|
+
error_code="UNSUPPORTED_ANALYZE_FILTER_OP",
|
|
856
|
+
fix_hint="Use one of eq/neq/in/not_in/gt/gte/lt/lte/between/contains/is_null/not_null.",
|
|
857
|
+
)
|
|
858
|
+
field = self._resolve_field_by_id(field_id, index, location=f"filters[{idx}]")
|
|
859
|
+
normalized_value = self._validate_analyze_filter_value(
|
|
860
|
+
field=field,
|
|
861
|
+
op=op,
|
|
862
|
+
value=item.get("value", item.get("values")),
|
|
863
|
+
location=f"filters[{idx}]",
|
|
864
|
+
)
|
|
865
|
+
compiled.append({"field": field, "field_id": field_id, "op": op, "value": normalized_value})
|
|
866
|
+
return compiled
|
|
867
|
+
|
|
868
|
+
def _compile_analyze_sort(self, sort: list[JSONObject], dimensions: list[JSONObject], metrics: list[JSONObject]) -> list[JSONObject]:
|
|
869
|
+
dimension_aliases = {str(item["alias"]) for item in dimensions}
|
|
870
|
+
metric_aliases = {str(item["alias"]) for item in metrics}
|
|
871
|
+
compiled: list[JSONObject] = []
|
|
872
|
+
for idx, item in enumerate(sort):
|
|
873
|
+
if not isinstance(item, dict):
|
|
874
|
+
raise RecordInputError(
|
|
875
|
+
message=f"sort[{idx}] must be an object",
|
|
876
|
+
error_code="INVALID_ANALYZE_SORT",
|
|
877
|
+
fix_hint="Pass sort like {'by': '记录数', 'order': 'desc'}.",
|
|
878
|
+
)
|
|
879
|
+
self._ensure_allowed_analyze_keys(
|
|
880
|
+
item,
|
|
881
|
+
location=f"sort[{idx}]",
|
|
882
|
+
allowed_keys={"by", "order"},
|
|
883
|
+
example="{'by': '记录数', 'order': 'desc'}",
|
|
884
|
+
)
|
|
885
|
+
by = _normalize_optional_text(item.get("by"))
|
|
886
|
+
if by is None:
|
|
887
|
+
raise RecordInputError(
|
|
888
|
+
message=f"sort[{idx}] requires by",
|
|
889
|
+
error_code="MISSING_SORT_KEY",
|
|
890
|
+
fix_hint="Use a dimension alias or metric alias in sort.by.",
|
|
891
|
+
)
|
|
892
|
+
order = (_normalize_optional_text(item.get("order")) or "asc").lower()
|
|
893
|
+
if order not in {"asc", "desc"}:
|
|
894
|
+
raise RecordInputError(
|
|
895
|
+
message=f"sort[{idx}] uses unsupported order '{order}'",
|
|
896
|
+
error_code="INVALID_SORT_ORDER",
|
|
897
|
+
fix_hint="Use asc or desc.",
|
|
898
|
+
)
|
|
899
|
+
if by in dimension_aliases:
|
|
900
|
+
compiled.append({"by": by, "order": order, "kind": "dimension"})
|
|
901
|
+
continue
|
|
902
|
+
if by in metric_aliases:
|
|
903
|
+
compiled.append({"by": by, "order": order, "kind": "metric"})
|
|
904
|
+
continue
|
|
905
|
+
raise RecordInputError(
|
|
906
|
+
message=f"sort[{idx}] references unknown alias '{by}'",
|
|
907
|
+
error_code="UNKNOWN_ANALYZE_SORT_KEY",
|
|
908
|
+
fix_hint="Use a dimension alias or metric alias defined in the same record_analyze request.",
|
|
909
|
+
)
|
|
910
|
+
return compiled
|
|
911
|
+
|
|
912
|
+
def _execute_record_analyze(
|
|
913
|
+
self,
|
|
914
|
+
*,
|
|
915
|
+
profile: str,
|
|
916
|
+
session_profile,
|
|
917
|
+
context, # type: ignore[no-untyped-def]
|
|
918
|
+
app_key: str,
|
|
919
|
+
index: FieldIndex,
|
|
920
|
+
view_selection: ViewSelection | None,
|
|
921
|
+
dimensions: list[JSONObject],
|
|
922
|
+
metrics: list[JSONObject],
|
|
923
|
+
filters: list[JSONObject],
|
|
924
|
+
sort: list[JSONObject],
|
|
925
|
+
limit: int,
|
|
926
|
+
strict_full: bool,
|
|
927
|
+
output_profile: str,
|
|
928
|
+
) -> JSONObject:
|
|
929
|
+
started_at = time.perf_counter()
|
|
930
|
+
analysis_paging = _fixed_analysis_scan_policy()
|
|
931
|
+
page_size = int(analysis_paging["page_size"])
|
|
932
|
+
requested_pages = int(analysis_paging["requested_pages"])
|
|
933
|
+
scan_max_pages = int(analysis_paging["scan_max_pages"])
|
|
934
|
+
auto_expand_pages = bool(analysis_paging["auto_expand_pages"])
|
|
935
|
+
query_id = _query_id()
|
|
936
|
+
pages_to_scan = min(max(requested_pages, 1), max(scan_max_pages, 1))
|
|
937
|
+
current_page = 1
|
|
938
|
+
scanned_pages = 0
|
|
939
|
+
source_pages: list[int] = []
|
|
940
|
+
result_amount: int | None = None
|
|
941
|
+
has_more = False
|
|
942
|
+
dept_member_cache: dict[int, set[int]] = {}
|
|
943
|
+
local_filtering = bool(filters) or bool(view_selection is not None and view_selection.conditions)
|
|
944
|
+
group_stats: dict[tuple[tuple[str, object], ...], JSONObject] = {}
|
|
945
|
+
overall_metrics = self._initialize_metric_states(metrics)
|
|
946
|
+
matched_rows = 0
|
|
947
|
+
scan_control: JSONObject = {
|
|
948
|
+
"requested_pages": max(requested_pages, 1),
|
|
949
|
+
"scan_max_pages": max(scan_max_pages, 1),
|
|
950
|
+
"auto_expand_pages": auto_expand_pages,
|
|
951
|
+
"auto_expand_applied": False,
|
|
952
|
+
"auto_expand_target_pages": pages_to_scan,
|
|
953
|
+
"auto_expand_page_cap": DEFAULT_ANALYSIS_AUTO_EXPAND_PAGE_CAP,
|
|
954
|
+
}
|
|
955
|
+
|
|
956
|
+
while scanned_pages < pages_to_scan:
|
|
957
|
+
page = self._search_page(
|
|
958
|
+
context,
|
|
959
|
+
app_key=app_key,
|
|
960
|
+
page_num=current_page,
|
|
961
|
+
page_size=page_size,
|
|
962
|
+
query_key=None,
|
|
963
|
+
match_rules=[],
|
|
964
|
+
sorts=[],
|
|
965
|
+
search_que_ids=None,
|
|
966
|
+
list_type=DEFAULT_RECORD_LIST_TYPE,
|
|
967
|
+
)
|
|
968
|
+
scanned_pages += 1
|
|
969
|
+
source_pages.append(current_page)
|
|
970
|
+
items = page.get("list") if isinstance(page.get("list"), list) else []
|
|
971
|
+
if result_amount is None:
|
|
972
|
+
result_amount = _effective_total(page, page_size)
|
|
973
|
+
pages_to_scan, scan_control = _compute_scan_limit(
|
|
974
|
+
requested_pages=requested_pages,
|
|
975
|
+
scan_max_pages=scan_max_pages,
|
|
976
|
+
auto_expand_pages=auto_expand_pages,
|
|
977
|
+
page=page,
|
|
978
|
+
page_size=page_size,
|
|
979
|
+
)
|
|
980
|
+
has_more = _page_has_more(page, current_page, page_size, len(items))
|
|
981
|
+
for item in items:
|
|
982
|
+
if not isinstance(item, dict):
|
|
983
|
+
continue
|
|
984
|
+
answers = item.get("answers")
|
|
985
|
+
answer_list = answers if isinstance(answers, list) else []
|
|
986
|
+
if not self._matches_view_selection(
|
|
987
|
+
context,
|
|
988
|
+
answer_list,
|
|
989
|
+
view_selection=view_selection,
|
|
990
|
+
dept_member_cache=dept_member_cache,
|
|
991
|
+
):
|
|
992
|
+
continue
|
|
993
|
+
if not self._matches_analyze_filters(answer_list, filters):
|
|
994
|
+
continue
|
|
995
|
+
matched_rows += 1
|
|
996
|
+
self._apply_metric_states(overall_metrics, metrics, answer_list)
|
|
997
|
+
if not dimensions:
|
|
998
|
+
continue
|
|
999
|
+
group_payload = self._build_analyze_group_payload(answer_list, dimensions)
|
|
1000
|
+
group_key = self._analysis_group_key(group_payload)
|
|
1001
|
+
bucket = group_stats.get(group_key)
|
|
1002
|
+
if bucket is None:
|
|
1003
|
+
bucket = {
|
|
1004
|
+
"dimensions": group_payload,
|
|
1005
|
+
"metrics_state": self._initialize_metric_states(metrics),
|
|
1006
|
+
}
|
|
1007
|
+
group_stats[group_key] = bucket
|
|
1008
|
+
bucket_metrics = cast(dict[str, JSONObject], bucket["metrics_state"])
|
|
1009
|
+
self._apply_metric_states(bucket_metrics, metrics, answer_list)
|
|
1010
|
+
if not has_more:
|
|
1011
|
+
break
|
|
1012
|
+
current_page += 1
|
|
1013
|
+
|
|
1014
|
+
metric_totals = self._render_metric_values(overall_metrics, metrics)
|
|
1015
|
+
if dimensions:
|
|
1016
|
+
all_rows = [
|
|
1017
|
+
{
|
|
1018
|
+
"dimensions": cast(JSONObject, bucket["dimensions"]),
|
|
1019
|
+
"metrics": self._render_metric_values(cast(dict[str, JSONObject], bucket["metrics_state"]), metrics),
|
|
1020
|
+
}
|
|
1021
|
+
for bucket in group_stats.values()
|
|
1022
|
+
]
|
|
1023
|
+
all_rows = self._sort_analyze_rows(all_rows, sort, dimensions, metrics)
|
|
1024
|
+
rows_truncated = len(all_rows) > limit
|
|
1025
|
+
limited_rows = all_rows[:limit]
|
|
1026
|
+
rows = limited_rows
|
|
1027
|
+
rows_returned = len(limited_rows)
|
|
1028
|
+
group_count = len(all_rows)
|
|
1029
|
+
statement_scope = "returned_groups_only" if rows_truncated else "full_population"
|
|
1030
|
+
else:
|
|
1031
|
+
rows_truncated = False
|
|
1032
|
+
rows = [{"dimensions": {}, "metrics": metric_totals}]
|
|
1033
|
+
rows_returned = 1
|
|
1034
|
+
group_count = 1
|
|
1035
|
+
statement_scope = "full_population"
|
|
1036
|
+
raw_scan_complete = not has_more
|
|
1037
|
+
completeness_status = "complete" if raw_scan_complete else "incomplete"
|
|
1038
|
+
reason_code = "LOCAL_VIEW_FILTERING" if local_filtering and raw_scan_complete else ("SOURCE_EXHAUSTED" if raw_scan_complete else "SCAN_LIMIT_HIT")
|
|
1039
|
+
totals = {
|
|
1040
|
+
"scanned_count": matched_rows,
|
|
1041
|
+
"group_count": group_count,
|
|
1042
|
+
"metric_totals": metric_totals,
|
|
1043
|
+
}
|
|
1044
|
+
data: JSONObject = {
|
|
1045
|
+
"query": {
|
|
1046
|
+
"app_key": app_key,
|
|
1047
|
+
"dimensions": [
|
|
1048
|
+
{
|
|
1049
|
+
"field_id": cast(FormField, item["field"]).que_id,
|
|
1050
|
+
"title": cast(FormField, item["field"]).que_title,
|
|
1051
|
+
"alias": item["alias"],
|
|
1052
|
+
"bucket": item["bucket"],
|
|
1053
|
+
}
|
|
1054
|
+
for item in dimensions
|
|
1055
|
+
],
|
|
1056
|
+
"metrics": [
|
|
1057
|
+
{
|
|
1058
|
+
"op": item["op"],
|
|
1059
|
+
"field_id": cast(FormField | None, item["field"]).que_id if item["field"] is not None else None,
|
|
1060
|
+
"title": cast(FormField | None, item["field"]).que_title if item["field"] is not None else None,
|
|
1061
|
+
"alias": item["alias"],
|
|
1062
|
+
}
|
|
1063
|
+
for item in metrics
|
|
1064
|
+
],
|
|
1065
|
+
"filters": [
|
|
1066
|
+
{
|
|
1067
|
+
"field_id": cast(FormField, item["field"]).que_id,
|
|
1068
|
+
"title": cast(FormField, item["field"]).que_title,
|
|
1069
|
+
"op": item["op"],
|
|
1070
|
+
"value": item.get("value"),
|
|
1071
|
+
}
|
|
1072
|
+
for item in filters
|
|
1073
|
+
],
|
|
1074
|
+
"applied_sort": [{"by": item["by"], "order": item["order"]} for item in sort],
|
|
1075
|
+
"view": _view_selection_payload(view_selection),
|
|
1076
|
+
},
|
|
1077
|
+
"rows": rows,
|
|
1078
|
+
"totals": totals,
|
|
1079
|
+
"completeness": {
|
|
1080
|
+
"status": completeness_status,
|
|
1081
|
+
"reason_code": reason_code,
|
|
1082
|
+
"local_filtering_applied": local_filtering,
|
|
1083
|
+
"safe_for_final_conclusion": completeness_status == "complete",
|
|
1084
|
+
},
|
|
1085
|
+
"presentation": {
|
|
1086
|
+
"row_limit": limit,
|
|
1087
|
+
"rows_returned": rows_returned,
|
|
1088
|
+
"rows_truncated": rows_truncated,
|
|
1089
|
+
"statement_scope": statement_scope,
|
|
1090
|
+
},
|
|
1091
|
+
"warnings": self._build_analyze_warnings(local_filtering=local_filtering, rows_truncated=rows_truncated),
|
|
1092
|
+
}
|
|
1093
|
+
response: JSONObject = {
|
|
1094
|
+
"profile": profile,
|
|
1095
|
+
"ws_id": session_profile.selected_ws_id,
|
|
1096
|
+
"ok": True,
|
|
1097
|
+
"status": "success" if raw_scan_complete else "partial",
|
|
1098
|
+
"query_id": query_id,
|
|
1099
|
+
"request_route": self._request_route_payload(context),
|
|
1100
|
+
"data": data,
|
|
1101
|
+
}
|
|
1102
|
+
if strict_full and not raw_scan_complete:
|
|
1103
|
+
response["ok"] = False
|
|
1104
|
+
response["status"] = "error"
|
|
1105
|
+
response["error"] = {
|
|
1106
|
+
"code": "INCOMPLETE_SCAN",
|
|
1107
|
+
"message": "record_analyze could not complete the scan within the fixed internal analysis budget.",
|
|
1108
|
+
"fix_hint": "Narrow the scope with view/filter constraints, or retry after reducing the dataset size.",
|
|
1109
|
+
}
|
|
1110
|
+
if output_profile == "verbose":
|
|
1111
|
+
response["data"]["debug"] = {
|
|
1112
|
+
"elapsed_ms": int((time.perf_counter() - started_at) * 1000),
|
|
1113
|
+
"backend_total_hint": scan_control.get("backend_total_count", result_amount),
|
|
1114
|
+
"backend_page_amount": scan_control.get("backend_page_amount"),
|
|
1115
|
+
"source_pages": source_pages,
|
|
1116
|
+
"raw_scan_complete": raw_scan_complete,
|
|
1117
|
+
"scan_control": scan_control,
|
|
1118
|
+
}
|
|
1119
|
+
return response
|
|
1120
|
+
|
|
1121
|
+
def _build_analyze_group_payload(self, answer_list: list[JSONValue], dimensions: list[JSONObject]) -> JSONObject:
|
|
1122
|
+
if not dimensions:
|
|
1123
|
+
return {}
|
|
1124
|
+
payload: JSONObject = {}
|
|
1125
|
+
for item in dimensions:
|
|
1126
|
+
field = cast(FormField, item["field"])
|
|
1127
|
+
alias = cast(str, item["alias"])
|
|
1128
|
+
bucket = cast(str | None, item["bucket"])
|
|
1129
|
+
value = _extract_field_value(answer_list, field)
|
|
1130
|
+
if bucket is not None:
|
|
1131
|
+
value = _to_time_bucket(value, bucket)
|
|
1132
|
+
payload[alias] = value
|
|
1133
|
+
return payload
|
|
1134
|
+
|
|
1135
|
+
def _initialize_metric_states(self, metrics: list[JSONObject]) -> dict[str, JSONObject]:
|
|
1136
|
+
states: dict[str, JSONObject] = {}
|
|
1137
|
+
for item in metrics:
|
|
1138
|
+
states[str(item["alias"])] = {
|
|
1139
|
+
"count": 0,
|
|
1140
|
+
"sum": 0.0,
|
|
1141
|
+
"min": None,
|
|
1142
|
+
"max": None,
|
|
1143
|
+
"seen": set(),
|
|
462
1144
|
}
|
|
1145
|
+
return states
|
|
463
1146
|
|
|
464
|
-
|
|
1147
|
+
def _analysis_group_key(self, payload: JSONObject) -> tuple[tuple[str, object], ...]:
|
|
1148
|
+
return tuple((key, self._freeze_group_key_value(value)) for key, value in payload.items())
|
|
1149
|
+
|
|
1150
|
+
def _freeze_group_key_value(self, value: JSONValue) -> object:
|
|
1151
|
+
if isinstance(value, dict):
|
|
1152
|
+
return tuple((key, self._freeze_group_key_value(item)) for key, item in sorted(value.items()))
|
|
1153
|
+
if isinstance(value, list):
|
|
1154
|
+
return tuple(self._freeze_group_key_value(item) for item in value)
|
|
1155
|
+
return value
|
|
1156
|
+
|
|
1157
|
+
def _apply_metric_states(self, states: dict[str, JSONObject], metrics: list[JSONObject], answer_list: list[JSONValue]) -> None:
|
|
1158
|
+
for item in metrics:
|
|
1159
|
+
alias = cast(str, item["alias"])
|
|
1160
|
+
op = cast(str, item["op"])
|
|
1161
|
+
field = cast(FormField | None, item["field"])
|
|
1162
|
+
state = states[alias]
|
|
1163
|
+
if op == "count":
|
|
1164
|
+
state["count"] = int(state["count"]) + 1
|
|
1165
|
+
continue
|
|
1166
|
+
value = _extract_field_value(answer_list, field)
|
|
1167
|
+
if op == "distinct_count":
|
|
1168
|
+
for entry in _flatten_distinct_values(value):
|
|
1169
|
+
cast(set[str], state["seen"]).add(entry)
|
|
1170
|
+
continue
|
|
1171
|
+
amount = _coerce_amount(value)
|
|
1172
|
+
if amount is None:
|
|
1173
|
+
continue
|
|
1174
|
+
state["count"] = int(state["count"]) + 1
|
|
1175
|
+
state["sum"] = float(state["sum"]) + amount
|
|
1176
|
+
state["min"] = amount if state["min"] is None else min(float(state["min"]), amount)
|
|
1177
|
+
state["max"] = amount if state["max"] is None else max(float(state["max"]), amount)
|
|
1178
|
+
|
|
1179
|
+
def _render_metric_values(self, states: dict[str, JSONObject], metrics: list[JSONObject]) -> JSONObject:
|
|
1180
|
+
rendered: JSONObject = {}
|
|
1181
|
+
for item in metrics:
|
|
1182
|
+
alias = cast(str, item["alias"])
|
|
1183
|
+
op = cast(str, item["op"])
|
|
1184
|
+
state = states[alias]
|
|
1185
|
+
count = int(state["count"] or 0)
|
|
1186
|
+
amount_sum = float(state["sum"] or 0.0)
|
|
1187
|
+
if op == "count":
|
|
1188
|
+
rendered[alias] = count
|
|
1189
|
+
elif op == "sum":
|
|
1190
|
+
rendered[alias] = amount_sum
|
|
1191
|
+
elif op == "avg":
|
|
1192
|
+
rendered[alias] = (amount_sum / count) if count else None
|
|
1193
|
+
elif op == "min":
|
|
1194
|
+
rendered[alias] = state["min"]
|
|
1195
|
+
elif op == "max":
|
|
1196
|
+
rendered[alias] = state["max"]
|
|
1197
|
+
elif op == "distinct_count":
|
|
1198
|
+
rendered[alias] = len(cast(set[str], state["seen"]))
|
|
1199
|
+
return rendered
|
|
1200
|
+
|
|
1201
|
+
def _matches_analyze_filters(self, answer_list: list[JSONValue], filters: list[JSONObject]) -> bool:
|
|
1202
|
+
for item in filters:
|
|
1203
|
+
field = cast(FormField, item["field"])
|
|
1204
|
+
if not _match_analyze_filter(_extract_field_value(answer_list, field), cast(str, item["op"]), item.get("value")):
|
|
1205
|
+
return False
|
|
1206
|
+
return True
|
|
1207
|
+
|
|
1208
|
+
def _sort_analyze_rows(
|
|
1209
|
+
self,
|
|
1210
|
+
rows: list[JSONObject],
|
|
1211
|
+
sort: list[JSONObject],
|
|
1212
|
+
dimensions: list[JSONObject],
|
|
1213
|
+
metrics: list[JSONObject],
|
|
1214
|
+
) -> list[JSONObject]:
|
|
1215
|
+
if not rows or not sort:
|
|
1216
|
+
if dimensions and any(item.get("bucket") for item in dimensions):
|
|
1217
|
+
return sorted(rows, key=lambda item: json.dumps(item.get("dimensions", {}), ensure_ascii=False, sort_keys=True))
|
|
1218
|
+
return rows
|
|
1219
|
+
sorted_rows = list(rows)
|
|
1220
|
+
for sort_item in reversed(sort):
|
|
1221
|
+
by = cast(str, sort_item["by"])
|
|
1222
|
+
reverse = cast(str, sort_item["order"]) == "desc"
|
|
1223
|
+
kind = cast(str, sort_item["kind"])
|
|
1224
|
+
sorted_rows.sort(
|
|
1225
|
+
key=lambda item: _sortable_value(
|
|
1226
|
+
cast(JSONObject, item["metrics" if kind == "metric" else "dimensions"]).get(by)
|
|
1227
|
+
),
|
|
1228
|
+
reverse=reverse,
|
|
1229
|
+
)
|
|
1230
|
+
return sorted_rows
|
|
1231
|
+
|
|
1232
|
+
def _build_analyze_warnings(self, *, local_filtering: bool, rows_truncated: bool) -> list[JSONObject]:
|
|
1233
|
+
warnings: list[JSONObject] = []
|
|
1234
|
+
if local_filtering:
|
|
1235
|
+
warnings.append(
|
|
1236
|
+
{
|
|
1237
|
+
"code": "LOCAL_FILTERING_APPLIED",
|
|
1238
|
+
"message": "Current analysis applies local filtering after scanning source pages.",
|
|
1239
|
+
}
|
|
1240
|
+
)
|
|
1241
|
+
if rows_truncated:
|
|
1242
|
+
warnings.append(
|
|
1243
|
+
{
|
|
1244
|
+
"code": "ROWS_TRUNCATED",
|
|
1245
|
+
"message": "Result rows were truncated by limit; totals remain based on the full analyzed result set.",
|
|
1246
|
+
}
|
|
1247
|
+
)
|
|
1248
|
+
return warnings
|
|
465
1249
|
|
|
466
1250
|
def record_write_plan(
|
|
467
1251
|
self,
|
|
@@ -558,7 +1342,7 @@ class RecordTools(ToolBase):
|
|
|
558
1342
|
blockers.append("payload writes readonly or system-managed fields")
|
|
559
1343
|
if question_relations:
|
|
560
1344
|
validation["warnings"].append("form contains questionRelations; linked visibility and runtime required rules may differ at submit time.")
|
|
561
|
-
actions = ["Use
|
|
1345
|
+
actions = ["Use record_schema_get when field titles or field ids are ambiguous."]
|
|
562
1346
|
if support_matrix["restricted"]:
|
|
563
1347
|
actions.append("Review write_format.required_presteps for restricted fields before submit.")
|
|
564
1348
|
if invalid_fields:
|
|
@@ -606,6 +1390,7 @@ class RecordTools(ToolBase):
|
|
|
606
1390
|
page_size: int,
|
|
607
1391
|
requested_pages: int,
|
|
608
1392
|
scan_max_pages: int,
|
|
1393
|
+
auto_expand_pages: bool = False,
|
|
609
1394
|
query_key: str | None,
|
|
610
1395
|
filters: list[JSONObject],
|
|
611
1396
|
sorts: list[JSONObject],
|
|
@@ -622,6 +1407,24 @@ class RecordTools(ToolBase):
|
|
|
622
1407
|
view_name: str | None = None,
|
|
623
1408
|
) -> JSONObject:
|
|
624
1409
|
resolved_mode = _resolve_query_mode(query_mode, apply_id=apply_id, amount_column=amount_column, time_range=time_range, stat_policy=stat_policy)
|
|
1410
|
+
if resolved_mode == "summary":
|
|
1411
|
+
raise_tool_error(
|
|
1412
|
+
QingflowApiError(
|
|
1413
|
+
category="config",
|
|
1414
|
+
message="query_mode='summary' is not supported",
|
|
1415
|
+
details={
|
|
1416
|
+
"error_code": "UNSUPPORTED_QUERY_MODE",
|
|
1417
|
+
"allowed_modes": ["auto", "list", "record"],
|
|
1418
|
+
"fix_hint": "Use record_schema_get followed by record_analyze for any statistical analysis.",
|
|
1419
|
+
},
|
|
1420
|
+
)
|
|
1421
|
+
)
|
|
1422
|
+
if resolved_mode == "list":
|
|
1423
|
+
list_paging = _fixed_list_scan_policy()
|
|
1424
|
+
page_size = int(list_paging["page_size"])
|
|
1425
|
+
requested_pages = int(list_paging["requested_pages"])
|
|
1426
|
+
scan_max_pages = int(list_paging["scan_max_pages"])
|
|
1427
|
+
auto_expand_pages = bool(list_paging["auto_expand_pages"])
|
|
625
1428
|
if resolved_mode == "record":
|
|
626
1429
|
return self._record_query_record(
|
|
627
1430
|
profile=profile,
|
|
@@ -632,259 +1435,25 @@ class RecordTools(ToolBase):
|
|
|
632
1435
|
output_profile=output_profile,
|
|
633
1436
|
list_type=list_type,
|
|
634
1437
|
)
|
|
635
|
-
if resolved_mode == "summary":
|
|
636
|
-
return self._record_query_summary(
|
|
637
|
-
profile=profile,
|
|
638
|
-
app_key=app_key,
|
|
639
|
-
page_num=page_num,
|
|
640
|
-
page_size=page_size,
|
|
641
|
-
requested_pages=requested_pages,
|
|
642
|
-
scan_max_pages=scan_max_pages,
|
|
643
|
-
query_key=query_key,
|
|
644
|
-
filters=filters,
|
|
645
|
-
sorts=sorts,
|
|
646
|
-
max_rows=max_rows,
|
|
647
|
-
max_columns=max_columns,
|
|
648
|
-
select_columns=select_columns,
|
|
649
|
-
amount_column=amount_column,
|
|
650
|
-
time_range=time_range,
|
|
651
|
-
stat_policy=stat_policy,
|
|
652
|
-
strict_full=strict_full,
|
|
653
|
-
output_profile=output_profile,
|
|
654
|
-
list_type=list_type,
|
|
655
|
-
view_key=view_key,
|
|
656
|
-
view_name=view_name,
|
|
657
|
-
)
|
|
658
1438
|
return self._record_query_list(
|
|
659
1439
|
profile=profile,
|
|
660
|
-
app_key=app_key,
|
|
661
|
-
page_num=page_num,
|
|
662
|
-
page_size=page_size,
|
|
663
|
-
requested_pages=requested_pages,
|
|
664
|
-
scan_max_pages=scan_max_pages,
|
|
665
|
-
query_key=query_key,
|
|
666
|
-
filters=filters,
|
|
667
|
-
sorts=sorts,
|
|
668
|
-
max_rows=max_rows,
|
|
669
|
-
max_columns=max_columns,
|
|
670
|
-
select_columns=select_columns,
|
|
671
|
-
time_range=time_range,
|
|
672
|
-
output_profile=output_profile,
|
|
673
|
-
list_type=list_type,
|
|
674
|
-
view_key=view_key,
|
|
675
|
-
view_name=view_name,
|
|
676
|
-
)
|
|
677
|
-
|
|
678
|
-
def record_aggregate(
|
|
679
|
-
self,
|
|
680
|
-
*,
|
|
681
|
-
profile: str,
|
|
682
|
-
app_key: str,
|
|
683
|
-
group_by: list[str | int],
|
|
684
|
-
amount_column: str | int | None,
|
|
685
|
-
metrics: list[str],
|
|
686
|
-
page_num: int,
|
|
687
|
-
page_size: int,
|
|
688
|
-
requested_pages: int,
|
|
689
|
-
scan_max_pages: int,
|
|
690
|
-
query_key: str | None,
|
|
691
|
-
filters: list[JSONObject],
|
|
692
|
-
sorts: list[JSONObject],
|
|
693
|
-
time_range: JSONObject,
|
|
694
|
-
time_bucket: str | None,
|
|
695
|
-
max_groups: int,
|
|
696
|
-
strict_full: bool,
|
|
697
|
-
output_profile: str,
|
|
698
|
-
list_type: int,
|
|
699
|
-
view_key: str | None = None,
|
|
700
|
-
view_name: str | None = None,
|
|
701
|
-
) -> JSONObject:
|
|
702
|
-
if not app_key:
|
|
703
|
-
raise_tool_error(QingflowApiError.config_error("app_key is required"))
|
|
704
|
-
if max_groups <= 0:
|
|
705
|
-
raise_tool_error(QingflowApiError.config_error("max_groups must be positive"))
|
|
706
|
-
|
|
707
|
-
def runner(session_profile, context):
|
|
708
|
-
index = self._get_field_index(profile, context, app_key, force_refresh=False)
|
|
709
|
-
view_selection = self._resolve_view_selection(profile, context, app_key, view_key=view_key, view_name=view_name)
|
|
710
|
-
dept_member_cache: dict[int, set[int]] = {}
|
|
711
|
-
group_fields = [self._resolve_field_selector(item, index, location="group_by") for item in group_by]
|
|
712
|
-
amount_field = self._resolve_field_selector(amount_column, index, location="amount_column") if amount_column is not None else None
|
|
713
|
-
time_field = self._resolve_time_range_column(time_range, index)
|
|
714
|
-
match_rules = self._resolve_match_rules(context, filters, index)
|
|
715
|
-
sort_rules = self._resolve_sorts(sorts, index)
|
|
716
|
-
match_rules = self._append_time_range_filter(match_rules, time_range, time_field)
|
|
717
|
-
metric_names = _normalize_metrics(metrics, include_sum=amount_field is not None)
|
|
718
|
-
query_id = _query_id()
|
|
719
|
-
pages_to_scan = min(max(requested_pages, 1), max(scan_max_pages, 1))
|
|
720
|
-
current_page = max(page_num, 1)
|
|
721
|
-
scanned_pages = 0
|
|
722
|
-
scanned_records = 0
|
|
723
|
-
source_pages: list[int] = []
|
|
724
|
-
result_amount: int | None = None
|
|
725
|
-
has_more = False
|
|
726
|
-
group_stats: dict[str, JSONObject] = {}
|
|
727
|
-
total_amount = 0.0
|
|
728
|
-
while scanned_pages < pages_to_scan:
|
|
729
|
-
page = self._search_page(
|
|
730
|
-
context,
|
|
731
|
-
app_key=app_key,
|
|
732
|
-
page_num=current_page,
|
|
733
|
-
page_size=page_size,
|
|
734
|
-
query_key=query_key,
|
|
735
|
-
match_rules=match_rules,
|
|
736
|
-
sorts=sort_rules,
|
|
737
|
-
search_que_ids=None,
|
|
738
|
-
list_type=list_type,
|
|
739
|
-
)
|
|
740
|
-
scanned_pages += 1
|
|
741
|
-
source_pages.append(current_page)
|
|
742
|
-
rows = page.get("list")
|
|
743
|
-
items = rows if isinstance(rows, list) else []
|
|
744
|
-
if result_amount is None:
|
|
745
|
-
result_amount = _effective_total(page, page_size)
|
|
746
|
-
has_more = _page_has_more(page, current_page, page_size, len(items))
|
|
747
|
-
for item in items:
|
|
748
|
-
if not isinstance(item, dict):
|
|
749
|
-
continue
|
|
750
|
-
answers = item.get("answers")
|
|
751
|
-
answer_list = answers if isinstance(answers, list) else []
|
|
752
|
-
if not self._matches_view_selection(
|
|
753
|
-
context,
|
|
754
|
-
answer_list,
|
|
755
|
-
view_selection=view_selection,
|
|
756
|
-
dept_member_cache=dept_member_cache,
|
|
757
|
-
):
|
|
758
|
-
continue
|
|
759
|
-
scanned_records += 1
|
|
760
|
-
group_payload = {
|
|
761
|
-
field.que_title: _extract_field_value(answer_list, field)
|
|
762
|
-
for field in group_fields
|
|
763
|
-
}
|
|
764
|
-
if time_bucket and time_field is not None:
|
|
765
|
-
group_payload["time_bucket"] = _to_time_bucket(_extract_field_value(answer_list, time_field), time_bucket)
|
|
766
|
-
group_key = json.dumps(group_payload, ensure_ascii=False, sort_keys=True)
|
|
767
|
-
bucket = group_stats.get(group_key)
|
|
768
|
-
if bucket is None:
|
|
769
|
-
bucket = {"group": group_payload, "count": 0, "amount_total": None, "metrics": {}}
|
|
770
|
-
group_stats[group_key] = bucket
|
|
771
|
-
bucket["count"] = int(bucket["count"]) + 1
|
|
772
|
-
amount_value = _coerce_amount(_extract_field_value(answer_list, amount_field)) if amount_field is not None else None
|
|
773
|
-
if amount_value is not None:
|
|
774
|
-
total_amount += amount_value
|
|
775
|
-
bucket["amount_total"] = float(bucket.get("amount_total") or 0.0) + amount_value
|
|
776
|
-
metrics_payload = bucket["metrics"] if isinstance(bucket.get("metrics"), dict) else {}
|
|
777
|
-
for metric in metric_names:
|
|
778
|
-
current_metric = metrics_payload.get(metric)
|
|
779
|
-
metric_state = current_metric if isinstance(current_metric, dict) else {"count": 0, "sum": 0.0, "min": None, "max": None}
|
|
780
|
-
metric_state["count"] = int(metric_state["count"]) + 1
|
|
781
|
-
if amount_value is not None:
|
|
782
|
-
metric_state["sum"] = float(metric_state["sum"]) + amount_value
|
|
783
|
-
metric_state["min"] = amount_value if metric_state["min"] is None else min(float(metric_state["min"]), amount_value)
|
|
784
|
-
metric_state["max"] = amount_value if metric_state["max"] is None else max(float(metric_state["max"]), amount_value)
|
|
785
|
-
metrics_payload[metric] = metric_state
|
|
786
|
-
bucket["metrics"] = metrics_payload
|
|
787
|
-
if len(group_stats) >= max_groups:
|
|
788
|
-
break
|
|
789
|
-
if len(group_stats) >= max_groups or not has_more:
|
|
790
|
-
break
|
|
791
|
-
current_page += 1
|
|
792
|
-
|
|
793
|
-
groups = []
|
|
794
|
-
for bucket in group_stats.values():
|
|
795
|
-
metrics_payload = bucket["metrics"] if isinstance(bucket.get("metrics"), dict) else {}
|
|
796
|
-
rendered_metrics: JSONObject = {}
|
|
797
|
-
for metric_name, metric_state in metrics_payload.items():
|
|
798
|
-
if not isinstance(metric_state, dict):
|
|
799
|
-
continue
|
|
800
|
-
count = int(metric_state.get("count", 0) or 0)
|
|
801
|
-
amount_sum = float(metric_state.get("sum", 0.0) or 0.0)
|
|
802
|
-
metric_result: JSONObject = {}
|
|
803
|
-
if metric_name == "count":
|
|
804
|
-
metric_result["value"] = count
|
|
805
|
-
else:
|
|
806
|
-
if metric_name == "sum":
|
|
807
|
-
metric_result["value"] = amount_sum
|
|
808
|
-
elif metric_name == "avg":
|
|
809
|
-
metric_result["value"] = (amount_sum / count) if count else None
|
|
810
|
-
elif metric_name == "min":
|
|
811
|
-
metric_result["value"] = metric_state.get("min")
|
|
812
|
-
elif metric_name == "max":
|
|
813
|
-
metric_result["value"] = metric_state.get("max")
|
|
814
|
-
rendered_metrics[metric_name] = metric_result
|
|
815
|
-
groups.append(
|
|
816
|
-
{
|
|
817
|
-
"group": bucket["group"],
|
|
818
|
-
"count": bucket["count"],
|
|
819
|
-
"count_ratio": (int(bucket["count"]) / scanned_records) if scanned_records else 0,
|
|
820
|
-
"amount_total": None if amount_field is None else _coerce_amount(bucket.get("amount_total")),
|
|
821
|
-
"amount_ratio": None,
|
|
822
|
-
"metrics": rendered_metrics,
|
|
823
|
-
}
|
|
824
|
-
)
|
|
825
|
-
groups.sort(key=lambda item: int(item["count"]), reverse=True)
|
|
826
|
-
if amount_field is not None and total_amount > 0:
|
|
827
|
-
for item in groups:
|
|
828
|
-
amount_total = _coerce_amount(item.get("amount_total"))
|
|
829
|
-
item["amount_ratio"] = (amount_total / total_amount) if amount_total is not None else None
|
|
830
|
-
effective_result_amount = scanned_records if view_selection is not None else (result_amount or scanned_records)
|
|
831
|
-
completeness = _build_completeness(
|
|
832
|
-
result_amount=effective_result_amount,
|
|
833
|
-
returned_items=len(groups),
|
|
834
|
-
fetched_pages=scanned_pages,
|
|
835
|
-
requested_pages=pages_to_scan,
|
|
836
|
-
has_more=has_more,
|
|
837
|
-
next_page_token=None,
|
|
838
|
-
is_complete=not has_more and len(groups) < max_groups,
|
|
839
|
-
omitted_items=max(0, effective_result_amount - len(groups)),
|
|
840
|
-
extra={
|
|
841
|
-
"raw_scan_complete": not has_more,
|
|
842
|
-
"scan_limit_hit": has_more,
|
|
843
|
-
"scanned_pages": scanned_pages,
|
|
844
|
-
"scan_limit": pages_to_scan,
|
|
845
|
-
"output_page_complete": len(groups) < max_groups,
|
|
846
|
-
"raw_next_page_token": None,
|
|
847
|
-
"output_next_page_token": None,
|
|
848
|
-
"stop_reason": "source_exhausted" if not has_more else "scan_limit",
|
|
849
|
-
},
|
|
850
|
-
)
|
|
851
|
-
evidence = {
|
|
852
|
-
"query_id": query_id,
|
|
853
|
-
"app_key": app_key,
|
|
854
|
-
"filters": _echo_filters(match_rules),
|
|
855
|
-
"selected_columns": [field.que_title for field in group_fields],
|
|
856
|
-
"time_range": time_range or None,
|
|
857
|
-
"source_pages": source_pages,
|
|
858
|
-
"view": _view_selection_payload(view_selection),
|
|
859
|
-
}
|
|
860
|
-
if strict_full and not bool(completeness.get("raw_scan_complete")):
|
|
861
|
-
self._raise_need_more_data(completeness, evidence, "Aggregate result is incomplete; increase requested_pages or scan_max_pages.")
|
|
862
|
-
response: JSONObject = {
|
|
863
|
-
"profile": profile,
|
|
864
|
-
"ws_id": session_profile.selected_ws_id,
|
|
865
|
-
"ok": True,
|
|
866
|
-
"request_route": self._request_route_payload(context),
|
|
867
|
-
"data": {
|
|
868
|
-
"app_key": app_key,
|
|
869
|
-
"view": _view_selection_payload(view_selection),
|
|
870
|
-
"summary": {
|
|
871
|
-
"total_count": scanned_records,
|
|
872
|
-
"total_amount": total_amount if amount_field is not None else None,
|
|
873
|
-
},
|
|
874
|
-
"groups": groups,
|
|
875
|
-
"completeness": completeness,
|
|
876
|
-
},
|
|
877
|
-
}
|
|
878
|
-
if output_profile == "verbose":
|
|
879
|
-
response["completeness"] = completeness
|
|
880
|
-
response["evidence"] = evidence
|
|
881
|
-
response["resolved_mappings"] = {
|
|
882
|
-
"group_by": [_field_mapping_entry("group_by", field, requested=field.que_title) for field in group_fields],
|
|
883
|
-
"amount_column": _field_mapping_entry("amount", amount_field, requested=amount_field.que_title) if amount_field is not None else None,
|
|
884
|
-
}
|
|
885
|
-
return response
|
|
886
|
-
|
|
887
|
-
return self._run_record_tool(profile, runner)
|
|
1440
|
+
app_key=app_key,
|
|
1441
|
+
page_num=page_num,
|
|
1442
|
+
page_size=page_size,
|
|
1443
|
+
requested_pages=requested_pages,
|
|
1444
|
+
scan_max_pages=scan_max_pages,
|
|
1445
|
+
query_key=query_key,
|
|
1446
|
+
filters=filters,
|
|
1447
|
+
sorts=sorts,
|
|
1448
|
+
max_rows=max_rows,
|
|
1449
|
+
max_columns=max_columns,
|
|
1450
|
+
select_columns=select_columns,
|
|
1451
|
+
time_range=time_range,
|
|
1452
|
+
output_profile=output_profile,
|
|
1453
|
+
list_type=list_type,
|
|
1454
|
+
view_key=view_key,
|
|
1455
|
+
view_name=view_name,
|
|
1456
|
+
)
|
|
888
1457
|
|
|
889
1458
|
def record_create(
|
|
890
1459
|
self,
|
|
@@ -1391,6 +1960,18 @@ class RecordTools(ToolBase):
|
|
|
1391
1960
|
"view": _view_selection_payload(view_selection),
|
|
1392
1961
|
"list": {
|
|
1393
1962
|
"rows": rows,
|
|
1963
|
+
"row_cap_hit": _list_row_cap_hit(returned_items=len(rows), row_cap=max_rows),
|
|
1964
|
+
"sample_only": _list_sample_only(
|
|
1965
|
+
returned_items=len(rows),
|
|
1966
|
+
row_cap=max_rows,
|
|
1967
|
+
result_amount=effective_result_amount,
|
|
1968
|
+
),
|
|
1969
|
+
"safe_for_final_conclusion": False,
|
|
1970
|
+
"analysis_warning": _list_sample_warning(
|
|
1971
|
+
returned_items=len(rows),
|
|
1972
|
+
row_cap=max_rows,
|
|
1973
|
+
result_amount=effective_result_amount,
|
|
1974
|
+
),
|
|
1394
1975
|
"pagination": {
|
|
1395
1976
|
"page_num": page_num,
|
|
1396
1977
|
"page_size": page_size,
|
|
@@ -1421,187 +2002,6 @@ class RecordTools(ToolBase):
|
|
|
1421
2002
|
|
|
1422
2003
|
return self._run_record_tool(profile, runner)
|
|
1423
2004
|
|
|
1424
|
-
def _record_query_summary(
|
|
1425
|
-
self,
|
|
1426
|
-
*,
|
|
1427
|
-
profile: str,
|
|
1428
|
-
app_key: str,
|
|
1429
|
-
page_num: int,
|
|
1430
|
-
page_size: int,
|
|
1431
|
-
requested_pages: int,
|
|
1432
|
-
scan_max_pages: int,
|
|
1433
|
-
query_key: str | None,
|
|
1434
|
-
filters: list[JSONObject],
|
|
1435
|
-
sorts: list[JSONObject],
|
|
1436
|
-
max_rows: int,
|
|
1437
|
-
max_columns: int | None,
|
|
1438
|
-
select_columns: list[str | int],
|
|
1439
|
-
amount_column: str | int | None,
|
|
1440
|
-
time_range: JSONObject,
|
|
1441
|
-
stat_policy: JSONObject,
|
|
1442
|
-
strict_full: bool,
|
|
1443
|
-
output_profile: str,
|
|
1444
|
-
list_type: int,
|
|
1445
|
-
view_key: str | None = None,
|
|
1446
|
-
view_name: str | None = None,
|
|
1447
|
-
) -> JSONObject:
|
|
1448
|
-
if not app_key:
|
|
1449
|
-
raise_tool_error(QingflowApiError.config_error("app_key is required"))
|
|
1450
|
-
|
|
1451
|
-
def runner(session_profile, context):
|
|
1452
|
-
index = self._get_field_index(profile, context, app_key, force_refresh=False)
|
|
1453
|
-
view_selection = self._resolve_view_selection(profile, context, app_key, view_key=view_key, view_name=view_name)
|
|
1454
|
-
dept_member_cache: dict[int, set[int]] = {}
|
|
1455
|
-
amount_field = self._resolve_field_selector(amount_column, index, location="amount_column") if amount_column is not None else None
|
|
1456
|
-
time_field = self._resolve_time_range_column(time_range, index)
|
|
1457
|
-
resolved_column_cap = _bounded_column_limit(
|
|
1458
|
-
max_columns,
|
|
1459
|
-
default_limit=MAX_SUMMARY_PREVIEW_COLUMN_LIMIT,
|
|
1460
|
-
hard_limit=MAX_SUMMARY_PREVIEW_COLUMN_LIMIT,
|
|
1461
|
-
)
|
|
1462
|
-
preview_fields = self._resolve_summary_preview_fields(select_columns, index, amount_field, time_field, max_columns=max_columns)
|
|
1463
|
-
match_rules = self._resolve_match_rules(context, filters, index)
|
|
1464
|
-
sort_rules = self._resolve_sorts(sorts, index)
|
|
1465
|
-
match_rules = self._append_time_range_filter(match_rules, time_range, time_field)
|
|
1466
|
-
include_negative = bool(stat_policy.get("include_negative", True))
|
|
1467
|
-
include_null = bool(stat_policy.get("include_null", False))
|
|
1468
|
-
scan_limit = min(max(requested_pages, 1), max(scan_max_pages, 1))
|
|
1469
|
-
current_page = max(page_num, 1)
|
|
1470
|
-
scanned_pages = 0
|
|
1471
|
-
scanned_records = 0
|
|
1472
|
-
result_amount: int | None = None
|
|
1473
|
-
has_more = False
|
|
1474
|
-
source_pages: list[int] = []
|
|
1475
|
-
preview_rows: list[JSONObject] = []
|
|
1476
|
-
total_amount = 0.0
|
|
1477
|
-
missing_count = 0
|
|
1478
|
-
by_day: dict[str, JSONObject] = {}
|
|
1479
|
-
while scanned_pages < scan_limit:
|
|
1480
|
-
page = self._search_page(
|
|
1481
|
-
context,
|
|
1482
|
-
app_key=app_key,
|
|
1483
|
-
page_num=current_page,
|
|
1484
|
-
page_size=page_size,
|
|
1485
|
-
query_key=query_key,
|
|
1486
|
-
match_rules=match_rules,
|
|
1487
|
-
sorts=sort_rules,
|
|
1488
|
-
search_que_ids=None,
|
|
1489
|
-
list_type=list_type,
|
|
1490
|
-
)
|
|
1491
|
-
scanned_pages += 1
|
|
1492
|
-
source_pages.append(current_page)
|
|
1493
|
-
page_rows = page.get("list")
|
|
1494
|
-
items = page_rows if isinstance(page_rows, list) else []
|
|
1495
|
-
if result_amount is None:
|
|
1496
|
-
result_amount = _effective_total(page, page_size)
|
|
1497
|
-
has_more = _page_has_more(page, current_page, page_size, len(items))
|
|
1498
|
-
for item in items:
|
|
1499
|
-
if not isinstance(item, dict):
|
|
1500
|
-
continue
|
|
1501
|
-
answers = item.get("answers")
|
|
1502
|
-
answer_list = answers if isinstance(answers, list) else []
|
|
1503
|
-
if not self._matches_view_selection(
|
|
1504
|
-
context,
|
|
1505
|
-
answer_list,
|
|
1506
|
-
view_selection=view_selection,
|
|
1507
|
-
dept_member_cache=dept_member_cache,
|
|
1508
|
-
):
|
|
1509
|
-
continue
|
|
1510
|
-
scanned_records += 1
|
|
1511
|
-
if len(preview_rows) < max_rows:
|
|
1512
|
-
apply_id = _coerce_count(item.get("applyId")) or _coerce_count(item.get("id"))
|
|
1513
|
-
preview_rows.append(_build_flat_row(answer_list, preview_fields, apply_id=apply_id))
|
|
1514
|
-
amount_value = _coerce_amount(_extract_field_value(answer_list, amount_field)) if amount_field is not None else None
|
|
1515
|
-
if amount_field is not None:
|
|
1516
|
-
if amount_value is None:
|
|
1517
|
-
if not include_null:
|
|
1518
|
-
missing_count += 1
|
|
1519
|
-
elif include_negative or amount_value >= 0:
|
|
1520
|
-
total_amount += amount_value
|
|
1521
|
-
day_key = _to_time_bucket(_extract_field_value(answer_list, time_field), "day") if time_field is not None else "all"
|
|
1522
|
-
bucket = by_day.get(day_key)
|
|
1523
|
-
if bucket is None:
|
|
1524
|
-
bucket = {"day": day_key, "count": 0, "amount_total": 0.0 if amount_field is not None else None}
|
|
1525
|
-
by_day[day_key] = bucket
|
|
1526
|
-
bucket["count"] = int(bucket["count"]) + 1
|
|
1527
|
-
if amount_field is not None and amount_value is not None and (include_negative or amount_value >= 0):
|
|
1528
|
-
bucket["amount_total"] = float(bucket.get("amount_total") or 0.0) + amount_value
|
|
1529
|
-
if not has_more:
|
|
1530
|
-
break
|
|
1531
|
-
current_page += 1
|
|
1532
|
-
raw_scan_complete = not has_more
|
|
1533
|
-
effective_result_amount = scanned_records if view_selection is not None else max(result_amount or 0, scanned_records)
|
|
1534
|
-
completeness = _build_completeness(
|
|
1535
|
-
result_amount=effective_result_amount,
|
|
1536
|
-
returned_items=len(preview_rows),
|
|
1537
|
-
fetched_pages=scanned_pages,
|
|
1538
|
-
requested_pages=scan_limit,
|
|
1539
|
-
has_more=has_more,
|
|
1540
|
-
next_page_token=None,
|
|
1541
|
-
is_complete=raw_scan_complete and len(preview_rows) < max_rows,
|
|
1542
|
-
omitted_items=max(0, effective_result_amount - len(preview_rows)),
|
|
1543
|
-
extra={
|
|
1544
|
-
"raw_scan_complete": raw_scan_complete,
|
|
1545
|
-
"scan_limit_hit": has_more,
|
|
1546
|
-
"scanned_pages": scanned_pages,
|
|
1547
|
-
"scan_limit": scan_limit,
|
|
1548
|
-
"output_page_complete": len(preview_rows) < max_rows,
|
|
1549
|
-
"raw_next_page_token": None,
|
|
1550
|
-
"output_next_page_token": None,
|
|
1551
|
-
"stop_reason": "source_exhausted" if raw_scan_complete else "scan_limit",
|
|
1552
|
-
},
|
|
1553
|
-
)
|
|
1554
|
-
evidence = {
|
|
1555
|
-
"query_id": _query_id(),
|
|
1556
|
-
"app_key": app_key,
|
|
1557
|
-
"filters": _echo_filters(match_rules),
|
|
1558
|
-
"selected_columns": [field.que_title for field in preview_fields],
|
|
1559
|
-
"time_range": time_range or None,
|
|
1560
|
-
"source_pages": source_pages,
|
|
1561
|
-
"view": _view_selection_payload(view_selection),
|
|
1562
|
-
}
|
|
1563
|
-
if strict_full and not raw_scan_complete:
|
|
1564
|
-
self._raise_need_more_data(completeness, evidence, "Summary is incomplete; increase requested_pages or scan_max_pages.")
|
|
1565
|
-
response: JSONObject = {
|
|
1566
|
-
"profile": profile,
|
|
1567
|
-
"ws_id": session_profile.selected_ws_id,
|
|
1568
|
-
"ok": True,
|
|
1569
|
-
"request_route": self._request_route_payload(context),
|
|
1570
|
-
"data": {
|
|
1571
|
-
"mode": "summary",
|
|
1572
|
-
"source_tool": "record_search",
|
|
1573
|
-
"view": _view_selection_payload(view_selection),
|
|
1574
|
-
"summary": {
|
|
1575
|
-
"summary": {
|
|
1576
|
-
"total_count": scanned_records,
|
|
1577
|
-
"total_amount": total_amount if amount_field is not None else None,
|
|
1578
|
-
"by_day": sorted(by_day.values(), key=lambda item: str(item.get("day"))),
|
|
1579
|
-
"missing_count": missing_count,
|
|
1580
|
-
},
|
|
1581
|
-
"rows": preview_rows,
|
|
1582
|
-
"completeness": completeness,
|
|
1583
|
-
"applied_limits": {
|
|
1584
|
-
"row_cap": max_rows,
|
|
1585
|
-
"column_cap": resolved_column_cap,
|
|
1586
|
-
"selected_columns": [field.que_title for field in preview_fields],
|
|
1587
|
-
},
|
|
1588
|
-
},
|
|
1589
|
-
},
|
|
1590
|
-
"output_profile": output_profile,
|
|
1591
|
-
"next_page_token": None,
|
|
1592
|
-
}
|
|
1593
|
-
if output_profile == "verbose":
|
|
1594
|
-
response["completeness"] = completeness
|
|
1595
|
-
response["evidence"] = evidence
|
|
1596
|
-
response["resolved_mappings"] = {
|
|
1597
|
-
"select_columns": [_field_mapping_entry("row", field, requested=field.que_title) for field in preview_fields],
|
|
1598
|
-
"amount_column": _field_mapping_entry("amount", amount_field, requested=amount_field.que_title) if amount_field is not None else None,
|
|
1599
|
-
"time_range": _field_mapping_entry("time", time_field, requested=time_field.que_title) if time_field is not None else None,
|
|
1600
|
-
}
|
|
1601
|
-
return response
|
|
1602
|
-
|
|
1603
|
-
return self._run_record_tool(profile, runner)
|
|
1604
|
-
|
|
1605
2005
|
def _get_form_schema(self, profile: str, context, app_key: str, *, force_refresh: bool) -> JSONObject: # type: ignore[no-untyped-def]
|
|
1606
2006
|
cache_key = (profile, app_key)
|
|
1607
2007
|
if not force_refresh and cache_key in self._form_cache:
|
|
@@ -1721,6 +2121,60 @@ class RecordTools(ToolBase):
|
|
|
1721
2121
|
return True
|
|
1722
2122
|
return False
|
|
1723
2123
|
|
|
2124
|
+
def _build_analysis_probe(
|
|
2125
|
+
self,
|
|
2126
|
+
profile: str,
|
|
2127
|
+
context, # type: ignore[no-untyped-def]
|
|
2128
|
+
*,
|
|
2129
|
+
app_key: str,
|
|
2130
|
+
arguments: JSONObject,
|
|
2131
|
+
view_selection: ViewSelection | None,
|
|
2132
|
+
) -> JSONObject:
|
|
2133
|
+
routed_mode = _resolve_query_mode(
|
|
2134
|
+
str(arguments.get("query_mode", "auto")),
|
|
2135
|
+
apply_id=_coerce_count(arguments.get("apply_id")),
|
|
2136
|
+
amount_column=arguments.get("amount_column"),
|
|
2137
|
+
time_range=cast(JSONObject, arguments.get("time_range") if isinstance(arguments.get("time_range"), dict) else {}),
|
|
2138
|
+
stat_policy=cast(JSONObject, arguments.get("stat_policy") if isinstance(arguments.get("stat_policy"), dict) else {}),
|
|
2139
|
+
)
|
|
2140
|
+
fixed_policy = _fixed_analysis_scan_policy() if routed_mode == "summary" else _fixed_list_scan_policy()
|
|
2141
|
+
page_size = int(fixed_policy["page_size"])
|
|
2142
|
+
query_key = _normalize_optional_text(arguments.get("query_key"))
|
|
2143
|
+
filters = _as_object_list(arguments.get("filters"))
|
|
2144
|
+
sorts = _as_object_list(arguments.get("sorts"))
|
|
2145
|
+
app_form = self._get_field_index(profile, context, app_key, force_refresh=False)
|
|
2146
|
+
time_range = cast(JSONObject, arguments.get("time_range") if isinstance(arguments.get("time_range"), dict) else {})
|
|
2147
|
+
match_rules = self._resolve_match_rules(context, filters, app_form)
|
|
2148
|
+
sort_rules = self._resolve_sorts(sorts, app_form)
|
|
2149
|
+
time_field = self._resolve_time_range_column(time_range, app_form)
|
|
2150
|
+
match_rules = self._append_time_range_filter(match_rules, time_range, time_field)
|
|
2151
|
+
probe_page = self._search_page(
|
|
2152
|
+
context,
|
|
2153
|
+
app_key=app_key,
|
|
2154
|
+
page_num=max(_coerce_count(arguments.get("page_num")) or 1, 1),
|
|
2155
|
+
page_size=page_size,
|
|
2156
|
+
query_key=query_key,
|
|
2157
|
+
match_rules=match_rules,
|
|
2158
|
+
sorts=sort_rules,
|
|
2159
|
+
search_que_ids=None,
|
|
2160
|
+
list_type=_coerce_count(arguments.get("list_type")) or DEFAULT_RECORD_LIST_TYPE,
|
|
2161
|
+
)
|
|
2162
|
+
backend_total_count = _effective_total(probe_page, page_size)
|
|
2163
|
+
page_amount = _coerce_count(probe_page.get("pageAmount"))
|
|
2164
|
+
estimated_full_scan_pages = page_amount
|
|
2165
|
+
if estimated_full_scan_pages is None and backend_total_count > 0:
|
|
2166
|
+
estimated_full_scan_pages = (backend_total_count + page_size - 1) // page_size
|
|
2167
|
+
current_budget = min(int(fixed_policy["requested_pages"]), int(fixed_policy["scan_max_pages"]))
|
|
2168
|
+
return {
|
|
2169
|
+
"page_size": page_size,
|
|
2170
|
+
"backend_total_count": backend_total_count,
|
|
2171
|
+
"backend_page_amount": page_amount,
|
|
2172
|
+
"estimated_full_scan_pages": estimated_full_scan_pages,
|
|
2173
|
+
"current_scan_budget": current_budget,
|
|
2174
|
+
"would_exceed_current_budget": bool(estimated_full_scan_pages is not None and estimated_full_scan_pages > current_budget),
|
|
2175
|
+
"local_filtering": bool(view_selection.conditions) if view_selection is not None else False,
|
|
2176
|
+
}
|
|
2177
|
+
|
|
1724
2178
|
def _search_page(
|
|
1725
2179
|
self,
|
|
1726
2180
|
context, # type: ignore[no-untyped-def]
|
|
@@ -2121,7 +2575,7 @@ class RecordTools(ToolBase):
|
|
|
2121
2575
|
raise RecordInputError(
|
|
2122
2576
|
message=f"{location} references unknown queId '{requested}'",
|
|
2123
2577
|
error_code="FIELD_NOT_FOUND",
|
|
2124
|
-
fix_hint="Use
|
|
2578
|
+
fix_hint="Use record_schema_get to confirm the exact field_id.",
|
|
2125
2579
|
details={"location": location, "requested": requested, "requested_key": requested_key},
|
|
2126
2580
|
)
|
|
2127
2581
|
matches = index.by_title.get(requested_key, [])
|
|
@@ -2131,7 +2585,7 @@ class RecordTools(ToolBase):
|
|
|
2131
2585
|
raise RecordInputError(
|
|
2132
2586
|
message=f"{location} field '{requested}' is ambiguous",
|
|
2133
2587
|
error_code="AMBIGUOUS_FIELD",
|
|
2134
|
-
fix_hint="Use numeric queId, or
|
|
2588
|
+
fix_hint="Use numeric queId, or inspect record_schema_get output to disambiguate the field.",
|
|
2135
2589
|
details={
|
|
2136
2590
|
"location": location,
|
|
2137
2591
|
"requested": requested,
|
|
@@ -2147,7 +2601,7 @@ class RecordTools(ToolBase):
|
|
|
2147
2601
|
raise RecordInputError(
|
|
2148
2602
|
message=f"{location} field '{requested}' is ambiguous",
|
|
2149
2603
|
error_code="AMBIGUOUS_FIELD",
|
|
2150
|
-
fix_hint="Use a more specific field title, or
|
|
2604
|
+
fix_hint="Use a more specific field title, or inspect record_schema_get aliases to disambiguate the field.",
|
|
2151
2605
|
details={
|
|
2152
2606
|
"location": location,
|
|
2153
2607
|
"requested": requested,
|
|
@@ -2159,7 +2613,7 @@ class RecordTools(ToolBase):
|
|
|
2159
2613
|
raise RecordInputError(
|
|
2160
2614
|
message=f"{location} cannot resolve field '{requested}'",
|
|
2161
2615
|
error_code="FIELD_NOT_FOUND",
|
|
2162
|
-
fix_hint="Use
|
|
2616
|
+
fix_hint="Use record_schema_get to confirm the exact field title or field_id.",
|
|
2163
2617
|
details={
|
|
2164
2618
|
"location": location,
|
|
2165
2619
|
"requested": requested,
|
|
@@ -2536,56 +2990,6 @@ class RecordTools(ToolBase):
|
|
|
2536
2990
|
"reason": error.message,
|
|
2537
2991
|
}
|
|
2538
2992
|
|
|
2539
|
-
def _resolve_plan_candidate(self, candidate: JSONObject, index: FieldIndex) -> JSONObject:
|
|
2540
|
-
requested = str(candidate.get("requested", "")).strip()
|
|
2541
|
-
role = str(candidate.get("role", "field"))
|
|
2542
|
-
try:
|
|
2543
|
-
field = self._resolve_field_selector(requested, index, location=role)
|
|
2544
|
-
return {
|
|
2545
|
-
"role": role,
|
|
2546
|
-
"requested": requested,
|
|
2547
|
-
"resolved": True,
|
|
2548
|
-
"que_id": field.que_id,
|
|
2549
|
-
"que_title": field.que_title,
|
|
2550
|
-
"que_type": field.que_type,
|
|
2551
|
-
"reason": None,
|
|
2552
|
-
}
|
|
2553
|
-
except RecordInputError as error:
|
|
2554
|
-
return {
|
|
2555
|
-
"role": role,
|
|
2556
|
-
"requested": requested,
|
|
2557
|
-
"resolved": False,
|
|
2558
|
-
"que_id": None,
|
|
2559
|
-
"que_title": None,
|
|
2560
|
-
"que_type": None,
|
|
2561
|
-
"reason": error.message,
|
|
2562
|
-
}
|
|
2563
|
-
|
|
2564
|
-
def _validate_plan_arguments(self, tool: str, arguments: JSONObject) -> JSONObject:
|
|
2565
|
-
missing_required: list[str] = []
|
|
2566
|
-
warnings: list[str] = []
|
|
2567
|
-
if tool in {"record_query", "record_aggregate"} and not arguments.get("app_key"):
|
|
2568
|
-
missing_required.append("app_key")
|
|
2569
|
-
if tool == "record_get":
|
|
2570
|
-
if not arguments.get("app_key"):
|
|
2571
|
-
missing_required.append("app_key")
|
|
2572
|
-
if not arguments.get("apply_id"):
|
|
2573
|
-
missing_required.append("apply_id")
|
|
2574
|
-
if not arguments.get("select_columns"):
|
|
2575
|
-
missing_required.append("select_columns")
|
|
2576
|
-
query_mode = _resolve_query_mode(
|
|
2577
|
-
str(arguments.get("query_mode", "auto")),
|
|
2578
|
-
apply_id=_coerce_count(arguments.get("apply_id")),
|
|
2579
|
-
amount_column=arguments.get("amount_column"),
|
|
2580
|
-
time_range=cast(JSONObject, arguments.get("time_range") if isinstance(arguments.get("time_range"), dict) else {}),
|
|
2581
|
-
stat_policy=cast(JSONObject, arguments.get("stat_policy") if isinstance(arguments.get("stat_policy"), dict) else {}),
|
|
2582
|
-
) if tool == "record_query" else None
|
|
2583
|
-
if tool == "record_query" and query_mode in {"list", "record"} and not arguments.get("select_columns"):
|
|
2584
|
-
missing_required.append("select_columns")
|
|
2585
|
-
if tool == "record_query" and query_mode == "summary" and not arguments.get("amount_column") and not arguments.get("time_range"):
|
|
2586
|
-
warnings.append("summary mode without amount_column or time_range only returns row counts")
|
|
2587
|
-
return {"valid": not missing_required, "missing_required": missing_required, "warnings": warnings}
|
|
2588
|
-
|
|
2589
2993
|
def _validate_app_and_record(self, app_key: str, apply_id: int | str) -> int:
|
|
2590
2994
|
if not app_key:
|
|
2591
2995
|
raise_tool_error(QingflowApiError.config_error("app_key is required"))
|
|
@@ -2910,143 +3314,44 @@ def _extract_question_options(question: JSONObject) -> list[str]:
|
|
|
2910
3314
|
return values
|
|
2911
3315
|
|
|
2912
3316
|
|
|
2913
|
-
def
|
|
2914
|
-
normalized = cast(JSONObject, _parse_json_like(arguments))
|
|
2915
|
-
alias_map = {
|
|
2916
|
-
"appKey": "app_key",
|
|
2917
|
-
"applyId": "apply_id",
|
|
2918
|
-
"queryMode": "query_mode",
|
|
2919
|
-
"pageNum": "page_num",
|
|
2920
|
-
"pageSize": "page_size",
|
|
2921
|
-
"requestedPages": "requested_pages",
|
|
2922
|
-
"scanMaxPages": "scan_max_pages",
|
|
2923
|
-
"queryKey": "query_key",
|
|
2924
|
-
"maxRows": "max_rows",
|
|
2925
|
-
"maxColumns": "max_columns",
|
|
2926
|
-
"selectColumns": "select_columns",
|
|
2927
|
-
"amountColumn": "amount_column",
|
|
2928
|
-
"timeRange": "time_range",
|
|
2929
|
-
"strictFull": "strict_full",
|
|
2930
|
-
"outputProfile": "output_profile",
|
|
2931
|
-
"listType": "list_type",
|
|
2932
|
-
"viewKey": "view_key",
|
|
2933
|
-
"viewName": "view_name",
|
|
2934
|
-
"groupBy": "group_by",
|
|
2935
|
-
"timeBucket": "time_bucket",
|
|
2936
|
-
"maxGroups": "max_groups",
|
|
2937
|
-
"forceRefreshForm": "force_refresh_form",
|
|
2938
|
-
}
|
|
2939
|
-
result = dict(normalized)
|
|
2940
|
-
for alias, canonical in alias_map.items():
|
|
2941
|
-
if alias in result and canonical not in result:
|
|
2942
|
-
result[canonical] = result[alias]
|
|
2943
|
-
if tool == "record_get" and "query_mode" not in result:
|
|
2944
|
-
result["query_mode"] = "record"
|
|
2945
|
-
return result
|
|
2946
|
-
|
|
2947
|
-
|
|
2948
|
-
def _collect_plan_field_candidates(tool: str, arguments: JSONObject) -> list[JSONObject]:
|
|
2949
|
-
candidates: list[JSONObject] = []
|
|
2950
|
-
if tool in {"record_query", "record_get"}:
|
|
2951
|
-
for item in _as_selector_list(arguments.get("select_columns")):
|
|
2952
|
-
candidates.append({"role": "select_columns", "requested": str(item)})
|
|
2953
|
-
amount = arguments.get("amount_column")
|
|
2954
|
-
if amount is not None:
|
|
2955
|
-
candidates.append({"role": "amount_column", "requested": str(amount)})
|
|
2956
|
-
time_range = arguments.get("time_range")
|
|
2957
|
-
if isinstance(time_range, dict) and time_range.get("column") is not None:
|
|
2958
|
-
candidates.append({"role": "time_range.column", "requested": str(time_range["column"])})
|
|
2959
|
-
if tool == "record_aggregate":
|
|
2960
|
-
for item in _as_selector_list(arguments.get("group_by")):
|
|
2961
|
-
candidates.append({"role": "group_by", "requested": str(item)})
|
|
2962
|
-
amount = arguments.get("amount_column")
|
|
2963
|
-
if amount is not None:
|
|
2964
|
-
candidates.append({"role": "amount_column", "requested": str(amount)})
|
|
2965
|
-
time_range = arguments.get("time_range")
|
|
2966
|
-
if isinstance(time_range, dict) and time_range.get("column") is not None:
|
|
2967
|
-
candidates.append({"role": "time_range.column", "requested": str(time_range["column"])})
|
|
2968
|
-
for item in _as_object_list(arguments.get("filters")):
|
|
2969
|
-
selector = _extract_filter_selector(item)
|
|
2970
|
-
if selector is not None:
|
|
2971
|
-
candidates.append({"role": "filter", "requested": str(selector)})
|
|
2972
|
-
for item in _as_object_list(arguments.get("sorts", arguments.get("sort"))):
|
|
2973
|
-
selector = _extract_sort_selector(item)
|
|
2974
|
-
if selector is not None:
|
|
2975
|
-
candidates.append({"role": "sort", "requested": str(selector)})
|
|
2976
|
-
return candidates
|
|
2977
|
-
|
|
2978
|
-
|
|
2979
|
-
def _build_plan_estimate(tool: str, arguments: JSONObject) -> JSONObject:
|
|
2980
|
-
page_size = _coerce_count(arguments.get("page_size")) or DEFAULT_QUERY_PAGE_SIZE
|
|
2981
|
-
requested_pages = _coerce_count(arguments.get("requested_pages")) or 1
|
|
2982
|
-
scan_max_pages = _coerce_count(arguments.get("scan_max_pages")) or requested_pages
|
|
2983
|
-
estimated_scan_pages = min(requested_pages, scan_max_pages)
|
|
2984
|
-
may_hit_limits = estimated_scan_pages > DEFAULT_SCAN_MAX_PAGES
|
|
2985
|
-
reasons = []
|
|
2986
|
-
if may_hit_limits:
|
|
2987
|
-
reasons.append("requested scan pages exceed the default analysis budget")
|
|
2988
|
-
if tool == "record_query":
|
|
2989
|
-
routed_mode = _resolve_query_mode(
|
|
2990
|
-
str(arguments.get("query_mode", "auto")),
|
|
2991
|
-
apply_id=_coerce_count(arguments.get("apply_id")),
|
|
2992
|
-
amount_column=arguments.get("amount_column"),
|
|
2993
|
-
time_range=cast(JSONObject, arguments.get("time_range") if isinstance(arguments.get("time_range"), dict) else {}),
|
|
2994
|
-
stat_policy=cast(JSONObject, arguments.get("stat_policy") if isinstance(arguments.get("stat_policy"), dict) else {}),
|
|
2995
|
-
)
|
|
2996
|
-
if routed_mode == "list":
|
|
2997
|
-
reasons.append("list mode is not a safe final-analysis endpoint")
|
|
3317
|
+
def _fixed_list_scan_policy() -> JSONObject:
|
|
2998
3318
|
return {
|
|
2999
|
-
"page_size":
|
|
3000
|
-
"requested_pages":
|
|
3001
|
-
"scan_max_pages":
|
|
3002
|
-
"
|
|
3003
|
-
"
|
|
3004
|
-
"may_hit_limits": may_hit_limits,
|
|
3005
|
-
"reasons": reasons,
|
|
3006
|
-
"probe": None,
|
|
3319
|
+
"page_size": DEFAULT_LIST_PAGE_SIZE,
|
|
3320
|
+
"requested_pages": 1,
|
|
3321
|
+
"scan_max_pages": 1,
|
|
3322
|
+
"auto_expand_pages": False,
|
|
3323
|
+
"public_inputs_exposed": False,
|
|
3007
3324
|
}
|
|
3008
3325
|
|
|
3009
3326
|
|
|
3010
|
-
def
|
|
3011
|
-
tool: str,
|
|
3012
|
-
arguments: JSONObject,
|
|
3013
|
-
validation: JSONObject,
|
|
3014
|
-
field_mapping: list[JSONObject],
|
|
3015
|
-
estimate: JSONObject,
|
|
3016
|
-
) -> JSONObject:
|
|
3017
|
-
blockers: list[str] = []
|
|
3018
|
-
actions: list[str] = []
|
|
3019
|
-
if not bool(validation.get("valid")):
|
|
3020
|
-
blockers.append("arguments are not valid")
|
|
3021
|
-
actions.append("Fix missing_required before execution.")
|
|
3022
|
-
unresolved = [item for item in field_mapping if not bool(item.get("resolved"))]
|
|
3023
|
-
if unresolved:
|
|
3024
|
-
blockers.append("one or more fields are unresolved")
|
|
3025
|
-
actions.append("Use record_field_resolve to resolve field ids before execution.")
|
|
3026
|
-
if tool == "record_query":
|
|
3027
|
-
routed_mode = _resolve_query_mode(
|
|
3028
|
-
str(arguments.get("query_mode", "auto")),
|
|
3029
|
-
apply_id=_coerce_count(arguments.get("apply_id")),
|
|
3030
|
-
amount_column=arguments.get("amount_column"),
|
|
3031
|
-
time_range=cast(JSONObject, arguments.get("time_range") if isinstance(arguments.get("time_range"), dict) else {}),
|
|
3032
|
-
stat_policy=cast(JSONObject, arguments.get("stat_policy") if isinstance(arguments.get("stat_policy"), dict) else {}),
|
|
3033
|
-
)
|
|
3034
|
-
if routed_mode == "list":
|
|
3035
|
-
blockers.append("list mode is not a safe final-analysis endpoint")
|
|
3036
|
-
actions.append("Use record_query(summary) or record_aggregate for final statistics.")
|
|
3037
|
-
if routed_mode == "record":
|
|
3038
|
-
blockers.append("record mode is a detail endpoint, not a final-analysis endpoint")
|
|
3039
|
-
if tool in {"record_query", "record_aggregate"} and not bool(arguments.get("strict_full")):
|
|
3040
|
-
blockers.append("strict_full should be true for final conclusions")
|
|
3041
|
-
actions.append("Set strict_full=true so incomplete scans block final conclusions.")
|
|
3042
|
-
actions.append("After execution, verify completeness before using the result as a final conclusion.")
|
|
3327
|
+
def _fixed_analysis_scan_policy() -> JSONObject:
|
|
3043
3328
|
return {
|
|
3044
|
-
"
|
|
3045
|
-
"
|
|
3046
|
-
"
|
|
3329
|
+
"page_size": DEFAULT_ANALYSIS_PAGE_SIZE,
|
|
3330
|
+
"requested_pages": DEFAULT_ANALYSIS_SCAN_MAX_PAGES,
|
|
3331
|
+
"scan_max_pages": DEFAULT_ANALYSIS_SCAN_MAX_PAGES,
|
|
3332
|
+
"auto_expand_pages": True,
|
|
3333
|
+
"public_inputs_exposed": False,
|
|
3047
3334
|
}
|
|
3048
3335
|
|
|
3049
3336
|
|
|
3337
|
+
def _list_row_cap_hit(*, returned_items: int, row_cap: int) -> bool:
|
|
3338
|
+
return row_cap > 0 and returned_items >= row_cap
|
|
3339
|
+
|
|
3340
|
+
|
|
3341
|
+
def _list_sample_only(*, returned_items: int, row_cap: int, result_amount: int | None) -> bool:
|
|
3342
|
+
if _list_row_cap_hit(returned_items=returned_items, row_cap=row_cap):
|
|
3343
|
+
return True
|
|
3344
|
+
if result_amount is None:
|
|
3345
|
+
return False
|
|
3346
|
+
return result_amount > returned_items
|
|
3347
|
+
|
|
3348
|
+
|
|
3349
|
+
def _list_sample_warning(*, returned_items: int, row_cap: int, result_amount: int | None) -> str:
|
|
3350
|
+
if _list_sample_only(returned_items=returned_items, row_cap=row_cap, result_amount=result_amount):
|
|
3351
|
+
return "当前仅返回样本,不适合最终统计结论。"
|
|
3352
|
+
return "record_query(list) 适合浏览或导出明细;最终统计结论请改用 record_schema_get -> record_analyze。"
|
|
3353
|
+
|
|
3354
|
+
|
|
3050
3355
|
def _resolve_query_mode(
|
|
3051
3356
|
query_mode: str,
|
|
3052
3357
|
*,
|
|
@@ -3059,8 +3364,6 @@ def _resolve_query_mode(
|
|
|
3059
3364
|
return query_mode
|
|
3060
3365
|
if apply_id is not None and apply_id > 0:
|
|
3061
3366
|
return "record"
|
|
3062
|
-
if amount_column is not None or time_range or stat_policy:
|
|
3063
|
-
return "summary"
|
|
3064
3367
|
return "list"
|
|
3065
3368
|
|
|
3066
3369
|
|
|
@@ -3154,6 +3457,69 @@ def _view_selection_payload(view_selection: ViewSelection | None) -> JSONObject
|
|
|
3154
3457
|
}
|
|
3155
3458
|
|
|
3156
3459
|
|
|
3460
|
+
def _compute_scan_limit(
|
|
3461
|
+
*,
|
|
3462
|
+
requested_pages: int,
|
|
3463
|
+
scan_max_pages: int,
|
|
3464
|
+
auto_expand_pages: bool,
|
|
3465
|
+
page: JSONObject | None = None,
|
|
3466
|
+
page_size: int | None = None,
|
|
3467
|
+
) -> tuple[int, JSONObject]:
|
|
3468
|
+
base_requested = max(requested_pages, 1)
|
|
3469
|
+
base_scan_max = max(scan_max_pages, 1)
|
|
3470
|
+
initial_limit = min(base_requested, base_scan_max)
|
|
3471
|
+
meta: JSONObject = {
|
|
3472
|
+
"requested_pages": base_requested,
|
|
3473
|
+
"scan_max_pages": base_scan_max,
|
|
3474
|
+
"auto_expand_pages": auto_expand_pages,
|
|
3475
|
+
"auto_expand_applied": False,
|
|
3476
|
+
"auto_expand_target_pages": initial_limit,
|
|
3477
|
+
"auto_expand_page_cap": DEFAULT_ANALYSIS_AUTO_EXPAND_PAGE_CAP,
|
|
3478
|
+
}
|
|
3479
|
+
if not auto_expand_pages or not isinstance(page, dict):
|
|
3480
|
+
return initial_limit, meta
|
|
3481
|
+
effective_page_size = max(page_size or 0, 1)
|
|
3482
|
+
backend_total = _effective_total(page, effective_page_size)
|
|
3483
|
+
page_amount = _coerce_count(page.get("pageAmount"))
|
|
3484
|
+
target_pages = page_amount
|
|
3485
|
+
if target_pages is None and backend_total > 0:
|
|
3486
|
+
target_pages = (backend_total + effective_page_size - 1) // effective_page_size
|
|
3487
|
+
if target_pages is None:
|
|
3488
|
+
return initial_limit, meta
|
|
3489
|
+
expanded_limit = min(max(initial_limit, target_pages), DEFAULT_ANALYSIS_AUTO_EXPAND_PAGE_CAP)
|
|
3490
|
+
if expanded_limit > initial_limit:
|
|
3491
|
+
meta["auto_expand_applied"] = True
|
|
3492
|
+
meta["auto_expand_target_pages"] = target_pages
|
|
3493
|
+
meta["backend_total_count"] = backend_total
|
|
3494
|
+
meta["backend_page_amount"] = page_amount
|
|
3495
|
+
return expanded_limit, meta
|
|
3496
|
+
|
|
3497
|
+
|
|
3498
|
+
def _build_analysis_counts(
|
|
3499
|
+
*,
|
|
3500
|
+
backend_total_count: int | None,
|
|
3501
|
+
scanned_count: int,
|
|
3502
|
+
grouped_count: int,
|
|
3503
|
+
local_filtering: bool,
|
|
3504
|
+
) -> JSONObject:
|
|
3505
|
+
unscanned_count: int | None = None
|
|
3506
|
+
if backend_total_count is not None and not local_filtering:
|
|
3507
|
+
unscanned_count = max(backend_total_count - scanned_count, 0)
|
|
3508
|
+
return {
|
|
3509
|
+
"backend_total_count": backend_total_count,
|
|
3510
|
+
"scanned_count": scanned_count,
|
|
3511
|
+
"grouped_count": grouped_count,
|
|
3512
|
+
"unscanned_count": unscanned_count,
|
|
3513
|
+
"local_filtering": local_filtering,
|
|
3514
|
+
}
|
|
3515
|
+
|
|
3516
|
+
|
|
3517
|
+
def _analysis_status_from_completeness(completeness: JSONObject) -> tuple[str, bool]:
|
|
3518
|
+
raw_scan_complete = bool(completeness.get("raw_scan_complete"))
|
|
3519
|
+
status = "success" if raw_scan_complete else "partial_success"
|
|
3520
|
+
return status, raw_scan_complete
|
|
3521
|
+
|
|
3522
|
+
|
|
3157
3523
|
def _build_completeness(
|
|
3158
3524
|
*,
|
|
3159
3525
|
result_amount: int,
|
|
@@ -3277,6 +3643,11 @@ def _to_time_bucket(value: JSONValue, bucket: str) -> str:
|
|
|
3277
3643
|
if bucket == "week":
|
|
3278
3644
|
iso_year, iso_week, _ = parsed.isocalendar()
|
|
3279
3645
|
return f"{iso_year}-W{iso_week:02d}"
|
|
3646
|
+
if bucket == "quarter":
|
|
3647
|
+
quarter = ((parsed.month - 1) // 3) + 1
|
|
3648
|
+
return f"{parsed.year}-Q{quarter}"
|
|
3649
|
+
if bucket == "year":
|
|
3650
|
+
return parsed.strftime("%Y")
|
|
3280
3651
|
return parsed.strftime("%Y-%m-%d")
|
|
3281
3652
|
|
|
3282
3653
|
|
|
@@ -3290,6 +3661,117 @@ def _parse_datetime_like(text: str) -> datetime | None:
|
|
|
3290
3661
|
return None
|
|
3291
3662
|
|
|
3292
3663
|
|
|
3664
|
+
def _flatten_distinct_values(value: JSONValue) -> list[str]:
|
|
3665
|
+
if value is None:
|
|
3666
|
+
return []
|
|
3667
|
+
if isinstance(value, list):
|
|
3668
|
+
items = value
|
|
3669
|
+
else:
|
|
3670
|
+
items = [value]
|
|
3671
|
+
flattened: list[str] = []
|
|
3672
|
+
for item in items:
|
|
3673
|
+
if item is None:
|
|
3674
|
+
continue
|
|
3675
|
+
if isinstance(item, (dict, list)):
|
|
3676
|
+
flattened.append(json.dumps(item, ensure_ascii=False, sort_keys=True))
|
|
3677
|
+
else:
|
|
3678
|
+
flattened.append(_stringify_json(item))
|
|
3679
|
+
return flattened
|
|
3680
|
+
|
|
3681
|
+
|
|
3682
|
+
def _coerce_comparable(value: JSONValue) -> JSONValue:
|
|
3683
|
+
amount = _coerce_amount(value)
|
|
3684
|
+
if amount is not None:
|
|
3685
|
+
return amount
|
|
3686
|
+
text = _normalize_optional_text(value)
|
|
3687
|
+
if text is None:
|
|
3688
|
+
return None
|
|
3689
|
+
parsed = _parse_datetime_like(text)
|
|
3690
|
+
if parsed is not None:
|
|
3691
|
+
return parsed
|
|
3692
|
+
return text
|
|
3693
|
+
|
|
3694
|
+
|
|
3695
|
+
def _match_analyze_filter(field_value: JSONValue, op: str, expected: JSONValue) -> bool:
|
|
3696
|
+
if op == "is_null":
|
|
3697
|
+
return field_value is None or field_value == [] or field_value == ""
|
|
3698
|
+
if op == "not_null":
|
|
3699
|
+
return not _match_analyze_filter(field_value, "is_null", expected)
|
|
3700
|
+
|
|
3701
|
+
values = field_value if isinstance(field_value, list) else [field_value]
|
|
3702
|
+
normalized_values = [_coerce_comparable(item) for item in values if item is not None]
|
|
3703
|
+
|
|
3704
|
+
if op == "contains":
|
|
3705
|
+
needle = _normalize_optional_text(expected)
|
|
3706
|
+
if needle is None:
|
|
3707
|
+
return False
|
|
3708
|
+
return any(needle in _stringify_json(item) for item in values if item is not None)
|
|
3709
|
+
|
|
3710
|
+
if op in {"in", "not_in"}:
|
|
3711
|
+
expected_values = expected if isinstance(expected, list) else [expected]
|
|
3712
|
+
matched = any(
|
|
3713
|
+
_analyze_values_equal(actual=item, expected=expected_item)
|
|
3714
|
+
for item in values
|
|
3715
|
+
if item is not None
|
|
3716
|
+
for expected_item in expected_values
|
|
3717
|
+
if expected_item is not None
|
|
3718
|
+
)
|
|
3719
|
+
return matched if op == "in" else not matched
|
|
3720
|
+
|
|
3721
|
+
if op in {"eq", "neq"}:
|
|
3722
|
+
matched = any(_analyze_values_equal(actual=item, expected=expected) for item in values if item is not None)
|
|
3723
|
+
return matched if op == "eq" else not matched
|
|
3724
|
+
|
|
3725
|
+
if op == "between":
|
|
3726
|
+
lower, upper = _coerce_filter_range(expected)
|
|
3727
|
+
lower_value = _coerce_comparable(lower)
|
|
3728
|
+
upper_value = _coerce_comparable(upper)
|
|
3729
|
+
for item in normalized_values:
|
|
3730
|
+
if item is None:
|
|
3731
|
+
continue
|
|
3732
|
+
try:
|
|
3733
|
+
if lower_value is not None and item < lower_value:
|
|
3734
|
+
continue
|
|
3735
|
+
if upper_value is not None and item > upper_value:
|
|
3736
|
+
continue
|
|
3737
|
+
return True
|
|
3738
|
+
except TypeError:
|
|
3739
|
+
continue
|
|
3740
|
+
return False
|
|
3741
|
+
|
|
3742
|
+
target = _coerce_comparable(expected)
|
|
3743
|
+
for item in normalized_values:
|
|
3744
|
+
if item is None or target is None:
|
|
3745
|
+
continue
|
|
3746
|
+
try:
|
|
3747
|
+
if op == "gt" and item > target:
|
|
3748
|
+
return True
|
|
3749
|
+
if op == "gte" and item >= target:
|
|
3750
|
+
return True
|
|
3751
|
+
if op == "lt" and item < target:
|
|
3752
|
+
return True
|
|
3753
|
+
if op == "lte" and item <= target:
|
|
3754
|
+
return True
|
|
3755
|
+
except TypeError:
|
|
3756
|
+
continue
|
|
3757
|
+
return False
|
|
3758
|
+
|
|
3759
|
+
|
|
3760
|
+
def _analyze_values_equal(*, actual: JSONValue, expected: JSONValue) -> bool:
|
|
3761
|
+
actual_comparable = _coerce_comparable(actual)
|
|
3762
|
+
expected_comparable = _coerce_comparable(expected)
|
|
3763
|
+
if actual_comparable is not None and expected_comparable is not None and type(actual_comparable) is type(expected_comparable):
|
|
3764
|
+
return actual_comparable == expected_comparable
|
|
3765
|
+
return _stringify_json(actual) == _stringify_json(expected)
|
|
3766
|
+
|
|
3767
|
+
|
|
3768
|
+
def _sortable_value(value: JSONValue) -> tuple[int, JSONValue]:
|
|
3769
|
+
if value is None:
|
|
3770
|
+
return (1, "")
|
|
3771
|
+
comparable = _coerce_comparable(value)
|
|
3772
|
+
return (0, comparable if comparable is not None else "")
|
|
3773
|
+
|
|
3774
|
+
|
|
3293
3775
|
def _echo_filters(filters: list[JSONObject]) -> list[JSONObject]:
|
|
3294
3776
|
return [dict(item) for item in filters]
|
|
3295
3777
|
|
|
@@ -3850,7 +4332,7 @@ def _option_value(raw_value: JSONValue, field: FormField) -> JSONObject:
|
|
|
3850
4332
|
raise RecordInputError(
|
|
3851
4333
|
message=f"field '{field.que_title}' uses unknown option '{text}'",
|
|
3852
4334
|
error_code="OPTION_NOT_FOUND",
|
|
3853
|
-
fix_hint="Use
|
|
4335
|
+
fix_hint="Use record_schema_get or inspect the form to confirm allowed option values.",
|
|
3854
4336
|
details={"field": _field_ref_payload(field), "expected_format": _write_format_for_field(field), "received_value": raw_value},
|
|
3855
4337
|
)
|
|
3856
4338
|
return {"value": text}
|