@roll-agent/octopus-agent 0.0.10 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +19 -1
- package/SKILL.md +77 -12
- package/package.json +1 -1
- package/pyproject.toml +4 -2
- package/references/env.yaml +21 -0
- package/scripts/refresh_dictionary.py +67 -0
- package/scripts/verify_binding.py +47 -0
- package/src/octopus_skill/_version.py +1 -1
- package/src/octopus_skill/agent.py +4045 -415
- package/src/octopus_skill/config.py +55 -10
- package/src/octopus_skill/exporter.py +66 -0
- package/src/octopus_skill/llm_result_renderer.py +101 -0
- package/src/octopus_skill/llm_sql_generator.py +17 -3
- package/src/octopus_skill/mcp_client.py +16 -0
- package/src/octopus_skill/octopus_run.py +370 -32
- package/src/octopus_skill/prompt_builder.py +427 -78
- package/src/octopus_skill/question_analyzer.py +388 -0
- package/src/octopus_skill/result_renderer.py +38 -14
- package/src/octopus_skill/schema_context_retriever.py +932 -159
- package/src/octopus_skill/sql_generator.py +2093 -1231
- package/src/octopus_skill/query_sql_cache.py +0 -72
|
@@ -8,25 +8,78 @@ from typing import Any
|
|
|
8
8
|
from .audit_logger import AuditLogger, sql_hash
|
|
9
9
|
from .config import AppConfig
|
|
10
10
|
from .context import UserContext, new_session_id, new_trace_id
|
|
11
|
+
from .exporter import export_result_to_csv
|
|
12
|
+
from .llm_result_renderer import render_result_with_llm
|
|
11
13
|
from .llm_sql_generator import Sampler, SamplingSqlGenerator
|
|
12
14
|
from .mcp_client import SpongeMcpClient
|
|
13
|
-
from .
|
|
15
|
+
from .prompt_builder import _business_examples, filter_agent_instructions_by_stage, format_numbered_instruction_lines
|
|
16
|
+
from .question_analyzer import QuestionAnalysis, QuestionAnalysisError, analyze_question
|
|
14
17
|
from .result_renderer import normalize_result_format, render_result
|
|
15
18
|
from .schema_cache import SchemaCache
|
|
16
|
-
from .schema_context_retriever import retrieve_schema_context
|
|
19
|
+
from .schema_context_retriever import match_concepts_for_question, narrow_schema_context_for_query_plan, retrieve_schema_context, _schema_agent_instructions
|
|
17
20
|
from .sql_generator import (
|
|
18
21
|
GeneratedSql,
|
|
19
|
-
RuleBasedSqlGenerator,
|
|
20
22
|
SqlGenerationError,
|
|
23
|
+
build_schema_validation_error,
|
|
24
|
+
build_execute_unknown_column_error,
|
|
25
|
+
build_sql_rule_validation_error,
|
|
21
26
|
enforce_sql_rules,
|
|
27
|
+
finalize_generated_sql,
|
|
28
|
+
_is_business_query,
|
|
22
29
|
is_low_quality_sql_for_question,
|
|
23
|
-
|
|
30
|
+
low_quality_sql_issue_for_question,
|
|
31
|
+
normalize_text_filters_to_like,
|
|
32
|
+
normalize_chinese_select_aliases,
|
|
24
33
|
sensitive_terms_in_text,
|
|
34
|
+
sql_missing_query_plan_filters,
|
|
35
|
+
sql_missing_query_plan_select_fields,
|
|
36
|
+
sql_satisfies_query_plan_filters,
|
|
37
|
+
sql_satisfies_query_plan_select_fields,
|
|
25
38
|
sql_satisfies_time_range,
|
|
39
|
+
apply_time_range_filters_from_plan,
|
|
40
|
+
apply_time_range_filters_from_question,
|
|
41
|
+
time_range_plan_filters,
|
|
26
42
|
validate_sql_against_schema,
|
|
27
43
|
)
|
|
28
44
|
|
|
29
45
|
|
|
46
|
+
# Entity types supported by Sponge search_entities; mapsTo values map here.
|
|
47
|
+
_SEARCHABLE_ENTITY_TYPES = frozenset({"brand", "project", "province", "city", "region"})
|
|
48
|
+
_MAPS_TO_SEARCH_ENTITY_TYPE = {
|
|
49
|
+
"brand": "brand",
|
|
50
|
+
"sponge_project": "project",
|
|
51
|
+
"project": "project",
|
|
52
|
+
"province": "province",
|
|
53
|
+
"city": "city",
|
|
54
|
+
"region": "region",
|
|
55
|
+
}
|
|
56
|
+
# Always available location dimensions; merged into catalog when schema omits them.
|
|
57
|
+
_CORE_LOCATION_DIMENSIONS: tuple[dict[str, str], ...] = (
|
|
58
|
+
{"term": "城市", "mapsTo": "city", "type": "dimension"},
|
|
59
|
+
{"term": "省份", "mapsTo": "province", "type": "dimension"},
|
|
60
|
+
{"term": "区域", "mapsTo": "region", "type": "dimension"},
|
|
61
|
+
)
|
|
62
|
+
_STORE_NAME_MARKERS = ("店", "广场", "路", "街", "中心", "大厦", "号楼", "商场", "地铁", "项目")
|
|
63
|
+
_EXPLICIT_ENTITY_TYPES_KEY = "__explicitTypes"
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
# Validate error codes treated as "caller has no data scope" -> a valid empty
|
|
67
|
+
# result, not a SQL failure.
|
|
68
|
+
#
|
|
69
|
+
# Per MCP source review (alignment checklist Q1), the *current* server emits NO
|
|
70
|
+
# error for empty scope: it injects an `IN (NULL)` scope filter and returns
|
|
71
|
+
# success with 0 rows, which the normal empty-result path already handles. These
|
|
72
|
+
# codes are therefore only a defensive net for OLDER ONLINE deployments still
|
|
73
|
+
# observed emitting MCP_SQL_MISSING_SCOPE in audit logs (plus the declared-but-
|
|
74
|
+
# currently-unthrown MCP_PERMISSION_EMPTY_SCOPE).
|
|
75
|
+
#
|
|
76
|
+
# This is intentionally NOT treated as the live MCP contract. Remove once every
|
|
77
|
+
# online MCP instance runs the no-error-code behaviour.
|
|
78
|
+
_LEGACY_EMPTY_SCOPE_CODES = frozenset(
|
|
79
|
+
{"MCP_PERMISSION_EMPTY_SCOPE", "MCP_SQL_MISSING_SCOPE"}
|
|
80
|
+
)
|
|
81
|
+
|
|
82
|
+
|
|
30
83
|
@dataclass(frozen=True)
|
|
31
84
|
class QueryResult:
|
|
32
85
|
answer: str
|
|
@@ -34,6 +87,13 @@ class QueryResult:
|
|
|
34
87
|
normalized_sql: str
|
|
35
88
|
repaired_sql: str | None
|
|
36
89
|
validation: dict[str, Any]
|
|
90
|
+
needs_clarification: bool = False
|
|
91
|
+
clarification_type: str | None = None
|
|
92
|
+
clarification_payload: dict[str, Any] | None = None
|
|
93
|
+
query_plan: dict[str, Any] | None = None
|
|
94
|
+
execution_steps: list[dict[str, Any]] = field(default_factory=list)
|
|
95
|
+
raw_result: dict[str, Any] | None = None
|
|
96
|
+
result_format: str | None = None
|
|
37
97
|
|
|
38
98
|
|
|
39
99
|
@dataclass
|
|
@@ -88,272 +148,764 @@ class SpongeAgent:
|
|
|
88
148
|
result_format: str | None = None,
|
|
89
149
|
fast_mode: bool = True,
|
|
90
150
|
) -> QueryResult:
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
151
|
+
steps = _ExecutionSteps()
|
|
152
|
+
steps.done(1, "已进入 query_sponge")
|
|
153
|
+
user_question = _isolate_current_question(original_question or question)
|
|
154
|
+
schema: dict[str, Any] | None = None
|
|
155
|
+
pre_sql_agent_instructions: list[dict[str, Any]] = []
|
|
156
|
+
try:
|
|
157
|
+
schema = self._ensure_schema()
|
|
158
|
+
pre_sql_agent_instructions = _schema_agent_instructions(schema)
|
|
159
|
+
except Exception:
|
|
160
|
+
schema = None
|
|
161
|
+
pre_sql_agent_instructions = []
|
|
162
|
+
# Step 2: LLM 需求分析
|
|
163
|
+
steps.running(2)
|
|
164
|
+
if sampler is None:
|
|
165
|
+
reason = "当前无模型可用,请确认配置文件。"
|
|
166
|
+
steps.error(2, reason)
|
|
167
|
+
return _attach_execution_steps(_query_only_result(reason), steps)
|
|
168
|
+
try:
|
|
169
|
+
analysis = analyze_question(
|
|
170
|
+
user_question,
|
|
171
|
+
sampler,
|
|
172
|
+
agent_instructions=pre_sql_agent_instructions,
|
|
173
|
+
)
|
|
174
|
+
except QuestionAnalysisError as exc:
|
|
175
|
+
reason = f"需求分析失败:{exc}"
|
|
176
|
+
steps.error(2, reason)
|
|
177
|
+
return _attach_execution_steps(_query_only_result(reason), steps)
|
|
178
|
+
if analysis.intent == "mutation":
|
|
179
|
+
reason = f"检测到写操作意图(置信度 {analysis.confidence:.0%});query_sponge 只支持只读查询。"
|
|
180
|
+
steps.error(2, reason)
|
|
181
|
+
return _attach_execution_steps(_non_query_result(reason), steps)
|
|
182
|
+
if analysis.intent == "non_query":
|
|
183
|
+
reason = analysis.clarification_reason or "请先输入明确的业务数据查询需求。"
|
|
184
|
+
steps.error(2, reason)
|
|
185
|
+
return _attach_execution_steps(_non_query_result(reason), steps)
|
|
186
|
+
# intent == "query"
|
|
187
|
+
sql_question = _preserve_limit_scope_in_normalized_question(
|
|
188
|
+
analysis.normalized_question or user_question,
|
|
189
|
+
user_question,
|
|
190
|
+
)
|
|
191
|
+
time_range_question = _time_range_source_question(user_question, sql_question)
|
|
192
|
+
steps.done(
|
|
193
|
+
2,
|
|
194
|
+
f"意图=query({analysis.confidence:.0%});"
|
|
195
|
+
f"对象={analysis.target_entities or '未提取'};"
|
|
196
|
+
f"规范化问题={sql_question}",
|
|
197
|
+
)
|
|
198
|
+
self.audit_logger.log(
|
|
199
|
+
{
|
|
200
|
+
"sessionId": session_id,
|
|
201
|
+
"question": user_question,
|
|
202
|
+
"tool": "question_analysis",
|
|
203
|
+
"intent": analysis.intent,
|
|
204
|
+
"confidence": analysis.confidence,
|
|
205
|
+
"targetEntities": analysis.target_entities,
|
|
206
|
+
"metrics": analysis.metrics,
|
|
207
|
+
"noiseValues": analysis.noise_values,
|
|
208
|
+
"requiredConcepts": analysis.required_concepts,
|
|
209
|
+
"mutationSignals": analysis.mutation_signals,
|
|
210
|
+
"normalizedQuestion": sql_question,
|
|
211
|
+
}
|
|
212
|
+
)
|
|
94
213
|
context = UserContext(
|
|
95
214
|
session_id=session_id or new_session_id(),
|
|
96
215
|
)
|
|
97
|
-
sql_question = _strip_result_format_instruction(user_question)
|
|
98
216
|
sensitive_terms = sensitive_terms_in_text(sql_question)
|
|
99
|
-
|
|
217
|
+
steps.running(3)
|
|
218
|
+
if schema is None:
|
|
219
|
+
try:
|
|
220
|
+
schema = self._ensure_schema()
|
|
221
|
+
except Exception as exc:
|
|
222
|
+
reason = f"{type(exc).__name__}: {exc}"
|
|
223
|
+
steps.error(3, reason)
|
|
224
|
+
return _attach_execution_steps(_query_only_result(reason), steps)
|
|
225
|
+
steps.done(3, f"schemaVersion={schema.get('schemaVersion') or 'unknown'}")
|
|
226
|
+
required_concept_gap = _required_schema_concepts_gap_message(analysis, schema)
|
|
227
|
+
if required_concept_gap is not None:
|
|
228
|
+
steps.error(4, required_concept_gap)
|
|
229
|
+
return _attach_execution_steps(_business_metric_clarification_result(required_concept_gap), steps)
|
|
100
230
|
unsupported_field_message = _unsupported_requested_field_message(sql_question, schema)
|
|
101
231
|
if unsupported_field_message is not None:
|
|
102
|
-
|
|
232
|
+
steps.error(4, unsupported_field_message)
|
|
233
|
+
return _attach_execution_steps(_business_metric_clarification_result(unsupported_field_message), steps)
|
|
103
234
|
business_clarification = _business_metric_clarification(sql_question, schema)
|
|
104
235
|
if business_clarification is not None:
|
|
105
|
-
|
|
106
|
-
|
|
236
|
+
steps.error(4, "业务指标口径不明确,需要用户先确认指标、时间、状态和聚合维度。")
|
|
237
|
+
return _attach_execution_steps(_business_metric_clarification_result(business_clarification), steps)
|
|
107
238
|
repaired_sql: str | None = None
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
if
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
if generated.sql == "" and repaired_sql is None:
|
|
154
|
-
return _query_only_result()
|
|
155
|
-
if fast_mode and generated.sql == "" and repaired_sql is None:
|
|
156
|
-
return _query_only_result()
|
|
157
|
-
if not fast_mode and generated.sql == "" and repaired_sql is None:
|
|
158
|
-
return _query_only_result()
|
|
159
|
-
elif cached_sql is None:
|
|
160
|
-
schema_context = None
|
|
161
|
-
generated = _generate_rule_based_sql(
|
|
239
|
+
steps.running(4)
|
|
240
|
+
try:
|
|
241
|
+
schema_context = retrieve_schema_context(
|
|
242
|
+
schema,
|
|
243
|
+
sql_question,
|
|
244
|
+
)
|
|
245
|
+
except Exception as exc:
|
|
246
|
+
reason = f"{type(exc).__name__}: {exc}"
|
|
247
|
+
steps.error(4, reason)
|
|
248
|
+
return _attach_execution_steps(_query_only_result(reason), steps)
|
|
249
|
+
steps.done(4, _schema_context_step_detail(schema_context))
|
|
250
|
+
if not _schema_context_has_semantic_match(sql_question, schema_context):
|
|
251
|
+
reason = "schemaContext 未匹配到可用表、字段、指标、概念或示例。"
|
|
252
|
+
steps.error(4, reason)
|
|
253
|
+
return _attach_execution_steps(_schema_no_match_result(sql_question), steps)
|
|
254
|
+
steps.running(5)
|
|
255
|
+
searched_entities = self._search_entities_for_question(
|
|
256
|
+
context,
|
|
257
|
+
sql_question,
|
|
258
|
+
schema_context,
|
|
259
|
+
schema,
|
|
260
|
+
sampler=sampler,
|
|
261
|
+
analysis=analysis,
|
|
262
|
+
)
|
|
263
|
+
steps.done(5, _entity_search_step_detail(searched_entities))
|
|
264
|
+
steps.running(6)
|
|
265
|
+
query_plan = _build_query_plan(
|
|
266
|
+
sql_question,
|
|
267
|
+
schema_context,
|
|
268
|
+
schema,
|
|
269
|
+
searched_entities=searched_entities,
|
|
270
|
+
sampler=sampler,
|
|
271
|
+
time_range_question=time_range_question,
|
|
272
|
+
analysis=analysis,
|
|
273
|
+
)
|
|
274
|
+
steps.done(6, "已生成查询计划并直接执行。")
|
|
275
|
+
sql_schema_context = (
|
|
276
|
+
narrow_schema_context_for_query_plan(schema_context, query_plan, full_schema=schema)
|
|
277
|
+
if query_plan
|
|
278
|
+
else schema_context
|
|
279
|
+
)
|
|
280
|
+
generator: SamplingSqlGenerator | None = None
|
|
281
|
+
steps.running(7)
|
|
282
|
+
if sampler is None:
|
|
283
|
+
generated = _generate_from_schema_example(
|
|
162
284
|
question=sql_question,
|
|
163
|
-
|
|
285
|
+
schema_context=sql_schema_context,
|
|
286
|
+
)
|
|
287
|
+
self.audit_logger.log(
|
|
288
|
+
{
|
|
289
|
+
"sessionId": context.session_id,
|
|
290
|
+
"question": sql_question,
|
|
291
|
+
"tool": "schema_example_fallback",
|
|
292
|
+
"matched": generated is not None,
|
|
293
|
+
"exampleCount": len(schema_context.examples),
|
|
294
|
+
"sqlHash": sql_hash(generated.sql) if generated is not None else None,
|
|
295
|
+
}
|
|
296
|
+
)
|
|
297
|
+
if generated is None:
|
|
298
|
+
reason = "未提供 sampler,且 schema examples 没有匹配到可复用 SQL。"
|
|
299
|
+
steps.error(7, reason)
|
|
300
|
+
return _attach_execution_steps(_query_only_result(reason), steps)
|
|
301
|
+
steps.done(7, "已从 schema examples 匹配 SQL。")
|
|
302
|
+
else:
|
|
303
|
+
generator = SamplingSqlGenerator(
|
|
304
|
+
sampler=sampler,
|
|
164
305
|
default_limit=self.config.sql.default_limit,
|
|
165
306
|
max_limit=self.config.sql.max_limit,
|
|
307
|
+
confirmed_query_plan=query_plan,
|
|
166
308
|
)
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
309
|
+
generated, repaired_sql, generation_repair_attempted = _generate_with_sampler(
|
|
310
|
+
generator=generator,
|
|
311
|
+
question=sql_question,
|
|
312
|
+
schema_context=sql_schema_context,
|
|
313
|
+
audit_logger=self.audit_logger,
|
|
314
|
+
session_id=context.session_id,
|
|
315
|
+
max_repair_attempts=self.config.sql.max_repair_attempts,
|
|
316
|
+
schema=schema,
|
|
317
|
+
)
|
|
318
|
+
generation_clarification = _sql_generation_clarification_result(
|
|
319
|
+
sql_question,
|
|
320
|
+
generated,
|
|
321
|
+
query_plan=query_plan,
|
|
322
|
+
)
|
|
323
|
+
if generation_clarification is not None:
|
|
324
|
+
steps.waiting(7, "LLM 判断需要用户补充信息。")
|
|
325
|
+
return _attach_execution_steps(generation_clarification, steps)
|
|
326
|
+
if generated.sql == "" and repaired_sql is None:
|
|
327
|
+
if generation_repair_attempted:
|
|
328
|
+
reason = "LLM SQL 生成失败,自动修复后仍未返回可用 SELECT SQL。"
|
|
329
|
+
steps.error(7, reason)
|
|
330
|
+
return _attach_execution_steps(_query_only_result(reason), steps)
|
|
331
|
+
repaired_sql, fail_reason = _repair_sql_with_llm_retries(
|
|
332
|
+
generator=generator,
|
|
333
|
+
question=sql_question,
|
|
334
|
+
sql="",
|
|
335
|
+
validation_error={
|
|
336
|
+
"code": "EMPTY_SQL",
|
|
337
|
+
"message": "schemaContext 已提供相关表、JOIN、枚举或示例时,不要直接返回空 SQL;请基于当前 schemaContext 生成完整 SELECT SQL。",
|
|
338
|
+
"originalSql": "",
|
|
339
|
+
},
|
|
340
|
+
schema_context=sql_schema_context,
|
|
341
|
+
audit_logger=self.audit_logger,
|
|
342
|
+
session_id=context.session_id,
|
|
343
|
+
max_attempts=self.config.sql.max_repair_attempts,
|
|
344
|
+
default_limit=self.config.sql.default_limit,
|
|
345
|
+
max_limit=self.config.sql.max_limit,
|
|
346
|
+
schema=schema,
|
|
347
|
+
check=lambda candidate: _require_non_empty_select(candidate),
|
|
348
|
+
steps=steps,
|
|
349
|
+
running_detail="LLM 返回空 SQL,尝试修复",
|
|
350
|
+
failure_detail="LLM 返回空 SQL,修复失败",
|
|
351
|
+
)
|
|
352
|
+
if repaired_sql is None:
|
|
353
|
+
steps.error(9, fail_reason)
|
|
354
|
+
return _attach_execution_steps(_query_only_result(fail_reason), steps)
|
|
355
|
+
steps.done(7, "已通过 LLM 生成 SQL。")
|
|
356
|
+
candidate_sql = repaired_sql or _finalize_candidate_sql(
|
|
171
357
|
generated.sql,
|
|
172
358
|
sql_question,
|
|
173
359
|
default_limit=self.config.sql.default_limit,
|
|
174
360
|
max_limit=self.config.sql.max_limit,
|
|
361
|
+
schema=schema,
|
|
175
362
|
)
|
|
176
|
-
if
|
|
363
|
+
if query_plan:
|
|
364
|
+
candidate_sql = apply_time_range_filters_from_plan(candidate_sql, query_plan)
|
|
365
|
+
if schema is not None:
|
|
366
|
+
candidate_sql = apply_time_range_filters_from_question(candidate_sql, time_range_question, schema)
|
|
367
|
+
if not sql_satisfies_time_range(candidate_sql, time_range_question, schema, query_plan=query_plan):
|
|
177
368
|
if sampler is None or schema_context is None or generator is None:
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
sql_question,
|
|
369
|
+
reason = "SQL 缺少用户问题要求的时间范围过滤,且当前没有 sampler 可修复。"
|
|
370
|
+
steps.error(8, reason)
|
|
371
|
+
return _attach_execution_steps(_query_only_result(reason), steps)
|
|
372
|
+
repaired_sql, fail_reason = _repair_sql_with_llm_retries(
|
|
373
|
+
generator=generator,
|
|
374
|
+
question=sql_question,
|
|
375
|
+
sql=candidate_sql,
|
|
376
|
+
validation_error={
|
|
377
|
+
"code": "MISSING_TIME_RANGE_FILTER",
|
|
378
|
+
"message": "用户问题包含时间范围,SQL 必须在 WHERE 条件中直接包含对应时间字段过滤。",
|
|
379
|
+
"originalSql": candidate_sql,
|
|
380
|
+
},
|
|
381
|
+
schema_context=sql_schema_context,
|
|
382
|
+
audit_logger=self.audit_logger,
|
|
383
|
+
session_id=context.session_id,
|
|
384
|
+
max_attempts=self.config.sql.max_repair_attempts,
|
|
195
385
|
default_limit=self.config.sql.default_limit,
|
|
196
386
|
max_limit=self.config.sql.max_limit,
|
|
387
|
+
schema=schema,
|
|
388
|
+
check=lambda candidate: _require_time_range(candidate, time_range_question, schema, query_plan),
|
|
389
|
+
steps=steps,
|
|
390
|
+
running_detail="SQL 缺少时间范围过滤,尝试修复",
|
|
391
|
+
failure_detail="SQL 缺少时间范围过滤,修复失败",
|
|
197
392
|
)
|
|
393
|
+
if repaired_sql is None:
|
|
394
|
+
steps.error(9, fail_reason)
|
|
395
|
+
return _attach_execution_steps(_query_only_result(fail_reason), steps)
|
|
396
|
+
generated = GeneratedSql(sql="", tables=[], placeholders=[])
|
|
198
397
|
candidate_sql = repaired_sql
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
398
|
+
steps.done(9, "已补充时间范围过滤。")
|
|
399
|
+
if query_plan and not sql_satisfies_query_plan_filters(candidate_sql, query_plan):
|
|
400
|
+
missing_filters = sql_missing_query_plan_filters(candidate_sql, query_plan)
|
|
202
401
|
if sampler is None or schema_context is None or generator is None:
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
402
|
+
reason = f"SQL 缺少查询计划中的筛选条件:{'; '.join(missing_filters)}"
|
|
403
|
+
steps.error(8, reason)
|
|
404
|
+
return _attach_execution_steps(_query_only_result(reason), steps)
|
|
405
|
+
repaired_sql, fail_reason = _repair_sql_with_llm_retries(
|
|
406
|
+
generator=generator,
|
|
407
|
+
question=sql_question,
|
|
408
|
+
sql=candidate_sql,
|
|
409
|
+
validation_error={
|
|
410
|
+
"code": "MISSING_QUERY_PLAN_FILTER",
|
|
411
|
+
"message": (
|
|
412
|
+
f"SQL 缺少查询计划中的筛选条件:{'; '.join(missing_filters)}。"
|
|
413
|
+
"必须 JOIN 相关表并在 WHERE 中包含全部 table.column 过滤。"
|
|
414
|
+
),
|
|
415
|
+
"originalSql": candidate_sql,
|
|
416
|
+
"missingFilters": missing_filters,
|
|
417
|
+
},
|
|
418
|
+
schema_context=sql_schema_context,
|
|
419
|
+
audit_logger=self.audit_logger,
|
|
420
|
+
session_id=context.session_id,
|
|
421
|
+
max_attempts=self.config.sql.max_repair_attempts,
|
|
220
422
|
default_limit=self.config.sql.default_limit,
|
|
221
423
|
max_limit=self.config.sql.max_limit,
|
|
424
|
+
schema=schema,
|
|
425
|
+
check=lambda candidate: sql_satisfies_query_plan_filters(candidate, query_plan),
|
|
426
|
+
steps=steps,
|
|
427
|
+
running_detail=f"SQL 缺少查询计划筛选条件,尝试修复:{'; '.join(missing_filters)}",
|
|
428
|
+
failure_detail="SQL 缺少查询计划筛选条件,修复失败",
|
|
222
429
|
)
|
|
430
|
+
if repaired_sql is None:
|
|
431
|
+
steps.error(9, fail_reason)
|
|
432
|
+
return _attach_execution_steps(_query_only_result(fail_reason), steps)
|
|
433
|
+
generated = GeneratedSql(sql="", tables=[], placeholders=[])
|
|
223
434
|
candidate_sql = repaired_sql
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
if schema_issues:
|
|
435
|
+
steps.done(9, "已补充查询计划筛选条件。")
|
|
436
|
+
if query_plan and not sql_satisfies_query_plan_select_fields(candidate_sql, query_plan):
|
|
437
|
+
missing_select_fields = sql_missing_query_plan_select_fields(candidate_sql, query_plan)
|
|
228
438
|
if sampler is None or schema_context is None or generator is None:
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
439
|
+
reason = f"SQL 缺少查询计划中的展示字段:{'; '.join(missing_select_fields)}"
|
|
440
|
+
steps.error(8, reason)
|
|
441
|
+
return _attach_execution_steps(_query_only_result(reason), steps)
|
|
442
|
+
repaired_sql, fail_reason = _repair_sql_with_llm_retries(
|
|
443
|
+
generator=generator,
|
|
444
|
+
question=sql_question,
|
|
445
|
+
sql=candidate_sql,
|
|
446
|
+
validation_error={
|
|
447
|
+
"code": "MISSING_QUERY_PLAN_SELECT_FIELD",
|
|
448
|
+
"message": (
|
|
449
|
+
f"SQL 缺少查询计划中的展示字段:{'; '.join(missing_select_fields)}。"
|
|
450
|
+
"同一别称命中多个实体类型时,必须在 SELECT 中展示各实体名称字段,方便用户自行缩小范围。"
|
|
451
|
+
),
|
|
452
|
+
"originalSql": candidate_sql,
|
|
453
|
+
"missingSelectFields": missing_select_fields,
|
|
454
|
+
},
|
|
455
|
+
schema_context=sql_schema_context,
|
|
456
|
+
audit_logger=self.audit_logger,
|
|
457
|
+
session_id=context.session_id,
|
|
458
|
+
max_attempts=self.config.sql.max_repair_attempts,
|
|
459
|
+
default_limit=self.config.sql.default_limit,
|
|
460
|
+
max_limit=self.config.sql.max_limit,
|
|
461
|
+
schema=schema,
|
|
462
|
+
check=lambda candidate: sql_satisfies_query_plan_select_fields(candidate, query_plan),
|
|
463
|
+
steps=steps,
|
|
464
|
+
running_detail=f"SQL 缺少查询计划展示字段,尝试修复:{'; '.join(missing_select_fields)}",
|
|
465
|
+
failure_detail="SQL 缺少查询计划展示字段,修复失败",
|
|
466
|
+
)
|
|
467
|
+
if repaired_sql is None:
|
|
468
|
+
steps.error(9, fail_reason)
|
|
469
|
+
return _attach_execution_steps(_query_only_result(fail_reason), steps)
|
|
242
470
|
generated = GeneratedSql(sql="", tables=[], placeholders=[])
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
471
|
+
candidate_sql = repaired_sql
|
|
472
|
+
steps.done(9, "已补充查询计划展示字段。")
|
|
473
|
+
low_quality_issue = low_quality_sql_issue_for_question(sql_question, candidate_sql, schema=schema)
|
|
474
|
+
if low_quality_issue is not None:
|
|
475
|
+
if sampler is None or schema_context is None or generator is None:
|
|
476
|
+
steps.error(8, low_quality_issue.message)
|
|
477
|
+
return _attach_execution_steps(_low_quality_sql_result(low_quality_issue.message, candidate_sql), steps)
|
|
478
|
+
repaired_sql, fail_reason = _repair_sql_with_llm_retries(
|
|
479
|
+
generator=generator,
|
|
480
|
+
question=sql_question,
|
|
481
|
+
sql=candidate_sql,
|
|
482
|
+
validation_error={
|
|
483
|
+
"code": low_quality_issue.code,
|
|
484
|
+
"message": low_quality_issue.message,
|
|
485
|
+
"originalSql": candidate_sql,
|
|
486
|
+
},
|
|
487
|
+
schema_context=sql_schema_context,
|
|
488
|
+
audit_logger=self.audit_logger,
|
|
489
|
+
session_id=context.session_id,
|
|
490
|
+
max_attempts=self.config.sql.max_repair_attempts,
|
|
246
491
|
default_limit=self.config.sql.default_limit,
|
|
247
492
|
max_limit=self.config.sql.max_limit,
|
|
493
|
+
schema=schema,
|
|
494
|
+
check=lambda candidate: _require_low_quality_pass(candidate, sql_question, schema),
|
|
495
|
+
steps=steps,
|
|
496
|
+
running_detail=f"SQL 质量检查失败,尝试修复:{low_quality_issue.message}",
|
|
497
|
+
failure_detail=f"SQL 质量检查失败,修复失败:{low_quality_issue.message}",
|
|
248
498
|
)
|
|
499
|
+
if repaired_sql is None:
|
|
500
|
+
steps.error(9, fail_reason)
|
|
501
|
+
return _attach_execution_steps(_low_quality_sql_result(fail_reason, candidate_sql), steps)
|
|
502
|
+
generated = GeneratedSql(sql="", tables=[], placeholders=[])
|
|
249
503
|
candidate_sql = repaired_sql
|
|
250
|
-
|
|
251
|
-
|
|
504
|
+
steps.done(9, "已修复 SQL 质量问题。")
|
|
505
|
+
prior_sql = candidate_sql
|
|
506
|
+
candidate_sql, schema_fail = _repair_sql_for_local_schema_issues(
|
|
507
|
+
candidate_sql=candidate_sql,
|
|
508
|
+
sql_question=sql_question,
|
|
509
|
+
schema=schema,
|
|
510
|
+
generator=generator,
|
|
511
|
+
schema_context=schema_context,
|
|
512
|
+
sql_schema_context=sql_schema_context,
|
|
513
|
+
audit_logger=self.audit_logger,
|
|
514
|
+
session_id=context.session_id,
|
|
515
|
+
max_repair_attempts=self.config.sql.max_repair_attempts,
|
|
516
|
+
default_limit=self.config.sql.default_limit,
|
|
517
|
+
max_limit=self.config.sql.max_limit,
|
|
518
|
+
steps=steps,
|
|
519
|
+
)
|
|
520
|
+
if schema_fail:
|
|
521
|
+
steps.error(8, schema_fail)
|
|
522
|
+
return _attach_execution_steps(_query_only_result(schema_fail), steps)
|
|
523
|
+
if candidate_sql != prior_sql:
|
|
524
|
+
generated = GeneratedSql(sql="", tables=[], placeholders=[])
|
|
525
|
+
candidate_sql = _finalize_candidate_sql(
|
|
526
|
+
candidate_sql,
|
|
527
|
+
sql_question,
|
|
528
|
+
default_limit=self.config.sql.default_limit,
|
|
529
|
+
max_limit=self.config.sql.max_limit,
|
|
530
|
+
schema=schema,
|
|
531
|
+
)
|
|
252
532
|
try:
|
|
533
|
+
steps.running(8)
|
|
253
534
|
enforce_sql_rules(candidate_sql, max_limit=self.config.sql.max_limit)
|
|
535
|
+
steps.done(8, "SQL 满足只读、安全、LIMIT、中文别名等规则。")
|
|
254
536
|
except SqlGenerationError as exc:
|
|
537
|
+
if sampler is None or schema_context is None or generator is None:
|
|
538
|
+
reason = f"SQL 违反本地规则,且当前没有 sampler 可修复:{exc}"
|
|
539
|
+
steps.error(8, reason)
|
|
540
|
+
return _attach_execution_steps(_query_only_result(reason), steps)
|
|
255
541
|
if _is_sensitive_sql_error(exc) and sensitive_terms:
|
|
256
|
-
|
|
257
|
-
|
|
542
|
+
validation_error = {
|
|
543
|
+
"code": "SENSITIVE_FIELD",
|
|
544
|
+
"message": "用户要求的敏感字段必须从 SELECT 中排除;如还有其他非敏感字段,请继续生成去敏后的 SQL。",
|
|
545
|
+
"originalSql": candidate_sql,
|
|
546
|
+
"violatingColumns": build_sql_rule_validation_error(
|
|
547
|
+
candidate_sql, exc, max_limit=self.config.sql.max_limit
|
|
548
|
+
).get("violatingColumns", []),
|
|
549
|
+
}
|
|
550
|
+
running_detail = f"SQL 包含敏感字段,尝试去敏修复:{exc}"
|
|
551
|
+
failure_detail = f"SQL 包含敏感字段,去敏修复失败"
|
|
552
|
+
else:
|
|
553
|
+
validation_error = build_sql_rule_validation_error(
|
|
554
|
+
candidate_sql,
|
|
555
|
+
exc,
|
|
556
|
+
max_limit=self.config.sql.max_limit,
|
|
557
|
+
max_attempts=self.config.sql.max_repair_attempts,
|
|
558
|
+
)
|
|
559
|
+
running_detail = f"SQL 违反本地规则,尝试修复:{exc}"
|
|
560
|
+
failure_detail = "SQL 违反本地规则,修复失败"
|
|
561
|
+
repaired_sql, fail_reason = _repair_sql_with_llm_retries(
|
|
562
|
+
generator=generator,
|
|
563
|
+
question=sql_question,
|
|
564
|
+
sql=candidate_sql,
|
|
565
|
+
validation_error=validation_error,
|
|
566
|
+
schema_context=sql_schema_context,
|
|
567
|
+
audit_logger=self.audit_logger,
|
|
568
|
+
session_id=context.session_id,
|
|
569
|
+
max_attempts=self.config.sql.max_repair_attempts,
|
|
570
|
+
default_limit=self.config.sql.default_limit,
|
|
571
|
+
max_limit=self.config.sql.max_limit,
|
|
572
|
+
schema=schema,
|
|
573
|
+
check=lambda candidate: enforce_sql_rules(candidate, max_limit=self.config.sql.max_limit),
|
|
574
|
+
steps=steps,
|
|
575
|
+
running_detail=running_detail,
|
|
576
|
+
failure_detail=failure_detail,
|
|
577
|
+
)
|
|
578
|
+
if repaired_sql is None:
|
|
579
|
+
steps.error(9, fail_reason)
|
|
580
|
+
if _is_sensitive_sql_error(exc) and sensitive_terms:
|
|
581
|
+
return _attach_execution_steps(_sensitive_query_only_result(sensitive_terms, reason=fail_reason), steps)
|
|
582
|
+
return _attach_execution_steps(_query_only_result(fail_reason), steps)
|
|
583
|
+
generated = GeneratedSql(sql="", tables=[], placeholders=[])
|
|
584
|
+
candidate_sql = repaired_sql
|
|
585
|
+
steps.done(8, "SQL 修复后满足本地规则。")
|
|
586
|
+
if not any(step["status"] == "done" and step["step"] == 9 for step in steps.steps):
|
|
587
|
+
steps.skipped(9, "SQL 无需修复。")
|
|
588
|
+
prior_sql = candidate_sql
|
|
589
|
+
candidate_sql, schema_fail = _repair_sql_for_local_schema_issues(
|
|
590
|
+
candidate_sql=candidate_sql,
|
|
591
|
+
sql_question=sql_question,
|
|
592
|
+
schema=schema,
|
|
593
|
+
generator=generator,
|
|
594
|
+
schema_context=schema_context,
|
|
595
|
+
sql_schema_context=sql_schema_context,
|
|
596
|
+
audit_logger=self.audit_logger,
|
|
597
|
+
session_id=context.session_id,
|
|
598
|
+
max_repair_attempts=self.config.sql.max_repair_attempts,
|
|
599
|
+
default_limit=self.config.sql.default_limit,
|
|
600
|
+
max_limit=self.config.sql.max_limit,
|
|
601
|
+
steps=steps,
|
|
602
|
+
repair_success_detail="validate_sql 前已修复本地 schema 校验问题。",
|
|
603
|
+
running_prefix="validate_sql 前本地 schema 校验失败,尝试修复",
|
|
604
|
+
)
|
|
605
|
+
if schema_fail:
|
|
606
|
+
steps.error(10, schema_fail)
|
|
607
|
+
return _attach_execution_steps(_query_only_result(schema_fail), steps)
|
|
608
|
+
if candidate_sql != prior_sql:
|
|
609
|
+
generated = GeneratedSql(sql="", tables=[], placeholders=[])
|
|
610
|
+
prior_sql = candidate_sql
|
|
611
|
+
if query_plan:
|
|
612
|
+
candidate_sql = apply_time_range_filters_from_plan(candidate_sql, query_plan)
|
|
613
|
+
if schema is not None:
|
|
614
|
+
candidate_sql = apply_time_range_filters_from_question(candidate_sql, time_range_question, schema)
|
|
615
|
+
if candidate_sql != prior_sql:
|
|
616
|
+
candidate_sql, schema_fail = _repair_sql_for_local_schema_issues(
|
|
617
|
+
candidate_sql=candidate_sql,
|
|
618
|
+
sql_question=sql_question,
|
|
619
|
+
schema=schema,
|
|
620
|
+
generator=generator,
|
|
621
|
+
schema_context=schema_context,
|
|
622
|
+
sql_schema_context=sql_schema_context,
|
|
623
|
+
audit_logger=self.audit_logger,
|
|
624
|
+
session_id=context.session_id,
|
|
625
|
+
max_repair_attempts=self.config.sql.max_repair_attempts,
|
|
626
|
+
default_limit=self.config.sql.default_limit,
|
|
627
|
+
max_limit=self.config.sql.max_limit,
|
|
628
|
+
steps=steps,
|
|
629
|
+
repair_success_detail="补充时间范围后已修复本地 schema 校验问题。",
|
|
630
|
+
running_prefix="补充时间范围后本地 schema 校验失败,尝试修复",
|
|
631
|
+
)
|
|
632
|
+
if schema_fail:
|
|
633
|
+
steps.error(10, schema_fail)
|
|
634
|
+
return _attach_execution_steps(_query_only_result(schema_fail), steps)
|
|
635
|
+
generated = GeneratedSql(sql="", tables=[], placeholders=[])
|
|
636
|
+
candidate_sql = _remove_noise_name_like_filters_from_sql(candidate_sql, analysis=analysis)
|
|
637
|
+
if candidate_sql != prior_sql:
|
|
638
|
+
generated = GeneratedSql(sql="", tables=[], placeholders=[])
|
|
639
|
+
repaired_sql = candidate_sql
|
|
640
|
+
steps.running(10)
|
|
641
|
+
try:
|
|
642
|
+
normalized_sql = self._validate_sql(context, user_question, candidate_sql)
|
|
643
|
+
except Exception as exc:
|
|
644
|
+
reason = f"{type(exc).__name__}: {exc}"
|
|
645
|
+
steps.error(10, reason)
|
|
646
|
+
return _attach_execution_steps(_query_only_result(reason), steps)
|
|
647
|
+
if normalized_sql is None and _is_auth_validation_error(self._validation_error_code()):
|
|
648
|
+
reason = _validation_error_message(self._last_validation_result) or "MCP 用户认证失败。"
|
|
649
|
+
steps.error(10, reason)
|
|
650
|
+
return _attach_execution_steps(
|
|
651
|
+
_auth_error_result(
|
|
652
|
+
self._last_validation_result,
|
|
653
|
+
generated_sql=generated.sql,
|
|
654
|
+
repaired_sql=repaired_sql,
|
|
655
|
+
),
|
|
656
|
+
steps,
|
|
657
|
+
)
|
|
658
|
+
if normalized_sql is None and self._validation_error_code() in _LEGACY_EMPTY_SCOPE_CODES:
|
|
659
|
+
# Legacy online-version compatibility only (see _LEGACY_EMPTY_SCOPE_CODES):
|
|
660
|
+
# the current MCP contract returns an empty success result for no-scope
|
|
661
|
+
# users, which the normal flow already renders. Older deployments still
|
|
662
|
+
# emit these codes, so treat them as an empty result instead of raising.
|
|
663
|
+
steps.done(10, "validate_sql 返回旧版空权限错误,按空结果兼容处理。")
|
|
664
|
+
return _attach_execution_steps(_empty_scope_result(user_question, result_format), steps)
|
|
665
|
+
if sampler is not None and normalized_sql is None and schema_context is not None and generator is not None:
|
|
666
|
+
current_sql = candidate_sql
|
|
667
|
+
validation_error: dict[str, Any] = dict(self._last_validation_error or {})
|
|
668
|
+
validation_error["originalSql"] = current_sql
|
|
669
|
+
validate_message = _validation_error_message(self._last_validation_result)
|
|
670
|
+
last_reason = f"validate_sql 失败,修复失败:{validate_message}"
|
|
671
|
+
for attempt in range(1, self.config.sql.max_repair_attempts + 1):
|
|
672
|
+
steps.running(9, f"validate_sql 失败,尝试修复:{validate_message}(第 {attempt}/{self.config.sql.max_repair_attempts} 次)")
|
|
673
|
+
repair_error = {
|
|
674
|
+
**validation_error,
|
|
675
|
+
"attempt": attempt,
|
|
676
|
+
"maxAttempts": self.config.sql.max_repair_attempts,
|
|
677
|
+
"originalSql": current_sql,
|
|
678
|
+
}
|
|
258
679
|
try:
|
|
259
680
|
repaired = generator.repair(
|
|
260
681
|
question=sql_question,
|
|
261
|
-
original_sql=
|
|
262
|
-
validation_error=
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
682
|
+
original_sql=current_sql,
|
|
683
|
+
validation_error=repair_error,
|
|
684
|
+
schema_context=sql_schema_context,
|
|
685
|
+
)
|
|
686
|
+
except SqlGenerationError as exc:
|
|
687
|
+
last_reason = f"validate_sql 失败,LLM 修复调用失败(第 {attempt}/{self.config.sql.max_repair_attempts} 次):{exc}"
|
|
688
|
+
_log_llm_sql_repair_audit(
|
|
689
|
+
self.audit_logger,
|
|
690
|
+
session_id=context.session_id,
|
|
691
|
+
question=sql_question,
|
|
692
|
+
success=False,
|
|
693
|
+
error=str(exc),
|
|
694
|
+
attempt=attempt,
|
|
267
695
|
)
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
repaired_sql = _apply_limit_policy(
|
|
696
|
+
continue
|
|
697
|
+
repaired_sql = _finalize_candidate_sql(
|
|
271
698
|
repaired.sql,
|
|
272
699
|
sql_question,
|
|
273
700
|
default_limit=self.config.sql.default_limit,
|
|
274
701
|
max_limit=self.config.sql.max_limit,
|
|
702
|
+
schema=schema,
|
|
703
|
+
)
|
|
704
|
+
if query_plan:
|
|
705
|
+
repaired_sql = apply_time_range_filters_from_plan(repaired_sql, query_plan)
|
|
706
|
+
if schema is not None:
|
|
707
|
+
repaired_sql = apply_time_range_filters_from_question(repaired_sql, time_range_question, schema)
|
|
708
|
+
_log_llm_sql_repair_audit(
|
|
709
|
+
self.audit_logger,
|
|
710
|
+
session_id=context.session_id,
|
|
711
|
+
question=sql_question,
|
|
712
|
+
success=bool(repaired_sql.strip()),
|
|
713
|
+
sql=repaired_sql if repaired_sql.strip() else None,
|
|
714
|
+
error=None if repaired_sql.strip() else "empty_sql",
|
|
715
|
+
attempt=attempt,
|
|
275
716
|
)
|
|
717
|
+
if not repaired_sql.strip():
|
|
718
|
+
last_reason = f"validate_sql 修复返回空 SQL(第 {attempt}/{self.config.sql.max_repair_attempts} 次)"
|
|
719
|
+
current_sql = repaired_sql
|
|
720
|
+
continue
|
|
276
721
|
try:
|
|
277
722
|
enforce_sql_rules(repaired_sql, max_limit=self.config.sql.max_limit)
|
|
278
|
-
except SqlGenerationError as
|
|
279
|
-
if _is_sensitive_sql_error(
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
if not fast_mode:
|
|
288
|
-
return _query_only_result()
|
|
289
|
-
try:
|
|
290
|
-
generated = _generate_rule_based_sql(
|
|
291
|
-
question=sql_question,
|
|
292
|
-
schema=schema,
|
|
293
|
-
default_limit=self.config.sql.default_limit,
|
|
294
|
-
max_limit=self.config.sql.max_limit,
|
|
295
|
-
)
|
|
296
|
-
repaired_sql = None
|
|
297
|
-
candidate_sql = _apply_limit_policy(
|
|
298
|
-
generated.sql,
|
|
299
|
-
sql_question,
|
|
300
|
-
default_limit=self.config.sql.default_limit,
|
|
723
|
+
except SqlGenerationError as exc:
|
|
724
|
+
if _is_sensitive_sql_error(exc) and sensitive_terms:
|
|
725
|
+
reason = f"validate_sql 修复后包含敏感字段:{exc}"
|
|
726
|
+
steps.error(9, reason)
|
|
727
|
+
return _attach_execution_steps(_sensitive_query_only_result(sensitive_terms, reason=reason), steps)
|
|
728
|
+
last_reason = f"validate_sql 修复后违反本地规则:{exc}"
|
|
729
|
+
validation_error = build_sql_rule_validation_error(
|
|
730
|
+
repaired_sql,
|
|
731
|
+
exc,
|
|
301
732
|
max_limit=self.config.sql.max_limit,
|
|
733
|
+
attempt=attempt,
|
|
734
|
+
max_attempts=self.config.sql.max_repair_attempts,
|
|
302
735
|
)
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
return _query_only_result()
|
|
306
|
-
normalized_sql = self._validate_sql(context, user_question, candidate_sql)
|
|
307
|
-
if sampler is not None and normalized_sql is None and schema_context is not None and generator is not None:
|
|
308
|
-
try:
|
|
309
|
-
repaired = generator.repair(
|
|
310
|
-
question=sql_question,
|
|
311
|
-
original_sql=generated.sql,
|
|
312
|
-
validation_error=self._last_validation_error,
|
|
313
|
-
schema_context=schema_context,
|
|
314
|
-
)
|
|
315
|
-
except SqlGenerationError:
|
|
736
|
+
current_sql = repaired_sql
|
|
737
|
+
continue
|
|
316
738
|
try:
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
739
|
+
_require_no_schema_issues(repaired_sql, schema)
|
|
740
|
+
except SqlGenerationError as exc:
|
|
741
|
+
schema_issues = validate_sql_against_schema(repaired_sql, schema)
|
|
742
|
+
last_reason = f"validate_sql 修复后本地 schema 校验失败:{exc}"
|
|
743
|
+
validation_error = build_schema_validation_error(
|
|
744
|
+
repaired_sql,
|
|
745
|
+
schema_issues,
|
|
746
|
+
schema,
|
|
747
|
+
attempt=attempt,
|
|
748
|
+
max_attempts=self.config.sql.max_repair_attempts,
|
|
324
749
|
)
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
750
|
+
current_sql = repaired_sql
|
|
751
|
+
continue
|
|
752
|
+
generated = GeneratedSql(sql="", tables=[], placeholders=[])
|
|
753
|
+
try:
|
|
754
|
+
normalized_sql = self._validate_sql(context, user_question, repaired_sql)
|
|
755
|
+
except Exception as exc:
|
|
756
|
+
reason = f"{type(exc).__name__}: {exc}"
|
|
757
|
+
steps.error(10, reason)
|
|
758
|
+
return _attach_execution_steps(_query_only_result(reason), steps)
|
|
759
|
+
if normalized_sql is not None:
|
|
760
|
+
candidate_sql = repaired_sql
|
|
761
|
+
steps.done(9, f"已根据 validate_sql 错误修复 SQL(第 {attempt}/{self.config.sql.max_repair_attempts} 次)。")
|
|
762
|
+
break
|
|
763
|
+
if _is_auth_validation_error(self._validation_error_code()):
|
|
764
|
+
reason = _validation_error_message(self._last_validation_result) or "MCP 用户认证失败。"
|
|
765
|
+
steps.error(10, reason)
|
|
766
|
+
return _attach_execution_steps(
|
|
767
|
+
_auth_error_result(
|
|
768
|
+
self._last_validation_result,
|
|
769
|
+
generated_sql=generated.sql,
|
|
770
|
+
repaired_sql=repaired_sql,
|
|
771
|
+
),
|
|
772
|
+
steps,
|
|
331
773
|
)
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
774
|
+
validate_message = _validation_error_message(self._last_validation_result)
|
|
775
|
+
last_reason = f"validate_sql 修复后仍失败:{validate_message}"
|
|
776
|
+
validation_error = dict(self._last_validation_error or {})
|
|
777
|
+
validation_error["originalSql"] = repaired_sql
|
|
778
|
+
current_sql = repaired_sql
|
|
336
779
|
else:
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
780
|
+
steps.error(9, last_reason)
|
|
781
|
+
steps.error(10, last_reason)
|
|
782
|
+
return _attach_execution_steps(_query_only_result(last_reason), steps)
|
|
783
|
+
if normalized_sql is None:
|
|
784
|
+
reason = f"SQL validation failed: {_validation_error_message(self._last_validation_result)}"
|
|
785
|
+
steps.error(10, reason)
|
|
786
|
+
return _attach_execution_steps(_query_only_result(reason), steps)
|
|
787
|
+
steps.done(10, "validate_sql 通过。")
|
|
788
|
+
steps.running(11)
|
|
789
|
+
try:
|
|
790
|
+
result = self._execute_sql(context, user_question, normalized_sql)
|
|
791
|
+
except Exception as exc:
|
|
792
|
+
unknown_col = _parse_execute_unknown_column(str(exc))
|
|
793
|
+
if (
|
|
794
|
+
unknown_col
|
|
795
|
+
and sampler is not None
|
|
796
|
+
and generator is not None
|
|
797
|
+
and schema_context is not None
|
|
798
|
+
):
|
|
799
|
+
exec_error = build_execute_unknown_column_error(
|
|
800
|
+
normalized_sql,
|
|
801
|
+
unknown_col,
|
|
802
|
+
schema,
|
|
803
|
+
max_attempts=self.config.sql.max_repair_attempts,
|
|
804
|
+
)
|
|
805
|
+
repaired_exec_sql, fail_reason = _repair_sql_for_local_schema_issues(
|
|
806
|
+
candidate_sql=normalized_sql,
|
|
807
|
+
sql_question=sql_question,
|
|
808
|
+
schema=schema,
|
|
809
|
+
generator=generator,
|
|
810
|
+
schema_context=schema_context,
|
|
811
|
+
sql_schema_context=sql_schema_context,
|
|
812
|
+
audit_logger=self.audit_logger,
|
|
813
|
+
session_id=context.session_id,
|
|
814
|
+
max_repair_attempts=self.config.sql.max_repair_attempts,
|
|
341
815
|
default_limit=self.config.sql.default_limit,
|
|
342
816
|
max_limit=self.config.sql.max_limit,
|
|
817
|
+
steps=steps,
|
|
818
|
+
repair_success_detail=f"已根据 execute 未知列 {unknown_col} 修复 SQL。",
|
|
819
|
+
running_prefix=f"execute_sql 报未知列 {unknown_col},尝试修复",
|
|
820
|
+
failure_prefix=f"execute_sql 未知列 {unknown_col} 修复失败",
|
|
821
|
+
validation_error=exec_error,
|
|
343
822
|
)
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
823
|
+
if repaired_exec_sql is not None:
|
|
824
|
+
prior_exec_sql = repaired_exec_sql
|
|
825
|
+
if query_plan:
|
|
826
|
+
repaired_exec_sql = apply_time_range_filters_from_plan(repaired_exec_sql, query_plan)
|
|
827
|
+
if schema is not None:
|
|
828
|
+
repaired_exec_sql = apply_time_range_filters_from_question(repaired_exec_sql, time_range_question, schema)
|
|
829
|
+
if repaired_exec_sql != prior_exec_sql:
|
|
830
|
+
repaired_exec_sql, schema_fail = _repair_sql_for_local_schema_issues(
|
|
831
|
+
candidate_sql=repaired_exec_sql,
|
|
832
|
+
sql_question=sql_question,
|
|
833
|
+
schema=schema,
|
|
834
|
+
generator=generator,
|
|
835
|
+
schema_context=schema_context,
|
|
836
|
+
sql_schema_context=sql_schema_context,
|
|
837
|
+
audit_logger=self.audit_logger,
|
|
838
|
+
session_id=context.session_id,
|
|
839
|
+
max_repair_attempts=self.config.sql.max_repair_attempts,
|
|
840
|
+
default_limit=self.config.sql.default_limit,
|
|
841
|
+
max_limit=self.config.sql.max_limit,
|
|
842
|
+
steps=steps,
|
|
843
|
+
repair_success_detail="execute 修复补充时间范围后已修复本地 schema 校验问题。",
|
|
844
|
+
running_prefix="execute 修复补充时间范围后本地 schema 校验失败,尝试修复",
|
|
845
|
+
)
|
|
846
|
+
if schema_fail:
|
|
847
|
+
steps.error(10, schema_fail)
|
|
848
|
+
return _attach_execution_steps(_query_only_result(schema_fail), steps)
|
|
849
|
+
try:
|
|
850
|
+
revalidated = self._validate_sql(context, user_question, repaired_exec_sql)
|
|
851
|
+
except Exception as validate_exc:
|
|
852
|
+
reason = f"{type(validate_exc).__name__}: {validate_exc}"
|
|
853
|
+
steps.error(10, reason)
|
|
854
|
+
return _attach_execution_steps(_query_only_result(reason), steps)
|
|
855
|
+
if revalidated is not None:
|
|
856
|
+
try:
|
|
857
|
+
result = self._execute_sql(context, user_question, revalidated)
|
|
858
|
+
normalized_sql = revalidated
|
|
859
|
+
repaired_sql = repaired_exec_sql
|
|
860
|
+
steps.done(11, f"execute_sql 完成,rowCount={result.get('rowCount')}")
|
|
861
|
+
except Exception as retry_exc:
|
|
862
|
+
reason = f"{type(retry_exc).__name__}: {retry_exc}"
|
|
863
|
+
steps.error(11, reason)
|
|
864
|
+
return _attach_execution_steps(_query_only_result(reason), steps)
|
|
865
|
+
else:
|
|
866
|
+
reason = (
|
|
867
|
+
f"execute 未知列修复后 validate_sql 失败:"
|
|
868
|
+
f"{_validation_error_message(self._last_validation_result)}"
|
|
869
|
+
)
|
|
870
|
+
steps.error(10, reason)
|
|
871
|
+
return _attach_execution_steps(_query_only_result(reason), steps)
|
|
872
|
+
else:
|
|
873
|
+
reason = fail_reason or f"{type(exc).__name__}: {exc}"
|
|
874
|
+
steps.error(11, reason)
|
|
875
|
+
return _attach_execution_steps(_query_only_result(reason), steps)
|
|
876
|
+
else:
|
|
877
|
+
reason = f"{type(exc).__name__}: {exc}"
|
|
878
|
+
steps.error(11, reason)
|
|
879
|
+
return _attach_execution_steps(_query_only_result(reason), steps)
|
|
880
|
+
else:
|
|
881
|
+
steps.done(11, f"execute_sql 完成,rowCount={result.get('rowCount')}")
|
|
882
|
+
result = self._ensure_limited_flag(result)
|
|
883
|
+
steps.running(12)
|
|
884
|
+
try:
|
|
885
|
+
if sampler is not None:
|
|
886
|
+
answer = render_result_with_llm(
|
|
887
|
+
question=user_question,
|
|
888
|
+
result=result,
|
|
889
|
+
agent_instructions=getattr(schema_context, "agent_instructions", []) or [],
|
|
890
|
+
sampler=sampler,
|
|
891
|
+
result_format=result_format,
|
|
892
|
+
)
|
|
893
|
+
else:
|
|
894
|
+
answer = render_result(user_question, result, result_format)
|
|
895
|
+
except Exception as exc:
|
|
896
|
+
reason = f"{type(exc).__name__}: {exc}"
|
|
897
|
+
steps.error(12, reason)
|
|
898
|
+
return _attach_execution_steps(_query_only_result(reason), steps)
|
|
899
|
+
if query_plan:
|
|
900
|
+
answer = ensure_query_plan_summary_answer(answer, user_question, result_format, query_plan)
|
|
901
|
+
export_notice = _csv_export_notice(user_question, result)
|
|
902
|
+
if export_notice:
|
|
903
|
+
answer = _append_default_limit_notice(
|
|
904
|
+
answer,
|
|
905
|
+
user_question,
|
|
906
|
+
result_format,
|
|
907
|
+
export_notice,
|
|
908
|
+
)
|
|
357
909
|
if _should_append_default_limit_notice(
|
|
358
910
|
question=sql_question,
|
|
359
911
|
result=result,
|
|
@@ -372,13 +924,17 @@ class SpongeAgent:
|
|
|
372
924
|
result_format,
|
|
373
925
|
_sensitive_field_notice(sensitive_terms),
|
|
374
926
|
)
|
|
375
|
-
|
|
927
|
+
steps.done(12, "LLM 已渲染结果并返回。" if sampler is not None else "结果已渲染并返回。")
|
|
928
|
+
return _attach_execution_steps(QueryResult(
|
|
376
929
|
answer=answer,
|
|
377
930
|
generated_sql=generated.sql,
|
|
378
931
|
normalized_sql=normalized_sql,
|
|
379
932
|
repaired_sql=repaired_sql,
|
|
380
933
|
validation=self._last_validation_result,
|
|
381
|
-
|
|
934
|
+
query_plan=query_plan,
|
|
935
|
+
raw_result=result,
|
|
936
|
+
result_format=result_format,
|
|
937
|
+
), steps)
|
|
382
938
|
|
|
383
939
|
def _ensure_schema(self) -> dict[str, Any]:
|
|
384
940
|
cached_schema = self.schema_cache.load()
|
|
@@ -429,6 +985,84 @@ class SpongeAgent:
|
|
|
429
985
|
return cached_schema
|
|
430
986
|
raise RuntimeError("Sponge get_schema did not return tables and no local schema cache exists")
|
|
431
987
|
|
|
988
|
+
def _search_entities_for_question(
|
|
989
|
+
self,
|
|
990
|
+
context: UserContext,
|
|
991
|
+
question: str,
|
|
992
|
+
schema_context: Any,
|
|
993
|
+
schema: dict[str, Any],
|
|
994
|
+
*,
|
|
995
|
+
sampler: Sampler | None = None,
|
|
996
|
+
analysis: QuestionAnalysis | None = None,
|
|
997
|
+
) -> dict[str, Any] | None:
|
|
998
|
+
search_candidates, schema_candidates = _collect_entity_resolution_candidates(
|
|
999
|
+
question,
|
|
1000
|
+
schema,
|
|
1001
|
+
schema_context,
|
|
1002
|
+
sampler,
|
|
1003
|
+
agent_instructions=getattr(schema_context, "agent_instructions", []) or [],
|
|
1004
|
+
analysis=analysis,
|
|
1005
|
+
)
|
|
1006
|
+
if not search_candidates and not schema_candidates:
|
|
1007
|
+
return None
|
|
1008
|
+
|
|
1009
|
+
search_results: list[dict[str, Any]] = []
|
|
1010
|
+
for candidate in search_candidates:
|
|
1011
|
+
entity_type = str(candidate.get("type") or "")
|
|
1012
|
+
name = str(candidate.get("value") or "")
|
|
1013
|
+
if not entity_type or not name:
|
|
1014
|
+
continue
|
|
1015
|
+
trace_id = new_trace_id("trace_entity")
|
|
1016
|
+
try:
|
|
1017
|
+
response = self.mcp_client.search_entities(
|
|
1018
|
+
trace_id=trace_id,
|
|
1019
|
+
session_id=context.session_id,
|
|
1020
|
+
entity_types=[entity_type],
|
|
1021
|
+
name=name,
|
|
1022
|
+
)
|
|
1023
|
+
except AttributeError:
|
|
1024
|
+
break
|
|
1025
|
+
except Exception as exc:
|
|
1026
|
+
self.audit_logger.log(
|
|
1027
|
+
{
|
|
1028
|
+
"traceId": trace_id,
|
|
1029
|
+
"sessionId": context.session_id,
|
|
1030
|
+
"question": question,
|
|
1031
|
+
"tool": "search_entities",
|
|
1032
|
+
"entityType": entity_type,
|
|
1033
|
+
"name": name,
|
|
1034
|
+
"error": f"{type(exc).__name__}: {exc}",
|
|
1035
|
+
}
|
|
1036
|
+
)
|
|
1037
|
+
continue
|
|
1038
|
+
self.audit_logger.log(
|
|
1039
|
+
{
|
|
1040
|
+
"traceId": trace_id,
|
|
1041
|
+
"sessionId": context.session_id,
|
|
1042
|
+
"question": question,
|
|
1043
|
+
"tool": "search_entities",
|
|
1044
|
+
"entityType": entity_type,
|
|
1045
|
+
"name": name,
|
|
1046
|
+
"matchedCount": len(_flatten_entity_search_items(response)),
|
|
1047
|
+
}
|
|
1048
|
+
)
|
|
1049
|
+
search_results.append(
|
|
1050
|
+
{
|
|
1051
|
+
"candidate": candidate,
|
|
1052
|
+
"searchedName": name,
|
|
1053
|
+
"entityType": entity_type,
|
|
1054
|
+
"response": response,
|
|
1055
|
+
}
|
|
1056
|
+
)
|
|
1057
|
+
return {
|
|
1058
|
+
"traceId": search_results[0]["response"].get("traceId") if search_results else "",
|
|
1059
|
+
"searchedName": search_candidates[0].get("value") or "" if search_candidates else "",
|
|
1060
|
+
"entityCandidates": search_candidates,
|
|
1061
|
+
"schemaCandidates": schema_candidates,
|
|
1062
|
+
"searchResults": search_results,
|
|
1063
|
+
"entities": _merge_entity_search_results(search_results),
|
|
1064
|
+
}
|
|
1065
|
+
|
|
432
1066
|
def _validate_sql(
|
|
433
1067
|
self,
|
|
434
1068
|
context: UserContext,
|
|
@@ -467,6 +1101,38 @@ class SpongeAgent:
|
|
|
467
1101
|
self._last_validation_trace_id = trace_id
|
|
468
1102
|
return str(result.get("normalizedSql") or sql)
|
|
469
1103
|
|
|
1104
|
+
def _validation_error_code(self) -> str | None:
|
|
1105
|
+
error = self._last_validation_error
|
|
1106
|
+
if isinstance(error, dict):
|
|
1107
|
+
code = error.get("code")
|
|
1108
|
+
return str(code) if code else None
|
|
1109
|
+
return None
|
|
1110
|
+
|
|
1111
|
+
def _ensure_limited_flag(self, result: dict[str, Any]) -> dict[str, Any]:
|
|
1112
|
+
"""Derive `limited` from the validate-stage LIMIT when execute omits it.
|
|
1113
|
+
|
|
1114
|
+
MCP execute_sql does not return a `limited` flag (alignment checklist
|
|
1115
|
+
supplementary #1); the agreed approach is to infer it client-side from
|
|
1116
|
+
the LIMIT that validate_sql parsed. The ceiling is never hardcoded here:
|
|
1117
|
+
the value always comes from MCP's validate response.
|
|
1118
|
+
"""
|
|
1119
|
+
if "limited" in result:
|
|
1120
|
+
return result
|
|
1121
|
+
limit: int | None = None
|
|
1122
|
+
if isinstance(self._last_validation_result, dict):
|
|
1123
|
+
raw_limit = self._last_validation_result.get("limit")
|
|
1124
|
+
if isinstance(raw_limit, int) and not isinstance(raw_limit, bool):
|
|
1125
|
+
limit = raw_limit
|
|
1126
|
+
if limit is None:
|
|
1127
|
+
return result
|
|
1128
|
+
row_count = result.get("rowCount")
|
|
1129
|
+
if row_count is None:
|
|
1130
|
+
rows = result.get("rows")
|
|
1131
|
+
row_count = len(rows) if isinstance(rows, list) else None
|
|
1132
|
+
if not isinstance(row_count, int):
|
|
1133
|
+
return result
|
|
1134
|
+
return {**result, "limited": row_count >= limit}
|
|
1135
|
+
|
|
470
1136
|
def _execute_sql(self, context: UserContext, question: str, sql: str) -> dict[str, Any]:
|
|
471
1137
|
if self._last_validation_trace_id is None:
|
|
472
1138
|
raise RuntimeError("SQL execution requires a prior successful validate_sql trace")
|
|
@@ -497,40 +1163,2871 @@ class SpongeAgent:
|
|
|
497
1163
|
return result
|
|
498
1164
|
|
|
499
1165
|
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
1166
|
+
class _ExecutionSteps:
|
|
1167
|
+
def __init__(self) -> None:
|
|
1168
|
+
self.steps: list[dict[str, Any]] = [
|
|
1169
|
+
{"step": 1, "name": "接收 query_sponge 调用", "status": "pending"},
|
|
1170
|
+
{"step": 2, "name": "LLM 需求分析", "status": "pending"},
|
|
1171
|
+
{"step": 3, "name": "获取 schema", "status": "pending"},
|
|
1172
|
+
{"step": 4, "name": "按读取顺序匹配 schemaContext", "status": "pending"},
|
|
1173
|
+
{"step": 5, "name": "识别并确认实体", "status": "pending"},
|
|
1174
|
+
{"step": 6, "name": "生成并确认查询计划", "status": "pending"},
|
|
1175
|
+
{"step": 7, "name": "按 policy 和 schemaContext 生成 SQL", "status": "pending"},
|
|
1176
|
+
{"step": 8, "name": "生成后按 policy 复查 SQL", "status": "pending"},
|
|
1177
|
+
{"step": 9, "name": "修复 SQL", "status": "pending"},
|
|
1178
|
+
{"step": 10, "name": "调用 validate_sql", "status": "pending"},
|
|
1179
|
+
{"step": 11, "name": "调用 execute_sql", "status": "pending"},
|
|
1180
|
+
{"step": 12, "name": "LLM 渲染并返回结果", "status": "pending"},
|
|
1181
|
+
]
|
|
512
1182
|
|
|
1183
|
+
def running(self, step: int, detail: str | None = None) -> None:
|
|
1184
|
+
self._set(step, "running", detail=detail)
|
|
513
1185
|
|
|
514
|
-
def
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
1186
|
+
def done(self, step: int, detail: str | None = None) -> None:
|
|
1187
|
+
self._set(step, "done", detail=detail)
|
|
1188
|
+
|
|
1189
|
+
def waiting(self, step: int, detail: str | None = None) -> None:
|
|
1190
|
+
self._set(step, "waiting", detail=detail)
|
|
1191
|
+
|
|
1192
|
+
def skipped(self, step: int, detail: str | None = None) -> None:
|
|
1193
|
+
self._set(step, "skipped", detail=detail)
|
|
1194
|
+
|
|
1195
|
+
def error(self, step: int, reason: str) -> None:
|
|
1196
|
+
self._set(step, "error", detail=reason, error=reason)
|
|
1197
|
+
|
|
1198
|
+
def snapshot(self) -> list[dict[str, Any]]:
|
|
1199
|
+
return [dict(step) for step in self.steps]
|
|
1200
|
+
|
|
1201
|
+
def _set(self, step: int, status: str, *, detail: str | None = None, error: str | None = None) -> None:
|
|
1202
|
+
for item in self.steps:
|
|
1203
|
+
if item["step"] != step:
|
|
1204
|
+
continue
|
|
1205
|
+
item["status"] = status
|
|
1206
|
+
if detail:
|
|
1207
|
+
item["detail"] = detail
|
|
1208
|
+
if error:
|
|
1209
|
+
item["error"] = error
|
|
1210
|
+
return
|
|
1211
|
+
|
|
1212
|
+
|
|
1213
|
+
def _attach_execution_steps(result: QueryResult, steps: _ExecutionSteps) -> QueryResult:
|
|
1214
|
+
snapshot = steps.snapshot()
|
|
1215
|
+
return QueryResult(
|
|
1216
|
+
answer=result.answer,
|
|
1217
|
+
generated_sql=result.generated_sql,
|
|
1218
|
+
normalized_sql=result.normalized_sql,
|
|
1219
|
+
repaired_sql=result.repaired_sql,
|
|
1220
|
+
validation=result.validation,
|
|
1221
|
+
needs_clarification=result.needs_clarification,
|
|
1222
|
+
clarification_type=result.clarification_type,
|
|
1223
|
+
clarification_payload=result.clarification_payload,
|
|
1224
|
+
query_plan=result.query_plan,
|
|
1225
|
+
execution_steps=snapshot,
|
|
1226
|
+
)
|
|
1227
|
+
|
|
1228
|
+
|
|
1229
|
+
def _schema_context_step_detail(schema_context: Any) -> str:
|
|
1230
|
+
parts = [
|
|
1231
|
+
("agentInstructions", getattr(schema_context, "agent_instructions", []) or []),
|
|
1232
|
+
("policy", getattr(schema_context, "policy", []) or []),
|
|
1233
|
+
("glossary", getattr(schema_context, "glossary_terms", []) or []),
|
|
1234
|
+
("schema.tables", getattr(schema_context, "tables", []) or []),
|
|
1235
|
+
("businessTerms", getattr(schema_context, "business_terms", []) or []),
|
|
1236
|
+
("concepts", getattr(schema_context, "concepts", []) or []),
|
|
1237
|
+
("metrics", getattr(schema_context, "metrics", []) or []),
|
|
1238
|
+
("tableDefaults", getattr(schema_context, "default_filters", []) or []),
|
|
1239
|
+
("joins", getattr(schema_context, "manual_joins", []) or []),
|
|
1240
|
+
("structure.relations", getattr(schema_context, "inferred_relations", []) or []),
|
|
1241
|
+
("example.joinPattern", getattr(schema_context, "example_join_patterns", []) or []),
|
|
1242
|
+
("enums", getattr(schema_context, "enums", []) or []),
|
|
1243
|
+
]
|
|
1244
|
+
return ";".join(f"{name}={len(value) if isinstance(value, list) else 0}" for name, value in parts)
|
|
1245
|
+
|
|
1246
|
+
|
|
1247
|
+
def _entity_search_step_detail(searched_entities: dict[str, Any] | None) -> str:
|
|
1248
|
+
if not searched_entities:
|
|
1249
|
+
return "未发现需要远程确认的实体。"
|
|
1250
|
+
items = _flatten_entity_search_items(searched_entities)
|
|
1251
|
+
schema_count = len(searched_entities.get("schemaCandidates") or [])
|
|
1252
|
+
return f"search_entities 返回 {len(items)} 个候选;schema 路径 {schema_count} 个实体。"
|
|
1253
|
+
|
|
1254
|
+
|
|
1255
|
+
def _sql_generation_clarification_result(
|
|
1256
|
+
question: str,
|
|
1257
|
+
generated: GeneratedSql,
|
|
1258
|
+
query_plan: dict[str, Any] | None = None,
|
|
1259
|
+
) -> QueryResult | None:
|
|
1260
|
+
clarification = generated.clarification
|
|
1261
|
+
if not isinstance(clarification, dict):
|
|
1262
|
+
return None
|
|
1263
|
+
clarification_type = str(clarification.get("type") or "")
|
|
1264
|
+
if clarification_type == "status_enum_confirmation":
|
|
1265
|
+
return None
|
|
1266
|
+
if clarification_type == "status_enum_missing":
|
|
1267
|
+
return None
|
|
1268
|
+
return None
|
|
1269
|
+
|
|
1270
|
+
|
|
1271
|
+
def _validation_error_message(validation: dict[str, Any] | None) -> str:
|
|
1272
|
+
if not isinstance(validation, dict):
|
|
1273
|
+
return ""
|
|
1274
|
+
error = validation.get("error")
|
|
1275
|
+
if not isinstance(error, dict):
|
|
1276
|
+
return ""
|
|
1277
|
+
code = str(error.get("code") or "")
|
|
1278
|
+
message = str(error.get("message") or error.get("detail") or "")
|
|
1279
|
+
if code and message:
|
|
1280
|
+
return f"{code}: {message}"
|
|
1281
|
+
return code or message
|
|
1282
|
+
|
|
1283
|
+
|
|
1284
|
+
def _build_query_plan(
|
|
1285
|
+
question: str,
|
|
1286
|
+
schema_context: Any,
|
|
1287
|
+
schema: dict[str, Any],
|
|
1288
|
+
*,
|
|
1289
|
+
searched_entities: dict[str, Any] | None = None,
|
|
1290
|
+
sampler: Sampler | None = None,
|
|
1291
|
+
time_range_question: str | None = None,
|
|
1292
|
+
analysis: QuestionAnalysis | None = None,
|
|
1293
|
+
) -> dict[str, Any]:
|
|
1294
|
+
tables = _query_plan_tables(getattr(schema_context, "tables", []) or [])
|
|
1295
|
+
fields = _query_plan_fields(question, getattr(schema_context, "tables", []) or [])
|
|
1296
|
+
# Extract open-valued entities once (schema-driven LLM), reused by both
|
|
1297
|
+
# filters and entities so the two views never diverge.
|
|
1298
|
+
question_entities = _extract_question_entities(
|
|
1299
|
+
question,
|
|
1300
|
+
schema,
|
|
1301
|
+
sampler,
|
|
1302
|
+
agent_instructions=getattr(schema_context, "agent_instructions", []) or [],
|
|
1303
|
+
analysis=analysis,
|
|
1304
|
+
)
|
|
1305
|
+
selected_concepts = match_concepts_for_question(getattr(schema_context, "concepts", []) or [], question)
|
|
1306
|
+
filters = _query_plan_filters(
|
|
1307
|
+
question,
|
|
1308
|
+
schema_context,
|
|
1309
|
+
searched_entities=searched_entities,
|
|
1310
|
+
question_entities=question_entities if searched_entities is None else None,
|
|
1311
|
+
schema=schema,
|
|
1312
|
+
time_range_question=time_range_question or question,
|
|
1313
|
+
selected_concepts=selected_concepts,
|
|
1314
|
+
analysis=analysis,
|
|
1315
|
+
)
|
|
1316
|
+
entities = _query_plan_entities(
|
|
1317
|
+
question,
|
|
1318
|
+
schema_context,
|
|
1319
|
+
searched_entities=searched_entities,
|
|
1320
|
+
question_entities=question_entities if searched_entities is None else None,
|
|
1321
|
+
analysis=analysis,
|
|
1322
|
+
)
|
|
1323
|
+
plan = {
|
|
1324
|
+
"question": question,
|
|
1325
|
+
"schemaVersion": schema.get("schemaVersion"),
|
|
1326
|
+
"steps": [
|
|
1327
|
+
{"step": 1, "name": "上层 Agent 调用 query_sponge", "status": "done"},
|
|
1328
|
+
{"step": 2, "name": "判断用户意图是查询", "status": "done"},
|
|
1329
|
+
{"step": 3, "name": "按 schema 中文语义匹配表、字段、筛选条件", "status": "done"},
|
|
1330
|
+
{"step": 4, "name": "固化查询计划并直接执行", "status": "done"},
|
|
1331
|
+
{"step": 5, "name": "按表关联关系组合单条 SQL", "status": "pending"},
|
|
1332
|
+
{"step": 6, "name": "按 schema 补充筛选条件和枚举值", "status": "pending"},
|
|
1333
|
+
{"step": 7, "name": "检查 SELECT、WHERE、LIKE、枚举和单 SQL 要求", "status": "pending"},
|
|
1334
|
+
{"step": 8, "name": "调用 validate_sql 校验 SQL", "status": "pending"},
|
|
1335
|
+
{"step": 9, "name": "同 traceId 调用 execute_sql 执行", "status": "pending"},
|
|
1336
|
+
{"step": 10, "name": "检查结果是否符合口径", "status": "pending"},
|
|
1337
|
+
{"step": 11, "name": "按指定格式渲染结果", "status": "pending"},
|
|
1338
|
+
{"step": 12, "name": "返回 content 和 structuredContent.answer", "status": "pending"},
|
|
1339
|
+
],
|
|
1340
|
+
"tables": tables,
|
|
1341
|
+
"fields": fields,
|
|
1342
|
+
"filters": filters,
|
|
1343
|
+
"entities": entities,
|
|
1344
|
+
"matchedConcepts": [
|
|
1345
|
+
{
|
|
1346
|
+
"name": str(concept.get("name") or ""),
|
|
1347
|
+
"comment": str(concept.get("comment") or ""),
|
|
1348
|
+
}
|
|
1349
|
+
for concept in selected_concepts
|
|
1350
|
+
],
|
|
1351
|
+
}
|
|
1352
|
+
_ensure_query_plan_concept_required_tables(plan, selected_concepts)
|
|
1353
|
+
_ensure_query_plan_bridge_tables(plan)
|
|
1354
|
+
_ensure_query_plan_entity_display_fields(plan)
|
|
1355
|
+
return plan
|
|
1356
|
+
|
|
1357
|
+
|
|
1358
|
+
def _ensure_query_plan_concept_required_tables(plan: dict[str, Any], concepts: list[dict[str, Any]]) -> None:
|
|
1359
|
+
"""Pin concept.requiredTables onto the query plan."""
|
|
1360
|
+
tables = plan.get("tables")
|
|
1361
|
+
if not isinstance(tables, list):
|
|
1362
|
+
tables = []
|
|
1363
|
+
plan["tables"] = tables
|
|
1364
|
+
names = {str(item.get("name") or "") for item in tables if isinstance(item, dict)}
|
|
1365
|
+
names.discard("")
|
|
1366
|
+
for concept in concepts:
|
|
1367
|
+
if not isinstance(concept, dict):
|
|
1368
|
+
continue
|
|
1369
|
+
for table_name in concept.get("requiredTables") or []:
|
|
1370
|
+
table = str(table_name or "").strip()
|
|
1371
|
+
if not table or table in names:
|
|
1372
|
+
continue
|
|
1373
|
+
tables.append(
|
|
1374
|
+
{
|
|
1375
|
+
"name": table,
|
|
1376
|
+
"description": str(concept.get("comment") or concept.get("description") or ""),
|
|
1377
|
+
"source": "schema.concepts",
|
|
1378
|
+
}
|
|
1379
|
+
)
|
|
1380
|
+
names.add(table)
|
|
1381
|
+
|
|
1382
|
+
|
|
1383
|
+
def _ensure_query_plan_bridge_tables(plan: dict[str, Any]) -> None:
|
|
1384
|
+
"""Pin bridge tables implied by entity/filter semantics onto the plan."""
|
|
1385
|
+
tables = plan.get("tables")
|
|
1386
|
+
if not isinstance(tables, list):
|
|
1387
|
+
return
|
|
1388
|
+
names = {str(item.get("name") or "") for item in tables if isinstance(item, dict)}
|
|
1389
|
+
names.discard("")
|
|
1390
|
+
job_tables = {"job", "job_basic_info", "job_address", "job_store", "job_hiring_requirement"}
|
|
1391
|
+
has_store = "store" in names
|
|
1392
|
+
if not has_store:
|
|
1393
|
+
for flt in plan.get("filters") or []:
|
|
1394
|
+
if isinstance(flt, dict) and str(flt.get("field") or "").startswith("store."):
|
|
1395
|
+
has_store = True
|
|
1396
|
+
break
|
|
1397
|
+
if not has_store:
|
|
1398
|
+
for entity in plan.get("entities") or []:
|
|
1399
|
+
if isinstance(entity, dict) and str(entity.get("type") or "") == "store":
|
|
1400
|
+
has_store = True
|
|
1401
|
+
break
|
|
1402
|
+
if has_store and names.intersection(job_tables) and "job_store" not in names:
|
|
1403
|
+
tables.append(
|
|
1404
|
+
{
|
|
1405
|
+
"name": "job_store",
|
|
1406
|
+
"description": "岗位门店关系持久化对象。 对应表:job_store。",
|
|
1407
|
+
"source": "schema.planRules",
|
|
1408
|
+
}
|
|
1409
|
+
)
|
|
1410
|
+
|
|
1411
|
+
|
|
1412
|
+
def _ensure_query_plan_entity_display_fields(plan: dict[str, Any]) -> None:
|
|
1413
|
+
"""Require display names for OR-matched entity dimensions."""
|
|
1414
|
+
fields = plan.get("fields")
|
|
1415
|
+
if not isinstance(fields, list):
|
|
1416
|
+
fields = []
|
|
1417
|
+
plan["fields"] = fields
|
|
1418
|
+
existing = {
|
|
1419
|
+
(str(item.get("table") or ""), str(item.get("field") or ""))
|
|
1420
|
+
for item in fields
|
|
1421
|
+
if isinstance(item, dict)
|
|
1422
|
+
}
|
|
1423
|
+
for flt in plan.get("filters") or []:
|
|
1424
|
+
if not isinstance(flt, dict):
|
|
1425
|
+
continue
|
|
1426
|
+
if str(flt.get("source") or "") != "search_entities":
|
|
1427
|
+
continue
|
|
1428
|
+
if not str(flt.get("logicalGroup") or ""):
|
|
1429
|
+
continue
|
|
1430
|
+
field = str(flt.get("field") or "")
|
|
1431
|
+
if "." not in field:
|
|
1432
|
+
continue
|
|
1433
|
+
table_name, column_name = field.split(".", 1)
|
|
1434
|
+
if column_name != "name":
|
|
1435
|
+
continue
|
|
1436
|
+
key = (table_name, column_name)
|
|
1437
|
+
label = _query_plan_filter_field_label(field)
|
|
1438
|
+
description = f"{label}名称" if label else "实体名称"
|
|
1439
|
+
if key in existing:
|
|
1440
|
+
for item in fields:
|
|
1441
|
+
if not isinstance(item, dict):
|
|
1442
|
+
continue
|
|
1443
|
+
if (str(item.get("table") or ""), str(item.get("field") or "")) != key:
|
|
1444
|
+
continue
|
|
1445
|
+
item["required"] = True
|
|
1446
|
+
item["source"] = "search_entities"
|
|
1447
|
+
if not item.get("description"):
|
|
1448
|
+
item["description"] = description
|
|
1449
|
+
break
|
|
1450
|
+
continue
|
|
1451
|
+
fields.append(
|
|
1452
|
+
{
|
|
1453
|
+
"table": table_name,
|
|
1454
|
+
"field": column_name,
|
|
1455
|
+
"description": description,
|
|
1456
|
+
"source": "search_entities",
|
|
1457
|
+
"required": True,
|
|
1458
|
+
}
|
|
1459
|
+
)
|
|
1460
|
+
existing.add(key)
|
|
1461
|
+
|
|
1462
|
+
|
|
1463
|
+
def _query_plan_tables(tables: list[dict[str, Any]]) -> list[dict[str, str]]:
|
|
1464
|
+
result: list[dict[str, str]] = []
|
|
1465
|
+
for table in tables[:10]:
|
|
1466
|
+
if not isinstance(table, dict):
|
|
1467
|
+
continue
|
|
1468
|
+
name = str(table.get("tableName") or table.get("name") or "")
|
|
1469
|
+
if not name:
|
|
1470
|
+
continue
|
|
1471
|
+
result.append(
|
|
1472
|
+
{
|
|
1473
|
+
"name": name,
|
|
1474
|
+
"description": str(table.get("comment") or table.get("description") or ""),
|
|
1475
|
+
"source": "schema.tables",
|
|
1476
|
+
}
|
|
1477
|
+
)
|
|
1478
|
+
return result
|
|
1479
|
+
|
|
1480
|
+
|
|
1481
|
+
def _question_requests_id_field(question: str) -> bool:
|
|
1482
|
+
lowered = question.lower()
|
|
1483
|
+
return "id" in lowered or "编号" in question or "主键" in question
|
|
1484
|
+
|
|
1485
|
+
|
|
1486
|
+
def _is_query_plan_id_column(column_name: str) -> bool:
|
|
1487
|
+
lowered = column_name.lower()
|
|
1488
|
+
return lowered == "id" or lowered.endswith("_id")
|
|
1489
|
+
|
|
1490
|
+
|
|
1491
|
+
def _is_query_plan_display_column(column_name: str) -> bool:
|
|
1492
|
+
lowered = column_name.lower()
|
|
1493
|
+
if lowered in {"name", "info", "title", "text", "remark", "remarks", "desc", "description", "job_name", "nick_name", "status"}:
|
|
1494
|
+
return True
|
|
1495
|
+
return lowered.endswith("_name") or lowered.endswith("_info") or lowered.endswith("_text")
|
|
1496
|
+
|
|
1497
|
+
|
|
1498
|
+
def _query_plan_field_description(column: dict[str, Any], description: str) -> str:
|
|
1499
|
+
enum = column.get("enum")
|
|
1500
|
+
if not isinstance(enum, dict):
|
|
1501
|
+
return description
|
|
1502
|
+
values = enum.get("values")
|
|
1503
|
+
if not isinstance(values, list):
|
|
1504
|
+
return description
|
|
1505
|
+
labels = [str(item.get("label") or "").strip() for item in values if isinstance(item, dict) and item.get("label")]
|
|
1506
|
+
if not labels:
|
|
1507
|
+
return description
|
|
1508
|
+
preview = "、".join(labels[:5])
|
|
1509
|
+
if len(labels) > 5:
|
|
1510
|
+
preview += "..."
|
|
1511
|
+
suffix = f"(枚举:{preview})"
|
|
1512
|
+
return f"{description}{suffix}" if description else suffix
|
|
1513
|
+
|
|
1514
|
+
|
|
1515
|
+
def _query_plan_fields(question: str, tables: list[dict[str, Any]]) -> list[dict[str, str]]:
|
|
1516
|
+
allow_id = _question_requests_id_field(question)
|
|
1517
|
+
tokens = _query_plan_tokens(question)
|
|
1518
|
+
matched: list[tuple[int, dict[str, str]]] = []
|
|
1519
|
+
fallback: list[dict[str, str]] = []
|
|
1520
|
+
for table in tables:
|
|
1521
|
+
if not isinstance(table, dict):
|
|
1522
|
+
continue
|
|
1523
|
+
table_name = str(table.get("tableName") or table.get("name") or "")
|
|
1524
|
+
columns = table.get("columns", [])
|
|
1525
|
+
if not isinstance(columns, list):
|
|
1526
|
+
continue
|
|
1527
|
+
for column in columns:
|
|
1528
|
+
if not isinstance(column, dict):
|
|
1529
|
+
continue
|
|
1530
|
+
column_name = str(column.get("columnName") or column.get("name") or "")
|
|
1531
|
+
if not column_name:
|
|
1532
|
+
continue
|
|
1533
|
+
description = _query_plan_field_description(
|
|
1534
|
+
column,
|
|
1535
|
+
str(column.get("comment") or column.get("description") or ""),
|
|
1536
|
+
)
|
|
1537
|
+
field = {
|
|
1538
|
+
"table": table_name,
|
|
1539
|
+
"field": column_name,
|
|
1540
|
+
"description": description,
|
|
1541
|
+
"source": "schema.columns",
|
|
1542
|
+
}
|
|
1543
|
+
text = f"{table_name} {column_name} {description}".lower()
|
|
1544
|
+
score = sum(1 for token in tokens if token and token in text)
|
|
1545
|
+
if score > 0:
|
|
1546
|
+
if _is_query_plan_display_column(column_name):
|
|
1547
|
+
score += 2
|
|
1548
|
+
if not allow_id and _is_query_plan_id_column(column_name):
|
|
1549
|
+
score -= 50
|
|
1550
|
+
if score > 0:
|
|
1551
|
+
matched.append((score, field))
|
|
1552
|
+
elif not allow_id and _is_query_plan_id_column(column_name):
|
|
1553
|
+
continue
|
|
1554
|
+
elif column_name in {"name", "job_name", "status", "info", "create_at", "created_at"} or _is_query_plan_display_column(column_name):
|
|
1555
|
+
fallback.append(field)
|
|
1556
|
+
elif allow_id and column_name == "id":
|
|
1557
|
+
fallback.append(field)
|
|
1558
|
+
matched.sort(key=lambda item: item[0], reverse=True)
|
|
1559
|
+
fields = [field for _, field in matched[:12]]
|
|
1560
|
+
if not fields:
|
|
1561
|
+
fields = fallback[:8]
|
|
1562
|
+
return fields
|
|
1563
|
+
|
|
1564
|
+
|
|
1565
|
+
def _schema_context_has_semantic_match(question: str, schema_context: Any) -> bool:
|
|
1566
|
+
if any(getattr(schema_context, attr, []) for attr in ("business_terms", "concepts", "metrics", "examples")):
|
|
1567
|
+
return True
|
|
1568
|
+
tokens = _schema_match_tokens(question)
|
|
1569
|
+
if not tokens:
|
|
1570
|
+
return bool(getattr(schema_context, "tables", []) or [])
|
|
1571
|
+
for table in getattr(schema_context, "tables", []) or []:
|
|
1572
|
+
if not isinstance(table, dict):
|
|
1573
|
+
continue
|
|
1574
|
+
text = _schema_table_search_text(table)
|
|
1575
|
+
if any(token in text for token in tokens):
|
|
1576
|
+
return True
|
|
1577
|
+
if getattr(schema_context, "tables", None) and _has_query_intent(question):
|
|
1578
|
+
return True
|
|
1579
|
+
return False
|
|
1580
|
+
|
|
1581
|
+
|
|
1582
|
+
def _schema_match_tokens(question: str) -> list[str]:
|
|
1583
|
+
ignored = {
|
|
1584
|
+
"查询",
|
|
1585
|
+
"查看",
|
|
1586
|
+
"看看",
|
|
1587
|
+
"搜索",
|
|
1588
|
+
"统计",
|
|
1589
|
+
"多少",
|
|
1590
|
+
"几个",
|
|
1591
|
+
"几条",
|
|
1592
|
+
"哪些",
|
|
1593
|
+
"有哪些",
|
|
1594
|
+
"列表",
|
|
1595
|
+
"明细",
|
|
1596
|
+
"数据",
|
|
1597
|
+
"目前",
|
|
1598
|
+
"当前",
|
|
1599
|
+
"现在",
|
|
1600
|
+
"帮我",
|
|
1601
|
+
"一下",
|
|
1602
|
+
"返回",
|
|
1603
|
+
"输出",
|
|
1604
|
+
}
|
|
1605
|
+
tokens = []
|
|
1606
|
+
for token in _query_plan_tokens(question):
|
|
1607
|
+
if token in ignored or len(token) < 2:
|
|
1608
|
+
continue
|
|
1609
|
+
tokens.append(token)
|
|
1610
|
+
return list(dict.fromkeys(tokens))
|
|
1611
|
+
|
|
1612
|
+
|
|
1613
|
+
def _schema_table_search_text(table: dict[str, Any]) -> str:
|
|
1614
|
+
parts = [
|
|
1615
|
+
str(table.get("tableName") or table.get("name") or ""),
|
|
1616
|
+
str(table.get("comment") or table.get("description") or ""),
|
|
1617
|
+
]
|
|
1618
|
+
columns = table.get("columns")
|
|
1619
|
+
if isinstance(columns, list):
|
|
1620
|
+
for column in columns:
|
|
1621
|
+
if isinstance(column, dict):
|
|
1622
|
+
parts.extend(
|
|
1623
|
+
str(column.get(key) or "")
|
|
1624
|
+
for key in ("columnName", "name", "comment", "description", "dataType", "type")
|
|
1625
|
+
)
|
|
1626
|
+
return " ".join(parts).lower()
|
|
1627
|
+
|
|
1628
|
+
|
|
1629
|
+
def _schema_entity_candidates(question: str, schema_context: Any) -> list[dict[str, str]]:
|
|
1630
|
+
return _dedupe_schema_entity_candidates(_schema_generic_entity_candidates(question, schema_context))
|
|
1631
|
+
|
|
1632
|
+
|
|
1633
|
+
def _search_entity_type_for_maps_to(maps_to: str) -> str | None:
|
|
1634
|
+
return _MAPS_TO_SEARCH_ENTITY_TYPE.get(maps_to)
|
|
1635
|
+
|
|
1636
|
+
|
|
1637
|
+
def _search_maps_to_for_entity_type(entity_type: str) -> str:
|
|
1638
|
+
reverse = {
|
|
1639
|
+
"brand": "brand",
|
|
1640
|
+
"project": "sponge_project",
|
|
1641
|
+
"province": "province",
|
|
1642
|
+
"city": "city",
|
|
1643
|
+
"region": "region",
|
|
1644
|
+
}
|
|
1645
|
+
return reverse.get(entity_type, entity_type)
|
|
1646
|
+
|
|
1647
|
+
|
|
1648
|
+
def _entity_table_name_column(maps_to: str, schema: dict[str, Any]) -> str | None:
|
|
1649
|
+
table_name = maps_to.strip()
|
|
1650
|
+
if not table_name:
|
|
1651
|
+
return None
|
|
1652
|
+
for table in schema.get("tables", []):
|
|
1653
|
+
if not isinstance(table, dict):
|
|
1654
|
+
continue
|
|
1655
|
+
current = str(table.get("tableName") or table.get("name") or "")
|
|
1656
|
+
if current != table_name:
|
|
1657
|
+
continue
|
|
1658
|
+
columns = table.get("columns")
|
|
1659
|
+
if not isinstance(columns, list):
|
|
1660
|
+
break
|
|
1661
|
+
preferred: list[str] = []
|
|
1662
|
+
fallback: list[str] = []
|
|
1663
|
+
for column in columns:
|
|
1664
|
+
if not isinstance(column, dict):
|
|
1665
|
+
continue
|
|
1666
|
+
column_name = str(column.get("columnName") or column.get("name") or "")
|
|
1667
|
+
if not column_name or _is_identifier_column(column_name):
|
|
1668
|
+
continue
|
|
1669
|
+
comment = str(column.get("comment") or column.get("description") or "")
|
|
1670
|
+
if column_name in {"name", "nick_name", "job_name"} or "名称" in comment:
|
|
1671
|
+
preferred.append(column_name)
|
|
1672
|
+
else:
|
|
1673
|
+
fallback.append(column_name)
|
|
1674
|
+
chosen = (preferred or fallback or ["name"])[0]
|
|
1675
|
+
return f"{table_name}.{chosen}"
|
|
1676
|
+
return f"{table_name}.name"
|
|
1677
|
+
|
|
1678
|
+
|
|
1679
|
+
def _resolve_query_plan_filter_field(field: str, schema: dict[str, Any] | None) -> str:
|
|
1680
|
+
if not field or "." in field or schema is None:
|
|
1681
|
+
return field
|
|
1682
|
+
for item in _entity_catalog(schema):
|
|
1683
|
+
if item["filterField"] == field:
|
|
1684
|
+
return _entity_table_name_column(item["type"], schema) or field
|
|
1685
|
+
return field
|
|
1686
|
+
|
|
1687
|
+
|
|
1688
|
+
def _looks_like_store_name(value: str) -> bool:
|
|
1689
|
+
cleaned = value.strip()
|
|
1690
|
+
if not cleaned:
|
|
1691
|
+
return False
|
|
1692
|
+
if re.search(r"PS\d", cleaned, re.IGNORECASE):
|
|
1693
|
+
return True
|
|
1694
|
+
return any(marker in cleaned for marker in _STORE_NAME_MARKERS)
|
|
1695
|
+
|
|
1696
|
+
|
|
1697
|
+
def _looks_like_region_name(value: str) -> bool:
|
|
1698
|
+
cleaned = value.strip()
|
|
1699
|
+
if not cleaned or _looks_like_store_name(cleaned):
|
|
1700
|
+
return False
|
|
1701
|
+
if re.search(r"\d", cleaned):
|
|
1702
|
+
return False
|
|
1703
|
+
return cleaned.endswith(("区", "县", "旗")) and len(cleaned) <= 12
|
|
1704
|
+
|
|
1705
|
+
|
|
1706
|
+
def _looks_like_city_name(value: str) -> bool:
|
|
1707
|
+
cleaned = value.strip()
|
|
1708
|
+
if not cleaned or _looks_like_store_name(cleaned) or _looks_like_region_name(cleaned):
|
|
1709
|
+
return False
|
|
1710
|
+
if re.search(r"\d", cleaned):
|
|
1711
|
+
return False
|
|
1712
|
+
if cleaned.endswith(("里", "村", "镇")):
|
|
1713
|
+
return False
|
|
1714
|
+
if cleaned.endswith("市") and len(cleaned) <= 8:
|
|
1715
|
+
return True
|
|
1716
|
+
return 2 <= len(cleaned) <= 4
|
|
1717
|
+
|
|
1718
|
+
|
|
1719
|
+
def _city_name_filter_values(name: str) -> list[str]:
|
|
1720
|
+
cleaned = name.strip()
|
|
1721
|
+
if not cleaned:
|
|
1722
|
+
return []
|
|
1723
|
+
if cleaned.endswith("市"):
|
|
1724
|
+
base = cleaned[:-1]
|
|
1725
|
+
return list(dict.fromkeys([base, cleaned]))
|
|
1726
|
+
return list(dict.fromkeys([cleaned, f"{cleaned}市"]))
|
|
1727
|
+
|
|
1728
|
+
|
|
1729
|
+
def _region_name_filter_values(name: str) -> list[str]:
|
|
1730
|
+
cleaned = name.strip()
|
|
1731
|
+
if not cleaned:
|
|
1732
|
+
return []
|
|
1733
|
+
if cleaned.endswith("区"):
|
|
1734
|
+
base = cleaned[:-1]
|
|
1735
|
+
return list(dict.fromkeys([cleaned, base]))
|
|
1736
|
+
if cleaned.endswith("县"):
|
|
1737
|
+
base = cleaned[:-1]
|
|
1738
|
+
return list(dict.fromkeys([cleaned, base]))
|
|
1739
|
+
return [cleaned]
|
|
1740
|
+
|
|
1741
|
+
|
|
1742
|
+
def _promote_region_search_candidate(
|
|
1743
|
+
search_candidates: list[dict[str, str]],
|
|
1744
|
+
schema_candidates: list[dict[str, str]],
|
|
1745
|
+
*,
|
|
1746
|
+
value: str,
|
|
1747
|
+
schema: dict[str, Any],
|
|
1748
|
+
seen_search: set[tuple[str, str]],
|
|
1749
|
+
) -> None:
|
|
1750
|
+
field = _entity_table_name_column("region", schema) or "region.name"
|
|
1751
|
+
key = ("region", value)
|
|
1752
|
+
if key in seen_search:
|
|
1753
|
+
return
|
|
1754
|
+
seen_search.add(key)
|
|
1755
|
+
search_candidates.append(
|
|
1756
|
+
{
|
|
1757
|
+
"type": "region",
|
|
1758
|
+
"label": "区域",
|
|
1759
|
+
"value": value,
|
|
1760
|
+
"field": field,
|
|
1761
|
+
"mapsTo": "region",
|
|
1762
|
+
"source": "search_entities.candidate",
|
|
1763
|
+
}
|
|
1764
|
+
)
|
|
1765
|
+
schema_candidates[:] = [
|
|
1766
|
+
item
|
|
1767
|
+
for item in schema_candidates
|
|
1768
|
+
if not (
|
|
1769
|
+
str(item.get("type") or "") in {"city", "store"}
|
|
1770
|
+
and str(item.get("value") or "") == value
|
|
1771
|
+
)
|
|
1772
|
+
]
|
|
1773
|
+
search_candidates[:] = [
|
|
1774
|
+
item
|
|
1775
|
+
for item in search_candidates
|
|
1776
|
+
if not (
|
|
1777
|
+
str(item.get("type") or "") == "city"
|
|
1778
|
+
and str(item.get("value") or "") == value
|
|
1779
|
+
)
|
|
1780
|
+
]
|
|
1781
|
+
|
|
1782
|
+
|
|
1783
|
+
def _promote_city_search_candidate(
|
|
1784
|
+
search_candidates: list[dict[str, str]],
|
|
1785
|
+
schema_candidates: list[dict[str, str]],
|
|
1786
|
+
*,
|
|
1787
|
+
value: str,
|
|
1788
|
+
schema: dict[str, Any],
|
|
1789
|
+
seen_search: set[tuple[str, str]],
|
|
1790
|
+
) -> None:
|
|
1791
|
+
field = _entity_table_name_column("city", schema) or "city.name"
|
|
1792
|
+
key = ("city", value)
|
|
1793
|
+
if key in seen_search:
|
|
1794
|
+
return
|
|
1795
|
+
seen_search.add(key)
|
|
1796
|
+
search_candidates.append(
|
|
1797
|
+
{
|
|
1798
|
+
"type": "city",
|
|
1799
|
+
"label": "城市",
|
|
1800
|
+
"value": value,
|
|
1801
|
+
"field": field,
|
|
1802
|
+
"mapsTo": "city",
|
|
1803
|
+
"source": "search_entities.candidate",
|
|
1804
|
+
}
|
|
1805
|
+
)
|
|
1806
|
+
schema_candidates[:] = [
|
|
1807
|
+
item
|
|
1808
|
+
for item in schema_candidates
|
|
1809
|
+
if not (str(item.get("type") or "") == "store" and str(item.get("value") or "") == value)
|
|
1810
|
+
]
|
|
1811
|
+
|
|
1812
|
+
|
|
1813
|
+
def _extract_region_tokens(question: str) -> list[str]:
|
|
1814
|
+
tokens: list[str] = []
|
|
1815
|
+
for match in re.finditer(r"([\u4e00-\u9fff]{2,4})区", question):
|
|
1816
|
+
district = match.group(0)
|
|
1817
|
+
tokens.append(district)
|
|
1818
|
+
core = match.group(1)
|
|
1819
|
+
for size in (2, 3):
|
|
1820
|
+
if len(core) >= size:
|
|
1821
|
+
suffix = f"{core[-size:]}区"
|
|
1822
|
+
if _looks_like_region_name(suffix):
|
|
1823
|
+
tokens.append(suffix)
|
|
1824
|
+
return list(dict.fromkeys(tokens))
|
|
1825
|
+
|
|
1826
|
+
|
|
1827
|
+
def _expand_brand_project_search_candidates(
|
|
1828
|
+
search_candidates: list[dict[str, str]],
|
|
1829
|
+
llm_values: dict[str, Any],
|
|
1830
|
+
schema: dict[str, Any],
|
|
1831
|
+
seen_search: set[tuple[str, str]],
|
|
1832
|
+
) -> None:
|
|
1833
|
+
"""Brand/project names are often ambiguous; search both when only one slot is filled."""
|
|
1834
|
+
if _explicit_brand_or_project_types(llm_values):
|
|
1835
|
+
return
|
|
1836
|
+
brand_value = _normalize_entity_search_value("brand", str(llm_values.get("brand") or "").strip())
|
|
1837
|
+
project_value = _normalize_entity_search_value(
|
|
1838
|
+
"sponge_project",
|
|
1839
|
+
str(llm_values.get("sponge_project") or "").strip(),
|
|
1840
|
+
)
|
|
1841
|
+
expansions: list[tuple[str, str, str]] = []
|
|
1842
|
+
if brand_value and not project_value:
|
|
1843
|
+
expansions.append(("project", brand_value, "sponge_project"))
|
|
1844
|
+
if project_value and not brand_value:
|
|
1845
|
+
expansions.append(("brand", project_value, "brand"))
|
|
1846
|
+
for search_type, value, maps_to in expansions:
|
|
1847
|
+
key = (search_type, value)
|
|
1848
|
+
if key in seen_search:
|
|
1849
|
+
continue
|
|
1850
|
+
seen_search.add(key)
|
|
1851
|
+
field = _entity_table_name_column(maps_to, schema) or f"{maps_to}.name"
|
|
1852
|
+
search_candidates.append(
|
|
1853
|
+
{
|
|
1854
|
+
"type": search_type,
|
|
1855
|
+
"label": _entity_type_label(search_type),
|
|
1856
|
+
"value": value,
|
|
1857
|
+
"field": field,
|
|
1858
|
+
"mapsTo": maps_to,
|
|
1859
|
+
"source": "search_entities.candidate",
|
|
1860
|
+
}
|
|
1861
|
+
)
|
|
1862
|
+
|
|
1863
|
+
|
|
1864
|
+
def _explicit_brand_or_project_types(llm_values: dict[str, Any]) -> set[str]:
|
|
1865
|
+
explicit_types = llm_values.get(_EXPLICIT_ENTITY_TYPES_KEY)
|
|
1866
|
+
if not isinstance(explicit_types, list):
|
|
1867
|
+
return set()
|
|
1868
|
+
return {str(item) for item in explicit_types if str(item) in {"brand", "sponge_project"}}
|
|
1869
|
+
|
|
1870
|
+
|
|
1871
|
+
def _reconcile_location_entity_candidates(
|
|
1872
|
+
question: str,
|
|
1873
|
+
search_candidates: list[dict[str, str]],
|
|
1874
|
+
schema_candidates: list[dict[str, str]],
|
|
1875
|
+
schema: dict[str, Any],
|
|
1876
|
+
) -> tuple[list[dict[str, str]], list[dict[str, str]]]:
|
|
1877
|
+
"""Move location tokens onto city/region search paths; keep store for store-like names."""
|
|
1878
|
+
seen_search = {(str(item.get("type") or ""), str(item.get("value") or "")) for item in search_candidates}
|
|
1879
|
+
for token in _extract_region_tokens(question):
|
|
1880
|
+
if _looks_like_region_name(token):
|
|
1881
|
+
_promote_region_search_candidate(
|
|
1882
|
+
search_candidates,
|
|
1883
|
+
schema_candidates,
|
|
1884
|
+
value=token,
|
|
1885
|
+
schema=schema,
|
|
1886
|
+
seen_search=seen_search,
|
|
1887
|
+
)
|
|
1888
|
+
for candidate in list(schema_candidates):
|
|
1889
|
+
value = str(candidate.get("value") or "").strip()
|
|
1890
|
+
if not value:
|
|
1891
|
+
continue
|
|
1892
|
+
if _looks_like_region_name(value):
|
|
1893
|
+
_promote_region_search_candidate(
|
|
1894
|
+
search_candidates,
|
|
1895
|
+
schema_candidates,
|
|
1896
|
+
value=value,
|
|
1897
|
+
schema=schema,
|
|
1898
|
+
seen_search=seen_search,
|
|
1899
|
+
)
|
|
1900
|
+
continue
|
|
1901
|
+
if str(candidate.get("type") or "") != "store":
|
|
1902
|
+
continue
|
|
1903
|
+
if _looks_like_city_name(value):
|
|
1904
|
+
_promote_city_search_candidate(
|
|
1905
|
+
search_candidates,
|
|
1906
|
+
schema_candidates,
|
|
1907
|
+
value=value,
|
|
1908
|
+
schema=schema,
|
|
1909
|
+
seen_search=seen_search,
|
|
1910
|
+
)
|
|
1911
|
+
for candidate in list(search_candidates):
|
|
1912
|
+
value = str(candidate.get("value") or "").strip()
|
|
1913
|
+
if not value:
|
|
1914
|
+
continue
|
|
1915
|
+
if str(candidate.get("type") or "") == "city" and _looks_like_region_name(value):
|
|
1916
|
+
search_candidates.remove(candidate)
|
|
1917
|
+
_promote_region_search_candidate(
|
|
1918
|
+
search_candidates,
|
|
1919
|
+
schema_candidates,
|
|
1920
|
+
value=value,
|
|
1921
|
+
schema=schema,
|
|
1922
|
+
seen_search=seen_search,
|
|
1923
|
+
)
|
|
1924
|
+
continue
|
|
1925
|
+
if str(candidate.get("type") or "") != "store":
|
|
1926
|
+
continue
|
|
1927
|
+
if _looks_like_city_name(value):
|
|
1928
|
+
search_candidates.remove(candidate)
|
|
1929
|
+
_promote_city_search_candidate(
|
|
1930
|
+
search_candidates,
|
|
1931
|
+
schema_candidates,
|
|
1932
|
+
value=value,
|
|
1933
|
+
schema=schema,
|
|
1934
|
+
seen_search=seen_search,
|
|
1935
|
+
)
|
|
1936
|
+
return search_candidates, schema_candidates
|
|
1937
|
+
|
|
1938
|
+
|
|
1939
|
+
def _collect_entity_resolution_candidates(
|
|
1940
|
+
question: str,
|
|
1941
|
+
schema: dict[str, Any],
|
|
1942
|
+
schema_context: Any,
|
|
1943
|
+
sampler: Sampler | None,
|
|
1944
|
+
*,
|
|
1945
|
+
agent_instructions: list[dict[str, Any]] | None = None,
|
|
1946
|
+
analysis: QuestionAnalysis | None = None,
|
|
1947
|
+
) -> tuple[list[dict[str, str]], list[dict[str, str]]]:
|
|
1948
|
+
"""Split entity mentions into search_entities candidates and schema-path candidates."""
|
|
1949
|
+
search_candidates: list[dict[str, str]] = []
|
|
1950
|
+
schema_candidates: list[dict[str, str]] = []
|
|
1951
|
+
seen_search: set[tuple[str, str]] = set()
|
|
1952
|
+
seen_schema: set[tuple[str, str]] = set()
|
|
1953
|
+
|
|
1954
|
+
catalog = _entity_catalog(schema)
|
|
1955
|
+
llm_values: dict[str, Any] = {}
|
|
1956
|
+
if sampler is not None and catalog:
|
|
1957
|
+
llm_values = _llm_extract_entity_values(question, catalog, sampler, agent_instructions)
|
|
1958
|
+
llm_values = _merge_analysis_entity_candidates_into_llm_values(llm_values, analysis, catalog)
|
|
1959
|
+
_drop_implicit_location_values_inside_named_entities(llm_values)
|
|
1960
|
+
for item in catalog:
|
|
1961
|
+
value = llm_values.get(item["type"]) or _extract_value_near_label(question, item["label"])
|
|
1962
|
+
if not value:
|
|
1963
|
+
continue
|
|
1964
|
+
value = _normalize_entity_search_value(item["type"], value)
|
|
1965
|
+
if _is_analysis_noise_value(value, analysis):
|
|
1966
|
+
continue
|
|
1967
|
+
search_type = _search_entity_type_for_maps_to(item["type"])
|
|
1968
|
+
field = _entity_table_name_column(item["type"], schema) or item["filterField"]
|
|
1969
|
+
if search_type:
|
|
1970
|
+
key = (search_type, value)
|
|
1971
|
+
if key in seen_search:
|
|
1972
|
+
continue
|
|
1973
|
+
seen_search.add(key)
|
|
1974
|
+
search_candidates.append(
|
|
1975
|
+
{
|
|
1976
|
+
"type": search_type,
|
|
1977
|
+
"label": item["label"],
|
|
1978
|
+
"value": value,
|
|
1979
|
+
"field": field,
|
|
1980
|
+
"mapsTo": item["type"],
|
|
1981
|
+
"source": "search_entities.candidate",
|
|
1982
|
+
}
|
|
1983
|
+
)
|
|
1984
|
+
else:
|
|
1985
|
+
key = (item["type"], value)
|
|
1986
|
+
if key in seen_schema:
|
|
1987
|
+
continue
|
|
1988
|
+
seen_schema.add(key)
|
|
1989
|
+
schema_candidates.append(
|
|
1990
|
+
{
|
|
1991
|
+
"type": item["type"],
|
|
1992
|
+
"label": item["label"],
|
|
1993
|
+
"value": value,
|
|
1994
|
+
"field": field,
|
|
1995
|
+
"mapsTo": item["type"],
|
|
1996
|
+
"source": "schema.businessTerms",
|
|
1997
|
+
}
|
|
1998
|
+
)
|
|
1999
|
+
|
|
2000
|
+
for candidate in _schema_generic_entity_candidates(question, schema_context):
|
|
2001
|
+
field = str(candidate.get("field") or "")
|
|
2002
|
+
value = str(candidate.get("value") or "")
|
|
2003
|
+
key = (field, value)
|
|
2004
|
+
if not value or key in seen_schema or _is_analysis_noise_value(value, analysis):
|
|
2005
|
+
continue
|
|
2006
|
+
seen_schema.add(key)
|
|
2007
|
+
schema_candidates.append(candidate)
|
|
2008
|
+
|
|
2009
|
+
_expand_brand_project_search_candidates(search_candidates, llm_values, schema, seen_search)
|
|
2010
|
+
search_candidates, schema_candidates = _reconcile_location_entity_candidates(
|
|
2011
|
+
question,
|
|
2012
|
+
search_candidates,
|
|
2013
|
+
schema_candidates,
|
|
2014
|
+
schema,
|
|
2015
|
+
)
|
|
2016
|
+
return search_candidates, schema_candidates
|
|
2017
|
+
|
|
2018
|
+
|
|
2019
|
+
def _schema_generic_entity_candidates(question: str, schema_context: Any) -> list[dict[str, str]]:
|
|
2020
|
+
candidates: list[dict[str, str]] = []
|
|
2021
|
+
for table in getattr(schema_context, "tables", []) or []:
|
|
2022
|
+
if not isinstance(table, dict):
|
|
2023
|
+
continue
|
|
2024
|
+
table_name = str(table.get("tableName") or table.get("name") or "")
|
|
2025
|
+
columns = table.get("columns")
|
|
2026
|
+
if not isinstance(columns, list):
|
|
2027
|
+
continue
|
|
2028
|
+
for column in columns:
|
|
2029
|
+
if not isinstance(column, dict):
|
|
2030
|
+
continue
|
|
2031
|
+
column_name = str(column.get("columnName") or column.get("name") or "")
|
|
2032
|
+
if _is_identifier_column(column_name):
|
|
2033
|
+
continue
|
|
2034
|
+
if column_name.endswith("_ids") or column_name.endswith("_id_list"):
|
|
2035
|
+
continue
|
|
2036
|
+
label = _generic_schema_entity_label(column)
|
|
2037
|
+
if not label or label not in question:
|
|
2038
|
+
continue
|
|
2039
|
+
value = _extract_value_near_label(question, label)
|
|
2040
|
+
if not value:
|
|
2041
|
+
continue
|
|
2042
|
+
candidates.append(
|
|
2043
|
+
{
|
|
2044
|
+
"type": table_name or column_name or label,
|
|
2045
|
+
"label": label,
|
|
2046
|
+
"value": value,
|
|
2047
|
+
"field": f"{table_name}.{column_name}" if table_name and column_name else f"{label}名称",
|
|
2048
|
+
"source": "schema.columns",
|
|
2049
|
+
}
|
|
2050
|
+
)
|
|
2051
|
+
return candidates
|
|
2052
|
+
|
|
2053
|
+
|
|
2054
|
+
def _is_identifier_column(column_name: str) -> bool:
|
|
2055
|
+
lowered = column_name.lower()
|
|
2056
|
+
return lowered == "id" or lowered.endswith("_id")
|
|
2057
|
+
|
|
2058
|
+
|
|
2059
|
+
def _generic_schema_entity_label(column: dict[str, Any]) -> str:
|
|
2060
|
+
raw = str(column.get("comment") or column.get("description") or "")
|
|
2061
|
+
if not raw:
|
|
2062
|
+
return ""
|
|
2063
|
+
label = re.split(r"[\s,,。;;((]", raw, maxsplit=1)[0]
|
|
2064
|
+
label = re.sub(r"(名称|名字|姓名|编号|编码|ID|Id|id)$", "", label).strip()
|
|
2065
|
+
if len(label) < 2 or len(label) > 12 or not _contains_cjk(label):
|
|
2066
|
+
return ""
|
|
2067
|
+
return label
|
|
2068
|
+
|
|
2069
|
+
|
|
2070
|
+
def _extract_value_near_label(question: str, label: str) -> str | None:
|
|
2071
|
+
escaped = re.escape(label)
|
|
2072
|
+
# The value char class intentionally excludes whitespace so the capture stops
|
|
2073
|
+
# at the next token boundary instead of eating across words (e.g. it must not
|
|
2074
|
+
# turn "项目:成都你六姐 门店:..." into "成都你六姐 门店").
|
|
2075
|
+
value_class = r"[\u4e00-\u9fffA-Za-z0-9][\u4e00-\u9fffA-Za-z0-9_\-]{0,39}"
|
|
2076
|
+
# A value is only trustworthy when an explicit connector links it to the label.
|
|
2077
|
+
# A leading label with no connector usually means the label is the query subject,
|
|
2078
|
+
# not a filter.
|
|
2079
|
+
forward = re.search(rf"{escaped}\s*(?:名称|名字)?\s*(?:为|是|叫|名为|=|:|:)\s*({value_class})", question)
|
|
2080
|
+
if forward is not None:
|
|
2081
|
+
value = _clean_entity_value(_truncate_entity_value(forward.group(1)), label)
|
|
2082
|
+
if value:
|
|
2083
|
+
return value
|
|
2084
|
+
# Reverse form "成都你六姐项目" is allowed, but reject captures that span a whole
|
|
2085
|
+
# clause (still contain a connector verb) so the query subject noun does not pull
|
|
2086
|
+
# in unrelated filter text.
|
|
2087
|
+
reverse = re.search(rf"({value_class})\s*{escaped}", question)
|
|
2088
|
+
if reverse is not None:
|
|
2089
|
+
raw = _truncate_entity_value(_strip_leading_connectives(reverse.group(1)))
|
|
2090
|
+
if not any(connector in raw for connector in ("为", "是", "=", ":", ":")):
|
|
2091
|
+
value = _clean_entity_value(raw, label)
|
|
2092
|
+
if value:
|
|
2093
|
+
return value
|
|
2094
|
+
return None
|
|
2095
|
+
|
|
2096
|
+
|
|
2097
|
+
# Chinese connective particles that mark the end of an entity value. They are CJK
|
|
2098
|
+
# characters, so the capture regex above would otherwise greedily consume them and
|
|
2099
|
+
# everything after a connector into one long phrase. We cut the value
|
|
2100
|
+
# at the first particle whose preceding segment already looks like a real value,
|
|
2101
|
+
# so multi-char names that legitimately contain the particle (e.g. "美的") survive.
|
|
2102
|
+
_ENTITY_VALUE_BOUNDARIES = "的及和与或"
|
|
2103
|
+
|
|
2104
|
+
|
|
2105
|
+
def _truncate_entity_value(value: str) -> str:
|
|
2106
|
+
for index, char in enumerate(value):
|
|
2107
|
+
if char in _ENTITY_VALUE_BOUNDARIES and index >= 2:
|
|
2108
|
+
return value[:index]
|
|
2109
|
+
return value
|
|
2110
|
+
|
|
2111
|
+
|
|
2112
|
+
def _strip_leading_connectives(value: str) -> str:
|
|
2113
|
+
return value.lstrip(_ENTITY_VALUE_BOUNDARIES)
|
|
2114
|
+
|
|
2115
|
+
|
|
2116
|
+
def _contains_cjk(value: str) -> bool:
|
|
2117
|
+
return bool(re.search(r"[\u4e00-\u9fff]", value))
|
|
2118
|
+
|
|
2119
|
+
|
|
2120
|
+
def _clean_entity_value(value: str, label: str = "") -> str:
|
|
2121
|
+
cleaned = value.strip(" \t\r\n,,。;;??!!()()[]【】")
|
|
2122
|
+
if label:
|
|
2123
|
+
cleaned = cleaned.replace(label, " ")
|
|
2124
|
+
for stop in (
|
|
2125
|
+
"帮我查询",
|
|
2126
|
+
"帮我查",
|
|
2127
|
+
"查询",
|
|
2128
|
+
"查看",
|
|
2129
|
+
"看看",
|
|
2130
|
+
"列出",
|
|
2131
|
+
"返回",
|
|
2132
|
+
"统计",
|
|
2133
|
+
"有多少",
|
|
2134
|
+
"有哪些",
|
|
2135
|
+
"列表",
|
|
2136
|
+
"明细",
|
|
2137
|
+
"数据",
|
|
2138
|
+
"目前",
|
|
2139
|
+
"当前",
|
|
2140
|
+
"现在",
|
|
2141
|
+
"的",
|
|
2142
|
+
):
|
|
2143
|
+
cleaned = cleaned.replace(stop, " ")
|
|
2144
|
+
cleaned = re.sub(r"\s+", " ", cleaned).strip(" -_::")
|
|
2145
|
+
if len(cleaned) < 2:
|
|
2146
|
+
return ""
|
|
2147
|
+
return cleaned[:40]
|
|
2148
|
+
|
|
2149
|
+
|
|
2150
|
+
def _schema_entity_candidate(entity_type: str, value: str, source: str) -> dict[str, str]:
|
|
2151
|
+
return {
|
|
2152
|
+
"type": entity_type,
|
|
2153
|
+
"label": _entity_type_label(entity_type),
|
|
2154
|
+
"value": value,
|
|
2155
|
+
"field": _entity_search_filter_field(entity_type),
|
|
2156
|
+
"source": source,
|
|
2157
|
+
}
|
|
2158
|
+
|
|
2159
|
+
|
|
2160
|
+
def _dedupe_schema_entity_candidates(candidates: list[dict[str, str]]) -> list[dict[str, str]]:
|
|
2161
|
+
result: list[dict[str, str]] = []
|
|
2162
|
+
seen: set[tuple[str, str]] = set()
|
|
2163
|
+
for candidate in candidates:
|
|
2164
|
+
key = (str(candidate.get("type") or ""), str(candidate.get("value") or ""))
|
|
2165
|
+
if not key[0] or not key[1] or key in seen:
|
|
2166
|
+
continue
|
|
2167
|
+
seen.add(key)
|
|
2168
|
+
result.append(candidate)
|
|
2169
|
+
return result
|
|
2170
|
+
|
|
2171
|
+
|
|
2172
|
+
def _merge_entity_search_results(search_results: list[dict[str, Any]]) -> dict[str, Any]:
|
|
2173
|
+
merged: dict[str, Any] = {}
|
|
2174
|
+
for search_result in search_results:
|
|
2175
|
+
if not isinstance(search_result, dict):
|
|
2176
|
+
continue
|
|
2177
|
+
response = search_result.get("response")
|
|
2178
|
+
if not isinstance(response, dict):
|
|
2179
|
+
continue
|
|
2180
|
+
entities = response.get("entities")
|
|
2181
|
+
if not isinstance(entities, dict):
|
|
2182
|
+
continue
|
|
2183
|
+
for entity_type, group in entities.items():
|
|
2184
|
+
if not isinstance(group, dict):
|
|
2185
|
+
continue
|
|
2186
|
+
target = merged.setdefault(str(entity_type), {key: group.get(key) for key in ("table", "idColumn", "nameColumn", "parentType")})
|
|
2187
|
+
target.setdefault("items", [])
|
|
2188
|
+
items = group.get("items")
|
|
2189
|
+
if isinstance(items, list):
|
|
2190
|
+
target["items"].extend(item for item in items if isinstance(item, dict))
|
|
2191
|
+
return merged
|
|
2192
|
+
|
|
2193
|
+
|
|
2194
|
+
def _flatten_entity_search_items(search_response: dict[str, Any] | None) -> list[dict[str, str]]:
|
|
2195
|
+
if not isinstance(search_response, dict):
|
|
2196
|
+
return []
|
|
2197
|
+
entities = search_response.get("entities")
|
|
2198
|
+
if not isinstance(entities, dict):
|
|
2199
|
+
return []
|
|
2200
|
+
result: list[dict[str, str]] = []
|
|
2201
|
+
for entity_type, group in entities.items():
|
|
2202
|
+
if not isinstance(group, dict):
|
|
2203
|
+
continue
|
|
2204
|
+
items = group.get("items")
|
|
2205
|
+
if not isinstance(items, list):
|
|
2206
|
+
continue
|
|
2207
|
+
for item in items:
|
|
2208
|
+
if not isinstance(item, dict):
|
|
2209
|
+
continue
|
|
2210
|
+
name = str(item.get("name") or "")
|
|
2211
|
+
if not name:
|
|
2212
|
+
continue
|
|
2213
|
+
aliases = item.get("aliases")
|
|
2214
|
+
result.append(
|
|
2215
|
+
{
|
|
2216
|
+
"type": str(entity_type),
|
|
2217
|
+
"id": str(item.get("id") or ""),
|
|
2218
|
+
"name": name,
|
|
2219
|
+
"aliases": "、".join(str(alias) for alias in aliases) if isinstance(aliases, list) else "",
|
|
2220
|
+
"pinyin": str(item.get("pinyin") or ""),
|
|
2221
|
+
"parentId": str(item.get("parentId") or ""),
|
|
2222
|
+
"table": str(group.get("table") or ""),
|
|
2223
|
+
"idColumn": str(group.get("idColumn") or ""),
|
|
2224
|
+
"nameColumn": str(group.get("nameColumn") or ""),
|
|
2225
|
+
"parentType": str(group.get("parentType") or ""),
|
|
2226
|
+
}
|
|
2227
|
+
)
|
|
2228
|
+
return result
|
|
2229
|
+
|
|
2230
|
+
|
|
2231
|
+
def _unique_entity_search_item(search_response: dict[str, Any] | None) -> dict[str, str] | None:
|
|
2232
|
+
items = _flatten_entity_search_items(search_response)
|
|
2233
|
+
return items[0] if len(items) == 1 else None
|
|
2234
|
+
|
|
2235
|
+
|
|
2236
|
+
def _unique_entity_search_items(search_response: dict[str, Any] | None) -> list[dict[str, str]]:
|
|
2237
|
+
if not isinstance(search_response, dict):
|
|
2238
|
+
return []
|
|
2239
|
+
search_results = search_response.get("searchResults")
|
|
2240
|
+
if not isinstance(search_results, list):
|
|
2241
|
+
item = _unique_entity_search_item(search_response)
|
|
2242
|
+
return [item] if item is not None else []
|
|
2243
|
+
result: list[dict[str, str]] = []
|
|
2244
|
+
for search_result in search_results:
|
|
2245
|
+
if not isinstance(search_result, dict):
|
|
2246
|
+
continue
|
|
2247
|
+
item = _unique_entity_search_item(search_result.get("response"))
|
|
2248
|
+
if item is not None:
|
|
2249
|
+
result.append(item)
|
|
2250
|
+
return result
|
|
2251
|
+
|
|
2252
|
+
|
|
2253
|
+
def _entity_search_filter_items(search_response: dict[str, Any] | None) -> list[dict[str, str]]:
|
|
2254
|
+
if not isinstance(search_response, dict):
|
|
2255
|
+
return []
|
|
2256
|
+
search_results = search_response.get("searchResults")
|
|
2257
|
+
if not isinstance(search_results, list):
|
|
2258
|
+
return _entity_search_filter_items_for_response(
|
|
2259
|
+
search_response,
|
|
2260
|
+
searched_name=str(search_response.get("searchedName") or ""),
|
|
2261
|
+
group_seed="entity_search",
|
|
2262
|
+
)
|
|
2263
|
+
result: list[dict[str, str]] = []
|
|
2264
|
+
group_counts: dict[str, int] = {}
|
|
2265
|
+
for search_result in search_results:
|
|
2266
|
+
if not isinstance(search_result, dict):
|
|
2267
|
+
continue
|
|
2268
|
+
response = search_result.get("response")
|
|
2269
|
+
if not isinstance(response, dict):
|
|
2270
|
+
continue
|
|
2271
|
+
group_key = _entity_search_alias_group_key(search_result)
|
|
2272
|
+
if not group_key:
|
|
2273
|
+
continue
|
|
2274
|
+
group_counts[group_key] = group_counts.get(group_key, 0) + len(_flatten_entity_search_items(response))
|
|
2275
|
+
for index, search_result in enumerate(search_results, start=1):
|
|
2276
|
+
if not isinstance(search_result, dict):
|
|
2277
|
+
continue
|
|
2278
|
+
response = search_result.get("response")
|
|
2279
|
+
if not isinstance(response, dict):
|
|
2280
|
+
continue
|
|
2281
|
+
searched_name = str(search_result.get("searchedName") or "")
|
|
2282
|
+
candidate = search_result.get("candidate")
|
|
2283
|
+
candidate_type = str(candidate.get("type") or "") if isinstance(candidate, dict) else ""
|
|
2284
|
+
group_key = _entity_search_alias_group_key(search_result)
|
|
2285
|
+
if group_key and group_counts.get(group_key, 0) > 1:
|
|
2286
|
+
group_seed = f"entity_search:{group_key}"
|
|
2287
|
+
else:
|
|
2288
|
+
group_seed = f"entity_search:{index}:{candidate_type}:{searched_name}"
|
|
2289
|
+
result.extend(
|
|
2290
|
+
_entity_search_filter_items_for_response(
|
|
2291
|
+
response,
|
|
2292
|
+
searched_name=searched_name,
|
|
2293
|
+
group_seed=group_seed,
|
|
2294
|
+
force_logical_group=bool(group_key and group_counts.get(group_key, 0) > 1),
|
|
2295
|
+
)
|
|
2296
|
+
)
|
|
2297
|
+
return result
|
|
2298
|
+
|
|
2299
|
+
|
|
2300
|
+
def _entity_search_alias_group_key(search_result: dict[str, Any]) -> str:
|
|
2301
|
+
candidate = search_result.get("candidate")
|
|
2302
|
+
raw = ""
|
|
2303
|
+
if isinstance(candidate, dict):
|
|
2304
|
+
raw = str(candidate.get("value") or "")
|
|
2305
|
+
if not raw:
|
|
2306
|
+
raw = str(search_result.get("searchedName") or "")
|
|
2307
|
+
return re.sub(r"[^0-9A-Za-z\u4e00-\u9fff]+", "", raw).lower()
|
|
2308
|
+
|
|
2309
|
+
|
|
2310
|
+
def _entity_search_filter_items_for_response(
|
|
2311
|
+
response: dict[str, Any],
|
|
2312
|
+
*,
|
|
2313
|
+
searched_name: str,
|
|
2314
|
+
group_seed: str,
|
|
2315
|
+
force_logical_group: bool = False,
|
|
2316
|
+
) -> list[dict[str, str]]:
|
|
2317
|
+
items = _flatten_entity_search_items(response)
|
|
2318
|
+
if not items:
|
|
2319
|
+
return []
|
|
2320
|
+
logical_group = group_seed if force_logical_group or len(items) > 1 else ""
|
|
2321
|
+
result: list[dict[str, str]] = []
|
|
2322
|
+
for item in items:
|
|
2323
|
+
enriched = dict(item)
|
|
2324
|
+
if searched_name:
|
|
2325
|
+
enriched["matchedName"] = searched_name
|
|
2326
|
+
if logical_group:
|
|
2327
|
+
enriched["logicalGroup"] = logical_group
|
|
2328
|
+
enriched["logicalOperator"] = "OR"
|
|
2329
|
+
result.append(enriched)
|
|
2330
|
+
return result
|
|
2331
|
+
|
|
2332
|
+
|
|
2333
|
+
def _direct_entity_candidates(search_response: dict[str, Any] | None) -> list[dict[str, str]]:
|
|
2334
|
+
if not isinstance(search_response, dict):
|
|
2335
|
+
return []
|
|
2336
|
+
candidates = [
|
|
2337
|
+
candidate
|
|
2338
|
+
for candidate in search_response.get("entityCandidates") or []
|
|
2339
|
+
if isinstance(candidate, dict) and candidate.get("type") and candidate.get("value")
|
|
2340
|
+
]
|
|
2341
|
+
search_results = search_response.get("searchResults")
|
|
2342
|
+
if not isinstance(search_results, list):
|
|
2343
|
+
return candidates
|
|
2344
|
+
resolved: set[tuple[str, str]] = set()
|
|
2345
|
+
for search_result in search_results:
|
|
2346
|
+
if not isinstance(search_result, dict):
|
|
2347
|
+
continue
|
|
2348
|
+
candidate = search_result.get("candidate")
|
|
2349
|
+
if not isinstance(candidate, dict):
|
|
2350
|
+
continue
|
|
2351
|
+
if _flatten_entity_search_items(search_result.get("response")):
|
|
2352
|
+
resolved.add((str(candidate.get("type") or ""), str(candidate.get("value") or "")))
|
|
2353
|
+
return [
|
|
2354
|
+
candidate
|
|
2355
|
+
for candidate in candidates
|
|
2356
|
+
if (str(candidate.get("type") or ""), str(candidate.get("value") or "")) not in resolved
|
|
2357
|
+
]
|
|
2358
|
+
|
|
2359
|
+
|
|
2360
|
+
def _entity_type_label(entity_type: str) -> str:
|
|
2361
|
+
return {
|
|
2362
|
+
"brand": "品牌",
|
|
2363
|
+
"project": "项目",
|
|
2364
|
+
"province": "省份",
|
|
2365
|
+
"city": "城市",
|
|
2366
|
+
"region": "区域",
|
|
2367
|
+
"store": "门店",
|
|
2368
|
+
"sponge_project": "项目",
|
|
2369
|
+
"supplier": "供应商",
|
|
2370
|
+
}.get(entity_type, entity_type)
|
|
2371
|
+
|
|
2372
|
+
|
|
2373
|
+
_ENTITY_TYPE_SUFFIXES: dict[str, tuple[str, ...]] = {
|
|
2374
|
+
"brand": ("品牌",),
|
|
2375
|
+
"project": ("项目",),
|
|
2376
|
+
"sponge_project": ("项目",),
|
|
2377
|
+
"store": ("门店",),
|
|
2378
|
+
}
|
|
2379
|
+
|
|
2380
|
+
|
|
2381
|
+
def _normalize_entity_search_value(entity_type: str, value: str) -> str:
|
|
2382
|
+
"""Drop trailing type words such as 项目/品牌/门店 when they are labels, not part of the name."""
|
|
2383
|
+
cleaned = value.strip()
|
|
2384
|
+
if not cleaned:
|
|
2385
|
+
return cleaned
|
|
2386
|
+
suffixes = _ENTITY_TYPE_SUFFIXES.get(entity_type, ())
|
|
2387
|
+
for suffix in suffixes:
|
|
2388
|
+
if cleaned.endswith(suffix) and len(cleaned) > len(suffix) + 1:
|
|
2389
|
+
return cleaned[: -len(suffix)].strip()
|
|
2390
|
+
return cleaned
|
|
2391
|
+
|
|
2392
|
+
|
|
2393
|
+
def _entity_search_filter_field(entity_type: str) -> str:
|
|
2394
|
+
return entity_type
|
|
2395
|
+
|
|
2396
|
+
|
|
2397
|
+
def _entity_catalog(schema: dict[str, Any]) -> list[dict[str, str]]:
|
|
2398
|
+
"""Build extractable entity slots from ``structure.businessTerms`` only."""
|
|
2399
|
+
catalog: list[dict[str, str]] = []
|
|
2400
|
+
seen_types: set[str] = set()
|
|
2401
|
+
for term in list(_schema_structure(schema).get("businessTerms", [])) + list(_CORE_LOCATION_DIMENSIONS):
|
|
2402
|
+
if not isinstance(term, dict):
|
|
2403
|
+
continue
|
|
2404
|
+
if term.get("type") not in ("dimension", "entity"):
|
|
2405
|
+
continue
|
|
2406
|
+
label = str(term.get("term") or "").strip()
|
|
2407
|
+
type_code = str(term.get("mapsTo") or "").strip()
|
|
2408
|
+
if not label or not type_code or type_code in seen_types:
|
|
2409
|
+
continue
|
|
2410
|
+
seen_types.add(type_code)
|
|
2411
|
+
filter_field = label if label.endswith("名称") else f"{label}名称"
|
|
2412
|
+
catalog.append({"type": type_code, "label": label, "filterField": filter_field})
|
|
2413
|
+
return catalog
|
|
2414
|
+
|
|
2415
|
+
|
|
2416
|
+
def _llm_extract_entity_values(
|
|
2417
|
+
question: str,
|
|
2418
|
+
catalog: list[dict[str, str]],
|
|
2419
|
+
sampler: Sampler,
|
|
2420
|
+
agent_instructions: list[dict[str, Any]] | None = None,
|
|
2421
|
+
) -> dict[str, Any]:
|
|
2422
|
+
"""One LLM call that extracts open values for every catalog type at once."""
|
|
2423
|
+
field_lines = "\n".join(f'- {item["type"]}: {item["label"]}' for item in catalog)
|
|
2424
|
+
instruction_lines = format_numbered_instruction_lines(
|
|
2425
|
+
filter_agent_instructions_by_stage(agent_instructions or [], "before")
|
|
2426
|
+
)
|
|
2427
|
+
system = (
|
|
2428
|
+
"你是一个严格的实体槽位抽取器。payload 中的 mandatoryAgentInstructions 必须逐条遵守。"
|
|
2429
|
+
"从用户问题中抽取字段值,"
|
|
2430
|
+
"只输出一个 JSON 对象,键为字段代码,值为问题中出现的实体名称原文;"
|
|
2431
|
+
"问题中没有提到的字段一律输出 null。不要输出任何解释或多余文本。"
|
|
2432
|
+
"城市(如上海、北京)填 city;区县(如杨浦区、浦东新区)填 region,不要填 city;"
|
|
2433
|
+
"具体门店(含店/路/广场等)填 store。"
|
|
2434
|
+
"品牌与项目可能同名:如果用户明确说这是项目,只填 sponge_project,不填 brand;"
|
|
2435
|
+
"如果用户明确说这是品牌,只填 brand,不填 sponge_project;"
|
|
2436
|
+
"只有问题没有明确标注品牌或项目时,brand 与 sponge_project 才都填同一检索词。"
|
|
2437
|
+
"实体名称必须保留用户原文中的括号补充说明,例如 Mstand(打包店员)不能改成 Mstand。"
|
|
2438
|
+
"遇到“名称 + 时间词 + 阶段/动作词 + 业务对象/指标”的连续短语时,必须拆分后只保留名称部分为实体值;"
|
|
2439
|
+
"例如“成都你六姐明天面试的工单数量”中,brand 和 sponge_project 都填“成都你六姐”,"
|
|
2440
|
+
"不要把“成都你六姐明天面试”填入任何字段,尤其不能填入工单、报名时间、面试时间等字段。"
|
|
2441
|
+
"时间、查询动作、统计意图、范围词和状态口径不要填为实体值;这些不是品牌、项目、门店、城市等名称。"
|
|
2442
|
+
"同时输出 explicitTypes 数组,放入用户明确标注的字段代码;未明确标注则输出空数组。"
|
|
2443
|
+
)
|
|
2444
|
+
payload = {
|
|
2445
|
+
"mandatoryAgentInstructions": instruction_lines,
|
|
2446
|
+
"fields": field_lines,
|
|
2447
|
+
"question": question,
|
|
2448
|
+
"outputExample": {
|
|
2449
|
+
"brand": None,
|
|
2450
|
+
"sponge_project": "Mstand(打包店员)",
|
|
2451
|
+
"explicitTypes": ["sponge_project"],
|
|
2452
|
+
},
|
|
2453
|
+
"compoundPhraseExample": {
|
|
2454
|
+
"question": "成都你六姐明天面试的工单数量",
|
|
2455
|
+
"brand": "成都你六姐",
|
|
2456
|
+
"sponge_project": "成都你六姐",
|
|
2457
|
+
"mvp_applied_jobs_by_helped_account": None,
|
|
2458
|
+
"sign_up_time": None,
|
|
2459
|
+
"explicitTypes": [],
|
|
2460
|
+
},
|
|
2461
|
+
}
|
|
2462
|
+
prompt = json.dumps(payload, ensure_ascii=False, indent=2)
|
|
2463
|
+
try:
|
|
2464
|
+
raw = sampler(system, prompt)
|
|
2465
|
+
except Exception: # noqa: BLE001 - extraction is best-effort, never fatal
|
|
2466
|
+
return {}
|
|
2467
|
+
payload = _load_json_object_safe(raw)
|
|
2468
|
+
values: dict[str, Any] = {}
|
|
2469
|
+
allowed = {item["type"] for item in catalog}
|
|
2470
|
+
value_payload = payload.get("values")
|
|
2471
|
+
if not isinstance(value_payload, dict):
|
|
2472
|
+
value_payload = payload
|
|
2473
|
+
for key, value in value_payload.items():
|
|
2474
|
+
if key in allowed and isinstance(value, str) and value.strip():
|
|
2475
|
+
values[key] = value.strip()
|
|
2476
|
+
explicit_types = _normalize_explicit_entity_types(payload.get("explicitTypes"), allowed)
|
|
2477
|
+
if explicit_types:
|
|
2478
|
+
values[_EXPLICIT_ENTITY_TYPES_KEY] = explicit_types
|
|
2479
|
+
return values
|
|
2480
|
+
|
|
2481
|
+
|
|
2482
|
+
def _normalize_explicit_entity_types(raw: Any, allowed: set[str]) -> list[str]:
|
|
2483
|
+
if not isinstance(raw, list):
|
|
2484
|
+
return []
|
|
2485
|
+
result: list[str] = []
|
|
2486
|
+
for item in raw:
|
|
2487
|
+
entity_type = _normalize_entity_slot_type(str(item or "").strip())
|
|
2488
|
+
if entity_type and entity_type in allowed and entity_type not in result:
|
|
2489
|
+
result.append(entity_type)
|
|
2490
|
+
return result
|
|
2491
|
+
|
|
2492
|
+
|
|
2493
|
+
def _normalize_entity_slot_type(entity_type: str) -> str:
|
|
2494
|
+
mapped = {
|
|
2495
|
+
"品牌": "brand",
|
|
2496
|
+
"项目": "sponge_project",
|
|
2497
|
+
"工程": "sponge_project",
|
|
2498
|
+
"门店": "store",
|
|
2499
|
+
"城市": "city",
|
|
2500
|
+
"省份": "province",
|
|
2501
|
+
"区域": "region",
|
|
2502
|
+
"区县": "region",
|
|
2503
|
+
}.get(entity_type)
|
|
2504
|
+
if mapped:
|
|
2505
|
+
return mapped
|
|
2506
|
+
if entity_type == "project":
|
|
2507
|
+
return "sponge_project"
|
|
2508
|
+
return entity_type
|
|
2509
|
+
|
|
2510
|
+
|
|
2511
|
+
def _merge_analysis_entity_candidates_into_llm_values(
|
|
2512
|
+
llm_values: dict[str, Any],
|
|
2513
|
+
analysis: QuestionAnalysis | None,
|
|
2514
|
+
catalog: list[dict[str, str]],
|
|
2515
|
+
) -> dict[str, Any]:
|
|
2516
|
+
"""Use the demand-analysis entity candidates as a fallback for entity search."""
|
|
2517
|
+
if analysis is None:
|
|
2518
|
+
return dict(llm_values)
|
|
2519
|
+
allowed = {item["type"] for item in catalog}
|
|
2520
|
+
merged = dict(llm_values)
|
|
2521
|
+
for candidate in analysis.entity_candidates:
|
|
2522
|
+
if not isinstance(candidate, dict) or candidate.get("isNoise") is True:
|
|
2523
|
+
continue
|
|
2524
|
+
entity_type = _normalize_entity_slot_type(str(candidate.get("type") or "").strip())
|
|
2525
|
+
value = str(candidate.get("value") or "").strip()
|
|
2526
|
+
if not entity_type or entity_type not in allowed or not value:
|
|
2527
|
+
continue
|
|
2528
|
+
if _is_analysis_noise_value(value, analysis):
|
|
2529
|
+
continue
|
|
2530
|
+
merged.setdefault(entity_type, value)
|
|
2531
|
+
return merged
|
|
2532
|
+
|
|
2533
|
+
|
|
2534
|
+
def _drop_implicit_location_values_inside_named_entities(llm_values: dict[str, Any]) -> None:
|
|
2535
|
+
"""Avoid treating the location-looking prefix of a brand/project name as a separate location."""
|
|
2536
|
+
explicit_types = llm_values.get(_EXPLICIT_ENTITY_TYPES_KEY)
|
|
2537
|
+
explicit = set(explicit_types) if isinstance(explicit_types, list) else set()
|
|
2538
|
+
if explicit.intersection({"province", "city", "region"}):
|
|
2539
|
+
return
|
|
2540
|
+
named_values = [
|
|
2541
|
+
str(llm_values.get(entity_type) or "").strip()
|
|
2542
|
+
for entity_type in ("brand", "sponge_project", "project", "store")
|
|
2543
|
+
if str(llm_values.get(entity_type) or "").strip()
|
|
2544
|
+
]
|
|
2545
|
+
if not named_values:
|
|
2546
|
+
return
|
|
2547
|
+
for entity_type in ("province", "city", "region"):
|
|
2548
|
+
value = str(llm_values.get(entity_type) or "").strip()
|
|
2549
|
+
if value and any(value != name and value in name for name in named_values):
|
|
2550
|
+
llm_values.pop(entity_type, None)
|
|
2551
|
+
|
|
2552
|
+
|
|
2553
|
+
def _is_analysis_noise_value(value: Any, analysis: QuestionAnalysis | None) -> bool:
|
|
2554
|
+
normalized = _normalize_analysis_noise_text(value)
|
|
2555
|
+
if not normalized:
|
|
2556
|
+
return False
|
|
2557
|
+
return normalized in _analysis_noise_value_set(analysis)
|
|
2558
|
+
|
|
2559
|
+
|
|
2560
|
+
def _analysis_noise_value_set(analysis: QuestionAnalysis | None) -> set[str]:
|
|
2561
|
+
if analysis is None:
|
|
2562
|
+
return set()
|
|
2563
|
+
values: set[str] = set()
|
|
2564
|
+
for item in analysis.noise_values:
|
|
2565
|
+
if not isinstance(item, dict):
|
|
2566
|
+
continue
|
|
2567
|
+
normalized = _normalize_analysis_noise_text(item.get("value"))
|
|
2568
|
+
if normalized:
|
|
2569
|
+
values.add(normalized)
|
|
2570
|
+
for item in analysis.entity_candidates:
|
|
2571
|
+
if not isinstance(item, dict) or item.get("isNoise") is not True:
|
|
2572
|
+
continue
|
|
2573
|
+
normalized = _normalize_analysis_noise_text(item.get("value"))
|
|
2574
|
+
if normalized:
|
|
2575
|
+
values.add(normalized)
|
|
2576
|
+
return values
|
|
2577
|
+
|
|
2578
|
+
|
|
2579
|
+
def _normalize_analysis_noise_text(value: Any) -> str:
|
|
2580
|
+
return str(value or "").strip().strip("%").strip().lower()
|
|
2581
|
+
|
|
2582
|
+
|
|
2583
|
+
def _load_json_object_safe(text: str) -> dict[str, Any]:
|
|
2584
|
+
cleaned = (text or "").strip()
|
|
2585
|
+
if cleaned.startswith("```"):
|
|
2586
|
+
cleaned = cleaned.strip("`")
|
|
2587
|
+
if cleaned.lower().startswith("json"):
|
|
2588
|
+
cleaned = cleaned[4:].strip()
|
|
2589
|
+
try:
|
|
2590
|
+
payload = json.loads(cleaned)
|
|
2591
|
+
except json.JSONDecodeError:
|
|
2592
|
+
start = cleaned.find("{")
|
|
2593
|
+
end = cleaned.rfind("}")
|
|
2594
|
+
if start == -1 or end == -1 or end <= start:
|
|
2595
|
+
return {}
|
|
2596
|
+
try:
|
|
2597
|
+
payload = json.loads(cleaned[start : end + 1])
|
|
2598
|
+
except json.JSONDecodeError:
|
|
2599
|
+
return {}
|
|
2600
|
+
return payload if isinstance(payload, dict) else {}
|
|
2601
|
+
|
|
2602
|
+
|
|
2603
|
+
def _extract_question_entities(
|
|
2604
|
+
question: str,
|
|
2605
|
+
schema: dict[str, Any],
|
|
2606
|
+
sampler: Sampler | None,
|
|
2607
|
+
*,
|
|
2608
|
+
agent_instructions: list[dict[str, Any]] | None = None,
|
|
2609
|
+
analysis: QuestionAnalysis | None = None,
|
|
2610
|
+
) -> list[dict[str, str]]:
|
|
2611
|
+
"""Unified schema-driven entity extraction used by both filters and entities."""
|
|
2612
|
+
results: list[dict[str, str]] = []
|
|
2613
|
+
catalog = _entity_catalog(schema)
|
|
2614
|
+
llm_values: dict[str, Any] = {}
|
|
2615
|
+
if sampler is not None and catalog:
|
|
2616
|
+
llm_values = _llm_extract_entity_values(question, catalog, sampler, agent_instructions)
|
|
2617
|
+
llm_values = _merge_analysis_entity_candidates_into_llm_values(llm_values, analysis, catalog)
|
|
2618
|
+
_drop_implicit_location_values_inside_named_entities(llm_values)
|
|
2619
|
+
for item in catalog:
|
|
2620
|
+
value = llm_values.get(item["type"]) or _extract_value_near_label(question, item["label"])
|
|
2621
|
+
if value and not _is_analysis_noise_value(value, analysis):
|
|
2622
|
+
results.append(
|
|
2623
|
+
{
|
|
2624
|
+
"type": item["type"],
|
|
2625
|
+
"label": item["label"],
|
|
2626
|
+
"filterField": item["filterField"],
|
|
2627
|
+
"value": value,
|
|
2628
|
+
}
|
|
2629
|
+
)
|
|
2630
|
+
return results
|
|
2631
|
+
|
|
2632
|
+
|
|
2633
|
+
def _time_range_source_question(original_question: str, normalized_question: str) -> str:
|
|
2634
|
+
"""Prefer the user's original wording for date parsing when they omitted an explicit year."""
|
|
2635
|
+
original = original_question.strip()
|
|
2636
|
+
normalized = normalized_question.strip()
|
|
2637
|
+
if not original:
|
|
2638
|
+
return normalized
|
|
2639
|
+
if re.search(r"20\d{2}\s*年", original):
|
|
2640
|
+
return original
|
|
2641
|
+
if re.search(r"20\d{2}\s*年", normalized):
|
|
2642
|
+
return original
|
|
2643
|
+
return normalized or original
|
|
2644
|
+
|
|
2645
|
+
|
|
2646
|
+
def _query_plan_filters(
|
|
2647
|
+
question: str,
|
|
2648
|
+
schema_context: Any,
|
|
2649
|
+
*,
|
|
2650
|
+
searched_entities: dict[str, Any] | None = None,
|
|
2651
|
+
question_entities: list[dict[str, str]] | None = None,
|
|
2652
|
+
schema: dict[str, Any] | None = None,
|
|
2653
|
+
time_range_question: str | None = None,
|
|
2654
|
+
selected_concepts: list[dict[str, Any]] | None = None,
|
|
2655
|
+
analysis: QuestionAnalysis | None = None,
|
|
2656
|
+
) -> list[dict[str, str]]:
|
|
2657
|
+
filters: list[dict[str, str]] = []
|
|
2658
|
+
|
|
2659
|
+
def _append_filter(item: dict[str, str]) -> None:
|
|
2660
|
+
field = _resolve_query_plan_filter_field(str(item.get("field") or ""), schema)
|
|
2661
|
+
normalized = {**item, "field": field}
|
|
2662
|
+
key = (field, str(normalized.get("operator") or ""), str(normalized.get("value") or ""))
|
|
2663
|
+
if any(existing for existing in filters if (existing.get("field"), existing.get("operator"), existing.get("value")) == key):
|
|
2664
|
+
return
|
|
2665
|
+
filters.append(normalized)
|
|
2666
|
+
|
|
2667
|
+
if isinstance(searched_entities, dict):
|
|
2668
|
+
for candidate in searched_entities.get("schemaCandidates") or []:
|
|
2669
|
+
if not isinstance(candidate, dict):
|
|
2670
|
+
continue
|
|
2671
|
+
entity_type = str(candidate.get("type") or "")
|
|
2672
|
+
normalized_value = _normalize_entity_search_value(entity_type, str(candidate.get("value") or ""))
|
|
2673
|
+
if _is_analysis_noise_value(normalized_value, analysis):
|
|
2674
|
+
continue
|
|
2675
|
+
_append_filter(
|
|
2676
|
+
{
|
|
2677
|
+
"field": str(candidate.get("field") or _entity_search_filter_field(entity_type)),
|
|
2678
|
+
"operator": "LIKE",
|
|
2679
|
+
"value": f"%{normalized_value}%",
|
|
2680
|
+
"source": str(candidate.get("source") or "schema"),
|
|
2681
|
+
}
|
|
2682
|
+
)
|
|
2683
|
+
|
|
2684
|
+
for entity in question_entities or []:
|
|
2685
|
+
if _is_analysis_noise_value(entity.get("value"), analysis):
|
|
2686
|
+
continue
|
|
2687
|
+
_append_filter(
|
|
2688
|
+
{
|
|
2689
|
+
"field": entity["filterField"],
|
|
2690
|
+
"operator": "LIKE",
|
|
2691
|
+
"value": f"%{entity['value']}%",
|
|
2692
|
+
"source": "用户问题",
|
|
2693
|
+
}
|
|
2694
|
+
)
|
|
2695
|
+
for filter_item in _query_plan_search_entity_filters(searched_entities, schema):
|
|
2696
|
+
_append_filter(filter_item)
|
|
2697
|
+
for candidate in _direct_entity_candidates(searched_entities):
|
|
2698
|
+
maps_to = str(candidate.get("mapsTo") or candidate.get("type") or "")
|
|
2699
|
+
field = str(candidate.get("field") or "")
|
|
2700
|
+
if not field or "." not in field:
|
|
2701
|
+
field = _entity_table_name_column(maps_to, schema) if schema and maps_to else _entity_search_filter_field(str(candidate.get("type") or ""))
|
|
2702
|
+
raw_value = str(candidate.get("value") or "")
|
|
2703
|
+
normalized_value = _normalize_entity_search_value(maps_to, _normalize_entity_search_value(str(candidate.get("type") or ""), raw_value))
|
|
2704
|
+
if _is_analysis_noise_value(normalized_value, analysis):
|
|
2705
|
+
continue
|
|
2706
|
+
_append_filter(
|
|
2707
|
+
{
|
|
2708
|
+
"field": field,
|
|
2709
|
+
"operator": "LIKE",
|
|
2710
|
+
"value": f"%{normalized_value}%",
|
|
2711
|
+
"source": str(candidate.get("source") or "schema.columns"),
|
|
2712
|
+
}
|
|
2713
|
+
)
|
|
2714
|
+
for enum_filter in _query_plan_enum_filters(question, getattr(schema_context, "enums", []) or []):
|
|
2715
|
+
filters.append(enum_filter)
|
|
2716
|
+
if selected_concepts is None:
|
|
2717
|
+
selected_concepts = match_concepts_for_question(getattr(schema_context, "concepts", []) or [], question)
|
|
2718
|
+
filters.extend(_query_plan_concept_filters(selected_concepts))
|
|
2719
|
+
for default_filter in getattr(schema_context, "table_defaults", []) or []:
|
|
2720
|
+
if not isinstance(default_filter, dict):
|
|
2721
|
+
continue
|
|
2722
|
+
condition = (
|
|
2723
|
+
default_filter.get("condition")
|
|
2724
|
+
or default_filter.get("where")
|
|
2725
|
+
or default_filter.get("predicate")
|
|
2726
|
+
or default_filter.get("expression")
|
|
2727
|
+
or default_filter.get("filter")
|
|
2728
|
+
)
|
|
2729
|
+
if condition:
|
|
2730
|
+
filters.append(
|
|
2731
|
+
{
|
|
2732
|
+
"field": str(default_filter.get("table") or ""),
|
|
2733
|
+
"operator": "DEFAULT",
|
|
2734
|
+
"value": str(condition),
|
|
2735
|
+
"source": "structure.defaultFilters",
|
|
2736
|
+
}
|
|
2737
|
+
)
|
|
2738
|
+
filters = _merge_query_plan_equality_filters(filters)
|
|
2739
|
+
filters = _drop_conflicting_status_concept_filters(filters)
|
|
2740
|
+
filters = _drop_schema_column_noise_filters(filters, analysis=analysis)
|
|
2741
|
+
filters = _drop_time_field_text_like_filters(filters, analysis=analysis)
|
|
2742
|
+
filters = _apply_query_plan_time_range_filters(
|
|
2743
|
+
filters,
|
|
2744
|
+
time_range_question or question,
|
|
2745
|
+
schema,
|
|
2746
|
+
selected_concepts=selected_concepts,
|
|
2747
|
+
analysis=analysis,
|
|
2748
|
+
)
|
|
2749
|
+
return filters
|
|
2750
|
+
|
|
2751
|
+
|
|
2752
|
+
def _query_plan_search_entity_filters(
|
|
2753
|
+
searched_entities: dict[str, Any] | None,
|
|
2754
|
+
schema: dict[str, Any] | None,
|
|
2755
|
+
) -> list[dict[str, str]]:
|
|
2756
|
+
result: list[dict[str, str]] = []
|
|
2757
|
+
for unique_entity in _entity_search_filter_items(searched_entities):
|
|
2758
|
+
maps_to = _search_maps_to_for_entity_type(unique_entity.get("type", ""))
|
|
2759
|
+
field = _entity_table_name_column(maps_to, schema) if schema and maps_to else _entity_search_filter_field(unique_entity["type"])
|
|
2760
|
+
if unique_entity.get("type") == "city":
|
|
2761
|
+
name_variants = _city_name_filter_values(unique_entity["name"])
|
|
2762
|
+
elif unique_entity.get("type") == "region":
|
|
2763
|
+
name_variants = _region_name_filter_values(unique_entity["name"])
|
|
2764
|
+
else:
|
|
2765
|
+
name_variants = [unique_entity["name"]]
|
|
2766
|
+
for variant in name_variants:
|
|
2767
|
+
filter_item = {
|
|
2768
|
+
"field": field,
|
|
2769
|
+
"operator": "LIKE",
|
|
2770
|
+
"value": f"%{variant}%",
|
|
2771
|
+
"source": "search_entities",
|
|
2772
|
+
}
|
|
2773
|
+
logical_group = unique_entity.get("logicalGroup")
|
|
2774
|
+
if logical_group:
|
|
2775
|
+
filter_item["logicalGroup"] = logical_group
|
|
2776
|
+
filter_item["logicalOperator"] = "OR"
|
|
2777
|
+
result.append(filter_item)
|
|
2778
|
+
return result
|
|
2779
|
+
|
|
2780
|
+
|
|
2781
|
+
def _apply_query_plan_time_range_filters(
|
|
2782
|
+
filters: list[dict[str, str]],
|
|
2783
|
+
question: str,
|
|
2784
|
+
schema: dict[str, Any] | None,
|
|
2785
|
+
*,
|
|
2786
|
+
selected_concepts: list[dict[str, Any]] | None = None,
|
|
2787
|
+
analysis: QuestionAnalysis | None = None,
|
|
2788
|
+
) -> list[dict[str, str]]:
|
|
2789
|
+
"""Replace noisy date LIKE filters with explicit range filters defaulting to current year."""
|
|
2790
|
+
time_filters = _analysis_time_range_plan_filters(analysis, selected_concepts=selected_concepts)
|
|
2791
|
+
if not time_filters:
|
|
2792
|
+
time_filters = time_range_plan_filters(question, schema, concepts=selected_concepts)
|
|
2793
|
+
if not time_filters:
|
|
2794
|
+
return filters
|
|
2795
|
+
qualified_fields = {str(item.get("field") or "") for item in time_filters if item.get("field")}
|
|
2796
|
+
column_names = {field.split(".", 1)[-1] for field in qualified_fields if field}
|
|
2797
|
+
cleaned: list[dict[str, str]] = []
|
|
2798
|
+
for item in filters:
|
|
2799
|
+
field = str(item.get("field") or "")
|
|
2800
|
+
operator = str(item.get("operator") or "").upper()
|
|
2801
|
+
if operator == "LIKE" and (
|
|
2802
|
+
field in qualified_fields
|
|
2803
|
+
or field.split(".", 1)[-1] in column_names
|
|
2804
|
+
or _looks_like_time_range_noise_like(item.get("value"), analysis=analysis)
|
|
2805
|
+
):
|
|
2806
|
+
continue
|
|
2807
|
+
cleaned.append(item)
|
|
2808
|
+
return cleaned + time_filters
|
|
2809
|
+
|
|
2810
|
+
|
|
2811
|
+
def _analysis_time_range_plan_filters(
|
|
2812
|
+
analysis: QuestionAnalysis | None,
|
|
2813
|
+
*,
|
|
2814
|
+
selected_concepts: list[dict[str, Any]] | None = None,
|
|
2815
|
+
) -> list[dict[str, str]]:
|
|
2816
|
+
if analysis is None or not isinstance(analysis.time_range, dict):
|
|
2817
|
+
return []
|
|
2818
|
+
start = str(analysis.time_range.get("start") or "").strip()
|
|
2819
|
+
end = str(analysis.time_range.get("endExclusive") or analysis.time_range.get("end") or "").strip()
|
|
2820
|
+
if not (_looks_like_iso_date(start) and _looks_like_iso_date(end)):
|
|
2821
|
+
return []
|
|
2822
|
+
field = _time_range_field_from_concepts(selected_concepts)
|
|
2823
|
+
if not field:
|
|
2824
|
+
return []
|
|
2825
|
+
original = str(analysis.time_range.get("originalText") or analysis.time_range.get("value") or "时间范围").strip()
|
|
2826
|
+
return [
|
|
2827
|
+
{
|
|
2828
|
+
"field": field,
|
|
2829
|
+
"operator": ">=",
|
|
2830
|
+
"value": start[:10],
|
|
2831
|
+
"description": f"{original}起",
|
|
2832
|
+
"source": "question_analysis.timeRange",
|
|
2833
|
+
},
|
|
2834
|
+
{
|
|
2835
|
+
"field": field,
|
|
2836
|
+
"operator": "<",
|
|
2837
|
+
"value": end[:10],
|
|
2838
|
+
"description": f"{original}止",
|
|
2839
|
+
"source": "question_analysis.timeRange",
|
|
2840
|
+
},
|
|
2841
|
+
]
|
|
2842
|
+
|
|
2843
|
+
|
|
2844
|
+
def _time_range_field_from_concepts(concepts: list[dict[str, Any]] | None) -> str:
|
|
2845
|
+
for concept in concepts or []:
|
|
2846
|
+
if not isinstance(concept, dict):
|
|
2847
|
+
continue
|
|
2848
|
+
field = str(concept.get("timeRangeField") or "").strip()
|
|
2849
|
+
if "." in field:
|
|
2850
|
+
return field
|
|
2851
|
+
return ""
|
|
2852
|
+
|
|
2853
|
+
|
|
2854
|
+
def _looks_like_iso_date(value: str) -> bool:
|
|
2855
|
+
return bool(re.fullmatch(r"20\d{2}-\d{2}-\d{2}", value[:10]))
|
|
2856
|
+
|
|
2857
|
+
|
|
2858
|
+
def _drop_time_field_text_like_filters(
|
|
2859
|
+
filters: list[dict[str, str]],
|
|
2860
|
+
*,
|
|
2861
|
+
analysis: QuestionAnalysis | None = None,
|
|
2862
|
+
) -> list[dict[str, str]]:
|
|
2863
|
+
cleaned: list[dict[str, str]] = []
|
|
2864
|
+
for item in filters:
|
|
2865
|
+
operator = str(item.get("operator") or "").upper()
|
|
2866
|
+
field = str(item.get("field") or "")
|
|
2867
|
+
value = str(item.get("value") or "")
|
|
2868
|
+
if operator == "LIKE" and _is_time_like_query_plan_field(field):
|
|
2869
|
+
stripped = value.strip().strip("%").strip()
|
|
2870
|
+
if _is_analysis_noise_value_or_part(stripped, analysis) or not _contains_date_like_text(stripped):
|
|
2871
|
+
continue
|
|
2872
|
+
cleaned.append(item)
|
|
2873
|
+
return cleaned
|
|
2874
|
+
|
|
2875
|
+
|
|
2876
|
+
def _is_time_like_query_plan_field(field: str) -> bool:
|
|
2877
|
+
column = field.rsplit(".", 1)[-1].strip("`").lower()
|
|
2878
|
+
return bool(column in {"date", "day"} or column.endswith("_time") or column.endswith("_date") or column.endswith("_at"))
|
|
2879
|
+
|
|
2880
|
+
|
|
2881
|
+
def _contains_date_like_text(value: str) -> bool:
|
|
2882
|
+
return bool(
|
|
2883
|
+
re.search(r"20\d{2}-\d{1,2}-\d{1,2}", value)
|
|
2884
|
+
or re.search(r"20\d{2}\s*年", value)
|
|
2885
|
+
or re.search(r"\d{1,2}\s*月\s*\d{1,2}\s*日", value)
|
|
2886
|
+
)
|
|
2887
|
+
|
|
2888
|
+
|
|
2889
|
+
def _looks_like_time_range_noise_like(value: Any, *, analysis: QuestionAnalysis | None = None) -> bool:
|
|
2890
|
+
text = str(value or "")
|
|
2891
|
+
if not text:
|
|
2892
|
+
return False
|
|
2893
|
+
stripped = text.strip("%")
|
|
2894
|
+
if _is_analysis_noise_value(stripped, analysis):
|
|
2895
|
+
return True
|
|
2896
|
+
return bool(re.search(r"\d{1,2}\s*月", stripped) and re.search(r"(到|至|日|整个|整月|之间)", stripped))
|
|
2897
|
+
|
|
2898
|
+
|
|
2899
|
+
def _is_schema_column_name_filter(field: str) -> bool:
|
|
2900
|
+
if "." not in field:
|
|
2901
|
+
return False
|
|
2902
|
+
column = field.rsplit(".", 1)[-1].lower()
|
|
2903
|
+
return column in {"name", "title"} or column.endswith("_name")
|
|
2904
|
+
|
|
2905
|
+
|
|
2906
|
+
_GENERIC_ENTITY_NAME_VALUES = frozenset(
|
|
2907
|
+
{
|
|
2908
|
+
"岗位",
|
|
2909
|
+
"职位",
|
|
2910
|
+
"工单",
|
|
2911
|
+
"门店",
|
|
2912
|
+
"品牌",
|
|
2913
|
+
"项目",
|
|
2914
|
+
"供应商",
|
|
2915
|
+
"城市",
|
|
2916
|
+
"区域",
|
|
2917
|
+
"省份",
|
|
2918
|
+
}
|
|
2919
|
+
)
|
|
2920
|
+
|
|
2921
|
+
|
|
2922
|
+
def _is_generic_entity_name_value(value: Any) -> bool:
|
|
2923
|
+
normalized = str(value or "").strip().strip("%").strip()
|
|
2924
|
+
return normalized in _GENERIC_ENTITY_NAME_VALUES
|
|
2925
|
+
|
|
2926
|
+
|
|
2927
|
+
def _drop_schema_column_noise_filters(
|
|
2928
|
+
filters: list[dict[str, str]],
|
|
2929
|
+
*,
|
|
2930
|
+
analysis: QuestionAnalysis | None = None,
|
|
2931
|
+
) -> list[dict[str, str]]:
|
|
2932
|
+
"""Drop name-field LIKE filters whose values are query-intent phrases, not entity names."""
|
|
2933
|
+
authoritative_tables: set[str] = set()
|
|
2934
|
+
for item in filters:
|
|
2935
|
+
source = str(item.get("source") or "")
|
|
2936
|
+
if source not in {"search_entities", "search_entities.candidate", "schema.businessTerms", "用户问题"}:
|
|
2937
|
+
continue
|
|
2938
|
+
field = str(item.get("field") or "")
|
|
2939
|
+
if "." in field:
|
|
2940
|
+
authoritative_tables.add(field.split(".", 1)[0])
|
|
2941
|
+
|
|
2942
|
+
cleaned: list[dict[str, str]] = []
|
|
2943
|
+
for item in filters:
|
|
2944
|
+
operator = str(item.get("operator") or "").upper()
|
|
2945
|
+
field = str(item.get("field") or "")
|
|
2946
|
+
source = str(item.get("source") or "")
|
|
2947
|
+
if operator == "LIKE" and _is_schema_column_name_filter(field):
|
|
2948
|
+
if _is_analysis_noise_value(item.get("value"), analysis) or _is_generic_entity_name_value(item.get("value")):
|
|
2949
|
+
continue
|
|
2950
|
+
if source != "schema.columns":
|
|
2951
|
+
cleaned.append(item)
|
|
2952
|
+
continue
|
|
2953
|
+
if operator != "LIKE":
|
|
2954
|
+
cleaned.append(item)
|
|
2955
|
+
continue
|
|
2956
|
+
if not _is_schema_column_name_filter(field):
|
|
2957
|
+
continue
|
|
2958
|
+
table = field.split(".", 1)[0] if "." in field else ""
|
|
2959
|
+
if table in authoritative_tables:
|
|
2960
|
+
continue
|
|
2961
|
+
cleaned.append(item)
|
|
2962
|
+
return cleaned
|
|
2963
|
+
|
|
2964
|
+
|
|
2965
|
+
def _remove_noise_name_like_filters_from_sql(sql: str, *, analysis: QuestionAnalysis | None = None) -> str:
|
|
2966
|
+
if not sql.strip() or not _analysis_noise_value_set(analysis):
|
|
2967
|
+
return sql
|
|
2968
|
+
where_bounds = _top_level_where_bounds(sql)
|
|
2969
|
+
if where_bounds is None:
|
|
2970
|
+
return sql
|
|
2971
|
+
where_start, body_start, body_end = where_bounds
|
|
2972
|
+
where_body = sql[body_start:body_end].strip()
|
|
2973
|
+
predicates = [predicate for predicate in _split_top_level_and_predicates(where_body) if predicate.strip()]
|
|
2974
|
+
kept = [
|
|
2975
|
+
predicate.strip()
|
|
2976
|
+
for predicate in predicates
|
|
2977
|
+
if not _is_noise_name_like_sql_predicate(predicate, analysis=analysis)
|
|
2978
|
+
]
|
|
2979
|
+
if len(kept) == len(predicates):
|
|
2980
|
+
return sql
|
|
2981
|
+
prefix = sql[:where_start].rstrip()
|
|
2982
|
+
suffix = sql[body_end:].strip()
|
|
2983
|
+
rebuilt = f"{prefix} WHERE {' AND '.join(kept)}" if kept else prefix
|
|
2984
|
+
if suffix:
|
|
2985
|
+
rebuilt = f"{rebuilt} {suffix}"
|
|
2986
|
+
return rebuilt.strip()
|
|
2987
|
+
|
|
2988
|
+
|
|
2989
|
+
def _top_level_where_bounds(sql: str) -> tuple[int, int, int] | None:
|
|
2990
|
+
where_start = _find_top_level_sql_keyword(sql, "where", start=0)
|
|
2991
|
+
if where_start is None:
|
|
2992
|
+
return None
|
|
2993
|
+
body_start = where_start + len("where")
|
|
2994
|
+
suffix_starts = [
|
|
2995
|
+
index
|
|
2996
|
+
for keyword in ("group by", "order by", "having", "limit")
|
|
2997
|
+
if (index := _find_top_level_sql_keyword(sql, keyword, start=body_start)) is not None
|
|
2998
|
+
]
|
|
2999
|
+
body_end = min(suffix_starts) if suffix_starts else len(sql)
|
|
3000
|
+
return where_start, body_start, body_end
|
|
3001
|
+
|
|
3002
|
+
|
|
3003
|
+
def _find_top_level_sql_keyword(sql: str, keyword: str, *, start: int) -> int | None:
|
|
3004
|
+
target = keyword.lower()
|
|
3005
|
+
depth = 0
|
|
3006
|
+
quote: str | None = None
|
|
3007
|
+
index = start
|
|
3008
|
+
while index < len(sql):
|
|
3009
|
+
char = sql[index]
|
|
3010
|
+
if quote:
|
|
3011
|
+
if char == quote:
|
|
3012
|
+
if quote == "'" and index + 1 < len(sql) and sql[index + 1] == "'":
|
|
3013
|
+
index += 2
|
|
3014
|
+
continue
|
|
3015
|
+
quote = None
|
|
3016
|
+
index += 1
|
|
3017
|
+
continue
|
|
3018
|
+
if char in {"'", '"', "`"}:
|
|
3019
|
+
quote = char
|
|
3020
|
+
index += 1
|
|
3021
|
+
continue
|
|
3022
|
+
if char == "(":
|
|
3023
|
+
depth += 1
|
|
3024
|
+
index += 1
|
|
3025
|
+
continue
|
|
3026
|
+
if char == ")":
|
|
3027
|
+
depth = max(0, depth - 1)
|
|
3028
|
+
index += 1
|
|
3029
|
+
continue
|
|
3030
|
+
if depth == 0 and sql[index : index + len(target)].lower() == target:
|
|
3031
|
+
before = sql[index - 1] if index > 0 else " "
|
|
3032
|
+
after = sql[index + len(target)] if index + len(target) < len(sql) else " "
|
|
3033
|
+
if not (before.isalnum() or before == "_") and not (after.isalnum() or after == "_"):
|
|
3034
|
+
return index
|
|
3035
|
+
index += 1
|
|
3036
|
+
return None
|
|
3037
|
+
|
|
3038
|
+
|
|
3039
|
+
def _split_top_level_and_predicates(where_body: str) -> list[str]:
|
|
3040
|
+
predicates: list[str] = []
|
|
3041
|
+
depth = 0
|
|
3042
|
+
quote: str | None = None
|
|
3043
|
+
start = 0
|
|
3044
|
+
index = 0
|
|
3045
|
+
while index < len(where_body):
|
|
3046
|
+
char = where_body[index]
|
|
3047
|
+
if quote:
|
|
3048
|
+
if char == quote:
|
|
3049
|
+
if quote == "'" and index + 1 < len(where_body) and where_body[index + 1] == "'":
|
|
3050
|
+
index += 2
|
|
3051
|
+
continue
|
|
3052
|
+
quote = None
|
|
3053
|
+
index += 1
|
|
3054
|
+
continue
|
|
3055
|
+
if char in {"'", '"', "`"}:
|
|
3056
|
+
quote = char
|
|
3057
|
+
index += 1
|
|
3058
|
+
continue
|
|
3059
|
+
if char == "(":
|
|
3060
|
+
depth += 1
|
|
3061
|
+
index += 1
|
|
3062
|
+
continue
|
|
3063
|
+
if char == ")":
|
|
3064
|
+
depth = max(0, depth - 1)
|
|
3065
|
+
index += 1
|
|
3066
|
+
continue
|
|
3067
|
+
if depth == 0 and where_body[index : index + 3].lower() == "and":
|
|
3068
|
+
before = where_body[index - 1] if index > 0 else " "
|
|
3069
|
+
after = where_body[index + 3] if index + 3 < len(where_body) else " "
|
|
3070
|
+
if not (before.isalnum() or before == "_") and not (after.isalnum() or after == "_"):
|
|
3071
|
+
predicates.append(where_body[start:index])
|
|
3072
|
+
start = index + 3
|
|
3073
|
+
index += 3
|
|
3074
|
+
continue
|
|
3075
|
+
index += 1
|
|
3076
|
+
predicates.append(where_body[start:])
|
|
3077
|
+
return predicates
|
|
3078
|
+
|
|
3079
|
+
|
|
3080
|
+
def _is_noise_name_like_sql_predicate(predicate: str, *, analysis: QuestionAnalysis | None) -> bool:
|
|
3081
|
+
match = re.fullmatch(
|
|
3082
|
+
r"\s*(?P<field>(?:`?[\w]+`?\.)?`?[\w]+`?)\s+LIKE\s+(?P<quote>'|\")(?P<value>.*)(?P=quote)\s*",
|
|
3083
|
+
predicate,
|
|
3084
|
+
flags=re.IGNORECASE | re.DOTALL,
|
|
3085
|
+
)
|
|
3086
|
+
if match is None:
|
|
3087
|
+
return False
|
|
3088
|
+
field = match.group("field").replace("`", "")
|
|
3089
|
+
if not _is_schema_column_name_filter(field):
|
|
3090
|
+
return False
|
|
3091
|
+
return _is_analysis_noise_value_or_part(match.group("value"), analysis)
|
|
3092
|
+
|
|
3093
|
+
|
|
3094
|
+
def _is_analysis_noise_value_or_part(value: Any, analysis: QuestionAnalysis | None) -> bool:
|
|
3095
|
+
normalized = _normalize_analysis_noise_text(value)
|
|
3096
|
+
if not normalized:
|
|
3097
|
+
return False
|
|
3098
|
+
for noise in _analysis_noise_value_set(analysis):
|
|
3099
|
+
if normalized == noise or normalized in noise or noise in normalized:
|
|
3100
|
+
return True
|
|
3101
|
+
return False
|
|
3102
|
+
|
|
3103
|
+
|
|
3104
|
+
def _merge_query_plan_equality_filters(filters: list[dict[str, str]]) -> list[dict[str, str]]:
|
|
3105
|
+
"""Merge multiple ``=`` filters on the same field into one ``IN`` filter."""
|
|
3106
|
+
passthrough: list[dict[str, str]] = []
|
|
3107
|
+
equality_by_field: dict[str, list[dict[str, str]]] = {}
|
|
3108
|
+
for item in filters:
|
|
3109
|
+
if not isinstance(item, dict):
|
|
3110
|
+
continue
|
|
3111
|
+
operator = str(item.get("operator") or "").upper()
|
|
3112
|
+
field = str(item.get("field") or "").strip()
|
|
3113
|
+
if operator == "=" and field:
|
|
3114
|
+
equality_by_field.setdefault(field, []).append(item)
|
|
3115
|
+
else:
|
|
3116
|
+
passthrough.append(item)
|
|
3117
|
+
merged: list[dict[str, str]] = []
|
|
3118
|
+
for field, items in equality_by_field.items():
|
|
3119
|
+
values: list[str] = []
|
|
3120
|
+
descriptions: list[str] = []
|
|
3121
|
+
source = str(items[0].get("source") or "")
|
|
3122
|
+
seen_values: set[str] = set()
|
|
3123
|
+
for item in items:
|
|
3124
|
+
value = str(item.get("value") or "")
|
|
3125
|
+
if not value or value in seen_values:
|
|
3126
|
+
continue
|
|
3127
|
+
seen_values.add(value)
|
|
3128
|
+
values.append(value)
|
|
3129
|
+
desc = str(item.get("description") or "").strip()
|
|
3130
|
+
if desc and desc not in descriptions:
|
|
3131
|
+
descriptions.append(desc)
|
|
3132
|
+
if len(values) <= 1:
|
|
3133
|
+
merged.extend(items)
|
|
3134
|
+
continue
|
|
3135
|
+
entry: dict[str, str] = {
|
|
3136
|
+
"field": field,
|
|
3137
|
+
"operator": "IN",
|
|
3138
|
+
"value": ", ".join(values),
|
|
3139
|
+
"source": source,
|
|
3140
|
+
}
|
|
3141
|
+
if descriptions:
|
|
3142
|
+
entry["description"] = "、".join(descriptions)
|
|
3143
|
+
merged.append(entry)
|
|
3144
|
+
return passthrough + merged
|
|
3145
|
+
|
|
3146
|
+
|
|
3147
|
+
def _drop_conflicting_status_concept_filters(filters: list[dict[str, str]]) -> list[dict[str, str]]:
|
|
3148
|
+
"""Drop single-status concept predicates when the plan already uses multi-value IN."""
|
|
3149
|
+
multi_in_fields: dict[str, set[str]] = {}
|
|
3150
|
+
for item in filters:
|
|
3151
|
+
if str(item.get("operator") or "").upper() != "IN":
|
|
3152
|
+
continue
|
|
3153
|
+
field = str(item.get("field") or "").strip()
|
|
3154
|
+
if not field or not field.endswith(".status"):
|
|
3155
|
+
continue
|
|
3156
|
+
values = {part.strip() for part in str(item.get("value") or "").split(",") if part.strip()}
|
|
3157
|
+
if len(values) > 1:
|
|
3158
|
+
multi_in_fields[field] = values
|
|
3159
|
+
if not multi_in_fields:
|
|
3160
|
+
return filters
|
|
3161
|
+
result: list[dict[str, str]] = []
|
|
3162
|
+
for item in filters:
|
|
3163
|
+
if str(item.get("operator") or "").upper() != "DEFAULT":
|
|
3164
|
+
result.append(item)
|
|
3165
|
+
continue
|
|
3166
|
+
if str(item.get("source") or "") != "schema.concepts":
|
|
3167
|
+
result.append(item)
|
|
3168
|
+
continue
|
|
3169
|
+
predicate = str(item.get("value") or "").strip()
|
|
3170
|
+
drop = False
|
|
3171
|
+
for field, allowed_values in multi_in_fields.items():
|
|
3172
|
+
if not predicate.startswith(f"{field} = "):
|
|
3173
|
+
continue
|
|
3174
|
+
single = predicate[len(f"{field} = ") :].strip()
|
|
3175
|
+
if single in allowed_values:
|
|
3176
|
+
drop = True
|
|
3177
|
+
break
|
|
3178
|
+
if not drop:
|
|
3179
|
+
result.append(item)
|
|
3180
|
+
return result
|
|
3181
|
+
|
|
3182
|
+
|
|
3183
|
+
def _query_plan_concept_filters(concepts: list[dict[str, Any]]) -> list[dict[str, str]]:
|
|
3184
|
+
filters: list[dict[str, str]] = []
|
|
3185
|
+
for concept in concepts:
|
|
3186
|
+
if not isinstance(concept, dict):
|
|
3187
|
+
continue
|
|
3188
|
+
predicates = concept.get("sqlPredicate")
|
|
3189
|
+
if not isinstance(predicates, list):
|
|
3190
|
+
continue
|
|
3191
|
+
description = str(concept.get("comment") or concept.get("name") or "")
|
|
3192
|
+
for predicate in predicates:
|
|
3193
|
+
predicate_text = str(predicate).strip()
|
|
3194
|
+
if not predicate_text:
|
|
3195
|
+
continue
|
|
3196
|
+
field = predicate_text.split(".", 1)[0] if "." in predicate_text else ""
|
|
3197
|
+
filters.append(
|
|
3198
|
+
{
|
|
3199
|
+
"field": field,
|
|
3200
|
+
"operator": "DEFAULT",
|
|
3201
|
+
"value": predicate_text,
|
|
3202
|
+
"source": "schema.concepts",
|
|
3203
|
+
"description": description,
|
|
3204
|
+
}
|
|
3205
|
+
)
|
|
3206
|
+
return filters
|
|
3207
|
+
|
|
3208
|
+
|
|
3209
|
+
def _query_plan_entities(
|
|
3210
|
+
question: str,
|
|
3211
|
+
schema_context: Any,
|
|
3212
|
+
*,
|
|
3213
|
+
searched_entities: dict[str, Any] | None = None,
|
|
3214
|
+
question_entities: list[dict[str, str]] | None = None,
|
|
3215
|
+
analysis: QuestionAnalysis | None = None,
|
|
3216
|
+
) -> list[dict[str, str]]:
|
|
3217
|
+
entities: list[dict[str, str]] = []
|
|
3218
|
+
if isinstance(searched_entities, dict):
|
|
3219
|
+
for candidate in searched_entities.get("schemaCandidates") or []:
|
|
3220
|
+
if not isinstance(candidate, dict):
|
|
3221
|
+
continue
|
|
3222
|
+
entities.append(
|
|
3223
|
+
{
|
|
3224
|
+
"type": str(candidate.get("type") or ""),
|
|
3225
|
+
"label": str(candidate.get("label") or _entity_type_label(str(candidate.get("type") or ""))),
|
|
3226
|
+
"value": str(candidate.get("value") or ""),
|
|
3227
|
+
"source": str(candidate.get("source") or "schema"),
|
|
3228
|
+
}
|
|
3229
|
+
)
|
|
3230
|
+
entities.extend(_query_plan_search_entity_entities(searched_entities))
|
|
3231
|
+
for candidate in _direct_entity_candidates(searched_entities):
|
|
3232
|
+
entities.append(
|
|
3233
|
+
{
|
|
3234
|
+
"type": str(candidate.get("type") or ""),
|
|
3235
|
+
"label": str(candidate.get("label") or _entity_type_label(str(candidate.get("type") or ""))),
|
|
3236
|
+
"value": str(candidate.get("value") or ""),
|
|
3237
|
+
"source": str(candidate.get("source") or "schema.columns"),
|
|
3238
|
+
}
|
|
3239
|
+
)
|
|
3240
|
+
for entity in question_entities or []:
|
|
3241
|
+
entities.append(
|
|
3242
|
+
{
|
|
3243
|
+
"type": entity["type"],
|
|
3244
|
+
"label": entity["label"],
|
|
3245
|
+
"value": entity["value"],
|
|
3246
|
+
"source": "用户问题",
|
|
3247
|
+
}
|
|
3248
|
+
)
|
|
3249
|
+
for enum_filter in _query_plan_enum_filters(question, getattr(schema_context, "enums", []) or []):
|
|
3250
|
+
description = str(enum_filter.get("description") or "")
|
|
3251
|
+
field = str(enum_filter.get("field") or "")
|
|
3252
|
+
value = str(enum_filter.get("value") or "")
|
|
3253
|
+
if description and value:
|
|
3254
|
+
entities.append(
|
|
3255
|
+
{
|
|
3256
|
+
"type": "status",
|
|
3257
|
+
"label": "状态",
|
|
3258
|
+
"value": description,
|
|
3259
|
+
"matchedField": field,
|
|
3260
|
+
"matchedValue": value,
|
|
3261
|
+
"source": "schema.enums",
|
|
3262
|
+
}
|
|
3263
|
+
)
|
|
3264
|
+
return _dedupe_query_plan_entities(
|
|
3265
|
+
entity for entity in entities if not _is_schema_column_noise_entity(entity, analysis=analysis)
|
|
3266
|
+
)
|
|
3267
|
+
|
|
3268
|
+
|
|
3269
|
+
def _query_plan_search_entity_entities(searched_entities: dict[str, Any] | None) -> list[dict[str, str]]:
|
|
3270
|
+
result: list[dict[str, str]] = []
|
|
3271
|
+
for unique_entity in _entity_search_filter_items(searched_entities):
|
|
3272
|
+
entity = {
|
|
3273
|
+
"type": unique_entity["type"],
|
|
3274
|
+
"label": _entity_type_label(unique_entity["type"]),
|
|
3275
|
+
"value": unique_entity["name"],
|
|
3276
|
+
"id": str(unique_entity.get("id") or ""),
|
|
3277
|
+
"source": "search_entities",
|
|
3278
|
+
"matchedName": str(unique_entity.get("matchedName") or ""),
|
|
3279
|
+
}
|
|
3280
|
+
logical_group = unique_entity.get("logicalGroup")
|
|
3281
|
+
if logical_group:
|
|
3282
|
+
entity["logicalGroup"] = logical_group
|
|
3283
|
+
entity["logicalOperator"] = "OR"
|
|
3284
|
+
result.append(entity)
|
|
3285
|
+
return result
|
|
3286
|
+
|
|
3287
|
+
|
|
3288
|
+
def _is_schema_column_noise_entity(
|
|
3289
|
+
entity: dict[str, str],
|
|
3290
|
+
*,
|
|
3291
|
+
analysis: QuestionAnalysis | None = None,
|
|
3292
|
+
) -> bool:
|
|
3293
|
+
if str(entity.get("source") or "") != "schema.columns":
|
|
3294
|
+
return False
|
|
3295
|
+
return _is_analysis_noise_value(entity.get("value"), analysis)
|
|
3296
|
+
|
|
3297
|
+
|
|
3298
|
+
def _dedupe_query_plan_entities(entities: list[dict[str, str]]) -> list[dict[str, str]]:
|
|
3299
|
+
result: list[dict[str, str]] = []
|
|
3300
|
+
seen: set[tuple[str, str, str]] = set()
|
|
3301
|
+
for entity in entities:
|
|
3302
|
+
key = (
|
|
3303
|
+
str(entity.get("type") or ""),
|
|
3304
|
+
str(entity.get("value") or ""),
|
|
3305
|
+
str(entity.get("matchedField") or ""),
|
|
3306
|
+
)
|
|
3307
|
+
if key in seen:
|
|
3308
|
+
continue
|
|
3309
|
+
seen.add(key)
|
|
3310
|
+
result.append(entity)
|
|
3311
|
+
return result
|
|
3312
|
+
|
|
3313
|
+
|
|
3314
|
+
def _query_plan_enum_filters(question: str, enums: list[dict[str, Any]]) -> list[dict[str, str]]:
|
|
3315
|
+
result: list[dict[str, str]] = []
|
|
3316
|
+
for enum in enums:
|
|
3317
|
+
if not isinstance(enum, dict):
|
|
3318
|
+
continue
|
|
3319
|
+
field = str(enum.get("field") or "")
|
|
3320
|
+
values = enum.get("values")
|
|
3321
|
+
if not isinstance(values, list):
|
|
3322
|
+
continue
|
|
3323
|
+
for item in values:
|
|
3324
|
+
if not isinstance(item, dict):
|
|
3325
|
+
continue
|
|
3326
|
+
label = str(item.get("label") or "")
|
|
3327
|
+
value = item.get("value")
|
|
3328
|
+
if not label or value is None:
|
|
3329
|
+
continue
|
|
3330
|
+
if any(term and term in question for term in _split_enum_label_terms(label)):
|
|
3331
|
+
result.append(
|
|
3332
|
+
{
|
|
3333
|
+
"field": field,
|
|
3334
|
+
"operator": "=",
|
|
3335
|
+
"value": str(value),
|
|
3336
|
+
"description": label,
|
|
3337
|
+
"source": "schema.enums",
|
|
3338
|
+
}
|
|
3339
|
+
)
|
|
3340
|
+
return result
|
|
3341
|
+
|
|
3342
|
+
|
|
3343
|
+
def _split_enum_label_terms(label: str) -> set[str]:
|
|
3344
|
+
terms = {label.strip()}
|
|
3345
|
+
terms.update(part.strip() for part in re.split(r"[/、,,;;\s]+", label) if part.strip())
|
|
3346
|
+
return {term for term in terms if term}
|
|
3347
|
+
|
|
3348
|
+
|
|
3349
|
+
def _query_plan_tokens(question: str) -> list[str]:
|
|
3350
|
+
tokens = [question.lower()]
|
|
3351
|
+
tokens.extend(part for part in re.split(r"[\s_\-::,,。;;]+", question.lower()) if part)
|
|
3352
|
+
for run in re.findall(r"[\u4e00-\u9fff]+", question):
|
|
3353
|
+
if len(run) == 1:
|
|
3354
|
+
tokens.append(run)
|
|
3355
|
+
for index in range(len(run) - 1):
|
|
3356
|
+
tokens.append(run[index : index + 2])
|
|
3357
|
+
return list(dict.fromkeys(tokens))
|
|
3358
|
+
|
|
3359
|
+
|
|
3360
|
+
def _prepend_query_plan_summary(
|
|
3361
|
+
answer: str,
|
|
3362
|
+
question: str,
|
|
3363
|
+
result_format: str | None,
|
|
3364
|
+
query_plan: dict[str, Any],
|
|
3365
|
+
) -> str:
|
|
3366
|
+
output_format = normalize_result_format(question, result_format)
|
|
3367
|
+
summary = _identification_summary(query_plan)
|
|
3368
|
+
filters = _query_plan_summary_filters(query_plan)
|
|
3369
|
+
entities = _query_plan_summary_entities(query_plan)
|
|
3370
|
+
if output_format == "json":
|
|
3371
|
+
try:
|
|
3372
|
+
payload = json.loads(answer)
|
|
3373
|
+
except json.JSONDecodeError:
|
|
3374
|
+
return f"{_format_identification_section(query_plan)}\n\n3. 查询结果:\n{answer}"
|
|
3375
|
+
return json.dumps(
|
|
3376
|
+
{
|
|
3377
|
+
"查询实体": entities,
|
|
3378
|
+
"筛选条件": filters,
|
|
3379
|
+
"查询结果": payload,
|
|
3380
|
+
},
|
|
3381
|
+
ensure_ascii=False,
|
|
3382
|
+
indent=2,
|
|
3383
|
+
) + "\n"
|
|
3384
|
+
if output_format == "yaml":
|
|
3385
|
+
return (
|
|
3386
|
+
"查询实体:\n"
|
|
3387
|
+
+ "".join(f" - {json.dumps(item, ensure_ascii=False)}\n" for item in entities)
|
|
3388
|
+
+ "筛选条件:\n"
|
|
3389
|
+
+ "".join(f" - {json.dumps(item, ensure_ascii=False)}\n" for item in filters)
|
|
3390
|
+
+ "查询结果:\n"
|
|
3391
|
+
+ "\n".join(f" {line}" if line else "" for line in answer.rstrip().splitlines())
|
|
3392
|
+
+ "\n"
|
|
3393
|
+
)
|
|
3394
|
+
if output_format == "html":
|
|
3395
|
+
return (
|
|
3396
|
+
"<section><p>1. 查询实体:</p><p>"
|
|
3397
|
+
+ (html_escape(";".join(entities)) or "无")
|
|
3398
|
+
+ "</p></section>"
|
|
3399
|
+
+ "<section><p>2. 筛选条件:</p><p>"
|
|
3400
|
+
+ (html_escape(";".join(filters)) or "无")
|
|
3401
|
+
+ "</p></section>"
|
|
3402
|
+
+ "<section><p>3. 查询结果:</p></section>"
|
|
3403
|
+
+ answer
|
|
3404
|
+
)
|
|
3405
|
+
return f"{summary}\n\n3. 查询结果:\n{answer}"
|
|
3406
|
+
|
|
3407
|
+
|
|
3408
|
+
def ensure_query_plan_summary_answer(
|
|
3409
|
+
answer: str,
|
|
3410
|
+
question: str,
|
|
3411
|
+
result_format: str | None,
|
|
3412
|
+
query_plan: dict[str, Any] | None,
|
|
3413
|
+
) -> str:
|
|
3414
|
+
if not isinstance(query_plan, dict) or not query_plan:
|
|
3415
|
+
return answer
|
|
3416
|
+
if _has_query_plan_summary_answer(answer, question, result_format):
|
|
3417
|
+
return _replace_query_plan_summary(answer, question, result_format, query_plan)
|
|
3418
|
+
return _prepend_query_plan_summary(answer, question, result_format, query_plan)
|
|
3419
|
+
|
|
3420
|
+
|
|
3421
|
+
def _replace_query_plan_summary(
|
|
3422
|
+
answer: str,
|
|
3423
|
+
question: str,
|
|
3424
|
+
result_format: str | None,
|
|
3425
|
+
query_plan: dict[str, Any],
|
|
3426
|
+
) -> str:
|
|
3427
|
+
output_format = normalize_result_format(question, result_format)
|
|
3428
|
+
filters = _query_plan_summary_filters(query_plan)
|
|
3429
|
+
entities = _query_plan_summary_entities(query_plan)
|
|
3430
|
+
if output_format == "json":
|
|
3431
|
+
try:
|
|
3432
|
+
payload = json.loads(answer)
|
|
3433
|
+
except json.JSONDecodeError:
|
|
3434
|
+
return _prepend_query_plan_summary(answer, question, result_format, query_plan)
|
|
3435
|
+
if isinstance(payload, dict):
|
|
3436
|
+
rebuilt: dict[str, Any] = {
|
|
3437
|
+
"查询实体": entities,
|
|
3438
|
+
"筛选条件": filters,
|
|
3439
|
+
"查询结果": payload.get("查询结果", payload),
|
|
3440
|
+
}
|
|
3441
|
+
return json.dumps(rebuilt, ensure_ascii=False, indent=2) + "\n"
|
|
3442
|
+
return _prepend_query_plan_summary(answer, question, result_format, query_plan)
|
|
3443
|
+
if output_format == "yaml":
|
|
3444
|
+
result_body = _yaml_query_result_body(answer)
|
|
3445
|
+
return _prepend_query_plan_summary(result_body, question, result_format, query_plan)
|
|
3446
|
+
if output_format == "html":
|
|
3447
|
+
result_match = re.search(r"(?is)<section><p>3\.\s*查询结果:</p></section>(?P<body>.*)$", answer)
|
|
3448
|
+
result_body = result_match.group("body") if result_match else answer
|
|
3449
|
+
return _prepend_query_plan_summary(result_body, question, result_format, query_plan)
|
|
3450
|
+
result_body = _plain_query_result_body(answer)
|
|
3451
|
+
return _prepend_query_plan_summary(result_body, question, result_format, query_plan)
|
|
3452
|
+
|
|
3453
|
+
|
|
3454
|
+
def _plain_query_result_body(answer: str) -> str:
|
|
3455
|
+
match = re.search(r"(?s)\b3\.\s*查询结果:\s*\n?(?P<body>.*)$", answer)
|
|
3456
|
+
if match is None:
|
|
3457
|
+
return answer
|
|
3458
|
+
return match.group("body").strip()
|
|
3459
|
+
|
|
3460
|
+
|
|
3461
|
+
def _yaml_query_result_body(answer: str) -> str:
|
|
3462
|
+
match = re.search(r"(?ms)^查询结果:\s*\n(?P<body>.*)$", answer)
|
|
3463
|
+
if match is None:
|
|
3464
|
+
return answer
|
|
3465
|
+
lines = []
|
|
3466
|
+
for line in match.group("body").splitlines():
|
|
3467
|
+
lines.append(line[2:] if line.startswith(" ") else line)
|
|
3468
|
+
return "\n".join(lines).strip()
|
|
3469
|
+
|
|
3470
|
+
|
|
3471
|
+
def _has_query_plan_summary_answer(answer: str, question: str, result_format: str | None) -> bool:
|
|
3472
|
+
output_format = normalize_result_format(question, result_format)
|
|
3473
|
+
if output_format == "json":
|
|
3474
|
+
try:
|
|
3475
|
+
payload = json.loads(answer)
|
|
3476
|
+
except json.JSONDecodeError:
|
|
3477
|
+
return False
|
|
3478
|
+
return (
|
|
3479
|
+
isinstance(payload, dict)
|
|
3480
|
+
and "查询实体" in payload
|
|
3481
|
+
and "筛选条件" in payload
|
|
3482
|
+
and "查询结果" in payload
|
|
3483
|
+
)
|
|
3484
|
+
if output_format == "yaml":
|
|
3485
|
+
return "查询实体:" in answer and "筛选条件:" in answer and "查询结果:" in answer
|
|
3486
|
+
return "1. 查询实体" in answer and "2. 筛选条件" in answer and "3. 查询结果" in answer
|
|
3487
|
+
|
|
3488
|
+
|
|
3489
|
+
def _format_identification_section(query_plan: dict[str, Any]) -> str:
|
|
3490
|
+
return _identification_summary(query_plan)
|
|
3491
|
+
|
|
3492
|
+
|
|
3493
|
+
def _identification_summary(query_plan: dict[str, Any]) -> str:
|
|
3494
|
+
entities = _query_plan_summary_entities(query_plan)
|
|
3495
|
+
filters = _query_plan_summary_filters(query_plan)
|
|
3496
|
+
return (
|
|
3497
|
+
"1. 查询实体:\n" + (";".join(entities) if entities else "无")
|
|
3498
|
+
+ "\n\n"
|
|
3499
|
+
+ "2. 筛选条件:\n" + (";".join(filters) if filters else "无")
|
|
3500
|
+
)
|
|
3501
|
+
|
|
3502
|
+
|
|
3503
|
+
def _query_plan_summary_entities(query_plan: dict[str, Any]) -> list[str]:
|
|
3504
|
+
entities = query_plan.get("entities") if isinstance(query_plan.get("entities"), list) else []
|
|
3505
|
+
result: list[str] = []
|
|
3506
|
+
for entity in entities:
|
|
3507
|
+
if not isinstance(entity, dict):
|
|
3508
|
+
continue
|
|
3509
|
+
if _entity_summary_as_filter_label(entity):
|
|
3510
|
+
continue
|
|
3511
|
+
label = str(entity.get("label") or entity.get("type") or "").strip()
|
|
3512
|
+
value = str(entity.get("value") or "").strip()
|
|
3513
|
+
if label in {"实体", "实体类型", "查询对象", "对象"} and value:
|
|
3514
|
+
label = value
|
|
3515
|
+
if label and label not in result:
|
|
3516
|
+
result.append(label)
|
|
3517
|
+
return result
|
|
3518
|
+
|
|
3519
|
+
|
|
3520
|
+
def _query_plan_summary_filters(query_plan: dict[str, Any]) -> list[str]:
|
|
3521
|
+
filters = query_plan.get("filters") if isinstance(query_plan.get("filters"), list) else []
|
|
3522
|
+
result: list[str] = []
|
|
3523
|
+
|
|
3524
|
+
def append_filter(display: str) -> None:
|
|
3525
|
+
if display and display not in result:
|
|
3526
|
+
result.append(display)
|
|
3527
|
+
|
|
3528
|
+
entity_filter_labels: set[str] = set()
|
|
3529
|
+
for display, label in _query_plan_entity_filter_summaries(query_plan):
|
|
3530
|
+
append_filter(display)
|
|
3531
|
+
entity_filter_labels.add(label)
|
|
3532
|
+
|
|
3533
|
+
for item in filters:
|
|
3534
|
+
if not isinstance(item, dict):
|
|
3535
|
+
continue
|
|
3536
|
+
field = str(item.get("field") or "")
|
|
3537
|
+
operator = str(item.get("operator") or "")
|
|
3538
|
+
value = str(item.get("value") or "")
|
|
3539
|
+
label = _query_plan_filter_display_label(field, item)
|
|
3540
|
+
display_value = _query_plan_filter_display_value(value, item)
|
|
3541
|
+
if label in entity_filter_labels and label in _FILTER_SUMMARY_ENTITY_LABELS:
|
|
3542
|
+
continue
|
|
3543
|
+
if label and display_value:
|
|
3544
|
+
append_filter(f"{label}{_query_plan_filter_display_operator(operator)}{display_value}" + _query_plan_filter_logic_label(item))
|
|
3545
|
+
continue
|
|
3546
|
+
if field or value:
|
|
3547
|
+
append_filter(" ".join(part for part in (field, operator, value) if part) + _query_plan_filter_logic_label(item))
|
|
3548
|
+
return result
|
|
3549
|
+
|
|
3550
|
+
|
|
3551
|
+
_FILTER_SUMMARY_ENTITY_LABELS = frozenset({"品牌", "项目", "省份", "城市", "区域", "门店", "供应商", "状态", "岗位状态"})
|
|
3552
|
+
|
|
3553
|
+
|
|
3554
|
+
def _query_plan_entity_filter_summaries(query_plan: dict[str, Any]) -> list[tuple[str, str]]:
|
|
3555
|
+
entities = query_plan.get("entities") if isinstance(query_plan.get("entities"), list) else []
|
|
3556
|
+
result: list[tuple[str, str]] = []
|
|
3557
|
+
for entity in entities:
|
|
3558
|
+
if not isinstance(entity, dict):
|
|
3559
|
+
continue
|
|
3560
|
+
label = _entity_summary_as_filter_label(entity)
|
|
3561
|
+
if not label:
|
|
3562
|
+
continue
|
|
3563
|
+
value = _query_plan_filter_display_value(str(entity.get("value") or ""), entity)
|
|
3564
|
+
if not value:
|
|
3565
|
+
continue
|
|
3566
|
+
result.append((f"{label}={value}" + _query_plan_filter_logic_label(entity), label))
|
|
3567
|
+
return result
|
|
3568
|
+
|
|
3569
|
+
|
|
3570
|
+
def _entity_summary_as_filter_label(entity: dict[str, Any]) -> str | None:
|
|
3571
|
+
value = str(entity.get("value") or "").strip()
|
|
3572
|
+
if not value:
|
|
3573
|
+
return None
|
|
3574
|
+
entity_type = str(entity.get("type") or "")
|
|
3575
|
+
label = str(entity.get("label") or "").strip()
|
|
3576
|
+
if entity_type == "status":
|
|
3577
|
+
field_label = _query_plan_filter_field_label(str(entity.get("matchedField") or ""))
|
|
3578
|
+
return field_label or "状态"
|
|
3579
|
+
normalized = _entity_type_label(entity_type)
|
|
3580
|
+
for candidate in (normalized, label):
|
|
3581
|
+
if candidate in _FILTER_SUMMARY_ENTITY_LABELS:
|
|
3582
|
+
return candidate
|
|
3583
|
+
return None
|
|
3584
|
+
|
|
3585
|
+
|
|
3586
|
+
def _query_plan_filter_display_label(field: str, item: dict[str, Any]) -> str:
|
|
3587
|
+
source = str(item.get("source") or "")
|
|
3588
|
+
if source == "schema.enums":
|
|
3589
|
+
field_label = _query_plan_filter_field_label(field)
|
|
3590
|
+
return field_label if field_label.endswith("状态") else "状态"
|
|
3591
|
+
return _query_plan_filter_field_label(field)
|
|
3592
|
+
|
|
3593
|
+
|
|
3594
|
+
def _query_plan_filter_field_label(field: str) -> str:
|
|
3595
|
+
cleaned = field.strip()
|
|
3596
|
+
lowered = cleaned.lower()
|
|
3597
|
+
if lowered.startswith("brand.") or cleaned in {"品牌", "品牌名称"}:
|
|
3598
|
+
return "品牌"
|
|
3599
|
+
if lowered.startswith("city.") or cleaned in {"城市", "城市名称"}:
|
|
3600
|
+
return "城市"
|
|
3601
|
+
if lowered.startswith("region.") or cleaned in {"区域", "区域名称"}:
|
|
3602
|
+
return "区域"
|
|
3603
|
+
if lowered.startswith("province.") or cleaned in {"省份", "省份名称"}:
|
|
3604
|
+
return "省份"
|
|
3605
|
+
if lowered.startswith(("sponge_project.", "project.")) or cleaned in {"项目", "项目名称"}:
|
|
3606
|
+
return "项目"
|
|
3607
|
+
if lowered.startswith("store.") or cleaned in {"门店", "门店名称"}:
|
|
3608
|
+
return "门店"
|
|
3609
|
+
if "supplier" in lowered or cleaned in {"供应商", "供应商名称"}:
|
|
3610
|
+
return "供应商"
|
|
3611
|
+
if lowered == "job_basic_info.status":
|
|
3612
|
+
return "岗位状态"
|
|
3613
|
+
if lowered.endswith(".status") or cleaned == "状态":
|
|
3614
|
+
return "状态"
|
|
3615
|
+
if cleaned.endswith("名称") and len(cleaned) > 2:
|
|
3616
|
+
return cleaned[: -len("名称")]
|
|
3617
|
+
return cleaned
|
|
3618
|
+
|
|
3619
|
+
|
|
3620
|
+
def _query_plan_filter_display_value(value: str, item: dict[str, Any]) -> str:
|
|
3621
|
+
description = str(item.get("description") or "").strip()
|
|
3622
|
+
if str(item.get("source") or "") == "schema.enums" and description:
|
|
3623
|
+
return description
|
|
3624
|
+
return value.strip().strip("'\"").strip("%").strip()
|
|
3625
|
+
|
|
3626
|
+
|
|
3627
|
+
def _query_plan_filter_display_operator(operator: str) -> str:
|
|
3628
|
+
normalized = operator.upper()
|
|
3629
|
+
if normalized in {"=", "LIKE"}:
|
|
3630
|
+
return "="
|
|
3631
|
+
if normalized:
|
|
3632
|
+
return f" {operator} "
|
|
3633
|
+
return "="
|
|
3634
|
+
|
|
3635
|
+
|
|
3636
|
+
def _query_plan_filter_logic_label(item: dict[str, Any]) -> str:
|
|
3637
|
+
logical_group = str(item.get("logicalGroup") or "")
|
|
3638
|
+
logical_operator = str(item.get("logicalOperator") or "").upper()
|
|
3639
|
+
if logical_group and logical_operator:
|
|
3640
|
+
return f"(同组{logical_operator})"
|
|
3641
|
+
return ""
|
|
3642
|
+
|
|
3643
|
+
|
|
3644
|
+
def html_escape(value: str) -> str:
|
|
3645
|
+
return (
|
|
3646
|
+
value.replace("&", "&")
|
|
3647
|
+
.replace("<", "<")
|
|
3648
|
+
.replace(">", ">")
|
|
3649
|
+
.replace('"', """)
|
|
3650
|
+
)
|
|
3651
|
+
|
|
3652
|
+
|
|
3653
|
+
_NEW_QUESTION_STARTERS = re.compile(
|
|
3654
|
+
r"(?=你得查|你得|请查|请帮|另外|接下来|然后|再查|继续查|现在查|帮我查|还要查|再帮我)"
|
|
3655
|
+
)
|
|
3656
|
+
|
|
3657
|
+
|
|
3658
|
+
def _split_question_clauses(question: str) -> list[str]:
|
|
3659
|
+
parts: list[str] = []
|
|
3660
|
+
for chunk in re.split(r"[。;;\n]+", question):
|
|
3661
|
+
chunk = chunk.strip()
|
|
3662
|
+
if not chunk:
|
|
3663
|
+
continue
|
|
3664
|
+
for part in _NEW_QUESTION_STARTERS.split(chunk):
|
|
3665
|
+
cleaned = part.strip(",, ")
|
|
3666
|
+
if cleaned:
|
|
3667
|
+
parts.append(cleaned)
|
|
3668
|
+
return parts
|
|
3669
|
+
|
|
3670
|
+
|
|
3671
|
+
def _isolate_current_question(question: str) -> str:
|
|
3672
|
+
"""When an orchestrator concatenates prior turns, keep the latest business clause."""
|
|
3673
|
+
normalized = question.strip()
|
|
3674
|
+
if not normalized:
|
|
3675
|
+
return question
|
|
3676
|
+
parts = _split_question_clauses(normalized)
|
|
3677
|
+
if len(parts) <= 1:
|
|
3678
|
+
return question
|
|
3679
|
+
business_parts = [part for part in parts if _is_business_query(part)]
|
|
3680
|
+
if not business_parts:
|
|
3681
|
+
return question
|
|
3682
|
+
if len(business_parts) == 1:
|
|
3683
|
+
return business_parts[0]
|
|
3684
|
+
return business_parts[-1]
|
|
3685
|
+
|
|
3686
|
+
|
|
3687
|
+
def _strip_result_format_instruction(question: str) -> str:
|
|
3688
|
+
cleaned = question
|
|
3689
|
+
patterns = [
|
|
3690
|
+
r"^\s*返回\s*(?:json|yaml|html|markdown|md)\s*格式\s*的?",
|
|
3691
|
+
r"[,,。;;]?\s*输出\s*(?:json|yaml|html|markdown|md)\s*(?:格式)?",
|
|
3692
|
+
r"[,,。;;]?\s*用\s*表格\s*展示",
|
|
3693
|
+
r"[,,。;;]?\s*以\s*(?:json|yaml|html|markdown|md)\s*(?:格式)?\s*输出",
|
|
3694
|
+
r"[,,。;;]?\s*return\s+(?:json|yaml|html|markdown|md)",
|
|
3695
|
+
]
|
|
3696
|
+
for pattern in patterns:
|
|
3697
|
+
cleaned = re.sub(pattern, "", cleaned, flags=re.IGNORECASE)
|
|
3698
|
+
return cleaned.strip() or question
|
|
3699
|
+
|
|
3700
|
+
|
|
3701
|
+
def _preserve_limit_scope_in_normalized_question(normalized_question: str, original_question: str) -> str:
|
|
3702
|
+
"""Keep scope words that the question-analysis LLM may omit while rewriting."""
|
|
3703
|
+
normalized = normalized_question.strip() or original_question
|
|
3704
|
+
if _question_requests_all_rows(original_question) and not _question_requests_all_rows(normalized):
|
|
3705
|
+
return f"{normalized};返回范围:所有数据"
|
|
3706
|
+
original_limit = _requested_limit(original_question)
|
|
3707
|
+
if original_limit is not None and _requested_limit(normalized) is None:
|
|
3708
|
+
return f"{normalized};返回条数:{original_limit}条"
|
|
3709
|
+
return normalized
|
|
3710
|
+
|
|
3711
|
+
|
|
3712
|
+
def _require_non_empty_select(sql: str) -> None:
|
|
3713
|
+
if not sql.strip():
|
|
3714
|
+
raise SqlGenerationError("SQL must not be empty")
|
|
3715
|
+
if not sql.strip().lower().startswith("select "):
|
|
3716
|
+
raise SqlGenerationError("SQL must be SELECT only")
|
|
3717
|
+
|
|
3718
|
+
|
|
3719
|
+
def _parse_execute_unknown_column(error_text: str) -> str | None:
|
|
3720
|
+
match = re.search(r"Unknown column ['`]([^'`]+)['`]", error_text, re.IGNORECASE)
|
|
3721
|
+
if not match:
|
|
3722
|
+
return None
|
|
3723
|
+
return match.group(1).strip()
|
|
3724
|
+
|
|
3725
|
+
|
|
3726
|
+
def _repair_sql_for_local_schema_issues(
|
|
3727
|
+
*,
|
|
3728
|
+
candidate_sql: str,
|
|
3729
|
+
sql_question: str,
|
|
3730
|
+
schema: dict[str, Any],
|
|
3731
|
+
generator: SamplingSqlGenerator | None,
|
|
3732
|
+
schema_context: Any,
|
|
3733
|
+
sql_schema_context: Any,
|
|
3734
|
+
audit_logger: AuditLogger,
|
|
3735
|
+
session_id: str | None,
|
|
3736
|
+
max_repair_attempts: int,
|
|
3737
|
+
default_limit: int,
|
|
3738
|
+
max_limit: int,
|
|
3739
|
+
steps: _ExecutionSteps,
|
|
3740
|
+
repair_success_detail: str = "已修复本地 schema 校验问题。",
|
|
3741
|
+
running_prefix: str = "本地 schema 校验失败,尝试修复",
|
|
3742
|
+
failure_prefix: str = "本地 schema 校验失败,修复失败",
|
|
3743
|
+
validation_error: dict[str, Any] | None = None,
|
|
3744
|
+
) -> tuple[str | None, str | None]:
|
|
3745
|
+
if validation_error is None:
|
|
3746
|
+
schema_issues = validate_sql_against_schema(candidate_sql, schema)
|
|
3747
|
+
if not schema_issues:
|
|
3748
|
+
return candidate_sql, None
|
|
3749
|
+
if generator is None or schema_context is None:
|
|
3750
|
+
return None, f"本地 schema 校验失败:{schema_issues[0].message}"
|
|
3751
|
+
validation_error = build_schema_validation_error(
|
|
3752
|
+
candidate_sql,
|
|
3753
|
+
schema_issues,
|
|
3754
|
+
schema,
|
|
3755
|
+
max_attempts=max_repair_attempts,
|
|
3756
|
+
)
|
|
3757
|
+
elif generator is None or schema_context is None:
|
|
3758
|
+
return None, str(validation_error.get("message") or "本地 schema 校验失败")
|
|
3759
|
+
repaired_sql, fail_reason = _repair_sql_with_llm_retries(
|
|
3760
|
+
generator=generator,
|
|
3761
|
+
question=sql_question,
|
|
3762
|
+
sql=candidate_sql,
|
|
3763
|
+
validation_error=validation_error,
|
|
3764
|
+
schema_context=sql_schema_context,
|
|
3765
|
+
audit_logger=audit_logger,
|
|
3766
|
+
session_id=session_id,
|
|
3767
|
+
max_attempts=max_repair_attempts,
|
|
3768
|
+
default_limit=default_limit,
|
|
3769
|
+
max_limit=max_limit,
|
|
3770
|
+
schema=schema,
|
|
3771
|
+
check=lambda candidate: _require_no_schema_issues(candidate, schema),
|
|
3772
|
+
steps=steps,
|
|
3773
|
+
running_detail=f"{running_prefix}:{validation_error['message']}",
|
|
3774
|
+
failure_detail=failure_prefix,
|
|
3775
|
+
)
|
|
3776
|
+
if repaired_sql is None:
|
|
3777
|
+
return None, fail_reason
|
|
3778
|
+
steps.done(9, repair_success_detail)
|
|
3779
|
+
return repaired_sql, None
|
|
3780
|
+
|
|
3781
|
+
|
|
3782
|
+
def _require_time_range(
|
|
3783
|
+
sql: str,
|
|
3784
|
+
question: str,
|
|
3785
|
+
schema: dict[str, Any],
|
|
3786
|
+
query_plan: dict[str, Any] | None = None,
|
|
3787
|
+
) -> None:
|
|
3788
|
+
if not sql_satisfies_time_range(sql, question, schema, query_plan=query_plan):
|
|
3789
|
+
raise SqlGenerationError("SQL must include the requested time range filter")
|
|
3790
|
+
|
|
3791
|
+
|
|
3792
|
+
def _require_no_schema_issues(sql: str, schema: dict[str, Any]) -> None:
|
|
3793
|
+
issues = validate_sql_against_schema(sql, schema)
|
|
3794
|
+
if issues:
|
|
3795
|
+
raise SqlGenerationError(issues[0].message)
|
|
3796
|
+
|
|
3797
|
+
|
|
3798
|
+
def _require_low_quality_pass(sql: str, question: str, schema: dict[str, Any]) -> None:
|
|
3799
|
+
issue = low_quality_sql_issue_for_question(question, sql, schema=schema)
|
|
3800
|
+
if issue is not None:
|
|
3801
|
+
raise SqlGenerationError(issue.message)
|
|
3802
|
+
|
|
3803
|
+
|
|
3804
|
+
def _log_llm_sql_repair_audit(
|
|
3805
|
+
audit_logger: AuditLogger,
|
|
3806
|
+
*,
|
|
3807
|
+
session_id: str | None,
|
|
3808
|
+
question: str,
|
|
3809
|
+
success: bool,
|
|
3810
|
+
sql: str | None = None,
|
|
3811
|
+
error: str | None = None,
|
|
3812
|
+
attempt: int | None = None,
|
|
3813
|
+
) -> None:
|
|
3814
|
+
audit_logger.log(
|
|
3815
|
+
{
|
|
3816
|
+
"sessionId": session_id,
|
|
3817
|
+
"question": question,
|
|
3818
|
+
"tool": "llm_sql_repair",
|
|
3819
|
+
"success": success,
|
|
3820
|
+
"attempt": attempt,
|
|
3821
|
+
"sqlHash": sql_hash(sql) if sql else None,
|
|
3822
|
+
"error": error,
|
|
3823
|
+
}
|
|
3824
|
+
)
|
|
3825
|
+
|
|
3826
|
+
|
|
3827
|
+
def _repair_sql_with_llm_retries(
|
|
3828
|
+
*,
|
|
3829
|
+
generator: SamplingSqlGenerator,
|
|
3830
|
+
question: str,
|
|
3831
|
+
sql: str,
|
|
3832
|
+
validation_error: dict[str, Any],
|
|
3833
|
+
schema_context: Any,
|
|
3834
|
+
audit_logger: AuditLogger,
|
|
3835
|
+
session_id: str | None,
|
|
3836
|
+
max_attempts: int,
|
|
3837
|
+
default_limit: int,
|
|
3838
|
+
max_limit: int,
|
|
3839
|
+
schema: dict[str, Any] | None,
|
|
3840
|
+
check: Any,
|
|
3841
|
+
steps: _ExecutionSteps,
|
|
3842
|
+
running_detail: str,
|
|
3843
|
+
failure_detail: str,
|
|
3844
|
+
) -> tuple[str | None, str]:
|
|
3845
|
+
current_sql = sql
|
|
3846
|
+
current_error = validation_error
|
|
3847
|
+
last_reason = failure_detail
|
|
3848
|
+
for attempt in range(1, max_attempts + 1):
|
|
3849
|
+
steps.running(9, f"{running_detail}(第 {attempt}/{max_attempts} 次)")
|
|
3850
|
+
repair_error = {
|
|
3851
|
+
**current_error,
|
|
3852
|
+
"attempt": attempt,
|
|
3853
|
+
"maxAttempts": max_attempts,
|
|
3854
|
+
"originalSql": current_sql,
|
|
3855
|
+
}
|
|
3856
|
+
try:
|
|
3857
|
+
repaired = generator.repair(
|
|
3858
|
+
question=question,
|
|
3859
|
+
original_sql=current_sql,
|
|
3860
|
+
validation_error=repair_error,
|
|
3861
|
+
schema_context=schema_context,
|
|
3862
|
+
)
|
|
3863
|
+
except SqlGenerationError as exc:
|
|
3864
|
+
last_reason = f"{failure_detail}:LLM 修复调用失败(第 {attempt}/{max_attempts} 次):{exc}"
|
|
3865
|
+
_log_llm_sql_repair_audit(
|
|
3866
|
+
audit_logger,
|
|
3867
|
+
session_id=session_id,
|
|
3868
|
+
question=question,
|
|
3869
|
+
success=False,
|
|
3870
|
+
error=str(exc),
|
|
3871
|
+
attempt=attempt,
|
|
3872
|
+
)
|
|
3873
|
+
continue
|
|
3874
|
+
repaired_sql = _finalize_candidate_sql(
|
|
3875
|
+
repaired.sql,
|
|
3876
|
+
question,
|
|
3877
|
+
default_limit=default_limit,
|
|
3878
|
+
max_limit=max_limit,
|
|
3879
|
+
schema=schema,
|
|
3880
|
+
)
|
|
3881
|
+
if schema is not None:
|
|
3882
|
+
repaired_sql = apply_time_range_filters_from_question(repaired_sql, question, schema)
|
|
3883
|
+
_log_llm_sql_repair_audit(
|
|
3884
|
+
audit_logger,
|
|
3885
|
+
session_id=session_id,
|
|
3886
|
+
question=question,
|
|
3887
|
+
success=bool(repaired_sql.strip()),
|
|
3888
|
+
sql=repaired_sql if repaired_sql.strip() else None,
|
|
3889
|
+
error=None if repaired_sql.strip() else "empty_sql",
|
|
3890
|
+
attempt=attempt,
|
|
3891
|
+
)
|
|
3892
|
+
if not repaired_sql.strip():
|
|
3893
|
+
last_reason = f"{failure_detail}:LLM 修复返回空 SQL(第 {attempt}/{max_attempts} 次)"
|
|
3894
|
+
current_sql = repaired_sql
|
|
3895
|
+
current_error = {**current_error, "message": last_reason}
|
|
3896
|
+
continue
|
|
3897
|
+
try:
|
|
3898
|
+
check(repaired_sql)
|
|
3899
|
+
steps.done(9, f"已修复 SQL(第 {attempt}/{max_attempts} 次)。")
|
|
3900
|
+
return repaired_sql, ""
|
|
3901
|
+
except SqlGenerationError as exc:
|
|
3902
|
+
if schema is not None:
|
|
3903
|
+
schema_issues = validate_sql_against_schema(repaired_sql, schema)
|
|
3904
|
+
if schema_issues:
|
|
3905
|
+
current_error = build_schema_validation_error(
|
|
3906
|
+
repaired_sql,
|
|
3907
|
+
schema_issues,
|
|
3908
|
+
schema,
|
|
3909
|
+
attempt=attempt,
|
|
3910
|
+
max_attempts=max_attempts,
|
|
3911
|
+
)
|
|
3912
|
+
last_reason = f"{failure_detail}:{current_error['message']}"
|
|
3913
|
+
current_sql = repaired_sql
|
|
3914
|
+
continue
|
|
3915
|
+
last_reason = f"{failure_detail}:{exc}"
|
|
3916
|
+
rule_error = build_sql_rule_validation_error(
|
|
3917
|
+
repaired_sql,
|
|
3918
|
+
exc,
|
|
3919
|
+
max_limit=max_limit,
|
|
3920
|
+
attempt=attempt,
|
|
3921
|
+
max_attempts=max_attempts,
|
|
3922
|
+
)
|
|
3923
|
+
if rule_error.get("violatingColumns") or validation_error.get("code") == "SQL_RULE_VIOLATION":
|
|
3924
|
+
current_error = rule_error
|
|
3925
|
+
else:
|
|
3926
|
+
current_error = {
|
|
3927
|
+
**validation_error,
|
|
3928
|
+
"message": str(exc),
|
|
3929
|
+
"originalSql": repaired_sql,
|
|
3930
|
+
}
|
|
3931
|
+
current_sql = repaired_sql
|
|
3932
|
+
return None, last_reason
|
|
3933
|
+
|
|
3934
|
+
|
|
3935
|
+
def _query_only_result(reason: str | None = None) -> QueryResult:
|
|
3936
|
+
answer = reason.strip() if isinstance(reason, str) and reason.strip() else "查询未能完成,请稍后重试。"
|
|
3937
|
+
return QueryResult(
|
|
3938
|
+
answer=answer,
|
|
3939
|
+
generated_sql="",
|
|
3940
|
+
normalized_sql="",
|
|
3941
|
+
repaired_sql=None,
|
|
3942
|
+
validation={},
|
|
3943
|
+
)
|
|
3944
|
+
|
|
3945
|
+
|
|
3946
|
+
def _has_query_intent(question: str) -> bool:
|
|
3947
|
+
return bool(
|
|
3948
|
+
_is_business_query(question)
|
|
3949
|
+
or re.search(
|
|
3950
|
+
r"(查询|查看|看看|搜索|统计|多少|几|哪些|有哪些|列表|明细|清单|展示|返回|输出|分析|报告|count)",
|
|
3951
|
+
question,
|
|
3952
|
+
flags=re.IGNORECASE,
|
|
3953
|
+
)
|
|
3954
|
+
)
|
|
3955
|
+
|
|
3956
|
+
|
|
3957
|
+
def _non_query_result(reason: str | None = None) -> QueryResult:
|
|
3958
|
+
return QueryResult(
|
|
3959
|
+
answer="目前仅支持查询功能",
|
|
3960
|
+
generated_sql="",
|
|
3961
|
+
normalized_sql="",
|
|
3962
|
+
repaired_sql=None,
|
|
3963
|
+
validation={},
|
|
521
3964
|
)
|
|
522
3965
|
|
|
523
3966
|
|
|
524
|
-
def
|
|
3967
|
+
def _schema_no_match_result(question: str) -> QueryResult:
|
|
525
3968
|
return QueryResult(
|
|
526
|
-
answer=f"
|
|
3969
|
+
answer=f"目前匹配不到 schema 里的任何表,暂不支持该查询。\n\n用户问题:{question}",
|
|
527
3970
|
generated_sql="",
|
|
528
3971
|
normalized_sql="",
|
|
529
3972
|
repaired_sql=None,
|
|
530
3973
|
validation={},
|
|
3974
|
+
needs_clarification=True,
|
|
3975
|
+
clarification_type="schema_no_match",
|
|
3976
|
+
clarification_payload={
|
|
3977
|
+
"type": "schema_no_match",
|
|
3978
|
+
"baseQuestion": question,
|
|
3979
|
+
"instruction": "请直接展示 answer,等待用户改写查询问题;不要替用户选择表或字段。",
|
|
3980
|
+
},
|
|
531
3981
|
)
|
|
532
3982
|
|
|
533
3983
|
|
|
3984
|
+
def _low_quality_sql_result(message: str, generated_sql: str) -> QueryResult:
|
|
3985
|
+
return QueryResult(
|
|
3986
|
+
answer=message,
|
|
3987
|
+
generated_sql=generated_sql,
|
|
3988
|
+
normalized_sql="",
|
|
3989
|
+
repaired_sql=None,
|
|
3990
|
+
validation={},
|
|
3991
|
+
)
|
|
3992
|
+
|
|
3993
|
+
|
|
3994
|
+
def _sensitive_query_only_result(terms: list[str], *, reason: str | None = None) -> QueryResult:
|
|
3995
|
+
notice = _sensitive_field_notice(terms)
|
|
3996
|
+
if isinstance(reason, str) and reason.strip():
|
|
3997
|
+
answer = f"该查询包含敏感字段,{notice}{reason.strip()}"
|
|
3998
|
+
else:
|
|
3999
|
+
answer = f"该查询包含敏感字段,{notice}无法生成去敏后的查询 SQL。"
|
|
4000
|
+
return QueryResult(
|
|
4001
|
+
answer=answer,
|
|
4002
|
+
generated_sql="",
|
|
4003
|
+
normalized_sql="",
|
|
4004
|
+
repaired_sql=None,
|
|
4005
|
+
validation={},
|
|
4006
|
+
)
|
|
4007
|
+
|
|
4008
|
+
|
|
4009
|
+
def _auth_error_result(
|
|
4010
|
+
validation: dict[str, Any] | None,
|
|
4011
|
+
*,
|
|
4012
|
+
generated_sql: str = "",
|
|
4013
|
+
repaired_sql: str | None = None,
|
|
4014
|
+
) -> QueryResult:
|
|
4015
|
+
error = validation.get("error") if isinstance(validation, dict) else None
|
|
4016
|
+
code = error.get("code") if isinstance(error, dict) else None
|
|
4017
|
+
suffix = f"(错误码:{code})" if code else ""
|
|
4018
|
+
return QueryResult(
|
|
4019
|
+
answer=f"当前 MCP 用户认证失败,无法校验和执行查询。请检查 MCP 访问 token 或重新登录后再试。{suffix}",
|
|
4020
|
+
generated_sql=generated_sql,
|
|
4021
|
+
normalized_sql="",
|
|
4022
|
+
repaired_sql=repaired_sql,
|
|
4023
|
+
validation=validation or {},
|
|
4024
|
+
)
|
|
4025
|
+
|
|
4026
|
+
|
|
4027
|
+
def _is_auth_validation_error(code: str | None) -> bool:
|
|
4028
|
+
return code in {"MCP_AUTH_INVALID_TOKEN", "MCP_AUTH_REQUIRED", "MCP_AUTH_FORBIDDEN"}
|
|
4029
|
+
|
|
4030
|
+
|
|
534
4031
|
def _business_metric_clarification_result(answer: str) -> QueryResult:
|
|
535
4032
|
return QueryResult(
|
|
536
4033
|
answer=answer,
|
|
@@ -541,6 +4038,21 @@ def _business_metric_clarification_result(answer: str) -> QueryResult:
|
|
|
541
4038
|
)
|
|
542
4039
|
|
|
543
4040
|
|
|
4041
|
+
def _empty_scope_result(question: str, result_format: str | None) -> QueryResult:
|
|
4042
|
+
# Reuses the standard empty-result rendering so the requested output format is
|
|
4043
|
+
# respected. Only used as legacy compatibility for older online MCP versions
|
|
4044
|
+
# (see _LEGACY_EMPTY_SCOPE_CODES); the current contract returns this shape
|
|
4045
|
+
# directly as a normal success result.
|
|
4046
|
+
answer = render_result(question, {"success": True, "rowCount": 0, "rows": []}, result_format)
|
|
4047
|
+
return QueryResult(
|
|
4048
|
+
answer=answer,
|
|
4049
|
+
generated_sql="",
|
|
4050
|
+
normalized_sql="",
|
|
4051
|
+
repaired_sql=None,
|
|
4052
|
+
validation={},
|
|
4053
|
+
)
|
|
4054
|
+
|
|
4055
|
+
|
|
544
4056
|
def _sensitive_field_notice(terms: list[str]) -> str:
|
|
545
4057
|
if not terms:
|
|
546
4058
|
return "已排除敏感字段。"
|
|
@@ -558,225 +4070,318 @@ def _generate_with_sampler(
|
|
|
558
4070
|
generator: SamplingSqlGenerator,
|
|
559
4071
|
question: str,
|
|
560
4072
|
schema_context: Any,
|
|
561
|
-
|
|
4073
|
+
audit_logger: AuditLogger | None = None,
|
|
4074
|
+
session_id: str | None = None,
|
|
4075
|
+
max_repair_attempts: int = 2,
|
|
4076
|
+
schema: dict[str, Any] | None = None,
|
|
4077
|
+
) -> tuple[GeneratedSql, str | None, bool]:
|
|
4078
|
+
def _audit(tool: str, **fields: Any) -> None:
|
|
4079
|
+
if audit_logger is None:
|
|
4080
|
+
return
|
|
4081
|
+
audit_logger.log(
|
|
4082
|
+
{
|
|
4083
|
+
"sessionId": session_id,
|
|
4084
|
+
"question": question,
|
|
4085
|
+
"tool": tool,
|
|
4086
|
+
**fields,
|
|
4087
|
+
}
|
|
4088
|
+
)
|
|
4089
|
+
|
|
562
4090
|
try:
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
schema_context=schema_context,
|
|
567
|
-
),
|
|
568
|
-
None,
|
|
4091
|
+
generated = generator.generate(
|
|
4092
|
+
question=question,
|
|
4093
|
+
schema_context=schema_context,
|
|
569
4094
|
)
|
|
570
4095
|
except SqlGenerationError as sampling_exc:
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
4096
|
+
_audit("llm_sql_generate", success=False, error=str(sampling_exc))
|
|
4097
|
+
validation_error = {"code": type(sampling_exc).__name__, "message": str(sampling_exc), "originalSql": ""}
|
|
4098
|
+
for attempt in range(1, max_repair_attempts + 1):
|
|
4099
|
+
try:
|
|
4100
|
+
repaired = generator.repair(
|
|
4101
|
+
question=question,
|
|
4102
|
+
original_sql="",
|
|
4103
|
+
validation_error={
|
|
4104
|
+
**validation_error,
|
|
4105
|
+
"attempt": attempt,
|
|
4106
|
+
"maxAttempts": max_repair_attempts,
|
|
4107
|
+
},
|
|
4108
|
+
schema_context=schema_context,
|
|
4109
|
+
)
|
|
4110
|
+
except SqlGenerationError as repair_exc:
|
|
4111
|
+
_audit("llm_sql_repair", success=False, error=str(repair_exc), attempt=attempt)
|
|
4112
|
+
continue
|
|
4113
|
+
repaired_sql = _finalize_candidate_sql(
|
|
583
4114
|
repaired.sql,
|
|
584
4115
|
question,
|
|
585
4116
|
default_limit=generator.default_limit,
|
|
586
4117
|
max_limit=generator.max_limit,
|
|
587
|
-
|
|
588
|
-
|
|
4118
|
+
schema=schema,
|
|
4119
|
+
)
|
|
4120
|
+
_audit(
|
|
4121
|
+
"llm_sql_repair",
|
|
4122
|
+
success=bool(repaired_sql.strip()),
|
|
4123
|
+
sqlHash=sql_hash(repaired_sql) if repaired_sql.strip() else None,
|
|
4124
|
+
error=None if repaired_sql.strip() else "empty_sql",
|
|
4125
|
+
attempt=attempt,
|
|
4126
|
+
)
|
|
4127
|
+
if repaired_sql.strip():
|
|
4128
|
+
return (
|
|
4129
|
+
GeneratedSql(sql="", tables=[], placeholders=[]),
|
|
4130
|
+
repaired_sql,
|
|
4131
|
+
True,
|
|
4132
|
+
)
|
|
4133
|
+
return GeneratedSql(sql="", tables=[], placeholders=[]), None, True
|
|
4134
|
+
if generated.sql.strip():
|
|
4135
|
+
_audit("llm_sql_generate", success=True, sqlHash=sql_hash(generated.sql))
|
|
4136
|
+
else:
|
|
4137
|
+
_audit("llm_sql_generate", success=False, error="empty_sql")
|
|
4138
|
+
return generated, None, False
|
|
589
4139
|
|
|
590
4140
|
|
|
591
|
-
def
|
|
4141
|
+
def _generate_from_schema_example(
|
|
592
4142
|
*,
|
|
593
4143
|
question: str,
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
4144
|
+
schema_context: Any,
|
|
4145
|
+
) -> GeneratedSql | None:
|
|
4146
|
+
examples = _business_examples(getattr(schema_context, "examples", []) or [])
|
|
4147
|
+
best: tuple[int, dict[str, Any]] | None = None
|
|
4148
|
+
for example in examples:
|
|
4149
|
+
if not isinstance(example, dict) or not isinstance(example.get("sql"), str):
|
|
4150
|
+
continue
|
|
4151
|
+
if not _is_high_match_example(question, str(example.get("question") or "")):
|
|
4152
|
+
continue
|
|
4153
|
+
score = _example_match_score(question, str(example.get("question") or ""))
|
|
4154
|
+
if best is None or score > best[0]:
|
|
4155
|
+
best = (score, example)
|
|
4156
|
+
if best is None:
|
|
4157
|
+
return None
|
|
4158
|
+
example = best[1]
|
|
4159
|
+
tables = example.get("tables") if isinstance(example.get("tables"), list) else []
|
|
4160
|
+
return GeneratedSql(
|
|
4161
|
+
sql=str(example["sql"]).strip(),
|
|
4162
|
+
tables=[str(table) for table in tables],
|
|
4163
|
+
placeholders=[],
|
|
605
4164
|
)
|
|
606
4165
|
|
|
607
4166
|
|
|
608
|
-
def
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
max_limit=max_limit,
|
|
622
|
-
)
|
|
4167
|
+
def _is_high_match_example(question: str, example_question: str) -> bool:
|
|
4168
|
+
if not question or not example_question:
|
|
4169
|
+
return False
|
|
4170
|
+
question_compact = _compact_match_text(question)
|
|
4171
|
+
example_compact = _compact_match_text(example_question)
|
|
4172
|
+
if len(example_compact) >= 12 and (example_compact in question_compact or question_compact in example_compact):
|
|
4173
|
+
return True
|
|
4174
|
+
question_tokens = set(_match_tokens(question))
|
|
4175
|
+
example_tokens = set(_match_tokens(example_question))
|
|
4176
|
+
if not question_tokens or not example_tokens:
|
|
4177
|
+
return False
|
|
4178
|
+
overlap = len(question_tokens & example_tokens)
|
|
4179
|
+
return overlap >= 8 and overlap / min(len(question_tokens), len(example_tokens)) >= 0.65
|
|
623
4180
|
|
|
624
4181
|
|
|
625
|
-
def
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
return
|
|
4182
|
+
def _example_match_score(question: str, example_question: str) -> int:
|
|
4183
|
+
question_tokens = set(_match_tokens(question))
|
|
4184
|
+
example_tokens = set(_match_tokens(example_question))
|
|
4185
|
+
return len(question_tokens & example_tokens)
|
|
629
4186
|
|
|
630
4187
|
|
|
631
|
-
def
|
|
632
|
-
|
|
633
|
-
r"(新增|新建|创建|添加|插入|录入|修改|更新|编辑|变更|调整|删除|移除|作废|发布|下架|保存|提交|审批|导入|同步|insert|update|delete|create|drop|alter|truncate)",
|
|
634
|
-
question,
|
|
635
|
-
flags=re.IGNORECASE,
|
|
636
|
-
)
|
|
637
|
-
if mutation_match is None:
|
|
638
|
-
return False
|
|
639
|
-
if _is_mutation_analytics_request(question):
|
|
640
|
-
return False
|
|
641
|
-
return True
|
|
4188
|
+
def _compact_match_text(text: str) -> str:
|
|
4189
|
+
return "".join(re.findall(r"[A-Za-z0-9\u4e00-\u9fff]+", text.lower()))
|
|
642
4190
|
|
|
643
4191
|
|
|
644
|
-
def
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
4192
|
+
def _match_tokens(text: str) -> list[str]:
|
|
4193
|
+
compact = _compact_match_text(text)
|
|
4194
|
+
tokens = re.findall(r"[a-z0-9]+", compact)
|
|
4195
|
+
cjk = "".join(re.findall(r"[\u4e00-\u9fff]+", compact))
|
|
4196
|
+
for index in range(max(0, len(cjk) - 1)):
|
|
4197
|
+
tokens.append(cjk[index : index + 2])
|
|
4198
|
+
return list(dict.fromkeys(tokens))
|
|
650
4199
|
|
|
651
4200
|
|
|
652
4201
|
def _business_metric_clarification(question: str, schema: dict[str, Any]) -> str | None:
|
|
653
|
-
|
|
654
|
-
return None
|
|
655
|
-
metric_terms = _ambiguous_business_metric_terms(question)
|
|
656
|
-
if not metric_terms:
|
|
657
|
-
return None
|
|
658
|
-
if _schema_has_metric_definition(question, schema, metric_terms):
|
|
659
|
-
return None
|
|
660
|
-
|
|
661
|
-
lines = [
|
|
662
|
-
"这个查询涉及业务口径,我暂时不直接生成 SQL,避免结果口径不准。",
|
|
663
|
-
"",
|
|
664
|
-
"请先确认:",
|
|
665
|
-
f"1. 指标定义:“{'、'.join(metric_terms)}”具体包含哪些数据?",
|
|
666
|
-
"2. 时间口径:按哪个时间字段统计?",
|
|
667
|
-
"3. 状态口径:哪些状态算有效/成功/完成/异常?",
|
|
668
|
-
"4. 聚合维度:需要按什么维度展示?",
|
|
669
|
-
]
|
|
670
|
-
return "\n".join(lines)
|
|
4202
|
+
return None
|
|
671
4203
|
|
|
672
4204
|
|
|
673
4205
|
def _unsupported_requested_field_message(question: str, schema: dict[str, Any]) -> str | None:
|
|
674
|
-
if "供应商" not in question:
|
|
675
|
-
return None
|
|
676
|
-
if not re.search(r"(报名|工单|候选人|人员|清单|明细)", question):
|
|
677
|
-
return None
|
|
678
|
-
if not _schema_has_table(schema, "supplier") or not _schema_has_column(schema, "supplier", "nick_name"):
|
|
679
|
-
return "当前 schema 不支持“供应商”字段的查询,可联系管理员补充。"
|
|
680
|
-
if not _schema_supports_work_order_supplier_relation(schema):
|
|
681
|
-
return "当前 schema 缺少“供应商”的关联关系,可联系管理员补充。"
|
|
682
4206
|
return None
|
|
683
4207
|
|
|
684
4208
|
|
|
685
|
-
def
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
)
|
|
693
|
-
|
|
4209
|
+
def _required_schema_concepts_gap_message(analysis: QuestionAnalysis, schema: dict[str, Any]) -> str | None:
|
|
4210
|
+
missing_labels: list[str] = []
|
|
4211
|
+
for required in analysis.required_concepts:
|
|
4212
|
+
if not isinstance(required, dict):
|
|
4213
|
+
continue
|
|
4214
|
+
if _required_concept_is_defined(required, schema):
|
|
4215
|
+
continue
|
|
4216
|
+
label = str(required.get("label") or required.get("name") or "").strip()
|
|
4217
|
+
if label:
|
|
4218
|
+
missing_labels.append(label)
|
|
4219
|
+
if not missing_labels:
|
|
4220
|
+
return None
|
|
4221
|
+
if len(missing_labels) == 1:
|
|
4222
|
+
return f"schema 缺少{missing_labels[0]}口径"
|
|
4223
|
+
return "schema 缺少以下口径:" + "、".join(missing_labels)
|
|
694
4224
|
|
|
695
4225
|
|
|
696
|
-
def
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
)
|
|
4226
|
+
def _required_concept_search_terms(required: dict[str, Any]) -> list[str]:
|
|
4227
|
+
terms: list[str] = []
|
|
4228
|
+
primary = str(required.get("name") or "").strip()
|
|
4229
|
+
if primary:
|
|
4230
|
+
terms.append(primary)
|
|
4231
|
+
for name in required.get("fallbackNames") or []:
|
|
4232
|
+
value = str(name or "").strip()
|
|
4233
|
+
if value and value not in terms:
|
|
4234
|
+
terms.append(value)
|
|
4235
|
+
label = str(required.get("label") or "").strip()
|
|
4236
|
+
if label and label not in terms:
|
|
4237
|
+
terms.append(label)
|
|
4238
|
+
return terms
|
|
702
4239
|
|
|
703
4240
|
|
|
704
|
-
def
|
|
705
|
-
|
|
706
|
-
if not
|
|
707
|
-
return
|
|
708
|
-
|
|
709
|
-
|
|
4241
|
+
def _required_concept_is_defined(required: dict[str, Any], schema: dict[str, Any]) -> bool:
|
|
4242
|
+
terms = _required_concept_search_terms(required)
|
|
4243
|
+
if not terms:
|
|
4244
|
+
return True
|
|
4245
|
+
business_items = _schema_concepts(schema) + _schema_metrics(schema)
|
|
4246
|
+
for item in business_items:
|
|
4247
|
+
if _business_definition_matches_required_terms(item, terms):
|
|
4248
|
+
return True
|
|
4249
|
+
glossary_targets = _schema_glossary_maps_to_targets(schema, terms)
|
|
4250
|
+
for item in business_items:
|
|
4251
|
+
identifiers = _business_definition_identifiers(item)
|
|
4252
|
+
if identifiers & glossary_targets:
|
|
4253
|
+
return True
|
|
4254
|
+
return False
|
|
4255
|
+
|
|
4256
|
+
|
|
4257
|
+
def _business_definition_matches_required_terms(item: dict[str, Any], terms: list[str]) -> bool:
|
|
4258
|
+
identifiers = _business_definition_identifiers(item)
|
|
4259
|
+
if any(term in identifiers for term in terms):
|
|
4260
|
+
return True
|
|
4261
|
+
for term in terms:
|
|
4262
|
+
if not term:
|
|
710
4263
|
continue
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
4264
|
+
for key in ("comment", "description"):
|
|
4265
|
+
body = str(item.get(key) or "")
|
|
4266
|
+
if term in body:
|
|
4267
|
+
return True
|
|
4268
|
+
for example in item.get("positiveExamples") or []:
|
|
4269
|
+
example_text = str(example or "").strip()
|
|
4270
|
+
if term == example_text or term in example_text:
|
|
4271
|
+
return True
|
|
719
4272
|
return False
|
|
720
4273
|
|
|
721
4274
|
|
|
722
|
-
def
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
4275
|
+
def _business_definition_identifiers(item: dict[str, Any]) -> set[str]:
|
|
4276
|
+
identifiers: set[str] = set()
|
|
4277
|
+
for key in ("name", "id", "term", "metric", "baseTable", "comment"):
|
|
4278
|
+
value = str(item.get(key) or "").strip()
|
|
4279
|
+
if value:
|
|
4280
|
+
identifiers.add(value)
|
|
4281
|
+
for key in ("requiredTables", "positiveExamples", "filters"):
|
|
4282
|
+
values = item.get(key)
|
|
4283
|
+
if not isinstance(values, list):
|
|
4284
|
+
continue
|
|
4285
|
+
for entry in values:
|
|
4286
|
+
value = str(entry or "").strip()
|
|
4287
|
+
if value:
|
|
4288
|
+
identifiers.add(value)
|
|
4289
|
+
return identifiers
|
|
4290
|
+
|
|
4291
|
+
|
|
4292
|
+
def _schema_glossary_maps_to_targets(schema: dict[str, Any], terms: list[str]) -> set[str]:
|
|
4293
|
+
targets: set[str] = set()
|
|
4294
|
+
for item in _schema_glossary(schema):
|
|
4295
|
+
if not isinstance(item, dict):
|
|
4296
|
+
continue
|
|
4297
|
+
maps_to = str(item.get("mapsTo") or "").strip()
|
|
4298
|
+
if not maps_to:
|
|
728
4299
|
continue
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
str(
|
|
4300
|
+
searchable = {
|
|
4301
|
+
maps_to,
|
|
4302
|
+
str(item.get("term") or "").strip(),
|
|
4303
|
+
str(item.get("description") or "").strip(),
|
|
732
4304
|
}
|
|
733
|
-
if
|
|
734
|
-
|
|
735
|
-
return
|
|
4305
|
+
if any(term in searchable or term in str(item.get("description") or "") for term in terms if term):
|
|
4306
|
+
targets.add(maps_to)
|
|
4307
|
+
return targets
|
|
736
4308
|
|
|
737
4309
|
|
|
738
|
-
def
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
if
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
4310
|
+
def _schema_glossary(schema: dict[str, Any]) -> list[dict[str, Any]]:
|
|
4311
|
+
glossary = schema.get("glossary")
|
|
4312
|
+
if isinstance(glossary, list):
|
|
4313
|
+
return [item for item in glossary if isinstance(item, dict)]
|
|
4314
|
+
return []
|
|
4315
|
+
|
|
4316
|
+
|
|
4317
|
+
def _schema_concepts(schema: dict[str, Any]) -> list[dict[str, Any]]:
|
|
4318
|
+
concepts = _schema_structure(schema).get("concepts")
|
|
4319
|
+
if isinstance(concepts, list):
|
|
4320
|
+
return [concept for concept in concepts if isinstance(concept, dict)]
|
|
4321
|
+
legacy_concepts = schema.get("concepts")
|
|
4322
|
+
if isinstance(legacy_concepts, list):
|
|
4323
|
+
return [concept for concept in legacy_concepts if isinstance(concept, dict)]
|
|
4324
|
+
return []
|
|
4325
|
+
|
|
4326
|
+
|
|
4327
|
+
def _schema_structure(schema: dict[str, Any]) -> dict[str, Any]:
|
|
4328
|
+
structure = schema.get("structure")
|
|
4329
|
+
return structure if isinstance(structure, dict) else {}
|
|
4330
|
+
|
|
4331
|
+
|
|
4332
|
+
def _schema_relations(schema: dict[str, Any]) -> list[dict[str, Any]]:
|
|
4333
|
+
relations = _schema_structure(schema).get("relations")
|
|
4334
|
+
if isinstance(relations, list):
|
|
4335
|
+
return [relation for relation in relations if isinstance(relation, dict)]
|
|
4336
|
+
joins = schema.get("joins")
|
|
4337
|
+
if isinstance(joins, list):
|
|
4338
|
+
return [join for join in joins if isinstance(join, dict)]
|
|
4339
|
+
return []
|
|
4340
|
+
|
|
4341
|
+
|
|
4342
|
+
def _schema_metrics(schema: dict[str, Any]) -> list[dict[str, Any]]:
|
|
4343
|
+
metrics = _schema_structure(schema).get("metrics")
|
|
4344
|
+
if isinstance(metrics, list):
|
|
4345
|
+
return [metric for metric in metrics if isinstance(metric, dict)]
|
|
4346
|
+
legacy_metrics = schema.get("metrics")
|
|
4347
|
+
if isinstance(legacy_metrics, list):
|
|
4348
|
+
return [metric for metric in legacy_metrics if isinstance(metric, dict)]
|
|
4349
|
+
return []
|
|
4350
|
+
|
|
4351
|
+
|
|
4352
|
+
def _relation_left(relation: dict[str, Any]) -> tuple[str, str]:
|
|
4353
|
+
table = str(relation.get("leftTable") or "")
|
|
4354
|
+
column = str(relation.get("leftColumn") or relation.get("fromColumn") or "")
|
|
4355
|
+
if table:
|
|
4356
|
+
return table, column
|
|
4357
|
+
return _split_field_ref(str(relation.get("from") or ""))
|
|
4358
|
+
|
|
4359
|
+
|
|
4360
|
+
def _relation_right(relation: dict[str, Any]) -> tuple[str, str]:
|
|
4361
|
+
table = str(relation.get("rightTable") or "")
|
|
4362
|
+
column = str(relation.get("rightColumn") or relation.get("toColumn") or "")
|
|
4363
|
+
if table:
|
|
4364
|
+
return table, column
|
|
4365
|
+
return _split_field_ref(str(relation.get("to") or ""))
|
|
778
4366
|
|
|
779
|
-
|
|
4367
|
+
|
|
4368
|
+
def _split_field_ref(value: str) -> tuple[str, str]:
|
|
4369
|
+
if "." not in value:
|
|
4370
|
+
return value, ""
|
|
4371
|
+
table, column = value.split(".", 1)
|
|
4372
|
+
return table, column
|
|
4373
|
+
|
|
4374
|
+
|
|
4375
|
+
def _finalize_candidate_sql(
|
|
4376
|
+
sql: str,
|
|
4377
|
+
question: str,
|
|
4378
|
+
*,
|
|
4379
|
+
default_limit: int,
|
|
4380
|
+
max_limit: int,
|
|
4381
|
+
schema: dict[str, Any] | None = None,
|
|
4382
|
+
) -> str:
|
|
4383
|
+
normalized = finalize_generated_sql(sql, question, schema=schema)
|
|
4384
|
+
return _apply_limit_policy(normalized, question, default_limit=default_limit, max_limit=max_limit)
|
|
780
4385
|
|
|
781
4386
|
|
|
782
4387
|
def _apply_limit_policy(sql: str, question: str, *, default_limit: int, max_limit: int) -> str:
|
|
@@ -795,7 +4400,10 @@ def _apply_default_limit(sql: str, question: str, default_limit: int) -> str:
|
|
|
795
4400
|
|
|
796
4401
|
|
|
797
4402
|
def _replace_limit(sql: str, limit: int) -> str:
|
|
798
|
-
|
|
4403
|
+
stripped = sql.strip().rstrip(";")
|
|
4404
|
+
if re.search(r"\blimit\s+\d+\b", stripped, flags=re.IGNORECASE):
|
|
4405
|
+
return re.sub(r"\blimit\s+\d+\b", f"LIMIT {limit}", stripped, count=1, flags=re.IGNORECASE)
|
|
4406
|
+
return f"{stripped} LIMIT {limit}"
|
|
799
4407
|
|
|
800
4408
|
|
|
801
4409
|
def _question_requests_limit(question: str) -> bool:
|
|
@@ -823,8 +4431,8 @@ def _question_requests_all_rows(question: str) -> bool:
|
|
|
823
4431
|
lowered = question.lower()
|
|
824
4432
|
return bool(
|
|
825
4433
|
re.search(r"(所有|全部|全量|完整)", question)
|
|
826
|
-
or re.search(r"(导出|生成|下载).{0,12}(excel|xlsx|表格|文件)", lowered)
|
|
827
|
-
or re.search(r"(excel|xlsx).{0,12}(导出|生成|下载)", lowered)
|
|
4434
|
+
or re.search(r"(导出|生成|下载).{0,12}(excel|xlsx|csv|表格|文件)", lowered)
|
|
4435
|
+
or re.search(r"(excel|xlsx|csv).{0,12}(导出|生成|下载)", lowered)
|
|
828
4436
|
)
|
|
829
4437
|
|
|
830
4438
|
|
|
@@ -832,6 +4440,28 @@ def _default_limit_notice(default_limit: int) -> str:
|
|
|
832
4440
|
return f"本次为默认查询 {default_limit} 条数据;如果需要查询更多数据,可以修改查询口径。"
|
|
833
4441
|
|
|
834
4442
|
|
|
4443
|
+
def _csv_export_notice(question: str, result: dict[str, Any]) -> str | None:
|
|
4444
|
+
if not _question_requests_all_rows(question):
|
|
4445
|
+
return None
|
|
4446
|
+
if not result.get("success", True):
|
|
4447
|
+
return None
|
|
4448
|
+
export = export_result_to_csv(result)
|
|
4449
|
+
if export is None:
|
|
4450
|
+
return None
|
|
4451
|
+
try:
|
|
4452
|
+
row_count = int(result.get("rowCount", export.row_count))
|
|
4453
|
+
except (TypeError, ValueError):
|
|
4454
|
+
row_count = export.row_count
|
|
4455
|
+
if row_count > export.row_count:
|
|
4456
|
+
count_text = f"查询匹配 {row_count} 条,本次 CSV 写入当前返回的 {export.row_count} 条。"
|
|
4457
|
+
else:
|
|
4458
|
+
count_text = f"查询返回 {export.row_count} 条数据。"
|
|
4459
|
+
return (
|
|
4460
|
+
f"{count_text}因为页面展示大量数据不太友好,已生成可用 Excel 打开的 CSV 文件:"
|
|
4461
|
+
f"{export.path}"
|
|
4462
|
+
)
|
|
4463
|
+
|
|
4464
|
+
|
|
835
4465
|
def _should_append_default_limit_notice(*, question: str, result: dict[str, Any], default_limit: int) -> bool:
|
|
836
4466
|
if _question_requests_limit(question):
|
|
837
4467
|
return False
|