@roll-agent/octopus-agent 0.0.11 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +18 -1
- package/SKILL.md +76 -35
- package/package.json +1 -1
- package/pyproject.toml +1 -1
- package/references/env.yaml +4 -0
- package/src/octopus_skill/_version.py +1 -1
- package/src/octopus_skill/agent.py +3787 -1110
- package/src/octopus_skill/config.py +22 -25
- package/src/octopus_skill/exporter.py +66 -0
- package/src/octopus_skill/llm_result_renderer.py +101 -0
- package/src/octopus_skill/llm_sql_generator.py +17 -6
- package/src/octopus_skill/mcp_client.py +4 -4
- package/src/octopus_skill/octopus_run.py +281 -295
- package/src/octopus_skill/prompt_builder.py +386 -80
- package/src/octopus_skill/question_analyzer.py +388 -0
- package/src/octopus_skill/result_renderer.py +38 -14
- package/src/octopus_skill/schema_context_retriever.py +759 -35
- package/src/octopus_skill/sql_generator.py +1956 -725
- package/src/octopus_skill/entity_binding.py +0 -523
- package/src/octopus_skill/entity_dictionary.py +0 -59
- package/src/octopus_skill/entity_resolver.py +0 -446
|
@@ -5,6 +5,8 @@ import re
|
|
|
5
5
|
from dataclasses import dataclass
|
|
6
6
|
from typing import Any
|
|
7
7
|
|
|
8
|
+
from .schema_context_retriever import _schema_enums as merged_schema_enums
|
|
9
|
+
|
|
8
10
|
|
|
9
11
|
class SqlGenerationError(RuntimeError):
|
|
10
12
|
pass
|
|
@@ -18,6 +20,7 @@ class GeneratedSql:
|
|
|
18
20
|
used_columns: list[str] | None = None
|
|
19
21
|
used_joins: list[str] | None = None
|
|
20
22
|
metric_refs: list[str] | None = None
|
|
23
|
+
clarification: dict[str, Any] | None = None
|
|
21
24
|
|
|
22
25
|
|
|
23
26
|
@dataclass(frozen=True)
|
|
@@ -26,12 +29,14 @@ class TimeRange:
|
|
|
26
29
|
alias: str
|
|
27
30
|
start: str
|
|
28
31
|
end: str
|
|
32
|
+
table_name: str = ""
|
|
29
33
|
|
|
30
34
|
|
|
31
35
|
@dataclass(frozen=True)
|
|
32
36
|
class FieldRef:
|
|
33
37
|
name: str
|
|
34
38
|
alias: str
|
|
39
|
+
table_name: str = ""
|
|
35
40
|
|
|
36
41
|
|
|
37
42
|
@dataclass(frozen=True)
|
|
@@ -46,13 +51,32 @@ class LowQualitySqlIssue:
|
|
|
46
51
|
message: str
|
|
47
52
|
|
|
48
53
|
|
|
54
|
+
_SELECT_AS_ALIAS_PATTERN = (
|
|
55
|
+
r"\s+as\s+(`[^`]+`|\"[^\"]+\"|'[^']+'|\w+)\s*$"
|
|
56
|
+
)
|
|
57
|
+
|
|
58
|
+
_FORBIDDEN_DML_KEYWORDS = re.compile(
|
|
59
|
+
r"\b(insert|update|delete|drop|alter|create|truncate)\b",
|
|
60
|
+
flags=re.IGNORECASE,
|
|
61
|
+
)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _sql_without_string_literals(sql: str) -> str:
|
|
65
|
+
return re.sub(r"'([^']|'')*'", "''", sql)
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def contains_forbidden_dml_keyword(sql: str) -> bool:
|
|
69
|
+
stripped = _sql_without_string_literals(sql.strip())
|
|
70
|
+
return _FORBIDDEN_DML_KEYWORDS.search(stripped) is not None
|
|
71
|
+
|
|
72
|
+
|
|
49
73
|
def enforce_sql_rules(sql: str, *, max_limit: int) -> None:
|
|
50
74
|
normalized = sql.strip().lower()
|
|
51
75
|
if not normalized.startswith("select "):
|
|
52
76
|
raise SqlGenerationError("SQL must be SELECT only")
|
|
53
77
|
if ";" in normalized.rstrip(";"):
|
|
54
78
|
raise SqlGenerationError("SQL must be a single statement")
|
|
55
|
-
if
|
|
79
|
+
if contains_forbidden_dml_keyword(sql):
|
|
56
80
|
raise SqlGenerationError("SQL contains a forbidden keyword")
|
|
57
81
|
if re.search(r"\b(password|token|secret|credential|private_key)\b", normalized):
|
|
58
82
|
raise SqlGenerationError("SQL contains a sensitive field")
|
|
@@ -61,8 +85,10 @@ def enforce_sql_rules(sql: str, *, max_limit: int) -> None:
|
|
|
61
85
|
if ":scope." in normalized:
|
|
62
86
|
raise SqlGenerationError("SQL must not contain scope placeholders")
|
|
63
87
|
_enforce_text_filters_use_like(sql)
|
|
88
|
+
_enforce_join_on_uses_equality_keys(sql)
|
|
64
89
|
_enforce_chinese_select_aliases(sql)
|
|
65
90
|
_enforce_no_null_placeholder_select_items(sql)
|
|
91
|
+
_enforce_no_tautological_join_conditions(sql)
|
|
66
92
|
limit = re.search(r"\blimit\s+(\d+)\b", normalized)
|
|
67
93
|
if limit is None:
|
|
68
94
|
raise SqlGenerationError("SQL must contain LIMIT")
|
|
@@ -70,6 +96,226 @@ def enforce_sql_rules(sql: str, *, max_limit: int) -> None:
|
|
|
70
96
|
raise SqlGenerationError(f"SQL LIMIT must not exceed {max_limit}")
|
|
71
97
|
|
|
72
98
|
|
|
99
|
+
def build_schema_validation_error(
|
|
100
|
+
sql: str,
|
|
101
|
+
issues: list[SchemaValidationIssue],
|
|
102
|
+
schema: dict[str, Any],
|
|
103
|
+
*,
|
|
104
|
+
attempt: int | None = None,
|
|
105
|
+
max_attempts: int | None = None,
|
|
106
|
+
) -> dict[str, Any]:
|
|
107
|
+
"""Build a repair payload listing every schema violation and column hints."""
|
|
108
|
+
schema_issues = [{"code": issue.code, "message": issue.message} for issue in issues]
|
|
109
|
+
codes = {issue.code for issue in issues}
|
|
110
|
+
if len(codes) == 1:
|
|
111
|
+
code = next(iter(codes))
|
|
112
|
+
else:
|
|
113
|
+
code = "SCHEMA_VALIDATION"
|
|
114
|
+
if len(issues) == 1:
|
|
115
|
+
message = issues[0].message
|
|
116
|
+
elif issues:
|
|
117
|
+
message = f"共 {len(issues)} 项 schema 校验失败:{issues[0].message}"
|
|
118
|
+
else:
|
|
119
|
+
message = "schema 校验失败"
|
|
120
|
+
payload: dict[str, Any] = {
|
|
121
|
+
"code": code,
|
|
122
|
+
"message": message,
|
|
123
|
+
"originalSql": sql,
|
|
124
|
+
"schemaIssues": schema_issues,
|
|
125
|
+
}
|
|
126
|
+
unknown_refs = _unknown_column_refs_from_issues(issues)
|
|
127
|
+
if unknown_refs:
|
|
128
|
+
payload["suggestedColumns"] = suggest_columns_for_unknown(unknown_refs, schema)
|
|
129
|
+
if any(issue.code == "MISSING_TABLE_REFERENCE" for issue in issues):
|
|
130
|
+
payload["tableReferenceRepairHint"] = (
|
|
131
|
+
"SQL 引用了未在 FROM/JOIN 中声明的表名或别名。"
|
|
132
|
+
"如果该字段确实需要参与查询,必须按 schema relations / joins 补 JOIN;"
|
|
133
|
+
"如果不需要该表,则删除对应 SELECT/WHERE/ON/GROUP/ORDER 条件。"
|
|
134
|
+
)
|
|
135
|
+
if any(issue.code == "INVALID_JOIN" for issue in issues):
|
|
136
|
+
payload["joinRepairHint"] = (
|
|
137
|
+
"禁止用 ON 1=0 / ON 1=1 / ON false / ON true 规避 JOIN 校验;"
|
|
138
|
+
"必须按 schemaContext.exampleGuidance、exampleJoinPatterns 与 structureHints 重建真实 JOIN。"
|
|
139
|
+
)
|
|
140
|
+
payload["suggestedJoinPatterns"] = suggest_join_patterns_for_schema(schema)
|
|
141
|
+
if attempt is not None:
|
|
142
|
+
payload["attempt"] = attempt
|
|
143
|
+
if max_attempts is not None:
|
|
144
|
+
payload["maxAttempts"] = max_attempts
|
|
145
|
+
return payload
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def build_execute_unknown_column_error(
|
|
149
|
+
sql: str,
|
|
150
|
+
column_ref: str,
|
|
151
|
+
schema: dict[str, Any],
|
|
152
|
+
*,
|
|
153
|
+
attempt: int | None = None,
|
|
154
|
+
max_attempts: int | None = None,
|
|
155
|
+
) -> dict[str, Any]:
|
|
156
|
+
"""Build a repair payload when execute_sql reports a missing column."""
|
|
157
|
+
table_name, column_name = _split_sql_column_ref(column_ref)
|
|
158
|
+
if table_name and column_name:
|
|
159
|
+
message = f"SQL 使用了数据库中不存在的列:{table_name}.{column_name}"
|
|
160
|
+
issue = SchemaValidationIssue(code="UNKNOWN_COLUMN", message=message)
|
|
161
|
+
else:
|
|
162
|
+
message = f"SQL 使用了数据库中不存在的列:{column_ref}"
|
|
163
|
+
issue = SchemaValidationIssue(code="UNKNOWN_COLUMN", message=message)
|
|
164
|
+
return build_schema_validation_error(
|
|
165
|
+
sql,
|
|
166
|
+
[issue],
|
|
167
|
+
schema,
|
|
168
|
+
attempt=attempt,
|
|
169
|
+
max_attempts=max_attempts,
|
|
170
|
+
)
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def _split_sql_column_ref(column_ref: str) -> tuple[str, str]:
|
|
174
|
+
cleaned = column_ref.strip().strip("`")
|
|
175
|
+
if "." not in cleaned:
|
|
176
|
+
return "", cleaned
|
|
177
|
+
table_name, column_name = cleaned.rsplit(".", 1)
|
|
178
|
+
return table_name, column_name
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
def _unknown_column_refs_from_issues(issues: list[SchemaValidationIssue]) -> list[tuple[str, str]]:
|
|
182
|
+
refs: list[tuple[str, str]] = []
|
|
183
|
+
seen: set[tuple[str, str]] = set()
|
|
184
|
+
for issue in issues:
|
|
185
|
+
if issue.code != "UNKNOWN_COLUMN":
|
|
186
|
+
continue
|
|
187
|
+
match = re.search(
|
|
188
|
+
r"不存在的列:([A-Za-z_][\w]*)\.([A-Za-z_][\w]*)",
|
|
189
|
+
issue.message,
|
|
190
|
+
)
|
|
191
|
+
if not match:
|
|
192
|
+
continue
|
|
193
|
+
key = (match.group(1).lower(), match.group(2).lower())
|
|
194
|
+
if key in seen:
|
|
195
|
+
continue
|
|
196
|
+
seen.add(key)
|
|
197
|
+
refs.append(key)
|
|
198
|
+
return refs
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
def suggest_columns_for_unknown(
|
|
202
|
+
missing: list[tuple[str, str]],
|
|
203
|
+
schema: dict[str, Any],
|
|
204
|
+
) -> list[dict[str, str]]:
|
|
205
|
+
"""Find same column name on other tables when SQL references a non-existent column."""
|
|
206
|
+
column_map = _schema_table_column_map(schema)
|
|
207
|
+
suggestions: list[dict[str, str]] = []
|
|
208
|
+
seen: set[tuple[str, str]] = set()
|
|
209
|
+
for wrong_table, column in missing:
|
|
210
|
+
for table, columns in sorted(column_map.items()):
|
|
211
|
+
if table == wrong_table or column not in columns:
|
|
212
|
+
continue
|
|
213
|
+
key = (table, column)
|
|
214
|
+
if key in seen:
|
|
215
|
+
continue
|
|
216
|
+
seen.add(key)
|
|
217
|
+
suggestions.append(
|
|
218
|
+
{
|
|
219
|
+
"table": table,
|
|
220
|
+
"column": column,
|
|
221
|
+
"message": f"列 {column} 存在于 {table},不存在于 {wrong_table}",
|
|
222
|
+
}
|
|
223
|
+
)
|
|
224
|
+
return suggestions
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
def build_sql_rule_validation_error(
|
|
228
|
+
sql: str,
|
|
229
|
+
exc: SqlGenerationError,
|
|
230
|
+
*,
|
|
231
|
+
max_limit: int,
|
|
232
|
+
attempt: int | None = None,
|
|
233
|
+
max_attempts: int | None = None,
|
|
234
|
+
) -> dict[str, Any]:
|
|
235
|
+
"""Build a repair payload with the failing SQL and concrete violating SELECT items."""
|
|
236
|
+
message = str(exc)
|
|
237
|
+
payload: dict[str, Any] = {
|
|
238
|
+
"code": "SQL_RULE_VIOLATION",
|
|
239
|
+
"message": message,
|
|
240
|
+
"originalSql": sql,
|
|
241
|
+
"violatingColumns": list_sql_rule_violating_items(sql, message, max_limit=max_limit),
|
|
242
|
+
}
|
|
243
|
+
if attempt is not None:
|
|
244
|
+
payload["attempt"] = attempt
|
|
245
|
+
if max_attempts is not None:
|
|
246
|
+
payload["maxAttempts"] = max_attempts
|
|
247
|
+
return payload
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
def list_sql_rule_violating_items(sql: str, message: str, *, max_limit: int) -> list[str]:
|
|
251
|
+
lowered = message.lower()
|
|
252
|
+
if "chinese alias" in lowered:
|
|
253
|
+
return _list_missing_chinese_alias_select_items(sql)
|
|
254
|
+
if "null placeholders" in lowered:
|
|
255
|
+
return _list_null_placeholder_select_items(sql)
|
|
256
|
+
if "sensitive field" in lowered:
|
|
257
|
+
return _list_sensitive_select_items(sql)
|
|
258
|
+
if "must be select only" in lowered:
|
|
259
|
+
return ["(entire statement is not a single SELECT)"]
|
|
260
|
+
if "must contain limit" in lowered:
|
|
261
|
+
return ["(missing LIMIT clause)"]
|
|
262
|
+
if "must not exceed" in lowered and "limit" in lowered:
|
|
263
|
+
return ["(LIMIT exceeds allowed maximum)"]
|
|
264
|
+
if "scope placeholders" in lowered:
|
|
265
|
+
return re.findall(r":scope\.\w+", sql, flags=re.IGNORECASE)
|
|
266
|
+
if "like" in lowered:
|
|
267
|
+
return _list_text_filter_violations(sql)
|
|
268
|
+
return []
|
|
269
|
+
|
|
270
|
+
|
|
271
|
+
def _list_missing_chinese_alias_select_items(sql: str) -> list[str]:
|
|
272
|
+
select_list = _top_level_select_list(sql)
|
|
273
|
+
if select_list is None:
|
|
274
|
+
return ["(missing top-level FROM clause)"]
|
|
275
|
+
violations: list[str] = []
|
|
276
|
+
for item in _split_top_level_select_items(select_list):
|
|
277
|
+
cleaned = item.strip()
|
|
278
|
+
if not cleaned:
|
|
279
|
+
continue
|
|
280
|
+
alias_match = re.search(
|
|
281
|
+
_SELECT_AS_ALIAS_PATTERN,
|
|
282
|
+
cleaned,
|
|
283
|
+
flags=re.IGNORECASE,
|
|
284
|
+
)
|
|
285
|
+
if alias_match is None:
|
|
286
|
+
violations.append(cleaned)
|
|
287
|
+
continue
|
|
288
|
+
alias = alias_match.group(1).strip("`\"'")
|
|
289
|
+
if re.search(r"[\u4e00-\u9fff]", alias) is None:
|
|
290
|
+
violations.append(cleaned)
|
|
291
|
+
return violations
|
|
292
|
+
|
|
293
|
+
|
|
294
|
+
def _list_null_placeholder_select_items(sql: str) -> list[str]:
|
|
295
|
+
select_list = _top_level_select_list(sql)
|
|
296
|
+
if select_list is None:
|
|
297
|
+
return []
|
|
298
|
+
return [
|
|
299
|
+
item.strip()
|
|
300
|
+
for item in _split_top_level_select_items(select_list)
|
|
301
|
+
if item.strip() and _is_null_placeholder_expression(_select_expression_without_alias(item))
|
|
302
|
+
]
|
|
303
|
+
|
|
304
|
+
|
|
305
|
+
def _list_sensitive_select_items(sql: str) -> list[str]:
|
|
306
|
+
select_clause = _top_level_select_clause(sql)
|
|
307
|
+
if select_clause is None:
|
|
308
|
+
return []
|
|
309
|
+
return [item.strip() for item in _split_select_items(select_clause) if _select_item_contains_sensitive_field(item)]
|
|
310
|
+
|
|
311
|
+
|
|
312
|
+
def _list_text_filter_violations(sql: str) -> list[str]:
|
|
313
|
+
return re.findall(
|
|
314
|
+
r"(?i)(?:\b[\w.]+\b)\s*=\s*'[^']*(?:名称|状态|品牌|城市|项目|门店|供应商|工单)[^']*'",
|
|
315
|
+
sql,
|
|
316
|
+
)
|
|
317
|
+
|
|
318
|
+
|
|
73
319
|
SENSITIVE_FIELD_NAMES = {
|
|
74
320
|
"phone",
|
|
75
321
|
"mobile",
|
|
@@ -206,6 +452,381 @@ def normalize_text_filters_to_like(sql: str) -> str:
|
|
|
206
452
|
return _TEXT_EQUALITY_FILTER_RE.sub(replace, sql)
|
|
207
453
|
|
|
208
454
|
|
|
455
|
+
def normalize_chinese_select_aliases(sql: str, schema: dict[str, Any] | None) -> str:
|
|
456
|
+
"""Add or replace SELECT aliases with Chinese labels from schema column comments."""
|
|
457
|
+
if not schema or not sql.strip():
|
|
458
|
+
return sql
|
|
459
|
+
select_list = _top_level_select_list(sql)
|
|
460
|
+
if select_list is None:
|
|
461
|
+
return sql
|
|
462
|
+
|
|
463
|
+
table_aliases = _sql_table_aliases(sql)
|
|
464
|
+
used_labels: set[str] = set()
|
|
465
|
+
new_items: list[str] = []
|
|
466
|
+
changed = False
|
|
467
|
+
for item in _split_top_level_select_items(select_list):
|
|
468
|
+
cleaned = item.strip()
|
|
469
|
+
if not cleaned:
|
|
470
|
+
new_items.append(cleaned)
|
|
471
|
+
continue
|
|
472
|
+
if _select_item_has_chinese_alias(cleaned):
|
|
473
|
+
alias = _extract_select_alias(cleaned)
|
|
474
|
+
if alias:
|
|
475
|
+
used_labels.add(alias)
|
|
476
|
+
new_items.append(cleaned)
|
|
477
|
+
continue
|
|
478
|
+
expr = _select_expression_without_alias(cleaned)
|
|
479
|
+
label = _infer_chinese_select_alias(expr, table_aliases=table_aliases, schema=schema)
|
|
480
|
+
if label is None or re.search(r"[\u4e00-\u9fff]", label) is None:
|
|
481
|
+
new_items.append(cleaned)
|
|
482
|
+
continue
|
|
483
|
+
label = _dedupe_chinese_alias(label, used_labels)
|
|
484
|
+
used_labels.add(label)
|
|
485
|
+
new_items.append(f"{expr} AS {label}")
|
|
486
|
+
changed = True
|
|
487
|
+
if not changed:
|
|
488
|
+
return sql
|
|
489
|
+
return _replace_top_level_select_list(sql, ", ".join(new_items))
|
|
490
|
+
|
|
491
|
+
|
|
492
|
+
def question_requests_id_field(question: str) -> bool:
|
|
493
|
+
lowered = question.lower()
|
|
494
|
+
return "id" in lowered or "编号" in question or "主键" in question
|
|
495
|
+
|
|
496
|
+
|
|
497
|
+
def strip_identifier_columns_from_select(sql: str, question: str) -> str:
|
|
498
|
+
"""Drop bare id / *_id columns from SELECT unless the question explicitly asks for ids."""
|
|
499
|
+
if question_requests_id_field(question):
|
|
500
|
+
return sql
|
|
501
|
+
select_list = _top_level_select_list(sql)
|
|
502
|
+
if select_list is None:
|
|
503
|
+
return sql
|
|
504
|
+
table_aliases = _sql_table_aliases(sql)
|
|
505
|
+
kept: list[str] = []
|
|
506
|
+
for item in _split_top_level_select_items(select_list):
|
|
507
|
+
cleaned = item.strip()
|
|
508
|
+
if not cleaned:
|
|
509
|
+
continue
|
|
510
|
+
expr = _select_expression_without_alias(cleaned)
|
|
511
|
+
if _select_expr_is_identifier_column(expr, table_aliases):
|
|
512
|
+
continue
|
|
513
|
+
kept.append(cleaned)
|
|
514
|
+
if not kept or len(kept) == len(_split_top_level_select_items(select_list)):
|
|
515
|
+
return sql
|
|
516
|
+
return _replace_top_level_select_list(sql, ", ".join(kept))
|
|
517
|
+
|
|
518
|
+
|
|
519
|
+
def apply_enum_case_to_select(sql: str, schema: dict[str, Any] | None) -> str:
|
|
520
|
+
"""Wrap enum-backed scalar columns in CASE WHEN so SELECT shows Chinese labels."""
|
|
521
|
+
if not schema or not sql.strip():
|
|
522
|
+
return sql
|
|
523
|
+
select_list = _top_level_select_list(sql)
|
|
524
|
+
if select_list is None:
|
|
525
|
+
return sql
|
|
526
|
+
enums_by_field = {
|
|
527
|
+
str(entry.get("field") or ""): entry.get("values")
|
|
528
|
+
for entry in merged_schema_enums(schema)
|
|
529
|
+
if isinstance(entry, dict) and entry.get("field")
|
|
530
|
+
}
|
|
531
|
+
table_aliases = _sql_table_aliases(sql)
|
|
532
|
+
used_labels: set[str] = set()
|
|
533
|
+
new_items: list[str] = []
|
|
534
|
+
changed = False
|
|
535
|
+
for item in _split_top_level_select_items(select_list):
|
|
536
|
+
cleaned = item.strip()
|
|
537
|
+
if not cleaned:
|
|
538
|
+
new_items.append(cleaned)
|
|
539
|
+
continue
|
|
540
|
+
if re.match(r"case\s+", _select_expression_without_alias(cleaned), flags=re.IGNORECASE):
|
|
541
|
+
alias = _extract_select_alias(cleaned)
|
|
542
|
+
if alias:
|
|
543
|
+
used_labels.add(alias)
|
|
544
|
+
new_items.append(cleaned)
|
|
545
|
+
continue
|
|
546
|
+
expr = _select_expression_without_alias(cleaned)
|
|
547
|
+
case_expr = _enum_case_expression_for_select_item(
|
|
548
|
+
expr,
|
|
549
|
+
table_aliases=table_aliases,
|
|
550
|
+
enums_by_field=enums_by_field,
|
|
551
|
+
schema=schema,
|
|
552
|
+
used_labels=used_labels,
|
|
553
|
+
existing_alias=_extract_select_alias(cleaned),
|
|
554
|
+
)
|
|
555
|
+
if case_expr is None:
|
|
556
|
+
alias = _extract_select_alias(cleaned)
|
|
557
|
+
if alias:
|
|
558
|
+
used_labels.add(alias)
|
|
559
|
+
new_items.append(cleaned)
|
|
560
|
+
continue
|
|
561
|
+
used_labels.add(_extract_select_alias(case_expr) or "")
|
|
562
|
+
new_items.append(case_expr)
|
|
563
|
+
changed = True
|
|
564
|
+
if not changed:
|
|
565
|
+
return sql
|
|
566
|
+
return _replace_top_level_select_list(sql, ", ".join(new_items))
|
|
567
|
+
|
|
568
|
+
|
|
569
|
+
def suggest_join_patterns_for_schema(schema: dict[str, Any]) -> list[dict[str, Any]]:
|
|
570
|
+
suggestions: list[dict[str, Any]] = []
|
|
571
|
+
for example in _schema_examples(schema):
|
|
572
|
+
if not isinstance(example, dict):
|
|
573
|
+
continue
|
|
574
|
+
join_pattern = example.get("joinPattern")
|
|
575
|
+
if not isinstance(join_pattern, dict):
|
|
576
|
+
continue
|
|
577
|
+
item: dict[str, Any] = {"joinPattern": join_pattern}
|
|
578
|
+
for key in ("id", "intent", "notes", "questionExamples"):
|
|
579
|
+
value = example.get(key)
|
|
580
|
+
if value:
|
|
581
|
+
item[key] = value
|
|
582
|
+
suggestions.append(item)
|
|
583
|
+
return suggestions[:6]
|
|
584
|
+
|
|
585
|
+
|
|
586
|
+
def finalize_generated_sql(
|
|
587
|
+
sql: str,
|
|
588
|
+
question: str,
|
|
589
|
+
*,
|
|
590
|
+
schema: dict[str, Any] | None = None,
|
|
591
|
+
) -> str:
|
|
592
|
+
normalized = normalize_text_filters_to_like(sql)
|
|
593
|
+
normalized = strip_identifier_columns_from_select(normalized, question)
|
|
594
|
+
normalized = apply_enum_case_to_select(normalized, schema)
|
|
595
|
+
normalized = normalize_chinese_select_aliases(normalized, schema)
|
|
596
|
+
return normalized
|
|
597
|
+
|
|
598
|
+
|
|
599
|
+
def _select_expr_is_identifier_column(expr: str, table_aliases: dict[str, str]) -> bool:
|
|
600
|
+
normalized = expr.strip()
|
|
601
|
+
match = re.fullmatch(r"([a-zA-Z_][\w]*)\.([a-zA-Z_][\w]*)", normalized)
|
|
602
|
+
if match is None:
|
|
603
|
+
return normalized.lower() in {"id"} or normalized.lower().endswith("_id")
|
|
604
|
+
alias, column = match.group(1).lower(), match.group(2).lower()
|
|
605
|
+
if column == "id" or column.endswith("_id"):
|
|
606
|
+
return True
|
|
607
|
+
table = table_aliases.get(alias)
|
|
608
|
+
if table and (column == "id" or column.endswith("_id")):
|
|
609
|
+
return True
|
|
610
|
+
return False
|
|
611
|
+
|
|
612
|
+
|
|
613
|
+
def _enum_case_expression_for_select_item(
|
|
614
|
+
expr: str,
|
|
615
|
+
*,
|
|
616
|
+
table_aliases: dict[str, str],
|
|
617
|
+
enums_by_field: dict[str, Any],
|
|
618
|
+
schema: dict[str, Any],
|
|
619
|
+
used_labels: set[str],
|
|
620
|
+
existing_alias: str | None,
|
|
621
|
+
) -> str | None:
|
|
622
|
+
match = re.fullmatch(r"([a-zA-Z_][\w]*)\.([a-zA-Z_][\w]*)", expr.strip())
|
|
623
|
+
if match is None:
|
|
624
|
+
return None
|
|
625
|
+
alias, column = match.group(1), match.group(2)
|
|
626
|
+
table = table_aliases.get(alias.lower())
|
|
627
|
+
if table is None:
|
|
628
|
+
return None
|
|
629
|
+
values = enums_by_field.get(f"{table}.{column}")
|
|
630
|
+
if not isinstance(values, list) or not values:
|
|
631
|
+
return None
|
|
632
|
+
when_parts: list[str] = []
|
|
633
|
+
for item in values:
|
|
634
|
+
if not isinstance(item, dict):
|
|
635
|
+
continue
|
|
636
|
+
value = item.get("value")
|
|
637
|
+
label = str(item.get("label") or "").strip()
|
|
638
|
+
if value is None or not label:
|
|
639
|
+
continue
|
|
640
|
+
literal = repr(str(value)) if isinstance(value, str) else str(value)
|
|
641
|
+
when_parts.append(f"WHEN {expr} = {literal} THEN {_sql_string_literal(label)}")
|
|
642
|
+
if not when_parts:
|
|
643
|
+
return None
|
|
644
|
+
alias_label = existing_alias or _chinese_label_for_column(schema, table, column) or column
|
|
645
|
+
alias_label = _dedupe_chinese_alias(alias_label, used_labels)
|
|
646
|
+
return f"CASE {' '.join(when_parts)} ELSE CAST({expr} AS CHAR) END AS {alias_label}"
|
|
647
|
+
|
|
648
|
+
|
|
649
|
+
def _sql_string_literal(value: str) -> str:
|
|
650
|
+
return "'" + value.replace("'", "''") + "'"
|
|
651
|
+
|
|
652
|
+
|
|
653
|
+
def _enforce_no_tautological_join_conditions(sql: str) -> None:
|
|
654
|
+
if re.search(
|
|
655
|
+
r"\b(?:left|right|inner|full|cross)?\s*join\b[\s\S]*?\bon\s+(?:1\s*=\s*0|0\s*=\s*1|1\s*=\s*1|true|false)\b",
|
|
656
|
+
sql,
|
|
657
|
+
flags=re.IGNORECASE,
|
|
658
|
+
):
|
|
659
|
+
raise SqlGenerationError(
|
|
660
|
+
"SQL JOIN must not use tautological ON conditions such as ON 1=0 or ON 1=1; "
|
|
661
|
+
"use schema exampleJoinPatterns and structureHints to build real joins"
|
|
662
|
+
)
|
|
663
|
+
|
|
664
|
+
|
|
665
|
+
_JOIN_ON_START_RE = re.compile(
|
|
666
|
+
r"\b(?:(?:left|right|inner|full|cross)\s+)?join\s+(?:\([^)]+\)|[a-zA-Z_][\w]*(?:\s+(?:as\s+)?[a-zA-Z_][\w]*)?)\s+on\s+",
|
|
667
|
+
re.IGNORECASE,
|
|
668
|
+
)
|
|
669
|
+
_JOIN_ON_END_RE = re.compile(
|
|
670
|
+
r"\b(where|group\s+by|order\s+by|limit|(?:left|right|inner|full|cross)\s+join)\b",
|
|
671
|
+
re.IGNORECASE,
|
|
672
|
+
)
|
|
673
|
+
_CROSS_TABLE_EQUALITY_RE = re.compile(
|
|
674
|
+
r"(?:"
|
|
675
|
+
r"\b[a-zA-Z_][a-zA-Z0-9_]*\.[a-zA-Z_][a-zA-Z0-9_]*\s*=\s*"
|
|
676
|
+
r"(?:cast\s*\(\s*[a-zA-Z_][a-zA-Z0-9_]*\.[a-zA-Z_][a-zA-Z0-9_]*\s+as\s+\w+\s*\)"
|
|
677
|
+
r"|[a-zA-Z_][a-zA-Z0-9_]*\.[a-zA-Z_][a-zA-Z0-9_]*)"
|
|
678
|
+
r"|"
|
|
679
|
+
r"cast\s*\(\s*[a-zA-Z_][a-zA-Z0-9_]*\.[a-zA-Z_][a-zA-Z0-9_]*\s+as\s+\w+\s*\)\s*=\s*"
|
|
680
|
+
r"[a-zA-Z_][a-zA-Z0-9_]*\.[a-zA-Z_][a-zA-Z0-9_]*)",
|
|
681
|
+
re.IGNORECASE,
|
|
682
|
+
)
|
|
683
|
+
|
|
684
|
+
|
|
685
|
+
def _iter_join_on_clauses(sql: str) -> list[str]:
|
|
686
|
+
clauses: list[str] = []
|
|
687
|
+
for match in _JOIN_ON_START_RE.finditer(sql):
|
|
688
|
+
rest = sql[match.end() :]
|
|
689
|
+
end_match = _JOIN_ON_END_RE.search(rest)
|
|
690
|
+
end = end_match.start() if end_match else len(rest)
|
|
691
|
+
clause = rest[:end].strip()
|
|
692
|
+
if clause:
|
|
693
|
+
clauses.append(clause)
|
|
694
|
+
return clauses
|
|
695
|
+
|
|
696
|
+
|
|
697
|
+
def _list_join_on_violations(sql: str) -> list[str]:
|
|
698
|
+
violations: list[str] = []
|
|
699
|
+
for on_clause in _iter_join_on_clauses(sql):
|
|
700
|
+
preview = re.sub(r"\s+", " ", on_clause).strip()[:120]
|
|
701
|
+
if re.search(r"\blike\b", on_clause, re.IGNORECASE):
|
|
702
|
+
violations.append(
|
|
703
|
+
"JOIN ON 禁止写 LIKE 筛选,请按 schema relations 写主表与关联表的等值键,"
|
|
704
|
+
f"把实体筛选放到 WHERE:{preview}"
|
|
705
|
+
)
|
|
706
|
+
continue
|
|
707
|
+
if re.search(r"\bin\s*\(", on_clause, re.IGNORECASE):
|
|
708
|
+
violations.append(
|
|
709
|
+
"JOIN ON 禁止写 IN 筛选,请按 schema relations 写主表与关联表的等值键,"
|
|
710
|
+
f"把实体筛选放到 WHERE:{preview}"
|
|
711
|
+
)
|
|
712
|
+
continue
|
|
713
|
+
if _CROSS_TABLE_EQUALITY_RE.search(on_clause) is None:
|
|
714
|
+
violations.append(
|
|
715
|
+
"JOIN ON 必须使用主表与关联表的等值关联(如 store.organization_id = sponge_project.id),"
|
|
716
|
+
f"不能只有筛选条件:{preview}"
|
|
717
|
+
)
|
|
718
|
+
return violations
|
|
719
|
+
|
|
720
|
+
|
|
721
|
+
def _enforce_join_on_uses_equality_keys(sql: str) -> None:
|
|
722
|
+
violations = _list_join_on_violations(sql)
|
|
723
|
+
if violations:
|
|
724
|
+
raise SqlGenerationError(violations[0])
|
|
725
|
+
|
|
726
|
+
|
|
727
|
+
def _select_item_has_chinese_alias(item: str) -> bool:
|
|
728
|
+
alias = _extract_select_alias(item)
|
|
729
|
+
return alias is not None and re.search(r"[\u4e00-\u9fff]", alias) is not None
|
|
730
|
+
|
|
731
|
+
|
|
732
|
+
def _extract_select_alias(item: str) -> str | None:
|
|
733
|
+
alias_match = re.search(
|
|
734
|
+
_SELECT_AS_ALIAS_PATTERN,
|
|
735
|
+
item.strip(),
|
|
736
|
+
flags=re.IGNORECASE,
|
|
737
|
+
)
|
|
738
|
+
if alias_match is None:
|
|
739
|
+
return None
|
|
740
|
+
return alias_match.group(1).strip("`\"'")
|
|
741
|
+
|
|
742
|
+
|
|
743
|
+
def _dedupe_chinese_alias(label: str, used: set[str]) -> str:
|
|
744
|
+
if label not in used:
|
|
745
|
+
return label
|
|
746
|
+
index = 2
|
|
747
|
+
while f"{label}{index}" in used:
|
|
748
|
+
index += 1
|
|
749
|
+
return f"{label}{index}"
|
|
750
|
+
|
|
751
|
+
|
|
752
|
+
def _infer_chinese_select_alias(
|
|
753
|
+
expr: str,
|
|
754
|
+
*,
|
|
755
|
+
table_aliases: dict[str, str],
|
|
756
|
+
schema: dict[str, Any],
|
|
757
|
+
) -> str | None:
|
|
758
|
+
normalized = expr.strip()
|
|
759
|
+
if not normalized:
|
|
760
|
+
return None
|
|
761
|
+
|
|
762
|
+
aggregate_match = re.match(r"(count|sum|avg|min|max)\s*\(", normalized, flags=re.IGNORECASE)
|
|
763
|
+
if aggregate_match is not None:
|
|
764
|
+
labels = {
|
|
765
|
+
"count": "数量",
|
|
766
|
+
"sum": "合计",
|
|
767
|
+
"avg": "平均值",
|
|
768
|
+
"min": "最小值",
|
|
769
|
+
"max": "最大值",
|
|
770
|
+
}
|
|
771
|
+
return labels.get(aggregate_match.group(1).lower())
|
|
772
|
+
|
|
773
|
+
if re.match(r"case\s+", normalized, flags=re.IGNORECASE):
|
|
774
|
+
return "结果"
|
|
775
|
+
|
|
776
|
+
qualified_match = re.search(
|
|
777
|
+
r"(?:([a-zA-Z_][a-zA-Z0-9_]*)\.)?([a-zA-Z_][a-zA-Z0-9_]*)\s*$",
|
|
778
|
+
normalized,
|
|
779
|
+
)
|
|
780
|
+
if qualified_match is not None:
|
|
781
|
+
alias_part, column_name = qualified_match.group(1), qualified_match.group(2)
|
|
782
|
+
if alias_part is not None:
|
|
783
|
+
table_name = table_aliases.get(alias_part.lower())
|
|
784
|
+
if table_name is not None:
|
|
785
|
+
return _chinese_label_for_column(schema, table_name, column_name)
|
|
786
|
+
return _chinese_label_for_bare_column(schema, table_aliases, column_name)
|
|
787
|
+
return None
|
|
788
|
+
|
|
789
|
+
|
|
790
|
+
def _chinese_label_for_column(schema: dict[str, Any], table_name: str, column_name: str) -> str | None:
|
|
791
|
+
for column in _table_columns(schema, table_name):
|
|
792
|
+
name = str(column.get("columnName") or column.get("name") or "")
|
|
793
|
+
if name.lower() != column_name.lower():
|
|
794
|
+
continue
|
|
795
|
+
label = _column_alias(name, str(column.get("comment") or ""))
|
|
796
|
+
if re.search(r"[\u4e00-\u9fff]", label):
|
|
797
|
+
return label
|
|
798
|
+
return None
|
|
799
|
+
|
|
800
|
+
|
|
801
|
+
def _chinese_label_for_bare_column(
|
|
802
|
+
schema: dict[str, Any],
|
|
803
|
+
table_aliases: dict[str, str],
|
|
804
|
+
column_name: str,
|
|
805
|
+
) -> str | None:
|
|
806
|
+
labels = [
|
|
807
|
+
label
|
|
808
|
+
for table_name in set(table_aliases.values())
|
|
809
|
+
if (label := _chinese_label_for_column(schema, table_name, column_name)) is not None
|
|
810
|
+
]
|
|
811
|
+
if len(labels) == 1:
|
|
812
|
+
return labels[0]
|
|
813
|
+
return None
|
|
814
|
+
|
|
815
|
+
|
|
816
|
+
def _replace_top_level_select_list(sql: str, select_list: str) -> str:
|
|
817
|
+
stripped = sql.strip().rstrip(";")
|
|
818
|
+
match = re.match(r"select\s+(?:distinct\s+)?", stripped, flags=re.IGNORECASE)
|
|
819
|
+
if match is None:
|
|
820
|
+
return sql
|
|
821
|
+
start = match.end()
|
|
822
|
+
from_index = _find_top_level_keyword(stripped, "from", start=start)
|
|
823
|
+
if from_index is None:
|
|
824
|
+
return sql
|
|
825
|
+
prefix = stripped[:start].rstrip()
|
|
826
|
+
suffix = stripped[from_index:].lstrip()
|
|
827
|
+
return f"{prefix} {select_list} {suffix}"
|
|
828
|
+
|
|
829
|
+
|
|
209
830
|
def _should_normalize_text_filter_to_like(column_name: str, value: str) -> bool:
|
|
210
831
|
if not value or _is_numeric_literal(value):
|
|
211
832
|
return False
|
|
@@ -237,159 +858,156 @@ def is_low_quality_sql_for_question(question: str, sql: str, *, schema: dict[str
|
|
|
237
858
|
|
|
238
859
|
|
|
239
860
|
def low_quality_sql_issue_for_question(question: str, sql: str, *, schema: dict[str, Any] | None = None) -> LowQualitySqlIssue | None:
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
)
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
)
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
)
|
|
278
|
-
job_name = _extract_job_name_filter(question)
|
|
279
|
-
if job_name is not None and _sql_references_table(sql, "job_basic_info") and not _sql_contains_job_name_filter(sql, job_name):
|
|
861
|
+
if schema is None:
|
|
862
|
+
return None
|
|
863
|
+
from .schema_context_retriever import match_concepts_for_question
|
|
864
|
+
|
|
865
|
+
matched_concepts = match_concepts_for_question(_schema_concepts(schema), question)
|
|
866
|
+
normalized = sql.strip().lower()
|
|
867
|
+
for concept in matched_concepts:
|
|
868
|
+
checks = concept.get("sqlQualityChecks")
|
|
869
|
+
if not _concept_quality_check_applies(concept, question):
|
|
870
|
+
continue
|
|
871
|
+
time_range_issue = _concept_time_range_field_issue(question, sql, concept)
|
|
872
|
+
if time_range_issue is not None:
|
|
873
|
+
return time_range_issue
|
|
874
|
+
if not isinstance(checks, dict):
|
|
875
|
+
continue
|
|
876
|
+
for table_name in checks.get("requireTables") or []:
|
|
877
|
+
table = str(table_name or "").strip().lower()
|
|
878
|
+
if table and table not in normalized:
|
|
879
|
+
message = str(checks.get("message") or f"SQL 必须引用 schema 要求的表:{table_name}")
|
|
880
|
+
return LowQualitySqlIssue(code="SCHEMA_SQL_QUALITY", message=message)
|
|
881
|
+
aggregate_issue = _aggregate_sql_quality_issue(question, sql)
|
|
882
|
+
if aggregate_issue is not None:
|
|
883
|
+
return aggregate_issue
|
|
884
|
+
return None
|
|
885
|
+
|
|
886
|
+
|
|
887
|
+
def _concept_time_range_field_issue(
|
|
888
|
+
question: str,
|
|
889
|
+
sql: str,
|
|
890
|
+
concept: dict[str, Any],
|
|
891
|
+
) -> LowQualitySqlIssue | None:
|
|
892
|
+
qualified = str(concept.get("timeRangeField") or "").strip()
|
|
893
|
+
if "." not in qualified or _extract_time_bounds(question) is None:
|
|
894
|
+
return None
|
|
895
|
+
table_name, field_name = qualified.rsplit(".", 1)
|
|
896
|
+
time_range = TimeRange(field=field_name, alias="时间范围", start="", end="", table_name=table_name)
|
|
897
|
+
if not _sql_has_time_filter_for_range(sql, time_range):
|
|
280
898
|
return LowQualitySqlIssue(
|
|
281
|
-
code="
|
|
282
|
-
message=
|
|
899
|
+
code="SCHEMA_TIME_RANGE_FIELD_REQUIRED",
|
|
900
|
+
message=(
|
|
901
|
+
f"用户问题包含时间范围,且 schema concept 指定时间字段为 {qualified};"
|
|
902
|
+
"SQL 必须使用该字段过滤时间范围,不要改用关联维表的同名时间字段。"
|
|
903
|
+
),
|
|
283
904
|
)
|
|
284
|
-
|
|
285
|
-
if
|
|
905
|
+
unexpected = _unexpected_time_filter_fields(sql, expected_field=qualified)
|
|
906
|
+
if unexpected:
|
|
286
907
|
return LowQualitySqlIssue(
|
|
287
|
-
code="
|
|
288
|
-
message=
|
|
908
|
+
code="SCHEMA_TIME_RANGE_FIELD_REQUIRED",
|
|
909
|
+
message=(
|
|
910
|
+
f"用户问题包含时间范围,且 schema concept 指定时间字段为 {qualified};"
|
|
911
|
+
f"SQL 不应同时使用 {'、'.join(unexpected)} 过滤时间范围。"
|
|
912
|
+
),
|
|
289
913
|
)
|
|
290
|
-
if _is_work_order_detail_query(question):
|
|
291
|
-
if "mvp_applied_jobs_by_helped_account" not in normalized:
|
|
292
|
-
return LowQualitySqlIssue(
|
|
293
|
-
code="MISSING_WORK_ORDER_TABLE",
|
|
294
|
-
message="本次查询未执行:你问的是报名/工单明细,但生成的 SQL 没有使用报名工单表。",
|
|
295
|
-
)
|
|
296
|
-
if re.search(r"\bfrom\s+brand\b", normalized) and "mvp_applied_jobs_by_helped_account" not in normalized:
|
|
297
|
-
return LowQualitySqlIssue(
|
|
298
|
-
code="WORK_ORDER_QUERY_BRAND_ONLY",
|
|
299
|
-
message="本次查询未执行:你问的是报名/工单明细,但生成的 SQL 只查询了品牌列表。",
|
|
300
|
-
)
|
|
301
|
-
if _has_work_order_job_id_to_job_pk_join(normalized):
|
|
302
|
-
return LowQualitySqlIssue(
|
|
303
|
-
code="WORK_ORDER_JOB_JOIN_MISMATCH",
|
|
304
|
-
message="本次查询未执行:报名工单和岗位表的关联方式不符合当前 schema 声明。",
|
|
305
|
-
)
|
|
306
|
-
if "面试成功" in question and "interview_pass_status" in normalized:
|
|
307
|
-
return LowQualitySqlIssue(
|
|
308
|
-
code="INTERVIEW_STATUS_FIELD_MISMATCH",
|
|
309
|
-
message="本次查询未执行:你问的是候选人面试成功状态,但生成的 SQL 使用了不符合当前口径的面试状态字段。",
|
|
310
|
-
)
|
|
311
914
|
return None
|
|
312
915
|
|
|
313
916
|
|
|
314
|
-
def
|
|
315
|
-
|
|
316
|
-
if
|
|
917
|
+
def _sql_has_time_filter_for_range(sql: str, time_range: TimeRange) -> bool:
|
|
918
|
+
where_index = _find_top_level_keyword(sql, "where", start=0)
|
|
919
|
+
if where_index is None:
|
|
317
920
|
return False
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
if
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
921
|
+
clause_start = where_index + len("where")
|
|
922
|
+
clause_end = _where_filter_insert_pos(sql[clause_start:])
|
|
923
|
+
if clause_end < len(sql[clause_start:]):
|
|
924
|
+
clause_end += clause_start
|
|
925
|
+
else:
|
|
926
|
+
clause_end = len(sql)
|
|
927
|
+
for predicate in _split_top_level_and_predicates(sql[clause_start:clause_end].strip()):
|
|
928
|
+
if _is_time_range_predicate(predicate, sql=sql, time_range=time_range):
|
|
929
|
+
return True
|
|
930
|
+
return False
|
|
326
931
|
|
|
327
932
|
|
|
328
|
-
def
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
933
|
+
def _unexpected_time_filter_fields(sql: str, *, expected_field: str) -> list[str]:
|
|
934
|
+
expected_table, expected_column = expected_field.lower().rsplit(".", 1)
|
|
935
|
+
aliases = _sql_table_aliases(sql)
|
|
936
|
+
unexpected: list[str] = []
|
|
937
|
+
seen: set[str] = set()
|
|
938
|
+
where_index = _find_top_level_keyword(sql, "where", start=0)
|
|
939
|
+
if where_index is None:
|
|
940
|
+
return []
|
|
941
|
+
clause_start = where_index + len("where")
|
|
942
|
+
clause_end = _where_filter_insert_pos(sql[clause_start:])
|
|
943
|
+
if clause_end < len(sql[clause_start:]):
|
|
944
|
+
clause_end += clause_start
|
|
945
|
+
else:
|
|
946
|
+
clause_end = len(sql)
|
|
947
|
+
for predicate in _split_top_level_and_predicates(sql[clause_start:clause_end].strip()):
|
|
948
|
+
if re.search(r"'20\d{2}-\d{1,2}-\d{1,2}", predicate) is None:
|
|
949
|
+
continue
|
|
950
|
+
if re.search(r"(?:\bbetween\b|>=|>|<=|<|=)", predicate, flags=re.IGNORECASE) is None:
|
|
951
|
+
continue
|
|
952
|
+
for match in re.finditer(r"\b([a-zA-Z_][\w]*)\.([a-zA-Z_][\w]*)\b", predicate):
|
|
953
|
+
qualifier = match.group(1).lower()
|
|
954
|
+
column = match.group(2).lower()
|
|
955
|
+
if column != expected_column:
|
|
956
|
+
continue
|
|
957
|
+
table = aliases.get(qualifier, qualifier)
|
|
958
|
+
qualified = f"{table}.{column}"
|
|
959
|
+
if qualified == f"{expected_table}.{expected_column}" or qualified in seen:
|
|
960
|
+
continue
|
|
961
|
+
seen.add(qualified)
|
|
962
|
+
unexpected.append(qualified)
|
|
963
|
+
return unexpected
|
|
340
964
|
|
|
341
965
|
|
|
342
|
-
def
|
|
343
|
-
if
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
):
|
|
347
|
-
return
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
966
|
+
def _aggregate_sql_quality_issue(question: str, sql: str) -> LowQualitySqlIssue | None:
|
|
967
|
+
if not _question_requires_aggregate_sql(question):
|
|
968
|
+
return None
|
|
969
|
+
normalized = sql.lower()
|
|
970
|
+
if re.search(r"\b(count|sum|avg|min|max)\s*\(", normalized):
|
|
971
|
+
return None
|
|
972
|
+
return LowQualitySqlIssue(
|
|
973
|
+
code="AGGREGATE_SQL_REQUIRED",
|
|
974
|
+
message="用户问题要求统计数量/最多值及对应维度,SQL 必须使用 COUNT/SUM/AVG/MIN/MAX 等聚合函数,不能只返回明细行。",
|
|
975
|
+
)
|
|
351
976
|
|
|
352
977
|
|
|
353
|
-
def
|
|
354
|
-
if re.search(r"
|
|
355
|
-
return True
|
|
356
|
-
requested_values = _requested_job_status_values(question, schema)
|
|
357
|
-
if requested_values is None:
|
|
978
|
+
def _question_requires_aggregate_sql(question: str) -> bool:
|
|
979
|
+
if not re.search(r"(计数|统计|数量|个数|总数|多少)", question):
|
|
358
980
|
return False
|
|
359
|
-
|
|
360
|
-
return True
|
|
361
|
-
status_enum = _status_enum(schema, "job_basic_info.status")
|
|
362
|
-
if not status_enum:
|
|
363
|
-
return True
|
|
364
|
-
recruiting_values: set[str] = set()
|
|
365
|
-
for item in status_enum:
|
|
366
|
-
value = item.get("value")
|
|
367
|
-
labels = _enum_label_terms(str(item.get("label") or ""))
|
|
368
|
-
if value is None:
|
|
369
|
-
continue
|
|
370
|
-
if any(label in {"在招", "已发布", "已上架"} or "在招" in label for label in labels):
|
|
371
|
-
recruiting_values.add(str(value))
|
|
372
|
-
return bool(requested_values & recruiting_values)
|
|
981
|
+
return bool(re.search(r"(最多|最少|最大|最小|排行|排名|对应|按.+?统计|根据.+?计数)", question))
|
|
373
982
|
|
|
374
983
|
|
|
375
|
-
def
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
984
|
+
def _concept_quality_check_applies(concept: dict[str, Any], question: str) -> bool:
|
|
985
|
+
question_text = _compact_quality_match_text(question)
|
|
986
|
+
if not question_text:
|
|
987
|
+
return False
|
|
988
|
+
for example in concept.get("negativeExamples") or []:
|
|
989
|
+
if _quality_phrase_matches(question_text, example):
|
|
990
|
+
return False
|
|
991
|
+
for example in concept.get("positiveExamples") or []:
|
|
992
|
+
if _quality_phrase_matches(question_text, example):
|
|
993
|
+
return True
|
|
994
|
+
for example in concept.get("questionExamples") or []:
|
|
995
|
+
if _quality_phrase_matches(question_text, example):
|
|
996
|
+
return True
|
|
997
|
+
for key in ("comment", "description", "intent", "name", "term"):
|
|
998
|
+
if _quality_phrase_matches(question_text, concept.get(key)):
|
|
999
|
+
return True
|
|
1000
|
+
return False
|
|
1001
|
+
|
|
1002
|
+
|
|
1003
|
+
def _quality_phrase_matches(question_text: str, phrase: Any) -> bool:
|
|
1004
|
+
phrase_text = _compact_quality_match_text(str(phrase or ""))
|
|
1005
|
+
return len(phrase_text) >= 2 and phrase_text in question_text
|
|
1006
|
+
|
|
1007
|
+
|
|
1008
|
+
def _compact_quality_match_text(text: str) -> str:
|
|
1009
|
+
normalized = text.translate(str.maketrans("0123456789", "0123456789")).lower()
|
|
1010
|
+
return "".join(re.findall(r"[a-z0-9\u4e00-\u9fff]+", normalized))
|
|
393
1011
|
|
|
394
1012
|
|
|
395
1013
|
def _sql_select_metric_aliases(sql: str) -> list[str]:
|
|
@@ -408,58 +1026,6 @@ def _sql_select_metric_aliases(sql: str) -> list[str]:
|
|
|
408
1026
|
return aliases
|
|
409
1027
|
|
|
410
1028
|
|
|
411
|
-
def _generic_job_count_sql_mismatches_question(
|
|
412
|
-
question: str,
|
|
413
|
-
sql: str,
|
|
414
|
-
schema: dict[str, Any] | None,
|
|
415
|
-
) -> bool:
|
|
416
|
-
if not _is_job_count_intent_question(question) or not _sql_counts_job_basic_info(sql):
|
|
417
|
-
return False
|
|
418
|
-
preferred_alias = _preferred_count_alias(question)
|
|
419
|
-
aliases = _sql_select_metric_aliases(sql)
|
|
420
|
-
if preferred_alias and aliases and preferred_alias not in aliases:
|
|
421
|
-
if any(alias in {"在招岗位数", "在招岗位", "已发布岗位数"} for alias in aliases):
|
|
422
|
-
return not _user_requests_recruiting_job_count(question, schema)
|
|
423
|
-
if preferred_alias == "总数" and any("在招" in alias for alias in aliases):
|
|
424
|
-
return not _user_requests_recruiting_job_count(question, schema)
|
|
425
|
-
if _user_requests_recruiting_job_count(question, schema):
|
|
426
|
-
return False
|
|
427
|
-
if any("在招" in alias for alias in aliases):
|
|
428
|
-
return True
|
|
429
|
-
return False
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
def _sql_counts_job_basic_info(sql: str) -> bool:
|
|
433
|
-
normalized = re.sub(r"\s+", " ", sql.strip().lower())
|
|
434
|
-
return "count(" in normalized and "job_basic_info" in normalized
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
def _strip_job_status_filters(sql: str) -> str:
|
|
438
|
-
table_aliases = _sql_table_aliases(sql)
|
|
439
|
-
job_aliases = {alias for alias, table in table_aliases.items() if table == "job_basic_info"}
|
|
440
|
-
if not job_aliases:
|
|
441
|
-
return sql
|
|
442
|
-
alias_pattern = "|".join(re.escape(alias) for alias in sorted(job_aliases, key=len, reverse=True))
|
|
443
|
-
columns = [r"(?<!\.)\bstatus"]
|
|
444
|
-
if alias_pattern:
|
|
445
|
-
columns.append(rf"\b(?:{alias_pattern})\.status")
|
|
446
|
-
cleaned = sql
|
|
447
|
-
for column in columns:
|
|
448
|
-
cleaned = re.sub(
|
|
449
|
-
rf"\s+and\s+{column}\s*=\s*-?\d+\b",
|
|
450
|
-
"",
|
|
451
|
-
cleaned,
|
|
452
|
-
flags=re.IGNORECASE,
|
|
453
|
-
)
|
|
454
|
-
cleaned = re.sub(
|
|
455
|
-
rf"\s+and\s+{column}\s+in\s*\([^)]*\)",
|
|
456
|
-
"",
|
|
457
|
-
cleaned,
|
|
458
|
-
flags=re.IGNORECASE,
|
|
459
|
-
)
|
|
460
|
-
return cleaned
|
|
461
|
-
|
|
462
|
-
|
|
463
1029
|
def _replace_primary_metric_alias(sql: str, alias: str) -> str:
|
|
464
1030
|
select_list = _top_level_select_list(sql)
|
|
465
1031
|
if select_list is None:
|
|
@@ -487,52 +1053,38 @@ def _replace_primary_metric_alias(sql: str, alias: str) -> str:
|
|
|
487
1053
|
)
|
|
488
1054
|
|
|
489
1055
|
|
|
490
|
-
def
|
|
491
|
-
if not _is_job_count_intent_question(question) or not _sql_counts_job_basic_info(sql):
|
|
492
|
-
return sql
|
|
493
|
-
preferred_alias = _preferred_count_alias(question)
|
|
494
|
-
normalized = sql
|
|
495
|
-
if not _user_requests_recruiting_job_count(question, schema):
|
|
496
|
-
normalized = _strip_job_status_filters(normalized)
|
|
497
|
-
if preferred_alias:
|
|
498
|
-
normalized = _replace_primary_metric_alias(normalized, preferred_alias)
|
|
499
|
-
elif any("在招" in alias for alias in _sql_select_metric_aliases(normalized)):
|
|
500
|
-
normalized = _replace_primary_metric_alias(normalized, "总数")
|
|
501
|
-
elif preferred_alias:
|
|
502
|
-
normalized = _replace_primary_metric_alias(normalized, preferred_alias)
|
|
503
|
-
return normalized
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
def _requested_job_status_values(question: str, schema: dict[str, Any] | None) -> set[str] | None:
|
|
507
|
-
if not schema:
|
|
508
|
-
return None
|
|
509
|
-
status_enum = _status_enum(schema, "job_basic_info.status")
|
|
510
|
-
if status_enum is None:
|
|
511
|
-
return None
|
|
512
|
-
matched: set[str] = set()
|
|
513
|
-
for item in status_enum:
|
|
514
|
-
value = item.get("value")
|
|
515
|
-
labels = _enum_label_terms(str(item.get("label") or ""))
|
|
516
|
-
if value is None or not labels:
|
|
517
|
-
continue
|
|
518
|
-
if any(label and label in question for label in labels):
|
|
519
|
-
matched.add(str(value))
|
|
520
|
-
return matched or None
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
def _status_enum(schema: dict[str, Any], field: str) -> list[dict[str, Any]] | None:
|
|
1056
|
+
def _schema_enums(schema: dict[str, Any]) -> list[dict[str, Any]]:
|
|
524
1057
|
enums = schema.get("enums")
|
|
525
|
-
if
|
|
526
|
-
return
|
|
527
|
-
|
|
528
|
-
|
|
1058
|
+
if isinstance(enums, list):
|
|
1059
|
+
return [enum for enum in enums if isinstance(enum, dict)]
|
|
1060
|
+
structure = schema.get("structure")
|
|
1061
|
+
concepts = structure.get("concepts") if isinstance(structure, dict) else None
|
|
1062
|
+
if not isinstance(concepts, list):
|
|
1063
|
+
return []
|
|
1064
|
+
by_field: dict[str, dict[Any, str]] = {}
|
|
1065
|
+
for concept in concepts:
|
|
1066
|
+
if not isinstance(concept, dict):
|
|
529
1067
|
continue
|
|
530
|
-
|
|
1068
|
+
enum_refs = concept.get("enumRefs")
|
|
1069
|
+
if not isinstance(enum_refs, list):
|
|
531
1070
|
continue
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
1071
|
+
label_parts = [
|
|
1072
|
+
str(concept.get("comment") or ""),
|
|
1073
|
+
str(concept.get("name") or ""),
|
|
1074
|
+
str(concept.get("description") or ""),
|
|
1075
|
+
]
|
|
1076
|
+
for enum_ref in enum_refs:
|
|
1077
|
+
if not isinstance(enum_ref, dict):
|
|
1078
|
+
continue
|
|
1079
|
+
field_name = str(enum_ref.get("field") or "")
|
|
1080
|
+
if not field_name or "value" not in enum_ref:
|
|
1081
|
+
continue
|
|
1082
|
+
label = " / ".join(part for part in label_parts + [str(enum_ref.get("meaning") or "")] if part)
|
|
1083
|
+
by_field.setdefault(field_name, {})[enum_ref["value"]] = label
|
|
1084
|
+
return [
|
|
1085
|
+
{"field": field_name, "values": [{"value": value, "label": label} for value, label in values.items()]}
|
|
1086
|
+
for field_name, values in by_field.items()
|
|
1087
|
+
]
|
|
536
1088
|
|
|
537
1089
|
|
|
538
1090
|
def _enum_label_terms(label: str) -> set[str]:
|
|
@@ -541,26 +1093,18 @@ def _enum_label_terms(label: str) -> set[str]:
|
|
|
541
1093
|
return {term for term in terms if term}
|
|
542
1094
|
|
|
543
1095
|
|
|
544
|
-
def _sql_status_filter_values(sql: str, job_aliases: set[str]) -> set[str] | None:
|
|
545
|
-
values: set[str] = set()
|
|
546
|
-
alias_pattern = "|".join(re.escape(alias) for alias in sorted(job_aliases, key=len, reverse=True))
|
|
547
|
-
columns = [r"(?<!\.)\bstatus"]
|
|
548
|
-
if alias_pattern:
|
|
549
|
-
columns.append(rf"\b(?:{alias_pattern})\.status")
|
|
550
|
-
for column in columns:
|
|
551
|
-
for match in re.finditer(rf"{column}\s*=\s*(?P<value>-?\d+)\b", sql, flags=re.IGNORECASE):
|
|
552
|
-
values.add(match.group("value"))
|
|
553
|
-
for match in re.finditer(rf"{column}\s+in\s*\((?P<values>[^)]*)\)", sql, flags=re.IGNORECASE):
|
|
554
|
-
parsed = re.findall(r"-?\d+", match.group("values"))
|
|
555
|
-
values.update(parsed)
|
|
556
|
-
return values or None
|
|
557
|
-
|
|
558
|
-
|
|
559
1096
|
def validate_sql_against_schema(sql: str, schema: dict[str, Any]) -> list[SchemaValidationIssue]:
|
|
560
1097
|
issues: list[SchemaValidationIssue] = []
|
|
1098
|
+
for subquery in _sql_derived_subqueries(sql).values():
|
|
1099
|
+
issues.extend(validate_sql_against_schema(subquery, schema))
|
|
561
1100
|
table_aliases = _sql_table_aliases(sql)
|
|
562
|
-
|
|
563
|
-
|
|
1101
|
+
derived_columns = _sql_derived_subquery_aliases(sql)
|
|
1102
|
+
schema_tables = _schema_table_column_map(schema)
|
|
1103
|
+
for alias, columns in derived_columns.items():
|
|
1104
|
+
schema_tables[alias] = columns
|
|
1105
|
+
for table in set(table_aliases.values()):
|
|
1106
|
+
if table in derived_columns:
|
|
1107
|
+
continue
|
|
564
1108
|
if table not in schema_tables:
|
|
565
1109
|
issues.append(
|
|
566
1110
|
SchemaValidationIssue(
|
|
@@ -568,118 +1112,1000 @@ def validate_sql_against_schema(sql: str, schema: dict[str, Any]) -> list[Schema
|
|
|
568
1112
|
message=f"SQL 使用了 schema 中不存在的表:{table}",
|
|
569
1113
|
)
|
|
570
1114
|
)
|
|
1115
|
+
issues.extend(_schema_missing_table_reference_issues(sql, table_aliases))
|
|
1116
|
+
issues.extend(_schema_unknown_column_issues(sql, table_aliases, schema_tables))
|
|
1117
|
+
for message in _list_join_on_violations(sql):
|
|
1118
|
+
issues.append(SchemaValidationIssue(code="JOIN_ON_FILTER", message=message))
|
|
1119
|
+
if not any(issue.code == "UNKNOWN_TABLE" for issue in issues):
|
|
1120
|
+
issues.extend(_schema_invalid_join_issues(sql, table_aliases, schema))
|
|
1121
|
+
return issues
|
|
1122
|
+
|
|
571
1123
|
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
1124
|
+
def _schema_table_column_map(schema: dict[str, Any]) -> dict[str, set[str]]:
|
|
1125
|
+
column_map: dict[str, set[str]] = {}
|
|
1126
|
+
tables = schema.get("tables")
|
|
1127
|
+
if not isinstance(tables, list):
|
|
1128
|
+
nested_schema = schema.get("schema")
|
|
1129
|
+
if isinstance(nested_schema, dict) and isinstance(nested_schema.get("tables"), list):
|
|
1130
|
+
tables = nested_schema["tables"]
|
|
1131
|
+
else:
|
|
1132
|
+
tables = []
|
|
1133
|
+
for table in tables:
|
|
1134
|
+
if not isinstance(table, dict):
|
|
1135
|
+
continue
|
|
1136
|
+
name = table.get("name") or table.get("tableName")
|
|
1137
|
+
if not name:
|
|
1138
|
+
continue
|
|
1139
|
+
table_key = str(name).lower()
|
|
1140
|
+
columns = table.get("columns")
|
|
1141
|
+
if not isinstance(columns, list):
|
|
1142
|
+
continue
|
|
1143
|
+
column_map[table_key] = {
|
|
1144
|
+
str(column.get("columnName") or column.get("name") or "").lower()
|
|
1145
|
+
for column in columns
|
|
1146
|
+
if isinstance(column, dict) and (column.get("columnName") or column.get("name"))
|
|
580
1147
|
}
|
|
581
|
-
|
|
1148
|
+
return column_map
|
|
1149
|
+
|
|
1150
|
+
|
|
1151
|
+
def _schema_missing_table_reference_issues(
|
|
1152
|
+
sql: str,
|
|
1153
|
+
table_aliases: dict[str, str],
|
|
1154
|
+
) -> list[SchemaValidationIssue]:
|
|
1155
|
+
declared = set(table_aliases)
|
|
1156
|
+
issues: list[SchemaValidationIssue] = []
|
|
1157
|
+
seen: set[tuple[str, str]] = set()
|
|
1158
|
+
for alias, column in _sql_qualified_column_refs(sql):
|
|
1159
|
+
if alias in declared:
|
|
1160
|
+
continue
|
|
1161
|
+
key = (alias, column)
|
|
1162
|
+
if key in seen:
|
|
1163
|
+
continue
|
|
1164
|
+
seen.add(key)
|
|
1165
|
+
issues.append(
|
|
1166
|
+
SchemaValidationIssue(
|
|
1167
|
+
code="MISSING_TABLE_REFERENCE",
|
|
1168
|
+
message=(
|
|
1169
|
+
"SQL 引用了未在 FROM/JOIN 中声明的表或别名:"
|
|
1170
|
+
f"{alias}.{column};如果需要该字段必须先 JOIN 对应表,否则删除该引用。"
|
|
1171
|
+
),
|
|
1172
|
+
)
|
|
1173
|
+
)
|
|
1174
|
+
return issues
|
|
1175
|
+
|
|
1176
|
+
|
|
1177
|
+
def _schema_unknown_column_issues(
|
|
1178
|
+
sql: str,
|
|
1179
|
+
table_aliases: dict[str, str],
|
|
1180
|
+
schema_tables: dict[str, set[str]],
|
|
1181
|
+
) -> list[SchemaValidationIssue]:
|
|
1182
|
+
issues: list[SchemaValidationIssue] = []
|
|
1183
|
+
seen: set[tuple[str, str]] = set()
|
|
1184
|
+
for alias, column in _sql_qualified_column_refs(sql):
|
|
1185
|
+
table = table_aliases.get(alias)
|
|
1186
|
+
if table is None or table not in schema_tables:
|
|
1187
|
+
continue
|
|
1188
|
+
key = (table, column)
|
|
1189
|
+
if key in seen:
|
|
1190
|
+
continue
|
|
1191
|
+
seen.add(key)
|
|
1192
|
+
if column not in schema_tables[table]:
|
|
582
1193
|
issues.append(
|
|
583
1194
|
SchemaValidationIssue(
|
|
584
|
-
code="
|
|
585
|
-
message="
|
|
1195
|
+
code="UNKNOWN_COLUMN",
|
|
1196
|
+
message=f"SQL 使用了 schema 中不存在的列:{table}.{column}",
|
|
586
1197
|
)
|
|
587
1198
|
)
|
|
588
1199
|
return issues
|
|
589
1200
|
|
|
590
1201
|
|
|
591
|
-
def
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
)
|
|
1202
|
+
def _schema_invalid_join_issues(
|
|
1203
|
+
sql: str,
|
|
1204
|
+
table_aliases: dict[str, str],
|
|
1205
|
+
schema: dict[str, Any],
|
|
1206
|
+
) -> list[SchemaValidationIssue]:
|
|
1207
|
+
allowed_pairs = _schema_allowed_join_pairs(schema)
|
|
1208
|
+
if not allowed_pairs:
|
|
1209
|
+
return []
|
|
1210
|
+
issues: list[SchemaValidationIssue] = []
|
|
1211
|
+
seen: set[tuple[tuple[str, str], tuple[str, str]]] = set()
|
|
1212
|
+
for pair in _sql_equality_pairs(sql, table_aliases):
|
|
1213
|
+
if pair in seen:
|
|
1214
|
+
continue
|
|
1215
|
+
seen.add(pair)
|
|
1216
|
+
if pair in allowed_pairs:
|
|
1217
|
+
continue
|
|
1218
|
+
left, right = pair
|
|
1219
|
+
issues.append(
|
|
1220
|
+
SchemaValidationIssue(
|
|
1221
|
+
code="INVALID_JOIN",
|
|
1222
|
+
message=(
|
|
1223
|
+
"SQL JOIN 使用了 schema relations / joinPattern 未定义的关联:"
|
|
1224
|
+
f"{left[0]}.{left[1]} = {right[0]}.{right[1]}"
|
|
1225
|
+
),
|
|
1226
|
+
)
|
|
1227
|
+
)
|
|
1228
|
+
return issues
|
|
1229
|
+
|
|
1230
|
+
|
|
1231
|
+
def _sql_qualified_column_refs(sql: str) -> list[tuple[str, str]]:
|
|
1232
|
+
refs: list[tuple[str, str]] = []
|
|
1233
|
+
seen: set[tuple[str, str]] = set()
|
|
1234
|
+
derived_spans = _sql_derived_subquery_spans(sql)
|
|
1235
|
+
for match in re.finditer(
|
|
1236
|
+
r"\b([a-zA-Z_][a-zA-Z0-9_]*)\.((?:[a-zA-Z_][a-zA-Z0-9_]*\.)*[a-zA-Z_][a-zA-Z0-9_]*)\b",
|
|
1237
|
+
sql,
|
|
1238
|
+
flags=re.IGNORECASE,
|
|
1239
|
+
):
|
|
1240
|
+
if _position_in_spans(match.start(), derived_spans):
|
|
1241
|
+
continue
|
|
1242
|
+
alias = match.group(1).lower()
|
|
1243
|
+
column_path = match.group(2).lower()
|
|
1244
|
+
column = column_path.rsplit(".", 1)[-1]
|
|
1245
|
+
key = (alias, column)
|
|
1246
|
+
if key in seen:
|
|
1247
|
+
continue
|
|
1248
|
+
seen.add(key)
|
|
1249
|
+
refs.append(key)
|
|
1250
|
+
return refs
|
|
1251
|
+
|
|
1252
|
+
|
|
1253
|
+
def _sql_table_aliases(sql: str) -> dict[str, str]:
|
|
1254
|
+
aliases = dict(_sql_top_level_table_aliases(sql))
|
|
1255
|
+
for alias in _sql_derived_subquery_aliases(sql):
|
|
1256
|
+
aliases[alias] = alias
|
|
1257
|
+
return aliases
|
|
1258
|
+
|
|
1259
|
+
|
|
1260
|
+
def _sql_top_level_table_aliases(sql: str) -> dict[str, str]:
|
|
1261
|
+
aliases: dict[str, str] = {}
|
|
1262
|
+
depth = 0
|
|
1263
|
+
quote: str | None = None
|
|
1264
|
+
index = 0
|
|
1265
|
+
length = len(sql)
|
|
1266
|
+
while index < length:
|
|
1267
|
+
char = sql[index]
|
|
1268
|
+
if quote is not None:
|
|
1269
|
+
if char == quote:
|
|
1270
|
+
quote = None
|
|
1271
|
+
elif char == "\\":
|
|
1272
|
+
index += 1
|
|
1273
|
+
index += 1
|
|
1274
|
+
continue
|
|
1275
|
+
if char in ("'", '"', "`"):
|
|
1276
|
+
quote = char
|
|
1277
|
+
index += 1
|
|
1278
|
+
continue
|
|
1279
|
+
if char == "(":
|
|
1280
|
+
depth += 1
|
|
1281
|
+
index += 1
|
|
1282
|
+
continue
|
|
1283
|
+
if char == ")":
|
|
1284
|
+
depth = max(0, depth - 1)
|
|
1285
|
+
index += 1
|
|
1286
|
+
continue
|
|
1287
|
+
if depth == 0:
|
|
1288
|
+
match = re.match(
|
|
1289
|
+
r"(?i)(?:from|join)\s+([a-zA-Z_][a-zA-Z0-9_]*)(?:\s+(?:as\s+)?([a-zA-Z_][a-zA-Z0-9_]*))?",
|
|
1290
|
+
sql[index:],
|
|
1291
|
+
)
|
|
1292
|
+
if match:
|
|
1293
|
+
table = match.group(1).lower()
|
|
1294
|
+
alias = (match.group(2) or table).lower()
|
|
1295
|
+
if alias in {"on", "where", "left", "right", "inner", "outer", "join"}:
|
|
1296
|
+
alias = table
|
|
1297
|
+
aliases[alias] = table
|
|
1298
|
+
aliases[table] = table
|
|
1299
|
+
index += match.end()
|
|
1300
|
+
continue
|
|
1301
|
+
index += 1
|
|
1302
|
+
return aliases
|
|
1303
|
+
|
|
1304
|
+
|
|
1305
|
+
def _sql_derived_subquery_aliases(sql: str) -> dict[str, set[str]]:
|
|
1306
|
+
return {
|
|
1307
|
+
alias: _extract_subquery_output_columns(subquery)
|
|
1308
|
+
for alias, subquery in _sql_derived_subqueries(sql).items()
|
|
1309
|
+
}
|
|
1310
|
+
|
|
1311
|
+
|
|
1312
|
+
def _sql_derived_subqueries(sql: str) -> dict[str, str]:
|
|
1313
|
+
derived: dict[str, str] = {}
|
|
1314
|
+
for alias, subquery_start, subquery_end in _sql_derived_subquery_matches(sql):
|
|
1315
|
+
derived[alias] = sql[subquery_start + 1 : subquery_end]
|
|
1316
|
+
return derived
|
|
1317
|
+
|
|
1318
|
+
|
|
1319
|
+
def _sql_derived_subquery_spans(sql: str) -> list[tuple[int, int]]:
|
|
1320
|
+
return [
|
|
1321
|
+
(subquery_start + 1, subquery_end)
|
|
1322
|
+
for _alias, subquery_start, subquery_end in _sql_derived_subquery_matches(sql)
|
|
1323
|
+
]
|
|
1324
|
+
|
|
1325
|
+
|
|
1326
|
+
def _sql_derived_subquery_matches(sql: str) -> list[tuple[str, int, int]]:
|
|
1327
|
+
matches: list[tuple[str, int, int]] = []
|
|
1328
|
+
for match in re.finditer(
|
|
1329
|
+
r"\)\s*(?:as\s+)?([a-zA-Z_][a-zA-Z0-9_]*)\s+on\b",
|
|
1330
|
+
sql,
|
|
1331
|
+
flags=re.IGNORECASE,
|
|
1332
|
+
):
|
|
1333
|
+
alias = match.group(1).lower()
|
|
1334
|
+
if alias in {"on", "where", "left", "right", "inner", "outer", "join", "select"}:
|
|
1335
|
+
continue
|
|
1336
|
+
subquery_start = _find_matching_open_paren(sql, match.start())
|
|
1337
|
+
if subquery_start is None:
|
|
1338
|
+
continue
|
|
1339
|
+
matches.append((alias, subquery_start, match.start()))
|
|
1340
|
+
return matches
|
|
1341
|
+
|
|
1342
|
+
|
|
1343
|
+
def _position_in_spans(position: int, spans: list[tuple[int, int]]) -> bool:
|
|
1344
|
+
return any(start <= position < end for start, end in spans)
|
|
1345
|
+
|
|
1346
|
+
|
|
1347
|
+
def _find_matching_open_paren(sql: str, close_index: int) -> int | None:
|
|
1348
|
+
depth = 0
|
|
1349
|
+
for index in range(close_index, -1, -1):
|
|
1350
|
+
char = sql[index]
|
|
1351
|
+
if char == ")":
|
|
1352
|
+
depth += 1
|
|
1353
|
+
elif char == "(":
|
|
1354
|
+
depth -= 1
|
|
1355
|
+
if depth == 0:
|
|
1356
|
+
return index
|
|
1357
|
+
return None
|
|
1358
|
+
|
|
1359
|
+
|
|
1360
|
+
def _extract_subquery_output_columns(subquery: str) -> set[str]:
|
|
1361
|
+
select_match = re.search(r"\bselect\b(.*?)\bfrom\b", subquery, flags=re.IGNORECASE | re.DOTALL)
|
|
1362
|
+
if select_match is None:
|
|
1363
|
+
return set()
|
|
1364
|
+
columns: set[str] = set()
|
|
1365
|
+
for expr in _split_select_items(select_match.group(1)):
|
|
1366
|
+
cleaned = expr.strip()
|
|
1367
|
+
if not cleaned:
|
|
1368
|
+
continue
|
|
1369
|
+
alias_match = re.search(r"\bas\s+([a-zA-Z_][a-zA-Z0-9_]*)\s*$", cleaned, flags=re.IGNORECASE)
|
|
1370
|
+
if alias_match:
|
|
1371
|
+
columns.add(alias_match.group(1).lower())
|
|
1372
|
+
continue
|
|
1373
|
+
qualified_match = re.match(
|
|
1374
|
+
r"^([a-zA-Z_][a-zA-Z0-9_]*)\.([a-zA-Z_][a-zA-Z0-9_]*)$",
|
|
1375
|
+
cleaned,
|
|
1376
|
+
flags=re.IGNORECASE,
|
|
1377
|
+
)
|
|
1378
|
+
if qualified_match:
|
|
1379
|
+
columns.add(qualified_match.group(2).lower())
|
|
1380
|
+
continue
|
|
1381
|
+
bare_match = re.match(r"^([a-zA-Z_][a-zA-Z0-9_]*)$", cleaned, flags=re.IGNORECASE)
|
|
1382
|
+
if bare_match:
|
|
1383
|
+
columns.add(bare_match.group(1).lower())
|
|
1384
|
+
return columns
|
|
1385
|
+
|
|
1386
|
+
|
|
1387
|
+
def _sql_equality_pairs(sql: str, table_aliases: dict[str, str]) -> set[tuple[tuple[str, str], tuple[str, str]]]:
|
|
1388
|
+
pairs: set[tuple[tuple[str, str], tuple[str, str]]] = set()
|
|
1389
|
+
derived_spans = _sql_derived_subquery_spans(sql)
|
|
1390
|
+
for match in re.finditer(
|
|
1391
|
+
r"\b([a-zA-Z_][a-zA-Z0-9_]*)\.([a-zA-Z_][a-zA-Z0-9_]*)\s*=\s*([a-zA-Z_][a-zA-Z0-9_]*)\.([a-zA-Z_][a-zA-Z0-9_]*)\b",
|
|
1392
|
+
sql,
|
|
1393
|
+
flags=re.IGNORECASE,
|
|
1394
|
+
):
|
|
1395
|
+
if _position_in_spans(match.start(), derived_spans):
|
|
1396
|
+
continue
|
|
1397
|
+
left_alias, left_column, right_alias, right_column = (part.lower() for part in match.groups())
|
|
1398
|
+
left_table = table_aliases.get(left_alias)
|
|
1399
|
+
right_table = table_aliases.get(right_alias)
|
|
1400
|
+
if left_table is None or right_table is None:
|
|
1401
|
+
continue
|
|
1402
|
+
pairs.add(_canonical_join_pair((left_table, left_column), (right_table, right_column)))
|
|
1403
|
+
return pairs
|
|
1404
|
+
|
|
1405
|
+
|
|
1406
|
+
def _schema_allowed_join_pairs(schema: dict[str, Any]) -> set[tuple[tuple[str, str], tuple[str, str]]]:
|
|
1407
|
+
pairs: set[tuple[tuple[str, str], tuple[str, str]]] = set()
|
|
1408
|
+
relations = _schema_relations(schema)
|
|
1409
|
+
if isinstance(relations, list):
|
|
1410
|
+
for relation in relations:
|
|
1411
|
+
if not isinstance(relation, dict):
|
|
1412
|
+
continue
|
|
1413
|
+
left_table, left_column = _relation_left(relation)
|
|
1414
|
+
right_table, right_column = _relation_right(relation)
|
|
1415
|
+
if left_table and left_column and right_table and right_column:
|
|
1416
|
+
pairs.add(_canonical_join_pair((left_table, left_column), (right_table, right_column)))
|
|
1417
|
+
for section in (_schema_metrics(schema), _schema_business_terms(schema), _schema_concepts(schema), _schema_examples(schema)):
|
|
1418
|
+
if isinstance(section, list):
|
|
1419
|
+
for item in section:
|
|
1420
|
+
if isinstance(item, dict):
|
|
1421
|
+
pairs.update(_join_pairs_from_text(str(item)))
|
|
1422
|
+
pairs.update(_schema_example_join_pattern_pairs(schema))
|
|
1423
|
+
pairs.update(_schema_work_order_failure_reason_join_pairs(schema))
|
|
1424
|
+
pairs.update(_schema_inferred_foreign_key_join_pairs(schema))
|
|
1425
|
+
return pairs
|
|
1426
|
+
|
|
1427
|
+
|
|
1428
|
+
def _schema_inferred_foreign_key_join_pairs(
|
|
1429
|
+
schema: dict[str, Any],
|
|
1430
|
+
) -> set[tuple[tuple[str, str], tuple[str, str]]]:
|
|
1431
|
+
"""Allow `{ref_table}_id` columns to join `{ref_table}.id` when no explicit relation exists."""
|
|
1432
|
+
pairs: set[tuple[tuple[str, str], tuple[str, str]]] = set()
|
|
1433
|
+
table_names = {
|
|
1434
|
+
str(table.get("tableName") or table.get("name") or "")
|
|
1435
|
+
for table in schema.get("tables") or []
|
|
1436
|
+
if isinstance(table, dict) and (table.get("tableName") or table.get("name"))
|
|
1437
|
+
}
|
|
1438
|
+
for table in schema.get("tables") or []:
|
|
1439
|
+
if not isinstance(table, dict):
|
|
1440
|
+
continue
|
|
1441
|
+
table_name = str(table.get("tableName") or table.get("name") or "")
|
|
1442
|
+
if not table_name:
|
|
1443
|
+
continue
|
|
1444
|
+
for column in table.get("columns") or []:
|
|
1445
|
+
if not isinstance(column, dict):
|
|
1446
|
+
continue
|
|
1447
|
+
column_name = str(column.get("columnName") or column.get("name") or "")
|
|
1448
|
+
if not column_name.endswith("_id") or column_name == "id":
|
|
1449
|
+
continue
|
|
1450
|
+
ref_table = column_name[:-3]
|
|
1451
|
+
if ref_table not in table_names:
|
|
1452
|
+
continue
|
|
1453
|
+
pairs.add(_canonical_join_pair((table_name, column_name), (ref_table, "id")))
|
|
1454
|
+
return pairs
|
|
1455
|
+
|
|
1456
|
+
|
|
1457
|
+
def _schema_work_order_failure_reason_join_pairs(
|
|
1458
|
+
schema: dict[str, Any],
|
|
1459
|
+
) -> set[tuple[tuple[str, str], tuple[str, str]]]:
|
|
1460
|
+
pairs: set[tuple[tuple[str, str], tuple[str, str]]] = set()
|
|
1461
|
+
mappings = _schema_structure(schema).get("workOrderFailureReasonMappings")
|
|
1462
|
+
if not isinstance(mappings, list):
|
|
1463
|
+
return pairs
|
|
1464
|
+
for mapping in mappings:
|
|
1465
|
+
if not isinstance(mapping, dict):
|
|
1466
|
+
continue
|
|
1467
|
+
reason_join = mapping.get("reasonJoin")
|
|
1468
|
+
if isinstance(reason_join, str) and reason_join.strip():
|
|
1469
|
+
pairs.update(_join_pairs_from_text(reason_join))
|
|
1470
|
+
unpivot = _schema_structure(schema).get("workOrderFailureReasonUnpivot")
|
|
1471
|
+
if isinstance(unpivot, dict):
|
|
1472
|
+
base_table = str(unpivot.get("baseTable") or "mvp_applied_jobs_by_helped_account")
|
|
1473
|
+
for branch in unpivot.get("branches") or []:
|
|
1474
|
+
if not isinstance(branch, dict):
|
|
1475
|
+
continue
|
|
1476
|
+
reason_field = str(branch.get("reasonField") or "")
|
|
1477
|
+
if reason_field:
|
|
1478
|
+
pairs.add(
|
|
1479
|
+
_canonical_join_pair(
|
|
1480
|
+
("failure_reasons", "id"),
|
|
1481
|
+
(base_table, reason_field),
|
|
1482
|
+
)
|
|
1483
|
+
)
|
|
1484
|
+
return pairs
|
|
1485
|
+
|
|
1486
|
+
|
|
1487
|
+
def _schema_example_join_pattern_pairs(schema: dict[str, Any]) -> set[tuple[tuple[str, str], tuple[str, str]]]:
|
|
1488
|
+
pairs: set[tuple[tuple[str, str], tuple[str, str]]] = set()
|
|
1489
|
+
example = schema.get("example")
|
|
1490
|
+
examples: list[Any] = []
|
|
1491
|
+
if isinstance(example, dict) and isinstance(example.get("examples"), list):
|
|
1492
|
+
examples = example["examples"]
|
|
1493
|
+
examples.extend(_schema_examples(schema))
|
|
1494
|
+
for item in examples:
|
|
1495
|
+
if not isinstance(item, dict):
|
|
1496
|
+
continue
|
|
1497
|
+
join_pattern = item.get("joinPattern")
|
|
1498
|
+
if isinstance(join_pattern, dict):
|
|
1499
|
+
pairs.update(_join_pattern_equality_pairs(join_pattern))
|
|
1500
|
+
return pairs
|
|
1501
|
+
|
|
1502
|
+
|
|
1503
|
+
def _join_pattern_equality_pairs(join_pattern: dict[str, Any]) -> set[tuple[tuple[str, str], tuple[str, str]]]:
|
|
1504
|
+
aliases = _parse_table_alias_clause(str(join_pattern.get("from") or ""))
|
|
1505
|
+
pairs: set[tuple[tuple[str, str], tuple[str, str]]] = set()
|
|
1506
|
+
joins = join_pattern.get("joins")
|
|
1507
|
+
if not isinstance(joins, list):
|
|
1508
|
+
return pairs
|
|
1509
|
+
for join in joins:
|
|
1510
|
+
if not isinstance(join, dict):
|
|
1511
|
+
continue
|
|
1512
|
+
aliases.update(_parse_table_alias_clause(str(join.get("table") or "")))
|
|
1513
|
+
on_conditions = join.get("on")
|
|
1514
|
+
if not isinstance(on_conditions, list):
|
|
1515
|
+
continue
|
|
1516
|
+
for condition in on_conditions:
|
|
1517
|
+
pairs.update(_join_pairs_from_on_text(str(condition), aliases))
|
|
1518
|
+
return pairs
|
|
1519
|
+
|
|
1520
|
+
|
|
1521
|
+
def _parse_table_alias_clause(clause: str) -> dict[str, str]:
|
|
1522
|
+
cleaned = clause.strip()
|
|
1523
|
+
if not cleaned:
|
|
1524
|
+
return {}
|
|
1525
|
+
match = re.match(
|
|
1526
|
+
r"^([a-zA-Z_][a-zA-Z0-9_]*)(?:\s+(?:as\s+)?([a-zA-Z_][a-zA-Z0-9_]*))?$",
|
|
1527
|
+
cleaned,
|
|
1528
|
+
flags=re.IGNORECASE,
|
|
1529
|
+
)
|
|
1530
|
+
if match is None:
|
|
1531
|
+
return {}
|
|
1532
|
+
table = match.group(1).lower()
|
|
1533
|
+
alias = (match.group(2) or table).lower()
|
|
1534
|
+
return {alias: table, table: table}
|
|
1535
|
+
|
|
1536
|
+
|
|
1537
|
+
def _join_pairs_from_on_text(
|
|
1538
|
+
on: str,
|
|
1539
|
+
aliases: dict[str, str],
|
|
1540
|
+
) -> set[tuple[tuple[str, str], tuple[str, str]]]:
|
|
1541
|
+
pairs: set[tuple[tuple[str, str], tuple[str, str]]] = set()
|
|
1542
|
+
for match in re.finditer(
|
|
1543
|
+
r"\b([a-zA-Z_][a-zA-Z0-9_]*)\.([a-zA-Z_][a-zA-Z0-9_]*)\s*=\s*([a-zA-Z_][a-zA-Z0-9_]*)\.([a-zA-Z_][a-zA-Z0-9_]*)\b",
|
|
1544
|
+
on,
|
|
1545
|
+
flags=re.IGNORECASE,
|
|
1546
|
+
):
|
|
1547
|
+
left_alias, left_column, right_alias, right_column = (part.lower() for part in match.groups())
|
|
1548
|
+
left_table = aliases.get(left_alias)
|
|
1549
|
+
right_table = aliases.get(right_alias)
|
|
1550
|
+
if left_table is None or right_table is None:
|
|
1551
|
+
continue
|
|
1552
|
+
pairs.add(_canonical_join_pair((left_table, left_column), (right_table, right_column)))
|
|
1553
|
+
return pairs
|
|
1554
|
+
|
|
1555
|
+
|
|
1556
|
+
def _schema_structure(schema: dict[str, Any]) -> dict[str, Any]:
|
|
1557
|
+
structure = schema.get("structure")
|
|
1558
|
+
return structure if isinstance(structure, dict) else {}
|
|
1559
|
+
|
|
1560
|
+
|
|
1561
|
+
def _schema_relations(schema: dict[str, Any]) -> list[dict[str, Any]]:
|
|
1562
|
+
relations = _schema_structure(schema).get("relations")
|
|
1563
|
+
if isinstance(relations, list):
|
|
1564
|
+
return [relation for relation in relations if isinstance(relation, dict)]
|
|
1565
|
+
joins = schema.get("joins")
|
|
1566
|
+
if isinstance(joins, list):
|
|
1567
|
+
return [join for join in joins if isinstance(join, dict)]
|
|
1568
|
+
return []
|
|
1569
|
+
|
|
1570
|
+
|
|
1571
|
+
def _schema_metrics(schema: dict[str, Any]) -> list[dict[str, Any]]:
|
|
1572
|
+
metrics = _schema_structure(schema).get("metrics")
|
|
1573
|
+
if isinstance(metrics, list):
|
|
1574
|
+
return [metric for metric in metrics if isinstance(metric, dict)]
|
|
1575
|
+
legacy_metrics = schema.get("metrics")
|
|
1576
|
+
if isinstance(legacy_metrics, list):
|
|
1577
|
+
return [metric for metric in legacy_metrics if isinstance(metric, dict)]
|
|
1578
|
+
return []
|
|
1579
|
+
|
|
1580
|
+
|
|
1581
|
+
def _schema_business_terms(schema: dict[str, Any]) -> list[dict[str, Any]]:
|
|
1582
|
+
business_terms = _schema_structure(schema).get("businessTerms")
|
|
1583
|
+
if isinstance(business_terms, list):
|
|
1584
|
+
return [term for term in business_terms if isinstance(term, dict)]
|
|
1585
|
+
legacy_glossary = schema.get("glossary")
|
|
1586
|
+
if isinstance(legacy_glossary, list):
|
|
1587
|
+
return [term for term in legacy_glossary if isinstance(term, dict)]
|
|
1588
|
+
return []
|
|
1589
|
+
|
|
1590
|
+
|
|
1591
|
+
def _schema_concepts(schema: dict[str, Any]) -> list[dict[str, Any]]:
|
|
1592
|
+
concepts = _schema_structure(schema).get("concepts")
|
|
1593
|
+
if isinstance(concepts, list):
|
|
1594
|
+
return [concept for concept in concepts if isinstance(concept, dict)]
|
|
1595
|
+
return []
|
|
1596
|
+
|
|
1597
|
+
|
|
1598
|
+
def _schema_examples(schema: dict[str, Any]) -> list[dict[str, Any]]:
|
|
1599
|
+
example = schema.get("example")
|
|
1600
|
+
if isinstance(example, dict) and isinstance(example.get("examples"), list):
|
|
1601
|
+
return [item for item in example["examples"] if isinstance(item, dict)]
|
|
1602
|
+
legacy_examples = schema.get("examples")
|
|
1603
|
+
if isinstance(legacy_examples, list):
|
|
1604
|
+
return [item for item in legacy_examples if isinstance(item, dict)]
|
|
1605
|
+
return []
|
|
1606
|
+
|
|
1607
|
+
|
|
1608
|
+
def _relation_left(relation: dict[str, Any]) -> tuple[str, str]:
|
|
1609
|
+
table = str(relation.get("leftTable") or "").lower()
|
|
1610
|
+
column = str(relation.get("leftColumn") or relation.get("fromColumn") or "").lower()
|
|
1611
|
+
if table:
|
|
1612
|
+
return table, column
|
|
1613
|
+
return _split_field_ref(str(relation.get("from") or "").lower())
|
|
1614
|
+
|
|
1615
|
+
|
|
1616
|
+
def _relation_right(relation: dict[str, Any]) -> tuple[str, str]:
|
|
1617
|
+
table = str(relation.get("rightTable") or "").lower()
|
|
1618
|
+
column = str(relation.get("rightColumn") or relation.get("toColumn") or "").lower()
|
|
1619
|
+
if table:
|
|
1620
|
+
return table, column
|
|
1621
|
+
return _split_field_ref(str(relation.get("to") or "").lower())
|
|
1622
|
+
|
|
1623
|
+
|
|
1624
|
+
def _split_field_ref(value: str) -> tuple[str, str]:
|
|
1625
|
+
if "." not in value:
|
|
1626
|
+
return value, ""
|
|
1627
|
+
return tuple(value.split(".", 1)) # type: ignore[return-value]
|
|
1628
|
+
|
|
1629
|
+
|
|
1630
|
+
def _join_pairs_from_text(text: str) -> set[tuple[tuple[str, str], tuple[str, str]]]:
|
|
1631
|
+
pairs: set[tuple[tuple[str, str], tuple[str, str]]] = set()
|
|
1632
|
+
for match in re.finditer(
|
|
1633
|
+
r"\b([a-zA-Z_][a-zA-Z0-9_]*)\.([a-zA-Z_][a-zA-Z0-9_]*)\s*=\s*([a-zA-Z_][a-zA-Z0-9_]*)\.([a-zA-Z_][a-zA-Z0-9_]*)\b",
|
|
1634
|
+
text,
|
|
1635
|
+
flags=re.IGNORECASE,
|
|
1636
|
+
):
|
|
1637
|
+
left_table, left_column, right_table, right_column = (part.lower() for part in match.groups())
|
|
1638
|
+
pairs.add(_canonical_join_pair((left_table, left_column), (right_table, right_column)))
|
|
1639
|
+
return pairs
|
|
1640
|
+
|
|
1641
|
+
|
|
1642
|
+
def _canonical_join_pair(
|
|
1643
|
+
left: tuple[str, str],
|
|
1644
|
+
right: tuple[str, str],
|
|
1645
|
+
) -> tuple[tuple[str, str], tuple[str, str]]:
|
|
1646
|
+
return tuple(sorted((left, right))) # type: ignore[return-value]
|
|
1647
|
+
|
|
1648
|
+
|
|
1649
|
+
def question_has_time_range(question: str) -> bool:
|
|
1650
|
+
return _extract_time_range(question) is not None
|
|
1651
|
+
|
|
1652
|
+
|
|
1653
|
+
def extract_time_range(question: str, schema: dict[str, Any] | None = None) -> TimeRange | None:
|
|
1654
|
+
"""Parse a question time window; month/day without year defaults to the current year."""
|
|
1655
|
+
return _extract_time_range(question, schema)
|
|
1656
|
+
|
|
1657
|
+
|
|
1658
|
+
def time_range_from_query_plan(query_plan: dict[str, Any] | None) -> TimeRange | None:
|
|
1659
|
+
"""Extract a confirmed time window from query-plan filters."""
|
|
1660
|
+
if not isinstance(query_plan, dict):
|
|
1661
|
+
return None
|
|
1662
|
+
start_value = ""
|
|
1663
|
+
end_value = ""
|
|
1664
|
+
field_name = ""
|
|
1665
|
+
for flt in query_plan.get("filters") or []:
|
|
1666
|
+
if not isinstance(flt, dict):
|
|
1667
|
+
continue
|
|
1668
|
+
if str(flt.get("source") or "") != "question.timeRange":
|
|
1669
|
+
continue
|
|
1670
|
+
operator = str(flt.get("operator") or "").strip()
|
|
1671
|
+
value = str(flt.get("value") or "").strip()
|
|
1672
|
+
field = str(flt.get("field") or "").strip()
|
|
1673
|
+
if operator == ">=":
|
|
1674
|
+
start_value = value
|
|
1675
|
+
field_name = field or field_name
|
|
1676
|
+
elif operator == "<":
|
|
1677
|
+
end_value = value
|
|
1678
|
+
field_name = field or field_name
|
|
1679
|
+
if not start_value or not end_value or not field_name:
|
|
1680
|
+
return None
|
|
1681
|
+
table_name, column = field_name.split(".", 1) if "." in field_name else ("", field_name)
|
|
1682
|
+
return TimeRange(
|
|
1683
|
+
field=column,
|
|
1684
|
+
alias="时间范围",
|
|
1685
|
+
start=start_value,
|
|
1686
|
+
end=end_value,
|
|
1687
|
+
table_name=table_name,
|
|
1688
|
+
)
|
|
1689
|
+
|
|
1690
|
+
|
|
1691
|
+
def resolve_time_range(
|
|
1692
|
+
question: str,
|
|
1693
|
+
schema: dict[str, Any] | None = None,
|
|
1694
|
+
*,
|
|
1695
|
+
query_plan: dict[str, Any] | None = None,
|
|
1696
|
+
) -> TimeRange | None:
|
|
1697
|
+
plan_range = time_range_from_query_plan(query_plan)
|
|
1698
|
+
if plan_range is not None:
|
|
1699
|
+
return plan_range
|
|
1700
|
+
return extract_time_range(question, schema)
|
|
1701
|
+
|
|
1702
|
+
|
|
1703
|
+
def time_range_plan_filters(
|
|
1704
|
+
question: str,
|
|
1705
|
+
schema: dict[str, Any] | None,
|
|
1706
|
+
*,
|
|
1707
|
+
concepts: list[dict[str, Any]] | None = None,
|
|
1708
|
+
) -> list[dict[str, str]]:
|
|
1709
|
+
"""Build query-plan filters for an extracted time range."""
|
|
1710
|
+
time_range = resolve_time_range_for_question(question, schema, concepts=concepts)
|
|
1711
|
+
if time_range is None:
|
|
1712
|
+
return []
|
|
1713
|
+
qualified_field = (
|
|
1714
|
+
f"{time_range.table_name}.{time_range.field}"
|
|
1715
|
+
if time_range.table_name
|
|
1716
|
+
else time_range.field
|
|
1717
|
+
)
|
|
1718
|
+
label = time_range.alias or "时间范围"
|
|
1719
|
+
return [
|
|
1720
|
+
{
|
|
1721
|
+
"field": qualified_field,
|
|
1722
|
+
"operator": ">=",
|
|
1723
|
+
"value": time_range.start[:10],
|
|
1724
|
+
"description": f"{label}起",
|
|
1725
|
+
"source": "question.timeRange",
|
|
1726
|
+
},
|
|
1727
|
+
{
|
|
1728
|
+
"field": qualified_field,
|
|
1729
|
+
"operator": "<",
|
|
1730
|
+
"value": time_range.end[:10],
|
|
1731
|
+
"description": f"{label}止",
|
|
1732
|
+
"source": "question.timeRange",
|
|
1733
|
+
},
|
|
1734
|
+
]
|
|
1735
|
+
|
|
1736
|
+
|
|
1737
|
+
_FILLED_JOB_PERIOD_CONCEPT = "filled_job_by_period"
|
|
1738
|
+
|
|
1739
|
+
|
|
1740
|
+
def _query_plan_has_matched_concept(query_plan: dict[str, Any], concept_name: str) -> bool:
|
|
1741
|
+
for concept in query_plan.get("matchedConcepts") or []:
|
|
1742
|
+
if isinstance(concept, dict) and str(concept.get("name") or "") == concept_name:
|
|
1743
|
+
return True
|
|
1744
|
+
return False
|
|
1745
|
+
|
|
1746
|
+
|
|
1747
|
+
def _uses_subquery_period_time_range(sql: str, query_plan: dict[str, Any] | None) -> bool:
|
|
1748
|
+
if isinstance(query_plan, dict) and _query_plan_has_matched_concept(
|
|
1749
|
+
query_plan,
|
|
1750
|
+
_FILLED_JOB_PERIOD_CONCEPT,
|
|
1751
|
+
):
|
|
1752
|
+
return True
|
|
1753
|
+
return "period_active_apply_count" in sql.lower()
|
|
1754
|
+
|
|
1755
|
+
|
|
1756
|
+
def _period_time_bounds(time_range: TimeRange) -> tuple[str, str]:
|
|
1757
|
+
start = time_range.start
|
|
1758
|
+
end = time_range.end
|
|
1759
|
+
if len(start) <= 10:
|
|
1760
|
+
start = f"{start[:10]} 00:00:00"
|
|
1761
|
+
if len(end) <= 10:
|
|
1762
|
+
end = f"{end[:10]} 00:00:00"
|
|
1763
|
+
return start, end
|
|
1764
|
+
|
|
1765
|
+
|
|
1766
|
+
def _apply_subquery_period_time_range(sql: str, time_range: TimeRange) -> str:
|
|
1767
|
+
"""Apply confirmed time bounds inside apply_stat CASE WHEN, not outer WHERE."""
|
|
1768
|
+
stripped = _remove_time_range_filters_from_sql(
|
|
1769
|
+
sql.rstrip().rstrip(";"),
|
|
1770
|
+
time_range,
|
|
1771
|
+
allow_any_alias=True,
|
|
1772
|
+
)
|
|
1773
|
+
period_start, period_end = _period_time_bounds(time_range)
|
|
1774
|
+
result = stripped
|
|
1775
|
+
for old, new in (
|
|
1776
|
+
("'<period_start>'", f"'{period_start}'"),
|
|
1777
|
+
("'<period_end>'", f"'{period_end}'"),
|
|
1778
|
+
("<period_start>", f"'{period_start}'"),
|
|
1779
|
+
("<period_end>", f"'{period_end}'"),
|
|
1780
|
+
):
|
|
1781
|
+
result = result.replace(old, new)
|
|
1782
|
+
field_pattern = rf"(?:\w+\.)?{re.escape(time_range.field)}"
|
|
1783
|
+
time_expr = (
|
|
1784
|
+
r"(?:date_add\s*\(\s*curdate\s*\(\s*\)\s*,\s*interval\s+\d+\s+day\s*\)"
|
|
1785
|
+
r"|curdate\s*\(\s*\)|current_date(?:\s*\(\s*\))?|'[^']+')"
|
|
1786
|
+
)
|
|
1787
|
+
return re.sub(
|
|
1788
|
+
rf"(?is)(case\s+when\s+{field_pattern}\s*>=\s*){time_expr}(\s+and\s+{field_pattern}\s*<\s*){time_expr}(\s+then)",
|
|
1789
|
+
lambda match: f"{match.group(1)}'{period_start}'{match.group(2)}'{period_end}'{match.group(3)}",
|
|
1790
|
+
result,
|
|
1791
|
+
)
|
|
1792
|
+
|
|
1793
|
+
|
|
1794
|
+
def apply_time_range_filters_from_plan(sql: str, query_plan: dict[str, Any] | None) -> str:
|
|
1795
|
+
"""Force generated SQL to use the confirmed query-plan time filters."""
|
|
1796
|
+
if not isinstance(query_plan, dict):
|
|
1797
|
+
return sql
|
|
1798
|
+
time_range = time_range_from_query_plan(query_plan)
|
|
1799
|
+
if time_range is None:
|
|
1800
|
+
return sql
|
|
1801
|
+
stripped = _remove_time_range_filters_from_sql(sql.rstrip().rstrip(";"), time_range)
|
|
1802
|
+
if _uses_subquery_period_time_range(stripped, query_plan):
|
|
1803
|
+
return _apply_subquery_period_time_range(stripped, time_range)
|
|
1804
|
+
qualified_field = _time_range_sql_qualified_field(stripped, time_range)
|
|
1805
|
+
extra = (
|
|
1806
|
+
f"{qualified_field} >= '{time_range.start[:10]}' "
|
|
1807
|
+
f"AND {qualified_field} < '{time_range.end[:10]}'"
|
|
1808
|
+
)
|
|
1809
|
+
insert_pos = _where_filter_insert_pos(stripped)
|
|
1810
|
+
head = stripped[:insert_pos].rstrip()
|
|
1811
|
+
tail = stripped[insert_pos:].lstrip()
|
|
1812
|
+
if _find_top_level_keyword(head, "where", start=0) is not None:
|
|
1813
|
+
return f"{head} AND {extra} {tail}".strip()
|
|
1814
|
+
return f"{head} WHERE {extra} {tail}".strip()
|
|
1815
|
+
|
|
1816
|
+
|
|
1817
|
+
def apply_time_range_filters_from_question(
|
|
1818
|
+
sql: str,
|
|
1819
|
+
question: str,
|
|
1820
|
+
schema: dict[str, Any] | None,
|
|
1821
|
+
) -> str:
|
|
1822
|
+
"""Force a question-level time range into SQL when queryPlan missed it."""
|
|
1823
|
+
time_range = resolve_time_range_for_sql(question, schema, sql)
|
|
1824
|
+
if time_range is None:
|
|
1825
|
+
return sql
|
|
1826
|
+
query_plan = {
|
|
1827
|
+
"filters": [
|
|
1828
|
+
{
|
|
1829
|
+
"field": _qualified_time_range_field(time_range),
|
|
1830
|
+
"operator": ">=",
|
|
1831
|
+
"value": time_range.start[:10],
|
|
1832
|
+
"source": "question.timeRange",
|
|
1833
|
+
},
|
|
1834
|
+
{
|
|
1835
|
+
"field": _qualified_time_range_field(time_range),
|
|
1836
|
+
"operator": "<",
|
|
1837
|
+
"value": time_range.end[:10],
|
|
1838
|
+
"source": "question.timeRange",
|
|
1839
|
+
},
|
|
1840
|
+
]
|
|
1841
|
+
}
|
|
1842
|
+
return apply_time_range_filters_from_plan(sql, query_plan)
|
|
1843
|
+
|
|
1844
|
+
|
|
1845
|
+
def resolve_time_range_for_sql(
|
|
1846
|
+
question: str,
|
|
1847
|
+
schema: dict[str, Any] | None,
|
|
1848
|
+
sql: str,
|
|
1849
|
+
) -> TimeRange | None:
|
|
1850
|
+
if not schema:
|
|
1851
|
+
return None
|
|
1852
|
+
from .schema_context_retriever import match_concepts_for_question
|
|
1853
|
+
|
|
1854
|
+
concepts = match_concepts_for_question(_schema_concepts(schema), question)
|
|
1855
|
+
sql_schema = _schema_limited_to_sql_tables(schema, sql)
|
|
1856
|
+
return resolve_time_range_for_question(question, sql_schema or schema, concepts=concepts)
|
|
1857
|
+
|
|
1858
|
+
|
|
1859
|
+
def _qualified_time_range_field(time_range: TimeRange) -> str:
|
|
1860
|
+
if time_range.table_name:
|
|
1861
|
+
return f"{time_range.table_name}.{time_range.field}"
|
|
1862
|
+
return time_range.field
|
|
1863
|
+
|
|
1864
|
+
|
|
1865
|
+
def _where_filter_insert_pos(sql: str) -> int:
|
|
1866
|
+
positions = [
|
|
1867
|
+
pos
|
|
1868
|
+
for keyword in ("group by", "order by", "limit")
|
|
1869
|
+
if (pos := _find_top_level_keyword(sql, keyword, start=0)) is not None
|
|
1870
|
+
]
|
|
1871
|
+
return min(positions) if positions else len(sql)
|
|
1872
|
+
|
|
1873
|
+
|
|
1874
|
+
def _remove_time_range_filters_from_sql(
|
|
1875
|
+
sql: str,
|
|
1876
|
+
time_range: TimeRange,
|
|
1877
|
+
*,
|
|
1878
|
+
allow_any_alias: bool = False,
|
|
1879
|
+
) -> str:
|
|
1880
|
+
where_index = _find_top_level_keyword(sql, "where", start=0)
|
|
1881
|
+
if where_index is None:
|
|
1882
|
+
return sql
|
|
1883
|
+
clause_start = where_index + len("where")
|
|
1884
|
+
clause_end = _where_filter_insert_pos(sql[clause_start:])
|
|
1885
|
+
if clause_end < len(sql[clause_start:]):
|
|
1886
|
+
clause_end += clause_start
|
|
1887
|
+
else:
|
|
1888
|
+
clause_end = len(sql)
|
|
1889
|
+
where_body = sql[clause_start:clause_end].strip()
|
|
1890
|
+
if not where_body:
|
|
1891
|
+
return sql
|
|
1892
|
+
predicates = _split_top_level_and_predicates(where_body)
|
|
1893
|
+
kept = [
|
|
1894
|
+
predicate.strip()
|
|
1895
|
+
for predicate in predicates
|
|
1896
|
+
if predicate.strip()
|
|
1897
|
+
and not _is_time_range_predicate(
|
|
1898
|
+
predicate,
|
|
1899
|
+
sql=sql,
|
|
1900
|
+
time_range=time_range,
|
|
1901
|
+
allow_any_alias=allow_any_alias,
|
|
1902
|
+
)
|
|
1903
|
+
]
|
|
1904
|
+
prefix = sql[:where_index].rstrip()
|
|
1905
|
+
suffix = sql[clause_end:].lstrip()
|
|
1906
|
+
if not kept:
|
|
1907
|
+
return f"{prefix} {suffix}".strip()
|
|
1908
|
+
return f"{prefix} WHERE {' AND '.join(kept)} {suffix}".strip()
|
|
1909
|
+
|
|
1910
|
+
|
|
1911
|
+
def _split_top_level_and_predicates(where_body: str) -> list[str]:
|
|
1912
|
+
predicates: list[str] = []
|
|
1913
|
+
depth = 0
|
|
1914
|
+
quote: str | None = None
|
|
1915
|
+
start = 0
|
|
1916
|
+
index = 0
|
|
1917
|
+
while index < len(where_body):
|
|
1918
|
+
char = where_body[index]
|
|
1919
|
+
if quote is not None:
|
|
1920
|
+
if char == quote:
|
|
1921
|
+
quote = None
|
|
1922
|
+
elif char == "\\":
|
|
1923
|
+
index += 1
|
|
1924
|
+
index += 1
|
|
1925
|
+
continue
|
|
1926
|
+
if char in ("'", '"', "`"):
|
|
1927
|
+
quote = char
|
|
1928
|
+
index += 1
|
|
1929
|
+
continue
|
|
1930
|
+
if char == "(":
|
|
1931
|
+
depth += 1
|
|
1932
|
+
index += 1
|
|
1933
|
+
continue
|
|
1934
|
+
if char == ")":
|
|
1935
|
+
depth = max(0, depth - 1)
|
|
1936
|
+
index += 1
|
|
1937
|
+
continue
|
|
1938
|
+
if depth == 0 and where_body[index : index + 3].lower() == "and":
|
|
1939
|
+
before = where_body[index - 1] if index > 0 else " "
|
|
1940
|
+
after_index = index + 3
|
|
1941
|
+
after = where_body[after_index] if after_index < len(where_body) else " "
|
|
1942
|
+
if not (before.isalnum() or before == "_") and not (after.isalnum() or after == "_"):
|
|
1943
|
+
predicates.append(where_body[start:index])
|
|
1944
|
+
start = after_index
|
|
1945
|
+
index = after_index
|
|
1946
|
+
continue
|
|
1947
|
+
index += 1
|
|
1948
|
+
predicates.append(where_body[start:])
|
|
1949
|
+
return predicates
|
|
1950
|
+
|
|
1951
|
+
|
|
1952
|
+
def _is_time_range_predicate(
|
|
1953
|
+
predicate: str,
|
|
1954
|
+
*,
|
|
1955
|
+
sql: str,
|
|
1956
|
+
time_range: TimeRange,
|
|
1957
|
+
allow_any_alias: bool = False,
|
|
1958
|
+
) -> bool:
|
|
1959
|
+
field_pattern = _time_range_sql_field_pattern(sql, time_range, allow_any_alias=allow_any_alias)
|
|
1960
|
+
if re.search(field_pattern, predicate, flags=re.IGNORECASE) is None:
|
|
1961
|
+
return False
|
|
1962
|
+
if re.search(r"'20\d{2}-\d{1,2}-\d{1,2}(?:\s+\d{1,2}:\d{2}:\d{2})?'", predicate) is None:
|
|
1963
|
+
return False
|
|
1964
|
+
return bool(re.search(r"(?:\bbetween\b|>=|>|<=|<|=)", predicate, flags=re.IGNORECASE))
|
|
602
1965
|
|
|
603
1966
|
|
|
604
|
-
def
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
)
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
1967
|
+
def _time_range_sql_field_pattern(
|
|
1968
|
+
sql: str,
|
|
1969
|
+
time_range: TimeRange,
|
|
1970
|
+
*,
|
|
1971
|
+
allow_any_alias: bool = False,
|
|
1972
|
+
) -> str:
|
|
1973
|
+
field = re.escape(time_range.field.lower())
|
|
1974
|
+
if allow_any_alias or not time_range.table_name:
|
|
1975
|
+
return rf"(?:\b\w+\.)?{field}\b"
|
|
1976
|
+
aliases = [
|
|
1977
|
+
re.escape(alias)
|
|
1978
|
+
for alias, table in _sql_table_aliases(sql).items()
|
|
1979
|
+
if table == time_range.table_name.lower()
|
|
1980
|
+
]
|
|
1981
|
+
aliases.append(re.escape(time_range.table_name.lower()))
|
|
1982
|
+
qualifier = "|".join(sorted(set(aliases), key=len, reverse=True))
|
|
1983
|
+
return rf"\b(?:{qualifier})\.{field}\b"
|
|
1984
|
+
|
|
1985
|
+
|
|
1986
|
+
def _time_range_sql_qualified_field(sql: str, time_range: TimeRange) -> str:
|
|
1987
|
+
if not time_range.table_name:
|
|
1988
|
+
return time_range.field
|
|
1989
|
+
aliases = _sql_top_level_table_aliases(sql)
|
|
1990
|
+
alias_candidates = [
|
|
1991
|
+
alias
|
|
1992
|
+
for alias, table in aliases.items()
|
|
1993
|
+
if table == time_range.table_name.lower() and alias != time_range.table_name.lower()
|
|
1994
|
+
]
|
|
1995
|
+
if alias_candidates:
|
|
1996
|
+
return f"{sorted(alias_candidates, key=len)[0]}.{time_range.field}"
|
|
1997
|
+
return f"{time_range.table_name}.{time_range.field}"
|
|
618
1998
|
|
|
619
1999
|
|
|
620
|
-
def
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
right_table = table_aliases.get(right_alias)
|
|
630
|
-
if left_table is None or right_table is None:
|
|
2000
|
+
def sql_missing_query_plan_filters(sql: str, query_plan: dict[str, Any] | None) -> list[str]:
|
|
2001
|
+
"""Return table.column LIKE filters from query_plan that are absent in sql."""
|
|
2002
|
+
if not isinstance(query_plan, dict):
|
|
2003
|
+
return []
|
|
2004
|
+
missing: list[str] = []
|
|
2005
|
+
normalized_sql = sql.lower()
|
|
2006
|
+
grouped_filters: dict[str, list[dict[str, Any]]] = {}
|
|
2007
|
+
for flt in query_plan.get("filters") or []:
|
|
2008
|
+
if not isinstance(flt, dict):
|
|
631
2009
|
continue
|
|
632
|
-
|
|
633
|
-
|
|
2010
|
+
if str(flt.get("operator") or "").upper() != "LIKE":
|
|
2011
|
+
continue
|
|
2012
|
+
logical_group = str(flt.get("logicalGroup") or "").strip()
|
|
2013
|
+
if logical_group:
|
|
2014
|
+
grouped_filters.setdefault(logical_group, []).append(flt)
|
|
2015
|
+
field = str(flt.get("field") or "").strip()
|
|
2016
|
+
if "." not in field:
|
|
2017
|
+
continue
|
|
2018
|
+
table, column = field.split(".", 1)
|
|
2019
|
+
value = str(flt.get("value") or "").strip().strip("%")
|
|
2020
|
+
if not value:
|
|
2021
|
+
continue
|
|
2022
|
+
table_l = table.lower()
|
|
2023
|
+
column_l = column.lower()
|
|
2024
|
+
table_present = bool(re.search(rf"\b{re.escape(table_l)}\b", normalized_sql))
|
|
2025
|
+
column_present = bool(re.search(rf"\b{re.escape(column_l)}\b", normalized_sql))
|
|
2026
|
+
value_present = value.lower() in normalized_sql
|
|
2027
|
+
if not table_present or not column_present or not value_present:
|
|
2028
|
+
missing.append(f"{field} LIKE {flt.get('value')}")
|
|
2029
|
+
for group_name, filters in grouped_filters.items():
|
|
2030
|
+
if len(filters) <= 1:
|
|
2031
|
+
continue
|
|
2032
|
+
group_missing = [
|
|
2033
|
+
f"{flt.get('field')} LIKE {flt.get('value')}"
|
|
2034
|
+
for flt in filters
|
|
2035
|
+
if f"{flt.get('field')} LIKE {flt.get('value')}" in missing
|
|
2036
|
+
]
|
|
2037
|
+
if group_missing:
|
|
2038
|
+
continue
|
|
2039
|
+
if not _or_group_filters_connected_by_or(normalized_sql, filters):
|
|
2040
|
+
missing.append(f"{group_name} requires OR between matched entities")
|
|
2041
|
+
return missing
|
|
634
2042
|
|
|
635
2043
|
|
|
636
|
-
def
|
|
637
|
-
|
|
638
|
-
joins = schema.get("joins", [])
|
|
639
|
-
if isinstance(joins, list):
|
|
640
|
-
for join in joins:
|
|
641
|
-
if not isinstance(join, dict):
|
|
642
|
-
continue
|
|
643
|
-
left_table = str(join.get("leftTable") or join.get("from") or "").lower()
|
|
644
|
-
left_column = str(join.get("leftColumn") or join.get("fromColumn") or "").lower()
|
|
645
|
-
right_table = str(join.get("rightTable") or join.get("to") or "").lower()
|
|
646
|
-
right_column = str(join.get("rightColumn") or join.get("toColumn") or "").lower()
|
|
647
|
-
if left_table and left_column and right_table and right_column:
|
|
648
|
-
pairs.add(_canonical_join_pair((left_table, left_column), (right_table, right_column)))
|
|
649
|
-
for section_name in ("metrics", "glossary", "examples"):
|
|
650
|
-
section = schema.get(section_name, [])
|
|
651
|
-
if isinstance(section, list):
|
|
652
|
-
for item in section:
|
|
653
|
-
if isinstance(item, dict):
|
|
654
|
-
pairs.update(_join_pairs_from_text(str(item)))
|
|
655
|
-
return pairs
|
|
2044
|
+
def sql_satisfies_query_plan_filters(sql: str, query_plan: dict[str, Any] | None) -> bool:
|
|
2045
|
+
return not sql_missing_query_plan_filters(sql, query_plan)
|
|
656
2046
|
|
|
657
2047
|
|
|
658
|
-
def
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
2048
|
+
def _or_group_filters_connected_by_or(normalized_sql: str, filters: list[dict[str, Any]]) -> bool:
|
|
2049
|
+
if not re.search(r"\bor\b", normalized_sql):
|
|
2050
|
+
return False
|
|
2051
|
+
positions: list[int] = []
|
|
2052
|
+
for flt in filters:
|
|
2053
|
+
value = str(flt.get("value") or "").strip().strip("%").lower()
|
|
2054
|
+
if not value:
|
|
2055
|
+
continue
|
|
2056
|
+
found = normalized_sql.find(value)
|
|
2057
|
+
if found >= 0:
|
|
2058
|
+
positions.append(found)
|
|
2059
|
+
if len(positions) < 2:
|
|
2060
|
+
return True
|
|
2061
|
+
start, end = min(positions), max(positions)
|
|
2062
|
+
between = normalized_sql[start:end]
|
|
2063
|
+
return bool(re.search(r"\bor\b", between))
|
|
668
2064
|
|
|
669
2065
|
|
|
670
|
-
def
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
2066
|
+
def sql_missing_query_plan_select_fields(sql: str, query_plan: dict[str, Any] | None) -> list[str]:
|
|
2067
|
+
"""Return required SELECT display fields from query_plan that are absent in sql."""
|
|
2068
|
+
if not isinstance(query_plan, dict):
|
|
2069
|
+
return []
|
|
2070
|
+
select_clause = _select_clause(sql).lower()
|
|
2071
|
+
if not select_clause:
|
|
2072
|
+
return []
|
|
2073
|
+
missing: list[str] = []
|
|
2074
|
+
for item in query_plan.get("fields") or []:
|
|
2075
|
+
if not isinstance(item, dict):
|
|
2076
|
+
continue
|
|
2077
|
+
if item.get("required") is not True and str(item.get("source") or "") != "search_entities":
|
|
2078
|
+
continue
|
|
2079
|
+
table = str(item.get("table") or "").strip()
|
|
2080
|
+
field = str(item.get("field") or "").strip()
|
|
2081
|
+
description = str(item.get("description") or "").strip()
|
|
2082
|
+
if not table or not field:
|
|
2083
|
+
continue
|
|
2084
|
+
qualified = f"{table}.{field}".lower()
|
|
2085
|
+
description_present = bool(description and description.lower() in select_clause)
|
|
2086
|
+
qualified_present = qualified in select_clause
|
|
2087
|
+
if not description_present and not qualified_present:
|
|
2088
|
+
missing.append(f"{qualified} AS {description or field}")
|
|
2089
|
+
return missing
|
|
675
2090
|
|
|
676
2091
|
|
|
677
|
-
def
|
|
678
|
-
return
|
|
2092
|
+
def sql_satisfies_query_plan_select_fields(sql: str, query_plan: dict[str, Any] | None) -> bool:
|
|
2093
|
+
return not sql_missing_query_plan_select_fields(sql, query_plan)
|
|
2094
|
+
|
|
2095
|
+
|
|
2096
|
+
def _select_clause(sql: str) -> str:
|
|
2097
|
+
match = re.search(r"\bselect\b(?P<select>.*?)\bfrom\b", sql, flags=re.IGNORECASE | re.DOTALL)
|
|
2098
|
+
return match.group("select") if match else ""
|
|
679
2099
|
|
|
680
2100
|
|
|
681
|
-
def sql_satisfies_time_range(
|
|
682
|
-
|
|
2101
|
+
def sql_satisfies_time_range(
|
|
2102
|
+
sql: str,
|
|
2103
|
+
question: str,
|
|
2104
|
+
schema: dict[str, Any] | None = None,
|
|
2105
|
+
*,
|
|
2106
|
+
query_plan: dict[str, Any] | None = None,
|
|
2107
|
+
) -> bool:
|
|
2108
|
+
time_range = resolve_time_range(question, schema, query_plan=query_plan)
|
|
683
2109
|
if time_range is None:
|
|
684
2110
|
return True
|
|
685
2111
|
normalized = sql.lower()
|
|
@@ -688,6 +2114,11 @@ def sql_satisfies_time_range(sql: str, question: str, schema: dict[str, Any] | N
|
|
|
688
2114
|
return False
|
|
689
2115
|
if time_range.start[:10] in sql and time_range.end[:10] in sql:
|
|
690
2116
|
return True
|
|
2117
|
+
end_date = _parse_date(time_range.end[:10])
|
|
2118
|
+
if end_date is not None:
|
|
2119
|
+
inclusive_end = (end_date - timedelta(days=1)).isoformat()
|
|
2120
|
+
if time_range.start[:10] in sql and inclusive_end in sql:
|
|
2121
|
+
return True
|
|
691
2122
|
year = time_range.start[:4]
|
|
692
2123
|
month = str(int(time_range.start[5:7]))
|
|
693
2124
|
padded_month = time_range.start[5:7]
|
|
@@ -697,6 +2128,61 @@ def sql_satisfies_time_range(sql: str, question: str, schema: dict[str, Any] | N
|
|
|
697
2128
|
)
|
|
698
2129
|
|
|
699
2130
|
|
|
2131
|
+
def _schema_limited_to_sql_tables(schema: dict[str, Any], sql: str) -> dict[str, Any] | None:
|
|
2132
|
+
used_tables = _sql_referenced_table_names(sql)
|
|
2133
|
+
if not used_tables:
|
|
2134
|
+
return None
|
|
2135
|
+
tables = schema.get("tables", [])
|
|
2136
|
+
if not isinstance(tables, list):
|
|
2137
|
+
return None
|
|
2138
|
+
matched_tables = []
|
|
2139
|
+
for table in tables:
|
|
2140
|
+
if not isinstance(table, dict):
|
|
2141
|
+
continue
|
|
2142
|
+
table_name = str(table.get("tableName") or table.get("name") or "").lower()
|
|
2143
|
+
if table_name in used_tables:
|
|
2144
|
+
matched_tables.append(table)
|
|
2145
|
+
if not matched_tables:
|
|
2146
|
+
return None
|
|
2147
|
+
limited = dict(schema)
|
|
2148
|
+
limited["tables"] = matched_tables
|
|
2149
|
+
return limited
|
|
2150
|
+
|
|
2151
|
+
|
|
2152
|
+
def _sql_referenced_table_names(sql: str) -> set[str]:
|
|
2153
|
+
normalized = _strip_sql_string_literals(sql)
|
|
2154
|
+
names: set[str] = set()
|
|
2155
|
+
for match in re.finditer(
|
|
2156
|
+
r"\b(?:from|join)\s+([a-zA-Z_][a-zA-Z0-9_]*)\b",
|
|
2157
|
+
normalized,
|
|
2158
|
+
flags=re.IGNORECASE,
|
|
2159
|
+
):
|
|
2160
|
+
names.add(match.group(1).lower())
|
|
2161
|
+
return names
|
|
2162
|
+
|
|
2163
|
+
|
|
2164
|
+
def _strip_sql_string_literals(sql: str) -> str:
|
|
2165
|
+
result: list[str] = []
|
|
2166
|
+
quote: str | None = None
|
|
2167
|
+
index = 0
|
|
2168
|
+
while index < len(sql):
|
|
2169
|
+
char = sql[index]
|
|
2170
|
+
if quote is not None:
|
|
2171
|
+
if char == quote:
|
|
2172
|
+
quote = None
|
|
2173
|
+
result.append(" ")
|
|
2174
|
+
index += 1
|
|
2175
|
+
continue
|
|
2176
|
+
if char in ("'", '"'):
|
|
2177
|
+
quote = char
|
|
2178
|
+
result.append(" ")
|
|
2179
|
+
index += 1
|
|
2180
|
+
continue
|
|
2181
|
+
result.append(char)
|
|
2182
|
+
index += 1
|
|
2183
|
+
return "".join(result)
|
|
2184
|
+
|
|
2185
|
+
|
|
700
2186
|
def _table_names(schema: dict[str, Any]) -> set[str]:
|
|
701
2187
|
tables = schema.get("tables", [])
|
|
702
2188
|
if not isinstance(tables, list):
|
|
@@ -719,7 +2205,7 @@ def _table_columns(schema: dict[str, Any], table_name: str) -> list[dict[str, An
|
|
|
719
2205
|
if not isinstance(table, dict):
|
|
720
2206
|
continue
|
|
721
2207
|
name = table.get("name") or table.get("tableName")
|
|
722
|
-
if name == table_name and isinstance(table.get("columns"), list):
|
|
2208
|
+
if str(name).lower() == str(table_name).lower() and isinstance(table.get("columns"), list):
|
|
723
2209
|
return [column for column in table["columns"] if isinstance(column, dict)]
|
|
724
2210
|
return []
|
|
725
2211
|
|
|
@@ -755,55 +2241,75 @@ def _has_column(schema: dict[str, Any], table_name: str, column_name: str) -> bo
|
|
|
755
2241
|
return any((column.get("columnName") or column.get("name")) == column_name for column in _table_columns(schema, table_name))
|
|
756
2242
|
|
|
757
2243
|
|
|
758
|
-
def
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
2244
|
+
def resolve_time_range_for_question(
|
|
2245
|
+
question: str,
|
|
2246
|
+
schema: dict[str, Any] | None = None,
|
|
2247
|
+
*,
|
|
2248
|
+
concepts: list[dict[str, Any]] | None = None,
|
|
2249
|
+
) -> TimeRange | None:
|
|
762
2250
|
bounds = _extract_time_bounds(question)
|
|
763
2251
|
if bounds is None:
|
|
764
2252
|
return None
|
|
765
|
-
|
|
2253
|
+
for concept in concepts or []:
|
|
2254
|
+
qualified = str(concept.get("timeRangeField") or "").strip()
|
|
2255
|
+
if "." not in qualified:
|
|
2256
|
+
continue
|
|
2257
|
+
table_name, field_name = qualified.rsplit(".", 1)
|
|
2258
|
+
return TimeRange(
|
|
2259
|
+
field=field_name,
|
|
2260
|
+
alias="时间范围",
|
|
2261
|
+
start=bounds[0],
|
|
2262
|
+
end=bounds[1],
|
|
2263
|
+
table_name=table_name,
|
|
2264
|
+
)
|
|
2265
|
+
field = _resolve_time_field(question, schema or {})
|
|
2266
|
+
if field is None:
|
|
2267
|
+
return None
|
|
2268
|
+
return TimeRange(
|
|
2269
|
+
field=field.name,
|
|
2270
|
+
alias=field.alias,
|
|
2271
|
+
start=bounds[0],
|
|
2272
|
+
end=bounds[1],
|
|
2273
|
+
table_name=field.table_name,
|
|
2274
|
+
)
|
|
2275
|
+
|
|
2276
|
+
|
|
2277
|
+
def _extract_time_range(question: str, schema: dict[str, Any] | None = None) -> TimeRange | None:
|
|
2278
|
+
return resolve_time_range_for_question(question, schema)
|
|
766
2279
|
|
|
767
2280
|
|
|
768
|
-
def
|
|
769
|
-
columns = _table_columns(schema, "job_basic_info")
|
|
2281
|
+
def _resolve_time_field(question: str, schema: dict[str, Any]) -> FieldRef | None:
|
|
770
2282
|
question_text = _normalize_semantic_text(question)
|
|
771
2283
|
best: tuple[int, FieldRef] | None = None
|
|
772
|
-
for
|
|
773
|
-
|
|
774
|
-
if not name:
|
|
775
|
-
continue
|
|
776
|
-
data_type = str(column.get("dataType") or column.get("type") or "")
|
|
777
|
-
comment = str(column.get("comment") or "")
|
|
778
|
-
if not _is_time_like_column(name, data_type, comment):
|
|
2284
|
+
for table in schema.get("tables", []) if isinstance(schema.get("tables"), list) else []:
|
|
2285
|
+
if not isinstance(table, dict):
|
|
779
2286
|
continue
|
|
780
|
-
|
|
781
|
-
if
|
|
2287
|
+
columns = table.get("columns")
|
|
2288
|
+
if not isinstance(columns, list):
|
|
782
2289
|
continue
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
2290
|
+
for column in columns:
|
|
2291
|
+
if not isinstance(column, dict):
|
|
2292
|
+
continue
|
|
2293
|
+
name = str(column.get("columnName") or column.get("name") or "")
|
|
2294
|
+
if not name:
|
|
2295
|
+
continue
|
|
2296
|
+
data_type = str(column.get("dataType") or column.get("type") or "")
|
|
2297
|
+
comment = str(column.get("comment") or "")
|
|
2298
|
+
if not _is_time_like_column(name, data_type, comment):
|
|
2299
|
+
continue
|
|
2300
|
+
score = _time_field_match_score(question_text, name, comment)
|
|
2301
|
+
if score <= 0:
|
|
2302
|
+
continue
|
|
2303
|
+
table_name = str(table.get("tableName") or table.get("name") or "")
|
|
2304
|
+
field_ref = FieldRef(
|
|
2305
|
+
name=name,
|
|
2306
|
+
alias=_column_alias(name, comment),
|
|
2307
|
+
table_name=table_name,
|
|
2308
|
+
)
|
|
2309
|
+
if best is None or score > best[0]:
|
|
2310
|
+
best = (score, field_ref)
|
|
786
2311
|
if best is not None:
|
|
787
2312
|
return best[1]
|
|
788
|
-
if _extract_time_bounds(question) is not None and re.search(r"(在招岗位数|岗位供给|招聘人数|招聘门店数|供给)", question):
|
|
789
|
-
created_column = _select_column(schema, "job_basic_info", ("创建时间",), ("create_at",))
|
|
790
|
-
if created_column is not None:
|
|
791
|
-
return FieldRef(created_column, _column_alias(created_column, "创建时间"))
|
|
792
|
-
fallback = _select_time_field(question)
|
|
793
|
-
if fallback is None:
|
|
794
|
-
return None
|
|
795
|
-
return fallback
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
def _select_time_field(question: str) -> FieldRef | None:
|
|
799
|
-
if re.search(r"(发布|上架)", question):
|
|
800
|
-
return FieldRef("publish_at", "发布时间")
|
|
801
|
-
if re.search(r"(下架)", question):
|
|
802
|
-
return FieldRef("off_at", "下架时间")
|
|
803
|
-
if re.search(r"(更新|修改)", question):
|
|
804
|
-
return FieldRef("update_at", "更新时间")
|
|
805
|
-
if re.search(r"(创建|新建|新增)", question):
|
|
806
|
-
return FieldRef("create_at", "创建时间")
|
|
807
2313
|
return None
|
|
808
2314
|
|
|
809
2315
|
|
|
@@ -818,9 +2324,6 @@ def _is_time_like_column(name: str, data_type: str, comment: str) -> bool:
|
|
|
818
2324
|
def _time_field_match_score(question_text: str, name: str, comment: str) -> int:
|
|
819
2325
|
field_text = _normalize_semantic_text(f"{name}{comment}")
|
|
820
2326
|
score = 0
|
|
821
|
-
for term in ("创建", "发布", "下架", "更新"):
|
|
822
|
-
if term in question_text and term in field_text:
|
|
823
|
-
score += 100
|
|
824
2327
|
comment_text = _normalize_semantic_text(comment)
|
|
825
2328
|
if comment_text and comment_text in question_text:
|
|
826
2329
|
score += 50
|
|
@@ -829,23 +2332,18 @@ def _time_field_match_score(question_text: str, name: str, comment: str) -> int:
|
|
|
829
2332
|
score += 30
|
|
830
2333
|
if "时间" in question_text and "时间" in field_text:
|
|
831
2334
|
score += 5
|
|
2335
|
+
if "日期" in question_text and ("时间" in field_text or "日期" in field_text):
|
|
2336
|
+
score += 5
|
|
2337
|
+
if "报名" in question_text and "报名" in comment_text:
|
|
2338
|
+
score += 40
|
|
2339
|
+
if re.search(r"(上架|发布|新上架)", question_text):
|
|
2340
|
+
if "publish" in name.lower() or "发布" in comment_text:
|
|
2341
|
+
score += 40
|
|
832
2342
|
return score
|
|
833
2343
|
|
|
834
2344
|
|
|
835
2345
|
def _normalize_semantic_text(text: str) -> str:
|
|
836
2346
|
normalized = text.lower()
|
|
837
|
-
replacements = {
|
|
838
|
-
"新建": "创建",
|
|
839
|
-
"新增": "创建",
|
|
840
|
-
"录入": "创建",
|
|
841
|
-
"生成": "创建",
|
|
842
|
-
"上架": "发布",
|
|
843
|
-
"上线": "发布",
|
|
844
|
-
"修改": "更新",
|
|
845
|
-
"下线": "下架",
|
|
846
|
-
}
|
|
847
|
-
for source, target in replacements.items():
|
|
848
|
-
normalized = normalized.replace(source, target)
|
|
849
2347
|
return re.sub(r"[^0-9a-z\u4e00-\u9fff]+", "", normalized)
|
|
850
2348
|
|
|
851
2349
|
|
|
@@ -853,24 +2351,73 @@ def _column_alias(name: str, comment: str) -> str:
|
|
|
853
2351
|
cleaned = re.sub(r"[。;;,,\s].*$", "", comment.strip())
|
|
854
2352
|
if cleaned:
|
|
855
2353
|
return cleaned
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
}
|
|
862
|
-
return fallback_aliases.get(name, name)
|
|
2354
|
+
return name
|
|
2355
|
+
|
|
2356
|
+
|
|
2357
|
+
def _normalize_fullwidth_digits(text: str) -> str:
|
|
2358
|
+
return text.translate(str.maketrans("0123456789", "0123456789"))
|
|
863
2359
|
|
|
864
2360
|
|
|
865
2361
|
def _extract_time_bounds(question: str) -> tuple[str, str] | None:
|
|
2362
|
+
question = _normalize_fullwidth_digits(question)
|
|
866
2363
|
date_range = _extract_explicit_date_range(question)
|
|
867
2364
|
if date_range is not None:
|
|
868
2365
|
return date_range
|
|
2366
|
+
if re.search(r"(近|最近|过去)\s*(?:1|一)\s*年", question):
|
|
2367
|
+
today = _today()
|
|
2368
|
+
return _format_datetime(_one_year_before(today)), _format_datetime(today + timedelta(days=1))
|
|
2369
|
+
if re.search(r"(今天|今日)", question):
|
|
2370
|
+
today = _today()
|
|
2371
|
+
return _format_datetime(today), _format_datetime(today + timedelta(days=1))
|
|
2372
|
+
if re.search(r"(昨天|昨日)", question):
|
|
2373
|
+
today = _today()
|
|
2374
|
+
yesterday = today - timedelta(days=1)
|
|
2375
|
+
return _format_datetime(yesterday), _format_datetime(today)
|
|
869
2376
|
relative_month_match = re.search(r"(?P<relative_year>今年|去年|前年|明年|后年)\s*(?P<month>\d{1,2})\s*月(?:份)?", question)
|
|
870
2377
|
if relative_month_match is not None:
|
|
871
2378
|
year = _today().year + _relative_year_offset(relative_month_match.group("relative_year"))
|
|
872
2379
|
month = int(relative_month_match.group("month"))
|
|
873
2380
|
return _month_bounds(year, month)
|
|
2381
|
+
if not re.search(r"20\d{2}\s*年", question):
|
|
2382
|
+
cn_day_range = re.search(
|
|
2383
|
+
r"(?P<month>\d{1,2})\s*月\s*(?P<start_day>\d{1,2})\s*日?\s*"
|
|
2384
|
+
r"(?:到|至|~|~|-)\s*"
|
|
2385
|
+
r"(?:(?P<end_month>\d{1,2})\s*月\s*)?(?P<end_day>\d{1,2})\s*日?",
|
|
2386
|
+
question,
|
|
2387
|
+
)
|
|
2388
|
+
if cn_day_range is not None:
|
|
2389
|
+
year = _today().year
|
|
2390
|
+
month = int(cn_day_range.group("month"))
|
|
2391
|
+
end_month_text = cn_day_range.group("end_month")
|
|
2392
|
+
end_month = int(end_month_text) if end_month_text else month
|
|
2393
|
+
if end_month != month:
|
|
2394
|
+
return None
|
|
2395
|
+
start_day = int(cn_day_range.group("start_day"))
|
|
2396
|
+
end_day = int(cn_day_range.group("end_day"))
|
|
2397
|
+
try:
|
|
2398
|
+
start = date(year, month, start_day)
|
|
2399
|
+
end = date(year, month, end_day)
|
|
2400
|
+
except ValueError:
|
|
2401
|
+
return None
|
|
2402
|
+
if end >= start:
|
|
2403
|
+
return _format_datetime(start), _format_datetime(end + timedelta(days=1))
|
|
2404
|
+
single_cn_day = re.search(r"(?P<month>\d{1,2})\s*月\s*(?P<day>\d{1,2})\s*日", question)
|
|
2405
|
+
if single_cn_day is not None and not re.search(r"(到|至|~|~|-)", question):
|
|
2406
|
+
year = _today().year
|
|
2407
|
+
try:
|
|
2408
|
+
day = date(year, int(single_cn_day.group("month")), int(single_cn_day.group("day")))
|
|
2409
|
+
except ValueError:
|
|
2410
|
+
day = None
|
|
2411
|
+
if day is not None:
|
|
2412
|
+
return _format_datetime(day), _format_datetime(day + timedelta(days=1))
|
|
2413
|
+
cn_month_only = re.search(
|
|
2414
|
+
r"(?:整个|整)\s*(?P<month>\d{1,2})\s*月(?:份)?|(?P<month_only>\d{1,2})\s*月(?:份)?(?!\s*\d{1,2}\s*日)",
|
|
2415
|
+
question,
|
|
2416
|
+
)
|
|
2417
|
+
if cn_month_only is not None:
|
|
2418
|
+
month_text = cn_month_only.group("month") or cn_month_only.group("month_only")
|
|
2419
|
+
if month_text is not None:
|
|
2420
|
+
return _month_bounds(_today().year, int(month_text))
|
|
874
2421
|
month_match = re.search(r"(?P<year>20\d{2})\s*年\s*(?P<month>\d{1,2})\s*月(?:份)?", question)
|
|
875
2422
|
if month_match is None:
|
|
876
2423
|
month_match = re.search(r"(?P<year>20\d{2})[-/](?P<month>\d{1,2})(?![-/]\d)", question)
|
|
@@ -892,6 +2439,13 @@ def _relative_year_offset(relative_year: str) -> int:
|
|
|
892
2439
|
return offsets[relative_year]
|
|
893
2440
|
|
|
894
2441
|
|
|
2442
|
+
def _one_year_before(value: date) -> date:
|
|
2443
|
+
try:
|
|
2444
|
+
return value.replace(year=value.year - 1)
|
|
2445
|
+
except ValueError:
|
|
2446
|
+
return value.replace(year=value.year - 1, day=28)
|
|
2447
|
+
|
|
2448
|
+
|
|
895
2449
|
def _month_bounds(year: int, month: int) -> tuple[str, str] | None:
|
|
896
2450
|
if not 1 <= month <= 12:
|
|
897
2451
|
return None
|
|
@@ -933,339 +2487,16 @@ def _today() -> date:
|
|
|
933
2487
|
return date.today()
|
|
934
2488
|
|
|
935
2489
|
|
|
936
|
-
def _extract_location_for_recruiting_job_question(question: str) -> str | None:
|
|
937
|
-
if not _is_recruiting_job_question(question):
|
|
938
|
-
return None
|
|
939
|
-
text = re.sub(r"(查询|查看|看看|列出|给我|帮我查|当前|目前|现在)", "", question)
|
|
940
|
-
text = re.sub(r"(所有|全部|每个|各个)", "", text)
|
|
941
|
-
patterns = [
|
|
942
|
-
r"(?P<location>[\u4e00-\u9fff]{2,12}?)(?:有)?(?:多少|几|数量|总数).*?(?:在招|招聘)(?:岗位|职位)",
|
|
943
|
-
r"(?P<location>[\u4e00-\u9fff]{2,12}?)(?:的)?(?:在招|招聘)(?:岗位|职位)",
|
|
944
|
-
r"(?P<location>[\u4e00-\u9fff]{2,12}?)(?:有|有哪些|有什么).*?(?:在招|招聘)(?:岗位|职位)",
|
|
945
|
-
r"(?P<location>[\u4e00-\u9fff]{2,12}?)(?:的)?(?:岗位|职位)",
|
|
946
|
-
r"(?:在|位于)(?P<location>[\u4e00-\u9fff]{2,12}?)(?:的)?(?:岗位|职位)",
|
|
947
|
-
]
|
|
948
|
-
for pattern in patterns:
|
|
949
|
-
match = re.search(pattern, text)
|
|
950
|
-
if match is not None:
|
|
951
|
-
return _clean_location_candidate(match.group("location"), require_location_signal=True)
|
|
952
|
-
return None
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
def _is_recruiting_job_question(question: str) -> bool:
|
|
956
|
-
return bool(re.search(r"(岗位|职位|招聘|在招)", question))
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
def _is_work_order_detail_query(question: str) -> bool:
|
|
960
|
-
return bool(
|
|
961
|
-
"品牌" in question
|
|
962
|
-
and re.search(r"(报名|工单)", question)
|
|
963
|
-
and re.search(r"(候选人状态|候选人|面试成功)", question)
|
|
964
|
-
and re.search(r"(姓名|名字|岗位|职位|项目)", question)
|
|
965
|
-
)
|
|
966
|
-
|
|
967
|
-
|
|
968
2490
|
def _is_business_query(question: str) -> bool:
|
|
969
2491
|
return bool(
|
|
970
2492
|
re.search(
|
|
971
|
-
r"(
|
|
2493
|
+
r"(查询|查看|看看|搜索|统计|多少|几|哪些|有哪些|列表|明细|清单|展示|返回|输出|分析|报告|count|sum|avg|min|max)",
|
|
972
2494
|
question,
|
|
2495
|
+
flags=re.IGNORECASE,
|
|
973
2496
|
)
|
|
974
2497
|
)
|
|
975
2498
|
|
|
976
2499
|
|
|
977
|
-
def _is_single_entity_lookup_sql(sql: str) -> bool:
|
|
978
|
-
normalized = re.sub(r"\s+", " ", sql.strip().lower())
|
|
979
|
-
table_aliases = _sql_table_aliases(normalized)
|
|
980
|
-
referenced_tables = set(table_aliases.values())
|
|
981
|
-
entity_tables = {"brand", "sponge_project", "city", "province", "region", "store"}
|
|
982
|
-
if len(referenced_tables) != 1 or not referenced_tables <= entity_tables:
|
|
983
|
-
return False
|
|
984
|
-
if " limit " not in f" {normalized} ":
|
|
985
|
-
return False
|
|
986
|
-
return bool(re.search(r"\b(?:id|name|job_name)\b", normalized))
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
_ENTITY_NAME_CHARS = r"[\u4e00-\u9fffA-Za-z0-9()()·_\-]{1,40}"
|
|
990
|
-
_ENTITY_SEPARATOR = r"[\s::,,、;;]+"
|
|
991
|
-
_FIELD_LABEL_BOUNDARY = (
|
|
992
|
-
r"(?:\s+(?=门店|品牌|项目|岗位|职位|城市|区域|有哪些|有什么|多少|包括|返回|状态)"
|
|
993
|
-
r"|(?=在(?!招)|位于|有哪些|有什么|多少|包括|返回|状态|岗位|职位|门店|品牌|项目|$|[,,。;;??]))"
|
|
994
|
-
)
|
|
995
|
-
_DIRECT_MUNICIPALITIES = ("上海", "北京", "天津", "重庆")
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
def _extract_brand_filter(question: str) -> tuple[str, str] | None:
|
|
999
|
-
contains_patterns = [
|
|
1000
|
-
r"品牌(?:名称)?(?:包含|含有|模糊匹配|like)(?P<brand>.+?)(?:,|,|。|;|;|的品牌|品牌信息|包括|返回|$)",
|
|
1001
|
-
r"(?:name|名称)\s+LIKE\s+[\"'“”‘’%]*(?P<brand>[\u4e00-\u9fffA-Za-z0-9()()·_\-]{1,40})",
|
|
1002
|
-
]
|
|
1003
|
-
for pattern in contains_patterns:
|
|
1004
|
-
match = re.search(pattern, question, flags=re.IGNORECASE)
|
|
1005
|
-
if match is None:
|
|
1006
|
-
continue
|
|
1007
|
-
brand = _clean_brand_candidate(match.group("brand"))
|
|
1008
|
-
if brand is not None:
|
|
1009
|
-
return "contains", brand
|
|
1010
|
-
|
|
1011
|
-
exact_patterns = [
|
|
1012
|
-
rf"品牌(?:名称)?(?:为|是|=|等于|:|:)(?:{_ENTITY_SEPARATOR})?(?P<brand>{_ENTITY_NAME_CHARS})",
|
|
1013
|
-
rf"品牌{_ENTITY_SEPARATOR}(?P<brand>{_ENTITY_NAME_CHARS})",
|
|
1014
|
-
rf"品牌(?P<brand>{_ENTITY_NAME_CHARS}?)(?={_FIELD_LABEL_BOUNDARY})",
|
|
1015
|
-
rf"(?P<brand>{_ENTITY_NAME_CHARS})(?:{_ENTITY_SEPARATOR})?品牌(?:下|的)?",
|
|
1016
|
-
]
|
|
1017
|
-
for pattern in exact_patterns:
|
|
1018
|
-
match = re.search(pattern, question)
|
|
1019
|
-
if match is None:
|
|
1020
|
-
continue
|
|
1021
|
-
brand = _clean_brand_candidate(match.group("brand"))
|
|
1022
|
-
if brand is not None:
|
|
1023
|
-
return "exact", brand
|
|
1024
|
-
return None
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
def _extract_brand_name(question: str) -> str | None:
|
|
1028
|
-
brand_filter = _extract_brand_filter(question)
|
|
1029
|
-
return brand_filter[1] if brand_filter is not None else None
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
def _extract_project_name(question: str) -> str | None:
|
|
1033
|
-
patterns = [
|
|
1034
|
-
r"项目(?:名称)?(?:包含|含有|模糊匹配|like)(?P<name>.+?)(?:,|,|。|;|;|项目信息|包括|返回|$)",
|
|
1035
|
-
rf"项目(?:名称)?(?:为|是|=|等于|:|:)(?:{_ENTITY_SEPARATOR})?(?P<name>{_ENTITY_NAME_CHARS})",
|
|
1036
|
-
rf"项目{_ENTITY_SEPARATOR}(?P<name>{_ENTITY_NAME_CHARS})",
|
|
1037
|
-
rf"(?P<name>{_ENTITY_NAME_CHARS})(?:{_ENTITY_SEPARATOR})?项目(?:下|的)?",
|
|
1038
|
-
rf"项目(?P<name>{_ENTITY_NAME_CHARS})(?={_FIELD_LABEL_BOUNDARY})",
|
|
1039
|
-
]
|
|
1040
|
-
for pattern in patterns:
|
|
1041
|
-
match = re.search(pattern, question, flags=re.IGNORECASE)
|
|
1042
|
-
if match is None:
|
|
1043
|
-
continue
|
|
1044
|
-
project = _clean_project_candidate(match.group("name"))
|
|
1045
|
-
if project is not None:
|
|
1046
|
-
return project
|
|
1047
|
-
return None
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
def _clean_project_candidate(value: str) -> str | None:
|
|
1051
|
-
return _clean_entity_candidate(
|
|
1052
|
-
value,
|
|
1053
|
-
extra_forbidden=("品牌", "岗位", "职位", "报名"),
|
|
1054
|
-
suffix_pattern=r"((?:在|位于)[\u4e00-\u9fff]{1,20}(?:市|省|区|县)?.*$|(?:的)?(?:岗位|职位).*$|的项目信息|项目信息|的项目|项目|的|下)$",
|
|
1055
|
-
reject_location_like=True,
|
|
1056
|
-
)
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
def _extract_job_name_filter(question: str) -> str | None:
|
|
1060
|
-
patterns = [
|
|
1061
|
-
r"(?:岗位|职位)(?:名称)?(?:包含|含有|模糊匹配|like)(?P<name>.+?)(?:,|,|。|;|;|岗位信息|职位信息|包括|返回|$)",
|
|
1062
|
-
rf"(?:岗位|职位)(?:名称)?(?:为|是|=|等于|:|:)(?:{_ENTITY_SEPARATOR})?(?P<name>{_ENTITY_NAME_CHARS})",
|
|
1063
|
-
rf"(?:招聘|在招|招|急招)(?:{_ENTITY_SEPARATOR})?(?P<name>{_ENTITY_NAME_CHARS})(?:岗位|职位)",
|
|
1064
|
-
]
|
|
1065
|
-
for pattern in patterns:
|
|
1066
|
-
match = re.search(pattern, question, flags=re.IGNORECASE)
|
|
1067
|
-
if match is None:
|
|
1068
|
-
continue
|
|
1069
|
-
job_name = _clean_entity_candidate(match.group("name"), extra_forbidden=("品牌", "项目", "门店", "报名"))
|
|
1070
|
-
if job_name is not None:
|
|
1071
|
-
return job_name
|
|
1072
|
-
return None
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
def _extract_store_name_filter(question: str) -> str | None:
|
|
1076
|
-
patterns = [
|
|
1077
|
-
r"门店(?:名称)?(?:包含|含有|模糊匹配|like)(?P<name>.+?)(?:,|,|。|;|;|门店信息|包括|返回|$)",
|
|
1078
|
-
rf"门店(?:名称)?(?:为|是|=|等于|:|:)(?:{_ENTITY_SEPARATOR})?(?P<name>{_ENTITY_NAME_CHARS})",
|
|
1079
|
-
rf"(?<!所属)门店(?P<name>{_ENTITY_NAME_CHARS})(?=[,,。;;??\s]|有哪些|有什么|多少|岗位|职位|在招|招聘|$)",
|
|
1080
|
-
rf"(?P<name>{_ENTITY_NAME_CHARS})(?:{_ENTITY_SEPARATOR})?的门店",
|
|
1081
|
-
]
|
|
1082
|
-
for pattern in patterns:
|
|
1083
|
-
match = re.search(pattern, question, flags=re.IGNORECASE)
|
|
1084
|
-
if match is None:
|
|
1085
|
-
continue
|
|
1086
|
-
store_name = _clean_entity_candidate(
|
|
1087
|
-
match.group("name"),
|
|
1088
|
-
extra_forbidden=("品牌", "项目", "岗位", "职位", "报名", "薪资", "学历", "学历要求", "年龄要求"),
|
|
1089
|
-
suffix_pattern=r"((?:的)?(?:岗位|职位).*$|门店信息|的门店|门店|的|下)$",
|
|
1090
|
-
reject_location_like=True,
|
|
1091
|
-
)
|
|
1092
|
-
if store_name is not None:
|
|
1093
|
-
return store_name
|
|
1094
|
-
return None
|
|
1095
|
-
|
|
1096
|
-
|
|
1097
|
-
def _clean_brand_candidate(value: str) -> str | None:
|
|
1098
|
-
return _clean_entity_candidate(
|
|
1099
|
-
value,
|
|
1100
|
-
extra_forbidden=("岗位", "候选人", "报名", "入职", "在招", "招聘", "数量", "多少"),
|
|
1101
|
-
suffix_pattern=r"((?:的)?(?:岗位|职位).*$|的品牌信息|品牌信息|的品牌|品牌|的|下)$",
|
|
1102
|
-
)
|
|
1103
|
-
|
|
1104
|
-
|
|
1105
|
-
def _clean_entity_candidate(
|
|
1106
|
-
value: str,
|
|
1107
|
-
*,
|
|
1108
|
-
extra_forbidden: tuple[str, ...] = (),
|
|
1109
|
-
suffix_pattern: str = r"(的|下)$",
|
|
1110
|
-
reject_location_like: bool = False,
|
|
1111
|
-
) -> str | None:
|
|
1112
|
-
brand = re.sub(r"\s+", "", value)
|
|
1113
|
-
brand = brand.strip("\"'“”‘’%::,,、;;")
|
|
1114
|
-
brand = re.sub(r"^(名称为|名称是|为|是|=|等于)", "", brand)
|
|
1115
|
-
brand = re.sub(suffix_pattern, "", brand)
|
|
1116
|
-
brand = brand.strip("\"'“”‘’%::,,、;;")
|
|
1117
|
-
forbidden = (
|
|
1118
|
-
"查询",
|
|
1119
|
-
"查看",
|
|
1120
|
-
"给我",
|
|
1121
|
-
"帮我",
|
|
1122
|
-
"根据",
|
|
1123
|
-
"roll-core",
|
|
1124
|
-
"信息",
|
|
1125
|
-
"数据",
|
|
1126
|
-
"列表",
|
|
1127
|
-
"明细",
|
|
1128
|
-
"情况",
|
|
1129
|
-
"哪些",
|
|
1130
|
-
"所有",
|
|
1131
|
-
"全部",
|
|
1132
|
-
"我能看到",
|
|
1133
|
-
) + extra_forbidden
|
|
1134
|
-
if reject_location_like and _is_location_like_entity_name(brand):
|
|
1135
|
-
return None
|
|
1136
|
-
if 2 <= len(brand) <= 40 and not any(term in brand for term in forbidden):
|
|
1137
|
-
return brand
|
|
1138
|
-
return None
|
|
1139
|
-
|
|
1140
|
-
|
|
1141
|
-
def _is_location_like_entity_name(value: str) -> bool:
|
|
1142
|
-
return bool(
|
|
1143
|
-
re.fullmatch(r"在[\u4e00-\u9fff]{1,20}(?:市|省|区|县)?", value)
|
|
1144
|
-
or re.fullmatch(r"位于[\u4e00-\u9fff]{1,20}(?:市|省|区|县)?", value)
|
|
1145
|
-
or value.startswith(("在", "位于"))
|
|
1146
|
-
)
|
|
1147
|
-
|
|
1148
|
-
|
|
1149
|
-
def _sql_references_brand_table(sql: str) -> bool:
|
|
1150
|
-
return _sql_references_table(sql, "brand")
|
|
1151
|
-
|
|
1152
|
-
|
|
1153
|
-
def _sql_contains_brand_name_filter(sql: str, brand_name: str) -> bool:
|
|
1154
|
-
return _sql_contains_name_filter(sql, brand_name)
|
|
1155
|
-
|
|
1156
|
-
|
|
1157
|
-
def _sql_references_table(sql: str, table_name: str) -> bool:
|
|
1158
|
-
return table_name in _sql_table_aliases(sql.lower()).values()
|
|
1159
|
-
|
|
1160
|
-
|
|
1161
|
-
def _sql_contains_name_filter(sql: str, name: str) -> bool:
|
|
1162
|
-
normalized = sql.lower()
|
|
1163
|
-
escaped = re.escape(_escape_sql_string(name).lower())
|
|
1164
|
-
return bool(
|
|
1165
|
-
re.search(rf"\b[a-zA-Z_][a-zA-Z0-9_]*\.name\s*(?:=|like)\s*['\"]%?{escaped}%?['\"]", normalized)
|
|
1166
|
-
or re.search(rf"\bname\s*(?:=|like)\s*['\"]%?{escaped}%?['\"]", normalized)
|
|
1167
|
-
)
|
|
1168
|
-
|
|
1169
|
-
|
|
1170
|
-
def _sql_contains_project_id_filter(sql: str) -> bool:
|
|
1171
|
-
normalized = sql.lower()
|
|
1172
|
-
table_aliases = _sql_table_aliases(normalized)
|
|
1173
|
-
project_aliases = [alias for alias, table in table_aliases.items() if table == "sponge_project"]
|
|
1174
|
-
if not project_aliases:
|
|
1175
|
-
return False
|
|
1176
|
-
alias_pattern = "|".join(re.escape(alias) for alias in sorted(set(project_aliases), key=len, reverse=True))
|
|
1177
|
-
return bool(re.search(rf"\b(?:{alias_pattern})\.id\s*=\s*\d+\b", normalized))
|
|
1178
|
-
|
|
1179
|
-
|
|
1180
|
-
def _sql_contains_job_name_filter(sql: str, job_name: str) -> bool:
|
|
1181
|
-
normalized = sql.lower()
|
|
1182
|
-
escaped = re.escape(_escape_sql_string(job_name).lower())
|
|
1183
|
-
return bool(
|
|
1184
|
-
re.search(rf"\bjbi\.job_name\s*(?:=|like)\s*['\"]%?{escaped}%?['\"]", normalized)
|
|
1185
|
-
or re.search(rf"\bjob_name\s*(?:=|like)\s*['\"]%?{escaped}%?['\"]", normalized)
|
|
1186
|
-
)
|
|
1187
|
-
|
|
1188
|
-
|
|
1189
|
-
def _sql_contains_store_name_filter(sql: str, store_name: str) -> bool:
|
|
1190
|
-
normalized = sql.lower()
|
|
1191
|
-
escaped = re.escape(_escape_sql_string(store_name).lower())
|
|
1192
|
-
table_aliases = _sql_table_aliases(normalized)
|
|
1193
|
-
store_aliases = [alias for alias, table in table_aliases.items() if table == "store"]
|
|
1194
|
-
if store_aliases:
|
|
1195
|
-
alias_pattern = "|".join(re.escape(alias) for alias in sorted(set(store_aliases), key=len, reverse=True))
|
|
1196
|
-
if re.search(rf"\b(?:{alias_pattern})\.name\s*(?:=|like)\s*['\"]%?{escaped}%?['\"]", normalized):
|
|
1197
|
-
return True
|
|
1198
|
-
return bool(re.search(rf"\bstore_name\s*(?:=|like)\s*['\"]%?{escaped}%?['\"]", normalized))
|
|
1199
|
-
|
|
1200
|
-
|
|
1201
|
-
def is_recruiting_match_effect_query(question: str) -> bool:
|
|
1202
|
-
return bool(
|
|
1203
|
-
"岗位" in question
|
|
1204
|
-
and re.search(r"(候选人|报名|入职|上岗)", question)
|
|
1205
|
-
and re.search(r"(匹配效果|报名多|入职少|在招多|报名少|候选人多|岗位少)", question)
|
|
1206
|
-
and re.search(r"(品牌|项目|城市)", question)
|
|
1207
|
-
)
|
|
1208
|
-
|
|
1209
|
-
|
|
1210
|
-
def _clean_location_candidate(candidate: str, *, require_location_signal: bool = False) -> str | None:
|
|
1211
|
-
cleaned = re.sub(r"[,,。;;??\s]", "", candidate)
|
|
1212
|
-
cleaned = re.sub(r"(有哪些|有什么|所有|全部|每个|各个|多少|数量|列表|明细|信息|数据|情况|都|的|有)$", "", cleaned)
|
|
1213
|
-
if not 2 <= len(cleaned) <= 12:
|
|
1214
|
-
return None
|
|
1215
|
-
if re.search(r"[\u4e00-\u9fff]", cleaned) is None:
|
|
1216
|
-
return None
|
|
1217
|
-
if re.search(r"(岗位|职位|招聘|在招|品牌|项目|查询|查看|当前|目前|现在)", cleaned):
|
|
1218
|
-
return None
|
|
1219
|
-
if _is_generic_recruiting_term(cleaned):
|
|
1220
|
-
return None
|
|
1221
|
-
if require_location_signal and not _has_location_signal(cleaned):
|
|
1222
|
-
return None
|
|
1223
|
-
return cleaned
|
|
1224
|
-
|
|
1225
|
-
|
|
1226
|
-
def _has_location_signal(candidate: str) -> bool:
|
|
1227
|
-
if re.search(r"(省|市|区|县|镇|乡|街道|路|街|大道|广场|商圈|附近|地铁|站|商场|门店)$", candidate):
|
|
1228
|
-
return True
|
|
1229
|
-
return 2 <= len(candidate) <= 4 and not _is_generic_recruiting_term(candidate)
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
def _is_generic_recruiting_term(candidate: str) -> bool:
|
|
1233
|
-
generic_terms = {
|
|
1234
|
-
"兼职",
|
|
1235
|
-
"全职",
|
|
1236
|
-
"小时工",
|
|
1237
|
-
"暑假工",
|
|
1238
|
-
"寒假工",
|
|
1239
|
-
"长期工",
|
|
1240
|
-
"短期工",
|
|
1241
|
-
"服务员",
|
|
1242
|
-
"店员",
|
|
1243
|
-
"骑手",
|
|
1244
|
-
"厨师",
|
|
1245
|
-
"收银员",
|
|
1246
|
-
"营业员",
|
|
1247
|
-
"理货员",
|
|
1248
|
-
"分拣",
|
|
1249
|
-
"补货",
|
|
1250
|
-
"打包",
|
|
1251
|
-
"餐厅",
|
|
1252
|
-
"门店",
|
|
1253
|
-
"品牌",
|
|
1254
|
-
"项目",
|
|
1255
|
-
"高薪",
|
|
1256
|
-
"急招",
|
|
1257
|
-
"最新",
|
|
1258
|
-
"全部",
|
|
1259
|
-
"所有",
|
|
1260
|
-
"热门",
|
|
1261
|
-
"附近",
|
|
1262
|
-
"本周",
|
|
1263
|
-
"今天",
|
|
1264
|
-
"明天",
|
|
1265
|
-
}
|
|
1266
|
-
return candidate in generic_terms
|
|
1267
|
-
|
|
1268
|
-
|
|
1269
2500
|
def _escape_sql_string(value: str) -> str:
|
|
1270
2501
|
return value.replace("\\", "\\\\").replace("'", "''")
|
|
1271
2502
|
|
|
@@ -1279,7 +2510,7 @@ def _enforce_chinese_select_aliases(sql: str) -> None:
|
|
|
1279
2510
|
if not cleaned:
|
|
1280
2511
|
continue
|
|
1281
2512
|
alias_match = re.search(
|
|
1282
|
-
|
|
2513
|
+
_SELECT_AS_ALIAS_PATTERN,
|
|
1283
2514
|
cleaned,
|
|
1284
2515
|
flags=re.IGNORECASE,
|
|
1285
2516
|
)
|
|
@@ -1302,7 +2533,7 @@ def _enforce_no_null_placeholder_select_items(sql: str) -> None:
|
|
|
1302
2533
|
|
|
1303
2534
|
def _select_expression_without_alias(item: str) -> str:
|
|
1304
2535
|
return re.sub(
|
|
1305
|
-
|
|
2536
|
+
_SELECT_AS_ALIAS_PATTERN,
|
|
1306
2537
|
"",
|
|
1307
2538
|
item.strip(),
|
|
1308
2539
|
flags=re.IGNORECASE,
|