@roll-agent/octopus-agent 0.1.3 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (36) hide show
  1. package/README.md +26 -190
  2. package/SKILL.md +27 -103
  3. package/package.json +12 -18
  4. package/references/env.yaml +1 -22
  5. package/scripts/start-octopus-agent.js +1 -125
  6. package/src/config.js +13 -0
  7. package/src/context.js +5 -0
  8. package/src/mcp-client.js +80 -0
  9. package/src/orchestrator.js +57 -0
  10. package/src/server-core.js +67 -0
  11. package/src/server.js +10 -0
  12. package/src/stdio.js +29 -0
  13. package/octopus_skill/__init__.py +0 -11
  14. package/pyproject.toml +0 -19
  15. package/scripts/refresh_dictionary.py +0 -67
  16. package/scripts/verify_binding.py +0 -47
  17. package/src/octopus_skill/__init__.py +0 -5
  18. package/src/octopus_skill/__main__.py +0 -4
  19. package/src/octopus_skill/_version.py +0 -3
  20. package/src/octopus_skill/agent.py +0 -5739
  21. package/src/octopus_skill/audit_logger.py +0 -29
  22. package/src/octopus_skill/config.py +0 -129
  23. package/src/octopus_skill/context.py +0 -17
  24. package/src/octopus_skill/device_info.py +0 -7
  25. package/src/octopus_skill/exporter.py +0 -66
  26. package/src/octopus_skill/llm_result_renderer.py +0 -101
  27. package/src/octopus_skill/llm_sql_generator.py +0 -197
  28. package/src/octopus_skill/main.py +0 -26
  29. package/src/octopus_skill/mcp_client.py +0 -114
  30. package/src/octopus_skill/octopus_run.py +0 -755
  31. package/src/octopus_skill/prompt_builder.py +0 -522
  32. package/src/octopus_skill/question_analyzer.py +0 -401
  33. package/src/octopus_skill/result_renderer.py +0 -367
  34. package/src/octopus_skill/schema_cache.py +0 -49
  35. package/src/octopus_skill/schema_context_retriever.py +0 -1053
  36. package/src/octopus_skill/sql_generator.py +0 -2866
@@ -1,2866 +0,0 @@
1
- from __future__ import annotations
2
-
3
- from datetime import date, timedelta
4
- import re
5
- from dataclasses import dataclass
6
- from typing import Any
7
-
8
- from .schema_context_retriever import _schema_enums as merged_schema_enums
9
-
10
-
11
- class SqlGenerationError(RuntimeError):
12
- pass
13
-
14
-
15
- @dataclass(frozen=True)
16
- class GeneratedSql:
17
- sql: str
18
- tables: list[str]
19
- placeholders: list[str]
20
- used_columns: list[str] | None = None
21
- used_joins: list[str] | None = None
22
- metric_refs: list[str] | None = None
23
- clarification: dict[str, Any] | None = None
24
-
25
-
26
- @dataclass(frozen=True)
27
- class TimeRange:
28
- field: str
29
- alias: str
30
- start: str
31
- end: str
32
- table_name: str = ""
33
-
34
-
35
- @dataclass(frozen=True)
36
- class FieldRef:
37
- name: str
38
- alias: str
39
- table_name: str = ""
40
-
41
-
42
- @dataclass(frozen=True)
43
- class SchemaValidationIssue:
44
- code: str
45
- message: str
46
-
47
-
48
- @dataclass(frozen=True)
49
- class LowQualitySqlIssue:
50
- code: str
51
- message: str
52
- missing_items: tuple[str, ...] = ()
53
- current_problem: str = ""
54
- repair_hint: str = ""
55
-
56
-
57
- _SELECT_AS_ALIAS_PATTERN = (
58
- r"\s+as\s+(`[^`]+`|\"[^\"]+\"|'[^']+'|\w+)\s*$"
59
- )
60
-
61
- _FORBIDDEN_DML_KEYWORDS = re.compile(
62
- r"\b(insert|update|delete|drop|alter|create|truncate)\b",
63
- flags=re.IGNORECASE,
64
- )
65
-
66
-
67
- def _sql_without_string_literals(sql: str) -> str:
68
- return re.sub(r"'([^']|'')*'", "''", sql)
69
-
70
-
71
- def contains_forbidden_dml_keyword(sql: str) -> bool:
72
- stripped = _sql_without_string_literals(sql.strip())
73
- return _FORBIDDEN_DML_KEYWORDS.search(stripped) is not None
74
-
75
-
76
- def enforce_sql_rules(sql: str, *, max_limit: int) -> None:
77
- normalized = sql.strip().lower()
78
- if not normalized.startswith("select "):
79
- raise SqlGenerationError("SQL must be SELECT only")
80
- if ";" in normalized.rstrip(";"):
81
- raise SqlGenerationError("SQL must be a single statement")
82
- if contains_forbidden_dml_keyword(sql):
83
- raise SqlGenerationError("SQL contains a forbidden keyword")
84
- if re.search(r"\b(password|token|secret|credential|private_key)\b", normalized):
85
- raise SqlGenerationError("SQL contains a sensitive field")
86
- if _sql_selects_sensitive_field(sql):
87
- raise SqlGenerationError("SQL contains a sensitive field")
88
- if ":scope." in normalized:
89
- raise SqlGenerationError("SQL must not contain scope placeholders")
90
- _enforce_text_filters_use_like(sql)
91
- _enforce_join_on_uses_equality_keys(sql)
92
- _enforce_no_aggregate_functions_in_row_filters(sql)
93
- _enforce_chinese_select_aliases(sql)
94
- _enforce_no_null_placeholder_select_items(sql)
95
- _enforce_no_tautological_join_conditions(sql)
96
- limit = re.search(r"\blimit\s+(\d+)\b", normalized)
97
- if limit is None:
98
- raise SqlGenerationError("SQL must contain LIMIT")
99
- if int(limit.group(1)) > max_limit:
100
- raise SqlGenerationError(f"SQL LIMIT must not exceed {max_limit}")
101
-
102
-
103
- def build_schema_validation_error(
104
- sql: str,
105
- issues: list[SchemaValidationIssue],
106
- schema: dict[str, Any],
107
- *,
108
- attempt: int | None = None,
109
- max_attempts: int | None = None,
110
- ) -> dict[str, Any]:
111
- """Build a repair payload listing every schema violation and column hints."""
112
- schema_issues = [{"code": issue.code, "message": issue.message} for issue in issues]
113
- codes = {issue.code for issue in issues}
114
- if len(codes) == 1:
115
- code = next(iter(codes))
116
- else:
117
- code = "SCHEMA_VALIDATION"
118
- if len(issues) == 1:
119
- message = issues[0].message
120
- elif issues:
121
- message = f"共 {len(issues)} 项 schema 校验失败:{issues[0].message}"
122
- else:
123
- message = "schema 校验失败"
124
- payload: dict[str, Any] = {
125
- "code": code,
126
- "message": message,
127
- "originalSql": sql,
128
- "schemaIssues": schema_issues,
129
- }
130
- unknown_refs = _unknown_column_refs_from_issues(issues)
131
- if unknown_refs:
132
- payload["suggestedColumns"] = suggest_columns_for_unknown(unknown_refs, schema)
133
- if any(issue.code == "MISSING_TABLE_REFERENCE" for issue in issues):
134
- payload["tableReferenceRepairHint"] = (
135
- "SQL 引用了未在 FROM/JOIN 中声明的表名或别名。"
136
- "如果该字段确实需要参与查询,必须按 schema relations / joins 补 JOIN;"
137
- "如果不需要该表,则删除对应 SELECT/WHERE/ON/GROUP/ORDER 条件。"
138
- )
139
- if any(issue.code == "INVALID_JOIN" for issue in issues):
140
- payload["joinRepairHint"] = (
141
- "禁止用 ON 1=0 / ON 1=1 / ON false / ON true 规避 JOIN 校验;"
142
- "必须按 schemaContext.exampleGuidance、exampleJoinPatterns 与 structureHints 重建真实 JOIN。"
143
- )
144
- payload["suggestedJoinPatterns"] = suggest_join_patterns_for_schema(schema)
145
- if attempt is not None:
146
- payload["attempt"] = attempt
147
- if max_attempts is not None:
148
- payload["maxAttempts"] = max_attempts
149
- return payload
150
-
151
-
152
- def build_execute_unknown_column_error(
153
- sql: str,
154
- column_ref: str,
155
- schema: dict[str, Any],
156
- *,
157
- attempt: int | None = None,
158
- max_attempts: int | None = None,
159
- ) -> dict[str, Any]:
160
- """Build a repair payload when execute_sql reports a missing column."""
161
- table_name, column_name = _split_sql_column_ref(column_ref)
162
- if table_name and column_name:
163
- message = f"SQL 使用了数据库中不存在的列:{table_name}.{column_name}"
164
- issue = SchemaValidationIssue(code="UNKNOWN_COLUMN", message=message)
165
- else:
166
- message = f"SQL 使用了数据库中不存在的列:{column_ref}"
167
- issue = SchemaValidationIssue(code="UNKNOWN_COLUMN", message=message)
168
- return build_schema_validation_error(
169
- sql,
170
- [issue],
171
- schema,
172
- attempt=attempt,
173
- max_attempts=max_attempts,
174
- )
175
-
176
-
177
- def _split_sql_column_ref(column_ref: str) -> tuple[str, str]:
178
- cleaned = column_ref.strip().strip("`")
179
- if "." not in cleaned:
180
- return "", cleaned
181
- table_name, column_name = cleaned.rsplit(".", 1)
182
- return table_name, column_name
183
-
184
-
185
- def _unknown_column_refs_from_issues(issues: list[SchemaValidationIssue]) -> list[tuple[str, str]]:
186
- refs: list[tuple[str, str]] = []
187
- seen: set[tuple[str, str]] = set()
188
- for issue in issues:
189
- if issue.code != "UNKNOWN_COLUMN":
190
- continue
191
- match = re.search(
192
- r"不存在的列:([A-Za-z_][\w]*)\.([A-Za-z_][\w]*)",
193
- issue.message,
194
- )
195
- if not match:
196
- continue
197
- key = (match.group(1).lower(), match.group(2).lower())
198
- if key in seen:
199
- continue
200
- seen.add(key)
201
- refs.append(key)
202
- return refs
203
-
204
-
205
- def suggest_columns_for_unknown(
206
- missing: list[tuple[str, str]],
207
- schema: dict[str, Any],
208
- ) -> list[dict[str, str]]:
209
- """Find same column name on other tables when SQL references a non-existent column."""
210
- column_map = _schema_table_column_map(schema)
211
- suggestions: list[dict[str, str]] = []
212
- seen: set[tuple[str, str]] = set()
213
- for wrong_table, column in missing:
214
- for table, columns in sorted(column_map.items()):
215
- if table == wrong_table or column not in columns:
216
- continue
217
- key = (table, column)
218
- if key in seen:
219
- continue
220
- seen.add(key)
221
- suggestions.append(
222
- {
223
- "table": table,
224
- "column": column,
225
- "message": f"列 {column} 存在于 {table},不存在于 {wrong_table}",
226
- }
227
- )
228
- return suggestions
229
-
230
-
231
- def build_sql_rule_validation_error(
232
- sql: str,
233
- exc: SqlGenerationError,
234
- *,
235
- max_limit: int,
236
- attempt: int | None = None,
237
- max_attempts: int | None = None,
238
- ) -> dict[str, Any]:
239
- """Build a repair payload with the failing SQL and concrete violating SELECT items."""
240
- message = str(exc)
241
- payload: dict[str, Any] = {
242
- "code": "SQL_RULE_VIOLATION",
243
- "message": message,
244
- "originalSql": sql,
245
- "violatingColumns": list_sql_rule_violating_items(sql, message, max_limit=max_limit),
246
- }
247
- if attempt is not None:
248
- payload["attempt"] = attempt
249
- if max_attempts is not None:
250
- payload["maxAttempts"] = max_attempts
251
- return payload
252
-
253
-
254
- def list_sql_rule_violating_items(sql: str, message: str, *, max_limit: int) -> list[str]:
255
- lowered = message.lower()
256
- if "chinese alias" in lowered:
257
- return _list_missing_chinese_alias_select_items(sql)
258
- if "null placeholders" in lowered:
259
- return _list_null_placeholder_select_items(sql)
260
- if "sensitive field" in lowered:
261
- return _list_sensitive_select_items(sql)
262
- if "must be select only" in lowered:
263
- return ["(entire statement is not a single SELECT)"]
264
- if "must contain limit" in lowered:
265
- return ["(missing LIMIT clause)"]
266
- if "must not exceed" in lowered and "limit" in lowered:
267
- return ["(LIMIT exceeds allowed maximum)"]
268
- if "scope placeholders" in lowered:
269
- return re.findall(r":scope\.\w+", sql, flags=re.IGNORECASE)
270
- if "like" in lowered:
271
- return _list_text_filter_violations(sql)
272
- return []
273
-
274
-
275
- def _list_missing_chinese_alias_select_items(sql: str) -> list[str]:
276
- select_list = _top_level_select_list(sql)
277
- if select_list is None:
278
- return ["(missing top-level FROM clause)"]
279
- violations: list[str] = []
280
- for item in _split_top_level_select_items(select_list):
281
- cleaned = item.strip()
282
- if not cleaned:
283
- continue
284
- alias_match = re.search(
285
- _SELECT_AS_ALIAS_PATTERN,
286
- cleaned,
287
- flags=re.IGNORECASE,
288
- )
289
- if alias_match is None:
290
- violations.append(cleaned)
291
- continue
292
- alias = alias_match.group(1).strip("`\"'")
293
- if re.search(r"[\u4e00-\u9fff]", alias) is None:
294
- violations.append(cleaned)
295
- return violations
296
-
297
-
298
- def _list_null_placeholder_select_items(sql: str) -> list[str]:
299
- select_list = _top_level_select_list(sql)
300
- if select_list is None:
301
- return []
302
- return [
303
- item.strip()
304
- for item in _split_top_level_select_items(select_list)
305
- if item.strip() and _is_null_placeholder_expression(_select_expression_without_alias(item))
306
- ]
307
-
308
-
309
- def _list_sensitive_select_items(sql: str) -> list[str]:
310
- select_clause = _top_level_select_clause(sql)
311
- if select_clause is None:
312
- return []
313
- return [item.strip() for item in _split_select_items(select_clause) if _select_item_contains_sensitive_field(item)]
314
-
315
-
316
- def _list_text_filter_violations(sql: str) -> list[str]:
317
- return re.findall(
318
- r"(?i)(?:\b[\w.]+\b)\s*=\s*'[^']*(?:名称|状态|品牌|城市|项目|门店|供应商|工单)[^']*'",
319
- sql,
320
- )
321
-
322
-
323
- SENSITIVE_FIELD_NAMES = {
324
- "phone",
325
- "mobile",
326
- "tel",
327
- "telephone",
328
- "phone_number",
329
- "mobile_number",
330
- "mobile_phone",
331
- "contact_phone",
332
- "contact_mobile",
333
- "contact_tel",
334
- "id_card",
335
- "idcard",
336
- "identity_card",
337
- "cert_no",
338
- "certificate_no",
339
- "password",
340
- "token",
341
- "secret",
342
- "credential",
343
- "private_key",
344
- }
345
- SENSITIVE_FIELD_SUFFIXES = ("_phone", "_mobile", "_tel")
346
- SENSITIVE_FIELD_LABELS = (
347
- "手机号",
348
- "手机号码",
349
- "电话",
350
- "电话号码",
351
- "联系方式",
352
- "身份证",
353
- "身份证号",
354
- "身份证号码",
355
- "密码",
356
- "令牌",
357
- "密钥",
358
- "凭证",
359
- )
360
-
361
-
362
- def sensitive_terms_in_text(text: str) -> list[str]:
363
- lowered = text.lower()
364
- terms: list[str] = []
365
- for label in SENSITIVE_FIELD_LABELS:
366
- if label in text:
367
- terms.append(label)
368
- english_terms = {
369
- "phone": "手机号",
370
- "mobile": "手机号",
371
- "tel": "电话",
372
- "telephone": "电话",
373
- "id_card": "身份证",
374
- "idcard": "身份证",
375
- "password": "密码",
376
- "token": "令牌",
377
- "secret": "密钥",
378
- }
379
- for term, label in english_terms.items():
380
- if re.search(rf"\b{re.escape(term)}\b", lowered) and label not in terms:
381
- terms.append(label)
382
- return list(dict.fromkeys(terms))
383
-
384
-
385
- def _sql_selects_sensitive_field(sql: str) -> bool:
386
- select_clause = _top_level_select_clause(sql)
387
- if select_clause is None:
388
- return False
389
- return any(_select_item_contains_sensitive_field(item) for item in _split_select_items(select_clause))
390
-
391
-
392
- def _top_level_select_clause(sql: str) -> str | None:
393
- match = re.search(r"\bselect\b(?P<select>.*?)\bfrom\b", sql, flags=re.IGNORECASE | re.DOTALL)
394
- if match is None:
395
- return None
396
- return match.group("select")
397
-
398
-
399
- def _split_select_items(select_clause: str) -> list[str]:
400
- items: list[str] = []
401
- start = 0
402
- depth = 0
403
- quote: str | None = None
404
- for index, char in enumerate(select_clause):
405
- if quote is not None:
406
- if char == quote:
407
- quote = None
408
- continue
409
- if char in {"'", '"', "`"}:
410
- quote = char
411
- elif char == "(":
412
- depth += 1
413
- elif char == ")" and depth > 0:
414
- depth -= 1
415
- elif char == "," and depth == 0:
416
- items.append(select_clause[start:index].strip())
417
- start = index + 1
418
- tail = select_clause[start:].strip()
419
- if tail:
420
- items.append(tail)
421
- return items
422
-
423
-
424
- def _select_item_contains_sensitive_field(item: str) -> bool:
425
- if any(label in item for label in SENSITIVE_FIELD_LABELS):
426
- return True
427
- identifiers = re.findall(r"\b[a-zA-Z_][a-zA-Z0-9_]*\b", item.lower())
428
- for identifier in identifiers:
429
- if identifier in {"select", "case", "when", "then", "else", "end", "as", "if", "null", "coalesce"}:
430
- continue
431
- if identifier in SENSITIVE_FIELD_NAMES or identifier.endswith(SENSITIVE_FIELD_SUFFIXES):
432
- return True
433
- return False
434
-
435
-
436
- _TEXT_EQUALITY_FILTER_RE = re.compile(
437
- r"(?P<prefix>\b(?:WHERE|AND|OR|HAVING)\s+\(?\s*)"
438
- r"(?P<column>(?:[a-zA-Z_][A-Za-z0-9_]*\.)?[a-zA-Z_][A-Za-z0-9_]*)"
439
- r"\s*=\s*'(?P<value>[^']*)'",
440
- flags=re.IGNORECASE,
441
- )
442
-
443
-
444
- def normalize_text_filters_to_like(sql: str) -> str:
445
- """Rewrite name/status text equality filters to LIKE before rule enforcement."""
446
-
447
- def replace(match: re.Match[str]) -> str:
448
- column = match.group("column")
449
- column_name = column.rsplit(".", 1)[-1].lower()
450
- value = match.group("value").strip()
451
- if not _should_normalize_text_filter_to_like(column_name, value):
452
- return match.group(0)
453
- like_value = value if value.startswith("%") and value.endswith("%") else f"%{value.strip('%')}%"
454
- return f"{match.group('prefix')}{column} LIKE '{like_value}'"
455
-
456
- return _TEXT_EQUALITY_FILTER_RE.sub(replace, sql)
457
-
458
-
459
- def normalize_chinese_select_aliases(sql: str, schema: dict[str, Any] | None) -> str:
460
- """Add or replace SELECT aliases with Chinese labels from schema column comments."""
461
- if not schema or not sql.strip():
462
- return sql
463
- select_list = _top_level_select_list(sql)
464
- if select_list is None:
465
- return sql
466
-
467
- table_aliases = _sql_table_aliases(sql)
468
- used_labels: set[str] = set()
469
- new_items: list[str] = []
470
- changed = False
471
- for item in _split_top_level_select_items(select_list):
472
- cleaned = item.strip()
473
- if not cleaned:
474
- new_items.append(cleaned)
475
- continue
476
- if _select_item_has_chinese_alias(cleaned):
477
- alias = _extract_select_alias(cleaned)
478
- if alias:
479
- used_labels.add(alias)
480
- new_items.append(cleaned)
481
- continue
482
- expr = _select_expression_without_alias(cleaned)
483
- label = _infer_chinese_select_alias(expr, table_aliases=table_aliases, schema=schema)
484
- if label is None or re.search(r"[\u4e00-\u9fff]", label) is None:
485
- new_items.append(cleaned)
486
- continue
487
- label = _dedupe_chinese_alias(label, used_labels)
488
- used_labels.add(label)
489
- new_items.append(f"{expr} AS {label}")
490
- changed = True
491
- if not changed:
492
- return sql
493
- return _replace_top_level_select_list(sql, ", ".join(new_items))
494
-
495
-
496
- def question_requests_id_field(question: str) -> bool:
497
- lowered = question.lower()
498
- return "id" in lowered or "编号" in question or "主键" in question
499
-
500
-
501
- def strip_identifier_columns_from_select(sql: str, question: str) -> str:
502
- """Drop bare id / *_id columns from SELECT unless the question explicitly asks for ids."""
503
- if question_requests_id_field(question):
504
- return sql
505
- select_list = _top_level_select_list(sql)
506
- if select_list is None:
507
- return sql
508
- table_aliases = _sql_table_aliases(sql)
509
- kept: list[str] = []
510
- for item in _split_top_level_select_items(select_list):
511
- cleaned = item.strip()
512
- if not cleaned:
513
- continue
514
- expr = _select_expression_without_alias(cleaned)
515
- if _select_expr_is_identifier_column(expr, table_aliases):
516
- continue
517
- kept.append(cleaned)
518
- if not kept or len(kept) == len(_split_top_level_select_items(select_list)):
519
- return sql
520
- return _replace_top_level_select_list(sql, ", ".join(kept))
521
-
522
-
523
- def apply_enum_case_to_select(sql: str, schema: dict[str, Any] | None) -> str:
524
- """Wrap enum-backed scalar columns in CASE WHEN so SELECT shows Chinese labels."""
525
- if not schema or not sql.strip():
526
- return sql
527
- select_list = _top_level_select_list(sql)
528
- if select_list is None:
529
- return sql
530
- enums_by_field = {
531
- str(entry.get("field") or ""): entry.get("values")
532
- for entry in merged_schema_enums(schema)
533
- if isinstance(entry, dict) and entry.get("field")
534
- }
535
- table_aliases = _sql_table_aliases(sql)
536
- used_labels: set[str] = set()
537
- new_items: list[str] = []
538
- changed = False
539
- for item in _split_top_level_select_items(select_list):
540
- cleaned = item.strip()
541
- if not cleaned:
542
- new_items.append(cleaned)
543
- continue
544
- if re.match(r"case\s+", _select_expression_without_alias(cleaned), flags=re.IGNORECASE):
545
- alias = _extract_select_alias(cleaned)
546
- if alias:
547
- used_labels.add(alias)
548
- new_items.append(cleaned)
549
- continue
550
- expr = _select_expression_without_alias(cleaned)
551
- case_expr = _enum_case_expression_for_select_item(
552
- expr,
553
- table_aliases=table_aliases,
554
- enums_by_field=enums_by_field,
555
- schema=schema,
556
- used_labels=used_labels,
557
- existing_alias=_extract_select_alias(cleaned),
558
- )
559
- if case_expr is None:
560
- alias = _extract_select_alias(cleaned)
561
- if alias:
562
- used_labels.add(alias)
563
- new_items.append(cleaned)
564
- continue
565
- used_labels.add(_extract_select_alias(case_expr) or "")
566
- new_items.append(case_expr)
567
- changed = True
568
- if not changed:
569
- return sql
570
- return _replace_top_level_select_list(sql, ", ".join(new_items))
571
-
572
-
573
- def suggest_join_patterns_for_schema(schema: dict[str, Any]) -> list[dict[str, Any]]:
574
- suggestions: list[dict[str, Any]] = []
575
- for example in _schema_examples(schema):
576
- if not isinstance(example, dict):
577
- continue
578
- join_pattern = example.get("joinPattern")
579
- if not isinstance(join_pattern, dict):
580
- continue
581
- item: dict[str, Any] = {"joinPattern": join_pattern}
582
- for key in ("id", "intent", "notes", "questionExamples"):
583
- value = example.get(key)
584
- if value:
585
- item[key] = value
586
- suggestions.append(item)
587
- return suggestions[:6]
588
-
589
-
590
- def finalize_generated_sql(
591
- sql: str,
592
- question: str,
593
- *,
594
- schema: dict[str, Any] | None = None,
595
- ) -> str:
596
- normalized = normalize_text_filters_to_like(sql)
597
- normalized = strip_identifier_columns_from_select(normalized, question)
598
- normalized = apply_enum_case_to_select(normalized, schema)
599
- normalized = normalize_chinese_select_aliases(normalized, schema)
600
- return normalized
601
-
602
-
603
- def _select_expr_is_identifier_column(expr: str, table_aliases: dict[str, str]) -> bool:
604
- normalized = expr.strip()
605
- match = re.fullmatch(r"([a-zA-Z_][\w]*)\.([a-zA-Z_][\w]*)", normalized)
606
- if match is None:
607
- return normalized.lower() in {"id"} or normalized.lower().endswith("_id")
608
- alias, column = match.group(1).lower(), match.group(2).lower()
609
- if column == "id" or column.endswith("_id"):
610
- return True
611
- table = table_aliases.get(alias)
612
- if table and (column == "id" or column.endswith("_id")):
613
- return True
614
- return False
615
-
616
-
617
- def _enum_case_expression_for_select_item(
618
- expr: str,
619
- *,
620
- table_aliases: dict[str, str],
621
- enums_by_field: dict[str, Any],
622
- schema: dict[str, Any],
623
- used_labels: set[str],
624
- existing_alias: str | None,
625
- ) -> str | None:
626
- match = re.fullmatch(r"([a-zA-Z_][\w]*)\.([a-zA-Z_][\w]*)", expr.strip())
627
- if match is None:
628
- return None
629
- alias, column = match.group(1), match.group(2)
630
- table = table_aliases.get(alias.lower())
631
- if table is None:
632
- return None
633
- values = enums_by_field.get(f"{table}.{column}")
634
- if not isinstance(values, list) or not values:
635
- return None
636
- when_parts: list[str] = []
637
- for item in values:
638
- if not isinstance(item, dict):
639
- continue
640
- value = item.get("value")
641
- label = str(item.get("label") or "").strip()
642
- if value is None or not label:
643
- continue
644
- literal = repr(str(value)) if isinstance(value, str) else str(value)
645
- when_parts.append(f"WHEN {expr} = {literal} THEN {_sql_string_literal(label)}")
646
- if not when_parts:
647
- return None
648
- alias_label = existing_alias or _chinese_label_for_column(schema, table, column) or column
649
- alias_label = _dedupe_chinese_alias(alias_label, used_labels)
650
- return f"CASE {' '.join(when_parts)} ELSE CAST({expr} AS CHAR) END AS {alias_label}"
651
-
652
-
653
- def _sql_string_literal(value: str) -> str:
654
- return "'" + value.replace("'", "''") + "'"
655
-
656
-
657
- def _enforce_no_tautological_join_conditions(sql: str) -> None:
658
- if re.search(
659
- r"\b(?:left|right|inner|full|cross)?\s*join\b[\s\S]*?\bon\s+(?:1\s*=\s*0|0\s*=\s*1|1\s*=\s*1|true|false)\b",
660
- sql,
661
- flags=re.IGNORECASE,
662
- ):
663
- raise SqlGenerationError(
664
- "SQL JOIN must not use tautological ON conditions such as ON 1=0 or ON 1=1; "
665
- "use schema exampleJoinPatterns and structureHints to build real joins"
666
- )
667
-
668
-
669
- _JOIN_ON_START_RE = re.compile(
670
- r"\b(?:(?:left|right|inner|full|cross)\s+)?join\s+(?:\([^)]+\)|[a-zA-Z_][\w]*(?:\s+(?:as\s+)?[a-zA-Z_][\w]*)?)\s+on\s+",
671
- re.IGNORECASE,
672
- )
673
- _JOIN_ON_END_RE = re.compile(
674
- r"\b(where|group\s+by|order\s+by|limit|(?:left|right|inner|full|cross)\s+join)\b",
675
- re.IGNORECASE,
676
- )
677
- _CROSS_TABLE_EQUALITY_RE = re.compile(
678
- r"(?:"
679
- r"\b[a-zA-Z_][a-zA-Z0-9_]*\.[a-zA-Z_][a-zA-Z0-9_]*\s*=\s*"
680
- r"(?:cast\s*\(\s*[a-zA-Z_][a-zA-Z0-9_]*\.[a-zA-Z_][a-zA-Z0-9_]*\s+as\s+\w+\s*\)"
681
- r"|[a-zA-Z_][a-zA-Z0-9_]*\.[a-zA-Z_][a-zA-Z0-9_]*)"
682
- r"|"
683
- r"cast\s*\(\s*[a-zA-Z_][a-zA-Z0-9_]*\.[a-zA-Z_][a-zA-Z0-9_]*\s+as\s+\w+\s*\)\s*=\s*"
684
- r"[a-zA-Z_][a-zA-Z0-9_]*\.[a-zA-Z_][a-zA-Z0-9_]*)",
685
- re.IGNORECASE,
686
- )
687
-
688
-
689
- def _iter_join_on_clauses(sql: str) -> list[str]:
690
- clauses: list[str] = []
691
- for match in _JOIN_ON_START_RE.finditer(sql):
692
- rest = sql[match.end() :]
693
- end_match = _JOIN_ON_END_RE.search(rest)
694
- end = end_match.start() if end_match else len(rest)
695
- clause = rest[:end].strip()
696
- if clause:
697
- clauses.append(clause)
698
- return clauses
699
-
700
-
701
- def _list_join_on_violations(sql: str) -> list[str]:
702
- violations: list[str] = []
703
- for on_clause in _iter_join_on_clauses(sql):
704
- preview = re.sub(r"\s+", " ", on_clause).strip()[:120]
705
- if re.search(r"\blike\b", on_clause, re.IGNORECASE):
706
- violations.append(
707
- "JOIN ON 禁止写 LIKE 筛选,请按 schema relations 写主表与关联表的等值键,"
708
- f"把实体筛选放到 WHERE:{preview}"
709
- )
710
- continue
711
- if re.search(r"\bin\s*\(", on_clause, re.IGNORECASE):
712
- violations.append(
713
- "JOIN ON 禁止写 IN 筛选,请按 schema relations 写主表与关联表的等值键,"
714
- f"把实体筛选放到 WHERE:{preview}"
715
- )
716
- continue
717
- if _CROSS_TABLE_EQUALITY_RE.search(on_clause) is None:
718
- violations.append(
719
- "JOIN ON 必须使用主表与关联表的等值关联(如 store.organization_id = sponge_project.id),"
720
- f"不能只有筛选条件:{preview}"
721
- )
722
- return violations
723
-
724
-
725
- def _enforce_join_on_uses_equality_keys(sql: str) -> None:
726
- violations = _list_join_on_violations(sql)
727
- if violations:
728
- raise SqlGenerationError(violations[0])
729
-
730
-
731
- _AGGREGATE_FUNCTION_RE = re.compile(
732
- r"\b(?:count|sum|avg|min|max|group_concat)\s*\(",
733
- flags=re.IGNORECASE,
734
- )
735
-
736
-
737
- def _enforce_no_aggregate_functions_in_row_filters(sql: str) -> None:
738
- for clause in _iter_join_on_clauses(sql):
739
- if _AGGREGATE_FUNCTION_RE.search(clause):
740
- raise SqlGenerationError(
741
- "SQL aggregate functions must not be used in JOIN ON; "
742
- "aggregate first in a subquery or move aggregate predicates to HAVING"
743
- )
744
- where_clause = _top_level_where_clause(sql)
745
- if where_clause and _AGGREGATE_FUNCTION_RE.search(where_clause):
746
- raise SqlGenerationError(
747
- "SQL aggregate functions must not be used in WHERE; "
748
- "move aggregate predicates to HAVING or aggregate first in a subquery"
749
- )
750
-
751
-
752
- def _top_level_where_clause(sql: str) -> str:
753
- where_index = _find_top_level_keyword(sql, "where", start=0)
754
- if where_index is None:
755
- return ""
756
- body_start = where_index + len("where")
757
- suffix_starts = [
758
- index
759
- for keyword in ("group by", "having", "order by", "limit")
760
- if (index := _find_top_level_keyword(sql, keyword, start=body_start)) is not None
761
- ]
762
- body_end = min(suffix_starts) if suffix_starts else len(sql)
763
- return sql[body_start:body_end]
764
-
765
-
766
- def _select_item_has_chinese_alias(item: str) -> bool:
767
- alias = _extract_select_alias(item)
768
- return alias is not None and re.search(r"[\u4e00-\u9fff]", alias) is not None
769
-
770
-
771
- def _extract_select_alias(item: str) -> str | None:
772
- alias_match = re.search(
773
- _SELECT_AS_ALIAS_PATTERN,
774
- item.strip(),
775
- flags=re.IGNORECASE,
776
- )
777
- if alias_match is None:
778
- return None
779
- return alias_match.group(1).strip("`\"'")
780
-
781
-
782
- def _dedupe_chinese_alias(label: str, used: set[str]) -> str:
783
- if label not in used:
784
- return label
785
- index = 2
786
- while f"{label}{index}" in used:
787
- index += 1
788
- return f"{label}{index}"
789
-
790
-
791
- def _infer_chinese_select_alias(
792
- expr: str,
793
- *,
794
- table_aliases: dict[str, str],
795
- schema: dict[str, Any],
796
- ) -> str | None:
797
- normalized = expr.strip()
798
- if not normalized:
799
- return None
800
-
801
- aggregate_match = re.match(r"(count|sum|avg|min|max)\s*\(", normalized, flags=re.IGNORECASE)
802
- if aggregate_match is not None:
803
- labels = {
804
- "count": "数量",
805
- "sum": "合计",
806
- "avg": "平均值",
807
- "min": "最小值",
808
- "max": "最大值",
809
- }
810
- return labels.get(aggregate_match.group(1).lower())
811
-
812
- if re.match(r"case\s+", normalized, flags=re.IGNORECASE):
813
- return "结果"
814
-
815
- qualified_match = re.search(
816
- r"(?:([a-zA-Z_][a-zA-Z0-9_]*)\.)?([a-zA-Z_][a-zA-Z0-9_]*)\s*$",
817
- normalized,
818
- )
819
- if qualified_match is not None:
820
- alias_part, column_name = qualified_match.group(1), qualified_match.group(2)
821
- if alias_part is not None:
822
- table_name = table_aliases.get(alias_part.lower())
823
- if table_name is not None:
824
- return _chinese_label_for_column(schema, table_name, column_name)
825
- return _chinese_label_for_bare_column(schema, table_aliases, column_name)
826
- return None
827
-
828
-
829
- def _chinese_label_for_column(schema: dict[str, Any], table_name: str, column_name: str) -> str | None:
830
- for column in _table_columns(schema, table_name):
831
- name = str(column.get("columnName") or column.get("name") or "")
832
- if name.lower() != column_name.lower():
833
- continue
834
- label = _column_alias(name, str(column.get("comment") or ""))
835
- if re.search(r"[\u4e00-\u9fff]", label):
836
- return label
837
- return None
838
-
839
-
840
- def _chinese_label_for_bare_column(
841
- schema: dict[str, Any],
842
- table_aliases: dict[str, str],
843
- column_name: str,
844
- ) -> str | None:
845
- labels = [
846
- label
847
- for table_name in set(table_aliases.values())
848
- if (label := _chinese_label_for_column(schema, table_name, column_name)) is not None
849
- ]
850
- if len(labels) == 1:
851
- return labels[0]
852
- return None
853
-
854
-
855
- def _replace_top_level_select_list(sql: str, select_list: str) -> str:
856
- stripped = sql.strip().rstrip(";")
857
- match = re.match(r"select\s+(?:distinct\s+)?", stripped, flags=re.IGNORECASE)
858
- if match is None:
859
- return sql
860
- start = match.end()
861
- from_index = _find_top_level_keyword(stripped, "from", start=start)
862
- if from_index is None:
863
- return sql
864
- prefix = stripped[:start].rstrip()
865
- suffix = stripped[from_index:].lstrip()
866
- return f"{prefix} {select_list} {suffix}"
867
-
868
-
869
- def _should_normalize_text_filter_to_like(column_name: str, value: str) -> bool:
870
- if not value or _is_numeric_literal(value):
871
- return False
872
- return _is_name_filter_column(column_name) or _is_status_text_filter_column(column_name)
873
-
874
-
875
- def _enforce_text_filters_use_like(sql: str) -> None:
876
- for match in _TEXT_EQUALITY_FILTER_RE.finditer(sql):
877
- column = match.group("column").rsplit(".", 1)[-1].lower()
878
- value = match.group("value").strip()
879
- if _should_normalize_text_filter_to_like(column, value):
880
- raise SqlGenerationError("SQL text filters must use LIKE")
881
-
882
-
883
- def _is_name_filter_column(column: str) -> bool:
884
- return column == "name" or column == "job_name" or column.endswith("_name")
885
-
886
-
887
- def _is_status_text_filter_column(column: str) -> bool:
888
- return column in {"status", "state"} or column.endswith("_status") or column.endswith("_state")
889
-
890
-
891
- def _is_numeric_literal(value: str) -> bool:
892
- return bool(re.fullmatch(r"-?\d+(?:\.\d+)?", value))
893
-
894
-
895
- def is_low_quality_sql_for_question(question: str, sql: str, *, schema: dict[str, Any] | None = None) -> bool:
896
- return low_quality_sql_issue_for_question(question, sql, schema=schema) is not None
897
-
898
-
899
- def low_quality_sql_issue_for_question(question: str, sql: str, *, schema: dict[str, Any] | None = None) -> LowQualitySqlIssue | None:
900
- if schema is None:
901
- return None
902
- from .schema_context_retriever import match_concepts_for_question
903
-
904
- matched_concepts = match_concepts_for_question(_schema_concepts(schema), question)
905
- normalized = sql.strip().lower()
906
- for concept in matched_concepts:
907
- checks = concept.get("sqlQualityChecks")
908
- if not _concept_quality_check_applies(concept, question):
909
- continue
910
- time_range_issue = _concept_time_range_field_issue(question, sql, concept)
911
- if time_range_issue is not None:
912
- return time_range_issue
913
- if not isinstance(checks, dict):
914
- continue
915
- for table_name in checks.get("requireTables") or []:
916
- table = str(table_name or "").strip().lower()
917
- if table and table not in normalized:
918
- message = str(checks.get("message") or f"SQL 必须引用 schema 要求的表:{table_name}")
919
- concept_label = str(concept.get("comment") or concept.get("name") or "命中的 schema concept")
920
- return LowQualitySqlIssue(
921
- code="SCHEMA_SQL_QUALITY",
922
- message=message,
923
- missing_items=(f"必须引用表 {table_name}",),
924
- current_problem=(
925
- f"当前 SQL 没有使用 {table_name},无法覆盖“{concept_label}”要求的查询口径。"
926
- ),
927
- repair_hint=(
928
- f"按 schema.concepts 中“{concept_label}”的定义补充 {table_name} 及对应 JOIN/WHERE 条件。"
929
- ),
930
- )
931
- aggregate_issue = _aggregate_sql_quality_issue(question, sql)
932
- if aggregate_issue is not None:
933
- return aggregate_issue
934
- return None
935
-
936
-
937
- def _concept_time_range_field_issue(
938
- question: str,
939
- sql: str,
940
- concept: dict[str, Any],
941
- ) -> LowQualitySqlIssue | None:
942
- qualified = str(concept.get("timeRangeField") or "").strip()
943
- if "." not in qualified or _extract_time_bounds(question) is None:
944
- return None
945
- table_name, field_name = qualified.rsplit(".", 1)
946
- time_range = TimeRange(field=field_name, alias="时间范围", start="", end="", table_name=table_name)
947
- if not _sql_has_time_filter_for_range(sql, time_range):
948
- return LowQualitySqlIssue(
949
- code="SCHEMA_TIME_RANGE_FIELD_REQUIRED",
950
- message=(
951
- f"用户问题包含时间范围,且 schema concept 指定时间字段为 {qualified};"
952
- "SQL 必须使用该字段过滤时间范围,不要改用关联维表的同名时间字段。"
953
- ),
954
- missing_items=(f"时间范围字段 {qualified}",),
955
- current_problem=f"当前 SQL 没有使用 {qualified} 过滤用户要求的时间范围。",
956
- repair_hint=f"在 WHERE 中使用 {qualified} 补充时间范围过滤。",
957
- )
958
- unexpected = _unexpected_time_filter_fields(sql, expected_field=qualified)
959
- if unexpected:
960
- return LowQualitySqlIssue(
961
- code="SCHEMA_TIME_RANGE_FIELD_REQUIRED",
962
- message=(
963
- f"用户问题包含时间范围,且 schema concept 指定时间字段为 {qualified};"
964
- f"SQL 不应同时使用 {'、'.join(unexpected)} 过滤时间范围。"
965
- ),
966
- missing_items=(f"唯一时间范围字段 {qualified}",),
967
- current_problem=f"当前 SQL 还使用了 {'、'.join(unexpected)} 过滤时间范围,可能把口径切到关联表时间。",
968
- repair_hint=f"删除关联表时间过滤,只保留 {qualified} 的时间范围条件。",
969
- )
970
- return None
971
-
972
-
973
- _TIME_RANGE_DATE_LITERAL = r"'20\d{2}-\d{1,2}-\d{1,2}(?:\s+\d{1,2}:\d{2}:\d{2})?'"
974
-
975
-
976
- def _sql_has_time_filter_for_range(sql: str, time_range: TimeRange) -> bool:
977
- if _scope_has_time_filter_for_range(sql, time_range):
978
- return True
979
- for subquery in _sql_derived_subqueries(sql).values():
980
- if _sql_has_time_filter_for_range(subquery, time_range):
981
- return True
982
- return False
983
-
984
-
985
- def _mask_derived_subqueries(sql: str) -> str:
986
- spans = _sql_derived_subquery_spans(sql)
987
- if not spans:
988
- return sql
989
- chars = list(sql)
990
- for start, end in spans:
991
- for index in range(start, min(end, len(chars))):
992
- chars[index] = " "
993
- return "".join(chars)
994
-
995
-
996
- def _scope_has_time_filter_for_range(sql: str, time_range: TimeRange) -> bool:
997
- """Return True if the designated time field is compared to a date literal in this
998
- query scope.
999
-
1000
- Nested derived subqueries are masked out here and evaluated separately by
1001
- ``_sql_has_time_filter_for_range`` so alias resolution stays scoped. The check
1002
- scans the whole scope (not just the top-level WHERE) so the field can satisfy the
1003
- requirement whether it is used in WHERE or inside a conditional aggregate such as
1004
- ``SUM(CASE WHEN sign_up_time >= ... THEN 1 ELSE 0 END)``, which some schema concepts
1005
- mandate for period metrics.
1006
- """
1007
- scope_text = _mask_derived_subqueries(sql)
1008
- op = r"(?:>=|<=|>|<|=)"
1009
- date = _TIME_RANGE_DATE_LITERAL
1010
- field_patterns: list[str] = [_time_range_sql_field_pattern(scope_text, time_range)]
1011
- base_tables = set(_sql_top_level_table_aliases(scope_text).values())
1012
- if not time_range.table_name or base_tables == {time_range.table_name.lower()}:
1013
- # Single base table in scope: an unqualified column reference is unambiguous.
1014
- field_patterns.append(rf"(?<![.\w]){re.escape(time_range.field.lower())}\b")
1015
- for field_pattern in field_patterns:
1016
- checks = (
1017
- rf"{field_pattern}\s*{op}\s*{date}",
1018
- rf"{date}\s*{op}\s*{field_pattern}",
1019
- rf"{field_pattern}\s+between\s+{date}\s+and\s+{date}",
1020
- )
1021
- if any(re.search(pattern, scope_text, flags=re.IGNORECASE) for pattern in checks):
1022
- return True
1023
- return False
1024
-
1025
-
1026
- def _unexpected_time_filter_fields(sql: str, *, expected_field: str) -> list[str]:
1027
- expected_table, expected_column = expected_field.lower().rsplit(".", 1)
1028
- aliases = _sql_table_aliases(sql)
1029
- unexpected: list[str] = []
1030
- seen: set[str] = set()
1031
- where_index = _find_top_level_keyword(sql, "where", start=0)
1032
- if where_index is None:
1033
- return []
1034
- clause_start = where_index + len("where")
1035
- clause_end = _where_filter_insert_pos(sql[clause_start:])
1036
- if clause_end < len(sql[clause_start:]):
1037
- clause_end += clause_start
1038
- else:
1039
- clause_end = len(sql)
1040
- for predicate in _split_top_level_and_predicates(sql[clause_start:clause_end].strip()):
1041
- if re.search(r"'20\d{2}-\d{1,2}-\d{1,2}", predicate) is None:
1042
- continue
1043
- if re.search(r"(?:\bbetween\b|>=|>|<=|<|=)", predicate, flags=re.IGNORECASE) is None:
1044
- continue
1045
- for match in re.finditer(r"\b([a-zA-Z_][\w]*)\.([a-zA-Z_][\w]*)\b", predicate):
1046
- qualifier = match.group(1).lower()
1047
- column = match.group(2).lower()
1048
- if column != expected_column:
1049
- continue
1050
- table = aliases.get(qualifier, qualifier)
1051
- qualified = f"{table}.{column}"
1052
- if qualified == f"{expected_table}.{expected_column}" or qualified in seen:
1053
- continue
1054
- seen.add(qualified)
1055
- unexpected.append(qualified)
1056
- return unexpected
1057
-
1058
-
1059
- def _aggregate_sql_quality_issue(question: str, sql: str) -> LowQualitySqlIssue | None:
1060
- if not _question_requires_aggregate_sql(question):
1061
- return None
1062
- normalized = sql.lower()
1063
- if re.search(r"\b(count|sum|avg|min|max)\s*\(", normalized):
1064
- return None
1065
- return LowQualitySqlIssue(
1066
- code="AGGREGATE_SQL_REQUIRED",
1067
- message="用户问题要求统计数量/最多值及对应维度,SQL 必须使用 COUNT/SUM/AVG/MIN/MAX 等聚合函数,不能只返回明细行。",
1068
- missing_items=("聚合函数 COUNT/SUM/AVG/MIN/MAX",),
1069
- current_problem="当前 SQL 返回明细行,不能回答数量、最多值、排名或按维度统计问题。",
1070
- repair_hint="按用户问题中的统计维度 GROUP BY,并用聚合函数生成指标。",
1071
- )
1072
-
1073
-
1074
- def format_sql_quality_issue(issue: LowQualitySqlIssue) -> str:
1075
- parts = [f"当前 SQL 不满足查询口径:{issue.message}"]
1076
- if issue.missing_items:
1077
- parts.append(f"缺失口径:{';'.join(issue.missing_items)}")
1078
- if issue.current_problem:
1079
- parts.append(f"当前 SQL 的问题:{issue.current_problem}")
1080
- if issue.repair_hint:
1081
- parts.append(f"建议修复方向:{issue.repair_hint}")
1082
- return ";".join(parts)
1083
-
1084
-
1085
- def _question_requires_aggregate_sql(question: str) -> bool:
1086
- if not re.search(r"(计数|统计|数量|个数|总数|多少)", question):
1087
- return False
1088
- return bool(re.search(r"(最多|最少|最大|最小|排行|排名|对应|按.+?统计|根据.+?计数)", question))
1089
-
1090
-
1091
- def _concept_quality_check_applies(concept: dict[str, Any], question: str) -> bool:
1092
- question_text = _compact_quality_match_text(question)
1093
- if not question_text:
1094
- return False
1095
- for example in concept.get("negativeExamples") or []:
1096
- if _quality_phrase_matches(question_text, example):
1097
- return False
1098
- for example in concept.get("positiveExamples") or []:
1099
- if _quality_phrase_matches(question_text, example):
1100
- return True
1101
- for example in concept.get("questionExamples") or []:
1102
- if _quality_phrase_matches(question_text, example):
1103
- return True
1104
- for key in ("comment", "description", "intent", "name", "term"):
1105
- if _quality_phrase_matches(question_text, concept.get(key)):
1106
- return True
1107
- return False
1108
-
1109
-
1110
- def _quality_phrase_matches(question_text: str, phrase: Any) -> bool:
1111
- phrase_text = _compact_quality_match_text(str(phrase or ""))
1112
- return len(phrase_text) >= 2 and phrase_text in question_text
1113
-
1114
-
1115
- def _compact_quality_match_text(text: str) -> str:
1116
- normalized = text.translate(str.maketrans("0123456789", "0123456789")).lower()
1117
- return "".join(re.findall(r"[a-z0-9\u4e00-\u9fff]+", normalized))
1118
-
1119
-
1120
- def _sql_select_metric_aliases(sql: str) -> list[str]:
1121
- select_list = _top_level_select_list(sql)
1122
- if select_list is None:
1123
- return []
1124
- aliases: list[str] = []
1125
- for item in _split_top_level_select_items(select_list):
1126
- alias_match = re.search(
1127
- r"\s+as\s+(`[^`]+`|\"[^\"]+\"|'[^']+'|[\u4e00-\u9fff][\w\u4e00-\u9fff]*)\s*$",
1128
- item.strip(),
1129
- flags=re.IGNORECASE,
1130
- )
1131
- if alias_match is not None:
1132
- aliases.append(alias_match.group(1).strip("`\"'"))
1133
- return aliases
1134
-
1135
-
1136
- def _replace_primary_metric_alias(sql: str, alias: str) -> str:
1137
- select_list = _top_level_select_list(sql)
1138
- if select_list is None:
1139
- return sql
1140
- items = _split_top_level_select_items(select_list)
1141
- if not items:
1142
- return sql
1143
- last_item = items[-1].strip()
1144
- updated_last = re.sub(
1145
- r"\s+as\s+(`[^`]+`|\"[^\"]+\"|'[^']+'|[\u4e00-\u9fff][\w\u4e00-\u9fff]*)\s*$",
1146
- f" AS {alias}",
1147
- last_item,
1148
- count=1,
1149
- flags=re.IGNORECASE,
1150
- )
1151
- if updated_last == last_item:
1152
- return sql
1153
- items[-1] = updated_last
1154
- rebuilt_select = ", ".join(items)
1155
- return re.sub(
1156
- r"(?is)\bselect\b(.*?)\bfrom\b",
1157
- lambda match: f"SELECT {rebuilt_select} FROM",
1158
- sql,
1159
- count=1,
1160
- )
1161
-
1162
-
1163
- def _schema_enums(schema: dict[str, Any]) -> list[dict[str, Any]]:
1164
- enums = schema.get("enums")
1165
- if isinstance(enums, list):
1166
- return [enum for enum in enums if isinstance(enum, dict)]
1167
- structure = schema.get("structure")
1168
- concepts = structure.get("concepts") if isinstance(structure, dict) else None
1169
- if not isinstance(concepts, list):
1170
- return []
1171
- by_field: dict[str, dict[Any, str]] = {}
1172
- for concept in concepts:
1173
- if not isinstance(concept, dict):
1174
- continue
1175
- enum_refs = concept.get("enumRefs")
1176
- if not isinstance(enum_refs, list):
1177
- continue
1178
- label_parts = [
1179
- str(concept.get("comment") or ""),
1180
- str(concept.get("name") or ""),
1181
- str(concept.get("description") or ""),
1182
- ]
1183
- for enum_ref in enum_refs:
1184
- if not isinstance(enum_ref, dict):
1185
- continue
1186
- field_name = str(enum_ref.get("field") or "")
1187
- if not field_name or "value" not in enum_ref:
1188
- continue
1189
- label = " / ".join(part for part in label_parts + [str(enum_ref.get("meaning") or "")] if part)
1190
- by_field.setdefault(field_name, {})[enum_ref["value"]] = label
1191
- return [
1192
- {"field": field_name, "values": [{"value": value, "label": label} for value, label in values.items()]}
1193
- for field_name, values in by_field.items()
1194
- ]
1195
-
1196
-
1197
- def _enum_label_terms(label: str) -> set[str]:
1198
- terms = {label.strip()}
1199
- terms.update(part.strip() for part in re.split(r"[/、,,;;\s]+", label) if part.strip())
1200
- return {term for term in terms if term}
1201
-
1202
-
1203
- def validate_sql_against_schema(sql: str, schema: dict[str, Any]) -> list[SchemaValidationIssue]:
1204
- issues: list[SchemaValidationIssue] = []
1205
- for subquery in _sql_derived_subqueries(sql).values():
1206
- issues.extend(validate_sql_against_schema(subquery, schema))
1207
- table_aliases = _sql_table_aliases(sql)
1208
- derived_columns = _sql_derived_subquery_aliases(sql)
1209
- schema_tables = _schema_table_column_map(schema)
1210
- for alias, columns in derived_columns.items():
1211
- schema_tables[alias] = columns
1212
- for table in set(table_aliases.values()):
1213
- if table in derived_columns:
1214
- continue
1215
- if table not in schema_tables:
1216
- issues.append(
1217
- SchemaValidationIssue(
1218
- code="UNKNOWN_TABLE",
1219
- message=f"SQL 使用了 schema 中不存在的表:{table}",
1220
- )
1221
- )
1222
- issues.extend(_schema_missing_table_reference_issues(sql, table_aliases))
1223
- issues.extend(_schema_unknown_column_issues(sql, table_aliases, schema_tables))
1224
- for message in _list_join_on_violations(sql):
1225
- issues.append(SchemaValidationIssue(code="JOIN_ON_FILTER", message=message))
1226
- if not any(issue.code == "UNKNOWN_TABLE" for issue in issues):
1227
- issues.extend(_schema_invalid_join_issues(sql, table_aliases, schema))
1228
- return issues
1229
-
1230
-
1231
- def _schema_table_column_map(schema: dict[str, Any]) -> dict[str, set[str]]:
1232
- column_map: dict[str, set[str]] = {}
1233
- tables = schema.get("tables")
1234
- if not isinstance(tables, list):
1235
- nested_schema = schema.get("schema")
1236
- if isinstance(nested_schema, dict) and isinstance(nested_schema.get("tables"), list):
1237
- tables = nested_schema["tables"]
1238
- else:
1239
- tables = []
1240
- for table in tables:
1241
- if not isinstance(table, dict):
1242
- continue
1243
- name = table.get("name") or table.get("tableName")
1244
- if not name:
1245
- continue
1246
- table_key = str(name).lower()
1247
- columns = table.get("columns")
1248
- if not isinstance(columns, list):
1249
- continue
1250
- column_map[table_key] = {
1251
- str(column.get("columnName") or column.get("name") or "").lower()
1252
- for column in columns
1253
- if isinstance(column, dict) and (column.get("columnName") or column.get("name"))
1254
- }
1255
- return column_map
1256
-
1257
-
1258
- def _schema_missing_table_reference_issues(
1259
- sql: str,
1260
- table_aliases: dict[str, str],
1261
- ) -> list[SchemaValidationIssue]:
1262
- declared = set(table_aliases)
1263
- issues: list[SchemaValidationIssue] = []
1264
- seen: set[tuple[str, str]] = set()
1265
- for alias, column in _sql_qualified_column_refs(sql):
1266
- if alias in declared:
1267
- continue
1268
- key = (alias, column)
1269
- if key in seen:
1270
- continue
1271
- seen.add(key)
1272
- issues.append(
1273
- SchemaValidationIssue(
1274
- code="MISSING_TABLE_REFERENCE",
1275
- message=(
1276
- "SQL 引用了未在 FROM/JOIN 中声明的表或别名:"
1277
- f"{alias}.{column};如果需要该字段必须先 JOIN 对应表,否则删除该引用。"
1278
- ),
1279
- )
1280
- )
1281
- return issues
1282
-
1283
-
1284
- def _schema_unknown_column_issues(
1285
- sql: str,
1286
- table_aliases: dict[str, str],
1287
- schema_tables: dict[str, set[str]],
1288
- ) -> list[SchemaValidationIssue]:
1289
- issues: list[SchemaValidationIssue] = []
1290
- seen: set[tuple[str, str]] = set()
1291
- for alias, column in _sql_qualified_column_refs(sql):
1292
- table = table_aliases.get(alias)
1293
- if table is None or table not in schema_tables:
1294
- continue
1295
- key = (table, column)
1296
- if key in seen:
1297
- continue
1298
- seen.add(key)
1299
- if column not in schema_tables[table]:
1300
- issues.append(
1301
- SchemaValidationIssue(
1302
- code="UNKNOWN_COLUMN",
1303
- message=f"SQL 使用了 schema 中不存在的列:{table}.{column}",
1304
- )
1305
- )
1306
- return issues
1307
-
1308
-
1309
- def _schema_invalid_join_issues(
1310
- sql: str,
1311
- table_aliases: dict[str, str],
1312
- schema: dict[str, Any],
1313
- ) -> list[SchemaValidationIssue]:
1314
- allowed_pairs = _schema_allowed_join_pairs(schema)
1315
- if not allowed_pairs:
1316
- return []
1317
- derived_aliases = set(_sql_derived_subquery_aliases(sql))
1318
- issues: list[SchemaValidationIssue] = []
1319
- seen: set[tuple[tuple[str, str], tuple[str, str]]] = set()
1320
- for pair in _sql_equality_pairs(sql, table_aliases):
1321
- if pair in seen:
1322
- continue
1323
- seen.add(pair)
1324
- if pair in allowed_pairs:
1325
- continue
1326
- left, right = pair
1327
- if left[0] in derived_aliases or right[0] in derived_aliases:
1328
- continue
1329
- issues.append(
1330
- SchemaValidationIssue(
1331
- code="INVALID_JOIN",
1332
- message=(
1333
- "SQL JOIN 使用了 schema relations / joinPattern 未定义的关联:"
1334
- f"{left[0]}.{left[1]} = {right[0]}.{right[1]}"
1335
- ),
1336
- )
1337
- )
1338
- return issues
1339
-
1340
-
1341
- def _sql_qualified_column_refs(sql: str) -> list[tuple[str, str]]:
1342
- refs: list[tuple[str, str]] = []
1343
- seen: set[tuple[str, str]] = set()
1344
- derived_spans = _sql_derived_subquery_spans(sql)
1345
- for match in re.finditer(
1346
- r"\b([a-zA-Z_][a-zA-Z0-9_]*)\.((?:[a-zA-Z_][a-zA-Z0-9_]*\.)*[a-zA-Z_][a-zA-Z0-9_]*)\b",
1347
- sql,
1348
- flags=re.IGNORECASE,
1349
- ):
1350
- if _position_in_spans(match.start(), derived_spans):
1351
- continue
1352
- alias = match.group(1).lower()
1353
- column_path = match.group(2).lower()
1354
- column = column_path.rsplit(".", 1)[-1]
1355
- key = (alias, column)
1356
- if key in seen:
1357
- continue
1358
- seen.add(key)
1359
- refs.append(key)
1360
- return refs
1361
-
1362
-
1363
- def _sql_table_aliases(sql: str) -> dict[str, str]:
1364
- aliases = dict(_sql_top_level_table_aliases(sql))
1365
- for alias in _sql_derived_subquery_aliases(sql):
1366
- aliases[alias] = alias
1367
- return aliases
1368
-
1369
-
1370
- def _sql_top_level_table_aliases(sql: str) -> dict[str, str]:
1371
- aliases: dict[str, str] = {}
1372
- depth = 0
1373
- quote: str | None = None
1374
- index = 0
1375
- length = len(sql)
1376
- while index < length:
1377
- char = sql[index]
1378
- if quote is not None:
1379
- if char == quote:
1380
- quote = None
1381
- elif char == "\\":
1382
- index += 1
1383
- index += 1
1384
- continue
1385
- if char in ("'", '"', "`"):
1386
- quote = char
1387
- index += 1
1388
- continue
1389
- if char == "(":
1390
- depth += 1
1391
- index += 1
1392
- continue
1393
- if char == ")":
1394
- depth = max(0, depth - 1)
1395
- index += 1
1396
- continue
1397
- if depth == 0:
1398
- match = re.match(
1399
- r"(?i)(?:from|join)\s+([a-zA-Z_][a-zA-Z0-9_]*)(?:\s+(?:as\s+)?([a-zA-Z_][a-zA-Z0-9_]*))?",
1400
- sql[index:],
1401
- )
1402
- if match:
1403
- table = match.group(1).lower()
1404
- alias = (match.group(2) or table).lower()
1405
- if alias in {"on", "where", "left", "right", "inner", "outer", "join"}:
1406
- alias = table
1407
- aliases[alias] = table
1408
- aliases[table] = table
1409
- index += match.end()
1410
- continue
1411
- index += 1
1412
- return aliases
1413
-
1414
-
1415
- def _sql_derived_subquery_aliases(sql: str) -> dict[str, set[str]]:
1416
- return {
1417
- alias: _extract_subquery_output_columns(subquery)
1418
- for alias, subquery in _sql_derived_subqueries(sql).items()
1419
- }
1420
-
1421
-
1422
- def _sql_derived_subqueries(sql: str) -> dict[str, str]:
1423
- derived: dict[str, str] = {}
1424
- for alias, subquery_start, subquery_end in _sql_derived_subquery_matches(sql):
1425
- derived[alias] = sql[subquery_start + 1 : subquery_end]
1426
- return derived
1427
-
1428
-
1429
- def _sql_derived_subquery_spans(sql: str) -> list[tuple[int, int]]:
1430
- return [
1431
- (subquery_start + 1, subquery_end)
1432
- for _alias, subquery_start, subquery_end in _sql_derived_subquery_matches(sql)
1433
- ]
1434
-
1435
-
1436
- _DERIVED_SUBQUERY_TRAILING = (
1437
- r"on\b"
1438
- r"|(?:(?:left|right|inner|outer|cross|natural)\s+)?join\b"
1439
- r"|where\b|group\b|order\b|having\b|limit\b"
1440
- r"|,"
1441
- r"|\)(?!\s*[a-zA-Z_])"
1442
- r"|;"
1443
- )
1444
- _DERIVED_SUBQUERY_ALIAS_PATTERN = re.compile(
1445
- rf"\)\s*(?:as\s+)?([a-zA-Z_][a-zA-Z0-9_]*)(?:\s+(?:{_DERIVED_SUBQUERY_TRAILING})|\s*$)",
1446
- flags=re.IGNORECASE,
1447
- )
1448
- _RESERVED_DERIVED_ALIASES = frozenset(
1449
- {"on", "where", "left", "right", "inner", "outer", "join", "select", "cross", "natural", "group", "order", "having", "limit"}
1450
- )
1451
-
1452
-
1453
- def _sql_derived_subquery_matches(sql: str) -> list[tuple[str, int, int]]:
1454
- matches: list[tuple[str, int, int]] = []
1455
- seen_ranges: set[tuple[int, int]] = set()
1456
- for match in _DERIVED_SUBQUERY_ALIAS_PATTERN.finditer(sql):
1457
- alias = match.group(1).lower()
1458
- if alias in _RESERVED_DERIVED_ALIASES:
1459
- continue
1460
- subquery_start = _find_matching_open_paren(sql, match.start())
1461
- if subquery_start is None:
1462
- continue
1463
- range_key = (subquery_start, match.start())
1464
- if range_key in seen_ranges:
1465
- continue
1466
- seen_ranges.add(range_key)
1467
- matches.append((alias, subquery_start, match.start()))
1468
- return matches
1469
-
1470
-
1471
- def _position_in_spans(position: int, spans: list[tuple[int, int]]) -> bool:
1472
- return any(start <= position < end for start, end in spans)
1473
-
1474
-
1475
- def _find_matching_open_paren(sql: str, close_index: int) -> int | None:
1476
- depth = 0
1477
- for index in range(close_index, -1, -1):
1478
- char = sql[index]
1479
- if char == ")":
1480
- depth += 1
1481
- elif char == "(":
1482
- depth -= 1
1483
- if depth == 0:
1484
- return index
1485
- return None
1486
-
1487
-
1488
- def _extract_subquery_output_columns(subquery: str) -> set[str]:
1489
- select_match = re.search(r"\bselect\b(.*?)\bfrom\b", subquery, flags=re.IGNORECASE | re.DOTALL)
1490
- if select_match is None:
1491
- return set()
1492
- columns: set[str] = set()
1493
- for expr in _split_select_items(select_match.group(1)):
1494
- cleaned = expr.strip()
1495
- if not cleaned:
1496
- continue
1497
- alias_match = re.search(r"\bas\s+([a-zA-Z_][a-zA-Z0-9_]*)\s*$", cleaned, flags=re.IGNORECASE)
1498
- if alias_match:
1499
- columns.add(alias_match.group(1).lower())
1500
- continue
1501
- qualified_match = re.match(
1502
- r"^([a-zA-Z_][a-zA-Z0-9_]*)\.([a-zA-Z_][a-zA-Z0-9_]*)$",
1503
- cleaned,
1504
- flags=re.IGNORECASE,
1505
- )
1506
- if qualified_match:
1507
- columns.add(qualified_match.group(2).lower())
1508
- continue
1509
- bare_match = re.match(r"^([a-zA-Z_][a-zA-Z0-9_]*)$", cleaned, flags=re.IGNORECASE)
1510
- if bare_match:
1511
- columns.add(bare_match.group(1).lower())
1512
- return columns
1513
-
1514
-
1515
- def _sql_equality_pairs(sql: str, table_aliases: dict[str, str]) -> set[tuple[tuple[str, str], tuple[str, str]]]:
1516
- pairs: set[tuple[tuple[str, str], tuple[str, str]]] = set()
1517
- derived_spans = _sql_derived_subquery_spans(sql)
1518
- for match in re.finditer(
1519
- r"\b([a-zA-Z_][a-zA-Z0-9_]*)\.([a-zA-Z_][a-zA-Z0-9_]*)\s*=\s*([a-zA-Z_][a-zA-Z0-9_]*)\.([a-zA-Z_][a-zA-Z0-9_]*)\b",
1520
- sql,
1521
- flags=re.IGNORECASE,
1522
- ):
1523
- if _position_in_spans(match.start(), derived_spans):
1524
- continue
1525
- left_alias, left_column, right_alias, right_column = (part.lower() for part in match.groups())
1526
- left_table = table_aliases.get(left_alias)
1527
- right_table = table_aliases.get(right_alias)
1528
- if left_table is None or right_table is None:
1529
- continue
1530
- pairs.add(_canonical_join_pair((left_table, left_column), (right_table, right_column)))
1531
- return pairs
1532
-
1533
-
1534
- def _schema_allowed_join_pairs(schema: dict[str, Any]) -> set[tuple[tuple[str, str], tuple[str, str]]]:
1535
- pairs: set[tuple[tuple[str, str], tuple[str, str]]] = set()
1536
- relations = _schema_relations(schema)
1537
- if isinstance(relations, list):
1538
- for relation in relations:
1539
- if not isinstance(relation, dict):
1540
- continue
1541
- left_table, left_column = _relation_left(relation)
1542
- right_table, right_column = _relation_right(relation)
1543
- if left_table and left_column and right_table and right_column:
1544
- pairs.add(_canonical_join_pair((left_table, left_column), (right_table, right_column)))
1545
- for section in (_schema_metrics(schema), _schema_business_terms(schema), _schema_concepts(schema), _schema_examples(schema)):
1546
- if isinstance(section, list):
1547
- for item in section:
1548
- if isinstance(item, dict):
1549
- pairs.update(_join_pairs_from_text(str(item)))
1550
- pairs.update(_schema_example_join_pattern_pairs(schema))
1551
- pairs.update(_schema_work_order_failure_reason_join_pairs(schema))
1552
- pairs.update(_schema_inferred_foreign_key_join_pairs(schema))
1553
- return pairs
1554
-
1555
-
1556
- def _schema_inferred_foreign_key_join_pairs(
1557
- schema: dict[str, Any],
1558
- ) -> set[tuple[tuple[str, str], tuple[str, str]]]:
1559
- """Allow `{ref_table}_id` columns to join `{ref_table}.id` when no explicit relation exists."""
1560
- pairs: set[tuple[tuple[str, str], tuple[str, str]]] = set()
1561
- table_names = {
1562
- str(table.get("tableName") or table.get("name") or "")
1563
- for table in schema.get("tables") or []
1564
- if isinstance(table, dict) and (table.get("tableName") or table.get("name"))
1565
- }
1566
- for table in schema.get("tables") or []:
1567
- if not isinstance(table, dict):
1568
- continue
1569
- table_name = str(table.get("tableName") or table.get("name") or "")
1570
- if not table_name:
1571
- continue
1572
- for column in table.get("columns") or []:
1573
- if not isinstance(column, dict):
1574
- continue
1575
- column_name = str(column.get("columnName") or column.get("name") or "")
1576
- if not column_name.endswith("_id") or column_name == "id":
1577
- continue
1578
- ref_table = column_name[:-3]
1579
- if ref_table not in table_names:
1580
- continue
1581
- pairs.add(_canonical_join_pair((table_name, column_name), (ref_table, "id")))
1582
- return pairs
1583
-
1584
-
1585
- def _schema_work_order_failure_reason_join_pairs(
1586
- schema: dict[str, Any],
1587
- ) -> set[tuple[tuple[str, str], tuple[str, str]]]:
1588
- pairs: set[tuple[tuple[str, str], tuple[str, str]]] = set()
1589
- mappings = _schema_structure(schema).get("workOrderFailureReasonMappings")
1590
- if not isinstance(mappings, list):
1591
- return pairs
1592
- for mapping in mappings:
1593
- if not isinstance(mapping, dict):
1594
- continue
1595
- reason_join = mapping.get("reasonJoin")
1596
- if isinstance(reason_join, str) and reason_join.strip():
1597
- pairs.update(_join_pairs_from_text(reason_join))
1598
- unpivot = _schema_structure(schema).get("workOrderFailureReasonUnpivot")
1599
- if isinstance(unpivot, dict):
1600
- base_table = str(unpivot.get("baseTable") or "mvp_applied_jobs_by_helped_account")
1601
- for branch in unpivot.get("branches") or []:
1602
- if not isinstance(branch, dict):
1603
- continue
1604
- reason_field = str(branch.get("reasonField") or "")
1605
- if reason_field:
1606
- pairs.add(
1607
- _canonical_join_pair(
1608
- ("failure_reasons", "id"),
1609
- (base_table, reason_field),
1610
- )
1611
- )
1612
- return pairs
1613
-
1614
-
1615
- def _schema_example_join_pattern_pairs(schema: dict[str, Any]) -> set[tuple[tuple[str, str], tuple[str, str]]]:
1616
- pairs: set[tuple[tuple[str, str], tuple[str, str]]] = set()
1617
- example = schema.get("example")
1618
- examples: list[Any] = []
1619
- if isinstance(example, dict) and isinstance(example.get("examples"), list):
1620
- examples = example["examples"]
1621
- examples.extend(_schema_examples(schema))
1622
- for item in examples:
1623
- if not isinstance(item, dict):
1624
- continue
1625
- join_pattern = item.get("joinPattern")
1626
- if isinstance(join_pattern, dict):
1627
- pairs.update(_join_pattern_equality_pairs(join_pattern))
1628
- return pairs
1629
-
1630
-
1631
- def _join_pattern_equality_pairs(join_pattern: dict[str, Any]) -> set[tuple[tuple[str, str], tuple[str, str]]]:
1632
- aliases = _parse_table_alias_clause(str(join_pattern.get("from") or ""))
1633
- pairs: set[tuple[tuple[str, str], tuple[str, str]]] = set()
1634
- joins = join_pattern.get("joins")
1635
- if not isinstance(joins, list):
1636
- return pairs
1637
- for join in joins:
1638
- if not isinstance(join, dict):
1639
- continue
1640
- aliases.update(_parse_table_alias_clause(str(join.get("table") or "")))
1641
- on_conditions = join.get("on")
1642
- if not isinstance(on_conditions, list):
1643
- continue
1644
- for condition in on_conditions:
1645
- pairs.update(_join_pairs_from_on_text(str(condition), aliases))
1646
- return pairs
1647
-
1648
-
1649
- def _parse_table_alias_clause(clause: str) -> dict[str, str]:
1650
- cleaned = clause.strip()
1651
- if not cleaned:
1652
- return {}
1653
- match = re.match(
1654
- r"^([a-zA-Z_][a-zA-Z0-9_]*)(?:\s+(?:as\s+)?([a-zA-Z_][a-zA-Z0-9_]*))?$",
1655
- cleaned,
1656
- flags=re.IGNORECASE,
1657
- )
1658
- if match is None:
1659
- return {}
1660
- table = match.group(1).lower()
1661
- alias = (match.group(2) or table).lower()
1662
- return {alias: table, table: table}
1663
-
1664
-
1665
- def _join_pairs_from_on_text(
1666
- on: str,
1667
- aliases: dict[str, str],
1668
- ) -> set[tuple[tuple[str, str], tuple[str, str]]]:
1669
- pairs: set[tuple[tuple[str, str], tuple[str, str]]] = set()
1670
- for match in re.finditer(
1671
- r"\b([a-zA-Z_][a-zA-Z0-9_]*)\.([a-zA-Z_][a-zA-Z0-9_]*)\s*=\s*([a-zA-Z_][a-zA-Z0-9_]*)\.([a-zA-Z_][a-zA-Z0-9_]*)\b",
1672
- on,
1673
- flags=re.IGNORECASE,
1674
- ):
1675
- left_alias, left_column, right_alias, right_column = (part.lower() for part in match.groups())
1676
- left_table = aliases.get(left_alias)
1677
- right_table = aliases.get(right_alias)
1678
- if left_table is None or right_table is None:
1679
- continue
1680
- pairs.add(_canonical_join_pair((left_table, left_column), (right_table, right_column)))
1681
- return pairs
1682
-
1683
-
1684
- def _schema_structure(schema: dict[str, Any]) -> dict[str, Any]:
1685
- structure = schema.get("structure")
1686
- return structure if isinstance(structure, dict) else {}
1687
-
1688
-
1689
- def _schema_relations(schema: dict[str, Any]) -> list[dict[str, Any]]:
1690
- relations = _schema_structure(schema).get("relations")
1691
- if isinstance(relations, list):
1692
- return [relation for relation in relations if isinstance(relation, dict)]
1693
- joins = schema.get("joins")
1694
- if isinstance(joins, list):
1695
- return [join for join in joins if isinstance(join, dict)]
1696
- return []
1697
-
1698
-
1699
- def _schema_metrics(schema: dict[str, Any]) -> list[dict[str, Any]]:
1700
- metrics = _schema_structure(schema).get("metrics")
1701
- if isinstance(metrics, list):
1702
- return [metric for metric in metrics if isinstance(metric, dict)]
1703
- legacy_metrics = schema.get("metrics")
1704
- if isinstance(legacy_metrics, list):
1705
- return [metric for metric in legacy_metrics if isinstance(metric, dict)]
1706
- return []
1707
-
1708
-
1709
- def _schema_business_terms(schema: dict[str, Any]) -> list[dict[str, Any]]:
1710
- business_terms = _schema_structure(schema).get("businessTerms")
1711
- if isinstance(business_terms, list):
1712
- return [term for term in business_terms if isinstance(term, dict)]
1713
- legacy_glossary = schema.get("glossary")
1714
- if isinstance(legacy_glossary, list):
1715
- return [term for term in legacy_glossary if isinstance(term, dict)]
1716
- return []
1717
-
1718
-
1719
- def _schema_concepts(schema: dict[str, Any]) -> list[dict[str, Any]]:
1720
- concepts = _schema_structure(schema).get("concepts")
1721
- if isinstance(concepts, list):
1722
- return [concept for concept in concepts if isinstance(concept, dict)]
1723
- return []
1724
-
1725
-
1726
- def _schema_examples(schema: dict[str, Any]) -> list[dict[str, Any]]:
1727
- example = schema.get("example")
1728
- if isinstance(example, dict) and isinstance(example.get("examples"), list):
1729
- return [item for item in example["examples"] if isinstance(item, dict)]
1730
- legacy_examples = schema.get("examples")
1731
- if isinstance(legacy_examples, list):
1732
- return [item for item in legacy_examples if isinstance(item, dict)]
1733
- return []
1734
-
1735
-
1736
- def _relation_left(relation: dict[str, Any]) -> tuple[str, str]:
1737
- table = str(relation.get("leftTable") or "").lower()
1738
- column = str(relation.get("leftColumn") or relation.get("fromColumn") or "").lower()
1739
- if table:
1740
- return table, column
1741
- return _split_field_ref(str(relation.get("from") or "").lower())
1742
-
1743
-
1744
- def _relation_right(relation: dict[str, Any]) -> tuple[str, str]:
1745
- table = str(relation.get("rightTable") or "").lower()
1746
- column = str(relation.get("rightColumn") or relation.get("toColumn") or "").lower()
1747
- if table:
1748
- return table, column
1749
- return _split_field_ref(str(relation.get("to") or "").lower())
1750
-
1751
-
1752
- def _split_field_ref(value: str) -> tuple[str, str]:
1753
- if "." not in value:
1754
- return value, ""
1755
- return tuple(value.split(".", 1)) # type: ignore[return-value]
1756
-
1757
-
1758
- def _join_pairs_from_text(text: str) -> set[tuple[tuple[str, str], tuple[str, str]]]:
1759
- pairs: set[tuple[tuple[str, str], tuple[str, str]]] = set()
1760
- for match in re.finditer(
1761
- r"\b([a-zA-Z_][a-zA-Z0-9_]*)\.([a-zA-Z_][a-zA-Z0-9_]*)\s*=\s*([a-zA-Z_][a-zA-Z0-9_]*)\.([a-zA-Z_][a-zA-Z0-9_]*)\b",
1762
- text,
1763
- flags=re.IGNORECASE,
1764
- ):
1765
- left_table, left_column, right_table, right_column = (part.lower() for part in match.groups())
1766
- pairs.add(_canonical_join_pair((left_table, left_column), (right_table, right_column)))
1767
- return pairs
1768
-
1769
-
1770
- def _canonical_join_pair(
1771
- left: tuple[str, str],
1772
- right: tuple[str, str],
1773
- ) -> tuple[tuple[str, str], tuple[str, str]]:
1774
- return tuple(sorted((left, right))) # type: ignore[return-value]
1775
-
1776
-
1777
- def question_has_time_range(question: str) -> bool:
1778
- return _extract_time_range(question) is not None
1779
-
1780
-
1781
- def extract_time_range(question: str, schema: dict[str, Any] | None = None) -> TimeRange | None:
1782
- """Parse a question time window; month/day without year defaults to the current year."""
1783
- return _extract_time_range(question, schema)
1784
-
1785
-
1786
- def time_range_from_query_plan(query_plan: dict[str, Any] | None) -> TimeRange | None:
1787
- """Extract a confirmed time window from query-plan filters."""
1788
- if not isinstance(query_plan, dict):
1789
- return None
1790
- start_value = ""
1791
- end_value = ""
1792
- field_name = ""
1793
- for flt in _query_plan_sql_filters(query_plan):
1794
- if not isinstance(flt, dict):
1795
- continue
1796
- if str(flt.get("source") or "") != "question.timeRange":
1797
- continue
1798
- operator = str(flt.get("operator") or "").strip()
1799
- value = str(flt.get("value") or "").strip()
1800
- field = str(flt.get("field") or "").strip()
1801
- if operator == ">=":
1802
- start_value = value
1803
- field_name = field or field_name
1804
- elif operator == "<":
1805
- end_value = value
1806
- field_name = field or field_name
1807
- if not start_value or not end_value or not field_name:
1808
- return None
1809
- table_name, column = field_name.split(".", 1) if "." in field_name else ("", field_name)
1810
- return TimeRange(
1811
- field=column,
1812
- alias="时间范围",
1813
- start=start_value,
1814
- end=end_value,
1815
- table_name=table_name,
1816
- )
1817
-
1818
-
1819
- def resolve_time_range(
1820
- question: str,
1821
- schema: dict[str, Any] | None = None,
1822
- *,
1823
- query_plan: dict[str, Any] | None = None,
1824
- ) -> TimeRange | None:
1825
- plan_range = time_range_from_query_plan(query_plan)
1826
- if plan_range is not None:
1827
- return plan_range
1828
- return extract_time_range(question, schema)
1829
-
1830
-
1831
- def time_range_plan_filters(
1832
- question: str,
1833
- schema: dict[str, Any] | None,
1834
- *,
1835
- concepts: list[dict[str, Any]] | None = None,
1836
- ) -> list[dict[str, str]]:
1837
- """Build query-plan filters for an extracted time range."""
1838
- time_range = resolve_time_range_for_question(question, schema, concepts=concepts)
1839
- if time_range is None:
1840
- return []
1841
- qualified_field = (
1842
- f"{time_range.table_name}.{time_range.field}"
1843
- if time_range.table_name
1844
- else time_range.field
1845
- )
1846
- label = time_range.alias or "时间范围"
1847
- return [
1848
- {
1849
- "field": qualified_field,
1850
- "operator": ">=",
1851
- "value": time_range.start[:10],
1852
- "description": f"{label}起",
1853
- "source": "question.timeRange",
1854
- },
1855
- {
1856
- "field": qualified_field,
1857
- "operator": "<",
1858
- "value": time_range.end[:10],
1859
- "description": f"{label}止",
1860
- "source": "question.timeRange",
1861
- },
1862
- ]
1863
-
1864
-
1865
- _FILLED_JOB_PERIOD_CONCEPT = "filled_job_by_period"
1866
-
1867
-
1868
- def _query_plan_has_matched_concept(query_plan: dict[str, Any], concept_name: str) -> bool:
1869
- for concept in query_plan.get("matchedConcepts") or []:
1870
- if isinstance(concept, dict) and str(concept.get("name") or "") == concept_name:
1871
- return True
1872
- return False
1873
-
1874
-
1875
- def _uses_subquery_period_time_range(sql: str, query_plan: dict[str, Any] | None) -> bool:
1876
- if isinstance(query_plan, dict) and _query_plan_has_matched_concept(
1877
- query_plan,
1878
- _FILLED_JOB_PERIOD_CONCEPT,
1879
- ):
1880
- return True
1881
- return "period_active_apply_count" in sql.lower()
1882
-
1883
-
1884
- def _period_time_bounds(time_range: TimeRange) -> tuple[str, str]:
1885
- start = time_range.start
1886
- end = time_range.end
1887
- if len(start) <= 10:
1888
- start = f"{start[:10]} 00:00:00"
1889
- if len(end) <= 10:
1890
- end = f"{end[:10]} 00:00:00"
1891
- return start, end
1892
-
1893
-
1894
- def _apply_subquery_period_time_range(sql: str, time_range: TimeRange) -> str:
1895
- """Apply confirmed time bounds inside apply_stat CASE WHEN, not outer WHERE."""
1896
- stripped = _remove_time_range_filters_from_sql(
1897
- sql.rstrip().rstrip(";"),
1898
- time_range,
1899
- allow_any_alias=True,
1900
- )
1901
- period_start, period_end = _period_time_bounds(time_range)
1902
- result = stripped
1903
- for old, new in (
1904
- ("'<period_start>'", f"'{period_start}'"),
1905
- ("'<period_end>'", f"'{period_end}'"),
1906
- ("<period_start>", f"'{period_start}'"),
1907
- ("<period_end>", f"'{period_end}'"),
1908
- ):
1909
- result = result.replace(old, new)
1910
- field_pattern = rf"(?:\w+\.)?{re.escape(time_range.field)}"
1911
- time_expr = (
1912
- r"(?:date_add\s*\(\s*curdate\s*\(\s*\)\s*,\s*interval\s+\d+\s+day\s*\)"
1913
- r"|curdate\s*\(\s*\)|current_date(?:\s*\(\s*\))?|'[^']+')"
1914
- )
1915
- return re.sub(
1916
- rf"(?is)(case\s+when\s+{field_pattern}\s*>=\s*){time_expr}(\s+and\s+{field_pattern}\s*<\s*){time_expr}(\s+then)",
1917
- lambda match: f"{match.group(1)}'{period_start}'{match.group(2)}'{period_end}'{match.group(3)}",
1918
- result,
1919
- )
1920
-
1921
-
1922
- def apply_time_range_filters_from_plan(sql: str, query_plan: dict[str, Any] | None) -> str:
1923
- """Force generated SQL to use the confirmed query-plan time filters."""
1924
- if not isinstance(query_plan, dict):
1925
- return sql
1926
- time_range = time_range_from_query_plan(query_plan)
1927
- if time_range is None:
1928
- return sql
1929
- stripped = _remove_time_range_filters_from_sql(sql.rstrip().rstrip(";"), time_range)
1930
- if _uses_subquery_period_time_range(stripped, query_plan):
1931
- return _apply_subquery_period_time_range(stripped, time_range)
1932
- qualified_field = _time_range_sql_qualified_field(stripped, time_range)
1933
- extra = (
1934
- f"{qualified_field} >= '{time_range.start[:10]}' "
1935
- f"AND {qualified_field} < '{time_range.end[:10]}'"
1936
- )
1937
- insert_pos = _where_filter_insert_pos(stripped)
1938
- head = stripped[:insert_pos].rstrip()
1939
- tail = stripped[insert_pos:].lstrip()
1940
- if _find_top_level_keyword(head, "where", start=0) is not None:
1941
- return f"{head} AND {extra} {tail}".strip()
1942
- return f"{head} WHERE {extra} {tail}".strip()
1943
-
1944
-
1945
- def apply_time_range_filters_from_question(
1946
- sql: str,
1947
- question: str,
1948
- schema: dict[str, Any] | None,
1949
- ) -> str:
1950
- """Force a question-level time range into SQL when queryPlan missed it."""
1951
- time_range = resolve_time_range_for_sql(question, schema, sql)
1952
- if time_range is None:
1953
- return sql
1954
- query_plan = {
1955
- "filters": [
1956
- {
1957
- "field": _qualified_time_range_field(time_range),
1958
- "operator": ">=",
1959
- "value": time_range.start[:10],
1960
- "source": "question.timeRange",
1961
- },
1962
- {
1963
- "field": _qualified_time_range_field(time_range),
1964
- "operator": "<",
1965
- "value": time_range.end[:10],
1966
- "source": "question.timeRange",
1967
- },
1968
- ]
1969
- }
1970
- return apply_time_range_filters_from_plan(sql, query_plan)
1971
-
1972
-
1973
- def resolve_time_range_for_sql(
1974
- question: str,
1975
- schema: dict[str, Any] | None,
1976
- sql: str,
1977
- ) -> TimeRange | None:
1978
- if not schema:
1979
- return None
1980
- from .schema_context_retriever import match_concepts_for_question
1981
-
1982
- concepts = match_concepts_for_question(_schema_concepts(schema), question)
1983
- sql_schema = _schema_limited_to_sql_tables(schema, sql)
1984
- if sql_schema is None and _sql_referenced_table_names(sql):
1985
- return None
1986
- if sql_schema is not None:
1987
- return resolve_time_range_for_question(
1988
- question,
1989
- sql_schema,
1990
- concepts=_concepts_limited_to_schema_tables(concepts, sql_schema),
1991
- )
1992
- return resolve_time_range_for_question(question, schema, concepts=concepts)
1993
-
1994
-
1995
- def _concepts_limited_to_schema_tables(
1996
- concepts: list[dict[str, Any]],
1997
- schema: dict[str, Any],
1998
- ) -> list[dict[str, Any]]:
1999
- table_names = {name.lower() for name in _table_names(schema)}
2000
- if not table_names:
2001
- return []
2002
- limited: list[dict[str, Any]] = []
2003
- for concept in concepts:
2004
- qualified = str(concept.get("timeRangeField") or "").strip()
2005
- if "." not in qualified:
2006
- limited.append(concept)
2007
- continue
2008
- table_name = qualified.split(".", 1)[0].lower()
2009
- if table_name in table_names:
2010
- limited.append(concept)
2011
- return limited
2012
-
2013
-
2014
- def _qualified_time_range_field(time_range: TimeRange) -> str:
2015
- if time_range.table_name:
2016
- return f"{time_range.table_name}.{time_range.field}"
2017
- return time_range.field
2018
-
2019
-
2020
- def _where_filter_insert_pos(sql: str) -> int:
2021
- positions = [
2022
- pos
2023
- for keyword in ("group by", "order by", "limit")
2024
- if (pos := _find_top_level_keyword(sql, keyword, start=0)) is not None
2025
- ]
2026
- return min(positions) if positions else len(sql)
2027
-
2028
-
2029
- def _remove_time_range_filters_from_sql(
2030
- sql: str,
2031
- time_range: TimeRange,
2032
- *,
2033
- allow_any_alias: bool = False,
2034
- ) -> str:
2035
- where_index = _find_top_level_keyword(sql, "where", start=0)
2036
- if where_index is None:
2037
- return sql
2038
- clause_start = where_index + len("where")
2039
- clause_end = _where_filter_insert_pos(sql[clause_start:])
2040
- if clause_end < len(sql[clause_start:]):
2041
- clause_end += clause_start
2042
- else:
2043
- clause_end = len(sql)
2044
- where_body = sql[clause_start:clause_end].strip()
2045
- if not where_body:
2046
- return sql
2047
- predicates = _split_top_level_and_predicates(where_body)
2048
- kept = [
2049
- predicate.strip()
2050
- for predicate in predicates
2051
- if predicate.strip()
2052
- and not _is_time_range_predicate(
2053
- predicate,
2054
- sql=sql,
2055
- time_range=time_range,
2056
- allow_any_alias=allow_any_alias,
2057
- )
2058
- ]
2059
- prefix = sql[:where_index].rstrip()
2060
- suffix = sql[clause_end:].lstrip()
2061
- if not kept:
2062
- return f"{prefix} {suffix}".strip()
2063
- return f"{prefix} WHERE {' AND '.join(kept)} {suffix}".strip()
2064
-
2065
-
2066
- def _split_top_level_and_predicates(where_body: str) -> list[str]:
2067
- predicates: list[str] = []
2068
- depth = 0
2069
- quote: str | None = None
2070
- start = 0
2071
- index = 0
2072
- while index < len(where_body):
2073
- char = where_body[index]
2074
- if quote is not None:
2075
- if char == quote:
2076
- quote = None
2077
- elif char == "\\":
2078
- index += 1
2079
- index += 1
2080
- continue
2081
- if char in ("'", '"', "`"):
2082
- quote = char
2083
- index += 1
2084
- continue
2085
- if char == "(":
2086
- depth += 1
2087
- index += 1
2088
- continue
2089
- if char == ")":
2090
- depth = max(0, depth - 1)
2091
- index += 1
2092
- continue
2093
- if depth == 0 and where_body[index : index + 3].lower() == "and":
2094
- before = where_body[index - 1] if index > 0 else " "
2095
- after_index = index + 3
2096
- after = where_body[after_index] if after_index < len(where_body) else " "
2097
- if not (before.isalnum() or before == "_") and not (after.isalnum() or after == "_"):
2098
- predicates.append(where_body[start:index])
2099
- start = after_index
2100
- index = after_index
2101
- continue
2102
- index += 1
2103
- predicates.append(where_body[start:])
2104
- return predicates
2105
-
2106
-
2107
- def _is_time_range_predicate(
2108
- predicate: str,
2109
- *,
2110
- sql: str,
2111
- time_range: TimeRange,
2112
- allow_any_alias: bool = False,
2113
- ) -> bool:
2114
- field_pattern = _time_range_sql_field_pattern(sql, time_range, allow_any_alias=allow_any_alias)
2115
- if re.search(field_pattern, predicate, flags=re.IGNORECASE) is None:
2116
- return False
2117
- if re.search(r"'20\d{2}-\d{1,2}-\d{1,2}(?:\s+\d{1,2}:\d{2}:\d{2})?'", predicate) is None:
2118
- return False
2119
- return bool(re.search(r"(?:\bbetween\b|>=|>|<=|<|=)", predicate, flags=re.IGNORECASE))
2120
-
2121
-
2122
- def _time_range_sql_field_pattern(
2123
- sql: str,
2124
- time_range: TimeRange,
2125
- *,
2126
- allow_any_alias: bool = False,
2127
- ) -> str:
2128
- field = re.escape(time_range.field.lower())
2129
- if allow_any_alias or not time_range.table_name:
2130
- return rf"(?:\b\w+\.)?{field}\b"
2131
- aliases = [
2132
- re.escape(alias)
2133
- for alias, table in _sql_table_aliases(sql).items()
2134
- if table == time_range.table_name.lower()
2135
- ]
2136
- aliases.append(re.escape(time_range.table_name.lower()))
2137
- qualifier = "|".join(sorted(set(aliases), key=len, reverse=True))
2138
- return rf"\b(?:{qualifier})\.{field}\b"
2139
-
2140
-
2141
- def _time_range_sql_qualified_field(sql: str, time_range: TimeRange) -> str:
2142
- if not time_range.table_name:
2143
- return time_range.field
2144
- aliases = _sql_top_level_table_aliases(sql)
2145
- alias_candidates = [
2146
- alias
2147
- for alias, table in aliases.items()
2148
- if table == time_range.table_name.lower() and alias != time_range.table_name.lower()
2149
- ]
2150
- if alias_candidates:
2151
- return f"{sorted(alias_candidates, key=len)[0]}.{time_range.field}"
2152
- return f"{time_range.table_name}.{time_range.field}"
2153
-
2154
-
2155
- def sql_missing_query_plan_filters(sql: str, query_plan: dict[str, Any] | None) -> list[str]:
2156
- """Return table.column LIKE filters from query_plan that are absent in sql."""
2157
- if not isinstance(query_plan, dict):
2158
- return []
2159
- missing: list[str] = []
2160
- normalized_sql = sql.lower()
2161
- grouped_filters: dict[str, list[dict[str, Any]]] = {}
2162
- plan_filters = [flt for flt in _query_plan_sql_filters(query_plan) if isinstance(flt, dict)]
2163
- for flt in plan_filters:
2164
- if str(flt.get("operator") or "").upper() != "LIKE":
2165
- continue
2166
- logical_group = str(flt.get("logicalGroup") or "").strip()
2167
- if logical_group:
2168
- grouped_filters.setdefault(logical_group, []).append(flt)
2169
- field = str(flt.get("field") or "").strip()
2170
- if "." not in field:
2171
- continue
2172
- table, column = field.split(".", 1)
2173
- value = str(flt.get("value") or "").strip().strip("%")
2174
- if not value:
2175
- continue
2176
- table_l = table.lower()
2177
- column_l = column.lower()
2178
- table_present = bool(re.search(rf"\b{re.escape(table_l)}\b", normalized_sql))
2179
- column_present = bool(re.search(rf"\b{re.escape(column_l)}\b", normalized_sql))
2180
- value_present = value.lower() in normalized_sql
2181
- if not table_present or not column_present or not value_present:
2182
- missing.append(f"{field} LIKE {flt.get('value')}")
2183
- for group_name, filters in grouped_filters.items():
2184
- if len(filters) <= 1:
2185
- continue
2186
- group_missing = [
2187
- f"{flt.get('field')} LIKE {flt.get('value')}"
2188
- for flt in filters
2189
- if f"{flt.get('field')} LIKE {flt.get('value')}" in missing
2190
- ]
2191
- if group_missing:
2192
- continue
2193
- if not _or_group_filters_connected_by_or(normalized_sql, filters):
2194
- missing.append(f"{group_name} requires OR between matched entities")
2195
- and_pairs = _hierarchical_location_and_filter_pairs(plan_filters)
2196
- for city_flt, region_flt in and_pairs:
2197
- city_token = f"{city_flt.get('field')} LIKE {city_flt.get('value')}"
2198
- region_token = f"{region_flt.get('field')} LIKE {region_flt.get('value')}"
2199
- if city_token in missing or region_token in missing:
2200
- continue
2201
- if _filters_connected_by_or(normalized_sql, [city_flt, region_flt]):
2202
- missing.append(
2203
- "city.name and region.name require AND for hierarchical location filters "
2204
- f"({city_flt.get('value')} AND {region_flt.get('value')})"
2205
- )
2206
- break
2207
- return missing
2208
-
2209
-
2210
- def sql_satisfies_query_plan_filters(sql: str, query_plan: dict[str, Any] | None) -> bool:
2211
- return not sql_missing_query_plan_filters(sql, query_plan)
2212
-
2213
-
2214
- def _query_plan_sql_filters(query_plan: dict[str, Any]) -> list[Any]:
2215
- filters = query_plan.get("_sqlFilters")
2216
- if isinstance(filters, list):
2217
- return filters
2218
- filters = query_plan.get("filters")
2219
- return filters if isinstance(filters, list) else []
2220
-
2221
-
2222
- def _location_place_key_for_filter(value: str) -> str:
2223
- cleaned = str(value or "").strip().strip("%")
2224
- if not cleaned:
2225
- return ""
2226
- for suffix in ("自治州", "地区", "市", "区", "县", "旗"):
2227
- if cleaned.endswith(suffix) and len(cleaned) > len(suffix):
2228
- return cleaned[: -len(suffix)]
2229
- return cleaned
2230
-
2231
-
2232
- def _hierarchical_location_and_filter_pairs(
2233
- filters: list[dict[str, Any]],
2234
- ) -> list[tuple[dict[str, Any], dict[str, Any]]]:
2235
- """City/region filters that must be AND-ed (distinct places, not same-place OR)."""
2236
- city_filters = [
2237
- flt
2238
- for flt in filters
2239
- if str(flt.get("field") or "") == "city.name" and str(flt.get("operator") or "").upper() == "LIKE"
2240
- ]
2241
- region_filters = [
2242
- flt
2243
- for flt in filters
2244
- if str(flt.get("field") or "") == "region.name" and str(flt.get("operator") or "").upper() == "LIKE"
2245
- ]
2246
- if not city_filters or not region_filters:
2247
- return []
2248
- pairs: list[tuple[dict[str, Any], dict[str, Any]]] = []
2249
- for city_flt in city_filters:
2250
- city_key = _location_place_key_for_filter(str(city_flt.get("value") or ""))
2251
- city_group = str(city_flt.get("logicalGroup") or "").strip()
2252
- city_op = str(city_flt.get("logicalOperator") or "").upper()
2253
- for region_flt in region_filters:
2254
- region_key = _location_place_key_for_filter(str(region_flt.get("value") or ""))
2255
- region_group = str(region_flt.get("logicalGroup") or "").strip()
2256
- region_op = str(region_flt.get("logicalOperator") or "").upper()
2257
- same_or_group = bool(
2258
- city_group
2259
- and city_group == region_group
2260
- and city_op == "OR"
2261
- and region_op == "OR"
2262
- )
2263
- if same_or_group:
2264
- continue
2265
- if city_key and region_key and city_key == region_key:
2266
- continue
2267
- pairs.append((city_flt, region_flt))
2268
- return pairs
2269
-
2270
-
2271
- def _or_group_filters_connected_by_or(normalized_sql: str, filters: list[dict[str, Any]]) -> bool:
2272
- if not re.search(r"\bor\b", normalized_sql):
2273
- return False
2274
- positions: list[int] = []
2275
- for flt in filters:
2276
- value = str(flt.get("value") or "").strip().strip("%").lower()
2277
- if not value:
2278
- continue
2279
- found = normalized_sql.find(value)
2280
- if found >= 0:
2281
- positions.append(found)
2282
- if len(positions) < 2:
2283
- return True
2284
- start, end = min(positions), max(positions)
2285
- between = normalized_sql[start:end]
2286
- return bool(re.search(r"\bor\b", between))
2287
-
2288
-
2289
- def _filters_connected_by_or(normalized_sql: str, filters: list[dict[str, Any]]) -> bool:
2290
- if not re.search(r"\bor\b", normalized_sql):
2291
- return False
2292
- positions: list[int] = []
2293
- for flt in filters:
2294
- value = str(flt.get("value") or "").strip().strip("%").lower()
2295
- if not value:
2296
- continue
2297
- found = normalized_sql.find(value)
2298
- if found >= 0:
2299
- positions.append(found)
2300
- if len(positions) < 2:
2301
- return False
2302
- start, end = min(positions), max(positions)
2303
- between = normalized_sql[start:end]
2304
- return bool(re.search(r"\bor\b", between))
2305
-
2306
-
2307
- def sql_missing_query_plan_select_fields(sql: str, query_plan: dict[str, Any] | None) -> list[str]:
2308
- """Return required SELECT display fields from query_plan that are absent in sql."""
2309
- if not isinstance(query_plan, dict):
2310
- return []
2311
- select_clause = _select_clause(sql).lower()
2312
- if not select_clause:
2313
- return []
2314
- missing: list[str] = []
2315
- for item in query_plan.get("fields") or []:
2316
- if not isinstance(item, dict):
2317
- continue
2318
- if item.get("required") is not True and str(item.get("source") or "") != "search_entities":
2319
- continue
2320
- table = str(item.get("table") or "").strip()
2321
- field = str(item.get("field") or "").strip()
2322
- description = str(item.get("description") or "").strip()
2323
- if not table or not field:
2324
- continue
2325
- qualified = f"{table}.{field}".lower()
2326
- description_present = bool(description and description.lower() in select_clause)
2327
- qualified_present = qualified in select_clause
2328
- if not description_present and not qualified_present:
2329
- missing.append(f"{qualified} AS {description or field}")
2330
- return missing
2331
-
2332
-
2333
- def sql_satisfies_query_plan_select_fields(sql: str, query_plan: dict[str, Any] | None) -> bool:
2334
- return not sql_missing_query_plan_select_fields(sql, query_plan)
2335
-
2336
-
2337
- def _select_clause(sql: str) -> str:
2338
- match = re.search(r"\bselect\b(?P<select>.*?)\bfrom\b", sql, flags=re.IGNORECASE | re.DOTALL)
2339
- return match.group("select") if match else ""
2340
-
2341
-
2342
- def sql_satisfies_time_range(
2343
- sql: str,
2344
- question: str,
2345
- schema: dict[str, Any] | None = None,
2346
- *,
2347
- query_plan: dict[str, Any] | None = None,
2348
- ) -> bool:
2349
- time_range = resolve_time_range(question, schema, query_plan=query_plan)
2350
- if time_range is None:
2351
- return True
2352
- normalized = sql.lower()
2353
- field_pattern = rf"(?:\b\w+\.)?{re.escape(time_range.field.lower())}\b"
2354
- if re.search(field_pattern, normalized) is None:
2355
- return False
2356
- if time_range.start[:10] in sql and time_range.end[:10] in sql:
2357
- return True
2358
- end_date = _parse_date(time_range.end[:10])
2359
- if end_date is not None:
2360
- inclusive_end = (end_date - timedelta(days=1)).isoformat()
2361
- if time_range.start[:10] in sql and inclusive_end in sql:
2362
- return True
2363
- year = time_range.start[:4]
2364
- month = str(int(time_range.start[5:7]))
2365
- padded_month = time_range.start[5:7]
2366
- return bool(
2367
- re.search(rf"\byear\s*\([^)]*{re.escape(time_range.field)}[^)]*\)\s*=\s*{year}\b", normalized)
2368
- and re.search(rf"\bmonth\s*\([^)]*{re.escape(time_range.field)}[^)]*\)\s*=\s*(?:{month}|{padded_month})\b", normalized)
2369
- )
2370
-
2371
-
2372
- def _schema_limited_to_sql_tables(schema: dict[str, Any], sql: str) -> dict[str, Any] | None:
2373
- used_tables = _sql_referenced_table_names(sql)
2374
- if not used_tables:
2375
- return None
2376
- tables = schema.get("tables", [])
2377
- if not isinstance(tables, list):
2378
- return None
2379
- matched_tables = []
2380
- for table in tables:
2381
- if not isinstance(table, dict):
2382
- continue
2383
- table_name = str(table.get("tableName") or table.get("name") or "").lower()
2384
- if table_name in used_tables:
2385
- matched_tables.append(table)
2386
- if not matched_tables:
2387
- return None
2388
- limited = dict(schema)
2389
- limited["tables"] = matched_tables
2390
- return limited
2391
-
2392
-
2393
- def _sql_referenced_table_names(sql: str) -> set[str]:
2394
- normalized = _strip_sql_string_literals(sql)
2395
- names: set[str] = set()
2396
- for match in re.finditer(
2397
- r"\b(?:from|join)\s+([a-zA-Z_][a-zA-Z0-9_]*)\b",
2398
- normalized,
2399
- flags=re.IGNORECASE,
2400
- ):
2401
- names.add(match.group(1).lower())
2402
- return names
2403
-
2404
-
2405
- def _strip_sql_string_literals(sql: str) -> str:
2406
- result: list[str] = []
2407
- quote: str | None = None
2408
- index = 0
2409
- while index < len(sql):
2410
- char = sql[index]
2411
- if quote is not None:
2412
- if char == quote:
2413
- quote = None
2414
- result.append(" ")
2415
- index += 1
2416
- continue
2417
- if char in ("'", '"'):
2418
- quote = char
2419
- result.append(" ")
2420
- index += 1
2421
- continue
2422
- result.append(char)
2423
- index += 1
2424
- return "".join(result)
2425
-
2426
-
2427
- def _table_names(schema: dict[str, Any]) -> set[str]:
2428
- tables = schema.get("tables", [])
2429
- if not isinstance(tables, list):
2430
- return set()
2431
- names: set[str] = set()
2432
- for table in tables:
2433
- if not isinstance(table, dict):
2434
- continue
2435
- name = table.get("name") or table.get("tableName")
2436
- if name:
2437
- names.add(str(name))
2438
- return names
2439
-
2440
-
2441
- def _table_columns(schema: dict[str, Any], table_name: str) -> list[dict[str, Any]]:
2442
- tables = schema.get("tables", [])
2443
- if not isinstance(tables, list):
2444
- return []
2445
- for table in tables:
2446
- if not isinstance(table, dict):
2447
- continue
2448
- name = table.get("name") or table.get("tableName")
2449
- if str(name).lower() == str(table_name).lower() and isinstance(table.get("columns"), list):
2450
- return [column for column in table["columns"] if isinstance(column, dict)]
2451
- return []
2452
-
2453
-
2454
- def _select_column(
2455
- schema: dict[str, Any],
2456
- table_name: str,
2457
- terms: tuple[str, ...],
2458
- fallback_names: tuple[str, ...],
2459
- ) -> str | None:
2460
- columns = _table_columns(schema, table_name)
2461
- best: tuple[int, str] | None = None
2462
- normalized_terms = [_normalize_semantic_text(term) for term in terms]
2463
- for column in columns:
2464
- name = str(column.get("columnName") or column.get("name") or "")
2465
- if not name:
2466
- continue
2467
- text = _normalize_semantic_text(f"{name}{column.get('comment') or ''}")
2468
- score = sum(10 for term in normalized_terms if term and term in text)
2469
- if name in fallback_names:
2470
- score += 5
2471
- if score > 0 and (best is None or score > best[0]):
2472
- best = (score, name)
2473
- if best is not None:
2474
- return best[1]
2475
- for name in fallback_names:
2476
- if _has_column(schema, table_name, name):
2477
- return name
2478
- return None
2479
-
2480
-
2481
- def _has_column(schema: dict[str, Any], table_name: str, column_name: str) -> bool:
2482
- return any((column.get("columnName") or column.get("name")) == column_name for column in _table_columns(schema, table_name))
2483
-
2484
-
2485
- def resolve_time_range_for_question(
2486
- question: str,
2487
- schema: dict[str, Any] | None = None,
2488
- *,
2489
- concepts: list[dict[str, Any]] | None = None,
2490
- ) -> TimeRange | None:
2491
- bounds = _extract_time_bounds(question)
2492
- if bounds is None:
2493
- return None
2494
- for concept in concepts or []:
2495
- qualified = str(concept.get("timeRangeField") or "").strip()
2496
- if "." not in qualified:
2497
- continue
2498
- table_name, field_name = qualified.rsplit(".", 1)
2499
- return TimeRange(
2500
- field=field_name,
2501
- alias="时间范围",
2502
- start=bounds[0],
2503
- end=bounds[1],
2504
- table_name=table_name,
2505
- )
2506
- field = _resolve_time_field(question, schema or {})
2507
- if field is None:
2508
- return None
2509
- return TimeRange(
2510
- field=field.name,
2511
- alias=field.alias,
2512
- start=bounds[0],
2513
- end=bounds[1],
2514
- table_name=field.table_name,
2515
- )
2516
-
2517
-
2518
- def _extract_time_range(question: str, schema: dict[str, Any] | None = None) -> TimeRange | None:
2519
- return resolve_time_range_for_question(question, schema)
2520
-
2521
-
2522
- def _resolve_time_field(question: str, schema: dict[str, Any]) -> FieldRef | None:
2523
- question_text = _normalize_semantic_text(question)
2524
- best: tuple[int, FieldRef] | None = None
2525
- for table in schema.get("tables", []) if isinstance(schema.get("tables"), list) else []:
2526
- if not isinstance(table, dict):
2527
- continue
2528
- columns = table.get("columns")
2529
- if not isinstance(columns, list):
2530
- continue
2531
- for column in columns:
2532
- if not isinstance(column, dict):
2533
- continue
2534
- name = str(column.get("columnName") or column.get("name") or "")
2535
- if not name:
2536
- continue
2537
- data_type = str(column.get("dataType") or column.get("type") or "")
2538
- comment = str(column.get("comment") or "")
2539
- if not _is_time_like_column(name, data_type, comment):
2540
- continue
2541
- score = _time_field_match_score(question_text, name, comment)
2542
- if score <= 0:
2543
- continue
2544
- table_name = str(table.get("tableName") or table.get("name") or "")
2545
- field_ref = FieldRef(
2546
- name=name,
2547
- alias=_column_alias(name, comment),
2548
- table_name=table_name,
2549
- )
2550
- if best is None or score > best[0]:
2551
- best = (score, field_ref)
2552
- if best is not None:
2553
- return best[1]
2554
- return None
2555
-
2556
-
2557
- def _is_time_like_column(name: str, data_type: str, comment: str) -> bool:
2558
- lowered = f"{name} {data_type}".lower()
2559
- return bool(
2560
- re.search(r"(date|time|datetime|timestamp|_at$)", lowered)
2561
- or re.search(r"(时间|日期)", comment)
2562
- )
2563
-
2564
-
2565
- def _time_field_match_score(question_text: str, name: str, comment: str) -> int:
2566
- field_text = _normalize_semantic_text(f"{name}{comment}")
2567
- score = 0
2568
- comment_text = _normalize_semantic_text(comment)
2569
- if comment_text and comment_text in question_text:
2570
- score += 50
2571
- name_text = _normalize_semantic_text(name)
2572
- if name_text and name_text in question_text:
2573
- score += 30
2574
- if "时间" in question_text and "时间" in field_text:
2575
- score += 5
2576
- if "日期" in question_text and ("时间" in field_text or "日期" in field_text):
2577
- score += 5
2578
- if "报名" in question_text and "报名" in comment_text:
2579
- score += 40
2580
- if re.search(r"(上架|发布|新上架)", question_text):
2581
- if "publish" in name.lower() or "发布" in comment_text:
2582
- score += 40
2583
- return score
2584
-
2585
-
2586
- def _normalize_semantic_text(text: str) -> str:
2587
- normalized = text.lower()
2588
- return re.sub(r"[^0-9a-z\u4e00-\u9fff]+", "", normalized)
2589
-
2590
-
2591
- def _column_alias(name: str, comment: str) -> str:
2592
- cleaned = re.sub(r"[。;;,,\s].*$", "", comment.strip())
2593
- if cleaned:
2594
- return cleaned
2595
- return name
2596
-
2597
-
2598
- def _normalize_fullwidth_digits(text: str) -> str:
2599
- return text.translate(str.maketrans("0123456789", "0123456789"))
2600
-
2601
-
2602
- def _extract_time_bounds(question: str) -> tuple[str, str] | None:
2603
- question = _normalize_fullwidth_digits(question)
2604
- date_range = _extract_explicit_date_range(question)
2605
- if date_range is not None:
2606
- return date_range
2607
- if re.search(r"(近|最近|过去)\s*(?:1|一)\s*年", question):
2608
- today = _today()
2609
- return _format_datetime(_one_year_before(today)), _format_datetime(today + timedelta(days=1))
2610
- if re.search(r"(今天|今日)", question):
2611
- today = _today()
2612
- return _format_datetime(today), _format_datetime(today + timedelta(days=1))
2613
- if re.search(r"(昨天|昨日)", question):
2614
- today = _today()
2615
- yesterday = today - timedelta(days=1)
2616
- return _format_datetime(yesterday), _format_datetime(today)
2617
- relative_month_match = re.search(r"(?P<relative_year>今年|去年|前年|明年|后年)\s*(?P<month>\d{1,2})\s*月(?:份)?", question)
2618
- if relative_month_match is not None:
2619
- year = _today().year + _relative_year_offset(relative_month_match.group("relative_year"))
2620
- month = int(relative_month_match.group("month"))
2621
- return _month_bounds(year, month)
2622
- if not re.search(r"20\d{2}\s*年", question):
2623
- cn_day_range = re.search(
2624
- r"(?P<month>\d{1,2})\s*月\s*(?P<start_day>\d{1,2})\s*日?\s*"
2625
- r"(?:到|至|~|~|-)\s*"
2626
- r"(?:(?P<end_month>\d{1,2})\s*月\s*)?(?P<end_day>\d{1,2})\s*日?",
2627
- question,
2628
- )
2629
- if cn_day_range is not None:
2630
- year = _today().year
2631
- month = int(cn_day_range.group("month"))
2632
- end_month_text = cn_day_range.group("end_month")
2633
- end_month = int(end_month_text) if end_month_text else month
2634
- if end_month != month:
2635
- return None
2636
- start_day = int(cn_day_range.group("start_day"))
2637
- end_day = int(cn_day_range.group("end_day"))
2638
- try:
2639
- start = date(year, month, start_day)
2640
- end = date(year, month, end_day)
2641
- except ValueError:
2642
- return None
2643
- if end >= start:
2644
- return _format_datetime(start), _format_datetime(end + timedelta(days=1))
2645
- single_cn_day = re.search(r"(?P<month>\d{1,2})\s*月\s*(?P<day>\d{1,2})\s*日", question)
2646
- if single_cn_day is not None and not re.search(r"(到|至|~|~|-)", question):
2647
- year = _today().year
2648
- try:
2649
- day = date(year, int(single_cn_day.group("month")), int(single_cn_day.group("day")))
2650
- except ValueError:
2651
- day = None
2652
- if day is not None:
2653
- return _format_datetime(day), _format_datetime(day + timedelta(days=1))
2654
- cn_month_only = re.search(
2655
- r"(?:整个|整)\s*(?P<month>\d{1,2})\s*月(?:份)?|(?P<month_only>\d{1,2})\s*月(?:份)?(?!\s*\d{1,2}\s*日)",
2656
- question,
2657
- )
2658
- if cn_month_only is not None:
2659
- month_text = cn_month_only.group("month") or cn_month_only.group("month_only")
2660
- if month_text is not None:
2661
- return _month_bounds(_today().year, int(month_text))
2662
- month_match = re.search(r"(?P<year>20\d{2})\s*年\s*(?P<month>\d{1,2})\s*月(?:份)?", question)
2663
- if month_match is None:
2664
- month_match = re.search(r"(?P<year>20\d{2})[-/](?P<month>\d{1,2})(?![-/]\d)", question)
2665
- if month_match is None:
2666
- return None
2667
- year = int(month_match.group("year"))
2668
- month = int(month_match.group("month"))
2669
- return _month_bounds(year, month)
2670
-
2671
-
2672
- def _relative_year_offset(relative_year: str) -> int:
2673
- offsets = {
2674
- "今年": 0,
2675
- "去年": -1,
2676
- "前年": -2,
2677
- "明年": 1,
2678
- "后年": 2,
2679
- }
2680
- return offsets[relative_year]
2681
-
2682
-
2683
- def _one_year_before(value: date) -> date:
2684
- try:
2685
- return value.replace(year=value.year - 1)
2686
- except ValueError:
2687
- return value.replace(year=value.year - 1, day=28)
2688
-
2689
-
2690
- def _month_bounds(year: int, month: int) -> tuple[str, str] | None:
2691
- if not 1 <= month <= 12:
2692
- return None
2693
- start = date(year, month, 1)
2694
- end = date(year + 1, 1, 1) if month == 12 else date(year, month + 1, 1)
2695
- return _format_datetime(start), _format_datetime(end)
2696
-
2697
-
2698
- def _extract_explicit_date_range(question: str) -> tuple[str, str] | None:
2699
- date_pattern = r"20\d{2}[-/]\d{1,2}[-/]\d{1,2}"
2700
- match = re.search(rf"(?P<start>{date_pattern})\s*(?:到|至|~|~)\s*(?P<end>{date_pattern})", question)
2701
- if match is None:
2702
- match = re.search(rf"(?P<start>{date_pattern})\s+-\s+(?P<end>{date_pattern})", question)
2703
- if match is None:
2704
- return None
2705
- start = _parse_date(match.group("start"))
2706
- end = _parse_date(match.group("end"))
2707
- if start is None or end is None or end < start:
2708
- return None
2709
- return _format_datetime(start), _format_datetime(end + timedelta(days=1))
2710
-
2711
-
2712
- def _parse_date(value: str) -> date | None:
2713
- normalized = value.replace("/", "-")
2714
- parts = normalized.split("-")
2715
- if len(parts) != 3:
2716
- return None
2717
- try:
2718
- return date(int(parts[0]), int(parts[1]), int(parts[2]))
2719
- except ValueError:
2720
- return None
2721
-
2722
-
2723
- def _format_datetime(value: date) -> str:
2724
- return f"{value.isoformat()} 00:00:00"
2725
-
2726
-
2727
- def _today() -> date:
2728
- return date.today()
2729
-
2730
-
2731
- def _is_business_query(question: str) -> bool:
2732
- return bool(
2733
- re.search(
2734
- r"(查询|查看|看看|搜索|统计|多少|几|哪些|有哪些|列表|明细|清单|展示|返回|输出|分析|报告|count|sum|avg|min|max)",
2735
- question,
2736
- flags=re.IGNORECASE,
2737
- )
2738
- )
2739
-
2740
-
2741
- def _escape_sql_string(value: str) -> str:
2742
- return value.replace("\\", "\\\\").replace("'", "''")
2743
-
2744
-
2745
- def _enforce_chinese_select_aliases(sql: str) -> None:
2746
- select_list = _top_level_select_list(sql)
2747
- if select_list is None:
2748
- raise SqlGenerationError("SQL must contain a top-level FROM")
2749
- for item in _split_top_level_select_items(select_list):
2750
- cleaned = item.strip()
2751
- if not cleaned:
2752
- continue
2753
- alias_match = re.search(
2754
- _SELECT_AS_ALIAS_PATTERN,
2755
- cleaned,
2756
- flags=re.IGNORECASE,
2757
- )
2758
- if alias_match is None:
2759
- raise SqlGenerationError("Every SELECT field must use AS with a Chinese alias")
2760
- alias = alias_match.group(1).strip("`\"'")
2761
- if re.search(r"[\u4e00-\u9fff]", alias) is None:
2762
- raise SqlGenerationError("Every SELECT alias must contain Chinese characters")
2763
-
2764
-
2765
- def _enforce_no_null_placeholder_select_items(sql: str) -> None:
2766
- select_list = _top_level_select_list(sql)
2767
- if select_list is None:
2768
- raise SqlGenerationError("SQL must contain a top-level FROM")
2769
- for item in _split_top_level_select_items(select_list):
2770
- expression = _select_expression_without_alias(item)
2771
- if _is_null_placeholder_expression(expression):
2772
- raise SqlGenerationError("SELECT fields must not use NULL placeholders")
2773
-
2774
-
2775
- def _select_expression_without_alias(item: str) -> str:
2776
- return re.sub(
2777
- _SELECT_AS_ALIAS_PATTERN,
2778
- "",
2779
- item.strip(),
2780
- flags=re.IGNORECASE,
2781
- ).strip()
2782
-
2783
-
2784
- def _is_null_placeholder_expression(expression: str) -> bool:
2785
- normalized = expression.strip().rstrip(";")
2786
- if re.fullmatch(r"null", normalized, flags=re.IGNORECASE):
2787
- return True
2788
- if re.fullmatch(r"cast\s*\(\s*null\s+as\s+[^)]+\)", normalized, flags=re.IGNORECASE):
2789
- return True
2790
- return bool(re.fullmatch(r"convert\s*\(\s*null\s*,\s*[^)]+\)", normalized, flags=re.IGNORECASE))
2791
-
2792
-
2793
- def _top_level_select_list(sql: str) -> str | None:
2794
- stripped = sql.strip().rstrip(";")
2795
- match = re.match(r"select\s+(?:distinct\s+)?", stripped, flags=re.IGNORECASE)
2796
- if match is None:
2797
- return None
2798
- start = match.end()
2799
- from_index = _find_top_level_keyword(stripped, "from", start=start)
2800
- if from_index is None:
2801
- return None
2802
- return stripped[start:from_index]
2803
-
2804
-
2805
- def _find_top_level_keyword(sql: str, keyword: str, *, start: int) -> int | None:
2806
- depth = 0
2807
- quote: str | None = None
2808
- keyword_lower = keyword.lower()
2809
- index = start
2810
- while index < len(sql):
2811
- char = sql[index]
2812
- if quote is not None:
2813
- if char == quote:
2814
- quote = None
2815
- elif char == "\\":
2816
- index += 1
2817
- index += 1
2818
- continue
2819
- if char in ("'", '"', "`"):
2820
- quote = char
2821
- index += 1
2822
- continue
2823
- if char == "(":
2824
- depth += 1
2825
- index += 1
2826
- continue
2827
- if char == ")":
2828
- depth = max(0, depth - 1)
2829
- index += 1
2830
- continue
2831
- if depth == 0 and sql[index : index + len(keyword_lower)].lower() == keyword_lower:
2832
- before = sql[index - 1] if index > 0 else " "
2833
- after_index = index + len(keyword_lower)
2834
- after = sql[after_index] if after_index < len(sql) else " "
2835
- if not (before.isalnum() or before == "_") and not (after.isalnum() or after == "_"):
2836
- return index
2837
- index += 1
2838
- return None
2839
-
2840
-
2841
- def _split_top_level_select_items(select_list: str) -> list[str]:
2842
- items: list[str] = []
2843
- depth = 0
2844
- quote: str | None = None
2845
- start = 0
2846
- for index, char in enumerate(select_list):
2847
- if quote is not None:
2848
- if char == quote:
2849
- quote = None
2850
- elif char == "\\":
2851
- continue
2852
- continue
2853
- if char in ("'", '"', "`"):
2854
- quote = char
2855
- continue
2856
- if char == "(":
2857
- depth += 1
2858
- continue
2859
- if char == ")":
2860
- depth = max(0, depth - 1)
2861
- continue
2862
- if char == "," and depth == 0:
2863
- items.append(select_list[start:index])
2864
- start = index + 1
2865
- items.append(select_list[start:])
2866
- return items