@roll-agent/octopus-agent 0.1.0 → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/pyproject.toml +1 -1
- package/src/octopus_skill/_version.py +1 -1
- package/src/octopus_skill/agent.py +512 -30
- package/src/octopus_skill/llm_sql_generator.py +55 -1
- package/src/octopus_skill/octopus_run.py +9 -1
- package/src/octopus_skill/prompt_builder.py +60 -3
- package/src/octopus_skill/question_analyzer.py +17 -6
- package/src/octopus_skill/sql_generator.py +136 -10
package/package.json
CHANGED
package/pyproject.toml
CHANGED
|
@@ -25,6 +25,7 @@ from .sql_generator import (
|
|
|
25
25
|
build_sql_rule_validation_error,
|
|
26
26
|
enforce_sql_rules,
|
|
27
27
|
finalize_generated_sql,
|
|
28
|
+
format_sql_quality_issue,
|
|
28
29
|
_is_business_query,
|
|
29
30
|
is_low_quality_sql_for_question,
|
|
30
31
|
low_quality_sql_issue_for_question,
|
|
@@ -365,8 +366,14 @@ class SpongeAgent:
|
|
|
365
366
|
if schema is not None:
|
|
366
367
|
candidate_sql = apply_time_range_filters_from_question(candidate_sql, time_range_question, schema)
|
|
367
368
|
if not sql_satisfies_time_range(candidate_sql, time_range_question, schema, query_plan=query_plan):
|
|
369
|
+
quality_message = _query_quality_failure_message(
|
|
370
|
+
"用户问题包含时间范围,SQL 必须在 WHERE 条件中直接包含对应时间字段过滤。",
|
|
371
|
+
missing_items=["用户要求的时间范围过滤"],
|
|
372
|
+
current_problem="当前 SQL 没有覆盖用户问题中的时间范围,执行后可能返回全量或错误时间段的数据。",
|
|
373
|
+
repair_hint="按 schema concepts/metrics 或查询计划中的时间字段补充 WHERE 时间范围条件。",
|
|
374
|
+
)
|
|
368
375
|
if sampler is None or schema_context is None or generator is None:
|
|
369
|
-
reason = "
|
|
376
|
+
reason = f"{quality_message};当前没有 sampler 可修复。"
|
|
370
377
|
steps.error(8, reason)
|
|
371
378
|
return _attach_execution_steps(_query_only_result(reason), steps)
|
|
372
379
|
repaired_sql, fail_reason = _repair_sql_with_llm_retries(
|
|
@@ -375,8 +382,11 @@ class SpongeAgent:
|
|
|
375
382
|
sql=candidate_sql,
|
|
376
383
|
validation_error={
|
|
377
384
|
"code": "MISSING_TIME_RANGE_FILTER",
|
|
378
|
-
"message":
|
|
385
|
+
"message": quality_message,
|
|
379
386
|
"originalSql": candidate_sql,
|
|
387
|
+
"missingItems": ["用户要求的时间范围过滤"],
|
|
388
|
+
"currentProblem": "当前 SQL 没有覆盖用户问题中的时间范围。",
|
|
389
|
+
"repairHint": "按 schema concepts/metrics 或查询计划中的时间字段补充 WHERE 时间范围条件。",
|
|
380
390
|
},
|
|
381
391
|
schema_context=sql_schema_context,
|
|
382
392
|
audit_logger=self.audit_logger,
|
|
@@ -387,8 +397,8 @@ class SpongeAgent:
|
|
|
387
397
|
schema=schema,
|
|
388
398
|
check=lambda candidate: _require_time_range(candidate, time_range_question, schema, query_plan),
|
|
389
399
|
steps=steps,
|
|
390
|
-
running_detail="SQL
|
|
391
|
-
failure_detail="SQL
|
|
400
|
+
running_detail=f"SQL 口径校验失败,尝试修复:{quality_message}",
|
|
401
|
+
failure_detail=f"SQL 口径校验失败,修复失败:{quality_message}",
|
|
392
402
|
)
|
|
393
403
|
if repaired_sql is None:
|
|
394
404
|
steps.error(9, fail_reason)
|
|
@@ -398,8 +408,14 @@ class SpongeAgent:
|
|
|
398
408
|
steps.done(9, "已补充时间范围过滤。")
|
|
399
409
|
if query_plan and not sql_satisfies_query_plan_filters(candidate_sql, query_plan):
|
|
400
410
|
missing_filters = sql_missing_query_plan_filters(candidate_sql, query_plan)
|
|
411
|
+
quality_message = _query_quality_failure_message(
|
|
412
|
+
f"SQL 缺少查询计划中的筛选条件:{'; '.join(missing_filters)}。",
|
|
413
|
+
missing_items=missing_filters,
|
|
414
|
+
current_problem="当前 SQL 没有覆盖用户已确认的关键筛选,执行后可能返回范围过宽或不相关的数据。",
|
|
415
|
+
repair_hint="必须 JOIN 相关表,并把缺失的 table.column 过滤写入 WHERE;同组 OR 条件需要用括号包裹。",
|
|
416
|
+
)
|
|
401
417
|
if sampler is None or schema_context is None or generator is None:
|
|
402
|
-
reason =
|
|
418
|
+
reason = quality_message
|
|
403
419
|
steps.error(8, reason)
|
|
404
420
|
return _attach_execution_steps(_query_only_result(reason), steps)
|
|
405
421
|
repaired_sql, fail_reason = _repair_sql_with_llm_retries(
|
|
@@ -408,12 +424,12 @@ class SpongeAgent:
|
|
|
408
424
|
sql=candidate_sql,
|
|
409
425
|
validation_error={
|
|
410
426
|
"code": "MISSING_QUERY_PLAN_FILTER",
|
|
411
|
-
"message":
|
|
412
|
-
f"SQL 缺少查询计划中的筛选条件:{'; '.join(missing_filters)}。"
|
|
413
|
-
"必须 JOIN 相关表并在 WHERE 中包含全部 table.column 过滤。"
|
|
414
|
-
),
|
|
427
|
+
"message": quality_message,
|
|
415
428
|
"originalSql": candidate_sql,
|
|
416
429
|
"missingFilters": missing_filters,
|
|
430
|
+
"missingItems": missing_filters,
|
|
431
|
+
"currentProblem": "当前 SQL 没有覆盖用户已确认的关键筛选。",
|
|
432
|
+
"repairHint": "必须 JOIN 相关表,并把缺失的 table.column 过滤写入 WHERE。",
|
|
417
433
|
},
|
|
418
434
|
schema_context=sql_schema_context,
|
|
419
435
|
audit_logger=self.audit_logger,
|
|
@@ -424,8 +440,8 @@ class SpongeAgent:
|
|
|
424
440
|
schema=schema,
|
|
425
441
|
check=lambda candidate: sql_satisfies_query_plan_filters(candidate, query_plan),
|
|
426
442
|
steps=steps,
|
|
427
|
-
running_detail=f"SQL
|
|
428
|
-
failure_detail="SQL
|
|
443
|
+
running_detail=f"SQL 口径校验失败,尝试修复:{quality_message}",
|
|
444
|
+
failure_detail=f"SQL 口径校验失败,修复失败:{quality_message}",
|
|
429
445
|
)
|
|
430
446
|
if repaired_sql is None:
|
|
431
447
|
steps.error(9, fail_reason)
|
|
@@ -435,8 +451,14 @@ class SpongeAgent:
|
|
|
435
451
|
steps.done(9, "已补充查询计划筛选条件。")
|
|
436
452
|
if query_plan and not sql_satisfies_query_plan_select_fields(candidate_sql, query_plan):
|
|
437
453
|
missing_select_fields = sql_missing_query_plan_select_fields(candidate_sql, query_plan)
|
|
454
|
+
quality_message = _query_quality_failure_message(
|
|
455
|
+
f"SQL 缺少查询计划中的展示字段:{'; '.join(missing_select_fields)}。",
|
|
456
|
+
missing_items=missing_select_fields,
|
|
457
|
+
current_problem="当前 SQL 没有展示用户已确认的必要字段,结果无法解释或无法区分命中的实体。",
|
|
458
|
+
repair_hint="把缺失字段加入 SELECT,并使用中文别名。",
|
|
459
|
+
)
|
|
438
460
|
if sampler is None or schema_context is None or generator is None:
|
|
439
|
-
reason =
|
|
461
|
+
reason = quality_message
|
|
440
462
|
steps.error(8, reason)
|
|
441
463
|
return _attach_execution_steps(_query_only_result(reason), steps)
|
|
442
464
|
repaired_sql, fail_reason = _repair_sql_with_llm_retries(
|
|
@@ -445,12 +467,12 @@ class SpongeAgent:
|
|
|
445
467
|
sql=candidate_sql,
|
|
446
468
|
validation_error={
|
|
447
469
|
"code": "MISSING_QUERY_PLAN_SELECT_FIELD",
|
|
448
|
-
"message":
|
|
449
|
-
f"SQL 缺少查询计划中的展示字段:{'; '.join(missing_select_fields)}。"
|
|
450
|
-
"同一别称命中多个实体类型时,必须在 SELECT 中展示各实体名称字段,方便用户自行缩小范围。"
|
|
451
|
-
),
|
|
470
|
+
"message": quality_message,
|
|
452
471
|
"originalSql": candidate_sql,
|
|
453
472
|
"missingSelectFields": missing_select_fields,
|
|
473
|
+
"missingItems": missing_select_fields,
|
|
474
|
+
"currentProblem": "当前 SQL 没有展示用户已确认的必要字段。",
|
|
475
|
+
"repairHint": "把缺失字段加入 SELECT,并使用中文别名。",
|
|
454
476
|
},
|
|
455
477
|
schema_context=sql_schema_context,
|
|
456
478
|
audit_logger=self.audit_logger,
|
|
@@ -461,8 +483,8 @@ class SpongeAgent:
|
|
|
461
483
|
schema=schema,
|
|
462
484
|
check=lambda candidate: sql_satisfies_query_plan_select_fields(candidate, query_plan),
|
|
463
485
|
steps=steps,
|
|
464
|
-
running_detail=f"SQL
|
|
465
|
-
failure_detail="SQL
|
|
486
|
+
running_detail=f"SQL 口径校验失败,尝试修复:{quality_message}",
|
|
487
|
+
failure_detail=f"SQL 口径校验失败,修复失败:{quality_message}",
|
|
466
488
|
)
|
|
467
489
|
if repaired_sql is None:
|
|
468
490
|
steps.error(9, fail_reason)
|
|
@@ -472,17 +494,21 @@ class SpongeAgent:
|
|
|
472
494
|
steps.done(9, "已补充查询计划展示字段。")
|
|
473
495
|
low_quality_issue = low_quality_sql_issue_for_question(sql_question, candidate_sql, schema=schema)
|
|
474
496
|
if low_quality_issue is not None:
|
|
497
|
+
quality_message = format_sql_quality_issue(low_quality_issue)
|
|
475
498
|
if sampler is None or schema_context is None or generator is None:
|
|
476
|
-
steps.error(8,
|
|
477
|
-
return _attach_execution_steps(_low_quality_sql_result(
|
|
499
|
+
steps.error(8, quality_message)
|
|
500
|
+
return _attach_execution_steps(_low_quality_sql_result(quality_message, candidate_sql), steps)
|
|
478
501
|
repaired_sql, fail_reason = _repair_sql_with_llm_retries(
|
|
479
502
|
generator=generator,
|
|
480
503
|
question=sql_question,
|
|
481
504
|
sql=candidate_sql,
|
|
482
505
|
validation_error={
|
|
483
506
|
"code": low_quality_issue.code,
|
|
484
|
-
"message":
|
|
507
|
+
"message": quality_message,
|
|
485
508
|
"originalSql": candidate_sql,
|
|
509
|
+
"missingItems": list(low_quality_issue.missing_items),
|
|
510
|
+
"currentProblem": low_quality_issue.current_problem,
|
|
511
|
+
"repairHint": low_quality_issue.repair_hint,
|
|
486
512
|
},
|
|
487
513
|
schema_context=sql_schema_context,
|
|
488
514
|
audit_logger=self.audit_logger,
|
|
@@ -493,8 +519,8 @@ class SpongeAgent:
|
|
|
493
519
|
schema=schema,
|
|
494
520
|
check=lambda candidate: _require_low_quality_pass(candidate, sql_question, schema),
|
|
495
521
|
steps=steps,
|
|
496
|
-
running_detail=f"SQL
|
|
497
|
-
failure_detail=f"SQL
|
|
522
|
+
running_detail=f"SQL 口径校验失败,尝试修复:{quality_message}",
|
|
523
|
+
failure_detail=f"SQL 口径校验失败,修复失败:{quality_message}",
|
|
498
524
|
)
|
|
499
525
|
if repaired_sql is None:
|
|
500
526
|
steps.error(9, fail_reason)
|
|
@@ -634,6 +660,28 @@ class SpongeAgent:
|
|
|
634
660
|
return _attach_execution_steps(_query_only_result(schema_fail), steps)
|
|
635
661
|
generated = GeneratedSql(sql="", tables=[], placeholders=[])
|
|
636
662
|
candidate_sql = _remove_noise_name_like_filters_from_sql(candidate_sql, analysis=analysis)
|
|
663
|
+
if candidate_sql != prior_sql:
|
|
664
|
+
generated = GeneratedSql(sql="", tables=[], placeholders=[])
|
|
665
|
+
repaired_sql = candidate_sql
|
|
666
|
+
prior_sql = candidate_sql
|
|
667
|
+
candidate_sql, expert_fail = _review_sql_with_llm_expert_if_needed(
|
|
668
|
+
generator=generator,
|
|
669
|
+
question=sql_question,
|
|
670
|
+
sql=candidate_sql,
|
|
671
|
+
schema_context=sql_schema_context,
|
|
672
|
+
schema=schema,
|
|
673
|
+
query_plan=query_plan,
|
|
674
|
+
time_range_question=time_range_question,
|
|
675
|
+
default_limit=self.config.sql.default_limit,
|
|
676
|
+
max_limit=self.config.sql.max_limit,
|
|
677
|
+
max_repair_attempts=self.config.sql.max_repair_attempts,
|
|
678
|
+
steps=steps,
|
|
679
|
+
audit_logger=self.audit_logger,
|
|
680
|
+
session_id=context.session_id,
|
|
681
|
+
)
|
|
682
|
+
if expert_fail:
|
|
683
|
+
steps.error(10, expert_fail)
|
|
684
|
+
return _attach_execution_steps(_query_only_result(expert_fail), steps)
|
|
637
685
|
if candidate_sql != prior_sql:
|
|
638
686
|
generated = GeneratedSql(sql="", tables=[], placeholders=[])
|
|
639
687
|
repaired_sql = candidate_sql
|
|
@@ -790,7 +838,66 @@ class SpongeAgent:
|
|
|
790
838
|
result = self._execute_sql(context, user_question, normalized_sql)
|
|
791
839
|
except Exception as exc:
|
|
792
840
|
unknown_col = _parse_execute_unknown_column(str(exc))
|
|
793
|
-
|
|
841
|
+
invalid_group_function = _is_execute_invalid_group_function(str(exc))
|
|
842
|
+
if invalid_group_function and sampler is not None and generator is not None and schema_context is not None:
|
|
843
|
+
exec_error = {
|
|
844
|
+
"code": "MYSQL_INVALID_GROUP_FUNCTION_USE",
|
|
845
|
+
"message": (
|
|
846
|
+
"MySQL 执行失败:Invalid use of group function。"
|
|
847
|
+
"通常是聚合函数写在 WHERE 或 JOIN ON 中,或聚合筛选没有放到 HAVING/子查询外层。"
|
|
848
|
+
),
|
|
849
|
+
"originalSql": normalized_sql,
|
|
850
|
+
"repairHint": "把聚合条件移动到 HAVING,或先用子查询聚合后在外层 WHERE 过滤;不要改变业务口径。",
|
|
851
|
+
}
|
|
852
|
+
repaired_exec_sql, fail_reason = _repair_sql_with_llm_retries(
|
|
853
|
+
generator=generator,
|
|
854
|
+
question=sql_question,
|
|
855
|
+
sql=normalized_sql,
|
|
856
|
+
validation_error=exec_error,
|
|
857
|
+
schema_context=sql_schema_context,
|
|
858
|
+
audit_logger=self.audit_logger,
|
|
859
|
+
session_id=context.session_id,
|
|
860
|
+
max_attempts=self.config.sql.max_repair_attempts,
|
|
861
|
+
default_limit=self.config.sql.default_limit,
|
|
862
|
+
max_limit=self.config.sql.max_limit,
|
|
863
|
+
schema=schema,
|
|
864
|
+
check=lambda candidate: _require_sql_expert_local_pass(candidate, schema, self.config.sql.max_limit),
|
|
865
|
+
steps=steps,
|
|
866
|
+
running_detail="execute_sql 报聚合函数用法错误,尝试修复",
|
|
867
|
+
failure_detail="execute_sql 聚合函数用法错误修复失败",
|
|
868
|
+
)
|
|
869
|
+
if repaired_exec_sql is None:
|
|
870
|
+
reason = fail_reason or f"{type(exc).__name__}: {exc}"
|
|
871
|
+
steps.error(11, reason)
|
|
872
|
+
return _attach_execution_steps(_query_only_result(reason), steps)
|
|
873
|
+
if query_plan:
|
|
874
|
+
repaired_exec_sql = apply_time_range_filters_from_plan(repaired_exec_sql, query_plan)
|
|
875
|
+
if schema is not None:
|
|
876
|
+
repaired_exec_sql = apply_time_range_filters_from_question(repaired_exec_sql, time_range_question, schema)
|
|
877
|
+
try:
|
|
878
|
+
_require_sql_expert_local_pass(repaired_exec_sql, schema, self.config.sql.max_limit)
|
|
879
|
+
revalidated = self._validate_sql(context, user_question, repaired_exec_sql)
|
|
880
|
+
except Exception as validate_exc:
|
|
881
|
+
reason = f"{type(validate_exc).__name__}: {validate_exc}"
|
|
882
|
+
steps.error(10, reason)
|
|
883
|
+
return _attach_execution_steps(_query_only_result(reason), steps)
|
|
884
|
+
if revalidated is None:
|
|
885
|
+
reason = (
|
|
886
|
+
f"execute 聚合函数用法修复后 validate_sql 失败:"
|
|
887
|
+
f"{_validation_error_message(self._last_validation_result)}"
|
|
888
|
+
)
|
|
889
|
+
steps.error(10, reason)
|
|
890
|
+
return _attach_execution_steps(_query_only_result(reason), steps)
|
|
891
|
+
try:
|
|
892
|
+
result = self._execute_sql(context, user_question, revalidated)
|
|
893
|
+
normalized_sql = revalidated
|
|
894
|
+
repaired_sql = repaired_exec_sql
|
|
895
|
+
steps.done(11, f"execute_sql 完成,rowCount={result.get('rowCount')}")
|
|
896
|
+
except Exception as retry_exc:
|
|
897
|
+
reason = f"{type(retry_exc).__name__}: {retry_exc}"
|
|
898
|
+
steps.error(11, reason)
|
|
899
|
+
return _attach_execution_steps(_query_only_result(reason), steps)
|
|
900
|
+
elif (
|
|
794
901
|
unknown_col
|
|
795
902
|
and sampler is not None
|
|
796
903
|
and generator is not None
|
|
@@ -1303,7 +1410,7 @@ def _build_query_plan(
|
|
|
1303
1410
|
analysis=analysis,
|
|
1304
1411
|
)
|
|
1305
1412
|
selected_concepts = match_concepts_for_question(getattr(schema_context, "concepts", []) or [], question)
|
|
1306
|
-
|
|
1413
|
+
sql_filters = _query_plan_sql_filters(
|
|
1307
1414
|
question,
|
|
1308
1415
|
schema_context,
|
|
1309
1416
|
searched_entities=searched_entities,
|
|
@@ -1313,6 +1420,11 @@ def _build_query_plan(
|
|
|
1313
1420
|
selected_concepts=selected_concepts,
|
|
1314
1421
|
analysis=analysis,
|
|
1315
1422
|
)
|
|
1423
|
+
filters = _query_plan_semantic_filters(
|
|
1424
|
+
question,
|
|
1425
|
+
sql_filters=sql_filters,
|
|
1426
|
+
analysis=analysis,
|
|
1427
|
+
)
|
|
1316
1428
|
entities = _query_plan_entities(
|
|
1317
1429
|
question,
|
|
1318
1430
|
schema_context,
|
|
@@ -1340,6 +1452,7 @@ def _build_query_plan(
|
|
|
1340
1452
|
"tables": tables,
|
|
1341
1453
|
"fields": fields,
|
|
1342
1454
|
"filters": filters,
|
|
1455
|
+
"_sqlFilters": sql_filters,
|
|
1343
1456
|
"entities": entities,
|
|
1344
1457
|
"matchedConcepts": [
|
|
1345
1458
|
{
|
|
@@ -1390,7 +1503,7 @@ def _ensure_query_plan_bridge_tables(plan: dict[str, Any]) -> None:
|
|
|
1390
1503
|
job_tables = {"job", "job_basic_info", "job_address", "job_store", "job_hiring_requirement"}
|
|
1391
1504
|
has_store = "store" in names
|
|
1392
1505
|
if not has_store:
|
|
1393
|
-
for flt in plan
|
|
1506
|
+
for flt in _plan_sql_filters(plan):
|
|
1394
1507
|
if isinstance(flt, dict) and str(flt.get("field") or "").startswith("store."):
|
|
1395
1508
|
has_store = True
|
|
1396
1509
|
break
|
|
@@ -1420,7 +1533,7 @@ def _ensure_query_plan_entity_display_fields(plan: dict[str, Any]) -> None:
|
|
|
1420
1533
|
for item in fields
|
|
1421
1534
|
if isinstance(item, dict)
|
|
1422
1535
|
}
|
|
1423
|
-
for flt in plan
|
|
1536
|
+
for flt in _plan_sql_filters(plan):
|
|
1424
1537
|
if not isinstance(flt, dict):
|
|
1425
1538
|
continue
|
|
1426
1539
|
if str(flt.get("source") or "") != "search_entities":
|
|
@@ -1460,6 +1573,14 @@ def _ensure_query_plan_entity_display_fields(plan: dict[str, Any]) -> None:
|
|
|
1460
1573
|
existing.add(key)
|
|
1461
1574
|
|
|
1462
1575
|
|
|
1576
|
+
def _plan_sql_filters(plan: dict[str, Any]) -> list[Any]:
|
|
1577
|
+
filters = plan.get("_sqlFilters")
|
|
1578
|
+
if isinstance(filters, list):
|
|
1579
|
+
return filters
|
|
1580
|
+
filters = plan.get("filters")
|
|
1581
|
+
return filters if isinstance(filters, list) else []
|
|
1582
|
+
|
|
1583
|
+
|
|
1463
1584
|
def _query_plan_tables(tables: list[dict[str, Any]]) -> list[dict[str, str]]:
|
|
1464
1585
|
result: list[dict[str, str]] = []
|
|
1465
1586
|
for table in tables[:10]:
|
|
@@ -2653,6 +2774,172 @@ def _query_plan_filters(
|
|
|
2653
2774
|
time_range_question: str | None = None,
|
|
2654
2775
|
selected_concepts: list[dict[str, Any]] | None = None,
|
|
2655
2776
|
analysis: QuestionAnalysis | None = None,
|
|
2777
|
+
) -> list[dict[str, str]]:
|
|
2778
|
+
sql_filters = _query_plan_sql_filters(
|
|
2779
|
+
question,
|
|
2780
|
+
schema_context,
|
|
2781
|
+
searched_entities=searched_entities,
|
|
2782
|
+
question_entities=question_entities,
|
|
2783
|
+
schema=schema,
|
|
2784
|
+
time_range_question=time_range_question,
|
|
2785
|
+
selected_concepts=selected_concepts,
|
|
2786
|
+
analysis=analysis,
|
|
2787
|
+
)
|
|
2788
|
+
return _query_plan_semantic_filters(
|
|
2789
|
+
question,
|
|
2790
|
+
sql_filters=sql_filters,
|
|
2791
|
+
analysis=analysis,
|
|
2792
|
+
)
|
|
2793
|
+
|
|
2794
|
+
|
|
2795
|
+
def _query_plan_semantic_filters(
|
|
2796
|
+
question: str,
|
|
2797
|
+
*,
|
|
2798
|
+
sql_filters: list[dict[str, str]],
|
|
2799
|
+
analysis: QuestionAnalysis | None = None,
|
|
2800
|
+
) -> list[dict[str, str]]:
|
|
2801
|
+
filters: list[dict[str, str]] = []
|
|
2802
|
+
|
|
2803
|
+
def append_filter(item: dict[str, str]) -> None:
|
|
2804
|
+
field = str(item.get("field") or "").strip()
|
|
2805
|
+
value = str(item.get("value") or "").strip()
|
|
2806
|
+
if not field or not value:
|
|
2807
|
+
return
|
|
2808
|
+
operator = str(item.get("operator") or "=").strip() or "="
|
|
2809
|
+
normalized = {**item, "field": field, "operator": operator, "value": value}
|
|
2810
|
+
key = (
|
|
2811
|
+
field,
|
|
2812
|
+
operator.upper(),
|
|
2813
|
+
value,
|
|
2814
|
+
str(normalized.get("logicalGroup") or ""),
|
|
2815
|
+
str(normalized.get("logicalOperator") or ""),
|
|
2816
|
+
)
|
|
2817
|
+
if any(
|
|
2818
|
+
(
|
|
2819
|
+
existing.get("field"),
|
|
2820
|
+
str(existing.get("operator") or "").upper(),
|
|
2821
|
+
existing.get("value"),
|
|
2822
|
+
str(existing.get("logicalGroup") or ""),
|
|
2823
|
+
str(existing.get("logicalOperator") or ""),
|
|
2824
|
+
)
|
|
2825
|
+
== key
|
|
2826
|
+
for existing in filters
|
|
2827
|
+
):
|
|
2828
|
+
return
|
|
2829
|
+
filters.append(normalized)
|
|
2830
|
+
|
|
2831
|
+
if analysis is not None:
|
|
2832
|
+
for item in analysis.filters:
|
|
2833
|
+
if not isinstance(item, dict):
|
|
2834
|
+
continue
|
|
2835
|
+
append_filter(
|
|
2836
|
+
{
|
|
2837
|
+
"field": str(item.get("field") or "筛选条件"),
|
|
2838
|
+
"operator": str(item.get("operator") or "="),
|
|
2839
|
+
"value": str(item.get("value") or ""),
|
|
2840
|
+
"source": "llm.semanticFilter",
|
|
2841
|
+
}
|
|
2842
|
+
)
|
|
2843
|
+
time_filter = _semantic_time_filter_from_analysis(analysis)
|
|
2844
|
+
if time_filter is not None:
|
|
2845
|
+
append_filter(time_filter)
|
|
2846
|
+
|
|
2847
|
+
if not any(str(item.get("source") or "") == "question.timeRange" for item in filters):
|
|
2848
|
+
time_filter = _semantic_time_filter_from_sql_filters(sql_filters)
|
|
2849
|
+
if time_filter is not None:
|
|
2850
|
+
append_filter(time_filter)
|
|
2851
|
+
|
|
2852
|
+
for item in sql_filters:
|
|
2853
|
+
semantic = _semantic_filter_from_sql_filter(item)
|
|
2854
|
+
if semantic is not None:
|
|
2855
|
+
append_filter(semantic)
|
|
2856
|
+
return filters
|
|
2857
|
+
|
|
2858
|
+
|
|
2859
|
+
def _semantic_time_filter_from_analysis(analysis: QuestionAnalysis) -> dict[str, str] | None:
|
|
2860
|
+
time_range = analysis.time_range
|
|
2861
|
+
if not isinstance(time_range, dict):
|
|
2862
|
+
return None
|
|
2863
|
+
value = str(time_range.get("originalText") or time_range.get("value") or "").strip()
|
|
2864
|
+
if not value:
|
|
2865
|
+
start = str(time_range.get("start") or "").strip()
|
|
2866
|
+
end = str(time_range.get("endExclusive") or "").strip()
|
|
2867
|
+
if start and end:
|
|
2868
|
+
value = f"{start} 至 {end}"
|
|
2869
|
+
if not value:
|
|
2870
|
+
return None
|
|
2871
|
+
return {
|
|
2872
|
+
"field": "时间范围",
|
|
2873
|
+
"operator": "=",
|
|
2874
|
+
"value": value,
|
|
2875
|
+
"source": "question.timeRange",
|
|
2876
|
+
}
|
|
2877
|
+
|
|
2878
|
+
|
|
2879
|
+
def _semantic_time_filter_from_sql_filters(filters: list[dict[str, str]]) -> dict[str, str] | None:
|
|
2880
|
+
start = ""
|
|
2881
|
+
end = ""
|
|
2882
|
+
for item in filters:
|
|
2883
|
+
if str(item.get("source") or "") != "question.timeRange":
|
|
2884
|
+
continue
|
|
2885
|
+
operator = str(item.get("operator") or "")
|
|
2886
|
+
value = str(item.get("value") or "").strip()
|
|
2887
|
+
if operator == ">=":
|
|
2888
|
+
start = value
|
|
2889
|
+
elif operator == "<":
|
|
2890
|
+
end = value
|
|
2891
|
+
if not start and not end:
|
|
2892
|
+
return None
|
|
2893
|
+
value = f"{start} 至 {end}" if start and end else start or end
|
|
2894
|
+
return {
|
|
2895
|
+
"field": "时间范围",
|
|
2896
|
+
"operator": "BETWEEN" if start and end else "=",
|
|
2897
|
+
"value": value,
|
|
2898
|
+
"source": "question.timeRange",
|
|
2899
|
+
}
|
|
2900
|
+
|
|
2901
|
+
|
|
2902
|
+
_TECHNICAL_QUERY_PLAN_FILTER_SOURCES = frozenset({"schema.concepts", "structure.defaultFilters"})
|
|
2903
|
+
|
|
2904
|
+
|
|
2905
|
+
def _semantic_filter_from_sql_filter(item: dict[str, str]) -> dict[str, str] | None:
|
|
2906
|
+
source = str(item.get("source") or "")
|
|
2907
|
+
operator = str(item.get("operator") or "").upper()
|
|
2908
|
+
if source in _TECHNICAL_QUERY_PLAN_FILTER_SOURCES or operator == "DEFAULT":
|
|
2909
|
+
return None
|
|
2910
|
+
if source in {"question.timeRange", "question_analysis.timeRange"}:
|
|
2911
|
+
return None
|
|
2912
|
+
field = str(item.get("field") or "").strip()
|
|
2913
|
+
value = str(item.get("value") or "").strip()
|
|
2914
|
+
if not field or not value:
|
|
2915
|
+
return None
|
|
2916
|
+
label = _query_plan_filter_display_label(field, item)
|
|
2917
|
+
display_value = _query_plan_filter_display_value(value, item)
|
|
2918
|
+
if not label or not display_value:
|
|
2919
|
+
return None
|
|
2920
|
+
display_operator = "=" if operator in {"", "LIKE"} else str(item.get("operator") or "=")
|
|
2921
|
+
semantic = {
|
|
2922
|
+
"field": label,
|
|
2923
|
+
"operator": display_operator,
|
|
2924
|
+
"value": display_value,
|
|
2925
|
+
"source": source or "queryPlan.sqlFilter",
|
|
2926
|
+
}
|
|
2927
|
+
for key in ("description", "logicalGroup", "logicalOperator"):
|
|
2928
|
+
if key in item:
|
|
2929
|
+
semantic[key] = str(item[key])
|
|
2930
|
+
return semantic
|
|
2931
|
+
|
|
2932
|
+
|
|
2933
|
+
def _query_plan_sql_filters(
|
|
2934
|
+
question: str,
|
|
2935
|
+
schema_context: Any,
|
|
2936
|
+
*,
|
|
2937
|
+
searched_entities: dict[str, Any] | None = None,
|
|
2938
|
+
question_entities: list[dict[str, str]] | None = None,
|
|
2939
|
+
schema: dict[str, Any] | None = None,
|
|
2940
|
+
time_range_question: str | None = None,
|
|
2941
|
+
selected_concepts: list[dict[str, Any]] | None = None,
|
|
2942
|
+
analysis: QuestionAnalysis | None = None,
|
|
2656
2943
|
) -> list[dict[str, str]]:
|
|
2657
2944
|
filters: list[dict[str, str]] = []
|
|
2658
2945
|
|
|
@@ -2787,7 +3074,12 @@ def _apply_query_plan_time_range_filters(
|
|
|
2787
3074
|
analysis: QuestionAnalysis | None = None,
|
|
2788
3075
|
) -> list[dict[str, str]]:
|
|
2789
3076
|
"""Replace noisy date LIKE filters with explicit range filters defaulting to current year."""
|
|
2790
|
-
time_filters = _analysis_time_range_plan_filters(
|
|
3077
|
+
time_filters = _analysis_time_range_plan_filters(
|
|
3078
|
+
analysis,
|
|
3079
|
+
selected_concepts=selected_concepts,
|
|
3080
|
+
question=question,
|
|
3081
|
+
schema=schema,
|
|
3082
|
+
)
|
|
2791
3083
|
if not time_filters:
|
|
2792
3084
|
time_filters = time_range_plan_filters(question, schema, concepts=selected_concepts)
|
|
2793
3085
|
if not time_filters:
|
|
@@ -2812,6 +3104,8 @@ def _analysis_time_range_plan_filters(
|
|
|
2812
3104
|
analysis: QuestionAnalysis | None,
|
|
2813
3105
|
*,
|
|
2814
3106
|
selected_concepts: list[dict[str, Any]] | None = None,
|
|
3107
|
+
question: str | None = None,
|
|
3108
|
+
schema: dict[str, Any] | None = None,
|
|
2815
3109
|
) -> list[dict[str, str]]:
|
|
2816
3110
|
if analysis is None or not isinstance(analysis.time_range, dict):
|
|
2817
3111
|
return []
|
|
@@ -2819,7 +3113,7 @@ def _analysis_time_range_plan_filters(
|
|
|
2819
3113
|
end = str(analysis.time_range.get("endExclusive") or analysis.time_range.get("end") or "").strip()
|
|
2820
3114
|
if not (_looks_like_iso_date(start) and _looks_like_iso_date(end)):
|
|
2821
3115
|
return []
|
|
2822
|
-
field = _time_range_field_from_concepts(selected_concepts)
|
|
3116
|
+
field = _time_range_field_from_question_schema(question, schema) or _time_range_field_from_concepts(selected_concepts)
|
|
2823
3117
|
if not field:
|
|
2824
3118
|
return []
|
|
2825
3119
|
original = str(analysis.time_range.get("originalText") or analysis.time_range.get("value") or "时间范围").strip()
|
|
@@ -2841,6 +3135,20 @@ def _analysis_time_range_plan_filters(
|
|
|
2841
3135
|
]
|
|
2842
3136
|
|
|
2843
3137
|
|
|
3138
|
+
def _time_range_field_from_question_schema(
|
|
3139
|
+
question: str | None,
|
|
3140
|
+
schema: dict[str, Any] | None,
|
|
3141
|
+
) -> str:
|
|
3142
|
+
if not question or schema is None:
|
|
3143
|
+
return ""
|
|
3144
|
+
filters = time_range_plan_filters(question, schema, concepts=[])
|
|
3145
|
+
for item in filters:
|
|
3146
|
+
field = str(item.get("field") or "").strip()
|
|
3147
|
+
if "." in field:
|
|
3148
|
+
return field
|
|
3149
|
+
return ""
|
|
3150
|
+
|
|
3151
|
+
|
|
2844
3152
|
def _time_range_field_from_concepts(concepts: list[dict[str, Any]] | None) -> str:
|
|
2845
3153
|
for concept in concepts or []:
|
|
2846
3154
|
if not isinstance(concept, dict):
|
|
@@ -3533,8 +3841,11 @@ def _query_plan_summary_filters(query_plan: dict[str, Any]) -> list[str]:
|
|
|
3533
3841
|
for item in filters:
|
|
3534
3842
|
if not isinstance(item, dict):
|
|
3535
3843
|
continue
|
|
3536
|
-
|
|
3844
|
+
source = str(item.get("source") or "")
|
|
3537
3845
|
operator = str(item.get("operator") or "")
|
|
3846
|
+
if source in _TECHNICAL_QUERY_PLAN_FILTER_SOURCES or operator.upper() == "DEFAULT":
|
|
3847
|
+
continue
|
|
3848
|
+
field = str(item.get("field") or "")
|
|
3538
3849
|
value = str(item.get("value") or "")
|
|
3539
3850
|
label = _query_plan_filter_display_label(field, item)
|
|
3540
3851
|
display_value = _query_plan_filter_display_value(value, item)
|
|
@@ -3723,6 +4034,10 @@ def _parse_execute_unknown_column(error_text: str) -> str | None:
|
|
|
3723
4034
|
return match.group(1).strip()
|
|
3724
4035
|
|
|
3725
4036
|
|
|
4037
|
+
def _is_execute_invalid_group_function(error_text: str) -> bool:
|
|
4038
|
+
return bool(re.search(r"invalid use of group function|ER_INVALID_GROUP_FUNC_USE", error_text, re.IGNORECASE))
|
|
4039
|
+
|
|
4040
|
+
|
|
3726
4041
|
def _repair_sql_for_local_schema_issues(
|
|
3727
4042
|
*,
|
|
3728
4043
|
candidate_sql: str,
|
|
@@ -3932,6 +4247,154 @@ def _repair_sql_with_llm_retries(
|
|
|
3932
4247
|
return None, last_reason
|
|
3933
4248
|
|
|
3934
4249
|
|
|
4250
|
+
def _review_sql_with_llm_expert_if_needed(
|
|
4251
|
+
*,
|
|
4252
|
+
generator: SamplingSqlGenerator | None,
|
|
4253
|
+
question: str,
|
|
4254
|
+
sql: str,
|
|
4255
|
+
schema_context: Any,
|
|
4256
|
+
schema: dict[str, Any],
|
|
4257
|
+
query_plan: dict[str, Any] | None,
|
|
4258
|
+
time_range_question: str,
|
|
4259
|
+
default_limit: int,
|
|
4260
|
+
max_limit: int,
|
|
4261
|
+
max_repair_attempts: int,
|
|
4262
|
+
steps: _ExecutionSteps,
|
|
4263
|
+
audit_logger: AuditLogger,
|
|
4264
|
+
session_id: str | None,
|
|
4265
|
+
) -> tuple[str, str | None]:
|
|
4266
|
+
if generator is None or schema_context is None or not _should_llm_sql_expert_review(sql, query_plan):
|
|
4267
|
+
return sql, None
|
|
4268
|
+
steps.running(8, "LLM SQL 专家复核高风险 SQL 写法。")
|
|
4269
|
+
try:
|
|
4270
|
+
review = generator.review_sql(
|
|
4271
|
+
question=question,
|
|
4272
|
+
sql=sql,
|
|
4273
|
+
schema_context=schema_context,
|
|
4274
|
+
)
|
|
4275
|
+
except SqlGenerationError as exc:
|
|
4276
|
+
audit_logger.log(
|
|
4277
|
+
{
|
|
4278
|
+
"sessionId": session_id,
|
|
4279
|
+
"question": question,
|
|
4280
|
+
"tool": "llm_sql_expert_review",
|
|
4281
|
+
"success": False,
|
|
4282
|
+
"error": str(exc),
|
|
4283
|
+
}
|
|
4284
|
+
)
|
|
4285
|
+
return sql, None
|
|
4286
|
+
audit_logger.log(
|
|
4287
|
+
{
|
|
4288
|
+
"sessionId": session_id,
|
|
4289
|
+
"question": question,
|
|
4290
|
+
"tool": "llm_sql_expert_review",
|
|
4291
|
+
"success": review.executable,
|
|
4292
|
+
"sqlHash": sql_hash(sql),
|
|
4293
|
+
"issues": review.issues,
|
|
4294
|
+
"repaired": bool(review.repaired_sql),
|
|
4295
|
+
}
|
|
4296
|
+
)
|
|
4297
|
+
if review.executable and not review.repaired_sql:
|
|
4298
|
+
steps.done(8, "LLM SQL 专家复核通过。")
|
|
4299
|
+
return sql, None
|
|
4300
|
+
validation_error = {
|
|
4301
|
+
"code": "LLM_SQL_EXPERT_REVIEW_FAILED",
|
|
4302
|
+
"message": _sql_expert_review_message(review.issues),
|
|
4303
|
+
"originalSql": sql,
|
|
4304
|
+
"issues": review.issues,
|
|
4305
|
+
"repairHint": "只修复 SQL 写法问题,不改变业务口径;聚合筛选放 HAVING 或先子查询聚合后外层过滤。",
|
|
4306
|
+
}
|
|
4307
|
+
candidate = review.repaired_sql
|
|
4308
|
+
if candidate:
|
|
4309
|
+
candidate = _finalize_candidate_sql(
|
|
4310
|
+
candidate,
|
|
4311
|
+
question,
|
|
4312
|
+
default_limit=default_limit,
|
|
4313
|
+
max_limit=max_limit,
|
|
4314
|
+
schema=schema,
|
|
4315
|
+
)
|
|
4316
|
+
if query_plan:
|
|
4317
|
+
candidate = apply_time_range_filters_from_plan(candidate, query_plan)
|
|
4318
|
+
candidate = apply_time_range_filters_from_question(candidate, time_range_question, schema)
|
|
4319
|
+
try:
|
|
4320
|
+
enforce_sql_rules(candidate, max_limit=max_limit)
|
|
4321
|
+
_require_no_schema_issues(candidate, schema)
|
|
4322
|
+
steps.done(9, "已根据 LLM SQL 专家复核修复 SQL 写法问题。")
|
|
4323
|
+
steps.done(8, "LLM SQL 专家复核后 SQL 满足本地规则。")
|
|
4324
|
+
return candidate, None
|
|
4325
|
+
except SqlGenerationError as exc:
|
|
4326
|
+
schema_issues = validate_sql_against_schema(candidate, schema)
|
|
4327
|
+
if schema_issues:
|
|
4328
|
+
validation_error = build_schema_validation_error(
|
|
4329
|
+
candidate,
|
|
4330
|
+
schema_issues,
|
|
4331
|
+
schema,
|
|
4332
|
+
max_attempts=max_repair_attempts,
|
|
4333
|
+
)
|
|
4334
|
+
else:
|
|
4335
|
+
validation_error = build_sql_rule_validation_error(
|
|
4336
|
+
candidate,
|
|
4337
|
+
exc,
|
|
4338
|
+
max_limit=max_limit,
|
|
4339
|
+
max_attempts=max_repair_attempts,
|
|
4340
|
+
)
|
|
4341
|
+
validation_error["message"] = f"LLM SQL 专家修复后仍有 SQL 写法问题:{exc}"
|
|
4342
|
+
sql = candidate
|
|
4343
|
+
repaired_sql, fail_reason = _repair_sql_with_llm_retries(
|
|
4344
|
+
generator=generator,
|
|
4345
|
+
question=question,
|
|
4346
|
+
sql=sql,
|
|
4347
|
+
validation_error=validation_error,
|
|
4348
|
+
schema_context=schema_context,
|
|
4349
|
+
audit_logger=audit_logger,
|
|
4350
|
+
session_id=session_id,
|
|
4351
|
+
max_attempts=max_repair_attempts,
|
|
4352
|
+
default_limit=default_limit,
|
|
4353
|
+
max_limit=max_limit,
|
|
4354
|
+
schema=schema,
|
|
4355
|
+
check=lambda candidate_sql: _require_sql_expert_local_pass(candidate_sql, schema, max_limit),
|
|
4356
|
+
steps=steps,
|
|
4357
|
+
running_detail=f"LLM SQL 专家发现写法问题,尝试修复:{validation_error['message']}",
|
|
4358
|
+
failure_detail="LLM SQL 专家发现写法问题,修复失败",
|
|
4359
|
+
)
|
|
4360
|
+
if repaired_sql is None:
|
|
4361
|
+
return sql, fail_reason
|
|
4362
|
+
steps.done(8, "LLM SQL 专家复核后 SQL 满足本地规则。")
|
|
4363
|
+
return repaired_sql, None
|
|
4364
|
+
|
|
4365
|
+
|
|
4366
|
+
def _require_sql_expert_local_pass(sql: str, schema: dict[str, Any], max_limit: int) -> None:
|
|
4367
|
+
enforce_sql_rules(sql, max_limit=max_limit)
|
|
4368
|
+
_require_no_schema_issues(sql, schema)
|
|
4369
|
+
|
|
4370
|
+
|
|
4371
|
+
def _should_llm_sql_expert_review(sql: str, query_plan: dict[str, Any] | None) -> bool:
|
|
4372
|
+
normalized = sql.lower()
|
|
4373
|
+
if re.search(r"\b(group\s+by|having|count|sum|avg|min|max|group_concat)\s*\b", normalized):
|
|
4374
|
+
return True
|
|
4375
|
+
if re.search(r"\(\s*select\b", normalized):
|
|
4376
|
+
return True
|
|
4377
|
+
if isinstance(query_plan, dict):
|
|
4378
|
+
for concept in query_plan.get("matchedConcepts") or []:
|
|
4379
|
+
if isinstance(concept, dict) and str(concept.get("name") or "") in {"filled_job_by_period"}:
|
|
4380
|
+
return True
|
|
4381
|
+
return False
|
|
4382
|
+
|
|
4383
|
+
|
|
4384
|
+
def _sql_expert_review_message(issues: list[dict[str, str]]) -> str:
|
|
4385
|
+
if not issues:
|
|
4386
|
+
return "LLM SQL 专家判断 SQL 可能存在写法问题,需修复后再执行。"
|
|
4387
|
+
parts: list[str] = []
|
|
4388
|
+
for issue in issues:
|
|
4389
|
+
message = str(issue.get("message") or issue.get("code") or "").strip()
|
|
4390
|
+
hint = str(issue.get("repairHint") or "").strip()
|
|
4391
|
+
if message and hint:
|
|
4392
|
+
parts.append(f"{message};建议:{hint}")
|
|
4393
|
+
elif message:
|
|
4394
|
+
parts.append(message)
|
|
4395
|
+
return ";".join(parts) or "LLM SQL 专家判断 SQL 可能存在写法问题,需修复后再执行。"
|
|
4396
|
+
|
|
4397
|
+
|
|
3935
4398
|
def _query_only_result(reason: str | None = None) -> QueryResult:
|
|
3936
4399
|
answer = reason.strip() if isinstance(reason, str) and reason.strip() else "查询未能完成,请稍后重试。"
|
|
3937
4400
|
return QueryResult(
|
|
@@ -3991,6 +4454,25 @@ def _low_quality_sql_result(message: str, generated_sql: str) -> QueryResult:
|
|
|
3991
4454
|
)
|
|
3992
4455
|
|
|
3993
4456
|
|
|
4457
|
+
def _query_quality_failure_message(
|
|
4458
|
+
message: str,
|
|
4459
|
+
*,
|
|
4460
|
+
missing_items: list[str] | tuple[str, ...] | None = None,
|
|
4461
|
+
current_problem: str = "",
|
|
4462
|
+
repair_hint: str = "",
|
|
4463
|
+
) -> str:
|
|
4464
|
+
parts = [f"当前 SQL 不满足查询口径:{message.strip()}"]
|
|
4465
|
+
if missing_items:
|
|
4466
|
+
missing_text = ";".join(str(item) for item in missing_items if str(item).strip())
|
|
4467
|
+
if missing_text:
|
|
4468
|
+
parts.append(f"缺失口径:{missing_text}")
|
|
4469
|
+
if current_problem:
|
|
4470
|
+
parts.append(f"当前 SQL 的问题:{current_problem}")
|
|
4471
|
+
if repair_hint:
|
|
4472
|
+
parts.append(f"建议修复方向:{repair_hint}")
|
|
4473
|
+
return ";".join(parts)
|
|
4474
|
+
|
|
4475
|
+
|
|
3994
4476
|
def _sensitive_query_only_result(terms: list[str], *, reason: str | None = None) -> QueryResult:
|
|
3995
4477
|
notice = _sensitive_field_notice(terms)
|
|
3996
4478
|
if isinstance(reason, str) and reason.strip():
|
|
@@ -5,7 +5,12 @@ import re
|
|
|
5
5
|
from dataclasses import dataclass
|
|
6
6
|
from typing import Any, Callable
|
|
7
7
|
|
|
8
|
-
from .prompt_builder import
|
|
8
|
+
from .prompt_builder import (
|
|
9
|
+
NL2SQL_SYSTEM_PROMPT,
|
|
10
|
+
build_nl2sql_prompt,
|
|
11
|
+
build_sql_expert_review_prompt,
|
|
12
|
+
build_sql_repair_prompt,
|
|
13
|
+
)
|
|
9
14
|
from .schema_context_retriever import SchemaContext
|
|
10
15
|
from .sql_generator import GeneratedSql, SqlGenerationError, contains_forbidden_dml_keyword
|
|
11
16
|
|
|
@@ -54,6 +59,30 @@ class SamplingSqlGenerator:
|
|
|
54
59
|
)
|
|
55
60
|
return _parse_generated_sql(self.sampler(NL2SQL_SYSTEM_PROMPT, prompt))
|
|
56
61
|
|
|
62
|
+
def review_sql(
|
|
63
|
+
self,
|
|
64
|
+
*,
|
|
65
|
+
question: str,
|
|
66
|
+
sql: str,
|
|
67
|
+
schema_context: SchemaContext,
|
|
68
|
+
) -> "SqlExpertReview":
|
|
69
|
+
prompt = build_sql_expert_review_prompt(
|
|
70
|
+
question=question,
|
|
71
|
+
sql=sql,
|
|
72
|
+
schema_context=schema_context,
|
|
73
|
+
default_limit=self.default_limit,
|
|
74
|
+
max_limit=self.max_limit,
|
|
75
|
+
confirmed_query_plan=self.confirmed_query_plan,
|
|
76
|
+
)
|
|
77
|
+
return _parse_sql_expert_review(self.sampler(NL2SQL_SYSTEM_PROMPT, prompt))
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
@dataclass(frozen=True)
|
|
81
|
+
class SqlExpertReview:
|
|
82
|
+
executable: bool
|
|
83
|
+
issues: list[dict[str, str]]
|
|
84
|
+
repaired_sql: str | None = None
|
|
85
|
+
|
|
57
86
|
|
|
58
87
|
def _parse_generated_sql(text: str) -> GeneratedSql:
|
|
59
88
|
try:
|
|
@@ -101,6 +130,31 @@ def _parse_generated_sql(text: str) -> GeneratedSql:
|
|
|
101
130
|
)
|
|
102
131
|
|
|
103
132
|
|
|
133
|
+
def _parse_sql_expert_review(text: str) -> SqlExpertReview:
|
|
134
|
+
payload = _load_json_object(text)
|
|
135
|
+
executable = bool(payload.get("executable", False))
|
|
136
|
+
issues_payload = payload.get("issues")
|
|
137
|
+
issues: list[dict[str, str]] = []
|
|
138
|
+
if isinstance(issues_payload, list):
|
|
139
|
+
for item in issues_payload:
|
|
140
|
+
if not isinstance(item, dict):
|
|
141
|
+
continue
|
|
142
|
+
issues.append(
|
|
143
|
+
{
|
|
144
|
+
"code": str(item.get("code") or ""),
|
|
145
|
+
"message": str(item.get("message") or ""),
|
|
146
|
+
"repairHint": str(item.get("repairHint") or ""),
|
|
147
|
+
}
|
|
148
|
+
)
|
|
149
|
+
repaired_sql = payload.get("repairedSql")
|
|
150
|
+
if isinstance(repaired_sql, str) and repaired_sql.strip():
|
|
151
|
+
repaired_sql = repaired_sql.strip()
|
|
152
|
+
_require_select_sql(repaired_sql)
|
|
153
|
+
else:
|
|
154
|
+
repaired_sql = None
|
|
155
|
+
return SqlExpertReview(executable=executable, issues=issues, repaired_sql=repaired_sql)
|
|
156
|
+
|
|
157
|
+
|
|
104
158
|
def _load_json_object(text: str) -> dict[str, Any]:
|
|
105
159
|
cleaned = text.strip()
|
|
106
160
|
if cleaned.startswith("```"):
|
|
@@ -469,7 +469,7 @@ def _query_result_payload(query_result: Any, *, include_sql: bool = False, base_
|
|
|
469
469
|
}
|
|
470
470
|
query_plan = getattr(query_result, "query_plan", None)
|
|
471
471
|
if isinstance(query_plan, dict):
|
|
472
|
-
payload["queryPlan"] = query_plan
|
|
472
|
+
payload["queryPlan"] = _public_query_plan(query_plan)
|
|
473
473
|
execution_steps = getattr(query_result, "execution_steps", None)
|
|
474
474
|
if isinstance(execution_steps, list):
|
|
475
475
|
payload["executionSteps"] = execution_steps
|
|
@@ -485,6 +485,14 @@ def _query_result_payload(query_result: Any, *, include_sql: bool = False, base_
|
|
|
485
485
|
return _jsonable(payload)
|
|
486
486
|
|
|
487
487
|
|
|
488
|
+
def _public_query_plan(query_plan: dict[str, Any]) -> Json:
|
|
489
|
+
return {
|
|
490
|
+
key: value
|
|
491
|
+
for key, value in query_plan.items()
|
|
492
|
+
if not str(key).startswith("_")
|
|
493
|
+
}
|
|
494
|
+
|
|
495
|
+
|
|
488
496
|
def _user_visible_answer(query_result: Any, *, base_question: str | None = None) -> str:
|
|
489
497
|
answer = str(getattr(query_result, "answer", ""))
|
|
490
498
|
question = base_question or ""
|
|
@@ -17,7 +17,8 @@ INSTRUCTION_STAGE_PREFIX: dict[InstructionStage, str] = {
|
|
|
17
17
|
}
|
|
18
18
|
|
|
19
19
|
|
|
20
|
-
NL2SQL_SYSTEM_PROMPT = """你是丸子Agent(Octopus Agent)的海绵数据 SQL
|
|
20
|
+
NL2SQL_SYSTEM_PROMPT = """你是丸子Agent(Octopus Agent)的海绵数据 SQL 专家,一名专业 MySQL SQL 专家。
|
|
21
|
+
你必须以 SQL 专家的标准生成 SQL:输出前逐句自检,确保 SQL 整体可以直接执行,没有任何 SQL 语法错误,也没有 SQL 写法问题(如括号/引号不闭合、SELECT 与 GROUP BY 不一致、聚合函数误用、JOIN 缺少等值键、别名冲突、字段/表拼写错误等)。
|
|
21
22
|
业务规则的唯一来源是 payload 中的 mandatoryAgentInstructions 与 schemaContext.policy;必须逐条遵守,不得使用训练记忆中的业务口径替代 schema 要求。
|
|
22
23
|
生成 SQL 前必须严格按 schemaContext.readOrder 顺序阅读 schemaContext 各段内容,再组合 SELECT、JOIN、WHERE、LIMIT。
|
|
23
24
|
只输出 JSON,结构见 outputSchema。返回 JSON 中的 sql 字段必须直接以 SELECT 开头;不要为了状态枚举单独返回 clarification。用户明确提出状态筛选时把状态条件写入 WHERE;用户未提出状态筛选或 schemaContext 缺少枚举时不要追问,首次查询保持宽泛,并把相关状态字段及中文含义加入 SELECT 结果。"""
|
|
@@ -76,6 +77,46 @@ def build_sql_repair_prompt(
|
|
|
76
77
|
)
|
|
77
78
|
|
|
78
79
|
|
|
80
|
+
def build_sql_expert_review_prompt(
|
|
81
|
+
*,
|
|
82
|
+
question: str,
|
|
83
|
+
sql: str,
|
|
84
|
+
schema_context: SchemaContext,
|
|
85
|
+
default_limit: int,
|
|
86
|
+
max_limit: int,
|
|
87
|
+
confirmed_query_plan: dict[str, Any] | None = None,
|
|
88
|
+
) -> str:
|
|
89
|
+
payload = _base_sql_prompt_payload(
|
|
90
|
+
question=question,
|
|
91
|
+
schema_context=schema_context,
|
|
92
|
+
default_limit=default_limit,
|
|
93
|
+
max_limit=max_limit,
|
|
94
|
+
confirmed_query_plan=confirmed_query_plan,
|
|
95
|
+
task="review_sql_executability",
|
|
96
|
+
original_sql=sql,
|
|
97
|
+
)
|
|
98
|
+
payload["reviewInstruction"] = (
|
|
99
|
+
"你是专业 MySQL SQL 专家,只检查 SQL 写法是否整体可执行,不改变业务口径。"
|
|
100
|
+
"重点检查:聚合函数是否误写在 WHERE/JOIN ON;GROUP BY/HAVING 是否合法;"
|
|
101
|
+
"子查询和派生表别名是否完整;外层引用的派生表字段是否存在;"
|
|
102
|
+
"SELECT 中非聚合字段与 GROUP BY 是否兼容;MySQL 低版本不支持 WITH。"
|
|
103
|
+
"如果 SQL 可执行,返回 executable=true 且 repairedSql 为空;"
|
|
104
|
+
"如果不可执行,返回 executable=false、issues,并在不改变业务口径和 schema 字段的前提下给出 repairedSql。"
|
|
105
|
+
)
|
|
106
|
+
payload["reviewOutputSchema"] = {
|
|
107
|
+
"executable": True,
|
|
108
|
+
"issues": [{"code": "ISSUE_CODE", "message": "中文问题说明", "repairHint": "中文修复建议"}],
|
|
109
|
+
"repairedSql": "SELECT ... LIMIT 50 或空字符串",
|
|
110
|
+
}
|
|
111
|
+
compact = confirmed_query_plan is not None
|
|
112
|
+
return json.dumps(
|
|
113
|
+
payload,
|
|
114
|
+
ensure_ascii=False,
|
|
115
|
+
indent=None if compact else 2,
|
|
116
|
+
separators=(",", ":") if compact else (", ", ": "),
|
|
117
|
+
)
|
|
118
|
+
|
|
119
|
+
|
|
79
120
|
def _base_sql_prompt_payload(
|
|
80
121
|
*,
|
|
81
122
|
question: str,
|
|
@@ -99,8 +140,13 @@ def _base_sql_prompt_payload(
|
|
|
99
140
|
"schemaContext 未提供对应口径时,不要凭训练记忆或字段名猜测 SQL。"
|
|
100
141
|
"涉及时间范围时,如果命中的 schemaContext.concepts 或 metrics 提供 timeRangeField,"
|
|
101
142
|
"必须使用该字段生成 WHERE 时间过滤,不要改用关联维表的同名时间字段。"
|
|
143
|
+
"JOIN ... ON 只能写 schemaContext.relations / joins / exampleJoinPatterns 中提供的等值关联键;"
|
|
144
|
+
"禁止凭字段名相似或训练记忆新增 schema 未定义 JOIN。"
|
|
102
145
|
"涉及“最多/最高/最大/排名/排行”等聚合极值诉求时,必须优先按聚合指标倒序排序,"
|
|
103
146
|
"再按时间等次要维度排序。"
|
|
147
|
+
"你必须作为专业 MySQL SQL 专家自检 SQL 整体可执行性:聚合函数不能写在 WHERE/JOIN ON,"
|
|
148
|
+
"聚合筛选必须放 HAVING 或先在子查询聚合后外层过滤;派生表字段和别名必须真实存在;"
|
|
149
|
+
"MySQL 低版本不支持 WITH。"
|
|
104
150
|
"不要把查询动作、状态变化、时间词或范围词当作名称筛选;"
|
|
105
151
|
"只有用户明确说“名称包含/岗位名称是/叫做/简称为”时,才允许对 name/title/job_name 写 LIKE。"
|
|
106
152
|
if not compact
|
|
@@ -112,8 +158,13 @@ def _base_sql_prompt_payload(
|
|
|
112
158
|
"schemaContext 未提供对应口径时,不要凭训练记忆或字段名猜测 SQL。"
|
|
113
159
|
"涉及时间范围时,如果命中的 schemaContext.concepts 或 metrics 提供 timeRangeField,"
|
|
114
160
|
"必须使用该字段生成 WHERE 时间过滤,不要改用关联维表的同名时间字段。"
|
|
161
|
+
"JOIN ... ON 只能写 schemaContext.relations / joins / exampleJoinPatterns 中提供的等值关联键;"
|
|
162
|
+
"禁止凭字段名相似或训练记忆新增 schema 未定义 JOIN。"
|
|
115
163
|
"涉及“最多/最高/最大/排名/排行”等聚合极值诉求时,必须优先按聚合指标倒序排序,"
|
|
116
164
|
"再按时间等次要维度排序。"
|
|
165
|
+
"你必须作为专业 MySQL SQL 专家自检 SQL 整体可执行性:聚合函数不能写在 WHERE/JOIN ON,"
|
|
166
|
+
"聚合筛选必须放 HAVING 或先在子查询聚合后外层过滤;派生表字段和别名必须真实存在;"
|
|
167
|
+
"MySQL 低版本不支持 WITH。"
|
|
117
168
|
"JOIN ... ON 只能写 schema relations / joins 中的等值关联键(如 a.id = b.x_id);"
|
|
118
169
|
"confirmedQueryPlan.filters 里的 LIKE/IN 筛选必须写在 WHERE,禁止写在 JOIN ON;"
|
|
119
170
|
"同一 logicalGroup 且 logicalOperator=OR 的筛选必须用 OR 连接,并用括号包裹;"
|
|
@@ -125,10 +176,13 @@ def _base_sql_prompt_payload(
|
|
|
125
176
|
"schemaContext": _schema_context_payload(schema_context, compact=compact),
|
|
126
177
|
"outputSchema": _sql_output_schema(default_limit, question),
|
|
127
178
|
}
|
|
128
|
-
if task
|
|
179
|
+
if task:
|
|
129
180
|
payload["task"] = task
|
|
181
|
+
if task == "repair_sql":
|
|
130
182
|
payload["originalSql"] = original_sql or ""
|
|
131
183
|
payload["validationError"] = validation_error or {}
|
|
184
|
+
elif task == "review_sql_executability":
|
|
185
|
+
payload["sqlToReview"] = original_sql or ""
|
|
132
186
|
if confirmed_query_plan:
|
|
133
187
|
payload["confirmedQueryPlan"] = _compact_query_plan_for_sql(confirmed_query_plan)
|
|
134
188
|
return payload
|
|
@@ -198,13 +252,16 @@ def _compact_query_plan_for_sql(query_plan: dict[str, Any]) -> dict[str, Any]:
|
|
|
198
252
|
]
|
|
199
253
|
if entities:
|
|
200
254
|
compact["entities"] = entities
|
|
255
|
+
plan_filters = query_plan.get("_sqlFilters")
|
|
256
|
+
if not isinstance(plan_filters, list):
|
|
257
|
+
plan_filters = query_plan.get("filters", [])
|
|
201
258
|
filters = [
|
|
202
259
|
{
|
|
203
260
|
key: item[key]
|
|
204
261
|
for key in ("field", "operator", "value", "description", "source", "logicalGroup", "logicalOperator")
|
|
205
262
|
if key in item
|
|
206
263
|
}
|
|
207
|
-
for item in
|
|
264
|
+
for item in plan_filters
|
|
208
265
|
if isinstance(item, dict)
|
|
209
266
|
]
|
|
210
267
|
if filters:
|
|
@@ -132,10 +132,15 @@ class QuestionAnalysisError(Exception):
|
|
|
132
132
|
"""Raised when question analysis fails."""
|
|
133
133
|
|
|
134
134
|
|
|
135
|
+
SAMPLING_MAX_RETRIES = 2
|
|
136
|
+
|
|
137
|
+
|
|
135
138
|
def analyze_question(
|
|
136
139
|
question: str,
|
|
137
140
|
sampler: Sampler,
|
|
138
141
|
agent_instructions: list[dict[str, Any]] | None = None,
|
|
142
|
+
*,
|
|
143
|
+
max_sampling_retries: int = SAMPLING_MAX_RETRIES,
|
|
139
144
|
) -> QuestionAnalysis:
|
|
140
145
|
"""Use LLM to analyze user question and extract structured intent + query targets.
|
|
141
146
|
|
|
@@ -152,12 +157,18 @@ def analyze_question(
|
|
|
152
157
|
"""
|
|
153
158
|
prompt = build_question_analysis_prompt(question, agent_instructions)
|
|
154
159
|
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
160
|
+
raw_response: str | None = None
|
|
161
|
+
for attempt in range(max_sampling_retries + 1):
|
|
162
|
+
try:
|
|
163
|
+
raw_response = sampler(QUESTION_ANALYSIS_SYSTEM_PROMPT, prompt)
|
|
164
|
+
break
|
|
165
|
+
except Exception as exc:
|
|
166
|
+
if attempt >= max_sampling_retries:
|
|
167
|
+
raise QuestionAnalysisError(
|
|
168
|
+
f"LLM 调用失败(已重试 {max_sampling_retries} 次):{type(exc).__name__}: {exc}"
|
|
169
|
+
) from exc
|
|
170
|
+
|
|
171
|
+
payload = _parse_analysis_response(raw_response or "")
|
|
161
172
|
return _build_analysis(payload, question)
|
|
162
173
|
|
|
163
174
|
|
|
@@ -49,6 +49,9 @@ class SchemaValidationIssue:
|
|
|
49
49
|
class LowQualitySqlIssue:
|
|
50
50
|
code: str
|
|
51
51
|
message: str
|
|
52
|
+
missing_items: tuple[str, ...] = ()
|
|
53
|
+
current_problem: str = ""
|
|
54
|
+
repair_hint: str = ""
|
|
52
55
|
|
|
53
56
|
|
|
54
57
|
_SELECT_AS_ALIAS_PATTERN = (
|
|
@@ -86,6 +89,7 @@ def enforce_sql_rules(sql: str, *, max_limit: int) -> None:
|
|
|
86
89
|
raise SqlGenerationError("SQL must not contain scope placeholders")
|
|
87
90
|
_enforce_text_filters_use_like(sql)
|
|
88
91
|
_enforce_join_on_uses_equality_keys(sql)
|
|
92
|
+
_enforce_no_aggregate_functions_in_row_filters(sql)
|
|
89
93
|
_enforce_chinese_select_aliases(sql)
|
|
90
94
|
_enforce_no_null_placeholder_select_items(sql)
|
|
91
95
|
_enforce_no_tautological_join_conditions(sql)
|
|
@@ -724,6 +728,41 @@ def _enforce_join_on_uses_equality_keys(sql: str) -> None:
|
|
|
724
728
|
raise SqlGenerationError(violations[0])
|
|
725
729
|
|
|
726
730
|
|
|
731
|
+
_AGGREGATE_FUNCTION_RE = re.compile(
|
|
732
|
+
r"\b(?:count|sum|avg|min|max|group_concat)\s*\(",
|
|
733
|
+
flags=re.IGNORECASE,
|
|
734
|
+
)
|
|
735
|
+
|
|
736
|
+
|
|
737
|
+
def _enforce_no_aggregate_functions_in_row_filters(sql: str) -> None:
|
|
738
|
+
for clause in _iter_join_on_clauses(sql):
|
|
739
|
+
if _AGGREGATE_FUNCTION_RE.search(clause):
|
|
740
|
+
raise SqlGenerationError(
|
|
741
|
+
"SQL aggregate functions must not be used in JOIN ON; "
|
|
742
|
+
"aggregate first in a subquery or move aggregate predicates to HAVING"
|
|
743
|
+
)
|
|
744
|
+
where_clause = _top_level_where_clause(sql)
|
|
745
|
+
if where_clause and _AGGREGATE_FUNCTION_RE.search(where_clause):
|
|
746
|
+
raise SqlGenerationError(
|
|
747
|
+
"SQL aggregate functions must not be used in WHERE; "
|
|
748
|
+
"move aggregate predicates to HAVING or aggregate first in a subquery"
|
|
749
|
+
)
|
|
750
|
+
|
|
751
|
+
|
|
752
|
+
def _top_level_where_clause(sql: str) -> str:
|
|
753
|
+
where_index = _find_top_level_keyword(sql, "where", start=0)
|
|
754
|
+
if where_index is None:
|
|
755
|
+
return ""
|
|
756
|
+
body_start = where_index + len("where")
|
|
757
|
+
suffix_starts = [
|
|
758
|
+
index
|
|
759
|
+
for keyword in ("group by", "having", "order by", "limit")
|
|
760
|
+
if (index := _find_top_level_keyword(sql, keyword, start=body_start)) is not None
|
|
761
|
+
]
|
|
762
|
+
body_end = min(suffix_starts) if suffix_starts else len(sql)
|
|
763
|
+
return sql[body_start:body_end]
|
|
764
|
+
|
|
765
|
+
|
|
727
766
|
def _select_item_has_chinese_alias(item: str) -> bool:
|
|
728
767
|
alias = _extract_select_alias(item)
|
|
729
768
|
return alias is not None and re.search(r"[\u4e00-\u9fff]", alias) is not None
|
|
@@ -877,7 +916,18 @@ def low_quality_sql_issue_for_question(question: str, sql: str, *, schema: dict[
|
|
|
877
916
|
table = str(table_name or "").strip().lower()
|
|
878
917
|
if table and table not in normalized:
|
|
879
918
|
message = str(checks.get("message") or f"SQL 必须引用 schema 要求的表:{table_name}")
|
|
880
|
-
|
|
919
|
+
concept_label = str(concept.get("comment") or concept.get("name") or "命中的 schema concept")
|
|
920
|
+
return LowQualitySqlIssue(
|
|
921
|
+
code="SCHEMA_SQL_QUALITY",
|
|
922
|
+
message=message,
|
|
923
|
+
missing_items=(f"必须引用表 {table_name}",),
|
|
924
|
+
current_problem=(
|
|
925
|
+
f"当前 SQL 没有使用 {table_name},无法覆盖“{concept_label}”要求的查询口径。"
|
|
926
|
+
),
|
|
927
|
+
repair_hint=(
|
|
928
|
+
f"按 schema.concepts 中“{concept_label}”的定义补充 {table_name} 及对应 JOIN/WHERE 条件。"
|
|
929
|
+
),
|
|
930
|
+
)
|
|
881
931
|
aggregate_issue = _aggregate_sql_quality_issue(question, sql)
|
|
882
932
|
if aggregate_issue is not None:
|
|
883
933
|
return aggregate_issue
|
|
@@ -901,6 +951,9 @@ def _concept_time_range_field_issue(
|
|
|
901
951
|
f"用户问题包含时间范围,且 schema concept 指定时间字段为 {qualified};"
|
|
902
952
|
"SQL 必须使用该字段过滤时间范围,不要改用关联维表的同名时间字段。"
|
|
903
953
|
),
|
|
954
|
+
missing_items=(f"时间范围字段 {qualified}",),
|
|
955
|
+
current_problem=f"当前 SQL 没有使用 {qualified} 过滤用户要求的时间范围。",
|
|
956
|
+
repair_hint=f"在 WHERE 中使用 {qualified} 补充时间范围过滤。",
|
|
904
957
|
)
|
|
905
958
|
unexpected = _unexpected_time_filter_fields(sql, expected_field=qualified)
|
|
906
959
|
if unexpected:
|
|
@@ -910,6 +963,9 @@ def _concept_time_range_field_issue(
|
|
|
910
963
|
f"用户问题包含时间范围,且 schema concept 指定时间字段为 {qualified};"
|
|
911
964
|
f"SQL 不应同时使用 {'、'.join(unexpected)} 过滤时间范围。"
|
|
912
965
|
),
|
|
966
|
+
missing_items=(f"唯一时间范围字段 {qualified}",),
|
|
967
|
+
current_problem=f"当前 SQL 还使用了 {'、'.join(unexpected)} 过滤时间范围,可能把口径切到关联表时间。",
|
|
968
|
+
repair_hint=f"删除关联表时间过滤,只保留 {qualified} 的时间范围条件。",
|
|
913
969
|
)
|
|
914
970
|
return None
|
|
915
971
|
|
|
@@ -972,9 +1028,23 @@ def _aggregate_sql_quality_issue(question: str, sql: str) -> LowQualitySqlIssue
|
|
|
972
1028
|
return LowQualitySqlIssue(
|
|
973
1029
|
code="AGGREGATE_SQL_REQUIRED",
|
|
974
1030
|
message="用户问题要求统计数量/最多值及对应维度,SQL 必须使用 COUNT/SUM/AVG/MIN/MAX 等聚合函数,不能只返回明细行。",
|
|
1031
|
+
missing_items=("聚合函数 COUNT/SUM/AVG/MIN/MAX",),
|
|
1032
|
+
current_problem="当前 SQL 返回明细行,不能回答数量、最多值、排名或按维度统计问题。",
|
|
1033
|
+
repair_hint="按用户问题中的统计维度 GROUP BY,并用聚合函数生成指标。",
|
|
975
1034
|
)
|
|
976
1035
|
|
|
977
1036
|
|
|
1037
|
+
def format_sql_quality_issue(issue: LowQualitySqlIssue) -> str:
|
|
1038
|
+
parts = [f"当前 SQL 不满足查询口径:{issue.message}"]
|
|
1039
|
+
if issue.missing_items:
|
|
1040
|
+
parts.append(f"缺失口径:{';'.join(issue.missing_items)}")
|
|
1041
|
+
if issue.current_problem:
|
|
1042
|
+
parts.append(f"当前 SQL 的问题:{issue.current_problem}")
|
|
1043
|
+
if issue.repair_hint:
|
|
1044
|
+
parts.append(f"建议修复方向:{issue.repair_hint}")
|
|
1045
|
+
return ";".join(parts)
|
|
1046
|
+
|
|
1047
|
+
|
|
978
1048
|
def _question_requires_aggregate_sql(question: str) -> bool:
|
|
979
1049
|
if not re.search(r"(计数|统计|数量|个数|总数|多少)", question):
|
|
980
1050
|
return False
|
|
@@ -1207,6 +1277,7 @@ def _schema_invalid_join_issues(
|
|
|
1207
1277
|
allowed_pairs = _schema_allowed_join_pairs(schema)
|
|
1208
1278
|
if not allowed_pairs:
|
|
1209
1279
|
return []
|
|
1280
|
+
derived_aliases = set(_sql_derived_subquery_aliases(sql))
|
|
1210
1281
|
issues: list[SchemaValidationIssue] = []
|
|
1211
1282
|
seen: set[tuple[tuple[str, str], tuple[str, str]]] = set()
|
|
1212
1283
|
for pair in _sql_equality_pairs(sql, table_aliases):
|
|
@@ -1216,6 +1287,8 @@ def _schema_invalid_join_issues(
|
|
|
1216
1287
|
if pair in allowed_pairs:
|
|
1217
1288
|
continue
|
|
1218
1289
|
left, right = pair
|
|
1290
|
+
if left[0] in derived_aliases or right[0] in derived_aliases:
|
|
1291
|
+
continue
|
|
1219
1292
|
issues.append(
|
|
1220
1293
|
SchemaValidationIssue(
|
|
1221
1294
|
code="INVALID_JOIN",
|
|
@@ -1323,19 +1396,37 @@ def _sql_derived_subquery_spans(sql: str) -> list[tuple[int, int]]:
|
|
|
1323
1396
|
]
|
|
1324
1397
|
|
|
1325
1398
|
|
|
1399
|
+
_DERIVED_SUBQUERY_TRAILING = (
|
|
1400
|
+
r"on\b"
|
|
1401
|
+
r"|(?:(?:left|right|inner|outer|cross|natural)\s+)?join\b"
|
|
1402
|
+
r"|where\b|group\b|order\b|having\b|limit\b"
|
|
1403
|
+
r"|,"
|
|
1404
|
+
r"|\)(?!\s*[a-zA-Z_])"
|
|
1405
|
+
r"|;"
|
|
1406
|
+
)
|
|
1407
|
+
_DERIVED_SUBQUERY_ALIAS_PATTERN = re.compile(
|
|
1408
|
+
rf"\)\s*(?:as\s+)?([a-zA-Z_][a-zA-Z0-9_]*)(?:\s+(?:{_DERIVED_SUBQUERY_TRAILING})|\s*$)",
|
|
1409
|
+
flags=re.IGNORECASE,
|
|
1410
|
+
)
|
|
1411
|
+
_RESERVED_DERIVED_ALIASES = frozenset(
|
|
1412
|
+
{"on", "where", "left", "right", "inner", "outer", "join", "select", "cross", "natural", "group", "order", "having", "limit"}
|
|
1413
|
+
)
|
|
1414
|
+
|
|
1415
|
+
|
|
1326
1416
|
def _sql_derived_subquery_matches(sql: str) -> list[tuple[str, int, int]]:
|
|
1327
1417
|
matches: list[tuple[str, int, int]] = []
|
|
1328
|
-
|
|
1329
|
-
|
|
1330
|
-
sql,
|
|
1331
|
-
flags=re.IGNORECASE,
|
|
1332
|
-
):
|
|
1418
|
+
seen_ranges: set[tuple[int, int]] = set()
|
|
1419
|
+
for match in _DERIVED_SUBQUERY_ALIAS_PATTERN.finditer(sql):
|
|
1333
1420
|
alias = match.group(1).lower()
|
|
1334
|
-
if alias in
|
|
1421
|
+
if alias in _RESERVED_DERIVED_ALIASES:
|
|
1335
1422
|
continue
|
|
1336
1423
|
subquery_start = _find_matching_open_paren(sql, match.start())
|
|
1337
1424
|
if subquery_start is None:
|
|
1338
1425
|
continue
|
|
1426
|
+
range_key = (subquery_start, match.start())
|
|
1427
|
+
if range_key in seen_ranges:
|
|
1428
|
+
continue
|
|
1429
|
+
seen_ranges.add(range_key)
|
|
1339
1430
|
matches.append((alias, subquery_start, match.start()))
|
|
1340
1431
|
return matches
|
|
1341
1432
|
|
|
@@ -1662,7 +1753,7 @@ def time_range_from_query_plan(query_plan: dict[str, Any] | None) -> TimeRange |
|
|
|
1662
1753
|
start_value = ""
|
|
1663
1754
|
end_value = ""
|
|
1664
1755
|
field_name = ""
|
|
1665
|
-
for flt in query_plan
|
|
1756
|
+
for flt in _query_plan_sql_filters(query_plan):
|
|
1666
1757
|
if not isinstance(flt, dict):
|
|
1667
1758
|
continue
|
|
1668
1759
|
if str(flt.get("source") or "") != "question.timeRange":
|
|
@@ -1853,7 +1944,34 @@ def resolve_time_range_for_sql(
|
|
|
1853
1944
|
|
|
1854
1945
|
concepts = match_concepts_for_question(_schema_concepts(schema), question)
|
|
1855
1946
|
sql_schema = _schema_limited_to_sql_tables(schema, sql)
|
|
1856
|
-
|
|
1947
|
+
if sql_schema is None and _sql_referenced_table_names(sql):
|
|
1948
|
+
return None
|
|
1949
|
+
if sql_schema is not None:
|
|
1950
|
+
return resolve_time_range_for_question(
|
|
1951
|
+
question,
|
|
1952
|
+
sql_schema,
|
|
1953
|
+
concepts=_concepts_limited_to_schema_tables(concepts, sql_schema),
|
|
1954
|
+
)
|
|
1955
|
+
return resolve_time_range_for_question(question, schema, concepts=concepts)
|
|
1956
|
+
|
|
1957
|
+
|
|
1958
|
+
def _concepts_limited_to_schema_tables(
|
|
1959
|
+
concepts: list[dict[str, Any]],
|
|
1960
|
+
schema: dict[str, Any],
|
|
1961
|
+
) -> list[dict[str, Any]]:
|
|
1962
|
+
table_names = {name.lower() for name in _table_names(schema)}
|
|
1963
|
+
if not table_names:
|
|
1964
|
+
return []
|
|
1965
|
+
limited: list[dict[str, Any]] = []
|
|
1966
|
+
for concept in concepts:
|
|
1967
|
+
qualified = str(concept.get("timeRangeField") or "").strip()
|
|
1968
|
+
if "." not in qualified:
|
|
1969
|
+
limited.append(concept)
|
|
1970
|
+
continue
|
|
1971
|
+
table_name = qualified.split(".", 1)[0].lower()
|
|
1972
|
+
if table_name in table_names:
|
|
1973
|
+
limited.append(concept)
|
|
1974
|
+
return limited
|
|
1857
1975
|
|
|
1858
1976
|
|
|
1859
1977
|
def _qualified_time_range_field(time_range: TimeRange) -> str:
|
|
@@ -2004,7 +2122,7 @@ def sql_missing_query_plan_filters(sql: str, query_plan: dict[str, Any] | None)
|
|
|
2004
2122
|
missing: list[str] = []
|
|
2005
2123
|
normalized_sql = sql.lower()
|
|
2006
2124
|
grouped_filters: dict[str, list[dict[str, Any]]] = {}
|
|
2007
|
-
for flt in query_plan
|
|
2125
|
+
for flt in _query_plan_sql_filters(query_plan):
|
|
2008
2126
|
if not isinstance(flt, dict):
|
|
2009
2127
|
continue
|
|
2010
2128
|
if str(flt.get("operator") or "").upper() != "LIKE":
|
|
@@ -2045,6 +2163,14 @@ def sql_satisfies_query_plan_filters(sql: str, query_plan: dict[str, Any] | None
|
|
|
2045
2163
|
return not sql_missing_query_plan_filters(sql, query_plan)
|
|
2046
2164
|
|
|
2047
2165
|
|
|
2166
|
+
def _query_plan_sql_filters(query_plan: dict[str, Any]) -> list[Any]:
|
|
2167
|
+
filters = query_plan.get("_sqlFilters")
|
|
2168
|
+
if isinstance(filters, list):
|
|
2169
|
+
return filters
|
|
2170
|
+
filters = query_plan.get("filters")
|
|
2171
|
+
return filters if isinstance(filters, list) else []
|
|
2172
|
+
|
|
2173
|
+
|
|
2048
2174
|
def _or_group_filters_connected_by_or(normalized_sql: str, filters: list[dict[str, Any]]) -> bool:
|
|
2049
2175
|
if not re.search(r"\bor\b", normalized_sql):
|
|
2050
2176
|
return False
|