@roll-agent/octopus-agent 0.1.3 → 0.2.0

package/src/octopus_skill/prompt_builder.py

@@ -1,522 +0,0 @@
-from __future__ import annotations
-
-import json
-import re
-from typing import Any, Literal
-
-from .schema_context_retriever import SchemaContext
-from .sql_generator import normalize_text_filters_to_like
-
-
-InstructionStage = Literal["before", "during", "after"]
-
-INSTRUCTION_STAGE_PREFIX: dict[InstructionStage, str] = {
-    "before": "【生成前】",
-    "during": "【生成中】",
-    "after": "【执行后】",
-}
-
-
-NL2SQL_SYSTEM_PROMPT = """你是丸子Agent（Octopus Agent）的海绵数据 SQL 专家，一名专业 MySQL SQL 专家。
-你必须以 SQL 专家的标准生成 SQL：输出前逐句自检，确保 SQL 整体可以直接执行，没有任何 SQL 语法错误，也没有 SQL 写法问题（如括号/引号不闭合、SELECT 与 GROUP BY 不一致、聚合函数误用、JOIN 缺少等值键、别名冲突、字段/表拼写错误等）。
-业务规则的唯一来源是 payload 中的 mandatoryAgentInstructions 与 schemaContext.policy；必须逐条遵守，不得使用训练记忆中的业务口径替代 schema 要求。
-生成 SQL 前必须严格按 schemaContext.readOrder 顺序阅读 schemaContext 各段内容，再组合 SELECT、JOIN、WHERE、LIMIT。
-只输出 JSON，结构见 outputSchema。返回 JSON 中的 sql 字段必须直接以 SELECT 开头；不要为了状态枚举单独返回 clarification。用户明确提出状态筛选时把状态条件写入 WHERE；用户未提出状态筛选或 schemaContext 缺少枚举时不要追问，首次查询保持宽泛，并把相关状态字段及中文含义加入 SELECT 结果。"""
-
-
-def build_nl2sql_prompt(
-    *,
-    question: str,
-    schema_context: SchemaContext,
-    default_limit: int,
-    max_limit: int,
-    confirmed_query_plan: dict[str, Any] | None = None,
-) -> str:
-    payload = _base_sql_prompt_payload(
-        question=question,
-        schema_context=schema_context,
-        default_limit=default_limit,
-        max_limit=max_limit,
-        confirmed_query_plan=confirmed_query_plan,
-    )
-    compact = confirmed_query_plan is not None
-    return json.dumps(
-        payload,
-        ensure_ascii=False,
-        indent=None if compact else 2,
-        separators=(",", ":") if compact else (", ", ": "),
-    )
-
-
-def build_sql_repair_prompt(
-    *,
-    question: str,
-    original_sql: str,
-    validation_error: dict[str, Any],
-    schema_context: SchemaContext,
-    default_limit: int,
-    max_limit: int,
-    confirmed_query_plan: dict[str, Any] | None = None,
-) -> str:
-    payload = _base_sql_prompt_payload(
-        question=question,
-        schema_context=schema_context,
-        default_limit=default_limit,
-        max_limit=max_limit,
-        confirmed_query_plan=confirmed_query_plan,
-        task="repair_sql",
-        original_sql=original_sql,
-        validation_error=validation_error,
-    )
-    compact = confirmed_query_plan is not None
-    return json.dumps(
-        payload,
-        ensure_ascii=False,
-        indent=None if compact else 2,
-        separators=(",", ":") if compact else (", ", ": "),
-    )
-
-
-def build_sql_expert_review_prompt(
-    *,
-    question: str,
-    sql: str,
-    schema_context: SchemaContext,
-    default_limit: int,
-    max_limit: int,
-    confirmed_query_plan: dict[str, Any] | None = None,
-) -> str:
-    payload = _base_sql_prompt_payload(
-        question=question,
-        schema_context=schema_context,
-        default_limit=default_limit,
-        max_limit=max_limit,
-        confirmed_query_plan=confirmed_query_plan,
-        task="review_sql_executability",
-        original_sql=sql,
-    )
-    payload["reviewInstruction"] = (
-        "你是专业 MySQL SQL 专家，只检查 SQL 写法是否整体可执行，不改变业务口径。"
-        "重点检查：聚合函数是否误写在 WHERE/JOIN ON；GROUP BY/HAVING 是否合法；"
-        "子查询和派生表别名是否完整；外层引用的派生表字段是否存在；"
-        "SELECT 中非聚合字段与 GROUP BY 是否兼容；MySQL 低版本不支持 WITH。"
-        "如果 SQL 可执行，返回 executable=true 且 repairedSql 为空；"
-        "如果不可执行，返回 executable=false、issues，并在不改变业务口径和 schema 字段的前提下给出 repairedSql。"
-    )
-    payload["reviewOutputSchema"] = {
-        "executable": True,
-        "issues": [{"code": "ISSUE_CODE", "message": "中文问题说明", "repairHint": "中文修复建议"}],
-        "repairedSql": "SELECT ... LIMIT 50 或空字符串",
-    }
-    compact = confirmed_query_plan is not None
-    return json.dumps(
-        payload,
-        ensure_ascii=False,
-        indent=None if compact else 2,
-        separators=(",", ":") if compact else (", ", ": "),
-    )
-
-
-def _base_sql_prompt_payload(
-    *,
-    question: str,
-    schema_context: SchemaContext,
-    default_limit: int,
-    max_limit: int,
-    confirmed_query_plan: dict[str, Any] | None = None,
-    task: str | None = None,
-    original_sql: str | None = None,
-    validation_error: dict[str, Any] | None = None,
-) -> dict[str, Any]:
-    compact = confirmed_query_plan is not None
-    instruction_lines = _agent_instruction_lines(schema_context)
-    payload: dict[str, Any] = {
-        "mandatoryAgentInstructions": instruction_lines,
-        "instruction": (
-            "生成或修复 SQL 前必须先完整阅读 mandatoryAgentInstructions，"
-            "再按 schemaContext.readOrder 阅读 schemaContext；"
-            "WHERE/SELECT/JOIN/LIMIT/字段别名/澄清策略均以 mandatoryAgentInstructions 与 schemaContext 为准。"
-            "涉及业务口径的查询必须优先使用 schemaContext.concepts、metrics、examples 或 exampleJoinPatterns；"
-            "schemaContext 未提供对应口径时，不要凭训练记忆或字段名猜测 SQL。"
-            "涉及时间范围时，如果命中的 schemaContext.concepts 或 metrics 提供 timeRangeField，"
-            "必须使用该字段生成 WHERE 时间过滤，不要改用关联维表的同名时间字段。"
-            "JOIN ... ON 只能写 schemaContext.relations / joins / exampleJoinPatterns 中提供的等值关联键；"
-            "禁止凭字段名相似或训练记忆新增 schema 未定义 JOIN。"
-            "涉及“最多/最高/最大/排名/排行”等聚合极值诉求时，必须优先按聚合指标倒序排序，"
-            "再按时间等次要维度排序。"
-            "你必须作为专业 MySQL SQL 专家自检 SQL 整体可执行性：聚合函数不能写在 WHERE/JOIN ON，"
-            "聚合筛选必须放 HAVING 或先在子查询聚合后外层过滤；派生表字段和别名必须真实存在；"
-            "MySQL 低版本不支持 WITH。"
-            "不要把查询动作、状态变化、时间词或范围词当作名称筛选；"
-            "只有用户明确说“名称包含/岗位名称是/叫做/简称为”时，才允许对 name/title/job_name 写 LIKE。"
-            if not compact
-            else
-            "生成或修复 SQL 前必须先完整阅读 mandatoryAgentInstructions，"
-            "再按 schemaContext.readOrder 阅读 schemaContext；"
-            "confirmedQueryPlan 提供实体、筛选与输出字段，schemaContext 提供表结构、JOIN 与 policy。"
-            "涉及业务口径的查询必须优先使用 schemaContext.concepts、metrics、examples 或 exampleJoinPatterns；"
-            "schemaContext 未提供对应口径时，不要凭训练记忆或字段名猜测 SQL。"
-            "涉及时间范围时，如果命中的 schemaContext.concepts 或 metrics 提供 timeRangeField，"
-            "必须使用该字段生成 WHERE 时间过滤，不要改用关联维表的同名时间字段。"
-            "JOIN ... ON 只能写 schemaContext.relations / joins / exampleJoinPatterns 中提供的等值关联键；"
-            "禁止凭字段名相似或训练记忆新增 schema 未定义 JOIN。"
-            "涉及“最多/最高/最大/排名/排行”等聚合极值诉求时，必须优先按聚合指标倒序排序，"
-            "再按时间等次要维度排序。"
-            "你必须作为专业 MySQL SQL 专家自检 SQL 整体可执行性：聚合函数不能写在 WHERE/JOIN ON，"
-            "聚合筛选必须放 HAVING 或先在子查询聚合后外层过滤；派生表字段和别名必须真实存在；"
-            "MySQL 低版本不支持 WITH。"
-            "JOIN ... ON 只能写 schema relations / joins 中的等值关联键（如 a.id = b.x_id）；"
-            "confirmedQueryPlan.filters 里的 LIKE/IN 筛选必须写在 WHERE，禁止写在 JOIN ON；"
-            "同一 logicalGroup 且 logicalOperator=OR 的筛选必须用 OR 连接，并用括号包裹；"
-            "没有 logicalGroup 的筛选默认用 AND 连接；"
-            "不同地点维度（city.name 与 region.name）若没有同一 logicalGroup，必须用 AND 连接，"
-            "例如 c.name LIKE '%苏州%' AND r.name LIKE '%昆山%'；禁止写成 (c.name LIKE '%苏州%' OR r.name LIKE '%昆山%')；"
-            "confirmedQueryPlan.fields 中 required=true 的字段必须出现在 SELECT 中；"
-            "不要把查询动作、状态变化、时间词或范围词当作名称筛选。"
-        ),
-        "question": question,
-        "limits": {"defaultLimit": default_limit, "maxLimit": max_limit},
-        "schemaContext": _schema_context_payload(schema_context, compact=compact),
-        "outputSchema": _sql_output_schema(default_limit, question),
-    }
-    if task:
-        payload["task"] = task
-    if task == "repair_sql":
-        payload["originalSql"] = original_sql or ""
-        payload["validationError"] = validation_error or {}
-    elif task == "review_sql_executability":
-        payload["sqlToReview"] = original_sql or ""
-    if confirmed_query_plan:
-        payload["confirmedQueryPlan"] = _compact_query_plan_for_sql(confirmed_query_plan)
-    return payload
-
-
-def instruction_text(item: dict[str, Any]) -> str:
-    return str(item.get("text") or item.get("instruction") or item.get("content") or "").strip()
-
-
-def filter_agent_instructions_by_stage(
-    instructions: list[dict[str, Any]],
-    stage: InstructionStage,
-) -> list[str]:
-    prefix = INSTRUCTION_STAGE_PREFIX[stage]
-    texts: list[str] = []
-    for item in instructions:
-        if not isinstance(item, dict):
-            continue
-        text = instruction_text(item)
-        if text.startswith(prefix):
-            texts.append(text)
-    return texts
-
-
-def format_numbered_instruction_lines(texts: list[str]) -> list[str]:
-    return [f"{index}. {text}" for index, text in enumerate(texts, start=1)]
-
-
-def _agent_instruction_lines(
-    schema_context: SchemaContext,
-    *,
-    stage: InstructionStage = "during",
-) -> list[str]:
-    texts = filter_agent_instructions_by_stage(schema_context.agent_instructions, stage)
-    return format_numbered_instruction_lines(texts)
-
-
-def _filtered_agent_instruction_items(
-    instructions: list[dict[str, Any]],
-    *,
-    stage: InstructionStage,
-) -> list[dict[str, Any]]:
-    prefix = INSTRUCTION_STAGE_PREFIX[stage]
-    items: list[dict[str, Any]] = []
-    for item in instructions:
-        if not isinstance(item, dict):
-            continue
-        text = instruction_text(item)
-        if text.startswith(prefix):
-            items.append({"text": text})
-    return items
-
-
-def _compact_query_plan_for_sql(query_plan: dict[str, Any]) -> dict[str, Any]:
-    compact: dict[str, Any] = {}
-    question = query_plan.get("question")
-    if isinstance(question, str) and question.strip():
-        compact["question"] = question.strip()
-    entities = [
-        {
-            key: entity[key]
-            for key in ("label", "type", "value", "source", "logicalGroup", "logicalOperator")
-            if key in entity
-        }
-        for entity in query_plan.get("entities", [])
-        if isinstance(entity, dict)
-    ]
-    if entities:
-        compact["entities"] = entities
-    plan_filters = query_plan.get("_sqlFilters")
-    if not isinstance(plan_filters, list):
-        plan_filters = query_plan.get("filters", [])
-    filters = [
-        {
-            key: item[key]
-            for key in ("field", "operator", "value", "description", "source", "logicalGroup", "logicalOperator")
-            if key in item
-        }
-        for item in plan_filters
-        if isinstance(item, dict)
-    ]
-    if filters:
-        compact["filters"] = filters
-    fields = [
-        {key: item[key] for key in ("table", "field", "description", "source", "required") if key in item}
-        for item in query_plan.get("fields", [])
-        if isinstance(item, dict)
-    ]
-    if fields:
-        compact["fields"] = fields
-    tables = [
-        {key: item[key] for key in ("name", "description") if key in item}
-        for item in query_plan.get("tables", [])
-        if isinstance(item, dict) and item.get("name")
-    ]
-    if tables:
-        compact["tables"] = tables
-    return compact
-
-
-def _schema_context_payload(schema_context: SchemaContext, *, compact: bool = False) -> dict[str, Any]:
-    selected_tables = {
-        str(table.get("tableName") or table.get("name") or "")
-        for table in schema_context.tables
-        if isinstance(table, dict)
-    }
-    selected_tables.discard("")
-    relations = schema_context.relations
-    policy = schema_context.policy
-    if compact:
-        relations = _slim_relations(relations, selected_tables)
-        policy = _slim_policy(policy, selected_tables)
-    payload: dict[str, Any] = {
-        "readOrder": _schema_read_order(compact=compact),
-        "policy": policy,
-        "schema": {"tables": schema_context.tables},
-        "joins": schema_context.manual_joins,
-        "relations": relations,
-        "defaultFilters": schema_context.default_filters,
-        "enums": schema_context.enums,
-        "businessTerms": schema_context.business_terms,
-        "concepts": schema_context.concepts,
-        "metrics": schema_context.metrics,
-        "exampleJoinPatterns": schema_context.example_join_patterns,
-        "exampleGuidance": schema_context.example_guidance,
-        "structureHints": schema_context.structure_hints,
-        "examples": _business_examples(schema_context.examples),
-    }
-    if not compact:
-        payload["agentInstructions"] = _filtered_agent_instruction_items(
-            schema_context.agent_instructions,
-            stage="during",
-        )
-        payload["glossary"] = schema_context.glossary_terms
-        payload["tables"] = schema_context.tables
-    if compact:
-        if not payload["businessTerms"]:
-            payload.pop("businessTerms", None)
-        if not payload["metrics"]:
-            payload.pop("metrics", None)
-        if not payload["exampleJoinPatterns"]:
-            payload.pop("exampleJoinPatterns", None)
-        if not payload.get("exampleGuidance"):
-            payload.pop("exampleGuidance", None)
-        if not payload.get("structureHints"):
-            payload.pop("structureHints", None)
-        if not payload["examples"]:
-            payload.pop("examples", None)
-        if not payload["enums"]:
-            payload.pop("enums", None)
-        if not payload["defaultFilters"]:
-            payload.pop("defaultFilters", None)
-        if not payload["joins"]:
-            payload.pop("joins", None)
-    else:
-        if not payload.get("glossary"):
-            payload.pop("glossary", None)
-        if not payload["businessTerms"]:
-            payload.pop("businessTerms", None)
-        if not payload["metrics"]:
-            payload.pop("metrics", None)
-        if not payload["exampleJoinPatterns"]:
-            payload.pop("exampleJoinPatterns", None)
-        if not payload.get("exampleGuidance"):
-            payload.pop("exampleGuidance", None)
-        if not payload.get("structureHints"):
-            payload.pop("structureHints", None)
-        if not payload["examples"]:
-            payload.pop("examples", None)
-        if not payload["enums"]:
-            payload.pop("enums", None)
-        if not payload["defaultFilters"]:
-            payload.pop("defaultFilters", None)
-    return payload
-
-
-def _slim_relations(
-    relations: list[dict[str, Any]],
-    selected_tables: set[str],
-) -> list[dict[str, Any]]:
-    slim: list[dict[str, Any]] = []
-    for relation in relations:
-        if not isinstance(relation, dict):
-            continue
-        left_table, right_table = _relation_endpoint_tables(relation)
-        if left_table not in selected_tables or right_table not in selected_tables:
-            continue
-        from_ref, to_ref = _relation_from_to_refs(relation)
-        if not from_ref or not to_ref:
-            continue
-        item: dict[str, Any] = {"from": from_ref, "to": to_ref}
-        join_type = relation.get("joinType")
-        if isinstance(join_type, str) and join_type.strip():
-            item["joinType"] = join_type.strip()
-        required_filters = relation.get("requiredFilters")
-        if isinstance(required_filters, list) and required_filters:
-            item["requiredFilters"] = [str(value) for value in required_filters if str(value).strip()]
-        slim.append(item)
-    return slim
-
-
-def _slim_policy(
-    policy_items: list[dict[str, Any]],
-    selected_tables: set[str],
-) -> list[dict[str, Any]]:
-    if not policy_items:
-        return []
-    merged: dict[str, Any] = {}
-    for item in policy_items:
-        if not isinstance(item, dict):
-            continue
-        for key, value in item.items():
-            if key in {"requiredScopes", "scopePlaceholders", "source"}:
-                continue
-            if key == "fieldBlacklist" and isinstance(value, list):
-                filtered = [
-                    {
-                        "table": entry.get("table"),
-                        "columns": entry.get("columns"),
-                    }
-                    for entry in value
-                    if isinstance(entry, dict)
-                    and str(entry.get("table") or "") in selected_tables
-                    and isinstance(entry.get("columns"), list)
-                ]
-                if filtered:
-                    merged["fieldBlacklist"] = filtered
-            elif key == "sqlRules" and value:
-                merged["sqlRules"] = value
-            elif key not in merged and value:
-                merged[key] = value
-    return [merged] if merged else []
-
-
-def _relation_endpoint_tables(relation: dict[str, Any]) -> tuple[str, str]:
-    left_table = str(relation.get("leftTable") or "")
-    right_table = str(relation.get("rightTable") or "")
-    if left_table and right_table:
-        return left_table, right_table
-    from_ref = str(relation.get("from") or "")
-    to_ref = str(relation.get("to") or "")
-    left = from_ref.split(".", 1)[0] if "." in from_ref else from_ref
-    right = to_ref.split(".", 1)[0] if "." in to_ref else to_ref
-    return left, right
-
-
-def _relation_from_to_refs(relation: dict[str, Any]) -> tuple[str | None, str | None]:
-    left_table = str(relation.get("leftTable") or "")
-    right_table = str(relation.get("rightTable") or "")
-    if left_table and right_table:
-        left_column = str(relation.get("leftColumn") or relation.get("fromColumn") or "")
-        right_column = str(relation.get("rightColumn") or relation.get("toColumn") or "")
-        from_ref = f"{left_table}.{left_column}".rstrip(".") if left_column else left_table
-        to_ref = f"{right_table}.{right_column}".rstrip(".") if right_column else right_table
-        return from_ref, to_ref
-    from_ref = str(relation.get("from") or "").strip()
-    to_ref = str(relation.get("to") or "").strip()
-    return (from_ref or None), (to_ref or None)
-
-
-def _sql_output_schema(default_limit: int, question: str) -> dict[str, Any]:
-    return {
-        "sql": f"SELECT column AS 中文含义 ... LIMIT {default_limit}",
-        "tables": ["table_name"],
-        "placeholders": [],
-        "usedColumns": ["table.column"],
-        "usedJoins": ["left_table.left_column = right_table.right_column"],
-        "metricRefs": ["schemaContext.metrics.name"],
-        "clarification": None,
-        "statusEnumPolicy": {
-            "explicitStatusFilter": "用户明确提出状态筛选时，把状态枚举条件写入 WHERE，并在 SELECT 中用 CASE 或枚举关联展示中文含义。",
-            "missingOrUnspecifiedStatus": "用户未提出状态筛选或 schemaContext 缺少枚举时，不要追问；首次查询保持宽泛，并把相关状态字段及中文含义加入 SELECT。",
-        },
-    }
-
-
-def _schema_read_order(*, compact: bool = False) -> list[str]:
-    if compact:
-        return ["policy", "schema", "concepts", "relations", "enums", "policy"]
-    return [
-        "agentInstructions",
-        "policy",
-        "glossary",
-        "schema",
-        "concepts/metrics/businessTerms",
-        "defaultFilters",
-        "joins",
-        "relations",
-        "exampleJoinPatterns",
-        "enums",
-        "policy",
-    ]
-
-
-def _business_examples(examples: list[dict[str, Any]]) -> list[dict[str, Any]]:
-    """Prepare examples for the LLM prompt.
-
-    The MCP examples carry data-scope predicates (``... IN (:scope.project_ids)``)
-    that the LLM must never emit. Rather than dropping every example that has
-    them (which removed almost all join guidance), strip the scope predicates
-    and keep the rest of the pattern. If an example still contains ``:scope.``
-    after stripping, drop it to stay safe.
-    """
-    result: list[dict[str, Any]] = []
-    for example in examples:
-        sql = str(example.get("sql") or "")
-        original_sql = sql
-        if ":scope." in sql:
-            sql = _strip_scope_predicates(sql)
-            if ":scope." in sql:
-                continue
-        sql = _normalize_example_text_filters(sql)
-        if sql != original_sql:
-            example = {**example, "sql": sql}
-        result.append(example)
-    return result
-
-
-def _strip_scope_predicates(sql: str) -> str:
-    # Remove "AND <column> IN (:scope.<name>)" predicates, then tidy a WHERE
-    # clause that may be left starting with a stray AND.
-    cleaned = re.sub(
-        r"\s+AND\s+[A-Za-z_][\w.]*\s+IN\s*\(\s*:scope\.\w+\s*\)",
-        "",
-        sql,
-        flags=re.IGNORECASE,
-    )
-    cleaned = re.sub(r"\bWHERE\s+AND\s+", "WHERE ", cleaned, flags=re.IGNORECASE)
-    return cleaned
-
-
-def _normalize_example_text_filters(sql: str) -> str:
-    """Align business examples with SQL guardrails before prompting."""
-    return normalize_text_filters_to_like(sql)