@roll-agent/octopus-agent 0.0.10 → 0.1.0

package/src/octopus_skill/schema_context_retriever.py

@@ -1,61 +1,451 @@
 from __future__ import annotations
 
 import re
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from typing import Any
 
 
 @dataclass(frozen=True)
 class SchemaContext:
     tables: list[dict[str, Any]]
-    joins: list[dict[str, Any]]
-    glossary: list[dict[str, Any]]
+    relations: list[dict[str, Any]]
+    business_terms: list[dict[str, Any]]
+    concepts: list[dict[str, Any]]
     metrics: list[dict[str, Any]]
     examples: list[dict[str, Any]]
+    default_filters: list[dict[str, Any]] = field(default_factory=list)
+    enums: list[dict[str, Any]] = field(default_factory=list)
+    agent_instructions: list[dict[str, Any]] = field(default_factory=list)
+    policy: list[dict[str, Any]] = field(default_factory=list)
+    glossary_terms: list[dict[str, Any]] = field(default_factory=list)
+    manual_joins: list[dict[str, Any]] = field(default_factory=list)
+    inferred_relations: list[dict[str, Any]] = field(default_factory=list)
+    example_join_patterns: list[dict[str, Any]] = field(default_factory=list)
+    example_guidance: list[dict[str, Any]] = field(default_factory=list)
+    structure_hints: list[dict[str, Any]] = field(default_factory=list)
 
+    @property
+    def joins(self) -> list[dict[str, Any]]:
+        return self.relations
+
+    @property
+    def glossary(self) -> list[dict[str, Any]]:
+        return self.glossary_terms + self.business_terms + self.concepts
+
+    @property
+    def table_defaults(self) -> list[dict[str, Any]]:
+        return self.default_filters
+
+
+def retrieve_schema_context(
+    schema: dict[str, Any],
+    question: str,
+    *,
+    max_tables: int = 20,
+    required_tables: set[str] | None = None,
+) -> SchemaContext:
+    """Assemble the schema slice handed to the LLM.
+
+    Everything is driven by the MCP ``get_schema`` payload: tables are ranked by
+    overlap with the question and with the glossary/metrics/examples that match
+    the question, then the schema's own ``joins`` graph is walked to pull in any
+    bridge tables needed to connect them. The agent stores no per-business table
+    lists, keyword triggers or hand-coded joins.
+    """
+    all_tables = _schema_tables(schema)
+    manual_joins = _schema_manual_joins(schema)
+    inferred_relations = _schema_inferred_relations(schema)
+    all_relations = _schema_relations(schema)
+    all_default_filters = _schema_default_filters(schema)
+    all_enums = _schema_enums(schema)
+    agent_instructions = _schema_agent_instructions(schema)
+    policy = _schema_policy(schema)
+    glossary_terms = _matching_items(
+        _schema_glossary(schema),
+        question,
+        keys=("term", "name", "description", "comment", "mapsTo", "type", "table", "field"),
+    )
+    business_terms = _matching_items(
+        _schema_business_terms(schema),
+        question,
+        keys=("term", "description", "comment", "mapsTo", "type"),
+    )
+    concepts = _matching_items(
+        _schema_concepts(schema),
+        question,
+        keys=("name", "term", "comment", "description", "positiveExamples", "sqlPredicate"),
+    )
+    metrics = _matching_items(
+        _schema_metrics(schema),
+        question,
+        keys=("name", "metric", "comment", "description", "definition", "expression"),
+    )
+    examples = _matching_items(
+        _schema_examples(schema),
+        question,
+        keys=("question", "description", "intent", "questionExamples"),
+    )
+    example_join_patterns = _schema_example_join_patterns(examples)
+    example_guidance = _schema_example_guidance(examples)
+    structure_hints = _schema_structure_hints(schema, question)
+    table_names_from_context = _target_table_names(glossary_terms + business_terms + concepts + metrics + examples)
+    example_table_names = _ordered_target_table_names(examples)
+    glossary_metric_table_names = _target_table_names(glossary_terms + business_terms + concepts + metrics)
+    required_tables = required_tables or set()
+    preferred = table_names_from_context | required_tables
+
+    tables = _rank_tables(all_tables, question, preferred)[:max_tables]
+    if isinstance(all_tables, list):
+        # Pin the must-have tables to the front in a deterministic order:
+        # required (resolved-entity) tables first, then tables referenced by the
+        # best matched examples, then glossary / metrics. Example table order is
+        # preserved because examples often encode the intended join path; this
+        # keeps late-declared fact tables from being evicted by weaker matches.
+        ordered_names = _ordered_preferred_names(
+            all_tables,
+            required_tables,
+            example_table_names,
+            glossary_metric_table_names,
+        )
+        if ordered_names:
+            tables = _prepend_named_tables(all_tables, tables, ordered_names, max_tables)
+        tables = _expand_via_join_graph(all_tables, all_relations, tables, max_tables)
 
-LOCATION_RECRUITING_TABLES = (
-    "province",
-    "city",
-    "region",
-    "job_basic_info",
-    "job_address",
-    "job_store",
-    "store",
-)
-
-WORK_ORDER_DETAIL_TABLES = (
-    "mvp_applied_jobs_by_helped_account",
-    "c_helped_account",
-    "resume",
-    "job",
-    "job_basic_info",
-    "brand",
-    "supplier",
-    "sponge_project",
-)
-
-
-def retrieve_schema_context(schema: dict[str, Any], question: str, *, max_tables: int = 12) -> SchemaContext:
-    glossary = _matching_items(schema.get("glossary", []), question, keys=("term", "description"))
-    metrics = _matching_items(schema.get("metrics", []), question, keys=("name", "metric", "description", "definition"))
-    examples = _matching_items(schema.get("examples", []), question, keys=("question", "description"))
-    table_names_from_context = _target_table_names(glossary + metrics + examples)
-    tables = _select_tables(schema.get("tables", []), question, table_names_from_context, max_tables)
     selected_table_names = {_table_name(table) for table in tables}
-    joins = _filter_joins(schema.get("joins", []), selected_table_names)
-    joins = _append_derived_joins(schema, joins, selected_table_names)
-    return SchemaContext(tables=tables, joins=joins, glossary=glossary, metrics=metrics, examples=examples)
+    relations = _filter_joins(all_relations, selected_table_names, required_tables)
+    default_filters = _filter_table_defaults(all_default_filters, selected_table_names)
+    enums = _filter_enums(all_enums, selected_table_names)
+    return SchemaContext(
+        tables=tables,
+        relations=relations,
+        business_terms=business_terms,
+        concepts=concepts,
+        metrics=metrics,
+        examples=examples,
+        default_filters=default_filters,
+        enums=enums,
+        agent_instructions=agent_instructions,
+        policy=policy,
+        glossary_terms=glossary_terms,
+        manual_joins=_filter_joins(manual_joins, selected_table_names, required_tables),
+        inferred_relations=_filter_joins(inferred_relations, selected_table_names, required_tables),
+        example_join_patterns=example_join_patterns,
+        example_guidance=example_guidance,
+        structure_hints=structure_hints,
+    )
+
+
+def narrow_schema_context_for_query_plan(
+    schema_context: SchemaContext,
+    query_plan: dict[str, Any],
+    *,
+    full_schema: dict[str, Any] | None = None,
+    max_tables: int = 12,
+    join_hops: int = 4,
+) -> SchemaContext:
+    """Shrink schemaContext for SQL sampling when a query plan is already confirmed.
+
+    Roll sampling models have limited context/output budgets; passing the full
+    ranked schema slice (~20 tables × all columns) often yields truncated or
+    invalid JSON. This keeps plan tables, join-path neighbours, plan fields,
+    and relation keys only.
+    """
+    required_table_names = _required_tables_from_query_plan(query_plan)
+    plan_table_names = {
+        str(item.get("name") or "").strip()
+        for item in query_plan.get("tables", [])
+        if isinstance(item, dict) and str(item.get("name") or "").strip()
+    }
+    plan_table_names |= required_table_names
+    if not plan_table_names:
+        return schema_context
+
+    all_tables = _schema_tables(full_schema) if full_schema else schema_context.tables
+    all_relations = _schema_relations(full_schema) if full_schema else schema_context.relations
+    tables_by_name = {_table_name(table): table for table in all_tables if isinstance(table, dict)}
+
+    selected_names = _expand_table_names_via_relations(
+        plan_table_names,
+        all_relations,
+        max_tables=max_tables,
+        join_hops=join_hops,
+        required_names=required_table_names,
+    )
+    keep_columns = _column_names_for_query_plan(query_plan, all_relations, selected_names)
+    ordered_tables = [
+        _trim_table_columns(tables_by_name[name], keep_columns.get(name, set()))
+        for name in sorted(selected_names)
+        if name in tables_by_name
+    ]
+
+    return SchemaContext(
+        tables=ordered_tables,
+        relations=_filter_joins(all_relations, selected_names, plan_table_names),
+        business_terms=schema_context.business_terms,
+        concepts=schema_context.concepts,
+        metrics=schema_context.metrics,
+        examples=[
+            example
+            for example in schema_context.examples
+            if _example_tables(example).issubset(selected_names) or not _example_tables(example)
+        ],
+        default_filters=_filter_table_defaults(schema_context.default_filters, selected_names),
+        enums=_filter_enums(schema_context.enums, selected_names),
+        agent_instructions=schema_context.agent_instructions,
+        policy=schema_context.policy,
+        glossary_terms=schema_context.glossary_terms,
+        manual_joins=_filter_joins(
+            _schema_manual_joins(full_schema) if full_schema else schema_context.manual_joins,
+            selected_names,
+            plan_table_names,
+        ),
+        inferred_relations=_filter_joins(
+            schema_context.inferred_relations,
+            selected_names,
+            plan_table_names,
+        ),
+        example_join_patterns=[
+            pattern
+            for pattern in schema_context.example_join_patterns
+            if _join_pattern_tables(pattern).issubset(selected_names) or not _join_pattern_tables(pattern)
+        ],
+        example_guidance=[
+            item
+            for item in schema_context.example_guidance
+            if _join_pattern_tables(item.get("joinPattern", {})).issubset(selected_names)
+            or not _join_pattern_tables(item.get("joinPattern", {}))
+        ],
+        structure_hints=schema_context.structure_hints,
+    )
+
+
+def _required_tables_from_query_plan(query_plan: dict[str, Any]) -> set[str]:
+    """Tables referenced by filters/entities that must survive schema narrowing."""
+    required: set[str] = set()
+    maps_to_table = {
+        "brand": "brand",
+        "project": "sponge_project",
+        "city": "city",
+        "region": "region",
+        "store": "store",
+    }
+    job_tables = {"job", "job_basic_info", "job_address", "job_store", "job_hiring_requirement"}
+    for flt in query_plan.get("filters", []):
+        if not isinstance(flt, dict):
+            continue
+        field = str(flt.get("field") or "").strip()
+        if "." in field:
+            required.add(field.split(".", 1)[0].strip())
+    for entity in query_plan.get("entities", []):
+        if not isinstance(entity, dict):
+            continue
+        entity_type = str(entity.get("type") or "").strip()
+        if entity_type in maps_to_table:
+            required.add(maps_to_table[entity_type])
+        elif entity_type and entity_type not in {"status", "name"}:
+            required.add(entity_type)
+    for field in query_plan.get("fields", []):
+        if not isinstance(field, dict):
+            continue
+        table = str(field.get("table") or "").strip()
+        if table:
+            required.add(table)
+    if required.intersection({"city", "region"}) and required.intersection(job_tables):
+        required.update({"job_address", "city"})
+    if "region" in required and required.intersection(job_tables):
+        required.add("region")
+    if "store" in required and required.intersection(job_tables):
+        required.add("job_store")
+    required.discard("")
+    return required
+
+
+def _expand_table_names_via_relations(
+    seed_names: set[str],
+    relations: list[dict[str, Any]],
+    *,
+    max_tables: int,
+    join_hops: int,
+    required_names: set[str] | None = None,
+) -> set[str]:
+    selected = {name for name in seed_names if name}
+    required = {name for name in (required_names or set()) if name}
+    for _ in range(join_hops):
+        if len(selected) >= max_tables:
+            break
+        expanded = set(selected)
+        for relation in relations:
+            if not isinstance(relation, dict):
+                continue
+            left_table, _ = _relation_left(relation)
+            right_table, _ = _relation_right(relation)
+            if left_table in selected:
+                expanded.add(right_table)
+            if right_table in selected:
+                expanded.add(left_table)
+        if expanded == selected:
+            break
+        selected = expanded
+    ordered = sorted(selected)
+    if len(ordered) <= max_tables:
+        return set(ordered)
+    optional = [name for name in ordered if name not in required]
+    required_list = [name for name in ordered if name in required]
+    slots_for_optional = max(0, max_tables - len(required_list))
+    return set(required_list + optional[:slots_for_optional])
+
+
+def _column_names_for_query_plan(
+    query_plan: dict[str, Any],
+    relations: list[dict[str, Any]],
+    selected_names: set[str],
+) -> dict[str, set[str]]:
+    keep: dict[str, set[str]] = {name: {"id"} for name in selected_names}
+    for field in query_plan.get("fields", []):
+        if not isinstance(field, dict):
+            continue
+        table = str(field.get("table") or "").strip()
+        column = str(field.get("field") or "").strip()
+        if table and column:
+            keep.setdefault(table, set()).add(column)
+    for relation in relations:
+        if not isinstance(relation, dict):
+            continue
+        left_table, left_column = _relation_left(relation)
+        right_table, right_column = _relation_right(relation)
+        if left_table in selected_names and left_column:
+            keep.setdefault(left_table, set()).add(left_column)
+        if right_table in selected_names and right_column:
+            keep.setdefault(right_table, set()).add(right_column)
+    for entity in query_plan.get("entities", []):
+        if not isinstance(entity, dict):
+            continue
+        entity_type = str(entity.get("type") or "")
+        if entity_type == "supplier":
+            keep.setdefault("supplier", set()).update({"id", "name", "nick_name"})
+    if "failure_reasons" in selected_names:
+        keep.setdefault("failure_reasons", set()).update({"id", "info"})
+    for flt in query_plan.get("filters", []):
+        if not isinstance(flt, dict):
+            continue
+        field = str(flt.get("field") or "").strip()
+        if "." in field:
+            table, column = _split_field_ref(field)
+            if table in selected_names and column:
+                keep.setdefault(table, set()).add(column)
+        for ref in _field_refs(str(flt.get("value") or "")):
+            table, column = _split_field_ref(ref)
+            if table in selected_names:
+                keep.setdefault(table, set()).add(column)
+    return keep
+
+
+def _trim_table_columns(table: dict[str, Any], keep_columns: set[str]) -> dict[str, Any]:
+    columns = table.get("columns")
+    if not isinstance(columns, list) or not keep_columns:
+        return table
+    trimmed = [
+        column
+        for column in columns
+        if isinstance(column, dict)
+        and str(column.get("columnName") or column.get("name") or "") in keep_columns
+    ]
+    if not trimmed:
+        return table
+    return {**table, "columns": trimmed}
+
+
+def _example_tables(example: dict[str, Any]) -> set[str]:
+    names: set[str] = set()
+    for key in ("requiredTables", "tables"):
+        value = example.get(key)
+        if isinstance(value, list):
+            names.update(str(item) for item in value if str(item).strip())
+    return names
 
 
-def _select_tables(tables: Any, question: str, preferred_names: set[str], max_tables: int) -> list[dict[str, Any]]:
-    ranked = _rank_tables(tables, question, preferred_names)
-    if _needs_work_order_detail_tables(question) and isinstance(tables, list):
-        return _prepend_named_tables(tables, ranked, WORK_ORDER_DETAIL_TABLES, max_tables)
-    if not _needs_location_recruiting_tables(question) or not isinstance(tables, list):
-        return ranked[:max_tables]
+def _join_pattern_tables(pattern: dict[str, Any]) -> set[str]:
+    names: set[str] = set()
+    from_clause = str(pattern.get("from") or "")
+    match = re.search(r"\b([A-Za-z_][A-Za-z0-9_]*)\b", from_clause)
+    if match:
+        names.add(match.group(1))
+    for join in pattern.get("joins", []) if isinstance(pattern.get("joins"), list) else []:
+        if not isinstance(join, dict):
+            continue
+        table_clause = str(join.get("table") or "")
+        match = re.search(r"\b([A-Za-z_][A-Za-z0-9_]*)\b", table_clause)
+        if match:
+            names.add(match.group(1))
+    return names
+
+
+def _ordered_preferred_names(
+    tables: Any,
+    required_tables: set[str],
+    example_tables: tuple[str, ...],
+    context_tables: set[str],
+) -> tuple[str, ...]:
+    """Deterministic priority order for must-have tables.
+
+    Required (resolved-entity) tables come first and follow schema declaration
+    order. Matched example tables come next in the example SQL/table order, then
+    glossary / metric tables follow schema order. Required tables always win the
+    max_tables budget, while example join paths stay intact.
+    """
+    if not isinstance(tables, list):
+        return ()
+    schema_order = [_table_name(table) for table in tables if isinstance(table, dict)]
+    schema_names = set(schema_order)
+    required_names = [name for name in schema_order if name in required_tables]
+    example_names = [
+        name
+        for name in example_tables
+        if name in schema_names and name not in required_tables
+    ]
+    context_names = [
+        name
+        for name in schema_order
+        if name in context_tables and name not in required_tables and name not in example_names
+    ]
+    return tuple(dict.fromkeys(required_names + example_names + context_names))
+
+
+def _expand_via_join_graph(
+    all_tables: list[dict[str, Any]],
+    relations: list[dict[str, Any]],
+    tables: list[dict[str, Any]],
+    max_tables: int,
+) -> list[dict[str, Any]]:
+    """Add bridge tables that connect two or more already-selected tables.
 
-    return _prepend_named_tables(tables, ranked, LOCATION_RECRUITING_TABLES, max_tables)
+    Relationship knowledge lives entirely in ``structure.relations``; this walks that
+    graph so indirect paths (e.g. brand -> brand_project -> sponge_project) are
+    complete without hard-coding any business-specific table set.
+    """
+    if not isinstance(all_tables, list) or not isinstance(relations, list):
+        return tables[:max_tables]
+    by_name = {_table_name(table): table for table in all_tables if isinstance(table, dict)}
+    selected = {_table_name(table) for table in tables}
+    adjacency: dict[str, set[str]] = {}
+    for relation in relations:
+        if not isinstance(relation, dict):
+            continue
+        left, _ = _relation_left(relation)
+        right, _ = _relation_right(relation)
+        if left and right:
+            adjacency.setdefault(left, set()).add(right)
+            adjacency.setdefault(right, set()).add(left)
+    result = list(tables)
+    for candidate, neighbours in adjacency.items():
+        if len(result) >= max_tables:
+            break
+        if candidate in selected or candidate not in by_name:
+            continue
+        if len(neighbours & selected) >= 2:
+            result.append(by_name[candidate])
+            selected.add(candidate)
+    return result[:max_tables]
 
 
 def _prepend_named_tables(
@@ -85,14 +475,13 @@ def _prepend_named_tables(
 def _rank_tables(tables: Any, question: str, preferred_names: set[str]) -> list[dict[str, Any]]:
     if not isinstance(tables, list):
         return []
+    tokens = _question_tokens(question)
     ranked: list[tuple[int, dict[str, Any]]] = []
     for table in tables:
         if not isinstance(table, dict):
             continue
         name = _table_name(table)
         score = 5 if name in preferred_names else 0
-        if _needs_work_order_detail_tables(question) and name in WORK_ORDER_DETAIL_TABLES:
-            score += 10
         text = " ".join(
             [
                 name,
@@ -100,7 +489,7 @@ def _rank_tables(tables: Any, question: str, preferred_names: set[str]) -> list[
                 " ".join(_column_text(column) for column in table.get("columns", []) if isinstance(column, dict)),
             ]
         ).lower()
-        for token in _question_tokens(question):
+        for token in tokens:
             if token and token in text:
                 score += 1
         if score > 0 or not preferred_names:
@@ -109,101 +498,496 @@ def _rank_tables(tables: Any, question: str, preferred_names: set[str]) -> list[
     return [table for _, table in ranked]
 
 
+def _schema_tables(schema: dict[str, Any]) -> list[dict[str, Any]]:
+    tables = schema.get("tables")
+    if isinstance(tables, list):
+        return [table for table in tables if isinstance(table, dict)]
+    nested_schema = schema.get("schema")
+    if isinstance(nested_schema, dict) and isinstance(nested_schema.get("tables"), list):
+        return [table for table in nested_schema["tables"] if isinstance(table, dict)]
+    return []
+
+
+def _schema_structure(schema: dict[str, Any]) -> dict[str, Any]:
+    structure = schema.get("structure")
+    return structure if isinstance(structure, dict) else {}
+
+
+def _schema_relations(schema: dict[str, Any]) -> list[dict[str, Any]]:
+    return _schema_manual_joins(schema) + _schema_inferred_relations(schema)
+
+
+def _schema_manual_joins(schema: dict[str, Any]) -> list[dict[str, Any]]:
+    legacy_joins = schema.get("joins")
+    if isinstance(legacy_joins, list):
+        return [_mark_relation_source(join, "joins") for join in legacy_joins if isinstance(join, dict)]
+    return []
+
+
+def _schema_inferred_relations(schema: dict[str, Any]) -> list[dict[str, Any]]:
+    structure_relations = _schema_structure(schema).get("relations")
+    if isinstance(structure_relations, list):
+        return [_mark_relation_source(relation, "structure.relations") for relation in structure_relations if isinstance(relation, dict)]
+    return []
+
+
+def _mark_relation_source(relation: dict[str, Any], source: str) -> dict[str, Any]:
+    return relation if relation.get("source") else {**relation, "source": source}
+
+
+def _schema_default_filters(schema: dict[str, Any]) -> list[dict[str, Any]]:
+    result: list[dict[str, Any]] = []
+    default_filters = _schema_structure(schema).get("defaultFilters")
+    if isinstance(default_filters, list):
+        result.extend(item for item in default_filters if isinstance(item, dict))
+    legacy_defaults = schema.get("tableDefaults")
+    if isinstance(legacy_defaults, list):
+        result.extend(_flatten_table_default(item) for item in legacy_defaults if isinstance(item, dict))
+    return result
+
+
+def _flatten_table_default(item: dict[str, Any]) -> dict[str, Any]:
+    if any(key in item for key in ("predicate", "condition", "where", "expression", "filter")):
+        return item
+    filters = item.get("defaultFilters")
+    if isinstance(filters, list) and filters:
+        return {**item, "condition": " AND ".join(str(value) for value in filters if value)}
+    return item
+
+
+def _schema_business_terms(schema: dict[str, Any]) -> list[dict[str, Any]]:
+    business_terms = _schema_structure(schema).get("businessTerms")
+    if isinstance(business_terms, list):
+        return [item for item in business_terms if isinstance(item, dict)]
+    legacy_glossary = schema.get("glossary")
+    if isinstance(legacy_glossary, list):
+        return [item for item in legacy_glossary if isinstance(item, dict)]
+    return []
+
+
+def _schema_agent_instructions(schema: dict[str, Any]) -> list[dict[str, Any]]:
+    return _normalize_instruction_items(_schema_list(schema, "agentInstructions"))
+
+
+def _normalize_instruction_items(items: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    """Flatten schema instruction payloads into numbered ``{"text": ...}`` entries for LLM prompts."""
+    normalized: list[dict[str, Any]] = []
+    for item in items:
+        if not isinstance(item, dict):
+            continue
+        for key in ("text", "instruction", "content"):
+            value = item.get(key)
+            if isinstance(value, str) and value.strip():
+                normalized.append({"text": value.strip()})
+                break
+        else:
+            nested = item.get("instructions")
+            if isinstance(nested, list):
+                for line in nested:
+                    if isinstance(line, str) and line.strip():
+                        normalized.append({"text": line.strip()})
+            elif isinstance(nested, str) and nested.strip():
+                normalized.append({"text": nested.strip()})
+            elif item.get("text") is None and item.get("instructions") is None:
+                normalized.append(item)
+    return normalized
+
+
+def _schema_policy(schema: dict[str, Any]) -> list[dict[str, Any]]:
+    return _schema_list(schema, "policy")
+
+
+def _schema_glossary(schema: dict[str, Any]) -> list[dict[str, Any]]:
+    return _schema_list(schema, "glossary")
+
+
+def _schema_list(schema: dict[str, Any], key: str) -> list[dict[str, Any]]:
+    value = schema.get(key)
+    if isinstance(value, list):
+        return [item for item in value if isinstance(item, dict)]
+    if isinstance(value, dict):
+        return [value]
+    if isinstance(value, str) and value.strip():
+        return [{"text": value.strip()}]
+    structure_value = _schema_structure(schema).get(key)
+    if isinstance(structure_value, list):
+        return [item for item in structure_value if isinstance(item, dict)]
+    if isinstance(structure_value, dict):
+        return [structure_value]
+    if isinstance(structure_value, str) and structure_value.strip():
+        return [{"text": structure_value.strip()}]
+    return []
+
+
+def _schema_concepts(schema: dict[str, Any]) -> list[dict[str, Any]]:
+    concepts = _schema_structure(schema).get("concepts")
+    if isinstance(concepts, list):
+        return [item for item in concepts if isinstance(item, dict)]
+    return []
+
+
+def _schema_metrics(schema: dict[str, Any]) -> list[dict[str, Any]]:
+    metrics = _schema_structure(schema).get("metrics")
+    if isinstance(metrics, list):
+        return [item for item in metrics if isinstance(item, dict)]
+    legacy_metrics = schema.get("metrics")
+    if isinstance(legacy_metrics, list):
+        return [item for item in legacy_metrics if isinstance(item, dict)]
+    return []
+
+
+def _schema_examples(schema: dict[str, Any]) -> list[dict[str, Any]]:
+    example = schema.get("example")
+    if isinstance(example, dict) and isinstance(example.get("examples"), list):
+        return [item for item in example["examples"] if isinstance(item, dict)]
+    legacy_examples = schema.get("examples")
+    if isinstance(legacy_examples, list):
+        return [item for item in legacy_examples if isinstance(item, dict)]
+    return []
+
+
+def _schema_example_join_patterns(examples: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    result: list[dict[str, Any]] = []
+    for example in examples:
+        join_pattern = example.get("joinPattern")
+        if isinstance(join_pattern, dict):
+            result.append(join_pattern)
+    return result
+
+
+def _schema_example_guidance(examples: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    result: list[dict[str, Any]] = []
+    for example in examples:
+        if not isinstance(example, dict):
+            continue
+        join_pattern = example.get("joinPattern")
+        if not isinstance(join_pattern, dict):
+            continue
+        item: dict[str, Any] = {"joinPattern": join_pattern}
+        for key in ("id", "intent", "notes", "questionExamples", "requiredTables"):
+            value = example.get(key)
+            if value:
+                item[key] = value
+        result.append(item)
+    return result
+
+
+def _schema_structure_hints(schema: dict[str, Any], question: str) -> list[dict[str, Any]]:
+    hints: list[dict[str, Any]] = []
+    if not any(token in question for token in ("原因", "失败原因", "工单原因", "失败阶段")):
+        return hints
+    structure = schema.get("structure")
+    if not isinstance(structure, dict):
+        return hints
+    unpivot = structure.get("workOrderFailureReasonUnpivot")
+    if isinstance(unpivot, dict):
+        hints.append({"type": "workOrderFailureReasonUnpivot", **unpivot})
+    mappings = structure.get("workOrderFailureReasonMappings")
+    if isinstance(mappings, list) and mappings:
+        hints.append({"type": "workOrderFailureReasonMappings", "mappings": mappings})
+    return hints
+
+
+def _schema_enums(schema: dict[str, Any]) -> list[dict[str, Any]]:
+    """Merge enum metadata from top-level enums, column.enum, and concept enumRefs."""
+    by_field: dict[str, dict[Any, str]] = {}
+
+    def add_value(field: str, value: Any, label: str, *, overwrite: bool = True) -> None:
+        if not field or value is None:
+            return
+        text = str(label or "").strip()
+        if not text:
+            return
+        bucket = by_field.setdefault(field, {})
+        if overwrite or value not in bucket:
+            bucket[value] = text
+
+    def add_enum_values(field: str, values: Any, *, overwrite: bool = True) -> None:
+        if not field or not isinstance(values, list):
+            return
+        for item in values:
+            if not isinstance(item, dict):
+                continue
+            value = item.get("value")
+            if value is None and "code" in item:
+                value = item.get("code")
+            add_value(field, value, str(item.get("label") or item.get("meaning") or ""), overwrite=overwrite)
+
+    top_level = schema.get("enums")
+    if isinstance(top_level, list):
+        for entry in top_level:
+            if not isinstance(entry, dict):
+                continue
+            field = str(entry.get("field") or "")
+            add_enum_values(field, entry.get("values"), overwrite=True)
+
+    for table in _schema_tables(schema):
+        table_name = _table_name(table)
+        if not table_name:
+            continue
+        columns = table.get("columns")
+        if not isinstance(columns, list):
+            continue
+        for column in columns:
+            if not isinstance(column, dict):
+                continue
+            column_name = str(column.get("columnName") or column.get("name") or "")
+            enum = column.get("enum")
+            if not column_name or not isinstance(enum, dict):
+                continue
+            add_enum_values(f"{table_name}.{column_name}", enum.get("values"), overwrite=True)
+
+    for concept in _schema_concepts(schema):
+        enum_refs = concept.get("enumRefs")
+        if not isinstance(enum_refs, list):
+            continue
+        label_parts = [
+            str(concept.get("comment") or ""),
+            str(concept.get("name") or ""),
+            str(concept.get("description") or ""),
+        ]
+        for enum_ref in enum_refs:
+            if not isinstance(enum_ref, dict):
+                continue
+            field = str(enum_ref.get("field") or "")
+            if not field:
+                continue
+            meaning = str(enum_ref.get("meaning") or "")
+            label = " / ".join(part for part in label_parts + [meaning] if part)
+            if "value" in enum_ref:
+                add_value(field, enum_ref["value"], label, overwrite=False)
+                continue
+            values = enum_ref.get("values")
+            if isinstance(values, list):
+                for value in values:
+                    add_value(field, value, meaning or label, overwrite=False)
+
+    return [
+        {
+            "field": field,
+            "values": [{"value": value, "label": label} for value, label in sorted(values.items(), key=lambda item: str(item[0]))],
+        }
+        for field, values in sorted(by_field.items())
+    ]
+
+
 def _matching_items(items: Any, question: str, *, keys: tuple[str, ...]) -> list[dict[str, Any]]:
     if not isinstance(items, list):
         return []
-    tokens = _question_tokens(question)
+    tokens = _semantic_tokens(question)
     matches: list[tuple[int, dict[str, Any]]] = []
     for item in items:
         if not isinstance(item, dict):
             continue
         text = " ".join(str(item.get(key) or "") for key in keys + ("sql",)).lower()
         score = sum(1 for token in tokens if token and token in text)
-        if _needs_work_order_detail_tables(question):
-            for term in (
-                "mvp_applied_jobs_by_helped_account",
-                "work_order_status",
-                "current_helper_status_id",
-                "job_basic_info.job_id",
-                "报名",
-                "工单",
-                "候选人状态",
-                "面试成功",
-            ):
-                if term.lower() in text:
-                    score += 5
         if score > 0:
             matches.append((score, item))
     matches.sort(key=lambda match: match[0], reverse=True)
     return [item for _, item in matches[:8]]
 
 
+def concept_question_match_score(concept: dict[str, Any], question: str) -> int:
+    """Score how well a schema concept matches the user question."""
+    if not isinstance(concept, dict):
+        return 0
+    question_text = _normalize_fullwidth_digits(str(question or ""))
+    score = 0
+    for example in concept.get("positiveExamples") or []:
+        example_text = str(example or "").strip()
+        if example_text and example_text in question_text:
+            score += 10
+    for example in concept.get("negativeExamples") or []:
+        example_text = str(example or "").strip()
+        if example_text and example_text in question_text:
+            score -= 100
+    tokens = _semantic_tokens(question_text)
+    blob = " ".join(
+        str(concept.get(key) or "")
+        for key in ("comment", "description", "name", "intent")
+    ).lower()
+    score += sum(1 for token in tokens if token and token in blob)
+    return score
+
+
+def match_concepts_for_question(concepts: Any, question: str) -> list[dict[str, Any]]:
+    """Pick schema concepts for query-plan filters, honoring conflictsWith."""
+    if not isinstance(concepts, list):
+        return []
+    scored: list[tuple[int, dict[str, Any]]] = []
+    for concept in concepts:
+        if not isinstance(concept, dict):
+            continue
+        match_score = concept_question_match_score(concept, question)
+        if match_score > 0:
+            scored.append((match_score, concept))
+    scored.sort(key=lambda item: item[0], reverse=True)
+    selected: list[dict[str, Any]] = []
+    excluded_names: set[str] = set()
+    for _, concept in scored:
+        name = str(concept.get("name") or "").strip()
+        if not name or name in excluded_names:
+            continue
+        if any(name in (item.get("conflictsWith") or []) for item in selected):
+            continue
+        selected.append(concept)
+        for conflict in concept.get("conflictsWith") or []:
+            conflict_name = str(conflict or "").strip()
+            if conflict_name:
+                excluded_names.add(conflict_name)
+    return selected
+
+
+def _normalize_fullwidth_digits(text: str) -> str:
+    return text.translate(str.maketrans("０１２３４５６７８９", "0123456789"))
+
+
+def _semantic_tokens(question: str) -> list[str]:
+    ignored = {
+        "查询",
+        "查看",
+        "看看",
+        "搜索",
+        "统计",
+        "多少",
+        "几个",
+        "几条",
+        "哪些",
+        "列表",
+        "明细",
+        "数据",
+        "目前",
+        "当前",
+        "现在",
+        "涉及",
+        "关联",
+        "关系",
+    }
+    return [token for token in _question_tokens(question) if token not in ignored]
+
+
 def _target_table_names(items: list[dict[str, Any]]) -> set[str]:
+    return set(_ordered_target_table_names(items))
+
+
+def _ordered_target_table_names(items: list[dict[str, Any]]) -> tuple[str, ...]:
     names: set[str] = set()
+    ordered: list[str] = []
     for item in items:
-        for key in ("tables", "targetTables"):
+        for key in ("tables", "targetTables", "requiredTables"):
             value = item.get(key)
             if isinstance(value, list):
-                names.update(str(name) for name in value)
+                for name in value:
+                    _append_unique_table(ordered, names, str(name))
+        for key in ("mapsTo", "baseTable"):
+            value = item.get(key)
+            if isinstance(value, str):
+                _append_unique_table(ordered, names, value)
         sql = item.get("sql")
         if isinstance(sql, str):
-            names.update(referenced_table for referenced_table in _referenced_tables(sql))
+            for referenced_table in _referenced_tables(sql):
+                _append_unique_table(ordered, names, referenced_table)
+        for field_key in ("expression", "predicate"):
+            value = item.get(field_key)
+            if isinstance(value, str):
+                for field_ref in _field_refs(value):
+                    _append_unique_table(ordered, names, field_ref.split(".", 1)[0])
+        predicates = item.get("sqlPredicate")
+        if isinstance(predicates, list):
+            for predicate in predicates:
+                for field_ref in _field_refs(str(predicate)):
+                    _append_unique_table(ordered, names, field_ref.split(".", 1)[0])
+        enum_refs = item.get("enumRefs")
+        if isinstance(enum_refs, list):
+            for enum_ref in enum_refs:
+                if isinstance(enum_ref, dict) and isinstance(enum_ref.get("field"), str):
+                    _append_unique_table(ordered, names, enum_ref["field"].split(".", 1)[0])
+        join_pattern = item.get("joinPattern")
+        if isinstance(join_pattern, dict):
+            from_clause = join_pattern.get("from")
+            if isinstance(from_clause, str):
+                for referenced_table in _referenced_tables("FROM " + from_clause):
+                    _append_unique_table(ordered, names, referenced_table)
+            joins = join_pattern.get("joins")
+            if isinstance(joins, list):
+                for join in joins:
+                    if isinstance(join, dict) and isinstance(join.get("table"), str):
+                        for referenced_table in _referenced_tables("JOIN " + join["table"]):
+                            _append_unique_table(ordered, names, referenced_table)
         targets = item.get("targets")
         if isinstance(targets, list):
             for target in targets:
                 if isinstance(target, str) and "." in target:
-                    names.add(target.split(".", 1)[0])
-    return names
+                    _append_unique_table(ordered, names, target.split(".", 1)[0])
+    return tuple(ordered)
+
+
+def _append_unique_table(ordered: list[str], seen: set[str], name: str) -> None:
+    if not name or name in seen:
+        return
+    ordered.append(name)
+    seen.add(name)
 
 
-def _filter_joins(joins: Any, selected_table_names: set[str]) -> list[dict[str, Any]]:
+def _filter_joins(
+    joins: Any,
+    selected_table_names: set[str],
+    priority_table_names: set[str] | None = None,
+) -> list[dict[str, Any]]:
     if not isinstance(joins, list):
         return []
-    result: list[dict[str, Any]] = []
-    for join in joins:
+    priority_table_names = priority_table_names or set()
+    direct: list[tuple[int, int, dict[str, Any]]] = []
+    adjacent: list[tuple[int, int, dict[str, Any]]] = []
+    for index, join in enumerate(joins):
         if not isinstance(join, dict):
             continue
-        join_tables = {
-            str(join.get("leftTable") or join.get("from") or ""),
-            str(join.get("rightTable") or join.get("to") or ""),
-        }
-        if join_tables & selected_table_names:
-            result.append(join)
-    return result[:12]
+        left_table, _ = _relation_left(join)
+        right_table, _ = _relation_right(join)
+        join_tables = {left_table, right_table}
+        join_tables.discard("")
+        matched = join_tables & selected_table_names
+        priority_score = len(join_tables & priority_table_names)
+        if join_tables <= selected_table_names:
+            direct.append((-priority_score, index, join))
+        elif matched:
+            adjacent.append((-priority_score, index, join))
+    direct.sort()
+    adjacent.sort()
+    return [join for _, _, join in direct + adjacent][:16]
 
 
-def _append_derived_joins(
-    schema: dict[str, Any],
-    joins: list[dict[str, Any]],
-    selected_table_names: set[str],
-) -> list[dict[str, Any]]:
-    result = list(joins)
-    existing = {
-        (
-            str(join.get("leftTable") or ""),
-            str(join.get("leftColumn") or ""),
-            str(join.get("rightTable") or ""),
-            str(join.get("rightColumn") or ""),
-        )
-        for join in result
-    }
-    if (
-        {"resume", "supplier"} <= selected_table_names
-        and _table_has_column(schema, "resume", "supplier_id")
-        and _table_has_column(schema, "supplier", "id")
-        and ("supplier", "id", "resume", "supplier_id") not in existing
-        and ("resume", "supplier_id", "supplier", "id") not in existing
-    ):
-        result.append(
-            {
-                "description": "简历和供应商关系；供应商名称可使用 supplier.nick_name",
-                "leftTable": "supplier",
-                "leftColumn": "id",
-                "rightTable": "resume",
-                "rightColumn": "supplier_id",
-            }
-        )
-    return result[:16]
+def _filter_table_defaults(table_defaults: Any, selected_table_names: set[str]) -> list[dict[str, Any]]:
+    """Pass through default filters for selected tables (data from get_schema)."""
+    if not isinstance(table_defaults, list):
+        return []
+    result: list[dict[str, Any]] = []
+    for entry in table_defaults:
+        if not isinstance(entry, dict):
+            continue
+        table = str(entry.get("table") or "")
+        if not table:
+            predicate = str(entry.get("predicate") or entry.get("condition") or "")
+            first_ref = _field_refs(predicate)
+            table = first_ref[0].split(".", 1)[0] if first_ref else ""
+        if table in selected_table_names:
+            result.append(entry)
+    return result
+
+
+def _filter_enums(enums: Any, selected_table_names: set[str]) -> list[dict[str, Any]]:
+    """Pass through enum definitions whose field belongs to a selected table."""
+    if not isinstance(enums, list):
+        return []
+    result: list[dict[str, Any]] = []
+    for entry in enums:
+        if not isinstance(entry, dict):
+            continue
+        field = str(entry.get("field") or "")
+        table = field.split(".", 1)[0] if "." in field else ""
+        if table in selected_table_names:
+            result.append(entry)
+    return result
 
 
 def _table_name(table: dict[str, Any]) -> str:
@@ -216,65 +1000,54 @@ def _column_text(column: dict[str, Any]) -> str:
     )
 
 
-def _table_has_column(schema: dict[str, Any], table_name: str, column_name: str) -> bool:
-    tables = schema.get("tables", [])
-    if not isinstance(tables, list):
-        return False
-    for table in tables:
-        if not isinstance(table, dict) or _table_name(table) != table_name:
-            continue
-        columns = table.get("columns", [])
-        if not isinstance(columns, list):
-            return False
-        return any(
-            isinstance(column, dict)
-            and str(column.get("columnName") or column.get("name") or "") == column_name
-            for column in columns
-        )
-    return False
+def _relation_left(relation: dict[str, Any]) -> tuple[str, str]:
+    table = str(relation.get("leftTable") or "")
+    column = str(relation.get("leftColumn") or relation.get("fromColumn") or "")
+    if table:
+        return table, column
+    return _split_field_ref(str(relation.get("from") or ""))
 
 
-def _question_tokens(question: str) -> list[str]:
-    lowered = question.lower()
-    tokens = [lowered]
-    tokens.extend(part for part in lowered.replace("_", " ").replace("-", " ").split() if part)
-    for word in (
-        "品牌",
-        "项目",
-        "岗位",
-        "门店",
-        "招聘",
-        "在招",
-        "数量",
-        "列表",
-        "报名",
-        "报名情况",
-        "报名工单",
-        "工单",
-        "候选人",
-        "候选人状态",
-        "面试成功",
-        "进行中",
-        "姓名",
-        "人员",
-        "清单",
-        "供应商",
-    ):
-        if word in question:
-            tokens.append(word)
-    return list(dict.fromkeys(tokens))
+def _relation_right(relation: dict[str, Any]) -> tuple[str, str]:
+    table = str(relation.get("rightTable") or "")
+    column = str(relation.get("rightColumn") or relation.get("toColumn") or "")
+    if table:
+        return table, column
+    return _split_field_ref(str(relation.get("to") or ""))
 
 
-def _referenced_tables(sql: str) -> set[str]:
-    return set(match.group(1) for match in re.finditer(r"\b(?:from|join)\s+([a-zA-Z_][a-zA-Z0-9_]*)\b", sql, re.IGNORECASE))
+def _split_field_ref(value: str) -> tuple[str, str]:
+    if "." not in value:
+        return value, ""
+    table, column = value.split(".", 1)
+    return table, column
 
 
-def _needs_location_recruiting_tables(question: str) -> bool:
-    return any(word in question for word in ("岗位", "门店", "招聘", "在招"))
+def _field_refs(text: str) -> list[str]:
+    return re.findall(r"\b[A-Za-z_][A-Za-z0-9_]*\.[A-Za-z_][A-Za-z0-9_]*\b", text)
 
 
-def _needs_work_order_detail_tables(question: str) -> bool:
-    return bool(
-        any(word in question for word in ("报名", "工单", "候选人"))
-        and any(word in question for word in ("姓名", "岗位", "项目", "候选人状态"))
-    )
+def _question_tokens(question: str) -> list[str]:
+    """Generic tokenization with no business vocabulary.
+
+    Produces the lowered question, its ascii word splits, and CJK character
+    bigrams so Chinese questions can still overlap Chinese table comments /
+    glossary text without any hand-maintained keyword list.
+    """
+    lowered = question.lower()
+    tokens = [lowered]
+    tokens.extend(part for part in re.split(r"[\s_\-]+", lowered) if part)
+    for run in re.findall(r"[\u4e00-\u9fff]+", question):
+        if len(run) == 1:
+            tokens.append(run)
+        for index in range(len(run) - 1):
+            tokens.append(run[index : index + 2])
+    return list(dict.fromkeys(tokens))
+
+
+def _referenced_tables(sql: str) -> tuple[str, ...]:
+    names: list[str] = []
+    seen: set[str] = set()
+    for match in re.finditer(r"\b(?:from|join)\s+([a-zA-Z_][a-zA-Z0-9_]*)\b", sql, re.IGNORECASE):
+        _append_unique_table(names, seen, match.group(1))
+    return tuple(names)