@roll-agent/octopus-agent 0.0.10 → 0.0.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/SKILL.md +25 -1
- package/package.json +1 -1
- package/pyproject.toml +4 -2
- package/references/env.yaml +17 -0
- package/scripts/refresh_dictionary.py +67 -0
- package/scripts/verify_binding.py +47 -0
- package/src/octopus_skill/_version.py +1 -1
- package/src/octopus_skill/agent.py +1099 -146
- package/src/octopus_skill/config.py +58 -10
- package/src/octopus_skill/entity_binding.py +523 -0
- package/src/octopus_skill/entity_dictionary.py +59 -0
- package/src/octopus_skill/entity_resolver.py +446 -0
- package/src/octopus_skill/llm_sql_generator.py +3 -0
- package/src/octopus_skill/mcp_client.py +16 -0
- package/src/octopus_skill/octopus_run.py +369 -17
- package/src/octopus_skill/prompt_builder.py +74 -31
- package/src/octopus_skill/schema_context_retriever.py +199 -150
- package/src/octopus_skill/sql_generator.py +432 -801
- package/src/octopus_skill/query_sql_cache.py +0 -72
|
@@ -24,13 +24,30 @@ class SchemaConfig:
|
|
|
24
24
|
refresh_interval_minutes: int
|
|
25
25
|
|
|
26
26
|
|
|
27
|
+
@dataclass(frozen=True)
|
|
28
|
+
class DictionaryConfig:
|
|
29
|
+
cache_path: Path
|
|
30
|
+
geo_alias_path: Path
|
|
31
|
+
refresh_on_start: bool
|
|
32
|
+
refresh_interval_minutes: int
|
|
33
|
+
entity_types: tuple[str, ...]
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
@dataclass(frozen=True)
|
|
37
|
+
class EntityBindingConfig:
|
|
38
|
+
# Compute bindings and log them, but do not change SQL generation yet.
|
|
39
|
+
shadow_mode: bool
|
|
40
|
+
# Entity types whose bindings are allowed to influence SQL (live rollout).
|
|
41
|
+
# In shadow_mode this is ignored.
|
|
42
|
+
enabled_entity_types: tuple[str, ...]
|
|
43
|
+
min_fuzzy_score: float
|
|
44
|
+
|
|
45
|
+
|
|
27
46
|
@dataclass(frozen=True)
|
|
28
47
|
class SqlConfig:
|
|
29
48
|
max_repair_attempts: int
|
|
30
49
|
default_limit: int
|
|
31
50
|
max_limit: int
|
|
32
|
-
query_cache_path: Path | None = None
|
|
33
|
-
query_cache_ttl_minutes: int = 30
|
|
34
51
|
|
|
35
52
|
|
|
36
53
|
@dataclass(frozen=True)
|
|
@@ -45,6 +62,8 @@ class AppConfig:
|
|
|
45
62
|
schema: SchemaConfig
|
|
46
63
|
sql: SqlConfig
|
|
47
64
|
audit: AuditConfig
|
|
65
|
+
dictionary: DictionaryConfig | None = None
|
|
66
|
+
binding: EntityBindingConfig | None = None
|
|
48
67
|
|
|
49
68
|
|
|
50
69
|
def load_config() -> AppConfig:
|
|
@@ -59,17 +78,29 @@ def load_config() -> AppConfig:
|
|
|
59
78
|
refresh_on_start=True,
|
|
60
79
|
refresh_interval_minutes=1440,
|
|
61
80
|
),
|
|
81
|
+
dictionary=DictionaryConfig(
|
|
82
|
+
cache_path=PACKAGE_ROOT / "data/sponge-entity-cache.json",
|
|
83
|
+
geo_alias_path=PACKAGE_ROOT / "data/geo-aliases.json",
|
|
84
|
+
refresh_on_start=True,
|
|
85
|
+
refresh_interval_minutes=1440,
|
|
86
|
+
entity_types=("brand", "province", "city", "region", "project"),
|
|
87
|
+
),
|
|
62
88
|
sql=SqlConfig(
|
|
63
89
|
max_repair_attempts=2,
|
|
64
|
-
default_limit=
|
|
90
|
+
default_limit=200,
|
|
65
91
|
max_limit=500,
|
|
66
|
-
query_cache_path=_optional_path("OCTOPUS_QUERY_SQL_CACHE_PATH"),
|
|
67
|
-
query_cache_ttl_minutes=30,
|
|
68
92
|
),
|
|
69
93
|
audit=AuditConfig(
|
|
70
94
|
enabled=True,
|
|
71
95
|
log_path=PACKAGE_ROOT / "logs/octopus-sponge-audit.log",
|
|
72
96
|
),
|
|
97
|
+
binding=EntityBindingConfig(
|
|
98
|
+
shadow_mode=_env_bool("OCTOPUS_BINDING_SHADOW", default=False),
|
|
99
|
+
enabled_entity_types=_env_csv(
|
|
100
|
+
"OCTOPUS_BINDING_ENABLED", default=("brand", "location", "project")
|
|
101
|
+
),
|
|
102
|
+
min_fuzzy_score=_env_float("OCTOPUS_BINDING_MIN_SCORE", default=0.6),
|
|
103
|
+
),
|
|
73
104
|
)
|
|
74
105
|
|
|
75
106
|
|
|
@@ -77,8 +108,25 @@ def load_config_from_env() -> AppConfig:
|
|
|
77
108
|
return load_config()
|
|
78
109
|
|
|
79
110
|
|
|
80
|
-
def
|
|
81
|
-
value = os.getenv(
|
|
82
|
-
if value is None or
|
|
83
|
-
return
|
|
84
|
-
return
|
|
111
|
+
def _env_bool(name: str, *, default: bool) -> bool:
|
|
112
|
+
value = os.getenv(name)
|
|
113
|
+
if value is None or value == "":
|
|
114
|
+
return default
|
|
115
|
+
return value.strip().lower() in {"1", "true", "yes", "on"}
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def _env_csv(name: str, *, default: tuple[str, ...] = ()) -> tuple[str, ...]:
|
|
119
|
+
value = os.getenv(name)
|
|
120
|
+
if value is None:
|
|
121
|
+
return default
|
|
122
|
+
return tuple(item.strip() for item in value.split(",") if item.strip())
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def _env_float(name: str, *, default: float) -> float:
|
|
126
|
+
value = os.getenv(name)
|
|
127
|
+
if value is None or value == "":
|
|
128
|
+
return default
|
|
129
|
+
try:
|
|
130
|
+
return float(value)
|
|
131
|
+
except ValueError:
|
|
132
|
+
return default
|
|
@@ -0,0 +1,523 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import re
|
|
4
|
+
from dataclasses import dataclass, field
|
|
5
|
+
from typing import Any, Iterable
|
|
6
|
+
|
|
7
|
+
from .entity_resolver import EntityResolver, Mention, ResolvedEntity, _geo_core, _norm
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
# brand/project are scanned for mentions; on equal-length text overlaps the
|
|
11
|
+
# higher-priority type wins.
|
|
12
|
+
_NAMED_TYPES = ("brand", "project")
|
|
13
|
+
_TYPE_PRIORITY = {"brand": 5, "project": 4}
|
|
14
|
+
_GEO_TYPES = ("province", "city", "region")
|
|
15
|
+
# Placeholder used to mask brand/project spans before geo resolution, so a city
|
|
16
|
+
# name embedded in a brand (e.g. "成都" in "成都你六姐") is not mis-detected.
|
|
17
|
+
_MASK_CHAR = "\u3000"
|
|
18
|
+
_STORE_NAME_CHARS = r"[\u4e00-\u9fffA-Za-z0-9()()·_\-]{2,40}"
|
|
19
|
+
_STORE_SEPARATOR = r"[\s::,,、;;]+"
|
|
20
|
+
_STORE_NAME_STOP = (
|
|
21
|
+
r"(?=[,,。;;??\s]|有哪些|有什么|哪些|多少|岗位|职位|在招|招聘|包括|返回|状态|$)"
|
|
22
|
+
)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@dataclass(frozen=True)
|
|
26
|
+
class EntityAmbiguity:
|
|
27
|
+
"""A user term that can refer to both brand and project."""
|
|
28
|
+
|
|
29
|
+
term: str
|
|
30
|
+
brands: list[ResolvedEntity] = field(default_factory=list)
|
|
31
|
+
projects: list[ResolvedEntity] = field(default_factory=list)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
@dataclass(frozen=True)
|
|
35
|
+
class EntityBindings:
|
|
36
|
+
"""Structured entities resolved from a question against the dictionary."""
|
|
37
|
+
|
|
38
|
+
brands: list[ResolvedEntity] = field(default_factory=list)
|
|
39
|
+
projects: list[ResolvedEntity] = field(default_factory=list)
|
|
40
|
+
location: dict[str, ResolvedEntity] = field(default_factory=dict)
|
|
41
|
+
ambiguity: EntityAmbiguity | None = None
|
|
42
|
+
|
|
43
|
+
def is_empty(self) -> bool:
|
|
44
|
+
return not self.brands and not self.projects and not self.location and self.ambiguity is None
|
|
45
|
+
|
|
46
|
+
def to_log_dict(self) -> dict[str, object]:
|
|
47
|
+
"""Compact form for shadow-mode audit logging."""
|
|
48
|
+
payload: dict[str, object] = {
|
|
49
|
+
"brands": [_entity_log(entity) for entity in self.brands],
|
|
50
|
+
"projects": [_entity_log(entity) for entity in self.projects],
|
|
51
|
+
"location": {level: _entity_log(entity) for level, entity in self.location.items()},
|
|
52
|
+
}
|
|
53
|
+
if self.ambiguity is not None:
|
|
54
|
+
payload["ambiguity"] = {
|
|
55
|
+
"term": self.ambiguity.term,
|
|
56
|
+
"brands": [_entity_log(entity) for entity in self.ambiguity.brands],
|
|
57
|
+
"projects": [_entity_log(entity) for entity in self.ambiguity.projects],
|
|
58
|
+
}
|
|
59
|
+
return payload
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def bind_entities(
|
|
63
|
+
question: str,
|
|
64
|
+
resolver: EntityResolver | None,
|
|
65
|
+
*,
|
|
66
|
+
fuzzy_candidates: dict[str, list[str]] | None = None,
|
|
67
|
+
min_fuzzy_score: float = 0.6,
|
|
68
|
+
) -> EntityBindings:
|
|
69
|
+
"""Resolve the entities mentioned in ``question``.
|
|
70
|
+
|
|
71
|
+
Pipeline:
|
|
72
|
+
|
|
73
|
+
1. **Named-entity mention scan** (brand/project) over the whole question.
|
|
74
|
+
Cross-type overlaps are resolved by longest-match, so a brand like
|
|
75
|
+
"成都你六姐" wins over the embedded city "成都".
|
|
76
|
+
2. **Geo resolution** on the question with the accepted brand/project spans
|
|
77
|
+
and explicit ``门店:…`` name spans masked out. ``resolve_location``
|
|
78
|
+
disambiguates duplicate region names (e.g. 宝山区 in 上海 vs 黑龙江)
|
|
79
|
+
using the co-mentioned province/city.
|
|
80
|
+
3. **Fuzzy fallback** for caller-provided spans (typically regex-extracted)
|
|
81
|
+
that the dictionary scan misses, e.g. the typo "刘姐" -> "成都你六姐".
|
|
82
|
+
"""
|
|
83
|
+
if resolver is None:
|
|
84
|
+
return EntityBindings()
|
|
85
|
+
|
|
86
|
+
explicit_type = _explicit_named_entity_type(question)
|
|
87
|
+
named_types = (explicit_type,) if explicit_type in _NAMED_TYPES else _NAMED_TYPES
|
|
88
|
+
named_mentions: list[Mention] = []
|
|
89
|
+
for entity_type in named_types:
|
|
90
|
+
named_mentions.extend(resolver.find_mentions(question, entity_type))
|
|
91
|
+
accepted = _resolve_overlaps(named_mentions)
|
|
92
|
+
preferred_scoped_project = False
|
|
93
|
+
if explicit_type is None:
|
|
94
|
+
accepted, preferred_scoped_project = _prefer_exact_project_for_scoped_job_query(
|
|
95
|
+
question,
|
|
96
|
+
resolver,
|
|
97
|
+
accepted,
|
|
98
|
+
named_mentions,
|
|
99
|
+
min_fuzzy_score=min_fuzzy_score,
|
|
100
|
+
)
|
|
101
|
+
ambiguity = (
|
|
102
|
+
_detect_brand_project_ambiguity(resolver, accepted, question=question)
|
|
103
|
+
if explicit_type is None and not preferred_scoped_project
|
|
104
|
+
else None
|
|
105
|
+
)
|
|
106
|
+
|
|
107
|
+
brands: dict[object, ResolvedEntity] = {}
|
|
108
|
+
projects: dict[object, ResolvedEntity] = {}
|
|
109
|
+
for mention in accepted:
|
|
110
|
+
entity = mention.entity
|
|
111
|
+
target = brands if entity.entity_type == "brand" else projects
|
|
112
|
+
target.setdefault(entity.id, entity)
|
|
113
|
+
|
|
114
|
+
masked_question = _mask_spans(_norm(question), accepted)
|
|
115
|
+
masked_question = _mask_ranges(masked_question, _store_name_ranges(masked_question))
|
|
116
|
+
location = resolver.resolve_location(masked_question, min_score=min_fuzzy_score)
|
|
117
|
+
|
|
118
|
+
candidate_ambiguity = _fuzzy_candidate_ambiguity(
|
|
119
|
+
resolver,
|
|
120
|
+
fuzzy_candidates or {},
|
|
121
|
+
explicit_type=explicit_type,
|
|
122
|
+
min_fuzzy_score=min_fuzzy_score,
|
|
123
|
+
brands=brands,
|
|
124
|
+
projects=projects,
|
|
125
|
+
)
|
|
126
|
+
if candidate_ambiguity is not None:
|
|
127
|
+
return EntityBindings(
|
|
128
|
+
brands=list(brands.values()),
|
|
129
|
+
projects=list(projects.values()),
|
|
130
|
+
location=location,
|
|
131
|
+
ambiguity=candidate_ambiguity,
|
|
132
|
+
)
|
|
133
|
+
|
|
134
|
+
_merge_fuzzy_candidates(
|
|
135
|
+
resolver,
|
|
136
|
+
fuzzy_candidates or {},
|
|
137
|
+
brands=brands,
|
|
138
|
+
projects=projects,
|
|
139
|
+
location=location,
|
|
140
|
+
min_fuzzy_score=min_fuzzy_score,
|
|
141
|
+
explicit_type=explicit_type,
|
|
142
|
+
)
|
|
143
|
+
|
|
144
|
+
return EntityBindings(
|
|
145
|
+
brands=list(brands.values()),
|
|
146
|
+
projects=list(projects.values()),
|
|
147
|
+
location=location,
|
|
148
|
+
ambiguity=ambiguity,
|
|
149
|
+
)
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def _explicit_named_entity_type(question: str) -> str | None:
|
|
153
|
+
has_brand = _has_prefix_type_marker(question, "品牌") or _has_postfix_type_marker(question, "品牌")
|
|
154
|
+
has_project = _has_prefix_type_marker(question, "项目") or _has_postfix_type_marker(question, "项目")
|
|
155
|
+
if has_brand == has_project:
|
|
156
|
+
return None
|
|
157
|
+
return "brand" if has_brand else "project"
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def _has_prefix_type_marker(question: str, marker: str) -> bool:
|
|
161
|
+
name_suffix = _FIELD_LABEL_NAME_SUFFIX
|
|
162
|
+
patterns = [
|
|
163
|
+
rf"{marker}(?:名称)?\s*[「“\"'《【((](?P<name>[A-Za-z0-9_\-\u4e00-\u9fff]{{2,40}})",
|
|
164
|
+
rf"{marker}(?:名称)?(?:为|是|=|等于|:|:)\s*[「“\"'《【((]?(?P<name>[A-Za-z0-9_\-\u4e00-\u9fff]{{2,40}})",
|
|
165
|
+
rf"{marker}\s+(?P<name>[A-Za-z0-9_\-\u4e00-\u9fff]{{2,40}})",
|
|
166
|
+
# 「项目成都你六姐」/「品牌必胜客」/「项目成都你六姐 门店:…」等写法。
|
|
167
|
+
rf"{marker}(?P<name>[A-Za-z0-9_\-\u4e00-\u9fff]{{2,40}}){name_suffix}",
|
|
168
|
+
]
|
|
169
|
+
return any(_specific_marker_match(pattern, question) for pattern in patterns)
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
_FIELD_LABEL_NAME_SUFFIX = (
|
|
173
|
+
r"(?:在(?!招)|的|下|有|还|目前|当前|现在|是|$|[,,。;;??]"
|
|
174
|
+
r"|\s+(?=门店|品牌|项目|岗位|职位|城市|区域|有哪些|有什么|多少|包括|返回|状态)"
|
|
175
|
+
r"|(?=位于|有哪些|有什么|多少|包括|返回|状态|岗位|职位|门店|品牌|项目))"
|
|
176
|
+
)
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
def _has_postfix_type_marker(question: str, marker: str) -> bool:
|
|
180
|
+
suffix = rf"(?:{marker})\s*(?:在|的|下|有|还|目前|当前|现在|是|$)"
|
|
181
|
+
for match in re.finditer(rf"(?P<name>[A-Za-z0-9_\-\u4e00-\u9fff]{{2,40}})\s*{suffix}", question):
|
|
182
|
+
if _is_specific_entity_name(match.group("name")):
|
|
183
|
+
return True
|
|
184
|
+
return False
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
def _specific_marker_match(pattern: str, question: str) -> bool:
|
|
188
|
+
for match in re.finditer(pattern, question):
|
|
189
|
+
if _is_specific_entity_name(match.group("name")):
|
|
190
|
+
return True
|
|
191
|
+
return False
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
def _is_specific_entity_name(value: str) -> bool:
|
|
195
|
+
name = re.sub(
|
|
196
|
+
r"^(帮我|请问|查询|查看|统计|搜索|看下|看一下|我想知道|想知道)",
|
|
197
|
+
"",
|
|
198
|
+
value,
|
|
199
|
+
)
|
|
200
|
+
generic_terms = {
|
|
201
|
+
"哪些",
|
|
202
|
+
"什么",
|
|
203
|
+
"所有",
|
|
204
|
+
"全部",
|
|
205
|
+
"查询",
|
|
206
|
+
"查看",
|
|
207
|
+
"招聘",
|
|
208
|
+
"在招",
|
|
209
|
+
"岗位",
|
|
210
|
+
"职位",
|
|
211
|
+
"有哪些",
|
|
212
|
+
}
|
|
213
|
+
generic_prefixes = ("在", "有哪些", "哪些", "什么", "所有", "全部", "目前", "当前", "现在")
|
|
214
|
+
return bool(name) and name not in generic_terms and not name.startswith(generic_prefixes)
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
def _detect_brand_project_ambiguity(
|
|
218
|
+
resolver: EntityResolver,
|
|
219
|
+
accepted: list[Mention],
|
|
220
|
+
*,
|
|
221
|
+
question: str = "",
|
|
222
|
+
) -> EntityAmbiguity | None:
|
|
223
|
+
for mention in accepted:
|
|
224
|
+
entity_type = mention.entity.entity_type
|
|
225
|
+
if entity_type not in _NAMED_TYPES:
|
|
226
|
+
continue
|
|
227
|
+
term = mention.text
|
|
228
|
+
if _field_marker_around_term(question, term, "项目"):
|
|
229
|
+
continue
|
|
230
|
+
if _field_marker_around_term(question, term, "品牌"):
|
|
231
|
+
continue
|
|
232
|
+
brands = resolver.find_containing(term, "brand")
|
|
233
|
+
projects = resolver.find_containing(term, "project")
|
|
234
|
+
if brands and projects:
|
|
235
|
+
return EntityAmbiguity(term=term, brands=brands, projects=projects)
|
|
236
|
+
return None
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
def _prefer_exact_project_for_scoped_job_query(
|
|
240
|
+
question: str,
|
|
241
|
+
resolver: EntityResolver,
|
|
242
|
+
accepted: list[Mention],
|
|
243
|
+
all_mentions: list[Mention],
|
|
244
|
+
*,
|
|
245
|
+
min_fuzzy_score: float,
|
|
246
|
+
) -> tuple[list[Mention], bool]:
|
|
247
|
+
if not accepted or not _has_job_query_signal(question):
|
|
248
|
+
return accepted, False
|
|
249
|
+
|
|
250
|
+
normalized = _norm(question)
|
|
251
|
+
masked_question = _mask_spans(normalized, accepted)
|
|
252
|
+
masked_question = _mask_ranges(masked_question, _store_name_ranges(masked_question))
|
|
253
|
+
has_location = bool(resolver.resolve_location(masked_question, min_score=min_fuzzy_score))
|
|
254
|
+
has_store_scope = "门店" in normalized
|
|
255
|
+
if not has_location and not has_store_scope:
|
|
256
|
+
return accepted, False
|
|
257
|
+
|
|
258
|
+
replacements: list[Mention] = []
|
|
259
|
+
changed = False
|
|
260
|
+
for mention in accepted:
|
|
261
|
+
if mention.entity.entity_type != "brand":
|
|
262
|
+
replacements.append(mention)
|
|
263
|
+
continue
|
|
264
|
+
project = _same_name_project_mention(mention, all_mentions)
|
|
265
|
+
if project is None:
|
|
266
|
+
replacements.append(mention)
|
|
267
|
+
continue
|
|
268
|
+
replacements.append(project)
|
|
269
|
+
changed = True
|
|
270
|
+
return (replacements, True) if changed else (accepted, False)
|
|
271
|
+
|
|
272
|
+
|
|
273
|
+
def _has_job_query_signal(question: str) -> bool:
|
|
274
|
+
return bool(re.search(r"岗位|职位|在招|招聘|招人", _norm(question)))
|
|
275
|
+
|
|
276
|
+
|
|
277
|
+
def _same_name_project_mention(brand_mention: Mention, all_mentions: list[Mention]) -> Mention | None:
|
|
278
|
+
brand_name = _norm(brand_mention.entity.canonical)
|
|
279
|
+
for mention in all_mentions:
|
|
280
|
+
if mention.entity.entity_type != "project":
|
|
281
|
+
continue
|
|
282
|
+
if mention.start != brand_mention.start or mention.end != brand_mention.end:
|
|
283
|
+
continue
|
|
284
|
+
if _norm(mention.entity.canonical) == brand_name:
|
|
285
|
+
return mention
|
|
286
|
+
return None
|
|
287
|
+
|
|
288
|
+
|
|
289
|
+
def _field_marker_around_term(question: str, term: str, marker: str) -> bool:
|
|
290
|
+
return _field_marker_before_term(question, term, marker) or _field_marker_after_term(question, term, marker)
|
|
291
|
+
|
|
292
|
+
|
|
293
|
+
def _field_marker_before_term(question: str, term: str, marker: str) -> bool:
|
|
294
|
+
"""True when ``term`` is introduced by an explicit field label like ``项目:``."""
|
|
295
|
+
normalized = _norm(question)
|
|
296
|
+
term_key = _norm(term)
|
|
297
|
+
start = normalized.find(term_key)
|
|
298
|
+
if start <= 0:
|
|
299
|
+
return False
|
|
300
|
+
prefix = normalized[:start]
|
|
301
|
+
if prefix.endswith(marker):
|
|
302
|
+
return True
|
|
303
|
+
return bool(re.search(rf"{marker}(?:名称)?(?:[::]\s*|\s+)$", prefix))
|
|
304
|
+
|
|
305
|
+
|
|
306
|
+
def _field_marker_after_term(question: str, term: str, marker: str) -> bool:
|
|
307
|
+
normalized = _norm(question)
|
|
308
|
+
term_key = re.escape(_norm(term))
|
|
309
|
+
if not term_key:
|
|
310
|
+
return False
|
|
311
|
+
return bool(
|
|
312
|
+
re.search(
|
|
313
|
+
rf"{term_key}\s*{marker}\s*(?:在|的|下|有|还|目前|当前|现在|是|$|[,,。;;??\s])",
|
|
314
|
+
normalized,
|
|
315
|
+
)
|
|
316
|
+
)
|
|
317
|
+
|
|
318
|
+
|
|
319
|
+
def _fuzzy_candidate_ambiguity(
|
|
320
|
+
resolver: EntityResolver,
|
|
321
|
+
fuzzy_candidates: dict[str, list[str]],
|
|
322
|
+
*,
|
|
323
|
+
explicit_type: str | None,
|
|
324
|
+
min_fuzzy_score: float,
|
|
325
|
+
brands: dict[object, ResolvedEntity] | None = None,
|
|
326
|
+
projects: dict[object, ResolvedEntity] | None = None,
|
|
327
|
+
) -> EntityAmbiguity | None:
|
|
328
|
+
bound_brands = brands or {}
|
|
329
|
+
bound_projects = projects or {}
|
|
330
|
+
for entity_type, spans in fuzzy_candidates.items():
|
|
331
|
+
if entity_type not in _NAMED_TYPES:
|
|
332
|
+
continue
|
|
333
|
+
if explicit_type in _NAMED_TYPES and entity_type != explicit_type:
|
|
334
|
+
continue
|
|
335
|
+
if explicit_type == "brand" and bound_brands:
|
|
336
|
+
continue
|
|
337
|
+
if explicit_type == "project" and bound_projects:
|
|
338
|
+
continue
|
|
339
|
+
for span in spans:
|
|
340
|
+
if _is_location_like_entity_span(span):
|
|
341
|
+
continue
|
|
342
|
+
direct = resolver.resolve(span, entity_type, limit=2, min_score=min_fuzzy_score)
|
|
343
|
+
if len(direct) == 1 and direct[0].matched_via in {"exact", "alias"}:
|
|
344
|
+
continue
|
|
345
|
+
candidates = resolver.find_containing(span, entity_type)
|
|
346
|
+
if not candidates:
|
|
347
|
+
candidates = resolver.resolve(span, entity_type, limit=10, min_score=min_fuzzy_score)
|
|
348
|
+
if entity_type == "brand":
|
|
349
|
+
return EntityAmbiguity(term=span, brands=candidates, projects=[])
|
|
350
|
+
return EntityAmbiguity(term=span, brands=[], projects=candidates)
|
|
351
|
+
return None
|
|
352
|
+
|
|
353
|
+
|
|
354
|
+
def _is_location_like_entity_span(span: str) -> bool:
|
|
355
|
+
from .sql_generator import _is_location_like_entity_name
|
|
356
|
+
|
|
357
|
+
return _is_location_like_entity_name(re.sub(r"\s+", "", span))
|
|
358
|
+
|
|
359
|
+
|
|
360
|
+
def _resolve_overlaps(mentions: Iterable[Mention]) -> list[Mention]:
|
|
361
|
+
ordered = sorted(
|
|
362
|
+
mentions,
|
|
363
|
+
key=lambda mention: (
|
|
364
|
+
mention.end - mention.start,
|
|
365
|
+
_TYPE_PRIORITY.get(mention.entity.entity_type, 0),
|
|
366
|
+
-mention.start,
|
|
367
|
+
),
|
|
368
|
+
reverse=True,
|
|
369
|
+
)
|
|
370
|
+
accepted: list[Mention] = []
|
|
371
|
+
for mention in ordered:
|
|
372
|
+
if any(_overlaps(mention, other) for other in accepted):
|
|
373
|
+
continue
|
|
374
|
+
accepted.append(mention)
|
|
375
|
+
return sorted(accepted, key=lambda mention: mention.start)
|
|
376
|
+
|
|
377
|
+
|
|
378
|
+
def _overlaps(a: Mention, b: Mention) -> bool:
|
|
379
|
+
return a.start < b.end and b.start < a.end
|
|
380
|
+
|
|
381
|
+
|
|
382
|
+
def _mask_spans(text: str, mentions: Iterable[Mention]) -> str:
|
|
383
|
+
return _mask_ranges(text, ((mention.start, mention.end) for mention in mentions))
|
|
384
|
+
|
|
385
|
+
|
|
386
|
+
def _mask_ranges(text: str, ranges: Iterable[tuple[int, int]]) -> str:
|
|
387
|
+
chars = list(text)
|
|
388
|
+
for start, end in ranges:
|
|
389
|
+
for index in range(start, min(end, len(chars))):
|
|
390
|
+
chars[index] = _MASK_CHAR
|
|
391
|
+
return "".join(chars)
|
|
392
|
+
|
|
393
|
+
|
|
394
|
+
def _store_name_ranges(normalized_question: str) -> list[tuple[int, int]]:
|
|
395
|
+
"""Character spans of explicit store names that must not feed geo resolution."""
|
|
396
|
+
patterns = [
|
|
397
|
+
rf"门店(?:名称)?(?:为|是|=|等于|:|:)\s*(?P<name>{_STORE_NAME_CHARS}){_STORE_NAME_STOP}",
|
|
398
|
+
rf"门店{_STORE_SEPARATOR}(?P<name>{_STORE_NAME_CHARS}){_STORE_NAME_STOP}",
|
|
399
|
+
rf"(?P<name>{_STORE_NAME_CHARS})的门店{_STORE_NAME_STOP}",
|
|
400
|
+
# Prefix marker only; skip the suffix marker in 「…的门店」.
|
|
401
|
+
rf"(?<!的)(?<![店])门店(?P<name>{_STORE_NAME_CHARS}){_STORE_NAME_STOP}",
|
|
402
|
+
]
|
|
403
|
+
ranges: list[tuple[int, int]] = []
|
|
404
|
+
for pattern in patterns:
|
|
405
|
+
for match in re.finditer(pattern, normalized_question):
|
|
406
|
+
name = _clean_store_name(match.group("name"))
|
|
407
|
+
if name is None:
|
|
408
|
+
continue
|
|
409
|
+
start = match.start("name")
|
|
410
|
+
ranges.append((start, start + len(name)))
|
|
411
|
+
return _merge_ranges(ranges)
|
|
412
|
+
|
|
413
|
+
|
|
414
|
+
def _clean_store_name(value: str) -> str | None:
|
|
415
|
+
name = re.sub(r"\s+", "", value)
|
|
416
|
+
name = name.strip("\"'“”‘’%::,,、;;")
|
|
417
|
+
forbidden = ("哪些", "什么", "所有", "全部", "查询", "查看", "门店", "岗位", "职位")
|
|
418
|
+
if name.startswith(("在", "位于", "有哪些", "有什么", "多少")):
|
|
419
|
+
return None
|
|
420
|
+
if 2 <= len(name) <= 40 and name not in forbidden:
|
|
421
|
+
return name
|
|
422
|
+
return None
|
|
423
|
+
|
|
424
|
+
|
|
425
|
+
def _merge_ranges(ranges: list[tuple[int, int]]) -> list[tuple[int, int]]:
|
|
426
|
+
if not ranges:
|
|
427
|
+
return []
|
|
428
|
+
ordered = sorted(ranges)
|
|
429
|
+
merged: list[tuple[int, int]] = [ordered[0]]
|
|
430
|
+
for start, end in ordered[1:]:
|
|
431
|
+
last_start, last_end = merged[-1]
|
|
432
|
+
if start <= last_end:
|
|
433
|
+
merged[-1] = (last_start, max(last_end, end))
|
|
434
|
+
else:
|
|
435
|
+
merged.append((start, end))
|
|
436
|
+
return merged
|
|
437
|
+
|
|
438
|
+
|
|
439
|
+
def _merge_fuzzy_candidates(
|
|
440
|
+
resolver: EntityResolver,
|
|
441
|
+
fuzzy_candidates: dict[str, list[str]],
|
|
442
|
+
*,
|
|
443
|
+
brands: dict[object, ResolvedEntity],
|
|
444
|
+
projects: dict[object, ResolvedEntity],
|
|
445
|
+
location: dict[str, ResolvedEntity],
|
|
446
|
+
min_fuzzy_score: float,
|
|
447
|
+
explicit_type: str | None = None,
|
|
448
|
+
) -> None:
|
|
449
|
+
for entity_type, spans in fuzzy_candidates.items():
|
|
450
|
+
if explicit_type in _NAMED_TYPES and entity_type != explicit_type:
|
|
451
|
+
continue
|
|
452
|
+
for span in spans:
|
|
453
|
+
if _is_location_like_entity_span(span):
|
|
454
|
+
continue
|
|
455
|
+
if entity_type in _GEO_TYPES or entity_type == "location":
|
|
456
|
+
if location:
|
|
457
|
+
continue
|
|
458
|
+
resolved = resolver.resolve_location(span, min_score=min_fuzzy_score)
|
|
459
|
+
for level, entity in resolved.items():
|
|
460
|
+
location.setdefault(level, entity)
|
|
461
|
+
continue
|
|
462
|
+
results = resolver.resolve(span, entity_type, limit=1, min_score=min_fuzzy_score)
|
|
463
|
+
if results and results[0].matched_via not in {"exact", "alias"}:
|
|
464
|
+
continue
|
|
465
|
+
if not results:
|
|
466
|
+
continue
|
|
467
|
+
target = brands if entity_type == "brand" else projects if entity_type == "project" else None
|
|
468
|
+
if target is not None:
|
|
469
|
+
for entity in results:
|
|
470
|
+
target.setdefault(entity.id, entity)
|
|
471
|
+
|
|
472
|
+
|
|
473
|
+
def _entity_log(entity: ResolvedEntity) -> dict[str, object]:
|
|
474
|
+
return {
|
|
475
|
+
"type": entity.entity_type,
|
|
476
|
+
"id": entity.id,
|
|
477
|
+
"canonical": entity.canonical,
|
|
478
|
+
"via": entity.matched_via,
|
|
479
|
+
"score": entity.score,
|
|
480
|
+
}
|
|
481
|
+
|
|
482
|
+
|
|
483
|
+
def missing_entities_in_sql(sql: str, resolved_entities: dict[str, Any] | None) -> list[str]:
|
|
484
|
+
"""Resolved entities that have NO trace at all in the SQL.
|
|
485
|
+
|
|
486
|
+
Deliberately a presence check, not a form check: an entity counts as
|
|
487
|
+
present if its id, canonical name, or (for geo) name core appears anywhere
|
|
488
|
+
in the SQL — regardless of whether it is used as ``id =``, ``name LIKE``, a
|
|
489
|
+
JOIN, or a subquery. This keeps false positives near-zero: only an entity
|
|
490
|
+
that was completely dropped is reported. A coincidental id collision causes
|
|
491
|
+
a (safe) false negative, never a false positive.
|
|
492
|
+
"""
|
|
493
|
+
if not resolved_entities or not sql:
|
|
494
|
+
return []
|
|
495
|
+
missing: list[str] = []
|
|
496
|
+
for brand in resolved_entities.get("brands") or []:
|
|
497
|
+
if not _entity_present(sql, brand):
|
|
498
|
+
missing.append(f"品牌「{brand.get('name')}」")
|
|
499
|
+
for project in resolved_entities.get("projects") or []:
|
|
500
|
+
if not _entity_present(sql, project):
|
|
501
|
+
missing.append(f"项目「{project.get('name')}」")
|
|
502
|
+
location = resolved_entities.get("location") or {}
|
|
503
|
+
finest = location.get("region") or location.get("city") or location.get("province")
|
|
504
|
+
if finest and not _geo_present(sql, finest):
|
|
505
|
+
missing.append(f"地点「{finest.get('name')}」")
|
|
506
|
+
return missing
|
|
507
|
+
|
|
508
|
+
|
|
509
|
+
def _entity_present(sql: str, entity: dict[str, Any]) -> bool:
|
|
510
|
+
name = str(entity.get("name") or "")
|
|
511
|
+
if name and name in sql:
|
|
512
|
+
return True
|
|
513
|
+
entity_id = entity.get("id")
|
|
514
|
+
if entity_id is not None and re.search(rf"\b{re.escape(str(entity_id))}\b", sql):
|
|
515
|
+
return True
|
|
516
|
+
return False
|
|
517
|
+
|
|
518
|
+
|
|
519
|
+
def _geo_present(sql: str, entity: dict[str, Any]) -> bool:
|
|
520
|
+
if _entity_present(sql, entity):
|
|
521
|
+
return True
|
|
522
|
+
core = _geo_core(_norm(str(entity.get("name") or "")))
|
|
523
|
+
return bool(core) and core in sql.lower()
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import time
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import Any
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class EntityDictionaryCacheError(RuntimeError):
|
|
11
|
+
"""Raised when the persisted entity dictionary cache cannot be used."""
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
@dataclass
|
|
15
|
+
class EntityDictionaryCache:
|
|
16
|
+
"""Persists the entity dictionary snapshot returned by the MCP server.
|
|
17
|
+
|
|
18
|
+
Mirrors ``SchemaCache``: the payload is a JSON object that carries a
|
|
19
|
+
``dictVersion`` field used for incremental refresh negotiation.
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
path: Path
|
|
23
|
+
|
|
24
|
+
def load(self) -> dict[str, Any] | None:
|
|
25
|
+
if not self.path.exists():
|
|
26
|
+
return None
|
|
27
|
+
try:
|
|
28
|
+
data = json.loads(self.path.read_text(encoding="utf-8"))
|
|
29
|
+
except json.JSONDecodeError as exc:
|
|
30
|
+
raise EntityDictionaryCacheError(
|
|
31
|
+
f"Invalid entity dictionary cache JSON: {self.path}"
|
|
32
|
+
) from exc
|
|
33
|
+
if not isinstance(data, dict):
|
|
34
|
+
raise EntityDictionaryCacheError(
|
|
35
|
+
f"Entity dictionary cache must be a JSON object: {self.path}"
|
|
36
|
+
)
|
|
37
|
+
return data
|
|
38
|
+
|
|
39
|
+
def save(self, dictionary: dict[str, Any]) -> None:
|
|
40
|
+
if not isinstance(dictionary, dict):
|
|
41
|
+
raise TypeError("dictionary must be a dict")
|
|
42
|
+
self.path.parent.mkdir(parents=True, exist_ok=True)
|
|
43
|
+
self.path.write_text(
|
|
44
|
+
json.dumps(dictionary, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
|
|
45
|
+
encoding="utf-8",
|
|
46
|
+
)
|
|
47
|
+
|
|
48
|
+
def version(self) -> str | None:
|
|
49
|
+
dictionary = self.load()
|
|
50
|
+
if dictionary is None:
|
|
51
|
+
return None
|
|
52
|
+
version = dictionary.get("dictVersion")
|
|
53
|
+
return str(version) if version is not None else None
|
|
54
|
+
|
|
55
|
+
def is_fresh(self, max_age_minutes: int) -> bool:
|
|
56
|
+
if max_age_minutes <= 0 or not self.path.exists():
|
|
57
|
+
return False
|
|
58
|
+
age_seconds = time.time() - self.path.stat().st_mtime
|
|
59
|
+
return age_seconds <= max_age_minutes * 60
|