@nikeandocean/carbon-factor-matcher 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +252 -0
- package/index.js +28 -0
- package/package.json +41 -0
- package/src/carbon_factor_matcher/__init__.py +3 -0
- package/src/carbon_factor_matcher/__main__.py +5 -0
- package/src/carbon_factor_matcher/adapters/__init__.py +6 -0
- package/src/carbon_factor_matcher/adapters/base.py +22 -0
- package/src/carbon_factor_matcher/adapters/ecoinvent.py +292 -0
- package/src/carbon_factor_matcher/api_client.py +94 -0
- package/src/carbon_factor_matcher/config.py +14 -0
- package/src/carbon_factor_matcher/factor_db.py +220 -0
- package/src/carbon_factor_matcher/license.py +80 -0
- package/src/carbon_factor_matcher/matcher.py +795 -0
- package/src/carbon_factor_matcher/models.py +71 -0
- package/src/carbon_factor_matcher/server.py +232 -0
- package/src/carbon_factor_matcher/usage_tracker.py +68 -0
|
@@ -0,0 +1,795 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Two-stage emission factor matching: hybrid retrieval + quality rating + LLM ranking.
|
|
3
|
+
|
|
4
|
+
Stage 0: Unit pre-filter (hard constraint)
|
|
5
|
+
Stage 1: Hybrid retrieval - keyword (0.3) + embedding (0.7)
|
|
6
|
+
Stage 2: Quality rating + LLM fine ranking
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
import logging
|
|
10
|
+
import math
|
|
11
|
+
import os
|
|
12
|
+
import re
|
|
13
|
+
from datetime import datetime
|
|
14
|
+
from typing import Optional
|
|
15
|
+
|
|
16
|
+
from openai import OpenAI
|
|
17
|
+
|
|
18
|
+
from .config import LLMConfig
|
|
19
|
+
from .factor_db import FactorDatabase
|
|
20
|
+
from .models import Activity, Factor, MatchResult
|
|
21
|
+
|
|
22
|
+
logger = logging.getLogger(__name__)
|
|
23
|
+
|
|
24
|
+
# Default weights for hybrid retrieval
|
|
25
|
+
KEYWORD_WEIGHT = 0.3
|
|
26
|
+
EMBEDDING_WEIGHT = 0.7
|
|
27
|
+
|
|
28
|
+
# Quality rating dimensions and weights (data quality assessment)
|
|
29
|
+
QUALITY_WEIGHTS = {
|
|
30
|
+
"tech_representativeness": 0.3, # 技术代表性
|
|
31
|
+
"geo_representativeness": 0.1, # 地理代表性
|
|
32
|
+
"source_reliability": 0.4, # 来源可靠性
|
|
33
|
+
"time_representativeness": 0.1, # 时间代表性
|
|
34
|
+
"factor_type": 0.1, # 因子类型
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
# Scoring criteria for quality dimensions
|
|
38
|
+
TECH_SCORES = {"exact_match": 1, "type_match": 2, "unrelated": 3}
|
|
39
|
+
GEO_SCORES = {"provincial": 1, "national": 2, "international": 3}
|
|
40
|
+
SOURCE_SCORES = {"measured": 1, "estimated": 2, "unknown": 3}
|
|
41
|
+
TIME_SCORES = {"recent": 1, "moderate": 2, "old": 3}
|
|
42
|
+
FACTOR_TYPE_SCORES = {"lca": 1, "process": 2, "unknown": 3}
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
class EmissionFactorMatcher:
|
|
46
|
+
"""Two-stage emission factor matcher.
|
|
47
|
+
|
|
48
|
+
Stage 0: Unit pre-filter (hard constraint)
|
|
49
|
+
Stage 1: Hybrid retrieval - keyword (0.3) + embedding (0.7)
|
|
50
|
+
Stage 2: Quality rating + LLM fine ranking
|
|
51
|
+
|
|
52
|
+
Args:
|
|
53
|
+
factors: List of Factor to search.
|
|
54
|
+
config: LLMConfig instance. If None, uses embedding_model/chat_model
|
|
55
|
+
parameters or defaults.
|
|
56
|
+
embedding_model: Name of local sentence embedding model
|
|
57
|
+
(default: shibing624/text2vec-base-chinese).
|
|
58
|
+
chat_model: Name of chat model for re-ranking (default: deepseek-v4-flash).
|
|
59
|
+
"""
|
|
60
|
+
|
|
61
|
+
def __init__(
|
|
62
|
+
self,
|
|
63
|
+
factors: list[Factor],
|
|
64
|
+
config: LLMConfig = None,
|
|
65
|
+
embedding_model: str = None,
|
|
66
|
+
chat_model: str = None,
|
|
67
|
+
skip_embedding: bool = False,
|
|
68
|
+
):
|
|
69
|
+
self.factors = factors
|
|
70
|
+
self._by_id = {f.id: f for f in factors}
|
|
71
|
+
|
|
72
|
+
# Use config if provided, otherwise use parameters or defaults
|
|
73
|
+
if config is None:
|
|
74
|
+
config = LLMConfig(
|
|
75
|
+
embedding_model=embedding_model or "shibing624/text2vec-base-chinese",
|
|
76
|
+
model=chat_model or "deepseek-v4-flash",
|
|
77
|
+
)
|
|
78
|
+
|
|
79
|
+
self._config = config
|
|
80
|
+
self.embedding_model_name = config.embedding_model
|
|
81
|
+
self.chat_model = config.model
|
|
82
|
+
|
|
83
|
+
# Initialize OpenAI client for LLM (DeepSeek)
|
|
84
|
+
self.llm_client = OpenAI(
|
|
85
|
+
api_key=config.api_key or os.getenv("DeepSeek_API_KEY", "sk-placeholder"),
|
|
86
|
+
base_url=config.base_url,
|
|
87
|
+
)
|
|
88
|
+
|
|
89
|
+
# Initialize local sentence embedding model
|
|
90
|
+
self._embedding_model = None
|
|
91
|
+
if not skip_embedding:
|
|
92
|
+
self._init_embedding_model()
|
|
93
|
+
|
|
94
|
+
# Build factor texts (lightweight, no embedding computation)
|
|
95
|
+
self._factor_texts = [self._build_factor_text(f) for f in factors]
|
|
96
|
+
# Cache for on-demand embeddings (populated during hybrid_search)
|
|
97
|
+
self._embedding_cache: dict[int, list[float]] = {}
|
|
98
|
+
|
|
99
|
+
def _init_embedding_model(self):
|
|
100
|
+
"""Initialize local sentence embedding model.
|
|
101
|
+
|
|
102
|
+
Tries local cache first (no network). Falls back to online download.
|
|
103
|
+
"""
|
|
104
|
+
try:
|
|
105
|
+
from sentence_transformers import SentenceTransformer
|
|
106
|
+
except ImportError:
|
|
107
|
+
logger.warning("sentence-transformers not installed, embedding disabled")
|
|
108
|
+
self._embedding_model = None
|
|
109
|
+
return
|
|
110
|
+
|
|
111
|
+
# Try local cache first — no network calls, instant load
|
|
112
|
+
try:
|
|
113
|
+
self._embedding_model = SentenceTransformer(
|
|
114
|
+
self.embedding_model_name,
|
|
115
|
+
device="cpu",
|
|
116
|
+
local_files_only=True,
|
|
117
|
+
)
|
|
118
|
+
logger.info(f"Loaded embedding model from cache: {self.embedding_model_name}")
|
|
119
|
+
return
|
|
120
|
+
except Exception:
|
|
121
|
+
logger.info("Model not in cache, downloading from HuggingFace...")
|
|
122
|
+
|
|
123
|
+
# Fallback: download from HuggingFace
|
|
124
|
+
try:
|
|
125
|
+
self._embedding_model = SentenceTransformer(
|
|
126
|
+
self.embedding_model_name,
|
|
127
|
+
device="cpu",
|
|
128
|
+
)
|
|
129
|
+
logger.info(f"Downloaded embedding model: {self.embedding_model_name}")
|
|
130
|
+
except Exception as e:
|
|
131
|
+
logger.warning(f"Failed to load embedding model: {e}")
|
|
132
|
+
self._embedding_model = None
|
|
133
|
+
|
|
134
|
+
def _compute_factor_embeddings(self) -> list[list[float]]:
|
|
135
|
+
"""Pre-compute embeddings for all factors."""
|
|
136
|
+
if self._embedding_model is None:
|
|
137
|
+
return [[] for _ in self.factors]
|
|
138
|
+
|
|
139
|
+
try:
|
|
140
|
+
embeddings = self._embedding_model.encode(
|
|
141
|
+
self._factor_texts,
|
|
142
|
+
batch_size=32,
|
|
143
|
+
show_progress_bar=False,
|
|
144
|
+
)
|
|
145
|
+
return [emb.tolist() for emb in embeddings]
|
|
146
|
+
except Exception as e:
|
|
147
|
+
logger.warning(f"Failed to compute factor embeddings: {e}")
|
|
148
|
+
return [[] for _ in self.factors]
|
|
149
|
+
|
|
150
|
+
def _build_factor_text(self, factor: Factor) -> str:
|
|
151
|
+
"""Build searchable text from factor fields."""
|
|
152
|
+
parts = [
|
|
153
|
+
factor.name,
|
|
154
|
+
factor.category,
|
|
155
|
+
factor.applicability,
|
|
156
|
+
factor.geography.get("location", factor.geography.get("country", "")),
|
|
157
|
+
]
|
|
158
|
+
return " ".join(p for p in parts if p)
|
|
159
|
+
|
|
160
|
+
def _unit_match(self, factor: Factor, target_unit: str) -> bool:
|
|
161
|
+
"""Hard constraint: check if factor unit matches target unit."""
|
|
162
|
+
if not target_unit:
|
|
163
|
+
return True
|
|
164
|
+
|
|
165
|
+
factor_unit = factor.unit.lower().strip()
|
|
166
|
+
target = target_unit.lower().strip()
|
|
167
|
+
|
|
168
|
+
if factor_unit == target:
|
|
169
|
+
return True
|
|
170
|
+
|
|
171
|
+
unit_aliases = {
|
|
172
|
+
"kg co2e": ["kg co2 eq", "kg co2-eq", "kg二氧化碳当量"],
|
|
173
|
+
"t co2e": ["t co2 eq", "t co2-eq", "ton co2e", "吨二氧化碳当量"],
|
|
174
|
+
"kg": ["千克", "公斤"],
|
|
175
|
+
"g": ["克"],
|
|
176
|
+
"t": ["ton", "tonne", "吨"],
|
|
177
|
+
"kwh": ["千瓦时", "度"],
|
|
178
|
+
"mj": ["兆焦"],
|
|
179
|
+
"m3": ["立方米", "立方"],
|
|
180
|
+
"m2": ["平方米", "平米"],
|
|
181
|
+
"l": ["liter", "litre", "升"],
|
|
182
|
+
"t*km": ["吨公里", "吨千米"],
|
|
183
|
+
"kbq": ["千贝克勒尔"],
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
for canonical, aliases in unit_aliases.items():
|
|
187
|
+
if target == canonical or target in aliases:
|
|
188
|
+
return factor_unit == canonical or factor_unit in aliases
|
|
189
|
+
|
|
190
|
+
return False
|
|
191
|
+
|
|
192
|
+
def _keyword_score(self, factor: Factor, query: str) -> float:
|
|
193
|
+
"""Compute keyword match score using exact and partial matching."""
|
|
194
|
+
query_lower = query.lower()
|
|
195
|
+
factor_text = self._build_factor_text(factor).lower()
|
|
196
|
+
|
|
197
|
+
if query_lower == factor.name.lower():
|
|
198
|
+
return 1.0
|
|
199
|
+
|
|
200
|
+
if query_lower in factor_text:
|
|
201
|
+
return 0.8
|
|
202
|
+
|
|
203
|
+
query_terms = query_lower.split()
|
|
204
|
+
if not query_terms:
|
|
205
|
+
return 0.0
|
|
206
|
+
|
|
207
|
+
matched = sum(1 for t in query_terms if t in factor_text)
|
|
208
|
+
return matched / len(query_terms)
|
|
209
|
+
|
|
210
|
+
def _compute_embedding_similarity(
|
|
211
|
+
self, query: str, factor_idx: int
|
|
212
|
+
) -> float:
|
|
213
|
+
"""Compute embedding similarity between query and factor (uses cache)."""
|
|
214
|
+
if self._embedding_model is None:
|
|
215
|
+
return 0.0
|
|
216
|
+
|
|
217
|
+
factor_emb = self._embedding_cache.get(factor_idx)
|
|
218
|
+
if not factor_emb:
|
|
219
|
+
return 0.0
|
|
220
|
+
|
|
221
|
+
try:
|
|
222
|
+
query_emb = self._embedding_model.encode([query])[0].tolist()
|
|
223
|
+
return self._cosine_similarity(query_emb, factor_emb)
|
|
224
|
+
except Exception:
|
|
225
|
+
return 0.0
|
|
226
|
+
|
|
227
|
+
def _batch_encode_factors(self, indices: list[int]) -> None:
|
|
228
|
+
"""Batch-encode factors and cache their embeddings."""
|
|
229
|
+
if self._embedding_model is None:
|
|
230
|
+
return
|
|
231
|
+
uncached = [i for i in indices if i not in self._embedding_cache]
|
|
232
|
+
if not uncached:
|
|
233
|
+
return
|
|
234
|
+
texts = [self._factor_texts[i] for i in uncached]
|
|
235
|
+
embeddings = self._embedding_model.encode(
|
|
236
|
+
texts, batch_size=64, show_progress_bar=False,
|
|
237
|
+
)
|
|
238
|
+
for idx, emb in zip(uncached, embeddings):
|
|
239
|
+
self._embedding_cache[idx] = emb.tolist()
|
|
240
|
+
|
|
241
|
+
def _cosine_similarity(self, a: list[float], b: list[float]) -> float:
|
|
242
|
+
"""Compute cosine similarity between two vectors."""
|
|
243
|
+
dot = sum(x * y for x, y in zip(a, b))
|
|
244
|
+
norm_a = math.sqrt(sum(x * x for x in a))
|
|
245
|
+
norm_b = math.sqrt(sum(x * x for x in b))
|
|
246
|
+
if norm_a == 0 or norm_b == 0:
|
|
247
|
+
return 0.0
|
|
248
|
+
return dot / (norm_a * norm_b)
|
|
249
|
+
|
|
250
|
+
def _compute_quality_ratings(self, factor: Factor, query: str) -> dict:
|
|
251
|
+
"""Compute quality ratings based on the 5-dimension assessment system."""
|
|
252
|
+
current_year = datetime.now().year
|
|
253
|
+
ratings = {}
|
|
254
|
+
|
|
255
|
+
# 1. 技术代表性 (Technology Representativeness)
|
|
256
|
+
query_lower = query.lower()
|
|
257
|
+
factor_name = factor.name.lower()
|
|
258
|
+
factor_applicability = (factor.applicability or "").lower()
|
|
259
|
+
|
|
260
|
+
if query_lower in factor_name or factor_name in query_lower:
|
|
261
|
+
ratings["tech_representativeness"] = TECH_SCORES["exact_match"]
|
|
262
|
+
elif any(t in factor_applicability for t in query_lower.split()):
|
|
263
|
+
ratings["tech_representativeness"] = TECH_SCORES["type_match"]
|
|
264
|
+
else:
|
|
265
|
+
ratings["tech_representativeness"] = TECH_SCORES["unrelated"]
|
|
266
|
+
|
|
267
|
+
# 2. 地理代表性 (Geographic Representativeness)
|
|
268
|
+
location = factor.geography.get("location", "").lower()
|
|
269
|
+
if any(kw in location for kw in ["省", "市", "区", "province", "city"]):
|
|
270
|
+
ratings["geo_representativeness"] = GEO_SCORES["provincial"]
|
|
271
|
+
elif any(kw in location for kw in ["中国", "china", "美国", "usa", "欧洲", "europe"]):
|
|
272
|
+
ratings["geo_representativeness"] = GEO_SCORES["national"]
|
|
273
|
+
else:
|
|
274
|
+
ratings["geo_representativeness"] = GEO_SCORES["international"]
|
|
275
|
+
|
|
276
|
+
# 3. 来源可靠性 (Source Reliability)
|
|
277
|
+
source_lower = factor.source.lower()
|
|
278
|
+
if any(kw in source_lower for kw in ["实测", "测量", "实绩", "measured", "actual"]):
|
|
279
|
+
ratings["source_reliability"] = SOURCE_SCORES["measured"]
|
|
280
|
+
elif any(kw in source_lower for kw in ["估算", "计算", "模型", "estimated", "modeled"]):
|
|
281
|
+
ratings["source_reliability"] = SOURCE_SCORES["estimated"]
|
|
282
|
+
else:
|
|
283
|
+
ratings["source_reliability"] = SOURCE_SCORES["unknown"]
|
|
284
|
+
|
|
285
|
+
# 4. 时间代表性 (Time Representativeness)
|
|
286
|
+
year = factor.source_year
|
|
287
|
+
if year == 0:
|
|
288
|
+
ratings["time_representativeness"] = TIME_SCORES["old"]
|
|
289
|
+
else:
|
|
290
|
+
age = current_year - year
|
|
291
|
+
if age <= 3:
|
|
292
|
+
ratings["time_representativeness"] = TIME_SCORES["recent"]
|
|
293
|
+
elif age <= 10:
|
|
294
|
+
ratings["time_representativeness"] = TIME_SCORES["moderate"]
|
|
295
|
+
else:
|
|
296
|
+
ratings["time_representativeness"] = TIME_SCORES["old"]
|
|
297
|
+
|
|
298
|
+
# 5. 因子类型 (Factor Type)
|
|
299
|
+
category = factor.category.lower()
|
|
300
|
+
if any(kw in category for kw in ["lca", "生命周期", "life cycle"]):
|
|
301
|
+
ratings["factor_type"] = FACTOR_TYPE_SCORES["lca"]
|
|
302
|
+
elif any(kw in category for kw in ["过程", "process", "清单", "inventory"]):
|
|
303
|
+
ratings["factor_type"] = FACTOR_TYPE_SCORES["process"]
|
|
304
|
+
else:
|
|
305
|
+
ratings["factor_type"] = FACTOR_TYPE_SCORES["unknown"]
|
|
306
|
+
|
|
307
|
+
return ratings
|
|
308
|
+
|
|
309
|
+
def _compute_quality_score(self, quality_ratings: dict) -> float:
|
|
310
|
+
"""Compute weighted quality score from dimension ratings.
|
|
311
|
+
|
|
312
|
+
Lower rating = better quality, so we invert the score.
|
|
313
|
+
Score = 1 - (weighted_average / 3)
|
|
314
|
+
"""
|
|
315
|
+
weighted_sum = sum(
|
|
316
|
+
quality_ratings[dim] * weight
|
|
317
|
+
for dim, weight in QUALITY_WEIGHTS.items()
|
|
318
|
+
)
|
|
319
|
+
return 1.0 - (weighted_sum - 1) / 2.0
|
|
320
|
+
|
|
321
|
+
def _build_quality_description(self, quality_ratings: dict) -> str:
|
|
322
|
+
"""Build human-readable quality description."""
|
|
323
|
+
dim_names = {
|
|
324
|
+
"tech_representativeness": "技术代表性",
|
|
325
|
+
"geo_representativeness": "地理代表性",
|
|
326
|
+
"source_reliability": "来源可靠性",
|
|
327
|
+
"time_representativeness": "时间代表性",
|
|
328
|
+
"factor_type": "因子类型",
|
|
329
|
+
}
|
|
330
|
+
level_labels = {1: "优", 2: "中", 3: "差"}
|
|
331
|
+
|
|
332
|
+
parts = []
|
|
333
|
+
for dim, rating in quality_ratings.items():
|
|
334
|
+
name = dim_names.get(dim, dim)
|
|
335
|
+
level = level_labels.get(rating, "未知")
|
|
336
|
+
parts.append(f"{name}: {level}({rating})")
|
|
337
|
+
|
|
338
|
+
return " | ".join(parts)
|
|
339
|
+
|
|
340
|
+
def _format_factor_for_llm(
|
|
341
|
+
self, idx: int, factor: Factor, quality_ratings: dict
|
|
342
|
+
) -> str:
|
|
343
|
+
"""Format a single factor for LLM re-ranking prompt."""
|
|
344
|
+
quality_desc = self._build_quality_description(quality_ratings)
|
|
345
|
+
return f"""[{idx}] {factor.name}
|
|
346
|
+
分类: {factor.category}
|
|
347
|
+
适用性: {factor.applicability or 'N/A'}
|
|
348
|
+
地理: {factor.geography.get('location', 'N/A')}
|
|
349
|
+
单位: {factor.unit}
|
|
350
|
+
数值: {factor.value}
|
|
351
|
+
来源: {factor.source} ({factor.source_year})
|
|
352
|
+
数据质量: {quality_desc}"""
|
|
353
|
+
|
|
354
|
+
def search(self, query: str) -> list[Factor]:
|
|
355
|
+
"""Simple keyword search across factor name, category, applicability."""
|
|
356
|
+
query_lower = query.lower()
|
|
357
|
+
return [
|
|
358
|
+
f for f in self.factors
|
|
359
|
+
if query_lower in f.name.lower()
|
|
360
|
+
or query_lower in f.category.lower()
|
|
361
|
+
or query_lower in f.applicability.lower()
|
|
362
|
+
]
|
|
363
|
+
|
|
364
|
+
def get_by_id(self, factor_id: str) -> Factor | None:
|
|
365
|
+
"""Get a factor by ID."""
|
|
366
|
+
return self._by_id.get(factor_id)
|
|
367
|
+
|
|
368
|
+
def hybrid_search(
|
|
369
|
+
self,
|
|
370
|
+
query: str,
|
|
371
|
+
target_unit: str = "",
|
|
372
|
+
top_k: int = 20,
|
|
373
|
+
keyword_top_n: int = 200,
|
|
374
|
+
) -> list[dict]:
|
|
375
|
+
"""Stage 0 + Stage 1: Unit pre-filter + Hybrid retrieval.
|
|
376
|
+
|
|
377
|
+
Two-stage optimization: keyword pre-filter to keyword_top_n,
|
|
378
|
+
then compute embeddings only on the narrowed candidates.
|
|
379
|
+
|
|
380
|
+
Args:
|
|
381
|
+
query: User query for emission factor.
|
|
382
|
+
target_unit: Required unit (hard constraint).
|
|
383
|
+
top_k: Number of candidates to return.
|
|
384
|
+
keyword_top_n: Top-N candidates from keyword stage to pass
|
|
385
|
+
to embedding stage. Lower = faster, higher = more thorough.
|
|
386
|
+
|
|
387
|
+
Returns:
|
|
388
|
+
Top-k candidates with hybrid scores.
|
|
389
|
+
"""
|
|
390
|
+
# Stage 0: Unit pre-filter
|
|
391
|
+
if target_unit:
|
|
392
|
+
eligible_factors = [
|
|
393
|
+
(i, f) for i, f in enumerate(self.factors)
|
|
394
|
+
if self._unit_match(f, target_unit)
|
|
395
|
+
]
|
|
396
|
+
if not eligible_factors:
|
|
397
|
+
logger.warning(f"No factors match unit '{target_unit}'")
|
|
398
|
+
return []
|
|
399
|
+
else:
|
|
400
|
+
eligible_factors = list(enumerate(self.factors))
|
|
401
|
+
|
|
402
|
+
# Stage 1a: Keyword scoring
|
|
403
|
+
keyword_results = []
|
|
404
|
+
for idx, factor in eligible_factors:
|
|
405
|
+
score = self._keyword_score(factor, query)
|
|
406
|
+
keyword_results.append((idx, score))
|
|
407
|
+
|
|
408
|
+
keyword_results.sort(key=lambda x: x[1], reverse=True)
|
|
409
|
+
max_kw = keyword_results[0][1] if keyword_results else 1.0
|
|
410
|
+
keyword_scores = {
|
|
411
|
+
idx: score / max_kw if max_kw > 0 else 0.0
|
|
412
|
+
for idx, score in keyword_results
|
|
413
|
+
}
|
|
414
|
+
|
|
415
|
+
# Stage 1b: Narrow candidates before embedding
|
|
416
|
+
if self._embedding_model is not None and len(keyword_results) > keyword_top_n:
|
|
417
|
+
narrowed = keyword_results[:keyword_top_n]
|
|
418
|
+
logger.info(
|
|
419
|
+
f"Keyword pre-filter: {len(keyword_results)} -> {keyword_top_n} candidates"
|
|
420
|
+
)
|
|
421
|
+
else:
|
|
422
|
+
narrowed = keyword_results
|
|
423
|
+
|
|
424
|
+
# Batch-encode narrowed candidates, then compute similarities
|
|
425
|
+
embedding_scores = {}
|
|
426
|
+
if self._embedding_model is not None:
|
|
427
|
+
narrowed_indices = [idx for idx, _ in narrowed]
|
|
428
|
+
self._batch_encode_factors(narrowed_indices)
|
|
429
|
+
query_emb = self._embedding_model.encode([query])[0].tolist()
|
|
430
|
+
for idx, _ in narrowed:
|
|
431
|
+
factor_emb = self._embedding_cache.get(idx, [])
|
|
432
|
+
if factor_emb:
|
|
433
|
+
embedding_scores[idx] = self._cosine_similarity(query_emb, factor_emb)
|
|
434
|
+
else:
|
|
435
|
+
embedding_scores[idx] = 0.0
|
|
436
|
+
|
|
437
|
+
# Combine scores
|
|
438
|
+
results = []
|
|
439
|
+
for idx, _ in narrowed:
|
|
440
|
+
factor = self.factors[idx]
|
|
441
|
+
kw_score = keyword_scores.get(idx, 0.0)
|
|
442
|
+
emb_score = embedding_scores.get(idx, 0.0)
|
|
443
|
+
|
|
444
|
+
if self._embedding_model is None:
|
|
445
|
+
hybrid = kw_score
|
|
446
|
+
else:
|
|
447
|
+
hybrid = KEYWORD_WEIGHT * kw_score + EMBEDDING_WEIGHT * emb_score
|
|
448
|
+
|
|
449
|
+
quality_ratings = self._compute_quality_ratings(factor, query)
|
|
450
|
+
quality_score = self._compute_quality_score(quality_ratings)
|
|
451
|
+
|
|
452
|
+
results.append({
|
|
453
|
+
"factor": factor,
|
|
454
|
+
"keyword_score": kw_score,
|
|
455
|
+
"embedding_score": emb_score,
|
|
456
|
+
"hybrid_score": hybrid,
|
|
457
|
+
"quality_ratings": quality_ratings,
|
|
458
|
+
"quality_score": quality_score,
|
|
459
|
+
"llm_reasoning": "",
|
|
460
|
+
"final_score": 0.0,
|
|
461
|
+
})
|
|
462
|
+
|
|
463
|
+
results.sort(key=lambda x: x["hybrid_score"], reverse=True)
|
|
464
|
+
return results[:top_k]
|
|
465
|
+
|
|
466
|
+
def match(
|
|
467
|
+
self,
|
|
468
|
+
query: str,
|
|
469
|
+
target_unit: str = "",
|
|
470
|
+
top_k: int = 5,
|
|
471
|
+
) -> MatchResult | None:
|
|
472
|
+
"""Full two-stage matching pipeline.
|
|
473
|
+
|
|
474
|
+
Stage 0: Unit pre-filter
|
|
475
|
+
Stage 1: Hybrid retrieval (keyword 0.3 + embedding 0.7)
|
|
476
|
+
Stage 2: Quality rating + LLM fine ranking
|
|
477
|
+
|
|
478
|
+
Args:
|
|
479
|
+
query: User query for emission factor.
|
|
480
|
+
target_unit: Required unit (hard constraint).
|
|
481
|
+
top_k: Number of final results to return.
|
|
482
|
+
|
|
483
|
+
Returns:
|
|
484
|
+
MatchResult with best factor, reason, confidence, and candidates, or None if no matches.
|
|
485
|
+
"""
|
|
486
|
+
candidates = self.hybrid_search(query, target_unit, top_k=20)
|
|
487
|
+
|
|
488
|
+
if not candidates:
|
|
489
|
+
return None
|
|
490
|
+
|
|
491
|
+
ranked = self._llm_rank(query, candidates)
|
|
492
|
+
|
|
493
|
+
if not ranked:
|
|
494
|
+
return None
|
|
495
|
+
|
|
496
|
+
# Return best match as MatchResult
|
|
497
|
+
best = ranked[0]
|
|
498
|
+
return MatchResult(
|
|
499
|
+
factor=best["factor"],
|
|
500
|
+
reason=best.get("llm_reasoning", ""),
|
|
501
|
+
confidence=best.get("final_score", 0.0),
|
|
502
|
+
candidates=[c["factor"] for c in ranked[:top_k]],
|
|
503
|
+
)
|
|
504
|
+
|
|
505
|
+
def match_activity(
|
|
506
|
+
self,
|
|
507
|
+
activity: Activity,
|
|
508
|
+
top_k: int = 5,
|
|
509
|
+
) -> MatchResult | None:
|
|
510
|
+
"""Match emission factor using structured activity data.
|
|
511
|
+
|
|
512
|
+
Args:
|
|
513
|
+
activity: Activity object with name, location, time, unit, etc.
|
|
514
|
+
top_k: Number of final results to return.
|
|
515
|
+
|
|
516
|
+
Returns:
|
|
517
|
+
MatchResult with best factor, reason, confidence, and candidates, or None if no matches.
|
|
518
|
+
"""
|
|
519
|
+
query = activity.to_query()
|
|
520
|
+
candidates = self.hybrid_search(query, activity.unit, top_k=20)
|
|
521
|
+
|
|
522
|
+
if not candidates:
|
|
523
|
+
return None
|
|
524
|
+
|
|
525
|
+
# Re-compute quality ratings using activity fields
|
|
526
|
+
for c in candidates:
|
|
527
|
+
c["quality_ratings"] = self._compute_quality_ratings_for_activity(
|
|
528
|
+
c["factor"], activity
|
|
529
|
+
)
|
|
530
|
+
c["quality_score"] = self._compute_quality_score(c["quality_ratings"])
|
|
531
|
+
|
|
532
|
+
ranked = self._llm_rank_activity(activity, candidates)
|
|
533
|
+
|
|
534
|
+
if not ranked:
|
|
535
|
+
return None
|
|
536
|
+
|
|
537
|
+
# Return best match as MatchResult
|
|
538
|
+
best = ranked[0]
|
|
539
|
+
return MatchResult(
|
|
540
|
+
factor=best["factor"],
|
|
541
|
+
reason=best.get("llm_reasoning", ""),
|
|
542
|
+
confidence=best.get("final_score", 0.0),
|
|
543
|
+
candidates=[c["factor"] for c in ranked[:top_k]],
|
|
544
|
+
)
|
|
545
|
+
|
|
546
|
+
def _compute_quality_ratings_for_activity(
|
|
547
|
+
self, factor: Factor, activity: Activity
|
|
548
|
+
) -> dict:
|
|
549
|
+
"""Compute quality ratings using structured activity data."""
|
|
550
|
+
current_year = datetime.now().year
|
|
551
|
+
ratings = {}
|
|
552
|
+
|
|
553
|
+
# 1. 技术代表性 - compare activity name/technology with factor
|
|
554
|
+
activity_name = activity.name.lower()
|
|
555
|
+
factor_name = factor.name.lower()
|
|
556
|
+
factor_applicability = (factor.applicability or "").lower()
|
|
557
|
+
activity_tech = (activity.technology or "").lower()
|
|
558
|
+
|
|
559
|
+
if activity_name in factor_name or factor_name in activity_name:
|
|
560
|
+
ratings["tech_representativeness"] = TECH_SCORES["exact_match"]
|
|
561
|
+
elif activity_tech and activity_tech in factor_applicability:
|
|
562
|
+
ratings["tech_representativeness"] = TECH_SCORES["exact_match"]
|
|
563
|
+
elif any(t in factor_applicability for t in activity_name.split()):
|
|
564
|
+
ratings["tech_representativeness"] = TECH_SCORES["type_match"]
|
|
565
|
+
else:
|
|
566
|
+
ratings["tech_representativeness"] = TECH_SCORES["unrelated"]
|
|
567
|
+
|
|
568
|
+
# 2. 地理代表性 - compare activity location with factor geography
|
|
569
|
+
factor_location = factor.geography.get("location", "").lower()
|
|
570
|
+
activity_location = activity.location.lower()
|
|
571
|
+
|
|
572
|
+
if activity_location and activity_location in factor_location:
|
|
573
|
+
ratings["geo_representativeness"] = GEO_SCORES["provincial"]
|
|
574
|
+
elif any(kw in factor_location for kw in ["省", "市", "区", "province", "city"]):
|
|
575
|
+
ratings["geo_representativeness"] = GEO_SCORES["provincial"]
|
|
576
|
+
elif any(kw in factor_location for kw in ["中国", "china"]):
|
|
577
|
+
ratings["geo_representativeness"] = GEO_SCORES["national"]
|
|
578
|
+
else:
|
|
579
|
+
ratings["geo_representativeness"] = GEO_SCORES["international"]
|
|
580
|
+
|
|
581
|
+
# 3. 来源可靠性
|
|
582
|
+
source_lower = factor.source.lower()
|
|
583
|
+
if any(kw in source_lower for kw in ["实测", "测量", "实绩", "measured", "actual"]):
|
|
584
|
+
ratings["source_reliability"] = SOURCE_SCORES["measured"]
|
|
585
|
+
elif any(kw in source_lower for kw in ["估算", "计算", "模型", "estimated", "modeled"]):
|
|
586
|
+
ratings["source_reliability"] = SOURCE_SCORES["estimated"]
|
|
587
|
+
else:
|
|
588
|
+
ratings["source_reliability"] = SOURCE_SCORES["unknown"]
|
|
589
|
+
|
|
590
|
+
# 4. 时间代表性 - compare activity time with factor source_year
|
|
591
|
+
factor_year = factor.source_year
|
|
592
|
+
if activity.time and factor_year > 0:
|
|
593
|
+
try:
|
|
594
|
+
activity_year = int(activity.time[:4])
|
|
595
|
+
age = activity_year - factor_year
|
|
596
|
+
if abs(age) <= 3:
|
|
597
|
+
ratings["time_representativeness"] = TIME_SCORES["recent"]
|
|
598
|
+
elif abs(age) <= 10:
|
|
599
|
+
ratings["time_representativeness"] = TIME_SCORES["moderate"]
|
|
600
|
+
else:
|
|
601
|
+
ratings["time_representativeness"] = TIME_SCORES["old"]
|
|
602
|
+
except ValueError:
|
|
603
|
+
ratings["time_representativeness"] = TIME_SCORES["old"]
|
|
604
|
+
elif factor_year == 0:
|
|
605
|
+
ratings["time_representativeness"] = TIME_SCORES["old"]
|
|
606
|
+
else:
|
|
607
|
+
age = current_year - factor_year
|
|
608
|
+
if age <= 3:
|
|
609
|
+
ratings["time_representativeness"] = TIME_SCORES["recent"]
|
|
610
|
+
elif age <= 10:
|
|
611
|
+
ratings["time_representativeness"] = TIME_SCORES["moderate"]
|
|
612
|
+
else:
|
|
613
|
+
ratings["time_representativeness"] = TIME_SCORES["old"]
|
|
614
|
+
|
|
615
|
+
# 5. 因子类型
|
|
616
|
+
category = factor.category.lower()
|
|
617
|
+
if any(kw in category for kw in ["lca", "生命周期", "life cycle"]):
|
|
618
|
+
ratings["factor_type"] = FACTOR_TYPE_SCORES["lca"]
|
|
619
|
+
elif any(kw in category for kw in ["过程", "process", "清单", "inventory"]):
|
|
620
|
+
ratings["factor_type"] = FACTOR_TYPE_SCORES["process"]
|
|
621
|
+
else:
|
|
622
|
+
ratings["factor_type"] = FACTOR_TYPE_SCORES["unknown"]
|
|
623
|
+
|
|
624
|
+
return ratings
|
|
625
|
+
|
|
626
|
+
def _llm_rank_activity(
|
|
627
|
+
self, activity: Activity, candidates: list[dict]
|
|
628
|
+
) -> list[dict]:
|
|
629
|
+
"""LLM fine ranking with structured activity context."""
|
|
630
|
+
factor_descriptions = []
|
|
631
|
+
for i, c in enumerate(candidates, 1):
|
|
632
|
+
desc = self._format_factor_for_llm(i, c["factor"], c["quality_ratings"])
|
|
633
|
+
factor_descriptions.append(desc)
|
|
634
|
+
|
|
635
|
+
factors_text = "\n\n".join(factor_descriptions)
|
|
636
|
+
|
|
637
|
+
prompt = f"""请根据用户活动数据,从候选列表中选择最匹配的排放因子。
|
|
638
|
+
|
|
639
|
+
用户活动数据:
|
|
640
|
+
活动名称: {activity.name}
|
|
641
|
+
地点: {activity.location or '未指定'}
|
|
642
|
+
时间: {activity.time or '未指定'}
|
|
643
|
+
工艺: {activity.technology or '未指定'}
|
|
644
|
+
单位: {activity.unit}
|
|
645
|
+
数量: {activity.amount or '未指定'}
|
|
646
|
+
|
|
647
|
+
候选因子(含数据质量评级):
|
|
648
|
+
{factors_text}
|
|
649
|
+
|
|
650
|
+
选择原则:
|
|
651
|
+
1. 地理位置必须匹配(优先同国家/地区)
|
|
652
|
+
2. 技术规格匹配(电压等级、温度、压力等)
|
|
653
|
+
3. 当用户明确指定规格时(如"230V"、"低电压"、"1kV-60kV"),严格匹配
|
|
654
|
+
4. 当用户未指定规格时,保持默认排序(第一个候选)不变
|
|
655
|
+
5. 仅当有明确理由时才改变排序
|
|
656
|
+
6. 单位必须完全匹配
|
|
657
|
+
7. 数据质量评级越高越好
|
|
658
|
+
|
|
659
|
+
请返回最匹配的因子编号(多个用逗号分隔),并简要说明选择理由。
|
|
660
|
+
格式: 1,3,5 | 理由: ..."""
|
|
661
|
+
|
|
662
|
+
try:
|
|
663
|
+
response = self.llm_client.chat.completions.create(
|
|
664
|
+
model=self.chat_model,
|
|
665
|
+
messages=[
|
|
666
|
+
{
|
|
667
|
+
"role": "system",
|
|
668
|
+
"content": (
|
|
669
|
+
"你是碳排放因子匹配专家。"
|
|
670
|
+
"数据质量评级说明:技术代表性/地理代表性/来源可靠性/时间代表性/因子类型,"
|
|
671
|
+
"1=优, 2=中, 3=差。来源可靠性权重最高(0.4)。"
|
|
672
|
+
"优先选择:地理匹配、时间接近、来源可靠的因子。"
|
|
673
|
+
"技术规格匹配:仔细对比用户查询与因子名称中的电压等级、温度、压力等参数。"
|
|
674
|
+
"重要:默认选择第一个候选(已按相关性排序)。仅当能明确证明第一个候选"
|
|
675
|
+
"在技术规格上不匹配时才改变排序(如用户明确写了'230V'但候选是'1kV-60kV')。"
|
|
676
|
+
),
|
|
677
|
+
},
|
|
678
|
+
{"role": "user", "content": prompt},
|
|
679
|
+
],
|
|
680
|
+
temperature=0.1,
|
|
681
|
+
)
|
|
682
|
+
|
|
683
|
+
answer = response.choices[0].message.content or ""
|
|
684
|
+
return self._parse_ranked_results(candidates, answer)
|
|
685
|
+
|
|
686
|
+
except Exception as e:
|
|
687
|
+
logger.warning(f"LLM ranking failed: {e}, using hybrid score")
|
|
688
|
+
return candidates
|
|
689
|
+
|
|
690
|
+
def _llm_rank(self, query: str, candidates: list[dict]) -> list[dict]:
|
|
691
|
+
"""Stage 2: LLM fine ranking with quality ratings context."""
|
|
692
|
+
factor_descriptions = []
|
|
693
|
+
for i, c in enumerate(candidates, 1):
|
|
694
|
+
desc = self._format_factor_for_llm(i, c["factor"], c["quality_ratings"])
|
|
695
|
+
factor_descriptions.append(desc)
|
|
696
|
+
|
|
697
|
+
factors_text = "\n\n".join(factor_descriptions)
|
|
698
|
+
units = list(set(c["factor"].unit for c in candidates))
|
|
699
|
+
units_text = ", ".join(units[:3])
|
|
700
|
+
|
|
701
|
+
prompt = f"""请根据用户查询,从候选列表中选择最匹配的排放因子。
|
|
702
|
+
|
|
703
|
+
用户查询: {query}
|
|
704
|
+
可选单位: {units_text}
|
|
705
|
+
|
|
706
|
+
候选因子(含数据质量评级):
|
|
707
|
+
{factors_text}
|
|
708
|
+
|
|
709
|
+
选择原则:
|
|
710
|
+
1. 地理位置必须匹配(优先同国家/地区)
|
|
711
|
+
2. 技术规格匹配(电压等级、温度、压力等)
|
|
712
|
+
3. 当用户明确指定规格时(如"230V"、"低电压"、"1kV-60kV"),严格匹配
|
|
713
|
+
4. 当用户未指定规格时,保持默认排序(第一个候选)不变
|
|
714
|
+
5. 仅当有明确理由时才改变排序
|
|
715
|
+
6. 单位必须完全匹配
|
|
716
|
+
7. 数据质量评级越高越好
|
|
717
|
+
|
|
718
|
+
请返回最匹配的因子编号(多个用逗号分隔),并简要说明选择理由。
|
|
719
|
+
格式: 1,3,5 | 理由: ..."""
|
|
720
|
+
|
|
721
|
+
try:
|
|
722
|
+
response = self.llm_client.chat.completions.create(
|
|
723
|
+
model=self.chat_model,
|
|
724
|
+
messages=[
|
|
725
|
+
{
|
|
726
|
+
"role": "system",
|
|
727
|
+
"content": (
|
|
728
|
+
"你是碳排放因子匹配专家。"
|
|
729
|
+
"数据质量评级说明:技术代表性/地理代表性/来源可靠性/时间代表性/因子类型,"
|
|
730
|
+
"1=优, 2=中, 3=差。来源可靠性权重最高(0.4)。"
|
|
731
|
+
"优先选择:地理匹配、时间接近、来源可靠的因子。"
|
|
732
|
+
"技术规格匹配:仔细对比用户查询与因子名称中的电压等级、温度、压力等参数。"
|
|
733
|
+
"重要:默认选择第一个候选(已按相关性排序)。仅当能明确证明第一个候选"
|
|
734
|
+
"在技术规格上不匹配时才改变排序(如用户明确写了'230V'但候选是'1kV-60kV')。"
|
|
735
|
+
),
|
|
736
|
+
},
|
|
737
|
+
{"role": "user", "content": prompt},
|
|
738
|
+
],
|
|
739
|
+
temperature=0.1,
|
|
740
|
+
)
|
|
741
|
+
|
|
742
|
+
answer = response.choices[0].message.content or ""
|
|
743
|
+
return self._parse_ranked_results(candidates, answer)
|
|
744
|
+
|
|
745
|
+
except Exception as e:
|
|
746
|
+
logger.warning(f"LLM ranking failed: {e}, using hybrid score")
|
|
747
|
+
return candidates
|
|
748
|
+
|
|
749
|
+
def _parse_ranked_results(
|
|
750
|
+
self, candidates: list[dict], llm_answer: str
|
|
751
|
+
) -> list[dict]:
|
|
752
|
+
"""Parse LLM ranking answer and reorder candidates."""
|
|
753
|
+
numbers = re.findall(
|
|
754
|
+
r"\d+",
|
|
755
|
+
llm_answer.split("|")[0] if "|" in llm_answer else llm_answer,
|
|
756
|
+
)
|
|
757
|
+
|
|
758
|
+
ranked = []
|
|
759
|
+
seen = set()
|
|
760
|
+
|
|
761
|
+
for num_str in numbers:
|
|
762
|
+
try:
|
|
763
|
+
idx = int(num_str) - 1
|
|
764
|
+
if 0 <= idx < len(candidates) and idx not in seen:
|
|
765
|
+
candidates[idx]["llm_reasoning"] = llm_answer
|
|
766
|
+
candidates[idx]["final_score"] = (
|
|
767
|
+
candidates[idx]["hybrid_score"] * 0.5
|
|
768
|
+
+ candidates[idx]["quality_score"] * 0.5
|
|
769
|
+
)
|
|
770
|
+
ranked.append(candidates[idx])
|
|
771
|
+
seen.add(idx)
|
|
772
|
+
except (ValueError, IndexError):
|
|
773
|
+
continue
|
|
774
|
+
|
|
775
|
+
for i, c in enumerate(candidates):
|
|
776
|
+
if i not in seen:
|
|
777
|
+
c["final_score"] = c["hybrid_score"] * 0.3
|
|
778
|
+
ranked.append(c)
|
|
779
|
+
|
|
780
|
+
return ranked
|
|
781
|
+
|
|
782
|
+
|
|
783
|
+
def create_matcher(db: Optional[FactorDatabase] = None) -> EmissionFactorMatcher:
|
|
784
|
+
"""Create a matcher with factors from the database.
|
|
785
|
+
|
|
786
|
+
Args:
|
|
787
|
+
db: FactorDatabase instance. If None, creates a new one.
|
|
788
|
+
|
|
789
|
+
Returns:
|
|
790
|
+
Initialized EmissionFactorMatcher.
|
|
791
|
+
"""
|
|
792
|
+
if db is None:
|
|
793
|
+
db = FactorDatabase()
|
|
794
|
+
|
|
795
|
+
return EmissionFactorMatcher(factors=db.factors)
|