@lzhzzzzwill/cofos 1.0.0 → 1.0.2

package/runtime/src/data_processing/utils.py

@@ -0,0 +1,74 @@
+#!/usr/bin/env python3
+"""
+COF-ROS-Reasoner: 数据处理工具函数
+"""
+
+import logging
+from pathlib import Path
+from typing import Dict, Any
+
+
+def setup_logging(config: Dict[str, Any]) -> None:
+    """设置日志配置"""
+    logs_dir = Path(config['paths']['outputs_dir']) / 'logs'
+    logs_dir.mkdir(parents=True, exist_ok=True)
+
+    log_file = logs_dir / f"process_{__name__.split('.')[-1]}.log"
+
+    logging.basicConfig(
+        level=logging.INFO,
+        format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
+        handlers=[
+            logging.FileHandler(log_file, encoding='utf-8'),
+            logging.StreamHandler()
+        ]
+    )
+
+
+def load_config(config_path: str) -> Dict[str, Any]:
+    """加载 YAML 配置文件"""
+    import yaml
+    with open(config_path, 'r', encoding='utf-8') as f:
+        return yaml.safe_load(f)
+
+
+def ensure_directory(path: Path) -> Path:
+    """确保目录存在"""
+    path.mkdir(parents=True, exist_ok=True)
+    return path
+
+
+def save_jsonl(records: list, file_path: Path) -> None:
+    """保存记录为 JSONL 格式"""
+    ensure_directory(file_path.parent)
+    with open(file_path, 'w', encoding='utf-8') as f:
+        for record in records:
+            f.write(json.dumps(record, ensure_ascii=False) + '\n')
+
+
+def load_jsonl(file_path: Path) -> list:
+    """加载 JSONL 文件"""
+    records = []
+    if file_path.exists():
+        with open(file_path, 'r', encoding='utf-8') as f:
+            for line in f:
+                if line.strip():
+                    records.append(json.loads(line))
+    return records
+
+
+def load_json(file_path: Path) -> Any:
+    """加载 JSON 文件"""
+    with open(file_path, 'r', encoding='utf-8') as f:
+        return json.load(f)
+
+
+def save_json(data: Any, file_path: Path) -> None:
+    """保存数据为 JSON 文件"""
+    ensure_directory(file_path.parent)
+    with open(file_path, 'w', encoding='utf-8') as f:
+        json.dump(data, f, ensure_ascii=False, indent=2)
+
+
+# 全局导入
+import json

package/runtime/src/inference/__init__.py

@@ -0,0 +1,8 @@
+#!/usr/bin/env python3
+"""
+COF-ROS-Reasoner: 推理模块初始化
+"""
+
+from .run_kg_grounded_inference import KGRAGInference
+
+__all__ = ['KGRAGInference']

package/runtime/src/inference/query_router.py

@@ -0,0 +1,33 @@
+"""Minimal conversational routing before domain retrieval."""
+
+import re
+from pathlib import Path
+from typing import Optional
+
+import yaml
+
+
+class QueryRouter:
+    """Handle only turns that should not invoke either retrieval or generation."""
+
+    _RESPONSES = None
+
+    @classmethod
+    def _load_responses(cls) -> dict:
+        if cls._RESPONSES is not None:
+            return cls._RESPONSES
+        prompt_path = Path("config/prompt_templates.yaml")
+        if prompt_path.exists():
+            with open(prompt_path, "r", encoding="utf-8") as f:
+                prompt_config = yaml.safe_load(f)
+            cls._RESPONSES = prompt_config.get("interaction", {}).get("greetings", {})
+        else:
+            cls._RESPONSES = {}
+        return cls._RESPONSES
+
+    @classmethod
+    def quick_response(cls, question: str) -> Optional[str]:
+        normalized = re.sub(
+            r"[^a-z0一-鿿]+", " ", question.lower()
+        ).strip()
+        return cls._load_responses().get(normalized)

package/runtime/src/inference/run_kg_grounded_inference.py

@@ -0,0 +1,370 @@
+#!/usr/bin/env python3
+"""
+COF-ROS-Reasoner: KG-RAG 推理 demo
+
+目标：
+- 实现基于知识图谱和检索的问答推理
+- 输出必须包含：主导物种、辅助物种、反应条件、证据依据、机理解释、不确定性
+"""
+
+import os
+import json
+import yaml
+import logging
+from pathlib import Path
+from typing import Dict, List, Any, Optional
+from datetime import datetime
+
+# 添加 src 路径
+
+try:
+    from openai import OpenAI
+except ImportError:
+    OpenAI = None
+
+try:
+    import requests
+except ImportError:
+    requests = None
+
+try:
+    from sentence_transformers import SentenceTransformer
+except ImportError:
+    SentenceTransformer = None
+
+try:
+    import faiss
+except ImportError:
+    faiss = None
+
+try:
+    from rank_bm25 import BM25Okapi
+except ImportError:
+    BM25Okapi = None
+
+from retrieval.retrieve_evidence import EvidenceRetriever
+from data_processing.utils import setup_logging, ensure_directory
+
+
+class KGRAGInference:
+    """KG-RAG 推理"""
+
+    def __init__(self, config: Dict[str, Any]):
+        self.config = config
+        self.retriever = None
+
+        # 设置日志
+        setup_logging(config)
+
+        # 推理配置
+        self.temperature = config.get('generation', {}).get('temperature', 0.2)
+        self.max_new_tokens = config.get('generation', {}).get('max_new_tokens', 1024)
+        llm_config = config.get('llm', {})
+        self.llm_provider = llm_config.get('provider', 'ollama')
+        self.generate_url = os.environ.get(
+            'COFROS_LLM_URL',
+            llm_config.get('generate_url', 'http://localhost:11434/api/generate')
+        )
+        self.model_name = (
+            os.environ.get('COFROS_LLM_MODEL')
+            or os.environ.get('COFROS_INFERENCE_MODEL')
+            or llm_config.get('inference_model')
+            or llm_config.get('teacher_model')
+            or llm_config.get('model')
+            or config.get('models', {}).get('teacher_model', 'qwen3.6:35b')
+        )
+        self.timeout = llm_config.get('timeout', 120)
+
+        # 加载检索器
+        self._load_retriever()
+
+        # 加载 KG facts
+        self.kg_facts = self._load_kg_facts()
+
+        # 加载 prompt 模板
+        self.system_prompt = self._load_system_prompt()
+        self.user_prompt_template = self._load_user_prompt_template()
+
+    def _load_retriever(self):
+        """加载检索器"""
+        try:
+            self.retriever = EvidenceRetriever(self.config)
+            logging.info("Retriever loaded successfully")
+        except Exception as e:
+            logging.error(f"Failed to load retriever: {e}")
+
+    def _load_kg_facts(self) -> List[Dict[str, Any]]:
+        """加载 KG facts"""
+        kg_path = Path(self.config['paths']['kg_dir']) / 'cofros_kg_triples.jsonl'
+        facts = []
+
+        if kg_path.exists():
+            try:
+                with open(kg_path, 'r', encoding='utf-8') as f:
+                    for line in f:
+                        if line.strip():
+                            facts.append(json.loads(line))
+                logging.info(f"Loaded {len(facts)} KG facts")
+            except Exception as e:
+                logging.error(f"Failed to load KG facts: {e}")
+
+        return facts
+
+    def _load_system_prompt(self) -> str:
+        """加载系统提示"""
+        prompt_path = Path('config/prompt_templates.yaml')
+        if not prompt_path.exists():
+            raise FileNotFoundError(f"Prompt template file not found: {prompt_path}")
+        with open(prompt_path, 'r', encoding='utf-8') as f:
+            prompt_config = yaml.safe_load(f) or {}
+        prompt = prompt_config.get('kg_rag_system_prompt')
+        if not prompt:
+            raise ValueError(f"kg_rag_system_prompt is missing from {prompt_path}")
+        return prompt
+
+    def _load_user_prompt_template(self) -> str:
+        """加载用户提示模板"""
+        prompt_path = Path('config/prompt_templates.yaml')
+        if not prompt_path.exists():
+            raise FileNotFoundError(f"Prompt template file not found: {prompt_path}")
+        with open(prompt_path, 'r', encoding='utf-8') as f:
+            prompt_config = yaml.safe_load(f) or {}
+        prompt = prompt_config.get('kg_rag_user_prompt')
+        if not prompt:
+            raise ValueError(f"kg_rag_user_prompt is missing from {prompt_path}")
+        return prompt
+
+    def format_kg_facts(self, kg_facts: List[Dict[str, Any]], max_items: int = 10) -> str:
+        """格式化 KG facts，标注可信度。"""
+        if not kg_facts:
+            return "No relevant knowledge graph facts found."
+
+        support_icon = {
+            'supported': '✓',
+            'partially_supported': '~',
+            'unsupported': '?',
+        }
+
+        formatted = []
+        for fact in kg_facts[:max_items]:
+            head = fact.get('head', '')
+            relation = fact.get('relation', '')
+            tail = fact.get('tail', '')
+            tail_category = fact.get('tail_category', '')
+
+            line = f"- {head} {relation} {tail}"
+            if tail_category:
+                line += f" ({tail_category})"
+
+            condition = fact.get('condition') or {}
+            if isinstance(condition, dict):
+                condition_text = '; '.join(
+                    str(value) for value in [
+                        condition.get('phase'),
+                        condition.get('oxidant_source'),
+                        condition.get('driving_force'),
+                    ]
+                    if value
+                )
+                if condition_text:
+                    line += f" | condition: {condition_text}"
+
+            evidence = fact.get('evidence') or {}
+            if isinstance(evidence, dict):
+                status = evidence.get('support_status') or ''
+                level = evidence.get('evidence_level') or ''
+                evidence_text = ' '.join(filter(None, [status, level]))
+                icon = support_icon.get(status, '')
+                if icon:
+                    evidence_text = f"{icon} {evidence_text}"
+            else:
+                evidence_text = str(evidence)
+            if evidence_text:
+                line += f" | {evidence_text}"
+
+            formatted.append(line)
+
+        return '\n'.join(formatted)
+
+    @staticmethod
+    def _truncate_at_sentence(text: str, max_chars: int = 500) -> str:
+        """Truncate *text* at the last sentence boundary before *max_chars*."""
+        if len(text) <= max_chars:
+            return text
+        snippet = text[:max_chars]
+        for sep in ('. ', '.\n', '? ', '?\n', '! ', '!\n', '。', '；'):
+            idx = snippet.rfind(sep)
+            if idx > max_chars * 0.5:
+                return snippet[:idx + len(sep.rstrip())]
+        last_space = snippet.rfind(' ')
+        if last_space > max_chars * 0.5:
+            return snippet[:last_space]
+        return snippet
+
+    def format_evidence(self, evidence_list: List[Dict[str, Any]], max_items: int = 10) -> str:
+        """格式化证据，标注可信度等级和验证状态。"""
+        if not evidence_list:
+            return "No relevant evidence found."
+
+        level_badge = {
+            'high': '[HIGH confidence]',
+            'medium': '[MEDIUM confidence]',
+            'low': '[LOW confidence]',
+        }
+        support_badge = {
+            'supported': 'verified',
+            'partially_supported': 'partial',
+            'unsupported': 'unverified',
+        }
+
+        formatted = []
+        for i, evidence in enumerate(evidence_list[:max_items], 1):
+            source_type = evidence.get('source_type', 'unknown')
+            material = evidence.get('material_context', {}).get('material_type', 'N/A')
+            dominant = [item.get('species', 'N/A') for item in evidence.get('dominant_ros_or_product', [])]
+            evidence_level = evidence.get('evidence_level', 'low')
+            support_status = evidence.get('support_status', 'unsupported')
+
+            level_str = level_badge.get(evidence_level, evidence_level)
+            support_str = support_badge.get(support_status, support_status)
+            text = self._truncate_at_sentence(evidence.get('text', ''), max_chars=500)
+
+            formatted.append(
+                f"[{i}] {source_type} {level_str} ({support_str})\n"
+                f"    Material: {material}\n"
+                f"    Dominant ROS: {', '.join(dominant)}\n"
+                f"    {text}"
+            )
+        return '\n\n'.join(formatted)
+
+    def call_llm(self, system_prompt: str, user_prompt: str) -> str:
+        """调用本地/内网 LLM。"""
+        if self.llm_provider != 'ollama':
+            raise ValueError(f"Unsupported LLM provider: {self.llm_provider}")
+
+        if requests is None:
+            raise ImportError("requests required. Run: pip install requests")
+
+        prompt = f"{system_prompt.strip()}\n\n{user_prompt.strip()}"
+        payload = {
+            "model": self.model_name,
+            "prompt": prompt,
+            "stream": False,
+            "options": {
+                "temperature": self.temperature,
+                "top_p": self.config.get('generation', {}).get('top_p', 0.9),
+                "num_predict": self.max_new_tokens,
+            },
+        }
+
+        response = requests.post(self.generate_url, json=payload, timeout=self.timeout)
+        response.raise_for_status()
+        data = response.json()
+        return data.get("response", "").strip()
+
+    def answer_question(self, question: str, top_k: int = 5) -> Dict[str, Any]:
+        """回答问题"""
+        result = {
+            'question': question,
+            'timestamp': datetime.now().isoformat()
+        }
+
+        if self.retriever is None:
+            result['kg_facts'] = []
+            result['evidence'] = []
+            result['answer'] = "Evidence retrieval is unavailable. Build retrieval indexes before running KG-RAG inference."
+            return result
+
+        # 1. 检索 KG facts
+        kg_facts = self.retriever.retrieve_kg_facts(question, top_k)
+        result['kg_facts'] = kg_facts
+
+        # 2. 检索文献证据
+        evidence = self.retriever.retrieve_with_sources(question, top_k)
+        result['evidence'] = evidence
+
+        # 3. 构造 prompt
+        kg_facts_str = self.format_kg_facts(kg_facts)
+        evidence_str = self.format_evidence(evidence)
+
+        user_prompt = self.user_prompt_template.format(
+            question=question,
+            kg_facts=kg_facts_str,
+            evidence=evidence_str
+        )
+
+        # 4. 调用 LLM
+        answer = self.call_llm(self.system_prompt, user_prompt)
+        result['answer'] = answer
+
+        return result
+
+    def run_demo(self):
+        """运行 demo"""
+        logging.info("KG-RAG Inference Demo")
+        logging.info("=" * 50)
+
+        # 从 prompt_templates.yaml 加载 demo 问题
+        import yaml
+        prompt_path = Path("config/prompt_templates.yaml")
+        test_questions = [
+            "What ROS does TT-T-COF generate under visible light photocatalysis?",
+            "Which COF materials are reported to generate H2O2 as the main product?",
+            "What is the mechanism of ROS generation in donor-acceptor COFs?",
+        ]
+        if prompt_path.exists():
+            with open(prompt_path, "r", encoding="utf-8") as f:
+                prompt_config = yaml.safe_load(f)
+            test_questions = prompt_config.get("demo", {}).get("questions", test_questions)
+
+        for question in test_questions:
+            print(f"\n{'=' * 50}")
+            print(f"Question: {question}")
+            print("=" * 50)
+
+            result = self.answer_question(question)
+
+            print(f"\nAnswer:")
+            print(result['answer'])
+
+    def run(self):
+        """运行推理服务"""
+        logging.info("KG-RAG Inference initialized")
+        logging.info(f"KG facts loaded: {len(self.kg_facts)}")
+
+
+def main():
+    """主函数"""
+    import argparse
+
+    parser = argparse.ArgumentParser(description='Run KG-RAG inference for COF-ROS questions')
+    parser.add_argument('--config', type=str, default='config/config.yaml',
+                        help='Path to config file')
+    parser.add_argument('--question', type=str, default=None,
+                        help='Single question to answer')
+    parser.add_argument('--demo', action='store_true',
+                        help='Run demo mode')
+    args = parser.parse_args()
+
+    # 加载配置
+    config_path = Path(args.config)
+    with open(config_path, 'r', encoding='utf-8') as f:
+        config = yaml.safe_load(f)
+
+    # 初始化推理器
+    inference = KGRAGInference(config)
+
+    if args.question:
+        # 单个问题
+        result = inference.answer_question(args.question)
+        print(f"\nQuestion: {args.question}")
+        print(f"\nAnswer:\n{result['answer']}")
+    elif args.demo:
+        # Demo 模式
+        inference.run_demo()
+    else:
+        inference.run()
+
+
+if __name__ == '__main__':
+    main()