@lzhzzzzwill/cofos 1.0.4 → 1.1.1

package/runtime/src/inference/run_kg_grounded_inference.py

@@ -1,370 +0,0 @@
-#!/usr/bin/env python3
-"""
-COF-ROS-Reasoner: KG-RAG 推理 demo
-
-目标：
-- 实现基于知识图谱和检索的问答推理
-- 输出必须包含：主导物种、辅助物种、反应条件、证据依据、机理解释、不确定性
-"""
-
-import os
-import json
-import yaml
-import logging
-from pathlib import Path
-from typing import Dict, List, Any, Optional
-from datetime import datetime
-
-# 添加 src 路径
-
-try:
-    from openai import OpenAI
-except ImportError:
-    OpenAI = None
-
-try:
-    import requests
-except ImportError:
-    requests = None
-
-try:
-    from sentence_transformers import SentenceTransformer
-except ImportError:
-    SentenceTransformer = None
-
-try:
-    import faiss
-except ImportError:
-    faiss = None
-
-try:
-    from rank_bm25 import BM25Okapi
-except ImportError:
-    BM25Okapi = None
-
-from retrieval.retrieve_evidence import EvidenceRetriever
-from data_processing.utils import setup_logging, ensure_directory
-
-
-class KGRAGInference:
-    """KG-RAG 推理"""
-
-    def __init__(self, config: Dict[str, Any]):
-        self.config = config
-        self.retriever = None
-
-        # 设置日志
-        setup_logging(config)
-
-        # 推理配置
-        self.temperature = config.get('generation', {}).get('temperature', 0.2)
-        self.max_new_tokens = config.get('generation', {}).get('max_new_tokens', 1024)
-        llm_config = config.get('llm', {})
-        self.llm_provider = llm_config.get('provider', 'ollama')
-        self.generate_url = os.environ.get(
-            'COFROS_LLM_URL',
-            llm_config.get('generate_url', 'http://localhost:11434/api/generate')
-        )
-        self.model_name = (
-            os.environ.get('COFROS_LLM_MODEL')
-            or os.environ.get('COFROS_INFERENCE_MODEL')
-            or llm_config.get('inference_model')
-            or llm_config.get('teacher_model')
-            or llm_config.get('model')
-            or config.get('models', {}).get('teacher_model', 'qwen3.6:35b')
-        )
-        self.timeout = llm_config.get('timeout', 120)
-
-        # 加载检索器
-        self._load_retriever()
-
-        # 加载 KG facts
-        self.kg_facts = self._load_kg_facts()
-
-        # 加载 prompt 模板
-        self.system_prompt = self._load_system_prompt()
-        self.user_prompt_template = self._load_user_prompt_template()
-
-    def _load_retriever(self):
-        """加载检索器"""
-        try:
-            self.retriever = EvidenceRetriever(self.config)
-            logging.info("Retriever loaded successfully")
-        except Exception as e:
-            logging.error(f"Failed to load retriever: {e}")
-
-    def _load_kg_facts(self) -> List[Dict[str, Any]]:
-        """加载 KG facts"""
-        kg_path = Path(self.config['paths']['kg_dir']) / 'cofros_kg_triples.jsonl'
-        facts = []
-
-        if kg_path.exists():
-            try:
-                with open(kg_path, 'r', encoding='utf-8') as f:
-                    for line in f:
-                        if line.strip():
-                            facts.append(json.loads(line))
-                logging.info(f"Loaded {len(facts)} KG facts")
-            except Exception as e:
-                logging.error(f"Failed to load KG facts: {e}")
-
-        return facts
-
-    def _load_system_prompt(self) -> str:
-        """加载系统提示"""
-        prompt_path = Path('config/prompt_templates.yaml')
-        if not prompt_path.exists():
-            raise FileNotFoundError(f"Prompt template file not found: {prompt_path}")
-        with open(prompt_path, 'r', encoding='utf-8') as f:
-            prompt_config = yaml.safe_load(f) or {}
-        prompt = prompt_config.get('kg_rag_system_prompt')
-        if not prompt:
-            raise ValueError(f"kg_rag_system_prompt is missing from {prompt_path}")
-        return prompt
-
-    def _load_user_prompt_template(self) -> str:
-        """加载用户提示模板"""
-        prompt_path = Path('config/prompt_templates.yaml')
-        if not prompt_path.exists():
-            raise FileNotFoundError(f"Prompt template file not found: {prompt_path}")
-        with open(prompt_path, 'r', encoding='utf-8') as f:
-            prompt_config = yaml.safe_load(f) or {}
-        prompt = prompt_config.get('kg_rag_user_prompt')
-        if not prompt:
-            raise ValueError(f"kg_rag_user_prompt is missing from {prompt_path}")
-        return prompt
-
-    def format_kg_facts(self, kg_facts: List[Dict[str, Any]], max_items: int = 10) -> str:
-        """格式化 KG facts，标注可信度。"""
-        if not kg_facts:
-            return "No relevant knowledge graph facts found."
-
-        support_icon = {
-            'supported': '✓',
-            'partially_supported': '~',
-            'unsupported': '?',
-        }
-
-        formatted = []
-        for fact in kg_facts[:max_items]:
-            head = fact.get('head', '')
-            relation = fact.get('relation', '')
-            tail = fact.get('tail', '')
-            tail_category = fact.get('tail_category', '')
-
-            line = f"- {head} {relation} {tail}"
-            if tail_category:
-                line += f" ({tail_category})"
-
-            condition = fact.get('condition') or {}
-            if isinstance(condition, dict):
-                condition_text = '; '.join(
-                    str(value) for value in [
-                        condition.get('phase'),
-                        condition.get('oxidant_source'),
-                        condition.get('driving_force'),
-                    ]
-                    if value
-                )
-                if condition_text:
-                    line += f" | condition: {condition_text}"
-
-            evidence = fact.get('evidence') or {}
-            if isinstance(evidence, dict):
-                status = evidence.get('support_status') or ''
-                level = evidence.get('evidence_level') or ''
-                evidence_text = ' '.join(filter(None, [status, level]))
-                icon = support_icon.get(status, '')
-                if icon:
-                    evidence_text = f"{icon} {evidence_text}"
-            else:
-                evidence_text = str(evidence)
-            if evidence_text:
-                line += f" | {evidence_text}"
-
-            formatted.append(line)
-
-        return '\n'.join(formatted)
-
-    @staticmethod
-    def _truncate_at_sentence(text: str, max_chars: int = 500) -> str:
-        """Truncate *text* at the last sentence boundary before *max_chars*."""
-        if len(text) <= max_chars:
-            return text
-        snippet = text[:max_chars]
-        for sep in ('. ', '.\n', '? ', '?\n', '! ', '!\n', '。', '；'):
-            idx = snippet.rfind(sep)
-            if idx > max_chars * 0.5:
-                return snippet[:idx + len(sep.rstrip())]
-        last_space = snippet.rfind(' ')
-        if last_space > max_chars * 0.5:
-            return snippet[:last_space]
-        return snippet
-
-    def format_evidence(self, evidence_list: List[Dict[str, Any]], max_items: int = 10) -> str:
-        """格式化证据，标注可信度等级和验证状态。"""
-        if not evidence_list:
-            return "No relevant evidence found."
-
-        level_badge = {
-            'high': '[HIGH confidence]',
-            'medium': '[MEDIUM confidence]',
-            'low': '[LOW confidence]',
-        }
-        support_badge = {
-            'supported': 'verified',
-            'partially_supported': 'partial',
-            'unsupported': 'unverified',
-        }
-
-        formatted = []
-        for i, evidence in enumerate(evidence_list[:max_items], 1):
-            source_type = evidence.get('source_type', 'unknown')
-            material = evidence.get('material_context', {}).get('material_type', 'N/A')
-            dominant = [item.get('species', 'N/A') for item in evidence.get('dominant_ros_or_product', [])]
-            evidence_level = evidence.get('evidence_level', 'low')
-            support_status = evidence.get('support_status', 'unsupported')
-
-            level_str = level_badge.get(evidence_level, evidence_level)
-            support_str = support_badge.get(support_status, support_status)
-            text = self._truncate_at_sentence(evidence.get('text', ''), max_chars=500)
-
-            formatted.append(
-                f"[{i}] {source_type} {level_str} ({support_str})\n"
-                f"    Material: {material}\n"
-                f"    Dominant ROS: {', '.join(dominant)}\n"
-                f"    {text}"
-            )
-        return '\n\n'.join(formatted)
-
-    def call_llm(self, system_prompt: str, user_prompt: str) -> str:
-        """调用本地/内网 LLM。"""
-        if self.llm_provider != 'ollama':
-            raise ValueError(f"Unsupported LLM provider: {self.llm_provider}")
-
-        if requests is None:
-            raise ImportError("requests required. Run: pip install requests")
-
-        prompt = f"{system_prompt.strip()}\n\n{user_prompt.strip()}"
-        payload = {
-            "model": self.model_name,
-            "prompt": prompt,
-            "stream": False,
-            "options": {
-                "temperature": self.temperature,
-                "top_p": self.config.get('generation', {}).get('top_p', 0.9),
-                "num_predict": self.max_new_tokens,
-            },
-        }
-
-        response = requests.post(self.generate_url, json=payload, timeout=self.timeout)
-        response.raise_for_status()
-        data = response.json()
-        return data.get("response", "").strip()
-
-    def answer_question(self, question: str, top_k: int = 5) -> Dict[str, Any]:
-        """回答问题"""
-        result = {
-            'question': question,
-            'timestamp': datetime.now().isoformat()
-        }
-
-        if self.retriever is None:
-            result['kg_facts'] = []
-            result['evidence'] = []
-            result['answer'] = "Evidence retrieval is unavailable. Build retrieval indexes before running KG-RAG inference."
-            return result
-
-        # 1. 检索 KG facts
-        kg_facts = self.retriever.retrieve_kg_facts(question, top_k)
-        result['kg_facts'] = kg_facts
-
-        # 2. 检索文献证据
-        evidence = self.retriever.retrieve_with_sources(question, top_k)
-        result['evidence'] = evidence
-
-        # 3. 构造 prompt
-        kg_facts_str = self.format_kg_facts(kg_facts)
-        evidence_str = self.format_evidence(evidence)
-
-        user_prompt = self.user_prompt_template.format(
-            question=question,
-            kg_facts=kg_facts_str,
-            evidence=evidence_str
-        )
-
-        # 4. 调用 LLM
-        answer = self.call_llm(self.system_prompt, user_prompt)
-        result['answer'] = answer
-
-        return result
-
-    def run_demo(self):
-        """运行 demo"""
-        logging.info("KG-RAG Inference Demo")
-        logging.info("=" * 50)
-
-        # 从 prompt_templates.yaml 加载 demo 问题
-        import yaml
-        prompt_path = Path("config/prompt_templates.yaml")
-        test_questions = [
-            "What ROS does TT-T-COF generate under visible light photocatalysis?",
-            "Which COF materials are reported to generate H2O2 as the main product?",
-            "What is the mechanism of ROS generation in donor-acceptor COFs?",
-        ]
-        if prompt_path.exists():
-            with open(prompt_path, "r", encoding="utf-8") as f:
-                prompt_config = yaml.safe_load(f)
-            test_questions = prompt_config.get("demo", {}).get("questions", test_questions)
-
-        for question in test_questions:
-            print(f"\n{'=' * 50}")
-            print(f"Question: {question}")
-            print("=" * 50)
-
-            result = self.answer_question(question)
-
-            print(f"\nAnswer:")
-            print(result['answer'])
-
-    def run(self):
-        """运行推理服务"""
-        logging.info("KG-RAG Inference initialized")
-        logging.info(f"KG facts loaded: {len(self.kg_facts)}")
-
-
-def main():
-    """主函数"""
-    import argparse
-
-    parser = argparse.ArgumentParser(description='Run KG-RAG inference for COF-ROS questions')
-    parser.add_argument('--config', type=str, default='config/config.yaml',
-                        help='Path to config file')
-    parser.add_argument('--question', type=str, default=None,
-                        help='Single question to answer')
-    parser.add_argument('--demo', action='store_true',
-                        help='Run demo mode')
-    args = parser.parse_args()
-
-    # 加载配置
-    config_path = Path(args.config)
-    with open(config_path, 'r', encoding='utf-8') as f:
-        config = yaml.safe_load(f)
-
-    # 初始化推理器
-    inference = KGRAGInference(config)
-
-    if args.question:
-        # 单个问题
-        result = inference.answer_question(args.question)
-        print(f"\nQuestion: {args.question}")
-        print(f"\nAnswer:\n{result['answer']}")
-    elif args.demo:
-        # Demo 模式
-        inference.run_demo()
-    else:
-        inference.run()
-
-
-if __name__ == '__main__':
-    main()

package/runtime/src/retrieval/build_faiss_index.py

@@ -1,328 +0,0 @@
-#!/usr/bin/env python3
-"""
-COF-ROS-Reasoner: FAISS 向量索引构建脚本
-
-目标：
-- 构建 FAISS 向量索引用于相似性检索
-- 支持来自 JSON evidence、PDF chunks、WOS abstracts 的文本
-"""
-
-import os
-import json
-import yaml
-import logging
-from pathlib import Path
-from typing import Dict, List, Any, Optional
-from datetime import datetime
-
-import numpy as np
-
-# 添加 src 路径
-
-try:
-    from sentence_transformers import SentenceTransformer
-except ImportError:
-    SentenceTransformer = None
-
-try:
-    import faiss
-except ImportError:
-    faiss = None
-
-from data_processing.utils import setup_logging, ensure_directory
-
-
-class FAISSIndexBuilder:
-    """构建 FAISS 向量索引"""
-
-    def __init__(self, config: Dict[str, Any]):
-        self.config = config
-        self.output_dir = Path(config['paths']['retrieval_store'])
-
-        # 设置日志
-        setup_logging(config)
-
-        # 嵌入模型配置
-        self.embedding_model_name = config.get('models', {}).get('embedding_model', '')
-
-        # 加载嵌入模型
-        self._load_embedding_model()
-
-        # 索引参数
-        self.dimension = 1024  # BGE-M3 的维度
-        self.index = None
-
-        # 存储元数据
-        self.metadata: List[Dict[str, Any]] = []
-
-    def _load_embedding_model(self):
-        """加载嵌入模型"""
-        if not self.embedding_model_name:
-            raise ValueError(
-                "FAISS is enabled but models.embedding_model is empty. "
-                "Set it to a local embedding model directory, or keep retrieval.enable_faiss=false."
-            )
-
-        if SentenceTransformer is None:
-            logging.error("sentence-transformers not installed. Run: pip install sentence-transformers")
-            raise ImportError("sentence-transformers required")
-
-        logging.info(f"Loading embedding model: {self.embedding_model_name}")
-        try:
-            self.model = SentenceTransformer(self.embedding_model_name)
-        except Exception as exc:
-            message = (
-                f"Failed to load embedding model '{self.embedding_model_name}'. "
-                "If this machine is offline, download the model on a machine with network access "
-                "and set models.embedding_model in config/config.yaml to the local model directory."
-            )
-            logging.error(message)
-            raise RuntimeError(message) from exc
-
-        # 获取模型维度
-        try:
-            self.dimension = self.model.get_sentence_embedding_dimension()
-        except Exception:
-            self.dimension = 1024
-
-        logging.info(f"Model dimension: {self.dimension}")
-
-    def create_index(self):
-        """创建 FAISS 索引"""
-        if faiss is None:
-            logging.error("faiss not installed. Run: pip install faiss-cpu")
-            raise ImportError("faiss required")
-
-        # 使用 IVFFlat 索引
-        nlist = 100  # 簇的数量
-        self.index = faiss.index_factory(self.dimension, f"IVF{nlist},Flat", faiss.METRIC_INNER_PRODUCT)
-
-        # 训练索引（需要一些样本）
-        logging.info("Training FAISS index...")
-        # 使用随机样本训练
-        samples = np.random.randn(1000, self.dimension).astype('float32')
-        self.index.train(samples)
-
-        logging.info("FAISS index created")
-
-    def embed_texts(self, texts: List[str]) -> np.ndarray:
-        """嵌入文本列表"""
-        embeddings = self.model.encode(texts, normalize_embeddings=True)
-        return embeddings.astype('float32')
-
-    def add_texts(self, texts: List[str], metadata: List[Dict[str, Any]]):
-        """添加文本到索引"""
-        if len(texts) == 0:
-            return
-
-        logging.info(f"Adding {len(texts)} texts to index...")
-
-        # 嵌入文本
-        embeddings = self.embed_texts(texts)
-
-        # 添加到 FAISS 索引
-        self.index.add(embeddings)
-
-        # 保存元数据
-        self.metadata.extend(metadata)
-
-        logging.info(f"Added {len(texts)} vectors. Total: {self.index.ntotal}")
-
-    def save_index(self):
-        """保存 FAISS 索引和元数据"""
-        ensure_directory(self.output_dir)
-
-        # 保存 FAISS 索引
-        index_path = self.output_dir / 'faiss_index.bin'
-        faiss.write_index(self.index, str(index_path))
-        logging.info(f"FAISS index saved to {index_path}")
-
-        # 保存元数据
-        metadata_path = self.output_dir / 'faiss_metadata.jsonl'
-        with open(metadata_path, 'w', encoding='utf-8') as f:
-            for meta in self.metadata:
-                f.write(json.dumps(meta, ensure_ascii=False) + '\n')
-        logging.info(f"Metadata saved to {metadata_path}")
-
-    def build_from_json_records(self, jsonl_path: Path):
-        """从 JSON 记录构建索引"""
-        logging.info(f"Building index from {jsonl_path}")
-
-        texts = []
-        metadata = []
-
-        if not jsonl_path.exists():
-            logging.warning(f"File not found: {jsonl_path}")
-            return
-
-        with open(jsonl_path, 'r', encoding='utf-8') as f:
-            for line in f:
-                if line.strip():
-                    record = json.loads(line)
-                    payload = record.get('extracted_json')
-                    if isinstance(payload, dict) and payload.get('status') != 'insufficient_evidence':
-                        source_record = payload
-                    else:
-                        source_record = record
-
-                    # 构造文本
-                    text_parts = []
-
-                    # 来自 JSON evidence
-                    dominant = source_record.get('dominant_ros_or_product', [])
-                    for item in dominant:
-                        species = item.get('species', '')
-                        evidence = item.get('evidence', '')
-                        if species:
-                            text_parts.append(f"Species: {species}")
-                        if evidence:
-                            text_parts.append(f"Evidence: {evidence}")
-
-                    secondary = source_record.get('secondary_or_intermediate_ros', [])
-                    for item in secondary:
-                        species = item.get('species', '')
-                        evidence = item.get('evidence', '')
-                        if species:
-                            text_parts.append(f"Secondary: {species}")
-                        if evidence:
-                            text_parts.append(f"Evidence: {evidence}")
-
-                    material = source_record.get('material_context', {})
-                    material_type = material.get('material_type', '')
-                    active_site = material.get('active_site', '')
-
-                    if material_type:
-                        text_parts.insert(0, f"Material: {material_type}")
-                    if active_site:
-                        text_parts.insert(1, f"Active site: {active_site}")
-
-                    reaction_env = source_record.get('reaction_environment', {})
-                    phase = reaction_env.get('phase', '')
-                    oxidant = reaction_env.get('oxidant_source', '')
-                    driving = reaction_env.get('driving_force', '')
-
-                    if phase:
-                        text_parts.append(f"Phase: {phase}")
-                    if oxidant:
-                        text_parts.append(f"Oxidant: {oxidant}")
-                    if driving:
-                        text_parts.append(f"Driving force: {driving}")
-
-                    mechanism = source_record.get('mechanism', [])
-                    for step in mechanism:
-                        if step:
-                            text_parts.append(f"Mechanism: {step}")
-
-                    # 合并文本
-                    text = ' '.join(text_parts)
-                    if not text.strip():
-                        continue
-
-                    texts.append(text)
-
-                    # 构造元数据
-                    meta = {
-                        'doc_id': record.get('source_chunk_id', record.get('chunk_id', '')),
-                        'source_type': 'gold_silver_json',
-                        'evidence_level': record.get('evidence_level', 'low'),
-                        'text': text
-                    }
-                    metadata.append(meta)
-
-        logging.info(f"Loaded {len(texts)} records from {jsonl_path}")
-        self.add_texts(texts, metadata)
-
-    def build_from_chunks(self, chunks_path: Path, source_type: str = 'pdf'):
-        """从 chunks 文件构建索引"""
-        logging.info(f"Building index from {chunks_path}")
-
-        texts = []
-        metadata = []
-
-        if not chunks_path.exists():
-            logging.warning(f"File not found: {chunks_path}")
-            return
-
-        with open(chunks_path, 'r', encoding='utf-8') as f:
-            for line in f:
-                if line.strip():
-                    record = json.loads(line)
-
-                    text = record.get('text', '')
-                    if not text.strip():
-                        continue
-
-                    # 如果只添加 ROS 相关的 chunks
-                    if source_type == 'pdf' and not record.get('is_ros_related', False):
-                        continue
-
-                    texts.append(text)
-
-                    meta = {
-                        'doc_id': record.get('chunk_id', ''),
-                        'source_type': source_type,
-                        'source': record.get('source_type', 'pdf_fulltext'),
-                        'title': record.get('title', ''),
-                        'doi': record.get('doi', ''),
-                        'page': record.get('page', 0),
-                        'is_ros_related': record.get('is_ros_related', False),
-                        'text': text
-                    }
-                    metadata.append(meta)
-
-        logging.info(f"Loaded {len(texts)} {source_type} chunks")
-        self.add_texts(texts, metadata)
-
-    def run(self):
-        """运行索引构建流程"""
-        logging.info("Starting FAISS index construction")
-
-        # 创建索引
-        self.create_index()
-
-        # 从 JSON 记录构建
-        gold_json_path = Path(self.config['paths']['processed_dir']) / 'json_records_cleaned.jsonl'
-        if gold_json_path.exists():
-            self.build_from_json_records(gold_json_path)
-
-        json_path = Path(self.config['paths']['processed_dir']) / 'silver_json_validated.jsonl'
-        if json_path.exists():
-            self.build_from_json_records(json_path)
-
-        # 从 PDF chunks 构建
-        pdf_path = Path(self.config['paths']['processed_dir']) / 'pdf_chunks.jsonl'
-        if pdf_path.exists():
-            self.build_from_chunks(pdf_path, 'pdf')
-
-        # 从 WoS abstracts 构建
-        wos_path = Path(self.config['paths']['processed_dir']) / 'wos_abstract_chunks.jsonl'
-        if wos_path.exists():
-            self.build_from_chunks(wos_path, 'wos')
-
-        # 保存索引
-        self.save_index()
-
-        logging.info(f"FAISS index construction completed. Total vectors: {self.index.ntotal}")
-
-
-def main():
-    """主函数"""
-    import argparse
-
-    parser = argparse.ArgumentParser(description='Build FAISS index for COF-ROS retrieval')
-    parser.add_argument('--config', type=str, default='config/config.yaml',
-                        help='Path to config file')
-    args = parser.parse_args()
-
-    # 加载配置
-    config_path = Path(args.config)
-    with open(config_path, 'r', encoding='utf-8') as f:
-        config = yaml.safe_load(f)
-
-    # 运行构建
-    builder = FAISSIndexBuilder(config)
-    builder.run()
-
-
-if __name__ == '__main__':
-    main()