@lzhzzzzwill/cofos 1.0.4 → 1.1.1

package/runtime/src/data_processing/clean_json_records.py

@@ -1,387 +0,0 @@
-#!/usr/bin/env python3
-"""
-COF-ROS-Reasoner: 人工 JSON 清洗脚本
-
-目标：
-- 把已有人工 JSON 统一成标准字段
-- 检查 JSON 合法性
-- 统一字段名
-- 标准化 ROS / product 名称
-- 保存坏 JSON 到 error log
-"""
-
-import os
-import json
-import logging
-from pathlib import Path
-from typing import Dict, List, Any, Optional, Tuple
-from datetime import datetime
-
-# 添加 src 路径
-
-from data_processing.utils import setup_logging
-
-
-class JSONCleaner:
-    """清洗和标准化人工标注的 JSON 数据"""
-
-    def __init__(self, config: Dict[str, Any]):
-        self.config = config
-        self.ros_normalization = config.get('data_processing', {}).get('ros_normalization', {})
-        self.raw_json_path = Path(config['paths']['raw_json'])
-        self.output_path = Path(config['paths']['processed_dir']) / 'json_records_cleaned.jsonl'
-        self.error_log_path = Path(config['paths']['outputs_dir']) / 'logs' / 'json_cleaning_errors.log'
-
-        # 设置日志
-        setup_logging(config)
-
-        # 常用字段映射
-        self.field_mappings = {
-            'dominant_ros': 'dominant_ros_or_product',
-            'secondary_ros': 'secondary_or_intermediate_ros',
-            'intermediates': 'mechanism_intermediates',
-            'reaction_conditions': 'reaction_environment',
-            'material': 'material_context',
-            'mechanism_path': 'mechanism',
-            'reaction_conditions': 'reaction_environment',
-        }
-
-        # 验证字段
-        self.required_fields = {
-            'material_context',
-            'reaction_environment',
-            'dominant_ros_or_product',
-            'secondary_or_intermediate_ros',
-            'mechanism_intermediates',
-            'mechanism'
-        }
-
-        self.material_context_fields = {'material_type', 'active_site', 'structural_feature'}
-        self.reaction_environment_fields = {'phase', 'oxidant_source', 'driving_force', 'notes'}
-        self.ros_fields = {'species', 'category', 'role', 'basis', 'evidence'}
-        self.secondary_ros_fields = {'species', 'role', 'basis', 'evidence'}
-        self.intermediate_fields = {'species', 'notation_in_paper', 'role', 'evidence'}
-
-    def safe_str(self, value: Any) -> str:
-        """Convert arbitrary JSON values to reviewable strings."""
-        if value is None:
-            return ""
-        if isinstance(value, str):
-            return value.strip()
-        if isinstance(value, (int, float, bool)):
-            return str(value)
-        return json.dumps(value, ensure_ascii=False)
-
-    def normalize_ros_name(self, name: str) -> str:
-        """标准化 ROS 名称"""
-        if not name:
-            return ""
-
-        name = name.strip()
-
-        # 直接匹配
-        if name in self.ros_normalization:
-            return self.ros_normalization[name]
-
-        # 忽略大小写匹配
-        name_lower = name.lower()
-        for key, value in self.ros_normalization.items():
-            if key.lower() == name_lower:
-                return value
-
-        # 返回原名称（如果无法标准化）
-        return name
-
-    def is_valid_ros_name(self, name: str) -> bool:
-        """检查 ROS 名称是否有效（可以被标准化或本身就是有效名称）"""
-        if not name:
-            return False
-
-        normalized = self.normalize_ros_name(name)
-        return normalized != ""
-
-    def clean_material_context(self, data: Dict[str, Any]) -> Dict[str, Any]:
-        """清洗 material_context 字段"""
-        material = {}
-
-        # 处理可能的字段名
-        raw_material = data.get('material_context', data.get('material', {}))
-
-        if isinstance(raw_material, dict):
-            for field in self.material_context_fields:
-                value = raw_material.get(field, '')
-                material[field] = self.safe_str(value)
-
-        return material
-
-    def clean_reaction_environment(self, data: Dict[str, Any]) -> Dict[str, Any]:
-        """清洗 reaction_environment 字段"""
-        env = {}
-
-        # 处理可能的字段名
-        raw_env = data.get('reaction_environment', data.get('reaction_conditions', data.get('conditions', {})))
-
-        if isinstance(raw_env, dict):
-            for field in self.reaction_environment_fields:
-                value = raw_env.get(field, '')
-                env[field] = self.safe_str(value)
-
-        return env
-
-    def clean_ros_list(self, ros_list: List[Dict[str, Any]], is_dominant: bool = True) -> List[Dict[str, Any]]:
-        """清洗 ROS 列表"""
-        if not isinstance(ros_list, list):
-            return []
-
-        cleaned = []
-        fields = self.ros_fields if is_dominant else self.secondary_ros_fields
-
-        for item in ros_list:
-            if not isinstance(item, dict):
-                continue
-
-            cleaned_item = {}
-            for field in fields:
-                value = item.get(field, '')
-                cleaned_item[field] = self.safe_str(value)
-
-            # 标准化 species
-            if 'species' in cleaned_item:
-                cleaned_item['species'] = self.normalize_ros_name(cleaned_item['species'])
-
-            # 如果是 dominant ROS，添加 category 字段（如果没有）
-            if is_dominant and 'category' not in cleaned_item:
-                species = cleaned_item.get('species', '').lower()
-                # H2O2 在 2e- ORR 中应为 oxygen-derived target product
-                if 'h2o2' in species or 'hydrogen peroxide' in species:
-                    cleaned_item['category'] = 'oxygen-derived target product'
-                else:
-                    cleaned_item['category'] = 'radical ROS'
-
-            cleaned.append(cleaned_item)
-
-        return cleaned
-
-    def clean_mechanism_intermediates(self, intermediates: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
-        """清洗 mechanism_intermediates 字段"""
-        if not isinstance(intermediates, list):
-            return []
-
-        cleaned = []
-        for item in intermediates:
-            if not isinstance(item, dict):
-                continue
-
-            cleaned_item = {}
-            for field in self.intermediate_fields:
-                value = item.get(field, '')
-                cleaned_item[field] = self.safe_str(value)
-
-            # 标准化 species
-            if 'species' in cleaned_item:
-                cleaned_item['species'] = self.normalize_ros_name(cleaned_item['species'])
-
-            cleaned.append(cleaned_item)
-
-        return cleaned
-
-    def clean_mechanism(self, mechanism: Any) -> List[str]:
-        """清洗 mechanism 字段"""
-        if isinstance(mechanism, list):
-            return [self.safe_str(step) for step in mechanism if self.safe_str(step)]
-        elif isinstance(mechanism, str):
-            if mechanism.strip():
-                return [mechanism.strip()]
-        elif mechanism:
-            return [self.safe_str(mechanism)]
-        return []
-
-    def validate_record(self, record: Dict[str, Any]) -> Tuple[bool, List[str]]:
-        """验证清洗后的记录"""
-        errors = []
-
-        # 检查 required fields
-        for field in self.required_fields:
-            if field not in record:
-                errors.append(f"Missing required field: {field}")
-
-        # 检查 material_context
-        material = record.get('material_context', {})
-        if not material.get('material_type'):
-            errors.append("material_context.material_type is empty")
-
-        # 检查 reaction_environment
-        env = record.get('reaction_environment', {})
-        if not env.get('phase'):
-            errors.append("reaction_environment.phase is empty")
-
-        # 检查至少有一个 dominant ROS
-        dominant = record.get('dominant_ros_or_product', [])
-        if not dominant:
-            errors.append("dominant_ros_or_product is empty")
-
-        # 检查 species 格式
-        for item in dominant:
-            species = item.get('species', '')
-            if species and not self.is_valid_ros_name(species):
-                errors.append(f"Invalid ROS species: {species}")
-
-        return len(errors) == 0, errors
-
-    def clean_record(self, record: Dict[str, Any]) -> Dict[str, Any]:
-        """清洗单个 JSON 记录"""
-        cleaned = {}
-
-        # 清洗各个字段
-        cleaned['material_context'] = self.clean_material_context(record)
-        cleaned['reaction_environment'] = self.clean_reaction_environment(record)
-        cleaned['dominant_ros_or_product'] = self.clean_ros_list(
-            record.get('dominant_ros_or_product', record.get('dominant_ros', [])),
-            is_dominant=True
-        )
-        cleaned['secondary_or_intermediate_ros'] = self.clean_ros_list(
-            record.get('secondary_or_intermediate_ros', record.get('secondary_ros', [])),
-            is_dominant=False
-        )
-        cleaned['mechanism_intermediates'] = self.clean_mechanism_intermediates(
-            record.get('mechanism_intermediates', [])
-        )
-        cleaned['mechanism'] = self.clean_mechanism(record.get('mechanism', []))
-
-        # 添加元数据
-        cleaned['_source'] = self.safe_str(record.get('_source', 'unknown'))
-        cleaned['_cleaned_at'] = datetime.now().isoformat()
-
-        return cleaned
-
-    def process_file(self, file_path: Path) -> Tuple[int, int]:
-        """处理单个 JSON 文件"""
-        success_count = 0
-        error_count = 0
-
-        try:
-            with open(file_path, 'r', encoding='utf-8') as f:
-                content = f.read()
-
-            # 尝试解析 JSON
-            try:
-                data = json.loads(content)
-            except json.JSONDecodeError:
-                # 可能是 JSONL 格式
-                lines = content.strip().split('\n')
-                for line in lines:
-                    if line.strip():
-                        try:
-                            data = json.loads(line)
-                            self.process_single_record(data, file_path)
-                            success_count += 1
-                        except json.JSONDecodeError as e:
-                            self._log_error(file_path, f"Invalid JSON: {e}")
-                            error_count += 1
-                return success_count, error_count
-
-            # 处理单个记录或记录列表
-            if isinstance(data, list):
-                for record in data:
-                    self.process_single_record(record, file_path)
-                    success_count += 1
-            else:
-                self.process_single_record(data, file_path)
-                success_count += 1
-
-        except Exception as e:
-            self._log_error(file_path, str(e))
-            error_count += 1
-
-        return success_count, error_count
-
-    def process_single_record(self, record: Dict[str, Any], source_file: str = ""):
-        """处理单个记录"""
-        try:
-            # 添加源文件信息
-            if source_file:
-                record['_source'] = str(source_file)
-
-            # 清洗记录
-            cleaned = self.clean_record(record)
-
-            # 验证
-            is_valid, errors = self.validate_record(cleaned)
-
-            if is_valid:
-                # 写入输出文件
-                with open(self.output_path, 'a', encoding='utf-8') as f:
-                    f.write(json.dumps(cleaned, ensure_ascii=False) + '\n')
-            else:
-                # 记录错误
-                error_msg = f"Validation errors in {source_file}: {errors}"
-                self._log_error(source_file, error_msg)
-
-        except Exception as e:
-            self._log_error(source_file, str(e))
-
-    def _log_error(self, source: str, message: str):
-        """记录错误到日志文件"""
-        error_entry = {
-            'timestamp': datetime.now().isoformat(),
-            'source': str(source),
-            'error': message
-        }
-        with open(self.error_log_path, 'a', encoding='utf-8') as f:
-            f.write(json.dumps(error_entry, ensure_ascii=False) + '\n')
-        logging.error(f"Error processing {source}: {message}")
-
-    def run(self):
-        """运行清洗流程"""
-        logging.info(f"Starting JSON cleaning from {self.raw_json_path}")
-        logging.info(f"Output will be saved to {self.output_path}")
-
-        # 清空输出文件
-        self.output_path.parent.mkdir(parents=True, exist_ok=True)
-        self.error_log_path.parent.mkdir(parents=True, exist_ok=True)
-        self.output_path.write_text('', encoding='utf-8')
-        self.error_log_path.write_text('', encoding='utf-8')
-
-        total_success = 0
-        total_error = 0
-
-        # 处理所有 JSON 文件
-        for file_path in self.raw_json_path.glob('*.json'):
-            logging.info(f"Processing {file_path.name}")
-            success, error = self.process_file(file_path)
-            total_success += success
-            total_error += error
-
-        # 处理 JSONL 文件
-        for file_path in self.raw_json_path.glob('*.jsonl'):
-            logging.info(f"Processing {file_path.name}")
-            success, error = self.process_file(file_path)
-            total_success += success
-            total_error += error
-
-        logging.info(f"JSON cleaning completed: {total_success} records cleaned, {total_error} errors")
-        logging.info(f"Cleaned records saved to {self.output_path}")
-        logging.info(f"Errors logged to {self.error_log_path}")
-
-
-def main():
-    """主函数"""
-    import argparse
-
-    parser = argparse.ArgumentParser(description='Clean and normalize COF-ROS JSON records')
-    parser.add_argument('--config', type=str, default='config/config.yaml',
-                        help='Path to config file')
-    args = parser.parse_args()
-
-    # 加载配置
-    config_path = Path(args.config)
-    with open(config_path, 'r', encoding='utf-8') as f:
-        config = yaml.safe_load(f)
-
-    # 运行清洗
-    cleaner = JSONCleaner(config)
-    cleaner.run()
-
-
-if __name__ == '__main__':
-    main()

package/runtime/src/data_processing/parse_wos_xlsx.py

@@ -1,255 +0,0 @@
-#!/usr/bin/env python3
-"""
-COF-ROS-Reasoner: WoS 摘要解析脚本
-
-目标：
-- 从 WoS xlsx 中提取摘要、题名、DOI、年份、期刊等信息
-- 输出为标准化的 JSONL 格式
-"""
-
-import os
-import json
-import yaml
-import logging
-from pathlib import Path
-from typing import Dict, List, Any, Optional
-from datetime import datetime
-import math
-
-import pandas as pd
-
-# 添加 src 路径
-
-from data_processing.utils import setup_logging, ensure_directory
-
-
-class WoSParser:
-    """解析 WoS 导出的 XLSX 文件"""
-
-    def __init__(self, config: Dict[str, Any]):
-        self.config = config
-        self.raw_wos_path = Path(config['paths']['raw_wos'])
-        self.output_path = Path(config['paths']['processed_dir']) / 'wos_abstract_chunks.jsonl'
-
-        # 设置日志
-        setup_logging(config)
-
-        # 保留字段映射
-        self.field_mappings = {
-            'Article Title': 'title',
-            'Abstract': 'abstract',
-            'Abstract Note': 'abstract',
-            'DOI': 'doi',
-            'Publication Year': 'year',
-            'Year Published': 'year',
-            'PY': 'year',
-            'Source Title': 'source_title',
-            'Publication Name': 'source_title',
-            'Author Keywords': 'author_keywords',
-            'Author Keywords (DE)': 'author_keywords',
-            'Keywords Plus': 'keywords_plus',
-            'Keywords Plus (ID)': 'keywords_plus',
-            'UT': 'ut',  # WOS Unique ID
-            'WOS Unique ID': 'ut',
-        }
-
-        # 需要解析的关键词字段
-        self.keyword_fields = ['Author Keywords', 'Keywords Plus']
-
-        # ROS 关键词（用于筛选）
-        self.ros_keywords = config.get('data_processing', {}).get('ros_keywords', [])
-
-    def normalize_column_key(self, value: Any) -> str:
-        """Normalize a column name for matching."""
-        return ' '.join(self.safe_str(value).lower().split())
-
-    def normalize_dataframe_columns(self, df: pd.DataFrame) -> pd.DataFrame:
-        """Normalize common WoS column names while preserving unknown columns."""
-        rename_map = {}
-        normalized_targets = set()
-        normalized_mapping = {
-            self.normalize_column_key(source): target
-            for source, target in self.field_mappings.items()
-        }
-
-        for column in df.columns:
-            raw = self.safe_str(column)
-            target = self.field_mappings.get(raw)
-            if not target:
-                target = self.field_mappings.get(raw.strip())
-            if not target:
-                target = normalized_mapping.get(self.normalize_column_key(raw))
-
-            if target and target not in normalized_targets:
-                rename_map[column] = target
-                normalized_targets.add(target)
-
-        normalized = df.rename(columns=rename_map)
-        logging.info(f"WoS columns: {list(df.columns)}")
-        logging.info(f"Normalized WoS columns: {list(normalized.columns)}")
-        return normalized
-
-    def normalize_field_name(self, field: str) -> str:
-        """标准化字段名"""
-        field = self.safe_str(field).strip()
-        return self.field_mappings.get(field, field)
-
-    def safe_str(self, value: Any) -> str:
-        """Convert Excel cell values to clean strings."""
-        if value is None:
-            return ""
-
-        try:
-            if pd.isna(value):
-                return ""
-        except TypeError:
-            pass
-
-        if isinstance(value, float) and math.isfinite(value) and value.is_integer():
-            return str(int(value))
-
-        return str(value).strip()
-
-    def parse_year(self, value: Any) -> Optional[int]:
-        """Safely parse publication year."""
-        text = self.safe_str(value)
-        if not text:
-            return None
-
-        try:
-            return int(float(text))
-        except ValueError:
-            logging.warning(f"Could not parse publication year: {text}")
-            return None
-
-    def parse_keywords(self, keywords_str: Any) -> List[str]:
-        """解析关键词字符串"""
-        keywords_str = self.safe_str(keywords_str)
-        if not keywords_str:
-            return []
-
-        # 分隔符可能是 ; 或 |
-        if ';' in keywords_str:
-            keywords = [k.strip() for k in keywords_str.split(';')]
-        elif '|' in keywords_str:
-            keywords = [k.strip() for k in keywords_str.split('|')]
-        else:
-            keywords = [keywords_str.strip()]
-
-        # 清理空值
-        return [k for k in keywords if k]
-
-    def is_ros_related(self, title: Any, abstract: Any, keywords: List[str] = None) -> bool:
-        """判断是否与 ROS 相关"""
-        keyword_text = ' '.join(keywords or [])
-        text = f"{self.safe_str(title)} {self.safe_str(abstract)} {keyword_text}"
-        text_lower = text.lower()
-
-        for keyword in self.ros_keywords:
-            if keyword.lower() in text_lower:
-                return True
-        return False
-
-    def parse_row(self, row: pd.Series, index: int) -> Optional[Dict[str, Any]]:
-        """解析单行数据"""
-        try:
-            # 提取基本信息
-            title = self.safe_str(row.get('title', ''))
-            abstract = self.safe_str(row.get('abstract', ''))
-            doi = self.safe_str(row.get('doi', ''))
-            year = self.parse_year(row.get('year', ''))
-            source_title = self.safe_str(row.get('source_title', ''))
-            ut = self.safe_str(row.get('ut', ''))
-
-            # 解析关键词
-            author_keywords = self.parse_keywords(row.get('author_keywords', ''))
-            keywords_plus = self.parse_keywords(row.get('keywords_plus', ''))
-
-            # 合并关键词
-            keywords = list(set(author_keywords + keywords_plus))
-
-            # 如果没有摘要，使用标题作为文本
-            if not abstract:
-                text = title
-            else:
-                text = abstract
-
-            # 构建记录
-            record = {
-                'doc_id': f"WOS:{ut}" if ut else f"WOS:index_{index}",
-                'doi': doi,
-                'title': title,
-                'source_type': 'wos_abstract',
-                'year': year,
-                'journal': source_title,
-                'text': text.strip(),
-                'keywords': keywords,
-                'is_ros_related': self.is_ros_related(title, abstract, keywords)
-            }
-
-            return record
-
-        except Exception as e:
-            logging.error(f"Error parsing row {index}: {e}")
-            return None
-
-    def run(self):
-        """运行解析流程"""
-        logging.info(f"Starting WoS parsing from {self.raw_wos_path}")
-        logging.info(f"Output will be saved to {self.output_path}")
-
-        # 检查输入文件
-        if not self.raw_wos_path.exists():
-            logging.error(f"Input file not found: {self.raw_wos_path}")
-            return
-
-        # 读取 XLSX
-        try:
-            df = pd.read_excel(self.raw_wos_path)
-            df = self.normalize_dataframe_columns(df)
-            logging.info(f"Loaded {len(df)} rows from {self.raw_wos_path}")
-        except Exception as e:
-            logging.error(f"Failed to read Excel file: {e}")
-            return
-
-        # 处理记录
-        records = []
-        for idx, row in df.iterrows():
-            record = self.parse_row(row, idx)
-            if record:
-                records.append(record)
-
-        # 保存输出
-        ensure_directory(self.output_path.parent)
-        with open(self.output_path, 'w', encoding='utf-8') as f:
-            for record in records:
-                f.write(json.dumps(record, ensure_ascii=False) + '\n')
-
-        # 统计信息
-        ros_related = sum(1 for r in records if r.get('is_ros_related', False))
-        logging.info(f"WoS parsing completed: {len(records)} records processed")
-        logging.info(f"ROS-related records: {ros_related}")
-        logging.info(f"Output saved to {self.output_path}")
-
-
-def main():
-    """主函数"""
-    import argparse
-
-    parser = argparse.ArgumentParser(description='Parse WoS abstracts from Excel file')
-    parser.add_argument('--config', type=str, default='config/config.yaml',
-                        help='Path to config file')
-    args = parser.parse_args()
-
-    # 加载配置
-    config_path = Path(args.config)
-    with open(config_path, 'r', encoding='utf-8') as f:
-        config = yaml.safe_load(f)
-
-    # 运行解析
-    parser = WoSParser(config)
-    parser.run()
-
-
-if __name__ == '__main__':
-    main()

package/runtime/src/data_processing/schema_utils.py

@@ -1,56 +0,0 @@
-"""Canonical schema-boundary normalization shared by downstream stages."""
-
-import json
-from typing import Any, Dict, List
-
-
-class SchemaNormalizationMixin:
-    """Normalize accepted legacy shapes once through a shared implementation."""
-
-    @staticmethod
-    def safe_str(value: Any) -> str:
-        if value is None:
-            return ""
-        if isinstance(value, str):
-            return value.strip()
-        if isinstance(value, (int, float, bool)):
-            return str(value)
-        return json.dumps(value, ensure_ascii=False)
-
-    @staticmethod
-    def as_list(value: Any) -> List[Any]:
-        if value is None:
-            return []
-        if isinstance(value, list):
-            return value
-        return [value]
-
-    def normalize_item_list(
-        self, value: Any, default_key: str = "species"
-    ) -> List[Dict[str, Any]]:
-        items = []
-        for item in self.as_list(value):
-            if isinstance(item, dict):
-                items.append(item)
-            elif item:
-                items.append({default_key: self.safe_str(item)})
-        return items
-
-    def normalize_ros(self, name: Any) -> str:
-        normalized_name = self.safe_str(name)
-        if not normalized_name:
-            return ""
-        mapping = self.config.get("data_processing", {}).get(
-            "ros_normalization", {}
-        )
-        return mapping.get(
-            normalized_name,
-            mapping.get(normalized_name.lower(), normalized_name),
-        )
-
-    def first_species(self, value: Any) -> str:
-        for item in self.normalize_item_list(value):
-            species = self.normalize_ros(item.get("species", ""))
-            if species:
-                return species
-        return ""