@lzhzzzzwill/cofos 1.0.1 → 1.0.4

package/runtime/src/data_processing/clean_json_records.py

@@ -0,0 +1,387 @@
+#!/usr/bin/env python3
+"""
+COF-ROS-Reasoner: 人工 JSON 清洗脚本
+
+目标：
+- 把已有人工 JSON 统一成标准字段
+- 检查 JSON 合法性
+- 统一字段名
+- 标准化 ROS / product 名称
+- 保存坏 JSON 到 error log
+"""
+
+import os
+import json
+import logging
+from pathlib import Path
+from typing import Dict, List, Any, Optional, Tuple
+from datetime import datetime
+
+# 添加 src 路径
+
+from data_processing.utils import setup_logging
+
+
+class JSONCleaner:
+    """清洗和标准化人工标注的 JSON 数据"""
+
+    def __init__(self, config: Dict[str, Any]):
+        self.config = config
+        self.ros_normalization = config.get('data_processing', {}).get('ros_normalization', {})
+        self.raw_json_path = Path(config['paths']['raw_json'])
+        self.output_path = Path(config['paths']['processed_dir']) / 'json_records_cleaned.jsonl'
+        self.error_log_path = Path(config['paths']['outputs_dir']) / 'logs' / 'json_cleaning_errors.log'
+
+        # 设置日志
+        setup_logging(config)
+
+        # 常用字段映射
+        self.field_mappings = {
+            'dominant_ros': 'dominant_ros_or_product',
+            'secondary_ros': 'secondary_or_intermediate_ros',
+            'intermediates': 'mechanism_intermediates',
+            'reaction_conditions': 'reaction_environment',
+            'material': 'material_context',
+            'mechanism_path': 'mechanism',
+            'reaction_conditions': 'reaction_environment',
+        }
+
+        # 验证字段
+        self.required_fields = {
+            'material_context',
+            'reaction_environment',
+            'dominant_ros_or_product',
+            'secondary_or_intermediate_ros',
+            'mechanism_intermediates',
+            'mechanism'
+        }
+
+        self.material_context_fields = {'material_type', 'active_site', 'structural_feature'}
+        self.reaction_environment_fields = {'phase', 'oxidant_source', 'driving_force', 'notes'}
+        self.ros_fields = {'species', 'category', 'role', 'basis', 'evidence'}
+        self.secondary_ros_fields = {'species', 'role', 'basis', 'evidence'}
+        self.intermediate_fields = {'species', 'notation_in_paper', 'role', 'evidence'}
+
+    def safe_str(self, value: Any) -> str:
+        """Convert arbitrary JSON values to reviewable strings."""
+        if value is None:
+            return ""
+        if isinstance(value, str):
+            return value.strip()
+        if isinstance(value, (int, float, bool)):
+            return str(value)
+        return json.dumps(value, ensure_ascii=False)
+
+    def normalize_ros_name(self, name: str) -> str:
+        """标准化 ROS 名称"""
+        if not name:
+            return ""
+
+        name = name.strip()
+
+        # 直接匹配
+        if name in self.ros_normalization:
+            return self.ros_normalization[name]
+
+        # 忽略大小写匹配
+        name_lower = name.lower()
+        for key, value in self.ros_normalization.items():
+            if key.lower() == name_lower:
+                return value
+
+        # 返回原名称（如果无法标准化）
+        return name
+
+    def is_valid_ros_name(self, name: str) -> bool:
+        """检查 ROS 名称是否有效（可以被标准化或本身就是有效名称）"""
+        if not name:
+            return False
+
+        normalized = self.normalize_ros_name(name)
+        return normalized != ""
+
+    def clean_material_context(self, data: Dict[str, Any]) -> Dict[str, Any]:
+        """清洗 material_context 字段"""
+        material = {}
+
+        # 处理可能的字段名
+        raw_material = data.get('material_context', data.get('material', {}))
+
+        if isinstance(raw_material, dict):
+            for field in self.material_context_fields:
+                value = raw_material.get(field, '')
+                material[field] = self.safe_str(value)
+
+        return material
+
+    def clean_reaction_environment(self, data: Dict[str, Any]) -> Dict[str, Any]:
+        """清洗 reaction_environment 字段"""
+        env = {}
+
+        # 处理可能的字段名
+        raw_env = data.get('reaction_environment', data.get('reaction_conditions', data.get('conditions', {})))
+
+        if isinstance(raw_env, dict):
+            for field in self.reaction_environment_fields:
+                value = raw_env.get(field, '')
+                env[field] = self.safe_str(value)
+
+        return env
+
+    def clean_ros_list(self, ros_list: List[Dict[str, Any]], is_dominant: bool = True) -> List[Dict[str, Any]]:
+        """清洗 ROS 列表"""
+        if not isinstance(ros_list, list):
+            return []
+
+        cleaned = []
+        fields = self.ros_fields if is_dominant else self.secondary_ros_fields
+
+        for item in ros_list:
+            if not isinstance(item, dict):
+                continue
+
+            cleaned_item = {}
+            for field in fields:
+                value = item.get(field, '')
+                cleaned_item[field] = self.safe_str(value)
+
+            # 标准化 species
+            if 'species' in cleaned_item:
+                cleaned_item['species'] = self.normalize_ros_name(cleaned_item['species'])
+
+            # 如果是 dominant ROS，添加 category 字段（如果没有）
+            if is_dominant and 'category' not in cleaned_item:
+                species = cleaned_item.get('species', '').lower()
+                # H2O2 在 2e- ORR 中应为 oxygen-derived target product
+                if 'h2o2' in species or 'hydrogen peroxide' in species:
+                    cleaned_item['category'] = 'oxygen-derived target product'
+                else:
+                    cleaned_item['category'] = 'radical ROS'
+
+            cleaned.append(cleaned_item)
+
+        return cleaned
+
+    def clean_mechanism_intermediates(self, intermediates: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+        """清洗 mechanism_intermediates 字段"""
+        if not isinstance(intermediates, list):
+            return []
+
+        cleaned = []
+        for item in intermediates:
+            if not isinstance(item, dict):
+                continue
+
+            cleaned_item = {}
+            for field in self.intermediate_fields:
+                value = item.get(field, '')
+                cleaned_item[field] = self.safe_str(value)
+
+            # 标准化 species
+            if 'species' in cleaned_item:
+                cleaned_item['species'] = self.normalize_ros_name(cleaned_item['species'])
+
+            cleaned.append(cleaned_item)
+
+        return cleaned
+
+    def clean_mechanism(self, mechanism: Any) -> List[str]:
+        """清洗 mechanism 字段"""
+        if isinstance(mechanism, list):
+            return [self.safe_str(step) for step in mechanism if self.safe_str(step)]
+        elif isinstance(mechanism, str):
+            if mechanism.strip():
+                return [mechanism.strip()]
+        elif mechanism:
+            return [self.safe_str(mechanism)]
+        return []
+
+    def validate_record(self, record: Dict[str, Any]) -> Tuple[bool, List[str]]:
+        """验证清洗后的记录"""
+        errors = []
+
+        # 检查 required fields
+        for field in self.required_fields:
+            if field not in record:
+                errors.append(f"Missing required field: {field}")
+
+        # 检查 material_context
+        material = record.get('material_context', {})
+        if not material.get('material_type'):
+            errors.append("material_context.material_type is empty")
+
+        # 检查 reaction_environment
+        env = record.get('reaction_environment', {})
+        if not env.get('phase'):
+            errors.append("reaction_environment.phase is empty")
+
+        # 检查至少有一个 dominant ROS
+        dominant = record.get('dominant_ros_or_product', [])
+        if not dominant:
+            errors.append("dominant_ros_or_product is empty")
+
+        # 检查 species 格式
+        for item in dominant:
+            species = item.get('species', '')
+            if species and not self.is_valid_ros_name(species):
+                errors.append(f"Invalid ROS species: {species}")
+
+        return len(errors) == 0, errors
+
+    def clean_record(self, record: Dict[str, Any]) -> Dict[str, Any]:
+        """清洗单个 JSON 记录"""
+        cleaned = {}
+
+        # 清洗各个字段
+        cleaned['material_context'] = self.clean_material_context(record)
+        cleaned['reaction_environment'] = self.clean_reaction_environment(record)
+        cleaned['dominant_ros_or_product'] = self.clean_ros_list(
+            record.get('dominant_ros_or_product', record.get('dominant_ros', [])),
+            is_dominant=True
+        )
+        cleaned['secondary_or_intermediate_ros'] = self.clean_ros_list(
+            record.get('secondary_or_intermediate_ros', record.get('secondary_ros', [])),
+            is_dominant=False
+        )
+        cleaned['mechanism_intermediates'] = self.clean_mechanism_intermediates(
+            record.get('mechanism_intermediates', [])
+        )
+        cleaned['mechanism'] = self.clean_mechanism(record.get('mechanism', []))
+
+        # 添加元数据
+        cleaned['_source'] = self.safe_str(record.get('_source', 'unknown'))
+        cleaned['_cleaned_at'] = datetime.now().isoformat()
+
+        return cleaned
+
+    def process_file(self, file_path: Path) -> Tuple[int, int]:
+        """处理单个 JSON 文件"""
+        success_count = 0
+        error_count = 0
+
+        try:
+            with open(file_path, 'r', encoding='utf-8') as f:
+                content = f.read()
+
+            # 尝试解析 JSON
+            try:
+                data = json.loads(content)
+            except json.JSONDecodeError:
+                # 可能是 JSONL 格式
+                lines = content.strip().split('\n')
+                for line in lines:
+                    if line.strip():
+                        try:
+                            data = json.loads(line)
+                            self.process_single_record(data, file_path)
+                            success_count += 1
+                        except json.JSONDecodeError as e:
+                            self._log_error(file_path, f"Invalid JSON: {e}")
+                            error_count += 1
+                return success_count, error_count
+
+            # 处理单个记录或记录列表
+            if isinstance(data, list):
+                for record in data:
+                    self.process_single_record(record, file_path)
+                    success_count += 1
+            else:
+                self.process_single_record(data, file_path)
+                success_count += 1
+
+        except Exception as e:
+            self._log_error(file_path, str(e))
+            error_count += 1
+
+        return success_count, error_count
+
+    def process_single_record(self, record: Dict[str, Any], source_file: str = ""):
+        """处理单个记录"""
+        try:
+            # 添加源文件信息
+            if source_file:
+                record['_source'] = str(source_file)
+
+            # 清洗记录
+            cleaned = self.clean_record(record)
+
+            # 验证
+            is_valid, errors = self.validate_record(cleaned)
+
+            if is_valid:
+                # 写入输出文件
+                with open(self.output_path, 'a', encoding='utf-8') as f:
+                    f.write(json.dumps(cleaned, ensure_ascii=False) + '\n')
+            else:
+                # 记录错误
+                error_msg = f"Validation errors in {source_file}: {errors}"
+                self._log_error(source_file, error_msg)
+
+        except Exception as e:
+            self._log_error(source_file, str(e))
+
+    def _log_error(self, source: str, message: str):
+        """记录错误到日志文件"""
+        error_entry = {
+            'timestamp': datetime.now().isoformat(),
+            'source': str(source),
+            'error': message
+        }
+        with open(self.error_log_path, 'a', encoding='utf-8') as f:
+            f.write(json.dumps(error_entry, ensure_ascii=False) + '\n')
+        logging.error(f"Error processing {source}: {message}")
+
+    def run(self):
+        """运行清洗流程"""
+        logging.info(f"Starting JSON cleaning from {self.raw_json_path}")
+        logging.info(f"Output will be saved to {self.output_path}")
+
+        # 清空输出文件
+        self.output_path.parent.mkdir(parents=True, exist_ok=True)
+        self.error_log_path.parent.mkdir(parents=True, exist_ok=True)
+        self.output_path.write_text('', encoding='utf-8')
+        self.error_log_path.write_text('', encoding='utf-8')
+
+        total_success = 0
+        total_error = 0
+
+        # 处理所有 JSON 文件
+        for file_path in self.raw_json_path.glob('*.json'):
+            logging.info(f"Processing {file_path.name}")
+            success, error = self.process_file(file_path)
+            total_success += success
+            total_error += error
+
+        # 处理 JSONL 文件
+        for file_path in self.raw_json_path.glob('*.jsonl'):
+            logging.info(f"Processing {file_path.name}")
+            success, error = self.process_file(file_path)
+            total_success += success
+            total_error += error
+
+        logging.info(f"JSON cleaning completed: {total_success} records cleaned, {total_error} errors")
+        logging.info(f"Cleaned records saved to {self.output_path}")
+        logging.info(f"Errors logged to {self.error_log_path}")
+
+
+def main():
+    """主函数"""
+    import argparse
+
+    parser = argparse.ArgumentParser(description='Clean and normalize COF-ROS JSON records')
+    parser.add_argument('--config', type=str, default='config/config.yaml',
+                        help='Path to config file')
+    args = parser.parse_args()
+
+    # 加载配置
+    config_path = Path(args.config)
+    with open(config_path, 'r', encoding='utf-8') as f:
+        config = yaml.safe_load(f)
+
+    # 运行清洗
+    cleaner = JSONCleaner(config)
+    cleaner.run()
+
+
+if __name__ == '__main__':
+    main()