@lzhzzzzwill/cofos 1.0.1 → 1.0.2

package/runtime/src/data_processing/parse_pdf_text.py

@@ -0,0 +1,332 @@
+#!/usr/bin/env python3
+"""
+COF-ROS-Reasoner: PDF 文本解析和 ROS 相关段落筛选脚本
+
+目标：
+- 从 PDF 中提取正文并切分为段落
+- 优先标记包含 ROS 相关关键词的段落
+- 输出为标准化的 JSONL 格式
+"""
+
+import os
+import json
+import yaml
+import logging
+import re
+from pathlib import Path
+from typing import Dict, List, Any, Optional, Tuple
+from datetime import datetime
+
+try:
+    import pdfplumber
+except ImportError:
+    pdfplumber = None
+
+try:
+    import fitz  # PyMuPDF
+except ImportError:
+    fitz = None
+
+# 添加 src 路径
+
+from data_processing.utils import setup_logging, ensure_directory
+
+
+class PDFParser:
+    """解析 PDF 文件"""
+
+    def __init__(self, config: Dict[str, Any]):
+        self.config = config
+        self.raw_pdfs_path = Path(config['paths']['raw_pdfs'])
+        self.output_path = Path(config['paths']['processed_dir']) / 'pdf_chunks.jsonl'
+
+        # 设置日志
+        setup_logging(config)
+
+        # chunk 配置
+        self.chunk_size = config.get('data_processing', {}).get('chunk_size', 768)
+        self.chunk_overlap = config.get('data_processing', {}).get('chunk_overlap', 50)
+
+        # ROS 关键词
+        self.ros_keywords = config.get('data_processing', {}).get('ros_keywords', [])
+
+        # 证据方法关键词
+        self.evidence_methods = config.get('data_processing', {}).get('evidence_methods', [])
+
+        # 验证 pdfplumber 是否可用
+        self.use_pdfplumber = pdfplumber is not None
+
+    def get_ros_score(self, text: str) -> Tuple[float, List[str]]:
+        """计算文本的 ROS 相关性得分"""
+        if not text:
+            return 0.0, []
+
+        text_lower = text.lower()
+        matched_keywords = []
+
+        for keyword in self.ros_keywords:
+            if keyword.lower() in text_lower:
+                matched_keywords.append(keyword)
+
+        # 得分 = 匹配的关键词数 / 总关键词数
+        score = len(matched_keywords) / len(self.ros_keywords) if self.ros_keywords else 0.0
+        return score, matched_keywords
+
+    def has_evidence_methods(self, text: str) -> bool:
+        """检查是否包含证据方法关键词"""
+        text_lower = text.lower()
+        for method in self.evidence_methods:
+            if method.lower() in text_lower:
+                return True
+        return False
+
+    def extract_text_from_pdf_plumber(self, pdf_path: Path) -> List[str]:
+        """使用 pdfplumber 提取文本"""
+        if not self.use_pdfplumber:
+            return []
+
+        pages_text = []
+        try:
+            with pdfplumber.open(pdf_path) as pdf:
+                for page_num, page in enumerate(pdf.pages, 1):
+                    text = page.extract_text() or ""
+                    pages_text.append({
+                        'page': page_num,
+                        'text': text
+                    })
+        except Exception as e:
+            logging.error(f"Error extracting text from {pdf_path}: {e}")
+
+        return pages_text
+
+    def extract_text_from_pymupdf(self, pdf_path: Path) -> List[str]:
+        """使用 PyMuPDF 提取文本"""
+        pages_text = []
+        try:
+            doc = fitz.open(pdf_path)
+            for page_num in range(len(doc)):
+                page = doc[page_num]
+                text = page.get_text() or ""
+                pages_text.append({
+                    'page': page_num + 1,
+                    'text': text
+                })
+            doc.close()
+        except Exception as e:
+            logging.error(f"Error extracting text from {pdf_path}: {e}")
+
+        return pages_text
+
+    def chunk_text(self, text: str, page: int) -> List[Dict[str, Any]]:
+        """将文本切分为 chunk"""
+        chunks = []
+
+        # 简单按句子切分
+        sentences = re.split(r'(?<=[.!?])\s+', text)
+
+        current_chunk = []
+        current_length = 0
+
+        for sentence in sentences:
+            sentence_length = len(sentence)
+
+            if current_length + sentence_length > self.chunk_size and current_chunk:
+                # 保存当前 chunk
+                chunk_text = ' '.join(current_chunk)
+                ros_score, matched_keywords = self.get_ros_score(chunk_text)
+                is_ros_related = ros_score > 0
+
+                chunks.append({
+                    'text': chunk_text,
+                    'page': page,
+                    'ros_score': ros_score,
+                    'matched_keywords': matched_keywords,
+                    'has_evidence_methods': self.has_evidence_methods(chunk_text),
+                    'is_ros_related': is_ros_related
+                })
+
+                # 开始新 chunk
+                current_chunk = [sentence]
+                current_length = sentence_length
+            else:
+                current_chunk.append(sentence)
+                current_length += sentence_length
+
+        # 处理剩余的 chunk
+        if current_chunk:
+            chunk_text = ' '.join(current_chunk)
+            ros_score, matched_keywords = self.get_ros_score(chunk_text)
+            is_ros_related = ros_score > 0
+
+            chunks.append({
+                'text': chunk_text,
+                'page': page,
+                'ros_score': ros_score,
+                'matched_keywords': matched_keywords,
+                'has_evidence_methods': self.has_evidence_methods(chunk_text),
+                'is_ros_related': is_ros_related
+            })
+
+        return chunks
+
+    def extract_title_from_pdf(self, pdf_path: Path) -> str:
+        """从 PDF 中提取标题（简单实现：取第一页第一行）"""
+        if self.use_pdfplumber:
+            try:
+                with pdfplumber.open(pdf_path) as pdf:
+                    if pdf.pages:
+                        first_page = pdf.pages[0]
+                        text = first_page.extract_text()
+                        if text:
+                            lines = text.split('\n')
+                            for line in lines:
+                                line = line.strip()
+                                if line and len(line) > 10:
+                                    return line[:100]
+            except Exception:
+                pass
+
+        return pdf_path.stem
+
+    def extract_doi_from_pdf(self, pdf_path: Path) -> str:
+        """从 PDF 中提取 DOI"""
+        try:
+            with open(pdf_path, 'rb') as f:
+                content = f.read().decode('utf-8', errors='ignore')
+
+            # 尝试查找 DOI
+            doi_patterns = [
+                r'10\.\d{4,}/[^\s]+',
+                r'DOI:\s*(10\.\d{4,}/[^\s]+)',
+                r'doi:\s*(10\.\d{4,}/[^\s]+)',
+            ]
+
+            for pattern in doi_patterns:
+                match = re.search(pattern, content, re.IGNORECASE)
+                if match:
+                    return match.group(0) if match.lastindex else match.group(0)
+
+        except Exception:
+            pass
+
+        return ""
+
+    def process_pdf(self, pdf_path: Path) -> List[Dict[str, Any]]:
+        """处理单个 PDF 文件"""
+        records = []
+
+        try:
+            # 提取标题和 DOI
+            title = self.extract_title_from_pdf(pdf_path)
+            doi = self.extract_doi_from_pdf(pdf_path)
+
+            # 提取文本
+            if self.use_pdfplumber:
+                pages_text = self.extract_text_from_pdf_plumber(pdf_path)
+            elif fitz:
+                pages_text = self.extract_text_from_pymupdf(pdf_path)
+            else:
+                logging.error("No PDF library available. Install pdfplumber or PyMuPDF.")
+                return []
+
+            # 处理每个页面
+            for page_data in pages_text:
+                page_num = page_data['page']
+                text = page_data['text']
+
+                # 切分文本
+                chunks = self.chunk_text(text, page_num)
+
+                for chunk_idx, chunk in enumerate(chunks):
+                    record = {
+                        'doc_id': f"{pdf_path.stem}_{page_num}_{chunk_idx:03d}",
+                        'source_type': 'pdf_fulltext',
+                        'title': title,
+                        'doi': doi,
+                        'page': page_num,
+                        'chunk_id': f"{pdf_path.stem}_{page_num}_{chunk_idx:03d}",
+                        'is_ros_related': chunk['is_ros_related'],
+                        'ros_score': chunk['ros_score'],
+                        'matched_keywords': chunk['matched_keywords'],
+                        'has_evidence_methods': chunk['has_evidence_methods'],
+                        'text': chunk['text']
+                    }
+                    records.append(record)
+
+        except Exception as e:
+            logging.error(f"Error processing PDF {pdf_path}: {e}")
+
+        return records
+
+    def run(self):
+        """运行解析流程"""
+        logging.info(f"Starting PDF parsing from {self.raw_pdfs_path}")
+        logging.info(f"Output will be saved to {self.output_path}")
+
+        # 检查输入目录
+        if not self.raw_pdfs_path.exists():
+            logging.error(f"Input directory not found: {self.raw_pdfs_path}")
+            return
+
+        # 收集所有 PDF 文件
+        pdf_files = list(self.raw_pdfs_path.glob('*.pdf'))
+        logging.info(f"Found {len(pdf_files)} PDF files")
+
+        if not pdf_files:
+            logging.warning("No PDF files found")
+            return
+
+        # 处理所有 PDF
+        all_records = []
+        for pdf_path in pdf_files:
+            logging.info(f"Processing {pdf_path.name}")
+            records = self.process_pdf(pdf_path)
+            all_records.extend(records)
+
+        # 按 ROS 相关性排序
+        all_records.sort(key=lambda x: x.get('ros_score', 0), reverse=True)
+
+        # 保存输出
+        ensure_directory(self.output_path.parent)
+        with open(self.output_path, 'w', encoding='utf-8') as f:
+            for record in all_records:
+                f.write(json.dumps(record, ensure_ascii=False) + '\n')
+
+        # 统计信息
+        ros_related = sum(1 for r in all_records if r.get('is_ros_related', False))
+        total_chunks = len(all_records)
+        avg_score = sum(r.get('ros_score', 0) for r in all_records) / total_chunks if total_chunks else 0
+
+        logging.info(f"PDF parsing completed: {total_chunks} chunks from {len(pdf_files)} PDFs")
+        logging.info(f"ROS-related chunks: {ros_related}")
+        logging.info(f"Average ROS score: {avg_score:.3f}")
+        logging.info(f"Output saved to {self.output_path}")
+
+
+def main():
+    """主函数"""
+    import argparse
+
+    parser = argparse.ArgumentParser(description='Parse PDF files and extract ROS-related chunks')
+    parser.add_argument('--config', type=str, default='config/config.yaml',
+                        help='Path to config file')
+    parser.add_argument('--pdf-path', type=str, default=None,
+                        help='Path to PDF file or directory (overrides config)')
+    args = parser.parse_args()
+
+    # 加载配置
+    config_path = Path(args.config)
+    with open(config_path, 'r', encoding='utf-8') as f:
+        config = yaml.safe_load(f)
+
+    # 如果指定了 PDF 路径，覆盖配置
+    if args.pdf_path:
+        config['paths']['raw_pdfs'] = args.pdf_path
+
+    # 运行解析
+    parser = PDFParser(config)
+    parser.run()
+
+
+if __name__ == '__main__':
+    main()

package/runtime/src/data_processing/parse_wos_xlsx.py

@@ -0,0 +1,255 @@
+#!/usr/bin/env python3
+"""
+COF-ROS-Reasoner: WoS 摘要解析脚本
+
+目标：
+- 从 WoS xlsx 中提取摘要、题名、DOI、年份、期刊等信息
+- 输出为标准化的 JSONL 格式
+"""
+
+import os
+import json
+import yaml
+import logging
+from pathlib import Path
+from typing import Dict, List, Any, Optional
+from datetime import datetime
+import math
+
+import pandas as pd
+
+# 添加 src 路径
+
+from data_processing.utils import setup_logging, ensure_directory
+
+
+class WoSParser:
+    """解析 WoS 导出的 XLSX 文件"""
+
+    def __init__(self, config: Dict[str, Any]):
+        self.config = config
+        self.raw_wos_path = Path(config['paths']['raw_wos'])
+        self.output_path = Path(config['paths']['processed_dir']) / 'wos_abstract_chunks.jsonl'
+
+        # 设置日志
+        setup_logging(config)
+
+        # 保留字段映射
+        self.field_mappings = {
+            'Article Title': 'title',
+            'Abstract': 'abstract',
+            'Abstract Note': 'abstract',
+            'DOI': 'doi',
+            'Publication Year': 'year',
+            'Year Published': 'year',
+            'PY': 'year',
+            'Source Title': 'source_title',
+            'Publication Name': 'source_title',
+            'Author Keywords': 'author_keywords',
+            'Author Keywords (DE)': 'author_keywords',
+            'Keywords Plus': 'keywords_plus',
+            'Keywords Plus (ID)': 'keywords_plus',
+            'UT': 'ut',  # WOS Unique ID
+            'WOS Unique ID': 'ut',
+        }
+
+        # 需要解析的关键词字段
+        self.keyword_fields = ['Author Keywords', 'Keywords Plus']
+
+        # ROS 关键词（用于筛选）
+        self.ros_keywords = config.get('data_processing', {}).get('ros_keywords', [])
+
+    def normalize_column_key(self, value: Any) -> str:
+        """Normalize a column name for matching."""
+        return ' '.join(self.safe_str(value).lower().split())
+
+    def normalize_dataframe_columns(self, df: pd.DataFrame) -> pd.DataFrame:
+        """Normalize common WoS column names while preserving unknown columns."""
+        rename_map = {}
+        normalized_targets = set()
+        normalized_mapping = {
+            self.normalize_column_key(source): target
+            for source, target in self.field_mappings.items()
+        }
+
+        for column in df.columns:
+            raw = self.safe_str(column)
+            target = self.field_mappings.get(raw)
+            if not target:
+                target = self.field_mappings.get(raw.strip())
+            if not target:
+                target = normalized_mapping.get(self.normalize_column_key(raw))
+
+            if target and target not in normalized_targets:
+                rename_map[column] = target
+                normalized_targets.add(target)
+
+        normalized = df.rename(columns=rename_map)
+        logging.info(f"WoS columns: {list(df.columns)}")
+        logging.info(f"Normalized WoS columns: {list(normalized.columns)}")
+        return normalized
+
+    def normalize_field_name(self, field: str) -> str:
+        """标准化字段名"""
+        field = self.safe_str(field).strip()
+        return self.field_mappings.get(field, field)
+
+    def safe_str(self, value: Any) -> str:
+        """Convert Excel cell values to clean strings."""
+        if value is None:
+            return ""
+
+        try:
+            if pd.isna(value):
+                return ""
+        except TypeError:
+            pass
+
+        if isinstance(value, float) and math.isfinite(value) and value.is_integer():
+            return str(int(value))
+
+        return str(value).strip()
+
+    def parse_year(self, value: Any) -> Optional[int]:
+        """Safely parse publication year."""
+        text = self.safe_str(value)
+        if not text:
+            return None
+
+        try:
+            return int(float(text))
+        except ValueError:
+            logging.warning(f"Could not parse publication year: {text}")
+            return None
+
+    def parse_keywords(self, keywords_str: Any) -> List[str]:
+        """解析关键词字符串"""
+        keywords_str = self.safe_str(keywords_str)
+        if not keywords_str:
+            return []
+
+        # 分隔符可能是 ; 或 |
+        if ';' in keywords_str:
+            keywords = [k.strip() for k in keywords_str.split(';')]
+        elif '|' in keywords_str:
+            keywords = [k.strip() for k in keywords_str.split('|')]
+        else:
+            keywords = [keywords_str.strip()]
+
+        # 清理空值
+        return [k for k in keywords if k]
+
+    def is_ros_related(self, title: Any, abstract: Any, keywords: List[str] = None) -> bool:
+        """判断是否与 ROS 相关"""
+        keyword_text = ' '.join(keywords or [])
+        text = f"{self.safe_str(title)} {self.safe_str(abstract)} {keyword_text}"
+        text_lower = text.lower()
+
+        for keyword in self.ros_keywords:
+            if keyword.lower() in text_lower:
+                return True
+        return False
+
+    def parse_row(self, row: pd.Series, index: int) -> Optional[Dict[str, Any]]:
+        """解析单行数据"""
+        try:
+            # 提取基本信息
+            title = self.safe_str(row.get('title', ''))
+            abstract = self.safe_str(row.get('abstract', ''))
+            doi = self.safe_str(row.get('doi', ''))
+            year = self.parse_year(row.get('year', ''))
+            source_title = self.safe_str(row.get('source_title', ''))
+            ut = self.safe_str(row.get('ut', ''))
+
+            # 解析关键词
+            author_keywords = self.parse_keywords(row.get('author_keywords', ''))
+            keywords_plus = self.parse_keywords(row.get('keywords_plus', ''))
+
+            # 合并关键词
+            keywords = list(set(author_keywords + keywords_plus))
+
+            # 如果没有摘要，使用标题作为文本
+            if not abstract:
+                text = title
+            else:
+                text = abstract
+
+            # 构建记录
+            record = {
+                'doc_id': f"WOS:{ut}" if ut else f"WOS:index_{index}",
+                'doi': doi,
+                'title': title,
+                'source_type': 'wos_abstract',
+                'year': year,
+                'journal': source_title,
+                'text': text.strip(),
+                'keywords': keywords,
+                'is_ros_related': self.is_ros_related(title, abstract, keywords)
+            }
+
+            return record
+
+        except Exception as e:
+            logging.error(f"Error parsing row {index}: {e}")
+            return None
+
+    def run(self):
+        """运行解析流程"""
+        logging.info(f"Starting WoS parsing from {self.raw_wos_path}")
+        logging.info(f"Output will be saved to {self.output_path}")
+
+        # 检查输入文件
+        if not self.raw_wos_path.exists():
+            logging.error(f"Input file not found: {self.raw_wos_path}")
+            return
+
+        # 读取 XLSX
+        try:
+            df = pd.read_excel(self.raw_wos_path)
+            df = self.normalize_dataframe_columns(df)
+            logging.info(f"Loaded {len(df)} rows from {self.raw_wos_path}")
+        except Exception as e:
+            logging.error(f"Failed to read Excel file: {e}")
+            return
+
+        # 处理记录
+        records = []
+        for idx, row in df.iterrows():
+            record = self.parse_row(row, idx)
+            if record:
+                records.append(record)
+
+        # 保存输出
+        ensure_directory(self.output_path.parent)
+        with open(self.output_path, 'w', encoding='utf-8') as f:
+            for record in records:
+                f.write(json.dumps(record, ensure_ascii=False) + '\n')
+
+        # 统计信息
+        ros_related = sum(1 for r in records if r.get('is_ros_related', False))
+        logging.info(f"WoS parsing completed: {len(records)} records processed")
+        logging.info(f"ROS-related records: {ros_related}")
+        logging.info(f"Output saved to {self.output_path}")
+
+
+def main():
+    """主函数"""
+    import argparse
+
+    parser = argparse.ArgumentParser(description='Parse WoS abstracts from Excel file')
+    parser.add_argument('--config', type=str, default='config/config.yaml',
+                        help='Path to config file')
+    args = parser.parse_args()
+
+    # 加载配置
+    config_path = Path(args.config)
+    with open(config_path, 'r', encoding='utf-8') as f:
+        config = yaml.safe_load(f)
+
+    # 运行解析
+    parser = WoSParser(config)
+    parser.run()
+
+
+if __name__ == '__main__':
+    main()

package/runtime/src/data_processing/schema_utils.py

@@ -0,0 +1,56 @@
+"""Canonical schema-boundary normalization shared by downstream stages."""
+
+import json
+from typing import Any, Dict, List
+
+
+class SchemaNormalizationMixin:
+    """Normalize accepted legacy shapes once through a shared implementation."""
+
+    @staticmethod
+    def safe_str(value: Any) -> str:
+        if value is None:
+            return ""
+        if isinstance(value, str):
+            return value.strip()
+        if isinstance(value, (int, float, bool)):
+            return str(value)
+        return json.dumps(value, ensure_ascii=False)
+
+    @staticmethod
+    def as_list(value: Any) -> List[Any]:
+        if value is None:
+            return []
+        if isinstance(value, list):
+            return value
+        return [value]
+
+    def normalize_item_list(
+        self, value: Any, default_key: str = "species"
+    ) -> List[Dict[str, Any]]:
+        items = []
+        for item in self.as_list(value):
+            if isinstance(item, dict):
+                items.append(item)
+            elif item:
+                items.append({default_key: self.safe_str(item)})
+        return items
+
+    def normalize_ros(self, name: Any) -> str:
+        normalized_name = self.safe_str(name)
+        if not normalized_name:
+            return ""
+        mapping = self.config.get("data_processing", {}).get(
+            "ros_normalization", {}
+        )
+        return mapping.get(
+            normalized_name,
+            mapping.get(normalized_name.lower(), normalized_name),
+        )
+
+    def first_species(self, value: Any) -> str:
+        for item in self.normalize_item_list(value):
+            species = self.normalize_ros(item.get("species", ""))
+            if species:
+                return species
+        return ""