dr-claw 1.1.2

package/server/scripts/research-news/scan_existing_notes.py

@@ -0,0 +1,266 @@
+#!/usr/bin/env python3
+"""
+扫描现有笔记构建索引
+用于 start-my-day skill，扫描 vault 中的现有笔记并构建关键词到笔记路径的映射表
+"""
+
+import os
+import re
+import json
+import sys
+import argparse
+import logging
+from pathlib import Path
+from typing import List, Dict, Set, Tuple
+import yaml
+
+from common_words import COMMON_WORDS
+
+logger = logging.getLogger(__name__)
+
+
+def parse_frontmatter(content: str) -> Dict:
+    """
+    解析 frontmatter (YAML 格式)
+
+    Args:
+        content: markdown 文件内容
+
+    Returns:
+        frontmatter 字典
+    """
+    # 查找 frontmatter 开始和结束标记
+    frontmatch = re.match(r'^---\s*\n(.*?)^---\s*\n', content, re.MULTILINE | re.DOTALL)
+
+    if not frontmatch:
+        return {}
+
+    try:
+        frontmatter_str = frontmatch.group(1)
+        frontmatter_data = yaml.safe_load(frontmatter_str)
+        return frontmatter_data or {}
+    except Exception as e:
+        logger.warning("Error parsing frontmatter: %s", e)
+        return {}
+
+
+def extract_keywords_from_title(title: str) -> List[str]:
+    """
+    从标题中提取关键词
+
+    Args:
+        title: 论文标题
+
+    Returns:
+        关键词列表
+    """
+    if not title:
+        return []
+
+    keywords = []
+
+    # 主要策略：提取论文的缩写或专有名词（大写开头的词）
+    # 例如：从 "BLIP: Bootstrapping..." 提取 "BLIP"
+    main_keyword = re.match(r'^([A-Z]{2,})(?:\s*:|\s+)', title)
+    if main_keyword:
+        keywords.append(main_keyword.group(1))
+
+    # 策略2：提取冒号前的完整标题（如果是缩写+冒号格式）
+    colon_match = title.split(':')
+    if len(colon_match) >= 2 and len(colon_match[0].strip()) > 2:
+        before_colon = colon_match[0].strip()
+        # 只添加长度在3-20之间的
+        if 3 <= len(before_colon) <= 20:
+            keywords.append(before_colon)
+
+    # 策略3：提取带连字符的术语（如 Vision-Language, Fine-Tuning, In-Context）
+    # 只匹配明确的技术术语，避免过度分割
+    tech_terms = re.findall(r'\b[A-Z][a-z]*(?:-[A-Z][a-z]*)+\b', title)
+    for term in tech_terms:
+        term_clean = term.strip()
+        # 只添加长度在3-20之间的技术术语
+        if 3 <= len(term_clean) <= 20:
+            # 过滤掉通用词
+            if term_clean.lower() not in COMMON_WORDS:
+                keywords.append(term_clean)
+
+    # 去重
+    keywords = list(dict.fromkeys(keywords))
+
+    return keywords
+
+
+def scan_notes_directory(papers_dir: Path) -> List[Dict]:
+    """
+    扫描 Papers 目录下的所有笔记
+
+    Args:
+        papers_dir: Papers 目录路径
+
+    Returns:
+        笔记列表
+    """
+    notes = []
+
+    # 递归查找所有 .md 文件
+    for md_file in papers_dir.rglob('*.md'):
+        try:
+            with open(md_file, 'r', encoding='utf-8') as f:
+                content = f.read()
+
+            # 解析 frontmatter
+            frontmatter = parse_frontmatter(content)
+
+            # 提取信息
+            # 计算相对于vault的路径（使用正斜杠）
+            rel_path = md_file.relative_to(papers_dir.parent.parent)
+            note_info = {
+                'path': str(rel_path).replace('\\', '/'),  # 使用正斜杠
+                'filename': md_file.name,
+                'short_name': md_file.stem,  # 文件名（不含.md扩展名），用于短链接
+                'path_str': str(rel_path),  # 添加路径的字符串表示，用于正确编码
+                'title': frontmatter.get('title', md_file.stem),
+                'tags': frontmatter.get('tags', []),
+            }
+
+            # 从标题提取关键词
+            title_keywords = extract_keywords_from_title(note_info['title'])
+            note_info['title_keywords'] = title_keywords
+
+            # 从 tags 提取关键词（保留有意义的tag）
+            tag_keywords = []
+            for tag in note_info['tags']:
+                if isinstance(tag, list):
+                    for sub_tag in tag:
+                        if isinstance(sub_tag, str):
+                            # 只添加长度3-20的tag，过滤通用词
+                            if 3 <= len(sub_tag) <= 20 and sub_tag.lower() not in COMMON_WORDS:
+                                tag_keywords.append(sub_tag)
+                elif isinstance(tag, str):
+                    if 3 <= len(tag) <= 20 and tag.lower() not in COMMON_WORDS:
+                        tag_keywords.append(tag)
+
+            note_info['tag_keywords'] = tag_keywords
+
+            notes.append(note_info)
+
+        except Exception as e:
+            logger.warning("Error reading %s: %s", md_file, e)
+            continue
+
+    return notes
+
+
+def build_keyword_index(notes: List[Dict]) -> Dict[str, List[str]]:
+    """
+    构建关键词到笔记路径的映射表
+
+    Args:
+        notes: 笔记列表
+
+    Returns:
+        关键词映射字典
+    """
+    keyword_index = {}
+
+    for note in notes:
+        # 优先使用从标题提取的关键词
+        for keyword in note['title_keywords']:
+            keyword_lower = keyword.lower()
+            if len(keyword_lower) >= 3 and len(keyword_lower) <= 30:
+                if keyword_lower not in COMMON_WORDS:
+                    if keyword_lower not in keyword_index:
+                        keyword_index[keyword_lower] = []
+                    if note['path'] not in keyword_index[keyword_lower]:
+                        keyword_index[keyword_lower].append(note['path'])
+
+        # 添加从 tags 提取的关键词
+        for keyword in note['tag_keywords']:
+            keyword_lower = keyword.lower()
+            if len(keyword_lower) >= 3 and len(keyword_lower) <= 30:
+                if keyword_lower not in COMMON_WORDS:
+                    if keyword_lower not in keyword_index:
+                        keyword_index[keyword_lower] = []
+                    if note['path'] not in keyword_index[keyword_lower]:
+                        keyword_index[keyword_lower].append(note['path'])
+
+        # 使用短名称（文件名）作为关键词，但只添加主要部分
+        if 'short_name' in note:
+            short_name = note['short_name']
+            # 移除版本号和常见后缀
+            clean_short = re.sub(r'-\d{4}\.\d{5}$', '', short_name)
+            clean_short = re.sub(r'-v\d+$', '', clean_short)
+
+            # 如果清理后的短名称长度合适，添加到索引
+            if 3 <= len(clean_short) <= 40 and clean_short.lower() not in COMMON_WORDS:
+                keyword_index[clean_short.lower()] = [note['path']]
+
+    return keyword_index
+
+
+def main():
+    """主函数"""
+    parser = argparse.ArgumentParser(description='Scan existing notes and build keyword index')
+    parser.add_argument('--vault', type=str,
+                        default=os.environ.get('OBSIDIAN_VAULT_PATH', ''),
+                        help='Path to Obsidian vault (or set OBSIDIAN_VAULT_PATH env var)')
+    parser.add_argument('--output', type=str, default='existing_notes_index.json',
+                        help='Output JSON file path')
+    parser.add_argument('--papers-dir', type=str,
+                        default='20_Research/Papers',
+                        help='Relative path to Papers directory')
+
+    args = parser.parse_args()
+
+    logging.basicConfig(
+        level=logging.INFO,
+        format='%(asctime)s [%(levelname)s] %(message)s',
+        datefmt='%H:%M:%S',
+        stream=sys.stderr,
+    )
+
+    if not args.vault:
+        logger.error("未指定 vault 路径。请通过 --vault 参数或 OBSIDIAN_VAULT_PATH 环境变量设置。")
+        sys.exit(1)
+
+    vault_path = Path(args.vault)
+    papers_dir = vault_path / args.papers_dir
+
+    if not papers_dir.exists():
+        logger.error("Papers directory not found: %s", papers_dir)
+        logger.error("Using vault path: %s", vault_path)
+        sys.exit(1)
+
+    logger.info("Scanning notes in: %s", papers_dir)
+
+    notes = scan_notes_directory(papers_dir)
+    logger.info("Found %d notes", len(notes))
+
+    keyword_index = build_keyword_index(notes)
+    logger.info("Built index with %d keywords", len(keyword_index))
+
+    # 准备输出
+    output = {
+        'notes': notes,
+        'keyword_to_notes': keyword_index
+    }
+
+    # 保存结果
+    with open(args.output, 'w', encoding='utf-8') as f:
+        json.dump(output, f, ensure_ascii=False, indent=2, default=str)
+
+    logger.info("Index saved to: %s", args.output)
+
+    logger.info("=== Keyword Index Statistics ===")
+    logger.info("Total notes: %d", len(notes))
+    logger.info("Total keywords: %d", len(keyword_index))
+
+    if len(keyword_index) > 0:
+        logger.info("=== Sample Keywords ===")
+        sample_keywords = sorted(keyword_index.items())[:10]
+        for keyword, paths in sample_keywords:
+            logger.info("  %s: %d notes", keyword, len(paths))
+
+
+if __name__ == '__main__':
+    main()

package/server/scripts/research-news/scoring_utils.py

@@ -0,0 +1,227 @@
+#!/usr/bin/env python3
+"""
+Shared scoring functions for paper recommendation.
+
+Extracted from search_arxiv.py so that other modules (e.g. search_semantic_scholar)
+can reuse the same scoring logic without duplication.
+"""
+
+from datetime import datetime
+from typing import Dict, List, Optional, Tuple
+
+# ---------------------------------------------------------------------------
+# 评分常量  —— 修改权重时只需编辑这里
+# ---------------------------------------------------------------------------
+
+# 各维度原始评分的满分值（归一化基准）
+SCORE_MAX = 3.0
+
+# 相关性评分：关键词在标题 / 摘要中匹配的加分
+RELEVANCE_TITLE_KEYWORD_BOOST = 0.5
+RELEVANCE_SUMMARY_KEYWORD_BOOST = 0.3
+RELEVANCE_CATEGORY_MATCH_BOOST = 1.0
+
+# 新近性阈值（天） -> 对应评分
+RECENCY_THRESHOLDS = [
+    (30, 3.0),
+    (90, 2.0),
+    (180, 1.0),
+]
+RECENCY_DEFAULT = 0.0
+
+# 热门度：高影响力引用数归一化到 0-SCORE_MAX
+# 含义：达到此引用数时视为满分
+POPULARITY_INFLUENTIAL_CITATION_FULL_SCORE = 100
+
+# 综合推荐评分权重（普通论文）
+WEIGHTS_NORMAL = {
+    'relevance': 0.40,
+    'recency': 0.20,
+    'popularity': 0.30,
+    'quality': 0.10,
+}
+# 综合推荐评分权重（高影响力论文：提高热门度，降低新近性）
+WEIGHTS_HOT = {
+    'relevance': 0.35,
+    'recency': 0.10,
+    'popularity': 0.45,
+    'quality': 0.10,
+}
+
+
+def calculate_relevance_score(
+    paper: Dict,
+    domains: Dict,
+    excluded_keywords: List[str]
+) -> Tuple[float, Optional[str], List[str]]:
+    """
+    计算论文与研究兴趣的相关性评分
+
+    Args:
+        paper: 论文信息
+        domains: 研究领域配置
+        excluded_keywords: 排除关键词
+
+    Returns:
+        (相关性评分, 匹配的领域, 匹配的关键词列表)
+    """
+    title = paper.get('title', '').lower()
+    summary = paper.get('summary', '').lower() if 'summary' in paper else paper.get('abstract', '').lower()
+    categories = set(paper.get('categories', []))
+
+    # 检查排除关键词
+    for keyword in excluded_keywords:
+        if keyword.lower() in title or keyword.lower() in summary:
+            return 0, None, []
+
+    max_score = 0
+    best_domain = None
+    matched_keywords = []
+
+    # 遍历所有领域
+    for domain_name, domain_config in domains.items():
+        score = 0
+        domain_matched_keywords = []
+
+        # 关键词匹配
+        keywords = domain_config.get('keywords', [])
+        for keyword in keywords:
+            keyword_lower = keyword.lower()
+            if keyword_lower in title:
+                score += RELEVANCE_TITLE_KEYWORD_BOOST
+                domain_matched_keywords.append(keyword)
+            elif keyword_lower in summary:
+                score += RELEVANCE_SUMMARY_KEYWORD_BOOST
+                domain_matched_keywords.append(keyword)
+
+        # 类别匹配
+        domain_categories = domain_config.get('arxiv_categories', [])
+        for cat in domain_categories:
+            if cat in categories:
+                score += RELEVANCE_CATEGORY_MATCH_BOOST
+                domain_matched_keywords.append(cat)
+
+        if score > max_score:
+            max_score = score
+            best_domain = domain_name
+            matched_keywords = domain_matched_keywords
+
+    return max_score, best_domain, matched_keywords
+
+
+def calculate_recency_score(published_date: Optional[datetime]) -> float:
+    """
+    根据发布日期计算新近性评分
+
+    Args:
+        published_date: 发布日期
+
+    Returns:
+        新近性评分 (0-3)
+    """
+    if published_date is None:
+        return 0
+
+    now = datetime.now(published_date.tzinfo) if published_date.tzinfo else datetime.now()
+    days_diff = (now - published_date).days
+
+    for max_days, score in RECENCY_THRESHOLDS:
+        if days_diff <= max_days:
+            return score
+    return RECENCY_DEFAULT
+
+
+def calculate_quality_score(summary: str) -> float:
+    """
+    从摘要推断质量评分
+
+    采用更细粒度的指标：强创新词权重高于弱创新词，
+    量化结果和对比实验也加分。
+
+    Args:
+        summary: 论文摘要
+
+    Returns:
+        质量评分 (0-3)
+    """
+    score = 0.0
+    summary_lower = summary.lower()
+
+    strong_innovation = [
+        'state-of-the-art', 'sota', 'breakthrough', 'first',
+        'surpass', 'outperform', 'pioneering'
+    ]
+    weak_innovation = [
+        'novel', 'propose', 'introduce', 'new approach',
+        'new method', 'innovative'
+    ]
+    method_indicators = [
+        'framework', 'architecture', 'algorithm', 'mechanism',
+        'pipeline', 'end-to-end'
+    ]
+    quantitative_indicators = [
+        'outperforms', 'improves by', 'achieves', 'accuracy',
+        'f1', 'bleu', 'rouge', 'beats', 'surpasses'
+    ]
+    experiment_indicators = [
+        'experiment', 'evaluation', 'benchmark', 'ablation',
+        'baseline', 'comparison'
+    ]
+
+    strong_count = sum(1 for ind in strong_innovation if ind in summary_lower)
+    if strong_count >= 2:
+        score += 1.0
+    elif strong_count == 1:
+        score += 0.7
+    else:
+        weak_count = sum(1 for ind in weak_innovation if ind in summary_lower)
+        if weak_count > 0:
+            score += 0.3
+
+    if any(ind in summary_lower for ind in method_indicators):
+        score += 0.5
+
+    if any(ind in summary_lower for ind in quantitative_indicators):
+        score += 0.8
+    elif any(ind in summary_lower for ind in experiment_indicators):
+        score += 0.4
+
+    return min(score, SCORE_MAX)
+
+
+def calculate_recommendation_score(
+    relevance_score: float,
+    recency_score: float,
+    popularity_score: float,
+    quality_score: float,
+    is_hot_paper: bool = False
+) -> float:
+    """
+    计算综合推荐评分
+
+    权重定义在模块顶部常量 WEIGHTS_NORMAL / WEIGHTS_HOT 中。
+    对于高影响力论文（来自 Semantic Scholar），使用 WEIGHTS_HOT 提高热门度权重。
+
+    Args:
+        relevance_score: 相关性评分 (0-SCORE_MAX)
+        recency_score: 新近性评分 (0-SCORE_MAX)
+        popularity_score: 热门度评分 (0-SCORE_MAX)
+        quality_score: 质量评分 (0-SCORE_MAX)
+        is_hot_paper: 是否是高影响力论文
+
+    Returns:
+        综合推荐评分 (0-10)
+    """
+    scores = {
+        'relevance': relevance_score,
+        'recency': recency_score,
+        'popularity': popularity_score,
+        'quality': quality_score,
+    }
+    # 归一化到 0-10 分
+    normalized = {k: (v / SCORE_MAX) * 10 for k, v in scores.items()}
+
+    weights = WEIGHTS_HOT if is_hot_paper else WEIGHTS_NORMAL
+    final_score = sum(normalized[k] * weights[k] for k in weights)
+
+    return round(final_score, 2)