@rudderhq/server 0.1.0-canary.0

package/resources/community-skills/deep-research/scripts/source_evaluator.py

@@ -0,0 +1,292 @@
+#!/usr/bin/env python3
+"""
+Source Credibility Evaluator
+Assesses source quality, credibility, and potential biases
+"""
+
+from dataclasses import dataclass
+from typing import List, Dict, Optional
+from urllib.parse import urlparse
+from datetime import datetime, timedelta
+import re
+
+
+@dataclass
+class CredibilityScore:
+    """Represents source credibility assessment"""
+    overall_score: float  # 0-100
+    domain_authority: float  # 0-100
+    recency: float  # 0-100
+    expertise: float  # 0-100
+    bias_score: float  # 0-100 (higher = more neutral)
+    factors: Dict[str, str]
+    recommendation: str  # "high_trust", "moderate_trust", "low_trust", "verify"
+
+
+class SourceEvaluator:
+    """Evaluates source credibility and quality"""
+
+    # Domain reputation tiers
+    HIGH_AUTHORITY_DOMAINS = {
+        # Academic & Research
+        'arxiv.org', 'nature.com', 'science.org', 'cell.com', 'nejm.org',
+        'thelancet.com', 'springer.com', 'sciencedirect.com', 'plos.org',
+        'ieee.org', 'acm.org', 'pubmed.ncbi.nlm.nih.gov',
+
+        # Government & International Organizations
+        'nih.gov', 'cdc.gov', 'who.int', 'fda.gov', 'nasa.gov',
+        'gov.uk', 'europa.eu', 'un.org',
+
+        # Established Tech Documentation
+        'docs.python.org', 'developer.mozilla.org', 'docs.microsoft.com',
+        'cloud.google.com', 'aws.amazon.com', 'kubernetes.io',
+
+        # Reputable News (Fact-check verified)
+        'reuters.com', 'apnews.com', 'bbc.com', 'economist.com',
+        'nature.com/news', 'scientificamerican.com'
+    }
+
+    MODERATE_AUTHORITY_DOMAINS = {
+        # Tech News & Analysis
+        'techcrunch.com', 'theverge.com', 'arstechnica.com', 'wired.com',
+        'zdnet.com', 'cnet.com',
+
+        # Industry Publications
+        'forbes.com', 'bloomberg.com', 'wsj.com', 'ft.com',
+
+        # Educational
+        'wikipedia.org', 'britannica.com', 'khanacademy.org',
+
+        # Tech Blogs (established)
+        'medium.com', 'dev.to', 'stackoverflow.com', 'github.com'
+    }
+
+    LOW_AUTHORITY_INDICATORS = [
+        'blogspot.com', 'wordpress.com', 'wix.com', 'substack.com'
+    ]
+
+    def __init__(self):
+        pass
+
+    def evaluate_source(
+        self,
+        url: str,
+        title: str,
+        content: Optional[str] = None,
+        publication_date: Optional[str] = None,
+        author: Optional[str] = None
+    ) -> CredibilityScore:
+        """Evaluate source credibility"""
+
+        domain = self._extract_domain(url)
+
+        # Calculate component scores
+        domain_score = self._evaluate_domain_authority(domain)
+        recency_score = self._evaluate_recency(publication_date)
+        expertise_score = self._evaluate_expertise(domain, title, author)
+        bias_score = self._evaluate_bias(domain, title, content)
+
+        # Calculate overall score (weighted average)
+        overall = (
+            domain_score * 0.35 +
+            recency_score * 0.20 +
+            expertise_score * 0.25 +
+            bias_score * 0.20
+        )
+
+        # Determine factors
+        factors = self._identify_factors(
+            domain, domain_score, recency_score, expertise_score, bias_score
+        )
+
+        # Generate recommendation
+        recommendation = self._generate_recommendation(overall)
+
+        return CredibilityScore(
+            overall_score=round(overall, 2),
+            domain_authority=round(domain_score, 2),
+            recency=round(recency_score, 2),
+            expertise=round(expertise_score, 2),
+            bias_score=round(bias_score, 2),
+            factors=factors,
+            recommendation=recommendation
+        )
+
+    def _extract_domain(self, url: str) -> str:
+        """Extract domain from URL"""
+        parsed = urlparse(url)
+        domain = parsed.netloc.lower()
+        # Remove www prefix
+        domain = domain.replace('www.', '')
+        return domain
+
+    def _evaluate_domain_authority(self, domain: str) -> float:
+        """Evaluate domain authority (0-100)"""
+        if domain in self.HIGH_AUTHORITY_DOMAINS:
+            return 90.0
+        elif domain in self.MODERATE_AUTHORITY_DOMAINS:
+            return 70.0
+        elif any(indicator in domain for indicator in self.LOW_AUTHORITY_INDICATORS):
+            return 40.0
+        else:
+            # Unknown domain - moderate skepticism
+            return 55.0
+
+    def _evaluate_recency(self, publication_date: Optional[str]) -> float:
+        """Evaluate information recency (0-100)"""
+        if not publication_date:
+            return 50.0  # Unknown date
+
+        try:
+            pub_date = datetime.fromisoformat(publication_date.replace('Z', '+00:00'))
+            age = datetime.now() - pub_date
+
+            # Recency scoring
+            if age < timedelta(days=90):  # < 3 months
+                return 100.0
+            elif age < timedelta(days=365):  # < 1 year
+                return 85.0
+            elif age < timedelta(days=730):  # < 2 years
+                return 70.0
+            elif age < timedelta(days=1825):  # < 5 years
+                return 50.0
+            else:
+                return 30.0
+
+        except Exception:
+            return 50.0
+
+    def _evaluate_expertise(
+        self,
+        domain: str,
+        title: str,
+        author: Optional[str]
+    ) -> float:
+        """Evaluate source expertise (0-100)"""
+        score = 50.0
+
+        # Academic/research domains get high expertise
+        if any(d in domain for d in ['arxiv', 'nature', 'science', 'ieee', 'acm']):
+            score += 30
+
+        # Government/official sources
+        if '.gov' in domain or 'who.int' in domain:
+            score += 25
+
+        # Technical documentation
+        if 'docs.' in domain or 'documentation' in title.lower():
+            score += 20
+
+        # Author credentials (if available)
+        if author:
+            if any(title in author.lower() for title in ['dr.', 'phd', 'professor']):
+                score += 15
+
+        return min(score, 100.0)
+
+    def _evaluate_bias(
+        self,
+        domain: str,
+        title: str,
+        content: Optional[str]
+    ) -> float:
+        """Evaluate potential bias (0-100, higher = more neutral)"""
+        score = 70.0  # Start neutral
+
+        # Check for sensationalism in title
+        sensational_indicators = [
+            '!', 'shocking', 'unbelievable', 'you won\'t believe',
+            'secret', 'they don\'t want you to know'
+        ]
+        title_lower = title.lower()
+        if any(indicator in title_lower for indicator in sensational_indicators):
+            score -= 20
+
+        # Academic sources are typically less biased
+        if any(d in domain for d in ['arxiv', 'nature', 'science', 'ieee']):
+            score += 20
+
+        # Check for balance in content (if available)
+        if content:
+            # Look for balanced language
+            balanced_indicators = ['however', 'although', 'on the other hand', 'critics argue']
+            if any(indicator in content.lower() for indicator in balanced_indicators):
+                score += 10
+
+        return min(max(score, 0), 100.0)
+
+    def _identify_factors(
+        self,
+        domain: str,
+        domain_score: float,
+        recency_score: float,
+        expertise_score: float,
+        bias_score: float
+    ) -> Dict[str, str]:
+        """Identify key credibility factors"""
+        factors = {}
+
+        if domain_score >= 85:
+            factors['domain'] = "High authority domain"
+        elif domain_score <= 45:
+            factors['domain'] = "Low authority domain - verify claims"
+
+        if recency_score >= 85:
+            factors['recency'] = "Recent information"
+        elif recency_score <= 40:
+            factors['recency'] = "Outdated information - verify currency"
+
+        if expertise_score >= 80:
+            factors['expertise'] = "Expert source"
+        elif expertise_score <= 45:
+            factors['expertise'] = "Limited expertise indicators"
+
+        if bias_score >= 80:
+            factors['bias'] = "Balanced perspective"
+        elif bias_score <= 50:
+            factors['bias'] = "Potential bias detected"
+
+        return factors
+
+    def _generate_recommendation(self, overall_score: float) -> str:
+        """Generate trust recommendation"""
+        if overall_score >= 80:
+            return "high_trust"
+        elif overall_score >= 60:
+            return "moderate_trust"
+        elif overall_score >= 40:
+            return "low_trust"
+        else:
+            return "verify"
+
+
+# Example usage
+if __name__ == '__main__':
+    evaluator = SourceEvaluator()
+
+    # Test sources
+    test_sources = [
+        {
+            'url': 'https://www.nature.com/articles/s41586-2025-12345',
+            'title': 'Breakthrough in Quantum Computing',
+            'publication_date': '2025-10-15'
+        },
+        {
+            'url': 'https://someblog.wordpress.com/shocking-discovery',
+            'title': 'SHOCKING! You Won\'t Believe This Discovery!',
+            'publication_date': '2020-01-01'
+        },
+        {
+            'url': 'https://docs.python.org/3/library/asyncio.html',
+            'title': 'asyncio — Asynchronous I/O',
+            'publication_date': '2025-11-01'
+        }
+    ]
+
+    for source in test_sources:
+        score = evaluator.evaluate_source(**source)
+        print(f"\nSource: {source['title']}")
+        print(f"URL: {source['url']}")
+        print(f"Overall Score: {score.overall_score}/100")
+        print(f"Recommendation: {score.recommendation}")
+        print(f"Factors: {score.factors}")

package/resources/community-skills/deep-research/scripts/validate_report.py

@@ -0,0 +1,403 @@
+#!/usr/bin/env python3
+"""
+Report Validation Script
+Ensures research reports meet quality standards before delivery
+"""
+
+import argparse
+import re
+import sys
+from pathlib import Path
+from typing import List
+
+
+class ReportValidator:
+    """Validates research report quality"""
+
+    def __init__(self, report_path: Path):
+        self.report_path = report_path
+        self.content = self._read_report()
+        self.errors: List[str] = []
+        self.warnings: List[str] = []
+
+    def _read_report(self) -> str:
+        """Read report file"""
+        try:
+            with open(self.report_path, 'r', encoding='utf-8') as f:
+                return f.read()
+        except Exception as e:
+            print(f"❌ ERROR: Cannot read report: {e}")
+            sys.exit(1)
+
+    def validate(self) -> bool:
+        """Run all validation checks"""
+        print(f"\n{'='*60}")
+        print(f"VALIDATING REPORT: {self.report_path.name}")
+        print(f"{'='*60}\n")
+
+        checks = [
+            ("Executive Summary", self._check_executive_summary),
+            ("Required Sections", self._check_required_sections),
+            ("Citations", self._check_citations),
+            ("Bibliography", self._check_bibliography),
+            ("Placeholder Text", self._check_placeholders),
+            ("Content Truncation", self._check_content_truncation),
+            ("Word Count", self._check_word_count),
+            ("Source Count", self._check_source_count),
+            ("Broken Links", self._check_broken_references),
+            ("Concrete Grounding", self._check_concrete_grounding),
+        ]
+
+        for check_name, check_func in checks:
+            print(f"⏳ Checking: {check_name}...", end=" ")
+            passed = check_func()
+            if passed:
+                print("✅ PASS")
+            else:
+                print("❌ FAIL")
+
+        self._print_summary()
+
+        return len(self.errors) == 0
+
+    def _check_executive_summary(self) -> bool:
+        """Check executive summary exists and is 200-400 words"""
+        pattern = r'## Executive Summary(.*?)(?=##|\Z)'
+        match = re.search(pattern, self.content, re.DOTALL | re.IGNORECASE)
+
+        if not match:
+            self.errors.append("Missing 'Executive Summary' section")
+            return False
+
+        summary = match.group(1).strip()
+        word_count = len(summary.split())
+
+        if word_count > 400:
+            self.warnings.append(f"Executive summary too long: {word_count} words (should be ≤400)")
+
+        if word_count < 50:
+            self.warnings.append(f"Executive summary too short: {word_count} words (should be ≥50)")
+
+        return True
+
+    def _check_required_sections(self) -> bool:
+        """Check all required sections are present"""
+        required = [
+            "Executive Summary",
+            "Introduction",
+            "Main Analysis",
+            "Synthesis",
+            "Limitations",
+            "Recommendations",
+            "Bibliography",
+            "Methodology"
+        ]
+
+        # Recommended sections (warnings if missing, not errors)
+        recommended = [
+            "Counterevidence Register",
+            "Claims-Evidence Table"
+        ]
+
+        missing = []
+        for section in required:
+            if not re.search(rf'##.*{section}', self.content, re.IGNORECASE):
+                missing.append(section)
+
+        if missing:
+            self.errors.append(f"Missing sections: {', '.join(missing)}")
+            return False
+
+        # Check recommended sections (warnings only)
+        missing_recommended = []
+        for section in recommended:
+            if not re.search(rf'##.*{section}', self.content, re.IGNORECASE):
+                missing_recommended.append(section)
+
+        if missing_recommended:
+            self.warnings.append(f"Missing recommended sections (for academic rigor): {', '.join(missing_recommended)}")
+
+        return True
+
+    def _check_citations(self) -> bool:
+        """Check citation format and presence"""
+        # Find all citation references [1], [2], etc.
+        citations = re.findall(r'\[(\d+)\]', self.content)
+
+        if not citations:
+            self.errors.append("No citations found in report")
+            return False
+
+        unique_citations = set(citations)
+
+        if len(unique_citations) < 10:
+            self.warnings.append(f"Only {len(unique_citations)} unique sources cited (recommended: ≥10)")
+
+        # Check for consecutive citation numbers
+        citation_nums = sorted([int(c) for c in unique_citations])
+        if citation_nums:
+            max_citation = max(citation_nums)
+            expected = set(range(1, max_citation + 1))
+            missing = expected - set(citation_nums)
+
+            if missing:
+                self.warnings.append(f"Non-consecutive citation numbers, missing: {sorted(missing)}")
+
+        return True
+
+    def _check_bibliography(self) -> bool:
+        """Check bibliography exists, matches citations, and has no truncation placeholders"""
+        pattern = r'## Bibliography(.*?)(?=##|\Z)'
+        match = re.search(pattern, self.content, re.DOTALL | re.IGNORECASE)
+
+        if not match:
+            self.errors.append("Missing 'Bibliography' section")
+            return False
+
+        bib_section = match.group(1)
+
+        # CRITICAL: Check for truncation placeholders (2025 CiteGuard enhancement)
+        truncation_patterns = [
+            (r'\[\d+-\d+\]', 'Citation range (e.g., [8-75])'),
+            (r'Additional.*citations', 'Phrase "Additional citations"'),
+            (r'would be included', 'Phrase "would be included"'),
+            (r'\[\.\.\.continue', 'Pattern "[...continue"'),
+            (r'\[Continue with', 'Pattern "[Continue with"'),
+            (r'etc\.(?!\w)', 'Standalone "etc."'),
+            (r'and so on', 'Phrase "and so on"'),
+        ]
+
+        for pattern_re, description in truncation_patterns:
+            if re.search(pattern_re, bib_section, re.IGNORECASE):
+                self.errors.append(f"⚠️ CRITICAL: Bibliography contains truncation placeholder: {description}")
+                self.errors.append(f"   This makes the report UNUSABLE - complete bibliography required")
+                return False
+
+        # Count bibliography entries [1], [2], etc.
+        bib_entries = re.findall(r'^\[(\d+)\]', bib_section, re.MULTILINE)
+
+        if not bib_entries:
+            self.errors.append("Bibliography has no entries")
+            return False
+
+        # Check citation number continuity (no gaps)
+        bib_nums = sorted([int(n) for n in bib_entries])
+        if bib_nums:
+            expected = list(range(1, bib_nums[-1] + 1))
+            actual = bib_nums
+            missing = [n for n in expected if n not in actual]
+            if missing:
+                self.errors.append(f"Bibliography has gaps in numbering: missing {missing}")
+                return False
+
+        # Find citations in text
+        text_citations = set(re.findall(r'\[(\d+)\]', self.content))
+        bib_citations = set(bib_entries)
+
+        # Check all citations have bibliography entries
+        missing_in_bib = text_citations - bib_citations
+        if missing_in_bib:
+            self.errors.append(f"Citations missing from bibliography: {sorted(missing_in_bib)}")
+            return False
+
+        # Check for unused bibliography entries
+        unused = bib_citations - text_citations
+        if unused:
+            self.warnings.append(f"Unused bibliography entries: {sorted(unused)}")
+
+        return True
+
+    def _check_placeholders(self) -> bool:
+        """Check for placeholder text that shouldn't be in final report"""
+        placeholders = [
+            'TBD', 'TODO', 'FIXME', 'XXX',
+            '[citation needed]', '[needs citation]',
+            '[placeholder]', '[TODO]', '[TBD]'
+        ]
+
+        found_placeholders = []
+        for placeholder in placeholders:
+            if placeholder in self.content:
+                found_placeholders.append(placeholder)
+
+        if found_placeholders:
+            self.errors.append(f"Found placeholder text: {', '.join(found_placeholders)}")
+            return False
+
+        return True
+
+    def _check_content_truncation(self) -> bool:
+        """Check for content truncation patterns (2025 Progressive Assembly enhancement)"""
+        truncation_patterns = [
+            (r'Content continues', 'Phrase "Content continues"'),
+            (r'Due to length', 'Phrase "Due to length"'),
+            (r'would continue', 'Phrase "would continue"'),
+            (r'\[Sections \d+-\d+', 'Pattern "[Sections X-Y"'),
+            (r'Additional sections', 'Phrase "Additional sections"'),
+            (r'comprehensive.*word document that continues', 'Pattern "comprehensive...document that continues"'),
+        ]
+
+        for pattern_re, description in truncation_patterns:
+            if re.search(pattern_re, self.content, re.IGNORECASE):
+                self.errors.append(f"⚠️ CRITICAL: Content truncation detected: {description}")
+                self.errors.append(f"   Report is INCOMPLETE and UNUSABLE - regenerate with progressive assembly")
+                return False
+
+        return True
+
+    def _check_word_count(self) -> bool:
+        """Check overall report length"""
+        word_count = len(self.content.split())
+
+        if word_count < 500:
+            self.warnings.append(f"Report is very short: {word_count} words (consider expanding)")
+        # No upper limit warning - progressive assembly supports unlimited lengths
+
+        return True
+
+    def _check_source_count(self) -> bool:
+        """Check minimum source count"""
+        pattern = r'## Bibliography(.*?)(?=##|\Z)'
+        match = re.search(pattern, self.content, re.DOTALL | re.IGNORECASE)
+
+        if not match:
+            return True  # Already caught in bibliography check
+
+        bib_section = match.group(1)
+        bib_entries = re.findall(r'^\[(\d+)\]', bib_section, re.MULTILINE)
+
+        source_count = len(set(bib_entries))
+
+        if source_count < 10:
+            self.warnings.append(f"Only {source_count} sources (recommended: ≥10)")
+
+        return True
+
+    def _check_broken_references(self) -> bool:
+        """Check for broken internal references"""
+        # Find all markdown links [text](./path)
+        internal_links = re.findall(r'\[.*?\]\((\.\/.*?)\)', self.content)
+
+        broken = []
+        for link in internal_links:
+            # Remove anchor if present
+            link_path = link.split('#')[0]
+            full_path = self.report_path.parent / link_path
+
+            if not full_path.exists():
+                broken.append(link)
+
+        if broken:
+            self.errors.append(f"Broken internal links: {', '.join(broken)}")
+            return False
+
+        return True
+
+    def _check_concrete_grounding(self) -> bool:
+        """Warn when major findings appear purely abstract with no grounding examples"""
+        pattern = r'## Main Analysis(.*?)(?=##|\Z)'
+        match = re.search(pattern, self.content, re.DOTALL | re.IGNORECASE)
+
+        if not match:
+            return True  # Already covered by required sections
+
+        main_analysis = match.group(1)
+        finding_count = len(re.findall(r'^###\s+Finding\b', main_analysis, re.MULTILINE | re.IGNORECASE))
+
+        if finding_count == 0:
+            return True
+
+        grounding_markers = [
+            r'Concrete Example',
+            r'Mini-Case',
+            r'Applied Scenario',
+            r'Counterexample',
+            r'Failure Mode',
+            r'Representative Case',
+            r'Illustrative Hypothetical',
+            r'Hypothetical Example',
+        ]
+        marker_count = 0
+        for marker in grounding_markers:
+            marker_count += len(re.findall(marker, main_analysis, re.IGNORECASE))
+
+        if marker_count == 0:
+            self.warnings.append(
+                "Main Analysis has no concrete example/case markers. "
+                "For abstract or technical findings, add source-grounded examples, mini-cases, "
+                "counterexamples, failure modes, or clearly labeled hypotheticals."
+            )
+
+        hypothetical_mentions = len(re.findall(r'\bhypothetical\b|\billustrative\b', self.content, re.IGNORECASE))
+        citation_like_hypotheticals = re.findall(
+            r'(?i)(hypothetical|illustrative)[^\n]{0,120}\[\d+\]',
+            self.content,
+        )
+        if hypothetical_mentions and citation_like_hypotheticals:
+            self.warnings.append(
+                "Hypothetical or illustrative examples appear next to citations. "
+                "Make sure invented scenarios are labeled clearly and not presented as sourced facts."
+            )
+
+        return True
+
+    def _print_summary(self):
+        """Print validation summary"""
+        print(f"\n{'='*60}")
+        print(f"VALIDATION SUMMARY")
+        print(f"{'='*60}\n")
+
+        if self.errors:
+            print(f"❌ ERRORS ({len(self.errors)}):")
+            for error in self.errors:
+                print(f"   • {error}")
+            print()
+
+        if self.warnings:
+            print(f"⚠️  WARNINGS ({len(self.warnings)}):")
+            for warning in self.warnings:
+                print(f"   • {warning}")
+            print()
+
+        if not self.errors and not self.warnings:
+            print("✅ ALL CHECKS PASSED - Report meets quality standards!\n")
+        elif not self.errors:
+            print("✅ VALIDATION PASSED (with warnings)\n")
+        else:
+            print("❌ VALIDATION FAILED - Please fix errors before delivery\n")
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Validate research report quality",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""
+Examples:
+  python validate_report.py --report report.md
+  python validate_report.py -r ~/.claude/research_output/research_report_20251104_153045.md
+        """
+    )
+
+    parser.add_argument(
+        '--report', '-r',
+        type=str,
+        required=True,
+        help='Path to research report markdown file'
+    )
+
+    args = parser.parse_args()
+
+    report_path = Path(args.report)
+
+    if not report_path.exists():
+        print(f"❌ ERROR: Report file not found: {report_path}")
+        sys.exit(1)
+
+    validator = ReportValidator(report_path)
+    passed = validator.validate()
+
+    sys.exit(0 if passed else 1)
+
+
+if __name__ == '__main__':
+    main()