@rudderhq/server 0.1.0-canary.0

package/resources/community-skills/deep-research/scripts/verify_citations.py

@@ -0,0 +1,426 @@
+#!/usr/bin/env python3
+"""
+Citation Verification Script
+
+Catches fabricated citations by checking:
+1. DOI resolution (via doi.org)
+2. Basic metadata matching (title similarity, year match)
+3. URL accessibility verification
+4. Hallucination pattern detection (generic titles, suspicious patterns)
+5. Flags suspicious entries for manual review
+
+Usage:
+    python verify_citations.py --report [path]
+    python verify_citations.py --report [path] --strict  # Fail on any unverified
+
+Does NOT require API keys - uses free DOI resolver and heuristics.
+"""
+
+import sys
+import argparse
+import re
+from pathlib import Path
+from typing import List, Dict, Tuple
+from urllib import request, error
+from urllib.parse import quote
+import json
+import time
+from datetime import datetime
+
+class CitationVerifier:
+    """Verify citations in research report"""
+
+    def __init__(self, report_path: Path, strict_mode: bool = False):
+        self.report_path = report_path
+        self.strict_mode = strict_mode
+        self.content = self._read_report()
+        self.suspicious = []
+        self.verified = []
+        self.errors = []
+
+        # Hallucination detection patterns (2025 CiteGuard enhancement)
+        self.suspicious_patterns = [
+            # Generic academic-sounding but fake patterns
+            (r'^(A |An |The )?(Study|Analysis|Review|Survey|Investigation) (of|on|into)',
+             "Generic academic title pattern"),
+            (r'^(Recent|Current|Modern|Contemporary) (Advances|Developments|Trends) in',
+             "Generic 'advances' title pattern"),
+            # Too perfect, templated titles
+            (r'^[A-Z][a-z]+ [A-Z][a-z]+: A (Comprehensive|Complete|Systematic) (Review|Analysis|Guide)$',
+             "Too perfect, templated structure"),
+        ]
+
+    def _read_report(self) -> str:
+        """Read report file"""
+        try:
+            with open(self.report_path, 'r', encoding='utf-8') as f:
+                return f.read()
+        except Exception as e:
+            print(f"L ERROR: Cannot read report: {e}")
+            sys.exit(1)
+
+    def extract_bibliography(self) -> List[Dict]:
+        """Extract bibliography entries from report"""
+        pattern = r'## Bibliography(.*?)(?=##|\Z)'
+        match = re.search(pattern, self.content, re.DOTALL | re.IGNORECASE)
+
+        if not match:
+            self.errors.append("No Bibliography section found")
+            return []
+
+        bib_section = match.group(1)
+
+        # Parse entries: [N] Author (Year). "Title". Venue. URL
+        entries = []
+        lines = bib_section.strip().split('\n')
+
+        current_entry = None
+        for line in lines:
+            line = line.strip()
+            if not line:
+                continue
+
+            # Check if starts with citation number [N]
+            match_num = re.match(r'^\[(\d+)\]\s+(.+)$', line)
+            if match_num:
+                if current_entry:
+                    entries.append(current_entry)
+
+                num = match_num.group(1)
+                rest = match_num.group(2)
+
+                # Try to parse: Author (Year). "Title". Venue. URL
+                year_match = re.search(r'\((\d{4})\)', rest)
+                title_match = re.search(r'"([^"]+)"', rest)
+                doi_match = re.search(r'doi\.org/(10\.\S+)', rest)
+                url_match = re.search(r'https?://[^\s\)]+', rest)
+
+                current_entry = {
+                    'num': num,
+                    'raw': rest,
+                    'year': year_match.group(1) if year_match else None,
+                    'title': title_match.group(1) if title_match else None,
+                    'doi': doi_match.group(1) if doi_match else None,
+                    'url': url_match.group(0) if url_match else None
+                }
+            elif current_entry:
+                # Multi-line entry, append to raw
+                current_entry['raw'] += ' ' + line
+
+        if current_entry:
+            entries.append(current_entry)
+
+        return entries
+
+    def verify_doi(self, doi: str) -> Tuple[bool, Dict]:
+        """
+        Verify DOI exists and get metadata.
+        Returns (success, metadata_dict)
+        """
+        if not doi:
+            return False, {}
+
+        try:
+            # Use content negotiation to get JSON metadata
+            url = f"https://doi.org/{quote(doi)}"
+            req = request.Request(url)
+            req.add_header('Accept', 'application/vnd.citationstyles.csl+json')
+
+            with request.urlopen(req, timeout=10) as response:
+                data = json.loads(response.read().decode('utf-8'))
+
+                return True, {
+                    'title': data.get('title', ''),
+                    'year': data.get('issued', {}).get('date-parts', [[None]])[0][0],
+                    'authors': [
+                        f"{a.get('family', '')} {a.get('given', '')}"
+                        for a in data.get('author', [])
+                    ],
+                    'venue': data.get('container-title', '')
+                }
+        except error.HTTPError as e:
+            if e.code == 404:
+                return False, {'error': 'DOI not found (404)'}
+            return False, {'error': f'HTTP {e.code}'}
+        except Exception as e:
+            return False, {'error': str(e)}
+
+    def verify_url(self, url: str) -> Tuple[bool, str]:
+        """
+        Verify URL is accessible (2025 CiteGuard enhancement).
+        Returns (accessible, status_message)
+        """
+        if not url:
+            return False, "No URL"
+
+        try:
+            # HEAD request to check accessibility without downloading
+            req = request.Request(url, method='HEAD')
+            req.add_header('User-Agent', 'Mozilla/5.0 (Research Citation Verifier)')
+
+            with request.urlopen(req, timeout=10) as response:
+                if response.status == 200:
+                    return True, "URL accessible"
+                else:
+                    return False, f"HTTP {response.status}"
+        except error.HTTPError as e:
+            return False, f"HTTP {e.code}"
+        except error.URLError as e:
+            return False, f"URL error: {e.reason}"
+        except Exception as e:
+            return False, f"Connection error: {str(e)[:50]}"
+
+    def detect_hallucination_patterns(self, entry: Dict) -> List[str]:
+        """
+        Detect common LLM hallucination patterns in citations (2025 CiteGuard).
+        Returns list of detected issues.
+        """
+        issues = []
+        title = entry.get('title', '')
+
+        if not title:
+            return issues
+
+        # Check against suspicious patterns
+        for pattern, description in self.suspicious_patterns:
+            if re.match(pattern, title, re.IGNORECASE):
+                issues.append(f"Suspicious title pattern: {description}")
+
+        # Check for overly generic titles
+        generic_words = ['overview', 'introduction', 'guide', 'handbook', 'manual']
+        if any(word in title.lower() for word in generic_words) and len(title.split()) < 5:
+            issues.append("Very generic short title")
+
+        # Check for placeholder-like titles
+        if any(x in title.lower() for x in ['tbd', 'todo', 'placeholder', 'example']):
+            issues.append("Placeholder text in title")
+
+        # Check for inconsistent metadata
+        if entry.get('year'):
+            year = int(entry['year'])
+            current_year = datetime.now().year
+            # Very recent without DOI or URL is suspicious
+            if year >= current_year - 1 and not entry.get('doi') and not entry.get('url'):
+                issues.append(f"Recent year ({year}) with no verification method")
+            # Future year is definitely wrong
+            if year > current_year:
+                issues.append(f"Future year: {year} (current: {current_year})")
+            # Very old with modern phrasing is suspicious
+            if year < 2000 and any(word in title.lower() for word in ['ai', 'llm', 'gpt', 'transformer']):
+                issues.append(f"Anachronistic: pre-2000 ({year}) citation mentioning modern AI terms")
+
+        return issues
+
+    def check_title_similarity(self, title1: str, title2: str) -> float:
+        """
+        Simple title similarity check (word overlap).
+        Returns score 0.0-1.0
+        """
+        if not title1 or not title2:
+            return 0.0
+
+        # Normalize: lowercase, remove punctuation, split
+        def normalize(s):
+            s = s.lower()
+            s = re.sub(r'[^\w\s]', ' ', s)
+            return set(s.split())
+
+        words1 = normalize(title1)
+        words2 = normalize(title2)
+
+        if not words1 or not words2:
+            return 0.0
+
+        overlap = len(words1 & words2)
+        total = len(words1 | words2)
+
+        return overlap / total if total > 0 else 0.0
+
+    def verify_entry(self, entry: Dict) -> Dict:
+        """Verify a single bibliography entry (Enhanced 2025 with CiteGuard)"""
+        result = {
+            'num': entry['num'],
+            'status': 'unknown',
+            'issues': [],
+            'metadata': {},
+            'verification_methods': []
+        }
+
+        # STEP 1: Run hallucination detection (CiteGuard 2025)
+        hallucination_issues = self.detect_hallucination_patterns(entry)
+        if hallucination_issues:
+            result['issues'].extend(hallucination_issues)
+            result['status'] = 'suspicious'
+
+        # STEP 2: Has DOI?
+        if entry['doi']:
+            print(f"  [{entry['num']}] Checking DOI {entry['doi']}...", end=' ')
+            success, metadata = self.verify_doi(entry['doi'])
+
+            if success:
+                result['metadata'] = metadata
+                result['status'] = 'verified'
+                print("")
+
+                # Check title similarity if we have both
+                if entry['title'] and metadata.get('title'):
+                    similarity = self.check_title_similarity(
+                        entry['title'],
+                        metadata['title']
+                    )
+
+                    if similarity < 0.5:
+                        result['issues'].append(
+                            f"Title mismatch (similarity: {similarity:.1%})"
+                        )
+                        result['status'] = 'suspicious'
+
+                # Check year match
+                if entry['year'] and metadata.get('year'):
+                    if int(entry['year']) != int(metadata['year']):
+                        result['issues'].append(
+                            f"Year mismatch: report says {entry['year']}, DOI says {metadata['year']}"
+                        )
+                        result['status'] = 'suspicious'
+
+            else:
+                print(f"✗ {metadata.get('error', 'Failed')}")
+                result['status'] = 'unverified'
+                result['issues'].append(f"DOI resolution failed: {metadata.get('error', 'unknown')}")
+
+        # STEP 3: Check URL accessibility (if no DOI or DOI failed)
+        if entry['url'] and result['status'] != 'verified':
+            url_ok, url_status = self.verify_url(entry['url'])
+            if url_ok:
+                result['verification_methods'].append('URL')
+                # Upgrade status if URL verifies
+                if result['status'] in ['unknown', 'no_doi', 'unverified']:
+                    result['status'] = 'url_verified'
+                print(f"  [{entry['num']}] URL accessible ✓")
+            else:
+                result['issues'].append(f"URL check failed: {url_status}")
+
+        # STEP 4: Final fallback - no verification method
+        if not entry['doi'] and not entry['url']:
+            if 'No DOI provided' not in ' '.join(result['issues']):
+                result['issues'].append("No DOI or URL - cannot verify")
+            result['status'] = 'suspicious'
+
+        return result
+
+    def verify_all(self):
+        """Verify all bibliography entries"""
+        print(f"\n{'='*60}")
+        print(f"CITATION VERIFICATION: {self.report_path.name}")
+        print(f"{'='*60}\n")
+
+        entries = self.extract_bibliography()
+
+        if not entries:
+            print("L No bibliography entries found\n")
+            return False
+
+        print(f"Found {len(entries)} citations\n")
+
+        results = []
+        for entry in entries:
+            result = self.verify_entry(entry)
+            results.append(result)
+
+            # Rate limiting
+            time.sleep(0.5)
+
+        # Summarize
+        print(f"\n{'='*60}")
+        print(f"VERIFICATION SUMMARY")
+        print(f"{'='*60}\n")
+
+        verified = [r for r in results if r['status'] == 'verified']
+        url_verified = [r for r in results if r['status'] == 'url_verified']
+        suspicious = [r for r in results if r['status'] == 'suspicious']
+        unverified = [r for r in results if r['status'] in ['unverified', 'no_doi', 'unknown']]
+
+        print(f'DOI Verified: {len(verified)}/{len(results)}')
+        print(f'URL Verified: {len(url_verified)}/{len(results)}')
+        print(f'Suspicious: {len(suspicious)}/{len(results)}')
+        print(f'Unverified: {len(unverified)}/{len(results)}')
+        print()
+
+        if suspicious:
+            print('SUSPICIOUS CITATIONS (Manual Review Needed):')
+            for r in suspicious:
+                print(f"\n  [{r['num']}]")
+                for issue in r['issues']:
+                    print(f"    - {issue}")
+            print()
+
+        if unverified and len(unverified) > 0:
+            print('UNVERIFIED CITATIONS (Could not check):')
+            for r in unverified:
+                print(f"  [{r['num']}] {r['issues'][0] if r['issues'] else 'Unknown'}")
+            print()
+
+        # Decision (Enhanced 2025 - includes URL-verified as acceptable)
+        total_verified = len(verified) + len(url_verified)
+
+        if suspicious:
+            print('WARNING: Suspicious citations detected')
+            if self.strict_mode:
+                print('  STRICT MODE: Failing due to suspicious citations')
+                return False
+            else:
+                print('  (Continuing in non-strict mode)')
+
+        if self.strict_mode and unverified:
+            print('STRICT MODE: Unverified citations found')
+            return False
+
+        if total_verified / len(results) < 0.5:
+            print('WARNING: Less than 50% citations verified')
+            return True  # Pass with warning
+        else:
+            print('CITATION VERIFICATION PASSED')
+            return True
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Verify citations in research report",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""
+Examples:
+  python verify_citations.py --report report.md
+
+Note: Requires internet connection to check DOIs.
+Uses free DOI resolver - no API key needed.
+        """
+    )
+
+    parser.add_argument(
+        '--report', '-r',
+        type=str,
+        required=True,
+        help='Path to research report markdown file'
+    )
+
+    parser.add_argument(
+        '--strict',
+        action='store_true',
+        help='Strict mode: fail on any unverified or suspicious citations'
+    )
+
+    args = parser.parse_args()
+    report_path = Path(args.report)
+
+    if not report_path.exists():
+        print(f"ERROR: Report file not found: {report_path}")
+        sys.exit(1)
+
+    verifier = CitationVerifier(report_path, strict_mode=args.strict)
+    passed = verifier.verify_all()
+
+    sys.exit(0 if passed else 1)
+
+
+if __name__ == '__main__':
+    main()

package/resources/community-skills/deep-research/scripts/verify_html.py

@@ -0,0 +1,220 @@
+#!/usr/bin/env python3
+"""
+HTML Report Verification Script
+Validates that HTML reports are properly generated with all sections from MD
+"""
+
+import argparse
+import re
+from pathlib import Path
+from typing import List, Tuple
+
+
+class HTMLVerifier:
+    """Verify HTML research reports"""
+
+    def __init__(self, html_path: Path, md_path: Path):
+        self.html_path = html_path
+        self.md_path = md_path
+        self.errors = []
+        self.warnings = []
+
+    def verify(self) -> bool:
+        """
+        Run all verification checks
+
+        Returns:
+            True if all checks pass, False otherwise
+        """
+        print(f"\n{'='*60}")
+        print(f"HTML REPORT VERIFICATION")
+        print(f"{'='*60}\n")
+
+        print(f"HTML File: {self.html_path}")
+        print(f"MD File: {self.md_path}\n")
+
+        # Read files
+        try:
+            html_content = self.html_path.read_text()
+            md_content = self.md_path.read_text()
+        except Exception as e:
+            self.errors.append(f"Failed to read files: {e}")
+            return False
+
+        # Run checks
+        self._check_sections(html_content, md_content)
+        self._check_no_placeholders(html_content)
+        self._check_no_emojis(html_content)
+        self._check_structure(html_content)
+        self._check_citations(html_content, md_content)
+        self._check_bibliography(html_content, md_content)
+
+        # Report results
+        self._print_results()
+
+        return len(self.errors) == 0
+
+    def _check_sections(self, html: str, md: str):
+        """Verify all markdown sections are present in HTML"""
+        # Extract section headings from markdown
+        md_sections = re.findall(r'^## (.+)$', md, re.MULTILINE)
+
+        # Extract sections from HTML
+        html_sections = re.findall(r'<h2 class="section-title">(.+?)</h2>', html)
+
+        # Check if we have placeholder sections like <div class="section">#</div>
+        placeholder_sections = re.findall(r'<div class="section">#</div>', html)
+
+        if placeholder_sections:
+            self.errors.append(
+                f"Found {len(placeholder_sections)} placeholder sections (empty '#' divs) - content not converted properly"
+            )
+
+        # Compare section counts
+        if len(md_sections) > len(html_sections) + 1:  # +1 for bibliography which is separate
+            self.errors.append(
+                f"Section count mismatch: MD has {len(md_sections)} sections, HTML has only {len(html_sections)} + bibliography"
+            )
+            missing = set(md_sections) - set(html_sections)
+            if missing:
+                self.errors.append(f"Missing sections in HTML: {missing}")
+
+        # Verify Executive Summary is present
+        if "Executive Summary" in md and "Executive Summary" not in html:
+            self.errors.append("Executive Summary missing from HTML")
+
+    def _check_no_placeholders(self, html: str):
+        """Check for common placeholders that shouldn't be in final report"""
+        placeholders = [
+            '{{TITLE}}', '{{DATE}}', '{{CONTENT}}', '{{BIBLIOGRAPHY}}',
+            '{{METRICS_DASHBOARD}}', '{{SOURCE_COUNT}}', 'TODO', 'TBD',
+            'PLACEHOLDER', 'FIXME'
+        ]
+
+        found = []
+        for placeholder in placeholders:
+            if placeholder in html:
+                found.append(placeholder)
+
+        if found:
+            self.errors.append(f"Found unreplaced placeholders: {', '.join(found)}")
+
+    def _check_no_emojis(self, html: str):
+        """Verify no emojis are present in HTML"""
+        # Common emoji patterns
+        emoji_pattern = re.compile(
+            "["
+            "\U0001F600-\U0001F64F"  # emoticons
+            "\U0001F300-\U0001F5FF"  # symbols & pictographs
+            "\U0001F680-\U0001F6FF"  # transport & map symbols
+            "\U0001F1E0-\U0001F1FF"  # flags
+            "\U00002702-\U000027B0"
+            "\U000024C2-\U0001F251"
+            "]+",
+            flags=re.UNICODE
+        )
+
+        emojis = emoji_pattern.findall(html)
+        if emojis:
+            unique_emojis = set(emojis)
+            self.errors.append(f"Found {len(emojis)} emojis in HTML (should be none): {unique_emojis}")
+
+    def _check_structure(self, html: str):
+        """Verify HTML has proper structure"""
+        required_elements = [
+            ('<html', 'HTML tag'),
+            ('<head', 'head tag'),
+            ('<body', 'body tag'),
+            ('<title>', 'title tag'),
+            ('class="header"', 'header section'),
+            ('class="content"', 'content section'),
+            ('class="bibliography"', 'bibliography section'),
+        ]
+
+        for element, name in required_elements:
+            if element not in html:
+                self.errors.append(f"Missing {name} in HTML")
+
+        # Check for unclosed tags (basic check)
+        open_divs = html.count('<div')
+        close_divs = html.count('</div>')
+
+        if abs(open_divs - close_divs) > 2:  # Allow small discrepancy
+            self.warnings.append(
+                f"Possible unclosed divs: {open_divs} opening tags, {close_divs} closing tags"
+            )
+
+    def _check_citations(self, html: str, md: str):
+        """Verify citations are present"""
+        # Extract citations from markdown
+        md_citations = set(re.findall(r'\[(\d+)\]', md))
+
+        # Extract citations from HTML (excluding bibliography)
+        html_content = html.split('class="bibliography"')[0] if 'class="bibliography"' in html else html
+        html_citations = set(re.findall(r'\[(\d+)\]', html_content))
+
+        if len(md_citations) > 0 and len(html_citations) == 0:
+            self.errors.append("No citations found in HTML content (but present in MD)")
+
+        if len(md_citations) > len(html_citations) * 1.5:  # Allow some variation
+            self.warnings.append(
+                f"Fewer citations in HTML ({len(html_citations)}) than MD ({len(md_citations)})"
+            )
+
+    def _check_bibliography(self, html: str, md: str):
+        """Verify bibliography is present and formatted"""
+        if '## Bibliography' in md:
+            if 'class="bibliography"' not in html:
+                self.errors.append("Bibliography section missing from HTML")
+            elif 'class="bib-entry"' not in html:
+                self.warnings.append("Bibliography present but entries not properly formatted")
+
+    def _print_results(self):
+        """Print verification results"""
+        print(f"\n{'-'*60}")
+        print("VERIFICATION RESULTS")
+        print(f"{'-'*60}\n")
+
+        if self.errors:
+            print(f"❌ ERRORS ({len(self.errors)}):")
+            for i, error in enumerate(self.errors, 1):
+                print(f"  {i}. {error}")
+            print()
+
+        if self.warnings:
+            print(f"⚠️  WARNINGS ({len(self.warnings)}):")
+            for i, warning in enumerate(self.warnings, 1):
+                print(f"  {i}. {warning}")
+            print()
+
+        if not self.errors and not self.warnings:
+            print("✅ All checks passed! HTML report is valid.")
+            print()
+
+        print(f"{'-'*60}\n")
+
+
+def main():
+    """Main entry point"""
+    parser = argparse.ArgumentParser(description='Verify HTML research report')
+    parser.add_argument('--html', type=Path, required=True, help='Path to HTML report')
+    parser.add_argument('--md', type=Path, required=True, help='Path to markdown report')
+
+    args = parser.parse_args()
+
+    if not args.html.exists():
+        print(f"Error: HTML file not found: {args.html}")
+        return 1
+
+    if not args.md.exists():
+        print(f"Error: Markdown file not found: {args.md}")
+        return 1
+
+    verifier = HTMLVerifier(args.html, args.md)
+    success = verifier.verify()
+
+    return 0 if success else 1
+
+
+if __name__ == "__main__":
+    exit(main())