fraim-framework 2.0.63 → 2.0.65

package/registry/scripts/scrape-site.py

@@ -1,302 +0,0 @@
-#!/usr/bin/env python3
-"""
-Reusable web scraper for any website
-Scrapes the entire site and collects HTML, CSS, JS, and structure information
-
-Usage:
-    python scrape_site.py <base_url> [--max-pages N] [--output-dir DIR]
-"""
-
-import requests
-from bs4 import BeautifulSoup
-from urllib.parse import urljoin, urlparse
-import json
-import time
-import sys
-import argparse
-from collections import defaultdict
-from pathlib import Path
-from datetime import datetime
-
-class SiteScraper:
-    def __init__(self, base_url, output_dir="."):
-        self.base_url = base_url.rstrip('/')
-        self.output_dir = Path(output_dir)
-        self.output_dir.mkdir(parents=True, exist_ok=True)
-        
-        self.visited_urls = set()
-        self.pages_data = []
-        self.css_files = set()
-        self.js_files = set()
-        self.images = set()
-        self.external_links = set()
-        self.forms = []
-        self.meta_tags = {}
-        self.session = requests.Session()
-        self.session.headers.update({
-            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
-        })
-        
-    def is_valid_url(self, url):
-        """Check if URL belongs to the same domain"""
-        parsed = urlparse(url)
-        base_parsed = urlparse(self.base_url)
-        return parsed.netloc == base_parsed.netloc or parsed.netloc == ''
-    
-    def normalize_url(self, url):
-        """Normalize URL to avoid duplicates"""
-        url = urljoin(self.base_url, url)
-        parsed = urlparse(url)
-        # Remove fragment
-        normalized = f"{parsed.scheme}://{parsed.netloc}{parsed.path}"
-        if parsed.query:
-            normalized += f"?{parsed.query}"
-        return normalized.rstrip('/')
-    
-    def extract_resources(self, soup, page_url):
-        """Extract CSS, JS, images, and other resources"""
-        # CSS files
-        for link in soup.find_all('link', rel='stylesheet'):
-            href = link.get('href')
-            if href:
-                full_url = urljoin(page_url, href)
-                self.css_files.add(full_url)
-        
-        # JavaScript files
-        for script in soup.find_all('script', src=True):
-            src = script.get('src')
-            if src:
-                full_url = urljoin(page_url, src)
-                self.js_files.add(full_url)
-        
-        # Images
-        for img in soup.find_all('img', src=True):
-            src = img.get('src')
-            if src:
-                full_url = urljoin(page_url, src)
-                self.images.add(full_url)
-        
-        # External links
-        for a in soup.find_all('a', href=True):
-            href = a.get('href')
-            if href:
-                parsed = urlparse(href)
-                if parsed.netloc and parsed.netloc != urlparse(self.base_url).netloc:
-                    self.external_links.add(href)
-    
-    def extract_forms(self, soup, page_url):
-        """Extract form information"""
-        for form in soup.find_all('form'):
-            form_data = {
-                'action': form.get('action', ''),
-                'method': form.get('method', 'GET').upper(),
-                'inputs': []
-            }
-            
-            for input_tag in form.find_all(['input', 'textarea', 'select']):
-                input_data = {
-                    'type': input_tag.get('type', input_tag.name),
-                    'name': input_tag.get('name', ''),
-                    'id': input_tag.get('id', ''),
-                    'placeholder': input_tag.get('placeholder', ''),
-                    'required': input_tag.has_attr('required'),
-                    'label': ''
-                }
-                
-                # Try to find associated label
-                if input_data['id']:
-                    label = soup.find('label', {'for': input_data['id']})
-                    if label:
-                        input_data['label'] = label.get_text(strip=True)
-                
-                form_data['inputs'].append(input_data)
-            
-            if form_data['inputs']:
-                self.forms.append({
-                    'page': page_url,
-                    'form': form_data
-                })
-    
-    def extract_meta_info(self, soup, page_url):
-        """Extract meta tags and page information"""
-        meta_info = {
-            'title': soup.title.string if soup.title else '',
-            'description': '',
-            'keywords': '',
-            'viewport': '',
-            'og_tags': {},
-            'other_meta': {}
-        }
-        
-        for meta in soup.find_all('meta'):
-            name = meta.get('name', '').lower()
-            property_attr = meta.get('property', '').lower()
-            content = meta.get('content', '')
-            
-            if name == 'description':
-                meta_info['description'] = content
-            elif name == 'keywords':
-                meta_info['keywords'] = content
-            elif name == 'viewport':
-                meta_info['viewport'] = content
-            elif property_attr.startswith('og:'):
-                meta_info['og_tags'][property_attr] = content
-            elif name:
-                meta_info['other_meta'][name] = content
-        
-        return meta_info
-    
-    def scrape_page(self, url):
-        """Scrape a single page"""
-        if url in self.visited_urls:
-            return None
-        
-        self.visited_urls.add(url)
-        print(f"  Scraping: {url}")
-        
-        try:
-            response = self.session.get(url, timeout=10)
-            response.raise_for_status()
-            
-            # Check if it's HTML
-            content_type = response.headers.get('Content-Type', '')
-            if 'text/html' not in content_type:
-                return None
-            
-            soup = BeautifulSoup(response.text, 'html.parser')
-            
-            # Extract page data
-            page_data = {
-                'url': url,
-                'status_code': response.status_code,
-                'title': soup.title.string if soup.title else '',
-                'meta': self.extract_meta_info(soup, url),
-                'headings': {
-                    'h1': [h.get_text(strip=True) for h in soup.find_all('h1')],
-                    'h2': [h.get_text(strip=True) for h in soup.find_all('h2')],
-                    'h3': [h.get_text(strip=True) for h in soup.find_all('h3')],
-                },
-                'links': [],
-                'text_content': soup.get_text(separator=' ', strip=True)[:5000],  # First 5000 chars
-                'html_structure': self.analyze_structure(soup),
-            }
-            
-            # Extract resources
-            self.extract_resources(soup, url)
-            self.extract_forms(soup, url)
-            
-            # Find internal links for further scraping
-            for link in soup.find_all('a', href=True):
-                href = link.get('href')
-                if href:
-                    normalized = self.normalize_url(href)
-                    if self.is_valid_url(normalized) and normalized not in self.visited_urls:
-                        page_data['links'].append(normalized)
-            
-            self.pages_data.append(page_data)
-            return page_data
-            
-        except Exception as e:
-            print(f"  ⚠️  Error scraping {url}: {e}")
-            return None
-    
-    def analyze_structure(self, soup):
-        """Analyze HTML structure"""
-        structure = {
-            'main_tags': defaultdict(int),
-            'ids': [],
-            'classes': [],
-            'data_attributes': []
-        }
-        
-        for tag in soup.find_all():
-            structure['main_tags'][tag.name] += 1
-            
-            if tag.get('id'):
-                structure['ids'].append(tag.get('id'))
-            
-            if tag.get('class'):
-                structure['classes'].extend(tag.get('class'))
-            
-            # Data attributes
-            for attr in tag.attrs:
-                if attr.startswith('data-'):
-                    structure['data_attributes'].append(attr)
-        
-        return {
-            'main_tags': dict(structure['main_tags']),
-            'unique_ids_count': len(set(structure['ids'])),
-            'unique_classes_count': len(set(structure['classes'])),
-            'data_attributes': list(set(structure['data_attributes']))
-        }
-    
-    def scrape_site(self, max_pages=50):
-        """Scrape the entire site starting from base URL"""
-        queue = [self.base_url]
-        
-        print(f"\n🚀 Starting to scrape {self.base_url}...")
-        print(f"   Max pages: {max_pages}\n")
-        
-        while queue and len(self.visited_urls) < max_pages:
-            current_url = queue.pop(0)
-            page_data = self.scrape_page(current_url)
-            
-            if page_data:
-                # Add new links to queue
-                for link in page_data.get('links', []):
-                    if link not in self.visited_urls and link not in queue:
-                        queue.append(link)
-            
-            time.sleep(0.5)  # Be polite
-    
-    def get_summary(self):
-        """Generate summary of scraped data"""
-        return {
-            'base_url': self.base_url,
-            'scrape_date': datetime.now().isoformat(),
-            'total_pages': len(self.pages_data),
-            'total_css_files': len(self.css_files),
-            'total_js_files': len(self.js_files),
-            'total_images': len(self.images),
-            'total_forms': len(self.forms),
-            'total_external_links': len(self.external_links),
-            'pages': self.pages_data,
-            'css_files': list(self.css_files),
-            'js_files': list(self.js_files),
-            'images': list(self.images)[:20],  # First 20
-            'external_links': list(self.external_links),
-            'forms': self.forms
-        }
-    
-    def save_results(self, filename='site_analysis.json'):
-        """Save results to JSON file"""
-        summary = self.get_summary()
-        output_path = self.output_dir / filename
-        
-        with open(output_path, 'w', encoding='utf-8') as f:
-            json.dump(summary, f, indent=2, ensure_ascii=False)
-        
-        print(f"\n✅ Scraping complete!")
-        print(f"   📄 Pages scraped: {summary['total_pages']}")
-        print(f"   🎨 CSS files: {summary['total_css_files']}")
-        print(f"   📜 JS files: {summary['total_js_files']}")
-        print(f"   🖼️  Images: {summary['total_images']}")
-        print(f"   📝 Forms: {summary['total_forms']}")
-        print(f"   🔗 External links: {summary['total_external_links']}")
-        print(f"\n   💾 Data saved to: {output_path}")
-
-def main():
-    parser = argparse.ArgumentParser(description='Scrape a website and analyze its structure')
-    parser.add_argument('url', help='Base URL of the website to scrape')
-    parser.add_argument('--max-pages', type=int, default=50, help='Maximum number of pages to scrape (default: 50)')
-    parser.add_argument('--output-dir', default='.', help='Output directory for results (default: current directory)')
-    parser.add_argument('--output-file', default='site_analysis.json', help='Output filename (default: site_analysis.json)')
-    
-    args = parser.parse_args()
-    
-    scraper = SiteScraper(args.url, args.output_dir)
-    scraper.scrape_site(max_pages=args.max_pages)
-    scraper.save_results(args.output_file)
-
-if __name__ == "__main__":
-    main()

package/registry/scripts/send-newsletter-simple.ts

@@ -1,102 +0,0 @@
-#!/usr/bin/env tsx
-
-/**
- * Send Newsletter - Deterministic Script
- * 
- * Simple script that users run after approving newsletter JSON.
- * No decisions, just sends the newsletter.
- * 
- * Usage:
- *   npm run send-newsletter                                    # Send to all executives and potential customers
- *   npm run send-newsletter -- --showonly                      # List recipients without sending
- *   npm run send-newsletter -- --no-potential-customers        # Send only to active executives
- *   npm run send-newsletter -- --only-potential-customers      # Send only to potential customers
- *   npm run send-newsletter -- --email=user@example.com        # Send to specific email
- *   npm run send-newsletter -- exec-XXX exec-YYY               # Send to specific executive IDs
- */
-
-import { sendNewsletterToExecutives } from './newsletter-helpers';
-import { readdirSync, existsSync } from 'fs';
-import { join } from 'path';
-
-async function main() {
-  console.log('📧 Weekly Newsletter Sender\n');
-
-  // Parse command line arguments
-  const args = process.argv.slice(2);
-  const showOnly = args.includes('--showonly');
-  const excludePotentialCustomers = args.includes('--no-potential-customers');
-  const onlyPotentialCustomers = args.includes('--only-potential-customers');
-  const filterEmails: string[] = [];
-  const filterExecIds: string[] = [];
-
-  // Parse email filters (--email=address)
-  args.forEach(arg => {
-    if (arg.startsWith('--email=')) {
-      filterEmails.push(arg.substring(8));
-    } else if (arg.startsWith('exec-')) {
-      filterExecIds.push(arg);
-    }
-  });
-
-  // Find the most recent newsletter
-  const newslettersDir = join(process.cwd(), 'docs', 'customer-development', 'newsletters');
-
-  if (!existsSync(newslettersDir)) {
-    console.error('❌ Error: No newsletters directory found');
-    console.error('   Expected: docs/customer-development/newsletters/');
-    console.error('   Generate a newsletter first using the AI agent workflow');
-    process.exit(1);
-  }
-
-  const files = readdirSync(newslettersDir)
-    .filter(f => f.startsWith('newsletter-') && f.endsWith('.json'))
-    .sort()
-    .reverse(); // Most recent first
-
-  if (files.length === 0) {
-    console.error('❌ Error: No newsletter files found');
-    console.error('   Generate a newsletter first using the AI agent workflow');
-    process.exit(1);
-  }
-
-  const latestNewsletter = files[0];
-  const newsletterPath = join(newslettersDir, latestNewsletter);
-
-  console.log(`📄 Found newsletter: ${latestNewsletter}`);
-
-  if (showOnly) {
-    console.log(`📋 Listing recipients (--showonly mode - no emails will be sent)...\n`);
-  } else if (filterEmails.length > 0 || filterExecIds.length > 0) {
-    console.log(`📧 Sending to filtered recipients...\n`);
-    console.log(`⚠️  This will send real emails. Make sure the newsletter is approved!`);
-  } else {
-    if (onlyPotentialCustomers) {
-      console.log(`📧 Sending to potential customers only...\n`);
-    } else {
-      console.log(`📧 Sending to all active executives${excludePotentialCustomers ? '' : ' and potential customers'}...\n`);
-    }
-    console.log(`⚠️  This will send real emails. Make sure the newsletter is approved!`);
-  }
-
-  console.log(`   Preview: ${newsletterPath.replace('.json', '.html')}\n`);
-
-  // Determine what to include based on flags
-  const includePotentialCustomers = onlyPotentialCustomers || !excludePotentialCustomers;
-  const includeExecutives = !onlyPotentialCustomers;
-
-  // Send newsletter (or show only)
-  await sendNewsletterToExecutives(
-    newsletterPath,
-    filterEmails.length > 0 ? filterEmails : undefined,
-    filterExecIds.length > 0 ? filterExecIds : undefined,
-    includePotentialCustomers,
-    showOnly,
-    includeExecutives
-  );
-}
-
-main().catch(error => {
-  console.error('❌ Error:', error.message);
-  process.exit(1);
-});

package/registry/scripts/send-thank-you-emails.ts

@@ -1,57 +0,0 @@
-#!/usr/bin/env tsx
-
-/**
- * Send Thank-You Emails from Candidates JSON File
- * 
- * Usage:
- *   npm run send-thank-you -- <filename> [exec-id-1] [exec-id-2] ...
- * 
- * Examples:
- *   npm run send-thank-you -- docs/customer-development/thank-you-notes/thank-you-candidates-2025-10-29.json
- *   npm run send-thank-you -- docs/customer-development/thank-you-notes/thank-you-candidates-2025-10-29.json exec-123 exec-456
- */
-
-import { sendCustomerMail } from './generate-engagement-emails';
-
-async function main() {
-  const args = process.argv.slice(2);
-
-  if (args.length === 0) {
-    console.error('❌ Error: Missing required argument: filename');
-    console.error('');
-    console.error('Usage: npm run send-thank-you -- <filename> [exec-id-1] [exec-id-2] ...');
-    console.error('');
-    console.error('Examples:');
-    console.error('  npm run send-thank-you -- docs/customer-development/thank-you-notes/thank-you-candidates-2025-10-29.json');
-    console.error('  npm run send-thank-you -- docs/customer-development/thank-you-notes/thank-you-candidates-2025-10-29.json exec-1760627251980-31qmjy5y4');
-    process.exit(1);
-  }
-
-  const filename = args[0];
-  const execIds = args.slice(1).filter(arg => arg.startsWith('exec-'));
-
-  if (execIds.length === 0) {
-    // Send to all pending candidates
-    console.log(`📧 Sending emails from: ${filename}`);
-    console.log(`🎯 Sending to ALL pending candidates`);
-    await sendCustomerMail(filename);
-  } else if (execIds.length === 1) {
-    // Send to specific executive
-    console.log(`📧 Sending emails from: ${filename}`);
-    console.log(`🎯 Sending to executive: ${execIds[0]}`);
-    await sendCustomerMail(filename, execIds[0]);
-  } else {
-    // Multiple executives - send to each one
-    console.log(`📧 Sending emails from: ${filename}`);
-    console.log(`🎯 Sending to ${execIds.length} executives: ${execIds.join(', ')}`);
-    for (const execId of execIds) {
-      console.log(`\n📧 Processing executive: ${execId}`);
-      await sendCustomerMail(filename, execId);
-    }
-  }
-}
-
-main().catch(error => {
-  console.error('❌ Error:', error);
-  process.exit(1);
-});