@musashishao/agent-kit 1.8.1 → 1.9.0

package/.agent/skills/browser-automation/SKILL.md

@@ -0,0 +1,408 @@
+---
+name: browser-automation
+description: "Browser automation with Playwright and Puppeteer. Web scraping, E2E testing, screenshot capture, and PDF generation."
+version: "1.0.0"
+---
+
+# 🌐 Browser Automation
+
+You are a browser automation expert who builds reliable scraping pipelines, E2E tests, and automation workflows. You handle dynamic content, anti-bot measures, and scale across multiple pages.
+
+---
+
+## When to Use This Skill
+
+- Web scraping and data extraction
+- E2E testing automation
+- Screenshot/PDF generation
+- Form filling and submission
+- Monitoring and alerting
+- Social media automation
+
+---
+
+## Capabilities
+
+- `playwright`
+- `puppeteer`
+- `web-scraping`
+- `e2e-testing`
+- `screenshot-capture`
+- `pdf-generation`
+
+---
+
+## 1. Playwright Setup
+
+```bash
+npm install playwright
+npx playwright install
+```
+
+### Basic Automation
+
+```typescript
+// scraper.ts
+import { chromium, Browser, Page } from 'playwright';
+
+async function scrape(url: string) {
+  const browser = await chromium.launch({
+    headless: true, // Set false for debugging
+  });
+  
+  const context = await browser.newContext({
+    userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36',
+    viewport: { width: 1920, height: 1080 },
+  });
+  
+  const page = await context.newPage();
+  
+  try {
+    await page.goto(url, { waitUntil: 'networkidle' });
+    
+    // Extract data
+    const data = await page.evaluate(() => {
+      const items = document.querySelectorAll('.product-card');
+      return Array.from(items).map(item => ({
+        title: item.querySelector('h2')?.textContent?.trim(),
+        price: item.querySelector('.price')?.textContent?.trim(),
+        image: item.querySelector('img')?.src,
+      }));
+    });
+    
+    return data;
+  } finally {
+    await browser.close();
+  }
+}
+```
+
+### Waiting Strategies
+
+```typescript
+// Wait for selector
+await page.waitForSelector('.content');
+
+// Wait for text
+await page.waitForSelector('text=Loading complete');
+
+// Wait for network idle
+await page.waitForLoadState('networkidle');
+
+// Wait for navigation
+await Promise.all([
+  page.waitForNavigation(),
+  page.click('a.next-page'),
+]);
+
+// Custom wait
+await page.waitForFunction(() => {
+  return document.querySelectorAll('.item').length >= 10;
+});
+
+// Wait with timeout
+await page.waitForSelector('.slow-content', { timeout: 30000 });
+```
+
+---
+
+## 2. Form Automation
+
+```typescript
+async function fillForm(page: Page) {
+  // Text input
+  await page.fill('input[name="email"]', 'user@example.com');
+  await page.fill('input[name="password"]', 'securepassword');
+  
+  // Select dropdown
+  await page.selectOption('select[name="country"]', 'US');
+  
+  // Checkbox
+  await page.check('input[name="terms"]');
+  
+  // Radio button
+  await page.click('input[value="express"]');
+  
+  // File upload
+  await page.setInputFiles('input[type="file"]', './document.pdf');
+  
+  // Date input
+  await page.fill('input[type="date"]', '2024-12-25');
+  
+  // Submit
+  await page.click('button[type="submit"]');
+  
+  // Wait for success
+  await page.waitForSelector('.success-message');
+}
+```
+
+---
+
+## 3. Authentication Handling
+
+```typescript
+// Save session
+async function loginAndSaveSession(page: Page) {
+  await page.goto('https://example.com/login');
+  await page.fill('#email', 'user@example.com');
+  await page.fill('#password', 'password');
+  await page.click('button[type="submit"]');
+  
+  await page.waitForURL('**/dashboard');
+  
+  // Save cookies
+  const cookies = await page.context().cookies();
+  await fs.writeFile('cookies.json', JSON.stringify(cookies));
+}
+
+// Restore session
+async function restoreSession(context: BrowserContext) {
+  const cookies = JSON.parse(await fs.readFile('cookies.json', 'utf-8'));
+  await context.addCookies(cookies);
+}
+
+// Using storage state
+async function saveStorageState() {
+  await context.storageState({ path: 'auth.json' });
+}
+
+// Restore storage state
+const context = await browser.newContext({
+  storageState: 'auth.json',
+});
+```
+
+---
+
+## 4. Handling Dynamic Content
+
+```typescript
+// Infinite scroll
+async function scrapeInfiniteScroll(page: Page, maxItems: number) {
+  const items: any[] = [];
+  
+  while (items.length < maxItems) {
+    // Get current items
+    const newItems = await page.$$eval('.item', els => 
+      els.map(el => el.textContent)
+    );
+    
+    items.push(...newItems.filter(item => !items.includes(item)));
+    
+    // Check if more content
+    const hasMore = await page.$('.load-more');
+    if (!hasMore) break;
+    
+    // Scroll to bottom
+    await page.evaluate(() => window.scrollTo(0, document.body.scrollHeight));
+    
+    // Wait for new content
+    await page.waitForTimeout(1000);
+  }
+  
+  return items.slice(0, maxItems);
+}
+
+// Handle lazy loading images
+async function loadAllImages(page: Page) {
+  await page.evaluate(async () => {
+    const images = document.querySelectorAll('img[data-src]');
+    images.forEach(img => {
+      img.src = img.getAttribute('data-src')!;
+    });
+    
+    // Wait for all images to load
+    await Promise.all(
+      Array.from(images).map(img => 
+        new Promise(resolve => {
+          if (img.complete) resolve(null);
+          else img.onload = () => resolve(null);
+        })
+      )
+    );
+  });
+}
+```
+
+---
+
+## 5. Screenshot & PDF
+
+```typescript
+// Full page screenshot
+await page.screenshot({
+  path: 'screenshot.png',
+  fullPage: true,
+});
+
+// Element screenshot
+const element = await page.$('.chart');
+await element?.screenshot({ path: 'chart.png' });
+
+// PDF generation
+await page.pdf({
+  path: 'document.pdf',
+  format: 'A4',
+  margin: { top: '20mm', bottom: '20mm', left: '15mm', right: '15mm' },
+  printBackground: true,
+});
+
+// Screenshot with clip
+await page.screenshot({
+  path: 'cropped.png',
+  clip: { x: 0, y: 0, width: 800, height: 600 },
+});
+```
+
+---
+
+## 6. Parallel Execution
+
+```typescript
+import { chromium, Browser } from 'playwright';
+
+async function scrapeMultipleUrls(urls: string[]) {
+  const browser = await chromium.launch();
+  
+  // Limit concurrency
+  const CONCURRENCY = 5;
+  const results: any[] = [];
+  
+  for (let i = 0; i < urls.length; i += CONCURRENCY) {
+    const batch = urls.slice(i, i + CONCURRENCY);
+    
+    const batchResults = await Promise.all(
+      batch.map(url => scrapePage(browser, url))
+    );
+    
+    results.push(...batchResults);
+  }
+  
+  await browser.close();
+  return results;
+}
+
+async function scrapePage(browser: Browser, url: string) {
+  const context = await browser.newContext();
+  const page = await context.newPage();
+  
+  try {
+    await page.goto(url);
+    const data = await extractData(page);
+    return { url, data, success: true };
+  } catch (error) {
+    return { url, error: error.message, success: false };
+  } finally {
+    await context.close();
+  }
+}
+```
+
+---
+
+## 7. Anti-Detection
+
+```typescript
+const context = await browser.newContext({
+  userAgent: randomUserAgent(),
+  viewport: { width: 1920, height: 1080 },
+  locale: 'en-US',
+  timezoneId: 'America/New_York',
+  geolocation: { latitude: 40.7128, longitude: -74.0060 },
+  permissions: ['geolocation'],
+});
+
+// Stealth mode
+await page.addInitScript(() => {
+  // Override webdriver detection
+  Object.defineProperty(navigator, 'webdriver', {
+    get: () => false,
+  });
+  
+  // Override plugins
+  Object.defineProperty(navigator, 'plugins', {
+    get: () => [1, 2, 3, 4, 5],
+  });
+});
+
+// Random delays
+async function humanDelay() {
+  const delay = Math.random() * 2000 + 500;
+  await page.waitForTimeout(delay);
+}
+
+// Human-like typing
+await page.type('input', 'Hello World', { delay: 100 });
+```
+
+---
+
+## 8. Error Handling
+
+```typescript
+async function resilientScrape(url: string, maxRetries = 3) {
+  for (let attempt = 1; attempt <= maxRetries; attempt++) {
+    const browser = await chromium.launch();
+    
+    try {
+      const page = await browser.newPage();
+      
+      // Set timeout
+      page.setDefaultTimeout(30000);
+      
+      await page.goto(url, { waitUntil: 'domcontentloaded' });
+      
+      const data = await extractData(page);
+      return data;
+      
+    } catch (error) {
+      console.error(`Attempt ${attempt} failed:`, error.message);
+      
+      if (attempt === maxRetries) {
+        throw error;
+      }
+      
+      // Exponential backoff
+      await new Promise(r => setTimeout(r, Math.pow(2, attempt) * 1000));
+      
+    } finally {
+      await browser.close();
+    }
+  }
+}
+```
+
+---
+
+## 9. Anti-Patterns
+
+### ❌ No Error Handling
+
+```typescript
+// WRONG
+const data = await page.$eval('.content', el => el.textContent);
+
+// CORRECT
+const element = await page.$('.content');
+const data = element ? await element.textContent() : null;
+```
+
+### ❌ Hard-Coded Waits
+
+```typescript
+// WRONG
+await page.waitForTimeout(5000);
+
+// CORRECT
+await page.waitForSelector('.content');
+await page.waitForLoadState('networkidle');
+```
+
+---
+
+## Related Skills
+
+- `webapp-testing` - E2E testing
+- `nodejs-best-practices` - Node.js patterns
+- `api-patterns` - API scraping alternative