@hasna/skills 0.0.2

package/skills/skill-apidocs/src/lib/crawler.ts

@@ -0,0 +1,427 @@
+/**
+ * Web crawler for documentation sites
+ *
+ * Two-phase approach:
+ * 1. Discovery: Aggressively crawl ALL pages within the domain
+ * 2. Extraction: Extract and convert content from each page
+ */
+
+import Anthropic from '@anthropic-ai/sdk';
+import type { CrawledPage, CrawlConfig, CrawlEvent, CrawlResult } from '../types/index.js';
+import { htmlToMarkdown } from './html-to-md.js';
+
+/**
+ * Normalize URL for deduplication
+ */
+function normalizeUrl(url: string): string {
+  try {
+    const parsed = new URL(url);
+    // Remove trailing slash, hash, and common tracking params
+    let normalized = `${parsed.protocol}//${parsed.host}${parsed.pathname}`.replace(/\/$/, '');
+    // Keep meaningful query params (like ?tab=, ?section=) but remove tracking
+    const dominated = ['utm_', 'ref', 'source', 'fbclid', 'gclid'];
+    const params = new URLSearchParams(parsed.search);
+    const cleanParams = new URLSearchParams();
+    for (const [key, value] of params) {
+      if (!dominated.some((d) => key.startsWith(d))) {
+        cleanParams.set(key, value);
+      }
+    }
+    const search = cleanParams.toString();
+    if (search) {
+      normalized += `?${search}`;
+    }
+    return normalized.toLowerCase();
+  } catch {
+    return url.toLowerCase();
+  }
+}
+
+/**
+ * Check if URL should be skipped entirely (non-content URLs)
+ */
+function shouldSkipUrl(url: string): boolean {
+  const skipPatterns = [
+    // Static assets
+    /\.(png|jpg|jpeg|gif|svg|ico|webp|pdf|zip|tar|gz|mp4|mp3|wav|woff|woff2|ttf|eot)$/i,
+    /\.(css|js|mjs|jsx|ts|tsx)$/i,
+    /\.(json|xml|yaml|yml)$/i,
+    /\/assets\//i,
+    /\/_next\//i,
+    /\/static\//i,
+    /\/chunks\//i,
+    // Protocol handlers
+    /^mailto:/i,
+    /^tel:/i,
+    /^javascript:/i,
+    /^#/,
+    // Feeds
+    /\/feed\/?$/i,
+    /\/rss\/?$/i,
+    /\/atom\/?$/i,
+    // Auth/account
+    /\/(login|logout|signin|signout|signup|register|auth)\/?/i,
+    /\/(account|profile|settings|dashboard)\/?$/i,
+    // E-commerce
+    /\/(cart|checkout|payment|order)\/?/i,
+    // Search
+    /\/search\?/i,
+    /\/search\/?$/i,
+  ];
+
+  return skipPatterns.some((pattern) => pattern.test(url));
+}
+
+/**
+ * Check if URL is likely a documentation page (loose check)
+ */
+function isLikelyDocPage(url: string): boolean {
+  // Skip obvious non-doc pages
+  const nonDocPatterns = [
+    /\/blog\//i,
+    /\/news\//i,
+    /\/press\//i,
+    /\/careers\//i,
+    /\/jobs\//i,
+    /\/about-us\/?$/i,
+    /\/contact-us\/?$/i,
+    /\/terms\/?$/i,
+    /\/privacy\/?$/i,
+    /\/legal\/?$/i,
+    /\/cookie/i,
+    /\/pricing\/?$/i,
+    /\/enterprise\/?$/i,
+    /\/customers\/?$/i,
+    /\/case-studies\//i,
+    /\/testimonials\//i,
+  ];
+
+  if (nonDocPatterns.some((p) => p.test(url))) {
+    return false;
+  }
+
+  return true;
+}
+
+/**
+ * Extract all links from HTML
+ */
+function extractLinks(html: string, baseUrl: string, baseDomain: string): string[] {
+  const links: string[] = [];
+  const seen = new Set<string>();
+
+  // Match href attributes
+  const hrefPattern = /href=["']([^"']+)["']/gi;
+  let match;
+
+  while ((match = hrefPattern.exec(html)) !== null) {
+    let href = match[1];
+
+    // Skip empty, anchors, javascript
+    if (!href || href.startsWith('#') || href.startsWith('javascript:') || href.startsWith('mailto:')) {
+      continue;
+    }
+
+    // Resolve relative URLs
+    try {
+      const resolved = new URL(href, baseUrl);
+
+      // Only follow same domain
+      if (!resolved.hostname.endsWith(baseDomain) && resolved.hostname !== baseDomain) {
+        continue;
+      }
+
+      const normalized = normalizeUrl(resolved.href);
+
+      // Skip if already seen or should be skipped
+      if (seen.has(normalized) || shouldSkipUrl(normalized)) {
+        continue;
+      }
+
+      seen.add(normalized);
+      links.push(resolved.href);
+    } catch {
+      // Invalid URL, skip
+    }
+  }
+
+  return links;
+}
+
+/**
+ * Fetch a URL and return HTML content
+ */
+async function fetchPage(url: string): Promise<{ html: string; finalUrl: string; title: string }> {
+  const response = await fetch(url, {
+    headers: {
+      'User-Agent':
+        'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+      Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+      'Accept-Language': 'en-US,en;q=0.9',
+    },
+    redirect: 'follow',
+  });
+
+  if (!response.ok) {
+    throw new Error(`HTTP ${response.status}: ${response.statusText}`);
+  }
+
+  const contentType = response.headers.get('content-type') || '';
+  if (!contentType.includes('text/html') && !contentType.includes('application/xhtml')) {
+    throw new Error(`Not HTML: ${contentType}`);
+  }
+
+  const html = await response.text();
+  const finalUrl = response.url;
+
+  // Extract title
+  const titleMatch = html.match(/<title[^>]*>([^<]+)<\/title>/i);
+  const title = titleMatch ? titleMatch[1].trim() : '';
+
+  return { html, finalUrl, title };
+}
+
+/**
+ * Extract main content from HTML
+ */
+function extractContent(html: string): string {
+  // Try to find main content area
+  const contentSelectors = [
+    /<main[^>]*>([\s\S]*?)<\/main>/i,
+    /<article[^>]*>([\s\S]*?)<\/article>/i,
+    /<div[^>]*class="[^"]*(?:content|docs|documentation|markdown|prose)[^"]*"[^>]*>([\s\S]*?)<\/div>/i,
+    /<div[^>]*id="(?:content|main|docs)"[^>]*>([\s\S]*?)<\/div>/i,
+    /<div[^>]*role="main"[^>]*>([\s\S]*?)<\/div>/i,
+  ];
+
+  for (const pattern of contentSelectors) {
+    const match = html.match(pattern);
+    if (match && match[1] && match[1].length > 500) {
+      return match[1];
+    }
+  }
+
+  // Fall back to body content
+  const bodyMatch = html.match(/<body[^>]*>([\s\S]*)<\/body>/i);
+  if (bodyMatch) {
+    // Remove obvious non-content elements
+    let content = bodyMatch[1];
+    content = content.replace(/<nav[^>]*>[\s\S]*?<\/nav>/gi, '');
+    content = content.replace(/<header[^>]*>[\s\S]*?<\/header>/gi, '');
+    content = content.replace(/<footer[^>]*>[\s\S]*?<\/footer>/gi, '');
+    content = content.replace(/<aside[^>]*>[\s\S]*?<\/aside>/gi, '');
+    content = content.replace(/<script[^>]*>[\s\S]*?<\/script>/gi, '');
+    content = content.replace(/<style[^>]*>[\s\S]*?<\/style>/gi, '');
+    content = content.replace(/<!--[\s\S]*?-->/g, '');
+    return content;
+  }
+
+  return html;
+}
+
+/**
+ * Check if content looks like documentation (has code, API references, etc.)
+ */
+function hasDocumentationContent(html: string): boolean {
+  const docIndicators = [
+    /<pre[^>]*>/i,
+    /<code[^>]*>/i,
+    /```/,
+    /class="[^"]*(?:highlight|syntax|code)[^"]*"/i,
+    /api/i,
+    /endpoint/i,
+    /parameter/i,
+    /request/i,
+    /response/i,
+    /example/i,
+    /usage/i,
+    /installation/i,
+    /getting.?started/i,
+    /quickstart/i,
+    /tutorial/i,
+    /guide/i,
+    /reference/i,
+    /documentation/i,
+  ];
+
+  const matchCount = docIndicators.filter((p) => p.test(html)).length;
+  return matchCount >= 2; // At least 2 indicators
+}
+
+/**
+ * Main crawler function
+ */
+export async function crawlDocumentation(
+  config: CrawlConfig,
+  onEvent: (event: CrawlEvent) => void = () => {}
+): Promise<CrawlResult> {
+  const startTime = Date.now();
+
+  // Parse base domain
+  const startUrlParsed = new URL(config.startUrl);
+  const baseDomain = startUrlParsed.hostname.replace(/^www\./, '');
+
+  // State
+  const visited = new Set<string>();
+  const toVisit: string[] = [config.startUrl];
+  const pages: CrawledPage[] = [];
+  const errors: string[] = [];
+  const contentHashes = new Set<string>(); // For deduplication
+
+  onEvent({ type: 'navigating', url: config.startUrl });
+
+  // Phase 1 & 2 combined: Crawl and extract in one pass
+  while (toVisit.length > 0 && pages.length < config.maxPages) {
+    const url = toVisit.shift()!;
+    const normalizedUrl = normalizeUrl(url);
+
+    // Skip if already visited
+    if (visited.has(normalizedUrl)) {
+      continue;
+    }
+    visited.add(normalizedUrl);
+
+    // Skip non-doc URLs
+    if (!isLikelyDocPage(url)) {
+      onEvent({ type: 'skipped', url, reason: 'non-documentation URL pattern' });
+      continue;
+    }
+
+    try {
+      onEvent({ type: 'navigating', url });
+
+      const { html, finalUrl, title } = await fetchPage(url);
+
+      // Also mark final URL as visited (in case of redirects)
+      visited.add(normalizeUrl(finalUrl));
+
+      // Extract links for discovery (do this for ALL pages)
+      const links = extractLinks(html, finalUrl, baseDomain);
+      for (const link of links) {
+        const normalizedLink = normalizeUrl(link);
+        if (!visited.has(normalizedLink)) {
+          toVisit.push(link);
+        }
+      }
+
+      // Check if page has documentation content
+      if (!hasDocumentationContent(html)) {
+        onEvent({ type: 'skipped', url: finalUrl, reason: 'no documentation content detected' });
+        continue;
+      }
+
+      // Extract main content
+      const mainContent = extractContent(html);
+
+      // Convert to markdown
+      const markdown = htmlToMarkdown(mainContent);
+
+      // Skip if markdown is too short (likely not useful)
+      if (markdown.length < 100) {
+        onEvent({ type: 'skipped', url: finalUrl, reason: 'content too short' });
+        continue;
+      }
+
+      // Deduplicate by content hash (first 500 chars)
+      const contentHash = markdown.substring(0, 500);
+      if (contentHashes.has(contentHash)) {
+        onEvent({ type: 'skipped', url: finalUrl, reason: 'duplicate content' });
+        continue;
+      }
+      contentHashes.add(contentHash);
+
+      // Parse path from URL
+      const urlObj = new URL(finalUrl);
+      const path = urlObj.pathname;
+
+      // Save page
+      const page: CrawledPage = {
+        url: finalUrl,
+        path,
+        title: title || path,
+        content: markdown,
+        html: mainContent,
+        crawledAt: new Date().toISOString(),
+      };
+
+      pages.push(page);
+      onEvent({
+        type: 'extracted',
+        url: finalUrl,
+        title: title || path,
+        pageCount: pages.length,
+        totalPages: config.maxPages,
+      });
+
+      // Small delay to be polite
+      await new Promise((resolve) => setTimeout(resolve, 100));
+    } catch (error) {
+      const errorMsg = (error as Error).message;
+      errors.push(`${url}: ${errorMsg}`);
+      onEvent({ type: 'error', url, error: errorMsg });
+    }
+  }
+
+  onEvent({ type: 'complete', pageCount: pages.length });
+
+  return {
+    pages,
+    totalPages: pages.length,
+    duration: Date.now() - startTime,
+    errors,
+  };
+}
+
+/**
+ * Alias for backwards compatibility
+ */
+export const crawlSimple = crawlDocumentation;
+
+/**
+ * Use Claude to clean up and enhance extracted content
+ * Call this after crawling to improve content quality
+ */
+export async function enhanceContentWithAgent(
+  pages: CrawledPage[],
+  onEvent: (event: CrawlEvent) => void = () => {}
+): Promise<CrawledPage[]> {
+  const client = new Anthropic();
+  const enhanced: CrawledPage[] = [];
+
+  for (const page of pages) {
+    try {
+      const response = await client.messages.create({
+        model: 'claude-sonnet-4-20250514',
+        max_tokens: 8192,
+        messages: [
+          {
+            role: 'user',
+            content: `Clean up this documentation content. Remove navigation elements, ads, cookie notices, and other non-documentation content. Keep all code examples, API references, and explanatory text. Output clean markdown only, no explanation.
+
+Title: ${page.title}
+URL: ${page.url}
+
+Content:
+${page.content}`,
+          },
+        ],
+      });
+
+      const textBlock = response.content.find((b) => b.type === 'text');
+      if (textBlock && textBlock.type === 'text') {
+        enhanced.push({
+          ...page,
+          content: textBlock.text,
+        });
+      } else {
+        enhanced.push(page);
+      }
+    } catch (error) {
+      // Keep original on error
+      enhanced.push(page);
+      onEvent({ type: 'error', url: page.url, error: (error as Error).message });
+    }
+  }
+
+  return enhanced;
+}

package/skills/skill-apidocs/src/lib/discovery.ts

@@ -0,0 +1,244 @@
+/**
+ * Agentic docs URL discovery using Claude and Playwright MCP
+ *
+ * When a user provides a base URL (e.g., stripe.com), this agent
+ * navigates the site to find the API documentation section.
+ */
+
+import Anthropic from '@anthropic-ai/sdk';
+import type { DiscoveryEvent, DiscoveryResult } from '../types/index.js';
+
+const DISCOVERY_SYSTEM_PROMPT = `You are a documentation discovery agent. Your task is to find the API documentation section of a website.
+
+STRATEGY:
+1. First, analyze the current page for links to documentation
+2. Look for links containing: docs, documentation, api, reference, developer, api-reference
+3. Prioritize links that seem to be API references over general documentation
+4. If you find an API documentation landing page, report it
+
+IMPORTANT:
+- Use browser_snapshot to see the current page content
+- Use browser_click to click on promising links
+- Look for navigation menus, footer links, and header links
+- Common patterns: /docs, /api, /api-reference, /developers, /documentation
+
+When you find the API documentation, respond with:
+FOUND: <url>
+
+If you cannot find API documentation after reasonable exploration, respond with:
+NOT_FOUND: <reason>`;
+
+/**
+ * Check if a URL looks like it's already a documentation URL
+ */
+export function isLikelyDocsUrl(url: string): boolean {
+  const docsPatterns = [
+    /\/docs\/?/i,
+    /\/documentation\/?/i,
+    /\/api-reference\/?/i,
+    /\/api\/?$/i,
+    /\/api\/v\d/i,
+    /\/reference\/?/i,
+    /\/developers?\/?/i,
+    /docs\./i,
+    /developers?\./i,
+    /api\./i,
+  ];
+
+  return docsPatterns.some((pattern) => pattern.test(url));
+}
+
+/**
+ * Discover the documentation URL from a base website URL
+ *
+ * This agent uses Claude with Playwright MCP to navigate a website
+ * and find the API documentation section.
+ */
+export async function discoverDocsUrl(
+  baseUrl: string,
+  onEvent: (event: DiscoveryEvent) => void = () => {}
+): Promise<DiscoveryResult> {
+  // If URL already looks like docs, return it directly
+  if (isLikelyDocsUrl(baseUrl)) {
+    onEvent({ type: 'found', docsUrl: baseUrl, message: 'URL already appears to be documentation' });
+    return { success: true, docsUrl: baseUrl };
+  }
+
+  const client = new Anthropic();
+
+  onEvent({ type: 'navigating', url: baseUrl, message: 'Starting documentation discovery' });
+
+  try {
+    // First, let's try common documentation URL patterns
+    const commonDocsPatterns = ['/docs', '/api', '/api-reference', '/documentation', '/developers', '/reference'];
+
+    const parsedUrl = new URL(baseUrl);
+    const baseOrigin = parsedUrl.origin;
+
+    // Try common patterns first (quick check)
+    for (const pattern of commonDocsPatterns) {
+      const testUrl = `${baseOrigin}${pattern}`;
+      try {
+        const response = await fetch(testUrl, {
+          method: 'HEAD',
+          redirect: 'follow',
+          headers: {
+            'User-Agent':
+              'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+          },
+        });
+        if (response.ok) {
+          onEvent({ type: 'found', docsUrl: testUrl, message: `Found docs at common pattern: ${pattern}` });
+          return { success: true, docsUrl: testUrl };
+        }
+      } catch {
+        // Pattern doesn't exist, continue
+      }
+    }
+
+    // Also check for subdomains
+    const domain = parsedUrl.hostname.replace(/^www\./, '');
+    const commonSubdomains = ['docs', 'api', 'developers', 'developer'];
+
+    for (const subdomain of commonSubdomains) {
+      const testUrl = `${parsedUrl.protocol}//${subdomain}.${domain}`;
+      try {
+        const response = await fetch(testUrl, {
+          method: 'HEAD',
+          redirect: 'follow',
+          headers: {
+            'User-Agent':
+              'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+          },
+        });
+        if (response.ok) {
+          onEvent({ type: 'found', docsUrl: testUrl, message: `Found docs at subdomain: ${subdomain}` });
+          return { success: true, docsUrl: testUrl };
+        }
+      } catch {
+        // Subdomain doesn't exist, continue
+      }
+    }
+
+    onEvent({ type: 'analyzing', url: baseUrl, message: 'Checking page content for documentation links' });
+
+    // If common patterns fail, fetch the page and analyze links
+    const pageResponse = await fetch(baseUrl, {
+      headers: {
+        'User-Agent':
+          'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+        Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+      },
+    });
+
+    if (!pageResponse.ok) {
+      return { success: false, error: `Failed to fetch base URL: ${pageResponse.status}` };
+    }
+
+    const html = await pageResponse.text();
+
+    // Use Claude to analyze the page and find docs links
+    const response = await client.messages.create({
+      model: 'claude-sonnet-4-20250514',
+      max_tokens: 1024,
+      messages: [
+        {
+          role: 'user',
+          content: `Analyze this HTML page and find the URL to API documentation.
+
+Look for:
+1. Links with text containing: "API", "Docs", "Documentation", "Reference", "Developers"
+2. Navigation links pointing to documentation sections
+3. Footer links to developer resources
+
+URL: ${baseUrl}
+
+HTML (truncated to key sections):
+${extractRelevantHtml(html)}
+
+Respond in this exact format:
+If found: DOCS_URL: <the full URL>
+If not found: NOT_FOUND: <brief reason>`,
+        },
+      ],
+    });
+
+    const textBlock = response.content.find((b) => b.type === 'text');
+    if (textBlock && textBlock.type === 'text') {
+      const text = textBlock.text;
+
+      if (text.includes('DOCS_URL:')) {
+        const match = text.match(/DOCS_URL:\s*(\S+)/);
+        if (match) {
+          let docsUrl = match[1].trim();
+
+          // Resolve relative URLs
+          if (docsUrl.startsWith('/')) {
+            docsUrl = `${baseOrigin}${docsUrl}`;
+          } else if (!docsUrl.startsWith('http')) {
+            docsUrl = `${baseOrigin}/${docsUrl}`;
+          }
+
+          onEvent({ type: 'found', docsUrl, message: 'Found documentation URL via page analysis' });
+          return { success: true, docsUrl };
+        }
+      }
+
+      if (text.includes('NOT_FOUND:')) {
+        const reason = text.split('NOT_FOUND:')[1]?.trim() || 'Could not find documentation links';
+        onEvent({ type: 'not_found', message: reason });
+        return { success: false, error: reason };
+      }
+    }
+
+    // Fallback: just use the base URL
+    onEvent({ type: 'not_found', message: 'Could not find specific documentation URL, using base URL' });
+    return { success: true, docsUrl: baseUrl };
+  } catch (error) {
+    const errorMsg = (error as Error).message;
+    onEvent({ type: 'error', error: errorMsg });
+    return { success: false, error: errorMsg };
+  }
+}
+
+/**
+ * Extract relevant HTML sections for analysis (nav, header, footer, main links)
+ */
+function extractRelevantHtml(html: string): string {
+  const sections: string[] = [];
+
+  // Extract nav content
+  const navMatch = html.match(/<nav[^>]*>([\s\S]*?)<\/nav>/gi);
+  if (navMatch) {
+    sections.push('=== Navigation ===');
+    sections.push(navMatch.slice(0, 3).join('\n').substring(0, 3000));
+  }
+
+  // Extract header content
+  const headerMatch = html.match(/<header[^>]*>([\s\S]*?)<\/header>/gi);
+  if (headerMatch) {
+    sections.push('=== Header ===');
+    sections.push(headerMatch.slice(0, 2).join('\n').substring(0, 2000));
+  }
+
+  // Extract footer content
+  const footerMatch = html.match(/<footer[^>]*>([\s\S]*?)<\/footer>/gi);
+  if (footerMatch) {
+    sections.push('=== Footer ===');
+    sections.push(footerMatch.slice(0, 2).join('\n').substring(0, 2000));
+  }
+
+  // Extract links with relevant text
+  const linkPattern = /<a[^>]*href=["']([^"']+)["'][^>]*>([^<]*(?:doc|api|reference|developer)[^<]*)<\/a>/gi;
+  const links: string[] = [];
+  let match;
+  while ((match = linkPattern.exec(html)) !== null) {
+    links.push(`<a href="${match[1]}">${match[2]}</a>`);
+  }
+  if (links.length > 0) {
+    sections.push('=== Relevant Links ===');
+    sections.push(links.slice(0, 20).join('\n'));
+  }
+
+  return sections.join('\n\n').substring(0, 10000);
+}