@hasna/skills 0.0.2

package/skills/skill-apidocs/src/lib/embeddings.ts

@@ -0,0 +1,136 @@
+import OpenAI from 'openai';
+
+const EMBEDDING_MODEL = 'text-embedding-3-small';
+export const EMBEDDING_DIMENSIONS = 1536;
+
+let openai: OpenAI | null = null;
+
+/**
+ * Get or create OpenAI client
+ */
+function getOpenAI(): OpenAI {
+  if (!openai) {
+    const apiKey = process.env.OPENAI_API_KEY;
+    if (!apiKey) {
+      throw new Error('OPENAI_API_KEY environment variable is required');
+    }
+    openai = new OpenAI({ apiKey });
+  }
+  return openai;
+}
+
+/**
+ * Generate embedding for a single text
+ */
+export async function generateEmbedding(text: string): Promise<number[]> {
+  const client = getOpenAI();
+
+  const response = await client.embeddings.create({
+    model: EMBEDDING_MODEL,
+    input: text,
+  });
+
+  return response.data[0].embedding;
+}
+
+/**
+ * Estimate token count for a text (rough approximation: ~4 chars per token)
+ */
+function estimateTokens(text: string): number {
+  return Math.ceil(text.length / 4);
+}
+
+/**
+ * Progress callback for embedding generation
+ */
+export type EmbeddingProgressCallback = (processed: number, total: number) => void;
+
+/**
+ * Generate embeddings for multiple texts in batch
+ * Respects both input count limit (2048) and token limit (300k)
+ */
+export async function generateEmbeddings(
+  texts: string[],
+  onProgress?: EmbeddingProgressCallback
+): Promise<number[][]> {
+  const client = getOpenAI();
+
+  // OpenAI limits
+  const maxInputsPerRequest = 2048;
+  const maxTokensPerRequest = 250000; // Use 250k to be safe (limit is 300k)
+
+  const embeddings: number[][] = [];
+  let currentBatch: string[] = [];
+  let currentTokenCount = 0;
+  let processedCount = 0;
+
+  for (const text of texts) {
+    const textTokens = estimateTokens(text);
+
+    // Check if adding this text would exceed limits
+    const wouldExceedInputs = currentBatch.length >= maxInputsPerRequest;
+    const wouldExceedTokens = currentTokenCount + textTokens > maxTokensPerRequest;
+
+    if (wouldExceedInputs || wouldExceedTokens) {
+      // Process current batch
+      if (currentBatch.length > 0) {
+        const batchEmbeddings = await processBatch(client, currentBatch);
+        embeddings.push(...batchEmbeddings);
+        processedCount += currentBatch.length;
+        onProgress?.(processedCount, texts.length);
+      }
+
+      // Reset batch
+      currentBatch = [];
+      currentTokenCount = 0;
+    }
+
+    currentBatch.push(text);
+    currentTokenCount += textTokens;
+  }
+
+  // Process remaining batch
+  if (currentBatch.length > 0) {
+    const batchEmbeddings = await processBatch(client, currentBatch);
+    embeddings.push(...batchEmbeddings);
+    processedCount += currentBatch.length;
+    onProgress?.(processedCount, texts.length);
+  }
+
+  return embeddings;
+}
+
+/**
+ * Process a single batch of texts
+ */
+async function processBatch(client: OpenAI, batch: string[]): Promise<number[][]> {
+  const response = await client.embeddings.create({
+    model: EMBEDDING_MODEL,
+    input: batch,
+  });
+
+  // Sort by index to ensure correct order
+  const sorted = response.data.sort((a, b) => a.index - b.index);
+  return sorted.map((d) => d.embedding);
+}
+
+/**
+ * Calculate cosine similarity between two vectors
+ */
+export function cosineSimilarity(a: number[], b: number[]): number {
+  if (a.length !== b.length) {
+    throw new Error('Vectors must have the same length');
+  }
+
+  let dotProduct = 0;
+  let normA = 0;
+  let normB = 0;
+
+  for (let i = 0; i < a.length; i++) {
+    dotProduct += a[i] * b[i];
+    normA += a[i] * a[i];
+    normB += b[i] * b[i];
+  }
+
+  return dotProduct / (Math.sqrt(normA) * Math.sqrt(normB));
+}

package/skills/skill-apidocs/src/lib/endpoint-extractor.ts

@@ -0,0 +1,359 @@
+/**
+ * API endpoint extraction using Claude
+ *
+ * Extracts structured API endpoint information from crawled documentation pages.
+ */
+
+import Anthropic from '@anthropic-ai/sdk';
+import { createHash } from 'crypto';
+import type {
+  CrawledPage,
+  APIEndpoint,
+  ExtractionEvent,
+  ExtractionResult,
+  HTTPMethod,
+} from '../types/index.js';
+
+const EXTRACTION_SYSTEM_PROMPT = `You are an API endpoint extraction agent. Analyze documentation content and extract structured API endpoint information.
+
+For each endpoint found, extract:
+- method: HTTP method (GET, POST, PUT, DELETE, PATCH, HEAD, OPTIONS)
+- path: The endpoint path (e.g., /users, /posts/{id}, /v1/chat/completions)
+- title: A short title/name for the endpoint
+- description: What the endpoint does
+- parameters: Path, query, header parameters with name, type, required, description
+- requestBody: Content type and example if available
+- responses: Status codes and descriptions
+- codeExamples: Any code examples (curl, python, javascript, etc.)
+
+Output a JSON array of endpoints. Only include endpoints that are clearly documented with at least method and path.
+
+If no API endpoints are found on a page, output an empty array: []`;
+
+const EXTRACTION_BATCH_SIZE = 5; // Process pages in batches
+
+/**
+ * Generate a unique ID for an endpoint based on method and path
+ */
+function generateEndpointId(method: string, path: string): string {
+  const hash = createHash('md5').update(`${method}:${path}`).digest('hex');
+  return hash.substring(0, 12);
+}
+
+/**
+ * Extract resource name from path
+ * /users/{id} -> users
+ * /v1/chat/completions -> chat
+ * /api/posts/{postId}/comments -> posts
+ */
+function extractResource(path: string): string {
+  // Remove version prefixes
+  let cleanPath = path.replace(/^\/?(v\d+|api)\/?/i, '/');
+
+  // Get first meaningful segment
+  const segments = cleanPath.split('/').filter(Boolean);
+
+  for (const segment of segments) {
+    // Skip path parameters
+    if (segment.startsWith('{') || segment.startsWith(':')) {
+      continue;
+    }
+    // Return first non-parameter segment
+    return segment.toLowerCase();
+  }
+
+  return 'general';
+}
+
+/**
+ * Validate and normalize extracted endpoints
+ */
+function validateEndpoint(raw: unknown, sourceUrl: string, sourceTitle: string): APIEndpoint | null {
+  if (!raw || typeof raw !== 'object') return null;
+
+  const obj = raw as Record<string, unknown>;
+
+  // Must have method and path
+  if (!obj.method || !obj.path) return null;
+
+  const method = String(obj.method).toUpperCase();
+  const validMethods: HTTPMethod[] = ['GET', 'POST', 'PUT', 'DELETE', 'PATCH', 'HEAD', 'OPTIONS'];
+  if (!validMethods.includes(method as HTTPMethod)) return null;
+
+  const path = String(obj.path);
+  if (!path.startsWith('/')) return null;
+
+  return {
+    id: generateEndpointId(method, path),
+    method: method as HTTPMethod,
+    path,
+    title: String(obj.title || `${method} ${path}`),
+    description: String(obj.description || ''),
+    resource: extractResource(path),
+    parameters: Array.isArray(obj.parameters)
+      ? obj.parameters.map((p: unknown) => {
+          const param = p as Record<string, unknown>;
+          return {
+            name: String(param.name || ''),
+            in: (['path', 'query', 'header', 'cookie'].includes(String(param.in))
+              ? String(param.in)
+              : 'query') as 'path' | 'query' | 'header' | 'cookie',
+            type: String(param.type || 'string'),
+            required: Boolean(param.required),
+            description: param.description ? String(param.description) : undefined,
+            default: param.default,
+          };
+        })
+      : undefined,
+    requestBody: obj.requestBody
+      ? {
+          contentType: String((obj.requestBody as Record<string, unknown>).contentType || 'application/json'),
+          schema: (obj.requestBody as Record<string, unknown>).schema as Record<string, unknown> | undefined,
+          example: (obj.requestBody as Record<string, unknown>).example,
+        }
+      : undefined,
+    responses: Array.isArray(obj.responses)
+      ? obj.responses.map((r: unknown) => {
+          const resp = r as Record<string, unknown>;
+          return {
+            status: Number(resp.status) || 200,
+            description: String(resp.description || ''),
+            example: resp.example,
+          };
+        })
+      : undefined,
+    codeExamples: Array.isArray(obj.codeExamples)
+      ? obj.codeExamples.map((e: unknown) => {
+          const ex = e as Record<string, unknown>;
+          return {
+            language: String(ex.language || 'text'),
+            code: String(ex.code || ''),
+            title: ex.title ? String(ex.title) : undefined,
+          };
+        })
+      : undefined,
+    sourceUrl,
+    sourcePageTitle: sourceTitle,
+  };
+}
+
+/**
+ * Extract endpoints from a single page using Claude
+ */
+async function extractEndpointsFromPage(
+  client: Anthropic,
+  page: CrawledPage
+): Promise<APIEndpoint[]> {
+  // Skip pages that don't look like API documentation
+  const apiIndicators = [
+    /\bGET\b/,
+    /\bPOST\b/,
+    /\bPUT\b/,
+    /\bDELETE\b/,
+    /\bPATCH\b/,
+    /endpoint/i,
+    /api/i,
+    /request/i,
+    /response/i,
+    /curl/i,
+    /http/i,
+  ];
+
+  const hasApiContent = apiIndicators.filter((p) => p.test(page.content)).length >= 2;
+  if (!hasApiContent) {
+    return [];
+  }
+
+  try {
+    const response = await client.messages.create({
+      model: 'claude-sonnet-4-20250514',
+      max_tokens: 8192,
+      system: EXTRACTION_SYSTEM_PROMPT,
+      messages: [
+        {
+          role: 'user',
+          content: `Extract API endpoints from this documentation page.
+
+Title: ${page.title}
+URL: ${page.url}
+
+Content:
+${page.content.substring(0, 30000)}`,
+        },
+      ],
+    });
+
+    const textBlock = response.content.find((b) => b.type === 'text');
+    if (!textBlock || textBlock.type !== 'text') {
+      return [];
+    }
+
+    // Parse JSON from response
+    const text = textBlock.text;
+    const jsonMatch = text.match(/\[[\s\S]*\]/);
+    if (!jsonMatch) {
+      return [];
+    }
+
+    try {
+      const parsed = JSON.parse(jsonMatch[0]);
+      if (!Array.isArray(parsed)) {
+        return [];
+      }
+
+      const endpoints: APIEndpoint[] = [];
+      for (const item of parsed) {
+        const validated = validateEndpoint(item, page.url, page.title);
+        if (validated) {
+          endpoints.push(validated);
+        }
+      }
+
+      return endpoints;
+    } catch {
+      // JSON parse error
+      return [];
+    }
+  } catch (error) {
+    console.error(`Error extracting from ${page.url}:`, (error as Error).message);
+    return [];
+  }
+}
+
+/**
+ * Extract API endpoints from crawled documentation pages
+ *
+ * Processes pages in batches and uses Claude to extract structured endpoint data.
+ */
+export async function extractEndpoints(
+  pages: CrawledPage[],
+  onEvent: (event: ExtractionEvent) => void = () => {}
+): Promise<ExtractionResult> {
+  const client = new Anthropic();
+  const allEndpoints: APIEndpoint[] = [];
+  const errors: string[] = [];
+  const seenEndpoints = new Set<string>(); // For deduplication by id
+
+  onEvent({ type: 'processing', pageIndex: 0, totalPages: pages.length });
+
+  // Process pages in batches
+  for (let i = 0; i < pages.length; i += EXTRACTION_BATCH_SIZE) {
+    const batch = pages.slice(i, i + EXTRACTION_BATCH_SIZE);
+
+    // Process batch in parallel
+    const batchPromises = batch.map(async (page, batchIndex) => {
+      const pageIndex = i + batchIndex;
+      onEvent({
+        type: 'processing',
+        pageIndex,
+        totalPages: pages.length,
+        pageUrl: page.url,
+      });
+
+      try {
+        const endpoints = await extractEndpointsFromPage(client, page);
+        return { endpoints, error: null };
+      } catch (error) {
+        return { endpoints: [], error: `${page.url}: ${(error as Error).message}` };
+      }
+    });
+
+    const results = await Promise.all(batchPromises);
+
+    for (const result of results) {
+      if (result.error) {
+        errors.push(result.error);
+      }
+
+      for (const endpoint of result.endpoints) {
+        // Deduplicate by endpoint ID (method:path hash)
+        if (!seenEndpoints.has(endpoint.id)) {
+          seenEndpoints.add(endpoint.id);
+          allEndpoints.push(endpoint);
+        }
+      }
+    }
+
+    onEvent({
+      type: 'extracted',
+      pageIndex: Math.min(i + EXTRACTION_BATCH_SIZE, pages.length),
+      totalPages: pages.length,
+      endpointCount: allEndpoints.length,
+    });
+
+    // Small delay between batches
+    if (i + EXTRACTION_BATCH_SIZE < pages.length) {
+      await new Promise((resolve) => setTimeout(resolve, 500));
+    }
+  }
+
+  // Sort endpoints by resource then path
+  allEndpoints.sort((a, b) => {
+    if (a.resource !== b.resource) {
+      return a.resource.localeCompare(b.resource);
+    }
+    return a.path.localeCompare(b.path);
+  });
+
+  onEvent({
+    type: 'complete',
+    endpointCount: allEndpoints.length,
+  });
+
+  return {
+    endpoints: allEndpoints,
+    processedPages: pages.length,
+    errors,
+  };
+}
+
+/**
+ * Get unique resources from endpoints
+ */
+export function getUniqueResources(endpoints: APIEndpoint[]): string[] {
+  const resources = new Set(endpoints.map((e) => e.resource));
+  return Array.from(resources).sort();
+}
+
+/**
+ * Group endpoints by resource
+ */
+export function groupByResource(endpoints: APIEndpoint[]): Map<string, APIEndpoint[]> {
+  const grouped = new Map<string, APIEndpoint[]>();
+
+  for (const endpoint of endpoints) {
+    const existing = grouped.get(endpoint.resource) || [];
+    existing.push(endpoint);
+    grouped.set(endpoint.resource, existing);
+  }
+
+  return grouped;
+}
+
+/**
+ * Group endpoints by HTTP method
+ */
+export function groupByMethod(endpoints: APIEndpoint[]): Map<string, APIEndpoint[]> {
+  const grouped = new Map<string, APIEndpoint[]>();
+  const methodOrder = ['GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'HEAD', 'OPTIONS'];
+
+  // Initialize in order
+  for (const method of methodOrder) {
+    grouped.set(method, []);
+  }
+
+  for (const endpoint of endpoints) {
+    const existing = grouped.get(endpoint.method) || [];
+    existing.push(endpoint);
+    grouped.set(endpoint.method, existing);
+  }
+
+  // Remove empty methods
+  for (const [method, list] of grouped) {
+    if (list.length === 0) {
+      grouped.delete(method);
+    }
+  }
+
+  return grouped;
+}

package/skills/skill-apidocs/src/lib/html-to-md.ts

@@ -0,0 +1,163 @@
+import TurndownService from 'turndown';
+
+/**
+ * HTML to Markdown converter using Turndown
+ */
+
+const turndownService = new TurndownService({
+  headingStyle: 'atx',
+  codeBlockStyle: 'fenced',
+  bulletListMarker: '-',
+  emDelimiter: '*',
+});
+
+// Preserve code blocks with language
+turndownService.addRule('fencedCodeBlock', {
+  filter: (node) => {
+    return (
+      node.nodeName === 'PRE' &&
+      node.firstChild !== null &&
+      node.firstChild.nodeName === 'CODE'
+    );
+  },
+  replacement: (_content, node) => {
+    const codeNode = node.firstChild;
+    const code = codeNode?.textContent || '';
+
+    // Try to extract language from class using getAttribute if available
+    const getAttr = (codeNode as { getAttribute?: (name: string) => string | null })?.getAttribute;
+    const className = getAttr ? getAttr.call(codeNode, 'class') || '' : '';
+    const langMatch = className.match(/language-(\w+)/);
+    const language = langMatch ? langMatch[1] : '';
+
+    return `\n\`\`\`${language}\n${code}\n\`\`\`\n`;
+  },
+});
+
+// Handle inline code
+turndownService.addRule('inlineCode', {
+  filter: (node) => {
+    return node.nodeName === 'CODE' && node.parentNode?.nodeName !== 'PRE';
+  },
+  replacement: (content) => {
+    if (content.includes('`')) {
+      return `\`\`${content}\`\``;
+    }
+    return `\`${content}\``;
+  },
+});
+
+// Remove navigation elements
+turndownService.addRule('removeNav', {
+  filter: (node) => {
+    const tagName = node.nodeName.toLowerCase();
+    return ['nav', 'header', 'footer', 'aside'].includes(tagName);
+  },
+  replacement: () => '',
+});
+
+// Remove scripts and styles
+turndownService.addRule('removeScripts', {
+  filter: ['script', 'style', 'noscript'],
+  replacement: () => '',
+});
+
+/**
+ * Convert HTML to Markdown
+ */
+export function htmlToMarkdown(html: string): string {
+  // Pre-process: remove common non-content elements
+  let cleaned = html
+    // Remove HTML comments
+    .replace(/<!--[\s\S]*?-->/g, '')
+    // Remove common cookie/banner divs
+    .replace(/<div[^>]*class="[^"]*(?:cookie|banner|popup|modal|overlay)[^"]*"[^>]*>[\s\S]*?<\/div>/gi, '')
+    // Remove aria-hidden elements
+    .replace(/<[^>]+aria-hidden="true"[^>]*>[\s\S]*?<\/[^>]+>/gi, '');
+
+  const markdown = turndownService.turndown(cleaned);
+
+  // Post-process: clean up markdown
+  return markdown
+    // Remove excessive newlines
+    .replace(/\n{4,}/g, '\n\n\n')
+    // Remove trailing whitespace
+    .replace(/[ \t]+$/gm, '')
+    // Ensure file ends with single newline
+    .trim() + '\n';
+}
+
+/**
+ * Extract main content from HTML using common selectors
+ */
+export function extractMainContent(html: string): string {
+  // Common documentation content selectors
+  const contentSelectors = [
+    'main article',
+    'main .content',
+    '[role="main"]',
+    '.documentation',
+    '.docs-content',
+    '.markdown-body',
+    '.prose',
+    'article.content',
+    'article',
+    'main',
+    '#content',
+    '.content',
+  ];
+
+  // Try to match content area with simple regex patterns
+  for (const selector of contentSelectors) {
+    // Convert selector to basic regex pattern
+    const pattern = selectorToPattern(selector);
+    if (pattern) {
+      const match = html.match(pattern);
+      if (match) {
+        return match[0];
+      }
+    }
+  }
+
+  // Fall back to body content
+  const bodyMatch = html.match(/<body[^>]*>([\s\S]*)<\/body>/i);
+  return bodyMatch ? bodyMatch[1] : html;
+}
+
+/**
+ * Convert CSS selector to regex pattern (simplified)
+ */
+function selectorToPattern(selector: string): RegExp | null {
+  // Handle class selectors
+  if (selector.startsWith('.')) {
+    const className = selector.slice(1).replace('.', '\\s+');
+    return new RegExp(`<[^>]+class="[^"]*${className}[^"]*"[^>]*>[\\s\\S]*?<\\/[^>]+>`, 'i');
+  }
+
+  // Handle ID selectors
+  if (selector.startsWith('#')) {
+    const id = selector.slice(1);
+    return new RegExp(`<[^>]+id="${id}"[^>]*>[\\s\\S]*?<\\/[^>]+>`, 'i');
+  }
+
+  // Handle attribute selectors
+  if (selector.includes('[')) {
+    const attrMatch = selector.match(/\[([^=]+)="([^"]+)"\]/);
+    if (attrMatch) {
+      return new RegExp(`<[^>]+${attrMatch[1]}="${attrMatch[2]}"[^>]*>[\\s\\S]*?<\\/[^>]+>`, 'i');
+    }
+  }
+
+  // Handle tag selectors
+  const tagMatch = selector.match(/^(\w+)(?:\s+\.(\w+))?$/);
+  if (tagMatch) {
+    const tag = tagMatch[1];
+    const className = tagMatch[2];
+    if (className) {
+      return new RegExp(`<${tag}[^>]+class="[^"]*${className}[^"]*"[^>]*>[\\s\\S]*?<\\/${tag}>`, 'i');
+    }
+    return new RegExp(`<${tag}[^>]*>[\\s\\S]*?<\\/${tag}>`, 'i');
+  }
+
+  return null;
+}