@hasna/skills 0.1.60 → 0.1.61

package/skills/apidocs/src/commands/sync.ts

@@ -1,227 +0,0 @@
-import type { Chunk, VectorData, DocFile } from '../types/index.js';
-import * as output from '../utils/output.js';
-import {
-  findLibrary,
-  listLibraries,
-  initStorage,
-  saveLibraryMetadata,
-  cacheChunks,
-  saveCrawledPages,
-  saveEndpoints,
-} from '../lib/storage.js';
-import { crawlSimple } from '../lib/crawler.js';
-import { parseMarkdown } from '../lib/parser.js';
-import { generateEmbeddings } from '../lib/embeddings.js';
-import { deleteIndex, createIndex, upsertVectors, indexExists } from '../lib/vectors.js';
-import { createIndexNameFromUrl } from '../utils/paths.js';
-import { extractEndpoints } from '../lib/endpoint-extractor.js';
-
-interface SyncOptions {
-  all?: boolean;
-  maxPages?: number;
-}
-
-/**
- * Sync a library's documentation
- */
-export async function sync(library: string | undefined, options: SyncOptions): Promise<void> {
-  await initStorage();
-
-  if (options.all) {
-    // Sync all libraries
-    const libraries = await listLibraries();
-
-    if (libraries.length === 0) {
-      output.info('No libraries to sync');
-      return;
-    }
-
-    output.info(`Syncing ${libraries.length} libraries...`);
-    console.log();
-
-    for (const lib of libraries) {
-      try {
-        await syncLibrary(lib.id, options.maxPages);
-        console.log();
-      } catch (error) {
-        output.error(`Failed to sync ${lib.name}: ${(error as Error).message}`);
-        console.log();
-      }
-    }
-
-    output.success('Sync complete');
-    return;
-  }
-
-  if (!library) {
-    output.error('Please specify a library or use --all');
-    process.exit(1);
-  }
-
-  // Find the library
-  const metadata = await findLibrary(library);
-
-  if (!metadata) {
-    output.error(`Library not found: ${library}`);
-    output.info('Use `service-apidocs list` to see indexed libraries');
-    process.exit(1);
-  }
-
-  await syncLibrary(metadata.id, options.maxPages);
-}
-
-/**
- * Sync a single library by ID
- */
-async function syncLibrary(libraryId: string, maxPages?: number): Promise<void> {
-  const metadata = await findLibrary(libraryId);
-  if (!metadata) {
-    throw new Error(`Library not found: ${libraryId}`);
-  }
-
-  output.info(`Syncing: ${metadata.name}`);
-  output.info(`URL: ${metadata.websiteUrl}`);
-
-  const crawlMaxPages = maxPages || metadata.pageCount * 2 || 500;
-
-  // Re-crawl the website
-  output.step('Re-crawling documentation...');
-  console.log();
-
-  const crawlResult = await crawlSimple(
-    {
-      startUrl: metadata.websiteUrl,
-      maxPages: crawlMaxPages,
-    },
-    (event) => {
-      switch (event.type) {
-        case 'navigating':
-          output.step(`Navigating: ${event.url}`);
-          break;
-        case 'extracted':
-          output.success(`[${event.pageCount}/${event.totalPages}] ${event.title || event.url}`);
-          break;
-        case 'error':
-          output.warn(`Error: ${event.error}`);
-          break;
-        case 'complete':
-          console.log();
-          output.success(`Crawl complete: ${event.pageCount} pages`);
-          break;
-      }
-    }
-  );
-
-  if (crawlResult.pages.length === 0) {
-    output.warn('No documentation pages found during sync');
-    return;
-  }
-
-  // Save crawled pages
-  await saveCrawledPages(libraryId, crawlResult.pages);
-
-  // Convert crawled pages to DocFiles for parsing
-  const files: DocFile[] = crawlResult.pages.map((page) => ({
-    path: page.path || new URL(page.url).pathname,
-    content: page.content,
-  }));
-
-  // Parse and chunk files
-  output.step('Parsing and chunking documentation...');
-  const allChunks: Chunk[] = [];
-
-  for (const file of files) {
-    try {
-      const chunks = parseMarkdown(file.content, file.path);
-      allChunks.push(...chunks);
-    } catch (error) {
-      output.warn(`Failed to parse ${file.path}: ${(error as Error).message}`);
-    }
-  }
-
-  output.success(`Created ${allChunks.length} chunks`);
-
-  if (allChunks.length === 0) {
-    output.error('No chunks created from documentation');
-    return;
-  }
-
-  // Generate embeddings
-  output.step('Generating embeddings...');
-  const texts = allChunks.map((chunk) => chunk.content);
-  const embeddings = await generateEmbeddings(texts, (processed, total) => {
-    output.step(`Generating embeddings... ${processed}/${total}`);
-  });
-  output.success(`Generated ${embeddings.length} embeddings`);
-
-  // Delete old index and recreate
-  const indexName = metadata.indexName;
-  output.step('Recreating vector index...');
-  try {
-    if (await indexExists(indexName)) {
-      await deleteIndex(indexName);
-    }
-  } catch {
-    // Index might not exist
-  }
-  await createIndex(indexName);
-  output.success('Vector index recreated');
-
-  // Prepare and upsert vectors
-  output.step('Uploading vectors...');
-  const vectors: VectorData[] = allChunks.map((chunk, i) => ({
-    key: chunk.id,
-    data: {
-      float32: embeddings[i],
-    },
-    metadata: {
-      libraryId,
-      version: new Date().toISOString().split('T')[0],
-      filePath: chunk.filePath,
-      chunkIndex: i,
-      title: chunk.title,
-      type: chunk.type,
-      content: chunk.content,
-    },
-  }));
-
-  await upsertVectors(indexName, vectors);
-  output.success(`Uploaded ${vectors.length} vectors`);
-
-  // Extract API endpoints
-  output.step('Extracting API endpoints...');
-  let endpointCount = 0;
-  try {
-    const extractionResult = await extractEndpoints(crawlResult.pages, (event) => {
-      if (event.type === 'extracted') {
-        output.step(`Processed ${event.pageIndex}/${event.totalPages} pages, found ${event.endpointCount} endpoints`);
-      } else if (event.type === 'complete') {
-        output.success(`Extraction complete: ${event.endpointCount} endpoints`);
-      }
-    });
-
-    if (extractionResult.endpoints.length > 0) {
-      await saveEndpoints(libraryId, extractionResult.endpoints);
-      endpointCount = extractionResult.endpoints.length;
-    }
-  } catch (error) {
-    output.warn(`Endpoint extraction failed: ${(error as Error).message}`);
-  }
-
-  // Update metadata
-  metadata.indexedAt = new Date().toISOString();
-  metadata.chunkCount = allChunks.length;
-  metadata.pageCount = crawlResult.pages.length;
-  metadata.endpointCount = endpointCount > 0 ? endpointCount : undefined;
-  metadata.crawledUrls = crawlResult.pages.map((p) => p.url);
-
-  await saveLibraryMetadata(libraryId, metadata);
-  await cacheChunks(libraryId, allChunks);
-
-  output.success(`Synced ${metadata.name} successfully!`);
-  output.info(`Pages: ${crawlResult.pages.length}`);
-  output.info(`Chunks: ${allChunks.length}`);
-  if (endpointCount > 0) {
-    output.info(`Endpoints: ${endpointCount}`);
-  }
-}

package/skills/apidocs/src/index.ts

@@ -1,3 +0,0 @@
-#!/usr/bin/env bun
-
-console.log("Usage: skills run apidocs -- <command> [options]");

package/skills/apidocs/src/lib/crawler.ts

@@ -1,427 +0,0 @@
-/**
- * Web crawler for documentation sites
- *
- * Two-phase approach:
- * 1. Discovery: Aggressively crawl ALL pages within the domain
- * 2. Extraction: Extract and convert content from each page
- */
-
-import Anthropic from '@anthropic-ai/sdk';
-import type { CrawledPage, CrawlConfig, CrawlEvent, CrawlResult } from '../types/index.js';
-import { htmlToMarkdown } from './html-to-md.js';
-
-/**
- * Normalize URL for deduplication
- */
-function normalizeUrl(url: string): string {
-  try {
-    const parsed = new URL(url);
-    // Remove trailing slash, hash, and common tracking params
-    let normalized = `${parsed.protocol}//${parsed.host}${parsed.pathname}`.replace(/\/$/, '');
-    // Keep meaningful query params (like ?tab=, ?section=) but remove tracking
-    const dominated = ['utm_', 'ref', 'source', 'fbclid', 'gclid'];
-    const params = new URLSearchParams(parsed.search);
-    const cleanParams = new URLSearchParams();
-    for (const [key, value] of params) {
-      if (!dominated.some((d) => key.startsWith(d))) {
-        cleanParams.set(key, value);
-      }
-    }
-    const search = cleanParams.toString();
-    if (search) {
-      normalized += `?${search}`;
-    }
-    return normalized.toLowerCase();
-  } catch {
-    return url.toLowerCase();
-  }
-}
-
-/**
- * Check if URL should be skipped entirely (non-content URLs)
- */
-function shouldSkipUrl(url: string): boolean {
-  const skipPatterns = [
-    // Static assets
-    /\.(png|jpg|jpeg|gif|svg|ico|webp|pdf|zip|tar|gz|mp4|mp3|wav|woff|woff2|ttf|eot)$/i,
-    /\.(css|js|mjs|jsx|ts|tsx)$/i,
-    /\.(json|xml|yaml|yml)$/i,
-    /\/assets\//i,
-    /\/_next\//i,
-    /\/static\//i,
-    /\/chunks\//i,
-    // Protocol handlers
-    /^mailto:/i,
-    /^tel:/i,
-    /^javascript:/i,
-    /^#/,
-    // Feeds
-    /\/feed\/?$/i,
-    /\/rss\/?$/i,
-    /\/atom\/?$/i,
-    // Auth/account
-    /\/(login|logout|signin|signout|signup|register|auth)\/?/i,
-    /\/(account|profile|settings|dashboard)\/?$/i,
-    // E-commerce
-    /\/(cart|checkout|payment|order)\/?/i,
-    // Search
-    /\/search\?/i,
-    /\/search\/?$/i,
-  ];
-
-  return skipPatterns.some((pattern) => pattern.test(url));
-}
-
-/**
- * Check if URL is likely a documentation page (loose check)
- */
-function isLikelyDocPage(url: string): boolean {
-  // Skip obvious non-doc pages
-  const nonDocPatterns = [
-    /\/blog\//i,
-    /\/news\//i,
-    /\/press\//i,
-    /\/careers\//i,
-    /\/jobs\//i,
-    /\/about-us\/?$/i,
-    /\/contact-us\/?$/i,
-    /\/terms\/?$/i,
-    /\/privacy\/?$/i,
-    /\/legal\/?$/i,
-    /\/cookie/i,
-    /\/pricing\/?$/i,
-    /\/enterprise\/?$/i,
-    /\/customers\/?$/i,
-    /\/case-studies\//i,
-    /\/testimonials\//i,
-  ];
-
-  if (nonDocPatterns.some((p) => p.test(url))) {
-    return false;
-  }
-
-  return true;
-}
-
-/**
- * Extract all links from HTML
- */
-function extractLinks(html: string, baseUrl: string, baseDomain: string): string[] {
-  const links: string[] = [];
-  const seen = new Set<string>();
-
-  // Match href attributes
-  const hrefPattern = /href=["']([^"']+)["']/gi;
-  let match;
-
-  while ((match = hrefPattern.exec(html)) !== null) {
-    let href = match[1];
-
-    // Skip empty, anchors, javascript
-    if (!href || href.startsWith('#') || href.startsWith('javascript:') || href.startsWith('mailto:')) {
-      continue;
-    }
-
-    // Resolve relative URLs
-    try {
-      const resolved = new URL(href, baseUrl);
-
-      // Only follow same domain
-      if (!resolved.hostname.endsWith(baseDomain) && resolved.hostname !== baseDomain) {
-        continue;
-      }
-
-      const normalized = normalizeUrl(resolved.href);
-
-      // Skip if already seen or should be skipped
-      if (seen.has(normalized) || shouldSkipUrl(normalized)) {
-        continue;
-      }
-
-      seen.add(normalized);
-      links.push(resolved.href);
-    } catch {
-      // Invalid URL, skip
-    }
-  }
-
-  return links;
-}
-
-/**
- * Fetch a URL and return HTML content
- */
-async function fetchPage(url: string): Promise<{ html: string; finalUrl: string; title: string }> {
-  const response = await fetch(url, {
-    headers: {
-      'User-Agent':
-        'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
-      Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
-      'Accept-Language': 'en-US,en;q=0.9',
-    },
-    redirect: 'follow',
-  });
-
-  if (!response.ok) {
-    throw new Error(`HTTP ${response.status}: ${response.statusText}`);
-  }
-
-  const contentType = response.headers.get('content-type') || '';
-  if (!contentType.includes('text/html') && !contentType.includes('application/xhtml')) {
-    throw new Error(`Not HTML: ${contentType}`);
-  }
-
-  const html = await response.text();
-  const finalUrl = response.url;
-
-  // Extract title
-  const titleMatch = html.match(/<title[^>]*>([^<]+)<\/title>/i);
-  const title = titleMatch ? titleMatch[1].trim() : '';
-
-  return { html, finalUrl, title };
-}
-
-/**
- * Extract main content from HTML
- */
-function extractContent(html: string): string {
-  // Try to find main content area
-  const contentSelectors = [
-    /<main[^>]*>([\s\S]*?)<\/main>/i,
-    /<article[^>]*>([\s\S]*?)<\/article>/i,
-    /<div[^>]*class="[^"]*(?:content|docs|documentation|markdown|prose)[^"]*"[^>]*>([\s\S]*?)<\/div>/i,
-    /<div[^>]*id="(?:content|main|docs)"[^>]*>([\s\S]*?)<\/div>/i,
-    /<div[^>]*role="main"[^>]*>([\s\S]*?)<\/div>/i,
-  ];
-
-  for (const pattern of contentSelectors) {
-    const match = html.match(pattern);
-    if (match && match[1] && match[1].length > 500) {
-      return match[1];
-    }
-  }
-
-  // Fall back to body content
-  const bodyMatch = html.match(/<body[^>]*>([\s\S]*)<\/body>/i);
-  if (bodyMatch) {
-    // Remove obvious non-content elements
-    let content = bodyMatch[1];
-    content = content.replace(/<nav[^>]*>[\s\S]*?<\/nav>/gi, '');
-    content = content.replace(/<header[^>]*>[\s\S]*?<\/header>/gi, '');
-    content = content.replace(/<footer[^>]*>[\s\S]*?<\/footer>/gi, '');
-    content = content.replace(/<aside[^>]*>[\s\S]*?<\/aside>/gi, '');
-    content = content.replace(/<script[^>]*>[\s\S]*?<\/script>/gi, '');
-    content = content.replace(/<style[^>]*>[\s\S]*?<\/style>/gi, '');
-    content = content.replace(/<!--[\s\S]*?-->/g, '');
-    return content;
-  }
-
-  return html;
-}
-
-/**
- * Check if content looks like documentation (has code, API references, etc.)
- */
-function hasDocumentationContent(html: string): boolean {
-  const docIndicators = [
-    /<pre[^>]*>/i,
-    /<code[^>]*>/i,
-    /```/,
-    /class="[^"]*(?:highlight|syntax|code)[^"]*"/i,
-    /api/i,
-    /endpoint/i,
-    /parameter/i,
-    /request/i,
-    /response/i,
-    /example/i,
-    /usage/i,
-    /installation/i,
-    /getting.?started/i,
-    /quickstart/i,
-    /tutorial/i,
-    /guide/i,
-    /reference/i,
-    /documentation/i,
-  ];
-
-  const matchCount = docIndicators.filter((p) => p.test(html)).length;
-  return matchCount >= 2; // At least 2 indicators
-}
-
-/**
- * Main crawler function
- */
-export async function crawlDocumentation(
-  config: CrawlConfig,
-  onEvent: (event: CrawlEvent) => void = () => {}
-): Promise<CrawlResult> {
-  const startTime = Date.now();
-
-  // Parse base domain
-  const startUrlParsed = new URL(config.startUrl);
-  const baseDomain = startUrlParsed.hostname.replace(/^www\./, '');
-
-  // State
-  const visited = new Set<string>();
-  const toVisit: string[] = [config.startUrl];
-  const pages: CrawledPage[] = [];
-  const errors: string[] = [];
-  const contentHashes = new Set<string>(); // For deduplication
-
-  onEvent({ type: 'navigating', url: config.startUrl });
-
-  // Phase 1 & 2 combined: Crawl and extract in one pass
-  while (toVisit.length > 0 && pages.length < config.maxPages) {
-    const url = toVisit.shift()!;
-    const normalizedUrl = normalizeUrl(url);
-
-    // Skip if already visited
-    if (visited.has(normalizedUrl)) {
-      continue;
-    }
-    visited.add(normalizedUrl);
-
-    // Skip non-doc URLs
-    if (!isLikelyDocPage(url)) {
-      onEvent({ type: 'skipped', url, reason: 'non-documentation URL pattern' });
-      continue;
-    }
-
-    try {
-      onEvent({ type: 'navigating', url });
-
-      const { html, finalUrl, title } = await fetchPage(url);
-
-      // Also mark final URL as visited (in case of redirects)
-      visited.add(normalizeUrl(finalUrl));
-
-      // Extract links for discovery (do this for ALL pages)
-      const links = extractLinks(html, finalUrl, baseDomain);
-      for (const link of links) {
-        const normalizedLink = normalizeUrl(link);
-        if (!visited.has(normalizedLink)) {
-          toVisit.push(link);
-        }
-      }
-
-      // Check if page has documentation content
-      if (!hasDocumentationContent(html)) {
-        onEvent({ type: 'skipped', url: finalUrl, reason: 'no documentation content detected' });
-        continue;
-      }
-
-      // Extract main content
-      const mainContent = extractContent(html);
-
-      // Convert to markdown
-      const markdown = htmlToMarkdown(mainContent);
-
-      // Skip if markdown is too short (likely not useful)
-      if (markdown.length < 100) {
-        onEvent({ type: 'skipped', url: finalUrl, reason: 'content too short' });
-        continue;
-      }
-
-      // Deduplicate by content hash (first 500 chars)
-      const contentHash = markdown.substring(0, 500);
-      if (contentHashes.has(contentHash)) {
-        onEvent({ type: 'skipped', url: finalUrl, reason: 'duplicate content' });
-        continue;
-      }
-      contentHashes.add(contentHash);
-
-      // Parse path from URL
-      const urlObj = new URL(finalUrl);
-      const path = urlObj.pathname;
-
-      // Save page
-      const page: CrawledPage = {
-        url: finalUrl,
-        path,
-        title: title || path,
-        content: markdown,
-        html: mainContent,
-        crawledAt: new Date().toISOString(),
-      };
-
-      pages.push(page);
-      onEvent({
-        type: 'extracted',
-        url: finalUrl,
-        title: title || path,
-        pageCount: pages.length,
-        totalPages: config.maxPages,
-      });
-
-      // Small delay to be polite
-      await new Promise((resolve) => setTimeout(resolve, 100));
-    } catch (error) {
-      const errorMsg = (error as Error).message;
-      errors.push(`${url}: ${errorMsg}`);
-      onEvent({ type: 'error', url, error: errorMsg });
-    }
-  }
-
-  onEvent({ type: 'complete', pageCount: pages.length });
-
-  return {
-    pages,
-    totalPages: pages.length,
-    duration: Date.now() - startTime,
-    errors,
-  };
-}
-
-/**
- * Alias for backwards compatibility
- */
-export const crawlSimple = crawlDocumentation;
-
-/**
- * Use Claude to clean up and enhance extracted content
- * Call this after crawling to improve content quality
- */
-export async function enhanceContentWithAgent(
-  pages: CrawledPage[],
-  onEvent: (event: CrawlEvent) => void = () => {}
-): Promise<CrawledPage[]> {
-  const client = new Anthropic();
-  const enhanced: CrawledPage[] = [];
-
-  for (const page of pages) {
-    try {
-      const response = await client.messages.create({
-        model: 'claude-sonnet-4-20250514',
-        max_tokens: 8192,
-        messages: [
-          {
-            role: 'user',
-            content: `Clean up this documentation content. Remove navigation elements, ads, cookie notices, and other non-documentation content. Keep all code examples, API references, and explanatory text. Output clean markdown only, no explanation.
-
-Title: ${page.title}
-URL: ${page.url}
-
-Content:
-${page.content}`,
-          },
-        ],
-      });
-
-      const textBlock = response.content.find((b) => b.type === 'text');
-      if (textBlock && textBlock.type === 'text') {
-        enhanced.push({
-          ...page,
-          content: textBlock.text,
-        });
-      } else {
-        enhanced.push(page);
-      }
-    } catch (error) {
-      // Keep original on error
-      enhanced.push(page);
-      onEvent({ type: 'error', url: page.url, error: (error as Error).message });
-    }
-  }
-
-  return enhanced;
-}