@hasna/skills 0.1.59 → 0.1.61

package/skills/apidocs/src/commands/add.ts

@@ -1,279 +0,0 @@
-import type { AddOptions, Chunk, VectorData, LibraryMetadata, DocFile } from '../types/index.js';
-import { parseWebsiteUrl, createLibraryIdFromUrl, createIndexNameFromUrl } from '../utils/paths.js';
-import * as output from '../utils/output.js';
-import { crawlDocumentation, crawlSimple } from '../lib/crawler.js';
-import { parseMarkdown } from '../lib/parser.js';
-import { generateEmbeddings } from '../lib/embeddings.js';
-import { createIndex, upsertVectors, indexExists, deleteIndex } from '../lib/vectors.js';
-import {
-  initStorage,
-  saveLibraryMetadata,
-  cacheChunks,
-  getLibraryMetadata,
-  saveCrawledPages,
-  saveEndpoints,
-} from '../lib/storage.js';
-import { discoverDocsUrl, isLikelyDocsUrl } from '../lib/discovery.js';
-import { extractEndpoints } from '../lib/endpoint-extractor.js';
-
-/**
- * Add a library from a documentation website
- */
-export async function add(websiteUrl: string, options: AddOptions): Promise<void> {
-  // Parse website URL
-  const parsed = parseWebsiteUrl(websiteUrl);
-  if (!parsed) {
-    output.error(`Invalid URL: ${websiteUrl}`);
-    process.exit(1);
-  }
-
-  const libraryId = createLibraryIdFromUrl(websiteUrl, options.name);
-  const libraryName = options.name || parsed.domain.replace(/^(www\.|docs\.|api\.|platform\.)/, '');
-
-  output.info(`Adding library: ${libraryName}`);
-  output.info(`URL: ${websiteUrl}`);
-
-  // Initialize storage
-  await initStorage();
-
-  // Check if already indexed
-  const existing = await getLibraryMetadata(libraryId);
-  if (existing) {
-    output.warn(`Library ${libraryName} is already indexed`);
-    output.info('Use `service-apidocs sync` to update the index');
-    return;
-  }
-
-  // Discover documentation URL if needed
-  let docsUrl = websiteUrl;
-  if (!isLikelyDocsUrl(websiteUrl)) {
-    output.step('Discovering documentation URL...');
-    const discovery = await discoverDocsUrl(websiteUrl, (event) => {
-      switch (event.type) {
-        case 'navigating':
-          output.step(`Checking: ${event.url || event.message}`);
-          break;
-        case 'analyzing':
-          output.step(event.message || 'Analyzing page...');
-          break;
-        case 'found':
-          output.success(`Found docs: ${event.docsUrl}`);
-          break;
-        case 'not_found':
-          output.warn(event.message || 'Could not find specific documentation URL');
-          break;
-        case 'error':
-          output.warn(`Discovery error: ${event.error}`);
-          break;
-      }
-    });
-
-    if (discovery.success && discovery.docsUrl) {
-      docsUrl = discovery.docsUrl;
-    }
-    console.log();
-  } else {
-    output.info('URL appears to be documentation, skipping discovery');
-  }
-
-  // Set max pages
-  const maxPages = options.maxPages || 500;
-  output.info(`Max pages: ${maxPages}`);
-
-  // Crawl documentation
-  output.step('Crawling documentation...');
-  console.log();
-
-  const crawlResult = await crawlSimple(
-    {
-      startUrl: docsUrl,
-      maxPages,
-    },
-    (event) => {
-      switch (event.type) {
-        case 'navigating':
-          output.step(`Navigating: ${event.url}`);
-          break;
-        case 'extracted':
-          output.success(`[${event.pageCount}/${event.totalPages}] ${event.title || event.url}`);
-          break;
-        case 'skipped':
-          // Don't log skipped pages to reduce noise
-          break;
-        case 'error':
-          output.warn(`Error: ${event.error}`);
-          break;
-        case 'complete':
-          console.log();
-          output.success(`Crawl complete: ${event.pageCount} pages`);
-          break;
-      }
-    }
-  );
-
-  if (crawlResult.pages.length === 0) {
-    output.error('No documentation pages found');
-    output.info('The crawler could not find any documentation content at this URL');
-    return;
-  }
-
-  // Save crawled pages
-  await saveCrawledPages(libraryId, crawlResult.pages);
-
-  // Convert crawled pages to DocFiles for parsing
-  const files: DocFile[] = crawlResult.pages.map((page) => ({
-    path: page.path || new URL(page.url).pathname,
-    content: page.content,
-  }));
-
-  // Parse and chunk files
-  output.step('Parsing and chunking documentation...');
-  const allChunks: Chunk[] = [];
-
-  for (const file of files) {
-    try {
-      const chunks = parseMarkdown(file.content, file.path);
-      allChunks.push(...chunks);
-    } catch (error) {
-      output.warn(`Failed to parse ${file.path}: ${(error as Error).message}`);
-    }
-  }
-
-  output.success(`Created ${allChunks.length} chunks`);
-
-  if (allChunks.length === 0) {
-    output.error('No chunks created from documentation');
-    return;
-  }
-
-  // Generate embeddings
-  output.step('Generating embeddings...');
-  const texts = allChunks.map((chunk) => chunk.content);
-  let embeddings: number[][];
-
-  try {
-    embeddings = await generateEmbeddings(texts, (processed, total) => {
-      output.step(`Generating embeddings... ${processed}/${total}`);
-    });
-    output.success(`Generated ${embeddings.length} embeddings`);
-  } catch (error) {
-    output.error(`Failed to generate embeddings: ${(error as Error).message}`);
-    output.info('Make sure OPENAI_API_KEY is set');
-    process.exit(1);
-  }
-
-  // Create vector index
-  const indexName = createIndexNameFromUrl(websiteUrl, options.name);
-  output.step(`Creating vector index: ${indexName}`);
-
-  try {
-    // Delete existing index if it exists
-    if (await indexExists(indexName)) {
-      await deleteIndex(indexName);
-    }
-    await createIndex(indexName);
-    output.success('Vector index created');
-  } catch (error) {
-    output.error(`Failed to create index: ${(error as Error).message}`);
-    process.exit(1);
-  }
-
-  // Prepare and upsert vectors
-  output.step('Uploading vectors...');
-  const vectors: VectorData[] = allChunks.map((chunk, i) => ({
-    key: chunk.id,
-    data: {
-      float32: embeddings[i],
-    },
-    metadata: {
-      libraryId,
-      version: new Date().toISOString().split('T')[0],
-      filePath: chunk.filePath,
-      chunkIndex: i,
-      title: chunk.title,
-      type: chunk.type,
-      content: chunk.content,
-    },
-  }));
-
-  try {
-    await upsertVectors(indexName, vectors);
-    output.success(`Uploaded ${vectors.length} vectors`);
-  } catch (error) {
-    output.error(`Failed to upload vectors: ${(error as Error).message}`);
-    process.exit(1);
-  }
-
-  // Extract API endpoints
-  output.step('Extracting API endpoints...');
-  console.log();
-
-  let endpointCount = 0;
-  try {
-    const extractionResult = await extractEndpoints(crawlResult.pages, (event) => {
-      switch (event.type) {
-        case 'processing':
-          if (event.pageUrl) {
-            output.step(`[${event.pageIndex}/${event.totalPages}] Processing: ${event.pageUrl}`);
-          }
-          break;
-        case 'extracted':
-          output.success(`Processed ${event.pageIndex}/${event.totalPages} pages, found ${event.endpointCount} endpoints`);
-          break;
-        case 'complete':
-          console.log();
-          output.success(`Extraction complete: ${event.endpointCount} endpoints`);
-          break;
-        case 'error':
-          output.warn(`Extraction error: ${event.error}`);
-          break;
-      }
-    });
-
-    if (extractionResult.endpoints.length > 0) {
-      await saveEndpoints(libraryId, extractionResult.endpoints);
-      endpointCount = extractionResult.endpoints.length;
-    }
-  } catch (error) {
-    output.warn(`Endpoint extraction failed: ${(error as Error).message}`);
-    output.info('Continuing without endpoint data...');
-  }
-
-  // Save metadata locally
-  output.step('Saving metadata...');
-  const metadata: LibraryMetadata = {
-    id: libraryId,
-    name: libraryName,
-    websiteUrl,
-    docsUrl: docsUrl !== websiteUrl ? docsUrl : undefined,
-    domain: parsed.domain,
-    indexedAt: new Date().toISOString(),
-    chunkCount: allChunks.length,
-    pageCount: crawlResult.pages.length,
-    endpointCount: endpointCount > 0 ? endpointCount : undefined,
-    indexName,
-    crawledUrls: crawlResult.pages.map((p) => p.url),
-  };
-
-  await saveLibraryMetadata(libraryId, metadata);
-  await cacheChunks(libraryId, allChunks);
-
-  console.log();
-  output.success(`Library "${libraryName}" indexed successfully!`);
-  output.info(`Pages: ${crawlResult.pages.length}`);
-  output.info(`Chunks: ${allChunks.length}`);
-  if (endpointCount > 0) {
-    output.info(`Endpoints: ${endpointCount}`);
-  }
-  output.info(`Index: ${indexName}`);
-  output.info(`Duration: ${(crawlResult.duration / 1000).toFixed(1)}s`);
-
-  if (crawlResult.errors.length > 0) {
-    console.log();
-    output.warn(`${crawlResult.errors.length} errors during crawl:`);
-    crawlResult.errors.slice(0, 5).forEach((err) => output.info(`  - ${err}`));
-    if (crawlResult.errors.length > 5) {
-      output.info(`  ... and ${crawlResult.errors.length - 5} more`);
-    }
-  }
-}

package/skills/apidocs/src/commands/endpoints.ts

@@ -1,157 +0,0 @@
-import chalk from 'chalk';
-import type { EndpointsOptions, APIEndpoint } from '../types/index.js';
-import * as output from '../utils/output.js';
-import { initStorage, findLibrary, loadEndpoints } from '../lib/storage.js';
-import { groupByResource, groupByMethod } from '../lib/endpoint-extractor.js';
-
-/**
- * Display API endpoints from indexed documentation
- */
-export async function endpoints(library: string, options: EndpointsOptions): Promise<void> {
-  await initStorage();
-
-  // Find library
-  const meta = await findLibrary(library);
-  if (!meta) {
-    output.error(`Library not found: ${library}`);
-    output.info('Use `service-apidocs list` to see indexed libraries');
-    process.exit(1);
-  }
-
-  // Load endpoints
-  const allEndpoints = await loadEndpoints(meta.id);
-
-  if (allEndpoints.length === 0) {
-    output.warn(`No API endpoints found for library: ${meta.name}`);
-    output.info('Endpoints are extracted during indexing. Try re-indexing with `service-apidocs sync`.');
-    return;
-  }
-
-  // Apply filter if specified
-  let filteredEndpoints = allEndpoints;
-  if (options.filter) {
-    const filterMethod = options.filter.toUpperCase();
-    filteredEndpoints = allEndpoints.filter((e) => e.method === filterMethod);
-
-    if (filteredEndpoints.length === 0) {
-      output.warn(`No ${filterMethod} endpoints found`);
-      return;
-    }
-  }
-
-  // JSON output
-  if (options.json) {
-    output.json({
-      library: meta.name,
-      libraryId: meta.id,
-      docsUrl: meta.docsUrl || meta.websiteUrl,
-      endpointCount: filteredEndpoints.length,
-      endpoints: filteredEndpoints,
-    });
-    return;
-  }
-
-  // Header
-  const docsUrl = meta.docsUrl || meta.websiteUrl;
-  console.log();
-  console.log(chalk.bold(`${meta.name} API Endpoints`) + chalk.dim(` (${filteredEndpoints.length} total)`));
-  console.log(chalk.dim(`Docs: ${docsUrl}`));
-  console.log();
-
-  // Group and display
-  const groupBy = options.groupBy || 'resource';
-
-  if (groupBy === 'method') {
-    displayGroupedByMethod(filteredEndpoints);
-  } else {
-    displayGroupedByResource(filteredEndpoints);
-  }
-}
-
-/**
- * Display endpoints grouped by resource
- */
-function displayGroupedByResource(endpoints: APIEndpoint[]): void {
-  const grouped = groupByResource(endpoints);
-
-  for (const [resource, resourceEndpoints] of grouped) {
-    // Resource header
-    console.log(chalk.bold.cyan(capitalizeFirst(resource)));
-
-    // Endpoints in this resource
-    for (const endpoint of resourceEndpoints) {
-      const methodColor = getMethodColor(endpoint.method);
-      const method = chalk[methodColor](endpoint.method.padEnd(7));
-      const path = endpoint.path;
-      const title = chalk.dim(truncate(endpoint.title, 50));
-
-      console.log(`  ${method} ${path}  ${title}`);
-    }
-
-    console.log(); // Space between resources
-  }
-}
-
-/**
- * Display endpoints grouped by HTTP method
- */
-function displayGroupedByMethod(endpoints: APIEndpoint[]): void {
-  const grouped = groupByMethod(endpoints);
-
-  for (const [method, methodEndpoints] of grouped) {
-    if (methodEndpoints.length === 0) continue;
-
-    // Method header
-    const methodColor = getMethodColor(method);
-    console.log(chalk.bold[methodColor](`${method}`) + chalk.dim(` (${methodEndpoints.length} endpoints)`));
-
-    // Endpoints with this method
-    for (const endpoint of methodEndpoints) {
-      const path = endpoint.path.padEnd(40);
-      const title = chalk.dim(truncate(endpoint.title, 40));
-
-      console.log(`  ${path} ${title}`);
-    }
-
-    console.log(); // Space between methods
-  }
-}
-
-/**
- * Get chalk color for HTTP method
- */
-function getMethodColor(method: string): 'green' | 'blue' | 'yellow' | 'red' | 'magenta' | 'cyan' {
-  switch (method) {
-    case 'GET':
-      return 'green';
-    case 'POST':
-      return 'blue';
-    case 'PUT':
-      return 'yellow';
-    case 'PATCH':
-      return 'yellow';
-    case 'DELETE':
-      return 'red';
-    case 'HEAD':
-      return 'magenta';
-    case 'OPTIONS':
-      return 'cyan';
-    default:
-      return 'cyan';
-  }
-}
-
-/**
- * Capitalize first letter
- */
-function capitalizeFirst(str: string): string {
-  return str.charAt(0).toUpperCase() + str.slice(1);
-}
-
-/**
- * Truncate string with ellipsis
- */
-function truncate(str: string, maxLength: number): string {
-  if (str.length <= maxLength) return str;
-  return str.substring(0, maxLength - 3) + '...';
-}

package/skills/apidocs/src/commands/list.ts

@@ -1,51 +0,0 @@
-import * as output from '../utils/output.js';
-import { listLibraries, initStorage } from '../lib/storage.js';
-import { formatNumber } from '../utils/output.js';
-import chalk from 'chalk';
-
-/**
- * List all indexed libraries
- */
-export async function list(): Promise<void> {
-  await initStorage();
-
-  const libraries = await listLibraries();
-
-  if (libraries.length === 0) {
-    output.info('No libraries indexed yet');
-    output.info('Use `service-apidocs add <website-url>` to add documentation');
-    return;
-  }
-
-  console.log();
-  console.log(chalk.bold('Indexed Libraries'));
-  console.log('─'.repeat(80));
-
-  const rows = libraries.map((lib) => {
-    const name = chalk.cyan(lib.name);
-    const domain = chalk.yellow(lib.domain);
-    const pages = chalk.gray(`${formatNumber(lib.pageCount || 0)} pages`);
-    const chunks = chalk.gray(`${formatNumber(lib.chunkCount)} chunks`);
-    const endpoints = lib.endpointCount
-      ? chalk.magenta(`${formatNumber(lib.endpointCount)} endpoints`)
-      : chalk.gray('-');
-    const date = new Date(lib.indexedAt).toLocaleDateString();
-
-    return [name, domain, pages, chunks, endpoints, chalk.gray(date)];
-  });
-
-  console.log(output.table(rows, ['Library', 'Domain', 'Pages', 'Chunks', 'Endpoints', 'Indexed']));
-
-  // Show docs URLs for libraries that have them
-  const withDocsUrl = libraries.filter((lib) => lib.docsUrl);
-  if (withDocsUrl.length > 0) {
-    console.log();
-    console.log(chalk.dim('Documentation URLs:'));
-    for (const lib of withDocsUrl) {
-      console.log(chalk.dim(`  ${lib.name}: ${lib.docsUrl}`));
-    }
-  }
-
-  console.log();
-  console.log(chalk.gray(`Total: ${libraries.length} libraries`));
-}

package/skills/apidocs/src/commands/query.ts

@@ -1,63 +0,0 @@
-import type { QueryOptions } from '../types/index.js';
-import * as output from '../utils/output.js';
-import { findLibrary, initStorage } from '../lib/storage.js';
-import { semanticSearch, searchAsJson } from '../lib/search.js';
-import { indexExists } from '../lib/vectors.js';
-
-/**
- * Query documentation semantically
- */
-export async function query(library: string, question: string, options: QueryOptions): Promise<void> {
-  await initStorage();
-
-  // Find the library
-  const metadata = await findLibrary(library);
-
-  if (!metadata) {
-    output.error(`Library not found: ${library}`);
-    output.info('Use `service-apidocs list` to see indexed libraries');
-    process.exit(1);
-  }
-
-  const { indexName, id: libraryId } = metadata;
-
-  // Verify index exists
-  const exists = await indexExists(indexName);
-  if (!exists) {
-    output.error(`Index not found: ${indexName}`);
-    output.info('The library may need to be re-indexed. Use `service-apidocs sync`');
-    process.exit(1);
-  }
-
-  // Determine output format
-  const maxTokens = options.tokens || 8000;
-  const topK = options.topK || 10;
-
-  if (options.json) {
-    // JSON output
-    try {
-      const result = await searchAsJson(indexName, question, { topK, maxTokens });
-      output.json(result);
-    } catch (error) {
-      output.error(`Search failed: ${(error as Error).message}`);
-      process.exit(1);
-    }
-  } else {
-    // Markdown output
-    try {
-      const result = await semanticSearch(indexName, question, { topK, maxTokens });
-
-      // Print header
-      console.log();
-      console.log(`# ${metadata.config?.projectTitle || metadata.name}`);
-      console.log(`> Query: "${question}"`);
-      console.log();
-
-      // Print content
-      output.markdown(result.content);
-    } catch (error) {
-      output.error(`Search failed: ${(error as Error).message}`);
-      process.exit(1);
-    }
-  }
-}

package/skills/apidocs/src/commands/remove.ts

@@ -1,39 +0,0 @@
-import * as output from '../utils/output.js';
-import { findLibrary, deleteLibrary, initStorage } from '../lib/storage.js';
-import { deleteIndex } from '../lib/vectors.js';
-
-/**
- * Remove a library from the index
- */
-export async function remove(library: string): Promise<void> {
-  await initStorage();
-
-  // Find the library
-  const metadata = await findLibrary(library);
-
-  if (!metadata) {
-    output.error(`Library not found: ${library}`);
-    output.info('Use `service-apidocs list` to see indexed libraries');
-    process.exit(1);
-  }
-
-  const { id: libraryId, indexName } = metadata;
-
-  output.info(`Removing: ${libraryId}`);
-
-  // Delete vector index
-  output.step('Deleting vector index...');
-  try {
-    await deleteIndex(indexName);
-    output.success('Vector index deleted');
-  } catch (error) {
-    output.warn(`Could not delete index: ${(error as Error).message}`);
-  }
-
-  // Delete local metadata
-  output.step('Removing local data...');
-  await deleteLibrary(libraryId);
-  output.success('Local data removed');
-
-  output.success(`Library ${libraryId} removed successfully`);
-}