@hasna/skills 0.1.60 → 0.1.61

package/skills/apidocs/src/lib/parser.ts

@@ -1,288 +0,0 @@
-import { unified } from 'unified';
-import remarkParse from 'remark-parse';
-import remarkGfm from 'remark-gfm';
-import remarkMdx from 'remark-mdx';
-import { visit } from 'unist-util-visit';
-import type { Root, Heading, Code, Text, Paragraph } from 'mdast';
-import type { Chunk } from '../types/index.js';
-import { createHash } from 'crypto';
-
-const MAX_CHUNK_TOKENS = 500;
-const CHARS_PER_TOKEN = 4; // Rough estimate
-
-/**
- * Create a unique chunk ID
- */
-function createChunkId(filePath: string, index: number, content: string): string {
-  const hash = createHash('md5').update(`${filePath}:${index}:${content}`).digest('hex').slice(0, 8);
-  return `${filePath.replace(/[\/\.]/g, '-')}-${index}-${hash}`;
-}
-
-/**
- * Estimate token count for text
- */
-function estimateTokens(text: string): number {
-  return Math.ceil(text.length / CHARS_PER_TOKEN);
-}
-
-/**
- * Extract text content from AST node
- */
-function extractText(node: unknown): string {
-  if (!node || typeof node !== 'object') return '';
-
-  const n = node as Record<string, unknown>;
-
-  if (n.type === 'text' && typeof n.value === 'string') {
-    return n.value;
-  }
-
-  if (n.type === 'code' && typeof n.value === 'string') {
-    return n.value;
-  }
-
-  if (Array.isArray(n.children)) {
-    return n.children.map((child) => extractText(child)).join('');
-  }
-
-  return '';
-}
-
-/**
- * Get heading text
- */
-function getHeadingText(node: Heading): string {
-  return extractText(node);
-}
-
-/**
- * Parse markdown content into chunks
- */
-export function parseMarkdown(content: string, filePath: string): Chunk[] {
-  const processor = unified()
-    .use(remarkParse)
-    .use(remarkGfm)
-    .use(remarkMdx);
-
-  let ast: Root;
-  try {
-    ast = processor.parse(content);
-  } catch {
-    // If MDX parsing fails, try plain markdown
-    ast = unified().use(remarkParse).use(remarkGfm).parse(content);
-  }
-
-  return chunkByHeadings(ast, filePath, MAX_CHUNK_TOKENS);
-}
-
-/**
- * Chunk content by headings
- */
-function chunkByHeadings(ast: Root, filePath: string, maxTokens: number): Chunk[] {
-  const chunks: Chunk[] = [];
-  const headingStack: { level: number; text: string }[] = [];
-
-  let currentContent: string[] = [];
-  let currentTitle = '';
-  let chunkIndex = 0;
-
-  // Get title from first heading or filename
-  let documentTitle = filePath.split('/').pop()?.replace(/\.(md|mdx)$/, '') || 'Document';
-
-  function flushChunk(type: 'text' | 'code' = 'text', codeLanguage?: string): void {
-    const content = currentContent.join('\n\n').trim();
-    if (!content) return;
-
-    const tokens = estimateTokens(content);
-
-    // If chunk is too large, split it
-    if (tokens > maxTokens) {
-      const subChunks = splitLargeChunk(content, maxTokens);
-      for (const subContent of subChunks) {
-        chunks.push({
-          id: createChunkId(filePath, chunkIndex, subContent),
-          content: subContent,
-          title: currentTitle || documentTitle,
-          type,
-          filePath,
-          headingHierarchy: headingStack.map((h) => h.text),
-          codeLanguage,
-          tokenCount: estimateTokens(subContent),
-        });
-        chunkIndex++;
-      }
-    } else {
-      chunks.push({
-        id: createChunkId(filePath, chunkIndex, content),
-        content,
-        title: currentTitle || documentTitle,
-        type,
-        filePath,
-        headingHierarchy: headingStack.map((h) => h.text),
-        codeLanguage,
-        tokenCount: tokens,
-      });
-      chunkIndex++;
-    }
-
-    currentContent = [];
-  }
-
-  visit(ast, (node) => {
-    if (node.type === 'heading') {
-      const heading = node as Heading;
-      const text = getHeadingText(heading);
-
-      // Set document title from first h1
-      if (heading.depth === 1 && !documentTitle) {
-        documentTitle = text;
-      }
-
-      // Flush previous content
-      if (currentContent.length > 0) {
-        flushChunk();
-      }
-
-      // Update heading stack
-      while (headingStack.length > 0 && headingStack[headingStack.length - 1].level >= heading.depth) {
-        headingStack.pop();
-      }
-      headingStack.push({ level: heading.depth, text });
-      currentTitle = text;
-
-      // Add heading to content
-      currentContent.push('#'.repeat(heading.depth) + ' ' + text);
-    } else if (node.type === 'code') {
-      const code = node as Code;
-      // Flush any previous text content
-      if (currentContent.length > 0) {
-        flushChunk();
-      }
-
-      // Create separate chunk for code
-      const codeContent = code.lang ? `\`\`\`${code.lang}\n${code.value}\n\`\`\`` : `\`\`\`\n${code.value}\n\`\`\``;
-
-      currentContent.push(codeContent);
-      flushChunk('code', code.lang || undefined);
-    } else if (node.type === 'paragraph') {
-      const text = extractText(node);
-      if (text.trim()) {
-        currentContent.push(text);
-      }
-    } else if (node.type === 'list') {
-      const text = stringifyList(node);
-      if (text.trim()) {
-        currentContent.push(text);
-      }
-    } else if (node.type === 'blockquote') {
-      const text = '> ' + extractText(node).split('\n').join('\n> ');
-      if (text.trim()) {
-        currentContent.push(text);
-      }
-    } else if (node.type === 'table') {
-      const text = stringifyTable(node);
-      if (text.trim()) {
-        currentContent.push(text);
-      }
-    }
-  });
-
-  // Flush remaining content
-  if (currentContent.length > 0) {
-    flushChunk();
-  }
-
-  return chunks;
-}
-
-/**
- * Split a large chunk into smaller pieces
- */
-function splitLargeChunk(content: string, maxTokens: number): string[] {
-  const chunks: string[] = [];
-  const lines = content.split('\n');
-  let current: string[] = [];
-  let currentTokens = 0;
-
-  for (const line of lines) {
-    const lineTokens = estimateTokens(line);
-
-    if (currentTokens + lineTokens > maxTokens && current.length > 0) {
-      chunks.push(current.join('\n'));
-      current = [];
-      currentTokens = 0;
-    }
-
-    current.push(line);
-    currentTokens += lineTokens;
-  }
-
-  if (current.length > 0) {
-    chunks.push(current.join('\n'));
-  }
-
-  return chunks;
-}
-
-/**
- * Stringify list node
- */
-function stringifyList(node: unknown): string {
-  const n = node as Record<string, unknown>;
-  if (!Array.isArray(n.children)) return '';
-
-  const ordered = n.ordered === true;
-  return n.children
-    .map((item, i) => {
-      const prefix = ordered ? `${i + 1}. ` : '- ';
-      const text = extractText(item);
-      return prefix + text;
-    })
-    .join('\n');
-}
-
-/**
- * Stringify table node
- */
-function stringifyTable(node: unknown): string {
-  const n = node as Record<string, unknown>;
-  if (!Array.isArray(n.children)) return '';
-
-  const rows = n.children.map((row) => {
-    const r = row as Record<string, unknown>;
-    if (!Array.isArray(r.children)) return '';
-    return '| ' + r.children.map((cell) => extractText(cell)).join(' | ') + ' |';
-  });
-
-  if (rows.length > 0) {
-    // Add header separator after first row
-    const headerCells = (n.children[0] as Record<string, unknown>).children as unknown[];
-    const separator = '| ' + headerCells.map(() => '---').join(' | ') + ' |';
-    rows.splice(1, 0, separator);
-  }
-
-  return rows.join('\n');
-}
-
-/**
- * Extract all code blocks from content
- */
-export function extractCodeBlocks(
-  content: string,
-  filePath: string
-): { content: string; language: string; filePath: string }[] {
-  const blocks: { content: string; language: string; filePath: string }[] = [];
-
-  const processor = unified().use(remarkParse).use(remarkGfm);
-  const ast = processor.parse(content);
-
-  visit(ast, 'code', (node: Code) => {
-    blocks.push({
-      content: node.value,
-      language: node.lang || '',
-      filePath,
-    });
-  });
-
-  return blocks;
-}

package/skills/apidocs/src/lib/search.ts

@@ -1,189 +0,0 @@
-import type { SearchResult, QueryResult, VectorMetadata } from '../types/index.js';
-import { generateEmbedding } from './embeddings.js';
-import { queryVectors } from './vectors.js';
-
-const DEFAULT_TOP_K = 10;
-const DEFAULT_MAX_TOKENS = 8000;
-const CHARS_PER_TOKEN = 4;
-
-/**
- * Perform semantic search on an index
- */
-export async function semanticSearch(
-  indexName: string,
-  query: string,
-  options: {
-    topK?: number;
-    maxTokens?: number;
-  } = {}
-): Promise<QueryResult> {
-  const { topK = DEFAULT_TOP_K, maxTokens = DEFAULT_MAX_TOKENS } = options;
-
-  // Generate embedding for the query
-  const queryEmbedding = await generateEmbedding(query);
-
-  // Query vectors
-  const results = await queryVectors(indexName, queryEmbedding, topK * 2); // Get more to filter
-
-  // Deduplicate and rank results
-  const uniqueChunks = deduplicateResults(results);
-
-  // Build response within token limit
-  const { content, sources, chunks } = buildResponse(uniqueChunks, maxTokens);
-
-  return { content, sources, chunks };
-}
-
-/**
- * Deduplicate results by content similarity
- */
-function deduplicateResults(results: SearchResult[]): SearchResult[] {
-  const seen = new Set<string>();
-  const unique: SearchResult[] = [];
-
-  for (const result of results) {
-    const contentKey = result.metadata.content?.substring(0, 100);
-    if (!seen.has(contentKey)) {
-      seen.add(contentKey);
-      unique.push(result);
-    }
-  }
-
-  return unique;
-}
-
-/**
- * Build formatted response from search results
- */
-function buildResponse(
-  results: SearchResult[],
-  maxTokens: number
-): { content: string; sources: string[]; chunks: SearchResult[] } {
-  const maxChars = maxTokens * CHARS_PER_TOKEN;
-  const sources = new Set<string>();
-  const includedChunks: SearchResult[] = [];
-
-  let content = '';
-  let totalChars = 0;
-
-  // Group results by file for better organization
-  const byFile = new Map<string, SearchResult[]>();
-  for (const result of results) {
-    const file = result.metadata.filePath;
-    if (!byFile.has(file)) {
-      byFile.set(file, []);
-    }
-    byFile.get(file)!.push(result);
-  }
-
-  // Build content from grouped results
-  for (const [file, chunks] of byFile) {
-    // Sort chunks by score within each file
-    chunks.sort((a, b) => b.score - a.score);
-
-    for (const chunk of chunks) {
-      const chunkContent = formatChunk(chunk);
-      const chunkChars = chunkContent.length;
-
-      if (totalChars + chunkChars > maxChars) {
-        // Check if we have at least some content
-        if (includedChunks.length > 0) {
-          break;
-        }
-        // Truncate if this is the first chunk
-        const truncated = chunkContent.substring(0, maxChars - totalChars - 50);
-        content += truncated + '\n\n...(truncated)';
-        sources.add(file);
-        includedChunks.push(chunk);
-        break;
-      }
-
-      content += chunkContent + '\n\n';
-      totalChars += chunkChars;
-      sources.add(file);
-      includedChunks.push(chunk);
-    }
-
-    if (totalChars >= maxChars) break;
-  }
-
-  // Add sources section
-  const sourcesList = Array.from(sources);
-  if (sourcesList.length > 0) {
-    content += '---\nSources:\n';
-    for (const source of sourcesList) {
-      content += `- ${source}\n`;
-    }
-  }
-
-  return {
-    content: content.trim(),
-    sources: sourcesList,
-    chunks: includedChunks,
-  };
-}
-
-/**
- * Format a single chunk for display
- */
-function formatChunk(result: SearchResult): string {
-  const { metadata } = result;
-  let formatted = '';
-
-  // Add title if available
-  if (metadata.title) {
-    const headingLevel = metadata.type === 'code' ? '###' : '##';
-    formatted += `${headingLevel} ${metadata.title}\n\n`;
-  }
-
-  // Add content
-  formatted += metadata.content;
-
-  return formatted;
-}
-
-/**
- * Search and format results as JSON
- */
-export async function searchAsJson(
-  indexName: string,
-  query: string,
-  options: {
-    topK?: number;
-    maxTokens?: number;
-  } = {}
-): Promise<{
-  query: string;
-  results: Array<{
-    title: string;
-    content: string;
-    filePath: string;
-    type: string;
-    score: number;
-  }>;
-  totalResults: number;
-}> {
-  const { topK = DEFAULT_TOP_K, maxTokens = DEFAULT_MAX_TOKENS } = options;
-
-  // Generate embedding for the query
-  const queryEmbedding = await generateEmbedding(query);
-
-  // Query vectors
-  const results = await queryVectors(indexName, queryEmbedding, topK);
-
-  // Deduplicate
-  const unique = deduplicateResults(results);
-
-  // Format as JSON
-  return {
-    query,
-    results: unique.slice(0, topK).map((r) => ({
-      title: r.metadata.title,
-      content: r.metadata.content,
-      filePath: r.metadata.filePath,
-      type: r.metadata.type,
-      score: r.score,
-    })),
-    totalResults: unique.length,
-  };
-}

package/skills/apidocs/src/lib/storage.ts

@@ -1,242 +0,0 @@
-import { readFile, writeFile, readdir, rm, mkdir } from 'fs/promises';
-import { existsSync } from 'fs';
-import type { LibraryMetadata, Chunk, ApidocsConfig, CrawledPage, APIEndpoint } from '../types/index.js';
-import {
-  SERVICE_DIR,
-  LIBRARIES_DIR,
-  CONFIG_FILE,
-  getLibraryDir,
-  getLibraryMetadataPath,
-  getLibraryCacheDir,
-  ensureDir,
-} from '../utils/paths.js';
-
-/**
- * Initialize storage directories
- */
-export async function initStorage(): Promise<void> {
-  await ensureDir(SERVICE_DIR);
-  await ensureDir(LIBRARIES_DIR);
-}
-
-/**
- * Get global config
- */
-export async function getConfig(): Promise<Record<string, unknown>> {
-  try {
-    const content = await readFile(CONFIG_FILE, 'utf-8');
-    return JSON.parse(content);
-  } catch {
-    return {};
-  }
-}
-
-/**
- * Save global config
- */
-export async function saveConfig(config: Record<string, unknown>): Promise<void> {
-  await ensureDir(SERVICE_DIR);
-  await writeFile(CONFIG_FILE, JSON.stringify(config, null, 2));
-}
-
-/**
- * Get library metadata
- */
-export async function getLibraryMetadata(libraryId: string): Promise<LibraryMetadata | null> {
-  const metadataPath = getLibraryMetadataPath(libraryId);
-  try {
-    const content = await readFile(metadataPath, 'utf-8');
-    return JSON.parse(content);
-  } catch {
-    return null;
-  }
-}
-
-/**
- * Save library metadata
- */
-export async function saveLibraryMetadata(libraryId: string, metadata: LibraryMetadata): Promise<void> {
-  const libraryDir = getLibraryDir(libraryId);
-  await ensureDir(libraryDir);
-  const metadataPath = getLibraryMetadataPath(libraryId);
-  await writeFile(metadataPath, JSON.stringify(metadata, null, 2));
-}
-
-/**
- * Delete library metadata and cache
- */
-export async function deleteLibrary(libraryId: string): Promise<void> {
-  const libraryDir = getLibraryDir(libraryId);
-  if (existsSync(libraryDir)) {
-    await rm(libraryDir, { recursive: true });
-  }
-}
-
-/**
- * List all indexed libraries
- */
-export async function listLibraries(): Promise<LibraryMetadata[]> {
-  await initStorage();
-
-  const libraries: LibraryMetadata[] = [];
-
-  try {
-    const dirs = await readdir(LIBRARIES_DIR);
-    for (const dir of dirs) {
-      // Try to load metadata using the directory name as ID
-      const metadata = await getLibraryMetadata(dir);
-      if (metadata) {
-        libraries.push(metadata);
-      }
-    }
-  } catch {
-    // Directory doesn't exist yet
-  }
-
-  return libraries;
-}
-
-/**
- * Find library by name (partial match)
- */
-export async function findLibrary(name: string): Promise<LibraryMetadata | null> {
-  const libraries = await listLibraries();
-  const searchLower = name.toLowerCase();
-
-  // Try exact match on id or name first
-  let match = libraries.find(
-    (lib) =>
-      lib.id === name ||
-      lib.name === name ||
-      lib.id?.toLowerCase() === searchLower ||
-      lib.name?.toLowerCase() === searchLower
-  );
-
-  if (match) return match;
-
-  // Try partial match on name or domain
-  match = libraries.find(
-    (lib) =>
-      lib.name?.toLowerCase().includes(searchLower) ||
-      lib.domain?.toLowerCase().includes(searchLower) ||
-      lib.id?.toLowerCase().includes(searchLower)
-  );
-
-  return match || null;
-}
-
-/**
- * Save crawled pages to cache
- */
-export async function saveCrawledPages(libraryId: string, pages: CrawledPage[]): Promise<void> {
-  const cacheDir = getLibraryCacheDir(libraryId);
-  await ensureDir(cacheDir);
-
-  // Save pages in batches of 50
-  const batchSize = 50;
-  for (let i = 0; i < pages.length; i += batchSize) {
-    const batch = pages.slice(i, i + batchSize);
-    const batchFile = `${cacheDir}/pages-${Math.floor(i / batchSize)}.json`;
-    await writeFile(batchFile, JSON.stringify(batch, null, 2));
-  }
-}
-
-/**
- * Load cached crawled pages
- */
-export async function loadCrawledPages(libraryId: string): Promise<CrawledPage[]> {
-  const cacheDir = getLibraryCacheDir(libraryId);
-  const pages: CrawledPage[] = [];
-
-  try {
-    const files = await readdir(cacheDir);
-    for (const file of files.filter((f) => f.startsWith('pages-'))) {
-      const content = await readFile(`${cacheDir}/${file}`, 'utf-8');
-      pages.push(...JSON.parse(content));
-    }
-  } catch {
-    // No cache
-  }
-
-  return pages;
-}
-
-/**
- * Cache chunks locally
- */
-export async function cacheChunks(libraryId: string, chunks: Chunk[]): Promise<void> {
-  const cacheDir = getLibraryCacheDir(libraryId);
-  await ensureDir(cacheDir);
-
-  // Save chunks in batches of 100
-  const batchSize = 100;
-  for (let i = 0; i < chunks.length; i += batchSize) {
-    const batch = chunks.slice(i, i + batchSize);
-    const batchFile = `${cacheDir}/chunks-${Math.floor(i / batchSize)}.json`;
-    await writeFile(batchFile, JSON.stringify(batch, null, 2));
-  }
-}
-
-/**
- * Load cached chunks
- */
-export async function loadCachedChunks(libraryId: string): Promise<Chunk[]> {
-  const cacheDir = getLibraryCacheDir(libraryId);
-  const chunks: Chunk[] = [];
-
-  try {
-    const files = await readdir(cacheDir);
-    for (const file of files.filter((f) => f.startsWith('chunks-'))) {
-      const content = await readFile(`${cacheDir}/${file}`, 'utf-8');
-      chunks.push(...JSON.parse(content));
-    }
-  } catch {
-    // No cache
-  }
-
-  return chunks;
-}
-
-/**
- * Save apidocs config
- */
-export async function saveApidocsConfig(libraryId: string, config: ApidocsConfig): Promise<void> {
-  const libraryDir = getLibraryDir(libraryId);
-  await ensureDir(libraryDir);
-  await writeFile(`${libraryDir}/apidocs.json`, JSON.stringify(config, null, 2));
-}
-
-/**
- * Load apidocs config
- */
-export async function loadApidocsConfig(libraryId: string): Promise<ApidocsConfig | null> {
-  const libraryDir = getLibraryDir(libraryId);
-  try {
-    const content = await readFile(`${libraryDir}/apidocs.json`, 'utf-8');
-    return JSON.parse(content);
-  } catch {
-    return null;
-  }
-}
-
-/**
- * Save extracted API endpoints
- */
-export async function saveEndpoints(libraryId: string, endpoints: APIEndpoint[]): Promise<void> {
-  const libraryDir = getLibraryDir(libraryId);
-  await ensureDir(libraryDir);
-  await writeFile(`${libraryDir}/endpoints.json`, JSON.stringify(endpoints, null, 2));
-}
-
-/**
- * Load extracted API endpoints
- */
-export async function loadEndpoints(libraryId: string): Promise<APIEndpoint[]> {
-  const libraryDir = getLibraryDir(libraryId);
-  try {
-    const content = await readFile(`${libraryDir}/endpoints.json`, 'utf-8');
-    return JSON.parse(content);
-  } catch {
-    return [];
-  }
-}