@hasna/skills 0.0.2

package/skills/skill-apidocs/src/lib/parser.ts

@@ -0,0 +1,288 @@
+import { unified } from 'unified';
+import remarkParse from 'remark-parse';
+import remarkGfm from 'remark-gfm';
+import remarkMdx from 'remark-mdx';
+import { visit } from 'unist-util-visit';
+import type { Root, Heading, Code, Text, Paragraph } from 'mdast';
+import type { Chunk } from '../types/index.js';
+import { createHash } from 'crypto';
+
+const MAX_CHUNK_TOKENS = 500;
+const CHARS_PER_TOKEN = 4; // Rough estimate
+
+/**
+ * Create a unique chunk ID
+ */
+function createChunkId(filePath: string, index: number, content: string): string {
+  const hash = createHash('md5').update(`${filePath}:${index}:${content}`).digest('hex').slice(0, 8);
+  return `${filePath.replace(/[\/\.]/g, '-')}-${index}-${hash}`;
+}
+
+/**
+ * Estimate token count for text
+ */
+function estimateTokens(text: string): number {
+  return Math.ceil(text.length / CHARS_PER_TOKEN);
+}
+
+/**
+ * Extract text content from AST node
+ */
+function extractText(node: unknown): string {
+  if (!node || typeof node !== 'object') return '';
+
+  const n = node as Record<string, unknown>;
+
+  if (n.type === 'text' && typeof n.value === 'string') {
+    return n.value;
+  }
+
+  if (n.type === 'code' && typeof n.value === 'string') {
+    return n.value;
+  }
+
+  if (Array.isArray(n.children)) {
+    return n.children.map((child) => extractText(child)).join('');
+  }
+
+  return '';
+}
+
+/**
+ * Get heading text
+ */
+function getHeadingText(node: Heading): string {
+  return extractText(node);
+}
+
+/**
+ * Parse markdown content into chunks
+ */
+export function parseMarkdown(content: string, filePath: string): Chunk[] {
+  const processor = unified()
+    .use(remarkParse)
+    .use(remarkGfm)
+    .use(remarkMdx);
+
+  let ast: Root;
+  try {
+    ast = processor.parse(content);
+  } catch {
+    // If MDX parsing fails, try plain markdown
+    ast = unified().use(remarkParse).use(remarkGfm).parse(content);
+  }
+
+  return chunkByHeadings(ast, filePath, MAX_CHUNK_TOKENS);
+}
+
+/**
+ * Chunk content by headings
+ */
+function chunkByHeadings(ast: Root, filePath: string, maxTokens: number): Chunk[] {
+  const chunks: Chunk[] = [];
+  const headingStack: { level: number; text: string }[] = [];
+
+  let currentContent: string[] = [];
+  let currentTitle = '';
+  let chunkIndex = 0;
+
+  // Get title from first heading or filename
+  let documentTitle = filePath.split('/').pop()?.replace(/\.(md|mdx)$/, '') || 'Document';
+
+  function flushChunk(type: 'text' | 'code' = 'text', codeLanguage?: string): void {
+    const content = currentContent.join('\n\n').trim();
+    if (!content) return;
+
+    const tokens = estimateTokens(content);
+
+    // If chunk is too large, split it
+    if (tokens > maxTokens) {
+      const subChunks = splitLargeChunk(content, maxTokens);
+      for (const subContent of subChunks) {
+        chunks.push({
+          id: createChunkId(filePath, chunkIndex, subContent),
+          content: subContent,
+          title: currentTitle || documentTitle,
+          type,
+          filePath,
+          headingHierarchy: headingStack.map((h) => h.text),
+          codeLanguage,
+          tokenCount: estimateTokens(subContent),
+        });
+        chunkIndex++;
+      }
+    } else {
+      chunks.push({
+        id: createChunkId(filePath, chunkIndex, content),
+        content,
+        title: currentTitle || documentTitle,
+        type,
+        filePath,
+        headingHierarchy: headingStack.map((h) => h.text),
+        codeLanguage,
+        tokenCount: tokens,
+      });
+      chunkIndex++;
+    }
+
+    currentContent = [];
+  }
+
+  visit(ast, (node) => {
+    if (node.type === 'heading') {
+      const heading = node as Heading;
+      const text = getHeadingText(heading);
+
+      // Set document title from first h1
+      if (heading.depth === 1 && !documentTitle) {
+        documentTitle = text;
+      }
+
+      // Flush previous content
+      if (currentContent.length > 0) {
+        flushChunk();
+      }
+
+      // Update heading stack
+      while (headingStack.length > 0 && headingStack[headingStack.length - 1].level >= heading.depth) {
+        headingStack.pop();
+      }
+      headingStack.push({ level: heading.depth, text });
+      currentTitle = text;
+
+      // Add heading to content
+      currentContent.push('#'.repeat(heading.depth) + ' ' + text);
+    } else if (node.type === 'code') {
+      const code = node as Code;
+      // Flush any previous text content
+      if (currentContent.length > 0) {
+        flushChunk();
+      }
+
+      // Create separate chunk for code
+      const codeContent = code.lang ? `\`\`\`${code.lang}\n${code.value}\n\`\`\`` : `\`\`\`\n${code.value}\n\`\`\``;
+
+      currentContent.push(codeContent);
+      flushChunk('code', code.lang || undefined);
+    } else if (node.type === 'paragraph') {
+      const text = extractText(node);
+      if (text.trim()) {
+        currentContent.push(text);
+      }
+    } else if (node.type === 'list') {
+      const text = stringifyList(node);
+      if (text.trim()) {
+        currentContent.push(text);
+      }
+    } else if (node.type === 'blockquote') {
+      const text = '> ' + extractText(node).split('\n').join('\n> ');
+      if (text.trim()) {
+        currentContent.push(text);
+      }
+    } else if (node.type === 'table') {
+      const text = stringifyTable(node);
+      if (text.trim()) {
+        currentContent.push(text);
+      }
+    }
+  });
+
+  // Flush remaining content
+  if (currentContent.length > 0) {
+    flushChunk();
+  }
+
+  return chunks;
+}
+
+/**
+ * Split a large chunk into smaller pieces
+ */
+function splitLargeChunk(content: string, maxTokens: number): string[] {
+  const chunks: string[] = [];
+  const lines = content.split('\n');
+  let current: string[] = [];
+  let currentTokens = 0;
+
+  for (const line of lines) {
+    const lineTokens = estimateTokens(line);
+
+    if (currentTokens + lineTokens > maxTokens && current.length > 0) {
+      chunks.push(current.join('\n'));
+      current = [];
+      currentTokens = 0;
+    }
+
+    current.push(line);
+    currentTokens += lineTokens;
+  }
+
+  if (current.length > 0) {
+    chunks.push(current.join('\n'));
+  }
+
+  return chunks;
+}
+
+/**
+ * Stringify list node
+ */
+function stringifyList(node: unknown): string {
+  const n = node as Record<string, unknown>;
+  if (!Array.isArray(n.children)) return '';
+
+  const ordered = n.ordered === true;
+  return n.children
+    .map((item, i) => {
+      const prefix = ordered ? `${i + 1}. ` : '- ';
+      const text = extractText(item);
+      return prefix + text;
+    })
+    .join('\n');
+}
+
+/**
+ * Stringify table node
+ */
+function stringifyTable(node: unknown): string {
+  const n = node as Record<string, unknown>;
+  if (!Array.isArray(n.children)) return '';
+
+  const rows = n.children.map((row) => {
+    const r = row as Record<string, unknown>;
+    if (!Array.isArray(r.children)) return '';
+    return '| ' + r.children.map((cell) => extractText(cell)).join(' | ') + ' |';
+  });
+
+  if (rows.length > 0) {
+    // Add header separator after first row
+    const headerCells = (n.children[0] as Record<string, unknown>).children as unknown[];
+    const separator = '| ' + headerCells.map(() => '---').join(' | ') + ' |';
+    rows.splice(1, 0, separator);
+  }
+
+  return rows.join('\n');
+}
+
+/**
+ * Extract all code blocks from content
+ */
+export function extractCodeBlocks(
+  content: string,
+  filePath: string
+): { content: string; language: string; filePath: string }[] {
+  const blocks: { content: string; language: string; filePath: string }[] = [];
+
+  const processor = unified().use(remarkParse).use(remarkGfm);
+  const ast = processor.parse(content);
+
+  visit(ast, 'code', (node: Code) => {
+    blocks.push({
+      content: node.value,
+      language: node.lang || '',
+      filePath,
+    });
+  });
+
+  return blocks;
+}

package/skills/skill-apidocs/src/lib/search.ts

@@ -0,0 +1,189 @@
+import type { SearchResult, QueryResult, VectorMetadata } from '../types/index.js';
+import { generateEmbedding } from './embeddings.js';
+import { queryVectors } from './vectors.js';
+
+const DEFAULT_TOP_K = 10;
+const DEFAULT_MAX_TOKENS = 8000;
+const CHARS_PER_TOKEN = 4;
+
+/**
+ * Perform semantic search on an index
+ */
+export async function semanticSearch(
+  indexName: string,
+  query: string,
+  options: {
+    topK?: number;
+    maxTokens?: number;
+  } = {}
+): Promise<QueryResult> {
+  const { topK = DEFAULT_TOP_K, maxTokens = DEFAULT_MAX_TOKENS } = options;
+
+  // Generate embedding for the query
+  const queryEmbedding = await generateEmbedding(query);
+
+  // Query vectors
+  const results = await queryVectors(indexName, queryEmbedding, topK * 2); // Get more to filter
+
+  // Deduplicate and rank results
+  const uniqueChunks = deduplicateResults(results);
+
+  // Build response within token limit
+  const { content, sources, chunks } = buildResponse(uniqueChunks, maxTokens);
+
+  return { content, sources, chunks };
+}
+
+/**
+ * Deduplicate results by content similarity
+ */
+function deduplicateResults(results: SearchResult[]): SearchResult[] {
+  const seen = new Set<string>();
+  const unique: SearchResult[] = [];
+
+  for (const result of results) {
+    const contentKey = result.metadata.content?.substring(0, 100);
+    if (!seen.has(contentKey)) {
+      seen.add(contentKey);
+      unique.push(result);
+    }
+  }
+
+  return unique;
+}
+
+/**
+ * Build formatted response from search results
+ */
+function buildResponse(
+  results: SearchResult[],
+  maxTokens: number
+): { content: string; sources: string[]; chunks: SearchResult[] } {
+  const maxChars = maxTokens * CHARS_PER_TOKEN;
+  const sources = new Set<string>();
+  const includedChunks: SearchResult[] = [];
+
+  let content = '';
+  let totalChars = 0;
+
+  // Group results by file for better organization
+  const byFile = new Map<string, SearchResult[]>();
+  for (const result of results) {
+    const file = result.metadata.filePath;
+    if (!byFile.has(file)) {
+      byFile.set(file, []);
+    }
+    byFile.get(file)!.push(result);
+  }
+
+  // Build content from grouped results
+  for (const [file, chunks] of byFile) {
+    // Sort chunks by score within each file
+    chunks.sort((a, b) => b.score - a.score);
+
+    for (const chunk of chunks) {
+      const chunkContent = formatChunk(chunk);
+      const chunkChars = chunkContent.length;
+
+      if (totalChars + chunkChars > maxChars) {
+        // Check if we have at least some content
+        if (includedChunks.length > 0) {
+          break;
+        }
+        // Truncate if this is the first chunk
+        const truncated = chunkContent.substring(0, maxChars - totalChars - 50);
+        content += truncated + '\n\n...(truncated)';
+        sources.add(file);
+        includedChunks.push(chunk);
+        break;
+      }
+
+      content += chunkContent + '\n\n';
+      totalChars += chunkChars;
+      sources.add(file);
+      includedChunks.push(chunk);
+    }
+
+    if (totalChars >= maxChars) break;
+  }
+
+  // Add sources section
+  const sourcesList = Array.from(sources);
+  if (sourcesList.length > 0) {
+    content += '---\nSources:\n';
+    for (const source of sourcesList) {
+      content += `- ${source}\n`;
+    }
+  }
+
+  return {
+    content: content.trim(),
+    sources: sourcesList,
+    chunks: includedChunks,
+  };
+}
+
+/**
+ * Format a single chunk for display
+ */
+function formatChunk(result: SearchResult): string {
+  const { metadata } = result;
+  let formatted = '';
+
+  // Add title if available
+  if (metadata.title) {
+    const headingLevel = metadata.type === 'code' ? '###' : '##';
+    formatted += `${headingLevel} ${metadata.title}\n\n`;
+  }
+
+  // Add content
+  formatted += metadata.content;
+
+  return formatted;
+}
+
+/**
+ * Search and format results as JSON
+ */
+export async function searchAsJson(
+  indexName: string,
+  query: string,
+  options: {
+    topK?: number;
+    maxTokens?: number;
+  } = {}
+): Promise<{
+  query: string;
+  results: Array<{
+    title: string;
+    content: string;
+    filePath: string;
+    type: string;
+    score: number;
+  }>;
+  totalResults: number;
+}> {
+  const { topK = DEFAULT_TOP_K, maxTokens = DEFAULT_MAX_TOKENS } = options;
+
+  // Generate embedding for the query
+  const queryEmbedding = await generateEmbedding(query);
+
+  // Query vectors
+  const results = await queryVectors(indexName, queryEmbedding, topK);
+
+  // Deduplicate
+  const unique = deduplicateResults(results);
+
+  // Format as JSON
+  return {
+    query,
+    results: unique.slice(0, topK).map((r) => ({
+      title: r.metadata.title,
+      content: r.metadata.content,
+      filePath: r.metadata.filePath,
+      type: r.metadata.type,
+      score: r.score,
+    })),
+    totalResults: unique.length,
+  };
+}

package/skills/skill-apidocs/src/lib/storage.ts

@@ -0,0 +1,242 @@
+import { readFile, writeFile, readdir, rm, mkdir } from 'fs/promises';
+import { existsSync } from 'fs';
+import type { LibraryMetadata, Chunk, ApidocsConfig, CrawledPage, APIEndpoint } from '../types/index.js';
+import {
+  SERVICE_DIR,
+  LIBRARIES_DIR,
+  CONFIG_FILE,
+  getLibraryDir,
+  getLibraryMetadataPath,
+  getLibraryCacheDir,
+  ensureDir,
+} from '../utils/paths.js';
+
+/**
+ * Initialize storage directories
+ */
+export async function initStorage(): Promise<void> {
+  await ensureDir(SERVICE_DIR);
+  await ensureDir(LIBRARIES_DIR);
+}
+
+/**
+ * Get global config
+ */
+export async function getConfig(): Promise<Record<string, unknown>> {
+  try {
+    const content = await readFile(CONFIG_FILE, 'utf-8');
+    return JSON.parse(content);
+  } catch {
+    return {};
+  }
+}
+
+/**
+ * Save global config
+ */
+export async function saveConfig(config: Record<string, unknown>): Promise<void> {
+  await ensureDir(SERVICE_DIR);
+  await writeFile(CONFIG_FILE, JSON.stringify(config, null, 2));
+}
+
+/**
+ * Get library metadata
+ */
+export async function getLibraryMetadata(libraryId: string): Promise<LibraryMetadata | null> {
+  const metadataPath = getLibraryMetadataPath(libraryId);
+  try {
+    const content = await readFile(metadataPath, 'utf-8');
+    return JSON.parse(content);
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Save library metadata
+ */
+export async function saveLibraryMetadata(libraryId: string, metadata: LibraryMetadata): Promise<void> {
+  const libraryDir = getLibraryDir(libraryId);
+  await ensureDir(libraryDir);
+  const metadataPath = getLibraryMetadataPath(libraryId);
+  await writeFile(metadataPath, JSON.stringify(metadata, null, 2));
+}
+
+/**
+ * Delete library metadata and cache
+ */
+export async function deleteLibrary(libraryId: string): Promise<void> {
+  const libraryDir = getLibraryDir(libraryId);
+  if (existsSync(libraryDir)) {
+    await rm(libraryDir, { recursive: true });
+  }
+}
+
+/**
+ * List all indexed libraries
+ */
+export async function listLibraries(): Promise<LibraryMetadata[]> {
+  await initStorage();
+
+  const libraries: LibraryMetadata[] = [];
+
+  try {
+    const dirs = await readdir(LIBRARIES_DIR);
+    for (const dir of dirs) {
+      // Try to load metadata using the directory name as ID
+      const metadata = await getLibraryMetadata(dir);
+      if (metadata) {
+        libraries.push(metadata);
+      }
+    }
+  } catch {
+    // Directory doesn't exist yet
+  }
+
+  return libraries;
+}
+
+/**
+ * Find library by name (partial match)
+ */
+export async function findLibrary(name: string): Promise<LibraryMetadata | null> {
+  const libraries = await listLibraries();
+  const searchLower = name.toLowerCase();
+
+  // Try exact match on id or name first
+  let match = libraries.find(
+    (lib) =>
+      lib.id === name ||
+      lib.name === name ||
+      lib.id?.toLowerCase() === searchLower ||
+      lib.name?.toLowerCase() === searchLower
+  );
+
+  if (match) return match;
+
+  // Try partial match on name or domain
+  match = libraries.find(
+    (lib) =>
+      lib.name?.toLowerCase().includes(searchLower) ||
+      lib.domain?.toLowerCase().includes(searchLower) ||
+      lib.id?.toLowerCase().includes(searchLower)
+  );
+
+  return match || null;
+}
+
+/**
+ * Save crawled pages to cache
+ */
+export async function saveCrawledPages(libraryId: string, pages: CrawledPage[]): Promise<void> {
+  const cacheDir = getLibraryCacheDir(libraryId);
+  await ensureDir(cacheDir);
+
+  // Save pages in batches of 50
+  const batchSize = 50;
+  for (let i = 0; i < pages.length; i += batchSize) {
+    const batch = pages.slice(i, i + batchSize);
+    const batchFile = `${cacheDir}/pages-${Math.floor(i / batchSize)}.json`;
+    await writeFile(batchFile, JSON.stringify(batch, null, 2));
+  }
+}
+
+/**
+ * Load cached crawled pages
+ */
+export async function loadCrawledPages(libraryId: string): Promise<CrawledPage[]> {
+  const cacheDir = getLibraryCacheDir(libraryId);
+  const pages: CrawledPage[] = [];
+
+  try {
+    const files = await readdir(cacheDir);
+    for (const file of files.filter((f) => f.startsWith('pages-'))) {
+      const content = await readFile(`${cacheDir}/${file}`, 'utf-8');
+      pages.push(...JSON.parse(content));
+    }
+  } catch {
+    // No cache
+  }
+
+  return pages;
+}
+
+/**
+ * Cache chunks locally
+ */
+export async function cacheChunks(libraryId: string, chunks: Chunk[]): Promise<void> {
+  const cacheDir = getLibraryCacheDir(libraryId);
+  await ensureDir(cacheDir);
+
+  // Save chunks in batches of 100
+  const batchSize = 100;
+  for (let i = 0; i < chunks.length; i += batchSize) {
+    const batch = chunks.slice(i, i + batchSize);
+    const batchFile = `${cacheDir}/chunks-${Math.floor(i / batchSize)}.json`;
+    await writeFile(batchFile, JSON.stringify(batch, null, 2));
+  }
+}
+
+/**
+ * Load cached chunks
+ */
+export async function loadCachedChunks(libraryId: string): Promise<Chunk[]> {
+  const cacheDir = getLibraryCacheDir(libraryId);
+  const chunks: Chunk[] = [];
+
+  try {
+    const files = await readdir(cacheDir);
+    for (const file of files.filter((f) => f.startsWith('chunks-'))) {
+      const content = await readFile(`${cacheDir}/${file}`, 'utf-8');
+      chunks.push(...JSON.parse(content));
+    }
+  } catch {
+    // No cache
+  }
+
+  return chunks;
+}
+
+/**
+ * Save apidocs config
+ */
+export async function saveApidocsConfig(libraryId: string, config: ApidocsConfig): Promise<void> {
+  const libraryDir = getLibraryDir(libraryId);
+  await ensureDir(libraryDir);
+  await writeFile(`${libraryDir}/apidocs.json`, JSON.stringify(config, null, 2));
+}
+
+/**
+ * Load apidocs config
+ */
+export async function loadApidocsConfig(libraryId: string): Promise<ApidocsConfig | null> {
+  const libraryDir = getLibraryDir(libraryId);
+  try {
+    const content = await readFile(`${libraryDir}/apidocs.json`, 'utf-8');
+    return JSON.parse(content);
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Save extracted API endpoints
+ */
+export async function saveEndpoints(libraryId: string, endpoints: APIEndpoint[]): Promise<void> {
+  const libraryDir = getLibraryDir(libraryId);
+  await ensureDir(libraryDir);
+  await writeFile(`${libraryDir}/endpoints.json`, JSON.stringify(endpoints, null, 2));
+}
+
+/**
+ * Load extracted API endpoints
+ */
+export async function loadEndpoints(libraryId: string): Promise<APIEndpoint[]> {
+  const libraryDir = getLibraryDir(libraryId);
+  try {
+    const content = await readFile(`${libraryDir}/endpoints.json`, 'utf-8');
+    return JSON.parse(content);
+  } catch {
+    return [];
+  }
+}