docshark 0.1.5 → 0.1.7

package/dist/http.js

@@ -0,0 +1,84 @@
+// src/http.ts — HTTP server (MCP + REST API + SSE + static dashboard)
+import { serve } from 'srvx';
+import { HttpTransport } from '@tmcp/transport-http';
+import { SseTransport } from '@tmcp/transport-sse';
+import { server, eventBus, db, searchEngine, jobManager, libraryService } from './server.js';
+import { createApiRouter } from './api/router.js';
+import { VERSION } from './version.js';
+export async function startHttpServer(port) {
+    const httpTransport = new HttpTransport(server, { path: '/mcp' });
+    const sseTransport = new SseTransport(server, { path: '/sse' });
+    const apiRouter = createApiRouter({ db, searchEngine, jobManager, libraryService, eventBus });
+    const httpServer = serve({
+        port,
+        async fetch(request) {
+            const url = new URL(request.url);
+            // 1. MCP Streamable HTTP transport
+            if (url.pathname.startsWith('/mcp')) {
+                const response = await httpTransport.respond(request);
+                if (response)
+                    return response;
+            }
+            // 2. MCP SSE transport
+            if (url.pathname.startsWith('/sse')) {
+                const response = await sseTransport.respond(request);
+                if (response)
+                    return response;
+            }
+            // 3. SSE endpoint for real-time crawl events
+            if (url.pathname === '/api/crawl-events') {
+                return handleCrawlSSE(request);
+            }
+            // 4. REST API for dashboard
+            if (url.pathname.startsWith('/api/')) {
+                return apiRouter.handle(request);
+            }
+            // 5. Root — server info
+            return new Response(JSON.stringify({
+                name: 'DocShark',
+                version: VERSION,
+                description: '🦈 Documentation MCP Server',
+                endpoints: {
+                    mcp: '/mcp',
+                    sse: '/sse',
+                    api: '/api',
+                    crawlEvents: '/api/crawl-events',
+                },
+            }), { headers: { 'Content-Type': 'application/json' } });
+        },
+    });
+    console.log(`\n🦈 DocShark running on http://localhost:${port}`);
+    console.log(`   MCP (HTTP):  http://localhost:${port}/mcp`);
+    console.log(`   MCP (SSE):   http://localhost:${port}/sse`);
+    console.log(`   REST API:    http://localhost:${port}/api`);
+    console.log(`   Health:      http://localhost:${port}/api/health\n`);
+    return httpServer;
+}
+/** SSE handler for real-time crawl progress */
+function handleCrawlSSE(request) {
+    const stream = new ReadableStream({
+        start(controller) {
+            const encoder = new TextEncoder();
+            const onProgress = (data) => {
+                controller.enqueue(encoder.encode(`data: ${JSON.stringify(data)}\n\n`));
+            };
+            eventBus.on('crawl:progress', onProgress);
+            eventBus.on('crawl:complete', onProgress);
+            eventBus.on('crawl:error', onProgress);
+            request.signal.addEventListener('abort', () => {
+                eventBus.off('crawl:progress', onProgress);
+                eventBus.off('crawl:complete', onProgress);
+                eventBus.off('crawl:error', onProgress);
+                controller.close();
+            });
+        },
+    });
+    return new Response(stream, {
+        headers: {
+            'Content-Type': 'text/event-stream',
+            'Cache-Control': 'no-cache',
+            Connection: 'keep-alive',
+            'Access-Control-Allow-Origin': '*',
+        },
+    });
+}

package/dist/index.js

@@ -1,4 +1,3 @@
-// src/index.ts
 export * from "./server.js";
 export * from "./types.js";
 export * from "./version.js";

package/dist/jobs/events.js

@@ -0,0 +1,15 @@
+// src/jobs/events.ts — EventBus for real-time crawl progress (SSE)
+export class EventBus {
+    listeners = new Map();
+    on(event, listener) {
+        if (!this.listeners.has(event))
+            this.listeners.set(event, new Set());
+        this.listeners.get(event).add(listener);
+    }
+    off(event, listener) {
+        this.listeners.get(event)?.delete(listener);
+    }
+    emit(event, data) {
+        this.listeners.get(event)?.forEach((fn) => fn(data));
+    }
+}

package/dist/jobs/manager.js

@@ -0,0 +1,49 @@
+// src/jobs/manager.ts — Crawl job queue manager
+import { nanoid } from 'nanoid';
+import { CrawlWorker } from './worker.js';
+export class JobManager {
+    db;
+    eventBus;
+    activeJobs = new Map();
+    constructor(db, eventBus) {
+        this.db = db;
+        this.eventBus = eventBus;
+    }
+    /** Start a crawl job for a library */
+    startCrawl(libraryId, opts) {
+        const jobId = nanoid();
+        const job = this.db.createJob({ id: jobId, libraryId });
+        // Run crawl async (non-blocking)
+        const worker = new CrawlWorker(this.db, this.eventBus);
+        this.activeJobs.set(jobId, worker);
+        // Fire and forget — runs in the background
+        worker
+            .crawl(libraryId, jobId)
+            .catch((err) => {
+            console.error(`[DocShark] Crawl job ${jobId} failed:`, err);
+        })
+            .finally(() => {
+            this.activeJobs.delete(jobId);
+        });
+        return job;
+    }
+    /** Get status of a specific job */
+    getJob(jobId) {
+        return this.db.getJob(jobId);
+    }
+    /** List all jobs, optionally filtered by library */
+    listJobs(libraryId) {
+        return this.db.listJobs(libraryId);
+    }
+    /** Check if a crawl is currently running for a library */
+    isRunning(libraryId) {
+        for (const [, worker] of this.activeJobs) {
+            // Check by iterating active jobs
+            const jobs = this.db.listJobs(libraryId);
+            if (jobs.some((j) => j.status === 'running' || j.status === 'queued')) {
+                return true;
+            }
+        }
+        return false;
+    }
+}

package/dist/jobs/worker.js

@@ -0,0 +1,120 @@
+// src/jobs/worker.ts — Crawl execution pipeline
+import { nanoid } from 'nanoid';
+import { createHash } from 'crypto';
+import { discoverPages } from '../scraper/discoverer.js';
+import { fetchPage } from '../scraper/fetcher.js';
+import { extractAndConvert } from '../processor/extractor.js';
+import { chunkMarkdown } from '../processor/chunker.js';
+import { RateLimiter } from '../scraper/rate-limiter.js';
+export class CrawlWorker {
+    db;
+    eventBus;
+    constructor(db, eventBus) {
+        this.db = db;
+        this.eventBus = eventBus;
+    }
+    async crawl(libraryId, jobId) {
+        const lib = this.db.getLibraryById(libraryId);
+        if (!lib)
+            throw new Error(`Library ${libraryId} not found`);
+        const config = lib.crawl_config ? JSON.parse(lib.crawl_config) : {};
+        this.db.updateLibraryStatus(libraryId, 'crawling');
+        this.db.updateJob(jobId, { status: 'running', started_at: new Date().toISOString() });
+        const rateLimiter = new RateLimiter(config.rateLimit ?? 500);
+        try {
+            // Phase 1: Discover pages
+            const urls = await discoverPages(lib.url, config);
+            this.db.updateJob(jobId, { pages_discovered: urls.length });
+            this.eventBus.emit('crawl:progress', {
+                jobId,
+                libraryId,
+                phase: 'discovering',
+                pagesDiscovered: urls.length,
+            });
+            console.log(`[DocShark] Discovered ${urls.length} pages for "${lib.display_name}"`);
+            let crawled = 0;
+            let failed = 0;
+            let totalChunks = 0;
+            // Phase 2-6: Fetch → Extract → Convert → Chunk → Index
+            for (const url of urls) {
+                try {
+                    await rateLimiter.wait();
+                    const result = await fetchPage(url, config.renderer);
+                    // Extract content + convert to markdown
+                    const { markdown, title, headings } = extractAndConvert(result.html, url);
+                    if (!markdown || markdown.length < 50) {
+                        crawled++;
+                        continue; // Skip essentially empty pages
+                    }
+                    const contentHash = createHash('sha256').update(markdown).digest('hex');
+                    // Store page
+                    const path = new URL(url).pathname;
+                    const pageId = this.db.upsertPage({
+                        id: nanoid(),
+                        libraryId,
+                        url,
+                        path,
+                        title,
+                        contentMarkdown: markdown,
+                        contentHash,
+                        headings,
+                    });
+                    // Delete old chunks for this page (for re-crawls)
+                    this.db.deleteChunksByPage(pageId);
+                    // Chunk and index
+                    const chunks = chunkMarkdown(markdown, headings);
+                    if (chunks.length > 0) {
+                        const chunkRecords = chunks.map((c, i) => ({
+                            id: nanoid(),
+                            pageId,
+                            libraryId,
+                            content: c.content,
+                            headingContext: c.headingContext,
+                            chunkIndex: i,
+                            tokenCount: c.tokenCount,
+                            hasCodeBlock: c.hasCodeBlock,
+                        }));
+                        this.db.insertChunks(chunkRecords);
+                        totalChunks += chunkRecords.length;
+                    }
+                    crawled++;
+                    // Emit progress
+                    this.eventBus.emit('crawl:progress', {
+                        jobId,
+                        libraryId,
+                        phase: 'crawling',
+                        pagesCrawled: crawled,
+                        pagesDiscovered: urls.length,
+                        currentUrl: url,
+                    });
+                    // Log progress every 10 pages
+                    if (crawled % 10 === 0) {
+                        console.log(`[DocShark] Progress: ${crawled}/${urls.length} pages (${totalChunks} chunks)`);
+                    }
+                }
+                catch (err) {
+                    failed++;
+                    console.error(`[DocShark] Failed to crawl ${url}:`, err.message);
+                }
+            }
+            // Update final stats
+            this.db.updateLibraryStats(libraryId, crawled, totalChunks);
+            this.db.updateLibraryStatus(libraryId, 'indexed');
+            this.db.updateJob(jobId, {
+                status: 'completed',
+                pages_crawled: crawled,
+                pages_failed: failed,
+                chunks_created: totalChunks,
+                completed_at: new Date().toISOString(),
+            });
+            this.eventBus.emit('crawl:complete', { jobId, libraryId, crawled, failed, totalChunks });
+            console.log(`[DocShark] ✅ Crawl complete: ${crawled} pages, ${totalChunks} chunks, ${failed} failed`);
+        }
+        catch (err) {
+            this.db.updateLibraryStatus(libraryId, 'error');
+            this.db.updateJob(jobId, { status: 'failed', error_message: err.message });
+            this.eventBus.emit('crawl:error', { jobId, libraryId, error: err.message });
+            console.error(`[DocShark] ❌ Crawl failed for "${lib.display_name}":`, err.message);
+        }
+    }
+}

package/dist/processor/chunker.js

@@ -0,0 +1,79 @@
+// src/processor/chunker.ts — Heading-based semantic chunking
+const MAX_TOKENS = 1200;
+const MIN_TOKENS = 50;
+export function chunkMarkdown(markdown, _headings) {
+    const sections = splitByHeadings(markdown);
+    const chunks = [];
+    for (const section of sections) {
+        const tokens = estimateTokens(section.content);
+        if (tokens < MIN_TOKENS)
+            continue;
+        if (tokens <= MAX_TOKENS) {
+            chunks.push({
+                content: section.content,
+                headingContext: section.headingPath,
+                tokenCount: tokens,
+                hasCodeBlock: section.content.includes('```'),
+            });
+        }
+        else {
+            // Split large sections by paragraphs
+            const paras = splitByParagraphs(section.content);
+            let buffer = '';
+            for (const para of paras) {
+                if (estimateTokens(buffer + para) > MAX_TOKENS && buffer) {
+                    chunks.push({
+                        content: buffer.trim(),
+                        headingContext: section.headingPath,
+                        tokenCount: estimateTokens(buffer),
+                        hasCodeBlock: buffer.includes('```'),
+                    });
+                    buffer = '';
+                }
+                buffer += para + '\n\n';
+            }
+            if (buffer.trim() && estimateTokens(buffer) >= MIN_TOKENS) {
+                chunks.push({
+                    content: buffer.trim(),
+                    headingContext: section.headingPath,
+                    tokenCount: estimateTokens(buffer),
+                    hasCodeBlock: buffer.includes('```'),
+                });
+            }
+        }
+    }
+    return chunks;
+}
+function splitByHeadings(md) {
+    const lines = md.split('\n');
+    const sections = [];
+    const headingStack = [];
+    let currentContent = '';
+    for (const line of lines) {
+        const headingMatch = line.match(/^(#{1,6})\s+(.+)$/);
+        if (headingMatch) {
+            if (currentContent.trim()) {
+                sections.push({ content: currentContent.trim(), headingPath: headingStack.join(' > ') });
+            }
+            const level = headingMatch[1].length;
+            const text = headingMatch[2];
+            while (headingStack.length >= level)
+                headingStack.pop();
+            headingStack.push(text);
+            currentContent = line + '\n';
+        }
+        else {
+            currentContent += line + '\n';
+        }
+    }
+    if (currentContent.trim()) {
+        sections.push({ content: currentContent.trim(), headingPath: headingStack.join(' > ') });
+    }
+    return sections;
+}
+function splitByParagraphs(text) {
+    return text.split(/\n{2,}/).filter(Boolean);
+}
+function estimateTokens(text) {
+    return Math.ceil(text.length / 4);
+}

package/dist/processor/extractor.js

@@ -0,0 +1,81 @@
+// src/processor/extractor.ts — HTML content extraction + Markdown conversion
+import { Readability } from '@mozilla/readability';
+import { parseHTML } from 'linkedom';
+import TurndownService from 'turndown';
+import { gfm } from 'turndown-plugin-gfm';
+const turndown = new TurndownService({
+    headingStyle: 'atx',
+    codeBlockStyle: 'fenced',
+    bulletListMarker: '-',
+});
+turndown.use(gfm);
+turndown.keep(['details', 'summary', 'kbd']);
+// Preserve language attribute on code blocks
+turndown.addRule('fencedCodeBlock', {
+    filter: (node) => node.nodeName === 'PRE' && node.querySelector('code'),
+    replacement: (_content, node) => {
+        const code = node.querySelector('code');
+        const lang = code?.className?.match(/language-(\w+)/)?.[1] || '';
+        const text = code?.textContent || '';
+        return `\n\`\`\`${lang}\n${text.trim()}\n\`\`\`\n`;
+    },
+});
+// Strip images (noisy for search context)
+turndown.addRule('removeImages', {
+    filter: 'img',
+    replacement: () => '',
+});
+export function extractAndConvert(html, url) {
+    const { document } = parseHTML(html);
+    // Set the document URL for Readability to resolve relative links
+    if (url) {
+        try {
+            const baseEl = document.createElement('base');
+            baseEl.setAttribute('href', url);
+            document.head.appendChild(baseEl);
+        }
+        catch {
+            // Ignore if head doesn't exist
+        }
+    }
+    // PRE-PROCESS: Rescue code blocks from aggressive Readability stripping
+    // Many doc sites wrap <pre> blocks in complex UI (like <figure>) that Readability deletes.
+    const pres = Array.from(document.querySelectorAll('pre'));
+    for (const pre of pres) {
+        const codeText = pre.textContent || '';
+        if (!codeText)
+            continue;
+        const lang = pre.getAttribute('data-language') ||
+            pre.className?.match(/language-(\w+)/)?.[1] ||
+            pre.querySelector('code')?.getAttribute('data-language') ||
+            pre.querySelector('code')?.className?.match(/language-(\w+)/)?.[1] ||
+            '';
+        let wrapper = pre;
+        // Walk up to find the container that exclusively holds this code block
+        while (wrapper.parentElement &&
+            !['BODY', 'MAIN', 'ARTICLE', 'SECTION'].includes(wrapper.parentElement.tagName.toUpperCase()) &&
+            wrapper.parentElement.textContent?.trim() === pre.textContent?.trim()) {
+            wrapper = wrapper.parentElement;
+        }
+        const cleanPre = document.createElement('pre');
+        const cleanCode = document.createElement('code');
+        if (lang)
+            cleanCode.className = `language-${lang}`;
+        cleanCode.textContent = codeText;
+        cleanPre.appendChild(cleanCode);
+        wrapper.replaceWith(cleanPre);
+    }
+    const reader = new Readability(document, { charThreshold: 100 });
+    const article = reader.parse();
+    const title = article?.title || document.querySelector('title')?.textContent || '';
+    const contentHtml = article?.content || document.body?.innerHTML || '';
+    const markdown = turndown.turndown(contentHtml).trim();
+    // Extract heading hierarchy from the markdown
+    const headings = [];
+    const headingRegex = /^(#{1,6})\s+(.+)$/gm;
+    let match;
+    while ((match = headingRegex.exec(markdown)) !== null) {
+        headings.push({ level: match[1].length, text: match[2].trim() });
+    }
+    return { markdown, title, headings };
+}

package/dist/scraper/discoverer.js

@@ -0,0 +1,206 @@
+// src/scraper/discoverer.ts — Page URL discovery via sitemap + link crawl
+import * as cheerio from 'cheerio';
+import { getRobotsParser, isAllowed } from './robots.js';
+import { RateLimiter } from './rate-limiter.js';
+const USER_AGENT = 'DocShark/1.0';
+/**
+ * Discover all documentation page URLs from a base URL.
+ * Strategy: sitemap.xml → link crawl fallback
+ */
+export async function discoverPages(baseUrl, config = {}) {
+    const maxDepth = config.maxDepth ?? 3;
+    const robots = await getRobotsParser(baseUrl);
+    // Strategy A: Try sitemap first
+    const sitemapUrls = await discoverFromSitemap(baseUrl, robots);
+    if (sitemapUrls.length > 0) {
+        console.log(`[DocShark] Found ${sitemapUrls.length} URLs from sitemap`);
+        return filterUrls(sitemapUrls, baseUrl, config, robots);
+    }
+    // Strategy B: BFS link crawl
+    console.log(`[DocShark] No sitemap found, crawling links (depth=${maxDepth})`);
+    const crawledUrls = await discoverByLinkCrawl(baseUrl, maxDepth, config, robots);
+    return crawledUrls;
+}
+/** Parse sitemap.xml for page URLs */
+async function discoverFromSitemap(baseUrl, robots) {
+    // Check for sitemap in robots.txt
+    const sitemapUrl = robots?.getSitemaps()?.[0] || new URL('/sitemap.xml', baseUrl).href;
+    try {
+        const response = await fetch(sitemapUrl, {
+            headers: { 'User-Agent': USER_AGENT },
+            signal: AbortSignal.timeout(15_000),
+        });
+        if (!response.ok)
+            return [];
+        const xml = await response.text();
+        const $ = cheerio.load(xml, { xmlMode: true });
+        const urls = [];
+        // Handle sitemap index (sitemapindex > sitemap > loc)
+        const sitemapLocs = $('sitemapindex > sitemap > loc');
+        if (sitemapLocs.length > 0) {
+            for (const el of sitemapLocs.toArray()) {
+                const childSitemapUrl = $(el).text().trim();
+                if (childSitemapUrl) {
+                    const childUrls = await fetchSitemapUrls(childSitemapUrl);
+                    urls.push(...childUrls);
+                }
+            }
+        }
+        else {
+            // Regular sitemap (urlset > url > loc)
+            $('urlset > url > loc').each((_, el) => {
+                const loc = $(el).text().trim();
+                if (loc)
+                    urls.push(loc);
+            });
+        }
+        return urls;
+    }
+    catch {
+        return [];
+    }
+}
+async function fetchSitemapUrls(sitemapUrl) {
+    try {
+        const response = await fetch(sitemapUrl, {
+            headers: { 'User-Agent': USER_AGENT },
+            signal: AbortSignal.timeout(15_000),
+        });
+        if (!response.ok)
+            return [];
+        const xml = await response.text();
+        const $ = cheerio.load(xml, { xmlMode: true });
+        const urls = [];
+        $('urlset > url > loc').each((_, el) => {
+            const loc = $(el).text().trim();
+            if (loc)
+                urls.push(loc);
+        });
+        return urls;
+    }
+    catch {
+        return [];
+    }
+}
+/** BFS link crawl from the base URL */
+async function discoverByLinkCrawl(baseUrl, maxDepth, config, robots) {
+    const visited = new Set();
+    const queue = [{ url: baseUrl, depth: 0 }];
+    const rateLimiter = new RateLimiter(config.rateLimit ?? 500);
+    const baseOrigin = new URL(baseUrl).origin;
+    const basePath = new URL(baseUrl).pathname;
+    while (queue.length > 0) {
+        const item = queue.shift();
+        if (visited.has(item.url) || item.depth > maxDepth)
+            continue;
+        if (!isAllowed(robots, item.url))
+            continue;
+        visited.add(item.url);
+        try {
+            await rateLimiter.wait();
+            const response = await fetch(item.url, {
+                headers: { 'User-Agent': USER_AGENT },
+                signal: AbortSignal.timeout(15_000),
+            });
+            if (!response.ok)
+                continue;
+            const contentType = response.headers.get('content-type') || '';
+            if (!contentType.includes('text/html'))
+                continue;
+            const html = await response.text();
+            const $ = cheerio.load(html);
+            $('a[href]').each((_, el) => {
+                try {
+                    const href = $(el).attr('href');
+                    if (!href)
+                        return;
+                    const resolved = new URL(href, item.url);
+                    // Strip hash and query
+                    resolved.hash = '';
+                    resolved.search = '';
+                    const resolvedUrl = resolved.href;
+                    // Only follow same-origin links under the base path
+                    if (resolved.origin === baseOrigin &&
+                        resolved.pathname.startsWith(basePath) &&
+                        !visited.has(resolvedUrl) &&
+                        !isNonDocUrl(resolvedUrl)) {
+                        queue.push({ url: resolvedUrl, depth: item.depth + 1 });
+                    }
+                }
+                catch {
+                    // Invalid URL, skip
+                }
+            });
+        }
+        catch {
+            // Fetch failed, skip
+        }
+    }
+    return filterUrls([...visited], baseUrl, config, robots);
+}
+/** Filter URLs based on config patterns */
+function filterUrls(urls, baseUrl, config, robots) {
+    const baseOrigin = new URL(baseUrl).origin;
+    const basePath = new URL(baseUrl).pathname;
+    return urls.filter((url) => {
+        try {
+            const parsed = new URL(url);
+            // Must be same origin and under base path
+            if (parsed.origin !== baseOrigin)
+                return false;
+            if (!parsed.pathname.startsWith(basePath))
+                return false;
+            // Check robots.txt
+            if (!isAllowed(robots, url))
+                return false;
+            // Skip non-doc URLs
+            if (isNonDocUrl(url))
+                return false;
+            // Include/exclude patterns
+            if (config.includePatterns?.length) {
+                const matches = config.includePatterns.some((p) => url.includes(p));
+                if (!matches)
+                    return false;
+            }
+            if (config.excludePatterns?.length) {
+                const excluded = config.excludePatterns.some((p) => url.includes(p));
+                if (excluded)
+                    return false;
+            }
+            return true;
+        }
+        catch {
+            return false;
+        }
+    });
+}
+/** Heuristic: skip non-documentation URLs */
+function isNonDocUrl(url) {
+    const skip = [
+        '/api/',
+        '/login',
+        '/signup',
+        '/auth/',
+        '.pdf',
+        '.zip',
+        '.tar',
+        '.gz',
+        '.png',
+        '.jpg',
+        '.jpeg',
+        '.gif',
+        '.svg',
+        '.ico',
+        '.css',
+        '.js',
+        '.woff',
+        '.woff2',
+        '.ttf',
+        '.eot',
+        '/feed',
+        '/rss',
+        '/atom',
+    ];
+    const lower = url.toLowerCase();
+    return skip.some((s) => lower.includes(s));
+}