@gulibs/safe-coder 0.0.26 → 0.0.27

package/dist/documentation/doc-crawler.d.ts

@@ -1,221 +0,0 @@
-import { HttpClient } from '../utils/http-client.js';
-export interface CrawlOptions {
-    crawlStrategy?: 'bfs' | 'dfs';
-    maxDepth?: number;
-    maxPages?: number;
-    includePaths?: string[];
-    excludePaths?: string[];
-    rateLimit?: number;
-    maxRetries?: number;
-    retryDelay?: number;
-    useBrowserAutomation?: boolean;
-    skipLlmsTxt?: boolean;
-    workers?: number;
-    checkpoint?: {
-        enabled: boolean;
-        interval: number;
-        file?: string;
-    };
-    resume?: boolean;
-    spaStrategy?: 'smart' | 'auto' | 'manual';
-    spaFallback?: 'warn' | 'skip' | 'error';
-    browserConfig?: {
-        executablePath?: string;
-        waitForTimeout?: number;
-        networkIdleTimeout?: number;
-    };
-}
-export interface CrawledPage {
-    url: string;
-    title: string;
-    content: string;
-    depth: number;
-    sections: Array<{
-        title: string;
-        content: string;
-        anchor?: string;
-    }>;
-    navigationLinks: Array<{
-        text: string;
-        url: string;
-        isInternal: boolean;
-    }>;
-    headings: Array<{
-        level: string;
-        text: string;
-        id?: string;
-    }>;
-    codeSamples: Array<{
-        code: string;
-        language: string;
-    }>;
-}
-export interface LinkDiscoveryStats {
-    totalLinksFound: number;
-    linksFiltered: {
-        notContent: number;
-        externalDomain: number;
-        alreadyVisited: number;
-        excludedPattern: number;
-        depthLimit: number;
-    };
-    linksQueued: number;
-    pagesDiscovered: number;
-    pagesCrawled: number;
-}
-export type AbandonReason = 'insufficient_content' | 'media_only' | 'empty_pages' | 'no_structured_content';
-export interface CrawlResult {
-    pages: CrawledPage[];
-    totalPages: number;
-    maxDepthReached: number;
-    errors: Array<{
-        url: string;
-        error: string;
-    }>;
-    linkDiscoveryStats: LinkDiscoveryStats;
-    abandoned?: boolean;
-    abandonReason?: AbandonReason;
-}
-export declare class DocumentationCrawler {
-    private browser;
-    private browserManager?;
-    private visitedUrls;
-    private urlQueue;
-    private crawledPages;
-    private errors;
-    private options;
-    private baseUrl;
-    private linkDiscoveryStats;
-    private checkpointManager?;
-    private pagesSinceLastCheckpoint;
-    private readonly DOCUMENTATION_PATTERNS;
-    private readonly EXCLUDED_PATTERNS;
-    constructor(httpClient?: HttpClient);
-    /**
-     * Crawl documentation starting from a root URL
-     * Uses HTTP client (axios) exclusively - no browser automation
-     * For SPA sites that require JavaScript rendering, use Cursor/Claude's built-in browser tools
-     * Supports both BFS (breadth-first) and DFS (depth-first) crawl strategies
-     */
-    crawl(rootUrl: string, options?: CrawlOptions): Promise<CrawlResult>;
-    /**
-     * Sequential crawling (single-threaded)
-     */
-    private crawlSequential;
-    /**
-     * Parallel crawling with multiple workers
-     */
-    private crawlWithWorkers;
-    /**
-     * Process a single page (shared by both sequential and parallel crawling)
-     */
-    private processPage;
-    /**
-     * Discover documentation links from a crawled page
-     */
-    private discoverDocumentationLinks;
-    /**
-     * Check if a path should be crawled (permissive - only exclude clearly non-content paths)
-     */
-    private isDocumentationPath;
-    /**
-     * Check if a path should be excluded
-     */
-    private shouldExclude;
-    /**
-     * Check if crawled content is sufficient for skill generation
-     * Enhanced with multi-dimensional quality metrics
-     */
-    private canGenerateSkill;
-    /**
-     * Evaluate content quality with multi-dimensional metrics
-     */
-    private evaluateContentQuality;
-    /**
-     * Check if should continue crawling based on content quality
-     */
-    private shouldContinueCrawling;
-    /**
-     * Fetch a page with retry logic
-     * Supports HTML pages, Markdown files, and SPA rendering
-     */
-    private fetchPageWithRetry;
-    /**
-     * Extract content from Markdown file
-     * Converts Markdown structure to WebDocumentationPage format
-     */
-    private extractMarkdownContent;
-    /**
-     * Parse Markdown content into structured data
-     */
-    private parseMarkdown;
-    /**
-     * Classify error type for better error messages
-     */
-    private classifyError;
-    /**
-     * Check if an error is retryable
-     */
-    private isRetryableError;
-    /**
-     * Get error breakdown by type
-     */
-    private getErrorBreakdown;
-    /**
-     * Try to detect and use llms.txt for optimized crawling
-     */
-    private tryLlmsTxt;
-    /**
-     * Check if a URL is valid for crawling
-     */
-    private isValidUrl;
-    /**
-     * Save checkpoint
-     */
-    private saveCheckpoint;
-    /**
-     * Load checkpoint and restore state
-     */
-    private loadCheckpoint;
-    /**
-     * Clear checkpoint after successful crawl
-     */
-    private clearCheckpoint;
-    /**
-     * Sanitize filename for checkpoint
-     */
-    private sanitizeFilename;
-    /**
-     * Check if browser rendering is needed
-     */
-    private shouldUseBrowser;
-    /**
-     * Fetch page using browser rendering
-     */
-    private fetchWithBrowser;
-    /**
-     * Parse browser-rendered HTML into WebDocumentationPage
-     */
-    private parseRenderedPage;
-    /**
-     * Handle browser rendering failure based on fallback strategy
-     */
-    private handleBrowserFailure;
-    /**
-     * Create empty page placeholder
-     */
-    private createEmptyPage;
-    /**
-     * Create page with browser installation guide
-     */
-    private createPageWithGuide;
-    /**
-     * Cleanup resources (browser, checkpoint, etc.)
-     */
-    cleanup(): Promise<void>;
-    /**
-     * Delay helper for rate limiting
-     */
-    private delay;
-}
-//# sourceMappingURL=doc-crawler.d.ts.map

package/dist/documentation/doc-crawler.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"doc-crawler.d.ts","sourceRoot":"","sources":["../../src/documentation/doc-crawler.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AAUrD,MAAM,WAAW,YAAY;IAC3B,aAAa,CAAC,EAAE,KAAK,GAAG,KAAK,CAAC;IAC9B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,oBAAoB,CAAC,EAAE,OAAO,CAAC;IAC/B,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE;QACX,OAAO,EAAE,OAAO,CAAC;QACjB,QAAQ,EAAE,MAAM,CAAC;QACjB,IAAI,CAAC,EAAE,MAAM,CAAC;KACf,CAAC;IACF,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,WAAW,CAAC,EAAE,OAAO,GAAG,MAAM,GAAG,QAAQ,CAAC;IAC1C,WAAW,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC;IACxC,aAAa,CAAC,EAAE;QACd,cAAc,CAAC,EAAE,MAAM,CAAC;QACxB,cAAc,CAAC,EAAE,MAAM,CAAC;QACxB,kBAAkB,CAAC,EAAE,MAAM,CAAC;KAC7B,CAAC;CACH;AAED,MAAM,WAAW,WAAW;IAC1B,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,KAAK,CAAC;QACd,KAAK,EAAE,MAAM,CAAC;QACd,OAAO,EAAE,MAAM,CAAC;QAChB,MAAM,CAAC,EAAE,MAAM,CAAC;KACjB,CAAC,CAAC;IACH,eAAe,EAAE,KAAK,CAAC;QACrB,IAAI,EAAE,MAAM,CAAC;QACb,GAAG,EAAE,MAAM,CAAC;QACZ,UAAU,EAAE,OAAO,CAAC;KACrB,CAAC,CAAC;IACH,QAAQ,EAAE,KAAK,CAAC;QACd,KAAK,EAAE,MAAM,CAAC;QACd,IAAI,EAAE,MAAM,CAAC;QACb,EAAE,CAAC,EAAE,MAAM,CAAC;KACb,CAAC,CAAC;IACH,WAAW,EAAE,KAAK,CAAC;QACjB,IAAI,EAAE,MAAM,CAAC;QACb,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC,CAAC;CACJ;AAED,MAAM,WAAW,kBAAkB;IACjC,eAAe,EAAE,MAAM,CAAC;IACxB,aAAa,EAAE;QACb,UAAU,EAAE,MAAM,CAAC;QACnB,cAAc,EAAE,MAAM,CAAC;QACvB,cAAc,EAAE,MAAM,CAAC;QACvB,eAAe,EAAE,MAAM,CAAC;QACxB,UAAU,EAAE,MAAM,CAAC;KACpB,CAAC;IACF,WAAW,EAAE,MAAM,CAAC;IACpB,eAAe,EAAE,MAAM,CAAC;IACxB,YAAY,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,MAAM,aAAa,GACrB,sBAAsB,GACtB,YAAY,GACZ,aAAa,GACb,uBAAuB,CAAC;AAE5B,MAAM,WAAW,WAAW;IAC1B,KAAK,EAAE,WAAW,EAAE,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,eAAe,EAAE,MAAM,CAAC;IACxB,MAAM,EAAE,KAAK,CAAC;QACZ,GAAG,EAAE,MAAM,CAAC;QACZ,KAAK,EAAE,MAAM,CAAC;KACf,CAAC,CAAC;IACH,kBAAkB,EAAE,kBAAkB,CAAC;IACvC,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,aAAa,CAAC,EAAE,aAAa,CAAC;CAC/B;AAOD,qBAAa,oBAAoB;IAC/B,OAAO,CAAC,OAAO,CAA0B;IACzC,OAAO,CAAC,cAAc,CAAC,CAAiB;IACxC,OAAO,CAAC,WAAW,CAAc;IACjC,OAAO,CAAC,QAAQ,CAAc;IAC9B,OAAO,CAAC,YAAY,CAAgB;IACpC,OAAO,CAAC,MAAM,CAAwC;IACtD,OAAO,CAAC,OAAO,CAcb;IACF,OAAO,CAAC,OAAO,CAAM;IACrB,OAAO,CAAC,kBAAkB,CAAqB;IAC/C,OAAO,CAAC,iBAAiB,CAAC,CAAoB;IAC9C,OAAO,CAAC,wBAAwB,CAAS;IACzC,OAAO,CAAC,QAAQ,CAAC,sBAAsB,CAWrC;IACF,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAUhC;gBAEU,UAAU,CAAC,EAAE,UAAU;IAuCnC;;;;;OAKG;IACG,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,GAAE,YAAiB,GAAG,OAAO,CAAC,WAAW,CAAC;IA4K9E;;OAEG;YACW,eAAe;IAyD7B;;OAEG;YACW,gBAAgB;IAoE9B;;OAEG;YACW,WAAW;IAgHzB;;OAEG;IACH,OAAO,CAAC,0BAA0B;IAiJlC;;OAEG;IACH,OAAO,CAAC,mBAAmB;IAmC3B;;OAEG;IACH,OAAO,CAAC,aAAa;IAIrB;;;OAGG;IACH,OAAO,CAAC,gBAAgB;IAyBxB;;OAEG;IACH,OAAO,CAAC,sBAAsB;IAoF9B;;OAEG;IACH,OAAO,CAAC,sBAAsB;IAsC9B;;;OAGG;YACW,kBAAkB;IA+DhC;;;OAGG;YACW,sBAAsB;IA4BpC;;OAEG;IACH,OAAO,CAAC,aAAa;IAyJrB;;OAEG;IACH,OAAO,CAAC,aAAa;IA0CrB;;OAEG;IACH,OAAO,CAAC,gBAAgB;IAmBxB;;OAEG;IACH,OAAO,CAAC,iBAAiB;IAWzB;;OAEG;YACW,UAAU;IAmExB;;OAEG;IACH,OAAO,CAAC,UAAU;IAiBlB;;OAEG;YACW,cAAc;IAuB5B;;OAEG;YACW,cAAc;IAgC5B;;OAEG;YACW,eAAe;IAY7B;;OAEG;IACH,OAAO,CAAC,gBAAgB;IAOxB;;OAEG;YACW,gBAAgB;IAiC9B;;OAEG;YACW,gBAAgB;IAuB9B;;OAEG;IACH,OAAO,CAAC,iBAAiB;IA6EzB;;OAEG;YACW,oBAAoB;IAkBlC;;OAEG;IACH,OAAO,CAAC,eAAe;IAcvB;;OAEG;IACH,OAAO,CAAC,mBAAmB;IAsD3B;;OAEG;IACG,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAO9B;;OAEG;IACH,OAAO,CAAC,KAAK;CAGd"}