@gulibs/safe-coder 0.0.23 → 0.0.25

package/dist/documentation/doc-crawler.d.ts

@@ -1,5 +1,6 @@
 import { HttpClient } from '../utils/http-client.js';
 export interface CrawlOptions {
+    crawlStrategy?: 'bfs' | 'dfs';
     maxDepth?: number;
     maxPages?: number;
     includePaths?: string[];
@@ -7,6 +8,15 @@ export interface CrawlOptions {
     rateLimit?: number;
     maxRetries?: number;
     retryDelay?: number;
+    useBrowserAutomation?: boolean;
+    skipLlmsTxt?: boolean;
+    workers?: number;
+    checkpoint?: {
+        enabled: boolean;
+        interval: number;
+        file?: string;
+    };
+    resume?: boolean;
 }
 export interface CrawledPage {
     url: string;
@@ -36,7 +46,7 @@ export interface CrawledPage {
 export interface LinkDiscoveryStats {
     totalLinksFound: number;
     linksFiltered: {
-        notDocumentation: number;
+        notContent: number;
         externalDomain: number;
         alreadyVisited: number;
         excludedPattern: number;
@@ -46,6 +56,7 @@ export interface LinkDiscoveryStats {
     pagesDiscovered: number;
     pagesCrawled: number;
 }
+export type AbandonReason = 'insufficient_content' | 'media_only' | 'empty_pages' | 'no_structured_content';
 export interface CrawlResult {
     pages: CrawledPage[];
     totalPages: number;
@@ -55,6 +66,8 @@ export interface CrawlResult {
         error: string;
     }>;
     linkDiscoveryStats: LinkDiscoveryStats;
+    abandoned?: boolean;
+    abandonReason?: AbandonReason;
 }
 export declare class DocumentationCrawler {
     private browser;
@@ -65,6 +78,8 @@ export declare class DocumentationCrawler {
     private options;
     private baseUrl;
     private linkDiscoveryStats;
+    private checkpointManager?;
+    private pagesSinceLastCheckpoint;
     private readonly DOCUMENTATION_PATTERNS;
     private readonly EXCLUDED_PATTERNS;
     constructor(httpClient?: HttpClient);
@@ -72,24 +87,60 @@ export declare class DocumentationCrawler {
      * Crawl documentation starting from a root URL
      * Uses HTTP client (axios) exclusively - no browser automation
      * For SPA sites that require JavaScript rendering, use Cursor/Claude's built-in browser tools
+     * Supports both BFS (breadth-first) and DFS (depth-first) crawl strategies
      */
     crawl(rootUrl: string, options?: CrawlOptions): Promise<CrawlResult>;
+    /**
+     * Sequential crawling (single-threaded)
+     */
+    private crawlSequential;
+    /**
+     * Parallel crawling with multiple workers
+     */
+    private crawlWithWorkers;
+    /**
+     * Process a single page (shared by both sequential and parallel crawling)
+     */
+    private processPage;
     /**
      * Discover documentation links from a crawled page
      */
     private discoverDocumentationLinks;
     /**
-     * Check if a path is a documentation path
+     * Check if a path should be crawled (permissive - only exclude clearly non-content paths)
      */
     private isDocumentationPath;
     /**
      * Check if a path should be excluded
      */
     private shouldExclude;
+    /**
+     * Check if crawled content is sufficient for skill generation
+     * Enhanced with multi-dimensional quality metrics
+     */
+    private canGenerateSkill;
+    /**
+     * Evaluate content quality with multi-dimensional metrics
+     */
+    private evaluateContentQuality;
+    /**
+     * Check if should continue crawling based on content quality
+     */
+    private shouldContinueCrawling;
     /**
      * Fetch a page with retry logic
+     * Supports both HTML pages and Markdown files
      */
     private fetchPageWithRetry;
+    /**
+     * Extract content from Markdown file
+     * Converts Markdown structure to WebDocumentationPage format
+     */
+    private extractMarkdownContent;
+    /**
+     * Parse Markdown content into structured data
+     */
+    private parseMarkdown;
     /**
      * Classify error type for better error messages
      */
@@ -102,6 +153,30 @@ export declare class DocumentationCrawler {
      * Get error breakdown by type
      */
     private getErrorBreakdown;
+    /**
+     * Try to detect and use llms.txt for optimized crawling
+     */
+    private tryLlmsTxt;
+    /**
+     * Check if a URL is valid for crawling
+     */
+    private isValidUrl;
+    /**
+     * Save checkpoint
+     */
+    private saveCheckpoint;
+    /**
+     * Load checkpoint and restore state
+     */
+    private loadCheckpoint;
+    /**
+     * Clear checkpoint after successful crawl
+     */
+    private clearCheckpoint;
+    /**
+     * Sanitize filename for checkpoint
+     */
+    private sanitizeFilename;
     /**
      * Delay helper for rate limiting
      */

package/dist/documentation/doc-crawler.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"doc-crawler.d.ts","sourceRoot":"","sources":["../../src/documentation/doc-crawler.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AAIrD,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,WAAW;IAC1B,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,KAAK,CAAC;QACd,KAAK,EAAE,MAAM,CAAC;QACd,OAAO,EAAE,MAAM,CAAC;QAChB,MAAM,CAAC,EAAE,MAAM,CAAC;KACjB,CAAC,CAAC;IACH,eAAe,EAAE,KAAK,CAAC;QACrB,IAAI,EAAE,MAAM,CAAC;QACb,GAAG,EAAE,MAAM,CAAC;QACZ,UAAU,EAAE,OAAO,CAAC;KACrB,CAAC,CAAC;IACH,QAAQ,EAAE,KAAK,CAAC;QACd,KAAK,EAAE,MAAM,CAAC;QACd,IAAI,EAAE,MAAM,CAAC;QACb,EAAE,CAAC,EAAE,MAAM,CAAC;KACb,CAAC,CAAC;IACH,WAAW,EAAE,KAAK,CAAC;QACjB,IAAI,EAAE,MAAM,CAAC;QACb,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC,CAAC;CACJ;AAED,MAAM,WAAW,kBAAkB;IACjC,eAAe,EAAE,MAAM,CAAC;IACxB,aAAa,EAAE;QACb,gBAAgB,EAAE,MAAM,CAAC;QACzB,cAAc,EAAE,MAAM,CAAC;QACvB,cAAc,EAAE,MAAM,CAAC;QACvB,eAAe,EAAE,MAAM,CAAC;QACxB,UAAU,EAAE,MAAM,CAAC;KACpB,CAAC;IACF,WAAW,EAAE,MAAM,CAAC;IACpB,eAAe,EAAE,MAAM,CAAC;IACxB,YAAY,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,WAAW;IAC1B,KAAK,EAAE,WAAW,EAAE,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,eAAe,EAAE,MAAM,CAAC;IACxB,MAAM,EAAE,KAAK,CAAC;QACZ,GAAG,EAAE,MAAM,CAAC;QACZ,KAAK,EAAE,MAAM,CAAC;KACf,CAAC,CAAC;IACH,kBAAkB,EAAE,kBAAkB,CAAC;CACxC;AAOD,qBAAa,oBAAoB;IAC/B,OAAO,CAAC,OAAO,CAA0B;IACzC,OAAO,CAAC,WAAW,CAAc;IACjC,OAAO,CAAC,QAAQ,CAAc;IAC9B,OAAO,CAAC,YAAY,CAAgB;IACpC,OAAO,CAAC,MAAM,CAAwC;IACtD,OAAO,CAAC,OAAO,CAAyB;IACxC,OAAO,CAAC,OAAO,CAAM;IACrB,OAAO,CAAC,kBAAkB,CAAqB;IAC/C,OAAO,CAAC,QAAQ,CAAC,sBAAsB,CAWrC;IACF,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAUhC;gBAEU,UAAU,CAAC,EAAE,UAAU;IAgCnC;;;;OAIG;IACG,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,GAAE,YAAiB,GAAG,OAAO,CAAC,WAAW,CAAC;IAqR9E;;OAEG;IACH,OAAO,CAAC,0BAA0B;IAiJlC;;OAEG;IACH,OAAO,CAAC,mBAAmB;IAgD3B;;OAEG;IACH,OAAO,CAAC,aAAa;IAIrB;;OAEG;YACW,kBAAkB;IA0BhC;;OAEG;IACH,OAAO,CAAC,aAAa;IA0CrB;;OAEG;IACH,OAAO,CAAC,gBAAgB;IAmBxB;;OAEG;IACH,OAAO,CAAC,iBAAiB;IAWzB;;OAEG;IACH,OAAO,CAAC,KAAK;CAGd"}
+{"version":3,"file":"doc-crawler.d.ts","sourceRoot":"","sources":["../../src/documentation/doc-crawler.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AAQrD,MAAM,WAAW,YAAY;IAC3B,aAAa,CAAC,EAAE,KAAK,GAAG,KAAK,CAAC;IAC9B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,oBAAoB,CAAC,EAAE,OAAO,CAAC;IAC/B,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE;QACX,OAAO,EAAE,OAAO,CAAC;QACjB,QAAQ,EAAE,MAAM,CAAC;QACjB,IAAI,CAAC,EAAE,MAAM,CAAC;KACf,CAAC;IACF,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,WAAW;IAC1B,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,KAAK,CAAC;QACd,KAAK,EAAE,MAAM,CAAC;QACd,OAAO,EAAE,MAAM,CAAC;QAChB,MAAM,CAAC,EAAE,MAAM,CAAC;KACjB,CAAC,CAAC;IACH,eAAe,EAAE,KAAK,CAAC;QACrB,IAAI,EAAE,MAAM,CAAC;QACb,GAAG,EAAE,MAAM,CAAC;QACZ,UAAU,EAAE,OAAO,CAAC;KACrB,CAAC,CAAC;IACH,QAAQ,EAAE,KAAK,CAAC;QACd,KAAK,EAAE,MAAM,CAAC;QACd,IAAI,EAAE,MAAM,CAAC;QACb,EAAE,CAAC,EAAE,MAAM,CAAC;KACb,CAAC,CAAC;IACH,WAAW,EAAE,KAAK,CAAC;QACjB,IAAI,EAAE,MAAM,CAAC;QACb,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC,CAAC;CACJ;AAED,MAAM,WAAW,kBAAkB;IACjC,eAAe,EAAE,MAAM,CAAC;IACxB,aAAa,EAAE;QACb,UAAU,EAAE,MAAM,CAAC;QACnB,cAAc,EAAE,MAAM,CAAC;QACvB,cAAc,EAAE,MAAM,CAAC;QACvB,eAAe,EAAE,MAAM,CAAC;QACxB,UAAU,EAAE,MAAM,CAAC;KACpB,CAAC;IACF,WAAW,EAAE,MAAM,CAAC;IACpB,eAAe,EAAE,MAAM,CAAC;IACxB,YAAY,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,MAAM,aAAa,GACrB,sBAAsB,GACtB,YAAY,GACZ,aAAa,GACb,uBAAuB,CAAC;AAE5B,MAAM,WAAW,WAAW;IAC1B,KAAK,EAAE,WAAW,EAAE,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,eAAe,EAAE,MAAM,CAAC;IACxB,MAAM,EAAE,KAAK,CAAC;QACZ,GAAG,EAAE,MAAM,CAAC;QACZ,KAAK,EAAE,MAAM,CAAC;KACf,CAAC,CAAC;IACH,kBAAkB,EAAE,kBAAkB,CAAC;IACvC,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,aAAa,CAAC,EAAE,aAAa,CAAC;CAC/B;AAOD,qBAAa,oBAAoB;IAC/B,OAAO,CAAC,OAAO,CAA0B;IACzC,OAAO,CAAC,WAAW,CAAc;IACjC,OAAO,CAAC,QAAQ,CAAc;IAC9B,OAAO,CAAC,YAAY,CAAgB;IACpC,OAAO,CAAC,MAAM,CAAwC;IACtD,OAAO,CAAC,OAAO,CAYb;IACF,OAAO,CAAC,OAAO,CAAM;IACrB,OAAO,CAAC,kBAAkB,CAAqB;IAC/C,OAAO,CAAC,iBAAiB,CAAC,CAAoB;IAC9C,OAAO,CAAC,wBAAwB,CAAS;IACzC,OAAO,CAAC,QAAQ,CAAC,sBAAsB,CAWrC;IACF,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAUhC;gBAEU,UAAU,CAAC,EAAE,UAAU;IAqCnC;;;;;OAKG;IACG,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,GAAE,YAAiB,GAAG,OAAO,CAAC,WAAW,CAAC;IA4K9E;;OAEG;YACW,eAAe;IAyD7B;;OAEG;YACW,gBAAgB;IAoE9B;;OAEG;YACW,WAAW;IAgHzB;;OAEG;IACH,OAAO,CAAC,0BAA0B;IAiJlC;;OAEG;IACH,OAAO,CAAC,mBAAmB;IAmC3B;;OAEG;IACH,OAAO,CAAC,aAAa;IAIrB;;;OAGG;IACH,OAAO,CAAC,gBAAgB;IAyBxB;;OAEG;IACH,OAAO,CAAC,sBAAsB;IAoF9B;;OAEG;IACH,OAAO,CAAC,sBAAsB;IAsC9B;;;OAGG;YACW,kBAAkB;IAgChC;;;OAGG;YACW,sBAAsB;IA4BpC;;OAEG;IACH,OAAO,CAAC,aAAa;IAyJrB;;OAEG;IACH,OAAO,CAAC,aAAa;IA0CrB;;OAEG;IACH,OAAO,CAAC,gBAAgB;IAmBxB;;OAEG;IACH,OAAO,CAAC,iBAAiB;IAWzB;;OAEG;YACW,UAAU;IAmExB;;OAEG;IACH,OAAO,CAAC,UAAU;IAiBlB;;OAEG;YACW,cAAc;IAuB5B;;OAEG;YACW,cAAc;IAgC5B;;OAEG;YACW,eAAe;IAY7B;;OAEG;IACH,OAAO,CAAC,gBAAgB;IAOxB;;OAEG;IACH,OAAO,CAAC,KAAK;CAGd"}