@crawlith/core 0.1.1 → 0.1.2

package/dist/crawler/crawler.d.ts

@@ -1,5 +1,6 @@
 import { Graph } from '../graph/graph.js';
 import { EngineContext } from '../events.js';
+import { PluginRegistry } from '../plugin-system/plugin-registry.js';
 export interface CrawlOptions {
     limit: number;
     depth: number;
@@ -7,7 +8,7 @@ export interface CrawlOptions {
     ignoreRobots?: boolean;
     stripQuery?: boolean;
     previousGraph?: Graph;
-    sitemap?: string;
+    sitemap?: string | boolean;
     debug?: boolean;
     detectSoft404?: boolean;
     detectTraps?: boolean;
@@ -19,12 +20,16 @@ export interface CrawlOptions {
     proxyUrl?: string;
     maxRedirects?: number;
     userAgent?: string;
-    snapshotType?: 'full' | 'partial' | 'incremental';
+    snapshotRunType?: 'completed' | 'incremental' | 'single';
+    registry?: PluginRegistry;
+    plugins?: any[];
+    robots?: any;
 }
 export declare class Crawler {
     private startUrl;
     private options;
     private context;
+    private registry?;
     private visited;
     private uniqueQueue;
     private queue;
@@ -41,27 +46,34 @@ export declare class Crawler {
     private metricsRepo;
     private siteId;
     private snapshotId;
+    private reusingSnapshot;
+    private runType;
     private rootOrigin;
     private discoveryDepths;
     private pageBuffer;
     private edgeBuffer;
     private metricsBuffer;
+    private pendingSitemaps;
+    private edgesFound;
+    private lastProgressEmitAt;
+    private progressPhase;
     private scopeManager;
     private fetcher;
     private parser;
     private sitemapFetcher;
-    private trapDetector;
     private robots;
     constructor(startUrl: string, options: CrawlOptions, context?: EngineContext);
+    private toStorageUrl;
     initialize(): Promise<void>;
     setupModules(): void;
-    fetchRobots(): Promise<void>;
+    private fetchRobots;
     shouldEnqueue(url: string, depth: number): boolean;
-    addToQueue(u: string, d: number): void;
+    addToQueue(u: string, d: number, data?: any): void;
     seedQueue(): Promise<void>;
     private bufferPage;
     private flushPages;
     private bufferEdge;
+    private emitProgress;
     private flushEdges;
     private bufferMetrics;
     private flushMetrics;