@jambudipa/spider 0.2.1 → 0.2.3

package/dist/lib/WebScrapingEngine/WebScrapingEngine.service.d.ts

@@ -0,0 +1,110 @@
+/**
+ * Web Scraping Engine Service
+ * Orchestrates all scraping capabilities including authentication, token management, and session handling
+ */
+import { Context, DateTime, Effect, HashMap, Layer } from 'effect';
+import { EnhancedHttpClient, type HttpResponse } from '../HttpClient/EnhancedHttpClient.js';
+import { CookieManager } from '../HttpClient/CookieManager.js';
+import { SessionStore, SessionError } from '../HttpClient/SessionStore.js';
+import { TokenExtractor } from '../HttpClient/TokenExtractor.js';
+import { StateManager, TokenType } from '../StateManager/StateManager.service.js';
+import { SpiderLogger } from '../Logging/SpiderLogger.service.js';
+import { NetworkError } from '../errors.js';
+import { ParseError, TimeoutError } from '../errors/effect-errors.js';
+import { JsonStringifyError } from '../utils/JsonUtils.js';
+declare const LoginError_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").Equals<A, {}> extends true ? void : { readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }) => import("effect/Cause").YieldableError & {
+    readonly _tag: "LoginError";
+} & Readonly<A>;
+export declare class LoginError extends LoginError_base<{
+    readonly status: number;
+    readonly message: string;
+}> {
+}
+declare const SessionNotValidError_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").Equals<A, {}> extends true ? void : { readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }) => import("effect/Cause").YieldableError & {
+    readonly _tag: "SessionNotValidError";
+} & Readonly<A>;
+export declare class SessionNotValidError extends SessionNotValidError_base<{
+    readonly message: string;
+}> {
+}
+declare const SessionLoadError_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").Equals<A, {}> extends true ? void : { readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }) => import("effect/Cause").YieldableError & {
+    readonly _tag: "SessionLoadError";
+} & Readonly<A>;
+export declare class SessionLoadError extends SessionLoadError_base<{
+    readonly message: string;
+}> {
+}
+export type WebScrapingEngineError = LoginError | SessionNotValidError | SessionLoadError;
+/**
+ * Combined error types for HTTP operations
+ */
+export type HttpOperationError = NetworkError | ParseError | TimeoutError;
+/**
+ * Combined error types for POST operations
+ */
+export type HttpPostOperationError = HttpOperationError | JsonStringifyError;
+export interface LoginCredentials {
+    username: string;
+    password: string;
+    loginUrl: string;
+    usernameField?: string;
+    passwordField?: string;
+    additionalFields?: Record<string, string>;
+}
+export interface ScrapingSession {
+    id: string;
+    authenticated: boolean;
+    tokens: HashMap.HashMap<TokenType, string>;
+    startTime: DateTime.Utc;
+}
+export interface WebScrapingEngineService {
+    /**
+     * Perform login with form submission
+     */
+    login: (_credentials: LoginCredentials) => Effect.Effect<ScrapingSession, HttpOperationError | SessionError | LoginError>;
+    /**
+     * Fetch authenticated content
+     */
+    fetchAuthenticated: (_url: string) => Effect.Effect<HttpResponse, HttpOperationError | SessionNotValidError>;
+    /**
+     * Submit form with CSRF protection
+     */
+    submitFormWithCSRF: (_url: string, _formData: Record<string, string>, _csrfUrl?: string) => Effect.Effect<HttpResponse, HttpOperationError>;
+    /**
+     * Make API request with token
+     */
+    makeAPIRequest: (_url: string, _method?: 'GET' | 'POST' | 'PUT' | 'DELETE', _data?: Record<string, unknown>) => Effect.Effect<HttpResponse, HttpPostOperationError>;
+    /**
+     * Create and save a scraping session
+     */
+    createSession: (_id?: string) => Effect.Effect<ScrapingSession>;
+    /**
+     * Load existing session
+     */
+    loadSession: (_id: string) => Effect.Effect<ScrapingSession, SessionError | SessionLoadError>;
+    /**
+     * Export session for persistence
+     */
+    exportSession: () => Effect.Effect<string, SessionError>;
+    /**
+     * Import session from persistence
+     */
+    importSession: (_data: string) => Effect.Effect<void, SessionError>;
+    /**
+     * Clear all state and sessions
+     */
+    clearAll: () => Effect.Effect<void>;
+}
+declare const WebScrapingEngine_base: Context.TagClass<WebScrapingEngine, "WebScrapingEngine", WebScrapingEngineService>;
+export declare class WebScrapingEngine extends WebScrapingEngine_base {
+}
+/**
+ * Create a WebScrapingEngine service implementation
+ */
+export declare const makeWebScrapingEngine: Effect.Effect<WebScrapingEngineService, never, SpiderLogger | CookieManager | EnhancedHttpClient | StateManager | SessionStore | TokenExtractor>;
+/**
+ * WebScrapingEngine Layer with all dependencies
+ */
+export declare const WebScrapingEngineLive: Layer.Layer<WebScrapingEngine, never, SpiderLogger | CookieManager | EnhancedHttpClient | StateManager | SessionStore | TokenExtractor>;
+export {};
+//# sourceMappingURL=WebScrapingEngine.service.d.ts.map

package/dist/lib/WebScrapingEngine/WebScrapingEngine.service.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"WebScrapingEngine.service.d.ts","sourceRoot":"","sources":["../../../src/lib/WebScrapingEngine/WebScrapingEngine.service.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,OAAO,EAAQ,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAU,MAAM,QAAQ,CAAC;AACjF,OAAO,EACL,kBAAkB,EAClB,KAAK,YAAY,EAClB,MAAM,qCAAqC,CAAC;AAC7C,OAAO,EAAE,aAAa,EAAE,MAAM,gCAAgC,CAAC;AAC/D,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,+BAA+B,CAAC;AAC3E,OAAO,EAAE,cAAc,EAAE,MAAM,iCAAiC,CAAC;AACjE,OAAO,EACL,YAAY,EACZ,SAAS,EACV,MAAM,yCAAyC,CAAC;AACjD,OAAO,EAAE,YAAY,EAAE,MAAM,oCAAoC,CAAC;AAClE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AACtE,OAAO,EAAE,kBAAkB,EAAE,MAAM,uBAAuB,CAAC;;;;AAM3D,qBAAa,UAAW,SAAQ,gBAA+B;IAC7D,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;CAC1B,CAAC;CAAG;;;;AAEL,qBAAa,oBAAqB,SAAQ,0BAAyC;IACjF,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;CAC1B,CAAC;CAAG;;;;AAEL,qBAAa,gBAAiB,SAAQ,sBAAqC;IACzE,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;CAC1B,CAAC;CAAG;AAEL,MAAM,MAAM,sBAAsB,GAAG,UAAU,GAAG,oBAAoB,GAAG,gBAAgB,CAAC;AAE1F;;GAEG;AACH,MAAM,MAAM,kBAAkB,GAAG,YAAY,GAAG,UAAU,GAAG,YAAY,CAAC;AAE1E;;GAEG;AACH,MAAM,MAAM,sBAAsB,GAAG,kBAAkB,GAAG,kBAAkB,CAAC;AAE7E,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,gBAAgB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAC3C;AAED,MAAM,WAAW,eAAe;IAC9B,EAAE,EAAE,MAAM,CAAC;IACX,aAAa,EAAE,OAAO,CAAC;IACvB,MAAM,EAAE,OAAO,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAC3C,SAAS,EAAE,QAAQ,CAAC,GAAG,CAAC;CACzB;AAED,MAAM,WAAW,wBAAwB;IACvC;;OAEG;IACH,KAAK,EAAE,CACL,YAAY,EAAE,gBAAgB,KAC3B,MAAM,CAAC,MAAM,CAAC,eAAe,EAAE,kBAAkB,GAAG,YAAY,GAAG,UAAU,CAAC,CAAC;IAEpF;;OAEG;IACH,kBAAkB,EAAE,CAClB,IAAI,EAAE,MAAM,KACT,MAAM,CAAC,MAAM,CAAC,YAAY,EAAE,kBAAkB,GAAG,oBAAoB,CAAC,CAAC;IAE5E;;OAEG;IACH,kBAAkB,EAAE,CAClB,IAAI,EAAE,MAAM,EACZ,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EACjC,QAAQ,CAAC,EAAE,MAAM,KACd,MAAM,CAAC,MAAM,CAAC,YAAY,EAAE,kBAAkB,CAAC,CAAC;IAErD;;OAEG;IACH,cAAc,EAAE,CACd,IAAI,EAAE,MAAM,EACZ,OAAO,CAAC,EAAE,KAAK,GAAG,MAAM,GAAG,KAAK,GAAG,QAAQ,EAC3C,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAC5B,MAAM,CAAC,MAAM,CAAC,YAAY,EAAE,sBAAsB,CAAC,CAAC;IAEzD;;OAEG;IACH,aAAa,EAAE,CAAC,GAAG,CAAC,EAAE,MAAM,KAAK,MAAM,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;IAEhE;;OAEG;IACH,WAAW,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,MAAM,CAAC,MAAM,CAAC,eAAe,EAAE,YAAY,GAAG,gBAAgB,CAAC,CAAC;IAE9F;;OAEG;IACH,aAAa,EAAE,MAAM,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;IAEzD;;OAEG;IACH,aAAa,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC;IAEpE;;OAEG;IACH,QAAQ,EAAE,MAAM,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;CACrC;;AAED,qBAAa,iBAAkB,SAAQ,sBAGpC;CAAG;AAEN;;GAEG;AACH,eAAO,MAAM,qBAAqB,kJA+RhC,CAAC;AAEH;;GAEG;AACH,eAAO,MAAM,qBAAqB,yIAGjC,CAAC"}

package/dist/lib/WebScrapingEngine/index.d.ts

@@ -0,0 +1,5 @@
+/**
+ * Web Scraping Engine module exports
+ */
+export * from './WebScrapingEngine.service.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/lib/WebScrapingEngine/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/lib/WebScrapingEngine/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,cAAc,gCAAgC,CAAC"}

package/dist/lib/WorkerHealth/WorkerHealthMonitor.service.d.ts

@@ -0,0 +1,39 @@
+import { DateTime, Effect } from 'effect';
+import { SpiderLogger } from '../Logging/SpiderLogger.service.js';
+interface WorkerStatus {
+    workerId: string;
+    domain: string;
+    currentUrl?: string;
+    lastActivity: DateTime.Utc;
+    fetchStartTime?: DateTime.Utc;
+}
+declare const WorkerHealthMonitor_base: Effect.Service.Class<WorkerHealthMonitor, "@jambudipa.io/WorkerHealthMonitor", {
+    readonly effect: Effect.Effect<{
+        /**
+         * Register a worker's activity
+         */
+        recordActivity: (workerId: string, domain: string, activity: {
+            url?: string;
+            fetchStart?: boolean;
+        }) => Effect.Effect<void, never, never>;
+        /**
+         * Remove a worker from monitoring
+         */
+        removeWorker: (workerId: string) => Effect.Effect<void, never, never>;
+        /**
+         * Get stuck workers
+         */
+        getStuckWorkers: Effect.Effect<WorkerStatus[], never, never>;
+        /**
+         * Monitor workers and log stuck ones
+         */
+        startMonitoring: Effect.Effect<void, never, never>;
+    }, never, SpiderLogger>;
+}>;
+/**
+ * Monitors worker health and kills stuck workers
+ */
+export declare class WorkerHealthMonitor extends WorkerHealthMonitor_base {
+}
+export {};
+//# sourceMappingURL=WorkerHealthMonitor.service.d.ts.map

package/dist/lib/WorkerHealth/WorkerHealthMonitor.service.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"WorkerHealthMonitor.service.d.ts","sourceRoot":"","sources":["../../../src/lib/WorkerHealth/WorkerHealthMonitor.service.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAY,MAAM,EAAkC,MAAM,QAAQ,CAAC;AACpF,OAAO,EAAE,YAAY,EAAE,MAAM,oCAAoC,CAAC;AAElE,UAAU,YAAY;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,QAAQ,CAAC,GAAG,CAAC;IAC3B,cAAc,CAAC,EAAE,QAAQ,CAAC,GAAG,CAAC;CAC/B;;;QAcO;;WAEG;mCAES,MAAM,UACR,MAAM,YACJ;YAAE,GAAG,CAAC,EAAE,MAAM,CAAC;YAAC,UAAU,CAAC,EAAE,OAAO,CAAA;SAAE;QA4BlD;;WAEG;iCACsB,MAAM;QAG/B;;WAEG;;QAgBH;;WAEG;;;;AAxEX;;GAEG;AACH,qBAAa,mBAAoB,SAAQ,wBAwHxC;CAAG"}

package/dist/lib/api-facades.d.ts

@@ -0,0 +1,313 @@
+/**
+ * Clean API facades that hide Effect.Service implementation details.
+ *
+ * These interfaces provide clean documentation without exposing
+ * internal Effect service machinery.
+ *
+ * @group Services
+ */
+import { Effect, Sink } from 'effect';
+import { CrawlResult, CrawlTask } from './Spider/Spider.service.js';
+import { PriorityRequest, SpiderState, SpiderStateKey, StatePersistence } from './Scheduler/SpiderScheduler.service.js';
+import { SpiderMiddleware, SpiderRequest, SpiderResponse } from './Middleware/SpiderMiddleware.js';
+import { MiddlewareError } from './errors.js';
+/**
+ * The main Spider service interface for web crawling.
+ *
+ * Orchestrates the entire crawling process including URL validation,
+ * robots.txt checking, concurrent processing, and result streaming.
+ *
+ * @example
+ * ```typescript
+ * const program = Effect.gen(function* () {
+ *   const spider = yield* Spider;
+ *   const collectSink = Sink.forEach<CrawlResult>(result =>
+ *     Effect.sync(() => console.log(result.pageData.url))
+ *   );
+ *
+ *   const stats = yield* spider.crawl('https://example.com', collectSink);
+ *   console.log(`Crawled ${stats.totalPages} pages`);
+ * });
+ * ```
+ *
+ * @group Services
+ * @public
+ */
+export interface ISpider {
+    /**
+     * Starts crawling from the specified URL and processes results through the provided sink.
+     *
+     * @param urlString - The starting URL for crawling
+     * @param sink - Sink to process crawl results as they're produced
+     * @returns Effect containing crawl statistics (total pages, completion status)
+     */
+    crawl<A, E, R>(_urlString: string, _sink: Sink.Sink<A, CrawlResult, E, R>): Effect.Effect<{
+        totalPages: number;
+        completed: boolean;
+    }, Error>;
+    /**
+     * Returns the list of URLs that have been visited during crawling.
+     *
+     * @returns Effect containing array of visited URLs
+     */
+    getVisitedUrls(): Effect.Effect<string[]>;
+}
+/**
+ * The SpiderSchedulerService service interface for request scheduling and persistence.
+ *
+ * Manages request queuing, prioritization, and state persistence for
+ * resumable crawling operations.
+ *
+ * @example
+ * ```typescript
+ * const program = Effect.gen(function* () {
+ *   const scheduler = yield* SpiderSchedulerService;
+ *
+ *   // Configure persistence
+ *   const stateKey = new SpiderStateKey({
+ *     id: 'my-crawl',
+ *     timestamp: new Date(),
+ *     name: 'Example Crawl'
+ *   });
+ *
+ *   yield* scheduler.configurePersistence(persistence, stateKey);
+ *
+ *   // Queue requests with priority
+ *   yield* scheduler.enqueue({ url: 'https://example.com', depth: 0 }, 10);
+ *
+ *   // Process requests
+ *   const request = yield* scheduler.dequeue();
+ *   console.log(`Processing: ${request.request.url}`);
+ * });
+ * ```
+ *
+ * @group Services
+ * @public
+ */
+export interface ISpiderScheduler {
+    /**
+     * Configures the scheduler to use a specific persistence layer with a state key.
+     *
+     * @param persistence - Implementation of StatePersistence interface
+     * @param stateKey - Unique identifier for the crawl session
+     */
+    configurePersistence(_persistence: StatePersistence, _stateKey: SpiderStateKey): Effect.Effect<void>;
+    /**
+     * Removes persistence configuration, disabling state saving.
+     */
+    clearPersistence(): Effect.Effect<void>;
+    /**
+     * Adds a crawl task to the processing queue with optional priority.
+     *
+     * @param request - Crawl task containing URL and depth
+     * @param priority - Optional priority (higher numbers = higher priority, default: 0)
+     * @returns Effect containing boolean indicating if task was added (false if duplicate)
+     */
+    enqueue(_request: CrawlTask, _priority?: number): Effect.Effect<boolean>;
+    /**
+     * Retrieves the next highest-priority task from the queue.
+     *
+     * @returns Effect containing the next priority request
+     */
+    dequeue(): Effect.Effect<PriorityRequest>;
+    /**
+     * Returns the current number of tasks in the queue.
+     */
+    size(): Effect.Effect<number>;
+    /**
+     * Checks if the queue is empty.
+     */
+    isEmpty(): Effect.Effect<boolean>;
+    /**
+     * Returns the current scheduler state for persistence.
+     */
+    getState(): Effect.Effect<SpiderState>;
+    /**
+     * Restores the scheduler from a previously saved state.
+     *
+     * @param state - Complete state to restore from
+     */
+    restoreFromState(_state: SpiderState): Effect.Effect<void>;
+    /**
+     * Attempts to restore state from a persistence layer.
+     *
+     * @param persistence - Persistence layer to load from
+     * @param stateKey - State key to restore
+     * @returns Effect containing boolean indicating if state was successfully restored
+     */
+    restore(_persistence: StatePersistence, _stateKey: SpiderStateKey): Effect.Effect<boolean>;
+}
+/**
+ * The MiddlewareManager service interface for pipeline processing.
+ *
+ * Orchestrates the execution of middleware in the correct order for
+ * request processing, response handling, and error recovery.
+ *
+ * @example
+ * ```typescript
+ * const program = Effect.gen(function* () {
+ *   const manager = yield* MiddlewareManager;
+ *
+ *   const middleware = [
+ *     rateLimitMiddleware,
+ *     loggingMiddleware,
+ *     userAgentMiddleware
+ *   ];
+ *
+ *   const processedRequest = yield* manager.processRequest(request, middleware);
+ *   console.log('Request processed through middleware pipeline');
+ * });
+ * ```
+ *
+ * @group Services
+ * @public
+ */
+export interface IMiddlewareManager {
+    /**
+     * Processes a request through the middleware pipeline.
+     *
+     * @param request - The initial request to process
+     * @param middlewares - Array of middleware to apply
+     * @returns Effect containing the processed request
+     */
+    processRequest(_request: SpiderRequest, _middlewares: SpiderMiddleware[]): Effect.Effect<SpiderRequest, MiddlewareError>;
+    /**
+     * Processes a response through the middleware pipeline in reverse order.
+     *
+     * @param response - The response to process
+     * @param request - The original request (for context)
+     * @param middlewares - Array of middleware to apply
+     * @returns Effect containing the processed response
+     */
+    processResponse(_response: SpiderResponse, _request: SpiderRequest, _middlewares: SpiderMiddleware[]): Effect.Effect<SpiderResponse, MiddlewareError>;
+    /**
+     * Processes an exception through the middleware pipeline in reverse order.
+     *
+     * @param error - The error that occurred
+     * @param request - The request that caused the error
+     * @param middlewares - Array of middleware to apply
+     * @returns Effect containing a recovered response or null
+     */
+    processException(_error: Error, _request: SpiderRequest, _middlewares: SpiderMiddleware[]): Effect.Effect<SpiderResponse | null, MiddlewareError>;
+}
+/**
+ * Rate limiting middleware service interface.
+ *
+ * Provides rate limiting functionality for respectful crawling,
+ * controlling request frequency at both global and per-domain levels.
+ *
+ * @group Middleware
+ * @public
+ */
+export interface IRateLimitMiddleware {
+    /**
+     * Creates a rate limiting middleware with the specified configuration.
+     *
+     * @param config - Rate limiting configuration options
+     * @returns Configured middleware instance
+     *
+     * @example
+     * ```typescript
+     * const rateLimiter = yield* RateLimitMiddleware;
+     * const middleware = rateLimiter.create({
+     *   maxConcurrentRequests: 5,
+     *   maxRequestsPerSecondPerDomain: 2,
+     *   requestDelayMs: 250
+     * });
+     * ```
+     */
+    create(_config: {
+        maxConcurrentRequests: number;
+        maxRequestsPerSecondPerDomain: number;
+        requestDelayMs?: number;
+    }): SpiderMiddleware;
+}
+/**
+ * Logging middleware service interface.
+ *
+ * Provides logging functionality using Effect.Logger for debugging
+ * and monitoring crawling operations.
+ *
+ * @group Middleware
+ * @public
+ */
+export interface ILoggingMiddleware {
+    /**
+     * Creates a logging middleware with optional configuration.
+     *
+     * @param config - Optional logging configuration
+     * @returns Configured middleware instance
+     *
+     * @example
+     * ```typescript
+     * const logger = yield* LoggingMiddleware;
+     * const middleware = logger.create({
+     *   logRequests: true,
+     *   logResponses: true,
+     *   logLevel: 'info'
+     * });
+     * ```
+     */
+    create(_config?: {
+        logRequests?: boolean;
+        logResponses?: boolean;
+        logErrors?: boolean;
+        logLevel?: 'debug' | 'info' | 'warn' | 'error';
+    }): SpiderMiddleware;
+}
+/**
+ * User agent middleware service interface.
+ *
+ * Adds consistent User-Agent headers to all requests for
+ * proper identification of your crawler.
+ *
+ * @group Middleware
+ * @public
+ */
+export interface IUserAgentMiddleware {
+    /**
+     * Creates a User-Agent middleware with the specified user agent string.
+     *
+     * @param userAgent - User agent string to add to requests
+     * @returns Configured middleware instance
+     *
+     * @example
+     * ```typescript
+     * const userAgent = yield* UserAgentMiddleware;
+     * const middleware = userAgent.create('MyBot/1.0 (+https://example.com)');
+     * ```
+     */
+    create(_userAgent: string): SpiderMiddleware;
+}
+/**
+ * Statistics middleware service interface.
+ *
+ * Collects comprehensive metrics about crawling activity including
+ * request counts, response codes, and performance statistics.
+ *
+ * @group Middleware
+ * @public
+ */
+export interface IStatsMiddleware {
+    /**
+     * Creates a statistics middleware and returns both the middleware and a stats getter.
+     *
+     * @returns Object containing the middleware instance and statistics retrieval function
+     *
+     * @example
+     * ```typescript
+     * const statsService = yield* StatsMiddleware;
+     * const { middleware, getStats } = statsService.create();
+     *
+     * // Use middleware in your pipeline
+     * // Later get statistics
+     * const stats = yield* getStats();
+     * console.log(`Processed ${stats.requests_processed} requests`);
+     * ```
+     */
+    create(): {
+        middleware: SpiderMiddleware;
+        getStats: () => Effect.Effect<Record<string, number>>;
+    };
+}
+//# sourceMappingURL=api-facades.d.ts.map

package/dist/lib/api-facades.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"api-facades.d.ts","sourceRoot":"","sources":["../../src/lib/api-facades.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,QAAQ,CAAC;AACtC,OAAO,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,4BAA4B,CAAC;AACpE,OAAO,EACL,eAAe,EACf,WAAW,EACX,cAAc,EACd,gBAAgB,EACjB,MAAM,wCAAwC,CAAC;AAChD,OAAO,EACL,gBAAgB,EAChB,aAAa,EACb,cAAc,EACf,MAAM,kCAAkC,CAAC;AAC1C,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAE9C;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,MAAM,WAAW,OAAO;IACtB;;;;;;OAMG;IACH,KAAK,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EACX,UAAU,EAAE,MAAM,EAClB,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,CAAC,CAAC,GACrC,MAAM,CAAC,MAAM,CAAC;QAAE,UAAU,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,OAAO,CAAA;KAAE,EAAE,KAAK,CAAC,CAAC;IAEpE;;;;OAIG;IACH,cAAc,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;CAC3C;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AACH,MAAM,WAAW,gBAAgB;IAC/B;;;;;OAKG;IACH,oBAAoB,CAClB,YAAY,EAAE,gBAAgB,EAC9B,SAAS,EAAE,cAAc,GACxB,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAEvB;;OAEG;IACH,gBAAgB,IAAI,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAExC;;;;;;OAMG;IACH,OAAO,CAAC,QAAQ,EAAE,SAAS,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAEzE;;;;OAIG;IACH,OAAO,IAAI,MAAM,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;IAE1C;;OAEG;IACH,IAAI,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAE9B;;OAEG;IACH,OAAO,IAAI,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAElC;;OAEG;IACH,QAAQ,IAAI,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;IAEvC;;;;OAIG;IACH,gBAAgB,CAAC,MAAM,EAAE,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAE3D;;;;;;OAMG;IACH,OAAO,CACL,YAAY,EAAE,gBAAgB,EAC9B,SAAS,EAAE,cAAc,GACxB,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;CAC3B;AAED;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,MAAM,WAAW,kBAAkB;IACjC;;;;;;OAMG;IACH,cAAc,CACZ,QAAQ,EAAE,aAAa,EACvB,YAAY,EAAE,gBAAgB,EAAE,GAC/B,MAAM,CAAC,MAAM,CAAC,aAAa,EAAE,eAAe,CAAC,CAAC;IAEjD;;;;;;;OAOG;IACH,eAAe,CACb,SAAS,EAAE,cAAc,EACzB,QAAQ,EAAE,aAAa,EACvB,YAAY,EAAE,gBAAgB,EAAE,GAC/B,MAAM,CAAC,MAAM,CAAC,cAAc,EAAE,eAAe,CAAC,CAAC;IAElD;;;;;;;OAOG;IACH,gBAAgB,CACd,MAAM,EAAE,KAAK,EACb,QAAQ,EAAE,aAAa,EACvB,YAAY,EAAE,gBAAgB,EAAE,GAC/B,MAAM,CAAC,MAAM,CAAC,cAAc,GAAG,IAAI,EAAE,eAAe,CAAC,CAAC;CAC1D;AAED;;;;;;;;GAQG;AACH,MAAM,WAAW,oBAAoB;IACnC;;;;;;;;;;;;;;;OAeG;IACH,MAAM,CAAC,OAAO,EAAE;QACd,qBAAqB,EAAE,MAAM,CAAC;QAC9B,6BAA6B,EAAE,MAAM,CAAC;QACtC,cAAc,CAAC,EAAE,MAAM,CAAC;KACzB,GAAG,gBAAgB,CAAC;CACtB;AAED;;;;;;;;GAQG;AACH,MAAM,WAAW,kBAAkB;IACjC;;;;;;;;;;;;;;;OAeG;IACH,MAAM,CAAC,OAAO,CAAC,EAAE;QACf,WAAW,CAAC,EAAE,OAAO,CAAC;QACtB,YAAY,CAAC,EAAE,OAAO,CAAC;QACvB,SAAS,CAAC,EAAE,OAAO,CAAC;QACpB,QAAQ,CAAC,EAAE,OAAO,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC;KAChD,GAAG,gBAAgB,CAAC;CACtB;AAED;;;;;;;;GAQG;AACH,MAAM,WAAW,oBAAoB;IACnC;;;;;;;;;;;OAWG;IACH,MAAM,CAAC,UAAU,EAAE,MAAM,GAAG,gBAAgB,CAAC;CAC9C;AAED;;;;;;;;GAQG;AACH,MAAM,WAAW,gBAAgB;IAC/B;;;;;;;;;;;;;;;OAeG;IACH,MAAM,IAAI;QACR,UAAU,EAAE,gBAAgB,CAAC;QAC7B,QAAQ,EAAE,MAAM,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;KACvD,CAAC;CACH"}

package/dist/lib/errors/effect-errors.d.ts

@@ -0,0 +1,179 @@
+/**
+ * Effect-based Error Types
+ * Comprehensive error hierarchy using Data.TaggedError for type-safe error handling
+ */
+declare const SpiderError_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").Equals<A, {}> extends true ? void : { readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }) => import("effect/Cause").YieldableError & {
+    readonly _tag: "SpiderError";
+} & Readonly<A>;
+/**
+ * Base error class for all Spider errors
+ */
+export declare class SpiderError extends SpiderError_base<{
+    readonly operation: string;
+    readonly details?: unknown;
+    readonly cause?: unknown;
+}> {
+    get message(): string;
+}
+declare const NetworkError_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").Equals<A, {}> extends true ? void : { readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }) => import("effect/Cause").YieldableError & {
+    readonly _tag: "NetworkError";
+} & Readonly<A>;
+export declare class NetworkError extends NetworkError_base<{
+    readonly url: string;
+    readonly statusCode?: number;
+    readonly method?: string;
+    readonly cause?: unknown;
+}> {
+    get message(): string;
+    static fromResponse(url: string, response: Response): NetworkError;
+    static fromCause(url: string, cause: unknown): NetworkError;
+}
+declare const TimeoutError_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").Equals<A, {}> extends true ? void : { readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }) => import("effect/Cause").YieldableError & {
+    readonly _tag: "TimeoutError";
+} & Readonly<A>;
+export declare class TimeoutError extends TimeoutError_base<{
+    readonly url: string;
+    readonly timeoutMs: number;
+    readonly operation: string;
+}> {
+    get message(): string;
+}
+declare const ParseError_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").Equals<A, {}> extends true ? void : { readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }) => import("effect/Cause").YieldableError & {
+    readonly _tag: "ParseError";
+} & Readonly<A>;
+export declare class ParseError extends ParseError_base<{
+    readonly input?: string;
+    readonly expected: string;
+    readonly cause?: unknown;
+}> {
+    get message(): string;
+    static json(input: string, cause?: unknown): ParseError;
+    static html(input: string, cause?: unknown): ParseError;
+}
+declare const ValidationError_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").Equals<A, {}> extends true ? void : { readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }) => import("effect/Cause").YieldableError & {
+    readonly _tag: "ValidationError";
+} & Readonly<A>;
+export declare class ValidationError extends ValidationError_base<{
+    readonly field: string;
+    readonly value?: unknown;
+    readonly constraint: string;
+}> {
+    get message(): string;
+    static url(url: string): ValidationError;
+}
+declare const BrowserError_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").Equals<A, {}> extends true ? void : { readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }) => import("effect/Cause").YieldableError & {
+    readonly _tag: "BrowserError";
+} & Readonly<A>;
+export declare class BrowserError extends BrowserError_base<{
+    readonly operation: string;
+    readonly browserId?: string;
+    readonly cause?: unknown;
+}> {
+    get message(): string;
+    static notLaunched(): BrowserError;
+    static launchFailed(cause: unknown): BrowserError;
+}
+declare const PageError_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").Equals<A, {}> extends true ? void : { readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }) => import("effect/Cause").YieldableError & {
+    readonly _tag: "PageError";
+} & Readonly<A>;
+export declare class PageError extends PageError_base<{
+    readonly url: string;
+    readonly operation: string;
+    readonly selector?: string;
+    readonly cause?: unknown;
+}> {
+    get message(): string;
+}
+declare const StateError_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").Equals<A, {}> extends true ? void : { readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }) => import("effect/Cause").YieldableError & {
+    readonly _tag: "StateError";
+} & Readonly<A>;
+export declare class StateError extends StateError_base<{
+    readonly operation: 'save' | 'load' | 'delete' | 'update';
+    readonly stateKey?: string;
+    readonly cause?: unknown;
+}> {
+    get message(): string;
+}
+declare const SessionError_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").Equals<A, {}> extends true ? void : { readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }) => import("effect/Cause").YieldableError & {
+    readonly _tag: "SessionError";
+} & Readonly<A>;
+export declare class SessionError extends SessionError_base<{
+    readonly sessionId?: string;
+    readonly operation: string;
+    readonly cause?: unknown;
+}> {
+    get message(): string;
+    static noActiveSession(): SessionError;
+}
+declare const FileSystemError_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").Equals<A, {}> extends true ? void : { readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }) => import("effect/Cause").YieldableError & {
+    readonly _tag: "FileSystemError";
+} & Readonly<A>;
+export declare class FileSystemError extends FileSystemError_base<{
+    readonly path: string;
+    readonly operation: 'read' | 'write' | 'delete' | 'create';
+    readonly cause?: unknown;
+}> {
+    get message(): string;
+}
+declare const CrawlError_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").Equals<A, {}> extends true ? void : { readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }) => import("effect/Cause").YieldableError & {
+    readonly _tag: "CrawlError";
+} & Readonly<A>;
+export declare class CrawlError extends CrawlError_base<{
+    readonly url: string;
+    readonly depth: number;
+    readonly reason: string;
+    readonly cause?: unknown;
+}> {
+    get message(): string;
+    static maxDepthReached(url: string, depth: number): CrawlError;
+    static robotsBlocked(url: string): CrawlError;
+}
+declare const QueueError_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").Equals<A, {}> extends true ? void : { readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }) => import("effect/Cause").YieldableError & {
+    readonly _tag: "QueueError";
+} & Readonly<A>;
+export declare class QueueError extends QueueError_base<{
+    readonly operation: 'enqueue' | 'dequeue' | 'peek';
+    readonly queueSize?: number;
+    readonly cause?: unknown;
+}> {
+    get message(): string;
+}
+declare const ConfigError_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").Equals<A, {}> extends true ? void : { readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }) => import("effect/Cause").YieldableError & {
+    readonly _tag: "ConfigError";
+} & Readonly<A>;
+export declare class ConfigError extends ConfigError_base<{
+    readonly field: string;
+    readonly value?: unknown;
+    readonly reason: string;
+}> {
+    get message(): string;
+    static invalid(field: string, value: unknown, expected: string): ConfigError;
+}
+declare const MiddlewareError_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").Equals<A, {}> extends true ? void : { readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }) => import("effect/Cause").YieldableError & {
+    readonly _tag: "MiddlewareError";
+} & Readonly<A>;
+export declare class MiddlewareError extends MiddlewareError_base<{
+    readonly middlewareName: string;
+    readonly phase: 'request' | 'response' | 'error';
+    readonly cause?: unknown;
+}> {
+    get message(): string;
+}
+/**
+ * Type guard for Spider errors
+ */
+export declare const isSpiderError: (error: unknown) => error is SpiderError;
+/**
+ * Type guard for network-related errors
+ */
+export declare const isNetworkError: (error: unknown) => error is NetworkError | TimeoutError;
+/**
+ * Type guard for browser-related errors
+ */
+export declare const isBrowserError: (error: unknown) => error is BrowserError | PageError;
+/**
+ * Union type of all Spider errors
+ */
+export type AllSpiderErrors = SpiderError | NetworkError | TimeoutError | ParseError | ValidationError | BrowserError | PageError | StateError | SessionError | FileSystemError | CrawlError | QueueError | ConfigError | MiddlewareError;
+export {};
+//# sourceMappingURL=effect-errors.d.ts.map