@jambudipa/spider 0.1.0

package/dist/lib/LinkExtractor/LinkExtractor.service.d.ts

@@ -0,0 +1,166 @@
+import { Effect } from 'effect';
+/**
+ * Configuration for link extraction behavior.
+ *
+ * Focuses purely on HOW to extract links from HTML documents,
+ * not on processing or validating the extracted URLs.
+ *
+ * @example
+ * ```typescript
+ * // Extract from specific CSS selectors
+ * const config: LinkExtractorConfig = {
+ *   restrictCss: ['a.product-link', 'form[action]'],
+ *   tags: ['a', 'form'],
+ *   attrs: ['href', 'action']
+ * };
+ *
+ * // Extract from all standard elements
+ * const config: LinkExtractorConfig = {
+ *   tags: ['a', 'area', 'form', 'frame', 'iframe'],
+ *   attrs: ['href', 'action', 'src']
+ * };
+ * ```
+ *
+ * @group LinkExtractor
+ * @public
+ */
+export interface LinkExtractorConfig {
+    /**
+     * CSS selectors to restrict extraction to specific elements.
+     * If specified, only elements matching these selectors will be processed.
+     *
+     * @example
+     * ```typescript
+     * restrictCss: [
+     *   'a.product-link',     // Only product links
+     *   '.content a',         // Links within content area
+     *   'form[method="post"]' // POST forms only
+     * ]
+     * ```
+     */
+    readonly restrictCss?: string[];
+    /**
+     * HTML tag names to extract links from.
+     * Defaults to common link-containing elements.
+     *
+     * @example ['a', 'area', 'form', 'frame', 'iframe', 'link']
+     */
+    readonly tags?: string[];
+    /**
+     * HTML attributes to extract URLs from.
+     * Defaults to common URL-containing attributes.
+     *
+     * @example ['href', 'action', 'src', 'data-url']
+     */
+    readonly attrs?: string[];
+    /**
+     * Whether to extract URLs from form input elements.
+     * Looks for hidden inputs with URL-like names/values.
+     *
+     * @default false
+     */
+    readonly extractFromInputs?: boolean;
+}
+/**
+ * Result of link extraction from an HTML document.
+ *
+ * Contains the raw extracted URLs without any processing or validation.
+ *
+ * @group LinkExtractor
+ * @public
+ */
+export interface LinkExtractionResult {
+    /**
+     * Raw URLs extracted from the HTML document.
+     * These are unprocessed and may be relative URLs, fragments, etc.
+     */
+    readonly links: string[];
+    /**
+     * Total number of potential URL-containing elements found.
+     * Includes elements that didn't yield valid URLs.
+     */
+    readonly totalElementsProcessed: number;
+    /**
+     * Breakdown of extraction by element type.
+     * Maps element types to the number of URLs extracted from them.
+     */
+    readonly extractionBreakdown: Record<string, number>;
+}
+declare const LinkExtractionError_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").Equals<A, {}> extends true ? void : { readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }) => import("effect/Cause").YieldableError & {
+    readonly _tag: "LinkExtractionError";
+} & Readonly<A>;
+/**
+ * Error that can occur during link extraction.
+ *
+ * @group Errors
+ * @public
+ */
+export declare class LinkExtractionError extends LinkExtractionError_base<{
+    readonly message: string;
+    readonly cause?: unknown;
+}> {
+}
+/**
+ * Service interface for extracting links from HTML documents.
+ *
+ * This service focuses purely on extraction - it does not process,
+ * validate, or filter the extracted URLs in any way.
+ *
+ * @group Services
+ * @public
+ */
+export interface LinkExtractorServiceInterface {
+    /**
+     * Extracts all URLs from an HTML document based on configuration.
+     *
+     * This method only extracts URLs from the HTML - it does not:
+     * - Validate URLs
+     * - Resolve relative URLs to absolute URLs
+     * - Apply domain or pattern filtering
+     * - Canonicalize URLs
+     *
+     * URL processing should be handled separately by the consumer.
+     *
+     * @param html - The HTML content to extract links from
+     * @param config - Configuration for extraction behavior
+     * @returns Effect containing the extraction result
+     *
+     * @example
+     * ```typescript
+     * const extractor = yield* LinkExtractorService;
+     * const result = yield* extractor.extractLinks(htmlContent, {
+     *   tags: ['a', 'form'],
+     *   attrs: ['href', 'action'],
+     *   restrictCss: ['.content a']
+     * });
+     *
+     * console.log(`Found ${result.links.length} raw URLs`);
+     * // URLs may be relative, absolute, fragments, etc.
+     * ```
+     */
+    extractLinks(html: string, config?: LinkExtractorConfig): Effect.Effect<LinkExtractionResult, LinkExtractionError>;
+}
+declare const LinkExtractorService_base: Effect.Service.Class<LinkExtractorService, "@jambudipa.io/LinkExtractorService", {
+    readonly effect: Effect.Effect<{
+        extractLinks: (html: string, config?: LinkExtractorConfig) => Effect.Effect<LinkExtractionResult, LinkExtractionError, never>;
+    }, never, never>;
+}>;
+/**
+ * Implementation of the LinkExtractorService.
+ *
+ * Provides pure HTML link extraction without any URL processing.
+ *
+ * @group Services
+ * @public
+ */
+export declare class LinkExtractorService extends LinkExtractorService_base {
+}
+/**
+ * Default layer for LinkExtractorService.
+ *
+ * @group Layers
+ * @public
+ */
+export declare const LinkExtractorServiceLayer: import("effect/Layer").Layer<LinkExtractorService, never, never>;
+export {};
+//# sourceMappingURL=LinkExtractor.service.d.ts.map

package/dist/lib/LinkExtractor/LinkExtractor.service.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"LinkExtractor.service.d.ts","sourceRoot":"","sources":["../../../src/lib/LinkExtractor/LinkExtractor.service.ts"],"names":[],"mappings":"AAAA,OAAO,EAAQ,MAAM,EAAE,MAAM,QAAQ,CAAC;AAItC;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,MAAM,WAAW,mBAAmB;IAClC;;;;;;;;;;;;OAYG;IACH,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IAEhC;;;;;OAKG;IACH,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAEzB;;;;;OAKG;IACH,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;IAE1B;;;;;OAKG;IACH,QAAQ,CAAC,iBAAiB,CAAC,EAAE,OAAO,CAAC;CACtC;AAED;;;;;;;GAOG;AACH,MAAM,WAAW,oBAAoB;IACnC;;;OAGG;IACH,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,CAAC;IAEzB;;;OAGG;IACH,QAAQ,CAAC,sBAAsB,EAAE,MAAM,CAAC;IAExC;;;OAGG;IACH,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CACtD;;;;AAED;;;;;GAKG;AACH,qBAAa,mBAAoB,SAAQ,yBAEvC;IACA,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,KAAK,CAAC,EAAE,OAAO,CAAC;CAC1B,CAAC;CAAG;AAEL;;;;;;;;GAQG;AACH,MAAM,WAAW,6BAA6B;IAC5C;;;;;;;;;;;;;;;;;;;;;;;;;;;OA2BG;IACH,YAAY,CACV,IAAI,EAAE,MAAM,EACZ,MAAM,CAAC,EAAE,mBAAmB,GAC3B,MAAM,CAAC,MAAM,CAAC,oBAAoB,EAAE,mBAAmB,CAAC,CAAC;CAC7D;;;6BAyB0B,MAAM,WAAW,mBAAmB;;;AAZ/D;;;;;;;GAOG;AACH,qBAAa,oBAAqB,SAAQ,yBAsBzC;CAAG;AAEJ;;;;;GAKG;AACH,eAAO,MAAM,yBAAyB,kEAA+B,CAAC"}

package/dist/lib/LinkExtractor/index.d.ts

@@ -0,0 +1,37 @@
+/**
+ * Advanced link extraction functionality for the Spider framework.
+ *
+ * This module provides Scrapy-equivalent link extraction capabilities with support for:
+ * - CSS selector-based extraction
+ * - Pattern-based filtering (allow/deny regex patterns)
+ * - Domain-based filtering
+ * - URL canonicalization
+ * - Duplicate removal
+ * - Comprehensive extraction statistics
+ *
+ * @example
+ * ```typescript
+ * import { LinkExtractorService, type LinkExtractorConfig } from '@jambudipa.io/spider/LinkExtractor';
+ *
+ * const program = Effect.gen(function* () {
+ *   const extractor = yield* LinkExtractorService;
+ *
+ *   const result = yield* extractor.extractLinks(
+ *     htmlContent,
+ *     'https://example.com',
+ *     {
+ *       allowPatterns: [/\/articles\/\d+/],
+ *       restrictCss: ['.content a'],
+ *       canonicalize: true
+ *     }
+ *   );
+ *
+ *   console.log(`Extracted ${result.links.length} links`);
+ * });
+ * ```
+ *
+ * @group LinkExtractor
+ * @public
+ */
+export { LinkExtractorService, LinkExtractorServiceLayer, type LinkExtractorConfig, type LinkExtractionResult, type LinkExtractorServiceInterface, LinkExtractionError, } from './LinkExtractor.service.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/lib/LinkExtractor/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/lib/LinkExtractor/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkCG;AAEH,OAAO,EACL,oBAAoB,EACpB,yBAAyB,EACzB,KAAK,mBAAmB,EACxB,KAAK,oBAAoB,EACzB,KAAK,6BAA6B,EAClC,mBAAmB,GACpB,MAAM,4BAA4B,CAAC"}

package/dist/lib/Logging/FetchLogger.d.ts

@@ -0,0 +1,8 @@
+import { Context, Effect } from 'effect';
+import { SpiderLogger } from './SpiderLogger.service.js';
+/**
+ * Wrapper for fetch that adds comprehensive logging
+ */
+export declare const makeLoggingFetch: Effect.Effect<(url: string, options?: RequestInit) => Promise<Response>, never, SpiderLogger>;
+export declare const LoggingFetch: Context.Tag<(url: string, options?: RequestInit) => Promise<Response>, (url: string, options?: RequestInit) => Promise<Response>>;
+//# sourceMappingURL=FetchLogger.d.ts.map

package/dist/lib/Logging/FetchLogger.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"FetchLogger.d.ts","sourceRoot":"","sources":["../../../src/lib/Logging/FetchLogger.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAC;AAEzD;;GAEG;AACH,eAAO,MAAM,gBAAgB,sBAGd,MAAM,YAAY,WAAW,4CAgG1C,CAAC;AAEH,eAAO,MAAM,YAAY,oBACE,MAAM,YAAY,WAAW,KAAK,OAAO,CAAC,QAAQ,CAAC,QAAnD,MAAM,YAAY,WAAW,KAAK,OAAO,CAAC,QAAQ,CAAC,CAE3E,CAAC"}

package/dist/lib/Logging/SpiderLogger.service.d.ts

@@ -0,0 +1,34 @@
+import { Context, Effect, Layer } from 'effect';
+export interface SpiderLogEvent {
+    timestamp: string;
+    type: 'domain_start' | 'domain_complete' | 'domain_error' | 'page_scraped' | 'queue_status' | 'worker_status' | 'rate_limit' | 'spider_lifecycle' | 'worker_lifecycle' | 'worker_state' | 'completion_monitor' | 'edge_case' | 'crawl_delay_capped';
+    domain?: string;
+    url?: string;
+    workerId?: string;
+    fiberId?: string;
+    message: string;
+    details?: Record<string, unknown>;
+}
+export interface SpiderLogger {
+    readonly logEvent: (event: Omit<SpiderLogEvent, 'timestamp'>) => Effect.Effect<void>;
+    readonly logDomainStart: (domain: string, startUrl: string) => Effect.Effect<void>;
+    readonly logDomainComplete: (domain: string, pagesScraped: number, reason: 'max_pages' | 'queue_empty' | 'error') => Effect.Effect<void>;
+    readonly logPageScraped: (url: string, domain: string, pageNumber: number) => Effect.Effect<void>;
+    readonly logQueueStatus: (domain: string, queueSize: number, activeWorkers: number) => Effect.Effect<void>;
+    readonly logRateLimit: (domain: string, requestsInWindow: number) => Effect.Effect<void>;
+    readonly logSpiderLifecycle: (event: 'start' | 'complete' | 'error', details?: Record<string, unknown>) => Effect.Effect<void>;
+    readonly logWorkerLifecycle: (workerId: string, domain: string, event: 'created' | 'entering_loop' | 'exiting_loop', reason?: string, details?: Record<string, unknown>) => Effect.Effect<void>;
+    readonly logWorkerState: (workerId: string, domain: string, event: 'taking_task' | 'marked_active' | 'marked_idle' | 'task_completed', details?: Record<string, unknown>) => Effect.Effect<void>;
+    readonly logCompletionMonitor: (domain: string, checkCount: number, queueSize: number, activeWorkers: number, stableCount: number, maxPagesReached: boolean, decision: string) => Effect.Effect<void>;
+    readonly logEdgeCase: (domain: string, caseType: string, details?: Record<string, unknown>) => Effect.Effect<void>;
+    readonly logDomainStatus: (domain: string, status: {
+        pagesScraped: number;
+        queueSize: number;
+        activeWorkers: number;
+        maxWorkers: number;
+    }) => Effect.Effect<void>;
+}
+export declare const SpiderLogger: Context.Tag<SpiderLogger, SpiderLogger>;
+export declare const makeSpiderLogger: (logDir?: string) => SpiderLogger;
+export declare const SpiderLoggerLive: Layer.Layer<SpiderLogger, never, never>;
+//# sourceMappingURL=SpiderLogger.service.d.ts.map

package/dist/lib/Logging/SpiderLogger.service.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"SpiderLogger.service.d.ts","sourceRoot":"","sources":["../../../src/lib/Logging/SpiderLogger.service.ts"],"names":[],"mappings":"AAAA,OAAO,EAAW,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,QAAQ,CAAC;AAIzD,MAAM,WAAW,cAAc;IAC7B,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EACA,cAAc,GACd,iBAAiB,GACjB,cAAc,GACd,cAAc,GACd,cAAc,GACd,eAAe,GACf,YAAY,GACZ,kBAAkB,GAClB,kBAAkB,GAClB,cAAc,GACd,oBAAoB,GACpB,WAAW,GACX,oBAAoB,CAAC;IACzB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACnC;AAED,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,QAAQ,EAAE,CACjB,KAAK,EAAE,IAAI,CAAC,cAAc,EAAE,WAAW,CAAC,KACrC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACzB,QAAQ,CAAC,cAAc,EAAE,CACvB,MAAM,EAAE,MAAM,EACd,QAAQ,EAAE,MAAM,KACb,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACzB,QAAQ,CAAC,iBAAiB,EAAE,CAC1B,MAAM,EAAE,MAAM,EACd,YAAY,EAAE,MAAM,EACpB,MAAM,EAAE,WAAW,GAAG,aAAa,GAAG,OAAO,KAC1C,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACzB,QAAQ,CAAC,cAAc,EAAE,CACvB,GAAG,EAAE,MAAM,EACX,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,KACf,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACzB,QAAQ,CAAC,cAAc,EAAE,CACvB,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,EACjB,aAAa,EAAE,MAAM,KAClB,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACzB,QAAQ,CAAC,YAAY,EAAE,CACrB,MAAM,EAAE,MAAM,EACd,gBAAgB,EAAE,MAAM,KACrB,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACzB,QAAQ,CAAC,kBAAkB,EAAE,CAC3B,KAAK,EAAE,OAAO,GAAG,UAAU,GAAG,OAAO,EACrC,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAC9B,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAGzB,QAAQ,CAAC,kBAAkB,EAAE,CAC3B,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,SAAS,GAAG,eAAe,GAAG,cAAc,EACnD,MAAM,CAAC,EAAE,MAAM,EACf,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAC9B,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACzB,QAAQ,CAAC,cAAc,EAAE,CACvB,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,aAAa,GAAG,eAAe,GAAG,aAAa,GAAG,gBAAgB,EACzE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAC9B,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACzB,QAAQ,CAAC,oBAAoB,EAAE,CAC7B,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,SAAS,EAAE,MAAM,EACjB,aAAa,EAAE,MAAM,EACrB,WAAW,EAAE,MAAM,EACnB,eAAe,EAAE,OAAO,EACxB,QAAQ,EAAE,MAAM,KACb,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACzB,QAAQ,CAAC,WAAW,EAAE,CACpB,MAAM,EAAE,MAAM,EACd,QAAQ,EAAE,MAAM,EAChB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAC9B,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACzB,QAAQ,CAAC,eAAe,EAAE,CACxB,MAAM,EAAE,MAAM,EACd,MAAM,EAAE;QACN,YAAY,EAAE,MAAM,CAAC;QACrB,SAAS,EAAE,MAAM,CAAC;QAClB,aAAa,EAAE,MAAM,CAAC;QACtB,UAAU,EAAE,MAAM,CAAC;KACpB,KACE,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;CAC1B;AAED,eAAO,MAAM,YAAY,yCAAmD,CAAC;AAE7E,eAAO,MAAM,gBAAgB,GAAI,eAAwB,KAAG,YA2R3D,CAAC;AAEF,eAAO,MAAM,gBAAgB,yCAAkD,CAAC"}

package/dist/lib/Middleware/SpiderMiddleware.d.ts

@@ -0,0 +1,276 @@
+import { Effect } from 'effect';
+import { CrawlTask } from '../Spider/Spider.service.js';
+import { PageData } from '../PageData/PageData.js';
+import { MiddlewareError } from '../errors.js';
+/**
+ * Request object used in the middleware pipeline.
+ *
+ * Contains the crawl task along with optional headers and metadata
+ * that can be modified by middleware during processing.
+ *
+ * @group Interfaces
+ * @public
+ */
+export interface SpiderRequest {
+    /** The crawl task containing URL and depth information */
+    task: CrawlTask;
+    /** HTTP headers to include with the request */
+    headers?: Record<string, string>;
+    /** Additional metadata that can be used by middleware */
+    meta?: Record<string, unknown>;
+}
+/**
+ * Response object used in the middleware pipeline.
+ *
+ * Contains the extracted page data along with optional HTTP response
+ * information and metadata from middleware processing.
+ *
+ * @group Interfaces
+ * @public
+ */
+export interface SpiderResponse {
+    /** The extracted page data including content, links, and metadata */
+    pageData: PageData;
+    /** HTTP status code of the response */
+    statusCode?: number;
+    /** HTTP response headers */
+    headers?: Record<string, string>;
+    /** Additional metadata added by middleware */
+    meta?: Record<string, unknown>;
+}
+/**
+ * Interface for implementing custom middleware components.
+ *
+ * Middleware can intercept and modify requests before they're sent,
+ * responses after they're received, and handle exceptions that occur
+ * during processing. All methods are optional.
+ *
+ * @example
+ * ```typescript
+ * const loggingMiddleware: SpiderMiddleware = {
+ *   processRequest: (request) => Effect.gen(function* () {
+ *     console.log(`Requesting: ${request.task.url}`);
+ *     return request;
+ *   }),
+ *
+ *   processResponse: (response, request) => Effect.gen(function* () {
+ *     console.log(`Response: ${response.statusCode} for ${request.task.url}`);
+ *     return response;
+ *   }),
+ *
+ *   processException: (error, request) => Effect.gen(function* () {
+ *     console.error(`Error processing ${request.task.url}: ${error.message}`);
+ *     return null; // Let the error propagate
+ *   })
+ * };
+ * ```
+ *
+ * @group Interfaces
+ * @public
+ */
+export interface SpiderMiddleware {
+    /**
+     * Process a request before it's sent to the target server.
+     * Can modify headers, metadata, or reject the request entirely.
+     */
+    processRequest?: (request: SpiderRequest) => Effect.Effect<SpiderRequest, MiddlewareError>;
+    /**
+     * Process a response after it's received from the target server.
+     * Can modify the response data or metadata.
+     */
+    processResponse?: (response: SpiderResponse, request: SpiderRequest) => Effect.Effect<SpiderResponse, MiddlewareError>;
+    /**
+     * Handle exceptions that occur during request processing.
+     * Can attempt recovery by returning a SpiderResponse, or return null to propagate the error.
+     */
+    processException?: (error: Error, request: SpiderRequest) => Effect.Effect<SpiderResponse | null, MiddlewareError>;
+}
+declare const MiddlewareManager_base: Effect.Service.Class<MiddlewareManager, "@jambudipa.io/MiddlewareManager", {
+    readonly effect: Effect.Effect<{
+        /**
+         * Processes a request through the middleware pipeline.
+         *
+         * Middleware are executed in order from first to last, with each middleware
+         * receiving the output of the previous middleware as input.
+         *
+         * @param request - The initial request to process
+         * @param middlewares - Array of middleware to apply
+         * @returns Effect containing the processed request
+         */
+        processRequest: (request: SpiderRequest, middlewares: SpiderMiddleware[]) => Effect.Effect<SpiderRequest, MiddlewareError, never>;
+        /**
+         * Processes a response through the middleware pipeline in reverse order.
+         *
+         * Middleware are executed in reverse order (last to first) to provide
+         * proper nesting of response processing.
+         *
+         * @param response - The response to process
+         * @param request - The original request (for context)
+         * @param middlewares - Array of middleware to apply
+         * @returns Effect containing the processed response
+         */
+        processResponse: (response: SpiderResponse, request: SpiderRequest, middlewares: SpiderMiddleware[]) => Effect.Effect<SpiderResponse, MiddlewareError, never>;
+        /**
+         * Processes an exception through the middleware pipeline in reverse order.
+         *
+         * Middleware are given a chance to handle or recover from exceptions.
+         * If a middleware returns a SpiderResponse, it indicates successful recovery.
+         * If it returns null, the exception continues to propagate.
+         *
+         * @param error - The error that occurred
+         * @param request - The request that caused the error
+         * @param middlewares - Array of middleware to apply
+         * @returns Effect containing a recovered response or null
+         */
+        processException: (error: Error, request: SpiderRequest, middlewares: SpiderMiddleware[]) => Effect.Effect<SpiderResponse | null, MiddlewareError, never>;
+    }, never, never>;
+}>;
+/**
+ * Manages the middleware pipeline for request and response processing.
+ *
+ * The MiddlewareManager orchestrates the execution of middleware in the correct order:
+ * - Requests are processed forward through the middleware array
+ * - Responses are processed in reverse order (last middleware first)
+ * - Exceptions are processed in reverse order for proper error handling
+ *
+ * @example
+ * ```typescript
+ * const program = Effect.gen(function* () {
+ *   const manager = yield* MiddlewareManager;
+ *
+ *   const middleware = [
+ *     rateLimitMiddleware,
+ *     loggingMiddleware,
+ *     userAgentMiddleware
+ *   ];
+ *
+ *   const request: SpiderRequest = {
+ *     task: { url: 'https://example.com', depth: 0 },
+ *     headers: {}
+ *   };
+ *
+ *   const processedRequest = yield* manager.processRequest(request, middleware);
+ *   console.log('Request processed through middleware pipeline');
+ * });
+ * ```
+ *
+ * @group Services
+ * @public
+ */
+export declare class MiddlewareManager extends MiddlewareManager_base {
+}
+declare const RateLimitMiddleware_base: Effect.Service.Class<RateLimitMiddleware, "@jambudipa.io/RateLimitMiddleware", {
+    readonly effect: Effect.Effect<{
+        create: (config: {
+            maxConcurrentRequests: number;
+            maxRequestsPerSecondPerDomain: number;
+            requestDelayMs?: number;
+        }) => SpiderMiddleware;
+    }, never, never>;
+}>;
+/**
+ * Provides rate limiting functionality for respectful crawling.
+ *
+ * Controls request frequency at both global and per-domain levels to prevent
+ * overwhelming target servers and avoid being blocked.
+ *
+ * @example
+ * ```typescript
+ * const rateLimiter = yield* RateLimitMiddleware;
+ * const middleware = rateLimiter.create({
+ *   maxConcurrentRequests: 5,
+ *   maxRequestsPerSecondPerDomain: 2,
+ *   requestDelayMs: 250
+ * });
+ * ```
+ *
+ * @group Middleware
+ * @public
+ */
+export declare class RateLimitMiddleware extends RateLimitMiddleware_base {
+}
+declare const LoggingMiddleware_base: Effect.Service.Class<LoggingMiddleware, "@jambudipa.io/LoggingMiddleware", {
+    readonly effect: Effect.Effect<{
+        create: (config?: {
+            logRequests?: boolean;
+            logResponses?: boolean;
+            logErrors?: boolean;
+            logLevel?: "debug" | "info" | "warn" | "error";
+        }) => SpiderMiddleware;
+    }, never, never>;
+}>;
+/**
+ * Provides logging functionality using Effect.Logger.
+ *
+ * Logs requests, responses, and errors at configurable levels for debugging
+ * and monitoring purposes.
+ *
+ * @example
+ * ```typescript
+ * const logger = yield* LoggingMiddleware;
+ * const middleware = logger.create({
+ *   logRequests: true,
+ *   logResponses: true,
+ *   logLevel: 'info'
+ * });
+ * ```
+ *
+ * @group Middleware
+ * @public
+ */
+export declare class LoggingMiddleware extends LoggingMiddleware_base {
+}
+declare const UserAgentMiddleware_base: Effect.Service.Class<UserAgentMiddleware, "@jambudipa.io/UserAgentMiddleware", {
+    readonly effect: Effect.Effect<{
+        create: (userAgent: string) => SpiderMiddleware;
+    }, never, never>;
+}>;
+/**
+ * Adds User-Agent headers to requests.
+ *
+ * Sets a consistent User-Agent string for all requests to identify
+ * your crawler to web servers.
+ *
+ * @example
+ * ```typescript
+ * const userAgent = yield* UserAgentMiddleware;
+ * const middleware = userAgent.create('MyBot/1.0 (+https://example.com)');
+ * ```
+ *
+ * @group Middleware
+ * @public
+ */
+export declare class UserAgentMiddleware extends UserAgentMiddleware_base {
+}
+declare const StatsMiddleware_base: Effect.Service.Class<StatsMiddleware, "@jambudipa.io/StatsMiddleware", {
+    readonly effect: Effect.Effect<{
+        create: () => {
+            middleware: SpiderMiddleware;
+            getStats: () => Effect.Effect<Record<string, number>>;
+        };
+    }, never, never>;
+}>;
+/**
+ * Collects statistics about crawling activity.
+ *
+ * Tracks various metrics including requests processed, response codes,
+ * bytes downloaded, and processing times for monitoring and optimization.
+ *
+ * @example
+ * ```typescript
+ * const statsService = yield* StatsMiddleware;
+ * const { middleware, getStats } = statsService.create();
+ *
+ * // Use middleware in your pipeline
+ * // Later get statistics
+ * const stats = yield* getStats();
+ * console.log(`Processed ${stats.requests_processed} requests`);
+ * ```
+ *
+ * @group Middleware
+ * @public
+ */
+export declare class StatsMiddleware extends StatsMiddleware_base {
+}
+export {};
+//# sourceMappingURL=SpiderMiddleware.d.ts.map

package/dist/lib/Middleware/SpiderMiddleware.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"SpiderMiddleware.d.ts","sourceRoot":"","sources":["../../../src/lib/Middleware/SpiderMiddleware.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAA0B,MAAM,QAAQ,CAAC;AACxD,OAAO,EAAE,SAAS,EAAE,MAAM,6BAA6B,CAAC;AACxD,OAAO,EAAE,QAAQ,EAAE,MAAM,yBAAyB,CAAC;AACnD,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAE/C;;;;;;;;GAQG;AACH,MAAM,WAAW,aAAa;IAC5B,0DAA0D;IAC1D,IAAI,EAAE,SAAS,CAAC;IAChB,+CAA+C;IAC/C,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,yDAAyD;IACzD,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAChC;AAED;;;;;;;;GAQG;AACH,MAAM,WAAW,cAAc;IAC7B,qEAAqE;IACrE,QAAQ,EAAE,QAAQ,CAAC;IACnB,uCAAuC;IACvC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,4BAA4B;IAC5B,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,8CAA8C;IAC9C,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAChC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACH,MAAM,WAAW,gBAAgB;IAC/B;;;OAGG;IACH,cAAc,CAAC,EAAE,CACf,OAAO,EAAE,aAAa,KACnB,MAAM,CAAC,MAAM,CAAC,aAAa,EAAE,eAAe,CAAC,CAAC;IAEnD;;;OAGG;IACH,eAAe,CAAC,EAAE,CAChB,QAAQ,EAAE,cAAc,EACxB,OAAO,EAAE,aAAa,KACnB,MAAM,CAAC,MAAM,CAAC,cAAc,EAAE,eAAe,CAAC,CAAC;IAEpD;;;OAGG;IACH,gBAAgB,CAAC,EAAE,CACjB,KAAK,EAAE,KAAK,EACZ,OAAO,EAAE,aAAa,KACnB,MAAM,CAAC,MAAM,CAAC,cAAc,GAAG,IAAI,EAAE,eAAe,CAAC,CAAC;CAC5D;;;QAsCK;;;;;;;;;WASG;kCAEQ,aAAa,eACT,gBAAgB,EAAE;QAQjC;;;;;;;;;;WAUG;oCAES,cAAc,WACf,aAAa,eACT,gBAAgB,EAAE;QAWjC;;;;;;;;;;;WAWG;kCAEM,KAAK,WACH,aAAa,eACT,gBAAgB,EAAE;;;AAhGvC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AACH,qBAAa,iBAAkB,SAAQ,sBA4EtC;CAAG;;;yBA8BqB;YACf,qBAAqB,EAAE,MAAM,CAAC;YAC9B,6BAA6B,EAAE,MAAM,CAAC;YACtC,cAAc,CAAC,EAAE,MAAM,CAAC;SACzB,KAAG,gBAAgB;;;AAhC5B;;;;;;;;;;;;;;;;;;GAkBG;AACH,qBAAa,mBAAoB,SAAQ,wBAmExC;CAAG;;;0BA0BY;YACN,WAAW,CAAC,EAAE,OAAO,CAAC;YACtB,YAAY,CAAC,EAAE,OAAO,CAAC;YACvB,SAAS,CAAC,EAAE,OAAO,CAAC;YACpB,QAAQ,CAAC,EAAE,OAAO,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC;SAChD,KACA,gBAAgB;;;AA9BzB;;;;;;;;;;;;;;;;;;GAkBG;AACH,qBAAa,iBAAkB,SAAQ,sBA4EtC;CAAG;;;4BAqBsB,MAAM,KAAG,gBAAgB;;;AAnBnD;;;;;;;;;;;;;;GAcG;AACH,qBAAa,mBAAoB,SAAQ,wBAgBxC;CAAG;;;sBA0Bc;YACV,UAAU,EAAE,gBAAgB,CAAC;YAC7B,QAAQ,EAAE,MAAM,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;SACvD;;;AA3BP;;;;;;;;;;;;;;;;;;;GAmBG;AACH,qBAAa,eAAgB,SAAQ,oBA4DpC;CAAG"}

package/dist/lib/PageData/PageData.d.ts

@@ -0,0 +1,28 @@
+import { Schema } from 'effect';
+export declare const PageDataSchema: Schema.Struct<{
+    url: Schema.filter<typeof Schema.String>;
+    html: typeof Schema.String;
+    title: Schema.optional<typeof Schema.String>;
+    /** All available metadata from meta tags */
+    metadata: Schema.Record$<typeof Schema.String, typeof Schema.String>;
+    /** Commonly used metadata fields for convenience */
+    commonMetadata: Schema.optional<Schema.Struct<{
+        description: Schema.optional<typeof Schema.String>;
+        keywords: Schema.optional<typeof Schema.String>;
+        author: Schema.optional<typeof Schema.String>;
+        robots: Schema.optional<typeof Schema.String>;
+    }>>;
+    statusCode: Schema.filter<Schema.filter<typeof Schema.Number>>;
+    /** All response headers */
+    headers: Schema.Record$<typeof Schema.String, typeof Schema.String>;
+    /** When the fetch operation started */
+    fetchedAt: typeof Schema.DateFromSelf;
+    /** How long the entire fetch and parse operation took in milliseconds */
+    scrapeDurationMs: typeof Schema.Number;
+    /** The crawl depth (number of hops from the starting URL) */
+    depth: Schema.filter<Schema.filter<typeof Schema.Number>>;
+    /** Optional extracted data from the page */
+    extractedData: Schema.optional<Schema.Record$<typeof Schema.String, typeof Schema.Unknown>>;
+}>;
+export type PageData = Schema.Schema.Type<typeof PageDataSchema>;
+//# sourceMappingURL=PageData.d.ts.map

package/dist/lib/PageData/PageData.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"PageData.d.ts","sourceRoot":"","sources":["../../../src/lib/PageData/PageData.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAEhC,eAAO,MAAM,cAAc;;;;IAkBzB,4CAA4C;;IAE5C,oDAAoD;;;;;;;;IAUpD,2BAA2B;;IAE3B,uCAAuC;;IAEvC,yEAAyE;;IAEzE,6DAA6D;;IAE7D,4CAA4C;;EAI5C,CAAC;AAEH,MAAM,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,cAAc,CAAC,CAAC"}