@jambudipa/spider 0.2.1 → 0.2.3

package/dist/lib/HttpClient/TokenExtractor.d.ts

@@ -0,0 +1,83 @@
+/**
+ * Token Extractor Service
+ * Extracts and manages various types of tokens from HTTP responses
+ */
+import { Context, Effect, Layer, Option } from 'effect';
+import { StateManager, TokenType } from '../StateManager/StateManager.service.js';
+import { EnhancedHttpClient, type HttpResponse } from './EnhancedHttpClient.js';
+import { SpiderLogger } from '../Logging/SpiderLogger.service.js';
+import { NetworkError, ParseError, TimeoutError } from '../errors/effect-errors.js';
+declare const TokenNotAvailableError_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").Equals<A, {}> extends true ? void : { readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }) => import("effect/Cause").YieldableError & {
+    readonly _tag: "TokenNotAvailableError";
+} & Readonly<A>;
+export declare class TokenNotAvailableError extends TokenNotAvailableError_base<{
+    readonly message: string;
+}> {
+}
+declare const TokenRefreshError_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").Equals<A, {}> extends true ? void : { readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }) => import("effect/Cause").YieldableError & {
+    readonly _tag: "TokenRefreshError";
+} & Readonly<A>;
+export declare class TokenRefreshError extends TokenRefreshError_base<{
+    readonly message: string;
+    readonly tokenType: TokenType;
+}> {
+}
+declare const NoRefreshUrlError_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").Equals<A, {}> extends true ? void : { readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }) => import("effect/Cause").YieldableError & {
+    readonly _tag: "NoRefreshUrlError";
+} & Readonly<A>;
+export declare class NoRefreshUrlError extends NoRefreshUrlError_base<{
+    readonly message: string;
+}> {
+}
+export interface TokenInfo {
+    type: TokenType;
+    value: string;
+    source: 'html' | 'header' | 'script' | 'json';
+    selector?: string;
+    pattern?: string;
+}
+type HttpRequestError = NetworkError | ParseError | TimeoutError;
+type TokenExtractorError = HttpRequestError | Error | TokenNotAvailableError | TokenRefreshError | NoRefreshUrlError;
+export interface TokenExtractorService {
+    /**
+     * Extract all tokens from an HTTP response
+     */
+    extractTokensFromResponse: (response: HttpResponse) => Effect.Effect<TokenInfo[]>;
+    /**
+     * Extract CSRF token from response
+     */
+    extractCSRFFromResponse: (response: HttpResponse) => Effect.Effect<Option.Option<string>>;
+    /**
+     * Extract API token from response
+     */
+    extractAPIFromResponse: (response: HttpResponse) => Effect.Effect<Option.Option<string>>;
+    /**
+     * Make authenticated request with automatic token injection
+     */
+    authenticatedRequest: (url: string, options?: {
+        requireCSRF?: boolean;
+        requireAPI?: boolean;
+        customHeaders?: Record<string, string>;
+    }) => Effect.Effect<HttpResponse, TokenExtractorError>;
+    /**
+     * Detect and handle token rotation
+     */
+    detectTokenRotation: (oldToken: string, response: HttpResponse, type: TokenType) => Effect.Effect<boolean>;
+    /**
+     * Refresh expired tokens
+     */
+    refreshToken: (type: TokenType, refreshUrl?: string) => Effect.Effect<string, TokenExtractorError>;
+}
+export type { TokenExtractorError };
+declare const TokenExtractor_base: Context.TagClass<TokenExtractor, "TokenExtractor", TokenExtractorService>;
+export declare class TokenExtractor extends TokenExtractor_base {
+}
+/**
+ * Create a TokenExtractor service implementation
+ */
+export declare const makeTokenExtractor: Effect.Effect<TokenExtractorService, never, SpiderLogger | EnhancedHttpClient | StateManager>;
+/**
+ * TokenExtractor Layer with dependencies
+ */
+export declare const TokenExtractorLive: Layer.Layer<TokenExtractor, never, SpiderLogger | EnhancedHttpClient | StateManager>;
+//# sourceMappingURL=TokenExtractor.d.ts.map

package/dist/lib/HttpClient/TokenExtractor.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"TokenExtractor.d.ts","sourceRoot":"","sources":["../../../src/lib/HttpClient/TokenExtractor.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAW,MAAM,EAAkB,MAAM,QAAQ,CAAC;AAEjF,OAAO,EACL,YAAY,EACZ,SAAS,EACV,MAAM,yCAAyC,CAAC;AACjD,OAAO,EAAE,kBAAkB,EAAE,KAAK,YAAY,EAAE,MAAM,yBAAyB,CAAC;AAChF,OAAO,EAAE,YAAY,EAAE,MAAM,oCAAoC,CAAC;AAClE,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;;;;AAGpF,qBAAa,sBAAuB,SAAQ,4BAA2C;IACrF,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;CAC1B,CAAC;CAAG;;;;AAEL,qBAAa,iBAAkB,SAAQ,uBAAsC;IAC3E,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,SAAS,EAAE,SAAS,CAAC;CAC/B,CAAC;CAAG;;;;AAEL,qBAAa,iBAAkB,SAAQ,uBAAsC;IAC3E,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;CAC1B,CAAC;CAAG;AAEL,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,SAAS,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,GAAG,QAAQ,GAAG,QAAQ,GAAG,MAAM,CAAC;IAC9C,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAGD,KAAK,gBAAgB,GAAG,YAAY,GAAG,UAAU,GAAG,YAAY,CAAC;AAGjE,KAAK,mBAAmB,GAAG,gBAAgB,GAAG,KAAK,GAAG,sBAAsB,GAAG,iBAAiB,GAAG,iBAAiB,CAAC;AAErH,MAAM,WAAW,qBAAqB;IACpC;;OAEG;IACH,yBAAyB,EAAE,CACzB,QAAQ,EAAE,YAAY,KACnB,MAAM,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC;IAEhC;;OAEG;IACH,uBAAuB,EAAE,CACvB,QAAQ,EAAE,YAAY,KACnB,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;IAE1C;;OAEG;IACH,sBAAsB,EAAE,CACtB,QAAQ,EAAE,YAAY,KACnB,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;IAE1C;;OAEG;IACH,oBAAoB,EAAE,CACpB,GAAG,EAAE,MAAM,EACX,OAAO,CAAC,EAAE;QACR,WAAW,CAAC,EAAE,OAAO,CAAC;QACtB,UAAU,CAAC,EAAE,OAAO,CAAC;QACrB,aAAa,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;KACxC,KACE,MAAM,CAAC,MAAM,CAAC,YAAY,EAAE,mBAAmB,CAAC,CAAC;IAEtD;;OAEG;IACH,mBAAmB,EAAE,CACnB,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,YAAY,EACtB,IAAI,EAAE,SAAS,KACZ,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAE5B;;OAEG;IACH,YAAY,EAAE,CACZ,IAAI,EAAE,SAAS,EACf,UAAU,CAAC,EAAE,MAAM,KAChB,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,mBAAmB,CAAC,CAAC;CACjD;AAED,YAAY,EAAE,mBAAmB,EAAE,CAAC;;AAEpC,qBAAa,cAAe,SAAQ,mBAGjC;CAAG;AAEN;;GAEG;AACH,eAAO,MAAM,kBAAkB,+FAkd7B,CAAC;AAEH;;GAEG;AACH,eAAO,MAAM,kBAAkB,sFAG9B,CAAC"}

package/dist/lib/HttpClient/index.d.ts

@@ -0,0 +1,8 @@
+/**
+ * HTTP Client module exports
+ */
+export * from './CookieManager.js';
+export * from './EnhancedHttpClient.js';
+export * from './SessionStore.js';
+export * from './TokenExtractor.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/lib/HttpClient/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/lib/HttpClient/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,cAAc,oBAAoB,CAAC;AACnC,cAAc,yBAAyB,CAAC;AACxC,cAAc,mBAAmB,CAAC;AAClC,cAAc,qBAAqB,CAAC"}

package/dist/lib/LinkExtractor/LinkExtractor.service.d.ts

@@ -0,0 +1,166 @@
+import { Effect } from 'effect';
+/**
+ * Configuration for link extraction behavior.
+ *
+ * Focuses purely on HOW to extract links from HTML documents,
+ * not on processing or validating the extracted URLs.
+ *
+ * @example
+ * ```typescript
+ * // Extract from specific CSS selectors
+ * const config: LinkExtractorConfig = {
+ *   restrictCss: ['a.product-link', 'form[action]'],
+ *   tags: ['a', 'form'],
+ *   attrs: ['href', 'action']
+ * };
+ *
+ * // Extract from all standard elements
+ * const config: LinkExtractorConfig = {
+ *   tags: ['a', 'area', 'form', 'frame', 'iframe'],
+ *   attrs: ['href', 'action', 'src']
+ * };
+ * ```
+ *
+ * @group LinkExtractor
+ * @public
+ */
+export interface LinkExtractorConfig {
+    /**
+     * CSS selectors to restrict extraction to specific elements.
+     * If specified, only elements matching these selectors will be processed.
+     *
+     * @example
+     * ```typescript
+     * restrictCss: [
+     *   'a.product-link',     // Only product links
+     *   '.content a',         // Links within content area
+     *   'form[method="post"]' // POST forms only
+     * ]
+     * ```
+     */
+    readonly restrictCss?: string[];
+    /**
+     * HTML tag names to extract links from.
+     * Defaults to common link-containing elements.
+     *
+     * @example ['a', 'area', 'form', 'frame', 'iframe', 'link']
+     */
+    readonly tags?: string[];
+    /**
+     * HTML attributes to extract URLs from.
+     * Defaults to common URL-containing attributes.
+     *
+     * @example ['href', 'action', 'src', 'data-url']
+     */
+    readonly attrs?: string[];
+    /**
+     * Whether to extract URLs from form input elements.
+     * Looks for hidden inputs with URL-like names/values.
+     *
+     * @default false
+     */
+    readonly extractFromInputs?: boolean;
+}
+/**
+ * Result of link extraction from an HTML document.
+ *
+ * Contains the raw extracted URLs without any processing or validation.
+ *
+ * @group LinkExtractor
+ * @public
+ */
+export interface LinkExtractionResult {
+    /**
+     * Raw URLs extracted from the HTML document.
+     * These are unprocessed and may be relative URLs, fragments, etc.
+     */
+    readonly links: string[];
+    /**
+     * Total number of potential URL-containing elements found.
+     * Includes elements that didn't yield valid URLs.
+     */
+    readonly totalElementsProcessed: number;
+    /**
+     * Breakdown of extraction by element type.
+     * Maps element types to the number of URLs extracted from them.
+     */
+    readonly extractionBreakdown: Record<string, number>;
+}
+declare const LinkExtractionError_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").Equals<A, {}> extends true ? void : { readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }) => import("effect/Cause").YieldableError & {
+    readonly _tag: "LinkExtractionError";
+} & Readonly<A>;
+/**
+ * Error that can occur during link extraction.
+ *
+ * @group Errors
+ * @public
+ */
+export declare class LinkExtractionError extends LinkExtractionError_base<{
+    readonly message: string;
+    readonly cause?: unknown;
+}> {
+}
+/**
+ * Service interface for extracting links from HTML documents.
+ *
+ * This service focuses purely on extraction - it does not process,
+ * validate, or filter the extracted URLs in any way.
+ *
+ * @group Services
+ * @public
+ */
+export interface LinkExtractorServiceInterface {
+    /**
+     * Extracts all URLs from an HTML document based on configuration.
+     *
+     * This method only extracts URLs from the HTML - it does not:
+     * - Validate URLs
+     * - Resolve relative URLs to absolute URLs
+     * - Apply domain or pattern filtering
+     * - Canonicalize URLs
+     *
+     * URL processing should be handled separately by the consumer.
+     *
+     * @param html - The HTML content to extract links from
+     * @param config - Configuration for extraction behavior
+     * @returns Effect containing the extraction result
+     *
+     * @example
+     * ```typescript
+     * const extractor = yield* LinkExtractorService;
+     * const result = yield* extractor.extractLinks(htmlContent, {
+     *   tags: ['a', 'form'],
+     *   attrs: ['href', 'action'],
+     *   restrictCss: ['.content a']
+     * });
+     *
+     * console.log(`Found ${result.links.length} raw URLs`);
+     * // URLs may be relative, absolute, fragments, etc.
+     * ```
+     */
+    extractLinks: (html: string, config?: LinkExtractorConfig) => Effect.Effect<LinkExtractionResult, LinkExtractionError>;
+}
+declare const LinkExtractorService_base: Effect.Service.Class<LinkExtractorService, "@jambudipa.io/LinkExtractorService", {
+    readonly effect: Effect.Effect<{
+        extractLinks: (html: string, config?: LinkExtractorConfig) => Effect.Effect<LinkExtractionResult, LinkExtractionError, never>;
+    }, never, never>;
+}>;
+/**
+ * Implementation of the LinkExtractorService.
+ *
+ * Provides pure HTML link extraction without any URL processing.
+ *
+ * @group Services
+ * @public
+ */
+export declare class LinkExtractorService extends LinkExtractorService_base {
+}
+/**
+ * Default layer for LinkExtractorService.
+ *
+ * @group Layers
+ * @public
+ */
+export declare const LinkExtractorServiceLayer: import("effect/Layer").Layer<LinkExtractorService, never, never>;
+export {};
+//# sourceMappingURL=LinkExtractor.service.d.ts.map

package/dist/lib/LinkExtractor/LinkExtractor.service.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"LinkExtractor.service.d.ts","sourceRoot":"","sources":["../../../src/lib/LinkExtractor/LinkExtractor.service.ts"],"names":[],"mappings":"AAAA,OAAO,EAAe,MAAM,EAAU,MAAM,QAAQ,CAAC;AAIrD;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,MAAM,WAAW,mBAAmB;IAClC;;;;;;;;;;;;OAYG;IACH,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IAEhC;;;;;OAKG;IACH,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAEzB;;;;;OAKG;IACH,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;IAE1B;;;;;OAKG;IACH,QAAQ,CAAC,iBAAiB,CAAC,EAAE,OAAO,CAAC;CACtC;AAED;;;;;;;GAOG;AACH,MAAM,WAAW,oBAAoB;IACnC;;;OAGG;IACH,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,CAAC;IAEzB;;;OAGG;IACH,QAAQ,CAAC,sBAAsB,EAAE,MAAM,CAAC;IAExC;;;OAGG;IACH,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CACtD;;;;AAED;;;;;GAKG;AACH,qBAAa,mBAAoB,SAAQ,yBAEvC;IACA,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,KAAK,CAAC,EAAE,OAAO,CAAC;CAC1B,CAAC;CAAG;AAEL;;;;;;;;GAQG;AACH,MAAM,WAAW,6BAA6B;IAC5C;;;;;;;;;;;;;;;;;;;;;;;;;;;OA2BG;IACH,YAAY,EAAE,CACZ,IAAI,EAAE,MAAM,EACZ,MAAM,CAAC,EAAE,mBAAmB,KACzB,MAAM,CAAC,MAAM,CAAC,oBAAoB,EAAE,mBAAmB,CAAC,CAAC;CAC/D;;;6BAyB0B,MAAM,WAAW,mBAAmB;;;AAZ/D;;;;;;;GAOG;AACH,qBAAa,oBAAqB,SAAQ,yBAoBzC;CAAG;AAEJ;;;;;GAKG;AACH,eAAO,MAAM,yBAAyB,kEAA+B,CAAC"}

package/dist/lib/LinkExtractor/index.d.ts

@@ -0,0 +1,37 @@
+/**
+ * Advanced link extraction functionality for the Spider framework.
+ *
+ * This module provides Scrapy-equivalent link extraction capabilities with support for:
+ * - CSS selector-based extraction
+ * - Pattern-based filtering (allow/deny regex patterns)
+ * - Domain-based filtering
+ * - URL canonicalization
+ * - Duplicate removal
+ * - Comprehensive extraction statistics
+ *
+ * @example
+ * ```typescript
+ * import { LinkExtractorService, type LinkExtractorConfig } from '@jambudipa/spider/LinkExtractor';
+ *
+ * const program = Effect.gen(function* () {
+ *   const extractor = yield* LinkExtractorService;
+ *
+ *   const result = yield* extractor.extractLinks(
+ *     htmlContent,
+ *     'https://example.com',
+ *     {
+ *       allowPatterns: [/\/articles\/\d+/],
+ *       restrictCss: ['.content a'],
+ *       canonicalize: true
+ *     }
+ *   );
+ *
+ *   console.log(`Extracted ${result.links.length} links`);
+ * });
+ * ```
+ *
+ * @group LinkExtractor
+ * @public
+ */
+export { LinkExtractorService, LinkExtractorServiceLayer, type LinkExtractorConfig, type LinkExtractionResult, type LinkExtractorServiceInterface, LinkExtractionError, } from './LinkExtractor.service.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/lib/LinkExtractor/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/lib/LinkExtractor/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkCG;AAEH,OAAO,EACL,oBAAoB,EACpB,yBAAyB,EACzB,KAAK,mBAAmB,EACxB,KAAK,oBAAoB,EACzB,KAAK,6BAA6B,EAClC,mBAAmB,GACpB,MAAM,4BAA4B,CAAC"}

package/dist/lib/Logging/FetchLogger.d.ts

@@ -0,0 +1,24 @@
+import { Context, Effect } from 'effect';
+import { SpiderLogger } from './SpiderLogger.service.js';
+declare const FetchError_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").Equals<A, {}> extends true ? void : { readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }) => import("effect/Cause").YieldableError & {
+    readonly _tag: "FetchError";
+} & Readonly<A>;
+/**
+ * Tagged error for fetch operations
+ */
+export declare class FetchError extends FetchError_base<{
+    readonly url: string;
+    readonly reason: 'timeout' | 'network' | 'unknown';
+    readonly durationMs: number;
+    readonly cause?: unknown;
+}> {
+    get message(): string;
+}
+/**
+ * Wrapper for fetch that adds comprehensive logging
+ */
+export declare const makeLoggingFetch: Effect.Effect<(url: string, options?: RequestInit) => Effect.Effect<Response, FetchError>, never, SpiderLogger>;
+export type LoggingFetchFn = (url: string, options?: RequestInit) => Effect.Effect<Response, FetchError>;
+export declare const LoggingFetch: Context.Tag<LoggingFetchFn, LoggingFetchFn>;
+export {};
+//# sourceMappingURL=FetchLogger.d.ts.map

package/dist/lib/Logging/FetchLogger.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"FetchLogger.d.ts","sourceRoot":"","sources":["../../../src/lib/Logging/FetchLogger.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAA4B,MAAM,EAAU,MAAM,QAAQ,CAAC;AAC3E,OAAO,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAC;;;;AAEzD;;GAEG;AACH,qBAAa,UAAW,SAAQ,gBAA+B;IAC7D,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,MAAM,EAAE,SAAS,GAAG,SAAS,GAAG,SAAS,CAAC;IACnD,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,KAAK,CAAC,EAAE,OAAO,CAAC;CAC1B,CAAC;IACA,IAAI,OAAO,IAAI,MAAM,CAEpB;CACF;AAED;;GAEG;AACH,eAAO,MAAM,gBAAgB,sBAGd,MAAM,YAAY,WAAW,KAAG,MAAM,CAAC,MAAM,CAAC,QAAQ,EAAE,UAAU,CAAC,sBA+HhF,CAAC;AAEH,MAAM,MAAM,cAAc,GAAG,CAC3B,GAAG,EAAE,MAAM,EACX,OAAO,CAAC,EAAE,WAAW,KAClB,MAAM,CAAC,MAAM,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;AAEzC,eAAO,MAAM,YAAY,6CAAqD,CAAC"}

package/dist/lib/Logging/SpiderLogger.service.d.ts

@@ -0,0 +1,37 @@
+import { Context, Effect, Layer } from 'effect';
+export interface SpiderLogEvent {
+    timestamp: string;
+    type: 'domain_start' | 'domain_complete' | 'domain_error' | 'page_scraped' | 'queue_status' | 'worker_status' | 'rate_limit' | 'spider_lifecycle' | 'worker_lifecycle' | 'worker_state' | 'completion_monitor' | 'edge_case' | 'crawl_delay_capped';
+    domain?: string;
+    url?: string;
+    workerId?: string;
+    fiberId?: string;
+    message: string;
+    details?: Record<string, unknown>;
+}
+export interface SpiderLoggerService {
+    readonly logEvent: (event: Omit<SpiderLogEvent, 'timestamp'>) => Effect.Effect<void>;
+    readonly logDomainStart: (domain: string, startUrl: string) => Effect.Effect<void>;
+    readonly logDomainComplete: (domain: string, pagesScraped: number, reason: 'max_pages' | 'queue_empty' | 'error') => Effect.Effect<void>;
+    readonly logPageScraped: (url: string, domain: string, pageNumber: number) => Effect.Effect<void>;
+    readonly logQueueStatus: (domain: string, queueSize: number, activeWorkers: number) => Effect.Effect<void>;
+    readonly logRateLimit: (domain: string, requestsInWindow: number) => Effect.Effect<void>;
+    readonly logSpiderLifecycle: (event: 'start' | 'complete' | 'error', details?: Record<string, unknown>) => Effect.Effect<void>;
+    readonly logWorkerLifecycle: (workerId: string, domain: string, event: 'created' | 'entering_loop' | 'exiting_loop', reason?: string, details?: Record<string, unknown>) => Effect.Effect<void>;
+    readonly logWorkerState: (workerId: string, domain: string, event: 'taking_task' | 'marked_active' | 'marked_idle' | 'task_completed', details?: Record<string, unknown>) => Effect.Effect<void>;
+    readonly logCompletionMonitor: (domain: string, checkCount: number, queueSize: number, activeWorkers: number, stableCount: number, maxPagesReached: boolean, decision: string) => Effect.Effect<void>;
+    readonly logEdgeCase: (domain: string, caseType: string, details?: Record<string, unknown>) => Effect.Effect<void>;
+    readonly logDomainStatus: (domain: string, status: {
+        pagesScraped: number;
+        queueSize: number;
+        activeWorkers: number;
+        maxWorkers: number;
+    }) => Effect.Effect<void>;
+}
+declare const SpiderLogger_base: Context.TagClass<SpiderLogger, "SpiderLogger", SpiderLoggerService>;
+export declare class SpiderLogger extends SpiderLogger_base {
+}
+export declare const makeSpiderLogger: (logDir?: string) => SpiderLoggerService;
+export declare const SpiderLoggerLive: Layer.Layer<SpiderLogger, never, never>;
+export {};
+//# sourceMappingURL=SpiderLogger.service.d.ts.map

package/dist/lib/Logging/SpiderLogger.service.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"SpiderLogger.service.d.ts","sourceRoot":"","sources":["../../../src/lib/Logging/SpiderLogger.service.ts"],"names":[],"mappings":"AAAA,OAAO,EAAW,OAAO,EAAY,MAAM,EAAE,KAAK,EAAU,MAAM,QAAQ,CAAC;AAI3E,MAAM,WAAW,cAAc;IAC7B,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EACA,cAAc,GACd,iBAAiB,GACjB,cAAc,GACd,cAAc,GACd,cAAc,GACd,eAAe,GACf,YAAY,GACZ,kBAAkB,GAClB,kBAAkB,GAClB,cAAc,GACd,oBAAoB,GACpB,WAAW,GACX,oBAAoB,CAAC;IACzB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACnC;AAED,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,QAAQ,EAAE,CACjB,KAAK,EAAE,IAAI,CAAC,cAAc,EAAE,WAAW,CAAC,KACrC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACzB,QAAQ,CAAC,cAAc,EAAE,CACvB,MAAM,EAAE,MAAM,EACd,QAAQ,EAAE,MAAM,KACb,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACzB,QAAQ,CAAC,iBAAiB,EAAE,CAC1B,MAAM,EAAE,MAAM,EACd,YAAY,EAAE,MAAM,EACpB,MAAM,EAAE,WAAW,GAAG,aAAa,GAAG,OAAO,KAC1C,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACzB,QAAQ,CAAC,cAAc,EAAE,CACvB,GAAG,EAAE,MAAM,EACX,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,KACf,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACzB,QAAQ,CAAC,cAAc,EAAE,CACvB,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,EACjB,aAAa,EAAE,MAAM,KAClB,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACzB,QAAQ,CAAC,YAAY,EAAE,CACrB,MAAM,EAAE,MAAM,EACd,gBAAgB,EAAE,MAAM,KACrB,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACzB,QAAQ,CAAC,kBAAkB,EAAE,CAC3B,KAAK,EAAE,OAAO,GAAG,UAAU,GAAG,OAAO,EACrC,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAC9B,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAGzB,QAAQ,CAAC,kBAAkB,EAAE,CAC3B,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,SAAS,GAAG,eAAe,GAAG,cAAc,EACnD,MAAM,CAAC,EAAE,MAAM,EACf,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAC9B,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACzB,QAAQ,CAAC,cAAc,EAAE,CACvB,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,aAAa,GAAG,eAAe,GAAG,aAAa,GAAG,gBAAgB,EACzE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAC9B,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACzB,QAAQ,CAAC,oBAAoB,EAAE,CAC7B,MAAM,EAAE,MAAM,EACd,UAAU,EAAE,MAAM,EAClB,SAAS,EAAE,MAAM,EACjB,aAAa,EAAE,MAAM,EACrB,WAAW,EAAE,MAAM,EACnB,eAAe,EAAE,OAAO,EACxB,QAAQ,EAAE,MAAM,KACb,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACzB,QAAQ,CAAC,WAAW,EAAE,CACpB,MAAM,EAAE,MAAM,EACd,QAAQ,EAAE,MAAM,EAChB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAC9B,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACzB,QAAQ,CAAC,eAAe,EAAE,CACxB,MAAM,EAAE,MAAM,EACd,MAAM,EAAE;QACN,YAAY,EAAE,MAAM,CAAC;QACrB,SAAS,EAAE,MAAM,CAAC;QAClB,aAAa,EAAE,MAAM,CAAC;QACtB,UAAU,EAAE,MAAM,CAAC;KACpB,KACE,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;CAC1B;;AAED,qBAAa,YAAa,SAAQ,iBAG/B;CAAG;AAiCN,eAAO,MAAM,gBAAgB,GAAI,eAAwB,KAAG,mBA2U3D,CAAC;AAEF,eAAO,MAAM,gBAAgB,yCAAkD,CAAC"}

package/dist/lib/Middleware/SpiderMiddleware.d.ts

@@ -0,0 +1,239 @@
+import { Effect, Option } from 'effect';
+import { MiddlewareError } from '../errors.js';
+import { SpiderRequest, SpiderResponse } from './types.js';
+export { SpiderRequest, SpiderResponse } from './types.js';
+/**
+ * Interface for implementing custom middleware components.
+ *
+ * Middleware can intercept and modify requests before they're sent,
+ * responses after they're received, and handle exceptions that occur
+ * during processing. All methods are optional.
+ *
+ * @example
+ * ```typescript
+ * const loggingMiddleware: SpiderMiddleware = {
+ *   processRequest: (request) => Effect.gen(function* () {
+ *     console.log(`Requesting: ${request.task.url}`);
+ *     return request;
+ *   }),
+ *
+ *   processResponse: (response, request) => Effect.gen(function* () {
+ *     console.log(`Response: ${response.statusCode} for ${request.task.url}`);
+ *     return response;
+ *   }),
+ *
+ *   processException: (error, request) => Effect.gen(function* () {
+ *     console.error(`Error processing ${request.task.url}: ${error.message}`);
+ *     return null; // Let the error propagate
+ *   })
+ * };
+ * ```
+ *
+ * @group Interfaces
+ * @public
+ */
+export interface SpiderMiddleware {
+    /**
+     * Process a request before it's sent to the target server.
+     * Can modify headers, metadata, or reject the request entirely.
+     */
+    processRequest?: (_request: SpiderRequest) => Effect.Effect<SpiderRequest, MiddlewareError>;
+    /**
+     * Process a response after it's received from the target server.
+     * Can modify the response data or metadata.
+     */
+    processResponse?: (_response: SpiderResponse, _request: SpiderRequest) => Effect.Effect<SpiderResponse, MiddlewareError>;
+    /**
+     * Handle exceptions that occur during request processing.
+     * Can attempt recovery by returning a SpiderResponse, or return Option.none() to propagate the error.
+     */
+    processException?: (_error: Error, _request: SpiderRequest) => Effect.Effect<Option.Option<SpiderResponse>, MiddlewareError>;
+}
+declare const MiddlewareManager_base: Effect.Service.Class<MiddlewareManager, "@jambudipa.io/MiddlewareManager", {
+    readonly effect: Effect.Effect<{
+        /**
+         * Processes a request through the middleware pipeline.
+         *
+         * Middleware are executed in order from first to last, with each middleware
+         * receiving the output of the previous middleware as input.
+         *
+         * @param request - The initial request to process
+         * @param middlewares - Array of middleware to apply
+         * @returns Effect containing the processed request
+         */
+        processRequest: (request: SpiderRequest, middlewares: SpiderMiddleware[]) => Effect.Effect<SpiderRequest, MiddlewareError, never>;
+        /**
+         * Processes a response through the middleware pipeline in reverse order.
+         *
+         * Middleware are executed in reverse order (last to first) to provide
+         * proper nesting of response processing.
+         *
+         * @param response - The response to process
+         * @param request - The original request (for context)
+         * @param middlewares - Array of middleware to apply
+         * @returns Effect containing the processed response
+         */
+        processResponse: (response: SpiderResponse, request: SpiderRequest, middlewares: SpiderMiddleware[]) => Effect.Effect<SpiderResponse, MiddlewareError, never>;
+        /**
+         * Processes an exception through the middleware pipeline in reverse order.
+         *
+         * Middleware are given a chance to handle or recover from exceptions.
+         * If a middleware returns Option.some(SpiderResponse), it indicates successful recovery.
+         * If it returns Option.none(), the exception continues to propagate.
+         *
+         * @param error - The error that occurred
+         * @param request - The request that caused the error
+         * @param middlewares - Array of middleware to apply
+         * @returns Effect containing a recovered response wrapped in Option
+         */
+        processException: (error: Error, request: SpiderRequest, middlewares: SpiderMiddleware[]) => Effect.Effect<Option.Option<SpiderResponse>, MiddlewareError, never>;
+    }, never, never>;
+}>;
+/**
+ * Manages the middleware pipeline for request and response processing.
+ *
+ * The MiddlewareManager orchestrates the execution of middleware in the correct order:
+ * - Requests are processed forward through the middleware array
+ * - Responses are processed in reverse order (last middleware first)
+ * - Exceptions are processed in reverse order for proper error handling
+ *
+ * @example
+ * ```typescript
+ * const program = Effect.gen(function* () {
+ *   const manager = yield* MiddlewareManager;
+ *
+ *   const middleware = [
+ *     rateLimitMiddleware,
+ *     loggingMiddleware,
+ *     userAgentMiddleware
+ *   ];
+ *
+ *   const request: SpiderRequest = {
+ *     task: { url: 'https://example.com', depth: 0 },
+ *     headers: {}
+ *   };
+ *
+ *   const processedRequest = yield* manager.processRequest(request, middleware);
+ *   console.log('Request processed through middleware pipeline');
+ * });
+ * ```
+ *
+ * @group Services
+ * @public
+ */
+export declare class MiddlewareManager extends MiddlewareManager_base {
+}
+declare const RateLimitMiddleware_base: Effect.Service.Class<RateLimitMiddleware, "@jambudipa.io/RateLimitMiddleware", {
+    readonly effect: Effect.Effect<{
+        create: (config: {
+            maxConcurrentRequests: number;
+            maxRequestsPerSecondPerDomain: number;
+            requestDelayMs?: number;
+        }) => SpiderMiddleware;
+    }, never, never>;
+}>;
+/**
+ * Provides rate limiting functionality for respectful crawling.
+ *
+ * Controls request frequency at both global and per-domain levels to prevent
+ * overwhelming target servers and avoid being blocked.
+ *
+ * @example
+ * ```typescript
+ * const rateLimiter = yield* RateLimitMiddleware;
+ * const middleware = rateLimiter.create({
+ *   maxConcurrentRequests: 5,
+ *   maxRequestsPerSecondPerDomain: 2,
+ *   requestDelayMs: 250
+ * });
+ * ```
+ *
+ * @group Middleware
+ * @public
+ */
+export declare class RateLimitMiddleware extends RateLimitMiddleware_base {
+}
+declare const LoggingMiddleware_base: Effect.Service.Class<LoggingMiddleware, "@jambudipa.io/LoggingMiddleware", {
+    readonly effect: Effect.Effect<{
+        create: (config?: {
+            logRequests?: boolean;
+            logResponses?: boolean;
+            logErrors?: boolean;
+            logLevel?: "debug" | "info" | "warn" | "error";
+        }) => SpiderMiddleware;
+    }, never, never>;
+}>;
+/**
+ * Provides logging functionality using Effect.Logger.
+ *
+ * Logs requests, responses, and errors at configurable levels for debugging
+ * and monitoring purposes.
+ *
+ * @example
+ * ```typescript
+ * const logger = yield* LoggingMiddleware;
+ * const middleware = logger.create({
+ *   logRequests: true,
+ *   logResponses: true,
+ *   logLevel: 'info'
+ * });
+ * ```
+ *
+ * @group Middleware
+ * @public
+ */
+export declare class LoggingMiddleware extends LoggingMiddleware_base {
+}
+declare const UserAgentMiddleware_base: Effect.Service.Class<UserAgentMiddleware, "@jambudipa.io/UserAgentMiddleware", {
+    readonly effect: Effect.Effect<{
+        create: (userAgent: string) => SpiderMiddleware;
+    }, never, never>;
+}>;
+/**
+ * Adds User-Agent headers to requests.
+ *
+ * Sets a consistent User-Agent string for all requests to identify
+ * your crawler to web servers.
+ *
+ * @example
+ * ```typescript
+ * const userAgent = yield* UserAgentMiddleware;
+ * const middleware = userAgent.create('MyBot/1.0 (+https://example.com)');
+ * ```
+ *
+ * @group Middleware
+ * @public
+ */
+export declare class UserAgentMiddleware extends UserAgentMiddleware_base {
+}
+declare const StatsMiddleware_base: Effect.Service.Class<StatsMiddleware, "@jambudipa.io/StatsMiddleware", {
+    readonly effect: Effect.Effect<{
+        create: () => {
+            middleware: SpiderMiddleware;
+            getStats: () => Effect.Effect<Record<string, number>>;
+        };
+    }, never, never>;
+}>;
+/**
+ * Collects statistics about crawling activity.
+ *
+ * Tracks various metrics including requests processed, response codes,
+ * bytes downloaded, and processing times for monitoring and optimization.
+ *
+ * @example
+ * ```typescript
+ * const statsService = yield* StatsMiddleware;
+ * const { middleware, getStats } = statsService.create();
+ *
+ * // Use middleware in your pipeline
+ * // Later get statistics
+ * const stats = yield* getStats();
+ * console.log(`Processed ${stats.requests_processed} requests`);
+ * ```
+ *
+ * @group Middleware
+ * @public
+ */
+export declare class StatsMiddleware extends StatsMiddleware_base {
+}
+//# sourceMappingURL=SpiderMiddleware.d.ts.map

package/dist/lib/Middleware/SpiderMiddleware.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"SpiderMiddleware.d.ts","sourceRoot":"","sources":["../../../src/lib/Middleware/SpiderMiddleware.ts"],"names":[],"mappings":"AAAA,OAAO,EAAY,MAAM,EAAkB,MAAM,EAAE,MAAM,QAAQ,CAAC;AAClE,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAC/C,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAE3D,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAE3D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACH,MAAM,WAAW,gBAAgB;IAC/B;;;OAGG;IACH,cAAc,CAAC,EAAE,CACf,QAAQ,EAAE,aAAa,KACpB,MAAM,CAAC,MAAM,CAAC,aAAa,EAAE,eAAe,CAAC,CAAC;IAEnD;;;OAGG;IACH,eAAe,CAAC,EAAE,CAChB,SAAS,EAAE,cAAc,EACzB,QAAQ,EAAE,aAAa,KACpB,MAAM,CAAC,MAAM,CAAC,cAAc,EAAE,eAAe,CAAC,CAAC;IAEpD;;;OAGG;IACH,gBAAgB,CAAC,EAAE,CACjB,MAAM,EAAE,KAAK,EACb,QAAQ,EAAE,aAAa,KACpB,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,EAAE,eAAe,CAAC,CAAC;CACpE;;;QAsCK;;;;;;;;;WASG;kCAEQ,aAAa,eACT,gBAAgB,EAAE;QAQjC;;;;;;;;;;WAUG;oCAES,cAAc,WACf,aAAa,eACT,gBAAgB,EAAE;QAWjC;;;;;;;;;;;WAWG;kCAEM,KAAK,WACH,aAAa,eACT,gBAAgB,EAAE;;;AAhGvC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AACH,qBAAa,iBAAkB,SAAQ,sBA4EtC;CAAG;;;yBA8BqB;YACf,qBAAqB,EAAE,MAAM,CAAC;YAC9B,6BAA6B,EAAE,MAAM,CAAC;YACtC,cAAc,CAAC,EAAE,MAAM,CAAC;SACzB,KAAG,gBAAgB;;;AAhC5B;;;;;;;;;;;;;;;;;;GAkBG;AACH,qBAAa,mBAAoB,SAAQ,wBAqExC;CAAG;;;0BA0BY;YACN,WAAW,CAAC,EAAE,OAAO,CAAC;YACtB,YAAY,CAAC,EAAE,OAAO,CAAC;YACvB,SAAS,CAAC,EAAE,OAAO,CAAC;YACpB,QAAQ,CAAC,EAAE,OAAO,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC;SAChD,KACA,gBAAgB;;;AA9BzB;;;;;;;;;;;;;;;;;;GAkBG;AACH,qBAAa,iBAAkB,SAAQ,sBA4EtC;CAAG;;;4BAqBsB,MAAM,KAAG,gBAAgB;;;AAnBnD;;;;;;;;;;;;;;GAcG;AACH,qBAAa,mBAAoB,SAAQ,wBAYxC;CAAG;;;sBA6BgB;YACV,UAAU,EAAE,gBAAgB,CAAC;YAC7B,QAAQ,EAAE,MAAM,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;SACvD;;;AA9BT;;;;;;;;;;;;;;;;;;;GAmBG;AACH,qBAAa,eAAgB,SAAQ,oBAqEpC;CAAG"}