@jambudipa/spider 0.2.1 → 0.2.3

package/README.md

@@ -37,12 +37,12 @@ A powerful, Effect-based web crawling framework for modern TypeScript applicatio
 
 > **Live Testing**: Our CI pipeline runs all 16 web scraping scenarios against real websites daily, ensuring Spider remains robust against changing web technologies.
 
-### 🔍 **Current Status** (Updated: Aug 2025)
+### 🔍 **Current Status** (Updated: Jan 2026)
 - ✅ **Core Functionality**: All web scraping scenarios working
 - ✅ **Type Safety**: Full TypeScript compilation without errors
 - ✅ **Build System**: Package builds successfully for distribution
-- ✅ **Test Suite**: 92+ scenario tests passing against live websites
-- ⚠️ **Code Quality**: 1,163 linting issues identified (technical debt - does not affect functionality)
+- ✅ **Test Suite**: 243 tests passing against live websites (25 test files)
+- ✅ **Code Quality**: Clean - only 3 linting warnings (skipped test suites)
 
 ## ✨ Key Features
 
@@ -419,33 +419,27 @@ npm run typecheck
 # Validate CI setup locally
 npm run ci:validate
 
-# Code quality (has known issues)
-npm run lint        # Shows 1,163 issues
+# Code quality
+npm run lint        # Shows 3 warnings (skipped tests)
 npm run format     # Formats code consistently
 ```
 
 ### 🛠️ Contributing & Code Quality
 
-**Current State**: The codebase is fully functional with comprehensive test coverage, but has technical debt in code style consistency.
+**Current State**: The codebase is fully functional with comprehensive test coverage and clean linting.
 
 - ✅ **Functional Changes**: All PRs must pass scenario tests
-- ✅ **Type Safety**: TypeScript compilation must succeed  
+- ✅ **Type Safety**: TypeScript compilation must succeed
 - ✅ **Build System**: Package must build without errors
-- 🔄 **Code Style**: Help wanted fixing linting issues (great first contribution!)
+- ✅ **Code Style**: ESLint configured with Effect-idiomatic rules
 
-**Contributing to Code Quality**:
+**Code Quality Commands**:
 ```bash
-# See specific linting issues
+# Check for linting issues
 npm run lint
 
 # Fix auto-fixable issues
 npm run lint:fix
-
-# Focus areas for improvement:
-# - Unused variable cleanup (877 issues)
-# - Return type annotations (286 issues)  
-# - Nullish coalescing operators
-# - Console.log removal in production code
 ```
 
 ## License

package/dist/browser/BrowserManager.d.ts

@@ -0,0 +1,63 @@
+/**
+ * Browser Manager for Playwright Integration
+ * Handles browser lifecycle, pooling, and resource management
+ */
+import { Effect } from 'effect';
+import { BrowserContext, Page, BrowserContextOptions } from 'playwright';
+import { BrowserError } from '../lib/errors';
+export interface BrowserConfig {
+    headless?: boolean;
+    timeout?: number;
+    poolSize?: number;
+    viewport?: {
+        width: number;
+        height: number;
+    };
+    userAgent?: string;
+    locale?: string;
+    extraHTTPHeaders?: Record<string, string>;
+}
+export declare class BrowserManager {
+    private browsers;
+    private contexts;
+    private config;
+    private isInitialised;
+    constructor(config?: BrowserConfig);
+    /**
+     * Initialise browser pool
+     */
+    initialise(): Effect.Effect<void, BrowserError>;
+    /**
+     * Launch a new browser instance
+     */
+    private launchBrowser;
+    /**
+     * Get or create a browser context
+     */
+    getContext(id: string, options?: BrowserContextOptions): Effect.Effect<BrowserContext, BrowserError>;
+    /**
+     * Create a new page in a context
+     */
+    createPage(contextId: string): Effect.Effect<Page, BrowserError>;
+    /**
+     * Get the browser with least contexts
+     */
+    private getLeastLoadedBrowser;
+    /**
+     * Close a specific context
+     */
+    closeContext(id: string): Effect.Effect<void, BrowserError>;
+    /**
+     * Close all resources
+     */
+    close(): Effect.Effect<void>;
+    /**
+     * Get statistics about browser pool
+     */
+    getStats(): {
+        browsers: number;
+        contexts: number;
+        pages: number;
+    };
+}
+//# sourceMappingURL=BrowserManager.d.ts.map

package/dist/browser/BrowserManager.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"BrowserManager.d.ts","sourceRoot":"","sources":["../../src/browser/BrowserManager.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,MAAM,EAAkC,MAAM,QAAQ,CAAC;AAChE,OAAO,EAAW,cAAc,EAAE,IAAI,EAAY,qBAAqB,EAAE,MAAM,YAAY,CAAC;AAC5F,OAAO,EAAuB,YAAY,EAAE,MAAM,eAAe,CAAC;AAElE,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;IAC7C,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,gBAAgB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAC3C;AAED,qBAAa,cAAc;IACzB,OAAO,CAAC,QAAQ,CAAiB;IACjC,OAAO,CAAC,QAAQ,CAAiF;IACjG,OAAO,CAAC,MAAM,CAA0B;IACxC,OAAO,CAAC,aAAa,CAAS;gBAElB,MAAM,GAAE,aAAkB;IAYtC;;OAEG;IACH,UAAU,IAAI,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,YAAY,CAAC;IAc/C;;OAEG;IACH,OAAO,CAAC,aAAa;IAmBrB;;OAEG;IACH,UAAU,CAAC,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,qBAAqB,GAAG,MAAM,CAAC,MAAM,CAAC,cAAc,EAAE,YAAY,CAAC;IA+BpG;;OAEG;IACH,UAAU,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,YAAY,CAAC;IAahE;;OAEG;IACH,OAAO,CAAC,qBAAqB;IAe7B;;OAEG;IACH,YAAY,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,YAAY,CAAC;IAc3D;;OAEG;IACH,KAAK,IAAI,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC;IAiD5B;;OAEG;IACH,QAAQ,IAAI;QACV,QAAQ,EAAE,MAAM,CAAC;QACjB,QAAQ,EAAE,MAAM,CAAC;QACjB,KAAK,EAAE,MAAM,CAAC;KACf;CAYF"}

package/dist/browser/PlaywrightAdapter.d.ts

@@ -0,0 +1,166 @@
+/**
+ * Playwright Adapter for Spider Integration
+ * Provides high-level browser automation capabilities
+ */
+import { Effect, Option } from 'effect';
+import { Page, Response as PlaywrightResponse, Route, Cookie, Request as PlaywrightRequest } from 'playwright';
+import { BrowserManager } from './BrowserManager';
+import { AdapterNotInitialisedError } from '../lib/errors';
+export type RequestHandler = (request: PlaywrightRequest) => void;
+export type ResponseHandler = (response: PlaywrightResponse) => void;
+export interface WaitOptions {
+    timeout?: number;
+    state?: 'load' | 'domcontentloaded' | 'networkidle';
+}
+export interface ScrollOptions {
+    delay?: number;
+    maxScrolls?: number;
+    scrollDistance?: number;
+}
+declare const PageClosedError_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").Equals<A, {}> extends true ? void : { readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }) => import("effect/Cause").YieldableError & {
+    readonly _tag: "PageClosedError";
+} & Readonly<A>;
+/**
+ * Error thrown when page is closed during an operation
+ */
+export declare class PageClosedError extends PageClosedError_base<{
+    readonly operation: string;
+    readonly message: string;
+}> {
+    static create(operation: string): PageClosedError;
+}
+declare const StreamReadError_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").Equals<A, {}> extends true ? void : { readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }) => import("effect/Cause").YieldableError & {
+    readonly _tag: "StreamReadError";
+} & Readonly<A>;
+/**
+ * Error thrown when stream reading fails
+ */
+export declare class StreamReadError extends StreamReadError_base<{
+    readonly cause: unknown;
+    readonly message: string;
+}> {
+    static fromCause(cause: unknown): StreamReadError;
+}
+export declare class PlaywrightAdapter {
+    private browserManager;
+    private page;
+    private contextId;
+    private requestHandlers;
+    private responseHandlers;
+    constructor(browserManager: BrowserManager, contextId: string);
+    /**
+     * Initialise the adapter with a new page
+     */
+    initialise(): Effect.Effect<Page, AdapterNotInitialisedError>;
+    /**
+     * Get the current page instance (Effect)
+     */
+    getPageEffect(): Effect.Effect<Page, AdapterNotInitialisedError>;
+    /**
+     * Get the current page instance (direct)
+     * Returns Option for type-safe handling
+     */
+    getPage(): Option.Option<Page>;
+    /**
+     * Internal helper to get page or fail
+     */
+    private requirePage;
+    /**
+     * Navigate to a URL
+     */
+    goto(url: string, options?: WaitOptions): Effect.Effect<Option.Option<PlaywrightResponse>, AdapterNotInitialisedError>;
+    /**
+     * Wait for dynamic content to load
+     */
+    waitForDynamicContent(selector: string, options?: WaitOptions): Effect.Effect<void, AdapterNotInitialisedError>;
+    /**
+     * Scroll to bottom progressively
+     */
+    scrollToBottom(options?: ScrollOptions): Effect.Effect<void, AdapterNotInitialisedError>;
+    /**
+     * Click an element and wait for navigation or content
+     */
+    clickAndWait(selector: string, waitFor?: string | WaitOptions): Effect.Effect<void, AdapterNotInitialisedError>;
+    /**
+     * Intercept requests
+     */
+    interceptRequests(handler: RequestHandler): Effect.Effect<void>;
+    /**
+     * Intercept responses
+     */
+    interceptResponses(handler: ResponseHandler): Effect.Effect<void>;
+    /**
+     * Route specific URLs
+     */
+    route(pattern: string | RegExp, handler: (route: Route) => void): Effect.Effect<void, AdapterNotInitialisedError>;
+    /**
+     * Execute JavaScript in page context
+     */
+    evaluate<T>(fn: () => T): Effect.Effect<T, AdapterNotInitialisedError>;
+    /**
+     * Take a screenshot
+     */
+    screenshot(path: string): Effect.Effect<void, AdapterNotInitialisedError>;
+    /**
+     * Get page content
+     */
+    content(): Effect.Effect<string, AdapterNotInitialisedError>;
+    /**
+     * Fill a form field
+     */
+    fill(selector: string, value: string): Effect.Effect<void, AdapterNotInitialisedError>;
+    /**
+     * Select an option
+     */
+    select(selector: string, value: string): Effect.Effect<void, AdapterNotInitialisedError>;
+    /**
+     * Check if element exists
+     */
+    exists(selector: string): Effect.Effect<boolean, AdapterNotInitialisedError>;
+    /**
+     * Wait for network idle
+     */
+    waitForNetworkIdle(options?: WaitOptions): Effect.Effect<void, AdapterNotInitialisedError>;
+    /**
+     * Handle new tabs/windows
+     */
+    handleNewTab(callback: (page: Page) => Effect.Effect<void>): Effect.Effect<void, AdapterNotInitialisedError>;
+    /**
+     * Get cookies
+     */
+    getCookies(): Effect.Effect<readonly Cookie[], AdapterNotInitialisedError>;
+    /**
+     * Set cookies
+     */
+    setCookies(cookies: readonly Cookie[]): Effect.Effect<void, AdapterNotInitialisedError>;
+    /**
+     * Clear cookies
+     */
+    clearCookies(): Effect.Effect<void, AdapterNotInitialisedError>;
+    /**
+     * Helper to read a Node stream as a Buffer using Effect Stream
+     */
+    private readStreamAsBuffer;
+    /**
+     * Download file from URL
+     */
+    downloadFile(url: string, filename?: string): Effect.Effect<{
+        buffer: Buffer;
+        filename: string;
+        mimeType: string;
+    }, AdapterNotInitialisedError | PageClosedError | StreamReadError>;
+    /**
+     * Trigger download by clicking element
+     */
+    downloadFromClick(selector: string): Effect.Effect<{
+        buffer: Buffer;
+        filename: string;
+        mimeType: string;
+    }, AdapterNotInitialisedError | PageClosedError | StreamReadError>;
+    /**
+     * Close the page
+     */
+    close(): Effect.Effect<void>;
+}
+export {};
+//# sourceMappingURL=PlaywrightAdapter.d.ts.map

package/dist/browser/PlaywrightAdapter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"PlaywrightAdapter.d.ts","sourceRoot":"","sources":["../../src/browser/PlaywrightAdapter.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAe,MAAM,EAAE,MAAM,EAAU,MAAM,QAAQ,CAAC;AAE7D,OAAO,EAAE,IAAI,EAAE,QAAQ,IAAI,kBAAkB,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,IAAI,iBAAiB,EAAE,MAAM,YAAY,CAAC;AAC/G,OAAO,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAClD,OAAO,EAAE,0BAA0B,EAAE,MAAM,eAAe,CAAC;AAE3D,MAAM,MAAM,cAAc,GAAG,CAAC,OAAO,EAAE,iBAAiB,KAAK,IAAI,CAAC;AAClE,MAAM,MAAM,eAAe,GAAG,CAAC,QAAQ,EAAE,kBAAkB,KAAK,IAAI,CAAC;AAErE,MAAM,WAAW,WAAW;IAC1B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,GAAG,kBAAkB,GAAG,aAAa,CAAC;CACrD;AAED,MAAM,WAAW,aAAa;IAC5B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;;;;AAED;;GAEG;AACH,qBAAa,eAAgB,SAAQ,qBAAoC;IACvE,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;CAC1B,CAAC;IACA,MAAM,CAAC,MAAM,CAAC,SAAS,EAAE,MAAM,GAAG,eAAe;CAMlD;;;;AAED;;GAEG;AACH,qBAAa,eAAgB,SAAQ,qBAAoC;IACvE,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC;IACxB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;CAC1B,CAAC;IACA,MAAM,CAAC,SAAS,CAAC,KAAK,EAAE,OAAO,GAAG,eAAe;CAMlD;AAED,qBAAa,iBAAiB;IAC5B,OAAO,CAAC,cAAc,CAAiB;IACvC,OAAO,CAAC,IAAI,CAAsC;IAClD,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,eAAe,CAA8C;IACrE,OAAO,CAAC,gBAAgB,CAA+C;gBAE3D,cAAc,EAAE,cAAc,EAAE,SAAS,EAAE,MAAM;IAK7D;;OAEG;IACH,UAAU,IAAI,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,0BAA0B,CAAC;IAyB7D;;OAEG;IACH,aAAa,IAAI,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,0BAA0B,CAAC;IAShE;;;OAGG;IACH,OAAO,IAAI,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC;IAI9B;;OAEG;IACH,OAAO,CAAC,WAAW;IAOnB;;OAEG;IACH,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,kBAAkB,CAAC,EAAE,0BAA0B,CAAC;IAetH;;OAEG;IACH,qBAAqB,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,0BAA0B,CAAC;IAc/G;;OAEG;IACH,cAAc,CAAC,OAAO,CAAC,EAAE,aAAa,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,0BAA0B,CAAC;IA+CxF;;OAEG;IACH,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,GAAG,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,0BAA0B,CAAC;IA0C/G;;OAEG;IACH,iBAAiB,CAAC,OAAO,EAAE,cAAc,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC;IAM/D;;OAEG;IACH,kBAAkB,CAAC,OAAO,EAAE,eAAe,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC;IAMjE;;OAEG;IACH,KAAK,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,OAAO,EAAE,CAAC,KAAK,EAAE,KAAK,KAAK,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,0BAA0B,CAAC;IAWjH;;OAEG;IACH,QAAQ,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,0BAA0B,CAAC;IAWtE;;OAEG;IACH,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,0BAA0B,CAAC;IAWzE;;OAEG;IACH,OAAO,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,0BAA0B,CAAC;IAW5D;;OAEG;IACH,IAAI,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,0BAA0B,CAAC;IAWtF;;OAEG;IACH,MAAM,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,0BAA0B,CAAC;IAWxF;;OAEG;IACH,MAAM,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,0BAA0B,CAAC;IAY5E;;OAEG;IACH,kBAAkB,CAAC,OAAO,CAAC,EAAE,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,0BAA0B,CAAC;IAa1F;;OAEG;IACH,YAAY,CAAC,QAAQ,EAAE,CAAC,IAAI,EAAE,IAAI,KAAK,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,0BAA0B,CAAC;IAmB5G;;OAEG;IACH,UAAU,IAAI,MAAM,CAAC,MAAM,CAAC,SAAS,MAAM,EAAE,EAAE,0BAA0B,CAAC;IAW1E;;OAEG;IACH,UAAU,CAAC,OAAO,EAAE,SAAS,MAAM,EAAE,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,0BAA0B,CAAC;IAWvF;;OAEG;IACH,YAAY,IAAI,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,0BAA0B,CAAC;IAW/D;;OAEG;IACH,OAAO,CAAC,kBAAkB;IAY1B;;OAEG;IACH,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;QAC1D,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,EAAE,MAAM,CAAC;QACjB,QAAQ,EAAE,MAAM,CAAC;KAClB,EAAE,0BAA0B,GAAG,eAAe,GAAG,eAAe,CAAC;IAiElE;;OAEG;IACH,iBAAiB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;QACjD,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,EAAE,MAAM,CAAC;QACjB,QAAQ,EAAE,MAAM,CAAC;KAClB,EAAE,0BAA0B,GAAG,eAAe,GAAG,eAAe,CAAC;IA2DlE;;OAEG;IACH,KAAK,IAAI,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC;CAsB7B"}

package/dist/examples/01-basic-crawl-working.d.ts

@@ -0,0 +1,13 @@
+/**
+ * Example 01: Basic Web Crawling
+ *
+ * This example demonstrates:
+ * - Basic Spider setup and configuration
+ * - Single URL crawling with depth control
+ * - Result collection and processing
+ * - Error handling
+ *
+ * Tests against: web-scraping.dev (static content)
+ */
+export {};
+//# sourceMappingURL=01-basic-crawl-working.d.ts.map

package/dist/examples/01-basic-crawl-working.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"01-basic-crawl-working.d.ts","sourceRoot":"","sources":["../../src/examples/01-basic-crawl-working.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG"}

package/dist/examples/02-multiple-urls-working.d.ts

@@ -0,0 +1,13 @@
+/**
+ * Example 02: Multiple Starting URLs and Domain Isolation
+ *
+ * This example demonstrates:
+ * - Crawling multiple domains concurrently
+ * - Domain isolation and deduplication
+ * - Metadata passing through crawl
+ * - Concurrent request handling
+ *
+ * Tests against: Multiple sections of web-scraping.dev
+ */
+export {};
+//# sourceMappingURL=02-multiple-urls-working.d.ts.map

package/dist/examples/02-multiple-urls-working.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"02-multiple-urls-working.d.ts","sourceRoot":"","sources":["../../src/examples/02-multiple-urls-working.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG"}

package/dist/examples/03-url-filtering.d.ts

@@ -0,0 +1,13 @@
+/**
+ * Example 03: URL Filtering and Domain Restrictions
+ *
+ * This example demonstrates:
+ * - Custom URL filtering with regex patterns
+ * - File extension filtering
+ * - Domain restrictions
+ * - Technical URL filtering (malformed, long URLs, etc.)
+ *
+ * Tests against: web-scraping.dev with various filters
+ */
+export {};
+//# sourceMappingURL=03-url-filtering.d.ts.map

package/dist/examples/03-url-filtering.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"03-url-filtering.d.ts","sourceRoot":"","sources":["../../src/examples/03-url-filtering.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG"}

package/dist/examples/04-robots-compliance.d.ts

@@ -0,0 +1,14 @@
+/**
+ * Example 04: Robots.txt Compliance and Rate Limiting
+ *
+ * This example demonstrates:
+ * - Robots.txt compliance checking
+ * - Crawl delay respect from robots.txt
+ * - Rate limiting per domain
+ * - Request delay configuration
+ * - Respectful crawling practices
+ *
+ * Tests against: web-scraping.dev robots.txt rules
+ */
+export {};
+//# sourceMappingURL=04-robots-compliance.d.ts.map

package/dist/examples/04-robots-compliance.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"04-robots-compliance.d.ts","sourceRoot":"","sources":["../../src/examples/04-robots-compliance.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG"}

package/dist/examples/05-link-extraction-selectors.d.ts

@@ -0,0 +1,14 @@
+/**
+ * Example 05: Advanced Link Extraction with CSS Selectors
+ *
+ * This example demonstrates:
+ * - Custom CSS selectors for targeted link extraction
+ * - Link extraction from different HTML elements
+ * - Form action URL extraction
+ * - Custom attribute extraction
+ * - Link extraction statistics and analysis
+ *
+ * Tests against: web-scraping.dev with targeted selectors
+ */
+export {};
+//# sourceMappingURL=05-link-extraction-selectors.d.ts.map

package/dist/examples/05-link-extraction-selectors.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"05-link-extraction-selectors.d.ts","sourceRoot":"","sources":["../../src/examples/05-link-extraction-selectors.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG"}

package/dist/examples/06-custom-middleware.d.ts

@@ -0,0 +1,18 @@
+/**
+ * Example 06: Custom Middleware Usage
+ *
+ * This example demonstrates:
+ * - Creating custom middleware for request/response processing
+ * - Using built-in middleware (RateLimit, Logging, UserAgent, Stats)
+ * - Middleware error handling
+ * - Request preprocessing and response postprocessing
+ * - Statistics collection through middleware
+ *
+ * Tests against: web-scraping.dev with custom middleware pipeline
+ */
+import { SpiderMiddleware } from '../index.js';
+declare const TimingMiddleware: SpiderMiddleware;
+declare const ContentAnalysisMiddleware: SpiderMiddleware;
+declare const PatternDetectionMiddleware: SpiderMiddleware;
+export { TimingMiddleware, ContentAnalysisMiddleware, PatternDetectionMiddleware };
+//# sourceMappingURL=06-custom-middleware.d.ts.map

package/dist/examples/06-custom-middleware.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"06-custom-middleware.d.ts","sourceRoot":"","sources":["../../src/examples/06-custom-middleware.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAGH,OAAO,EAKL,gBAAgB,EAIjB,MAAM,aAAa,CAAC;AAGrB,QAAA,MAAM,gBAAgB,EAAE,gBAyCvB,CAAC;AAGF,QAAA,MAAM,yBAAyB,EAAE,gBAiBhC,CAAC;AAGF,QAAA,MAAM,0BAA0B,EAAE,gBAiBjC,CAAC;AAGF,OAAO,EAAE,gBAAgB,EAAE,yBAAyB,EAAE,0BAA0B,EAAE,CAAC"}

package/dist/examples/07-resumability-demo.d.ts

@@ -0,0 +1,14 @@
+/**
+ * Example 07: Resumability and State Persistence
+ *
+ * This example demonstrates:
+ * - Resumable crawling with state persistence
+ * - Multiple storage backends (File, Redis, Postgres)
+ * - Session management and restoration
+ * - State persistence strategies (Full, Delta, Hybrid)
+ * - Graceful recovery from interruptions
+ *
+ * Tests against: web-scraping.dev with resumable sessions
+ */
+export {};
+//# sourceMappingURL=07-resumability-demo.d.ts.map

package/dist/examples/07-resumability-demo.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"07-resumability-demo.d.ts","sourceRoot":"","sources":["../../src/examples/07-resumability-demo.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG"}

package/dist/examples/08-worker-monitoring.d.ts

@@ -0,0 +1,15 @@
+/**
+ * Example 08: Worker Health Monitoring and Performance Analysis
+ *
+ * This example demonstrates:
+ * - Worker health monitoring and stuck worker detection
+ * - Memory usage monitoring and limits
+ * - Performance metrics collection
+ * - Concurrent worker management
+ * - Worker lifecycle tracking
+ * - Domain failure detection and recovery
+ *
+ * Tests against: web-scraping.dev with worker monitoring
+ */
+export {};
+//# sourceMappingURL=08-worker-monitoring.d.ts.map

package/dist/examples/08-worker-monitoring.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"08-worker-monitoring.d.ts","sourceRoot":"","sources":["../../src/examples/08-worker-monitoring.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG"}

package/dist/examples/09-error-handling-recovery.d.ts

@@ -0,0 +1,15 @@
+/**
+ * Example 09: Error Handling and Recovery Strategies
+ *
+ * This example demonstrates:
+ * - Comprehensive error handling for network failures
+ * - Recovery strategies for failed requests
+ * - Timeout handling and request cancellation
+ * - Domain failure detection and recovery
+ * - Graceful degradation under adverse conditions
+ * - Error classification and reporting
+ *
+ * Tests against: web-scraping.dev with simulated failures
+ */
+export {};
+//# sourceMappingURL=09-error-handling-recovery.d.ts.map

package/dist/examples/09-error-handling-recovery.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"09-error-handling-recovery.d.ts","sourceRoot":"","sources":["../../src/examples/09-error-handling-recovery.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG"}

package/dist/index.d.ts

@@ -0,0 +1,33 @@
+export type { ISpider, ISpiderScheduler, IMiddlewareManager, IRateLimitMiddleware, ILoggingMiddleware, IUserAgentMiddleware, IStatsMiddleware, } from './lib/api-facades.js';
+export * from './lib/Spider/Spider.service.js';
+export * from './lib/Robots/Robots.service.js';
+export * from './lib/Scraper/Scraper.service.js';
+export * from './lib/PageData/PageData.js';
+export type { SpiderConfigOptions, SpiderConfigService, } from './lib/Config/SpiderConfig.service.js';
+export { SpiderConfig, makeSpiderConfig, } from './lib/Config/SpiderConfig.service.js';
+export type { IUrlDeduplicator } from './lib/UrlDeduplicator/UrlDeduplicator.service.js';
+export { UrlDeduplicatorService } from './lib/UrlDeduplicator/UrlDeduplicator.service.js';
+export type { StatePersistence } from './lib/Scheduler/SpiderScheduler.service.js';
+export { SpiderSchedulerService, SpiderStateKey, PriorityRequest, SpiderState, } from './lib/Scheduler/SpiderScheduler.service.js';
+export type { SpiderMiddleware, SpiderRequest, SpiderResponse, } from './lib/Middleware/SpiderMiddleware.js';
+export { MiddlewareManager, RateLimitMiddleware, LoggingMiddleware, UserAgentMiddleware, StatsMiddleware, } from './lib/Middleware/SpiderMiddleware.js';
+export type { LinkExtractorConfig, LinkExtractionResult, LinkExtractorServiceInterface, } from './lib/LinkExtractor/LinkExtractor.service.js';
+export { LinkExtractorService, LinkExtractorServiceLayer, LinkExtractionError, } from './lib/LinkExtractor/LinkExtractor.service.js';
+export type { CrawlResult, CrawlTask, SpiderLinkExtractionOptions, } from './lib/Spider/Spider.service.js';
+export type { PersistenceStrategy, StateOperation, StorageBackend, StorageCapabilities, HybridPersistenceConfig, } from './lib/Resumability/types.js';
+export type { ResumabilityConfig } from './lib/Resumability/Resumability.service.js';
+export { StateDelta, PersistenceError as ResumabilityError, DEFAULT_HYBRID_CONFIG, } from './lib/Resumability/types.js';
+export { ResumabilityService, ResumabilityConfigs, createStateOperation, } from './lib/Resumability/Resumability.service.js';
+export { FullStatePersistence, DeltaPersistence, HybridPersistence, } from './lib/Resumability/strategies.js';
+export { FileStorageBackend } from './lib/Resumability/backends/FileStorageBackend.js';
+export { NetworkError, ResponseError, RobotsTxtError, ConfigurationError, MiddlewareError, FileSystemError, PersistenceError, } from './lib/errors.js';
+export type { SpiderError } from './lib/errors.js';
+export type { SpiderLogEvent, SpiderLogger, } from './lib/Logging/SpiderLogger.service.js';
+export { SpiderLogger as SpiderLoggerTag, makeSpiderLogger, SpiderLoggerLive, } from './lib/Logging/SpiderLogger.service.js';
+export type { CookieManagerService, EnhancedHttpClientService, HttpRequestOptions, HttpResponse, Session, Credentials, SessionStoreService, TokenInfo, TokenExtractorService, } from './lib/HttpClient/index.js';
+export { CookieManager, makeCookieManager, CookieManagerLive, EnhancedHttpClient, makeEnhancedHttpClient, EnhancedHttpClientLive, SessionStore, makeSessionStore, SessionStoreLive, TokenExtractor, makeTokenExtractor, TokenExtractorLive, } from './lib/HttpClient/index.js';
+export type { Token, StateManagerService } from './lib/StateManager/index.js';
+export { TokenType, StateManager, makeStateManager, StateManagerLive, } from './lib/StateManager/index.js';
+export type { LoginCredentials, ScrapingSession, WebScrapingEngineService, } from './lib/WebScrapingEngine/index.js';
+export { WebScrapingEngine, makeWebScrapingEngine, WebScrapingEngineLive, } from './lib/WebScrapingEngine/index.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,YAAY,EACV,OAAO,EACP,gBAAgB,EAChB,kBAAkB,EAClB,oBAAoB,EACpB,kBAAkB,EAClB,oBAAoB,EACpB,gBAAgB,GACjB,MAAM,sBAAsB,CAAC;AAG9B,cAAc,gCAAgC,CAAC;AAC/C,cAAc,gCAAgC,CAAC;AAC/C,cAAc,kCAAkC,CAAC;AACjD,cAAc,4BAA4B,CAAC;AAG3C,YAAY,EACV,mBAAmB,EACnB,mBAAmB,GACpB,MAAM,sCAAsC,CAAC;AAC9C,OAAO,EACL,YAAY,EACZ,gBAAgB,GACjB,MAAM,sCAAsC,CAAC;AAG9C,YAAY,EAAE,gBAAgB,EAAE,MAAM,kDAAkD,CAAC;AACzF,OAAO,EAAE,sBAAsB,EAAE,MAAM,kDAAkD,CAAC;AAG1F,YAAY,EAAE,gBAAgB,EAAE,MAAM,4CAA4C,CAAC;AACnF,OAAO,EACL,sBAAsB,EACtB,cAAc,EACd,eAAe,EACf,WAAW,GACZ,MAAM,4CAA4C,CAAC;AAGpD,YAAY,EACV,gBAAgB,EAChB,aAAa,EACb,cAAc,GACf,MAAM,sCAAsC,CAAC;AAC9C,OAAO,EACL,iBAAiB,EACjB,mBAAmB,EACnB,iBAAiB,EACjB,mBAAmB,EACnB,eAAe,GAChB,MAAM,sCAAsC,CAAC;AAG9C,YAAY,EACV,mBAAmB,EACnB,oBAAoB,EACpB,6BAA6B,GAC9B,MAAM,8CAA8C,CAAC;AACtD,OAAO,EACL,oBAAoB,EACpB,yBAAyB,EACzB,mBAAmB,GACpB,MAAM,8CAA8C,CAAC;AAGtD,YAAY,EACV,WAAW,EACX,SAAS,EACT,2BAA2B,GAC5B,MAAM,gCAAgC,CAAC;AAGxC,YAAY,EACV,mBAAmB,EACnB,cAAc,EACd,cAAc,EACd,mBAAmB,EACnB,uBAAuB,GACxB,MAAM,6BAA6B,CAAC;AACrC,YAAY,EAAE,kBAAkB,EAAE,MAAM,4CAA4C,CAAC;AACrF,OAAO,EACL,UAAU,EACV,gBAAgB,IAAI,iBAAiB,EACrC,qBAAqB,GACtB,MAAM,6BAA6B,CAAC;AACrC,OAAO,EACL,mBAAmB,EACnB,mBAAmB,EACnB,oBAAoB,GACrB,MAAM,4CAA4C,CAAC;AACpD,OAAO,EACL,oBAAoB,EACpB,gBAAgB,EAChB,iBAAiB,GAClB,MAAM,kCAAkC,CAAC;AAC1C,OAAO,EAAE,kBAAkB,EAAE,MAAM,mDAAmD,CAAC;AAGvF,OAAO,EACL,YAAY,EACZ,aAAa,EACb,cAAc,EACd,kBAAkB,EAClB,eAAe,EACf,eAAe,EACf,gBAAgB,GACjB,MAAM,iBAAiB,CAAC;AACzB,YAAY,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAGnD,YAAY,EACV,cAAc,EACd,YAAY,GACb,MAAM,uCAAuC,CAAC;AAC/C,OAAO,EACL,YAAY,IAAI,eAAe,EAC/B,gBAAgB,EAChB,gBAAgB,GACjB,MAAM,uCAAuC,CAAC;AAG/C,YAAY,EACV,oBAAoB,EACpB,yBAAyB,EACzB,kBAAkB,EAClB,YAAY,EACZ,OAAO,EACP,WAAW,EACX,mBAAmB,EACnB,SAAS,EACT,qBAAqB,GACtB,MAAM,2BAA2B,CAAC;AACnC,OAAO,EACL,aAAa,EACb,iBAAiB,EACjB,iBAAiB,EACjB,kBAAkB,EAClB,sBAAsB,EACtB,sBAAsB,EACtB,YAAY,EACZ,gBAAgB,EAChB,gBAAgB,EAChB,cAAc,EACd,kBAAkB,EAClB,kBAAkB,GACnB,MAAM,2BAA2B,CAAC;AAGnC,YAAY,EAAE,KAAK,EAAE,mBAAmB,EAAE,MAAM,6BAA6B,CAAC;AAC9E,OAAO,EACL,SAAS,EACT,YAAY,EACZ,gBAAgB,EAChB,gBAAgB,GACjB,MAAM,6BAA6B,CAAC;AAGrC,YAAY,EACV,gBAAgB,EAChB,eAAe,EACf,wBAAwB,GACzB,MAAM,kCAAkC,CAAC;AAC1C,OAAO,EACL,iBAAiB,EACjB,qBAAqB,EACrB,qBAAqB,GACtB,MAAM,kCAAkC,CAAC"}