@jambudipa/spider 0.1.0

package/dist/lib/Spider/Spider.service.d.ts

@@ -0,0 +1,194 @@
+import { Effect, Sink } from 'effect';
+import { UrlDeduplicatorService } from '../UrlDeduplicator/UrlDeduplicator.service.js';
+import { ScraperService } from '../Scraper/Scraper.service.js';
+import { PageData } from '../PageData/PageData.js';
+import { RobotsService } from '../Robots/Robots.service.js';
+import { type LinkExtractorConfig, LinkExtractorService } from '../LinkExtractor/index.js';
+import { SpiderLogger } from '../Logging/SpiderLogger.service.js';
+/**
+ * Represents a single crawling task with URL and depth information.
+ *
+ * @group Data Types
+ * @public
+ */
+interface CrawlTask {
+    /** The URL to be crawled */
+    url: string;
+    /** The depth level of this URL relative to the starting URL */
+    depth: number;
+    /** The URL from which this URL was discovered (optional) */
+    fromUrl?: string;
+    /** Optional metadata to be passed through to the result */
+    metadata?: Record<string, unknown>;
+    /** Optional data extraction configuration */
+    extractData?: Record<string, any>;
+}
+/**
+ * The result of a successful crawl operation.
+ *
+ * Contains all extracted information from a crawled page along with
+ * metadata about when and at what depth it was processed.
+ *
+ * @group Data Types
+ * @public
+ */
+interface CrawlResult {
+    /** The extracted page data including content, links, and metadata */
+    pageData: PageData;
+    /** The depth at which this page was crawled */
+    depth: number;
+    /** When this page was crawled */
+    timestamp: Date;
+    /** Optional metadata passed through from the original request */
+    metadata?: Record<string, unknown>;
+}
+/**
+ * The main Spider service that orchestrates web crawling operations.
+ *
+ * This service provides the core functionality for crawling websites, including:
+ * - URL validation and filtering based on configuration
+ * - Robots.txt compliance checking
+ * - Concurrent crawling with configurable worker pools
+ * - Request scheduling and rate limiting
+ * - Result streaming through Effect sinks
+ *
+ * @example
+ * ```typescript
+ * const program = Effect.gen(function* () {
+ *   const spider = yield* Spider;
+ *   const collectSink = Sink.forEach<CrawlResult>(result =>
+ *     Effect.sync(() => console.log(result.pageData.url))
+ *   );
+ *
+ *   const stats = yield* spider.crawl('https://example.com', collectSink);
+ *   console.log(`Crawled ${stats.totalPages} pages`);
+ * });
+ * ```
+ *
+ * @group Services
+ * @public
+ */
+/**
+ * Options for enhanced link extraction during crawling.
+ *
+ * @group Configuration
+ * @public
+ */
+export interface SpiderLinkExtractionOptions {
+    /** Configuration for the LinkExtractorService */
+    readonly linkExtractorConfig?: LinkExtractorConfig;
+    /** Whether to use enhanced extraction in addition to basic extraction (default: false) */
+    readonly useEnhancedExtraction?: boolean;
+    /** Whether to replace basic extraction with enhanced extraction (default: true) */
+    readonly replaceBasicExtraction?: boolean;
+    /** Data extraction configuration for structured data extraction */
+    readonly extractData?: Record<string, any>;
+}
+declare const SpiderService_base: Effect.Service.Class<SpiderService, "@jambudipa.io/Spider", {
+    readonly effect: Effect.Effect<{
+        /**
+         * Starts crawling from the specified URL and processes results through the provided sink.
+         *
+         * This method:
+         * 1. Validates the starting URL against configuration rules
+         * 2. Starts a configurable number of worker fibers
+         * 3. Each worker processes URLs from a shared queue
+         * 4. Results are streamed through the provided sink
+         * 5. New URLs discovered are queued for processing
+         *
+         * @param startingUrls - The starting URL(s) for crawling (single string or array)
+         * @param sink - Sink to process crawl results as they're produced
+         * @param options - Optional enhanced link extraction configuration
+         * @returns Effect containing crawl statistics (total pages, completion status)
+         *
+         * @example
+         * Basic usage:
+         * ```typescript
+         * const collectSink = Sink.forEach<CrawlResult>(result =>
+         *   Effect.sync(() => console.log(`Found: ${result.pageData.title}`))
+         * );
+         *
+         * const stats = yield* spider.crawl('https://example.com', collectSink);
+         * ```
+         *
+         * With multiple starting URLs:
+         * ```typescript
+         * const stats = yield* spider.crawl([
+         *   'https://example.com',
+         *   'https://other-domain.com'
+         * ], collectSink);
+         * ```
+         *
+         * With enhanced link extraction:
+         * ```typescript
+         * const stats = yield* spider.crawl('https://example.com', collectSink, {
+         *   useEnhancedExtraction: true,
+         *   linkExtractorConfig: {
+         *     allowPatterns: [/\/articles\//],
+         *     restrictCss: ['.content a']
+         *   }
+         * });
+         * ```
+         */
+        crawl: <A, E, R>(startingUrls: string | string[] | {
+            url: string;
+            metadata?: Record<string, unknown>;
+        } | {
+            url: string;
+            metadata?: Record<string, unknown>;
+        }[], sink: Sink.Sink<A, CrawlResult, E, R>, options?: SpiderLinkExtractionOptions) => Effect.Effect<{
+            completed: boolean;
+        }, unknown, import("../Config/SpiderConfig.service.js").SpiderConfigService | SpiderLogger>;
+        crawlSingle: <A, E, R>(urlString: string, sink: Sink.Sink<A, CrawlResult, E, R>, options?: SpiderLinkExtractionOptions, initialMetadata?: Record<string, unknown>, restrictToStartingDomain?: boolean) => Effect.Effect<{
+            completed: boolean;
+            pagesScraped: number;
+            domain: string;
+        }, unknown, import("../Config/SpiderConfig.service.js").SpiderConfigService | SpiderLogger>;
+        /**
+         * Resume a previous crawling session from persistent storage.
+         *
+         * This method requires resumability to be enabled in the SpiderConfig and
+         * a StatePersistence implementation to be configured. It will restore the
+         * crawling state and continue processing from where it left off.
+         *
+         * @param stateKey - The unique identifier for the session to resume
+         * @param sink - Sink to process crawl results as they're produced
+         * @param persistence - Optional persistence implementation (uses configured one if not provided)
+         * @returns Effect containing crawl statistics
+         *
+         * @example
+         * ```typescript
+         * const stateKey = new SpiderStateKey({
+         *   id: 'my-crawl-session',
+         *   timestamp: new Date('2024-01-01'),
+         *   name: 'Example Crawl'
+         * });
+         *
+         * const collectSink = Sink.forEach<CrawlResult>(result =>
+         *   Effect.sync(() => console.log(`Resumed: ${result.pageData.title}`))
+         * );
+         *
+         * const stats = yield* spider.resume(stateKey, collectSink);
+         * ```
+         */
+        resume: <A, E, R>(stateKey: import("../Scheduler/SpiderScheduler.service.js").SpiderStateKey, _sink: Sink.Sink<A, CrawlResult, E, R>, _persistence?: import("../Scheduler/SpiderScheduler.service.js").StatePersistence) => Effect.Effect<{
+            completed: boolean;
+            resumed: boolean;
+        }, Error, import("../Config/SpiderConfig.service.js").SpiderConfigService>;
+        /**
+         * Returns the list of URLs that have been visited during crawling.
+         *
+         * @returns Effect containing array of visited URLs
+         *
+         * @remarks
+         * This is currently a placeholder implementation. In a future version,
+         * this will return the actual list of visited URLs from the current session.
+         */
+        getVisitedUrls: () => Effect.Effect<string[], never, never>;
+    }, never, SpiderLogger | ScraperService | RobotsService | LinkExtractorService>;
+    readonly dependencies: readonly [import("effect/Layer").Layer<RobotsService, never, never>, import("effect/Layer").Layer<ScraperService, never, never>, import("effect/Layer").Layer<UrlDeduplicatorService, never, never>, import("effect/Layer").Layer<import("../Config/SpiderConfig.service.js").SpiderConfigService, never, never>, import("effect/Layer").Layer<LinkExtractorService, never, never>, import("effect/Layer").Layer<SpiderLogger, never, never>];
+}>;
+export declare class SpiderService extends SpiderService_base {
+}
+export type { CrawlResult, CrawlTask };
+//# sourceMappingURL=Spider.service.d.ts.map

package/dist/lib/Spider/Spider.service.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"Spider.service.d.ts","sourceRoot":"","sources":["../../../src/lib/Spider/Spider.service.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,MAAM,EAQN,IAAI,EAEL,MAAM,QAAQ,CAAC;AAGhB,OAAO,EAAE,sBAAsB,EAAE,MAAM,+CAA+C,CAAC;AACvF,OAAO,EAAE,cAAc,EAAE,MAAM,+BAA+B,CAAC;AAC/D,OAAO,EAAE,QAAQ,EAAE,MAAM,yBAAyB,CAAC;AACnD,OAAO,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AAC5D,OAAO,EACL,KAAK,mBAAmB,EACxB,oBAAoB,EACrB,MAAM,2BAA2B,CAAC;AAEnC,OAAO,EACL,YAAY,EAEb,MAAM,oCAAoC,CAAC;AAE5C;;;;;GAKG;AACH,UAAU,SAAS;IACjB,4BAA4B;IAC5B,GAAG,EAAE,MAAM,CAAC;IACZ,+DAA+D;IAC/D,KAAK,EAAE,MAAM,CAAC;IACd,4DAA4D;IAC5D,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,2DAA2D;IAC3D,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACnC,6CAA6C;IAC7C,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;CACnC;AAED;;;;;;;;GAQG;AACH,UAAU,WAAW;IACnB,qEAAqE;IACrE,QAAQ,EAAE,QAAQ,CAAC;IACnB,+CAA+C;IAC/C,KAAK,EAAE,MAAM,CAAC;IACd,iCAAiC;IACjC,SAAS,EAAE,IAAI,CAAC;IAChB,iEAAiE;IACjE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH;;;;;GAKG;AACH,MAAM,WAAW,2BAA2B;IAC1C,iDAAiD;IACjD,QAAQ,CAAC,mBAAmB,CAAC,EAAE,mBAAmB,CAAC;IACnD,0FAA0F;IAC1F,QAAQ,CAAC,qBAAqB,CAAC,EAAE,OAAO,CAAC;IACzC,mFAAmF;IACnF,QAAQ,CAAC,sBAAsB,CAAC,EAAE,OAAO,CAAC;IAC1C,mEAAmE;IACnE,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;CAC5C;;;QAuBO;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;WA2CG;gBACK,CAAC,EAAE,CAAC,EAAE,CAAC,gBAET,MAAM,GACN,MAAM,EAAE,GACR;YAAE,GAAG,EAAE,MAAM,CAAC;YAAC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;SAAE,GACnD;YAAE,GAAG,EAAE,MAAM,CAAC;YAAC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;SAAE,EAAE,QACnD,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,CAAC,CAAC,YAC3B,2BAA2B;;;sBA+HzB,CAAC,EAAE,CAAC,EAAE,CAAC,aACR,MAAM,QACX,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,CAAC,CAAC,YAC3B,2BAA2B,oBACnB,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,6BACd,OAAO;;;;;QA28BpC;;;;;;;;;;;;;;;;;;;;;;;;;;WA0BG;iBACM,CAAC,EAAE,CAAC,EAAE,CAAC,YACJ,OAAO,yCAAyC,EAAE,cAAc,SAEnE,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,CAAC,CAAC,iBAEvB,OAAO,yCAAyC,EAAE,gBAAgB;;;;QA0CnF;;;;;;;;WAQG;;;;;AAzuCX,qBAAa,aAAc,SAAQ,kBAwvClC;CAAG;AAEJ,YAAY,EAAE,WAAW,EAAE,SAAS,EAAE,CAAC"}

package/dist/lib/StateManager/StateManager.service.d.ts

@@ -0,0 +1,68 @@
+/**
+ * State Manager Service
+ * Manages tokens, sessions, and client-side storage simulation
+ */
+import { Context, Effect, Layer } from 'effect';
+export declare enum TokenType {
+    CSRF = "csrf",
+    API = "api",
+    AUTH = "auth",
+    REFRESH = "refresh"
+}
+export interface Token {
+    type: TokenType;
+    value: string;
+    expiry?: Date;
+    scope?: string[];
+}
+export interface StateManagerService {
+    /**
+     * Extract CSRF token from HTML
+     */
+    extractCSRFToken: (html: string) => Effect.Effect<string, Error, never>;
+    /**
+     * Extract API token from JavaScript
+     */
+    extractAPIToken: (scripts: string[]) => Effect.Effect<string, Error, never>;
+    /**
+     * Store a token
+     */
+    storeToken: (type: TokenType, token: string, expiry?: Date) => Effect.Effect<void, never, never>;
+    /**
+     * Get a stored token
+     */
+    getToken: (type: TokenType) => Effect.Effect<string, Error, never>;
+    /**
+     * Check if token is valid (not expired)
+     */
+    isTokenValid: (type: TokenType) => Effect.Effect<boolean, never, never>;
+    /**
+     * Simulate local storage
+     */
+    setLocalStorage: (key: string, value: string) => Effect.Effect<void, never, never>;
+    getLocalStorage: (key: string) => Effect.Effect<string, Error, never>;
+    clearLocalStorage: () => Effect.Effect<void, never, never>;
+    /**
+     * Simulate session storage
+     */
+    setSessionStorage: (key: string, value: string) => Effect.Effect<void, never, never>;
+    getSessionStorage: (key: string) => Effect.Effect<string, Error, never>;
+    clearSessionStorage: () => Effect.Effect<void, never, never>;
+    /**
+     * Clear all state
+     */
+    clearState: () => Effect.Effect<void, never, never>;
+}
+declare const StateManager_base: Context.TagClass<StateManager, "StateManager", StateManagerService>;
+export declare class StateManager extends StateManager_base {
+}
+/**
+ * Create a StateManager service implementation
+ */
+export declare const makeStateManager: () => Effect.Effect<StateManagerService, never, never>;
+/**
+ * StateManager Layer
+ */
+export declare const StateManagerLive: Layer.Layer<StateManager, never, never>;
+export {};
+//# sourceMappingURL=StateManager.service.d.ts.map

package/dist/lib/StateManager/StateManager.service.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"StateManager.service.d.ts","sourceRoot":"","sources":["../../../src/lib/StateManager/StateManager.service.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAO,MAAM,QAAQ,CAAC;AAGrD,oBAAY,SAAS;IACnB,IAAI,SAAS;IACb,GAAG,QAAQ;IACX,IAAI,SAAS;IACb,OAAO,YAAY;CACpB;AAED,MAAM,WAAW,KAAK;IACpB,IAAI,EAAE,SAAS,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,IAAI,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;CAClB;AAED,MAAM,WAAW,mBAAmB;IAClC;;OAEG;IACH,gBAAgB,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;IAExE;;OAEG;IACH,eAAe,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;IAE5E;;OAEG;IACH,UAAU,EAAE,CACV,IAAI,EAAE,SAAS,EACf,KAAK,EAAE,MAAM,EACb,MAAM,CAAC,EAAE,IAAI,KACV,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;IAEvC;;OAEG;IACH,QAAQ,EAAE,CAAC,IAAI,EAAE,SAAS,KAAK,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;IAEnE;;OAEG;IACH,YAAY,EAAE,CAAC,IAAI,EAAE,SAAS,KAAK,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;IAExE;;OAEG;IACH,eAAe,EAAE,CACf,GAAG,EAAE,MAAM,EACX,KAAK,EAAE,MAAM,KACV,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;IACvC,eAAe,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;IACtE,iBAAiB,EAAE,MAAM,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;IAE3D;;OAEG;IACH,iBAAiB,EAAE,CACjB,GAAG,EAAE,MAAM,EACX,KAAK,EAAE,MAAM,KACV,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;IACvC,iBAAiB,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;IACxE,mBAAmB,EAAE,MAAM,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;IAE7D;;OAEG;IACH,UAAU,EAAE,MAAM,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;CACrD;;AAED,qBAAa,YAAa,SAAQ,iBAG/B;CAAG;AAEN;;GAEG;AACH,eAAO,MAAM,gBAAgB,QAAO,MAAM,CAAC,MAAM,CAC/C,mBAAmB,EACnB,KAAK,EACL,KAAK,CAgNH,CAAC;AAEL;;GAEG;AACH,eAAO,MAAM,gBAAgB,yCAAiD,CAAC"}

package/dist/lib/StateManager/index.d.ts

@@ -0,0 +1,5 @@
+/**
+ * State Manager module exports
+ */
+export * from './StateManager.service.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/lib/StateManager/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/lib/StateManager/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,cAAc,2BAA2B,CAAC"}

package/dist/lib/UrlDeduplicator/UrlDeduplicator.service.d.ts

@@ -0,0 +1,58 @@
+import { Effect, MutableHashSet } from 'effect';
+/**
+ * Thread-safe URL deduplication service with built-in normalization.
+ *
+ * Provides atomic operations for checking and adding URLs to prevent
+ * race conditions in concurrent environments. URLs are normalized
+ * before storage to ensure consistent deduplication.
+ *
+ * @group Services
+ * @public
+ */
+export interface IUrlDeduplicator {
+    /**
+     * Attempts to add a URL to the deduplication set.
+     *
+     * @param url - The URL to add
+     * @returns Effect containing boolean - true if URL was added (first time seen), false if already exists
+     */
+    tryAdd(url: string): Effect.Effect<boolean>;
+    /**
+     * Checks if a URL has already been seen.
+     *
+     * @param url - The URL to check
+     * @returns Effect containing boolean - true if URL exists, false otherwise
+     */
+    contains(url: string): Effect.Effect<boolean>;
+    /**
+     * Returns the current number of unique URLs in the set.
+     *
+     * @returns Effect containing the count
+     */
+    size(): Effect.Effect<number>;
+    /**
+     * Clears all URLs from the deduplication set.
+     *
+     * @returns Effect containing void
+     */
+    clear(): Effect.Effect<void>;
+}
+declare const UrlDeduplicatorService_base: Effect.Service.Class<UrlDeduplicatorService, "@jambudipa.io/UrlDeduplicatorService", {
+    readonly effect: Effect.Effect<{
+        tryAdd: (url: string) => Effect.Effect<boolean, never, never>;
+        contains: (url: string) => Effect.Effect<boolean, never, never>;
+        size: () => Effect.Effect<number, never, never>;
+        clear: () => Effect.Effect<MutableHashSet.MutableHashSet<string>, never, never>;
+    }, never, import("../Config/SpiderConfig.service.js").SpiderConfigService>;
+    readonly dependencies: readonly [import("effect/Layer").Layer<import("../Config/SpiderConfig.service.js").SpiderConfigService, never, never>];
+}>;
+/**
+ * URL deduplication service as an Effect Service.
+ *
+ * @group Services
+ * @public
+ */
+export declare class UrlDeduplicatorService extends UrlDeduplicatorService_base {
+}
+export {};
+//# sourceMappingURL=UrlDeduplicator.service.d.ts.map

package/dist/lib/UrlDeduplicator/UrlDeduplicator.service.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"UrlDeduplicator.service.d.ts","sourceRoot":"","sources":["../../../src/lib/UrlDeduplicator/UrlDeduplicator.service.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,QAAQ,CAAC;AAGhD;;;;;;;;;GASG;AACH,MAAM,WAAW,gBAAgB;IAC/B;;;;;OAKG;IACH,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAE5C;;;;;OAKG;IACH,QAAQ,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAE9C;;;;OAIG;IACH,IAAI,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAE9B;;;;OAIG;IACH,KAAK,IAAI,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;CAC9B;;;sBA2EqB,MAAM;wBAcJ,MAAM;;;;;;AAvF9B;;;;;GAKG;AACH,qBAAa,sBAAuB,SAAQ,2BAsG3C;CAAG"}

package/dist/lib/WebScrapingEngine/WebScrapingEngine.service.d.ts

@@ -0,0 +1,77 @@
+/**
+ * Web Scraping Engine Service
+ * Orchestrates all scraping capabilities including authentication, token management, and session handling
+ */
+import { Context, Effect, Layer } from 'effect';
+import { ScraperService } from '../Scraper/Scraper.service.js';
+import { EnhancedHttpClient, type HttpResponse } from '../HttpClient/EnhancedHttpClient.js';
+import { CookieManager } from '../HttpClient/CookieManager.js';
+import { SessionStore } from '../HttpClient/SessionStore.js';
+import { TokenExtractor } from '../HttpClient/TokenExtractor.js';
+import { StateManager, TokenType } from '../StateManager/StateManager.service.js';
+import { SpiderLogger } from '../Logging/SpiderLogger.service.js';
+export interface LoginCredentials {
+    username: string;
+    password: string;
+    loginUrl: string;
+    usernameField?: string;
+    passwordField?: string;
+    additionalFields?: Record<string, string>;
+}
+export interface ScrapingSession {
+    id: string;
+    authenticated: boolean;
+    tokens: Map<TokenType, string>;
+    startTime: Date;
+}
+export interface WebScrapingEngineService {
+    /**
+     * Perform login with form submission
+     */
+    login: (credentials: LoginCredentials) => Effect.Effect<ScrapingSession, Error, never>;
+    /**
+     * Fetch authenticated content
+     */
+    fetchAuthenticated: (url: string) => Effect.Effect<HttpResponse, Error, never>;
+    /**
+     * Submit form with CSRF protection
+     */
+    submitFormWithCSRF: (url: string, formData: Record<string, string>, csrfUrl?: string) => Effect.Effect<HttpResponse, Error, never>;
+    /**
+     * Make API request with token
+     */
+    makeAPIRequest: (url: string, method?: 'GET' | 'POST' | 'PUT' | 'DELETE', data?: any) => Effect.Effect<HttpResponse, Error, never>;
+    /**
+     * Create and save a scraping session
+     */
+    createSession: (id?: string) => Effect.Effect<ScrapingSession, Error, never>;
+    /**
+     * Load existing session
+     */
+    loadSession: (id: string) => Effect.Effect<ScrapingSession, Error, never>;
+    /**
+     * Export session for persistence
+     */
+    exportSession: () => Effect.Effect<string, Error, never>;
+    /**
+     * Import session from persistence
+     */
+    importSession: (data: string) => Effect.Effect<void, Error, never>;
+    /**
+     * Clear all state and sessions
+     */
+    clearAll: () => Effect.Effect<void, never, never>;
+}
+declare const WebScrapingEngine_base: Context.TagClass<WebScrapingEngine, "WebScrapingEngine", WebScrapingEngineService>;
+export declare class WebScrapingEngine extends WebScrapingEngine_base {
+}
+/**
+ * Create a WebScrapingEngine service implementation
+ */
+export declare const makeWebScrapingEngine: Effect.Effect<WebScrapingEngineService, never, SpiderLogger | ScraperService | CookieManager | EnhancedHttpClient | StateManager | SessionStore | TokenExtractor>;
+/**
+ * WebScrapingEngine Layer with all dependencies
+ */
+export declare const WebScrapingEngineLive: Layer.Layer<WebScrapingEngine, never, SpiderLogger | ScraperService | CookieManager | EnhancedHttpClient | StateManager | SessionStore | TokenExtractor>;
+export {};
+//# sourceMappingURL=WebScrapingEngine.service.d.ts.map

package/dist/lib/WebScrapingEngine/WebScrapingEngine.service.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"WebScrapingEngine.service.d.ts","sourceRoot":"","sources":["../../../src/lib/WebScrapingEngine/WebScrapingEngine.service.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,QAAQ,CAAC;AAChD,OAAO,EAAE,cAAc,EAAE,MAAM,+BAA+B,CAAC;AAC/D,OAAO,EACL,kBAAkB,EAClB,KAAK,YAAY,EAClB,MAAM,qCAAqC,CAAC;AAC7C,OAAO,EAAE,aAAa,EAAE,MAAM,gCAAgC,CAAC;AAC/D,OAAO,EAAE,YAAY,EAAE,MAAM,+BAA+B,CAAC;AAC7D,OAAO,EAAE,cAAc,EAAE,MAAM,iCAAiC,CAAC;AACjE,OAAO,EACL,YAAY,EACZ,SAAS,EACV,MAAM,yCAAyC,CAAC;AACjD,OAAO,EAAE,YAAY,EAAE,MAAM,oCAAoC,CAAC;AAElE,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,gBAAgB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAC3C;AAED,MAAM,WAAW,eAAe;IAC9B,EAAE,EAAE,MAAM,CAAC;IACX,aAAa,EAAE,OAAO,CAAC;IACvB,MAAM,EAAE,GAAG,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAC/B,SAAS,EAAE,IAAI,CAAC;CACjB;AAED,MAAM,WAAW,wBAAwB;IACvC;;OAEG;IACH,KAAK,EAAE,CACL,WAAW,EAAE,gBAAgB,KAC1B,MAAM,CAAC,MAAM,CAAC,eAAe,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;IAElD;;OAEG;IACH,kBAAkB,EAAE,CAClB,GAAG,EAAE,MAAM,KACR,MAAM,CAAC,MAAM,CAAC,YAAY,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;IAE/C;;OAEG;IACH,kBAAkB,EAAE,CAClB,GAAG,EAAE,MAAM,EACX,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EAChC,OAAO,CAAC,EAAE,MAAM,KACb,MAAM,CAAC,MAAM,CAAC,YAAY,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;IAE/C;;OAEG;IACH,cAAc,EAAE,CACd,GAAG,EAAE,MAAM,EACX,MAAM,CAAC,EAAE,KAAK,GAAG,MAAM,GAAG,KAAK,GAAG,QAAQ,EAC1C,IAAI,CAAC,EAAE,GAAG,KACP,MAAM,CAAC,MAAM,CAAC,YAAY,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;IAE/C;;OAEG;IACH,aAAa,EAAE,CAAC,EAAE,CAAC,EAAE,MAAM,KAAK,MAAM,CAAC,MAAM,CAAC,eAAe,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;IAE7E;;OAEG;IACH,WAAW,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,MAAM,CAAC,MAAM,CAAC,eAAe,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;IAE1E;;OAEG;IACH,aAAa,EAAE,MAAM,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;IAEzD;;OAEG;IACH,aAAa,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;IAEnE;;OAEG;IACH,QAAQ,EAAE,MAAM,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;CACnD;;AAED,qBAAa,iBAAkB,SAAQ,sBAGpC;CAAG;AAEN;;GAEG;AACH,eAAO,MAAM,qBAAqB,mKAoRhC,CAAC;AAEH;;GAEG;AACH,eAAO,MAAM,qBAAqB,0JAGjC,CAAC"}

package/dist/lib/WebScrapingEngine/index.d.ts

@@ -0,0 +1,5 @@
+/**
+ * Web Scraping Engine module exports
+ */
+export * from './WebScrapingEngine.service.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/lib/WebScrapingEngine/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/lib/WebScrapingEngine/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,cAAc,gCAAgC,CAAC"}

package/dist/lib/WorkerHealth/WorkerHealthMonitor.service.d.ts

@@ -0,0 +1,39 @@
+import { Effect } from 'effect';
+import { SpiderLogger } from '../Logging/SpiderLogger.service.js';
+interface WorkerStatus {
+    workerId: string;
+    domain: string;
+    currentUrl?: string;
+    lastActivity: Date;
+    fetchStartTime?: Date;
+}
+declare const WorkerHealthMonitor_base: Effect.Service.Class<WorkerHealthMonitor, "@jambudipa.io/WorkerHealthMonitor", {
+    readonly effect: Effect.Effect<{
+        /**
+         * Register a worker's activity
+         */
+        recordActivity: (workerId: string, domain: string, activity: {
+            url?: string;
+            fetchStart?: boolean;
+        }) => Effect.Effect<void, never, never>;
+        /**
+         * Remove a worker from monitoring
+         */
+        removeWorker: (workerId: string) => Effect.Effect<void, never, never>;
+        /**
+         * Get stuck workers
+         */
+        getStuckWorkers: Effect.Effect<WorkerStatus[], never, never>;
+        /**
+         * Monitor workers and log stuck ones
+         */
+        startMonitoring: Effect.Effect<void, never, never>;
+    }, never, SpiderLogger>;
+}>;
+/**
+ * Monitors worker health and kills stuck workers
+ */
+export declare class WorkerHealthMonitor extends WorkerHealthMonitor_base {
+}
+export {};
+//# sourceMappingURL=WorkerHealthMonitor.service.d.ts.map

package/dist/lib/WorkerHealth/WorkerHealthMonitor.service.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"WorkerHealthMonitor.service.d.ts","sourceRoot":"","sources":["../../../src/lib/WorkerHealth/WorkerHealthMonitor.service.ts"],"names":[],"mappings":"AAAA,OAAO,EAAY,MAAM,EAAiB,MAAM,QAAQ,CAAC;AACzD,OAAO,EAAE,YAAY,EAAE,MAAM,oCAAoC,CAAC;AAElE,UAAU,YAAY;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,IAAI,CAAC;IACnB,cAAc,CAAC,EAAE,IAAI,CAAC;CACvB;;;QAcO;;WAEG;mCAES,MAAM,UACR,MAAM,YACJ;YAAE,GAAG,CAAC,EAAE,MAAM,CAAC;YAAC,UAAU,CAAC,EAAE,OAAO,CAAA;SAAE;QAuBlD;;WAEG;iCACsB,MAAM;QAO/B;;WAEG;;QAgBH;;WAEG;;;;AAvEX;;GAEG;AACH,qBAAa,mBAAoB,SAAQ,wBAmHxC;CAAG"}