@jambudipa/spider 0.2.0 → 0.2.3

package/dist/lib/Scraper/Scraper.service.d.ts

@@ -0,0 +1,123 @@
+import { Effect } from 'effect';
+import { NetworkError, ResponseError, ContentTypeError, RequestAbortError } from '../errors.js';
+import { SpiderLogger } from '../Logging/SpiderLogger.service.js';
+declare const ScraperService_base: Effect.Service.Class<ScraperService, "@jambudipa.io/ScraperService", {
+    readonly effect: Effect.Effect<{
+        /**
+         * Fetches a URL and parses the HTML to extract basic page information.
+         *
+         * This method performs the following operations:
+         * 1. Fetches the URL with configurable timeout (30 seconds)
+         * 2. Validates content type (skips binary files)
+         * 3. Parses HTML content with cheerio
+         * 4. Extracts basic page metadata (title, description, etc.)
+         * 5. Returns structured PageData object
+         *
+         * The method uses AbortController for proper timeout handling to prevent
+         * workers from hanging on malformed URLs or slow responses.
+         *
+         * @param url - The URL to fetch and parse
+         * @param depth - The crawl depth for logging purposes (default: 0)
+         * @returns Effect containing PageData with extracted information
+         * @throws NetworkError for network-related failures
+         * @throws ResponseError for HTTP error responses
+         *
+         * @example
+         * Basic usage:
+         * ```typescript
+         * const pageData = yield* scraper.fetchAndParse('https://example.com');
+         * console.log(`Page title: ${pageData.title}`);
+         * ```
+         *
+         * With depth tracking:
+         * ```typescript
+         * const pageData = yield* scraper.fetchAndParse('https://example.com/page', 2);
+         * ```
+         *
+         * Error handling:
+         * ```typescript
+         * const result = yield* scraper.fetchAndParse('https://example.com').pipe(
+         *   Effect.catchTags({
+         *     NetworkError: (error) => {
+         *       console.log('Network error:', error.message);
+         *       return Effect.succeed(null);
+         *     },
+         *     ResponseError: (error) => {
+         *       console.log('HTTP error:', error.statusCode);
+         *       return Effect.succeed(null);
+         *     }
+         *   })
+         * );
+         * ```
+         *
+         * @performance
+         * - Request timeout: 30 seconds
+         * - Response parsing timeout: 10 seconds
+         * - Memory usage: ~2-5MB per page depending on content size
+         *
+         * @security
+         * - Validates content types to prevent processing binary files
+         * - Uses AbortController to prevent hanging requests
+         * - No execution of JavaScript content (static HTML parsing only)
+         */
+        fetchAndParse: (url: string, depth?: number) => Effect.Effect<{
+            readonly url: string;
+            readonly html: string;
+            readonly title?: string | undefined;
+            readonly metadata: {
+                readonly [x: string]: string;
+            };
+            readonly commonMetadata?: {
+                readonly description?: string | undefined;
+                readonly keywords?: string | undefined;
+                readonly author?: string | undefined;
+                readonly robots?: string | undefined;
+            } | undefined;
+            readonly statusCode: number;
+            readonly headers: {
+                readonly [x: string]: string;
+            };
+            readonly fetchedAt: Date;
+            readonly scrapeDurationMs: number;
+            readonly depth: number;
+            readonly extractedData?: {
+                readonly [x: string]: unknown;
+            } | undefined;
+        }, NetworkError | ResponseError | ContentTypeError | RequestAbortError | import("effect/ParseResult").ParseError, SpiderLogger>;
+    }, never, never>;
+}>;
+/**
+ * Service responsible for fetching HTML content and parsing basic page information.
+ *
+ * The ScraperService handles the core HTTP fetching and HTML parsing functionality
+ * for the Spider framework. It provides robust error handling, timeout management,
+ * and content type validation to ensure reliable data extraction.
+ *
+ * **Key Features:**
+ * - Automatic timeout handling with AbortController
+ * - Content type validation (skips binary files)
+ * - Comprehensive error handling with typed errors
+ * - Performance monitoring and logging
+ * - Effect integration for composability
+ *
+ * **Note:** This service focuses solely on fetching and parsing HTML content.
+ * Link extraction is handled separately by LinkExtractorService for better
+ * separation of concerns and modularity.
+ *
+ * @example
+ * ```typescript
+ * const program = Effect.gen(function* () {
+ *   const scraper = yield* ScraperService;
+ *   const pageData = yield* scraper.fetchAndParse('https://example.com', 0);
+ *   console.log(`Title: ${pageData.title}`);
+ *   console.log(`Content length: ${pageData.html.length}`);
+ * });
+ * ```
+ *
+ * @group Services
+ * @public
+ */
+export declare class ScraperService extends ScraperService_base {
+}
+export {};
+//# sourceMappingURL=Scraper.service.d.ts.map

package/dist/lib/Scraper/Scraper.service.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"Scraper.service.d.ts","sourceRoot":"","sources":["../../../src/lib/Scraper/Scraper.service.ts"],"names":[],"mappings":"AAAA,OAAO,EAAsB,MAAM,EAAkB,MAAM,QAAQ,CAAC;AAGpE,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MAAM,cAAc,CAAC;AAChG,OAAO,EAAE,YAAY,EAAE,MAAM,oCAAoC,CAAC;;;QAqC5D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;WAwDG;6BACkB,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;AA5FjC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AACH,qBAAa,cAAe,SAAQ,mBAoOnC;CAAG"}

package/dist/lib/Spider/Spider.service.d.ts

@@ -0,0 +1,249 @@
+import { Effect, Sink } from 'effect';
+import { UrlDeduplicatorService } from '../UrlDeduplicator/UrlDeduplicator.service.js';
+import { ScraperService } from '../Scraper/Scraper.service.js';
+import { PageData } from '../PageData/PageData.js';
+import { RobotsService } from '../Robots/Robots.service.js';
+import { type LinkExtractorConfig, LinkExtractorService } from '../LinkExtractor/index.js';
+import { SpiderSchedulerService } from '../Scheduler/SpiderScheduler.service.js';
+import { SpiderLogger } from '../Logging/SpiderLogger.service.js';
+/**
+ * Configuration for extracting a nested field from an element.
+ *
+ * @group Data Types
+ * @public
+ */
+interface NestedFieldConfig {
+    /** CSS selector to find the nested element */
+    readonly selector: string;
+    /** HTML attribute to extract (if not specified, extracts text content) */
+    readonly attribute?: string;
+}
+/**
+ * Configuration for extracting a single field from the page.
+ *
+ * @group Data Types
+ * @public
+ */
+interface FieldExtractionConfig {
+    /** CSS selector to find the element */
+    readonly selector: string;
+    /** Extract text content (default: true) */
+    readonly text?: boolean;
+    /** HTML attribute to extract instead of text */
+    readonly attribute?: string;
+    /** Extract multiple matching elements */
+    readonly multiple?: boolean;
+    /** Check if element exists (returns boolean) */
+    readonly exists?: boolean;
+    /** Nested fields to extract from each matched element */
+    readonly fields?: Record<string, NestedFieldConfig>;
+}
+/**
+ * Data extraction configuration - either a simple CSS selector string
+ * or a detailed field extraction configuration.
+ *
+ * @group Data Types
+ * @public
+ */
+type DataExtractionFieldConfig = string | FieldExtractionConfig;
+/**
+ * Configuration for extracting structured data from pages.
+ *
+ * @group Data Types
+ * @public
+ */
+type DataExtractionConfig = Record<string, DataExtractionFieldConfig>;
+/**
+ * Represents a single crawling task with URL and depth information.
+ *
+ * @group Data Types
+ * @public
+ */
+interface CrawlTask {
+    /** The URL to be crawled */
+    url: string;
+    /** The depth level of this URL relative to the starting URL */
+    depth: number;
+    /** The URL from which this URL was discovered (optional) */
+    fromUrl?: string;
+    /** Optional metadata to be passed through to the result */
+    metadata?: Record<string, unknown>;
+    /** Optional data extraction configuration */
+    extractData?: DataExtractionConfig;
+}
+/**
+ * The result of a successful crawl operation.
+ *
+ * Contains all extracted information from a crawled page along with
+ * metadata about when and at what depth it was processed.
+ *
+ * @group Data Types
+ * @public
+ */
+interface CrawlResult {
+    /** The extracted page data including content, links, and metadata */
+    pageData: PageData;
+    /** The depth at which this page was crawled */
+    depth: number;
+    /** When this page was crawled */
+    timestamp: Date;
+    /** Optional metadata passed through from the original request */
+    metadata?: Record<string, unknown>;
+}
+/**
+ * The main Spider service that orchestrates web crawling operations.
+ *
+ * This service provides the core functionality for crawling websites, including:
+ * - URL validation and filtering based on configuration
+ * - Robots.txt compliance checking
+ * - Concurrent crawling with configurable worker pools
+ * - Request scheduling and rate limiting
+ * - Result streaming through Effect sinks
+ *
+ * @example
+ * ```typescript
+ * const program = Effect.gen(function* () {
+ *   const spider = yield* Spider;
+ *   const collectSink = Sink.forEach<CrawlResult>(result =>
+ *     Effect.sync(() => console.log(result.pageData.url))
+ *   );
+ *
+ *   const stats = yield* spider.crawl('https://example.com', collectSink);
+ *   console.log(`Crawled ${stats.totalPages} pages`);
+ * });
+ * ```
+ *
+ * @group Services
+ * @public
+ */
+/**
+ * Options for enhanced link extraction during crawling.
+ *
+ * @group Configuration
+ * @public
+ */
+export interface SpiderLinkExtractionOptions {
+    /** Configuration for the LinkExtractorService */
+    readonly linkExtractorConfig?: LinkExtractorConfig;
+    /** Whether to use enhanced extraction in addition to basic extraction (default: false) */
+    readonly useEnhancedExtraction?: boolean;
+    /** Whether to replace basic extraction with enhanced extraction (default: true) */
+    readonly replaceBasicExtraction?: boolean;
+    /** Data extraction configuration for structured data extraction */
+    readonly extractData?: DataExtractionConfig;
+}
+declare const SpiderService_base: Effect.Service.Class<SpiderService, "@jambudipa/spider", {
+    readonly effect: Effect.Effect<{
+        /**
+         * Starts crawling from the specified URL and processes results through the provided sink.
+         *
+         * This method:
+         * 1. Validates the starting URL against configuration rules
+         * 2. Starts a configurable number of worker fibers
+         * 3. Each worker processes URLs from a shared queue
+         * 4. Results are streamed through the provided sink
+         * 5. New URLs discovered are queued for processing
+         *
+         * @param startingUrls - The starting URL(s) for crawling (single string or array)
+         * @param sink - Sink to process crawl results as they're produced
+         * @param options - Optional enhanced link extraction configuration
+         * @returns Effect containing crawl statistics (total pages, completion status)
+         *
+         * @example
+         * Basic usage:
+         * ```typescript
+         * const collectSink = Sink.forEach<CrawlResult>(result =>
+         *   Effect.sync(() => console.log(`Found: ${result.pageData.title}`))
+         * );
+         *
+         * const stats = yield* spider.crawl('https://example.com', collectSink);
+         * ```
+         *
+         * With multiple starting URLs:
+         * ```typescript
+         * const stats = yield* spider.crawl([
+         *   'https://example.com',
+         *   'https://other-domain.com'
+         * ], collectSink);
+         * ```
+         *
+         * With enhanced link extraction:
+         * ```typescript
+         * const stats = yield* spider.crawl('https://example.com', collectSink, {
+         *   useEnhancedExtraction: true,
+         *   linkExtractorConfig: {
+         *     allowPatterns: [/\/articles\//],
+         *     restrictCss: ['.content a']
+         *   }
+         * });
+         * ```
+         */
+        crawl: <A, E, R>(startingUrls: string | string[] | {
+            url: string;
+            metadata?: Record<string, unknown>;
+        } | {
+            url: string;
+            metadata?: Record<string, unknown>;
+        }[], sink: Sink.Sink<A, CrawlResult, E, R>, options?: SpiderLinkExtractionOptions) => Effect.Effect<{
+            completed: boolean;
+        }, unknown, import("../Config/SpiderConfig.service.js").SpiderConfigService | SpiderLogger>;
+        crawlSingle: <A, E, R>(urlString: string, sink: Sink.Sink<A, CrawlResult, E, R>, options?: SpiderLinkExtractionOptions, initialMetadata?: Record<string, unknown>, restrictToStartingDomain?: boolean) => Effect.Effect<{
+            completed: boolean;
+            pagesScraped: number;
+            domain: string;
+        }, unknown, import("../Config/SpiderConfig.service.js").SpiderConfigService | SpiderLogger>;
+        /**
+         * Resume a previous crawling session from persistent storage.
+         *
+         * This method requires resumability to be enabled in the SpiderConfig and
+         * a StatePersistence implementation to be configured. It will restore the
+         * crawling state and continue processing from where it left off.
+         *
+         * @param stateKey - The unique identifier for the session to resume
+         * @param sink - Sink to process crawl results as they're produced
+         * @param persistence - Optional persistence implementation (uses configured one if not provided)
+         * @returns Effect containing crawl statistics
+         *
+         * @example
+         * ```typescript
+         * const stateKey = new SpiderStateKey({
+         *   id: 'my-crawl-session',
+         *   timestamp: new Date('2024-01-01'),
+         *   name: 'Example Crawl'
+         * });
+         *
+         * const collectSink = Sink.forEach<CrawlResult>(result =>
+         *   Effect.sync(() => console.log(`Resumed: ${result.pageData.title}`))
+         * );
+         *
+         * const stats = yield* spider.resume(stateKey, collectSink);
+         * ```
+         */
+        resume: <A, E, R>(stateKey: import("../Scheduler/SpiderScheduler.service.js").SpiderStateKey, resumeSink: Sink.Sink<A, CrawlResult, E, R>, _persistence?: import("../Scheduler/SpiderScheduler.service.js").StatePersistence) => Effect.Effect<{
+            resumed: boolean;
+            sessionId: string;
+            completed: boolean;
+            urlsProcessed?: undefined;
+        } | {
+            completed: boolean;
+            resumed: boolean;
+            sessionId: string;
+            urlsProcessed: number;
+        }, unknown, import("../Config/SpiderConfig.service.js").SpiderConfigService | SpiderLogger | SpiderSchedulerService>;
+        /**
+         * Returns the list of URLs that have been visited during crawling.
+         *
+         * @returns Effect containing array of visited URLs
+         *
+         * @remarks
+         * This is currently a placeholder implementation. In a future version,
+         * this will return the actual list of visited URLs from the current session.
+         */
+        getVisitedUrls: () => Effect.Effect<string[]>;
+    }, never, SpiderLogger | ScraperService | RobotsService | LinkExtractorService>;
+    readonly dependencies: readonly [import("effect/Layer").Layer<RobotsService, never, never>, import("effect/Layer").Layer<ScraperService, never, never>, import("effect/Layer").Layer<UrlDeduplicatorService, never, never>, import("effect/Layer").Layer<import("../Config/SpiderConfig.service.js").SpiderConfigService, never, never>, import("effect/Layer").Layer<LinkExtractorService, never, never>, import("effect/Layer").Layer<SpiderLogger, never, never>];
+}>;
+export declare class SpiderService extends SpiderService_base {
+}
+export type { CrawlResult, CrawlTask };
+//# sourceMappingURL=Spider.service.d.ts.map

package/dist/lib/Spider/Spider.service.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"Spider.service.d.ts","sourceRoot":"","sources":["../../../src/lib/Spider/Spider.service.ts"],"names":[],"mappings":"AAAA,OAAO,EAGL,MAAM,EASN,IAAI,EAEL,MAAM,QAAQ,CAAC;AAIhB,OAAO,EAAE,sBAAsB,EAAE,MAAM,+CAA+C,CAAC;AACvF,OAAO,EAAE,cAAc,EAAE,MAAM,+BAA+B,CAAC;AAC/D,OAAO,EAAE,QAAQ,EAAE,MAAM,yBAAyB,CAAC;AACnD,OAAO,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AAC5D,OAAO,EACL,KAAK,mBAAmB,EACxB,oBAAoB,EACrB,MAAM,2BAA2B,CAAC;AACnC,OAAO,EAAE,sBAAsB,EAAE,MAAM,yCAAyC,CAAC;AAEjF,OAAO,EACL,YAAY,EAEb,MAAM,oCAAoC,CAAC;AAG5C;;;;;GAKG;AACH,UAAU,iBAAiB;IACzB,8CAA8C;IAC9C,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,0EAA0E;IAC1E,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;CAC7B;AAED;;;;;GAKG;AACH,UAAU,qBAAqB;IAC7B,uCAAuC;IACvC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,2CAA2C;IAC3C,QAAQ,CAAC,IAAI,CAAC,EAAE,OAAO,CAAC;IACxB,gDAAgD;IAChD,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,yCAAyC;IACzC,QAAQ,CAAC,QAAQ,CAAC,EAAE,OAAO,CAAC;IAC5B,gDAAgD;IAChD,QAAQ,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC;IAC1B,yDAAyD;IACzD,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,iBAAiB,CAAC,CAAC;CACrD;AAED;;;;;;GAMG;AACH,KAAK,yBAAyB,GAAG,MAAM,GAAG,qBAAqB,CAAC;AAEhE;;;;;GAKG;AACH,KAAK,oBAAoB,GAAG,MAAM,CAAC,MAAM,EAAE,yBAAyB,CAAC,CAAC;AAEtE;;;;;GAKG;AACH,UAAU,SAAS;IACjB,4BAA4B;IAC5B,GAAG,EAAE,MAAM,CAAC;IACZ,+DAA+D;IAC/D,KAAK,EAAE,MAAM,CAAC;IACd,4DAA4D;IAC5D,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,2DAA2D;IAC3D,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACnC,6CAA6C;IAC7C,WAAW,CAAC,EAAE,oBAAoB,CAAC;CACpC;AAED;;;;;;;;GAQG;AACH,UAAU,WAAW;IACnB,qEAAqE;IACrE,QAAQ,EAAE,QAAQ,CAAC;IACnB,+CAA+C;IAC/C,KAAK,EAAE,MAAM,CAAC;IACd,iCAAiC;IACjC,SAAS,EAAE,IAAI,CAAC;IAChB,iEAAiE;IACjE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH;;;;;GAKG;AACH,MAAM,WAAW,2BAA2B;IAC1C,iDAAiD;IACjD,QAAQ,CAAC,mBAAmB,CAAC,EAAE,mBAAmB,CAAC;IACnD,0FAA0F;IAC1F,QAAQ,CAAC,qBAAqB,CAAC,EAAE,OAAO,CAAC;IACzC,mFAAmF;IACnF,QAAQ,CAAC,sBAAsB,CAAC,EAAE,OAAO,CAAC;IAC1C,mEAAmE;IACnE,QAAQ,CAAC,WAAW,CAAC,EAAE,oBAAoB,CAAC;CAC7C;;;QAqBO;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;WA2CG;gBACK,CAAC,EAAE,CAAC,EAAE,CAAC,gBAET,MAAM,GACN,MAAM,EAAE,GACR;YAAE,GAAG,EAAE,MAAM,CAAC;YAAC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;SAAE,GACnD;YAAE,GAAG,EAAE,MAAM,CAAC;YAAC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;SAAE,EAAE,QACnD,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,CAAC,CAAC,YAC3B,2BAA2B;;;sBA0HzB,CAAC,EAAE,CAAC,EAAE,CAAC,aACR,MAAM,QACX,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,CAAC,CAAC,YAC3B,2BAA2B,oBACnB,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,6BACd,OAAO;;;;;QA6gCpC;;;;;;;;;;;;;;;;;;;;;;;;;;WA0BG;iBACM,CAAC,EAAE,CAAC,EAAE,CAAC,YACJ,OAAO,yCAAyC,EAAE,cAAc,cAC9D,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,CAAC,CAAC,iBAC5B,OAAO,yCAAyC,EAAE,gBAAgB;;;;;;;;;;;QAyInF;;;;;;;;WAQG;8BACiB,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;;;;AAl4CnD,qBAAa,aAAc,SAAQ,kBAg5ClC;CAAG;AAEJ,YAAY,EAAE,WAAW,EAAE,SAAS,EAAE,CAAC"}

package/dist/lib/StateManager/StateManager.service.d.ts

@@ -0,0 +1,107 @@
+/**
+ * State Manager Service
+ * Manages tokens, sessions, and client-side storage simulation
+ */
+import { Context, Effect, Layer } from 'effect';
+declare const CSRFTokenNotFoundError_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").Equals<A, {}> extends true ? void : { readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }) => import("effect/Cause").YieldableError & {
+    readonly _tag: "CSRFTokenNotFoundError";
+} & Readonly<A>;
+export declare class CSRFTokenNotFoundError extends CSRFTokenNotFoundError_base<{
+    readonly message: string;
+}> {
+}
+declare const APITokenNotFoundError_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").Equals<A, {}> extends true ? void : { readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }) => import("effect/Cause").YieldableError & {
+    readonly _tag: "APITokenNotFoundError";
+} & Readonly<A>;
+export declare class APITokenNotFoundError extends APITokenNotFoundError_base<{
+    readonly message: string;
+}> {
+}
+declare const TokenNotFoundError_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").Equals<A, {}> extends true ? void : { readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }) => import("effect/Cause").YieldableError & {
+    readonly _tag: "TokenNotFoundError";
+} & Readonly<A>;
+export declare class TokenNotFoundError extends TokenNotFoundError_base<{
+    readonly message: string;
+    readonly tokenType: TokenType;
+}> {
+}
+declare const TokenExpiredError_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").Equals<A, {}> extends true ? void : { readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }) => import("effect/Cause").YieldableError & {
+    readonly _tag: "TokenExpiredError";
+} & Readonly<A>;
+export declare class TokenExpiredError extends TokenExpiredError_base<{
+    readonly message: string;
+    readonly tokenType: TokenType;
+}> {
+}
+declare const StorageKeyNotFoundError_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").Equals<A, {}> extends true ? void : { readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }) => import("effect/Cause").YieldableError & {
+    readonly _tag: "StorageKeyNotFoundError";
+} & Readonly<A>;
+export declare class StorageKeyNotFoundError extends StorageKeyNotFoundError_base<{
+    readonly message: string;
+    readonly key: string;
+    readonly storageType: 'local' | 'session';
+}> {
+}
+export declare enum TokenType {
+    CSRF = "csrf",
+    API = "api",
+    AUTH = "auth",
+    REFRESH = "refresh"
+}
+export interface Token {
+    type: TokenType;
+    value: string;
+    expiry?: Date;
+    scope?: string[];
+}
+export interface StateManagerService {
+    /**
+     * Extract CSRF token from HTML
+     */
+    extractCSRFToken: (html: string) => Effect.Effect<string, CSRFTokenNotFoundError>;
+    /**
+     * Extract API token from JavaScript
+     */
+    extractAPIToken: (scripts: string[]) => Effect.Effect<string, APITokenNotFoundError>;
+    /**
+     * Store a token
+     */
+    storeToken: (type: TokenType, token: string, expiry?: Date) => Effect.Effect<void>;
+    /**
+     * Get a stored token
+     */
+    getToken: (type: TokenType) => Effect.Effect<string, TokenNotFoundError | TokenExpiredError>;
+    /**
+     * Check if token is valid (not expired)
+     */
+    isTokenValid: (type: TokenType) => Effect.Effect<boolean>;
+    /**
+     * Simulate local storage
+     */
+    setLocalStorage: (key: string, value: string) => Effect.Effect<void>;
+    getLocalStorage: (key: string) => Effect.Effect<string, StorageKeyNotFoundError>;
+    clearLocalStorage: () => Effect.Effect<void>;
+    /**
+     * Simulate session storage
+     */
+    setSessionStorage: (key: string, value: string) => Effect.Effect<void>;
+    getSessionStorage: (key: string) => Effect.Effect<string, StorageKeyNotFoundError>;
+    clearSessionStorage: () => Effect.Effect<void>;
+    /**
+     * Clear all state
+     */
+    clearState: () => Effect.Effect<void>;
+}
+declare const StateManager_base: Context.TagClass<StateManager, "StateManager", StateManagerService>;
+export declare class StateManager extends StateManager_base {
+}
+/**
+ * Create a StateManager service implementation
+ */
+export declare const makeStateManager: () => Effect.Effect<StateManagerService>;
+/**
+ * StateManager Layer
+ */
+export declare const StateManagerLive: Layer.Layer<StateManager, never, never>;
+export {};
+//# sourceMappingURL=StateManager.service.d.ts.map

package/dist/lib/StateManager/StateManager.service.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"StateManager.service.d.ts","sourceRoot":"","sources":["../../../src/lib/StateManager/StateManager.service.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,OAAO,EAAkB,MAAM,EAAW,KAAK,EAAe,MAAM,QAAQ,CAAC;;;;AAItF,qBAAa,sBAAuB,SAAQ,4BAA2C;IACrF,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;CAC1B,CAAC;CAAG;;;;AAEL,qBAAa,qBAAsB,SAAQ,2BAA0C;IACnF,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;CAC1B,CAAC;CAAG;;;;AAEL,qBAAa,kBAAmB,SAAQ,wBAAuC;IAC7E,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,SAAS,EAAE,SAAS,CAAC;CAC/B,CAAC;CAAG;;;;AAEL,qBAAa,iBAAkB,SAAQ,uBAAsC;IAC3E,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,SAAS,EAAE,SAAS,CAAC;CAC/B,CAAC;CAAG;;;;AAEL,qBAAa,uBAAwB,SAAQ,6BAA4C;IACvF,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,WAAW,EAAE,OAAO,GAAG,SAAS,CAAC;CAC3C,CAAC;CAAG;AAEL,oBAAY,SAAS;IACnB,IAAI,SAAS;IACb,GAAG,QAAQ;IACX,IAAI,SAAS;IACb,OAAO,YAAY;CACpB;AAED,MAAM,WAAW,KAAK;IACpB,IAAI,EAAE,SAAS,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,IAAI,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;CAClB;AAED,MAAM,WAAW,mBAAmB;IAClC;;OAEG;IACH,gBAAgB,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,sBAAsB,CAAC,CAAC;IAElF;;OAEG;IACH,eAAe,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,qBAAqB,CAAC,CAAC;IAErF;;OAEG;IACH,UAAU,EAAE,CACV,IAAI,EAAE,SAAS,EACf,KAAK,EAAE,MAAM,EACb,MAAM,CAAC,EAAE,IAAI,KACV,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAEzB;;OAEG;IACH,QAAQ,EAAE,CAAC,IAAI,EAAE,SAAS,KAAK,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,kBAAkB,GAAG,iBAAiB,CAAC,CAAC;IAE7F;;OAEG;IACH,YAAY,EAAE,CAAC,IAAI,EAAE,SAAS,KAAK,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAE1D;;OAEG;IACH,eAAe,EAAE,CACf,GAAG,EAAE,MAAM,EACX,KAAK,EAAE,MAAM,KACV,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACzB,eAAe,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,uBAAuB,CAAC,CAAC;IACjF,iBAAiB,EAAE,MAAM,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAE7C;;OAEG;IACH,iBAAiB,EAAE,CACjB,GAAG,EAAE,MAAM,EACX,KAAK,EAAE,MAAM,KACV,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACzB,iBAAiB,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,uBAAuB,CAAC,CAAC;IACnF,mBAAmB,EAAE,MAAM,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAE/C;;OAEG;IACH,UAAU,EAAE,MAAM,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;CACvC;;AAED,qBAAa,YAAa,SAAQ,iBAG/B;CAAG;AAEN;;GAEG;AACH,eAAO,MAAM,gBAAgB,QAAO,MAAM,CAAC,MAAM,CAAC,mBAAmB,CAqNjE,CAAC;AAEL;;GAEG;AACH,eAAO,MAAM,gBAAgB,yCAAiD,CAAC"}

package/dist/lib/StateManager/index.d.ts

@@ -0,0 +1,5 @@
+/**
+ * State Manager module exports
+ */
+export * from './StateManager.service.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/lib/StateManager/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/lib/StateManager/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,cAAc,2BAA2B,CAAC"}

package/dist/lib/UrlDeduplicator/UrlDeduplicator.service.d.ts

@@ -0,0 +1,58 @@
+import { Effect, MutableHashSet } from 'effect';
+/**
+ * Thread-safe URL deduplication service with built-in normalization.
+ *
+ * Provides atomic operations for checking and adding URLs to prevent
+ * race conditions in concurrent environments. URLs are normalized
+ * before storage to ensure consistent deduplication.
+ *
+ * @group Services
+ * @public
+ */
+export interface IUrlDeduplicator {
+    /**
+     * Attempts to add a URL to the deduplication set.
+     *
+     * @param url - The URL to add
+     * @returns Effect containing boolean - true if URL was added (first time seen), false if already exists
+     */
+    tryAdd(_url: string): Effect.Effect<boolean>;
+    /**
+     * Checks if a URL has already been seen.
+     *
+     * @param url - The URL to check
+     * @returns Effect containing boolean - true if URL exists, false otherwise
+     */
+    contains(_url: string): Effect.Effect<boolean>;
+    /**
+     * Returns the current number of unique URLs in the set.
+     *
+     * @returns Effect containing the count
+     */
+    size(): Effect.Effect<number>;
+    /**
+     * Clears all URLs from the deduplication set.
+     *
+     * @returns Effect containing void
+     */
+    clear(): Effect.Effect<void>;
+}
+declare const UrlDeduplicatorService_base: Effect.Service.Class<UrlDeduplicatorService, "@jambudipa.io/UrlDeduplicatorService", {
+    readonly effect: Effect.Effect<{
+        tryAdd: (url: string) => Effect.Effect<boolean, never, never>;
+        contains: (url: string) => Effect.Effect<boolean, never, never>;
+        size: () => Effect.Effect<number, never, never>;
+        clear: () => Effect.Effect<MutableHashSet.MutableHashSet<string>, never, never>;
+    }, never, import("../Config/SpiderConfig.service.js").SpiderConfigService>;
+    readonly dependencies: readonly [import("effect/Layer").Layer<import("../Config/SpiderConfig.service.js").SpiderConfigService, never, never>];
+}>;
+/**
+ * URL deduplication service as an Effect Service.
+ *
+ * @group Services
+ * @public
+ */
+export declare class UrlDeduplicatorService extends UrlDeduplicatorService_base {
+}
+export {};
+//# sourceMappingURL=UrlDeduplicator.service.d.ts.map

package/dist/lib/UrlDeduplicator/UrlDeduplicator.service.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"UrlDeduplicator.service.d.ts","sourceRoot":"","sources":["../../../src/lib/UrlDeduplicator/UrlDeduplicator.service.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,QAAQ,CAAC;AAGhD;;;;;;;;;GASG;AACH,MAAM,WAAW,gBAAgB;IAC/B;;;;;OAKG;IACH,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAE7C;;;;;OAKG;IACH,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAE/C;;;;OAIG;IACH,IAAI,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAE9B;;;;OAIG;IACH,KAAK,IAAI,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;CAC9B;;;sBA4EqB,MAAM;wBAcJ,MAAM;;;;;;AAxF9B;;;;;GAKG;AACH,qBAAa,sBAAuB,SAAQ,2BAuG3C;CAAG"}