@jambudipa/spider 0.1.0

package/dist/lib/api-facades.d.ts

@@ -0,0 +1,313 @@
+/**
+ * Clean API facades that hide Effect.Service implementation details.
+ *
+ * These interfaces provide clean documentation without exposing
+ * internal Effect.js service machinery.
+ *
+ * @group Services
+ */
+import { Effect, Sink } from 'effect';
+import { CrawlResult, CrawlTask } from './Spider/Spider.service.js';
+import { PriorityRequest, SpiderState, SpiderStateKey, StatePersistence } from './Scheduler/SpiderScheduler.service.js';
+import { SpiderMiddleware, SpiderRequest, SpiderResponse } from './Middleware/SpiderMiddleware.js';
+import { MiddlewareError } from './errors.js';
+/**
+ * The main Spider service interface for web crawling.
+ *
+ * Orchestrates the entire crawling process including URL validation,
+ * robots.txt checking, concurrent processing, and result streaming.
+ *
+ * @example
+ * ```typescript
+ * const program = Effect.gen(function* () {
+ *   const spider = yield* Spider;
+ *   const collectSink = Sink.forEach<CrawlResult>(result =>
+ *     Effect.sync(() => console.log(result.pageData.url))
+ *   );
+ *
+ *   const stats = yield* spider.crawl('https://example.com', collectSink);
+ *   console.log(`Crawled ${stats.totalPages} pages`);
+ * });
+ * ```
+ *
+ * @group Services
+ * @public
+ */
+export interface ISpider {
+    /**
+     * Starts crawling from the specified URL and processes results through the provided sink.
+     *
+     * @param urlString - The starting URL for crawling
+     * @param sink - Sink to process crawl results as they're produced
+     * @returns Effect containing crawl statistics (total pages, completion status)
+     */
+    crawl<A, E, R>(urlString: string, sink: Sink.Sink<A, CrawlResult, E, R>): Effect.Effect<{
+        totalPages: number;
+        completed: boolean;
+    }, Error>;
+    /**
+     * Returns the list of URLs that have been visited during crawling.
+     *
+     * @returns Effect containing array of visited URLs
+     */
+    getVisitedUrls(): Effect.Effect<string[]>;
+}
+/**
+ * The SpiderSchedulerService service interface for request scheduling and persistence.
+ *
+ * Manages request queuing, prioritization, and state persistence for
+ * resumable crawling operations.
+ *
+ * @example
+ * ```typescript
+ * const program = Effect.gen(function* () {
+ *   const scheduler = yield* SpiderSchedulerService;
+ *
+ *   // Configure persistence
+ *   const stateKey = new SpiderStateKey({
+ *     id: 'my-crawl',
+ *     timestamp: new Date(),
+ *     name: 'Example Crawl'
+ *   });
+ *
+ *   yield* scheduler.configurePersistence(persistence, stateKey);
+ *
+ *   // Queue requests with priority
+ *   yield* scheduler.enqueue({ url: 'https://example.com', depth: 0 }, 10);
+ *
+ *   // Process requests
+ *   const request = yield* scheduler.dequeue();
+ *   console.log(`Processing: ${request.request.url}`);
+ * });
+ * ```
+ *
+ * @group Services
+ * @public
+ */
+export interface ISpiderScheduler {
+    /**
+     * Configures the scheduler to use a specific persistence layer with a state key.
+     *
+     * @param persistence - Implementation of StatePersistence interface
+     * @param stateKey - Unique identifier for the crawl session
+     */
+    configurePersistence(persistence: StatePersistence, stateKey: SpiderStateKey): Effect.Effect<void>;
+    /**
+     * Removes persistence configuration, disabling state saving.
+     */
+    clearPersistence(): Effect.Effect<void>;
+    /**
+     * Adds a crawl task to the processing queue with optional priority.
+     *
+     * @param request - Crawl task containing URL and depth
+     * @param priority - Optional priority (higher numbers = higher priority, default: 0)
+     * @returns Effect containing boolean indicating if task was added (false if duplicate)
+     */
+    enqueue(request: CrawlTask, priority?: number): Effect.Effect<boolean>;
+    /**
+     * Retrieves the next highest-priority task from the queue.
+     *
+     * @returns Effect containing the next priority request
+     */
+    dequeue(): Effect.Effect<PriorityRequest>;
+    /**
+     * Returns the current number of tasks in the queue.
+     */
+    size(): Effect.Effect<number>;
+    /**
+     * Checks if the queue is empty.
+     */
+    isEmpty(): Effect.Effect<boolean>;
+    /**
+     * Returns the current scheduler state for persistence.
+     */
+    getState(): Effect.Effect<SpiderState>;
+    /**
+     * Restores the scheduler from a previously saved state.
+     *
+     * @param state - Complete state to restore from
+     */
+    restoreFromState(state: SpiderState): Effect.Effect<void>;
+    /**
+     * Attempts to restore state from a persistence layer.
+     *
+     * @param persistence - Persistence layer to load from
+     * @param stateKey - State key to restore
+     * @returns Effect containing boolean indicating if state was successfully restored
+     */
+    restore(persistence: StatePersistence, stateKey: SpiderStateKey): Effect.Effect<boolean>;
+}
+/**
+ * The MiddlewareManager service interface for pipeline processing.
+ *
+ * Orchestrates the execution of middleware in the correct order for
+ * request processing, response handling, and error recovery.
+ *
+ * @example
+ * ```typescript
+ * const program = Effect.gen(function* () {
+ *   const manager = yield* MiddlewareManager;
+ *
+ *   const middleware = [
+ *     rateLimitMiddleware,
+ *     loggingMiddleware,
+ *     userAgentMiddleware
+ *   ];
+ *
+ *   const processedRequest = yield* manager.processRequest(request, middleware);
+ *   console.log('Request processed through middleware pipeline');
+ * });
+ * ```
+ *
+ * @group Services
+ * @public
+ */
+export interface IMiddlewareManager {
+    /**
+     * Processes a request through the middleware pipeline.
+     *
+     * @param request - The initial request to process
+     * @param middlewares - Array of middleware to apply
+     * @returns Effect containing the processed request
+     */
+    processRequest(request: SpiderRequest, middlewares: SpiderMiddleware[]): Effect.Effect<SpiderRequest, MiddlewareError>;
+    /**
+     * Processes a response through the middleware pipeline in reverse order.
+     *
+     * @param response - The response to process
+     * @param request - The original request (for context)
+     * @param middlewares - Array of middleware to apply
+     * @returns Effect containing the processed response
+     */
+    processResponse(response: SpiderResponse, request: SpiderRequest, middlewares: SpiderMiddleware[]): Effect.Effect<SpiderResponse, MiddlewareError>;
+    /**
+     * Processes an exception through the middleware pipeline in reverse order.
+     *
+     * @param error - The error that occurred
+     * @param request - The request that caused the error
+     * @param middlewares - Array of middleware to apply
+     * @returns Effect containing a recovered response or null
+     */
+    processException(error: Error, request: SpiderRequest, middlewares: SpiderMiddleware[]): Effect.Effect<SpiderResponse | null, MiddlewareError>;
+}
+/**
+ * Rate limiting middleware service interface.
+ *
+ * Provides rate limiting functionality for respectful crawling,
+ * controlling request frequency at both global and per-domain levels.
+ *
+ * @group Middleware
+ * @public
+ */
+export interface IRateLimitMiddleware {
+    /**
+     * Creates a rate limiting middleware with the specified configuration.
+     *
+     * @param config - Rate limiting configuration options
+     * @returns Configured middleware instance
+     *
+     * @example
+     * ```typescript
+     * const rateLimiter = yield* RateLimitMiddleware;
+     * const middleware = rateLimiter.create({
+     *   maxConcurrentRequests: 5,
+     *   maxRequestsPerSecondPerDomain: 2,
+     *   requestDelayMs: 250
+     * });
+     * ```
+     */
+    create(config: {
+        maxConcurrentRequests: number;
+        maxRequestsPerSecondPerDomain: number;
+        requestDelayMs?: number;
+    }): SpiderMiddleware;
+}
+/**
+ * Logging middleware service interface.
+ *
+ * Provides logging functionality using Effect.Logger for debugging
+ * and monitoring crawling operations.
+ *
+ * @group Middleware
+ * @public
+ */
+export interface ILoggingMiddleware {
+    /**
+     * Creates a logging middleware with optional configuration.
+     *
+     * @param config - Optional logging configuration
+     * @returns Configured middleware instance
+     *
+     * @example
+     * ```typescript
+     * const logger = yield* LoggingMiddleware;
+     * const middleware = logger.create({
+     *   logRequests: true,
+     *   logResponses: true,
+     *   logLevel: 'info'
+     * });
+     * ```
+     */
+    create(config?: {
+        logRequests?: boolean;
+        logResponses?: boolean;
+        logErrors?: boolean;
+        logLevel?: 'debug' | 'info' | 'warn' | 'error';
+    }): SpiderMiddleware;
+}
+/**
+ * User agent middleware service interface.
+ *
+ * Adds consistent User-Agent headers to all requests for
+ * proper identification of your crawler.
+ *
+ * @group Middleware
+ * @public
+ */
+export interface IUserAgentMiddleware {
+    /**
+     * Creates a User-Agent middleware with the specified user agent string.
+     *
+     * @param userAgent - User agent string to add to requests
+     * @returns Configured middleware instance
+     *
+     * @example
+     * ```typescript
+     * const userAgent = yield* UserAgentMiddleware;
+     * const middleware = userAgent.create('MyBot/1.0 (+https://example.com)');
+     * ```
+     */
+    create(userAgent: string): SpiderMiddleware;
+}
+/**
+ * Statistics middleware service interface.
+ *
+ * Collects comprehensive metrics about crawling activity including
+ * request counts, response codes, and performance statistics.
+ *
+ * @group Middleware
+ * @public
+ */
+export interface IStatsMiddleware {
+    /**
+     * Creates a statistics middleware and returns both the middleware and a stats getter.
+     *
+     * @returns Object containing the middleware instance and statistics retrieval function
+     *
+     * @example
+     * ```typescript
+     * const statsService = yield* StatsMiddleware;
+     * const { middleware, getStats } = statsService.create();
+     *
+     * // Use middleware in your pipeline
+     * // Later get statistics
+     * const stats = yield* getStats();
+     * console.log(`Processed ${stats.requests_processed} requests`);
+     * ```
+     */
+    create(): {
+        middleware: SpiderMiddleware;
+        getStats: () => Effect.Effect<Record<string, number>>;
+    };
+}
+//# sourceMappingURL=api-facades.d.ts.map

package/dist/lib/api-facades.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"api-facades.d.ts","sourceRoot":"","sources":["../../src/lib/api-facades.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,QAAQ,CAAC;AACtC,OAAO,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,4BAA4B,CAAC;AACpE,OAAO,EACL,eAAe,EACf,WAAW,EACX,cAAc,EACd,gBAAgB,EACjB,MAAM,wCAAwC,CAAC;AAChD,OAAO,EACL,gBAAgB,EAChB,aAAa,EACb,cAAc,EACf,MAAM,kCAAkC,CAAC;AAC1C,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAE9C;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,MAAM,WAAW,OAAO;IACtB;;;;;;OAMG;IACH,KAAK,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EACX,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,CAAC,CAAC,GACpC,MAAM,CAAC,MAAM,CAAC;QAAE,UAAU,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,OAAO,CAAA;KAAE,EAAE,KAAK,CAAC,CAAC;IAEpE;;;;OAIG;IACH,cAAc,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;CAC3C;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AACH,MAAM,WAAW,gBAAgB;IAC/B;;;;;OAKG;IACH,oBAAoB,CAClB,WAAW,EAAE,gBAAgB,EAC7B,QAAQ,EAAE,cAAc,GACvB,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAEvB;;OAEG;IACH,gBAAgB,IAAI,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAExC;;;;;;OAMG;IACH,OAAO,CAAC,OAAO,EAAE,SAAS,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAEvE;;;;OAIG;IACH,OAAO,IAAI,MAAM,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;IAE1C;;OAEG;IACH,IAAI,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAE9B;;OAEG;IACH,OAAO,IAAI,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAElC;;OAEG;IACH,QAAQ,IAAI,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;IAEvC;;;;OAIG;IACH,gBAAgB,CAAC,KAAK,EAAE,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAE1D;;;;;;OAMG;IACH,OAAO,CACL,WAAW,EAAE,gBAAgB,EAC7B,QAAQ,EAAE,cAAc,GACvB,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;CAC3B;AAED;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,MAAM,WAAW,kBAAkB;IACjC;;;;;;OAMG;IACH,cAAc,CACZ,OAAO,EAAE,aAAa,EACtB,WAAW,EAAE,gBAAgB,EAAE,GAC9B,MAAM,CAAC,MAAM,CAAC,aAAa,EAAE,eAAe,CAAC,CAAC;IAEjD;;;;;;;OAOG;IACH,eAAe,CACb,QAAQ,EAAE,cAAc,EACxB,OAAO,EAAE,aAAa,EACtB,WAAW,EAAE,gBAAgB,EAAE,GAC9B,MAAM,CAAC,MAAM,CAAC,cAAc,EAAE,eAAe,CAAC,CAAC;IAElD;;;;;;;OAOG;IACH,gBAAgB,CACd,KAAK,EAAE,KAAK,EACZ,OAAO,EAAE,aAAa,EACtB,WAAW,EAAE,gBAAgB,EAAE,GAC9B,MAAM,CAAC,MAAM,CAAC,cAAc,GAAG,IAAI,EAAE,eAAe,CAAC,CAAC;CAC1D;AAED;;;;;;;;GAQG;AACH,MAAM,WAAW,oBAAoB;IACnC;;;;;;;;;;;;;;;OAeG;IACH,MAAM,CAAC,MAAM,EAAE;QACb,qBAAqB,EAAE,MAAM,CAAC;QAC9B,6BAA6B,EAAE,MAAM,CAAC;QACtC,cAAc,CAAC,EAAE,MAAM,CAAC;KACzB,GAAG,gBAAgB,CAAC;CACtB;AAED;;;;;;;;GAQG;AACH,MAAM,WAAW,kBAAkB;IACjC;;;;;;;;;;;;;;;OAeG;IACH,MAAM,CAAC,MAAM,CAAC,EAAE;QACd,WAAW,CAAC,EAAE,OAAO,CAAC;QACtB,YAAY,CAAC,EAAE,OAAO,CAAC;QACvB,SAAS,CAAC,EAAE,OAAO,CAAC;QACpB,QAAQ,CAAC,EAAE,OAAO,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC;KAChD,GAAG,gBAAgB,CAAC;CACtB;AAED;;;;;;;;GAQG;AACH,MAAM,WAAW,oBAAoB;IACnC;;;;;;;;;;;OAWG;IACH,MAAM,CAAC,SAAS,EAAE,MAAM,GAAG,gBAAgB,CAAC;CAC7C;AAED;;;;;;;;GAQG;AACH,MAAM,WAAW,gBAAgB;IAC/B;;;;;;;;;;;;;;;OAeG;IACH,MAAM,IAAI;QACR,UAAU,EAAE,gBAAgB,CAAC;QAC7B,QAAQ,EAAE,MAAM,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;KACvD,CAAC;CACH"}

package/dist/lib/errors.d.ts

@@ -0,0 +1,99 @@
+declare const NetworkError_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").Equals<A, {}> extends true ? void : { readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }) => import("effect/Cause").YieldableError & {
+    readonly _tag: "NetworkError";
+} & Readonly<A>;
+/**
+ * Network-related errors (fetch failures, timeouts, etc.)
+ */
+export declare class NetworkError extends NetworkError_base<{
+    readonly url: string;
+    readonly cause?: unknown;
+    readonly message: string;
+}> {
+    static fromCause(url: string, cause: unknown): NetworkError;
+}
+declare const ResponseError_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").Equals<A, {}> extends true ? void : { readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }) => import("effect/Cause").YieldableError & {
+    readonly _tag: "ResponseError";
+} & Readonly<A>;
+/**
+ * Response processing errors (invalid content, parsing failures)
+ */
+export declare class ResponseError extends ResponseError_base<{
+    readonly url: string;
+    readonly cause?: unknown;
+    readonly message: string;
+}> {
+    static fromCause(url: string, cause: unknown): ResponseError;
+}
+declare const RobotsTxtError_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").Equals<A, {}> extends true ? void : { readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }) => import("effect/Cause").YieldableError & {
+    readonly _tag: "RobotsTxtError";
+} & Readonly<A>;
+/**
+ * Robots.txt fetching errors
+ */
+export declare class RobotsTxtError extends RobotsTxtError_base<{
+    readonly url: string;
+    readonly cause?: unknown;
+    readonly message: string;
+}> {
+    static fromCause(url: string, cause: unknown): RobotsTxtError;
+}
+declare const ConfigurationError_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").Equals<A, {}> extends true ? void : { readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }) => import("effect/Cause").YieldableError & {
+    readonly _tag: "ConfigurationError";
+} & Readonly<A>;
+/**
+ * Configuration errors
+ */
+export declare class ConfigurationError extends ConfigurationError_base<{
+    readonly message: string;
+    readonly details?: unknown;
+}> {
+}
+declare const MiddlewareError_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").Equals<A, {}> extends true ? void : { readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }) => import("effect/Cause").YieldableError & {
+    readonly _tag: "MiddlewareError";
+} & Readonly<A>;
+/**
+ * Middleware processing errors
+ */
+export declare class MiddlewareError extends MiddlewareError_base<{
+    readonly phase: 'transform' | 'error';
+    readonly middlewareName: string;
+    readonly cause?: unknown;
+    readonly message: string;
+}> {
+    static transform(middlewareName: string, cause: unknown): MiddlewareError;
+    static error(middlewareName: string, cause: unknown): MiddlewareError;
+}
+declare const FileSystemError_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").Equals<A, {}> extends true ? void : { readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }) => import("effect/Cause").YieldableError & {
+    readonly _tag: "FileSystemError";
+} & Readonly<A>;
+/**
+ * File system errors
+ */
+export declare class FileSystemError extends FileSystemError_base<{
+    readonly operation: 'read' | 'write' | 'create' | 'delete';
+    readonly path: string;
+    readonly cause?: unknown;
+    readonly message: string;
+}> {
+    static write(path: string, cause: unknown): FileSystemError;
+    static create(path: string, cause: unknown): FileSystemError;
+}
+declare const PersistenceError_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").Equals<A, {}> extends true ? void : { readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }) => import("effect/Cause").YieldableError & {
+    readonly _tag: "PersistenceError";
+} & Readonly<A>;
+/**
+ * Persistence layer errors
+ */
+export declare class PersistenceError extends PersistenceError_base<{
+    readonly operation: 'save' | 'load' | 'delete';
+    readonly key?: string;
+    readonly cause?: unknown;
+    readonly message: string;
+}> {
+    static save(cause: unknown, key?: string): PersistenceError;
+    static load(cause: unknown, key?: string): PersistenceError;
+    static delete(cause: unknown, key?: string): PersistenceError;
+}
+export type SpiderError = NetworkError | ResponseError | RobotsTxtError | ConfigurationError | MiddlewareError | FileSystemError | PersistenceError;
+export {};
+//# sourceMappingURL=errors.d.ts.map

package/dist/lib/errors.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"errors.d.ts","sourceRoot":"","sources":["../../src/lib/errors.ts"],"names":[],"mappings":";;;AAEA;;GAEG;AACH,qBAAa,YAAa,SAAQ,kBAAiC;IACjE,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,KAAK,CAAC,EAAE,OAAO,CAAC;IACzB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;CAC1B,CAAC;IACA,MAAM,CAAC,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,GAAG,YAAY;CAO5D;;;;AAED;;GAEG;AACH,qBAAa,aAAc,SAAQ,mBAAkC;IACnE,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,KAAK,CAAC,EAAE,OAAO,CAAC;IACzB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;CAC1B,CAAC;IACA,MAAM,CAAC,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,GAAG,aAAa;CAO7D;;;;AAED;;GAEG;AACH,qBAAa,cAAe,SAAQ,oBAAmC;IACrE,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,KAAK,CAAC,EAAE,OAAO,CAAC;IACzB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;CAC1B,CAAC;IACA,MAAM,CAAC,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,GAAG,cAAc;CAO9D;;;;AAED;;GAEG;AACH,qBAAa,kBAAmB,SAAQ,wBAAuC;IAC7E,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,OAAO,CAAC,EAAE,OAAO,CAAC;CAC5B,CAAC;CAAG;;;;AAEL;;GAEG;AACH,qBAAa,eAAgB,SAAQ,qBAAoC;IACvE,QAAQ,CAAC,KAAK,EAAE,WAAW,GAAG,OAAO,CAAC;IACtC,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,KAAK,CAAC,EAAE,OAAO,CAAC;IACzB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;CAC1B,CAAC;IACA,MAAM,CAAC,SAAS,CAAC,cAAc,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,GAAG,eAAe;IASzE,MAAM,CAAC,KAAK,CAAC,cAAc,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,GAAG,eAAe;CAQtE;;;;AAED;;GAEG;AACH,qBAAa,eAAgB,SAAQ,qBAAoC;IACvE,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,GAAG,QAAQ,GAAG,QAAQ,CAAC;IAC3D,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,KAAK,CAAC,EAAE,OAAO,CAAC;IACzB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;CAC1B,CAAC;IACA,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,GAAG,eAAe;IAS3D,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,GAAG,eAAe;CAQ7D;;;;AAED;;GAEG;AACH,qBAAa,gBAAiB,SAAQ,sBAAqC;IACzE,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,GAAG,QAAQ,CAAC;IAC/C,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,KAAK,CAAC,EAAE,OAAO,CAAC;IACzB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;CAC1B,CAAC;IACA,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,OAAO,EAAE,GAAG,CAAC,EAAE,MAAM,GAAG,gBAAgB;IAW3D,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,OAAO,EAAE,GAAG,CAAC,EAAE,MAAM,GAAG,gBAAgB;IAW3D,MAAM,CAAC,MAAM,CAAC,KAAK,EAAE,OAAO,EAAE,GAAG,CAAC,EAAE,MAAM,GAAG,gBAAgB;CAU9D;AAGD,MAAM,MAAM,WAAW,GACnB,YAAY,GACZ,aAAa,GACb,cAAc,GACd,kBAAkB,GAClB,eAAe,GACf,eAAe,GACf,gBAAgB,CAAC"}

package/package.json

@@ -0,0 +1,108 @@
+{
+  "name": "@jambudipa/spider",
+  "version": "0.1.0",
+  "description": "A comprehensive web scraping library with resumable operations, middleware support, and built-in rate limiting",
+  "type": "module",
+  "main": "./dist/index.js",
+  "module": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js",
+      "default": "./dist/index.js"
+    },
+    "./package.json": "./package.json"
+  },
+  "files": [
+    "dist",
+    "README.md",
+    "LICENSE"
+  ],
+  "engines": {
+    "node": ">=18.0.0"
+  },
+  "scripts": {
+    "build": "vite build && tsc --emitDeclarationOnly --outDir dist",
+    "dev": "vite build --watch",
+    "test": "vitest",
+    "test:ui": "vitest --ui",
+    "test:coverage": "vitest --coverage",
+    "test:run": "vitest run",
+    "lint": "eslint src",
+    "lint:fix": "eslint src --fix",
+    "format": "prettier --write \"src/**/*.ts\"",
+    "format:check": "prettier --check \"src/**/*.ts\"",
+    "typecheck": "tsc --noEmit",
+    "typecheck:test": "tsc --noEmit --skipLibCheck --downlevelIteration --project tsconfig.test.json",
+    "typecheck:examples": "tsc --noEmit --project tsconfig.examples.json",
+    "clean": "rm -rf dist",
+    "prepublishOnly": "npm run clean && npm run build && npm run typecheck",
+    "prepack": "npm run build",
+    "changeset": "changeset",
+    "version": "changeset version",
+    "release": "npm run build && changeset publish",
+    "docs:api": "typedoc",
+    "docs:build": "npm run docs:api && npm run examples:validate",
+    "examples:run": "tsx",
+    "examples:validate": "tsx scripts/analyse-exports.ts",
+    "examples:run-all": "tsx examples/run-all.ts",
+    "test:examples": "vitest run tests/examples.test.ts",
+    "test:core": "vitest run tests/core",
+    "test:all": "npm run test:examples && npm run test:core"
+  },
+  "keywords": [
+    "web-scraping",
+    "crawler",
+    "spider",
+    "http",
+    "middleware",
+    "resumable",
+    "rate-limiting",
+    "typescript"
+  ],
+  "author": "Jambudipa.io",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/jambudipa/spider.git"
+  },
+  "bugs": {
+    "url": "https://github.com/jambudipa/spider/issues"
+  },
+  "homepage": "https://github.com/jambudipa/spider#readme",
+  "peerDependencies": {
+    "@types/tough-cookie": "^4.0.5",
+    "cheerio": "^1.1.2",
+    "domhandler": "^5.0.3",
+    "effect": "^3.16.16",
+    "tough-cookie": "^6.0.0",
+    "tslib": "^2.3.0"
+  },
+  "devDependencies": {
+    "@changesets/cli": "^2.29.5",
+    "@types/node": "^20.0.0",
+    "@typescript-eslint/eslint-plugin": "^7.0.0",
+    "@typescript-eslint/parser": "^7.0.0",
+    "@typescript-eslint/typescript-estree": "^8.38.0",
+    "@vitest/coverage-v8": "^1.6.0",
+    "@vitest/ui": "^1.6.0",
+    "@types/tough-cookie": "^4.0.5",
+    "cheerio": "^1.1.2",
+    "domhandler": "^5.0.3",
+    "effect": "^3.16.16",
+    "eslint": "^8.57.0",
+    "prettier": "^3.6.2",
+    "tough-cookie": "^6.0.0",
+    "tsx": "^4.20.3",
+    "typedoc": "^0.28.9",
+    "typedoc-plugin-markdown": "^4.8.0",
+    "tslib": "^2.3.0",
+    "typescript": "^5.4.0",
+    "vite": "^5.0.0",
+    "vitest": "^1.6.0"
+  },
+  "publishConfig": {
+    "access": "public"
+  }
+}