@jambudipa/spider 0.1.0

package/dist/lib/Resumability/types.d.ts

@@ -0,0 +1,201 @@
+import { Effect, Schema } from 'effect';
+import { PriorityRequest, SpiderState, SpiderStateKey } from '../Scheduler/SpiderScheduler.service.js';
+export { SpiderStateKey, PriorityRequest, SpiderState };
+declare const StateDelta_base: Schema.Class<StateDelta, {
+    /** Session this delta applies to */
+    stateKey: typeof Schema.String;
+    /** Sequence number for ordering deltas */
+    sequence: typeof Schema.Number;
+    /** When this delta was created */
+    timestamp: typeof Schema.Date;
+    /** The operation that created this delta */
+    operation: Schema.Union<[Schema.Struct<{
+        type: Schema.Literal<["enqueue"]>;
+        request: typeof PriorityRequest;
+    }>, Schema.Struct<{
+        type: Schema.Literal<["dequeue"]>;
+        fingerprint: typeof Schema.String;
+    }>, Schema.Struct<{
+        type: Schema.Literal<["mark_visited"]>;
+        fingerprint: typeof Schema.String;
+    }>]>;
+}, Schema.Struct.Encoded<{
+    /** Session this delta applies to */
+    stateKey: typeof Schema.String;
+    /** Sequence number for ordering deltas */
+    sequence: typeof Schema.Number;
+    /** When this delta was created */
+    timestamp: typeof Schema.Date;
+    /** The operation that created this delta */
+    operation: Schema.Union<[Schema.Struct<{
+        type: Schema.Literal<["enqueue"]>;
+        request: typeof PriorityRequest;
+    }>, Schema.Struct<{
+        type: Schema.Literal<["dequeue"]>;
+        fingerprint: typeof Schema.String;
+    }>, Schema.Struct<{
+        type: Schema.Literal<["mark_visited"]>;
+        fingerprint: typeof Schema.String;
+    }>]>;
+}>, never, {
+    readonly operation: {
+        readonly type: "enqueue";
+        readonly request: PriorityRequest;
+    } | {
+        readonly type: "dequeue";
+        readonly fingerprint: string;
+    } | {
+        readonly type: "mark_visited";
+        readonly fingerprint: string;
+    };
+} & {
+    readonly timestamp: Date;
+} & {
+    readonly stateKey: string;
+} & {
+    readonly sequence: number;
+}, {}, {}>;
+/**
+ * Delta operation that represents a single state change.
+ *
+ * Used for incremental persistence instead of saving the entire state
+ * on every operation, which is much more efficient for large crawls.
+ *
+ * @group Delta Updates
+ * @public
+ */
+export declare class StateDelta extends StateDelta_base {
+}
+/**
+ * Represents a state change operation with both the delta and resulting state.
+ *
+ * This allows persistence strategies to choose whether to save deltas,
+ * full state, or both depending on their optimization needs.
+ *
+ * @group Operations
+ * @public
+ */
+export interface StateOperation {
+    /** The incremental change */
+    readonly delta: StateDelta;
+    /** The complete state after applying this operation */
+    readonly resultingState: SpiderState;
+    /** Whether this operation should trigger a snapshot */
+    readonly shouldSnapshot: boolean;
+}
+declare const PersistenceError_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").Equals<A, {}> extends true ? void : { readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }) => import("effect/Cause").YieldableError & {
+    readonly _tag: "PersistenceError";
+} & Readonly<A>;
+/**
+ * Error that can occur during persistence operations.
+ *
+ * @group Errors
+ * @public
+ */
+export declare class PersistenceError extends PersistenceError_base<{
+    readonly message: string;
+    readonly cause?: unknown;
+    readonly operation?: string;
+}> {
+}
+/**
+ * Storage backend capabilities that determine optimal persistence strategy.
+ *
+ * Backends advertise their capabilities so the ResumabilityService can
+ * choose the best strategy automatically.
+ *
+ * @group Storage
+ * @public
+ */
+export interface StorageCapabilities {
+    /** Can efficiently store and retrieve delta operations */
+    readonly supportsDelta: boolean;
+    /** Can efficiently store full state snapshots */
+    readonly supportsSnapshot: boolean;
+    /** Can handle streaming/batch operations */
+    readonly supportsStreaming: boolean;
+    /** Can handle concurrent access safely */
+    readonly supportsConcurrency: boolean;
+    /** Estimated latency category */
+    readonly latency: 'low' | 'medium' | 'high';
+}
+/**
+ * Generic storage backend interface that persistence strategies use.
+ *
+ * Backends implement the storage operations they support best.
+ * Not all methods need to be implemented - strategies will adapt.
+ *
+ * @group Storage
+ * @public
+ */
+export interface StorageBackend {
+    /** Backend capabilities for strategy selection */
+    readonly capabilities: StorageCapabilities;
+    /** Storage backend identifier */
+    readonly name: string;
+    /** Initialize the backend (create tables, connections, etc.) */
+    initialize(): Effect.Effect<void, PersistenceError, never>;
+    /** Cleanup backend resources */
+    cleanup(): Effect.Effect<void, PersistenceError, never>;
+    saveState?(key: SpiderStateKey, state: SpiderState): Effect.Effect<void, PersistenceError, never>;
+    loadState?(key: SpiderStateKey): Effect.Effect<SpiderState | null, PersistenceError, never>;
+    deleteState?(key: SpiderStateKey): Effect.Effect<void, PersistenceError, never>;
+    saveDelta?(delta: StateDelta): Effect.Effect<void, PersistenceError, never>;
+    saveDeltas?(deltas: StateDelta[]): Effect.Effect<void, PersistenceError, never>;
+    loadDeltas?(key: SpiderStateKey, fromSequence?: number): Effect.Effect<StateDelta[], PersistenceError, never>;
+    saveSnapshot?(key: SpiderStateKey, state: SpiderState, sequence: number): Effect.Effect<void, PersistenceError, never>;
+    loadLatestSnapshot?(key: SpiderStateKey): Effect.Effect<{
+        state: SpiderState;
+        sequence: number;
+    } | null, PersistenceError, never>;
+    compactDeltas?(key: SpiderStateKey, beforeSequence: number): Effect.Effect<void, PersistenceError, never>;
+    listSessions?(): Effect.Effect<SpiderStateKey[], PersistenceError, never>;
+}
+/**
+ * Core strategy interface for different persistence approaches.
+ *
+ * Strategies implement the logic for when and how to persist state,
+ * using the storage backend for actual I/O operations.
+ *
+ * @group Strategies
+ * @public
+ */
+export interface PersistenceStrategy {
+    /** Persist a state operation */
+    persist(operation: StateOperation): Effect.Effect<void, PersistenceError, never>;
+    /** Restore state from storage */
+    restore(key: SpiderStateKey): Effect.Effect<SpiderState | null, PersistenceError, never>;
+    /** Clean up old data */
+    cleanup(key: SpiderStateKey): Effect.Effect<void, PersistenceError, never>;
+    /** Get strategy information */
+    getInfo(): {
+        readonly name: string;
+        readonly description: string;
+        readonly capabilities: string[];
+    };
+}
+/**
+ * Configuration for hybrid persistence strategy.
+ *
+ * Controls when to save snapshots vs deltas for optimal performance.
+ *
+ * @group Configuration
+ * @public
+ */
+export interface HybridPersistenceConfig {
+    /** Save a full snapshot every N operations */
+    readonly snapshotInterval: number;
+    /** Maximum deltas to accumulate before forcing a snapshot */
+    readonly maxDeltasBeforeSnapshot: number;
+    /** Whether to compact old deltas after snapshots */
+    readonly compactionEnabled: boolean;
+    /** Batch multiple deltas together for efficiency */
+    readonly batchDeltas: boolean;
+    /** Batch size for delta operations */
+    readonly deltaBatchSize: number;
+}
+/**
+ * Default hybrid persistence configuration.
+ */
+export declare const DEFAULT_HYBRID_CONFIG: HybridPersistenceConfig;
+//# sourceMappingURL=types.d.ts.map

package/dist/lib/Resumability/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/lib/Resumability/types.ts"],"names":[],"mappings":"AAAA,OAAO,EAAQ,MAAM,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAC9C,OAAO,EACL,eAAe,EACf,WAAW,EACX,cAAc,EACf,MAAM,yCAAyC,CAAC;AAGjD,OAAO,EAAE,cAAc,EAAE,eAAe,EAAE,WAAW,EAAE,CAAC;;IAYtD,oCAAoC;;IAEpC,0CAA0C;;IAE1C,kCAAkC;;IAElC,4CAA4C;;;;;;;;;;;;IAN5C,oCAAoC;;IAEpC,0CAA0C;;IAE1C,kCAAkC;;IAElC,4CAA4C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAhB9C;;;;;;;;GAQG;AACH,qBAAa,UAAW,SAAQ,eAsB9B;CAAG;AAEL;;;;;;;;GAQG;AACH,MAAM,WAAW,cAAc;IAC7B,6BAA6B;IAC7B,QAAQ,CAAC,KAAK,EAAE,UAAU,CAAC;IAC3B,uDAAuD;IACvD,QAAQ,CAAC,cAAc,EAAE,WAAW,CAAC;IACrC,uDAAuD;IACvD,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAC;CAClC;;;;AAED;;;;;GAKG;AACH,qBAAa,gBAAiB,SAAQ,sBAAqC;IACzE,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,KAAK,CAAC,EAAE,OAAO,CAAC;IACzB,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;CAC7B,CAAC;CAAG;AAEL;;;;;;;;GAQG;AACH,MAAM,WAAW,mBAAmB;IAClC,0DAA0D;IAC1D,QAAQ,CAAC,aAAa,EAAE,OAAO,CAAC;IAChC,iDAAiD;IACjD,QAAQ,CAAC,gBAAgB,EAAE,OAAO,CAAC;IACnC,4CAA4C;IAC5C,QAAQ,CAAC,iBAAiB,EAAE,OAAO,CAAC;IACpC,0CAA0C;IAC1C,QAAQ,CAAC,mBAAmB,EAAE,OAAO,CAAC;IACtC,iCAAiC;IACjC,QAAQ,CAAC,OAAO,EAAE,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;CAC7C;AAED;;;;;;;;GAQG;AACH,MAAM,WAAW,cAAc;IAC7B,kDAAkD;IAClD,QAAQ,CAAC,YAAY,EAAE,mBAAmB,CAAC;IAE3C,iCAAiC;IACjC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IAEtB,gEAAgE;IAChE,UAAU,IAAI,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,gBAAgB,EAAE,KAAK,CAAC,CAAC;IAE3D,gCAAgC;IAChC,OAAO,IAAI,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,gBAAgB,EAAE,KAAK,CAAC,CAAC;IAGxD,SAAS,CAAC,CACR,GAAG,EAAE,cAAc,EACnB,KAAK,EAAE,WAAW,GACjB,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,gBAAgB,EAAE,KAAK,CAAC,CAAC;IAChD,SAAS,CAAC,CACR,GAAG,EAAE,cAAc,GAClB,MAAM,CAAC,MAAM,CAAC,WAAW,GAAG,IAAI,EAAE,gBAAgB,EAAE,KAAK,CAAC,CAAC;IAC9D,WAAW,CAAC,CACV,GAAG,EAAE,cAAc,GAClB,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,gBAAgB,EAAE,KAAK,CAAC,CAAC;IAGhD,SAAS,CAAC,CAAC,KAAK,EAAE,UAAU,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,gBAAgB,EAAE,KAAK,CAAC,CAAC;IAC5E,UAAU,CAAC,CACT,MAAM,EAAE,UAAU,EAAE,GACnB,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,gBAAgB,EAAE,KAAK,CAAC,CAAC;IAChD,UAAU,CAAC,CACT,GAAG,EAAE,cAAc,EACnB,YAAY,CAAC,EAAE,MAAM,GACpB,MAAM,CAAC,MAAM,CAAC,UAAU,EAAE,EAAE,gBAAgB,EAAE,KAAK,CAAC,CAAC;IAGxD,YAAY,CAAC,CACX,GAAG,EAAE,cAAc,EACnB,KAAK,EAAE,WAAW,EAClB,QAAQ,EAAE,MAAM,GACf,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,gBAAgB,EAAE,KAAK,CAAC,CAAC;IAChD,kBAAkB,CAAC,CACjB,GAAG,EAAE,cAAc,GAClB,MAAM,CAAC,MAAM,CACd;QAAE,KAAK,EAAE,WAAW,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,EAC/C,gBAAgB,EAChB,KAAK,CACN,CAAC;IAGF,aAAa,CAAC,CACZ,GAAG,EAAE,cAAc,EACnB,cAAc,EAAE,MAAM,GACrB,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,gBAAgB,EAAE,KAAK,CAAC,CAAC;IAChD,YAAY,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,cAAc,EAAE,EAAE,gBAAgB,EAAE,KAAK,CAAC,CAAC;CAC3E;AAED;;;;;;;;GAQG;AACH,MAAM,WAAW,mBAAmB;IAClC,gCAAgC;IAChC,OAAO,CACL,SAAS,EAAE,cAAc,GACxB,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,gBAAgB,EAAE,KAAK,CAAC,CAAC;IAEhD,iCAAiC;IACjC,OAAO,CACL,GAAG,EAAE,cAAc,GAClB,MAAM,CAAC,MAAM,CAAC,WAAW,GAAG,IAAI,EAAE,gBAAgB,EAAE,KAAK,CAAC,CAAC;IAE9D,wBAAwB;IACxB,OAAO,CAAC,GAAG,EAAE,cAAc,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,gBAAgB,EAAE,KAAK,CAAC,CAAC;IAE3E,+BAA+B;IAC/B,OAAO,IAAI;QACT,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;QACtB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;QAC7B,QAAQ,CAAC,YAAY,EAAE,MAAM,EAAE,CAAC;KACjC,CAAC;CACH;AAED;;;;;;;GAOG;AACH,MAAM,WAAW,uBAAuB;IACtC,8CAA8C;IAC9C,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;IAClC,6DAA6D;IAC7D,QAAQ,CAAC,uBAAuB,EAAE,MAAM,CAAC;IACzC,oDAAoD;IACpD,QAAQ,CAAC,iBAAiB,EAAE,OAAO,CAAC;IACpC,oDAAoD;IACpD,QAAQ,CAAC,WAAW,EAAE,OAAO,CAAC;IAC9B,sCAAsC;IACtC,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;CACjC;AAED;;GAEG;AACH,eAAO,MAAM,qBAAqB,EAAE,uBAMnC,CAAC"}

package/dist/lib/Robots/Robots.service.d.ts

@@ -0,0 +1,78 @@
+import { Effect, Option } from 'effect';
+/**
+ * Parsed robots.txt rules for a specific user agent.
+ *
+ * Contains the disallowed paths and crawl delay settings extracted
+ * from a robots.txt file for a particular user agent string.
+ *
+ * @group Data Types
+ * @internal
+ */
+interface RobotsRules {
+    /** Set of URL paths that are disallowed for this user agent */
+    disallowedPaths: Set<string>;
+    /** Optional crawl delay in seconds specified in robots.txt */
+    crawlDelay?: number;
+    /** The user agent these rules apply to */
+    userAgent: string;
+}
+declare const RobotsService_base: Effect.Service.Class<RobotsService, "@jambudipa.io/RobotsService", {
+    readonly effect: Effect.Effect<{
+        checkUrl: (urlString: string) => Effect.Effect<{
+            allowed: boolean;
+            crawlDelay?: undefined;
+        } | {
+            allowed: boolean;
+            crawlDelay: number | undefined;
+        }, never, never>;
+        getRules: (domain: string) => Effect.Effect<Option.Option<RobotsRules>, never, never>;
+    }, never, never>;
+}>;
+/**
+ * Service for parsing and enforcing robots.txt compliance.
+ *
+ * The RobotsService handles fetching, parsing, and caching robots.txt files
+ * to ensure compliant web crawling. It provides efficient URL checking with
+ * automatic caching to minimise network requests.
+ *
+ * **Key Features:**
+ * - Automatic robots.txt fetching and parsing
+ * - Intelligent caching to reduce redundant requests
+ * - User agent-specific rule enforcement
+ * - Crawl delay extraction and enforcement
+ * - Graceful error handling for malformed robots.txt files
+ *
+ * **Standards Compliance:**
+ * - Follows the Robots Exclusion Standard (RFC 9309)
+ * - Supports User-agent, Disallow, and Crawl-delay directives
+ * - Handles wildcard (*) user agent specifications
+ * - Case-insensitive user agent matching
+ *
+ * @example
+ * ```typescript
+ * const program = Effect.gen(function* () {
+ *   const robots = yield* RobotsService;
+ *
+ *   // Check if URL is allowed
+ *   const check = yield* robots.checkUrl('https://example.com/admin');
+ *   if (!check.allowed) {
+ *     console.log('URL blocked by robots.txt');
+ *     return;
+ *   }
+ *
+ *   // Apply crawl delay if specified
+ *   if (check.crawlDelay) {
+ *     yield* Effect.sleep(`${check.crawlDelay} seconds`);
+ *   }
+ *
+ *   // Proceed with crawling...
+ * });
+ * ```
+ *
+ * @group Services
+ * @public
+ */
+export declare class RobotsService extends RobotsService_base {
+}
+export {};
+//# sourceMappingURL=Robots.service.d.ts.map

package/dist/lib/Robots/Robots.service.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"Robots.service.d.ts","sourceRoot":"","sources":["../../../src/lib/Robots/Robots.service.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAkB,MAAM,EAAE,MAAM,QAAQ,CAAC;AAGxD;;;;;;;;GAQG;AACH,UAAU,WAAW;IACnB,+DAA+D;IAC/D,eAAe,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IAC7B,8DAA8D;IAC9D,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,0CAA0C;IAC1C,SAAS,EAAE,MAAM,CAAC;CACnB;;;8BA6I6B,MAAM;;;;;;;2BAqDT,MAAM;;;AAhMjC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2CG;AACH,qBAAa,aAAc,SAAQ,kBA6JlC;CAAG"}

package/dist/lib/Scheduler/SpiderScheduler.service.d.ts

@@ -0,0 +1,211 @@
+import { Effect, Schema } from 'effect';
+import { CrawlTask } from '../Spider/Spider.service.js';
+import { ConfigurationError } from '../errors.js';
+declare const SpiderStateKey_base: Schema.Class<SpiderStateKey, {
+    /** Unique identifier for the session */
+    id: typeof Schema.String;
+    /** When the session was created */
+    timestamp: typeof Schema.Date;
+    /** Human-readable name for the session */
+    name: typeof Schema.String;
+}, Schema.Struct.Encoded<{
+    /** Unique identifier for the session */
+    id: typeof Schema.String;
+    /** When the session was created */
+    timestamp: typeof Schema.Date;
+    /** Human-readable name for the session */
+    name: typeof Schema.String;
+}>, never, {
+    readonly name: string;
+} & {
+    readonly timestamp: Date;
+} & {
+    readonly id: string;
+}, {}, {}>;
+/**
+ * Unique identifier for a spider crawling session.
+ *
+ * Used to identify and restore specific crawl sessions when using
+ * persistent storage. Each crawl session should have a unique key.
+ *
+ * @group Data Types
+ * @public
+ */
+export declare class SpiderStateKey extends SpiderStateKey_base {
+}
+declare const PriorityRequest_base: Schema.Class<PriorityRequest, {
+    /** The crawl task containing URL and depth information */
+    request: Schema.Struct<{
+        url: typeof Schema.String;
+        depth: typeof Schema.Number;
+        fromUrl: Schema.optional<typeof Schema.String>;
+    }>;
+    /** Priority level (higher numbers processed first) */
+    priority: typeof Schema.Number;
+    /** When this request was created */
+    timestamp: typeof Schema.Date;
+    /** Unique fingerprint for deduplication */
+    fingerprint: typeof Schema.String;
+}, Schema.Struct.Encoded<{
+    /** The crawl task containing URL and depth information */
+    request: Schema.Struct<{
+        url: typeof Schema.String;
+        depth: typeof Schema.Number;
+        fromUrl: Schema.optional<typeof Schema.String>;
+    }>;
+    /** Priority level (higher numbers processed first) */
+    priority: typeof Schema.Number;
+    /** When this request was created */
+    timestamp: typeof Schema.Date;
+    /** Unique fingerprint for deduplication */
+    fingerprint: typeof Schema.String;
+}>, never, {
+    readonly timestamp: Date;
+} & {
+    readonly priority: number;
+} & {
+    readonly fingerprint: string;
+} & {
+    readonly request: {
+        readonly url: string;
+        readonly depth: number;
+        readonly fromUrl?: string | undefined;
+    };
+}, {}, {}>;
+/**
+ * A crawl request with priority and metadata for scheduling.
+ *
+ * Requests are processed in priority order (higher numbers first),
+ * with FIFO ordering within the same priority level.
+ *
+ * @group Data Types
+ * @public
+ */
+export declare class PriorityRequest extends PriorityRequest_base {
+}
+declare const SpiderState_base: Schema.Class<SpiderState, {
+    /** The state key identifying this session */
+    key: typeof SpiderStateKey;
+    /** All requests waiting to be processed */
+    pendingRequests: Schema.Array$<typeof PriorityRequest>;
+    /** Fingerprints of URLs already visited (for deduplication) */
+    visitedFingerprints: Schema.Array$<typeof Schema.String>;
+    /** Total number of requests processed so far */
+    totalProcessed: typeof Schema.Number;
+}, Schema.Struct.Encoded<{
+    /** The state key identifying this session */
+    key: typeof SpiderStateKey;
+    /** All requests waiting to be processed */
+    pendingRequests: Schema.Array$<typeof PriorityRequest>;
+    /** Fingerprints of URLs already visited (for deduplication) */
+    visitedFingerprints: Schema.Array$<typeof Schema.String>;
+    /** Total number of requests processed so far */
+    totalProcessed: typeof Schema.Number;
+}>, never, {
+    readonly key: SpiderStateKey;
+} & {
+    readonly totalProcessed: number;
+} & {
+    readonly pendingRequests: readonly PriorityRequest[];
+} & {
+    readonly visitedFingerprints: readonly string[];
+}, {}, {}>;
+/**
+ * Complete state snapshot of a spider crawling session.
+ *
+ * This contains all information needed to resume a crawl session,
+ * including pending requests, visited URLs, and progress counters.
+ *
+ * @group Data Types
+ * @public
+ */
+export declare class SpiderState extends SpiderState_base {
+}
+/**
+ * Generic interface for persisting spider state.
+ *
+ * Implementations can use any storage backend (filesystem, database, etc.)
+ * to save and restore crawling sessions. All operations are Effect-based
+ * for composability and error handling.
+ *
+ * @example
+ * ```typescript
+ * class FilePersistence implements StatePersistence {
+ *   saveState = (key: SpiderStateKey, state: SpiderState) =>
+ *     Effect.tryPromise(() => fs.writeFile(key.id + '.json', JSON.stringify(state)))
+ *
+ *   loadState = (key: SpiderStateKey) =>
+ *     Effect.tryPromise(() => fs.readFile(key.id + '.json').then(JSON.parse))
+ *
+ *   deleteState = (key: SpiderStateKey) =>
+ *     Effect.tryPromise(() => fs.unlink(key.id + '.json'))
+ * }
+ * ```
+ *
+ * @group Interfaces
+ * @public
+ */
+export interface StatePersistence {
+    /** Saves the complete spider state to persistent storage */
+    saveState: (key: SpiderStateKey, state: SpiderState) => Effect.Effect<void, Error>;
+    /** Loads spider state from persistent storage, returns null if not found */
+    loadState: (key: SpiderStateKey) => Effect.Effect<SpiderState | null, Error>;
+    /** Deletes spider state from persistent storage */
+    deleteState: (key: SpiderStateKey) => Effect.Effect<void, Error>;
+}
+declare const SpiderSchedulerService_base: Effect.Service.Class<SpiderSchedulerService, "@jambudipa.io/SpiderSchedulerService", {
+    readonly effect: Effect.Effect<{
+        configurePersistence: (persistence: StatePersistence, stateKey: SpiderStateKey) => Effect.Effect<void, never, never>;
+        clearPersistence: () => Effect.Effect<void, never, never>;
+        enqueue: (request: CrawlTask, priority?: number) => Effect.Effect<boolean, Error, never>;
+        dequeue: () => Effect.Effect<PriorityRequest, Error, never>;
+        size: () => Effect.Effect<number, never, never>;
+        isEmpty: () => Effect.Effect<boolean, never, never>;
+        getState: () => Effect.Effect<SpiderState, ConfigurationError, never>;
+        restoreFromState: (state: SpiderState) => Effect.Effect<void, Error>;
+        restore: (persistence: StatePersistence, stateKey: SpiderStateKey) => Effect.Effect<boolean, Error, never>;
+    }, never, import("../Config/SpiderConfig.service.js").SpiderConfigService>;
+    readonly dependencies: readonly [import("effect/Layer").Layer<import("../Config/SpiderConfig.service.js").SpiderConfigService, never, never>];
+}>;
+/**
+ * Manages request scheduling, prioritization, and state persistence for web crawling.
+ *
+ * The SpiderSchedulerService provides a priority-based request queue with optional persistence
+ * capabilities. It handles:
+ * - Request deduplication via fingerprinting
+ * - Priority-based scheduling (higher numbers processed first)
+ * - State persistence for resumable crawling
+ * - Atomic state operations
+ *
+ * @example
+ * ```typescript
+ * const program = Effect.gen(function* () {
+ *   const scheduler = yield* SpiderSchedulerService;
+ *
+ *   // Configure persistence
+ *   const persistence = new FilePersistence('./state');
+ *   const stateKey = new SpiderStateKey({
+ *     id: 'my-crawl',
+ *     timestamp: new Date(),
+ *     name: 'Example Crawl'
+ *   });
+ *
+ *   yield* scheduler.configurePersistence(persistence, stateKey);
+ *
+ *   // Queue requests
+ *   yield* scheduler.enqueue({ url: 'https://example.com', depth: 0 }, 10);
+ *   yield* scheduler.enqueue({ url: 'https://example.com/about', depth: 1 }, 5);
+ *
+ *   // Process requests
+ *   const request = yield* scheduler.dequeue();
+ *   console.log(`Processing: ${request.request.url}`);
+ * });
+ * ```
+ *
+ * @group Services
+ * @public
+ */
+export declare class SpiderSchedulerService extends SpiderSchedulerService_base {
+}
+export {};
+//# sourceMappingURL=SpiderScheduler.service.d.ts.map

package/dist/lib/Scheduler/SpiderScheduler.service.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"SpiderScheduler.service.d.ts","sourceRoot":"","sources":["../../../src/lib/Scheduler/SpiderScheduler.service.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAyB,MAAM,EAAE,MAAM,QAAQ,CAAC;AAC/D,OAAO,EAAE,SAAS,EAAE,MAAM,6BAA6B,CAAC;AACxD,OAAO,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAC;;IAehD,wCAAwC;;IAExC,mCAAmC;;IAEnC,0CAA0C;;;IAJ1C,wCAAwC;;IAExC,mCAAmC;;IAEnC,0CAA0C;;;;;;;;;AAhB5C;;;;;;;;GAQG;AACH,qBAAa,cAAe,SAAQ,mBASlC;CAAG;;IAcH,0DAA0D;;;;;;IAM1D,sDAAsD;;IAEtD,oCAAoC;;IAEpC,2CAA2C;;;IAV3C,0DAA0D;;;;;;IAM1D,sDAAsD;;IAEtD,oCAAoC;;IAEpC,2CAA2C;;;;;;;;;;;;;;;AAtB7C;;;;;;;;GAQG;AACH,qBAAa,eAAgB,SAAQ,oBAenC;CAAG;;IAYH,6CAA6C;;IAE7C,2CAA2C;;IAE3C,+DAA+D;;IAE/D,gDAAgD;;;IANhD,6CAA6C;;IAE7C,2CAA2C;;IAE3C,+DAA+D;;IAE/D,gDAAgD;;;;;;;;;;;AAhBlD;;;;;;;;GAQG;AACH,qBAAa,WAAY,SAAQ,gBAS/B;CAAG;AAEL;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,MAAM,WAAW,gBAAgB;IAC/B,4DAA4D;IAC5D,SAAS,EAAE,CACT,GAAG,EAAE,cAAc,EACnB,KAAK,EAAE,WAAW,KACf,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IAChC,4EAA4E;IAC5E,SAAS,EAAE,CAAC,GAAG,EAAE,cAAc,KAAK,MAAM,CAAC,MAAM,CAAC,WAAW,GAAG,IAAI,EAAE,KAAK,CAAC,CAAC;IAC7E,mDAAmD;IACnD,WAAW,EAAE,CAAC,GAAG,EAAE,cAAc,KAAK,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;CAClE;;;4CA6LsB,gBAAgB,YACnB,cAAc;;2BAeP,SAAS;;;;;kCAhDrB,WAAW,KACjB,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC;+BA4HJ,gBAAgB,YAAY,cAAc;;;;AAxRzE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAqCG;AACH,qBAAa,sBAAuB,SAAQ,2BAgQ3C;CAAG"}

package/dist/lib/Scraper/Scraper.service.d.ts

@@ -0,0 +1,123 @@
+import { Effect } from 'effect';
+import { NetworkError, ResponseError } from '../errors.js';
+import { SpiderLogger } from '../Logging/SpiderLogger.service.js';
+declare const ScraperService_base: Effect.Service.Class<ScraperService, "@jambudipa.io/ScraperService", {
+    readonly effect: Effect.Effect<{
+        /**
+         * Fetches a URL and parses the HTML to extract basic page information.
+         *
+         * This method performs the following operations:
+         * 1. Fetches the URL with configurable timeout (30 seconds)
+         * 2. Validates content type (skips binary files)
+         * 3. Parses HTML content with cheerio
+         * 4. Extracts basic page metadata (title, description, etc.)
+         * 5. Returns structured PageData object
+         *
+         * The method uses AbortController for proper timeout handling to prevent
+         * workers from hanging on malformed URLs or slow responses.
+         *
+         * @param url - The URL to fetch and parse
+         * @param depth - The crawl depth for logging purposes (default: 0)
+         * @returns Effect containing PageData with extracted information
+         * @throws NetworkError for network-related failures
+         * @throws ResponseError for HTTP error responses
+         *
+         * @example
+         * Basic usage:
+         * ```typescript
+         * const pageData = yield* scraper.fetchAndParse('https://example.com');
+         * console.log(`Page title: ${pageData.title}`);
+         * ```
+         *
+         * With depth tracking:
+         * ```typescript
+         * const pageData = yield* scraper.fetchAndParse('https://example.com/page', 2);
+         * ```
+         *
+         * Error handling:
+         * ```typescript
+         * const result = yield* scraper.fetchAndParse('https://example.com').pipe(
+         *   Effect.catchTags({
+         *     NetworkError: (error) => {
+         *       console.log('Network error:', error.message);
+         *       return Effect.succeed(null);
+         *     },
+         *     ResponseError: (error) => {
+         *       console.log('HTTP error:', error.statusCode);
+         *       return Effect.succeed(null);
+         *     }
+         *   })
+         * );
+         * ```
+         *
+         * @performance
+         * - Request timeout: 30 seconds
+         * - Response parsing timeout: 10 seconds
+         * - Memory usage: ~2-5MB per page depending on content size
+         *
+         * @security
+         * - Validates content types to prevent processing binary files
+         * - Uses AbortController to prevent hanging requests
+         * - No execution of JavaScript content (static HTML parsing only)
+         */
+        fetchAndParse: (url: string, depth?: number) => Effect.Effect<{
+            readonly url: string;
+            readonly html: string;
+            readonly title?: string | undefined;
+            readonly metadata: {
+                readonly [x: string]: string;
+            };
+            readonly commonMetadata?: {
+                readonly description?: string | undefined;
+                readonly keywords?: string | undefined;
+                readonly author?: string | undefined;
+                readonly robots?: string | undefined;
+            } | undefined;
+            readonly statusCode: number;
+            readonly headers: {
+                readonly [x: string]: string;
+            };
+            readonly fetchedAt: Date;
+            readonly scrapeDurationMs: number;
+            readonly depth: number;
+            readonly extractedData?: {
+                readonly [x: string]: unknown;
+            } | undefined;
+        }, NetworkError | ResponseError | import("effect/ParseResult").ParseError, SpiderLogger>;
+    }, never, never>;
+}>;
+/**
+ * Service responsible for fetching HTML content and parsing basic page information.
+ *
+ * The ScraperService handles the core HTTP fetching and HTML parsing functionality
+ * for the Spider framework. It provides robust error handling, timeout management,
+ * and content type validation to ensure reliable data extraction.
+ *
+ * **Key Features:**
+ * - Automatic timeout handling with AbortController
+ * - Content type validation (skips binary files)
+ * - Comprehensive error handling with typed errors
+ * - Performance monitoring and logging
+ * - Effect.js integration for composability
+ *
+ * **Note:** This service focuses solely on fetching and parsing HTML content.
+ * Link extraction is handled separately by LinkExtractorService for better
+ * separation of concerns and modularity.
+ *
+ * @example
+ * ```typescript
+ * const program = Effect.gen(function* () {
+ *   const scraper = yield* ScraperService;
+ *   const pageData = yield* scraper.fetchAndParse('https://example.com', 0);
+ *   console.log(`Title: ${pageData.title}`);
+ *   console.log(`Content length: ${pageData.html.length}`);
+ * });
+ * ```
+ *
+ * @group Services
+ * @public
+ */
+export declare class ScraperService extends ScraperService_base {
+}
+export {};
+//# sourceMappingURL=Scraper.service.d.ts.map

package/dist/lib/Scraper/Scraper.service.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"Scraper.service.d.ts","sourceRoot":"","sources":["../../../src/lib/Scraper/Scraper.service.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAU,MAAM,QAAQ,CAAC;AAGxC,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAC3D,OAAO,EAAE,YAAY,EAAE,MAAM,oCAAoC,CAAC;;;QAqC5D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;WAwDG;6BACkB,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;AA5FjC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AACH,qBAAa,cAAe,SAAQ,mBAqOnC;CAAG"}