@jambudipa/spider 0.2.1 → 0.2.3

package/dist/lib/Resumability/strategies.d.ts

@@ -0,0 +1,76 @@
+import { Effect, Option } from 'effect';
+import { SpiderState, SpiderStateKey } from '../Scheduler/SpiderScheduler.service.js';
+import { HybridPersistenceConfig, PersistenceError, PersistenceStrategy, StateOperation, StorageBackend } from './types.js';
+/**
+ * Full state persistence strategy.
+ *
+ * Saves the complete spider state on every operation. Simple and reliable,
+ * but can be inefficient for large crawls with many URLs.
+ *
+ * @group Strategies
+ * @public
+ */
+export declare class FullStatePersistence implements PersistenceStrategy {
+    private readonly backend;
+    constructor(backend: StorageBackend);
+    persist: (operation: StateOperation) => Effect.Effect<void, PersistenceError>;
+    restore: (key: SpiderStateKey) => Effect.Effect<Option.Option<SpiderState>, PersistenceError>;
+    cleanup: (key: SpiderStateKey) => Effect.Effect<void, PersistenceError>;
+    getInfo: () => {
+        name: string;
+        description: string;
+        capabilities: string[];
+    };
+}
+/**
+ * Delta persistence strategy.
+ *
+ * Saves only incremental changes (deltas) instead of the full state.
+ * Much more efficient for large crawls, but requires delta replay for restoration.
+ *
+ * @group Strategies
+ * @public
+ */
+export declare class DeltaPersistence implements PersistenceStrategy {
+    private readonly backend;
+    constructor(backend: StorageBackend);
+    persist: (operation: StateOperation) => Effect.Effect<void, PersistenceError>;
+    restore: (key: SpiderStateKey) => Effect.Effect<Option.Option<SpiderState>, PersistenceError>;
+    cleanup: (key: SpiderStateKey) => Effect.Effect<void, PersistenceError>;
+    reconstructStateFromDeltas: (key: SpiderStateKey, deltas: ReadonlyArray<import("./types.js").StateDelta>) => Effect.Effect<SpiderState, PersistenceError>;
+    getInfo: () => {
+        name: string;
+        description: string;
+        capabilities: string[];
+    };
+}
+/**
+ * Hybrid persistence strategy.
+ *
+ * Combines delta and full state approaches for optimal performance.
+ * Saves deltas for efficiency, with periodic snapshots for fast recovery.
+ *
+ * @group Strategies
+ * @public
+ */
+export declare class HybridPersistence implements PersistenceStrategy {
+    private readonly backend;
+    private readonly config;
+    private operationCount;
+    private lastSnapshotSequence;
+    private pendingDeltas;
+    constructor(backend: StorageBackend, config?: HybridPersistenceConfig);
+    persist: (operation: StateOperation) => Effect.Effect<void, PersistenceError>;
+    private saveSnapshot;
+    private saveDelta;
+    private flushPendingDeltas;
+    restore: (key: SpiderStateKey) => Effect.Effect<Option.Option<SpiderState>, PersistenceError>;
+    private applyDeltasToState;
+    cleanup: (key: SpiderStateKey) => Effect.Effect<void, PersistenceError>;
+    getInfo: () => {
+        name: string;
+        description: string;
+        capabilities: string[];
+    };
+}
+//# sourceMappingURL=strategies.d.ts.map

package/dist/lib/Resumability/strategies.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"strategies.d.ts","sourceRoot":"","sources":["../../../src/lib/Resumability/strategies.ts"],"names":[],"mappings":"AAAA,OAAO,EAAS,MAAM,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAC/C,OAAO,EACL,WAAW,EACX,cAAc,EACf,MAAM,yCAAyC,CAAC;AACjD,OAAO,EAEL,uBAAuB,EACvB,gBAAgB,EAChB,mBAAmB,EACnB,cAAc,EACd,cAAc,EACf,MAAM,YAAY,CAAC;AAEpB;;;;;;;;GAQG;AACH,qBAAa,oBAAqB,YAAW,mBAAmB;IAClD,OAAO,CAAC,QAAQ,CAAC,OAAO;gBAAP,OAAO,EAAE,cAAc;IAEpD,OAAO,GACL,WAAW,cAAc,KACxB,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,gBAAgB,CAAC,CAiBtC;IAEF,OAAO,GACL,KAAK,cAAc,KAClB,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,EAAE,gBAAgB,CAAC,CAc5D;IAEF,OAAO,GAAI,KAAK,cAAc,KAAG,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,gBAAgB,CAAC,CAcpE;IAEF,OAAO;;;;MAKJ;CACJ;AAED;;;;;;;;GAQG;AACH,qBAAa,gBAAiB,YAAW,mBAAmB;IAC9C,OAAO,CAAC,QAAQ,CAAC,OAAO;gBAAP,OAAO,EAAE,cAAc;IAEpD,OAAO,GACL,WAAW,cAAc,KACxB,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,gBAAgB,CAAC,CActC;IAEF,OAAO,GACL,KAAK,cAAc,KAClB,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,EAAE,gBAAgB,CAAC,CAqB5D;IAEF,OAAO,GAAI,KAAK,cAAc,KAAG,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,gBAAgB,CAAC,CAmBpE;IAEF,0BAA0B,GACxB,KAAK,cAAc,EACnB,QAAQ,aAAa,CAAC,OAAO,YAAY,EAAE,UAAU,CAAC,KACrD,MAAM,CAAC,MAAM,CAAC,WAAW,EAAE,gBAAgB,CAAC,CA4D1C;IAEL,OAAO;;;;MAKJ;CACJ;AAED;;;;;;;;GAQG;AACH,qBAAa,iBAAkB,YAAW,mBAAmB;IAMzD,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,MAAM;IANzB,OAAO,CAAC,cAAc,CAAK;IAC3B,OAAO,CAAC,oBAAoB,CAAK;IACjC,OAAO,CAAC,aAAa,CAAyC;gBAG3C,OAAO,EAAE,cAAc,EACvB,MAAM,GAAE,uBAA+C;IAG1E,OAAO,GACL,WAAW,cAAc,KACxB,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,gBAAgB,CAAC,CA+BtC;IAEF,OAAO,CAAC,YAAY,CAkClB;IAEF,OAAO,CAAC,SAAS,CAmBf;IAEF,OAAO,CAAC,kBAAkB,CAwBxB;IAEF,OAAO,GACL,KAAK,cAAc,KAClB,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,EAAE,gBAAgB,CAAC,CA0C5D;IAEF,OAAO,CAAC,kBAAkB,CA0ExB;IAEF,OAAO,GAAI,KAAK,cAAc,KAAG,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,gBAAgB,CAAC,CAepE;IAEF,OAAO;;;;MAWJ;CACJ"}

package/dist/lib/Resumability/types.d.ts

@@ -0,0 +1,201 @@
+import { Effect, Option, Schema } from 'effect';
+import { PriorityRequest, SpiderState, SpiderStateKey } from '../Scheduler/SpiderScheduler.service.js';
+export { SpiderStateKey, PriorityRequest, SpiderState };
+declare const StateDelta_base: Schema.Class<StateDelta, {
+    /** Session this delta applies to */
+    stateKey: typeof Schema.String;
+    /** Sequence number for ordering deltas */
+    sequence: typeof Schema.Number;
+    /** When this delta was created */
+    timestamp: typeof Schema.Date;
+    /** The operation that created this delta */
+    operation: Schema.Union<[Schema.Struct<{
+        type: Schema.Literal<["enqueue"]>;
+        request: typeof PriorityRequest;
+    }>, Schema.Struct<{
+        type: Schema.Literal<["dequeue"]>;
+        fingerprint: typeof Schema.String;
+    }>, Schema.Struct<{
+        type: Schema.Literal<["mark_visited"]>;
+        fingerprint: typeof Schema.String;
+    }>]>;
+}, Schema.Struct.Encoded<{
+    /** Session this delta applies to */
+    stateKey: typeof Schema.String;
+    /** Sequence number for ordering deltas */
+    sequence: typeof Schema.Number;
+    /** When this delta was created */
+    timestamp: typeof Schema.Date;
+    /** The operation that created this delta */
+    operation: Schema.Union<[Schema.Struct<{
+        type: Schema.Literal<["enqueue"]>;
+        request: typeof PriorityRequest;
+    }>, Schema.Struct<{
+        type: Schema.Literal<["dequeue"]>;
+        fingerprint: typeof Schema.String;
+    }>, Schema.Struct<{
+        type: Schema.Literal<["mark_visited"]>;
+        fingerprint: typeof Schema.String;
+    }>]>;
+}>, never, {
+    readonly operation: {
+        readonly type: "enqueue";
+        readonly request: PriorityRequest;
+    } | {
+        readonly type: "dequeue";
+        readonly fingerprint: string;
+    } | {
+        readonly type: "mark_visited";
+        readonly fingerprint: string;
+    };
+} & {
+    readonly timestamp: Date;
+} & {
+    readonly stateKey: string;
+} & {
+    readonly sequence: number;
+}, {}, {}>;
+/**
+ * Delta operation that represents a single state change.
+ *
+ * Used for incremental persistence instead of saving the entire state
+ * on every operation, which is much more efficient for large crawls.
+ *
+ * @group Delta Updates
+ * @public
+ */
+export declare class StateDelta extends StateDelta_base {
+}
+/**
+ * Represents a state change operation with both the delta and resulting state.
+ *
+ * This allows persistence strategies to choose whether to save deltas,
+ * full state, or both depending on their optimization needs.
+ *
+ * @group Operations
+ * @public
+ */
+export interface StateOperation {
+    /** The incremental change */
+    readonly delta: StateDelta;
+    /** The complete state after applying this operation */
+    readonly resultingState: SpiderState;
+    /** Whether this operation should trigger a snapshot */
+    readonly shouldSnapshot: boolean;
+}
+declare const PersistenceError_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").Equals<A, {}> extends true ? void : { readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }) => import("effect/Cause").YieldableError & {
+    readonly _tag: "PersistenceError";
+} & Readonly<A>;
+/**
+ * Error that can occur during persistence operations.
+ *
+ * @group Errors
+ * @public
+ */
+export declare class PersistenceError extends PersistenceError_base<{
+    readonly message: string;
+    readonly cause?: unknown;
+    readonly operation?: string;
+}> {
+}
+/**
+ * Storage backend capabilities that determine optimal persistence strategy.
+ *
+ * Backends advertise their capabilities so the ResumabilityService can
+ * choose the best strategy automatically.
+ *
+ * @group Storage
+ * @public
+ */
+export interface StorageCapabilities {
+    /** Can efficiently store and retrieve delta operations */
+    readonly supportsDelta: boolean;
+    /** Can efficiently store full state snapshots */
+    readonly supportsSnapshot: boolean;
+    /** Can handle streaming/batch operations */
+    readonly supportsStreaming: boolean;
+    /** Can handle concurrent access safely */
+    readonly supportsConcurrency: boolean;
+    /** Estimated latency category */
+    readonly latency: 'low' | 'medium' | 'high';
+}
+/**
+ * Generic storage backend interface that persistence strategies use.
+ *
+ * Backends implement the storage operations they support best.
+ * Not all methods need to be implemented - strategies will adapt.
+ *
+ * @group Storage
+ * @public
+ */
+export interface StorageBackend {
+    /** Backend capabilities for strategy selection */
+    readonly capabilities: StorageCapabilities;
+    /** Storage backend identifier */
+    readonly name: string;
+    /** Initialize the backend (create tables, connections, etc.) */
+    initialize(): Effect.Effect<void, PersistenceError>;
+    /** Cleanup backend resources */
+    cleanup(): Effect.Effect<void, PersistenceError>;
+    saveState?(key: SpiderStateKey, state: SpiderState): Effect.Effect<void, PersistenceError>;
+    loadState?(key: SpiderStateKey): Effect.Effect<Option.Option<SpiderState>, PersistenceError>;
+    deleteState?(key: SpiderStateKey): Effect.Effect<void, PersistenceError>;
+    saveDelta?(delta: StateDelta): Effect.Effect<void, PersistenceError>;
+    saveDeltas?(deltas: readonly StateDelta[]): Effect.Effect<void, PersistenceError>;
+    loadDeltas?(key: SpiderStateKey, fromSequence?: number): Effect.Effect<readonly StateDelta[], PersistenceError>;
+    saveSnapshot?(key: SpiderStateKey, state: SpiderState, sequence: number): Effect.Effect<void, PersistenceError>;
+    loadLatestSnapshot?(key: SpiderStateKey): Effect.Effect<Option.Option<{
+        state: SpiderState;
+        sequence: number;
+    }>, PersistenceError>;
+    compactDeltas?(key: SpiderStateKey, beforeSequence: number): Effect.Effect<void, PersistenceError>;
+    listSessions?(): Effect.Effect<readonly SpiderStateKey[], PersistenceError>;
+}
+/**
+ * Core strategy interface for different persistence approaches.
+ *
+ * Strategies implement the logic for when and how to persist state,
+ * using the storage backend for actual I/O operations.
+ *
+ * @group Strategies
+ * @public
+ */
+export interface PersistenceStrategy {
+    /** Persist a state operation */
+    persist(operation: StateOperation): Effect.Effect<void, PersistenceError>;
+    /** Restore state from storage */
+    restore(key: SpiderStateKey): Effect.Effect<Option.Option<SpiderState>, PersistenceError>;
+    /** Clean up old data */
+    cleanup(key: SpiderStateKey): Effect.Effect<void, PersistenceError>;
+    /** Get strategy information */
+    getInfo(): {
+        readonly name: string;
+        readonly description: string;
+        readonly capabilities: string[];
+    };
+}
+/**
+ * Configuration for hybrid persistence strategy.
+ *
+ * Controls when to save snapshots vs deltas for optimal performance.
+ *
+ * @group Configuration
+ * @public
+ */
+export interface HybridPersistenceConfig {
+    /** Save a full snapshot every N operations */
+    readonly snapshotInterval: number;
+    /** Maximum deltas to accumulate before forcing a snapshot */
+    readonly maxDeltasBeforeSnapshot: number;
+    /** Whether to compact old deltas after snapshots */
+    readonly compactionEnabled: boolean;
+    /** Batch multiple deltas together for efficiency */
+    readonly batchDeltas: boolean;
+    /** Batch size for delta operations */
+    readonly deltaBatchSize: number;
+}
+/**
+ * Default hybrid persistence configuration.
+ */
+export declare const DEFAULT_HYBRID_CONFIG: HybridPersistenceConfig;
+//# sourceMappingURL=types.d.ts.map

package/dist/lib/Resumability/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/lib/Resumability/types.ts"],"names":[],"mappings":"AAAA,OAAO,EAAQ,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AACtD,OAAO,EACL,eAAe,EACf,WAAW,EACX,cAAc,EACf,MAAM,yCAAyC,CAAC;AAGjD,OAAO,EAAE,cAAc,EAAE,eAAe,EAAE,WAAW,EAAE,CAAC;;IAYtD,oCAAoC;;IAEpC,0CAA0C;;IAE1C,kCAAkC;;IAElC,4CAA4C;;;;;;;;;;;;IAN5C,oCAAoC;;IAEpC,0CAA0C;;IAE1C,kCAAkC;;IAElC,4CAA4C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAhB9C;;;;;;;;GAQG;AACH,qBAAa,UAAW,SAAQ,eAsB9B;CAAG;AAEL;;;;;;;;GAQG;AACH,MAAM,WAAW,cAAc;IAC7B,6BAA6B;IAC7B,QAAQ,CAAC,KAAK,EAAE,UAAU,CAAC;IAC3B,uDAAuD;IACvD,QAAQ,CAAC,cAAc,EAAE,WAAW,CAAC;IACrC,uDAAuD;IACvD,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAC;CAClC;;;;AAED;;;;;GAKG;AACH,qBAAa,gBAAiB,SAAQ,sBAAqC;IACzE,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,KAAK,CAAC,EAAE,OAAO,CAAC;IACzB,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;CAC7B,CAAC;CAAG;AAEL;;;;;;;;GAQG;AACH,MAAM,WAAW,mBAAmB;IAClC,0DAA0D;IAC1D,QAAQ,CAAC,aAAa,EAAE,OAAO,CAAC;IAChC,iDAAiD;IACjD,QAAQ,CAAC,gBAAgB,EAAE,OAAO,CAAC;IACnC,4CAA4C;IAC5C,QAAQ,CAAC,iBAAiB,EAAE,OAAO,CAAC;IACpC,0CAA0C;IAC1C,QAAQ,CAAC,mBAAmB,EAAE,OAAO,CAAC;IACtC,iCAAiC;IACjC,QAAQ,CAAC,OAAO,EAAE,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;CAC7C;AAED;;;;;;;;GAQG;AACH,MAAM,WAAW,cAAc;IAC7B,kDAAkD;IAClD,QAAQ,CAAC,YAAY,EAAE,mBAAmB,CAAC;IAE3C,iCAAiC;IACjC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IAEtB,gEAAgE;IAChE,UAAU,IAAI,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,gBAAgB,CAAC,CAAC;IAEpD,gCAAgC;IAChC,OAAO,IAAI,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,gBAAgB,CAAC,CAAC;IAGjD,SAAS,CAAC,CACR,GAAG,EAAE,cAAc,EACnB,KAAK,EAAE,WAAW,GACjB,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,gBAAgB,CAAC,CAAC;IACzC,SAAS,CAAC,CACR,GAAG,EAAE,cAAc,GAClB,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,EAAE,gBAAgB,CAAC,CAAC;IAC/D,WAAW,CAAC,CACV,GAAG,EAAE,cAAc,GAClB,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,gBAAgB,CAAC,CAAC;IAGzC,SAAS,CAAC,CAAC,KAAK,EAAE,UAAU,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,gBAAgB,CAAC,CAAC;IACrE,UAAU,CAAC,CACT,MAAM,EAAE,SAAS,UAAU,EAAE,GAC5B,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,gBAAgB,CAAC,CAAC;IACzC,UAAU,CAAC,CACT,GAAG,EAAE,cAAc,EACnB,YAAY,CAAC,EAAE,MAAM,GACpB,MAAM,CAAC,MAAM,CAAC,SAAS,UAAU,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAG1D,YAAY,CAAC,CACX,GAAG,EAAE,cAAc,EACnB,KAAK,EAAE,WAAW,EAClB,QAAQ,EAAE,MAAM,GACf,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,gBAAgB,CAAC,CAAC;IACzC,kBAAkB,CAAC,CACjB,GAAG,EAAE,cAAc,GAClB,MAAM,CAAC,MAAM,CACd,MAAM,CAAC,MAAM,CAAC;QAAE,KAAK,EAAE,WAAW,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC,EACvD,gBAAgB,CACjB,CAAC;IAGF,aAAa,CAAC,CACZ,GAAG,EAAE,cAAc,EACnB,cAAc,EAAE,MAAM,GACrB,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,gBAAgB,CAAC,CAAC;IACzC,YAAY,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,SAAS,cAAc,EAAE,EAAE,gBAAgB,CAAC,CAAC;CAC7E;AAED;;;;;;;;GAQG;AACH,MAAM,WAAW,mBAAmB;IAClC,gCAAgC;IAChC,OAAO,CACL,SAAS,EAAE,cAAc,GACxB,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,gBAAgB,CAAC,CAAC;IAEzC,iCAAiC;IACjC,OAAO,CACL,GAAG,EAAE,cAAc,GAClB,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,EAAE,gBAAgB,CAAC,CAAC;IAE/D,wBAAwB;IACxB,OAAO,CAAC,GAAG,EAAE,cAAc,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,gBAAgB,CAAC,CAAC;IAEpE,+BAA+B;IAC/B,OAAO,IAAI;QACT,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;QACtB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;QAC7B,QAAQ,CAAC,YAAY,EAAE,MAAM,EAAE,CAAC;KACjC,CAAC;CACH;AAED;;;;;;;GAOG;AACH,MAAM,WAAW,uBAAuB;IACtC,8CAA8C;IAC9C,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;IAClC,6DAA6D;IAC7D,QAAQ,CAAC,uBAAuB,EAAE,MAAM,CAAC;IACzC,oDAAoD;IACpD,QAAQ,CAAC,iBAAiB,EAAE,OAAO,CAAC;IACpC,oDAAoD;IACpD,QAAQ,CAAC,WAAW,EAAE,OAAO,CAAC;IAC9B,sCAAsC;IACtC,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;CACjC;AAED;;GAEG;AACH,eAAO,MAAM,qBAAqB,EAAE,uBAMnC,CAAC"}

package/dist/lib/Robots/Robots.service.d.ts

@@ -0,0 +1,78 @@
+import { Effect, MutableHashSet, Option } from 'effect';
+/**
+ * Parsed robots.txt rules for a specific user agent.
+ *
+ * Contains the disallowed paths and crawl delay settings extracted
+ * from a robots.txt file for a particular user agent string.
+ *
+ * @group Data Types
+ * @internal
+ */
+interface RobotsRules {
+    /** Set of URL paths that are disallowed for this user agent */
+    disallowedPaths: MutableHashSet.MutableHashSet<string>;
+    /** Optional crawl delay in seconds specified in robots.txt */
+    crawlDelay?: number;
+    /** The user agent these rules apply to */
+    userAgent: string;
+}
+declare const RobotsService_base: Effect.Service.Class<RobotsService, "@jambudipa.io/RobotsService", {
+    readonly effect: Effect.Effect<{
+        checkUrl: (urlString: string) => Effect.Effect<{
+            allowed: boolean;
+            crawlDelay?: undefined;
+        } | {
+            allowed: boolean;
+            crawlDelay: number | undefined;
+        }, never, never>;
+        getRules: (domain: string) => Effect.Effect<Option.Option<RobotsRules>, never, never>;
+    }, never, never>;
+}>;
+/**
+ * Service for parsing and enforcing robots.txt compliance.
+ *
+ * The RobotsService handles fetching, parsing, and caching robots.txt files
+ * to ensure compliant web crawling. It provides efficient URL checking with
+ * automatic caching to minimise network requests.
+ *
+ * **Key Features:**
+ * - Automatic robots.txt fetching and parsing
+ * - Intelligent caching to reduce redundant requests
+ * - User agent-specific rule enforcement
+ * - Crawl delay extraction and enforcement
+ * - Graceful error handling for malformed robots.txt files
+ *
+ * **Standards Compliance:**
+ * - Follows the Robots Exclusion Standard (RFC 9309)
+ * - Supports User-agent, Disallow, and Crawl-delay directives
+ * - Handles wildcard (*) user agent specifications
+ * - Case-insensitive user agent matching
+ *
+ * @example
+ * ```typescript
+ * const program = Effect.gen(function* () {
+ *   const robots = yield* RobotsService;
+ *
+ *   // Check if URL is allowed
+ *   const check = yield* robots.checkUrl('https://example.com/admin');
+ *   if (!check.allowed) {
+ *     console.log('URL blocked by robots.txt');
+ *     return;
+ *   }
+ *
+ *   // Apply crawl delay if specified
+ *   if (check.crawlDelay) {
+ *     yield* Effect.sleep(`${check.crawlDelay} seconds`);
+ *   }
+ *
+ *   // Proceed with crawling...
+ * });
+ * ```
+ *
+ * @group Services
+ * @public
+ */
+export declare class RobotsService extends RobotsService_base {
+}
+export {};
+//# sourceMappingURL=Robots.service.d.ts.map

package/dist/lib/Robots/Robots.service.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"Robots.service.d.ts","sourceRoot":"","sources":["../../../src/lib/Robots/Robots.service.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAkB,cAAc,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAGxE;;;;;;;;GAQG;AACH,UAAU,WAAW;IACnB,+DAA+D;IAC/D,eAAe,EAAE,cAAc,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;IACvD,8DAA8D;IAC9D,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,0CAA0C;IAC1C,SAAS,EAAE,MAAM,CAAC;CACnB;;;8BAuK6B,MAAM;;;;;;;2BA+CT,MAAM;;;AApNjC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2CG;AACH,qBAAa,aAAc,SAAQ,kBAiLlC;CAAG"}

package/dist/lib/Scheduler/SpiderScheduler.service.d.ts

@@ -0,0 +1,211 @@
+import { Effect, Option, Schema } from 'effect';
+import { CrawlTask } from '../Spider/Spider.service.js';
+import { ConfigurationError } from '../errors.js';
+declare const SpiderStateKey_base: Schema.Class<SpiderStateKey, {
+    /** Unique identifier for the session */
+    id: typeof Schema.String;
+    /** When the session was created */
+    timestamp: typeof Schema.Date;
+    /** Human-readable name for the session */
+    name: typeof Schema.String;
+}, Schema.Struct.Encoded<{
+    /** Unique identifier for the session */
+    id: typeof Schema.String;
+    /** When the session was created */
+    timestamp: typeof Schema.Date;
+    /** Human-readable name for the session */
+    name: typeof Schema.String;
+}>, never, {
+    readonly name: string;
+} & {
+    readonly timestamp: Date;
+} & {
+    readonly id: string;
+}, {}, {}>;
+/**
+ * Unique identifier for a spider crawling session.
+ *
+ * Used to identify and restore specific crawl sessions when using
+ * persistent storage. Each crawl session should have a unique key.
+ *
+ * @group Data Types
+ * @public
+ */
+export declare class SpiderStateKey extends SpiderStateKey_base {
+}
+declare const PriorityRequest_base: Schema.Class<PriorityRequest, {
+    /** The crawl task containing URL and depth information */
+    request: Schema.Struct<{
+        url: typeof Schema.String;
+        depth: typeof Schema.Number;
+        fromUrl: Schema.optional<typeof Schema.String>;
+    }>;
+    /** Priority level (higher numbers processed first) */
+    priority: typeof Schema.Number;
+    /** When this request was created */
+    timestamp: typeof Schema.Date;
+    /** Unique fingerprint for deduplication */
+    fingerprint: typeof Schema.String;
+}, Schema.Struct.Encoded<{
+    /** The crawl task containing URL and depth information */
+    request: Schema.Struct<{
+        url: typeof Schema.String;
+        depth: typeof Schema.Number;
+        fromUrl: Schema.optional<typeof Schema.String>;
+    }>;
+    /** Priority level (higher numbers processed first) */
+    priority: typeof Schema.Number;
+    /** When this request was created */
+    timestamp: typeof Schema.Date;
+    /** Unique fingerprint for deduplication */
+    fingerprint: typeof Schema.String;
+}>, never, {
+    readonly timestamp: Date;
+} & {
+    readonly priority: number;
+} & {
+    readonly fingerprint: string;
+} & {
+    readonly request: {
+        readonly url: string;
+        readonly depth: number;
+        readonly fromUrl?: string | undefined;
+    };
+}, {}, {}>;
+/**
+ * A crawl request with priority and metadata for scheduling.
+ *
+ * Requests are processed in priority order (higher numbers first),
+ * with FIFO ordering within the same priority level.
+ *
+ * @group Data Types
+ * @public
+ */
+export declare class PriorityRequest extends PriorityRequest_base {
+}
+declare const SpiderState_base: Schema.Class<SpiderState, {
+    /** The state key identifying this session */
+    key: typeof SpiderStateKey;
+    /** All requests waiting to be processed */
+    pendingRequests: Schema.Array$<typeof PriorityRequest>;
+    /** Fingerprints of URLs already visited (for deduplication) */
+    visitedFingerprints: Schema.Array$<typeof Schema.String>;
+    /** Total number of requests processed so far */
+    totalProcessed: typeof Schema.Number;
+}, Schema.Struct.Encoded<{
+    /** The state key identifying this session */
+    key: typeof SpiderStateKey;
+    /** All requests waiting to be processed */
+    pendingRequests: Schema.Array$<typeof PriorityRequest>;
+    /** Fingerprints of URLs already visited (for deduplication) */
+    visitedFingerprints: Schema.Array$<typeof Schema.String>;
+    /** Total number of requests processed so far */
+    totalProcessed: typeof Schema.Number;
+}>, never, {
+    readonly key: SpiderStateKey;
+} & {
+    readonly totalProcessed: number;
+} & {
+    readonly pendingRequests: readonly PriorityRequest[];
+} & {
+    readonly visitedFingerprints: readonly string[];
+}, {}, {}>;
+/**
+ * Complete state snapshot of a spider crawling session.
+ *
+ * This contains all information needed to resume a crawl session,
+ * including pending requests, visited URLs, and progress counters.
+ *
+ * @group Data Types
+ * @public
+ */
+export declare class SpiderState extends SpiderState_base {
+}
+/**
+ * Generic interface for persisting spider state.
+ *
+ * Implementations can use any storage backend (filesystem, database, etc.)
+ * to save and restore crawling sessions. All operations are Effect-based
+ * for composability and error handling.
+ *
+ * @example
+ * ```typescript
+ * class FilePersistence implements StatePersistence {
+ *   saveState = (key: SpiderStateKey, state: SpiderState) =>
+ *     Effect.tryPromise(() => fs.writeFile(key.id + '.json', JSON.stringify(state)))
+ *
+ *   loadState = (key: SpiderStateKey) =>
+ *     Effect.tryPromise(() => fs.readFile(key.id + '.json').then(JSON.parse))
+ *
+ *   deleteState = (key: SpiderStateKey) =>
+ *     Effect.tryPromise(() => fs.unlink(key.id + '.json'))
+ * }
+ * ```
+ *
+ * @group Interfaces
+ * @public
+ */
+export interface StatePersistence {
+    /** Saves the complete spider state to persistent storage */
+    saveState: (_key: SpiderStateKey, _state: SpiderState) => Effect.Effect<void, Error>;
+    /** Loads spider state from persistent storage, returns Option.none if not found */
+    loadState: (_key: SpiderStateKey) => Effect.Effect<Option.Option<SpiderState>, Error>;
+    /** Deletes spider state from persistent storage */
+    deleteState: (_key: SpiderStateKey) => Effect.Effect<void, Error>;
+}
+declare const SpiderSchedulerService_base: Effect.Service.Class<SpiderSchedulerService, "@jambudipa/spiderSchedulerService", {
+    readonly effect: Effect.Effect<{
+        configurePersistence: (persistence: StatePersistence, stateKey: SpiderStateKey) => Effect.Effect<void, never, never>;
+        clearPersistence: () => Effect.Effect<void, never, never>;
+        enqueue: (request: CrawlTask, priority?: number) => Effect.Effect<boolean, Error, never>;
+        dequeue: () => Effect.Effect<PriorityRequest, Error, never>;
+        size: () => Effect.Effect<number, never, never>;
+        isEmpty: () => Effect.Effect<boolean, never, never>;
+        getState: () => Effect.Effect<SpiderState, ConfigurationError, never>;
+        restoreFromState: (state: SpiderState) => Effect.Effect<void, Error>;
+        restore: (persistence: StatePersistence, stateKey: SpiderStateKey) => Effect.Effect<boolean, Error, never>;
+    }, never, import("../Config/SpiderConfig.service.js").SpiderConfigService>;
+    readonly dependencies: readonly [import("effect/Layer").Layer<import("../Config/SpiderConfig.service.js").SpiderConfigService, never, never>];
+}>;
+/**
+ * Manages request scheduling, prioritization, and state persistence for web crawling.
+ *
+ * The SpiderSchedulerService provides a priority-based request queue with optional persistence
+ * capabilities. It handles:
+ * - Request deduplication via fingerprinting
+ * - Priority-based scheduling (higher numbers processed first)
+ * - State persistence for resumable crawling
+ * - Atomic state operations
+ *
+ * @example
+ * ```typescript
+ * const program = Effect.gen(function* () {
+ *   const scheduler = yield* SpiderSchedulerService;
+ *
+ *   // Configure persistence
+ *   const persistence = new FilePersistence('./state');
+ *   const stateKey = new SpiderStateKey({
+ *     id: 'my-crawl',
+ *     timestamp: new Date(),
+ *     name: 'Example Crawl'
+ *   });
+ *
+ *   yield* scheduler.configurePersistence(persistence, stateKey);
+ *
+ *   // Queue requests
+ *   yield* scheduler.enqueue({ url: 'https://example.com', depth: 0 }, 10);
+ *   yield* scheduler.enqueue({ url: 'https://example.com/about', depth: 1 }, 5);
+ *
+ *   // Process requests
+ *   const request = yield* scheduler.dequeue();
+ *   console.log(`Processing: ${request.request.url}`);
+ * });
+ * ```
+ *
+ * @group Services
+ * @public
+ */
+export declare class SpiderSchedulerService extends SpiderSchedulerService_base {
+}
+export {};
+//# sourceMappingURL=SpiderScheduler.service.d.ts.map

package/dist/lib/Scheduler/SpiderScheduler.service.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"SpiderScheduler.service.d.ts","sourceRoot":"","sources":["../../../src/lib/Scheduler/SpiderScheduler.service.ts"],"names":[],"mappings":"AAAA,OAAO,EAAY,MAAM,EAAkB,MAAM,EAAS,MAAM,EAAE,MAAM,QAAQ,CAAC;AACjF,OAAO,EAAE,SAAS,EAAE,MAAM,6BAA6B,CAAC;AACxD,OAAO,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAC;;IAehD,wCAAwC;;IAExC,mCAAmC;;IAEnC,0CAA0C;;;IAJ1C,wCAAwC;;IAExC,mCAAmC;;IAEnC,0CAA0C;;;;;;;;;AAhB5C;;;;;;;;GAQG;AACH,qBAAa,cAAe,SAAQ,mBASlC;CAAG;;IAcH,0DAA0D;;;;;;IAM1D,sDAAsD;;IAEtD,oCAAoC;;IAEpC,2CAA2C;;;IAV3C,0DAA0D;;;;;;IAM1D,sDAAsD;;IAEtD,oCAAoC;;IAEpC,2CAA2C;;;;;;;;;;;;;;;AAtB7C;;;;;;;;GAQG;AACH,qBAAa,eAAgB,SAAQ,oBAenC;CAAG;;IAYH,6CAA6C;;IAE7C,2CAA2C;;IAE3C,+DAA+D;;IAE/D,gDAAgD;;;IANhD,6CAA6C;;IAE7C,2CAA2C;;IAE3C,+DAA+D;;IAE/D,gDAAgD;;;;;;;;;;;AAhBlD;;;;;;;;GAQG;AACH,qBAAa,WAAY,SAAQ,gBAS/B;CAAG;AAEL;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,MAAM,WAAW,gBAAgB;IAC/B,4DAA4D;IAC5D,SAAS,EAAE,CACT,IAAI,EAAE,cAAc,EACpB,MAAM,EAAE,WAAW,KAChB,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IAChC,mFAAmF;IACnF,SAAS,EAAE,CAAC,IAAI,EAAE,cAAc,KAAK,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,EAAE,KAAK,CAAC,CAAC;IACtF,mDAAmD;IACnD,WAAW,EAAE,CAAC,IAAI,EAAE,cAAc,KAAK,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;CACnE;;;4CAiMsB,gBAAgB,YACnB,cAAc;;2BAeP,SAAS;;;;;kCAhDrB,WAAW,KACjB,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC;+BA4HJ,gBAAgB,YAAY,cAAc;;;;AA5RzE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAqCG;AACH,qBAAa,sBAAuB,SAAQ,2BAoQ3C;CAAG"}