@jambudipa/spider 0.2.1 → 0.2.3

package/dist/lib/Middleware/types.d.ts

@@ -0,0 +1,99 @@
+/**
+ * Data type definitions for Spider Middleware
+ * Using Effect's Data.Class for immutability and built-in equality
+ */
+import { Data, Option } from 'effect';
+import { PageData } from '../PageData/PageData.js';
+/**
+ * Represents a single crawling task with URL and depth information.
+ * Used internally by the Spider service for task management.
+ */
+export interface CrawlTask {
+    /** The URL to be crawled */
+    url: string;
+    /** The depth level of this URL relative to the starting URL */
+    depth: number;
+    /** The URL from which this URL was discovered (optional) */
+    fromUrl?: string;
+    /** Optional metadata to be passed through to the result */
+    metadata?: Record<string, unknown>;
+    /** Optional data extraction configuration */
+    extractData?: Record<string, unknown>;
+}
+/**
+ * Request object used in the middleware pipeline.
+ *
+ * Contains the crawl task along with optional headers and metadata
+ * that can be modified by middleware during processing.
+ *
+ * Uses Data.Class for:
+ * - Built-in equality checking
+ * - Immutability by default
+ * - Better pattern matching support
+ *
+ * @group Data Types
+ * @public
+ */
+export declare class SpiderRequest extends Data.Class<{
+    /** The crawl task containing URL and depth information */
+    readonly task: CrawlTask;
+    /** HTTP headers to include with the request */
+    readonly headers: Option.Option<Record<string, string>>;
+    /** Additional metadata that can be used by middleware */
+    readonly meta: Option.Option<Record<string, unknown>>;
+}> {
+    /**
+     * Create a SpiderRequest from a CrawlTask
+     */
+    static fromTask(task: CrawlTask, headers?: Record<string, string>, meta?: Record<string, unknown>): SpiderRequest;
+    /**
+     * Add or update headers
+     */
+    withHeaders(headers: Record<string, string>): SpiderRequest;
+    /**
+     * Add or update metadata
+     */
+    withMeta(meta: Record<string, unknown>): SpiderRequest;
+}
+/**
+ * Response object used in the middleware pipeline.
+ *
+ * Contains the extracted page data along with optional HTTP response
+ * information and metadata from middleware processing.
+ *
+ * Uses Data.Class for:
+ * - Built-in equality checking
+ * - Immutability by default
+ * - Better pattern matching support
+ *
+ * @group Data Types
+ * @public
+ */
+export declare class SpiderResponse extends Data.Class<{
+    /** The extracted page data including content, links, and metadata */
+    readonly pageData: PageData;
+    /** HTTP status code of the response */
+    readonly statusCode: Option.Option<number>;
+    /** HTTP response headers */
+    readonly headers: Option.Option<Record<string, string>>;
+    /** Additional metadata from middleware processing */
+    readonly meta: Option.Option<Record<string, unknown>>;
+}> {
+    /**
+     * Create a SpiderResponse from PageData
+     */
+    static fromPageData(pageData: PageData, statusCode?: number, headers?: Record<string, string>, meta?: Record<string, unknown>): SpiderResponse;
+    /**
+     * Update the page data
+     */
+    withPageData(pageData: PageData): SpiderResponse;
+    /**
+     * Add or update metadata
+     */
+    withMeta(meta: Record<string, unknown>): SpiderResponse;
+    /**
+     * Check if the response was successful (2xx status code)
+     */
+    isSuccessful(): boolean;
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/lib/Middleware/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/lib/Middleware/types.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AACtC,OAAO,EAAE,QAAQ,EAAE,MAAM,yBAAyB,CAAC;AAEnD;;;GAGG;AACH,MAAM,WAAW,SAAS;IACxB,4BAA4B;IAC5B,GAAG,EAAE,MAAM,CAAC;IACZ,+DAA+D;IAC/D,KAAK,EAAE,MAAM,CAAC;IACd,4DAA4D;IAC5D,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,2DAA2D;IAC3D,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACnC,6CAA6C;IAC7C,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACvC;AAED;;;;;;;;;;;;;GAaG;AACH,qBAAa,aAAc,SAAQ,IAAI,CAAC,KAAK,CAAC;IAC5C,0DAA0D;IAC1D,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAC;IACzB,+CAA+C;IAC/C,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IACxD,yDAAyD;IACzD,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;CACvD,CAAC;IACA;;OAEG;IACH,MAAM,CAAC,QAAQ,CACb,IAAI,EAAE,SAAS,EACf,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EAChC,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAC7B,aAAa;IAQhB;;OAEG;IACH,WAAW,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,aAAa;IAQ3D;;OAEG;IACH,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,aAAa;CAOvD;AAED;;;;;;;;;;;;;GAaG;AACH,qBAAa,cAAe,SAAQ,IAAI,CAAC,KAAK,CAAC;IAC7C,qEAAqE;IACrE,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC;IAC5B,uCAAuC;IACvC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAC3C,4BAA4B;IAC5B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IACxD,qDAAqD;IACrD,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;CACvD,CAAC;IACA;;OAEG;IACH,MAAM,CAAC,YAAY,CACjB,QAAQ,EAAE,QAAQ,EAClB,UAAU,CAAC,EAAE,MAAM,EACnB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EAChC,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAC7B,cAAc;IASjB;;OAEG;IACH,YAAY,CAAC,QAAQ,EAAE,QAAQ,GAAG,cAAc;IAOhD;;OAEG;IACH,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,cAAc;IAQvD;;OAEG;IACH,YAAY,IAAI,OAAO;CAMxB"}

package/dist/lib/PageData/PageData.d.ts

@@ -0,0 +1,28 @@
+import { Schema } from 'effect';
+export declare const PageDataSchema: Schema.Struct<{
+    url: Schema.filter<typeof Schema.String>;
+    html: typeof Schema.String;
+    title: Schema.optional<typeof Schema.String>;
+    /** All available metadata from meta tags */
+    metadata: Schema.Record$<typeof Schema.String, typeof Schema.String>;
+    /** Commonly used metadata fields for convenience */
+    commonMetadata: Schema.optional<Schema.Struct<{
+        description: Schema.optional<typeof Schema.String>;
+        keywords: Schema.optional<typeof Schema.String>;
+        author: Schema.optional<typeof Schema.String>;
+        robots: Schema.optional<typeof Schema.String>;
+    }>>;
+    statusCode: Schema.filter<Schema.filter<typeof Schema.Number>>;
+    /** All response headers */
+    headers: Schema.Record$<typeof Schema.String, typeof Schema.String>;
+    /** When the fetch operation started */
+    fetchedAt: typeof Schema.DateFromSelf;
+    /** How long the entire fetch and parse operation took in milliseconds */
+    scrapeDurationMs: typeof Schema.Number;
+    /** The crawl depth (number of hops from the starting URL) */
+    depth: Schema.filter<Schema.filter<typeof Schema.Number>>;
+    /** Optional extracted data from the page */
+    extractedData: Schema.optional<Schema.Record$<typeof Schema.String, typeof Schema.Unknown>>;
+}>;
+export type PageData = Schema.Schema.Type<typeof PageDataSchema>;
+//# sourceMappingURL=PageData.d.ts.map

package/dist/lib/PageData/PageData.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"PageData.d.ts","sourceRoot":"","sources":["../../../src/lib/PageData/PageData.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAEhC,eAAO,MAAM,cAAc;;;;IAQzB,4CAA4C;;IAE5C,oDAAoD;;;;;;;;IAUpD,2BAA2B;;IAE3B,uCAAuC;;IAEvC,yEAAyE;;IAEzE,6DAA6D;;IAE7D,4CAA4C;;EAI5C,CAAC;AAEH,MAAM,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,cAAc,CAAC,CAAC"}

package/dist/lib/Resumability/Resumability.service.d.ts

@@ -0,0 +1,178 @@
+import { Effect, Option } from 'effect';
+import { SpiderState, SpiderStateKey } from '../Scheduler/SpiderScheduler.service.js';
+import { HybridPersistenceConfig, PersistenceError, StateDelta, StateOperation, StorageBackend } from './types.js';
+import { type RedisClientInterface } from './backends/RedisStorageBackend.js';
+import { type DatabaseClientInterface, type PostgresStorageConfig } from './backends/PostgresStorageBackend.js';
+/**
+ * Configuration for the ResumabilityService.
+ *
+ * Allows choosing between different persistence strategies and
+ * configuring their behavior based on use case requirements.
+ *
+ * @group Configuration
+ * @public
+ */
+export interface ResumabilityConfig {
+    /** Persistence strategy to use */
+    strategy: 'full-state' | 'delta' | 'hybrid' | 'auto';
+    /** Storage backend implementation */
+    backend: StorageBackend;
+    /** Configuration for hybrid strategy (only used when strategy is 'hybrid') */
+    hybridConfig?: HybridPersistenceConfig;
+}
+declare const ResumabilityService_base: Effect.Service.Class<ResumabilityService, "@jambudipa.io/ResumabilityService", {
+    readonly effect: Effect.Effect<{
+        /**
+         * Configure the resumability service with a specific strategy and backend.
+         *
+         * This method initializes the storage backend and creates the appropriate
+         * persistence strategy based on the configuration.
+         *
+         * @param config - Resumability configuration
+         * @returns Effect that completes when configuration is applied
+         */
+        configure: (config: ResumabilityConfig) => Effect.Effect<void, PersistenceError, never>;
+        /**
+         * Persist a state operation using the configured strategy.
+         *
+         * @param operation - State operation to persist
+         * @returns Effect that completes when operation is persisted
+         */
+        persistOperation: (operation: StateOperation) => Effect.Effect<undefined, PersistenceError, never>;
+        /**
+         * Restore spider state from persistent storage.
+         *
+         * @param key - State key identifying the session to restore
+         * @returns Effect containing the restored state, or null if not found
+         */
+        restore: (key: SpiderStateKey) => Effect.Effect<Option.Option<SpiderState>, PersistenceError, never>;
+        /**
+         * Clean up old state data for a session.
+         *
+         * @param key - State key identifying the session to clean up
+         * @returns Effect that completes when cleanup is finished
+         */
+        cleanup: (key: SpiderStateKey) => Effect.Effect<undefined, PersistenceError, never>;
+        /**
+         * List all available sessions in storage.
+         *
+         * @returns Effect containing array of session keys
+         */
+        listSessions: () => Effect.Effect<readonly SpiderStateKey[], PersistenceError, never>;
+        /**
+         * Get information about the current configuration.
+         *
+         * @returns Information about strategy and backend
+         */
+        getInfo: () => Effect.Effect<{
+            strategy: {
+                readonly name: string;
+                readonly description: string;
+                readonly capabilities: string[];
+            };
+            backend: {
+                name: string;
+                capabilities: import("./types.js").StorageCapabilities;
+            };
+        }, PersistenceError, never>;
+        /**
+         * Reconfigure the service with new settings.
+         *
+         * This will clean up the current backend and reinitialize with new config.
+         *
+         * @param config - New configuration
+         * @returns Effect that completes when reconfiguration is finished
+         */
+        reconfigure: (config: ResumabilityConfig) => Effect.Effect<void, PersistenceError, never>;
+    }, never, never>;
+}>;
+/**
+ * Service for resumable spider crawling with configurable persistence strategies.
+ *
+ * Provides a unified interface for different persistence approaches:
+ * - Full state: Simple, saves complete state on every change
+ * - Delta: Efficient, saves only incremental changes
+ * - Hybrid: Best of both worlds, deltas + periodic snapshots
+ * - Auto: Automatically chooses best strategy based on backend capabilities
+ *
+ * @example
+ * ```typescript
+ * // File-based full state persistence
+ * const resumabilityLayer = ResumabilityService.fromConfig({
+ *   strategy: 'full-state',
+ *   backend: new FileStorageBackend('./spider-state')
+ * });
+ *
+ * // Redis-based hybrid persistence
+ * const resumabilityLayer = ResumabilityService.fromConfig({
+ *   strategy: 'hybrid',
+ *   backend: new RedisStorageBackend(redisClient),
+ *   hybridConfig: {
+ *     snapshotInterval: 1000,
+ *     maxDeltasBeforeSnapshot: 500
+ *   }
+ * });
+ *
+ * // Auto-selected strategy based on backend
+ * const resumabilityLayer = ResumabilityService.fromConfig({
+ *   strategy: 'auto',
+ *   backend: new PostgresStorageBackend(pgClient)
+ * });
+ * ```
+ *
+ * @group Services
+ * @public
+ */
+export declare class ResumabilityService extends ResumabilityService_base {
+    /**
+     * Create a ResumabilityService layer from configuration.
+     *
+     * This is the primary way to create and configure the ResumabilityService.
+     *
+     * @param config - Resumability configuration
+     * @returns Effect layer providing the configured ResumabilityService
+     */
+    static fromConfig: (config: ResumabilityConfig) => Effect.Effect<ResumabilityService, PersistenceError, never>;
+}
+/**
+ * Utility function to create a state operation.
+ *
+ * @param delta - The delta operation
+ * @param resultingState - The complete state after applying the delta
+ * @param shouldSnapshot - Whether this operation should trigger a snapshot
+ * @returns StateOperation object
+ */
+export declare const createStateOperation: (delta: StateDelta, resultingState: SpiderState, shouldSnapshot?: boolean) => StateOperation;
+/**
+ * Factory functions for creating common resumability configurations.
+ */
+export declare const ResumabilityConfigs: {
+    /**
+     * Create a file-based configuration.
+     *
+     * @param baseDir - Directory to store state files
+     * @param strategy - Persistence strategy (defaults to 'auto')
+     * @returns ResumabilityConfig
+     */
+    file: (baseDir: string, strategy?: "full-state" | "delta" | "hybrid" | "auto") => ResumabilityConfig;
+    /**
+     * Create a Redis-based configuration.
+     *
+     * @param redisClient - Redis client instance
+     * @param strategy - Persistence strategy (defaults to 'hybrid')
+     * @param keyPrefix - Redis key prefix (defaults to 'spider')
+     * @returns ResumabilityConfig
+     */
+    redis: (redisClient: RedisClientInterface, strategy?: "full-state" | "delta" | "hybrid" | "auto", keyPrefix?: string) => ResumabilityConfig;
+    /**
+     * Create a PostgreSQL-based configuration.
+     *
+     * @param dbClient - Database client instance
+     * @param strategy - Persistence strategy (defaults to 'hybrid')
+     * @param config - PostgreSQL configuration
+     * @returns ResumabilityConfig
+     */
+    postgres: (dbClient: DatabaseClientInterface, strategy?: "full-state" | "delta" | "hybrid" | "auto", config?: PostgresStorageConfig) => ResumabilityConfig;
+};
+export {};
+//# sourceMappingURL=Resumability.service.d.ts.map

package/dist/lib/Resumability/Resumability.service.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"Resumability.service.d.ts","sourceRoot":"","sources":["../../../src/lib/Resumability/Resumability.service.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AACxC,OAAO,EACL,WAAW,EACX,cAAc,EACf,MAAM,yCAAyC,CAAC;AACjD,OAAO,EAEL,uBAAuB,EACvB,gBAAgB,EAEhB,UAAU,EACV,cAAc,EACd,cAAc,EACf,MAAM,YAAY,CAAC;AAOpB,OAAO,EAEL,KAAK,oBAAoB,EAC1B,MAAM,mCAAmC,CAAC;AAC3C,OAAO,EAEL,KAAK,uBAAuB,EAC5B,KAAK,qBAAqB,EAC3B,MAAM,sCAAsC,CAAC;AAE9C;;;;;;;;GAQG;AACH,MAAM,WAAW,kBAAkB;IACjC,kCAAkC;IAClC,QAAQ,EAAE,YAAY,GAAG,OAAO,GAAG,QAAQ,GAAG,MAAM,CAAC;IACrD,qCAAqC;IACrC,OAAO,EAAE,cAAc,CAAC;IACxB,8EAA8E;IAC9E,YAAY,CAAC,EAAE,uBAAuB,CAAC;CACxC;;;QAmDO;;;;;;;;WAQG;4BACiB,kBAAkB;QAWtC;;;;;WAKG;sCAC2B,cAAc;QAe5C;;;;;WAKG;uBACY,cAAc;QAe7B;;;;;WAKG;uBACY,cAAc;QAe7B;;;;WAIG;;QA0BH;;;;WAIG;;;;;;;;;;;;QAwBH;;;;;;;WAOG;8BACmB,kBAAkB;;;AAtMhD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCG;AACH,qBAAa,mBAAoB,SAAQ,wBAgLxC;IACC;;;;;;;OAOG;IACH,MAAM,CAAC,UAAU,GAAI,QAAQ,kBAAkB,iEAKQ;CACxD;AAwDD;;;;;;;GAOG;AACH,eAAO,MAAM,oBAAoB,GAC/B,OAAO,UAAU,EACjB,gBAAgB,WAAW,EAC3B,wBAAsB,KACrB,cAID,CAAC;AAEH;;GAEG;AACH,eAAO,MAAM,mBAAmB;IAC9B;;;;;;OAMG;oBAEQ,MAAM,aACL,YAAY,GAAG,OAAO,GAAG,QAAQ,GAAG,MAAM,KACnD,kBAAkB;IAKrB;;;;;;;OAOG;yBAEY,oBAAoB,aACvB,YAAY,GAAG,OAAO,GAAG,QAAQ,GAAG,MAAM,yBAEnD,kBAAkB;IAKrB;;;;;;;OAOG;yBAES,uBAAuB,aACvB,YAAY,GAAG,OAAO,GAAG,QAAQ,GAAG,MAAM,WAC3C,qBAAqB,KAC7B,kBAAkB;CAItB,CAAC"}

package/dist/lib/Resumability/backends/FileStorageBackend.d.ts

@@ -0,0 +1,47 @@
+import { Effect, Option } from 'effect';
+import { PersistenceError, SpiderState, SpiderStateKey, StateDelta, StorageBackend, StorageCapabilities } from '../types.js';
+/**
+ * File system storage backend for spider state persistence.
+ *
+ * Stores state and deltas as JSON files in a directory structure.
+ * Good for development, testing, and single-machine deployments.
+ *
+ * Directory structure:
+ * ```
+ * baseDir/
+ *   sessions/
+ *     sessionId/
+ *       state.json        # Full state
+ *       snapshot.json     # Latest snapshot
+ *       deltas/
+ *         0001.json       # Delta files
+ *         0002.json
+ *         ...
+ * ```
+ *
+ * @group Backends
+ * @public
+ */
+export declare class FileStorageBackend implements StorageBackend {
+    readonly capabilities: StorageCapabilities;
+    readonly name = "FileStorageBackend";
+    private readonly storageDir;
+    constructor(baseDir: string);
+    initialize: () => Effect.Effect<void, PersistenceError>;
+    cleanup: () => Effect.Effect<void, PersistenceError>;
+    saveState: (key: SpiderStateKey, state: SpiderState) => Effect.Effect<void, PersistenceError>;
+    loadState: (key: SpiderStateKey) => Effect.Effect<Option.Option<SpiderState>, PersistenceError>;
+    deleteState: (key: SpiderStateKey) => Effect.Effect<void, PersistenceError>;
+    saveDelta: (delta: StateDelta) => Effect.Effect<void, PersistenceError>;
+    saveDeltas: (deltas: StateDelta[]) => Effect.Effect<void, PersistenceError>;
+    loadDeltas: (key: SpiderStateKey, fromSequence?: number) => Effect.Effect<StateDelta[], PersistenceError>;
+    saveSnapshot: (key: SpiderStateKey, state: SpiderState, sequence: number) => Effect.Effect<void, PersistenceError>;
+    loadLatestSnapshot: (key: SpiderStateKey) => Effect.Effect<Option.Option<{
+        state: SpiderState;
+        sequence: number;
+    }>, PersistenceError>;
+    compactDeltas: (key: SpiderStateKey, beforeSequence: number) => Effect.Effect<void, PersistenceError>;
+    listSessions: () => Effect.Effect<SpiderStateKey[], PersistenceError>;
+    private getSessionDir;
+}
+//# sourceMappingURL=FileStorageBackend.d.ts.map

package/dist/lib/Resumability/backends/FileStorageBackend.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"FileStorageBackend.d.ts","sourceRoot":"","sources":["../../../../src/lib/Resumability/backends/FileStorageBackend.ts"],"names":[],"mappings":"AAAA,OAAO,EAAmB,MAAM,EAAE,MAAM,EAAU,MAAM,QAAQ,CAAC;AAGjE,OAAO,EACL,gBAAgB,EAChB,WAAW,EACX,cAAc,EACd,UAAU,EACV,cAAc,EACd,mBAAmB,EACpB,MAAM,aAAa,CAAC;AA4BrB;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,qBAAa,kBAAmB,YAAW,cAAc;IACvD,QAAQ,CAAC,YAAY,EAAE,mBAAmB,CAMxC;IAEF,QAAQ,CAAC,IAAI,wBAAwB;IAErC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAS;gBAExB,OAAO,EAAE,MAAM;IAI3B,UAAU,QAAO,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,gBAAgB,CAAC,CAuBpD;IAEF,OAAO,QAAO,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,gBAAgB,CAAC,CAAgB;IAGnE,SAAS,GACP,KAAK,cAAc,EACnB,OAAO,WAAW,KACjB,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,gBAAgB,CAAC,CAqCtC;IAEF,SAAS,GACP,KAAK,cAAc,KAClB,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,EAAE,gBAAgB,CAAC,CA0C5D;IAEF,WAAW,GACT,KAAK,cAAc,KAClB,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,gBAAgB,CAAC,CAetC;IAGF,SAAS,GAAI,OAAO,UAAU,KAAG,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,gBAAgB,CAAC,CA6CpE;IAEF,UAAU,GACR,QAAQ,UAAU,EAAE,KACnB,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,gBAAgB,CAAC,CAQtC;IAEF,UAAU,GACR,KAAK,cAAc,EACnB,qBAAgB,KACf,MAAM,CAAC,MAAM,CAAC,UAAU,EAAE,EAAE,gBAAgB,CAAC,CA6D9C;IAGF,YAAY,GACV,KAAK,cAAc,EACnB,OAAO,WAAW,EAClB,UAAU,MAAM,KACf,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,gBAAgB,CAAC,CA0CtC;IAEF,kBAAkB,GAChB,KAAK,cAAc,KAClB,MAAM,CAAC,MAAM,CACd,MAAM,CAAC,MAAM,CAAC;QAAE,KAAK,EAAE,WAAW,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC,EACvD,gBAAgB,CACjB,CA6CC;IAGF,aAAa,GACX,KAAK,cAAc,EACnB,gBAAgB,MAAM,KACrB,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,gBAAgB,CAAC,CA6CtC;IAEF,YAAY,QAAO,MAAM,CAAC,MAAM,CAAC,cAAc,EAAE,EAAE,gBAAgB,CAAC,CA6DlE;IAEF,OAAO,CAAC,aAAa,CAEnB;CACH"}

package/dist/lib/Resumability/backends/PostgresStorageBackend.d.ts

@@ -0,0 +1,95 @@
+import { Effect, Option } from 'effect';
+import { SpiderState, SpiderStateKey } from '../../Scheduler/SpiderScheduler.service.js';
+import { PersistenceError, StateDelta, StorageBackend, StorageCapabilities } from '../types.js';
+/**
+ * Database client interface for dependency injection.
+ *
+ * This allows users to provide their own database client implementation
+ * (pg, node-postgres, prisma, drizzle, etc.) without tight coupling.
+ *
+ * @group Backends
+ * @public
+ */
+export interface DatabaseClientInterface {
+    query<T = unknown>(sql: string, params?: readonly unknown[]): Promise<{
+        rows: readonly T[];
+        rowCount: number;
+    }>;
+    transaction?<T>(callback: (client: DatabaseClientInterface) => Promise<T>): Promise<T>;
+}
+/**
+ * Configuration for PostgreSQL storage backend.
+ */
+export interface PostgresStorageConfig {
+    /** Table prefix for spider tables */
+    tablePrefix?: string;
+    /** Schema name (defaults to 'public') */
+    schema?: string;
+    /** Whether to auto-create tables */
+    autoCreateTables?: boolean;
+}
+/**
+ * PostgreSQL storage backend for spider state persistence.
+ *
+ * Uses PostgreSQL for robust, ACID-compliant state persistence with
+ * excellent support for concurrent access and complex queries.
+ *
+ * Database schema:
+ * ```sql
+ * CREATE TABLE spider_sessions (
+ *   id VARCHAR(255) PRIMARY KEY,
+ *   name VARCHAR(255) NOT NULL,
+ *   created_at TIMESTAMP NOT NULL,
+ *   state_data JSONB,
+ *   updated_at TIMESTAMP DEFAULT NOW()
+ * );
+ *
+ * CREATE TABLE spider_deltas (
+ *   id SERIAL PRIMARY KEY,
+ *   session_id VARCHAR(255) NOT NULL REFERENCES spider_sessions(id),
+ *   sequence_number BIGINT NOT NULL,
+ *   operation_type VARCHAR(50) NOT NULL,
+ *   operation_data JSONB NOT NULL,
+ *   created_at TIMESTAMP DEFAULT NOW(),
+ *   UNIQUE(session_id, sequence_number)
+ * );
+ *
+ * CREATE TABLE spider_snapshots (
+ *   id SERIAL PRIMARY KEY,
+ *   session_id VARCHAR(255) NOT NULL REFERENCES spider_sessions(id),
+ *   sequence_number BIGINT NOT NULL,
+ *   state_data JSONB NOT NULL,
+ *   created_at TIMESTAMP DEFAULT NOW()
+ * );
+ * ```
+ *
+ * @group Backends
+ * @public
+ */
+export declare class PostgresStorageBackend implements StorageBackend {
+    readonly db: DatabaseClientInterface;
+    readonly capabilities: StorageCapabilities;
+    readonly name = "PostgresStorageBackend";
+    private readonly tablePrefix;
+    private readonly schema;
+    private readonly autoCreateTables;
+    constructor(db: DatabaseClientInterface, config?: PostgresStorageConfig);
+    initialize: () => Effect.Effect<void, PersistenceError>;
+    cleanup: () => Effect.Effect<void, PersistenceError>;
+    saveState: (key: SpiderStateKey, state: SpiderState) => Effect.Effect<void, PersistenceError>;
+    loadState: (key: SpiderStateKey) => Effect.Effect<Option.Option<SpiderState>, PersistenceError>;
+    deleteState: (key: SpiderStateKey) => Effect.Effect<void, PersistenceError>;
+    saveDelta: (delta: StateDelta) => Effect.Effect<void, PersistenceError>;
+    saveDeltas: (deltas: readonly StateDelta[]) => Effect.Effect<void, PersistenceError>;
+    loadDeltas: (key: SpiderStateKey, fromSequence?: number) => Effect.Effect<StateDelta[], PersistenceError>;
+    saveSnapshot: (key: SpiderStateKey, state: SpiderState, sequence: number) => Effect.Effect<void, PersistenceError>;
+    loadLatestSnapshot: (key: SpiderStateKey) => Effect.Effect<Option.Option<{
+        state: SpiderState;
+        sequence: number;
+    }>, PersistenceError>;
+    compactDeltas: (key: SpiderStateKey, beforeSequence: number) => Effect.Effect<void, PersistenceError>;
+    listSessions: () => Effect.Effect<SpiderStateKey[], PersistenceError>;
+    private createTables;
+    private getTableName;
+}
+//# sourceMappingURL=PostgresStorageBackend.d.ts.map

package/dist/lib/Resumability/backends/PostgresStorageBackend.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"PostgresStorageBackend.d.ts","sourceRoot":"","sources":["../../../../src/lib/Resumability/backends/PostgresStorageBackend.ts"],"names":[],"mappings":"AAAA,OAAO,EAAmB,MAAM,EAAE,MAAM,EAAU,MAAM,QAAQ,CAAC;AACjE,OAAO,EACL,WAAW,EACX,cAAc,EACf,MAAM,4CAA4C,CAAC;AACpD,OAAO,EACL,gBAAgB,EAChB,UAAU,EACV,cAAc,EACd,mBAAmB,EACpB,MAAM,aAAa,CAAC;AAQrB;;;;;;;;GAQG;AACH,MAAM,WAAW,uBAAuB;IACtC,KAAK,CAAC,CAAC,GAAG,OAAO,EACf,GAAG,EAAE,MAAM,EACX,MAAM,CAAC,EAAE,SAAS,OAAO,EAAE,GAC1B,OAAO,CAAC;QAAE,IAAI,EAAE,SAAS,CAAC,EAAE,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACrD,WAAW,CAAC,CAAC,CAAC,EACZ,QAAQ,EAAE,CAAC,MAAM,EAAE,uBAAuB,KAAK,OAAO,CAAC,CAAC,CAAC,GACxD,OAAO,CAAC,CAAC,CAAC,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC,qCAAqC;IACrC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,yCAAyC;IACzC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,oCAAoC;IACpC,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC5B;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAqCG;AACH,qBAAa,sBAAuB,YAAW,cAAc;IAgBzD,QAAQ,CAAC,EAAE,EAAE,uBAAuB;IAftC,QAAQ,CAAC,YAAY,EAAE,mBAAmB,CAMxC;IAEF,QAAQ,CAAC,IAAI,4BAA4B;IAEzC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAU;gBAGhC,EAAE,EAAE,uBAAuB,EACpC,MAAM,CAAC,EAAE,qBAAqB;IAOhC,UAAU,QAAO,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,gBAAgB,CAAC,CAOpD;IAEF,OAAO,QAAO,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,gBAAgB,CAAC,CAAgB;IAGnE,SAAS,GACP,KAAK,cAAc,EACnB,OAAO,WAAW,KACjB,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,gBAAgB,CAAC,CAyCtC;IAEF,SAAS,GACP,KAAK,cAAc,KAClB,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,EAAE,gBAAgB,CAAC,CAoC5D;IAEF,WAAW,GACT,KAAK,cAAc,KAClB,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,gBAAgB,CAAC,CAmFtC;IAGF,SAAS,GAAI,OAAO,UAAU,KAAG,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,gBAAgB,CAAC,CAoCpE;IAEF,UAAU,GACR,QAAQ,SAAS,UAAU,EAAE,KAC5B,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,gBAAgB,CAAC,CA+DtC;IAEF,UAAU,GACR,KAAK,cAAc,EACnB,qBAAgB,KACf,MAAM,CAAC,MAAM,CAAC,UAAU,EAAE,EAAE,gBAAgB,CAAC,CA6C9C;IAGF,YAAY,GACV,KAAK,cAAc,EACnB,OAAO,WAAW,EAClB,UAAU,MAAM,KACf,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,gBAAgB,CAAC,CA+BtC;IAEF,kBAAkB,GAChB,KAAK,cAAc,KAClB,MAAM,CAAC,MAAM,CACd,MAAM,CAAC,MAAM,CAAC;QAAE,KAAK,EAAE,WAAW,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC,EACvD,gBAAgB,CACjB,CA4CC;IAGF,aAAa,GACX,KAAK,cAAc,EACnB,gBAAgB,MAAM,KACrB,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,gBAAgB,CAAC,CAkBtC;IAEF,YAAY,QAAO,MAAM,CAAC,MAAM,CAAC,cAAc,EAAE,EAAE,gBAAgB,CAAC,CAgClE;IAGF,OAAO,CAAC,YAAY,CAoFlB;IAEF,OAAO,CAAC,YAAY,CAElB;CACH"}

package/dist/lib/Resumability/backends/RedisStorageBackend.d.ts

@@ -0,0 +1,92 @@
+import { Effect, Option } from 'effect';
+import { SpiderState, SpiderStateKey } from '../../Scheduler/SpiderScheduler.service.js';
+import { PersistenceError, StateDelta, StorageBackend, StorageCapabilities } from '../types.js';
+/**
+ * Redis client interface for dependency injection.
+ *
+ * This allows users to provide their own Redis client implementation
+ * (node_redis, ioredis, etc.) without tight coupling.
+ *
+ * @group Backends
+ * @public
+ */
+export interface RedisClientInterface {
+    get(_key: string): Promise<string | null>;
+    set(_key: string, _value: string): Promise<void>;
+    del(_key: string): Promise<void>;
+    exists(_key: string): Promise<boolean>;
+    hget(_key: string, _field: string): Promise<string | null>;
+    hset(_key: string, _field: string, _value: string): Promise<void>;
+    hdel(_key: string, _field: string): Promise<void>;
+    hgetall(_key: string): Promise<Record<string, string>>;
+    zadd(_key: string, _score: number, _member: string): Promise<void>;
+    zrange(_key: string, _start: number, _stop: number): Promise<string[]>;
+    zrangebyscore(_key: string, _min: number | string, _max: number | string): Promise<string[]>;
+    zrem(_key: string, _member: string): Promise<void>;
+    zremrangebyscore(_key: string, _min: number | string, _max: number | string): Promise<void>;
+    keys(_pattern: string): Promise<string[]>;
+    pipeline?(): RedisPipeline;
+    multi?(): RedisMulti;
+}
+/**
+ * Redis pipeline interface for batch operations.
+ */
+export interface RedisPipeline {
+    zadd(_key: string, _score: number, _member: string): RedisPipeline;
+    exec(): Promise<unknown[]>;
+}
+/**
+ * Redis multi/transaction interface.
+ */
+export interface RedisMulti {
+    zadd(_key: string, _score: number, _member: string): RedisMulti;
+    exec(): Promise<unknown[]>;
+}
+/**
+ * Redis storage backend for spider state persistence.
+ *
+ * Uses Redis data structures for efficient storage:
+ * - Hashes for full state and snapshots
+ * - Sorted sets for deltas (ordered by sequence number)
+ * - TTL support for automatic cleanup
+ *
+ * Redis key structure:
+ * ```
+ * spider:state:{sessionId}           # Hash: full state
+ * spider:snapshot:{sessionId}        # Hash: latest snapshot + sequence
+ * spider:deltas:{sessionId}          # Sorted set: sequence -> delta JSON
+ * spider:sessions                    # Set: all session IDs
+ * ```
+ *
+ * @group Backends
+ * @public
+ */
+export declare class RedisStorageBackend implements StorageBackend {
+    readonly capabilities: StorageCapabilities;
+    readonly name = "RedisStorageBackend";
+    private readonly redis;
+    private readonly keyPrefix;
+    constructor(redis: RedisClientInterface, keyPrefix?: string);
+    initialize: () => Effect.Effect<void, PersistenceError>;
+    cleanup: () => Effect.Effect<void, PersistenceError>;
+    saveState: (key: SpiderStateKey, state: SpiderState) => Effect.Effect<void, PersistenceError>;
+    loadState: (key: SpiderStateKey) => Effect.Effect<Option.Option<SpiderState>, PersistenceError>;
+    deleteState: (key: SpiderStateKey) => Effect.Effect<void, PersistenceError>;
+    saveDelta: (delta: StateDelta) => Effect.Effect<void, PersistenceError>;
+    saveDeltas: (deltas: StateDelta[]) => Effect.Effect<void, PersistenceError>;
+    loadDeltas: (key: SpiderStateKey, fromSequence?: number) => Effect.Effect<StateDelta[], PersistenceError>;
+    saveSnapshot: (key: SpiderStateKey, state: SpiderState, sequence: number) => Effect.Effect<void, PersistenceError>;
+    loadLatestSnapshot: (key: SpiderStateKey) => Effect.Effect<Option.Option<{
+        state: SpiderState;
+        sequence: number;
+    }>, PersistenceError>;
+    compactDeltas: (key: SpiderStateKey, beforeSequence: number) => Effect.Effect<void, PersistenceError>;
+    listSessions: () => Effect.Effect<SpiderStateKey[], PersistenceError>;
+    private getStateKey;
+    private getSnapshotKey;
+    private getDeltasKey;
+    private getSessionsKey;
+    private addToSessionsList;
+    private removeFromSessionsList;
+}
+//# sourceMappingURL=RedisStorageBackend.d.ts.map

package/dist/lib/Resumability/backends/RedisStorageBackend.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"RedisStorageBackend.d.ts","sourceRoot":"","sources":["../../../../src/lib/Resumability/backends/RedisStorageBackend.ts"],"names":[],"mappings":"AAAA,OAAO,EAAmB,MAAM,EAAW,MAAM,EAAU,MAAM,QAAQ,CAAC;AAC1E,OAAO,EACL,WAAW,EACX,cAAc,EACf,MAAM,4CAA4C,CAAC;AACpD,OAAO,EACL,gBAAgB,EAChB,UAAU,EACV,cAAc,EACd,mBAAmB,EACpB,MAAM,aAAa,CAAC;AAarB;;;;;;;;GAQG;AACH,MAAM,WAAW,oBAAoB;IACnC,GAAG,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;IAC1C,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACjD,GAAG,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACjC,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IACvC,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;IAC3D,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAClE,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAClD,OAAO,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IACvD,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACnE,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IACvE,aAAa,CACX,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,MAAM,GAAG,MAAM,EACrB,IAAI,EAAE,MAAM,GAAG,MAAM,GACpB,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IACrB,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACnD,gBAAgB,CACd,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,MAAM,GAAG,MAAM,EACrB,IAAI,EAAE,MAAM,GAAG,MAAM,GACpB,OAAO,CAAC,IAAI,CAAC,CAAC;IACjB,IAAI,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IAC1C,QAAQ,CAAC,IAAI,aAAa,CAAC;IAC3B,KAAK,CAAC,IAAI,UAAU,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,aAAa,CAAC;IACnE,IAAI,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;CAC5B;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,UAAU,CAAC;IAChE,IAAI,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;CAC5B;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,qBAAa,mBAAoB,YAAW,cAAc;IACxD,QAAQ,CAAC,YAAY,EAAE,mBAAmB,CAMxC;IAEF,QAAQ,CAAC,IAAI,yBAAyB;IAEtC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAuB;IAC7C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;gBAEvB,KAAK,EAAE,oBAAoB,EAAE,SAAS,SAAW;IAK7D,UAAU,QAAO,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,gBAAgB,CAAC,CACxC;IAEd,OAAO,QAAO,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,gBAAgB,CAAC,CACrC;IAGd,SAAS,GACP,KAAK,cAAc,EACnB,OAAO,WAAW,KACjB,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,gBAAgB,CAAC,CA4BtC;IAEF,SAAS,GACP,KAAK,cAAc,KAClB,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,EAAE,gBAAgB,CAAC,CAgC5D;IAEF,WAAW,GACT,KAAK,cAAc,KAClB,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,gBAAgB,CAAC,CAoCtC;IAGF,SAAS,GAAI,OAAO,UAAU,KAAG,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,gBAAgB,CAAC,CAoCpE;IAEF,UAAU,GACR,QAAQ,UAAU,EAAE,KACnB,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,gBAAgB,CAAC,CAqEtC;IAEF,UAAU,GACR,KAAK,cAAc,EACnB,qBAAgB,KACf,MAAM,CAAC,MAAM,CAAC,UAAU,EAAE,EAAE,gBAAgB,CAAC,CAkC9C;IAGF,YAAY,GACV,KAAK,cAAc,EACnB,OAAO,WAAW,EAClB,UAAU,MAAM,KACf,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,gBAAgB,CAAC,CAmCtC;IAEF,kBAAkB,GAChB,KAAK,cAAc,KAClB,MAAM,CAAC,MAAM,CACd,MAAM,CAAC,MAAM,CAAC;QAAE,KAAK,EAAE,WAAW,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC,EACvD,gBAAgB,CACjB,CAwCC;IAGF,aAAa,GACX,KAAK,cAAc,EACnB,gBAAgB,MAAM,KACrB,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,gBAAgB,CAAC,CAetC;IAEF,YAAY,QAAO,MAAM,CAAC,MAAM,CAAC,cAAc,EAAE,EAAE,gBAAgB,CAAC,CA2ClE;IAGF,OAAO,CAAC,WAAW,CACmB;IAEtC,OAAO,CAAC,cAAc,CACmB;IAEzC,OAAO,CAAC,YAAY,CACmB;IAEvC,OAAO,CAAC,cAAc,CAA8C;IAEpE,OAAO,CAAC,iBAAiB,CAkBvB;IAEF,OAAO,CAAC,sBAAsB,CAgB5B;CACH"}

package/dist/lib/Resumability/index.d.ts

@@ -0,0 +1,51 @@
+/**
+ * Resumable spider crawling with configurable persistence strategies.
+ *
+ * This module provides a complete solution for resumable web crawling with
+ * support for different persistence strategies and storage backends.
+ *
+ * ## Key Features
+ *
+ * - **Multiple Strategies**: Full state, delta, hybrid, and auto-selection
+ * - **Multiple Backends**: File system, Redis, PostgreSQL with extensible interface
+ * - **Effect Native**: Full integration with Effect ecosystem
+ * - **Type Safe**: Complete TypeScript support with runtime validation
+ * - **Production Ready**: Handles concurrency, errors, and edge cases
+ *
+ * ## Quick Start
+ *
+ * ```typescript
+ * import { ResumabilityService, ResumabilityConfigs } from '@jambudipa/spider/resumability';
+ *
+ * // File-based resumability
+ * const resumabilityLayer = ResumabilityService.fromConfig(
+ *   ResumabilityConfigs.file('./spider-state', 'hybrid')
+ * );
+ *
+ * // Use with Spider
+ * const program = Effect.gen(function* () {
+ *   const spider = yield* Spider;
+ *   const resumability = yield* ResumabilityService;
+ *
+ *   // Configure resumable crawling...
+ * });
+ *
+ * Effect.runPromise(
+ *   program.pipe(
+ *     Effect.provide(Spider.Default),
+ *     Effect.provide(resumabilityLayer)
+ *   )
+ * );
+ * ```
+ *
+ * @group Resumability
+ * @public
+ */
+export type { StorageBackend, StorageCapabilities, PersistenceStrategy, StateOperation, HybridPersistenceConfig, } from './types.js';
+export { StateDelta, PersistenceError, DEFAULT_HYBRID_CONFIG, } from './types.js';
+export { FullStatePersistence, DeltaPersistence, HybridPersistence, } from './strategies.js';
+export { FileStorageBackend } from './backends/FileStorageBackend.js';
+export { RedisStorageBackend, type RedisClientInterface, type RedisPipeline, type RedisMulti, } from './backends/RedisStorageBackend.js';
+export { PostgresStorageBackend, type DatabaseClientInterface, type PostgresStorageConfig, } from './backends/PostgresStorageBackend.js';
+export { ResumabilityService, ResumabilityConfigs, createStateOperation, type ResumabilityConfig, } from './Resumability.service.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/lib/Resumability/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/lib/Resumability/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA0CG;AAGH,YAAY,EACV,cAAc,EACd,mBAAmB,EACnB,mBAAmB,EACnB,cAAc,EACd,uBAAuB,GACxB,MAAM,YAAY,CAAC;AAEpB,OAAO,EACL,UAAU,EACV,gBAAgB,EAChB,qBAAqB,GACtB,MAAM,YAAY,CAAC;AAGpB,OAAO,EACL,oBAAoB,EACpB,gBAAgB,EAChB,iBAAiB,GAClB,MAAM,iBAAiB,CAAC;AAGzB,OAAO,EAAE,kBAAkB,EAAE,MAAM,kCAAkC,CAAC;AACtE,OAAO,EACL,mBAAmB,EACnB,KAAK,oBAAoB,EACzB,KAAK,aAAa,EAClB,KAAK,UAAU,GAChB,MAAM,mCAAmC,CAAC;AAC3C,OAAO,EACL,sBAAsB,EACtB,KAAK,uBAAuB,EAC5B,KAAK,qBAAqB,GAC3B,MAAM,sCAAsC,CAAC;AAG9C,OAAO,EACL,mBAAmB,EACnB,mBAAmB,EACnB,oBAAoB,EACpB,KAAK,kBAAkB,GACxB,MAAM,2BAA2B,CAAC"}