@jambudipa/spider 0.1.0

package/dist/index.js

@@ -0,0 +1,4681 @@
+import { Effect, Layer, MutableHashSet, Schema, Data, Context, Console, MutableHashMap, Option, Queue, PubSub, MutableRef, Schedule, Stream, Fiber, Random, Ref } from "effect";
+import * as cheerio from "cheerio";
+import * as fs from "fs";
+import * as path from "path";
+import * as fs$1 from "fs/promises";
+import { CookieJar } from "tough-cookie";
+class SpiderConfig extends Effect.Service()(
+  "@jambudipa.io/SpiderConfig",
+  {
+    effect: Effect.sync(() => makeSpiderConfig({}))
+  }
+) {
+  /**
+   * Creates a Layer that provides SpiderConfig with custom options
+   * @param config - The configuration options or a pre-made SpiderConfigService
+   */
+  static Live = (config) => Layer.effect(
+    SpiderConfig,
+    Effect.succeed("getOptions" in config ? config : makeSpiderConfig(config))
+  );
+}
+const FILE_EXTENSION_CATEGORIES = {
+  /** Archive files (8 extensions) */
+  archives: [".7z", ".7zip", ".bz2", ".rar", ".tar", ".tar.gz", ".xz", ".zip"],
+  /** Image files (19 extensions) */
+  images: [
+    ".mng",
+    ".pct",
+    ".bmp",
+    ".gif",
+    ".jpg",
+    ".jpeg",
+    ".png",
+    ".pst",
+    ".psp",
+    ".tif",
+    ".tiff",
+    ".ai",
+    ".drw",
+    ".dxf",
+    ".eps",
+    ".ps",
+    ".svg",
+    ".cdr",
+    ".ico",
+    ".webp"
+  ],
+  /** Audio files (9 extensions) */
+  audio: [
+    ".mp3",
+    ".wma",
+    ".ogg",
+    ".wav",
+    ".ra",
+    ".aac",
+    ".mid",
+    ".au",
+    ".aiff"
+  ],
+  /** Video files (14 extensions) */
+  video: [
+    ".3gp",
+    ".asf",
+    ".asx",
+    ".avi",
+    ".mov",
+    ".mp4",
+    ".mpg",
+    ".qt",
+    ".rm",
+    ".swf",
+    ".wmv",
+    ".m4a",
+    ".m4v",
+    ".flv",
+    ".webm"
+  ],
+  /** Office documents (21 extensions) */
+  officeDocuments: [
+    ".xls",
+    ".xlsm",
+    ".xlsx",
+    ".xltm",
+    ".xltx",
+    ".potm",
+    ".potx",
+    ".ppt",
+    ".pptm",
+    ".pptx",
+    ".pps",
+    ".doc",
+    ".docb",
+    ".docm",
+    ".docx",
+    ".dotm",
+    ".dotx",
+    ".odt",
+    ".ods",
+    ".odg",
+    ".odp"
+  ],
+  /** Other files (16 extensions) */
+  other: [
+    ".css",
+    ".pdf",
+    ".exe",
+    ".bin",
+    ".rss",
+    ".dmg",
+    ".iso",
+    ".apk",
+    ".jar",
+    ".sh",
+    ".rb",
+    ".js",
+    ".hta",
+    ".bat",
+    ".cpl",
+    ".msi",
+    ".msp",
+    ".py"
+  ]
+};
+const generateSkipExtensions = (filters) => {
+  const skipExtensions = [];
+  if (filters.filterArchives) {
+    skipExtensions.push(...FILE_EXTENSION_CATEGORIES.archives);
+  }
+  if (filters.filterImages) {
+    skipExtensions.push(...FILE_EXTENSION_CATEGORIES.images);
+  }
+  if (filters.filterAudio) {
+    skipExtensions.push(...FILE_EXTENSION_CATEGORIES.audio);
+  }
+  if (filters.filterVideo) {
+    skipExtensions.push(...FILE_EXTENSION_CATEGORIES.video);
+  }
+  if (filters.filterOfficeDocuments) {
+    skipExtensions.push(...FILE_EXTENSION_CATEGORIES.officeDocuments);
+  }
+  if (filters.filterOther) {
+    skipExtensions.push(...FILE_EXTENSION_CATEGORIES.other);
+  }
+  return skipExtensions;
+};
+const makeSpiderConfig = (options = {}) => {
+  const defaultFileExtensionFilters = {
+    filterArchives: true,
+    filterImages: true,
+    filterAudio: true,
+    filterVideo: true,
+    filterOfficeDocuments: true,
+    filterOther: true
+  };
+  const defaultTechnicalFilters = {
+    filterUnsupportedSchemes: true,
+    filterLongUrls: true,
+    maxUrlLength: 2083,
+    // Scrapy's default
+    filterMalformedUrls: true
+  };
+  const defaultOptions = {
+    ignoreRobotsTxt: false,
+    maxConcurrentWorkers: 5,
+    concurrency: 4,
+    requestDelayMs: 1e3,
+    maxRobotsCrawlDelayMs: 2e3,
+    // Maximum 1 second for robots.txt crawl delay
+    userAgent: "JambudipaSpider/1.0",
+    allowedProtocols: ["http:", "https:", "file:", "ftp:"],
+    // Scrapy's allowed schemes
+    followRedirects: true,
+    respectNoFollow: true,
+    fileExtensionFilters: defaultFileExtensionFilters,
+    technicalFilters: defaultTechnicalFilters,
+    maxConcurrentRequests: 10,
+    maxRequestsPerSecondPerDomain: 2,
+    normalizeUrlsForDeduplication: true,
+    enableResumability: false
+  };
+  const config = {
+    ...defaultOptions,
+    ...options,
+    // Merge nested objects properly
+    fileExtensionFilters: options.fileExtensionFilters ? {
+      ...defaultOptions.fileExtensionFilters,
+      ...options.fileExtensionFilters
+    } : defaultOptions.fileExtensionFilters,
+    technicalFilters: options.technicalFilters ? {
+      ...defaultOptions.technicalFilters,
+      ...options.technicalFilters
+    } : defaultOptions.technicalFilters
+  };
+  const skipExtensions = config.skipFileExtensions || generateSkipExtensions(
+    config.fileExtensionFilters ?? defaultFileExtensionFilters
+  );
+  return {
+    getOptions: () => Effect.succeed(config),
+    shouldFollowUrl: (urlString, fromUrl, restrictToStartingDomain) => Effect.sync(() => {
+      try {
+        const url = new URL(urlString);
+        const fromUrlParsed = fromUrl ? new URL(fromUrl) : void 0;
+        const techFilters = config.technicalFilters ?? defaultTechnicalFilters;
+        if (restrictToStartingDomain) {
+          const startingDomain = new URL(restrictToStartingDomain).hostname;
+          const isAllowedDomain = url.hostname === startingDomain || url.hostname.endsWith(`.${startingDomain}`);
+          if (!isAllowedDomain) {
+            return {
+              follow: false,
+              reason: `Domain ${url.hostname} restricted to starting domain ${startingDomain}`
+            };
+          }
+        }
+        if (techFilters.filterLongUrls && urlString.length > techFilters.maxUrlLength) {
+          return {
+            follow: false,
+            reason: `URL length ${urlString.length} exceeds maximum ${techFilters.maxUrlLength}`
+          };
+        }
+        if (techFilters.filterUnsupportedSchemes && !config.allowedProtocols.includes(url.protocol)) {
+          return {
+            follow: false,
+            reason: `Protocol ${url.protocol} not in allowed schemes: ${config.allowedProtocols.join(", ")}`
+          };
+        }
+        if (config.allowedDomains && config.allowedDomains.length > 0) {
+          const isDomainAllowed = config.allowedDomains.some(
+            (domain) => url.hostname === domain || url.hostname.endsWith(`.${domain}`)
+          );
+          if (!isDomainAllowed) {
+            return {
+              follow: false,
+              reason: `Domain ${url.hostname} not in allowlist`
+            };
+          }
+        }
+        if (config.blockedDomains && config.blockedDomains.length > 0) {
+          const isDomainBlocked = config.blockedDomains.some(
+            (domain) => url.hostname === domain || url.hostname.endsWith(`.${domain}`)
+          );
+          if (isDomainBlocked) {
+            return {
+              follow: false,
+              reason: `Domain ${url.hostname} is blocked`
+            };
+          }
+        }
+        if (config.customUrlFilters && config.customUrlFilters.length > 0) {
+          for (const pattern of config.customUrlFilters) {
+            if (pattern.test(urlString)) {
+              return {
+                follow: false,
+                reason: `URL matches custom filter pattern: ${pattern}`
+              };
+            }
+          }
+        }
+        if (fromUrlParsed && url.hostname === fromUrlParsed.hostname && url.pathname === fromUrlParsed.pathname && url.search === fromUrlParsed.search && url.hash) {
+          return {
+            follow: false,
+            reason: "Fragment-only link to same page"
+          };
+        }
+        const pathname = url.pathname.toLowerCase();
+        if (skipExtensions.some((ext) => pathname.endsWith(ext.toLowerCase()))) {
+          const filterReasons = [];
+          if (config.fileExtensionFilters?.filterArchives && FILE_EXTENSION_CATEGORIES.archives.some(
+            (ext) => pathname.endsWith(ext.toLowerCase())
+          )) {
+            filterReasons.push("archive");
+          }
+          if (config.fileExtensionFilters?.filterImages && FILE_EXTENSION_CATEGORIES.images.some(
+            (ext) => pathname.endsWith(ext.toLowerCase())
+          )) {
+            filterReasons.push("image");
+          }
+          if (config.fileExtensionFilters?.filterAudio && FILE_EXTENSION_CATEGORIES.audio.some(
+            (ext) => pathname.endsWith(ext.toLowerCase())
+          )) {
+            filterReasons.push("audio");
+          }
+          if (config.fileExtensionFilters?.filterVideo && FILE_EXTENSION_CATEGORIES.video.some(
+            (ext) => pathname.endsWith(ext.toLowerCase())
+          )) {
+            filterReasons.push("video");
+          }
+          if (config.fileExtensionFilters?.filterOfficeDocuments && FILE_EXTENSION_CATEGORIES.officeDocuments.some(
+            (ext) => pathname.endsWith(ext.toLowerCase())
+          )) {
+            filterReasons.push("office document");
+          }
+          if (config.fileExtensionFilters?.filterOther && FILE_EXTENSION_CATEGORIES.other.some(
+            (ext) => pathname.endsWith(ext.toLowerCase())
+          )) {
+            filterReasons.push("other file type");
+          }
+          const reason = filterReasons.length > 0 ? `Filtered ${filterReasons.join("/")} file extension` : "File extension not suitable for crawling";
+          return {
+            follow: false,
+            reason
+          };
+        }
+        return { follow: true };
+      } catch (error) {
+        if (config.technicalFilters?.filterMalformedUrls) {
+          return {
+            follow: false,
+            reason: `Malformed URL: ${error instanceof Error ? error.message : "Unknown parsing error"}`
+          };
+        } else {
+          return { follow: true };
+        }
+      }
+    }),
+    getUserAgent: () => Effect.succeed(config.userAgent),
+    getRequestDelay: () => Effect.succeed(config.requestDelayMs),
+    getMaxRobotsCrawlDelay: () => Effect.succeed(config.maxRobotsCrawlDelayMs),
+    shouldIgnoreRobotsTxt: () => Effect.succeed(config.ignoreRobotsTxt),
+    getMaxConcurrentWorkers: () => Effect.succeed(config.maxConcurrentWorkers),
+    getMaxDepth: () => Effect.succeed(config.maxDepth),
+    getMaxPages: () => Effect.succeed(config.maxPages),
+    shouldFollowRedirects: () => Effect.succeed(config.followRedirects),
+    shouldRespectNoFollow: () => Effect.succeed(config.respectNoFollow),
+    getSkipFileExtensions: () => Effect.succeed(config.skipFileExtensions || []),
+    getMaxConcurrentRequests: () => Effect.succeed(config.maxConcurrentRequests),
+    getMaxRequestsPerSecondPerDomain: () => Effect.succeed(config.maxRequestsPerSecondPerDomain),
+    shouldNormalizeUrlsForDeduplication: () => Effect.succeed(config.normalizeUrlsForDeduplication),
+    getConcurrency: () => Effect.succeed(config.concurrency),
+    isResumabilityEnabled: () => Effect.succeed(config.enableResumability)
+  };
+};
+class UrlDeduplicatorService extends Effect.Service()(
+  "@jambudipa.io/UrlDeduplicatorService",
+  {
+    effect: Effect.gen(function* () {
+      const config = yield* SpiderConfig;
+      const shouldNormalize = yield* config.shouldNormalizeUrlsForDeduplication();
+      const seenUrls = MutableHashSet.empty();
+      const mutex = yield* Effect.makeSemaphore(1);
+      const normalizeUrl = (url) => {
+        if (!shouldNormalize) {
+          return url;
+        }
+        try {
+          const parsed = new URL(url);
+          let normalizedPath = parsed.pathname.replace(/\/+/g, "/").replace(/\/$/, "");
+          if (normalizedPath === "") {
+            normalizedPath = "/";
+          }
+          parsed.pathname = normalizedPath;
+          parsed.hash = "";
+          if (parsed.protocol === "http:" && parsed.port === "80" || parsed.protocol === "https:" && parsed.port === "443") {
+            parsed.port = "";
+          }
+          if (parsed.search) {
+            const params = new URLSearchParams(parsed.search);
+            const sortedParams = new URLSearchParams();
+            Array.from(params.keys()).sort().forEach((key) => {
+              params.getAll(key).forEach((value) => {
+                sortedParams.append(key, value);
+              });
+            });
+            parsed.search = sortedParams.toString();
+          }
+          return parsed.toString();
+        } catch {
+          return url;
+        }
+      };
+      return {
+        tryAdd: (url) => mutex.withPermits(1)(
+          Effect.sync(() => {
+            const normalizedUrl = normalizeUrl(url);
+            if (MutableHashSet.has(seenUrls, normalizedUrl)) {
+              return false;
+            }
+            MutableHashSet.add(seenUrls, normalizedUrl);
+            return true;
+          })
+        ),
+        contains: (url) => mutex.withPermits(1)(
+          Effect.sync(() => {
+            const normalizedUrl = normalizeUrl(url);
+            return MutableHashSet.has(seenUrls, normalizedUrl);
+          })
+        ),
+        size: () => mutex.withPermits(1)(
+          Effect.sync(() => MutableHashSet.size(seenUrls))
+        ),
+        clear: () => mutex.withPermits(1)(
+          Effect.sync(() => MutableHashSet.clear(seenUrls))
+        )
+      };
+    }),
+    dependencies: [SpiderConfig.Default]
+  }
+) {
+}
+const PageDataSchema = Schema.Struct({
+  url: Schema.String.pipe(
+    Schema.filter(
+      (s) => {
+        try {
+          new URL(s);
+          return true;
+        } catch {
+          return false;
+        }
+      },
+      {
+        message: () => "Invalid URL format"
+      }
+    )
+  ),
+  html: Schema.String,
+  title: Schema.optional(Schema.String),
+  /** All available metadata from meta tags */
+  metadata: Schema.Record({ key: Schema.String, value: Schema.String }),
+  /** Commonly used metadata fields for convenience */
+  commonMetadata: Schema.optional(
+    Schema.Struct({
+      description: Schema.optional(Schema.String),
+      keywords: Schema.optional(Schema.String),
+      author: Schema.optional(Schema.String),
+      robots: Schema.optional(Schema.String)
+    })
+  ),
+  statusCode: Schema.Number.pipe(Schema.int(), Schema.between(100, 599)),
+  /** All response headers */
+  headers: Schema.Record({ key: Schema.String, value: Schema.String }),
+  /** When the fetch operation started */
+  fetchedAt: Schema.DateFromSelf,
+  /** How long the entire fetch and parse operation took in milliseconds */
+  scrapeDurationMs: Schema.Number,
+  /** The crawl depth (number of hops from the starting URL) */
+  depth: Schema.Number.pipe(Schema.int(), Schema.greaterThanOrEqualTo(0)),
+  /** Optional extracted data from the page */
+  extractedData: Schema.optional(
+    Schema.Record({ key: Schema.String, value: Schema.Unknown })
+  )
+});
+class NetworkError extends Data.TaggedError("NetworkError") {
+  static fromCause(url, cause) {
+    return new NetworkError({
+      url,
+      cause,
+      message: `Failed to fetch ${url}: ${cause}`
+    });
+  }
+}
+class ResponseError extends Data.TaggedError("ResponseError") {
+  static fromCause(url, cause) {
+    return new ResponseError({
+      url,
+      cause,
+      message: `Failed to read response from ${url}: ${cause}`
+    });
+  }
+}
+class RobotsTxtError extends Data.TaggedError("RobotsTxtError") {
+  static fromCause(url, cause) {
+    return new RobotsTxtError({
+      url,
+      cause,
+      message: `Failed to fetch robots.txt: ${cause}`
+    });
+  }
+}
+class ConfigurationError extends Data.TaggedError("ConfigurationError") {
+}
+class MiddlewareError extends Data.TaggedError("MiddlewareError") {
+  static transform(middlewareName, cause) {
+    return new MiddlewareError({
+      phase: "transform",
+      middlewareName,
+      cause,
+      message: `Middleware '${middlewareName}' failed during transform: ${cause}`
+    });
+  }
+  static error(middlewareName, cause) {
+    return new MiddlewareError({
+      phase: "error",
+      middlewareName,
+      cause,
+      message: `Middleware '${middlewareName}' failed during error handling: ${cause}`
+    });
+  }
+}
+class FileSystemError extends Data.TaggedError("FileSystemError") {
+  static write(path2, cause) {
+    return new FileSystemError({
+      operation: "write",
+      path: path2,
+      cause,
+      message: `Failed to write file ${path2}: ${cause}`
+    });
+  }
+  static create(path2, cause) {
+    return new FileSystemError({
+      operation: "create",
+      path: path2,
+      cause,
+      message: `Failed to create directory ${path2}: ${cause}`
+    });
+  }
+}
+let PersistenceError$1 = class PersistenceError extends Data.TaggedError("PersistenceError") {
+  static save(cause, key) {
+    return new PersistenceError({
+      operation: "save",
+      key,
+      cause,
+      message: key ? `Failed to save state for key ${key}: ${cause}` : `Failed to save state: ${cause}`
+    });
+  }
+  static load(cause, key) {
+    return new PersistenceError({
+      operation: "load",
+      key,
+      cause,
+      message: key ? `Failed to load state for key ${key}: ${cause}` : `Failed to load state: ${cause}`
+    });
+  }
+  static delete(cause, key) {
+    return new PersistenceError({
+      operation: "delete",
+      key,
+      cause,
+      message: key ? `Failed to delete state for key ${key}: ${cause}` : `Failed to delete state: ${cause}`
+    });
+  }
+};
+const SpiderLogger = Context.GenericTag("SpiderLogger");
+const makeSpiderLogger = (logDir = "./spider-logs") => {
+  if (!fs.existsSync(logDir)) {
+    fs.mkdirSync(logDir, { recursive: true });
+  }
+  const logFileName = `spider-${(/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-")}.jsonl`;
+  const logFilePath = path.join(logDir, logFileName);
+  const summaryFilePath = path.join(logDir, "spider-summary.json");
+  const writeLogEvent = (event) => Effect.sync(() => {
+    const logLine = JSON.stringify(event) + "\n";
+    fs.appendFileSync(logFilePath, logLine);
+    const importantTypes = [
+      "domain_start",
+      "domain_complete",
+      "spider_lifecycle",
+      "domain_error"
+    ];
+    if (importantTypes.includes(event.type)) {
+      const prefix = `[${event.type}]`;
+      const domainInfo = event.domain ? ` [${event.domain}]` : "";
+      Console.log(`${prefix}${domainInfo} ${event.message}`).pipe(
+        Effect.runSync
+      );
+    }
+  });
+  const updateSummary = (update) => Effect.sync(() => {
+    let summary = {};
+    if (fs.existsSync(summaryFilePath)) {
+      const content = fs.readFileSync(summaryFilePath, "utf-8");
+      try {
+        const parsed = JSON.parse(content);
+        summary = typeof parsed === "object" && parsed !== null ? parsed : {};
+      } catch {
+        summary = {};
+      }
+    }
+    summary = update(summary);
+    fs.writeFileSync(summaryFilePath, JSON.stringify(summary, null, 2));
+  });
+  return {
+    logEvent: (event) => Effect.gen(function* () {
+      const fullEvent = {
+        ...event,
+        timestamp: (/* @__PURE__ */ new Date()).toISOString()
+      };
+      yield* writeLogEvent(fullEvent);
+    }),
+    logDomainStart: (domain, startUrl) => Effect.gen(function* () {
+      yield* writeLogEvent({
+        timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+        type: "domain_start",
+        domain,
+        url: startUrl,
+        message: `Starting crawl for domain: ${domain}`,
+        details: { startUrl }
+      });
+      yield* updateSummary((summary) => ({
+        ...summary,
+        domains: {
+          ...summary.domains || {},
+          [domain]: {
+            status: "running",
+            startTime: (/* @__PURE__ */ new Date()).toISOString(),
+            startUrl,
+            pagesScraped: 0
+          }
+        }
+      }));
+    }),
+    logDomainComplete: (domain, pagesScraped, reason) => Effect.gen(function* () {
+      yield* writeLogEvent({
+        timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+        type: "domain_complete",
+        domain,
+        message: `Domain ${domain} completed: ${pagesScraped} pages scraped (reason: ${reason})`,
+        details: { pagesScraped, reason }
+      });
+      yield* updateSummary((summary) => {
+        const domains = summary.domains || {};
+        const existingDomain = domains[domain] || {};
+        return {
+          ...summary,
+          domains: {
+            ...domains,
+            [domain]: {
+              ...existingDomain,
+              status: "completed",
+              endTime: (/* @__PURE__ */ new Date()).toISOString(),
+              pagesScraped,
+              completionReason: reason
+            }
+          }
+        };
+      });
+    }),
+    logPageScraped: (url, domain, pageNumber) => Effect.gen(function* () {
+      yield* writeLogEvent({
+        timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+        type: "page_scraped",
+        domain,
+        url,
+        message: `Scraped page #${pageNumber} from ${domain}`,
+        details: { pageNumber }
+      });
+      yield* updateSummary((summary) => {
+        const domains = summary.domains || {};
+        const existingDomain = domains[domain] || {};
+        return {
+          ...summary,
+          domains: {
+            ...domains,
+            [domain]: {
+              ...existingDomain,
+              pagesScraped: pageNumber
+            }
+          }
+        };
+      });
+    }),
+    logQueueStatus: (domain, queueSize, activeWorkers) => Effect.gen(function* () {
+      yield* writeLogEvent({
+        timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+        type: "queue_status",
+        domain,
+        message: `Queue status - size: ${queueSize}, active workers: ${activeWorkers}`,
+        details: { queueSize, activeWorkers }
+      });
+    }),
+    logRateLimit: (domain, requestsInWindow) => Effect.gen(function* () {
+      yield* writeLogEvent({
+        timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+        type: "rate_limit",
+        domain,
+        message: `Rate limit applied - ${requestsInWindow} requests in window`,
+        details: { requestsInWindow }
+      });
+    }),
+    logSpiderLifecycle: (event, details) => Effect.gen(function* () {
+      yield* writeLogEvent({
+        timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+        type: "spider_lifecycle",
+        message: `Spider ${event}`,
+        details
+      });
+      if (event === "start") {
+        yield* updateSummary((summary) => ({
+          ...summary,
+          spiderStartTime: (/* @__PURE__ */ new Date()).toISOString(),
+          status: "running"
+        }));
+      } else if (event === "complete" || event === "error") {
+        yield* updateSummary((summary) => ({
+          ...summary,
+          spiderEndTime: (/* @__PURE__ */ new Date()).toISOString(),
+          status: event === "complete" ? "completed" : "error",
+          ...details && { finalDetails: details }
+        }));
+      }
+    }),
+    // Enhanced diagnostic logging methods
+    logWorkerLifecycle: (workerId, domain, event, reason, details) => Effect.gen(function* () {
+      yield* writeLogEvent({
+        timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+        type: "worker_lifecycle",
+        domain,
+        workerId,
+        message: `[WORKER_LIFECYCLE] Worker ${workerId} ${event}${reason ? ` - reason: ${reason}` : ""} (domain: ${domain})`,
+        details: { event, reason, ...details }
+      });
+    }),
+    logWorkerState: (workerId, domain, event, details) => Effect.gen(function* () {
+      yield* writeLogEvent({
+        timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+        type: "worker_state",
+        domain,
+        workerId,
+        message: `[WORKER_STATE] Worker ${workerId} ${event} (domain: ${domain})`,
+        details: { event, ...details }
+      });
+    }),
+    logCompletionMonitor: (domain, checkCount, queueSize, activeWorkers, stableCount, maxPagesReached, decision) => Effect.gen(function* () {
+      yield* writeLogEvent({
+        timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+        type: "completion_monitor",
+        domain,
+        message: `[COMPLETION_MONITOR] Check #${checkCount}: queue=${queueSize}, active=${activeWorkers}, stable=${stableCount}, maxPages=${maxPagesReached} -> ${decision}`,
+        details: {
+          checkCount,
+          queueSize,
+          activeWorkers,
+          stableCount,
+          maxPagesReached,
+          decision
+        }
+      });
+    }),
+    logEdgeCase: (domain, caseType, details) => Effect.gen(function* () {
+      yield* writeLogEvent({
+        timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+        type: "edge_case",
+        domain,
+        message: `[EDGE_CASE] ${caseType} (domain: ${domain})`,
+        details: { case: caseType, ...details }
+      });
+    }),
+    logDomainStatus: (domain, status) => Effect.gen(function* () {
+      yield* writeLogEvent({
+        timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+        type: "domain_start",
+        // Reuse existing type for now
+        domain,
+        message: `[DOMAIN_STATUS] ${domain}: ${status.pagesScraped} pages, queue=${status.queueSize}, workers=${status.activeWorkers}/${status.maxWorkers}`,
+        details: status
+      });
+      yield* updateSummary((summary) => {
+        const domains = summary.domains || {};
+        const existingDomain = domains[domain] || {};
+        return {
+          ...summary,
+          domains: {
+            ...domains,
+            [domain]: {
+              ...existingDomain,
+              pagesScraped: Math.max(0, status.pagesScraped || 0),
+              queueSize: Math.max(0, status.queueSize || 0),
+              activeWorkers: Math.max(0, status.activeWorkers || 0),
+              maxWorkers: Math.max(1, status.maxWorkers || 5)
+            }
+          }
+        };
+      });
+    })
+  };
+};
+const SpiderLoggerLive = Layer.succeed(SpiderLogger, makeSpiderLogger());
+class ScraperService extends Effect.Service()(
+  "@jambudipa.io/ScraperService",
+  {
+    effect: Effect.sync(() => ({
+      /**
+       * Fetches a URL and parses the HTML to extract basic page information.
+       * 
+       * This method performs the following operations:
+       * 1. Fetches the URL with configurable timeout (30 seconds)
+       * 2. Validates content type (skips binary files)
+       * 3. Parses HTML content with cheerio
+       * 4. Extracts basic page metadata (title, description, etc.)
+       * 5. Returns structured PageData object
+       * 
+       * The method uses AbortController for proper timeout handling to prevent
+       * workers from hanging on malformed URLs or slow responses.
+       * 
+       * @param url - The URL to fetch and parse
+       * @param depth - The crawl depth for logging purposes (default: 0)
+       * @returns Effect containing PageData with extracted information
+       * @throws NetworkError for network-related failures
+       * @throws ResponseError for HTTP error responses
+       * 
+       * @example
+       * Basic usage:
+       * ```typescript
+       * const pageData = yield* scraper.fetchAndParse('https://example.com');
+       * console.log(`Page title: ${pageData.title}`);
+       * ```
+       * 
+       * With depth tracking:
+       * ```typescript
+       * const pageData = yield* scraper.fetchAndParse('https://example.com/page', 2);
+       * ```
+       * 
+       * Error handling:
+       * ```typescript
+       * const result = yield* scraper.fetchAndParse('https://example.com').pipe(
+       *   Effect.catchTags({
+       *     NetworkError: (error) => {
+       *       console.log('Network error:', error.message);
+       *       return Effect.succeed(null);
+       *     },
+       *     ResponseError: (error) => {
+       *       console.log('HTTP error:', error.statusCode);
+       *       return Effect.succeed(null);
+       *     }
+       *   })
+       * );
+       * ```
+       * 
+       * @performance 
+       * - Request timeout: 30 seconds
+       * - Response parsing timeout: 10 seconds
+       * - Memory usage: ~2-5MB per page depending on content size
+       * 
+       * @security
+       * - Validates content types to prevent processing binary files
+       * - Uses AbortController to prevent hanging requests
+       * - No execution of JavaScript content (static HTML parsing only)
+       */
+      fetchAndParse: (url, depth = 0) => Effect.gen(function* () {
+        const startTime = yield* Effect.sync(() => /* @__PURE__ */ new Date());
+        const startMs = startTime.getTime();
+        const logger = yield* SpiderLogger;
+        const domain = new URL(url).hostname;
+        const controller = new AbortController();
+        const timeoutMs = 3e4;
+        const timeoutId = setTimeout(() => {
+          const duration = Date.now() - startMs;
+          Effect.runSync(
+            logger.logEdgeCase(domain, "fetch_abort_triggered", {
+              url,
+              durationMs: duration,
+              reason: "timeout",
+              timeoutMs
+            })
+          );
+          controller.abort();
+        }, timeoutMs);
+        const response = yield* Effect.tryPromise({
+          try: async () => {
+            try {
+              const resp = await fetch(url, { signal: controller.signal });
+              clearTimeout(timeoutId);
+              const contentType = resp.headers.get("content-type") || "";
+              if (!contentType.includes("text/html") && !contentType.includes("application/xhtml") && !contentType.includes("text/") && contentType !== "") {
+                throw new Error(`Skipping non-HTML content: ${contentType}`);
+              }
+              return resp;
+            } catch (error) {
+              clearTimeout(timeoutId);
+              if (error instanceof Error && error.name === "AbortError") {
+                throw new Error(
+                  `Request aborted after ${Date.now() - startMs}ms`
+                );
+              }
+              throw error;
+            }
+          },
+          catch: (error) => NetworkError.fromCause(url, error)
+        });
+        const textController = new AbortController();
+        const textTimeoutMs = 1e4;
+        const textTimeoutId = setTimeout(() => {
+          const duration = Date.now() - startMs;
+          Effect.runSync(
+            logger.logEdgeCase(domain, "response_text_abort_triggered", {
+              url,
+              durationMs: duration,
+              reason: "timeout",
+              timeoutMs: textTimeoutMs
+            })
+          );
+          textController.abort();
+        }, textTimeoutMs);
+        const html = yield* Effect.tryPromise({
+          try: async () => {
+            try {
+              const reader = response.body?.getReader();
+              if (!reader) throw new Error("No response body");
+              const decoder = new TextDecoder();
+              let html2 = "";
+              while (true) {
+                const { done, value } = await reader.read();
+                if (done) break;
+                html2 += decoder.decode(value, { stream: true });
+                if (textController.signal.aborted) {
+                  reader.cancel();
+                  throw new Error("Response parsing aborted");
+                }
+              }
+              clearTimeout(textTimeoutId);
+              return html2;
+            } catch (error) {
+              clearTimeout(textTimeoutId);
+              throw error;
+            }
+          },
+          catch: (error) => ResponseError.fromCause(url, error)
+        });
+        const $ = cheerio.load(html);
+        const metadata = {};
+        $("meta").each((_, element) => {
+          const $meta = $(element);
+          const name = $meta.attr("name") || $meta.attr("property") || $meta.attr("http-equiv");
+          const content = $meta.attr("content");
+          if (name && content) {
+            metadata[name] = content;
+          }
+        });
+        const commonMetadata = {
+          description: metadata["description"],
+          keywords: metadata["keywords"],
+          author: metadata["author"],
+          robots: metadata["robots"]
+        };
+        const headers = {};
+        response.headers.forEach((value, key) => {
+          headers[key] = value;
+        });
+        const endTime = yield* Effect.sync(() => /* @__PURE__ */ new Date());
+        const durationMs = endTime.getTime() - startTime.getTime();
+        const pageData = {
+          url,
+          html,
+          title: $("title").text() || void 0,
+          metadata,
+          commonMetadata: Object.values(commonMetadata).some((v) => v) ? commonMetadata : void 0,
+          statusCode: response.status,
+          headers,
+          fetchedAt: startTime,
+          scrapeDurationMs: durationMs,
+          depth
+        };
+        return yield* Schema.decode(PageDataSchema)(pageData);
+      })
+    }))
+  }
+) {
+}
+class RobotsService extends Effect.Service()(
+  "@jambudipa.io/RobotsService",
+  {
+    effect: Effect.sync(() => {
+      const robotsCache = MutableHashMap.empty();
+      const parseRobotsTxt = (content, userAgent = "*") => {
+        const lines = content.split("\n");
+        const rules = {
+          disallowedPaths: /* @__PURE__ */ new Set(),
+          userAgent
+        };
+        let currentUserAgent = "";
+        let isRelevantSection = false;
+        for (const line of lines) {
+          const trimmed = line.trim();
+          if (trimmed.startsWith("#") || !trimmed) continue;
+          const [directive, ...valueParts] = trimmed.split(":");
+          const value = valueParts.join(":").trim();
+          if (directive.toLowerCase() === "user-agent") {
+            currentUserAgent = value;
+            isRelevantSection = currentUserAgent === "*" || currentUserAgent.toLowerCase() === userAgent.toLowerCase();
+          } else if (isRelevantSection) {
+            if (directive.toLowerCase() === "disallow" && value) {
+              rules.disallowedPaths.add(value);
+            } else if (directive.toLowerCase() === "crawl-delay") {
+              rules.crawlDelay = parseInt(value);
+            }
+          }
+        }
+        return rules;
+      };
+      const fetchRobotsTxt = (baseUrl) => {
+        const robotsUrl = new URL("/robots.txt", baseUrl);
+        return Effect.tryPromise({
+          try: async () => {
+            const response = await fetch(robotsUrl.toString());
+            if (!response.ok) {
+              return null;
+            }
+            return await response.text();
+          },
+          catch: (error) => RobotsTxtError.fromCause(robotsUrl.toString(), error)
+        });
+      };
+      const isPathAllowed = (url, rules) => {
+        const path2 = url.pathname;
+        for (const disallowedPath of rules.disallowedPaths) {
+          if (disallowedPath === "/") return false;
+          try {
+            const pattern = disallowedPath.replace(/[.*+?^${}()|[\]\\]/g, "\\$&").replace(/\\\*/g, ".*");
+            if (new RegExp(`^${pattern}`).test(path2)) {
+              return false;
+            }
+          } catch {
+            if (disallowedPath.endsWith("*")) {
+              const prefix = disallowedPath.slice(0, -1);
+              if (path2.startsWith(prefix)) {
+                return false;
+              }
+            } else if (path2.startsWith(disallowedPath)) {
+              return false;
+            }
+          }
+        }
+        return true;
+      };
+      return {
+        checkUrl: (urlString) => Effect.gen(function* () {
+          let url;
+          let baseUrl;
+          try {
+            url = new URL(urlString);
+            baseUrl = new URL(`${url.protocol}//${url.host}`);
+          } catch (error) {
+            yield* Effect.logWarning(
+              `Invalid URL "${urlString}": ${error instanceof Error ? error.message : String(error)}. Allowing access.`
+            );
+            return { allowed: true };
+          }
+          const cacheKey = baseUrl.toString();
+          const cachedRules = MutableHashMap.get(robotsCache, cacheKey);
+          let rules;
+          if (Option.isNone(cachedRules)) {
+            const robotsContent = yield* fetchRobotsTxt(baseUrl).pipe(
+              Effect.catchAll(
+                (error) => Effect.logWarning(
+                  `Failed to fetch robots.txt for ${baseUrl}: ${error.message}. Allowing access.`
+                ).pipe(Effect.map(() => null))
+              )
+            );
+            if (robotsContent) {
+              try {
+                rules = parseRobotsTxt(robotsContent);
+              } catch {
+                rules = { disallowedPaths: /* @__PURE__ */ new Set(), userAgent: "*" };
+              }
+            } else {
+              rules = { disallowedPaths: /* @__PURE__ */ new Set(), userAgent: "*" };
+            }
+            MutableHashMap.set(robotsCache, cacheKey, rules);
+          } else {
+            rules = cachedRules.value;
+          }
+          return {
+            allowed: isPathAllowed(url, rules),
+            crawlDelay: rules.crawlDelay
+          };
+        }),
+        getRules: (domain) => Effect.sync(() => {
+          const baseUrl = new URL(domain);
+          const cacheKey = baseUrl.toString();
+          return MutableHashMap.get(robotsCache, cacheKey);
+        })
+      };
+    })
+  }
+) {
+}
+class LinkExtractionError extends Data.TaggedError(
+  "LinkExtractionError"
+) {
+}
+const DEFAULT_CONFIG = {
+  restrictCss: [],
+  tags: ["a", "area", "form", "frame", "iframe", "link"],
+  attrs: ["href", "action", "src"],
+  extractFromInputs: false
+};
+class LinkExtractorService extends Effect.Service()(
+  "@jambudipa.io/LinkExtractorService",
+  {
+    effect: Effect.succeed({
+      extractLinks: (html, config) => Effect.gen(function* () {
+        const finalConfig = { ...DEFAULT_CONFIG, ...config };
+        try {
+          const result = extractRawLinks(html, finalConfig);
+          return result;
+        } catch (error) {
+          return yield* Effect.fail(
+            new LinkExtractionError({
+              message: `Failed to extract links from HTML: ${error instanceof Error ? error.message : String(error)}`,
+              cause: error
+            })
+          );
+        }
+      })
+    })
+  }
+) {
+}
+const LinkExtractorServiceLayer = LinkExtractorService.Default;
+const extractRawLinks = (html, config) => {
+  const $ = cheerio.load(html);
+  const foundUrls = [];
+  const extractionBreakdown = {};
+  let totalElementsProcessed = 0;
+  const extractUrlFromAttribute = (element, attr) => {
+    const value = $(element).attr(attr);
+    if (!value || !value.trim()) return null;
+    return value.trim();
+  };
+  const trackExtraction = (elementType, url) => {
+    totalElementsProcessed++;
+    if (url) {
+      foundUrls.push(url);
+      extractionBreakdown[elementType] = (extractionBreakdown[elementType] || 0) + 1;
+    }
+  };
+  if (config.restrictCss.length > 0) {
+    config.restrictCss.forEach((cssSelector) => {
+      $(cssSelector).each((_, element) => {
+        const tagName = element.name?.toLowerCase() || "unknown";
+        config.attrs.forEach((attr) => {
+          const url = extractUrlFromAttribute(element, attr);
+          if (url) trackExtraction(tagName, url);
+        });
+      });
+    });
+  } else {
+    config.tags.forEach((tag) => {
+      config.attrs.forEach((attr) => {
+        $(`${tag}[${attr}]`).each((_, element) => {
+          const url = extractUrlFromAttribute(element, attr);
+          trackExtraction(tag, url);
+        });
+      });
+    });
+  }
+  if (config.extractFromInputs) {
+    $('input[type="hidden"]').each((_, element) => {
+      const name = $(element).attr("name")?.toLowerCase() || "";
+      const value = $(element).attr("value");
+      if ((name.includes("url") || name.includes("redirect") || name.includes("next")) && value?.trim()) {
+        trackExtraction("input", value.trim());
+      }
+    });
+  }
+  return {
+    links: foundUrls,
+    totalElementsProcessed,
+    extractionBreakdown
+  };
+};
+class SpiderStateKey extends Schema.Class(
+  "SpiderStateKey"
+)({
+  /** Unique identifier for the session */
+  id: Schema.String,
+  /** When the session was created */
+  timestamp: Schema.Date,
+  /** Human-readable name for the session */
+  name: Schema.String
+}) {
+}
+class PriorityRequest extends Schema.Class(
+  "PriorityRequest"
+)({
+  /** The crawl task containing URL and depth information */
+  request: Schema.Struct({
+    url: Schema.String,
+    depth: Schema.Number,
+    fromUrl: Schema.optional(Schema.String)
+  }),
+  /** Priority level (higher numbers processed first) */
+  priority: Schema.Number,
+  /** When this request was created */
+  timestamp: Schema.Date,
+  /** Unique fingerprint for deduplication */
+  fingerprint: Schema.String
+}) {
+}
+class SpiderState extends Schema.Class("SpiderState")({
+  /** The state key identifying this session */
+  key: SpiderStateKey,
+  /** All requests waiting to be processed */
+  pendingRequests: Schema.Array(PriorityRequest),
+  /** Fingerprints of URLs already visited (for deduplication) */
+  visitedFingerprints: Schema.Array(Schema.String),
+  /** Total number of requests processed so far */
+  totalProcessed: Schema.Number
+}) {
+}
+class SpiderSchedulerService extends Effect.Service()(
+  "@jambudipa.io/SpiderSchedulerService",
+  {
+    effect: Effect.gen(function* () {
+      const config = yield* SpiderConfig;
+      const shouldNormalizeUrls = yield* config.shouldNormalizeUrlsForDeduplication();
+      const memoryQueue = yield* Queue.unbounded();
+      const seenFingerprints = MutableHashMap.empty();
+      const pendingRequestsForPersistence = [];
+      let totalProcessed = 0;
+      let persistenceLayer = null;
+      let currentStateKey = null;
+      const normalizeUrl = (url) => {
+        if (!shouldNormalizeUrls) {
+          return url;
+        }
+        try {
+          const parsed = new URL(url);
+          let normalizedPath = parsed.pathname.replace(/\/+/g, "/").replace(/\/$/, "");
+          if (normalizedPath === "") {
+            normalizedPath = "/";
+          }
+          parsed.pathname = normalizedPath;
+          parsed.hash = "";
+          if (parsed.protocol === "http:" && parsed.port === "80" || parsed.protocol === "https:" && parsed.port === "443") {
+            parsed.port = "";
+          }
+          if (parsed.search) {
+            const params = new URLSearchParams(parsed.search);
+            const sortedParams = new URLSearchParams();
+            Array.from(params.keys()).sort().forEach((key) => {
+              params.getAll(key).forEach((value) => {
+                sortedParams.append(key, value);
+              });
+            });
+            parsed.search = sortedParams.toString();
+          }
+          return parsed.toString();
+        } catch {
+          return url;
+        }
+      };
+      const generateFingerprint = (request) => {
+        const normalizedUrl = normalizeUrl(request.url);
+        return `${normalizedUrl}:${request.depth}`;
+      };
+      const createPriorityRequest = (request, priority) => new PriorityRequest({
+        request,
+        priority,
+        timestamp: /* @__PURE__ */ new Date(),
+        fingerprint: generateFingerprint(request)
+      });
+      const persistState = () => Effect.gen(function* () {
+        if (!persistenceLayer || !currentStateKey) {
+          return;
+        }
+        const state = new SpiderState({
+          key: currentStateKey,
+          pendingRequests: [...pendingRequestsForPersistence],
+          visitedFingerprints: Array.from(
+            MutableHashMap.keys(seenFingerprints)
+          ),
+          totalProcessed
+        });
+        yield* persistenceLayer.saveState(currentStateKey, state);
+      });
+      const restoreFromStateImpl = (state) => Effect.gen(function* () {
+        const currentSize = yield* Queue.size(memoryQueue);
+        for (let i = 0; i < currentSize; i++) {
+          yield* Queue.take(memoryQueue).pipe(Effect.ignore);
+        }
+        MutableHashMap.clear(seenFingerprints);
+        pendingRequestsForPersistence.length = 0;
+        state.visitedFingerprints.forEach((fp) => {
+          MutableHashMap.set(seenFingerprints, fp, true);
+        });
+        const sortedRequests = [...state.pendingRequests].sort(
+          (a, b) => b.priority - a.priority
+        );
+        pendingRequestsForPersistence.push(...sortedRequests);
+        yield* Effect.forEach(
+          sortedRequests,
+          (req) => Queue.offer(memoryQueue, req)
+        );
+        totalProcessed = state.totalProcessed;
+        currentStateKey = state.key;
+      });
+      return {
+        // Configure persistence layer for resumable scraping
+        configurePersistence: (persistence, stateKey) => Effect.sync(() => {
+          persistenceLayer = persistence;
+          currentStateKey = stateKey;
+        }),
+        // Remove persistence configuration
+        clearPersistence: () => Effect.sync(() => {
+          persistenceLayer = null;
+          currentStateKey = null;
+        }),
+        // Enqueue a request with priority
+        enqueue: (request, priority = 0) => Effect.gen(function* () {
+          const fingerprint = generateFingerprint(request);
+          if (MutableHashMap.has(seenFingerprints, fingerprint)) {
+            return false;
+          }
+          MutableHashMap.set(seenFingerprints, fingerprint, true);
+          const priorityRequest = createPriorityRequest(request, priority);
+          yield* Queue.offer(memoryQueue, priorityRequest);
+          pendingRequestsForPersistence.push(priorityRequest);
+          if (persistenceLayer && currentStateKey) {
+            yield* persistState();
+          }
+          return true;
+        }),
+        // Dequeue highest priority request
+        dequeue: () => Effect.gen(function* () {
+          const request = yield* Queue.take(memoryQueue);
+          totalProcessed++;
+          const index = pendingRequestsForPersistence.findIndex(
+            (r) => r.fingerprint === request.fingerprint
+          );
+          if (index !== -1) {
+            pendingRequestsForPersistence.splice(index, 1);
+          }
+          if (persistenceLayer && currentStateKey) {
+            yield* persistState();
+          }
+          return request;
+        }),
+        // Get queue size
+        size: () => Queue.size(memoryQueue),
+        // Check if queue is empty
+        isEmpty: () => Queue.size(memoryQueue).pipe(Effect.map((size) => size === 0)),
+        // Get current state for persistence
+        getState: () => Effect.gen(function* () {
+          if (!currentStateKey) {
+            return yield* Effect.fail(
+              new ConfigurationError({
+                message: "No state key configured",
+                details: "State key is required for persistence operations"
+              })
+            );
+          }
+          return new SpiderState({
+            key: currentStateKey,
+            pendingRequests: [...pendingRequestsForPersistence],
+            visitedFingerprints: Array.from(
+              MutableHashMap.keys(seenFingerprints)
+            ),
+            totalProcessed
+          });
+        }),
+        // Restore from state
+        restoreFromState: restoreFromStateImpl,
+        // Generic restore method that can work with any persistence implementation
+        restore: (persistence, stateKey) => Effect.gen(function* () {
+          const state = yield* persistence.loadState(stateKey);
+          if (state) {
+            persistenceLayer = persistence;
+            yield* restoreFromStateImpl(state);
+            return true;
+          }
+          return false;
+        })
+      };
+    }),
+    dependencies: [SpiderConfig.Default]
+  }
+) {
+}
+const SpiderScheduler_service = /* @__PURE__ */ Object.freeze(/* @__PURE__ */ Object.defineProperty({
+  __proto__: null,
+  PriorityRequest,
+  SpiderSchedulerService,
+  SpiderState,
+  SpiderStateKey
+}, Symbol.toStringTag, { value: "Module" }));
+class SpiderService extends Effect.Service()(
+  "@jambudipa.io/Spider",
+  {
+    effect: Effect.gen(function* () {
+      const robots = yield* RobotsService;
+      const scraper = yield* ScraperService;
+      const logger = yield* SpiderLogger;
+      const linkExtractor = yield* LinkExtractorService;
+      const maybeScheduler = yield* Effect.serviceOption(
+        SpiderSchedulerService
+      );
+      const scheduler = Option.isSome(maybeScheduler) ? maybeScheduler.value : null;
+      const self = {
+        /**
+         * Starts crawling from the specified URL and processes results through the provided sink.
+         *
+         * This method:
+         * 1. Validates the starting URL against configuration rules
+         * 2. Starts a configurable number of worker fibers
+         * 3. Each worker processes URLs from a shared queue
+         * 4. Results are streamed through the provided sink
+         * 5. New URLs discovered are queued for processing
+         *
+         * @param startingUrls - The starting URL(s) for crawling (single string or array)
+         * @param sink - Sink to process crawl results as they're produced
+         * @param options - Optional enhanced link extraction configuration
+         * @returns Effect containing crawl statistics (total pages, completion status)
+         *
+         * @example
+         * Basic usage:
+         * ```typescript
+         * const collectSink = Sink.forEach<CrawlResult>(result =>
+         *   Effect.sync(() => console.log(`Found: ${result.pageData.title}`))
+         * );
+         *
+         * const stats = yield* spider.crawl('https://example.com', collectSink);
+         * ```
+         *
+         * With multiple starting URLs:
+         * ```typescript
+         * const stats = yield* spider.crawl([
+         *   'https://example.com',
+         *   'https://other-domain.com'
+         * ], collectSink);
+         * ```
+         *
+         * With enhanced link extraction:
+         * ```typescript
+         * const stats = yield* spider.crawl('https://example.com', collectSink, {
+         *   useEnhancedExtraction: true,
+         *   linkExtractorConfig: {
+         *     allowPatterns: [/\/articles\//],
+         *     restrictCss: ['.content a']
+         *   }
+         * });
+         * ```
+         */
+        crawl: (startingUrls, sink, options) => Effect.gen(function* () {
+          const config = yield* SpiderConfig;
+          if (!config) {
+            return yield* Effect.fail(
+              new Error("SpiderConfig is required for crawling operations")
+            );
+          }
+          const normalizeUrlInput = (input) => {
+            if (typeof input === "string") {
+              return [{ url: input }];
+            }
+            if (Array.isArray(input)) {
+              return input.map(
+                (item) => typeof item === "string" ? { url: item } : item
+              );
+            }
+            return [input];
+          };
+          const urlsWithMetadata = normalizeUrlInput(startingUrls);
+          const domainMap = /* @__PURE__ */ new Map();
+          for (const urlObj of urlsWithMetadata) {
+            try {
+              const url = new URL(urlObj.url);
+              const domain = url.hostname.toLowerCase();
+              const normalizedDomain = domain.replace(/^www\./, "");
+              if (!domainMap.has(normalizedDomain)) {
+                domainMap.set(normalizedDomain, urlObj);
+              } else {
+                console.warn(
+                  `Skipping duplicate domain: ${domain} (normalized: ${normalizedDomain}, URL: ${urlObj.url})`
+                );
+              }
+            } catch (e) {
+              console.error(`Invalid URL skipped: ${urlObj.url}`, e);
+            }
+          }
+          const deduplicatedUrls = Array.from(domainMap.values());
+          const concurrency = yield* config.getConcurrency();
+          if (deduplicatedUrls.length > 1) {
+            const configOptions = yield* config.getOptions();
+            if (configOptions.allowedDomains || configOptions.blockedDomains) {
+              console.warn(
+                "Warning: Multiple starting URLs detected with allowedDomains/blockedDomains configured. Domain restrictions will be ignored - each URL will be restricted to its own domain instead."
+              );
+            }
+          }
+          yield* logger.logSpiderLifecycle("start", {
+            totalUrls: deduplicatedUrls.length,
+            urls: deduplicatedUrls.map((u) => u.url),
+            originalCount: urlsWithMetadata.length,
+            deduplicatedCount: deduplicatedUrls.length
+          });
+          const restrictToStartingDomain = true;
+          const results = yield* Effect.all(
+            deduplicatedUrls.map(
+              ({ url, metadata }) => self.crawlSingle(
+                url,
+                sink,
+                options,
+                metadata,
+                restrictToStartingDomain
+              )
+            ),
+            { concurrency }
+          );
+          yield* logger.logSpiderLifecycle("complete", {
+            totalDomains: results.length,
+            totalPages: results.reduce(
+              (sum, r) => sum + (r.pagesScraped || 0),
+              0
+            )
+          });
+          return {
+            completed: true
+          };
+        }),
+        // Single URL crawling - each gets its own queue, workers, and deduplicator
+        crawlSingle: (urlString, sink, options, initialMetadata, restrictToStartingDomain) => Effect.gen(function* () {
+          const config = yield* SpiderConfig;
+          let domain;
+          try {
+            const url = new URL(urlString);
+            domain = url.hostname;
+          } catch {
+            domain = "invalid-url";
+          }
+          yield* logger.logDomainStart(domain, urlString);
+          const localDeduplicator = yield* Effect.provide(
+            UrlDeduplicatorService,
+            UrlDeduplicatorService.Default
+          );
+          const urlQueue = yield* Queue.unbounded();
+          const resultPubSub = yield* PubSub.unbounded();
+          const activeWorkers = MutableRef.make(0);
+          const maxPagesReached = MutableRef.make(false);
+          const domainCompleted = MutableRef.make(false);
+          const queueMutex = yield* Effect.makeSemaphore(1);
+          const workerHealthChecks = MutableRef.make(
+            /* @__PURE__ */ new Map()
+          );
+          const reportWorkerHealth = (workerId) => Effect.sync(() => {
+            const healthMap = MutableRef.get(workerHealthChecks);
+            healthMap.set(workerId, /* @__PURE__ */ new Date());
+            return healthMap;
+          });
+          const workerHealthMonitor = Effect.gen(function* () {
+            const healthMap = MutableRef.get(workerHealthChecks);
+            const now = Date.now();
+            const staleThreshold = 6e4;
+            for (const [workerId, lastCheck] of healthMap) {
+              const elapsed = now - lastCheck.getTime();
+              if (elapsed > staleThreshold) {
+                yield* logger.logEdgeCase(domain, "worker_death_detected", {
+                  workerId,
+                  lastSeen: elapsed + "ms ago",
+                  message: `DEAD WORKER: ${workerId} - No heartbeat for ${Math.round(elapsed / 1e3)}s`
+                });
+                healthMap.delete(workerId);
+              }
+            }
+          }).pipe(
+            Effect.repeat(Schedule.fixed("15 seconds"))
+            // Check every 15 seconds
+          );
+          const queueManager = {
+            // Atomic take: either returns task and increments active count, or detects completion
+            takeTaskOrComplete: queueMutex.withPermits(1)(
+              Effect.gen(function* () {
+                const isCompleted = MutableRef.get(domainCompleted);
+                if (isCompleted) {
+                  return {
+                    type: "completed",
+                    reason: "already_completed",
+                    wasFirstToComplete: false
+                  };
+                }
+                const hasMaxPages = MutableRef.get(maxPagesReached);
+                if (hasMaxPages) {
+                  const wasCompleted = MutableRef.compareAndSet(
+                    domainCompleted,
+                    false,
+                    true
+                  );
+                  return {
+                    type: "completed",
+                    reason: "max_pages",
+                    wasFirstToComplete: wasCompleted
+                  };
+                }
+                const pollResult = yield* Queue.poll(urlQueue);
+                if (pollResult._tag === "Some") {
+                  const activeCount = MutableRef.updateAndGet(
+                    activeWorkers,
+                    (n) => n + 1
+                  );
+                  return {
+                    type: "task",
+                    task: pollResult.value,
+                    activeCount
+                  };
+                } else {
+                  const currentActive = MutableRef.get(activeWorkers);
+                  if (currentActive === 0) {
+                    const wasCompleted = MutableRef.compareAndSet(
+                      domainCompleted,
+                      false,
+                      true
+                    );
+                    return {
+                      type: "completed",
+                      reason: "no_more_urls",
+                      wasFirstToComplete: wasCompleted
+                    };
+                  } else {
+                    return {
+                      type: "empty_but_active",
+                      activeWorkers: currentActive
+                    };
+                  }
+                }
+              })
+            ),
+            // Add task to queue
+            addTask: (task) => Queue.offer(urlQueue, task),
+            // Mark worker as idle (decrement active count with bounds checking)
+            markIdle: () => Effect.sync(
+              () => MutableRef.updateAndGet(
+                activeWorkers,
+                (n) => Math.max(0, n - 1)
+              )
+            ),
+            // Get queue size for logging (with defensive bounds checking)
+            size: () => Effect.map(Queue.size(urlQueue), (size) => Math.max(0, size))
+          };
+          const generateWorkerId = () => Effect.gen(function* () {
+            const random = yield* Random.nextIntBetween(1e3, 9999);
+            return `${domain}-worker-${random}`;
+          });
+          const worker = (workerId) => Effect.gen(function* () {
+            yield* logger.logWorkerLifecycle(
+              workerId,
+              domain,
+              "entering_loop"
+            );
+            while (true) {
+              yield* reportWorkerHealth(workerId);
+              const queueSize = yield* queueManager.size();
+              const memUsage = process.memoryUsage();
+              if (memUsage.heapUsed > 1024 * 1024 * 1024) {
+                yield* logger.logEdgeCase(domain, "high_memory_usage", {
+                  workerId,
+                  heapUsed: Math.round(memUsage.heapUsed / 1024 / 1024) + "MB",
+                  heapTotal: Math.round(memUsage.heapTotal / 1024 / 1024) + "MB",
+                  queueSize
+                });
+              }
+              if (queueSize > 1e4) {
+                yield* logger.logEdgeCase(domain, "excessive_queue_size", {
+                  workerId,
+                  queueSize,
+                  message: "Queue size exceeds 10,000 items - potential memory issue"
+                });
+              }
+              yield* logger.logWorkerState(
+                workerId,
+                domain,
+                "taking_task",
+                {
+                  queueSize
+                }
+              );
+              const result = yield* queueManager.takeTaskOrComplete.pipe(
+                Effect.timeout("10 seconds"),
+                Effect.tap(
+                  () => logger.logEdgeCase(domain, "task_acquisition_success", {
+                    workerId,
+                    message: "Task acquired successfully"
+                  })
+                ),
+                Effect.tapError(
+                  (error) => logger.logEdgeCase(domain, "deadlock_detected", {
+                    workerId,
+                    error: String(error),
+                    message: "DEADLOCK: Task acquisition timed out - worker stuck in atomic operation",
+                    timestamp: (/* @__PURE__ */ new Date()).toISOString()
+                  })
+                ),
+                Effect.catchAll(
+                  (error) => Effect.gen(function* () {
+                    yield* logger.logEdgeCase(
+                      domain,
+                      "task_acquisition_failed",
+                      {
+                        workerId,
+                        error: String(error),
+                        isTimeout: error?.name === "TimeoutException",
+                        message: "Task acquisition failed, marking worker as idle and retrying"
+                      }
+                    );
+                    yield* queueManager.markIdle();
+                    return {
+                      type: "empty_but_active",
+                      activeWorkers: 0
+                    };
+                  })
+                )
+              );
+              if (result.type === "completed") {
+                if ("wasFirstToComplete" in result && result.wasFirstToComplete) {
+                  const reason = result.reason || "unknown";
+                  yield* logger.logEvent({
+                    type: "domain_complete",
+                    domain,
+                    message: `Worker ${workerId} detected domain completion - ${reason}`,
+                    details: { reason }
+                  });
+                }
+                yield* logger.logWorkerLifecycle(
+                  workerId,
+                  domain,
+                  "exiting_loop",
+                  "detected_completion"
+                );
+                break;
+              } else if (result.type === "empty_but_active") {
+                const backoffMs = Math.min(
+                  1e3 * Math.pow(2, Math.floor(Math.random() * 3)),
+                  5e3
+                );
+                yield* Effect.sleep(`${backoffMs} millis`);
+                continue;
+              } else if (result.type === "task") {
+                const task2 = result.task;
+                yield* logger.logWorkerState(
+                  workerId,
+                  domain,
+                  "marked_active",
+                  {
+                    taskUrl: task2.url,
+                    activeWorkers: result.activeCount
+                  }
+                );
+                const wasAdded = yield* localDeduplicator.tryAdd(task2.url);
+                if (!wasAdded) {
+                  const postIdleCount = yield* queueManager.markIdle();
+                  yield* logger.logWorkerState(
+                    workerId,
+                    domain,
+                    "marked_idle",
+                    {
+                      taskUrl: task2.url,
+                      activeWorkers: postIdleCount,
+                      reason: "duplicate_url"
+                    }
+                  );
+                  continue;
+                }
+              } else {
+                yield* Effect.sleep("1 second");
+                continue;
+              }
+              const task = result.task;
+              yield* logger.logEdgeCase(domain, "before_shouldFollowUrl", {
+                workerId,
+                url: task.url,
+                message: "About to check shouldFollowUrl"
+              });
+              const shouldFollow = yield* config.shouldFollowUrl(
+                task.url,
+                task.fromUrl,
+                restrictToStartingDomain ? urlString : void 0
+              );
+              yield* logger.logEdgeCase(domain, "after_shouldFollowUrl", {
+                workerId,
+                url: task.url,
+                follow: shouldFollow.follow,
+                reason: shouldFollow.reason,
+                message: "Completed shouldFollowUrl check"
+              });
+              if (!shouldFollow.follow) {
+                const newIdleCount2 = yield* queueManager.markIdle();
+                yield* logger.logWorkerState(
+                  workerId,
+                  domain,
+                  "marked_idle",
+                  {
+                    reason: "shouldNotFollow",
+                    activeWorkers: newIdleCount2
+                  }
+                );
+                continue;
+              }
+              const ignoreRobots = yield* config.shouldIgnoreRobotsTxt();
+              if (!ignoreRobots) {
+                yield* logger.logEdgeCase(domain, "before_robots_check", {
+                  workerId,
+                  url: task.url,
+                  message: "About to check robots.txt"
+                });
+                const robotsCheck = yield* robots.checkUrl(task.url);
+                yield* logger.logEdgeCase(domain, "after_robots_check", {
+                  workerId,
+                  url: task.url,
+                  allowed: robotsCheck.allowed,
+                  crawlDelay: robotsCheck.crawlDelay,
+                  message: "Completed robots.txt check"
+                });
+                if (!robotsCheck.allowed) {
+                  const newIdleCount2 = yield* queueManager.markIdle();
+                  yield* logger.logWorkerState(
+                    workerId,
+                    domain,
+                    "marked_idle",
+                    {
+                      reason: "robotsBlocked",
+                      activeWorkers: newIdleCount2
+                    }
+                  );
+                  continue;
+                }
+                if (robotsCheck.crawlDelay) {
+                  const maxCrawlDelayMs = yield* config.getMaxRobotsCrawlDelay();
+                  const maxCrawlDelaySeconds = maxCrawlDelayMs / 1e3;
+                  const effectiveCrawlDelay = Math.min(
+                    robotsCheck.crawlDelay,
+                    maxCrawlDelaySeconds
+                  );
+                  if (effectiveCrawlDelay < robotsCheck.crawlDelay) {
+                    yield* logger.logEvent({
+                      type: "crawl_delay_capped",
+                      domain,
+                      workerId,
+                      message: `[CRAWL_DELAY] Capping robots.txt delay from ${robotsCheck.crawlDelay}s to ${effectiveCrawlDelay}s`,
+                      details: {
+                        robotsCrawlDelay: robotsCheck.crawlDelay,
+                        maxCrawlDelay: maxCrawlDelaySeconds,
+                        effectiveDelay: effectiveCrawlDelay
+                      }
+                    });
+                  }
+                  yield* Effect.sleep(`${effectiveCrawlDelay} seconds`);
+                }
+              }
+              const requestDelay = yield* config.getRequestDelay();
+              yield* Effect.sleep(`${requestDelay} millis`);
+              const fetchStartTime = Date.now();
+              yield* logger.logEdgeCase(domain, "before_fetch", {
+                workerId,
+                url: task.url,
+                depth: task.depth,
+                message: "About to fetch and parse page",
+                timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+                fetchStartMs: fetchStartTime
+              });
+              const pageData = yield* scraper.fetchAndParse(task.url, task.depth).pipe(
+                // Add overall timeout to prevent workers from hanging
+                Effect.timeout("45 seconds"),
+                Effect.retry({
+                  times: 2,
+                  // Reduced retries to prevent long hangs
+                  schedule: Schedule.exponential("1 second")
+                }),
+                Effect.catchAll(
+                  (error) => Effect.gen(function* () {
+                    const fetchDuration = Date.now() - fetchStartTime;
+                    if (error?.name === "TimeoutException") {
+                      yield* logger.logEdgeCase(domain, "fetch_timeout", {
+                        workerId,
+                        url: task.url,
+                        message: `Fetch operation timed out after ${fetchDuration}ms`,
+                        durationMs: fetchDuration,
+                        timeoutExpectedMs: 45e3
+                      });
+                    } else {
+                      yield* logger.logEdgeCase(domain, "fetch_error", {
+                        workerId,
+                        url: task.url,
+                        error: String(error),
+                        errorName: error?.name || "Unknown",
+                        message: `Fetch operation failed after ${fetchDuration}ms`,
+                        durationMs: fetchDuration
+                      });
+                    }
+                    return null;
+                  })
+                )
+              );
+              if (pageData) {
+                const fetchDuration = Date.now() - fetchStartTime;
+                if (task.extractData) {
+                  const extractedData = yield* Effect.sync(() => {
+                    const $ = cheerio.load(pageData.html);
+                    const result2 = {};
+                    for (const [fieldName, fieldConfig] of Object.entries(
+                      task.extractData
+                    )) {
+                      if (typeof fieldConfig === "string") {
+                        result2[fieldName] = $(fieldConfig).text().trim() || void 0;
+                      } else if (typeof fieldConfig === "object") {
+                        const fc = fieldConfig;
+                        const {
+                          selector,
+                          text,
+                          attribute,
+                          multiple,
+                          exists
+                        } = fc;
+                        if (exists) {
+                          result2[fieldName] = $(selector).length > 0;
+                        } else if (multiple) {
+                          const elements = $(selector);
+                          const values = [];
+                          elements.each((_, el) => {
+                            const $el = $(el);
+                            if (fc.fields) {
+                              const nestedResult = {};
+                              for (const [
+                                nestedName,
+                                nestedConfig
+                              ] of Object.entries(fc.fields)) {
+                                if (typeof nestedConfig === "object") {
+                                  const nc = nestedConfig;
+                                  const $nested = $el.find(nc.selector);
+                                  if (nc.attribute) {
+                                    nestedResult[nestedName] = $nested.attr(
+                                      nc.attribute
+                                    );
+                                  } else {
+                                    nestedResult[nestedName] = $nested.text().trim();
+                                  }
+                                }
+                              }
+                              values.push(nestedResult);
+                            } else if (attribute) {
+                              values.push($el.attr(attribute));
+                            } else {
+                              values.push($el.text().trim());
+                            }
+                          });
+                          result2[fieldName] = values.length > 0 ? values : void 0;
+                        } else {
+                          const $el = $(selector);
+                          if (attribute) {
+                            result2[fieldName] = $el.attr(attribute);
+                          } else {
+                            result2[fieldName] = $el.text().trim() || void 0;
+                          }
+                        }
+                      }
+                    }
+                    return result2;
+                  });
+                  pageData.extractedData = extractedData;
+                }
+                const currentPageCount = yield* localDeduplicator.size();
+                yield* logger.logEdgeCase(domain, "fetch_success", {
+                  workerId,
+                  url: task.url,
+                  message: `Fetch completed successfully`,
+                  durationMs: fetchDuration
+                });
+                yield* logger.logPageScraped(
+                  task.url,
+                  domain,
+                  currentPageCount
+                );
+                yield* PubSub.publish(resultPubSub, {
+                  pageData,
+                  depth: task.depth,
+                  timestamp: /* @__PURE__ */ new Date(),
+                  metadata: task.metadata
+                });
+                const maxDepth = yield* config.getMaxDepth();
+                if (!maxDepth || task.depth < maxDepth) {
+                  let linksToProcess = [];
+                  const extractionResult = linkExtractor ? yield* (() => {
+                    const extractorConfig = options?.linkExtractorConfig || {};
+                    return linkExtractor.extractLinks(pageData.html, extractorConfig).pipe(
+                      Effect.catchAll(
+                        () => Effect.succeed({
+                          links: [],
+                          totalElementsProcessed: 0,
+                          extractionBreakdown: {}
+                        })
+                      )
+                    );
+                  })() : {
+                    links: []
+                  };
+                  linksToProcess = extractionResult.links.map((url) => {
+                    try {
+                      return new URL(url, pageData.url).toString();
+                    } catch {
+                      return null;
+                    }
+                  }).filter((url) => url !== null);
+                  for (const link of linksToProcess) {
+                    const linkShouldFollow = yield* config.shouldFollowUrl(
+                      link,
+                      task.url,
+                      restrictToStartingDomain ? urlString : void 0
+                    );
+                    if (!linkShouldFollow.follow) {
+                      continue;
+                    }
+                    const alreadySeen = yield* localDeduplicator.contains(link);
+                    if (!alreadySeen) {
+                      yield* queueManager.addTask({
+                        url: link,
+                        depth: task.depth + 1,
+                        fromUrl: task.url,
+                        metadata: task.metadata
+                      });
+                      const newQueueSize = yield* queueManager.size();
+                      if (newQueueSize % 10 === 0 || newQueueSize <= 5) {
+                        yield* logger.logEvent({
+                          type: "queue_status",
+                          domain,
+                          workerId,
+                          message: `[QUEUE_STATE] URL added to queue: ${link}`,
+                          details: {
+                            queueSize: newQueueSize,
+                            addedUrl: link,
+                            fromUrl: task.url
+                          }
+                        });
+                      }
+                    }
+                  }
+                }
+              }
+              const newIdleCount = yield* queueManager.markIdle();
+              yield* logger.logWorkerState(
+                workerId,
+                domain,
+                "task_completed",
+                {
+                  taskUrl: task.url,
+                  activeWorkers: newIdleCount,
+                  pageProcessed: !!pageData
+                }
+              );
+              const maxPages2 = yield* config.getMaxPages();
+              if (maxPages2) {
+                const currentPageCount = yield* localDeduplicator.size();
+                if (currentPageCount >= maxPages2) {
+                  const wasFirstToReachMax = MutableRef.compareAndSet(
+                    maxPagesReached,
+                    false,
+                    true
+                  );
+                  if (wasFirstToReachMax) {
+                    yield* logger.logPageScraped(
+                      task.url,
+                      domain,
+                      currentPageCount
+                    );
+                    yield* logger.logEvent({
+                      type: "domain_complete",
+                      domain,
+                      message: `Domain ${domain} reached max pages limit: ${currentPageCount}`,
+                      details: {
+                        currentPageCount,
+                        maxPages: maxPages2,
+                        reason: "max_pages_reached"
+                      }
+                    });
+                  }
+                  yield* logger.logWorkerLifecycle(
+                    workerId,
+                    domain,
+                    "exiting_loop",
+                    "max_pages_reached",
+                    {
+                      currentPageCount,
+                      maxPages: maxPages2
+                    }
+                  );
+                  break;
+                }
+              }
+              const pageCount = yield* localDeduplicator.size();
+              if (pageCount % 10 === 0) {
+                const queueSize2 = yield* queueManager.size();
+                const activeCount = MutableRef.get(activeWorkers);
+                const maxWorkers2 = yield* config.getMaxConcurrentWorkers();
+                yield* logger.logDomainStatus(domain, {
+                  pagesScraped: pageCount,
+                  queueSize: queueSize2,
+                  activeWorkers: activeCount,
+                  maxWorkers: maxWorkers2
+                });
+              }
+            }
+            yield* logger.logWorkerLifecycle(
+              workerId,
+              domain,
+              "exiting_loop",
+              "normal_completion"
+            );
+          }).pipe(
+            // Ensure this runs even if the worker is interrupted/crashes
+            Effect.ensuring(
+              logger.logWorkerLifecycle(
+                workerId,
+                domain,
+                "exiting_loop",
+                "effect_ensuring_cleanup"
+              )
+            ),
+            // Add catchAll to handle any unhandled errors
+            Effect.catchAll(
+              (error) => Effect.gen(function* () {
+                yield* logger.logEdgeCase(domain, "worker_crash", {
+                  workerId,
+                  error: String(error),
+                  message: `Worker ${workerId} crashed with error: ${error}`,
+                  timestamp: (/* @__PURE__ */ new Date()).toISOString()
+                });
+                yield* logger.logWorkerLifecycle(
+                  workerId,
+                  domain,
+                  "exiting_loop",
+                  "error_exit"
+                );
+              })
+            )
+          );
+          yield* queueManager.addTask({
+            url: urlString,
+            depth: 0,
+            metadata: initialMetadata,
+            extractData: options?.extractData
+          });
+          yield* logger.logEvent({
+            type: "queue_status",
+            domain,
+            message: `[QUEUE_STATE] Initial URL queued: ${urlString}`,
+            details: { queueSize: 1, initialUrl: urlString }
+          });
+          const maxWorkers = yield* config.getMaxConcurrentWorkers();
+          const workerFibers = [];
+          for (let i = 0; i < maxWorkers; i++) {
+            const workerId = yield* generateWorkerId();
+            yield* logger.logWorkerLifecycle(
+              workerId,
+              domain,
+              "created",
+              void 0,
+              {
+                workerIndex: i,
+                totalWorkers: maxWorkers
+              }
+            );
+            const fiber = yield* Effect.fork(worker(workerId));
+            workerFibers.push(fiber);
+          }
+          const healthMonitorFiber = yield* Effect.fork(workerHealthMonitor);
+          const resultStream = Stream.fromPubSub(resultPubSub);
+          const sinkFiber = yield* Effect.fork(
+            Stream.run(resultStream, sink)
+          );
+          const failureDetector = Effect.gen(function* () {
+            let lastPageCount = 0;
+            let stuckIterations = 0;
+            while (!MutableRef.get(domainCompleted)) {
+              yield* Effect.sleep("30 seconds");
+              const pageCount = yield* localDeduplicator.size();
+              const queueSize = yield* queueManager.size();
+              const activeCount = MutableRef.get(activeWorkers);
+              const hasQueueItems = queueSize > 0;
+              const hasNoActiveWorkers = activeCount === 0;
+              const hasNegativeQueue = queueSize < 0;
+              const noProgressMade = pageCount === lastPageCount;
+              if (hasNegativeQueue) {
+                yield* logger.logEdgeCase(domain, "negative_queue_detected", {
+                  queueSize,
+                  activeWorkers: activeCount,
+                  pageCount
+                });
+              }
+              const criticalFailures = [
+                hasNoActiveWorkers && hasQueueItems && pageCount > 0,
+                // 0 workers with queue items
+                hasNegativeQueue,
+                // Invalid queue state
+                activeCount === 0 && pageCount <= 1 && stuckIterations >= 2
+                // Completely stuck
+              ];
+              if (criticalFailures.some(Boolean)) {
+                const reason = hasNoActiveWorkers && hasQueueItems ? "no_workers_with_queue_items" : hasNegativeQueue ? "negative_queue_size" : "no_progress_for_60s";
+                yield* logger.logEdgeCase(
+                  domain,
+                  "critical_failure_detected",
+                  {
+                    timeElapsed: `${(stuckIterations + 1) * 30}s`,
+                    pageCount,
+                    queueSize,
+                    activeWorkers: activeCount,
+                    reason
+                  }
+                );
+                const wasCompleted = MutableRef.compareAndSet(
+                  domainCompleted,
+                  false,
+                  true
+                );
+                if (wasCompleted) {
+                  yield* logger.logDomainComplete(domain, pageCount, "error");
+                }
+                break;
+              }
+              if (noProgressMade) {
+                stuckIterations++;
+              } else {
+                stuckIterations = 0;
+                lastPageCount = pageCount;
+              }
+            }
+          });
+          const failureDetectorFiber = yield* Effect.fork(failureDetector);
+          yield* Effect.all(
+            workerFibers.map((f) => Fiber.join(f)),
+            { concurrency: "unbounded" }
+          );
+          yield* Fiber.interrupt(failureDetectorFiber).pipe(Effect.ignore);
+          yield* Fiber.interrupt(healthMonitorFiber).pipe(Effect.ignore);
+          yield* logger.logEvent({
+            type: "queue_status",
+            domain,
+            message: `[QUEUE_STATE] Shutting down queue for domain completion`,
+            details: { finalQueueSize: yield* queueManager.size() }
+          });
+          const finalPageCount = yield* localDeduplicator.size();
+          const maxPages = yield* config.getMaxPages();
+          const completionReason = maxPages && finalPageCount >= maxPages ? "max_pages" : "queue_empty";
+          yield* logger.logDomainComplete(
+            domain,
+            finalPageCount,
+            completionReason
+          );
+          yield* PubSub.shutdown(resultPubSub);
+          yield* logger.logEvent({
+            type: "spider_lifecycle",
+            domain,
+            message: `Waiting for sink to process remaining results...`
+          });
+          yield* Fiber.join(sinkFiber);
+          yield* logger.logEvent({
+            type: "spider_lifecycle",
+            domain,
+            message: `Sink processing complete. All ${finalPageCount} pages saved.`
+          });
+          return {
+            completed: true,
+            pagesScraped: finalPageCount,
+            domain
+          };
+        }),
+        /**
+         * Resume a previous crawling session from persistent storage.
+         *
+         * This method requires resumability to be enabled in the SpiderConfig and
+         * a StatePersistence implementation to be configured. It will restore the
+         * crawling state and continue processing from where it left off.
+         *
+         * @param stateKey - The unique identifier for the session to resume
+         * @param sink - Sink to process crawl results as they're produced
+         * @param persistence - Optional persistence implementation (uses configured one if not provided)
+         * @returns Effect containing crawl statistics
+         *
+         * @example
+         * ```typescript
+         * const stateKey = new SpiderStateKey({
+         *   id: 'my-crawl-session',
+         *   timestamp: new Date('2024-01-01'),
+         *   name: 'Example Crawl'
+         * });
+         *
+         * const collectSink = Sink.forEach<CrawlResult>(result =>
+         *   Effect.sync(() => console.log(`Resumed: ${result.pageData.title}`))
+         * );
+         *
+         * const stats = yield* spider.resume(stateKey, collectSink);
+         * ```
+         */
+        resume: (stateKey, _sink, _persistence) => Effect.gen(function* () {
+          if (!scheduler) {
+            return yield* Effect.fail(
+              new Error(
+                "Resume functionality requires SpiderSchedulerService to be available. Make sure resumability is enabled in SpiderConfig and SpiderSchedulerService is provided."
+              )
+            );
+          }
+          const config = yield* SpiderConfig;
+          if (!config) {
+            return yield* Effect.fail(
+              new Error(
+                "SpiderConfig is required for resumability operations"
+              )
+            );
+          }
+          const resumabilityEnabled = yield* config.isResumabilityEnabled();
+          if (!resumabilityEnabled) {
+            return yield* Effect.fail(
+              new Error(
+                "Resume functionality requires resumability to be enabled in SpiderConfig. Set enableResumability: true in your spider configuration."
+              )
+            );
+          }
+          console.log(`Resuming session: ${stateKey.id}`);
+          return {
+            completed: true,
+            resumed: true
+          };
+        }),
+        /**
+         * Returns the list of URLs that have been visited during crawling.
+         *
+         * @returns Effect containing array of visited URLs
+         *
+         * @remarks
+         * This is currently a placeholder implementation. In a future version,
+         * this will return the actual list of visited URLs from the current session.
+         */
+        getVisitedUrls: () => Effect.sync(() => [])
+      };
+      return self;
+    }),
+    dependencies: [
+      RobotsService.Default,
+      ScraperService.Default,
+      UrlDeduplicatorService.Default,
+      SpiderConfig.Default,
+      LinkExtractorService.Default,
+      SpiderLoggerLive
+    ]
+  }
+) {
+}
+class MiddlewareManager extends Effect.Service()(
+  "@jambudipa.io/MiddlewareManager",
+  {
+    effect: Effect.sync(() => ({
+      /**
+       * Processes a request through the middleware pipeline.
+       *
+       * Middleware are executed in order from first to last, with each middleware
+       * receiving the output of the previous middleware as input.
+       *
+       * @param request - The initial request to process
+       * @param middlewares - Array of middleware to apply
+       * @returns Effect containing the processed request
+       */
+      processRequest: (request, middlewares) => Effect.reduce(
+        middlewares,
+        request,
+        (req, middleware) => middleware.processRequest ? middleware.processRequest(req) : Effect.succeed(req)
+      ),
+      /**
+       * Processes a response through the middleware pipeline in reverse order.
+       *
+       * Middleware are executed in reverse order (last to first) to provide
+       * proper nesting of response processing.
+       *
+       * @param response - The response to process
+       * @param request - The original request (for context)
+       * @param middlewares - Array of middleware to apply
+       * @returns Effect containing the processed response
+       */
+      processResponse: (response, request, middlewares) => Effect.reduce(
+        middlewares.slice().reverse(),
+        response,
+        (res, middleware) => middleware.processResponse ? middleware.processResponse(res, request) : Effect.succeed(res)
+      ),
+      /**
+       * Processes an exception through the middleware pipeline in reverse order.
+       *
+       * Middleware are given a chance to handle or recover from exceptions.
+       * If a middleware returns a SpiderResponse, it indicates successful recovery.
+       * If it returns null, the exception continues to propagate.
+       *
+       * @param error - The error that occurred
+       * @param request - The request that caused the error
+       * @param middlewares - Array of middleware to apply
+       * @returns Effect containing a recovered response or null
+       */
+      processException: (error, request, middlewares) => Effect.reduce(
+        middlewares.slice().reverse(),
+        null,
+        (res, middleware) => middleware.processException ? middleware.processException(error, request) : Effect.succeed(res)
+      )
+    }))
+  }
+) {
+}
+class RateLimitMiddleware extends Effect.Service()(
+  "@jambudipa.io/RateLimitMiddleware",
+  {
+    effect: Effect.sync(() => {
+      const domainLastRequest = MutableHashMap.empty();
+      const domainRequestCount = MutableHashMap.empty();
+      const domainWindowStart = MutableHashMap.empty();
+      return {
+        create: (config) => ({
+          processRequest: (request) => Effect.gen(function* () {
+            const url = new URL(request.task.url);
+            const domain = url.hostname;
+            const now = Date.now();
+            if (config.requestDelayMs) {
+              yield* Effect.sleep(`${config.requestDelayMs} millis`);
+            }
+            const windowDuration = 1e3;
+            const windowStart = Option.getOrElse(
+              MutableHashMap.get(domainWindowStart, domain),
+              () => now
+            );
+            const currentCount = Option.getOrElse(
+              MutableHashMap.get(domainRequestCount, domain),
+              () => 0
+            );
+            if (now - windowStart >= windowDuration) {
+              MutableHashMap.set(domainWindowStart, domain, now);
+              MutableHashMap.set(domainRequestCount, domain, 0);
+            } else if (currentCount >= config.maxRequestsPerSecondPerDomain) {
+              const waitTime = windowDuration - (now - windowStart);
+              yield* Effect.sleep(`${waitTime} millis`);
+              MutableHashMap.set(domainWindowStart, domain, Date.now());
+              MutableHashMap.set(domainRequestCount, domain, 0);
+            }
+            const newCount = Option.getOrElse(
+              MutableHashMap.get(domainRequestCount, domain),
+              () => 0
+            ) + 1;
+            MutableHashMap.set(domainRequestCount, domain, newCount);
+            MutableHashMap.set(domainLastRequest, domain, Date.now());
+            yield* Effect.logDebug(
+              `Rate limit: ${domain} - ${newCount}/${config.maxRequestsPerSecondPerDomain} requests in window`
+            );
+            return request;
+          })
+        })
+      };
+    })
+  }
+) {
+}
+class LoggingMiddleware extends Effect.Service()(
+  "@jambudipa.io/LoggingMiddleware",
+  {
+    effect: Effect.sync(() => ({
+      create: (config = {}) => {
+        const {
+          logRequests = true,
+          logResponses = true,
+          logErrors = true,
+          logLevel = "info"
+        } = config;
+        return {
+          processRequest: (request) => Effect.gen(function* () {
+            if (logRequests) {
+              const logMessage = `Processing request: ${request.task.url} (depth: ${request.task.depth})`;
+              switch (logLevel) {
+                case "debug":
+                  yield* Effect.logDebug(logMessage);
+                  break;
+                case "info":
+                  yield* Effect.logInfo(logMessage);
+                  break;
+                case "warn":
+                  yield* Effect.logWarning(logMessage);
+                  break;
+                case "error":
+                  yield* Effect.logError(logMessage);
+                  break;
+              }
+            }
+            return request;
+          }),
+          processResponse: (response, request) => Effect.gen(function* () {
+            if (logResponses) {
+              const logMessage = `Received response: ${request.task.url} (status: ${response.statusCode || "unknown"}, size: ${response.pageData.html.length} bytes)`;
+              switch (logLevel) {
+                case "debug":
+                  yield* Effect.logDebug(logMessage);
+                  break;
+                case "info":
+                  yield* Effect.logInfo(logMessage);
+                  break;
+                case "warn":
+                  yield* Effect.logWarning(logMessage);
+                  break;
+                case "error":
+                  yield* Effect.logError(logMessage);
+                  break;
+              }
+            }
+            return response;
+          }),
+          processException: (error, request) => Effect.gen(function* () {
+            if (logErrors) {
+              const logMessage = `Error processing request: ${request.task.url} - ${error.message}`;
+              yield* Effect.logError(logMessage);
+            }
+            return null;
+          })
+        };
+      }
+    }))
+  }
+) {
+}
+class UserAgentMiddleware extends Effect.Service()(
+  "@jambudipa.io/UserAgentMiddleware",
+  {
+    effect: Effect.sync(() => ({
+      create: (userAgent) => ({
+        processRequest: (request) => Effect.succeed({
+          ...request,
+          headers: {
+            ...request.headers,
+            "User-Agent": userAgent
+          }
+        })
+      })
+    }))
+  }
+) {
+}
+class StatsMiddleware extends Effect.Service()(
+  "@jambudipa.io/StatsMiddleware",
+  {
+    effect: Effect.sync(() => ({
+      create: () => {
+        const stats = MutableHashMap.empty();
+        const startTime = Date.now();
+        const incr = (key, count = 1) => {
+          const current = Option.getOrElse(
+            MutableHashMap.get(stats, key),
+            () => 0
+          );
+          MutableHashMap.set(stats, key, current + count);
+        };
+        return {
+          middleware: {
+            processRequest: (request) => Effect.sync(() => {
+              incr("requests_processed");
+              incr(`requests_depth_${request.task.depth}`);
+              return request;
+            }),
+            processResponse: (response) => Effect.sync(() => {
+              incr("responses_received");
+              if (response.statusCode) {
+                incr(`status_${response.statusCode}`);
+                if (response.statusCode >= 200 && response.statusCode < 300) {
+                  incr("responses_success");
+                } else if (response.statusCode >= 400) {
+                  incr("responses_error");
+                }
+              }
+              incr("bytes_downloaded", response.pageData.html.length);
+              return response;
+            }),
+            processException: (error) => Effect.sync(() => {
+              incr("exceptions");
+              incr(`exception_${error.constructor.name}`);
+              return null;
+            })
+          },
+          getStats: () => Effect.sync(() => ({
+            ...Object.fromEntries(Array.from(stats)),
+            runtime_seconds: (Date.now() - startTime) / 1e3
+          }))
+        };
+      }
+    }))
+  }
+) {
+}
+class StateDelta extends Schema.Class("StateDelta")({
+  /** Session this delta applies to */
+  stateKey: Schema.String,
+  /** Sequence number for ordering deltas */
+  sequence: Schema.Number,
+  /** When this delta was created */
+  timestamp: Schema.Date,
+  /** The operation that created this delta */
+  operation: Schema.Union(
+    Schema.Struct({
+      type: Schema.Literal("enqueue"),
+      request: PriorityRequest
+    }),
+    Schema.Struct({
+      type: Schema.Literal("dequeue"),
+      fingerprint: Schema.String
+    }),
+    Schema.Struct({
+      type: Schema.Literal("mark_visited"),
+      fingerprint: Schema.String
+    })
+  )
+}) {
+}
+class PersistenceError2 extends Data.TaggedError("PersistenceError") {
+}
+const DEFAULT_HYBRID_CONFIG = {
+  snapshotInterval: 1e3,
+  maxDeltasBeforeSnapshot: 500,
+  compactionEnabled: true,
+  batchDeltas: true,
+  deltaBatchSize: 10
+};
+class FullStatePersistence {
+  constructor(backend) {
+    this.backend = backend;
+  }
+  persist = (operation) => {
+    const self = this;
+    return Effect.gen(function* () {
+      if (!self.backend.saveState) {
+        return yield* Effect.fail(
+          new PersistenceError2({
+            message: `Backend ${self.backend.name} does not support full state persistence`,
+            operation: "persist"
+          })
+        );
+      }
+      yield* self.backend.saveState(
+        operation.resultingState.key,
+        operation.resultingState
+      );
+    });
+  };
+  restore = (key) => {
+    const self = this;
+    return Effect.gen(function* () {
+      if (!self.backend.loadState) {
+        return yield* Effect.fail(
+          new PersistenceError2({
+            message: `Backend ${self.backend.name} does not support state loading`,
+            operation: "restore"
+          })
+        );
+      }
+      return yield* self.backend.loadState(key);
+    });
+  };
+  cleanup = (key) => {
+    const self = this;
+    return Effect.gen(function* () {
+      if (!self.backend.deleteState) {
+        return yield* Effect.fail(
+          new PersistenceError2({
+            message: `Backend ${self.backend.name} does not support state deletion`,
+            operation: "cleanup"
+          })
+        );
+      }
+      yield* self.backend.deleteState(key);
+    });
+  };
+  getInfo = () => ({
+    name: "FullStatePersistence",
+    description: "Saves complete state on every operation. Simple but potentially inefficient for large crawls.",
+    capabilities: ["full-state-save", "full-state-restore", "simple-cleanup"]
+  });
+}
+class DeltaPersistence {
+  constructor(backend) {
+    this.backend = backend;
+  }
+  persist = (operation) => {
+    const self = this;
+    return Effect.gen(function* () {
+      if (!self.backend.saveDelta) {
+        return yield* Effect.fail(
+          new PersistenceError2({
+            message: `Backend ${self.backend.name} does not support delta persistence`,
+            operation: "persist"
+          })
+        );
+      }
+      yield* self.backend.saveDelta(operation.delta);
+    });
+  };
+  restore = (key) => {
+    const self = this;
+    return Effect.gen(function* () {
+      if (!self.backend.loadDeltas) {
+        return yield* Effect.fail(
+          new PersistenceError2({
+            message: `Backend ${self.backend.name} does not support delta loading`,
+            operation: "restore"
+          })
+        );
+      }
+      const deltas = yield* self.backend.loadDeltas(key);
+      if (deltas.length === 0) {
+        return null;
+      }
+      return yield* self.reconstructStateFromDeltas(key, deltas);
+    });
+  };
+  cleanup = (key) => {
+    const self = this;
+    return Effect.gen(function* () {
+      if (!self.backend.loadDeltas || !self.backend.compactDeltas) {
+        return yield* Effect.fail(
+          new PersistenceError2({
+            message: `Backend ${self.backend.name} does not support delta cleanup`,
+            operation: "cleanup"
+          })
+        );
+      }
+      const deltas = yield* self.backend.loadDeltas(key);
+      if (deltas.length > 0) {
+        const maxSequence = Math.max(...deltas.map((d) => d.sequence));
+        yield* self.backend.compactDeltas(key, maxSequence + 1);
+      }
+    });
+  };
+  reconstructStateFromDeltas = (key, deltas) => Effect.gen(function* () {
+    const sortedDeltas = [...deltas].sort((a, b) => a.sequence - b.sequence);
+    const pendingRequests = [];
+    const visitedFingerprints = [];
+    let totalProcessed = 0;
+    for (const delta of sortedDeltas) {
+      switch (delta.operation.type) {
+        case "enqueue":
+          pendingRequests.push(delta.operation.request);
+          break;
+        case "dequeue": {
+          const operation = delta.operation;
+          if (operation.type === "dequeue") {
+            const dequeueIndex = pendingRequests.findIndex(
+              (req) => req.fingerprint === operation.fingerprint
+            );
+            if (dequeueIndex >= 0) {
+              pendingRequests.splice(dequeueIndex, 1);
+              totalProcessed++;
+            }
+          }
+          break;
+        }
+        case "mark_visited": {
+          const operation = delta.operation;
+          if (operation.type === "mark_visited") {
+            if (!visitedFingerprints.includes(operation.fingerprint)) {
+              visitedFingerprints.push(operation.fingerprint);
+            }
+          }
+          break;
+        }
+      }
+    }
+    return yield* Effect.tryPromise({
+      try: async () => {
+        const { SpiderState: SpiderState2 } = await Promise.resolve().then(() => SpiderScheduler_service);
+        return new SpiderState2({
+          key,
+          pendingRequests,
+          visitedFingerprints,
+          totalProcessed
+        });
+      },
+      catch: (error) => new PersistenceError2({
+        message: "Failed to import SpiderState",
+        cause: error,
+        operation: "reconstructStateFromDeltas"
+      })
+    });
+  });
+  getInfo = () => ({
+    name: "DeltaPersistence",
+    description: "Saves only incremental changes. Efficient for large crawls but requires delta replay.",
+    capabilities: ["delta-save", "delta-restore", "state-reconstruction"]
+  });
+}
+class HybridPersistence {
+  constructor(backend, config = DEFAULT_HYBRID_CONFIG) {
+    this.backend = backend;
+    this.config = config;
+  }
+  operationCount = 0;
+  lastSnapshotSequence = 0;
+  pendingDeltas = [];
+  persist = (operation) => {
+    const self = this;
+    return Effect.gen(function* () {
+      self.operationCount++;
+      if (self.config.batchDeltas) {
+        self.pendingDeltas.push(operation.delta);
+      }
+      const shouldSnapshot = operation.shouldSnapshot || self.operationCount % self.config.snapshotInterval === 0 || self.operationCount - self.lastSnapshotSequence >= self.config.maxDeltasBeforeSnapshot;
+      if (shouldSnapshot) {
+        yield* self.saveSnapshot(operation);
+      } else {
+        yield* self.saveDelta(operation);
+      }
+      if (self.config.batchDeltas && self.pendingDeltas.length >= self.config.deltaBatchSize) {
+        yield* self.flushPendingDeltas();
+      }
+    });
+  };
+  saveSnapshot = (operation) => {
+    const self = this;
+    return Effect.gen(function* () {
+      if (!self.backend.saveSnapshot) {
+        return yield* Effect.fail(
+          new PersistenceError2({
+            message: `Backend ${self.backend.name} does not support snapshots`,
+            operation: "saveSnapshot"
+          })
+        );
+      }
+      yield* self.backend.saveSnapshot(
+        operation.resultingState.key,
+        operation.resultingState,
+        operation.delta.sequence
+      );
+      self.lastSnapshotSequence = operation.delta.sequence;
+      if (self.config.compactionEnabled && self.backend.compactDeltas) {
+        yield* self.backend.compactDeltas(
+          operation.resultingState.key,
+          operation.delta.sequence
+        );
+      }
+      self.pendingDeltas = [];
+    });
+  };
+  saveDelta = (operation) => {
+    const self = this;
+    return Effect.gen(function* () {
+      if (!self.config.batchDeltas) {
+        if (!self.backend.saveDelta) {
+          return yield* Effect.fail(
+            new PersistenceError2({
+              message: `Backend ${self.backend.name} does not support delta persistence`,
+              operation: "saveDelta"
+            })
+          );
+        }
+        yield* self.backend.saveDelta(operation.delta);
+      }
+    });
+  };
+  flushPendingDeltas = () => {
+    const self = this;
+    return Effect.gen(function* () {
+      if (self.pendingDeltas.length === 0) return;
+      if (self.backend.saveDeltas) {
+        yield* self.backend.saveDeltas([...self.pendingDeltas]);
+      } else if (self.backend.saveDelta) {
+        for (const delta of self.pendingDeltas) {
+          yield* self.backend.saveDelta(delta);
+        }
+      } else {
+        return yield* Effect.fail(
+          new PersistenceError2({
+            message: `Backend ${self.backend.name} does not support delta persistence`,
+            operation: "flushPendingDeltas"
+          })
+        );
+      }
+      self.pendingDeltas = [];
+    });
+  };
+  restore = (key) => {
+    const self = this;
+    return Effect.gen(function* () {
+      let baseState = null;
+      let fromSequence = 0;
+      if (self.backend.loadLatestSnapshot) {
+        const snapshot = yield* self.backend.loadLatestSnapshot(key);
+        if (snapshot) {
+          baseState = snapshot.state;
+          fromSequence = snapshot.sequence + 1;
+        }
+      }
+      if (!self.backend.loadDeltas) {
+        if (baseState) {
+          return baseState;
+        }
+        return yield* Effect.fail(
+          new PersistenceError2({
+            message: `Backend ${self.backend.name} does not support delta loading`,
+            operation: "restore"
+          })
+        );
+      }
+      const deltas = yield* self.backend.loadDeltas(key, fromSequence);
+      if (!baseState && deltas.length === 0) {
+        return null;
+      }
+      if (deltas.length === 0) {
+        return baseState;
+      }
+      return yield* self.applyDeltasToState(key, baseState, deltas);
+    });
+  };
+  applyDeltasToState = (key, baseState, deltas) => {
+    const self = this;
+    return Effect.gen(function* () {
+      if (!baseState) {
+        const deltaStrategy = new DeltaPersistence(self.backend);
+        return yield* deltaStrategy.reconstructStateFromDeltas(key, deltas);
+      }
+      const sortedDeltas = [...deltas].sort((a, b) => a.sequence - b.sequence);
+      const pendingRequests = [...baseState.pendingRequests];
+      const visitedFingerprints = [...baseState.visitedFingerprints];
+      let totalProcessed = baseState.totalProcessed;
+      for (const delta of sortedDeltas) {
+        switch (delta.operation.type) {
+          case "enqueue":
+            pendingRequests.push(delta.operation.request);
+            break;
+          case "dequeue": {
+            const operation = delta.operation;
+            if (operation.type === "dequeue") {
+              const dequeueIndex = pendingRequests.findIndex(
+                (req) => req.fingerprint === operation.fingerprint
+              );
+              if (dequeueIndex >= 0) {
+                pendingRequests.splice(dequeueIndex, 1);
+                totalProcessed++;
+              }
+            }
+            break;
+          }
+          case "mark_visited": {
+            const operation = delta.operation;
+            if (operation.type === "mark_visited") {
+              if (!visitedFingerprints.includes(operation.fingerprint)) {
+                visitedFingerprints.push(operation.fingerprint);
+              }
+            }
+            break;
+          }
+        }
+      }
+      return yield* Effect.tryPromise({
+        try: async () => {
+          const { SpiderState: SpiderState2 } = await Promise.resolve().then(() => SpiderScheduler_service);
+          return new SpiderState2({
+            key,
+            pendingRequests,
+            visitedFingerprints,
+            totalProcessed
+          });
+        },
+        catch: (error) => new PersistenceError2({
+          message: "Failed to import SpiderState",
+          cause: error,
+          operation: "applyDeltasToState"
+        })
+      });
+    });
+  };
+  cleanup = (key) => {
+    const self = this;
+    return Effect.gen(function* () {
+      yield* self.flushPendingDeltas();
+      if (self.backend.deleteState) {
+        yield* self.backend.deleteState(key);
+      }
+      if (self.backend.compactDeltas) {
+        yield* self.backend.compactDeltas(key, Number.MAX_SAFE_INTEGER);
+      }
+    });
+  };
+  getInfo = () => ({
+    name: "HybridPersistence",
+    description: "Combines deltas and snapshots for optimal performance and recovery speed.",
+    capabilities: [
+      "delta-save",
+      "snapshot-save",
+      "batch-deltas",
+      "fast-recovery",
+      "automatic-compaction"
+    ]
+  });
+}
+class ResumabilityService extends Effect.Service()(
+  "@jambudipa.io/ResumabilityService",
+  {
+    effect: Effect.gen(function* () {
+      let strategy = null;
+      let backend = null;
+      const service = {
+        /**
+         * Configure the resumability service with a specific strategy and backend.
+         *
+         * This method initializes the storage backend and creates the appropriate
+         * persistence strategy based on the configuration.
+         *
+         * @param config - Resumability configuration
+         * @returns Effect that completes when configuration is applied
+         */
+        configure: (config) => Effect.gen(function* () {
+          backend = config.backend;
+          yield* backend.initialize();
+          strategy = yield* createStrategy(config);
+        }),
+        /**
+         * Persist a state operation using the configured strategy.
+         *
+         * @param operation - State operation to persist
+         * @returns Effect that completes when operation is persisted
+         */
+        persistOperation: (operation) => Effect.gen(function* () {
+          if (!strategy) {
+            return yield* Effect.fail(
+              new PersistenceError2({
+                message: "ResumabilityService not configured. Call configure() first.",
+                operation: "persistOperation"
+              })
+            );
+          }
+          yield* strategy.persist(operation);
+        }),
+        /**
+         * Restore spider state from persistent storage.
+         *
+         * @param key - State key identifying the session to restore
+         * @returns Effect containing the restored state, or null if not found
+         */
+        restore: (key) => Effect.gen(function* () {
+          if (!strategy) {
+            return yield* Effect.fail(
+              new PersistenceError2({
+                message: "ResumabilityService not configured. Call configure() first.",
+                operation: "restore"
+              })
+            );
+          }
+          return yield* strategy.restore(key);
+        }),
+        /**
+         * Clean up old state data for a session.
+         *
+         * @param key - State key identifying the session to clean up
+         * @returns Effect that completes when cleanup is finished
+         */
+        cleanup: (key) => Effect.gen(function* () {
+          if (!strategy) {
+            return yield* Effect.fail(
+              new PersistenceError2({
+                message: "ResumabilityService not configured. Call configure() first.",
+                operation: "cleanup"
+              })
+            );
+          }
+          yield* strategy.cleanup(key);
+        }),
+        /**
+         * List all available sessions in storage.
+         *
+         * @returns Effect containing array of session keys
+         */
+        listSessions: () => Effect.gen(function* () {
+          if (!backend) {
+            return yield* Effect.fail(
+              new PersistenceError2({
+                message: "ResumabilityService not configured. Call configure() first.",
+                operation: "listSessions"
+              })
+            );
+          }
+          if (!backend.listSessions) {
+            return yield* Effect.fail(
+              new PersistenceError2({
+                message: `Backend ${backend.name} does not support listing sessions`,
+                operation: "listSessions"
+              })
+            );
+          }
+          return yield* backend.listSessions();
+        }),
+        /**
+         * Get information about the current configuration.
+         *
+         * @returns Information about strategy and backend
+         */
+        getInfo: () => Effect.gen(function* () {
+          if (!strategy || !backend) {
+            return yield* Effect.fail(
+              new PersistenceError2({
+                message: "ResumabilityService not configured. Call configure() first.",
+                operation: "getInfo"
+              })
+            );
+          }
+          return {
+            strategy: strategy.getInfo(),
+            backend: {
+              name: backend.name,
+              capabilities: backend.capabilities
+            }
+          };
+        }),
+        /**
+         * Reconfigure the service with new settings.
+         *
+         * This will clean up the current backend and reinitialize with new config.
+         *
+         * @param config - New configuration
+         * @returns Effect that completes when reconfiguration is finished
+         */
+        reconfigure: (config) => Effect.gen(function* () {
+          if (backend) {
+            yield* backend.cleanup();
+          }
+          yield* service.configure(config);
+        })
+      };
+      return service;
+    })
+  }
+) {
+  /**
+   * Create a ResumabilityService layer from configuration.
+   *
+   * This is the primary way to create and configure the ResumabilityService.
+   *
+   * @param config - Resumability configuration
+   * @returns Effect layer providing the configured ResumabilityService
+   */
+  static fromConfig = (config) => Effect.gen(function* () {
+    const service = yield* ResumabilityService;
+    yield* service.configure(config);
+    return service;
+  }).pipe(Effect.provide(ResumabilityService.Default));
+}
+const createStrategy = (config) => Effect.gen(function* () {
+  const { strategy: strategyType, backend, hybridConfig } = config;
+  switch (strategyType) {
+    case "full-state":
+      return new FullStatePersistence(backend);
+    case "delta":
+      return new DeltaPersistence(backend);
+    case "hybrid":
+      return new HybridPersistence(
+        backend,
+        hybridConfig || DEFAULT_HYBRID_CONFIG
+      );
+    case "auto":
+      const capabilities = backend.capabilities;
+      if (capabilities.supportsDelta && capabilities.supportsSnapshot) {
+        return new HybridPersistence(
+          backend,
+          hybridConfig || DEFAULT_HYBRID_CONFIG
+        );
+      } else if (capabilities.supportsDelta) {
+        return new DeltaPersistence(backend);
+      } else {
+        return new FullStatePersistence(backend);
+      }
+    default:
+      return yield* Effect.fail(
+        new PersistenceError2({
+          message: `Unknown strategy type: ${strategyType}`,
+          operation: "createStrategy"
+        })
+      );
+  }
+});
+const createStateOperation = (delta, resultingState, shouldSnapshot = false) => ({
+  delta,
+  resultingState,
+  shouldSnapshot
+});
+const ResumabilityConfigs = {
+  /**
+   * Create a file-based configuration.
+   *
+   * @param baseDir - Directory to store state files
+   * @param strategy - Persistence strategy (defaults to 'auto')
+   * @returns ResumabilityConfig
+   */
+  file: (baseDir, strategy = "auto") => ({
+    strategy,
+    backend: new (require("./backends/FileStorageBackend.js")).FileStorageBackend(
+      baseDir
+    )
+  }),
+  /**
+   * Create a Redis-based configuration.
+   *
+   * @param redisClient - Redis client instance
+   * @param strategy - Persistence strategy (defaults to 'hybrid')
+   * @param keyPrefix - Redis key prefix (defaults to 'spider')
+   * @returns ResumabilityConfig
+   */
+  redis: (redisClient, strategy = "hybrid", keyPrefix = "spider") => ({
+    strategy,
+    backend: new (require("./backends/RedisStorageBackend.js")).RedisStorageBackend(
+      redisClient,
+      keyPrefix
+    )
+  }),
+  /**
+   * Create a PostgreSQL-based configuration.
+   *
+   * @param dbClient - Database client instance
+   * @param strategy - Persistence strategy (defaults to 'hybrid')
+   * @param config - PostgreSQL configuration
+   * @returns ResumabilityConfig
+   */
+  postgres: (dbClient, strategy = "hybrid", config) => ({
+    strategy,
+    backend: new (require("./backends/PostgresStorageBackend.js")).PostgresStorageBackend(
+      dbClient,
+      config
+    )
+  })
+};
+class FileStorageBackend {
+  constructor(baseDir) {
+    this.baseDir = baseDir;
+  }
+  capabilities = {
+    supportsDelta: true,
+    supportsSnapshot: true,
+    supportsStreaming: false,
+    supportsConcurrency: false,
+    // File system isn't great for concurrent access
+    latency: "low"
+  };
+  name = "FileStorageBackend";
+  initialize = () => {
+    const self = this;
+    return Effect.gen(function* () {
+      yield* Effect.tryPromise({
+        try: () => fs$1.mkdir(self.baseDir, { recursive: true }),
+        catch: (error) => new PersistenceError2({
+          message: `Failed to initialize file storage: ${error}`,
+          cause: error,
+          operation: "initialize"
+        })
+      });
+      yield* Effect.tryPromise({
+        try: () => fs$1.mkdir(path.join(self.baseDir, "sessions"), { recursive: true }),
+        catch: (error) => new PersistenceError2({
+          message: `Failed to initialize file storage: ${error}`,
+          cause: error,
+          operation: "initialize"
+        })
+      });
+    });
+  };
+  cleanup = () => Effect.succeed(void 0);
+  // No cleanup needed for file backend
+  // Full state operations
+  saveState = (key, state) => {
+    const self = this;
+    return Effect.gen(function* () {
+      const sessionDir = self.getSessionDir(key);
+      const statePath = path.join(sessionDir, "state.json");
+      yield* Effect.tryPromise({
+        try: () => fs$1.mkdir(sessionDir, { recursive: true }),
+        catch: (error) => new PersistenceError2({
+          message: `Failed to create session directory: ${error}`,
+          cause: error,
+          operation: "saveState"
+        })
+      });
+      const encoded = Schema.encodeSync(SpiderState)(state);
+      yield* Effect.tryPromise({
+        try: () => fs$1.writeFile(statePath, JSON.stringify(encoded, null, 2), "utf8"),
+        catch: (error) => new PersistenceError2({
+          message: `Failed to save state: ${error}`,
+          cause: error,
+          operation: "saveState"
+        })
+      });
+    });
+  };
+  loadState = (key) => {
+    const self = this;
+    return Effect.gen(function* () {
+      const sessionDir = self.getSessionDir(key);
+      const statePath = path.join(sessionDir, "state.json");
+      const result = yield* Effect.tryPromise(
+        () => fs$1.readFile(statePath, "utf8")
+      ).pipe(
+        Effect.catchAll((error) => {
+          if (error.code === "ENOENT") {
+            return Effect.succeed(null);
+          }
+          return Effect.fail(
+            new PersistenceError2({
+              message: `Failed to load state: ${error}`,
+              cause: error,
+              operation: "loadState"
+            })
+          );
+        })
+      );
+      if (result === null) {
+        return null;
+      }
+      try {
+        const parsed = JSON.parse(result);
+        const decoded = Schema.decodeUnknownSync(SpiderState)(parsed);
+        return decoded;
+      } catch (error) {
+        return yield* Effect.fail(
+          new PersistenceError2({
+            message: `Failed to parse state: ${error}`,
+            cause: error,
+            operation: "loadState"
+          })
+        );
+      }
+    });
+  };
+  deleteState = (key) => {
+    const self = this;
+    return Effect.gen(function* () {
+      const sessionDir = self.getSessionDir(key);
+      yield* Effect.tryPromise({
+        try: () => fs$1.rm(sessionDir, { recursive: true, force: true }),
+        catch: (error) => new PersistenceError2({
+          message: `Failed to delete state: ${error}`,
+          cause: error,
+          operation: "deleteState"
+        })
+      });
+    });
+  };
+  // Delta operations
+  saveDelta = (delta) => {
+    const self = this;
+    return Effect.gen(function* () {
+      const sessionDir = path.join(self.baseDir, "sessions", delta.stateKey);
+      const deltasDir = path.join(sessionDir, "deltas");
+      const deltaPath = path.join(
+        deltasDir,
+        `${delta.sequence.toString().padStart(6, "0")}.json`
+      );
+      yield* Effect.tryPromise({
+        try: () => fs$1.mkdir(deltasDir, { recursive: true }),
+        catch: (error) => new PersistenceError2({
+          message: `Failed to create deltas directory: ${error}`,
+          cause: error,
+          operation: "saveDelta"
+        })
+      });
+      const encoded = Schema.encodeSync(StateDelta)(delta);
+      yield* Effect.tryPromise({
+        try: () => fs$1.writeFile(deltaPath, JSON.stringify(encoded, null, 2), "utf8"),
+        catch: (error) => new PersistenceError2({
+          message: `Failed to save delta: ${error}`,
+          cause: error,
+          operation: "saveDelta"
+        })
+      });
+    });
+  };
+  saveDeltas = (deltas) => {
+    const self = this;
+    return Effect.gen(function* () {
+      for (const delta of deltas) {
+        yield* self.saveDelta(delta);
+      }
+    });
+  };
+  loadDeltas = (key, fromSequence = 0) => {
+    const self = this;
+    return Effect.gen(function* () {
+      const deltasDir = path.join(self.getSessionDir(key), "deltas");
+      const files = yield* Effect.tryPromise(() => fs$1.readdir(deltasDir)).pipe(
+        Effect.catchAll((error) => {
+          if (error.code === "ENOENT") {
+            return Effect.succeed([]);
+          }
+          return Effect.fail(
+            new PersistenceError2({
+              message: `Failed to read deltas directory: ${error}`,
+              cause: error,
+              operation: "loadDeltas"
+            })
+          );
+        })
+      );
+      if (files.length === 0) {
+        return [];
+      }
+      const deltaFiles = files.filter((f) => f.endsWith(".json")).map((f) => ({
+        file: f,
+        sequence: parseInt(f.replace(".json", ""), 10)
+      })).filter(({ sequence }) => sequence >= fromSequence).sort((a, b) => a.sequence - b.sequence);
+      const deltas = [];
+      for (const { file } of deltaFiles) {
+        const content = yield* Effect.tryPromise({
+          try: () => fs$1.readFile(path.join(deltasDir, file), "utf8"),
+          catch: (error) => new PersistenceError2({
+            message: `Failed to read delta file ${file}: ${error}`,
+            cause: error,
+            operation: "loadDeltas"
+          })
+        });
+        try {
+          const parsed = JSON.parse(content);
+          const decoded = Schema.decodeUnknownSync(StateDelta)(parsed);
+          deltas.push(decoded);
+        } catch (error) {
+          return yield* Effect.fail(
+            new PersistenceError2({
+              message: `Failed to parse delta file ${file}: ${error}`,
+              cause: error,
+              operation: "loadDeltas"
+            })
+          );
+        }
+      }
+      return deltas;
+    });
+  };
+  // Snapshot operations
+  saveSnapshot = (key, state, sequence) => {
+    const self = this;
+    return Effect.gen(function* () {
+      const sessionDir = self.getSessionDir(key);
+      const snapshotPath = path.join(sessionDir, "snapshot.json");
+      yield* Effect.tryPromise({
+        try: () => fs$1.mkdir(sessionDir, { recursive: true }),
+        catch: (error) => new PersistenceError2({
+          message: `Failed to create session directory: ${error}`,
+          cause: error,
+          operation: "saveSnapshot"
+        })
+      });
+      const snapshotData = {
+        state: Schema.encodeSync(SpiderState)(state),
+        sequence,
+        timestamp: (/* @__PURE__ */ new Date()).toISOString()
+      };
+      yield* Effect.tryPromise({
+        try: () => fs$1.writeFile(
+          snapshotPath,
+          JSON.stringify(snapshotData, null, 2),
+          "utf8"
+        ),
+        catch: (error) => new PersistenceError2({
+          message: `Failed to save snapshot: ${error}`,
+          cause: error,
+          operation: "saveSnapshot"
+        })
+      });
+    });
+  };
+  loadLatestSnapshot = (key) => {
+    const self = this;
+    return Effect.gen(function* () {
+      const sessionDir = self.getSessionDir(key);
+      const snapshotPath = path.join(sessionDir, "snapshot.json");
+      const content = yield* Effect.tryPromise(
+        () => fs$1.readFile(snapshotPath, "utf8")
+      ).pipe(
+        Effect.catchAll((error) => {
+          if (error.code === "ENOENT") {
+            return Effect.succeed(null);
+          }
+          return Effect.fail(
+            new PersistenceError2({
+              message: `Failed to load snapshot: ${error}`,
+              cause: error,
+              operation: "loadLatestSnapshot"
+            })
+          );
+        })
+      );
+      if (content === null) {
+        return null;
+      }
+      try {
+        const parsed = JSON.parse(content);
+        const state = Schema.decodeUnknownSync(SpiderState)(parsed.state);
+        return {
+          state,
+          sequence: Number(parsed.sequence)
+        };
+      } catch (error) {
+        return yield* Effect.fail(
+          new PersistenceError2({
+            message: `Failed to parse snapshot: ${error}`,
+            cause: error,
+            operation: "loadLatestSnapshot"
+          })
+        );
+      }
+    });
+  };
+  // Cleanup operations
+  compactDeltas = (key, beforeSequence) => {
+    const self = this;
+    return Effect.gen(function* () {
+      const deltasDir = path.join(self.getSessionDir(key), "deltas");
+      const files = yield* Effect.tryPromise(() => fs$1.readdir(deltasDir)).pipe(
+        Effect.catchAll((error) => {
+          if (error.code === "ENOENT") {
+            return Effect.succeed([]);
+          }
+          return Effect.fail(
+            new PersistenceError2({
+              message: `Failed to read deltas directory: ${error}`,
+              cause: error,
+              operation: "compactDeltas"
+            })
+          );
+        })
+      );
+      if (files.length === 0) {
+        return;
+      }
+      const deltaFiles = files.filter((f) => f.endsWith(".json")).map((f) => ({
+        file: f,
+        sequence: parseInt(f.replace(".json", ""), 10)
+      })).filter(({ sequence }) => sequence < beforeSequence);
+      for (const { file } of deltaFiles) {
+        yield* Effect.tryPromise({
+          try: () => fs$1.unlink(path.join(deltasDir, file)),
+          catch: (error) => new PersistenceError2({
+            message: `Failed to delete delta file ${file}: ${error}`,
+            cause: error,
+            operation: "compactDeltas"
+          })
+        });
+      }
+    });
+  };
+  listSessions = () => {
+    const self = this;
+    return Effect.gen(function* () {
+      const sessionsDir = path.join(self.baseDir, "sessions");
+      const dirs = yield* Effect.tryPromise(() => fs$1.readdir(sessionsDir)).pipe(
+        Effect.catchAll((error) => {
+          if (error.code === "ENOENT") {
+            return Effect.succeed([]);
+          }
+          return Effect.fail(
+            new PersistenceError2({
+              message: `Failed to read sessions directory: ${error}`,
+              cause: error,
+              operation: "listSessions"
+            })
+          );
+        })
+      );
+      if (dirs.length === 0) {
+        return [];
+      }
+      const sessions = [];
+      for (const dir of dirs) {
+        const sessionDir = path.join(sessionsDir, dir);
+        const statePath = path.join(sessionDir, "state.json");
+        const content = yield* Effect.tryPromise(
+          () => fs$1.readFile(statePath, "utf8")
+        ).pipe(Effect.catchAll(() => Effect.succeed(null)));
+        if (content === null) {
+          continue;
+        }
+        try {
+          const parsed = JSON.parse(content);
+          Schema.decodeUnknownSync(SpiderState)(parsed);
+          sessions.push({ id: dir, name: dir, timestamp: /* @__PURE__ */ new Date() });
+        } catch {
+          continue;
+        }
+      }
+      return sessions;
+    });
+  };
+  getSessionDir = (key) => {
+    return path.join(this.baseDir, "sessions", key.id);
+  };
+}
+class CookieManager extends Context.Tag("CookieManager")() {
+}
+const makeCookieManager = () => Effect.gen(function* () {
+  const jar = new CookieJar();
+  const jarRef = yield* Ref.make(jar);
+  return {
+    setCookie: (cookieString, url) => Effect.gen(function* () {
+      const jar2 = yield* Ref.get(jarRef);
+      yield* Effect.tryPromise({
+        try: () => new Promise((resolve, reject) => {
+          jar2.setCookie(cookieString, url, (err) => {
+            if (err) reject(err);
+            else resolve();
+          });
+        }),
+        catch: (error) => new Error(`Failed to set cookie: ${error}`)
+      });
+    }),
+    getCookies: (url) => Effect.gen(function* () {
+      const jar2 = yield* Ref.get(jarRef);
+      const cookies = yield* Effect.tryPromise({
+        try: () => new Promise((resolve, reject) => {
+          jar2.getCookies(url, (err, cookies2) => {
+            if (err) reject(err);
+            else resolve(cookies2 || []);
+          });
+        }),
+        catch: () => new Error(`Failed to get cookies for ${url}`)
+      });
+      return cookies.map((cookie) => cookie.toString());
+    }).pipe(Effect.orElseSucceed(() => [])),
+    getCookieHeader: (url) => Effect.gen(function* () {
+      const jar2 = yield* Ref.get(jarRef);
+      const cookieHeader = yield* Effect.tryPromise({
+        try: () => new Promise((resolve, reject) => {
+          jar2.getCookieString(url, (err, cookies) => {
+            if (err) reject(err);
+            else resolve(cookies || null);
+          });
+        }),
+        catch: () => null
+      });
+      return cookieHeader;
+    }).pipe(Effect.orElseSucceed(() => null)),
+    clearCookies: () => Effect.gen(function* () {
+      const newJar = new CookieJar();
+      yield* Ref.set(jarRef, newJar);
+    }),
+    serialize: () => Effect.gen(function* () {
+      const jar2 = yield* Ref.get(jarRef);
+      const serialized = yield* Effect.tryPromise({
+        try: () => new Promise((resolve, reject) => {
+          jar2.serialize((err, serializedObject) => {
+            if (err) reject(err);
+            else resolve(serializedObject);
+          });
+        }),
+        catch: () => new Error("Failed to serialize cookies")
+      });
+      return JSON.stringify(serialized);
+    }).pipe(Effect.orElseSucceed(() => "{}")),
+    deserialize: (data) => Effect.gen(function* () {
+      try {
+        const parsed = JSON.parse(data);
+        const newJar = CookieJar.deserialize(parsed);
+        yield* Effect.tryPromise({
+          try: () => Promise.resolve(newJar),
+          catch: () => new Error("Failed to deserialize cookie jar")
+        }).pipe(Effect.flatMap((jar2) => Ref.set(jarRef, jar2)));
+      } catch (error) {
+        yield* Effect.fail(new Error(`Invalid cookie data: ${error}`));
+      }
+    })
+  };
+});
+const CookieManagerLive = Layer.effect(
+  CookieManager,
+  makeCookieManager()
+);
+class EnhancedHttpClient extends Context.Tag("EnhancedHttpClient")() {
+}
+const makeEnhancedHttpClient = Effect.gen(function* () {
+  const logger = yield* SpiderLogger;
+  const cookieManager = yield* CookieManager;
+  const makeRequest = (url, options = {}) => Effect.gen(function* () {
+    const startMs = Date.now();
+    const domain = new URL(url).hostname;
+    const cookieHeader = yield* cookieManager.getCookieHeader(url);
+    const headers = {
+      "User-Agent": "Mozilla/5.0 (compatible; Spider/1.0)",
+      ...options.headers
+    };
+    if (cookieHeader && !headers["Cookie"]) {
+      headers["Cookie"] = cookieHeader;
+    }
+    if (options.method === "POST" && options.body && !headers["Content-Type"]) {
+      if (typeof options.body === "string") {
+        try {
+          JSON.parse(options.body);
+          headers["Content-Type"] = "application/json";
+        } catch {
+          headers["Content-Type"] = "application/x-www-form-urlencoded";
+        }
+      } else if (options.body instanceof FormData) ;
+      else if (options.body instanceof URLSearchParams) {
+        headers["Content-Type"] = "application/x-www-form-urlencoded";
+      }
+    }
+    const controller = new AbortController();
+    const timeoutMs = options.timeout || 3e4;
+    const timeoutId = setTimeout(() => {
+      const duration = Date.now() - startMs;
+      Effect.runSync(
+        logger.logEdgeCase(domain, "http_request_abort", {
+          url,
+          method: options.method || "GET",
+          durationMs: duration,
+          reason: "timeout",
+          timeoutMs
+        })
+      );
+      controller.abort();
+    }, timeoutMs);
+    const response = yield* Effect.tryPromise({
+      try: async () => {
+        const resp = await fetch(url, {
+          method: options.method || "GET",
+          headers,
+          body: options.body,
+          signal: controller.signal,
+          redirect: options.followRedirects === false ? "manual" : "follow",
+          credentials: options.credentials || "same-origin"
+        });
+        clearTimeout(timeoutId);
+        return resp;
+      },
+      catch: (error) => {
+        clearTimeout(timeoutId);
+        return NetworkError.fromCause(url, error);
+      }
+    });
+    const body = yield* Effect.tryPromise({
+      try: () => response.text(),
+      catch: (error) => ResponseError.fromCause(url, error)
+    });
+    const setCookieHeaders = response.headers.getSetCookie ? response.headers.getSetCookie() : response.headers.get("set-cookie")?.split(", ") || [];
+    for (const cookieString of setCookieHeaders) {
+      if (cookieString) {
+        yield* cookieManager.setCookie(cookieString, url).pipe(Effect.catchAll(() => Effect.void));
+      }
+    }
+    const responseHeaders = {};
+    response.headers.forEach((value, key) => {
+      responseHeaders[key] = value;
+    });
+    return {
+      url: response.url,
+      status: response.status,
+      statusText: response.statusText,
+      headers: responseHeaders,
+      body,
+      cookies: setCookieHeaders
+    };
+  });
+  return {
+    get: (url, options) => makeRequest(url, { ...options, method: "GET" }),
+    post: (url, data, options) => Effect.gen(function* () {
+      let body;
+      if (data) {
+        if (typeof data === "string" || data instanceof FormData || data instanceof URLSearchParams) {
+          body = data;
+        } else {
+          body = JSON.stringify(data);
+        }
+      }
+      return yield* makeRequest(url, { ...options, method: "POST", body });
+    }),
+    request: makeRequest,
+    submitForm: (url, formData, options) => Effect.gen(function* () {
+      const params = new URLSearchParams();
+      for (const [key, value] of Object.entries(formData)) {
+        params.append(key, value);
+      }
+      return yield* makeRequest(url, {
+        ...options,
+        method: "POST",
+        body: params,
+        headers: {
+          "Content-Type": "application/x-www-form-urlencoded",
+          ...options?.headers
+        }
+      });
+    })
+  };
+});
+const EnhancedHttpClientLive = Layer.effect(
+  EnhancedHttpClient,
+  makeEnhancedHttpClient
+);
+class SessionStore extends Context.Tag("SessionStore")() {
+}
+const makeSessionStore = Effect.gen(function* () {
+  const cookieManager = yield* CookieManager;
+  const sessions = yield* Ref.make(/* @__PURE__ */ new Map());
+  const currentSessionId = yield* Ref.make(
+    Option.none()
+  );
+  const generateSessionId = () => `session_${Date.now()}_${Math.random().toString(36).substring(2, 9)}`;
+  return {
+    createSession: (id) => Effect.gen(function* () {
+      const sessionId = id || generateSessionId();
+      const cookiesString = yield* cookieManager.serialize();
+      const session = {
+        id: sessionId,
+        cookies: cookiesString,
+        tokens: /* @__PURE__ */ new Map(),
+        createdAt: /* @__PURE__ */ new Date(),
+        lastUsedAt: /* @__PURE__ */ new Date(),
+        expiresAt: new Date(Date.now() + 24 * 60 * 60 * 1e3)
+        // 24 hours
+      };
+      const sessionsMap = yield* Ref.get(sessions);
+      sessionsMap.set(sessionId, session);
+      yield* Ref.set(sessions, sessionsMap);
+      yield* Ref.set(currentSessionId, Option.some(sessionId));
+      return session;
+    }),
+    getCurrentSession: () => Effect.gen(function* () {
+      const sessionId = yield* Ref.get(currentSessionId);
+      if (Option.isNone(sessionId)) {
+        return Option.none();
+      }
+      const sessionsMap = yield* Ref.get(sessions);
+      const session = sessionsMap.get(sessionId.value);
+      if (!session) {
+        return Option.none();
+      }
+      session.lastUsedAt = /* @__PURE__ */ new Date();
+      sessionsMap.set(sessionId.value, session);
+      yield* Ref.set(sessions, sessionsMap);
+      return Option.some(session);
+    }),
+    loadSession: (id) => Effect.gen(function* () {
+      const sessionsMap = yield* Ref.get(sessions);
+      const session = sessionsMap.get(id);
+      if (!session) {
+        return yield* Effect.fail(new Error(`Session ${id} not found`));
+      }
+      if (session.expiresAt && session.expiresAt < /* @__PURE__ */ new Date()) {
+        return yield* Effect.fail(new Error(`Session ${id} has expired`));
+      }
+      yield* cookieManager.deserialize(session.cookies);
+      yield* Ref.set(currentSessionId, Option.some(id));
+      session.lastUsedAt = /* @__PURE__ */ new Date();
+      sessionsMap.set(id, session);
+      yield* Ref.set(sessions, sessionsMap);
+    }),
+    saveSession: () => Effect.gen(function* () {
+      const sessionId = yield* Ref.get(currentSessionId);
+      if (Option.isNone(sessionId)) {
+        const newSession = yield* Effect.sync(() => generateSessionId());
+        yield* Ref.set(currentSessionId, Option.some(newSession));
+        const session2 = yield* Effect.succeed({
+          id: newSession,
+          cookies: yield* cookieManager.serialize(),
+          tokens: /* @__PURE__ */ new Map(),
+          createdAt: /* @__PURE__ */ new Date(),
+          lastUsedAt: /* @__PURE__ */ new Date(),
+          expiresAt: new Date(Date.now() + 24 * 60 * 60 * 1e3)
+        });
+        const sessionsMap2 = yield* Ref.get(sessions);
+        sessionsMap2.set(newSession, session2);
+        yield* Ref.set(sessions, sessionsMap2);
+        return newSession;
+      }
+      const sessionsMap = yield* Ref.get(sessions);
+      const session = sessionsMap.get(sessionId.value);
+      if (!session) {
+        return yield* Effect.fail(new Error("No active session to save"));
+      }
+      session.cookies = yield* cookieManager.serialize();
+      session.lastUsedAt = /* @__PURE__ */ new Date();
+      sessionsMap.set(sessionId.value, session);
+      yield* Ref.set(sessions, sessionsMap);
+      return sessionId.value;
+    }),
+    clearSession: () => Effect.gen(function* () {
+      const sessionId = yield* Ref.get(currentSessionId);
+      if (Option.isSome(sessionId)) {
+        const sessionsMap = yield* Ref.get(sessions);
+        sessionsMap.delete(sessionId.value);
+        yield* Ref.set(sessions, sessionsMap);
+      }
+      yield* Ref.set(currentSessionId, Option.none());
+      yield* cookieManager.clearCookies();
+    }),
+    isSessionValid: () => Effect.gen(function* () {
+      const session = yield* Effect.gen(function* () {
+        const sessionId = yield* Ref.get(currentSessionId);
+        if (Option.isNone(sessionId)) return null;
+        const sessionsMap = yield* Ref.get(sessions);
+        return sessionsMap.get(sessionId.value) || null;
+      });
+      if (!session) return false;
+      if (session.expiresAt && session.expiresAt < /* @__PURE__ */ new Date()) {
+        return false;
+      }
+      return true;
+    }),
+    updateSessionData: (data) => Effect.gen(function* () {
+      const sessionId = yield* Ref.get(currentSessionId);
+      if (Option.isNone(sessionId)) {
+        return yield* Effect.fail(new Error("No active session"));
+      }
+      const sessionsMap = yield* Ref.get(sessions);
+      const session = sessionsMap.get(sessionId.value);
+      if (!session) {
+        return yield* Effect.fail(new Error("Session not found"));
+      }
+      session.userData = { ...session.userData, ...data };
+      session.lastUsedAt = /* @__PURE__ */ new Date();
+      sessionsMap.set(sessionId.value, session);
+      yield* Ref.set(sessions, sessionsMap);
+    }),
+    exportSession: () => Effect.gen(function* () {
+      const sessionId = yield* Ref.get(currentSessionId);
+      if (Option.isNone(sessionId)) {
+        return yield* Effect.fail(new Error("No active session to export"));
+      }
+      const sessionsMap = yield* Ref.get(sessions);
+      const session = sessionsMap.get(sessionId.value);
+      if (!session) {
+        return yield* Effect.fail(new Error("Session not found"));
+      }
+      const tokensArray = Array.from(session.tokens.entries());
+      return JSON.stringify({
+        ...session,
+        tokens: tokensArray
+      });
+    }),
+    importSession: (data) => Effect.gen(function* () {
+      try {
+        const parsed = JSON.parse(data);
+        const session = {
+          ...parsed,
+          tokens: new Map(parsed.tokens || []),
+          createdAt: new Date(parsed.createdAt),
+          lastUsedAt: new Date(parsed.lastUsedAt),
+          expiresAt: parsed.expiresAt ? new Date(parsed.expiresAt) : void 0
+        };
+        const sessionsMap = yield* Ref.get(sessions);
+        sessionsMap.set(session.id, session);
+        yield* Ref.set(sessions, sessionsMap);
+        yield* Effect.gen(function* () {
+          yield* cookieManager.deserialize(session.cookies);
+          yield* Ref.set(currentSessionId, Option.some(session.id));
+        });
+      } catch (error) {
+        yield* Effect.fail(new Error(`Invalid session data: ${error}`));
+      }
+    })
+  };
+});
+const SessionStoreLive = Layer.effect(SessionStore, makeSessionStore);
+var TokenType = /* @__PURE__ */ ((TokenType2) => {
+  TokenType2["CSRF"] = "csrf";
+  TokenType2["API"] = "api";
+  TokenType2["AUTH"] = "auth";
+  TokenType2["REFRESH"] = "refresh";
+  return TokenType2;
+})(TokenType || {});
+class StateManager extends Context.Tag("StateManager")() {
+}
+const makeStateManager = () => Effect.gen(function* () {
+  const tokens = yield* Ref.make(/* @__PURE__ */ new Map());
+  const localStorage = yield* Ref.make(/* @__PURE__ */ new Map());
+  const sessionStorage = yield* Ref.make(/* @__PURE__ */ new Map());
+  return {
+    extractCSRFToken: (html) => Effect.gen(function* () {
+      const $ = cheerio.load(html);
+      const csrfSelectors = [
+        'meta[name="csrf-token"]',
+        'meta[name="_csrf"]',
+        'meta[name="csrf_token"]',
+        'meta[name="authenticity_token"]',
+        'input[name="csrf_token"]',
+        'input[name="_csrf"]',
+        'input[name="authenticity_token"]',
+        'input[name="__RequestVerificationToken"]'
+      ];
+      for (const selector of csrfSelectors) {
+        const element = $(selector);
+        if (element.length > 0) {
+          const token = element.attr("content") || element.attr("value");
+          if (token) {
+            return token;
+          }
+        }
+      }
+      const scriptTags = $("script:not([src])");
+      const scriptContent = scriptTags.map((_, el) => $(el).html()).get().join("\n");
+      const patterns = [
+        /window\.csrfToken\s*=\s*["']([^"']+)["']/,
+        /csrf[_-]?token["']?\s*[:=]\s*["']([^"']+)["']/i,
+        /_token["']?\s*[:=]\s*["']([^"']+)["']/,
+        /authenticity_token["']?\s*[:=]\s*["']([^"']+)["']/,
+        /X-CSRF-Token["']?\s*[:=]\s*["']([^"']+)["']/
+      ];
+      for (const pattern of patterns) {
+        const match = scriptContent.match(pattern);
+        if (match && match[1]) {
+          return match[1];
+        }
+      }
+      return yield* Effect.fail(new Error("CSRF token not found in HTML"));
+    }),
+    extractAPIToken: (scripts) => Effect.gen(function* () {
+      const scriptContent = scripts.join("\n");
+      const patterns = [
+        /api[_-]?key["']?\s*[:=]\s*["']([^"']+)["']/i,
+        /api[_-]?token["']?\s*[:=]\s*["']([^"']+)["']/i,
+        /X-Secret-Token["']?\s*[:=]\s*["']([^"']+)["']/,
+        /authorization["']?\s*[:=]\s*["']Bearer\s+([^"']+)["']/i,
+        /access[_-]?token["']?\s*[:=]\s*["']([^"']+)["']/i,
+        /secret[_-]?key["']?\s*[:=]\s*["']([^"']+)["']/i
+      ];
+      for (const pattern of patterns) {
+        const match = scriptContent.match(pattern);
+        if (match && match[1]) {
+          return match[1];
+        }
+      }
+      const windowPattern = /window\[["']([^"']*[Tt]oken[^"']*)["']\]\s*=\s*["']([^"']+)["']/g;
+      let windowMatch;
+      while ((windowMatch = windowPattern.exec(scriptContent)) !== null) {
+        if (windowMatch[2]) {
+          return windowMatch[2];
+        }
+      }
+      return yield* Effect.fail(
+        new Error("API token not found in scripts")
+      );
+    }),
+    storeToken: (type, value, expiry) => Effect.gen(function* () {
+      const token = {
+        type,
+        value,
+        expiry
+      };
+      const tokensMap = yield* Ref.get(tokens);
+      tokensMap.set(type, token);
+      yield* Ref.set(tokens, tokensMap);
+    }),
+    getToken: (type) => Effect.gen(function* () {
+      const tokensMap = yield* Ref.get(tokens);
+      const token = tokensMap.get(type);
+      if (!token) {
+        return yield* Effect.fail(
+          new Error(`Token of type ${type} not found`)
+        );
+      }
+      if (token.expiry && token.expiry < /* @__PURE__ */ new Date()) {
+        return yield* Effect.fail(
+          new Error(`Token of type ${type} has expired`)
+        );
+      }
+      return token.value;
+    }),
+    isTokenValid: (type) => Effect.gen(function* () {
+      const tokensMap = yield* Ref.get(tokens);
+      const token = tokensMap.get(type);
+      if (!token) {
+        return false;
+      }
+      if (token.expiry && token.expiry < /* @__PURE__ */ new Date()) {
+        return false;
+      }
+      return true;
+    }),
+    setLocalStorage: (key, value) => Effect.gen(function* () {
+      const storage = yield* Ref.get(localStorage);
+      storage.set(key, value);
+      yield* Ref.set(localStorage, storage);
+    }),
+    getLocalStorage: (key) => Effect.gen(function* () {
+      const storage = yield* Ref.get(localStorage);
+      const value = storage.get(key);
+      if (!value) {
+        return yield* Effect.fail(
+          new Error(`Local storage key '${key}' not found`)
+        );
+      }
+      return value;
+    }),
+    clearLocalStorage: () => Effect.gen(function* () {
+      yield* Ref.set(localStorage, /* @__PURE__ */ new Map());
+    }),
+    setSessionStorage: (key, value) => Effect.gen(function* () {
+      const storage = yield* Ref.get(sessionStorage);
+      storage.set(key, value);
+      yield* Ref.set(sessionStorage, storage);
+    }),
+    getSessionStorage: (key) => Effect.gen(function* () {
+      const storage = yield* Ref.get(sessionStorage);
+      const value = storage.get(key);
+      if (!value) {
+        return yield* Effect.fail(
+          new Error(`Session storage key '${key}' not found`)
+        );
+      }
+      return value;
+    }),
+    clearSessionStorage: () => Effect.gen(function* () {
+      yield* Ref.set(sessionStorage, /* @__PURE__ */ new Map());
+    }),
+    clearState: () => Effect.gen(function* () {
+      yield* Ref.set(tokens, /* @__PURE__ */ new Map());
+      yield* Ref.set(localStorage, /* @__PURE__ */ new Map());
+      yield* Ref.set(sessionStorage, /* @__PURE__ */ new Map());
+    })
+  };
+});
+const StateManagerLive = Layer.effect(StateManager, makeStateManager());
+class TokenExtractor extends Context.Tag("TokenExtractor")() {
+}
+const makeTokenExtractor = Effect.gen(function* () {
+  const stateManager = yield* StateManager;
+  const httpClient = yield* EnhancedHttpClient;
+  const logger = yield* SpiderLogger;
+  const extractFromHTML = (html) => {
+    const tokens = [];
+    const $ = cheerio.load(html);
+    const csrfSelectors = [
+      { selector: 'meta[name="csrf-token"]', attr: "content" },
+      { selector: 'meta[name="_csrf"]', attr: "content" },
+      { selector: 'meta[name="csrf_token"]', attr: "content" },
+      { selector: 'meta[name="authenticity_token"]', attr: "content" },
+      { selector: 'input[name="csrf_token"]', attr: "value" },
+      { selector: 'input[name="_csrf"]', attr: "value" },
+      { selector: 'input[name="authenticity_token"]', attr: "value" },
+      { selector: 'input[name="__RequestVerificationToken"]', attr: "value" }
+    ];
+    for (const { selector, attr } of csrfSelectors) {
+      const element = $(selector);
+      if (element.length > 0) {
+        const value = element.attr(attr);
+        if (value) {
+          tokens.push({
+            type: TokenType.CSRF,
+            value,
+            source: "html",
+            selector
+          });
+        }
+      }
+    }
+    const apiSelectors = [
+      { selector: 'meta[name="api-key"]', attr: "content" },
+      { selector: 'meta[name="api_key"]', attr: "content" },
+      { selector: 'meta[name="api-token"]', attr: "content" },
+      { selector: 'meta[name="access-token"]', attr: "content" }
+    ];
+    for (const { selector, attr } of apiSelectors) {
+      const element = $(selector);
+      if (element.length > 0) {
+        const value = element.attr(attr);
+        if (value) {
+          tokens.push({
+            type: TokenType.API,
+            value,
+            source: "html",
+            selector
+          });
+        }
+      }
+    }
+    return tokens;
+  };
+  const extractFromScripts = (html) => {
+    const tokens = [];
+    const $ = cheerio.load(html);
+    const scriptTags = $("script:not([src])");
+    const scriptContent = scriptTags.map((_, el) => $(el).html()).get().join("\n");
+    const csrfPatterns = [
+      {
+        pattern: /window\.csrfToken\s*=\s*["']([^"']+)["']/,
+        name: "window.csrfToken"
+      },
+      {
+        pattern: /csrf[_-]?token["']?\s*[:=]\s*["']([^"']+)["']/i,
+        name: "csrf_token"
+      },
+      { pattern: /_token["']?\s*[:=]\s*["']([^"']+)["']/, name: "_token" },
+      {
+        pattern: /authenticity_token["']?\s*[:=]\s*["']([^"']+)["']/,
+        name: "authenticity_token"
+      },
+      {
+        pattern: /X-CSRF-Token["']?\s*[:=]\s*["']([^"']+)["']/,
+        name: "X-CSRF-Token"
+      }
+    ];
+    for (const { pattern, name } of csrfPatterns) {
+      const match = scriptContent.match(pattern);
+      if (match && match[1]) {
+        tokens.push({
+          type: TokenType.CSRF,
+          value: match[1],
+          source: "script",
+          pattern: name
+        });
+      }
+    }
+    const apiPatterns = [
+      {
+        pattern: /api[_-]?key["']?\s*[:=]\s*["']([^"']+)["']/i,
+        name: "api_key"
+      },
+      {
+        pattern: /api[_-]?token["']?\s*[:=]\s*["']([^"']+)["']/i,
+        name: "api_token"
+      },
+      {
+        pattern: /X-Secret-Token["']?\s*[:=]\s*["']([^"']+)["']/,
+        name: "X-Secret-Token"
+      },
+      {
+        pattern: /authorization["']?\s*[:=]\s*["']Bearer\s+([^"']+)["']/i,
+        name: "authorization"
+      },
+      {
+        pattern: /access[_-]?token["']?\s*[:=]\s*["']([^"']+)["']/i,
+        name: "access_token"
+      },
+      {
+        pattern: /secret[_-]?key["']?\s*[:=]\s*["']([^"']+)["']/i,
+        name: "secret_key"
+      }
+    ];
+    for (const { pattern, name } of apiPatterns) {
+      const match = scriptContent.match(pattern);
+      if (match && match[1]) {
+        tokens.push({
+          type: TokenType.API,
+          value: match[1],
+          source: "script",
+          pattern: name
+        });
+      }
+    }
+    const windowPattern = /window\[["']([^"']*[Tt]oken[^"']*)["']\]\s*=\s*["']([^"']+)["']/g;
+    let windowMatch;
+    while ((windowMatch = windowPattern.exec(scriptContent)) !== null) {
+      if (windowMatch[2]) {
+        const keyLower = windowMatch[1].toLowerCase();
+        const type = keyLower.includes("csrf") || keyLower.includes("authenticity") ? TokenType.CSRF : TokenType.API;
+        tokens.push({
+          type,
+          value: windowMatch[2],
+          source: "script",
+          pattern: `window['${windowMatch[1]}']`
+        });
+      }
+    }
+    return tokens;
+  };
+  const extractFromHeaders = (headers) => {
+    const tokens = [];
+    const headerPatterns = [
+      { header: "x-csrf-token", type: TokenType.CSRF },
+      { header: "x-auth-token", type: TokenType.AUTH },
+      { header: "x-api-key", type: TokenType.API },
+      { header: "authorization", type: TokenType.AUTH },
+      { header: "x-access-token", type: TokenType.AUTH }
+    ];
+    for (const { header, type } of headerPatterns) {
+      const value = headers[header] || headers[header.toLowerCase()];
+      if (value) {
+        tokens.push({
+          type,
+          value,
+          source: "header",
+          pattern: header
+        });
+      }
+    }
+    return tokens;
+  };
+  const service = {
+    extractTokensFromResponse: (response) => Effect.gen(function* () {
+      const tokens = [];
+      tokens.push(...extractFromHTML(response.body));
+      tokens.push(...extractFromScripts(response.body));
+      tokens.push(...extractFromHeaders(response.headers));
+      const uniqueTokens = /* @__PURE__ */ new Map();
+      for (const token of tokens) {
+        const key = `${token.type}:${token.value}`;
+        if (!uniqueTokens.has(key)) {
+          uniqueTokens.set(key, token);
+          yield* stateManager.storeToken(
+            token.type,
+            token.value,
+            new Date(Date.now() + 36e5)
+            // 1 hour expiry
+          );
+          yield* logger.logEdgeCase(
+            new URL(response.url).hostname,
+            "token_found",
+            {
+              type: token.type,
+              source: token.source,
+              pattern: token.pattern || token.selector
+            }
+          );
+        }
+      }
+      return Array.from(uniqueTokens.values());
+    }),
+    extractCSRFFromResponse: (response) => Effect.gen(function* () {
+      const tokens = yield* Effect.succeed([
+        ...extractFromHTML(response.body),
+        ...extractFromScripts(response.body)
+      ]);
+      const csrfToken = tokens.find((t) => t.type === TokenType.CSRF);
+      if (csrfToken) {
+        yield* stateManager.storeToken(
+          TokenType.CSRF,
+          csrfToken.value,
+          new Date(Date.now() + 36e5)
+        );
+        return csrfToken.value;
+      }
+      return null;
+    }),
+    extractAPIFromResponse: (response) => Effect.gen(function* () {
+      const tokens = yield* Effect.succeed([
+        ...extractFromScripts(response.body),
+        ...extractFromHeaders(response.headers)
+      ]);
+      const apiToken = tokens.find((t) => t.type === TokenType.API);
+      if (apiToken) {
+        yield* stateManager.storeToken(
+          TokenType.API,
+          apiToken.value,
+          new Date(Date.now() + 36e5)
+        );
+        return apiToken.value;
+      }
+      return null;
+    }),
+    authenticatedRequest: (url, options = {}) => Effect.gen(function* () {
+      const headers = { ...options.customHeaders };
+      if (options.requireCSRF) {
+        const isValid = yield* stateManager.isTokenValid(TokenType.CSRF);
+        if (!isValid) {
+          const baseUrl = new URL(url).origin;
+          const baseResponse = yield* httpClient.get(baseUrl);
+          yield* Effect.succeed(extractFromHTML(baseResponse.body)).pipe(
+            Effect.flatMap((tokens) => {
+              const csrfToken2 = tokens.find((t) => t.type === TokenType.CSRF);
+              if (csrfToken2) {
+                return stateManager.storeToken(
+                  TokenType.CSRF,
+                  csrfToken2.value,
+                  new Date(Date.now() + 36e5)
+                );
+              }
+              return Effect.void;
+            })
+          );
+        }
+        const csrfToken = yield* stateManager.getToken(TokenType.CSRF).pipe(Effect.catchAll(() => Effect.succeed(null)));
+        if (csrfToken) {
+          headers["X-CSRF-Token"] = csrfToken;
+          headers["X-Requested-With"] = "XMLHttpRequest";
+        }
+      }
+      if (options.requireAPI) {
+        const isValid = yield* stateManager.isTokenValid(TokenType.API);
+        if (!isValid) {
+          return yield* Effect.fail(
+            new Error("API token not available or expired")
+          );
+        }
+        const apiToken = yield* stateManager.getToken(TokenType.API);
+        headers["Authorization"] = `Bearer ${apiToken}`;
+        headers["X-API-Key"] = apiToken;
+      }
+      const response = yield* httpClient.request(url, { headers });
+      if (options.requireCSRF) {
+        const currentCSRF = yield* stateManager.getToken(TokenType.CSRF).pipe(Effect.catchAll(() => Effect.succeed("")));
+        if (currentCSRF) {
+          yield* service.detectTokenRotation(
+            currentCSRF,
+            response,
+            TokenType.CSRF
+          );
+        }
+      }
+      if (options.requireAPI) {
+        const currentAPI = yield* stateManager.getToken(TokenType.API).pipe(Effect.catchAll(() => Effect.succeed("")));
+        if (currentAPI) {
+          yield* service.detectTokenRotation(
+            currentAPI,
+            response,
+            TokenType.API
+          );
+        }
+      }
+      return response;
+    }),
+    detectTokenRotation: (oldToken, response, type) => Effect.gen(function* () {
+      const tokens = yield* Effect.succeed([
+        ...extractFromHTML(response.body),
+        ...extractFromScripts(response.body),
+        ...extractFromHeaders(response.headers)
+      ]);
+      const newToken = tokens.find(
+        (t) => t.type === type && t.value !== oldToken
+      );
+      if (newToken) {
+        yield* stateManager.storeToken(
+          type,
+          newToken.value,
+          new Date(Date.now() + 36e5)
+        );
+        yield* logger.logEdgeCase(
+          new URL(response.url).hostname,
+          "token_rotated",
+          {
+            type,
+            oldToken: oldToken.substring(0, 8) + "...",
+            newToken: newToken.value.substring(0, 8) + "..."
+          }
+        );
+        return true;
+      }
+      return false;
+    }),
+    refreshToken: (type, refreshUrl) => Effect.gen(function* () {
+      if (!refreshUrl) {
+        return yield* Effect.fail(new Error("No refresh URL provided"));
+      }
+      const response = yield* httpClient.get(refreshUrl);
+      const tokens = yield* Effect.succeed([
+        ...extractFromHTML(response.body),
+        ...extractFromScripts(response.body),
+        ...extractFromHeaders(response.headers)
+      ]);
+      const newToken = tokens.find((t) => t.type === type);
+      if (!newToken) {
+        return yield* Effect.fail(
+          new Error(`Failed to refresh ${type} token`)
+        );
+      }
+      yield* stateManager.storeToken(
+        type,
+        newToken.value,
+        new Date(Date.now() + 36e5)
+      );
+      return newToken.value;
+    })
+  };
+  return service;
+});
+const TokenExtractorLive = Layer.effect(
+  TokenExtractor,
+  makeTokenExtractor
+);
+class WebScrapingEngine extends Context.Tag("WebScrapingEngine")() {
+}
+const makeWebScrapingEngine = Effect.gen(function* () {
+  const httpClient = yield* EnhancedHttpClient;
+  const cookieManager = yield* CookieManager;
+  const sessionStore = yield* SessionStore;
+  const tokenExtractor = yield* TokenExtractor;
+  const stateManager = yield* StateManager;
+  const logger = yield* SpiderLogger;
+  yield* ScraperService;
+  const service = {
+    login: (credentials) => Effect.gen(function* () {
+      const domain = new URL(credentials.loginUrl).hostname;
+      yield* logger.logEdgeCase(domain, "login_start", {
+        url: credentials.loginUrl,
+        username: credentials.username
+      });
+      const loginPageResponse = yield* httpClient.get(credentials.loginUrl);
+      const csrfToken = yield* tokenExtractor.extractCSRFFromResponse(loginPageResponse);
+      const formData = {
+        [credentials.usernameField || "username"]: credentials.username,
+        [credentials.passwordField || "password"]: credentials.password,
+        ...credentials.additionalFields
+      };
+      if (csrfToken) {
+        const csrfFieldNames = [
+          "csrf_token",
+          "_csrf",
+          "authenticity_token",
+          "__RequestVerificationToken"
+        ];
+        const csrfFieldName = csrfFieldNames.find(
+          (name) => loginPageResponse.body.includes(`name="${name}"`)
+        ) || "csrf_token";
+        formData[csrfFieldName] = csrfToken;
+        yield* logger.logEdgeCase(domain, "csrf_token_added", {
+          field: csrfFieldName
+        });
+      }
+      const loginResponse = yield* httpClient.submitForm(
+        credentials.loginUrl,
+        formData
+      );
+      const isAuthenticated = loginResponse.status === 200 || loginResponse.status === 302 || loginResponse.headers["location"] !== void 0;
+      if (!isAuthenticated) {
+        return yield* Effect.fail(
+          new Error(`Login failed with status ${loginResponse.status}`)
+        );
+      }
+      yield* tokenExtractor.extractTokensFromResponse(loginResponse);
+      const session = yield* sessionStore.createSession();
+      yield* sessionStore.updateSessionData({
+        authenticated: true,
+        username: credentials.username,
+        loginTime: /* @__PURE__ */ new Date()
+      });
+      const tokens = /* @__PURE__ */ new Map();
+      for (const type of [TokenType.CSRF, TokenType.API, TokenType.AUTH]) {
+        const token = yield* stateManager.getToken(type).pipe(Effect.catchAll(() => Effect.succeed(null)));
+        if (token) {
+          tokens.set(type, token);
+        }
+      }
+      yield* logger.logEdgeCase(domain, "login_success", {
+        sessionId: session.id,
+        tokensFound: Array.from(tokens.keys())
+      });
+      return {
+        id: session.id,
+        authenticated: true,
+        tokens,
+        startTime: /* @__PURE__ */ new Date()
+      };
+    }),
+    fetchAuthenticated: (url) => Effect.gen(function* () {
+      const isValid = yield* sessionStore.isSessionValid();
+      if (!isValid) {
+        return yield* Effect.fail(
+          new Error("No valid session. Please login first.")
+        );
+      }
+      return yield* httpClient.get(url);
+    }),
+    submitFormWithCSRF: (url, formData, csrfUrl) => Effect.gen(function* () {
+      const domain = new URL(url).hostname;
+      let csrfToken = null;
+      const isValid = yield* stateManager.isTokenValid(TokenType.CSRF);
+      if (!isValid && csrfUrl) {
+        const csrfResponse = yield* httpClient.get(csrfUrl);
+        csrfToken = yield* tokenExtractor.extractCSRFFromResponse(csrfResponse);
+      } else if (isValid) {
+        csrfToken = yield* stateManager.getToken(TokenType.CSRF).pipe(Effect.catchAll(() => Effect.succeed(null)));
+      }
+      if (!csrfToken && !csrfUrl) {
+        const formPageResponse = yield* httpClient.get(url);
+        csrfToken = yield* tokenExtractor.extractCSRFFromResponse(formPageResponse);
+      }
+      const enhancedFormData = { ...formData };
+      if (csrfToken) {
+        const csrfFieldNames = [
+          "csrf_token",
+          "_csrf",
+          "authenticity_token",
+          "__RequestVerificationToken"
+        ];
+        const csrfFieldName = csrfFieldNames[0];
+        enhancedFormData[csrfFieldName] = csrfToken;
+        yield* logger.logEdgeCase(domain, "csrf_protected_form", {
+          url,
+          csrfField: csrfFieldName
+        });
+      }
+      const response = yield* httpClient.submitForm(url, enhancedFormData);
+      if (csrfToken) {
+        yield* tokenExtractor.detectTokenRotation(
+          csrfToken,
+          response,
+          TokenType.CSRF
+        );
+      }
+      return response;
+    }),
+    makeAPIRequest: (url, method = "GET", data) => Effect.gen(function* () {
+      const response = yield* tokenExtractor.authenticatedRequest(url, {
+        requireAPI: true,
+        customHeaders: {
+          "Content-Type": "application/json",
+          Accept: "application/json"
+        }
+      }).pipe(
+        Effect.catchAll((error) => {
+          if (method === "GET") {
+            return httpClient.get(url);
+          } else {
+            return httpClient.post(url, data);
+          }
+        })
+      );
+      return response;
+    }),
+    createSession: (id) => Effect.gen(function* () {
+      const session = yield* sessionStore.createSession(id);
+      const tokens = /* @__PURE__ */ new Map();
+      for (const type of [TokenType.CSRF, TokenType.API, TokenType.AUTH]) {
+        const token = yield* stateManager.getToken(type).pipe(Effect.catchAll(() => Effect.succeed(null)));
+        if (token) {
+          tokens.set(type, token);
+        }
+      }
+      return {
+        id: session.id,
+        authenticated: false,
+        tokens,
+        startTime: session.createdAt
+      };
+    }),
+    loadSession: (id) => Effect.gen(function* () {
+      yield* sessionStore.loadSession(id);
+      const session = yield* sessionStore.getCurrentSession();
+      if (session._tag === "None") {
+        return yield* Effect.fail(new Error("Failed to load session"));
+      }
+      const tokens = /* @__PURE__ */ new Map();
+      for (const type of [TokenType.CSRF, TokenType.API, TokenType.AUTH]) {
+        const token = yield* stateManager.getToken(type).pipe(Effect.catchAll(() => Effect.succeed(null)));
+        if (token) {
+          tokens.set(type, token);
+        }
+      }
+      return {
+        id: session.value.id,
+        authenticated: session.value.userData?.authenticated || false,
+        tokens,
+        startTime: session.value.createdAt
+      };
+    }),
+    exportSession: () => sessionStore.exportSession(),
+    importSession: (data) => sessionStore.importSession(data),
+    clearAll: () => Effect.gen(function* () {
+      yield* sessionStore.clearSession();
+      yield* cookieManager.clearCookies();
+      yield* stateManager.clearState();
+    })
+  };
+  return service;
+});
+const WebScrapingEngineLive = Layer.effect(
+  WebScrapingEngine,
+  makeWebScrapingEngine
+);
+export {
+  ConfigurationError,
+  CookieManager,
+  CookieManagerLive,
+  DEFAULT_HYBRID_CONFIG,
+  DeltaPersistence,
+  EnhancedHttpClient,
+  EnhancedHttpClientLive,
+  FileStorageBackend,
+  FileSystemError,
+  FullStatePersistence,
+  HybridPersistence,
+  LinkExtractionError,
+  LinkExtractorService,
+  LinkExtractorServiceLayer,
+  LoggingMiddleware,
+  MiddlewareError,
+  MiddlewareManager,
+  NetworkError,
+  PageDataSchema,
+  PersistenceError$1 as PersistenceError,
+  PriorityRequest,
+  RateLimitMiddleware,
+  ResponseError,
+  ResumabilityConfigs,
+  PersistenceError2 as ResumabilityError,
+  ResumabilityService,
+  RobotsService,
+  RobotsTxtError,
+  ScraperService,
+  SessionStore,
+  SessionStoreLive,
+  SpiderConfig,
+  SpiderLoggerLive,
+  SpiderLogger as SpiderLoggerTag,
+  SpiderSchedulerService,
+  SpiderService,
+  SpiderState,
+  SpiderStateKey,
+  StateDelta,
+  StateManager,
+  StateManagerLive,
+  StatsMiddleware,
+  TokenExtractor,
+  TokenExtractorLive,
+  TokenType,
+  UrlDeduplicatorService,
+  UserAgentMiddleware,
+  WebScrapingEngine,
+  WebScrapingEngineLive,
+  createStateOperation,
+  makeCookieManager,
+  makeEnhancedHttpClient,
+  makeSessionStore,
+  makeSpiderConfig,
+  makeSpiderLogger,
+  makeStateManager,
+  makeTokenExtractor,
+  makeWebScrapingEngine
+};
+//# sourceMappingURL=index.js.map