@jambudipa/spider 0.2.3 → 0.3.1

package/dist/index.js

@@ -1,4 +1,4 @@
-import { Effect, Layer, Option, Chunk, MutableHashSet, Schema, Data, Context, DateTime, Console, Duration, MutableHashMap, Queue, Ref, HashMap, pipe, PubSub, MutableRef, Schedule, Stream, Fiber, Random, Struct } from "effect";
+import { Effect, Layer, Option, Chunk, MutableHashSet, Schema, Data, pipe, Context, DateTime, Console, Duration, MutableHashMap, Queue, HashMap, PubSub, MutableRef, Schedule, Stream, Fiber, Random, Struct, Ref } from "effect";
 import * as cheerio from "cheerio";
 import * as fs from "fs";
 import * as path from "path";
@@ -342,11 +342,12 @@ class UrlDeduplicatorService extends Effect.Service()(
             if (normalizedPath === "") {
               normalizedPath = "/";
             }
-            parsed.pathname = normalizedPath;
-            parsed.hash = "";
+            const hash = "";
+            let port = parsed.port;
             if (parsed.protocol === "http:" && parsed.port === "80" || parsed.protocol === "https:" && parsed.port === "443") {
-              parsed.port = "";
+              port = "";
             }
+            let search = parsed.search;
             if (parsed.search) {
               const params = new URLSearchParams(parsed.search);
               const sortedParams = new URLSearchParams();
@@ -355,9 +356,12 @@ class UrlDeduplicatorService extends Effect.Service()(
                   sortedParams.append(key, value);
                 });
               });
-              parsed.search = sortedParams.toString();
+              const sortedStr = sortedParams.toString();
+              search = sortedStr ? `?${sortedStr}` : "";
             }
-            return parsed.toString();
+            const auth = parsed.username ? `${parsed.username}${parsed.password ? ":" + parsed.password : ""}@` : "";
+            const portStr = port ? `:${port}` : "";
+            return `${parsed.protocol}//${auth}${parsed.hostname}${portStr}${normalizedPath}${search}${hash}`;
           }),
           // If URL parsing fails, return original
           () => Effect.succeed(url)
@@ -425,15 +429,49 @@ const PageDataSchema = Schema.Struct({
     Schema.Record({ key: Schema.String, value: Schema.Unknown })
   )
 });
-let NetworkError$1 = class NetworkError extends Data.TaggedError("NetworkError") {
-  static fromCause(url, cause) {
+class SpiderError extends Data.TaggedError("SpiderError") {
+  get message() {
+    const detailsStr = Option.fromNullable(this.details).pipe(
+      Option.map((d) => `: ${String(d)}`),
+      Option.getOrElse(() => "")
+    );
+    return `Spider operation '${this.operation}' failed${detailsStr}`;
+  }
+}
+class NetworkError extends Data.TaggedError("NetworkError") {
+  get message() {
+    const parts = pipe(
+      Chunk.make(`Network request to ${this.url} failed`),
+      (chunk) => this.statusCode ? Chunk.append(chunk, `with status ${this.statusCode}`) : chunk,
+      (chunk) => this.cause ? Chunk.append(chunk, `${this.cause}`) : chunk
+    );
+    return Chunk.toArray(parts).join(" ");
+  }
+  static fromResponse(url, response) {
     return new NetworkError({
+      url,
+      statusCode: response.status,
+      method: "GET"
+    });
+  }
+  static fromCause(url, cause) {
+    return new NetworkError({ url, cause });
+  }
+}
+class TimeoutError extends Data.TaggedError("TimeoutError") {
+  get message() {
+    return `Operation '${this.operation}' timed out after ${this.timeoutMs}ms for ${this.url}`;
+  }
+}
+class RobotsTxtError extends Data.TaggedError("RobotsTxtError") {
+  static fromCause(url, cause) {
+    return new RobotsTxtError({
       url,
       cause,
-      message: `Failed to fetch ${url}: ${cause}`
+      message: `Failed to fetch robots.txt: ${cause}`
     });
   }
-};
+}
 class ResponseError extends Data.TaggedError("ResponseError") {
   static fromCause(url, cause) {
     return new ResponseError({
@@ -443,53 +481,89 @@ class ResponseError extends Data.TaggedError("ResponseError") {
     });
   }
 }
-class RobotsTxtError extends Data.TaggedError("RobotsTxtError") {
-  static fromCause(url, cause) {
-    return new RobotsTxtError({
-      url,
-      cause,
-      message: `Failed to fetch robots.txt: ${cause}`
+class ParseError extends Data.TaggedError("ParseError") {
+  get message() {
+    return `Failed to parse ${this.expected}${this.input ? ` from input: ${this.input.substring(0, 100)}...` : ""}`;
+  }
+  static json(input, cause) {
+    return new ParseError({
+      input,
+      expected: "JSON",
+      cause
+    });
+  }
+  static html(input, cause) {
+    return new ParseError({
+      input,
+      expected: "HTML",
+      cause
+    });
+  }
+}
+class ValidationError extends Data.TaggedError("ValidationError") {
+  get message() {
+    return `Validation failed for field '${this.field}': ${this.constraint}`;
+  }
+  static url(url) {
+    return new ValidationError({
+      field: "url",
+      value: url,
+      constraint: "Invalid URL format"
     });
   }
 }
 class ConfigurationError extends Data.TaggedError("ConfigurationError") {
 }
-let MiddlewareError$1 = class MiddlewareError extends Data.TaggedError("MiddlewareError") {
+class ConfigError extends Data.TaggedError("ConfigError") {
+  get message() {
+    return `Configuration error for '${this.field}': ${this.reason}`;
+  }
+  static invalid(field, value, expected) {
+    return new ConfigError({
+      field,
+      value,
+      reason: `Expected ${expected}, got ${typeof value}`
+    });
+  }
+}
+class MiddlewareError extends Data.TaggedError("MiddlewareError") {
+  get message() {
+    return `Middleware '${this.middlewareName}' failed during ${this.phase} phase`;
+  }
   static transform(middlewareName, cause) {
     return new MiddlewareError({
       phase: "transform",
       middlewareName,
-      cause,
-      message: `Middleware '${middlewareName}' failed during transform: ${cause}`
+      cause
     });
   }
   static error(middlewareName, cause) {
     return new MiddlewareError({
       phase: "error",
       middlewareName,
-      cause,
-      message: `Middleware '${middlewareName}' failed during error handling: ${cause}`
+      cause
     });
   }
-};
-let FileSystemError$1 = class FileSystemError extends Data.TaggedError("FileSystemError") {
+}
+class FileSystemError extends Data.TaggedError("FileSystemError") {
+  get message() {
+    return `File system ${this.operation} operation failed for path: ${this.path}`;
+  }
   static write(path2, cause) {
     return new FileSystemError({
       operation: "write",
       path: path2,
-      cause,
-      message: `Failed to write file ${path2}: ${cause}`
+      cause
     });
   }
   static create(path2, cause) {
     return new FileSystemError({
       operation: "create",
       path: path2,
-      cause,
-      message: `Failed to create directory ${path2}: ${cause}`
+      cause
     });
   }
-};
+}
 let PersistenceError$1 = class PersistenceError extends Data.TaggedError("PersistenceError") {
   static save(cause, key) {
     return new PersistenceError({
@@ -553,36 +627,32 @@ class AdapterNotInitialisedError extends Data.TaggedError("AdapterNotInitialised
     });
   }
 }
-let BrowserError$1 = class BrowserError extends Data.TaggedError("BrowserError") {
+class BrowserError extends Data.TaggedError("BrowserError") {
+  get message() {
+    return `Browser operation '${this.operation}' failed${this.browserId ? ` for browser ${this.browserId}` : ""}${this.cause ? `: ${this.cause}` : ""}`;
+  }
   static launch(cause) {
-    return new BrowserError({
-      operation: "launch",
-      cause,
-      message: `Failed to launch browser: ${cause}`
-    });
+    return new BrowserError({ operation: "launch", cause });
   }
   static createContext(cause) {
-    return new BrowserError({
-      operation: "createContext",
-      cause,
-      message: `Failed to create browser context: ${cause}`
-    });
+    return new BrowserError({ operation: "createContext", cause });
   }
   static createPage(cause) {
-    return new BrowserError({
-      operation: "createPage",
-      cause,
-      message: `Failed to create page: ${cause}`
-    });
+    return new BrowserError({ operation: "createPage", cause });
   }
   static closeContext(cause) {
+    return new BrowserError({ operation: "closeContext", cause });
+  }
+  static notLaunched() {
     return new BrowserError({
-      operation: "closeContext",
-      cause,
-      message: `Failed to close context: ${cause}`
+      operation: "access",
+      cause: "Browser not launched"
     });
   }
-};
+  static launchFailed(cause) {
+    return new BrowserError({ operation: "launch", cause });
+  }
+}
 class BrowserCleanupError extends Data.TaggedError("BrowserCleanupError") {
   static context(id, cause) {
     return new BrowserCleanupError({
@@ -601,6 +671,64 @@ class BrowserCleanupError extends Data.TaggedError("BrowserCleanupError") {
     });
   }
 }
+class PageError extends Data.TaggedError("PageError") {
+  get message() {
+    return `Page operation '${this.operation}' failed for ${this.url}${this.selector ? ` with selector '${this.selector}'` : ""}`;
+  }
+}
+class StateError extends Data.TaggedError("StateError") {
+  get message() {
+    return `State ${this.operation} operation failed${this.stateKey ? ` for key '${this.stateKey}'` : ""}`;
+  }
+}
+let SessionError$1 = class SessionError extends Data.TaggedError("SessionError") {
+  get message() {
+    return `Session operation '${this.operation}' failed${this.sessionId ? ` for session ${this.sessionId}` : ""}`;
+  }
+  static noActiveSession() {
+    return new SessionError({
+      operation: "access",
+      cause: "No active session"
+    });
+  }
+};
+class CrawlError extends Data.TaggedError("CrawlError") {
+  get message() {
+    return `Failed to crawl ${this.url} at depth ${this.depth}: ${this.reason}`;
+  }
+  static maxDepthReached(url, depth) {
+    return new CrawlError({
+      url,
+      depth,
+      reason: "Maximum depth reached"
+    });
+  }
+  static robotsBlocked(url) {
+    return new CrawlError({
+      url,
+      depth: 0,
+      reason: "Blocked by robots.txt"
+    });
+  }
+}
+class QueueError extends Data.TaggedError("QueueError") {
+  get message() {
+    const sizeStr = Option.fromNullable(this.queueSize).pipe(
+      Option.map((size) => ` (queue size: ${size})`),
+      Option.getOrElse(() => "")
+    );
+    return `Queue ${this.operation} operation failed${sizeStr}`;
+  }
+}
+const isSpiderError = (error) => {
+  return error instanceof SpiderError;
+};
+const isNetworkError = (error) => {
+  return error instanceof NetworkError || error instanceof TimeoutError;
+};
+const isBrowserError = (error) => {
+  return error instanceof BrowserError || error instanceof PageError;
+};
 class SpiderLogger extends Context.Tag("SpiderLogger")() {
 }
 const SummarySchema = Schema.Record({
@@ -651,9 +779,9 @@ const makeSpiderLogger = (logDir = "./spider-logs") => {
     }
     return "{}";
   };
-  const writeLogEvent = (event) => Effect.sync(() => {
+  const writeLogEvent = (event) => Effect.gen(function* () {
     const logLine = stringifyForLog(event) + "\n";
-    fs.appendFileSync(logFilePath, logLine);
+    yield* Effect.sync(() => fs.appendFileSync(logFilePath, logLine));
     const importantTypes = [
       "domain_start",
       "domain_complete",
@@ -663,9 +791,7 @@ const makeSpiderLogger = (logDir = "./spider-logs") => {
     if (importantTypes.includes(event.type)) {
       const prefix = `[${event.type}]`;
       const domainInfo = event.domain ? ` [${event.domain}]` : "";
-      Console.log(`${prefix}${domainInfo} ${event.message}`).pipe(
-        Effect.runSync
-      );
+      yield* Console.log(`${prefix}${domainInfo} ${event.message}`);
     }
   });
   const updateSummary = (update) => Effect.sync(() => {
@@ -973,7 +1099,7 @@ class ScraperService extends Effect.Service()(
             if (error instanceof Error && error.name === "AbortError") {
               return RequestAbortError.timeout(url, timeoutMs);
             }
-            return NetworkError$1.fromCause(url, error);
+            return NetworkError.fromCause(url, error);
           }
         });
         const fetchWithTimeout = fetchEffect.pipe(
@@ -1367,11 +1493,11 @@ class SpiderSchedulerService extends Effect.Service()(
               if (normalizedPath === "") {
                 normalizedPath = "/";
               }
-              parsed.pathname = normalizedPath;
-              parsed.hash = "";
+              let port = parsed.port;
               if (parsed.protocol === "http:" && parsed.port === "80" || parsed.protocol === "https:" && parsed.port === "443") {
-                parsed.port = "";
+                port = "";
               }
+              let search = parsed.search;
               if (parsed.search) {
                 const params = new URLSearchParams(parsed.search);
                 const sortedParams = new URLSearchParams();
@@ -1380,9 +1506,12 @@ class SpiderSchedulerService extends Effect.Service()(
                     sortedParams.append(key, value);
                   });
                 });
-                parsed.search = sortedParams.toString();
+                const sortedStr = sortedParams.toString();
+                search = sortedStr ? `?${sortedStr}` : "";
               }
-              return parsed.toString();
+              const auth = parsed.username ? `${parsed.username}${parsed.password ? ":" + parsed.password : ""}@` : "";
+              const portStr = port ? `:${port}` : "";
+              return `${parsed.protocol}//${auth}${parsed.hostname}${portStr}${normalizedPath}${search}`;
             }
           }
         );
@@ -1516,154 +1645,6 @@ class SpiderSchedulerService extends Effect.Service()(
   }
 ) {
 }
-class SpiderError extends Data.TaggedError("SpiderError") {
-  get message() {
-    const detailsStr = Option.fromNullable(this.details).pipe(
-      Option.map((d) => `: ${String(d)}`),
-      Option.getOrElse(() => "")
-    );
-    return `Spider operation '${this.operation}' failed${detailsStr}`;
-  }
-}
-class NetworkError2 extends Data.TaggedError("NetworkError") {
-  get message() {
-    return `Network request to ${this.url} failed${this.statusCode ? ` with status ${this.statusCode}` : ""}`;
-  }
-  static fromResponse(url, response) {
-    return new NetworkError2({
-      url,
-      statusCode: response.status,
-      method: "GET"
-    });
-  }
-  static fromCause(url, cause) {
-    return new NetworkError2({ url, cause });
-  }
-}
-class TimeoutError extends Data.TaggedError("TimeoutError") {
-  get message() {
-    return `Operation '${this.operation}' timed out after ${this.timeoutMs}ms for ${this.url}`;
-  }
-}
-class ParseError extends Data.TaggedError("ParseError") {
-  get message() {
-    return `Failed to parse ${this.expected}${this.input ? ` from input: ${this.input.substring(0, 100)}...` : ""}`;
-  }
-  static json(input, cause) {
-    return new ParseError({
-      input,
-      expected: "JSON",
-      cause
-    });
-  }
-  static html(input, cause) {
-    return new ParseError({
-      input,
-      expected: "HTML",
-      cause
-    });
-  }
-}
-class ValidationError extends Data.TaggedError("ValidationError") {
-  get message() {
-    return `Validation failed for field '${this.field}': ${this.constraint}`;
-  }
-  static url(url) {
-    return new ValidationError({
-      field: "url",
-      value: url,
-      constraint: "Invalid URL format"
-    });
-  }
-}
-class BrowserError2 extends Data.TaggedError("BrowserError") {
-  get message() {
-    return `Browser operation '${this.operation}' failed${this.browserId ? ` for browser ${this.browserId}` : ""}`;
-  }
-  static notLaunched() {
-    return new BrowserError2({
-      operation: "access",
-      cause: "Browser not launched"
-    });
-  }
-  static launchFailed(cause) {
-    return new BrowserError2({
-      operation: "launch",
-      cause
-    });
-  }
-}
-class PageError extends Data.TaggedError("PageError") {
-  get message() {
-    return `Page operation '${this.operation}' failed for ${this.url}${this.selector ? ` with selector '${this.selector}'` : ""}`;
-  }
-}
-class StateError extends Data.TaggedError("StateError") {
-  get message() {
-    return `State ${this.operation} operation failed${this.stateKey ? ` for key '${this.stateKey}'` : ""}`;
-  }
-}
-let SessionError$1 = class SessionError extends Data.TaggedError("SessionError") {
-  get message() {
-    return `Session operation '${this.operation}' failed${this.sessionId ? ` for session ${this.sessionId}` : ""}`;
-  }
-  static noActiveSession() {
-    return new SessionError({
-      operation: "access",
-      cause: "No active session"
-    });
-  }
-};
-class FileSystemError2 extends Data.TaggedError("FileSystemError") {
-  get message() {
-    return `File system ${this.operation} operation failed for path: ${this.path}`;
-  }
-}
-class CrawlError extends Data.TaggedError("CrawlError") {
-  get message() {
-    return `Failed to crawl ${this.url} at depth ${this.depth}: ${this.reason}`;
-  }
-  static maxDepthReached(url, depth) {
-    return new CrawlError({
-      url,
-      depth,
-      reason: "Maximum depth reached"
-    });
-  }
-  static robotsBlocked(url) {
-    return new CrawlError({
-      url,
-      depth: 0,
-      reason: "Blocked by robots.txt"
-    });
-  }
-}
-class QueueError extends Data.TaggedError("QueueError") {
-  get message() {
-    const sizeStr = Option.fromNullable(this.queueSize).pipe(
-      Option.map((size) => ` (queue size: ${size})`),
-      Option.getOrElse(() => "")
-    );
-    return `Queue ${this.operation} operation failed${sizeStr}`;
-  }
-}
-class ConfigError extends Data.TaggedError("ConfigError") {
-  get message() {
-    return `Configuration error for '${this.field}': ${this.reason}`;
-  }
-  static invalid(field, value, expected) {
-    return new ConfigError({
-      field,
-      value,
-      reason: `Expected ${expected}, got ${typeof value}`
-    });
-  }
-}
-class MiddlewareError2 extends Data.TaggedError("MiddlewareError") {
-  get message() {
-    return `Middleware '${this.middlewareName}' failed during ${this.phase} phase`;
-  }
-}
 const DEFAULT_DEDUPLICATION_STRATEGY = {
   wwwHandling: "ignore",
   protocolHandling: "prefer-https",
@@ -1677,9 +1658,7 @@ const parseUrl = (url) => Effect.try({
 });
 const normalizeUrl = (url, strategy = DEFAULT_DEDUPLICATION_STRATEGY) => Effect.gen(function* () {
   const parsed = yield* parseUrl(url);
-  if (strategy.protocolHandling === "prefer-https") {
-    parsed.protocol = "https:";
-  }
+  const protocol = strategy.protocolHandling === "prefer-https" ? "https:" : parsed.protocol;
   let domain = parsed.hostname.toLowerCase();
   const hasWww = domain.startsWith("www.");
   const domainWithoutWww = hasWww ? domain.substring(4) : domain;
@@ -1687,104 +1666,105 @@ const normalizeUrl = (url, strategy = DEFAULT_DEDUPLICATION_STRATEGY) => Effect.
     case "ignore":
     case "prefer-non-www":
       domain = domainWithoutWww;
-      parsed.hostname = domain;
       break;
     case "prefer-www":
       if (!hasWww) {
         domain = `www.${domain}`;
-        parsed.hostname = domain;
       }
       break;
   }
+  let pathname = parsed.pathname;
   if (strategy.trailingSlashHandling === "ignore") {
-    parsed.pathname = parsed.pathname.replace(/\/$/, "") || "/";
+    pathname = pathname.replace(/\/$/, "") || "/";
   }
+  let search = "";
   if (strategy.queryParamHandling === "ignore") {
-    parsed.search = "";
+    search = "";
   } else if (strategy.queryParamHandling === "sort") {
     const params = new URLSearchParams(parsed.search);
     const sorted = Array.from(params.entries()).sort(([a], [b]) => a.localeCompare(b));
-    parsed.search = new URLSearchParams(sorted).toString();
-  }
-  if (strategy.fragmentHandling === "ignore") {
-    parsed.hash = "";
+    const sortedSearch = new URLSearchParams(sorted).toString();
+    search = sortedSearch ? `?${sortedSearch}` : "";
+  } else {
+    search = parsed.search;
   }
+  const hash = strategy.fragmentHandling === "ignore" ? "" : parsed.hash;
+  const auth = parsed.username ? `${parsed.username}${parsed.password ? ":" + parsed.password : ""}@` : "";
+  const port = parsed.port ? `:${parsed.port}` : "";
+  const normalized = `${protocol}//${auth}${domain}${port}${pathname}${search}${hash}`;
   return {
     original: url,
-    normalized: parsed.toString(),
+    normalized,
     domain: domainWithoutWww
   };
 });
 const deduplicateUrls = (urls, strategy = DEFAULT_DEDUPLICATION_STRATEGY) => Effect.gen(function* () {
-  const domainMap = yield* Ref.make(HashMap.empty());
-  const skipped = yield* Ref.make([]);
+  let domainMap = HashMap.empty();
+  const skipped = [];
   let invalidCount = 0;
-  yield* Effect.all(
-    urls.map(
-      (urlObj) => pipe(
-        normalizeUrl(urlObj.url, strategy),
-        Effect.tap(
-          (normalized) => Effect.gen(function* () {
-            const currentMap = yield* Ref.get(domainMap);
-            const key = strategy.wwwHandling === "preserve" ? normalized.normalized : normalized.domain;
-            const existingOption = HashMap.get(currentMap, key);
-            if (Option.isNone(existingOption)) {
-              yield* Ref.set(domainMap, HashMap.set(currentMap, key, urlObj));
-            } else {
-              const existing = existingOption.value;
-              let shouldReplace = false;
-              if (strategy.wwwHandling === "prefer-www") {
-                const existingHasWww = existing.url.includes("://www.");
-                const newHasWww = urlObj.url.includes("://www.");
-                shouldReplace = !existingHasWww && newHasWww;
-              } else if (strategy.wwwHandling === "prefer-non-www") {
-                const existingHasWww = existing.url.includes("://www.");
-                const newHasWww = urlObj.url.includes("://www.");
-                shouldReplace = existingHasWww && !newHasWww;
-              }
-              if (shouldReplace) {
-                yield* Ref.set(domainMap, HashMap.set(currentMap, key, urlObj));
-                yield* Ref.update(skipped, (arr) => [
-                  ...arr,
-                  { url: existing.url, reason: `Replaced by preferred variant: ${urlObj.url}` }
-                ]);
-              } else {
-                yield* Ref.update(skipped, (arr) => [
-                  ...arr,
-                  { url: urlObj.url, reason: `Duplicate of: ${existing.url}` }
-                ]);
-              }
-            }
-          })
-        ),
-        Effect.catchAll(
-          (error) => Effect.gen(function* () {
-            invalidCount++;
-            yield* Ref.update(skipped, (arr) => [
-              ...arr,
-              { url: urlObj.url, reason: `Invalid URL: ${error.message}` }
-            ]);
-            yield* Effect.logWarning(`Invalid URL skipped: ${urlObj.url}`);
-          })
-        )
-      )
-    ),
-    { concurrency: "unbounded" }
-  );
-  const finalMap = yield* Ref.get(domainMap);
-  const finalSkipped = yield* Ref.get(skipped);
-  const deduplicated = Array.from(HashMap.values(finalMap));
+  for (const urlObj of urls) {
+    const normalizeResult = yield* Effect.either(normalizeUrl(urlObj.url, strategy));
+    if (normalizeResult._tag === "Left") {
+      invalidCount++;
+      skipped.push({ url: urlObj.url, reason: `Invalid URL: ${normalizeResult.left.message}` });
+      yield* Effect.logWarning(`Invalid URL skipped: ${urlObj.url}`);
+      continue;
+    }
+    const normalized = normalizeResult.right;
+    const key = strategy.wwwHandling === "preserve" ? normalized.normalized : normalized.domain;
+    const existingOption = HashMap.get(domainMap, key);
+    if (Option.isNone(existingOption)) {
+      domainMap = HashMap.set(domainMap, key, urlObj);
+    } else {
+      const existing = existingOption.value;
+      let shouldReplace = false;
+      if (strategy.wwwHandling === "prefer-www") {
+        const existingHasWww = existing.url.includes("://www.");
+        const newHasWww = urlObj.url.includes("://www.");
+        shouldReplace = !existingHasWww && newHasWww;
+      } else if (strategy.wwwHandling === "prefer-non-www") {
+        const existingHasWww = existing.url.includes("://www.");
+        const newHasWww = urlObj.url.includes("://www.");
+        shouldReplace = existingHasWww && !newHasWww;
+      }
+      if (shouldReplace) {
+        domainMap = HashMap.set(domainMap, key, urlObj);
+        skipped.push({ url: existing.url, reason: `Replaced by preferred variant: ${urlObj.url}` });
+      } else {
+        skipped.push({ url: urlObj.url, reason: `Duplicate of: ${existing.url}` });
+      }
+    }
+  }
+  const deduplicated = Array.from(HashMap.values(domainMap));
   return {
     deduplicated,
-    skipped: finalSkipped,
+    skipped,
     stats: {
       total: urls.length,
       unique: deduplicated.length,
-      duplicates: finalSkipped.filter((s) => s.reason.startsWith("Duplicate")).length,
+      duplicates: skipped.filter((s) => s.reason.startsWith("Duplicate")).length,
       invalid: invalidCount
     }
   };
 });
+const SPIDER_DEFAULTS = Object.freeze({
+  /** Threshold in ms after which a worker is considered stale (60s) */
+  STALE_WORKER_THRESHOLD_MS: 6e4,
+  /** Interval for health check monitoring */
+  HEALTH_CHECK_INTERVAL: "15 seconds",
+  /** Memory usage threshold in bytes (1GB) before logging warnings */
+  MEMORY_THRESHOLD_BYTES: 1024 * 1024 * 1024,
+  /** Queue size threshold before logging warnings */
+  QUEUE_SIZE_THRESHOLD: 1e4,
+  /** Timeout for task acquisition from queue */
+  TASK_ACQUISITION_TIMEOUT: "10 seconds",
+  /** Timeout for page fetch operations */
+  FETCH_TIMEOUT: "45 seconds",
+  /** Number of retry attempts for fetch operations */
+  FETCH_RETRY_COUNT: 2,
+  /** Interval for domain failure detection checks */
+  FAILURE_DETECTOR_INTERVAL: "30 seconds"
+});
 class SpiderService extends Effect.Service()(
   "@jambudipa/spider",
   {
@@ -1954,7 +1934,7 @@ class SpiderService extends Effect.Service()(
           const workerHealthMonitor = Effect.gen(function* () {
             const healthMap = MutableRef.get(workerHealthChecks);
             const now = yield* DateTime.now;
-            const staleThreshold = 6e4;
+            const staleThreshold = SPIDER_DEFAULTS.STALE_WORKER_THRESHOLD_MS;
             let staleWorkersChunk = Chunk.empty();
             for (const [workerId, lastCheck] of healthMap) {
               const elapsed = DateTime.toEpochMillis(now) - DateTime.toEpochMillis(lastCheck);
@@ -1976,8 +1956,7 @@ class SpiderService extends Effect.Service()(
               MutableRef.set(workerHealthChecks, updatedMap);
             }
           }).pipe(
-            Effect.repeat(Schedule.fixed("15 seconds"))
-            // Check every 15 seconds
+            Effect.repeat(Schedule.fixed(SPIDER_DEFAULTS.HEALTH_CHECK_INTERVAL))
           );
           const queueManager = {
             // Atomic take: either returns task and increments active count, or detects completion
@@ -2063,7 +2042,7 @@ class SpiderService extends Effect.Service()(
               yield* reportWorkerHealth(workerId);
               const queueSize = yield* queueManager.size();
               const memUsage = process.memoryUsage();
-              if (memUsage.heapUsed > 1024 * 1024 * 1024) {
+              if (memUsage.heapUsed > SPIDER_DEFAULTS.MEMORY_THRESHOLD_BYTES) {
                 yield* logger.logEdgeCase(domain, "high_memory_usage", {
                   workerId,
                   heapUsed: Math.round(memUsage.heapUsed / 1024 / 1024) + "MB",
@@ -2071,7 +2050,7 @@ class SpiderService extends Effect.Service()(
                   queueSize
                 });
               }
-              if (queueSize > 1e4) {
+              if (queueSize > SPIDER_DEFAULTS.QUEUE_SIZE_THRESHOLD) {
                 yield* logger.logEdgeCase(domain, "excessive_queue_size", {
                   workerId,
                   queueSize,
@@ -2087,7 +2066,7 @@ class SpiderService extends Effect.Service()(
                 }
               );
               const result = yield* queueManager.takeTaskOrComplete.pipe(
-                Effect.timeout("10 seconds"),
+                Effect.timeout(SPIDER_DEFAULTS.TASK_ACQUISITION_TIMEOUT),
                 Effect.tap(
                   () => logger.logEdgeCase(domain, "task_acquisition_success", {
                     workerId,
@@ -2186,10 +2165,11 @@ class SpiderService extends Effect.Service()(
                 url: task.url,
                 message: "About to check shouldFollowUrl"
               });
+              const restrictToStartingDomainOption = restrictToStartingDomain ? Option.some(urlString) : Option.none();
               const shouldFollow = yield* config.shouldFollowUrl(
                 task.url,
                 task.fromUrl,
-                restrictToStartingDomain ? urlString : Option.none().pipe(Option.getOrUndefined)
+                Option.getOrUndefined(restrictToStartingDomainOption)
               );
               yield* logger.logEdgeCase(domain, "after_shouldFollowUrl", {
                 workerId,
@@ -2276,10 +2256,9 @@ class SpiderService extends Effect.Service()(
               });
               const pageData = yield* scraper.fetchAndParse(task.url, task.depth).pipe(
                 // Add overall timeout to prevent workers from hanging
-                Effect.timeout("45 seconds"),
+                Effect.timeout(SPIDER_DEFAULTS.FETCH_TIMEOUT),
                 Effect.retry({
-                  times: 2,
-                  // Reduced retries to prevent long hangs
+                  times: SPIDER_DEFAULTS.FETCH_RETRY_COUNT,
                   schedule: Schedule.exponential("1 second")
                 }),
                 Effect.catchAll(
@@ -2448,10 +2427,11 @@ class SpiderService extends Effect.Service()(
                   );
                   linksToProcess = resolvedLinks.filter(Option.isSome).map((opt) => opt.value);
                   for (const link of linksToProcess) {
+                    const linkRestrictOption = restrictToStartingDomain ? Option.some(urlString) : Option.none();
                     const linkShouldFollow = yield* config.shouldFollowUrl(
                       link,
                       task.url,
-                      restrictToStartingDomain ? urlString : Option.none().pipe(Option.getOrUndefined)
+                      Option.getOrUndefined(linkRestrictOption)
                     );
                     if (!linkShouldFollow.follow) {
                       continue;
@@ -2619,7 +2599,7 @@ class SpiderService extends Effect.Service()(
             let lastPageCount = 0;
             let stuckIterations = 0;
             while (!MutableRef.get(domainCompleted)) {
-              yield* Effect.sleep("30 seconds");
+              yield* Effect.sleep(SPIDER_DEFAULTS.FAILURE_DETECTOR_INTERVAL);
               const pageCount = yield* localDeduplicator.size();
               const queueSize = yield* queueManager.size();
               const activeCount = MutableRef.get(activeWorkers);
@@ -2844,17 +2824,7 @@ class SpiderService extends Effect.Service()(
             sessionId: stateKey.id,
             urlsProcessed: 0
           };
-        }),
-        /**
-         * Returns the list of URLs that have been visited during crawling.
-         *
-         * @returns Effect containing array of visited URLs
-         *
-         * @remarks
-         * This is currently a placeholder implementation. In a future version,
-         * this will return the actual list of visited URLs from the current session.
-         */
-        getVisitedUrls: () => Effect.sync(() => [])
+        })
       };
       return self;
     }),
@@ -4502,34 +4472,25 @@ class PostgresStorageBackend {
     return Effect.gen(function* () {
       if (self.db.transaction) {
         yield* Effect.tryPromise({
+          // pg transaction callback requires a Promise, not Effect — .then() chains are intentional
+          /* eslint-disable effect/no-promise-then-catch */
           try: () => self.db.transaction(
-            (tx) => (
-              // Delete in correct order due to foreign key constraints
-              // Use Effect.runPromise to execute Effect chain within the Promise callback
-              Effect.runPromise(
-                Effect.gen(function* () {
-                  yield* Effect.promise(
-                    () => tx.query(
-                      `DELETE FROM ${self.getTableName("snapshots")} WHERE session_id = $1`,
-                      [key.id]
-                    )
-                  );
-                  yield* Effect.promise(
-                    () => tx.query(
-                      `DELETE FROM ${self.getTableName("deltas")} WHERE session_id = $1`,
-                      [key.id]
-                    )
-                  );
-                  yield* Effect.promise(
-                    () => tx.query(
-                      `DELETE FROM ${self.getTableName("sessions")} WHERE id = $1`,
-                      [key.id]
-                    )
-                  );
-                })
+            (tx) => tx.query(
+              `DELETE FROM ${self.getTableName("snapshots")} WHERE session_id = $1`,
+              [key.id]
+            ).then(
+              () => tx.query(
+                `DELETE FROM ${self.getTableName("deltas")} WHERE session_id = $1`,
+                [key.id]
+              )
+            ).then(
+              () => tx.query(
+                `DELETE FROM ${self.getTableName("sessions")} WHERE id = $1`,
+                [key.id]
               )
             )
           ),
+          /* eslint-enable effect/no-promise-then-catch */
           catch: (error) => new PersistenceError2({
             message: `Failed to delete state from PostgreSQL: ${error}`,
             cause: error,
@@ -5136,7 +5097,7 @@ class JsonSchemaValidationError extends Data.TaggedError("JsonSchemaValidationEr
     return `JSON validation failed for schema "${this.schemaName}": ${this.cause}`;
   }
 }
-const isNonNullObject = (value) => typeof value === "object" && !Array.isArray(value) && Option.fromNullable(value)._tag === "Some";
+const isNonNullObject = (value) => typeof value === "object" && !Array.isArray(value) && Option.isSome(Option.fromNullable(value));
 const applyReplacer = (value, replacer) => {
   const transform = (key, val) => {
     const replaced = replacer(key, val);
@@ -5559,7 +5520,7 @@ const makeEnhancedHttpClient = Effect.gen(function* () {
         redirect: options.followRedirects === false ? "manual" : "follow",
         credentials: options.credentials ?? "same-origin"
       }),
-      catch: (error) => new NetworkError2({
+      catch: (error) => new NetworkError({
         url,
         method: options.method ?? "GET",
         cause: error
@@ -5593,7 +5554,7 @@ const makeEnhancedHttpClient = Effect.gen(function* () {
     );
     const response = yield* fetchWithTimeout;
     if (!response.ok) {
-      return yield* Effect.fail(new NetworkError2({
+      return yield* Effect.fail(new NetworkError({
         url: response.url,
         statusCode: response.status,
         method: options.method ?? "GET",
@@ -5656,7 +5617,7 @@ const makeEnhancedHttpClient = Effect.gen(function* () {
       Effect.retry({
         schedule: retrySchedule,
         while: (error) => {
-          if (error instanceof NetworkError2) {
+          if (error instanceof NetworkError) {
             if (error.statusCode && error.statusCode >= 400 && error.statusCode < 500) {
               return false;
             }
@@ -6775,28 +6736,38 @@ const WebScrapingEngineLive = Layer.effect(
   makeWebScrapingEngine
 );
 export {
+  AdapterNotInitialisedError,
+  BrowserCleanupError,
+  BrowserError,
+  ConfigError,
   ConfigurationError,
+  ContentTypeError,
   CookieManager,
   CookieManagerLive,
+  CrawlError,
   DEFAULT_HYBRID_CONFIG,
   DeltaPersistence,
   EnhancedHttpClient,
   EnhancedHttpClientLive,
   FileStorageBackend,
-  FileSystemError$1 as FileSystemError,
+  FileSystemError,
   FullStatePersistence,
   HybridPersistence,
   LinkExtractionError,
   LinkExtractorService,
   LinkExtractorServiceLayer,
   LoggingMiddleware,
-  MiddlewareError$1 as MiddlewareError,
+  MiddlewareError,
   MiddlewareManager,
-  NetworkError$1 as NetworkError,
+  NetworkError,
   PageDataSchema,
+  PageError,
+  ParseError,
   PersistenceError$1 as PersistenceError,
   PriorityRequest,
+  QueueError,
   RateLimitMiddleware,
+  RequestAbortError,
   ResponseError,
   ResumabilityConfigs,
   PersistenceError2 as ResumabilityError,
@@ -6804,6 +6775,7 @@ export {
   RobotsService,
   RobotsTxtError,
   ScraperService,
+  SessionError$1 as SessionError,
   SessionStore,
   SessionStoreLive,
   SpiderConfig,
@@ -6814,17 +6786,23 @@ export {
   SpiderState,
   SpiderStateKey,
   StateDelta,
+  StateError,
   StateManager,
   StateManagerLive,
   StatsMiddleware,
+  TimeoutError,
   TokenExtractor,
   TokenExtractorLive,
   TokenType,
   UrlDeduplicatorService,
   UserAgentMiddleware,
+  ValidationError,
   WebScrapingEngine,
   WebScrapingEngineLive,
   createStateOperation,
+  isBrowserError,
+  isNetworkError,
+  isSpiderError,
   makeCookieManager,
   makeEnhancedHttpClient,
   makeSessionStore,