@jambudipa/spider 0.1.0 → 0.2.1

package/dist/index.js

@@ -1,11 +1,11 @@
-import { Effect, Layer, MutableHashSet, Schema, Data, Context, Console, MutableHashMap, Option, Queue, PubSub, MutableRef, Schedule, Stream, Fiber, Random, Ref } from "effect";
+import { Effect, Layer, MutableHashSet, Schema, Data, Context, Console, MutableHashMap, Option, Queue, Ref, pipe, PubSub, MutableRef, Schedule, Stream, Fiber, Random, Duration } from "effect";
 import * as cheerio from "cheerio";
 import * as fs from "fs";
 import * as path from "path";
 import * as fs$1 from "fs/promises";
 import { CookieJar } from "tough-cookie";
 class SpiderConfig extends Effect.Service()(
-  "@jambudipa.io/SpiderConfig",
+  "@jambudipa/spiderConfig",
   {
     effect: Effect.sync(() => makeSpiderConfig({}))
   }
@@ -337,7 +337,7 @@ class UrlDeduplicatorService extends Effect.Service()(
       const shouldNormalize = yield* config.shouldNormalizeUrlsForDeduplication();
       const seenUrls = MutableHashSet.empty();
       const mutex = yield* Effect.makeSemaphore(1);
-      const normalizeUrl = (url) => {
+      const normalizeUrl2 = (url) => {
         if (!shouldNormalize) {
           return url;
         }
@@ -370,7 +370,7 @@ class UrlDeduplicatorService extends Effect.Service()(
       return {
         tryAdd: (url) => mutex.withPermits(1)(
           Effect.sync(() => {
-            const normalizedUrl = normalizeUrl(url);
+            const normalizedUrl = normalizeUrl2(url);
             if (MutableHashSet.has(seenUrls, normalizedUrl)) {
               return false;
             }
@@ -380,7 +380,7 @@ class UrlDeduplicatorService extends Effect.Service()(
         ),
         contains: (url) => mutex.withPermits(1)(
           Effect.sync(() => {
-            const normalizedUrl = normalizeUrl(url);
+            const normalizedUrl = normalizeUrl2(url);
             return MutableHashSet.has(seenUrls, normalizedUrl);
           })
         ),
@@ -439,7 +439,7 @@ const PageDataSchema = Schema.Struct({
     Schema.Record({ key: Schema.String, value: Schema.Unknown })
   )
 });
-class NetworkError extends Data.TaggedError("NetworkError") {
+let NetworkError$1 = class NetworkError extends Data.TaggedError("NetworkError") {
   static fromCause(url, cause) {
     return new NetworkError({
       url,
@@ -447,7 +447,7 @@ class NetworkError extends Data.TaggedError("NetworkError") {
       message: `Failed to fetch ${url}: ${cause}`
     });
   }
-}
+};
 class ResponseError extends Data.TaggedError("ResponseError") {
   static fromCause(url, cause) {
     return new ResponseError({
@@ -468,7 +468,7 @@ class RobotsTxtError extends Data.TaggedError("RobotsTxtError") {
 }
 class ConfigurationError extends Data.TaggedError("ConfigurationError") {
 }
-class MiddlewareError extends Data.TaggedError("MiddlewareError") {
+let MiddlewareError$1 = class MiddlewareError extends Data.TaggedError("MiddlewareError") {
   static transform(middlewareName, cause) {
     return new MiddlewareError({
       phase: "transform",
@@ -485,8 +485,8 @@ class MiddlewareError extends Data.TaggedError("MiddlewareError") {
       message: `Middleware '${middlewareName}' failed during error handling: ${cause}`
     });
   }
-}
-class FileSystemError extends Data.TaggedError("FileSystemError") {
+};
+let FileSystemError$1 = class FileSystemError extends Data.TaggedError("FileSystemError") {
   static write(path2, cause) {
     return new FileSystemError({
       operation: "write",
@@ -503,7 +503,7 @@ class FileSystemError extends Data.TaggedError("FileSystemError") {
       message: `Failed to create directory ${path2}: ${cause}`
     });
   }
-}
+};
 let PersistenceError$1 = class PersistenceError extends Data.TaggedError("PersistenceError") {
   static save(cause, key) {
     return new PersistenceError({
@@ -530,6 +530,61 @@ let PersistenceError$1 = class PersistenceError extends Data.TaggedError("Persis
     });
   }
 };
+class ContentTypeError extends Data.TaggedError("ContentTypeError") {
+  static create(url, contentType, expectedTypes) {
+    return new ContentTypeError({
+      url,
+      contentType,
+      expectedTypes,
+      message: `Invalid content type '${contentType}' for ${url}. Expected one of: ${expectedTypes.join(", ")}`
+    });
+  }
+}
+class RequestAbortError extends Data.TaggedError("RequestAbortError") {
+  static timeout(url, duration) {
+    return new RequestAbortError({
+      url,
+      duration,
+      reason: "timeout",
+      message: `Request to ${url} aborted after ${duration}ms due to timeout`
+    });
+  }
+  static cancelled(url, duration) {
+    return new RequestAbortError({
+      url,
+      duration,
+      reason: "cancelled",
+      message: `Request to ${url} cancelled after ${duration}ms`
+    });
+  }
+}
+class AdapterNotInitialisedError extends Data.TaggedError("AdapterNotInitialisedError") {
+  static create(adapterId, operation) {
+    return new AdapterNotInitialisedError({
+      adapterId,
+      operation,
+      message: `Adapter '${adapterId}' not initialised. Cannot perform operation: ${operation}`
+    });
+  }
+}
+class BrowserCleanupError extends Data.TaggedError("BrowserCleanupError") {
+  static context(id, cause) {
+    return new BrowserCleanupError({
+      resourceType: "context",
+      resourceId: id,
+      cause,
+      message: `Failed to close browser context '${id}': ${cause}`
+    });
+  }
+  static browser(id, cause) {
+    return new BrowserCleanupError({
+      resourceType: "browser",
+      resourceId: id,
+      cause,
+      message: `Failed to close browser '${id}': ${cause}`
+    });
+  }
+}
 const SpiderLogger = Context.GenericTag("SpiderLogger");
 const makeSpiderLogger = (logDir = "./spider-logs") => {
   if (!fs.existsSync(logDir)) {
@@ -852,20 +907,22 @@ class ScraperService extends Effect.Service()(
               clearTimeout(timeoutId);
               const contentType = resp.headers.get("content-type") || "";
               if (!contentType.includes("text/html") && !contentType.includes("application/xhtml") && !contentType.includes("text/") && contentType !== "") {
-                throw new Error(`Skipping non-HTML content: ${contentType}`);
+                throw ContentTypeError.create(
+                  url,
+                  contentType,
+                  ["text/html", "application/xhtml+xml", "text/*"]
+                );
               }
               return resp;
             } catch (error) {
               clearTimeout(timeoutId);
               if (error instanceof Error && error.name === "AbortError") {
-                throw new Error(
-                  `Request aborted after ${Date.now() - startMs}ms`
-                );
+                throw RequestAbortError.timeout(url, Date.now() - startMs);
               }
               throw error;
             }
           },
-          catch: (error) => NetworkError.fromCause(url, error)
+          catch: (error) => NetworkError$1.fromCause(url, error)
         });
         const textController = new AbortController();
         const textTimeoutMs = 1e4;
@@ -885,7 +942,7 @@ class ScraperService extends Effect.Service()(
           try: async () => {
             try {
               const reader = response.body?.getReader();
-              if (!reader) throw new Error("No response body");
+              if (!reader) throw ResponseError.fromCause(url, "No response body");
               const decoder = new TextDecoder();
               let html2 = "";
               while (true) {
@@ -894,7 +951,7 @@ class ScraperService extends Effect.Service()(
                 html2 += decoder.decode(value, { stream: true });
                 if (textController.signal.aborted) {
                   reader.cancel();
-                  throw new Error("Response parsing aborted");
+                  throw RequestAbortError.cancelled(url, Date.now() - startMs);
                 }
               }
               clearTimeout(textTimeoutId);
@@ -1189,7 +1246,7 @@ class SpiderState extends Schema.Class("SpiderState")({
 }) {
 }
 class SpiderSchedulerService extends Effect.Service()(
-  "@jambudipa.io/SpiderSchedulerService",
+  "@jambudipa/spiderSchedulerService",
   {
     effect: Effect.gen(function* () {
       const config = yield* SpiderConfig;
@@ -1200,7 +1257,7 @@ class SpiderSchedulerService extends Effect.Service()(
       let totalProcessed = 0;
       let persistenceLayer = null;
       let currentStateKey = null;
-      const normalizeUrl = (url) => {
+      const normalizeUrl2 = (url) => {
         if (!shouldNormalizeUrls) {
           return url;
         }
@@ -1231,7 +1288,7 @@ class SpiderSchedulerService extends Effect.Service()(
         }
       };
       const generateFingerprint = (request) => {
-        const normalizedUrl = normalizeUrl(request.url);
+        const normalizedUrl = normalizeUrl2(request.url);
         return `${normalizedUrl}:${request.depth}`;
       };
       const createPriorityRequest = (request, priority) => new PriorityRequest({
@@ -1364,8 +1421,270 @@ const SpiderScheduler_service = /* @__PURE__ */ Object.freeze(/* @__PURE__ */ Ob
   SpiderState,
   SpiderStateKey
 }, Symbol.toStringTag, { value: "Module" }));
+class SpiderError extends Data.TaggedError("SpiderError") {
+  get message() {
+    return `Spider operation '${this.operation}' failed${this.details ? `: ${JSON.stringify(this.details)}` : ""}`;
+  }
+}
+class NetworkError2 extends Data.TaggedError("NetworkError") {
+  get message() {
+    return `Network request to ${this.url} failed${this.statusCode ? ` with status ${this.statusCode}` : ""}`;
+  }
+  static fromResponse(url, response) {
+    return new NetworkError2({
+      url,
+      statusCode: response.status,
+      method: "GET"
+    });
+  }
+  static fromCause(url, cause) {
+    return new NetworkError2({ url, cause });
+  }
+}
+class TimeoutError extends Data.TaggedError("TimeoutError") {
+  get message() {
+    return `Operation '${this.operation}' timed out after ${this.timeoutMs}ms for ${this.url}`;
+  }
+}
+class ParseError extends Data.TaggedError("ParseError") {
+  get message() {
+    return `Failed to parse ${this.expected}${this.input ? ` from input: ${this.input.substring(0, 100)}...` : ""}`;
+  }
+  static json(input, cause) {
+    return new ParseError({
+      input,
+      expected: "JSON",
+      cause
+    });
+  }
+  static html(input, cause) {
+    return new ParseError({
+      input,
+      expected: "HTML",
+      cause
+    });
+  }
+}
+class ValidationError extends Data.TaggedError("ValidationError") {
+  get message() {
+    return `Validation failed for field '${this.field}': ${this.constraint}`;
+  }
+  static url(url) {
+    return new ValidationError({
+      field: "url",
+      value: url,
+      constraint: "Invalid URL format"
+    });
+  }
+}
+class BrowserError extends Data.TaggedError("BrowserError") {
+  get message() {
+    return `Browser operation '${this.operation}' failed${this.browserId ? ` for browser ${this.browserId}` : ""}`;
+  }
+  static notLaunched() {
+    return new BrowserError({
+      operation: "access",
+      cause: "Browser not launched"
+    });
+  }
+  static launchFailed(cause) {
+    return new BrowserError({
+      operation: "launch",
+      cause
+    });
+  }
+}
+class PageError extends Data.TaggedError("PageError") {
+  get message() {
+    return `Page operation '${this.operation}' failed for ${this.url}${this.selector ? ` with selector '${this.selector}'` : ""}`;
+  }
+}
+class StateError extends Data.TaggedError("StateError") {
+  get message() {
+    return `State ${this.operation} operation failed${this.stateKey ? ` for key '${this.stateKey}'` : ""}`;
+  }
+}
+class SessionError extends Data.TaggedError("SessionError") {
+  get message() {
+    return `Session operation '${this.operation}' failed${this.sessionId ? ` for session ${this.sessionId}` : ""}`;
+  }
+  static noActiveSession() {
+    return new SessionError({
+      operation: "access",
+      cause: "No active session"
+    });
+  }
+}
+class FileSystemError2 extends Data.TaggedError("FileSystemError") {
+  get message() {
+    return `File system ${this.operation} operation failed for path: ${this.path}`;
+  }
+}
+class CrawlError extends Data.TaggedError("CrawlError") {
+  get message() {
+    return `Failed to crawl ${this.url} at depth ${this.depth}: ${this.reason}`;
+  }
+  static maxDepthReached(url, depth) {
+    return new CrawlError({
+      url,
+      depth,
+      reason: "Maximum depth reached"
+    });
+  }
+  static robotsBlocked(url) {
+    return new CrawlError({
+      url,
+      depth: 0,
+      reason: "Blocked by robots.txt"
+    });
+  }
+}
+class QueueError extends Data.TaggedError("QueueError") {
+  get message() {
+    return `Queue ${this.operation} operation failed${this.queueSize !== void 0 ? ` (queue size: ${this.queueSize})` : ""}`;
+  }
+}
+class ConfigError extends Data.TaggedError("ConfigError") {
+  get message() {
+    return `Configuration error for '${this.field}': ${this.reason}`;
+  }
+  static invalid(field, value, expected) {
+    return new ConfigError({
+      field,
+      value,
+      reason: `Expected ${expected}, got ${typeof value}`
+    });
+  }
+}
+class MiddlewareError2 extends Data.TaggedError("MiddlewareError") {
+  get message() {
+    return `Middleware '${this.middlewareName}' failed during ${this.phase} phase`;
+  }
+}
+const DEFAULT_DEDUPLICATION_STRATEGY = {
+  wwwHandling: "ignore",
+  protocolHandling: "prefer-https",
+  trailingSlashHandling: "ignore",
+  queryParamHandling: "preserve",
+  fragmentHandling: "ignore"
+};
+const parseUrl = (url) => Effect.try({
+  try: () => new URL(url),
+  catch: () => ValidationError.url(url)
+});
+const normalizeUrl = (url, strategy = DEFAULT_DEDUPLICATION_STRATEGY) => Effect.gen(function* () {
+  const parsed = yield* parseUrl(url);
+  if (strategy.protocolHandling === "prefer-https") {
+    parsed.protocol = "https:";
+  }
+  let domain = parsed.hostname.toLowerCase();
+  const hasWww = domain.startsWith("www.");
+  const domainWithoutWww = hasWww ? domain.substring(4) : domain;
+  switch (strategy.wwwHandling) {
+    case "ignore":
+    case "prefer-non-www":
+      domain = domainWithoutWww;
+      parsed.hostname = domain;
+      break;
+    case "prefer-www":
+      if (!hasWww) {
+        domain = `www.${domain}`;
+        parsed.hostname = domain;
+      }
+      break;
+  }
+  if (strategy.trailingSlashHandling === "ignore") {
+    parsed.pathname = parsed.pathname.replace(/\/$/, "") || "/";
+  }
+  if (strategy.queryParamHandling === "ignore") {
+    parsed.search = "";
+  } else if (strategy.queryParamHandling === "sort") {
+    const params = new URLSearchParams(parsed.search);
+    const sorted = Array.from(params.entries()).sort(([a], [b]) => a.localeCompare(b));
+    parsed.search = new URLSearchParams(sorted).toString();
+  }
+  if (strategy.fragmentHandling === "ignore") {
+    parsed.hash = "";
+  }
+  return {
+    original: url,
+    normalized: parsed.toString(),
+    domain: domainWithoutWww
+  };
+});
+const deduplicateUrls = (urls, strategy = DEFAULT_DEDUPLICATION_STRATEGY) => Effect.gen(function* () {
+  const domainMap = yield* Ref.make(/* @__PURE__ */ new Map());
+  const skipped = yield* Ref.make([]);
+  let invalidCount = 0;
+  yield* Effect.all(
+    urls.map(
+      (urlObj) => pipe(
+        normalizeUrl(urlObj.url, strategy),
+        Effect.tap(
+          (normalized) => Effect.gen(function* () {
+            const currentMap = yield* Ref.get(domainMap);
+            const key = strategy.wwwHandling === "preserve" ? normalized.normalized : normalized.domain;
+            if (!currentMap.has(key)) {
+              currentMap.set(key, urlObj);
+              yield* Ref.set(domainMap, currentMap);
+            } else {
+              const existing = currentMap.get(key);
+              let shouldReplace = false;
+              if (strategy.wwwHandling === "prefer-www") {
+                const existingHasWww = existing.url.includes("://www.");
+                const newHasWww = urlObj.url.includes("://www.");
+                shouldReplace = !existingHasWww && newHasWww;
+              } else if (strategy.wwwHandling === "prefer-non-www") {
+                const existingHasWww = existing.url.includes("://www.");
+                const newHasWww = urlObj.url.includes("://www.");
+                shouldReplace = existingHasWww && !newHasWww;
+              }
+              if (shouldReplace) {
+                currentMap.set(key, urlObj);
+                yield* Ref.set(domainMap, currentMap);
+                yield* Ref.update(skipped, (arr) => [
+                  ...arr,
+                  { url: existing.url, reason: `Replaced by preferred variant: ${urlObj.url}` }
+                ]);
+              } else {
+                yield* Ref.update(skipped, (arr) => [
+                  ...arr,
+                  { url: urlObj.url, reason: `Duplicate of: ${existing.url}` }
+                ]);
+              }
+            }
+          })
+        ),
+        Effect.catchAll(
+          (error) => Effect.gen(function* () {
+            invalidCount++;
+            yield* Ref.update(skipped, (arr) => [
+              ...arr,
+              { url: urlObj.url, reason: `Invalid URL: ${error.message}` }
+            ]);
+            yield* Effect.logWarning(`Invalid URL skipped: ${urlObj.url}`);
+          })
+        )
+      )
+    ),
+    { concurrency: "unbounded" }
+  );
+  const finalMap = yield* Ref.get(domainMap);
+  const finalSkipped = yield* Ref.get(skipped);
+  const deduplicated = Array.from(finalMap.values());
+  return {
+    deduplicated,
+    skipped: finalSkipped,
+    stats: {
+      total: urls.length,
+      unique: deduplicated.length,
+      duplicates: finalSkipped.filter((s) => s.reason.startsWith("Duplicate")).length,
+      invalid: invalidCount
+    }
+  };
+});
 class SpiderService extends Effect.Service()(
-  "@jambudipa.io/Spider",
+  "@jambudipa/spider",
   {
     effect: Effect.gen(function* () {
       const robots = yield* RobotsService;
@@ -1375,7 +1694,7 @@ class SpiderService extends Effect.Service()(
       const maybeScheduler = yield* Effect.serviceOption(
         SpiderSchedulerService
       );
-      const scheduler = Option.isSome(maybeScheduler) ? maybeScheduler.value : null;
+      Option.isSome(maybeScheduler) ? maybeScheduler.value : null;
       const self = {
         /**
          * Starts crawling from the specified URL and processes results through the provided sink.
@@ -1440,24 +1759,27 @@ class SpiderService extends Effect.Service()(
             return [input];
           };
           const urlsWithMetadata = normalizeUrlInput(startingUrls);
-          const domainMap = /* @__PURE__ */ new Map();
-          for (const urlObj of urlsWithMetadata) {
-            try {
-              const url = new URL(urlObj.url);
-              const domain = url.hostname.toLowerCase();
-              const normalizedDomain = domain.replace(/^www\./, "");
-              if (!domainMap.has(normalizedDomain)) {
-                domainMap.set(normalizedDomain, urlObj);
-              } else {
-                console.warn(
-                  `Skipping duplicate domain: ${domain} (normalized: ${normalizedDomain}, URL: ${urlObj.url})`
-                );
-              }
-            } catch (e) {
-              console.error(`Invalid URL skipped: ${urlObj.url}`, e);
+          const deduplicationResult = yield* deduplicateUrls(
+            urlsWithMetadata,
+            {
+              // Strategy: Treat www and non-www as the same domain by default
+              // This can be configured via Spider options if needed
+              wwwHandling: "ignore",
+              protocolHandling: "prefer-https",
+              trailingSlashHandling: "ignore",
+              queryParamHandling: "preserve",
+              fragmentHandling: "ignore"
             }
+          );
+          const deduplicatedUrls = deduplicationResult.deduplicated;
+          if (deduplicationResult.stats.duplicates > 0) {
+            yield* Effect.logInfo(
+              `URL deduplication: ${deduplicationResult.stats.total} total, ${deduplicationResult.stats.unique} unique, ${deduplicationResult.stats.duplicates} duplicates removed`
+            );
+          }
+          for (const skipped of deduplicationResult.skipped) {
+            yield* Effect.logDebug(`Skipped URL: ${skipped.url} - Reason: ${skipped.reason}`);
           }
-          const deduplicatedUrls = Array.from(domainMap.values());
           const concurrency = yield* config.getConcurrency();
           if (deduplicatedUrls.length > 1) {
             const configOptions = yield* config.getOptions();
@@ -2268,13 +2590,6 @@ class SpiderService extends Effect.Service()(
          * ```
          */
         resume: (stateKey, _sink, _persistence) => Effect.gen(function* () {
-          if (!scheduler) {
-            return yield* Effect.fail(
-              new Error(
-                "Resume functionality requires SpiderSchedulerService to be available. Make sure resumability is enabled in SpiderConfig and SpiderSchedulerService is provided."
-              )
-            );
-          }
           const config = yield* SpiderConfig;
           if (!config) {
             return yield* Effect.fail(
@@ -2291,10 +2606,75 @@ class SpiderService extends Effect.Service()(
               )
             );
           }
-          console.log(`Resuming session: ${stateKey.id}`);
+          const scheduler2 = yield* SpiderSchedulerService;
+          const logger2 = yield* SpiderLogger;
+          yield* logger2.logSpiderLifecycle("start", {
+            sessionId: stateKey.id,
+            timestamp: (/* @__PURE__ */ new Date()).toISOString()
+          });
+          const savedState = yield* Effect.tryPromise({
+            try: async () => {
+              return scheduler2.getState ? await Effect.runPromise(scheduler2.getState()) : null;
+            },
+            catch: (error) => new StateError({
+              operation: "load",
+              stateKey: stateKey.id,
+              cause: error
+            })
+          });
+          if (!savedState) {
+            return yield* Effect.fail(
+              new StateError({
+                operation: "load",
+                stateKey: stateKey.id,
+                cause: "No saved state found for session"
+              })
+            );
+          }
+          const restoredUrls = yield* Effect.try({
+            try: () => {
+              const urls = [];
+              if (savedState && typeof savedState === "object") {
+                if ("pendingUrls" in savedState && Array.isArray(savedState.pendingUrls)) {
+                  urls.push(...savedState.pendingUrls);
+                }
+                if ("visitedUrls" in savedState && Array.isArray(savedState.visitedUrls)) ;
+              }
+              return urls;
+            },
+            catch: (error) => new ParseError({
+              input: "saved state",
+              expected: "crawl state",
+              cause: error
+            })
+          });
+          yield* logger2.logSpiderLifecycle("start", {
+            sessionId: stateKey.id,
+            pendingUrls: restoredUrls.length,
+            timestamp: (/* @__PURE__ */ new Date()).toISOString()
+          });
+          if (restoredUrls.length > 0) {
+            const crawlResult = yield* self.crawl(
+              restoredUrls,
+              _sink,
+              {}
+            );
+            yield* logger2.logSpiderLifecycle("complete", {
+              sessionId: stateKey.id,
+              urlsProcessed: restoredUrls.length,
+              timestamp: (/* @__PURE__ */ new Date()).toISOString()
+            });
+            return {
+              ...crawlResult,
+              resumed: true,
+              sessionId: stateKey.id
+            };
+          }
           return {
             completed: true,
-            resumed: true
+            resumed: true,
+            sessionId: stateKey.id,
+            urlsProcessed: 0
           };
         }),
         /**
@@ -2321,6 +2701,80 @@ class SpiderService extends Effect.Service()(
   }
 ) {
 }
+class SpiderRequest extends Data.Class {
+  /**
+   * Create a SpiderRequest from a CrawlTask
+   */
+  static fromTask(task, headers, meta) {
+    return new SpiderRequest({
+      task,
+      headers: Option.fromNullable(headers),
+      meta: Option.fromNullable(meta)
+    });
+  }
+  /**
+   * Add or update headers
+   */
+  withHeaders(headers) {
+    const existingHeaders = Option.getOrElse(this.headers, () => ({}));
+    return new SpiderRequest({
+      ...this,
+      headers: Option.some({ ...existingHeaders, ...headers })
+    });
+  }
+  /**
+   * Add or update metadata
+   */
+  withMeta(meta) {
+    const existingMeta = Option.getOrElse(this.meta, () => ({}));
+    return new SpiderRequest({
+      ...this,
+      meta: Option.some({ ...existingMeta, ...meta })
+    });
+  }
+}
+class SpiderResponse extends Data.Class {
+  /**
+   * Create a SpiderResponse from PageData
+   */
+  static fromPageData(pageData, statusCode, headers, meta) {
+    return new SpiderResponse({
+      pageData,
+      statusCode: Option.fromNullable(statusCode),
+      headers: Option.fromNullable(headers),
+      meta: Option.fromNullable(meta)
+    });
+  }
+  /**
+   * Update the page data
+   */
+  withPageData(pageData) {
+    return new SpiderResponse({
+      ...this,
+      pageData
+    });
+  }
+  /**
+   * Add or update metadata
+   */
+  withMeta(meta) {
+    const existingMeta = Option.getOrElse(this.meta, () => ({}));
+    return new SpiderResponse({
+      ...this,
+      meta: Option.some({ ...existingMeta, ...meta })
+    });
+  }
+  /**
+   * Check if the response was successful (2xx status code)
+   */
+  isSuccessful() {
+    return Option.match(this.statusCode, {
+      onNone: () => true,
+      // Assume success if no status code
+      onSome: (code) => code >= 200 && code < 300
+    });
+  }
+}
 class MiddlewareManager extends Effect.Service()(
   "@jambudipa.io/MiddlewareManager",
   {
@@ -2498,13 +2952,9 @@ class UserAgentMiddleware extends Effect.Service()(
   {
     effect: Effect.sync(() => ({
       create: (userAgent) => ({
-        processRequest: (request) => Effect.succeed({
-          ...request,
-          headers: {
-            ...request.headers,
-            "User-Agent": userAgent
-          }
-        })
+        processRequest: (request) => Effect.succeed(
+          request.withHeaders({ "User-Agent": userAgent })
+        )
       })
     }))
   }
@@ -2533,14 +2983,18 @@ class StatsMiddleware extends Effect.Service()(
             }),
             processResponse: (response) => Effect.sync(() => {
               incr("responses_received");
-              if (response.statusCode) {
-                incr(`status_${response.statusCode}`);
-                if (response.statusCode >= 200 && response.statusCode < 300) {
-                  incr("responses_success");
-                } else if (response.statusCode >= 400) {
-                  incr("responses_error");
+              Option.match(response.statusCode, {
+                onNone: () => {
+                },
+                onSome: (statusCode) => {
+                  incr(`status_${statusCode}`);
+                  if (statusCode >= 200 && statusCode < 300) {
+                    incr("responses_success");
+                  } else if (statusCode >= 400) {
+                    incr("responses_error");
+                  }
                 }
-              }
+              });
               incr("bytes_downloaded", response.pageData.html.length);
               return response;
             }),
@@ -3559,6 +4013,207 @@ class FileStorageBackend {
     return path.join(this.baseDir, "sessions", key.id);
   };
 }
+class JsonParseError extends Data.TaggedError("JsonParseError") {
+  get message() {
+    const preview = this.input.length > 100 ? `${this.input.substring(0, 100)}...` : this.input;
+    return `Failed to parse JSON: ${this.cause}. Input: "${preview}"`;
+  }
+}
+class JsonStringifyError extends Data.TaggedError("JsonStringifyError") {
+  get message() {
+    const typeInfo = typeof this.input === "object" ? this.input?.constructor?.name || "Object" : typeof this.input;
+    return `Failed to stringify value of type ${typeInfo}: ${this.cause}`;
+  }
+}
+class JsonSchemaValidationError extends Data.TaggedError("JsonSchemaValidationError") {
+  get message() {
+    return `JSON validation failed for schema "${this.schemaName}": ${this.cause}`;
+  }
+}
+const JsonUtils = {
+  /**
+   * Safely parse JSON string
+   * 
+   * @example
+   * ```ts
+   * const result = yield* JsonUtils.parse('{"name": "test"}');
+   * // result: { name: "test" }
+   * ```
+   */
+  parse: (input) => Effect.try({
+    try: () => JSON.parse(input),
+    catch: (cause) => new JsonParseError({ input, cause })
+  }),
+  /**
+   * Parse JSON with schema validation
+   * 
+   * @example
+   * ```ts
+   * const UserSchema = Schema.Struct({
+   *   name: Schema.String,
+   *   age: Schema.Number
+   * });
+   * 
+   * const user = yield* JsonUtils.parseWithSchema(
+   *   '{"name": "Alice", "age": 30}',
+   *   UserSchema
+   * );
+   * ```
+   */
+  parseWithSchema: (input, schema, options) => Effect.gen(function* () {
+    const parsed = yield* JsonUtils.parse(input);
+    return yield* Effect.try({
+      try: () => {
+        const parseResult = Schema.decodeUnknownSync(schema, {
+          errors: "all",
+          ...options
+        })(parsed);
+        return parseResult;
+      },
+      catch: (cause) => new JsonSchemaValidationError({
+        input: parsed,
+        schemaName: schema.ast._tag || "Unknown",
+        cause
+      })
+    });
+  }),
+  /**
+   * Safely stringify value to JSON
+   * 
+   * @example
+   * ```ts
+   * const json = yield* JsonUtils.stringify({ name: "test" });
+   * // json: '{"name":"test"}'
+   * 
+   * const pretty = yield* JsonUtils.stringify({ name: "test" }, 2);
+   * // pretty: '{\n  "name": "test"\n}'
+   * ```
+   */
+  stringify: (value, space, replacer) => Effect.try({
+    try: () => JSON.stringify(value, replacer, space),
+    catch: (cause) => new JsonStringifyError({ input: value, cause })
+  }),
+  /**
+   * Parse JSON with fallback value
+   * 
+   * @example
+   * ```ts
+   * const config = yield* JsonUtils.parseOrDefault(
+   *   configStr,
+   *   { debug: false }
+   * );
+   * ```
+   */
+  parseOrDefault: (input, defaultValue) => JsonUtils.parse(input).pipe(
+    Effect.catchAll(() => Effect.succeed(defaultValue))
+  ),
+  /**
+   * Parse JSON and return null on failure
+   * 
+   * @example
+   * ```ts
+   * const data = yield* JsonUtils.parseOrNull(input);
+   * if (data !== null) {
+   *   // Use parsed data
+   * }
+   * ```
+   */
+  parseOrNull: (input) => JsonUtils.parse(input).pipe(
+    Effect.catchAll(() => Effect.succeed(null))
+  ),
+  /**
+   * Try to parse JSON and return boolean success
+   * 
+   * @example
+   * ```ts
+   * const isValid = yield* JsonUtils.isValid('{"valid": true}');
+   * // isValid: true
+   * ```
+   */
+  isValid: (input) => JsonUtils.parse(input).pipe(
+    Effect.map(() => true),
+    Effect.catchAll(() => Effect.succeed(false))
+  ),
+  /**
+   * Pretty print JSON with indentation
+   * 
+   * @example
+   * ```ts
+   * const pretty = yield* JsonUtils.prettyPrint({ complex: { data: true } });
+   * ```
+   */
+  prettyPrint: (value, indent = 2) => JsonUtils.stringify(value, indent),
+  /**
+   * Deep clone an object via JSON serialization
+   * Note: This will lose functions, undefined values, symbols, etc.
+   * 
+   * @example
+   * ```ts
+   * const clone = yield* JsonUtils.deepClone(originalObject);
+   * ```
+   */
+  deepClone: (value) => Effect.gen(function* () {
+    const json = yield* JsonUtils.stringify(value);
+    return yield* JsonUtils.parse(json);
+  }),
+  /**
+   * Merge two JSON objects
+   * 
+   * @example
+   * ```ts
+   * const merged = yield* JsonUtils.merge(
+   *   { a: 1 },
+   *   { b: 2 }
+   * );
+   * // merged: { a: 1, b: 2 }
+   * ```
+   */
+  merge: (target, source) => Effect.gen(function* () {
+    const clonedTarget = yield* JsonUtils.deepClone(target);
+    const clonedSource = yield* JsonUtils.deepClone(source);
+    return { ...clonedTarget, ...clonedSource };
+  }),
+  /**
+   * Extract a subset of JSON properties
+   * 
+   * @example
+   * ```ts
+   * const subset = yield* JsonUtils.pick(
+   *   { a: 1, b: 2, c: 3 },
+   *   ['a', 'c']
+   * );
+   * // subset: { a: 1, c: 3 }
+   * ```
+   */
+  pick: (obj, keys) => Effect.succeed(
+    keys.reduce((acc, key) => {
+      if (key in obj) {
+        acc[key] = obj[key];
+      }
+      return acc;
+    }, {})
+  ),
+  /**
+   * Omit properties from JSON object
+   * 
+   * @example
+   * ```ts
+   * const result = yield* JsonUtils.omit(
+   *   { a: 1, b: 2, c: 3 },
+   *   ['b']
+   * );
+   * // result: { a: 1, c: 3 }
+   * ```
+   */
+  omit: (obj, keys) => Effect.succeed(
+    Object.keys(obj).reduce((acc, key) => {
+      if (!keys.includes(key)) {
+        acc[key] = obj[key];
+      }
+      return acc;
+    }, {})
+  )
+};
 class CookieManager extends Context.Tag("CookieManager")() {
 }
 const makeCookieManager = () => Effect.gen(function* () {
@@ -3618,19 +4273,17 @@ const makeCookieManager = () => Effect.gen(function* () {
         }),
         catch: () => new Error("Failed to serialize cookies")
       });
-      return JSON.stringify(serialized);
+      return yield* JsonUtils.stringify(serialized);
     }).pipe(Effect.orElseSucceed(() => "{}")),
     deserialize: (data) => Effect.gen(function* () {
-      try {
-        const parsed = JSON.parse(data);
-        const newJar = CookieJar.deserialize(parsed);
-        yield* Effect.tryPromise({
-          try: () => Promise.resolve(newJar),
-          catch: () => new Error("Failed to deserialize cookie jar")
-        }).pipe(Effect.flatMap((jar2) => Ref.set(jarRef, jar2)));
-      } catch (error) {
-        yield* Effect.fail(new Error(`Invalid cookie data: ${error}`));
-      }
+      const parsed = yield* JsonUtils.parse(data).pipe(
+        Effect.mapError((error) => new Error(`Invalid cookie JSON format: ${error.message}`))
+      );
+      const newJar = yield* Effect.tryPromise({
+        try: () => CookieJar.deserialize(parsed),
+        catch: (error) => new Error(`Failed to deserialize cookie jar: ${error}`)
+      });
+      yield* Ref.set(jarRef, newJar);
     })
   };
 });
@@ -3643,7 +4296,7 @@ class EnhancedHttpClient extends Context.Tag("EnhancedHttpClient")() {
 const makeEnhancedHttpClient = Effect.gen(function* () {
   const logger = yield* SpiderLogger;
   const cookieManager = yield* CookieManager;
-  const makeRequest = (url, options = {}) => Effect.gen(function* () {
+  const makeRequest = (url, options = {}) => Effect.gen(function* (_) {
     const startMs = Date.now();
     const domain = new URL(url).hostname;
     const cookieHeader = yield* cookieManager.getCookieHeader(url);
@@ -3656,12 +4309,15 @@ const makeEnhancedHttpClient = Effect.gen(function* () {
     }
     if (options.method === "POST" && options.body && !headers["Content-Type"]) {
       if (typeof options.body === "string") {
-        try {
-          JSON.parse(options.body);
-          headers["Content-Type"] = "application/json";
-        } catch {
-          headers["Content-Type"] = "application/x-www-form-urlencoded";
-        }
+        const isJson = yield* Effect.succeed((() => {
+          try {
+            JSON.parse(options.body);
+            return true;
+          } catch {
+            return false;
+          }
+        })());
+        headers["Content-Type"] = isJson ? "application/json" : "application/x-www-form-urlencoded";
       } else if (options.body instanceof FormData) ;
       else if (options.body instanceof URLSearchParams) {
         headers["Content-Type"] = "application/x-www-form-urlencoded";
@@ -3697,34 +4353,93 @@ const makeEnhancedHttpClient = Effect.gen(function* () {
       },
       catch: (error) => {
         clearTimeout(timeoutId);
-        return NetworkError.fromCause(url, error);
+        if (error instanceof Error && error.name === "AbortError") {
+          return new TimeoutError({
+            operation: `HTTP ${options.method || "GET"}`,
+            timeoutMs,
+            url
+          });
+        }
+        return new NetworkError2({
+          url,
+          method: options.method || "GET",
+          cause: error
+        });
       }
     });
+    if (!response.ok) {
+      return yield* Effect.fail(new NetworkError2({
+        url: response.url,
+        statusCode: response.status,
+        method: options.method || "GET",
+        cause: `HTTP ${response.status}: ${response.statusText}`
+      }));
+    }
     const body = yield* Effect.tryPromise({
       try: () => response.text(),
-      catch: (error) => ResponseError.fromCause(url, error)
+      catch: (error) => new ParseError({
+        input: url,
+        expected: "text",
+        cause: error
+      })
     });
     const setCookieHeaders = response.headers.getSetCookie ? response.headers.getSetCookie() : response.headers.get("set-cookie")?.split(", ") || [];
     for (const cookieString of setCookieHeaders) {
       if (cookieString) {
-        yield* cookieManager.setCookie(cookieString, url).pipe(Effect.catchAll(() => Effect.void));
+        yield* cookieManager.setCookie(cookieString, url).pipe(Effect.catchAll(() => Effect.succeed(void 0)));
       }
     }
     const responseHeaders = {};
     response.headers.forEach((value, key) => {
       responseHeaders[key] = value;
     });
-    return {
+    const result = {
       url: response.url,
       status: response.status,
       statusText: response.statusText,
       headers: responseHeaders,
       body,
-      cookies: setCookieHeaders
+      cookies: setCookieHeaders.length > 0 ? setCookieHeaders : void 0
     };
+    return result;
   });
+  const makeRequestWithRetry = (url, options = {}) => {
+    const retries = options.retries ?? 3;
+    const retryDelay = options.retryDelay ?? 1e3;
+    const retrySchedule = Schedule.exponential(Duration.millis(retryDelay), 2).pipe(
+      Schedule.compose(Schedule.recurs(retries)),
+      Schedule.tapInput(
+        (error) => Effect.gen(function* () {
+          yield* logger.logEdgeCase(
+            new URL(url).hostname,
+            "http_request_retry",
+            {
+              url,
+              method: options.method || "GET",
+              error: error instanceof Error ? error.message : String(error),
+              attempt: retries
+            }
+          );
+        })
+      )
+    );
+    return makeRequest(url, options).pipe(
+      Effect.retry({
+        schedule: retrySchedule,
+        while: (error) => {
+          if (error instanceof NetworkError2) {
+            if (error.statusCode && error.statusCode >= 400 && error.statusCode < 500) {
+              return false;
+            }
+            return true;
+          }
+          return error instanceof TimeoutError;
+        }
+      })
+    );
+  };
   return {
-    get: (url, options) => makeRequest(url, { ...options, method: "GET" }),
+    get: (url, options) => makeRequestWithRetry(url, { ...options, method: "GET" }),
     post: (url, data, options) => Effect.gen(function* () {
       let body;
       if (data) {
@@ -3734,15 +4449,15 @@ const makeEnhancedHttpClient = Effect.gen(function* () {
           body = JSON.stringify(data);
         }
       }
-      return yield* makeRequest(url, { ...options, method: "POST", body });
+      return yield* makeRequestWithRetry(url, { ...options, method: "POST", body });
     }),
-    request: makeRequest,
+    request: makeRequestWithRetry,
     submitForm: (url, formData, options) => Effect.gen(function* () {
       const params = new URLSearchParams();
       for (const [key, value] of Object.entries(formData)) {
         params.append(key, value);
       }
-      return yield* makeRequest(url, {
+      return yield* makeRequestWithRetry(url, {
         ...options,
         method: "POST",
         body: params,
@@ -3856,17 +4571,23 @@ const makeSessionStore = Effect.gen(function* () {
       yield* cookieManager.clearCookies();
     }),
     isSessionValid: () => Effect.gen(function* () {
-      const session = yield* Effect.gen(function* () {
+      const sessionOption = yield* Effect.gen(function* () {
         const sessionId = yield* Ref.get(currentSessionId);
-        if (Option.isNone(sessionId)) return null;
+        if (Option.isNone(sessionId)) {
+          return Option.none();
+        }
         const sessionsMap = yield* Ref.get(sessions);
-        return sessionsMap.get(sessionId.value) || null;
+        return Option.fromNullable(sessionsMap.get(sessionId.value));
+      });
+      return Option.match(sessionOption, {
+        onNone: () => false,
+        onSome: (session) => {
+          if (session.expiresAt && session.expiresAt < /* @__PURE__ */ new Date()) {
+            return false;
+          }
+          return true;
+        }
       });
-      if (!session) return false;
-      if (session.expiresAt && session.expiresAt < /* @__PURE__ */ new Date()) {
-        return false;
-      }
-      return true;
     }),
     updateSessionData: (data) => Effect.gen(function* () {
       const sessionId = yield* Ref.get(currentSessionId);
@@ -3900,25 +4621,25 @@ const makeSessionStore = Effect.gen(function* () {
       });
     }),
     importSession: (data) => Effect.gen(function* () {
-      try {
-        const parsed = JSON.parse(data);
-        const session = {
+      const parsed = yield* Effect.try({
+        try: () => JSON.parse(data),
+        catch: (error) => new Error(`Invalid session JSON: ${error}`)
+      });
+      const session = yield* Effect.try({
+        try: () => ({
           ...parsed,
           tokens: new Map(parsed.tokens || []),
           createdAt: new Date(parsed.createdAt),
           lastUsedAt: new Date(parsed.lastUsedAt),
           expiresAt: parsed.expiresAt ? new Date(parsed.expiresAt) : void 0
-        };
-        const sessionsMap = yield* Ref.get(sessions);
-        sessionsMap.set(session.id, session);
-        yield* Ref.set(sessions, sessionsMap);
-        yield* Effect.gen(function* () {
-          yield* cookieManager.deserialize(session.cookies);
-          yield* Ref.set(currentSessionId, Option.some(session.id));
-        });
-      } catch (error) {
-        yield* Effect.fail(new Error(`Invalid session data: ${error}`));
-      }
+        }),
+        catch: (error) => new Error(`Failed to reconstruct session: ${error}`)
+      });
+      const sessionsMap = yield* Ref.get(sessions);
+      sessionsMap.set(session.id, session);
+      yield* Ref.set(sessions, sessionsMap);
+      yield* cookieManager.deserialize(session.cookies);
+      yield* Ref.set(currentSessionId, Option.some(session.id));
     })
   };
 });
@@ -4627,16 +5348,16 @@ export {
   EnhancedHttpClient,
   EnhancedHttpClientLive,
   FileStorageBackend,
-  FileSystemError,
+  FileSystemError$1 as FileSystemError,
   FullStatePersistence,
   HybridPersistence,
   LinkExtractionError,
   LinkExtractorService,
   LinkExtractorServiceLayer,
   LoggingMiddleware,
-  MiddlewareError,
+  MiddlewareError$1 as MiddlewareError,
   MiddlewareManager,
-  NetworkError,
+  NetworkError$1 as NetworkError,
   PageDataSchema,
   PersistenceError$1 as PersistenceError,
   PriorityRequest,