@jambudipa/spider 0.3.3 → 0.4.0

package/dist/index.d.ts

@@ -1,5 +1,6 @@
 export type { ISpider, ISpiderScheduler, IMiddlewareManager, IRateLimitMiddleware, ILoggingMiddleware, IUserAgentMiddleware, IStatsMiddleware, } from './lib/api-facades.js';
 export * from './lib/Spider/Spider.service.js';
+export { SPIDER_DEFAULTS } from './lib/Spider/Spider.defaults.js';
 export * from './lib/Robots/Robots.service.js';
 export * from './lib/Scraper/Scraper.service.js';
 export * from './lib/PageData/PageData.js';
@@ -20,16 +21,24 @@ export { StateDelta, PersistenceError as ResumabilityError, DEFAULT_HYBRID_CONFI
 export { ResumabilityService, ResumabilityConfigs, createStateOperation, } from './lib/Resumability/Resumability.service.js';
 export { FullStatePersistence, DeltaPersistence, HybridPersistence, } from './lib/Resumability/strategies.js';
 export { FileStorageBackend } from './lib/Resumability/backends/FileStorageBackend.js';
+export type { DatabaseClientInterface, PostgresStorageConfig, } from './lib/Resumability/backends/PostgresStorageBackend.js';
+export { PostgresStorageBackend } from './lib/Resumability/backends/PostgresStorageBackend.js';
 export { NetworkError, ResponseError, RobotsTxtError, ConfigurationError, MiddlewareError, FileSystemError, PersistenceError, ContentTypeError, RequestAbortError, AdapterNotInitialisedError, BrowserError, BrowserCleanupError, TimeoutError, ParseError, ValidationError, PageError, StateError, SessionError, CrawlError, QueueError, ConfigError, isSpiderError, isNetworkError, isBrowserError, } from './lib/errors/effect-errors.js';
 export type { SpiderError, AllSpiderErrors } from './lib/errors/effect-errors.js';
 export type { SpiderLogEvent, SpiderLogger, } from './lib/Logging/SpiderLogger.service.js';
 export { SpiderLogger as SpiderLoggerTag, makeSpiderLogger, SpiderLoggerLive, } from './lib/Logging/SpiderLogger.service.js';
+export type { LoggingFetchFn } from './lib/Logging/FetchLogger.js';
+export { FetchError, makeLoggingFetch, LoggingFetch, } from './lib/Logging/FetchLogger.js';
 export type { CookieManagerService, EnhancedHttpClientService, HttpRequestOptions, HttpResponse, Session, Credentials, SessionStoreService, TokenInfo, TokenExtractorService, } from './lib/HttpClient/index.js';
 export { CookieManager, makeCookieManager, CookieManagerLive, EnhancedHttpClient, makeEnhancedHttpClient, EnhancedHttpClientLive, SessionStore, makeSessionStore, SessionStoreLive, TokenExtractor, makeTokenExtractor, TokenExtractorLive, } from './lib/HttpClient/index.js';
 export type { Token, StateManagerService } from './lib/StateManager/index.js';
-export { TokenType, StateManager, makeStateManager, StateManagerLive, } from './lib/StateManager/index.js';
+export { TokenType, StateManager, makeStateManager, StateManagerLive, CSRFTokenNotFoundError, APITokenNotFoundError, TokenNotFoundError, TokenExpiredError, StorageKeyNotFoundError, } from './lib/StateManager/index.js';
 export type { BrowserEngineConfig, BrowserEngineServiceInterface, PageElement, } from './lib/BrowserEngine/BrowserEngine.service.js';
 export { BrowserEngineService, BrowserEngineLive, BrowserEngineWithConfig, withBrowser, } from './lib/BrowserEngine/BrowserEngine.service.js';
-export type { LoginCredentials, ScrapingSession, WebScrapingEngineService, } from './lib/WebScrapingEngine/index.js';
-export { WebScrapingEngine, makeWebScrapingEngine, WebScrapingEngineLive, } from './lib/WebScrapingEngine/index.js';
+export type { LoginCredentials, ScrapingSession, WebScrapingEngineService, WebScrapingEngineError, HttpOperationError, HttpPostOperationError, } from './lib/WebScrapingEngine/index.js';
+export { WebScrapingEngine, makeWebScrapingEngine, WebScrapingEngineLive, LoginError, SessionNotValidError, SessionLoadError, } from './lib/WebScrapingEngine/index.js';
+export { WorkerHealthMonitor } from './lib/WorkerHealth/WorkerHealthMonitor.service.js';
+export type { DeduplicationStrategy, UrlWithMetadata, NormalizedUrl, } from './lib/utils/url-deduplication.js';
+export { DEFAULT_DEDUPLICATION_STRATEGY, parseUrl, normalizeUrl, deduplicateUrls, createUrlDeduplicator, } from './lib/utils/url-deduplication.js';
+export { safeJsonParse, toOption, fromPromise, cleanupResources, matchOption, } from './lib/utils/effect-migration.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,YAAY,EACV,OAAO,EACP,gBAAgB,EAChB,kBAAkB,EAClB,oBAAoB,EACpB,kBAAkB,EAClB,oBAAoB,EACpB,gBAAgB,GACjB,MAAM,sBAAsB,CAAC;AAG9B,cAAc,gCAAgC,CAAC;AAC/C,cAAc,gCAAgC,CAAC;AAC/C,cAAc,kCAAkC,CAAC;AACjD,cAAc,4BAA4B,CAAC;AAG3C,YAAY,EACV,mBAAmB,EACnB,mBAAmB,GACpB,MAAM,sCAAsC,CAAC;AAC9C,OAAO,EACL,YAAY,EACZ,gBAAgB,GACjB,MAAM,sCAAsC,CAAC;AAG9C,YAAY,EAAE,gBAAgB,EAAE,MAAM,kDAAkD,CAAC;AACzF,OAAO,EAAE,sBAAsB,EAAE,MAAM,kDAAkD,CAAC;AAG1F,YAAY,EAAE,gBAAgB,EAAE,MAAM,4CAA4C,CAAC;AACnF,OAAO,EACL,sBAAsB,EACtB,cAAc,EACd,eAAe,EACf,WAAW,GACZ,MAAM,4CAA4C,CAAC;AAGpD,YAAY,EACV,gBAAgB,EAChB,aAAa,EACb,cAAc,GACf,MAAM,sCAAsC,CAAC;AAC9C,OAAO,EACL,iBAAiB,EACjB,mBAAmB,EACnB,iBAAiB,EACjB,mBAAmB,EACnB,eAAe,GAChB,MAAM,sCAAsC,CAAC;AAG9C,YAAY,EACV,mBAAmB,EACnB,oBAAoB,EACpB,6BAA6B,GAC9B,MAAM,8CAA8C,CAAC;AACtD,OAAO,EACL,oBAAoB,EACpB,yBAAyB,EACzB,mBAAmB,GACpB,MAAM,8CAA8C,CAAC;AAGtD,YAAY,EACV,WAAW,EACX,SAAS,EACT,2BAA2B,GAC5B,MAAM,gCAAgC,CAAC;AAGxC,YAAY,EACV,mBAAmB,EACnB,cAAc,EACd,cAAc,EACd,mBAAmB,EACnB,uBAAuB,GACxB,MAAM,6BAA6B,CAAC;AACrC,YAAY,EAAE,kBAAkB,EAAE,MAAM,4CAA4C,CAAC;AACrF,OAAO,EACL,UAAU,EACV,gBAAgB,IAAI,iBAAiB,EACrC,qBAAqB,GACtB,MAAM,6BAA6B,CAAC;AACrC,OAAO,EACL,mBAAmB,EACnB,mBAAmB,EACnB,oBAAoB,GACrB,MAAM,4CAA4C,CAAC;AACpD,OAAO,EACL,oBAAoB,EACpB,gBAAgB,EAChB,iBAAiB,GAClB,MAAM,kCAAkC,CAAC;AAC1C,OAAO,EAAE,kBAAkB,EAAE,MAAM,mDAAmD,CAAC;AAGvF,OAAO,EACL,YAAY,EACZ,aAAa,EACb,cAAc,EACd,kBAAkB,EAClB,eAAe,EACf,eAAe,EACf,gBAAgB,EAChB,gBAAgB,EAChB,iBAAiB,EACjB,0BAA0B,EAC1B,YAAY,EACZ,mBAAmB,EACnB,YAAY,EACZ,UAAU,EACV,eAAe,EACf,SAAS,EACT,UAAU,EACV,YAAY,EACZ,UAAU,EACV,UAAU,EACV,WAAW,EACX,aAAa,EACb,cAAc,EACd,cAAc,GACf,MAAM,+BAA+B,CAAC;AACvC,YAAY,EAAE,WAAW,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAC;AAGlF,YAAY,EACV,cAAc,EACd,YAAY,GACb,MAAM,uCAAuC,CAAC;AAC/C,OAAO,EACL,YAAY,IAAI,eAAe,EAC/B,gBAAgB,EAChB,gBAAgB,GACjB,MAAM,uCAAuC,CAAC;AAG/C,YAAY,EACV,oBAAoB,EACpB,yBAAyB,EACzB,kBAAkB,EAClB,YAAY,EACZ,OAAO,EACP,WAAW,EACX,mBAAmB,EACnB,SAAS,EACT,qBAAqB,GACtB,MAAM,2BAA2B,CAAC;AACnC,OAAO,EACL,aAAa,EACb,iBAAiB,EACjB,iBAAiB,EACjB,kBAAkB,EAClB,sBAAsB,EACtB,sBAAsB,EACtB,YAAY,EACZ,gBAAgB,EAChB,gBAAgB,EAChB,cAAc,EACd,kBAAkB,EAClB,kBAAkB,GACnB,MAAM,2BAA2B,CAAC;AAGnC,YAAY,EAAE,KAAK,EAAE,mBAAmB,EAAE,MAAM,6BAA6B,CAAC;AAC9E,OAAO,EACL,SAAS,EACT,YAAY,EACZ,gBAAgB,EAChB,gBAAgB,GACjB,MAAM,6BAA6B,CAAC;AAGrC,YAAY,EACV,mBAAmB,EACnB,6BAA6B,EAC7B,WAAW,GACZ,MAAM,8CAA8C,CAAC;AACtD,OAAO,EACL,oBAAoB,EACpB,iBAAiB,EACjB,uBAAuB,EACvB,WAAW,GACZ,MAAM,8CAA8C,CAAC;AAGtD,YAAY,EACV,gBAAgB,EAChB,eAAe,EACf,wBAAwB,GACzB,MAAM,kCAAkC,CAAC;AAC1C,OAAO,EACL,iBAAiB,EACjB,qBAAqB,EACrB,qBAAqB,GACtB,MAAM,kCAAkC,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,YAAY,EACV,OAAO,EACP,gBAAgB,EAChB,kBAAkB,EAClB,oBAAoB,EACpB,kBAAkB,EAClB,oBAAoB,EACpB,gBAAgB,GACjB,MAAM,sBAAsB,CAAC;AAG9B,cAAc,gCAAgC,CAAC;AAC/C,OAAO,EAAE,eAAe,EAAE,MAAM,iCAAiC,CAAC;AAClE,cAAc,gCAAgC,CAAC;AAC/C,cAAc,kCAAkC,CAAC;AACjD,cAAc,4BAA4B,CAAC;AAG3C,YAAY,EACV,mBAAmB,EACnB,mBAAmB,GACpB,MAAM,sCAAsC,CAAC;AAC9C,OAAO,EACL,YAAY,EACZ,gBAAgB,GACjB,MAAM,sCAAsC,CAAC;AAG9C,YAAY,EAAE,gBAAgB,EAAE,MAAM,kDAAkD,CAAC;AACzF,OAAO,EAAE,sBAAsB,EAAE,MAAM,kDAAkD,CAAC;AAG1F,YAAY,EAAE,gBAAgB,EAAE,MAAM,4CAA4C,CAAC;AACnF,OAAO,EACL,sBAAsB,EACtB,cAAc,EACd,eAAe,EACf,WAAW,GACZ,MAAM,4CAA4C,CAAC;AAGpD,YAAY,EACV,gBAAgB,EAChB,aAAa,EACb,cAAc,GACf,MAAM,sCAAsC,CAAC;AAC9C,OAAO,EACL,iBAAiB,EACjB,mBAAmB,EACnB,iBAAiB,EACjB,mBAAmB,EACnB,eAAe,GAChB,MAAM,sCAAsC,CAAC;AAG9C,YAAY,EACV,mBAAmB,EACnB,oBAAoB,EACpB,6BAA6B,GAC9B,MAAM,8CAA8C,CAAC;AACtD,OAAO,EACL,oBAAoB,EACpB,yBAAyB,EACzB,mBAAmB,GACpB,MAAM,8CAA8C,CAAC;AAGtD,YAAY,EACV,WAAW,EACX,SAAS,EACT,2BAA2B,GAC5B,MAAM,gCAAgC,CAAC;AAGxC,YAAY,EACV,mBAAmB,EACnB,cAAc,EACd,cAAc,EACd,mBAAmB,EACnB,uBAAuB,GACxB,MAAM,6BAA6B,CAAC;AACrC,YAAY,EAAE,kBAAkB,EAAE,MAAM,4CAA4C,CAAC;AACrF,OAAO,EACL,UAAU,EACV,gBAAgB,IAAI,iBAAiB,EACrC,qBAAqB,GACtB,MAAM,6BAA6B,CAAC;AACrC,OAAO,EACL,mBAAmB,EACnB,mBAAmB,EACnB,oBAAoB,GACrB,MAAM,4CAA4C,CAAC;AACpD,OAAO,EACL,oBAAoB,EACpB,gBAAgB,EAChB,iBAAiB,GAClB,MAAM,kCAAkC,CAAC;AAC1C,OAAO,EAAE,kBAAkB,EAAE,MAAM,mDAAmD,CAAC;AACvF,YAAY,EACV,uBAAuB,EACvB,qBAAqB,GACtB,MAAM,uDAAuD,CAAC;AAC/D,OAAO,EAAE,sBAAsB,EAAE,MAAM,uDAAuD,CAAC;AAG/F,OAAO,EACL,YAAY,EACZ,aAAa,EACb,cAAc,EACd,kBAAkB,EAClB,eAAe,EACf,eAAe,EACf,gBAAgB,EAChB,gBAAgB,EAChB,iBAAiB,EACjB,0BAA0B,EAC1B,YAAY,EACZ,mBAAmB,EACnB,YAAY,EACZ,UAAU,EACV,eAAe,EACf,SAAS,EACT,UAAU,EACV,YAAY,EACZ,UAAU,EACV,UAAU,EACV,WAAW,EACX,aAAa,EACb,cAAc,EACd,cAAc,GACf,MAAM,+BAA+B,CAAC;AACvC,YAAY,EAAE,WAAW,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAC;AAGlF,YAAY,EACV,cAAc,EACd,YAAY,GACb,MAAM,uCAAuC,CAAC;AAC/C,OAAO,EACL,YAAY,IAAI,eAAe,EAC/B,gBAAgB,EAChB,gBAAgB,GACjB,MAAM,uCAAuC,CAAC;AAG/C,YAAY,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AACnE,OAAO,EACL,UAAU,EACV,gBAAgB,EAChB,YAAY,GACb,MAAM,8BAA8B,CAAC;AAGtC,YAAY,EACV,oBAAoB,EACpB,yBAAyB,EACzB,kBAAkB,EAClB,YAAY,EACZ,OAAO,EACP,WAAW,EACX,mBAAmB,EACnB,SAAS,EACT,qBAAqB,GACtB,MAAM,2BAA2B,CAAC;AACnC,OAAO,EACL,aAAa,EACb,iBAAiB,EACjB,iBAAiB,EACjB,kBAAkB,EAClB,sBAAsB,EACtB,sBAAsB,EACtB,YAAY,EACZ,gBAAgB,EAChB,gBAAgB,EAChB,cAAc,EACd,kBAAkB,EAClB,kBAAkB,GACnB,MAAM,2BAA2B,CAAC;AAGnC,YAAY,EAAE,KAAK,EAAE,mBAAmB,EAAE,MAAM,6BAA6B,CAAC;AAC9E,OAAO,EACL,SAAS,EACT,YAAY,EACZ,gBAAgB,EAChB,gBAAgB,EAChB,sBAAsB,EACtB,qBAAqB,EACrB,kBAAkB,EAClB,iBAAiB,EACjB,uBAAuB,GACxB,MAAM,6BAA6B,CAAC;AAGrC,YAAY,EACV,mBAAmB,EACnB,6BAA6B,EAC7B,WAAW,GACZ,MAAM,8CAA8C,CAAC;AACtD,OAAO,EACL,oBAAoB,EACpB,iBAAiB,EACjB,uBAAuB,EACvB,WAAW,GACZ,MAAM,8CAA8C,CAAC;AAGtD,YAAY,EACV,gBAAgB,EAChB,eAAe,EACf,wBAAwB,EACxB,sBAAsB,EACtB,kBAAkB,EAClB,sBAAsB,GACvB,MAAM,kCAAkC,CAAC;AAC1C,OAAO,EACL,iBAAiB,EACjB,qBAAqB,EACrB,qBAAqB,EACrB,UAAU,EACV,oBAAoB,EACpB,gBAAgB,GACjB,MAAM,kCAAkC,CAAC;AAG1C,OAAO,EAAE,mBAAmB,EAAE,MAAM,mDAAmD,CAAC;AAGxF,YAAY,EACV,qBAAqB,EACrB,eAAe,EACf,aAAa,GACd,MAAM,kCAAkC,CAAC;AAC1C,OAAO,EACL,8BAA8B,EAC9B,QAAQ,EACR,YAAY,EACZ,eAAe,EACf,qBAAqB,GACtB,MAAM,kCAAkC,CAAC;AAG1C,OAAO,EACL,aAAa,EACb,QAAQ,EACR,WAAW,EACX,gBAAgB,EAChB,WAAW,GACZ,MAAM,iCAAiC,CAAC"}

package/dist/index.js

@@ -1,4 +1,4 @@
-import { Effect, Layer, Option, Chunk, MutableHashSet, Schema, Data, pipe, Context, DateTime, Console, Duration, MutableHashMap, Queue, HashMap, PubSub, MutableRef, Schedule, Stream, Fiber, Random, Struct, Ref } from "effect";
+import { Effect, Layer, Option, Chunk, MutableHashSet, Schema, Data, pipe, Context, DateTime, Console, Duration, MutableHashMap, Queue, HashMap, Ref, HashSet, PubSub, MutableRef, Schedule, Stream, Fiber, Random, Struct } from "effect";
 import * as cheerio from "cheerio";
 import * as fs from "fs";
 import * as path from "path";
@@ -1747,6 +1747,62 @@ const deduplicateUrls = (urls, strategy = DEFAULT_DEDUPLICATION_STRATEGY) => Eff
     }
   };
 });
+const createUrlDeduplicator = (strategy = DEFAULT_DEDUPLICATION_STRATEGY) => Effect.gen(function* () {
+  const seenUrls = yield* Ref.make(HashSet.empty());
+  const urlStats = yield* Ref.make({
+    processed: 0,
+    unique: 0,
+    duplicates: 0
+  });
+  return {
+    /**
+     * Check if a URL has been seen (after normalization)
+     */
+    hasSeenUrl: (url) => Effect.gen(function* () {
+      const normalized = yield* normalizeUrl(url, strategy);
+      const seen = yield* Ref.get(seenUrls);
+      return HashSet.has(seen, normalized.normalized);
+    }),
+    /**
+     * Add a URL to the seen set
+     */
+    markUrlSeen: (url) => Effect.gen(function* () {
+      const normalized = yield* normalizeUrl(url, strategy);
+      const seen = yield* Ref.get(seenUrls);
+      if (HashSet.has(seen, normalized.normalized)) {
+        yield* Ref.update(urlStats, (stats) => ({
+          ...stats,
+          processed: stats.processed + 1,
+          duplicates: stats.duplicates + 1
+        }));
+        return false;
+      } else {
+        yield* Ref.set(seenUrls, HashSet.add(seen, normalized.normalized));
+        yield* Ref.update(urlStats, (stats) => ({
+          ...stats,
+          processed: stats.processed + 1,
+          unique: stats.unique + 1
+        }));
+        return true;
+      }
+    }),
+    /**
+     * Get deduplication statistics
+     */
+    getStats: () => Ref.get(urlStats),
+    /**
+     * Reset the deduplicator
+     */
+    reset: () => Effect.gen(function* () {
+      yield* Ref.set(seenUrls, HashSet.empty());
+      yield* Ref.set(urlStats, {
+        processed: 0,
+        unique: 0,
+        duplicates: 0
+      });
+    })
+  };
+});
 const SPIDER_DEFAULTS = Object.freeze({
   /** Threshold in ms after which a worker is considered stale (60s) */
   STALE_WORKER_THRESHOLD_MS: 6e4,
@@ -5080,6 +5136,120 @@ const ResumabilityConfigs = {
     backend: new PostgresStorageBackend(dbClient, config)
   })
 };
+class FetchError extends Data.TaggedError("FetchError") {
+  get message() {
+    return `Fetch failed for ${this.url}: ${this.reason} after ${this.durationMs}ms`;
+  }
+}
+const makeLoggingFetch = Effect.gen(function* () {
+  const logger = yield* SpiderLogger;
+  return (url, options) => Effect.gen(function* () {
+    const startTime = yield* DateTime.now;
+    const startMs = DateTime.toEpochMillis(startTime);
+    const domain = new URL(url).hostname;
+    const optionDetails = Option.fromNullable(options).pipe(
+      Option.map((opts) => ({
+        method: opts.method,
+        headers: Object.keys(opts.headers ?? {})
+      }))
+    );
+    yield* logger.logEvent({
+      type: "edge_case",
+      domain,
+      url,
+      message: "[FETCH_START] Starting fetch request",
+      details: {
+        case: "fetch_start",
+        url,
+        timestamp: DateTime.formatIso(startTime),
+        options: Option.getOrUndefined(optionDetails)
+      }
+    });
+    const fetchEffect = Effect.tryPromise({
+      try: () => globalThis.fetch(url, options),
+      catch: (error) => new FetchError({
+        url,
+        reason: "network",
+        durationMs: 0,
+        // Duration will be calculated in error handler
+        cause: error
+      })
+    });
+    const timeoutDuration = Duration.seconds(30);
+    const fetchWithTimeout = fetchEffect.pipe(
+      Effect.timeoutOption(timeoutDuration),
+      Effect.flatMap(
+        (maybeResponse) => Option.match(maybeResponse, {
+          onNone: () => Effect.gen(function* () {
+            const currentTime = yield* DateTime.now;
+            const durationMs2 = DateTime.toEpochMillis(currentTime) - startMs;
+            yield* logger.logEvent({
+              type: "edge_case",
+              domain,
+              url,
+              message: `[FETCH_ABORT] Aborting fetch after ${durationMs2}ms`,
+              details: {
+                case: "fetch_abort",
+                url,
+                durationMs: durationMs2,
+                reason: "timeout"
+              }
+            });
+            return yield* Effect.fail(
+              new FetchError({
+                url,
+                reason: "timeout",
+                durationMs: Number(durationMs2)
+              })
+            );
+          }),
+          onSome: (response2) => Effect.succeed(response2)
+        })
+      )
+    );
+    const response = yield* fetchWithTimeout.pipe(
+      Effect.catchAll(
+        (error) => Effect.gen(function* () {
+          const currentTime = yield* DateTime.now;
+          const durationMs2 = DateTime.toEpochMillis(currentTime) - startMs;
+          yield* logger.logEvent({
+            type: "edge_case",
+            domain,
+            url,
+            message: `[FETCH_ERROR] Failed after ${durationMs2}ms`,
+            details: {
+              case: "fetch_failed",
+              url,
+              durationMs: durationMs2,
+              error: error._tag,
+              message: error.message,
+              isAborted: error.reason === "timeout"
+            }
+          });
+          return yield* Effect.fail(error);
+        })
+      )
+    );
+    const endTime = yield* DateTime.now;
+    const durationMs = DateTime.toEpochMillis(endTime) - startMs;
+    yield* logger.logEvent({
+      type: "edge_case",
+      domain,
+      url,
+      message: `[FETCH_SUCCESS] Got response in ${durationMs}ms`,
+      details: {
+        case: "fetch_success",
+        url,
+        durationMs,
+        status: response.status,
+        statusText: response.statusText,
+        contentType: response.headers.get("content-type")
+      }
+    });
+    return response;
+  });
+});
+const LoggingFetch = Context.GenericTag("LoggingFetch");
 class JsonParseError extends Data.TaggedError("JsonParseError") {
   get message() {
     const preview = this.input.length > 100 ? `${this.input.substring(0, 100)}...` : this.input;
@@ -6980,23 +7150,153 @@ const WebScrapingEngineLive = Layer.effect(
   WebScrapingEngine,
   makeWebScrapingEngine
 );
+class WorkerHealthMonitor extends Effect.Service()(
+  "@jambudipa.io/WorkerHealthMonitor",
+  {
+    effect: Effect.gen(function* () {
+      const logger = yield* SpiderLogger;
+      const workers = yield* Ref.make(HashMap.empty());
+      const stuckThresholdMs = 6e4;
+      return {
+        /**
+         * Register a worker's activity
+         */
+        recordActivity: (workerId, domain, activity) => Effect.gen(function* () {
+          const now = DateTime.unsafeNow();
+          yield* Ref.update(workers, (map) => {
+            const current = HashMap.get(map, workerId).pipe(
+              (opt) => opt._tag === "Some" ? opt.value : {
+                workerId,
+                domain,
+                lastActivity: now
+              }
+            );
+            const updated = {
+              ...current,
+              domain,
+              lastActivity: now,
+              currentUrl: activity.url ?? current.currentUrl,
+              fetchStartTime: activity.fetchStart ? now : current.fetchStartTime
+            };
+            return HashMap.set(map, workerId, updated);
+          });
+        }),
+        /**
+         * Remove a worker from monitoring
+         */
+        removeWorker: (workerId) => Ref.update(workers, (map) => HashMap.remove(map, workerId)),
+        /**
+         * Get stuck workers
+         */
+        getStuckWorkers: Effect.gen(function* () {
+          const now = DateTime.unsafeNow();
+          const workerMap = yield* Ref.get(workers);
+          const stuck = [];
+          for (const [, status] of workerMap) {
+            const inactiveMs = DateTime.toEpochMillis(now) - DateTime.toEpochMillis(status.lastActivity);
+            if (inactiveMs > stuckThresholdMs) {
+              stuck.push(status);
+            }
+          }
+          return stuck;
+        }),
+        /**
+         * Monitor workers and log stuck ones
+         */
+        startMonitoring: Effect.gen(function* () {
+          const self = {
+            getStuckWorkers: Effect.gen(function* () {
+              const now = DateTime.unsafeNow();
+              const workerMap = yield* Ref.get(workers);
+              const stuck = [];
+              for (const [, status] of workerMap) {
+                const inactiveMs = DateTime.toEpochMillis(now) - DateTime.toEpochMillis(status.lastActivity);
+                if (inactiveMs > stuckThresholdMs) {
+                  stuck.push(status);
+                }
+              }
+              return stuck;
+            })
+          };
+          yield* Effect.repeat(
+            Effect.gen(function* () {
+              const stuck = yield* self.getStuckWorkers;
+              if (stuck.length > 0) {
+                for (const worker of stuck) {
+                  const nowMillis = DateTime.toEpochMillis(DateTime.unsafeNow());
+                  const inactiveMs = nowMillis - DateTime.toEpochMillis(worker.lastActivity);
+                  yield* logger.logEdgeCase(
+                    worker.domain,
+                    "worker_stuck_detected",
+                    {
+                      workerId: worker.workerId,
+                      currentUrl: worker.currentUrl,
+                      lastActivity: DateTime.formatIso(worker.lastActivity),
+                      inactiveMs,
+                      fetchStartTime: Option.fromNullable(worker.fetchStartTime).pipe(
+                        Option.map(DateTime.formatIso),
+                        Option.getOrElse(() => "N/A")
+                      )
+                    }
+                  );
+                }
+              }
+            }),
+            Schedule.fixed(Duration.seconds(30))
+          );
+        })
+      };
+    })
+  }
+) {
+}
+const safeJsonParse = (data, onError) => Schema.decodeUnknown(Schema.parseJson(Schema.Unknown))(data).pipe(
+  Effect.mapError(onError)
+);
+const toOption = (value, logContext) => {
+  const result = Option.fromNullable(value);
+  if (logContext && Option.isNone(result)) {
+    return Effect.logDebug(`[Migration] Null value encountered: ${logContext}`).pipe(
+      Effect.map(() => result)
+    );
+  }
+  return Effect.succeed(result);
+};
+const fromPromise = (promise, onError) => Effect.tryPromise({
+  try: promise,
+  catch: onError
+});
+const cleanupResources = (resources) => Effect.all(
+  resources.map(
+    ({ id, cleanup, onError }) => Effect.tryPromise({
+      try: cleanup,
+      catch: (error) => onError(id, error)
+    })
+  ),
+  { mode: "either" }
+);
+const matchOption = (option, onNone, onSome) => Option.match(option, { onNone, onSome });
 export {
+  APITokenNotFoundError,
   AdapterNotInitialisedError,
   BrowserCleanupError,
   BrowserEngineLive,
   BrowserEngineService,
   BrowserEngineWithConfig,
   BrowserError,
+  CSRFTokenNotFoundError,
   ConfigError,
   ConfigurationError,
   ContentTypeError,
   CookieManager,
   CookieManagerLive,
   CrawlError,
+  DEFAULT_DEDUPLICATION_STRATEGY,
   DEFAULT_HYBRID_CONFIG,
   DeltaPersistence,
   EnhancedHttpClient,
   EnhancedHttpClientLive,
+  FetchError,
   FileStorageBackend,
   FileSystemError,
   FullStatePersistence,
@@ -7004,7 +7304,9 @@ export {
   LinkExtractionError,
   LinkExtractorService,
   LinkExtractorServiceLayer,
+  LoggingFetch,
   LoggingMiddleware,
+  LoginError,
   MiddlewareError,
   MiddlewareManager,
   NetworkError,
@@ -7012,6 +7314,7 @@ export {
   PageError,
   ParseError,
   PersistenceError$1 as PersistenceError,
+  PostgresStorageBackend,
   PriorityRequest,
   QueueError,
   RateLimitMiddleware,
@@ -7022,8 +7325,11 @@ export {
   ResumabilityService,
   RobotsService,
   RobotsTxtError,
+  SPIDER_DEFAULTS,
   ScraperService,
   SessionError$1 as SessionError,
+  SessionLoadError,
+  SessionNotValidError,
   SessionStore,
   SessionStoreLive,
   SpiderConfig,
@@ -7038,27 +7344,41 @@ export {
   StateManager,
   StateManagerLive,
   StatsMiddleware,
+  StorageKeyNotFoundError,
   TimeoutError,
+  TokenExpiredError,
   TokenExtractor,
   TokenExtractorLive,
+  TokenNotFoundError,
   TokenType,
   UrlDeduplicatorService,
   UserAgentMiddleware,
   ValidationError,
   WebScrapingEngine,
   WebScrapingEngineLive,
+  WorkerHealthMonitor,
+  cleanupResources,
   createStateOperation,
+  createUrlDeduplicator,
+  deduplicateUrls,
+  fromPromise,
   isBrowserError,
   isNetworkError,
   isSpiderError,
   makeCookieManager,
   makeEnhancedHttpClient,
+  makeLoggingFetch,
   makeSessionStore,
   makeSpiderConfig,
   makeSpiderLogger,
   makeStateManager,
   makeTokenExtractor,
   makeWebScrapingEngine,
+  matchOption,
+  normalizeUrl,
+  parseUrl,
+  safeJsonParse,
+  toOption,
   withBrowser
 };
 //# sourceMappingURL=index.js.map