@jambudipa/spider 0.3.1 → 0.4.0

package/dist/index.js

@@ -1,4 +1,4 @@
-import { Effect, Layer, Option, Chunk, MutableHashSet, Schema, Data, pipe, Context, DateTime, Console, Duration, MutableHashMap, Queue, HashMap, PubSub, MutableRef, Schedule, Stream, Fiber, Random, Struct, Ref } from "effect";
+import { Effect, Layer, Option, Chunk, MutableHashSet, Schema, Data, pipe, Context, DateTime, Console, Duration, MutableHashMap, Queue, HashMap, Ref, HashSet, PubSub, MutableRef, Schedule, Stream, Fiber, Random, Struct } from "effect";
 import * as cheerio from "cheerio";
 import * as fs from "fs";
 import * as path from "path";
@@ -1747,6 +1747,62 @@ const deduplicateUrls = (urls, strategy = DEFAULT_DEDUPLICATION_STRATEGY) => Eff
     }
   };
 });
+const createUrlDeduplicator = (strategy = DEFAULT_DEDUPLICATION_STRATEGY) => Effect.gen(function* () {
+  const seenUrls = yield* Ref.make(HashSet.empty());
+  const urlStats = yield* Ref.make({
+    processed: 0,
+    unique: 0,
+    duplicates: 0
+  });
+  return {
+    /**
+     * Check if a URL has been seen (after normalization)
+     */
+    hasSeenUrl: (url) => Effect.gen(function* () {
+      const normalized = yield* normalizeUrl(url, strategy);
+      const seen = yield* Ref.get(seenUrls);
+      return HashSet.has(seen, normalized.normalized);
+    }),
+    /**
+     * Add a URL to the seen set
+     */
+    markUrlSeen: (url) => Effect.gen(function* () {
+      const normalized = yield* normalizeUrl(url, strategy);
+      const seen = yield* Ref.get(seenUrls);
+      if (HashSet.has(seen, normalized.normalized)) {
+        yield* Ref.update(urlStats, (stats) => ({
+          ...stats,
+          processed: stats.processed + 1,
+          duplicates: stats.duplicates + 1
+        }));
+        return false;
+      } else {
+        yield* Ref.set(seenUrls, HashSet.add(seen, normalized.normalized));
+        yield* Ref.update(urlStats, (stats) => ({
+          ...stats,
+          processed: stats.processed + 1,
+          unique: stats.unique + 1
+        }));
+        return true;
+      }
+    }),
+    /**
+     * Get deduplication statistics
+     */
+    getStats: () => Ref.get(urlStats),
+    /**
+     * Reset the deduplicator
+     */
+    reset: () => Effect.gen(function* () {
+      yield* Ref.set(seenUrls, HashSet.empty());
+      yield* Ref.set(urlStats, {
+        processed: 0,
+        unique: 0,
+        duplicates: 0
+      });
+    })
+  };
+});
 const SPIDER_DEFAULTS = Object.freeze({
   /** Threshold in ms after which a worker is considered stale (60s) */
   STALE_WORKER_THRESHOLD_MS: 6e4,
@@ -5080,6 +5136,120 @@ const ResumabilityConfigs = {
     backend: new PostgresStorageBackend(dbClient, config)
   })
 };
+class FetchError extends Data.TaggedError("FetchError") {
+  get message() {
+    return `Fetch failed for ${this.url}: ${this.reason} after ${this.durationMs}ms`;
+  }
+}
+const makeLoggingFetch = Effect.gen(function* () {
+  const logger = yield* SpiderLogger;
+  return (url, options) => Effect.gen(function* () {
+    const startTime = yield* DateTime.now;
+    const startMs = DateTime.toEpochMillis(startTime);
+    const domain = new URL(url).hostname;
+    const optionDetails = Option.fromNullable(options).pipe(
+      Option.map((opts) => ({
+        method: opts.method,
+        headers: Object.keys(opts.headers ?? {})
+      }))
+    );
+    yield* logger.logEvent({
+      type: "edge_case",
+      domain,
+      url,
+      message: "[FETCH_START] Starting fetch request",
+      details: {
+        case: "fetch_start",
+        url,
+        timestamp: DateTime.formatIso(startTime),
+        options: Option.getOrUndefined(optionDetails)
+      }
+    });
+    const fetchEffect = Effect.tryPromise({
+      try: () => globalThis.fetch(url, options),
+      catch: (error) => new FetchError({
+        url,
+        reason: "network",
+        durationMs: 0,
+        // Duration will be calculated in error handler
+        cause: error
+      })
+    });
+    const timeoutDuration = Duration.seconds(30);
+    const fetchWithTimeout = fetchEffect.pipe(
+      Effect.timeoutOption(timeoutDuration),
+      Effect.flatMap(
+        (maybeResponse) => Option.match(maybeResponse, {
+          onNone: () => Effect.gen(function* () {
+            const currentTime = yield* DateTime.now;
+            const durationMs2 = DateTime.toEpochMillis(currentTime) - startMs;
+            yield* logger.logEvent({
+              type: "edge_case",
+              domain,
+              url,
+              message: `[FETCH_ABORT] Aborting fetch after ${durationMs2}ms`,
+              details: {
+                case: "fetch_abort",
+                url,
+                durationMs: durationMs2,
+                reason: "timeout"
+              }
+            });
+            return yield* Effect.fail(
+              new FetchError({
+                url,
+                reason: "timeout",
+                durationMs: Number(durationMs2)
+              })
+            );
+          }),
+          onSome: (response2) => Effect.succeed(response2)
+        })
+      )
+    );
+    const response = yield* fetchWithTimeout.pipe(
+      Effect.catchAll(
+        (error) => Effect.gen(function* () {
+          const currentTime = yield* DateTime.now;
+          const durationMs2 = DateTime.toEpochMillis(currentTime) - startMs;
+          yield* logger.logEvent({
+            type: "edge_case",
+            domain,
+            url,
+            message: `[FETCH_ERROR] Failed after ${durationMs2}ms`,
+            details: {
+              case: "fetch_failed",
+              url,
+              durationMs: durationMs2,
+              error: error._tag,
+              message: error.message,
+              isAborted: error.reason === "timeout"
+            }
+          });
+          return yield* Effect.fail(error);
+        })
+      )
+    );
+    const endTime = yield* DateTime.now;
+    const durationMs = DateTime.toEpochMillis(endTime) - startMs;
+    yield* logger.logEvent({
+      type: "edge_case",
+      domain,
+      url,
+      message: `[FETCH_SUCCESS] Got response in ${durationMs}ms`,
+      details: {
+        case: "fetch_success",
+        url,
+        durationMs,
+        status: response.status,
+        statusText: response.statusText,
+        contentType: response.headers.get("content-type")
+      }
+    });
+    return response;
+  });
+});
+const LoggingFetch = Context.GenericTag("LoggingFetch");
 class JsonParseError extends Data.TaggedError("JsonParseError") {
   get message() {
     const preview = this.input.length > 100 ? `${this.input.substring(0, 100)}...` : this.input;
@@ -6527,6 +6697,251 @@ const TokenExtractorLive = Layer.effect(
   TokenExtractor,
   makeTokenExtractor
 );
+class BrowserEngineService extends Effect.Service()(
+  "@jambudipa.io/BrowserEngine",
+  {
+    effect: Effect.gen(function* () {
+      const browserRef = yield* Ref.make(Option.none());
+      const contextRef = yield* Ref.make(Option.none());
+      const pageRef = yield* Ref.make(Option.none());
+      const configRef = yield* Ref.make({
+        headless: true,
+        timeout: 3e4,
+        viewport: { width: 1920, height: 1080 },
+        userAgent: "Mozilla/5.0 (compatible; Spider/1.0)",
+        locale: "en-GB"
+      });
+      const ensureBrowser = () => Effect.gen(function* () {
+        const browserOpt = yield* Ref.get(browserRef);
+        if (Option.isSome(browserOpt)) {
+          return browserOpt.value;
+        }
+        const { chromium } = yield* Effect.tryPromise({
+          try: () => import("playwright"),
+          catch: () => BrowserError.launchFailed("Playwright not installed")
+        });
+        const config = yield* Ref.get(configRef);
+        const browser = yield* Effect.tryPromise({
+          try: () => chromium.launch({
+            headless: config.headless,
+            timeout: config.timeout
+          }),
+          catch: (error) => BrowserError.launchFailed(error)
+        });
+        yield* Ref.set(browserRef, Option.some(browser));
+        return browser;
+      });
+      const ensureContext = () => Effect.gen(function* () {
+        const contextOpt = yield* Ref.get(contextRef);
+        if (Option.isSome(contextOpt)) {
+          return contextOpt.value;
+        }
+        const browser = yield* ensureBrowser();
+        const config = yield* Ref.get(configRef);
+        const context = yield* Effect.tryPromise({
+          try: () => browser.newContext({
+            viewport: config.viewport,
+            userAgent: config.userAgent,
+            locale: config.locale
+          }),
+          catch: (error) => new BrowserError({
+            operation: "newContext",
+            cause: error
+          })
+        });
+        yield* Ref.set(contextRef, Option.some(context));
+        return context;
+      });
+      const getCurrentPage = () => Effect.gen(function* () {
+        const pageOpt = yield* Ref.get(pageRef);
+        return yield* Option.match(pageOpt, {
+          onNone: () => Effect.fail(new PageError({
+            url: "unknown",
+            operation: "getCurrentPage",
+            cause: "No active page"
+          })),
+          onSome: (page) => Effect.succeed(page)
+        });
+      });
+      return {
+        launch: () => Effect.gen(function* () {
+          yield* ensureBrowser();
+          yield* Effect.log("Browser launched successfully");
+        }),
+        createPage: () => Effect.gen(function* () {
+          const context = yield* ensureContext();
+          const page = yield* Effect.tryPromise({
+            try: () => context.newPage(),
+            catch: (error) => new BrowserError({
+              operation: "newPage",
+              cause: error
+            })
+          });
+          yield* Ref.set(pageRef, Option.some(page));
+          yield* Effect.log("New page created");
+          return page;
+        }),
+        navigateTo: (url) => Effect.gen(function* () {
+          const page = yield* getCurrentPage();
+          yield* Effect.tryPromise({
+            try: () => page.goto(url, { waitUntil: "networkidle" }),
+            catch: (error) => new PageError({
+              url,
+              operation: "navigate",
+              cause: error
+            })
+          });
+          yield* Effect.logDebug(`Navigated to ${url}`);
+        }),
+        waitForSelector: (selector, timeout) => Effect.gen(function* () {
+          const page = yield* getCurrentPage();
+          const config = yield* Ref.get(configRef);
+          yield* Effect.tryPromise({
+            try: () => page.waitForSelector(selector, {
+              timeout: timeout ?? config.timeout
+            }),
+            catch: (error) => new PageError({
+              url: page.url(),
+              operation: "waitForSelector",
+              selector,
+              cause: error
+            })
+          });
+        }),
+        click: (selector) => Effect.gen(function* () {
+          const page = yield* getCurrentPage();
+          yield* Effect.tryPromise({
+            try: () => page.click(selector),
+            catch: (error) => new PageError({
+              url: page.url(),
+              operation: "click",
+              selector,
+              cause: error
+            })
+          });
+          yield* Effect.logDebug(`Clicked element: ${selector}`);
+        }),
+        fill: (selector, value) => Effect.gen(function* () {
+          const page = yield* getCurrentPage();
+          yield* Effect.tryPromise({
+            try: () => page.fill(selector, value),
+            catch: (error) => new PageError({
+              url: page.url(),
+              operation: "fill",
+              selector,
+              cause: error
+            })
+          });
+          yield* Effect.logDebug(`Filled ${selector} with value`);
+        }),
+        scroll: (distance) => Effect.gen(function* () {
+          const page = yield* getCurrentPage();
+          yield* Effect.ignore(
+            Effect.tryPromise({
+              try: () => page.evaluate((d) => {
+                window.scrollBy(0, d);
+              }, distance),
+              catch: (error) => error
+            })
+          );
+          yield* Effect.logDebug(`Scrolled ${distance}px`);
+        }),
+        evaluate: (script) => Effect.gen(function* () {
+          const page = yield* getCurrentPage();
+          return yield* Effect.tryPromise({
+            try: () => page.evaluate(script),
+            catch: (error) => new PageError({
+              url: page.url(),
+              operation: "evaluate",
+              cause: error
+            })
+          });
+        }),
+        getHTML: () => Effect.gen(function* () {
+          const page = yield* getCurrentPage();
+          return yield* Effect.tryPromise({
+            try: () => page.content(),
+            catch: (error) => new PageError({
+              url: page.url(),
+              operation: "getHTML",
+              cause: error
+            })
+          });
+        }),
+        screenshot: (path2) => Effect.gen(function* () {
+          const page = yield* getCurrentPage();
+          const buffer = yield* Effect.tryPromise({
+            try: () => page.screenshot({ path: path2, fullPage: true }),
+            catch: (error) => new PageError({
+              url: page.url(),
+              operation: "screenshot",
+              cause: error
+            })
+          });
+          yield* Effect.log(`Screenshot taken${path2 ? ` and saved to ${path2}` : ""}`);
+          return buffer;
+        }),
+        closePage: () => Effect.gen(function* () {
+          const pageOpt = yield* Ref.get(pageRef);
+          if (Option.isSome(pageOpt)) {
+            yield* Effect.ignore(
+              Effect.tryPromise({
+                try: () => pageOpt.value.close(),
+                catch: (error) => error
+              })
+            );
+            yield* Ref.set(pageRef, Option.none());
+            yield* Effect.log("Page closed");
+          }
+        }),
+        close: () => Effect.gen(function* () {
+          const pageOpt = yield* Ref.get(pageRef);
+          if (Option.isSome(pageOpt)) {
+            yield* Effect.ignore(
+              Effect.tryPromise({
+                try: () => pageOpt.value.close(),
+                catch: (error) => error
+              })
+            );
+          }
+          const contextOpt = yield* Ref.get(contextRef);
+          if (Option.isSome(contextOpt)) {
+            yield* Effect.ignore(
+              Effect.tryPromise({
+                try: () => contextOpt.value.close(),
+                catch: (error) => error
+              })
+            );
+          }
+          const browserOpt = yield* Ref.get(browserRef);
+          if (Option.isSome(browserOpt)) {
+            yield* Effect.ignore(
+              Effect.tryPromise({
+                try: () => browserOpt.value.close(),
+                catch: (error) => error
+              })
+            );
+          }
+          yield* Ref.set(pageRef, Option.none());
+          yield* Ref.set(contextRef, Option.none());
+          yield* Ref.set(browserRef, Option.none());
+          yield* Effect.log("Browser engine closed");
+        })
+      };
+    })
+  }
+) {
+}
+const BrowserEngineLive = BrowserEngineService.Default;
+const BrowserEngineWithConfig = (_config) => BrowserEngineService.Default;
+const withBrowser = (operation) => Effect.gen(function* () {
+  const engine = yield* BrowserEngineService;
+  return yield* Effect.acquireUseRelease(
+    Effect.succeed(engine),
+    operation,
+    (engine2) => engine2.close()
+  );
+});
 class LoginError extends Data.TaggedError("LoginError") {
 }
 class SessionNotValidError extends Data.TaggedError("SessionNotValidError") {
@@ -6735,20 +7150,153 @@ const WebScrapingEngineLive = Layer.effect(
   WebScrapingEngine,
   makeWebScrapingEngine
 );
+class WorkerHealthMonitor extends Effect.Service()(
+  "@jambudipa.io/WorkerHealthMonitor",
+  {
+    effect: Effect.gen(function* () {
+      const logger = yield* SpiderLogger;
+      const workers = yield* Ref.make(HashMap.empty());
+      const stuckThresholdMs = 6e4;
+      return {
+        /**
+         * Register a worker's activity
+         */
+        recordActivity: (workerId, domain, activity) => Effect.gen(function* () {
+          const now = DateTime.unsafeNow();
+          yield* Ref.update(workers, (map) => {
+            const current = HashMap.get(map, workerId).pipe(
+              (opt) => opt._tag === "Some" ? opt.value : {
+                workerId,
+                domain,
+                lastActivity: now
+              }
+            );
+            const updated = {
+              ...current,
+              domain,
+              lastActivity: now,
+              currentUrl: activity.url ?? current.currentUrl,
+              fetchStartTime: activity.fetchStart ? now : current.fetchStartTime
+            };
+            return HashMap.set(map, workerId, updated);
+          });
+        }),
+        /**
+         * Remove a worker from monitoring
+         */
+        removeWorker: (workerId) => Ref.update(workers, (map) => HashMap.remove(map, workerId)),
+        /**
+         * Get stuck workers
+         */
+        getStuckWorkers: Effect.gen(function* () {
+          const now = DateTime.unsafeNow();
+          const workerMap = yield* Ref.get(workers);
+          const stuck = [];
+          for (const [, status] of workerMap) {
+            const inactiveMs = DateTime.toEpochMillis(now) - DateTime.toEpochMillis(status.lastActivity);
+            if (inactiveMs > stuckThresholdMs) {
+              stuck.push(status);
+            }
+          }
+          return stuck;
+        }),
+        /**
+         * Monitor workers and log stuck ones
+         */
+        startMonitoring: Effect.gen(function* () {
+          const self = {
+            getStuckWorkers: Effect.gen(function* () {
+              const now = DateTime.unsafeNow();
+              const workerMap = yield* Ref.get(workers);
+              const stuck = [];
+              for (const [, status] of workerMap) {
+                const inactiveMs = DateTime.toEpochMillis(now) - DateTime.toEpochMillis(status.lastActivity);
+                if (inactiveMs > stuckThresholdMs) {
+                  stuck.push(status);
+                }
+              }
+              return stuck;
+            })
+          };
+          yield* Effect.repeat(
+            Effect.gen(function* () {
+              const stuck = yield* self.getStuckWorkers;
+              if (stuck.length > 0) {
+                for (const worker of stuck) {
+                  const nowMillis = DateTime.toEpochMillis(DateTime.unsafeNow());
+                  const inactiveMs = nowMillis - DateTime.toEpochMillis(worker.lastActivity);
+                  yield* logger.logEdgeCase(
+                    worker.domain,
+                    "worker_stuck_detected",
+                    {
+                      workerId: worker.workerId,
+                      currentUrl: worker.currentUrl,
+                      lastActivity: DateTime.formatIso(worker.lastActivity),
+                      inactiveMs,
+                      fetchStartTime: Option.fromNullable(worker.fetchStartTime).pipe(
+                        Option.map(DateTime.formatIso),
+                        Option.getOrElse(() => "N/A")
+                      )
+                    }
+                  );
+                }
+              }
+            }),
+            Schedule.fixed(Duration.seconds(30))
+          );
+        })
+      };
+    })
+  }
+) {
+}
+const safeJsonParse = (data, onError) => Schema.decodeUnknown(Schema.parseJson(Schema.Unknown))(data).pipe(
+  Effect.mapError(onError)
+);
+const toOption = (value, logContext) => {
+  const result = Option.fromNullable(value);
+  if (logContext && Option.isNone(result)) {
+    return Effect.logDebug(`[Migration] Null value encountered: ${logContext}`).pipe(
+      Effect.map(() => result)
+    );
+  }
+  return Effect.succeed(result);
+};
+const fromPromise = (promise, onError) => Effect.tryPromise({
+  try: promise,
+  catch: onError
+});
+const cleanupResources = (resources) => Effect.all(
+  resources.map(
+    ({ id, cleanup, onError }) => Effect.tryPromise({
+      try: cleanup,
+      catch: (error) => onError(id, error)
+    })
+  ),
+  { mode: "either" }
+);
+const matchOption = (option, onNone, onSome) => Option.match(option, { onNone, onSome });
 export {
+  APITokenNotFoundError,
   AdapterNotInitialisedError,
   BrowserCleanupError,
+  BrowserEngineLive,
+  BrowserEngineService,
+  BrowserEngineWithConfig,
   BrowserError,
+  CSRFTokenNotFoundError,
   ConfigError,
   ConfigurationError,
   ContentTypeError,
   CookieManager,
   CookieManagerLive,
   CrawlError,
+  DEFAULT_DEDUPLICATION_STRATEGY,
   DEFAULT_HYBRID_CONFIG,
   DeltaPersistence,
   EnhancedHttpClient,
   EnhancedHttpClientLive,
+  FetchError,
   FileStorageBackend,
   FileSystemError,
   FullStatePersistence,
@@ -6756,7 +7304,9 @@ export {
   LinkExtractionError,
   LinkExtractorService,
   LinkExtractorServiceLayer,
+  LoggingFetch,
   LoggingMiddleware,
+  LoginError,
   MiddlewareError,
   MiddlewareManager,
   NetworkError,
@@ -6764,6 +7314,7 @@ export {
   PageError,
   ParseError,
   PersistenceError$1 as PersistenceError,
+  PostgresStorageBackend,
   PriorityRequest,
   QueueError,
   RateLimitMiddleware,
@@ -6774,8 +7325,11 @@ export {
   ResumabilityService,
   RobotsService,
   RobotsTxtError,
+  SPIDER_DEFAULTS,
   ScraperService,
   SessionError$1 as SessionError,
+  SessionLoadError,
+  SessionNotValidError,
   SessionStore,
   SessionStoreLive,
   SpiderConfig,
@@ -6790,26 +7344,41 @@ export {
   StateManager,
   StateManagerLive,
   StatsMiddleware,
+  StorageKeyNotFoundError,
   TimeoutError,
+  TokenExpiredError,
   TokenExtractor,
   TokenExtractorLive,
+  TokenNotFoundError,
   TokenType,
   UrlDeduplicatorService,
   UserAgentMiddleware,
   ValidationError,
   WebScrapingEngine,
   WebScrapingEngineLive,
+  WorkerHealthMonitor,
+  cleanupResources,
   createStateOperation,
+  createUrlDeduplicator,
+  deduplicateUrls,
+  fromPromise,
   isBrowserError,
   isNetworkError,
   isSpiderError,
   makeCookieManager,
   makeEnhancedHttpClient,
+  makeLoggingFetch,
   makeSessionStore,
   makeSpiderConfig,
   makeSpiderLogger,
   makeStateManager,
   makeTokenExtractor,
-  makeWebScrapingEngine
+  makeWebScrapingEngine,
+  matchOption,
+  normalizeUrl,
+  parseUrl,
+  safeJsonParse,
+  toOption,
+  withBrowser
 };
 //# sourceMappingURL=index.js.map