@jambudipa/spider 0.3.1 → 0.3.3

package/dist/index.d.ts

@@ -28,6 +28,8 @@ export type { CookieManagerService, EnhancedHttpClientService, HttpRequestOption
 export { CookieManager, makeCookieManager, CookieManagerLive, EnhancedHttpClient, makeEnhancedHttpClient, EnhancedHttpClientLive, SessionStore, makeSessionStore, SessionStoreLive, TokenExtractor, makeTokenExtractor, TokenExtractorLive, } from './lib/HttpClient/index.js';
 export type { Token, StateManagerService } from './lib/StateManager/index.js';
 export { TokenType, StateManager, makeStateManager, StateManagerLive, } from './lib/StateManager/index.js';
+export type { BrowserEngineConfig, BrowserEngineServiceInterface, PageElement, } from './lib/BrowserEngine/BrowserEngine.service.js';
+export { BrowserEngineService, BrowserEngineLive, BrowserEngineWithConfig, withBrowser, } from './lib/BrowserEngine/BrowserEngine.service.js';
 export type { LoginCredentials, ScrapingSession, WebScrapingEngineService, } from './lib/WebScrapingEngine/index.js';
 export { WebScrapingEngine, makeWebScrapingEngine, WebScrapingEngineLive, } from './lib/WebScrapingEngine/index.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,YAAY,EACV,OAAO,EACP,gBAAgB,EAChB,kBAAkB,EAClB,oBAAoB,EACpB,kBAAkB,EAClB,oBAAoB,EACpB,gBAAgB,GACjB,MAAM,sBAAsB,CAAC;AAG9B,cAAc,gCAAgC,CAAC;AAC/C,cAAc,gCAAgC,CAAC;AAC/C,cAAc,kCAAkC,CAAC;AACjD,cAAc,4BAA4B,CAAC;AAG3C,YAAY,EACV,mBAAmB,EACnB,mBAAmB,GACpB,MAAM,sCAAsC,CAAC;AAC9C,OAAO,EACL,YAAY,EACZ,gBAAgB,GACjB,MAAM,sCAAsC,CAAC;AAG9C,YAAY,EAAE,gBAAgB,EAAE,MAAM,kDAAkD,CAAC;AACzF,OAAO,EAAE,sBAAsB,EAAE,MAAM,kDAAkD,CAAC;AAG1F,YAAY,EAAE,gBAAgB,EAAE,MAAM,4CAA4C,CAAC;AACnF,OAAO,EACL,sBAAsB,EACtB,cAAc,EACd,eAAe,EACf,WAAW,GACZ,MAAM,4CAA4C,CAAC;AAGpD,YAAY,EACV,gBAAgB,EAChB,aAAa,EACb,cAAc,GACf,MAAM,sCAAsC,CAAC;AAC9C,OAAO,EACL,iBAAiB,EACjB,mBAAmB,EACnB,iBAAiB,EACjB,mBAAmB,EACnB,eAAe,GAChB,MAAM,sCAAsC,CAAC;AAG9C,YAAY,EACV,mBAAmB,EACnB,oBAAoB,EACpB,6BAA6B,GAC9B,MAAM,8CAA8C,CAAC;AACtD,OAAO,EACL,oBAAoB,EACpB,yBAAyB,EACzB,mBAAmB,GACpB,MAAM,8CAA8C,CAAC;AAGtD,YAAY,EACV,WAAW,EACX,SAAS,EACT,2BAA2B,GAC5B,MAAM,gCAAgC,CAAC;AAGxC,YAAY,EACV,mBAAmB,EACnB,cAAc,EACd,cAAc,EACd,mBAAmB,EACnB,uBAAuB,GACxB,MAAM,6BAA6B,CAAC;AACrC,YAAY,EAAE,kBAAkB,EAAE,MAAM,4CAA4C,CAAC;AACrF,OAAO,EACL,UAAU,EACV,gBAAgB,IAAI,iBAAiB,EACrC,qBAAqB,GACtB,MAAM,6BAA6B,CAAC;AACrC,OAAO,EACL,mBAAmB,EACnB,mBAAmB,EACnB,oBAAoB,GACrB,MAAM,4CAA4C,CAAC;AACpD,OAAO,EACL,oBAAoB,EACpB,gBAAgB,EAChB,iBAAiB,GAClB,MAAM,kCAAkC,CAAC;AAC1C,OAAO,EAAE,kBAAkB,EAAE,MAAM,mDAAmD,CAAC;AAGvF,OAAO,EACL,YAAY,EACZ,aAAa,EACb,cAAc,EACd,kBAAkB,EAClB,eAAe,EACf,eAAe,EACf,gBAAgB,EAChB,gBAAgB,EAChB,iBAAiB,EACjB,0BAA0B,EAC1B,YAAY,EACZ,mBAAmB,EACnB,YAAY,EACZ,UAAU,EACV,eAAe,EACf,SAAS,EACT,UAAU,EACV,YAAY,EACZ,UAAU,EACV,UAAU,EACV,WAAW,EACX,aAAa,EACb,cAAc,EACd,cAAc,GACf,MAAM,+BAA+B,CAAC;AACvC,YAAY,EAAE,WAAW,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAC;AAGlF,YAAY,EACV,cAAc,EACd,YAAY,GACb,MAAM,uCAAuC,CAAC;AAC/C,OAAO,EACL,YAAY,IAAI,eAAe,EAC/B,gBAAgB,EAChB,gBAAgB,GACjB,MAAM,uCAAuC,CAAC;AAG/C,YAAY,EACV,oBAAoB,EACpB,yBAAyB,EACzB,kBAAkB,EAClB,YAAY,EACZ,OAAO,EACP,WAAW,EACX,mBAAmB,EACnB,SAAS,EACT,qBAAqB,GACtB,MAAM,2BAA2B,CAAC;AACnC,OAAO,EACL,aAAa,EACb,iBAAiB,EACjB,iBAAiB,EACjB,kBAAkB,EAClB,sBAAsB,EACtB,sBAAsB,EACtB,YAAY,EACZ,gBAAgB,EAChB,gBAAgB,EAChB,cAAc,EACd,kBAAkB,EAClB,kBAAkB,GACnB,MAAM,2BAA2B,CAAC;AAGnC,YAAY,EAAE,KAAK,EAAE,mBAAmB,EAAE,MAAM,6BAA6B,CAAC;AAC9E,OAAO,EACL,SAAS,EACT,YAAY,EACZ,gBAAgB,EAChB,gBAAgB,GACjB,MAAM,6BAA6B,CAAC;AAGrC,YAAY,EACV,gBAAgB,EAChB,eAAe,EACf,wBAAwB,GACzB,MAAM,kCAAkC,CAAC;AAC1C,OAAO,EACL,iBAAiB,EACjB,qBAAqB,EACrB,qBAAqB,GACtB,MAAM,kCAAkC,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,YAAY,EACV,OAAO,EACP,gBAAgB,EAChB,kBAAkB,EAClB,oBAAoB,EACpB,kBAAkB,EAClB,oBAAoB,EACpB,gBAAgB,GACjB,MAAM,sBAAsB,CAAC;AAG9B,cAAc,gCAAgC,CAAC;AAC/C,cAAc,gCAAgC,CAAC;AAC/C,cAAc,kCAAkC,CAAC;AACjD,cAAc,4BAA4B,CAAC;AAG3C,YAAY,EACV,mBAAmB,EACnB,mBAAmB,GACpB,MAAM,sCAAsC,CAAC;AAC9C,OAAO,EACL,YAAY,EACZ,gBAAgB,GACjB,MAAM,sCAAsC,CAAC;AAG9C,YAAY,EAAE,gBAAgB,EAAE,MAAM,kDAAkD,CAAC;AACzF,OAAO,EAAE,sBAAsB,EAAE,MAAM,kDAAkD,CAAC;AAG1F,YAAY,EAAE,gBAAgB,EAAE,MAAM,4CAA4C,CAAC;AACnF,OAAO,EACL,sBAAsB,EACtB,cAAc,EACd,eAAe,EACf,WAAW,GACZ,MAAM,4CAA4C,CAAC;AAGpD,YAAY,EACV,gBAAgB,EAChB,aAAa,EACb,cAAc,GACf,MAAM,sCAAsC,CAAC;AAC9C,OAAO,EACL,iBAAiB,EACjB,mBAAmB,EACnB,iBAAiB,EACjB,mBAAmB,EACnB,eAAe,GAChB,MAAM,sCAAsC,CAAC;AAG9C,YAAY,EACV,mBAAmB,EACnB,oBAAoB,EACpB,6BAA6B,GAC9B,MAAM,8CAA8C,CAAC;AACtD,OAAO,EACL,oBAAoB,EACpB,yBAAyB,EACzB,mBAAmB,GACpB,MAAM,8CAA8C,CAAC;AAGtD,YAAY,EACV,WAAW,EACX,SAAS,EACT,2BAA2B,GAC5B,MAAM,gCAAgC,CAAC;AAGxC,YAAY,EACV,mBAAmB,EACnB,cAAc,EACd,cAAc,EACd,mBAAmB,EACnB,uBAAuB,GACxB,MAAM,6BAA6B,CAAC;AACrC,YAAY,EAAE,kBAAkB,EAAE,MAAM,4CAA4C,CAAC;AACrF,OAAO,EACL,UAAU,EACV,gBAAgB,IAAI,iBAAiB,EACrC,qBAAqB,GACtB,MAAM,6BAA6B,CAAC;AACrC,OAAO,EACL,mBAAmB,EACnB,mBAAmB,EACnB,oBAAoB,GACrB,MAAM,4CAA4C,CAAC;AACpD,OAAO,EACL,oBAAoB,EACpB,gBAAgB,EAChB,iBAAiB,GAClB,MAAM,kCAAkC,CAAC;AAC1C,OAAO,EAAE,kBAAkB,EAAE,MAAM,mDAAmD,CAAC;AAGvF,OAAO,EACL,YAAY,EACZ,aAAa,EACb,cAAc,EACd,kBAAkB,EAClB,eAAe,EACf,eAAe,EACf,gBAAgB,EAChB,gBAAgB,EAChB,iBAAiB,EACjB,0BAA0B,EAC1B,YAAY,EACZ,mBAAmB,EACnB,YAAY,EACZ,UAAU,EACV,eAAe,EACf,SAAS,EACT,UAAU,EACV,YAAY,EACZ,UAAU,EACV,UAAU,EACV,WAAW,EACX,aAAa,EACb,cAAc,EACd,cAAc,GACf,MAAM,+BAA+B,CAAC;AACvC,YAAY,EAAE,WAAW,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAC;AAGlF,YAAY,EACV,cAAc,EACd,YAAY,GACb,MAAM,uCAAuC,CAAC;AAC/C,OAAO,EACL,YAAY,IAAI,eAAe,EAC/B,gBAAgB,EAChB,gBAAgB,GACjB,MAAM,uCAAuC,CAAC;AAG/C,YAAY,EACV,oBAAoB,EACpB,yBAAyB,EACzB,kBAAkB,EAClB,YAAY,EACZ,OAAO,EACP,WAAW,EACX,mBAAmB,EACnB,SAAS,EACT,qBAAqB,GACtB,MAAM,2BAA2B,CAAC;AACnC,OAAO,EACL,aAAa,EACb,iBAAiB,EACjB,iBAAiB,EACjB,kBAAkB,EAClB,sBAAsB,EACtB,sBAAsB,EACtB,YAAY,EACZ,gBAAgB,EAChB,gBAAgB,EAChB,cAAc,EACd,kBAAkB,EAClB,kBAAkB,GACnB,MAAM,2BAA2B,CAAC;AAGnC,YAAY,EAAE,KAAK,EAAE,mBAAmB,EAAE,MAAM,6BAA6B,CAAC;AAC9E,OAAO,EACL,SAAS,EACT,YAAY,EACZ,gBAAgB,EAChB,gBAAgB,GACjB,MAAM,6BAA6B,CAAC;AAGrC,YAAY,EACV,mBAAmB,EACnB,6BAA6B,EAC7B,WAAW,GACZ,MAAM,8CAA8C,CAAC;AACtD,OAAO,EACL,oBAAoB,EACpB,iBAAiB,EACjB,uBAAuB,EACvB,WAAW,GACZ,MAAM,8CAA8C,CAAC;AAGtD,YAAY,EACV,gBAAgB,EAChB,eAAe,EACf,wBAAwB,GACzB,MAAM,kCAAkC,CAAC;AAC1C,OAAO,EACL,iBAAiB,EACjB,qBAAqB,EACrB,qBAAqB,GACtB,MAAM,kCAAkC,CAAC"}

package/dist/index.js

@@ -6527,6 +6527,251 @@ const TokenExtractorLive = Layer.effect(
   TokenExtractor,
   makeTokenExtractor
 );
+class BrowserEngineService extends Effect.Service()(
+  "@jambudipa.io/BrowserEngine",
+  {
+    effect: Effect.gen(function* () {
+      const browserRef = yield* Ref.make(Option.none());
+      const contextRef = yield* Ref.make(Option.none());
+      const pageRef = yield* Ref.make(Option.none());
+      const configRef = yield* Ref.make({
+        headless: true,
+        timeout: 3e4,
+        viewport: { width: 1920, height: 1080 },
+        userAgent: "Mozilla/5.0 (compatible; Spider/1.0)",
+        locale: "en-GB"
+      });
+      const ensureBrowser = () => Effect.gen(function* () {
+        const browserOpt = yield* Ref.get(browserRef);
+        if (Option.isSome(browserOpt)) {
+          return browserOpt.value;
+        }
+        const { chromium } = yield* Effect.tryPromise({
+          try: () => import("playwright"),
+          catch: () => BrowserError.launchFailed("Playwright not installed")
+        });
+        const config = yield* Ref.get(configRef);
+        const browser = yield* Effect.tryPromise({
+          try: () => chromium.launch({
+            headless: config.headless,
+            timeout: config.timeout
+          }),
+          catch: (error) => BrowserError.launchFailed(error)
+        });
+        yield* Ref.set(browserRef, Option.some(browser));
+        return browser;
+      });
+      const ensureContext = () => Effect.gen(function* () {
+        const contextOpt = yield* Ref.get(contextRef);
+        if (Option.isSome(contextOpt)) {
+          return contextOpt.value;
+        }
+        const browser = yield* ensureBrowser();
+        const config = yield* Ref.get(configRef);
+        const context = yield* Effect.tryPromise({
+          try: () => browser.newContext({
+            viewport: config.viewport,
+            userAgent: config.userAgent,
+            locale: config.locale
+          }),
+          catch: (error) => new BrowserError({
+            operation: "newContext",
+            cause: error
+          })
+        });
+        yield* Ref.set(contextRef, Option.some(context));
+        return context;
+      });
+      const getCurrentPage = () => Effect.gen(function* () {
+        const pageOpt = yield* Ref.get(pageRef);
+        return yield* Option.match(pageOpt, {
+          onNone: () => Effect.fail(new PageError({
+            url: "unknown",
+            operation: "getCurrentPage",
+            cause: "No active page"
+          })),
+          onSome: (page) => Effect.succeed(page)
+        });
+      });
+      return {
+        launch: () => Effect.gen(function* () {
+          yield* ensureBrowser();
+          yield* Effect.log("Browser launched successfully");
+        }),
+        createPage: () => Effect.gen(function* () {
+          const context = yield* ensureContext();
+          const page = yield* Effect.tryPromise({
+            try: () => context.newPage(),
+            catch: (error) => new BrowserError({
+              operation: "newPage",
+              cause: error
+            })
+          });
+          yield* Ref.set(pageRef, Option.some(page));
+          yield* Effect.log("New page created");
+          return page;
+        }),
+        navigateTo: (url) => Effect.gen(function* () {
+          const page = yield* getCurrentPage();
+          yield* Effect.tryPromise({
+            try: () => page.goto(url, { waitUntil: "networkidle" }),
+            catch: (error) => new PageError({
+              url,
+              operation: "navigate",
+              cause: error
+            })
+          });
+          yield* Effect.logDebug(`Navigated to ${url}`);
+        }),
+        waitForSelector: (selector, timeout) => Effect.gen(function* () {
+          const page = yield* getCurrentPage();
+          const config = yield* Ref.get(configRef);
+          yield* Effect.tryPromise({
+            try: () => page.waitForSelector(selector, {
+              timeout: timeout ?? config.timeout
+            }),
+            catch: (error) => new PageError({
+              url: page.url(),
+              operation: "waitForSelector",
+              selector,
+              cause: error
+            })
+          });
+        }),
+        click: (selector) => Effect.gen(function* () {
+          const page = yield* getCurrentPage();
+          yield* Effect.tryPromise({
+            try: () => page.click(selector),
+            catch: (error) => new PageError({
+              url: page.url(),
+              operation: "click",
+              selector,
+              cause: error
+            })
+          });
+          yield* Effect.logDebug(`Clicked element: ${selector}`);
+        }),
+        fill: (selector, value) => Effect.gen(function* () {
+          const page = yield* getCurrentPage();
+          yield* Effect.tryPromise({
+            try: () => page.fill(selector, value),
+            catch: (error) => new PageError({
+              url: page.url(),
+              operation: "fill",
+              selector,
+              cause: error
+            })
+          });
+          yield* Effect.logDebug(`Filled ${selector} with value`);
+        }),
+        scroll: (distance) => Effect.gen(function* () {
+          const page = yield* getCurrentPage();
+          yield* Effect.ignore(
+            Effect.tryPromise({
+              try: () => page.evaluate((d) => {
+                window.scrollBy(0, d);
+              }, distance),
+              catch: (error) => error
+            })
+          );
+          yield* Effect.logDebug(`Scrolled ${distance}px`);
+        }),
+        evaluate: (script) => Effect.gen(function* () {
+          const page = yield* getCurrentPage();
+          return yield* Effect.tryPromise({
+            try: () => page.evaluate(script),
+            catch: (error) => new PageError({
+              url: page.url(),
+              operation: "evaluate",
+              cause: error
+            })
+          });
+        }),
+        getHTML: () => Effect.gen(function* () {
+          const page = yield* getCurrentPage();
+          return yield* Effect.tryPromise({
+            try: () => page.content(),
+            catch: (error) => new PageError({
+              url: page.url(),
+              operation: "getHTML",
+              cause: error
+            })
+          });
+        }),
+        screenshot: (path2) => Effect.gen(function* () {
+          const page = yield* getCurrentPage();
+          const buffer = yield* Effect.tryPromise({
+            try: () => page.screenshot({ path: path2, fullPage: true }),
+            catch: (error) => new PageError({
+              url: page.url(),
+              operation: "screenshot",
+              cause: error
+            })
+          });
+          yield* Effect.log(`Screenshot taken${path2 ? ` and saved to ${path2}` : ""}`);
+          return buffer;
+        }),
+        closePage: () => Effect.gen(function* () {
+          const pageOpt = yield* Ref.get(pageRef);
+          if (Option.isSome(pageOpt)) {
+            yield* Effect.ignore(
+              Effect.tryPromise({
+                try: () => pageOpt.value.close(),
+                catch: (error) => error
+              })
+            );
+            yield* Ref.set(pageRef, Option.none());
+            yield* Effect.log("Page closed");
+          }
+        }),
+        close: () => Effect.gen(function* () {
+          const pageOpt = yield* Ref.get(pageRef);
+          if (Option.isSome(pageOpt)) {
+            yield* Effect.ignore(
+              Effect.tryPromise({
+                try: () => pageOpt.value.close(),
+                catch: (error) => error
+              })
+            );
+          }
+          const contextOpt = yield* Ref.get(contextRef);
+          if (Option.isSome(contextOpt)) {
+            yield* Effect.ignore(
+              Effect.tryPromise({
+                try: () => contextOpt.value.close(),
+                catch: (error) => error
+              })
+            );
+          }
+          const browserOpt = yield* Ref.get(browserRef);
+          if (Option.isSome(browserOpt)) {
+            yield* Effect.ignore(
+              Effect.tryPromise({
+                try: () => browserOpt.value.close(),
+                catch: (error) => error
+              })
+            );
+          }
+          yield* Ref.set(pageRef, Option.none());
+          yield* Ref.set(contextRef, Option.none());
+          yield* Ref.set(browserRef, Option.none());
+          yield* Effect.log("Browser engine closed");
+        })
+      };
+    })
+  }
+) {
+}
+const BrowserEngineLive = BrowserEngineService.Default;
+const BrowserEngineWithConfig = (_config) => BrowserEngineService.Default;
+const withBrowser = (operation) => Effect.gen(function* () {
+  const engine = yield* BrowserEngineService;
+  return yield* Effect.acquireUseRelease(
+    Effect.succeed(engine),
+    operation,
+    (engine2) => engine2.close()
+  );
+});
 class LoginError extends Data.TaggedError("LoginError") {
 }
 class SessionNotValidError extends Data.TaggedError("SessionNotValidError") {
@@ -6738,6 +6983,9 @@ const WebScrapingEngineLive = Layer.effect(
 export {
   AdapterNotInitialisedError,
   BrowserCleanupError,
+  BrowserEngineLive,
+  BrowserEngineService,
+  BrowserEngineWithConfig,
   BrowserError,
   ConfigError,
   ConfigurationError,
@@ -6810,6 +7058,7 @@ export {
   makeSpiderLogger,
   makeStateManager,
   makeTokenExtractor,
-  makeWebScrapingEngine
+  makeWebScrapingEngine,
+  withBrowser
 };
 //# sourceMappingURL=index.js.map