html-renderer-api 1.0.0

package/src/scrapers/scraper-login.ts

@@ -0,0 +1,271 @@
+/**
+ * @file scraper-login.ts
+ * @description Google login automation via the BrowserDurableObject.
+ * Handles the multi-step Google sign-in flow (email → password → redirect)
+ * and returns the resulting auth cookies for NotebookLM access.
+ *
+ * Also provides a fetch-as-session endpoint that executes HTTP requests
+ * with the session's stored cookies.
+ */
+
+import puppeteer from "@cloudflare/puppeteer";
+import type { Browser, Page, Cookie } from "@cloudflare/puppeteer";
+import { applyStealthEvasions } from "./scraper-stealth.js";
+import type { Env } from "./scraper-utils.js";
+
+export interface LoginRequest {
+  action: "google-login";
+  sessionId: string;
+  email: string;
+  password: string;
+  targetUrl: string;
+}
+
+export interface LoginResponse {
+  success: boolean;
+  cookies?: Cookie[];
+  finalUrl?: string;
+  error?: string;
+}
+
+export interface FetchAsSessionRequest {
+  sessionId: string;
+  url: string;
+  method?: string;
+  body?: string;
+  cookies?: Cookie[];
+  headers?: Record<string, string>;
+}
+
+/**
+ * Handles POST /api/login — automates Google OAuth login flow via Puppeteer.
+ *
+ * Flow:
+ * 1. Navigate to targetUrl (e.g. NotebookLM) which redirects to Google login
+ * 2. Enter email, click Next
+ * 3. Enter password, click Next
+ * 4. Wait for redirect back to targetUrl
+ * 5. Return all cookies from the authenticated session
+ */
+export async function handleLogin(
+  request: Request,
+  env: Env,
+): Promise<Response> {
+  let body: LoginRequest;
+  try {
+    body = await request.json();
+  } catch {
+    return jsonResponse({ success: false, error: "Invalid JSON body" }, 400);
+  }
+
+  if (!body.email || !body.password || !body.targetUrl) {
+    return jsonResponse(
+      { success: false, error: "email, password, and targetUrl are required" },
+      400,
+    );
+  }
+
+  const id = env.BROWSER_DO.idFromName(`browser-${body.sessionId || "login"}`);
+  const browserDO = env.BROWSER_DO.get(id);
+
+  // Forward the login request to the Durable Object
+  const doRequest = new Request(request.url, {
+    method: "POST",
+    headers: { "Content-Type": "application/json" },
+    body: JSON.stringify({ ...body, _action: "login" }),
+  });
+
+  return browserDO.fetch(doRequest);
+}
+
+/**
+ * Handles POST /api/fetch — execute an HTTP request using the session's
+ * stored browser cookies. This allows the qwksearch-web backend to make
+ * authenticated requests to NotebookLM's internal APIs.
+ */
+export async function handleFetchAsSession(
+  request: Request,
+  env: Env,
+): Promise<Response> {
+  let body: FetchAsSessionRequest;
+  try {
+    body = await request.json();
+  } catch {
+    return jsonResponse({ error: "Invalid JSON body" }, 400);
+  }
+
+  if (!body.url || !body.sessionId) {
+    return jsonResponse({ error: "url and sessionId are required" }, 400);
+  }
+
+  // Build the proxied request with the user's cookies
+  const cookieHeader = body.cookies
+    ?.map((c) => `${c.name}=${c.value}`)
+    .join("; ");
+
+  const headers: Record<string, string> = {
+    ...body.headers,
+    "User-Agent":
+      "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+  };
+
+  if (cookieHeader) {
+    headers["Cookie"] = cookieHeader;
+  }
+
+  try {
+    const response = await fetch(body.url, {
+      method: body.method || "GET",
+      headers,
+      body: body.body,
+    });
+
+    const responseHeaders = new Headers();
+    responseHeaders.set(
+      "Content-Type",
+      response.headers.get("Content-Type") || "application/json",
+    );
+    responseHeaders.set("Access-Control-Allow-Origin", "*");
+
+    return new Response(response.body, {
+      status: response.status,
+      headers: responseHeaders,
+    });
+  } catch (err) {
+    return jsonResponse(
+      { error: `Fetch failed: ${(err as Error).message}` },
+      502,
+    );
+  }
+}
+
+/**
+ * Google login flow executed inside a Durable Object's persistent browser.
+ * Called from BrowserDurableObject when it receives a login action request.
+ */
+export async function executeGoogleLogin(
+  page: Page,
+  email: string,
+  password: string,
+  targetUrl: string,
+): Promise<LoginResponse> {
+  try {
+    // Navigate to the target (NotebookLM) which will redirect to Google login
+    await page.goto(targetUrl, {
+      waitUntil: "networkidle2",
+      timeout: 30000,
+    });
+
+    // Check if already logged in
+    if (page.url().includes("notebooklm.google.com") && !page.url().includes("accounts.google.com")) {
+      const cookies = await page.cookies();
+      return { success: true, cookies, finalUrl: page.url() };
+    }
+
+    // Wait for the email input field
+    await page.waitForSelector('input[type="email"]', { timeout: 10000 });
+    await randomDelay(500, 1000);
+
+    // Type email with human-like delays
+    await page.type('input[type="email"]', email, { delay: 50 + Math.random() * 80 });
+    await randomDelay(300, 700);
+
+    // Click "Next" button
+    const nextButton = await page.$(
+      '#identifierNext, button[type="submit"], [data-idom-class*="next"]',
+    );
+    if (nextButton) {
+      await nextButton.click();
+    } else {
+      await page.keyboard.press("Enter");
+    }
+
+    // Wait for password field to appear
+    await page.waitForSelector('input[type="password"]', {
+      visible: true,
+      timeout: 10000,
+    });
+    await randomDelay(800, 1500);
+
+    // Type password
+    await page.type('input[type="password"]', password, { delay: 40 + Math.random() * 60 });
+    await randomDelay(300, 700);
+
+    // Click "Next" / "Sign in"
+    const signInButton = await page.$(
+      '#passwordNext, button[type="submit"], [data-idom-class*="next"]',
+    );
+    if (signInButton) {
+      await signInButton.click();
+    } else {
+      await page.keyboard.press("Enter");
+    }
+
+    // Wait for navigation to complete (either to target or 2FA/consent)
+    await page.waitForNavigation({
+      waitUntil: "networkidle2",
+      timeout: 30000,
+    });
+
+    // Check for 2FA challenge
+    const currentUrl = page.url();
+    if (
+      currentUrl.includes("challenge") ||
+      currentUrl.includes("signin/v2/challenge")
+    ) {
+      return {
+        success: false,
+        error:
+          "Two-factor authentication required. Please use an app password or disable 2FA temporarily.",
+        finalUrl: currentUrl,
+      };
+    }
+
+    // Check for consent screen
+    if (currentUrl.includes("consent") || currentUrl.includes("oauthchooseaccount")) {
+      // Try to click "Allow" or "Continue"
+      const allowBtn = await page.$(
+        'button[id="submit_approve_access"], [data-value="true"], button:has-text("Allow"), button:has-text("Continue")',
+      );
+      if (allowBtn) {
+        await allowBtn.click();
+        await page.waitForNavigation({ waitUntil: "networkidle2", timeout: 15000 });
+      }
+    }
+
+    // Give the page a moment to settle after login
+    await randomDelay(2000, 3000);
+
+    const finalUrl = page.url();
+    const cookies = await page.cookies();
+
+    return {
+      success: finalUrl.includes("notebooklm.google.com"),
+      cookies,
+      finalUrl,
+      error: finalUrl.includes("notebooklm.google.com")
+        ? undefined
+        : "Did not reach NotebookLM after login",
+    };
+  } catch (err) {
+    return {
+      success: false,
+      error: `Login flow error: ${(err as Error).message}`,
+      finalUrl: page.url(),
+    };
+  }
+}
+
+function randomDelay(min: number, max: number): Promise<void> {
+  return new Promise((r) => setTimeout(r, min + Math.random() * (max - min)));
+}
+
+function jsonResponse(data: unknown, status = 200): Response {
+  return new Response(JSON.stringify(data), {
+    status,
+    headers: {
+      "Content-Type": "application/json",
+      "Access-Control-Allow-Origin": "*",
+    },
+  });
+}

package/src/scrapers/scraper-openapi.ts

@@ -0,0 +1,363 @@
+/**
+ * @file scraper-openapi.ts
+ * @description OpenAPI 3.0 specification and Swagger UI endpoints for the
+ * Puppeteer rendering API.  `serveSwagger` returns the interactive Swagger UI
+ * page; `serveOpenAPI` returns the raw JSON spec consumed by that UI.
+ */
+
+/**
+ * Returns an HTML page that renders the Swagger UI pointed at `./openapi.json`.
+ *
+ * @returns HTTP response with `Content-Type: text/html`.
+ */
+export function serveSwagger(): Response {
+  const html = `
+<!DOCTYPE html>
+<html>
+<head>
+    <title>Puppeteer API Documentation</title>
+    <link rel="stylesheet" type="text/css" href="https://unpkg.com/swagger-ui-dist@4.15.5/swagger-ui.css" />
+    <style>
+        html { box-sizing: border-box; overflow: -moz-scrollbars-vertical; overflow-y: scroll; }
+        *, *:before, *:after { box-sizing: inherit; }
+        body { margin:0; background: #fafafa; }
+    </style>
+</head>
+<body>
+    <div id="swagger-ui"></div>
+    <script src="https://unpkg.com/swagger-ui-dist@4.15.5/swagger-ui-bundle.js"></script>
+    <script>
+        SwaggerUIBundle({
+            url: './openapi.json',
+            dom_id: '#swagger-ui',
+            deepLinking: true,
+            presets: [
+                SwaggerUIBundle.presets.apis,
+                SwaggerUIBundle.presets.standalone
+            ],
+            plugins: [
+                SwaggerUIBundle.plugins.DownloadUrl
+            ]
+        });
+    </script>
+</body>
+</html>`;
+
+  return new Response(html, {
+    headers: { "Content-Type": "text/html" },
+  });
+}
+
+/** Minimal OpenAPI 3.0 schema type used for the spec object below. */
+interface OpenAPISpec {
+  openapi: string;
+  info: {
+    title: string;
+    description: string;
+    version: string;
+    contact: { name: string };
+  };
+  servers: Array<{ url: string; description: string }>;
+  security: Array<Record<string, string[]>>;
+  paths: Record<string, unknown>;
+  components: {
+    securitySchemes: Record<string, unknown>;
+  };
+}
+
+/**
+ * Returns the OpenAPI 3.0.3 JSON specification for the rendering API.
+ * Documents both the GET and POST `/render` endpoints including all
+ * parameters, request body fields, and response schemas.
+ *
+ * @returns HTTP response with `Content-Type: application/json`.
+ */
+export function serveOpenAPI(): Response {
+  const spec: OpenAPISpec = {
+    openapi: "3.0.3",
+    info: {
+      title: "Puppeteer Rendering API",
+      description:
+        "A powerful web scraping and rendering API using Puppeteer with Cloudflare Workers and Durable Objects. Includes automatic Cloudflare challenge bypass.",
+      version: "2.1.0",
+      contact: { name: "API Support" },
+    },
+    servers: [{ url: "/api", description: "API Server" }],
+    security: [{ bearerAuth: [] }, { passwordAuth: [] }],
+    paths: {
+      "/render": {
+        get: {
+          summary: "Render webpage (GET)",
+          description:
+            "Render a webpage using Puppeteer and return the HTML content. Automatically bypasses Cloudflare challenges.",
+          parameters: [
+            {
+              name: "url",
+              in: "query",
+              required: true,
+              schema: { type: "string", format: "uri" },
+              description: "The URL to render",
+            },
+            {
+              name: "SCRAPER_API_KEY",
+              in: "query",
+              required: false,
+              schema: { type: "string" },
+              description: "API key (if required)",
+            },
+            {
+              name: "wait",
+              in: "query",
+              required: false,
+              schema: { type: "integer", minimum: 0, maximum: 30000 },
+              description: "Additional wait time in milliseconds",
+            },
+            {
+              name: "blockImages",
+              in: "query",
+              required: false,
+              schema: { type: "boolean" },
+              description: "Block image loading to save bandwidth",
+            },
+            {
+              name: "sessionId",
+              in: "query",
+              required: false,
+              schema: { type: "string" },
+              description:
+                "Session ID for browser reuse and cookie persistence",
+            },
+            {
+              name: "timeout",
+              in: "query",
+              required: false,
+              schema: { type: "integer", minimum: 5000, maximum: 60000 },
+              description: "Page load timeout in milliseconds",
+            },
+            {
+              name: "waitUntil",
+              in: "query",
+              required: false,
+              schema: {
+                type: "string",
+                enum: [
+                  "load",
+                  "domcontentloaded",
+                  "networkidle0",
+                  "networkidle2",
+                ],
+              },
+              description: "When to consider navigation succeeded",
+            },
+            {
+              name: "cookies",
+              in: "query",
+              required: false,
+              schema: { type: "string" },
+              description: "JSON string of cookies to set",
+            },
+            {
+              name: "format",
+              in: "query",
+              required: false,
+              schema: { type: "string", enum: ["html", "json"] },
+              description: "Response format",
+            },
+            {
+              name: "proxyUrl",
+              in: "query",
+              required: false,
+              schema: { type: "string", format: "uri" },
+              description:
+                "Proxy server URL (e.g., http://proxy.example.com:8080)",
+            },
+            {
+              name: "proxyUser",
+              in: "query",
+              required: false,
+              schema: { type: "string" },
+              description: "Proxy username (if proxy requires authentication)",
+            },
+            {
+              name: "proxyPass",
+              in: "query",
+              required: false,
+              schema: { type: "string" },
+              description: "Proxy password (if proxy requires authentication)",
+            },
+            {
+              name: "bypassCaptcha",
+              in: "query",
+              required: false,
+              schema: { type: "boolean", default: true },
+              description:
+                "Enable Cloudflare challenge bypass (enabled by default)",
+            },
+            {
+              name: "challengeMatch",
+              in: "query",
+              required: false,
+              schema: { type: "string" },
+              description:
+                "Custom string to detect challenge pages (default: 'challenge-platform')",
+            },
+            {
+              name: "maxRetries",
+              in: "query",
+              required: false,
+              schema: { type: "integer", minimum: 1, maximum: 20, default: 10 },
+              description: "Maximum retries for challenge bypass",
+            },
+            {
+              name: "challengeTimeout",
+              in: "query",
+              required: false,
+              schema: {
+                type: "integer",
+                minimum: 1000,
+                maximum: 30000,
+                default: 5000,
+              },
+              description: "Timeout for each challenge retry in milliseconds",
+            },
+            {
+              name: "twoCaptchaKey",
+              in: "query",
+              required: false,
+              schema: { type: "string" },
+              description:
+                "2captcha API key for solving reCAPTCHA/Turnstile challenges (optional, for sites with harder protection)",
+            },
+          ],
+          responses: {
+            "200": {
+              description: "Successfully rendered webpage",
+              content: {
+                "text/html": { schema: { type: "string" } },
+                "application/json": {
+                  schema: {
+                    type: "object",
+                    properties: {
+                      html: { type: "string" },
+                      url: { type: "string" },
+                      title: { type: "string" },
+                      cookies: { type: "array" },
+                      performance: { type: "object" },
+                      challengeBypassed: { type: "boolean" },
+                      retryCount: { type: "integer" },
+                    },
+                  },
+                },
+              },
+            },
+            "400": { description: "Bad request - missing URL or invalid parameters" },
+            "401": { description: "Unauthorized - invalid or missing password" },
+            "500": { description: "Internal server error" },
+          },
+        },
+        post: {
+          summary: "Render webpage (POST)",
+          description:
+            "Render a webpage using Puppeteer with advanced options via POST body. Automatically bypasses Cloudflare challenges.",
+          requestBody: {
+            required: true,
+            content: {
+              "application/json": {
+                schema: {
+                  type: "object",
+                  required: ["url"],
+                  properties: {
+                    url: { type: "string", format: "uri" },
+                    SCRAPER_API_KEY: { type: "string" },
+                    wait: { type: "integer", minimum: 0, maximum: 30000 },
+                    blockImages: { type: "boolean" },
+                    sessionId: { type: "string" },
+                    timeout: {
+                      type: "integer",
+                      minimum: 5000,
+                      maximum: 60000,
+                    },
+                    waitUntil: {
+                      type: "string",
+                      enum: [
+                        "load",
+                        "domcontentloaded",
+                        "networkidle0",
+                        "networkidle2",
+                      ],
+                    },
+                    cookies: { type: "string" },
+                    headers: {
+                      type: "object",
+                      additionalProperties: { type: "string" },
+                    },
+                    format: { type: "string", enum: ["html", "json"] },
+                    proxyUrl: { type: "string", format: "uri" },
+                    proxyUser: { type: "string" },
+                    proxyPass: { type: "string" },
+                    bypassCaptcha: { type: "boolean", default: true },
+                    challengeMatch: { type: "string" },
+                    maxRetries: {
+                      type: "integer",
+                      minimum: 1,
+                      maximum: 20,
+                      default: 10,
+                    },
+                    challengeTimeout: {
+                      type: "integer",
+                      minimum: 1000,
+                      maximum: 30000,
+                      default: 5000,
+                    },
+                    twoCaptchaKey: { type: "string" },
+                  },
+                },
+              },
+            },
+          },
+          responses: {
+            "200": {
+              description: "Successfully rendered webpage",
+              content: {
+                "text/html": { schema: { type: "string" } },
+                "application/json": {
+                  schema: {
+                    type: "object",
+                    properties: {
+                      html: { type: "string" },
+                      url: { type: "string" },
+                      title: { type: "string" },
+                      cookies: { type: "array" },
+                      performance: { type: "object" },
+                      challengeBypassed: { type: "boolean" },
+                      retryCount: { type: "integer" },
+                    },
+                  },
+                },
+              },
+            },
+          },
+        },
+      },
+    },
+    components: {
+      securitySchemes: {
+        bearerAuth: {
+          type: "http",
+          scheme: "bearer",
+          description: "Use your API key as the bearer token",
+        },
+        apiKeyAuth: {
+          type: "apiKey",
+          in: "query",
+          name: "SCRAPER_API_KEY",
+          description: "API key as query parameter",
+        },
+      },
+    },
+  };
+
+  return new Response(JSON.stringify(spec, null, 2), {
+    headers: { "Content-Type": "application/json" },
+  });
+}