automate-google-login-scraper 0.1.9

package/dist/worker.js

@@ -0,0 +1,309 @@
+import { n as normalizeStorageState, s as summarizeStorageState, r as redactStorageState, b as applyStorageState, e as extractStorageState, i as isAuthorized, A as AUTH_HEADER } from "./secret-B4HOopFT.js";
+const KEY_STATE = "storage-state";
+function json(body, status = 200) {
+  return new Response(JSON.stringify(body, null, 2), {
+    status,
+    headers: { "content-type": "application/json; charset=utf-8" }
+  });
+}
+async function handleRequest(request, env) {
+  if (!env.TEST_AUTH_SECRET) {
+    return json(
+      {
+        error: "not_configured",
+        message: "TEST_AUTH_SECRET is not set, so this Worker refuses every request. Set it with `wrangler secret put TEST_AUTH_SECRET`."
+      },
+      503
+    );
+  }
+  if (!isAuthorized(request.headers, env.TEST_AUTH_SECRET)) {
+    return new Response("Unauthorized", { status: 401, headers: { "x-required-header": AUTH_HEADER } });
+  }
+  const url = new URL(request.url);
+  if (url.pathname === "/health") return json({ ok: true });
+  const session = url.searchParams.get("session") || "default";
+  const stub = env.BROWSER_SESSION.get(env.BROWSER_SESSION.idFromName(session));
+  return stub.fetch(request);
+}
+const worker = {
+  fetch: handleRequest
+};
+let puppeteerOverride;
+function setPuppeteer(implementation) {
+  puppeteerOverride = implementation;
+}
+async function getPuppeteer() {
+  if (puppeteerOverride) return puppeteerOverride;
+  const loaded = await import("@cloudflare/puppeteer");
+  const implementation = loaded.default ?? loaded;
+  if (typeof implementation?.launch !== "function") {
+    throw new TypeError(
+      "Could not load @cloudflare/puppeteer. Install it in the Worker that deploys this, or call setPuppeteer() with your own implementation."
+    );
+  }
+  return implementation;
+}
+class GoogleLoginBrowser {
+  storage;
+  env;
+  browser;
+  constructor(state, env) {
+    this.storage = state.storage;
+    this.env = env;
+  }
+  get idleMs() {
+    const minutes = Number.parseFloat(this.env.BROWSER_IDLE_MINUTES ?? "3");
+    return (Number.isFinite(minutes) && minutes > 0 ? minutes : 3) * 6e4;
+  }
+  async fetch(request) {
+    const url = new URL(request.url);
+    try {
+      switch (`${request.method} ${url.pathname}`) {
+        case "POST /state":
+          return await this.putState(request);
+        case "GET /state":
+          return await this.describeState();
+        case "DELETE /state":
+          return await this.clearState();
+        case "POST /check":
+          return await this.check(request);
+        case "POST /login":
+          return await this.login(request);
+        case "POST /close":
+          await this.closeBrowser();
+          return json({ closed: true });
+        default:
+          return json({ error: "not_found", message: `No route for ${request.method} ${url.pathname}` }, 404);
+      }
+    } catch (error) {
+      return json({ error: "failed", message: error.message }, 500);
+    }
+  }
+  /**
+   * Store a storage state for this session.
+   *
+   * Validated on the way in, so a malformed upload fails here with a precise
+   * reason rather than as an opaque CDP error during the next check.
+   */
+  async putState(request) {
+    const state = normalizeStorageState(await request.json());
+    await this.storage.put(KEY_STATE, state);
+    return json({ stored: true, summary: summarizeStorageState(state) });
+  }
+  /**
+   * Describe the stored state — redacted, always.
+   *
+   * There is no route that returns the live values. That is the point: this Worker
+   * can be handed to CI without handing CI a way to read the session back out.
+   */
+  async describeState() {
+    const state = await this.storage.get(KEY_STATE);
+    if (!state) return json({ present: false }, 404);
+    return json({
+      present: true,
+      summary: summarizeStorageState(state),
+      redacted: redactStorageState(state)
+    });
+  }
+  async clearState() {
+    const existed = await this.storage.delete(KEY_STATE);
+    await this.closeBrowser();
+    return json({ cleared: existed });
+  }
+  /**
+   * Replay the stored session against a URL and report whether it is signed in.
+   *
+   * "Signed in" is whatever you say it is: `expectSelector` is checked in the page
+   * and `rejectUrl` catches the redirect-to-login case, which is the failure that
+   * otherwise looks like a passing 200.
+   */
+  async check(request) {
+    const body = await request.json();
+    if (!body.url) return json({ error: "bad_request", message: "`url` is required" }, 400);
+    const state = await this.storage.get(KEY_STATE);
+    if (!state) {
+      return json(
+        {
+          error: "no_state",
+          message: "No stored session for this name. POST /state with a Playwright storage state first."
+        },
+        409
+      );
+    }
+    const browser = await this.ensureBrowser();
+    const page = await browser.newPage();
+    try {
+      const applied = await applyStorageState(page, state, { origins: body.origins });
+      const response = await page.goto(body.url, { waitUntil: "domcontentloaded" });
+      const landedUrl = page.url();
+      const title = await page.title();
+      let authenticated = true;
+      let reason = "navigated without being redirected away";
+      if (body.rejectUrl && landedUrl.includes(body.rejectUrl)) {
+        authenticated = false;
+        reason = `redirected to ${body.rejectUrl} — the session was not accepted`;
+      } else if (body.expectSelector) {
+        const found = await page.evaluate(
+          ((selector) => document.querySelector(selector) !== null),
+          body.expectSelector
+        );
+        authenticated = found === true;
+        reason = found ? `found ${body.expectSelector}` : `${body.expectSelector} is not on the page — probably signed out`;
+      }
+      const result = {
+        authenticated,
+        status: typeof response?.status === "function" ? response.status() : null,
+        url: landedUrl,
+        title,
+        reason,
+        applied
+      };
+      if (body.screenshot) {
+        const shot = await page.screenshot({ type: "png" });
+        result.screenshot = toBase64(shot);
+      }
+      return json(result);
+    } finally {
+      await page.close();
+      await this.touch();
+    }
+  }
+  /**
+   * Sign in to Google for real, in the Worker, and store the resulting session.
+   *
+   * Gated twice — the flag and the credentials — and still the path you should
+   * reach for last. Read {@link signInWithGoogle}'s notes in `google-login.ts`:
+   * they apply here in full, with the extra wrinkle that a datacentre IP makes
+   * Google's risk checks more likely, not less.
+   */
+  async login(request) {
+    if (this.env.ALLOW_REAL_GOOGLE_LOGIN !== "true") {
+      return json(
+        {
+          error: "disabled",
+          message: 'Real Google sign-in is disabled. Set ALLOW_REAL_GOOGLE_LOGIN="true" and the GOOGLE_TEST_EMAIL / GOOGLE_TEST_PASSWORD secrets to enable POST /login — and prefer capturing the session locally with `playwright codegen` instead.'
+        },
+        403
+      );
+    }
+    const email = this.env.GOOGLE_TEST_EMAIL;
+    const password = this.env.GOOGLE_TEST_PASSWORD;
+    if (!email || !password) {
+      return json(
+        { error: "disabled", message: "GOOGLE_TEST_EMAIL and GOOGLE_TEST_PASSWORD must both be set." },
+        403
+      );
+    }
+    const body = await request.json();
+    if (!body.loginUrl) return json({ error: "bad_request", message: "`loginUrl` is required" }, 400);
+    if (!body.expectUrl) {
+      return json(
+        {
+          error: "bad_request",
+          message: '`expectUrl` is required — a substring of the URL you land on once signed in, e.g. "/dashboard". It is not defaulted to your origin, because the login page is on that origin too and the wait would pass before sign-in even started.'
+        },
+        400
+      );
+    }
+    const browser = await this.ensureBrowser();
+    const page = await browser.newPage();
+    try {
+      await page.goto(body.loginUrl, { waitUntil: "domcontentloaded" });
+      await clickSelector(page, body.signInSelector ?? 'a[href*="google"], button[data-provider="google"]');
+      await typeInto(page, 'input[type="email"]', email);
+      await clickSelector(page, "#identifierNext button, #identifierNext");
+      await typeInto(page, 'input[type="password"]', password);
+      await clickSelector(page, "#passwordNext button, #passwordNext");
+      await waitForUrlContaining(
+        page,
+        body.expectUrl,
+        // Capped rather than trusted: an unbounded wait pins a billed browser
+        // session open for as long as the caller likes.
+        Math.min(Math.max(body.timeoutMs ?? 3e4, 250), 12e4)
+      );
+      const state = await extractStorageState(page, { origins: body.origins });
+      await this.storage.put(KEY_STATE, state);
+      return json({ stored: true, summary: summarizeStorageState(state) });
+    } finally {
+      await page.close();
+      await this.touch();
+    }
+  }
+  async ensureBrowser() {
+    if (this.browser?.isConnected?.() === false) this.browser = void 0;
+    if (!this.browser) {
+      const puppeteer = await getPuppeteer();
+      this.browser = await puppeteer.launch(this.env.BROWSER);
+    }
+    await this.touch();
+    return this.browser;
+  }
+  /** Push the idle alarm out. Called after every request that used the browser. */
+  async touch() {
+    await this.storage.setAlarm(Date.now() + this.idleMs);
+  }
+  /**
+   * Close the browser when the session goes quiet.
+   *
+   * Not an optimisation — a Browser Rendering session left open is billed and
+   * counts against the account's concurrency limit, so the alarm is the difference
+   * between a test tool and a surprise invoice.
+   */
+  async alarm() {
+    await this.closeBrowser();
+  }
+  async closeBrowser() {
+    const browser = this.browser;
+    this.browser = void 0;
+    if (!browser) return;
+    try {
+      await browser.close();
+    } catch {
+    }
+  }
+}
+async function clickSelector(page, selector) {
+  await page.evaluate(
+    ((css) => {
+      const element = document.querySelector(css);
+      if (!element) throw new Error(`No element matched ${css}`);
+      element.click();
+    }),
+    selector
+  );
+}
+async function typeInto(page, selector, value) {
+  await page.evaluate(
+    (([css, text]) => {
+      const element = document.querySelector(css);
+      if (!element) throw new Error(`No element matched ${css}`);
+      element.focus();
+      element.value = text;
+      element.dispatchEvent(new Event("input", { bubbles: true }));
+      element.dispatchEvent(new Event("change", { bubbles: true }));
+    }),
+    [selector, value]
+  );
+}
+async function waitForUrlContaining(page, fragment, timeoutMs = 3e4) {
+  const deadline = Date.now() + timeoutMs;
+  while (Date.now() < deadline) {
+    if (page.url().includes(fragment)) return;
+    await new Promise((resolve) => setTimeout(resolve, 250));
+  }
+  throw new Error(
+    `Timed out waiting for a URL containing "${fragment}" — currently ${page.url()}. A consent screen, a device check or MFA looks like this, and none of them should be automated: capture the session by hand instead.`
+  );
+}
+function toBase64(bytes) {
+  const view = bytes instanceof Uint8Array ? bytes : new Uint8Array(bytes);
+  let binary = "";
+  for (const byte of view) binary += String.fromCharCode(byte);
+  return btoa(binary);
+}
+export {
+  GoogleLoginBrowser,
+  worker as default,
+  handleRequest,
+  setPuppeteer
+};

package/examples/auth.setup.ts

@@ -0,0 +1,42 @@
+/**
+ * The setup project. Pick one of the two bodies below.
+ *
+ * **Verify a hand-captured session** (the default, and what a laptop should do):
+ * no credentials anywhere, no Google traffic, and a sub-millisecond check.
+ *
+ * **Mint your app's own session** (what CI should do): your app creates the same
+ * session a successful Google callback creates, so the run holds one secret of
+ * yours instead of a Google password.
+ */
+import { test as setup } from "@playwright/test";
+import { bootstrapAppSession, requireStoredSession } from "test-google-login";
+
+const SECRET = process.env.E2E_TEST_AUTH_SECRET;
+
+setup("an authenticated session is available", async ({ page, context, request }) => {
+  if (SECRET) {
+    // CI path. Regenerates state every run, so nothing persists between jobs.
+    const file = await bootstrapAppSession({
+      request,
+      page,
+      context,
+      secret: SECRET,
+      endpoint: process.env.E2E_TEST_AUTH_ENDPOINT ?? "/api/test-auth/google-user",
+      landingPath: "/dashboard",
+    });
+
+    console.log(`minted a fresh app session → ${file}`);
+    return;
+  }
+
+  // Local path. Throws with the exact `playwright codegen` command on the first
+  // run, which is the whole onboarding story for a new contributor.
+  const summary = requireStoredSession({ baseUrl: process.env.E2E_BASE_URL });
+
+  console.log(
+    `session ok — ${summary.cookieCount} cookies across ${summary.domains.join(", ")}` +
+      (summary.expiresInSeconds === null
+        ? " (no expiry; dies with the browser)"
+        : `, earliest expiry in ${Math.round(summary.expiresInSeconds / 60)} min`),
+  );
+});

package/examples/dashboard.spec.ts

@@ -0,0 +1,20 @@
+/**
+ * What the tests look like once the setup project has done its work: nothing
+ * about auth, because the browser is already signed in.
+ */
+import { expect, test } from "@playwright/test";
+
+test("an authenticated user can open the dashboard", async ({ page }) => {
+  await page.goto("/dashboard");
+
+  await expect(page.getByRole("heading", { name: /dashboard/i })).toBeVisible();
+});
+
+test("the session survives a reload", async ({ page }) => {
+  await page.goto("/dashboard");
+  await page.reload();
+
+  // The assertion worth making explicitly: a storageState that restores cookies
+  // but not localStorage often passes the first navigation and fails this one.
+  await expect(page).not.toHaveURL(/\/login/);
+});

package/examples/playwright.config.ts

@@ -0,0 +1,38 @@
+/**
+ * The setup-project layout: capture (or verify) the session once, then every
+ * browser project starts signed in.
+ *
+ * Copy this next to your tests. The two projects matter — `chromium` declares
+ * `dependencies: ["auth-setup"]`, so Playwright runs the setup once per run
+ * rather than once per test file, and a failure there fails fast with a message
+ * that says what to do instead of timing out on a login form.
+ */
+import { defineConfig, devices } from "@playwright/test";
+import { DEFAULT_AUTH_FILE } from "test-google-login";
+
+export default defineConfig({
+  testDir: "./tests/e2e",
+  timeout: 30_000,
+
+  use: {
+    baseURL: process.env.E2E_BASE_URL ?? "http://localhost:3000",
+    trace: "on-first-retry",
+  },
+
+  projects: [
+    {
+      name: "auth-setup",
+      testMatch: /.*\.setup\.ts/,
+    },
+    {
+      name: "chromium",
+      dependencies: ["auth-setup"],
+      use: {
+        ...devices["Desktop Chrome"],
+        // Imported rather than written out, so the config and the CLI cannot
+        // drift onto different paths — which looks exactly like an expired session.
+        storageState: DEFAULT_AUTH_FILE,
+      },
+    },
+  ],
+});

package/examples/test-auth-endpoint.ts

@@ -0,0 +1,58 @@
+/**
+ * The test-only endpoint `bootstrapAppSession` calls — the piece that lives in
+ * *your* app, not in this package.
+ *
+ * Written for a Next.js route handler; the shape is the same anywhere. Four
+ * things are not optional:
+ *
+ * 1. **It must not exist in production.** The 404 below is the floor, not the
+ *    ceiling — prefer excluding the file from the production build entirely, and
+ *    write a deployment test that asserts the route 404s in prod.
+ * 2. **It must require a strong secret**, compared in constant time.
+ * 3. **It must create the same session** a real Google callback creates: same
+ *    format, same claims, same roles, same cookie attributes. A subtly different
+ *    session turns your whole suite into a test of a code path that never ships.
+ * 4. **It must not accept a user id from the request** beyond what you are
+ *    willing to let any caller with the secret create.
+ */
+import { constantTimeEqual } from "test-google-login";
+
+// Stand-ins for your app's own modules.
+declare const db: {
+  user: {
+    upsert(args: Record<string, unknown>): Promise<{ id: string }>;
+  };
+};
+declare function createSessionForUser(userId: string): Promise<{ id: string }>;
+declare function buildSessionCookie(session: { id: string }): string;
+
+export async function POST(request: Request): Promise<Response> {
+  // Defence in depth. The real protection is this file not being in the bundle.
+  if (process.env.NODE_ENV === "production") {
+    return new Response("Not found", { status: 404 });
+  }
+
+  if (!constantTimeEqual(request.headers.get("x-e2e-auth-secret"), process.env.E2E_TEST_AUTH_SECRET)) {
+    return new Response("Unauthorized", { status: 401 });
+  }
+
+  const user = await db.user.upsert({
+    where: { email: "google-e2e@example.test" },
+    create: {
+      email: "google-e2e@example.test",
+      name: "Google E2E User",
+      googleSubject: "google-e2e-subject-001",
+      emailVerified: true,
+    },
+    update: {},
+  });
+
+  const session = await createSessionForUser(user.id);
+
+  return new Response(JSON.stringify({ ok: true }), {
+    headers: {
+      "content-type": "application/json",
+      "set-cookie": buildSessionCookie(session),
+    },
+  });
+}

package/examples/worker-check.ts

@@ -0,0 +1,49 @@
+/**
+ * Driving the Durable Object from a test, so one captured session can be checked
+ * against a deployed environment from Cloudflare's network.
+ *
+ * Worth doing when "does the session work?" depends on where the request comes
+ * from — geo-routed auth, Cloudflare Access in front of the app, a WAF rule that
+ * treats your CI runner's IP differently from a real visitor.
+ */
+import { AUTH_HEADER, loadStorageStateFor } from "test-google-login";
+
+const WORKER = process.env.TGL_WORKER_URL ?? "https://test-google-login.example.workers.dev";
+const SECRET = process.env.TGL_WORKER_SECRET;
+
+if (!SECRET) throw new Error("TGL_WORKER_SECRET is required.");
+
+async function worker(path: string, body?: unknown) {
+  const response = await fetch(`${WORKER}${path}`, {
+    method: body === undefined ? "GET" : "POST",
+    headers: { [AUTH_HEADER]: SECRET!, "content-type": "application/json" },
+    body: body === undefined ? undefined : JSON.stringify(body),
+  });
+
+  if (!response.ok) throw new Error(`${path} → ${response.status}: ${await response.text()}`);
+  return response.json();
+}
+
+// Upload the session captured locally. This is the one request that carries live
+// cookies, so it goes over HTTPS to a Worker that only accepts the shared secret —
+// and there is deliberately no route that reads them back out.
+await worker("/state?session=signed-in", loadStorageStateFor());
+
+const result = (await worker("/check?session=signed-in", {
+  url: "https://app.example.test/dashboard",
+  expectSelector: "[data-testid=user-menu]",
+  rejectUrl: "/login",
+  // Narrow the origins: restoring accounts.google.com storage means navigating a
+  // datacentre browser to Google, which is slow and trips its risk checks.
+  origins: ["https://app.example.test"],
+  screenshot: true,
+})) as { authenticated: boolean; reason: string; screenshot?: string };
+
+console.log(result.authenticated ? `signed in — ${result.reason}` : `NOT signed in — ${result.reason}`);
+
+// GET /state returns a redacted summary, never the values. Safe to log in CI.
+console.log(await worker("/state?session=signed-in"));
+
+// Close the browser rather than waiting for the idle alarm: an open Browser
+// Rendering session is billed.
+await worker("/close?session=signed-in", {});

package/package.json

@@ -0,0 +1,82 @@
+{
+  "name": "automate-google-login-scraper",
+  "version": "0.1.9",
+  "description": "End-to-end test harness for Google sign-in: persist a Playwright storageState once, reuse it everywhere, and replay it from a Cloudflare Workers Browser Rendering Durable Object",
+  "type": "module",
+  "main": "./dist/index.js",
+  "module": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    },
+    "./worker": {
+      "types": "./dist/worker.d.ts",
+      "import": "./dist/worker.js"
+    }
+  },
+  "bin": {
+    "test-google-login": "./bin/cli.js"
+  },
+  "files": [
+    "dist",
+    "bin",
+    "examples"
+  ],
+  "scripts": {
+    "build": "vite build",
+    "prepublishOnly": "bun run build",
+    "typecheck": "tsc --noEmit",
+    "test": "vitest run",
+    "test:watch": "vitest",
+    "test:ci": "vitest run --reporter=default --reporter=junit --outputFile=junit.xml --coverage",
+    "coverage": "vitest run --coverage",
+    "deploy": "wrangler deploy",
+    "dev:worker": "wrangler dev",
+    "ship": "npx standard-version --release-as patch; npm publish"
+  },
+  "keywords": [
+    "playwright",
+    "puppeteer",
+    "google-login",
+    "oauth",
+    "storage-state",
+    "e2e",
+    "testing",
+    "cloudflare-workers",
+    "browser-rendering",
+    "durable-objects"
+  ],
+  "author": "vtempest",
+  "license": "MIT",
+  "devDependencies": {
+    "@cloudflare/puppeteer": "^1.4.0",
+    "@cloudflare/workers-types": "^4.20260507.1",
+    "@types/node": "^22.10.0",
+    "@vitest/coverage-v8": "^4.1.0",
+    "typescript": "^5.9.3",
+    "vite": "^6.3.5",
+    "vite-plugin-dts": "^4.5.4",
+    "vitest": "^4.1.0",
+    "wrangler": "^4.24.0"
+  },
+  "peerDependencies": {
+    "@cloudflare/puppeteer": "*",
+    "@playwright/test": "*"
+  },
+  "peerDependenciesMeta": {
+    "@playwright/test": {
+      "optional": true
+    },
+    "@cloudflare/puppeteer": {
+      "optional": true
+    }
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/OpenSourceAGI/dev-tools-starter-agent.git",
+    "directory": "packages/test-google-login"
+  },
+  "homepage": "https://github.com/OpenSourceAGI/dev-tools-starter-agent/tree/master/packages/test-google-login"
+}