@apitap/core 1.0.7 → 1.0.9

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@apitap/core",
-  "version": "1.0.7",
+  "version": "1.0.9",
   "description": "Intercept web API traffic during browsing. Generate portable skill files so AI agents can call APIs directly instead of scraping.",
   "type": "module",
   "main": "dist/index.js",

package/src/auth/handoff.ts

@@ -1,6 +1,7 @@
 // src/auth/handoff.ts
 import type { AuthManager } from './manager.js';
 import type { StoredSession, StoredAuth } from '../types.js';
+import { launchBrowser } from '../capture/browser.js';
 
 export interface HandoffOptions {
   domain: string;
@@ -98,15 +99,12 @@ async function doHandoff(
   const loginUrl = options.loginUrl || `https://${domain}`;
   const timeout = options.timeout ?? 300_000; // 5 minutes
 
-  const { chromium } = await import('playwright');
-
-  const browser = await chromium.launch({ headless: false });
+  const { browser, context } = await launchBrowser({ headless: false });
 
   try {
-    const context = await browser.newContext();
 
     // Restore existing session cookies if available (warm start)
-    const cachedSession = await authManager.retrieveSession(domain);
+    const cachedSession = await authManager.retrieveSessionWithFallback(domain);
     if (cachedSession?.cookies?.length) {
       await context.addCookies(cachedSession.cookies);
     }
@@ -161,8 +159,13 @@ async function doHandoff(
       );
 
       if (hasSessionCookie || authDetected) {
-        // Wait a bit more for any final redirects/requests
-        await page.waitForTimeout(2000);
+        // Grace period: 4 additional polls at 2s each (~8s total)
+        // Allows time for MFA, CAPTCHAs, and post-login redirects
+        for (let grace = 0; grace < 4; grace++) {
+          if (page.isClosed()) break;
+          await page.waitForLoadState('networkidle', { timeout: 3000 }).catch(() => {});
+          await page.waitForTimeout(2000);
+        }
         loginDetected = true;
         break;
       }

package/src/auth/manager.ts

@@ -66,6 +66,25 @@ export class AuthManager {
     return all[domain]?.session ?? null;
   }
 
+  /**
+   * Retrieve session with subdomain fallback.
+   * Tries exact match first, then walks up parent domains.
+   * e.g., dashboard.twitch.tv → twitch.tv
+   */
+  async retrieveSessionWithFallback(domain: string): Promise<StoredSession | null> {
+    // Try exact match first
+    const exact = await this.retrieveSession(domain);
+    if (exact) return exact;
+
+    // Try parent domains
+    for (const parent of getParentDomains(domain)) {
+      const session = await this.retrieveSession(parent);
+      if (session) return session;
+    }
+
+    return null;
+  }
+
   /** Store OAuth credentials for a domain (merges with existing auth). */
   async storeOAuthCredentials(domain: string, creds: { refreshToken?: string; clientSecret?: string }): Promise<void> {
     const all = await this.loadAll();
@@ -122,6 +141,24 @@ export class AuthManager {
   }
 }
 
+/**
+ * Get parent domains for subdomain fallback.
+ * dashboard.twitch.tv → ["twitch.tv"]
+ * a.b.example.com → ["b.example.com", "example.com"]
+ * twitch.tv → [] (already base, 2 labels)
+ */
+export function getParentDomains(domain: string): string[] {
+  const parts = domain.split('.');
+  const parents: string[] = [];
+
+  // Stop at 2 labels (e.g., "example.com" is the minimum)
+  for (let i = 1; i < parts.length - 1; i++) {
+    parents.push(parts.slice(i).join('.'));
+  }
+
+  return parents;
+}
+
 /**
  * Get the machine ID for key derivation.
  * Linux: /etc/machine-id

package/src/auth/oauth-refresh.ts

@@ -56,9 +56,11 @@ export async function refreshOAuth(
     'oauth2.googleapis.com', 'accounts.google.com',
     'login.microsoftonline.com', 'github.com',
     'oauth.reddit.com', 'api.twitter.com',
+    'auth0.com', 'okta.com',
+    'securetoken.googleapis.com',
   ];
   const tokenHost = new URL(oauthConfig.tokenEndpoint).hostname;
-  if (tokenHost !== domain && !tokenHost.endsWith('.' + domain) && !KNOWN_OAUTH_HOSTS.includes(tokenHost)) {
+  if (tokenHost !== domain && !tokenHost.endsWith('.' + domain) && !isKnownOAuthHost(tokenHost, KNOWN_OAUTH_HOSTS)) {
     return { success: false, error: `Token endpoint domain mismatch: ${tokenHost} vs ${domain}` };
   }
 
@@ -118,3 +120,13 @@ export async function refreshOAuth(
     };
   }
 }
+
+/**
+ * Check if a hostname matches a known OAuth provider.
+ * Supports exact match and subdomain match (e.g., tenant.auth0.com matches auth0.com).
+ */
+function isKnownOAuthHost(tokenHost: string, knownHosts: string[]): boolean {
+  return knownHosts.some(known =>
+    tokenHost === known || tokenHost.endsWith('.' + known)
+  );
+}

package/src/auth/refresh.ts

@@ -2,6 +2,7 @@
 import type { SkillFile, StoredToken, StoredSession } from '../types.js';
 import type { AuthManager } from './manager.js';
 import { refreshOAuth, type OAuthRefreshResult } from './oauth-refresh.js';
+import { launchBrowser } from '../capture/browser.js';
 
 export interface RefreshOptions {
   domain: string;
@@ -197,22 +198,19 @@ async function doBrowserRefresh(
     return { success: oauthRefreshed, tokens: {}, oauthRefreshed: oauthRefreshed || undefined };
   }
 
-  const { chromium } = await import('playwright');
-
   const browserMode = options.browserMode || skill.auth?.browserMode || 'headless';
   const refreshUrl = options.refreshUrl || skill.auth?.refreshUrl || skill.baseUrl;
   const timeout = options.timeout || (skill.auth?.captchaRisk ? 300_000 : 30_000);
 
   // Try to restore session from cache
-  const cachedSession = await authManager.retrieveSession(options.domain);
+  const cachedSession = await authManager.retrieveSessionWithFallback(options.domain);
   const sessionValid = cachedSession && isSessionValid(cachedSession);
 
-  const browser = await chromium.launch({
+  const { browser, context } = await launchBrowser({
     headless: browserMode === 'headless',
   });
 
   try {
-    const context = await browser.newContext();
 
     // Restore cookies if session is valid
     if (sessionValid && cachedSession) {

package/src/capture/browser.ts

@@ -0,0 +1,51 @@
+// src/capture/browser.ts
+import type { Browser, BrowserContext } from 'playwright';
+
+const CHROME_USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36';
+
+/**
+ * Launch args that reduce Playwright's automation fingerprint.
+ */
+export function getLaunchArgs(): string[] {
+  return [
+    '--disable-blink-features=AutomationControlled',
+  ];
+}
+
+/**
+ * Realistic Chrome user-agent string for anti-detection.
+ */
+export function getChromeUserAgent(): string {
+  return CHROME_USER_AGENT;
+}
+
+/**
+ * Launch a Chromium browser with anti-detection measures.
+ *
+ * Three layers:
+ * 1. --disable-blink-features=AutomationControlled in launch args
+ * 2. Realistic Chrome UA on context
+ * 3. navigator.webdriver = false via addInitScript
+ * 4. Viewport 1920x1080
+ */
+export async function launchBrowser(options: { headless: boolean }): Promise<{ browser: Browser; context: BrowserContext }> {
+  const { chromium } = await import('playwright');
+
+  const browser = await chromium.launch({
+    headless: options.headless,
+    args: getLaunchArgs(),
+  });
+
+  const context = await browser.newContext({
+    userAgent: CHROME_USER_AGENT,
+    viewport: { width: 1920, height: 1080 },
+  });
+
+  await context.addInitScript(() => {
+    Object.defineProperty(navigator, 'webdriver', {
+      get: () => false,
+    });
+  });
+
+  return { browser, context };
+}

package/src/capture/monitor.ts

@@ -5,6 +5,7 @@ import { isDomainMatch } from './domain.js';
 import { SkillGenerator, type GeneratorOptions } from '../skill/generator.js';
 import { IdleTracker } from './idle.js';
 import { detectCaptcha } from '../auth/refresh.js';
+import { launchBrowser } from './browser.js';
 import type { CapturedExchange } from '../types.js';
 
 export interface CaptureOptions {
@@ -31,7 +32,7 @@ export interface CaptureResult {
 
 const DEFAULT_CDP_PORTS = [18792, 18800, 9222];
 
-async function connectToBrowser(options: CaptureOptions): Promise<{ browser: Browser; launched: boolean }> {
+async function connectToBrowser(options: CaptureOptions): Promise<{ browser: Browser; launched: boolean; launchContext?: import('playwright').BrowserContext }> {
   if (!options.launch) {
     const ports = options.port ? [options.port] : DEFAULT_CDP_PORTS;
     for (const port of ports) {
@@ -49,12 +50,12 @@ async function connectToBrowser(options: CaptureOptions): Promise<{ browser: Bro
     throw new Error(`No browser found on CDP ports: ${ports.join(', ')}. Is a Chromium browser running with remote debugging?`);
   }
 
-  const browser = await chromium.launch({ headless: options.headless ?? (process.env.DISPLAY ? false : true) });
-  return { browser, launched: true };
+  const { browser, context } = await launchBrowser({ headless: options.headless ?? (process.env.DISPLAY ? false : true) });
+  return { browser, launched: true, launchContext: context };
 }
 
 export async function capture(options: CaptureOptions): Promise<CaptureResult> {
-  const { browser, launched } = await connectToBrowser(options);
+  const { browser, launched, launchContext } = await connectToBrowser(options);
   const generators = new Map<string, SkillGenerator>();
   let totalRequests = 0;
   let filteredRequests = 0;
@@ -73,7 +74,10 @@ export async function capture(options: CaptureOptions): Promise<CaptureResult> {
   let idleInterval: ReturnType<typeof setInterval> | null = null;
 
   let page: Page;
-  if (launched) {
+  if (launched && launchContext) {
+    page = await launchContext.newPage();
+  } else if (launched) {
+    // Fallback: shouldn't happen, but handle gracefully
     const context = await browser.newContext();
     page = await context.newPage();
   } else {

package/src/capture/oauth-detector.ts

@@ -27,18 +27,48 @@ export function isOAuthTokenRequest(req: {
   const urlLower = req.url.toLowerCase();
   if (!urlLower.includes('/token') && !urlLower.includes('/oauth')) return null;
 
+  // Parse URL once — reused for query param fallback and Firebase detection
+  let parsedUrl: URL;
+  try {
+    parsedUrl = new URL(req.url);
+  } catch {
+    return null;
+  }
+
   if (!req.postData) return null;
 
   // Parse body — support URL-encoded and JSON
   const params = parseBody(req.postData, req.headers['content-type'] ?? '');
   if (!params) return null;
 
-  const grantType = params.get('grant_type');
+  // grant_type: body takes precedence, URL query param as fallback (Supabase GoTrue)
+  let grantType = params.get('grant_type');
+  if (!grantType) {
+    grantType = parsedUrl.searchParams.get('grant_type') ?? undefined;
+  }
   if (!grantType) return null;
 
   // Only refreshable flows
   if (grantType !== 'refresh_token' && grantType !== 'client_credentials') return null;
 
+  // Firebase provider-specific detection:
+  // securetoken.googleapis.com uses ?key= instead of client_id
+  if (
+    parsedUrl.hostname === 'securetoken.googleapis.com' &&
+    parsedUrl.searchParams.has('key') &&
+    grantType === 'refresh_token'
+  ) {
+    const firebaseKey = parsedUrl.searchParams.get('key')!;
+    const result: OAuthInfo = {
+      tokenEndpoint: req.url, // Keep full URL with ?key= param
+      clientId: firebaseKey,
+      grantType: 'refresh_token',
+    };
+    const refreshToken = params.get('refresh_token');
+    if (refreshToken) result.refreshToken = refreshToken;
+    return result;
+  }
+
   // Extract client_id — may also be in Basic auth header
   let clientId = params.get('client_id') ?? '';
   let clientSecret = params.get('client_secret');

package/src/capture/session.ts

@@ -1,7 +1,8 @@
 // src/capture/session.ts
-import { chromium, type Browser, type Page } from 'playwright';
+import { type Browser, type Page } from 'playwright';
 import { randomUUID } from 'node:crypto';
 import { shouldCapture } from './filter.js';
+import { launchBrowser } from './browser.js';
 import { isDomainMatch } from './domain.js';
 import { SkillGenerator, type GeneratorOptions } from '../skill/generator.js';
 import { detectCaptcha } from '../auth/refresh.js';
@@ -53,8 +54,8 @@ export class CaptureSession {
     this.targetUrl = url.startsWith('http') ? url : `https://${url}`;
     const headless = this.options.headless ?? true;
 
-    this.browser = await chromium.launch({ headless });
-    const context = await this.browser.newContext();
+    const { browser, context } = await launchBrowser({ headless });
+    this.browser = browser;
 
     // Inject cached session cookies if available
     try {
@@ -62,7 +63,7 @@ export class CaptureSession {
       const machineId = await getMachineId();
       const authManager = new AuthManager(authDir, machineId);
       const domain = new URL(this.targetUrl).hostname;
-      const cachedSession = await authManager.retrieveSession(domain);
+      const cachedSession = await authManager.retrieveSessionWithFallback(domain);
       if (cachedSession?.cookies?.length) {
         await context.addCookies(cachedSession.cookies);
       }
@@ -74,13 +75,8 @@ export class CaptureSession {
 
     this.setupResponseListener();
 
-    // Auto-timeout to prevent leaked browsers (unref so it doesn't block process exit)
-    const timeoutMs = this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS;
-    this.timeoutTimer = setTimeout(() => {
-      this.expired = true;
-      this.cleanup().catch(() => {});
-    }, timeoutMs);
-    if (this.timeoutTimer.unref) this.timeoutTimer.unref();
+    // Auto-timeout to prevent leaked browsers (resets on each interaction)
+    this.resetTimeout();
 
     await this.page.goto(this.targetUrl, { waitUntil: 'domcontentloaded' });
     return this.takeSnapshot();
@@ -91,6 +87,9 @@ export class CaptureSession {
     if (this.closed) return { success: false, error: 'Session closed', snapshot: this.emptySnapshot() };
     if (!this.page) return { success: false, error: 'Session not started', snapshot: this.emptySnapshot() };
 
+    // Reset timeout on each interaction — active sessions aren't leaked browsers
+    this.resetTimeout();
+
     try {
       switch (action.action) {
         case 'snapshot':
@@ -467,6 +466,17 @@ export class CaptureSession {
     };
   }
 
+  /** Reset the auto-timeout timer (called on each interaction to prevent premature close) */
+  private resetTimeout(): void {
+    if (this.timeoutTimer) clearTimeout(this.timeoutTimer);
+    const timeoutMs = this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS;
+    this.timeoutTimer = setTimeout(() => {
+      this.expired = true;
+      this.cleanup().catch(() => {});
+    }, timeoutMs);
+    if (this.timeoutTimer.unref) this.timeoutTimer.unref();
+  }
+
   private async cleanup(): Promise<void> {
     if (this.closed) return;
     this.closed = true;