@apitap/core 1.0.0

package/src/auth/refresh.ts

@@ -0,0 +1,300 @@
+// src/auth/refresh.ts
+import type { SkillFile, StoredToken, StoredSession } from '../types.js';
+import type { AuthManager } from './manager.js';
+import { refreshOAuth, type OAuthRefreshResult } from './oauth-refresh.js';
+
+export interface RefreshOptions {
+  domain: string;
+  refreshUrl?: string;
+  browserMode?: 'headless' | 'visible';
+  timeout?: number; // ms, default 30000, extended to 300000 for captcha
+  /** @internal Skip SSRF check — for testing only */
+  _skipSsrfCheck?: boolean;
+}
+
+export interface RefreshResult {
+  success: boolean;
+  tokens: Record<string, string>;
+  captchaDetected?: 'cloudflare' | 'recaptcha' | 'hcaptcha';
+  oauthRefreshed?: boolean;
+  error?: string;
+}
+
+// Mutex to prevent concurrent refreshes for the same domain
+const refreshLocks = new Map<string, Promise<RefreshResult>>();
+
+/**
+ * Extract token values from a request body string.
+ *
+ * @param body - Raw request body (string)
+ * @param tokenNames - JSON paths of tokens to extract (e.g., ["csrf_token", "data.nonce"])
+ * @returns Map of token name to value
+ */
+export function extractTokensFromRequest(
+  body: string,
+  tokenNames: string[]
+): Record<string, string> {
+  const result: Record<string, string> = {};
+
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(body);
+  } catch {
+    return result;
+  }
+
+  if (typeof parsed !== 'object' || parsed === null) {
+    return result;
+  }
+
+  for (const path of tokenNames) {
+    const value = getNestedValue(parsed as Record<string, unknown>, path);
+    if (typeof value === 'string') {
+      result[path] = value;
+    }
+  }
+
+  return result;
+}
+
+function getNestedValue(obj: Record<string, unknown>, path: string): unknown {
+  const parts = path.split('.');
+  let current: unknown = obj;
+
+  for (const part of parts) {
+    if (typeof current !== 'object' || current === null) {
+      return undefined;
+    }
+    current = (current as Record<string, unknown>)[part];
+  }
+
+  return current;
+}
+
+/**
+ * Detect captcha challenge in page content.
+ */
+export function detectCaptcha(
+  html: string
+): 'cloudflare' | 'recaptcha' | 'hcaptcha' | null {
+  // Cloudflare challenge
+  if (
+    html.includes('Just a moment...') ||
+    html.includes('cdn-cgi/challenge-platform') ||
+    html.includes('cf-browser-verification')
+  ) {
+    return 'cloudflare';
+  }
+
+  // reCAPTCHA
+  if (
+    html.includes('g-recaptcha') ||
+    html.includes('google.com/recaptcha')
+  ) {
+    return 'recaptcha';
+  }
+
+  // hCaptcha
+  if (
+    html.includes('h-captcha') ||
+    html.includes('hcaptcha.com')
+  ) {
+    return 'hcaptcha';
+  }
+
+  return null;
+}
+
+/**
+ * Refresh tokens by spawning a browser and intercepting requests.
+ *
+ * This is the main entry point for token refresh. It:
+ * 1. Launches a browser (visible if captchaRisk is true)
+ * 2. Navigates to the refresh URL
+ * 3. Intercepts outgoing requests to capture fresh token values
+ * 4. Returns the captured tokens
+ *
+ * @param skill - Skill file with auth config and endpoint info
+ * @param authManager - Auth manager for storing refreshed tokens
+ * @param options - Refresh options
+ */
+export async function refreshTokens(
+  skill: SkillFile,
+  authManager: AuthManager,
+  options: RefreshOptions
+): Promise<RefreshResult> {
+  const { domain } = options;
+
+  // Check for existing refresh in progress (mutex)
+  const existingRefresh = refreshLocks.get(domain);
+  if (existingRefresh) {
+    return existingRefresh;
+  }
+
+  const refreshPromise = doRefresh(skill, authManager, options);
+  refreshLocks.set(domain, refreshPromise);
+
+  try {
+    return await refreshPromise;
+  } finally {
+    refreshLocks.delete(domain);
+  }
+}
+
+async function doRefresh(
+  skill: SkillFile,
+  authManager: AuthManager,
+  options: RefreshOptions
+): Promise<RefreshResult> {
+  let oauthRefreshed = false;
+
+  // Step 1: OAuth path — if oauthConfig + stored credentials available
+  const oauthConfig = skill.auth?.oauthConfig;
+  if (oauthConfig) {
+    const oauthCreds = await authManager.retrieveOAuthCredentials(options.domain);
+    const canOAuth =
+      (oauthConfig.grantType === 'refresh_token' && oauthCreds?.refreshToken) ||
+      (oauthConfig.grantType === 'client_credentials');
+
+    if (canOAuth) {
+      const oauthResult = await refreshOAuth(options.domain, oauthConfig, authManager, { _skipSsrfCheck: options._skipSsrfCheck });
+      if (oauthResult.success) {
+        oauthRefreshed = true;
+      } else {
+        // OAuth failed — fall through to browser path if available
+      }
+    }
+  }
+
+  // Step 2: Browser path — if refreshable tokens exist or refreshUrl is present
+  const tokenNames = new Set<string>();
+  for (const endpoint of skill.endpoints) {
+    if (endpoint.requestBody?.refreshableTokens) {
+      for (const name of endpoint.requestBody.refreshableTokens) {
+        tokenNames.add(name);
+      }
+    }
+  }
+
+  const needsBrowser = tokenNames.size > 0 || (skill.auth?.refreshUrl && !oauthRefreshed);
+
+  if (!needsBrowser) {
+    // No browser refresh needed — return OAuth result
+    return { success: oauthRefreshed, tokens: {}, oauthRefreshed: oauthRefreshed || undefined };
+  }
+
+  return doBrowserRefresh(skill, authManager, options, tokenNames, oauthRefreshed);
+}
+
+async function doBrowserRefresh(
+  skill: SkillFile,
+  authManager: AuthManager,
+  options: RefreshOptions,
+  tokenNames: Set<string>,
+  oauthRefreshed: boolean
+): Promise<RefreshResult> {
+  if (tokenNames.size === 0 && !skill.auth?.refreshUrl) {
+    return { success: oauthRefreshed, tokens: {}, oauthRefreshed: oauthRefreshed || undefined };
+  }
+
+  const { chromium } = await import('playwright');
+
+  const browserMode = options.browserMode || skill.auth?.browserMode || 'headless';
+  const refreshUrl = options.refreshUrl || skill.auth?.refreshUrl || skill.baseUrl;
+  const timeout = options.timeout || (skill.auth?.captchaRisk ? 300_000 : 30_000);
+
+  // Try to restore session from cache
+  const cachedSession = await authManager.retrieveSession(options.domain);
+  const sessionValid = cachedSession && isSessionValid(cachedSession);
+
+  const browser = await chromium.launch({
+    headless: browserMode === 'headless',
+  });
+
+  try {
+    const context = await browser.newContext();
+
+    // Restore cookies if session is valid
+    if (sessionValid && cachedSession) {
+      await context.addCookies(cachedSession.cookies);
+    }
+
+    const page = await context.newPage();
+    const capturedTokens: Record<string, string> = {};
+    let captchaDetected: 'cloudflare' | 'recaptcha' | 'hcaptcha' | null = null;
+
+    // Intercept requests to capture token values
+    if (tokenNames.size > 0) {
+      page.on('request', (request) => {
+        const body = request.postData();
+        if (body) {
+          const extracted = extractTokensFromRequest(body, [...tokenNames]);
+          Object.assign(capturedTokens, extracted);
+        }
+      });
+    }
+
+    // Navigate and wait for network idle
+    await page.goto(refreshUrl, { waitUntil: 'networkidle', timeout });
+
+    // Check for captcha
+    const content = await page.content();
+    captchaDetected = detectCaptcha(content);
+
+    if (captchaDetected) {
+      // Extended timeout for captcha solving
+      console.error(`\u26a0\ufe0f  Captcha detected (${captchaDetected}). Please solve it in the browser window.`);
+      await page.waitForTimeout(timeout);
+
+      // Re-check for tokens after captcha
+      // User interaction will trigger requests containing tokens
+    }
+
+    // Wait a bit for any final requests
+    await page.waitForTimeout(2000);
+
+    // Save session for next time
+    const cookies = await context.cookies();
+    await authManager.storeSession(options.domain, {
+      cookies,
+      savedAt: new Date().toISOString(),
+      maxAgeMs: 24 * 60 * 60 * 1000, // 24 hours
+    });
+
+    // Store captured tokens
+    if (Object.keys(capturedTokens).length > 0) {
+      const storedTokens: Record<string, StoredToken> = {};
+      for (const [name, value] of Object.entries(capturedTokens)) {
+        storedTokens[name] = {
+          value,
+          refreshedAt: new Date().toISOString(),
+        };
+      }
+      await authManager.storeTokens(options.domain, storedTokens);
+    }
+
+    const browserSuccess = tokenNames.size === 0 || Object.keys(capturedTokens).length > 0;
+
+    return {
+      success: oauthRefreshed || browserSuccess,
+      tokens: capturedTokens,
+      captchaDetected: captchaDetected || undefined,
+      oauthRefreshed: oauthRefreshed || undefined,
+    };
+  } catch (error) {
+    return {
+      success: oauthRefreshed, // OAuth may have succeeded even if browser failed
+      tokens: {},
+      error: error instanceof Error ? error.message : String(error),
+      oauthRefreshed: oauthRefreshed || undefined,
+    };
+  } finally {
+    await browser.close();
+  }
+}
+
+function isSessionValid(session: StoredSession): boolean {
+  const maxAge = session.maxAgeMs || 24 * 60 * 60 * 1000;
+  const savedAt = new Date(session.savedAt).getTime();
+  return Date.now() - savedAt < maxAge;
+}

package/src/capture/anti-bot.ts

@@ -0,0 +1,63 @@
+// src/capture/anti-bot.ts
+
+export type AntiBotSignal = 'cloudflare' | 'akamai' | 'rate-limited' | 'captcha' | 'challenge';
+
+export interface AntiBotResult {
+  detected: boolean;
+  signals: AntiBotSignal[];
+}
+
+/**
+ * Detect anti-bot protection signals from response headers and body.
+ */
+export function detectAntiBot(options: {
+  headers: Record<string, string>;
+  cookies?: string;
+  body?: string;
+  status?: number;
+  contentType?: string;
+}): AntiBotResult {
+  const signals: AntiBotSignal[] = [];
+  const { headers, cookies, body, status, contentType } = options;
+  const headerLower = lowerKeys(headers);
+
+  // Cloudflare: cf-ray header or __cf_bm cookie
+  if (headerLower['cf-ray'] || headerLower['cf-cache-status'] || cookies?.includes('__cf_bm')) {
+    signals.push('cloudflare');
+  }
+
+  // Akamai: _abck cookie
+  if (cookies?.includes('_abck')) {
+    signals.push('akamai');
+  }
+
+  // Rate limiting: X-RateLimit-* or Retry-After
+  if (headerLower['retry-after'] ||
+      Object.keys(headerLower).some(k => k.startsWith('x-ratelimit'))) {
+    signals.push('rate-limited');
+  }
+
+  // CAPTCHA in response body
+  if (body && /\b(captcha|hcaptcha|recaptcha|cf-turnstile)\b/i.test(body)) {
+    signals.push('captcha');
+  }
+
+  // Challenge page: HTML response when JSON expected + 403
+  if (status === 403 && contentType?.includes('text/html') &&
+      !contentType.includes('json')) {
+    signals.push('challenge');
+  }
+
+  return {
+    detected: signals.length > 0,
+    signals: [...new Set(signals)],
+  };
+}
+
+function lowerKeys(obj: Record<string, string>): Record<string, string> {
+  const result: Record<string, string> = {};
+  for (const [k, v] of Object.entries(obj)) {
+    result[k.toLowerCase()] = v;
+  }
+  return result;
+}

package/src/capture/blocklist.ts

@@ -0,0 +1,75 @@
+// src/capture/blocklist.ts
+
+const BLOCKLIST = new Set([
+  // Analytics
+  'google-analytics.com',
+  'analytics.google.com',
+  'googletagmanager.com',
+  'segment.io',
+  'cdn.segment.com',
+  'mixpanel.com',
+  'amplitude.com',
+  'hotjar.com',
+  'heapanalytics.com',
+  'plausible.io',
+  'posthog.com',
+  'clarity.ms',
+  'fullstory.com',
+
+  // Ads
+  'doubleclick.net',
+  'googlesyndication.com',
+  'googleadservices.com',
+  'facebook.net',
+  'connect.facebook.net',
+  'adsrvr.org',
+  'adnxs.com',
+  'criteo.com',
+  'outbrain.com',
+  'taboola.com',
+
+  // Error tracking / monitoring
+  'sentry.io',
+  'datadoghq.com',
+  'browser-intake-datadoghq.com',
+  'newrelic.com',
+  'bam.nr-data.net',
+  'logrocket.com',
+  'logr-ingest.com',
+  'bugsnag.com',
+  'rollbar.com',
+
+  // Social tracking
+  'bat.bing.com',
+  'ct.pinterest.com',
+  'snap.licdn.com',
+  'px.ads.linkedin.com',
+  'analytics.twitter.com',
+  'analytics.tiktok.com',
+
+  // Customer engagement
+  'intercom.io',
+  'widget.intercom.io',
+  'api-iam.intercom.io',
+  'zendesk.com',
+  'drift.com',
+  'crisp.chat',
+]);
+
+/**
+ * Check if a hostname is on the blocklist.
+ * Matches exact hostnames and subdomains of blocklisted domains.
+ * e.g. "sentry.io" blocks "o123.ingest.sentry.io"
+ */
+export function isBlocklisted(hostname: string): boolean {
+  if (BLOCKLIST.has(hostname)) return true;
+
+  // Check parent domains: "a.b.sentry.io" → "b.sentry.io" → "sentry.io"
+  const parts = hostname.split('.');
+  for (let i = 1; i < parts.length - 1; i++) {
+    const parent = parts.slice(i).join('.');
+    if (BLOCKLIST.has(parent)) return true;
+  }
+
+  return false;
+}

package/src/capture/body-diff.ts

@@ -0,0 +1,109 @@
+// src/capture/body-diff.ts
+
+/**
+ * Cross-request body diffing (Strategy 1).
+ *
+ * Compare request bodies across multiple captures of the same endpoint.
+ * Any field whose value changed between requests is dynamic by definition.
+ * Returns JSON paths of changed fields.
+ */
+export function diffBodies(bodies: string[]): string[] {
+  if (bodies.length < 2) return [];
+
+  // Try JSON first
+  const parsed: unknown[] = [];
+  for (const body of bodies) {
+    try {
+      parsed.push(JSON.parse(body));
+    } catch {
+      // Fall back to form-encoded diffing
+      return diffFormEncoded(bodies);
+    }
+  }
+
+  // All parsed as JSON — diff objects
+  const changed = new Set<string>();
+  for (let i = 1; i < parsed.length; i++) {
+    diffObjects(parsed[0], parsed[i], '', changed);
+  }
+  return [...changed].sort();
+}
+
+function diffObjects(a: unknown, b: unknown, prefix: string, changed: Set<string>): void {
+  // Different types → the whole path is dynamic
+  if (typeof a !== typeof b || Array.isArray(a) !== Array.isArray(b)) {
+    if (prefix) changed.add(prefix);
+    return;
+  }
+
+  // Both arrays
+  if (Array.isArray(a) && Array.isArray(b)) {
+    if (a.length !== b.length) {
+      // Different lengths → mark whole array as dynamic
+      if (prefix) changed.add(prefix);
+      return;
+    }
+    for (let i = 0; i < a.length; i++) {
+      diffObjects(a[i], b[i], prefix ? `${prefix}[${i}]` : `[${i}]`, changed);
+    }
+    return;
+  }
+
+  // Both objects
+  if (a && b && typeof a === 'object' && typeof b === 'object' && !Array.isArray(a)) {
+    const aObj = a as Record<string, unknown>;
+    const bObj = b as Record<string, unknown>;
+    const allKeys = new Set([...Object.keys(aObj), ...Object.keys(bObj)]);
+
+    for (const key of allKeys) {
+      const path = prefix ? `${prefix}.${key}` : key;
+
+      if (!(key in aObj) || !(key in bObj)) {
+        // Key only exists in one → dynamic
+        changed.add(path);
+      } else {
+        diffObjects(aObj[key], bObj[key], path, changed);
+      }
+    }
+    return;
+  }
+
+  // Primitive comparison
+  if (a !== b && prefix) {
+    changed.add(prefix);
+  }
+}
+
+function diffFormEncoded(bodies: string[]): string[] {
+  const parsed = bodies.map(parseFormEncoded);
+  if (parsed.some(m => m.size === 0)) return [];
+
+  const changed = new Set<string>();
+  const firstMap = parsed[0];
+
+  for (let i = 1; i < parsed.length; i++) {
+    const otherMap = parsed[i];
+    const allKeys = new Set([...firstMap.keys(), ...otherMap.keys()]);
+
+    for (const key of allKeys) {
+      if (firstMap.get(key) !== otherMap.get(key)) {
+        changed.add(key);
+      }
+    }
+  }
+
+  return [...changed].sort();
+}
+
+function parseFormEncoded(body: string): Map<string, string> {
+  const map = new Map<string, string>();
+  const pairs = body.split('&');
+  for (const pair of pairs) {
+    const idx = pair.indexOf('=');
+    if (idx === -1) continue;
+    const key = decodeURIComponent(pair.slice(0, idx).replace(/\+/g, ' '));
+    const val = decodeURIComponent(pair.slice(idx + 1).replace(/\+/g, ' '));
+    map.set(key, val);
+  }
+  return map;
+}

package/src/capture/body-variables.ts

@@ -0,0 +1,156 @@
+// src/capture/body-variables.ts
+
+// Strategy 2: Name-based heuristics — keys implying dynamic values
+const DYNAMIC_KEY_PATTERNS = [
+  // Time
+  /timestamp/i, /\btime\b/i, /\bdate\b/i, /created[_-]?at/i, /updated[_-]?at/i,
+  /\bsince\b/i, /\buntil\b/i, /\bbefore\b/i, /\bafter\b/i, /\bexpires?\b/i,
+  // Pagination
+  /\bcursor\b/i, /\boffset\b/i, /\bpage\b/i, /page[_-]?number/i,
+  /next[_-]?token/i, /continuation/i,
+  // Identity
+  /request[_-]?id/i, /correlation[_-]?id/i, /trace[_-]?id/i,
+  /\bnonce\b/i, /idempotency[_-]?key/i,
+  // Session
+  /session[_-]?id/i, /\bcsrf\b/i, /\bxsrf\b/i,
+  // Geolocation
+  /\bgeo(code|loc(ation)?)\b/i, /\blat(itude)?\b/i, /\blo?ng(itude)?\b/i,
+  /\bcoord/i, /\bzip\b/i, /\bpostal/i,
+  // Search / user input
+  /\bquery\b/i, /\bsearch/i, /\bkeyword/i, /\bterm\b/i, /\bfilter\b/i,
+];
+
+function isDynamicKeyName(key: string): boolean {
+  return DYNAMIC_KEY_PATTERNS.some(p => p.test(key));
+}
+
+// Strategy 3: Pattern-based detection — value patterns implying dynamic data
+function isTimestampOrPattern(value: string | number): boolean {
+  if (typeof value === 'number') {
+    // Unix epoch seconds (roughly 2001–2603)
+    if (Number.isInteger(value) && value >= 1e9 && value < 2e10) return true;
+    // Unix epoch milliseconds
+    if (Number.isInteger(value) && value >= 1e12 && value < 2e13) return true;
+    return false;
+  }
+
+  // ISO 8601 datetime
+  if (/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}/.test(value)) return true;
+  // ISO 8601 date only
+  if (/^\d{4}-\d{2}-\d{2}$/.test(value)) return true;
+  // Prefixed IDs (req_xxx, id_xxx, txn_xxx, msg_xxx, evt_xxx)
+  if (/^(req|id|txn|msg|evt)_[a-zA-Z0-9]+$/.test(value)) return true;
+
+  return false;
+}
+
+/**
+ * Detect which fields in a JSON body are likely dynamic variables.
+ * Uses three strategies:
+ *   Strategy 1 (cross-request diffing) is in body-diff.ts
+ *   Strategy 2: Name-based key heuristics
+ *   Strategy 3: Pattern-based value detection
+ *   Plus existing: numeric values, UUIDs, base64 cursors, numeric strings
+ */
+export function detectBodyVariables(
+  body: unknown,
+  prefix = '',
+): string[] {
+  if (!body || typeof body !== 'object' || Array.isArray(body)) {
+    return [];
+  }
+
+  const detected: string[] = [];
+  const obj = body as Record<string, unknown>;
+
+  for (const [key, value] of Object.entries(obj)) {
+    const path = prefix ? `${prefix}.${key}` : key;
+
+    // Strategy 2: key name implies dynamic value
+    if (isDynamicKeyName(key) && value != null) {
+      detected.push(path);
+      continue;
+    }
+
+    if (typeof value === 'number') {
+      // Strategy 3: epoch timestamp detection
+      if (isTimestampOrPattern(value)) {
+        detected.push(path);
+      } else {
+        // Original: numeric values are often IDs
+        detected.push(path);
+      }
+    } else if (typeof value === 'string') {
+      // Strategy 3: timestamp/pattern detection (checked first, catches more)
+      if (isTimestampOrPattern(value)) {
+        detected.push(path);
+      } else if (isLikelyDynamic(value)) {
+        detected.push(path);
+      }
+    } else if (value && typeof value === 'object' && !Array.isArray(value)) {
+      // Recurse into nested objects
+      detected.push(...detectBodyVariables(value, path));
+    }
+  }
+
+  return detected;
+}
+
+function isLikelyDynamic(value: string): boolean {
+  // UUID
+  if (/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(value)) {
+    return true;
+  }
+  // Base64-ish cursor (long alphanumeric with optional padding)
+  if (value.length > 15 && /^[a-zA-Z0-9+/=_-]+$/.test(value)) {
+    return true;
+  }
+  // Numeric string (ID)
+  if (/^\d{4,}$/.test(value)) {
+    return true;
+  }
+  return false;
+}
+
+/**
+ * Substitute variables in a body template.
+ */
+export function substituteBodyVariables(
+  template: string | Record<string, unknown>,
+  values: Record<string, string>,
+): string | Record<string, unknown> {
+  if (typeof template === 'string') {
+    // String template with :param placeholders
+    return template.replace(/:([a-zA-Z_][a-zA-Z0-9_]*)/g, (match, name) => {
+      return values[name] ?? match;
+    });
+  }
+
+  // Deep clone and substitute
+  const result = JSON.parse(JSON.stringify(template)) as Record<string, unknown>;
+
+  for (const [path, value] of Object.entries(values)) {
+    setNestedValue(result, path, value);
+  }
+
+  return result;
+}
+
+function setNestedValue(obj: Record<string, unknown>, path: string, value: string): void {
+  const parts = path.split('.');
+  let current = obj;
+
+  for (let i = 0; i < parts.length - 1; i++) {
+    const part = parts[i];
+    if (current[part] && typeof current[part] === 'object') {
+      current = current[part] as Record<string, unknown>;
+    } else {
+      return; // Path doesn't exist
+    }
+  }
+
+  const lastPart = parts[parts.length - 1];
+  if (lastPart in current) {
+    current[lastPart] = value;
+  }
+}