@apitap/core 1.0.0

package/src/capture/domain.ts

@@ -0,0 +1,34 @@
+// src/capture/domain.ts
+
+/**
+ * Check if a hostname matches the target domain.
+ * Uses dot-prefix matching to prevent evil-example.com matching example.com.
+ *
+ * @param hostname - The hostname to check (e.g. "api.example.com")
+ * @param target - The target domain or URL (e.g. "example.com" or "https://example.com/path")
+ */
+export function isDomainMatch(hostname: string, target: string): boolean {
+  // Extract hostname from URL if target looks like a URL
+  let targetHost: string;
+  try {
+    if (target.includes('://')) {
+      targetHost = new URL(target).hostname;
+    } else {
+      targetHost = target;
+    }
+  } catch {
+    targetHost = target;
+  }
+
+  // Strip www. prefix from target for broader matching
+  if (targetHost.startsWith('www.')) {
+    targetHost = targetHost.slice(4);
+  }
+
+  // Exact match
+  if (hostname === targetHost) return true;
+
+  // Dot-prefix suffix match: hostname must end with ".targetHost"
+  // This prevents evil-example.com from matching example.com
+  return hostname.endsWith('.' + targetHost);
+}

package/src/capture/entropy.ts

@@ -0,0 +1,121 @@
+// src/capture/entropy.ts
+
+const MIN_TOKEN_LENGTH = 16;
+const UUID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
+
+export interface TokenClassification {
+  isToken: boolean;
+  confidence: 'high' | 'medium';
+  format: 'jwt' | 'opaque';
+  jwtClaims?: JwtClaims;
+}
+
+export interface JwtClaims {
+  exp?: number;
+  iat?: number;
+  iss?: string;
+  aud?: string;
+  scope?: string;
+}
+
+/**
+ * Calculate Shannon entropy (bits per character) of a string.
+ * Higher values indicate more randomness.
+ */
+export function shannonEntropy(value: string): number {
+  if (value.length === 0) return 0;
+
+  const freq = new Map<string, number>();
+  for (const ch of value) {
+    freq.set(ch, (freq.get(ch) ?? 0) + 1);
+  }
+
+  let entropy = 0;
+  const len = value.length;
+  for (const count of freq.values()) {
+    const p = count / len;
+    entropy -= p * Math.log2(p);
+  }
+
+  return entropy;
+}
+
+/**
+ * Parse JWT claims from a token string.
+ * Returns null if not a valid JWT structure.
+ */
+export function parseJwtClaims(token: string): JwtClaims | null {
+  // JWT: starts with eyJ, has exactly 2 dots
+  if (!token.startsWith('eyJ')) return null;
+  const parts = token.split('.');
+  if (parts.length !== 3) return null;
+
+  try {
+    // Decode payload (second part), base64url → JSON
+    const payload = parts[1]!;
+    const padded = payload.replace(/-/g, '+').replace(/_/g, '/');
+    const json = Buffer.from(padded, 'base64').toString('utf-8');
+    const claims = JSON.parse(json);
+
+    if (typeof claims !== 'object' || claims === null) return null;
+
+    const result: JwtClaims = {};
+    if (typeof claims.exp === 'number') result.exp = claims.exp;
+    if (typeof claims.iat === 'number') result.iat = claims.iat;
+    if (typeof claims.iss === 'string') result.iss = claims.iss;
+    if (typeof claims.aud === 'string') result.aud = claims.aud;
+    if (typeof claims.scope === 'string') result.scope = claims.scope;
+
+    return result;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Classify whether a header/cookie value is likely an auth token.
+ *
+ * Detection hierarchy:
+ * 1. JWT (eyJ prefix, 2 dots) → decode and classify with rich metadata
+ * 2. UUID → skip (entity ID, not token)
+ * 3. Short values (<16 chars) → skip
+ * 4. High-entropy opaque string → classify by entropy threshold
+ */
+export function isLikelyToken(name: string, value: string): TokenClassification {
+  // Strip "Bearer " prefix for analysis
+  const raw = value.startsWith('Bearer ') ? value.slice(7) : value;
+
+  // JWT detection — takes priority
+  const jwtClaims = parseJwtClaims(raw);
+  if (jwtClaims) {
+    return {
+      isToken: true,
+      confidence: 'high',
+      format: 'jwt',
+      jwtClaims,
+    };
+  }
+
+  // UUID exclusion — almost always entity IDs, not tokens
+  if (UUID_PATTERN.test(raw)) {
+    return { isToken: false, confidence: 'medium', format: 'opaque' };
+  }
+
+  // Minimum length gate
+  if (raw.length < MIN_TOKEN_LENGTH) {
+    return { isToken: false, confidence: 'medium', format: 'opaque' };
+  }
+
+  // Entropy-based classification
+  const entropy = shannonEntropy(raw);
+
+  if (entropy >= 4.5) {
+    return { isToken: true, confidence: 'high', format: 'opaque' };
+  }
+  if (entropy >= 3.5) {
+    return { isToken: true, confidence: 'medium', format: 'opaque' };
+  }
+
+  // Below threshold — not a token
+  return { isToken: false, confidence: 'medium', format: 'opaque' };
+}

package/src/capture/filter.ts

@@ -0,0 +1,56 @@
+// src/capture/filter.ts
+import { isBlocklisted } from './blocklist.js';
+
+export interface FilterableResponse {
+  url: string;
+  status: number;
+  contentType: string;
+}
+
+const JSON_CONTENT_TYPES = [
+  'application/json',
+  'application/vnd.api+json',
+  'text/json',
+];
+
+/** Exact path matches that are telemetry/framework noise */
+const NOISE_PATHS = new Set([
+  '/monitoring',
+  '/telemetry',
+  '/track',
+  '/manifest.json',
+]);
+
+/**
+ * Check if a URL path is framework or telemetry noise.
+ * Exported for testing.
+ */
+export function isPathNoise(pathname: string): boolean {
+  // Exact match noise paths
+  if (NOISE_PATHS.has(pathname)) return true;
+
+  // Next.js static build assets (not data routes)
+  if (pathname.startsWith('/_next/static/')) return true;
+
+  return false;
+}
+
+export function shouldCapture(response: FilterableResponse): boolean {
+  // Only keep 2xx success responses
+  if (response.status < 200 || response.status >= 300) return false;
+
+  // Content-type must indicate JSON
+  const ct = response.contentType.toLowerCase().split(';')[0].trim();
+  if (!JSON_CONTENT_TYPES.some(t => ct === t)) return false;
+
+  // Check domain and path
+  try {
+    const url = new URL(response.url);
+    if (isBlocklisted(url.hostname)) return false;
+    if (isPathNoise(url.pathname)) return false;
+  } catch {
+    return false;
+  }
+
+  return true;
+}

package/src/capture/graphql.ts

@@ -0,0 +1,124 @@
+// src/capture/graphql.ts
+
+export interface GraphQLParsed {
+  operationName: string | null;
+  query: string;
+  variables: Record<string, unknown> | null;
+}
+
+/**
+ * Detect if a request is to a GraphQL endpoint.
+ */
+export function isGraphQLEndpoint(
+  path: string,
+  contentType: string,
+  body: string | null,
+): boolean {
+  // Path contains /graphql
+  if (path.includes('/graphql')) {
+    return true;
+  }
+
+  // Content-Type is application/graphql
+  if (contentType.includes('application/graphql')) {
+    return true;
+  }
+
+  // Body contains a "query" field (GraphQL-style)
+  if (body) {
+    try {
+      const parsed = JSON.parse(body);
+      if (typeof parsed.query === 'string') {
+        return true;
+      }
+    } catch {
+      // Not JSON
+    }
+  }
+
+  return false;
+}
+
+/**
+ * Parse a GraphQL request body.
+ */
+export function parseGraphQLBody(body: string): GraphQLParsed | null {
+  try {
+    const parsed = JSON.parse(body);
+    if (typeof parsed.query !== 'string') {
+      return null;
+    }
+    return {
+      operationName: parsed.operationName ?? null,
+      query: parsed.query,
+      variables: parsed.variables ?? null,
+    };
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Extract operation name from query string or explicit operationName.
+ */
+export function extractOperationName(
+  query: string,
+  explicitName: string | null,
+): string {
+  if (explicitName) {
+    return explicitName;
+  }
+
+  // Match "query Name" or "mutation Name" at start
+  const match = query.match(/^\s*(query|mutation|subscription)\s+(\w+)/);
+  if (match) {
+    return match[2];
+  }
+
+  return 'Anonymous';
+}
+
+/**
+ * Detect which variables are likely dynamic (IDs, cursors, pagination).
+ */
+export function detectGraphQLVariables(
+  variables: Record<string, unknown> | null,
+  prefix = '',
+): string[] {
+  if (!variables || typeof variables !== 'object') {
+    return [];
+  }
+
+  const detected: string[] = [];
+
+  for (const [key, value] of Object.entries(variables)) {
+    const path = prefix ? `${prefix}.${key}` : key;
+
+    if (typeof value === 'number') {
+      // Numbers are often IDs or pagination values
+      detected.push(path);
+    } else if (typeof value === 'string') {
+      // Cursor-like strings (base64, long alphanumeric)
+      if (isLikelyCursor(value)) {
+        detected.push(path);
+      }
+    } else if (value && typeof value === 'object' && !Array.isArray(value)) {
+      // Recurse into nested objects
+      detected.push(...detectGraphQLVariables(value as Record<string, unknown>, path));
+    }
+  }
+
+  return detected;
+}
+
+function isLikelyCursor(value: string): boolean {
+  // Base64-ish: long alphanumeric, possibly with = padding
+  if (value.length > 10 && /^[a-zA-Z0-9+/=_-]+$/.test(value)) {
+    return true;
+  }
+  // UUID-like
+  if (/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(value)) {
+    return true;
+  }
+  return false;
+}

package/src/capture/idle.ts

@@ -0,0 +1,45 @@
+// src/capture/idle.ts
+
+/**
+ * Tracks unique endpoint discoveries and detects idle periods.
+ * Used during interactive capture to nudge the user when no new
+ * endpoints have been found for a while.
+ */
+export class IdleTracker {
+  private seen = new Set<string>();
+  private lastNewTime: number;
+  private thresholdMs: number;
+  private fired = false;
+  private now: () => number;
+
+  constructor(thresholdMs = 15000, now: () => number = Date.now) {
+    this.thresholdMs = thresholdMs;
+    this.now = now;
+    this.lastNewTime = this.now();
+  }
+
+  /**
+   * Record an endpoint key (e.g. "GET /api/items").
+   * Returns true if it's genuinely new (not seen before).
+   */
+  recordEndpoint(key: string): boolean {
+    if (this.seen.has(key)) return false;
+    this.seen.add(key);
+    this.lastNewTime = this.now();
+    this.fired = false;
+    return true;
+  }
+
+  /**
+   * Check if the idle threshold has been exceeded.
+   * Returns true exactly once per idle period (until reset by a new endpoint).
+   */
+  checkIdle(): boolean {
+    if (this.fired) return false;
+    if (this.now() - this.lastNewTime >= this.thresholdMs) {
+      this.fired = true;
+      return true;
+    }
+    return false;
+  }
+}

package/src/capture/monitor.ts

@@ -0,0 +1,224 @@
+// src/capture/monitor.ts
+import { chromium, type Browser, type Page } from 'playwright';
+import { shouldCapture } from './filter.js';
+import { isDomainMatch } from './domain.js';
+import { SkillGenerator, type GeneratorOptions } from '../skill/generator.js';
+import { IdleTracker } from './idle.js';
+import { detectCaptcha } from '../auth/refresh.js';
+import type { CapturedExchange } from '../types.js';
+
+export interface CaptureOptions {
+  url: string;
+  port?: number;
+  launch?: boolean;
+  attach?: boolean;
+  headless?: boolean;  // default: false (interactive capture shows browser)
+  duration?: number;
+  allDomains?: boolean;
+  enablePreview?: boolean;
+  scrub?: boolean;
+  onEndpoint?: (endpoint: { id: string; method: string; path: string }) => void;
+  onFiltered?: () => void;
+  onIdle?: () => void;
+}
+
+export interface CaptureResult {
+  generators: Map<string, SkillGenerator>;
+  totalRequests: number;
+  filteredRequests: number;
+  domBytes?: number; // v1.0: measured DOM size for browser cost comparison
+}
+
+const DEFAULT_CDP_PORTS = [18792, 18800, 9222];
+
+async function connectToBrowser(options: CaptureOptions): Promise<{ browser: Browser; launched: boolean }> {
+  if (!options.launch) {
+    const ports = options.port ? [options.port] : DEFAULT_CDP_PORTS;
+    for (const port of ports) {
+      try {
+        const browser = await chromium.connectOverCDP(`http://localhost:${port}`, { timeout: 3000 });
+        return { browser, launched: false };
+      } catch {
+        continue;
+      }
+    }
+  }
+
+  if (options.attach) {
+    const ports = options.port ? [options.port] : DEFAULT_CDP_PORTS;
+    throw new Error(`No browser found on CDP ports: ${ports.join(', ')}. Is a Chromium browser running with remote debugging?`);
+  }
+
+  const browser = await chromium.launch({ headless: options.headless ?? (process.env.DISPLAY ? false : true) });
+  return { browser, launched: true };
+}
+
+export async function capture(options: CaptureOptions): Promise<CaptureResult> {
+  const { browser, launched } = await connectToBrowser(options);
+  const generators = new Map<string, SkillGenerator>();
+  let totalRequests = 0;
+  let filteredRequests = 0;
+  const captchaDetectedDomains = new Set<string>();
+
+  // Extract target domain for domain-only filtering
+  const targetUrl = options.url;
+
+  const generatorOptions: GeneratorOptions = {
+    enablePreview: options.enablePreview ?? false,
+    scrub: options.scrub ?? true,
+  };
+
+  // Idle tracking: only active during interactive capture (no --duration)
+  const idleTracker = !options.duration ? new IdleTracker() : null;
+  let idleInterval: ReturnType<typeof setInterval> | null = null;
+
+  let page: Page;
+  if (launched) {
+    const context = await browser.newContext();
+    page = await context.newPage();
+  } else {
+    const contexts = browser.contexts();
+    if (contexts.length > 0 && contexts[0].pages().length > 0) {
+      page = contexts[0].pages()[0];
+    } else {
+      const context = contexts[0] ?? await browser.newContext();
+      page = await context.newPage();
+    }
+  }
+
+  page.on('response', async (response) => {
+    totalRequests++;
+
+    const url = response.url();
+    const status = response.status();
+    const contentType = response.headers()['content-type'] ?? '';
+
+    // Domain-only filtering (before any other processing)
+    if (!options.allDomains) {
+      const hostname = safeHostname(url);
+      if (hostname && !isDomainMatch(hostname, targetUrl)) {
+        filteredRequests++;
+        options.onFiltered?.();
+        return;
+      }
+    }
+
+    if (!shouldCapture({ url, status, contentType })) {
+      filteredRequests++;
+      const hostname = safeHostname(url);
+      if (hostname) {
+        const gen = generators.get(hostname);
+        if (gen) gen.recordFiltered();
+      }
+      // Track network bytes from headers for filtered responses (browser cost measurement)
+      const contentLength = parseInt(response.headers()['content-length'] ?? '0', 10);
+      if (contentLength > 0) {
+        const filteredHostname = safeHostname(url);
+        if (filteredHostname && generators.has(filteredHostname)) {
+          generators.get(filteredHostname)!.addNetworkBytes(contentLength);
+        }
+      }
+      options.onFiltered?.();
+      return;
+    }
+
+    try {
+      const body = await response.text();
+      const hostname = new URL(url).hostname;
+
+      // Check for captcha in HTML responses (v0.8 captcha risk detection)
+      if (contentType.includes('text/html') && detectCaptcha(body)) {
+        captchaDetectedDomains.add(hostname);
+      }
+
+      if (!generators.has(hostname)) {
+        generators.set(hostname, new SkillGenerator(generatorOptions));
+      }
+      const gen = generators.get(hostname)!;
+
+      const exchange: CapturedExchange = {
+        request: {
+          url,
+          method: response.request().method(),
+          headers: response.request().headers(),
+          postData: response.request().postData() ?? undefined,
+        },
+        response: {
+          status,
+          headers: response.headers(),
+          body,
+          contentType,
+        },
+        timestamp: new Date().toISOString(),
+      };
+
+      const endpoint = gen.addExchange(exchange);
+      if (endpoint) {
+        options.onEndpoint?.({ id: endpoint.id, method: endpoint.method, path: endpoint.path });
+
+        // Track for idle detection using parameterized key
+        if (idleTracker) {
+          const paramKey = `${endpoint.method} ${endpoint.path}`;
+          idleTracker.recordEndpoint(paramKey);
+        }
+      }
+    } catch {
+      // Response body may not be available (e.g. redirects); skip silently
+    }
+  });
+
+  await page.goto(options.url, { waitUntil: 'domcontentloaded' });
+
+  // Start idle check interval (every 5s) for interactive capture
+  if (idleTracker && options.onIdle) {
+    idleInterval = setInterval(() => {
+      if (idleTracker.checkIdle()) {
+        options.onIdle!();
+      }
+    }, 5000);
+  }
+
+  // Wait for duration or until interrupted
+  // SIGINT always resolves gracefully so skill files get written
+  if (options.duration) {
+    await new Promise<void>(resolve => {
+      const timer = setTimeout(resolve, options.duration! * 1000);
+      process.once('SIGINT', () => { clearTimeout(timer); resolve(); });
+    });
+  } else {
+    await new Promise<void>(resolve => {
+      process.once('SIGINT', resolve);
+    });
+  }
+
+  // Clean up idle interval
+  if (idleInterval) clearInterval(idleInterval);
+
+  // Measure DOM size for browser cost comparison (v1.0)
+  let domBytes: number | undefined;
+  try {
+    const html = await page.content();
+    domBytes = html.length;
+  } catch { /* page may have navigated away */ }
+
+  if (launched) {
+    await browser.close();
+  }
+
+  // Mark generators for domains where captcha was detected
+  for (const [hostname, gen] of generators) {
+    if (captchaDetectedDomains.has(hostname)) {
+      gen.setCaptchaRisk(true);
+    }
+  }
+
+  return { generators, totalRequests, filteredRequests, domBytes };
+}
+
+function safeHostname(url: string): string | null {
+  try {
+    return new URL(url).hostname;
+  } catch {
+    return null;
+  }
+}