@apitap/core 1.0.0

package/src/discovery/auth.ts

@@ -0,0 +1,99 @@
+// src/discovery/auth.ts
+
+export interface AuthDetectionResult {
+  authRequired: boolean;
+  signals: string[];
+  loginUrl?: string;  // detected login page URL if found
+}
+
+// Paths that indicate auth/login
+const AUTH_PATH_PATTERNS = [
+  /\/login/i, /\/signin/i, /\/sign-in/i, /\/auth\//i,
+  /\/sso/i, /\/saml/i, /\/oauth/i, /\/cas\/login/i,
+];
+
+// OAuth provider patterns in URLs
+const OAUTH_PATTERNS = [
+  /accounts\.google\.com\/o\/oauth/i,
+  /github\.com\/login\/oauth/i,
+  /login\.microsoftonline\.com/i,
+  /facebook\.com\/v\d+.*\/dialog\/oauth/i,
+  /appleid\.apple\.com\/auth/i,
+];
+
+/**
+ * Scan fetched HTML and response headers for indicators that a site requires authentication.
+ *
+ * Checks for:
+ * - Login forms (password inputs)
+ * - Meta redirects to auth paths
+ * - OAuth login links
+ * - WWW-Authenticate response header
+ * - Location header redirecting to login
+ * - SAML/SSO form patterns
+ */
+export function detectAuthRequired(
+  html: string,
+  url: string,
+  headers: Record<string, string>,
+): AuthDetectionResult {
+  const signals: string[] = [];
+  let loginUrl: string | undefined;
+
+  // 1. WWW-Authenticate header
+  if (headers['www-authenticate']) {
+    signals.push(`WWW-Authenticate header: ${headers['www-authenticate']}`);
+  }
+
+  // 2. Location header redirecting to auth path
+  const location = headers['location'];
+  if (location) {
+    if (AUTH_PATH_PATTERNS.some(p => p.test(location))) {
+      signals.push(`Location redirect to auth path: ${location}`);
+      loginUrl = loginUrl ?? location;
+    }
+  }
+
+  // 3. Login form with password input
+  const hasPasswordInput = /<input[^>]*type\s*=\s*["']password["'][^>]*>/i.test(html);
+  const hasFormAction = /<form[^>]*action\s*=\s*["'][^"']*(?:login|signin|sign-in|auth)[^"']*["'][^>]*>/i.test(html);
+  if (hasPasswordInput && hasFormAction) {
+    signals.push('Detected login form with password input');
+    // Try to extract login URL from form action
+    const formMatch = html.match(/<form[^>]*action\s*=\s*["']([^"']*(?:login|signin|sign-in|auth)[^"']*)["']/i);
+    if (formMatch) {
+      loginUrl = loginUrl ?? formMatch[1];
+    }
+  } else if (hasPasswordInput) {
+    signals.push('Detected login form with password input');
+  }
+
+  // 4. Meta redirect to auth path
+  const metaRefresh = html.match(/<meta[^>]*http-equiv\s*=\s*["']refresh["'][^>]*content\s*=\s*["'][^"']*url\s*=\s*([^"'\s>]+)/i);
+  if (metaRefresh) {
+    const redirectUrl = metaRefresh[1];
+    if (AUTH_PATH_PATTERNS.some(p => p.test(redirectUrl))) {
+      signals.push(`Meta redirect to auth path: ${redirectUrl}`);
+      loginUrl = loginUrl ?? redirectUrl;
+    }
+  }
+
+  // 5. OAuth provider links
+  const oauthMatch = OAUTH_PATTERNS.find(p => p.test(html));
+  if (oauthMatch) {
+    signals.push('OAuth provider login link detected');
+  }
+
+  // 6. SAML/SSO form
+  const hasSaml = /SAMLRequest/i.test(html) || /saml/i.test(html);
+  const hasSsoForm = /<form[^>]*action\s*=\s*["'][^"']*(?:sso|saml)[^"']*["'][^>]*>/i.test(html);
+  if (hasSaml && hasSsoForm) {
+    signals.push('SSO/SAML authentication form detected');
+  }
+
+  return {
+    authRequired: signals.length > 0,
+    signals,
+    loginUrl,
+  };
+}

package/src/discovery/fetch.ts

@@ -0,0 +1,85 @@
+// src/discovery/fetch.ts
+import { validateUrl } from '../skill/ssrf.js';
+
+export interface FetchResult {
+  status: number;
+  headers: Record<string, string>;
+  body: string;
+  contentType: string;
+}
+
+export interface SafeFetchOptions {
+  timeout?: number;
+  method?: 'GET' | 'HEAD';
+  maxBodySize?: number;
+  skipSsrf?: boolean; // bypass SSRF check (for testing with local servers)
+}
+
+const DEFAULT_TIMEOUT = 5000;
+const DEFAULT_MAX_BODY = 512 * 1024; // 512KB
+const USER_AGENT = 'ApiTap-Discovery/1.0';
+
+/**
+ * Fetch a URL with SSRF protection, timeout, and size limits.
+ * Returns null on any failure (network error, SSRF blocked, timeout).
+ */
+export async function safeFetch(
+  url: string,
+  options: SafeFetchOptions = {},
+): Promise<FetchResult | null> {
+  // SSRF check
+  if (!options.skipSsrf) {
+    const ssrfResult = validateUrl(url);
+    if (!ssrfResult.safe) return null;
+  }
+
+  const timeout = options.timeout ?? DEFAULT_TIMEOUT;
+  const method = options.method ?? 'GET';
+  const maxBody = options.maxBodySize ?? DEFAULT_MAX_BODY;
+
+  try {
+    const controller = new AbortController();
+    const timer = setTimeout(() => controller.abort(), timeout);
+
+    const response = await fetch(url, {
+      method,
+      signal: controller.signal,
+      headers: {
+        'User-Agent': USER_AGENT,
+        'Accept': 'text/html,application/json,*/*',
+      },
+      redirect: 'follow',
+    });
+
+    clearTimeout(timer);
+
+    // Extract headers
+    const headers: Record<string, string> = {};
+    response.headers.forEach((value, key) => {
+      headers[key.toLowerCase()] = value;
+    });
+
+    const contentType = headers['content-type'] || '';
+
+    // For HEAD requests, don't read body
+    if (method === 'HEAD') {
+      return { status: response.status, headers, body: '', contentType };
+    }
+
+    // Read body with size limit
+    const body = await readBodyLimited(response, maxBody);
+
+    return { status: response.status, headers, body, contentType };
+  } catch {
+    return null;
+  }
+}
+
+async function readBodyLimited(response: Response, maxSize: number): Promise<string> {
+  // Use text() with a size check — for discovery we don't need huge bodies
+  const text = await response.text();
+  if (text.length > maxSize) {
+    return text.slice(0, maxSize);
+  }
+  return text;
+}

package/src/discovery/frameworks.ts

@@ -0,0 +1,231 @@
+// src/discovery/frameworks.ts
+import type { DetectedFramework } from '../types.js';
+
+export interface PageInfo {
+  html: string;
+  headers: Record<string, string>;
+  url: string;
+}
+
+interface FrameworkDetector {
+  name: string;
+  detect(page: PageInfo): DetectedFramework | null;
+}
+
+const detectors: FrameworkDetector[] = [
+  {
+    name: 'wordpress',
+    detect({ html, headers }) {
+      const signals: string[] = [];
+      if (html.includes('/wp-json/')) signals.push('wp-json link');
+      if (html.includes('/wp-content/')) signals.push('wp-content');
+      if (html.includes('/wp-includes/')) signals.push('wp-includes');
+      if (headers['link']?.includes('/wp-json/')) signals.push('Link header');
+      if (headers['x-powered-by']?.toLowerCase().includes('wordpress')) signals.push('X-Powered-By');
+
+      if (signals.length === 0) return null;
+      return {
+        name: 'WordPress',
+        confidence: signals.length >= 2 ? 'high' : 'medium',
+        apiPatterns: [
+          '/wp-json/wp/v2/posts',
+          '/wp-json/wp/v2/pages',
+          '/wp-json/wp/v2/categories',
+          '/wp-json/wp/v2/tags',
+          '/wp-json/wp/v2/media',
+          '/wp-json/wp/v2/users',
+          '/wp-json/wp/v2/comments',
+          '/wp-json/wp/v2/search',
+        ],
+      };
+    },
+  },
+  {
+    name: 'shopify',
+    detect({ html, url }) {
+      const signals: string[] = [];
+      if (html.includes('cdn.shopify.com')) signals.push('Shopify CDN');
+      if (html.includes('Shopify.theme')) signals.push('Shopify.theme');
+      if (html.includes('myshopify.com')) signals.push('myshopify domain');
+      if (html.includes('shopify-section')) signals.push('shopify-section');
+
+      if (signals.length === 0) return null;
+      const origin = new URL(url).origin;
+      return {
+        name: 'Shopify',
+        confidence: signals.length >= 2 ? 'high' : 'medium',
+        apiPatterns: [
+          '/products.json',
+          '/collections.json',
+          '/cart.json',
+          '/search/suggest.json',
+        ],
+      };
+    },
+  },
+  {
+    name: 'nextjs',
+    detect({ html, headers }) {
+      const signals: string[] = [];
+      if (html.includes('__NEXT_DATA__')) signals.push('__NEXT_DATA__');
+      if (html.includes('/_next/')) signals.push('_next assets');
+      if (headers['x-nextjs-cache']) signals.push('X-Nextjs-Cache');
+      if (headers['x-powered-by']?.toLowerCase().includes('next.js')) signals.push('X-Powered-By');
+
+      if (signals.length === 0) return null;
+
+      // Extract build ID from __NEXT_DATA__ if available
+      const buildIdMatch = html.match(/"buildId"\s*:\s*"([^"]+)"/);
+      const patterns: string[] = ['/api/'];
+      if (buildIdMatch) {
+        patterns.push(`/_next/data/${buildIdMatch[1]}/`);
+      }
+
+      return {
+        name: 'Next.js',
+        confidence: signals.length >= 2 ? 'high' : 'medium',
+        apiPatterns: patterns,
+      };
+    },
+  },
+  {
+    name: 'nuxt',
+    detect({ html, headers }) {
+      const signals: string[] = [];
+      if (html.includes('__NUXT__')) signals.push('__NUXT__');
+      if (html.includes('/_nuxt/')) signals.push('_nuxt assets');
+      if (html.includes('_payload.json')) signals.push('_payload.json');
+      if (html.includes('nuxt-link')) signals.push('nuxt-link');
+
+      if (signals.length === 0) return null;
+      return {
+        name: 'Nuxt',
+        confidence: signals.length >= 2 ? 'high' : 'medium',
+        apiPatterns: ['/api/', '/_payload.json'],
+      };
+    },
+  },
+  {
+    name: 'graphql',
+    detect({ html }) {
+      const signals: string[] = [];
+      if (html.includes('/graphql')) signals.push('/graphql reference');
+      if (html.includes('__APOLLO_STATE__')) signals.push('Apollo state');
+      if (html.includes('apollo-client')) signals.push('apollo-client');
+      if (html.includes('relay-')) signals.push('Relay');
+      if (html.includes('urql')) signals.push('urql');
+
+      if (signals.length === 0) return null;
+      return {
+        name: 'GraphQL',
+        confidence: signals.length >= 2 ? 'high' : 'medium',
+        apiPatterns: ['/graphql', '/gql', '/api/graphql'],
+      };
+    },
+  },
+  {
+    name: 'drupal',
+    detect({ html, headers }) {
+      const signals: string[] = [];
+      if (headers['x-drupal-cache']) signals.push('X-Drupal-Cache');
+      if (headers['x-drupal-dynamic-cache']) signals.push('X-Drupal-Dynamic-Cache');
+      if (headers['x-generator']?.toLowerCase().includes('drupal')) signals.push('X-Generator');
+      if (html.includes('/jsonapi/')) signals.push('jsonapi');
+      if (html.includes('drupal-settings-json')) signals.push('drupal-settings');
+
+      if (signals.length === 0) return null;
+      return {
+        name: 'Drupal',
+        confidence: signals.length >= 2 ? 'high' : 'medium',
+        apiPatterns: [
+          '/jsonapi/node/article',
+          '/jsonapi/node/page',
+          '/jsonapi/taxonomy_term',
+        ],
+      };
+    },
+  },
+  {
+    name: 'rails',
+    detect({ headers }) {
+      const signals: string[] = [];
+      if (headers['x-request-id'] && headers['x-runtime']) signals.push('Rails headers');
+      if (headers['x-powered-by']?.toLowerCase().includes('phusion')) signals.push('Phusion');
+
+      if (signals.length === 0) return null;
+      return {
+        name: 'Rails',
+        confidence: 'low',
+        apiPatterns: ['/api/v1/'],
+      };
+    },
+  },
+  {
+    name: 'django-rest',
+    detect({ headers, html }) {
+      const signals: string[] = [];
+      if (headers['x-frame-options'] && headers['vary']?.includes('Cookie')) signals.push('Django-like headers');
+      if (html.includes('csrfmiddlewaretoken')) signals.push('CSRF middleware');
+      if (html.includes('django')) signals.push('django reference');
+
+      if (signals.length === 0) return null;
+      return {
+        name: 'Django',
+        confidence: 'low',
+        apiPatterns: ['/api/', '/api/v1/', '/rest/'],
+      };
+    },
+  },
+  {
+    name: 'laravel',
+    detect({ html, headers }) {
+      const signals: string[] = [];
+      if (html.includes('csrf-token') && html.includes('laravel')) signals.push('Laravel meta');
+      if (headers['set-cookie']?.includes('laravel_session')) signals.push('laravel_session');
+
+      if (signals.length === 0) return null;
+      return {
+        name: 'Laravel',
+        confidence: 'medium',
+        apiPatterns: ['/api/', '/api/v1/'],
+      };
+    },
+  },
+  {
+    name: 'strapi',
+    detect({ headers }) {
+      const signals: string[] = [];
+      if (headers['x-powered-by']?.toLowerCase().includes('strapi')) signals.push('X-Powered-By');
+
+      if (signals.length === 0) return null;
+      return {
+        name: 'Strapi',
+        confidence: 'high',
+        apiPatterns: ['/api/', '/api/content-types', '/api/articles', '/api/pages'],
+      };
+    },
+  },
+];
+
+/**
+ * Detect web frameworks from a page's HTML and response headers.
+ * Returns all detected frameworks, sorted by confidence (high first).
+ */
+export function detectFrameworks(page: PageInfo): DetectedFramework[] {
+  const results: DetectedFramework[] = [];
+  // Lowercase headers for consistent matching
+  const normalizedHeaders: Record<string, string> = {};
+  for (const [key, value] of Object.entries(page.headers)) {
+    normalizedHeaders[key.toLowerCase()] = value;
+  }
+  const normalizedPage = { ...page, headers: normalizedHeaders };
+
+  for (const detector of detectors) {
+    const result = detector.detect(normalizedPage);
+    if (result) results.push(result);
+  }
+
+  const order = { high: 0, medium: 1, low: 2 };
+  results.sort((a, b) => order[a.confidence] - order[b.confidence]);
+  return results;
+}

package/src/discovery/index.ts

@@ -0,0 +1,256 @@
+// src/discovery/index.ts
+import type { DiscoveryResult, SkillFile, SkillEndpoint, DetectedFramework } from '../types.js';
+import { validateUrl } from '../skill/ssrf.js';
+import { safeFetch } from './fetch.js';
+import { detectFrameworks } from './frameworks.js';
+import { discoverSpecs, parseSpecToSkillFile } from './openapi.js';
+import { probeApiPaths } from './probes.js';
+import { detectAuthRequired } from './auth.js';
+
+export interface DiscoveryOptions {
+  timeout?: number;         // overall timeout in ms (default: 30000)
+  skipProbes?: boolean;     // skip API path probing
+  skipSpecs?: boolean;      // skip OpenAPI spec discovery
+  skipFrameworks?: boolean; // skip framework detection
+  skipSsrf?: boolean;       // bypass SSRF check (for testing with local servers)
+}
+
+/**
+ * Run smart discovery on a URL to detect APIs without launching a browser.
+ *
+ * Flow:
+ * 1. SSRF validation
+ * 2. Fetch homepage HTML + headers
+ * 3. Run detection strategies in parallel:
+ *    - Framework detection (from HTML/headers)
+ *    - OpenAPI spec discovery (probe common paths)
+ *    - Common API pattern probing
+ * 4. Synthesize results into a DiscoveryResult
+ */
+export async function discover(
+  url: string,
+  options: DiscoveryOptions = {},
+): Promise<DiscoveryResult> {
+  const start = Date.now();
+  const fullUrl = url.startsWith('http') ? url : `https://${url}`;
+
+  // SSRF check
+  if (!options.skipSsrf) {
+    const ssrfResult = validateUrl(fullUrl);
+    if (!ssrfResult.safe) {
+      return {
+        confidence: 'none',
+        hints: [`SSRF blocked: ${ssrfResult.reason}`],
+        duration: Date.now() - start,
+      };
+    }
+  }
+
+  let domain: string;
+  let origin: string;
+  try {
+    const parsed = new URL(fullUrl);
+    domain = parsed.hostname;
+    origin = parsed.origin;
+  } catch {
+    return {
+      confidence: 'none',
+      hints: ['Invalid URL'],
+      duration: Date.now() - start,
+    };
+  }
+
+  // Fetch homepage
+  const homepage = await safeFetch(fullUrl, { timeout: options.timeout ?? 10000, skipSsrf: options.skipSsrf });
+  if (!homepage) {
+    return {
+      confidence: 'none',
+      hints: ['Failed to fetch homepage — site may be down or blocking requests'],
+      duration: Date.now() - start,
+    };
+  }
+
+  const ssrfOpts = { skipSsrf: options.skipSsrf };
+
+  // Auth detection (runs on homepage HTML + headers)
+  const authResult = detectAuthRequired(homepage.body, fullUrl, homepage.headers);
+  const authFields = authResult.authRequired ? {
+    authRequired: true as const,
+    authSignals: authResult.signals,
+    ...(authResult.loginUrl ? { loginUrl: authResult.loginUrl } : {}),
+  } : {};
+
+  // Run all detection strategies in parallel
+  const [frameworks, specs, probes] = await Promise.all([
+    options.skipFrameworks
+      ? []
+      : detectFrameworks({ html: homepage.body, headers: homepage.headers, url: fullUrl }),
+    options.skipSpecs
+      ? []
+      : discoverSpecs(origin, homepage.headers, ssrfOpts),
+    options.skipProbes
+      ? []
+      : probeApiPaths(origin, ssrfOpts),
+  ]);
+
+  const hints: string[] = [];
+
+  // Strategy 1: OpenAPI spec found → parse into skill file (highest confidence)
+  if (specs.length > 0) {
+    const bestSpec = specs[0];
+    const skillFile = await parseSpecToSkillFile(bestSpec.url, domain, origin, ssrfOpts);
+    if (skillFile && skillFile.endpoints.length > 0) {
+      hints.push(`OpenAPI spec found at ${bestSpec.url} (${bestSpec.version})`);
+      if (frameworks.length > 0) hints.push(`Framework: ${frameworks.map(f => f.name).join(', ')}`);
+      addProbeHints(hints, probes);
+
+      return {
+        confidence: 'high',
+        skillFile,
+        hints,
+        frameworks: frameworks.length > 0 ? frameworks : undefined,
+        specs,
+        probes: probes.length > 0 ? probes : undefined,
+        duration: Date.now() - start,
+        ...authFields,
+      };
+    }
+  }
+
+  // Strategy 2: Framework detected → generate skeleton skill file
+  const highConfidence = frameworks.filter(f => f.confidence === 'high');
+  if (highConfidence.length > 0) {
+    const skillFile = buildFrameworkSkillFile(domain, origin, highConfidence);
+    hints.push(`Detected: ${highConfidence.map(f => f.name).join(', ')}`);
+    addProbeHints(hints, probes);
+    if (specs.length > 0) hints.push(`Spec found but could not parse: ${specs.map(s => s.url).join(', ')}`);
+
+    return {
+      confidence: 'medium',
+      skillFile,
+      hints,
+      frameworks,
+      specs: specs.length > 0 ? specs : undefined,
+      probes: probes.length > 0 ? probes : undefined,
+      duration: Date.now() - start,
+      ...authFields,
+    };
+  }
+
+  // Strategy 3: Medium-confidence framework or API probes found → hints only
+  const apiProbes = probes.filter(p => p.isApi);
+  const mediumFrameworks = frameworks.filter(f => f.confidence === 'medium');
+
+  if (mediumFrameworks.length > 0 || apiProbes.length > 0) {
+    if (mediumFrameworks.length > 0) {
+      const skillFile = buildFrameworkSkillFile(domain, origin, mediumFrameworks);
+      hints.push(`Possibly: ${mediumFrameworks.map(f => f.name).join(', ')}`);
+      addProbeHints(hints, probes);
+
+      return {
+        confidence: 'low',
+        skillFile,
+        hints,
+        frameworks,
+        probes: probes.length > 0 ? probes : undefined,
+        duration: Date.now() - start,
+        ...authFields,
+      };
+    }
+
+    // Only probes found
+    hints.push('API paths detected via probing');
+    addProbeHints(hints, probes);
+
+    return {
+      confidence: 'low',
+      hints,
+      frameworks: frameworks.length > 0 ? frameworks : undefined,
+      probes,
+      duration: Date.now() - start,
+      ...authFields,
+    };
+  }
+
+  // Nothing found
+  if (frameworks.length > 0) {
+    hints.push(`Low-confidence signals: ${frameworks.map(f => f.name).join(', ')}`);
+  }
+  hints.push('No API patterns detected — auto-capture recommended');
+
+  return {
+    confidence: 'none',
+    hints,
+    frameworks: frameworks.length > 0 ? frameworks : undefined,
+    probes: probes.length > 0 ? probes : undefined,
+    duration: Date.now() - start,
+    ...authFields,
+  };
+}
+
+function addProbeHints(hints: string[], probes: import('../types.js').ProbeResult[]): void {
+  const apiProbes = probes.filter(p => p.isApi);
+  if (apiProbes.length > 0) {
+    hints.push(`API paths found: ${apiProbes.map(p => `${p.path} (${p.status})`).join(', ')}`);
+  }
+}
+
+/**
+ * Build a skeleton skill file from detected frameworks.
+ * Endpoints are unverified predictions — replayability is 'unknown'.
+ */
+function buildFrameworkSkillFile(
+  domain: string,
+  baseUrl: string,
+  frameworks: DetectedFramework[],
+): SkillFile {
+  const endpoints: SkillEndpoint[] = [];
+  const seen = new Set<string>();
+
+  for (const framework of frameworks) {
+    for (const pattern of framework.apiPatterns) {
+      const key = `GET ${pattern}`;
+      if (seen.has(key)) continue;
+      seen.add(key);
+
+      const id = generateId('GET', pattern);
+      endpoints.push({
+        id,
+        method: 'GET',
+        path: pattern,
+        queryParams: {},
+        headers: {},
+        responseShape: { type: 'unknown' },
+        examples: {
+          request: { url: `${baseUrl}${pattern}`, headers: {} },
+          responsePreview: null,
+        },
+        replayability: {
+          tier: 'unknown',
+          verified: false,
+          signals: [`discovered-from-${framework.name.toLowerCase()}`],
+        },
+      });
+    }
+  }
+
+  return {
+    version: '1.2',
+    domain,
+    capturedAt: new Date().toISOString(),
+    baseUrl,
+    endpoints,
+    metadata: {
+      captureCount: 0,
+      filteredCount: 0,
+      toolVersion: '1.0.0',
+    },
+    provenance: 'unsigned',
+  };
+}
+
+function generateId(method: string, path: string): string {
+  const segments = path.split('/').filter(s => s !== '' && !s.startsWith(':'));
+  const slug = segments.join('-').replace(/[^a-z0-9-]/gi, '').toLowerCase() || 'root';
+  return `${method.toLowerCase()}-${slug}`;
+}