@apitap/core 1.0.0

package/src/discovery/openapi.ts

@@ -0,0 +1,230 @@
+// src/discovery/openapi.ts
+import type { SkillEndpoint, SkillFile, DiscoveredSpec } from '../types.js';
+import { safeFetch } from './fetch.js';
+
+/** Paths to check for API specs, in priority order */
+const SPEC_PATHS = [
+  '/openapi.json',
+  '/swagger.json',
+  '/api-docs',
+  '/api/docs',
+  '/.well-known/openapi',
+  '/v1/openapi.json',
+  '/v2/openapi.json',
+  '/v3/openapi.json',
+  '/docs/api.json',
+  '/api/swagger.json',
+];
+
+interface OpenApiSpec {
+  openapi?: string;    // "3.x.x"
+  swagger?: string;    // "2.0"
+  info?: { title?: string; version?: string };
+  paths?: Record<string, Record<string, OpenApiOperation>>;
+  servers?: { url: string }[];
+  host?: string;       // Swagger 2.0
+  basePath?: string;   // Swagger 2.0
+}
+
+interface OpenApiOperation {
+  operationId?: string;
+  summary?: string;
+  parameters?: OpenApiParameter[];
+  requestBody?: {
+    content?: Record<string, { schema?: unknown }>;
+  };
+  responses?: Record<string, unknown>;
+}
+
+interface OpenApiParameter {
+  name: string;
+  in: 'query' | 'path' | 'header' | 'cookie';
+  required?: boolean;
+  schema?: { type?: string };
+}
+
+export interface SpecDiscoveryOptions {
+  skipSsrf?: boolean;
+}
+
+/**
+ * Check for API specs at common paths and in Link headers.
+ * Returns discovered specs with their URLs.
+ */
+export async function discoverSpecs(
+  baseUrl: string,
+  homepageHeaders?: Record<string, string>,
+  options: SpecDiscoveryOptions = {},
+): Promise<DiscoveredSpec[]> {
+  const specs: DiscoveredSpec[] = [];
+  const origin = new URL(baseUrl).origin;
+
+  // Check Link header from homepage for rel="describedby"
+  if (homepageHeaders) {
+    const linkHeader = homepageHeaders['link'] || homepageHeaders['Link'];
+    if (linkHeader) {
+      const describedBy = parseLinkHeader(linkHeader, 'describedby');
+      if (describedBy) {
+        const specUrl = describedBy.startsWith('http') ? describedBy : `${origin}${describedBy}`;
+        const result = await tryFetchSpec(specUrl, options);
+        if (result) specs.push(result);
+      }
+    }
+  }
+
+  // Probe common spec paths in parallel
+  const checks = SPEC_PATHS.map(async (path) => {
+    const specUrl = `${origin}${path}`;
+    return tryFetchSpec(specUrl, options);
+  });
+
+  const results = await Promise.all(checks);
+  for (const result of results) {
+    if (result && !specs.some(s => s.url === result.url)) {
+      specs.push(result);
+    }
+  }
+
+  return specs;
+}
+
+async function tryFetchSpec(url: string, options: SpecDiscoveryOptions = {}): Promise<DiscoveredSpec | null> {
+  const result = await safeFetch(url, { timeout: 5000, skipSsrf: options.skipSsrf });
+  if (!result || result.status !== 200) return null;
+
+  // Must look like JSON
+  const ct = result.contentType.toLowerCase();
+  if (!ct.includes('json') && !ct.includes('yaml') && !ct.includes('text/plain')) return null;
+
+  try {
+    const spec = JSON.parse(result.body) as OpenApiSpec;
+    if (spec.openapi || spec.swagger) {
+      const endpointCount = spec.paths ? Object.keys(spec.paths).reduce((sum, path) => {
+        return sum + Object.keys(spec.paths![path]).filter(m => ['get', 'post', 'put', 'patch', 'delete'].includes(m)).length;
+      }, 0) : 0;
+
+      return {
+        type: spec.openapi ? 'openapi' : 'swagger',
+        url,
+        version: spec.openapi || spec.swagger,
+        endpointCount,
+      };
+    }
+  } catch {
+    // Not valid JSON or not an API spec
+  }
+  return null;
+}
+
+/**
+ * Parse an OpenAPI/Swagger spec into a SkillFile.
+ */
+export async function parseSpecToSkillFile(
+  specUrl: string,
+  domain: string,
+  baseUrl: string,
+  options: SpecDiscoveryOptions = {},
+): Promise<SkillFile | null> {
+  const result = await safeFetch(specUrl, { timeout: 10000, skipSsrf: options.skipSsrf });
+  if (!result || result.status !== 200) return null;
+
+  let spec: OpenApiSpec;
+  try {
+    spec = JSON.parse(result.body);
+  } catch {
+    return null;
+  }
+
+  if (!spec.paths) return null;
+
+  // Determine API base URL
+  let apiBase = baseUrl;
+  if (spec.servers?.[0]?.url) {
+    const serverUrl = spec.servers[0].url;
+    apiBase = serverUrl.startsWith('http') ? serverUrl : `${baseUrl}${serverUrl}`;
+  } else if (spec.host) {
+    const scheme = specUrl.startsWith('https') ? 'https' : 'http';
+    apiBase = `${scheme}://${spec.host}${spec.basePath || ''}`;
+  }
+
+  const endpoints: SkillEndpoint[] = [];
+
+  for (const [path, methods] of Object.entries(spec.paths)) {
+    for (const [method, operation] of Object.entries(methods)) {
+      if (!['get', 'post', 'put', 'patch', 'delete'].includes(method)) continue;
+      const op = operation as OpenApiOperation;
+
+      // Parameterize path: {id} → :id
+      const paramPath = path.replace(/\{([^}]+)\}/g, ':$1');
+
+      // Extract query params
+      const queryParams: Record<string, { type: string; example: string }> = {};
+      if (op.parameters) {
+        for (const param of op.parameters) {
+          if (param.in === 'query') {
+            queryParams[param.name] = {
+              type: param.schema?.type || 'string',
+              example: '',
+            };
+          }
+        }
+      }
+
+      // Generate endpoint ID
+      const id = op.operationId
+        ? method.toLowerCase() + '-' + op.operationId.replace(/[^a-z0-9]/gi, '-').toLowerCase()
+        : generateId(method, paramPath);
+
+      endpoints.push({
+        id,
+        method: method.toUpperCase(),
+        path: paramPath,
+        queryParams,
+        headers: {},
+        responseShape: { type: 'unknown' },
+        examples: {
+          request: { url: `${apiBase}${path}`, headers: {} },
+          responsePreview: null,
+        },
+        replayability: {
+          tier: 'unknown',
+          verified: false,
+          signals: ['discovered-from-spec'],
+        },
+      });
+    }
+  }
+
+  if (endpoints.length === 0) return null;
+
+  return {
+    version: '1.2',
+    domain,
+    capturedAt: new Date().toISOString(),
+    baseUrl: apiBase,
+    endpoints,
+    metadata: {
+      captureCount: 0,
+      filteredCount: 0,
+      toolVersion: '1.0.0',
+    },
+    provenance: 'unsigned',
+  };
+}
+
+function generateId(method: string, path: string): string {
+  const segments = path.split('/').filter(s => s !== '' && !s.startsWith(':'));
+  const slug = segments.join('-').replace(/[^a-z0-9-]/gi, '').toLowerCase() || 'root';
+  return `${method.toLowerCase()}-${slug}`;
+}
+
+function parseLinkHeader(header: string, rel: string): string | null {
+  const parts = header.split(',');
+  for (const part of parts) {
+    const match = part.match(/<([^>]+)>.*rel\s*=\s*"?([^",;]+)"?/);
+    if (match && match[2].trim() === rel) {
+      return match[1];
+    }
+  }
+  return null;
+}

package/src/discovery/probes.ts

@@ -0,0 +1,76 @@
+// src/discovery/probes.ts
+import type { ProbeResult } from '../types.js';
+import { safeFetch } from './fetch.js';
+
+/** Common API paths to probe */
+const PROBE_PATHS = [
+  '/api/',
+  '/api/v1/',
+  '/api/v2/',
+  '/_api/',
+  '/rest/',
+  '/graphql',
+  '/gql',
+  '/api/graphql',
+];
+
+export interface ProbeOptions {
+  skipSsrf?: boolean;
+}
+
+/**
+ * Probe common API paths with GET requests.
+ * Returns results for paths that respond with API-like content types.
+ */
+export async function probeApiPaths(baseUrl: string, options: ProbeOptions = {}): Promise<ProbeResult[]> {
+  const origin = new URL(baseUrl).origin;
+  const results: ProbeResult[] = [];
+
+  const checks = PROBE_PATHS.map(async (path): Promise<ProbeResult | null> => {
+    const url = `${origin}${path}`;
+    const result = await safeFetch(url, { timeout: 5000, method: 'GET', maxBodySize: 4096, skipSsrf: options.skipSsrf });
+    if (!result) return null;
+
+    // Don't count redirects to login pages or error pages
+    if (result.status >= 400 && result.status !== 401 && result.status !== 403) return null;
+
+    const ct = result.contentType.toLowerCase();
+    const isApi = isApiContentType(ct, result.body, result.status);
+
+    return {
+      method: 'GET',
+      path,
+      status: result.status,
+      contentType: result.contentType,
+      isApi,
+    };
+  });
+
+  const settled = await Promise.all(checks);
+  for (const result of settled) {
+    if (result) results.push(result);
+  }
+
+  return results;
+}
+
+function isApiContentType(contentType: string, body: string, status: number): boolean {
+  // JSON responses are API
+  if (contentType.includes('json')) return true;
+  // XML/SOAP
+  if (contentType.includes('xml')) return true;
+  // 401/403 at an API path means something is there (but needs auth)
+  if ((status === 401 || status === 403) && !contentType.includes('html')) return true;
+  // GraphQL introspection response
+  if (body.includes('"data"') && body.includes('"__schema"')) return true;
+  // Check if body looks like JSON even without proper content-type
+  if (body.trim().startsWith('{') || body.trim().startsWith('[')) {
+    try {
+      JSON.parse(body);
+      return true;
+    } catch {
+      // Not JSON
+    }
+  }
+  return false;
+}

package/src/index.ts

@@ -0,0 +1,26 @@
+// src/index.ts
+export { capture, type CaptureOptions, type CaptureResult } from './capture/monitor.js';
+export { shouldCapture } from './capture/filter.js';
+export { isBlocklisted } from './capture/blocklist.js';
+export { isDomainMatch } from './capture/domain.js';
+export { scrubPII } from './capture/scrubber.js';
+export { SkillGenerator } from './skill/generator.js';
+export { writeSkillFile, readSkillFile, listSkillFiles } from './skill/store.js';
+export { signSkillFile, verifySignature } from './skill/signing.js';
+export { validateImport, importSkillFile } from './skill/importer.js';
+export { validateUrl, validateSkillFileUrls, resolveAndValidateUrl, resolveAndValidateSkillFileUrls } from './skill/ssrf.js';
+export { replayEndpoint, type ReplayResult } from './replay/engine.js';
+export { peek, read, type PeekOptions, type ReadOptions } from './read/index.js';
+export type { PeekResult, ReadResult, Decoder } from './read/types.js';
+export { AuthManager, getMachineId } from './auth/manager.js';
+export { parameterizePath, cleanFrameworkPath } from './capture/parameterize.js';
+export { detectPagination } from './capture/pagination.js';
+export { verifyEndpoints } from './capture/verifier.js';
+export { IdleTracker } from './capture/idle.js';
+export { isPathNoise } from './capture/filter.js';
+export { searchSkills, type SearchResult, type SearchResponse } from './skill/search.js';
+export { createPlugin, type Plugin, type ToolDefinition, type PluginOptions } from './plugin.js';
+export { shannonEntropy, isLikelyToken, parseJwtClaims, type TokenClassification, type JwtClaims } from './capture/entropy.js';
+export { isOAuthTokenRequest, type OAuthInfo } from './capture/oauth-detector.js';
+export { CaptureSession, type SessionOptions, type InteractionAction } from './capture/session.js';
+export type { SkillFile, SkillEndpoint, SkillSummary, CapturedExchange, StoredAuth, OAuthConfig, Replayability, PaginationInfo, PageSnapshot, PageElement, InteractionResult, FinishResult } from './types.js';

package/src/inspect/report.ts

@@ -0,0 +1,247 @@
+// src/inspect/report.ts
+import type { SkillFile, SkillEndpoint } from '../types.js';
+import { detectAntiBot, type AntiBotSignal } from '../capture/anti-bot.js';
+
+export interface InspectReport {
+  domain: string;
+  scanDuration: number;
+  totalRequests: number;
+  filteredRequests: number;
+  domBytes?: number;
+  endpoints: InspectEndpoint[];
+  antiBot: AntiBotSignal[];
+  summary: {
+    total: number;
+    replayable: number;
+    authRequired: number;
+    framework: string | null;
+    browserTokens: number;
+    replayTokens: number;
+    savingsPercent: number;
+  };
+}
+
+interface InspectEndpoint {
+  method: string;
+  path: string;
+  tier: string;
+  auth: string;
+  responseBytes: number;
+  responseShape: { type: string; fields?: string[] };
+  graphql: { operations: string[] } | null;
+  pagination: { type: string; paramName: string } | null;
+}
+
+/**
+ * Build an inspect report from capture results.
+ */
+export function buildInspectReport(options: {
+  skills: Map<string, SkillFile>;
+  totalRequests: number;
+  filteredRequests: number;
+  duration: number;
+  domBytes?: number;
+  antiBotSignals: AntiBotSignal[];
+  targetDomain: string;
+}): InspectReport {
+  const { skills, totalRequests, filteredRequests, duration, domBytes, antiBotSignals, targetDomain } = options;
+
+  // Merge all endpoints across domains for the report
+  const allEndpoints: InspectEndpoint[] = [];
+  let totalResponseBytes = 0;
+  let authCount = 0;
+
+  for (const skill of skills.values()) {
+    for (const ep of skill.endpoints) {
+      const auth = getAuthLabel(ep);
+      if (auth !== 'none') authCount++;
+
+      const respBytes = ep.responseBytes ?? 0;
+      totalResponseBytes += respBytes;
+
+      allEndpoints.push({
+        method: ep.method,
+        path: ep.path,
+        tier: ep.replayability?.tier ?? 'unknown',
+        auth,
+        responseBytes: respBytes,
+        responseShape: ep.responseShape,
+        graphql: isGraphQL(ep) ? { operations: [ep.id.replace(/^(get|post)-graphql-/, '')] } : null,
+        pagination: ep.pagination ? { type: ep.pagination.type, paramName: ep.pagination.paramName } : null,
+      });
+    }
+  }
+
+  const replayable = allEndpoints.filter(ep =>
+    ep.tier === 'green' || ep.tier === 'yellow',
+  ).length;
+
+  const browserTokens = domBytes ? Math.round(domBytes / 4) : 0;
+  const replayTokens = Math.round(totalResponseBytes / 4);
+  const savingsPercent = browserTokens > 0
+    ? Math.round((1 - replayTokens / browserTokens) * 1000) / 10
+    : 0;
+
+  const framework = detectFramework(allEndpoints);
+
+  return {
+    domain: targetDomain,
+    scanDuration: duration,
+    totalRequests,
+    filteredRequests,
+    domBytes,
+    endpoints: allEndpoints,
+    antiBot: antiBotSignals,
+    summary: {
+      total: allEndpoints.length,
+      replayable,
+      authRequired: authCount,
+      framework,
+      browserTokens,
+      replayTokens,
+      savingsPercent,
+    },
+  };
+}
+
+export function formatInspectHuman(report: InspectReport): string {
+  const lines: string[] = [
+    '',
+    `  ${report.domain} — API Discovery Report`,
+    '  ' + '═'.repeat(report.domain.length + 25),
+    '',
+    `  Scan duration:    ${report.scanDuration}s`,
+    `  Total requests:   ${report.totalRequests}`,
+    `  Filtered (noise): ${report.filteredRequests} (${pct(report.filteredRequests, report.totalRequests)})`,
+    `  API endpoints:    ${report.summary.total}` + endpointBreakdown(report.endpoints),
+    '',
+  ];
+
+  // Auth and anti-bot info
+  if (report.summary.authRequired > 0) {
+    lines.push(`  Auth required:    ${report.summary.authRequired} endpoints`);
+  } else {
+    lines.push('  Auth required:    None');
+  }
+
+  if (report.antiBot.length > 0) {
+    lines.push(`  Anti-bot:         ${report.antiBot.join(', ')} detected`);
+  } else {
+    lines.push('  Anti-bot:         None detected');
+  }
+
+  if (report.summary.framework) {
+    lines.push(`  Framework:        ${report.summary.framework}`);
+  }
+
+  lines.push('');
+
+  // Endpoint table
+  if (report.endpoints.length > 0) {
+    lines.push('  Endpoints:');
+    const methodW = 8;
+    const pathW = 32;
+    const tierW = 8;
+    const authW = 8;
+    const sizeW = 10;
+
+    const sep = '  ' + '─'.repeat(methodW + pathW + tierW + authW + sizeW);
+    lines.push(sep);
+    lines.push('  ' + pad('Method', methodW) + pad('Path', pathW) + pad('Tier', tierW) + pad('Auth', authW) + 'Size');
+    lines.push(sep);
+
+    for (const ep of report.endpoints) {
+      const pathStr = ep.path.length > pathW - 2 ? ep.path.slice(0, pathW - 3) + '…' : ep.path;
+      lines.push('  ' +
+        pad(ep.method, methodW) +
+        pad(pathStr, pathW) +
+        pad(ep.tier, tierW) +
+        pad(ep.auth, authW) +
+        formatBytes(ep.responseBytes),
+      );
+    }
+    lines.push(sep);
+  }
+
+  lines.push('');
+
+  // Data shapes
+  const shapedEndpoints = report.endpoints.filter(ep => ep.responseShape.fields?.length);
+  if (shapedEndpoints.length > 0) {
+    lines.push('  Data shapes:');
+    for (const ep of shapedEndpoints.slice(0, 5)) {
+      const fields = ep.responseShape.fields!.slice(0, 6).join(', ');
+      const suffix = ep.responseShape.fields!.length > 6 ? ', ...' : '';
+      lines.push(`    ${ep.method} ${ep.path} → { type: "${ep.responseShape.type}", fields: [${fields}${suffix}] }`);
+    }
+    lines.push('');
+  }
+
+  // Summary
+  lines.push('  Summary:');
+  lines.push(`    Replayable: ${report.summary.replayable} of ${report.summary.total} endpoints`);
+  if (report.domBytes && report.summary.browserTokens > 0) {
+    lines.push(`    DOM size:   ${formatBytes(report.domBytes)} = ${formatTokens(report.summary.browserTokens)} (what browser automation sends to LLM)`);
+    lines.push(`    API replay: ${formatTokens(report.summary.replayTokens)}`);
+    lines.push(`    Savings:    ${report.summary.savingsPercent}%`);
+    if (report.summary.savingsPercent < 0) {
+      lines.push('              API responses exceed DOM size — browser automation may be more token-efficient');
+    }
+  }
+  lines.push('');
+  lines.push(`  To capture these endpoints: apitap capture ${report.domain}`);
+  lines.push('');
+
+  return lines.join('\n');
+}
+
+function getAuthLabel(ep: SkillEndpoint): string {
+  const headers = ep.headers;
+  if (headers.authorization?.includes('[stored]')) return 'Bearer';
+  if (headers['x-api-key']?.includes('[stored]')) return 'API Key';
+  for (const [key, val] of Object.entries(headers)) {
+    if (val === '[stored]') return key;
+  }
+  return 'none';
+}
+
+function isGraphQL(ep: SkillEndpoint): boolean {
+  return ep.id.includes('graphql');
+}
+
+function detectFramework(endpoints: InspectEndpoint[]): string | null {
+  const paths = endpoints.map(e => e.path);
+  if (paths.some(p => p.includes('_next/'))) return 'Next.js';
+  if (paths.some(p => p.includes('__nuxt'))) return 'Nuxt';
+  if (endpoints.some(e => e.graphql)) return 'GraphQL';
+  return null;
+}
+
+function endpointBreakdown(endpoints: InspectEndpoint[]): string {
+  const rest = endpoints.filter(e => !e.graphql).length;
+  const gql = endpoints.filter(e => e.graphql).length;
+  if (gql > 0) return ` (${rest} REST, ${gql} GraphQL)`;
+  return '';
+}
+
+function pct(part: number, total: number): string {
+  if (total === 0) return '0%';
+  return `${Math.round((part / total) * 100)}%`;
+}
+
+function pad(s: string, width: number): string {
+  return s.length >= width ? s : s + ' '.repeat(width - s.length);
+}
+
+function formatBytes(bytes: number): string {
+  if (bytes >= 1_000_000) return `${(bytes / 1_000_000).toFixed(1)} MB`;
+  if (bytes >= 1_000) return `${(bytes / 1_000).toFixed(1)} KB`;
+  if (bytes > 0) return `${bytes} B`;
+  return '-';
+}
+
+function formatTokens(tokens: number): string {
+  if (tokens >= 1_000_000) return `${(tokens / 1_000_000).toFixed(2)}M tokens`;
+  if (tokens >= 1_000) return `${(tokens / 1_000).toFixed(1)}K tokens`;
+  return `${tokens} tokens`;
+}