@apitap/core 1.5.2 → 1.5.4

package/src/capture/parameterize.ts

@@ -3,10 +3,129 @@
 const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
 const PURE_NUMERIC_RE = /^\d+$/;
 const LONG_DIGITS_RE = /\d{8,}/;
+const LONG_HEX_RE = /^[0-9a-f]{16,}$/i;
 const NEXT_DATA_PREFIX_RE = /^\/_next\/data\/[^/]+\//;
 
 /**
- * Check if a path segment is a dynamic value that should be parameterized.
+ * Resource nouns: segments that name a collection of resources.
+ * The value is the param name(s) for the slot(s) that follow.
+ * e.g. "repos" expects two dynamic segments: :owner/:repo
+ */
+const RESOURCE_NOUNS = new Map<string, string[]>([
+  // Code hosting / generic REST
+  ['repos',         [':owner', ':repo']],
+  ['users',         [':username']],
+  ['orgs',          [':org']],
+  ['organizations', [':org']],
+  ['teams',         [':team']],
+  ['members',       [':member']],
+  ['projects',      [':project']],
+  ['gists',         [':gist_id']],
+  ['issues',        [':issue_number']],
+  ['pulls',         [':pull_number']],
+  ['commits',       [':sha']],
+  ['branches',      [':branch']],
+  ['tags',          [':tag']],
+  ['releases',      [':release_id']],
+  ['milestones',    [':milestone']],
+  ['labels',        [':label']],
+  ['hooks',         [':hook_id']],
+  ['keys',          [':key_id']],
+  ['deployments',   [':deployment_id']],
+  ['environments',  [':env']],
+  ['runs',          [':run_id']],
+  ['jobs',          [':job_id']],
+  ['artifacts',     [':artifact_id']],
+  ['packages',      [':package']],
+
+  // Content / social
+  ['posts',         [':post_id']],
+  ['comments',      [':comment_id']],
+  ['articles',      [':article_id']],
+  ['stories',       [':story_id']],
+  ['threads',       [':thread_id']],
+  ['messages',      [':message_id']],
+  ['channels',      [':channel']],
+  ['videos',        [':video_id']],
+  ['playlists',     [':playlist_id']],
+  ['tracks',        [':track_id']],
+  ['albums',        [':album_id']],
+  ['artists',       [':artist_id']],
+  ['images',        [':image_id']],
+  ['files',         [':file_id']],
+  ['documents',     [':doc_id']],
+  ['folders',       [':folder_id']],
+  ['collections',   [':collection_id']],
+  ['categories',    [':category']],
+
+  // E-commerce
+  ['products',      [':product_id']],
+  ['items',         [':item_id']],
+  ['orders',        [':order_id']],
+  ['customers',     [':customer_id']],
+  ['carts',         [':cart_id']],
+  ['stores',        [':store_id']],
+  ['reviews',       [':review_id']],
+
+  // Infrastructure / ops
+  ['accounts',      [':account_id']],
+  ['workspaces',    [':workspace']],
+  ['databases',     [':database']],
+  ['tables',        [':table']],
+  ['namespaces',    [':namespace']],
+  ['clusters',      [':cluster']],
+  ['instances',     [':instance']],
+  ['regions',       [':region']],
+  ['zones',         [':zone']],
+  ['resources',     [':resource_id']],
+  ['subscriptions', [':subscription_id']],
+  ['tenants',       [':tenant_id']],
+  ['groups',        [':group_id']],
+  ['roles',         [':role']],
+  ['policies',      [':policy']],
+  ['tokens',        [':token_id']],
+  ['sessions',      [':session_id']],
+  ['events',        [':event_id']],
+  ['logs',          [':log_id']],
+  ['metrics',       [':metric']],
+  ['alerts',        [':alert_id']],
+  ['notifications', [':notification_id']],
+  ['webhooks',      [':webhook_id']],
+
+  // Media
+  ['media',         [':media_id']],
+  ['assets',        [':asset_id']],
+  ['uploads',       [':upload_id']],
+]);
+
+/**
+ * Segments that are always structural (never parameterized).
+ * Includes version prefixes, action verbs, and all RESOURCE_NOUNS keys.
+ */
+const STRUCTURAL_SEGMENTS = new Set<string>([
+  // Version prefixes
+  'api', 'v1', 'v2', 'v3', 'v4', 'v5', 'v6', 'v7', 'v8', 'v9', 'v10',
+  // Actions / sub-resources
+  'search', 'filter', 'sort', 'query', 'list', 'create', 'update', 'delete',
+  'status', 'config', 'settings', 'preferences', 'profile', 'info', 'details',
+  'stats', 'analytics', 'count', 'batch', 'bulk', 'export', 'import',
+  'auth', 'login', 'logout', 'register', 'signup', 'signin', 'callback',
+  'oauth', 'token', 'refresh', 'verify', 'confirm', 'reset', 'activate',
+  'public', 'private', 'internal', 'external', 'admin', 'management',
+  'graphql', 'gql', 'rest', 'rpc', 'ws', 'websocket', 'stream', 'feed',
+  'health', 'ping', 'version', 'manifest', 'metadata', 'schema',
+  'upload', 'download', 'preview', 'thumbnail', 'embed',
+  'latest', 'trending', 'popular', 'featured', 'recommended', 'top', 'new',
+  'web', 'app', 'mobile', 'desktop', 'data', 'raw', 'render',
+  'consent', 'wrapper', 'widget', 'integrity', 'pathfinder', 'rum',
+  // All resource noun keys are also structural — this means a noun after itself
+  // (e.g. /repos/repos) treats the second "repos" as structural, not a param slot.
+  // That's a degenerate case and the correct behavior: we preserve the literal.
+  ...RESOURCE_NOUNS.keys(),
+]);
+
+/**
+ * Check if a path segment is a dynamic value based on its structure alone.
  * Returns the parameter name (:id, :hash, :slug) or null if static.
  */
 function classifySegment(segment: string): string | null {
@@ -16,6 +135,9 @@ function classifySegment(segment: string): string | null {
   // UUID → :id
   if (UUID_RE.test(segment)) return ':id';
 
+  // Long hex string (16+ hex chars) → :hash
+  if (LONG_HEX_RE.test(segment)) return ':hash';
+
   // Slug with embedded long number (8+ consecutive digits) — check before hash
   // because slugs like "btc-updown-15m-1770254100" would also match the hash rule
   if (LONG_DIGITS_RE.test(segment)) {
@@ -34,20 +156,94 @@ function classifySegment(segment: string): string | null {
 }
 
 /**
- * Replace dynamic path segments with :param placeholders.
+ * Check if a segment looks like a lowercase word or hyphenated compound word.
+ * e.g. "search", "location-metadata", "top-rated" → true
+ * e.g. "n1byn1kt", "OxItOzEC", "ABC-123" → false
+ */
+function looksLikeWord(segment: string): boolean {
+  return /^[a-z][a-z-]*[a-z]$/.test(segment)
+    && segment.split('-').every(part => /^[a-z]{2,}$/.test(part));
+}
+
+/**
+ * Replace dynamic path segments with named :param placeholders.
  *
- * Rules:
- * - Pure numeric → :id
- * - UUID → :id
- * - 12+ alphanum with mixed letters+digits → :hash
- * - Contains 8+ consecutive digits → :slug
+ * Three-layer approach:
+ * 1. Structural detection: UUIDs, numbers, hashes, long-digit slugs
+ * 2. Context-aware: segments following a known resource noun get a
+ *    semantically named param (e.g. /repos/:owner/:repo)
+ * 3. Heuristic fallback: non-word segments after structural segments
+ *    are parameterized as :id
  */
 export function parameterizePath(path: string): string {
   const segments = path.split('/');
-  const result = segments.map(seg => {
-    if (seg === '') return seg;
-    return classifySegment(seg) ?? seg;
-  });
+  const result: string[] = [];
+  let nounSlots: string[] = [];
+
+  for (let i = 0; i < segments.length; i++) {
+    const seg = segments[i];
+
+    // Preserve empty segments (leading/trailing slashes)
+    if (seg === '') { result.push(seg); continue; }
+
+    const lower = seg.toLowerCase();
+
+    // Layer 1: Always parameterize structurally obvious dynamic values
+    // Order matters: pure numeric before long-digits (1770254100 is numeric, not a slug)
+    if (UUID_RE.test(seg)) { result.push(':id'); nounSlots = []; continue; }
+
+    if (PURE_NUMERIC_RE.test(seg)) {
+      // Use noun-derived name if available, otherwise :id
+      const name = nounSlots.length > 0 ? nounSlots.shift()! : ':id';
+      result.push(name);
+      continue;
+    }
+
+    if (LONG_HEX_RE.test(seg)) { result.push(':hash'); nounSlots = []; continue; }
+
+    // Slug with embedded long number (8+ digits mixed with text)
+    // Pure-numeric already handled above, so this only fires on mixed segments
+    if (LONG_DIGITS_RE.test(seg)) { result.push(':slug'); nounSlots = []; continue; }
+
+    // Hash-like: 12+ mixed alphanum (catches remaining patterns)
+    const structural = classifySegment(seg);
+    if (structural) { result.push(structural); nounSlots = []; continue; }
+
+    // Layer 2: Known resource noun → keep it, queue param names for following segments
+    if (RESOURCE_NOUNS.has(lower)) {
+      result.push(seg);
+      nounSlots = [...RESOURCE_NOUNS.get(lower)!];
+      continue;
+    }
+
+    // Structural segment → keep as-is, reset slots
+    if (STRUCTURAL_SEGMENTS.has(lower)) {
+      result.push(seg);
+      nounSlots = [];
+      continue;
+    }
+
+    // Fill a queued noun slot (e.g. "n1byn1kt" after "repos")
+    if (nounSlots.length > 0) {
+      result.push(nounSlots.shift()!);
+      continue;
+    }
+
+    // Layer 3: Heuristic — segment after a structural segment that doesn't
+    // look like a plain English word is likely a dynamic value
+    const prevSeg = i > 0 ? segments[i - 1]?.toLowerCase() : '';
+    const prevIsStructural = STRUCTURAL_SEGMENTS.has(prevSeg) || RESOURCE_NOUNS.has(prevSeg);
+
+    if (prevIsStructural && seg.length >= 2 && !looksLikeWord(seg)) {
+      result.push(':id');
+      continue;
+    }
+
+    // Default: keep as-is
+    result.push(seg);
+    nounSlots = [];
+  }
+
   return result.join('/');
 }
 

package/src/capture/session.ts

@@ -4,7 +4,7 @@ import { randomUUID } from 'node:crypto';
 import { shouldCapture } from './filter.js';
 import { launchBrowser, normalizeCookiesForStorageState } from './browser.js';
 import { isDomainMatch } from './domain.js';
-import { SkillGenerator, type GeneratorOptions } from '../skill/generator.js';
+import { SkillGenerator, deduplicateAuth, type GeneratorOptions } from '../skill/generator.js';
 import { detectCaptcha } from '../auth/refresh.js';
 import { verifyEndpoints } from './verifier.js';
 import { signSkillFile } from '../skill/signing.js';
@@ -133,11 +133,21 @@ export class CaptureSession {
         case 'navigate': {
           if (!action.url) return { success: false, error: 'url required for navigate', snapshot: await this.takeSnapshot() };
 
-          // M7 fix: Full SSRF validation on navigate URLs (same checks as ssrf.ts)
-          const { validateUrl: validateNavUrl } = await import('../skill/ssrf.js');
-          const navResult = validateNavUrl(action.url);
-          if (!navResult.safe) {
-            return { success: false, error: `Navigation blocked: ${navResult.reason}`, snapshot: await this.takeSnapshot() };
+          // M7 fix: SSRF validation on navigate URLs — skip for same-origin
+          // (session.start() already navigated to targetUrl, so same-origin is trusted)
+          let sameOrigin = false;
+          try {
+            sameOrigin = this.targetUrl !== '' && new URL(action.url).origin === new URL(this.targetUrl).origin;
+          } catch {
+            // Invalid URL — let validateUrl produce the error
+          }
+
+          if (!sameOrigin) {
+            const { validateUrl: validateNavUrl } = await import('../skill/ssrf.js');
+            const navResult = validateNavUrl(action.url);
+            if (!navResult.safe) {
+              return { success: false, error: `Navigation blocked: ${navResult.reason}`, snapshot: await this.takeSnapshot() };
+            }
           }
 
           await this.page.goto(action.url, { waitUntil: 'domcontentloaded' });
@@ -208,10 +218,10 @@ export class CaptureSession {
 
       if (skill.endpoints.length === 0) continue;
 
-      // Store extracted auth
-      const extractedAuth = generator.getExtractedAuth();
-      if (extractedAuth.length > 0) {
-        await authManager.store(domain, extractedAuth[0]);
+      // Store extracted auth credentials
+      const auth = deduplicateAuth(generator.getExtractedAuth());
+      if (auth) {
+        await authManager.store(domain, auth);
       }
 
       // Store OAuth credentials if detected

package/src/cli.ts

@@ -26,6 +26,7 @@ import { readFileSync } from 'node:fs';
 import { stat, unlink } from 'node:fs/promises';
 import { fileURLToPath } from 'node:url';
 import { createMcpServer } from './mcp.js';
+import { attach, parseDomainPatterns } from './capture/cdp-attach.js';
 
 const __dirname = fileURLToPath(new URL('.', import.meta.url));
 const pkg = JSON.parse(readFileSync(join(__dirname, '..', 'package.json'), 'utf-8'));
@@ -66,6 +67,8 @@ function printUsage(): void {
 
   Usage:
     apitap capture <url>       Capture API traffic from a website
+    apitap attach [--port 9222] [--domain *.github.com]
+                               Attach to running Chrome and capture API traffic
     apitap discover <url>      Detect APIs without a browser (fast recon)
     apitap inspect <url>       Discover APIs without saving (X-ray vision)
     apitap search <query>      Search skill files for a domain or endpoint
@@ -1233,6 +1236,20 @@ async function handleExtension(positional: string[], flags: Record<string, strin
   process.exit(1);
 }
 
+async function handleAttach(_positional: string[], flags: Record<string, string | boolean>): Promise<void> {
+  const port = typeof flags.port === 'string' ? parseInt(flags.port, 10) : 9222;
+  const domainPatterns = parseDomainPatterns(
+    typeof flags.domain === 'string' ? flags.domain : undefined,
+  );
+  const json = flags.json === true;
+
+  const result = await attach({ port, domainPatterns, json });
+
+  if (json) {
+    console.log(JSON.stringify(result, null, 2));
+  }
+}
+
 async function main(): Promise<void> {
   const { command, positional, flags } = parseArgs(process.argv.slice(2));
 
@@ -1300,6 +1317,9 @@ async function main(): Promise<void> {
     case 'extension':
       await handleExtension(positional, flags);
       break;
+    case 'attach':
+      await handleAttach(positional, flags);
+      break;
     default:
       printUsage();
   }

package/src/index.ts

@@ -14,6 +14,7 @@ export { peek, read, type PeekOptions, type ReadOptions } from './read/index.js'
 export type { PeekResult, ReadResult, Decoder } from './read/types.js';
 export { AuthManager, getMachineId } from './auth/manager.js';
 export { parameterizePath, cleanFrameworkPath } from './capture/parameterize.js';
+export { attach, matchesDomainGlob, parseDomainPatterns } from './capture/cdp-attach.js';
 export { detectPagination } from './capture/pagination.js';
 export { verifyEndpoints } from './capture/verifier.js';
 export { IdleTracker } from './capture/idle.js';

package/src/mcp.ts

@@ -265,6 +265,12 @@ export function createMcpServer(options: McpServerOptions = {}): McpServer {
       },
     },
     async ({ requests, maxBytes }) => {
+      // Consume one rate-limit token per batch item
+      for (let i = 0; i < requests.length; i++) {
+        if (!rateLimiter.check()) {
+          return { content: [{ type: 'text' as const, text: `Rate limit exceeded after ${i} of ${requests.length} items. Try again in a moment.` }], isError: true };
+        }
+      }
       const { replayMultiple } = await import('./replay/engine.js');
       const typed = requests.map(r => ({
         domain: r.domain,
@@ -334,6 +340,9 @@ export function createMcpServer(options: McpServerOptions = {}): McpServer {
       },
     },
     async ({ url }) => {
+      if (!rateLimiter.check()) {
+        return { content: [{ type: 'text' as const, text: 'Rate limit exceeded. Try again in a moment.' }], isError: true };
+      }
       try {
         if (!options._skipSsrfCheck) {
           const validation = await resolveAndValidateUrl(url);
@@ -415,6 +424,9 @@ export function createMcpServer(options: McpServerOptions = {}): McpServer {
       },
     },
     async ({ url, duration }) => {
+      if (!rateLimiter.check()) {
+        return { content: [{ type: 'text' as const, text: 'Rate limit exceeded. Try again in a moment.' }], isError: true };
+      }
       if (!options._skipSsrfCheck) {
         const validation = await resolveAndValidateUrl(url);
         if (!validation.safe) {

package/src/native-host.ts

@@ -237,6 +237,13 @@ export async function startSocketServer(
 
       conn.on('data', (chunk) => {
         buffer += chunk.toString();
+
+        // Guard against unbounded buffer growth (max 10MB)
+        if (buffer.length > 10 * 1024 * 1024) {
+          conn.destroy();
+          return;
+        }
+
         const newlineIdx = buffer.indexOf('\n');
         if (newlineIdx === -1) return;
 
@@ -299,7 +306,7 @@ function readMessage(): Promise<NativeRequest | null> {
       }
 
       const messageLength = headerBuf.readUInt32LE(0);
-      if (messageLength > 1024 * 1024) {
+      if (messageLength > 10 * 1024 * 1024) {
         process.stderr.write(`Message too large: ${messageLength}\n`);
         resolve(null);
         return;
@@ -416,6 +423,10 @@ if (isMainModule) {
 
       // Otherwise, handle as a direct extension message (save_skill, etc.)
       const response = await handleNativeMessage(message);
+      // Echo _portMsgId so extension can match response to request
+      if ((message as any)._portMsgId) {
+        (response as any)._portMsgId = (message as any)._portMsgId;
+      }
       sendMessage(response);
     }
 

package/src/plugin.ts

@@ -3,6 +3,7 @@ import { searchSkills } from './skill/search.js';
 import { readSkillFile } from './skill/store.js';
 import { replayEndpoint } from './replay/engine.js';
 import { AuthManager, getMachineId } from './auth/manager.js';
+import { deriveSigningKey } from './auth/crypto.js';
 import { homedir } from 'node:os';
 import { join } from 'node:path';
 
@@ -29,7 +30,11 @@ const APITAP_DIR = join(homedir(), '.apitap');
 
 /** M20: Mark plugin responses as untrusted external content */
 function wrapUntrusted(data: unknown): unknown {
-  return { ...data as Record<string, unknown>, _meta: { externalContent: { untrusted: true } } };
+  const meta = { externalContent: { untrusted: true } };
+  if (Array.isArray(data)) {
+    return { results: data, _meta: meta };
+  }
+  return { ...data as Record<string, unknown>, _meta: meta };
 }
 
 export function createPlugin(options: PluginOptions = {}): Plugin {
@@ -93,7 +98,9 @@ export function createPlugin(options: PluginOptions = {}): Plugin {
       const endpointId = args.endpointId as string;
       const params = args.params as Record<string, string> | undefined;
 
-      const skill = await readSkillFile(domain, skillsDir, { trustUnsigned: true });
+      const machineId = await getMachineId();
+      const signingKey = deriveSigningKey(machineId);
+      const skill = await readSkillFile(domain, skillsDir, { verifySignature: true, signingKey, trustUnsigned: true });
       if (!skill) {
         return {
           error: `No skill file found for "${domain}". Use apitap_capture to capture it first.`,
@@ -159,7 +166,7 @@ export function createPlugin(options: PluginOptions = {}): Plugin {
       const { promisify } = await import('node:util');
       const execFileAsync = promisify(execFile);
 
-      const cliArgs = ['--import', 'tsx', 'src/cli.ts', 'capture', url, '--duration', String(duration), '--json', '--no-verify'];
+      const cliArgs = ['--import', 'tsx', 'src/cli.ts', 'capture', url, '--duration', String(duration), '--json'];
       if (allDomains) cliArgs.push('--all-domains');
 
       try {

package/src/skill/generator.ts

@@ -46,6 +46,18 @@ const STRIP_HEADERS = new Set([
   'cookie',
 ]);
 
+/** Headers that should never be treated as auth by entropy detection. */
+const NOT_AUTH_HEADERS = new Set([
+  'referer', 'user-agent', 'content-type', 'accept', 'accept-language',
+  'origin', 'host', 'content-length', 'cache-control', 'pragma', 'if-none-match',
+  'if-modified-since', 'dnt', 'upgrade-insecure-requests',
+  // Observability/tracing (high entropy but not auth)
+  'traceparent', 'tracestate', 'tracecontext', 'newrelic', 'sentry-trace',
+  'baggage', 'x-request-id', 'x-correlation-id', 'x-trace-id', 'x-span-id',
+  'x-datadog-trace-id', 'x-datadog-parent-id', 'x-datadog-sampling-priority',
+  'x-amzn-trace-id', 'x-cloud-trace-context',
+]);
+
 const AUTH_HEADERS = new Set([
   'authorization',
   'x-api-key',
@@ -107,7 +119,8 @@ function extractAuth(headers: Record<string, string>): [StoredAuth[], Set<string
       });
     } else if (lower === 'x-api-key' && value) {
       auth.push({ type: 'api-key', header: lower, value });
-    } else if (!AUTH_HEADERS.has(lower) && value) {
+    } else if (!AUTH_HEADERS.has(lower) && !STRIP_HEADERS.has(lower) && !NOT_AUTH_HEADERS.has(lower)
+      && !lower.startsWith('sec-') && value) {
       // Entropy-based detection for non-standard headers
       const classification = isLikelyToken(lower, value);
       if (classification.isToken) {
@@ -119,6 +132,25 @@ function extractAuth(headers: Record<string, string>): [StoredAuth[], Set<string
   return [auth, entropyDetected];
 }
 
+/**
+ * Deduplicate extracted auth entries by header name and build a StoredAuth
+ * object with all unique headers. Entries are expected to be pre-sorted
+ * by priority (bearer > api-key > custom) via getExtractedAuth().
+ */
+export function deduplicateAuth(extractedAuth: StoredAuth[]): StoredAuth | null {
+  if (extractedAuth.length === 0) return null;
+  const seen = new Set<string>();
+  const headers: Array<{ header: string; value: string }> = [];
+  for (const a of extractedAuth) {
+    if (!seen.has(a.header)) {
+      seen.add(a.header);
+      headers.push({ header: a.header, value: a.value });
+    }
+  }
+  const primary = extractedAuth[0];
+  return { type: primary.type, header: primary.header, value: primary.value, headers };
+}
+
 function generateEndpointId(method: string, parameterizedPath: string): string {
   // Clean framework noise for the ID (but not for the stored path)
   let cleaned = cleanFrameworkPath(parameterizedPath);
@@ -476,9 +508,12 @@ export class SkillGenerator {
     this.filteredCount++;
   }
 
-  /** Get auth credentials extracted during capture. */
+  /** Get auth credentials extracted during capture, prioritized by type. */
   getExtractedAuth(): StoredAuth[] {
-    return this.extractedAuthList;
+    const priority: Record<string, number> = { bearer: 0, 'api-key': 1, custom: 2 };
+    return [...this.extractedAuthList].sort(
+      (a, b) => (priority[a.type] ?? 3) - (priority[b.type] ?? 3),
+    );
   }
 
   /** Mark this domain as having captcha risk (detected during capture). */