@apitap/core 1.5.3 → 1.6.0

package/src/plugin.ts

@@ -3,6 +3,7 @@ import { searchSkills } from './skill/search.js';
 import { readSkillFile } from './skill/store.js';
 import { replayEndpoint } from './replay/engine.js';
 import { AuthManager, getMachineId } from './auth/manager.js';
+import { deriveSigningKey } from './auth/crypto.js';
 import { homedir } from 'node:os';
 import { join } from 'node:path';
 
@@ -29,7 +30,11 @@ const APITAP_DIR = join(homedir(), '.apitap');
 
 /** M20: Mark plugin responses as untrusted external content */
 function wrapUntrusted(data: unknown): unknown {
-  return { ...data as Record<string, unknown>, _meta: { externalContent: { untrusted: true } } };
+  const meta = { externalContent: { untrusted: true } };
+  if (Array.isArray(data)) {
+    return { results: data, _meta: meta };
+  }
+  return { ...data as Record<string, unknown>, _meta: meta };
 }
 
 export function createPlugin(options: PluginOptions = {}): Plugin {
@@ -93,7 +98,9 @@ export function createPlugin(options: PluginOptions = {}): Plugin {
       const endpointId = args.endpointId as string;
       const params = args.params as Record<string, string> | undefined;
 
-      const skill = await readSkillFile(domain, skillsDir, { trustUnsigned: true });
+      const machineId = await getMachineId();
+      const signingKey = deriveSigningKey(machineId);
+      const skill = await readSkillFile(domain, skillsDir, { verifySignature: true, signingKey, trustUnsigned: true });
       if (!skill) {
         return {
           error: `No skill file found for "${domain}". Use apitap_capture to capture it first.`,
@@ -159,7 +166,7 @@ export function createPlugin(options: PluginOptions = {}): Plugin {
       const { promisify } = await import('node:util');
       const execFileAsync = promisify(execFile);
 
-      const cliArgs = ['--import', 'tsx', 'src/cli.ts', 'capture', url, '--duration', String(duration), '--json', '--no-verify'];
+      const cliArgs = ['--import', 'tsx', 'src/cli.ts', 'capture', url, '--duration', String(duration), '--json'];
       if (allDomains) cliArgs.push('--all-domains');
 
       try {

package/src/replay/engine.ts

@@ -204,6 +204,24 @@ function wrapAuthError(
   };
 }
 
+/**
+ * Returns a user-facing hint about confidence level, or null if confidence is high enough.
+ */
+export function getConfidenceHint(confidence: number | undefined): string | null {
+  const c = confidence ?? 1.0;
+  if (c >= 0.85) return null;
+  if (c >= 0.7) return '(imported from spec — params may need adjustment)';
+  return '(imported from spec — provide params explicitly, no captured examples available)';
+}
+
+/**
+ * Returns true if a query param should be omitted from the request.
+ * Omits spec-derived params that have no real example value.
+ */
+export function shouldOmitQueryParam(param: { type: string; example: string; fromSpec?: boolean }): boolean {
+  return param.fromSpec === true && param.example === '';
+}
+
 /**
  * Replay a captured API endpoint.
  *
@@ -240,6 +258,7 @@ export async function replayEndpoint(
 
   // Apply query params: start with captured defaults, override with provided params
   for (const [key, val] of Object.entries(endpoint.queryParams)) {
+    if (shouldOmitQueryParam(val)) continue;
     url.searchParams.set(key, val.example);
   }
   if (params) {

package/src/skill/apis-guru.ts

@@ -0,0 +1,163 @@
+// src/skill/apis-guru.ts
+import { resolveAndValidateUrl } from '../skill/ssrf.js';
+
+export interface ApisGuruEntry {
+  apiId: string;           // e.g., "twilio.com:api"
+  providerName: string;    // e.g., "twilio.com"
+  title: string;
+  specUrl: string;         // direct URL to OpenAPI JSON spec
+  openapiVer: string;
+  updated: string;
+}
+
+const APIS_GURU_LIST_URL = 'https://api.apis.guru/v2/list.json';
+
+/**
+ * Parse raw APIs.guru list.json response into ApisGuruEntry array.
+ * For each API, use the preferred version's data.
+ */
+export function parseApisGuruList(raw: Record<string, any>): ApisGuruEntry[] {
+  const entries: ApisGuruEntry[] = [];
+
+  for (const apiId of Object.keys(raw)) {
+    const apiData = raw[apiId];
+    if (!apiData || typeof apiData !== 'object') continue;
+
+    const preferred: string = apiData.preferred;
+    if (!preferred) continue;
+
+    const versions = apiData.versions;
+    if (!versions || typeof versions !== 'object') continue;
+
+    const versionData = versions[preferred];
+    if (!versionData || typeof versionData !== 'object') continue;
+
+    const swaggerUrl: string | undefined = versionData.swaggerUrl;
+    if (!swaggerUrl) continue;
+
+    const info = versionData.info ?? {};
+    const title: string = info.title ?? '';
+    const openapiVer: string = versionData.openapiVer ?? '';
+    const updated: string = versionData.updated ?? '';
+
+    // providerName: prefer info.x-providerName, else split apiId on ':', else use apiId
+    let providerName: string;
+    if (info['x-providerName']) {
+      providerName = info['x-providerName'];
+    } else {
+      const colonIdx = apiId.indexOf(':');
+      providerName = colonIdx >= 0 ? apiId.slice(0, colonIdx) : apiId;
+    }
+
+    entries.push({
+      apiId,
+      providerName,
+      title,
+      specUrl: swaggerUrl,
+      openapiVer,
+      updated,
+    });
+  }
+
+  return entries;
+}
+
+export interface FilterOptions {
+  search?: string;
+  limit?: number;
+  noAuthOnly?: boolean;
+  preferOpenapi3?: boolean;
+}
+
+/**
+ * Filter and sort ApisGuruEntry array.
+ * - search: substring match (case-insensitive) on providerName or title
+ * - preferOpenapi3: sort 3.x entries before 2.x, then by recency within groups
+ * - default sort: by recency (updated desc)
+ * - limit: cap result count
+ */
+export function filterEntries(
+  entries: ApisGuruEntry[],
+  options: FilterOptions,
+): ApisGuruEntry[] {
+  const { search, limit, preferOpenapi3 } = options;
+
+  let result = entries;
+
+  // Filter by search term
+  if (search) {
+    const lower = search.toLowerCase();
+    result = result.filter(
+      e =>
+        e.providerName.toLowerCase().includes(lower) ||
+        e.title.toLowerCase().includes(lower),
+    );
+  }
+
+  // Sort
+  result = [...result].sort((a, b) => {
+    if (preferOpenapi3) {
+      const aIs3 = a.openapiVer.startsWith('3') ? 0 : 1;
+      const bIs3 = b.openapiVer.startsWith('3') ? 0 : 1;
+      if (aIs3 !== bIs3) return aIs3 - bIs3;
+    }
+    // Within same group (or when not preferring 3.x), sort by recency desc
+    return b.updated.localeCompare(a.updated);
+  });
+
+  // Apply limit
+  if (typeof limit === 'number' && limit > 0) {
+    result = result.slice(0, limit);
+  }
+
+  return result;
+}
+
+/**
+ * Fetch the APIs.guru list.json and parse it into ApisGuruEntry array.
+ */
+export async function fetchApisGuruList(): Promise<ApisGuruEntry[]> {
+  const ssrf = await resolveAndValidateUrl(APIS_GURU_LIST_URL);
+  if (!ssrf.safe) {
+    throw new Error(`SSRF check failed for APIs.guru list URL: ${ssrf.reason}`);
+  }
+  const response = await fetch(APIS_GURU_LIST_URL, {
+    signal: AbortSignal.timeout(30_000),
+  });
+  if (!response.ok) {
+    throw new Error(
+      `Failed to fetch APIs.guru list: ${response.status} ${response.statusText}`,
+    );
+  }
+  const text = await response.text();
+  try {
+    return parseApisGuruList(JSON.parse(text) as Record<string, any>);
+  } catch {
+    throw new Error(`Invalid JSON from ${APIS_GURU_LIST_URL}: ${text.slice(0, 100)}`);
+  }
+}
+
+/**
+ * Fetch a single OpenAPI spec by URL and return the parsed JSON.
+ */
+export async function fetchSpec(specUrl: string): Promise<Record<string, any>> {
+  const ssrf = await resolveAndValidateUrl(specUrl);
+  if (!ssrf.safe) {
+    throw new Error(`SSRF check failed for spec URL ${specUrl}: ${ssrf.reason}`);
+  }
+  const response = await fetch(specUrl, {
+    signal: AbortSignal.timeout(30_000),
+    redirect: 'error',
+  });
+  if (!response.ok) {
+    throw new Error(
+      `Failed to fetch spec at ${specUrl}: ${response.status} ${response.statusText}`,
+    );
+  }
+  const text = await response.text();
+  try {
+    return JSON.parse(text) as Record<string, any>;
+  } catch {
+    throw new Error(`Invalid JSON from ${specUrl}: ${text.slice(0, 100)}`);
+  }
+}

package/src/skill/generator.ts

@@ -46,6 +46,18 @@ const STRIP_HEADERS = new Set([
   'cookie',
 ]);
 
+/** Headers that should never be treated as auth by entropy detection. */
+const NOT_AUTH_HEADERS = new Set([
+  'referer', 'user-agent', 'content-type', 'accept', 'accept-language',
+  'origin', 'host', 'content-length', 'cache-control', 'pragma', 'if-none-match',
+  'if-modified-since', 'dnt', 'upgrade-insecure-requests',
+  // Observability/tracing (high entropy but not auth)
+  'traceparent', 'tracestate', 'tracecontext', 'newrelic', 'sentry-trace',
+  'baggage', 'x-request-id', 'x-correlation-id', 'x-trace-id', 'x-span-id',
+  'x-datadog-trace-id', 'x-datadog-parent-id', 'x-datadog-sampling-priority',
+  'x-amzn-trace-id', 'x-cloud-trace-context',
+]);
+
 const AUTH_HEADERS = new Set([
   'authorization',
   'x-api-key',
@@ -107,7 +119,8 @@ function extractAuth(headers: Record<string, string>): [StoredAuth[], Set<string
       });
     } else if (lower === 'x-api-key' && value) {
       auth.push({ type: 'api-key', header: lower, value });
-    } else if (!AUTH_HEADERS.has(lower) && value) {
+    } else if (!AUTH_HEADERS.has(lower) && !STRIP_HEADERS.has(lower) && !NOT_AUTH_HEADERS.has(lower)
+      && !lower.startsWith('sec-') && value) {
       // Entropy-based detection for non-standard headers
       const classification = isLikelyToken(lower, value);
       if (classification.isToken) {
@@ -119,6 +132,25 @@ function extractAuth(headers: Record<string, string>): [StoredAuth[], Set<string
   return [auth, entropyDetected];
 }
 
+/**
+ * Deduplicate extracted auth entries by header name and build a StoredAuth
+ * object with all unique headers. Entries are expected to be pre-sorted
+ * by priority (bearer > api-key > custom) via getExtractedAuth().
+ */
+export function deduplicateAuth(extractedAuth: StoredAuth[]): StoredAuth | null {
+  if (extractedAuth.length === 0) return null;
+  const seen = new Set<string>();
+  const headers: Array<{ header: string; value: string }> = [];
+  for (const a of extractedAuth) {
+    if (!seen.has(a.header)) {
+      seen.add(a.header);
+      headers.push({ header: a.header, value: a.value });
+    }
+  }
+  const primary = extractedAuth[0];
+  return { type: primary.type, header: primary.header, value: primary.value, headers };
+}
+
 function generateEndpointId(method: string, parameterizedPath: string): string {
   // Clean framework noise for the ID (but not for the stored path)
   let cleaned = cleanFrameworkPath(parameterizedPath);
@@ -476,9 +508,12 @@ export class SkillGenerator {
     this.filteredCount++;
   }
 
-  /** Get auth credentials extracted during capture. */
+  /** Get auth credentials extracted during capture, prioritized by type. */
   getExtractedAuth(): StoredAuth[] {
-    return this.extractedAuthList;
+    const priority: Record<string, number> = { bearer: 0, 'api-key': 1, custom: 2 };
+    return [...this.extractedAuthList].sort(
+      (a, b) => (priority[a.type] ?? 3) - (priority[b.type] ?? 3),
+    );
   }
 
   /** Mark this domain as having captcha risk (detected during capture). */

package/src/skill/merge.ts

@@ -0,0 +1,281 @@
+// src/skill/merge.ts
+import type { SkillFile, SkillEndpoint, ImportMeta, MergeResult } from '../types.js';
+
+/**
+ * Normalize a parameterized path by replacing all named param placeholders
+ * with the generic `:_` placeholder, enabling matching across different param
+ * naming conventions (e.g. `:id` vs `:userId` vs `:user_id`).
+ *
+ * @example
+ *   normalizePath('/repos/:owner/:repo') // → '/repos/:_/:_'
+ *   normalizePath('/users/list')          // → '/users/list'
+ */
+export function normalizePath(path: string): string {
+  return path.replace(/:[a-zA-Z_]\w*/g, ':_');
+}
+
+/**
+ * Build a match key for endpoint deduplication: METHOD + normalized path.
+ */
+function matchKey(method: string, path: string): string {
+  return `${method.toUpperCase()} ${normalizePath(path)}`;
+}
+
+/**
+ * Merge query params from a captured endpoint with params from an imported
+ * spec endpoint.
+ *
+ * Rules:
+ * - Captured `example` values are preserved (captured data is sacred).
+ * - Spec `type`, `required`, `enum` augment the captured params.
+ * - New params that only exist in the spec are added wholesale.
+ */
+function mergeQueryParams(
+  captured: SkillEndpoint['queryParams'],
+  specParams: SkillEndpoint['queryParams'],
+): SkillEndpoint['queryParams'] {
+  const merged = { ...captured };
+
+  for (const [name, specParam] of Object.entries(specParams)) {
+    if (name in merged) {
+      // Param exists in captured — keep captured example, augment with spec metadata
+      const existing = merged[name];
+      merged[name] = {
+        ...existing,
+        // Take spec type only if captured has generic 'string' and spec is more specific
+        type: existing.type === 'string' && specParam.type !== 'string' ? specParam.type : existing.type,
+        // Always preserve captured example value
+        example: existing.example,
+        // Add spec enum if present
+        ...(specParam.enum !== undefined ? { enum: specParam.enum } : {}),
+        // Add spec required flag if present
+        ...(specParam.required !== undefined ? { required: specParam.required } : {}),
+        // Mark as also coming from spec
+        ...(specParam.fromSpec ? { fromSpec: true } : {}),
+      };
+    } else {
+      // New param only in spec — add it
+      merged[name] = specParam;
+    }
+  }
+
+  return merged;
+}
+
+/**
+ * Determine whether an imported endpoint would enrich an existing captured
+ * endpoint (i.e. adds new metadata not already present).
+ */
+function wouldEnrich(existing: SkillEndpoint, imported: SkillEndpoint): boolean {
+  if (!existing.description && imported.description) return true;
+  if (!existing.specSource && imported.specSource) return true;
+
+  // Check if any new query params would be added or enriched
+  for (const [name, specParam] of Object.entries(imported.queryParams)) {
+    if (!(name in existing.queryParams)) return true;
+    const ep = existing.queryParams[name];
+    if (!ep.enum && specParam.enum) return true;
+    if (ep.required === undefined && specParam.required !== undefined) return true;
+    if (ep.type === 'string' && specParam.type !== 'string') return true;
+  }
+
+  return false;
+}
+
+/**
+ * Pure function — no I/O.
+ *
+ * Merges imported OpenAPI endpoints into an existing skill file.
+ *
+ * Captured data is sacred: it always wins on confidence, examples, and
+ * endpoint provenance. Spec data can only enrich (add description, specSource,
+ * query param enum/required/type) or fill gaps (add missing endpoints).
+ *
+ * Match logic: METHOD + normalizePath(path). This allows `:owner` and `:user`
+ * to be considered the same parameter slot.
+ *
+ * @param existing  The existing skill file on disk, or null if none exists.
+ * @param imported  Endpoints parsed from the OpenAPI spec.
+ * @param importMeta  Metadata about the import (spec URL, version, etc.).
+ * @returns MergeResult with the updated SkillFile and a diff summary.
+ */
+export function mergeSkillFile(
+  existing: SkillFile | null,
+  imported: SkillEndpoint[],
+  importMeta: ImportMeta,
+): MergeResult {
+  const now = new Date().toISOString();
+
+  let preserved = 0;
+  let added = 0;
+  let enriched = 0;
+  let skipped = 0;
+
+  // --- Case: no existing file — create a new SkillFile from imported endpoints ---
+  if (existing === null) {
+    const endpoints = imported.map(ep => ({
+      ...ep,
+      normalizedPath: normalizePath(ep.path),
+    }));
+
+    const skillFile: SkillFile = {
+      version: '1.2',
+      domain: extractDomainFromMeta(importMeta),
+      capturedAt: now,
+      baseUrl: extractBaseUrlFromMeta(importMeta),
+      endpoints,
+      metadata: {
+        captureCount: 0,
+        filteredCount: 0,
+        toolVersion: '1.0.0',
+        importHistory: [{
+          specUrl: importMeta.specUrl,
+          specVersion: importMeta.specVersion,
+          importedAt: now,
+          endpointsAdded: endpoints.length,
+          endpointsEnriched: 0,
+        }],
+      },
+      provenance: 'imported',
+    };
+
+    added = endpoints.length;
+
+    return {
+      skillFile,
+      diff: { preserved, added, enriched, skipped },
+    };
+  }
+
+  // --- Case: existing file present — merge into it ---
+
+  // Build a map from match-key → existing endpoint (mutable copy)
+  const existingMap = new Map<string, SkillEndpoint>();
+  for (const ep of existing.endpoints) {
+    existingMap.set(matchKey(ep.method, ep.path), ep);
+  }
+
+  // Build a map from match-key → imported endpoint
+  const importedMap = new Map<string, SkillEndpoint>();
+  for (const ep of imported) {
+    const key = matchKey(ep.method, ep.path);
+    // If multiple imported endpoints map to the same key, last wins
+    if (importedMap.has(key)) {
+      process.stderr.write(`[openapi-import] Warning: ${ep.method} ${ep.path} collides with existing import after normalization\n`);
+    }
+    importedMap.set(key, ep);
+  }
+
+  // Process: update or preserve existing endpoints
+  const resultEndpoints: SkillEndpoint[] = [];
+
+  for (const [key, existingEp] of existingMap) {
+    const importedEp = importedMap.get(key);
+
+    if (!importedEp) {
+      // Not in import — preserve as-is (captured endpoint with no match in spec)
+      resultEndpoints.push({
+        ...existingEp,
+        normalizedPath: normalizePath(existingEp.path),
+      });
+      preserved++;
+      continue;
+    }
+
+    // Imported endpoint matches an existing one — check if it would add anything new
+    if (!wouldEnrich(existingEp, importedEp)) {
+      resultEndpoints.push({
+        ...existingEp,
+        normalizedPath: normalizePath(existingEp.path),
+      });
+      // "skipped" means the import is redundant — existing already has spec metadata.
+      // "preserved" means the captured endpoint is untouched (import had nothing for it,
+      // or both sides are bare with no spec data to exchange).
+      const existingHasSpecData = !!(existingEp.specSource || existingEp.description);
+      const importHasSpecData = !!(importedEp.specSource || importedEp.description);
+      if (existingHasSpecData || importHasSpecData) {
+        // Spec data was already integrated (or import tried to add it but it's already present)
+        skipped++;
+      } else {
+        // Neither side has spec enrichment data — captured endpoint simply preserved
+        preserved++;
+      }
+      continue;
+    }
+
+    // Enrich the captured endpoint with spec metadata
+    const mergedQueryParams = mergeQueryParams(existingEp.queryParams, importedEp.queryParams);
+
+    const enrichedEp: SkillEndpoint = {
+      ...existingEp,
+      normalizedPath: normalizePath(existingEp.path),
+      // Augment with spec fields (only if not already present)
+      ...(importedEp.description && !existingEp.description ? { description: importedEp.description } : {}),
+      ...(importedEp.specSource && !existingEp.specSource ? { specSource: importedEp.specSource } : {}),
+      // Confidence never downgrades
+      confidence: Math.max(existingEp.confidence ?? 0, importedEp.confidence ?? 0) || existingEp.confidence,
+      // Keep captured provenance
+      endpointProvenance: existingEp.endpointProvenance,
+      queryParams: mergedQueryParams,
+    };
+
+    resultEndpoints.push(enrichedEp);
+    enriched++;
+  }
+
+  // Add endpoints from import that don't exist in the existing file
+  for (const [key, importedEp] of importedMap) {
+    if (!existingMap.has(key)) {
+      resultEndpoints.push({
+        ...importedEp,
+        normalizedPath: normalizePath(importedEp.path),
+      });
+      added++;
+    }
+  }
+
+  // Build updated import history
+  const prevHistory = existing.metadata.importHistory ?? [];
+  const newHistoryEntry = {
+    specUrl: importMeta.specUrl,
+    specVersion: importMeta.specVersion,
+    importedAt: now,
+    endpointsAdded: added,
+    endpointsEnriched: enriched,
+  };
+
+  const skillFile: SkillFile = {
+    ...existing,
+    endpoints: resultEndpoints,
+    metadata: {
+      ...existing.metadata,
+      importHistory: [...prevHistory, newHistoryEntry],
+    },
+  };
+
+  return {
+    skillFile,
+    diff: { preserved, added, enriched, skipped },
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Internal helpers
+// ---------------------------------------------------------------------------
+
+function extractDomainFromMeta(meta: ImportMeta): string {
+  try {
+    return new URL(meta.specUrl).hostname;
+  } catch {
+    throw new Error(`Cannot determine domain from specUrl: ${meta.specUrl}`);
+  }
+}
+
+function extractBaseUrlFromMeta(meta: ImportMeta): string {
+  try {
+    const u = new URL(meta.specUrl);
+    return `${u.protocol}//${u.hostname}`;
+  } catch {
+    throw new Error(`Cannot determine base URL from specUrl: ${meta.specUrl}`);
+  }
+}