@apitap/core 1.5.4 → 1.6.1

package/src/replay/engine.ts

@@ -45,6 +45,15 @@ const BLOCKED_REPLAY_HEADERS = new Set([
   'sec-fetch-user',
 ]);
 
+export function safeParseJson(text: string): unknown {
+  if (text.length === 0) return text;
+  try {
+    return JSON.parse(text);
+  } catch {
+    return text;
+  }
+}
+
 export interface ReplayOptions {
   /** User-provided parameters for path, query, and body substitution */
   params?: Record<string, string>;
@@ -70,6 +79,8 @@ export interface ReplayResult {
   truncated?: boolean;
   /** Contract warnings from schema drift detection */
   contractWarnings?: ContractWarning[];
+  /** Upgrade hint: set when a low-confidence endpoint gets a 2xx response */
+  upgrade?: { confidence: 1.0; endpointProvenance: 'captured' };
 }
 
 /**
@@ -204,6 +215,24 @@ function wrapAuthError(
   };
 }
 
+/**
+ * Returns a user-facing hint about confidence level, or null if confidence is high enough.
+ */
+export function getConfidenceHint(confidence: number | undefined): string | null {
+  const c = confidence ?? 1.0;
+  if (c >= 0.85) return null;
+  if (c >= 0.7) return '(imported from spec — params may need adjustment)';
+  return '(imported from spec — provide params explicitly, no captured examples available)';
+}
+
+/**
+ * Returns true if a query param should be omitted from the request.
+ * Omits spec-derived params that have no real example value.
+ */
+export function shouldOmitQueryParam(param: { type: string; example: string; fromSpec?: boolean }): boolean {
+  return param.fromSpec === true && param.example === '';
+}
+
 /**
  * Replay a captured API endpoint.
  *
@@ -240,6 +269,7 @@ export async function replayEndpoint(
 
   // Apply query params: start with captured defaults, override with provided params
   for (const [key, val] of Object.entries(endpoint.queryParams)) {
+    if (shouldOmitQueryParam(val)) continue;
     url.searchParams.set(key, val.example);
   }
   if (params) {
@@ -529,8 +559,8 @@ export async function replayEndpoint(
       let retryData: unknown;
       const retryCt = retryResponse.headers.get('content-type') ?? '';
       const retryText = await retryResponse.text();
-      if (retryCt.includes('json') && retryText.length > 0) {
-        retryData = JSON.parse(retryText);
+      if (retryCt.includes('json')) {
+        retryData = safeParseJson(retryText);
       } else {
         retryData = retryText;
       }
@@ -539,6 +569,11 @@ export async function replayEndpoint(
         ? wrapAuthError(retryResponse.status, retryData, skill.domain)
         : retryData;
 
+      const retryUpgrade = (endpoint.confidence !== undefined && endpoint.confidence < 1.0
+        && retryResponse.status >= 200 && retryResponse.status < 300)
+        ? { confidence: 1.0 as const, endpointProvenance: 'captured' as const }
+        : undefined;
+
       if (options.maxBytes) {
         const truncated = truncateResponse(retryFinalData, { maxBytes: options.maxBytes });
         return {
@@ -547,6 +582,7 @@ export async function replayEndpoint(
           data: truncated.data,
           refreshed,
           ...(truncated.truncated ? { truncated: true } : {}),
+          ...(retryUpgrade ? { upgrade: retryUpgrade } : {}),
         };
       }
 
@@ -555,6 +591,7 @@ export async function replayEndpoint(
         headers: retryHeaders,
         data: retryFinalData,
         refreshed,
+        ...(retryUpgrade ? { upgrade: retryUpgrade } : {}),
       };
     }
   }
@@ -567,8 +604,8 @@ export async function replayEndpoint(
   let data: unknown;
   const ct = response.headers.get('content-type') ?? '';
   const text = await response.text();
-  if (ct.includes('json') && text.length > 0) {
-    data = JSON.parse(text);
+  if (ct.includes('json')) {
+    data = safeParseJson(text);
   } else {
     data = text;
   }
@@ -587,6 +624,11 @@ export async function replayEndpoint(
     }
   }
 
+  const upgrade = (endpoint.confidence !== undefined && endpoint.confidence < 1.0
+    && response.status >= 200 && response.status < 300)
+    ? { confidence: 1.0 as const, endpointProvenance: 'captured' as const }
+    : undefined;
+
   // Apply truncation if maxBytes is set
   if (options.maxBytes) {
     const truncated = truncateResponse(finalData, { maxBytes: options.maxBytes });
@@ -597,10 +639,11 @@ export async function replayEndpoint(
       ...(refreshed ? { refreshed } : {}),
       ...(truncated.truncated ? { truncated: true } : {}),
       ...(contractWarnings ? { contractWarnings } : {}),
+      ...(upgrade ? { upgrade } : {}),
     };
   }
 
-  return { status: response.status, headers: responseHeaders, data: finalData, ...(refreshed ? { refreshed } : {}), ...(contractWarnings ? { contractWarnings } : {}) };
+  return { status: response.status, headers: responseHeaders, data: finalData, ...(refreshed ? { refreshed } : {}), ...(contractWarnings ? { contractWarnings } : {}), ...(upgrade ? { upgrade } : {}) };
 }
 
 // --- Batch replay ---

package/src/skill/apis-guru.ts

@@ -0,0 +1,169 @@
+// src/skill/apis-guru.ts
+import { resolveAndValidateUrl } from '../skill/ssrf.js';
+
+const MAX_SPEC_SIZE = 10 * 1024 * 1024;   // 10 MB per spec
+const MAX_LIST_SIZE = 100 * 1024 * 1024;  // 100 MB for APIs.guru list
+
+async function fetchWithSizeLimit(url: string, maxBytes: number, options?: RequestInit): Promise<string> {
+  const response = await fetch(url, {
+    signal: AbortSignal.timeout(30_000),
+    ...options,
+    headers: { 'User-Agent': 'apitap-import/1.0', ...(options?.headers as Record<string, string> || {}) },
+  });
+  if (!response.ok) {
+    throw new Error(`HTTP ${response.status} ${response.statusText} for ${url}`);
+  }
+  const contentLength = response.headers.get('content-length');
+  if (contentLength && parseInt(contentLength, 10) > maxBytes) {
+    throw new Error(`Response too large: ${contentLength} bytes (limit: ${maxBytes})`);
+  }
+  const text = await response.text();
+  if (text.length > maxBytes) {
+    throw new Error(`Response body too large: ${text.length} bytes (limit: ${maxBytes})`);
+  }
+  return text;
+}
+
+export interface ApisGuruEntry {
+  apiId: string;           // e.g., "twilio.com:api"
+  providerName: string;    // e.g., "twilio.com"
+  title: string;
+  specUrl: string;         // direct URL to OpenAPI JSON spec
+  openapiVer: string;
+  updated: string;
+}
+
+const APIS_GURU_LIST_URL = 'https://api.apis.guru/v2/list.json';
+
+/**
+ * Parse raw APIs.guru list.json response into ApisGuruEntry array.
+ * For each API, use the preferred version's data.
+ */
+export function parseApisGuruList(raw: Record<string, any>): ApisGuruEntry[] {
+  const entries: ApisGuruEntry[] = [];
+
+  for (const apiId of Object.keys(raw)) {
+    const apiData = raw[apiId];
+    if (!apiData || typeof apiData !== 'object') continue;
+
+    const preferred: string = apiData.preferred;
+    if (!preferred) continue;
+
+    const versions = apiData.versions;
+    if (!versions || typeof versions !== 'object') continue;
+
+    const versionData = versions[preferred];
+    if (!versionData || typeof versionData !== 'object') continue;
+
+    const swaggerUrl: string | undefined = versionData.swaggerUrl;
+    if (!swaggerUrl) continue;
+
+    const info = versionData.info ?? {};
+    const title: string = info.title ?? '';
+    const openapiVer: string = versionData.openapiVer ?? '';
+    const updated: string = versionData.updated ?? '';
+
+    // providerName: prefer info.x-providerName, else split apiId on ':', else use apiId
+    let providerName: string;
+    if (info['x-providerName']) {
+      providerName = info['x-providerName'];
+    } else {
+      const colonIdx = apiId.indexOf(':');
+      providerName = colonIdx >= 0 ? apiId.slice(0, colonIdx) : apiId;
+    }
+
+    entries.push({
+      apiId,
+      providerName,
+      title,
+      specUrl: swaggerUrl,
+      openapiVer,
+      updated,
+    });
+  }
+
+  return entries;
+}
+
+export interface FilterOptions {
+  search?: string;
+  limit?: number;
+  noAuthOnly?: boolean;
+  preferOpenapi3?: boolean;
+}
+
+/**
+ * Filter and sort ApisGuruEntry array.
+ * - search: substring match (case-insensitive) on providerName or title
+ * - preferOpenapi3: sort 3.x entries before 2.x, then by recency within groups
+ * - default sort: by recency (updated desc)
+ * - limit: cap result count
+ */
+export function filterEntries(
+  entries: ApisGuruEntry[],
+  options: FilterOptions,
+): ApisGuruEntry[] {
+  const { search, limit, preferOpenapi3 } = options;
+
+  let result = entries;
+
+  // Filter by search term
+  if (search) {
+    const lower = search.toLowerCase();
+    result = result.filter(
+      e =>
+        e.providerName.toLowerCase().includes(lower) ||
+        e.title.toLowerCase().includes(lower),
+    );
+  }
+
+  // Sort
+  result = [...result].sort((a, b) => {
+    if (preferOpenapi3) {
+      const aIs3 = a.openapiVer.startsWith('3') ? 0 : 1;
+      const bIs3 = b.openapiVer.startsWith('3') ? 0 : 1;
+      if (aIs3 !== bIs3) return aIs3 - bIs3;
+    }
+    // Within same group (or when not preferring 3.x), sort by recency desc
+    return b.updated.localeCompare(a.updated);
+  });
+
+  // Apply limit
+  if (typeof limit === 'number' && limit > 0) {
+    result = result.slice(0, limit);
+  }
+
+  return result;
+}
+
+/**
+ * Fetch the APIs.guru list.json and parse it into ApisGuruEntry array.
+ */
+export async function fetchApisGuruList(): Promise<ApisGuruEntry[]> {
+  const ssrf = await resolveAndValidateUrl(APIS_GURU_LIST_URL);
+  if (!ssrf.safe) {
+    throw new Error(`SSRF check failed for APIs.guru list URL: ${ssrf.reason}`);
+  }
+  const text = await fetchWithSizeLimit(APIS_GURU_LIST_URL, MAX_LIST_SIZE);
+  try {
+    return parseApisGuruList(JSON.parse(text) as Record<string, any>);
+  } catch {
+    throw new Error(`Invalid JSON from ${APIS_GURU_LIST_URL}: ${text.slice(0, 100)}`);
+  }
+}
+
+/**
+ * Fetch a single OpenAPI spec by URL and return the parsed JSON.
+ */
+export async function fetchSpec(specUrl: string): Promise<Record<string, any>> {
+  const ssrf = await resolveAndValidateUrl(specUrl);
+  if (!ssrf.safe) {
+    throw new Error(`SSRF check failed for spec URL ${specUrl}: ${ssrf.reason}`);
+  }
+  const text = await fetchWithSizeLimit(specUrl, MAX_SPEC_SIZE, { redirect: 'error' });
+  try {
+    return JSON.parse(text) as Record<string, any>;
+  } catch {
+    throw new Error(`Invalid JSON from ${specUrl}: ${text.slice(0, 100)}`);
+  }
+}

package/src/skill/merge.ts

@@ -0,0 +1,281 @@
+// src/skill/merge.ts
+import type { SkillFile, SkillEndpoint, ImportMeta, MergeResult } from '../types.js';
+
+/**
+ * Normalize a parameterized path by replacing all named param placeholders
+ * with the generic `:_` placeholder, enabling matching across different param
+ * naming conventions (e.g. `:id` vs `:userId` vs `:user_id`).
+ *
+ * @example
+ *   normalizePath('/repos/:owner/:repo') // → '/repos/:_/:_'
+ *   normalizePath('/users/list')          // → '/users/list'
+ */
+export function normalizePath(path: string): string {
+  return path.replace(/:[a-zA-Z_]\w*/g, ':_');
+}
+
+/**
+ * Build a match key for endpoint deduplication: METHOD + normalized path.
+ */
+function matchKey(method: string, path: string): string {
+  return `${method.toUpperCase()} ${normalizePath(path)}`;
+}
+
+/**
+ * Merge query params from a captured endpoint with params from an imported
+ * spec endpoint.
+ *
+ * Rules:
+ * - Captured `example` values are preserved (captured data is sacred).
+ * - Spec `type`, `required`, `enum` augment the captured params.
+ * - New params that only exist in the spec are added wholesale.
+ */
+function mergeQueryParams(
+  captured: SkillEndpoint['queryParams'],
+  specParams: SkillEndpoint['queryParams'],
+): SkillEndpoint['queryParams'] {
+  const merged = { ...captured };
+
+  for (const [name, specParam] of Object.entries(specParams)) {
+    if (name in merged) {
+      // Param exists in captured — keep captured example, augment with spec metadata
+      const existing = merged[name];
+      merged[name] = {
+        ...existing,
+        // Take spec type only if captured has generic 'string' and spec is more specific
+        type: existing.type === 'string' && specParam.type !== 'string' ? specParam.type : existing.type,
+        // Always preserve captured example value
+        example: existing.example,
+        // Add spec enum if present
+        ...(specParam.enum !== undefined ? { enum: specParam.enum } : {}),
+        // Add spec required flag if present
+        ...(specParam.required !== undefined ? { required: specParam.required } : {}),
+        // Mark as also coming from spec
+        ...(specParam.fromSpec ? { fromSpec: true } : {}),
+      };
+    } else {
+      // New param only in spec — add it
+      merged[name] = specParam;
+    }
+  }
+
+  return merged;
+}
+
+/**
+ * Determine whether an imported endpoint would enrich an existing captured
+ * endpoint (i.e. adds new metadata not already present).
+ */
+function wouldEnrich(existing: SkillEndpoint, imported: SkillEndpoint): boolean {
+  if (!existing.description && imported.description) return true;
+  if (!existing.specSource && imported.specSource) return true;
+
+  // Check if any new query params would be added or enriched
+  for (const [name, specParam] of Object.entries(imported.queryParams)) {
+    if (!(name in existing.queryParams)) return true;
+    const ep = existing.queryParams[name];
+    if (!ep.enum && specParam.enum) return true;
+    if (ep.required === undefined && specParam.required !== undefined) return true;
+    if (ep.type === 'string' && specParam.type !== 'string') return true;
+  }
+
+  return false;
+}
+
+/**
+ * Pure function — no I/O.
+ *
+ * Merges imported OpenAPI endpoints into an existing skill file.
+ *
+ * Captured data is sacred: it always wins on confidence, examples, and
+ * endpoint provenance. Spec data can only enrich (add description, specSource,
+ * query param enum/required/type) or fill gaps (add missing endpoints).
+ *
+ * Match logic: METHOD + normalizePath(path). This allows `:owner` and `:user`
+ * to be considered the same parameter slot.
+ *
+ * @param existing  The existing skill file on disk, or null if none exists.
+ * @param imported  Endpoints parsed from the OpenAPI spec.
+ * @param importMeta  Metadata about the import (spec URL, version, etc.).
+ * @returns MergeResult with the updated SkillFile and a diff summary.
+ */
+export function mergeSkillFile(
+  existing: SkillFile | null,
+  imported: SkillEndpoint[],
+  importMeta: ImportMeta,
+): MergeResult {
+  const now = new Date().toISOString();
+
+  let preserved = 0;
+  let added = 0;
+  let enriched = 0;
+  let skipped = 0;
+
+  // --- Case: no existing file — create a new SkillFile from imported endpoints ---
+  if (existing === null) {
+    const endpoints = imported.map(ep => ({
+      ...ep,
+      normalizedPath: normalizePath(ep.path),
+    }));
+
+    const skillFile: SkillFile = {
+      version: '1.2',
+      domain: extractDomainFromMeta(importMeta),
+      capturedAt: now,
+      baseUrl: extractBaseUrlFromMeta(importMeta),
+      endpoints,
+      metadata: {
+        captureCount: 0,
+        filteredCount: 0,
+        toolVersion: '1.0.0',
+        importHistory: [{
+          specUrl: importMeta.specUrl,
+          specVersion: importMeta.specVersion,
+          importedAt: now,
+          endpointsAdded: endpoints.length,
+          endpointsEnriched: 0,
+        }],
+      },
+      provenance: 'imported',
+    };
+
+    added = endpoints.length;
+
+    return {
+      skillFile,
+      diff: { preserved, added, enriched, skipped },
+    };
+  }
+
+  // --- Case: existing file present — merge into it ---
+
+  // Build a map from match-key → existing endpoint (mutable copy)
+  const existingMap = new Map<string, SkillEndpoint>();
+  for (const ep of existing.endpoints) {
+    existingMap.set(matchKey(ep.method, ep.path), ep);
+  }
+
+  // Build a map from match-key → imported endpoint
+  const importedMap = new Map<string, SkillEndpoint>();
+  for (const ep of imported) {
+    const key = matchKey(ep.method, ep.path);
+    // If multiple imported endpoints map to the same key, last wins
+    if (importedMap.has(key)) {
+      process.stderr.write(`[openapi-import] Warning: ${ep.method} ${ep.path} collides with existing import after normalization\n`);
+    }
+    importedMap.set(key, ep);
+  }
+
+  // Process: update or preserve existing endpoints
+  const resultEndpoints: SkillEndpoint[] = [];
+
+  for (const [key, existingEp] of existingMap) {
+    const importedEp = importedMap.get(key);
+
+    if (!importedEp) {
+      // Not in import — preserve as-is (captured endpoint with no match in spec)
+      resultEndpoints.push({
+        ...existingEp,
+        normalizedPath: normalizePath(existingEp.path),
+      });
+      preserved++;
+      continue;
+    }
+
+    // Imported endpoint matches an existing one — check if it would add anything new
+    if (!wouldEnrich(existingEp, importedEp)) {
+      resultEndpoints.push({
+        ...existingEp,
+        normalizedPath: normalizePath(existingEp.path),
+      });
+      // "skipped" means the import is redundant — existing already has spec metadata.
+      // "preserved" means the captured endpoint is untouched (import had nothing for it,
+      // or both sides are bare with no spec data to exchange).
+      const existingHasSpecData = !!(existingEp.specSource || existingEp.description);
+      const importHasSpecData = !!(importedEp.specSource || importedEp.description);
+      if (existingHasSpecData || importHasSpecData) {
+        // Spec data was already integrated (or import tried to add it but it's already present)
+        skipped++;
+      } else {
+        // Neither side has spec enrichment data — captured endpoint simply preserved
+        preserved++;
+      }
+      continue;
+    }
+
+    // Enrich the captured endpoint with spec metadata
+    const mergedQueryParams = mergeQueryParams(existingEp.queryParams, importedEp.queryParams);
+
+    const enrichedEp: SkillEndpoint = {
+      ...existingEp,
+      normalizedPath: normalizePath(existingEp.path),
+      // Augment with spec fields (only if not already present)
+      ...(importedEp.description && !existingEp.description ? { description: importedEp.description } : {}),
+      ...(importedEp.specSource && !existingEp.specSource ? { specSource: importedEp.specSource } : {}),
+      // Confidence never downgrades
+      confidence: Math.max(existingEp.confidence ?? 0, importedEp.confidence ?? 0) || existingEp.confidence,
+      // Keep captured provenance
+      endpointProvenance: existingEp.endpointProvenance,
+      queryParams: mergedQueryParams,
+    };
+
+    resultEndpoints.push(enrichedEp);
+    enriched++;
+  }
+
+  // Add endpoints from import that don't exist in the existing file
+  for (const [key, importedEp] of importedMap) {
+    if (!existingMap.has(key)) {
+      resultEndpoints.push({
+        ...importedEp,
+        normalizedPath: normalizePath(importedEp.path),
+      });
+      added++;
+    }
+  }
+
+  // Build updated import history
+  const prevHistory = existing.metadata.importHistory ?? [];
+  const newHistoryEntry = {
+    specUrl: importMeta.specUrl,
+    specVersion: importMeta.specVersion,
+    importedAt: now,
+    endpointsAdded: added,
+    endpointsEnriched: enriched,
+  };
+
+  const skillFile: SkillFile = {
+    ...existing,
+    endpoints: resultEndpoints,
+    metadata: {
+      ...existing.metadata,
+      importHistory: [...prevHistory, newHistoryEntry],
+    },
+  };
+
+  return {
+    skillFile,
+    diff: { preserved, added, enriched, skipped },
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Internal helpers
+// ---------------------------------------------------------------------------
+
+function extractDomainFromMeta(meta: ImportMeta): string {
+  try {
+    return new URL(meta.specUrl).hostname;
+  } catch {
+    throw new Error(`Cannot determine domain from specUrl: ${meta.specUrl}`);
+  }
+}
+
+function extractBaseUrlFromMeta(meta: ImportMeta): string {
+  try {
+    const u = new URL(meta.specUrl);
+    return `${u.protocol}//${u.hostname}`;
+  } catch {
+    throw new Error(`Cannot determine base URL from specUrl: ${meta.specUrl}`);
+  }
+}