@apitap/core 1.5.4 → 1.6.0

package/src/skill/apis-guru.ts

@@ -0,0 +1,163 @@
+// src/skill/apis-guru.ts
+import { resolveAndValidateUrl } from '../skill/ssrf.js';
+
+export interface ApisGuruEntry {
+  apiId: string;           // e.g., "twilio.com:api"
+  providerName: string;    // e.g., "twilio.com"
+  title: string;
+  specUrl: string;         // direct URL to OpenAPI JSON spec
+  openapiVer: string;
+  updated: string;
+}
+
+const APIS_GURU_LIST_URL = 'https://api.apis.guru/v2/list.json';
+
+/**
+ * Parse raw APIs.guru list.json response into ApisGuruEntry array.
+ * For each API, use the preferred version's data.
+ */
+export function parseApisGuruList(raw: Record<string, any>): ApisGuruEntry[] {
+  const entries: ApisGuruEntry[] = [];
+
+  for (const apiId of Object.keys(raw)) {
+    const apiData = raw[apiId];
+    if (!apiData || typeof apiData !== 'object') continue;
+
+    const preferred: string = apiData.preferred;
+    if (!preferred) continue;
+
+    const versions = apiData.versions;
+    if (!versions || typeof versions !== 'object') continue;
+
+    const versionData = versions[preferred];
+    if (!versionData || typeof versionData !== 'object') continue;
+
+    const swaggerUrl: string | undefined = versionData.swaggerUrl;
+    if (!swaggerUrl) continue;
+
+    const info = versionData.info ?? {};
+    const title: string = info.title ?? '';
+    const openapiVer: string = versionData.openapiVer ?? '';
+    const updated: string = versionData.updated ?? '';
+
+    // providerName: prefer info.x-providerName, else split apiId on ':', else use apiId
+    let providerName: string;
+    if (info['x-providerName']) {
+      providerName = info['x-providerName'];
+    } else {
+      const colonIdx = apiId.indexOf(':');
+      providerName = colonIdx >= 0 ? apiId.slice(0, colonIdx) : apiId;
+    }
+
+    entries.push({
+      apiId,
+      providerName,
+      title,
+      specUrl: swaggerUrl,
+      openapiVer,
+      updated,
+    });
+  }
+
+  return entries;
+}
+
+export interface FilterOptions {
+  search?: string;
+  limit?: number;
+  noAuthOnly?: boolean;
+  preferOpenapi3?: boolean;
+}
+
+/**
+ * Filter and sort ApisGuruEntry array.
+ * - search: substring match (case-insensitive) on providerName or title
+ * - preferOpenapi3: sort 3.x entries before 2.x, then by recency within groups
+ * - default sort: by recency (updated desc)
+ * - limit: cap result count
+ */
+export function filterEntries(
+  entries: ApisGuruEntry[],
+  options: FilterOptions,
+): ApisGuruEntry[] {
+  const { search, limit, preferOpenapi3 } = options;
+
+  let result = entries;
+
+  // Filter by search term
+  if (search) {
+    const lower = search.toLowerCase();
+    result = result.filter(
+      e =>
+        e.providerName.toLowerCase().includes(lower) ||
+        e.title.toLowerCase().includes(lower),
+    );
+  }
+
+  // Sort
+  result = [...result].sort((a, b) => {
+    if (preferOpenapi3) {
+      const aIs3 = a.openapiVer.startsWith('3') ? 0 : 1;
+      const bIs3 = b.openapiVer.startsWith('3') ? 0 : 1;
+      if (aIs3 !== bIs3) return aIs3 - bIs3;
+    }
+    // Within same group (or when not preferring 3.x), sort by recency desc
+    return b.updated.localeCompare(a.updated);
+  });
+
+  // Apply limit
+  if (typeof limit === 'number' && limit > 0) {
+    result = result.slice(0, limit);
+  }
+
+  return result;
+}
+
+/**
+ * Fetch the APIs.guru list.json and parse it into ApisGuruEntry array.
+ */
+export async function fetchApisGuruList(): Promise<ApisGuruEntry[]> {
+  const ssrf = await resolveAndValidateUrl(APIS_GURU_LIST_URL);
+  if (!ssrf.safe) {
+    throw new Error(`SSRF check failed for APIs.guru list URL: ${ssrf.reason}`);
+  }
+  const response = await fetch(APIS_GURU_LIST_URL, {
+    signal: AbortSignal.timeout(30_000),
+  });
+  if (!response.ok) {
+    throw new Error(
+      `Failed to fetch APIs.guru list: ${response.status} ${response.statusText}`,
+    );
+  }
+  const text = await response.text();
+  try {
+    return parseApisGuruList(JSON.parse(text) as Record<string, any>);
+  } catch {
+    throw new Error(`Invalid JSON from ${APIS_GURU_LIST_URL}: ${text.slice(0, 100)}`);
+  }
+}
+
+/**
+ * Fetch a single OpenAPI spec by URL and return the parsed JSON.
+ */
+export async function fetchSpec(specUrl: string): Promise<Record<string, any>> {
+  const ssrf = await resolveAndValidateUrl(specUrl);
+  if (!ssrf.safe) {
+    throw new Error(`SSRF check failed for spec URL ${specUrl}: ${ssrf.reason}`);
+  }
+  const response = await fetch(specUrl, {
+    signal: AbortSignal.timeout(30_000),
+    redirect: 'error',
+  });
+  if (!response.ok) {
+    throw new Error(
+      `Failed to fetch spec at ${specUrl}: ${response.status} ${response.statusText}`,
+    );
+  }
+  const text = await response.text();
+  try {
+    return JSON.parse(text) as Record<string, any>;
+  } catch {
+    throw new Error(`Invalid JSON from ${specUrl}: ${text.slice(0, 100)}`);
+  }
+}

package/src/skill/merge.ts

@@ -0,0 +1,281 @@
+// src/skill/merge.ts
+import type { SkillFile, SkillEndpoint, ImportMeta, MergeResult } from '../types.js';
+
+/**
+ * Normalize a parameterized path by replacing all named param placeholders
+ * with the generic `:_` placeholder, enabling matching across different param
+ * naming conventions (e.g. `:id` vs `:userId` vs `:user_id`).
+ *
+ * @example
+ *   normalizePath('/repos/:owner/:repo') // → '/repos/:_/:_'
+ *   normalizePath('/users/list')          // → '/users/list'
+ */
+export function normalizePath(path: string): string {
+  return path.replace(/:[a-zA-Z_]\w*/g, ':_');
+}
+
+/**
+ * Build a match key for endpoint deduplication: METHOD + normalized path.
+ */
+function matchKey(method: string, path: string): string {
+  return `${method.toUpperCase()} ${normalizePath(path)}`;
+}
+
+/**
+ * Merge query params from a captured endpoint with params from an imported
+ * spec endpoint.
+ *
+ * Rules:
+ * - Captured `example` values are preserved (captured data is sacred).
+ * - Spec `type`, `required`, `enum` augment the captured params.
+ * - New params that only exist in the spec are added wholesale.
+ */
+function mergeQueryParams(
+  captured: SkillEndpoint['queryParams'],
+  specParams: SkillEndpoint['queryParams'],
+): SkillEndpoint['queryParams'] {
+  const merged = { ...captured };
+
+  for (const [name, specParam] of Object.entries(specParams)) {
+    if (name in merged) {
+      // Param exists in captured — keep captured example, augment with spec metadata
+      const existing = merged[name];
+      merged[name] = {
+        ...existing,
+        // Take spec type only if captured has generic 'string' and spec is more specific
+        type: existing.type === 'string' && specParam.type !== 'string' ? specParam.type : existing.type,
+        // Always preserve captured example value
+        example: existing.example,
+        // Add spec enum if present
+        ...(specParam.enum !== undefined ? { enum: specParam.enum } : {}),
+        // Add spec required flag if present
+        ...(specParam.required !== undefined ? { required: specParam.required } : {}),
+        // Mark as also coming from spec
+        ...(specParam.fromSpec ? { fromSpec: true } : {}),
+      };
+    } else {
+      // New param only in spec — add it
+      merged[name] = specParam;
+    }
+  }
+
+  return merged;
+}
+
+/**
+ * Determine whether an imported endpoint would enrich an existing captured
+ * endpoint (i.e. adds new metadata not already present).
+ */
+function wouldEnrich(existing: SkillEndpoint, imported: SkillEndpoint): boolean {
+  if (!existing.description && imported.description) return true;
+  if (!existing.specSource && imported.specSource) return true;
+
+  // Check if any new query params would be added or enriched
+  for (const [name, specParam] of Object.entries(imported.queryParams)) {
+    if (!(name in existing.queryParams)) return true;
+    const ep = existing.queryParams[name];
+    if (!ep.enum && specParam.enum) return true;
+    if (ep.required === undefined && specParam.required !== undefined) return true;
+    if (ep.type === 'string' && specParam.type !== 'string') return true;
+  }
+
+  return false;
+}
+
+/**
+ * Pure function — no I/O.
+ *
+ * Merges imported OpenAPI endpoints into an existing skill file.
+ *
+ * Captured data is sacred: it always wins on confidence, examples, and
+ * endpoint provenance. Spec data can only enrich (add description, specSource,
+ * query param enum/required/type) or fill gaps (add missing endpoints).
+ *
+ * Match logic: METHOD + normalizePath(path). This allows `:owner` and `:user`
+ * to be considered the same parameter slot.
+ *
+ * @param existing  The existing skill file on disk, or null if none exists.
+ * @param imported  Endpoints parsed from the OpenAPI spec.
+ * @param importMeta  Metadata about the import (spec URL, version, etc.).
+ * @returns MergeResult with the updated SkillFile and a diff summary.
+ */
+export function mergeSkillFile(
+  existing: SkillFile | null,
+  imported: SkillEndpoint[],
+  importMeta: ImportMeta,
+): MergeResult {
+  const now = new Date().toISOString();
+
+  let preserved = 0;
+  let added = 0;
+  let enriched = 0;
+  let skipped = 0;
+
+  // --- Case: no existing file — create a new SkillFile from imported endpoints ---
+  if (existing === null) {
+    const endpoints = imported.map(ep => ({
+      ...ep,
+      normalizedPath: normalizePath(ep.path),
+    }));
+
+    const skillFile: SkillFile = {
+      version: '1.2',
+      domain: extractDomainFromMeta(importMeta),
+      capturedAt: now,
+      baseUrl: extractBaseUrlFromMeta(importMeta),
+      endpoints,
+      metadata: {
+        captureCount: 0,
+        filteredCount: 0,
+        toolVersion: '1.0.0',
+        importHistory: [{
+          specUrl: importMeta.specUrl,
+          specVersion: importMeta.specVersion,
+          importedAt: now,
+          endpointsAdded: endpoints.length,
+          endpointsEnriched: 0,
+        }],
+      },
+      provenance: 'imported',
+    };
+
+    added = endpoints.length;
+
+    return {
+      skillFile,
+      diff: { preserved, added, enriched, skipped },
+    };
+  }
+
+  // --- Case: existing file present — merge into it ---
+
+  // Build a map from match-key → existing endpoint (mutable copy)
+  const existingMap = new Map<string, SkillEndpoint>();
+  for (const ep of existing.endpoints) {
+    existingMap.set(matchKey(ep.method, ep.path), ep);
+  }
+
+  // Build a map from match-key → imported endpoint
+  const importedMap = new Map<string, SkillEndpoint>();
+  for (const ep of imported) {
+    const key = matchKey(ep.method, ep.path);
+    // If multiple imported endpoints map to the same key, last wins
+    if (importedMap.has(key)) {
+      process.stderr.write(`[openapi-import] Warning: ${ep.method} ${ep.path} collides with existing import after normalization\n`);
+    }
+    importedMap.set(key, ep);
+  }
+
+  // Process: update or preserve existing endpoints
+  const resultEndpoints: SkillEndpoint[] = [];
+
+  for (const [key, existingEp] of existingMap) {
+    const importedEp = importedMap.get(key);
+
+    if (!importedEp) {
+      // Not in import — preserve as-is (captured endpoint with no match in spec)
+      resultEndpoints.push({
+        ...existingEp,
+        normalizedPath: normalizePath(existingEp.path),
+      });
+      preserved++;
+      continue;
+    }
+
+    // Imported endpoint matches an existing one — check if it would add anything new
+    if (!wouldEnrich(existingEp, importedEp)) {
+      resultEndpoints.push({
+        ...existingEp,
+        normalizedPath: normalizePath(existingEp.path),
+      });
+      // "skipped" means the import is redundant — existing already has spec metadata.
+      // "preserved" means the captured endpoint is untouched (import had nothing for it,
+      // or both sides are bare with no spec data to exchange).
+      const existingHasSpecData = !!(existingEp.specSource || existingEp.description);
+      const importHasSpecData = !!(importedEp.specSource || importedEp.description);
+      if (existingHasSpecData || importHasSpecData) {
+        // Spec data was already integrated (or import tried to add it but it's already present)
+        skipped++;
+      } else {
+        // Neither side has spec enrichment data — captured endpoint simply preserved
+        preserved++;
+      }
+      continue;
+    }
+
+    // Enrich the captured endpoint with spec metadata
+    const mergedQueryParams = mergeQueryParams(existingEp.queryParams, importedEp.queryParams);
+
+    const enrichedEp: SkillEndpoint = {
+      ...existingEp,
+      normalizedPath: normalizePath(existingEp.path),
+      // Augment with spec fields (only if not already present)
+      ...(importedEp.description && !existingEp.description ? { description: importedEp.description } : {}),
+      ...(importedEp.specSource && !existingEp.specSource ? { specSource: importedEp.specSource } : {}),
+      // Confidence never downgrades
+      confidence: Math.max(existingEp.confidence ?? 0, importedEp.confidence ?? 0) || existingEp.confidence,
+      // Keep captured provenance
+      endpointProvenance: existingEp.endpointProvenance,
+      queryParams: mergedQueryParams,
+    };
+
+    resultEndpoints.push(enrichedEp);
+    enriched++;
+  }
+
+  // Add endpoints from import that don't exist in the existing file
+  for (const [key, importedEp] of importedMap) {
+    if (!existingMap.has(key)) {
+      resultEndpoints.push({
+        ...importedEp,
+        normalizedPath: normalizePath(importedEp.path),
+      });
+      added++;
+    }
+  }
+
+  // Build updated import history
+  const prevHistory = existing.metadata.importHistory ?? [];
+  const newHistoryEntry = {
+    specUrl: importMeta.specUrl,
+    specVersion: importMeta.specVersion,
+    importedAt: now,
+    endpointsAdded: added,
+    endpointsEnriched: enriched,
+  };
+
+  const skillFile: SkillFile = {
+    ...existing,
+    endpoints: resultEndpoints,
+    metadata: {
+      ...existing.metadata,
+      importHistory: [...prevHistory, newHistoryEntry],
+    },
+  };
+
+  return {
+    skillFile,
+    diff: { preserved, added, enriched, skipped },
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Internal helpers
+// ---------------------------------------------------------------------------
+
+function extractDomainFromMeta(meta: ImportMeta): string {
+  try {
+    return new URL(meta.specUrl).hostname;
+  } catch {
+    throw new Error(`Cannot determine domain from specUrl: ${meta.specUrl}`);
+  }
+}
+
+function extractBaseUrlFromMeta(meta: ImportMeta): string {
+  try {
+    const u = new URL(meta.specUrl);
+    return `${u.protocol}//${u.hostname}`;
+  } catch {
+    throw new Error(`Cannot determine base URL from specUrl: ${meta.specUrl}`);
+  }
+}