@apitap/core 1.5.4 → 1.6.1

package/src/cli.ts

@@ -2,10 +2,10 @@
 // src/cli.ts
 import { capture } from './capture/monitor.js';
 import { writeSkillFile, readSkillFile, listSkillFiles } from './skill/store.js';
-import { replayEndpoint } from './replay/engine.js';
+import { replayEndpoint, getConfidenceHint } from './replay/engine.js';
 import { AuthManager, getMachineId } from './auth/manager.js';
 import { deriveSigningKey } from './auth/crypto.js';
-import { signSkillFile } from './skill/signing.js';
+import { signSkillFile, signSkillFileAs } from './skill/signing.js';
 import { importSkillFile } from './skill/importer.js';
 import { resolveAndValidateUrl } from './skill/ssrf.js';
 import { verifyEndpoints } from './capture/verifier.js';
@@ -27,6 +27,9 @@ import { stat, unlink } from 'node:fs/promises';
 import { fileURLToPath } from 'node:url';
 import { createMcpServer } from './mcp.js';
 import { attach, parseDomainPatterns } from './capture/cdp-attach.js';
+import { isOpenAPISpec, convertOpenAPISpec } from './skill/openapi-converter.js';
+import { mergeSkillFile } from './skill/merge.js';
+import { fetchApisGuruList, filterEntries, fetchSpec } from './skill/apis-guru.js';
 
 const __dirname = fileURLToPath(new URL('.', import.meta.url));
 const pkg = JSON.parse(readFileSync(join(__dirname, '..', 'package.json'), 'utf-8'));
@@ -76,7 +79,9 @@ function printUsage(): void {
     apitap show <domain>       Show endpoints for a domain
     apitap replay <domain> <endpoint-id> [key=value...]
                                Replay an API endpoint
-    apitap import <file>       Import a skill file with safety validation
+    apitap import <file|url>   Import a skill file or OpenAPI spec
+    apitap import --from apis-guru
+                               Bulk-import from APIs.guru directory
     apitap refresh <domain>    Refresh auth tokens via browser
     apitap auth [domain]       View or manage stored auth
     apitap mcp                 Run the full ApiTap MCP server over stdio
@@ -128,6 +133,11 @@ function printUsage(): void {
 
   Import options:
     --yes                      Skip confirmation prompt
+    --dry-run                  Show what would change without writing
+    --json                     Output machine-readable JSON
+    --from apis-guru           Bulk-import from APIs.guru directory
+    --search <term>            Filter APIs.guru entries by name/title
+    --limit <n>                Max APIs to import (default: 100)
 
   Serve options:
     --json                     Output tool list as JSON on stderr
@@ -448,6 +458,20 @@ async function handleReplay(positional: string[], flags: Record<string, string |
     _skipSsrfCheck: dangerDisableSsrf,
   });
 
+  // Auto-upgrade imported endpoints on successful replay
+  if ((result as any).upgrade && endpoint) {
+    endpoint.confidence = 1.0;
+    endpoint.endpointProvenance = 'captured';
+    // Update example with the actual successful request
+    endpoint.examples.responsePreview = typeof result.data === 'object' ? result.data : null;
+    // Re-sign and write
+    const signed = signSkillFile(skill, signingKey);
+    await writeSkillFile(signed, SKILLS_DIR);
+    if (!json) {
+      console.error('  \u2713 Endpoint upgraded to captured (confidence 1.0)');
+    }
+  }
+
   if (json) {
     console.log(JSON.stringify({
       status: result.status,
@@ -455,6 +479,10 @@ async function handleReplay(positional: string[], flags: Record<string, string |
       ...(result.contractWarnings?.length ? { contractWarnings: result.contractWarnings } : {}),
     }, null, 2));
   } else {
+    const hint = endpoint ? getConfidenceHint(endpoint.confidence) : null;
+    if (hint) {
+      console.error(`  Note: ${hint}`);
+    }
     console.log(`\n  Status: ${result.status}\n`);
     console.log(JSON.stringify(result.data, null, 2));
     console.log();
@@ -462,35 +490,130 @@ async function handleReplay(positional: string[], flags: Record<string, string |
 }
 
 async function handleImport(positional: string[], flags: Record<string, string | boolean>): Promise<void> {
-  const filePath = positional[0];
-  if (!filePath) {
-    console.error('Error: File path required. Usage: apitap import <file>');
+  // --from apis-guru: bulk import mode
+  if (flags['from'] === 'apis-guru') {
+    await handleApisGuruImport(flags);
+    return;
+  }
+
+  const source = positional[0];
+  if (!source) {
+    console.error('Error: File path or URL required. Usage: apitap import <file|url>');
     process.exit(1);
   }
 
   const json = flags.json === true;
 
+  // Load content — from URL or file
+  let rawText: string;
+  let sourceUrl: string = source;
+  const isUrl = source.startsWith('http://') || source.startsWith('https://');
+
+  if (isUrl) {
+    try {
+      const ssrfCheck = await resolveAndValidateUrl(source);
+      if (!ssrfCheck.safe) {
+        throw new Error(`SSRF check failed: ${ssrfCheck.reason}`);
+      }
+      const response = await fetch(source, { signal: AbortSignal.timeout(30_000) });
+      if (!response.ok) {
+        throw new Error(`HTTP ${response.status} ${response.statusText}`);
+      }
+      rawText = await response.text();
+    } catch (err: any) {
+      const msg = `Failed to fetch ${source}: ${err.message}`;
+      if (json) {
+        console.log(JSON.stringify({ success: false, reason: msg }));
+      } else {
+        console.error(`Error: ${msg}`);
+      }
+      process.exit(1);
+    }
+  } else {
+    try {
+      const { readFile } = await import('node:fs/promises');
+      rawText = await readFile(source, 'utf-8');
+      sourceUrl = `file://${resolve(source)}`;
+    } catch (err: any) {
+      const msg = `Failed to read ${source}: ${err.message}`;
+      if (json) {
+        console.log(JSON.stringify({ success: false, reason: msg }));
+      } else {
+        console.error(`Error: ${msg}`);
+      }
+      process.exit(1);
+    }
+  }
+
+  // Parse JSON or YAML
+  let parsed: any;
+  try {
+    parsed = JSON.parse(rawText);
+  } catch {
+    try {
+      const yaml = await import('js-yaml');
+      parsed = yaml.load(rawText);
+      if (typeof parsed !== 'object' || parsed === null) {
+        throw new Error('YAML parsed to non-object');
+      }
+    } catch (yamlErr: any) {
+      if (json) {
+        console.log(JSON.stringify({ success: false, reason: `Invalid JSON/YAML: ${yamlErr.message}` }));
+      } else {
+        console.error(`Error: Could not parse as JSON or YAML: ${yamlErr.message}`);
+      }
+      process.exit(1);
+    }
+  }
+
+  // Route: OpenAPI spec vs SkillFile
+  if (isOpenAPISpec(parsed)) {
+    await handleOpenAPIImport(parsed, sourceUrl, flags);
+    return;
+  }
+
+  // --- Existing SkillFile import flow (unchanged) ---
+  if (isUrl) {
+    // SkillFile imports from URLs: write to temp file first
+    const { writeFile: writeTmp } = await import('node:fs/promises');
+    const { tmpdir } = await import('node:os');
+    const tmpPath = join(tmpdir(), `apitap-import-${Date.now()}.json`);
+    await writeTmp(tmpPath, rawText);
+    try {
+      await handleSkillFileImport(tmpPath, json);
+    } finally {
+      const { unlink: unlinkTmp } = await import('node:fs/promises');
+      await unlinkTmp(tmpPath).catch(() => {});
+    }
+  } else {
+    await handleSkillFileImport(source, json);
+  }
+}
+
+async function handleSkillFileImport(filePath: string, json: boolean): Promise<void> {
   // Get local key for signature verification
   const machineId = await getMachineId();
   const key = deriveSigningKey(machineId);
 
   // DNS-resolving SSRF check before importing (prevents DNS rebinding attacks)
+  let raw: any;
   try {
-    const raw = JSON.parse(await import('node:fs/promises').then(fs => fs.readFile(filePath, 'utf-8')));
-    if (raw.baseUrl) {
-      const dnsCheck = await resolveAndValidateUrl(raw.baseUrl);
-      if (!dnsCheck.safe) {
-        const msg = `DNS rebinding risk: ${dnsCheck.reason}`;
-        if (json) {
-          console.log(JSON.stringify({ success: false, reason: msg }));
-        } else {
-          console.error(`Error: ${msg}`);
-        }
-        process.exit(1);
+    raw = JSON.parse(await import('node:fs/promises').then(fs => fs.readFile(filePath, 'utf-8')));
+  } catch {
+    // Parse errors will be caught by importSkillFile below
+  }
+
+  if (raw?.baseUrl) {
+    const dnsCheck = await resolveAndValidateUrl(raw.baseUrl);
+    if (!dnsCheck.safe) {
+      const msg = `DNS rebinding risk: ${dnsCheck.reason}`;
+      if (json) {
+        console.log(JSON.stringify({ success: false, reason: msg }));
+      } else {
+        console.error(`Error: ${msg}`);
       }
+      process.exit(1);
     }
-  } catch {
-    // Parse errors will be caught by importSkillFile
   }
 
   const result = await importSkillFile(filePath, undefined, key);
@@ -511,6 +634,306 @@ async function handleImport(positional: string[], flags: Record<string, string |
   }
 }
 
+async function handleOpenAPIImport(
+  spec: Record<string, any>,
+  specUrl: string,
+  flags: Record<string, string | boolean>,
+): Promise<void> {
+  const json = flags.json === true;
+  const dryRun = flags['dry-run'] === true;
+  const update = flags.update === true;
+  const force = flags.force === true;
+  const skillsDir = SKILLS_DIR || join(APITAP_DIR, 'skills');
+
+  // Convert OpenAPI spec to endpoints
+  let importResult;
+  try {
+    importResult = convertOpenAPISpec(spec, specUrl);
+  } catch (err: any) {
+    const msg = `Failed to convert OpenAPI spec: ${err.message}`;
+    if (json) {
+      console.log(JSON.stringify({ success: false, reason: msg }));
+    } else {
+      console.error(`Error: ${msg}`);
+    }
+    process.exit(1);
+  }
+
+  const { domain, endpoints, meta } = importResult;
+  const specVersion = spec.openapi || spec.swagger || 'unknown';
+
+  if (!json) {
+    console.log(`\n  Importing ${domain} from OpenAPI ${specVersion} spec...\n`);
+  }
+
+  // SSRF validate the domain
+  const dnsCheck = await resolveAndValidateUrl(`https://${domain}`);
+  if (!dnsCheck.safe) {
+    const msg = `DNS rebinding risk for ${domain}: ${dnsCheck.reason}`;
+    if (json) {
+      console.log(JSON.stringify({ success: false, reason: msg }));
+    } else {
+      console.error(`Error: ${msg}`);
+    }
+    process.exit(1);
+  }
+
+  // Read existing skill file (if any)
+  let existing = null;
+  try {
+    existing = await readSkillFile(domain, skillsDir, {
+      verifySignature: true,
+      trustUnsigned: true,
+    });
+  } catch (err: any) {
+    if ((err as NodeJS.ErrnoException)?.code !== 'ENOENT') {
+      if (!json) console.error(`  Warning: could not read existing skill file for ${domain}: ${err.message}`);
+    }
+  }
+
+  if (!force && update && existing?.metadata.importHistory?.some(h => h.specUrl === specUrl)) {
+    if (json) {
+      console.log(JSON.stringify({ success: true, skipped: true, reason: 'Already imported from this spec URL' }));
+    } else {
+      console.log('  Already imported from this spec URL. Use --force to reimport.\n');
+    }
+    return;
+  }
+
+  // Merge
+  const { skillFile, diff } = mergeSkillFile(existing, endpoints, meta);
+
+  // Ensure domain and baseUrl reflect the API, not the spec source URL
+  skillFile.domain = domain;
+  skillFile.baseUrl = `https://${domain}`;
+
+  if (dryRun) {
+    if (json) {
+      console.log(JSON.stringify({
+        success: true,
+        dryRun: true,
+        domain,
+        diff,
+        totalEndpoints: skillFile.endpoints.length,
+      }));
+    } else {
+      printOpenAPIDiff(diff, skillFile, skillsDir);
+      console.log('  (dry run — no changes written)\n');
+    }
+    return;
+  }
+
+  // Sign and write
+  const machineId = await getMachineId();
+  const key = deriveSigningKey(machineId);
+  // Determine provenance: 'self' if file has any captured endpoints, 'imported-signed' if all from import
+  const hasCaptured = skillFile.endpoints.some(
+    ep => !ep.endpointProvenance || ep.endpointProvenance === 'captured'
+  );
+  const signed = signSkillFileAs(skillFile, key, hasCaptured ? 'self' : 'imported-signed');
+  const filePath = await writeSkillFile(signed, skillsDir);
+
+  if (json) {
+    console.log(JSON.stringify({
+      success: true,
+      domain,
+      skillFile: filePath,
+      diff,
+      totalEndpoints: signed.endpoints.length,
+    }));
+  } else {
+    printOpenAPIDiff(diff, signed, filePath);
+  }
+}
+
+function printOpenAPIDiff(
+  diff: { preserved: number; added: number; enriched: number; skipped: number },
+  skillFile: { endpoints: { length: number } },
+  pathOrDir: string,
+): void {
+  console.log(`  ✓ ${diff.preserved} existing captured endpoints preserved`);
+  console.log(`  + ${diff.added} new endpoints added from OpenAPI spec`);
+  console.log(`  ~ ${diff.enriched} endpoints enriched with spec metadata`);
+  console.log(`  · ${diff.skipped} skipped (already imported)`);
+  console.log();
+  console.log(`  Skill file: ${pathOrDir} (${skillFile.endpoints.length} endpoints)\n`);
+}
+
+async function handleApisGuruImport(flags: Record<string, string | boolean>): Promise<void> {
+  const json = flags.json === true;
+  const dryRun = flags['dry-run'] === true;
+  const update = flags.update === true;
+  const force = flags.force === true;
+  const limit = typeof flags.limit === 'string' ? parseInt(flags.limit, 10) : 100;
+  const search = typeof flags.search === 'string' ? flags.search : undefined;
+  const skillsDir = SKILLS_DIR || join(APITAP_DIR, 'skills');
+
+  if (!json) {
+    console.log(`\n  Importing from APIs.guru (limit: ${limit})...\n`);
+  }
+
+  // Fetch and filter
+  let entries;
+  try {
+    const allEntries = await fetchApisGuruList();
+    entries = filterEntries(allEntries, { search, limit, preferOpenapi3: true });
+  } catch (err: any) {
+    const msg = `Failed to fetch APIs.guru list: ${err.message}`;
+    if (json) {
+      console.log(JSON.stringify({ success: false, reason: msg }));
+    } else {
+      console.error(`Error: ${msg}`);
+    }
+    process.exit(1);
+  }
+
+  if (entries.length === 0) {
+    if (json) {
+      console.log(JSON.stringify({ success: true, imported: 0, failed: 0, skipped: 0, totalEndpoints: 0 }));
+    } else {
+      console.log('  No matching APIs found.\n');
+    }
+    return;
+  }
+
+  const total = entries.length;
+  let imported = 0;
+  let failed = 0;
+  let skippedApis = 0;
+  let totalEndpointsAdded = 0;
+
+  const machineId = await getMachineId();
+  const key = deriveSigningKey(machineId);
+
+  const results: Array<{
+    index: number;
+    status: 'ok' | 'fail' | 'skip';
+    domain: string;
+    title: string;
+    endpointsAdded: number;
+    error?: string;
+  }> = [];
+
+  for (let i = 0; i < entries.length; i++) {
+    const entry = entries[i];
+    const idx = String(i + 1).padStart(String(total).length, ' ');
+
+    try {
+      // Fetch spec
+      const spec = await fetchSpec(entry.specUrl);
+
+      // Convert
+      const importResult = convertOpenAPISpec(spec, entry.specUrl);
+      const { domain, endpoints, meta } = importResult;
+
+      if (endpoints.length === 0) {
+        if (!json) {
+          console.log(`  [${idx}/${total}] SKIP ${domain.padEnd(24)} 0 endpoints  (${entry.title})`);
+        }
+        results.push({ index: i + 1, status: 'skip', domain, title: entry.title, endpointsAdded: 0 });
+        skippedApis++;
+        continue;
+      }
+
+      // SSRF validate
+      const dnsCheck = await resolveAndValidateUrl(`https://${domain}`);
+      if (!dnsCheck.safe) {
+        if (!json) {
+          console.log(`  [${idx}/${total}] SKIP ${domain.padEnd(24)} SSRF risk  (${entry.title})`);
+        }
+        results.push({ index: i + 1, status: 'skip', domain, title: entry.title, endpointsAdded: 0 });
+        skippedApis++;
+        continue;
+      }
+
+      // Read existing
+      let existing = null;
+      try {
+        existing = await readSkillFile(domain, skillsDir, {
+          verifySignature: true,
+          trustUnsigned: true,
+        });
+      } catch (err: any) {
+        if ((err as NodeJS.ErrnoException)?.code !== 'ENOENT') {
+          if (!json) console.error(`  Warning: could not read existing skill file for ${domain}: ${err.message}`);
+        }
+      }
+
+      if (!force && update && existing?.metadata.importHistory?.length) {
+        const lastImport = existing.metadata.importHistory[existing.metadata.importHistory.length - 1];
+        if (lastImport.importedAt >= entry.updated) {
+          if (!json) console.log(`  [${idx}/${total}] SKIP ${domain.padEnd(24)} up to date`);
+          results.push({ index: i + 1, status: 'skip', domain, title: entry.title, endpointsAdded: 0 });
+          skippedApis++;
+          continue;
+        }
+      }
+
+      // Merge
+      const { skillFile, diff } = mergeSkillFile(existing, endpoints, meta);
+
+      // Ensure domain and baseUrl reflect the API, not the spec source URL
+      skillFile.domain = domain;
+      skillFile.baseUrl = `https://${domain}`;
+
+      if (!dryRun) {
+        // Sign and write
+        // Determine provenance: 'self' if file has any captured endpoints, 'imported-signed' if all from import
+        const hasCaptured = skillFile.endpoints.some(
+          ep => !ep.endpointProvenance || ep.endpointProvenance === 'captured'
+        );
+        const signed = signSkillFileAs(skillFile, key, hasCaptured ? 'self' : 'imported-signed');
+        await writeSkillFile(signed, skillsDir);
+      }
+
+      if (!json) {
+        console.log(`  [${idx}/${total}] OK   ${domain.padEnd(24)} +${diff.added} endpoints  (${entry.title})`);
+      }
+      results.push({ index: i + 1, status: 'ok', domain, title: entry.title, endpointsAdded: diff.added });
+      imported++;
+      totalEndpointsAdded += diff.added;
+    } catch (err: any) {
+      if (!json) {
+        console.log(`  [${idx}/${total}] FAIL ${entry.providerName.padEnd(24)} ${err.message}`);
+      }
+      results.push({
+        index: i + 1,
+        status: 'fail',
+        domain: entry.providerName,
+        title: entry.title,
+        endpointsAdded: 0,
+        error: err.message,
+      });
+      failed++;
+    }
+
+    // Small delay between requests to be polite to APIs.guru
+    if (i < entries.length - 1) {
+      await new Promise(r => setTimeout(r, 100));
+    }
+  }
+
+  if (json) {
+    console.log(JSON.stringify({
+      success: true,
+      dryRun,
+      imported,
+      failed,
+      skipped: skippedApis,
+      totalEndpoints: totalEndpointsAdded,
+      results,
+    }));
+  } else {
+    console.log();
+    console.log(`  Done: ${imported} imported, ${failed} failed, ${skippedApis} skipped`);
+    console.log(`       ${totalEndpointsAdded.toLocaleString()} endpoints added across ${imported} APIs`);
+    if (dryRun) {
+      console.log('  (dry run — no changes written)');
+    }
+    console.log();
+  }
+}
+
 async function handleRefresh(positional: string[], flags: Record<string, string | boolean>): Promise<void> {
   const domain = positional[0];
   if (!domain) {

package/src/discovery/openapi.ts

@@ -1,6 +1,7 @@
 // src/discovery/openapi.ts
 import type { SkillEndpoint, SkillFile, DiscoveredSpec } from '../types.js';
 import { safeFetch } from './fetch.js';
+import { convertOpenAPISpec } from '../skill/openapi-converter.js';
 
 /** Paths to check for API specs, in priority order */
 const SPEC_PATHS = [
@@ -137,7 +138,7 @@ export async function parseSpecToSkillFile(
 
   if (!spec.paths) return null;
 
-  // Determine API base URL
+  // Determine API base URL (discovery uses the passed-in baseUrl as starting point)
   let apiBase = baseUrl;
   if (spec.servers?.[0]?.url) {
     const serverUrl = spec.servers[0].url;
@@ -147,55 +148,29 @@ export async function parseSpecToSkillFile(
     apiBase = `${scheme}://${spec.host}${spec.basePath || ''}`;
   }
 
-  const endpoints: SkillEndpoint[] = [];
-
-  for (const [path, methods] of Object.entries(spec.paths)) {
-    for (const [method, operation] of Object.entries(methods)) {
-      if (!['get', 'post', 'put', 'patch', 'delete'].includes(method)) continue;
-      const op = operation as OpenApiOperation;
-
-      // Parameterize path: {id} → :id
-      const paramPath = path.replace(/\{([^}]+)\}/g, ':$1');
-
-      // Extract query params
-      const queryParams: Record<string, { type: string; example: string }> = {};
-      if (op.parameters) {
-        for (const param of op.parameters) {
-          if (param.in === 'query') {
-            queryParams[param.name] = {
-              type: param.schema?.type || 'string',
-              example: '',
-            };
-          }
-        }
-      }
-
-      // Generate endpoint ID
-      const id = op.operationId
-        ? method.toLowerCase() + '-' + op.operationId.replace(/[^a-z0-9]/gi, '-').toLowerCase()
-        : generateId(method, paramPath);
-
-      endpoints.push({
-        id,
-        method: method.toUpperCase(),
-        path: paramPath,
-        queryParams,
-        headers: {},
-        responseShape: { type: 'unknown' },
-        examples: {
-          request: { url: `${apiBase}${path}`, headers: {} },
-          responsePreview: null,
-        },
-        replayability: {
-          tier: 'unknown',
-          verified: false,
-          signals: ['discovered-from-spec'],
-        },
-      });
-    }
-  }
-
-  if (endpoints.length === 0) return null;
+  // Delegate endpoint extraction to the shared converter
+  const { endpoints: convertedEndpoints } = convertOpenAPISpec(spec, specUrl);
+
+  if (convertedEndpoints.length === 0) return null;
+
+  // Post-process: add replayability signal and fix example URLs to use apiBase
+  const endpoints: SkillEndpoint[] = convertedEndpoints.map(ep => {
+    // Rebuild example URL using apiBase (converter uses https://<domain> but discovery
+    // should use the baseUrl passed in, which may be http:// or a different host)
+    const exampleUrl = ep.examples.request.url.replace(/^https?:\/\/[^/]+/, apiBase.replace(/\/$/, ''));
+    return {
+      ...ep,
+      examples: {
+        ...ep.examples,
+        request: { ...ep.examples.request, url: exampleUrl },
+      },
+      replayability: {
+        tier: 'unknown',
+        verified: false,
+        signals: ['discovered-from-spec'],
+      },
+    };
+  });
 
   return {
     version: '1.2',
@@ -212,12 +187,6 @@ export async function parseSpecToSkillFile(
   };
 }
 
-function generateId(method: string, path: string): string {
-  const segments = path.split('/').filter(s => s !== '' && !s.startsWith(':'));
-  const slug = segments.join('-').replace(/[^a-z0-9-]/gi, '').toLowerCase() || 'root';
-  return `${method.toLowerCase()}-${slug}`;
-}
-
 function parseLinkHeader(header: string, rel: string): string | null {
   const parts = header.split(',');
   for (const part of parts) {