@builtbyecho/public-api-finder 0.5.5 → 0.5.7

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@builtbyecho/public-api-finder",
-  "version": "0.5.5",
+  "version": "0.5.7",
   "description": "Find free/public APIs for agents and prototypes.",
   "type": "module",
   "bin": {

package/scripts/eval-hardening.mjs

@@ -0,0 +1,78 @@
+#!/usr/bin/env node
+import { spawnSync } from 'node:child_process';
+
+const cases = [
+  { q: 'crypto prices', args: ['--no-auth','--https','--cors','Yes'], topCategory: /cryptocurrency/i, top3Category: /cryptocurrency/i, mustName: /dexscreener|dexpaprika|defillama|geckoterminal|coinpaprika/i, forbid: /weather|joke|tvmaze|jobs/i },
+  { q: 'stock prices', args: ['--no-auth','--https','--cors','Yes'], topCategory: /finance/i, top3Category: /finance/i, forbid: /weather|joke|tvmaze|jobs/i },
+  { q: 'free stock quote API no auth', args: ['--no-auth','--https'], topCategory: /finance/i, mustName: /stooq|portfolio|alpha|finnhub|twelve/i, forbid: /weather|joke|anime|tv/i },
+  { q: 'solana token price market cap dex no key', args: ['--no-auth','--https'], topCategory: /cryptocurrency/i, top3Category: /cryptocurrency/i, mustName: /dexscreener|dexpaprika|geckoterminal|defillama/i },
+  { q: 'wallet balance transactions erc20 transfers api', args: [], topCategory: /cryptocurrency|openapi/i, mustName: /etherscan|alchemy|moralis|covalent|block/i },
+  { q: 'weather forecast no auth cors', args: ['--no-auth','--https','--cors','Yes'], topCategory: /weather/i, top3Category: /weather/i, mustName: /open-meteo|national weather|pirate/i, forbid: /joke|crypto|stocks/i },
+  { q: 'historical weather api', args: [], topCategory: /weather/i, mustName: /open-meteo|weather/i },
+  { q: 'weather alerts us api', args: ['--no-auth'], topCategory: /weather/i, mustName: /national weather|weather/i },
+  { q: 'geocoding api no auth', args: ['--no-auth','--https'], topCategory: /geocoding|location/i, mustName: /nominatim|geocod|zippopotam|ipinfo/i },
+  { q: 'reverse geocoding api', args: [], topCategory: /geocoding|location/i, mustName: /nominatim|geocod|mapbox|google/i },
+  { q: 'maps routing api', args: [], topCategory: /geocoding|location/i, mustName: /graphhopper|mapbox|google|here|tomtom/i, forbid: /linkedin/i },
+  { q: 'public transit api', args: [], topCategory: /transport|transit/i, mustName: /transitland|transport|mbta/i },
+  { q: 'jobs api remote', args: [], topCategory: /jobs|agents/i, mustName: /graphql jobs|adzuna|usajobs|search.gov|linkedin jobs/i },
+  { q: 'government jobs api', args: [], topCategory: /jobs|government/i, mustName: /usajobs|search.gov/i },
+  { q: 'company enrichment api', args: [], topCategory: /business|marketing|openapi/i, mustName: /clearbit|brandfetch|opencorporates|company/i },
+  { q: 'email validation api', args: [], topCategory: /email/i, mustName: /abstract|email|mail/i },
+  { q: 'phone number lookup api', args: [], topCategory: /communication|telecom|security|openapi/i, mustName: /numverify|twilio|phone|abstract/i },
+  { q: 'sms api', args: [], topCategory: /communication|messaging|telecom|openapi/i, mustName: /twilio|telnyx|vonage|sms|message/i },
+  { q: 'webhook testing api', args: [], mustName: /webhook|requestbin|pipedream|beeceptor|alchemy/i },
+  { q: 'payment processing api', args: [], topCategory: /payments|financial|openapi/i, mustName: /stripe|paypal|checkout|payment/i },
+  { q: 'subscriptions billing api', args: [], topCategory: /payments|openapi/i, mustName: /stripe|billing|subscription|chargebee/i },
+  { q: 'oauth identity api', args: [], topCategory: /authentication|security|development|openapi/i, mustName: /auth0|clerk|okta|oauth|openid/i },
+  { q: 'passwordless auth api', args: [], topCategory: /authentication|security|openapi/i, mustName: /auth0|clerk|magic|stytch|passwordless/i },
+  { q: 'screenshot api', args: [], topCategory: /development|media|documents|openapi/i, mustName: /urlbox|microlink|screenshot/i },
+  { q: 'link preview api', args: [], topCategory: /development|media|utility|openapi/i, mustName: /microlink|linkpreview|metadata|open graph/i },
+  { q: 'website metadata api', args: [], topCategory: /development|media|utility|openapi/i, mustName: /microlink|metadata|open graph|urlbox/i },
+  { q: 'pdf generation api', args: [], topCategory: /documents|development|utility|openapi/i, mustName: /pdf|document/i },
+  { q: 'ocr api', args: [], topCategory: /machine learning|ai|documents|openapi/i, mustName: /ocr|vision|mindee/i },
+  { q: 'image generation api', args: [], topCategory: /ai|machine learning|media|openapi/i, mustName: /stability|openai|replicate|image/i },
+  { q: 'text to speech api', args: [], topCategory: /ai|audio|machine learning|openapi/i, mustName: /elevenlabs|speech|voice|tts/i },
+  { q: 'speech to text api', args: [], topCategory: /ai|audio|machine learning|openapi/i, mustName: /assemblyai|deepgram|whisper|speech/i },
+  { q: 'vulnerability database api', args: [], topCategory: /security|development|open data/i, mustName: /osv|nvd|cve|vulnerab/i },
+  { q: 'cve lookup api', args: [], topCategory: /security|open data/i, mustName: /nvd|cve|osv/i },
+  { q: 'npm package metadata api', args: ['--no-auth'], topCategory: /development|open data/i, mustName: /npm|libraries.io|package/i },
+  { q: 'github repo stats api', args: [], topCategory: /development|open data/i, mustName: /github|gitlab/i },
+  { q: 'domain dns lookup api', args: [], topCategory: /security|development|openapi/i, mustName: /dns|google dns|whois|ssl/i },
+  { q: 'open data demographics api', args: [], topCategory: /government|open data/i, mustName: /census|data.gov|demographics/i },
+  { q: 'fake ecommerce api for testing', args: ['--no-auth','--https'], topCategory: /test data|shopping/i, mustName: /fake store|dummyjson|jsonplaceholder/i },
+  { q: 'joke api', args: ['--no-auth'], topCategory: /entertainment|jokes/i, mustName: /joke/i, forbid: /weather|crypto|stock/i },
+  { q: 'cat images random dog facts no auth', args: ['--no-auth'], mustName: /cat|dog|thecatapi|dog ceo/i, forbid: /weather|stock|crypto/i },
+  { q: 'food barcode ingredients allergens nutrition no auth cors', args: ['--no-auth','--https'], topCategory: /food/i, mustName: /open food facts/i },
+  { q: 'recipe from pantry ingredients avoid allergens api', args: [], topCategory: /food/i, mustName: /spoonacular|edamam|recipe|meal/i },
+  { q: 'air quality by coordinates pollutant measurements', args: [], topCategory: /environment|science|weather/i, mustName: /openaq|air quality|aqicn/i },
+  { q: 'historical currency exchange rates no auth cors', args: ['--no-auth','--https'], topCategory: /currency exchange/i, mustName: /frankfurter|currency/i, forbid: /crypto|bitcoin|dex/i },
+  { q: 'public holidays by country no auth', args: ['--no-auth','--https'], topCategory: /calendar|date/i, mustName: /nager|holiday|calendarific/i },
+  { q: 'vehicle vin decode recall safety data no auth', args: ['--no-auth'], topCategory: /transportation|government|open data/i, mustName: /nhtsa|vin|vehicle/i },
+  { q: 'real estate property value rent estimate api', args: [], topCategory: /real estate|property|openapi|finance/i, mustName: /rentcast|attom|zillow|real estate|property/i },
+  { q: 'flight status airport arrivals departures api', args: [], topCategory: /transportation|travel|openapi/i, mustName: /aviation|flight|airport|amadeus/i },
+  { q: 'hotel search booking availability api', args: [], topCategory: /travel|commerce|openapi/i, mustName: /hotel|booking|amadeus/i },
+  { q: 'brand colors fonts logo company domain enrichment', args: [], topCategory: /business|marketing|development|openapi/i, mustName: /brandfetch|clearbit|logo|brand/i },
+];
+
+let failures = 0;
+for (const [i, c] of cases.entries()) {
+  const res = spawnSync(process.execPath, ['src/cli.js', c.q, ...c.args, '--limit', '5', '--json'], { encoding: 'utf8' });
+  const rows = res.status === 0 ? JSON.parse(res.stdout || '[]') : [];
+  const top = rows[0];
+  const checks = [];
+  if (!top) checks.push('no results');
+  if (top && c.topCategory && !c.topCategory.test(top.category || '')) checks.push(`top category ${top.category} !~ ${c.topCategory}`);
+  if (top && c.top3Category) {
+    const bad = rows.slice(0, 3).filter(r => !c.top3Category.test(r.category || '')).map(r => `${r.name} (${r.category})`);
+    if (bad.length) checks.push(`top3 off-domain: ${bad.join(', ')}`);
+  }
+  if (c.mustName && !rows.slice(0, 3).some(r => c.mustName.test(`${r.name} ${r.description} ${r.url}`))) checks.push(`top3 missing ${c.mustName}`);
+  if (c.forbid && rows.slice(0, 5).some(r => c.forbid.test(`${r.name} ${r.category} ${r.description}`))) checks.push(`forbidden result ${c.forbid}`);
+  const ok = checks.length === 0;
+  if (!ok) failures++;
+  console.log(`${ok ? 'PASS' : 'FAIL'} ${i + 1}. ${c.q}`);
+  rows.slice(0, 3).forEach((r, idx) => console.log(`  ${idx + 1}. ${r.name} | ${r.category} | auth=${r.auth} | cors=${r.cors} | score=${r.score}`));
+  if (!ok) console.log(`  Reasons: ${checks.join('; ')}`);
+}
+console.log(`\n${cases.length - failures}/${cases.length} hardening cases passed`);
+process.exitCode = failures ? 1 : 0;

package/src/cli.js

@@ -9,10 +9,11 @@ const SOURCES = {
   publicApiLists: 'https://public-api-lists.github.io/public-api-lists/api/all.json',
   publicApisReadme: 'https://raw.githubusercontent.com/public-apis/public-apis/master/README.md',
   apisGuru: 'https://api.apis.guru/v2/list.json',
+  apiMegaList: 'https://raw.githubusercontent.com/cporter202/API-mega-list/main/README.md',
 };
 const CACHE_PATH = process.env.PUBLIC_API_FINDER_CACHE || join(homedir(), '.cache', 'public-api-finder', 'all.json');
 const CACHE_TTL_MS = 24 * 60 * 60 * 1000;
-const DATA_VERSION = 14;
+const DATA_VERSION = 16;
 
 const ENRICHMENT_FIELDS = [
   'tags',
@@ -53,6 +54,15 @@ const TARGETED_BOOSTS = [
   [/\b(sanctions|ofac|pep|kyc|aml)\b/, /\b(opensanctions|ofac|sanctions|pep|kyc|aml|chainalysis)\b/i, 135],
   [/\b(wallet risk|crypto sanctions|blockchain wallet risk)\b/, /\b(chainalysis|trm|elliptic|sanctions|wallet risk)\b/i, 95],
   [/\b(crypto token metadata|token metadata|token logos|coin logos|logos contract addresses)\b/, /\b(coinmarketcap|coingecko|coinpaprika|coinbase|token logos|coin metadata|coin images)\b/i, 90],
+  [/\b(stock|stocks|stock quote|stock prices|equity|equities|ticker|tickers)\b/, /\b(stooq|portfolio optimizer|alpha vantage|polygon|finnhub|twelve data|stock|stocks|equity|ticker|quote)\b/i, 90],
+  [/\b(reverse geocoding|geocoding|maps routing|routing api|distance matrix|places)\b/, /\b(nominatim|geocod\.io|graphhopper|mapbox|foursquare|geocoding|routing|distance matrix|places)\b/i, 120],
+  [/\b(webhook testing|webhook debug|request bin|mock api)\b/, /\b(webhook\.site|beeceptor|requestbin|webhook|mock api|request bin)\b/i, 180],
+  [/\b(oauth identity|passwordless auth|openid|social auth|login oauth)\b/, /\b(auth0|clerk|stytch|magic|openid|oauth|passwordless|authentication|identity)\b/i, 190],
+  [/\b(cve lookup|vulnerability database|vulnerability data|package security)\b/, /\b(osv|nvd|cve|cvss|vulnerability|security advisories)\b/i, 190],
+  [/\b(cve lookup|cve api|cve search)\b/, /\b(osv|nvd|cve)\b/i, 260],
+  [/\b(open data demographics|demographics|census data)\b/, /\b(census|data\.gov|demographics|open data|government data)\b/i, 145],
+  [/\b(joke|jokes|meme|memes|random quote)\b/, /\b(official joke|joke api|jokes|quotable|quote)\b/i, 110],
+  [/\b(github repo|repo stats|stars issues commits)\b/, /\b(github rest|github|gitlab|repositories|stars|issues|commits)\b/i, 100],
   [/\b(zipcode|zip code|postal code)\b/, /\b(zippopotam|zip|postal|census)\b/i, 80],
   [/\b(real estate|property value|rent estimate)\b/, /\b(rentcast|attom|zillow|real estate|property|rent estimate)\b/i, 135],
   [/\b(mortgage|mortgage rates|loan calculator|loan rate|home loan)\b/, /\b(mortgage|home loan|loan rate)\b/i, 230],
@@ -387,9 +397,13 @@ const CURATED_APIS = [
   { name: 'GitHub REST API', url: 'https://docs.github.com/en/rest', description: 'GitHub repositories, stars, issues, commits, pull requests, releases, users, and org data API.', auth: 'No', https: true, cors: 'Yes', category: 'Development', source: 'curated', sourceWeight: 5 },
   { name: 'npm Registry API', url: 'https://github.com/npm/registry/blob/main/docs/REGISTRY-API.md', description: 'No-auth npm package metadata, versions, downloads, dist-tags, registry documents, and package data.', auth: 'No', https: true, cors: 'Yes', category: 'Development', source: 'curated', sourceWeight: 5 },
   { name: 'Docker Hub', url: 'https://docs.docker.com/docker-hub/api/latest/', description: 'Docker image repositories, tags, registry metadata, namespaces, vulnerabilities, and container image data API.', auth: 'apiKey', https: true, cors: 'Unknown', category: 'Development', source: 'curated', sourceWeight: 5 },
+  { name: 'Webhook.site', url: 'https://webhook.site/', description: 'Webhook testing, inspect HTTP requests, debug callbacks, request bins, and temporary webhook endpoints.', auth: 'No', https: true, cors: 'Yes', category: 'Development', source: 'curated', sourceWeight: 5 },
+  { name: 'Beeceptor', url: 'https://beeceptor.com/docs/', description: 'Webhook testing, mock APIs, request bin inspection, HTTP debugging, and endpoint simulation.', auth: 'No', https: true, cors: 'Unknown', category: 'Development', source: 'curated', sourceWeight: 5 },
 
   { name: 'Auth0', url: 'https://auth0.com/docs/api', description: 'OAuth, OpenID Connect, login, user profiles, social auth, authentication, and identity APIs.', auth: 'apiKey', https: true, cors: 'Unknown', category: 'Authentication', source: 'curated', sourceWeight: 5 },
   { name: 'Clerk', url: 'https://clerk.com/docs/reference/backend-api', description: 'Authentication, user profiles, organizations, sessions, OAuth, social login, and identity APIs.', auth: 'apiKey', https: true, cors: 'Unknown', category: 'Authentication', source: 'curated', sourceWeight: 5 },
+  { name: 'Stytch', url: 'https://stytch.com/docs/api', description: 'Passwordless login, magic links, OTP, OAuth, passkeys, sessions, and user authentication API.', auth: 'apiKey', https: true, cors: 'Unknown', category: 'Authentication', source: 'curated', sourceWeight: 5 },
+  { name: 'Magic', url: 'https://magic.link/docs/api', description: 'Passwordless authentication, magic links, wallets, login, users, and identity API.', auth: 'apiKey', https: true, cors: 'Unknown', category: 'Authentication', source: 'curated', sourceWeight: 5 },
   { name: 'Mail.tm', url: 'https://docs.mail.tm/', description: 'No-auth temporary email inboxes, disposable mail accounts, receive messages, and testing email API.', auth: 'No', https: true, cors: 'Yes', category: 'Email', source: 'curated', sourceWeight: 5 },
   { name: 'Twilio Verify', url: 'https://www.twilio.com/docs/verify/api', description: 'SMS OTP verification, phone verification, one-time passcodes, factors, and verification checks.', auth: 'apiKey', https: true, cors: 'Unknown', category: 'Telecom', source: 'curated', sourceWeight: 5 },
   { name: 'numverify', url: 'https://numverify.com/documentation', description: 'Phone number validation, carrier, country, location, line type, and international number lookup API.', auth: 'apiKey', https: true, cors: 'Unknown', category: 'Telecom', source: 'curated', sourceWeight: 5 },
@@ -683,7 +697,7 @@ Usage:
 
 Options:
   --category <name>  Filter by category substring
-  --source <name>    Filter by source: public-api-lists, public-apis, apis-guru, curated
+  --source <name>    Filter by source: public-api-lists, public-apis, apis-guru, api-mega-list, curated
   --no-auth          Only APIs with Auth = No
   --https            Only HTTPS APIs
   --cors <value>     Filter by CORS: Yes, No, Unknown
@@ -835,6 +849,37 @@ function intentPenalty(entry, queryText) {
     if (/\b(favicon|website preview|open graph|link preview|screenshot)\b/.test(queryText) && !/\b(microlink|urlbox|favicon|website metadata|open graph|link preview|screenshot)\b/.test(text)) penalty += 120;
   }
 
+  if (/\b(school district|school boundary|district boundary|reverse geocoding|maps routing|routing api|open data demographics|demographics|census data)\b/.test(queryText) && /\b(linkedin|jobs scraper|lead|sales|recruiting|amazon product scraper|tiktok profile scraper)\b/.test(text)) {
+    penalty += 120;
+  }
+  if (/\b(stock|stocks|stock quote|stock prices|equity|equities|ticker|tickers)\b/.test(queryText) && !/finance|financial|stock|stocks|equity|ticker|market data|portfolio|stooq|finnhub|polygon|alpha vantage|twelve data/.test(text)) {
+    penalty += 80;
+  }
+  if (/\b(stock|stocks|stock quote|stock prices|equity|equities|ticker|tickers)\b/.test(queryText) && /\b(quotable|joke|entertainment|food|weather|test data|placeholder)\b/.test(text)) {
+    penalty += 140;
+  }
+  if (/\bcrypto\b/.test(queryText) && /\bnot stocks?\b/.test(queryText) && /\b(finance|financial|stock|stocks|equity|ticker|portfolio|stooq|finnhub|polygon|sec edgar|predscope|valueray|econdb)\b/.test(text)) {
+    penalty += 240;
+  }
+  if (/\b(oauth identity|passwordless auth|openid|social auth|login oauth)\b/.test(queryText) && !/authentication|auth0|clerk|stytch|magic|openid|identity|passwordless|social auth/.test(text)) {
+    penalty += 160;
+  }
+  if (/\b(oauth identity|passwordless auth|openid|social auth|login oauth)\b/.test(queryText) && /\b(calendar|nager|holiday|open food|plaid|clearbit logo)\b/.test(text)) {
+    penalty += 180;
+  }
+  if (/\b(webhook testing|webhook debug|request bin|mock api)\b/.test(queryText) && !/webhook|beeceptor|requestbin|mock api|pipedream/.test(text)) {
+    penalty += 80;
+  }
+  if (/\b(cve lookup|vulnerability database|vulnerability data|package security)\b/.test(queryText) && !/osv|nvd|cve|cvss|vulnerability|security advisories/.test(text)) {
+    penalty += 95;
+  }
+  if (/\b(cve lookup|cve api|cve search)\b/.test(queryText) && !/osv|nvd|cve/.test(text)) {
+    penalty += 180;
+  }
+  if (/\b(joke api|jokes?|memes?)\b/.test(queryText) && !/joke|meme|quote/.test(text)) {
+    penalty += 70;
+  }
+
   if (!cat.includes('cryptocurrency') && /\b(wallet address|identicon|avatar|profile picture)\b/.test(queryText) && /\b(avatar|identicon|profile picture)\b/.test(text)) {
     penalty -= 20;
   }
@@ -926,6 +971,42 @@ function parsePublicApisReadme(readme) {
   return entries;
 }
 
+
+function parseApiMegaList(readme) {
+  const entries = [];
+  let category = '';
+  for (const raw of readme.split('\n')) {
+    const heading = raw.match(/^##\s+(.+?)\s*$/) || raw.match(/^###\s+(.+?)\s*$/);
+    if (heading) {
+      const text = heading[1].replace(/[#*_`]/g, '').trim();
+      if (text && !/table of contents|repository stats|star this|join my|contributing|license/i.test(text)) category = normalizeCategory(text.replace(/^\d+\.\s*/, ''));
+      continue;
+    }
+    if (!raw.startsWith('| [')) continue;
+    const cells = raw.split('|').slice(1, -1).map(c => c.trim());
+    if (cells.length < 2) continue;
+    if (/^-+$/.test(cells[0]) || /^api name$/i.test(cells[0])) continue;
+    const link = cells[0].match(/\[([^\]]+)\]\(([^)]+)\)/);
+    if (!link) continue;
+    const name = link[1].replace(/<[^>]+>/g, '').trim();
+    const url = link[2].trim();
+    const description = cleanDescription(cells[1] || `${name} API`);
+    if (!name || !/^https?:\/\//i.test(url)) continue;
+    entries.push({
+      name,
+      url,
+      description,
+      auth: 'Unknown',
+      https: /^https:/i.test(url),
+      cors: 'Unknown',
+      category: category || 'Unknown',
+      source: 'api-mega-list',
+      sourceWeight: 1,
+    });
+  }
+  return entries;
+}
+
 function parseApisGuru(data) {
   const entries = [];
   for (const [providerName, item] of Object.entries(data || {})) {
@@ -950,10 +1031,11 @@ function parseApisGuru(data) {
 }
 
 async function buildData() {
-  const [pal, publicApisReadme, guru] = await Promise.allSettled([
+  const [pal, publicApisReadme, guru, megaList] = await Promise.allSettled([
     fetchJson(SOURCES.publicApiLists),
     fetchText(SOURCES.publicApisReadme),
     fetchJson(SOURCES.apisGuru),
+    fetchText(SOURCES.apiMegaList),
   ]);
   const entries = [];
   const sourceStatus = {};
@@ -971,6 +1053,11 @@ async function buildData() {
     sourceStatus['apis-guru'] = rows.length;
     entries.push(...rows);
   } else sourceStatus['apis-guru'] = `error: ${guru.reason.message}`;
+  if (megaList.status === 'fulfilled') {
+    const rows = parseApiMegaList(megaList.value);
+    sourceStatus['api-mega-list'] = rows.length;
+    entries.push(...rows);
+  } else sourceStatus['api-mega-list'] = `error: ${megaList.reason.message}`;
   sourceStatus.curated = ENRICHED_CURATED_APIS.length;
   entries.push(...ENRICHED_CURATED_APIS);
   const deduped = dedupe(entries).map(enrichApiMetadata);