@builtbyecho/public-api-finder 0.5.4 → 0.5.6

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@builtbyecho/public-api-finder",
-  "version": "0.5.4",
+  "version": "0.5.6",
   "description": "Find free/public APIs for agents and prototypes.",
   "type": "module",
   "bin": {

package/src/cli.js

@@ -9,10 +9,11 @@ const SOURCES = {
   publicApiLists: 'https://public-api-lists.github.io/public-api-lists/api/all.json',
   publicApisReadme: 'https://raw.githubusercontent.com/public-apis/public-apis/master/README.md',
   apisGuru: 'https://api.apis.guru/v2/list.json',
+  apiMegaList: 'https://raw.githubusercontent.com/cporter202/API-mega-list/main/README.md',
 };
 const CACHE_PATH = process.env.PUBLIC_API_FINDER_CACHE || join(homedir(), '.cache', 'public-api-finder', 'all.json');
 const CACHE_TTL_MS = 24 * 60 * 60 * 1000;
-const DATA_VERSION = 13;
+const DATA_VERSION = 15;
 
 const ENRICHMENT_FIELDS = [
   'tags',
@@ -574,23 +575,51 @@ function normalizeAuthType(auth) {
   return value && value !== 'unknown' ? value : 'unknown';
 }
 
-function enrichCuratedApi(api) {
+function wordsFrom(value) {
+  return String(value || '')
+    .replace(/&/g, ' and ')
+    .replace(/([a-z])([A-Z])/g, '$1 $2')
+    .toLowerCase()
+    .match(/[a-z0-9]+/g) || [];
+}
+
+function categoryTags(category) {
+  const words = wordsFrom(category).filter(word => !['and', 'api', 'apis', 'openapi', 'unknown'].includes(word));
+  const tags = [];
+  const categoryText = String(category || '').toLowerCase();
+  if (categoryText && !['unknown', 'openapi'].includes(categoryText)) tags.push(categoryText.replace(/\s+/g, ' '));
+  for (const word of words) tags.push(word);
+  return tags;
+}
+
+function inferTags(api, specific = {}) {
+  const text = `${api.name || ''} ${api.category || ''} ${api.description || ''} ${api.url || ''} ${api.provider || ''} ${(specific.tags || []).join(' ')}`;
+  return INTENT_TAG_RULES.filter(([pattern]) => pattern.test(text)).map(([, tag]) => tag);
+}
+
+function enrichApiMetadata(api) {
   const categoryBase = CURATED_CATEGORY_ENRICHMENTS[api.category] || {};
   const specific = CURATED_API_ENRICHMENTS[api.name] || {};
-  const text = `${api.name || ''} ${api.description || ''} ${(specific.tags || []).join(' ')}`;
-  const inferredTags = INTENT_TAG_RULES.filter(([pattern]) => pattern.test(text)).map(([, tag]) => tag);
   const merged = { ...categoryBase, ...specific, ...api };
+  const tags = uniqueStrings([
+    ...(categoryBase.tags || []),
+    ...(specific.tags || []),
+    ...inferTags(api, specific),
+    ...(api.tags || []),
+    ...categoryTags(api.category),
+  ]);
+
   merged.authType = api.authType || specific.authType || categoryBase.authType || normalizeAuthType(api.auth);
   merged.providerType = api.providerType || specific.providerType || categoryBase.providerType || 'public-api';
-  merged.tags = uniqueStrings([...(categoryBase.tags || []), ...(specific.tags || []), ...inferredTags, ...(api.tags || [])]);
-  merged.domains = uniqueStrings([...(categoryBase.domains || []), ...(specific.domains || []), ...(api.domains || [])]);
+  merged.tags = tags.length ? tags : ['public api'];
+  merged.domains = uniqueStrings([...(categoryBase.domains || []), ...(specific.domains || []), ...(api.domains || []), ...categoryTags(api.category).slice(0, 2)]);
   merged.useCases = uniqueStrings([...(categoryBase.useCases || []), ...(specific.useCases || []), ...(api.useCases || [])]);
   merged.caveats = uniqueStrings([...(categoryBase.caveats || []), ...(specific.caveats || []), ...(api.caveats || [])]);
   merged.bestFor = api.bestFor || specific.bestFor || categoryBase.bestFor;
   return merged;
 }
 
-const ENRICHED_CURATED_APIS = CURATED_APIS.map(enrichCuratedApi);
+const ENRICHED_CURATED_APIS = CURATED_APIS.map(enrichApiMetadata);
 
 export function getCuratedApis() {
   return ENRICHED_CURATED_APIS.map(api => ({ ...api, tags: [...(api.tags || [])], domains: [...(api.domains || [])], useCases: [...(api.useCases || [])] }));
@@ -655,7 +684,7 @@ Usage:
 
 Options:
   --category <name>  Filter by category substring
-  --source <name>    Filter by source: public-api-lists, public-apis, apis-guru, curated
+  --source <name>    Filter by source: public-api-lists, public-apis, apis-guru, api-mega-list, curated
   --no-auth          Only APIs with Auth = No
   --https            Only HTTPS APIs
   --cors <value>     Filter by CORS: Yes, No, Unknown
@@ -705,8 +734,10 @@ function applyQueryHints(args) {
   if (!args.cors && /\b(cors|frontend-safe|browser-safe|frontend safe|browser safe)\b/.test(query)) args.cors = 'Yes';
 }
 
+const SEARCH_STOPWORDS = new Set(['a', 'an', 'and', 'api', 'apis', 'for', 'from', 'in', 'no', 'of', 'on', 'or', 'the', 'to', 'with']);
+
 function tokenSet(text) {
-  return new Set(String(text).toLowerCase().match(/[a-z0-9]+/g)?.filter(t => t.length > 1) || []);
+  return new Set(String(text).toLowerCase().match(/[a-z0-9]+/g)?.filter(t => t.length > 1 && !SEARCH_STOPWORDS.has(t)) || []);
 }
 
 function intersectionCount(a, b) {
@@ -805,6 +836,10 @@ function intentPenalty(entry, queryText) {
     if (/\b(favicon|website preview|open graph|link preview|screenshot)\b/.test(queryText) && !/\b(microlink|urlbox|favicon|website metadata|open graph|link preview|screenshot)\b/.test(text)) penalty += 120;
   }
 
+  if (/\b(school district|school boundary|district boundary)\b/.test(queryText) && /\b(linkedin|jobs scraper|lead|sales|recruiting)\b/.test(text)) {
+    penalty += 95;
+  }
+
   if (!cat.includes('cryptocurrency') && /\b(wallet address|identicon|avatar|profile picture)\b/.test(queryText) && /\b(avatar|identicon|profile picture)\b/.test(text)) {
     penalty -= 20;
   }
@@ -896,6 +931,42 @@ function parsePublicApisReadme(readme) {
   return entries;
 }
 
+
+function parseApiMegaList(readme) {
+  const entries = [];
+  let category = '';
+  for (const raw of readme.split('\n')) {
+    const heading = raw.match(/^##\s+(.+?)\s*$/) || raw.match(/^###\s+(.+?)\s*$/);
+    if (heading) {
+      const text = heading[1].replace(/[#*_`]/g, '').trim();
+      if (text && !/table of contents|repository stats|star this|join my|contributing|license/i.test(text)) category = normalizeCategory(text.replace(/^\d+\.\s*/, ''));
+      continue;
+    }
+    if (!raw.startsWith('| [')) continue;
+    const cells = raw.split('|').slice(1, -1).map(c => c.trim());
+    if (cells.length < 2) continue;
+    if (/^-+$/.test(cells[0]) || /^api name$/i.test(cells[0])) continue;
+    const link = cells[0].match(/\[([^\]]+)\]\(([^)]+)\)/);
+    if (!link) continue;
+    const name = link[1].replace(/<[^>]+>/g, '').trim();
+    const url = link[2].trim();
+    const description = cleanDescription(cells[1] || `${name} API`);
+    if (!name || !/^https?:\/\//i.test(url)) continue;
+    entries.push({
+      name,
+      url,
+      description,
+      auth: 'Unknown',
+      https: /^https:/i.test(url),
+      cors: 'Unknown',
+      category: category || 'Unknown',
+      source: 'api-mega-list',
+      sourceWeight: 1,
+    });
+  }
+  return entries;
+}
+
 function parseApisGuru(data) {
   const entries = [];
   for (const [providerName, item] of Object.entries(data || {})) {
@@ -920,10 +991,11 @@ function parseApisGuru(data) {
 }
 
 async function buildData() {
-  const [pal, publicApisReadme, guru] = await Promise.allSettled([
+  const [pal, publicApisReadme, guru, megaList] = await Promise.allSettled([
     fetchJson(SOURCES.publicApiLists),
     fetchText(SOURCES.publicApisReadme),
     fetchJson(SOURCES.apisGuru),
+    fetchText(SOURCES.apiMegaList),
   ]);
   const entries = [];
   const sourceStatus = {};
@@ -941,9 +1013,15 @@ async function buildData() {
     sourceStatus['apis-guru'] = rows.length;
     entries.push(...rows);
   } else sourceStatus['apis-guru'] = `error: ${guru.reason.message}`;
+  if (megaList.status === 'fulfilled') {
+    const rows = parseApiMegaList(megaList.value);
+    sourceStatus['api-mega-list'] = rows.length;
+    entries.push(...rows);
+  } else sourceStatus['api-mega-list'] = `error: ${megaList.reason.message}`;
   sourceStatus.curated = ENRICHED_CURATED_APIS.length;
   entries.push(...ENRICHED_CURATED_APIS);
-  return { dataVersion: DATA_VERSION, generatedAt: new Date().toISOString(), sourceStatus, entries: dedupe(entries) };
+  const deduped = dedupe(entries).map(enrichApiMetadata);
+  return { dataVersion: DATA_VERSION, generatedAt: new Date().toISOString(), sourceStatus, entries: deduped };
 }
 
 function keyFor(entry) {
@@ -1006,7 +1084,7 @@ function dedupe(entries) {
 async function loadData(refresh = false) {
   if (!refresh && await cacheIsFresh()) {
     const cached = JSON.parse(await readFile(CACHE_PATH, 'utf8'));
-    if (cached.dataVersion === DATA_VERSION) return cached.entries || [];
+    if (cached.dataVersion === DATA_VERSION) return (cached.entries || []).map(enrichApiMetadata);
   }
   const data = await buildData();
   await mkdir(dirname(CACHE_PATH), { recursive: true });
@@ -1084,6 +1162,10 @@ async function checkRows(rows) {
   return checked;
 }
 
+export async function buildSearchIndex(options = {}) {
+  return loadData(Boolean(options.refresh));
+}
+
 export async function searchApis(options = {}) {
   const args = {
     query: options.query || '',