@apitap/core 1.8.2 → 1.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/cli.ts CHANGED
@@ -32,6 +32,18 @@ import { mergeSkillFile } from './skill/merge.js';
32
32
  import { fetchApisGuruList, filterEntries, fetchSpec } from './skill/apis-guru.js';
33
33
  import { searchSwaggerHub, fetchSwaggerHubSpec } from './skill/swaggerhub.js';
34
34
  import { buildIndex, removeFromIndex } from './skill/index.js';
35
+ import {
36
+ resolveGitHubToken,
37
+ searchOrgSpecs,
38
+ searchTopicSpecs,
39
+ fetchGitHubSpec,
40
+ filterResults,
41
+ hasServerUrl,
42
+ isLocalhostSpec,
43
+ normalizeSpecServerUrls,
44
+ CANONICAL_TOPICS,
45
+ type RateLimit,
46
+ } from './skill/github.js';
35
47
 
36
48
  const __dirname = fileURLToPath(new URL('.', import.meta.url));
37
49
  const pkg = JSON.parse(readFileSync(join(__dirname, '..', 'package.json'), 'utf-8'));
@@ -86,6 +98,10 @@ function printUsage(): void {
86
98
  Bulk-import from APIs.guru directory
87
99
  apitap import --from swaggerhub --query <term>
88
100
  Import from SwaggerHub (785K+ public specs)
101
+ apitap import --from github --org <name>
102
+ Import specs from a GitHub org's repos
103
+ apitap import --from github --topic [name]
104
+ Import from topic-tagged repos (--query to filter)
89
105
  apitap refresh <domain> Refresh auth tokens via browser
90
106
  apitap auth [domain] View or manage stored auth
91
107
  apitap mcp Run the full ApiTap MCP server over stdio
@@ -522,6 +538,12 @@ async function handleImport(positional: string[], flags: Record<string, string |
522
538
  return;
523
539
  }
524
540
 
541
+ // --from github: GitHub import mode
542
+ if (flags['from'] === 'github') {
543
+ await handleGitHubImport(flags);
544
+ return;
545
+ }
546
+
525
547
  const source = positional[0];
526
548
  if (!source) {
527
549
  console.error('Error: File path or URL required. Usage: apitap import <file|url>');
@@ -1131,6 +1153,352 @@ async function handleSwaggerHubImport(flags: Record<string, string | boolean>):
1131
1153
  }
1132
1154
  }
1133
1155
 
1156
+ async function handleGitHubImport(flags: Record<string, string | boolean>): Promise<void> {
1157
+ const json = flags.json === true;
1158
+ const force = flags.force === true;
1159
+ const update = flags.update === true;
1160
+ const dryRun = flags['dry-run'] === true;
1161
+ const noAuthOnly = flags['no-auth-only'] === true;
1162
+ const includeStale = flags['include-stale'] === true;
1163
+ const limit = typeof flags.limit === 'string' ? parseInt(flags.limit, 10) : 20;
1164
+ const org = typeof flags.org === 'string' ? flags.org : undefined;
1165
+ const topicFlag = flags.topic;
1166
+ const query = typeof flags.query === 'string' ? flags.query : undefined;
1167
+ const skillsDir = SKILLS_DIR || join(APITAP_DIR, 'skills');
1168
+
1169
+ // Mutual exclusion
1170
+ if (org && topicFlag) {
1171
+ const msg = '--org and --topic are mutually exclusive';
1172
+ if (json) {
1173
+ console.log(JSON.stringify({ success: false, reason: msg }));
1174
+ } else {
1175
+ console.error(`Error: ${msg}`);
1176
+ }
1177
+ process.exit(1);
1178
+ }
1179
+ if (!org && !topicFlag) {
1180
+ const msg = '--org or --topic required. Usage: apitap import --from github --org <name> OR --topic [name]';
1181
+ if (json) {
1182
+ console.log(JSON.stringify({ success: false, reason: msg }));
1183
+ } else {
1184
+ console.error(`Error: ${msg}`);
1185
+ }
1186
+ process.exit(1);
1187
+ }
1188
+
1189
+ // --min-stars: different defaults per mode
1190
+ const minStarsDefault = org ? 0 : 10;
1191
+ const minStars = typeof flags['min-stars'] === 'string'
1192
+ ? parseInt(flags['min-stars'], 10) : minStarsDefault;
1193
+
1194
+ // --topic parsing: bare --topic = all four topics, --topic openapi = just that one
1195
+ const topics = typeof topicFlag === 'string'
1196
+ ? [topicFlag]
1197
+ : CANONICAL_TOPICS;
1198
+
1199
+ // Auth requirement for --org
1200
+ const token = await resolveGitHubToken();
1201
+ if (org && token === null) {
1202
+ const msg = "--org requires a GitHub token. Run 'gh auth login' or set GITHUB_TOKEN.";
1203
+ if (json) {
1204
+ console.log(JSON.stringify({ success: false, reason: msg }));
1205
+ } else {
1206
+ console.error(`Error: ${msg}`);
1207
+ }
1208
+ process.exit(1);
1209
+ }
1210
+
1211
+ // Discovery phase
1212
+ if (!json) {
1213
+ if (org) {
1214
+ console.log(`\n Scanning GitHub org "${org}" for OpenAPI specs (limit: ${limit})...\n`);
1215
+ } else {
1216
+ const topicStr = topics.join(', ');
1217
+ console.log(`\n Searching GitHub topics [${topicStr}] (min-stars: ${minStars}, limit: ${limit})...\n`);
1218
+ }
1219
+ }
1220
+
1221
+ let rawResults: import('./skill/github.js').GitHubSpecResult[];
1222
+ // Rate limit tracking: stored in a mutable container so TypeScript doesn't
1223
+ // narrow to `null` based on control flow. Currently not populated because
1224
+ // searchOrgSpecs/searchTopicSpecs don't expose rate limit info, but the
1225
+ // plumbing is in place for when they do.
1226
+ const rateLimitRef: { value: RateLimit | null } = { value: null };
1227
+
1228
+ try {
1229
+ if (org) {
1230
+ rawResults = await searchOrgSpecs(org, token);
1231
+ } else {
1232
+ rawResults = await searchTopicSpecs(topics, token, { minStars, query });
1233
+ }
1234
+ } catch (err: any) {
1235
+ const msg = `GitHub discovery failed: ${err.message}`;
1236
+ if (json) {
1237
+ console.log(JSON.stringify({ success: false, reason: msg }));
1238
+ } else {
1239
+ console.error(`Error: ${msg}`);
1240
+ }
1241
+ process.exit(1);
1242
+ }
1243
+
1244
+ // Filter: forks, archived, stale, min-stars
1245
+ const { passed, skips: repoSkips } = filterResults(rawResults, { includeStale, minStars });
1246
+
1247
+ if (!json && repoSkips.length > 0) {
1248
+ for (const skip of repoSkips) {
1249
+ console.log(` SKIP repo ${skip.repo.padEnd(40)} ${skip.reason}`);
1250
+ }
1251
+ console.log();
1252
+ }
1253
+
1254
+ // Cap to limit
1255
+ const capped = passed.slice(0, limit);
1256
+
1257
+ if (capped.length === 0) {
1258
+ if (json) {
1259
+ console.log(JSON.stringify({
1260
+ success: true,
1261
+ imported: 0,
1262
+ failed: 0,
1263
+ skipped: 0,
1264
+ totalEndpoints: 0,
1265
+ repoSkips,
1266
+ specSkips: [],
1267
+ results: [],
1268
+ }));
1269
+ } else {
1270
+ console.log(' No matching specs found.\n');
1271
+ }
1272
+ return;
1273
+ }
1274
+
1275
+ if (!json) {
1276
+ console.log(` Found ${passed.length} specs, importing top ${capped.length}...\n`);
1277
+ }
1278
+
1279
+ const total = capped.length;
1280
+ let imported = 0;
1281
+ let failed = 0;
1282
+ let skippedSpecs = 0;
1283
+ let totalEndpointsAdded = 0;
1284
+ const specSkips: Array<{ domain: string; reason: string }> = [];
1285
+
1286
+ const machineId = await getMachineId();
1287
+ const key = deriveSigningKey(machineId);
1288
+
1289
+ const results: Array<{
1290
+ index: number;
1291
+ status: 'ok' | 'fail' | 'skip';
1292
+ domain: string;
1293
+ title: string;
1294
+ endpointsAdded: number;
1295
+ htmlUrl: string;
1296
+ error?: string;
1297
+ }> = [];
1298
+
1299
+ for (let i = 0; i < capped.length; i++) {
1300
+ const result = capped[i];
1301
+ const idx = String(i + 1).padStart(String(total).length, ' ');
1302
+
1303
+ try {
1304
+ // Fetch spec
1305
+ const spec = await fetchGitHubSpec(result.specUrl, token);
1306
+
1307
+ // Pre-conversion content filters
1308
+ if (!hasServerUrl(spec)) {
1309
+ if (!json) {
1310
+ console.log(` [${idx}/${total}] SKIP ${result.repoFullName.padEnd(40)} no server URL`);
1311
+ }
1312
+ specSkips.push({ domain: result.repoFullName, reason: 'no server URL' });
1313
+ skippedSpecs++;
1314
+ continue;
1315
+ }
1316
+
1317
+ if (isLocalhostSpec(spec)) {
1318
+ specSkips.push({ domain: result.repoFullName, reason: 'localhost/placeholder' });
1319
+ skippedSpecs++;
1320
+ continue;
1321
+ }
1322
+
1323
+ // Normalize templated domains before conversion
1324
+ normalizeSpecServerUrls(spec);
1325
+
1326
+ // Convert
1327
+ const importResult = convertOpenAPISpec(spec, result.specUrl);
1328
+ const { domain, endpoints, meta } = importResult;
1329
+
1330
+ if (endpoints.length === 0) {
1331
+ if (!json) {
1332
+ console.log(` [${idx}/${total}] SKIP ${domain.padEnd(40)} 0 endpoints`);
1333
+ }
1334
+ specSkips.push({ domain, reason: '0 endpoints' });
1335
+ skippedSpecs++;
1336
+ continue;
1337
+ }
1338
+
1339
+ // SSRF validate
1340
+ const dnsCheck = await resolveAndValidateUrl(`https://${domain}`);
1341
+ if (!dnsCheck.safe) {
1342
+ if (!json) {
1343
+ console.log(` [${idx}/${total}] SKIP ${domain.padEnd(40)} SSRF risk`);
1344
+ }
1345
+ specSkips.push({ domain, reason: 'SSRF risk' });
1346
+ skippedSpecs++;
1347
+ continue;
1348
+ }
1349
+
1350
+ // Skip auth-required APIs when --no-auth-only is set
1351
+ if (noAuthOnly && meta.requiresAuth) {
1352
+ if (!json) {
1353
+ console.log(` [${idx}/${total}] SKIP ${domain.padEnd(40)} requires auth`);
1354
+ }
1355
+ specSkips.push({ domain, reason: 'requires auth' });
1356
+ skippedSpecs++;
1357
+ continue;
1358
+ }
1359
+
1360
+ // Read existing — skip HMAC since we only need endpoints for merge
1361
+ let existing = null;
1362
+ try {
1363
+ existing = await readSkillFile(domain, skillsDir, { verifySignature: false });
1364
+ } catch (err: any) {
1365
+ if ((err as NodeJS.ErrnoException)?.code !== 'ENOENT') {
1366
+ if (!json) console.error(` Warning: could not read existing skill file for ${domain}: ${err.message}`);
1367
+ }
1368
+ }
1369
+
1370
+ // --update check: skip if existing import from same specUrl is newer than repo push
1371
+ if (!force && update && existing?.metadata.importHistory?.some(
1372
+ (h: any) => h.specUrl === result.specUrl && h.importedAt >= result.pushedAt
1373
+ )) {
1374
+ if (!json) console.log(` [${idx}/${total}] SKIP ${domain.padEnd(40)} up to date`);
1375
+ skippedSpecs++;
1376
+ specSkips.push({ domain, reason: 'up to date' });
1377
+ continue;
1378
+ }
1379
+
1380
+ // Skip if already exists and not --force and not --update
1381
+ if (!force && !update && existing?.endpoints.length) {
1382
+ if (!json) {
1383
+ console.log(` [${idx}/${total}] SKIP ${domain.padEnd(40)} already exists (${existing.endpoints.length} endpoints)`);
1384
+ }
1385
+ specSkips.push({ domain, reason: 'already exists' });
1386
+ skippedSpecs++;
1387
+ continue;
1388
+ }
1389
+
1390
+ // Merge
1391
+ const { skillFile, diff } = mergeSkillFile(existing, endpoints, meta);
1392
+ skillFile.domain = domain;
1393
+ skillFile.baseUrl = `https://${domain}`;
1394
+
1395
+ if (dryRun) {
1396
+ if (!json) {
1397
+ console.log(` (dry run) [${idx}/${total}] OK ${domain.padEnd(40)} +${diff.added} endpoints`);
1398
+ console.log(` -> ${result.htmlUrl}`);
1399
+ }
1400
+ results.push({
1401
+ index: i + 1,
1402
+ status: 'ok',
1403
+ domain,
1404
+ title: meta.title || result.description,
1405
+ endpointsAdded: diff.added,
1406
+ htmlUrl: result.htmlUrl,
1407
+ });
1408
+ imported++;
1409
+ totalEndpointsAdded += diff.added;
1410
+ continue;
1411
+ }
1412
+
1413
+ // Sign and write
1414
+ const hasCaptured = skillFile.endpoints.some(
1415
+ ep => !ep.endpointProvenance || ep.endpointProvenance === 'captured'
1416
+ );
1417
+ const signed = signSkillFileAs(skillFile, key, hasCaptured ? 'self' : 'imported-signed');
1418
+ await writeSkillFile(signed, skillsDir);
1419
+
1420
+ if (!json) {
1421
+ console.log(` [${idx}/${total}] OK ${domain.padEnd(40)} +${diff.added} endpoints (${result.repoFullName})`);
1422
+ }
1423
+ results.push({
1424
+ index: i + 1,
1425
+ status: 'ok',
1426
+ domain,
1427
+ title: meta.title || result.description,
1428
+ endpointsAdded: diff.added,
1429
+ htmlUrl: result.htmlUrl,
1430
+ });
1431
+ imported++;
1432
+ totalEndpointsAdded += diff.added;
1433
+ } catch (err: any) {
1434
+ if (!json) {
1435
+ console.log(` [${idx}/${total}] FAIL ${result.repoFullName.padEnd(40)} ${err.message.slice(0, 60)}`);
1436
+ }
1437
+ results.push({
1438
+ index: i + 1,
1439
+ status: 'fail',
1440
+ domain: result.repoFullName,
1441
+ title: result.description,
1442
+ endpointsAdded: 0,
1443
+ htmlUrl: result.htmlUrl,
1444
+ error: err.message,
1445
+ });
1446
+ failed++;
1447
+ }
1448
+
1449
+ // Polite delay between spec fetches
1450
+ if (i < capped.length - 1) {
1451
+ await new Promise(r => setTimeout(r, 100));
1452
+ }
1453
+ }
1454
+
1455
+ if (json) {
1456
+ console.log(JSON.stringify({
1457
+ success: true,
1458
+ imported,
1459
+ failed,
1460
+ skipped: skippedSpecs,
1461
+ totalEndpoints: totalEndpointsAdded,
1462
+ repoSkips,
1463
+ specSkips,
1464
+ results,
1465
+ ...(rateLimitRef.value ? {
1466
+ githubApiUsage: {
1467
+ used: rateLimitRef.value.limit - rateLimitRef.value.remaining,
1468
+ limit: rateLimitRef.value.limit,
1469
+ resetAt: rateLimitRef.value.resetAt.toISOString(),
1470
+ },
1471
+ } : {}),
1472
+ }));
1473
+ } else {
1474
+ console.log(`\n Imported ${imported} specs: ${totalEndpointsAdded.toLocaleString()} endpoints across ${imported} domains`);
1475
+ if (repoSkips.length) {
1476
+ const reasons = new Map<string, number>();
1477
+ for (const s of repoSkips) reasons.set(s.reason, (reasons.get(s.reason) ?? 0) + 1);
1478
+ const parts = [...reasons.entries()].map(([r, n]) => `${n} ${r}`);
1479
+ console.log(` Repo skips: ${repoSkips.length} (${parts.join(', ')})`);
1480
+ }
1481
+ if (specSkips.length) {
1482
+ const reasons = new Map<string, number>();
1483
+ for (const s of specSkips) reasons.set(s.reason, (reasons.get(s.reason) ?? 0) + 1);
1484
+ const parts = [...reasons.entries()].map(([r, n]) => `${n} ${r}`);
1485
+ console.log(` Spec skips: ${specSkips.length} (${parts.join(', ')})`);
1486
+ }
1487
+ if (rateLimitRef.value) {
1488
+ const used = rateLimitRef.value.limit - rateLimitRef.value.remaining;
1489
+ let rateLine = ` GitHub API: ${used}/${rateLimitRef.value.limit} requests used`;
1490
+ if (rateLimitRef.value.remaining < 100) {
1491
+ rateLine += ` (resets ${rateLimitRef.value.resetAt.toLocaleTimeString()})`;
1492
+ }
1493
+ console.log(rateLine);
1494
+ }
1495
+ if (dryRun) {
1496
+ console.log(' (dry run — no changes written)');
1497
+ }
1498
+ console.log();
1499
+ }
1500
+ }
1501
+
1134
1502
  async function handleRefresh(positional: string[], flags: Record<string, string | boolean>): Promise<void> {
1135
1503
  const domain = positional[0];
1136
1504
  if (!domain) {