@apitap/core 1.5.3 → 1.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +28 -8
- package/dist/auth/handoff.js +1 -1
- package/dist/auth/handoff.js.map +1 -1
- package/dist/capture/cdp-attach.d.ts +60 -0
- package/dist/capture/cdp-attach.js +422 -0
- package/dist/capture/cdp-attach.js.map +1 -0
- package/dist/capture/filter.js +6 -0
- package/dist/capture/filter.js.map +1 -1
- package/dist/capture/parameterize.d.ts +7 -6
- package/dist/capture/parameterize.js +204 -12
- package/dist/capture/parameterize.js.map +1 -1
- package/dist/capture/session.js +20 -10
- package/dist/capture/session.js.map +1 -1
- package/dist/cli.js +387 -20
- package/dist/cli.js.map +1 -1
- package/dist/discovery/openapi.js +23 -50
- package/dist/discovery/openapi.js.map +1 -1
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -1
- package/dist/mcp.js +12 -0
- package/dist/mcp.js.map +1 -1
- package/dist/native-host.js +5 -0
- package/dist/native-host.js.map +1 -1
- package/dist/plugin.js +10 -3
- package/dist/plugin.js.map +1 -1
- package/dist/replay/engine.d.ts +13 -0
- package/dist/replay/engine.js +20 -0
- package/dist/replay/engine.js.map +1 -1
- package/dist/skill/apis-guru.d.ts +35 -0
- package/dist/skill/apis-guru.js +128 -0
- package/dist/skill/apis-guru.js.map +1 -0
- package/dist/skill/generator.d.ts +7 -1
- package/dist/skill/generator.js +35 -3
- package/dist/skill/generator.js.map +1 -1
- package/dist/skill/merge.d.ts +29 -0
- package/dist/skill/merge.js +252 -0
- package/dist/skill/merge.js.map +1 -0
- package/dist/skill/openapi-converter.d.ts +31 -0
- package/dist/skill/openapi-converter.js +383 -0
- package/dist/skill/openapi-converter.js.map +1 -0
- package/dist/types.d.ts +41 -0
- package/package.json +1 -1
- package/src/auth/handoff.ts +1 -1
- package/src/capture/cdp-attach.ts +501 -0
- package/src/capture/filter.ts +5 -0
- package/src/capture/parameterize.ts +207 -11
- package/src/capture/session.ts +20 -10
- package/src/cli.ts +420 -18
- package/src/discovery/openapi.ts +25 -56
- package/src/index.ts +1 -0
- package/src/mcp.ts +12 -0
- package/src/native-host.ts +7 -0
- package/src/plugin.ts +10 -3
- package/src/replay/engine.ts +19 -0
- package/src/skill/apis-guru.ts +163 -0
- package/src/skill/generator.ts +38 -3
- package/src/skill/merge.ts +281 -0
- package/src/skill/openapi-converter.ts +426 -0
- package/src/types.ts +42 -1
package/src/cli.ts
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
// src/cli.ts
|
|
3
3
|
import { capture } from './capture/monitor.js';
|
|
4
4
|
import { writeSkillFile, readSkillFile, listSkillFiles } from './skill/store.js';
|
|
5
|
-
import { replayEndpoint } from './replay/engine.js';
|
|
5
|
+
import { replayEndpoint, getConfidenceHint } from './replay/engine.js';
|
|
6
6
|
import { AuthManager, getMachineId } from './auth/manager.js';
|
|
7
7
|
import { deriveSigningKey } from './auth/crypto.js';
|
|
8
8
|
import { signSkillFile } from './skill/signing.js';
|
|
@@ -26,6 +26,10 @@ import { readFileSync } from 'node:fs';
|
|
|
26
26
|
import { stat, unlink } from 'node:fs/promises';
|
|
27
27
|
import { fileURLToPath } from 'node:url';
|
|
28
28
|
import { createMcpServer } from './mcp.js';
|
|
29
|
+
import { attach, parseDomainPatterns } from './capture/cdp-attach.js';
|
|
30
|
+
import { isOpenAPISpec, convertOpenAPISpec } from './skill/openapi-converter.js';
|
|
31
|
+
import { mergeSkillFile } from './skill/merge.js';
|
|
32
|
+
import { fetchApisGuruList, filterEntries, fetchSpec } from './skill/apis-guru.js';
|
|
29
33
|
|
|
30
34
|
const __dirname = fileURLToPath(new URL('.', import.meta.url));
|
|
31
35
|
const pkg = JSON.parse(readFileSync(join(__dirname, '..', 'package.json'), 'utf-8'));
|
|
@@ -66,6 +70,8 @@ function printUsage(): void {
|
|
|
66
70
|
|
|
67
71
|
Usage:
|
|
68
72
|
apitap capture <url> Capture API traffic from a website
|
|
73
|
+
apitap attach [--port 9222] [--domain *.github.com]
|
|
74
|
+
Attach to running Chrome and capture API traffic
|
|
69
75
|
apitap discover <url> Detect APIs without a browser (fast recon)
|
|
70
76
|
apitap inspect <url> Discover APIs without saving (X-ray vision)
|
|
71
77
|
apitap search <query> Search skill files for a domain or endpoint
|
|
@@ -73,7 +79,9 @@ function printUsage(): void {
|
|
|
73
79
|
apitap show <domain> Show endpoints for a domain
|
|
74
80
|
apitap replay <domain> <endpoint-id> [key=value...]
|
|
75
81
|
Replay an API endpoint
|
|
76
|
-
apitap import <file>
|
|
82
|
+
apitap import <file|url> Import a skill file or OpenAPI spec
|
|
83
|
+
apitap import --from apis-guru
|
|
84
|
+
Bulk-import from APIs.guru directory
|
|
77
85
|
apitap refresh <domain> Refresh auth tokens via browser
|
|
78
86
|
apitap auth [domain] View or manage stored auth
|
|
79
87
|
apitap mcp Run the full ApiTap MCP server over stdio
|
|
@@ -125,6 +133,11 @@ function printUsage(): void {
|
|
|
125
133
|
|
|
126
134
|
Import options:
|
|
127
135
|
--yes Skip confirmation prompt
|
|
136
|
+
--dry-run Show what would change without writing
|
|
137
|
+
--json Output machine-readable JSON
|
|
138
|
+
--from apis-guru Bulk-import from APIs.guru directory
|
|
139
|
+
--search <term> Filter APIs.guru entries by name/title
|
|
140
|
+
--limit <n> Max APIs to import (default: 100)
|
|
128
141
|
|
|
129
142
|
Serve options:
|
|
130
143
|
--json Output tool list as JSON on stderr
|
|
@@ -452,6 +465,10 @@ async function handleReplay(positional: string[], flags: Record<string, string |
|
|
|
452
465
|
...(result.contractWarnings?.length ? { contractWarnings: result.contractWarnings } : {}),
|
|
453
466
|
}, null, 2));
|
|
454
467
|
} else {
|
|
468
|
+
const hint = endpoint ? getConfidenceHint(endpoint.confidence) : null;
|
|
469
|
+
if (hint) {
|
|
470
|
+
console.error(` Note: ${hint}`);
|
|
471
|
+
}
|
|
455
472
|
console.log(`\n Status: ${result.status}\n`);
|
|
456
473
|
console.log(JSON.stringify(result.data, null, 2));
|
|
457
474
|
console.log();
|
|
@@ -459,35 +476,134 @@ async function handleReplay(positional: string[], flags: Record<string, string |
|
|
|
459
476
|
}
|
|
460
477
|
|
|
461
478
|
async function handleImport(positional: string[], flags: Record<string, string | boolean>): Promise<void> {
|
|
462
|
-
|
|
463
|
-
if (
|
|
464
|
-
|
|
479
|
+
// --from apis-guru: bulk import mode
|
|
480
|
+
if (flags['from'] === 'apis-guru') {
|
|
481
|
+
await handleApisGuruImport(flags);
|
|
482
|
+
return;
|
|
483
|
+
}
|
|
484
|
+
|
|
485
|
+
const source = positional[0];
|
|
486
|
+
if (!source) {
|
|
487
|
+
console.error('Error: File path or URL required. Usage: apitap import <file|url>');
|
|
465
488
|
process.exit(1);
|
|
466
489
|
}
|
|
467
490
|
|
|
468
491
|
const json = flags.json === true;
|
|
469
492
|
|
|
493
|
+
// Reject YAML files with helpful message
|
|
494
|
+
if (/\.ya?ml$/i.test(source)) {
|
|
495
|
+
const msg = 'YAML specs not yet supported. Convert to JSON first: npx swagger-cli bundle spec.yaml -o spec.json';
|
|
496
|
+
if (json) {
|
|
497
|
+
console.log(JSON.stringify({ success: false, reason: msg }));
|
|
498
|
+
} else {
|
|
499
|
+
console.error(`Error: ${msg}`);
|
|
500
|
+
}
|
|
501
|
+
process.exit(1);
|
|
502
|
+
}
|
|
503
|
+
|
|
504
|
+
// Load content — from URL or file
|
|
505
|
+
let rawText: string;
|
|
506
|
+
let sourceUrl: string = source;
|
|
507
|
+
const isUrl = source.startsWith('http://') || source.startsWith('https://');
|
|
508
|
+
|
|
509
|
+
if (isUrl) {
|
|
510
|
+
try {
|
|
511
|
+
const ssrfCheck = await resolveAndValidateUrl(source);
|
|
512
|
+
if (!ssrfCheck.safe) {
|
|
513
|
+
throw new Error(`SSRF check failed: ${ssrfCheck.reason}`);
|
|
514
|
+
}
|
|
515
|
+
const response = await fetch(source, { signal: AbortSignal.timeout(30_000) });
|
|
516
|
+
if (!response.ok) {
|
|
517
|
+
throw new Error(`HTTP ${response.status} ${response.statusText}`);
|
|
518
|
+
}
|
|
519
|
+
rawText = await response.text();
|
|
520
|
+
} catch (err: any) {
|
|
521
|
+
const msg = `Failed to fetch ${source}: ${err.message}`;
|
|
522
|
+
if (json) {
|
|
523
|
+
console.log(JSON.stringify({ success: false, reason: msg }));
|
|
524
|
+
} else {
|
|
525
|
+
console.error(`Error: ${msg}`);
|
|
526
|
+
}
|
|
527
|
+
process.exit(1);
|
|
528
|
+
}
|
|
529
|
+
} else {
|
|
530
|
+
try {
|
|
531
|
+
const { readFile } = await import('node:fs/promises');
|
|
532
|
+
rawText = await readFile(source, 'utf-8');
|
|
533
|
+
sourceUrl = `file://${resolve(source)}`;
|
|
534
|
+
} catch (err: any) {
|
|
535
|
+
const msg = `Failed to read ${source}: ${err.message}`;
|
|
536
|
+
if (json) {
|
|
537
|
+
console.log(JSON.stringify({ success: false, reason: msg }));
|
|
538
|
+
} else {
|
|
539
|
+
console.error(`Error: ${msg}`);
|
|
540
|
+
}
|
|
541
|
+
process.exit(1);
|
|
542
|
+
}
|
|
543
|
+
}
|
|
544
|
+
|
|
545
|
+
// Parse JSON
|
|
546
|
+
let parsed: any;
|
|
547
|
+
try {
|
|
548
|
+
parsed = JSON.parse(rawText);
|
|
549
|
+
} catch {
|
|
550
|
+
const msg = `Invalid JSON in ${source}`;
|
|
551
|
+
if (json) {
|
|
552
|
+
console.log(JSON.stringify({ success: false, reason: msg }));
|
|
553
|
+
} else {
|
|
554
|
+
console.error(`Error: ${msg}`);
|
|
555
|
+
}
|
|
556
|
+
process.exit(1);
|
|
557
|
+
}
|
|
558
|
+
|
|
559
|
+
// Route: OpenAPI spec vs SkillFile
|
|
560
|
+
if (isOpenAPISpec(parsed)) {
|
|
561
|
+
await handleOpenAPIImport(parsed, sourceUrl, flags);
|
|
562
|
+
return;
|
|
563
|
+
}
|
|
564
|
+
|
|
565
|
+
// --- Existing SkillFile import flow (unchanged) ---
|
|
566
|
+
if (isUrl) {
|
|
567
|
+
// SkillFile imports from URLs: write to temp file first
|
|
568
|
+
const { writeFile: writeTmp } = await import('node:fs/promises');
|
|
569
|
+
const { tmpdir } = await import('node:os');
|
|
570
|
+
const tmpPath = join(tmpdir(), `apitap-import-${Date.now()}.json`);
|
|
571
|
+
await writeTmp(tmpPath, rawText);
|
|
572
|
+
try {
|
|
573
|
+
await handleSkillFileImport(tmpPath, json);
|
|
574
|
+
} finally {
|
|
575
|
+
const { unlink: unlinkTmp } = await import('node:fs/promises');
|
|
576
|
+
await unlinkTmp(tmpPath).catch(() => {});
|
|
577
|
+
}
|
|
578
|
+
} else {
|
|
579
|
+
await handleSkillFileImport(source, json);
|
|
580
|
+
}
|
|
581
|
+
}
|
|
582
|
+
|
|
583
|
+
async function handleSkillFileImport(filePath: string, json: boolean): Promise<void> {
|
|
470
584
|
// Get local key for signature verification
|
|
471
585
|
const machineId = await getMachineId();
|
|
472
586
|
const key = deriveSigningKey(machineId);
|
|
473
587
|
|
|
474
588
|
// DNS-resolving SSRF check before importing (prevents DNS rebinding attacks)
|
|
589
|
+
let raw: any;
|
|
475
590
|
try {
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
591
|
+
raw = JSON.parse(await import('node:fs/promises').then(fs => fs.readFile(filePath, 'utf-8')));
|
|
592
|
+
} catch {
|
|
593
|
+
// Parse errors will be caught by importSkillFile below
|
|
594
|
+
}
|
|
595
|
+
|
|
596
|
+
if (raw?.baseUrl) {
|
|
597
|
+
const dnsCheck = await resolveAndValidateUrl(raw.baseUrl);
|
|
598
|
+
if (!dnsCheck.safe) {
|
|
599
|
+
const msg = `DNS rebinding risk: ${dnsCheck.reason}`;
|
|
600
|
+
if (json) {
|
|
601
|
+
console.log(JSON.stringify({ success: false, reason: msg }));
|
|
602
|
+
} else {
|
|
603
|
+
console.error(`Error: ${msg}`);
|
|
487
604
|
}
|
|
605
|
+
process.exit(1);
|
|
488
606
|
}
|
|
489
|
-
} catch {
|
|
490
|
-
// Parse errors will be caught by importSkillFile
|
|
491
607
|
}
|
|
492
608
|
|
|
493
609
|
const result = await importSkillFile(filePath, undefined, key);
|
|
@@ -508,6 +624,275 @@ async function handleImport(positional: string[], flags: Record<string, string |
|
|
|
508
624
|
}
|
|
509
625
|
}
|
|
510
626
|
|
|
627
|
+
async function handleOpenAPIImport(
|
|
628
|
+
spec: Record<string, any>,
|
|
629
|
+
specUrl: string,
|
|
630
|
+
flags: Record<string, string | boolean>,
|
|
631
|
+
): Promise<void> {
|
|
632
|
+
const json = flags.json === true;
|
|
633
|
+
const dryRun = flags['dry-run'] === true;
|
|
634
|
+
const skillsDir = SKILLS_DIR || join(APITAP_DIR, 'skills');
|
|
635
|
+
|
|
636
|
+
// Convert OpenAPI spec to endpoints
|
|
637
|
+
let importResult;
|
|
638
|
+
try {
|
|
639
|
+
importResult = convertOpenAPISpec(spec, specUrl);
|
|
640
|
+
} catch (err: any) {
|
|
641
|
+
const msg = `Failed to convert OpenAPI spec: ${err.message}`;
|
|
642
|
+
if (json) {
|
|
643
|
+
console.log(JSON.stringify({ success: false, reason: msg }));
|
|
644
|
+
} else {
|
|
645
|
+
console.error(`Error: ${msg}`);
|
|
646
|
+
}
|
|
647
|
+
process.exit(1);
|
|
648
|
+
}
|
|
649
|
+
|
|
650
|
+
const { domain, endpoints, meta } = importResult;
|
|
651
|
+
const specVersion = spec.openapi || spec.swagger || 'unknown';
|
|
652
|
+
|
|
653
|
+
if (!json) {
|
|
654
|
+
console.log(`\n Importing ${domain} from OpenAPI ${specVersion} spec...\n`);
|
|
655
|
+
}
|
|
656
|
+
|
|
657
|
+
// SSRF validate the domain
|
|
658
|
+
const dnsCheck = await resolveAndValidateUrl(`https://${domain}`);
|
|
659
|
+
if (!dnsCheck.safe) {
|
|
660
|
+
const msg = `DNS rebinding risk for ${domain}: ${dnsCheck.reason}`;
|
|
661
|
+
if (json) {
|
|
662
|
+
console.log(JSON.stringify({ success: false, reason: msg }));
|
|
663
|
+
} else {
|
|
664
|
+
console.error(`Error: ${msg}`);
|
|
665
|
+
}
|
|
666
|
+
process.exit(1);
|
|
667
|
+
}
|
|
668
|
+
|
|
669
|
+
// Read existing skill file (if any)
|
|
670
|
+
let existing = null;
|
|
671
|
+
try {
|
|
672
|
+
existing = await readSkillFile(domain, skillsDir, {
|
|
673
|
+
verifySignature: true,
|
|
674
|
+
trustUnsigned: true,
|
|
675
|
+
});
|
|
676
|
+
} catch (err: any) {
|
|
677
|
+
if ((err as NodeJS.ErrnoException)?.code !== 'ENOENT') {
|
|
678
|
+
if (!json) console.error(` Warning: could not read existing skill file for ${domain}: ${err.message}`);
|
|
679
|
+
}
|
|
680
|
+
}
|
|
681
|
+
|
|
682
|
+
// Merge
|
|
683
|
+
const { skillFile, diff } = mergeSkillFile(existing, endpoints, meta);
|
|
684
|
+
|
|
685
|
+
// Ensure domain and baseUrl reflect the API, not the spec source URL
|
|
686
|
+
skillFile.domain = domain;
|
|
687
|
+
skillFile.baseUrl = `https://${domain}`;
|
|
688
|
+
|
|
689
|
+
if (dryRun) {
|
|
690
|
+
if (json) {
|
|
691
|
+
console.log(JSON.stringify({
|
|
692
|
+
success: true,
|
|
693
|
+
dryRun: true,
|
|
694
|
+
domain,
|
|
695
|
+
diff,
|
|
696
|
+
totalEndpoints: skillFile.endpoints.length,
|
|
697
|
+
}));
|
|
698
|
+
} else {
|
|
699
|
+
printOpenAPIDiff(diff, skillFile, skillsDir);
|
|
700
|
+
console.log(' (dry run — no changes written)\n');
|
|
701
|
+
}
|
|
702
|
+
return;
|
|
703
|
+
}
|
|
704
|
+
|
|
705
|
+
// Sign and write
|
|
706
|
+
const machineId = await getMachineId();
|
|
707
|
+
const key = deriveSigningKey(machineId);
|
|
708
|
+
const signed = signSkillFile(skillFile, key);
|
|
709
|
+
const filePath = await writeSkillFile(signed, skillsDir);
|
|
710
|
+
|
|
711
|
+
if (json) {
|
|
712
|
+
console.log(JSON.stringify({
|
|
713
|
+
success: true,
|
|
714
|
+
domain,
|
|
715
|
+
skillFile: filePath,
|
|
716
|
+
diff,
|
|
717
|
+
totalEndpoints: signed.endpoints.length,
|
|
718
|
+
}));
|
|
719
|
+
} else {
|
|
720
|
+
printOpenAPIDiff(diff, signed, filePath);
|
|
721
|
+
}
|
|
722
|
+
}
|
|
723
|
+
|
|
724
|
+
function printOpenAPIDiff(
|
|
725
|
+
diff: { preserved: number; added: number; enriched: number; skipped: number },
|
|
726
|
+
skillFile: { endpoints: { length: number } },
|
|
727
|
+
pathOrDir: string,
|
|
728
|
+
): void {
|
|
729
|
+
console.log(` ✓ ${diff.preserved} existing captured endpoints preserved`);
|
|
730
|
+
console.log(` + ${diff.added} new endpoints added from OpenAPI spec`);
|
|
731
|
+
console.log(` ~ ${diff.enriched} endpoints enriched with spec metadata`);
|
|
732
|
+
console.log(` · ${diff.skipped} skipped (already imported)`);
|
|
733
|
+
console.log();
|
|
734
|
+
console.log(` Skill file: ${pathOrDir} (${skillFile.endpoints.length} endpoints)\n`);
|
|
735
|
+
}
|
|
736
|
+
|
|
737
|
+
async function handleApisGuruImport(flags: Record<string, string | boolean>): Promise<void> {
|
|
738
|
+
const json = flags.json === true;
|
|
739
|
+
const dryRun = flags['dry-run'] === true;
|
|
740
|
+
const limit = typeof flags.limit === 'string' ? parseInt(flags.limit, 10) : 100;
|
|
741
|
+
const search = typeof flags.search === 'string' ? flags.search : undefined;
|
|
742
|
+
const skillsDir = SKILLS_DIR || join(APITAP_DIR, 'skills');
|
|
743
|
+
|
|
744
|
+
if (!json) {
|
|
745
|
+
console.log(`\n Importing from APIs.guru (limit: ${limit})...\n`);
|
|
746
|
+
}
|
|
747
|
+
|
|
748
|
+
// Fetch and filter
|
|
749
|
+
let entries;
|
|
750
|
+
try {
|
|
751
|
+
const allEntries = await fetchApisGuruList();
|
|
752
|
+
entries = filterEntries(allEntries, { search, limit, preferOpenapi3: true });
|
|
753
|
+
} catch (err: any) {
|
|
754
|
+
const msg = `Failed to fetch APIs.guru list: ${err.message}`;
|
|
755
|
+
if (json) {
|
|
756
|
+
console.log(JSON.stringify({ success: false, reason: msg }));
|
|
757
|
+
} else {
|
|
758
|
+
console.error(`Error: ${msg}`);
|
|
759
|
+
}
|
|
760
|
+
process.exit(1);
|
|
761
|
+
}
|
|
762
|
+
|
|
763
|
+
if (entries.length === 0) {
|
|
764
|
+
if (json) {
|
|
765
|
+
console.log(JSON.stringify({ success: true, imported: 0, failed: 0, skipped: 0, totalEndpoints: 0 }));
|
|
766
|
+
} else {
|
|
767
|
+
console.log(' No matching APIs found.\n');
|
|
768
|
+
}
|
|
769
|
+
return;
|
|
770
|
+
}
|
|
771
|
+
|
|
772
|
+
const total = entries.length;
|
|
773
|
+
let imported = 0;
|
|
774
|
+
let failed = 0;
|
|
775
|
+
let skippedApis = 0;
|
|
776
|
+
let totalEndpointsAdded = 0;
|
|
777
|
+
|
|
778
|
+
const machineId = await getMachineId();
|
|
779
|
+
const key = deriveSigningKey(machineId);
|
|
780
|
+
|
|
781
|
+
const results: Array<{
|
|
782
|
+
index: number;
|
|
783
|
+
status: 'ok' | 'fail' | 'skip';
|
|
784
|
+
domain: string;
|
|
785
|
+
title: string;
|
|
786
|
+
endpointsAdded: number;
|
|
787
|
+
error?: string;
|
|
788
|
+
}> = [];
|
|
789
|
+
|
|
790
|
+
for (let i = 0; i < entries.length; i++) {
|
|
791
|
+
const entry = entries[i];
|
|
792
|
+
const idx = String(i + 1).padStart(String(total).length, ' ');
|
|
793
|
+
|
|
794
|
+
try {
|
|
795
|
+
// Fetch spec
|
|
796
|
+
const spec = await fetchSpec(entry.specUrl);
|
|
797
|
+
|
|
798
|
+
// Convert
|
|
799
|
+
const importResult = convertOpenAPISpec(spec, entry.specUrl);
|
|
800
|
+
const { domain, endpoints, meta } = importResult;
|
|
801
|
+
|
|
802
|
+
if (endpoints.length === 0) {
|
|
803
|
+
if (!json) {
|
|
804
|
+
console.log(` [${idx}/${total}] SKIP ${domain.padEnd(24)} 0 endpoints (${entry.title})`);
|
|
805
|
+
}
|
|
806
|
+
results.push({ index: i + 1, status: 'skip', domain, title: entry.title, endpointsAdded: 0 });
|
|
807
|
+
skippedApis++;
|
|
808
|
+
continue;
|
|
809
|
+
}
|
|
810
|
+
|
|
811
|
+
// SSRF validate
|
|
812
|
+
const dnsCheck = await resolveAndValidateUrl(`https://${domain}`);
|
|
813
|
+
if (!dnsCheck.safe) {
|
|
814
|
+
if (!json) {
|
|
815
|
+
console.log(` [${idx}/${total}] SKIP ${domain.padEnd(24)} SSRF risk (${entry.title})`);
|
|
816
|
+
}
|
|
817
|
+
results.push({ index: i + 1, status: 'skip', domain, title: entry.title, endpointsAdded: 0 });
|
|
818
|
+
skippedApis++;
|
|
819
|
+
continue;
|
|
820
|
+
}
|
|
821
|
+
|
|
822
|
+
// Read existing
|
|
823
|
+
let existing = null;
|
|
824
|
+
try {
|
|
825
|
+
existing = await readSkillFile(domain, skillsDir, {
|
|
826
|
+
verifySignature: true,
|
|
827
|
+
trustUnsigned: true,
|
|
828
|
+
});
|
|
829
|
+
} catch (err: any) {
|
|
830
|
+
if ((err as NodeJS.ErrnoException)?.code !== 'ENOENT') {
|
|
831
|
+
if (!json) console.error(` Warning: could not read existing skill file for ${domain}: ${err.message}`);
|
|
832
|
+
}
|
|
833
|
+
}
|
|
834
|
+
|
|
835
|
+
// Merge
|
|
836
|
+
const { skillFile, diff } = mergeSkillFile(existing, endpoints, meta);
|
|
837
|
+
|
|
838
|
+
// Ensure domain and baseUrl reflect the API, not the spec source URL
|
|
839
|
+
skillFile.domain = domain;
|
|
840
|
+
skillFile.baseUrl = `https://${domain}`;
|
|
841
|
+
|
|
842
|
+
if (!dryRun) {
|
|
843
|
+
// Sign and write
|
|
844
|
+
const signed = signSkillFile(skillFile, key);
|
|
845
|
+
await writeSkillFile(signed, skillsDir);
|
|
846
|
+
}
|
|
847
|
+
|
|
848
|
+
if (!json) {
|
|
849
|
+
console.log(` [${idx}/${total}] OK ${domain.padEnd(24)} +${diff.added} endpoints (${entry.title})`);
|
|
850
|
+
}
|
|
851
|
+
results.push({ index: i + 1, status: 'ok', domain, title: entry.title, endpointsAdded: diff.added });
|
|
852
|
+
imported++;
|
|
853
|
+
totalEndpointsAdded += diff.added;
|
|
854
|
+
} catch (err: any) {
|
|
855
|
+
if (!json) {
|
|
856
|
+
console.log(` [${idx}/${total}] FAIL ${entry.providerName.padEnd(24)} ${err.message}`);
|
|
857
|
+
}
|
|
858
|
+
results.push({
|
|
859
|
+
index: i + 1,
|
|
860
|
+
status: 'fail',
|
|
861
|
+
domain: entry.providerName,
|
|
862
|
+
title: entry.title,
|
|
863
|
+
endpointsAdded: 0,
|
|
864
|
+
error: err.message,
|
|
865
|
+
});
|
|
866
|
+
failed++;
|
|
867
|
+
}
|
|
868
|
+
|
|
869
|
+
// Small delay between requests to be polite to APIs.guru
|
|
870
|
+
if (i < entries.length - 1) {
|
|
871
|
+
await new Promise(r => setTimeout(r, 100));
|
|
872
|
+
}
|
|
873
|
+
}
|
|
874
|
+
|
|
875
|
+
if (json) {
|
|
876
|
+
console.log(JSON.stringify({
|
|
877
|
+
success: true,
|
|
878
|
+
dryRun,
|
|
879
|
+
imported,
|
|
880
|
+
failed,
|
|
881
|
+
skipped: skippedApis,
|
|
882
|
+
totalEndpoints: totalEndpointsAdded,
|
|
883
|
+
results,
|
|
884
|
+
}));
|
|
885
|
+
} else {
|
|
886
|
+
console.log();
|
|
887
|
+
console.log(` Done: ${imported} imported, ${failed} failed, ${skippedApis} skipped`);
|
|
888
|
+
console.log(` ${totalEndpointsAdded.toLocaleString()} endpoints added across ${imported} APIs`);
|
|
889
|
+
if (dryRun) {
|
|
890
|
+
console.log(' (dry run — no changes written)');
|
|
891
|
+
}
|
|
892
|
+
console.log();
|
|
893
|
+
}
|
|
894
|
+
}
|
|
895
|
+
|
|
511
896
|
async function handleRefresh(positional: string[], flags: Record<string, string | boolean>): Promise<void> {
|
|
512
897
|
const domain = positional[0];
|
|
513
898
|
if (!domain) {
|
|
@@ -1233,6 +1618,20 @@ async function handleExtension(positional: string[], flags: Record<string, strin
|
|
|
1233
1618
|
process.exit(1);
|
|
1234
1619
|
}
|
|
1235
1620
|
|
|
1621
|
+
async function handleAttach(_positional: string[], flags: Record<string, string | boolean>): Promise<void> {
|
|
1622
|
+
const port = typeof flags.port === 'string' ? parseInt(flags.port, 10) : 9222;
|
|
1623
|
+
const domainPatterns = parseDomainPatterns(
|
|
1624
|
+
typeof flags.domain === 'string' ? flags.domain : undefined,
|
|
1625
|
+
);
|
|
1626
|
+
const json = flags.json === true;
|
|
1627
|
+
|
|
1628
|
+
const result = await attach({ port, domainPatterns, json });
|
|
1629
|
+
|
|
1630
|
+
if (json) {
|
|
1631
|
+
console.log(JSON.stringify(result, null, 2));
|
|
1632
|
+
}
|
|
1633
|
+
}
|
|
1634
|
+
|
|
1236
1635
|
async function main(): Promise<void> {
|
|
1237
1636
|
const { command, positional, flags } = parseArgs(process.argv.slice(2));
|
|
1238
1637
|
|
|
@@ -1300,6 +1699,9 @@ async function main(): Promise<void> {
|
|
|
1300
1699
|
case 'extension':
|
|
1301
1700
|
await handleExtension(positional, flags);
|
|
1302
1701
|
break;
|
|
1702
|
+
case 'attach':
|
|
1703
|
+
await handleAttach(positional, flags);
|
|
1704
|
+
break;
|
|
1303
1705
|
default:
|
|
1304
1706
|
printUsage();
|
|
1305
1707
|
}
|
package/src/discovery/openapi.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
// src/discovery/openapi.ts
|
|
2
2
|
import type { SkillEndpoint, SkillFile, DiscoveredSpec } from '../types.js';
|
|
3
3
|
import { safeFetch } from './fetch.js';
|
|
4
|
+
import { convertOpenAPISpec } from '../skill/openapi-converter.js';
|
|
4
5
|
|
|
5
6
|
/** Paths to check for API specs, in priority order */
|
|
6
7
|
const SPEC_PATHS = [
|
|
@@ -137,7 +138,7 @@ export async function parseSpecToSkillFile(
|
|
|
137
138
|
|
|
138
139
|
if (!spec.paths) return null;
|
|
139
140
|
|
|
140
|
-
// Determine API base URL
|
|
141
|
+
// Determine API base URL (discovery uses the passed-in baseUrl as starting point)
|
|
141
142
|
let apiBase = baseUrl;
|
|
142
143
|
if (spec.servers?.[0]?.url) {
|
|
143
144
|
const serverUrl = spec.servers[0].url;
|
|
@@ -147,55 +148,29 @@ export async function parseSpecToSkillFile(
|
|
|
147
148
|
apiBase = `${scheme}://${spec.host}${spec.basePath || ''}`;
|
|
148
149
|
}
|
|
149
150
|
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
// Generate endpoint ID
|
|
174
|
-
const id = op.operationId
|
|
175
|
-
? method.toLowerCase() + '-' + op.operationId.replace(/[^a-z0-9]/gi, '-').toLowerCase()
|
|
176
|
-
: generateId(method, paramPath);
|
|
177
|
-
|
|
178
|
-
endpoints.push({
|
|
179
|
-
id,
|
|
180
|
-
method: method.toUpperCase(),
|
|
181
|
-
path: paramPath,
|
|
182
|
-
queryParams,
|
|
183
|
-
headers: {},
|
|
184
|
-
responseShape: { type: 'unknown' },
|
|
185
|
-
examples: {
|
|
186
|
-
request: { url: `${apiBase}${path}`, headers: {} },
|
|
187
|
-
responsePreview: null,
|
|
188
|
-
},
|
|
189
|
-
replayability: {
|
|
190
|
-
tier: 'unknown',
|
|
191
|
-
verified: false,
|
|
192
|
-
signals: ['discovered-from-spec'],
|
|
193
|
-
},
|
|
194
|
-
});
|
|
195
|
-
}
|
|
196
|
-
}
|
|
197
|
-
|
|
198
|
-
if (endpoints.length === 0) return null;
|
|
151
|
+
// Delegate endpoint extraction to the shared converter
|
|
152
|
+
const { endpoints: convertedEndpoints } = convertOpenAPISpec(spec, specUrl);
|
|
153
|
+
|
|
154
|
+
if (convertedEndpoints.length === 0) return null;
|
|
155
|
+
|
|
156
|
+
// Post-process: add replayability signal and fix example URLs to use apiBase
|
|
157
|
+
const endpoints: SkillEndpoint[] = convertedEndpoints.map(ep => {
|
|
158
|
+
// Rebuild example URL using apiBase (converter uses https://<domain> but discovery
|
|
159
|
+
// should use the baseUrl passed in, which may be http:// or a different host)
|
|
160
|
+
const exampleUrl = ep.examples.request.url.replace(/^https?:\/\/[^/]+/, apiBase.replace(/\/$/, ''));
|
|
161
|
+
return {
|
|
162
|
+
...ep,
|
|
163
|
+
examples: {
|
|
164
|
+
...ep.examples,
|
|
165
|
+
request: { ...ep.examples.request, url: exampleUrl },
|
|
166
|
+
},
|
|
167
|
+
replayability: {
|
|
168
|
+
tier: 'unknown',
|
|
169
|
+
verified: false,
|
|
170
|
+
signals: ['discovered-from-spec'],
|
|
171
|
+
},
|
|
172
|
+
};
|
|
173
|
+
});
|
|
199
174
|
|
|
200
175
|
return {
|
|
201
176
|
version: '1.2',
|
|
@@ -212,12 +187,6 @@ export async function parseSpecToSkillFile(
|
|
|
212
187
|
};
|
|
213
188
|
}
|
|
214
189
|
|
|
215
|
-
function generateId(method: string, path: string): string {
|
|
216
|
-
const segments = path.split('/').filter(s => s !== '' && !s.startsWith(':'));
|
|
217
|
-
const slug = segments.join('-').replace(/[^a-z0-9-]/gi, '').toLowerCase() || 'root';
|
|
218
|
-
return `${method.toLowerCase()}-${slug}`;
|
|
219
|
-
}
|
|
220
|
-
|
|
221
190
|
function parseLinkHeader(header: string, rel: string): string | null {
|
|
222
191
|
const parts = header.split(',');
|
|
223
192
|
for (const part of parts) {
|
package/src/index.ts
CHANGED
|
@@ -14,6 +14,7 @@ export { peek, read, type PeekOptions, type ReadOptions } from './read/index.js'
|
|
|
14
14
|
export type { PeekResult, ReadResult, Decoder } from './read/types.js';
|
|
15
15
|
export { AuthManager, getMachineId } from './auth/manager.js';
|
|
16
16
|
export { parameterizePath, cleanFrameworkPath } from './capture/parameterize.js';
|
|
17
|
+
export { attach, matchesDomainGlob, parseDomainPatterns } from './capture/cdp-attach.js';
|
|
17
18
|
export { detectPagination } from './capture/pagination.js';
|
|
18
19
|
export { verifyEndpoints } from './capture/verifier.js';
|
|
19
20
|
export { IdleTracker } from './capture/idle.js';
|
package/src/mcp.ts
CHANGED
|
@@ -265,6 +265,12 @@ export function createMcpServer(options: McpServerOptions = {}): McpServer {
|
|
|
265
265
|
},
|
|
266
266
|
},
|
|
267
267
|
async ({ requests, maxBytes }) => {
|
|
268
|
+
// Consume one rate-limit token per batch item
|
|
269
|
+
for (let i = 0; i < requests.length; i++) {
|
|
270
|
+
if (!rateLimiter.check()) {
|
|
271
|
+
return { content: [{ type: 'text' as const, text: `Rate limit exceeded after ${i} of ${requests.length} items. Try again in a moment.` }], isError: true };
|
|
272
|
+
}
|
|
273
|
+
}
|
|
268
274
|
const { replayMultiple } = await import('./replay/engine.js');
|
|
269
275
|
const typed = requests.map(r => ({
|
|
270
276
|
domain: r.domain,
|
|
@@ -334,6 +340,9 @@ export function createMcpServer(options: McpServerOptions = {}): McpServer {
|
|
|
334
340
|
},
|
|
335
341
|
},
|
|
336
342
|
async ({ url }) => {
|
|
343
|
+
if (!rateLimiter.check()) {
|
|
344
|
+
return { content: [{ type: 'text' as const, text: 'Rate limit exceeded. Try again in a moment.' }], isError: true };
|
|
345
|
+
}
|
|
337
346
|
try {
|
|
338
347
|
if (!options._skipSsrfCheck) {
|
|
339
348
|
const validation = await resolveAndValidateUrl(url);
|
|
@@ -415,6 +424,9 @@ export function createMcpServer(options: McpServerOptions = {}): McpServer {
|
|
|
415
424
|
},
|
|
416
425
|
},
|
|
417
426
|
async ({ url, duration }) => {
|
|
427
|
+
if (!rateLimiter.check()) {
|
|
428
|
+
return { content: [{ type: 'text' as const, text: 'Rate limit exceeded. Try again in a moment.' }], isError: true };
|
|
429
|
+
}
|
|
418
430
|
if (!options._skipSsrfCheck) {
|
|
419
431
|
const validation = await resolveAndValidateUrl(url);
|
|
420
432
|
if (!validation.safe) {
|
package/src/native-host.ts
CHANGED
|
@@ -237,6 +237,13 @@ export async function startSocketServer(
|
|
|
237
237
|
|
|
238
238
|
conn.on('data', (chunk) => {
|
|
239
239
|
buffer += chunk.toString();
|
|
240
|
+
|
|
241
|
+
// Guard against unbounded buffer growth (max 10MB)
|
|
242
|
+
if (buffer.length > 10 * 1024 * 1024) {
|
|
243
|
+
conn.destroy();
|
|
244
|
+
return;
|
|
245
|
+
}
|
|
246
|
+
|
|
240
247
|
const newlineIdx = buffer.indexOf('\n');
|
|
241
248
|
if (newlineIdx === -1) return;
|
|
242
249
|
|