@apitap/core 1.4.0 → 1.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -2
- package/dist/auth/crypto.d.ts +10 -0
- package/dist/auth/crypto.js +30 -6
- package/dist/auth/crypto.js.map +1 -1
- package/dist/auth/handoff.js +20 -1
- package/dist/auth/handoff.js.map +1 -1
- package/dist/auth/manager.d.ts +1 -0
- package/dist/auth/manager.js +35 -9
- package/dist/auth/manager.js.map +1 -1
- package/dist/capture/monitor.js +4 -0
- package/dist/capture/monitor.js.map +1 -1
- package/dist/capture/scrubber.js +10 -0
- package/dist/capture/scrubber.js.map +1 -1
- package/dist/capture/session.js +7 -17
- package/dist/capture/session.js.map +1 -1
- package/dist/cli.js +74 -17
- package/dist/cli.js.map +1 -1
- package/dist/discovery/fetch.js +3 -3
- package/dist/discovery/fetch.js.map +1 -1
- package/dist/mcp.d.ts +2 -0
- package/dist/mcp.js +59 -33
- package/dist/mcp.js.map +1 -1
- package/dist/native-host.js +2 -2
- package/dist/native-host.js.map +1 -1
- package/dist/orchestration/browse.js +13 -4
- package/dist/orchestration/browse.js.map +1 -1
- package/dist/plugin.d.ts +1 -1
- package/dist/plugin.js +14 -4
- package/dist/plugin.js.map +1 -1
- package/dist/read/decoders/reddit.js +4 -0
- package/dist/read/decoders/reddit.js.map +1 -1
- package/dist/replay/engine.js +60 -17
- package/dist/replay/engine.js.map +1 -1
- package/dist/serve.d.ts +2 -0
- package/dist/serve.js +8 -1
- package/dist/serve.js.map +1 -1
- package/dist/skill/generator.d.ts +5 -0
- package/dist/skill/generator.js +30 -4
- package/dist/skill/generator.js.map +1 -1
- package/dist/skill/search.js +1 -1
- package/dist/skill/search.js.map +1 -1
- package/dist/skill/signing.js +19 -1
- package/dist/skill/signing.js.map +1 -1
- package/dist/skill/ssrf.js +71 -2
- package/dist/skill/ssrf.js.map +1 -1
- package/dist/skill/store.d.ts +2 -0
- package/dist/skill/store.js +23 -10
- package/dist/skill/store.js.map +1 -1
- package/dist/skill/validate.d.ts +10 -0
- package/dist/skill/validate.js +106 -0
- package/dist/skill/validate.js.map +1 -0
- package/package.json +1 -1
- package/src/auth/crypto.ts +14 -6
- package/src/auth/handoff.ts +19 -1
- package/src/auth/manager.ts +22 -5
- package/src/capture/monitor.ts +4 -0
- package/src/capture/scrubber.ts +12 -0
- package/src/capture/session.ts +5 -14
- package/src/cli.ts +215 -12
- package/src/discovery/fetch.ts +2 -2
- package/src/index/reader.ts +65 -0
- package/src/mcp.ts +64 -31
- package/src/native-host.ts +29 -2
- package/src/orchestration/browse.ts +13 -4
- package/src/plugin.ts +17 -5
- package/src/read/decoders/reddit.ts +3 -3
- package/src/replay/engine.ts +65 -15
- package/src/serve.ts +10 -1
- package/src/skill/generator.ts +32 -4
- package/src/skill/search.ts +1 -1
- package/src/skill/signing.ts +20 -1
- package/src/skill/ssrf.ts +69 -2
- package/src/skill/store.ts +29 -11
- package/src/skill/validate.ts +48 -0
package/src/cli.ts
CHANGED
|
@@ -17,11 +17,13 @@ import { buildInspectReport, formatInspectHuman } from './inspect/report.js';
|
|
|
17
17
|
import { generateStatsReport, formatStatsHuman } from './stats/report.js';
|
|
18
18
|
import { detectAntiBot, type AntiBotSignal } from './capture/anti-bot.js';
|
|
19
19
|
import { discover } from './discovery/index.js';
|
|
20
|
+
import { readIndexEntry } from './index/reader.js';
|
|
20
21
|
import { peek } from './read/peek.js';
|
|
21
22
|
import { read } from './read/index.js';
|
|
22
23
|
import { homedir } from 'node:os';
|
|
23
24
|
import { join, resolve, dirname } from 'node:path';
|
|
24
25
|
import { readFileSync } from 'node:fs';
|
|
26
|
+
import { stat, unlink } from 'node:fs/promises';
|
|
25
27
|
import { fileURLToPath } from 'node:url';
|
|
26
28
|
import { createMcpServer } from './mcp.js';
|
|
27
29
|
|
|
@@ -79,6 +81,8 @@ function printUsage(): void {
|
|
|
79
81
|
apitap browse <url> Browse a URL (discover + replay in one step)
|
|
80
82
|
apitap peek <url> Zero-cost triage (HEAD only)
|
|
81
83
|
apitap read <url> Extract content without a browser
|
|
84
|
+
apitap audit Audit stored skill files and credentials
|
|
85
|
+
apitap forget <domain> Remove skill file and credentials for a domain
|
|
82
86
|
apitap stats Show token savings report
|
|
83
87
|
apitap extension install Register native messaging host for Chrome
|
|
84
88
|
|
|
@@ -158,6 +162,15 @@ async function handleCapture(positional: string[], flags: Record<string, string
|
|
|
158
162
|
const skipVerify = flags['no-verify'] === true;
|
|
159
163
|
const verifyPosts = flags['verify-posts'] === true;
|
|
160
164
|
|
|
165
|
+
// SSRF validation for CLI (H6 fix)
|
|
166
|
+
if (flags['danger-disable-ssrf'] !== true) {
|
|
167
|
+
const ssrfCheck = await resolveAndValidateUrl(fullUrl);
|
|
168
|
+
if (!ssrfCheck.safe) {
|
|
169
|
+
console.error(`Error: URL blocked (SSRF): ${ssrfCheck.reason}`);
|
|
170
|
+
process.exit(1);
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
|
|
161
174
|
if (!json) {
|
|
162
175
|
const domainOnly = flags['all-domains'] !== true;
|
|
163
176
|
console.log(`\n 🔍 Capturing ${url}...${duration ? ` (${duration}s)` : ' (Ctrl+C to stop)'}${domainOnly ? ' [domain-only]' : ' [all domains]'}\n`);
|
|
@@ -341,7 +354,7 @@ async function handleShow(positional: string[], flags: Record<string, string | b
|
|
|
341
354
|
process.exit(1);
|
|
342
355
|
}
|
|
343
356
|
|
|
344
|
-
const skill = await readSkillFile(domain, SKILLS_DIR);
|
|
357
|
+
const skill = await readSkillFile(domain, SKILLS_DIR, { trustUnsigned: true });
|
|
345
358
|
if (!skill) {
|
|
346
359
|
console.error(`Error: No skill file found for "${domain}". Run \`apitap capture\` first.`);
|
|
347
360
|
process.exit(1);
|
|
@@ -380,7 +393,8 @@ async function handleReplay(positional: string[], flags: Record<string, string |
|
|
|
380
393
|
|
|
381
394
|
const machineId = await getMachineId();
|
|
382
395
|
const signingKey = deriveSigningKey(machineId);
|
|
383
|
-
const
|
|
396
|
+
const trustUnsigned = flags['trust-unsigned'] === true;
|
|
397
|
+
const skill = await readSkillFile(domain, SKILLS_DIR, { verifySignature: true, signingKey, trustUnsigned });
|
|
384
398
|
if (!skill) {
|
|
385
399
|
console.error(`Error: No skill file found for "${domain}".`);
|
|
386
400
|
process.exit(1);
|
|
@@ -417,6 +431,10 @@ async function handleReplay(positional: string[], flags: Record<string, string |
|
|
|
417
431
|
const fresh = flags.fresh === true;
|
|
418
432
|
const json = flags.json === true;
|
|
419
433
|
const maxBytes = typeof flags['max-bytes'] === 'string' ? parseInt(flags['max-bytes'], 10) : undefined;
|
|
434
|
+
const dangerDisableSsrf = flags['danger-disable-ssrf'] === true;
|
|
435
|
+
if (dangerDisableSsrf) {
|
|
436
|
+
console.error('[apitap] WARNING: SSRF protection is disabled via --danger-disable-ssrf');
|
|
437
|
+
}
|
|
420
438
|
|
|
421
439
|
const result = await replayEndpoint(skill, endpointId, {
|
|
422
440
|
params: Object.keys(params).length > 0 ? params : undefined,
|
|
@@ -424,7 +442,7 @@ async function handleReplay(positional: string[], flags: Record<string, string |
|
|
|
424
442
|
domain,
|
|
425
443
|
fresh,
|
|
426
444
|
maxBytes,
|
|
427
|
-
_skipSsrfCheck:
|
|
445
|
+
_skipSsrfCheck: dangerDisableSsrf,
|
|
428
446
|
});
|
|
429
447
|
|
|
430
448
|
if (json) {
|
|
@@ -493,7 +511,8 @@ async function handleRefresh(positional: string[], flags: Record<string, string
|
|
|
493
511
|
process.exit(1);
|
|
494
512
|
}
|
|
495
513
|
|
|
496
|
-
const
|
|
514
|
+
const trustUnsigned = flags['trust-unsigned'] === true;
|
|
515
|
+
const skill = await readSkillFile(domain, SKILLS_DIR, { trustUnsigned });
|
|
497
516
|
if (!skill) {
|
|
498
517
|
console.error(`Error: No skill file found for "${domain}".`);
|
|
499
518
|
process.exit(1);
|
|
@@ -597,8 +616,8 @@ async function handleAuth(positional: string[], flags: Record<string, string | b
|
|
|
597
616
|
}
|
|
598
617
|
}
|
|
599
618
|
|
|
600
|
-
// Read skill file for OAuth config (non-secret)
|
|
601
|
-
const skill = await readSkillFile(domain, SKILLS_DIR);
|
|
619
|
+
// Read skill file for OAuth config (non-secret) — trustUnsigned for display only
|
|
620
|
+
const skill = await readSkillFile(domain, SKILLS_DIR, { trustUnsigned: true });
|
|
602
621
|
const oauthConfig = skill?.auth?.oauthConfig;
|
|
603
622
|
|
|
604
623
|
const status = {
|
|
@@ -660,7 +679,7 @@ async function handleServe(positional: string[], flags: Record<string, string |
|
|
|
660
679
|
});
|
|
661
680
|
|
|
662
681
|
// Print tool list to stderr (stdout is the MCP transport)
|
|
663
|
-
const skill = await readSkillFile(domain, SKILLS_DIR);
|
|
682
|
+
const skill = await readSkillFile(domain, SKILLS_DIR, { trustUnsigned: true });
|
|
664
683
|
const tools = buildServeTools(skill!);
|
|
665
684
|
|
|
666
685
|
if (json) {
|
|
@@ -682,10 +701,13 @@ async function handleServe(positional: string[], flags: Record<string, string |
|
|
|
682
701
|
}
|
|
683
702
|
}
|
|
684
703
|
|
|
685
|
-
async function handleMcp(): Promise<void> {
|
|
704
|
+
async function handleMcp(flags: Record<string, string | boolean>): Promise<void> {
|
|
705
|
+
if (flags['danger-disable-ssrf'] === true) {
|
|
706
|
+
console.error('[apitap] WARNING: SSRF protection is disabled via --danger-disable-ssrf');
|
|
707
|
+
}
|
|
686
708
|
const server = createMcpServer({
|
|
687
709
|
skillsDir: SKILLS_DIR,
|
|
688
|
-
_skipSsrfCheck:
|
|
710
|
+
_skipSsrfCheck: flags['danger-disable-ssrf'] === true,
|
|
689
711
|
});
|
|
690
712
|
const { StdioServerTransport } = await import('@modelcontextprotocol/sdk/server/stdio.js');
|
|
691
713
|
const transport = new StdioServerTransport();
|
|
@@ -710,6 +732,16 @@ async function handleInspect(positional: string[], flags: Record<string, string
|
|
|
710
732
|
process.exit(1);
|
|
711
733
|
}
|
|
712
734
|
|
|
735
|
+
// SSRF validation for CLI (H6 fix)
|
|
736
|
+
const fullInspectUrl = url.startsWith('http') ? url : `https://${url}`;
|
|
737
|
+
if (flags['danger-disable-ssrf'] !== true) {
|
|
738
|
+
const ssrfCheck = await resolveAndValidateUrl(fullInspectUrl);
|
|
739
|
+
if (!ssrfCheck.safe) {
|
|
740
|
+
console.error(`Error: URL blocked (SSRF): ${ssrfCheck.reason}`);
|
|
741
|
+
process.exit(1);
|
|
742
|
+
}
|
|
743
|
+
}
|
|
744
|
+
|
|
713
745
|
const json = flags.json === true;
|
|
714
746
|
const duration = typeof flags.duration === 'string' ? parseInt(flags.duration, 10) : 30;
|
|
715
747
|
|
|
@@ -799,14 +831,37 @@ async function handleDiscover(positional: string[], flags: Record<string, string
|
|
|
799
831
|
const json = flags.json === true;
|
|
800
832
|
const save = flags.save === true;
|
|
801
833
|
|
|
834
|
+
// SSRF validation for CLI (H6 fix)
|
|
835
|
+
const fullDiscoverUrl = url.startsWith('http') ? url : `https://${url}`;
|
|
836
|
+
if (flags['danger-disable-ssrf'] !== true) {
|
|
837
|
+
const ssrfCheck = await resolveAndValidateUrl(fullDiscoverUrl);
|
|
838
|
+
if (!ssrfCheck.safe) {
|
|
839
|
+
if (json) {
|
|
840
|
+
console.log(JSON.stringify({ error: `URL blocked (SSRF): ${ssrfCheck.reason}` }));
|
|
841
|
+
} else {
|
|
842
|
+
console.error(`Error: URL blocked (SSRF): ${ssrfCheck.reason}`);
|
|
843
|
+
}
|
|
844
|
+
process.exit(1);
|
|
845
|
+
}
|
|
846
|
+
}
|
|
847
|
+
|
|
802
848
|
if (!json) {
|
|
803
849
|
console.log(`\n Discovering APIs for ${url}...\n`);
|
|
804
850
|
}
|
|
805
851
|
|
|
806
852
|
const result = await discover(url);
|
|
807
853
|
|
|
854
|
+
// Look up passive index data for this domain
|
|
855
|
+
let domain: string;
|
|
856
|
+
try {
|
|
857
|
+
domain = new URL(fullDiscoverUrl).hostname;
|
|
858
|
+
} catch {
|
|
859
|
+
domain = url;
|
|
860
|
+
}
|
|
861
|
+
const indexEntry = await readIndexEntry(domain);
|
|
862
|
+
|
|
808
863
|
if (json) {
|
|
809
|
-
console.log(JSON.stringify(result, null, 2));
|
|
864
|
+
console.log(JSON.stringify({ ...result, indexEntry: indexEntry ?? undefined }, null, 2));
|
|
810
865
|
} else {
|
|
811
866
|
// Confidence summary
|
|
812
867
|
const confidenceLabels: Record<string, string> = {
|
|
@@ -853,6 +908,24 @@ async function handleDiscover(positional: string[], flags: Record<string, string
|
|
|
853
908
|
console.log(`\n Skill file: ${result.skillFile.endpoints.length} endpoints predicted`);
|
|
854
909
|
}
|
|
855
910
|
|
|
911
|
+
if (indexEntry) {
|
|
912
|
+
console.log(`\n Passive Index (from browser extension):`);
|
|
913
|
+
console.log(` Domain: ${indexEntry.domain}`);
|
|
914
|
+
console.log(` Total hits: ${indexEntry.totalHits}`);
|
|
915
|
+
console.log(` Endpoints: ${indexEntry.endpoints.length}`);
|
|
916
|
+
console.log(` Promoted: ${indexEntry.promoted ? 'yes' : 'no'}`);
|
|
917
|
+
if (indexEntry.endpoints.length > 0) {
|
|
918
|
+
console.log(` Observed endpoints:`);
|
|
919
|
+
for (const ep of indexEntry.endpoints) {
|
|
920
|
+
const methods = ep.methods.join(', ');
|
|
921
|
+
const auth = ep.authType ? ` [${ep.authType}]` : '';
|
|
922
|
+
const pagination = ep.pagination ? ` (${ep.pagination})` : '';
|
|
923
|
+
const gql = ep.type === 'graphql' ? ' [GraphQL]' : '';
|
|
924
|
+
console.log(` ${methods} ${ep.path} \u2014 ${ep.hits} hits${auth}${pagination}${gql}`);
|
|
925
|
+
}
|
|
926
|
+
}
|
|
927
|
+
}
|
|
928
|
+
|
|
856
929
|
if (result.confidence === 'none') {
|
|
857
930
|
console.log(`\n Recommendation: Run \`apitap capture ${url}\` for browser-based discovery`);
|
|
858
931
|
}
|
|
@@ -902,7 +975,7 @@ async function handleBrowse(positional: string[], flags: Record<string, string |
|
|
|
902
975
|
skillsDir: SKILLS_DIR,
|
|
903
976
|
cache: new SessionCache(),
|
|
904
977
|
maxBytes,
|
|
905
|
-
_skipSsrfCheck:
|
|
978
|
+
_skipSsrfCheck: flags['danger-disable-ssrf'] === true,
|
|
906
979
|
});
|
|
907
980
|
|
|
908
981
|
if (json) {
|
|
@@ -933,6 +1006,15 @@ async function handlePeek(positional: string[], flags: Record<string, string | b
|
|
|
933
1006
|
const json = flags.json === true;
|
|
934
1007
|
const fullUrl = url.startsWith('http') ? url : `https://${url}`;
|
|
935
1008
|
|
|
1009
|
+
// SSRF validation for CLI (H6 fix)
|
|
1010
|
+
if (flags['danger-disable-ssrf'] !== true) {
|
|
1011
|
+
const ssrfCheck = await resolveAndValidateUrl(fullUrl);
|
|
1012
|
+
if (!ssrfCheck.safe) {
|
|
1013
|
+
console.error(`Error: URL blocked (SSRF): ${ssrfCheck.reason}`);
|
|
1014
|
+
process.exit(1);
|
|
1015
|
+
}
|
|
1016
|
+
}
|
|
1017
|
+
|
|
936
1018
|
if (!json) {
|
|
937
1019
|
console.log(`\n Peeking at ${url}...\n`);
|
|
938
1020
|
}
|
|
@@ -964,6 +1046,15 @@ async function handleRead(positional: string[], flags: Record<string, string | b
|
|
|
964
1046
|
const fullUrl = url.startsWith('http') ? url : `https://${url}`;
|
|
965
1047
|
const maxBytes = typeof flags['max-bytes'] === 'string' ? parseInt(flags['max-bytes'], 10) : undefined;
|
|
966
1048
|
|
|
1049
|
+
// SSRF validation for CLI (H6 fix)
|
|
1050
|
+
if (flags['danger-disable-ssrf'] !== true) {
|
|
1051
|
+
const ssrfCheck = await resolveAndValidateUrl(fullUrl);
|
|
1052
|
+
if (!ssrfCheck.safe) {
|
|
1053
|
+
console.error(`Error: URL blocked (SSRF): ${ssrfCheck.reason}`);
|
|
1054
|
+
process.exit(1);
|
|
1055
|
+
}
|
|
1056
|
+
}
|
|
1057
|
+
|
|
967
1058
|
if (!json) {
|
|
968
1059
|
console.log(`\n Reading ${url}...\n`);
|
|
969
1060
|
}
|
|
@@ -990,6 +1081,112 @@ async function handleRead(positional: string[], flags: Record<string, string | b
|
|
|
990
1081
|
console.log();
|
|
991
1082
|
}
|
|
992
1083
|
|
|
1084
|
+
async function handleAudit(flags: Record<string, string | boolean>): Promise<void> {
|
|
1085
|
+
const skillsDir = SKILLS_DIR || join(APITAP_DIR, 'skills');
|
|
1086
|
+
const summaries = await listSkillFiles(skillsDir);
|
|
1087
|
+
const machineId = await getEffectiveMachineId();
|
|
1088
|
+
const authManager = new AuthManager(APITAP_DIR, machineId);
|
|
1089
|
+
const authDomains = new Set(await authManager.listDomains());
|
|
1090
|
+
|
|
1091
|
+
// Get skill file last-modified dates
|
|
1092
|
+
const rows: { domain: string; endpoints: number; auth: string; modified: string }[] = [];
|
|
1093
|
+
for (const s of summaries) {
|
|
1094
|
+
let modified = 'unknown';
|
|
1095
|
+
try {
|
|
1096
|
+
const st = await stat(s.skillFile);
|
|
1097
|
+
modified = st.mtime.toISOString().slice(0, 10);
|
|
1098
|
+
} catch {}
|
|
1099
|
+
rows.push({
|
|
1100
|
+
domain: s.domain,
|
|
1101
|
+
endpoints: s.endpointCount,
|
|
1102
|
+
auth: authDomains.has(s.domain) ? 'yes' : 'no',
|
|
1103
|
+
modified,
|
|
1104
|
+
});
|
|
1105
|
+
}
|
|
1106
|
+
|
|
1107
|
+
// Get auth.enc last-modified
|
|
1108
|
+
let authModified = 'none';
|
|
1109
|
+
try {
|
|
1110
|
+
const authStat = await stat(join(APITAP_DIR, 'auth.enc'));
|
|
1111
|
+
authModified = authStat.mtime.toISOString().slice(0, 10);
|
|
1112
|
+
} catch {}
|
|
1113
|
+
|
|
1114
|
+
if (flags.json === true) {
|
|
1115
|
+
console.log(JSON.stringify({ domains: rows, authFileModified: authModified }, null, 2));
|
|
1116
|
+
return;
|
|
1117
|
+
}
|
|
1118
|
+
|
|
1119
|
+
if (rows.length === 0) {
|
|
1120
|
+
console.log('\n No skill files found.\n');
|
|
1121
|
+
return;
|
|
1122
|
+
}
|
|
1123
|
+
|
|
1124
|
+
// Table header
|
|
1125
|
+
const domW = Math.max(6, ...rows.map(r => r.domain.length));
|
|
1126
|
+
console.log();
|
|
1127
|
+
console.log(` ${'DOMAIN'.padEnd(domW)} ${'ENDPOINTS'.padStart(9)} ${'AUTH'.padEnd(4)} MODIFIED`);
|
|
1128
|
+
console.log(` ${''.padEnd(domW, '-')} ${''.padEnd(9, '-')} ${''.padEnd(4, '-')} ${''.padEnd(10, '-')}`);
|
|
1129
|
+
for (const r of rows) {
|
|
1130
|
+
console.log(` ${r.domain.padEnd(domW)} ${String(r.endpoints).padStart(9)} ${r.auth.padEnd(4)} ${r.modified}`);
|
|
1131
|
+
}
|
|
1132
|
+
console.log();
|
|
1133
|
+
console.log(` auth.enc last modified: ${authModified}`);
|
|
1134
|
+
console.log();
|
|
1135
|
+
}
|
|
1136
|
+
|
|
1137
|
+
async function handleForget(positional: string[]): Promise<void> {
|
|
1138
|
+
const domain = positional[0];
|
|
1139
|
+
if (!domain) {
|
|
1140
|
+
console.error('Error: Domain required. Usage: apitap forget <domain>');
|
|
1141
|
+
process.exit(1);
|
|
1142
|
+
}
|
|
1143
|
+
|
|
1144
|
+
if (!/^[a-zA-Z0-9][a-zA-Z0-9._-]*$/.test(domain) || domain.includes('..')) {
|
|
1145
|
+
console.error('Error: Invalid domain name');
|
|
1146
|
+
process.exit(1);
|
|
1147
|
+
}
|
|
1148
|
+
|
|
1149
|
+
const skillsDir = SKILLS_DIR || join(APITAP_DIR, 'skills');
|
|
1150
|
+
let skillRemoved = false;
|
|
1151
|
+
let authRemoved = false;
|
|
1152
|
+
let notFound = true;
|
|
1153
|
+
|
|
1154
|
+
// Remove skill file
|
|
1155
|
+
const skillFilePath = join(skillsDir, `${domain}.json`);
|
|
1156
|
+
try {
|
|
1157
|
+
await stat(skillFilePath);
|
|
1158
|
+
notFound = false;
|
|
1159
|
+
await unlink(skillFilePath);
|
|
1160
|
+
skillRemoved = true;
|
|
1161
|
+
} catch {}
|
|
1162
|
+
|
|
1163
|
+
// Remove stored auth
|
|
1164
|
+
try {
|
|
1165
|
+
const machineId = await getEffectiveMachineId();
|
|
1166
|
+
const authManager = new AuthManager(APITAP_DIR, machineId);
|
|
1167
|
+
const hasAuth = await authManager.has(domain);
|
|
1168
|
+
if (hasAuth) {
|
|
1169
|
+
notFound = false;
|
|
1170
|
+
await authManager.clear(domain);
|
|
1171
|
+
authRemoved = true;
|
|
1172
|
+
}
|
|
1173
|
+
} catch (err: any) {
|
|
1174
|
+
if (skillRemoved) {
|
|
1175
|
+
console.error(`Warning: Could not remove auth credentials: ${err.message}`);
|
|
1176
|
+
}
|
|
1177
|
+
}
|
|
1178
|
+
|
|
1179
|
+
if (notFound) {
|
|
1180
|
+
console.log(`${domain} not found`);
|
|
1181
|
+
return;
|
|
1182
|
+
}
|
|
1183
|
+
|
|
1184
|
+
const parts: string[] = [];
|
|
1185
|
+
if (skillRemoved) parts.push('skill file');
|
|
1186
|
+
if (authRemoved) parts.push('credentials');
|
|
1187
|
+
console.log(`Forgot ${domain} — ${parts.join(' and ')} removed`);
|
|
1188
|
+
}
|
|
1189
|
+
|
|
993
1190
|
async function handleExtension(positional: string[], flags: Record<string, string | boolean>): Promise<void> {
|
|
994
1191
|
const subcommand = positional[0];
|
|
995
1192
|
|
|
@@ -1073,7 +1270,7 @@ async function main(): Promise<void> {
|
|
|
1073
1270
|
await handleServe(positional, flags);
|
|
1074
1271
|
break;
|
|
1075
1272
|
case 'mcp':
|
|
1076
|
-
await handleMcp();
|
|
1273
|
+
await handleMcp(flags);
|
|
1077
1274
|
break;
|
|
1078
1275
|
case 'inspect':
|
|
1079
1276
|
await handleInspect(positional, flags);
|
|
@@ -1090,6 +1287,12 @@ async function main(): Promise<void> {
|
|
|
1090
1287
|
case 'read':
|
|
1091
1288
|
await handleRead(positional, flags);
|
|
1092
1289
|
break;
|
|
1290
|
+
case 'audit':
|
|
1291
|
+
await handleAudit(flags);
|
|
1292
|
+
break;
|
|
1293
|
+
case 'forget':
|
|
1294
|
+
await handleForget(positional);
|
|
1295
|
+
break;
|
|
1093
1296
|
case 'extension':
|
|
1094
1297
|
await handleExtension(positional, flags);
|
|
1095
1298
|
break;
|
package/src/discovery/fetch.ts
CHANGED
|
@@ -27,9 +27,9 @@ export async function safeFetch(
|
|
|
27
27
|
url: string,
|
|
28
28
|
options: SafeFetchOptions = {},
|
|
29
29
|
): Promise<FetchResult | null> {
|
|
30
|
-
// SSRF check
|
|
30
|
+
// M18: Use DNS-resolving SSRF check to prevent rebinding attacks
|
|
31
31
|
if (!options.skipSsrf) {
|
|
32
|
-
const ssrfResult =
|
|
32
|
+
const ssrfResult = await resolveAndValidateUrl(url);
|
|
33
33
|
if (!ssrfResult.safe) return null;
|
|
34
34
|
}
|
|
35
35
|
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
import { promises as fs } from 'node:fs';
|
|
2
|
+
import path from 'node:path';
|
|
3
|
+
import os from 'node:os';
|
|
4
|
+
|
|
5
|
+
// Types re-declared here (extension types can't be imported due to different tsconfig).
|
|
6
|
+
// Keep in sync with extension/src/types.ts IndexFile/IndexEntry/IndexEndpoint.
|
|
7
|
+
export interface IndexFile {
|
|
8
|
+
v: 1;
|
|
9
|
+
updatedAt: string;
|
|
10
|
+
entries: IndexEntry[];
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
export interface IndexEntry {
|
|
14
|
+
domain: string;
|
|
15
|
+
firstSeen: string;
|
|
16
|
+
lastSeen: string;
|
|
17
|
+
totalHits: number;
|
|
18
|
+
promoted: boolean;
|
|
19
|
+
lastPromoted?: string;
|
|
20
|
+
skillFileSource?: 'extension' | 'cli';
|
|
21
|
+
endpoints: IndexEndpoint[];
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
export interface IndexEndpoint {
|
|
25
|
+
path: string;
|
|
26
|
+
methods: string[];
|
|
27
|
+
authType?: string;
|
|
28
|
+
hasBody: boolean;
|
|
29
|
+
hits: number;
|
|
30
|
+
lastSeen: string;
|
|
31
|
+
pagination?: string;
|
|
32
|
+
type?: 'graphql';
|
|
33
|
+
queryParamNames?: string[];
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
const DEFAULT_APITAP_DIR = path.join(os.homedir(), '.apitap');
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* Read the full passive index from disk.
|
|
40
|
+
* Returns null if index.json doesn't exist or is invalid.
|
|
41
|
+
*/
|
|
42
|
+
export async function readIndex(apitapDir: string = DEFAULT_APITAP_DIR): Promise<IndexFile | null> {
|
|
43
|
+
const indexPath = path.join(apitapDir, 'index.json');
|
|
44
|
+
try {
|
|
45
|
+
const raw = await fs.readFile(indexPath, 'utf-8');
|
|
46
|
+
const parsed = JSON.parse(raw);
|
|
47
|
+
if (parsed.v !== 1 || !Array.isArray(parsed.entries)) return null;
|
|
48
|
+
return parsed as IndexFile;
|
|
49
|
+
} catch {
|
|
50
|
+
return null;
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/**
|
|
55
|
+
* Read a single domain's index entry.
|
|
56
|
+
* Returns null if the domain is not in the index.
|
|
57
|
+
*/
|
|
58
|
+
export async function readIndexEntry(
|
|
59
|
+
domain: string,
|
|
60
|
+
apitapDir: string = DEFAULT_APITAP_DIR,
|
|
61
|
+
): Promise<IndexEntry | null> {
|
|
62
|
+
const index = await readIndex(apitapDir);
|
|
63
|
+
if (!index) return null;
|
|
64
|
+
return index.entries.find(e => e.domain === domain) ?? null;
|
|
65
|
+
}
|
package/src/mcp.ts
CHANGED
|
@@ -11,6 +11,7 @@ import { deriveSigningKey } from './auth/crypto.js';
|
|
|
11
11
|
import { requestAuth } from './auth/handoff.js';
|
|
12
12
|
import { CaptureSession } from './capture/session.js';
|
|
13
13
|
import { discover } from './discovery/index.js';
|
|
14
|
+
import { readIndexEntry } from './index/reader.js';
|
|
14
15
|
import { SessionCache } from './orchestration/cache.js';
|
|
15
16
|
import { peek } from './read/peek.js';
|
|
16
17
|
import { read } from './read/index.js';
|
|
@@ -49,14 +50,40 @@ export interface McpServerOptions {
|
|
|
49
50
|
skillsDir?: string;
|
|
50
51
|
/** @internal Skip SSRF check in replay — for testing only */
|
|
51
52
|
_skipSsrfCheck?: boolean;
|
|
53
|
+
/** Rate limit: max outbound requests per minute (default: 60). Set 0 to disable. */
|
|
54
|
+
rateLimitPerMinute?: number;
|
|
52
55
|
}
|
|
53
56
|
|
|
54
57
|
const MAX_SESSIONS = 3;
|
|
55
58
|
|
|
59
|
+
/**
|
|
60
|
+
* M23: Simple sliding-window rate limiter for outbound MCP tool calls.
|
|
61
|
+
* Prevents misconfigured agents from flooding target APIs.
|
|
62
|
+
*/
|
|
63
|
+
class RateLimiter {
|
|
64
|
+
private timestamps: number[] = [];
|
|
65
|
+
private readonly maxPerMinute: number;
|
|
66
|
+
|
|
67
|
+
constructor(maxPerMinute: number) {
|
|
68
|
+
this.maxPerMinute = maxPerMinute;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
check(): boolean {
|
|
72
|
+
if (this.maxPerMinute <= 0) return true; // Disabled
|
|
73
|
+
const now = Date.now();
|
|
74
|
+
const windowStart = now - 60_000;
|
|
75
|
+
this.timestamps = this.timestamps.filter(t => t > windowStart);
|
|
76
|
+
if (this.timestamps.length >= this.maxPerMinute) return false;
|
|
77
|
+
this.timestamps.push(now);
|
|
78
|
+
return true;
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
|
|
56
82
|
export function createMcpServer(options: McpServerOptions = {}): McpServer {
|
|
57
83
|
const skillsDir = options.skillsDir;
|
|
58
84
|
const sessions = new Map<string, CaptureSession>();
|
|
59
85
|
const sessionCache = new SessionCache();
|
|
86
|
+
const rateLimiter = new RateLimiter(options.rateLimitPerMinute ?? 60);
|
|
60
87
|
|
|
61
88
|
const server = new McpServer({
|
|
62
89
|
name: 'apitap',
|
|
@@ -80,9 +107,7 @@ export function createMcpServer(options: McpServerOptions = {}): McpServer {
|
|
|
80
107
|
},
|
|
81
108
|
async ({ query }) => {
|
|
82
109
|
const result = await searchSkills(query, skillsDir);
|
|
83
|
-
return
|
|
84
|
-
content: [{ type: 'text' as const, text: JSON.stringify(result) }],
|
|
85
|
-
};
|
|
110
|
+
return wrapExternalContent(result, 'apitap_search');
|
|
86
111
|
},
|
|
87
112
|
);
|
|
88
113
|
|
|
@@ -103,6 +128,9 @@ export function createMcpServer(options: McpServerOptions = {}): McpServer {
|
|
|
103
128
|
},
|
|
104
129
|
},
|
|
105
130
|
async ({ url }) => {
|
|
131
|
+
if (!rateLimiter.check()) {
|
|
132
|
+
return { content: [{ type: 'text' as const, text: 'Rate limit exceeded. Try again in a moment.' }], isError: true };
|
|
133
|
+
}
|
|
106
134
|
try {
|
|
107
135
|
if (!options._skipSsrfCheck) {
|
|
108
136
|
const validation = await resolveAndValidateUrl(url);
|
|
@@ -112,18 +140,23 @@ export function createMcpServer(options: McpServerOptions = {}): McpServer {
|
|
|
112
140
|
}
|
|
113
141
|
const result = await discover(url);
|
|
114
142
|
|
|
115
|
-
//
|
|
143
|
+
// Look up passive index data for this domain
|
|
144
|
+
let domain: string;
|
|
145
|
+
try { domain = new URL(url).hostname; } catch { domain = url; }
|
|
146
|
+
const indexEntry = await readIndexEntry(domain);
|
|
147
|
+
|
|
148
|
+
// If we got a skill file, sign and save it automatically
|
|
116
149
|
if (result.skillFile && (result.confidence === 'high' || result.confidence === 'medium')) {
|
|
117
150
|
const { writeSkillFile } = await import('./skill/store.js');
|
|
151
|
+
const { signSkillFile } = await import('./skill/signing.js');
|
|
152
|
+
const machineId = await getMachineId();
|
|
153
|
+
const sigKey = deriveSigningKey(machineId);
|
|
154
|
+
result.skillFile = signSkillFile(result.skillFile, sigKey);
|
|
118
155
|
const path = await writeSkillFile(result.skillFile, skillsDir);
|
|
119
|
-
return {
|
|
120
|
-
content: [{ type: 'text' as const, text: JSON.stringify({ ...result, savedTo: path }) }],
|
|
121
|
-
};
|
|
156
|
+
return wrapExternalContent({ ...result, savedTo: path, indexEntry: indexEntry ?? undefined }, 'apitap_discover');
|
|
122
157
|
}
|
|
123
158
|
|
|
124
|
-
return {
|
|
125
|
-
content: [{ type: 'text' as const, text: JSON.stringify(result) }],
|
|
126
|
-
};
|
|
159
|
+
return wrapExternalContent({ ...result, indexEntry: indexEntry ?? undefined }, 'apitap_discover');
|
|
127
160
|
} catch (err: any) {
|
|
128
161
|
return {
|
|
129
162
|
content: [{ type: 'text' as const, text: `Discovery failed: ${err.message}` }],
|
|
@@ -154,6 +187,9 @@ export function createMcpServer(options: McpServerOptions = {}): McpServer {
|
|
|
154
187
|
},
|
|
155
188
|
},
|
|
156
189
|
async ({ domain, endpointId, params, fresh, maxBytes }) => {
|
|
190
|
+
if (!rateLimiter.check()) {
|
|
191
|
+
return { content: [{ type: 'text' as const, text: 'Rate limit exceeded. Try again in a moment.' }], isError: true };
|
|
192
|
+
}
|
|
157
193
|
const machineId = await getMachineId();
|
|
158
194
|
const signingKey = deriveSigningKey(machineId);
|
|
159
195
|
const skill = await readSkillFile(domain, skillsDir, { verifySignature: true, signingKey });
|
|
@@ -258,6 +294,9 @@ export function createMcpServer(options: McpServerOptions = {}): McpServer {
|
|
|
258
294
|
},
|
|
259
295
|
},
|
|
260
296
|
async ({ url, task, maxBytes }) => {
|
|
297
|
+
if (!rateLimiter.check()) {
|
|
298
|
+
return { content: [{ type: 'text' as const, text: 'Rate limit exceeded. Try again in a moment.' }], isError: true };
|
|
299
|
+
}
|
|
261
300
|
if (!options._skipSsrfCheck) {
|
|
262
301
|
const validation = await resolveAndValidateUrl(url);
|
|
263
302
|
if (!validation.safe) {
|
|
@@ -274,13 +313,8 @@ export function createMcpServer(options: McpServerOptions = {}): McpServer {
|
|
|
274
313
|
// In test mode, disable bridge to avoid connecting to real socket
|
|
275
314
|
...(options._skipSsrfCheck ? { _bridgeSocketPath: '/nonexistent' } : {}),
|
|
276
315
|
});
|
|
277
|
-
//
|
|
278
|
-
|
|
279
|
-
return wrapExternalContent(result, 'apitap_browse');
|
|
280
|
-
}
|
|
281
|
-
return {
|
|
282
|
-
content: [{ type: 'text' as const, text: JSON.stringify(result) }],
|
|
283
|
-
};
|
|
316
|
+
// Always mark as untrusted — failed results may contain attacker-controlled strings (H7 fix)
|
|
317
|
+
return wrapExternalContent(result, 'apitap_browse');
|
|
284
318
|
},
|
|
285
319
|
);
|
|
286
320
|
|
|
@@ -336,6 +370,9 @@ export function createMcpServer(options: McpServerOptions = {}): McpServer {
|
|
|
336
370
|
},
|
|
337
371
|
},
|
|
338
372
|
async ({ url, maxBytes }) => {
|
|
373
|
+
if (!rateLimiter.check()) {
|
|
374
|
+
return { content: [{ type: 'text' as const, text: 'Rate limit exceeded. Try again in a moment.' }], isError: true };
|
|
375
|
+
}
|
|
339
376
|
try {
|
|
340
377
|
if (!options._skipSsrfCheck) {
|
|
341
378
|
const validation = await resolveAndValidateUrl(url);
|
|
@@ -404,9 +441,7 @@ export function createMcpServer(options: McpServerOptions = {}): McpServer {
|
|
|
404
441
|
setTimeout(() => reject(new Error('Capture timed out')), timeoutMs),
|
|
405
442
|
),
|
|
406
443
|
]);
|
|
407
|
-
return
|
|
408
|
-
content: [{ type: 'text' as const, text: JSON.stringify(result) }],
|
|
409
|
-
};
|
|
444
|
+
return wrapExternalContent(result, 'apitap_capture');
|
|
410
445
|
} catch (err: any) {
|
|
411
446
|
try { await session.abort(); } catch { /* already closed */ }
|
|
412
447
|
return {
|
|
@@ -457,9 +492,8 @@ export function createMcpServer(options: McpServerOptions = {}): McpServer {
|
|
|
457
492
|
});
|
|
458
493
|
const snapshot = await session.start(url);
|
|
459
494
|
sessions.set(session.id, session);
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
};
|
|
495
|
+
// Mark as untrusted — snapshot contains external page content (H5 fix)
|
|
496
|
+
return wrapExternalContent({ sessionId: session.id, snapshot }, 'apitap_capture_start');
|
|
463
497
|
} catch (err: any) {
|
|
464
498
|
return {
|
|
465
499
|
content: [{ type: 'text' as const, text: `Failed to start capture session: ${err.message}` }],
|
|
@@ -518,8 +552,10 @@ export function createMcpServer(options: McpServerOptions = {}): McpServer {
|
|
|
518
552
|
submit,
|
|
519
553
|
});
|
|
520
554
|
|
|
555
|
+
// Mark as untrusted — result contains external page content (H5 fix)
|
|
556
|
+
const wrapped = wrapExternalContent(result, 'apitap_capture_interact');
|
|
521
557
|
return {
|
|
522
|
-
|
|
558
|
+
...wrapped,
|
|
523
559
|
...(result.success ? {} : { isError: true }),
|
|
524
560
|
};
|
|
525
561
|
},
|
|
@@ -560,15 +596,11 @@ export function createMcpServer(options: McpServerOptions = {}): McpServer {
|
|
|
560
596
|
try {
|
|
561
597
|
if (shouldAbort) {
|
|
562
598
|
await session.abort();
|
|
563
|
-
return {
|
|
564
|
-
content: [{ type: 'text' as const, text: JSON.stringify({ aborted: true, domains: [] }) }],
|
|
565
|
-
};
|
|
599
|
+
return wrapExternalContent({ aborted: true, domains: [] }, 'apitap_capture_finish');
|
|
566
600
|
}
|
|
567
601
|
|
|
568
602
|
const result = await session.finish();
|
|
569
|
-
return
|
|
570
|
-
content: [{ type: 'text' as const, text: JSON.stringify(result) }],
|
|
571
|
-
};
|
|
603
|
+
return wrapExternalContent(result, 'apitap_capture_finish');
|
|
572
604
|
} catch (err: any) {
|
|
573
605
|
return {
|
|
574
606
|
content: [{ type: 'text' as const, text: `Finish failed: ${err.message}` }],
|
|
@@ -609,8 +641,9 @@ export function createMcpServer(options: McpServerOptions = {}): McpServer {
|
|
|
609
641
|
timeout: timeout ? timeout * 1000 : undefined,
|
|
610
642
|
});
|
|
611
643
|
|
|
644
|
+
const wrapped = wrapExternalContent(result, 'apitap_auth_request');
|
|
612
645
|
return {
|
|
613
|
-
|
|
646
|
+
...wrapped,
|
|
614
647
|
...(result.success ? {} : { isError: true }),
|
|
615
648
|
};
|
|
616
649
|
} catch (err: any) {
|