@apitap/core 1.2.0 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/auth/crypto.js +2 -2
- package/dist/auth/crypto.js.map +1 -1
- package/dist/auth/oauth-refresh.d.ts +0 -1
- package/dist/auth/oauth-refresh.js +9 -2
- package/dist/auth/oauth-refresh.js.map +1 -1
- package/dist/capture/body-variables.js +3 -0
- package/dist/capture/body-variables.js.map +1 -1
- package/dist/capture/verifier.d.ts +2 -0
- package/dist/capture/verifier.js +18 -4
- package/dist/capture/verifier.js.map +1 -1
- package/dist/cli.js +6 -1
- package/dist/cli.js.map +1 -1
- package/dist/discovery/fetch.js +3 -3
- package/dist/discovery/fetch.js.map +1 -1
- package/dist/mcp.js +37 -3
- package/dist/mcp.js.map +1 -1
- package/dist/native-host.js +5 -1
- package/dist/native-host.js.map +1 -1
- package/dist/read/decoders/deepwiki.js +8 -1
- package/dist/read/decoders/deepwiki.js.map +1 -1
- package/dist/replay/engine.js +22 -2
- package/dist/replay/engine.js.map +1 -1
- package/dist/serve.js +1 -1
- package/dist/serve.js.map +1 -1
- package/dist/skill/generator.js +33 -4
- package/dist/skill/generator.js.map +1 -1
- package/dist/skill/ssrf.js +10 -2
- package/dist/skill/ssrf.js.map +1 -1
- package/dist/skill/store.js +4 -1
- package/dist/skill/store.js.map +1 -1
- package/package.json +3 -2
- package/src/auth/crypto.ts +2 -2
- package/src/auth/oauth-refresh.ts +7 -3
- package/src/capture/body-variables.ts +3 -0
- package/src/capture/verifier.ts +20 -2
- package/src/cli.ts +6 -1
- package/src/discovery/fetch.ts +3 -3
- package/src/mcp.ts +37 -3
- package/src/native-host.ts +5 -1
- package/src/read/decoders/deepwiki.ts +9 -1
- package/src/replay/engine.ts +21 -2
- package/src/serve.ts +1 -1
- package/src/skill/generator.ts +32 -4
- package/src/skill/ssrf.ts +11 -2
- package/src/skill/store.ts +3 -1
package/dist/skill/store.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"store.js","sourceRoot":"","sources":["../../src/skill/store.ts"],"names":[],"mappings":"AAAA,qBAAqB;AACrB,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC/E,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAGlC,MAAM,kBAAkB,GAAG,IAAI,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC;AAEhE,MAAM,cAAc,GAAG;;;CAGtB,CAAC;AAEF,SAAS,SAAS,CAAC,MAAc,EAAE,SAAiB;IAClD,IAAI,CAAC,8BAA8B,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;QACjD,MAAM,IAAI,KAAK,CAAC,mBAAmB,MAAM,EAAE,CAAC,CAAC;IAC/C,CAAC;IACD,OAAO,IAAI,CAAC,SAAS,EAAE,GAAG,MAAM,OAAO,CAAC,CAAC;AAC3C,CAAC;AAED,KAAK,UAAU,eAAe,CAAC,SAAiB;IAC9C,MAAM,OAAO,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC;IACnC,MAAM,aAAa,GAAG,IAAI,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;IAElD,IAAI,CAAC;QACH,MAAM,MAAM,CAAC,aAAa,CAAC,CAAC;QAC5B,+BAA+B;IACjC,CAAC;IAAC,MAAM,CAAC;QACP,gCAAgC;QAChC,MAAM,KAAK,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC1C,MAAM,SAAS,CAAC,aAAa,EAAE,cAAc,CAAC,CAAC;IACjD,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,KAAgB,EAChB,YAAoB,kBAAkB;IAEtC,MAAM,KAAK,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC5C,MAAM,eAAe,CAAC,SAAS,CAAC,CAAC;IACjC,MAAM,QAAQ,GAAG,SAAS,CAAC,KAAK,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;IACpD,MAAM,SAAS,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"store.js","sourceRoot":"","sources":["../../src/skill/store.ts"],"names":[],"mappings":"AAAA,qBAAqB;AACrB,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC/E,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAGlC,MAAM,kBAAkB,GAAG,IAAI,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC;AAEhE,MAAM,cAAc,GAAG;;;CAGtB,CAAC;AAEF,SAAS,SAAS,CAAC,MAAc,EAAE,SAAiB;IAClD,IAAI,CAAC,8BAA8B,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;QACjD,MAAM,IAAI,KAAK,CAAC,mBAAmB,MAAM,EAAE,CAAC,CAAC;IAC/C,CAAC;IACD,OAAO,IAAI,CAAC,SAAS,EAAE,GAAG,MAAM,OAAO,CAAC,CAAC;AAC3C,CAAC;AAED,KAAK,UAAU,eAAe,CAAC,SAAiB;IAC9C,MAAM,OAAO,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC;IACnC,MAAM,aAAa,GAAG,IAAI,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;IAElD,IAAI,CAAC;QACH,MAAM,MAAM,CAAC,aAAa,CAAC,CAAC;QAC5B,+BAA+B;IACjC,CAAC;IAAC,MAAM,CAAC;QACP,gCAAgC;QAChC,MAAM,KAAK,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC1C,MAAM,SAAS,CAAC,aAAa,EAAE,cAAc,CAAC,CAAC;IACjD,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,KAAgB,EAChB,YAAoB,kBAAkB;IAEtC,MAAM,KAAK,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC5C,MAAM,eAAe,CAAC,SAAS,CAAC,CAAC;IACjC,MAAM,QAAQ,GAAG,SAAS,CAAC,KAAK,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;IACpD,MAAM,SAAS,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IAClF,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,MAAc,EACd,YAAoB,kBAAkB,EACtC,OAA4D;IAE5D,iFAAiF;IACjF,MAAM,IAAI,GAAG,SAAS,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;IAC1C,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QAC9C,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAc,CAAC;QAE/C,6CAA6C;QAC7C,IAAI,OAAO,EAAE,eAAe,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;YACnD,IAAI,KAAK,CAAC,UAAU,KAAK,UAAU,EAAE,CAAC;gBACpC,0EAA0E;gBAC1E,2CAA2C;YAC7C,CAAC;iBAAM,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC;gBAC5B,4CAA4C;gBAC5C,MAAM,IAAI,KAAK,CAAC,kBAAkB,MAAM,0CAA0C,CAAC,CAAC;YACtF,CAAC;iBAAM,CAAC;gBACN,MAAM,EAAE,eAAe,EAAE,GAAG,MAAM,MAAM,CAAC,cAAc,CAAC,CAAC;gBACzD,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC;oBAChD,MAAM,IAAI,KAAK,CAAC,gDAAgD,MAAM,yBAAyB,CAAC,CAAC;gBACnG,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,KAAK,CAAC;IACf,CAAC;IAAC,OAAO,CAAU,EAAE,CAAC;QACpB,IAAK,CAA2B,CAAC,IAAI,KAAK,QAAQ;YAAE,OAAO,IAAI,CAAC;QAChE,MAAM,CAAC,CAAC;IACV,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,YAAoB,kBAAkB;IAEtC,IAAI,KAAe,CAAC;IACpB,IAAI,CAAC;QACH,KAAK,GAAG,MAAM,OAAO,CAAC,SAAS,CAAC,CAAC;IACnC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,MAAM,SAAS,GAAmB,EAAE,CAAC;IACrC,MAAM,SAAS,GAAG,8BAA8B,CAAC;IACjD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC;YAAE,SAAS;QACtC,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;QAC3C,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC;YAAE,SAAS,CAAC,gCAAgC;QACvE,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;QACrD,IAAI,KAAK,EAAE,CAAC;YACV,SAAS,CAAC,IAAI,CAAC;gBACb,MAAM,EAAE,KAAK,CAAC,MAAM;gBACpB,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC;gBAChC,aAAa,EAAE,KAAK,CAAC,SAAS,CAAC,MAAM;gBACrC,UAAU,EAAE,KAAK,CAAC,UAAU;gBAC5B,UAAU,EAAE,KAAK,CAAC,UAAU,IAAI,UAAU;aAC3C,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,OAAO,SAAS,CAAC;AACnB,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@apitap/core",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.3.0",
|
|
4
4
|
"description": "Intercept web API traffic during browsing. Generate portable skill files so AI agents can call APIs directly instead of scraping.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "dist/index.js",
|
|
@@ -14,7 +14,8 @@
|
|
|
14
14
|
"dev": "tsx src/cli.ts",
|
|
15
15
|
"test": "node --import tsx --test 'test/**/*.test.ts'",
|
|
16
16
|
"typecheck": "tsc --noEmit",
|
|
17
|
-
"prepublishOnly": "npm run build"
|
|
17
|
+
"prepublishOnly": "npm run build",
|
|
18
|
+
"postpublish": "gh release create v$(node -p \"require('./package.json').version\") --title \"v$(node -p \\\"require('./package.json').version\\\")\" --generate-notes && echo '✅ GitHub release created'"
|
|
18
19
|
},
|
|
19
20
|
"license": "BSL-1.1",
|
|
20
21
|
"repository": {
|
package/src/auth/crypto.ts
CHANGED
|
@@ -10,7 +10,7 @@ import {
|
|
|
10
10
|
|
|
11
11
|
const ALGORITHM = 'aes-256-gcm';
|
|
12
12
|
const KEY_LENGTH = 32; // 256 bits
|
|
13
|
-
const IV_LENGTH =
|
|
13
|
+
const IV_LENGTH = 12; // NIST SP 800-38D recommended for GCM
|
|
14
14
|
const PBKDF2_ITERATIONS = 100_000;
|
|
15
15
|
// const PBKDF2_SALT = 'apitap-v0.2-key-derivation'; // fallback for migration
|
|
16
16
|
import { readFileSync, writeFileSync, existsSync, mkdirSync } from 'node:fs';
|
|
@@ -66,7 +66,7 @@ export function encrypt(plaintext: string, key: Buffer): EncryptedData {
|
|
|
66
66
|
const tag = cipher.getAuthTag();
|
|
67
67
|
|
|
68
68
|
return {
|
|
69
|
-
salt: '
|
|
69
|
+
salt: 'v2', // Format version marker — actual salt is per-install
|
|
70
70
|
iv: iv.toString('hex'),
|
|
71
71
|
ciphertext,
|
|
72
72
|
tag: tag.toString('hex'),
|
|
@@ -5,7 +5,6 @@ import { resolveAndValidateUrl } from '../skill/ssrf.js';
|
|
|
5
5
|
|
|
6
6
|
export interface OAuthRefreshResult {
|
|
7
7
|
success: boolean;
|
|
8
|
-
accessToken?: string;
|
|
9
8
|
tokenRotated?: boolean;
|
|
10
9
|
error?: string;
|
|
11
10
|
}
|
|
@@ -74,9 +73,14 @@ export async function refreshOAuth(
|
|
|
74
73
|
|
|
75
74
|
if (!response.ok) {
|
|
76
75
|
const errorText = await response.text().catch(() => '');
|
|
76
|
+
let errorSummary = `${response.status}`;
|
|
77
|
+
try {
|
|
78
|
+
const errJson = JSON.parse(errorText);
|
|
79
|
+
if (typeof errJson.error === 'string') errorSummary += `: ${errJson.error}`;
|
|
80
|
+
} catch { /* non-JSON, omit body */ }
|
|
77
81
|
return {
|
|
78
82
|
success: false,
|
|
79
|
-
error: `Token endpoint returned ${
|
|
83
|
+
error: `Token endpoint returned ${errorSummary}`,
|
|
80
84
|
};
|
|
81
85
|
}
|
|
82
86
|
|
|
@@ -119,7 +123,7 @@ export async function refreshOAuth(
|
|
|
119
123
|
tokenRotated = true;
|
|
120
124
|
}
|
|
121
125
|
|
|
122
|
-
return { success: true,
|
|
126
|
+
return { success: true, tokenRotated };
|
|
123
127
|
} catch (error) {
|
|
124
128
|
return {
|
|
125
129
|
success: false,
|
|
@@ -136,8 +136,11 @@ export function substituteBodyVariables(
|
|
|
136
136
|
return result;
|
|
137
137
|
}
|
|
138
138
|
|
|
139
|
+
const FORBIDDEN_KEYS = new Set(['__proto__', 'constructor', 'prototype']);
|
|
140
|
+
|
|
139
141
|
function setNestedValue(obj: Record<string, unknown>, path: string, value: string): void {
|
|
140
142
|
const parts = path.split('.');
|
|
143
|
+
if (parts.some(p => FORBIDDEN_KEYS.has(p))) return; // block prototype pollution
|
|
141
144
|
let current = obj;
|
|
142
145
|
|
|
143
146
|
for (let i = 0; i < parts.length - 1; i++) {
|
package/src/capture/verifier.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
// src/capture/verifier.ts
|
|
2
2
|
import type { SkillFile, SkillEndpoint, Replayability } from '../types.js';
|
|
3
|
+
import { validateUrl } from '../skill/ssrf.js';
|
|
3
4
|
|
|
4
5
|
/**
|
|
5
6
|
* Heuristic tier classification for non-GET endpoints (or when verification is skipped).
|
|
@@ -35,10 +36,17 @@ export function classifyHeuristic(endpoint: SkillEndpoint): Replayability {
|
|
|
35
36
|
*/
|
|
36
37
|
async function verifySingle(
|
|
37
38
|
endpoint: SkillEndpoint,
|
|
39
|
+
skipSsrf = false,
|
|
38
40
|
): Promise<Replayability> {
|
|
39
41
|
const url = endpoint.examples.request.url;
|
|
40
42
|
if (!url) return classifyHeuristic(endpoint);
|
|
41
43
|
|
|
44
|
+
// SSRF check before verification fetch
|
|
45
|
+
if (!skipSsrf) {
|
|
46
|
+
const ssrfResult = validateUrl(url);
|
|
47
|
+
if (!ssrfResult.safe) return classifyHeuristic(endpoint);
|
|
48
|
+
}
|
|
49
|
+
|
|
42
50
|
// Build headers: use endpoint headers but exclude [stored] auth placeholders
|
|
43
51
|
const headers: Record<string, string> = {};
|
|
44
52
|
for (const [k, v] of Object.entries(endpoint.headers)) {
|
|
@@ -89,10 +97,17 @@ async function verifySingle(
|
|
|
89
97
|
*/
|
|
90
98
|
async function verifySinglePost(
|
|
91
99
|
endpoint: SkillEndpoint,
|
|
100
|
+
skipSsrf = false,
|
|
92
101
|
): Promise<Replayability> {
|
|
93
102
|
const url = endpoint.examples.request.url;
|
|
94
103
|
if (!url || !endpoint.requestBody) return classifyHeuristic(endpoint);
|
|
95
104
|
|
|
105
|
+
// SSRF check before verification fetch
|
|
106
|
+
if (!skipSsrf) {
|
|
107
|
+
const ssrfResult = validateUrl(url);
|
|
108
|
+
if (!ssrfResult.safe) return classifyHeuristic(endpoint);
|
|
109
|
+
}
|
|
110
|
+
|
|
96
111
|
const headers: Record<string, string> = {};
|
|
97
112
|
for (const [k, v] of Object.entries(endpoint.headers)) {
|
|
98
113
|
if (v !== '[stored]') {
|
|
@@ -144,6 +159,8 @@ async function verifySinglePost(
|
|
|
144
159
|
export interface VerifyOptions {
|
|
145
160
|
/** Verify POST/PUT/PATCH endpoints by replaying them (opt-in, may cause side effects). */
|
|
146
161
|
verifyPosts?: boolean;
|
|
162
|
+
/** @internal Skip SSRF check — for testing only */
|
|
163
|
+
_skipSsrfCheck?: boolean;
|
|
147
164
|
}
|
|
148
165
|
|
|
149
166
|
/**
|
|
@@ -156,10 +173,11 @@ export async function verifyEndpoints(skill: SkillFile, opts?: VerifyOptions): P
|
|
|
156
173
|
const verifiedEndpoints = await Promise.all(
|
|
157
174
|
skill.endpoints.map(async (ep) => {
|
|
158
175
|
let replayability: Replayability;
|
|
176
|
+
const skipSsrf = opts?._skipSsrfCheck ?? false;
|
|
159
177
|
if (ep.method === 'GET') {
|
|
160
|
-
replayability = await verifySingle(ep);
|
|
178
|
+
replayability = await verifySingle(ep, skipSsrf);
|
|
161
179
|
} else if (opts?.verifyPosts) {
|
|
162
|
-
replayability = await verifySinglePost(ep);
|
|
180
|
+
replayability = await verifySinglePost(ep, skipSsrf);
|
|
163
181
|
} else {
|
|
164
182
|
replayability = classifyHeuristic(ep);
|
|
165
183
|
}
|
package/src/cli.ts
CHANGED
|
@@ -512,7 +512,12 @@ async function handleRefresh(positional: string[], flags: Record<string, string
|
|
|
512
512
|
});
|
|
513
513
|
|
|
514
514
|
if (json) {
|
|
515
|
-
|
|
515
|
+
// Redact token values from JSON output to prevent credential leaks in logs
|
|
516
|
+
const safeResult = {
|
|
517
|
+
...result,
|
|
518
|
+
tokens: Object.fromEntries(Object.keys(result.tokens).map(k => [k, '[redacted]'])),
|
|
519
|
+
};
|
|
520
|
+
console.log(JSON.stringify(safeResult, null, 2));
|
|
516
521
|
} else if (result.success) {
|
|
517
522
|
if (result.oauthRefreshed) {
|
|
518
523
|
console.log(` ✓ OAuth token refreshed via token endpoint`);
|
package/src/discovery/fetch.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// src/discovery/fetch.ts
|
|
2
|
-
import { validateUrl } from '../skill/ssrf.js';
|
|
2
|
+
import { validateUrl, resolveAndValidateUrl } from '../skill/ssrf.js';
|
|
3
3
|
|
|
4
4
|
export interface FetchResult {
|
|
5
5
|
status: number;
|
|
@@ -53,12 +53,12 @@ export async function safeFetch(
|
|
|
53
53
|
|
|
54
54
|
clearTimeout(timer);
|
|
55
55
|
|
|
56
|
-
// SSRF-safe manual redirect (one hop max)
|
|
56
|
+
// SSRF-safe manual redirect (one hop max, with DNS resolution check)
|
|
57
57
|
if (response.status >= 300 && response.status < 400 && response.headers.has('location')) {
|
|
58
58
|
const location = response.headers.get('location');
|
|
59
59
|
if (!location) return null;
|
|
60
60
|
const redirectUrl = new URL(location, url).toString();
|
|
61
|
-
const ssrfResult =
|
|
61
|
+
const ssrfResult = await resolveAndValidateUrl(redirectUrl);
|
|
62
62
|
if (!ssrfResult.safe) return null;
|
|
63
63
|
// Follow one redirect hop only
|
|
64
64
|
return await safeFetch(redirectUrl, { ...options, skipSsrf: true });
|
package/src/mcp.ts
CHANGED
|
@@ -103,6 +103,12 @@ export function createMcpServer(options: McpServerOptions = {}): McpServer {
|
|
|
103
103
|
},
|
|
104
104
|
async ({ url }) => {
|
|
105
105
|
try {
|
|
106
|
+
if (!options._skipSsrfCheck) {
|
|
107
|
+
const validation = await resolveAndValidateUrl(url);
|
|
108
|
+
if (!validation.safe) {
|
|
109
|
+
return { content: [{ type: 'text' as const, text: `Blocked: ${validation.reason}` }], isError: true };
|
|
110
|
+
}
|
|
111
|
+
}
|
|
106
112
|
const result = await discover(url);
|
|
107
113
|
|
|
108
114
|
// If we got a skill file, save it automatically
|
|
@@ -250,6 +256,12 @@ export function createMcpServer(options: McpServerOptions = {}): McpServer {
|
|
|
250
256
|
},
|
|
251
257
|
},
|
|
252
258
|
async ({ url, task, maxBytes }) => {
|
|
259
|
+
if (!options._skipSsrfCheck) {
|
|
260
|
+
const validation = await resolveAndValidateUrl(url);
|
|
261
|
+
if (!validation.safe) {
|
|
262
|
+
return { content: [{ type: 'text' as const, text: `Blocked: ${validation.reason}` }], isError: true };
|
|
263
|
+
}
|
|
264
|
+
}
|
|
253
265
|
const { browse: doBrowse } = await import('./orchestration/browse.js');
|
|
254
266
|
const result = await doBrowse(url, {
|
|
255
267
|
skillsDir,
|
|
@@ -257,6 +269,8 @@ export function createMcpServer(options: McpServerOptions = {}): McpServer {
|
|
|
257
269
|
task,
|
|
258
270
|
maxBytes: maxBytes ?? 50_000,
|
|
259
271
|
_skipSsrfCheck: options._skipSsrfCheck,
|
|
272
|
+
// In test mode, disable bridge to avoid connecting to real socket
|
|
273
|
+
...(options._skipSsrfCheck ? { _bridgeSocketPath: '/nonexistent' } : {}),
|
|
260
274
|
});
|
|
261
275
|
// Only mark as untrusted if it contains external data
|
|
262
276
|
if (result.success && result.data) {
|
|
@@ -285,6 +299,12 @@ export function createMcpServer(options: McpServerOptions = {}): McpServer {
|
|
|
285
299
|
},
|
|
286
300
|
async ({ url }) => {
|
|
287
301
|
try {
|
|
302
|
+
if (!options._skipSsrfCheck) {
|
|
303
|
+
const validation = await resolveAndValidateUrl(url);
|
|
304
|
+
if (!validation.safe) {
|
|
305
|
+
return { content: [{ type: 'text' as const, text: `Blocked: ${validation.reason}` }], isError: true };
|
|
306
|
+
}
|
|
307
|
+
}
|
|
288
308
|
const result = await peek(url);
|
|
289
309
|
// Peek returns metadata, not content — but still from external source
|
|
290
310
|
return wrapExternalContent(result, 'apitap_peek');
|
|
@@ -315,9 +335,11 @@ export function createMcpServer(options: McpServerOptions = {}): McpServer {
|
|
|
315
335
|
},
|
|
316
336
|
async ({ url, maxBytes }) => {
|
|
317
337
|
try {
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
338
|
+
if (!options._skipSsrfCheck) {
|
|
339
|
+
const validation = await resolveAndValidateUrl(url);
|
|
340
|
+
if (!validation.safe) {
|
|
341
|
+
throw new Error(validation.reason ?? 'URL validation failed');
|
|
342
|
+
}
|
|
321
343
|
}
|
|
322
344
|
const result = await read(url, { maxBytes: maxBytes ?? undefined });
|
|
323
345
|
if (!result) {
|
|
@@ -354,6 +376,12 @@ export function createMcpServer(options: McpServerOptions = {}): McpServer {
|
|
|
354
376
|
},
|
|
355
377
|
},
|
|
356
378
|
async ({ url, duration }) => {
|
|
379
|
+
if (!options._skipSsrfCheck) {
|
|
380
|
+
const validation = await resolveAndValidateUrl(url);
|
|
381
|
+
if (!validation.safe) {
|
|
382
|
+
return { content: [{ type: 'text' as const, text: `Blocked: ${validation.reason}` }], isError: true };
|
|
383
|
+
}
|
|
384
|
+
}
|
|
357
385
|
const dur = duration ?? 30;
|
|
358
386
|
const timeoutMs = (dur + 60) * 1000; // generous timeout: capture duration + 60s for start/finish
|
|
359
387
|
|
|
@@ -406,6 +434,12 @@ export function createMcpServer(options: McpServerOptions = {}): McpServer {
|
|
|
406
434
|
},
|
|
407
435
|
},
|
|
408
436
|
async ({ url, headless, allDomains }) => {
|
|
437
|
+
if (!options._skipSsrfCheck) {
|
|
438
|
+
const validation = await resolveAndValidateUrl(url);
|
|
439
|
+
if (!validation.safe) {
|
|
440
|
+
return { content: [{ type: 'text' as const, text: `Blocked: ${validation.reason}` }], isError: true };
|
|
441
|
+
}
|
|
442
|
+
}
|
|
409
443
|
if (sessions.size >= MAX_SESSIONS) {
|
|
410
444
|
return {
|
|
411
445
|
content: [{ type: 'text' as const, text: `Maximum ${MAX_SESSIONS} concurrent sessions. Finish or abort an existing session first.` }],
|
package/src/native-host.ts
CHANGED
|
@@ -193,7 +193,11 @@ export async function startSocketServer(
|
|
|
193
193
|
});
|
|
194
194
|
|
|
195
195
|
socketServer.on('error', reject);
|
|
196
|
-
socketServer.listen(socketPath, () =>
|
|
196
|
+
socketServer.listen(socketPath, () => {
|
|
197
|
+
// Restrict socket permissions to owner only
|
|
198
|
+
fs.chmod(socketPath, 0o600).catch(() => {});
|
|
199
|
+
resolve();
|
|
200
|
+
});
|
|
197
201
|
});
|
|
198
202
|
}
|
|
199
203
|
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
// src/read/decoders/deepwiki.ts
|
|
2
2
|
import type { Decoder, ReadResult } from '../types.js';
|
|
3
|
+
import { validateUrl } from '../../skill/ssrf.js';
|
|
3
4
|
|
|
4
5
|
function estimateTokens(text: string): number {
|
|
5
6
|
return Math.ceil(text.length / 4);
|
|
@@ -35,8 +36,15 @@ export const deepwikiDecoder: Decoder = {
|
|
|
35
36
|
const repo = match[3];
|
|
36
37
|
const pagePath = match[4] || '';
|
|
37
38
|
|
|
39
|
+
// SSRF check
|
|
40
|
+
const ssrfResult = validateUrl(url);
|
|
41
|
+
if (!ssrfResult.safe) return null;
|
|
42
|
+
|
|
43
|
+
// Sanitize extracted values for header injection
|
|
44
|
+
const sanitize = (s: string) => s.replace(/[\r\n]/g, '');
|
|
45
|
+
|
|
38
46
|
// Construct the path for the RSC request
|
|
39
|
-
const fullPath = `/${org}/${repo}${pagePath}
|
|
47
|
+
const fullPath = sanitize(`/${org}/${repo}${pagePath}`);
|
|
40
48
|
|
|
41
49
|
try {
|
|
42
50
|
const response = await fetch(url, {
|
package/src/replay/engine.ts
CHANGED
|
@@ -222,11 +222,18 @@ export async function replayEndpoint(
|
|
|
222
222
|
let body: string | undefined;
|
|
223
223
|
const headers = { ...endpoint.headers };
|
|
224
224
|
|
|
225
|
-
// Filter headers from skill file — block dangerous headers
|
|
225
|
+
// Filter headers from skill file — block dangerous headers and sanitize values
|
|
226
|
+
const allowedMethods = new Set(['GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'HEAD', 'OPTIONS']);
|
|
227
|
+
if (!allowedMethods.has(endpoint.method)) {
|
|
228
|
+
throw new Error(`Blocked: unsupported HTTP method "${endpoint.method}"`);
|
|
229
|
+
}
|
|
226
230
|
for (const key of Object.keys(headers)) {
|
|
227
231
|
const lower = key.toLowerCase();
|
|
228
232
|
if (BLOCKED_REPLAY_HEADERS.has(lower) || lower.startsWith('sec-')) {
|
|
229
233
|
delete headers[key];
|
|
234
|
+
} else {
|
|
235
|
+
// Sanitize CRLF from header values to prevent header injection
|
|
236
|
+
headers[key] = headers[key].replace(/[\r\n]/g, '');
|
|
230
237
|
}
|
|
231
238
|
}
|
|
232
239
|
|
|
@@ -381,10 +388,22 @@ export async function replayEndpoint(
|
|
|
381
388
|
throw new Error(`Redirect blocked (SSRF): ${redirectCheck.reason}`);
|
|
382
389
|
}
|
|
383
390
|
}
|
|
391
|
+
// Strip auth headers before cross-domain redirect
|
|
392
|
+
const redirectHeaders = { ...headers };
|
|
393
|
+
const originalHost = url.hostname;
|
|
394
|
+
const redirectHost = redirectUrl.hostname;
|
|
395
|
+
if (redirectHost !== originalHost && !redirectHost.endsWith('.' + originalHost)) {
|
|
396
|
+
delete redirectHeaders['authorization'];
|
|
397
|
+
for (const key of Object.keys(redirectHeaders)) {
|
|
398
|
+
if (key.toLowerCase() === 'authorization' || redirectHeaders[key] === '[stored]') {
|
|
399
|
+
delete redirectHeaders[key];
|
|
400
|
+
}
|
|
401
|
+
}
|
|
402
|
+
}
|
|
384
403
|
// Follow the redirect manually (single hop to prevent chains)
|
|
385
404
|
response = await fetch(redirectFetchUrl, {
|
|
386
405
|
method: 'GET', // Redirects typically become GET
|
|
387
|
-
headers,
|
|
406
|
+
headers: redirectHeaders,
|
|
388
407
|
signal: AbortSignal.timeout(30_000),
|
|
389
408
|
redirect: 'manual', // Prevent chaining
|
|
390
409
|
});
|
package/src/serve.ts
CHANGED
|
@@ -153,7 +153,7 @@ export async function createServeServer(
|
|
|
153
153
|
}],
|
|
154
154
|
};
|
|
155
155
|
} catch (err: any) {
|
|
156
|
-
console.error('Replay failed:', err);
|
|
156
|
+
console.error('Replay failed:', err instanceof Error ? err.message : String(err));
|
|
157
157
|
return {
|
|
158
158
|
content: [{
|
|
159
159
|
type: 'text' as const,
|
package/src/skill/generator.ts
CHANGED
|
@@ -49,6 +49,12 @@ const STRIP_HEADERS = new Set([
|
|
|
49
49
|
const AUTH_HEADERS = new Set([
|
|
50
50
|
'authorization',
|
|
51
51
|
'x-api-key',
|
|
52
|
+
'x-guest-token',
|
|
53
|
+
'x-csrf-token',
|
|
54
|
+
'x-xsrf-token',
|
|
55
|
+
'x-auth-token',
|
|
56
|
+
'x-access-token',
|
|
57
|
+
'x-session-token',
|
|
52
58
|
]);
|
|
53
59
|
|
|
54
60
|
export interface GeneratorOptions {
|
|
@@ -165,25 +171,47 @@ function extractQueryParams(url: URL): Record<string, { type: string; example: s
|
|
|
165
171
|
return params;
|
|
166
172
|
}
|
|
167
173
|
|
|
174
|
+
/** Query param names that carry API keys or tokens */
|
|
175
|
+
const SENSITIVE_QUERY_KEYS = /api.?key|token|secret|credential|key|access.?key/i;
|
|
176
|
+
|
|
168
177
|
function scrubQueryParams(
|
|
169
178
|
params: Record<string, { type: string; example: string }>,
|
|
170
179
|
): Record<string, { type: string; example: string }> {
|
|
171
180
|
const scrubbed: Record<string, { type: string; example: string }> = {};
|
|
172
181
|
for (const [key, val] of Object.entries(params)) {
|
|
173
|
-
|
|
182
|
+
// Scrub known sensitive query param names
|
|
183
|
+
if (SENSITIVE_QUERY_KEYS.test(key)) {
|
|
184
|
+
scrubbed[key] = { type: val.type, example: '[scrubbed]' };
|
|
185
|
+
} else {
|
|
186
|
+
// Also apply entropy-based detection for unknown high-entropy values
|
|
187
|
+
const classification = isLikelyToken(key, val.example);
|
|
188
|
+
if (classification.isToken) {
|
|
189
|
+
scrubbed[key] = { type: val.type, example: '[scrubbed]' };
|
|
190
|
+
} else {
|
|
191
|
+
scrubbed[key] = { type: val.type, example: scrubPII(val.example) };
|
|
192
|
+
}
|
|
193
|
+
}
|
|
174
194
|
}
|
|
175
195
|
return scrubbed;
|
|
176
196
|
}
|
|
177
197
|
|
|
198
|
+
/** Body field names that must always be scrubbed (credentials in POST bodies) */
|
|
199
|
+
const SENSITIVE_BODY_KEYS = /^(password|passwd|pass|secret|client_secret|refresh_token|access_token|api_key|apikey|token|csrf_token|_csrf|xsrf_token|private_key|credential)$/i;
|
|
200
|
+
|
|
178
201
|
function scrubBody(body: unknown, doScrub: boolean): unknown {
|
|
179
202
|
if (!doScrub) return body;
|
|
180
203
|
if (typeof body === 'string') {
|
|
181
204
|
return scrubPII(body);
|
|
182
205
|
}
|
|
206
|
+
if (Array.isArray(body)) {
|
|
207
|
+
return body.map(item => scrubBody(item, doScrub));
|
|
208
|
+
}
|
|
183
209
|
if (body && typeof body === 'object') {
|
|
184
210
|
const scrubbed: Record<string, unknown> = {};
|
|
185
211
|
for (const [key, value] of Object.entries(body as Record<string, unknown>)) {
|
|
186
|
-
if (typeof value === 'string') {
|
|
212
|
+
if (SENSITIVE_BODY_KEYS.test(key) && typeof value === 'string') {
|
|
213
|
+
scrubbed[key] = '[scrubbed]';
|
|
214
|
+
} else if (typeof value === 'string') {
|
|
187
215
|
scrubbed[key] = scrubPII(value);
|
|
188
216
|
} else if (value && typeof value === 'object') {
|
|
189
217
|
scrubbed[key] = scrubBody(value, doScrub);
|
|
@@ -310,8 +338,8 @@ export class SkillGenerator {
|
|
|
310
338
|
let responsePreview: unknown = null;
|
|
311
339
|
if (this.options.enablePreview) {
|
|
312
340
|
const preview = truncatePreview(exchange.response.body);
|
|
313
|
-
responsePreview = this.options.scrub
|
|
314
|
-
?
|
|
341
|
+
responsePreview = this.options.scrub
|
|
342
|
+
? scrubBody(preview, true)
|
|
315
343
|
: preview;
|
|
316
344
|
}
|
|
317
345
|
|
package/src/skill/ssrf.ts
CHANGED
|
@@ -11,7 +11,7 @@ export interface ValidationResult {
|
|
|
11
11
|
}
|
|
12
12
|
|
|
13
13
|
const INTERNAL_HOSTNAMES = ['localhost'];
|
|
14
|
-
const INTERNAL_SUFFIXES = ['.local', '.internal'];
|
|
14
|
+
const INTERNAL_SUFFIXES = ['.local', '.internal', '.localhost', '.corp', '.intranet', '.lan', '.test', '.invalid', '.example'];
|
|
15
15
|
|
|
16
16
|
/**
|
|
17
17
|
* Check if a URL is safe to replay (not targeting internal infrastructure).
|
|
@@ -126,8 +126,17 @@ function isPrivateIp(ip: string): string | null {
|
|
|
126
126
|
const v4mapped = ip.match(/^::ffff:(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})$/i);
|
|
127
127
|
const ipv4 = v4mapped ? v4mapped[1] : ip;
|
|
128
128
|
|
|
129
|
+
// IPv4-mapped IPv6 hex form (e.g. ::ffff:7f00:1)
|
|
130
|
+
const v4mappedHex = ip.match(/^::ffff:([0-9a-f]{1,4}):([0-9a-f]{1,4})$/i);
|
|
131
|
+
if (v4mappedHex) {
|
|
132
|
+
const hi = parseInt(v4mappedHex[1], 16);
|
|
133
|
+
const lo = parseInt(v4mappedHex[2], 16);
|
|
134
|
+
const reconstructed = `${(hi >> 8) & 0xff}.${hi & 0xff}.${(lo >> 8) & 0xff}.${lo & 0xff}`;
|
|
135
|
+
return isPrivateIp(reconstructed);
|
|
136
|
+
}
|
|
137
|
+
|
|
129
138
|
const parts = ipv4.match(/^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/);
|
|
130
|
-
if (!parts) return
|
|
139
|
+
if (!parts) return 'unrecognized IP format'; // Fail closed for unrecognized formats
|
|
131
140
|
|
|
132
141
|
const [, a, b] = parts;
|
|
133
142
|
const first = Number(a);
|
package/src/skill/store.ts
CHANGED
|
@@ -39,7 +39,7 @@ export async function writeSkillFile(
|
|
|
39
39
|
await mkdir(skillsDir, { recursive: true });
|
|
40
40
|
await ensureGitignore(skillsDir);
|
|
41
41
|
const filePath = skillPath(skill.domain, skillsDir);
|
|
42
|
-
await writeFile(filePath, JSON.stringify(skill, null, 2) + '\n');
|
|
42
|
+
await writeFile(filePath, JSON.stringify(skill, null, 2) + '\n', { mode: 0o600 });
|
|
43
43
|
return filePath;
|
|
44
44
|
}
|
|
45
45
|
|
|
@@ -88,9 +88,11 @@ export async function listSkillFiles(
|
|
|
88
88
|
}
|
|
89
89
|
|
|
90
90
|
const summaries: SkillSummary[] = [];
|
|
91
|
+
const DOMAIN_RE = /^[a-zA-Z0-9][a-zA-Z0-9._-]*$/;
|
|
91
92
|
for (const file of files) {
|
|
92
93
|
if (!file.endsWith('.json')) continue;
|
|
93
94
|
const domain = file.replace(/\.json$/, '');
|
|
95
|
+
if (!DOMAIN_RE.test(domain)) continue; // skip non-conforming filenames
|
|
94
96
|
const skill = await readSkillFile(domain, skillsDir);
|
|
95
97
|
if (skill) {
|
|
96
98
|
summaries.push({
|