@apitap/core 1.1.0 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +38 -2
- package/dist/auth/crypto.js +2 -2
- package/dist/auth/crypto.js.map +1 -1
- package/dist/auth/oauth-refresh.d.ts +0 -1
- package/dist/auth/oauth-refresh.js +9 -2
- package/dist/auth/oauth-refresh.js.map +1 -1
- package/dist/bridge/client.d.ts +19 -0
- package/dist/bridge/client.js +52 -0
- package/dist/bridge/client.js.map +1 -0
- package/dist/capture/body-variables.js +3 -0
- package/dist/capture/body-variables.js.map +1 -1
- package/dist/capture/verifier.d.ts +2 -0
- package/dist/capture/verifier.js +18 -4
- package/dist/capture/verifier.js.map +1 -1
- package/dist/cli.js +6 -1
- package/dist/cli.js.map +1 -1
- package/dist/discovery/fetch.js +3 -3
- package/dist/discovery/fetch.js.map +1 -1
- package/dist/mcp.js +37 -3
- package/dist/mcp.js.map +1 -1
- package/dist/native-host.d.ts +5 -1
- package/dist/native-host.js +111 -3
- package/dist/native-host.js.map +1 -1
- package/dist/orchestration/browse.d.ts +5 -1
- package/dist/orchestration/browse.js +109 -0
- package/dist/orchestration/browse.js.map +1 -1
- package/dist/read/decoders/deepwiki.js +8 -1
- package/dist/read/decoders/deepwiki.js.map +1 -1
- package/dist/replay/engine.js +22 -2
- package/dist/replay/engine.js.map +1 -1
- package/dist/serve.js +1 -1
- package/dist/serve.js.map +1 -1
- package/dist/skill/generator.js +33 -4
- package/dist/skill/generator.js.map +1 -1
- package/dist/skill/ssrf.js +10 -2
- package/dist/skill/ssrf.js.map +1 -1
- package/dist/skill/store.js +4 -1
- package/dist/skill/store.js.map +1 -1
- package/package.json +3 -2
- package/src/auth/crypto.ts +2 -2
- package/src/auth/oauth-refresh.ts +7 -3
- package/src/bridge/client.ts +67 -0
- package/src/capture/body-variables.ts +3 -0
- package/src/capture/verifier.ts +20 -2
- package/src/cli.ts +6 -1
- package/src/discovery/fetch.ts +3 -3
- package/src/mcp.ts +37 -3
- package/src/native-host.ts +145 -4
- package/src/orchestration/browse.ts +122 -1
- package/src/read/decoders/deepwiki.ts +9 -1
- package/src/replay/engine.ts +21 -2
- package/src/serve.ts +1 -1
- package/src/skill/generator.ts +32 -4
- package/src/skill/ssrf.ts +11 -2
- package/src/skill/store.ts +3 -1
package/src/cli.ts
CHANGED
|
@@ -512,7 +512,12 @@ async function handleRefresh(positional: string[], flags: Record<string, string
|
|
|
512
512
|
});
|
|
513
513
|
|
|
514
514
|
if (json) {
|
|
515
|
-
|
|
515
|
+
// Redact token values from JSON output to prevent credential leaks in logs
|
|
516
|
+
const safeResult = {
|
|
517
|
+
...result,
|
|
518
|
+
tokens: Object.fromEntries(Object.keys(result.tokens).map(k => [k, '[redacted]'])),
|
|
519
|
+
};
|
|
520
|
+
console.log(JSON.stringify(safeResult, null, 2));
|
|
516
521
|
} else if (result.success) {
|
|
517
522
|
if (result.oauthRefreshed) {
|
|
518
523
|
console.log(` ✓ OAuth token refreshed via token endpoint`);
|
package/src/discovery/fetch.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// src/discovery/fetch.ts
|
|
2
|
-
import { validateUrl } from '../skill/ssrf.js';
|
|
2
|
+
import { validateUrl, resolveAndValidateUrl } from '../skill/ssrf.js';
|
|
3
3
|
|
|
4
4
|
export interface FetchResult {
|
|
5
5
|
status: number;
|
|
@@ -53,12 +53,12 @@ export async function safeFetch(
|
|
|
53
53
|
|
|
54
54
|
clearTimeout(timer);
|
|
55
55
|
|
|
56
|
-
// SSRF-safe manual redirect (one hop max)
|
|
56
|
+
// SSRF-safe manual redirect (one hop max, with DNS resolution check)
|
|
57
57
|
if (response.status >= 300 && response.status < 400 && response.headers.has('location')) {
|
|
58
58
|
const location = response.headers.get('location');
|
|
59
59
|
if (!location) return null;
|
|
60
60
|
const redirectUrl = new URL(location, url).toString();
|
|
61
|
-
const ssrfResult =
|
|
61
|
+
const ssrfResult = await resolveAndValidateUrl(redirectUrl);
|
|
62
62
|
if (!ssrfResult.safe) return null;
|
|
63
63
|
// Follow one redirect hop only
|
|
64
64
|
return await safeFetch(redirectUrl, { ...options, skipSsrf: true });
|
package/src/mcp.ts
CHANGED
|
@@ -103,6 +103,12 @@ export function createMcpServer(options: McpServerOptions = {}): McpServer {
|
|
|
103
103
|
},
|
|
104
104
|
async ({ url }) => {
|
|
105
105
|
try {
|
|
106
|
+
if (!options._skipSsrfCheck) {
|
|
107
|
+
const validation = await resolveAndValidateUrl(url);
|
|
108
|
+
if (!validation.safe) {
|
|
109
|
+
return { content: [{ type: 'text' as const, text: `Blocked: ${validation.reason}` }], isError: true };
|
|
110
|
+
}
|
|
111
|
+
}
|
|
106
112
|
const result = await discover(url);
|
|
107
113
|
|
|
108
114
|
// If we got a skill file, save it automatically
|
|
@@ -250,6 +256,12 @@ export function createMcpServer(options: McpServerOptions = {}): McpServer {
|
|
|
250
256
|
},
|
|
251
257
|
},
|
|
252
258
|
async ({ url, task, maxBytes }) => {
|
|
259
|
+
if (!options._skipSsrfCheck) {
|
|
260
|
+
const validation = await resolveAndValidateUrl(url);
|
|
261
|
+
if (!validation.safe) {
|
|
262
|
+
return { content: [{ type: 'text' as const, text: `Blocked: ${validation.reason}` }], isError: true };
|
|
263
|
+
}
|
|
264
|
+
}
|
|
253
265
|
const { browse: doBrowse } = await import('./orchestration/browse.js');
|
|
254
266
|
const result = await doBrowse(url, {
|
|
255
267
|
skillsDir,
|
|
@@ -257,6 +269,8 @@ export function createMcpServer(options: McpServerOptions = {}): McpServer {
|
|
|
257
269
|
task,
|
|
258
270
|
maxBytes: maxBytes ?? 50_000,
|
|
259
271
|
_skipSsrfCheck: options._skipSsrfCheck,
|
|
272
|
+
// In test mode, disable bridge to avoid connecting to real socket
|
|
273
|
+
...(options._skipSsrfCheck ? { _bridgeSocketPath: '/nonexistent' } : {}),
|
|
260
274
|
});
|
|
261
275
|
// Only mark as untrusted if it contains external data
|
|
262
276
|
if (result.success && result.data) {
|
|
@@ -285,6 +299,12 @@ export function createMcpServer(options: McpServerOptions = {}): McpServer {
|
|
|
285
299
|
},
|
|
286
300
|
async ({ url }) => {
|
|
287
301
|
try {
|
|
302
|
+
if (!options._skipSsrfCheck) {
|
|
303
|
+
const validation = await resolveAndValidateUrl(url);
|
|
304
|
+
if (!validation.safe) {
|
|
305
|
+
return { content: [{ type: 'text' as const, text: `Blocked: ${validation.reason}` }], isError: true };
|
|
306
|
+
}
|
|
307
|
+
}
|
|
288
308
|
const result = await peek(url);
|
|
289
309
|
// Peek returns metadata, not content — but still from external source
|
|
290
310
|
return wrapExternalContent(result, 'apitap_peek');
|
|
@@ -315,9 +335,11 @@ export function createMcpServer(options: McpServerOptions = {}): McpServer {
|
|
|
315
335
|
},
|
|
316
336
|
async ({ url, maxBytes }) => {
|
|
317
337
|
try {
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
338
|
+
if (!options._skipSsrfCheck) {
|
|
339
|
+
const validation = await resolveAndValidateUrl(url);
|
|
340
|
+
if (!validation.safe) {
|
|
341
|
+
throw new Error(validation.reason ?? 'URL validation failed');
|
|
342
|
+
}
|
|
321
343
|
}
|
|
322
344
|
const result = await read(url, { maxBytes: maxBytes ?? undefined });
|
|
323
345
|
if (!result) {
|
|
@@ -354,6 +376,12 @@ export function createMcpServer(options: McpServerOptions = {}): McpServer {
|
|
|
354
376
|
},
|
|
355
377
|
},
|
|
356
378
|
async ({ url, duration }) => {
|
|
379
|
+
if (!options._skipSsrfCheck) {
|
|
380
|
+
const validation = await resolveAndValidateUrl(url);
|
|
381
|
+
if (!validation.safe) {
|
|
382
|
+
return { content: [{ type: 'text' as const, text: `Blocked: ${validation.reason}` }], isError: true };
|
|
383
|
+
}
|
|
384
|
+
}
|
|
357
385
|
const dur = duration ?? 30;
|
|
358
386
|
const timeoutMs = (dur + 60) * 1000; // generous timeout: capture duration + 60s for start/finish
|
|
359
387
|
|
|
@@ -406,6 +434,12 @@ export function createMcpServer(options: McpServerOptions = {}): McpServer {
|
|
|
406
434
|
},
|
|
407
435
|
},
|
|
408
436
|
async ({ url, headless, allDomains }) => {
|
|
437
|
+
if (!options._skipSsrfCheck) {
|
|
438
|
+
const validation = await resolveAndValidateUrl(url);
|
|
439
|
+
if (!validation.safe) {
|
|
440
|
+
return { content: [{ type: 'text' as const, text: `Blocked: ${validation.reason}` }], isError: true };
|
|
441
|
+
}
|
|
442
|
+
}
|
|
409
443
|
if (sessions.size >= MAX_SESSIONS) {
|
|
410
444
|
return {
|
|
411
445
|
content: [{ type: 'text' as const, text: `Maximum ${MAX_SESSIONS} concurrent sessions. Finish or abort an existing session first.` }],
|
package/src/native-host.ts
CHANGED
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
import { promises as fs } from 'node:fs';
|
|
6
6
|
import path from 'node:path';
|
|
7
7
|
import os from 'node:os';
|
|
8
|
+
import net from 'node:net';
|
|
8
9
|
import { signSkillFile } from './skill/signing.js';
|
|
9
10
|
import { deriveKey } from './auth/crypto.js';
|
|
10
11
|
import { getMachineId } from './auth/manager.js';
|
|
@@ -26,7 +27,7 @@ async function signSkillJson(skillJson: string): Promise<string> {
|
|
|
26
27
|
}
|
|
27
28
|
|
|
28
29
|
export interface NativeRequest {
|
|
29
|
-
action: 'save_skill' | 'save_batch' | 'ping';
|
|
30
|
+
action: 'save_skill' | 'save_batch' | 'ping' | 'capture_request';
|
|
30
31
|
domain?: string;
|
|
31
32
|
skillJson?: string;
|
|
32
33
|
skills?: Array<{ domain: string; skillJson: string }>;
|
|
@@ -120,6 +121,94 @@ export async function handleNativeMessage(
|
|
|
120
121
|
}
|
|
121
122
|
}
|
|
122
123
|
|
|
124
|
+
// --- Relay handler ---
|
|
125
|
+
|
|
126
|
+
// Actions handled locally by the native host (filesystem operations)
|
|
127
|
+
const LOCAL_ACTIONS = new Set(['save_skill', 'save_batch', 'ping']);
|
|
128
|
+
|
|
129
|
+
// Actions relayed to the extension (browser operations)
|
|
130
|
+
const EXTENSION_ACTIONS = new Set(['capture_request']);
|
|
131
|
+
|
|
132
|
+
export function createRelayHandler(
|
|
133
|
+
sendToExtension: (msg: any) => Promise<any>,
|
|
134
|
+
skillsDir: string = SKILLS_DIR,
|
|
135
|
+
): MessageHandler {
|
|
136
|
+
return async (message: any) => {
|
|
137
|
+
if (LOCAL_ACTIONS.has(message.action)) {
|
|
138
|
+
return handleNativeMessage(message, skillsDir);
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
if (EXTENSION_ACTIONS.has(message.action)) {
|
|
142
|
+
try {
|
|
143
|
+
return await sendToExtension(message);
|
|
144
|
+
} catch (err) {
|
|
145
|
+
return { success: false, error: String(err) };
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
return { success: false, error: `Unknown action: ${message.action}` };
|
|
150
|
+
};
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
// --- Unix socket server for CLI relay ---
|
|
154
|
+
|
|
155
|
+
export type MessageHandler = (message: any) => Promise<any>;
|
|
156
|
+
|
|
157
|
+
let socketServer: net.Server | null = null;
|
|
158
|
+
|
|
159
|
+
export async function startSocketServer(
|
|
160
|
+
socketPath: string,
|
|
161
|
+
handler: MessageHandler,
|
|
162
|
+
): Promise<void> {
|
|
163
|
+
// Clean up stale socket
|
|
164
|
+
try { await fs.unlink(socketPath); } catch { /* doesn't exist — fine */ }
|
|
165
|
+
|
|
166
|
+
return new Promise((resolve, reject) => {
|
|
167
|
+
socketServer = net.createServer((conn) => {
|
|
168
|
+
let buffer = '';
|
|
169
|
+
|
|
170
|
+
conn.on('data', (chunk) => {
|
|
171
|
+
buffer += chunk.toString();
|
|
172
|
+
const newlineIdx = buffer.indexOf('\n');
|
|
173
|
+
if (newlineIdx === -1) return;
|
|
174
|
+
|
|
175
|
+
const line = buffer.slice(0, newlineIdx);
|
|
176
|
+
buffer = buffer.slice(newlineIdx + 1);
|
|
177
|
+
|
|
178
|
+
let request: any;
|
|
179
|
+
try {
|
|
180
|
+
request = JSON.parse(line);
|
|
181
|
+
} catch {
|
|
182
|
+
conn.end(JSON.stringify({ success: false, error: 'Invalid JSON' }) + '\n');
|
|
183
|
+
return;
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
handler(request).then(
|
|
187
|
+
(response) => conn.end(JSON.stringify(response) + '\n'),
|
|
188
|
+
(err) => conn.end(JSON.stringify({ success: false, error: String(err) }) + '\n'),
|
|
189
|
+
);
|
|
190
|
+
});
|
|
191
|
+
|
|
192
|
+
conn.on('error', () => { /* client disconnect — ignore */ });
|
|
193
|
+
});
|
|
194
|
+
|
|
195
|
+
socketServer.on('error', reject);
|
|
196
|
+
socketServer.listen(socketPath, () => {
|
|
197
|
+
// Restrict socket permissions to owner only
|
|
198
|
+
fs.chmod(socketPath, 0o600).catch(() => {});
|
|
199
|
+
resolve();
|
|
200
|
+
});
|
|
201
|
+
});
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
export async function stopSocketServer(): Promise<void> {
|
|
205
|
+
if (!socketServer) return;
|
|
206
|
+
return new Promise((resolve) => {
|
|
207
|
+
socketServer!.close(() => resolve());
|
|
208
|
+
socketServer = null;
|
|
209
|
+
});
|
|
210
|
+
}
|
|
211
|
+
|
|
123
212
|
// --- stdio framing (only runs when executed directly, not when imported for tests) ---
|
|
124
213
|
|
|
125
214
|
function readMessage(): Promise<NativeRequest | null> {
|
|
@@ -206,12 +295,64 @@ const isMainModule = process.argv[1] &&
|
|
|
206
295
|
(process.argv[1].endsWith('native-host.ts') || process.argv[1].endsWith('native-host.js'));
|
|
207
296
|
|
|
208
297
|
if (isMainModule) {
|
|
298
|
+
const bridgeDir = path.join(os.homedir(), '.apitap');
|
|
299
|
+
const socketPath = path.join(bridgeDir, 'bridge.sock');
|
|
300
|
+
|
|
301
|
+
// Pending CLI requests waiting for extension responses
|
|
302
|
+
const pendingRequests = new Map<string, {
|
|
303
|
+
resolve: (value: any) => void;
|
|
304
|
+
timer: ReturnType<typeof setTimeout>;
|
|
305
|
+
}>();
|
|
306
|
+
let requestCounter = 0;
|
|
307
|
+
|
|
308
|
+
// Send a message to the extension via stdout and wait for response
|
|
309
|
+
function sendToExtension(message: any): Promise<any> {
|
|
310
|
+
return new Promise((resolve) => {
|
|
311
|
+
const id = String(++requestCounter);
|
|
312
|
+
const timer = setTimeout(() => {
|
|
313
|
+
pendingRequests.delete(id);
|
|
314
|
+
resolve({ success: false, error: 'approval_timeout' });
|
|
315
|
+
}, 60_000);
|
|
316
|
+
|
|
317
|
+
pendingRequests.set(id, { resolve, timer });
|
|
318
|
+
|
|
319
|
+
// Tag message with ID so we can match the response
|
|
320
|
+
sendMessage({ ...message, _relayId: id } as any);
|
|
321
|
+
});
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
const handler = createRelayHandler(sendToExtension);
|
|
325
|
+
|
|
209
326
|
(async () => {
|
|
327
|
+
// Ensure bridge directory exists
|
|
328
|
+
await fs.mkdir(bridgeDir, { recursive: true });
|
|
329
|
+
|
|
330
|
+
// Start socket server for CLI connections
|
|
331
|
+
await startSocketServer(socketPath, handler);
|
|
332
|
+
|
|
333
|
+
// Read messages from extension via stdin
|
|
210
334
|
while (true) {
|
|
211
|
-
const
|
|
212
|
-
if (!
|
|
213
|
-
|
|
335
|
+
const message = await readMessage();
|
|
336
|
+
if (!message) break;
|
|
337
|
+
|
|
338
|
+
// Check if this is a response to a relayed request
|
|
339
|
+
const relayId = (message as any)._relayId;
|
|
340
|
+
if (relayId && pendingRequests.has(relayId)) {
|
|
341
|
+
const pending = pendingRequests.get(relayId)!;
|
|
342
|
+
clearTimeout(pending.timer);
|
|
343
|
+
pendingRequests.delete(relayId);
|
|
344
|
+
const { _relayId, ...response } = message as any;
|
|
345
|
+
pending.resolve(response);
|
|
346
|
+
continue;
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
// Otherwise, handle as a direct extension message (save_skill, etc.)
|
|
350
|
+
const response = await handleNativeMessage(message);
|
|
214
351
|
sendMessage(response);
|
|
215
352
|
}
|
|
353
|
+
|
|
354
|
+
// Extension disconnected — clean up
|
|
355
|
+
await stopSocketServer();
|
|
356
|
+
try { await fs.unlink(socketPath); } catch { /* already gone */ }
|
|
216
357
|
})();
|
|
217
358
|
}
|
|
@@ -3,6 +3,7 @@ import { readSkillFile } from '../skill/store.js';
|
|
|
3
3
|
import { replayEndpoint } from '../replay/engine.js';
|
|
4
4
|
import { SessionCache } from './cache.js';
|
|
5
5
|
import { read } from '../read/index.js';
|
|
6
|
+
import { bridgeAvailable, requestBridgeCapture, DEFAULT_SOCKET } from '../bridge/client.js';
|
|
6
7
|
|
|
7
8
|
export interface BrowseOptions {
|
|
8
9
|
skillsDir?: string;
|
|
@@ -13,6 +14,10 @@ export interface BrowseOptions {
|
|
|
13
14
|
maxBytes?: number;
|
|
14
15
|
/** @internal Skip SSRF check — for testing only */
|
|
15
16
|
_skipSsrfCheck?: boolean;
|
|
17
|
+
/** @internal Override bridge socket path — for testing only */
|
|
18
|
+
_bridgeSocketPath?: string;
|
|
19
|
+
/** @internal Override bridge timeout — for testing only */
|
|
20
|
+
_bridgeTimeout?: number;
|
|
16
21
|
}
|
|
17
22
|
|
|
18
23
|
export interface BrowseSuccess {
|
|
@@ -22,7 +27,7 @@ export interface BrowseSuccess {
|
|
|
22
27
|
domain: string;
|
|
23
28
|
endpointId: string;
|
|
24
29
|
tier: string;
|
|
25
|
-
skillSource: 'disk' | 'discovered' | 'captured';
|
|
30
|
+
skillSource: 'disk' | 'discovered' | 'captured' | 'bridge';
|
|
26
31
|
capturedAt: string;
|
|
27
32
|
task?: string;
|
|
28
33
|
truncated?: boolean;
|
|
@@ -40,6 +45,106 @@ export interface BrowseGuidance {
|
|
|
40
45
|
|
|
41
46
|
export type BrowseResult = BrowseSuccess | BrowseGuidance;
|
|
42
47
|
|
|
48
|
+
/**
|
|
49
|
+
* Try escalating to the Chrome extension bridge for authenticated capture.
|
|
50
|
+
* Returns a BrowseResult if the bridge handled it, or null to fall through.
|
|
51
|
+
*/
|
|
52
|
+
async function tryBridgeCapture(
|
|
53
|
+
domain: string,
|
|
54
|
+
fullUrl: string,
|
|
55
|
+
options: BrowseOptions,
|
|
56
|
+
): Promise<BrowseResult | null> {
|
|
57
|
+
const socketPath = options._bridgeSocketPath ?? DEFAULT_SOCKET;
|
|
58
|
+
if (!await bridgeAvailable(socketPath)) return null;
|
|
59
|
+
|
|
60
|
+
const result = await requestBridgeCapture(domain, socketPath, { timeout: options._bridgeTimeout });
|
|
61
|
+
|
|
62
|
+
if (result.success && result.skillFiles && result.skillFiles.length > 0) {
|
|
63
|
+
const skillFiles = result.skillFiles;
|
|
64
|
+
// Save each skill file to disk
|
|
65
|
+
try {
|
|
66
|
+
const { writeSkillFile: writeSF } = await import('../skill/store.js');
|
|
67
|
+
for (const skill of skillFiles) {
|
|
68
|
+
await writeSF(skill, options.skillsDir);
|
|
69
|
+
}
|
|
70
|
+
} catch {
|
|
71
|
+
// Saving failed — still have the data in memory
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
// Find the skill file matching the requested domain
|
|
75
|
+
const primarySkill = skillFiles.find((s: any) => s.domain === domain)
|
|
76
|
+
?? skillFiles[0];
|
|
77
|
+
|
|
78
|
+
if (primarySkill?.endpoints?.length > 0) {
|
|
79
|
+
// Pick the best endpoint and replay it
|
|
80
|
+
let urlPath = '/';
|
|
81
|
+
try { urlPath = new URL(fullUrl).pathname; } catch { /* use default */ }
|
|
82
|
+
const endpoint = pickEndpoint(primarySkill, urlPath);
|
|
83
|
+
|
|
84
|
+
if (endpoint) {
|
|
85
|
+
try {
|
|
86
|
+
const replayResult = await replayEndpoint(primarySkill, endpoint.id, {
|
|
87
|
+
maxBytes: options.maxBytes,
|
|
88
|
+
_skipSsrfCheck: options._skipSsrfCheck,
|
|
89
|
+
});
|
|
90
|
+
if (replayResult.status >= 200 && replayResult.status < 300) {
|
|
91
|
+
return {
|
|
92
|
+
success: true,
|
|
93
|
+
data: replayResult.data,
|
|
94
|
+
status: replayResult.status,
|
|
95
|
+
domain,
|
|
96
|
+
endpointId: endpoint.id,
|
|
97
|
+
tier: endpoint.replayability?.tier ?? 'unknown',
|
|
98
|
+
skillSource: 'bridge',
|
|
99
|
+
capturedAt: primarySkill.capturedAt ?? new Date().toISOString(),
|
|
100
|
+
task: options.task,
|
|
101
|
+
...(replayResult.truncated ? { truncated: true } : {}),
|
|
102
|
+
};
|
|
103
|
+
}
|
|
104
|
+
} catch {
|
|
105
|
+
// Replay failed — but skill file is saved for next time
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
// Skill file saved but replay didn't work
|
|
111
|
+
return {
|
|
112
|
+
success: false,
|
|
113
|
+
reason: 'bridge_capture_saved',
|
|
114
|
+
suggestion: `Captured ${skillFiles.length} skill file(s) from browser. Replay failed — try 'apitap replay ${domain}'.`,
|
|
115
|
+
domain,
|
|
116
|
+
url: fullUrl,
|
|
117
|
+
task: options.task,
|
|
118
|
+
};
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
// Bridge returned an error
|
|
122
|
+
if (result.error === 'user_denied') {
|
|
123
|
+
return {
|
|
124
|
+
success: false,
|
|
125
|
+
reason: 'user_denied',
|
|
126
|
+
suggestion: `User denied browser access to ${domain}. Use 'apitap auth request ${domain}' for manual login instead.`,
|
|
127
|
+
domain,
|
|
128
|
+
url: fullUrl,
|
|
129
|
+
task: options.task,
|
|
130
|
+
};
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
if (result.error === 'approval_timeout') {
|
|
134
|
+
return {
|
|
135
|
+
success: false,
|
|
136
|
+
reason: 'approval_timeout',
|
|
137
|
+
suggestion: `User approval pending for ${domain}. Click Allow in the ApiTap extension and try again.`,
|
|
138
|
+
domain,
|
|
139
|
+
url: fullUrl,
|
|
140
|
+
task: options.task,
|
|
141
|
+
};
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
// Other bridge errors — fall through to existing fallback
|
|
145
|
+
return null;
|
|
146
|
+
}
|
|
147
|
+
|
|
43
148
|
/**
|
|
44
149
|
* High-level browse: check cache → disk → discover → replay.
|
|
45
150
|
* Auto-escalates cheap steps. Returns guidance for expensive ones.
|
|
@@ -121,6 +226,10 @@ export async function browse(
|
|
|
121
226
|
} catch {
|
|
122
227
|
// Read failed — fall through to capture_needed
|
|
123
228
|
}
|
|
229
|
+
// Try extension bridge before giving up
|
|
230
|
+
const bridgeResult1 = await tryBridgeCapture(domain, fullUrl, options);
|
|
231
|
+
if (bridgeResult1) return bridgeResult1;
|
|
232
|
+
|
|
124
233
|
return {
|
|
125
234
|
success: false,
|
|
126
235
|
reason: 'no_replayable_endpoints',
|
|
@@ -158,6 +267,10 @@ export async function browse(
|
|
|
158
267
|
// Read failed — fall through to capture_needed
|
|
159
268
|
}
|
|
160
269
|
}
|
|
270
|
+
// Try extension bridge before giving up
|
|
271
|
+
const bridgeResult2 = await tryBridgeCapture(domain, fullUrl, options);
|
|
272
|
+
if (bridgeResult2) return bridgeResult2;
|
|
273
|
+
|
|
161
274
|
return {
|
|
162
275
|
success: false,
|
|
163
276
|
reason: 'no_skill_file',
|
|
@@ -171,6 +284,10 @@ export async function browse(
|
|
|
171
284
|
// Step 4: Pick best endpoint
|
|
172
285
|
const endpoint = pickEndpoint(skill, urlPath);
|
|
173
286
|
if (!endpoint) {
|
|
287
|
+
// Try extension bridge before giving up
|
|
288
|
+
const bridgeResult3 = await tryBridgeCapture(domain, fullUrl, options);
|
|
289
|
+
if (bridgeResult3) return bridgeResult3;
|
|
290
|
+
|
|
174
291
|
return {
|
|
175
292
|
success: false,
|
|
176
293
|
reason: 'no_replayable_endpoints',
|
|
@@ -213,6 +330,10 @@ export async function browse(
|
|
|
213
330
|
...(result.truncated ? { truncated: true } : {}),
|
|
214
331
|
};
|
|
215
332
|
} catch {
|
|
333
|
+
// Try extension bridge before giving up
|
|
334
|
+
const bridgeResult4 = await tryBridgeCapture(domain, fullUrl, options);
|
|
335
|
+
if (bridgeResult4) return bridgeResult4;
|
|
336
|
+
|
|
216
337
|
return {
|
|
217
338
|
success: false,
|
|
218
339
|
reason: 'replay_failed',
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
// src/read/decoders/deepwiki.ts
|
|
2
2
|
import type { Decoder, ReadResult } from '../types.js';
|
|
3
|
+
import { validateUrl } from '../../skill/ssrf.js';
|
|
3
4
|
|
|
4
5
|
function estimateTokens(text: string): number {
|
|
5
6
|
return Math.ceil(text.length / 4);
|
|
@@ -35,8 +36,15 @@ export const deepwikiDecoder: Decoder = {
|
|
|
35
36
|
const repo = match[3];
|
|
36
37
|
const pagePath = match[4] || '';
|
|
37
38
|
|
|
39
|
+
// SSRF check
|
|
40
|
+
const ssrfResult = validateUrl(url);
|
|
41
|
+
if (!ssrfResult.safe) return null;
|
|
42
|
+
|
|
43
|
+
// Sanitize extracted values for header injection
|
|
44
|
+
const sanitize = (s: string) => s.replace(/[\r\n]/g, '');
|
|
45
|
+
|
|
38
46
|
// Construct the path for the RSC request
|
|
39
|
-
const fullPath = `/${org}/${repo}${pagePath}
|
|
47
|
+
const fullPath = sanitize(`/${org}/${repo}${pagePath}`);
|
|
40
48
|
|
|
41
49
|
try {
|
|
42
50
|
const response = await fetch(url, {
|
package/src/replay/engine.ts
CHANGED
|
@@ -222,11 +222,18 @@ export async function replayEndpoint(
|
|
|
222
222
|
let body: string | undefined;
|
|
223
223
|
const headers = { ...endpoint.headers };
|
|
224
224
|
|
|
225
|
-
// Filter headers from skill file — block dangerous headers
|
|
225
|
+
// Filter headers from skill file — block dangerous headers and sanitize values
|
|
226
|
+
const allowedMethods = new Set(['GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'HEAD', 'OPTIONS']);
|
|
227
|
+
if (!allowedMethods.has(endpoint.method)) {
|
|
228
|
+
throw new Error(`Blocked: unsupported HTTP method "${endpoint.method}"`);
|
|
229
|
+
}
|
|
226
230
|
for (const key of Object.keys(headers)) {
|
|
227
231
|
const lower = key.toLowerCase();
|
|
228
232
|
if (BLOCKED_REPLAY_HEADERS.has(lower) || lower.startsWith('sec-')) {
|
|
229
233
|
delete headers[key];
|
|
234
|
+
} else {
|
|
235
|
+
// Sanitize CRLF from header values to prevent header injection
|
|
236
|
+
headers[key] = headers[key].replace(/[\r\n]/g, '');
|
|
230
237
|
}
|
|
231
238
|
}
|
|
232
239
|
|
|
@@ -381,10 +388,22 @@ export async function replayEndpoint(
|
|
|
381
388
|
throw new Error(`Redirect blocked (SSRF): ${redirectCheck.reason}`);
|
|
382
389
|
}
|
|
383
390
|
}
|
|
391
|
+
// Strip auth headers before cross-domain redirect
|
|
392
|
+
const redirectHeaders = { ...headers };
|
|
393
|
+
const originalHost = url.hostname;
|
|
394
|
+
const redirectHost = redirectUrl.hostname;
|
|
395
|
+
if (redirectHost !== originalHost && !redirectHost.endsWith('.' + originalHost)) {
|
|
396
|
+
delete redirectHeaders['authorization'];
|
|
397
|
+
for (const key of Object.keys(redirectHeaders)) {
|
|
398
|
+
if (key.toLowerCase() === 'authorization' || redirectHeaders[key] === '[stored]') {
|
|
399
|
+
delete redirectHeaders[key];
|
|
400
|
+
}
|
|
401
|
+
}
|
|
402
|
+
}
|
|
384
403
|
// Follow the redirect manually (single hop to prevent chains)
|
|
385
404
|
response = await fetch(redirectFetchUrl, {
|
|
386
405
|
method: 'GET', // Redirects typically become GET
|
|
387
|
-
headers,
|
|
406
|
+
headers: redirectHeaders,
|
|
388
407
|
signal: AbortSignal.timeout(30_000),
|
|
389
408
|
redirect: 'manual', // Prevent chaining
|
|
390
409
|
});
|
package/src/serve.ts
CHANGED
|
@@ -153,7 +153,7 @@ export async function createServeServer(
|
|
|
153
153
|
}],
|
|
154
154
|
};
|
|
155
155
|
} catch (err: any) {
|
|
156
|
-
console.error('Replay failed:', err);
|
|
156
|
+
console.error('Replay failed:', err instanceof Error ? err.message : String(err));
|
|
157
157
|
return {
|
|
158
158
|
content: [{
|
|
159
159
|
type: 'text' as const,
|
package/src/skill/generator.ts
CHANGED
|
@@ -49,6 +49,12 @@ const STRIP_HEADERS = new Set([
|
|
|
49
49
|
const AUTH_HEADERS = new Set([
|
|
50
50
|
'authorization',
|
|
51
51
|
'x-api-key',
|
|
52
|
+
'x-guest-token',
|
|
53
|
+
'x-csrf-token',
|
|
54
|
+
'x-xsrf-token',
|
|
55
|
+
'x-auth-token',
|
|
56
|
+
'x-access-token',
|
|
57
|
+
'x-session-token',
|
|
52
58
|
]);
|
|
53
59
|
|
|
54
60
|
export interface GeneratorOptions {
|
|
@@ -165,25 +171,47 @@ function extractQueryParams(url: URL): Record<string, { type: string; example: s
|
|
|
165
171
|
return params;
|
|
166
172
|
}
|
|
167
173
|
|
|
174
|
+
/** Query param names that carry API keys or tokens */
|
|
175
|
+
const SENSITIVE_QUERY_KEYS = /api.?key|token|secret|credential|key|access.?key/i;
|
|
176
|
+
|
|
168
177
|
function scrubQueryParams(
|
|
169
178
|
params: Record<string, { type: string; example: string }>,
|
|
170
179
|
): Record<string, { type: string; example: string }> {
|
|
171
180
|
const scrubbed: Record<string, { type: string; example: string }> = {};
|
|
172
181
|
for (const [key, val] of Object.entries(params)) {
|
|
173
|
-
|
|
182
|
+
// Scrub known sensitive query param names
|
|
183
|
+
if (SENSITIVE_QUERY_KEYS.test(key)) {
|
|
184
|
+
scrubbed[key] = { type: val.type, example: '[scrubbed]' };
|
|
185
|
+
} else {
|
|
186
|
+
// Also apply entropy-based detection for unknown high-entropy values
|
|
187
|
+
const classification = isLikelyToken(key, val.example);
|
|
188
|
+
if (classification.isToken) {
|
|
189
|
+
scrubbed[key] = { type: val.type, example: '[scrubbed]' };
|
|
190
|
+
} else {
|
|
191
|
+
scrubbed[key] = { type: val.type, example: scrubPII(val.example) };
|
|
192
|
+
}
|
|
193
|
+
}
|
|
174
194
|
}
|
|
175
195
|
return scrubbed;
|
|
176
196
|
}
|
|
177
197
|
|
|
198
|
+
/** Body field names that must always be scrubbed (credentials in POST bodies) */
|
|
199
|
+
const SENSITIVE_BODY_KEYS = /^(password|passwd|pass|secret|client_secret|refresh_token|access_token|api_key|apikey|token|csrf_token|_csrf|xsrf_token|private_key|credential)$/i;
|
|
200
|
+
|
|
178
201
|
function scrubBody(body: unknown, doScrub: boolean): unknown {
|
|
179
202
|
if (!doScrub) return body;
|
|
180
203
|
if (typeof body === 'string') {
|
|
181
204
|
return scrubPII(body);
|
|
182
205
|
}
|
|
206
|
+
if (Array.isArray(body)) {
|
|
207
|
+
return body.map(item => scrubBody(item, doScrub));
|
|
208
|
+
}
|
|
183
209
|
if (body && typeof body === 'object') {
|
|
184
210
|
const scrubbed: Record<string, unknown> = {};
|
|
185
211
|
for (const [key, value] of Object.entries(body as Record<string, unknown>)) {
|
|
186
|
-
if (typeof value === 'string') {
|
|
212
|
+
if (SENSITIVE_BODY_KEYS.test(key) && typeof value === 'string') {
|
|
213
|
+
scrubbed[key] = '[scrubbed]';
|
|
214
|
+
} else if (typeof value === 'string') {
|
|
187
215
|
scrubbed[key] = scrubPII(value);
|
|
188
216
|
} else if (value && typeof value === 'object') {
|
|
189
217
|
scrubbed[key] = scrubBody(value, doScrub);
|
|
@@ -310,8 +338,8 @@ export class SkillGenerator {
|
|
|
310
338
|
let responsePreview: unknown = null;
|
|
311
339
|
if (this.options.enablePreview) {
|
|
312
340
|
const preview = truncatePreview(exchange.response.body);
|
|
313
|
-
responsePreview = this.options.scrub
|
|
314
|
-
?
|
|
341
|
+
responsePreview = this.options.scrub
|
|
342
|
+
? scrubBody(preview, true)
|
|
315
343
|
: preview;
|
|
316
344
|
}
|
|
317
345
|
|
package/src/skill/ssrf.ts
CHANGED
|
@@ -11,7 +11,7 @@ export interface ValidationResult {
|
|
|
11
11
|
}
|
|
12
12
|
|
|
13
13
|
const INTERNAL_HOSTNAMES = ['localhost'];
|
|
14
|
-
const INTERNAL_SUFFIXES = ['.local', '.internal'];
|
|
14
|
+
const INTERNAL_SUFFIXES = ['.local', '.internal', '.localhost', '.corp', '.intranet', '.lan', '.test', '.invalid', '.example'];
|
|
15
15
|
|
|
16
16
|
/**
|
|
17
17
|
* Check if a URL is safe to replay (not targeting internal infrastructure).
|
|
@@ -126,8 +126,17 @@ function isPrivateIp(ip: string): string | null {
|
|
|
126
126
|
const v4mapped = ip.match(/^::ffff:(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})$/i);
|
|
127
127
|
const ipv4 = v4mapped ? v4mapped[1] : ip;
|
|
128
128
|
|
|
129
|
+
// IPv4-mapped IPv6 hex form (e.g. ::ffff:7f00:1)
|
|
130
|
+
const v4mappedHex = ip.match(/^::ffff:([0-9a-f]{1,4}):([0-9a-f]{1,4})$/i);
|
|
131
|
+
if (v4mappedHex) {
|
|
132
|
+
const hi = parseInt(v4mappedHex[1], 16);
|
|
133
|
+
const lo = parseInt(v4mappedHex[2], 16);
|
|
134
|
+
const reconstructed = `${(hi >> 8) & 0xff}.${hi & 0xff}.${(lo >> 8) & 0xff}.${lo & 0xff}`;
|
|
135
|
+
return isPrivateIp(reconstructed);
|
|
136
|
+
}
|
|
137
|
+
|
|
129
138
|
const parts = ipv4.match(/^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/);
|
|
130
|
-
if (!parts) return
|
|
139
|
+
if (!parts) return 'unrecognized IP format'; // Fail closed for unrecognized formats
|
|
131
140
|
|
|
132
141
|
const [, a, b] = parts;
|
|
133
142
|
const first = Number(a);
|
package/src/skill/store.ts
CHANGED
|
@@ -39,7 +39,7 @@ export async function writeSkillFile(
|
|
|
39
39
|
await mkdir(skillsDir, { recursive: true });
|
|
40
40
|
await ensureGitignore(skillsDir);
|
|
41
41
|
const filePath = skillPath(skill.domain, skillsDir);
|
|
42
|
-
await writeFile(filePath, JSON.stringify(skill, null, 2) + '\n');
|
|
42
|
+
await writeFile(filePath, JSON.stringify(skill, null, 2) + '\n', { mode: 0o600 });
|
|
43
43
|
return filePath;
|
|
44
44
|
}
|
|
45
45
|
|
|
@@ -88,9 +88,11 @@ export async function listSkillFiles(
|
|
|
88
88
|
}
|
|
89
89
|
|
|
90
90
|
const summaries: SkillSummary[] = [];
|
|
91
|
+
const DOMAIN_RE = /^[a-zA-Z0-9][a-zA-Z0-9._-]*$/;
|
|
91
92
|
for (const file of files) {
|
|
92
93
|
if (!file.endsWith('.json')) continue;
|
|
93
94
|
const domain = file.replace(/\.json$/, '');
|
|
95
|
+
if (!DOMAIN_RE.test(domain)) continue; // skip non-conforming filenames
|
|
94
96
|
const skill = await readSkillFile(domain, skillsDir);
|
|
95
97
|
if (skill) {
|
|
96
98
|
summaries.push({
|