@apitap/core 1.0.6 → 1.0.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -3
- package/dist/auth/crypto.d.ts +1 -1
- package/dist/auth/crypto.js +27 -4
- package/dist/auth/crypto.js.map +1 -1
- package/dist/auth/manager.d.ts +1 -1
- package/dist/auth/manager.js +2 -2
- package/dist/auth/manager.js.map +1 -1
- package/dist/discovery/fetch.js +13 -1
- package/dist/discovery/fetch.js.map +1 -1
- package/package.json +1 -1
- package/src/auth/crypto.ts +27 -4
- package/src/auth/handoff.ts +10 -7
- package/src/auth/manager.ts +39 -2
- package/src/auth/oauth-refresh.ts +13 -1
- package/src/auth/refresh.ts +3 -5
- package/src/capture/browser.ts +51 -0
- package/src/capture/monitor.ts +9 -5
- package/src/capture/oauth-detector.ts +34 -1
- package/src/capture/session.ts +11 -6
- package/src/cli.ts +6 -2
- package/src/discovery/fetch.ts +12 -1
- package/src/skill/generator.ts +7 -0
package/README.md
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
# ApiTap
|
|
2
2
|
|
|
3
3
|
[](https://www.npmjs.com/package/@apitap/core)
|
|
4
|
-
[](https://github.com/n1byn1kt/apitap)
|
|
5
5
|
[](./LICENSE)
|
|
6
6
|
|
|
7
7
|
**The MCP server that turns any website into an API — no docs, no SDK, no browser.**
|
|
@@ -28,6 +28,8 @@ apitap replay gamma-api.polymarket.com get-events # Call the API directly
|
|
|
28
28
|
|
|
29
29
|
No scraping. No browser. Just the API.
|
|
30
30
|
|
|
31
|
+

|
|
32
|
+
|
|
31
33
|
---
|
|
32
34
|
|
|
33
35
|
## How It Works
|
|
@@ -332,7 +334,7 @@ Endpoint replayed
|
|
|
332
334
|
|
|
333
335
|
This is especially relevant now that [MCP servers are being used as attack vectors in the wild](https://cloud.google.com/blog/topics/threat-intelligence/distillation-experimentation-integration-ai-adversarial-use) — Google's Threat Intelligence Group recently documented underground toolkits built on compromised MCP servers. ApiTap is designed to be safe even when processing untrusted inputs.
|
|
334
336
|
|
|
335
|
-
|
|
337
|
+
|
|
336
338
|
|
|
337
339
|
## CLI Reference
|
|
338
340
|
|
|
@@ -376,7 +378,7 @@ All commands support `--json` for machine-readable output.
|
|
|
376
378
|
git clone https://github.com/n1byn1kt/apitap.git
|
|
377
379
|
cd apitap
|
|
378
380
|
npm install
|
|
379
|
-
npm test #
|
|
381
|
+
npm test # 789 tests, Node built-in test runner
|
|
380
382
|
npm run typecheck # Type checking
|
|
381
383
|
npm run build # Compile to dist/
|
|
382
384
|
npx tsx src/cli.ts capture <url> # Run from source
|
package/dist/auth/crypto.d.ts
CHANGED
|
@@ -9,7 +9,7 @@ export interface EncryptedData {
|
|
|
9
9
|
* Uses a fixed application salt — the entropy comes from the machine ID
|
|
10
10
|
* being stretched through 100K iterations.
|
|
11
11
|
*/
|
|
12
|
-
export declare function deriveKey(machineId: string): Buffer;
|
|
12
|
+
export declare function deriveKey(machineId: string, saltFile?: string): Buffer;
|
|
13
13
|
/**
|
|
14
14
|
* Encrypt plaintext using AES-256-GCM.
|
|
15
15
|
* Each call generates a random IV for semantic security.
|
package/dist/auth/crypto.js
CHANGED
|
@@ -4,14 +4,37 @@ const ALGORITHM = 'aes-256-gcm';
|
|
|
4
4
|
const KEY_LENGTH = 32; // 256 bits
|
|
5
5
|
const IV_LENGTH = 16;
|
|
6
6
|
const PBKDF2_ITERATIONS = 100_000;
|
|
7
|
-
const PBKDF2_SALT = 'apitap-v0.2-key-derivation';
|
|
7
|
+
// const PBKDF2_SALT = 'apitap-v0.2-key-derivation'; // fallback for migration
|
|
8
|
+
import { readFileSync, writeFileSync, existsSync, mkdirSync } from 'node:fs';
|
|
9
|
+
import { homedir } from 'node:os';
|
|
10
|
+
function getInstallSalt(saltFile) {
|
|
11
|
+
const saltPath = saltFile || `${homedir()}/.apitap/install-salt`;
|
|
12
|
+
if (existsSync(saltPath)) {
|
|
13
|
+
return readFileSync(saltPath, 'utf8').trim();
|
|
14
|
+
}
|
|
15
|
+
// Generate and save new salt
|
|
16
|
+
const salt = randomBytes(32).toString('hex');
|
|
17
|
+
try {
|
|
18
|
+
mkdirSync(saltPath.replace(/\/[^/]+$/, ''), { recursive: true });
|
|
19
|
+
writeFileSync(saltPath, salt, { mode: 0o600 });
|
|
20
|
+
}
|
|
21
|
+
catch { }
|
|
22
|
+
return salt;
|
|
23
|
+
}
|
|
8
24
|
/**
|
|
9
25
|
* Derive a 256-bit key from a machine identifier using PBKDF2.
|
|
10
26
|
* Uses a fixed application salt — the entropy comes from the machine ID
|
|
11
27
|
* being stretched through 100K iterations.
|
|
12
28
|
*/
|
|
13
|
-
export function deriveKey(machineId) {
|
|
14
|
-
|
|
29
|
+
export function deriveKey(machineId, saltFile) {
|
|
30
|
+
// Try per-install salt first, fallback to old constant for migration
|
|
31
|
+
try {
|
|
32
|
+
return pbkdf2Sync(machineId, getInstallSalt(saltFile), PBKDF2_ITERATIONS, KEY_LENGTH, 'sha512');
|
|
33
|
+
}
|
|
34
|
+
catch {
|
|
35
|
+
// Fallback for old installs (migration note: remove after all users migrated)
|
|
36
|
+
return pbkdf2Sync(machineId, 'apitap-v0.2-key-derivation', PBKDF2_ITERATIONS, KEY_LENGTH, 'sha512');
|
|
37
|
+
}
|
|
15
38
|
}
|
|
16
39
|
/**
|
|
17
40
|
* Encrypt plaintext using AES-256-GCM.
|
|
@@ -24,7 +47,7 @@ export function encrypt(plaintext, key) {
|
|
|
24
47
|
ciphertext += cipher.final('hex');
|
|
25
48
|
const tag = cipher.getAuthTag();
|
|
26
49
|
return {
|
|
27
|
-
salt:
|
|
50
|
+
salt: 'apitap-v0.2-key-derivation', // Legacy: stored for reference, not used in decrypt
|
|
28
51
|
iv: iv.toString('hex'),
|
|
29
52
|
ciphertext,
|
|
30
53
|
tag: tag.toString('hex'),
|
package/dist/auth/crypto.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"crypto.js","sourceRoot":"","sources":["../../src/auth/crypto.ts"],"names":[],"mappings":"AAAA,qBAAqB;AACrB,OAAO,EACL,cAAc,EACd,gBAAgB,EAChB,WAAW,EACX,UAAU,EACV,UAAU,EACV,eAAe,GAChB,MAAM,aAAa,CAAC;AAErB,MAAM,SAAS,GAAG,aAAa,CAAC;AAChC,MAAM,UAAU,GAAG,EAAE,CAAC,CAAC,WAAW;AAClC,MAAM,SAAS,GAAG,EAAE,CAAC;AACrB,MAAM,iBAAiB,GAAG,OAAO,CAAC;AAClC,MAAM,
|
|
1
|
+
{"version":3,"file":"crypto.js","sourceRoot":"","sources":["../../src/auth/crypto.ts"],"names":[],"mappings":"AAAA,qBAAqB;AACrB,OAAO,EACL,cAAc,EACd,gBAAgB,EAChB,WAAW,EACX,UAAU,EACV,UAAU,EACV,eAAe,GAChB,MAAM,aAAa,CAAC;AAErB,MAAM,SAAS,GAAG,aAAa,CAAC;AAChC,MAAM,UAAU,GAAG,EAAE,CAAC,CAAC,WAAW;AAClC,MAAM,SAAS,GAAG,EAAE,CAAC;AACrB,MAAM,iBAAiB,GAAG,OAAO,CAAC;AAClC,8EAA8E;AAC9E,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAC7E,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAElC,SAAS,cAAc,CAAC,QAAiB;IACvC,MAAM,QAAQ,GAAG,QAAQ,IAAI,GAAG,OAAO,EAAE,uBAAuB,CAAC;IACjE,IAAI,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;QACzB,OAAO,YAAY,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC;IAC/C,CAAC;IACD,6BAA6B;IAC7B,MAAM,IAAI,GAAG,WAAW,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IAC7C,IAAI,CAAC;QACH,SAAS,CAAC,QAAQ,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACjE,aAAa,CAAC,QAAQ,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IACjD,CAAC;IAAC,MAAM,CAAC,CAAA,CAAC;IACV,OAAO,IAAI,CAAC;AACd,CAAC;AAUD;;;;GAIG;AACH,MAAM,UAAU,SAAS,CAAC,SAAiB,EAAE,QAAiB;IAC5D,qEAAqE;IACrE,IAAI,CAAC;QACH,OAAO,UAAU,CAAC,SAAS,EAAE,cAAc,CAAC,QAAQ,CAAC,EAAE,iBAAiB,EAAE,UAAU,EAAE,QAAQ,CAAC,CAAC;IAClG,CAAC;IAAC,MAAM,CAAC;QACP,8EAA8E;QAC9E,OAAO,UAAU,CAAC,SAAS,EAAE,4BAA4B,EAAE,iBAAiB,EAAE,UAAU,EAAE,QAAQ,CAAC,CAAC;IACtG,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,OAAO,CAAC,SAAiB,EAAE,GAAW;IACpD,MAAM,EAAE,GAAG,WAAW,CAAC,SAAS,CAAC,CAAC;IAClC,MAAM,MAAM,GAAG,cAAc,CAAC,SAAS,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC;IAElD,IAAI,UAAU,GAAG,MAAM,CAAC,MAAM,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC;IACzD,UAAU,IAAI,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAClC,MAAM,GAAG,GAAG,MAAM,CAAC,UAAU,EAAE,CAAC;IAEhC,OAAO;QACL,IAAI,EAAE,4BAA4B,EAAE,oDAAoD;QACxF,EAAE,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC;QACtB,UAAU;QACV,GAAG,EAAE,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC;KACzB,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,OAAO,CAAC,IAAmB,EAAE,GAAW;IACtD,MAAM,QAAQ,GAAG,gBAAgB,CAC/B,SAAS,EACT,GAAG,EACH,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,KAAK,CAAC,CAC5B,CAAC;IACF,QAAQ,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC;IAElD,IAAI,SAAS,GAAG,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,UAAU,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC;IAChE,SAAS,IAAI,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IACpC,OAAO,SAAS,CAAC;AACnB,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,QAAQ,CAAC,IAAY,EAAE,GAAW;IAChD,MAAM,IAAI,GAAG,UAAU,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;IACvC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAClB,OAAO,eAAe,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;AAC7C,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,UAAU,CAAC,IAAY,EAAE,SAAiB,EAAE,GAAW;IACrE,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,cAAc,CAAC;QAAE,OAAO,KAAK,CAAC;IAExD,MAAM,QAAQ,GAAG,QAAQ,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;IACrC,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IACtC,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAErC,IAAI,MAAM,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM;QAAE,OAAO,KAAK,CAAC;IAClD,OAAO,eAAe,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AACzC,CAAC"}
|
package/dist/auth/manager.d.ts
CHANGED
|
@@ -6,7 +6,7 @@ import type { StoredAuth, StoredToken, StoredSession } from '../types.js';
|
|
|
6
6
|
export declare class AuthManager {
|
|
7
7
|
private key;
|
|
8
8
|
private authPath;
|
|
9
|
-
constructor(baseDir: string, machineId: string);
|
|
9
|
+
constructor(baseDir: string, machineId: string, saltFile?: string);
|
|
10
10
|
/** Store auth credentials for a domain (overwrites existing). */
|
|
11
11
|
store(domain: string, auth: StoredAuth): Promise<void>;
|
|
12
12
|
/** Retrieve auth credentials for a domain. Returns null if not found or decryption fails. */
|
package/dist/auth/manager.js
CHANGED
|
@@ -10,8 +10,8 @@ const AUTH_FILENAME = 'auth.enc';
|
|
|
10
10
|
export class AuthManager {
|
|
11
11
|
key;
|
|
12
12
|
authPath;
|
|
13
|
-
constructor(baseDir, machineId) {
|
|
14
|
-
this.key = deriveKey(machineId);
|
|
13
|
+
constructor(baseDir, machineId, saltFile) {
|
|
14
|
+
this.key = deriveKey(machineId, saltFile);
|
|
15
15
|
this.authPath = join(baseDir, AUTH_FILENAME);
|
|
16
16
|
}
|
|
17
17
|
/** Store auth credentials for a domain (overwrites existing). */
|
package/dist/auth/manager.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"manager.js","sourceRoot":"","sources":["../../src/auth/manager.ts"],"names":[],"mappings":"AAAA,sBAAsB;AACtB,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACrE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,SAAS,EAAsB,MAAM,aAAa,CAAC;AAG9E,MAAM,aAAa,GAAG,UAAU,CAAC;AAEjC;;;GAGG;AACH,MAAM,OAAO,WAAW;IACd,GAAG,CAAS;IACZ,QAAQ,CAAS;IAEzB,YAAY,OAAe,EAAE,SAAiB;
|
|
1
|
+
{"version":3,"file":"manager.js","sourceRoot":"","sources":["../../src/auth/manager.ts"],"names":[],"mappings":"AAAA,sBAAsB;AACtB,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACrE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,SAAS,EAAsB,MAAM,aAAa,CAAC;AAG9E,MAAM,aAAa,GAAG,UAAU,CAAC;AAEjC;;;GAGG;AACH,MAAM,OAAO,WAAW;IACd,GAAG,CAAS;IACZ,QAAQ,CAAS;IAEzB,YAAY,OAAe,EAAE,SAAiB,EAAE,QAAiB;QAC/D,IAAI,CAAC,GAAG,GAAG,SAAS,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;QAC1C,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;IAC/C,CAAC;IAED,iEAAiE;IACjE,KAAK,CAAC,KAAK,CAAC,MAAc,EAAE,IAAgB;QAC1C,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;QACrC,OAAO,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC;QACvB,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IAC9B,CAAC;IAED,6FAA6F;IAC7F,KAAK,CAAC,QAAQ,CAAC,MAAc;QAC3B,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;QACrC,OAAO,OAAO,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC;IACjC,CAAC;IAED,mEAAmE;IACnE,KAAK,CAAC,GAAG,CAAC,MAAc;QACtB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;QACrC,OAAO,MAAM,IAAI,OAAO,CAAC;IAC3B,CAAC;IAED,yEAAyE;IACzE,KAAK,CAAC,WAAW,CAAC,MAAc,EAAE,MAAmC;QACnE,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;QACjC,MAAM,QAAQ,GAAG,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,IAAI,EAAE,QAAiB,EAAE,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC;QACnF,GAAG,CAAC,MAAM,CAAC,GAAG,EAAE,GAAG,QAAQ,EAAE,MAAM,EAAE,CAAC;QACtC,MAAM,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAC1B,CAAC;IAED,gDAAgD;IAChD,KAAK,CAAC,cAAc,CAAC,MAAc;QACjC,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;QACjC,OAAO,GAAG,CAAC,MAAM,CAAC,EAAE,MAAM,IAAI,IAAI,CAAC;IACrC,CAAC;IAED,gFAAgF;IAChF,KAAK,CAAC,YAAY,CAAC,MAAc,EAAE,OAAsB;QACvD,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;QACjC,MAAM,QAAQ,GAAG,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,IAAI,EAAE,QAAiB,EAAE,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC;QACnF,GAAG,CAAC,MAAM,CAAC,GAAG,EAAE,GAAG,QAAQ,EAAE,OAAO,EAAE,CAAC;QACvC,MAAM,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAC1B,CAAC;IAED,6CAA6C;IAC7C,KAAK,CAAC,eAAe,CAAC,MAAc;QAClC,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;QACjC,OAAO,GAAG,CAAC,MAAM,CAAC,EAAE,OAAO,IAAI,IAAI,CAAC;IACtC,CAAC;IAED,wEAAwE;IACxE,KAAK,CAAC,qBAAqB,CAAC,MAAc,EAAE,KAAuD;QACjG,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;QACjC,MAAM,QAAQ,GAAG,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,IAAI,EAAE,QAAiB,EAAE,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC;QACnF,IAAI,KAAK,CAAC,YAAY,KAAK,SAAS;YAAE,QAAQ,CAAC,YAAY,GAAG,KAAK,CAAC,YAAY,CAAC;QACjF,IAAI,KAAK,CAAC,YAAY,KAAK,SAAS;YAAE,QAAQ,CAAC,YAAY,GAAG,KAAK,CAAC,YAAY,CAAC;QACjF,GAAG,CAAC,MAAM,CAAC,GAAG,QAAQ,CAAC;QACvB,MAAM,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAC1B,CAAC;IAED,+CAA+C;IAC/C,KAAK,CAAC,wBAAwB,CAAC,MAAc;QAC3C,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;QACjC,MAAM,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC;QACzB,IAAI,CAAC,IAAI;YAAE,OAAO,IAAI,CAAC;QACvB,IAAI,CAAC,IAAI,CAAC,YAAY,IAAI,CAAC,IAAI,CAAC,YAAY;YAAE,OAAO,IAAI,CAAC;QAC1D,OAAO,EAAE,YAAY,EAAE,IAAI,CAAC,YAAY,EAAE,YAAY,EAAE,IAAI,CAAC,YAAY,EAAE,CAAC;IAC9E,CAAC;IAED,yCAAyC;IACzC,KAAK,CAAC,WAAW;QACf,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;QACjC,OAAO,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC1B,CAAC;IAED,mCAAmC;IACnC,KAAK,CAAC,KAAK,CAAC,MAAc;QACxB,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;QACjC,OAAO,GAAG,CAAC,MAAM,CAAC,CAAC;QACnB,MAAM,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAC1B,CAAC;IAEO,KAAK,CAAC,OAAO;QACnB,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;YACvD,MAAM,SAAS,GAAkB,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YACrD,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;YAC/C,OAAO,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;QAC/B,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,EAAE,CAAC;QACZ,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,OAAO,CAAC,IAAgC;QACpD,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;QACtC,MAAM,KAAK,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAEtC,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QACvC,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;QAE/C,MAAM,SAAS,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;QAC3F,+DAA+D;QAC/D,MAAM,KAAK,CAAC,IAAI,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;IACpC,CAAC;CACF;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY;IAChC,IAAI,CAAC;QACH,MAAM,EAAE,GAAG,MAAM,QAAQ,CAAC,iBAAiB,EAAE,OAAO,CAAC,CAAC;QACtD,OAAO,EAAE,CAAC,IAAI,EAAE,CAAC;IACnB,CAAC;IAAC,MAAM,CAAC;QACP,iCAAiC;QACjC,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,MAAM,CAAC,SAAS,CAAC,CAAC;QAC7C,MAAM,EAAE,OAAO,EAAE,GAAG,MAAM,MAAM,CAAC,SAAS,CAAC,CAAC;QAC5C,OAAO,GAAG,QAAQ,EAAE,IAAI,OAAO,EAAE,EAAE,CAAC;IACtC,CAAC;AACH,CAAC"}
|
package/dist/discovery/fetch.js
CHANGED
|
@@ -27,9 +27,21 @@ export async function safeFetch(url, options = {}) {
|
|
|
27
27
|
'User-Agent': USER_AGENT,
|
|
28
28
|
'Accept': 'text/html,application/json,*/*',
|
|
29
29
|
},
|
|
30
|
-
redirect: '
|
|
30
|
+
redirect: 'manual',
|
|
31
31
|
});
|
|
32
32
|
clearTimeout(timer);
|
|
33
|
+
// SSRF-safe manual redirect (one hop max)
|
|
34
|
+
if (response.status >= 300 && response.status < 400 && response.headers.has('location')) {
|
|
35
|
+
const location = response.headers.get('location');
|
|
36
|
+
if (!location)
|
|
37
|
+
return null;
|
|
38
|
+
const redirectUrl = new URL(location, url).toString();
|
|
39
|
+
const ssrfResult = validateUrl(redirectUrl);
|
|
40
|
+
if (!ssrfResult.safe)
|
|
41
|
+
return null;
|
|
42
|
+
// Follow one redirect hop only
|
|
43
|
+
return await safeFetch(redirectUrl, { ...options, skipSsrf: true });
|
|
44
|
+
}
|
|
33
45
|
// Extract headers
|
|
34
46
|
const headers = {};
|
|
35
47
|
response.headers.forEach((value, key) => {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fetch.js","sourceRoot":"","sources":["../../src/discovery/fetch.ts"],"names":[],"mappings":"AAAA,yBAAyB;AACzB,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAgB/C,MAAM,eAAe,GAAG,IAAI,CAAC;AAC7B,MAAM,gBAAgB,GAAG,GAAG,GAAG,IAAI,CAAC,CAAC,QAAQ;AAC7C,MAAM,UAAU,GAAG,sBAAsB,CAAC;AAE1C;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,GAAW,EACX,UAA4B,EAAE;IAE9B,aAAa;IACb,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC;QACtB,MAAM,UAAU,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC;QACpC,IAAI,CAAC,UAAU,CAAC,IAAI;YAAE,OAAO,IAAI,CAAC;IACpC,CAAC;IAED,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,eAAe,CAAC;IACnD,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,KAAK,CAAC;IACvC,MAAM,OAAO,GAAG,OAAO,CAAC,WAAW,IAAI,gBAAgB,CAAC;IAExD,IAAI,CAAC;QACH,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,OAAO,CAAC,CAAC;QAE5D,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;YAChC,MAAM;YACN,MAAM,EAAE,UAAU,CAAC,MAAM;YACzB,OAAO,EAAE;gBACP,YAAY,EAAE,UAAU;gBACxB,QAAQ,EAAE,gCAAgC;aAC3C;YACD,QAAQ,EAAE,QAAQ;SACnB,CAAC,CAAC;QAEH,YAAY,CAAC,KAAK,CAAC,CAAC;QAEpB,kBAAkB;QAClB,MAAM,OAAO,GAA2B,EAAE,CAAC;QAC3C,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE;YACtC,OAAO,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC,GAAG,KAAK,CAAC;QACrC,CAAC,CAAC,CAAC;QAEH,MAAM,WAAW,GAAG,OAAO,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;QAElD,qCAAqC;QACrC,IAAI,MAAM,KAAK,MAAM,EAAE,CAAC;YACtB,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE,EAAE,WAAW,EAAE,CAAC;QACrE,CAAC;QAED,4BAA4B;QAC5B,MAAM,IAAI,GAAG,MAAM,eAAe,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAEtD,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC;IACjE,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,KAAK,UAAU,eAAe,CAAC,QAAkB,EAAE,OAAe;IAChE,yEAAyE;IACzE,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;IACnC,IAAI,IAAI,CAAC,MAAM,GAAG,OAAO,EAAE,CAAC;QAC1B,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;IAChC,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC"}
|
|
1
|
+
{"version":3,"file":"fetch.js","sourceRoot":"","sources":["../../src/discovery/fetch.ts"],"names":[],"mappings":"AAAA,yBAAyB;AACzB,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAgB/C,MAAM,eAAe,GAAG,IAAI,CAAC;AAC7B,MAAM,gBAAgB,GAAG,GAAG,GAAG,IAAI,CAAC,CAAC,QAAQ;AAC7C,MAAM,UAAU,GAAG,sBAAsB,CAAC;AAE1C;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,GAAW,EACX,UAA4B,EAAE;IAE9B,aAAa;IACb,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC;QACtB,MAAM,UAAU,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC;QACpC,IAAI,CAAC,UAAU,CAAC,IAAI;YAAE,OAAO,IAAI,CAAC;IACpC,CAAC;IAED,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,eAAe,CAAC;IACnD,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,KAAK,CAAC;IACvC,MAAM,OAAO,GAAG,OAAO,CAAC,WAAW,IAAI,gBAAgB,CAAC;IAExD,IAAI,CAAC;QACH,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,OAAO,CAAC,CAAC;QAE5D,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;YAChC,MAAM;YACN,MAAM,EAAE,UAAU,CAAC,MAAM;YACzB,OAAO,EAAE;gBACP,YAAY,EAAE,UAAU;gBACxB,QAAQ,EAAE,gCAAgC;aAC3C;YACD,QAAQ,EAAE,QAAQ;SACnB,CAAC,CAAC;QAEH,YAAY,CAAC,KAAK,CAAC,CAAC;QAEpB,0CAA0C;QAC1C,IAAI,QAAQ,CAAC,MAAM,IAAI,GAAG,IAAI,QAAQ,CAAC,MAAM,GAAG,GAAG,IAAI,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC;YACxF,MAAM,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;YAClD,IAAI,CAAC,QAAQ;gBAAE,OAAO,IAAI,CAAC;YAC3B,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC,QAAQ,EAAE,CAAC;YACtD,MAAM,UAAU,GAAG,WAAW,CAAC,WAAW,CAAC,CAAC;YAC5C,IAAI,CAAC,UAAU,CAAC,IAAI;gBAAE,OAAO,IAAI,CAAC;YAClC,+BAA+B;YAC/B,OAAO,MAAM,SAAS,CAAC,WAAW,EAAE,EAAE,GAAG,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;QACtE,CAAC;QAED,kBAAkB;QAClB,MAAM,OAAO,GAA2B,EAAE,CAAC;QAC3C,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE;YACtC,OAAO,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC,GAAG,KAAK,CAAC;QACrC,CAAC,CAAC,CAAC;QAEH,MAAM,WAAW,GAAG,OAAO,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;QAElD,qCAAqC;QACrC,IAAI,MAAM,KAAK,MAAM,EAAE,CAAC;YACtB,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE,EAAE,WAAW,EAAE,CAAC;QACrE,CAAC;QAED,4BAA4B;QAC5B,MAAM,IAAI,GAAG,MAAM,eAAe,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAEtD,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC;IACjE,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,KAAK,UAAU,eAAe,CAAC,QAAkB,EAAE,OAAe;IAChE,yEAAyE;IACzE,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;IACnC,IAAI,IAAI,CAAC,MAAM,GAAG,OAAO,EAAE,CAAC;QAC1B,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;IAChC,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@apitap/core",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.8",
|
|
4
4
|
"description": "Intercept web API traffic during browsing. Generate portable skill files so AI agents can call APIs directly instead of scraping.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "dist/index.js",
|
package/src/auth/crypto.ts
CHANGED
|
@@ -12,7 +12,24 @@ const ALGORITHM = 'aes-256-gcm';
|
|
|
12
12
|
const KEY_LENGTH = 32; // 256 bits
|
|
13
13
|
const IV_LENGTH = 16;
|
|
14
14
|
const PBKDF2_ITERATIONS = 100_000;
|
|
15
|
-
const PBKDF2_SALT = 'apitap-v0.2-key-derivation';
|
|
15
|
+
// const PBKDF2_SALT = 'apitap-v0.2-key-derivation'; // fallback for migration
|
|
16
|
+
import { readFileSync, writeFileSync, existsSync, mkdirSync } from 'node:fs';
|
|
17
|
+
import { homedir } from 'node:os';
|
|
18
|
+
|
|
19
|
+
function getInstallSalt(saltFile?: string): string {
|
|
20
|
+
const saltPath = saltFile || `${homedir()}/.apitap/install-salt`;
|
|
21
|
+
if (existsSync(saltPath)) {
|
|
22
|
+
return readFileSync(saltPath, 'utf8').trim();
|
|
23
|
+
}
|
|
24
|
+
// Generate and save new salt
|
|
25
|
+
const salt = randomBytes(32).toString('hex');
|
|
26
|
+
try {
|
|
27
|
+
mkdirSync(saltPath.replace(/\/[^/]+$/, ''), { recursive: true });
|
|
28
|
+
writeFileSync(saltPath, salt, { mode: 0o600 });
|
|
29
|
+
} catch {}
|
|
30
|
+
return salt;
|
|
31
|
+
}
|
|
32
|
+
|
|
16
33
|
|
|
17
34
|
export interface EncryptedData {
|
|
18
35
|
salt: string;
|
|
@@ -26,8 +43,14 @@ export interface EncryptedData {
|
|
|
26
43
|
* Uses a fixed application salt — the entropy comes from the machine ID
|
|
27
44
|
* being stretched through 100K iterations.
|
|
28
45
|
*/
|
|
29
|
-
export function deriveKey(machineId: string): Buffer {
|
|
30
|
-
|
|
46
|
+
export function deriveKey(machineId: string, saltFile?: string): Buffer {
|
|
47
|
+
// Try per-install salt first, fallback to old constant for migration
|
|
48
|
+
try {
|
|
49
|
+
return pbkdf2Sync(machineId, getInstallSalt(saltFile), PBKDF2_ITERATIONS, KEY_LENGTH, 'sha512');
|
|
50
|
+
} catch {
|
|
51
|
+
// Fallback for old installs (migration note: remove after all users migrated)
|
|
52
|
+
return pbkdf2Sync(machineId, 'apitap-v0.2-key-derivation', PBKDF2_ITERATIONS, KEY_LENGTH, 'sha512');
|
|
53
|
+
}
|
|
31
54
|
}
|
|
32
55
|
|
|
33
56
|
/**
|
|
@@ -43,7 +66,7 @@ export function encrypt(plaintext: string, key: Buffer): EncryptedData {
|
|
|
43
66
|
const tag = cipher.getAuthTag();
|
|
44
67
|
|
|
45
68
|
return {
|
|
46
|
-
salt:
|
|
69
|
+
salt: 'apitap-v0.2-key-derivation', // Legacy: stored for reference, not used in decrypt
|
|
47
70
|
iv: iv.toString('hex'),
|
|
48
71
|
ciphertext,
|
|
49
72
|
tag: tag.toString('hex'),
|
package/src/auth/handoff.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
// src/auth/handoff.ts
|
|
2
2
|
import type { AuthManager } from './manager.js';
|
|
3
3
|
import type { StoredSession, StoredAuth } from '../types.js';
|
|
4
|
+
import { launchBrowser } from '../capture/browser.js';
|
|
4
5
|
|
|
5
6
|
export interface HandoffOptions {
|
|
6
7
|
domain: string;
|
|
@@ -98,15 +99,12 @@ async function doHandoff(
|
|
|
98
99
|
const loginUrl = options.loginUrl || `https://${domain}`;
|
|
99
100
|
const timeout = options.timeout ?? 300_000; // 5 minutes
|
|
100
101
|
|
|
101
|
-
const {
|
|
102
|
-
|
|
103
|
-
const browser = await chromium.launch({ headless: false });
|
|
102
|
+
const { browser, context } = await launchBrowser({ headless: false });
|
|
104
103
|
|
|
105
104
|
try {
|
|
106
|
-
const context = await browser.newContext();
|
|
107
105
|
|
|
108
106
|
// Restore existing session cookies if available (warm start)
|
|
109
|
-
const cachedSession = await authManager.
|
|
107
|
+
const cachedSession = await authManager.retrieveSessionWithFallback(domain);
|
|
110
108
|
if (cachedSession?.cookies?.length) {
|
|
111
109
|
await context.addCookies(cachedSession.cookies);
|
|
112
110
|
}
|
|
@@ -161,8 +159,13 @@ async function doHandoff(
|
|
|
161
159
|
);
|
|
162
160
|
|
|
163
161
|
if (hasSessionCookie || authDetected) {
|
|
164
|
-
//
|
|
165
|
-
|
|
162
|
+
// Grace period: 4 additional polls at 2s each (~8s total)
|
|
163
|
+
// Allows time for MFA, CAPTCHAs, and post-login redirects
|
|
164
|
+
for (let grace = 0; grace < 4; grace++) {
|
|
165
|
+
if (page.isClosed()) break;
|
|
166
|
+
await page.waitForLoadState('networkidle', { timeout: 3000 }).catch(() => {});
|
|
167
|
+
await page.waitForTimeout(2000);
|
|
168
|
+
}
|
|
166
169
|
loginDetected = true;
|
|
167
170
|
break;
|
|
168
171
|
}
|
package/src/auth/manager.ts
CHANGED
|
@@ -14,8 +14,8 @@ export class AuthManager {
|
|
|
14
14
|
private key: Buffer;
|
|
15
15
|
private authPath: string;
|
|
16
16
|
|
|
17
|
-
constructor(baseDir: string, machineId: string) {
|
|
18
|
-
this.key = deriveKey(machineId);
|
|
17
|
+
constructor(baseDir: string, machineId: string, saltFile?: string) {
|
|
18
|
+
this.key = deriveKey(machineId, saltFile);
|
|
19
19
|
this.authPath = join(baseDir, AUTH_FILENAME);
|
|
20
20
|
}
|
|
21
21
|
|
|
@@ -66,6 +66,25 @@ export class AuthManager {
|
|
|
66
66
|
return all[domain]?.session ?? null;
|
|
67
67
|
}
|
|
68
68
|
|
|
69
|
+
/**
|
|
70
|
+
* Retrieve session with subdomain fallback.
|
|
71
|
+
* Tries exact match first, then walks up parent domains.
|
|
72
|
+
* e.g., dashboard.twitch.tv → twitch.tv
|
|
73
|
+
*/
|
|
74
|
+
async retrieveSessionWithFallback(domain: string): Promise<StoredSession | null> {
|
|
75
|
+
// Try exact match first
|
|
76
|
+
const exact = await this.retrieveSession(domain);
|
|
77
|
+
if (exact) return exact;
|
|
78
|
+
|
|
79
|
+
// Try parent domains
|
|
80
|
+
for (const parent of getParentDomains(domain)) {
|
|
81
|
+
const session = await this.retrieveSession(parent);
|
|
82
|
+
if (session) return session;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
return null;
|
|
86
|
+
}
|
|
87
|
+
|
|
69
88
|
/** Store OAuth credentials for a domain (merges with existing auth). */
|
|
70
89
|
async storeOAuthCredentials(domain: string, creds: { refreshToken?: string; clientSecret?: string }): Promise<void> {
|
|
71
90
|
const all = await this.loadAll();
|
|
@@ -122,6 +141,24 @@ export class AuthManager {
|
|
|
122
141
|
}
|
|
123
142
|
}
|
|
124
143
|
|
|
144
|
+
/**
|
|
145
|
+
* Get parent domains for subdomain fallback.
|
|
146
|
+
* dashboard.twitch.tv → ["twitch.tv"]
|
|
147
|
+
* a.b.example.com → ["b.example.com", "example.com"]
|
|
148
|
+
* twitch.tv → [] (already base, 2 labels)
|
|
149
|
+
*/
|
|
150
|
+
export function getParentDomains(domain: string): string[] {
|
|
151
|
+
const parts = domain.split('.');
|
|
152
|
+
const parents: string[] = [];
|
|
153
|
+
|
|
154
|
+
// Stop at 2 labels (e.g., "example.com" is the minimum)
|
|
155
|
+
for (let i = 1; i < parts.length - 1; i++) {
|
|
156
|
+
parents.push(parts.slice(i).join('.'));
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
return parents;
|
|
160
|
+
}
|
|
161
|
+
|
|
125
162
|
/**
|
|
126
163
|
* Get the machine ID for key derivation.
|
|
127
164
|
* Linux: /etc/machine-id
|
|
@@ -56,9 +56,11 @@ export async function refreshOAuth(
|
|
|
56
56
|
'oauth2.googleapis.com', 'accounts.google.com',
|
|
57
57
|
'login.microsoftonline.com', 'github.com',
|
|
58
58
|
'oauth.reddit.com', 'api.twitter.com',
|
|
59
|
+
'auth0.com', 'okta.com',
|
|
60
|
+
'securetoken.googleapis.com',
|
|
59
61
|
];
|
|
60
62
|
const tokenHost = new URL(oauthConfig.tokenEndpoint).hostname;
|
|
61
|
-
if (tokenHost !== domain && !tokenHost.endsWith('.' + domain) && !
|
|
63
|
+
if (tokenHost !== domain && !tokenHost.endsWith('.' + domain) && !isKnownOAuthHost(tokenHost, KNOWN_OAUTH_HOSTS)) {
|
|
62
64
|
return { success: false, error: `Token endpoint domain mismatch: ${tokenHost} vs ${domain}` };
|
|
63
65
|
}
|
|
64
66
|
|
|
@@ -118,3 +120,13 @@ export async function refreshOAuth(
|
|
|
118
120
|
};
|
|
119
121
|
}
|
|
120
122
|
}
|
|
123
|
+
|
|
124
|
+
/**
|
|
125
|
+
* Check if a hostname matches a known OAuth provider.
|
|
126
|
+
* Supports exact match and subdomain match (e.g., tenant.auth0.com matches auth0.com).
|
|
127
|
+
*/
|
|
128
|
+
function isKnownOAuthHost(tokenHost: string, knownHosts: string[]): boolean {
|
|
129
|
+
return knownHosts.some(known =>
|
|
130
|
+
tokenHost === known || tokenHost.endsWith('.' + known)
|
|
131
|
+
);
|
|
132
|
+
}
|
package/src/auth/refresh.ts
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
import type { SkillFile, StoredToken, StoredSession } from '../types.js';
|
|
3
3
|
import type { AuthManager } from './manager.js';
|
|
4
4
|
import { refreshOAuth, type OAuthRefreshResult } from './oauth-refresh.js';
|
|
5
|
+
import { launchBrowser } from '../capture/browser.js';
|
|
5
6
|
|
|
6
7
|
export interface RefreshOptions {
|
|
7
8
|
domain: string;
|
|
@@ -197,22 +198,19 @@ async function doBrowserRefresh(
|
|
|
197
198
|
return { success: oauthRefreshed, tokens: {}, oauthRefreshed: oauthRefreshed || undefined };
|
|
198
199
|
}
|
|
199
200
|
|
|
200
|
-
const { chromium } = await import('playwright');
|
|
201
|
-
|
|
202
201
|
const browserMode = options.browserMode || skill.auth?.browserMode || 'headless';
|
|
203
202
|
const refreshUrl = options.refreshUrl || skill.auth?.refreshUrl || skill.baseUrl;
|
|
204
203
|
const timeout = options.timeout || (skill.auth?.captchaRisk ? 300_000 : 30_000);
|
|
205
204
|
|
|
206
205
|
// Try to restore session from cache
|
|
207
|
-
const cachedSession = await authManager.
|
|
206
|
+
const cachedSession = await authManager.retrieveSessionWithFallback(options.domain);
|
|
208
207
|
const sessionValid = cachedSession && isSessionValid(cachedSession);
|
|
209
208
|
|
|
210
|
-
const browser = await
|
|
209
|
+
const { browser, context } = await launchBrowser({
|
|
211
210
|
headless: browserMode === 'headless',
|
|
212
211
|
});
|
|
213
212
|
|
|
214
213
|
try {
|
|
215
|
-
const context = await browser.newContext();
|
|
216
214
|
|
|
217
215
|
// Restore cookies if session is valid
|
|
218
216
|
if (sessionValid && cachedSession) {
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
// src/capture/browser.ts
|
|
2
|
+
import type { Browser, BrowserContext } from 'playwright';
|
|
3
|
+
|
|
4
|
+
const CHROME_USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36';
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Launch args that reduce Playwright's automation fingerprint.
|
|
8
|
+
*/
|
|
9
|
+
export function getLaunchArgs(): string[] {
|
|
10
|
+
return [
|
|
11
|
+
'--disable-blink-features=AutomationControlled',
|
|
12
|
+
];
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* Realistic Chrome user-agent string for anti-detection.
|
|
17
|
+
*/
|
|
18
|
+
export function getChromeUserAgent(): string {
|
|
19
|
+
return CHROME_USER_AGENT;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* Launch a Chromium browser with anti-detection measures.
|
|
24
|
+
*
|
|
25
|
+
* Three layers:
|
|
26
|
+
* 1. --disable-blink-features=AutomationControlled in launch args
|
|
27
|
+
* 2. Realistic Chrome UA on context
|
|
28
|
+
* 3. navigator.webdriver = false via addInitScript
|
|
29
|
+
* 4. Viewport 1920x1080
|
|
30
|
+
*/
|
|
31
|
+
export async function launchBrowser(options: { headless: boolean }): Promise<{ browser: Browser; context: BrowserContext }> {
|
|
32
|
+
const { chromium } = await import('playwright');
|
|
33
|
+
|
|
34
|
+
const browser = await chromium.launch({
|
|
35
|
+
headless: options.headless,
|
|
36
|
+
args: getLaunchArgs(),
|
|
37
|
+
});
|
|
38
|
+
|
|
39
|
+
const context = await browser.newContext({
|
|
40
|
+
userAgent: CHROME_USER_AGENT,
|
|
41
|
+
viewport: { width: 1920, height: 1080 },
|
|
42
|
+
});
|
|
43
|
+
|
|
44
|
+
await context.addInitScript(() => {
|
|
45
|
+
Object.defineProperty(navigator, 'webdriver', {
|
|
46
|
+
get: () => false,
|
|
47
|
+
});
|
|
48
|
+
});
|
|
49
|
+
|
|
50
|
+
return { browser, context };
|
|
51
|
+
}
|
package/src/capture/monitor.ts
CHANGED
|
@@ -5,6 +5,7 @@ import { isDomainMatch } from './domain.js';
|
|
|
5
5
|
import { SkillGenerator, type GeneratorOptions } from '../skill/generator.js';
|
|
6
6
|
import { IdleTracker } from './idle.js';
|
|
7
7
|
import { detectCaptcha } from '../auth/refresh.js';
|
|
8
|
+
import { launchBrowser } from './browser.js';
|
|
8
9
|
import type { CapturedExchange } from '../types.js';
|
|
9
10
|
|
|
10
11
|
export interface CaptureOptions {
|
|
@@ -31,7 +32,7 @@ export interface CaptureResult {
|
|
|
31
32
|
|
|
32
33
|
const DEFAULT_CDP_PORTS = [18792, 18800, 9222];
|
|
33
34
|
|
|
34
|
-
async function connectToBrowser(options: CaptureOptions): Promise<{ browser: Browser; launched: boolean }> {
|
|
35
|
+
async function connectToBrowser(options: CaptureOptions): Promise<{ browser: Browser; launched: boolean; launchContext?: import('playwright').BrowserContext }> {
|
|
35
36
|
if (!options.launch) {
|
|
36
37
|
const ports = options.port ? [options.port] : DEFAULT_CDP_PORTS;
|
|
37
38
|
for (const port of ports) {
|
|
@@ -49,12 +50,12 @@ async function connectToBrowser(options: CaptureOptions): Promise<{ browser: Bro
|
|
|
49
50
|
throw new Error(`No browser found on CDP ports: ${ports.join(', ')}. Is a Chromium browser running with remote debugging?`);
|
|
50
51
|
}
|
|
51
52
|
|
|
52
|
-
const browser = await
|
|
53
|
-
return { browser, launched: true };
|
|
53
|
+
const { browser, context } = await launchBrowser({ headless: options.headless ?? (process.env.DISPLAY ? false : true) });
|
|
54
|
+
return { browser, launched: true, launchContext: context };
|
|
54
55
|
}
|
|
55
56
|
|
|
56
57
|
export async function capture(options: CaptureOptions): Promise<CaptureResult> {
|
|
57
|
-
const { browser, launched } = await connectToBrowser(options);
|
|
58
|
+
const { browser, launched, launchContext } = await connectToBrowser(options);
|
|
58
59
|
const generators = new Map<string, SkillGenerator>();
|
|
59
60
|
let totalRequests = 0;
|
|
60
61
|
let filteredRequests = 0;
|
|
@@ -73,7 +74,10 @@ export async function capture(options: CaptureOptions): Promise<CaptureResult> {
|
|
|
73
74
|
let idleInterval: ReturnType<typeof setInterval> | null = null;
|
|
74
75
|
|
|
75
76
|
let page: Page;
|
|
76
|
-
if (launched) {
|
|
77
|
+
if (launched && launchContext) {
|
|
78
|
+
page = await launchContext.newPage();
|
|
79
|
+
} else if (launched) {
|
|
80
|
+
// Fallback: shouldn't happen, but handle gracefully
|
|
77
81
|
const context = await browser.newContext();
|
|
78
82
|
page = await context.newPage();
|
|
79
83
|
} else {
|
|
@@ -6,6 +6,7 @@ export interface OAuthInfo {
|
|
|
6
6
|
grantType: 'refresh_token' | 'client_credentials';
|
|
7
7
|
scope?: string;
|
|
8
8
|
clientSecret?: string;
|
|
9
|
+
refreshToken?: string;
|
|
9
10
|
}
|
|
10
11
|
|
|
11
12
|
/**
|
|
@@ -26,18 +27,48 @@ export function isOAuthTokenRequest(req: {
|
|
|
26
27
|
const urlLower = req.url.toLowerCase();
|
|
27
28
|
if (!urlLower.includes('/token') && !urlLower.includes('/oauth')) return null;
|
|
28
29
|
|
|
30
|
+
// Parse URL once — reused for query param fallback and Firebase detection
|
|
31
|
+
let parsedUrl: URL;
|
|
32
|
+
try {
|
|
33
|
+
parsedUrl = new URL(req.url);
|
|
34
|
+
} catch {
|
|
35
|
+
return null;
|
|
36
|
+
}
|
|
37
|
+
|
|
29
38
|
if (!req.postData) return null;
|
|
30
39
|
|
|
31
40
|
// Parse body — support URL-encoded and JSON
|
|
32
41
|
const params = parseBody(req.postData, req.headers['content-type'] ?? '');
|
|
33
42
|
if (!params) return null;
|
|
34
43
|
|
|
35
|
-
|
|
44
|
+
// grant_type: body takes precedence, URL query param as fallback (Supabase GoTrue)
|
|
45
|
+
let grantType = params.get('grant_type');
|
|
46
|
+
if (!grantType) {
|
|
47
|
+
grantType = parsedUrl.searchParams.get('grant_type') ?? undefined;
|
|
48
|
+
}
|
|
36
49
|
if (!grantType) return null;
|
|
37
50
|
|
|
38
51
|
// Only refreshable flows
|
|
39
52
|
if (grantType !== 'refresh_token' && grantType !== 'client_credentials') return null;
|
|
40
53
|
|
|
54
|
+
// Firebase provider-specific detection:
|
|
55
|
+
// securetoken.googleapis.com uses ?key= instead of client_id
|
|
56
|
+
if (
|
|
57
|
+
parsedUrl.hostname === 'securetoken.googleapis.com' &&
|
|
58
|
+
parsedUrl.searchParams.has('key') &&
|
|
59
|
+
grantType === 'refresh_token'
|
|
60
|
+
) {
|
|
61
|
+
const firebaseKey = parsedUrl.searchParams.get('key')!;
|
|
62
|
+
const result: OAuthInfo = {
|
|
63
|
+
tokenEndpoint: req.url, // Keep full URL with ?key= param
|
|
64
|
+
clientId: firebaseKey,
|
|
65
|
+
grantType: 'refresh_token',
|
|
66
|
+
};
|
|
67
|
+
const refreshToken = params.get('refresh_token');
|
|
68
|
+
if (refreshToken) result.refreshToken = refreshToken;
|
|
69
|
+
return result;
|
|
70
|
+
}
|
|
71
|
+
|
|
41
72
|
// Extract client_id — may also be in Basic auth header
|
|
42
73
|
let clientId = params.get('client_id') ?? '';
|
|
43
74
|
let clientSecret = params.get('client_secret');
|
|
@@ -61,6 +92,8 @@ export function isOAuthTokenRequest(req: {
|
|
|
61
92
|
const scope = params.get('scope');
|
|
62
93
|
if (scope) result.scope = scope;
|
|
63
94
|
if (clientSecret) result.clientSecret = clientSecret;
|
|
95
|
+
const refreshToken = params.get('refresh_token');
|
|
96
|
+
if (refreshToken) result.refreshToken = refreshToken;
|
|
64
97
|
|
|
65
98
|
return result;
|
|
66
99
|
}
|
package/src/capture/session.ts
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
// src/capture/session.ts
|
|
2
|
-
import {
|
|
2
|
+
import { type Browser, type Page } from 'playwright';
|
|
3
3
|
import { randomUUID } from 'node:crypto';
|
|
4
4
|
import { shouldCapture } from './filter.js';
|
|
5
|
+
import { launchBrowser } from './browser.js';
|
|
5
6
|
import { isDomainMatch } from './domain.js';
|
|
6
7
|
import { SkillGenerator, type GeneratorOptions } from '../skill/generator.js';
|
|
7
8
|
import { detectCaptcha } from '../auth/refresh.js';
|
|
@@ -53,8 +54,8 @@ export class CaptureSession {
|
|
|
53
54
|
this.targetUrl = url.startsWith('http') ? url : `https://${url}`;
|
|
54
55
|
const headless = this.options.headless ?? true;
|
|
55
56
|
|
|
56
|
-
|
|
57
|
-
|
|
57
|
+
const { browser, context } = await launchBrowser({ headless });
|
|
58
|
+
this.browser = browser;
|
|
58
59
|
|
|
59
60
|
// Inject cached session cookies if available
|
|
60
61
|
try {
|
|
@@ -62,7 +63,7 @@ export class CaptureSession {
|
|
|
62
63
|
const machineId = await getMachineId();
|
|
63
64
|
const authManager = new AuthManager(authDir, machineId);
|
|
64
65
|
const domain = new URL(this.targetUrl).hostname;
|
|
65
|
-
const cachedSession = await authManager.
|
|
66
|
+
const cachedSession = await authManager.retrieveSessionWithFallback(domain);
|
|
66
67
|
if (cachedSession?.cookies?.length) {
|
|
67
68
|
await context.addCookies(cachedSession.cookies);
|
|
68
69
|
}
|
|
@@ -224,8 +225,12 @@ export class CaptureSession {
|
|
|
224
225
|
const oauthConfig = generator.getOAuthConfig();
|
|
225
226
|
if (oauthConfig) {
|
|
226
227
|
const clientSecret = generator.getOAuthClientSecret();
|
|
227
|
-
|
|
228
|
-
|
|
228
|
+
const refreshToken = generator.getOAuthRefreshToken();
|
|
229
|
+
if (clientSecret || refreshToken) {
|
|
230
|
+
await authManager.storeOAuthCredentials(domain, {
|
|
231
|
+
...(clientSecret ? { clientSecret } : {}),
|
|
232
|
+
...(refreshToken ? { refreshToken } : {}),
|
|
233
|
+
});
|
|
229
234
|
}
|
|
230
235
|
}
|
|
231
236
|
|
package/src/cli.ts
CHANGED
|
@@ -211,8 +211,12 @@ async function handleCapture(positional: string[], flags: Record<string, string
|
|
|
211
211
|
const oauthConfig = generator.getOAuthConfig();
|
|
212
212
|
if (oauthConfig) {
|
|
213
213
|
const clientSecret = generator.getOAuthClientSecret();
|
|
214
|
-
|
|
215
|
-
|
|
214
|
+
const refreshToken = generator.getOAuthRefreshToken();
|
|
215
|
+
if (clientSecret || refreshToken) {
|
|
216
|
+
await authManager.storeOAuthCredentials(domain, {
|
|
217
|
+
...(clientSecret ? { clientSecret } : {}),
|
|
218
|
+
...(refreshToken ? { refreshToken } : {}),
|
|
219
|
+
});
|
|
216
220
|
}
|
|
217
221
|
}
|
|
218
222
|
|
package/src/discovery/fetch.ts
CHANGED
|
@@ -48,11 +48,22 @@ export async function safeFetch(
|
|
|
48
48
|
'User-Agent': USER_AGENT,
|
|
49
49
|
'Accept': 'text/html,application/json,*/*',
|
|
50
50
|
},
|
|
51
|
-
redirect: '
|
|
51
|
+
redirect: 'manual',
|
|
52
52
|
});
|
|
53
53
|
|
|
54
54
|
clearTimeout(timer);
|
|
55
55
|
|
|
56
|
+
// SSRF-safe manual redirect (one hop max)
|
|
57
|
+
if (response.status >= 300 && response.status < 400 && response.headers.has('location')) {
|
|
58
|
+
const location = response.headers.get('location');
|
|
59
|
+
if (!location) return null;
|
|
60
|
+
const redirectUrl = new URL(location, url).toString();
|
|
61
|
+
const ssrfResult = validateUrl(redirectUrl);
|
|
62
|
+
if (!ssrfResult.safe) return null;
|
|
63
|
+
// Follow one redirect hop only
|
|
64
|
+
return await safeFetch(redirectUrl, { ...options, skipSsrf: true });
|
|
65
|
+
}
|
|
66
|
+
|
|
56
67
|
// Extract headers
|
|
57
68
|
const headers: Record<string, string> = {};
|
|
58
69
|
response.headers.forEach((value, key) => {
|
package/src/skill/generator.ts
CHANGED
|
@@ -180,6 +180,7 @@ export class SkillGenerator {
|
|
|
180
180
|
private captchaRisk = false;
|
|
181
181
|
private oauthConfig: OAuthConfig | null = null;
|
|
182
182
|
private oauthClientSecret: string | undefined;
|
|
183
|
+
private oauthRefreshToken: string | undefined;
|
|
183
184
|
private totalNetworkBytes = 0; // v1.0: accumulate all response sizes
|
|
184
185
|
|
|
185
186
|
/** Number of unique endpoints captured so far */
|
|
@@ -255,6 +256,7 @@ export class SkillGenerator {
|
|
|
255
256
|
...(oauthInfo.scope ? { scope: oauthInfo.scope } : {}),
|
|
256
257
|
};
|
|
257
258
|
this.oauthClientSecret = oauthInfo.clientSecret;
|
|
259
|
+
this.oauthRefreshToken = oauthInfo.refreshToken;
|
|
258
260
|
}
|
|
259
261
|
|
|
260
262
|
// Extract auth before filtering headers (includes entropy-based detection)
|
|
@@ -402,6 +404,11 @@ export class SkillGenerator {
|
|
|
402
404
|
return this.oauthClientSecret;
|
|
403
405
|
}
|
|
404
406
|
|
|
407
|
+
/** Get the refresh token captured from OAuth traffic (for encrypted storage). */
|
|
408
|
+
getOAuthRefreshToken(): string | undefined {
|
|
409
|
+
return this.oauthRefreshToken;
|
|
410
|
+
}
|
|
411
|
+
|
|
405
412
|
/** Check if any endpoint has refreshable tokens. */
|
|
406
413
|
private hasRefreshableTokens(): boolean {
|
|
407
414
|
for (const endpoint of this.endpoints.values()) {
|