@apitap/core 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +60 -0
- package/README.md +362 -0
- package/SKILL.md +270 -0
- package/dist/auth/crypto.d.ts +31 -0
- package/dist/auth/crypto.js +66 -0
- package/dist/auth/crypto.js.map +1 -0
- package/dist/auth/handoff.d.ts +29 -0
- package/dist/auth/handoff.js +180 -0
- package/dist/auth/handoff.js.map +1 -0
- package/dist/auth/manager.d.ts +46 -0
- package/dist/auth/manager.js +127 -0
- package/dist/auth/manager.js.map +1 -0
- package/dist/auth/oauth-refresh.d.ts +16 -0
- package/dist/auth/oauth-refresh.js +91 -0
- package/dist/auth/oauth-refresh.js.map +1 -0
- package/dist/auth/refresh.d.ts +43 -0
- package/dist/auth/refresh.js +217 -0
- package/dist/auth/refresh.js.map +1 -0
- package/dist/capture/anti-bot.d.ts +15 -0
- package/dist/capture/anti-bot.js +43 -0
- package/dist/capture/anti-bot.js.map +1 -0
- package/dist/capture/blocklist.d.ts +6 -0
- package/dist/capture/blocklist.js +70 -0
- package/dist/capture/blocklist.js.map +1 -0
- package/dist/capture/body-diff.d.ts +8 -0
- package/dist/capture/body-diff.js +102 -0
- package/dist/capture/body-diff.js.map +1 -0
- package/dist/capture/body-variables.d.ts +13 -0
- package/dist/capture/body-variables.js +142 -0
- package/dist/capture/body-variables.js.map +1 -0
- package/dist/capture/domain.d.ts +8 -0
- package/dist/capture/domain.js +34 -0
- package/dist/capture/domain.js.map +1 -0
- package/dist/capture/entropy.d.ts +33 -0
- package/dist/capture/entropy.js +100 -0
- package/dist/capture/entropy.js.map +1 -0
- package/dist/capture/filter.d.ts +11 -0
- package/dist/capture/filter.js +49 -0
- package/dist/capture/filter.js.map +1 -0
- package/dist/capture/graphql.d.ts +21 -0
- package/dist/capture/graphql.js +99 -0
- package/dist/capture/graphql.js.map +1 -0
- package/dist/capture/idle.d.ts +23 -0
- package/dist/capture/idle.js +44 -0
- package/dist/capture/idle.js.map +1 -0
- package/dist/capture/monitor.d.ts +26 -0
- package/dist/capture/monitor.js +183 -0
- package/dist/capture/monitor.js.map +1 -0
- package/dist/capture/oauth-detector.d.ts +18 -0
- package/dist/capture/oauth-detector.js +96 -0
- package/dist/capture/oauth-detector.js.map +1 -0
- package/dist/capture/pagination.d.ts +9 -0
- package/dist/capture/pagination.js +40 -0
- package/dist/capture/pagination.js.map +1 -0
- package/dist/capture/parameterize.d.ts +17 -0
- package/dist/capture/parameterize.js +63 -0
- package/dist/capture/parameterize.js.map +1 -0
- package/dist/capture/scrubber.d.ts +5 -0
- package/dist/capture/scrubber.js +38 -0
- package/dist/capture/scrubber.js.map +1 -0
- package/dist/capture/session.d.ts +46 -0
- package/dist/capture/session.js +445 -0
- package/dist/capture/session.js.map +1 -0
- package/dist/capture/token-detector.d.ts +16 -0
- package/dist/capture/token-detector.js +62 -0
- package/dist/capture/token-detector.js.map +1 -0
- package/dist/capture/verifier.d.ts +17 -0
- package/dist/capture/verifier.js +147 -0
- package/dist/capture/verifier.js.map +1 -0
- package/dist/cli.d.ts +2 -0
- package/dist/cli.js +930 -0
- package/dist/cli.js.map +1 -0
- package/dist/discovery/auth.d.ts +17 -0
- package/dist/discovery/auth.js +81 -0
- package/dist/discovery/auth.js.map +1 -0
- package/dist/discovery/fetch.d.ts +17 -0
- package/dist/discovery/fetch.js +59 -0
- package/dist/discovery/fetch.js.map +1 -0
- package/dist/discovery/frameworks.d.ts +11 -0
- package/dist/discovery/frameworks.js +249 -0
- package/dist/discovery/frameworks.js.map +1 -0
- package/dist/discovery/index.d.ts +21 -0
- package/dist/discovery/index.js +219 -0
- package/dist/discovery/index.js.map +1 -0
- package/dist/discovery/openapi.d.ts +13 -0
- package/dist/discovery/openapi.js +175 -0
- package/dist/discovery/openapi.js.map +1 -0
- package/dist/discovery/probes.d.ts +9 -0
- package/dist/discovery/probes.js +70 -0
- package/dist/discovery/probes.js.map +1 -0
- package/dist/index.d.ts +25 -0
- package/dist/index.js +25 -0
- package/dist/index.js.map +1 -0
- package/dist/inspect/report.d.ts +52 -0
- package/dist/inspect/report.js +191 -0
- package/dist/inspect/report.js.map +1 -0
- package/dist/mcp.d.ts +8 -0
- package/dist/mcp.js +526 -0
- package/dist/mcp.js.map +1 -0
- package/dist/orchestration/browse.d.ts +38 -0
- package/dist/orchestration/browse.js +198 -0
- package/dist/orchestration/browse.js.map +1 -0
- package/dist/orchestration/cache.d.ts +15 -0
- package/dist/orchestration/cache.js +24 -0
- package/dist/orchestration/cache.js.map +1 -0
- package/dist/plugin.d.ts +17 -0
- package/dist/plugin.js +158 -0
- package/dist/plugin.js.map +1 -0
- package/dist/read/decoders/deepwiki.d.ts +2 -0
- package/dist/read/decoders/deepwiki.js +148 -0
- package/dist/read/decoders/deepwiki.js.map +1 -0
- package/dist/read/decoders/grokipedia.d.ts +2 -0
- package/dist/read/decoders/grokipedia.js +210 -0
- package/dist/read/decoders/grokipedia.js.map +1 -0
- package/dist/read/decoders/hackernews.d.ts +2 -0
- package/dist/read/decoders/hackernews.js +168 -0
- package/dist/read/decoders/hackernews.js.map +1 -0
- package/dist/read/decoders/index.d.ts +2 -0
- package/dist/read/decoders/index.js +12 -0
- package/dist/read/decoders/index.js.map +1 -0
- package/dist/read/decoders/reddit.d.ts +2 -0
- package/dist/read/decoders/reddit.js +142 -0
- package/dist/read/decoders/reddit.js.map +1 -0
- package/dist/read/decoders/twitter.d.ts +12 -0
- package/dist/read/decoders/twitter.js +187 -0
- package/dist/read/decoders/twitter.js.map +1 -0
- package/dist/read/decoders/wikipedia.d.ts +2 -0
- package/dist/read/decoders/wikipedia.js +66 -0
- package/dist/read/decoders/wikipedia.js.map +1 -0
- package/dist/read/decoders/youtube.d.ts +2 -0
- package/dist/read/decoders/youtube.js +69 -0
- package/dist/read/decoders/youtube.js.map +1 -0
- package/dist/read/extract.d.ts +25 -0
- package/dist/read/extract.js +320 -0
- package/dist/read/extract.js.map +1 -0
- package/dist/read/index.d.ts +14 -0
- package/dist/read/index.js +66 -0
- package/dist/read/index.js.map +1 -0
- package/dist/read/peek.d.ts +9 -0
- package/dist/read/peek.js +137 -0
- package/dist/read/peek.js.map +1 -0
- package/dist/read/types.d.ts +44 -0
- package/dist/read/types.js +3 -0
- package/dist/read/types.js.map +1 -0
- package/dist/replay/engine.d.ts +53 -0
- package/dist/replay/engine.js +441 -0
- package/dist/replay/engine.js.map +1 -0
- package/dist/replay/truncate.d.ts +16 -0
- package/dist/replay/truncate.js +92 -0
- package/dist/replay/truncate.js.map +1 -0
- package/dist/serve.d.ts +31 -0
- package/dist/serve.js +149 -0
- package/dist/serve.js.map +1 -0
- package/dist/skill/generator.d.ts +44 -0
- package/dist/skill/generator.js +419 -0
- package/dist/skill/generator.js.map +1 -0
- package/dist/skill/importer.d.ts +26 -0
- package/dist/skill/importer.js +80 -0
- package/dist/skill/importer.js.map +1 -0
- package/dist/skill/search.d.ts +19 -0
- package/dist/skill/search.js +51 -0
- package/dist/skill/search.js.map +1 -0
- package/dist/skill/signing.d.ts +16 -0
- package/dist/skill/signing.js +34 -0
- package/dist/skill/signing.js.map +1 -0
- package/dist/skill/ssrf.d.ts +27 -0
- package/dist/skill/ssrf.js +210 -0
- package/dist/skill/ssrf.js.map +1 -0
- package/dist/skill/store.d.ts +7 -0
- package/dist/skill/store.js +93 -0
- package/dist/skill/store.js.map +1 -0
- package/dist/stats/report.d.ts +26 -0
- package/dist/stats/report.js +157 -0
- package/dist/stats/report.js.map +1 -0
- package/dist/types.d.ts +214 -0
- package/dist/types.js +3 -0
- package/dist/types.js.map +1 -0
- package/package.json +58 -0
- package/src/auth/crypto.ts +92 -0
- package/src/auth/handoff.ts +229 -0
- package/src/auth/manager.ts +140 -0
- package/src/auth/oauth-refresh.ts +120 -0
- package/src/auth/refresh.ts +300 -0
- package/src/capture/anti-bot.ts +63 -0
- package/src/capture/blocklist.ts +75 -0
- package/src/capture/body-diff.ts +109 -0
- package/src/capture/body-variables.ts +156 -0
- package/src/capture/domain.ts +34 -0
- package/src/capture/entropy.ts +121 -0
- package/src/capture/filter.ts +56 -0
- package/src/capture/graphql.ts +124 -0
- package/src/capture/idle.ts +45 -0
- package/src/capture/monitor.ts +224 -0
- package/src/capture/oauth-detector.ts +106 -0
- package/src/capture/pagination.ts +49 -0
- package/src/capture/parameterize.ts +68 -0
- package/src/capture/scrubber.ts +49 -0
- package/src/capture/session.ts +502 -0
- package/src/capture/token-detector.ts +76 -0
- package/src/capture/verifier.ts +171 -0
- package/src/cli.ts +1031 -0
- package/src/discovery/auth.ts +99 -0
- package/src/discovery/fetch.ts +85 -0
- package/src/discovery/frameworks.ts +231 -0
- package/src/discovery/index.ts +256 -0
- package/src/discovery/openapi.ts +230 -0
- package/src/discovery/probes.ts +76 -0
- package/src/index.ts +26 -0
- package/src/inspect/report.ts +247 -0
- package/src/mcp.ts +618 -0
- package/src/orchestration/browse.ts +250 -0
- package/src/orchestration/cache.ts +37 -0
- package/src/plugin.ts +188 -0
- package/src/read/decoders/deepwiki.ts +180 -0
- package/src/read/decoders/grokipedia.ts +246 -0
- package/src/read/decoders/hackernews.ts +198 -0
- package/src/read/decoders/index.ts +15 -0
- package/src/read/decoders/reddit.ts +158 -0
- package/src/read/decoders/twitter.ts +211 -0
- package/src/read/decoders/wikipedia.ts +75 -0
- package/src/read/decoders/youtube.ts +75 -0
- package/src/read/extract.ts +396 -0
- package/src/read/index.ts +78 -0
- package/src/read/peek.ts +175 -0
- package/src/read/types.ts +37 -0
- package/src/replay/engine.ts +559 -0
- package/src/replay/truncate.ts +116 -0
- package/src/serve.ts +189 -0
- package/src/skill/generator.ts +473 -0
- package/src/skill/importer.ts +107 -0
- package/src/skill/search.ts +76 -0
- package/src/skill/signing.ts +36 -0
- package/src/skill/ssrf.ts +238 -0
- package/src/skill/store.ts +107 -0
- package/src/stats/report.ts +208 -0
- package/src/types.ts +233 -0
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import type { SkillFile } from '../types.js';
|
|
2
|
+
import type { AuthManager } from './manager.js';
|
|
3
|
+
export interface RefreshOptions {
|
|
4
|
+
domain: string;
|
|
5
|
+
refreshUrl?: string;
|
|
6
|
+
browserMode?: 'headless' | 'visible';
|
|
7
|
+
timeout?: number;
|
|
8
|
+
/** @internal Skip SSRF check — for testing only */
|
|
9
|
+
_skipSsrfCheck?: boolean;
|
|
10
|
+
}
|
|
11
|
+
export interface RefreshResult {
|
|
12
|
+
success: boolean;
|
|
13
|
+
tokens: Record<string, string>;
|
|
14
|
+
captchaDetected?: 'cloudflare' | 'recaptcha' | 'hcaptcha';
|
|
15
|
+
oauthRefreshed?: boolean;
|
|
16
|
+
error?: string;
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* Extract token values from a request body string.
|
|
20
|
+
*
|
|
21
|
+
* @param body - Raw request body (string)
|
|
22
|
+
* @param tokenNames - JSON paths of tokens to extract (e.g., ["csrf_token", "data.nonce"])
|
|
23
|
+
* @returns Map of token name to value
|
|
24
|
+
*/
|
|
25
|
+
export declare function extractTokensFromRequest(body: string, tokenNames: string[]): Record<string, string>;
|
|
26
|
+
/**
|
|
27
|
+
* Detect captcha challenge in page content.
|
|
28
|
+
*/
|
|
29
|
+
export declare function detectCaptcha(html: string): 'cloudflare' | 'recaptcha' | 'hcaptcha' | null;
|
|
30
|
+
/**
|
|
31
|
+
* Refresh tokens by spawning a browser and intercepting requests.
|
|
32
|
+
*
|
|
33
|
+
* This is the main entry point for token refresh. It:
|
|
34
|
+
* 1. Launches a browser (visible if captchaRisk is true)
|
|
35
|
+
* 2. Navigates to the refresh URL
|
|
36
|
+
* 3. Intercepts outgoing requests to capture fresh token values
|
|
37
|
+
* 4. Returns the captured tokens
|
|
38
|
+
*
|
|
39
|
+
* @param skill - Skill file with auth config and endpoint info
|
|
40
|
+
* @param authManager - Auth manager for storing refreshed tokens
|
|
41
|
+
* @param options - Refresh options
|
|
42
|
+
*/
|
|
43
|
+
export declare function refreshTokens(skill: SkillFile, authManager: AuthManager, options: RefreshOptions): Promise<RefreshResult>;
|
|
@@ -0,0 +1,217 @@
|
|
|
1
|
+
import { refreshOAuth } from './oauth-refresh.js';
|
|
2
|
+
// Mutex to prevent concurrent refreshes for the same domain
|
|
3
|
+
const refreshLocks = new Map();
|
|
4
|
+
/**
|
|
5
|
+
* Extract token values from a request body string.
|
|
6
|
+
*
|
|
7
|
+
* @param body - Raw request body (string)
|
|
8
|
+
* @param tokenNames - JSON paths of tokens to extract (e.g., ["csrf_token", "data.nonce"])
|
|
9
|
+
* @returns Map of token name to value
|
|
10
|
+
*/
|
|
11
|
+
export function extractTokensFromRequest(body, tokenNames) {
|
|
12
|
+
const result = {};
|
|
13
|
+
let parsed;
|
|
14
|
+
try {
|
|
15
|
+
parsed = JSON.parse(body);
|
|
16
|
+
}
|
|
17
|
+
catch {
|
|
18
|
+
return result;
|
|
19
|
+
}
|
|
20
|
+
if (typeof parsed !== 'object' || parsed === null) {
|
|
21
|
+
return result;
|
|
22
|
+
}
|
|
23
|
+
for (const path of tokenNames) {
|
|
24
|
+
const value = getNestedValue(parsed, path);
|
|
25
|
+
if (typeof value === 'string') {
|
|
26
|
+
result[path] = value;
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
return result;
|
|
30
|
+
}
|
|
31
|
+
function getNestedValue(obj, path) {
|
|
32
|
+
const parts = path.split('.');
|
|
33
|
+
let current = obj;
|
|
34
|
+
for (const part of parts) {
|
|
35
|
+
if (typeof current !== 'object' || current === null) {
|
|
36
|
+
return undefined;
|
|
37
|
+
}
|
|
38
|
+
current = current[part];
|
|
39
|
+
}
|
|
40
|
+
return current;
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* Detect captcha challenge in page content.
|
|
44
|
+
*/
|
|
45
|
+
export function detectCaptcha(html) {
|
|
46
|
+
// Cloudflare challenge
|
|
47
|
+
if (html.includes('Just a moment...') ||
|
|
48
|
+
html.includes('cdn-cgi/challenge-platform') ||
|
|
49
|
+
html.includes('cf-browser-verification')) {
|
|
50
|
+
return 'cloudflare';
|
|
51
|
+
}
|
|
52
|
+
// reCAPTCHA
|
|
53
|
+
if (html.includes('g-recaptcha') ||
|
|
54
|
+
html.includes('google.com/recaptcha')) {
|
|
55
|
+
return 'recaptcha';
|
|
56
|
+
}
|
|
57
|
+
// hCaptcha
|
|
58
|
+
if (html.includes('h-captcha') ||
|
|
59
|
+
html.includes('hcaptcha.com')) {
|
|
60
|
+
return 'hcaptcha';
|
|
61
|
+
}
|
|
62
|
+
return null;
|
|
63
|
+
}
|
|
64
|
+
/**
|
|
65
|
+
* Refresh tokens by spawning a browser and intercepting requests.
|
|
66
|
+
*
|
|
67
|
+
* This is the main entry point for token refresh. It:
|
|
68
|
+
* 1. Launches a browser (visible if captchaRisk is true)
|
|
69
|
+
* 2. Navigates to the refresh URL
|
|
70
|
+
* 3. Intercepts outgoing requests to capture fresh token values
|
|
71
|
+
* 4. Returns the captured tokens
|
|
72
|
+
*
|
|
73
|
+
* @param skill - Skill file with auth config and endpoint info
|
|
74
|
+
* @param authManager - Auth manager for storing refreshed tokens
|
|
75
|
+
* @param options - Refresh options
|
|
76
|
+
*/
|
|
77
|
+
export async function refreshTokens(skill, authManager, options) {
|
|
78
|
+
const { domain } = options;
|
|
79
|
+
// Check for existing refresh in progress (mutex)
|
|
80
|
+
const existingRefresh = refreshLocks.get(domain);
|
|
81
|
+
if (existingRefresh) {
|
|
82
|
+
return existingRefresh;
|
|
83
|
+
}
|
|
84
|
+
const refreshPromise = doRefresh(skill, authManager, options);
|
|
85
|
+
refreshLocks.set(domain, refreshPromise);
|
|
86
|
+
try {
|
|
87
|
+
return await refreshPromise;
|
|
88
|
+
}
|
|
89
|
+
finally {
|
|
90
|
+
refreshLocks.delete(domain);
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
async function doRefresh(skill, authManager, options) {
|
|
94
|
+
let oauthRefreshed = false;
|
|
95
|
+
// Step 1: OAuth path — if oauthConfig + stored credentials available
|
|
96
|
+
const oauthConfig = skill.auth?.oauthConfig;
|
|
97
|
+
if (oauthConfig) {
|
|
98
|
+
const oauthCreds = await authManager.retrieveOAuthCredentials(options.domain);
|
|
99
|
+
const canOAuth = (oauthConfig.grantType === 'refresh_token' && oauthCreds?.refreshToken) ||
|
|
100
|
+
(oauthConfig.grantType === 'client_credentials');
|
|
101
|
+
if (canOAuth) {
|
|
102
|
+
const oauthResult = await refreshOAuth(options.domain, oauthConfig, authManager, { _skipSsrfCheck: options._skipSsrfCheck });
|
|
103
|
+
if (oauthResult.success) {
|
|
104
|
+
oauthRefreshed = true;
|
|
105
|
+
}
|
|
106
|
+
else {
|
|
107
|
+
// OAuth failed — fall through to browser path if available
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
// Step 2: Browser path — if refreshable tokens exist or refreshUrl is present
|
|
112
|
+
const tokenNames = new Set();
|
|
113
|
+
for (const endpoint of skill.endpoints) {
|
|
114
|
+
if (endpoint.requestBody?.refreshableTokens) {
|
|
115
|
+
for (const name of endpoint.requestBody.refreshableTokens) {
|
|
116
|
+
tokenNames.add(name);
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
const needsBrowser = tokenNames.size > 0 || (skill.auth?.refreshUrl && !oauthRefreshed);
|
|
121
|
+
if (!needsBrowser) {
|
|
122
|
+
// No browser refresh needed — return OAuth result
|
|
123
|
+
return { success: oauthRefreshed, tokens: {}, oauthRefreshed: oauthRefreshed || undefined };
|
|
124
|
+
}
|
|
125
|
+
return doBrowserRefresh(skill, authManager, options, tokenNames, oauthRefreshed);
|
|
126
|
+
}
|
|
127
|
+
async function doBrowserRefresh(skill, authManager, options, tokenNames, oauthRefreshed) {
|
|
128
|
+
if (tokenNames.size === 0 && !skill.auth?.refreshUrl) {
|
|
129
|
+
return { success: oauthRefreshed, tokens: {}, oauthRefreshed: oauthRefreshed || undefined };
|
|
130
|
+
}
|
|
131
|
+
const { chromium } = await import('playwright');
|
|
132
|
+
const browserMode = options.browserMode || skill.auth?.browserMode || 'headless';
|
|
133
|
+
const refreshUrl = options.refreshUrl || skill.auth?.refreshUrl || skill.baseUrl;
|
|
134
|
+
const timeout = options.timeout || (skill.auth?.captchaRisk ? 300_000 : 30_000);
|
|
135
|
+
// Try to restore session from cache
|
|
136
|
+
const cachedSession = await authManager.retrieveSession(options.domain);
|
|
137
|
+
const sessionValid = cachedSession && isSessionValid(cachedSession);
|
|
138
|
+
const browser = await chromium.launch({
|
|
139
|
+
headless: browserMode === 'headless',
|
|
140
|
+
});
|
|
141
|
+
try {
|
|
142
|
+
const context = await browser.newContext();
|
|
143
|
+
// Restore cookies if session is valid
|
|
144
|
+
if (sessionValid && cachedSession) {
|
|
145
|
+
await context.addCookies(cachedSession.cookies);
|
|
146
|
+
}
|
|
147
|
+
const page = await context.newPage();
|
|
148
|
+
const capturedTokens = {};
|
|
149
|
+
let captchaDetected = null;
|
|
150
|
+
// Intercept requests to capture token values
|
|
151
|
+
if (tokenNames.size > 0) {
|
|
152
|
+
page.on('request', (request) => {
|
|
153
|
+
const body = request.postData();
|
|
154
|
+
if (body) {
|
|
155
|
+
const extracted = extractTokensFromRequest(body, [...tokenNames]);
|
|
156
|
+
Object.assign(capturedTokens, extracted);
|
|
157
|
+
}
|
|
158
|
+
});
|
|
159
|
+
}
|
|
160
|
+
// Navigate and wait for network idle
|
|
161
|
+
await page.goto(refreshUrl, { waitUntil: 'networkidle', timeout });
|
|
162
|
+
// Check for captcha
|
|
163
|
+
const content = await page.content();
|
|
164
|
+
captchaDetected = detectCaptcha(content);
|
|
165
|
+
if (captchaDetected) {
|
|
166
|
+
// Extended timeout for captcha solving
|
|
167
|
+
console.error(`\u26a0\ufe0f Captcha detected (${captchaDetected}). Please solve it in the browser window.`);
|
|
168
|
+
await page.waitForTimeout(timeout);
|
|
169
|
+
// Re-check for tokens after captcha
|
|
170
|
+
// User interaction will trigger requests containing tokens
|
|
171
|
+
}
|
|
172
|
+
// Wait a bit for any final requests
|
|
173
|
+
await page.waitForTimeout(2000);
|
|
174
|
+
// Save session for next time
|
|
175
|
+
const cookies = await context.cookies();
|
|
176
|
+
await authManager.storeSession(options.domain, {
|
|
177
|
+
cookies,
|
|
178
|
+
savedAt: new Date().toISOString(),
|
|
179
|
+
maxAgeMs: 24 * 60 * 60 * 1000, // 24 hours
|
|
180
|
+
});
|
|
181
|
+
// Store captured tokens
|
|
182
|
+
if (Object.keys(capturedTokens).length > 0) {
|
|
183
|
+
const storedTokens = {};
|
|
184
|
+
for (const [name, value] of Object.entries(capturedTokens)) {
|
|
185
|
+
storedTokens[name] = {
|
|
186
|
+
value,
|
|
187
|
+
refreshedAt: new Date().toISOString(),
|
|
188
|
+
};
|
|
189
|
+
}
|
|
190
|
+
await authManager.storeTokens(options.domain, storedTokens);
|
|
191
|
+
}
|
|
192
|
+
const browserSuccess = tokenNames.size === 0 || Object.keys(capturedTokens).length > 0;
|
|
193
|
+
return {
|
|
194
|
+
success: oauthRefreshed || browserSuccess,
|
|
195
|
+
tokens: capturedTokens,
|
|
196
|
+
captchaDetected: captchaDetected || undefined,
|
|
197
|
+
oauthRefreshed: oauthRefreshed || undefined,
|
|
198
|
+
};
|
|
199
|
+
}
|
|
200
|
+
catch (error) {
|
|
201
|
+
return {
|
|
202
|
+
success: oauthRefreshed, // OAuth may have succeeded even if browser failed
|
|
203
|
+
tokens: {},
|
|
204
|
+
error: error instanceof Error ? error.message : String(error),
|
|
205
|
+
oauthRefreshed: oauthRefreshed || undefined,
|
|
206
|
+
};
|
|
207
|
+
}
|
|
208
|
+
finally {
|
|
209
|
+
await browser.close();
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
function isSessionValid(session) {
|
|
213
|
+
const maxAge = session.maxAgeMs || 24 * 60 * 60 * 1000;
|
|
214
|
+
const savedAt = new Date(session.savedAt).getTime();
|
|
215
|
+
return Date.now() - savedAt < maxAge;
|
|
216
|
+
}
|
|
217
|
+
//# sourceMappingURL=refresh.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"refresh.js","sourceRoot":"","sources":["../../src/auth/refresh.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,YAAY,EAA2B,MAAM,oBAAoB,CAAC;AAmB3E,4DAA4D;AAC5D,MAAM,YAAY,GAAG,IAAI,GAAG,EAAkC,CAAC;AAE/D;;;;;;GAMG;AACH,MAAM,UAAU,wBAAwB,CACtC,IAAY,EACZ,UAAoB;IAEpB,MAAM,MAAM,GAA2B,EAAE,CAAC;IAE1C,IAAI,MAAe,CAAC;IACpB,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC5B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,IAAI,EAAE,CAAC;QAClD,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;QAC9B,MAAM,KAAK,GAAG,cAAc,CAAC,MAAiC,EAAE,IAAI,CAAC,CAAC;QACtE,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;YAC9B,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC;QACvB,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,cAAc,CAAC,GAA4B,EAAE,IAAY;IAChE,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC9B,IAAI,OAAO,GAAY,GAAG,CAAC;IAE3B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,OAAO,OAAO,KAAK,QAAQ,IAAI,OAAO,KAAK,IAAI,EAAE,CAAC;YACpD,OAAO,SAAS,CAAC;QACnB,CAAC;QACD,OAAO,GAAI,OAAmC,CAAC,IAAI,CAAC,CAAC;IACvD,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,aAAa,CAC3B,IAAY;IAEZ,uBAAuB;IACvB,IACE,IAAI,CAAC,QAAQ,CAAC,kBAAkB,CAAC;QACjC,IAAI,CAAC,QAAQ,CAAC,4BAA4B,CAAC;QAC3C,IAAI,CAAC,QAAQ,CAAC,yBAAyB,CAAC,EACxC,CAAC;QACD,OAAO,YAAY,CAAC;IACtB,CAAC;IAED,YAAY;IACZ,IACE,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC;QAC5B,IAAI,CAAC,QAAQ,CAAC,sBAAsB,CAAC,EACrC,CAAC;QACD,OAAO,WAAW,CAAC;IACrB,CAAC;IAED,WAAW;IACX,IACE,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC;QAC1B,IAAI,CAAC,QAAQ,CAAC,cAAc,CAAC,EAC7B,CAAC;QACD,OAAO,UAAU,CAAC;IACpB,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,KAAgB,EAChB,WAAwB,EACxB,OAAuB;IAEvB,MAAM,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;IAE3B,iDAAiD;IACjD,MAAM,eAAe,GAAG,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IACjD,IAAI,eAAe,EAAE,CAAC;QACpB,OAAO,eAAe,CAAC;IACzB,CAAC;IAED,MAAM,cAAc,GAAG,SAAS,CAAC,KAAK,EAAE,WAAW,EAAE,OAAO,CAAC,CAAC;IAC9D,YAAY,CAAC,GAAG,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC;IAEzC,IAAI,CAAC;QACH,OAAO,MAAM,cAAc,CAAC;IAC9B,CAAC;YAAS,CAAC;QACT,YAAY,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAC9B,CAAC;AACH,CAAC;AAED,KAAK,UAAU,SAAS,CACtB,KAAgB,EAChB,WAAwB,EACxB,OAAuB;IAEvB,IAAI,cAAc,GAAG,KAAK,CAAC;IAE3B,qEAAqE;IACrE,MAAM,WAAW,GAAG,KAAK,CAAC,IAAI,EAAE,WAAW,CAAC;IAC5C,IAAI,WAAW,EAAE,CAAC;QAChB,MAAM,UAAU,GAAG,MAAM,WAAW,CAAC,wBAAwB,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAC9E,MAAM,QAAQ,GACZ,CAAC,WAAW,CAAC,SAAS,KAAK,eAAe,IAAI,UAAU,EAAE,YAAY,CAAC;YACvE,CAAC,WAAW,CAAC,SAAS,KAAK,oBAAoB,CAAC,CAAC;QAEnD,IAAI,QAAQ,EAAE,CAAC;YACb,MAAM,WAAW,GAAG,MAAM,YAAY,CAAC,OAAO,CAAC,MAAM,EAAE,WAAW,EAAE,WAAW,EAAE,EAAE,cAAc,EAAE,OAAO,CAAC,cAAc,EAAE,CAAC,CAAC;YAC7H,IAAI,WAAW,CAAC,OAAO,EAAE,CAAC;gBACxB,cAAc,GAAG,IAAI,CAAC;YACxB,CAAC;iBAAM,CAAC;gBACN,2DAA2D;YAC7D,CAAC;QACH,CAAC;IACH,CAAC;IAED,8EAA8E;IAC9E,MAAM,UAAU,GAAG,IAAI,GAAG,EAAU,CAAC;IACrC,KAAK,MAAM,QAAQ,IAAI,KAAK,CAAC,SAAS,EAAE,CAAC;QACvC,IAAI,QAAQ,CAAC,WAAW,EAAE,iBAAiB,EAAE,CAAC;YAC5C,KAAK,MAAM,IAAI,IAAI,QAAQ,CAAC,WAAW,CAAC,iBAAiB,EAAE,CAAC;gBAC1D,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YACvB,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,YAAY,GAAG,UAAU,CAAC,IAAI,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,UAAU,IAAI,CAAC,cAAc,CAAC,CAAC;IAExF,IAAI,CAAC,YAAY,EAAE,CAAC;QAClB,kDAAkD;QAClD,OAAO,EAAE,OAAO,EAAE,cAAc,EAAE,MAAM,EAAE,EAAE,EAAE,cAAc,EAAE,cAAc,IAAI,SAAS,EAAE,CAAC;IAC9F,CAAC;IAED,OAAO,gBAAgB,CAAC,KAAK,EAAE,WAAW,EAAE,OAAO,EAAE,UAAU,EAAE,cAAc,CAAC,CAAC;AACnF,CAAC;AAED,KAAK,UAAU,gBAAgB,CAC7B,KAAgB,EAChB,WAAwB,EACxB,OAAuB,EACvB,UAAuB,EACvB,cAAuB;IAEvB,IAAI,UAAU,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,UAAU,EAAE,CAAC;QACrD,OAAO,EAAE,OAAO,EAAE,cAAc,EAAE,MAAM,EAAE,EAAE,EAAE,cAAc,EAAE,cAAc,IAAI,SAAS,EAAE,CAAC;IAC9F,CAAC;IAED,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,MAAM,CAAC,YAAY,CAAC,CAAC;IAEhD,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,KAAK,CAAC,IAAI,EAAE,WAAW,IAAI,UAAU,CAAC;IACjF,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,KAAK,CAAC,IAAI,EAAE,UAAU,IAAI,KAAK,CAAC,OAAO,CAAC;IACjF,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;IAEhF,oCAAoC;IACpC,MAAM,aAAa,GAAG,MAAM,WAAW,CAAC,eAAe,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IACxE,MAAM,YAAY,GAAG,aAAa,IAAI,cAAc,CAAC,aAAa,CAAC,CAAC;IAEpE,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC;QACpC,QAAQ,EAAE,WAAW,KAAK,UAAU;KACrC,CAAC,CAAC;IAEH,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,UAAU,EAAE,CAAC;QAE3C,sCAAsC;QACtC,IAAI,YAAY,IAAI,aAAa,EAAE,CAAC;YAClC,MAAM,OAAO,CAAC,UAAU,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;QAClD,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;QACrC,MAAM,cAAc,GAA2B,EAAE,CAAC;QAClD,IAAI,eAAe,GAAmD,IAAI,CAAC;QAE3E,6CAA6C;QAC7C,IAAI,UAAU,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;YACxB,IAAI,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,OAAO,EAAE,EAAE;gBAC7B,MAAM,IAAI,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;gBAChC,IAAI,IAAI,EAAE,CAAC;oBACT,MAAM,SAAS,GAAG,wBAAwB,CAAC,IAAI,EAAE,CAAC,GAAG,UAAU,CAAC,CAAC,CAAC;oBAClE,MAAM,CAAC,MAAM,CAAC,cAAc,EAAE,SAAS,CAAC,CAAC;gBAC3C,CAAC;YACH,CAAC,CAAC,CAAC;QACL,CAAC;QAED,qCAAqC;QACrC,MAAM,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,aAAa,EAAE,OAAO,EAAE,CAAC,CAAC;QAEnE,oBAAoB;QACpB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;QACrC,eAAe,GAAG,aAAa,CAAC,OAAO,CAAC,CAAC;QAEzC,IAAI,eAAe,EAAE,CAAC;YACpB,uCAAuC;YACvC,OAAO,CAAC,KAAK,CAAC,mCAAmC,eAAe,2CAA2C,CAAC,CAAC;YAC7G,MAAM,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;YAEnC,oCAAoC;YACpC,2DAA2D;QAC7D,CAAC;QAED,oCAAoC;QACpC,MAAM,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;QAEhC,6BAA6B;QAC7B,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;QACxC,MAAM,WAAW,CAAC,YAAY,CAAC,OAAO,CAAC,MAAM,EAAE;YAC7C,OAAO;YACP,OAAO,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACjC,QAAQ,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,EAAE,WAAW;SAC3C,CAAC,CAAC;QAEH,wBAAwB;QACxB,IAAI,MAAM,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC3C,MAAM,YAAY,GAAgC,EAAE,CAAC;YACrD,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,cAAc,CAAC,EAAE,CAAC;gBAC3D,YAAY,CAAC,IAAI,CAAC,GAAG;oBACnB,KAAK;oBACL,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;iBACtC,CAAC;YACJ,CAAC;YACD,MAAM,WAAW,CAAC,WAAW,CAAC,OAAO,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;QAC9D,CAAC;QAED,MAAM,cAAc,GAAG,UAAU,CAAC,IAAI,KAAK,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;QAEvF,OAAO;YACL,OAAO,EAAE,cAAc,IAAI,cAAc;YACzC,MAAM,EAAE,cAAc;YACtB,eAAe,EAAE,eAAe,IAAI,SAAS;YAC7C,cAAc,EAAE,cAAc,IAAI,SAAS;SAC5C,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO;YACL,OAAO,EAAE,cAAc,EAAE,kDAAkD;YAC3E,MAAM,EAAE,EAAE;YACV,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;YAC7D,cAAc,EAAE,cAAc,IAAI,SAAS;SAC5C,CAAC;IACJ,CAAC;YAAS,CAAC;QACT,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;IACxB,CAAC;AACH,CAAC;AAED,SAAS,cAAc,CAAC,OAAsB;IAC5C,MAAM,MAAM,GAAG,OAAO,CAAC,QAAQ,IAAI,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;IACvD,MAAM,OAAO,GAAG,IAAI,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,CAAC;IACpD,OAAO,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,GAAG,MAAM,CAAC;AACvC,CAAC"}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
export type AntiBotSignal = 'cloudflare' | 'akamai' | 'rate-limited' | 'captcha' | 'challenge';
|
|
2
|
+
export interface AntiBotResult {
|
|
3
|
+
detected: boolean;
|
|
4
|
+
signals: AntiBotSignal[];
|
|
5
|
+
}
|
|
6
|
+
/**
|
|
7
|
+
* Detect anti-bot protection signals from response headers and body.
|
|
8
|
+
*/
|
|
9
|
+
export declare function detectAntiBot(options: {
|
|
10
|
+
headers: Record<string, string>;
|
|
11
|
+
cookies?: string;
|
|
12
|
+
body?: string;
|
|
13
|
+
status?: number;
|
|
14
|
+
contentType?: string;
|
|
15
|
+
}): AntiBotResult;
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
// src/capture/anti-bot.ts
|
|
2
|
+
/**
|
|
3
|
+
* Detect anti-bot protection signals from response headers and body.
|
|
4
|
+
*/
|
|
5
|
+
export function detectAntiBot(options) {
|
|
6
|
+
const signals = [];
|
|
7
|
+
const { headers, cookies, body, status, contentType } = options;
|
|
8
|
+
const headerLower = lowerKeys(headers);
|
|
9
|
+
// Cloudflare: cf-ray header or __cf_bm cookie
|
|
10
|
+
if (headerLower['cf-ray'] || headerLower['cf-cache-status'] || cookies?.includes('__cf_bm')) {
|
|
11
|
+
signals.push('cloudflare');
|
|
12
|
+
}
|
|
13
|
+
// Akamai: _abck cookie
|
|
14
|
+
if (cookies?.includes('_abck')) {
|
|
15
|
+
signals.push('akamai');
|
|
16
|
+
}
|
|
17
|
+
// Rate limiting: X-RateLimit-* or Retry-After
|
|
18
|
+
if (headerLower['retry-after'] ||
|
|
19
|
+
Object.keys(headerLower).some(k => k.startsWith('x-ratelimit'))) {
|
|
20
|
+
signals.push('rate-limited');
|
|
21
|
+
}
|
|
22
|
+
// CAPTCHA in response body
|
|
23
|
+
if (body && /\b(captcha|hcaptcha|recaptcha|cf-turnstile)\b/i.test(body)) {
|
|
24
|
+
signals.push('captcha');
|
|
25
|
+
}
|
|
26
|
+
// Challenge page: HTML response when JSON expected + 403
|
|
27
|
+
if (status === 403 && contentType?.includes('text/html') &&
|
|
28
|
+
!contentType.includes('json')) {
|
|
29
|
+
signals.push('challenge');
|
|
30
|
+
}
|
|
31
|
+
return {
|
|
32
|
+
detected: signals.length > 0,
|
|
33
|
+
signals: [...new Set(signals)],
|
|
34
|
+
};
|
|
35
|
+
}
|
|
36
|
+
function lowerKeys(obj) {
|
|
37
|
+
const result = {};
|
|
38
|
+
for (const [k, v] of Object.entries(obj)) {
|
|
39
|
+
result[k.toLowerCase()] = v;
|
|
40
|
+
}
|
|
41
|
+
return result;
|
|
42
|
+
}
|
|
43
|
+
//# sourceMappingURL=anti-bot.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"anti-bot.js","sourceRoot":"","sources":["../../src/capture/anti-bot.ts"],"names":[],"mappings":"AAAA,0BAA0B;AAS1B;;GAEG;AACH,MAAM,UAAU,aAAa,CAAC,OAM7B;IACC,MAAM,OAAO,GAAoB,EAAE,CAAC;IACpC,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,OAAO,CAAC;IAChE,MAAM,WAAW,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC;IAEvC,8CAA8C;IAC9C,IAAI,WAAW,CAAC,QAAQ,CAAC,IAAI,WAAW,CAAC,iBAAiB,CAAC,IAAI,OAAO,EAAE,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;QAC5F,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IAC7B,CAAC;IAED,uBAAuB;IACvB,IAAI,OAAO,EAAE,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;QAC/B,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACzB,CAAC;IAED,8CAA8C;IAC9C,IAAI,WAAW,CAAC,aAAa,CAAC;QAC1B,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC,EAAE,CAAC;QACpE,OAAO,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;IAC/B,CAAC;IAED,2BAA2B;IAC3B,IAAI,IAAI,IAAI,gDAAgD,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QACxE,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IAC1B,CAAC;IAED,yDAAyD;IACzD,IAAI,MAAM,KAAK,GAAG,IAAI,WAAW,EAAE,QAAQ,CAAC,WAAW,CAAC;QACpD,CAAC,WAAW,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;QAClC,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IAC5B,CAAC;IAED,OAAO;QACL,QAAQ,EAAE,OAAO,CAAC,MAAM,GAAG,CAAC;QAC5B,OAAO,EAAE,CAAC,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC;KAC/B,CAAC;AACJ,CAAC;AAED,SAAS,SAAS,CAAC,GAA2B;IAC5C,MAAM,MAAM,GAA2B,EAAE,CAAC;IAC1C,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;QACzC,MAAM,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,GAAG,CAAC,CAAC;IAC9B,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
// src/capture/blocklist.ts
|
|
2
|
+
const BLOCKLIST = new Set([
|
|
3
|
+
// Analytics
|
|
4
|
+
'google-analytics.com',
|
|
5
|
+
'analytics.google.com',
|
|
6
|
+
'googletagmanager.com',
|
|
7
|
+
'segment.io',
|
|
8
|
+
'cdn.segment.com',
|
|
9
|
+
'mixpanel.com',
|
|
10
|
+
'amplitude.com',
|
|
11
|
+
'hotjar.com',
|
|
12
|
+
'heapanalytics.com',
|
|
13
|
+
'plausible.io',
|
|
14
|
+
'posthog.com',
|
|
15
|
+
'clarity.ms',
|
|
16
|
+
'fullstory.com',
|
|
17
|
+
// Ads
|
|
18
|
+
'doubleclick.net',
|
|
19
|
+
'googlesyndication.com',
|
|
20
|
+
'googleadservices.com',
|
|
21
|
+
'facebook.net',
|
|
22
|
+
'connect.facebook.net',
|
|
23
|
+
'adsrvr.org',
|
|
24
|
+
'adnxs.com',
|
|
25
|
+
'criteo.com',
|
|
26
|
+
'outbrain.com',
|
|
27
|
+
'taboola.com',
|
|
28
|
+
// Error tracking / monitoring
|
|
29
|
+
'sentry.io',
|
|
30
|
+
'datadoghq.com',
|
|
31
|
+
'browser-intake-datadoghq.com',
|
|
32
|
+
'newrelic.com',
|
|
33
|
+
'bam.nr-data.net',
|
|
34
|
+
'logrocket.com',
|
|
35
|
+
'logr-ingest.com',
|
|
36
|
+
'bugsnag.com',
|
|
37
|
+
'rollbar.com',
|
|
38
|
+
// Social tracking
|
|
39
|
+
'bat.bing.com',
|
|
40
|
+
'ct.pinterest.com',
|
|
41
|
+
'snap.licdn.com',
|
|
42
|
+
'px.ads.linkedin.com',
|
|
43
|
+
'analytics.twitter.com',
|
|
44
|
+
'analytics.tiktok.com',
|
|
45
|
+
// Customer engagement
|
|
46
|
+
'intercom.io',
|
|
47
|
+
'widget.intercom.io',
|
|
48
|
+
'api-iam.intercom.io',
|
|
49
|
+
'zendesk.com',
|
|
50
|
+
'drift.com',
|
|
51
|
+
'crisp.chat',
|
|
52
|
+
]);
|
|
53
|
+
/**
|
|
54
|
+
* Check if a hostname is on the blocklist.
|
|
55
|
+
* Matches exact hostnames and subdomains of blocklisted domains.
|
|
56
|
+
* e.g. "sentry.io" blocks "o123.ingest.sentry.io"
|
|
57
|
+
*/
|
|
58
|
+
export function isBlocklisted(hostname) {
|
|
59
|
+
if (BLOCKLIST.has(hostname))
|
|
60
|
+
return true;
|
|
61
|
+
// Check parent domains: "a.b.sentry.io" → "b.sentry.io" → "sentry.io"
|
|
62
|
+
const parts = hostname.split('.');
|
|
63
|
+
for (let i = 1; i < parts.length - 1; i++) {
|
|
64
|
+
const parent = parts.slice(i).join('.');
|
|
65
|
+
if (BLOCKLIST.has(parent))
|
|
66
|
+
return true;
|
|
67
|
+
}
|
|
68
|
+
return false;
|
|
69
|
+
}
|
|
70
|
+
//# sourceMappingURL=blocklist.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"blocklist.js","sourceRoot":"","sources":["../../src/capture/blocklist.ts"],"names":[],"mappings":"AAAA,2BAA2B;AAE3B,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC;IACxB,YAAY;IACZ,sBAAsB;IACtB,sBAAsB;IACtB,sBAAsB;IACtB,YAAY;IACZ,iBAAiB;IACjB,cAAc;IACd,eAAe;IACf,YAAY;IACZ,mBAAmB;IACnB,cAAc;IACd,aAAa;IACb,YAAY;IACZ,eAAe;IAEf,MAAM;IACN,iBAAiB;IACjB,uBAAuB;IACvB,sBAAsB;IACtB,cAAc;IACd,sBAAsB;IACtB,YAAY;IACZ,WAAW;IACX,YAAY;IACZ,cAAc;IACd,aAAa;IAEb,8BAA8B;IAC9B,WAAW;IACX,eAAe;IACf,8BAA8B;IAC9B,cAAc;IACd,iBAAiB;IACjB,eAAe;IACf,iBAAiB;IACjB,aAAa;IACb,aAAa;IAEb,kBAAkB;IAClB,cAAc;IACd,kBAAkB;IAClB,gBAAgB;IAChB,qBAAqB;IACrB,uBAAuB;IACvB,sBAAsB;IAEtB,sBAAsB;IACtB,aAAa;IACb,oBAAoB;IACpB,qBAAqB;IACrB,aAAa;IACb,WAAW;IACX,YAAY;CACb,CAAC,CAAC;AAEH;;;;GAIG;AACH,MAAM,UAAU,aAAa,CAAC,QAAgB;IAC5C,IAAI,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC;QAAE,OAAO,IAAI,CAAC;IAEzC,sEAAsE;IACtE,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAClC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC1C,MAAM,MAAM,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACxC,IAAI,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC;YAAE,OAAO,IAAI,CAAC;IACzC,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC"}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Cross-request body diffing (Strategy 1).
|
|
3
|
+
*
|
|
4
|
+
* Compare request bodies across multiple captures of the same endpoint.
|
|
5
|
+
* Any field whose value changed between requests is dynamic by definition.
|
|
6
|
+
* Returns JSON paths of changed fields.
|
|
7
|
+
*/
|
|
8
|
+
export declare function diffBodies(bodies: string[]): string[];
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
// src/capture/body-diff.ts
|
|
2
|
+
/**
|
|
3
|
+
* Cross-request body diffing (Strategy 1).
|
|
4
|
+
*
|
|
5
|
+
* Compare request bodies across multiple captures of the same endpoint.
|
|
6
|
+
* Any field whose value changed between requests is dynamic by definition.
|
|
7
|
+
* Returns JSON paths of changed fields.
|
|
8
|
+
*/
|
|
9
|
+
export function diffBodies(bodies) {
|
|
10
|
+
if (bodies.length < 2)
|
|
11
|
+
return [];
|
|
12
|
+
// Try JSON first
|
|
13
|
+
const parsed = [];
|
|
14
|
+
for (const body of bodies) {
|
|
15
|
+
try {
|
|
16
|
+
parsed.push(JSON.parse(body));
|
|
17
|
+
}
|
|
18
|
+
catch {
|
|
19
|
+
// Fall back to form-encoded diffing
|
|
20
|
+
return diffFormEncoded(bodies);
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
// All parsed as JSON — diff objects
|
|
24
|
+
const changed = new Set();
|
|
25
|
+
for (let i = 1; i < parsed.length; i++) {
|
|
26
|
+
diffObjects(parsed[0], parsed[i], '', changed);
|
|
27
|
+
}
|
|
28
|
+
return [...changed].sort();
|
|
29
|
+
}
|
|
30
|
+
function diffObjects(a, b, prefix, changed) {
|
|
31
|
+
// Different types → the whole path is dynamic
|
|
32
|
+
if (typeof a !== typeof b || Array.isArray(a) !== Array.isArray(b)) {
|
|
33
|
+
if (prefix)
|
|
34
|
+
changed.add(prefix);
|
|
35
|
+
return;
|
|
36
|
+
}
|
|
37
|
+
// Both arrays
|
|
38
|
+
if (Array.isArray(a) && Array.isArray(b)) {
|
|
39
|
+
if (a.length !== b.length) {
|
|
40
|
+
// Different lengths → mark whole array as dynamic
|
|
41
|
+
if (prefix)
|
|
42
|
+
changed.add(prefix);
|
|
43
|
+
return;
|
|
44
|
+
}
|
|
45
|
+
for (let i = 0; i < a.length; i++) {
|
|
46
|
+
diffObjects(a[i], b[i], prefix ? `${prefix}[${i}]` : `[${i}]`, changed);
|
|
47
|
+
}
|
|
48
|
+
return;
|
|
49
|
+
}
|
|
50
|
+
// Both objects
|
|
51
|
+
if (a && b && typeof a === 'object' && typeof b === 'object' && !Array.isArray(a)) {
|
|
52
|
+
const aObj = a;
|
|
53
|
+
const bObj = b;
|
|
54
|
+
const allKeys = new Set([...Object.keys(aObj), ...Object.keys(bObj)]);
|
|
55
|
+
for (const key of allKeys) {
|
|
56
|
+
const path = prefix ? `${prefix}.${key}` : key;
|
|
57
|
+
if (!(key in aObj) || !(key in bObj)) {
|
|
58
|
+
// Key only exists in one → dynamic
|
|
59
|
+
changed.add(path);
|
|
60
|
+
}
|
|
61
|
+
else {
|
|
62
|
+
diffObjects(aObj[key], bObj[key], path, changed);
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
return;
|
|
66
|
+
}
|
|
67
|
+
// Primitive comparison
|
|
68
|
+
if (a !== b && prefix) {
|
|
69
|
+
changed.add(prefix);
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
function diffFormEncoded(bodies) {
|
|
73
|
+
const parsed = bodies.map(parseFormEncoded);
|
|
74
|
+
if (parsed.some(m => m.size === 0))
|
|
75
|
+
return [];
|
|
76
|
+
const changed = new Set();
|
|
77
|
+
const firstMap = parsed[0];
|
|
78
|
+
for (let i = 1; i < parsed.length; i++) {
|
|
79
|
+
const otherMap = parsed[i];
|
|
80
|
+
const allKeys = new Set([...firstMap.keys(), ...otherMap.keys()]);
|
|
81
|
+
for (const key of allKeys) {
|
|
82
|
+
if (firstMap.get(key) !== otherMap.get(key)) {
|
|
83
|
+
changed.add(key);
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
return [...changed].sort();
|
|
88
|
+
}
|
|
89
|
+
function parseFormEncoded(body) {
|
|
90
|
+
const map = new Map();
|
|
91
|
+
const pairs = body.split('&');
|
|
92
|
+
for (const pair of pairs) {
|
|
93
|
+
const idx = pair.indexOf('=');
|
|
94
|
+
if (idx === -1)
|
|
95
|
+
continue;
|
|
96
|
+
const key = decodeURIComponent(pair.slice(0, idx).replace(/\+/g, ' '));
|
|
97
|
+
const val = decodeURIComponent(pair.slice(idx + 1).replace(/\+/g, ' '));
|
|
98
|
+
map.set(key, val);
|
|
99
|
+
}
|
|
100
|
+
return map;
|
|
101
|
+
}
|
|
102
|
+
//# sourceMappingURL=body-diff.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"body-diff.js","sourceRoot":"","sources":["../../src/capture/body-diff.ts"],"names":[],"mappings":"AAAA,2BAA2B;AAE3B;;;;;;GAMG;AACH,MAAM,UAAU,UAAU,CAAC,MAAgB;IACzC,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,EAAE,CAAC;IAEjC,iBAAiB;IACjB,MAAM,MAAM,GAAc,EAAE,CAAC;IAC7B,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;QAC1B,IAAI,CAAC;YACH,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;QAChC,CAAC;QAAC,MAAM,CAAC;YACP,oCAAoC;YACpC,OAAO,eAAe,CAAC,MAAM,CAAC,CAAC;QACjC,CAAC;IACH,CAAC;IAED,oCAAoC;IACpC,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;IAClC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,OAAO,CAAC,CAAC;IACjD,CAAC;IACD,OAAO,CAAC,GAAG,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC;AAC7B,CAAC;AAED,SAAS,WAAW,CAAC,CAAU,EAAE,CAAU,EAAE,MAAc,EAAE,OAAoB;IAC/E,8CAA8C;IAC9C,IAAI,OAAO,CAAC,KAAK,OAAO,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;QACnE,IAAI,MAAM;YAAE,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAChC,OAAO;IACT,CAAC;IAED,cAAc;IACd,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;QACzC,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,MAAM,EAAE,CAAC;YAC1B,kDAAkD;YAClD,IAAI,MAAM;gBAAE,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YAChC,OAAO;QACT,CAAC;QACD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAClC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QAC1E,CAAC;QACD,OAAO;IACT,CAAC;IAED,eAAe;IACf,IAAI,CAAC,IAAI,CAAC,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;QAClF,MAAM,IAAI,GAAG,CAA4B,CAAC;QAC1C,MAAM,IAAI,GAAG,CAA4B,CAAC;QAC1C,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAEtE,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;YAC1B,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,IAAI,GAAG,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;YAE/C,IAAI,CAAC,CAAC,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,IAAI,IAAI,CAAC,EAAE,CAAC;gBACrC,mCAAmC;gBACnC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YACpB,CAAC;iBAAM,CAAC;gBACN,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;YACnD,CAAC;QACH,CAAC;QACD,OAAO;IACT,CAAC;IAED,uBAAuB;IACvB,IAAI,CAAC,KAAK,CAAC,IAAI,MAAM,EAAE,CAAC;QACtB,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IACtB,CAAC;AACH,CAAC;AAED,SAAS,eAAe,CAAC,MAAgB;IACvC,MAAM,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;IAC5C,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC;QAAE,OAAO,EAAE,CAAC;IAE9C,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;IAClC,MAAM,QAAQ,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;IAE3B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,MAAM,QAAQ,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QAC3B,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,QAAQ,CAAC,IAAI,EAAE,EAAE,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QAElE,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;YAC1B,IAAI,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC5C,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACnB,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,CAAC,GAAG,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC;AAC7B,CAAC;AAED,SAAS,gBAAgB,CAAC,IAAY;IACpC,MAAM,GAAG,GAAG,IAAI,GAAG,EAAkB,CAAC;IACtC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC9B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAC9B,IAAI,GAAG,KAAK,CAAC,CAAC;YAAE,SAAS;QACzB,MAAM,GAAG,GAAG,kBAAkB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC;QACvE,MAAM,GAAG,GAAG,kBAAkB,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC;QACxE,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;IACpB,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC"}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Detect which fields in a JSON body are likely dynamic variables.
|
|
3
|
+
* Uses three strategies:
|
|
4
|
+
* Strategy 1 (cross-request diffing) is in body-diff.ts
|
|
5
|
+
* Strategy 2: Name-based key heuristics
|
|
6
|
+
* Strategy 3: Pattern-based value detection
|
|
7
|
+
* Plus existing: numeric values, UUIDs, base64 cursors, numeric strings
|
|
8
|
+
*/
|
|
9
|
+
export declare function detectBodyVariables(body: unknown, prefix?: string): string[];
|
|
10
|
+
/**
|
|
11
|
+
* Substitute variables in a body template.
|
|
12
|
+
*/
|
|
13
|
+
export declare function substituteBodyVariables(template: string | Record<string, unknown>, values: Record<string, string>): string | Record<string, unknown>;
|