@apitap/core 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +60 -0
- package/README.md +362 -0
- package/SKILL.md +270 -0
- package/dist/auth/crypto.d.ts +31 -0
- package/dist/auth/crypto.js +66 -0
- package/dist/auth/crypto.js.map +1 -0
- package/dist/auth/handoff.d.ts +29 -0
- package/dist/auth/handoff.js +180 -0
- package/dist/auth/handoff.js.map +1 -0
- package/dist/auth/manager.d.ts +46 -0
- package/dist/auth/manager.js +127 -0
- package/dist/auth/manager.js.map +1 -0
- package/dist/auth/oauth-refresh.d.ts +16 -0
- package/dist/auth/oauth-refresh.js +91 -0
- package/dist/auth/oauth-refresh.js.map +1 -0
- package/dist/auth/refresh.d.ts +43 -0
- package/dist/auth/refresh.js +217 -0
- package/dist/auth/refresh.js.map +1 -0
- package/dist/capture/anti-bot.d.ts +15 -0
- package/dist/capture/anti-bot.js +43 -0
- package/dist/capture/anti-bot.js.map +1 -0
- package/dist/capture/blocklist.d.ts +6 -0
- package/dist/capture/blocklist.js +70 -0
- package/dist/capture/blocklist.js.map +1 -0
- package/dist/capture/body-diff.d.ts +8 -0
- package/dist/capture/body-diff.js +102 -0
- package/dist/capture/body-diff.js.map +1 -0
- package/dist/capture/body-variables.d.ts +13 -0
- package/dist/capture/body-variables.js +142 -0
- package/dist/capture/body-variables.js.map +1 -0
- package/dist/capture/domain.d.ts +8 -0
- package/dist/capture/domain.js +34 -0
- package/dist/capture/domain.js.map +1 -0
- package/dist/capture/entropy.d.ts +33 -0
- package/dist/capture/entropy.js +100 -0
- package/dist/capture/entropy.js.map +1 -0
- package/dist/capture/filter.d.ts +11 -0
- package/dist/capture/filter.js +49 -0
- package/dist/capture/filter.js.map +1 -0
- package/dist/capture/graphql.d.ts +21 -0
- package/dist/capture/graphql.js +99 -0
- package/dist/capture/graphql.js.map +1 -0
- package/dist/capture/idle.d.ts +23 -0
- package/dist/capture/idle.js +44 -0
- package/dist/capture/idle.js.map +1 -0
- package/dist/capture/monitor.d.ts +26 -0
- package/dist/capture/monitor.js +183 -0
- package/dist/capture/monitor.js.map +1 -0
- package/dist/capture/oauth-detector.d.ts +18 -0
- package/dist/capture/oauth-detector.js +96 -0
- package/dist/capture/oauth-detector.js.map +1 -0
- package/dist/capture/pagination.d.ts +9 -0
- package/dist/capture/pagination.js +40 -0
- package/dist/capture/pagination.js.map +1 -0
- package/dist/capture/parameterize.d.ts +17 -0
- package/dist/capture/parameterize.js +63 -0
- package/dist/capture/parameterize.js.map +1 -0
- package/dist/capture/scrubber.d.ts +5 -0
- package/dist/capture/scrubber.js +38 -0
- package/dist/capture/scrubber.js.map +1 -0
- package/dist/capture/session.d.ts +46 -0
- package/dist/capture/session.js +445 -0
- package/dist/capture/session.js.map +1 -0
- package/dist/capture/token-detector.d.ts +16 -0
- package/dist/capture/token-detector.js +62 -0
- package/dist/capture/token-detector.js.map +1 -0
- package/dist/capture/verifier.d.ts +17 -0
- package/dist/capture/verifier.js +147 -0
- package/dist/capture/verifier.js.map +1 -0
- package/dist/cli.d.ts +2 -0
- package/dist/cli.js +930 -0
- package/dist/cli.js.map +1 -0
- package/dist/discovery/auth.d.ts +17 -0
- package/dist/discovery/auth.js +81 -0
- package/dist/discovery/auth.js.map +1 -0
- package/dist/discovery/fetch.d.ts +17 -0
- package/dist/discovery/fetch.js +59 -0
- package/dist/discovery/fetch.js.map +1 -0
- package/dist/discovery/frameworks.d.ts +11 -0
- package/dist/discovery/frameworks.js +249 -0
- package/dist/discovery/frameworks.js.map +1 -0
- package/dist/discovery/index.d.ts +21 -0
- package/dist/discovery/index.js +219 -0
- package/dist/discovery/index.js.map +1 -0
- package/dist/discovery/openapi.d.ts +13 -0
- package/dist/discovery/openapi.js +175 -0
- package/dist/discovery/openapi.js.map +1 -0
- package/dist/discovery/probes.d.ts +9 -0
- package/dist/discovery/probes.js +70 -0
- package/dist/discovery/probes.js.map +1 -0
- package/dist/index.d.ts +25 -0
- package/dist/index.js +25 -0
- package/dist/index.js.map +1 -0
- package/dist/inspect/report.d.ts +52 -0
- package/dist/inspect/report.js +191 -0
- package/dist/inspect/report.js.map +1 -0
- package/dist/mcp.d.ts +8 -0
- package/dist/mcp.js +526 -0
- package/dist/mcp.js.map +1 -0
- package/dist/orchestration/browse.d.ts +38 -0
- package/dist/orchestration/browse.js +198 -0
- package/dist/orchestration/browse.js.map +1 -0
- package/dist/orchestration/cache.d.ts +15 -0
- package/dist/orchestration/cache.js +24 -0
- package/dist/orchestration/cache.js.map +1 -0
- package/dist/plugin.d.ts +17 -0
- package/dist/plugin.js +158 -0
- package/dist/plugin.js.map +1 -0
- package/dist/read/decoders/deepwiki.d.ts +2 -0
- package/dist/read/decoders/deepwiki.js +148 -0
- package/dist/read/decoders/deepwiki.js.map +1 -0
- package/dist/read/decoders/grokipedia.d.ts +2 -0
- package/dist/read/decoders/grokipedia.js +210 -0
- package/dist/read/decoders/grokipedia.js.map +1 -0
- package/dist/read/decoders/hackernews.d.ts +2 -0
- package/dist/read/decoders/hackernews.js +168 -0
- package/dist/read/decoders/hackernews.js.map +1 -0
- package/dist/read/decoders/index.d.ts +2 -0
- package/dist/read/decoders/index.js +12 -0
- package/dist/read/decoders/index.js.map +1 -0
- package/dist/read/decoders/reddit.d.ts +2 -0
- package/dist/read/decoders/reddit.js +142 -0
- package/dist/read/decoders/reddit.js.map +1 -0
- package/dist/read/decoders/twitter.d.ts +12 -0
- package/dist/read/decoders/twitter.js +187 -0
- package/dist/read/decoders/twitter.js.map +1 -0
- package/dist/read/decoders/wikipedia.d.ts +2 -0
- package/dist/read/decoders/wikipedia.js +66 -0
- package/dist/read/decoders/wikipedia.js.map +1 -0
- package/dist/read/decoders/youtube.d.ts +2 -0
- package/dist/read/decoders/youtube.js +69 -0
- package/dist/read/decoders/youtube.js.map +1 -0
- package/dist/read/extract.d.ts +25 -0
- package/dist/read/extract.js +320 -0
- package/dist/read/extract.js.map +1 -0
- package/dist/read/index.d.ts +14 -0
- package/dist/read/index.js +66 -0
- package/dist/read/index.js.map +1 -0
- package/dist/read/peek.d.ts +9 -0
- package/dist/read/peek.js +137 -0
- package/dist/read/peek.js.map +1 -0
- package/dist/read/types.d.ts +44 -0
- package/dist/read/types.js +3 -0
- package/dist/read/types.js.map +1 -0
- package/dist/replay/engine.d.ts +53 -0
- package/dist/replay/engine.js +441 -0
- package/dist/replay/engine.js.map +1 -0
- package/dist/replay/truncate.d.ts +16 -0
- package/dist/replay/truncate.js +92 -0
- package/dist/replay/truncate.js.map +1 -0
- package/dist/serve.d.ts +31 -0
- package/dist/serve.js +149 -0
- package/dist/serve.js.map +1 -0
- package/dist/skill/generator.d.ts +44 -0
- package/dist/skill/generator.js +419 -0
- package/dist/skill/generator.js.map +1 -0
- package/dist/skill/importer.d.ts +26 -0
- package/dist/skill/importer.js +80 -0
- package/dist/skill/importer.js.map +1 -0
- package/dist/skill/search.d.ts +19 -0
- package/dist/skill/search.js +51 -0
- package/dist/skill/search.js.map +1 -0
- package/dist/skill/signing.d.ts +16 -0
- package/dist/skill/signing.js +34 -0
- package/dist/skill/signing.js.map +1 -0
- package/dist/skill/ssrf.d.ts +27 -0
- package/dist/skill/ssrf.js +210 -0
- package/dist/skill/ssrf.js.map +1 -0
- package/dist/skill/store.d.ts +7 -0
- package/dist/skill/store.js +93 -0
- package/dist/skill/store.js.map +1 -0
- package/dist/stats/report.d.ts +26 -0
- package/dist/stats/report.js +157 -0
- package/dist/stats/report.js.map +1 -0
- package/dist/types.d.ts +214 -0
- package/dist/types.js +3 -0
- package/dist/types.js.map +1 -0
- package/package.json +58 -0
- package/src/auth/crypto.ts +92 -0
- package/src/auth/handoff.ts +229 -0
- package/src/auth/manager.ts +140 -0
- package/src/auth/oauth-refresh.ts +120 -0
- package/src/auth/refresh.ts +300 -0
- package/src/capture/anti-bot.ts +63 -0
- package/src/capture/blocklist.ts +75 -0
- package/src/capture/body-diff.ts +109 -0
- package/src/capture/body-variables.ts +156 -0
- package/src/capture/domain.ts +34 -0
- package/src/capture/entropy.ts +121 -0
- package/src/capture/filter.ts +56 -0
- package/src/capture/graphql.ts +124 -0
- package/src/capture/idle.ts +45 -0
- package/src/capture/monitor.ts +224 -0
- package/src/capture/oauth-detector.ts +106 -0
- package/src/capture/pagination.ts +49 -0
- package/src/capture/parameterize.ts +68 -0
- package/src/capture/scrubber.ts +49 -0
- package/src/capture/session.ts +502 -0
- package/src/capture/token-detector.ts +76 -0
- package/src/capture/verifier.ts +171 -0
- package/src/cli.ts +1031 -0
- package/src/discovery/auth.ts +99 -0
- package/src/discovery/fetch.ts +85 -0
- package/src/discovery/frameworks.ts +231 -0
- package/src/discovery/index.ts +256 -0
- package/src/discovery/openapi.ts +230 -0
- package/src/discovery/probes.ts +76 -0
- package/src/index.ts +26 -0
- package/src/inspect/report.ts +247 -0
- package/src/mcp.ts +618 -0
- package/src/orchestration/browse.ts +250 -0
- package/src/orchestration/cache.ts +37 -0
- package/src/plugin.ts +188 -0
- package/src/read/decoders/deepwiki.ts +180 -0
- package/src/read/decoders/grokipedia.ts +246 -0
- package/src/read/decoders/hackernews.ts +198 -0
- package/src/read/decoders/index.ts +15 -0
- package/src/read/decoders/reddit.ts +158 -0
- package/src/read/decoders/twitter.ts +211 -0
- package/src/read/decoders/wikipedia.ts +75 -0
- package/src/read/decoders/youtube.ts +75 -0
- package/src/read/extract.ts +396 -0
- package/src/read/index.ts +78 -0
- package/src/read/peek.ts +175 -0
- package/src/read/types.ts +37 -0
- package/src/replay/engine.ts +559 -0
- package/src/replay/truncate.ts +116 -0
- package/src/serve.ts +189 -0
- package/src/skill/generator.ts +473 -0
- package/src/skill/importer.ts +107 -0
- package/src/skill/search.ts +76 -0
- package/src/skill/signing.ts +36 -0
- package/src/skill/ssrf.ts +238 -0
- package/src/skill/store.ts +107 -0
- package/src/stats/report.ts +208 -0
- package/src/types.ts +233 -0
|
@@ -0,0 +1,183 @@
|
|
|
1
|
+
// src/capture/monitor.ts
|
|
2
|
+
import { chromium } from 'playwright';
|
|
3
|
+
import { shouldCapture } from './filter.js';
|
|
4
|
+
import { isDomainMatch } from './domain.js';
|
|
5
|
+
import { SkillGenerator } from '../skill/generator.js';
|
|
6
|
+
import { IdleTracker } from './idle.js';
|
|
7
|
+
import { detectCaptcha } from '../auth/refresh.js';
|
|
8
|
+
const DEFAULT_CDP_PORTS = [18792, 18800, 9222];
|
|
9
|
+
async function connectToBrowser(options) {
|
|
10
|
+
if (!options.launch) {
|
|
11
|
+
const ports = options.port ? [options.port] : DEFAULT_CDP_PORTS;
|
|
12
|
+
for (const port of ports) {
|
|
13
|
+
try {
|
|
14
|
+
const browser = await chromium.connectOverCDP(`http://localhost:${port}`, { timeout: 3000 });
|
|
15
|
+
return { browser, launched: false };
|
|
16
|
+
}
|
|
17
|
+
catch {
|
|
18
|
+
continue;
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
if (options.attach) {
|
|
23
|
+
const ports = options.port ? [options.port] : DEFAULT_CDP_PORTS;
|
|
24
|
+
throw new Error(`No browser found on CDP ports: ${ports.join(', ')}. Is a Chromium browser running with remote debugging?`);
|
|
25
|
+
}
|
|
26
|
+
const browser = await chromium.launch({ headless: options.headless ?? (process.env.DISPLAY ? false : true) });
|
|
27
|
+
return { browser, launched: true };
|
|
28
|
+
}
|
|
29
|
+
export async function capture(options) {
|
|
30
|
+
const { browser, launched } = await connectToBrowser(options);
|
|
31
|
+
const generators = new Map();
|
|
32
|
+
let totalRequests = 0;
|
|
33
|
+
let filteredRequests = 0;
|
|
34
|
+
const captchaDetectedDomains = new Set();
|
|
35
|
+
// Extract target domain for domain-only filtering
|
|
36
|
+
const targetUrl = options.url;
|
|
37
|
+
const generatorOptions = {
|
|
38
|
+
enablePreview: options.enablePreview ?? false,
|
|
39
|
+
scrub: options.scrub ?? true,
|
|
40
|
+
};
|
|
41
|
+
// Idle tracking: only active during interactive capture (no --duration)
|
|
42
|
+
const idleTracker = !options.duration ? new IdleTracker() : null;
|
|
43
|
+
let idleInterval = null;
|
|
44
|
+
let page;
|
|
45
|
+
if (launched) {
|
|
46
|
+
const context = await browser.newContext();
|
|
47
|
+
page = await context.newPage();
|
|
48
|
+
}
|
|
49
|
+
else {
|
|
50
|
+
const contexts = browser.contexts();
|
|
51
|
+
if (contexts.length > 0 && contexts[0].pages().length > 0) {
|
|
52
|
+
page = contexts[0].pages()[0];
|
|
53
|
+
}
|
|
54
|
+
else {
|
|
55
|
+
const context = contexts[0] ?? await browser.newContext();
|
|
56
|
+
page = await context.newPage();
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
page.on('response', async (response) => {
|
|
60
|
+
totalRequests++;
|
|
61
|
+
const url = response.url();
|
|
62
|
+
const status = response.status();
|
|
63
|
+
const contentType = response.headers()['content-type'] ?? '';
|
|
64
|
+
// Domain-only filtering (before any other processing)
|
|
65
|
+
if (!options.allDomains) {
|
|
66
|
+
const hostname = safeHostname(url);
|
|
67
|
+
if (hostname && !isDomainMatch(hostname, targetUrl)) {
|
|
68
|
+
filteredRequests++;
|
|
69
|
+
options.onFiltered?.();
|
|
70
|
+
return;
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
if (!shouldCapture({ url, status, contentType })) {
|
|
74
|
+
filteredRequests++;
|
|
75
|
+
const hostname = safeHostname(url);
|
|
76
|
+
if (hostname) {
|
|
77
|
+
const gen = generators.get(hostname);
|
|
78
|
+
if (gen)
|
|
79
|
+
gen.recordFiltered();
|
|
80
|
+
}
|
|
81
|
+
// Track network bytes from headers for filtered responses (browser cost measurement)
|
|
82
|
+
const contentLength = parseInt(response.headers()['content-length'] ?? '0', 10);
|
|
83
|
+
if (contentLength > 0) {
|
|
84
|
+
const filteredHostname = safeHostname(url);
|
|
85
|
+
if (filteredHostname && generators.has(filteredHostname)) {
|
|
86
|
+
generators.get(filteredHostname).addNetworkBytes(contentLength);
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
options.onFiltered?.();
|
|
90
|
+
return;
|
|
91
|
+
}
|
|
92
|
+
try {
|
|
93
|
+
const body = await response.text();
|
|
94
|
+
const hostname = new URL(url).hostname;
|
|
95
|
+
// Check for captcha in HTML responses (v0.8 captcha risk detection)
|
|
96
|
+
if (contentType.includes('text/html') && detectCaptcha(body)) {
|
|
97
|
+
captchaDetectedDomains.add(hostname);
|
|
98
|
+
}
|
|
99
|
+
if (!generators.has(hostname)) {
|
|
100
|
+
generators.set(hostname, new SkillGenerator(generatorOptions));
|
|
101
|
+
}
|
|
102
|
+
const gen = generators.get(hostname);
|
|
103
|
+
const exchange = {
|
|
104
|
+
request: {
|
|
105
|
+
url,
|
|
106
|
+
method: response.request().method(),
|
|
107
|
+
headers: response.request().headers(),
|
|
108
|
+
postData: response.request().postData() ?? undefined,
|
|
109
|
+
},
|
|
110
|
+
response: {
|
|
111
|
+
status,
|
|
112
|
+
headers: response.headers(),
|
|
113
|
+
body,
|
|
114
|
+
contentType,
|
|
115
|
+
},
|
|
116
|
+
timestamp: new Date().toISOString(),
|
|
117
|
+
};
|
|
118
|
+
const endpoint = gen.addExchange(exchange);
|
|
119
|
+
if (endpoint) {
|
|
120
|
+
options.onEndpoint?.({ id: endpoint.id, method: endpoint.method, path: endpoint.path });
|
|
121
|
+
// Track for idle detection using parameterized key
|
|
122
|
+
if (idleTracker) {
|
|
123
|
+
const paramKey = `${endpoint.method} ${endpoint.path}`;
|
|
124
|
+
idleTracker.recordEndpoint(paramKey);
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
catch {
|
|
129
|
+
// Response body may not be available (e.g. redirects); skip silently
|
|
130
|
+
}
|
|
131
|
+
});
|
|
132
|
+
await page.goto(options.url, { waitUntil: 'domcontentloaded' });
|
|
133
|
+
// Start idle check interval (every 5s) for interactive capture
|
|
134
|
+
if (idleTracker && options.onIdle) {
|
|
135
|
+
idleInterval = setInterval(() => {
|
|
136
|
+
if (idleTracker.checkIdle()) {
|
|
137
|
+
options.onIdle();
|
|
138
|
+
}
|
|
139
|
+
}, 5000);
|
|
140
|
+
}
|
|
141
|
+
// Wait for duration or until interrupted
|
|
142
|
+
// SIGINT always resolves gracefully so skill files get written
|
|
143
|
+
if (options.duration) {
|
|
144
|
+
await new Promise(resolve => {
|
|
145
|
+
const timer = setTimeout(resolve, options.duration * 1000);
|
|
146
|
+
process.once('SIGINT', () => { clearTimeout(timer); resolve(); });
|
|
147
|
+
});
|
|
148
|
+
}
|
|
149
|
+
else {
|
|
150
|
+
await new Promise(resolve => {
|
|
151
|
+
process.once('SIGINT', resolve);
|
|
152
|
+
});
|
|
153
|
+
}
|
|
154
|
+
// Clean up idle interval
|
|
155
|
+
if (idleInterval)
|
|
156
|
+
clearInterval(idleInterval);
|
|
157
|
+
// Measure DOM size for browser cost comparison (v1.0)
|
|
158
|
+
let domBytes;
|
|
159
|
+
try {
|
|
160
|
+
const html = await page.content();
|
|
161
|
+
domBytes = html.length;
|
|
162
|
+
}
|
|
163
|
+
catch { /* page may have navigated away */ }
|
|
164
|
+
if (launched) {
|
|
165
|
+
await browser.close();
|
|
166
|
+
}
|
|
167
|
+
// Mark generators for domains where captcha was detected
|
|
168
|
+
for (const [hostname, gen] of generators) {
|
|
169
|
+
if (captchaDetectedDomains.has(hostname)) {
|
|
170
|
+
gen.setCaptchaRisk(true);
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
return { generators, totalRequests, filteredRequests, domBytes };
|
|
174
|
+
}
|
|
175
|
+
function safeHostname(url) {
|
|
176
|
+
try {
|
|
177
|
+
return new URL(url).hostname;
|
|
178
|
+
}
|
|
179
|
+
catch {
|
|
180
|
+
return null;
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
//# sourceMappingURL=monitor.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"monitor.js","sourceRoot":"","sources":["../../src/capture/monitor.ts"],"names":[],"mappings":"AAAA,yBAAyB;AACzB,OAAO,EAAE,QAAQ,EAA2B,MAAM,YAAY,CAAC;AAC/D,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,cAAc,EAAyB,MAAM,uBAAuB,CAAC;AAC9E,OAAO,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AACxC,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAyBnD,MAAM,iBAAiB,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,IAAI,CAAC,CAAC;AAE/C,KAAK,UAAU,gBAAgB,CAAC,OAAuB;IACrD,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;QACpB,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,iBAAiB,CAAC;QAChE,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,CAAC;gBACH,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,cAAc,CAAC,oBAAoB,IAAI,EAAE,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;gBAC7F,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC;YACtC,CAAC;YAAC,MAAM,CAAC;gBACP,SAAS;YACX,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;QACnB,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,iBAAiB,CAAC;QAChE,MAAM,IAAI,KAAK,CAAC,kCAAkC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,wDAAwD,CAAC,CAAC;IAC9H,CAAC;IAED,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAC9G,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;AACrC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,OAAuB;IACnD,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,MAAM,gBAAgB,CAAC,OAAO,CAAC,CAAC;IAC9D,MAAM,UAAU,GAAG,IAAI,GAAG,EAA0B,CAAC;IACrD,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,IAAI,gBAAgB,GAAG,CAAC,CAAC;IACzB,MAAM,sBAAsB,GAAG,IAAI,GAAG,EAAU,CAAC;IAEjD,kDAAkD;IAClD,MAAM,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC;IAE9B,MAAM,gBAAgB,GAAqB;QACzC,aAAa,EAAE,OAAO,CAAC,aAAa,IAAI,KAAK;QAC7C,KAAK,EAAE,OAAO,CAAC,KAAK,IAAI,IAAI;KAC7B,CAAC;IAEF,wEAAwE;IACxE,MAAM,WAAW,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,WAAW,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;IACjE,IAAI,YAAY,GAA0C,IAAI,CAAC;IAE/D,IAAI,IAAU,CAAC;IACf,IAAI,QAAQ,EAAE,CAAC;QACb,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,UAAU,EAAE,CAAC;QAC3C,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;IACjC,CAAC;SAAM,CAAC;QACN,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;QACpC,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,IAAI,QAAQ,CAAC,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC1D,IAAI,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC;QAChC,CAAC;aAAM,CAAC;YACN,MAAM,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAC,IAAI,MAAM,OAAO,CAAC,UAAU,EAAE,CAAC;YAC1D,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;QACjC,CAAC;IACH,CAAC;IAED,IAAI,CAAC,EAAE,CAAC,UAAU,EAAE,KAAK,EAAE,QAAQ,EAAE,EAAE;QACrC,aAAa,EAAE,CAAC;QAEhB,MAAM,GAAG,GAAG,QAAQ,CAAC,GAAG,EAAE,CAAC;QAC3B,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC;QACjC,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,EAAE,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;QAE7D,sDAAsD;QACtD,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC;YACxB,MAAM,QAAQ,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC;YACnC,IAAI,QAAQ,IAAI,CAAC,aAAa,CAAC,QAAQ,EAAE,SAAS,CAAC,EAAE,CAAC;gBACpD,gBAAgB,EAAE,CAAC;gBACnB,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC;gBACvB,OAAO;YACT,CAAC;QACH,CAAC;QAED,IAAI,CAAC,aAAa,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC,EAAE,CAAC;YACjD,gBAAgB,EAAE,CAAC;YACnB,MAAM,QAAQ,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC;YACnC,IAAI,QAAQ,EAAE,CAAC;gBACb,MAAM,GAAG,GAAG,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;gBACrC,IAAI,GAAG;oBAAE,GAAG,CAAC,cAAc,EAAE,CAAC;YAChC,CAAC;YACD,qFAAqF;YACrF,MAAM,aAAa,GAAG,QAAQ,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC,gBAAgB,CAAC,IAAI,GAAG,EAAE,EAAE,CAAC,CAAC;YAChF,IAAI,aAAa,GAAG,CAAC,EAAE,CAAC;gBACtB,MAAM,gBAAgB,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC;gBAC3C,IAAI,gBAAgB,IAAI,UAAU,CAAC,GAAG,CAAC,gBAAgB,CAAC,EAAE,CAAC;oBACzD,UAAU,CAAC,GAAG,CAAC,gBAAgB,CAAE,CAAC,eAAe,CAAC,aAAa,CAAC,CAAC;gBACnE,CAAC;YACH,CAAC;YACD,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC;YACvB,OAAO;QACT,CAAC;QAED,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YACnC,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;YAEvC,oEAAoE;YACpE,IAAI,WAAW,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,aAAa,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC7D,sBAAsB,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YACvC,CAAC;YAED,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;gBAC9B,UAAU,CAAC,GAAG,CAAC,QAAQ,EAAE,IAAI,cAAc,CAAC,gBAAgB,CAAC,CAAC,CAAC;YACjE,CAAC;YACD,MAAM,GAAG,GAAG,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAE,CAAC;YAEtC,MAAM,QAAQ,GAAqB;gBACjC,OAAO,EAAE;oBACP,GAAG;oBACH,MAAM,EAAE,QAAQ,CAAC,OAAO,EAAE,CAAC,MAAM,EAAE;oBACnC,OAAO,EAAE,QAAQ,CAAC,OAAO,EAAE,CAAC,OAAO,EAAE;oBACrC,QAAQ,EAAE,QAAQ,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,IAAI,SAAS;iBACrD;gBACD,QAAQ,EAAE;oBACR,MAAM;oBACN,OAAO,EAAE,QAAQ,CAAC,OAAO,EAAE;oBAC3B,IAAI;oBACJ,WAAW;iBACZ;gBACD,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;aACpC,CAAC;YAEF,MAAM,QAAQ,GAAG,GAAG,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;YAC3C,IAAI,QAAQ,EAAE,CAAC;gBACb,OAAO,CAAC,UAAU,EAAE,CAAC,EAAE,EAAE,EAAE,QAAQ,CAAC,EAAE,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,IAAI,EAAE,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;gBAExF,mDAAmD;gBACnD,IAAI,WAAW,EAAE,CAAC;oBAChB,MAAM,QAAQ,GAAG,GAAG,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,IAAI,EAAE,CAAC;oBACvD,WAAW,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;gBACvC,CAAC;YACH,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,qEAAqE;QACvE,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,kBAAkB,EAAE,CAAC,CAAC;IAEhE,+DAA+D;IAC/D,IAAI,WAAW,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;QAClC,YAAY,GAAG,WAAW,CAAC,GAAG,EAAE;YAC9B,IAAI,WAAW,CAAC,SAAS,EAAE,EAAE,CAAC;gBAC5B,OAAO,CAAC,MAAO,EAAE,CAAC;YACpB,CAAC;QACH,CAAC,EAAE,IAAI,CAAC,CAAC;IACX,CAAC;IAED,yCAAyC;IACzC,+DAA+D;IAC/D,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;QACrB,MAAM,IAAI,OAAO,CAAO,OAAO,CAAC,EAAE;YAChC,MAAM,KAAK,GAAG,UAAU,CAAC,OAAO,EAAE,OAAO,CAAC,QAAS,GAAG,IAAI,CAAC,CAAC;YAC5D,OAAO,CAAC,IAAI,CAAC,QAAQ,EAAE,GAAG,EAAE,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QACpE,CAAC,CAAC,CAAC;IACL,CAAC;SAAM,CAAC;QACN,MAAM,IAAI,OAAO,CAAO,OAAO,CAAC,EAAE;YAChC,OAAO,CAAC,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAClC,CAAC,CAAC,CAAC;IACL,CAAC;IAED,yBAAyB;IACzB,IAAI,YAAY;QAAE,aAAa,CAAC,YAAY,CAAC,CAAC;IAE9C,sDAAsD;IACtD,IAAI,QAA4B,CAAC;IACjC,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;QAClC,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC;IACzB,CAAC;IAAC,MAAM,CAAC,CAAC,kCAAkC,CAAC,CAAC;IAE9C,IAAI,QAAQ,EAAE,CAAC;QACb,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;IACxB,CAAC;IAED,yDAAyD;IACzD,KAAK,MAAM,CAAC,QAAQ,EAAE,GAAG,CAAC,IAAI,UAAU,EAAE,CAAC;QACzC,IAAI,sBAAsB,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;YACzC,GAAG,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;QAC3B,CAAC;IACH,CAAC;IAED,OAAO,EAAE,UAAU,EAAE,aAAa,EAAE,gBAAgB,EAAE,QAAQ,EAAE,CAAC;AACnE,CAAC;AAED,SAAS,YAAY,CAAC,GAAW;IAC/B,IAAI,CAAC;QACH,OAAO,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;IAC/B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
export interface OAuthInfo {
|
|
2
|
+
tokenEndpoint: string;
|
|
3
|
+
clientId: string;
|
|
4
|
+
grantType: 'refresh_token' | 'client_credentials';
|
|
5
|
+
scope?: string;
|
|
6
|
+
clientSecret?: string;
|
|
7
|
+
}
|
|
8
|
+
/**
|
|
9
|
+
* Detect OAuth2 token endpoint requests from captured traffic.
|
|
10
|
+
* Only recognizes refreshable flows (refresh_token, client_credentials).
|
|
11
|
+
* Ignores authorization_code (initial auth, not refreshable in isolation).
|
|
12
|
+
*/
|
|
13
|
+
export declare function isOAuthTokenRequest(req: {
|
|
14
|
+
url: string;
|
|
15
|
+
method: string;
|
|
16
|
+
headers: Record<string, string>;
|
|
17
|
+
postData?: string;
|
|
18
|
+
}): OAuthInfo | null;
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
// src/capture/oauth-detector.ts
|
|
2
|
+
/**
|
|
3
|
+
* Detect OAuth2 token endpoint requests from captured traffic.
|
|
4
|
+
* Only recognizes refreshable flows (refresh_token, client_credentials).
|
|
5
|
+
* Ignores authorization_code (initial auth, not refreshable in isolation).
|
|
6
|
+
*/
|
|
7
|
+
export function isOAuthTokenRequest(req) {
|
|
8
|
+
// Only POST requests
|
|
9
|
+
if (req.method.toUpperCase() !== 'POST')
|
|
10
|
+
return null;
|
|
11
|
+
// URL heuristic: must contain /token or /oauth
|
|
12
|
+
const urlLower = req.url.toLowerCase();
|
|
13
|
+
if (!urlLower.includes('/token') && !urlLower.includes('/oauth'))
|
|
14
|
+
return null;
|
|
15
|
+
if (!req.postData)
|
|
16
|
+
return null;
|
|
17
|
+
// Parse body — support URL-encoded and JSON
|
|
18
|
+
const params = parseBody(req.postData, req.headers['content-type'] ?? '');
|
|
19
|
+
if (!params)
|
|
20
|
+
return null;
|
|
21
|
+
const grantType = params.get('grant_type');
|
|
22
|
+
if (!grantType)
|
|
23
|
+
return null;
|
|
24
|
+
// Only refreshable flows
|
|
25
|
+
if (grantType !== 'refresh_token' && grantType !== 'client_credentials')
|
|
26
|
+
return null;
|
|
27
|
+
// Extract client_id — may also be in Basic auth header
|
|
28
|
+
let clientId = params.get('client_id') ?? '';
|
|
29
|
+
let clientSecret = params.get('client_secret');
|
|
30
|
+
if (!clientId) {
|
|
31
|
+
const basic = parseBasicAuth(req.headers['authorization'] ?? '');
|
|
32
|
+
if (basic) {
|
|
33
|
+
clientId = basic.username;
|
|
34
|
+
if (!clientSecret)
|
|
35
|
+
clientSecret = basic.password;
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
if (!clientId)
|
|
39
|
+
return null;
|
|
40
|
+
const result = {
|
|
41
|
+
tokenEndpoint: req.url.split('?')[0], // strip query params
|
|
42
|
+
clientId,
|
|
43
|
+
grantType: grantType,
|
|
44
|
+
};
|
|
45
|
+
const scope = params.get('scope');
|
|
46
|
+
if (scope)
|
|
47
|
+
result.scope = scope;
|
|
48
|
+
if (clientSecret)
|
|
49
|
+
result.clientSecret = clientSecret;
|
|
50
|
+
return result;
|
|
51
|
+
}
|
|
52
|
+
function parseBody(body, contentType) {
|
|
53
|
+
try {
|
|
54
|
+
if (contentType.includes('application/json')) {
|
|
55
|
+
const obj = JSON.parse(body);
|
|
56
|
+
if (typeof obj !== 'object' || obj === null)
|
|
57
|
+
return null;
|
|
58
|
+
const map = new Map();
|
|
59
|
+
for (const [k, v] of Object.entries(obj)) {
|
|
60
|
+
if (typeof v === 'string')
|
|
61
|
+
map.set(k, v);
|
|
62
|
+
}
|
|
63
|
+
return map;
|
|
64
|
+
}
|
|
65
|
+
// Default: URL-encoded (+ is space in application/x-www-form-urlencoded)
|
|
66
|
+
const map = new Map();
|
|
67
|
+
const pairs = body.split('&');
|
|
68
|
+
for (const pair of pairs) {
|
|
69
|
+
const idx = pair.indexOf('=');
|
|
70
|
+
if (idx === -1)
|
|
71
|
+
continue;
|
|
72
|
+
const key = decodeURIComponent(pair.slice(0, idx).replace(/\+/g, ' '));
|
|
73
|
+
const val = decodeURIComponent(pair.slice(idx + 1).replace(/\+/g, ' '));
|
|
74
|
+
map.set(key, val);
|
|
75
|
+
}
|
|
76
|
+
return map.size > 0 ? map : null;
|
|
77
|
+
}
|
|
78
|
+
catch {
|
|
79
|
+
return null;
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
function parseBasicAuth(header) {
|
|
83
|
+
if (!header.startsWith('Basic '))
|
|
84
|
+
return null;
|
|
85
|
+
try {
|
|
86
|
+
const decoded = Buffer.from(header.slice(6), 'base64').toString('utf-8');
|
|
87
|
+
const idx = decoded.indexOf(':');
|
|
88
|
+
if (idx === -1)
|
|
89
|
+
return null;
|
|
90
|
+
return { username: decoded.slice(0, idx), password: decoded.slice(idx + 1) };
|
|
91
|
+
}
|
|
92
|
+
catch {
|
|
93
|
+
return null;
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
//# sourceMappingURL=oauth-detector.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"oauth-detector.js","sourceRoot":"","sources":["../../src/capture/oauth-detector.ts"],"names":[],"mappings":"AAAA,gCAAgC;AAUhC;;;;GAIG;AACH,MAAM,UAAU,mBAAmB,CAAC,GAKnC;IACC,qBAAqB;IACrB,IAAI,GAAG,CAAC,MAAM,CAAC,WAAW,EAAE,KAAK,MAAM;QAAE,OAAO,IAAI,CAAC;IAErD,+CAA+C;IAC/C,MAAM,QAAQ,GAAG,GAAG,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC;IACvC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,QAAQ,CAAC;QAAE,OAAO,IAAI,CAAC;IAE9E,IAAI,CAAC,GAAG,CAAC,QAAQ;QAAE,OAAO,IAAI,CAAC;IAE/B,4CAA4C;IAC5C,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC,QAAQ,EAAE,GAAG,CAAC,OAAO,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC,CAAC;IAC1E,IAAI,CAAC,MAAM;QAAE,OAAO,IAAI,CAAC;IAEzB,MAAM,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;IAC3C,IAAI,CAAC,SAAS;QAAE,OAAO,IAAI,CAAC;IAE5B,yBAAyB;IACzB,IAAI,SAAS,KAAK,eAAe,IAAI,SAAS,KAAK,oBAAoB;QAAE,OAAO,IAAI,CAAC;IAErF,uDAAuD;IACvD,IAAI,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC;IAC7C,IAAI,YAAY,GAAG,MAAM,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC;IAE/C,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,MAAM,KAAK,GAAG,cAAc,CAAC,GAAG,CAAC,OAAO,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC,CAAC;QACjE,IAAI,KAAK,EAAE,CAAC;YACV,QAAQ,GAAG,KAAK,CAAC,QAAQ,CAAC;YAC1B,IAAI,CAAC,YAAY;gBAAE,YAAY,GAAG,KAAK,CAAC,QAAQ,CAAC;QACnD,CAAC;IACH,CAAC;IAED,IAAI,CAAC,QAAQ;QAAE,OAAO,IAAI,CAAC;IAE3B,MAAM,MAAM,GAAc;QACxB,aAAa,EAAE,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAE,EAAE,qBAAqB;QAC5D,QAAQ;QACR,SAAS,EAAE,SAAmD;KAC/D,CAAC;IAEF,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IAClC,IAAI,KAAK;QAAE,MAAM,CAAC,KAAK,GAAG,KAAK,CAAC;IAChC,IAAI,YAAY;QAAE,MAAM,CAAC,YAAY,GAAG,YAAY,CAAC;IAErD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,SAAS,CAAC,IAAY,EAAE,WAAmB;IAClD,IAAI,CAAC;QACH,IAAI,WAAW,CAAC,QAAQ,CAAC,kBAAkB,CAAC,EAAE,CAAC;YAC7C,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAC7B,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,KAAK,IAAI;gBAAE,OAAO,IAAI,CAAC;YACzD,MAAM,GAAG,GAAG,IAAI,GAAG,EAAkB,CAAC;YACtC,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;gBACzC,IAAI,OAAO,CAAC,KAAK,QAAQ;oBAAE,GAAG,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YAC3C,CAAC;YACD,OAAO,GAAG,CAAC;QACb,CAAC;QAED,yEAAyE;QACzE,MAAM,GAAG,GAAG,IAAI,GAAG,EAAkB,CAAC;QACtC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC9B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;YAC9B,IAAI,GAAG,KAAK,CAAC,CAAC;gBAAE,SAAS;YACzB,MAAM,GAAG,GAAG,kBAAkB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC;YACvE,MAAM,GAAG,GAAG,kBAAkB,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC;YACxE,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;QACpB,CAAC;QACD,OAAO,GAAG,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC;IACnC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,SAAS,cAAc,CAAC,MAAc;IACpC,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC;QAAE,OAAO,IAAI,CAAC;IAC9C,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;QACzE,MAAM,GAAG,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QACjC,IAAI,GAAG,KAAK,CAAC,CAAC;YAAE,OAAO,IAAI,CAAC;QAC5B,OAAO,EAAE,QAAQ,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,QAAQ,EAAE,OAAO,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC,EAAE,CAAC;IAC/E,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import type { PaginationInfo } from '../types.js';
|
|
2
|
+
/**
|
|
3
|
+
* Detect pagination patterns from query parameters.
|
|
4
|
+
* Returns null if no pagination pattern is detected.
|
|
5
|
+
*/
|
|
6
|
+
export declare function detectPagination(queryParams: Record<string, {
|
|
7
|
+
type: string;
|
|
8
|
+
example: string;
|
|
9
|
+
}>): PaginationInfo | null;
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
const OFFSET_PARAMS = new Set(['offset', 'skip']);
|
|
2
|
+
const CURSOR_PARAMS = new Set(['cursor', 'after', 'before', 'next_cursor', 'starting_after']);
|
|
3
|
+
const PAGE_PARAMS = new Set(['page', 'p', 'page_number']);
|
|
4
|
+
const LIMIT_PARAMS = new Set(['limit', 'per_page', 'page_size', 'count', 'size']);
|
|
5
|
+
/**
|
|
6
|
+
* Detect pagination patterns from query parameters.
|
|
7
|
+
* Returns null if no pagination pattern is detected.
|
|
8
|
+
*/
|
|
9
|
+
export function detectPagination(queryParams) {
|
|
10
|
+
const paramNames = Object.keys(queryParams);
|
|
11
|
+
const limitParam = paramNames.find(p => LIMIT_PARAMS.has(p.toLowerCase()));
|
|
12
|
+
// Check offset-based (offset/skip + optional limit)
|
|
13
|
+
for (const name of paramNames) {
|
|
14
|
+
if (OFFSET_PARAMS.has(name.toLowerCase())) {
|
|
15
|
+
return {
|
|
16
|
+
type: 'offset',
|
|
17
|
+
paramName: name,
|
|
18
|
+
...(limitParam ? { limitParam } : {}),
|
|
19
|
+
};
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
// Check cursor-based
|
|
23
|
+
for (const name of paramNames) {
|
|
24
|
+
if (CURSOR_PARAMS.has(name.toLowerCase())) {
|
|
25
|
+
return { type: 'cursor', paramName: name };
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
// Check page-based
|
|
29
|
+
for (const name of paramNames) {
|
|
30
|
+
if (PAGE_PARAMS.has(name.toLowerCase())) {
|
|
31
|
+
return {
|
|
32
|
+
type: 'page',
|
|
33
|
+
paramName: name,
|
|
34
|
+
...(limitParam && limitParam !== name ? { limitParam } : {}),
|
|
35
|
+
};
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
return null;
|
|
39
|
+
}
|
|
40
|
+
//# sourceMappingURL=pagination.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"pagination.js","sourceRoot":"","sources":["../../src/capture/pagination.ts"],"names":[],"mappings":"AAGA,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC;AAClD,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC,CAAC,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,aAAa,EAAE,gBAAgB,CAAC,CAAC,CAAC;AAC9F,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,CAAC,MAAM,EAAE,GAAG,EAAE,aAAa,CAAC,CAAC,CAAC;AAC1D,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,CAAC,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC;AAElF;;;GAGG;AACH,MAAM,UAAU,gBAAgB,CAC9B,WAA8D;IAE9D,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IAC5C,MAAM,UAAU,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;IAE3E,oDAAoD;IACpD,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;QAC9B,IAAI,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,EAAE,CAAC;YAC1C,OAAO;gBACL,IAAI,EAAE,QAAQ;gBACd,SAAS,EAAE,IAAI;gBACf,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aACtC,CAAC;QACJ,CAAC;IACH,CAAC;IAED,qBAAqB;IACrB,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;QAC9B,IAAI,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,EAAE,CAAC;YAC1C,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;QAC7C,CAAC;IACH,CAAC;IAED,mBAAmB;IACnB,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;QAC9B,IAAI,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,EAAE,CAAC;YACxC,OAAO;gBACL,IAAI,EAAE,MAAM;gBACZ,SAAS,EAAE,IAAI;gBACf,GAAG,CAAC,UAAU,IAAI,UAAU,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aAC7D,CAAC;QACJ,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC"}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Replace dynamic path segments with :param placeholders.
|
|
3
|
+
*
|
|
4
|
+
* Rules:
|
|
5
|
+
* - Pure numeric → :id
|
|
6
|
+
* - UUID → :id
|
|
7
|
+
* - 12+ alphanum with mixed letters+digits → :hash
|
|
8
|
+
* - Contains 8+ consecutive digits → :slug
|
|
9
|
+
*/
|
|
10
|
+
export declare function parameterizePath(path: string): string;
|
|
11
|
+
/**
|
|
12
|
+
* Strip framework-specific path noise for clean endpoint IDs.
|
|
13
|
+
*
|
|
14
|
+
* - Strips /_next/data/<hash>/ prefix (Next.js data routes)
|
|
15
|
+
* - Strips .json suffix
|
|
16
|
+
*/
|
|
17
|
+
export declare function cleanFrameworkPath(path: string): string;
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
// src/capture/parameterize.ts
|
|
2
|
+
const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
|
3
|
+
const PURE_NUMERIC_RE = /^\d+$/;
|
|
4
|
+
const LONG_DIGITS_RE = /\d{8,}/;
|
|
5
|
+
const NEXT_DATA_PREFIX_RE = /^\/_next\/data\/[^/]+\//;
|
|
6
|
+
/**
|
|
7
|
+
* Check if a path segment is a dynamic value that should be parameterized.
|
|
8
|
+
* Returns the parameter name (:id, :hash, :slug) or null if static.
|
|
9
|
+
*/
|
|
10
|
+
function classifySegment(segment) {
|
|
11
|
+
// Pure numeric → :id
|
|
12
|
+
if (PURE_NUMERIC_RE.test(segment))
|
|
13
|
+
return ':id';
|
|
14
|
+
// UUID → :id
|
|
15
|
+
if (UUID_RE.test(segment))
|
|
16
|
+
return ':id';
|
|
17
|
+
// Slug with embedded long number (8+ consecutive digits) — check before hash
|
|
18
|
+
// because slugs like "btc-updown-15m-1770254100" would also match the hash rule
|
|
19
|
+
if (LONG_DIGITS_RE.test(segment)) {
|
|
20
|
+
return ':slug';
|
|
21
|
+
}
|
|
22
|
+
// Strip hyphens/underscores for character analysis
|
|
23
|
+
const stripped = segment.replace(/[-_]/g, '');
|
|
24
|
+
// Hash-like: 12+ alphanumeric chars with both letters and digits
|
|
25
|
+
if (stripped.length >= 12 && /[a-zA-Z]/.test(stripped) && /\d/.test(stripped)) {
|
|
26
|
+
return ':hash';
|
|
27
|
+
}
|
|
28
|
+
return null;
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* Replace dynamic path segments with :param placeholders.
|
|
32
|
+
*
|
|
33
|
+
* Rules:
|
|
34
|
+
* - Pure numeric → :id
|
|
35
|
+
* - UUID → :id
|
|
36
|
+
* - 12+ alphanum with mixed letters+digits → :hash
|
|
37
|
+
* - Contains 8+ consecutive digits → :slug
|
|
38
|
+
*/
|
|
39
|
+
export function parameterizePath(path) {
|
|
40
|
+
const segments = path.split('/');
|
|
41
|
+
const result = segments.map(seg => {
|
|
42
|
+
if (seg === '')
|
|
43
|
+
return seg;
|
|
44
|
+
return classifySegment(seg) ?? seg;
|
|
45
|
+
});
|
|
46
|
+
return result.join('/');
|
|
47
|
+
}
|
|
48
|
+
/**
|
|
49
|
+
* Strip framework-specific path noise for clean endpoint IDs.
|
|
50
|
+
*
|
|
51
|
+
* - Strips /_next/data/<hash>/ prefix (Next.js data routes)
|
|
52
|
+
* - Strips .json suffix
|
|
53
|
+
*/
|
|
54
|
+
export function cleanFrameworkPath(path) {
|
|
55
|
+
let cleaned = path;
|
|
56
|
+
// Strip _next/data/<hash>/ prefix
|
|
57
|
+
cleaned = cleaned.replace(NEXT_DATA_PREFIX_RE, '/');
|
|
58
|
+
// Strip .json suffix
|
|
59
|
+
cleaned = cleaned.replace(/\.json$/, '');
|
|
60
|
+
// Ensure we have at least /
|
|
61
|
+
return cleaned || '/';
|
|
62
|
+
}
|
|
63
|
+
//# sourceMappingURL=parameterize.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"parameterize.js","sourceRoot":"","sources":["../../src/capture/parameterize.ts"],"names":[],"mappings":"AAAA,8BAA8B;AAE9B,MAAM,OAAO,GAAG,iEAAiE,CAAC;AAClF,MAAM,eAAe,GAAG,OAAO,CAAC;AAChC,MAAM,cAAc,GAAG,QAAQ,CAAC;AAChC,MAAM,mBAAmB,GAAG,yBAAyB,CAAC;AAEtD;;;GAGG;AACH,SAAS,eAAe,CAAC,OAAe;IACtC,qBAAqB;IACrB,IAAI,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC;QAAE,OAAO,KAAK,CAAC;IAEhD,aAAa;IACb,IAAI,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC;QAAE,OAAO,KAAK,CAAC;IAExC,6EAA6E;IAC7E,gFAAgF;IAChF,IAAI,cAAc,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QACjC,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,mDAAmD;IACnD,MAAM,QAAQ,GAAG,OAAO,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;IAE9C,iEAAiE;IACjE,IAAI,QAAQ,CAAC,MAAM,IAAI,EAAE,IAAI,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC9E,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,gBAAgB,CAAC,IAAY;IAC3C,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACjC,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE;QAChC,IAAI,GAAG,KAAK,EAAE;YAAE,OAAO,GAAG,CAAC;QAC3B,OAAO,eAAe,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC;IACrC,CAAC,CAAC,CAAC;IACH,OAAO,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAC1B,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,kBAAkB,CAAC,IAAY;IAC7C,IAAI,OAAO,GAAG,IAAI,CAAC;IACnB,kCAAkC;IAClC,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,mBAAmB,EAAE,GAAG,CAAC,CAAC;IACpD,qBAAqB;IACrB,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;IACzC,4BAA4B;IAC5B,OAAO,OAAO,IAAI,GAAG,CAAC;AACxB,CAAC"}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
// src/capture/scrubber.ts
|
|
2
|
+
// Email: standard pattern
|
|
3
|
+
const EMAIL_RE = /[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}/g;
|
|
4
|
+
// Phone (international): requires + prefix
|
|
5
|
+
const PHONE_INTL_RE = /\+[1-9]\d{7,14}/g;
|
|
6
|
+
// Phone (US): requires separators — (123) 456-7890 or 123-456-7890 or 123.456.7890
|
|
7
|
+
const PHONE_US_RE = /\(\d{3}\)[-.\s]\d{3}[-.\s]\d{4}|\d{3}[-.\s]\d{3}[-.\s]\d{4}/g;
|
|
8
|
+
// IPv4: four octets, each 0-255, validated programmatically
|
|
9
|
+
const IPV4_RE = /\b(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})\b/g;
|
|
10
|
+
// Credit card: 16 digits with optional dashes or spaces every 4
|
|
11
|
+
const CARD_RE = /\b\d{4}[- ]?\d{4}[- ]?\d{4}[- ]?\d{4}\b/g;
|
|
12
|
+
// US SSN: 123-45-6789
|
|
13
|
+
const SSN_RE = /\b\d{3}-\d{2}-\d{4}\b/g;
|
|
14
|
+
/**
|
|
15
|
+
* Scrub PII from a string. Returns the string with PII replaced by placeholders.
|
|
16
|
+
* Order matters: SSN before phone (SSN is more specific).
|
|
17
|
+
*/
|
|
18
|
+
export function scrubPII(input) {
|
|
19
|
+
let result = input;
|
|
20
|
+
// Email first (most distinctive pattern)
|
|
21
|
+
result = result.replace(EMAIL_RE, '[email]');
|
|
22
|
+
// SSN before phone (SSN pattern 123-45-6789 could be confused)
|
|
23
|
+
result = result.replace(SSN_RE, '[ssn]');
|
|
24
|
+
// Credit cards
|
|
25
|
+
result = result.replace(CARD_RE, '[card]');
|
|
26
|
+
// IPv4 with octet validation
|
|
27
|
+
result = result.replace(IPV4_RE, (_match, o1, o2, o3, o4) => {
|
|
28
|
+
const octets = [o1, o2, o3, o4].map(Number);
|
|
29
|
+
if (octets.every(o => o <= 255))
|
|
30
|
+
return '[ip]';
|
|
31
|
+
return _match;
|
|
32
|
+
});
|
|
33
|
+
// Phone (international, then US)
|
|
34
|
+
result = result.replace(PHONE_INTL_RE, '[phone]');
|
|
35
|
+
result = result.replace(PHONE_US_RE, '[phone]');
|
|
36
|
+
return result;
|
|
37
|
+
}
|
|
38
|
+
//# sourceMappingURL=scrubber.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"scrubber.js","sourceRoot":"","sources":["../../src/capture/scrubber.ts"],"names":[],"mappings":"AAAA,0BAA0B;AAE1B,0BAA0B;AAC1B,MAAM,QAAQ,GAAG,iDAAiD,CAAC;AAEnE,2CAA2C;AAC3C,MAAM,aAAa,GAAG,kBAAkB,CAAC;AAEzC,mFAAmF;AACnF,MAAM,WAAW,GAAG,8DAA8D,CAAC;AAEnF,4DAA4D;AAC5D,MAAM,OAAO,GAAG,iDAAiD,CAAC;AAElE,gEAAgE;AAChE,MAAM,OAAO,GAAG,0CAA0C,CAAC;AAE3D,sBAAsB;AACtB,MAAM,MAAM,GAAG,wBAAwB,CAAC;AAExC;;;GAGG;AACH,MAAM,UAAU,QAAQ,CAAC,KAAa;IACpC,IAAI,MAAM,GAAG,KAAK,CAAC;IAEnB,yCAAyC;IACzC,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;IAE7C,+DAA+D;IAC/D,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAEzC,eAAe;IACf,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;IAE3C,6BAA6B;IAC7B,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,MAAM,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE;QAC1D,MAAM,MAAM,GAAG,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAC5C,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,GAAG,CAAC;YAAE,OAAO,MAAM,CAAC;QAC/C,OAAO,MAAM,CAAC;IAChB,CAAC,CAAC,CAAC;IAEH,iCAAiC;IACjC,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,aAAa,EAAE,SAAS,CAAC,CAAC;IAClD,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,WAAW,EAAE,SAAS,CAAC,CAAC;IAEhD,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import type { PageSnapshot, InteractionResult, FinishResult } from '../types.js';
|
|
2
|
+
export interface SessionOptions {
|
|
3
|
+
headless?: boolean;
|
|
4
|
+
allDomains?: boolean;
|
|
5
|
+
timeoutMs?: number;
|
|
6
|
+
skillsDir?: string;
|
|
7
|
+
authDir?: string;
|
|
8
|
+
}
|
|
9
|
+
export declare class CaptureSession {
|
|
10
|
+
readonly id: string;
|
|
11
|
+
private browser;
|
|
12
|
+
private page;
|
|
13
|
+
private generators;
|
|
14
|
+
private totalRequests;
|
|
15
|
+
private filteredRequests;
|
|
16
|
+
private targetUrl;
|
|
17
|
+
private options;
|
|
18
|
+
private captchaDetectedDomains;
|
|
19
|
+
private recentEndpoints;
|
|
20
|
+
private timeoutTimer;
|
|
21
|
+
private expired;
|
|
22
|
+
private closed;
|
|
23
|
+
constructor(options?: SessionOptions);
|
|
24
|
+
start(url: string): Promise<PageSnapshot>;
|
|
25
|
+
interact(action: InteractionAction): Promise<InteractionResult>;
|
|
26
|
+
finish(): Promise<FinishResult>;
|
|
27
|
+
abort(): Promise<void>;
|
|
28
|
+
/** Whether session has been terminated (expired, closed, or aborted) */
|
|
29
|
+
get isActive(): boolean;
|
|
30
|
+
private setupResponseListener;
|
|
31
|
+
private takeSnapshot;
|
|
32
|
+
private extractElements;
|
|
33
|
+
private resolveRef;
|
|
34
|
+
private emptySnapshot;
|
|
35
|
+
private cleanup;
|
|
36
|
+
}
|
|
37
|
+
export interface InteractionAction {
|
|
38
|
+
action: 'snapshot' | 'click' | 'type' | 'select' | 'navigate' | 'scroll' | 'wait';
|
|
39
|
+
ref?: string;
|
|
40
|
+
text?: string;
|
|
41
|
+
value?: string;
|
|
42
|
+
url?: string;
|
|
43
|
+
direction?: 'up' | 'down';
|
|
44
|
+
seconds?: number;
|
|
45
|
+
submit?: boolean;
|
|
46
|
+
}
|