@cosmocoder/mcp-web-docs 1.4.4 → 1.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/crawler/browser-config.js +9 -9
- package/build/crawler/browser-config.js.map +1 -1
- package/build/crawler/browser-config.test.js +9 -9
- package/build/crawler/browser-config.test.js.map +1 -1
- package/build/crawler/crawlee-crawler.d.ts +8 -0
- package/build/crawler/crawlee-crawler.js +78 -0
- package/build/crawler/crawlee-crawler.js.map +1 -1
- package/build/crawler/crawlee-crawler.test.js +1 -0
- package/build/crawler/crawlee-crawler.test.js.map +1 -1
- package/build/crawler/docs-crawler.d.ts +0 -1
- package/build/crawler/docs-crawler.js +6 -6
- package/build/crawler/docs-crawler.js.map +1 -1
- package/build/crawler/docs-crawler.test.js +16 -14
- package/build/crawler/docs-crawler.test.js.map +1 -1
- package/build/crawler/llms-txt.d.ts +10 -0
- package/build/crawler/llms-txt.js +107 -0
- package/build/crawler/llms-txt.js.map +1 -0
- package/build/crawler/llms-txt.test.d.ts +1 -0
- package/build/crawler/llms-txt.test.js +113 -0
- package/build/crawler/llms-txt.test.js.map +1 -0
- package/build/crawler/queue-manager.d.ts +10 -0
- package/build/crawler/queue-manager.js +45 -8
- package/build/crawler/queue-manager.js.map +1 -1
- package/build/crawler/queue-manager.test.js +104 -1
- package/build/crawler/queue-manager.test.js.map +1 -1
- package/build/util/docs.d.ts +12 -0
- package/build/util/docs.js +15 -0
- package/build/util/docs.js.map +1 -1
- package/build/util/docs.test.js +25 -1
- package/build/util/docs.test.js.map +1 -1
- package/package.json +3 -3
|
@@ -5,15 +5,16 @@ log.setLevel(log.LEVELS.OFF);
|
|
|
5
5
|
export const getBrowserConfig = (requestQueue) => ({
|
|
6
6
|
maxRequestsPerCrawl: 1000,
|
|
7
7
|
requestQueue,
|
|
8
|
-
maxConcurrency:
|
|
9
|
-
maxRequestsPerMinute:
|
|
10
|
-
maxRequestRetries:
|
|
11
|
-
navigationTimeoutSecs:
|
|
8
|
+
maxConcurrency: 5,
|
|
9
|
+
maxRequestsPerMinute: 120,
|
|
10
|
+
maxRequestRetries: 2,
|
|
11
|
+
navigationTimeoutSecs: 30,
|
|
12
|
+
requestHandlerTimeoutSecs: 60,
|
|
12
13
|
browserPoolOptions: {
|
|
13
|
-
maxOpenPagesPerBrowser:
|
|
14
|
-
useFingerprints:
|
|
15
|
-
operationTimeoutSecs:
|
|
16
|
-
closeInactiveBrowserAfterSecs:
|
|
14
|
+
maxOpenPagesPerBrowser: 3,
|
|
15
|
+
useFingerprints: true,
|
|
16
|
+
operationTimeoutSecs: 30,
|
|
17
|
+
closeInactiveBrowserAfterSecs: 20,
|
|
17
18
|
},
|
|
18
19
|
preNavigationHooks: [
|
|
19
20
|
async ({ page }) => {
|
|
@@ -21,7 +22,6 @@ export const getBrowserConfig = (requestQueue) => ({
|
|
|
21
22
|
await page.setExtraHTTPHeaders({
|
|
22
23
|
'Accept-Language': 'en-US,en;q=0.9',
|
|
23
24
|
Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
|
|
24
|
-
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
|
|
25
25
|
});
|
|
26
26
|
},
|
|
27
27
|
],
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"browser-config.js","sourceRoot":"","sources":["../../src/crawler/browser-config.ts"],"names":[],"mappings":"AAAA,OAAO,EAA4B,GAAG,EAAE,MAAM,SAAS,CAAC;AAExD,0DAA0D;AAC1D,2DAA2D;AAC3D,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;AAE7B,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,YAAsD,EAAqC,EAAE,CAAC,CAAC;IAC9H,mBAAmB,EAAE,IAAI;IACzB,YAAY;IACZ,cAAc,EAAE,
|
|
1
|
+
{"version":3,"file":"browser-config.js","sourceRoot":"","sources":["../../src/crawler/browser-config.ts"],"names":[],"mappings":"AAAA,OAAO,EAA4B,GAAG,EAAE,MAAM,SAAS,CAAC;AAExD,0DAA0D;AAC1D,2DAA2D;AAC3D,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;AAE7B,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,YAAsD,EAAqC,EAAE,CAAC,CAAC;IAC9H,mBAAmB,EAAE,IAAI;IACzB,YAAY;IACZ,cAAc,EAAE,CAAC;IACjB,oBAAoB,EAAE,GAAG;IACzB,iBAAiB,EAAE,CAAC;IACpB,qBAAqB,EAAE,EAAE;IACzB,yBAAyB,EAAE,EAAE;IAC7B,kBAAkB,EAAE;QAClB,sBAAsB,EAAE,CAAC;QACzB,eAAe,EAAE,IAAI;QACrB,oBAAoB,EAAE,EAAE;QACxB,6BAA6B,EAAE,EAAE;KAClC;IACD,kBAAkB,EAAE;QAClB,KAAK,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE;YACjB,MAAM,IAAI,CAAC,eAAe,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;YAC1D,MAAM,IAAI,CAAC,mBAAmB,CAAC;gBAC7B,iBAAiB,EAAE,gBAAgB;gBACnC,MAAM,EAAE,4EAA4E;aACrF,CAAC,CAAC;QACL,CAAC;KACF;CACF,CAAC,CAAC"}
|
|
@@ -5,10 +5,11 @@ describe('Browser Config', () => {
|
|
|
5
5
|
const mockQueue = {};
|
|
6
6
|
const config = getBrowserConfig(mockQueue);
|
|
7
7
|
expect(config.maxRequestsPerCrawl).toBe(1000);
|
|
8
|
-
expect(config.maxConcurrency).toBe(
|
|
9
|
-
expect(config.maxRequestsPerMinute).toBe(
|
|
10
|
-
expect(config.maxRequestRetries).toBe(
|
|
11
|
-
expect(config.navigationTimeoutSecs).toBe(
|
|
8
|
+
expect(config.maxConcurrency).toBe(5);
|
|
9
|
+
expect(config.maxRequestsPerMinute).toBe(120);
|
|
10
|
+
expect(config.maxRequestRetries).toBe(2);
|
|
11
|
+
expect(config.navigationTimeoutSecs).toBe(30);
|
|
12
|
+
expect(config.requestHandlerTimeoutSecs).toBe(60);
|
|
12
13
|
});
|
|
13
14
|
it('should return config with requestQueue', () => {
|
|
14
15
|
const mockQueue = { name: 'test-queue' };
|
|
@@ -19,10 +20,10 @@ describe('Browser Config', () => {
|
|
|
19
20
|
const mockQueue = {};
|
|
20
21
|
const config = getBrowserConfig(mockQueue);
|
|
21
22
|
expect(config.browserPoolOptions).toBeDefined();
|
|
22
|
-
expect(config.browserPoolOptions?.maxOpenPagesPerBrowser).toBe(
|
|
23
|
-
expect(config.browserPoolOptions?.useFingerprints).toBe(
|
|
24
|
-
expect(config.browserPoolOptions?.operationTimeoutSecs).toBe(
|
|
25
|
-
expect(config.browserPoolOptions?.closeInactiveBrowserAfterSecs).toBe(
|
|
23
|
+
expect(config.browserPoolOptions?.maxOpenPagesPerBrowser).toBe(3);
|
|
24
|
+
expect(config.browserPoolOptions?.useFingerprints).toBe(true);
|
|
25
|
+
expect(config.browserPoolOptions?.operationTimeoutSecs).toBe(30);
|
|
26
|
+
expect(config.browserPoolOptions?.closeInactiveBrowserAfterSecs).toBe(20);
|
|
26
27
|
});
|
|
27
28
|
it('should have preNavigationHooks', () => {
|
|
28
29
|
const mockQueue = {};
|
|
@@ -48,7 +49,6 @@ describe('Browser Config', () => {
|
|
|
48
49
|
expect(mockPage.setExtraHTTPHeaders).toHaveBeenCalledWith(expect.objectContaining({
|
|
49
50
|
'Accept-Language': expect.any(String),
|
|
50
51
|
Accept: expect.any(String),
|
|
51
|
-
'User-Agent': expect.any(String),
|
|
52
52
|
}));
|
|
53
53
|
});
|
|
54
54
|
});
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"browser-config.test.js","sourceRoot":"","sources":["../../src/crawler/browser-config.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AAIvD,QAAQ,CAAC,gBAAgB,EAAE,GAAG,EAAE;IAC9B,QAAQ,CAAC,kBAAkB,EAAE,GAAG,EAAE;QAChC,EAAE,CAAC,0CAA0C,EAAE,GAAG,EAAE;YAClD,MAAM,SAAS,GAAG,EAA4C,CAAC;YAC/D,MAAM,MAAM,GAAG,gBAAgB,CAAC,SAAS,CAAC,CAAC;YAE3C,MAAM,CAAC,MAAM,CAAC,mBAAmB,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC9C,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,
|
|
1
|
+
{"version":3,"file":"browser-config.test.js","sourceRoot":"","sources":["../../src/crawler/browser-config.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AAIvD,QAAQ,CAAC,gBAAgB,EAAE,GAAG,EAAE;IAC9B,QAAQ,CAAC,kBAAkB,EAAE,GAAG,EAAE;QAChC,EAAE,CAAC,0CAA0C,EAAE,GAAG,EAAE;YAClD,MAAM,SAAS,GAAG,EAA4C,CAAC;YAC/D,MAAM,MAAM,GAAG,gBAAgB,CAAC,SAAS,CAAC,CAAC;YAE3C,MAAM,CAAC,MAAM,CAAC,mBAAmB,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC9C,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACtC,MAAM,CAAC,MAAM,CAAC,oBAAoB,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAC9C,MAAM,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACzC,MAAM,CAAC,MAAM,CAAC,qBAAqB,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAC9C,MAAM,CAAC,MAAM,CAAC,yBAAyB,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACpD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,wCAAwC,EAAE,GAAG,EAAE;YAChD,MAAM,SAAS,GAAG,EAAE,IAAI,EAAE,YAAY,EAA4C,CAAC;YACnF,MAAM,MAAM,GAAG,gBAAgB,CAAC,SAAS,CAAC,CAAC;YAE3C,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAC9C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,kCAAkC,EAAE,GAAG,EAAE;YAC1C,MAAM,SAAS,GAAG,EAA4C,CAAC;YAC/D,MAAM,MAAM,GAAG,gBAAgB,CAAC,SAAS,CAAC,CAAC;YAE3C,MAAM,CAAC,MAAM,CAAC,kBAAkB,CAAC,CAAC,WAAW,EAAE,CAAC;YAChD,MAAM,CAAC,MAAM,CAAC,kBAAkB,EAAE,sBAAsB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAClE,MAAM,CAAC,MAAM,CAAC,kBAAkB,EAAE,eAAe,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC9D,MAAM,CAAC,MAAM,CAAC,kBAAkB,EAAE,oBAAoB,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACjE,MAAM,CAAC,MAAM,CAAC,kBAAkB,EAAE,6BAA6B,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC5E,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gCAAgC,EAAE,GAAG,EAAE;YACxC,MAAM,SAAS,GAAG,EAA4C,CAAC;YAC/D,MAAM,MAAM,GAAG,gBAAgB,CAAC,SAAS,CAAC,CAAC;YAE3C,MAAM,CAAC,MAAM,CAAC,kBAAkB,CAAC,CAAC,WAAW,EAAE,CAAC;YAChD,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,kBAAkB,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC5D,MAAM,CAAC,MAAM,CAAC,kBAAkB,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACpD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,4CAA4C,EAAE,KAAK,IAAI,EAAE;YAC1D,MAAM,SAAS,GAAG,EAA4C,CAAC;YAC/D,MAAM,MAAM,GAAG,gBAAgB,CAAC,SAAS,CAAC,CAAC;YAE3C,MAAM,QAAQ,GAAG;gBACf,eAAe,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,SAAS,CAAC;gBACrD,mBAAmB,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,SAAS,CAAC;aACvC,CAAC;YAErB,MAAM,WAAW,GAAG,EAAE,IAAI,EAAE,QAAQ,EAA+B,CAAC;YACpE,MAAM,eAAe,GAAG,EAA2B,CAAC;YAEpD,MAAM,IAAI,GAAG,MAAM,CAAC,kBAAkB,EAAE,CAAC,CAAC,CAAC,CAAC;YAC5C,IAAI,IAAI,EAAE,CAAC;gBACT,MAAM,IAAI,CAAC,WAAW,EAAE,eAAe,CAAC,CAAC;YAC3C,CAAC;YAED,MAAM,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC,oBAAoB,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;YACrF,MAAM,CAAC,QAAQ,CAAC,mBAAmB,CAAC,CAAC,oBAAoB,CACvD,MAAM,CAAC,gBAAgB,CAAC;gBACtB,iBAAiB,EAAE,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC;gBACrC,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC;aAC3B,CAAC,CACH,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
@@ -50,6 +50,14 @@ export declare class CrawleeCrawler extends BaseCrawler {
|
|
|
50
50
|
* This is used to detect expired sessions during crawling.
|
|
51
51
|
*/
|
|
52
52
|
private checkForLoginPage;
|
|
53
|
+
/**
|
|
54
|
+
* Wait for the page to fully stabilize after navigation, handling:
|
|
55
|
+
* - Client-side redirects (meta-refresh, JavaScript location changes) common in Docusaurus
|
|
56
|
+
* - Cloudflare challenge interstitials
|
|
57
|
+
*
|
|
58
|
+
* Must be called after initial waitForLoadState, before content extraction.
|
|
59
|
+
*/
|
|
60
|
+
private waitForPageStabilization;
|
|
53
61
|
private findContentFrame;
|
|
54
62
|
private evaluateExtractor;
|
|
55
63
|
private extractContent;
|
|
@@ -89,6 +89,80 @@ export class CrawleeCrawler extends BaseCrawler {
|
|
|
89
89
|
}
|
|
90
90
|
return false;
|
|
91
91
|
}
|
|
92
|
+
/**
|
|
93
|
+
* Wait for the page to fully stabilize after navigation, handling:
|
|
94
|
+
* - Client-side redirects (meta-refresh, JavaScript location changes) common in Docusaurus
|
|
95
|
+
* - Cloudflare challenge interstitials
|
|
96
|
+
*
|
|
97
|
+
* Must be called after initial waitForLoadState, before content extraction.
|
|
98
|
+
*/
|
|
99
|
+
async waitForPageStabilization(page) {
|
|
100
|
+
const initialUrl = page.url();
|
|
101
|
+
// Detect client-side redirect pages (Docusaurus generates these for /docs/ → /docs/intro/)
|
|
102
|
+
// These are tiny HTML pages with a meta-refresh and/or JS redirect but no real content.
|
|
103
|
+
let isRedirectPage = false;
|
|
104
|
+
try {
|
|
105
|
+
isRedirectPage = await page.evaluate(() => {
|
|
106
|
+
const metaRefresh = document.querySelector('meta[http-equiv="refresh"]');
|
|
107
|
+
if (metaRefresh)
|
|
108
|
+
return true;
|
|
109
|
+
const body = document.body;
|
|
110
|
+
const hasMinimalContent = !body || (body.textContent?.trim().length || 0) < 100;
|
|
111
|
+
const hasNoMainContent = !document.querySelector('main, article, [role="main"], .content, #content');
|
|
112
|
+
return hasMinimalContent && hasNoMainContent;
|
|
113
|
+
});
|
|
114
|
+
}
|
|
115
|
+
catch {
|
|
116
|
+
// evaluate failed — page is likely mid-navigation already, which is fine
|
|
117
|
+
isRedirectPage = true;
|
|
118
|
+
}
|
|
119
|
+
if (isRedirectPage) {
|
|
120
|
+
logger.debug(`[CrawleeCrawler] Redirect/minimal page detected at ${initialUrl}, waiting for navigation...`);
|
|
121
|
+
try {
|
|
122
|
+
await page.waitForURL((url) => url.href !== initialUrl, { timeout: 10000 });
|
|
123
|
+
await Promise.all([
|
|
124
|
+
page.waitForLoadState('domcontentloaded'),
|
|
125
|
+
page.waitForLoadState('networkidle', { timeout: 10000 }).catch(() => { }),
|
|
126
|
+
]);
|
|
127
|
+
logger.debug(`[CrawleeCrawler] Redirect completed: ${initialUrl} → ${page.url()}`);
|
|
128
|
+
}
|
|
129
|
+
catch {
|
|
130
|
+
logger.debug(`[CrawleeCrawler] No redirect detected within timeout, continuing with current page`);
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
// Detect Cloudflare challenge pages (after any redirect has settled)
|
|
134
|
+
try {
|
|
135
|
+
const isChallenge = await page.evaluate(() => {
|
|
136
|
+
const bodyText = document.body?.textContent || '';
|
|
137
|
+
const hasChallengeText = bodyText.includes('Checking your browser') ||
|
|
138
|
+
bodyText.includes('Verify you are human') ||
|
|
139
|
+
bodyText.includes('Enable JavaScript and cookies');
|
|
140
|
+
const hasChallengeElement = document.querySelector('#challenge-running, #challenge-stage, .cf-browser-verification, #cf-wrapper') !== null;
|
|
141
|
+
return hasChallengeText || hasChallengeElement;
|
|
142
|
+
});
|
|
143
|
+
if (isChallenge) {
|
|
144
|
+
logger.info(`[CrawleeCrawler] Cloudflare challenge detected, waiting for resolution...`);
|
|
145
|
+
try {
|
|
146
|
+
await page.waitForFunction(() => {
|
|
147
|
+
const bodyText = document.body?.textContent || '';
|
|
148
|
+
const stillChallenge = bodyText.includes('Checking your browser') || bodyText.includes('Verify you are human');
|
|
149
|
+
const hasChallengeElement = document.querySelector('#challenge-running, #challenge-stage, .cf-browser-verification') !== null;
|
|
150
|
+
return !stillChallenge && !hasChallengeElement;
|
|
151
|
+
}, { timeout: 15000 });
|
|
152
|
+
await page.waitForLoadState('domcontentloaded');
|
|
153
|
+
await page.waitForLoadState('networkidle', { timeout: 5000 }).catch(() => { });
|
|
154
|
+
logger.info(`[CrawleeCrawler] Cloudflare challenge resolved`);
|
|
155
|
+
}
|
|
156
|
+
catch {
|
|
157
|
+
logger.warn(`[CrawleeCrawler] Cloudflare challenge did not resolve within timeout`);
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
catch {
|
|
162
|
+
// evaluate failed during challenge detection — page may be navigating, proceed anyway
|
|
163
|
+
logger.debug(`[CrawleeCrawler] Could not check for Cloudflare challenge, continuing`);
|
|
164
|
+
}
|
|
165
|
+
}
|
|
92
166
|
async findContentFrame(page) {
|
|
93
167
|
const frames = await page.frames();
|
|
94
168
|
const contentFrames = await Promise.all(frames.map(async (frame) => {
|
|
@@ -178,6 +252,8 @@ export class CrawleeCrawler extends BaseCrawler {
|
|
|
178
252
|
this.allowedHostname = '';
|
|
179
253
|
}
|
|
180
254
|
await this.queueManager.initialize(url, this.pathPrefix);
|
|
255
|
+
// Seed URLs from llms.txt for better coverage on bot-protected sites
|
|
256
|
+
await this.queueManager.seedFromLlmsTxt(url);
|
|
181
257
|
// Build crawler options with optional authentication
|
|
182
258
|
const crawlerOptions = getBrowserConfig(this.queueManager.getRequestQueue() ?? undefined);
|
|
183
259
|
// If we have storage state (auth cookies), configure the browser to use them
|
|
@@ -223,6 +299,8 @@ export class CrawleeCrawler extends BaseCrawler {
|
|
|
223
299
|
page.waitForLoadState('domcontentloaded'),
|
|
224
300
|
page.waitForLoadState('networkidle', { timeout: 5000 }).catch(() => log.debug('Network idle timeout - continuing anyway')),
|
|
225
301
|
]);
|
|
302
|
+
// Handle client-side redirects (Docusaurus) and Cloudflare challenges
|
|
303
|
+
await this.waitForPageStabilization(page);
|
|
226
304
|
// Get the actual URL after any redirects
|
|
227
305
|
const actualUrl = page.url();
|
|
228
306
|
// Check if the page redirected outside the allowed domain
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"crawlee-crawler.js","sourceRoot":"","sources":["../../src/crawler/crawlee-crawler.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,SAAS,CAAC;AAE5C,OAAO,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAExC,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAE5C,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AACvD,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAC3C,OAAO,EAAE,eAAe,EAAE,cAAc,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAoB3F,MAAM,OAAO,cAAe,SAAQ,WAAW;IACrC,OAAO,GAA6B,IAAI,CAAC;IACzC,YAAY,GAAiB,IAAI,YAAY,EAAE,CAAC;IAChD,YAAY,CAAgB;IAC5B,WAAW,GAAY,IAAI,CAAC;IAC5B,mBAAmB,GAA+B,IAAI,CAAC;IACvD,WAAW,GAAW,EAAE,CAAC;IACjC,gFAAgF;IACxE,eAAe,GAAW,EAAE,CAAC;IACrC,iDAAiD;IACzC,oBAAoB,GAAW,CAAC,CAAC;IACzC,gDAAgD;IACxC,UAAU,CAAU;IAE5B;;OAEG;IACH,eAAe,CAAC,KAAmB;QACjC,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC;QAC1B,MAAM,CAAC,IAAI,CAAC,2CAA2C,KAAK,CAAC,OAAO,EAAE,MAAM,IAAI,CAAC,UAAU,CAAC,CAAC;IAC/F,CAAC;IAED;;;;;;OAMG;IACK,qBAAqB,CAAC,GAAW;QACvC,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC;YAC1B,OAAO,IAAI,CAAC,CAAC,4BAA4B;QAC3C,CAAC;QAED,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;YAC5B,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;YAC/C,MAAM,OAAO,GAAG,IAAI,CAAC,eAAe,CAAC,WAAW,EAAE,CAAC;YAEnD,cAAc;YACd,IAAI,QAAQ,KAAK,OAAO,EAAE,CAAC;gBACzB,OAAO,IAAI,CAAC;YACd,CAAC;YAED,wEAAwE;YACxE,2EAA2E;YAC3E,IAAI,QAAQ,CAAC,QAAQ,CAAC,GAAG,GAAG,OAAO,CAAC,EAAE,CAAC;gBACrC,OAAO,IAAI,CAAC;YACd,CAAC;YAED,OAAO,KAAK,CAAC;QACf,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,iBAAiB,CAAC,IAAU,EAAE,UAAkB;QAC5D,+CAA+C;QAC/C,uEAAuE;QACvE,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YACtB,OAAO,KAAK,CAAC;QACf,CAAC;QAED,iCAAiC;QACjC,IAAI,cAAc,CAAC,UAAU,CAAC,EAAE,CAAC;YAC/B,MAAM,CAAC,IAAI,CAAC,0DAA0D,UAAU,EAAE,CAAC,CAAC;YACpF,OAAO,IAAI,CAAC;QACd,CAAC;QAED,qBAAqB;QACrB,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,IAAI,EAAE,WAAW,IAAI,EAAE,CAAC,CAAC;YAC7E,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;YACtC,MAAM,SAAS,GAAG,eAAe,CAAC,QAAQ,GAAG,QAAQ,EAAE,UAAU,CAAC,CAAC;YAEnE,IAAI,SAAS,CAAC,WAAW,IAAI,SAAS,CAAC,UAAU,IAAI,GAAG,EAAE,CAAC;gBACzD,MAAM,CAAC,IAAI,CAAC,uEAAuE,SAAS,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;gBACvH,MAAM,CAAC,KAAK,CAAC,uCAAuC,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gBAEpF,wEAAwE;gBACxE,IAAI,CAAC,mBAAmB,GAAG,IAAI,mBAAmB,CAChD,mEAAmE,EACnE,IAAI,CAAC,WAAW,EAChB,UAAU,EACV,SAAS,CACV,CAAC;gBACF,OAAO,IAAI,CAAC;YACd,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,KAAK,CAAC,iDAAiD,EAAE,KAAK,CAAC,CAAC;QACzE,CAAC;QAED,OAAO,KAAK,CAAC;IACf,CAAC;IAEO,KAAK,CAAC,gBAAgB,CAAC,IAAU;QACvC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,EAAE,CAAC;QACnC,MAAM,aAAa,GAAG,MAAM,OAAO,CAAC,GAAG,CACrC,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE;YACzB,IAAI,CAAC;gBACH,MAAM,UAAU,GAAG,MAAM,KAAK;qBAC3B,QAAQ,CAAC,GAAG,EAAE;oBACb,OAAO,QAAQ,CAAC,aAAa,CAAC,6DAA6D,CAAC,KAAK,IAAI,CAAC;gBACxG,CAAC,CAAC;qBACD,KAAK,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC;gBAEtB,IAAI,UAAU,EAAE,CAAC;oBACf,MAAM,OAAO,CAAC,GAAG,CAAC;wBAChB,KAAK,CAAC,gBAAgB,CAAC,kBAAkB,CAAC;wBAC1C,KAAK;6BACF,gBAAgB,CAAC,aAAa,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;6BAClD,KAAK,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,gDAAgD,CAAC,CAAC;qBAC/E,CAAC,CAAC;oBACH,OAAO,KAAK,CAAC;gBACf,CAAC;YACH,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,CAAC,KAAK,CAAC,sBAAsB,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YACjE,CAAC;YACD,OAAO,IAAI,CAAC;QACd,CAAC,CAAC,CACH,CAAC;QAEF,MAAM,KAAK,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,IAAI,IAAI,CAAC;QAC5D,IAAI,KAAK,EAAE,CAAC;YACV,MAAM,CAAC,KAAK,CAAC,yBAAyB,CAAC,CAAC;YACxC,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC;QAC5D,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IAEO,KAAK,CAAC,iBAAiB,CAAC,OAAqB,EAAE,SAA2B;QAChF,MAAM,aAAa,GAAG,SAAS,CAAC,WAAW,CAAC,QAAQ,EAAE,CAAC;QACvD,OAAO,OAAO,CAAC,QAAQ,CAAC,KAAK,EAAE,IAAY,EAAE,EAAE;YAC7C,MAAM,cAAc,GAAG,IAAI,QAAQ,CAAC,UAAU,IAAI,EAAE,CAAC,EAAE,CAAC;YACxD,MAAM,SAAS,GAAG,IAAI,cAAc,EAAE,CAAC;YACvC,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;YACxD,OAAO,MAAM,CAAC,OAAO,CAAC;QACxB,CAAC,EAAE,aAAa,CAAC,CAAC;IACpB,CAAC;IAEO,KAAK,CAAC,cAAc,CAC1B,IAAU,EACV,QAAgB,EAChB,SAA2B;QAE3B,IAAI,OAAO,GAAG,EAAE,CAAC;QACjB,IAAI,aAAa,GAAG,SAAS,CAAC,WAAW,CAAC,IAAI,CAAC;QAE/C,IAAI,CAAC;YACH,IAAI,QAAQ,KAAK,WAAW,EAAE,CAAC;gBAC7B,mBAAmB;gBACnB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC;gBAChD,IAAI,KAAK,EAAE,CAAC;oBACV,OAAO,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;gBAC3D,CAAC;gBAED,wBAAwB;gBACxB,IAAI,CAAC,OAAO,EAAE,CAAC;oBACb,OAAO,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;gBAC1D,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,OAAO,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;YAC1D,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAS,GAAG,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC;YAC7E,aAAa,GAAG,eAAe,CAAC;QAClC,CAAC;QAED,OAAO,EAAE,OAAO,EAAE,aAAa,EAAE,CAAC;IACpC,CAAC;IAED;;;OAGG;IACH,aAAa,CAAC,MAAc;QAC1B,IAAI,CAAC,UAAU,GAAG,MAAM,CAAC;QACzB,MAAM,CAAC,IAAI,CAAC,iDAAiD,MAAM,EAAE,CAAC,CAAC;IACzE,CAAC;IAED,KAAK,CAAC,CAAC,KAAK,CAAC,GAAW;QACtB,MAAM,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,wBAAwB,GAAG,EAAE,CAAC,CAAC;QAErE,6BAA6B;QAC7B,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;QACxB,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC;QAChC,IAAI,CAAC,WAAW,GAAG,GAAG,CAAC;QACvB,IAAI,CAAC,oBAAoB,GAAG,CAAC,CAAC;QAE9B,8DAA8D;QAC9D,IAAI,CAAC;YACH,IAAI,CAAC,eAAe,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;YAC7C,MAAM,CAAC,IAAI,CAAC,+DAA+D,IAAI,CAAC,eAAe,EAAE,CAAC,CAAC;QACrG,CAAC;QAAC,MAAM,CAAC;YACP,IAAI,CAAC,eAAe,GAAG,EAAE,CAAC;QAC5B,CAAC;QAED,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;QAEzD,qDAAqD;QACrD,MAAM,cAAc,GAAG,gBAAgB,CAAC,IAAI,CAAC,YAAY,CAAC,eAAe,EAAE,IAAI,SAAS,CAAC,CAAC;QAE1F,6EAA6E;QAC7E,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACtB,MAAM,CAAC,IAAI,CAAC,qDAAqD,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE,MAAM,IAAI,CAAC,UAAU,CAAC,CAAC;YACnH,cAAc,CAAC,aAAa,GAAG;gBAC7B,GAAG,cAAc,CAAC,aAAa;gBAC/B,aAAa,EAAE;oBACb,GAAG,cAAc,CAAC,aAAa,EAAE,aAAa;iBAC/C;aACF,CAAC;YACF,cAAc,CAAC,kBAAkB,GAAG;gBAClC,GAAG,cAAc,CAAC,kBAAkB;gBACpC,cAAc,EAAE;oBACd,KAAK,EAAE,MAAM,EAAE,EAAE;wBACf,0DAA0D;wBAC1D,MAAM,CAAC,KAAK,CAAC,+CAA+C,MAAM,EAAE,CAAC,CAAC;oBACxE,CAAC;iBACF;aACF,CAAC;YACF,qCAAqC;YACrC,MAAM,aAAa,GAAG,cAAc,CAAC,kBAAkB,IAAI,EAAE,CAAC;YAC9D,cAAc,CAAC,kBAAkB,GAAG;gBAClC,GAAG,aAAa;gBAChB,KAAK,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE;oBACjB,IAAI,IAAI,CAAC,YAAY,EAAE,OAAO,EAAE,CAAC;wBAC/B,MAAM,CAAC,KAAK,CAAC,4BAA4B,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,MAAM,4BAA4B,CAAC,CAAC;wBACvG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;oBAC7D,CAAC;gBACH,CAAC;aACF,CAAC;QACJ,CAAC;QAED,IAAI,CAAC,OAAO,GAAG,IAAI,iBAAiB,CAAC;YACnC,GAAG,cAAc;YACjB,cAAc,EAAE,KAAK,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,GAAG,EAAE,EAAE,EAAE;gBAC7D,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;oBACpB,GAAG,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;oBAC3B,OAAO;gBACT,CAAC;gBAED,IAAI,CAAC;oBACH,6BAA6B;oBAC7B,MAAM,OAAO,CAAC,GAAG,CAAC;wBAChB,IAAI,CAAC,gBAAgB,CAAC,kBAAkB,CAAC;wBACzC,IAAI,CAAC,gBAAgB,CAAC,aAAa,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,0CAA0C,CAAC,CAAC;qBAC3H,CAAC,CAAC;oBAEH,yCAAyC;oBACzC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;oBAE7B,0DAA0D;oBAC1D,IAAI,CAAC,IAAI,CAAC,qBAAqB,CAAC,SAAS,CAAC,EAAE,CAAC;wBAC3C,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;wBACxD,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC,QAAQ,CAAC;wBAEnD,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;4BACrB,gFAAgF;4BAChF,MAAM,CAAC,IAAI,CAAC,kEAAkE,iBAAiB,MAAM,cAAc,EAAE,CAAC,CAAC;4BAEvH,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;gCACtB,mDAAmD;gCACnD,IAAI,CAAC,mBAAmB,GAAG,IAAI,mBAAmB,CAChD,4EAA4E,cAAc,GAAG,EAC7F,IAAI,CAAC,WAAW,EAChB,SAAS,EACT,EAAE,WAAW,EAAE,IAAI,EAAE,UAAU,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,mBAAmB,iBAAiB,OAAO,cAAc,EAAE,CAAC,EAAE,CAC/G,CAAC;gCACF,GAAG,CAAC,KAAK,CAAC,oDAAoD,cAAc,mBAAmB,CAAC,CAAC;gCACjG,IAAI,CAAC,KAAK,EAAE,CAAC;gCACb,OAAO;4BACT,CAAC;iCAAM,CAAC;gCACN,0DAA0D;gCAC1D,GAAG,CAAC,KAAK,CAAC,6CAA6C,cAAc,mBAAmB,CAAC,CAAC;gCAC1F,IAAI,CAAC,KAAK,EAAE,CAAC;gCACb,OAAO;4BACT,CAAC;wBACH,CAAC;6BAAM,CAAC;4BACN,sDAAsD;4BACtD,IAAI,CAAC,oBAAoB,EAAE,CAAC;4BAC5B,GAAG,CAAC,OAAO,CACT,iDAAiD,OAAO,CAAC,GAAG,MAAM,SAAS,aAAa,IAAI,CAAC,oBAAoB,kBAAkB,CACpI,CAAC;4BACF,OAAO;wBACT,CAAC;oBACH,CAAC;oBAED,gEAAgE;oBAChE,IAAI,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;wBAC1C,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;wBAClE,IAAI,WAAW,EAAE,CAAC;4BAChB,GAAG,CAAC,KAAK,CAAC,uEAAuE,CAAC,CAAC;4BACnF,IAAI,CAAC,KAAK,EAAE,CAAC;4BACb,OAAO;wBACT,CAAC;wBACD,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;oBAC3B,CAAC;yBAAM,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;wBAC5B,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;oBAC3B,CAAC;oBAED,qCAAqC;oBACrC,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;wBAC7B,IAAI,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;4BAC5B,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;gCACjB,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;4BAChC,CAAC;4BAED,MAAM,IAAI,CAAC,YAAY,CAAC,mBAAmB,CAAC,YAAY,EAAE,GAAG,EAAE,IAAI,CAAC,CAAC;4BAErE,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;4BACjC,MAAM,EAAE,OAAO,EAAE,aAAa,EAAE,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;4BAE9F,MAAM,MAAM,GAAgB;gCAC1B,GAAG,EAAE,OAAO,CAAC,GAAG;gCAChB,IAAI,EAAE,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,QAAQ,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM;gCACjE,OAAO,EAAE,YAAY,CAAC,OAAO,CAAC;gCAC9B,KAAK;gCACL,aAAa;6BACd,CAAC;4BAEF,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;4BACpC,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;4BACnC,MAAM;wBACR,CAAC;oBACH,CAAC;gBACH,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,MAAM,YAAY,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;oBAC5E,GAAG,CAAC,KAAK,CAAC,oBAAoB,OAAO,CAAC,GAAG,KAAK,YAAY,EAAE,CAAC,CAAC;gBAChE,CAAC;YACH,CAAC;SACF,CAAC,CAAC;QAEH,IAAI,CAAC;YACH,MAAM,cAAc,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC;YAE1C,OAAO,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC;gBACxB,IAAI,IAAI,CAAC,YAAY,CAAC,gBAAgB,EAAE,EAAE,CAAC;oBACzC,KAAK,MAAM,MAAM,IAAI,MAAM,IAAI,CAAC,YAAY,CAAC,YAAY,EAAE,EAAE,CAAC;wBAC5D,MAAM,MAAM,CAAC;oBACf,CAAC;gBACH,CAAC;gBAED,IAAI,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,cAAc,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,EAAE,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;oBAAE,MAAM;YACpI,CAAC;YAED,MAAM,cAAc,CAAC;YACrB,MAAM,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC;YAEjC,4CAA4C;YAC5C,IAAI,IAAI,CAAC,oBAAoB,GAAG,CAAC,EAAE,CAAC;gBAClC,MAAM,CAAC,IAAI,CACT,4BAA4B,IAAI,CAAC,oBAAoB,sDAAsD,IAAI,CAAC,eAAe,GAAG,CACnI,CAAC;YACJ,CAAC;YAED,0DAA0D;YAC1D,IAAI,IAAI,CAAC,mBAAmB,EAAE,CAAC;gBAC7B,MAAM,IAAI,CAAC,mBAAmB,CAAC;YACjC,CAAC;YAED,KAAK,MAAM,MAAM,IAAI,MAAM,IAAI,CAAC,YAAY,CAAC,YAAY,EAAE,EAAE,CAAC;gBAC5D,MAAM,MAAM,CAAC;YACf,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,wCAAwC;YACxC,IAAI,KAAK,YAAY,mBAAmB,EAAE,CAAC;gBACzC,MAAM,KAAK,CAAC;YACd,CAAC;YACD,MAAM,CAAC,KAAK,CAAC,gBAAgB,EAAE,KAAK,CAAC,CAAC;YACtC,MAAM,KAAK,CAAC;QACd,CAAC;gBAAS,CAAC;YACT,MAAM,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE,CAAC;YAClC,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACtB,CAAC;IACH,CAAC;IAED,KAAK;QACH,KAAK,CAAC,KAAK,EAAE,CAAC;QACd,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,6BAA6B,EAAE,GAAG,CAAC,CAAC,CAAC;QAC3F,CAAC;IACH,CAAC;CACF"}
|
|
1
|
+
{"version":3,"file":"crawlee-crawler.js","sourceRoot":"","sources":["../../src/crawler/crawlee-crawler.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,SAAS,CAAC;AAE5C,OAAO,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAExC,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAE5C,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AACvD,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAC3C,OAAO,EAAE,eAAe,EAAE,cAAc,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAoB3F,MAAM,OAAO,cAAe,SAAQ,WAAW;IACrC,OAAO,GAA6B,IAAI,CAAC;IACzC,YAAY,GAAiB,IAAI,YAAY,EAAE,CAAC;IAChD,YAAY,CAAgB;IAC5B,WAAW,GAAY,IAAI,CAAC;IAC5B,mBAAmB,GAA+B,IAAI,CAAC;IACvD,WAAW,GAAW,EAAE,CAAC;IACjC,gFAAgF;IACxE,eAAe,GAAW,EAAE,CAAC;IACrC,iDAAiD;IACzC,oBAAoB,GAAW,CAAC,CAAC;IACzC,gDAAgD;IACxC,UAAU,CAAU;IAE5B;;OAEG;IACH,eAAe,CAAC,KAAmB;QACjC,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC;QAC1B,MAAM,CAAC,IAAI,CAAC,2CAA2C,KAAK,CAAC,OAAO,EAAE,MAAM,IAAI,CAAC,UAAU,CAAC,CAAC;IAC/F,CAAC;IAED;;;;;;OAMG;IACK,qBAAqB,CAAC,GAAW;QACvC,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC;YAC1B,OAAO,IAAI,CAAC,CAAC,4BAA4B;QAC3C,CAAC;QAED,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;YAC5B,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;YAC/C,MAAM,OAAO,GAAG,IAAI,CAAC,eAAe,CAAC,WAAW,EAAE,CAAC;YAEnD,cAAc;YACd,IAAI,QAAQ,KAAK,OAAO,EAAE,CAAC;gBACzB,OAAO,IAAI,CAAC;YACd,CAAC;YAED,wEAAwE;YACxE,2EAA2E;YAC3E,IAAI,QAAQ,CAAC,QAAQ,CAAC,GAAG,GAAG,OAAO,CAAC,EAAE,CAAC;gBACrC,OAAO,IAAI,CAAC;YACd,CAAC;YAED,OAAO,KAAK,CAAC;QACf,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,iBAAiB,CAAC,IAAU,EAAE,UAAkB;QAC5D,+CAA+C;QAC/C,uEAAuE;QACvE,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YACtB,OAAO,KAAK,CAAC;QACf,CAAC;QAED,iCAAiC;QACjC,IAAI,cAAc,CAAC,UAAU,CAAC,EAAE,CAAC;YAC/B,MAAM,CAAC,IAAI,CAAC,0DAA0D,UAAU,EAAE,CAAC,CAAC;YACpF,OAAO,IAAI,CAAC;QACd,CAAC;QAED,qBAAqB;QACrB,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,IAAI,EAAE,WAAW,IAAI,EAAE,CAAC,CAAC;YAC7E,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;YACtC,MAAM,SAAS,GAAG,eAAe,CAAC,QAAQ,GAAG,QAAQ,EAAE,UAAU,CAAC,CAAC;YAEnE,IAAI,SAAS,CAAC,WAAW,IAAI,SAAS,CAAC,UAAU,IAAI,GAAG,EAAE,CAAC;gBACzD,MAAM,CAAC,IAAI,CAAC,uEAAuE,SAAS,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;gBACvH,MAAM,CAAC,KAAK,CAAC,uCAAuC,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gBAEpF,wEAAwE;gBACxE,IAAI,CAAC,mBAAmB,GAAG,IAAI,mBAAmB,CAChD,mEAAmE,EACnE,IAAI,CAAC,WAAW,EAChB,UAAU,EACV,SAAS,CACV,CAAC;gBACF,OAAO,IAAI,CAAC;YACd,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,KAAK,CAAC,iDAAiD,EAAE,KAAK,CAAC,CAAC;QACzE,CAAC;QAED,OAAO,KAAK,CAAC;IACf,CAAC;IAED;;;;;;OAMG;IACK,KAAK,CAAC,wBAAwB,CAAC,IAAU;QAC/C,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAE9B,2FAA2F;QAC3F,wFAAwF;QACxF,IAAI,cAAc,GAAG,KAAK,CAAC;QAC3B,IAAI,CAAC;YACH,cAAc,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE;gBACxC,MAAM,WAAW,GAAG,QAAQ,CAAC,aAAa,CAAC,4BAA4B,CAAC,CAAC;gBACzE,IAAI,WAAW;oBAAE,OAAO,IAAI,CAAC;gBAC7B,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC;gBAC3B,MAAM,iBAAiB,GAAG,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,EAAE,CAAC,MAAM,IAAI,CAAC,CAAC,GAAG,GAAG,CAAC;gBAChF,MAAM,gBAAgB,GAAG,CAAC,QAAQ,CAAC,aAAa,CAAC,kDAAkD,CAAC,CAAC;gBACrG,OAAO,iBAAiB,IAAI,gBAAgB,CAAC;YAC/C,CAAC,CAAC,CAAC;QACL,CAAC;QAAC,MAAM,CAAC;YACP,yEAAyE;YACzE,cAAc,GAAG,IAAI,CAAC;QACxB,CAAC;QAED,IAAI,cAAc,EAAE,CAAC;YACnB,MAAM,CAAC,KAAK,CAAC,sDAAsD,UAAU,6BAA6B,CAAC,CAAC;YAC5G,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,UAAU,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,KAAK,UAAU,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;gBAC5E,MAAM,OAAO,CAAC,GAAG,CAAC;oBAChB,IAAI,CAAC,gBAAgB,CAAC,kBAAkB,CAAC;oBACzC,IAAI,CAAC,gBAAgB,CAAC,aAAa,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC;iBACzE,CAAC,CAAC;gBACH,MAAM,CAAC,KAAK,CAAC,wCAAwC,UAAU,MAAM,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YACrF,CAAC;YAAC,MAAM,CAAC;gBACP,MAAM,CAAC,KAAK,CAAC,oFAAoF,CAAC,CAAC;YACrG,CAAC;QACH,CAAC;QAED,qEAAqE;QACrE,IAAI,CAAC;YACH,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE;gBAC3C,MAAM,QAAQ,GAAG,QAAQ,CAAC,IAAI,EAAE,WAAW,IAAI,EAAE,CAAC;gBAClD,MAAM,gBAAgB,GACpB,QAAQ,CAAC,QAAQ,CAAC,uBAAuB,CAAC;oBAC1C,QAAQ,CAAC,QAAQ,CAAC,sBAAsB,CAAC;oBACzC,QAAQ,CAAC,QAAQ,CAAC,+BAA+B,CAAC,CAAC;gBACrD,MAAM,mBAAmB,GACvB,QAAQ,CAAC,aAAa,CAAC,6EAA6E,CAAC,KAAK,IAAI,CAAC;gBACjH,OAAO,gBAAgB,IAAI,mBAAmB,CAAC;YACjD,CAAC,CAAC,CAAC;YAEH,IAAI,WAAW,EAAE,CAAC;gBAChB,MAAM,CAAC,IAAI,CAAC,2EAA2E,CAAC,CAAC;gBACzF,IAAI,CAAC;oBACH,MAAM,IAAI,CAAC,eAAe,CACxB,GAAG,EAAE;wBACH,MAAM,QAAQ,GAAG,QAAQ,CAAC,IAAI,EAAE,WAAW,IAAI,EAAE,CAAC;wBAClD,MAAM,cAAc,GAAG,QAAQ,CAAC,QAAQ,CAAC,uBAAuB,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,sBAAsB,CAAC,CAAC;wBAC/G,MAAM,mBAAmB,GAAG,QAAQ,CAAC,aAAa,CAAC,gEAAgE,CAAC,KAAK,IAAI,CAAC;wBAC9H,OAAO,CAAC,cAAc,IAAI,CAAC,mBAAmB,CAAC;oBACjD,CAAC,EACD,EAAE,OAAO,EAAE,KAAK,EAAE,CACnB,CAAC;oBACF,MAAM,IAAI,CAAC,gBAAgB,CAAC,kBAAkB,CAAC,CAAC;oBAChD,MAAM,IAAI,CAAC,gBAAgB,CAAC,aAAa,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;oBAC9E,MAAM,CAAC,IAAI,CAAC,gDAAgD,CAAC,CAAC;gBAChE,CAAC;gBAAC,MAAM,CAAC;oBACP,MAAM,CAAC,IAAI,CAAC,sEAAsE,CAAC,CAAC;gBACtF,CAAC;YACH,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,sFAAsF;YACtF,MAAM,CAAC,KAAK,CAAC,uEAAuE,CAAC,CAAC;QACxF,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,gBAAgB,CAAC,IAAU;QACvC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,EAAE,CAAC;QACnC,MAAM,aAAa,GAAG,MAAM,OAAO,CAAC,GAAG,CACrC,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE;YACzB,IAAI,CAAC;gBACH,MAAM,UAAU,GAAG,MAAM,KAAK;qBAC3B,QAAQ,CAAC,GAAG,EAAE;oBACb,OAAO,QAAQ,CAAC,aAAa,CAAC,6DAA6D,CAAC,KAAK,IAAI,CAAC;gBACxG,CAAC,CAAC;qBACD,KAAK,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC;gBAEtB,IAAI,UAAU,EAAE,CAAC;oBACf,MAAM,OAAO,CAAC,GAAG,CAAC;wBAChB,KAAK,CAAC,gBAAgB,CAAC,kBAAkB,CAAC;wBAC1C,KAAK;6BACF,gBAAgB,CAAC,aAAa,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;6BAClD,KAAK,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,gDAAgD,CAAC,CAAC;qBAC/E,CAAC,CAAC;oBACH,OAAO,KAAK,CAAC;gBACf,CAAC;YACH,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,CAAC,KAAK,CAAC,sBAAsB,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YACjE,CAAC;YACD,OAAO,IAAI,CAAC;QACd,CAAC,CAAC,CACH,CAAC;QAEF,MAAM,KAAK,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,IAAI,IAAI,CAAC;QAC5D,IAAI,KAAK,EAAE,CAAC;YACV,MAAM,CAAC,KAAK,CAAC,yBAAyB,CAAC,CAAC;YACxC,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC;QAC5D,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IAEO,KAAK,CAAC,iBAAiB,CAAC,OAAqB,EAAE,SAA2B;QAChF,MAAM,aAAa,GAAG,SAAS,CAAC,WAAW,CAAC,QAAQ,EAAE,CAAC;QACvD,OAAO,OAAO,CAAC,QAAQ,CAAC,KAAK,EAAE,IAAY,EAAE,EAAE;YAC7C,MAAM,cAAc,GAAG,IAAI,QAAQ,CAAC,UAAU,IAAI,EAAE,CAAC,EAAE,CAAC;YACxD,MAAM,SAAS,GAAG,IAAI,cAAc,EAAE,CAAC;YACvC,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;YACxD,OAAO,MAAM,CAAC,OAAO,CAAC;QACxB,CAAC,EAAE,aAAa,CAAC,CAAC;IACpB,CAAC;IAEO,KAAK,CAAC,cAAc,CAC1B,IAAU,EACV,QAAgB,EAChB,SAA2B;QAE3B,IAAI,OAAO,GAAG,EAAE,CAAC;QACjB,IAAI,aAAa,GAAG,SAAS,CAAC,WAAW,CAAC,IAAI,CAAC;QAE/C,IAAI,CAAC;YACH,IAAI,QAAQ,KAAK,WAAW,EAAE,CAAC;gBAC7B,mBAAmB;gBACnB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC;gBAChD,IAAI,KAAK,EAAE,CAAC;oBACV,OAAO,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;gBAC3D,CAAC;gBAED,wBAAwB;gBACxB,IAAI,CAAC,OAAO,EAAE,CAAC;oBACb,OAAO,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;gBAC1D,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,OAAO,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;YAC1D,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAS,GAAG,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC;YAC7E,aAAa,GAAG,eAAe,CAAC;QAClC,CAAC;QAED,OAAO,EAAE,OAAO,EAAE,aAAa,EAAE,CAAC;IACpC,CAAC;IAED;;;OAGG;IACH,aAAa,CAAC,MAAc;QAC1B,IAAI,CAAC,UAAU,GAAG,MAAM,CAAC;QACzB,MAAM,CAAC,IAAI,CAAC,iDAAiD,MAAM,EAAE,CAAC,CAAC;IACzE,CAAC;IAED,KAAK,CAAC,CAAC,KAAK,CAAC,GAAW;QACtB,MAAM,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,wBAAwB,GAAG,EAAE,CAAC,CAAC;QAErE,6BAA6B;QAC7B,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;QACxB,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC;QAChC,IAAI,CAAC,WAAW,GAAG,GAAG,CAAC;QACvB,IAAI,CAAC,oBAAoB,GAAG,CAAC,CAAC;QAE9B,8DAA8D;QAC9D,IAAI,CAAC;YACH,IAAI,CAAC,eAAe,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;YAC7C,MAAM,CAAC,IAAI,CAAC,+DAA+D,IAAI,CAAC,eAAe,EAAE,CAAC,CAAC;QACrG,CAAC;QAAC,MAAM,CAAC;YACP,IAAI,CAAC,eAAe,GAAG,EAAE,CAAC;QAC5B,CAAC;QAED,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;QAEzD,qEAAqE;QACrE,MAAM,IAAI,CAAC,YAAY,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC;QAE7C,qDAAqD;QACrD,MAAM,cAAc,GAAG,gBAAgB,CAAC,IAAI,CAAC,YAAY,CAAC,eAAe,EAAE,IAAI,SAAS,CAAC,CAAC;QAE1F,6EAA6E;QAC7E,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACtB,MAAM,CAAC,IAAI,CAAC,qDAAqD,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE,MAAM,IAAI,CAAC,UAAU,CAAC,CAAC;YACnH,cAAc,CAAC,aAAa,GAAG;gBAC7B,GAAG,cAAc,CAAC,aAAa;gBAC/B,aAAa,EAAE;oBACb,GAAG,cAAc,CAAC,aAAa,EAAE,aAAa;iBAC/C;aACF,CAAC;YACF,cAAc,CAAC,kBAAkB,GAAG;gBAClC,GAAG,cAAc,CAAC,kBAAkB;gBACpC,cAAc,EAAE;oBACd,KAAK,EAAE,MAAM,EAAE,EAAE;wBACf,0DAA0D;wBAC1D,MAAM,CAAC,KAAK,CAAC,+CAA+C,MAAM,EAAE,CAAC,CAAC;oBACxE,CAAC;iBACF;aACF,CAAC;YACF,qCAAqC;YACrC,MAAM,aAAa,GAAG,cAAc,CAAC,kBAAkB,IAAI,EAAE,CAAC;YAC9D,cAAc,CAAC,kBAAkB,GAAG;gBAClC,GAAG,aAAa;gBAChB,KAAK,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE;oBACjB,IAAI,IAAI,CAAC,YAAY,EAAE,OAAO,EAAE,CAAC;wBAC/B,MAAM,CAAC,KAAK,CAAC,4BAA4B,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,MAAM,4BAA4B,CAAC,CAAC;wBACvG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;oBAC7D,CAAC;gBACH,CAAC;aACF,CAAC;QACJ,CAAC;QAED,IAAI,CAAC,OAAO,GAAG,IAAI,iBAAiB,CAAC;YACnC,GAAG,cAAc;YACjB,cAAc,EAAE,KAAK,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,GAAG,EAAE,EAAE,EAAE;gBAC7D,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;oBACpB,GAAG,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;oBAC3B,OAAO;gBACT,CAAC;gBAED,IAAI,CAAC;oBACH,6BAA6B;oBAC7B,MAAM,OAAO,CAAC,GAAG,CAAC;wBAChB,IAAI,CAAC,gBAAgB,CAAC,kBAAkB,CAAC;wBACzC,IAAI,CAAC,gBAAgB,CAAC,aAAa,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,0CAA0C,CAAC,CAAC;qBAC3H,CAAC,CAAC;oBAEH,sEAAsE;oBACtE,MAAM,IAAI,CAAC,wBAAwB,CAAC,IAAI,CAAC,CAAC;oBAE1C,yCAAyC;oBACzC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;oBAE7B,0DAA0D;oBAC1D,IAAI,CAAC,IAAI,CAAC,qBAAqB,CAAC,SAAS,CAAC,EAAE,CAAC;wBAC3C,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;wBACxD,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC,QAAQ,CAAC;wBAEnD,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;4BACrB,gFAAgF;4BAChF,MAAM,CAAC,IAAI,CAAC,kEAAkE,iBAAiB,MAAM,cAAc,EAAE,CAAC,CAAC;4BAEvH,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;gCACtB,mDAAmD;gCACnD,IAAI,CAAC,mBAAmB,GAAG,IAAI,mBAAmB,CAChD,4EAA4E,cAAc,GAAG,EAC7F,IAAI,CAAC,WAAW,EAChB,SAAS,EACT,EAAE,WAAW,EAAE,IAAI,EAAE,UAAU,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,mBAAmB,iBAAiB,OAAO,cAAc,EAAE,CAAC,EAAE,CAC/G,CAAC;gCACF,GAAG,CAAC,KAAK,CAAC,oDAAoD,cAAc,mBAAmB,CAAC,CAAC;gCACjG,IAAI,CAAC,KAAK,EAAE,CAAC;gCACb,OAAO;4BACT,CAAC;iCAAM,CAAC;gCACN,0DAA0D;gCAC1D,GAAG,CAAC,KAAK,CAAC,6CAA6C,cAAc,mBAAmB,CAAC,CAAC;gCAC1F,IAAI,CAAC,KAAK,EAAE,CAAC;gCACb,OAAO;4BACT,CAAC;wBACH,CAAC;6BAAM,CAAC;4BACN,sDAAsD;4BACtD,IAAI,CAAC,oBAAoB,EAAE,CAAC;4BAC5B,GAAG,CAAC,OAAO,CACT,iDAAiD,OAAO,CAAC,GAAG,MAAM,SAAS,aAAa,IAAI,CAAC,oBAAoB,kBAAkB,CACpI,CAAC;4BACF,OAAO;wBACT,CAAC;oBACH,CAAC;oBAED,gEAAgE;oBAChE,IAAI,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;wBAC1C,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;wBAClE,IAAI,WAAW,EAAE,CAAC;4BAChB,GAAG,CAAC,KAAK,CAAC,uEAAuE,CAAC,CAAC;4BACnF,IAAI,CAAC,KAAK,EAAE,CAAC;4BACb,OAAO;wBACT,CAAC;wBACD,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;oBAC3B,CAAC;yBAAM,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;wBAC5B,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;oBAC3B,CAAC;oBAED,qCAAqC;oBACrC,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;wBAC7B,IAAI,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;4BAC5B,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;gCACjB,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;4BAChC,CAAC;4BAED,MAAM,IAAI,CAAC,YAAY,CAAC,mBAAmB,CAAC,YAAY,EAAE,GAAG,EAAE,IAAI,CAAC,CAAC;4BAErE,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;4BACjC,MAAM,EAAE,OAAO,EAAE,aAAa,EAAE,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;4BAE9F,MAAM,MAAM,GAAgB;gCAC1B,GAAG,EAAE,OAAO,CAAC,GAAG;gCAChB,IAAI,EAAE,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,QAAQ,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM;gCACjE,OAAO,EAAE,YAAY,CAAC,OAAO,CAAC;gCAC9B,KAAK;gCACL,aAAa;6BACd,CAAC;4BAEF,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;4BACpC,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;4BACnC,MAAM;wBACR,CAAC;oBACH,CAAC;gBACH,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,MAAM,YAAY,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;oBAC5E,GAAG,CAAC,KAAK,CAAC,oBAAoB,OAAO,CAAC,GAAG,KAAK,YAAY,EAAE,CAAC,CAAC;gBAChE,CAAC;YACH,CAAC;SACF,CAAC,CAAC;QAEH,IAAI,CAAC;YACH,MAAM,cAAc,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC;YAE1C,OAAO,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC;gBACxB,IAAI,IAAI,CAAC,YAAY,CAAC,gBAAgB,EAAE,EAAE,CAAC;oBACzC,KAAK,MAAM,MAAM,IAAI,MAAM,IAAI,CAAC,YAAY,CAAC,YAAY,EAAE,EAAE,CAAC;wBAC5D,MAAM,MAAM,CAAC;oBACf,CAAC;gBACH,CAAC;gBAED,IAAI,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,cAAc,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,EAAE,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;oBAAE,MAAM;YACpI,CAAC;YAED,MAAM,cAAc,CAAC;YACrB,MAAM,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC;YAEjC,4CAA4C;YAC5C,IAAI,IAAI,CAAC,oBAAoB,GAAG,CAAC,EAAE,CAAC;gBAClC,MAAM,CAAC,IAAI,CACT,4BAA4B,IAAI,CAAC,oBAAoB,sDAAsD,IAAI,CAAC,eAAe,GAAG,CACnI,CAAC;YACJ,CAAC;YAED,0DAA0D;YAC1D,IAAI,IAAI,CAAC,mBAAmB,EAAE,CAAC;gBAC7B,MAAM,IAAI,CAAC,mBAAmB,CAAC;YACjC,CAAC;YAED,KAAK,MAAM,MAAM,IAAI,MAAM,IAAI,CAAC,YAAY,CAAC,YAAY,EAAE,EAAE,CAAC;gBAC5D,MAAM,MAAM,CAAC;YACf,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,wCAAwC;YACxC,IAAI,KAAK,YAAY,mBAAmB,EAAE,CAAC;gBACzC,MAAM,KAAK,CAAC;YACd,CAAC;YACD,MAAM,CAAC,KAAK,CAAC,gBAAgB,EAAE,KAAK,CAAC,CAAC;YACtC,MAAM,KAAK,CAAC;QACd,CAAC;gBAAS,CAAC;YACT,MAAM,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE,CAAC;YAClC,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACtB,CAAC;IACH,CAAC;IAED,KAAK;QACH,KAAK,CAAC,KAAK,EAAE,CAAC;QACd,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,6BAA6B,EAAE,GAAG,CAAC,CAAC,CAAC;QAC3F,CAAC;IACH,CAAC;CACF"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"crawlee-crawler.test.js","sourceRoot":"","sources":["../../src/crawler/crawlee-crawler.test.ts"],"names":[],"mappings":"AAEA,MAAM,gBAAgB,GAAG;IACvB,UAAU,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,SAAS,CAAC;IAChD,eAAe,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC,EAAE,CAAC;IAC5C,mBAAmB,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,SAAS,CAAC;IACzD,SAAS,EAAE,EAAE,CAAC,EAAE,EAAE;IAClB,gBAAgB,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC,KAAK,CAAC;IAChD,YAAY,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,EAAE,CAAC;IAC3C,OAAO,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,SAAS,CAAC;CAC9C,CAAC;AAEF,EAAE,CAAC,IAAI,CAAC,oBAAoB,EAAE,GAAG,EAAE,CAAC,CAAC;IACnC,YAAY,EAAE;QACZ,OAAO,gBAAgB,CAAC;IAC1B,CAAC;CACF,CAAC,CAAC,CAAC;AAEJ,EAAE,CAAC,IAAI,CAAC,qBAAqB,EAAE,GAAG,EAAE,CAAC,CAAC;IACpC,gBAAgB,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC;QACxC,YAAY,EAAE,EAAE;QAChB,kBAAkB,EAAE,EAAE;QACtB,aAAa,EAAE,EAAE;QACjB,kBAAkB,EAAE,EAAE;KACvB,CAAC;CACH,CAAC,CAAC,CAAC;AAEJ,EAAE,CAAC,IAAI,CAAC,iBAAiB,EAAE,GAAG,EAAE,CAAC,CAAC;IAChC,SAAS,EAAE;QACT;YACE,IAAI,EAAE,SAAS;YACf,SAAS,EAAE;gBACT,cAAc,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,EAAE,OAAO,EAAE,mBAAmB,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC;aAC1F;YACD,MAAM,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,IAAI,CAAC;SACxC;KACF;CACF,CAAC,CAAC,CAAC;AAEJ,yBAAyB;AACzB,MAAM,cAAc,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;AAC5D,MAAM,mBAAmB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;AAEjE,EAAE,CAAC,IAAI,CAAC,SAAS,EAAE,GAAG,EAAE,CAAC,CAAC;IACxB,iBAAiB,EAAE,UAAU,OAAqC;QAChE,wCAAwC;QACvC,MAAyC,CAAC,gBAAgB,GAAG,OAAO,CAAC,cAAc,CAAC;QACrF,OAAO;YACL,GAAG,EAAE,cAAc;YACnB,QAAQ,EAAE,mBAAmB;SAC9B,CAAC;IACJ,CAAC;CACF,CAAC,CAAC,CAAC;AAEJ,uBAAuB;AACvB,OAAO,EAAE,cAAc,EAAgB,MAAM,sBAAsB,CAAC;AAEpE,QAAQ,CAAC,gBAAgB,EAAE,GAAG,EAAE;IAC9B,IAAI,OAAuB,CAAC;IAE5B,UAAU,CAAC,GAAG,EAAE;QACd,EAAE,CAAC,aAAa,EAAE,CAAC;QACnB,OAAO,GAAG,IAAI,cAAc,EAAE,CAAC;QAC/B,gBAAgB,CAAC,gBAAgB,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;QACzD,gBAAgB,CAAC,YAAY,CAAC,iBAAiB,CAAC,EAAE,CAAC,CAAC;IACtD,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,aAAa,EAAE,GAAG,EAAE;QAC3B,EAAE,CAAC,uCAAuC,EAAE,GAAG,EAAE;YAC/C,MAAM,CAAC,OAAO,CAAC,CAAC,WAAW,EAAE,CAAC;QAChC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uDAAuD,EAAE,GAAG,EAAE;YAC/D,MAAM,aAAa,GAAG,IAAI,cAAc,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;YAClD,MAAM,CAAC,aAAa,CAAC,CAAC,WAAW,EAAE,CAAC;QACtC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,iCAAiC,EAAE,GAAG,EAAE;YACzC,MAAM,UAAU,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3B,MAAM,eAAe,GAAG,IAAI,cAAc,CAAC,CAAC,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC;YAChE,MAAM,CAAC,eAAe,CAAC,CAAC,WAAW,EAAE,CAAC;QACxC,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,iBAAiB,EAAE,GAAG,EAAE;QAC/B,EAAE,CAAC,6BAA6B,EAAE,GAAG,EAAE;YACrC,MAAM,KAAK,GAAiB;gBAC1B,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,aAAa,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC;aAClF,CAAC;YAEF,OAAO,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;YAE/B,yBAAyB;YACzB,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC1B,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,0CAA0C,EAAE,GAAG,EAAE;YAClD,MAAM,KAAK,GAAiB;gBAC1B,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,aAAa,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC;gBACjF,OAAO,EAAE;oBACP;wBACE,MAAM,EAAE,qBAAqB;wBAC7B,YAAY,EAAE,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;qBAChD;iBACF;aACF,CAAC;YAEF,OAAO,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;YAC/B,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC1B,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,OAAO,EAAE,GAAG,EAAE;QACrB,EAAE,CAAC,0CAA0C,EAAE,KAAK,IAAI,EAAE;YACxD,qEAAqE;YACrE,cAAc,CAAC,qBAAqB,CAAC,SAAS,CAAC,CAAC;YAChD,gBAAgB,CAAC,YAAY,CAAC,qBAAqB,CAAC,EAAE,CAAC,CAAC;YAExD,MAAM,OAAO,GAAkB,EAAE,CAAC;YAClC,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,OAAO,CAAC,KAAK,CAAC,0BAA0B,CAAC,EAAE,CAAC;gBACrE,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;YAED,MAAM,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,oBAAoB,CAAC,0BAA0B,EAAE,SAAS,CAAC,CAAC;QAClG,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yCAAyC,EAAE,KAAK,IAAI,EAAE;YACvD,MAAM,WAAW,GAAkB;gBACjC,EAAE,GAAG,EAAE,2BAA2B,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE;gBACxF,EAAE,GAAG,EAAE,2BAA2B,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE;aACzF,CAAC;YAEF,yEAAyE;YACzE,uFAAuF;YACvF,gBAAgB,CAAC,YAAY,CAAC,qBAAqB,CAAC,WAAW,CAAC,CAAC;YAEjE,MAAM,OAAO,GAAkB,EAAE,CAAC;YAClC,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,OAAO,CAAC,KAAK,CAAC,qBAAqB,CAAC,EAAE,CAAC;gBAChE,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;YAED,MAAM,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;QACvC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,0CAA0C,EAAE,KAAK,IAAI,EAAE;YACxD,6DAA6D;YAC7D,IAAI,KAAK,EAAE,MAAM,CAAC,IAAI,OAAO,CAAC,KAAK,CAAC,qBAAqB,CAAC,EAAE,CAAC;gBAC3D,uBAAuB;YACzB,CAAC;YAED,MAAM,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC,gBAAgB,EAAE,CAAC;QACtD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,sDAAsD,EAAE,KAAK,IAAI,EAAE;YACpE,MAAM,WAAW,GAAkB,CAAC,EAAE,GAAG,EAAE,2BAA2B,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC;YAE9H,gBAAgB,CAAC,gBAAgB,CAAC,mBAAmB,CAAC,IAAI,CAAC,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;YACnF,gBAAgB,CAAC,YAAY,CAAC,qBAAqB,CAAC,WAAW,CAAC,CAAC,qBAAqB,CAAC,EAAE,CAAC,CAAC;YAE3F,MAAM,OAAO,GAAkB,EAAE,CAAC;YAClC,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,OAAO,CAAC,KAAK,CAAC,qBAAqB,CAAC,EAAE,CAAC;gBAChE,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;YAED,MAAM,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAClC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,sCAAsC,EAAE,KAAK,IAAI,EAAE;YACpD,6DAA6D;YAC7D,IAAI,KAAK,EAAE,MAAM,CAAC,IAAI,OAAO,CAAC,KAAK,CAAC,gCAAgC,CAAC,EAAE,CAAC;gBACtE,uBAAuB;YACzB,CAAC;YAED,qCAAqC;YACrC,MAAM,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,oBAAoB,CAAC,gCAAgC,EAAE,SAAS,CAAC,CAAC;QACxG,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,OAAO,EAAE,GAAG,EAAE;QACrB,EAAE,CAAC,yBAAyB,EAAE,KAAK,IAAI,EAAE;YACvC,qFAAqF;YACrF,IAAI,UAAsB,CAAC;YAC3B,MAAM,UAAU,GAAG,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;gBAC/C,UAAU,GAAG,OAAO,CAAC;YACvB,CAAC,CAAC,CAAC;YACH,cAAc,CAAC,mBAAmB,CAAC,UAAU,CAAC,CAAC;YAE/C,qCAAqC;YACrC,MAAM,gBAAgB,GAAG,IAAI,cAAc,EAAE,CAAC;YAE9C,2DAA2D;YAC3D,MAAM,SAAS,GAAG,gBAAgB,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC;YAEhE,wEAAwE;YACxE,MAAM,kBAAkB,GAAG,SAAS,CAAC,IAAI,EAAE,CAAC;YAE5C,0DAA0D;YAC1D,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;YAExD,+CAA+C;YAC/C,gBAAgB,CAAC,KAAK,EAAE,CAAC;YAEzB,qDAAqD;YACrD,UAAW,EAAE,CAAC;YAEd,qCAAqC;YACrC,MAAM,kBAAkB,CAAC;YACzB,MAAM,OAAO,GAAkB,EAAE,CAAC;YAClC,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,SAAS,EAAE,CAAC;gBACrC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;YAED,MAAM,CAAC,mBAAmB,CAAC,CAAC,gBAAgB,EAAE,CAAC;QACjD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,oBAAoB,EAAE,GAAG,EAAE;QAClC,EAAE,CAAC,yDAAyD,EAAE,KAAK,IAAI,EAAE;YACvE,6DAA6D;YAC7D,IAAI,KAAK,EAAE,MAAM,CAAC,IAAI,OAAO,CAAC,KAAK,CAAC,oCAAoC,CAAC,EAAE,CAAC;gBAC1E,uBAAuB;YACzB,CAAC;YAED,MAAM,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,oBAAoB,CAAC,oCAAoC,EAAE,SAAS,CAAC,CAAC;QAC5G,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,mDAAmD,EAAE,KAAK,IAAI,EAAE;YACjE,OAAO,CAAC,aAAa,CAAC,WAAW,CAAC,CAAC;YAEnC,6DAA6D;YAC7D,IAAI,KAAK,EAAE,MAAM,CAAC,IAAI,OAAO,CAAC,KAAK,CAAC,8BAA8B,CAAC,EAAE,CAAC;gBACpE,uBAAuB;YACzB,CAAC;YAED,MAAM,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,oBAAoB,CAAC,8BAA8B,EAAE,WAAW,CAAC,CAAC;QACxG,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,gBAAgB,EAAE,GAAG,EAAE;QAC9B,EAAE,CAAC,sDAAsD,EAAE,KAAK,IAAI,EAAE;YACpE,MAAM,KAAK,GAAiB;gBAC1B,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,EAAE,aAAa,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC;aACjF,CAAC;YAEF,OAAO,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;YAE/B,6DAA6D;YAC7D,IAAI,KAAK,EAAE,MAAM,CAAC,IAAI,OAAO,CAAC,KAAK,CAAC,qBAAqB,CAAC,EAAE,CAAC;gBAC3D,uBAAuB;YACzB,CAAC;YAED,oEAAoE;YACpE,MAAM,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,gBAAgB,EAAE,CAAC;QACzD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,uBAAuB,EAAE,GAAG,EAAE;QACrC,oEAAoE;QACpE,EAAE,CAAC,4CAA4C,EAAE,KAAK,IAAI,EAAE;YAC1D,qDAAqD;YACrD,6DAA6D;YAC7D,IAAI,KAAK,EAAE,MAAM,CAAC,IAAI,OAAO,CAAC,KAAK,CAAC,qBAAqB,CAAC,EAAE,CAAC;gBAC3D,uBAAuB;YACzB,CAAC;YAED,wCAAwC;YACxC,MAAM,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,gBAAgB,EAAE,CAAC;QACzD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,gBAAgB,EAAE,GAAG,EAAE;QAC9B,EAAE,CAAC,yBAAyB,EAAE,KAAK,IAAI,EAAE;YACvC,cAAc,CAAC,qBAAqB,CAAC,IAAI,KAAK,CAAC,cAAc,CAAC,CAAC,CAAC;YAEhE,MAAM,MAAM,CAAC,KAAK,IAAI,EAAE;gBACtB,6DAA6D;gBAC7D,IAAI,KAAK,EAAE,MAAM,CAAC,IAAI,OAAO,CAAC,KAAK,CAAC,qBAAqB,CAAC,EAAE,CAAC;oBAC3D,uBAAuB;gBACzB,CAAC;YACH,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;YAEnC,MAAM,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC,gBAAgB,EAAE,CAAC;QACtD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uCAAuC,EAAE,KAAK,IAAI,EAAE;YACrD,4CAA4C;YAC5C,6DAA6D;YAC7D,IAAI,KAAK,EAAE,MAAM,CAAC,IAAI,OAAO,CAAC,KAAK,CAAC,qBAAqB,CAAC,EAAE,CAAC;gBAC3D,uBAAuB;YACzB,CAAC;YAED,MAAM,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,gBAAgB,EAAE,CAAC;QACzD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,wBAAwB,EAAE,GAAG,EAAE;IACtC,EAAE,CAAC,mDAAmD,EAAE,GAAG,EAAE;QAC3D,MAAM,KAAK,GAAiB;YAC1B,OAAO,EAAE;gBACP;oBACE,IAAI,EAAE,SAAS;oBACf,KAAK,EAAE,KAAK;oBACZ,MAAM,EAAE,aAAa;oBACrB,IAAI,EAAE,GAAG;oBACT,OAAO,EAAE,UAAU;oBACnB,QAAQ,EAAE,IAAI;oBACd,MAAM,EAAE,IAAI;oBACZ,QAAQ,EAAE,QAAQ;iBACnB;aACF;SACF,CAAC;QAEF,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACtC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACnD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,wCAAwC,EAAE,GAAG,EAAE;QAChD,MAAM,KAAK,GAAiB;YAC1B,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,UAAU,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC;SAC1E,CAAC;QAEF,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,aAAa,EAAE,CAAC;IACnD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,uCAAuC,EAAE,GAAG,EAAE;QAC/C,MAAM,KAAK,GAAiB;YAC1B,OAAO,EAAE,EAAE;YACX,OAAO,EAAE;gBACP;oBACE,MAAM,EAAE,qBAAqB;oBAC7B,YAAY,EAAE;wBACZ,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE;wBACjC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE;qBAClC;iBACF;aACF;SACF,CAAC;QAEF,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACtC,MAAM,CAAC,KAAK,CAAC,OAAQ,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;IACzD,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
1
|
+
{"version":3,"file":"crawlee-crawler.test.js","sourceRoot":"","sources":["../../src/crawler/crawlee-crawler.test.ts"],"names":[],"mappings":"AAEA,MAAM,gBAAgB,GAAG;IACvB,UAAU,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,SAAS,CAAC;IAChD,eAAe,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,CAAC,CAAC;IAC7C,eAAe,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC,EAAE,CAAC;IAC5C,mBAAmB,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,SAAS,CAAC;IACzD,SAAS,EAAE,EAAE,CAAC,EAAE,EAAE;IAClB,gBAAgB,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC,KAAK,CAAC;IAChD,YAAY,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,EAAE,CAAC;IAC3C,OAAO,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,SAAS,CAAC;CAC9C,CAAC;AAEF,EAAE,CAAC,IAAI,CAAC,oBAAoB,EAAE,GAAG,EAAE,CAAC,CAAC;IACnC,YAAY,EAAE;QACZ,OAAO,gBAAgB,CAAC;IAC1B,CAAC;CACF,CAAC,CAAC,CAAC;AAEJ,EAAE,CAAC,IAAI,CAAC,qBAAqB,EAAE,GAAG,EAAE,CAAC,CAAC;IACpC,gBAAgB,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC;QACxC,YAAY,EAAE,EAAE;QAChB,kBAAkB,EAAE,EAAE;QACtB,aAAa,EAAE,EAAE;QACjB,kBAAkB,EAAE,EAAE;KACvB,CAAC;CACH,CAAC,CAAC,CAAC;AAEJ,EAAE,CAAC,IAAI,CAAC,iBAAiB,EAAE,GAAG,EAAE,CAAC,CAAC;IAChC,SAAS,EAAE;QACT;YACE,IAAI,EAAE,SAAS;YACf,SAAS,EAAE;gBACT,cAAc,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,EAAE,OAAO,EAAE,mBAAmB,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC;aAC1F;YACD,MAAM,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,IAAI,CAAC;SACxC;KACF;CACF,CAAC,CAAC,CAAC;AAEJ,yBAAyB;AACzB,MAAM,cAAc,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;AAC5D,MAAM,mBAAmB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;AAEjE,EAAE,CAAC,IAAI,CAAC,SAAS,EAAE,GAAG,EAAE,CAAC,CAAC;IACxB,iBAAiB,EAAE,UAAU,OAAqC;QAChE,wCAAwC;QACvC,MAAyC,CAAC,gBAAgB,GAAG,OAAO,CAAC,cAAc,CAAC;QACrF,OAAO;YACL,GAAG,EAAE,cAAc;YACnB,QAAQ,EAAE,mBAAmB;SAC9B,CAAC;IACJ,CAAC;CACF,CAAC,CAAC,CAAC;AAEJ,uBAAuB;AACvB,OAAO,EAAE,cAAc,EAAgB,MAAM,sBAAsB,CAAC;AAEpE,QAAQ,CAAC,gBAAgB,EAAE,GAAG,EAAE;IAC9B,IAAI,OAAuB,CAAC;IAE5B,UAAU,CAAC,GAAG,EAAE;QACd,EAAE,CAAC,aAAa,EAAE,CAAC;QACnB,OAAO,GAAG,IAAI,cAAc,EAAE,CAAC;QAC/B,gBAAgB,CAAC,gBAAgB,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;QACzD,gBAAgB,CAAC,YAAY,CAAC,iBAAiB,CAAC,EAAE,CAAC,CAAC;IACtD,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,aAAa,EAAE,GAAG,EAAE;QAC3B,EAAE,CAAC,uCAAuC,EAAE,GAAG,EAAE;YAC/C,MAAM,CAAC,OAAO,CAAC,CAAC,WAAW,EAAE,CAAC;QAChC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uDAAuD,EAAE,GAAG,EAAE;YAC/D,MAAM,aAAa,GAAG,IAAI,cAAc,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;YAClD,MAAM,CAAC,aAAa,CAAC,CAAC,WAAW,EAAE,CAAC;QACtC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,iCAAiC,EAAE,GAAG,EAAE;YACzC,MAAM,UAAU,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3B,MAAM,eAAe,GAAG,IAAI,cAAc,CAAC,CAAC,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC;YAChE,MAAM,CAAC,eAAe,CAAC,CAAC,WAAW,EAAE,CAAC;QACxC,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,iBAAiB,EAAE,GAAG,EAAE;QAC/B,EAAE,CAAC,6BAA6B,EAAE,GAAG,EAAE;YACrC,MAAM,KAAK,GAAiB;gBAC1B,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,aAAa,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC;aAClF,CAAC;YAEF,OAAO,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;YAE/B,yBAAyB;YACzB,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC1B,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,0CAA0C,EAAE,GAAG,EAAE;YAClD,MAAM,KAAK,GAAiB;gBAC1B,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,aAAa,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC;gBACjF,OAAO,EAAE;oBACP;wBACE,MAAM,EAAE,qBAAqB;wBAC7B,YAAY,EAAE,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;qBAChD;iBACF;aACF,CAAC;YAEF,OAAO,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;YAC/B,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC1B,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,OAAO,EAAE,GAAG,EAAE;QACrB,EAAE,CAAC,0CAA0C,EAAE,KAAK,IAAI,EAAE;YACxD,qEAAqE;YACrE,cAAc,CAAC,qBAAqB,CAAC,SAAS,CAAC,CAAC;YAChD,gBAAgB,CAAC,YAAY,CAAC,qBAAqB,CAAC,EAAE,CAAC,CAAC;YAExD,MAAM,OAAO,GAAkB,EAAE,CAAC;YAClC,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,OAAO,CAAC,KAAK,CAAC,0BAA0B,CAAC,EAAE,CAAC;gBACrE,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;YAED,MAAM,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,oBAAoB,CAAC,0BAA0B,EAAE,SAAS,CAAC,CAAC;QAClG,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yCAAyC,EAAE,KAAK,IAAI,EAAE;YACvD,MAAM,WAAW,GAAkB;gBACjC,EAAE,GAAG,EAAE,2BAA2B,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE;gBACxF,EAAE,GAAG,EAAE,2BAA2B,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE;aACzF,CAAC;YAEF,yEAAyE;YACzE,uFAAuF;YACvF,gBAAgB,CAAC,YAAY,CAAC,qBAAqB,CAAC,WAAW,CAAC,CAAC;YAEjE,MAAM,OAAO,GAAkB,EAAE,CAAC;YAClC,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,OAAO,CAAC,KAAK,CAAC,qBAAqB,CAAC,EAAE,CAAC;gBAChE,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;YAED,MAAM,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;QACvC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,0CAA0C,EAAE,KAAK,IAAI,EAAE;YACxD,6DAA6D;YAC7D,IAAI,KAAK,EAAE,MAAM,CAAC,IAAI,OAAO,CAAC,KAAK,CAAC,qBAAqB,CAAC,EAAE,CAAC;gBAC3D,uBAAuB;YACzB,CAAC;YAED,MAAM,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC,gBAAgB,EAAE,CAAC;QACtD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,sDAAsD,EAAE,KAAK,IAAI,EAAE;YACpE,MAAM,WAAW,GAAkB,CAAC,EAAE,GAAG,EAAE,2BAA2B,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC;YAE9H,gBAAgB,CAAC,gBAAgB,CAAC,mBAAmB,CAAC,IAAI,CAAC,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;YACnF,gBAAgB,CAAC,YAAY,CAAC,qBAAqB,CAAC,WAAW,CAAC,CAAC,qBAAqB,CAAC,EAAE,CAAC,CAAC;YAE3F,MAAM,OAAO,GAAkB,EAAE,CAAC;YAClC,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,OAAO,CAAC,KAAK,CAAC,qBAAqB,CAAC,EAAE,CAAC;gBAChE,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;YAED,MAAM,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAClC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,sCAAsC,EAAE,KAAK,IAAI,EAAE;YACpD,6DAA6D;YAC7D,IAAI,KAAK,EAAE,MAAM,CAAC,IAAI,OAAO,CAAC,KAAK,CAAC,gCAAgC,CAAC,EAAE,CAAC;gBACtE,uBAAuB;YACzB,CAAC;YAED,qCAAqC;YACrC,MAAM,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,oBAAoB,CAAC,gCAAgC,EAAE,SAAS,CAAC,CAAC;QACxG,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,OAAO,EAAE,GAAG,EAAE;QACrB,EAAE,CAAC,yBAAyB,EAAE,KAAK,IAAI,EAAE;YACvC,qFAAqF;YACrF,IAAI,UAAsB,CAAC;YAC3B,MAAM,UAAU,GAAG,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;gBAC/C,UAAU,GAAG,OAAO,CAAC;YACvB,CAAC,CAAC,CAAC;YACH,cAAc,CAAC,mBAAmB,CAAC,UAAU,CAAC,CAAC;YAE/C,qCAAqC;YACrC,MAAM,gBAAgB,GAAG,IAAI,cAAc,EAAE,CAAC;YAE9C,2DAA2D;YAC3D,MAAM,SAAS,GAAG,gBAAgB,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC;YAEhE,wEAAwE;YACxE,MAAM,kBAAkB,GAAG,SAAS,CAAC,IAAI,EAAE,CAAC;YAE5C,0DAA0D;YAC1D,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;YAExD,+CAA+C;YAC/C,gBAAgB,CAAC,KAAK,EAAE,CAAC;YAEzB,qDAAqD;YACrD,UAAW,EAAE,CAAC;YAEd,qCAAqC;YACrC,MAAM,kBAAkB,CAAC;YACzB,MAAM,OAAO,GAAkB,EAAE,CAAC;YAClC,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,SAAS,EAAE,CAAC;gBACrC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;YAED,MAAM,CAAC,mBAAmB,CAAC,CAAC,gBAAgB,EAAE,CAAC;QACjD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,oBAAoB,EAAE,GAAG,EAAE;QAClC,EAAE,CAAC,yDAAyD,EAAE,KAAK,IAAI,EAAE;YACvE,6DAA6D;YAC7D,IAAI,KAAK,EAAE,MAAM,CAAC,IAAI,OAAO,CAAC,KAAK,CAAC,oCAAoC,CAAC,EAAE,CAAC;gBAC1E,uBAAuB;YACzB,CAAC;YAED,MAAM,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,oBAAoB,CAAC,oCAAoC,EAAE,SAAS,CAAC,CAAC;QAC5G,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,mDAAmD,EAAE,KAAK,IAAI,EAAE;YACjE,OAAO,CAAC,aAAa,CAAC,WAAW,CAAC,CAAC;YAEnC,6DAA6D;YAC7D,IAAI,KAAK,EAAE,MAAM,CAAC,IAAI,OAAO,CAAC,KAAK,CAAC,8BAA8B,CAAC,EAAE,CAAC;gBACpE,uBAAuB;YACzB,CAAC;YAED,MAAM,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,oBAAoB,CAAC,8BAA8B,EAAE,WAAW,CAAC,CAAC;QACxG,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,gBAAgB,EAAE,GAAG,EAAE;QAC9B,EAAE,CAAC,sDAAsD,EAAE,KAAK,IAAI,EAAE;YACpE,MAAM,KAAK,GAAiB;gBAC1B,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,EAAE,aAAa,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC;aACjF,CAAC;YAEF,OAAO,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;YAE/B,6DAA6D;YAC7D,IAAI,KAAK,EAAE,MAAM,CAAC,IAAI,OAAO,CAAC,KAAK,CAAC,qBAAqB,CAAC,EAAE,CAAC;gBAC3D,uBAAuB;YACzB,CAAC;YAED,oEAAoE;YACpE,MAAM,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,gBAAgB,EAAE,CAAC;QACzD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,uBAAuB,EAAE,GAAG,EAAE;QACrC,oEAAoE;QACpE,EAAE,CAAC,4CAA4C,EAAE,KAAK,IAAI,EAAE;YAC1D,qDAAqD;YACrD,6DAA6D;YAC7D,IAAI,KAAK,EAAE,MAAM,CAAC,IAAI,OAAO,CAAC,KAAK,CAAC,qBAAqB,CAAC,EAAE,CAAC;gBAC3D,uBAAuB;YACzB,CAAC;YAED,wCAAwC;YACxC,MAAM,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,gBAAgB,EAAE,CAAC;QACzD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,gBAAgB,EAAE,GAAG,EAAE;QAC9B,EAAE,CAAC,yBAAyB,EAAE,KAAK,IAAI,EAAE;YACvC,cAAc,CAAC,qBAAqB,CAAC,IAAI,KAAK,CAAC,cAAc,CAAC,CAAC,CAAC;YAEhE,MAAM,MAAM,CAAC,KAAK,IAAI,EAAE;gBACtB,6DAA6D;gBAC7D,IAAI,KAAK,EAAE,MAAM,CAAC,IAAI,OAAO,CAAC,KAAK,CAAC,qBAAqB,CAAC,EAAE,CAAC;oBAC3D,uBAAuB;gBACzB,CAAC;YACH,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;YAEnC,MAAM,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC,gBAAgB,EAAE,CAAC;QACtD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uCAAuC,EAAE,KAAK,IAAI,EAAE;YACrD,4CAA4C;YAC5C,6DAA6D;YAC7D,IAAI,KAAK,EAAE,MAAM,CAAC,IAAI,OAAO,CAAC,KAAK,CAAC,qBAAqB,CAAC,EAAE,CAAC;gBAC3D,uBAAuB;YACzB,CAAC;YAED,MAAM,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,gBAAgB,EAAE,CAAC;QACzD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,wBAAwB,EAAE,GAAG,EAAE;IACtC,EAAE,CAAC,mDAAmD,EAAE,GAAG,EAAE;QAC3D,MAAM,KAAK,GAAiB;YAC1B,OAAO,EAAE;gBACP;oBACE,IAAI,EAAE,SAAS;oBACf,KAAK,EAAE,KAAK;oBACZ,MAAM,EAAE,aAAa;oBACrB,IAAI,EAAE,GAAG;oBACT,OAAO,EAAE,UAAU;oBACnB,QAAQ,EAAE,IAAI;oBACd,MAAM,EAAE,IAAI;oBACZ,QAAQ,EAAE,QAAQ;iBACnB;aACF;SACF,CAAC;QAEF,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACtC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACnD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,wCAAwC,EAAE,GAAG,EAAE;QAChD,MAAM,KAAK,GAAiB;YAC1B,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,UAAU,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC;SAC1E,CAAC;QAEF,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,aAAa,EAAE,CAAC;IACnD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,uCAAuC,EAAE,GAAG,EAAE;QAC/C,MAAM,KAAK,GAAiB;YAC1B,OAAO,EAAE,EAAE;YACX,OAAO,EAAE;gBACP;oBACE,MAAM,EAAE,qBAAqB;oBAC7B,YAAY,EAAE;wBACZ,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE;wBACjC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE;qBAClC;iBACF;aACF;SACF,CAAC;QAEF,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACtC,MAAM,CAAC,KAAK,CAAC,OAAQ,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;IACzD,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
@@ -8,7 +8,6 @@ export class DocsCrawler {
|
|
|
8
8
|
githubToken;
|
|
9
9
|
onProgress;
|
|
10
10
|
GITHUB_HOST = 'github.com';
|
|
11
|
-
MIN_PAGES = 2; // Require at least 2 pages for component libraries
|
|
12
11
|
isAborting = false;
|
|
13
12
|
storageState;
|
|
14
13
|
pathPrefix;
|
|
@@ -78,12 +77,13 @@ export class DocsCrawler {
|
|
|
78
77
|
pageCount++;
|
|
79
78
|
yield page;
|
|
80
79
|
}
|
|
81
|
-
if (pageCount
|
|
82
|
-
logger.
|
|
83
|
-
return 'crawlee';
|
|
80
|
+
if (pageCount === 0) {
|
|
81
|
+
logger.warn(`[DocsCrawler] Crawlee crawler found no pages for ${url}`);
|
|
84
82
|
}
|
|
85
|
-
|
|
86
|
-
|
|
83
|
+
else {
|
|
84
|
+
logger.debug(`[DocsCrawler] Crawlee crawler completed (${pageCount} pages)`);
|
|
85
|
+
}
|
|
86
|
+
return 'crawlee';
|
|
87
87
|
}
|
|
88
88
|
catch (e) {
|
|
89
89
|
logger.debug('[DocsCrawler] Crawlee crawler failed:', e);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"docs-crawler.js","sourceRoot":"","sources":["../../src/crawler/docs-crawler.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,KAAK,CAAC;AAE1B,OAAO,EAAE,cAAc,EAAgB,MAAM,sBAAsB,CAAC;AACpE,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAE3C,MAAM,OAAO,WAAW;
|
|
1
|
+
{"version":3,"file":"docs-crawler.js","sourceRoot":"","sources":["../../src/crawler/docs-crawler.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,KAAK,CAAC;AAE1B,OAAO,EAAE,cAAc,EAAgB,MAAM,sBAAsB,CAAC;AACpE,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAE3C,MAAM,OAAO,WAAW;IAOH;IACA;IACA;IACA;IATF,WAAW,GAAG,YAAY,CAAC;IACpC,UAAU,GAAG,KAAK,CAAC;IACnB,YAAY,CAAgB;IAC5B,UAAU,CAAU;IAE5B,YACmB,WAAmB,CAAC,EACpB,sBAA8B,IAAI,EAClC,WAAoB,EACpB,UAA4D;QAH5D,aAAQ,GAAR,QAAQ,CAAY;QACpB,wBAAmB,GAAnB,mBAAmB,CAAe;QAClC,gBAAW,GAAX,WAAW,CAAS;QACpB,eAAU,GAAV,UAAU,CAAkD;IAC5E,CAAC;IAEJ;;;;OAIG;IACH,aAAa,CAAC,MAAc;QAC1B,IAAI,CAAC,UAAU,GAAG,MAAM,CAAC;QACzB,MAAM,CAAC,IAAI,CAAC,8CAA8C,MAAM,EAAE,CAAC,CAAC;IACtE,CAAC;IAED;;OAEG;IACH,eAAe,CAAC,KAAmB;QACjC,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC;QAC1B,MAAM,CAAC,IAAI,CAAC,wCAAwC,KAAK,CAAC,OAAO,EAAE,MAAM,IAAI,CAAC,UAAU,CAAC,CAAC;IAC5F,CAAC;IAED,KAAK,CAAC,CAAC,KAAK,CAAC,GAAW;QACtB,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAC9B,MAAM,CAAC,KAAK,CAAC,mCAAmC,QAAQ,EAAE,CAAC,CAAC;QAE5D,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACpB,MAAM,CAAC,KAAK,CAAC,6BAA6B,CAAC,CAAC;YAC5C,OAAO,SAAS,CAAC;QACnB,CAAC;QAED,6BAA6B;QAC7B,IAAI,QAAQ,CAAC,IAAI,KAAK,IAAI,CAAC,WAAW,EAAE,CAAC;YACvC,MAAM,CAAC,KAAK,CAAC,0CAA0C,CAAC,CAAC;YACzD,MAAM,aAAa,GAAG,IAAI,aAAa,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,mBAAmB,EAAE,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;YAEpH,IAAI,CAAC;gBACH,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,aAAa,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;oBAClD,IAAI,IAAI,CAAC,UAAU;wBAAE,MAAM;oBAC3B,MAAM,IAAI,CAAC;gBACb,CAAC;gBACD,OAAO,QAAQ,CAAC;YAClB,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,MAAM,CAAC,KAAK,CAAC,sCAAsC,EAAE,CAAC,CAAC,CAAC;gBACxD,uDAAuD;gBACvD,MAAM,CAAC,CAAC;YACV,CAAC;QACH,CAAC;QAED,kCAAkC;QAClC,MAAM,CAAC,KAAK,CAAC,qCAAqC,CAAC,CAAC;QACpD,MAAM,cAAc,GAAG,IAAI,cAAc,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,mBAAmB,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;QAEpG,mCAAmC;QACnC,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACtB,cAAc,CAAC,eAAe,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QACpD,CAAC;QAED,6CAA6C;QAC7C,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACpB,cAAc,CAAC,aAAa,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAChD,CAAC;QAED,IAAI,SAAS,GAAG,CAAC,CAAC;QAElB,IAAI,CAAC;YACH,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,cAAc,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;gBACnD,IAAI,IAAI,CAAC,UAAU;oBAAE,MAAM;gBAC3B,SAAS,EAAE,CAAC;gBACZ,MAAM,IAAI,CAAC;YACb,CAAC;YAED,IAAI,SAAS,KAAK,CAAC,EAAE,CAAC;gBACpB,MAAM,CAAC,IAAI,CAAC,oDAAoD,GAAG,EAAE,CAAC,CAAC;YACzE,CAAC;iBAAM,CAAC;gBACN,MAAM,CAAC,KAAK,CAAC,4CAA4C,SAAS,SAAS,CAAC,CAAC;YAC/E,CAAC;YACD,OAAO,SAAS,CAAC;QACnB,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,MAAM,CAAC,KAAK,CAAC,uCAAuC,EAAE,CAAC,CAAC,CAAC;YACzD,MAAM,CAAC,CAAC;QACV,CAAC;IACH,CAAC;IAED,KAAK;QACH,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;IACzB,CAAC;CACF"}
|
|
@@ -9,11 +9,13 @@ vi.mock('./github.js', () => ({
|
|
|
9
9
|
}));
|
|
10
10
|
const mockCrawleeCrawl = vi.fn();
|
|
11
11
|
const mockSetStorageState = vi.fn();
|
|
12
|
+
const mockSetPathPrefix = vi.fn();
|
|
12
13
|
vi.mock('./crawlee-crawler.js', () => ({
|
|
13
14
|
CrawleeCrawler: function () {
|
|
14
15
|
return {
|
|
15
16
|
crawl: mockCrawleeCrawl,
|
|
16
17
|
setStorageState: mockSetStorageState,
|
|
18
|
+
setPathPrefix: mockSetPathPrefix,
|
|
17
19
|
};
|
|
18
20
|
},
|
|
19
21
|
}));
|
|
@@ -95,32 +97,32 @@ describe('DocsCrawler', () => {
|
|
|
95
97
|
}
|
|
96
98
|
expect(results).toHaveLength(2);
|
|
97
99
|
});
|
|
98
|
-
it('should return crawlee type
|
|
100
|
+
it('should return crawlee type regardless of page count', async () => {
|
|
99
101
|
mockCrawleeCrawl.mockImplementation(async function* () {
|
|
100
102
|
yield { url: 'https://example.com/page1', path: '/page1', content: 'Page 1', title: 'Page 1' };
|
|
101
|
-
yield { url: 'https://example.com/page2', path: '/page2', content: 'Page 2', title: 'Page 2' };
|
|
102
103
|
});
|
|
104
|
+
const results = [];
|
|
103
105
|
const generator = crawler.crawl('https://example.com');
|
|
104
|
-
// Manually iterate to capture the return value
|
|
105
106
|
let result = await generator.next();
|
|
106
107
|
while (!result.done) {
|
|
108
|
+
results.push(result.value);
|
|
107
109
|
result = await generator.next();
|
|
108
110
|
}
|
|
109
111
|
const crawlerType = result.value;
|
|
112
|
+
expect(results).toHaveLength(1);
|
|
110
113
|
expect(crawlerType).toBe('crawlee');
|
|
111
114
|
});
|
|
112
|
-
it('should
|
|
113
|
-
mockCrawleeCrawl.mockImplementation(
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
});
|
|
115
|
+
it('should return crawlee type even with 0 pages found', async () => {
|
|
116
|
+
mockCrawleeCrawl.mockImplementation(() => ({
|
|
117
|
+
[Symbol.asyncIterator]: () => ({ next: () => Promise.resolve({ done: true, value: undefined }) }),
|
|
118
|
+
}));
|
|
117
119
|
const generator = crawler.crawl('https://example.com');
|
|
118
|
-
await
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
120
|
+
let result = await generator.next();
|
|
121
|
+
while (!result.done) {
|
|
122
|
+
result = await generator.next();
|
|
123
|
+
}
|
|
124
|
+
const crawlerType = result.value;
|
|
125
|
+
expect(crawlerType).toBe('crawlee');
|
|
124
126
|
});
|
|
125
127
|
});
|
|
126
128
|
describe('abort', () => {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"docs-crawler.test.js","sourceRoot":"","sources":["../../src/crawler/docs-crawler.test.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEhD,MAAM,eAAe,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;AAChC,EAAE,CAAC,IAAI,CAAC,aAAa,EAAE,GAAG,EAAE,CAAC,CAAC;IAC5B,aAAa,EAAE;QACb,OAAO;YACL,KAAK,EAAE,eAAe;SACvB,CAAC;IACJ,CAAC;CACF,CAAC,CAAC,CAAC;AAEJ,MAAM,gBAAgB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;AACjC,MAAM,mBAAmB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;AACpC,EAAE,CAAC,IAAI,CAAC,sBAAsB,EAAE,GAAG,EAAE,CAAC,CAAC;IACrC,cAAc,EAAE;QACd,OAAO;YACL,KAAK,EAAE,gBAAgB;YACvB,eAAe,EAAE,mBAAmB;
|
|
1
|
+
{"version":3,"file":"docs-crawler.test.js","sourceRoot":"","sources":["../../src/crawler/docs-crawler.test.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEhD,MAAM,eAAe,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;AAChC,EAAE,CAAC,IAAI,CAAC,aAAa,EAAE,GAAG,EAAE,CAAC,CAAC;IAC5B,aAAa,EAAE;QACb,OAAO;YACL,KAAK,EAAE,eAAe;SACvB,CAAC;IACJ,CAAC;CACF,CAAC,CAAC,CAAC;AAEJ,MAAM,gBAAgB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;AACjC,MAAM,mBAAmB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;AACpC,MAAM,iBAAiB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;AAClC,EAAE,CAAC,IAAI,CAAC,sBAAsB,EAAE,GAAG,EAAE,CAAC,CAAC;IACrC,cAAc,EAAE;QACd,OAAO;YACL,KAAK,EAAE,gBAAgB;YACvB,eAAe,EAAE,mBAAmB;YACpC,aAAa,EAAE,iBAAiB;SACjC,CAAC;IACJ,CAAC;CACF,CAAC,CAAC,CAAC;AAEJ,QAAQ,CAAC,aAAa,EAAE,GAAG,EAAE;IAC3B,IAAI,OAAoB,CAAC;IAEzB,UAAU,CAAC,GAAG,EAAE;QACd,EAAE,CAAC,aAAa,EAAE,CAAC;QACnB,OAAO,GAAG,IAAI,WAAW,EAAE,CAAC;IAC9B,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,aAAa,EAAE,GAAG,EAAE;QAC3B,EAAE,CAAC,uCAAuC,EAAE,GAAG,EAAE;YAC/C,MAAM,CAAC,OAAO,CAAC,CAAC,WAAW,EAAE,CAAC;QAChC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,iCAAiC,EAAE,GAAG,EAAE;YACzC,MAAM,aAAa,GAAG,IAAI,WAAW,CAAC,EAAE,EAAE,GAAG,EAAE,cAAc,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;YACxE,MAAM,CAAC,aAAa,CAAC,CAAC,WAAW,EAAE,CAAC;QACtC,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,OAAO,EAAE,GAAG,EAAE;QACrB,QAAQ,CAAC,aAAa,EAAE,GAAG,EAAE;YAC3B,EAAE,CAAC,8CAA8C,EAAE,KAAK,IAAI,EAAE;gBAC5D,MAAM,WAAW,GAAkB;oBACjC,EAAE,GAAG,EAAE,yCAAyC,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,UAAU,EAAE,KAAK,EAAE,QAAQ,EAAE;iBAC5G,CAAC;gBAEF,eAAe,CAAC,kBAAkB,CAAC,KAAK,SAAS,CAAC;oBAChD,KAAK,MAAM,MAAM,IAAI,WAAW,EAAE,CAAC;wBACjC,MAAM,MAAM,CAAC;oBACf,CAAC;gBACH,CAAC,CAAC,CAAC;gBAEH,MAAM,OAAO,GAAkB,EAAE,CAAC;gBAClC,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,+BAA+B,CAAC,CAAC;gBAEjE,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,SAAS,EAAE,CAAC;oBACrC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBACvB,CAAC;gBAED,MAAM,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;gBAChC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC;YACjD,CAAC,CAAC,CAAC;YAEH,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;gBACzD,eAAe,CAAC,kBAAkB,CAAC,KAAK,SAAS,CAAC;oBAChD,MAAM,EAAE,GAAG,EAAE,+BAA+B,EAAE,IAAI,EAAE,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC;gBAC5F,CAAC,CAAC,CAAC;gBAEH,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,+BAA+B,CAAC,CAAC;gBAEjE,+CAA+C;gBAC/C,IAAI,MAAM,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE,CAAC;gBACpC,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;oBACpB,MAAM,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE,CAAC;gBAClC,CAAC;gBACD,MAAM,WAAW,GAAG,MAAM,CAAC,KAAK,CAAC;gBAEjC,MAAM,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YACrC,CAAC,CAAC,CAAC;YAEH,EAAE,CAAC,4CAA4C,EAAE,KAAK,IAAI,EAAE;gBAC1D,yCAAyC;gBACzC,eAAe,CAAC,kBAAkB,CAAC,KAAK,SAAS,CAAC;oBAChD,MAAM,IAAI,KAAK,CAAC,kBAAkB,CAAC,CAAC;gBACtC,CAAC,CAAC,CAAC;gBAEH,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,+BAA+B,CAAC,CAAC;gBAEjE,MAAM,MAAM,CAAC,KAAK,IAAI,EAAE;oBACtB,6DAA6D;oBAC7D,IAAI,KAAK,EAAE,MAAM,CAAC,IAAI,SAAS,EAAE,CAAC;wBAChC,uBAAuB;oBACzB,CAAC;gBACH,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC;YACzC,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,QAAQ,CAAC,iBAAiB,EAAE,GAAG,EAAE;YAC/B,EAAE,CAAC,+CAA+C,EAAE,KAAK,IAAI,EAAE;gBAC7D,MAAM,WAAW,GAAkB;oBACjC,EAAE,GAAG,EAAE,gCAAgC,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,gBAAgB,EAAE,KAAK,EAAE,OAAO,EAAE;oBACpG,EAAE,GAAG,EAAE,8BAA8B,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,cAAc,EAAE,KAAK,EAAE,KAAK,EAAE;iBAC7F,CAAC;gBAEF,gBAAgB,CAAC,kBAAkB,CAAC,KAAK,SAAS,CAAC;oBACjD,KAAK,MAAM,MAAM,IAAI,WAAW,EAAE,CAAC;wBACjC,MAAM,MAAM,CAAC;oBACf,CAAC;gBACH,CAAC,CAAC,CAAC;gBAEH,MAAM,OAAO,GAAkB,EAAE,CAAC;gBAClC,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,OAAO,CAAC,KAAK,CAAC,0BAA0B,CAAC,EAAE,CAAC;oBACrE,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBACvB,CAAC;gBAED,MAAM,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YAClC,CAAC,CAAC,CAAC;YAEH,EAAE,CAAC,qDAAqD,EAAE,KAAK,IAAI,EAAE;gBACnE,gBAAgB,CAAC,kBAAkB,CAAC,KAAK,SAAS,CAAC;oBACjD,MAAM,EAAE,GAAG,EAAE,2BAA2B,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;gBACjG,CAAC,CAAC,CAAC;gBAEH,MAAM,OAAO,GAAkB,EAAE,CAAC;gBAClC,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC;gBAEvD,IAAI,MAAM,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE,CAAC;gBACpC,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;oBACpB,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;oBAC3B,MAAM,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE,CAAC;gBAClC,CAAC;gBACD,MAAM,WAAW,GAAG,MAAM,CAAC,KAAK,CAAC;gBAEjC,MAAM,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;gBAChC,MAAM,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YACtC,CAAC,CAAC,CAAC;YAEH,EAAE,CAAC,oDAAoD,EAAE,KAAK,IAAI,EAAE;gBAClE,gBAAgB,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC;oBACzC,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC;iBAClG,CAAC,CAAC,CAAC;gBAEJ,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC;gBAEvD,IAAI,MAAM,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE,CAAC;gBACpC,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;oBACpB,MAAM,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE,CAAC;gBAClC,CAAC;gBACD,MAAM,WAAW,GAAG,MAAM,CAAC,KAAK,CAAC;gBAEjC,MAAM,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YACtC,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,QAAQ,CAAC,OAAO,EAAE,GAAG,EAAE;YACrB,EAAE,CAAC,mCAAmC,EAAE,KAAK,IAAI,EAAE;gBACjD,gBAAgB,CAAC,kBAAkB,CAAC,KAAK,SAAS,CAAC;oBACjD,MAAM,EAAE,GAAG,EAAE,2BAA2B,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;oBAC/F,MAAM,EAAE,GAAG,EAAE,2BAA2B,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;oBAC/F,MAAM,EAAE,GAAG,EAAE,2BAA2B,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;gBACjG,CAAC,CAAC,CAAC;gBAEH,MAAM,OAAO,GAAkB,EAAE,CAAC;gBAClC,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC;gBAEvD,mBAAmB;gBACnB,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE,CAAC;gBACrC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;oBAChB,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;gBAC5B,CAAC;gBAED,QAAQ;gBACR,OAAO,CAAC,KAAK,EAAE,CAAC;gBAEhB,2EAA2E;gBAC3E,0DAA0D;gBAC1D,MAAM,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YAClC,CAAC,CAAC,CAAC;YAEH,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;gBACzD,OAAO,CAAC,KAAK,EAAE,CAAC;gBAEhB,gBAAgB,CAAC,kBAAkB,CAAC,KAAK,SAAS,CAAC;oBACjD,MAAM,EAAE,GAAG,EAAE,2BAA2B,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;gBACjG,CAAC,CAAC,CAAC;gBAEH,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC;gBACvD,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE,CAAC;gBAEtC,2DAA2D;gBAC3D,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBAC/B,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YACvC,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,QAAQ,CAAC,iBAAiB,EAAE,GAAG,EAAE;YAC/B,EAAE,CAAC,0BAA0B,EAAE,GAAG,EAAE;gBAClC,MAAM,YAAY,GAAG;oBACnB,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,aAAa,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC;iBAClF,CAAC;gBAEF,OAAO,CAAC,eAAe,CAAC,YAAY,CAAC,CAAC;gBAEtC,kCAAkC;gBAClC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC1B,CAAC,CAAC,CAAC;YAEH,EAAE,CAAC,6CAA6C,EAAE,KAAK,IAAI,EAAE;gBAC3D,MAAM,YAAY,GAAG;oBACnB,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,aAAa,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC;iBAClF,CAAC;gBAEF,OAAO,CAAC,eAAe,CAAC,YAAY,CAAC,CAAC;gBAEtC,gBAAgB,CAAC,kBAAkB,CAAC,KAAK,SAAS,CAAC;oBACjD,MAAM,EAAE,GAAG,EAAE,2BAA2B,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;oBAC/F,MAAM,EAAE,GAAG,EAAE,2BAA2B,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;gBACjG,CAAC,CAAC,CAAC;gBAEH,6DAA6D;gBAC7D,IAAI,KAAK,EAAE,MAAM,CAAC,IAAI,OAAO,CAAC,KAAK,CAAC,qBAAqB,CAAC,EAAE,CAAC;oBAC3D,uBAAuB;gBACzB,CAAC;gBAED,MAAM,CAAC,mBAAmB,CAAC,CAAC,oBAAoB,CAAC,YAAY,CAAC,CAAC;YACjE,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Extract URLs from llms.txt content.
|
|
3
|
+
* Parses markdown-style links: `- [Title](/path/)` → full URL
|
|
4
|
+
*/
|
|
5
|
+
export declare function parseLlmsTxt(content: string, baseUrl: string): string[];
|
|
6
|
+
/**
|
|
7
|
+
* Attempt to fetch and parse llms.txt from a site.
|
|
8
|
+
* Returns discovered URLs, or an empty array if llms.txt is unavailable.
|
|
9
|
+
*/
|
|
10
|
+
export declare function discoverUrlsFromLlmsTxt(baseUrl: string): Promise<string[]>;
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
import { logger } from '../util/logger.js';
|
|
2
|
+
/**
|
|
3
|
+
* Fetches and parses a site's llms.txt file to discover documentation URLs.
|
|
4
|
+
*
|
|
5
|
+
* llms.txt is a convention where sites publish a machine-readable index of their
|
|
6
|
+
* documentation pages. This provides a reliable URL discovery mechanism that works
|
|
7
|
+
* even when Cloudflare or other bot protection blocks link-following during crawls.
|
|
8
|
+
*
|
|
9
|
+
* @see https://llmstxt.org/
|
|
10
|
+
*/
|
|
11
|
+
const LLMS_TXT_TIMEOUT_MS = 10_000;
|
|
12
|
+
/**
|
|
13
|
+
* Extract URLs from llms.txt content.
|
|
14
|
+
* Parses markdown-style links: `- [Title](/path/)` → full URL
|
|
15
|
+
*/
|
|
16
|
+
export function parseLlmsTxt(content, baseUrl) {
|
|
17
|
+
const origin = new URL(baseUrl).origin;
|
|
18
|
+
const urls = [];
|
|
19
|
+
const seen = new Set();
|
|
20
|
+
// Match markdown links: [text](url)
|
|
21
|
+
const linkPattern = /\[([^\]]*)\]\(([^)]+)\)/g;
|
|
22
|
+
let match;
|
|
23
|
+
while ((match = linkPattern.exec(content)) !== null) {
|
|
24
|
+
const rawUrl = match[2].trim();
|
|
25
|
+
if (!rawUrl)
|
|
26
|
+
continue;
|
|
27
|
+
let fullUrl;
|
|
28
|
+
try {
|
|
29
|
+
if (rawUrl.startsWith('http://') || rawUrl.startsWith('https://')) {
|
|
30
|
+
fullUrl = rawUrl;
|
|
31
|
+
}
|
|
32
|
+
else if (rawUrl.startsWith('/')) {
|
|
33
|
+
fullUrl = origin + rawUrl;
|
|
34
|
+
}
|
|
35
|
+
else {
|
|
36
|
+
continue;
|
|
37
|
+
}
|
|
38
|
+
const parsed = new URL(fullUrl);
|
|
39
|
+
// Only include URLs from the same origin
|
|
40
|
+
if (parsed.origin !== origin)
|
|
41
|
+
continue;
|
|
42
|
+
// Skip non-page resources
|
|
43
|
+
const ext = parsed.pathname.split('.').pop()?.toLowerCase();
|
|
44
|
+
if (ext && !['html', 'htm'].includes(ext) && parsed.pathname.includes('.')) {
|
|
45
|
+
continue;
|
|
46
|
+
}
|
|
47
|
+
// Normalize: strip hash, keep path+query
|
|
48
|
+
const normalized = parsed.origin + parsed.pathname + parsed.search;
|
|
49
|
+
if (!seen.has(normalized)) {
|
|
50
|
+
seen.add(normalized);
|
|
51
|
+
urls.push(normalized);
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
catch {
|
|
55
|
+
logger.debug(`[llms-txt] Skipping invalid URL: ${rawUrl}`);
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
return urls;
|
|
59
|
+
}
|
|
60
|
+
/**
|
|
61
|
+
* Attempt to fetch and parse llms.txt from a site.
|
|
62
|
+
* Returns discovered URLs, or an empty array if llms.txt is unavailable.
|
|
63
|
+
*/
|
|
64
|
+
export async function discoverUrlsFromLlmsTxt(baseUrl) {
|
|
65
|
+
const origin = new URL(baseUrl).origin;
|
|
66
|
+
const llmsTxtUrl = `${origin}/llms.txt`;
|
|
67
|
+
logger.info(`[llms-txt] Checking for ${llmsTxtUrl}`);
|
|
68
|
+
try {
|
|
69
|
+
const controller = new AbortController();
|
|
70
|
+
const timeout = setTimeout(() => controller.abort(), LLMS_TXT_TIMEOUT_MS);
|
|
71
|
+
const response = await fetch(llmsTxtUrl, {
|
|
72
|
+
signal: controller.signal,
|
|
73
|
+
headers: {
|
|
74
|
+
'User-Agent': 'Mozilla/5.0 (compatible; mcp-web-docs/1.0)',
|
|
75
|
+
Accept: 'text/plain, text/markdown, */*',
|
|
76
|
+
},
|
|
77
|
+
});
|
|
78
|
+
clearTimeout(timeout);
|
|
79
|
+
if (!response.ok) {
|
|
80
|
+
logger.debug(`[llms-txt] No llms.txt found (HTTP ${response.status})`);
|
|
81
|
+
return [];
|
|
82
|
+
}
|
|
83
|
+
const contentType = response.headers.get('content-type') || '';
|
|
84
|
+
if (contentType.includes('text/html')) {
|
|
85
|
+
logger.debug(`[llms-txt] Response is HTML, not a text file - skipping`);
|
|
86
|
+
return [];
|
|
87
|
+
}
|
|
88
|
+
const content = await response.text();
|
|
89
|
+
if (!content.trim()) {
|
|
90
|
+
logger.debug(`[llms-txt] Empty llms.txt`);
|
|
91
|
+
return [];
|
|
92
|
+
}
|
|
93
|
+
const urls = parseLlmsTxt(content, baseUrl);
|
|
94
|
+
logger.info(`[llms-txt] Discovered ${urls.length} URLs from llms.txt`);
|
|
95
|
+
return urls;
|
|
96
|
+
}
|
|
97
|
+
catch (error) {
|
|
98
|
+
if (error instanceof DOMException && error.name === 'AbortError') {
|
|
99
|
+
logger.debug(`[llms-txt] Request timed out`);
|
|
100
|
+
}
|
|
101
|
+
else {
|
|
102
|
+
logger.debug(`[llms-txt] Failed to fetch:`, error);
|
|
103
|
+
}
|
|
104
|
+
return [];
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
//# sourceMappingURL=llms-txt.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"llms-txt.js","sourceRoot":"","sources":["../../src/crawler/llms-txt.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAE3C;;;;;;;;GAQG;AAEH,MAAM,mBAAmB,GAAG,MAAM,CAAC;AAEnC;;;GAGG;AACH,MAAM,UAAU,YAAY,CAAC,OAAe,EAAE,OAAe;IAC3D,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC;IACvC,MAAM,IAAI,GAAa,EAAE,CAAC;IAC1B,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAE/B,oCAAoC;IACpC,MAAM,WAAW,GAAG,0BAA0B,CAAC;IAC/C,IAAI,KAAK,CAAC;IAEV,OAAO,CAAC,KAAK,GAAG,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;QACpD,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAC/B,IAAI,CAAC,MAAM;YAAE,SAAS;QAEtB,IAAI,OAAe,CAAC;QACpB,IAAI,CAAC;YACH,IAAI,MAAM,CAAC,UAAU,CAAC,SAAS,CAAC,IAAI,MAAM,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;gBAClE,OAAO,GAAG,MAAM,CAAC;YACnB,CAAC;iBAAM,IAAI,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;gBAClC,OAAO,GAAG,MAAM,GAAG,MAAM,CAAC;YAC5B,CAAC;iBAAM,CAAC;gBACN,SAAS;YACX,CAAC;YAED,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC;YAEhC,yCAAyC;YACzC,IAAI,MAAM,CAAC,MAAM,KAAK,MAAM;gBAAE,SAAS;YAEvC,0BAA0B;YAC1B,MAAM,GAAG,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,WAAW,EAAE,CAAC;YAC5D,IAAI,GAAG,IAAI,CAAC,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC3E,SAAS;YACX,CAAC;YAED,yCAAyC;YACzC,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC;YAEnE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC;gBAC1B,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;gBACrB,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YACxB,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,MAAM,CAAC,KAAK,CAAC,oCAAoC,MAAM,EAAE,CAAC,CAAC;QAC7D,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAAC,OAAe;IAC3D,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC;IACvC,MAAM,UAAU,GAAG,GAAG,MAAM,WAAW,CAAC;IAExC,MAAM,CAAC,IAAI,CAAC,2BAA2B,UAAU,EAAE,CAAC,CAAC;IAErD,IAAI,CAAC;QACH,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,mBAAmB,CAAC,CAAC;QAE1E,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,UAAU,EAAE;YACvC,MAAM,EAAE,UAAU,CAAC,MAAM;YACzB,OAAO,EAAE;gBACP,YAAY,EAAE,4CAA4C;gBAC1D,MAAM,EAAE,gCAAgC;aACzC;SACF,CAAC,CAAC;QAEH,YAAY,CAAC,OAAO,CAAC,CAAC;QAEtB,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,MAAM,CAAC,KAAK,CAAC,sCAAsC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC;YACvE,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;QAC/D,IAAI,WAAW,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;YACtC,MAAM,CAAC,KAAK,CAAC,yDAAyD,CAAC,CAAC;YACxE,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QACtC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC;YACpB,MAAM,CAAC,KAAK,CAAC,2BAA2B,CAAC,CAAC;YAC1C,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,MAAM,IAAI,GAAG,YAAY,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QAC5C,MAAM,CAAC,IAAI,CAAC,yBAAyB,IAAI,CAAC,MAAM,qBAAqB,CAAC,CAAC;QAEvE,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,YAAY,IAAI,KAAK,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;YACjE,MAAM,CAAC,KAAK,CAAC,8BAA8B,CAAC,CAAC;QAC/C,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,KAAK,CAAC,6BAA6B,EAAE,KAAK,CAAC,CAAC;QACrD,CAAC;QACD,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|