@cosmocoder/mcp-web-docs 1.4.4 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/crawler/browser-config.js +9 -9
- package/build/crawler/browser-config.js.map +1 -1
- package/build/crawler/browser-config.test.js +9 -9
- package/build/crawler/browser-config.test.js.map +1 -1
- package/build/crawler/crawlee-crawler.d.ts +8 -0
- package/build/crawler/crawlee-crawler.js +78 -0
- package/build/crawler/crawlee-crawler.js.map +1 -1
- package/build/crawler/crawlee-crawler.test.js +1 -0
- package/build/crawler/crawlee-crawler.test.js.map +1 -1
- package/build/crawler/docs-crawler.d.ts +0 -1
- package/build/crawler/docs-crawler.js +6 -6
- package/build/crawler/docs-crawler.js.map +1 -1
- package/build/crawler/docs-crawler.test.js +16 -14
- package/build/crawler/docs-crawler.test.js.map +1 -1
- package/build/crawler/llms-txt.d.ts +10 -0
- package/build/crawler/llms-txt.js +107 -0
- package/build/crawler/llms-txt.js.map +1 -0
- package/build/crawler/llms-txt.test.d.ts +1 -0
- package/build/crawler/llms-txt.test.js +113 -0
- package/build/crawler/llms-txt.test.js.map +1 -0
- package/build/crawler/queue-manager.d.ts +10 -0
- package/build/crawler/queue-manager.js +45 -8
- package/build/crawler/queue-manager.js.map +1 -1
- package/build/crawler/queue-manager.test.js +104 -1
- package/build/crawler/queue-manager.test.js.map +1 -1
- package/build/util/docs.d.ts +12 -0
- package/build/util/docs.js +15 -0
- package/build/util/docs.js.map +1 -1
- package/build/util/docs.test.js +25 -1
- package/build/util/docs.test.js.map +1 -1
- package/package.json +1 -1
|
@@ -5,15 +5,16 @@ log.setLevel(log.LEVELS.OFF);
|
|
|
5
5
|
export const getBrowserConfig = (requestQueue) => ({
|
|
6
6
|
maxRequestsPerCrawl: 1000,
|
|
7
7
|
requestQueue,
|
|
8
|
-
maxConcurrency:
|
|
9
|
-
maxRequestsPerMinute:
|
|
10
|
-
maxRequestRetries:
|
|
11
|
-
navigationTimeoutSecs:
|
|
8
|
+
maxConcurrency: 5,
|
|
9
|
+
maxRequestsPerMinute: 120,
|
|
10
|
+
maxRequestRetries: 2,
|
|
11
|
+
navigationTimeoutSecs: 30,
|
|
12
|
+
requestHandlerTimeoutSecs: 60,
|
|
12
13
|
browserPoolOptions: {
|
|
13
|
-
maxOpenPagesPerBrowser:
|
|
14
|
-
useFingerprints:
|
|
15
|
-
operationTimeoutSecs:
|
|
16
|
-
closeInactiveBrowserAfterSecs:
|
|
14
|
+
maxOpenPagesPerBrowser: 3,
|
|
15
|
+
useFingerprints: true,
|
|
16
|
+
operationTimeoutSecs: 30,
|
|
17
|
+
closeInactiveBrowserAfterSecs: 20,
|
|
17
18
|
},
|
|
18
19
|
preNavigationHooks: [
|
|
19
20
|
async ({ page }) => {
|
|
@@ -21,7 +22,6 @@ export const getBrowserConfig = (requestQueue) => ({
|
|
|
21
22
|
await page.setExtraHTTPHeaders({
|
|
22
23
|
'Accept-Language': 'en-US,en;q=0.9',
|
|
23
24
|
Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
|
|
24
|
-
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
|
|
25
25
|
});
|
|
26
26
|
},
|
|
27
27
|
],
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"browser-config.js","sourceRoot":"","sources":["../../src/crawler/browser-config.ts"],"names":[],"mappings":"AAAA,OAAO,EAA4B,GAAG,EAAE,MAAM,SAAS,CAAC;AAExD,0DAA0D;AAC1D,2DAA2D;AAC3D,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;AAE7B,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,YAAsD,EAAqC,EAAE,CAAC,CAAC;IAC9H,mBAAmB,EAAE,IAAI;IACzB,YAAY;IACZ,cAAc,EAAE,
|
|
1
|
+
{"version":3,"file":"browser-config.js","sourceRoot":"","sources":["../../src/crawler/browser-config.ts"],"names":[],"mappings":"AAAA,OAAO,EAA4B,GAAG,EAAE,MAAM,SAAS,CAAC;AAExD,0DAA0D;AAC1D,2DAA2D;AAC3D,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;AAE7B,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,YAAsD,EAAqC,EAAE,CAAC,CAAC;IAC9H,mBAAmB,EAAE,IAAI;IACzB,YAAY;IACZ,cAAc,EAAE,CAAC;IACjB,oBAAoB,EAAE,GAAG;IACzB,iBAAiB,EAAE,CAAC;IACpB,qBAAqB,EAAE,EAAE;IACzB,yBAAyB,EAAE,EAAE;IAC7B,kBAAkB,EAAE;QAClB,sBAAsB,EAAE,CAAC;QACzB,eAAe,EAAE,IAAI;QACrB,oBAAoB,EAAE,EAAE;QACxB,6BAA6B,EAAE,EAAE;KAClC;IACD,kBAAkB,EAAE;QAClB,KAAK,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE;YACjB,MAAM,IAAI,CAAC,eAAe,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;YAC1D,MAAM,IAAI,CAAC,mBAAmB,CAAC;gBAC7B,iBAAiB,EAAE,gBAAgB;gBACnC,MAAM,EAAE,4EAA4E;aACrF,CAAC,CAAC;QACL,CAAC;KACF;CACF,CAAC,CAAC"}
|
|
@@ -5,10 +5,11 @@ describe('Browser Config', () => {
|
|
|
5
5
|
const mockQueue = {};
|
|
6
6
|
const config = getBrowserConfig(mockQueue);
|
|
7
7
|
expect(config.maxRequestsPerCrawl).toBe(1000);
|
|
8
|
-
expect(config.maxConcurrency).toBe(
|
|
9
|
-
expect(config.maxRequestsPerMinute).toBe(
|
|
10
|
-
expect(config.maxRequestRetries).toBe(
|
|
11
|
-
expect(config.navigationTimeoutSecs).toBe(
|
|
8
|
+
expect(config.maxConcurrency).toBe(5);
|
|
9
|
+
expect(config.maxRequestsPerMinute).toBe(120);
|
|
10
|
+
expect(config.maxRequestRetries).toBe(2);
|
|
11
|
+
expect(config.navigationTimeoutSecs).toBe(30);
|
|
12
|
+
expect(config.requestHandlerTimeoutSecs).toBe(60);
|
|
12
13
|
});
|
|
13
14
|
it('should return config with requestQueue', () => {
|
|
14
15
|
const mockQueue = { name: 'test-queue' };
|
|
@@ -19,10 +20,10 @@ describe('Browser Config', () => {
|
|
|
19
20
|
const mockQueue = {};
|
|
20
21
|
const config = getBrowserConfig(mockQueue);
|
|
21
22
|
expect(config.browserPoolOptions).toBeDefined();
|
|
22
|
-
expect(config.browserPoolOptions?.maxOpenPagesPerBrowser).toBe(
|
|
23
|
-
expect(config.browserPoolOptions?.useFingerprints).toBe(
|
|
24
|
-
expect(config.browserPoolOptions?.operationTimeoutSecs).toBe(
|
|
25
|
-
expect(config.browserPoolOptions?.closeInactiveBrowserAfterSecs).toBe(
|
|
23
|
+
expect(config.browserPoolOptions?.maxOpenPagesPerBrowser).toBe(3);
|
|
24
|
+
expect(config.browserPoolOptions?.useFingerprints).toBe(true);
|
|
25
|
+
expect(config.browserPoolOptions?.operationTimeoutSecs).toBe(30);
|
|
26
|
+
expect(config.browserPoolOptions?.closeInactiveBrowserAfterSecs).toBe(20);
|
|
26
27
|
});
|
|
27
28
|
it('should have preNavigationHooks', () => {
|
|
28
29
|
const mockQueue = {};
|
|
@@ -48,7 +49,6 @@ describe('Browser Config', () => {
|
|
|
48
49
|
expect(mockPage.setExtraHTTPHeaders).toHaveBeenCalledWith(expect.objectContaining({
|
|
49
50
|
'Accept-Language': expect.any(String),
|
|
50
51
|
Accept: expect.any(String),
|
|
51
|
-
'User-Agent': expect.any(String),
|
|
52
52
|
}));
|
|
53
53
|
});
|
|
54
54
|
});
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"browser-config.test.js","sourceRoot":"","sources":["../../src/crawler/browser-config.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AAIvD,QAAQ,CAAC,gBAAgB,EAAE,GAAG,EAAE;IAC9B,QAAQ,CAAC,kBAAkB,EAAE,GAAG,EAAE;QAChC,EAAE,CAAC,0CAA0C,EAAE,GAAG,EAAE;YAClD,MAAM,SAAS,GAAG,EAA4C,CAAC;YAC/D,MAAM,MAAM,GAAG,gBAAgB,CAAC,SAAS,CAAC,CAAC;YAE3C,MAAM,CAAC,MAAM,CAAC,mBAAmB,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC9C,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,
|
|
1
|
+
{"version":3,"file":"browser-config.test.js","sourceRoot":"","sources":["../../src/crawler/browser-config.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AAIvD,QAAQ,CAAC,gBAAgB,EAAE,GAAG,EAAE;IAC9B,QAAQ,CAAC,kBAAkB,EAAE,GAAG,EAAE;QAChC,EAAE,CAAC,0CAA0C,EAAE,GAAG,EAAE;YAClD,MAAM,SAAS,GAAG,EAA4C,CAAC;YAC/D,MAAM,MAAM,GAAG,gBAAgB,CAAC,SAAS,CAAC,CAAC;YAE3C,MAAM,CAAC,MAAM,CAAC,mBAAmB,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC9C,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACtC,MAAM,CAAC,MAAM,CAAC,oBAAoB,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAC9C,MAAM,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACzC,MAAM,CAAC,MAAM,CAAC,qBAAqB,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAC9C,MAAM,CAAC,MAAM,CAAC,yBAAyB,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACpD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,wCAAwC,EAAE,GAAG,EAAE;YAChD,MAAM,SAAS,GAAG,EAAE,IAAI,EAAE,YAAY,EAA4C,CAAC;YACnF,MAAM,MAAM,GAAG,gBAAgB,CAAC,SAAS,CAAC,CAAC;YAE3C,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAC9C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,kCAAkC,EAAE,GAAG,EAAE;YAC1C,MAAM,SAAS,GAAG,EAA4C,CAAC;YAC/D,MAAM,MAAM,GAAG,gBAAgB,CAAC,SAAS,CAAC,CAAC;YAE3C,MAAM,CAAC,MAAM,CAAC,kBAAkB,CAAC,CAAC,WAAW,EAAE,CAAC;YAChD,MAAM,CAAC,MAAM,CAAC,kBAAkB,EAAE,sBAAsB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAClE,MAAM,CAAC,MAAM,CAAC,kBAAkB,EAAE,eAAe,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC9D,MAAM,CAAC,MAAM,CAAC,kBAAkB,EAAE,oBAAoB,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACjE,MAAM,CAAC,MAAM,CAAC,kBAAkB,EAAE,6BAA6B,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC5E,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gCAAgC,EAAE,GAAG,EAAE;YACxC,MAAM,SAAS,GAAG,EAA4C,CAAC;YAC/D,MAAM,MAAM,GAAG,gBAAgB,CAAC,SAAS,CAAC,CAAC;YAE3C,MAAM,CAAC,MAAM,CAAC,kBAAkB,CAAC,CAAC,WAAW,EAAE,CAAC;YAChD,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,kBAAkB,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC5D,MAAM,CAAC,MAAM,CAAC,kBAAkB,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACpD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,4CAA4C,EAAE,KAAK,IAAI,EAAE;YAC1D,MAAM,SAAS,GAAG,EAA4C,CAAC;YAC/D,MAAM,MAAM,GAAG,gBAAgB,CAAC,SAAS,CAAC,CAAC;YAE3C,MAAM,QAAQ,GAAG;gBACf,eAAe,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,SAAS,CAAC;gBACrD,mBAAmB,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,SAAS,CAAC;aACvC,CAAC;YAErB,MAAM,WAAW,GAAG,EAAE,IAAI,EAAE,QAAQ,EAA+B,CAAC;YACpE,MAAM,eAAe,GAAG,EAA2B,CAAC;YAEpD,MAAM,IAAI,GAAG,MAAM,CAAC,kBAAkB,EAAE,CAAC,CAAC,CAAC,CAAC;YAC5C,IAAI,IAAI,EAAE,CAAC;gBACT,MAAM,IAAI,CAAC,WAAW,EAAE,eAAe,CAAC,CAAC;YAC3C,CAAC;YAED,MAAM,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC,oBAAoB,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;YACrF,MAAM,CAAC,QAAQ,CAAC,mBAAmB,CAAC,CAAC,oBAAoB,CACvD,MAAM,CAAC,gBAAgB,CAAC;gBACtB,iBAAiB,EAAE,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC;gBACrC,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC;aAC3B,CAAC,CACH,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
@@ -50,6 +50,14 @@ export declare class CrawleeCrawler extends BaseCrawler {
|
|
|
50
50
|
* This is used to detect expired sessions during crawling.
|
|
51
51
|
*/
|
|
52
52
|
private checkForLoginPage;
|
|
53
|
+
/**
|
|
54
|
+
* Wait for the page to fully stabilize after navigation, handling:
|
|
55
|
+
* - Client-side redirects (meta-refresh, JavaScript location changes) common in Docusaurus
|
|
56
|
+
* - Cloudflare challenge interstitials
|
|
57
|
+
*
|
|
58
|
+
* Must be called after initial waitForLoadState, before content extraction.
|
|
59
|
+
*/
|
|
60
|
+
private waitForPageStabilization;
|
|
53
61
|
private findContentFrame;
|
|
54
62
|
private evaluateExtractor;
|
|
55
63
|
private extractContent;
|
|
@@ -89,6 +89,80 @@ export class CrawleeCrawler extends BaseCrawler {
|
|
|
89
89
|
}
|
|
90
90
|
return false;
|
|
91
91
|
}
|
|
92
|
+
/**
|
|
93
|
+
* Wait for the page to fully stabilize after navigation, handling:
|
|
94
|
+
* - Client-side redirects (meta-refresh, JavaScript location changes) common in Docusaurus
|
|
95
|
+
* - Cloudflare challenge interstitials
|
|
96
|
+
*
|
|
97
|
+
* Must be called after initial waitForLoadState, before content extraction.
|
|
98
|
+
*/
|
|
99
|
+
async waitForPageStabilization(page) {
|
|
100
|
+
const initialUrl = page.url();
|
|
101
|
+
// Detect client-side redirect pages (Docusaurus generates these for /docs/ → /docs/intro/)
|
|
102
|
+
// These are tiny HTML pages with a meta-refresh and/or JS redirect but no real content.
|
|
103
|
+
let isRedirectPage = false;
|
|
104
|
+
try {
|
|
105
|
+
isRedirectPage = await page.evaluate(() => {
|
|
106
|
+
const metaRefresh = document.querySelector('meta[http-equiv="refresh"]');
|
|
107
|
+
if (metaRefresh)
|
|
108
|
+
return true;
|
|
109
|
+
const body = document.body;
|
|
110
|
+
const hasMinimalContent = !body || (body.textContent?.trim().length || 0) < 100;
|
|
111
|
+
const hasNoMainContent = !document.querySelector('main, article, [role="main"], .content, #content');
|
|
112
|
+
return hasMinimalContent && hasNoMainContent;
|
|
113
|
+
});
|
|
114
|
+
}
|
|
115
|
+
catch {
|
|
116
|
+
// evaluate failed — page is likely mid-navigation already, which is fine
|
|
117
|
+
isRedirectPage = true;
|
|
118
|
+
}
|
|
119
|
+
if (isRedirectPage) {
|
|
120
|
+
logger.debug(`[CrawleeCrawler] Redirect/minimal page detected at ${initialUrl}, waiting for navigation...`);
|
|
121
|
+
try {
|
|
122
|
+
await page.waitForURL((url) => url.href !== initialUrl, { timeout: 10000 });
|
|
123
|
+
await Promise.all([
|
|
124
|
+
page.waitForLoadState('domcontentloaded'),
|
|
125
|
+
page.waitForLoadState('networkidle', { timeout: 10000 }).catch(() => { }),
|
|
126
|
+
]);
|
|
127
|
+
logger.debug(`[CrawleeCrawler] Redirect completed: ${initialUrl} → ${page.url()}`);
|
|
128
|
+
}
|
|
129
|
+
catch {
|
|
130
|
+
logger.debug(`[CrawleeCrawler] No redirect detected within timeout, continuing with current page`);
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
// Detect Cloudflare challenge pages (after any redirect has settled)
|
|
134
|
+
try {
|
|
135
|
+
const isChallenge = await page.evaluate(() => {
|
|
136
|
+
const bodyText = document.body?.textContent || '';
|
|
137
|
+
const hasChallengeText = bodyText.includes('Checking your browser') ||
|
|
138
|
+
bodyText.includes('Verify you are human') ||
|
|
139
|
+
bodyText.includes('Enable JavaScript and cookies');
|
|
140
|
+
const hasChallengeElement = document.querySelector('#challenge-running, #challenge-stage, .cf-browser-verification, #cf-wrapper') !== null;
|
|
141
|
+
return hasChallengeText || hasChallengeElement;
|
|
142
|
+
});
|
|
143
|
+
if (isChallenge) {
|
|
144
|
+
logger.info(`[CrawleeCrawler] Cloudflare challenge detected, waiting for resolution...`);
|
|
145
|
+
try {
|
|
146
|
+
await page.waitForFunction(() => {
|
|
147
|
+
const bodyText = document.body?.textContent || '';
|
|
148
|
+
const stillChallenge = bodyText.includes('Checking your browser') || bodyText.includes('Verify you are human');
|
|
149
|
+
const hasChallengeElement = document.querySelector('#challenge-running, #challenge-stage, .cf-browser-verification') !== null;
|
|
150
|
+
return !stillChallenge && !hasChallengeElement;
|
|
151
|
+
}, { timeout: 15000 });
|
|
152
|
+
await page.waitForLoadState('domcontentloaded');
|
|
153
|
+
await page.waitForLoadState('networkidle', { timeout: 5000 }).catch(() => { });
|
|
154
|
+
logger.info(`[CrawleeCrawler] Cloudflare challenge resolved`);
|
|
155
|
+
}
|
|
156
|
+
catch {
|
|
157
|
+
logger.warn(`[CrawleeCrawler] Cloudflare challenge did not resolve within timeout`);
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
catch {
|
|
162
|
+
// evaluate failed during challenge detection — page may be navigating, proceed anyway
|
|
163
|
+
logger.debug(`[CrawleeCrawler] Could not check for Cloudflare challenge, continuing`);
|
|
164
|
+
}
|
|
165
|
+
}
|
|
92
166
|
async findContentFrame(page) {
|
|
93
167
|
const frames = await page.frames();
|
|
94
168
|
const contentFrames = await Promise.all(frames.map(async (frame) => {
|
|
@@ -178,6 +252,8 @@ export class CrawleeCrawler extends BaseCrawler {
|
|
|
178
252
|
this.allowedHostname = '';
|
|
179
253
|
}
|
|
180
254
|
await this.queueManager.initialize(url, this.pathPrefix);
|
|
255
|
+
// Seed URLs from llms.txt for better coverage on bot-protected sites
|
|
256
|
+
await this.queueManager.seedFromLlmsTxt(url);
|
|
181
257
|
// Build crawler options with optional authentication
|
|
182
258
|
const crawlerOptions = getBrowserConfig(this.queueManager.getRequestQueue() ?? undefined);
|
|
183
259
|
// If we have storage state (auth cookies), configure the browser to use them
|
|
@@ -223,6 +299,8 @@ export class CrawleeCrawler extends BaseCrawler {
|
|
|
223
299
|
page.waitForLoadState('domcontentloaded'),
|
|
224
300
|
page.waitForLoadState('networkidle', { timeout: 5000 }).catch(() => log.debug('Network idle timeout - continuing anyway')),
|
|
225
301
|
]);
|
|
302
|
+
// Handle client-side redirects (Docusaurus) and Cloudflare challenges
|
|
303
|
+
await this.waitForPageStabilization(page);
|
|
226
304
|
// Get the actual URL after any redirects
|
|
227
305
|
const actualUrl = page.url();
|
|
228
306
|
// Check if the page redirected outside the allowed domain
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"crawlee-crawler.js","sourceRoot":"","sources":["../../src/crawler/crawlee-crawler.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,SAAS,CAAC;AAE5C,OAAO,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAExC,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAE5C,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AACvD,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAC3C,OAAO,EAAE,eAAe,EAAE,cAAc,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAoB3F,MAAM,OAAO,cAAe,SAAQ,WAAW;IACrC,OAAO,GAA6B,IAAI,CAAC;IACzC,YAAY,GAAiB,IAAI,YAAY,EAAE,CAAC;IAChD,YAAY,CAAgB;IAC5B,WAAW,GAAY,IAAI,CAAC;IAC5B,mBAAmB,GAA+B,IAAI,CAAC;IACvD,WAAW,GAAW,EAAE,CAAC;IACjC,gFAAgF;IACxE,eAAe,GAAW,EAAE,CAAC;IACrC,iDAAiD;IACzC,oBAAoB,GAAW,CAAC,CAAC;IACzC,gDAAgD;IACxC,UAAU,CAAU;IAE5B;;OAEG;IACH,eAAe,CAAC,KAAmB;QACjC,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC;QAC1B,MAAM,CAAC,IAAI,CAAC,2CAA2C,KAAK,CAAC,OAAO,EAAE,MAAM,IAAI,CAAC,UAAU,CAAC,CAAC;IAC/F,CAAC;IAED;;;;;;OAMG;IACK,qBAAqB,CAAC,GAAW;QACvC,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC;YAC1B,OAAO,IAAI,CAAC,CAAC,4BAA4B;QAC3C,CAAC;QAED,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;YAC5B,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;YAC/C,MAAM,OAAO,GAAG,IAAI,CAAC,eAAe,CAAC,WAAW,EAAE,CAAC;YAEnD,cAAc;YACd,IAAI,QAAQ,KAAK,OAAO,EAAE,CAAC;gBACzB,OAAO,IAAI,CAAC;YACd,CAAC;YAED,wEAAwE;YACxE,2EAA2E;YAC3E,IAAI,QAAQ,CAAC,QAAQ,CAAC,GAAG,GAAG,OAAO,CAAC,EAAE,CAAC;gBACrC,OAAO,IAAI,CAAC;YACd,CAAC;YAED,OAAO,KAAK,CAAC;QACf,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,iBAAiB,CAAC,IAAU,EAAE,UAAkB;QAC5D,+CAA+C;QAC/C,uEAAuE;QACvE,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YACtB,OAAO,KAAK,CAAC;QACf,CAAC;QAED,iCAAiC;QACjC,IAAI,cAAc,CAAC,UAAU,CAAC,EAAE,CAAC;YAC/B,MAAM,CAAC,IAAI,CAAC,0DAA0D,UAAU,EAAE,CAAC,CAAC;YACpF,OAAO,IAAI,CAAC;QACd,CAAC;QAED,qBAAqB;QACrB,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,IAAI,EAAE,WAAW,IAAI,EAAE,CAAC,CAAC;YAC7E,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;YACtC,MAAM,SAAS,GAAG,eAAe,CAAC,QAAQ,GAAG,QAAQ,EAAE,UAAU,CAAC,CAAC;YAEnE,IAAI,SAAS,CAAC,WAAW,IAAI,SAAS,CAAC,UAAU,IAAI,GAAG,EAAE,CAAC;gBACzD,MAAM,CAAC,IAAI,CAAC,uEAAuE,SAAS,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;gBACvH,MAAM,CAAC,KAAK,CAAC,uCAAuC,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gBAEpF,wEAAwE;gBACxE,IAAI,CAAC,mBAAmB,GAAG,IAAI,mBAAmB,CAChD,mEAAmE,EACnE,IAAI,CAAC,WAAW,EAChB,UAAU,EACV,SAAS,CACV,CAAC;gBACF,OAAO,IAAI,CAAC;YACd,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,KAAK,CAAC,iDAAiD,EAAE,KAAK,CAAC,CAAC;QACzE,CAAC;QAED,OAAO,KAAK,CAAC;IACf,CAAC;IAEO,KAAK,CAAC,gBAAgB,CAAC,IAAU;QACvC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,EAAE,CAAC;QACnC,MAAM,aAAa,GAAG,MAAM,OAAO,CAAC,GAAG,CACrC,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE;YACzB,IAAI,CAAC;gBACH,MAAM,UAAU,GAAG,MAAM,KAAK;qBAC3B,QAAQ,CAAC,GAAG,EAAE;oBACb,OAAO,QAAQ,CAAC,aAAa,CAAC,6DAA6D,CAAC,KAAK,IAAI,CAAC;gBACxG,CAAC,CAAC;qBACD,KAAK,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC;gBAEtB,IAAI,UAAU,EAAE,CAAC;oBACf,MAAM,OAAO,CAAC,GAAG,CAAC;wBAChB,KAAK,CAAC,gBAAgB,CAAC,kBAAkB,CAAC;wBAC1C,KAAK;6BACF,gBAAgB,CAAC,aAAa,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;6BAClD,KAAK,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,gDAAgD,CAAC,CAAC;qBAC/E,CAAC,CAAC;oBACH,OAAO,KAAK,CAAC;gBACf,CAAC;YACH,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,CAAC,KAAK,CAAC,sBAAsB,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YACjE,CAAC;YACD,OAAO,IAAI,CAAC;QACd,CAAC,CAAC,CACH,CAAC;QAEF,MAAM,KAAK,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,IAAI,IAAI,CAAC;QAC5D,IAAI,KAAK,EAAE,CAAC;YACV,MAAM,CAAC,KAAK,CAAC,yBAAyB,CAAC,CAAC;YACxC,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC;QAC5D,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IAEO,KAAK,CAAC,iBAAiB,CAAC,OAAqB,EAAE,SAA2B;QAChF,MAAM,aAAa,GAAG,SAAS,CAAC,WAAW,CAAC,QAAQ,EAAE,CAAC;QACvD,OAAO,OAAO,CAAC,QAAQ,CAAC,KAAK,EAAE,IAAY,EAAE,EAAE;YAC7C,MAAM,cAAc,GAAG,IAAI,QAAQ,CAAC,UAAU,IAAI,EAAE,CAAC,EAAE,CAAC;YACxD,MAAM,SAAS,GAAG,IAAI,cAAc,EAAE,CAAC;YACvC,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;YACxD,OAAO,MAAM,CAAC,OAAO,CAAC;QACxB,CAAC,EAAE,aAAa,CAAC,CAAC;IACpB,CAAC;IAEO,KAAK,CAAC,cAAc,CAC1B,IAAU,EACV,QAAgB,EAChB,SAA2B;QAE3B,IAAI,OAAO,GAAG,EAAE,CAAC;QACjB,IAAI,aAAa,GAAG,SAAS,CAAC,WAAW,CAAC,IAAI,CAAC;QAE/C,IAAI,CAAC;YACH,IAAI,QAAQ,KAAK,WAAW,EAAE,CAAC;gBAC7B,mBAAmB;gBACnB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC;gBAChD,IAAI,KAAK,EAAE,CAAC;oBACV,OAAO,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;gBAC3D,CAAC;gBAED,wBAAwB;gBACxB,IAAI,CAAC,OAAO,EAAE,CAAC;oBACb,OAAO,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;gBAC1D,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,OAAO,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;YAC1D,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAS,GAAG,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC;YAC7E,aAAa,GAAG,eAAe,CAAC;QAClC,CAAC;QAED,OAAO,EAAE,OAAO,EAAE,aAAa,EAAE,CAAC;IACpC,CAAC;IAED;;;OAGG;IACH,aAAa,CAAC,MAAc;QAC1B,IAAI,CAAC,UAAU,GAAG,MAAM,CAAC;QACzB,MAAM,CAAC,IAAI,CAAC,iDAAiD,MAAM,EAAE,CAAC,CAAC;IACzE,CAAC;IAED,KAAK,CAAC,CAAC,KAAK,CAAC,GAAW;QACtB,MAAM,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,wBAAwB,GAAG,EAAE,CAAC,CAAC;QAErE,6BAA6B;QAC7B,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;QACxB,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC;QAChC,IAAI,CAAC,WAAW,GAAG,GAAG,CAAC;QACvB,IAAI,CAAC,oBAAoB,GAAG,CAAC,CAAC;QAE9B,8DAA8D;QAC9D,IAAI,CAAC;YACH,IAAI,CAAC,eAAe,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;YAC7C,MAAM,CAAC,IAAI,CAAC,+DAA+D,IAAI,CAAC,eAAe,EAAE,CAAC,CAAC;QACrG,CAAC;QAAC,MAAM,CAAC;YACP,IAAI,CAAC,eAAe,GAAG,EAAE,CAAC;QAC5B,CAAC;QAED,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;QAEzD,qDAAqD;QACrD,MAAM,cAAc,GAAG,gBAAgB,CAAC,IAAI,CAAC,YAAY,CAAC,eAAe,EAAE,IAAI,SAAS,CAAC,CAAC;QAE1F,6EAA6E;QAC7E,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACtB,MAAM,CAAC,IAAI,CAAC,qDAAqD,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE,MAAM,IAAI,CAAC,UAAU,CAAC,CAAC;YACnH,cAAc,CAAC,aAAa,GAAG;gBAC7B,GAAG,cAAc,CAAC,aAAa;gBAC/B,aAAa,EAAE;oBACb,GAAG,cAAc,CAAC,aAAa,EAAE,aAAa;iBAC/C;aACF,CAAC;YACF,cAAc,CAAC,kBAAkB,GAAG;gBAClC,GAAG,cAAc,CAAC,kBAAkB;gBACpC,cAAc,EAAE;oBACd,KAAK,EAAE,MAAM,EAAE,EAAE;wBACf,0DAA0D;wBAC1D,MAAM,CAAC,KAAK,CAAC,+CAA+C,MAAM,EAAE,CAAC,CAAC;oBACxE,CAAC;iBACF;aACF,CAAC;YACF,qCAAqC;YACrC,MAAM,aAAa,GAAG,cAAc,CAAC,kBAAkB,IAAI,EAAE,CAAC;YAC9D,cAAc,CAAC,kBAAkB,GAAG;gBAClC,GAAG,aAAa;gBAChB,KAAK,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE;oBACjB,IAAI,IAAI,CAAC,YAAY,EAAE,OAAO,EAAE,CAAC;wBAC/B,MAAM,CAAC,KAAK,CAAC,4BAA4B,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,MAAM,4BAA4B,CAAC,CAAC;wBACvG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;oBAC7D,CAAC;gBACH,CAAC;aACF,CAAC;QACJ,CAAC;QAED,IAAI,CAAC,OAAO,GAAG,IAAI,iBAAiB,CAAC;YACnC,GAAG,cAAc;YACjB,cAAc,EAAE,KAAK,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,GAAG,EAAE,EAAE,EAAE;gBAC7D,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;oBACpB,GAAG,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;oBAC3B,OAAO;gBACT,CAAC;gBAED,IAAI,CAAC;oBACH,6BAA6B;oBAC7B,MAAM,OAAO,CAAC,GAAG,CAAC;wBAChB,IAAI,CAAC,gBAAgB,CAAC,kBAAkB,CAAC;wBACzC,IAAI,CAAC,gBAAgB,CAAC,aAAa,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,0CAA0C,CAAC,CAAC;qBAC3H,CAAC,CAAC;oBAEH,yCAAyC;oBACzC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;oBAE7B,0DAA0D;oBAC1D,IAAI,CAAC,IAAI,CAAC,qBAAqB,CAAC,SAAS,CAAC,EAAE,CAAC;wBAC3C,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;wBACxD,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC,QAAQ,CAAC;wBAEnD,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;4BACrB,gFAAgF;4BAChF,MAAM,CAAC,IAAI,CAAC,kEAAkE,iBAAiB,MAAM,cAAc,EAAE,CAAC,CAAC;4BAEvH,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;gCACtB,mDAAmD;gCACnD,IAAI,CAAC,mBAAmB,GAAG,IAAI,mBAAmB,CAChD,4EAA4E,cAAc,GAAG,EAC7F,IAAI,CAAC,WAAW,EAChB,SAAS,EACT,EAAE,WAAW,EAAE,IAAI,EAAE,UAAU,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,mBAAmB,iBAAiB,OAAO,cAAc,EAAE,CAAC,EAAE,CAC/G,CAAC;gCACF,GAAG,CAAC,KAAK,CAAC,oDAAoD,cAAc,mBAAmB,CAAC,CAAC;gCACjG,IAAI,CAAC,KAAK,EAAE,CAAC;gCACb,OAAO;4BACT,CAAC;iCAAM,CAAC;gCACN,0DAA0D;gCAC1D,GAAG,CAAC,KAAK,CAAC,6CAA6C,cAAc,mBAAmB,CAAC,CAAC;gCAC1F,IAAI,CAAC,KAAK,EAAE,CAAC;gCACb,OAAO;4BACT,CAAC;wBACH,CAAC;6BAAM,CAAC;4BACN,sDAAsD;4BACtD,IAAI,CAAC,oBAAoB,EAAE,CAAC;4BAC5B,GAAG,CAAC,OAAO,CACT,iDAAiD,OAAO,CAAC,GAAG,MAAM,SAAS,aAAa,IAAI,CAAC,oBAAoB,kBAAkB,CACpI,CAAC;4BACF,OAAO;wBACT,CAAC;oBACH,CAAC;oBAED,gEAAgE;oBAChE,IAAI,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;wBAC1C,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;wBAClE,IAAI,WAAW,EAAE,CAAC;4BAChB,GAAG,CAAC,KAAK,CAAC,uEAAuE,CAAC,CAAC;4BACnF,IAAI,CAAC,KAAK,EAAE,CAAC;4BACb,OAAO;wBACT,CAAC;wBACD,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;oBAC3B,CAAC;yBAAM,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;wBAC5B,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;oBAC3B,CAAC;oBAED,qCAAqC;oBACrC,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;wBAC7B,IAAI,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;4BAC5B,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;gCACjB,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;4BAChC,CAAC;4BAED,MAAM,IAAI,CAAC,YAAY,CAAC,mBAAmB,CAAC,YAAY,EAAE,GAAG,EAAE,IAAI,CAAC,CAAC;4BAErE,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;4BACjC,MAAM,EAAE,OAAO,EAAE,aAAa,EAAE,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;4BAE9F,MAAM,MAAM,GAAgB;gCAC1B,GAAG,EAAE,OAAO,CAAC,GAAG;gCAChB,IAAI,EAAE,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,QAAQ,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM;gCACjE,OAAO,EAAE,YAAY,CAAC,OAAO,CAAC;gCAC9B,KAAK;gCACL,aAAa;6BACd,CAAC;4BAEF,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;4BACpC,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;4BACnC,MAAM;wBACR,CAAC;oBACH,CAAC;gBACH,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,MAAM,YAAY,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;oBAC5E,GAAG,CAAC,KAAK,CAAC,oBAAoB,OAAO,CAAC,GAAG,KAAK,YAAY,EAAE,CAAC,CAAC;gBAChE,CAAC;YACH,CAAC;SACF,CAAC,CAAC;QAEH,IAAI,CAAC;YACH,MAAM,cAAc,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC;YAE1C,OAAO,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC;gBACxB,IAAI,IAAI,CAAC,YAAY,CAAC,gBAAgB,EAAE,EAAE,CAAC;oBACzC,KAAK,MAAM,MAAM,IAAI,MAAM,IAAI,CAAC,YAAY,CAAC,YAAY,EAAE,EAAE,CAAC;wBAC5D,MAAM,MAAM,CAAC;oBACf,CAAC;gBACH,CAAC;gBAED,IAAI,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,cAAc,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,EAAE,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;oBAAE,MAAM;YACpI,CAAC;YAED,MAAM,cAAc,CAAC;YACrB,MAAM,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC;YAEjC,4CAA4C;YAC5C,IAAI,IAAI,CAAC,oBAAoB,GAAG,CAAC,EAAE,CAAC;gBAClC,MAAM,CAAC,IAAI,CACT,4BAA4B,IAAI,CAAC,oBAAoB,sDAAsD,IAAI,CAAC,eAAe,GAAG,CACnI,CAAC;YACJ,CAAC;YAED,0DAA0D;YAC1D,IAAI,IAAI,CAAC,mBAAmB,EAAE,CAAC;gBAC7B,MAAM,IAAI,CAAC,mBAAmB,CAAC;YACjC,CAAC;YAED,KAAK,MAAM,MAAM,IAAI,MAAM,IAAI,CAAC,YAAY,CAAC,YAAY,EAAE,EAAE,CAAC;gBAC5D,MAAM,MAAM,CAAC;YACf,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,wCAAwC;YACxC,IAAI,KAAK,YAAY,mBAAmB,EAAE,CAAC;gBACzC,MAAM,KAAK,CAAC;YACd,CAAC;YACD,MAAM,CAAC,KAAK,CAAC,gBAAgB,EAAE,KAAK,CAAC,CAAC;YACtC,MAAM,KAAK,CAAC;QACd,CAAC;gBAAS,CAAC;YACT,MAAM,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE,CAAC;YAClC,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACtB,CAAC;IACH,CAAC;IAED,KAAK;QACH,KAAK,CAAC,KAAK,EAAE,CAAC;QACd,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,6BAA6B,EAAE,GAAG,CAAC,CAAC,CAAC;QAC3F,CAAC;IACH,CAAC;CACF"}
|
|
1
|
+
{"version":3,"file":"crawlee-crawler.js","sourceRoot":"","sources":["../../src/crawler/crawlee-crawler.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,SAAS,CAAC;AAE5C,OAAO,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAExC,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAE5C,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AACvD,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAC3C,OAAO,EAAE,eAAe,EAAE,cAAc,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAoB3F,MAAM,OAAO,cAAe,SAAQ,WAAW;IACrC,OAAO,GAA6B,IAAI,CAAC;IACzC,YAAY,GAAiB,IAAI,YAAY,EAAE,CAAC;IAChD,YAAY,CAAgB;IAC5B,WAAW,GAAY,IAAI,CAAC;IAC5B,mBAAmB,GAA+B,IAAI,CAAC;IACvD,WAAW,GAAW,EAAE,CAAC;IACjC,gFAAgF;IACxE,eAAe,GAAW,EAAE,CAAC;IACrC,iDAAiD;IACzC,oBAAoB,GAAW,CAAC,CAAC;IACzC,gDAAgD;IACxC,UAAU,CAAU;IAE5B;;OAEG;IACH,eAAe,CAAC,KAAmB;QACjC,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC;QAC1B,MAAM,CAAC,IAAI,CAAC,2CAA2C,KAAK,CAAC,OAAO,EAAE,MAAM,IAAI,CAAC,UAAU,CAAC,CAAC;IAC/F,CAAC;IAED;;;;;;OAMG;IACK,qBAAqB,CAAC,GAAW;QACvC,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC;YAC1B,OAAO,IAAI,CAAC,CAAC,4BAA4B;QAC3C,CAAC;QAED,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;YAC5B,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;YAC/C,MAAM,OAAO,GAAG,IAAI,CAAC,eAAe,CAAC,WAAW,EAAE,CAAC;YAEnD,cAAc;YACd,IAAI,QAAQ,KAAK,OAAO,EAAE,CAAC;gBACzB,OAAO,IAAI,CAAC;YACd,CAAC;YAED,wEAAwE;YACxE,2EAA2E;YAC3E,IAAI,QAAQ,CAAC,QAAQ,CAAC,GAAG,GAAG,OAAO,CAAC,EAAE,CAAC;gBACrC,OAAO,IAAI,CAAC;YACd,CAAC;YAED,OAAO,KAAK,CAAC;QACf,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,iBAAiB,CAAC,IAAU,EAAE,UAAkB;QAC5D,+CAA+C;QAC/C,uEAAuE;QACvE,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YACtB,OAAO,KAAK,CAAC;QACf,CAAC;QAED,iCAAiC;QACjC,IAAI,cAAc,CAAC,UAAU,CAAC,EAAE,CAAC;YAC/B,MAAM,CAAC,IAAI,CAAC,0DAA0D,UAAU,EAAE,CAAC,CAAC;YACpF,OAAO,IAAI,CAAC;QACd,CAAC;QAED,qBAAqB;QACrB,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,IAAI,EAAE,WAAW,IAAI,EAAE,CAAC,CAAC;YAC7E,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;YACtC,MAAM,SAAS,GAAG,eAAe,CAAC,QAAQ,GAAG,QAAQ,EAAE,UAAU,CAAC,CAAC;YAEnE,IAAI,SAAS,CAAC,WAAW,IAAI,SAAS,CAAC,UAAU,IAAI,GAAG,EAAE,CAAC;gBACzD,MAAM,CAAC,IAAI,CAAC,uEAAuE,SAAS,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;gBACvH,MAAM,CAAC,KAAK,CAAC,uCAAuC,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gBAEpF,wEAAwE;gBACxE,IAAI,CAAC,mBAAmB,GAAG,IAAI,mBAAmB,CAChD,mEAAmE,EACnE,IAAI,CAAC,WAAW,EAChB,UAAU,EACV,SAAS,CACV,CAAC;gBACF,OAAO,IAAI,CAAC;YACd,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,KAAK,CAAC,iDAAiD,EAAE,KAAK,CAAC,CAAC;QACzE,CAAC;QAED,OAAO,KAAK,CAAC;IACf,CAAC;IAED;;;;;;OAMG;IACK,KAAK,CAAC,wBAAwB,CAAC,IAAU;QAC/C,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAE9B,2FAA2F;QAC3F,wFAAwF;QACxF,IAAI,cAAc,GAAG,KAAK,CAAC;QAC3B,IAAI,CAAC;YACH,cAAc,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE;gBACxC,MAAM,WAAW,GAAG,QAAQ,CAAC,aAAa,CAAC,4BAA4B,CAAC,CAAC;gBACzE,IAAI,WAAW;oBAAE,OAAO,IAAI,CAAC;gBAC7B,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC;gBAC3B,MAAM,iBAAiB,GAAG,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,EAAE,CAAC,MAAM,IAAI,CAAC,CAAC,GAAG,GAAG,CAAC;gBAChF,MAAM,gBAAgB,GAAG,CAAC,QAAQ,CAAC,aAAa,CAAC,kDAAkD,CAAC,CAAC;gBACrG,OAAO,iBAAiB,IAAI,gBAAgB,CAAC;YAC/C,CAAC,CAAC,CAAC;QACL,CAAC;QAAC,MAAM,CAAC;YACP,yEAAyE;YACzE,cAAc,GAAG,IAAI,CAAC;QACxB,CAAC;QAED,IAAI,cAAc,EAAE,CAAC;YACnB,MAAM,CAAC,KAAK,CAAC,sDAAsD,UAAU,6BAA6B,CAAC,CAAC;YAC5G,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,UAAU,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,KAAK,UAAU,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;gBAC5E,MAAM,OAAO,CAAC,GAAG,CAAC;oBAChB,IAAI,CAAC,gBAAgB,CAAC,kBAAkB,CAAC;oBACzC,IAAI,CAAC,gBAAgB,CAAC,aAAa,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC;iBACzE,CAAC,CAAC;gBACH,MAAM,CAAC,KAAK,CAAC,wCAAwC,UAAU,MAAM,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YACrF,CAAC;YAAC,MAAM,CAAC;gBACP,MAAM,CAAC,KAAK,CAAC,oFAAoF,CAAC,CAAC;YACrG,CAAC;QACH,CAAC;QAED,qEAAqE;QACrE,IAAI,CAAC;YACH,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE;gBAC3C,MAAM,QAAQ,GAAG,QAAQ,CAAC,IAAI,EAAE,WAAW,IAAI,EAAE,CAAC;gBAClD,MAAM,gBAAgB,GACpB,QAAQ,CAAC,QAAQ,CAAC,uBAAuB,CAAC;oBAC1C,QAAQ,CAAC,QAAQ,CAAC,sBAAsB,CAAC;oBACzC,QAAQ,CAAC,QAAQ,CAAC,+BAA+B,CAAC,CAAC;gBACrD,MAAM,mBAAmB,GACvB,QAAQ,CAAC,aAAa,CAAC,6EAA6E,CAAC,KAAK,IAAI,CAAC;gBACjH,OAAO,gBAAgB,IAAI,mBAAmB,CAAC;YACjD,CAAC,CAAC,CAAC;YAEH,IAAI,WAAW,EAAE,CAAC;gBAChB,MAAM,CAAC,IAAI,CAAC,2EAA2E,CAAC,CAAC;gBACzF,IAAI,CAAC;oBACH,MAAM,IAAI,CAAC,eAAe,CACxB,GAAG,EAAE;wBACH,MAAM,QAAQ,GAAG,QAAQ,CAAC,IAAI,EAAE,WAAW,IAAI,EAAE,CAAC;wBAClD,MAAM,cAAc,GAAG,QAAQ,CAAC,QAAQ,CAAC,uBAAuB,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,sBAAsB,CAAC,CAAC;wBAC/G,MAAM,mBAAmB,GAAG,QAAQ,CAAC,aAAa,CAAC,gEAAgE,CAAC,KAAK,IAAI,CAAC;wBAC9H,OAAO,CAAC,cAAc,IAAI,CAAC,mBAAmB,CAAC;oBACjD,CAAC,EACD,EAAE,OAAO,EAAE,KAAK,EAAE,CACnB,CAAC;oBACF,MAAM,IAAI,CAAC,gBAAgB,CAAC,kBAAkB,CAAC,CAAC;oBAChD,MAAM,IAAI,CAAC,gBAAgB,CAAC,aAAa,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;oBAC9E,MAAM,CAAC,IAAI,CAAC,gDAAgD,CAAC,CAAC;gBAChE,CAAC;gBAAC,MAAM,CAAC;oBACP,MAAM,CAAC,IAAI,CAAC,sEAAsE,CAAC,CAAC;gBACtF,CAAC;YACH,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,sFAAsF;YACtF,MAAM,CAAC,KAAK,CAAC,uEAAuE,CAAC,CAAC;QACxF,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,gBAAgB,CAAC,IAAU;QACvC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,EAAE,CAAC;QACnC,MAAM,aAAa,GAAG,MAAM,OAAO,CAAC,GAAG,CACrC,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE;YACzB,IAAI,CAAC;gBACH,MAAM,UAAU,GAAG,MAAM,KAAK;qBAC3B,QAAQ,CAAC,GAAG,EAAE;oBACb,OAAO,QAAQ,CAAC,aAAa,CAAC,6DAA6D,CAAC,KAAK,IAAI,CAAC;gBACxG,CAAC,CAAC;qBACD,KAAK,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC;gBAEtB,IAAI,UAAU,EAAE,CAAC;oBACf,MAAM,OAAO,CAAC,GAAG,CAAC;wBAChB,KAAK,CAAC,gBAAgB,CAAC,kBAAkB,CAAC;wBAC1C,KAAK;6BACF,gBAAgB,CAAC,aAAa,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;6BAClD,KAAK,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,gDAAgD,CAAC,CAAC;qBAC/E,CAAC,CAAC;oBACH,OAAO,KAAK,CAAC;gBACf,CAAC;YACH,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,CAAC,KAAK,CAAC,sBAAsB,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YACjE,CAAC;YACD,OAAO,IAAI,CAAC;QACd,CAAC,CAAC,CACH,CAAC;QAEF,MAAM,KAAK,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,IAAI,IAAI,CAAC;QAC5D,IAAI,KAAK,EAAE,CAAC;YACV,MAAM,CAAC,KAAK,CAAC,yBAAyB,CAAC,CAAC;YACxC,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC;QAC5D,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IAEO,KAAK,CAAC,iBAAiB,CAAC,OAAqB,EAAE,SAA2B;QAChF,MAAM,aAAa,GAAG,SAAS,CAAC,WAAW,CAAC,QAAQ,EAAE,CAAC;QACvD,OAAO,OAAO,CAAC,QAAQ,CAAC,KAAK,EAAE,IAAY,EAAE,EAAE;YAC7C,MAAM,cAAc,GAAG,IAAI,QAAQ,CAAC,UAAU,IAAI,EAAE,CAAC,EAAE,CAAC;YACxD,MAAM,SAAS,GAAG,IAAI,cAAc,EAAE,CAAC;YACvC,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;YACxD,OAAO,MAAM,CAAC,OAAO,CAAC;QACxB,CAAC,EAAE,aAAa,CAAC,CAAC;IACpB,CAAC;IAEO,KAAK,CAAC,cAAc,CAC1B,IAAU,EACV,QAAgB,EAChB,SAA2B;QAE3B,IAAI,OAAO,GAAG,EAAE,CAAC;QACjB,IAAI,aAAa,GAAG,SAAS,CAAC,WAAW,CAAC,IAAI,CAAC;QAE/C,IAAI,CAAC;YACH,IAAI,QAAQ,KAAK,WAAW,EAAE,CAAC;gBAC7B,mBAAmB;gBACnB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC;gBAChD,IAAI,KAAK,EAAE,CAAC;oBACV,OAAO,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;gBAC3D,CAAC;gBAED,wBAAwB;gBACxB,IAAI,CAAC,OAAO,EAAE,CAAC;oBACb,OAAO,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;gBAC1D,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,OAAO,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;YAC1D,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAS,GAAG,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC;YAC7E,aAAa,GAAG,eAAe,CAAC;QAClC,CAAC;QAED,OAAO,EAAE,OAAO,EAAE,aAAa,EAAE,CAAC;IACpC,CAAC;IAED;;;OAGG;IACH,aAAa,CAAC,MAAc;QAC1B,IAAI,CAAC,UAAU,GAAG,MAAM,CAAC;QACzB,MAAM,CAAC,IAAI,CAAC,iDAAiD,MAAM,EAAE,CAAC,CAAC;IACzE,CAAC;IAED,KAAK,CAAC,CAAC,KAAK,CAAC,GAAW;QACtB,MAAM,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,wBAAwB,GAAG,EAAE,CAAC,CAAC;QAErE,6BAA6B;QAC7B,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;QACxB,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC;QAChC,IAAI,CAAC,WAAW,GAAG,GAAG,CAAC;QACvB,IAAI,CAAC,oBAAoB,GAAG,CAAC,CAAC;QAE9B,8DAA8D;QAC9D,IAAI,CAAC;YACH,IAAI,CAAC,eAAe,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;YAC7C,MAAM,CAAC,IAAI,CAAC,+DAA+D,IAAI,CAAC,eAAe,EAAE,CAAC,CAAC;QACrG,CAAC;QAAC,MAAM,CAAC;YACP,IAAI,CAAC,eAAe,GAAG,EAAE,CAAC;QAC5B,CAAC;QAED,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;QAEzD,qEAAqE;QACrE,MAAM,IAAI,CAAC,YAAY,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC;QAE7C,qDAAqD;QACrD,MAAM,cAAc,GAAG,gBAAgB,CAAC,IAAI,CAAC,YAAY,CAAC,eAAe,EAAE,IAAI,SAAS,CAAC,CAAC;QAE1F,6EAA6E;QAC7E,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACtB,MAAM,CAAC,IAAI,CAAC,qDAAqD,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE,MAAM,IAAI,CAAC,UAAU,CAAC,CAAC;YACnH,cAAc,CAAC,aAAa,GAAG;gBAC7B,GAAG,cAAc,CAAC,aAAa;gBAC/B,aAAa,EAAE;oBACb,GAAG,cAAc,CAAC,aAAa,EAAE,aAAa;iBAC/C;aACF,CAAC;YACF,cAAc,CAAC,kBAAkB,GAAG;gBAClC,GAAG,cAAc,CAAC,kBAAkB;gBACpC,cAAc,EAAE;oBACd,KAAK,EAAE,MAAM,EAAE,EAAE;wBACf,0DAA0D;wBAC1D,MAAM,CAAC,KAAK,CAAC,+CAA+C,MAAM,EAAE,CAAC,CAAC;oBACxE,CAAC;iBACF;aACF,CAAC;YACF,qCAAqC;YACrC,MAAM,aAAa,GAAG,cAAc,CAAC,kBAAkB,IAAI,EAAE,CAAC;YAC9D,cAAc,CAAC,kBAAkB,GAAG;gBAClC,GAAG,aAAa;gBAChB,KAAK,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE;oBACjB,IAAI,IAAI,CAAC,YAAY,EAAE,OAAO,EAAE,CAAC;wBAC/B,MAAM,CAAC,KAAK,CAAC,4BAA4B,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,MAAM,4BAA4B,CAAC,CAAC;wBACvG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;oBAC7D,CAAC;gBACH,CAAC;aACF,CAAC;QACJ,CAAC;QAED,IAAI,CAAC,OAAO,GAAG,IAAI,iBAAiB,CAAC;YACnC,GAAG,cAAc;YACjB,cAAc,EAAE,KAAK,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,GAAG,EAAE,EAAE,EAAE;gBAC7D,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;oBACpB,GAAG,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;oBAC3B,OAAO;gBACT,CAAC;gBAED,IAAI,CAAC;oBACH,6BAA6B;oBAC7B,MAAM,OAAO,CAAC,GAAG,CAAC;wBAChB,IAAI,CAAC,gBAAgB,CAAC,kBAAkB,CAAC;wBACzC,IAAI,CAAC,gBAAgB,CAAC,aAAa,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,0CAA0C,CAAC,CAAC;qBAC3H,CAAC,CAAC;oBAEH,sEAAsE;oBACtE,MAAM,IAAI,CAAC,wBAAwB,CAAC,IAAI,CAAC,CAAC;oBAE1C,yCAAyC;oBACzC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;oBAE7B,0DAA0D;oBAC1D,IAAI,CAAC,IAAI,CAAC,qBAAqB,CAAC,SAAS,CAAC,EAAE,CAAC;wBAC3C,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;wBACxD,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC,QAAQ,CAAC;wBAEnD,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;4BACrB,gFAAgF;4BAChF,MAAM,CAAC,IAAI,CAAC,kEAAkE,iBAAiB,MAAM,cAAc,EAAE,CAAC,CAAC;4BAEvH,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;gCACtB,mDAAmD;gCACnD,IAAI,CAAC,mBAAmB,GAAG,IAAI,mBAAmB,CAChD,4EAA4E,cAAc,GAAG,EAC7F,IAAI,CAAC,WAAW,EAChB,SAAS,EACT,EAAE,WAAW,EAAE,IAAI,EAAE,UAAU,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,mBAAmB,iBAAiB,OAAO,cAAc,EAAE,CAAC,EAAE,CAC/G,CAAC;gCACF,GAAG,CAAC,KAAK,CAAC,oDAAoD,cAAc,mBAAmB,CAAC,CAAC;gCACjG,IAAI,CAAC,KAAK,EAAE,CAAC;gCACb,OAAO;4BACT,CAAC;iCAAM,CAAC;gCACN,0DAA0D;gCAC1D,GAAG,CAAC,KAAK,CAAC,6CAA6C,cAAc,mBAAmB,CAAC,CAAC;gCAC1F,IAAI,CAAC,KAAK,EAAE,CAAC;gCACb,OAAO;4BACT,CAAC;wBACH,CAAC;6BAAM,CAAC;4BACN,sDAAsD;4BACtD,IAAI,CAAC,oBAAoB,EAAE,CAAC;4BAC5B,GAAG,CAAC,OAAO,CACT,iDAAiD,OAAO,CAAC,GAAG,MAAM,SAAS,aAAa,IAAI,CAAC,oBAAoB,kBAAkB,CACpI,CAAC;4BACF,OAAO;wBACT,CAAC;oBACH,CAAC;oBAED,gEAAgE;oBAChE,IAAI,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;wBAC1C,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;wBAClE,IAAI,WAAW,EAAE,CAAC;4BAChB,GAAG,CAAC,KAAK,CAAC,uEAAuE,CAAC,CAAC;4BACnF,IAAI,CAAC,KAAK,EAAE,CAAC;4BACb,OAAO;wBACT,CAAC;wBACD,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;oBAC3B,CAAC;yBAAM,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;wBAC5B,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;oBAC3B,CAAC;oBAED,qCAAqC;oBACrC,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;wBAC7B,IAAI,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;4BAC5B,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;gCACjB,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;4BAChC,CAAC;4BAED,MAAM,IAAI,CAAC,YAAY,CAAC,mBAAmB,CAAC,YAAY,EAAE,GAAG,EAAE,IAAI,CAAC,CAAC;4BAErE,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;4BACjC,MAAM,EAAE,OAAO,EAAE,aAAa,EAAE,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;4BAE9F,MAAM,MAAM,GAAgB;gCAC1B,GAAG,EAAE,OAAO,CAAC,GAAG;gCAChB,IAAI,EAAE,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,QAAQ,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM;gCACjE,OAAO,EAAE,YAAY,CAAC,OAAO,CAAC;gCAC9B,KAAK;gCACL,aAAa;6BACd,CAAC;4BAEF,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;4BACpC,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;4BACnC,MAAM;wBACR,CAAC;oBACH,CAAC;gBACH,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,MAAM,YAAY,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;oBAC5E,GAAG,CAAC,KAAK,CAAC,oBAAoB,OAAO,CAAC,GAAG,KAAK,YAAY,EAAE,CAAC,CAAC;gBAChE,CAAC;YACH,CAAC;SACF,CAAC,CAAC;QAEH,IAAI,CAAC;YACH,MAAM,cAAc,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC;YAE1C,OAAO,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC;gBACxB,IAAI,IAAI,CAAC,YAAY,CAAC,gBAAgB,EAAE,EAAE,CAAC;oBACzC,KAAK,MAAM,MAAM,IAAI,MAAM,IAAI,CAAC,YAAY,CAAC,YAAY,EAAE,EAAE,CAAC;wBAC5D,MAAM,MAAM,CAAC;oBACf,CAAC;gBACH,CAAC;gBAED,IAAI,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,cAAc,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,EAAE,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;oBAAE,MAAM;YACpI,CAAC;YAED,MAAM,cAAc,CAAC;YACrB,MAAM,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC;YAEjC,4CAA4C;YAC5C,IAAI,IAAI,CAAC,oBAAoB,GAAG,CAAC,EAAE,CAAC;gBAClC,MAAM,CAAC,IAAI,CACT,4BAA4B,IAAI,CAAC,oBAAoB,sDAAsD,IAAI,CAAC,eAAe,GAAG,CACnI,CAAC;YACJ,CAAC;YAED,0DAA0D;YAC1D,IAAI,IAAI,CAAC,mBAAmB,EAAE,CAAC;gBAC7B,MAAM,IAAI,CAAC,mBAAmB,CAAC;YACjC,CAAC;YAED,KAAK,MAAM,MAAM,IAAI,MAAM,IAAI,CAAC,YAAY,CAAC,YAAY,EAAE,EAAE,CAAC;gBAC5D,MAAM,MAAM,CAAC;YACf,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,wCAAwC;YACxC,IAAI,KAAK,YAAY,mBAAmB,EAAE,CAAC;gBACzC,MAAM,KAAK,CAAC;YACd,CAAC;YACD,MAAM,CAAC,KAAK,CAAC,gBAAgB,EAAE,KAAK,CAAC,CAAC;YACtC,MAAM,KAAK,CAAC;QACd,CAAC;gBAAS,CAAC;YACT,MAAM,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE,CAAC;YAClC,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACtB,CAAC;IACH,CAAC;IAED,KAAK;QACH,KAAK,CAAC,KAAK,EAAE,CAAC;QACd,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,6BAA6B,EAAE,GAAG,CAAC,CAAC,CAAC;QAC3F,CAAC;IACH,CAAC;CACF"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"crawlee-crawler.test.js","sourceRoot":"","sources":["../../src/crawler/crawlee-crawler.test.ts"],"names":[],"mappings":"AAEA,MAAM,gBAAgB,GAAG;IACvB,UAAU,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,SAAS,CAAC;IAChD,eAAe,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC,EAAE,CAAC;IAC5C,mBAAmB,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,SAAS,CAAC;IACzD,SAAS,EAAE,EAAE,CAAC,EAAE,EAAE;IAClB,gBAAgB,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC,KAAK,CAAC;IAChD,YAAY,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,EAAE,CAAC;IAC3C,OAAO,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,SAAS,CAAC;CAC9C,CAAC;AAEF,EAAE,CAAC,IAAI,CAAC,oBAAoB,EAAE,GAAG,EAAE,CAAC,CAAC;IACnC,YAAY,EAAE;QACZ,OAAO,gBAAgB,CAAC;IAC1B,CAAC;CACF,CAAC,CAAC,CAAC;AAEJ,EAAE,CAAC,IAAI,CAAC,qBAAqB,EAAE,GAAG,EAAE,CAAC,CAAC;IACpC,gBAAgB,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC;QACxC,YAAY,EAAE,EAAE;QAChB,kBAAkB,EAAE,EAAE;QACtB,aAAa,EAAE,EAAE;QACjB,kBAAkB,EAAE,EAAE;KACvB,CAAC;CACH,CAAC,CAAC,CAAC;AAEJ,EAAE,CAAC,IAAI,CAAC,iBAAiB,EAAE,GAAG,EAAE,CAAC,CAAC;IAChC,SAAS,EAAE;QACT;YACE,IAAI,EAAE,SAAS;YACf,SAAS,EAAE;gBACT,cAAc,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,EAAE,OAAO,EAAE,mBAAmB,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC;aAC1F;YACD,MAAM,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,IAAI,CAAC;SACxC;KACF;CACF,CAAC,CAAC,CAAC;AAEJ,yBAAyB;AACzB,MAAM,cAAc,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;AAC5D,MAAM,mBAAmB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;AAEjE,EAAE,CAAC,IAAI,CAAC,SAAS,EAAE,GAAG,EAAE,CAAC,CAAC;IACxB,iBAAiB,EAAE,UAAU,OAAqC;QAChE,wCAAwC;QACvC,MAAyC,CAAC,gBAAgB,GAAG,OAAO,CAAC,cAAc,CAAC;QACrF,OAAO;YACL,GAAG,EAAE,cAAc;YACnB,QAAQ,EAAE,mBAAmB;SAC9B,CAAC;IACJ,CAAC;CACF,CAAC,CAAC,CAAC;AAEJ,uBAAuB;AACvB,OAAO,EAAE,cAAc,EAAgB,MAAM,sBAAsB,CAAC;AAEpE,QAAQ,CAAC,gBAAgB,EAAE,GAAG,EAAE;IAC9B,IAAI,OAAuB,CAAC;IAE5B,UAAU,CAAC,GAAG,EAAE;QACd,EAAE,CAAC,aAAa,EAAE,CAAC;QACnB,OAAO,GAAG,IAAI,cAAc,EAAE,CAAC;QAC/B,gBAAgB,CAAC,gBAAgB,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;QACzD,gBAAgB,CAAC,YAAY,CAAC,iBAAiB,CAAC,EAAE,CAAC,CAAC;IACtD,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,aAAa,EAAE,GAAG,EAAE;QAC3B,EAAE,CAAC,uCAAuC,EAAE,GAAG,EAAE;YAC/C,MAAM,CAAC,OAAO,CAAC,CAAC,WAAW,EAAE,CAAC;QAChC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uDAAuD,EAAE,GAAG,EAAE;YAC/D,MAAM,aAAa,GAAG,IAAI,cAAc,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;YAClD,MAAM,CAAC,aAAa,CAAC,CAAC,WAAW,EAAE,CAAC;QACtC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,iCAAiC,EAAE,GAAG,EAAE;YACzC,MAAM,UAAU,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3B,MAAM,eAAe,GAAG,IAAI,cAAc,CAAC,CAAC,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC;YAChE,MAAM,CAAC,eAAe,CAAC,CAAC,WAAW,EAAE,CAAC;QACxC,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,iBAAiB,EAAE,GAAG,EAAE;QAC/B,EAAE,CAAC,6BAA6B,EAAE,GAAG,EAAE;YACrC,MAAM,KAAK,GAAiB;gBAC1B,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,aAAa,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC;aAClF,CAAC;YAEF,OAAO,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;YAE/B,yBAAyB;YACzB,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC1B,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,0CAA0C,EAAE,GAAG,EAAE;YAClD,MAAM,KAAK,GAAiB;gBAC1B,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,aAAa,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC;gBACjF,OAAO,EAAE;oBACP;wBACE,MAAM,EAAE,qBAAqB;wBAC7B,YAAY,EAAE,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;qBAChD;iBACF;aACF,CAAC;YAEF,OAAO,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;YAC/B,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC1B,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,OAAO,EAAE,GAAG,EAAE;QACrB,EAAE,CAAC,0CAA0C,EAAE,KAAK,IAAI,EAAE;YACxD,qEAAqE;YACrE,cAAc,CAAC,qBAAqB,CAAC,SAAS,CAAC,CAAC;YAChD,gBAAgB,CAAC,YAAY,CAAC,qBAAqB,CAAC,EAAE,CAAC,CAAC;YAExD,MAAM,OAAO,GAAkB,EAAE,CAAC;YAClC,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,OAAO,CAAC,KAAK,CAAC,0BAA0B,CAAC,EAAE,CAAC;gBACrE,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;YAED,MAAM,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,oBAAoB,CAAC,0BAA0B,EAAE,SAAS,CAAC,CAAC;QAClG,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yCAAyC,EAAE,KAAK,IAAI,EAAE;YACvD,MAAM,WAAW,GAAkB;gBACjC,EAAE,GAAG,EAAE,2BAA2B,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE;gBACxF,EAAE,GAAG,EAAE,2BAA2B,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE;aACzF,CAAC;YAEF,yEAAyE;YACzE,uFAAuF;YACvF,gBAAgB,CAAC,YAAY,CAAC,qBAAqB,CAAC,WAAW,CAAC,CAAC;YAEjE,MAAM,OAAO,GAAkB,EAAE,CAAC;YAClC,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,OAAO,CAAC,KAAK,CAAC,qBAAqB,CAAC,EAAE,CAAC;gBAChE,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;YAED,MAAM,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;QACvC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,0CAA0C,EAAE,KAAK,IAAI,EAAE;YACxD,6DAA6D;YAC7D,IAAI,KAAK,EAAE,MAAM,CAAC,IAAI,OAAO,CAAC,KAAK,CAAC,qBAAqB,CAAC,EAAE,CAAC;gBAC3D,uBAAuB;YACzB,CAAC;YAED,MAAM,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC,gBAAgB,EAAE,CAAC;QACtD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,sDAAsD,EAAE,KAAK,IAAI,EAAE;YACpE,MAAM,WAAW,GAAkB,CAAC,EAAE,GAAG,EAAE,2BAA2B,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC;YAE9H,gBAAgB,CAAC,gBAAgB,CAAC,mBAAmB,CAAC,IAAI,CAAC,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;YACnF,gBAAgB,CAAC,YAAY,CAAC,qBAAqB,CAAC,WAAW,CAAC,CAAC,qBAAqB,CAAC,EAAE,CAAC,CAAC;YAE3F,MAAM,OAAO,GAAkB,EAAE,CAAC;YAClC,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,OAAO,CAAC,KAAK,CAAC,qBAAqB,CAAC,EAAE,CAAC;gBAChE,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;YAED,MAAM,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAClC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,sCAAsC,EAAE,KAAK,IAAI,EAAE;YACpD,6DAA6D;YAC7D,IAAI,KAAK,EAAE,MAAM,CAAC,IAAI,OAAO,CAAC,KAAK,CAAC,gCAAgC,CAAC,EAAE,CAAC;gBACtE,uBAAuB;YACzB,CAAC;YAED,qCAAqC;YACrC,MAAM,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,oBAAoB,CAAC,gCAAgC,EAAE,SAAS,CAAC,CAAC;QACxG,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,OAAO,EAAE,GAAG,EAAE;QACrB,EAAE,CAAC,yBAAyB,EAAE,KAAK,IAAI,EAAE;YACvC,qFAAqF;YACrF,IAAI,UAAsB,CAAC;YAC3B,MAAM,UAAU,GAAG,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;gBAC/C,UAAU,GAAG,OAAO,CAAC;YACvB,CAAC,CAAC,CAAC;YACH,cAAc,CAAC,mBAAmB,CAAC,UAAU,CAAC,CAAC;YAE/C,qCAAqC;YACrC,MAAM,gBAAgB,GAAG,IAAI,cAAc,EAAE,CAAC;YAE9C,2DAA2D;YAC3D,MAAM,SAAS,GAAG,gBAAgB,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC;YAEhE,wEAAwE;YACxE,MAAM,kBAAkB,GAAG,SAAS,CAAC,IAAI,EAAE,CAAC;YAE5C,0DAA0D;YAC1D,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;YAExD,+CAA+C;YAC/C,gBAAgB,CAAC,KAAK,EAAE,CAAC;YAEzB,qDAAqD;YACrD,UAAW,EAAE,CAAC;YAEd,qCAAqC;YACrC,MAAM,kBAAkB,CAAC;YACzB,MAAM,OAAO,GAAkB,EAAE,CAAC;YAClC,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,SAAS,EAAE,CAAC;gBACrC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;YAED,MAAM,CAAC,mBAAmB,CAAC,CAAC,gBAAgB,EAAE,CAAC;QACjD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,oBAAoB,EAAE,GAAG,EAAE;QAClC,EAAE,CAAC,yDAAyD,EAAE,KAAK,IAAI,EAAE;YACvE,6DAA6D;YAC7D,IAAI,KAAK,EAAE,MAAM,CAAC,IAAI,OAAO,CAAC,KAAK,CAAC,oCAAoC,CAAC,EAAE,CAAC;gBAC1E,uBAAuB;YACzB,CAAC;YAED,MAAM,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,oBAAoB,CAAC,oCAAoC,EAAE,SAAS,CAAC,CAAC;QAC5G,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,mDAAmD,EAAE,KAAK,IAAI,EAAE;YACjE,OAAO,CAAC,aAAa,CAAC,WAAW,CAAC,CAAC;YAEnC,6DAA6D;YAC7D,IAAI,KAAK,EAAE,MAAM,CAAC,IAAI,OAAO,CAAC,KAAK,CAAC,8BAA8B,CAAC,EAAE,CAAC;gBACpE,uBAAuB;YACzB,CAAC;YAED,MAAM,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,oBAAoB,CAAC,8BAA8B,EAAE,WAAW,CAAC,CAAC;QACxG,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,gBAAgB,EAAE,GAAG,EAAE;QAC9B,EAAE,CAAC,sDAAsD,EAAE,KAAK,IAAI,EAAE;YACpE,MAAM,KAAK,GAAiB;gBAC1B,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,EAAE,aAAa,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC;aACjF,CAAC;YAEF,OAAO,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;YAE/B,6DAA6D;YAC7D,IAAI,KAAK,EAAE,MAAM,CAAC,IAAI,OAAO,CAAC,KAAK,CAAC,qBAAqB,CAAC,EAAE,CAAC;gBAC3D,uBAAuB;YACzB,CAAC;YAED,oEAAoE;YACpE,MAAM,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,gBAAgB,EAAE,CAAC;QACzD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,uBAAuB,EAAE,GAAG,EAAE;QACrC,oEAAoE;QACpE,EAAE,CAAC,4CAA4C,EAAE,KAAK,IAAI,EAAE;YAC1D,qDAAqD;YACrD,6DAA6D;YAC7D,IAAI,KAAK,EAAE,MAAM,CAAC,IAAI,OAAO,CAAC,KAAK,CAAC,qBAAqB,CAAC,EAAE,CAAC;gBAC3D,uBAAuB;YACzB,CAAC;YAED,wCAAwC;YACxC,MAAM,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,gBAAgB,EAAE,CAAC;QACzD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,gBAAgB,EAAE,GAAG,EAAE;QAC9B,EAAE,CAAC,yBAAyB,EAAE,KAAK,IAAI,EAAE;YACvC,cAAc,CAAC,qBAAqB,CAAC,IAAI,KAAK,CAAC,cAAc,CAAC,CAAC,CAAC;YAEhE,MAAM,MAAM,CAAC,KAAK,IAAI,EAAE;gBACtB,6DAA6D;gBAC7D,IAAI,KAAK,EAAE,MAAM,CAAC,IAAI,OAAO,CAAC,KAAK,CAAC,qBAAqB,CAAC,EAAE,CAAC;oBAC3D,uBAAuB;gBACzB,CAAC;YACH,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;YAEnC,MAAM,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC,gBAAgB,EAAE,CAAC;QACtD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uCAAuC,EAAE,KAAK,IAAI,EAAE;YACrD,4CAA4C;YAC5C,6DAA6D;YAC7D,IAAI,KAAK,EAAE,MAAM,CAAC,IAAI,OAAO,CAAC,KAAK,CAAC,qBAAqB,CAAC,EAAE,CAAC;gBAC3D,uBAAuB;YACzB,CAAC;YAED,MAAM,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,gBAAgB,EAAE,CAAC;QACzD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,wBAAwB,EAAE,GAAG,EAAE;IACtC,EAAE,CAAC,mDAAmD,EAAE,GAAG,EAAE;QAC3D,MAAM,KAAK,GAAiB;YAC1B,OAAO,EAAE;gBACP;oBACE,IAAI,EAAE,SAAS;oBACf,KAAK,EAAE,KAAK;oBACZ,MAAM,EAAE,aAAa;oBACrB,IAAI,EAAE,GAAG;oBACT,OAAO,EAAE,UAAU;oBACnB,QAAQ,EAAE,IAAI;oBACd,MAAM,EAAE,IAAI;oBACZ,QAAQ,EAAE,QAAQ;iBACnB;aACF;SACF,CAAC;QAEF,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACtC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACnD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,wCAAwC,EAAE,GAAG,EAAE;QAChD,MAAM,KAAK,GAAiB;YAC1B,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,UAAU,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC;SAC1E,CAAC;QAEF,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,aAAa,EAAE,CAAC;IACnD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,uCAAuC,EAAE,GAAG,EAAE;QAC/C,MAAM,KAAK,GAAiB;YAC1B,OAAO,EAAE,EAAE;YACX,OAAO,EAAE;gBACP;oBACE,MAAM,EAAE,qBAAqB;oBAC7B,YAAY,EAAE;wBACZ,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE;wBACjC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE;qBAClC;iBACF;aACF;SACF,CAAC;QAEF,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACtC,MAAM,CAAC,KAAK,CAAC,OAAQ,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;IACzD,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
1
|
+
{"version":3,"file":"crawlee-crawler.test.js","sourceRoot":"","sources":["../../src/crawler/crawlee-crawler.test.ts"],"names":[],"mappings":"AAEA,MAAM,gBAAgB,GAAG;IACvB,UAAU,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,SAAS,CAAC;IAChD,eAAe,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,CAAC,CAAC;IAC7C,eAAe,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC,EAAE,CAAC;IAC5C,mBAAmB,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,SAAS,CAAC;IACzD,SAAS,EAAE,EAAE,CAAC,EAAE,EAAE;IAClB,gBAAgB,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC,KAAK,CAAC;IAChD,YAAY,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,EAAE,CAAC;IAC3C,OAAO,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,SAAS,CAAC;CAC9C,CAAC;AAEF,EAAE,CAAC,IAAI,CAAC,oBAAoB,EAAE,GAAG,EAAE,CAAC,CAAC;IACnC,YAAY,EAAE;QACZ,OAAO,gBAAgB,CAAC;IAC1B,CAAC;CACF,CAAC,CAAC,CAAC;AAEJ,EAAE,CAAC,IAAI,CAAC,qBAAqB,EAAE,GAAG,EAAE,CAAC,CAAC;IACpC,gBAAgB,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC;QACxC,YAAY,EAAE,EAAE;QAChB,kBAAkB,EAAE,EAAE;QACtB,aAAa,EAAE,EAAE;QACjB,kBAAkB,EAAE,EAAE;KACvB,CAAC;CACH,CAAC,CAAC,CAAC;AAEJ,EAAE,CAAC,IAAI,CAAC,iBAAiB,EAAE,GAAG,EAAE,CAAC,CAAC;IAChC,SAAS,EAAE;QACT;YACE,IAAI,EAAE,SAAS;YACf,SAAS,EAAE;gBACT,cAAc,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,EAAE,OAAO,EAAE,mBAAmB,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC;aAC1F;YACD,MAAM,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,IAAI,CAAC;SACxC;KACF;CACF,CAAC,CAAC,CAAC;AAEJ,yBAAyB;AACzB,MAAM,cAAc,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;AAC5D,MAAM,mBAAmB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;AAEjE,EAAE,CAAC,IAAI,CAAC,SAAS,EAAE,GAAG,EAAE,CAAC,CAAC;IACxB,iBAAiB,EAAE,UAAU,OAAqC;QAChE,wCAAwC;QACvC,MAAyC,CAAC,gBAAgB,GAAG,OAAO,CAAC,cAAc,CAAC;QACrF,OAAO;YACL,GAAG,EAAE,cAAc;YACnB,QAAQ,EAAE,mBAAmB;SAC9B,CAAC;IACJ,CAAC;CACF,CAAC,CAAC,CAAC;AAEJ,uBAAuB;AACvB,OAAO,EAAE,cAAc,EAAgB,MAAM,sBAAsB,CAAC;AAEpE,QAAQ,CAAC,gBAAgB,EAAE,GAAG,EAAE;IAC9B,IAAI,OAAuB,CAAC;IAE5B,UAAU,CAAC,GAAG,EAAE;QACd,EAAE,CAAC,aAAa,EAAE,CAAC;QACnB,OAAO,GAAG,IAAI,cAAc,EAAE,CAAC;QAC/B,gBAAgB,CAAC,gBAAgB,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;QACzD,gBAAgB,CAAC,YAAY,CAAC,iBAAiB,CAAC,EAAE,CAAC,CAAC;IACtD,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,aAAa,EAAE,GAAG,EAAE;QAC3B,EAAE,CAAC,uCAAuC,EAAE,GAAG,EAAE;YAC/C,MAAM,CAAC,OAAO,CAAC,CAAC,WAAW,EAAE,CAAC;QAChC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uDAAuD,EAAE,GAAG,EAAE;YAC/D,MAAM,aAAa,GAAG,IAAI,cAAc,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;YAClD,MAAM,CAAC,aAAa,CAAC,CAAC,WAAW,EAAE,CAAC;QACtC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,iCAAiC,EAAE,GAAG,EAAE;YACzC,MAAM,UAAU,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3B,MAAM,eAAe,GAAG,IAAI,cAAc,CAAC,CAAC,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC;YAChE,MAAM,CAAC,eAAe,CAAC,CAAC,WAAW,EAAE,CAAC;QACxC,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,iBAAiB,EAAE,GAAG,EAAE;QAC/B,EAAE,CAAC,6BAA6B,EAAE,GAAG,EAAE;YACrC,MAAM,KAAK,GAAiB;gBAC1B,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,aAAa,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC;aAClF,CAAC;YAEF,OAAO,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;YAE/B,yBAAyB;YACzB,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC1B,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,0CAA0C,EAAE,GAAG,EAAE;YAClD,MAAM,KAAK,GAAiB;gBAC1B,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,aAAa,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC;gBACjF,OAAO,EAAE;oBACP;wBACE,MAAM,EAAE,qBAAqB;wBAC7B,YAAY,EAAE,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;qBAChD;iBACF;aACF,CAAC;YAEF,OAAO,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;YAC/B,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC1B,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,OAAO,EAAE,GAAG,EAAE;QACrB,EAAE,CAAC,0CAA0C,EAAE,KAAK,IAAI,EAAE;YACxD,qEAAqE;YACrE,cAAc,CAAC,qBAAqB,CAAC,SAAS,CAAC,CAAC;YAChD,gBAAgB,CAAC,YAAY,CAAC,qBAAqB,CAAC,EAAE,CAAC,CAAC;YAExD,MAAM,OAAO,GAAkB,EAAE,CAAC;YAClC,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,OAAO,CAAC,KAAK,CAAC,0BAA0B,CAAC,EAAE,CAAC;gBACrE,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;YAED,MAAM,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,oBAAoB,CAAC,0BAA0B,EAAE,SAAS,CAAC,CAAC;QAClG,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yCAAyC,EAAE,KAAK,IAAI,EAAE;YACvD,MAAM,WAAW,GAAkB;gBACjC,EAAE,GAAG,EAAE,2BAA2B,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE;gBACxF,EAAE,GAAG,EAAE,2BAA2B,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE;aACzF,CAAC;YAEF,yEAAyE;YACzE,uFAAuF;YACvF,gBAAgB,CAAC,YAAY,CAAC,qBAAqB,CAAC,WAAW,CAAC,CAAC;YAEjE,MAAM,OAAO,GAAkB,EAAE,CAAC;YAClC,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,OAAO,CAAC,KAAK,CAAC,qBAAqB,CAAC,EAAE,CAAC;gBAChE,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;YAED,MAAM,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;QACvC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,0CAA0C,EAAE,KAAK,IAAI,EAAE;YACxD,6DAA6D;YAC7D,IAAI,KAAK,EAAE,MAAM,CAAC,IAAI,OAAO,CAAC,KAAK,CAAC,qBAAqB,CAAC,EAAE,CAAC;gBAC3D,uBAAuB;YACzB,CAAC;YAED,MAAM,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC,gBAAgB,EAAE,CAAC;QACtD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,sDAAsD,EAAE,KAAK,IAAI,EAAE;YACpE,MAAM,WAAW,GAAkB,CAAC,EAAE,GAAG,EAAE,2BAA2B,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC;YAE9H,gBAAgB,CAAC,gBAAgB,CAAC,mBAAmB,CAAC,IAAI,CAAC,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;YACnF,gBAAgB,CAAC,YAAY,CAAC,qBAAqB,CAAC,WAAW,CAAC,CAAC,qBAAqB,CAAC,EAAE,CAAC,CAAC;YAE3F,MAAM,OAAO,GAAkB,EAAE,CAAC;YAClC,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,OAAO,CAAC,KAAK,CAAC,qBAAqB,CAAC,EAAE,CAAC;gBAChE,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;YAED,MAAM,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAClC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,sCAAsC,EAAE,KAAK,IAAI,EAAE;YACpD,6DAA6D;YAC7D,IAAI,KAAK,EAAE,MAAM,CAAC,IAAI,OAAO,CAAC,KAAK,CAAC,gCAAgC,CAAC,EAAE,CAAC;gBACtE,uBAAuB;YACzB,CAAC;YAED,qCAAqC;YACrC,MAAM,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,oBAAoB,CAAC,gCAAgC,EAAE,SAAS,CAAC,CAAC;QACxG,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,OAAO,EAAE,GAAG,EAAE;QACrB,EAAE,CAAC,yBAAyB,EAAE,KAAK,IAAI,EAAE;YACvC,qFAAqF;YACrF,IAAI,UAAsB,CAAC;YAC3B,MAAM,UAAU,GAAG,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;gBAC/C,UAAU,GAAG,OAAO,CAAC;YACvB,CAAC,CAAC,CAAC;YACH,cAAc,CAAC,mBAAmB,CAAC,UAAU,CAAC,CAAC;YAE/C,qCAAqC;YACrC,MAAM,gBAAgB,GAAG,IAAI,cAAc,EAAE,CAAC;YAE9C,2DAA2D;YAC3D,MAAM,SAAS,GAAG,gBAAgB,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC;YAEhE,wEAAwE;YACxE,MAAM,kBAAkB,GAAG,SAAS,CAAC,IAAI,EAAE,CAAC;YAE5C,0DAA0D;YAC1D,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;YAExD,+CAA+C;YAC/C,gBAAgB,CAAC,KAAK,EAAE,CAAC;YAEzB,qDAAqD;YACrD,UAAW,EAAE,CAAC;YAEd,qCAAqC;YACrC,MAAM,kBAAkB,CAAC;YACzB,MAAM,OAAO,GAAkB,EAAE,CAAC;YAClC,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,SAAS,EAAE,CAAC;gBACrC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;YAED,MAAM,CAAC,mBAAmB,CAAC,CAAC,gBAAgB,EAAE,CAAC;QACjD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,oBAAoB,EAAE,GAAG,EAAE;QAClC,EAAE,CAAC,yDAAyD,EAAE,KAAK,IAAI,EAAE;YACvE,6DAA6D;YAC7D,IAAI,KAAK,EAAE,MAAM,CAAC,IAAI,OAAO,CAAC,KAAK,CAAC,oCAAoC,CAAC,EAAE,CAAC;gBAC1E,uBAAuB;YACzB,CAAC;YAED,MAAM,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,oBAAoB,CAAC,oCAAoC,EAAE,SAAS,CAAC,CAAC;QAC5G,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,mDAAmD,EAAE,KAAK,IAAI,EAAE;YACjE,OAAO,CAAC,aAAa,CAAC,WAAW,CAAC,CAAC;YAEnC,6DAA6D;YAC7D,IAAI,KAAK,EAAE,MAAM,CAAC,IAAI,OAAO,CAAC,KAAK,CAAC,8BAA8B,CAAC,EAAE,CAAC;gBACpE,uBAAuB;YACzB,CAAC;YAED,MAAM,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,oBAAoB,CAAC,8BAA8B,EAAE,WAAW,CAAC,CAAC;QACxG,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,gBAAgB,EAAE,GAAG,EAAE;QAC9B,EAAE,CAAC,sDAAsD,EAAE,KAAK,IAAI,EAAE;YACpE,MAAM,KAAK,GAAiB;gBAC1B,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,EAAE,aAAa,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC;aACjF,CAAC;YAEF,OAAO,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;YAE/B,6DAA6D;YAC7D,IAAI,KAAK,EAAE,MAAM,CAAC,IAAI,OAAO,CAAC,KAAK,CAAC,qBAAqB,CAAC,EAAE,CAAC;gBAC3D,uBAAuB;YACzB,CAAC;YAED,oEAAoE;YACpE,MAAM,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,gBAAgB,EAAE,CAAC;QACzD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,uBAAuB,EAAE,GAAG,EAAE;QACrC,oEAAoE;QACpE,EAAE,CAAC,4CAA4C,EAAE,KAAK,IAAI,EAAE;YAC1D,qDAAqD;YACrD,6DAA6D;YAC7D,IAAI,KAAK,EAAE,MAAM,CAAC,IAAI,OAAO,CAAC,KAAK,CAAC,qBAAqB,CAAC,EAAE,CAAC;gBAC3D,uBAAuB;YACzB,CAAC;YAED,wCAAwC;YACxC,MAAM,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,gBAAgB,EAAE,CAAC;QACzD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,gBAAgB,EAAE,GAAG,EAAE;QAC9B,EAAE,CAAC,yBAAyB,EAAE,KAAK,IAAI,EAAE;YACvC,cAAc,CAAC,qBAAqB,CAAC,IAAI,KAAK,CAAC,cAAc,CAAC,CAAC,CAAC;YAEhE,MAAM,MAAM,CAAC,KAAK,IAAI,EAAE;gBACtB,6DAA6D;gBAC7D,IAAI,KAAK,EAAE,MAAM,CAAC,IAAI,OAAO,CAAC,KAAK,CAAC,qBAAqB,CAAC,EAAE,CAAC;oBAC3D,uBAAuB;gBACzB,CAAC;YACH,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;YAEnC,MAAM,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC,gBAAgB,EAAE,CAAC;QACtD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uCAAuC,EAAE,KAAK,IAAI,EAAE;YACrD,4CAA4C;YAC5C,6DAA6D;YAC7D,IAAI,KAAK,EAAE,MAAM,CAAC,IAAI,OAAO,CAAC,KAAK,CAAC,qBAAqB,CAAC,EAAE,CAAC;gBAC3D,uBAAuB;YACzB,CAAC;YAED,MAAM,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,gBAAgB,EAAE,CAAC;QACzD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,wBAAwB,EAAE,GAAG,EAAE;IACtC,EAAE,CAAC,mDAAmD,EAAE,GAAG,EAAE;QAC3D,MAAM,KAAK,GAAiB;YAC1B,OAAO,EAAE;gBACP;oBACE,IAAI,EAAE,SAAS;oBACf,KAAK,EAAE,KAAK;oBACZ,MAAM,EAAE,aAAa;oBACrB,IAAI,EAAE,GAAG;oBACT,OAAO,EAAE,UAAU;oBACnB,QAAQ,EAAE,IAAI;oBACd,MAAM,EAAE,IAAI;oBACZ,QAAQ,EAAE,QAAQ;iBACnB;aACF;SACF,CAAC;QAEF,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACtC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACnD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,wCAAwC,EAAE,GAAG,EAAE;QAChD,MAAM,KAAK,GAAiB;YAC1B,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,UAAU,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC;SAC1E,CAAC;QAEF,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,aAAa,EAAE,CAAC;IACnD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,uCAAuC,EAAE,GAAG,EAAE;QAC/C,MAAM,KAAK,GAAiB;YAC1B,OAAO,EAAE,EAAE;YACX,OAAO,EAAE;gBACP;oBACE,MAAM,EAAE,qBAAqB;oBAC7B,YAAY,EAAE;wBACZ,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE;wBACjC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE;qBAClC;iBACF;aACF;SACF,CAAC;QAEF,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACtC,MAAM,CAAC,KAAK,CAAC,OAAQ,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;IACzD,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
@@ -8,7 +8,6 @@ export class DocsCrawler {
|
|
|
8
8
|
githubToken;
|
|
9
9
|
onProgress;
|
|
10
10
|
GITHUB_HOST = 'github.com';
|
|
11
|
-
MIN_PAGES = 2; // Require at least 2 pages for component libraries
|
|
12
11
|
isAborting = false;
|
|
13
12
|
storageState;
|
|
14
13
|
pathPrefix;
|
|
@@ -78,12 +77,13 @@ export class DocsCrawler {
|
|
|
78
77
|
pageCount++;
|
|
79
78
|
yield page;
|
|
80
79
|
}
|
|
81
|
-
if (pageCount
|
|
82
|
-
logger.
|
|
83
|
-
return 'crawlee';
|
|
80
|
+
if (pageCount === 0) {
|
|
81
|
+
logger.warn(`[DocsCrawler] Crawlee crawler found no pages for ${url}`);
|
|
84
82
|
}
|
|
85
|
-
|
|
86
|
-
|
|
83
|
+
else {
|
|
84
|
+
logger.debug(`[DocsCrawler] Crawlee crawler completed (${pageCount} pages)`);
|
|
85
|
+
}
|
|
86
|
+
return 'crawlee';
|
|
87
87
|
}
|
|
88
88
|
catch (e) {
|
|
89
89
|
logger.debug('[DocsCrawler] Crawlee crawler failed:', e);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"docs-crawler.js","sourceRoot":"","sources":["../../src/crawler/docs-crawler.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,KAAK,CAAC;AAE1B,OAAO,EAAE,cAAc,EAAgB,MAAM,sBAAsB,CAAC;AACpE,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAE3C,MAAM,OAAO,WAAW;
|
|
1
|
+
{"version":3,"file":"docs-crawler.js","sourceRoot":"","sources":["../../src/crawler/docs-crawler.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,KAAK,CAAC;AAE1B,OAAO,EAAE,cAAc,EAAgB,MAAM,sBAAsB,CAAC;AACpE,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAE3C,MAAM,OAAO,WAAW;IAOH;IACA;IACA;IACA;IATF,WAAW,GAAG,YAAY,CAAC;IACpC,UAAU,GAAG,KAAK,CAAC;IACnB,YAAY,CAAgB;IAC5B,UAAU,CAAU;IAE5B,YACmB,WAAmB,CAAC,EACpB,sBAA8B,IAAI,EAClC,WAAoB,EACpB,UAA4D;QAH5D,aAAQ,GAAR,QAAQ,CAAY;QACpB,wBAAmB,GAAnB,mBAAmB,CAAe;QAClC,gBAAW,GAAX,WAAW,CAAS;QACpB,eAAU,GAAV,UAAU,CAAkD;IAC5E,CAAC;IAEJ;;;;OAIG;IACH,aAAa,CAAC,MAAc;QAC1B,IAAI,CAAC,UAAU,GAAG,MAAM,CAAC;QACzB,MAAM,CAAC,IAAI,CAAC,8CAA8C,MAAM,EAAE,CAAC,CAAC;IACtE,CAAC;IAED;;OAEG;IACH,eAAe,CAAC,KAAmB;QACjC,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC;QAC1B,MAAM,CAAC,IAAI,CAAC,wCAAwC,KAAK,CAAC,OAAO,EAAE,MAAM,IAAI,CAAC,UAAU,CAAC,CAAC;IAC5F,CAAC;IAED,KAAK,CAAC,CAAC,KAAK,CAAC,GAAW;QACtB,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAC9B,MAAM,CAAC,KAAK,CAAC,mCAAmC,QAAQ,EAAE,CAAC,CAAC;QAE5D,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACpB,MAAM,CAAC,KAAK,CAAC,6BAA6B,CAAC,CAAC;YAC5C,OAAO,SAAS,CAAC;QACnB,CAAC;QAED,6BAA6B;QAC7B,IAAI,QAAQ,CAAC,IAAI,KAAK,IAAI,CAAC,WAAW,EAAE,CAAC;YACvC,MAAM,CAAC,KAAK,CAAC,0CAA0C,CAAC,CAAC;YACzD,MAAM,aAAa,GAAG,IAAI,aAAa,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,mBAAmB,EAAE,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;YAEpH,IAAI,CAAC;gBACH,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,aAAa,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;oBAClD,IAAI,IAAI,CAAC,UAAU;wBAAE,MAAM;oBAC3B,MAAM,IAAI,CAAC;gBACb,CAAC;gBACD,OAAO,QAAQ,CAAC;YAClB,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,MAAM,CAAC,KAAK,CAAC,sCAAsC,EAAE,CAAC,CAAC,CAAC;gBACxD,uDAAuD;gBACvD,MAAM,CAAC,CAAC;YACV,CAAC;QACH,CAAC;QAED,kCAAkC;QAClC,MAAM,CAAC,KAAK,CAAC,qCAAqC,CAAC,CAAC;QACpD,MAAM,cAAc,GAAG,IAAI,cAAc,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,mBAAmB,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;QAEpG,mCAAmC;QACnC,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACtB,cAAc,CAAC,eAAe,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QACpD,CAAC;QAED,6CAA6C;QAC7C,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACpB,cAAc,CAAC,aAAa,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAChD,CAAC;QAED,IAAI,SAAS,GAAG,CAAC,CAAC;QAElB,IAAI,CAAC;YACH,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,cAAc,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;gBACnD,IAAI,IAAI,CAAC,UAAU;oBAAE,MAAM;gBAC3B,SAAS,EAAE,CAAC;gBACZ,MAAM,IAAI,CAAC;YACb,CAAC;YAED,IAAI,SAAS,KAAK,CAAC,EAAE,CAAC;gBACpB,MAAM,CAAC,IAAI,CAAC,oDAAoD,GAAG,EAAE,CAAC,CAAC;YACzE,CAAC;iBAAM,CAAC;gBACN,MAAM,CAAC,KAAK,CAAC,4CAA4C,SAAS,SAAS,CAAC,CAAC;YAC/E,CAAC;YACD,OAAO,SAAS,CAAC;QACnB,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,MAAM,CAAC,KAAK,CAAC,uCAAuC,EAAE,CAAC,CAAC,CAAC;YACzD,MAAM,CAAC,CAAC;QACV,CAAC;IACH,CAAC;IAED,KAAK;QACH,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;IACzB,CAAC;CACF"}
|
|
@@ -9,11 +9,13 @@ vi.mock('./github.js', () => ({
|
|
|
9
9
|
}));
|
|
10
10
|
const mockCrawleeCrawl = vi.fn();
|
|
11
11
|
const mockSetStorageState = vi.fn();
|
|
12
|
+
const mockSetPathPrefix = vi.fn();
|
|
12
13
|
vi.mock('./crawlee-crawler.js', () => ({
|
|
13
14
|
CrawleeCrawler: function () {
|
|
14
15
|
return {
|
|
15
16
|
crawl: mockCrawleeCrawl,
|
|
16
17
|
setStorageState: mockSetStorageState,
|
|
18
|
+
setPathPrefix: mockSetPathPrefix,
|
|
17
19
|
};
|
|
18
20
|
},
|
|
19
21
|
}));
|
|
@@ -95,32 +97,32 @@ describe('DocsCrawler', () => {
|
|
|
95
97
|
}
|
|
96
98
|
expect(results).toHaveLength(2);
|
|
97
99
|
});
|
|
98
|
-
it('should return crawlee type
|
|
100
|
+
it('should return crawlee type regardless of page count', async () => {
|
|
99
101
|
mockCrawleeCrawl.mockImplementation(async function* () {
|
|
100
102
|
yield { url: 'https://example.com/page1', path: '/page1', content: 'Page 1', title: 'Page 1' };
|
|
101
|
-
yield { url: 'https://example.com/page2', path: '/page2', content: 'Page 2', title: 'Page 2' };
|
|
102
103
|
});
|
|
104
|
+
const results = [];
|
|
103
105
|
const generator = crawler.crawl('https://example.com');
|
|
104
|
-
// Manually iterate to capture the return value
|
|
105
106
|
let result = await generator.next();
|
|
106
107
|
while (!result.done) {
|
|
108
|
+
results.push(result.value);
|
|
107
109
|
result = await generator.next();
|
|
108
110
|
}
|
|
109
111
|
const crawlerType = result.value;
|
|
112
|
+
expect(results).toHaveLength(1);
|
|
110
113
|
expect(crawlerType).toBe('crawlee');
|
|
111
114
|
});
|
|
112
|
-
it('should
|
|
113
|
-
mockCrawleeCrawl.mockImplementation(
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
});
|
|
115
|
+
it('should return crawlee type even with 0 pages found', async () => {
|
|
116
|
+
mockCrawleeCrawl.mockImplementation(() => ({
|
|
117
|
+
[Symbol.asyncIterator]: () => ({ next: () => Promise.resolve({ done: true, value: undefined }) }),
|
|
118
|
+
}));
|
|
117
119
|
const generator = crawler.crawl('https://example.com');
|
|
118
|
-
await
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
120
|
+
let result = await generator.next();
|
|
121
|
+
while (!result.done) {
|
|
122
|
+
result = await generator.next();
|
|
123
|
+
}
|
|
124
|
+
const crawlerType = result.value;
|
|
125
|
+
expect(crawlerType).toBe('crawlee');
|
|
124
126
|
});
|
|
125
127
|
});
|
|
126
128
|
describe('abort', () => {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"docs-crawler.test.js","sourceRoot":"","sources":["../../src/crawler/docs-crawler.test.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEhD,MAAM,eAAe,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;AAChC,EAAE,CAAC,IAAI,CAAC,aAAa,EAAE,GAAG,EAAE,CAAC,CAAC;IAC5B,aAAa,EAAE;QACb,OAAO;YACL,KAAK,EAAE,eAAe;SACvB,CAAC;IACJ,CAAC;CACF,CAAC,CAAC,CAAC;AAEJ,MAAM,gBAAgB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;AACjC,MAAM,mBAAmB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;AACpC,EAAE,CAAC,IAAI,CAAC,sBAAsB,EAAE,GAAG,EAAE,CAAC,CAAC;IACrC,cAAc,EAAE;QACd,OAAO;YACL,KAAK,EAAE,gBAAgB;YACvB,eAAe,EAAE,mBAAmB;
|
|
1
|
+
{"version":3,"file":"docs-crawler.test.js","sourceRoot":"","sources":["../../src/crawler/docs-crawler.test.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEhD,MAAM,eAAe,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;AAChC,EAAE,CAAC,IAAI,CAAC,aAAa,EAAE,GAAG,EAAE,CAAC,CAAC;IAC5B,aAAa,EAAE;QACb,OAAO;YACL,KAAK,EAAE,eAAe;SACvB,CAAC;IACJ,CAAC;CACF,CAAC,CAAC,CAAC;AAEJ,MAAM,gBAAgB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;AACjC,MAAM,mBAAmB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;AACpC,MAAM,iBAAiB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;AAClC,EAAE,CAAC,IAAI,CAAC,sBAAsB,EAAE,GAAG,EAAE,CAAC,CAAC;IACrC,cAAc,EAAE;QACd,OAAO;YACL,KAAK,EAAE,gBAAgB;YACvB,eAAe,EAAE,mBAAmB;YACpC,aAAa,EAAE,iBAAiB;SACjC,CAAC;IACJ,CAAC;CACF,CAAC,CAAC,CAAC;AAEJ,QAAQ,CAAC,aAAa,EAAE,GAAG,EAAE;IAC3B,IAAI,OAAoB,CAAC;IAEzB,UAAU,CAAC,GAAG,EAAE;QACd,EAAE,CAAC,aAAa,EAAE,CAAC;QACnB,OAAO,GAAG,IAAI,WAAW,EAAE,CAAC;IAC9B,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,aAAa,EAAE,GAAG,EAAE;QAC3B,EAAE,CAAC,uCAAuC,EAAE,GAAG,EAAE;YAC/C,MAAM,CAAC,OAAO,CAAC,CAAC,WAAW,EAAE,CAAC;QAChC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,iCAAiC,EAAE,GAAG,EAAE;YACzC,MAAM,aAAa,GAAG,IAAI,WAAW,CAAC,EAAE,EAAE,GAAG,EAAE,cAAc,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;YACxE,MAAM,CAAC,aAAa,CAAC,CAAC,WAAW,EAAE,CAAC;QACtC,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,OAAO,EAAE,GAAG,EAAE;QACrB,QAAQ,CAAC,aAAa,EAAE,GAAG,EAAE;YAC3B,EAAE,CAAC,8CAA8C,EAAE,KAAK,IAAI,EAAE;gBAC5D,MAAM,WAAW,GAAkB;oBACjC,EAAE,GAAG,EAAE,yCAAyC,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,UAAU,EAAE,KAAK,EAAE,QAAQ,EAAE;iBAC5G,CAAC;gBAEF,eAAe,CAAC,kBAAkB,CAAC,KAAK,SAAS,CAAC;oBAChD,KAAK,MAAM,MAAM,IAAI,WAAW,EAAE,CAAC;wBACjC,MAAM,MAAM,CAAC;oBACf,CAAC;gBACH,CAAC,CAAC,CAAC;gBAEH,MAAM,OAAO,GAAkB,EAAE,CAAC;gBAClC,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,+BAA+B,CAAC,CAAC;gBAEjE,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,SAAS,EAAE,CAAC;oBACrC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBACvB,CAAC;gBAED,MAAM,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;gBAChC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC;YACjD,CAAC,CAAC,CAAC;YAEH,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;gBACzD,eAAe,CAAC,kBAAkB,CAAC,KAAK,SAAS,CAAC;oBAChD,MAAM,EAAE,GAAG,EAAE,+BAA+B,EAAE,IAAI,EAAE,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC;gBAC5F,CAAC,CAAC,CAAC;gBAEH,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,+BAA+B,CAAC,CAAC;gBAEjE,+CAA+C;gBAC/C,IAAI,MAAM,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE,CAAC;gBACpC,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;oBACpB,MAAM,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE,CAAC;gBAClC,CAAC;gBACD,MAAM,WAAW,GAAG,MAAM,CAAC,KAAK,CAAC;gBAEjC,MAAM,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YACrC,CAAC,CAAC,CAAC;YAEH,EAAE,CAAC,4CAA4C,EAAE,KAAK,IAAI,EAAE;gBAC1D,yCAAyC;gBACzC,eAAe,CAAC,kBAAkB,CAAC,KAAK,SAAS,CAAC;oBAChD,MAAM,IAAI,KAAK,CAAC,kBAAkB,CAAC,CAAC;gBACtC,CAAC,CAAC,CAAC;gBAEH,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,+BAA+B,CAAC,CAAC;gBAEjE,MAAM,MAAM,CAAC,KAAK,IAAI,EAAE;oBACtB,6DAA6D;oBAC7D,IAAI,KAAK,EAAE,MAAM,CAAC,IAAI,SAAS,EAAE,CAAC;wBAChC,uBAAuB;oBACzB,CAAC;gBACH,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC;YACzC,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,QAAQ,CAAC,iBAAiB,EAAE,GAAG,EAAE;YAC/B,EAAE,CAAC,+CAA+C,EAAE,KAAK,IAAI,EAAE;gBAC7D,MAAM,WAAW,GAAkB;oBACjC,EAAE,GAAG,EAAE,gCAAgC,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,gBAAgB,EAAE,KAAK,EAAE,OAAO,EAAE;oBACpG,EAAE,GAAG,EAAE,8BAA8B,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,cAAc,EAAE,KAAK,EAAE,KAAK,EAAE;iBAC7F,CAAC;gBAEF,gBAAgB,CAAC,kBAAkB,CAAC,KAAK,SAAS,CAAC;oBACjD,KAAK,MAAM,MAAM,IAAI,WAAW,EAAE,CAAC;wBACjC,MAAM,MAAM,CAAC;oBACf,CAAC;gBACH,CAAC,CAAC,CAAC;gBAEH,MAAM,OAAO,GAAkB,EAAE,CAAC;gBAClC,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,OAAO,CAAC,KAAK,CAAC,0BAA0B,CAAC,EAAE,CAAC;oBACrE,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBACvB,CAAC;gBAED,MAAM,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YAClC,CAAC,CAAC,CAAC;YAEH,EAAE,CAAC,qDAAqD,EAAE,KAAK,IAAI,EAAE;gBACnE,gBAAgB,CAAC,kBAAkB,CAAC,KAAK,SAAS,CAAC;oBACjD,MAAM,EAAE,GAAG,EAAE,2BAA2B,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;gBACjG,CAAC,CAAC,CAAC;gBAEH,MAAM,OAAO,GAAkB,EAAE,CAAC;gBAClC,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC;gBAEvD,IAAI,MAAM,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE,CAAC;gBACpC,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;oBACpB,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;oBAC3B,MAAM,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE,CAAC;gBAClC,CAAC;gBACD,MAAM,WAAW,GAAG,MAAM,CAAC,KAAK,CAAC;gBAEjC,MAAM,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;gBAChC,MAAM,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YACtC,CAAC,CAAC,CAAC;YAEH,EAAE,CAAC,oDAAoD,EAAE,KAAK,IAAI,EAAE;gBAClE,gBAAgB,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,CAAC;oBACzC,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC;iBAClG,CAAC,CAAC,CAAC;gBAEJ,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC;gBAEvD,IAAI,MAAM,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE,CAAC;gBACpC,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;oBACpB,MAAM,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE,CAAC;gBAClC,CAAC;gBACD,MAAM,WAAW,GAAG,MAAM,CAAC,KAAK,CAAC;gBAEjC,MAAM,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YACtC,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,QAAQ,CAAC,OAAO,EAAE,GAAG,EAAE;YACrB,EAAE,CAAC,mCAAmC,EAAE,KAAK,IAAI,EAAE;gBACjD,gBAAgB,CAAC,kBAAkB,CAAC,KAAK,SAAS,CAAC;oBACjD,MAAM,EAAE,GAAG,EAAE,2BAA2B,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;oBAC/F,MAAM,EAAE,GAAG,EAAE,2BAA2B,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;oBAC/F,MAAM,EAAE,GAAG,EAAE,2BAA2B,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;gBACjG,CAAC,CAAC,CAAC;gBAEH,MAAM,OAAO,GAAkB,EAAE,CAAC;gBAClC,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC;gBAEvD,mBAAmB;gBACnB,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE,CAAC;gBACrC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;oBAChB,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;gBAC5B,CAAC;gBAED,QAAQ;gBACR,OAAO,CAAC,KAAK,EAAE,CAAC;gBAEhB,2EAA2E;gBAC3E,0DAA0D;gBAC1D,MAAM,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YAClC,CAAC,CAAC,CAAC;YAEH,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;gBACzD,OAAO,CAAC,KAAK,EAAE,CAAC;gBAEhB,gBAAgB,CAAC,kBAAkB,CAAC,KAAK,SAAS,CAAC;oBACjD,MAAM,EAAE,GAAG,EAAE,2BAA2B,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;gBACjG,CAAC,CAAC,CAAC;gBAEH,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC;gBACvD,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE,CAAC;gBAEtC,2DAA2D;gBAC3D,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBAC/B,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YACvC,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,QAAQ,CAAC,iBAAiB,EAAE,GAAG,EAAE;YAC/B,EAAE,CAAC,0BAA0B,EAAE,GAAG,EAAE;gBAClC,MAAM,YAAY,GAAG;oBACnB,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,aAAa,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC;iBAClF,CAAC;gBAEF,OAAO,CAAC,eAAe,CAAC,YAAY,CAAC,CAAC;gBAEtC,kCAAkC;gBAClC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC1B,CAAC,CAAC,CAAC;YAEH,EAAE,CAAC,6CAA6C,EAAE,KAAK,IAAI,EAAE;gBAC3D,MAAM,YAAY,GAAG;oBACnB,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,aAAa,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC;iBAClF,CAAC;gBAEF,OAAO,CAAC,eAAe,CAAC,YAAY,CAAC,CAAC;gBAEtC,gBAAgB,CAAC,kBAAkB,CAAC,KAAK,SAAS,CAAC;oBACjD,MAAM,EAAE,GAAG,EAAE,2BAA2B,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;oBAC/F,MAAM,EAAE,GAAG,EAAE,2BAA2B,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;gBACjG,CAAC,CAAC,CAAC;gBAEH,6DAA6D;gBAC7D,IAAI,KAAK,EAAE,MAAM,CAAC,IAAI,OAAO,CAAC,KAAK,CAAC,qBAAqB,CAAC,EAAE,CAAC;oBAC3D,uBAAuB;gBACzB,CAAC;gBAED,MAAM,CAAC,mBAAmB,CAAC,CAAC,oBAAoB,CAAC,YAAY,CAAC,CAAC;YACjE,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Extract URLs from llms.txt content.
|
|
3
|
+
* Parses markdown-style links: `- [Title](/path/)` → full URL
|
|
4
|
+
*/
|
|
5
|
+
export declare function parseLlmsTxt(content: string, baseUrl: string): string[];
|
|
6
|
+
/**
|
|
7
|
+
* Attempt to fetch and parse llms.txt from a site.
|
|
8
|
+
* Returns discovered URLs, or an empty array if llms.txt is unavailable.
|
|
9
|
+
*/
|
|
10
|
+
export declare function discoverUrlsFromLlmsTxt(baseUrl: string): Promise<string[]>;
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
import { logger } from '../util/logger.js';
|
|
2
|
+
/**
|
|
3
|
+
* Fetches and parses a site's llms.txt file to discover documentation URLs.
|
|
4
|
+
*
|
|
5
|
+
* llms.txt is a convention where sites publish a machine-readable index of their
|
|
6
|
+
* documentation pages. This provides a reliable URL discovery mechanism that works
|
|
7
|
+
* even when Cloudflare or other bot protection blocks link-following during crawls.
|
|
8
|
+
*
|
|
9
|
+
* @see https://llmstxt.org/
|
|
10
|
+
*/
|
|
11
|
+
const LLMS_TXT_TIMEOUT_MS = 10_000;
|
|
12
|
+
/**
|
|
13
|
+
* Extract URLs from llms.txt content.
|
|
14
|
+
* Parses markdown-style links: `- [Title](/path/)` → full URL
|
|
15
|
+
*/
|
|
16
|
+
export function parseLlmsTxt(content, baseUrl) {
|
|
17
|
+
const origin = new URL(baseUrl).origin;
|
|
18
|
+
const urls = [];
|
|
19
|
+
const seen = new Set();
|
|
20
|
+
// Match markdown links: [text](url)
|
|
21
|
+
const linkPattern = /\[([^\]]*)\]\(([^)]+)\)/g;
|
|
22
|
+
let match;
|
|
23
|
+
while ((match = linkPattern.exec(content)) !== null) {
|
|
24
|
+
const rawUrl = match[2].trim();
|
|
25
|
+
if (!rawUrl)
|
|
26
|
+
continue;
|
|
27
|
+
let fullUrl;
|
|
28
|
+
try {
|
|
29
|
+
if (rawUrl.startsWith('http://') || rawUrl.startsWith('https://')) {
|
|
30
|
+
fullUrl = rawUrl;
|
|
31
|
+
}
|
|
32
|
+
else if (rawUrl.startsWith('/')) {
|
|
33
|
+
fullUrl = origin + rawUrl;
|
|
34
|
+
}
|
|
35
|
+
else {
|
|
36
|
+
continue;
|
|
37
|
+
}
|
|
38
|
+
const parsed = new URL(fullUrl);
|
|
39
|
+
// Only include URLs from the same origin
|
|
40
|
+
if (parsed.origin !== origin)
|
|
41
|
+
continue;
|
|
42
|
+
// Skip non-page resources
|
|
43
|
+
const ext = parsed.pathname.split('.').pop()?.toLowerCase();
|
|
44
|
+
if (ext && !['html', 'htm'].includes(ext) && parsed.pathname.includes('.')) {
|
|
45
|
+
continue;
|
|
46
|
+
}
|
|
47
|
+
// Normalize: strip hash, keep path+query
|
|
48
|
+
const normalized = parsed.origin + parsed.pathname + parsed.search;
|
|
49
|
+
if (!seen.has(normalized)) {
|
|
50
|
+
seen.add(normalized);
|
|
51
|
+
urls.push(normalized);
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
catch {
|
|
55
|
+
logger.debug(`[llms-txt] Skipping invalid URL: ${rawUrl}`);
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
return urls;
|
|
59
|
+
}
|
|
60
|
+
/**
|
|
61
|
+
* Attempt to fetch and parse llms.txt from a site.
|
|
62
|
+
* Returns discovered URLs, or an empty array if llms.txt is unavailable.
|
|
63
|
+
*/
|
|
64
|
+
export async function discoverUrlsFromLlmsTxt(baseUrl) {
|
|
65
|
+
const origin = new URL(baseUrl).origin;
|
|
66
|
+
const llmsTxtUrl = `${origin}/llms.txt`;
|
|
67
|
+
logger.info(`[llms-txt] Checking for ${llmsTxtUrl}`);
|
|
68
|
+
try {
|
|
69
|
+
const controller = new AbortController();
|
|
70
|
+
const timeout = setTimeout(() => controller.abort(), LLMS_TXT_TIMEOUT_MS);
|
|
71
|
+
const response = await fetch(llmsTxtUrl, {
|
|
72
|
+
signal: controller.signal,
|
|
73
|
+
headers: {
|
|
74
|
+
'User-Agent': 'Mozilla/5.0 (compatible; mcp-web-docs/1.0)',
|
|
75
|
+
Accept: 'text/plain, text/markdown, */*',
|
|
76
|
+
},
|
|
77
|
+
});
|
|
78
|
+
clearTimeout(timeout);
|
|
79
|
+
if (!response.ok) {
|
|
80
|
+
logger.debug(`[llms-txt] No llms.txt found (HTTP ${response.status})`);
|
|
81
|
+
return [];
|
|
82
|
+
}
|
|
83
|
+
const contentType = response.headers.get('content-type') || '';
|
|
84
|
+
if (contentType.includes('text/html')) {
|
|
85
|
+
logger.debug(`[llms-txt] Response is HTML, not a text file - skipping`);
|
|
86
|
+
return [];
|
|
87
|
+
}
|
|
88
|
+
const content = await response.text();
|
|
89
|
+
if (!content.trim()) {
|
|
90
|
+
logger.debug(`[llms-txt] Empty llms.txt`);
|
|
91
|
+
return [];
|
|
92
|
+
}
|
|
93
|
+
const urls = parseLlmsTxt(content, baseUrl);
|
|
94
|
+
logger.info(`[llms-txt] Discovered ${urls.length} URLs from llms.txt`);
|
|
95
|
+
return urls;
|
|
96
|
+
}
|
|
97
|
+
catch (error) {
|
|
98
|
+
if (error instanceof DOMException && error.name === 'AbortError') {
|
|
99
|
+
logger.debug(`[llms-txt] Request timed out`);
|
|
100
|
+
}
|
|
101
|
+
else {
|
|
102
|
+
logger.debug(`[llms-txt] Failed to fetch:`, error);
|
|
103
|
+
}
|
|
104
|
+
return [];
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
//# sourceMappingURL=llms-txt.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"llms-txt.js","sourceRoot":"","sources":["../../src/crawler/llms-txt.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAE3C;;;;;;;;GAQG;AAEH,MAAM,mBAAmB,GAAG,MAAM,CAAC;AAEnC;;;GAGG;AACH,MAAM,UAAU,YAAY,CAAC,OAAe,EAAE,OAAe;IAC3D,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC;IACvC,MAAM,IAAI,GAAa,EAAE,CAAC;IAC1B,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAE/B,oCAAoC;IACpC,MAAM,WAAW,GAAG,0BAA0B,CAAC;IAC/C,IAAI,KAAK,CAAC;IAEV,OAAO,CAAC,KAAK,GAAG,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;QACpD,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAC/B,IAAI,CAAC,MAAM;YAAE,SAAS;QAEtB,IAAI,OAAe,CAAC;QACpB,IAAI,CAAC;YACH,IAAI,MAAM,CAAC,UAAU,CAAC,SAAS,CAAC,IAAI,MAAM,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;gBAClE,OAAO,GAAG,MAAM,CAAC;YACnB,CAAC;iBAAM,IAAI,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;gBAClC,OAAO,GAAG,MAAM,GAAG,MAAM,CAAC;YAC5B,CAAC;iBAAM,CAAC;gBACN,SAAS;YACX,CAAC;YAED,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC;YAEhC,yCAAyC;YACzC,IAAI,MAAM,CAAC,MAAM,KAAK,MAAM;gBAAE,SAAS;YAEvC,0BAA0B;YAC1B,MAAM,GAAG,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,WAAW,EAAE,CAAC;YAC5D,IAAI,GAAG,IAAI,CAAC,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC3E,SAAS;YACX,CAAC;YAED,yCAAyC;YACzC,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC;YAEnE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC;gBAC1B,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;gBACrB,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YACxB,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,MAAM,CAAC,KAAK,CAAC,oCAAoC,MAAM,EAAE,CAAC,CAAC;QAC7D,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAAC,OAAe;IAC3D,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC;IACvC,MAAM,UAAU,GAAG,GAAG,MAAM,WAAW,CAAC;IAExC,MAAM,CAAC,IAAI,CAAC,2BAA2B,UAAU,EAAE,CAAC,CAAC;IAErD,IAAI,CAAC;QACH,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,mBAAmB,CAAC,CAAC;QAE1E,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,UAAU,EAAE;YACvC,MAAM,EAAE,UAAU,CAAC,MAAM;YACzB,OAAO,EAAE;gBACP,YAAY,EAAE,4CAA4C;gBAC1D,MAAM,EAAE,gCAAgC;aACzC;SACF,CAAC,CAAC;QAEH,YAAY,CAAC,OAAO,CAAC,CAAC;QAEtB,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,MAAM,CAAC,KAAK,CAAC,sCAAsC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC;YACvE,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;QAC/D,IAAI,WAAW,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;YACtC,MAAM,CAAC,KAAK,CAAC,yDAAyD,CAAC,CAAC;YACxE,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QACtC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC;YACpB,MAAM,CAAC,KAAK,CAAC,2BAA2B,CAAC,CAAC;YAC1C,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,MAAM,IAAI,GAAG,YAAY,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QAC5C,MAAM,CAAC,IAAI,CAAC,yBAAyB,IAAI,CAAC,MAAM,qBAAqB,CAAC,CAAC;QAEvE,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,YAAY,IAAI,KAAK,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;YACjE,MAAM,CAAC,KAAK,CAAC,8BAA8B,CAAC,CAAC;QAC/C,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,KAAK,CAAC,6BAA6B,EAAE,KAAK,CAAC,CAAC;QACrD,CAAC;QACD,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
import { parseLlmsTxt, discoverUrlsFromLlmsTxt } from './llms-txt.js';
|
|
2
|
+
describe('llms-txt', () => {
|
|
3
|
+
describe('parseLlmsTxt', () => {
|
|
4
|
+
const baseUrl = 'https://www.example.com/docs/';
|
|
5
|
+
it('should parse markdown links with relative paths', () => {
|
|
6
|
+
const content = `# Example Docs
|
|
7
|
+
|
|
8
|
+
## Docs
|
|
9
|
+
- [Getting Started](/docs/getting-started/)
|
|
10
|
+
- [API Reference](/docs/api/)
|
|
11
|
+
- [Configuration](/docs/config/)
|
|
12
|
+
`;
|
|
13
|
+
const urls = parseLlmsTxt(content, baseUrl);
|
|
14
|
+
expect(urls).toEqual([
|
|
15
|
+
'https://www.example.com/docs/getting-started/',
|
|
16
|
+
'https://www.example.com/docs/api/',
|
|
17
|
+
'https://www.example.com/docs/config/',
|
|
18
|
+
]);
|
|
19
|
+
});
|
|
20
|
+
it('should parse absolute URLs from the same origin', () => {
|
|
21
|
+
const content = `- [Page](https://www.example.com/docs/page1/)
|
|
22
|
+
- [Page 2](https://www.example.com/docs/page2/)`;
|
|
23
|
+
const urls = parseLlmsTxt(content, baseUrl);
|
|
24
|
+
expect(urls).toEqual(['https://www.example.com/docs/page1/', 'https://www.example.com/docs/page2/']);
|
|
25
|
+
});
|
|
26
|
+
it('should skip URLs from different origins', () => {
|
|
27
|
+
const content = `- [External](https://other-site.com/docs/page/)
|
|
28
|
+
- [Local](/docs/local/)`;
|
|
29
|
+
const urls = parseLlmsTxt(content, baseUrl);
|
|
30
|
+
expect(urls).toEqual(['https://www.example.com/docs/local/']);
|
|
31
|
+
});
|
|
32
|
+
it('should deduplicate URLs', () => {
|
|
33
|
+
const content = `- [Page](/docs/page/)
|
|
34
|
+
- [Same Page](/docs/page/)
|
|
35
|
+
- [Also Same](https://www.example.com/docs/page/)`;
|
|
36
|
+
const urls = parseLlmsTxt(content, baseUrl);
|
|
37
|
+
expect(urls).toEqual(['https://www.example.com/docs/page/']);
|
|
38
|
+
});
|
|
39
|
+
it('should strip hash fragments', () => {
|
|
40
|
+
const content = `- [Section](/docs/page/#section)
|
|
41
|
+
- [Page](/docs/page/)`;
|
|
42
|
+
const urls = parseLlmsTxt(content, baseUrl);
|
|
43
|
+
expect(urls).toEqual(['https://www.example.com/docs/page/']);
|
|
44
|
+
});
|
|
45
|
+
it('should skip non-page resources', () => {
|
|
46
|
+
const content = `- [Image](/images/logo.png)
|
|
47
|
+
- [PDF](/files/guide.pdf)
|
|
48
|
+
- [Page](/docs/guide/)`;
|
|
49
|
+
const urls = parseLlmsTxt(content, baseUrl);
|
|
50
|
+
expect(urls).toEqual(['https://www.example.com/docs/guide/']);
|
|
51
|
+
});
|
|
52
|
+
it('should return empty array when no links are present', () => {
|
|
53
|
+
expect(parseLlmsTxt('', baseUrl)).toEqual([]);
|
|
54
|
+
expect(parseLlmsTxt('# Just some text\nNo links here.', baseUrl)).toEqual([]);
|
|
55
|
+
});
|
|
56
|
+
it('should handle inline markdown links', () => {
|
|
57
|
+
const content = `Check out the [Getting Started](/docs/start/) guide and [API](/docs/api/).`;
|
|
58
|
+
const urls = parseLlmsTxt(content, baseUrl);
|
|
59
|
+
expect(urls).toEqual(['https://www.example.com/docs/start/', 'https://www.example.com/docs/api/']);
|
|
60
|
+
});
|
|
61
|
+
it('should skip relative URLs that are not absolute paths', () => {
|
|
62
|
+
const content = `- [Relative](relative/path/)
|
|
63
|
+
- [Absolute](/docs/absolute/)`;
|
|
64
|
+
const urls = parseLlmsTxt(content, baseUrl);
|
|
65
|
+
expect(urls).toEqual(['https://www.example.com/docs/absolute/']);
|
|
66
|
+
});
|
|
67
|
+
});
|
|
68
|
+
describe('discoverUrlsFromLlmsTxt', () => {
|
|
69
|
+
beforeEach(() => {
|
|
70
|
+
fetchMock.resetMocks();
|
|
71
|
+
});
|
|
72
|
+
it('should fetch and parse llms.txt', async () => {
|
|
73
|
+
const llmsContent = `# Docs
|
|
74
|
+
- [Page 1](/docs/page1/)
|
|
75
|
+
- [Page 2](/docs/page2/)`;
|
|
76
|
+
fetchMock.mockResponseOnce(llmsContent, {
|
|
77
|
+
headers: { 'content-type': 'text/plain' },
|
|
78
|
+
});
|
|
79
|
+
const urls = await discoverUrlsFromLlmsTxt('https://www.example.com/docs/');
|
|
80
|
+
expect(fetchMock).toHaveBeenCalledWith('https://www.example.com/llms.txt', expect.objectContaining({
|
|
81
|
+
headers: expect.objectContaining({
|
|
82
|
+
Accept: 'text/plain, text/markdown, */*',
|
|
83
|
+
}),
|
|
84
|
+
}));
|
|
85
|
+
expect(urls).toEqual(['https://www.example.com/docs/page1/', 'https://www.example.com/docs/page2/']);
|
|
86
|
+
});
|
|
87
|
+
it('should return empty array on 404', async () => {
|
|
88
|
+
fetchMock.mockResponseOnce('Not Found', { status: 404 });
|
|
89
|
+
const urls = await discoverUrlsFromLlmsTxt('https://www.example.com/docs/');
|
|
90
|
+
expect(urls).toEqual([]);
|
|
91
|
+
});
|
|
92
|
+
it('should return empty array on network error', async () => {
|
|
93
|
+
fetchMock.mockRejectOnce(new Error('Network error'));
|
|
94
|
+
const urls = await discoverUrlsFromLlmsTxt('https://www.example.com/docs/');
|
|
95
|
+
expect(urls).toEqual([]);
|
|
96
|
+
});
|
|
97
|
+
it('should skip HTML responses (likely a redirect to homepage)', async () => {
|
|
98
|
+
fetchMock.mockResponseOnce('<html><body>Not found</body></html>', {
|
|
99
|
+
headers: { 'content-type': 'text/html; charset=utf-8' },
|
|
100
|
+
});
|
|
101
|
+
const urls = await discoverUrlsFromLlmsTxt('https://www.example.com/docs/');
|
|
102
|
+
expect(urls).toEqual([]);
|
|
103
|
+
});
|
|
104
|
+
it('should return empty array for empty response', async () => {
|
|
105
|
+
fetchMock.mockResponseOnce('', {
|
|
106
|
+
headers: { 'content-type': 'text/plain' },
|
|
107
|
+
});
|
|
108
|
+
const urls = await discoverUrlsFromLlmsTxt('https://www.example.com/docs/');
|
|
109
|
+
expect(urls).toEqual([]);
|
|
110
|
+
});
|
|
111
|
+
});
|
|
112
|
+
});
|
|
113
|
+
//# sourceMappingURL=llms-txt.test.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"llms-txt.test.js","sourceRoot":"","sources":["../../src/crawler/llms-txt.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,uBAAuB,EAAE,MAAM,eAAe,CAAC;AAEtE,QAAQ,CAAC,UAAU,EAAE,GAAG,EAAE;IACxB,QAAQ,CAAC,cAAc,EAAE,GAAG,EAAE;QAC5B,MAAM,OAAO,GAAG,+BAA+B,CAAC;QAEhD,EAAE,CAAC,iDAAiD,EAAE,GAAG,EAAE;YACzD,MAAM,OAAO,GAAG;;;;;;CAMrB,CAAC;YACI,MAAM,IAAI,GAAG,YAAY,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;YAE5C,MAAM,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC;gBACnB,+CAA+C;gBAC/C,mCAAmC;gBACnC,sCAAsC;aACvC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,iDAAiD,EAAE,GAAG,EAAE;YACzD,MAAM,OAAO,GAAG;gDAC0B,CAAC;YAE3C,MAAM,IAAI,GAAG,YAAY,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;YAE5C,MAAM,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,qCAAqC,EAAE,qCAAqC,CAAC,CAAC,CAAC;QACvG,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yCAAyC,EAAE,GAAG,EAAE;YACjD,MAAM,OAAO,GAAG;wBACE,CAAC;YAEnB,MAAM,IAAI,GAAG,YAAY,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;YAE5C,MAAM,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,qCAAqC,CAAC,CAAC,CAAC;QAChE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yBAAyB,EAAE,GAAG,EAAE;YACjC,MAAM,OAAO,GAAG;;kDAE4B,CAAC;YAE7C,MAAM,IAAI,GAAG,YAAY,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;YAE5C,MAAM,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,oCAAoC,CAAC,CAAC,CAAC;QAC/D,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,6BAA6B,EAAE,GAAG,EAAE;YACrC,MAAM,OAAO,GAAG;sBACA,CAAC;YAEjB,MAAM,IAAI,GAAG,YAAY,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;YAE5C,MAAM,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,oCAAoC,CAAC,CAAC,CAAC;QAC/D,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gCAAgC,EAAE,GAAG,EAAE;YACxC,MAAM,OAAO,GAAG;;uBAEC,CAAC;YAElB,MAAM,IAAI,GAAG,YAAY,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;YAE5C,MAAM,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,qCAAqC,CAAC,CAAC,CAAC;QAChE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,qDAAqD,EAAE,GAAG,EAAE;YAC7D,MAAM,CAAC,YAAY,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;YAC9C,MAAM,CAAC,YAAY,CAAC,kCAAkC,EAAE,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QAChF,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,qCAAqC,EAAE,GAAG,EAAE;YAC7C,MAAM,OAAO,GAAG,4EAA4E,CAAC;YAE7F,MAAM,IAAI,GAAG,YAAY,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;YAE5C,MAAM,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,qCAAqC,EAAE,mCAAmC,CAAC,CAAC,CAAC;QACrG,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uDAAuD,EAAE,GAAG,EAAE;YAC/D,MAAM,OAAO,GAAG;8BACQ,CAAC;YAEzB,MAAM,IAAI,GAAG,YAAY,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;YAE5C,MAAM,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,wCAAwC,CAAC,CAAC,CAAC;QACnE,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,yBAAyB,EAAE,GAAG,EAAE;QACvC,UAAU,CAAC,GAAG,EAAE;YACd,SAAS,CAAC,UAAU,EAAE,CAAC;QACzB,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,iCAAiC,EAAE,KAAK,IAAI,EAAE;YAC/C,MAAM,WAAW,GAAG;;yBAED,CAAC;YAEpB,SAAS,CAAC,gBAAgB,CAAC,WAAW,EAAE;gBACtC,OAAO,EAAE,EAAE,cAAc,EAAE,YAAY,EAAE;aAC1C,CAAC,CAAC;YAEH,MAAM,IAAI,GAAG,MAAM,uBAAuB,CAAC,+BAA+B,CAAC,CAAC;YAE5E,MAAM,CAAC,SAAS,CAAC,CAAC,oBAAoB,CACpC,kCAAkC,EAClC,MAAM,CAAC,gBAAgB,CAAC;gBACtB,OAAO,EAAE,MAAM,CAAC,gBAAgB,CAAC;oBAC/B,MAAM,EAAE,gCAAgC;iBACzC,CAAC;aACH,CAAC,CACH,CAAC;YACF,MAAM,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,qCAAqC,EAAE,qCAAqC,CAAC,CAAC,CAAC;QACvG,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,kCAAkC,EAAE,KAAK,IAAI,EAAE;YAChD,SAAS,CAAC,gBAAgB,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC;YAEzD,MAAM,IAAI,GAAG,MAAM,uBAAuB,CAAC,+BAA+B,CAAC,CAAC;YAE5E,MAAM,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QAC3B,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,4CAA4C,EAAE,KAAK,IAAI,EAAE;YAC1D,SAAS,CAAC,cAAc,CAAC,IAAI,KAAK,CAAC,eAAe,CAAC,CAAC,CAAC;YAErD,MAAM,IAAI,GAAG,MAAM,uBAAuB,CAAC,+BAA+B,CAAC,CAAC;YAE5E,MAAM,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QAC3B,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,4DAA4D,EAAE,KAAK,IAAI,EAAE;YAC1E,SAAS,CAAC,gBAAgB,CAAC,qCAAqC,EAAE;gBAChE,OAAO,EAAE,EAAE,cAAc,EAAE,0BAA0B,EAAE;aACxD,CAAC,CAAC;YAEH,MAAM,IAAI,GAAG,MAAM,uBAAuB,CAAC,+BAA+B,CAAC,CAAC;YAE5E,MAAM,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QAC3B,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,8CAA8C,EAAE,KAAK,IAAI,EAAE;YAC5D,SAAS,CAAC,gBAAgB,CAAC,EAAE,EAAE;gBAC7B,OAAO,EAAE,EAAE,cAAc,EAAE,YAAY,EAAE;aAC1C,CAAC,CAAC;YAEH,MAAM,IAAI,GAAG,MAAM,uBAAuB,CAAC,+BAA+B,CAAC,CAAC;YAE5E,MAAM,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QAC3B,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
@@ -15,6 +15,16 @@ export declare class QueueManager {
|
|
|
15
15
|
/** Count of URLs filtered due to hostname mismatch */
|
|
16
16
|
private filteredByHostnameCount;
|
|
17
17
|
initialize(url: string, pathPrefix?: string): Promise<void>;
|
|
18
|
+
/**
|
|
19
|
+
* Attempt to discover URLs from the site's llms.txt and add them to the queue.
|
|
20
|
+
* This provides upfront URL discovery that bypasses bot-protection on link following.
|
|
21
|
+
*/
|
|
22
|
+
seedFromLlmsTxt(baseUrl: string): Promise<number>;
|
|
23
|
+
/**
|
|
24
|
+
* Add a batch of URLs to the request queue as seeds.
|
|
25
|
+
* Applies the same hostname and path prefix filters as enqueueLinks.
|
|
26
|
+
*/
|
|
27
|
+
addSeedUrls(urls: string[]): Promise<number>;
|
|
18
28
|
getFilteredByPathCount(): number;
|
|
19
29
|
getFilteredByHostnameCount(): number;
|
|
20
30
|
/**
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { RequestQueue, Dataset } from 'crawlee';
|
|
2
|
-
import { generateDocId } from '../util/docs.js';
|
|
2
|
+
import { generateDocId, isPathAllowed } from '../util/docs.js';
|
|
3
3
|
import { logger } from '../util/logger.js';
|
|
4
|
+
import { discoverUrlsFromLlmsTxt } from './llms-txt.js';
|
|
4
5
|
export class QueueManager {
|
|
5
6
|
requestQueue = null;
|
|
6
7
|
websiteId = '';
|
|
@@ -40,6 +41,45 @@ export class QueueManager {
|
|
|
40
41
|
const dataset = await Dataset.open(this.websiteId);
|
|
41
42
|
await dataset.drop();
|
|
42
43
|
}
|
|
44
|
+
/**
|
|
45
|
+
* Attempt to discover URLs from the site's llms.txt and add them to the queue.
|
|
46
|
+
* This provides upfront URL discovery that bypasses bot-protection on link following.
|
|
47
|
+
*/
|
|
48
|
+
async seedFromLlmsTxt(baseUrl) {
|
|
49
|
+
const urls = await discoverUrlsFromLlmsTxt(baseUrl);
|
|
50
|
+
if (urls.length === 0)
|
|
51
|
+
return 0;
|
|
52
|
+
return this.addSeedUrls(urls);
|
|
53
|
+
}
|
|
54
|
+
/**
|
|
55
|
+
* Add a batch of URLs to the request queue as seeds.
|
|
56
|
+
* Applies the same hostname and path prefix filters as enqueueLinks.
|
|
57
|
+
*/
|
|
58
|
+
async addSeedUrls(urls) {
|
|
59
|
+
if (!this.requestQueue) {
|
|
60
|
+
logger.warn(`[QueueManager] Cannot add seed URLs: queue not initialized`);
|
|
61
|
+
return 0;
|
|
62
|
+
}
|
|
63
|
+
let added = 0;
|
|
64
|
+
for (const rawUrl of urls) {
|
|
65
|
+
try {
|
|
66
|
+
const parsed = new URL(rawUrl);
|
|
67
|
+
if (!this.isHostnameAllowed(parsed.hostname))
|
|
68
|
+
continue;
|
|
69
|
+
if (this.pathPrefix && !isPathAllowed(parsed.pathname, this.pathPrefix))
|
|
70
|
+
continue;
|
|
71
|
+
const normalizedUrl = parsed.origin + parsed.pathname + parsed.search;
|
|
72
|
+
const uniqueKey = parsed.pathname + parsed.search;
|
|
73
|
+
await this.requestQueue.addRequest({ url: normalizedUrl, uniqueKey }, { forefront: false });
|
|
74
|
+
added++;
|
|
75
|
+
}
|
|
76
|
+
catch {
|
|
77
|
+
logger.debug(`[QueueManager] Skipping invalid seed URL: ${rawUrl}`);
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
logger.info(`[QueueManager] Seeded ${added} URLs from llms.txt (${urls.length - added} filtered)`);
|
|
81
|
+
return added;
|
|
82
|
+
}
|
|
43
83
|
getFilteredByPathCount() {
|
|
44
84
|
return this.filteredByPathCount;
|
|
45
85
|
}
|
|
@@ -109,13 +149,10 @@ export class QueueManager {
|
|
|
109
149
|
}
|
|
110
150
|
// Skip URLs outside the path prefix (if configured)
|
|
111
151
|
// Use precise matching: /docs matches /docs and /docs/foo but NOT /documentation
|
|
112
|
-
if (pathPrefix) {
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
incrementFilteredByPath();
|
|
117
|
-
return false;
|
|
118
|
-
}
|
|
152
|
+
if (pathPrefix && !isPathAllowed(url.pathname, pathPrefix)) {
|
|
153
|
+
logger.debug(`[QueueManager] Skipping URL outside path prefix: ${url.pathname} (prefix: ${pathPrefix})`);
|
|
154
|
+
incrementFilteredByPath();
|
|
155
|
+
return false;
|
|
119
156
|
}
|
|
120
157
|
// Return the request with a normalized URL (without hash) and unique key
|
|
121
158
|
return {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"queue-manager.js","sourceRoot":"","sources":["../../src/crawler/queue-manager.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,OAAO,EAA6C,MAAM,SAAS,CAAC;AAC3F,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;
|
|
1
|
+
{"version":3,"file":"queue-manager.js","sourceRoot":"","sources":["../../src/crawler/queue-manager.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,OAAO,EAA6C,MAAM,SAAS,CAAC;AAC3F,OAAO,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAG/D,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAC3C,OAAO,EAAE,uBAAuB,EAAE,MAAM,eAAe,CAAC;AAExD,MAAM,OAAO,YAAY;IACf,YAAY,GAAwB,IAAI,CAAC;IACzC,SAAS,GAAW,EAAE,CAAC;IACvB,OAAO,GAAkB,EAAE,CAAC;IAC5B,MAAM,CAAU,UAAU,GAAG,EAAE,CAAC;IACxC,wEAAwE;IAChE,UAAU,GAAW,EAAE,CAAC;IAChC,gGAAgG;IACxF,eAAe,GAAW,EAAE,CAAC;IACrC,yDAAyD;IACjD,mBAAmB,GAAW,CAAC,CAAC;IACxC,sDAAsD;IAC9C,uBAAuB,GAAW,CAAC,CAAC;IAE5C,KAAK,CAAC,UAAU,CAAC,GAAW,EAAE,UAAmB;QAC/C,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAC/B,IAAI,CAAC,SAAS,GAAG,aAAa,CAAC,GAAG,EAAE,SAAS,CAAC,QAAQ,CAAC,CAAC;QACxD,IAAI,CAAC,UAAU,GAAG,UAAU,IAAI,EAAE,CAAC;QACnC,IAAI,CAAC,eAAe,GAAG,SAAS,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;QACxD,IAAI,CAAC,mBAAmB,GAAG,CAAC,CAAC;QAC7B,IAAI,CAAC,uBAAuB,GAAG,CAAC,CAAC;QAEjC,MAAM,CAAC,IAAI,CAAC,8DAA8D,IAAI,CAAC,eAAe,uBAAuB,CAAC,CAAC;QACvH,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACpB,MAAM,CAAC,IAAI,CAAC,qEAAqE,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC;QACtG,CAAC;QACD,MAAM,CAAC,KAAK,CAAC,oCAAoC,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;QAEnE,mBAAmB;QACnB,IAAI,CAAC,YAAY,GAAG,MAAM,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAC5D,MAAM,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC;QAC/B,IAAI,CAAC,YAAY,GAAG,MAAM,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAE5D,gDAAgD;QAChD,MAAM,QAAQ,GAAG,SAAS,CAAC,MAAM,GAAG,SAAS,CAAC,QAAQ,GAAG,SAAS,CAAC,MAAM,CAAC;QAC1E,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC;YACjC,GAAG,EAAE,QAAQ;YACb,SAAS,EAAE,SAAS,CAAC,QAAQ,GAAG,SAAS,CAAC,MAAM;SACjD,CAAC,CAAC;QAEH,yBAAyB;QACzB,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACnD,MAAM,OAAO,CAAC,IAAI,EAAE,CAAC;IACvB,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,eAAe,CAAC,OAAe;QACnC,MAAM,IAAI,GAAG,MAAM,uBAAuB,CAAC,OAAO,CAAC,CAAC;QACpD,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,CAAC,CAAC;QAEhC,OAAO,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;IAChC,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,WAAW,CAAC,IAAc;QAC9B,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;YACvB,MAAM,CAAC,IAAI,CAAC,4DAA4D,CAAC,CAAC;YAC1E,OAAO,CAAC,CAAC;QACX,CAAC;QAED,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,KAAK,MAAM,MAAM,IAAI,IAAI,EAAE,CAAC;YAC1B,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC;gBAE/B,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,QAAQ,CAAC;oBAAE,SAAS;gBAEvD,IAAI,IAAI,CAAC,UAAU,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,QAAQ,EAAE,IAAI,CAAC,UAAU,CAAC;oBAAE,SAAS;gBAElF,MAAM,aAAa,GAAG,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC;gBACtE,MAAM,SAAS,GAAG,MAAM,CAAC,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC;gBAElD,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,EAAE,GAAG,EAAE,aAAa,EAAE,SAAS,EAAE,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC;gBAC5F,KAAK,EAAE,CAAC;YACV,CAAC;YAAC,MAAM,CAAC;gBACP,MAAM,CAAC,KAAK,CAAC,6CAA6C,MAAM,EAAE,CAAC,CAAC;YACtE,CAAC;QACH,CAAC;QAED,MAAM,CAAC,IAAI,CAAC,yBAAyB,KAAK,wBAAwB,IAAI,CAAC,MAAM,GAAG,KAAK,YAAY,CAAC,CAAC;QACnG,OAAO,KAAK,CAAC;IACf,CAAC;IAED,sBAAsB;QACpB,OAAO,IAAI,CAAC,mBAAmB,CAAC;IAClC,CAAC;IAED,0BAA0B;QACxB,OAAO,IAAI,CAAC,uBAAuB,CAAC;IACtC,CAAC;IAED;;;;;;;;;;;OAWG;IACK,iBAAiB,CAAC,QAAgB;QACxC,MAAM,CAAC,GAAG,QAAQ,CAAC,WAAW,EAAE,CAAC;QACjC,MAAM,OAAO,GAAG,IAAI,CAAC,eAAe,CAAC;QAErC,cAAc;QACd,IAAI,CAAC,KAAK,OAAO,EAAE,CAAC;YAClB,OAAO,IAAI,CAAC;QACd,CAAC;QAED,8FAA8F;QAC9F,IAAI,CAAC,CAAC,QAAQ,CAAC,GAAG,GAAG,OAAO,CAAC,EAAE,CAAC;YAC9B,OAAO,IAAI,CAAC;QACd,CAAC;QAED,OAAO,KAAK,CAAC;IACf,CAAC;IAED,KAAK,CAAC,mBAAmB,CACvB,YAAuG,EACvG,GAAQ,EACR,IAAuB;QAEvB,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,YAAa,CAAC,OAAO,EAAE,CAAC;QACrD,IAAI,SAAS,EAAE,CAAC;YACd,GAAG,CAAC,IAAI,CAAC,eAAe,EAAE;gBACxB,YAAY,EAAE,SAAS,CAAC,mBAAmB,IAAI,CAAC;gBAChD,YAAY,EAAE,SAAS,CAAC,mBAAmB,IAAI,CAAC;gBAChD,UAAU,EAAE,SAAS,CAAC,iBAAiB,IAAI,CAAC;aAC7C,CAAC,CAAC;QACL,CAAC;QAED,wDAAwD;QACxD,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;QACnC,MAAM,eAAe,GAAG,IAAI,CAAC,eAAe,CAAC;QAC7C,MAAM,iBAAiB,GAAG,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC5D,MAAM,uBAAuB,GAAG,GAAG,EAAE;YACnC,IAAI,CAAC,mBAAmB,EAAE,CAAC;QAC7B,CAAC,CAAC;QACF,MAAM,2BAA2B,GAAG,GAAG,EAAE;YACvC,IAAI,CAAC,uBAAuB,EAAE,CAAC;QACjC,CAAC,CAAC;QAEF,MAAM,cAAc,GAAwB;YAC1C,QAAQ,EAAE,aAAgC;YAC1C,wBAAwB,CAAC,GAAG;gBAC1B,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;gBAE7B,oDAAoD;gBACpD,4DAA4D;gBAC5D,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC;oBACb,MAAM,CAAC,KAAK,CAAC,wCAAwC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC;oBAChE,OAAO,KAAK,CAAC;gBACf,CAAC;gBAED,yEAAyE;gBACzE,+GAA+G;gBAC/G,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;oBACrC,MAAM,CAAC,KAAK,CAAC,wDAAwD,GAAG,CAAC,QAAQ,cAAc,eAAe,GAAG,CAAC,CAAC;oBACnH,2BAA2B,EAAE,CAAC;oBAC9B,OAAO,KAAK,CAAC;gBACf,CAAC;gBAED,oDAAoD;gBACpD,iFAAiF;gBACjF,IAAI,UAAU,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,QAAQ,EAAE,UAAU,CAAC,EAAE,CAAC;oBAC3D,MAAM,CAAC,KAAK,CAAC,oDAAoD,GAAG,CAAC,QAAQ,aAAa,UAAU,GAAG,CAAC,CAAC;oBACzG,uBAAuB,EAAE,CAAC;oBAC1B,OAAO,KAAK,CAAC;gBACf,CAAC;gBAED,yEAAyE;gBACzE,OAAO;oBACL,GAAG,GAAG;oBACN,GAAG,EAAE,GAAG,CAAC,MAAM,GAAG,GAAG,CAAC,QAAQ,GAAG,GAAG,CAAC,MAAM,EAAE,sBAAsB;oBACnE,SAAS,EAAE,GAAG,CAAC,QAAQ,GAAG,GAAG,CAAC,MAAM;iBACrC,CAAC;YACJ,CAAC;SACF,CAAC;QAEF,+CAA+C;QAC/C,IAAI,IAAI,CAAC,aAAa,EAAE,MAAM,EAAE,CAAC;YAC/B,cAAc,CAAC,QAAQ,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC1D,CAAC;QAED,MAAM,aAAa,GAAG,MAAM,YAAY,CAAC,cAAc,CAAC,CAAC;QAEzD,GAAG,CAAC,IAAI,CAAC,iBAAiB,EAAE;YAC1B,cAAc,EAAE,aAAa,CAAC,iBAAiB,CAAC,MAAM;YACtD,IAAI,EAAE,aAAa,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAwB,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;YACpF,GAAG,CAAC,IAAI,CAAC,uBAAuB,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,kBAAkB,EAAE,IAAI,CAAC,uBAAuB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACjG,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,IAAI,CAAC,UAAU,EAAE,cAAc,EAAE,IAAI,CAAC,mBAAmB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACtG,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,YAAY;QAChB,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAEzC,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACnD,MAAM,gBAAgB,GAAG,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC;QAC3C,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC;QAElB,iDAAiD;QACjD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,gBAAgB,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;YACpD,MAAM,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QAC3D,CAAC;QAED,OAAO,gBAAgB,CAAC;IAC1B,CAAC;IAED,SAAS,CAAC,MAAmB;QAC3B,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC5B,CAAC;IAED,gBAAgB;QACd,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,IAAI,YAAY,CAAC,UAAU,CAAC;IACxD,CAAC;IAED,eAAe;QACb,OAAO,IAAI,CAAC,YAAY,CAAC;IAC3B,CAAC;IAED,KAAK,CAAC,OAAO;QACX,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC;QAClB,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACtB,MAAM,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,+BAA+B,EAAE,GAAG,CAAC,CAAC,CAAC;YAClG,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;QAC3B,CAAC;IACH,CAAC"}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { QueueManager } from './queue-manager.js';
|
|
2
|
-
const { mockRequestQueue, mockDataset } = vi.hoisted(() => ({
|
|
2
|
+
const { mockRequestQueue, mockDataset, mockDiscoverUrls } = vi.hoisted(() => ({
|
|
3
3
|
mockRequestQueue: {
|
|
4
4
|
drop: vi.fn().mockResolvedValue(undefined),
|
|
5
5
|
addRequest: vi.fn().mockResolvedValue(undefined),
|
|
@@ -13,6 +13,10 @@ const { mockRequestQueue, mockDataset } = vi.hoisted(() => ({
|
|
|
13
13
|
drop: vi.fn().mockResolvedValue(undefined),
|
|
14
14
|
pushData: vi.fn().mockResolvedValue(undefined),
|
|
15
15
|
},
|
|
16
|
+
mockDiscoverUrls: vi.fn().mockResolvedValue([]),
|
|
17
|
+
}));
|
|
18
|
+
vi.mock('./llms-txt.js', () => ({
|
|
19
|
+
discoverUrlsFromLlmsTxt: mockDiscoverUrls,
|
|
16
20
|
}));
|
|
17
21
|
vi.mock('crawlee', () => ({
|
|
18
22
|
RequestQueue: {
|
|
@@ -271,6 +275,42 @@ describe('QueueManager', () => {
|
|
|
271
275
|
uniqueKey: '/docs',
|
|
272
276
|
});
|
|
273
277
|
});
|
|
278
|
+
it('should allow subpaths when pathPrefix has a trailing slash', async () => {
|
|
279
|
+
await queueManager.initialize('https://example.com/docs/', '/docs/');
|
|
280
|
+
let capturedOptions = null;
|
|
281
|
+
const mockEnqueueLinks = vi.fn().mockImplementation((options) => {
|
|
282
|
+
capturedOptions = options;
|
|
283
|
+
return Promise.resolve({ processedRequests: [] });
|
|
284
|
+
});
|
|
285
|
+
await queueManager.handleQueueAndLinks(mockEnqueueLinks, mockLog, mockRule);
|
|
286
|
+
const transformFn = capturedOptions.transformRequestFunction;
|
|
287
|
+
if (transformFn) {
|
|
288
|
+
// Subpaths must not be filtered out due to double-slash bug
|
|
289
|
+
const subpath = transformFn({
|
|
290
|
+
url: 'https://example.com/docs/intro',
|
|
291
|
+
uniqueKey: 'original',
|
|
292
|
+
});
|
|
293
|
+
expect(subpath).not.toBe(false);
|
|
294
|
+
// Deeper subpaths should also be allowed
|
|
295
|
+
const deepSubpath = transformFn({
|
|
296
|
+
url: 'https://example.com/docs/guides/getting-started',
|
|
297
|
+
uniqueKey: 'original',
|
|
298
|
+
});
|
|
299
|
+
expect(deepSubpath).not.toBe(false);
|
|
300
|
+
// The root path itself should match
|
|
301
|
+
const rootPath = transformFn({
|
|
302
|
+
url: 'https://example.com/docs/',
|
|
303
|
+
uniqueKey: 'original',
|
|
304
|
+
});
|
|
305
|
+
expect(rootPath).not.toBe(false);
|
|
306
|
+
// Unrelated paths must still be filtered
|
|
307
|
+
const outside = transformFn({
|
|
308
|
+
url: 'https://example.com/blog/post',
|
|
309
|
+
uniqueKey: 'original',
|
|
310
|
+
});
|
|
311
|
+
expect(outside).toBe(false);
|
|
312
|
+
}
|
|
313
|
+
});
|
|
274
314
|
});
|
|
275
315
|
describe('hostname filtering', () => {
|
|
276
316
|
const mockLog = {
|
|
@@ -482,6 +522,69 @@ describe('QueueManager', () => {
|
|
|
482
522
|
expect(queueManager.getRequestQueue()).toBe(mockRequestQueue);
|
|
483
523
|
});
|
|
484
524
|
});
|
|
525
|
+
describe('addSeedUrls', () => {
|
|
526
|
+
beforeEach(async () => {
|
|
527
|
+
await queueManager.initialize('https://example.com/docs');
|
|
528
|
+
mockRequestQueue.addRequest.mockClear();
|
|
529
|
+
});
|
|
530
|
+
it('should add valid URLs to the queue', async () => {
|
|
531
|
+
const added = await queueManager.addSeedUrls(['https://example.com/docs/page1/', 'https://example.com/docs/page2/']);
|
|
532
|
+
expect(added).toBe(2);
|
|
533
|
+
expect(mockRequestQueue.addRequest).toHaveBeenCalledTimes(2);
|
|
534
|
+
expect(mockRequestQueue.addRequest).toHaveBeenCalledWith({ url: 'https://example.com/docs/page1/', uniqueKey: '/docs/page1/' }, { forefront: false });
|
|
535
|
+
});
|
|
536
|
+
it('should filter URLs with different hostname', async () => {
|
|
537
|
+
const added = await queueManager.addSeedUrls(['https://example.com/docs/page1/', 'https://other.com/docs/page2/']);
|
|
538
|
+
expect(added).toBe(1);
|
|
539
|
+
expect(mockRequestQueue.addRequest).toHaveBeenCalledTimes(1);
|
|
540
|
+
});
|
|
541
|
+
it('should filter URLs outside path prefix', async () => {
|
|
542
|
+
await queueManager.initialize('https://example.com/docs', '/docs');
|
|
543
|
+
mockRequestQueue.addRequest.mockClear();
|
|
544
|
+
const added = await queueManager.addSeedUrls(['https://example.com/docs/page1/', 'https://example.com/blog/post1/']);
|
|
545
|
+
expect(added).toBe(1);
|
|
546
|
+
});
|
|
547
|
+
it('should allow subpaths when pathPrefix has a trailing slash', async () => {
|
|
548
|
+
await queueManager.initialize('https://example.com/docs/', '/docs/');
|
|
549
|
+
mockRequestQueue.addRequest.mockClear();
|
|
550
|
+
const added = await queueManager.addSeedUrls([
|
|
551
|
+
'https://example.com/docs/intro',
|
|
552
|
+
'https://example.com/docs/guides/getting-started',
|
|
553
|
+
'https://example.com/blog/post1',
|
|
554
|
+
]);
|
|
555
|
+
expect(added).toBe(2);
|
|
556
|
+
});
|
|
557
|
+
it('should return 0 when queue is not initialized', async () => {
|
|
558
|
+
const freshManager = new QueueManager();
|
|
559
|
+
const added = await freshManager.addSeedUrls(['https://example.com/page']);
|
|
560
|
+
expect(added).toBe(0);
|
|
561
|
+
});
|
|
562
|
+
it('should skip invalid URLs', async () => {
|
|
563
|
+
const added = await queueManager.addSeedUrls(['not-a-url', 'https://example.com/docs/valid/']);
|
|
564
|
+
expect(added).toBe(1);
|
|
565
|
+
});
|
|
566
|
+
});
|
|
567
|
+
describe('seedFromLlmsTxt', () => {
|
|
568
|
+
beforeEach(async () => {
|
|
569
|
+
await queueManager.initialize('https://example.com/docs');
|
|
570
|
+
mockRequestQueue.addRequest.mockClear();
|
|
571
|
+
});
|
|
572
|
+
it('should seed URLs from llms.txt discovery', async () => {
|
|
573
|
+
mockDiscoverUrls.mockResolvedValueOnce([
|
|
574
|
+
'https://example.com/docs/page1/',
|
|
575
|
+
'https://example.com/docs/page2/',
|
|
576
|
+
'https://example.com/docs/page3/',
|
|
577
|
+
]);
|
|
578
|
+
const added = await queueManager.seedFromLlmsTxt('https://example.com/docs');
|
|
579
|
+
expect(mockDiscoverUrls).toHaveBeenCalledWith('https://example.com/docs');
|
|
580
|
+
expect(added).toBe(3);
|
|
581
|
+
});
|
|
582
|
+
it('should return 0 when no llms.txt exists', async () => {
|
|
583
|
+
mockDiscoverUrls.mockResolvedValueOnce([]);
|
|
584
|
+
const added = await queueManager.seedFromLlmsTxt('https://example.com/docs');
|
|
585
|
+
expect(added).toBe(0);
|
|
586
|
+
});
|
|
587
|
+
});
|
|
485
588
|
describe('cleanup', () => {
|
|
486
589
|
it('should clear results and drop queue', async () => {
|
|
487
590
|
await queueManager.initialize('https://example.com');
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"queue-manager.test.js","sourceRoot":"","sources":["../../src/crawler/queue-manager.test.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAIlD,MAAM,EAAE,gBAAgB,EAAE,WAAW,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;IAC1D,gBAAgB,EAAE;QAChB,IAAI,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,SAAS,CAAC;QAC1C,UAAU,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,SAAS,CAAC;QAChD,OAAO,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC;YACjC,mBAAmB,EAAE,CAAC;YACtB,mBAAmB,EAAE,EAAE;YACvB,iBAAiB,EAAE,EAAE;SACtB,CAAC;KACH;IACD,WAAW,EAAE;QACX,IAAI,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,SAAS,CAAC;QAC1C,QAAQ,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,SAAS,CAAC;KAC/C;CACF,CAAC,CAAC,CAAC;AAEJ,EAAE,CAAC,IAAI,CAAC,SAAS,EAAE,GAAG,EAAE,CAAC,CAAC;IACxB,YAAY,EAAE;QACZ,IAAI,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,gBAAgB,CAAC;KAClD;IACD,OAAO,EAAE;QACP,IAAI,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,WAAW,CAAC;KAC7C;IACD,eAAe,EAAE;QACf,UAAU,EAAE,aAAa;KAC1B;CACF,CAAC,CAAC,CAAC;AAEJ,QAAQ,CAAC,cAAc,EAAE,GAAG,EAAE;IAC5B,IAAI,YAA0B,CAAC;IAE/B,UAAU,CAAC,GAAG,EAAE;QACd,EAAE,CAAC,aAAa,EAAE,CAAC;QACnB,YAAY,GAAG,IAAI,YAAY,EAAE,CAAC;IACpC,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,YAAY,EAAE,GAAG,EAAE;QAC1B,EAAE,CAAC,8BAA8B,EAAE,KAAK,IAAI,EAAE;YAC5C,MAAM,YAAY,CAAC,UAAU,CAAC,0BAA0B,CAAC,CAAC;YAE1D,MAAM,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,gBAAgB,EAAE,CAAC;YACjD,MAAM,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,oBAAoB,CAAC;gBACvD,GAAG,EAAE,0BAA0B;gBAC/B,SAAS,EAAE,OAAO;aACnB,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,6CAA6C,EAAE,KAAK,IAAI,EAAE;YAC3D,MAAM,YAAY,CAAC,UAAU,CAAC,0CAA0C,CAAC,CAAC;YAE1E,MAAM,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,oBAAoB,CAAC;gBACvD,GAAG,EAAE,0CAA0C;gBAC/C,SAAS,EAAE,uBAAuB;aACnC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,+BAA+B,EAAE,KAAK,IAAI,EAAE;YAC7C,MAAM,YAAY,CAAC,UAAU,CAAC,qBAAqB,CAAC,CAAC;YAErD,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,gBAAgB,EAAE,CAAC;QAC9C,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,qBAAqB,EAAE,GAAG,EAAE;QACnC,MAAM,OAAO,GAAQ;YACnB,IAAI,EAAE,EAAE,CAAC,EAAE,EAAE;YACb,KAAK,EAAE,EAAE,CAAC,EAAE,EAAE;YACd,OAAO,EAAE,EAAE,CAAC,EAAE,EAAE;YAChB,KAAK,EAAE,EAAE,CAAC,EAAE,EAAE;SACG,CAAC;QAEpB,MAAM,QAAQ,GAAsB;YAClC,IAAI,EAAE,SAAS;YACf,SAAS,EAAE,EAAE,cAAc,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE;YACtC,MAAM,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,IAAI,CAAC;SACxC,CAAC;QAEF,UAAU,CAAC,KAAK,IAAI,EAAE;YACpB,MAAM,YAAY,CAAC,UAAU,CAAC,qBAAqB,CAAC,CAAC;QACvD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yBAAyB,EAAE,KAAK,IAAI,EAAE;YACvC,MAAM,gBAAgB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC;gBACjD,iBAAiB,EAAE,EAAE;aACtB,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,mBAAmB,CAAC,gBAAgB,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;YAE5E,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,oBAAoB,CAAC,eAAe,EAAE;gBACzD,YAAY,EAAE,CAAC;gBACf,YAAY,EAAE,EAAE;gBAChB,UAAU,EAAE,EAAE;aACf,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gDAAgD,EAAE,KAAK,IAAI,EAAE;YAC9D,MAAM,gBAAgB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC;gBACjD,iBAAiB,EAAE,CAAC,EAAE,SAAS,EAAE,QAAQ,EAAE,EAAE,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC;aACtE,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,mBAAmB,CAAC,gBAAgB,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;YAE5E,MAAM,CAAC,gBAAgB,CAAC,CAAC,oBAAoB,CAC3C,MAAM,CAAC,gBAAgB,CAAC;gBACtB,QAAQ,EAAE,aAAa;aACxB,CAAC,CACH,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,mDAAmD,EAAE,KAAK,IAAI,EAAE;YACjE,MAAM,iBAAiB,GAAsB;gBAC3C,GAAG,QAAQ;gBACX,aAAa,EAAE,CAAC,QAAQ,EAAE,YAAY,CAAC;aACxC,CAAC;YAEF,MAAM,gBAAgB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC;gBACjD,iBAAiB,EAAE,EAAE;aACtB,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,mBAAmB,CAAC,gBAAgB,EAAE,OAAO,EAAE,iBAAiB,CAAC,CAAC;YAErF,MAAM,CAAC,gBAAgB,CAAC,CAAC,oBAAoB,CAC3C,MAAM,CAAC,gBAAgB,CAAC;gBACtB,QAAQ,EAAE,oBAAoB;aAC/B,CAAC,CACH,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,iCAAiC,EAAE,KAAK,IAAI,EAAE;YAC/C,MAAM,gBAAgB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC;gBACjD,iBAAiB,EAAE,CAAC,EAAE,SAAS,EAAE,QAAQ,EAAE,EAAE,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC;aACtE,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,mBAAmB,CAAC,gBAAgB,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;YAE5E,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,oBAAoB,CAAC,iBAAiB,EAAE;gBAC3D,cAAc,EAAE,CAAC;gBACjB,IAAI,EAAE,CAAC,QAAQ,EAAE,QAAQ,CAAC;aAC3B,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,6DAA6D,EAAE,KAAK,IAAI,EAAE;YAC3E,IAAI,eAAe,GAA+B,IAAI,CAAC;YAEvD,MAAM,gBAAgB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,CAAC,OAA4B,EAAE,EAAE;gBACnF,eAAe,GAAG,OAAO,CAAC;gBAC1B,OAAO,OAAO,CAAC,OAAO,CAAC,EAAE,iBAAiB,EAAE,EAAE,EAAE,CAAC,CAAC;YACpD,CAAC,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,mBAAmB,CAAC,gBAAgB,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;YAE5E,MAAM,CAAC,eAAe,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;YACvC,MAAM,WAAW,GAAG,eAAgB,CAAC,wBAAwB,CAAC;YAC9D,MAAM,CAAC,WAAW,CAAC,CAAC,WAAW,EAAE,CAAC;YAElC,IAAI,WAAW,EAAE,CAAC;gBAChB,MAAM,WAAW,GAAG,WAAW,CAAC;oBAC9B,GAAG,EAAE,sCAAsC;oBAC3C,SAAS,EAAE,UAAU;iBACe,CAAC,CAAC;gBAExC,MAAM,CAAC,WAAW,CAAC,CAAC,OAAO,CACzB,MAAM,CAAC,gBAAgB,CAAC;oBACtB,GAAG,EAAE,sCAAsC;oBAC3C,SAAS,EAAE,mBAAmB;iBAC/B,CAAC,CACH,CAAC;YACJ,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,8CAA8C,EAAE,KAAK,IAAI,EAAE;YAC5D,IAAI,eAAe,GAA+B,IAAI,CAAC;YAEvD,MAAM,gBAAgB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,CAAC,OAA4B,EAAE,EAAE;gBACnF,eAAe,GAAG,OAAO,CAAC;gBAC1B,OAAO,OAAO,CAAC,OAAO,CAAC,EAAE,iBAAiB,EAAE,EAAE,EAAE,CAAC,CAAC;YACpD,CAAC,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,mBAAmB,CAAC,gBAAgB,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;YAE5E,MAAM,CAAC,eAAe,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;YACvC,MAAM,WAAW,GAAG,eAAgB,CAAC,wBAAwB,CAAC;YAC9D,MAAM,CAAC,WAAW,CAAC,CAAC,WAAW,EAAE,CAAC;YAElC,IAAI,WAAW,EAAE,CAAC;gBAChB,yDAAyD;gBACzD,MAAM,YAAY,GAAG,WAAW,CAAC;oBAC/B,GAAG,EAAE,kCAAkC;oBACvC,SAAS,EAAE,UAAU;iBACe,CAAC,CAAC;gBAExC,MAAM,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACnC,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,+DAA+D,EAAE,KAAK,IAAI,EAAE;YAC7E,IAAI,eAAe,GAA+B,IAAI,CAAC;YAEvD,MAAM,gBAAgB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,CAAC,OAA4B,EAAE,EAAE;gBACnF,eAAe,GAAG,OAAO,CAAC;gBAC1B,OAAO,OAAO,CAAC,OAAO,CAAC,EAAE,iBAAiB,EAAE,EAAE,EAAE,CAAC,CAAC;YACpD,CAAC,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,mBAAmB,CAAC,gBAAgB,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;YAE5E,MAAM,WAAW,GAAG,eAAgB,CAAC,wBAAwB,CAAC;YAC9D,IAAI,WAAW,EAAE,CAAC;gBAChB,qDAAqD;gBACrD,MAAM,WAAW,GAAG,WAAW,CAAC;oBAC9B,GAAG,EAAE,0BAA0B;oBAC/B,SAAS,EAAE,UAAU;iBACe,CAAC,CAAC;gBAExC,MAAM,CAAC,WAAW,CAAC,CAAC,OAAO,CACzB,MAAM,CAAC,gBAAgB,CAAC;oBACtB,GAAG,EAAE,0BAA0B;oBAC/B,SAAS,EAAE,OAAO;iBACnB,CAAC,CACH,CAAC;YACJ,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,uBAAuB,EAAE,GAAG,EAAE;QACrC,MAAM,OAAO,GAAQ;YACnB,IAAI,EAAE,EAAE,CAAC,EAAE,EAAE;YACb,KAAK,EAAE,EAAE,CAAC,EAAE,EAAE;YACd,OAAO,EAAE,EAAE,CAAC,EAAE,EAAE;YAChB,KAAK,EAAE,EAAE,CAAC,EAAE,EAAE;SACG,CAAC;QAEpB,MAAM,QAAQ,GAAsB;YAClC,IAAI,EAAE,SAAS;YACf,SAAS,EAAE,EAAE,cAAc,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE;YACtC,MAAM,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,IAAI,CAAC;SACxC,CAAC;QAEF,EAAE,CAAC,oCAAoC,EAAE,KAAK,IAAI,EAAE;YAClD,MAAM,YAAY,CAAC,UAAU,CAAC,8BAA8B,EAAE,WAAW,CAAC,CAAC;YAE3E,uFAAuF;YACvF,MAAM,CAAC,YAAY,CAAC,sBAAsB,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACxD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,sCAAsC,EAAE,KAAK,IAAI,EAAE;YACpD,MAAM,YAAY,CAAC,UAAU,CAAC,0BAA0B,EAAE,OAAO,CAAC,CAAC;YAEnE,IAAI,eAAe,GAA+B,IAAI,CAAC;YACvD,MAAM,gBAAgB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,CAAC,OAA4B,EAAE,EAAE;gBACnF,eAAe,GAAG,OAAO,CAAC;gBAC1B,OAAO,OAAO,CAAC,OAAO,CAAC,EAAE,iBAAiB,EAAE,EAAE,EAAE,CAAC,CAAC;YACpD,CAAC,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,mBAAmB,CAAC,gBAAgB,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;YAE5E,MAAM,WAAW,GAAG,eAAgB,CAAC,wBAAwB,CAAC;YAC9D,IAAI,WAAW,EAAE,CAAC;gBAChB,cAAc;gBACd,MAAM,UAAU,GAAG,WAAW,CAAC;oBAC7B,GAAG,EAAE,0BAA0B;oBAC/B,SAAS,EAAE,UAAU;iBACe,CAAC,CAAC;gBACxC,MAAM,CAAC,UAAU,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBAEnC,UAAU;gBACV,MAAM,OAAO,GAAG,WAAW,CAAC;oBAC1B,GAAG,EAAE,iCAAiC;oBACtC,SAAS,EAAE,UAAU;iBACe,CAAC,CAAC;gBACxC,MAAM,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAClC,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,wCAAwC,EAAE,KAAK,IAAI,EAAE;YACtD,MAAM,YAAY,CAAC,UAAU,CAAC,0BAA0B,EAAE,OAAO,CAAC,CAAC;YAEnE,IAAI,eAAe,GAA+B,IAAI,CAAC;YACvD,MAAM,gBAAgB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,CAAC,OAA4B,EAAE,EAAE;gBACnF,eAAe,GAAG,OAAO,CAAC;gBAC1B,OAAO,OAAO,CAAC,OAAO,CAAC,EAAE,iBAAiB,EAAE,EAAE,EAAE,CAAC,CAAC;YACpD,CAAC,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,mBAAmB,CAAC,gBAAgB,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;YAE5E,MAAM,WAAW,GAAG,eAAgB,CAAC,wBAAwB,CAAC;YAC9D,IAAI,WAAW,EAAE,CAAC;gBAChB,0BAA0B;gBAC1B,MAAM,aAAa,GAAG,WAAW,CAAC;oBAChC,GAAG,EAAE,+BAA+B;oBACpC,SAAS,EAAE,UAAU;iBACe,CAAC,CAAC;gBACxC,MAAM,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACpC,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2DAA2D,EAAE,KAAK,IAAI,EAAE;YACzE,MAAM,YAAY,CAAC,UAAU,CAAC,0BAA0B,EAAE,OAAO,CAAC,CAAC;YAEnE,IAAI,eAAe,GAA+B,IAAI,CAAC;YACvD,MAAM,gBAAgB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,CAAC,OAA4B,EAAE,EAAE;gBACnF,eAAe,GAAG,OAAO,CAAC;gBAC1B,OAAO,OAAO,CAAC,OAAO,CAAC,EAAE,iBAAiB,EAAE,EAAE,EAAE,CAAC,CAAC;YACpD,CAAC,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,mBAAmB,CAAC,gBAAgB,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;YAE5E,MAAM,WAAW,GAAG,eAAgB,CAAC,wBAAwB,CAAC;YAC9D,IAAI,WAAW,EAAE,CAAC;gBAChB,iEAAiE;gBACjE,MAAM,mBAAmB,GAAG,WAAW,CAAC;oBACtC,GAAG,EAAE,mCAAmC;oBACxC,SAAS,EAAE,UAAU;iBACe,CAAC,CAAC;gBACxC,MAAM,CAAC,mBAAmB,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC1C,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,iCAAiC,EAAE,KAAK,IAAI,EAAE;YAC/C,MAAM,YAAY,CAAC,UAAU,CAAC,0BAA0B,EAAE,OAAO,CAAC,CAAC;YAEnE,IAAI,eAAe,GAA+B,IAAI,CAAC;YACvD,MAAM,gBAAgB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,CAAC,OAA4B,EAAE,EAAE;gBACnF,eAAe,GAAG,OAAO,CAAC;gBAC1B,OAAO,OAAO,CAAC,OAAO,CAAC,EAAE,iBAAiB,EAAE,EAAE,EAAE,CAAC,CAAC;YACpD,CAAC,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,mBAAmB,CAAC,gBAAgB,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;YAE5E,MAAM,WAAW,GAAG,eAAgB,CAAC,wBAAwB,CAAC;YAC9D,IAAI,WAAW,EAAE,CAAC;gBAChB,mBAAmB;gBACnB,WAAW,CAAC,EAAE,GAAG,EAAE,0BAA0B,EAAE,SAAS,EAAE,GAAG,EAAuC,CAAC,CAAC;gBACtG,WAAW,CAAC,EAAE,GAAG,EAAE,2BAA2B,EAAE,SAAS,EAAE,GAAG,EAAuC,CAAC,CAAC;YACzG,CAAC;YAED,MAAM,CAAC,YAAY,CAAC,sBAAsB,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACxD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,oCAAoC,EAAE,KAAK,IAAI,EAAE;YAClD,MAAM,YAAY,CAAC,UAAU,CAAC,gCAAgC,CAAC,CAAC;YAEhE,MAAM,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,oBAAoB,CAAC;gBACvD,GAAG,EAAE,0BAA0B;gBAC/B,SAAS,EAAE,OAAO;aACnB,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,oBAAoB,EAAE,GAAG,EAAE;QAClC,MAAM,OAAO,GAAQ;YACnB,IAAI,EAAE,EAAE,CAAC,EAAE,EAAE;YACb,KAAK,EAAE,EAAE,CAAC,EAAE,EAAE;YACd,OAAO,EAAE,EAAE,CAAC,EAAE,EAAE;YAChB,KAAK,EAAE,EAAE,CAAC,EAAE,EAAE;SACG,CAAC;QAEpB,MAAM,QAAQ,GAAsB;YAClC,IAAI,EAAE,SAAS;YACf,SAAS,EAAE,EAAE,cAAc,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE;YACtC,MAAM,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,IAAI,CAAC;SACxC,CAAC;QAEF,EAAE,CAAC,6CAA6C,EAAE,KAAK,IAAI,EAAE;YAC3D,MAAM,YAAY,CAAC,UAAU,CAAC,8BAA8B,CAAC,CAAC;YAE9D,IAAI,eAAe,GAA+B,IAAI,CAAC;YACvD,MAAM,gBAAgB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,CAAC,OAA4B,EAAE,EAAE;gBACnF,eAAe,GAAG,OAAO,CAAC;gBAC1B,OAAO,OAAO,CAAC,OAAO,CAAC,EAAE,iBAAiB,EAAE,EAAE,EAAE,CAAC,CAAC;YACpD,CAAC,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,mBAAmB,CAAC,gBAAgB,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;YAE5E,MAAM,WAAW,GAAG,eAAgB,CAAC,wBAAwB,CAAC;YAC9D,IAAI,WAAW,EAAE,CAAC;gBAChB,MAAM,MAAM,GAAG,WAAW,CAAC;oBACzB,GAAG,EAAE,gCAAgC;oBACrC,SAAS,EAAE,UAAU;iBACe,CAAC,CAAC;gBACxC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACjC,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uDAAuD,EAAE,KAAK,IAAI,EAAE;YACrE,MAAM,YAAY,CAAC,UAAU,CAAC,8BAA8B,CAAC,CAAC;YAE9D,IAAI,eAAe,GAA+B,IAAI,CAAC;YACvD,MAAM,gBAAgB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,CAAC,OAA4B,EAAE,EAAE;gBACnF,eAAe,GAAG,OAAO,CAAC;gBAC1B,OAAO,OAAO,CAAC,OAAO,CAAC,EAAE,iBAAiB,EAAE,EAAE,EAAE,CAAC,CAAC;YACpD,CAAC,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,mBAAmB,CAAC,gBAAgB,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;YAE5E,MAAM,WAAW,GAAG,eAAgB,CAAC,wBAAwB,CAAC;YAC9D,IAAI,WAAW,EAAE,CAAC;gBAChB,8EAA8E;gBAC9E,MAAM,MAAM,GAAG,WAAW,CAAC;oBACzB,GAAG,EAAE,uCAAuC;oBAC5C,SAAS,EAAE,UAAU;iBACe,CAAC,CAAC;gBACxC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACjC,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;YACzD,MAAM,YAAY,CAAC,UAAU,CAAC,8BAA8B,CAAC,CAAC;YAE9D,IAAI,eAAe,GAA+B,IAAI,CAAC;YACvD,MAAM,gBAAgB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,CAAC,OAA4B,EAAE,EAAE;gBACnF,eAAe,GAAG,OAAO,CAAC;gBAC1B,OAAO,OAAO,CAAC,OAAO,CAAC,EAAE,iBAAiB,EAAE,EAAE,EAAE,CAAC,CAAC;YACpD,CAAC,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,mBAAmB,CAAC,gBAAgB,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;YAE5E,MAAM,WAAW,GAAG,eAAgB,CAAC,wBAAwB,CAAC;YAC9D,IAAI,WAAW,EAAE,CAAC;gBAChB,iFAAiF;gBACjF,MAAM,MAAM,GAAG,WAAW,CAAC;oBACzB,GAAG,EAAE,kCAAkC;oBACvC,SAAS,EAAE,UAAU;iBACe,CAAC,CAAC;gBACxC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC7B,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uCAAuC,EAAE,KAAK,IAAI,EAAE;YACrD,MAAM,YAAY,CAAC,UAAU,CAAC,8BAA8B,CAAC,CAAC;YAE9D,IAAI,eAAe,GAA+B,IAAI,CAAC;YACvD,MAAM,gBAAgB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,CAAC,OAA4B,EAAE,EAAE;gBACnF,eAAe,GAAG,OAAO,CAAC;gBAC1B,OAAO,OAAO,CAAC,OAAO,CAAC,EAAE,iBAAiB,EAAE,EAAE,EAAE,CAAC,CAAC;YACpD,CAAC,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,mBAAmB,CAAC,gBAAgB,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;YAE5E,MAAM,WAAW,GAAG,eAAgB,CAAC,wBAAwB,CAAC;YAC9D,IAAI,WAAW,EAAE,CAAC;gBAChB,uDAAuD;gBACvD,MAAM,MAAM,GAAG,WAAW,CAAC;oBACzB,GAAG,EAAE,0BAA0B;oBAC/B,SAAS,EAAE,UAAU;iBACe,CAAC,CAAC;gBACxC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC7B,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,sCAAsC,EAAE,KAAK,IAAI,EAAE;YACpD,MAAM,YAAY,CAAC,UAAU,CAAC,8BAA8B,CAAC,CAAC;YAE9D,IAAI,eAAe,GAA+B,IAAI,CAAC;YACvD,MAAM,gBAAgB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,CAAC,OAA4B,EAAE,EAAE;gBACnF,eAAe,GAAG,OAAO,CAAC;gBAC1B,OAAO,OAAO,CAAC,OAAO,CAAC,EAAE,iBAAiB,EAAE,EAAE,EAAE,CAAC,CAAC;YACpD,CAAC,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,mBAAmB,CAAC,gBAAgB,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;YAE5E,MAAM,WAAW,GAAG,eAAgB,CAAC,wBAAwB,CAAC;YAC9D,IAAI,WAAW,EAAE,CAAC;gBAChB,wCAAwC;gBACxC,WAAW,CAAC,EAAE,GAAG,EAAE,8BAA8B,EAAE,SAAS,EAAE,GAAG,EAAuC,CAAC,CAAC;gBAC1G,WAAW,CAAC,EAAE,GAAG,EAAE,6BAA6B,EAAE,SAAS,EAAE,GAAG,EAAuC,CAAC,CAAC;YAC3G,CAAC;YAED,MAAM,CAAC,YAAY,CAAC,0BAA0B,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC5D,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,kDAAkD,EAAE,KAAK,IAAI,EAAE;YAChE,MAAM,YAAY,CAAC,UAAU,CAAC,8BAA8B,CAAC,CAAC;YAE9D,IAAI,eAAe,GAA+B,IAAI,CAAC;YACvD,MAAM,gBAAgB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,CAAC,OAA4B,EAAE,EAAE;gBACnF,eAAe,GAAG,OAAO,CAAC;gBAC1B,OAAO,OAAO,CAAC,OAAO,CAAC,EAAE,iBAAiB,EAAE,EAAE,EAAE,CAAC,CAAC;YACpD,CAAC,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,mBAAmB,CAAC,gBAAgB,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;YAE5E,MAAM,WAAW,GAAG,eAAgB,CAAC,wBAAwB,CAAC;YAC9D,IAAI,WAAW,EAAE,CAAC;gBAChB,MAAM,MAAM,GAAG,WAAW,CAAC;oBACzB,GAAG,EAAE,gCAAgC;oBACrC,SAAS,EAAE,UAAU;iBACe,CAAC,CAAC;gBACxC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACjC,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,4BAA4B,EAAE,GAAG,EAAE;QAC1C,UAAU,CAAC,KAAK,IAAI,EAAE;YACpB,MAAM,YAAY,CAAC,UAAU,CAAC,qBAAqB,CAAC,CAAC;QACvD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,oBAAoB,EAAE,GAAG,EAAE;YAC5B,MAAM,MAAM,GAAgB;gBAC1B,GAAG,EAAE,0BAA0B;gBAC/B,IAAI,EAAE,OAAO;gBACb,OAAO,EAAE,cAAc;gBACvB,KAAK,EAAE,WAAW;aACnB,CAAC;YAEF,YAAY,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;YAE/B,MAAM,CAAC,YAAY,CAAC,gBAAgB,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACtD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yCAAyC,EAAE,KAAK,IAAI,EAAE;YACvD,MAAM,MAAM,GAAgB;gBAC1B,GAAG,EAAE,0BAA0B;gBAC/B,IAAI,EAAE,OAAO;gBACb,OAAO,EAAE,cAAc;gBACvB,KAAK,EAAE,WAAW;aACnB,CAAC;YAEF,YAAY,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;YAE/B,MAAM,SAAS,GAAG,MAAM,YAAY,CAAC,YAAY,EAAE,CAAC;YAEpD,MAAM,CAAC,SAAS,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YAClC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;YACrC,MAAM,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,gBAAgB,EAAE,CAAC;QAClD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;YACzD,MAAM,SAAS,GAAG,MAAM,YAAY,CAAC,YAAY,EAAE,CAAC;YAEpD,MAAM,CAAC,SAAS,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACpC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uCAAuC,EAAE,KAAK,IAAI,EAAE;YACrD,YAAY,CAAC,SAAS,CAAC;gBACrB,GAAG,EAAE,0BAA0B;gBAC/B,IAAI,EAAE,OAAO;gBACb,OAAO,EAAE,MAAM;gBACf,KAAK,EAAE,MAAM;aACd,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,YAAY,EAAE,CAAC;YAClC,MAAM,WAAW,GAAG,MAAM,YAAY,CAAC,YAAY,EAAE,CAAC;YAEtD,MAAM,CAAC,WAAW,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACtC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uCAAuC,EAAE,KAAK,IAAI,EAAE;YACrD,gBAAgB;YAChB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC3B,YAAY,CAAC,SAAS,CAAC;oBACrB,GAAG,EAAE,2BAA2B,CAAC,EAAE;oBACnC,IAAI,EAAE,QAAQ,CAAC,EAAE;oBACjB,OAAO,EAAE,WAAW,CAAC,EAAE;oBACvB,KAAK,EAAE,QAAQ,CAAC,EAAE;iBACnB,CAAC,CAAC;YACL,CAAC;YAED,MAAM,YAAY,CAAC,YAAY,EAAE,CAAC;YAElC,kCAAkC;YAClC,MAAM,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;QACxD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,kBAAkB,EAAE,GAAG,EAAE;QAChC,UAAU,CAAC,KAAK,IAAI,EAAE;YACpB,MAAM,YAAY,CAAC,UAAU,CAAC,qBAAqB,CAAC,CAAC;QACvD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2CAA2C,EAAE,GAAG,EAAE;YACnD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC5B,YAAY,CAAC,SAAS,CAAC;oBACrB,GAAG,EAAE,2BAA2B,CAAC,EAAE;oBACnC,IAAI,EAAE,QAAQ,CAAC,EAAE;oBACjB,OAAO,EAAE,WAAW,CAAC,EAAE;oBACvB,KAAK,EAAE,QAAQ,CAAC,EAAE;iBACnB,CAAC,CAAC;YACL,CAAC;YAED,MAAM,CAAC,YAAY,CAAC,gBAAgB,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACtD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uCAAuC,EAAE,GAAG,EAAE;YAC/C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC5B,YAAY,CAAC,SAAS,CAAC;oBACrB,GAAG,EAAE,2BAA2B,CAAC,EAAE;oBACnC,IAAI,EAAE,QAAQ,CAAC,EAAE;oBACjB,OAAO,EAAE,WAAW,CAAC,EAAE;oBACvB,KAAK,EAAE,QAAQ,CAAC,EAAE;iBACnB,CAAC,CAAC;YACL,CAAC;YAED,MAAM,CAAC,YAAY,CAAC,gBAAgB,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACrD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,iBAAiB,EAAE,GAAG,EAAE;QAC/B,EAAE,CAAC,0CAA0C,EAAE,GAAG,EAAE;YAClD,MAAM,CAAC,YAAY,CAAC,eAAe,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC;QACpD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,kDAAkD,EAAE,KAAK,IAAI,EAAE;YAChE,MAAM,YAAY,CAAC,UAAU,CAAC,qBAAqB,CAAC,CAAC;YAErD,MAAM,CAAC,YAAY,CAAC,eAAe,EAAE,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;QAChE,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,SAAS,EAAE,GAAG,EAAE;QACvB,EAAE,CAAC,qCAAqC,EAAE,KAAK,IAAI,EAAE;YACnD,MAAM,YAAY,CAAC,UAAU,CAAC,qBAAqB,CAAC,CAAC;YAErD,YAAY,CAAC,SAAS,CAAC;gBACrB,GAAG,EAAE,0BAA0B;gBAC/B,IAAI,EAAE,OAAO;gBACb,OAAO,EAAE,MAAM;gBACf,KAAK,EAAE,MAAM;aACd,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,OAAO,EAAE,CAAC;YAE7B,MAAM,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,gBAAgB,EAAE,CAAC;YACjD,MAAM,CAAC,YAAY,CAAC,eAAe,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC;YAElD,4BAA4B;YAC5B,MAAM,SAAS,GAAG,MAAM,YAAY,CAAC,YAAY,EAAE,CAAC;YACpD,MAAM,CAAC,SAAS,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACpC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yCAAyC,EAAE,KAAK,IAAI,EAAE;YACvD,MAAM,YAAY,CAAC,UAAU,CAAC,qBAAqB,CAAC,CAAC;YAErD,gBAAgB,CAAC,IAAI,CAAC,qBAAqB,CAAC,IAAI,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC;YAEtE,mBAAmB;YACnB,MAAM,MAAM,CAAC,YAAY,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,CAAC,aAAa,EAAE,CAAC;QAChE,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
1
|
+
{"version":3,"file":"queue-manager.test.js","sourceRoot":"","sources":["../../src/crawler/queue-manager.test.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAIlD,MAAM,EAAE,gBAAgB,EAAE,WAAW,EAAE,gBAAgB,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;IAC5E,gBAAgB,EAAE;QAChB,IAAI,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,SAAS,CAAC;QAC1C,UAAU,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,SAAS,CAAC;QAChD,OAAO,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC;YACjC,mBAAmB,EAAE,CAAC;YACtB,mBAAmB,EAAE,EAAE;YACvB,iBAAiB,EAAE,EAAE;SACtB,CAAC;KACH;IACD,WAAW,EAAE;QACX,IAAI,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,SAAS,CAAC;QAC1C,QAAQ,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,SAAS,CAAC;KAC/C;IACD,gBAAgB,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,EAAE,CAAC;CAChD,CAAC,CAAC,CAAC;AAEJ,EAAE,CAAC,IAAI,CAAC,eAAe,EAAE,GAAG,EAAE,CAAC,CAAC;IAC9B,uBAAuB,EAAE,gBAAgB;CAC1C,CAAC,CAAC,CAAC;AAEJ,EAAE,CAAC,IAAI,CAAC,SAAS,EAAE,GAAG,EAAE,CAAC,CAAC;IACxB,YAAY,EAAE;QACZ,IAAI,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,gBAAgB,CAAC;KAClD;IACD,OAAO,EAAE;QACP,IAAI,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,WAAW,CAAC;KAC7C;IACD,eAAe,EAAE;QACf,UAAU,EAAE,aAAa;KAC1B;CACF,CAAC,CAAC,CAAC;AAEJ,QAAQ,CAAC,cAAc,EAAE,GAAG,EAAE;IAC5B,IAAI,YAA0B,CAAC;IAE/B,UAAU,CAAC,GAAG,EAAE;QACd,EAAE,CAAC,aAAa,EAAE,CAAC;QACnB,YAAY,GAAG,IAAI,YAAY,EAAE,CAAC;IACpC,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,YAAY,EAAE,GAAG,EAAE;QAC1B,EAAE,CAAC,8BAA8B,EAAE,KAAK,IAAI,EAAE;YAC5C,MAAM,YAAY,CAAC,UAAU,CAAC,0BAA0B,CAAC,CAAC;YAE1D,MAAM,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,gBAAgB,EAAE,CAAC;YACjD,MAAM,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,oBAAoB,CAAC;gBACvD,GAAG,EAAE,0BAA0B;gBAC/B,SAAS,EAAE,OAAO;aACnB,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,6CAA6C,EAAE,KAAK,IAAI,EAAE;YAC3D,MAAM,YAAY,CAAC,UAAU,CAAC,0CAA0C,CAAC,CAAC;YAE1E,MAAM,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,oBAAoB,CAAC;gBACvD,GAAG,EAAE,0CAA0C;gBAC/C,SAAS,EAAE,uBAAuB;aACnC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,+BAA+B,EAAE,KAAK,IAAI,EAAE;YAC7C,MAAM,YAAY,CAAC,UAAU,CAAC,qBAAqB,CAAC,CAAC;YAErD,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,gBAAgB,EAAE,CAAC;QAC9C,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,qBAAqB,EAAE,GAAG,EAAE;QACnC,MAAM,OAAO,GAAQ;YACnB,IAAI,EAAE,EAAE,CAAC,EAAE,EAAE;YACb,KAAK,EAAE,EAAE,CAAC,EAAE,EAAE;YACd,OAAO,EAAE,EAAE,CAAC,EAAE,EAAE;YAChB,KAAK,EAAE,EAAE,CAAC,EAAE,EAAE;SACG,CAAC;QAEpB,MAAM,QAAQ,GAAsB;YAClC,IAAI,EAAE,SAAS;YACf,SAAS,EAAE,EAAE,cAAc,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE;YACtC,MAAM,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,IAAI,CAAC;SACxC,CAAC;QAEF,UAAU,CAAC,KAAK,IAAI,EAAE;YACpB,MAAM,YAAY,CAAC,UAAU,CAAC,qBAAqB,CAAC,CAAC;QACvD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yBAAyB,EAAE,KAAK,IAAI,EAAE;YACvC,MAAM,gBAAgB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC;gBACjD,iBAAiB,EAAE,EAAE;aACtB,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,mBAAmB,CAAC,gBAAgB,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;YAE5E,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,oBAAoB,CAAC,eAAe,EAAE;gBACzD,YAAY,EAAE,CAAC;gBACf,YAAY,EAAE,EAAE;gBAChB,UAAU,EAAE,EAAE;aACf,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gDAAgD,EAAE,KAAK,IAAI,EAAE;YAC9D,MAAM,gBAAgB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC;gBACjD,iBAAiB,EAAE,CAAC,EAAE,SAAS,EAAE,QAAQ,EAAE,EAAE,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC;aACtE,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,mBAAmB,CAAC,gBAAgB,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;YAE5E,MAAM,CAAC,gBAAgB,CAAC,CAAC,oBAAoB,CAC3C,MAAM,CAAC,gBAAgB,CAAC;gBACtB,QAAQ,EAAE,aAAa;aACxB,CAAC,CACH,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,mDAAmD,EAAE,KAAK,IAAI,EAAE;YACjE,MAAM,iBAAiB,GAAsB;gBAC3C,GAAG,QAAQ;gBACX,aAAa,EAAE,CAAC,QAAQ,EAAE,YAAY,CAAC;aACxC,CAAC;YAEF,MAAM,gBAAgB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC;gBACjD,iBAAiB,EAAE,EAAE;aACtB,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,mBAAmB,CAAC,gBAAgB,EAAE,OAAO,EAAE,iBAAiB,CAAC,CAAC;YAErF,MAAM,CAAC,gBAAgB,CAAC,CAAC,oBAAoB,CAC3C,MAAM,CAAC,gBAAgB,CAAC;gBACtB,QAAQ,EAAE,oBAAoB;aAC/B,CAAC,CACH,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,iCAAiC,EAAE,KAAK,IAAI,EAAE;YAC/C,MAAM,gBAAgB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC;gBACjD,iBAAiB,EAAE,CAAC,EAAE,SAAS,EAAE,QAAQ,EAAE,EAAE,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC;aACtE,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,mBAAmB,CAAC,gBAAgB,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;YAE5E,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,oBAAoB,CAAC,iBAAiB,EAAE;gBAC3D,cAAc,EAAE,CAAC;gBACjB,IAAI,EAAE,CAAC,QAAQ,EAAE,QAAQ,CAAC;aAC3B,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,6DAA6D,EAAE,KAAK,IAAI,EAAE;YAC3E,IAAI,eAAe,GAA+B,IAAI,CAAC;YAEvD,MAAM,gBAAgB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,CAAC,OAA4B,EAAE,EAAE;gBACnF,eAAe,GAAG,OAAO,CAAC;gBAC1B,OAAO,OAAO,CAAC,OAAO,CAAC,EAAE,iBAAiB,EAAE,EAAE,EAAE,CAAC,CAAC;YACpD,CAAC,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,mBAAmB,CAAC,gBAAgB,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;YAE5E,MAAM,CAAC,eAAe,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;YACvC,MAAM,WAAW,GAAG,eAAgB,CAAC,wBAAwB,CAAC;YAC9D,MAAM,CAAC,WAAW,CAAC,CAAC,WAAW,EAAE,CAAC;YAElC,IAAI,WAAW,EAAE,CAAC;gBAChB,MAAM,WAAW,GAAG,WAAW,CAAC;oBAC9B,GAAG,EAAE,sCAAsC;oBAC3C,SAAS,EAAE,UAAU;iBACe,CAAC,CAAC;gBAExC,MAAM,CAAC,WAAW,CAAC,CAAC,OAAO,CACzB,MAAM,CAAC,gBAAgB,CAAC;oBACtB,GAAG,EAAE,sCAAsC;oBAC3C,SAAS,EAAE,mBAAmB;iBAC/B,CAAC,CACH,CAAC;YACJ,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,8CAA8C,EAAE,KAAK,IAAI,EAAE;YAC5D,IAAI,eAAe,GAA+B,IAAI,CAAC;YAEvD,MAAM,gBAAgB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,CAAC,OAA4B,EAAE,EAAE;gBACnF,eAAe,GAAG,OAAO,CAAC;gBAC1B,OAAO,OAAO,CAAC,OAAO,CAAC,EAAE,iBAAiB,EAAE,EAAE,EAAE,CAAC,CAAC;YACpD,CAAC,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,mBAAmB,CAAC,gBAAgB,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;YAE5E,MAAM,CAAC,eAAe,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;YACvC,MAAM,WAAW,GAAG,eAAgB,CAAC,wBAAwB,CAAC;YAC9D,MAAM,CAAC,WAAW,CAAC,CAAC,WAAW,EAAE,CAAC;YAElC,IAAI,WAAW,EAAE,CAAC;gBAChB,yDAAyD;gBACzD,MAAM,YAAY,GAAG,WAAW,CAAC;oBAC/B,GAAG,EAAE,kCAAkC;oBACvC,SAAS,EAAE,UAAU;iBACe,CAAC,CAAC;gBAExC,MAAM,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACnC,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,+DAA+D,EAAE,KAAK,IAAI,EAAE;YAC7E,IAAI,eAAe,GAA+B,IAAI,CAAC;YAEvD,MAAM,gBAAgB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,CAAC,OAA4B,EAAE,EAAE;gBACnF,eAAe,GAAG,OAAO,CAAC;gBAC1B,OAAO,OAAO,CAAC,OAAO,CAAC,EAAE,iBAAiB,EAAE,EAAE,EAAE,CAAC,CAAC;YACpD,CAAC,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,mBAAmB,CAAC,gBAAgB,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;YAE5E,MAAM,WAAW,GAAG,eAAgB,CAAC,wBAAwB,CAAC;YAC9D,IAAI,WAAW,EAAE,CAAC;gBAChB,qDAAqD;gBACrD,MAAM,WAAW,GAAG,WAAW,CAAC;oBAC9B,GAAG,EAAE,0BAA0B;oBAC/B,SAAS,EAAE,UAAU;iBACe,CAAC,CAAC;gBAExC,MAAM,CAAC,WAAW,CAAC,CAAC,OAAO,CACzB,MAAM,CAAC,gBAAgB,CAAC;oBACtB,GAAG,EAAE,0BAA0B;oBAC/B,SAAS,EAAE,OAAO;iBACnB,CAAC,CACH,CAAC;YACJ,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,uBAAuB,EAAE,GAAG,EAAE;QACrC,MAAM,OAAO,GAAQ;YACnB,IAAI,EAAE,EAAE,CAAC,EAAE,EAAE;YACb,KAAK,EAAE,EAAE,CAAC,EAAE,EAAE;YACd,OAAO,EAAE,EAAE,CAAC,EAAE,EAAE;YAChB,KAAK,EAAE,EAAE,CAAC,EAAE,EAAE;SACG,CAAC;QAEpB,MAAM,QAAQ,GAAsB;YAClC,IAAI,EAAE,SAAS;YACf,SAAS,EAAE,EAAE,cAAc,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE;YACtC,MAAM,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,IAAI,CAAC;SACxC,CAAC;QAEF,EAAE,CAAC,oCAAoC,EAAE,KAAK,IAAI,EAAE;YAClD,MAAM,YAAY,CAAC,UAAU,CAAC,8BAA8B,EAAE,WAAW,CAAC,CAAC;YAE3E,uFAAuF;YACvF,MAAM,CAAC,YAAY,CAAC,sBAAsB,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACxD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,sCAAsC,EAAE,KAAK,IAAI,EAAE;YACpD,MAAM,YAAY,CAAC,UAAU,CAAC,0BAA0B,EAAE,OAAO,CAAC,CAAC;YAEnE,IAAI,eAAe,GAA+B,IAAI,CAAC;YACvD,MAAM,gBAAgB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,CAAC,OAA4B,EAAE,EAAE;gBACnF,eAAe,GAAG,OAAO,CAAC;gBAC1B,OAAO,OAAO,CAAC,OAAO,CAAC,EAAE,iBAAiB,EAAE,EAAE,EAAE,CAAC,CAAC;YACpD,CAAC,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,mBAAmB,CAAC,gBAAgB,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;YAE5E,MAAM,WAAW,GAAG,eAAgB,CAAC,wBAAwB,CAAC;YAC9D,IAAI,WAAW,EAAE,CAAC;gBAChB,cAAc;gBACd,MAAM,UAAU,GAAG,WAAW,CAAC;oBAC7B,GAAG,EAAE,0BAA0B;oBAC/B,SAAS,EAAE,UAAU;iBACe,CAAC,CAAC;gBACxC,MAAM,CAAC,UAAU,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBAEnC,UAAU;gBACV,MAAM,OAAO,GAAG,WAAW,CAAC;oBAC1B,GAAG,EAAE,iCAAiC;oBACtC,SAAS,EAAE,UAAU;iBACe,CAAC,CAAC;gBACxC,MAAM,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAClC,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,wCAAwC,EAAE,KAAK,IAAI,EAAE;YACtD,MAAM,YAAY,CAAC,UAAU,CAAC,0BAA0B,EAAE,OAAO,CAAC,CAAC;YAEnE,IAAI,eAAe,GAA+B,IAAI,CAAC;YACvD,MAAM,gBAAgB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,CAAC,OAA4B,EAAE,EAAE;gBACnF,eAAe,GAAG,OAAO,CAAC;gBAC1B,OAAO,OAAO,CAAC,OAAO,CAAC,EAAE,iBAAiB,EAAE,EAAE,EAAE,CAAC,CAAC;YACpD,CAAC,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,mBAAmB,CAAC,gBAAgB,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;YAE5E,MAAM,WAAW,GAAG,eAAgB,CAAC,wBAAwB,CAAC;YAC9D,IAAI,WAAW,EAAE,CAAC;gBAChB,0BAA0B;gBAC1B,MAAM,aAAa,GAAG,WAAW,CAAC;oBAChC,GAAG,EAAE,+BAA+B;oBACpC,SAAS,EAAE,UAAU;iBACe,CAAC,CAAC;gBACxC,MAAM,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACpC,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2DAA2D,EAAE,KAAK,IAAI,EAAE;YACzE,MAAM,YAAY,CAAC,UAAU,CAAC,0BAA0B,EAAE,OAAO,CAAC,CAAC;YAEnE,IAAI,eAAe,GAA+B,IAAI,CAAC;YACvD,MAAM,gBAAgB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,CAAC,OAA4B,EAAE,EAAE;gBACnF,eAAe,GAAG,OAAO,CAAC;gBAC1B,OAAO,OAAO,CAAC,OAAO,CAAC,EAAE,iBAAiB,EAAE,EAAE,EAAE,CAAC,CAAC;YACpD,CAAC,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,mBAAmB,CAAC,gBAAgB,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;YAE5E,MAAM,WAAW,GAAG,eAAgB,CAAC,wBAAwB,CAAC;YAC9D,IAAI,WAAW,EAAE,CAAC;gBAChB,iEAAiE;gBACjE,MAAM,mBAAmB,GAAG,WAAW,CAAC;oBACtC,GAAG,EAAE,mCAAmC;oBACxC,SAAS,EAAE,UAAU;iBACe,CAAC,CAAC;gBACxC,MAAM,CAAC,mBAAmB,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC1C,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,iCAAiC,EAAE,KAAK,IAAI,EAAE;YAC/C,MAAM,YAAY,CAAC,UAAU,CAAC,0BAA0B,EAAE,OAAO,CAAC,CAAC;YAEnE,IAAI,eAAe,GAA+B,IAAI,CAAC;YACvD,MAAM,gBAAgB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,CAAC,OAA4B,EAAE,EAAE;gBACnF,eAAe,GAAG,OAAO,CAAC;gBAC1B,OAAO,OAAO,CAAC,OAAO,CAAC,EAAE,iBAAiB,EAAE,EAAE,EAAE,CAAC,CAAC;YACpD,CAAC,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,mBAAmB,CAAC,gBAAgB,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;YAE5E,MAAM,WAAW,GAAG,eAAgB,CAAC,wBAAwB,CAAC;YAC9D,IAAI,WAAW,EAAE,CAAC;gBAChB,mBAAmB;gBACnB,WAAW,CAAC,EAAE,GAAG,EAAE,0BAA0B,EAAE,SAAS,EAAE,GAAG,EAAuC,CAAC,CAAC;gBACtG,WAAW,CAAC,EAAE,GAAG,EAAE,2BAA2B,EAAE,SAAS,EAAE,GAAG,EAAuC,CAAC,CAAC;YACzG,CAAC;YAED,MAAM,CAAC,YAAY,CAAC,sBAAsB,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACxD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,oCAAoC,EAAE,KAAK,IAAI,EAAE;YAClD,MAAM,YAAY,CAAC,UAAU,CAAC,gCAAgC,CAAC,CAAC;YAEhE,MAAM,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,oBAAoB,CAAC;gBACvD,GAAG,EAAE,0BAA0B;gBAC/B,SAAS,EAAE,OAAO;aACnB,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,4DAA4D,EAAE,KAAK,IAAI,EAAE;YAC1E,MAAM,YAAY,CAAC,UAAU,CAAC,2BAA2B,EAAE,QAAQ,CAAC,CAAC;YAErE,IAAI,eAAe,GAA+B,IAAI,CAAC;YACvD,MAAM,gBAAgB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,CAAC,OAA4B,EAAE,EAAE;gBACnF,eAAe,GAAG,OAAO,CAAC;gBAC1B,OAAO,OAAO,CAAC,OAAO,CAAC,EAAE,iBAAiB,EAAE,EAAE,EAAE,CAAC,CAAC;YACpD,CAAC,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,mBAAmB,CAAC,gBAAgB,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;YAE5E,MAAM,WAAW,GAAG,eAAgB,CAAC,wBAAwB,CAAC;YAC9D,IAAI,WAAW,EAAE,CAAC;gBAChB,4DAA4D;gBAC5D,MAAM,OAAO,GAAG,WAAW,CAAC;oBAC1B,GAAG,EAAE,gCAAgC;oBACrC,SAAS,EAAE,UAAU;iBACe,CAAC,CAAC;gBACxC,MAAM,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBAEhC,yCAAyC;gBACzC,MAAM,WAAW,GAAG,WAAW,CAAC;oBAC9B,GAAG,EAAE,iDAAiD;oBACtD,SAAS,EAAE,UAAU;iBACe,CAAC,CAAC;gBACxC,MAAM,CAAC,WAAW,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBAEpC,oCAAoC;gBACpC,MAAM,QAAQ,GAAG,WAAW,CAAC;oBAC3B,GAAG,EAAE,2BAA2B;oBAChC,SAAS,EAAE,UAAU;iBACe,CAAC,CAAC;gBACxC,MAAM,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBAEjC,yCAAyC;gBACzC,MAAM,OAAO,GAAG,WAAW,CAAC;oBAC1B,GAAG,EAAE,+BAA+B;oBACpC,SAAS,EAAE,UAAU;iBACe,CAAC,CAAC;gBACxC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC9B,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,oBAAoB,EAAE,GAAG,EAAE;QAClC,MAAM,OAAO,GAAQ;YACnB,IAAI,EAAE,EAAE,CAAC,EAAE,EAAE;YACb,KAAK,EAAE,EAAE,CAAC,EAAE,EAAE;YACd,OAAO,EAAE,EAAE,CAAC,EAAE,EAAE;YAChB,KAAK,EAAE,EAAE,CAAC,EAAE,EAAE;SACG,CAAC;QAEpB,MAAM,QAAQ,GAAsB;YAClC,IAAI,EAAE,SAAS;YACf,SAAS,EAAE,EAAE,cAAc,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE;YACtC,MAAM,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,IAAI,CAAC;SACxC,CAAC;QAEF,EAAE,CAAC,6CAA6C,EAAE,KAAK,IAAI,EAAE;YAC3D,MAAM,YAAY,CAAC,UAAU,CAAC,8BAA8B,CAAC,CAAC;YAE9D,IAAI,eAAe,GAA+B,IAAI,CAAC;YACvD,MAAM,gBAAgB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,CAAC,OAA4B,EAAE,EAAE;gBACnF,eAAe,GAAG,OAAO,CAAC;gBAC1B,OAAO,OAAO,CAAC,OAAO,CAAC,EAAE,iBAAiB,EAAE,EAAE,EAAE,CAAC,CAAC;YACpD,CAAC,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,mBAAmB,CAAC,gBAAgB,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;YAE5E,MAAM,WAAW,GAAG,eAAgB,CAAC,wBAAwB,CAAC;YAC9D,IAAI,WAAW,EAAE,CAAC;gBAChB,MAAM,MAAM,GAAG,WAAW,CAAC;oBACzB,GAAG,EAAE,gCAAgC;oBACrC,SAAS,EAAE,UAAU;iBACe,CAAC,CAAC;gBACxC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACjC,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uDAAuD,EAAE,KAAK,IAAI,EAAE;YACrE,MAAM,YAAY,CAAC,UAAU,CAAC,8BAA8B,CAAC,CAAC;YAE9D,IAAI,eAAe,GAA+B,IAAI,CAAC;YACvD,MAAM,gBAAgB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,CAAC,OAA4B,EAAE,EAAE;gBACnF,eAAe,GAAG,OAAO,CAAC;gBAC1B,OAAO,OAAO,CAAC,OAAO,CAAC,EAAE,iBAAiB,EAAE,EAAE,EAAE,CAAC,CAAC;YACpD,CAAC,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,mBAAmB,CAAC,gBAAgB,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;YAE5E,MAAM,WAAW,GAAG,eAAgB,CAAC,wBAAwB,CAAC;YAC9D,IAAI,WAAW,EAAE,CAAC;gBAChB,8EAA8E;gBAC9E,MAAM,MAAM,GAAG,WAAW,CAAC;oBACzB,GAAG,EAAE,uCAAuC;oBAC5C,SAAS,EAAE,UAAU;iBACe,CAAC,CAAC;gBACxC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACjC,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;YACzD,MAAM,YAAY,CAAC,UAAU,CAAC,8BAA8B,CAAC,CAAC;YAE9D,IAAI,eAAe,GAA+B,IAAI,CAAC;YACvD,MAAM,gBAAgB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,CAAC,OAA4B,EAAE,EAAE;gBACnF,eAAe,GAAG,OAAO,CAAC;gBAC1B,OAAO,OAAO,CAAC,OAAO,CAAC,EAAE,iBAAiB,EAAE,EAAE,EAAE,CAAC,CAAC;YACpD,CAAC,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,mBAAmB,CAAC,gBAAgB,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;YAE5E,MAAM,WAAW,GAAG,eAAgB,CAAC,wBAAwB,CAAC;YAC9D,IAAI,WAAW,EAAE,CAAC;gBAChB,iFAAiF;gBACjF,MAAM,MAAM,GAAG,WAAW,CAAC;oBACzB,GAAG,EAAE,kCAAkC;oBACvC,SAAS,EAAE,UAAU;iBACe,CAAC,CAAC;gBACxC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC7B,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uCAAuC,EAAE,KAAK,IAAI,EAAE;YACrD,MAAM,YAAY,CAAC,UAAU,CAAC,8BAA8B,CAAC,CAAC;YAE9D,IAAI,eAAe,GAA+B,IAAI,CAAC;YACvD,MAAM,gBAAgB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,CAAC,OAA4B,EAAE,EAAE;gBACnF,eAAe,GAAG,OAAO,CAAC;gBAC1B,OAAO,OAAO,CAAC,OAAO,CAAC,EAAE,iBAAiB,EAAE,EAAE,EAAE,CAAC,CAAC;YACpD,CAAC,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,mBAAmB,CAAC,gBAAgB,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;YAE5E,MAAM,WAAW,GAAG,eAAgB,CAAC,wBAAwB,CAAC;YAC9D,IAAI,WAAW,EAAE,CAAC;gBAChB,uDAAuD;gBACvD,MAAM,MAAM,GAAG,WAAW,CAAC;oBACzB,GAAG,EAAE,0BAA0B;oBAC/B,SAAS,EAAE,UAAU;iBACe,CAAC,CAAC;gBACxC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC7B,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,sCAAsC,EAAE,KAAK,IAAI,EAAE;YACpD,MAAM,YAAY,CAAC,UAAU,CAAC,8BAA8B,CAAC,CAAC;YAE9D,IAAI,eAAe,GAA+B,IAAI,CAAC;YACvD,MAAM,gBAAgB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,CAAC,OAA4B,EAAE,EAAE;gBACnF,eAAe,GAAG,OAAO,CAAC;gBAC1B,OAAO,OAAO,CAAC,OAAO,CAAC,EAAE,iBAAiB,EAAE,EAAE,EAAE,CAAC,CAAC;YACpD,CAAC,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,mBAAmB,CAAC,gBAAgB,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;YAE5E,MAAM,WAAW,GAAG,eAAgB,CAAC,wBAAwB,CAAC;YAC9D,IAAI,WAAW,EAAE,CAAC;gBAChB,wCAAwC;gBACxC,WAAW,CAAC,EAAE,GAAG,EAAE,8BAA8B,EAAE,SAAS,EAAE,GAAG,EAAuC,CAAC,CAAC;gBAC1G,WAAW,CAAC,EAAE,GAAG,EAAE,6BAA6B,EAAE,SAAS,EAAE,GAAG,EAAuC,CAAC,CAAC;YAC3G,CAAC;YAED,MAAM,CAAC,YAAY,CAAC,0BAA0B,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC5D,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,kDAAkD,EAAE,KAAK,IAAI,EAAE;YAChE,MAAM,YAAY,CAAC,UAAU,CAAC,8BAA8B,CAAC,CAAC;YAE9D,IAAI,eAAe,GAA+B,IAAI,CAAC;YACvD,MAAM,gBAAgB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,CAAC,OAA4B,EAAE,EAAE;gBACnF,eAAe,GAAG,OAAO,CAAC;gBAC1B,OAAO,OAAO,CAAC,OAAO,CAAC,EAAE,iBAAiB,EAAE,EAAE,EAAE,CAAC,CAAC;YACpD,CAAC,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,mBAAmB,CAAC,gBAAgB,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;YAE5E,MAAM,WAAW,GAAG,eAAgB,CAAC,wBAAwB,CAAC;YAC9D,IAAI,WAAW,EAAE,CAAC;gBAChB,MAAM,MAAM,GAAG,WAAW,CAAC;oBACzB,GAAG,EAAE,gCAAgC;oBACrC,SAAS,EAAE,UAAU;iBACe,CAAC,CAAC;gBACxC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACjC,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,4BAA4B,EAAE,GAAG,EAAE;QAC1C,UAAU,CAAC,KAAK,IAAI,EAAE;YACpB,MAAM,YAAY,CAAC,UAAU,CAAC,qBAAqB,CAAC,CAAC;QACvD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,oBAAoB,EAAE,GAAG,EAAE;YAC5B,MAAM,MAAM,GAAgB;gBAC1B,GAAG,EAAE,0BAA0B;gBAC/B,IAAI,EAAE,OAAO;gBACb,OAAO,EAAE,cAAc;gBACvB,KAAK,EAAE,WAAW;aACnB,CAAC;YAEF,YAAY,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;YAE/B,MAAM,CAAC,YAAY,CAAC,gBAAgB,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACtD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yCAAyC,EAAE,KAAK,IAAI,EAAE;YACvD,MAAM,MAAM,GAAgB;gBAC1B,GAAG,EAAE,0BAA0B;gBAC/B,IAAI,EAAE,OAAO;gBACb,OAAO,EAAE,cAAc;gBACvB,KAAK,EAAE,WAAW;aACnB,CAAC;YAEF,YAAY,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;YAE/B,MAAM,SAAS,GAAG,MAAM,YAAY,CAAC,YAAY,EAAE,CAAC;YAEpD,MAAM,CAAC,SAAS,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YAClC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;YACrC,MAAM,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,gBAAgB,EAAE,CAAC;QAClD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;YACzD,MAAM,SAAS,GAAG,MAAM,YAAY,CAAC,YAAY,EAAE,CAAC;YAEpD,MAAM,CAAC,SAAS,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACpC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uCAAuC,EAAE,KAAK,IAAI,EAAE;YACrD,YAAY,CAAC,SAAS,CAAC;gBACrB,GAAG,EAAE,0BAA0B;gBAC/B,IAAI,EAAE,OAAO;gBACb,OAAO,EAAE,MAAM;gBACf,KAAK,EAAE,MAAM;aACd,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,YAAY,EAAE,CAAC;YAClC,MAAM,WAAW,GAAG,MAAM,YAAY,CAAC,YAAY,EAAE,CAAC;YAEtD,MAAM,CAAC,WAAW,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACtC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uCAAuC,EAAE,KAAK,IAAI,EAAE;YACrD,gBAAgB;YAChB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC3B,YAAY,CAAC,SAAS,CAAC;oBACrB,GAAG,EAAE,2BAA2B,CAAC,EAAE;oBACnC,IAAI,EAAE,QAAQ,CAAC,EAAE;oBACjB,OAAO,EAAE,WAAW,CAAC,EAAE;oBACvB,KAAK,EAAE,QAAQ,CAAC,EAAE;iBACnB,CAAC,CAAC;YACL,CAAC;YAED,MAAM,YAAY,CAAC,YAAY,EAAE,CAAC;YAElC,kCAAkC;YAClC,MAAM,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;QACxD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,kBAAkB,EAAE,GAAG,EAAE;QAChC,UAAU,CAAC,KAAK,IAAI,EAAE;YACpB,MAAM,YAAY,CAAC,UAAU,CAAC,qBAAqB,CAAC,CAAC;QACvD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2CAA2C,EAAE,GAAG,EAAE;YACnD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC5B,YAAY,CAAC,SAAS,CAAC;oBACrB,GAAG,EAAE,2BAA2B,CAAC,EAAE;oBACnC,IAAI,EAAE,QAAQ,CAAC,EAAE;oBACjB,OAAO,EAAE,WAAW,CAAC,EAAE;oBACvB,KAAK,EAAE,QAAQ,CAAC,EAAE;iBACnB,CAAC,CAAC;YACL,CAAC;YAED,MAAM,CAAC,YAAY,CAAC,gBAAgB,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACtD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uCAAuC,EAAE,GAAG,EAAE;YAC/C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC5B,YAAY,CAAC,SAAS,CAAC;oBACrB,GAAG,EAAE,2BAA2B,CAAC,EAAE;oBACnC,IAAI,EAAE,QAAQ,CAAC,EAAE;oBACjB,OAAO,EAAE,WAAW,CAAC,EAAE;oBACvB,KAAK,EAAE,QAAQ,CAAC,EAAE;iBACnB,CAAC,CAAC;YACL,CAAC;YAED,MAAM,CAAC,YAAY,CAAC,gBAAgB,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACrD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,iBAAiB,EAAE,GAAG,EAAE;QAC/B,EAAE,CAAC,0CAA0C,EAAE,GAAG,EAAE;YAClD,MAAM,CAAC,YAAY,CAAC,eAAe,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC;QACpD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,kDAAkD,EAAE,KAAK,IAAI,EAAE;YAChE,MAAM,YAAY,CAAC,UAAU,CAAC,qBAAqB,CAAC,CAAC;YAErD,MAAM,CAAC,YAAY,CAAC,eAAe,EAAE,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;QAChE,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,aAAa,EAAE,GAAG,EAAE;QAC3B,UAAU,CAAC,KAAK,IAAI,EAAE;YACpB,MAAM,YAAY,CAAC,UAAU,CAAC,0BAA0B,CAAC,CAAC;YAC1D,gBAAgB,CAAC,UAAU,CAAC,SAAS,EAAE,CAAC;QAC1C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,oCAAoC,EAAE,KAAK,IAAI,EAAE;YAClD,MAAM,KAAK,GAAG,MAAM,YAAY,CAAC,WAAW,CAAC,CAAC,iCAAiC,EAAE,iCAAiC,CAAC,CAAC,CAAC;YAErH,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACtB,MAAM,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;YAC7D,MAAM,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,oBAAoB,CACtD,EAAE,GAAG,EAAE,iCAAiC,EAAE,SAAS,EAAE,cAAc,EAAE,EACrE,EAAE,SAAS,EAAE,KAAK,EAAE,CACrB,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,4CAA4C,EAAE,KAAK,IAAI,EAAE;YAC1D,MAAM,KAAK,GAAG,MAAM,YAAY,CAAC,WAAW,CAAC,CAAC,iCAAiC,EAAE,+BAA+B,CAAC,CAAC,CAAC;YAEnH,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACtB,MAAM,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;QAC/D,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,wCAAwC,EAAE,KAAK,IAAI,EAAE;YACtD,MAAM,YAAY,CAAC,UAAU,CAAC,0BAA0B,EAAE,OAAO,CAAC,CAAC;YACnE,gBAAgB,CAAC,UAAU,CAAC,SAAS,EAAE,CAAC;YAExC,MAAM,KAAK,GAAG,MAAM,YAAY,CAAC,WAAW,CAAC,CAAC,iCAAiC,EAAE,iCAAiC,CAAC,CAAC,CAAC;YAErH,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACxB,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,4DAA4D,EAAE,KAAK,IAAI,EAAE;YAC1E,MAAM,YAAY,CAAC,UAAU,CAAC,2BAA2B,EAAE,QAAQ,CAAC,CAAC;YACrE,gBAAgB,CAAC,UAAU,CAAC,SAAS,EAAE,CAAC;YAExC,MAAM,KAAK,GAAG,MAAM,YAAY,CAAC,WAAW,CAAC;gBAC3C,gCAAgC;gBAChC,iDAAiD;gBACjD,gCAAgC;aACjC,CAAC,CAAC;YAEH,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACxB,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,+CAA+C,EAAE,KAAK,IAAI,EAAE;YAC7D,MAAM,YAAY,GAAG,IAAI,YAAY,EAAE,CAAC;YACxC,MAAM,KAAK,GAAG,MAAM,YAAY,CAAC,WAAW,CAAC,CAAC,0BAA0B,CAAC,CAAC,CAAC;YAE3E,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACxB,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,0BAA0B,EAAE,KAAK,IAAI,EAAE;YACxC,MAAM,KAAK,GAAG,MAAM,YAAY,CAAC,WAAW,CAAC,CAAC,WAAW,EAAE,iCAAiC,CAAC,CAAC,CAAC;YAE/F,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACxB,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,iBAAiB,EAAE,GAAG,EAAE;QAC/B,UAAU,CAAC,KAAK,IAAI,EAAE;YACpB,MAAM,YAAY,CAAC,UAAU,CAAC,0BAA0B,CAAC,CAAC;YAC1D,gBAAgB,CAAC,UAAU,CAAC,SAAS,EAAE,CAAC;QAC1C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,0CAA0C,EAAE,KAAK,IAAI,EAAE;YACxD,gBAAgB,CAAC,qBAAqB,CAAC;gBACrC,iCAAiC;gBACjC,iCAAiC;gBACjC,iCAAiC;aAClC,CAAC,CAAC;YAEH,MAAM,KAAK,GAAG,MAAM,YAAY,CAAC,eAAe,CAAC,0BAA0B,CAAC,CAAC;YAE7E,MAAM,CAAC,gBAAgB,CAAC,CAAC,oBAAoB,CAAC,0BAA0B,CAAC,CAAC;YAC1E,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACxB,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yCAAyC,EAAE,KAAK,IAAI,EAAE;YACvD,gBAAgB,CAAC,qBAAqB,CAAC,EAAE,CAAC,CAAC;YAE3C,MAAM,KAAK,GAAG,MAAM,YAAY,CAAC,eAAe,CAAC,0BAA0B,CAAC,CAAC;YAE7E,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACxB,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,SAAS,EAAE,GAAG,EAAE;QACvB,EAAE,CAAC,qCAAqC,EAAE,KAAK,IAAI,EAAE;YACnD,MAAM,YAAY,CAAC,UAAU,CAAC,qBAAqB,CAAC,CAAC;YAErD,YAAY,CAAC,SAAS,CAAC;gBACrB,GAAG,EAAE,0BAA0B;gBAC/B,IAAI,EAAE,OAAO;gBACb,OAAO,EAAE,MAAM;gBACf,KAAK,EAAE,MAAM;aACd,CAAC,CAAC;YAEH,MAAM,YAAY,CAAC,OAAO,EAAE,CAAC;YAE7B,MAAM,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,gBAAgB,EAAE,CAAC;YACjD,MAAM,CAAC,YAAY,CAAC,eAAe,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC;YAElD,4BAA4B;YAC5B,MAAM,SAAS,GAAG,MAAM,YAAY,CAAC,YAAY,EAAE,CAAC;YACpD,MAAM,CAAC,SAAS,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACpC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yCAAyC,EAAE,KAAK,IAAI,EAAE;YACvD,MAAM,YAAY,CAAC,UAAU,CAAC,qBAAqB,CAAC,CAAC;YAErD,gBAAgB,CAAC,IAAI,CAAC,qBAAqB,CAAC,IAAI,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC;YAEtE,mBAAmB;YACnB,MAAM,MAAM,CAAC,YAAY,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,CAAC,aAAa,EAAE,CAAC;QAChE,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
package/build/util/docs.d.ts
CHANGED
|
@@ -1 +1,13 @@
|
|
|
1
1
|
export declare function generateDocId(url: string, title: string): string;
|
|
2
|
+
/**
|
|
3
|
+
* Check if a URL pathname falls within a given path prefix.
|
|
4
|
+
* Strips trailing slashes from the prefix before comparing so that
|
|
5
|
+
* `/docs` and `/docs/` behave identically.
|
|
6
|
+
*
|
|
7
|
+
* @example
|
|
8
|
+
* isPathAllowed('/docs/intro', '/docs/') → true
|
|
9
|
+
* isPathAllowed('/docs/', '/docs/') → true
|
|
10
|
+
* isPathAllowed('/blog/post', '/docs/') → false
|
|
11
|
+
* isPathAllowed('/documentation', '/docs') → false
|
|
12
|
+
*/
|
|
13
|
+
export declare function isPathAllowed(pathname: string, prefix: string): boolean;
|
package/build/util/docs.js
CHANGED
|
@@ -23,4 +23,19 @@ export function generateDocId(url, title) {
|
|
|
23
23
|
}
|
|
24
24
|
return urlObj.hostname;
|
|
25
25
|
}
|
|
26
|
+
/**
|
|
27
|
+
* Check if a URL pathname falls within a given path prefix.
|
|
28
|
+
* Strips trailing slashes from the prefix before comparing so that
|
|
29
|
+
* `/docs` and `/docs/` behave identically.
|
|
30
|
+
*
|
|
31
|
+
* @example
|
|
32
|
+
* isPathAllowed('/docs/intro', '/docs/') → true
|
|
33
|
+
* isPathAllowed('/docs/', '/docs/') → true
|
|
34
|
+
* isPathAllowed('/blog/post', '/docs/') → false
|
|
35
|
+
* isPathAllowed('/documentation', '/docs') → false
|
|
36
|
+
*/
|
|
37
|
+
export function isPathAllowed(pathname, prefix) {
|
|
38
|
+
const normalizedPrefix = prefix.replace(/\/$/, '');
|
|
39
|
+
return pathname === normalizedPrefix || pathname.startsWith(normalizedPrefix + '/');
|
|
40
|
+
}
|
|
26
41
|
//# sourceMappingURL=docs.js.map
|
package/build/util/docs.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"docs.js","sourceRoot":"","sources":["../../src/util/docs.ts"],"names":[],"mappings":"AAAA,MAAM,UAAU,aAAa,CAAC,GAAW,EAAE,KAAa;IACtD,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IAC5B,MAAM,SAAS,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAE7D,qDAAqD;IACrD,IAAI,MAAM,CAAC,QAAQ,KAAK,WAAW,IAAI,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC;QAC9E,MAAM,GAAG,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAC1C,MAAM,IAAI,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;QAC1B,OAAO,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;IAC1B,CAAC;IAED,iDAAiD;IACjD,IAAI,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QACxB,OAAO,KAAK,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IACxE,CAAC;IAED,2DAA2D;IAC3D,MAAM,SAAS,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC7C,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACzB,MAAM,QAAQ,GAAG,SAAS,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;QACtE,gEAAgE;QAChE,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,IAAI,SAAS,CAAC,CAAC,CAAC,KAAK,MAAM,EAAE,CAAC;YACpD,OAAO,GAAG,QAAQ,IAAI,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC;QACvC,CAAC;QACD,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,OAAO,MAAM,CAAC,QAAQ,CAAC;AACzB,CAAC"}
|
|
1
|
+
{"version":3,"file":"docs.js","sourceRoot":"","sources":["../../src/util/docs.ts"],"names":[],"mappings":"AAAA,MAAM,UAAU,aAAa,CAAC,GAAW,EAAE,KAAa;IACtD,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IAC5B,MAAM,SAAS,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAE7D,qDAAqD;IACrD,IAAI,MAAM,CAAC,QAAQ,KAAK,WAAW,IAAI,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC;QAC9E,MAAM,GAAG,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAC1C,MAAM,IAAI,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;QAC1B,OAAO,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;IAC1B,CAAC;IAED,iDAAiD;IACjD,IAAI,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QACxB,OAAO,KAAK,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IACxE,CAAC;IAED,2DAA2D;IAC3D,MAAM,SAAS,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC7C,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACzB,MAAM,QAAQ,GAAG,SAAS,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;QACtE,gEAAgE;QAChE,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,IAAI,SAAS,CAAC,CAAC,CAAC,KAAK,MAAM,EAAE,CAAC;YACpD,OAAO,GAAG,QAAQ,IAAI,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC;QACvC,CAAC;QACD,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,OAAO,MAAM,CAAC,QAAQ,CAAC;AACzB,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,UAAU,aAAa,CAAC,QAAgB,EAAE,MAAc;IAC5D,MAAM,gBAAgB,GAAG,MAAM,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IACnD,OAAO,QAAQ,KAAK,gBAAgB,IAAI,QAAQ,CAAC,UAAU,CAAC,gBAAgB,GAAG,GAAG,CAAC,CAAC;AACtF,CAAC"}
|
package/build/util/docs.test.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { generateDocId } from './docs.js';
|
|
1
|
+
import { generateDocId, isPathAllowed } from './docs.js';
|
|
2
2
|
describe('Docs Utilities', () => {
|
|
3
3
|
describe('generateDocId', () => {
|
|
4
4
|
it('should generate ID for GitHub Pages URLs', () => {
|
|
@@ -45,5 +45,29 @@ describe('Docs Utilities', () => {
|
|
|
45
45
|
expect(result).toBe('example-guide');
|
|
46
46
|
});
|
|
47
47
|
});
|
|
48
|
+
describe('isPathAllowed', () => {
|
|
49
|
+
it('should match exact prefix without trailing slash', () => {
|
|
50
|
+
expect(isPathAllowed('/docs', '/docs')).toBe(true);
|
|
51
|
+
});
|
|
52
|
+
it('should match exact prefix with trailing slash', () => {
|
|
53
|
+
expect(isPathAllowed('/docs/', '/docs/')).toBe(true);
|
|
54
|
+
});
|
|
55
|
+
it('should match subpaths', () => {
|
|
56
|
+
expect(isPathAllowed('/docs/intro', '/docs')).toBe(true);
|
|
57
|
+
expect(isPathAllowed('/docs/guides/getting-started', '/docs')).toBe(true);
|
|
58
|
+
});
|
|
59
|
+
it('should match subpaths when prefix has a trailing slash', () => {
|
|
60
|
+
expect(isPathAllowed('/docs/intro', '/docs/')).toBe(true);
|
|
61
|
+
expect(isPathAllowed('/docs/guides/getting-started', '/docs/')).toBe(true);
|
|
62
|
+
});
|
|
63
|
+
it('should reject paths outside the prefix', () => {
|
|
64
|
+
expect(isPathAllowed('/blog/post', '/docs')).toBe(false);
|
|
65
|
+
expect(isPathAllowed('/', '/docs')).toBe(false);
|
|
66
|
+
});
|
|
67
|
+
it('should not match paths that only share a string prefix', () => {
|
|
68
|
+
expect(isPathAllowed('/documentation', '/docs')).toBe(false);
|
|
69
|
+
expect(isPathAllowed('/docs-old/page', '/docs')).toBe(false);
|
|
70
|
+
});
|
|
71
|
+
});
|
|
48
72
|
});
|
|
49
73
|
//# sourceMappingURL=docs.test.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"docs.test.js","sourceRoot":"","sources":["../../src/util/docs.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;
|
|
1
|
+
{"version":3,"file":"docs.test.js","sourceRoot":"","sources":["../../src/util/docs.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAEzD,QAAQ,CAAC,gBAAgB,EAAE,GAAG,EAAE;IAC9B,QAAQ,CAAC,eAAe,EAAE,GAAG,EAAE;QAC7B,EAAE,CAAC,0CAA0C,EAAE,GAAG,EAAE;YAClD,MAAM,MAAM,GAAG,aAAa,CAAC,mCAAmC,EAAE,eAAe,CAAC,CAAC;YACnF,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAClC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uDAAuD,EAAE,GAAG,EAAE;YAC/D,MAAM,MAAM,GAAG,aAAa,CAAC,+CAA+C,EAAE,OAAO,CAAC,CAAC;YACvF,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;QAC1C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,8CAA8C,EAAE,GAAG,EAAE;YACtD,MAAM,MAAM,GAAG,aAAa,CAAC,0BAA0B,EAAE,mBAAmB,CAAC,CAAC;YAC9E,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;QAC3C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,mDAAmD,EAAE,GAAG,EAAE;YAC3D,MAAM,MAAM,GAAG,aAAa,CAAC,gCAAgC,EAAE,eAAe,CAAC,CAAC;YAChF,iDAAiD;YACjD,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QACpC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,6CAA6C,EAAE,GAAG,EAAE;YACrD,MAAM,MAAM,GAAG,aAAa,CAAC,sCAAsC,EAAE,YAAY,CAAC,CAAC;YACnF,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;QAClD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,+BAA+B,EAAE,GAAG,EAAE;YACvC,MAAM,MAAM,GAAG,aAAa,CAAC,0CAA0C,EAAE,iBAAiB,CAAC,CAAC;YAC5F,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACjC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,0BAA0B,EAAE,GAAG,EAAE;YAClC,MAAM,MAAM,GAAG,aAAa,CAAC,6BAA6B,EAAE,eAAe,CAAC,CAAC;YAC7E,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QACrC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,+BAA+B,EAAE,GAAG,EAAE;YACvC,MAAM,MAAM,GAAG,aAAa,CAAC,yBAAyB,EAAE,YAAY,CAAC,CAAC;YACtE,+CAA+C;YAC/C,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QACnC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,0CAA0C,EAAE,GAAG,EAAE;YAClD,MAAM,MAAM,GAAG,aAAa,CAAC,qCAAqC,EAAE,QAAQ,CAAC,CAAC;YAC9E,qBAAqB;YACrB,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAClC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,mCAAmC,EAAE,GAAG,EAAE;YAC3C,MAAM,MAAM,GAAG,aAAa,CAAC,mCAAmC,EAAE,OAAO,CAAC,CAAC;YAC3E,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QACvC,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,eAAe,EAAE,GAAG,EAAE;QAC7B,EAAE,CAAC,kDAAkD,EAAE,GAAG,EAAE;YAC1D,MAAM,CAAC,aAAa,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACrD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,+CAA+C,EAAE,GAAG,EAAE;YACvD,MAAM,CAAC,aAAa,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACvD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uBAAuB,EAAE,GAAG,EAAE;YAC/B,MAAM,CAAC,aAAa,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACzD,MAAM,CAAC,aAAa,CAAC,8BAA8B,EAAE,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC5E,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,wDAAwD,EAAE,GAAG,EAAE;YAChE,MAAM,CAAC,aAAa,CAAC,aAAa,EAAE,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC1D,MAAM,CAAC,aAAa,CAAC,8BAA8B,EAAE,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7E,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,wCAAwC,EAAE,GAAG,EAAE;YAChD,MAAM,CAAC,aAAa,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACzD,MAAM,CAAC,aAAa,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAClD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,wDAAwD,EAAE,GAAG,EAAE;YAChE,MAAM,CAAC,aAAa,CAAC,gBAAgB,EAAE,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC7D,MAAM,CAAC,aAAa,CAAC,gBAAgB,EAAE,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC/D,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|