@cosmocoder/mcp-web-docs 2.0.19 → 2.0.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/config.d.ts +0 -18
- package/build/config.js +0 -20
- package/build/config.js.map +1 -1
- package/build/config.test.js +1 -17
- package/build/config.test.js.map +1 -1
- package/build/crawler/base.d.ts +2 -4
- package/build/crawler/base.js +1 -5
- package/build/crawler/base.js.map +1 -1
- package/build/crawler/base.test.js +4 -16
- package/build/crawler/base.test.js.map +1 -1
- package/build/crawler/crawlee-crawler.js +4 -8
- package/build/crawler/crawlee-crawler.js.map +1 -1
- package/build/crawler/crawlee-crawler.test.js +23 -6
- package/build/crawler/crawlee-crawler.test.js.map +1 -1
- package/build/crawler/docs-crawler.d.ts +4 -6
- package/build/crawler/docs-crawler.js +5 -10
- package/build/crawler/docs-crawler.js.map +1 -1
- package/build/crawler/docs-crawler.test.js +18 -50
- package/build/crawler/docs-crawler.test.js.map +1 -1
- package/build/crawler/github.d.ts +1 -1
- package/build/crawler/github.js +2 -2
- package/build/crawler/github.js.map +1 -1
- package/build/crawler/github.test.js +1 -20
- package/build/crawler/github.test.js.map +1 -1
- package/build/index.js +105 -62
- package/build/index.js.map +1 -1
- package/build/index.test.js +176 -15
- package/build/index.test.js.map +1 -1
- package/build/indexing/queue-manager.d.ts +2 -0
- package/build/indexing/queue-manager.js +12 -18
- package/build/indexing/queue-manager.js.map +1 -1
- package/build/indexing/queue-manager.test.js +5 -12
- package/build/indexing/queue-manager.test.js.map +1 -1
- package/build/types.d.ts +0 -15
- package/package.json +1 -1
package/build/config.d.ts
CHANGED
|
@@ -1,9 +1,6 @@
|
|
|
1
1
|
export interface DocsConfig {
|
|
2
2
|
githubToken?: string;
|
|
3
|
-
maxDepth: number;
|
|
4
|
-
maxRequestsPerCrawl: number;
|
|
5
3
|
maxChunkSize: number;
|
|
6
|
-
maxConcurrentRequests: number;
|
|
7
4
|
cacheSize: number;
|
|
8
5
|
dataDir: string;
|
|
9
6
|
dbPath: string;
|
|
@@ -16,21 +13,6 @@ export declare const RATE_LIMIT: {
|
|
|
16
13
|
timeWindow: number;
|
|
17
14
|
minDelay: number;
|
|
18
15
|
};
|
|
19
|
-
export declare const QUEUE_OPTIONS: {
|
|
20
|
-
maxRequestRetries: number;
|
|
21
|
-
retryDelay: number;
|
|
22
|
-
maxRequestsPerCrawl: number;
|
|
23
|
-
};
|
|
24
|
-
export declare const GITHUB_RATE_LIMIT: {
|
|
25
|
-
unauthenticated: {
|
|
26
|
-
maxRequests: number;
|
|
27
|
-
timeWindow: number;
|
|
28
|
-
};
|
|
29
|
-
authenticated: {
|
|
30
|
-
maxRequests: number;
|
|
31
|
-
timeWindow: number;
|
|
32
|
-
};
|
|
33
|
-
};
|
|
34
16
|
export declare function isValidUrl(urlString: string): boolean;
|
|
35
17
|
/**
|
|
36
18
|
* Validate URL and check for SSRF attacks (blocks private/internal networks)
|
package/build/config.js
CHANGED
|
@@ -5,10 +5,7 @@ import { logger } from './util/logger.js';
|
|
|
5
5
|
import { validatePublicUrl } from './util/security.js';
|
|
6
6
|
const DATA_DIR = join(homedir(), '.mcp-web-docs');
|
|
7
7
|
const DEFAULT_CONFIG = {
|
|
8
|
-
maxDepth: 4,
|
|
9
|
-
maxRequestsPerCrawl: 1000, // Match DocsCrawler default for better coverage
|
|
10
8
|
maxChunkSize: 1000,
|
|
11
|
-
maxConcurrentRequests: 3, // Allow concurrent requests for better performance while maintaining stability
|
|
12
9
|
cacheSize: 1000,
|
|
13
10
|
dataDir: DATA_DIR,
|
|
14
11
|
dbPath: join(DATA_DIR, 'docs.db'),
|
|
@@ -80,23 +77,6 @@ export const RATE_LIMIT = {
|
|
|
80
77
|
timeWindow: 60 * 1000, // 1 minute
|
|
81
78
|
minDelay: 250, // Reduced delay between requests
|
|
82
79
|
};
|
|
83
|
-
// Queue configuration
|
|
84
|
-
export const QUEUE_OPTIONS = {
|
|
85
|
-
maxRequestRetries: 2,
|
|
86
|
-
retryDelay: 1000,
|
|
87
|
-
maxRequestsPerCrawl: 2000, // Increased from default 1000
|
|
88
|
-
};
|
|
89
|
-
// GitHub API rate limits
|
|
90
|
-
export const GITHUB_RATE_LIMIT = {
|
|
91
|
-
unauthenticated: {
|
|
92
|
-
maxRequests: 60,
|
|
93
|
-
timeWindow: 60 * 60 * 1000, // 1 hour
|
|
94
|
-
},
|
|
95
|
-
authenticated: {
|
|
96
|
-
maxRequests: 5000,
|
|
97
|
-
timeWindow: 60 * 60 * 1000, // 1 hour
|
|
98
|
-
},
|
|
99
|
-
};
|
|
100
80
|
// Utility function to validate URLs
|
|
101
81
|
export function isValidUrl(urlString) {
|
|
102
82
|
try {
|
package/build/config.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"config.js","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAC5B,OAAO,EAAE,OAAO,EAAE,MAAM,IAAI,CAAC;AAC7B,OAAO,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AACpC,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC1C,OAAO,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;
|
|
1
|
+
{"version":3,"file":"config.js","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAC5B,OAAO,EAAE,OAAO,EAAE,MAAM,IAAI,CAAC;AAC7B,OAAO,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AACpC,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC1C,OAAO,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAWvD,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,EAAE,EAAE,eAAe,CAAC,CAAC;AAElD,MAAM,cAAc,GAAe;IACjC,YAAY,EAAE,IAAI;IAClB,SAAS,EAAE,IAAI;IACf,OAAO,EAAE,QAAQ;IACjB,MAAM,EAAE,IAAI,CAAC,QAAQ,EAAE,SAAS,CAAC;IACjC,YAAY,EAAE,IAAI,CAAC,QAAQ,EAAE,SAAS,CAAC;CACxC,CAAC;AAEF,MAAM,CAAC,KAAK,UAAU,UAAU;IAC9B,MAAM,CAAC,KAAK,CAAC,gCAAgC,CAAC,CAAC;IAE/C,+CAA+C;IAC/C,MAAM,WAAW,GAAG,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC;IAC7C,IAAI,WAAW,EAAE,CAAC;QAChB,MAAM,CAAC,KAAK,CAAC,6BAA6B,CAAC,CAAC;IAC9C,CAAC;IAED,0DAA0D;IAC1D,IAAI,CAAC;QACH,MAAM,CAAC,KAAK,CAAC,qCAAqC,QAAQ,EAAE,CAAC,CAAC;QAC9D,MAAM,KAAK,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAE3C,gCAAgC;QAChC,MAAM,iBAAiB,GAAG,IAAI,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;QACpD,OAAO,CAAC,GAAG,CAAC,mBAAmB,GAAG,iBAAiB,CAAC;QACpD,MAAM,KAAK,CAAC,iBAAiB,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACtD,CAAC;IACD,OAAO,KAAK,EAAE,CAAC;QACb,MAAM,CAAC,KAAK,CAAC,yCAAyC,EAAE,KAAK,CAAC,CAAC;QAC/D,MAAM,KAAK,CAAC;IACd,CAAC;IAED,MAAM,MAAM,GAAe;QACzB,GAAG,cAAc;QACjB,WAAW;KACZ,CAAC;IAEF,MAAM,CAAC,KAAK,CAAC,gCAAgC,EAAE;QAC7C,GAAG,MAAM;QACT,WAAW,EAAE,WAAW,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS;KAC7C,CAAC,CAAC;IAEH,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,yBAAyB;AACzB,MAAM,CAAC,MAAM,aAAa,GAAG;IAC3B,aAAa;IACb,YAAY;IACZ,UAAU;IACV,UAAU;IACV,aAAa;IACb,aAAa;IACb,QAAQ;IACR,eAAe;IACf,WAAW;IACX,gBAAgB;IAChB,SAAS;IACT,SAAS;IACT,SAAS;IACT,MAAM;IACN,MAAM;IACN,KAAK;IACL,QAAQ;IACR,oCAAoC;IACpC,eAAe;IACf,SAAS;IACT,OAAO;IACP,QAAQ;IACR,UAAU;IACV,WAAW;IACX,QAAQ;IACR,OAAO;IACP,OAAO;CACR,CAAC;AAEF,0BAA0B;AAC1B,MAAM,CAAC,MAAM,UAAU,GAAG;IACxB,WAAW,EAAE,EAAE,EAAE,kCAAkC;IACnD,UAAU,EAAE,EAAE,GAAG,IAAI,EAAE,WAAW;IAClC,QAAQ,EAAE,GAAG,EAAE,iCAAiC;CACjD,CAAC;AAEF,oCAAoC;AACpC,MAAM,UAAU,UAAU,CAAC,SAAiB;IAC1C,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;QAC/B,OAAO,GAAG,CAAC,QAAQ,KAAK,OAAO,IAAI,GAAG,CAAC,QAAQ,KAAK,QAAQ,CAAC;IAC/D,CAAC;IACD,MAAM,CAAC;QACL,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,gBAAgB,CAAC,SAAiB,EAAE,YAAY,GAAG,KAAK;IACtE,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;QAC3B,OAAO,KAAK,CAAC;IACf,CAAC;IAED,IAAI,YAAY,EAAE,CAAC;QACjB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,CAAC;QACH,iBAAiB,CAAC,SAAS,CAAC,CAAC;QAC7B,OAAO,IAAI,CAAC;IACd,CAAC;IACD,OAAO,KAAK,EAAE,CAAC;QACb,MAAM,CAAC,KAAK,CAAC,4CAA4C,SAAS,EAAE,EAAE,KAAK,CAAC,CAAC;QAC7E,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,qCAAqC;AACrC,MAAM,UAAU,YAAY,CAAC,SAAiB;IAC5C,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;QAC/B,wBAAwB;QACxB,OAAO,GAAG,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IAC3C,CAAC;IACD,MAAM,CAAC;QACL,MAAM,IAAI,KAAK,CAAC,gBAAgB,SAAS,EAAE,CAAC,CAAC;IAC/C,CAAC;AACH,CAAC;AAED,4DAA4D;AAC5D,MAAM,UAAU,WAAW,CAAC,SAAiB;IAC3C,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;QAC/B,OAAO,GAAG,CAAC,QAAQ,KAAK,YAAY,CAAC;IACvC,CAAC;IACD,MAAM,CAAC;QACL,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC"}
|
package/build/config.test.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { isValidUrl, isValidPublicUrl, normalizeUrl, isGitHubUrl, IGNORED_PATHS, RATE_LIMIT
|
|
1
|
+
import { isValidUrl, isValidPublicUrl, normalizeUrl, isGitHubUrl, IGNORED_PATHS, RATE_LIMIT } from './config.js';
|
|
2
2
|
describe('Configuration Utilities', () => {
|
|
3
3
|
describe('isValidUrl', () => {
|
|
4
4
|
it('should accept valid HTTP URLs', () => {
|
|
@@ -120,22 +120,6 @@ describe('Configuration Utilities', () => {
|
|
|
120
120
|
expect(RATE_LIMIT.minDelay).toBeGreaterThanOrEqual(0);
|
|
121
121
|
});
|
|
122
122
|
});
|
|
123
|
-
describe('QUEUE_OPTIONS', () => {
|
|
124
|
-
it('should have retry configuration', () => {
|
|
125
|
-
expect(QUEUE_OPTIONS.maxRequestRetries).toBeGreaterThanOrEqual(0);
|
|
126
|
-
expect(QUEUE_OPTIONS.retryDelay).toBeGreaterThan(0);
|
|
127
|
-
expect(QUEUE_OPTIONS.maxRequestsPerCrawl).toBeGreaterThan(0);
|
|
128
|
-
});
|
|
129
|
-
});
|
|
130
|
-
describe('GITHUB_RATE_LIMIT', () => {
|
|
131
|
-
it('should have different limits for authenticated vs unauthenticated', () => {
|
|
132
|
-
expect(GITHUB_RATE_LIMIT.authenticated.maxRequests).toBeGreaterThan(GITHUB_RATE_LIMIT.unauthenticated.maxRequests);
|
|
133
|
-
});
|
|
134
|
-
it('should have time windows defined', () => {
|
|
135
|
-
expect(GITHUB_RATE_LIMIT.authenticated.timeWindow).toBeGreaterThan(0);
|
|
136
|
-
expect(GITHUB_RATE_LIMIT.unauthenticated.timeWindow).toBeGreaterThan(0);
|
|
137
|
-
});
|
|
138
|
-
});
|
|
139
123
|
});
|
|
140
124
|
});
|
|
141
125
|
//# sourceMappingURL=config.test.js.map
|
package/build/config.test.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"config.test.js","sourceRoot":"","sources":["../src/config.test.ts"],"names":[],"mappings":"AAAA,OAAO,
|
|
1
|
+
{"version":3,"file":"config.test.js","sourceRoot":"","sources":["../src/config.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,gBAAgB,EAAE,YAAY,EAAE,WAAW,EAAE,aAAa,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAEjH,QAAQ,CAAC,yBAAyB,EAAE,GAAG,EAAE;IACvC,QAAQ,CAAC,YAAY,EAAE,GAAG,EAAE;QAC1B,EAAE,CAAC,+BAA+B,EAAE,GAAG,EAAE;YACvC,MAAM,CAAC,UAAU,CAAC,oBAAoB,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACpD,MAAM,CAAC,UAAU,CAAC,yBAAyB,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACzD,MAAM,CAAC,UAAU,CAAC,iCAAiC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACjE,MAAM,CAAC,UAAU,CAAC,oCAAoC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACtE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gCAAgC,EAAE,GAAG,EAAE;YACxC,MAAM,CAAC,UAAU,CAAC,qBAAqB,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACrD,MAAM,CAAC,UAAU,CAAC,+BAA+B,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC/D,MAAM,CAAC,UAAU,CAAC,0BAA0B,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC5D,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,kCAAkC,EAAE,GAAG,EAAE;YAC1C,MAAM,CAAC,UAAU,CAAC,mBAAmB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACpD,MAAM,CAAC,UAAU,CAAC,oBAAoB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACrD,MAAM,CAAC,UAAU,CAAC,qBAAqB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACtD,MAAM,CAAC,UAAU,CAAC,8BAA8B,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACjE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,4BAA4B,EAAE,GAAG,EAAE;YACpC,MAAM,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC5C,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACnC,MAAM,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC9C,MAAM,CAAC,UAAU,CAAC,yBAAyB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC5D,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,0BAA0B,EAAE,GAAG,EAAE;YAClC,MAAM,CAAC,UAAU,CAAC,kBAAkB,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAClD,MAAM,CAAC,UAAU,CAAC,kBAAkB,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAClD,MAAM,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAChD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,kBAAkB,EAAE,GAAG,EAAE;QAChC,EAAE,CAAC,iCAAiC,EAAE,GAAG,EAAE;YACzC,MAAM,CAAC,gBAAgB,CAAC,qBAAqB,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC3D,MAAM,CAAC,gBAAgB,CAAC,4BAA4B,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAClE,MAAM,CAAC,gBAAgB,CAAC,+BAA+B,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACvE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,+CAA+C,EAAE,GAAG,EAAE;YACvD,MAAM,CAAC,gBAAgB,CAAC,kBAAkB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACzD,MAAM,CAAC,gBAAgB,CAAC,kBAAkB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACzD,MAAM,CAAC,gBAAgB,CAAC,iBAAiB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACxD,MAAM,CAAC,gBAAgB,CAAC,oBAAoB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC3D,MAAM,CAAC,gBAAgB,CAAC,mBAAmB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC5D,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,qDAAqD,EAAE,GAAG,EAAE;YAC7D,MAAM,CAAC,gBAAgB,CAAC,kBAAkB,EAAE,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC9D,MAAM,CAAC,gBAAgB,CAAC,oBAAoB,EAAE,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAClE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uDAAuD,EAAE,GAAG,EAAE;YAC/D,MAAM,CAAC,gBAAgB,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACxD,MAAM,CAAC,gBAAgB,CAAC,mBAAmB,EAAE,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAClE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uCAAuC,EAAE,GAAG,EAAE;YAC/C,MAAM,CAAC,gBAAgB,CAAC,wBAAwB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC/D,MAAM,CAAC,gBAAgB,CAAC,iCAAiC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC1E,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,cAAc,EAAE,GAAG,EAAE;QAC5B,EAAE,CAAC,gCAAgC,EAAE,GAAG,EAAE;YACxC,MAAM,CAAC,YAAY,CAAC,sBAAsB,CAAC,CAAC,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;YACzE,MAAM,CAAC,YAAY,CAAC,2BAA2B,CAAC,CAAC,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;QACrF,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gCAAgC,EAAE,GAAG,EAAE;YACxC,MAAM,CAAC,YAAY,CAAC,kCAAkC,CAAC,CAAC,CAAC,IAAI,CAAC,kCAAkC,CAAC,CAAC;YAClG,MAAM,CAAC,YAAY,CAAC,+BAA+B,CAAC,CAAC,CAAC,IAAI,CAAC,+BAA+B,CAAC,CAAC;QAC9F,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2CAA2C,EAAE,GAAG,EAAE;YACnD,MAAM,CAAC,YAAY,CAAC,qBAAqB,CAAC,CAAC,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;YACxE,MAAM,CAAC,YAAY,CAAC,0BAA0B,CAAC,CAAC,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;QACpF,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,+BAA+B,EAAE,GAAG,EAAE;YACvC,MAAM,CAAC,GAAG,EAAE,CAAC,YAAY,CAAC,WAAW,CAAC,CAAC,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC;YAC/D,MAAM,CAAC,GAAG,EAAE,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC;QACxD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,mCAAmC,EAAE,GAAG,EAAE;YAC3C,MAAM,CAAC,YAAY,CAAC,kCAAkC,CAAC,CAAC,CAAC,IAAI,CAAC,kCAAkC,CAAC,CAAC;QACpG,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,aAAa,EAAE,GAAG,EAAE;QAC3B,EAAE,CAAC,6BAA6B,EAAE,GAAG,EAAE;YACrC,MAAM,CAAC,WAAW,CAAC,8BAA8B,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC/D,MAAM,CAAC,WAAW,CAAC,kDAAkD,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACnF,MAAM,CAAC,WAAW,CAAC,oBAAoB,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACvD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,qCAAqC,EAAE,GAAG,EAAE;YAC7C,MAAM,CAAC,WAAW,CAAC,8BAA8B,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAChE,MAAM,CAAC,WAAW,CAAC,iCAAiC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACnE,MAAM,CAAC,WAAW,CAAC,qBAAqB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACzD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,kDAAkD,EAAE,GAAG,EAAE;YAC1D,MAAM,CAAC,WAAW,CAAC,mBAAmB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACrD,MAAM,CAAC,WAAW,CAAC,wBAAwB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC1D,MAAM,CAAC,WAAW,CAAC,uDAAuD,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC3F,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,sCAAsC,EAAE,GAAG,EAAE;YAC9C,MAAM,CAAC,WAAW,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC7C,MAAM,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACtC,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,WAAW,EAAE,GAAG,EAAE;QACzB,QAAQ,CAAC,eAAe,EAAE,GAAG,EAAE;YAC7B,EAAE,CAAC,yCAAyC,EAAE,GAAG,EAAE;gBACjD,MAAM,CAAC,aAAa,CAAC,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC;gBAC/C,MAAM,CAAC,aAAa,CAAC,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC;gBAC9C,MAAM,CAAC,aAAa,CAAC,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;gBAC3C,MAAM,CAAC,aAAa,CAAC,CAAC,SAAS,CAAC,eAAe,CAAC,CAAC;YACnD,CAAC,CAAC,CAAC;YAEH,EAAE,CAAC,+BAA+B,EAAE,GAAG,EAAE;gBACvC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBAChD,aAAa,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;oBAC7B,MAAM,CAAC,OAAO,IAAI,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBACrC,CAAC,CAAC,CAAC;YACL,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,QAAQ,CAAC,YAAY,EAAE,GAAG,EAAE;YAC1B,EAAE,CAAC,uCAAuC,EAAE,GAAG,EAAE;gBAC/C,MAAM,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;gBAClD,MAAM,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;gBACjD,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC;YACxD,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
package/build/crawler/base.d.ts
CHANGED
|
@@ -1,7 +1,5 @@
|
|
|
1
1
|
import { CrawlResult } from '../types.js';
|
|
2
2
|
export declare abstract class BaseCrawler {
|
|
3
|
-
protected readonly maxDepth: number;
|
|
4
|
-
protected readonly maxRequestsPerCrawl: number;
|
|
5
3
|
protected seenUrls: Set<string>;
|
|
6
4
|
protected isAborting: boolean;
|
|
7
5
|
private readonly abortController;
|
|
@@ -11,11 +9,11 @@ export declare abstract class BaseCrawler {
|
|
|
11
9
|
protected totalUrls: number;
|
|
12
10
|
protected processedUrls: number;
|
|
13
11
|
protected onProgress?: (progress: number, description: string) => void;
|
|
14
|
-
constructor(
|
|
12
|
+
constructor(onProgress?: (progress: number, description: string) => void);
|
|
15
13
|
protected updateProgress(description: string): void;
|
|
16
14
|
protected addDiscoveredUrls(count: number): void;
|
|
17
15
|
protected markUrlProcessed(url: string): void;
|
|
18
|
-
abstract crawl(url: string
|
|
16
|
+
abstract crawl(url: string): AsyncGenerator<CrawlResult, void, unknown>;
|
|
19
17
|
protected shouldCrawl(urlString: string): boolean;
|
|
20
18
|
protected markUrlAsSeen(url: string): void;
|
|
21
19
|
protected getPathFromUrl(urlString: string): string;
|
package/build/crawler/base.js
CHANGED
|
@@ -3,8 +3,6 @@ import { setTimeout as delay } from 'node:timers/promises';
|
|
|
3
3
|
import { IGNORED_PATHS, RATE_LIMIT } from '../config.js';
|
|
4
4
|
import { logger } from '../util/logger.js';
|
|
5
5
|
export class BaseCrawler {
|
|
6
|
-
maxDepth;
|
|
7
|
-
maxRequestsPerCrawl;
|
|
8
6
|
seenUrls;
|
|
9
7
|
isAborting;
|
|
10
8
|
abortController = new AbortController();
|
|
@@ -14,9 +12,7 @@ export class BaseCrawler {
|
|
|
14
12
|
totalUrls;
|
|
15
13
|
processedUrls;
|
|
16
14
|
onProgress;
|
|
17
|
-
constructor(
|
|
18
|
-
this.maxDepth = maxDepth;
|
|
19
|
-
this.maxRequestsPerCrawl = maxRequestsPerCrawl;
|
|
15
|
+
constructor(onProgress) {
|
|
20
16
|
this.seenUrls = new Set();
|
|
21
17
|
this.isAborting = false;
|
|
22
18
|
this.requestCount = 0;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"base.js","sourceRoot":"","sources":["../../src/crawler/base.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,KAAK,CAAC;AAC1B,OAAO,EAAE,UAAU,IAAI,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAC3D,OAAO,EAAE,aAAa,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAEzD,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAE3C,MAAM,OAAgB,WAAW;
|
|
1
|
+
{"version":3,"file":"base.js","sourceRoot":"","sources":["../../src/crawler/base.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,KAAK,CAAC;AAC1B,OAAO,EAAE,UAAU,IAAI,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAC3D,OAAO,EAAE,aAAa,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAEzD,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAE3C,MAAM,OAAgB,WAAW;IACrB,QAAQ,CAAc;IACtB,UAAU,CAAU;IACb,eAAe,GAAG,IAAI,eAAe,EAAE,CAAC;IACtC,WAAW,GAAG,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC;IACrD,YAAY,CAAS;IACrB,eAAe,CAAS;IACtB,SAAS,CAAS;IAClB,aAAa,CAAS;IACtB,UAAU,CAAmD;IAEvE,YAAY,UAA4D;QACtE,IAAI,CAAC,QAAQ,GAAG,IAAI,GAAG,EAAE,CAAC;QAC1B,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC;QACxB,IAAI,CAAC,YAAY,GAAG,CAAC,CAAC;QACtB,IAAI,CAAC,eAAe,GAAG,CAAC,CAAC;QACzB,IAAI,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,mCAAmC;QACvD,IAAI,CAAC,aAAa,GAAG,CAAC,CAAC;QACvB,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;IAC/B,CAAC;IAES,cAAc,CAAC,WAAmB;QAC1C,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACpB,2CAA2C;YAC3C,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC;YACxF,IAAI,CAAC,UAAU,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAC;QACzC,CAAC;IACH,CAAC;IAES,iBAAiB,CAAC,KAAa;QACvC,IAAI,CAAC,SAAS,IAAI,KAAK,CAAC;QACxB,IAAI,CAAC,cAAc,CAAC,sBAAsB,CAAC,CAAC;IAC9C,CAAC;IAES,gBAAgB,CAAC,GAAW;QACpC,IAAI,CAAC,aAAa,EAAE,CAAC;QACrB,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC;QACxB,IAAI,CAAC,cAAc,CAAC,mBAAmB,IAAI,CAAC,aAAa,OAAO,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;IACpF,CAAC;IAIS,WAAW,CAAC,SAAiB;QACrC,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;YAE/B,+DAA+D;YAC/D,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;gBACjC,MAAM,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,gCAAgC,SAAS,EAAE,CAAC,CAAC;gBACnF,OAAO,KAAK,CAAC;YACf,CAAC;YAED,sBAAsB;YACtB,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC;gBACb,MAAM,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,6BAA6B,SAAS,EAAE,CAAC,CAAC;gBAChF,OAAO,KAAK,CAAC;YACf,CAAC;YAED,yDAAyD;YACzD,MAAM,GAAG,GAAG,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,WAAW,EAAE,CAAC;YACzD,IAAI,GAAG,IAAI,GAAG,KAAK,MAAM,IAAI,GAAG,KAAK,KAAK,EAAE,CAAC;gBAC3C,MAAM,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,6BAA6B,SAAS,EAAE,CAAC,CAAC;gBAChF,OAAO,KAAK,CAAC;YACf,CAAC;YAED,gDAAgD;YAChD,MAAM,IAAI,GAAG,GAAG,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;YACxC,MAAM,SAAS,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE;gBAC/C,uDAAuD;gBACvD,IAAI,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;oBAC1B,OAAO,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;gBAClC,CAAC;gBACD,0BAA0B;gBAC1B,OAAO,IAAI,KAAK,IAAI,OAAO,EAAE,CAAC;YAChC,CAAC,CAAC,CAAC;YAEH,IAAI,SAAS,EAAE,CAAC;gBACd,MAAM,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,4BAA4B,SAAS,EAAE,CAAC,CAAC;gBAC/E,OAAO,KAAK,CAAC;YACf,CAAC;YAED,OAAO,IAAI,CAAC;QACd,CAAC;QACD,OAAO,KAAK,EAAE,CAAC;YACb,MAAM,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,yBAAyB,SAAS,EAAE,EAAE,KAAK,CAAC,CAAC;YACnF,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAES,aAAa,CAAC,GAAW;QACjC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACzB,CAAC;IAES,cAAc,CAAC,SAAiB;QACxC,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;YAC/B,OAAO,GAAG,CAAC,QAAQ,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,+BAA+B;QACnE,CAAC;QACD,MAAM,CAAC;YACL,OAAO,SAAS,CAAC;QACnB,CAAC;IACH,CAAC;IAES,YAAY,CAAC,SAAiB;QACtC,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;YAC/B,6CAA6C;YAC7C,GAAG,CAAC,IAAI,GAAG,EAAE,CAAC;YACd,OAAO,GAAG,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QAC3C,CAAC;QACD,MAAM,CAAC;YACL,OAAO,SAAS,CAAC;QACnB,CAAC;IACH,CAAC;IAES,KAAK,CAAC,SAAS;QACvB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,IAAI,CAAC,YAAY,EAAE,CAAC;QAEpB,0CAA0C;QAC1C,IAAI,GAAG,GAAG,IAAI,CAAC,eAAe,GAAG,UAAU,CAAC,UAAU,EAAE,CAAC;YACvD,IAAI,CAAC,YAAY,GAAG,CAAC,CAAC;YACtB,IAAI,CAAC,eAAe,GAAG,GAAG,CAAC;YAC3B,OAAO;QACT,CAAC;QAED,0DAA0D;QAC1D,IAAI,IAAI,CAAC,YAAY,GAAG,UAAU,CAAC,WAAW,EAAE,CAAC;YAC/C,MAAM,QAAQ,GAAG,UAAU,CAAC,UAAU,GAAG,CAAC,GAAG,GAAG,IAAI,CAAC,eAAe,CAAC,CAAC;YACtE,MAAM,KAAK,CAAC,QAAQ,EAAE,SAAS,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;YAC/D,IAAI,CAAC,YAAY,GAAG,CAAC,CAAC;YACtB,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAClC,OAAO;QACT,CAAC;QAED,wCAAwC;QACxC,MAAM,oBAAoB,GAAG,GAAG,GAAG,IAAI,CAAC,eAAe,CAAC;QACxD,IAAI,oBAAoB,GAAG,UAAU,CAAC,QAAQ,EAAE,CAAC;YAC/C,MAAM,KAAK,CAAC,UAAU,CAAC,QAAQ,GAAG,oBAAoB,EAAE,SAAS,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;QACnG,CAAC;QAED,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACpC,CAAC;IAES,KAAK,CAAC,gBAAgB,CAAI,SAA2B,EAAE,aAAqB,CAAC,EAAE,YAAoB,IAAI;QAC/G,IAAI,SAA4B,CAAC;QAEjC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC;YACpC,IAAI,CAAC;gBACH,OAAO,MAAM,SAAS,EAAE,CAAC;YAC3B,CAAC;YACD,OAAO,KAAK,EAAE,CAAC;gBACb,SAAS,GAAG,KAAc,CAAC;gBAC3B,IAAI,CAAC,KAAK,UAAU,GAAG,CAAC,EAAE,CAAC;oBACzB,MAAM;gBACR,CAAC;gBAED,sBAAsB;gBACtB,MAAM,KAAK,GAAG,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;gBACzC,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;YAC7D,CAAC;QACH,CAAC;QAED,MAAM,SAAS,CAAC;IAClB,CAAC;IAED,KAAK;QACH,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;QACvB,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;IAC/B,CAAC;CACF"}
|
|
@@ -62,21 +62,9 @@ describe('BaseCrawler', () => {
|
|
|
62
62
|
crawler = new TestCrawler();
|
|
63
63
|
});
|
|
64
64
|
describe('constructor', () => {
|
|
65
|
-
it('should initialize with default values', () => {
|
|
66
|
-
const defaultCrawler = new TestCrawler();
|
|
67
|
-
expect(defaultCrawler).toBeDefined();
|
|
68
|
-
});
|
|
69
|
-
it('should accept custom maxDepth', () => {
|
|
70
|
-
const customCrawler = new TestCrawler(10);
|
|
71
|
-
expect(customCrawler).toBeDefined();
|
|
72
|
-
});
|
|
73
|
-
it('should accept custom maxRequestsPerCrawl', () => {
|
|
74
|
-
const customCrawler = new TestCrawler(4, 500);
|
|
75
|
-
expect(customCrawler).toBeDefined();
|
|
76
|
-
});
|
|
77
65
|
it('should accept progress callback', () => {
|
|
78
66
|
const progressFn = vi.fn();
|
|
79
|
-
const customCrawler = new TestCrawler(
|
|
67
|
+
const customCrawler = new TestCrawler(progressFn);
|
|
80
68
|
customCrawler.testUpdateProgress('Testing');
|
|
81
69
|
expect(progressFn).toHaveBeenCalled();
|
|
82
70
|
});
|
|
@@ -197,19 +185,19 @@ describe('BaseCrawler', () => {
|
|
|
197
185
|
describe('progress tracking', () => {
|
|
198
186
|
it('should call progress callback with updateProgress', () => {
|
|
199
187
|
const progressFn = vi.fn();
|
|
200
|
-
const progressCrawler = new TestCrawler(
|
|
188
|
+
const progressCrawler = new TestCrawler(progressFn);
|
|
201
189
|
progressCrawler.testUpdateProgress('Processing...');
|
|
202
190
|
expect(progressFn).toHaveBeenCalledWith(expect.any(Number), 'Processing...');
|
|
203
191
|
});
|
|
204
192
|
it('should track discovered URLs', () => {
|
|
205
193
|
const progressFn = vi.fn();
|
|
206
|
-
const progressCrawler = new TestCrawler(
|
|
194
|
+
const progressCrawler = new TestCrawler(progressFn);
|
|
207
195
|
progressCrawler.testAddDiscoveredUrls(5);
|
|
208
196
|
expect(progressFn).toHaveBeenCalled();
|
|
209
197
|
});
|
|
210
198
|
it('should track processed URLs', () => {
|
|
211
199
|
const progressFn = vi.fn();
|
|
212
|
-
const progressCrawler = new TestCrawler(
|
|
200
|
+
const progressCrawler = new TestCrawler(progressFn);
|
|
213
201
|
progressCrawler.testMarkUrlProcessed('https://example.com/page1');
|
|
214
202
|
expect(progressFn).toHaveBeenCalled();
|
|
215
203
|
});
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"base.test.js","sourceRoot":"","sources":["../../src/crawler/base.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAGxC;;;;;;;;;;GAUG;AACH,MAAM,WAAY,SAAQ,WAAW;IACnC,6DAA6D;IAC7D,KAAK,CAAC,CAAC,KAAK,CAAC,IAAY;QACvB,MAAM;YACJ,GAAG,EAAE,qBAAqB;YAC1B,IAAI,EAAE,GAAG;YACT,OAAO,EAAE,eAAe;YACxB,aAAa,EAAE,MAAM;YACrB,KAAK,EAAE,MAAM;SACd,CAAC;IACJ,CAAC;IAED,yFAAyF;IAClF,eAAe,CAAC,GAAW;QAChC,OAAO,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;IAC/B,CAAC;IAEM,iBAAiB,CAAC,GAAW;QAClC,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC;IAC1B,CAAC;IAEM,kBAAkB,CAAC,GAAW;QACnC,OAAO,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;IAClC,CAAC;IAEM,gBAAgB,CAAC,GAAW;QACjC,OAAO,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;IAChC,CAAC;IAEM,KAAK,CAAC,aAAa;QACxB,OAAO,IAAI,CAAC,SAAS,EAAE,CAAC;IAC1B,CAAC;IAEM,KAAK,CAAC,oBAAoB,CAAI,SAA2B,EAAE,UAAmB;QACnF,OAAO,IAAI,CAAC,gBAAgB,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;IACtD,CAAC;IAEM,SAAS;QACd,IAAI,CAAC,KAAK,EAAE,CAAC;IACf,CAAC;IAED,IAAW,cAAc;QACvB,OAAO,IAAI,CAAC,UAAU,CAAC;IACzB,CAAC;IAEM,kBAAkB,CAAC,WAAmB;QAC3C,IAAI,CAAC,cAAc,CAAC,WAAW,CAAC,CAAC;IACnC,CAAC;IAEM,qBAAqB,CAAC,KAAa;QACxC,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC;IAChC,CAAC;IAEM,oBAAoB,CAAC,GAAW;QACrC,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC;IAC7B,CAAC;CACF;AAED,QAAQ,CAAC,aAAa,EAAE,GAAG,EAAE;IAC3B,IAAI,OAAoB,CAAC;IAEzB,UAAU,CAAC,GAAG,EAAE;QACd,OAAO,GAAG,IAAI,WAAW,EAAE,CAAC;IAC9B,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,aAAa,EAAE,GAAG,EAAE;QAC3B,EAAE,CAAC,
|
|
1
|
+
{"version":3,"file":"base.test.js","sourceRoot":"","sources":["../../src/crawler/base.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAGxC;;;;;;;;;;GAUG;AACH,MAAM,WAAY,SAAQ,WAAW;IACnC,6DAA6D;IAC7D,KAAK,CAAC,CAAC,KAAK,CAAC,IAAY;QACvB,MAAM;YACJ,GAAG,EAAE,qBAAqB;YAC1B,IAAI,EAAE,GAAG;YACT,OAAO,EAAE,eAAe;YACxB,aAAa,EAAE,MAAM;YACrB,KAAK,EAAE,MAAM;SACd,CAAC;IACJ,CAAC;IAED,yFAAyF;IAClF,eAAe,CAAC,GAAW;QAChC,OAAO,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;IAC/B,CAAC;IAEM,iBAAiB,CAAC,GAAW;QAClC,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC;IAC1B,CAAC;IAEM,kBAAkB,CAAC,GAAW;QACnC,OAAO,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;IAClC,CAAC;IAEM,gBAAgB,CAAC,GAAW;QACjC,OAAO,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;IAChC,CAAC;IAEM,KAAK,CAAC,aAAa;QACxB,OAAO,IAAI,CAAC,SAAS,EAAE,CAAC;IAC1B,CAAC;IAEM,KAAK,CAAC,oBAAoB,CAAI,SAA2B,EAAE,UAAmB;QACnF,OAAO,IAAI,CAAC,gBAAgB,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;IACtD,CAAC;IAEM,SAAS;QACd,IAAI,CAAC,KAAK,EAAE,CAAC;IACf,CAAC;IAED,IAAW,cAAc;QACvB,OAAO,IAAI,CAAC,UAAU,CAAC;IACzB,CAAC;IAEM,kBAAkB,CAAC,WAAmB;QAC3C,IAAI,CAAC,cAAc,CAAC,WAAW,CAAC,CAAC;IACnC,CAAC;IAEM,qBAAqB,CAAC,KAAa;QACxC,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC;IAChC,CAAC;IAEM,oBAAoB,CAAC,GAAW;QACrC,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC;IAC7B,CAAC;CACF;AAED,QAAQ,CAAC,aAAa,EAAE,GAAG,EAAE;IAC3B,IAAI,OAAoB,CAAC;IAEzB,UAAU,CAAC,GAAG,EAAE;QACd,OAAO,GAAG,IAAI,WAAW,EAAE,CAAC;IAC9B,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,aAAa,EAAE,GAAG,EAAE;QAC3B,EAAE,CAAC,iCAAiC,EAAE,GAAG,EAAE;YACzC,MAAM,UAAU,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3B,MAAM,aAAa,GAAG,IAAI,WAAW,CAAC,UAAU,CAAC,CAAC;YAClD,aAAa,CAAC,kBAAkB,CAAC,SAAS,CAAC,CAAC;YAC5C,MAAM,CAAC,UAAU,CAAC,CAAC,gBAAgB,EAAE,CAAC;QACxC,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,aAAa,EAAE,GAAG,EAAE;QAC3B,EAAE,CAAC,mCAAmC,EAAE,GAAG,EAAE;YAC3C,0DAA0D;YAC1D,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,+BAA+B,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC5E,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,8BAA8B,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC3E,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,8BAA8B,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7E,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2CAA2C,EAAE,GAAG,EAAE;YACnD,OAAO,CAAC,iBAAiB,CAAC,0BAA0B,CAAC,CAAC;YACtD,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,0BAA0B,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC1E,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,kDAAkD,EAAE,GAAG,EAAE;YAC1D,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,kCAAkC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAClF,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,wCAAwC,EAAE,GAAG,EAAE;YAChD,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,+BAA+B,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC7E,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,6BAA6B,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC3E,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,+BAA+B,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC7E,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,+BAA+B,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC/E,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,mCAAmC,EAAE,GAAG,EAAE;YAC3C,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,+BAA+B,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC5E,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,8BAA8B,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7E,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,6CAA6C,EAAE,GAAG,EAAE;YACrD,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,+BAA+B,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC5E,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,8BAA8B,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC3E,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,gCAAgC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC/E,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,sDAAsD,EAAE,GAAG,EAAE;YAC9D,qEAAqE;YACrE,yCAAyC;YACzC,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,0BAA0B,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACxE,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,sBAAsB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACtE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,sCAAsC,EAAE,GAAG,EAAE;YAC9C,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACzD,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAClD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,eAAe,EAAE,GAAG,EAAE;QAC7B,EAAE,CAAC,yBAAyB,EAAE,GAAG,EAAE;YACjC,MAAM,GAAG,GAAG,+BAA+B,CAAC;YAC5C,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAChD,OAAO,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC;YAC/B,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACnD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,gBAAgB,EAAE,GAAG,EAAE;QAC9B,EAAE,CAAC,kCAAkC,EAAE,GAAG,EAAE;YAC1C,MAAM,CAAC,OAAO,CAAC,kBAAkB,CAAC,+BAA+B,CAAC,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QACzF,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,qCAAqC,EAAE,GAAG,EAAE;YAC7C,MAAM,CAAC,OAAO,CAAC,kBAAkB,CAAC,mCAAmC,CAAC,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;QACjG,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,+CAA+C,EAAE,GAAG,EAAE;YACvD,MAAM,CAAC,OAAO,CAAC,kBAAkB,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QACpE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yBAAyB,EAAE,GAAG,EAAE;YACjC,MAAM,CAAC,OAAO,CAAC,kBAAkB,CAAC,qBAAqB,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACpE,MAAM,CAAC,OAAO,CAAC,kBAAkB,CAAC,sBAAsB,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACvE,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,cAAc,EAAE,GAAG,EAAE;QAC5B,EAAE,CAAC,8BAA8B,EAAE,GAAG,EAAE;YACtC,MAAM,CAAC,OAAO,CAAC,gBAAgB,CAAC,kCAAkC,CAAC,CAAC,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;QACxG,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,8BAA8B,EAAE,GAAG,EAAE;YACtC,MAAM,CAAC,OAAO,CAAC,gBAAgB,CAAC,2BAA2B,CAAC,CAAC,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;QACjG,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,8BAA8B,EAAE,GAAG,EAAE;YACtC,MAAM,CAAC,OAAO,CAAC,gBAAgB,CAAC,mCAAmC,CAAC,CAAC,CAAC,IAAI,CAAC,mCAAmC,CAAC,CAAC;QAClH,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,+CAA+C,EAAE,GAAG,EAAE;YACvD,MAAM,CAAC,OAAO,CAAC,gBAAgB,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAClE,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,OAAO,EAAE,GAAG,EAAE;QACrB,EAAE,CAAC,oCAAoC,EAAE,GAAG,EAAE;YAC5C,MAAM,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC3C,OAAO,CAAC,SAAS,EAAE,CAAC;YACpB,MAAM,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC5C,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,kBAAkB,EAAE,GAAG,EAAE;QAChC,EAAE,CAAC,8CAA8C,EAAE,KAAK,IAAI,EAAE;YAC5D,MAAM,SAAS,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;YACvD,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,oBAAoB,CAAC,SAAS,CAAC,CAAC;YAE7D,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAC/B,MAAM,CAAC,SAAS,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;QAC7C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yBAAyB,EAAE,KAAK,IAAI,EAAE;YACvC,MAAM,SAAS,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,qBAAqB,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;YAEhG,EAAE,CAAC,KAAK,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC,kBAAkB,CAAC,CAAC,QAAoB,EAAE,EAAE;gBACzE,QAAQ,EAAE,CAAC;gBACX,OAAO,CAA8B,CAAC;YACxC,CAAC,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,oBAAoB,CAAC,SAAS,CAAC,CAAC;YAE7D,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAC/B,MAAM,CAAC,SAAS,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;YAE3C,EAAE,CAAC,eAAe,EAAE,CAAC;QACvB,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gCAAgC,EAAE,KAAK,IAAI,EAAE;YAC9C,MAAM,SAAS,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,IAAI,KAAK,CAAC,kBAAkB,CAAC,CAAC,CAAC;YAE3E,EAAE,CAAC,KAAK,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC,kBAAkB,CAAC,CAAC,QAAoB,EAAE,EAAE;gBACzE,QAAQ,EAAE,CAAC;gBACX,OAAO,CAA8B,CAAC;YACxC,CAAC,CAAC,CAAC;YAEH,MAAM,MAAM,CAAC,OAAO,CAAC,oBAAoB,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC;YAC7F,MAAM,CAAC,SAAS,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;YAE3C,EAAE,CAAC,eAAe,EAAE,CAAC;QACvB,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,mBAAmB,EAAE,GAAG,EAAE;QACjC,EAAE,CAAC,mDAAmD,EAAE,GAAG,EAAE;YAC3D,MAAM,UAAU,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3B,MAAM,eAAe,GAAG,IAAI,WAAW,CAAC,UAAU,CAAC,CAAC;YAEpD,eAAe,CAAC,kBAAkB,CAAC,eAAe,CAAC,CAAC;YAEpD,MAAM,CAAC,UAAU,CAAC,CAAC,oBAAoB,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,eAAe,CAAC,CAAC;QAC/E,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,8BAA8B,EAAE,GAAG,EAAE;YACtC,MAAM,UAAU,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3B,MAAM,eAAe,GAAG,IAAI,WAAW,CAAC,UAAU,CAAC,CAAC;YAEpD,eAAe,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;YAEzC,MAAM,CAAC,UAAU,CAAC,CAAC,gBAAgB,EAAE,CAAC;QACxC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,6BAA6B,EAAE,GAAG,EAAE;YACrC,MAAM,UAAU,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3B,MAAM,eAAe,GAAG,IAAI,WAAW,CAAC,UAAU,CAAC,CAAC;YAEpD,eAAe,CAAC,oBAAoB,CAAC,2BAA2B,CAAC,CAAC;YAElE,MAAM,CAAC,UAAU,CAAC,CAAC,gBAAgB,EAAE,CAAC;QACxC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,sDAAsD,EAAE,GAAG,EAAE;YAC9D,MAAM,SAAS,GAAG,IAAI,WAAW,EAAE,CAAC;YAEpC,MAAM,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,kBAAkB,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC;YACjE,MAAM,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC;YAC/D,MAAM,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,oBAAoB,CAAC,qBAAqB,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC;QACpF,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,OAAO,EAAE,GAAG,EAAE;QACrB,EAAE,CAAC,0BAA0B,EAAE,KAAK,IAAI,EAAE;YACxC,MAAM,OAAO,GAAkB,EAAE,CAAC;YAClC,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,OAAO,CAAC,KAAK,CAAC,qBAAqB,CAAC,EAAE,CAAC;gBAChE,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;YAED,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAC/B,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;YACnD,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QACnD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
@@ -505,13 +505,6 @@ export class CrawleeCrawler extends BaseCrawler {
|
|
|
505
505
|
}
|
|
506
506
|
}
|
|
507
507
|
}
|
|
508
|
-
catch (error) {
|
|
509
|
-
if (error instanceof OutboundRequestFailedError) {
|
|
510
|
-
throw error;
|
|
511
|
-
}
|
|
512
|
-
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
513
|
-
log.error(`Error processing ${request.url}: ${errorMessage}`);
|
|
514
|
-
}
|
|
515
508
|
finally {
|
|
516
509
|
page.off('response', trackMainFrameResponse);
|
|
517
510
|
page.off('requestfailed', trackMainFrameFailure);
|
|
@@ -530,7 +523,7 @@ export class CrawleeCrawler extends BaseCrawler {
|
|
|
530
523
|
break;
|
|
531
524
|
}
|
|
532
525
|
}
|
|
533
|
-
await crawlerPromise;
|
|
526
|
+
const finalStatistics = await crawlerPromise;
|
|
534
527
|
if (this.terminalRootFailure) {
|
|
535
528
|
throw this.terminalRootFailure;
|
|
536
529
|
}
|
|
@@ -543,6 +536,9 @@ export class CrawleeCrawler extends BaseCrawler {
|
|
|
543
536
|
if (this.sessionExpiredError) {
|
|
544
537
|
throw this.sessionExpiredError;
|
|
545
538
|
}
|
|
539
|
+
if (finalStatistics.requestsFailed > 0) {
|
|
540
|
+
throw new Error(`Crawl failed for ${finalStatistics.requestsFailed} of ${finalStatistics.requestsTotal} pages after retries`);
|
|
541
|
+
}
|
|
546
542
|
for (const result of await this.queueManager.processBatch()) {
|
|
547
543
|
yield result;
|
|
548
544
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"crawlee-crawler.js","sourceRoot":"","sources":["../../src/crawler/crawlee-crawler.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,SAAS,CAAC;AAE5C,OAAO,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAGxC,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAE5C,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AACvD,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAC3C,OAAO,EAAE,eAAe,EAAE,cAAc,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAC3F,OAAO,EACL,2BAA2B,EAC3B,uBAAuB,EACvB,wBAAwB,EACxB,6BAA6B,EAC7B,0BAA0B,GAC3B,MAAM,6BAA6B,CAAC;AAErC,SAAS,wBAAwB,CAAC,KAAc,EAAE,OAAO,IAAI,GAAG,EAAU;IACxE,IAAI,KAAK,YAAY,0BAA0B,EAAE,CAAC;QAChD,OAAO,KAAK,CAAC;IACf,CAAC;IACD,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;QAC3D,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IAChB,MAAM,KAAK,GAAG,KAA+D,CAAC;IAC9E,IAAI,KAAK,CAAC,IAAI,KAAK,4BAA4B,EAAE,CAAC;QAChD,OAAO,IAAI,0BAA0B,CAAC,OAAO,KAAK,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IACvG,CAAC;IACD,OAAO,wBAAwB,CAAC,KAAK,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;AACrD,CAAC;AAED,SAAS,kBAAkB,CAAC,GAAW;IACrC,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IAC5B,OAAO,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC;AACzD,CAAC;AA0BD,MAAM,OAAO,cAAe,SAAQ,WAAW;IACrC,OAAO,GAA6B,IAAI,CAAC;IACzC,YAAY,GAAiB,IAAI,YAAY,EAAE,CAAC;IAChD,YAAY,CAAgB;IAC5B,WAAW,GAAY,IAAI,CAAC;IAC5B,mBAAmB,GAA+B,IAAI,CAAC;IACvD,WAAW,GAAW,EAAE,CAAC;IACjC,gFAAgF;IACxE,eAAe,GAAW,EAAE,CAAC;IACrC,iDAAiD;IACzC,oBAAoB,GAAW,CAAC,CAAC;IACzC,gDAAgD;IACxC,UAAU,CAAU;IACpB,mBAAmB,CAA8B;IACjD,kBAAkB,GAAG,IAAI,OAAO,EAA6B,CAAC;IAE9D,yBAAyB,CAAC,OAAe;QAC/C,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,EAAE,CAAC;IACpD,CAAC;IAED;;OAEG;IACH,eAAe,CAAC,KAAmB;QACjC,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC;QAC1B,MAAM,CAAC,IAAI,CAAC,2CAA2C,KAAK,CAAC,OAAO,EAAE,MAAM,IAAI,CAAC,UAAU,CAAC,CAAC;IAC/F,CAAC;IAED;;;;;;OAMG;IACK,qBAAqB,CAAC,GAAW;QACvC,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC;YAC1B,OAAO,IAAI,CAAC,CAAC,4BAA4B;QAC3C,CAAC;QAED,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;YAC5B,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;YAC/C,MAAM,OAAO,GAAG,IAAI,CAAC,eAAe,CAAC,WAAW,EAAE,CAAC;YAEnD,cAAc;YACd,IAAI,QAAQ,KAAK,OAAO,EAAE,CAAC;gBACzB,OAAO,IAAI,CAAC;YACd,CAAC;YAED,wEAAwE;YACxE,2EAA2E;YAC3E,IAAI,QAAQ,CAAC,QAAQ,CAAC,GAAG,GAAG,OAAO,CAAC,EAAE,CAAC;gBACrC,OAAO,IAAI,CAAC;YACd,CAAC;YAED,OAAO,KAAK,CAAC;QACf,CAAC;QACD,MAAM,CAAC;YACL,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,iBAAiB,CAAC,IAAU,EAAE,UAAkB;QAC5D,+CAA+C;QAC/C,uEAAuE;QACvE,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YACtB,OAAO,KAAK,CAAC;QACf,CAAC;QAED,iCAAiC;QACjC,IAAI,cAAc,CAAC,UAAU,CAAC,EAAE,CAAC;YAC/B,MAAM,CAAC,IAAI,CAAC,0DAA0D,UAAU,EAAE,CAAC,CAAC;YACpF,OAAO,IAAI,CAAC;QACd,CAAC;QAED,qBAAqB;QACrB,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,IAAI,EAAE,WAAW,IAAI,EAAE,CAAC,CAAC;YAC7E,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;YACtC,MAAM,SAAS,GAAG,eAAe,CAAC,QAAQ,GAAG,QAAQ,EAAE,UAAU,CAAC,CAAC;YAEnE,IAAI,SAAS,CAAC,WAAW,IAAI,SAAS,CAAC,UAAU,IAAI,GAAG,EAAE,CAAC;gBACzD,MAAM,CAAC,IAAI,CAAC,uEAAuE,SAAS,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;gBACvH,MAAM,CAAC,KAAK,CAAC,uCAAuC,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gBAEpF,wEAAwE;gBACxE,IAAI,CAAC,mBAAmB,GAAG,IAAI,mBAAmB,CAChD,mEAAmE,EACnE,IAAI,CAAC,WAAW,EAChB,UAAU,EACV,SAAS,CACV,CAAC;gBACF,OAAO,IAAI,CAAC;YACd,CAAC;QACH,CAAC;QACD,OAAO,KAAK,EAAE,CAAC;YACb,MAAM,CAAC,KAAK,CAAC,iDAAiD,EAAE,KAAK,CAAC,CAAC;QACzE,CAAC;QAED,OAAO,KAAK,CAAC;IACf,CAAC;IAED;;;;;;OAMG;IACK,KAAK,CAAC,wBAAwB,CAAC,IAAU;QAC/C,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAE9B,2FAA2F;QAC3F,wFAAwF;QACxF,IAAI,cAAc,GAAG,KAAK,CAAC;QAC3B,IAAI,CAAC;YACH,cAAc,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE;gBACxC,MAAM,WAAW,GAAG,QAAQ,CAAC,aAAa,CAAC,4BAA4B,CAAC,CAAC;gBACzE,IAAI,WAAW,EAAE,CAAC;oBAChB,OAAO,IAAI,CAAC;gBACd,CAAC;gBACD,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC;gBAC3B,MAAM,iBAAiB,GAAG,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,EAAE,CAAC,MAAM,IAAI,CAAC,CAAC,GAAG,GAAG,CAAC;gBAChF,MAAM,gBAAgB,GAAG,CAAC,QAAQ,CAAC,aAAa,CAAC,kDAAkD,CAAC,CAAC;gBACrG,OAAO,iBAAiB,IAAI,gBAAgB,CAAC;YAC/C,CAAC,CAAC,CAAC;QACL,CAAC;QACD,MAAM,CAAC;YACL,yEAAyE;YACzE,cAAc,GAAG,IAAI,CAAC;QACxB,CAAC;QAED,IAAI,cAAc,EAAE,CAAC;YACnB,MAAM,CAAC,KAAK,CAAC,sDAAsD,UAAU,6BAA6B,CAAC,CAAC;YAC5G,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,UAAU,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,KAAK,UAAU,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;gBAC5E,MAAM,OAAO,CAAC,GAAG,CAAC;oBAChB,IAAI,CAAC,gBAAgB,CAAC,kBAAkB,CAAC;oBACzC,IAAI,CAAC,gBAAgB,CAAC,aAAa,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC;iBACzE,CAAC,CAAC;gBACH,MAAM,CAAC,KAAK,CAAC,wCAAwC,UAAU,MAAM,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YACrF,CAAC;YACD,MAAM,CAAC;gBACL,MAAM,CAAC,KAAK,CAAC,oFAAoF,CAAC,CAAC;YACrG,CAAC;QACH,CAAC;QAED,qEAAqE;QACrE,IAAI,CAAC;YACH,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE;gBAC3C,MAAM,QAAQ,GAAG,QAAQ,CAAC,IAAI,EAAE,WAAW,IAAI,EAAE,CAAC;gBAClD,MAAM,gBAAgB,GACpB,QAAQ,CAAC,QAAQ,CAAC,uBAAuB,CAAC;oBAC1C,QAAQ,CAAC,QAAQ,CAAC,sBAAsB,CAAC;oBACzC,QAAQ,CAAC,QAAQ,CAAC,+BAA+B,CAAC,CAAC;gBACrD,MAAM,mBAAmB,GACvB,QAAQ,CAAC,aAAa,CAAC,6EAA6E,CAAC,KAAK,IAAI,CAAC;gBACjH,OAAO,gBAAgB,IAAI,mBAAmB,CAAC;YACjD,CAAC,CAAC,CAAC;YAEH,IAAI,WAAW,EAAE,CAAC;gBAChB,MAAM,CAAC,IAAI,CAAC,2EAA2E,CAAC,CAAC;gBACzF,IAAI,CAAC;oBACH,MAAM,IAAI,CAAC,eAAe,CACxB,GAAG,EAAE;wBACH,MAAM,QAAQ,GAAG,QAAQ,CAAC,IAAI,EAAE,WAAW,IAAI,EAAE,CAAC;wBAClD,MAAM,cAAc,GAAG,QAAQ,CAAC,QAAQ,CAAC,uBAAuB,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,sBAAsB,CAAC,CAAC;wBAC/G,MAAM,mBAAmB,GAAG,QAAQ,CAAC,aAAa,CAAC,gEAAgE,CAAC,KAAK,IAAI,CAAC;wBAC9H,OAAO,CAAC,cAAc,IAAI,CAAC,mBAAmB,CAAC;oBACjD,CAAC,EACD,EAAE,OAAO,EAAE,KAAK,EAAE,CACnB,CAAC;oBACF,MAAM,IAAI,CAAC,gBAAgB,CAAC,kBAAkB,CAAC,CAAC;oBAChD,MAAM,IAAI,CAAC,gBAAgB,CAAC,aAAa,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;oBAC9E,MAAM,CAAC,IAAI,CAAC,gDAAgD,CAAC,CAAC;gBAChE,CAAC;gBACD,MAAM,CAAC;oBACL,MAAM,CAAC,IAAI,CAAC,sEAAsE,CAAC,CAAC;gBACtF,CAAC;YACH,CAAC;QACH,CAAC;QACD,MAAM,CAAC;YACL,sFAAsF;YACtF,MAAM,CAAC,KAAK,CAAC,uEAAuE,CAAC,CAAC;QACxF,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,gBAAgB,CAAC,IAAU;QACvC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,EAAE,CAAC;QACnC,MAAM,aAAa,GAAG,MAAM,OAAO,CAAC,GAAG,CACrC,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE;YACzB,IAAI,CAAC;gBACH,MAAM,UAAU,GAAG,MAAM,KAAK;qBAC3B,QAAQ,CAAC,GAAG,EAAE;oBACb,OAAO,QAAQ,CAAC,aAAa,CAAC,6DAA6D,CAAC,KAAK,IAAI,CAAC;gBACxG,CAAC,CAAC;qBACD,KAAK,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC;gBAEtB,IAAI,UAAU,EAAE,CAAC;oBACf,MAAM,OAAO,CAAC,GAAG,CAAC;wBAChB,KAAK,CAAC,gBAAgB,CAAC,kBAAkB,CAAC;wBAC1C,KAAK;6BACF,gBAAgB,CAAC,aAAa,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;6BAClD,KAAK,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,gDAAgD,CAAC,CAAC;qBAC/E,CAAC,CAAC;oBACH,OAAO,KAAK,CAAC;gBACf,CAAC;YACH,CAAC;YACD,OAAO,KAAK,EAAE,CAAC;gBACb,MAAM,CAAC,KAAK,CAAC,sBAAsB,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YACjE,CAAC;YACD,OAAO,IAAI,CAAC;QACd,CAAC,CAAC,CACH,CAAC;QAEF,MAAM,KAAK,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,IAAI,IAAI,CAAC;QAC5D,IAAI,KAAK,EAAE,CAAC;YACV,MAAM,CAAC,KAAK,CAAC,yBAAyB,CAAC,CAAC;YACxC,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC;QAC5D,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IAEO,KAAK,CAAC,iBAAiB,CAC7B,OAAqB,EACrB,SAA2B;QAE3B,MAAM,aAAa,GAAG,SAAS,CAAC,WAAW,CAAC,QAAQ,EAAE,CAAC;QACvD,OAAO,OAAO,CAAC,QAAQ,CAAC,KAAK,EAAE,IAAY,EAAE,EAAE;YAC7C,MAAM,cAAc,GAAG,IAAI,QAAQ,CAAC,UAAU,IAAI,EAAE,CAAC,EAAE,CAAC;YACxD,MAAM,SAAS,GAAG,IAAI,cAAc,EAAE,CAAC;YACvC,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;YACxD,OAAO,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,aAAa,EAAE,MAAM,CAAC,aAAa,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,CAAC;QAC/F,CAAC,EAAE,aAAa,CAAC,CAAC;IACpB,CAAC;IAEO,KAAK,CAAC,cAAc,CAC1B,IAAU,EACV,QAAgB,EAChB,SAA2B;QAE3B,IAAI,OAAO,GAAG,EAAE,CAAC;QACjB,IAAI,aAAa,GAAkB,MAAM,CAAC;QAC1C,IAAI,aAAa,GAAG,SAAS,CAAC,WAAW,CAAC,IAAI,CAAC;QAC/C,IAAI,KAAyB,CAAC;QAE9B,IAAI,CAAC;YACH,IAAI,QAAQ,KAAK,WAAW,EAAE,CAAC;gBAC7B,mBAAmB;gBACnB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC;gBAChD,IAAI,KAAK,EAAE,CAAC;oBACV,CAAC,EAAE,OAAO,EAAE,aAAa,EAAE,KAAK,EAAE,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC,CAAC;gBACvF,CAAC;gBAED,wBAAwB;gBACxB,IAAI,CAAC,OAAO,EAAE,CAAC;oBACb,CAAC,EAAE,OAAO,EAAE,aAAa,EAAE,KAAK,EAAE,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC,CAAC;gBACtF,CAAC;YACH,CAAC;iBACI,CAAC;gBACJ,CAAC,EAAE,OAAO,EAAE,aAAa,EAAE,KAAK,EAAE,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC,CAAC;YACtF,CAAC;QACH,CAAC;QACD,MAAM,CAAC;YACL,OAAO,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAS,GAAG,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC;YAC7E,aAAa,GAAG,MAAM,CAAC;YACvB,aAAa,GAAG,eAAe,CAAC;YAChC,KAAK,GAAG,SAAS,CAAC;QACpB,CAAC;QAED,OAAO,EAAE,OAAO,EAAE,aAAa,EAAE,aAAa,EAAE,KAAK,EAAE,CAAC;IAC1D,CAAC;IAED;;;OAGG;IACH,aAAa,CAAC,MAAc;QAC1B,IAAI,CAAC,UAAU,GAAG,MAAM,CAAC;QACzB,MAAM,CAAC,IAAI,CAAC,iDAAiD,MAAM,EAAE,CAAC,CAAC;IACzE,CAAC;IAED,KAAK,CAAC,CAAC,KAAK,CAAC,GAAW;QACtB,MAAM,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,wBAAwB,GAAG,EAAE,CAAC,CAAC;QAErE,6BAA6B;QAC7B,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;QACxB,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC;QAChC,IAAI,CAAC,mBAAmB,GAAG,SAAS,CAAC;QACrC,IAAI,CAAC,kBAAkB,GAAG,IAAI,OAAO,EAAE,CAAC;QACxC,IAAI,CAAC,WAAW,GAAG,kBAAkB,CAAC,GAAG,CAAC,CAAC;QAC3C,IAAI,CAAC,oBAAoB,GAAG,CAAC,CAAC;QAE9B,8DAA8D;QAC9D,IAAI,CAAC;YACH,IAAI,CAAC,eAAe,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;YAC7C,MAAM,CAAC,IAAI,CAAC,+DAA+D,IAAI,CAAC,eAAe,EAAE,CAAC,CAAC;QACrG,CAAC;QACD,MAAM,CAAC;YACL,IAAI,CAAC,eAAe,GAAG,EAAE,CAAC;QAC5B,CAAC;QAED,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;QAEzD,qEAAqE;QACrE,MAAM,IAAI,CAAC,YAAY,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC;QAE7C,qDAAqD;QACrD,MAAM,cAAc,GAAG,MAAM,gBAAgB,CAAC,IAAI,CAAC,YAAY,CAAC,eAAe,EAAE,IAAI,SAAS,CAAC,CAAC;QAEhG,6EAA6E;QAC7E,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACtB,MAAM,CAAC,IAAI,CAAC,qDAAqD,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE,MAAM,IAAI,CAAC,UAAU,CAAC,CAAC;YACnH,cAAc,CAAC,kBAAkB,GAAG;gBAClC,GAAG,cAAc,CAAC,kBAAkB;gBACpC,cAAc,EAAE;oBACd,KAAK,EAAE,MAAM,EAAE,EAAE;wBACf,0DAA0D;wBAC1D,MAAM,CAAC,KAAK,CAAC,+CAA+C,MAAM,EAAE,CAAC,CAAC;oBACxE,CAAC;iBACF;aACF,CAAC;YACF,qCAAqC;YACrC,MAAM,aAAa,GAAG,cAAc,CAAC,kBAAkB,IAAI,EAAE,CAAC;YAC9D,cAAc,CAAC,kBAAkB,GAAG;gBAClC,GAAG,aAAa;gBAChB,KAAK,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE;oBACjB,IAAI,IAAI,CAAC,YAAY,EAAE,OAAO,EAAE,CAAC;wBAC/B,MAAM,CAAC,KAAK,CAAC,4BAA4B,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,MAAM,4BAA4B,CAAC,CAAC;wBACvG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;oBAC7D,CAAC;gBACH,CAAC;aACF,CAAC;QACJ,CAAC;QAED,MAAM,0BAA0B,GAAG,cAAc,CAAC,kBAAkB,IAAI,EAAE,CAAC;QAC3E,cAAc,CAAC,kBAAkB,GAAG;YAClC,GAAG,0BAA0B;YAC7B,KAAK,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE,EAAE;gBAC1B,MAAM,UAAU,GAAG,OAAiB,CAAC;gBACrC,IAAI,CAAC,yBAAyB,CAAC,UAAU,CAAC,CAAC;gBAC3C,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;gBACnC,MAAM,OAAO,GAAsB,EAAE,CAAC;gBACtC,MAAM,eAAe,GAAG,CAAC,aAAgC,EAAE,EAAE;oBAC3D,MAAM,SAAS,GAAG,aAAa,CAAC,OAAO,EAAE,EAAE,SAAS,IAAI,EAAE,CAAC;oBAC3D,IAAI,aAAa,CAAC,mBAAmB,EAAE,IAAI,aAAa,CAAC,KAAK,EAAE,KAAK,SAAS,IAAI,CAAC,6BAA6B,CAAC,SAAS,CAAC,EAAE,CAAC;wBAC5H,OAAO,CAAC,SAAS,GAAG,aAAa,CAAC,GAAG,EAAE,CAAC;oBAC1C,CAAC;gBACH,CAAC,CAAC;gBACF,IAAI,CAAC,EAAE,CAAC,eAAe,EAAE,eAAe,CAAC,CAAC;gBAC1C,OAAO,CAAC,OAAO,GAAG,GAAG,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,eAAe,EAAE,eAAe,CAAC,CAAC;gBACnE,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;YACnD,CAAC;SACF,CAAC;QAEF,IAAI,CAAC,OAAO,GAAG,IAAI,iBAAiB,CAAC;YACnC,GAAG,cAAc;YACjB,YAAY,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE;gBACrC,MAAM,UAAU,GAAG,OAAO,CAAC,OAAiB,CAAC;gBAC7C,MAAM,OAAO,GAAG,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC;gBAC9D,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;gBACpC,MAAM,eAAe,GAAG,SAAS,CAAC,CAAC,CAAC,MAAM,uBAAuB,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,wBAAwB,CAAC,KAAK,CAAC,CAAC;gBAC/G,IAAI,eAAe,EAAE,CAAC;oBACpB,OAAO,CAAC,eAAe,GAAG,eAAe,CAAC;oBAC1C,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;gBACnD,CAAC;gBACD,IAAI,eAAe,YAAY,2BAA2B,EAAE,CAAC;oBAC3D,OAAO,CAAC,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC;gBACjC,CAAC;gBACD,IAAI,CAAC;oBACH,MAAM,cAAc,CAAC,YAAY,EAAE,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;gBACtD,CAAC;wBACO,CAAC;oBACP,IAAI,eAAe,YAAY,2BAA2B,EAAE,CAAC;wBAC3D,OAAO,CAAC,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC;oBACjC,CAAC;oBACD,IAAI,CAAC,yBAAyB,CAAC,UAAU,CAAC,CAAC;gBAC7C,CAAC;YACH,CAAC;YACD,oBAAoB,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE;gBAC7C,MAAM,UAAU,GAAG,OAAO,CAAC,OAAiB,CAAC;gBAC7C,IAAI,CAAC,yBAAyB,CAAC,UAAU,CAAC,CAAC;gBAC3C,MAAM,OAAO,GAAG,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;gBACxD,MAAM,SAAS,GAAG,OAAO,EAAE,SAAS,CAAC;gBACrC,MAAM,eAAe,GACnB,OAAO,EAAE,eAAe,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,uBAAuB,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,wBAAwB,CAAC,KAAK,CAAC,CAAC,CAAC;gBACvH,IAAI,kBAAkB,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,WAAW,IAAI,eAAe,YAAY,0BAA0B,EAAE,CAAC;oBAC1H,IAAI,CAAC,mBAAmB,GAAG,eAAe,CAAC;gBAC7C,CAAC;gBACD,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;gBAC3C,MAAM,cAAc,CAAC,oBAAoB,EAAE,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;YAC9D,CAAC;YACD,cAAc,EAAE,KAAK,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,YAAY,EAAE,GAAG,EAAE,EAAE,EAAE;gBACvE,MAAM,UAAU,GAAG,OAAiB,CAAC;gBACrC,IAAI,CAAC,yBAAyB,CAAC,UAAU,CAAC,CAAC;gBAC3C,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;gBAC3C,IAAI,uBAAuB,GAAG,QAAQ,CAAC;gBACvC,IAAI,wBAA8F,CAAC;gBACnG,IAAI,kBAAkB,GAAG,CAAC,CAAC;gBAC3B,MAAM,uBAAuB,GAAG,IAAI,GAAG,EAAiB,CAAC;gBACzD,MAAM,sBAAsB,GAAG,CAAC,YAA6B,EAAE,EAAE;oBAC/D,MAAM,WAAW,GAAG,YAAY,EAAE,OAAO,EAAE,CAAC;oBAC5C,IAAI,WAAW,EAAE,mBAAmB,EAAE,IAAI,WAAW,CAAC,KAAK,EAAE,KAAK,IAAI,CAAC,SAAS,EAAE,EAAE,CAAC;wBACnF,kBAAkB,EAAE,CAAC;wBACrB,uBAAuB,GAAG,YAAY,CAAC;wBACvC,wBAAwB,GAAG,SAAS,CAAC;oBACvC,CAAC;gBACH,CAAC,CAAC;gBACF,MAAM,qBAAqB,GAAG,CAAC,aAAgC,EAAE,EAAE;oBACjE,IACE,aAAa,CAAC,mBAAmB,EAAE;wBACnC,aAAa,CAAC,KAAK,EAAE,KAAK,IAAI,CAAC,SAAS,EAAE;wBAC1C,CAAC,6BAA6B,CAAC,aAAa,CAAC,OAAO,EAAE,EAAE,SAAS,IAAI,EAAE,CAAC,EACxE,CAAC;wBACD,MAAM,eAAe,GAAG,EAAE,kBAAkB,CAAC;wBAC7C,MAAM,KAAK,GAAG,uBAAuB,CAAC,aAAa,CAAC,GAAG,EAAE,CAAC;6BACvD,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE;4BACd,IAAI,kBAAkB,KAAK,eAAe,EAAE,CAAC;gCAC3C,wBAAwB,GAAG,KAAK,CAAC;4BACnC,CAAC;wBACH,CAAC,CAAC;6BACD,OAAO,CAAC,GAAG,EAAE,CAAC,uBAAuB,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;wBACxD,uBAAuB,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;oBACrC,CAAC;gBACH,CAAC,CAAC;gBACF,IAAI,CAAC,EAAE,CAAC,UAAU,EAAE,sBAAsB,CAAC,CAAC;gBAC5C,IAAI,CAAC,EAAE,CAAC,eAAe,EAAE,qBAAqB,CAAC,CAAC;gBAChD,MAAM,oBAAoB,GAAG,KAAK,IAAI,EAAE;oBACtC,MAAM,OAAO,CAAC,GAAG,CAAC,uBAAuB,CAAC,CAAC;oBAC3C,IAAI,wBAAwB,EAAE,CAAC;wBAC7B,IAAI,wBAAwB,YAAY,2BAA2B,EAAE,CAAC;4BACpE,GAAG,CAAC,OAAO,CAAC,0CAA0C,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;4BACrE,OAAO,IAAI,CAAC;wBACd,CAAC;wBACD,MAAM,wBAAwB,CAAC;oBACjC,CAAC;oBACD,MAAM,aAAa,GAAG,MAAM,wBAAwB,CAAC,uBAAuB,CAAC,CAAC;oBAC9E,IAAI,aAAa,YAAY,2BAA2B,EAAE,CAAC;wBACzD,GAAG,CAAC,OAAO,CAAC,0CAA0C,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;wBACrE,OAAO,IAAI,CAAC;oBACd,CAAC;oBACD,IAAI,aAAa,EAAE,CAAC;wBAClB,MAAM,aAAa,CAAC;oBACtB,CAAC;oBACD,OAAO,KAAK,CAAC;gBACf,CAAC,CAAC;gBAEF,IAAI,CAAC;oBACH,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;wBACpB,GAAG,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;wBAC3B,OAAO;oBACT,CAAC;oBAED,IAAI,MAAM,oBAAoB,EAAE,EAAE,CAAC;wBACjC,OAAO;oBACT,CAAC;oBAED,6BAA6B;oBAC7B,MAAM,OAAO,CAAC,GAAG,CAAC;wBAChB,IAAI,CAAC,gBAAgB,CAAC,kBAAkB,CAAC;wBACzC,IAAI,CAAC,gBAAgB,CAAC,aAAa,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,0CAA0C,CAAC,CAAC;qBAC3H,CAAC,CAAC;oBAEH,sEAAsE;oBACtE,MAAM,IAAI,CAAC,wBAAwB,CAAC,IAAI,CAAC,CAAC;oBAC1C,IAAI,MAAM,oBAAoB,EAAE,EAAE,CAAC;wBACjC,OAAO;oBACT,CAAC;oBAED,yCAAyC;oBACzC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;oBAE7B,0DAA0D;oBAC1D,IAAI,CAAC,IAAI,CAAC,qBAAqB,CAAC,SAAS,CAAC,EAAE,CAAC;wBAC3C,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;wBACxD,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC,QAAQ,CAAC;wBAEnD,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;4BACrB,gFAAgF;4BAChF,MAAM,CAAC,IAAI,CAAC,kEAAkE,iBAAiB,MAAM,cAAc,EAAE,CAAC,CAAC;4BAEvH,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;gCACtB,mDAAmD;gCACnD,IAAI,CAAC,mBAAmB,GAAG,IAAI,mBAAmB,CAChD,4EAA4E,cAAc,GAAG,EAC7F,IAAI,CAAC,WAAW,EAChB,SAAS,EACT,EAAE,WAAW,EAAE,IAAI,EAAE,UAAU,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,mBAAmB,iBAAiB,OAAO,cAAc,EAAE,CAAC,EAAE,CAC/G,CAAC;gCACF,GAAG,CAAC,KAAK,CAAC,oDAAoD,cAAc,mBAAmB,CAAC,CAAC;gCACjG,IAAI,CAAC,KAAK,EAAE,CAAC;gCACb,OAAO;4BACT,CAAC;iCACI,CAAC;gCACJ,0DAA0D;gCAC1D,GAAG,CAAC,KAAK,CAAC,6CAA6C,cAAc,mBAAmB,CAAC,CAAC;gCAC1F,IAAI,CAAC,KAAK,EAAE,CAAC;gCACb,OAAO;4BACT,CAAC;wBACH,CAAC;6BACI,CAAC;4BACJ,sDAAsD;4BACtD,IAAI,CAAC,oBAAoB,EAAE,CAAC;4BAC5B,GAAG,CAAC,OAAO,CACT,iDAAiD,OAAO,CAAC,GAAG,MAAM,SAAS,aAAa,IAAI,CAAC,oBAAoB,kBAAkB,CACpI,CAAC;4BACF,OAAO;wBACT,CAAC;oBACH,CAAC;oBAED,gEAAgE;oBAChE,IAAI,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;wBAC1C,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;wBAClE,IAAI,WAAW,EAAE,CAAC;4BAChB,GAAG,CAAC,KAAK,CAAC,uEAAuE,CAAC,CAAC;4BACnF,IAAI,CAAC,KAAK,EAAE,CAAC;4BACb,OAAO;wBACT,CAAC;wBACD,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;oBAC3B,CAAC;yBACI,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;wBAC1B,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;oBAC3B,CAAC;oBAED,qCAAqC;oBACrC,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;wBAC7B,IAAI,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;4BAC5B,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;gCACjB,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;4BAChC,CAAC;4BAED,MAAM,IAAI,CAAC,YAAY,CAAC,mBAAmB,CAAC,YAAY,EAAE,GAAG,EAAE,IAAI,CAAC,CAAC;4BAErE,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;4BACrC,MAAM,EAAE,OAAO,EAAE,aAAa,EAAE,aAAa,EAAE,KAAK,EAAE,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;4BAEpH,MAAM,MAAM,GAAgB;gCAC1B,GAAG,EAAE,OAAO,CAAC,GAAG;gCAChB,IAAI,EAAE,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,QAAQ,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM;gCACjE,OAAO,EAAE,aAAa,KAAK,MAAM,CAAC,CAAC,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO;gCACnE,aAAa;gCACb,KAAK,EAAE,KAAK,IAAI,SAAS;gCACzB,aAAa;6BACd,CAAC;4BAEF,IAAI,MAAM,oBAAoB,EAAE,EAAE,CAAC;gCACjC,OAAO;4BACT,CAAC;4BACD,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;4BACpC,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;4BACnC,MAAM;wBACR,CAAC;oBACH,CAAC;gBACH,CAAC;gBACD,OAAO,KAAK,EAAE,CAAC;oBACb,IAAI,KAAK,YAAY,0BAA0B,EAAE,CAAC;wBAChD,MAAM,KAAK,CAAC;oBACd,CAAC;oBACD,MAAM,YAAY,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;oBAC5E,GAAG,CAAC,KAAK,CAAC,oBAAoB,OAAO,CAAC,GAAG,KAAK,YAAY,EAAE,CAAC,CAAC;gBAChE,CAAC;wBACO,CAAC;oBACP,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,sBAAsB,CAAC,CAAC;oBAC7C,IAAI,CAAC,GAAG,CAAC,eAAe,EAAE,qBAAqB,CAAC,CAAC;gBACnD,CAAC;YACH,CAAC;SACF,CAAC,CAAC;QAEH,IAAI,CAAC;YACH,MAAM,cAAc,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC;YAE1C,OAAO,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC;gBACxB,IAAI,IAAI,CAAC,YAAY,CAAC,gBAAgB,EAAE,EAAE,CAAC;oBACzC,KAAK,MAAM,MAAM,IAAI,MAAM,IAAI,CAAC,YAAY,CAAC,YAAY,EAAE,EAAE,CAAC;wBAC5D,MAAM,MAAM,CAAC;oBACf,CAAC;gBACH,CAAC;gBAED,IAAI,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,cAAc,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,EAAE,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;oBAC3H,MAAM;gBACR,CAAC;YACH,CAAC;YAED,MAAM,cAAc,CAAC;YACrB,IAAI,IAAI,CAAC,mBAAmB,EAAE,CAAC;gBAC7B,MAAM,IAAI,CAAC,mBAAmB,CAAC;YACjC,CAAC;YACD,MAAM,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC;YAEjC,4CAA4C;YAC5C,IAAI,IAAI,CAAC,oBAAoB,GAAG,CAAC,EAAE,CAAC;gBAClC,MAAM,CAAC,IAAI,CACT,4BAA4B,IAAI,CAAC,oBAAoB,sDAAsD,IAAI,CAAC,eAAe,GAAG,CACnI,CAAC;YACJ,CAAC;YAED,0DAA0D;YAC1D,IAAI,IAAI,CAAC,mBAAmB,EAAE,CAAC;gBAC7B,MAAM,IAAI,CAAC,mBAAmB,CAAC;YACjC,CAAC;YAED,KAAK,MAAM,MAAM,IAAI,MAAM,IAAI,CAAC,YAAY,CAAC,YAAY,EAAE,EAAE,CAAC;gBAC5D,MAAM,MAAM,CAAC;YACf,CAAC;QACH,CAAC;QACD,OAAO,KAAK,EAAE,CAAC;YACb,wCAAwC;YACxC,IAAI,KAAK,YAAY,mBAAmB,EAAE,CAAC;gBACzC,MAAM,KAAK,CAAC;YACd,CAAC;YACD,MAAM,CAAC,KAAK,CAAC,gBAAgB,EAAE,KAAK,CAAC,CAAC;YACtC,MAAM,KAAK,CAAC;QACd,CAAC;gBACO,CAAC;YACP,MAAM,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE,CAAC;YAClC,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACtB,CAAC;IACH,CAAC;IAED,KAAK;QACH,KAAK,CAAC,KAAK,EAAE,CAAC;QACd,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,6BAA6B,EAAE,GAAG,CAAC,CAAC,CAAC;QAC3F,CAAC;IACH,CAAC;CACF"}
|
|
1
|
+
{"version":3,"file":"crawlee-crawler.js","sourceRoot":"","sources":["../../src/crawler/crawlee-crawler.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,SAAS,CAAC;AAE5C,OAAO,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAGxC,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAE5C,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AACvD,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAC3C,OAAO,EAAE,eAAe,EAAE,cAAc,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAC3F,OAAO,EACL,2BAA2B,EAC3B,uBAAuB,EACvB,wBAAwB,EACxB,6BAA6B,EAC7B,0BAA0B,GAC3B,MAAM,6BAA6B,CAAC;AAErC,SAAS,wBAAwB,CAAC,KAAc,EAAE,OAAO,IAAI,GAAG,EAAU;IACxE,IAAI,KAAK,YAAY,0BAA0B,EAAE,CAAC;QAChD,OAAO,KAAK,CAAC;IACf,CAAC;IACD,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;QAC3D,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IAChB,MAAM,KAAK,GAAG,KAA+D,CAAC;IAC9E,IAAI,KAAK,CAAC,IAAI,KAAK,4BAA4B,EAAE,CAAC;QAChD,OAAO,IAAI,0BAA0B,CAAC,OAAO,KAAK,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IACvG,CAAC;IACD,OAAO,wBAAwB,CAAC,KAAK,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;AACrD,CAAC;AAED,SAAS,kBAAkB,CAAC,GAAW;IACrC,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IAC5B,OAAO,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC;AACzD,CAAC;AA0BD,MAAM,OAAO,cAAe,SAAQ,WAAW;IACrC,OAAO,GAA6B,IAAI,CAAC;IACzC,YAAY,GAAiB,IAAI,YAAY,EAAE,CAAC;IAChD,YAAY,CAAgB;IAC5B,WAAW,GAAY,IAAI,CAAC;IAC5B,mBAAmB,GAA+B,IAAI,CAAC;IACvD,WAAW,GAAW,EAAE,CAAC;IACjC,gFAAgF;IACxE,eAAe,GAAW,EAAE,CAAC;IACrC,iDAAiD;IACzC,oBAAoB,GAAW,CAAC,CAAC;IACzC,gDAAgD;IACxC,UAAU,CAAU;IACpB,mBAAmB,CAA8B;IACjD,kBAAkB,GAAG,IAAI,OAAO,EAA6B,CAAC;IAE9D,yBAAyB,CAAC,OAAe;QAC/C,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,EAAE,CAAC;IACpD,CAAC;IAED;;OAEG;IACH,eAAe,CAAC,KAAmB;QACjC,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC;QAC1B,MAAM,CAAC,IAAI,CAAC,2CAA2C,KAAK,CAAC,OAAO,EAAE,MAAM,IAAI,CAAC,UAAU,CAAC,CAAC;IAC/F,CAAC;IAED;;;;;;OAMG;IACK,qBAAqB,CAAC,GAAW;QACvC,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC;YAC1B,OAAO,IAAI,CAAC,CAAC,4BAA4B;QAC3C,CAAC;QAED,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;YAC5B,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;YAC/C,MAAM,OAAO,GAAG,IAAI,CAAC,eAAe,CAAC,WAAW,EAAE,CAAC;YAEnD,cAAc;YACd,IAAI,QAAQ,KAAK,OAAO,EAAE,CAAC;gBACzB,OAAO,IAAI,CAAC;YACd,CAAC;YAED,wEAAwE;YACxE,2EAA2E;YAC3E,IAAI,QAAQ,CAAC,QAAQ,CAAC,GAAG,GAAG,OAAO,CAAC,EAAE,CAAC;gBACrC,OAAO,IAAI,CAAC;YACd,CAAC;YAED,OAAO,KAAK,CAAC;QACf,CAAC;QACD,MAAM,CAAC;YACL,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,iBAAiB,CAAC,IAAU,EAAE,UAAkB;QAC5D,+CAA+C;QAC/C,uEAAuE;QACvE,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YACtB,OAAO,KAAK,CAAC;QACf,CAAC;QAED,iCAAiC;QACjC,IAAI,cAAc,CAAC,UAAU,CAAC,EAAE,CAAC;YAC/B,MAAM,CAAC,IAAI,CAAC,0DAA0D,UAAU,EAAE,CAAC,CAAC;YACpF,OAAO,IAAI,CAAC;QACd,CAAC;QAED,qBAAqB;QACrB,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,IAAI,EAAE,WAAW,IAAI,EAAE,CAAC,CAAC;YAC7E,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;YACtC,MAAM,SAAS,GAAG,eAAe,CAAC,QAAQ,GAAG,QAAQ,EAAE,UAAU,CAAC,CAAC;YAEnE,IAAI,SAAS,CAAC,WAAW,IAAI,SAAS,CAAC,UAAU,IAAI,GAAG,EAAE,CAAC;gBACzD,MAAM,CAAC,IAAI,CAAC,uEAAuE,SAAS,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;gBACvH,MAAM,CAAC,KAAK,CAAC,uCAAuC,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gBAEpF,wEAAwE;gBACxE,IAAI,CAAC,mBAAmB,GAAG,IAAI,mBAAmB,CAChD,mEAAmE,EACnE,IAAI,CAAC,WAAW,EAChB,UAAU,EACV,SAAS,CACV,CAAC;gBACF,OAAO,IAAI,CAAC;YACd,CAAC;QACH,CAAC;QACD,OAAO,KAAK,EAAE,CAAC;YACb,MAAM,CAAC,KAAK,CAAC,iDAAiD,EAAE,KAAK,CAAC,CAAC;QACzE,CAAC;QAED,OAAO,KAAK,CAAC;IACf,CAAC;IAED;;;;;;OAMG;IACK,KAAK,CAAC,wBAAwB,CAAC,IAAU;QAC/C,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAE9B,2FAA2F;QAC3F,wFAAwF;QACxF,IAAI,cAAc,GAAG,KAAK,CAAC;QAC3B,IAAI,CAAC;YACH,cAAc,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE;gBACxC,MAAM,WAAW,GAAG,QAAQ,CAAC,aAAa,CAAC,4BAA4B,CAAC,CAAC;gBACzE,IAAI,WAAW,EAAE,CAAC;oBAChB,OAAO,IAAI,CAAC;gBACd,CAAC;gBACD,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC;gBAC3B,MAAM,iBAAiB,GAAG,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,EAAE,CAAC,MAAM,IAAI,CAAC,CAAC,GAAG,GAAG,CAAC;gBAChF,MAAM,gBAAgB,GAAG,CAAC,QAAQ,CAAC,aAAa,CAAC,kDAAkD,CAAC,CAAC;gBACrG,OAAO,iBAAiB,IAAI,gBAAgB,CAAC;YAC/C,CAAC,CAAC,CAAC;QACL,CAAC;QACD,MAAM,CAAC;YACL,yEAAyE;YACzE,cAAc,GAAG,IAAI,CAAC;QACxB,CAAC;QAED,IAAI,cAAc,EAAE,CAAC;YACnB,MAAM,CAAC,KAAK,CAAC,sDAAsD,UAAU,6BAA6B,CAAC,CAAC;YAC5G,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,UAAU,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,KAAK,UAAU,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;gBAC5E,MAAM,OAAO,CAAC,GAAG,CAAC;oBAChB,IAAI,CAAC,gBAAgB,CAAC,kBAAkB,CAAC;oBACzC,IAAI,CAAC,gBAAgB,CAAC,aAAa,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC;iBACzE,CAAC,CAAC;gBACH,MAAM,CAAC,KAAK,CAAC,wCAAwC,UAAU,MAAM,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YACrF,CAAC;YACD,MAAM,CAAC;gBACL,MAAM,CAAC,KAAK,CAAC,oFAAoF,CAAC,CAAC;YACrG,CAAC;QACH,CAAC;QAED,qEAAqE;QACrE,IAAI,CAAC;YACH,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE;gBAC3C,MAAM,QAAQ,GAAG,QAAQ,CAAC,IAAI,EAAE,WAAW,IAAI,EAAE,CAAC;gBAClD,MAAM,gBAAgB,GACpB,QAAQ,CAAC,QAAQ,CAAC,uBAAuB,CAAC;oBAC1C,QAAQ,CAAC,QAAQ,CAAC,sBAAsB,CAAC;oBACzC,QAAQ,CAAC,QAAQ,CAAC,+BAA+B,CAAC,CAAC;gBACrD,MAAM,mBAAmB,GACvB,QAAQ,CAAC,aAAa,CAAC,6EAA6E,CAAC,KAAK,IAAI,CAAC;gBACjH,OAAO,gBAAgB,IAAI,mBAAmB,CAAC;YACjD,CAAC,CAAC,CAAC;YAEH,IAAI,WAAW,EAAE,CAAC;gBAChB,MAAM,CAAC,IAAI,CAAC,2EAA2E,CAAC,CAAC;gBACzF,IAAI,CAAC;oBACH,MAAM,IAAI,CAAC,eAAe,CACxB,GAAG,EAAE;wBACH,MAAM,QAAQ,GAAG,QAAQ,CAAC,IAAI,EAAE,WAAW,IAAI,EAAE,CAAC;wBAClD,MAAM,cAAc,GAAG,QAAQ,CAAC,QAAQ,CAAC,uBAAuB,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,sBAAsB,CAAC,CAAC;wBAC/G,MAAM,mBAAmB,GAAG,QAAQ,CAAC,aAAa,CAAC,gEAAgE,CAAC,KAAK,IAAI,CAAC;wBAC9H,OAAO,CAAC,cAAc,IAAI,CAAC,mBAAmB,CAAC;oBACjD,CAAC,EACD,EAAE,OAAO,EAAE,KAAK,EAAE,CACnB,CAAC;oBACF,MAAM,IAAI,CAAC,gBAAgB,CAAC,kBAAkB,CAAC,CAAC;oBAChD,MAAM,IAAI,CAAC,gBAAgB,CAAC,aAAa,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;oBAC9E,MAAM,CAAC,IAAI,CAAC,gDAAgD,CAAC,CAAC;gBAChE,CAAC;gBACD,MAAM,CAAC;oBACL,MAAM,CAAC,IAAI,CAAC,sEAAsE,CAAC,CAAC;gBACtF,CAAC;YACH,CAAC;QACH,CAAC;QACD,MAAM,CAAC;YACL,sFAAsF;YACtF,MAAM,CAAC,KAAK,CAAC,uEAAuE,CAAC,CAAC;QACxF,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,gBAAgB,CAAC,IAAU;QACvC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,EAAE,CAAC;QACnC,MAAM,aAAa,GAAG,MAAM,OAAO,CAAC,GAAG,CACrC,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE;YACzB,IAAI,CAAC;gBACH,MAAM,UAAU,GAAG,MAAM,KAAK;qBAC3B,QAAQ,CAAC,GAAG,EAAE;oBACb,OAAO,QAAQ,CAAC,aAAa,CAAC,6DAA6D,CAAC,KAAK,IAAI,CAAC;gBACxG,CAAC,CAAC;qBACD,KAAK,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC;gBAEtB,IAAI,UAAU,EAAE,CAAC;oBACf,MAAM,OAAO,CAAC,GAAG,CAAC;wBAChB,KAAK,CAAC,gBAAgB,CAAC,kBAAkB,CAAC;wBAC1C,KAAK;6BACF,gBAAgB,CAAC,aAAa,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;6BAClD,KAAK,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,gDAAgD,CAAC,CAAC;qBAC/E,CAAC,CAAC;oBACH,OAAO,KAAK,CAAC;gBACf,CAAC;YACH,CAAC;YACD,OAAO,KAAK,EAAE,CAAC;gBACb,MAAM,CAAC,KAAK,CAAC,sBAAsB,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YACjE,CAAC;YACD,OAAO,IAAI,CAAC;QACd,CAAC,CAAC,CACH,CAAC;QAEF,MAAM,KAAK,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,IAAI,IAAI,CAAC;QAC5D,IAAI,KAAK,EAAE,CAAC;YACV,MAAM,CAAC,KAAK,CAAC,yBAAyB,CAAC,CAAC;YACxC,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC;QAC5D,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IAEO,KAAK,CAAC,iBAAiB,CAC7B,OAAqB,EACrB,SAA2B;QAE3B,MAAM,aAAa,GAAG,SAAS,CAAC,WAAW,CAAC,QAAQ,EAAE,CAAC;QACvD,OAAO,OAAO,CAAC,QAAQ,CAAC,KAAK,EAAE,IAAY,EAAE,EAAE;YAC7C,MAAM,cAAc,GAAG,IAAI,QAAQ,CAAC,UAAU,IAAI,EAAE,CAAC,EAAE,CAAC;YACxD,MAAM,SAAS,GAAG,IAAI,cAAc,EAAE,CAAC;YACvC,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;YACxD,OAAO,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,aAAa,EAAE,MAAM,CAAC,aAAa,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,CAAC;QAC/F,CAAC,EAAE,aAAa,CAAC,CAAC;IACpB,CAAC;IAEO,KAAK,CAAC,cAAc,CAC1B,IAAU,EACV,QAAgB,EAChB,SAA2B;QAE3B,IAAI,OAAO,GAAG,EAAE,CAAC;QACjB,IAAI,aAAa,GAAkB,MAAM,CAAC;QAC1C,IAAI,aAAa,GAAG,SAAS,CAAC,WAAW,CAAC,IAAI,CAAC;QAC/C,IAAI,KAAyB,CAAC;QAE9B,IAAI,CAAC;YACH,IAAI,QAAQ,KAAK,WAAW,EAAE,CAAC;gBAC7B,mBAAmB;gBACnB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC;gBAChD,IAAI,KAAK,EAAE,CAAC;oBACV,CAAC,EAAE,OAAO,EAAE,aAAa,EAAE,KAAK,EAAE,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC,CAAC;gBACvF,CAAC;gBAED,wBAAwB;gBACxB,IAAI,CAAC,OAAO,EAAE,CAAC;oBACb,CAAC,EAAE,OAAO,EAAE,aAAa,EAAE,KAAK,EAAE,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC,CAAC;gBACtF,CAAC;YACH,CAAC;iBACI,CAAC;gBACJ,CAAC,EAAE,OAAO,EAAE,aAAa,EAAE,KAAK,EAAE,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC,CAAC;YACtF,CAAC;QACH,CAAC;QACD,MAAM,CAAC;YACL,OAAO,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAS,GAAG,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC;YAC7E,aAAa,GAAG,MAAM,CAAC;YACvB,aAAa,GAAG,eAAe,CAAC;YAChC,KAAK,GAAG,SAAS,CAAC;QACpB,CAAC;QAED,OAAO,EAAE,OAAO,EAAE,aAAa,EAAE,aAAa,EAAE,KAAK,EAAE,CAAC;IAC1D,CAAC;IAED;;;OAGG;IACH,aAAa,CAAC,MAAc;QAC1B,IAAI,CAAC,UAAU,GAAG,MAAM,CAAC;QACzB,MAAM,CAAC,IAAI,CAAC,iDAAiD,MAAM,EAAE,CAAC,CAAC;IACzE,CAAC;IAED,KAAK,CAAC,CAAC,KAAK,CAAC,GAAW;QACtB,MAAM,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,wBAAwB,GAAG,EAAE,CAAC,CAAC;QAErE,6BAA6B;QAC7B,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;QACxB,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC;QAChC,IAAI,CAAC,mBAAmB,GAAG,SAAS,CAAC;QACrC,IAAI,CAAC,kBAAkB,GAAG,IAAI,OAAO,EAAE,CAAC;QACxC,IAAI,CAAC,WAAW,GAAG,kBAAkB,CAAC,GAAG,CAAC,CAAC;QAC3C,IAAI,CAAC,oBAAoB,GAAG,CAAC,CAAC;QAE9B,8DAA8D;QAC9D,IAAI,CAAC;YACH,IAAI,CAAC,eAAe,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;YAC7C,MAAM,CAAC,IAAI,CAAC,+DAA+D,IAAI,CAAC,eAAe,EAAE,CAAC,CAAC;QACrG,CAAC;QACD,MAAM,CAAC;YACL,IAAI,CAAC,eAAe,GAAG,EAAE,CAAC;QAC5B,CAAC;QAED,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;QAEzD,qEAAqE;QACrE,MAAM,IAAI,CAAC,YAAY,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC;QAE7C,qDAAqD;QACrD,MAAM,cAAc,GAAG,MAAM,gBAAgB,CAAC,IAAI,CAAC,YAAY,CAAC,eAAe,EAAE,IAAI,SAAS,CAAC,CAAC;QAEhG,6EAA6E;QAC7E,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACtB,MAAM,CAAC,IAAI,CAAC,qDAAqD,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE,MAAM,IAAI,CAAC,UAAU,CAAC,CAAC;YACnH,cAAc,CAAC,kBAAkB,GAAG;gBAClC,GAAG,cAAc,CAAC,kBAAkB;gBACpC,cAAc,EAAE;oBACd,KAAK,EAAE,MAAM,EAAE,EAAE;wBACf,0DAA0D;wBAC1D,MAAM,CAAC,KAAK,CAAC,+CAA+C,MAAM,EAAE,CAAC,CAAC;oBACxE,CAAC;iBACF;aACF,CAAC;YACF,qCAAqC;YACrC,MAAM,aAAa,GAAG,cAAc,CAAC,kBAAkB,IAAI,EAAE,CAAC;YAC9D,cAAc,CAAC,kBAAkB,GAAG;gBAClC,GAAG,aAAa;gBAChB,KAAK,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE;oBACjB,IAAI,IAAI,CAAC,YAAY,EAAE,OAAO,EAAE,CAAC;wBAC/B,MAAM,CAAC,KAAK,CAAC,4BAA4B,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,MAAM,4BAA4B,CAAC,CAAC;wBACvG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;oBAC7D,CAAC;gBACH,CAAC;aACF,CAAC;QACJ,CAAC;QAED,MAAM,0BAA0B,GAAG,cAAc,CAAC,kBAAkB,IAAI,EAAE,CAAC;QAC3E,cAAc,CAAC,kBAAkB,GAAG;YAClC,GAAG,0BAA0B;YAC7B,KAAK,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE,EAAE;gBAC1B,MAAM,UAAU,GAAG,OAAiB,CAAC;gBACrC,IAAI,CAAC,yBAAyB,CAAC,UAAU,CAAC,CAAC;gBAC3C,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;gBACnC,MAAM,OAAO,GAAsB,EAAE,CAAC;gBACtC,MAAM,eAAe,GAAG,CAAC,aAAgC,EAAE,EAAE;oBAC3D,MAAM,SAAS,GAAG,aAAa,CAAC,OAAO,EAAE,EAAE,SAAS,IAAI,EAAE,CAAC;oBAC3D,IAAI,aAAa,CAAC,mBAAmB,EAAE,IAAI,aAAa,CAAC,KAAK,EAAE,KAAK,SAAS,IAAI,CAAC,6BAA6B,CAAC,SAAS,CAAC,EAAE,CAAC;wBAC5H,OAAO,CAAC,SAAS,GAAG,aAAa,CAAC,GAAG,EAAE,CAAC;oBAC1C,CAAC;gBACH,CAAC,CAAC;gBACF,IAAI,CAAC,EAAE,CAAC,eAAe,EAAE,eAAe,CAAC,CAAC;gBAC1C,OAAO,CAAC,OAAO,GAAG,GAAG,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,eAAe,EAAE,eAAe,CAAC,CAAC;gBACnE,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;YACnD,CAAC;SACF,CAAC;QAEF,IAAI,CAAC,OAAO,GAAG,IAAI,iBAAiB,CAAC;YACnC,GAAG,cAAc;YACjB,YAAY,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE;gBACrC,MAAM,UAAU,GAAG,OAAO,CAAC,OAAiB,CAAC;gBAC7C,MAAM,OAAO,GAAG,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC;gBAC9D,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;gBACpC,MAAM,eAAe,GAAG,SAAS,CAAC,CAAC,CAAC,MAAM,uBAAuB,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,wBAAwB,CAAC,KAAK,CAAC,CAAC;gBAC/G,IAAI,eAAe,EAAE,CAAC;oBACpB,OAAO,CAAC,eAAe,GAAG,eAAe,CAAC;oBAC1C,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;gBACnD,CAAC;gBACD,IAAI,eAAe,YAAY,2BAA2B,EAAE,CAAC;oBAC3D,OAAO,CAAC,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC;gBACjC,CAAC;gBACD,IAAI,CAAC;oBACH,MAAM,cAAc,CAAC,YAAY,EAAE,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;gBACtD,CAAC;wBACO,CAAC;oBACP,IAAI,eAAe,YAAY,2BAA2B,EAAE,CAAC;wBAC3D,OAAO,CAAC,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC;oBACjC,CAAC;oBACD,IAAI,CAAC,yBAAyB,CAAC,UAAU,CAAC,CAAC;gBAC7C,CAAC;YACH,CAAC;YACD,oBAAoB,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE;gBAC7C,MAAM,UAAU,GAAG,OAAO,CAAC,OAAiB,CAAC;gBAC7C,IAAI,CAAC,yBAAyB,CAAC,UAAU,CAAC,CAAC;gBAC3C,MAAM,OAAO,GAAG,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;gBACxD,MAAM,SAAS,GAAG,OAAO,EAAE,SAAS,CAAC;gBACrC,MAAM,eAAe,GACnB,OAAO,EAAE,eAAe,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,uBAAuB,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,wBAAwB,CAAC,KAAK,CAAC,CAAC,CAAC;gBACvH,IAAI,kBAAkB,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,WAAW,IAAI,eAAe,YAAY,0BAA0B,EAAE,CAAC;oBAC1H,IAAI,CAAC,mBAAmB,GAAG,eAAe,CAAC;gBAC7C,CAAC;gBACD,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;gBAC3C,MAAM,cAAc,CAAC,oBAAoB,EAAE,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;YAC9D,CAAC;YACD,cAAc,EAAE,KAAK,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,YAAY,EAAE,GAAG,EAAE,EAAE,EAAE;gBACvE,MAAM,UAAU,GAAG,OAAiB,CAAC;gBACrC,IAAI,CAAC,yBAAyB,CAAC,UAAU,CAAC,CAAC;gBAC3C,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;gBAC3C,IAAI,uBAAuB,GAAG,QAAQ,CAAC;gBACvC,IAAI,wBAA8F,CAAC;gBACnG,IAAI,kBAAkB,GAAG,CAAC,CAAC;gBAC3B,MAAM,uBAAuB,GAAG,IAAI,GAAG,EAAiB,CAAC;gBACzD,MAAM,sBAAsB,GAAG,CAAC,YAA6B,EAAE,EAAE;oBAC/D,MAAM,WAAW,GAAG,YAAY,EAAE,OAAO,EAAE,CAAC;oBAC5C,IAAI,WAAW,EAAE,mBAAmB,EAAE,IAAI,WAAW,CAAC,KAAK,EAAE,KAAK,IAAI,CAAC,SAAS,EAAE,EAAE,CAAC;wBACnF,kBAAkB,EAAE,CAAC;wBACrB,uBAAuB,GAAG,YAAY,CAAC;wBACvC,wBAAwB,GAAG,SAAS,CAAC;oBACvC,CAAC;gBACH,CAAC,CAAC;gBACF,MAAM,qBAAqB,GAAG,CAAC,aAAgC,EAAE,EAAE;oBACjE,IACE,aAAa,CAAC,mBAAmB,EAAE;wBACnC,aAAa,CAAC,KAAK,EAAE,KAAK,IAAI,CAAC,SAAS,EAAE;wBAC1C,CAAC,6BAA6B,CAAC,aAAa,CAAC,OAAO,EAAE,EAAE,SAAS,IAAI,EAAE,CAAC,EACxE,CAAC;wBACD,MAAM,eAAe,GAAG,EAAE,kBAAkB,CAAC;wBAC7C,MAAM,KAAK,GAAG,uBAAuB,CAAC,aAAa,CAAC,GAAG,EAAE,CAAC;6BACvD,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE;4BACd,IAAI,kBAAkB,KAAK,eAAe,EAAE,CAAC;gCAC3C,wBAAwB,GAAG,KAAK,CAAC;4BACnC,CAAC;wBACH,CAAC,CAAC;6BACD,OAAO,CAAC,GAAG,EAAE,CAAC,uBAAuB,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;wBACxD,uBAAuB,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;oBACrC,CAAC;gBACH,CAAC,CAAC;gBACF,IAAI,CAAC,EAAE,CAAC,UAAU,EAAE,sBAAsB,CAAC,CAAC;gBAC5C,IAAI,CAAC,EAAE,CAAC,eAAe,EAAE,qBAAqB,CAAC,CAAC;gBAChD,MAAM,oBAAoB,GAAG,KAAK,IAAI,EAAE;oBACtC,MAAM,OAAO,CAAC,GAAG,CAAC,uBAAuB,CAAC,CAAC;oBAC3C,IAAI,wBAAwB,EAAE,CAAC;wBAC7B,IAAI,wBAAwB,YAAY,2BAA2B,EAAE,CAAC;4BACpE,GAAG,CAAC,OAAO,CAAC,0CAA0C,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;4BACrE,OAAO,IAAI,CAAC;wBACd,CAAC;wBACD,MAAM,wBAAwB,CAAC;oBACjC,CAAC;oBACD,MAAM,aAAa,GAAG,MAAM,wBAAwB,CAAC,uBAAuB,CAAC,CAAC;oBAC9E,IAAI,aAAa,YAAY,2BAA2B,EAAE,CAAC;wBACzD,GAAG,CAAC,OAAO,CAAC,0CAA0C,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;wBACrE,OAAO,IAAI,CAAC;oBACd,CAAC;oBACD,IAAI,aAAa,EAAE,CAAC;wBAClB,MAAM,aAAa,CAAC;oBACtB,CAAC;oBACD,OAAO,KAAK,CAAC;gBACf,CAAC,CAAC;gBAEF,IAAI,CAAC;oBACH,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;wBACpB,GAAG,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;wBAC3B,OAAO;oBACT,CAAC;oBAED,IAAI,MAAM,oBAAoB,EAAE,EAAE,CAAC;wBACjC,OAAO;oBACT,CAAC;oBAED,6BAA6B;oBAC7B,MAAM,OAAO,CAAC,GAAG,CAAC;wBAChB,IAAI,CAAC,gBAAgB,CAAC,kBAAkB,CAAC;wBACzC,IAAI,CAAC,gBAAgB,CAAC,aAAa,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,0CAA0C,CAAC,CAAC;qBAC3H,CAAC,CAAC;oBAEH,sEAAsE;oBACtE,MAAM,IAAI,CAAC,wBAAwB,CAAC,IAAI,CAAC,CAAC;oBAC1C,IAAI,MAAM,oBAAoB,EAAE,EAAE,CAAC;wBACjC,OAAO;oBACT,CAAC;oBAED,yCAAyC;oBACzC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;oBAE7B,0DAA0D;oBAC1D,IAAI,CAAC,IAAI,CAAC,qBAAqB,CAAC,SAAS,CAAC,EAAE,CAAC;wBAC3C,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;wBACxD,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC,QAAQ,CAAC;wBAEnD,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;4BACrB,gFAAgF;4BAChF,MAAM,CAAC,IAAI,CAAC,kEAAkE,iBAAiB,MAAM,cAAc,EAAE,CAAC,CAAC;4BAEvH,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;gCACtB,mDAAmD;gCACnD,IAAI,CAAC,mBAAmB,GAAG,IAAI,mBAAmB,CAChD,4EAA4E,cAAc,GAAG,EAC7F,IAAI,CAAC,WAAW,EAChB,SAAS,EACT,EAAE,WAAW,EAAE,IAAI,EAAE,UAAU,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,mBAAmB,iBAAiB,OAAO,cAAc,EAAE,CAAC,EAAE,CAC/G,CAAC;gCACF,GAAG,CAAC,KAAK,CAAC,oDAAoD,cAAc,mBAAmB,CAAC,CAAC;gCACjG,IAAI,CAAC,KAAK,EAAE,CAAC;gCACb,OAAO;4BACT,CAAC;iCACI,CAAC;gCACJ,0DAA0D;gCAC1D,GAAG,CAAC,KAAK,CAAC,6CAA6C,cAAc,mBAAmB,CAAC,CAAC;gCAC1F,IAAI,CAAC,KAAK,EAAE,CAAC;gCACb,OAAO;4BACT,CAAC;wBACH,CAAC;6BACI,CAAC;4BACJ,sDAAsD;4BACtD,IAAI,CAAC,oBAAoB,EAAE,CAAC;4BAC5B,GAAG,CAAC,OAAO,CACT,iDAAiD,OAAO,CAAC,GAAG,MAAM,SAAS,aAAa,IAAI,CAAC,oBAAoB,kBAAkB,CACpI,CAAC;4BACF,OAAO;wBACT,CAAC;oBACH,CAAC;oBAED,gEAAgE;oBAChE,IAAI,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;wBAC1C,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;wBAClE,IAAI,WAAW,EAAE,CAAC;4BAChB,GAAG,CAAC,KAAK,CAAC,uEAAuE,CAAC,CAAC;4BACnF,IAAI,CAAC,KAAK,EAAE,CAAC;4BACb,OAAO;wBACT,CAAC;wBACD,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;oBAC3B,CAAC;yBACI,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;wBAC1B,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;oBAC3B,CAAC;oBAED,qCAAqC;oBACrC,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;wBAC7B,IAAI,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;4BAC5B,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;gCACjB,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;4BAChC,CAAC;4BAED,MAAM,IAAI,CAAC,YAAY,CAAC,mBAAmB,CAAC,YAAY,EAAE,GAAG,EAAE,IAAI,CAAC,CAAC;4BAErE,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;4BACrC,MAAM,EAAE,OAAO,EAAE,aAAa,EAAE,aAAa,EAAE,KAAK,EAAE,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;4BAEpH,MAAM,MAAM,GAAgB;gCAC1B,GAAG,EAAE,OAAO,CAAC,GAAG;gCAChB,IAAI,EAAE,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,QAAQ,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM;gCACjE,OAAO,EAAE,aAAa,KAAK,MAAM,CAAC,CAAC,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO;gCACnE,aAAa;gCACb,KAAK,EAAE,KAAK,IAAI,SAAS;gCACzB,aAAa;6BACd,CAAC;4BAEF,IAAI,MAAM,oBAAoB,EAAE,EAAE,CAAC;gCACjC,OAAO;4BACT,CAAC;4BACD,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;4BACpC,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;4BACnC,MAAM;wBACR,CAAC;oBACH,CAAC;gBACH,CAAC;wBACO,CAAC;oBACP,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,sBAAsB,CAAC,CAAC;oBAC7C,IAAI,CAAC,GAAG,CAAC,eAAe,EAAE,qBAAqB,CAAC,CAAC;gBACnD,CAAC;YACH,CAAC;SACF,CAAC,CAAC;QAEH,IAAI,CAAC;YACH,MAAM,cAAc,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC;YAE1C,OAAO,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC;gBACxB,IAAI,IAAI,CAAC,YAAY,CAAC,gBAAgB,EAAE,EAAE,CAAC;oBACzC,KAAK,MAAM,MAAM,IAAI,MAAM,IAAI,CAAC,YAAY,CAAC,YAAY,EAAE,EAAE,CAAC;wBAC5D,MAAM,MAAM,CAAC;oBACf,CAAC;gBACH,CAAC;gBAED,IAAI,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,cAAc,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,EAAE,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;oBAC3H,MAAM;gBACR,CAAC;YACH,CAAC;YAED,MAAM,eAAe,GAAG,MAAM,cAAc,CAAC;YAC7C,IAAI,IAAI,CAAC,mBAAmB,EAAE,CAAC;gBAC7B,MAAM,IAAI,CAAC,mBAAmB,CAAC;YACjC,CAAC;YACD,MAAM,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC;YAEjC,4CAA4C;YAC5C,IAAI,IAAI,CAAC,oBAAoB,GAAG,CAAC,EAAE,CAAC;gBAClC,MAAM,CAAC,IAAI,CACT,4BAA4B,IAAI,CAAC,oBAAoB,sDAAsD,IAAI,CAAC,eAAe,GAAG,CACnI,CAAC;YACJ,CAAC;YAED,0DAA0D;YAC1D,IAAI,IAAI,CAAC,mBAAmB,EAAE,CAAC;gBAC7B,MAAM,IAAI,CAAC,mBAAmB,CAAC;YACjC,CAAC;YAED,IAAI,eAAe,CAAC,cAAc,GAAG,CAAC,EAAE,CAAC;gBACvC,MAAM,IAAI,KAAK,CAAC,oBAAoB,eAAe,CAAC,cAAc,OAAO,eAAe,CAAC,aAAa,sBAAsB,CAAC,CAAC;YAChI,CAAC;YAED,KAAK,MAAM,MAAM,IAAI,MAAM,IAAI,CAAC,YAAY,CAAC,YAAY,EAAE,EAAE,CAAC;gBAC5D,MAAM,MAAM,CAAC;YACf,CAAC;QACH,CAAC;QACD,OAAO,KAAK,EAAE,CAAC;YACb,wCAAwC;YACxC,IAAI,KAAK,YAAY,mBAAmB,EAAE,CAAC;gBACzC,MAAM,KAAK,CAAC;YACd,CAAC;YACD,MAAM,CAAC,KAAK,CAAC,gBAAgB,EAAE,KAAK,CAAC,CAAC;YACtC,MAAM,KAAK,CAAC;QACd,CAAC;gBACO,CAAC;YACP,MAAM,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE,CAAC;YAClC,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACtB,CAAC;IACH,CAAC;IAED,KAAK;QACH,KAAK,CAAC,KAAK,EAAE,CAAC;QACd,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,6BAA6B,EAAE,GAAG,CAAC,CAAC,CAAC;QAC3F,CAAC;IACH,CAAC;CACF"}
|
|
@@ -37,7 +37,8 @@ vi.mock('./site-rules.js', () => ({
|
|
|
37
37
|
],
|
|
38
38
|
}));
|
|
39
39
|
// Mock PlaywrightCrawler
|
|
40
|
-
const
|
|
40
|
+
const successfulRunStats = { requestsFailed: 0, requestsTotal: 0 };
|
|
41
|
+
const mockCrawlerRun = vi.fn().mockResolvedValue(successfulRunStats);
|
|
41
42
|
const mockCrawlerTeardown = vi.fn().mockResolvedValue(undefined);
|
|
42
43
|
vi.mock('crawlee', () => ({
|
|
43
44
|
PlaywrightCrawler: function (options) {
|
|
@@ -47,7 +48,7 @@ vi.mock('crawlee', () => ({
|
|
|
47
48
|
global.__errorHandler = options.errorHandler;
|
|
48
49
|
global.__failedRequestHandler = options.failedRequestHandler;
|
|
49
50
|
return {
|
|
50
|
-
run: mockCrawlerRun,
|
|
51
|
+
run: async () => (await mockCrawlerRun()) ?? successfulRunStats,
|
|
51
52
|
teardown: mockCrawlerTeardown,
|
|
52
53
|
};
|
|
53
54
|
},
|
|
@@ -294,9 +295,6 @@ describe('CrawleeCrawler', () => {
|
|
|
294
295
|
expect(page.off).toHaveBeenCalledWith('requestfailed', listeners.requestfailed);
|
|
295
296
|
});
|
|
296
297
|
it('should initialize queue manager with URL', async () => {
|
|
297
|
-
// Set up processBatch to return results immediately to end the crawl
|
|
298
|
-
mockCrawlerRun.mockResolvedValueOnce(undefined);
|
|
299
|
-
mockQueueManager.processBatch.mockResolvedValueOnce([]);
|
|
300
298
|
await collect(crawler, 'https://example.com/docs');
|
|
301
299
|
expect(mockQueueManager.initialize).toHaveBeenCalledWith('https://example.com/docs', undefined);
|
|
302
300
|
});
|
|
@@ -344,7 +342,7 @@ describe('CrawleeCrawler', () => {
|
|
|
344
342
|
// Now abort - the crawler exists at this point
|
|
345
343
|
abortableCrawler.abort();
|
|
346
344
|
// Let the run() complete so the generator can finish
|
|
347
|
-
resolveRun();
|
|
345
|
+
resolveRun(successfulRunStats);
|
|
348
346
|
// Wait for the generator to complete
|
|
349
347
|
await firstResultPromise;
|
|
350
348
|
expect(mockCrawlerTeardown).toHaveBeenCalled();
|
|
@@ -369,6 +367,25 @@ describe('CrawleeCrawler', () => {
|
|
|
369
367
|
});
|
|
370
368
|
});
|
|
371
369
|
describe('error handling', () => {
|
|
370
|
+
it('should reject after Crawlee exhausts retries for a request handler error', async () => {
|
|
371
|
+
mockCrawlerRun.mockImplementationOnce(async () => {
|
|
372
|
+
try {
|
|
373
|
+
await runRequestHandler({
|
|
374
|
+
mainFrame: vi.fn().mockReturnValue({}),
|
|
375
|
+
on: vi.fn(),
|
|
376
|
+
off: vi.fn(),
|
|
377
|
+
waitForLoadState: vi.fn().mockRejectedValueOnce(new Error('Page load failed')).mockResolvedValue(undefined),
|
|
378
|
+
}, 'https://example.com');
|
|
379
|
+
}
|
|
380
|
+
catch {
|
|
381
|
+
return { requestsFailed: 1, requestsTotal: 1 };
|
|
382
|
+
}
|
|
383
|
+
return successfulRunStats;
|
|
384
|
+
});
|
|
385
|
+
await expect(collect(crawler, 'https://example.com')).rejects.toThrow('Crawl failed for 1 of 1 pages after retries');
|
|
386
|
+
expect(mockQueueManager.processBatch).not.toHaveBeenCalled();
|
|
387
|
+
expect(mockQueueManager.cleanup).toHaveBeenCalled();
|
|
388
|
+
});
|
|
372
389
|
it('should cleanup on error', async () => {
|
|
373
390
|
mockCrawlerRun.mockRejectedValueOnce(new Error('Crawl failed'));
|
|
374
391
|
await expect(collect(crawler, 'https://example.com')).rejects.toThrow('Crawl failed');
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"crawlee-crawler.test.js","sourceRoot":"","sources":["../../src/crawler/crawlee-crawler.test.ts"],"names":[],"mappings":"AAEA,MAAM,gBAAgB,GAAG;IACvB,UAAU,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,SAAS,CAAC;IAChD,eAAe,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,CAAC,CAAC;IAC7C,eAAe,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC,EAAE,CAAC;IAC5C,mBAAmB,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,SAAS,CAAC;IACzD,SAAS,EAAE,EAAE,CAAC,EAAE,EAAE;IAClB,gBAAgB,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC,KAAK,CAAC;IAChD,YAAY,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,EAAE,CAAC;IAC3C,OAAO,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,SAAS,CAAC;CAC9C,CAAC;AACF,MAAM,EAAE,0BAA0B,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;IACvD,0BAA0B,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,SAAS,CAAC;CACjE,CAAC,CAAC,CAAC;AAEJ,EAAE,CAAC,IAAI,CAAC,oBAAoB,EAAE,GAAG,EAAE,CAAC,CAAC;IACnC,YAAY,EAAE;QACZ,OAAO,gBAAgB,CAAC;IAC1B,CAAC;CACF,CAAC,CAAC,CAAC;AAEJ,EAAE,CAAC,IAAI,CAAC,qBAAqB,EAAE,GAAG,EAAE,CAAC,CAAC;IACpC,gBAAgB,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC;QACxC,YAAY,EAAE,EAAE;QAChB,kBAAkB,EAAE,EAAE;QACtB,aAAa,EAAE,EAAE;QACjB,kBAAkB,EAAE,EAAE;QACtB,YAAY,EAAE,0BAA0B;KACzC,CAAC;CACH,CAAC,CAAC,CAAC;AAEJ,EAAE,CAAC,IAAI,CAAC,iBAAiB,EAAE,GAAG,EAAE,CAAC,CAAC;IAChC,SAAS,EAAE;QACT;YACE,IAAI,EAAE,SAAS;YACf,SAAS,EAAE;gBACT,cAAc,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,EAAE,OAAO,EAAE,mBAAmB,EAAE,aAAa,EAAE,MAAM,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC;aACjH;YACD,MAAM,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,IAAI,CAAC;SACxC;KACF;CACF,CAAC,CAAC,CAAC;AAEJ,yBAAyB;AACzB,MAAM,cAAc,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;AAC5D,MAAM,mBAAmB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;AAEjE,EAAE,CAAC,IAAI,CAAC,SAAS,EAAE,GAAG,EAAE,CAAC,CAAC;IACxB,iBAAiB,EAAE,UAAU,OAK5B;QACC,wCAAwC;QACvC,MAA6C,CAAC,oBAAoB,GAAG,OAAO,CAAC,kBAAkB,CAAC;QAChG,MAAyC,CAAC,gBAAgB,GAAG,OAAO,CAAC,cAAc,CAAC;QACpF,MAAuC,CAAC,cAAc,GAAG,OAAO,CAAC,YAAY,CAAC;QAC9E,MAA+C,CAAC,sBAAsB,GAAG,OAAO,CAAC,oBAAoB,CAAC;QACvG,OAAO;YACL,GAAG,EAAE,cAAc;YACnB,QAAQ,EAAE,mBAAmB;SAC9B,CAAC;IACJ,CAAC;CACF,CAAC,CAAC,CAAC;AAEJ,uBAAuB;AACvB,OAAO,EAAE,cAAc,EAAgB,MAAM,sBAAsB,CAAC;AAMpE,SAAS,iBAAiB;IACxB,OAAQ,UAA8D,CAAC,gBAAgB,CAAC;AAC1F,CAAC;AAED,SAAS,eAAe,CAAC,IAAiD;IACxE,OAAQ,UAAsD,CAAC,IAAI,CAAC,CAAC;AACvE,CAAC;AAED,KAAK,UAAU,OAAO,CAAC,OAAuB,EAAE,GAAW;IACzD,MAAM,OAAO,GAAkB,EAAE,CAAC;IAClC,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;QAC9C,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACvB,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,QAAQ,CAAC,MAAM,GAAG,GAAG,EAAE,OAAO,GAAG,KAAK,EAAE,SAAkB;IACjE,OAAO;QACL,MAAM,EAAE,GAAG,EAAE,CAAC,MAAM;QACpB,WAAW,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC;QAC5D,GAAG,CAAC,SAAS,IAAI,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,mBAAmB,EAAE,GAAG,EAAE,CAAC,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,SAAS,EAAE,CAAC,EAAE,CAAC;KACnG,CAAC;AACJ,CAAC;AAED,SAAS,cAAc,CACrB,GAAW,EACX,MAAkI,EAClI,QAAQ,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,KAAK,CAAC;IAE3C,MAAM,SAAS,GAAG,EAAE,CAAC;IACrB,MAAM,SAAS,GAAsE,EAAE,CAAC;IACxF,IAAI,MAAM,GAAG,KAAK,CAAC;IACnB,MAAM,IAAI,GAAG;QACX,SAAS,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC,SAAS,CAAC;QAC7C,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,KAAmC,EAAE,QAA4B,EAAE,EAAE;YAC9E,SAAS,CAAC,KAAK,CAAC,GAAG,QAAQ,CAAC;QAC9B,CAAC,CAAC;QACF,GAAG,EAAE,EAAE,CAAC,EAAE,EAAE;QACZ,gBAAgB,EAAE,EAAE,CAAC,EAAE,CAAC,KAAK,IAAI,EAAE;YACjC,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,MAAM,GAAG,IAAI,CAAC;gBACd,MAAM,CAAC;oBACL,QAAQ,EAAE,CAAC,MAAM,GAAG,GAAG,EAAE,OAAO,GAAG,KAAK,EAAE,EAAE,CAAC,SAAS,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,MAAM,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;oBACvG,OAAO,EAAE,CAAC,KAAK,EAAE,SAAS,EAAE,EAAE,CAC5B,SAAS,CAAC,aAAa,EAAE,CAAC;wBACxB,mBAAmB,EAAE,GAAG,EAAE,CAAC,IAAI;wBAC/B,KAAK,EAAE,GAAG,EAAE,CAAC,SAAS;wBACtB,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC;wBACrC,GAAG,CAAC,SAAS,IAAI,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,SAAS,EAAE,CAAC;qBAC3C,CAAC;iBACL,CAAC,CAAC;YACL,CAAC;QACH,CAAC,CAAC;QACF,QAAQ;QACR,GAAG,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC,GAAG,CAAC;QACjC,KAAK,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,MAAM,CAAC;KACzC,CAAC;IACF,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC;AACxC,CAAC;AAED,KAAK,UAAU,iBAAiB,CAAC,IAAY,EAAE,GAAW,EAAE,eAAe,GAAG,QAAQ,EAAE,EAAE,OAAO,GAAG,EAAE,CAAC,EAAE,EAAE;IACzG,MAAM,iBAAiB,EAAE,CAAC;QACxB,OAAO,EAAE,EAAE,GAAG,EAAE;QAChB,QAAQ,EAAE,eAAe;QACzB,IAAI;QACJ,YAAY,EAAE,EAAE,CAAC,EAAE,EAAE;QACrB,GAAG,EAAE,EAAE,KAAK,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,OAAO,EAAE;KACjD,CAAC,CAAC;AACL,CAAC;AAED,KAAK,UAAU,wBAAwB,CAAC,OAAe,EAAE,SAAiB;IACxE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,SAAS,EAAE,GAAG,cAAc,CAAC,EAAE,EAAE,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IACpE,MAAM,KAAK,GAAI,MAAgG,CAAC,oBAAqB,CAAC;IACtI,MAAM,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,CAAE,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC;IACvC,SAAS,CAAC,aAAa,EAAE,CAAC;QACxB,mBAAmB,EAAE,GAAG,EAAE,CAAC,IAAI;QAC/B,KAAK,EAAE,GAAG,EAAE,CAAC,SAAS;QACtB,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,SAAS,EAAE,mCAAmC,EAAE,CAAC;QACnE,GAAG,EAAE,GAAG,EAAE,CAAC,SAAS;KACrB,CAAC,CAAC;AACL,CAAC;AAED,QAAQ,CAAC,gBAAgB,EAAE,GAAG,EAAE;IAC9B,IAAI,OAAuB,CAAC;IAE5B,UAAU,CAAC,GAAG,EAAE;QACd,EAAE,CAAC,aAAa,EAAE,CAAC;QACnB,OAAO,GAAG,IAAI,cAAc,EAAE,CAAC;QAC/B,gBAAgB,CAAC,gBAAgB,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;QACzD,gBAAgB,CAAC,YAAY,CAAC,iBAAiB,CAAC,EAAE,CAAC,CAAC;IACtD,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,OAAO,EAAE,GAAG,EAAE;QACrB,EAAE,CAAC,4EAA4E,EAAE,KAAK,IAAI,EAAE;YAC1F,MAAM,iBAAiB;gBACrB,KAAK,CAAC,cAAc;oBAClB,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,aAAa,EAAE,UAAU,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,IAAI,EAAE,UAAU,EAAE,EAAE,CAAC;gBAC3G,CAAC;aACF;YAED,MAAM,cAAc,GAClB,OAOD,CAAC,cAAc,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAC/B,MAAM,WAAW,GAAG;gBAClB,QAAQ,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,EAAE,OAAO,EAAE,SAAS,EAAE,aAAa,EAAE,UAAU,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;aACvG,CAAC;YAEF,MAAM,MAAM,CAAC,cAAc,CAAC,WAAW,EAAE,SAAS,EAAE,IAAI,iBAAiB,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,aAAa,CAAC;gBACnG,OAAO,EAAE,SAAS;gBAClB,aAAa,EAAE,UAAU;gBACzB,aAAa,EAAE,mBAAmB;gBAClC,KAAK,EAAE,OAAO;aACf,CAAC,CAAC;YAEH,MAAM,YAAY,GAAG;gBACnB,QAAQ,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,qBAAqB,CAAC,IAAI,KAAK,CAAC,kBAAkB,CAAC,CAAC,CAAC,qBAAqB,CAAC,gBAAgB,CAAC;aAC/G,CAAC;YACF,MAAM,MAAM,CAAC,cAAc,CAAC,YAAY,EAAE,SAAS,EAAE,IAAI,iBAAiB,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,aAAa,CAAC;gBACpG,OAAO,EAAE,gBAAgB;gBACzB,aAAa,EAAE,MAAM;gBACrB,aAAa,EAAE,eAAe;aAC/B,CAAC,CAAC;YAEH,MAAM,mBAAmB,GAAG,EAAE;iBAC3B,KAAK,CACJ,OAEC,EACD,kBAAkB,CACnB;iBACA,iBAAiB,CAAC;gBACjB,QAAQ,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE,aAAa,EAAE,UAAU,EAAE,KAAK,EAAE,uBAAuB,EAAE,CAAC;aAChH,CAAC,CAAC;YACL,IAAI,CAAC;gBACH,MAAM,qBAAqB,GAAG;oBAC5B,QAAQ,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,qBAAqB,CAAC,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC,CAAC,qBAAqB,CAAC,yBAAyB,CAAC;iBAC9H,CAAC;gBACF,MAAM,MAAM,CAAC,cAAc,CAAC,qBAAqB,EAAE,WAAW,EAAE,IAAI,iBAAiB,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC;oBACzG,OAAO,EAAE,yBAAyB;oBAClC,aAAa,EAAE,MAAM;oBACrB,aAAa,EAAE,eAAe;oBAC9B,KAAK,EAAE,SAAS;iBACjB,CAAC,CAAC;YACL,CAAC;oBACO,CAAC;gBACP,mBAAmB,CAAC,WAAW,EAAE,CAAC;YACpC,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,IAAI,CAAC;YACN;gBACE,KAAK,EAAE,gBAAgB;gBACvB,YAAY,EAAE,qBAAqB;gBACnC,SAAS,EAAE,qBAAqB;gBAChC,OAAO,EAAE,kBAAkB;aAC5B;YACD;gBACE,KAAK,EAAE,2BAA2B;gBAClC,YAAY,EAAE,8BAA8B;gBAC5C,SAAS,EAAE,sBAAsB;gBACjC,OAAO,EAAE,6BAA6B;aACvC;SACF,CAAC,CAAC,qDAAqD,EAAE,KAAK,EAAE,EAAE,YAAY,EAAE,SAAS,EAAE,OAAO,EAAE,EAAE,EAAE;YACvG,cAAc,CAAC,sBAAsB,CAAC,KAAK,IAAI,EAAE;gBAC/C,MAAM,eAAe,CAAC,wBAAwB,CAAC,CAC7C,EAAE,OAAO,EAAE,EAAE,GAAG,EAAE,SAAS,EAAE,EAAE,EAC/B,MAAM,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,uBAAuB,CAAC,EAAE;oBAChD,KAAK,EAAE,EAAE,IAAI,EAAE,4BAA4B,EAAE,OAAO,EAAE;iBACvD,CAAC,CACH,CAAC;YACJ,CAAC,CAAC,CAAC;YAEH,MAAM,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QACxE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gEAAgE,EAAE,KAAK,IAAI,EAAE;YAC9E,MAAM,OAAO,GAAG,EAAE,GAAG,EAAE,sBAAsB,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;YAChE,0BAA0B,CAAC,sBAAsB,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE;gBAClE,OAAO,CAAC,OAAO,CAAC,OAAO,GAAG,KAAK,CAAC;YAClC,CAAC,CAAC,CAAC;YACH,cAAc,CAAC,sBAAsB,CAAC,KAAK,IAAI,EAAE;gBAC/C,MAAM,wBAAwB,CAAC,OAAO,EAAE,0BAA0B,CAAC,CAAC;gBACpE,MAAM,KAAK,GAAG,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;gBAC7D,MAAM,eAAe,CAAC,gBAAgB,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE,KAAK,CAAC,CAAC;gBAC5D,MAAM,eAAe,CAAC,wBAAwB,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE,KAAK,CAAC,CAAC;YACtE,CAAC,CAAC,CAAC;YAEH,MAAM,OAAO,CAAC,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC;YAEpC,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACnC,MAAM,CAAC,0BAA0B,CAAC,CAAC,oBAAoB,CAAC,EAAE,OAAO,EAAE,EAAE,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;QAC1F,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uFAAuF,EAAE,KAAK,IAAI,EAAE;YACrG,MAAM,OAAO,GAAG,EAAE,GAAG,EAAE,sBAAsB,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;YAChE,cAAc,CAAC,sBAAsB,CAAC,KAAK,IAAI,EAAE;gBAC/C,MAAM,wBAAwB,CAAC,OAAO,EAAE,4BAA4B,CAAC,CAAC;gBACtE,MAAM,KAAK,GAAG,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;gBAC7D,MAAM,eAAe,CAAC,gBAAgB,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE,KAAK,CAAC,CAAC;gBAC5D,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBACpC,MAAM,eAAe,CAAC,wBAAwB,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE,KAAK,CAAC,CAAC;YACtE,CAAC,CAAC,CAAC;YAEH,MAAM,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,kCAAkC,CAAC,CAAC;QAClG,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,oDAAoD,EAAE,KAAK,IAAI,EAAE;YAClE,MAAM,IAAI,GAAG;gBACX,gBAAgB,EAAE,EAAE,CAAC,EAAE,EAAE;gBACzB,SAAS,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC,EAAE,CAAC;gBACtC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE;gBACX,GAAG,EAAE,EAAE,CAAC,EAAE,EAAE;aACb,CAAC;YACF,MAAM,OAAO,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;YACxB,cAAc,CAAC,sBAAsB,CAAC,KAAK,IAAI,EAAE;gBAC/C,MAAM,iBAAiB,CAAC,IAAI,EAAE,6BAA6B,EAAE,QAAQ,CAAC,GAAG,EAAE,IAAI,CAAC,EAAE,OAAO,CAAC,CAAC;YAC7F,CAAC,CAAC,CAAC;YAEH,MAAM,OAAO,CAAC,OAAO,EAAE,6BAA6B,CAAC,CAAC;YAEtD,MAAM,CAAC,OAAO,CAAC,CAAC,oBAAoB,CAAC,MAAM,CAAC,gBAAgB,CAAC,8BAA8B,CAAC,CAAC,CAAC;YAC9F,MAAM,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC,GAAG,CAAC,gBAAgB,EAAE,CAAC;YACrD,MAAM,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,gBAAgB,EAAE,CAAC;QAC5D,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,4EAA4E,EAAE,KAAK,IAAI,EAAE;YAC1F,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,GAAG,cAAc,CAAC,0BAA0B,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC;YAC3G,MAAM,OAAO,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;YACxB,cAAc,CAAC,sBAAsB,CAAC,KAAK,IAAI,EAAE;gBAC/C,MAAM,iBAAiB,CAAC,IAAI,EAAE,0BAA0B,EAAE,QAAQ,EAAE,EAAE,OAAO,CAAC,CAAC;YACjF,CAAC,CAAC,CAAC;YAEH,MAAM,OAAO,CAAC,OAAO,EAAE,0BAA0B,CAAC,CAAC;YAEnD,MAAM,CAAC,OAAO,CAAC,CAAC,oBAAoB,CAAC,MAAM,CAAC,gBAAgB,CAAC,8BAA8B,CAAC,CAAC,CAAC;YAC9F,MAAM,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,gBAAgB,EAAE,CAAC;YAC1D,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,oBAAoB,CAAC,UAAU,EAAE,SAAS,CAAC,QAAQ,CAAC,CAAC;QACxE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,qEAAqE,EAAE,KAAK,IAAI,EAAE;YACnF,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,GAAG,cAAc,CAAC,sBAAsB,EAAE,CAAC,IAAI,EAAE,EAAE,CAC1E,IAAI,CAAC,OAAO,CAAC,6BAA6B,EAAE,sBAAsB,CAAC,CACpE,CAAC;YACF,MAAM,OAAO,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;YACxB,cAAc,CAAC,sBAAsB,CAAC,KAAK,IAAI,EAAE;gBAC/C,MAAM,iBAAiB,CAAC,IAAI,EAAE,sBAAsB,EAAE,QAAQ,EAAE,EAAE,OAAO,CAAC,CAAC;YAC7E,CAAC,CAAC,CAAC;YAEH,MAAM,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE,sBAAsB,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,kCAAkC,CAAC,CAAC;YAE3G,MAAM,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,gBAAgB,EAAE,CAAC;YACvC,MAAM,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,gBAAgB,EAAE,CAAC;YAC1D,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,oBAAoB,CAAC,eAAe,EAAE,SAAS,CAAC,aAAa,CAAC,CAAC;QAClF,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,mEAAmE,EAAE,KAAK,IAAI,EAAE;YACjF,MAAM,EAAE,IAAI,EAAE,GAAG,cAAc,CAAC,0BAA0B,EAAE,CAAC,IAAI,EAAE,EAAE,CACnE,IAAI,CAAC,OAAO,CAAC,mCAAmC,EAAE,uBAAuB,CAAC,CAC3E,CAAC;YACF,MAAM,OAAO,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;YACxB,cAAc,CAAC,sBAAsB,CAAC,KAAK,IAAI,EAAE;gBAC/C,MAAM,iBAAiB,CAAC,IAAI,EAAE,0BAA0B,EAAE,QAAQ,EAAE,EAAE,OAAO,CAAC,CAAC;YACjF,CAAC,CAAC,CAAC;YAEH,MAAM,OAAO,CAAC,OAAO,EAAE,0BAA0B,CAAC,CAAC;YAEnD,MAAM,CAAC,OAAO,CAAC,CAAC,oBAAoB,CAAC,MAAM,CAAC,gBAAgB,CAAC,8BAA8B,CAAC,CAAC,CAAC;YAC9F,MAAM,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,gBAAgB,EAAE,CAAC;QAC5D,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,iGAAiG,EAAE,KAAK,IAAI,EAAE;YAC/G,MAAM,QAAQ,GAAG,wFAAwF,CAAC;YAC1G,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,GAAG,cAAc,CACxC,0BAA0B,EAC1B,CAAC,IAAI,EAAE,EAAE;gBACP,KAAK,MAAM,KAAK,IAAI,CAAC,kBAAkB,EAAE,oBAAoB,EAAE,uBAAuB,CAAC,EAAE,CAAC;oBACxF,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;gBACtB,CAAC;gBACD,IAAI,CAAC,OAAO,CAAC,4BAA4B,EAAE,0BAA0B,CAAC,CAAC;gBACvE,IAAI,CAAC,QAAQ,EAAE,CAAC;YAClB,CAAC,EACD,EAAE;iBACC,EAAE,EAAE;iBACJ,qBAAqB,CAAC,KAAK,CAAC;iBAC5B,qBAAqB,CAAC,KAAK,CAAC;iBAC5B,iBAAiB,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,aAAa,EAAE,UAAU,EAAE,CAAC,CACvE,CAAC;YACF,cAAc,CAAC,sBAAsB,CAAC,KAAK,IAAI,EAAE;gBAC/C,MAAM,iBAAiB,CAAC,IAAI,EAAE,0BAA0B,CAAC,CAAC;YAC5D,CAAC,CAAC,CAAC;YAEH,MAAM,OAAO,CAAC,OAAO,EAAE,0BAA0B,CAAC,CAAC;YAEnD,MAAM,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC,oBAAoB,CACrD,MAAM,CAAC,gBAAgB,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,aAAa,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CACzF,CAAC;YACF,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,oBAAoB,CAAC,UAAU,EAAE,SAAS,CAAC,QAAQ,CAAC,CAAC;YACtE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,oBAAoB,CAAC,eAAe,EAAE,SAAS,CAAC,aAAa,CAAC,CAAC;QAClF,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,0CAA0C,EAAE,KAAK,IAAI,EAAE;YACxD,qEAAqE;YACrE,cAAc,CAAC,qBAAqB,CAAC,SAAS,CAAC,CAAC;YAChD,gBAAgB,CAAC,YAAY,CAAC,qBAAqB,CAAC,EAAE,CAAC,CAAC;YAExD,MAAM,OAAO,CAAC,OAAO,EAAE,0BAA0B,CAAC,CAAC;YAEnD,MAAM,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,oBAAoB,CAAC,0BAA0B,EAAE,SAAS,CAAC,CAAC;QAClG,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yCAAyC,EAAE,KAAK,IAAI,EAAE;YACvD,MAAM,WAAW,GAAkB;gBACjC,EAAE,GAAG,EAAE,2BAA2B,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,aAAa,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE;gBAC/G,EAAE,GAAG,EAAE,2BAA2B,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,aAAa,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE;aAChH,CAAC;YAEF,yEAAyE;YACzE,uFAAuF;YACvF,gBAAgB,CAAC,YAAY,CAAC,qBAAqB,CAAC,WAAW,CAAC,CAAC;YAEjE,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,qBAAqB,CAAC,CAAC;YAE9D,MAAM,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;QACvC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,0CAA0C,EAAE,KAAK,IAAI,EAAE;YACxD,MAAM,OAAO,CAAC,OAAO,EAAE,qBAAqB,CAAC,CAAC;YAE9C,MAAM,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC,gBAAgB,EAAE,CAAC;QACtD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,sDAAsD,EAAE,KAAK,IAAI,EAAE;YACpE,MAAM,WAAW,GAAkB;gBACjC,EAAE,GAAG,EAAE,2BAA2B,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,aAAa,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE;aAChH,CAAC;YAEF,gBAAgB,CAAC,gBAAgB,CAAC,mBAAmB,CAAC,IAAI,CAAC,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;YACnF,gBAAgB,CAAC,YAAY,CAAC,qBAAqB,CAAC,WAAW,CAAC,CAAC,qBAAqB,CAAC,EAAE,CAAC,CAAC;YAE3F,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,qBAAqB,CAAC,CAAC;YAE9D,MAAM,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAClC,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,OAAO,EAAE,GAAG,EAAE;QACrB,EAAE,CAAC,yBAAyB,EAAE,KAAK,IAAI,EAAE;YACvC,qFAAqF;YACrF,IAAI,UAAsB,CAAC;YAC3B,MAAM,UAAU,GAAG,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;gBAC/C,UAAU,GAAG,OAAO,CAAC;YACvB,CAAC,CAAC,CAAC;YACH,cAAc,CAAC,mBAAmB,CAAC,UAAU,CAAC,CAAC;YAE/C,qCAAqC;YACrC,MAAM,gBAAgB,GAAG,IAAI,cAAc,EAAE,CAAC;YAE9C,2DAA2D;YAC3D,MAAM,SAAS,GAAG,gBAAgB,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC;YAEhE,wEAAwE;YACxE,MAAM,kBAAkB,GAAG,SAAS,CAAC,IAAI,EAAE,CAAC;YAE5C,0DAA0D;YAC1D,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;YAExD,+CAA+C;YAC/C,gBAAgB,CAAC,KAAK,EAAE,CAAC;YAEzB,qDAAqD;YACrD,UAAW,EAAE,CAAC;YAEd,qCAAqC;YACrC,MAAM,kBAAkB,CAAC;YAEzB,MAAM,CAAC,mBAAmB,CAAC,CAAC,gBAAgB,EAAE,CAAC;QACjD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,oBAAoB,EAAE,GAAG,EAAE;QAClC,EAAE,CAAC,mDAAmD,EAAE,KAAK,IAAI,EAAE;YACjE,OAAO,CAAC,aAAa,CAAC,WAAW,CAAC,CAAC;YAEnC,MAAM,OAAO,CAAC,OAAO,EAAE,8BAA8B,CAAC,CAAC;YAEvD,MAAM,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,oBAAoB,CAAC,8BAA8B,EAAE,WAAW,CAAC,CAAC;QACxG,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,gBAAgB,EAAE,GAAG,EAAE;QAC9B,EAAE,CAAC,sDAAsD,EAAE,KAAK,IAAI,EAAE;YACpE,MAAM,KAAK,GAAiB;gBAC1B,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,EAAE,aAAa,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC;aACjF,CAAC;YAEF,OAAO,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;YAE/B,MAAM,OAAO,CAAC,OAAO,EAAE,qBAAqB,CAAC,CAAC;YAE9C,oEAAoE;YACpE,MAAM,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,gBAAgB,EAAE,CAAC;QACzD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,gBAAgB,EAAE,GAAG,EAAE;QAC9B,EAAE,CAAC,yBAAyB,EAAE,KAAK,IAAI,EAAE;YACvC,cAAc,CAAC,qBAAqB,CAAC,IAAI,KAAK,CAAC,cAAc,CAAC,CAAC,CAAC;YAEhE,MAAM,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE,qBAAqB,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;YAEtF,MAAM,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC,gBAAgB,EAAE,CAAC;QACtD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
1
|
+
{"version":3,"file":"crawlee-crawler.test.js","sourceRoot":"","sources":["../../src/crawler/crawlee-crawler.test.ts"],"names":[],"mappings":"AAEA,MAAM,gBAAgB,GAAG;IACvB,UAAU,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,SAAS,CAAC;IAChD,eAAe,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,CAAC,CAAC;IAC7C,eAAe,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC,EAAE,CAAC;IAC5C,mBAAmB,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,SAAS,CAAC;IACzD,SAAS,EAAE,EAAE,CAAC,EAAE,EAAE;IAClB,gBAAgB,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC,KAAK,CAAC;IAChD,YAAY,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,EAAE,CAAC;IAC3C,OAAO,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,SAAS,CAAC;CAC9C,CAAC;AACF,MAAM,EAAE,0BAA0B,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;IACvD,0BAA0B,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,SAAS,CAAC;CACjE,CAAC,CAAC,CAAC;AAEJ,EAAE,CAAC,IAAI,CAAC,oBAAoB,EAAE,GAAG,EAAE,CAAC,CAAC;IACnC,YAAY,EAAE;QACZ,OAAO,gBAAgB,CAAC;IAC1B,CAAC;CACF,CAAC,CAAC,CAAC;AAEJ,EAAE,CAAC,IAAI,CAAC,qBAAqB,EAAE,GAAG,EAAE,CAAC,CAAC;IACpC,gBAAgB,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC;QACxC,YAAY,EAAE,EAAE;QAChB,kBAAkB,EAAE,EAAE;QACtB,aAAa,EAAE,EAAE;QACjB,kBAAkB,EAAE,EAAE;QACtB,YAAY,EAAE,0BAA0B;KACzC,CAAC;CACH,CAAC,CAAC,CAAC;AAEJ,EAAE,CAAC,IAAI,CAAC,iBAAiB,EAAE,GAAG,EAAE,CAAC,CAAC;IAChC,SAAS,EAAE;QACT;YACE,IAAI,EAAE,SAAS;YACf,SAAS,EAAE;gBACT,cAAc,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,EAAE,OAAO,EAAE,mBAAmB,EAAE,aAAa,EAAE,MAAM,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC;aACjH;YACD,MAAM,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,IAAI,CAAC;SACxC;KACF;CACF,CAAC,CAAC,CAAC;AAEJ,yBAAyB;AACzB,MAAM,kBAAkB,GAAG,EAAE,cAAc,EAAE,CAAC,EAAE,aAAa,EAAE,CAAC,EAAE,CAAC;AACnE,MAAM,cAAc,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,kBAAkB,CAAC,CAAC;AACrE,MAAM,mBAAmB,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;AAEjE,EAAE,CAAC,IAAI,CAAC,SAAS,EAAE,GAAG,EAAE,CAAC,CAAC;IACxB,iBAAiB,EAAE,UAAU,OAK5B;QACC,wCAAwC;QACvC,MAA6C,CAAC,oBAAoB,GAAG,OAAO,CAAC,kBAAkB,CAAC;QAChG,MAAyC,CAAC,gBAAgB,GAAG,OAAO,CAAC,cAAc,CAAC;QACpF,MAAuC,CAAC,cAAc,GAAG,OAAO,CAAC,YAAY,CAAC;QAC9E,MAA+C,CAAC,sBAAsB,GAAG,OAAO,CAAC,oBAAoB,CAAC;QACvG,OAAO;YACL,GAAG,EAAE,KAAK,IAAI,EAAE,CAAC,CAAC,MAAM,cAAc,EAAE,CAAC,IAAI,kBAAkB;YAC/D,QAAQ,EAAE,mBAAmB;SAC9B,CAAC;IACJ,CAAC;CACF,CAAC,CAAC,CAAC;AAEJ,uBAAuB;AACvB,OAAO,EAAE,cAAc,EAAgB,MAAM,sBAAsB,CAAC;AAMpE,SAAS,iBAAiB;IACxB,OAAQ,UAA8D,CAAC,gBAAgB,CAAC;AAC1F,CAAC;AAED,SAAS,eAAe,CAAC,IAAiD;IACxE,OAAQ,UAAsD,CAAC,IAAI,CAAC,CAAC;AACvE,CAAC;AAED,KAAK,UAAU,OAAO,CAAC,OAAuB,EAAE,GAAW;IACzD,MAAM,OAAO,GAAkB,EAAE,CAAC;IAClC,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;QAC9C,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACvB,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,QAAQ,CAAC,MAAM,GAAG,GAAG,EAAE,OAAO,GAAG,KAAK,EAAE,SAAkB;IACjE,OAAO;QACL,MAAM,EAAE,GAAG,EAAE,CAAC,MAAM;QACpB,WAAW,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC;QAC5D,GAAG,CAAC,SAAS,IAAI,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,mBAAmB,EAAE,GAAG,EAAE,CAAC,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,SAAS,EAAE,CAAC,EAAE,CAAC;KACnG,CAAC;AACJ,CAAC;AAED,SAAS,cAAc,CACrB,GAAW,EACX,MAAkI,EAClI,QAAQ,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,KAAK,CAAC;IAE3C,MAAM,SAAS,GAAG,EAAE,CAAC;IACrB,MAAM,SAAS,GAAsE,EAAE,CAAC;IACxF,IAAI,MAAM,GAAG,KAAK,CAAC;IACnB,MAAM,IAAI,GAAG;QACX,SAAS,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC,SAAS,CAAC;QAC7C,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,KAAmC,EAAE,QAA4B,EAAE,EAAE;YAC9E,SAAS,CAAC,KAAK,CAAC,GAAG,QAAQ,CAAC;QAC9B,CAAC,CAAC;QACF,GAAG,EAAE,EAAE,CAAC,EAAE,EAAE;QACZ,gBAAgB,EAAE,EAAE,CAAC,EAAE,CAAC,KAAK,IAAI,EAAE;YACjC,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,MAAM,GAAG,IAAI,CAAC;gBACd,MAAM,CAAC;oBACL,QAAQ,EAAE,CAAC,MAAM,GAAG,GAAG,EAAE,OAAO,GAAG,KAAK,EAAE,EAAE,CAAC,SAAS,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,MAAM,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;oBACvG,OAAO,EAAE,CAAC,KAAK,EAAE,SAAS,EAAE,EAAE,CAC5B,SAAS,CAAC,aAAa,EAAE,CAAC;wBACxB,mBAAmB,EAAE,GAAG,EAAE,CAAC,IAAI;wBAC/B,KAAK,EAAE,GAAG,EAAE,CAAC,SAAS;wBACtB,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC;wBACrC,GAAG,CAAC,SAAS,IAAI,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,SAAS,EAAE,CAAC;qBAC3C,CAAC;iBACL,CAAC,CAAC;YACL,CAAC;QACH,CAAC,CAAC;QACF,QAAQ;QACR,GAAG,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC,GAAG,CAAC;QACjC,KAAK,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,MAAM,CAAC;KACzC,CAAC;IACF,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC;AACxC,CAAC;AAED,KAAK,UAAU,iBAAiB,CAAC,IAAY,EAAE,GAAW,EAAE,eAAe,GAAG,QAAQ,EAAE,EAAE,OAAO,GAAG,EAAE,CAAC,EAAE,EAAE;IACzG,MAAM,iBAAiB,EAAE,CAAC;QACxB,OAAO,EAAE,EAAE,GAAG,EAAE;QAChB,QAAQ,EAAE,eAAe;QACzB,IAAI;QACJ,YAAY,EAAE,EAAE,CAAC,EAAE,EAAE;QACrB,GAAG,EAAE,EAAE,KAAK,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,OAAO,EAAE;KACjD,CAAC,CAAC;AACL,CAAC;AAED,KAAK,UAAU,wBAAwB,CAAC,OAAe,EAAE,SAAiB;IACxE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,SAAS,EAAE,GAAG,cAAc,CAAC,EAAE,EAAE,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IACpE,MAAM,KAAK,GAAI,MAAgG,CAAC,oBAAqB,CAAC;IACtI,MAAM,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,CAAE,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC;IACvC,SAAS,CAAC,aAAa,EAAE,CAAC;QACxB,mBAAmB,EAAE,GAAG,EAAE,CAAC,IAAI;QAC/B,KAAK,EAAE,GAAG,EAAE,CAAC,SAAS;QACtB,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,SAAS,EAAE,mCAAmC,EAAE,CAAC;QACnE,GAAG,EAAE,GAAG,EAAE,CAAC,SAAS;KACrB,CAAC,CAAC;AACL,CAAC;AAED,QAAQ,CAAC,gBAAgB,EAAE,GAAG,EAAE;IAC9B,IAAI,OAAuB,CAAC;IAE5B,UAAU,CAAC,GAAG,EAAE;QACd,EAAE,CAAC,aAAa,EAAE,CAAC;QACnB,OAAO,GAAG,IAAI,cAAc,EAAE,CAAC;QAC/B,gBAAgB,CAAC,gBAAgB,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;QACzD,gBAAgB,CAAC,YAAY,CAAC,iBAAiB,CAAC,EAAE,CAAC,CAAC;IACtD,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,OAAO,EAAE,GAAG,EAAE;QACrB,EAAE,CAAC,4EAA4E,EAAE,KAAK,IAAI,EAAE;YAC1F,MAAM,iBAAiB;gBACrB,KAAK,CAAC,cAAc;oBAClB,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,aAAa,EAAE,UAAU,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,IAAI,EAAE,UAAU,EAAE,EAAE,CAAC;gBAC3G,CAAC;aACF;YAED,MAAM,cAAc,GAClB,OAOD,CAAC,cAAc,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAC/B,MAAM,WAAW,GAAG;gBAClB,QAAQ,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,EAAE,OAAO,EAAE,SAAS,EAAE,aAAa,EAAE,UAAU,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;aACvG,CAAC;YAEF,MAAM,MAAM,CAAC,cAAc,CAAC,WAAW,EAAE,SAAS,EAAE,IAAI,iBAAiB,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,aAAa,CAAC;gBACnG,OAAO,EAAE,SAAS;gBAClB,aAAa,EAAE,UAAU;gBACzB,aAAa,EAAE,mBAAmB;gBAClC,KAAK,EAAE,OAAO;aACf,CAAC,CAAC;YAEH,MAAM,YAAY,GAAG;gBACnB,QAAQ,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,qBAAqB,CAAC,IAAI,KAAK,CAAC,kBAAkB,CAAC,CAAC,CAAC,qBAAqB,CAAC,gBAAgB,CAAC;aAC/G,CAAC;YACF,MAAM,MAAM,CAAC,cAAc,CAAC,YAAY,EAAE,SAAS,EAAE,IAAI,iBAAiB,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,aAAa,CAAC;gBACpG,OAAO,EAAE,gBAAgB;gBACzB,aAAa,EAAE,MAAM;gBACrB,aAAa,EAAE,eAAe;aAC/B,CAAC,CAAC;YAEH,MAAM,mBAAmB,GAAG,EAAE;iBAC3B,KAAK,CACJ,OAEC,EACD,kBAAkB,CACnB;iBACA,iBAAiB,CAAC;gBACjB,QAAQ,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE,aAAa,EAAE,UAAU,EAAE,KAAK,EAAE,uBAAuB,EAAE,CAAC;aAChH,CAAC,CAAC;YACL,IAAI,CAAC;gBACH,MAAM,qBAAqB,GAAG;oBAC5B,QAAQ,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,qBAAqB,CAAC,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC,CAAC,qBAAqB,CAAC,yBAAyB,CAAC;iBAC9H,CAAC;gBACF,MAAM,MAAM,CAAC,cAAc,CAAC,qBAAqB,EAAE,WAAW,EAAE,IAAI,iBAAiB,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC;oBACzG,OAAO,EAAE,yBAAyB;oBAClC,aAAa,EAAE,MAAM;oBACrB,aAAa,EAAE,eAAe;oBAC9B,KAAK,EAAE,SAAS;iBACjB,CAAC,CAAC;YACL,CAAC;oBACO,CAAC;gBACP,mBAAmB,CAAC,WAAW,EAAE,CAAC;YACpC,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,IAAI,CAAC;YACN;gBACE,KAAK,EAAE,gBAAgB;gBACvB,YAAY,EAAE,qBAAqB;gBACnC,SAAS,EAAE,qBAAqB;gBAChC,OAAO,EAAE,kBAAkB;aAC5B;YACD;gBACE,KAAK,EAAE,2BAA2B;gBAClC,YAAY,EAAE,8BAA8B;gBAC5C,SAAS,EAAE,sBAAsB;gBACjC,OAAO,EAAE,6BAA6B;aACvC;SACF,CAAC,CAAC,qDAAqD,EAAE,KAAK,EAAE,EAAE,YAAY,EAAE,SAAS,EAAE,OAAO,EAAE,EAAE,EAAE;YACvG,cAAc,CAAC,sBAAsB,CAAC,KAAK,IAAI,EAAE;gBAC/C,MAAM,eAAe,CAAC,wBAAwB,CAAC,CAC7C,EAAE,OAAO,EAAE,EAAE,GAAG,EAAE,SAAS,EAAE,EAAE,EAC/B,MAAM,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,uBAAuB,CAAC,EAAE;oBAChD,KAAK,EAAE,EAAE,IAAI,EAAE,4BAA4B,EAAE,OAAO,EAAE;iBACvD,CAAC,CACH,CAAC;YACJ,CAAC,CAAC,CAAC;YAEH,MAAM,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QACxE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gEAAgE,EAAE,KAAK,IAAI,EAAE;YAC9E,MAAM,OAAO,GAAG,EAAE,GAAG,EAAE,sBAAsB,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;YAChE,0BAA0B,CAAC,sBAAsB,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE;gBAClE,OAAO,CAAC,OAAO,CAAC,OAAO,GAAG,KAAK,CAAC;YAClC,CAAC,CAAC,CAAC;YACH,cAAc,CAAC,sBAAsB,CAAC,KAAK,IAAI,EAAE;gBAC/C,MAAM,wBAAwB,CAAC,OAAO,EAAE,0BAA0B,CAAC,CAAC;gBACpE,MAAM,KAAK,GAAG,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;gBAC7D,MAAM,eAAe,CAAC,gBAAgB,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE,KAAK,CAAC,CAAC;gBAC5D,MAAM,eAAe,CAAC,wBAAwB,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE,KAAK,CAAC,CAAC;YACtE,CAAC,CAAC,CAAC;YAEH,MAAM,OAAO,CAAC,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC;YAEpC,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACnC,MAAM,CAAC,0BAA0B,CAAC,CAAC,oBAAoB,CAAC,EAAE,OAAO,EAAE,EAAE,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;QAC1F,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uFAAuF,EAAE,KAAK,IAAI,EAAE;YACrG,MAAM,OAAO,GAAG,EAAE,GAAG,EAAE,sBAAsB,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;YAChE,cAAc,CAAC,sBAAsB,CAAC,KAAK,IAAI,EAAE;gBAC/C,MAAM,wBAAwB,CAAC,OAAO,EAAE,4BAA4B,CAAC,CAAC;gBACtE,MAAM,KAAK,GAAG,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;gBAC7D,MAAM,eAAe,CAAC,gBAAgB,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE,KAAK,CAAC,CAAC;gBAC5D,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBACpC,MAAM,eAAe,CAAC,wBAAwB,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE,KAAK,CAAC,CAAC;YACtE,CAAC,CAAC,CAAC;YAEH,MAAM,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,kCAAkC,CAAC,CAAC;QAClG,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,oDAAoD,EAAE,KAAK,IAAI,EAAE;YAClE,MAAM,IAAI,GAAG;gBACX,gBAAgB,EAAE,EAAE,CAAC,EAAE,EAAE;gBACzB,SAAS,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC,EAAE,CAAC;gBACtC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE;gBACX,GAAG,EAAE,EAAE,CAAC,EAAE,EAAE;aACb,CAAC;YACF,MAAM,OAAO,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;YACxB,cAAc,CAAC,sBAAsB,CAAC,KAAK,IAAI,EAAE;gBAC/C,MAAM,iBAAiB,CAAC,IAAI,EAAE,6BAA6B,EAAE,QAAQ,CAAC,GAAG,EAAE,IAAI,CAAC,EAAE,OAAO,CAAC,CAAC;YAC7F,CAAC,CAAC,CAAC;YAEH,MAAM,OAAO,CAAC,OAAO,EAAE,6BAA6B,CAAC,CAAC;YAEtD,MAAM,CAAC,OAAO,CAAC,CAAC,oBAAoB,CAAC,MAAM,CAAC,gBAAgB,CAAC,8BAA8B,CAAC,CAAC,CAAC;YAC9F,MAAM,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC,GAAG,CAAC,gBAAgB,EAAE,CAAC;YACrD,MAAM,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,gBAAgB,EAAE,CAAC;QAC5D,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,4EAA4E,EAAE,KAAK,IAAI,EAAE;YAC1F,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,GAAG,cAAc,CAAC,0BAA0B,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC;YAC3G,MAAM,OAAO,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;YACxB,cAAc,CAAC,sBAAsB,CAAC,KAAK,IAAI,EAAE;gBAC/C,MAAM,iBAAiB,CAAC,IAAI,EAAE,0BAA0B,EAAE,QAAQ,EAAE,EAAE,OAAO,CAAC,CAAC;YACjF,CAAC,CAAC,CAAC;YAEH,MAAM,OAAO,CAAC,OAAO,EAAE,0BAA0B,CAAC,CAAC;YAEnD,MAAM,CAAC,OAAO,CAAC,CAAC,oBAAoB,CAAC,MAAM,CAAC,gBAAgB,CAAC,8BAA8B,CAAC,CAAC,CAAC;YAC9F,MAAM,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,gBAAgB,EAAE,CAAC;YAC1D,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,oBAAoB,CAAC,UAAU,EAAE,SAAS,CAAC,QAAQ,CAAC,CAAC;QACxE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,qEAAqE,EAAE,KAAK,IAAI,EAAE;YACnF,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,GAAG,cAAc,CAAC,sBAAsB,EAAE,CAAC,IAAI,EAAE,EAAE,CAC1E,IAAI,CAAC,OAAO,CAAC,6BAA6B,EAAE,sBAAsB,CAAC,CACpE,CAAC;YACF,MAAM,OAAO,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;YACxB,cAAc,CAAC,sBAAsB,CAAC,KAAK,IAAI,EAAE;gBAC/C,MAAM,iBAAiB,CAAC,IAAI,EAAE,sBAAsB,EAAE,QAAQ,EAAE,EAAE,OAAO,CAAC,CAAC;YAC7E,CAAC,CAAC,CAAC;YAEH,MAAM,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE,sBAAsB,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,kCAAkC,CAAC,CAAC;YAE3G,MAAM,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,gBAAgB,EAAE,CAAC;YACvC,MAAM,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,gBAAgB,EAAE,CAAC;YAC1D,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,oBAAoB,CAAC,eAAe,EAAE,SAAS,CAAC,aAAa,CAAC,CAAC;QAClF,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,mEAAmE,EAAE,KAAK,IAAI,EAAE;YACjF,MAAM,EAAE,IAAI,EAAE,GAAG,cAAc,CAAC,0BAA0B,EAAE,CAAC,IAAI,EAAE,EAAE,CACnE,IAAI,CAAC,OAAO,CAAC,mCAAmC,EAAE,uBAAuB,CAAC,CAC3E,CAAC;YACF,MAAM,OAAO,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;YACxB,cAAc,CAAC,sBAAsB,CAAC,KAAK,IAAI,EAAE;gBAC/C,MAAM,iBAAiB,CAAC,IAAI,EAAE,0BAA0B,EAAE,QAAQ,EAAE,EAAE,OAAO,CAAC,CAAC;YACjF,CAAC,CAAC,CAAC;YAEH,MAAM,OAAO,CAAC,OAAO,EAAE,0BAA0B,CAAC,CAAC;YAEnD,MAAM,CAAC,OAAO,CAAC,CAAC,oBAAoB,CAAC,MAAM,CAAC,gBAAgB,CAAC,8BAA8B,CAAC,CAAC,CAAC;YAC9F,MAAM,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,gBAAgB,EAAE,CAAC;QAC5D,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,iGAAiG,EAAE,KAAK,IAAI,EAAE;YAC/G,MAAM,QAAQ,GAAG,wFAAwF,CAAC;YAC1G,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,GAAG,cAAc,CACxC,0BAA0B,EAC1B,CAAC,IAAI,EAAE,EAAE;gBACP,KAAK,MAAM,KAAK,IAAI,CAAC,kBAAkB,EAAE,oBAAoB,EAAE,uBAAuB,CAAC,EAAE,CAAC;oBACxF,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;gBACtB,CAAC;gBACD,IAAI,CAAC,OAAO,CAAC,4BAA4B,EAAE,0BAA0B,CAAC,CAAC;gBACvE,IAAI,CAAC,QAAQ,EAAE,CAAC;YAClB,CAAC,EACD,EAAE;iBACC,EAAE,EAAE;iBACJ,qBAAqB,CAAC,KAAK,CAAC;iBAC5B,qBAAqB,CAAC,KAAK,CAAC;iBAC5B,iBAAiB,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,aAAa,EAAE,UAAU,EAAE,CAAC,CACvE,CAAC;YACF,cAAc,CAAC,sBAAsB,CAAC,KAAK,IAAI,EAAE;gBAC/C,MAAM,iBAAiB,CAAC,IAAI,EAAE,0BAA0B,CAAC,CAAC;YAC5D,CAAC,CAAC,CAAC;YAEH,MAAM,OAAO,CAAC,OAAO,EAAE,0BAA0B,CAAC,CAAC;YAEnD,MAAM,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC,oBAAoB,CACrD,MAAM,CAAC,gBAAgB,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,aAAa,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CACzF,CAAC;YACF,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,oBAAoB,CAAC,UAAU,EAAE,SAAS,CAAC,QAAQ,CAAC,CAAC;YACtE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,oBAAoB,CAAC,eAAe,EAAE,SAAS,CAAC,aAAa,CAAC,CAAC;QAClF,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,0CAA0C,EAAE,KAAK,IAAI,EAAE;YACxD,MAAM,OAAO,CAAC,OAAO,EAAE,0BAA0B,CAAC,CAAC;YAEnD,MAAM,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,oBAAoB,CAAC,0BAA0B,EAAE,SAAS,CAAC,CAAC;QAClG,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yCAAyC,EAAE,KAAK,IAAI,EAAE;YACvD,MAAM,WAAW,GAAkB;gBACjC,EAAE,GAAG,EAAE,2BAA2B,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,aAAa,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE;gBAC/G,EAAE,GAAG,EAAE,2BAA2B,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,aAAa,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE;aAChH,CAAC;YAEF,yEAAyE;YACzE,uFAAuF;YACvF,gBAAgB,CAAC,YAAY,CAAC,qBAAqB,CAAC,WAAW,CAAC,CAAC;YAEjE,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,qBAAqB,CAAC,CAAC;YAE9D,MAAM,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;QACvC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,0CAA0C,EAAE,KAAK,IAAI,EAAE;YACxD,MAAM,OAAO,CAAC,OAAO,EAAE,qBAAqB,CAAC,CAAC;YAE9C,MAAM,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC,gBAAgB,EAAE,CAAC;QACtD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,sDAAsD,EAAE,KAAK,IAAI,EAAE;YACpE,MAAM,WAAW,GAAkB;gBACjC,EAAE,GAAG,EAAE,2BAA2B,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,aAAa,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE;aAChH,CAAC;YAEF,gBAAgB,CAAC,gBAAgB,CAAC,mBAAmB,CAAC,IAAI,CAAC,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;YACnF,gBAAgB,CAAC,YAAY,CAAC,qBAAqB,CAAC,WAAW,CAAC,CAAC,qBAAqB,CAAC,EAAE,CAAC,CAAC;YAE3F,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,qBAAqB,CAAC,CAAC;YAE9D,MAAM,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAClC,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,OAAO,EAAE,GAAG,EAAE;QACrB,EAAE,CAAC,yBAAyB,EAAE,KAAK,IAAI,EAAE;YACvC,qFAAqF;YACrF,IAAI,UAA2D,CAAC;YAChE,MAAM,UAAU,GAAG,IAAI,OAAO,CAA4B,CAAC,OAAO,EAAE,EAAE;gBACpE,UAAU,GAAG,OAAO,CAAC;YACvB,CAAC,CAAC,CAAC;YACH,cAAc,CAAC,mBAAmB,CAAC,UAAU,CAAC,CAAC;YAE/C,qCAAqC;YACrC,MAAM,gBAAgB,GAAG,IAAI,cAAc,EAAE,CAAC;YAE9C,2DAA2D;YAC3D,MAAM,SAAS,GAAG,gBAAgB,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC;YAEhE,wEAAwE;YACxE,MAAM,kBAAkB,GAAG,SAAS,CAAC,IAAI,EAAE,CAAC;YAE5C,0DAA0D;YAC1D,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;YAExD,+CAA+C;YAC/C,gBAAgB,CAAC,KAAK,EAAE,CAAC;YAEzB,qDAAqD;YACrD,UAAW,CAAC,kBAAkB,CAAC,CAAC;YAEhC,qCAAqC;YACrC,MAAM,kBAAkB,CAAC;YAEzB,MAAM,CAAC,mBAAmB,CAAC,CAAC,gBAAgB,EAAE,CAAC;QACjD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,oBAAoB,EAAE,GAAG,EAAE;QAClC,EAAE,CAAC,mDAAmD,EAAE,KAAK,IAAI,EAAE;YACjE,OAAO,CAAC,aAAa,CAAC,WAAW,CAAC,CAAC;YAEnC,MAAM,OAAO,CAAC,OAAO,EAAE,8BAA8B,CAAC,CAAC;YAEvD,MAAM,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,oBAAoB,CAAC,8BAA8B,EAAE,WAAW,CAAC,CAAC;QACxG,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,gBAAgB,EAAE,GAAG,EAAE;QAC9B,EAAE,CAAC,sDAAsD,EAAE,KAAK,IAAI,EAAE;YACpE,MAAM,KAAK,GAAiB;gBAC1B,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,EAAE,aAAa,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC;aACjF,CAAC;YAEF,OAAO,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;YAE/B,MAAM,OAAO,CAAC,OAAO,EAAE,qBAAqB,CAAC,CAAC;YAE9C,oEAAoE;YACpE,MAAM,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,gBAAgB,EAAE,CAAC;QACzD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,gBAAgB,EAAE,GAAG,EAAE;QAC9B,EAAE,CAAC,0EAA0E,EAAE,KAAK,IAAI,EAAE;YACxF,cAAc,CAAC,sBAAsB,CAAC,KAAK,IAAI,EAAE;gBAC/C,IAAI,CAAC;oBACH,MAAM,iBAAiB,CACrB;wBACE,SAAS,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC,EAAE,CAAC;wBACtC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE;wBACX,GAAG,EAAE,EAAE,CAAC,EAAE,EAAE;wBACZ,gBAAgB,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,qBAAqB,CAAC,IAAI,KAAK,CAAC,kBAAkB,CAAC,CAAC,CAAC,iBAAiB,CAAC,SAAS,CAAC;qBAC5G,EACD,qBAAqB,CACtB,CAAC;gBACJ,CAAC;gBACD,MAAM,CAAC;oBACL,OAAO,EAAE,cAAc,EAAE,CAAC,EAAE,aAAa,EAAE,CAAC,EAAE,CAAC;gBACjD,CAAC;gBAED,OAAO,kBAAkB,CAAC;YAC5B,CAAC,CAAC,CAAC;YAEH,MAAM,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE,qBAAqB,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,6CAA6C,CAAC,CAAC;YAErH,MAAM,CAAC,gBAAgB,CAAC,YAAY,CAAC,CAAC,GAAG,CAAC,gBAAgB,EAAE,CAAC;YAC7D,MAAM,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC,gBAAgB,EAAE,CAAC;QACtD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yBAAyB,EAAE,KAAK,IAAI,EAAE;YACvC,cAAc,CAAC,qBAAqB,CAAC,IAAI,KAAK,CAAC,cAAc,CAAC,CAAC,CAAC;YAEhE,MAAM,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE,qBAAqB,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;YAEtF,MAAM,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC,gBAAgB,EAAE,CAAC;QACtD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
@@ -1,8 +1,6 @@
|
|
|
1
|
-
import { CrawlResult
|
|
1
|
+
import { CrawlResult } from '../types.js';
|
|
2
2
|
import { StorageState } from './crawlee-crawler.js';
|
|
3
|
-
export declare class DocsCrawler
|
|
4
|
-
private readonly maxDepth;
|
|
5
|
-
private readonly maxRequestsPerCrawl;
|
|
3
|
+
export declare class DocsCrawler {
|
|
6
4
|
private readonly githubToken?;
|
|
7
5
|
private readonly onProgress?;
|
|
8
6
|
private readonly GITHUB_HOST;
|
|
@@ -10,7 +8,7 @@ export declare class DocsCrawler implements WebCrawler {
|
|
|
10
8
|
private storageState?;
|
|
11
9
|
private pathPrefix?;
|
|
12
10
|
private activeCrawler?;
|
|
13
|
-
constructor(
|
|
11
|
+
constructor(githubToken?: string | undefined, onProgress?: ((progress: number, description: string) => void) | undefined);
|
|
14
12
|
/**
|
|
15
13
|
* Set an optional path prefix to restrict crawling to URLs under this path.
|
|
16
14
|
* Only pages whose path starts with this prefix will be crawled.
|
|
@@ -21,6 +19,6 @@ export declare class DocsCrawler implements WebCrawler {
|
|
|
21
19
|
* Set authentication storage state (cookies) to use when crawling
|
|
22
20
|
*/
|
|
23
21
|
setStorageState(state: StorageState): void;
|
|
24
|
-
crawl(url: string): AsyncGenerator<CrawlResult,
|
|
22
|
+
crawl(url: string): AsyncGenerator<CrawlResult, void, unknown>;
|
|
25
23
|
abort(): void;
|
|
26
24
|
}
|