@crawlee/http 4.0.0-beta.12 → 4.0.0-beta.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -1
- package/internals/file-download.d.ts +28 -7
- package/internals/file-download.d.ts.map +1 -1
- package/internals/file-download.js +63 -32
- package/internals/file-download.js.map +1 -1
- package/internals/http-crawler.d.ts +23 -7
- package/internals/http-crawler.d.ts.map +1 -1
- package/internals/http-crawler.js +40 -68
- package/internals/http-crawler.js.map +1 -1
- package/internals/utils.d.ts +1 -1
- package/internals/utils.d.ts.map +1 -1
- package/internals/utils.js +2 -2
- package/internals/utils.js.map +1 -1
- package/package.json +6 -6
- package/tsconfig.build.tsbuildinfo +0 -1
package/README.md
CHANGED
|
@@ -9,6 +9,10 @@
|
|
|
9
9
|
<small>A web scraping and browser automation library</small>
|
|
10
10
|
</h1>
|
|
11
11
|
|
|
12
|
+
<p align=center>
|
|
13
|
+
<a href="https://trendshift.io/repositories/5179" target="_blank"><img src="https://trendshift.io/api/badge/repositories/5179" alt="apify%2Fcrawlee | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
|
|
14
|
+
</p>
|
|
15
|
+
|
|
12
16
|
<p align=center>
|
|
13
17
|
<a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/v/@crawlee/core.svg" alt="NPM latest version" data-canonical-src="https://img.shields.io/npm/v/@crawlee/core/next.svg" style="max-width: 100%;"></a>
|
|
14
18
|
<a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/dm/@crawlee/core.svg" alt="Downloads" data-canonical-src="https://img.shields.io/npm/dm/@crawlee/core.svg" style="max-width: 100%;"></a>
|
|
@@ -24,7 +28,7 @@ Crawlee is available as the [`crawlee`](https://www.npmjs.com/package/crawlee) N
|
|
|
24
28
|
|
|
25
29
|
> 👉 **View full documentation, guides and examples on the [Crawlee project website](https://crawlee.dev)** 👈
|
|
26
30
|
|
|
27
|
-
>
|
|
31
|
+
> Do you prefer 🐍 Python instead of JavaScript? [👉 Checkout Crawlee for Python 👈](https://github.com/apify/crawlee-python).
|
|
28
32
|
|
|
29
33
|
## Installation
|
|
30
34
|
|
|
@@ -1,22 +1,45 @@
|
|
|
1
|
-
import
|
|
1
|
+
import { Transform } from 'node:stream';
|
|
2
2
|
import type { BasicCrawlerOptions } from '@crawlee/basic';
|
|
3
3
|
import { BasicCrawler } from '@crawlee/basic';
|
|
4
|
-
import type { CrawlingContext,
|
|
4
|
+
import type { CrawlingContext, LoadedRequest, Request } from '@crawlee/core';
|
|
5
5
|
import type { Dictionary } from '@crawlee/types';
|
|
6
6
|
import type { ErrorHandler, GetUserDataFromRequest, InternalHttpHook, RequestHandler, RouterRoutes } from '../index.js';
|
|
7
7
|
export type FileDownloadErrorHandler<UserData extends Dictionary = any> = ErrorHandler<FileDownloadCrawlingContext<UserData>>;
|
|
8
8
|
export type FileDownloadHook<UserData extends Dictionary = any> = InternalHttpHook<FileDownloadCrawlingContext<UserData>>;
|
|
9
9
|
export interface FileDownloadCrawlingContext<UserData extends Dictionary = any> extends CrawlingContext<UserData> {
|
|
10
10
|
request: LoadedRequest<Request<UserData>>;
|
|
11
|
-
response:
|
|
12
|
-
body: Promise<Buffer>;
|
|
13
|
-
stream: Readable;
|
|
11
|
+
response: Response;
|
|
14
12
|
contentType: {
|
|
15
13
|
type: string;
|
|
16
14
|
encoding: BufferEncoding;
|
|
17
15
|
};
|
|
18
16
|
}
|
|
19
17
|
export type FileDownloadRequestHandler<UserData extends Dictionary = any> = RequestHandler<FileDownloadCrawlingContext<UserData>>;
|
|
18
|
+
/**
|
|
19
|
+
* Creates a transform stream that throws an error if the source data speed is below the specified minimum speed.
|
|
20
|
+
* This `Transform` checks the amount of data every `checkProgressInterval` milliseconds.
|
|
21
|
+
* If the stream has received less than `minSpeedKbps * historyLengthMs / 1000` bytes in the last `historyLengthMs` milliseconds,
|
|
22
|
+
* it will throw an error.
|
|
23
|
+
*
|
|
24
|
+
* Can be used e.g. to abort a download if the network speed is too slow.
|
|
25
|
+
* @returns Transform stream that monitors the speed of the incoming data.
|
|
26
|
+
*/
|
|
27
|
+
export declare function MinimumSpeedStream({ minSpeedKbps, historyLengthMs, checkProgressInterval: checkProgressIntervalMs, }: {
|
|
28
|
+
minSpeedKbps: number;
|
|
29
|
+
historyLengthMs?: number;
|
|
30
|
+
checkProgressInterval?: number;
|
|
31
|
+
}): Transform;
|
|
32
|
+
/**
|
|
33
|
+
* Creates a transform stream that logs the progress of the incoming data.
|
|
34
|
+
* This `Transform` calls the `logProgress` function every `loggingInterval` milliseconds with the number of bytes received so far.
|
|
35
|
+
*
|
|
36
|
+
* Can be used e.g. to log the progress of a download.
|
|
37
|
+
* @returns Transform stream logging the progress of the incoming data.
|
|
38
|
+
*/
|
|
39
|
+
export declare function ByteCounterStream({ logTransferredBytes, loggingInterval, }: {
|
|
40
|
+
logTransferredBytes: (transferredBytes: number) => void;
|
|
41
|
+
loggingInterval?: number;
|
|
42
|
+
}): Transform;
|
|
20
43
|
/**
|
|
21
44
|
* Provides a framework for downloading files in parallel using plain HTTP requests. The URLs to download are fed either from a static list of URLs or they can be added on the fly from another crawler.
|
|
22
45
|
*
|
|
@@ -61,10 +84,8 @@ export type FileDownloadRequestHandler<UserData extends Dictionary = any> = Requ
|
|
|
61
84
|
* ```
|
|
62
85
|
*/
|
|
63
86
|
export declare class FileDownload extends BasicCrawler<FileDownloadCrawlingContext> {
|
|
64
|
-
#private;
|
|
65
87
|
constructor(options?: BasicCrawlerOptions<FileDownloadCrawlingContext>);
|
|
66
88
|
private initiateDownload;
|
|
67
|
-
private cleanupDownload;
|
|
68
89
|
}
|
|
69
90
|
/**
|
|
70
91
|
* Creates new {@link Router} instance that works based on request labels.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"file-download.d.ts","sourceRoot":"","sources":["../../src/internals/file-download.ts"],"names":[],"mappings":"AAAA,OAAO,
|
|
1
|
+
{"version":3,"file":"file-download.d.ts","sourceRoot":"","sources":["../../src/internals/file-download.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAGxC,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AAC1D,OAAO,EAAE,YAAY,EAAmB,MAAM,gBAAgB,CAAC;AAC/D,OAAO,KAAK,EAAE,eAAe,EAAE,aAAa,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAC7E,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAEjD,OAAO,KAAK,EAAE,YAAY,EAAE,sBAAsB,EAAE,gBAAgB,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAIxH,MAAM,MAAM,wBAAwB,CAChC,QAAQ,SAAS,UAAU,GAAG,GAAG,IACjC,YAAY,CAAC,2BAA2B,CAAC,QAAQ,CAAC,CAAC,CAAC;AAExD,MAAM,MAAM,gBAAgB,CACxB,QAAQ,SAAS,UAAU,GAAG,GAAG,IACjC,gBAAgB,CAAC,2BAA2B,CAAC,QAAQ,CAAC,CAAC,CAAC;AAE5D,MAAM,WAAW,2BAA2B,CACxC,QAAQ,SAAS,UAAU,GAAG,GAAG,CACnC,SAAQ,eAAe,CAAC,QAAQ,CAAC;IAC/B,OAAO,EAAE,aAAa,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC1C,QAAQ,EAAE,QAAQ,CAAC;IACnB,WAAW,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,cAAc,CAAA;KAAE,CAAC;CAC3D;AAED,MAAM,MAAM,0BAA0B,CAClC,QAAQ,SAAS,UAAU,GAAG,GAAG,IACjC,cAAc,CAAC,2BAA2B,CAAC,QAAQ,CAAC,CAAC,CAAC;AAE1D;;;;;;;;GAQG;AACH,wBAAgB,kBAAkB,CAAC,EAC/B,YAAY,EACZ,eAAsB,EACtB,qBAAqB,EAAE,uBAA6B,GACvD,EAAE;IACC,YAAY,EAAE,MAAM,CAAC;IACrB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,qBAAqB,CAAC,EAAE,MAAM,CAAC;CAClC,GAAG,SAAS,CA4BZ;AAED;;;;;;GAMG;AACH,wBAAgB,iBAAiB,CAAC,EAC9B,mBAAmB,EACnB,eAAsB,GACzB,EAAE;IACC,mBAAmB,EAAE,CAAC,gBAAgB,EAAE,MAAM,KAAK,IAAI,CAAC;IACxD,eAAe,CAAC,EAAE,MAAM,CAAC;CAC5B,GAAG,SAAS,CAoBZ;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA0CG;AACH,qBAAa,YAAa,SAAQ,YAAY,CAAC,2BAA2B,CAAC;gBAE3D,OAAO,GAAE,mBAAmB,CAAC,2BAA2B,CAAM;YAa5D,gBAAgB;CAmBjC;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,wBAAgB,gBAAgB,CAC5B,OAAO,SAAS,2BAA2B,GAAG,2BAA2B,EACzE,QAAQ,SAAS,UAAU,GAAG,sBAAsB,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,EAC1E,MAAM,CAAC,EAAE,YAAY,CAAC,OAAO,EAAE,QAAQ,CAAC,mDAEzC"}
|
|
@@ -1,8 +1,66 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { Transform } from 'node:stream';
|
|
2
2
|
import { finished } from 'node:stream/promises';
|
|
3
3
|
import { BasicCrawler, ContextPipeline } from '@crawlee/basic';
|
|
4
4
|
import { Router } from '../index.js';
|
|
5
5
|
import { parseContentTypeFromResponse } from './utils.js';
|
|
6
|
+
/**
|
|
7
|
+
* Creates a transform stream that throws an error if the source data speed is below the specified minimum speed.
|
|
8
|
+
* This `Transform` checks the amount of data every `checkProgressInterval` milliseconds.
|
|
9
|
+
* If the stream has received less than `minSpeedKbps * historyLengthMs / 1000` bytes in the last `historyLengthMs` milliseconds,
|
|
10
|
+
* it will throw an error.
|
|
11
|
+
*
|
|
12
|
+
* Can be used e.g. to abort a download if the network speed is too slow.
|
|
13
|
+
* @returns Transform stream that monitors the speed of the incoming data.
|
|
14
|
+
*/
|
|
15
|
+
export function MinimumSpeedStream({ minSpeedKbps, historyLengthMs = 10e3, checkProgressInterval: checkProgressIntervalMs = 5e3, }) {
|
|
16
|
+
let snapshots = [];
|
|
17
|
+
const checkInterval = setInterval(() => {
|
|
18
|
+
const now = Date.now();
|
|
19
|
+
snapshots = snapshots.filter((snapshot) => now - snapshot.timestamp < historyLengthMs);
|
|
20
|
+
const totalBytes = snapshots.reduce((acc, snapshot) => acc + snapshot.bytes, 0);
|
|
21
|
+
const elapsed = (now - (snapshots[0]?.timestamp ?? 0)) / 1000;
|
|
22
|
+
if (totalBytes / 1024 / elapsed < minSpeedKbps) {
|
|
23
|
+
clearInterval(checkInterval);
|
|
24
|
+
stream.emit('error', new Error(`Stream speed too slow, aborting...`));
|
|
25
|
+
}
|
|
26
|
+
}, checkProgressIntervalMs);
|
|
27
|
+
const stream = new Transform({
|
|
28
|
+
transform: (chunk, _, callback) => {
|
|
29
|
+
snapshots.push({ timestamp: Date.now(), bytes: chunk.length });
|
|
30
|
+
callback(null, chunk);
|
|
31
|
+
},
|
|
32
|
+
final: (callback) => {
|
|
33
|
+
clearInterval(checkInterval);
|
|
34
|
+
callback();
|
|
35
|
+
},
|
|
36
|
+
});
|
|
37
|
+
return stream;
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* Creates a transform stream that logs the progress of the incoming data.
|
|
41
|
+
* This `Transform` calls the `logProgress` function every `loggingInterval` milliseconds with the number of bytes received so far.
|
|
42
|
+
*
|
|
43
|
+
* Can be used e.g. to log the progress of a download.
|
|
44
|
+
* @returns Transform stream logging the progress of the incoming data.
|
|
45
|
+
*/
|
|
46
|
+
export function ByteCounterStream({ logTransferredBytes, loggingInterval = 5000, }) {
|
|
47
|
+
let transferredBytes = 0;
|
|
48
|
+
let lastLogTime = Date.now();
|
|
49
|
+
return new Transform({
|
|
50
|
+
transform: (chunk, _, callback) => {
|
|
51
|
+
transferredBytes += chunk.length;
|
|
52
|
+
if (Date.now() - lastLogTime > loggingInterval) {
|
|
53
|
+
lastLogTime = Date.now();
|
|
54
|
+
logTransferredBytes(transferredBytes);
|
|
55
|
+
}
|
|
56
|
+
callback(null, chunk);
|
|
57
|
+
},
|
|
58
|
+
flush: (callback) => {
|
|
59
|
+
logTransferredBytes(transferredBytes);
|
|
60
|
+
callback();
|
|
61
|
+
},
|
|
62
|
+
});
|
|
63
|
+
}
|
|
6
64
|
/**
|
|
7
65
|
* Provides a framework for downloading files in parallel using plain HTTP requests. The URLs to download are fed either from a static list of URLs or they can be added on the fly from another crawler.
|
|
8
66
|
*
|
|
@@ -47,14 +105,15 @@ import { parseContentTypeFromResponse } from './utils.js';
|
|
|
47
105
|
* ```
|
|
48
106
|
*/
|
|
49
107
|
export class FileDownload extends BasicCrawler {
|
|
50
|
-
#contextInternals = Symbol('contextInternals');
|
|
51
108
|
// TODO hooks
|
|
52
109
|
constructor(options = {}) {
|
|
53
110
|
super({
|
|
54
111
|
...options,
|
|
55
112
|
contextPipelineBuilder: () => ContextPipeline.create().compose({
|
|
56
|
-
action: this.initiateDownload
|
|
57
|
-
cleanup:
|
|
113
|
+
action: async (context) => this.initiateDownload(context),
|
|
114
|
+
cleanup: async (context) => {
|
|
115
|
+
await (context.response.body ? finished(context.response.body) : Promise.resolve());
|
|
116
|
+
},
|
|
58
117
|
}),
|
|
59
118
|
});
|
|
60
119
|
}
|
|
@@ -66,41 +125,13 @@ export class FileDownload extends BasicCrawler {
|
|
|
66
125
|
});
|
|
67
126
|
const { type, charset: encoding } = parseContentTypeFromResponse(response);
|
|
68
127
|
context.request.url = response.url;
|
|
69
|
-
const pollingInterval = setInterval(() => {
|
|
70
|
-
const { total, transferred } = response.downloadProgress;
|
|
71
|
-
if (transferred > 0) {
|
|
72
|
-
context.log.debug(`Downloaded ${transferred} bytes of ${total ?? 0} bytes from ${context.request.url}.`);
|
|
73
|
-
}
|
|
74
|
-
}, 5000);
|
|
75
128
|
const contextExtension = {
|
|
76
|
-
[this.#contextInternals]: { pollingInterval },
|
|
77
129
|
request: context.request,
|
|
78
130
|
response,
|
|
79
131
|
contentType: { type, encoding },
|
|
80
|
-
stream: response.stream,
|
|
81
|
-
get body() {
|
|
82
|
-
return buffer(response.stream);
|
|
83
|
-
},
|
|
84
132
|
};
|
|
85
133
|
return contextExtension;
|
|
86
134
|
}
|
|
87
|
-
async cleanupDownload(context, error) {
|
|
88
|
-
clearInterval(context[this.#contextInternals].pollingInterval);
|
|
89
|
-
// If there was no error and the stream is still readable, wait for it to be consumed before proceeding
|
|
90
|
-
if (error === undefined) {
|
|
91
|
-
if (!context.stream.destroyed && context.stream.readable) {
|
|
92
|
-
try {
|
|
93
|
-
await finished(context.stream);
|
|
94
|
-
}
|
|
95
|
-
catch {
|
|
96
|
-
// Stream might have encountered an error or been closed, which is fine
|
|
97
|
-
}
|
|
98
|
-
}
|
|
99
|
-
}
|
|
100
|
-
else {
|
|
101
|
-
context.stream.destroy();
|
|
102
|
-
}
|
|
103
|
-
}
|
|
104
135
|
}
|
|
105
136
|
/**
|
|
106
137
|
* Creates new {@link Router} instance that works based on request labels.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"file-download.js","sourceRoot":"","sources":["../../src/internals/file-download.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"file-download.js","sourceRoot":"","sources":["../../src/internals/file-download.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AACxC,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAGhD,OAAO,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AAK/D,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AACrC,OAAO,EAAE,4BAA4B,EAAE,MAAM,YAAY,CAAC;AAsB1D;;;;;;;;GAQG;AACH,MAAM,UAAU,kBAAkB,CAAC,EAC/B,YAAY,EACZ,eAAe,GAAG,IAAI,EACtB,qBAAqB,EAAE,uBAAuB,GAAG,GAAG,GAKvD;IACG,IAAI,SAAS,GAA2C,EAAE,CAAC;IAE3D,MAAM,aAAa,GAAG,WAAW,CAAC,GAAG,EAAE;QACnC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAEvB,SAAS,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,GAAG,GAAG,QAAQ,CAAC,SAAS,GAAG,eAAe,CAAC,CAAC;QACvF,MAAM,UAAU,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,QAAQ,EAAE,EAAE,CAAC,GAAG,GAAG,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;QAChF,MAAM,OAAO,GAAG,CAAC,GAAG,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;QAE9D,IAAI,UAAU,GAAG,IAAI,GAAG,OAAO,GAAG,YAAY,EAAE,CAAC;YAC7C,aAAa,CAAC,aAAa,CAAC,CAAC;YAC7B,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAC,CAAC;QAC1E,CAAC;IACL,CAAC,EAAE,uBAAuB,CAAC,CAAC;IAE5B,MAAM,MAAM,GAAG,IAAI,SAAS,CAAC;QACzB,SAAS,EAAE,CAAC,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,EAAE;YAC9B,SAAS,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,KAAK,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;YAC/D,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QAC1B,CAAC;QACD,KAAK,EAAE,CAAC,QAAQ,EAAE,EAAE;YAChB,aAAa,CAAC,aAAa,CAAC,CAAC;YAC7B,QAAQ,EAAE,CAAC;QACf,CAAC;KACJ,CAAC,CAAC;IAEH,OAAO,MAAM,CAAC;AAClB,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,iBAAiB,CAAC,EAC9B,mBAAmB,EACnB,eAAe,GAAG,IAAI,GAIzB;IACG,IAAI,gBAAgB,GAAG,CAAC,CAAC;IACzB,IAAI,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAE7B,OAAO,IAAI,SAAS,CAAC;QACjB,SAAS,EAAE,CAAC,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,EAAE;YAC9B,gBAAgB,IAAI,KAAK,CAAC,MAAM,CAAC;YAEjC,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,WAAW,GAAG,eAAe,EAAE,CAAC;gBAC7C,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;gBACzB,mBAAmB,CAAC,gBAAgB,CAAC,CAAC;YAC1C,CAAC;YAED,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QAC1B,CAAC;QACD,KAAK,EAAE,CAAC,QAAQ,EAAE,EAAE;YAChB,mBAAmB,CAAC,gBAAgB,CAAC,CAAC;YACtC,QAAQ,EAAE,CAAC;QACf,CAAC;KACJ,CAAC,CAAC;AACP,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA0CG;AACH,MAAM,OAAO,YAAa,SAAQ,YAAyC;IACvE,aAAa;IACb,YAAY,UAA4D,EAAE;QACtE,KAAK,CAAC;YACF,GAAG,OAAO;YACV,sBAAsB,EAAE,GAAG,EAAE,CACzB,eAAe,CAAC,MAAM,EAAmB,CAAC,OAAO,CAAC;gBAC9C,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC;gBACzD,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;oBACvB,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAW,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;gBAC/F,CAAC;aACJ,CAAC;SACT,CAAC,CAAC;IACP,CAAC;IAEO,KAAK,CAAC,gBAAgB,CAAC,OAAwB;QACnD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC;YAC1C,GAAG,EAAE,OAAO,CAAC,OAAO,CAAC,GAAG;YACxB,OAAO,EAAE,EAAE,OAAO,EAAE,SAAS,EAAE;YAC/B,QAAQ,EAAE,OAAO,CAAC,SAAS,EAAE,GAAG;SACnC,CAAC,CAAC;QAEH,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,4BAA4B,CAAC,QAAQ,CAAC,CAAC;QAE3E,OAAO,CAAC,OAAO,CAAC,GAAG,GAAG,QAAQ,CAAC,GAAG,CAAC;QAEnC,MAAM,gBAAgB,GAAG;YACrB,OAAO,EAAE,OAAO,CAAC,OAAiC;YAClD,QAAQ;YACR,WAAW,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;SAClC,CAAC;QAEF,OAAO,gBAAgB,CAAC;IAC5B,CAAC;CACJ;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,MAAM,UAAU,gBAAgB,CAG9B,MAAwC;IACtC,OAAO,MAAM,CAAC,MAAM,CAAU,MAAM,CAAC,CAAC;AAC1C,CAAC"}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { IncomingMessage } from 'node:http';
|
|
2
|
-
import
|
|
2
|
+
import { Readable } from 'node:stream';
|
|
3
3
|
import type { BasicCrawlerOptions, CrawlingContext, ErrorHandler, GetUserDataFromRequest, Request, RequestHandler, RequireContextPipeline, RouterRoutes, Session } from '@crawlee/basic';
|
|
4
4
|
import { BasicCrawler, Configuration, ContextPipeline } from '@crawlee/basic';
|
|
5
5
|
import type { HttpResponse, LoadedRequest } from '@crawlee/core';
|
|
@@ -116,7 +116,7 @@ interface CrawlingContextWithReponse<UserData extends Dictionary = any> extends
|
|
|
116
116
|
/**
|
|
117
117
|
* The HTTP response object containing status code, headers, and other response metadata.
|
|
118
118
|
*/
|
|
119
|
-
response:
|
|
119
|
+
response: Response;
|
|
120
120
|
}
|
|
121
121
|
/**
|
|
122
122
|
* @internal
|
|
@@ -296,6 +296,8 @@ export declare class HttpCrawler<Context extends InternalHttpCrawlingContext<any
|
|
|
296
296
|
maxSessionRotations: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
|
|
297
297
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
298
298
|
maxRequestsPerCrawl: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
|
|
299
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
300
|
+
maxCrawlDepth: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
|
|
299
301
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
300
302
|
autoscaledPoolOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
301
303
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
@@ -311,7 +313,7 @@ export declare class HttpCrawler<Context extends InternalHttpCrawlingContext<any
|
|
|
311
313
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
312
314
|
retryOnBlocked: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
|
|
313
315
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
314
|
-
respectRobotsTxtFile: import("ow").
|
|
316
|
+
respectRobotsTxtFile: import("ow").AnyPredicate<boolean | object>;
|
|
315
317
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
316
318
|
onSkippedRequest: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
|
|
317
319
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
@@ -349,11 +351,25 @@ export declare class HttpCrawler<Context extends InternalHttpCrawlingContext<any
|
|
|
349
351
|
* on the request such as only downloading the request body if the
|
|
350
352
|
* received content type matches text/html, application/xml, application/xhtml+xml.
|
|
351
353
|
*/
|
|
352
|
-
protected _requestFunction({ request, session, proxyUrl, gotOptions, }: RequestFunctionOptions): Promise<
|
|
354
|
+
protected _requestFunction({ request, session, proxyUrl, gotOptions, }: RequestFunctionOptions): Promise<Response>;
|
|
353
355
|
/**
|
|
354
356
|
* Encodes and parses response according to the provided content type
|
|
355
357
|
*/
|
|
356
|
-
|
|
358
|
+
protected _parseResponse(request: Request, response: Response): Promise<{
|
|
359
|
+
response: Response;
|
|
360
|
+
contentType: {
|
|
361
|
+
type: string;
|
|
362
|
+
encoding: BufferEncoding;
|
|
363
|
+
};
|
|
364
|
+
body: string;
|
|
365
|
+
} | {
|
|
366
|
+
body: Buffer<ArrayBuffer>;
|
|
367
|
+
response: Response;
|
|
368
|
+
contentType: {
|
|
369
|
+
type: string;
|
|
370
|
+
encoding: BufferEncoding;
|
|
371
|
+
};
|
|
372
|
+
}>;
|
|
357
373
|
/**
|
|
358
374
|
* Combines the provided `requestOptions` with mandatory (non-overridable) values.
|
|
359
375
|
*/
|
|
@@ -430,9 +446,9 @@ export declare class HttpCrawler<Context extends InternalHttpCrawlingContext<any
|
|
|
430
446
|
} & import("got-scraping").Context & Required<Pick<OptionsInit, "url">> & {
|
|
431
447
|
isStream: true;
|
|
432
448
|
};
|
|
433
|
-
protected _encodeResponse(request: Request, response:
|
|
449
|
+
protected _encodeResponse(request: Request, response: Response, encoding: BufferEncoding): {
|
|
434
450
|
encoding: BufferEncoding;
|
|
435
|
-
response:
|
|
451
|
+
response: Response;
|
|
436
452
|
};
|
|
437
453
|
/**
|
|
438
454
|
* Checks and extends supported mime types
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"http-crawler.d.ts","sourceRoot":"","sources":["../../src/internals/http-crawler.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,
|
|
1
|
+
{"version":3,"file":"http-crawler.d.ts","sourceRoot":"","sources":["../../src/internals/http-crawler.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AACjD,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAGvC,OAAO,KAAK,EAER,mBAAmB,EACnB,eAAe,EACf,YAAY,EACZ,sBAAsB,EACtB,OAAO,EACP,cAAc,EACd,sBAAsB,EACtB,YAAY,EACZ,OAAO,EACV,MAAM,gBAAgB,CAAC;AACxB,OAAO,EACH,YAAY,EAEZ,aAAa,EACb,eAAe,EAOlB,MAAM,gBAAgB,CAAC;AACxB,OAAO,KAAK,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AACjE,OAAO,KAAK,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAC5D,OAAO,EAAE,KAAK,WAAW,EAAuB,MAAM,gBAAgB,CAAC;AAEvE,OAAO,KAAK,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAE9D,OAAO,KAAK,EAAE,MAAM,EAAE,WAAW,EAAqC,MAAM,cAAc,CAAC;AAG3F,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAQ3C;;;GAGG;AACH,MAAM,MAAM,aAAa,GAAG,IAAI,CAAC,YAAY,EAAE,MAAM,CAAC,GAClD,eAAe,GAAG;IACd,IAAI,CAAC,EAAE,OAAO,CAAC;CAClB,CAAC;AAkBN,MAAM,MAAM,gBAAgB,CACxB,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,SAAS,GAAG,GAAG,IAChC,YAAY,CAAC,mBAAmB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;AAE1D,MAAM,WAAW,kBAAkB,CAC/B,OAAO,SAAS,2BAA2B,GAAG,2BAA2B,EACzE,eAAe,SAAS,OAAO,GAAG,OAAO,CAC3C,SAAQ,mBAAmB,CAAC,OAAO,EAAE,eAAe,CAAC;IACnD;;OAEG;IACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAE/B;;OAEG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;IAE1B;;;;;;;;;;;;;;;OAeG;IACH,kBAAkB,CAAC,EAAE,gBAAgB,CAAC,eAAe,CAAC,EAAE,CAAC;IAEzD;;;;;;;;;;;OAWG;IACH,mBAAmB,CAAC,EAAE,CAAC,CAAC,eAAe,EAAE,0BAA0B,KAAK,SAAS,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;IAE3F;;;OAGG;IACH,mBAAmB,CAAC,EAAE,MAAM,EAAE,CAAC;IAE/B;;;;;;;;;;OAUG;IACH,uBAAuB,CAAC,EAAE,MAAM,CAAC;IAEjC;;;;;;;;OAQG;IACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAE/B;;;;;OAKG;IACH,wBAAwB,CAAC,EAAE,OAAO,CAAC;IAEnC;;;OAGG;IACH,0BAA0B,CAAC,EAAE,MAAM,EAAE,CAAC;IAEtC;;;OAGG;IACH,8BAA8B,CAAC,EAAE,MAAM,EAAE,CAAC;CAC7C;AAED;;GAEG;AACH,MAAM,MAAM,gBAAgB,CAAC,OAAO,IAAI,CAAC,eAAe,EAAE,OAAO,EAAE,UAAU,EAAE,WAAW,KAAK,SAAS,CAAC,IAAI,CAAC,CAAC;AAE/G,MAAM,MAAM,QAAQ,CAChB,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,SAAS,GAAG,GAAG,IAChC,gBAAgB,CAAC,mBAAmB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;AAE9D,UAAU,0BAA0B,CAChC,QAAQ,SAAS,UAAU,GAAG,GAAG,CACnC,SAAQ,eAAe,CAAC,QAAQ,CAAC;IAC/B;;OAEG;IACH,OAAO,EAAE,aAAa,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;IAE1C;;OAEG;IACH,QAAQ,EAAE,QAAQ,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,2BAA2B,CACxC,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,SAAS,GAAG,GAAG,CAClC,SAAQ,0BAA0B,CAAC,QAAQ,CAAC;IAC1C;;;;;OAKG;IACH,IAAI,EAAE,MAAM,GAAG,MAAM,CAAC;IAEtB;;OAEG;IACH,IAAI,EAAE,QAAQ,CAAC;IAEf;;OAEG;IACH,WAAW,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,cAAc,CAAA;KAAE,CAAC;IAExD;;;;;;;;;;;OAWG;IACH,eAAe,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAErE;;;;;;;;;;;OAWG;IACH,gBAAgB,CAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC;CACjF;AAED,MAAM,WAAW,mBAAmB,CAAC,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,QAAQ,SAAS,SAAS,GAAG,GAAG,CACpG,SAAQ,2BAA2B,CAAC,QAAQ,EAAE,QAAQ,CAAC;CAAG;AAE9D,MAAM,MAAM,kBAAkB,CAC1B,QAAQ,SAAS,UAAU,GAAG,GAAG,EAAE,2EAA2E;AAC9G,QAAQ,SAAS,SAAS,GAAG,GAAG,IAChC,cAAc,CAAC,mBAAmB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;AAE5D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkEG;AACH,qBAAa,WAAW,CACpB,OAAO,SAAS,2BAA2B,CAAC,GAAG,EAAE,GAAG,CAAC,GAAG,2BAA2B,EACnF,gBAAgB,GAAG,EAAE,EACrB,eAAe,SAAS,OAAO,GAAG,OAAO,GAAG,gBAAgB,CAC9D,SAAQ,YAAY,CAAC,OAAO,EAAE,eAAe,CAAC;aAmCtB,MAAM;IAlC5B,SAAS,CAAC,kBAAkB,EAAE,gBAAgB,CAAC,eAAe,CAAC,EAAE,CAAC;IAClE,SAAS,CAAC,mBAAmB,EAAE,CAAC,CAAC,eAAe,EAAE,0BAA0B,KAAK,SAAS,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;IACpG,SAAS,CAAC,wBAAwB,EAAE,OAAO,CAAC;IAC5C,SAAS,CAAC,uBAAuB,EAAE,MAAM,CAAC;IAC1C,SAAS,CAAC,eAAe,EAAE,OAAO,CAAC;IACnC,SAAS,CAAC,uBAAuB,CAAC,EAAE,MAAM,CAAC;IAC3C,SAAS,CAAC,qBAAqB,CAAC,EAAE,MAAM,CAAC;IACzC,SAAS,CAAC,8BAA8B,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IACtD,SAAS,CAAC,0BAA0B,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IAClD,SAAS,CAAC,QAAQ,CAAC,kBAAkB,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IAEnD,iBAA0B,YAAY;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;MAepC;IAEF;;OAEG;gBAEC,OAAO,GAAE,kBAAkB,CAAC,OAAO,EAAE,eAAe,CAAC,GACjD,sBAAsB,CAAC,2BAA2B,EAAE,OAAO,CAAa,EAC1D,MAAM,gBAAkC;IAkE9D,SAAS,CAAC,oBAAoB,IAAI,eAAe,CAAC,eAAe,EAAE,2BAA2B,CAAC;YASjF,eAAe;YAmDf,mBAAmB;YAuEnB,6BAA6B;cAQ3B,gBAAgB,CAAC,eAAe,EAAE,2BAA2B,GAAG,OAAO,CAAC,MAAM,GAAG,KAAK,CAAC;IAyBvG;;OAEG;IACH,SAAS,CAAC,aAAa,CACnB,EAAE,OAAO,EAAE,OAAO,EAAE,EAAE,eAAe,EACrC,UAAU,EAAE,WAAW,EACvB,cAAc,EAAE,MAAM,EACtB,eAAe,EAAE,MAAM;IAkD3B;;;;OAIG;cACa,gBAAgB,CAAC,EAC7B,OAAO,EACP,OAAO,EACP,QAAQ,EACR,UAAU,GACb,EAAE,sBAAsB,GAAG,OAAO,CAAC,QAAQ,CAAC;IAwB7C;;OAEG;cACa,cAAc,CAAC,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ;;;;;;;;;;;;;;;IA2CnE;;OAEG;IACH,SAAS,CAAC,kBAAkB,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,OAAO,EAAE,QAAQ,CAAC,EAAE,MAAM,EAAE,UAAU,CAAC,EAAE,WAAW;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBACpB,IAAI;;IAgC7F,SAAS,CAAC,eAAe,CACrB,OAAO,EAAE,OAAO,EAChB,QAAQ,EAAE,QAAQ,EAClB,QAAQ,EAAE,cAAc,GACzB;QACC,QAAQ,EAAE,cAAc,CAAC;QACzB,QAAQ,EAAE,QAAQ,CAAC;KACtB;IAsCD;;OAEG;IACH,SAAS,CAAC,yBAAyB,CAAC,mBAAmB,EAAE,CAAC,MAAM,GAAG,WAAW,GAAG,YAAY,CAAC,EAAE;IAgBhG;;OAEG;IACH,SAAS,CAAC,qBAAqB,CAAC,OAAO,CAAC,EAAE,OAAO;IAKjD,OAAO,CAAC,oBAAoB;IAkB5B;;OAEG;IACH,OAAO,CAAC,iBAAiB,CAuBvB;CACL;AAED,UAAU,sBAAsB;IAC5B,OAAO,EAAE,OAAO,CAAC;IACjB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,WAAW,CAAC;CAC3B;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,wBAAgB,gBAAgB,CAC5B,OAAO,SAAS,mBAAmB,GAAG,mBAAmB,EACzD,QAAQ,SAAS,UAAU,GAAG,sBAAsB,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,EAC1E,MAAM,CAAC,EAAE,YAAY,CAAC,OAAO,EAAE,QAAQ,CAAC,mDAEzC"}
|
|
@@ -1,12 +1,12 @@
|
|
|
1
|
+
import { Readable } from 'node:stream';
|
|
1
2
|
import util from 'node:util';
|
|
2
|
-
import { BasicCrawler, Configuration, ContextPipeline, mergeCookies, processHttpRequestOptions, RequestState, Router, SessionError, } from '@crawlee/basic';
|
|
3
|
+
import { BasicCrawler, BLOCKED_STATUS_CODES, Configuration, ContextPipeline, mergeCookies, processHttpRequestOptions, RequestState, ResponseWithUrl, Router, SessionError, } from '@crawlee/basic';
|
|
3
4
|
import { RETRY_CSS_SELECTORS } from '@crawlee/utils';
|
|
4
5
|
import * as cheerio from 'cheerio';
|
|
5
6
|
import contentTypeParser from 'content-type';
|
|
6
7
|
import iconv from 'iconv-lite';
|
|
7
8
|
import ow from 'ow';
|
|
8
9
|
import { addTimeoutToPromise, tryCancel } from '@apify/timeout';
|
|
9
|
-
import { concatStreamToBuffer, readStreamToString } from '@apify/utilities';
|
|
10
10
|
import { parseContentTypeFromResponse } from './utils.js';
|
|
11
11
|
let TimeoutError;
|
|
12
12
|
/**
|
|
@@ -196,7 +196,7 @@ export class HttpCrawler extends BasicCrawler {
|
|
|
196
196
|
const proxyUrl = crawlingContext.proxyInfo?.url;
|
|
197
197
|
const httpResponse = await addTimeoutToPromise(async () => this._requestFunction({ request, session, proxyUrl, gotOptions }), this.navigationTimeoutMillis, `request timed out after ${this.navigationTimeoutMillis / 1000} seconds.`);
|
|
198
198
|
tryCancel();
|
|
199
|
-
request.loadedUrl = httpResponse
|
|
199
|
+
request.loadedUrl = httpResponse?.url;
|
|
200
200
|
request.state = RequestState.AFTER_NAV;
|
|
201
201
|
return { request: request, response: httpResponse };
|
|
202
202
|
}
|
|
@@ -240,7 +240,7 @@ export class HttpCrawler extends BasicCrawler {
|
|
|
240
240
|
return $;
|
|
241
241
|
};
|
|
242
242
|
if (this.useSessionPool) {
|
|
243
|
-
this._throwOnBlockedRequest(crawlingContext.session, response.
|
|
243
|
+
this._throwOnBlockedRequest(crawlingContext.session, response.status);
|
|
244
244
|
}
|
|
245
245
|
if (this.persistCookiesPerSession) {
|
|
246
246
|
crawlingContext.session.setCookiesFromResponse(response);
|
|
@@ -274,6 +274,15 @@ export class HttpCrawler extends BasicCrawler {
|
|
|
274
274
|
return `Found selectors: ${foundSelectors.join(', ')}`;
|
|
275
275
|
}
|
|
276
276
|
}
|
|
277
|
+
const blockedStatusCodes =
|
|
278
|
+
// eslint-disable-next-line dot-notation
|
|
279
|
+
(this.sessionPool?.['blockedStatusCodes'].length ?? 0) > 0
|
|
280
|
+
? // eslint-disable-next-line dot-notation
|
|
281
|
+
this.sessionPool['blockedStatusCodes']
|
|
282
|
+
: BLOCKED_STATUS_CODES;
|
|
283
|
+
if (blockedStatusCodes.includes(crawlingContext.response.status)) {
|
|
284
|
+
return `Blocked by status code ${crawlingContext.response.status}`;
|
|
285
|
+
}
|
|
277
286
|
return false;
|
|
278
287
|
}
|
|
279
288
|
/**
|
|
@@ -333,7 +342,7 @@ export class HttpCrawler extends BasicCrawler {
|
|
|
333
342
|
catch (e) {
|
|
334
343
|
if (e instanceof TimeoutError) {
|
|
335
344
|
this._handleRequestTimeout(session);
|
|
336
|
-
return
|
|
345
|
+
return new Response(); // this will never happen, as _handleRequestTimeout always throws
|
|
337
346
|
}
|
|
338
347
|
if (this.isProxyError(e)) {
|
|
339
348
|
throw new SessionError(this._getMessageFromError(e));
|
|
@@ -346,18 +355,18 @@ export class HttpCrawler extends BasicCrawler {
|
|
|
346
355
|
/**
|
|
347
356
|
* Encodes and parses response according to the provided content type
|
|
348
357
|
*/
|
|
349
|
-
async _parseResponse(request,
|
|
350
|
-
const {
|
|
351
|
-
const { type, charset } = parseContentTypeFromResponse(
|
|
352
|
-
const { response, encoding } = this._encodeResponse(request,
|
|
358
|
+
async _parseResponse(request, response) {
|
|
359
|
+
const { status } = response;
|
|
360
|
+
const { type, charset } = parseContentTypeFromResponse(response);
|
|
361
|
+
const { response: reencodedResponse, encoding } = this._encodeResponse(request, response, charset);
|
|
353
362
|
const contentType = { type, encoding };
|
|
354
|
-
if (
|
|
355
|
-
this.stats.registerStatusCode(
|
|
363
|
+
if (status >= 400 && status <= 599) {
|
|
364
|
+
this.stats.registerStatusCode(status);
|
|
356
365
|
}
|
|
357
|
-
const excludeError = this.ignoreHttpErrorStatusCodes.has(
|
|
358
|
-
const includeError = this.additionalHttpErrorStatusCodes.has(
|
|
359
|
-
if ((
|
|
360
|
-
const body = await
|
|
366
|
+
const excludeError = this.ignoreHttpErrorStatusCodes.has(status);
|
|
367
|
+
const includeError = this.additionalHttpErrorStatusCodes.has(status);
|
|
368
|
+
if ((status >= 500 && !excludeError) || includeError) {
|
|
369
|
+
const body = await reencodedResponse.text(); // TODO - this always uses UTF-8 (see https://developer.mozilla.org/en-US/docs/Web/API/Request/text)
|
|
361
370
|
// Errors are often sent as JSON, so attempt to parse them,
|
|
362
371
|
// despite Accept header being set to text/html.
|
|
363
372
|
if (type === APPLICATION_JSON_MIME_TYPE) {
|
|
@@ -365,19 +374,19 @@ export class HttpCrawler extends BasicCrawler {
|
|
|
365
374
|
let { message } = errorResponse;
|
|
366
375
|
if (!message)
|
|
367
376
|
message = util.inspect(errorResponse, { depth: 1, maxArrayLength: 10 });
|
|
368
|
-
throw new Error(`${
|
|
377
|
+
throw new Error(`${status} - ${message}`);
|
|
369
378
|
}
|
|
370
379
|
if (includeError) {
|
|
371
|
-
throw new Error(`${
|
|
380
|
+
throw new Error(`${status} - Error status code was set by user.`);
|
|
372
381
|
}
|
|
373
382
|
// It's not a JSON, so it's probably some text. Get the first 100 chars of it.
|
|
374
|
-
throw new Error(`${
|
|
383
|
+
throw new Error(`${status} - Internal Server Error: ${body.slice(0, 100)}`);
|
|
375
384
|
}
|
|
376
385
|
else if (HTML_AND_XML_MIME_TYPES.includes(type)) {
|
|
377
|
-
return { response, contentType, body: await
|
|
386
|
+
return { response, contentType, body: await response.text() };
|
|
378
387
|
}
|
|
379
388
|
else {
|
|
380
|
-
const body = await
|
|
389
|
+
const body = Buffer.from(await response.bytes());
|
|
381
390
|
return {
|
|
382
391
|
body,
|
|
383
392
|
response,
|
|
@@ -394,6 +403,7 @@ export class HttpCrawler extends BasicCrawler {
|
|
|
394
403
|
method: request.method,
|
|
395
404
|
proxyUrl,
|
|
396
405
|
timeout: { request: this.navigationTimeoutMillis },
|
|
406
|
+
cookieJar: this.persistCookiesPerSession ? session?.cookieJar : undefined,
|
|
397
407
|
sessionToken: session,
|
|
398
408
|
...gotOptions,
|
|
399
409
|
headers: { ...request.headers, ...gotOptions?.headers },
|
|
@@ -435,13 +445,13 @@ export class HttpCrawler extends BasicCrawler {
|
|
|
435
445
|
if (iconv.encodingExists(encoding)) {
|
|
436
446
|
const encodeStream = iconv.encodeStream(utf8);
|
|
437
447
|
const decodeStream = iconv.decodeStream(encoding).on('error', (err) => encodeStream.emit('error', err));
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
448
|
+
const reencodedBody = response.body
|
|
449
|
+
? Readable.toWeb(Readable.from(Readable.fromWeb(response.body)
|
|
450
|
+
.pipe(decodeStream)
|
|
451
|
+
.pipe(encodeStream)))
|
|
452
|
+
: null;
|
|
443
453
|
return {
|
|
444
|
-
response:
|
|
454
|
+
response: new ResponseWithUrl(reencodedBody, response),
|
|
445
455
|
encoding: utf8,
|
|
446
456
|
};
|
|
447
457
|
}
|
|
@@ -473,12 +483,12 @@ export class HttpCrawler extends BasicCrawler {
|
|
|
473
483
|
throw new Error(`request timed out after ${this.navigationTimeoutMillis / 1000} seconds.`);
|
|
474
484
|
}
|
|
475
485
|
_abortDownloadOfBody(request, response) {
|
|
476
|
-
const {
|
|
486
|
+
const { status } = response;
|
|
477
487
|
const { type } = parseContentTypeFromResponse(response);
|
|
478
488
|
// eslint-disable-next-line dot-notation -- accessing private property
|
|
479
489
|
const blockedStatusCodes = this.sessionPool ? this.sessionPool['blockedStatusCodes'] : [];
|
|
480
490
|
// if we retry the request, can the Content-Type change?
|
|
481
|
-
const isTransientContentType =
|
|
491
|
+
const isTransientContentType = status >= 500 || blockedStatusCodes.includes(status);
|
|
482
492
|
if (!this.supportedMimeTypes.has(type) && !this.supportedMimeTypes.has('*/*') && !isTransientContentType) {
|
|
483
493
|
request.noRetry = true;
|
|
484
494
|
throw new Error(`Resource ${request.url} served Content-Type ${type}, ` +
|
|
@@ -491,7 +501,7 @@ export class HttpCrawler extends BasicCrawler {
|
|
|
491
501
|
_requestAsBrowser = async (options, session) => {
|
|
492
502
|
const response = await this.httpClient.stream(processHttpRequestOptions({
|
|
493
503
|
...options,
|
|
494
|
-
cookieJar: options.cookieJar,
|
|
504
|
+
cookieJar: options.cookieJar,
|
|
495
505
|
responseType: 'text',
|
|
496
506
|
}), (redirectResponse, updatedRequest) => {
|
|
497
507
|
if (this.persistCookiesPerSession) {
|
|
@@ -502,47 +512,9 @@ export class HttpCrawler extends BasicCrawler {
|
|
|
502
512
|
}
|
|
503
513
|
}
|
|
504
514
|
});
|
|
505
|
-
return
|
|
515
|
+
return response;
|
|
506
516
|
};
|
|
507
517
|
}
|
|
508
|
-
/**
|
|
509
|
-
* The stream object returned from got does not have the below properties.
|
|
510
|
-
* At the same time, you can't read data directly from the response stream,
|
|
511
|
-
* because they won't get emitted unless you also read from the primary
|
|
512
|
-
* got stream. To be able to work with only one stream, we move the expected props
|
|
513
|
-
* from the response stream to the got stream.
|
|
514
|
-
* @internal
|
|
515
|
-
*/
|
|
516
|
-
function addResponsePropertiesToStream(stream, response) {
|
|
517
|
-
const properties = [
|
|
518
|
-
'statusCode',
|
|
519
|
-
'statusMessage',
|
|
520
|
-
'headers',
|
|
521
|
-
'complete',
|
|
522
|
-
'httpVersion',
|
|
523
|
-
'rawHeaders',
|
|
524
|
-
'rawTrailers',
|
|
525
|
-
'trailers',
|
|
526
|
-
'url',
|
|
527
|
-
'request',
|
|
528
|
-
];
|
|
529
|
-
stream.on('end', () => {
|
|
530
|
-
// @ts-expect-error
|
|
531
|
-
if (stream.rawTrailers)
|
|
532
|
-
stream.rawTrailers = response.rawTrailers; // TODO BC with got - remove in 4.0
|
|
533
|
-
// @ts-expect-error
|
|
534
|
-
if (stream.trailers)
|
|
535
|
-
stream.trailers = response.trailers;
|
|
536
|
-
// @ts-expect-error
|
|
537
|
-
stream.complete = response.complete;
|
|
538
|
-
});
|
|
539
|
-
for (const prop of properties) {
|
|
540
|
-
if (!(prop in stream)) {
|
|
541
|
-
stream[prop] = response[prop];
|
|
542
|
-
}
|
|
543
|
-
}
|
|
544
|
-
return stream;
|
|
545
|
-
}
|
|
546
518
|
/**
|
|
547
519
|
* Creates new {@link Router} instance that works based on request labels.
|
|
548
520
|
* This instance can then serve as a `requestHandler` of your {@link HttpCrawler}.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"http-crawler.js","sourceRoot":"","sources":["../../src/internals/http-crawler.ts"],"names":[],"mappings":"AAEA,OAAO,IAAI,MAAM,WAAW,CAAC;AAc7B,OAAO,EACH,YAAY,EACZ,aAAa,EACb,eAAe,EACf,YAAY,EACZ,yBAAyB,EACzB,YAAY,EACZ,MAAM,EACN,YAAY,GACf,MAAM,gBAAgB,CAAC;AAGxB,OAAO,EAAoB,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AACvE,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAEnC,OAAO,iBAAiB,MAAM,cAAc,CAAC;AAE7C,OAAO,KAAK,MAAM,YAAY,CAAC;AAC/B,OAAO,EAAE,MAAM,IAAI,CAAC;AAGpB,OAAO,EAAE,mBAAmB,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAChE,OAAO,EAAE,oBAAoB,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AAE5E,OAAO,EAAE,4BAA4B,EAAE,MAAM,YAAY,CAAC;AAE1D,IAAI,YAAsC,CAAC;AAW3C;;GAEG;AACH,MAAM,uBAAuB,GAAG,CAAC,WAAW,EAAE,UAAU,EAAE,uBAAuB,EAAE,iBAAiB,CAAC,CAAC;AACtG,MAAM,0BAA0B,GAAG,kBAAkB,CAAC;AACtD,MAAM,sCAAsC,GAA0B;IAClE,kBAAkB,EAAE,EAAE;IACtB,kBAAkB,EAAE;QAChB,6BAA6B,EAAE,CAAC;QAChC,gBAAgB,EAAE,GAAG;KACxB;IACD,mBAAmB,EAAE;QACjB,2BAA2B,EAAE,GAAG;KACnC;CACJ,CAAC;AA8LF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkEG;AACH,MAAM,OAAO,WAIX,SAAQ,YAAsC;IAmCtB;IAlCZ,kBAAkB,CAAsC;IACxD,mBAAmB,CAAuE;IAC1F,wBAAwB,CAAU;IAClC,uBAAuB,CAAS;IAChC,eAAe,CAAU;IACzB,uBAAuB,CAAU;IACjC,qBAAqB,CAAU;IAC/B,8BAA8B,CAAc;IAC5C,0BAA0B,CAAc;IAC/B,kBAAkB,CAAc;IAEzC,MAAM,CAAU,YAAY,GAAG;QACrC,GAAG,YAAY,CAAC,YAAY;QAE5B,qBAAqB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QACzC,eAAe,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;QACpC,mBAAmB,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC;QACxD,uBAAuB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QAC3C,qBAAqB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QACzC,wBAAwB,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;QAE7C,8BAA8B,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC;QACnE,0BAA0B,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC;QAE/D,kBAAkB,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK;QACrC,mBAAmB,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK;KACzC,CAAC;IAEF;;OAEG;IACH,YACI,UACmE,EAAS,EAC1D,SAAS,aAAa,CAAC,eAAe,EAAE;QAE1D,EAAE,CAAC,OAAO,EAAE,oBAAoB,EAAE,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,WAAW,CAAC,YAAY,CAAC,CAAC,CAAC;QAElF,MAAM,EACF,qBAAqB,GAAG,EAAE,EAC1B,eAAe,GAAG,IAAI,EACtB,mBAAmB,GAAG,EAAE,EACxB,uBAAuB,EACvB,qBAAqB,EACrB,wBAAwB,EACxB,kBAAkB,GAAG,EAAE,EACvB,mBAAmB,GAAG,EAAE,EACxB,8BAA8B,GAAG,EAAE,EACnC,0BAA0B,GAAG,EAAE;QAE/B,eAAe;QACf,qBAAqB,GAAG,sCAAsC,EAC9D,sBAAsB,EACtB,GAAG,mBAAmB,EACzB,GAAG,OAAO,CAAC;QAEZ,KAAK,CACD;YACI,GAAG,mBAAmB;YACtB,qBAAqB;YACrB,sBAAsB,EAClB,sBAAsB;gBACtB,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,oBAAoB,EAA+C,CAAC;SACvF,EACD,MAAM,CACT,CAAC;QA/BgB,WAAM,GAAN,MAAM,CAAkC;QAiC1D,uEAAuE;QACvE,IAAI,CAAC,IAAI,CAAC,cAAc,IAAI,wBAAwB,EAAE,CAAC;YACnD,MAAM,IAAI,KAAK,CAAC,iFAAiF,CAAC,CAAC;QACvG,CAAC;QAED,IAAI,CAAC,kBAAkB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,uBAAuB,EAAE,0BAA0B,CAAC,CAAC,CAAC;QAC5F,IAAI,mBAAmB,CAAC,MAAM;YAAE,IAAI,CAAC,yBAAyB,CAAC,mBAAmB,CAAC,CAAC;QAEpF,IAAI,uBAAuB,IAAI,qBAAqB,EAAE,CAAC;YACnD,IAAI,CAAC,GAAG,CAAC,OAAO,CACZ,sGAAsG,CACzG,CAAC;QACN,CAAC;QAED,IAAI,CAAC,uBAAuB,GAAG,qBAAqB,GAAG,IAAI,CAAC;QAC5D,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;QACvC,IAAI,CAAC,uBAAuB,GAAG,uBAAuB,CAAC;QACvD,IAAI,CAAC,qBAAqB,GAAG,qBAAqB,CAAC;QACnD,IAAI,CAAC,8BAA8B,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,8BAA8B,CAAC,CAAC,CAAC;QACnF,IAAI,CAAC,0BAA0B,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,0BAA0B,CAAC,CAAC,CAAC;QAC3E,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C,IAAI,CAAC,mBAAmB,GAAG;YACvB,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,oBAAoB,CAAC,OAAO,EAAE,QAAS,CAAC;YACxE,GAAG,mBAAmB;SACzB,CAAC;QAEF,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACtB,IAAI,CAAC,wBAAwB,GAAG,wBAAwB,IAAI,IAAI,CAAC;QACrE,CAAC;aAAM,CAAC;YACJ,IAAI,CAAC,wBAAwB,GAAG,KAAK,CAAC;QAC1C,CAAC;IACL,CAAC;IAES,oBAAoB;QAC1B,OAAO,eAAe,CAAC,MAAM,EAAmB;aAC3C,OAAO,CAAC;YACL,MAAM,EAAE,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC;SAC1C,CAAC;aACD,OAAO,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;aACxD,OAAO,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,6BAA6B,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAC5E,CAAC;IAEO,KAAK,CAAC,eAAe,CACzB,eAAgC;QAEhC,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,eAAe,CAAC;QAE7C,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;YACzB,OAAO;gBACH,OAAO,EAAE,IAAI,KAAK,CAAC,OAAO,EAAE;oBACxB,GAAG,CAAC,MAAM,EAAE,YAAY,EAAE,QAAQ;wBAC9B,IAAI,YAAY,KAAK,WAAW,EAAE,CAAC;4BAC/B,MAAM,IAAI,KAAK,CACX,+EAA+E,CAClF,CAAC;wBACN,CAAC;wBACD,OAAO,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,YAAY,EAAE,QAAQ,CAAC,CAAC;oBACvD,CAAC;iBACJ,CAA2B;gBAC5B,IAAI,QAAQ;oBACR,MAAM,IAAI,KAAK,CAAC,sEAAsE,CAAC,CAAC;gBAC5F,CAAC;aACJ,CAAC;QACN,CAAC;QAED,MAAM,UAAU,GAAG,EAAiB,CAAC;QACrC,MAAM,yBAAyB,GAAG,IAAI,CAAC,2BAA2B,CAAC,OAAO,CAAC,CAAC;QAE5E,OAAO,CAAC,KAAK,GAAG,YAAY,CAAC,UAAU,CAAC;QACxC,qEAAqE;QACrE,8CAA8C;QAC9C,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,kBAAkB,EAAE,eAAe,EAAE,UAAU,CAAC,CAAC;QAC/E,SAAS,EAAE,CAAC;QAEZ,MAAM,0BAA0B,GAAG,IAAI,CAAC,2BAA2B,CAAC,OAAO,CAAC,CAAC;QAE7E,IAAI,CAAC,aAAa,CAAC,eAAe,EAAE,UAAU,EAAE,yBAAyB,EAAE,0BAA0B,CAAC,CAAC;QAEvG,MAAM,QAAQ,GAAG,eAAe,CAAC,SAAS,EAAE,GAAG,CAAC;QAEhD,MAAM,YAAY,GAAG,MAAM,mBAAmB,CAC1C,KAAK,IAAI,EAAE,CAAC,IAAI,CAAC,gBAAgB,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,CAAC,EAC7E,IAAI,CAAC,uBAAuB,EAC5B,2BAA2B,IAAI,CAAC,uBAAuB,GAAG,IAAI,WAAW,CAC5E,CAAC;QACF,SAAS,EAAE,CAAC;QAEZ,OAAO,CAAC,SAAS,GAAG,YAAY,CAAC,GAAG,CAAC;QACrC,OAAO,CAAC,KAAK,GAAG,YAAY,CAAC,SAAS,CAAC;QAEvC,OAAO,EAAE,OAAO,EAAE,OAAiC,EAAE,QAAQ,EAAE,YAAY,EAAE,CAAC;IAClF,CAAC;IAEO,KAAK,CAAC,mBAAmB,CAC7B,eAA2C;QAI3C,IAAI,eAAe,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC;YACzC,OAAO;gBACH,IAAI,WAAW;oBACX,MAAM,IAAI,KAAK,CAAC,yEAAyE,CAAC,CAAC;gBAC/F,CAAC;gBACD,IAAI,IAAI;oBACJ,MAAM,IAAI,KAAK,CAAC,kEAAkE,CAAC,CAAC;gBACxF,CAAC;gBACD,IAAI,IAAI;oBACJ,MAAM,IAAI,KAAK,CAAC,kEAAkE,CAAC,CAAC;gBACxF,CAAC;gBACD,IAAI,eAAe;oBACf,MAAM,IAAI,KAAK,CAAC,2EAA2E,CAAC,CAAC;gBACjG,CAAC;gBACD,IAAI,gBAAgB;oBAChB,MAAM,IAAI,KAAK,CAAC,4EAA4E,CAAC,CAAC;gBAClG,CAAC;aACJ,CAAC;QACN,CAAC;QAED,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,mBAAmB,EAAE,eAAe,CAAC,CAAC;QACpE,SAAS,EAAE,CAAC;QAEZ,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,eAAe,CAAC,OAAO,EAAE,eAAe,CAAC,QAAQ,CAAC,CAAC;QAC5F,SAAS,EAAE,CAAC;QACZ,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAS,CAAC;QAClC,MAAM,WAAW,GAAG,MAAM,CAAC,WAAY,CAAC;QAExC,MAAM,eAAe,GAAG,KAAK,EAAE,QAAgB,EAAE,UAAmB,EAAE,EAAE;YACpE,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,IAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;YAEhD,IAAI,CAAC,CAAC,QAAQ,CAAC,CAAC,GAAG,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACjC,MAAM,IAAI,KAAK,CAAC,aAAa,QAAQ,cAAc,CAAC,CAAC;YACzD,CAAC;QACL,CAAC,CAAC;QACF,MAAM,gBAAgB,GAAG,KAAK,EAAE,QAAiB,EAAE,SAAkB,EAAE,EAAE;YACrE,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,IAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;YAEhD,IAAI,QAAQ,EAAE,CAAC;gBACX,MAAO,eAA+C,CAAC,eAAe,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;YAChG,CAAC;YAED,OAAO,CAAC,CAAC;QACb,CAAC,CAAC;QAEF,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACtB,IAAI,CAAC,sBAAsB,CAAC,eAAe,CAAC,OAAQ,EAAE,QAAQ,CAAC,UAAW,CAAC,CAAC;QAChF,CAAC;QAED,IAAI,IAAI,CAAC,wBAAwB,EAAE,CAAC;YAChC,eAAe,CAAC,OAAQ,CAAC,sBAAsB,CAAC,QAAQ,CAAC,CAAC;QAC9D,CAAC;QAED,OAAO;YACH,IAAI,IAAI;gBACJ,IAAI,WAAW,CAAC,IAAI,KAAK,0BAA0B;oBAAE,OAAO,IAAI,CAAC;gBACjE,MAAM,UAAU,GAAG,MAAM,CAAC,IAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;gBAC/D,OAAO,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;YAClC,CAAC;YACD,eAAe;YACf,gBAAgB;YAChB,WAAW;YACX,IAAI,EAAE,MAAM,CAAC,IAAK;SACrB,CAAC;IACN,CAAC;IAEO,KAAK,CAAC,6BAA6B,CAAC,eAA4C;QACpF,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACtB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,eAAe,CAAC,CAAC;YAC3D,IAAI,KAAK;gBAAE,MAAM,IAAI,YAAY,CAAC,KAAK,CAAC,CAAC;QAC7C,CAAC;QACD,OAAO,EAAE,CAAC;IACd,CAAC;IAES,KAAK,CAAC,gBAAgB,CAAC,eAA4C;QACzE,IAAI,uBAAuB,CAAC,QAAQ,CAAC,eAAe,CAAC,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC;YACrE,MAAM,CAAC,GAAG,MAAM,eAAe,CAAC,gBAAgB,EAAE,CAAC;YAEnD,MAAM,cAAc,GAAG,mBAAmB,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YAExF,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC5B,OAAO,oBAAoB,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAC3D,CAAC;QACL,CAAC;QACD,OAAO,KAAK,CAAC;IACjB,CAAC;IAED;;OAEG;IACO,aAAa,CACnB,EAAE,OAAO,EAAE,OAAO,EAAmB,EACrC,UAAuB,EACvB,cAAsB,EACtB,eAAuB;QAEvB,MAAM,aAAa,GAAG,OAAO,EAAE,eAAe,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;QAClE,IAAI,wBAAwB,GAAG,UAAU,CAAC,OAAO,EAAE,MAAM,IAAI,UAAU,CAAC,OAAO,EAAE,MAAM,IAAI,EAAE,CAAC;QAE9F,IAAI,UAAU,CAAC,OAAO,EAAE,MAAM,IAAI,UAAU,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;YAC3D,MAAM,EAAE,MAAM,EAAE,eAAe,EAAE,MAAM,EAAE,eAAe,EAAE,GAAG,UAAU,CAAC,OAAO,CAAC;YAEhF,IAAI,CAAC,GAAG,CAAC,OAAO,CACZ,kFAAkF,OAAO,CAAC,GAAG,KAAK,OAAO,CAAC,EAAE,gCAAgC,CAC/I,CAAC;YAEF,MAAM,aAAa,GAAG,EAAE,CAAC;YAEzB,IAAI,KAAK,CAAC,OAAO,CAAC,eAAe,CAAC,EAAE,CAAC;gBACjC,aAAa,CAAC,IAAI,CAAC,GAAG,eAAe,CAAC,CAAC;YAC3C,CAAC;iBAAM,CAAC;gBACJ,aAAa,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;YACxC,CAAC;YAED,IAAI,KAAK,CAAC,OAAO,CAAC,eAAe,CAAC,EAAE,CAAC;gBACjC,aAAa,CAAC,IAAI,CAAC,GAAG,eAAe,CAAC,CAAC;YAC3C,CAAC;iBAAM,CAAC;gBACJ,aAAa,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;YACxC,CAAC;YAED,wBAAwB,GAAG,YAAY,CAAC,OAAO,CAAC,GAAG,EAAE,aAAa,CAAC,CAAC;QACxE,CAAC;QAED,MAAM,aAAa,GAAG,CAAC,aAAa,EAAE,cAAc,CAAC,CAAC;QAEtD,IAAI,KAAK,CAAC,OAAO,CAAC,wBAAwB,CAAC,EAAE,CAAC;YAC1C,aAAa,CAAC,IAAI,CAAC,GAAG,wBAAwB,CAAC,CAAC;QACpD,CAAC;aAAM,CAAC;YACJ,aAAa,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;QACjD,CAAC;QAED,aAAa,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QAEpC,MAAM,YAAY,GAAG,YAAY,CAAC,OAAO,CAAC,GAAG,EAAE,aAAa,CAAC,CAAC;QAE9D,UAAU,CAAC,OAAO,KAAK,EAAE,CAAC;QAC1B,OAAO,CAAC,cAAc,CAAC,UAAU,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;QACrD,OAAO,CAAC,cAAc,CAAC,UAAU,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;QAErD,IAAI,YAAY,KAAK,EAAE,EAAE,CAAC;YACtB,UAAU,CAAC,OAAO,CAAC,MAAM,GAAG,YAAY,CAAC;QAC7C,CAAC;IACL,CAAC;IAED;;;;OAIG;IACO,KAAK,CAAC,gBAAgB,CAAC,EAC7B,OAAO,EACP,OAAO,EACP,QAAQ,EACR,UAAU,GACW;QACrB,IAAI,CAAC,YAAY,EAAE,CAAC;YAChB,aAAa;YACb,CAAC,EAAE,YAAY,EAAE,GAAG,MAAM,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC;QACtD,CAAC;QAED,MAAM,IAAI,GAAG,IAAI,CAAC,kBAAkB,CAAC,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,UAAU,CAAC,CAAC;QAE7E,IAAI,CAAC;YACD,OAAO,MAAM,IAAI,CAAC,iBAAiB,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QACvD,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,IAAI,CAAC,YAAY,YAAY,EAAE,CAAC;gBAC5B,IAAI,CAAC,qBAAqB,CAAC,OAAO,CAAC,CAAC;gBACpC,OAAO,SAAqC,CAAC;YACjD,CAAC;YAED,IAAI,IAAI,CAAC,YAAY,CAAC,CAAU,CAAC,EAAE,CAAC;gBAChC,MAAM,IAAI,YAAY,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAU,CAAW,CAAC,CAAC;YAC5E,CAAC;iBAAM,CAAC;gBACJ,MAAM,CAAC,CAAC;YACZ,CAAC;QACL,CAAC;IACL,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,cAAc,CAAC,OAAgB,EAAE,cAA+B;QAC1E,MAAM,EAAE,UAAU,EAAE,GAAG,cAAc,CAAC;QACtC,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,4BAA4B,CAAC,cAAc,CAAC,CAAC;QACvE,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,cAAc,EAAE,OAAO,CAAC,CAAC;QACtF,MAAM,WAAW,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;QAEvC,IAAI,UAAW,IAAI,GAAG,IAAI,UAAW,IAAI,GAAG,EAAE,CAAC;YAC3C,IAAI,CAAC,KAAK,CAAC,kBAAkB,CAAC,UAAW,CAAC,CAAC;QAC/C,CAAC;QAED,MAAM,YAAY,GAAG,IAAI,CAAC,0BAA0B,CAAC,GAAG,CAAC,UAAW,CAAC,CAAC;QACtE,MAAM,YAAY,GAAG,IAAI,CAAC,8BAA8B,CAAC,GAAG,CAAC,UAAW,CAAC,CAAC;QAE1E,IAAI,CAAC,UAAW,IAAI,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,YAAY,EAAE,CAAC;YACxD,MAAM,IAAI,GAAG,MAAM,kBAAkB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;YAE1D,2DAA2D;YAC3D,gDAAgD;YAChD,IAAI,IAAI,KAAK,0BAA0B,EAAE,CAAC;gBACtC,MAAM,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBACvC,IAAI,EAAE,OAAO,EAAE,GAAG,aAAa,CAAC;gBAChC,IAAI,CAAC,OAAO;oBAAE,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,cAAc,EAAE,EAAE,EAAE,CAAC,CAAC;gBACtF,MAAM,IAAI,KAAK,CAAC,GAAG,UAAU,MAAM,OAAO,EAAE,CAAC,CAAC;YAClD,CAAC;YAED,IAAI,YAAY,EAAE,CAAC;gBACf,MAAM,IAAI,KAAK,CAAC,GAAG,UAAU,uCAAuC,CAAC,CAAC;YAC1E,CAAC;YAED,8EAA8E;YAC9E,MAAM,IAAI,KAAK,CAAC,GAAG,UAAU,6BAA6B,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;QACpF,CAAC;aAAM,IAAI,uBAAuB,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YAChD,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC/E,CAAC;aAAM,CAAC;YACJ,MAAM,IAAI,GAAG,MAAM,oBAAoB,CAAC,QAAQ,CAAC,CAAC;YAClD,OAAO;gBACH,IAAI;gBACJ,QAAQ;gBACR,WAAW;aACd,CAAC;QACN,CAAC;IACL,CAAC;IAED;;OAEG;IACO,kBAAkB,CAAC,OAAgB,EAAE,OAAiB,EAAE,QAAiB,EAAE,UAAwB;QACzG,MAAM,cAAc,GAA0E;YAC1F,GAAG,EAAE,OAAO,CAAC,GAAG;YAChB,MAAM,EAAE,OAAO,CAAC,MAAgB;YAChC,QAAQ;YACR,OAAO,EAAE,EAAE,OAAO,EAAE,IAAI,CAAC,uBAAuB,EAAE;YAClD,YAAY,EAAE,OAAO;YACrB,GAAG,UAAU;YACb,OAAO,EAAE,EAAE,GAAG,OAAO,CAAC,OAAO,EAAE,GAAG,UAAU,EAAE,OAAO,EAAE;YACvD,KAAK,EAAE;gBACH,GAAG,UAAU,EAAE,KAAK;gBACpB,kBAAkB,EAAE,CAAC,IAAI,CAAC,eAAe;aAC5C;YACD,QAAQ,EAAE,IAAI;SACjB,CAAC;QAEF,yHAAyH;QACzH,OAAO,CAAC,cAAc,CAAC,cAAc,CAAC,OAAQ,EAAE,QAAQ,CAAC,CAAC;QAE1D,4CAA4C;QAC5C,IAAI,OAAO,EAAE,SAAS,EAAE,eAAe,EAAE,CAAC;YACtC,cAAc,CAAC,KAAK,GAAG;gBACnB,GAAG,cAAc,CAAC,KAAK;gBACvB,kBAAkB,EAAE,KAAK;aAC5B,CAAC;QACN,CAAC;QAED,IAAI,gBAAgB,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;YAAE,cAAc,CAAC,IAAI,GAAG,OAAO,CAAC,OAAO,IAAI,EAAE,CAAC;QAEvF,OAAO,cAAc,CAAC;IAC1B,CAAC;IAES,eAAe,CACrB,OAAgB,EAChB,QAAyB,EACzB,QAAwB;QAKxB,IAAI,IAAI,CAAC,qBAAqB,EAAE,CAAC;YAC7B,QAAQ,GAAG,IAAI,CAAC,qBAAuC,CAAC;QAC5D,CAAC;aAAM,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,uBAAuB,EAAE,CAAC;YACnD,QAAQ,GAAG,IAAI,CAAC,uBAAyC,CAAC;QAC9D,CAAC;QAED,sDAAsD;QACtD,MAAM,IAAI,GAAG,MAAM,CAAC;QACpB,IAAI,CAAC,QAAQ;YAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;QAEnD,2DAA2D;QAC3D,+CAA+C;QAC/C,IAAI,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC;YAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC;QAE/D,+DAA+D;QAC/D,IAAI,KAAK,CAAC,cAAc,CAAC,QAAQ,CAAC,EAAE,CAAC;YACjC,MAAM,YAAY,GAAG,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;YAC9C,MAAM,YAAY,GAAG,KAAK,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;YACxG,QAAQ,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAU,EAAE,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;YACtE,MAAM,eAAe,GAAG,QAAQ,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,YAAY,CAIpE,CAAC;YACF,eAAe,CAAC,UAAU,GAAG,QAAQ,CAAC,UAAU,CAAC;YACjD,eAAe,CAAC,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAC;YAC3C,eAAe,CAAC,GAAG,GAAG,QAAQ,CAAC,GAAG,CAAC;YACnC,OAAO;gBACH,QAAQ,EAAE,eAAsB;gBAChC,QAAQ,EAAE,IAAI;aACjB,CAAC;QACN,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,YAAY,OAAO,CAAC,GAAG,8CAA8C,QAAQ,EAAE,CAAC,CAAC;IACrG,CAAC;IAED;;OAEG;IACO,yBAAyB,CAAC,mBAA4D;QAC5F,KAAK,MAAM,QAAQ,IAAI,mBAAmB,EAAE,CAAC;YACzC,IAAI,QAAQ,KAAK,KAAK,EAAE,CAAC;gBACrB,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;gBACtC,SAAS;YACb,CAAC;YAED,IAAI,CAAC;gBACD,MAAM,UAAU,GAAG,iBAAiB,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;gBACrD,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;YACjD,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,MAAM,IAAI,KAAK,CAAC,2BAA2B,QAAQ,sCAAsC,CAAC,CAAC;YAC/F,CAAC;QACL,CAAC;IACL,CAAC;IAED;;OAEG;IACO,qBAAqB,CAAC,OAAiB;QAC7C,OAAO,EAAE,OAAO,EAAE,CAAC;QACnB,MAAM,IAAI,KAAK,CAAC,2BAA2B,IAAI,CAAC,uBAAuB,GAAG,IAAI,WAAW,CAAC,CAAC;IAC/F,CAAC;IAEO,oBAAoB,CAAC,OAAgB,EAAE,QAAyB;QACpE,MAAM,EAAE,UAAU,EAAE,GAAG,QAAQ,CAAC;QAChC,MAAM,EAAE,IAAI,EAAE,GAAG,4BAA4B,CAAC,QAAQ,CAAC,CAAC;QAExD,sEAAsE;QACtE,MAAM,kBAAkB,GAAG,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC1F,wDAAwD;QACxD,MAAM,sBAAsB,GAAG,UAAW,IAAI,GAAG,IAAI,kBAAkB,CAAC,QAAQ,CAAC,UAAW,CAAC,CAAC;QAE9F,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,sBAAsB,EAAE,CAAC;YACvG,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC;YACvB,MAAM,IAAI,KAAK,CACX,YAAY,OAAO,CAAC,GAAG,wBAAwB,IAAI,IAAI;gBACnD,YAAY,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,kCAAkC,CACnG,CAAC;QACN,CAAC;IACL,CAAC;IAED;;OAEG;IACK,iBAAiB,GAAG,KAAK,EAC7B,OAA4D,EAC5D,OAAiB,EACnB,EAAE;QACA,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,MAAM,CACzC,yBAAyB,CAAC;YACtB,GAAG,OAAO;YACV,SAAS,EAAE,OAAO,CAAC,SAAgB,EAAE,oDAAoD;YACzF,YAAY,EAAE,MAAM;SACvB,CAAC,EACF,CAAC,gBAAgB,EAAE,cAAc,EAAE,EAAE;YACjC,IAAI,IAAI,CAAC,wBAAwB,EAAE,CAAC;gBAChC,OAAQ,CAAC,sBAAsB,CAAC,gBAAgB,CAAC,CAAC;gBAElD,MAAM,YAAY,GAAG,OAAQ,CAAC,eAAe,CAAC,cAAc,CAAC,GAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;gBAC9E,IAAI,YAAY,KAAK,EAAE,EAAE,CAAC;oBACtB,cAAc,CAAC,OAAO,CAAC,MAAM,GAAG,YAAY,CAAC;gBACjD,CAAC;YACL,CAAC;QACL,CAAC,CACJ,CAAC;QAEF,OAAO,6BAA6B,CAAC,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;IACpE,CAAC,CAAC;;AAUN;;;;;;;GAOG;AACH,SAAS,6BAA6B,CAAC,MAAgB,EAAE,QAA+B;IACpF,MAAM,UAAU,GAA4B;QACxC,YAAY;QACZ,eAAe;QACf,SAAS;QACT,UAAU;QACV,aAAa;QACb,YAAY;QACZ,aAAa;QACb,UAAU;QACV,KAAK;QACL,SAAS;KACZ,CAAC;IAEF,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;QAClB,mBAAmB;QACnB,IAAI,MAAM,CAAC,WAAW;YAAE,MAAM,CAAC,WAAW,GAAG,QAAQ,CAAC,WAAW,CAAC,CAAC,mCAAmC;QAEtG,mBAAmB;QACnB,IAAI,MAAM,CAAC,QAAQ;YAAE,MAAM,CAAC,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC;QAEzD,mBAAmB;QACnB,MAAM,CAAC,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC;IACxC,CAAC,CAAC,CAAC;IAEH,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;QAC5B,IAAI,CAAC,CAAC,IAAI,IAAI,MAAM,CAAC,EAAE,CAAC;YACnB,MAAc,CAAC,IAAI,CAAC,GAAI,QAAgB,CAAC,IAAI,CAAC,CAAC;QACpD,CAAC;IACL,CAAC;IAED,OAAO,MAAkC,CAAC;AAC9C,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,MAAM,UAAU,gBAAgB,CAG9B,MAAwC;IACtC,OAAO,MAAM,CAAC,MAAM,CAAU,MAAM,CAAC,CAAC;AAC1C,CAAC"}
|
|
1
|
+
{"version":3,"file":"http-crawler.js","sourceRoot":"","sources":["../../src/internals/http-crawler.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AACvC,OAAO,IAAI,MAAM,WAAW,CAAC;AAc7B,OAAO,EACH,YAAY,EACZ,oBAAoB,EACpB,aAAa,EACb,eAAe,EACf,YAAY,EACZ,yBAAyB,EACzB,YAAY,EACZ,eAAe,EACf,MAAM,EACN,YAAY,GACf,MAAM,gBAAgB,CAAC;AAGxB,OAAO,EAAoB,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AACvE,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAEnC,OAAO,iBAAiB,MAAM,cAAc,CAAC;AAE7C,OAAO,KAAK,MAAM,YAAY,CAAC;AAC/B,OAAO,EAAE,MAAM,IAAI,CAAC;AAGpB,OAAO,EAAE,mBAAmB,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAEhE,OAAO,EAAE,4BAA4B,EAAE,MAAM,YAAY,CAAC;AAE1D,IAAI,YAAsC,CAAC;AAW3C;;GAEG;AACH,MAAM,uBAAuB,GAAG,CAAC,WAAW,EAAE,UAAU,EAAE,uBAAuB,EAAE,iBAAiB,CAAC,CAAC;AACtG,MAAM,0BAA0B,GAAG,kBAAkB,CAAC;AACtD,MAAM,sCAAsC,GAA0B;IAClE,kBAAkB,EAAE,EAAE;IACtB,kBAAkB,EAAE;QAChB,6BAA6B,EAAE,CAAC;QAChC,gBAAgB,EAAE,GAAG;KACxB;IACD,mBAAmB,EAAE;QACjB,2BAA2B,EAAE,GAAG;KACnC;CACJ,CAAC;AA8LF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkEG;AACH,MAAM,OAAO,WAIX,SAAQ,YAAsC;IAmCtB;IAlCZ,kBAAkB,CAAsC;IACxD,mBAAmB,CAAuE;IAC1F,wBAAwB,CAAU;IAClC,uBAAuB,CAAS;IAChC,eAAe,CAAU;IACzB,uBAAuB,CAAU;IACjC,qBAAqB,CAAU;IAC/B,8BAA8B,CAAc;IAC5C,0BAA0B,CAAc;IAC/B,kBAAkB,CAAc;IAEzC,MAAM,CAAU,YAAY,GAAG;QACrC,GAAG,YAAY,CAAC,YAAY;QAE5B,qBAAqB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QACzC,eAAe,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;QACpC,mBAAmB,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC;QACxD,uBAAuB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QAC3C,qBAAqB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;QACzC,wBAAwB,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;QAE7C,8BAA8B,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC;QACnE,0BAA0B,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC;QAE/D,kBAAkB,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK;QACrC,mBAAmB,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK;KACzC,CAAC;IAEF;;OAEG;IACH,YACI,UACmE,EAAS,EAC1D,SAAS,aAAa,CAAC,eAAe,EAAE;QAE1D,EAAE,CAAC,OAAO,EAAE,oBAAoB,EAAE,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,WAAW,CAAC,YAAY,CAAC,CAAC,CAAC;QAElF,MAAM,EACF,qBAAqB,GAAG,EAAE,EAC1B,eAAe,GAAG,IAAI,EACtB,mBAAmB,GAAG,EAAE,EACxB,uBAAuB,EACvB,qBAAqB,EACrB,wBAAwB,EACxB,kBAAkB,GAAG,EAAE,EACvB,mBAAmB,GAAG,EAAE,EACxB,8BAA8B,GAAG,EAAE,EACnC,0BAA0B,GAAG,EAAE;QAE/B,eAAe;QACf,qBAAqB,GAAG,sCAAsC,EAC9D,sBAAsB,EACtB,GAAG,mBAAmB,EACzB,GAAG,OAAO,CAAC;QAEZ,KAAK,CACD;YACI,GAAG,mBAAmB;YACtB,qBAAqB;YACrB,sBAAsB,EAClB,sBAAsB;gBACtB,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,oBAAoB,EAA+C,CAAC;SACvF,EACD,MAAM,CACT,CAAC;QA/BgB,WAAM,GAAN,MAAM,CAAkC;QAiC1D,uEAAuE;QACvE,IAAI,CAAC,IAAI,CAAC,cAAc,IAAI,wBAAwB,EAAE,CAAC;YACnD,MAAM,IAAI,KAAK,CAAC,iFAAiF,CAAC,CAAC;QACvG,CAAC;QAED,IAAI,CAAC,kBAAkB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,uBAAuB,EAAE,0BAA0B,CAAC,CAAC,CAAC;QAC5F,IAAI,mBAAmB,CAAC,MAAM;YAAE,IAAI,CAAC,yBAAyB,CAAC,mBAAmB,CAAC,CAAC;QAEpF,IAAI,uBAAuB,IAAI,qBAAqB,EAAE,CAAC;YACnD,IAAI,CAAC,GAAG,CAAC,OAAO,CACZ,sGAAsG,CACzG,CAAC;QACN,CAAC;QAED,IAAI,CAAC,uBAAuB,GAAG,qBAAqB,GAAG,IAAI,CAAC;QAC5D,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;QACvC,IAAI,CAAC,uBAAuB,GAAG,uBAAuB,CAAC;QACvD,IAAI,CAAC,qBAAqB,GAAG,qBAAqB,CAAC;QACnD,IAAI,CAAC,8BAA8B,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,8BAA8B,CAAC,CAAC,CAAC;QACnF,IAAI,CAAC,0BAA0B,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,0BAA0B,CAAC,CAAC,CAAC;QAC3E,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C,IAAI,CAAC,mBAAmB,GAAG;YACvB,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,oBAAoB,CAAC,OAAO,EAAE,QAAS,CAAC;YACxE,GAAG,mBAAmB;SACzB,CAAC;QAEF,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACtB,IAAI,CAAC,wBAAwB,GAAG,wBAAwB,IAAI,IAAI,CAAC;QACrE,CAAC;aAAM,CAAC;YACJ,IAAI,CAAC,wBAAwB,GAAG,KAAK,CAAC;QAC1C,CAAC;IACL,CAAC;IAES,oBAAoB;QAC1B,OAAO,eAAe,CAAC,MAAM,EAAmB;aAC3C,OAAO,CAAC;YACL,MAAM,EAAE,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC;SAC1C,CAAC;aACD,OAAO,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;aACxD,OAAO,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,6BAA6B,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAC5E,CAAC;IAEO,KAAK,CAAC,eAAe,CACzB,eAAgC;QAEhC,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,eAAe,CAAC;QAE7C,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;YACzB,OAAO;gBACH,OAAO,EAAE,IAAI,KAAK,CAAC,OAAO,EAAE;oBACxB,GAAG,CAAC,MAAM,EAAE,YAAY,EAAE,QAAQ;wBAC9B,IAAI,YAAY,KAAK,WAAW,EAAE,CAAC;4BAC/B,MAAM,IAAI,KAAK,CACX,+EAA+E,CAClF,CAAC;wBACN,CAAC;wBACD,OAAO,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,YAAY,EAAE,QAAQ,CAAC,CAAC;oBACvD,CAAC;iBACJ,CAA2B;gBAC5B,IAAI,QAAQ;oBACR,MAAM,IAAI,KAAK,CAAC,sEAAsE,CAAC,CAAC;gBAC5F,CAAC;aACJ,CAAC;QACN,CAAC;QAED,MAAM,UAAU,GAAG,EAAiB,CAAC;QACrC,MAAM,yBAAyB,GAAG,IAAI,CAAC,2BAA2B,CAAC,OAAO,CAAC,CAAC;QAE5E,OAAO,CAAC,KAAK,GAAG,YAAY,CAAC,UAAU,CAAC;QACxC,qEAAqE;QACrE,8CAA8C;QAC9C,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,kBAAkB,EAAE,eAAe,EAAE,UAAU,CAAC,CAAC;QAC/E,SAAS,EAAE,CAAC;QAEZ,MAAM,0BAA0B,GAAG,IAAI,CAAC,2BAA2B,CAAC,OAAO,CAAC,CAAC;QAE7E,IAAI,CAAC,aAAa,CAAC,eAAe,EAAE,UAAU,EAAE,yBAAyB,EAAE,0BAA0B,CAAC,CAAC;QAEvG,MAAM,QAAQ,GAAG,eAAe,CAAC,SAAS,EAAE,GAAG,CAAC;QAEhD,MAAM,YAAY,GAAG,MAAM,mBAAmB,CAC1C,KAAK,IAAI,EAAE,CAAC,IAAI,CAAC,gBAAgB,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,CAAC,EAC7E,IAAI,CAAC,uBAAuB,EAC5B,2BAA2B,IAAI,CAAC,uBAAuB,GAAG,IAAI,WAAW,CAC5E,CAAC;QACF,SAAS,EAAE,CAAC;QAEZ,OAAO,CAAC,SAAS,GAAG,YAAY,EAAE,GAAG,CAAC;QACtC,OAAO,CAAC,KAAK,GAAG,YAAY,CAAC,SAAS,CAAC;QAEvC,OAAO,EAAE,OAAO,EAAE,OAAiC,EAAE,QAAQ,EAAE,YAAY,EAAE,CAAC;IAClF,CAAC;IAEO,KAAK,CAAC,mBAAmB,CAC7B,eAA2C;QAI3C,IAAI,eAAe,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC;YACzC,OAAO;gBACH,IAAI,WAAW;oBACX,MAAM,IAAI,KAAK,CAAC,yEAAyE,CAAC,CAAC;gBAC/F,CAAC;gBACD,IAAI,IAAI;oBACJ,MAAM,IAAI,KAAK,CAAC,kEAAkE,CAAC,CAAC;gBACxF,CAAC;gBACD,IAAI,IAAI;oBACJ,MAAM,IAAI,KAAK,CAAC,kEAAkE,CAAC,CAAC;gBACxF,CAAC;gBACD,IAAI,eAAe;oBACf,MAAM,IAAI,KAAK,CAAC,2EAA2E,CAAC,CAAC;gBACjG,CAAC;gBACD,IAAI,gBAAgB;oBAChB,MAAM,IAAI,KAAK,CAAC,4EAA4E,CAAC,CAAC;gBAClG,CAAC;aACJ,CAAC;QACN,CAAC;QAED,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,mBAAmB,EAAE,eAAe,CAAC,CAAC;QACpE,SAAS,EAAE,CAAC;QAEZ,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,eAAe,CAAC,OAAO,EAAE,eAAe,CAAC,QAAQ,CAAC,CAAC;QAC5F,SAAS,EAAE,CAAC;QACZ,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAS,CAAC;QAClC,MAAM,WAAW,GAAG,MAAM,CAAC,WAAY,CAAC;QAExC,MAAM,eAAe,GAAG,KAAK,EAAE,QAAgB,EAAE,UAAmB,EAAE,EAAE;YACpE,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,IAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;YAEhD,IAAI,CAAC,CAAC,QAAQ,CAAC,CAAC,GAAG,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACjC,MAAM,IAAI,KAAK,CAAC,aAAa,QAAQ,cAAc,CAAC,CAAC;YACzD,CAAC;QACL,CAAC,CAAC;QACF,MAAM,gBAAgB,GAAG,KAAK,EAAE,QAAiB,EAAE,SAAkB,EAAE,EAAE;YACrE,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,IAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;YAEhD,IAAI,QAAQ,EAAE,CAAC;gBACX,MAAO,eAA+C,CAAC,eAAe,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;YAChG,CAAC;YAED,OAAO,CAAC,CAAC;QACb,CAAC,CAAC;QAEF,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACtB,IAAI,CAAC,sBAAsB,CAAC,eAAe,CAAC,OAAQ,EAAE,QAAQ,CAAC,MAAO,CAAC,CAAC;QAC5E,CAAC;QAED,IAAI,IAAI,CAAC,wBAAwB,EAAE,CAAC;YAChC,eAAe,CAAC,OAAQ,CAAC,sBAAsB,CAAC,QAAQ,CAAC,CAAC;QAC9D,CAAC;QAED,OAAO;YACH,IAAI,IAAI;gBACJ,IAAI,WAAW,CAAC,IAAI,KAAK,0BAA0B;oBAAE,OAAO,IAAI,CAAC;gBACjE,MAAM,UAAU,GAAG,MAAM,CAAC,IAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;gBAC/D,OAAO,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;YAClC,CAAC;YACD,eAAe;YACf,gBAAgB;YAChB,WAAW;YACX,IAAI,EAAE,MAAM,CAAC,IAAI;SACpB,CAAC;IACN,CAAC;IAEO,KAAK,CAAC,6BAA6B,CAAC,eAA4C;QACpF,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACtB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,eAAe,CAAC,CAAC;YAC3D,IAAI,KAAK;gBAAE,MAAM,IAAI,YAAY,CAAC,KAAK,CAAC,CAAC;QAC7C,CAAC;QACD,OAAO,EAAE,CAAC;IACd,CAAC;IAES,KAAK,CAAC,gBAAgB,CAAC,eAA4C;QACzE,IAAI,uBAAuB,CAAC,QAAQ,CAAC,eAAe,CAAC,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC;YACrE,MAAM,CAAC,GAAG,MAAM,eAAe,CAAC,gBAAgB,EAAE,CAAC;YAEnD,MAAM,cAAc,GAAG,mBAAmB,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YAExF,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC5B,OAAO,oBAAoB,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAC3D,CAAC;QACL,CAAC;QAED,MAAM,kBAAkB;QACpB,wCAAwC;QACxC,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,oBAAoB,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,GAAG,CAAC;YACtD,CAAC,CAAC,wCAAwC;gBACxC,IAAI,CAAC,WAAY,CAAC,oBAAoB,CAAC;YACzC,CAAC,CAAC,oBAAoB,CAAC;QAE/B,IAAI,kBAAkB,CAAC,QAAQ,CAAC,eAAe,CAAC,QAAQ,CAAC,MAAO,CAAC,EAAE,CAAC;YAChE,OAAO,0BAA0B,eAAe,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;QACvE,CAAC;QAED,OAAO,KAAK,CAAC;IACjB,CAAC;IAED;;OAEG;IACO,aAAa,CACnB,EAAE,OAAO,EAAE,OAAO,EAAmB,EACrC,UAAuB,EACvB,cAAsB,EACtB,eAAuB;QAEvB,MAAM,aAAa,GAAG,OAAO,EAAE,eAAe,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;QAClE,IAAI,wBAAwB,GAAG,UAAU,CAAC,OAAO,EAAE,MAAM,IAAI,UAAU,CAAC,OAAO,EAAE,MAAM,IAAI,EAAE,CAAC;QAE9F,IAAI,UAAU,CAAC,OAAO,EAAE,MAAM,IAAI,UAAU,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;YAC3D,MAAM,EAAE,MAAM,EAAE,eAAe,EAAE,MAAM,EAAE,eAAe,EAAE,GAAG,UAAU,CAAC,OAAO,CAAC;YAEhF,IAAI,CAAC,GAAG,CAAC,OAAO,CACZ,kFAAkF,OAAO,CAAC,GAAG,KAAK,OAAO,CAAC,EAAE,gCAAgC,CAC/I,CAAC;YAEF,MAAM,aAAa,GAAG,EAAE,CAAC;YAEzB,IAAI,KAAK,CAAC,OAAO,CAAC,eAAe,CAAC,EAAE,CAAC;gBACjC,aAAa,CAAC,IAAI,CAAC,GAAG,eAAe,CAAC,CAAC;YAC3C,CAAC;iBAAM,CAAC;gBACJ,aAAa,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;YACxC,CAAC;YAED,IAAI,KAAK,CAAC,OAAO,CAAC,eAAe,CAAC,EAAE,CAAC;gBACjC,aAAa,CAAC,IAAI,CAAC,GAAG,eAAe,CAAC,CAAC;YAC3C,CAAC;iBAAM,CAAC;gBACJ,aAAa,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;YACxC,CAAC;YAED,wBAAwB,GAAG,YAAY,CAAC,OAAO,CAAC,GAAG,EAAE,aAAa,CAAC,CAAC;QACxE,CAAC;QAED,MAAM,aAAa,GAAG,CAAC,aAAa,EAAE,cAAc,CAAC,CAAC;QAEtD,IAAI,KAAK,CAAC,OAAO,CAAC,wBAAwB,CAAC,EAAE,CAAC;YAC1C,aAAa,CAAC,IAAI,CAAC,GAAG,wBAAwB,CAAC,CAAC;QACpD,CAAC;aAAM,CAAC;YACJ,aAAa,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;QACjD,CAAC;QAED,aAAa,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QAEpC,MAAM,YAAY,GAAG,YAAY,CAAC,OAAO,CAAC,GAAG,EAAE,aAAa,CAAC,CAAC;QAE9D,UAAU,CAAC,OAAO,KAAK,EAAE,CAAC;QAC1B,OAAO,CAAC,cAAc,CAAC,UAAU,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;QACrD,OAAO,CAAC,cAAc,CAAC,UAAU,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;QAErD,IAAI,YAAY,KAAK,EAAE,EAAE,CAAC;YACtB,UAAU,CAAC,OAAO,CAAC,MAAM,GAAG,YAAY,CAAC;QAC7C,CAAC;IACL,CAAC;IAED;;;;OAIG;IACO,KAAK,CAAC,gBAAgB,CAAC,EAC7B,OAAO,EACP,OAAO,EACP,QAAQ,EACR,UAAU,GACW;QACrB,IAAI,CAAC,YAAY,EAAE,CAAC;YAChB,aAAa;YACb,CAAC,EAAE,YAAY,EAAE,GAAG,MAAM,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC;QACtD,CAAC;QAED,MAAM,IAAI,GAAG,IAAI,CAAC,kBAAkB,CAAC,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,UAAU,CAAC,CAAC;QAE7E,IAAI,CAAC;YACD,OAAO,MAAM,IAAI,CAAC,iBAAiB,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QACvD,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,IAAI,CAAC,YAAY,YAAY,EAAE,CAAC;gBAC5B,IAAI,CAAC,qBAAqB,CAAC,OAAO,CAAC,CAAC;gBACpC,OAAO,IAAI,QAAQ,EAAE,CAAC,CAAC,iEAAiE;YAC5F,CAAC;YAED,IAAI,IAAI,CAAC,YAAY,CAAC,CAAU,CAAC,EAAE,CAAC;gBAChC,MAAM,IAAI,YAAY,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAU,CAAW,CAAC,CAAC;YAC5E,CAAC;iBAAM,CAAC;gBACJ,MAAM,CAAC,CAAC;YACZ,CAAC;QACL,CAAC;IACL,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,cAAc,CAAC,OAAgB,EAAE,QAAkB;QAC/D,MAAM,EAAE,MAAM,EAAE,GAAG,QAAQ,CAAC;QAC5B,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,4BAA4B,CAAC,QAAQ,CAAC,CAAC;QACjE,MAAM,EAAE,QAAQ,EAAE,iBAAiB,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;QACnG,MAAM,WAAW,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;QAEvC,IAAI,MAAM,IAAI,GAAG,IAAI,MAAM,IAAI,GAAG,EAAE,CAAC;YACjC,IAAI,CAAC,KAAK,CAAC,kBAAkB,CAAC,MAAM,CAAC,CAAC;QAC1C,CAAC;QAED,MAAM,YAAY,GAAG,IAAI,CAAC,0BAA0B,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACjE,MAAM,YAAY,GAAG,IAAI,CAAC,8BAA8B,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAErE,IAAI,CAAC,MAAM,IAAI,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,YAAY,EAAE,CAAC;YACnD,MAAM,IAAI,GAAG,MAAM,iBAAiB,CAAC,IAAI,EAAE,CAAC,CAAC,oGAAoG;YAEjJ,2DAA2D;YAC3D,gDAAgD;YAChD,IAAI,IAAI,KAAK,0BAA0B,EAAE,CAAC;gBACtC,MAAM,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBACvC,IAAI,EAAE,OAAO,EAAE,GAAG,aAAa,CAAC;gBAChC,IAAI,CAAC,OAAO;oBAAE,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,cAAc,EAAE,EAAE,EAAE,CAAC,CAAC;gBACtF,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,MAAM,OAAO,EAAE,CAAC,CAAC;YAC9C,CAAC;YAED,IAAI,YAAY,EAAE,CAAC;gBACf,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,uCAAuC,CAAC,CAAC;YACtE,CAAC;YAED,8EAA8E;YAC9E,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,6BAA6B,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;QAChF,CAAC;aAAM,IAAI,uBAAuB,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YAChD,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAE,IAAI,EAAE,MAAM,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC;QAClE,CAAC;aAAM,CAAC;YACJ,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC;YACjD,OAAO;gBACH,IAAI;gBACJ,QAAQ;gBACR,WAAW;aACd,CAAC;QACN,CAAC;IACL,CAAC;IAED;;OAEG;IACO,kBAAkB,CAAC,OAAgB,EAAE,OAAiB,EAAE,QAAiB,EAAE,UAAwB;QACzG,MAAM,cAAc,GAA0E;YAC1F,GAAG,EAAE,OAAO,CAAC,GAAG;YAChB,MAAM,EAAE,OAAO,CAAC,MAAgB;YAChC,QAAQ;YACR,OAAO,EAAE,EAAE,OAAO,EAAE,IAAI,CAAC,uBAAuB,EAAE;YAClD,SAAS,EAAE,IAAI,CAAC,wBAAwB,CAAC,CAAC,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC,CAAC,SAAS;YACzE,YAAY,EAAE,OAAO;YACrB,GAAG,UAAU;YACb,OAAO,EAAE,EAAE,GAAG,OAAO,CAAC,OAAO,EAAE,GAAG,UAAU,EAAE,OAAO,EAAE;YACvD,KAAK,EAAE;gBACH,GAAG,UAAU,EAAE,KAAK;gBACpB,kBAAkB,EAAE,CAAC,IAAI,CAAC,eAAe;aAC5C;YACD,QAAQ,EAAE,IAAI;SACjB,CAAC;QAEF,yHAAyH;QACzH,OAAO,CAAC,cAAc,CAAC,cAAc,CAAC,OAAQ,EAAE,QAAQ,CAAC,CAAC;QAE1D,4CAA4C;QAC5C,IAAI,OAAO,EAAE,SAAS,EAAE,eAAe,EAAE,CAAC;YACtC,cAAc,CAAC,KAAK,GAAG;gBACnB,GAAG,cAAc,CAAC,KAAK;gBACvB,kBAAkB,EAAE,KAAK;aAC5B,CAAC;QACN,CAAC;QAED,IAAI,gBAAgB,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;YAAE,cAAc,CAAC,IAAI,GAAG,OAAO,CAAC,OAAO,IAAI,EAAE,CAAC;QAEvF,OAAO,cAAc,CAAC;IAC1B,CAAC;IAES,eAAe,CACrB,OAAgB,EAChB,QAAkB,EAClB,QAAwB;QAKxB,IAAI,IAAI,CAAC,qBAAqB,EAAE,CAAC;YAC7B,QAAQ,GAAG,IAAI,CAAC,qBAAuC,CAAC;QAC5D,CAAC;aAAM,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,uBAAuB,EAAE,CAAC;YACnD,QAAQ,GAAG,IAAI,CAAC,uBAAyC,CAAC;QAC9D,CAAC;QAED,sDAAsD;QACtD,MAAM,IAAI,GAAG,MAAM,CAAC;QACpB,IAAI,CAAC,QAAQ;YAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;QAEnD,2DAA2D;QAC3D,+CAA+C;QAC/C,IAAI,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC;YAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC;QAE/D,+DAA+D;QAC/D,IAAI,KAAK,CAAC,cAAc,CAAC,QAAQ,CAAC,EAAE,CAAC;YACjC,MAAM,YAAY,GAAG,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;YAC9C,MAAM,YAAY,GAAG,KAAK,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;YACxG,MAAM,aAAa,GAAG,QAAQ,CAAC,IAAI;gBAC/B,CAAC,CAAC,QAAQ,CAAC,KAAK,CACV,QAAQ,CAAC,IAAI,CACT,QAAQ,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAW,CAAC;qBACjC,IAAI,CAAC,YAAY,CAAC;qBAClB,IAAI,CAAC,YAAY,CAAC,CAC1B,CACJ;gBACH,CAAC,CAAC,IAAI,CAAC;YAEX,OAAO;gBACH,QAAQ,EAAE,IAAI,eAAe,CAAC,aAAoB,EAAE,QAAQ,CAAC;gBAC7D,QAAQ,EAAE,IAAI;aACjB,CAAC;QACN,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,YAAY,OAAO,CAAC,GAAG,8CAA8C,QAAQ,EAAE,CAAC,CAAC;IACrG,CAAC;IAED;;OAEG;IACO,yBAAyB,CAAC,mBAA4D;QAC5F,KAAK,MAAM,QAAQ,IAAI,mBAAmB,EAAE,CAAC;YACzC,IAAI,QAAQ,KAAK,KAAK,EAAE,CAAC;gBACrB,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;gBACtC,SAAS;YACb,CAAC;YAED,IAAI,CAAC;gBACD,MAAM,UAAU,GAAG,iBAAiB,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;gBACrD,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;YACjD,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,MAAM,IAAI,KAAK,CAAC,2BAA2B,QAAQ,sCAAsC,CAAC,CAAC;YAC/F,CAAC;QACL,CAAC;IACL,CAAC;IAED;;OAEG;IACO,qBAAqB,CAAC,OAAiB;QAC7C,OAAO,EAAE,OAAO,EAAE,CAAC;QACnB,MAAM,IAAI,KAAK,CAAC,2BAA2B,IAAI,CAAC,uBAAuB,GAAG,IAAI,WAAW,CAAC,CAAC;IAC/F,CAAC;IAEO,oBAAoB,CAAC,OAAgB,EAAE,QAAkB;QAC7D,MAAM,EAAE,MAAM,EAAE,GAAG,QAAQ,CAAC;QAC5B,MAAM,EAAE,IAAI,EAAE,GAAG,4BAA4B,CAAC,QAAQ,CAAC,CAAC;QAExD,sEAAsE;QACtE,MAAM,kBAAkB,GAAG,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC1F,wDAAwD;QACxD,MAAM,sBAAsB,GAAG,MAAM,IAAI,GAAG,IAAI,kBAAkB,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QAEpF,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,sBAAsB,EAAE,CAAC;YACvG,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC;YACvB,MAAM,IAAI,KAAK,CACX,YAAY,OAAO,CAAC,GAAG,wBAAwB,IAAI,IAAI;gBACnD,YAAY,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,kCAAkC,CACnG,CAAC;QACN,CAAC;IACL,CAAC;IAED;;OAEG;IACK,iBAAiB,GAAG,KAAK,EAC7B,OAA4D,EAC5D,OAAiB,EACnB,EAAE;QACA,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,MAAM,CACzC,yBAAyB,CAAC;YACtB,GAAI,OAAe;YACnB,SAAS,EAAE,OAAO,CAAC,SAAS;YAC5B,YAAY,EAAE,MAAM;SACvB,CAAC,EACF,CAAC,gBAAgB,EAAE,cAAc,EAAE,EAAE;YACjC,IAAI,IAAI,CAAC,wBAAwB,EAAE,CAAC;gBAChC,OAAQ,CAAC,sBAAsB,CAAC,gBAAgB,CAAC,CAAC;gBAElD,MAAM,YAAY,GAAG,OAAQ,CAAC,eAAe,CAAC,cAAc,CAAC,GAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;gBAC9E,IAAI,YAAY,KAAK,EAAE,EAAE,CAAC;oBACtB,cAAc,CAAC,OAAO,CAAC,MAAM,GAAG,YAAY,CAAC;gBACjD,CAAC;YACL,CAAC;QACL,CAAC,CACJ,CAAC;QAEF,OAAO,QAAQ,CAAC;IACpB,CAAC,CAAC;;AAUN;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,MAAM,UAAU,gBAAgB,CAG9B,MAAwC;IACtC,OAAO,MAAM,CAAC,MAAM,CAAU,MAAM,CAAC,CAAC;AAC1C,CAAC"}
|