@crawlee/core 4.0.0-beta.2 → 4.0.0-beta.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -1
- package/autoscaling/snapshotter.d.ts.map +1 -1
- package/autoscaling/snapshotter.js +4 -11
- package/autoscaling/snapshotter.js.map +1 -1
- package/configuration.d.ts +0 -7
- package/configuration.d.ts.map +1 -1
- package/configuration.js +0 -4
- package/configuration.js.map +1 -1
- package/cookie_utils.d.ts +1 -1
- package/cookie_utils.d.ts.map +1 -1
- package/cookie_utils.js +4 -6
- package/cookie_utils.js.map +1 -1
- package/crawlers/context_pipeline.d.ts +61 -0
- package/crawlers/context_pipeline.d.ts.map +1 -0
- package/crawlers/context_pipeline.js +99 -0
- package/crawlers/context_pipeline.js.map +1 -0
- package/crawlers/crawler_commons.d.ts +12 -16
- package/crawlers/crawler_commons.d.ts.map +1 -1
- package/crawlers/crawler_commons.js +0 -8
- package/crawlers/crawler_commons.js.map +1 -1
- package/crawlers/error_snapshotter.d.ts +3 -2
- package/crawlers/error_snapshotter.d.ts.map +1 -1
- package/crawlers/error_snapshotter.js +2 -2
- package/crawlers/error_snapshotter.js.map +1 -1
- package/crawlers/error_tracker.d.ts +2 -1
- package/crawlers/error_tracker.d.ts.map +1 -1
- package/crawlers/error_tracker.js.map +1 -1
- package/crawlers/index.d.ts +1 -1
- package/crawlers/index.d.ts.map +1 -1
- package/crawlers/index.js +1 -1
- package/crawlers/index.js.map +1 -1
- package/crawlers/internals/types.d.ts +8 -0
- package/crawlers/internals/types.d.ts.map +1 -0
- package/crawlers/internals/types.js +2 -0
- package/crawlers/internals/types.js.map +1 -0
- package/enqueue_links/enqueue_links.d.ts +22 -15
- package/enqueue_links/enqueue_links.d.ts.map +1 -1
- package/enqueue_links/enqueue_links.js +33 -14
- package/enqueue_links/enqueue_links.js.map +1 -1
- package/enqueue_links/shared.d.ts +8 -2
- package/enqueue_links/shared.d.ts.map +1 -1
- package/enqueue_links/shared.js +40 -18
- package/enqueue_links/shared.js.map +1 -1
- package/errors.d.ts +12 -0
- package/errors.d.ts.map +1 -1
- package/errors.js +20 -0
- package/errors.js.map +1 -1
- package/events/local_event_manager.d.ts +0 -2
- package/events/local_event_manager.d.ts.map +1 -1
- package/events/local_event_manager.js +3 -32
- package/events/local_event_manager.js.map +1 -1
- package/http_clients/base-http-client.d.ts +26 -69
- package/http_clients/base-http-client.d.ts.map +1 -1
- package/http_clients/base-http-client.js +20 -9
- package/http_clients/base-http-client.js.map +1 -1
- package/http_clients/got-scraping-http-client.d.ts +10 -3
- package/http_clients/got-scraping-http-client.d.ts.map +1 -1
- package/http_clients/got-scraping-http-client.js +86 -39
- package/http_clients/got-scraping-http-client.js.map +1 -1
- package/index.d.ts +1 -0
- package/index.d.ts.map +1 -1
- package/index.js +1 -0
- package/index.js.map +1 -1
- package/log.d.ts +4 -2
- package/log.d.ts.map +1 -1
- package/log.js.map +1 -1
- package/package.json +6 -6
- package/proxy_configuration.d.ts +21 -40
- package/proxy_configuration.d.ts.map +1 -1
- package/proxy_configuration.js +15 -51
- package/proxy_configuration.js.map +1 -1
- package/recoverable_state.d.ts +123 -0
- package/recoverable_state.d.ts.map +1 -0
- package/recoverable_state.js +140 -0
- package/recoverable_state.js.map +1 -0
- package/request.d.ts +31 -5
- package/request.d.ts.map +1 -1
- package/request.js +35 -12
- package/request.js.map +1 -1
- package/session_pool/session.d.ts +6 -2
- package/session_pool/session.d.ts.map +1 -1
- package/session_pool/session.js +9 -2
- package/session_pool/session.js.map +1 -1
- package/session_pool/session_pool.d.ts +9 -2
- package/session_pool/session_pool.d.ts.map +1 -1
- package/session_pool/session_pool.js +13 -1
- package/session_pool/session_pool.js.map +1 -1
- package/storages/dataset.d.ts +7 -0
- package/storages/dataset.d.ts.map +1 -1
- package/storages/dataset.js +10 -1
- package/storages/dataset.js.map +1 -1
- package/storages/index.d.ts +2 -0
- package/storages/index.d.ts.map +1 -1
- package/storages/index.js +2 -0
- package/storages/index.js.map +1 -1
- package/storages/key_value_store.d.ts +13 -1
- package/storages/key_value_store.d.ts.map +1 -1
- package/storages/key_value_store.js +11 -6
- package/storages/key_value_store.js.map +1 -1
- package/storages/request_list.d.ts +0 -6
- package/storages/request_list.d.ts.map +1 -1
- package/storages/request_list.js.map +1 -1
- package/storages/request_list_adapter.d.ts +58 -0
- package/storages/request_list_adapter.d.ts.map +1 -0
- package/storages/request_list_adapter.js +81 -0
- package/storages/request_list_adapter.js.map +1 -0
- package/storages/request_manager_tandem.d.ts +68 -0
- package/storages/request_manager_tandem.d.ts.map +1 -0
- package/storages/request_manager_tandem.js +124 -0
- package/storages/request_manager_tandem.js.map +1 -0
- package/storages/request_provider.d.ts +68 -4
- package/storages/request_provider.d.ts.map +1 -1
- package/storages/request_provider.js +74 -44
- package/storages/request_provider.js.map +1 -1
- package/storages/request_queue_v2.d.ts +2 -2
- package/storages/request_queue_v2.d.ts.map +1 -1
- package/storages/request_queue_v2.js.map +1 -1
- package/storages/sitemap_request_list.d.ts.map +1 -1
- package/storages/sitemap_request_list.js.map +1 -1
- package/storages/storage_manager.d.ts.map +1 -1
- package/storages/storage_manager.js +1 -1
- package/storages/storage_manager.js.map +1 -1
- package/typedefs.d.ts +1 -1
- package/typedefs.d.ts.map +1 -1
- package/crawlers/crawler_extension.d.ts +0 -12
- package/crawlers/crawler_extension.d.ts.map +0 -1
- package/crawlers/crawler_extension.js +0 -14
- package/crawlers/crawler_extension.js.map +0 -1
- package/tsconfig.build.tsbuildinfo +0 -1
package/README.md
CHANGED
|
@@ -9,6 +9,10 @@
|
|
|
9
9
|
<small>A web scraping and browser automation library</small>
|
|
10
10
|
</h1>
|
|
11
11
|
|
|
12
|
+
<p align=center>
|
|
13
|
+
<a href="https://trendshift.io/repositories/5179" target="_blank"><img src="https://trendshift.io/api/badge/repositories/5179" alt="apify%2Fcrawlee | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
|
|
14
|
+
</p>
|
|
15
|
+
|
|
12
16
|
<p align=center>
|
|
13
17
|
<a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/v/@crawlee/core.svg" alt="NPM latest version" data-canonical-src="https://img.shields.io/npm/v/@crawlee/core/next.svg" style="max-width: 100%;"></a>
|
|
14
18
|
<a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/dm/@crawlee/core.svg" alt="Downloads" data-canonical-src="https://img.shields.io/npm/dm/@crawlee/core.svg" style="max-width: 100%;"></a>
|
|
@@ -24,7 +28,7 @@ Crawlee is available as the [`crawlee`](https://www.npmjs.com/package/crawlee) N
|
|
|
24
28
|
|
|
25
29
|
> 👉 **View full documentation, guides and examples on the [Crawlee project website](https://crawlee.dev)** 👈
|
|
26
30
|
|
|
27
|
-
>
|
|
31
|
+
> Do you prefer 🐍 Python instead of JavaScript? [👉 Checkout Crawlee for Python 👈](https://github.com/apify/crawlee-python).
|
|
28
32
|
|
|
29
33
|
## Installation
|
|
30
34
|
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"snapshotter.d.ts","sourceRoot":"","sources":["../../src/autoscaling/snapshotter.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAIpD,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AACtC,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAGzD,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAG/D,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAMrD,MAAM,WAAW,kBAAkB;IAC/B;;;OAGG;IACH,6BAA6B,CAAC,EAAE,MAAM,CAAC;IAEvC;;;;OAIG;IACH,0BAA0B,CAAC,EAAE,MAAM,CAAC;IAEpC;;;;OAIG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAE1B;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAE5B;;;;OAIG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IAEzB;;;;OAIG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAE7B,gBAAgB;IAChB,GAAG,CAAC,EAAE,GAAG,CAAC;IAEV,gBAAgB;IAChB,MAAM,CAAC,EAAE,aAAa,CAAC;IAEvB,gBAAgB;IAChB,MAAM,CAAC,EAAE,aAAa,CAAC;CAC1B;AAED,UAAU,cAAc;IACpB,SAAS,EAAE,IAAI,CAAC;IAChB,YAAY,EAAE,OAAO,CAAC;IACtB,SAAS,CAAC,EAAE,MAAM,CAAC;CACtB;AACD,UAAU,WAAW;IACjB,SAAS,EAAE,IAAI,CAAC;IAChB,YAAY,EAAE,OAAO,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;CAC3C;AACD,UAAU,iBAAiB;IACvB,SAAS,EAAE,IAAI,CAAC;IAChB,YAAY,EAAE,OAAO,CAAC;IACtB,cAAc,EAAE,MAAM,CAAC;CAC1B;AACD,UAAU,cAAc;IACpB,SAAS,EAAE,IAAI,CAAC;IAChB,YAAY,EAAE,OAAO,CAAC;IACtB,mBAAmB,EAAE,MAAM,CAAC;CAC/B;AAED;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,qBAAa,WAAW;IACpB,GAAG,EAAE,GAAG,CAAC;IACT,MAAM,EAAE,aAAa,CAAC;IACtB,MAAM,EAAE,aAAa,CAAC;IACtB,MAAM,EAAE,YAAY,CAAC;IACrB,+BAA+B,EAAE,MAAM,CAAC;IACxC,4BAA4B,EAAE,MAAM,CAAC;IACrC,qBAAqB,EAAE,MAAM,CAAC;IAC9B,gBAAgB,EAAE,MAAM,CAAC;IACzB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,eAAe,EAAE,MAAM,CAAC;IACxB,cAAc,EAAG,MAAM,CAAC;IAExB,YAAY,EAAE,WAAW,EAAE,CAAM;IACjC,kBAAkB,EAAE,iBAAiB,EAAE,CAAM;IAC7C,eAAe,EAAE,cAAc,EAAE,CAAM;IACvC,eAAe,EAAE,cAAc,EAAE,CAAM;IAEvC,iBAAiB,EAAE,gBAAgB,CAAS;IAC5C,cAAc,EAAE,gBAAgB,CAAS;IAEzC,kCAAkC,EAAE,IAAI,GAAG,IAAI,CAAQ;IAEvD;;OAEG;gBACS,OAAO,GAAE,kBAAuB;IA6C5C;;OAEG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;
|
|
1
|
+
{"version":3,"file":"snapshotter.d.ts","sourceRoot":"","sources":["../../src/autoscaling/snapshotter.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAIpD,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AACtC,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAGzD,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAG/D,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAMrD,MAAM,WAAW,kBAAkB;IAC/B;;;OAGG;IACH,6BAA6B,CAAC,EAAE,MAAM,CAAC;IAEvC;;;;OAIG;IACH,0BAA0B,CAAC,EAAE,MAAM,CAAC;IAEpC;;;;OAIG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAE1B;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAE5B;;;;OAIG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IAEzB;;;;OAIG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAE7B,gBAAgB;IAChB,GAAG,CAAC,EAAE,GAAG,CAAC;IAEV,gBAAgB;IAChB,MAAM,CAAC,EAAE,aAAa,CAAC;IAEvB,gBAAgB;IAChB,MAAM,CAAC,EAAE,aAAa,CAAC;CAC1B;AAED,UAAU,cAAc;IACpB,SAAS,EAAE,IAAI,CAAC;IAChB,YAAY,EAAE,OAAO,CAAC;IACtB,SAAS,CAAC,EAAE,MAAM,CAAC;CACtB;AACD,UAAU,WAAW;IACjB,SAAS,EAAE,IAAI,CAAC;IAChB,YAAY,EAAE,OAAO,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;CAC3C;AACD,UAAU,iBAAiB;IACvB,SAAS,EAAE,IAAI,CAAC;IAChB,YAAY,EAAE,OAAO,CAAC;IACtB,cAAc,EAAE,MAAM,CAAC;CAC1B;AACD,UAAU,cAAc;IACpB,SAAS,EAAE,IAAI,CAAC;IAChB,YAAY,EAAE,OAAO,CAAC;IACtB,mBAAmB,EAAE,MAAM,CAAC;CAC/B;AAED;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,qBAAa,WAAW;IACpB,GAAG,EAAE,GAAG,CAAC;IACT,MAAM,EAAE,aAAa,CAAC;IACtB,MAAM,EAAE,aAAa,CAAC;IACtB,MAAM,EAAE,YAAY,CAAC;IACrB,+BAA+B,EAAE,MAAM,CAAC;IACxC,4BAA4B,EAAE,MAAM,CAAC;IACrC,qBAAqB,EAAE,MAAM,CAAC;IAC9B,gBAAgB,EAAE,MAAM,CAAC;IACzB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,eAAe,EAAE,MAAM,CAAC;IACxB,cAAc,EAAG,MAAM,CAAC;IAExB,YAAY,EAAE,WAAW,EAAE,CAAM;IACjC,kBAAkB,EAAE,iBAAiB,EAAE,CAAM;IAC7C,eAAe,EAAE,cAAc,EAAE,CAAM;IACvC,eAAe,EAAE,cAAc,EAAE,CAAM;IAEvC,iBAAiB,EAAE,gBAAgB,CAAS;IAC5C,cAAc,EAAE,gBAAgB,CAAS;IAEzC,kCAAkC,EAAE,IAAI,GAAG,IAAI,CAAQ;IAEvD;;OAEG;gBACS,OAAO,GAAE,kBAAuB;IA6C5C;;OAEG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IA2B5B;;OAEG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAW3B;;;OAGG;IACH,eAAe,CAAC,oBAAoB,CAAC,EAAE,MAAM;IAI7C;;;OAGG;IACH,kBAAkB,CAAC,oBAAoB,CAAC,EAAE,MAAM;IAIhD;;;OAGG;IACH,YAAY,CAAC,oBAAoB,CAAC,EAAE,MAAM;IAI1C;;;OAGG;IACH,eAAe,CAAC,oBAAoB,CAAC,EAAE,MAAM;IAI7C;;OAEG;IACH,SAAS,CAAC,UAAU,CAAC,CAAC,SAAS;QAAE,SAAS,EAAE,IAAI,CAAA;KAAE,EAAE,SAAS,EAAE,CAAC,EAAE,EAAE,oBAAoB,CAAC,EAAE,MAAM,GAAG,CAAC,EAAE;IAoBvG;;;OAGG;IACH,SAAS,CAAC,eAAe,CAAC,UAAU,EAAE,UAAU;IAchD;;OAEG;IACH,SAAS,CAAC,sBAAsB,CAAC,UAAU,EAAE,UAAU;IA2BvD;;OAEG;IACH,SAAS,CAAC,kBAAkB,CAAC,gBAAgB,EAAE,MAAM,OAAO;IAuB5D;;OAEG;IACH,SAAS,CAAC,YAAY,CAAC,UAAU,EAAE,UAAU;IAY7C;;;;;;;OAOG;IACH,SAAS,CAAC,eAAe,CAAC,gBAAgB,EAAE,MAAM,OAAO;IAwBzD;;;OAGG;IACH,SAAS,CAAC,eAAe,CACrB,SAAS,EAAE,cAAc,EAAE,GAAG,WAAW,EAAE,GAAG,iBAAiB,EAAE,GAAG,cAAc,EAAE,EACpF,GAAG,EAAE,IAAI;CAUhB"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { getMemoryInfo,
|
|
1
|
+
import { getMemoryInfo, isContainerized } from '@crawlee/utils';
|
|
2
2
|
import ow from 'ow';
|
|
3
3
|
import { betterClearInterval, betterSetInterval } from '@apify/utilities';
|
|
4
4
|
import { Configuration } from '../configuration.js';
|
|
@@ -89,16 +89,9 @@ export class Snapshotter {
|
|
|
89
89
|
this.maxMemoryBytes = memoryMbytes * 1024 * 1024;
|
|
90
90
|
}
|
|
91
91
|
else {
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
const memInfo = await getMemoryInfoV2(containerized);
|
|
96
|
-
totalBytes = memInfo.totalBytes;
|
|
97
|
-
}
|
|
98
|
-
else {
|
|
99
|
-
const memInfo = await getMemoryInfo();
|
|
100
|
-
totalBytes = memInfo.totalBytes;
|
|
101
|
-
}
|
|
92
|
+
const containerized = this.config.get('containerized', await isContainerized());
|
|
93
|
+
const memInfo = await getMemoryInfo(containerized);
|
|
94
|
+
const totalBytes = memInfo.totalBytes;
|
|
102
95
|
this.maxMemoryBytes = Math.ceil(totalBytes * this.config.get('availableMemoryRatio'));
|
|
103
96
|
this.log.debug(`Setting max memory of this run to ${Math.round(this.maxMemoryBytes / 1024 / 1024)} MB. ` +
|
|
104
97
|
'Use the CRAWLEE_MEMORY_MBYTES or CRAWLEE_AVAILABLE_MEMORY_RATIO environment variable to override it.');
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"snapshotter.js","sourceRoot":"","sources":["../../src/autoscaling/snapshotter.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,aAAa,EAAE,eAAe,EAAE,
|
|
1
|
+
{"version":3,"file":"snapshotter.js","sourceRoot":"","sources":["../../src/autoscaling/snapshotter.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,aAAa,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AAChE,OAAO,EAAE,MAAM,IAAI,CAAC;AAIpB,OAAO,EAAE,mBAAmB,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAE1E,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAGpD,OAAO,EAAE,GAAG,IAAI,UAAU,EAAE,MAAM,WAAW,CAAC;AAG9C,MAAM,oBAAoB,GAAG,GAAG,CAAC;AACjC,MAAM,mCAAmC,GAAG,CAAC,CAAC;AAC9C,MAAM,mCAAmC,GAAG,KAAK,CAAC;AA4ElD;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,MAAM,OAAO,WAAW;IACpB,GAAG,CAAM;IACT,MAAM,CAAgB;IACtB,MAAM,CAAgB;IACtB,MAAM,CAAe;IACrB,+BAA+B,CAAS;IACxC,4BAA4B,CAAS;IACrC,qBAAqB,CAAS;IAC9B,gBAAgB,CAAS;IACzB,kBAAkB,CAAS;IAC3B,eAAe,CAAS;IACxB,cAAc,CAAU;IAExB,YAAY,GAAkB,EAAE,CAAC;IACjC,kBAAkB,GAAwB,EAAE,CAAC;IAC7C,eAAe,GAAqB,EAAE,CAAC;IACvC,eAAe,GAAqB,EAAE,CAAC;IAEvC,iBAAiB,GAAqB,IAAK,CAAC;IAC5C,cAAc,GAAqB,IAAK,CAAC;IAEzC,kCAAkC,GAAgB,IAAI,CAAC;IAEvD;;OAEG;IACH,YAAY,UAA8B,EAAE;QACxC,EAAE,CACE,OAAO,EACP,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YACjB,6BAA6B,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACjD,0BAA0B,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YAC9C,mBAAmB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACvC,gBAAgB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACpC,kBAAkB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACtC,eAAe,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACnC,GAAG,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACvB,MAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YAC1B,MAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;SAC7B,CAAC,CACL,CAAC;QAEF,MAAM,EACF,6BAA6B,GAAG,GAAG,EACnC,0BAA0B,GAAG,CAAC,EAC9B,mBAAmB,GAAG,EAAE,EACxB,gBAAgB,GAAG,EAAE,EACrB,kBAAkB,GAAG,GAAG,EACxB,eAAe,GAAG,CAAC,EACnB,GAAG,GAAG,UAAU,EAChB,MAAM,GAAG,aAAa,CAAC,eAAe,EAAE,EACxC,MAAM,GAAG,MAAM,CAAC,gBAAgB,EAAE,GACrC,GAAG,OAAO,CAAC;QAEZ,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,aAAa,EAAE,CAAC,CAAC;QAChD,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,eAAe,EAAE,CAAC;QAE5C,IAAI,CAAC,+BAA+B,GAAG,6BAA6B,GAAG,IAAI,CAAC;QAC5E,IAAI,CAAC,4BAA4B,GAAG,0BAA0B,GAAG,IAAI,CAAC;QACtE,IAAI,CAAC,qBAAqB,GAAG,mBAAmB,GAAG,IAAI,CAAC;QACxD,IAAI,CAAC,gBAAgB,GAAG,gBAAgB,CAAC;QACzC,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;QAEvC,mFAAmF;QACnF,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACjD,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC3D,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK;QACP,MAAM,YAAY,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC;QAExD,IAAI,YAAY,GAAG,CAAC,EAAE,CAAC;YACnB,IAAI,CAAC,cAAc,GAAG,YAAY,GAAG,IAAI,GAAG,IAAI,CAAC;QACrD,CAAC;aAAM,CAAC;YACJ,MAAM,aAAa,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,eAAe,EAAE,MAAM,eAAe,EAAE,CAAC,CAAC;YAChF,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,aAAa,CAAC,CAAC;YACnD,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC;YAEtC,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,sBAAsB,CAAE,CAAC,CAAC;YACvF,IAAI,CAAC,GAAG,CAAC,KAAK,CACV,qCAAqC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,cAAc,GAAG,IAAI,GAAG,IAAI,CAAC,OAAO;gBACrF,sGAAsG,CAC7G,CAAC;QACN,CAAC;QAED,sBAAsB;QACtB,IAAI,CAAC,iBAAiB,GAAG,iBAAiB,CACtC,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC,EAClC,IAAI,CAAC,+BAA+B,CACvC,CAAC;QACF,IAAI,CAAC,cAAc,GAAG,iBAAiB,CAAC,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,4BAA4B,CAAC,CAAC;QAC5G,IAAI,CAAC,MAAM,CAAC,EAAE,2CAAwB,IAAI,CAAC,YAAY,CAAC,CAAC;QACzD,IAAI,CAAC,MAAM,CAAC,EAAE,2CAAwB,IAAI,CAAC,eAAe,CAAC,CAAC;IAChE,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,IAAI;QACN,mBAAmB,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;QAC5C,mBAAmB,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QACzC,IAAI,CAAC,MAAM,CAAC,GAAG,2CAAwB,IAAI,CAAC,YAAY,CAAC,CAAC;QAC1D,IAAI,CAAC,MAAM,CAAC,GAAG,2CAAwB,IAAI,CAAC,eAAe,CAAC,CAAC;QAC7D,uDAAuD;QACvD,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;YAC1B,YAAY,CAAC,OAAO,CAAC,CAAC;QAC1B,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;;OAGG;IACH,eAAe,CAAC,oBAA6B;QACzC,OAAO,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,eAAe,EAAE,oBAAoB,CAAC,CAAC;IACvE,CAAC;IAED;;;OAGG;IACH,kBAAkB,CAAC,oBAA6B;QAC5C,OAAO,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,kBAAkB,EAAE,oBAAoB,CAAC,CAAC;IAC1E,CAAC;IAED;;;OAGG;IACH,YAAY,CAAC,oBAA6B;QACtC,OAAO,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,YAAY,EAAE,oBAAoB,CAAC,CAAC;IACpE,CAAC;IAED;;;OAGG;IACH,eAAe,CAAC,oBAA6B;QACzC,OAAO,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,eAAe,EAAE,oBAAoB,CAAC,CAAC;IACvE,CAAC;IAED;;OAEG;IACO,UAAU,CAAgC,SAAc,EAAE,oBAA6B;QAC7F,IAAI,CAAC,oBAAoB;YAAE,OAAO,SAAS,CAAC;QAE5C,MAAM,MAAM,GAAQ,EAAE,CAAC;QACvB,IAAI,GAAG,GAAG,SAAS,CAAC,MAAM,CAAC;QAC3B,IAAI,CAAC,GAAG;YAAE,OAAO,MAAM,CAAC;QAExB,MAAM,UAAU,GAAG,SAAS,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC;QAChD,OAAO,GAAG,EAAE,EAAE,CAAC;YACX,MAAM,QAAQ,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC;YAChC,IAAI,CAAC,UAAU,GAAG,CAAC,QAAQ,CAAC,SAAS,IAAI,oBAAoB,EAAE,CAAC;gBAC5D,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;YAC7B,CAAC;iBAAM,CAAC;gBACJ,MAAM;YACV,CAAC;QACL,CAAC;QAED,OAAO,MAAM,CAAC;IAClB,CAAC;IAED;;;OAGG;IACO,eAAe,CAAC,UAAsB;QAC5C,MAAM,SAAS,GAAG,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC;QACrF,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,eAAe,EAAE,SAAS,CAAC,CAAC;QACtD,MAAM,EAAE,eAAe,EAAE,GAAG,UAAU,CAAC;QACvC,MAAM,QAAQ,GAAmB;YAC7B,SAAS;YACT,YAAY,EAAE,eAAgB,GAAG,IAAI,CAAC,cAAe,GAAG,IAAI,CAAC,kBAAkB;YAC/E,SAAS,EAAE,eAAe;SAC7B,CAAC;QAEF,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACpC,IAAI,CAAC,sBAAsB,CAAC,UAAU,CAAC,CAAC;IAC5C,CAAC;IAED;;OAEG;IACO,sBAAsB,CAAC,UAAsB;QACnD,MAAM,EAAE,eAAe,EAAE,GAAG,UAAU,CAAC;QACvC,MAAM,SAAS,GAAG,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC;QACrF,IACI,IAAI,CAAC,kCAAkC;YACvC,CAAC,SAAS,GAAG,CAAC,IAAI,CAAC,kCAAkC,GAAG,mCAAmC;YAE3F,OAAO;QAEX,MAAM,qBAAqB,GAAG,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC,cAAe,CAAC;QAC7E,MAAM,aAAa,GAAG,IAAI,CAAC,cAAe,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,kBAAkB,CAAC,GAAG,oBAAoB,CAAC;QAClG,MAAM,qBAAqB,GAAG,qBAAqB,GAAG,aAAa,CAAC;QACpE,MAAM,kBAAkB,GAAG,eAAgB,GAAG,qBAAqB,CAAC;QAEpE,IAAI,kBAAkB,EAAE,CAAC;YACrB,MAAM,cAAc,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,eAAgB,GAAG,IAAI,CAAC,cAAe,CAAC,GAAG,GAAG,CAAC,CAAC;YACnF,MAAM,IAAI,GAAG,CAAC,KAAa,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,IAAI,IAAI,CAAC,CAAC,CAAC;YAC9D,IAAI,CAAC,GAAG,CAAC,OAAO,CACZ,mCAAmC;gBAC/B,SAAS,IAAI,CAAC,eAAgB,CAAC,UAAU,IAAI,CACzC,IAAI,CAAC,cAAe,CACvB,QAAQ,cAAc,2CAA2C,CACzE,CAAC;YACF,IAAI,CAAC,kCAAkC,GAAG,SAAS,CAAC;QACxD,CAAC;IACL,CAAC;IAED;;OAEG;IACO,kBAAkB,CAAC,gBAA+B;QACxD,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,kBAAkB,EAAE,GAAG,CAAC,CAAC;QAEnD,MAAM,QAAQ,GAAG;YACb,SAAS,EAAE,GAAG;YACd,YAAY,EAAE,KAAK;YACnB,cAAc,EAAE,CAAC;SACpB,CAAC;QAEF,MAAM,gBAAgB,GAAG,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,kBAAkB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACrF,IAAI,gBAAgB,EAAE,CAAC;YACnB,MAAM,EAAE,SAAS,EAAE,GAAG,gBAAgB,CAAC;YACvC,MAAM,KAAK,GAAG,GAAG,CAAC,OAAO,EAAE,GAAG,CAAC,SAAS,GAAG,IAAI,CAAC,+BAA+B,CAAC;YAEhF,IAAI,KAAK,GAAG,IAAI,CAAC,gBAAgB;gBAAE,QAAQ,CAAC,YAAY,GAAG,IAAI,CAAC;YAChE,QAAQ,CAAC,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,IAAI,CAAC,gBAAgB,EAAE,CAAC,CAAC,CAAC;QACzE,CAAC;QAED,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACvC,gBAAgB,EAAE,CAAC;IACvB,CAAC;IAED;;OAEG;IACO,YAAY,CAAC,UAAsB;QACzC,MAAM,EAAE,eAAe,EAAE,eAAe,EAAE,GAAG,UAAU,CAAC;QACxD,MAAM,SAAS,GAAG,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC;QACrF,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,YAAY,EAAE,SAAS,CAAC,CAAC;QAEnD,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC;YACnB,SAAS;YACT,YAAY,EAAE,eAAgB;YAC9B,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,eAAgB,GAAG,GAAG,CAAC;SAC/C,CAAC,CAAC;IACP,CAAC;IAED;;;;;;;OAOG;IACO,eAAe,CAAC,gBAA+B;QACrD,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,eAAe,EAAE,GAAG,CAAC,CAAC;QAEhD,MAAM,cAAc,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,eAAe,IAAI,EAAE,CAAC,CAAC,wCAAwC;QACzG,MAAM,eAAe,GAAG,cAAc,CAAC,mCAAmC,CAAC,IAAI,CAAC,CAAC;QAEjF,+BAA+B;QAC/B,MAAM,QAAQ,GAAG;YACb,SAAS,EAAE,GAAG;YACd,YAAY,EAAE,KAAK;YACnB,mBAAmB,EAAE,eAAe;SACvC,CAAC;QACF,MAAM,gBAAgB,GAAG,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,eAAe,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAC/E,IAAI,gBAAgB,EAAE,CAAC;YACnB,MAAM,EAAE,mBAAmB,EAAE,GAAG,gBAAgB,CAAC;YACjD,MAAM,KAAK,GAAG,eAAe,GAAG,mBAAmB,CAAC;YACpD,IAAI,KAAK,GAAG,IAAI,CAAC,eAAe;gBAAE,QAAQ,CAAC,YAAY,GAAG,IAAI,CAAC;QACnE,CAAC;QAED,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACpC,gBAAgB,EAAE,CAAC;IACvB,CAAC;IAED;;;OAGG;IACO,eAAe,CACrB,SAAoF,EACpF,GAAS;QAET,IAAI,QAAQ,GAAG,CAAC,CAAC;QACjB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACxC,MAAM,EAAE,SAAS,EAAE,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;YACnC,IAAI,GAAG,CAAC,OAAO,EAAE,GAAG,IAAI,IAAI,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,qBAAqB;gBAAE,QAAQ,EAAE,CAAC;;gBACtF,MAAM;QACf,CAAC;QACD,SAAS,CAAC,MAAM,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;IAClC,CAAC;CACJ"}
|
package/configuration.d.ts
CHANGED
|
@@ -136,12 +136,6 @@ export interface ConfigurationOptions {
|
|
|
136
136
|
* Alternative to `CRAWLEE_PERSIST_STORAGE` environment variable.
|
|
137
137
|
*/
|
|
138
138
|
persistStorage?: boolean;
|
|
139
|
-
/**
|
|
140
|
-
* Defines whether to use the systemInfoV2 metric collection experiment.
|
|
141
|
-
*
|
|
142
|
-
* Alternative to `CRAWLEE_SYSTEM_INFO_V2` environment variable.
|
|
143
|
-
*/
|
|
144
|
-
systemInfoV2?: boolean;
|
|
145
139
|
/**
|
|
146
140
|
* Used in place of `isContainerized()` when collecting system metrics.
|
|
147
141
|
*
|
|
@@ -212,7 +206,6 @@ export interface ConfigurationOptions {
|
|
|
212
206
|
* `defaultBrowserPath` | `CRAWLEE_DEFAULT_BROWSER_PATH` | -
|
|
213
207
|
* `disableBrowserSandbox` | `CRAWLEE_DISABLE_BROWSER_SANDBOX` | -
|
|
214
208
|
* `availableMemoryRatio` | `CRAWLEE_AVAILABLE_MEMORY_RATIO` | `0.25`
|
|
215
|
-
* `systemInfoV2` | `CRAWLEE_SYSTEM_INFO_V2` | false
|
|
216
209
|
* `containerized | `CRAWLEE_CONTAINERIZED | -
|
|
217
210
|
*/
|
|
218
211
|
export declare class Configuration {
|
package/configuration.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"configuration.d.ts","sourceRoot":"","sources":["../src/configuration.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAKrD,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AACpE,OAAO,EAAE,aAAa,EAAE,MAAM,yBAAyB,CAAC;AACxD,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAGhE,OAAY,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAE3C,OAAO,EAAE,KAAK,YAAY,EAAE,MAAM,2BAA2B,CAAC;AAE9D,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,+BAA+B,CAAC;AACpE,OAAO,EAAE,KAAK,WAAW,EAAW,MAAM,eAAe,CAAC;AAE1D,MAAM,WAAW,oBAAoB;IACjC;;;OAGG;IACH,aAAa,CAAC,EAAE,aAAa,CAAC;IAE9B;;;OAGG;IACH,YAAY,CAAC,EAAE,YAAY,CAAC;IAE5B;;;OAGG;IACH,oBAAoB,CAAC,EAAE,UAAU,CAAC;IAElC;;;;;OAKG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAE1B;;;;;OAKG;IACH,YAAY,CAAC,EAAE,OAAO,CAAC;IAEvB;;;;;OAKG;IACH,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAEhC;;;;;OAKG;IACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAE/B;;;;OAIG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IAEzB;;;;;;OAMG;IACH,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAE9B;;;;;OAKG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IAEtB;;;;;OAKG;IACH,0BAA0B,CAAC,EAAE,MAAM,CAAC;IAEpC;;;OAGG;IACH,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAElC;;;;;;OAMG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB;;;;;OAKG;IACH,QAAQ,CAAC,EAAE,OAAO,CAAC;IAEnB;;;;;OAKG;IACH,IAAI,CAAC,EAAE,OAAO,CAAC;IAEf;;;;OAIG;IACH,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAE9B;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAE5B;;;;OAIG;IACH,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAEhC;;;;;OAKG;IACH,QAAQ,CAAC,EAAE,QAAQ,GAAG,QAAQ,CAAC,MAAM,QAAQ,CAAC,CAAC;IAE/C;;;;OAIG;IACH,cAAc,CAAC,EAAE,OAAO,CAAC;IAEzB;;;;OAIG;IACH,
|
|
1
|
+
{"version":3,"file":"configuration.d.ts","sourceRoot":"","sources":["../src/configuration.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAKrD,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AACpE,OAAO,EAAE,aAAa,EAAE,MAAM,yBAAyB,CAAC;AACxD,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAGhE,OAAY,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAE3C,OAAO,EAAE,KAAK,YAAY,EAAE,MAAM,2BAA2B,CAAC;AAE9D,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,+BAA+B,CAAC;AACpE,OAAO,EAAE,KAAK,WAAW,EAAW,MAAM,eAAe,CAAC;AAE1D,MAAM,WAAW,oBAAoB;IACjC;;;OAGG;IACH,aAAa,CAAC,EAAE,aAAa,CAAC;IAE9B;;;OAGG;IACH,YAAY,CAAC,EAAE,YAAY,CAAC;IAE5B;;;OAGG;IACH,oBAAoB,CAAC,EAAE,UAAU,CAAC;IAElC;;;;;OAKG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAE1B;;;;;OAKG;IACH,YAAY,CAAC,EAAE,OAAO,CAAC;IAEvB;;;;;OAKG;IACH,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAEhC;;;;;OAKG;IACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAE/B;;;;OAIG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IAEzB;;;;;;OAMG;IACH,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAE9B;;;;;OAKG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IAEtB;;;;;OAKG;IACH,0BAA0B,CAAC,EAAE,MAAM,CAAC;IAEpC;;;OAGG;IACH,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAElC;;;;;;OAMG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB;;;;;OAKG;IACH,QAAQ,CAAC,EAAE,OAAO,CAAC;IAEnB;;;;;OAKG;IACH,IAAI,CAAC,EAAE,OAAO,CAAC;IAEf;;;;OAIG;IACH,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAE9B;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAE5B;;;;OAIG;IACH,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAEhC;;;;;OAKG;IACH,QAAQ,CAAC,EAAE,QAAQ,GAAG,QAAQ,CAAC,MAAM,QAAQ,CAAC,CAAC;IAE/C;;;;OAIG;IACH,cAAc,CAAC,EAAE,OAAO,CAAC;IAEzB;;;;OAIG;IACH,aAAa,CAAC,EAAE,OAAO,CAAC;CAC3B;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgEG;AACH,qBAAa,aAAa;IACtB;;OAEG;IACH,SAAS,CAAC,MAAM,CAAC,OAAO,EAAE,UAAU,CAiBlC;IAEF,SAAS,CAAC,MAAM,CAAC,YAAY,WAO3B;IAEF,SAAS,CAAC,MAAM,CAAC,YAAY,WAA8E;IAE3G,SAAS,CAAC,MAAM,CAAC,yBAAyB,EAAE,MAAM,EAAE,CAAM;IAE1D,SAAS,CAAC,MAAM,CAAC,QAAQ,EAAE,UAAU,CAanC;IAEF;;;OAGG;IACH,MAAM,CAAC,OAAO,mCAA0C;IAExD,SAAS,CAAC,OAAO,EAAG,GAAG,CAAC,MAAM,oBAAoB,EAAE,oBAAoB,CAAC,MAAM,oBAAoB,CAAC,CAAC,CAAC;IACtG,SAAS,CAAC,QAAQ,uBAA8B;IAEhD,gBAAgB;IAChB,MAAM,CAAC,YAAY,CAAC,EAAE,aAAa,CAAC;IAEpC,SAAgB,eAAe,qFAA0C;IAEzE;;OAEG;gBACS,OAAO,GAAE,oBAAyB;IAiB9C;;;;OAIG;IACH,GAAG,CAAC,CAAC,SAAS,MAAM,oBAAoB,EAAE,CAAC,SAAS,oBAAoB,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,YAAY,CAAC,EAAE,CAAC,GAAG,CAAC;IA2BzG,SAAS,CAAC,aAAa,CAAC,GAAG,EAAE,MAAM,oBAAoB,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO;IAoBzF;;;OAGG;IACH,GAAG,CAAC,GAAG,EAAE,MAAM,oBAAoB,EAAE,KAAK,CAAC,EAAE,GAAG,GAAG,IAAI;IAIvD;;;OAGG;IACH,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,oBAAoB,EAAE,KAAK,CAAC,EAAE,GAAG,GAAG,IAAI;IAI9D;;;;;;;;OAQG;IACH,gBAAgB,IAAI,aAAa;IASjC,eAAe,IAAI,YAAY;IAe/B;;;OAGG;IACH,mBAAmB,CAAC,OAAO,GAAE,oBAAyB,GAAG,aAAa;IAiBtE,gBAAgB,CAAC,MAAM,EAAE,aAAa,GAAG,IAAI;IAI7C,MAAM,CAAC,gBAAgB,CAAC,MAAM,EAAE,aAAa,GAAG,IAAI;IAIpD,eAAe,CAAC,MAAM,EAAE,YAAY,GAAG,IAAI;IAI3C;;OAEG;IACH,MAAM,CAAC,eAAe,IAAI,aAAa;IASvC;;OAEG;IACH,MAAM,CAAC,gBAAgB,IAAI,aAAa;IAIxC;;OAEG;IACH,MAAM,CAAC,eAAe,IAAI,YAAY;IAItC;;;OAGG;IACH,MAAM,CAAC,gBAAgB,IAAI,IAAI;IAI/B,SAAS,CAAC,YAAY,CAAC,OAAO,EAAE,oBAAoB;CAgBvD"}
|
package/configuration.js
CHANGED
|
@@ -70,7 +70,6 @@ import { entries } from './typedefs.js';
|
|
|
70
70
|
* `defaultBrowserPath` | `CRAWLEE_DEFAULT_BROWSER_PATH` | -
|
|
71
71
|
* `disableBrowserSandbox` | `CRAWLEE_DISABLE_BROWSER_SANDBOX` | -
|
|
72
72
|
* `availableMemoryRatio` | `CRAWLEE_AVAILABLE_MEMORY_RATIO` | `0.25`
|
|
73
|
-
* `systemInfoV2` | `CRAWLEE_SYSTEM_INFO_V2` | false
|
|
74
73
|
* `containerized | `CRAWLEE_CONTAINERIZED | -
|
|
75
74
|
*/
|
|
76
75
|
export class Configuration {
|
|
@@ -93,7 +92,6 @@ export class Configuration {
|
|
|
93
92
|
CRAWLEE_DISABLE_BROWSER_SANDBOX: 'disableBrowserSandbox',
|
|
94
93
|
CRAWLEE_LOG_LEVEL: 'logLevel',
|
|
95
94
|
CRAWLEE_PERSIST_STORAGE: 'persistStorage',
|
|
96
|
-
CRAWLEE_SYSTEM_INFO_V2: 'systemInfoV2',
|
|
97
95
|
CRAWLEE_CONTAINERIZED: 'containerized',
|
|
98
96
|
};
|
|
99
97
|
static BOOLEAN_VARS = [
|
|
@@ -102,7 +100,6 @@ export class Configuration {
|
|
|
102
100
|
'xvfb',
|
|
103
101
|
'disableBrowserSandbox',
|
|
104
102
|
'persistStorage',
|
|
105
|
-
'systemInfoV2',
|
|
106
103
|
'containerized',
|
|
107
104
|
];
|
|
108
105
|
static INTEGER_VARS = ['memoryMbytes', 'persistStateIntervalMillis', 'systemInfoIntervalMillis'];
|
|
@@ -120,7 +117,6 @@ export class Configuration {
|
|
|
120
117
|
persistStateIntervalMillis: 60_000,
|
|
121
118
|
systemInfoIntervalMillis: 1_000,
|
|
122
119
|
persistStorage: true,
|
|
123
|
-
systemInfoV2: false,
|
|
124
120
|
};
|
|
125
121
|
/**
|
|
126
122
|
* Provides access to the current-instance-scoped Configuration without passing it around in parameters.
|
package/configuration.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"configuration.js","sourceRoot":"","sources":["../src/configuration.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAGjC,OAAO,EAAE,aAAa,EAAE,MAAM,yBAAyB,CAAC;AAExD,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAE9C,OAAO,GAAG,EAAE,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAG3C,OAAO,EAAE,iBAAiB,EAAE,MAAM,iCAAiC,CAAC;AAEpE,OAAO,EAAoB,OAAO,EAAE,MAAM,eAAe,CAAC;
|
|
1
|
+
{"version":3,"file":"configuration.js","sourceRoot":"","sources":["../src/configuration.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAGjC,OAAO,EAAE,aAAa,EAAE,MAAM,yBAAyB,CAAC;AAExD,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAE9C,OAAO,GAAG,EAAE,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAG3C,OAAO,EAAE,iBAAiB,EAAE,MAAM,iCAAiC,CAAC;AAEpE,OAAO,EAAoB,OAAO,EAAE,MAAM,eAAe,CAAC;AAgK1D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgEG;AACH,MAAM,OAAO,aAAa;IACtB;;OAEG;IACO,MAAM,CAAC,OAAO,GAAe;QACnC,8BAA8B,EAAE,sBAAsB;QACtD,sBAAsB,EAAE,cAAc;QACtC,qBAAqB,EAAE,cAAc;QACrC,0BAA0B,EAAE,kBAAkB;QAC9C,kCAAkC,EAAE,wBAAwB;QAC5D,gCAAgC,EAAE,uBAAuB;QACzD,iBAAiB,EAAE,UAAU;QAC7B,qCAAqC,EAAE,4BAA4B;QACnE,gBAAgB,EAAE,UAAU;QAC5B,YAAY,EAAE,MAAM;QACpB,8BAA8B,EAAE,sBAAsB;QACtD,4BAA4B,EAAE,oBAAoB;QAClD,+BAA+B,EAAE,uBAAuB;QACxD,iBAAiB,EAAE,UAAU;QAC7B,uBAAuB,EAAE,gBAAgB;QACzC,qBAAqB,EAAE,eAAe;KACzC,CAAC;IAEQ,MAAM,CAAC,YAAY,GAAG;QAC5B,cAAc;QACd,UAAU;QACV,MAAM;QACN,uBAAuB;QACvB,gBAAgB;QAChB,eAAe;KAClB,CAAC;IAEQ,MAAM,CAAC,YAAY,GAAG,CAAC,cAAc,EAAE,4BAA4B,EAAE,0BAA0B,CAAC,CAAC;IAEjG,MAAM,CAAC,yBAAyB,GAAa,EAAE,CAAC;IAEhD,MAAM,CAAC,QAAQ,GAAe;QACpC,sBAAsB,EAAE,SAAS;QACjC,gBAAgB,EAAE,SAAS;QAC3B,qBAAqB,EAAE,SAAS;QAChC,QAAQ,EAAE,OAAO;QACjB,eAAe,EAAE,IAAI;QACrB,oBAAoB,EAAE,IAAI;QAC1B,oBAAoB,EAAE,EAAE;QACxB,YAAY,EAAE,IAAI;QAClB,QAAQ,EAAE,IAAI;QACd,0BAA0B,EAAE,MAAM;QAClC,wBAAwB,EAAE,KAAK;QAC/B,cAAc,EAAE,IAAI;KACvB,CAAC;IAEF;;;OAGG;IACH,MAAM,CAAC,OAAO,GAAG,IAAI,iBAAiB,EAAiB,CAAC;IAE9C,OAAO,CAAqF;IAC5F,QAAQ,GAAG,IAAI,GAAG,EAAmB,CAAC;IAEhD,gBAAgB;IAChB,MAAM,CAAC,YAAY,CAAiB;IAEpB,eAAe,GAAG,IAAI,GAAG,EAA+B,CAAC;IAEzE;;OAEG;IACH,YAAY,UAAgC,EAAE;QAC1C,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;QAE3B,oEAAoE;QACpE,YAAY,CAAC,mBAAmB,GAAG,EAAE,CAAC;QAEtC,6DAA6D;QAC7D,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QAEtC,IAAI,QAAQ,EAAE,CAAC;YACX,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC,QAAQ,CAAC;gBACpC,CAAC,CAAC,CAAC,QAAQ;gBACX,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAyB,CAAC,CAAC;YACtE,GAAG,CAAC,QAAQ,CAAC,KAAiB,CAAC,CAAC;QACpC,CAAC;IACL,CAAC;IAED;;;;OAIG;IACH,GAAG,CAA0E,GAAM,EAAE,YAAgB;QACjG,iHAAiH;QACjH,IAAI,QAA4B,CAAC;QAEjC,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,OAAO,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,CAAC;YAClD,IAAI,GAAG,KAAK,CAAC,EAAE,CAAC;gBACZ,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,CAAW,CAAC,CAAC;gBAEpC,IAAI,QAAQ,EAAE,CAAC;oBACX,MAAM;gBACV,CAAC;YACL,CAAC;QACL,CAAC;QAED,IAAI,QAAQ,IAAI,IAAI,EAAE,CAAC;YACnB,OAAO,IAAI,CAAC,aAAa,CAAC,GAAG,EAAE,QAAQ,CAAM,CAAC;QAClD,CAAC;QAED,+BAA+B;QAC/B,IAAI,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YACxB,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAM,CAAC;QACtC,CAAC;QAED,uBAAuB;QACvB,OAAO,CAAC,YAAY,IAAI,aAAa,CAAC,QAAQ,CAAC,GAA0C,CAAC,IAAI,QAAQ,CAAM,CAAC;IACjH,CAAC;IAES,aAAa,CAAC,GAA+B,EAAE,KAAgC;QACrF,IAAI,aAAa,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YAC3C,OAAO,CAAC,KAAK,CAAC;QAClB,CAAC;QAED,IAAI,aAAa,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YAC3C,wDAAwD;YACxD,OAAO,CAAC,CAAC,GAAG,EAAE,OAAO,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;QACrE,CAAC;QAED,IAAI,aAAa,CAAC,yBAAyB,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YACxD,IAAI,CAAC,KAAK;gBAAE,OAAO,EAAE,CAAC;YACtB,OAAO,MAAM,CAAC,KAAK,CAAC;iBACf,KAAK,CAAC,GAAG,CAAC;iBACV,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QAC9B,CAAC;QAED,OAAO,KAAK,CAAC;IACjB,CAAC;IAED;;;OAGG;IACH,GAAG,CAAC,GAA+B,EAAE,KAAW;QAC5C,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;IACjC,CAAC;IAED;;;OAGG;IACH,MAAM,CAAC,GAAG,CAAC,GAA+B,EAAE,KAAW;QACnD,IAAI,CAAC,eAAe,EAAE,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;IAC3C,CAAC;IAED;;;;;;;;OAQG;IACH,gBAAgB;QACZ,IAAI,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,EAAE,CAAC;YACpC,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAkB,CAAC;QAC9D,CAAC;QAED,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAe,CAAC;QACvE,OAAO,IAAI,CAAC,mBAAmB,CAAC,OAAO,CAAC,CAAC;IAC7C,CAAC;IAED,eAAe;QACX,IAAI,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,EAAE,CAAC;YACnC,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAiB,CAAC;QAC5D,CAAC;QAED,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,cAAc,CAAC,EAAE,CAAC;YACpC,OAAO,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,cAAc,CAAiB,CAAC;QAC7D,CAAC;QAED,MAAM,YAAY,GAAG,IAAI,iBAAiB,CAAC,IAAI,CAAC,CAAC;QACjD,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,cAAc,EAAE,YAAY,CAAC,CAAC;QAEhD,OAAO,YAAY,CAAC;IACxB,CAAC;IAED;;;OAGG;IACH,mBAAmB,CAAC,UAAgC,EAAE;QAClD,MAAM,QAAQ,GAAG,iBAAiB,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,CAAC;QAE5D,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC9B,OAAO,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAkB,CAAC;QACxD,CAAC;QAED,MAAM,OAAO,GAAG,IAAI,aAAa,CAAC;YAC9B,cAAc,EAAE,IAAI,CAAC,GAAG,CAAC,gBAAgB,CAAC;YAC1C,uEAAuE;YACvE,GAAG,OAAO;SACb,CAAC,CAAC;QACH,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAErC,OAAO,OAAO,CAAC;IACnB,CAAC;IAED,gBAAgB,CAAC,MAAqB;QAClC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,EAAE,MAAM,CAAC,CAAC;IAC9C,CAAC;IAED,MAAM,CAAC,gBAAgB,CAAC,MAAqB;QACzC,IAAI,CAAC,eAAe,EAAE,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC;IACpD,CAAC;IAED,eAAe,CAAC,MAAoB;QAChC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,MAAM,CAAC,CAAC;IAC7C,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,eAAe;QAClB,IAAI,aAAa,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC;YACnC,OAAO,aAAa,CAAC,OAAO,CAAC,QAAQ,EAAG,CAAC;QAC7C,CAAC;QAED,aAAa,CAAC,YAAY,KAAK,IAAI,aAAa,EAAE,CAAC;QACnD,OAAO,aAAa,CAAC,YAAY,CAAC;IACtC,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,gBAAgB;QACnB,OAAO,IAAI,CAAC,eAAe,EAAE,CAAC,gBAAgB,EAAE,CAAC;IACrD,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,eAAe;QAClB,OAAO,IAAI,CAAC,eAAe,EAAE,CAAC,eAAe,EAAE,CAAC;IACpD,CAAC;IAED;;;OAGG;IACH,MAAM,CAAC,gBAAgB;QACnB,OAAO,IAAI,CAAC,YAAY,CAAC;IAC7B,CAAC;IAES,YAAY,CAAC,OAA6B;QAChD,8DAA8D;QAC9D,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,cAAc,CAAC,CAAC;QAEjD,IAAI,cAAc,CAAC,IAAI,CAAC,EAAE,CAAC;YACvB,IAAI,CAAC;gBACD,MAAM,IAAI,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;gBAChC,MAAM,qBAAqB,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;gBAC1D,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,qBAAqB,CAAC,CAAC;YAClD,CAAC;YAAC,MAAM,CAAC;gBACL,SAAS;YACb,CAAC;QACL,CAAC;QAED,IAAI,CAAC,OAAO,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC;IAC7C,CAAC"}
|
package/cookie_utils.d.ts
CHANGED
|
@@ -7,7 +7,7 @@ export interface ResponseLike {
|
|
|
7
7
|
/**
|
|
8
8
|
* @internal
|
|
9
9
|
*/
|
|
10
|
-
export declare function getCookiesFromResponse(response:
|
|
10
|
+
export declare function getCookiesFromResponse(response: Response): Cookie[];
|
|
11
11
|
/**
|
|
12
12
|
* Calculate cookie expiration date
|
|
13
13
|
* @param maxAgeSecs
|
package/cookie_utils.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cookie_utils.d.ts","sourceRoot":"","sources":["../src/cookie_utils.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,IAAI,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC7D,OAAO,EAAE,MAAM,EAAa,MAAM,cAAc,CAAC;AAKjD,MAAM,WAAW,YAAY;IACzB,GAAG,CAAC,EAAE,MAAM,GAAG,CAAC,MAAM,MAAM,CAAC,CAAC;IAC9B,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,SAAS,CAAC,GAAG,CAAC,MAAM,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,SAAS,CAAC,CAAC,CAAC;CACnH;AAED;;GAEG;AACH,wBAAgB,sBAAsB,CAAC,QAAQ,EAAE,
|
|
1
|
+
{"version":3,"file":"cookie_utils.d.ts","sourceRoot":"","sources":["../src/cookie_utils.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,IAAI,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC7D,OAAO,EAAE,MAAM,EAAa,MAAM,cAAc,CAAC;AAKjD,MAAM,WAAW,YAAY;IACzB,GAAG,CAAC,EAAE,MAAM,GAAG,CAAC,MAAM,MAAM,CAAC,CAAC;IAC9B,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,SAAS,CAAC,GAAG,CAAC,MAAM,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,SAAS,CAAC,CAAC,CAAC;CACnH;AAED;;GAEG;AACH,wBAAgB,sBAAsB,CAAC,QAAQ,EAAE,QAAQ,GAAG,MAAM,EAAE,CASnE;AAED;;;;;GAKG;AACH,wBAAgB,8BAA8B,CAAC,UAAU,EAAE,MAAM,QAEhE;AAED;;;;;GAKG;AACH,wBAAgB,8BAA8B,CAAC,WAAW,EAAE,MAAM,GAAG,YAAY,CAehF;AAED;;;;GAIG;AACH,wBAAgB,8BAA8B,CAAC,YAAY,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,UAiB5F;AAED;;;;GAIG;AACH,wBAAgB,yBAAyB,CAAC,YAAY,EAAE,MAAM,uBAQ7D;AAED;;;;GAIG;AACH,wBAAgB,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,GAAG,MAAM,CA8BzE"}
|
package/cookie_utils.js
CHANGED
|
@@ -5,15 +5,13 @@ import { CookieParseError } from './session_pool/errors.js';
|
|
|
5
5
|
* @internal
|
|
6
6
|
*/
|
|
7
7
|
export function getCookiesFromResponse(response) {
|
|
8
|
-
const headers =
|
|
9
|
-
const
|
|
8
|
+
const headers = response.headers;
|
|
9
|
+
const cookieHeaders = headers.getSetCookie();
|
|
10
10
|
try {
|
|
11
|
-
return
|
|
12
|
-
? cookieHeader.map((cookie) => Cookie.parse(cookie))
|
|
13
|
-
: [Cookie.parse(cookieHeader)];
|
|
11
|
+
return cookieHeaders.map((cookie) => Cookie.parse(cookie));
|
|
14
12
|
}
|
|
15
13
|
catch (e) {
|
|
16
|
-
throw new CookieParseError(
|
|
14
|
+
throw new CookieParseError(cookieHeaders);
|
|
17
15
|
}
|
|
18
16
|
}
|
|
19
17
|
/**
|
package/cookie_utils.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cookie_utils.js","sourceRoot":"","sources":["../src/cookie_utils.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAEjD,OAAO,EAAE,GAAG,EAAE,MAAM,UAAU,CAAC;AAC/B,OAAO,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AAO5D;;GAEG;AACH,MAAM,UAAU,sBAAsB,CAAC,
|
|
1
|
+
{"version":3,"file":"cookie_utils.js","sourceRoot":"","sources":["../src/cookie_utils.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAEjD,OAAO,EAAE,GAAG,EAAE,MAAM,UAAU,CAAC;AAC/B,OAAO,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AAO5D;;GAEG;AACH,MAAM,UAAU,sBAAsB,CAAC,QAAkB;IACrD,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAC;IACjC,MAAM,aAAa,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;IAE7C,IAAI,CAAC;QACD,OAAO,aAAa,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAE,CAAC,CAAC;IAChE,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACT,MAAM,IAAI,gBAAgB,CAAC,aAAa,CAAC,CAAC;IAC9C,CAAC;AACL,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,8BAA8B,CAAC,UAAkB;IAC7D,OAAO,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,UAAU,GAAG,IAAI,CAAC,CAAC;AACpD,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,8BAA8B,CAAC,WAAmB;IAC9D,OAAO;QACH,IAAI,EAAE,WAAW,CAAC,GAAG;QACrB,KAAK,EAAE,WAAW,CAAC,KAAK;QACxB,iFAAiF;QACjF,6FAA6F;QAC7F,OAAO,EACH,WAAW,CAAC,OAAO,IAAI,IAAI,IAAI,WAAW,CAAC,OAAO,KAAK,UAAU;YAC7D,CAAC,CAAC,SAAS;YACX,CAAC,CAAC,WAAW,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,IAAI;QAC9C,MAAM,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,WAAW,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,GAAG,WAAW,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,SAAS;QAClG,IAAI,EAAE,WAAW,CAAC,IAAI,IAAI,SAAS;QACnC,MAAM,EAAE,WAAW,CAAC,MAAM;QAC1B,QAAQ,EAAE,WAAW,CAAC,QAAQ;KACjC,CAAC;AACN,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,8BAA8B,CAAC,YAA0B,EAAE,UAAkB;IACzF,MAAM,cAAc,GAAG,YAAY,CAAC,OAAO,IAAI,OAAO,YAAY,CAAC,OAAO,KAAK,QAAQ,IAAI,YAAY,CAAC,OAAO,GAAG,CAAC,CAAC;IACpH,MAAM,OAAO,GAAG,cAAc;QAC1B,CAAC,CAAC,IAAI,IAAI,CAAC,YAAY,CAAC,OAAQ,GAAG,IAAI,CAAC;QACxC,CAAC,CAAC,8BAA8B,CAAC,UAAU,CAAC,CAAC;IACjD,MAAM,mBAAmB,GAAG,YAAY,CAAC,MAAM,EAAE,UAAU,EAAE,CAAC,GAAG,CAAC,CAAC;IACnE,MAAM,MAAM,GAAG,mBAAmB,CAAC,CAAC,CAAC,YAAY,CAAC,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,MAAM,CAAC;IAC3F,OAAO,IAAI,MAAM,CAAC;QACd,GAAG,EAAE,YAAY,CAAC,IAAI;QACtB,KAAK,EAAE,YAAY,CAAC,KAAK;QACzB,OAAO;QACP,MAAM;QACN,IAAI,EAAE,YAAY,CAAC,IAAI;QACvB,MAAM,EAAE,YAAY,CAAC,MAAM;QAC3B,QAAQ,EAAE,YAAY,CAAC,QAAQ;QAC/B,QAAQ,EAAE,CAAC,mBAAmB;KACjC,CAAC,CAAC;AACP,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,yBAAyB,CAAC,YAAoB;IAC1D,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;IAE1C,IAAI,MAAM,EAAE,CAAC;QACT,OAAO,8BAA8B,CAAC,MAAM,CAAC,CAAC;IAClD,CAAC;IAED,OAAO,IAAI,CAAC;AAChB,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,YAAY,CAAC,GAAW,EAAE,aAAuB;IAC7D,MAAM,GAAG,GAAG,IAAI,SAAS,EAAE,CAAC;IAE5B,uBAAuB;IACvB,KAAK,MAAM,kBAAkB,IAAI,aAAa,EAAE,CAAC;QAC7C,uBAAuB;QACvB,IAAI,CAAC,kBAAkB;YAAE,SAAS;QAElC,MAAM,OAAO,GAAG,kBAAkB,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAElD,KAAK,MAAM,YAAY,IAAI,OAAO,EAAE,CAAC;YACjC,sBAAsB;YACtB,IAAI,CAAC,YAAY;gBAAE,SAAS;YAE5B,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,YAAY,CAAE,CAAC;YAC3C,MAAM,gBAAgB,GAAG,GAAG,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE;gBACxD,OAAO,MAAM,CAAC,GAAG,KAAK,CAAC,CAAC,GAAG,IAAI,MAAM,CAAC,GAAG,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC;YACpF,CAAC,CAAC,CAAC;YAEH,IAAI,gBAAgB,EAAE,CAAC;gBACnB,GAAG,CAAC,WAAW,CACX,2DAA2D,MAAM,CAAC,GAAG,UAAU,gBAAgB,CAAC,GAAG,GAAG,CACzG,CAAC;YACN,CAAC;YAED,GAAG,CAAC,aAAa,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QACnC,CAAC;IACL,CAAC;IAED,OAAO,GAAG,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAC;AACxC,CAAC"}
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
import type { Awaitable } from '@crawlee/types';
|
|
2
|
+
/**
|
|
3
|
+
* Represents a middleware step in the context pipeline.
|
|
4
|
+
*
|
|
5
|
+
* @template TCrawlingContext - The input context type for this middleware
|
|
6
|
+
* @template TCrawlingContextExtension - The enhanced output context type
|
|
7
|
+
*/
|
|
8
|
+
export interface ContextMiddleware<TCrawlingContext, TCrawlingContextExtension> {
|
|
9
|
+
/** The main middleware function that enhances the context */
|
|
10
|
+
action: (context: TCrawlingContext) => Awaitable<TCrawlingContextExtension>;
|
|
11
|
+
/** Optional cleanup function called after the consumer finishes or fails */
|
|
12
|
+
cleanup?: (context: TCrawlingContext & TCrawlingContextExtension, error?: unknown) => Awaitable<void>;
|
|
13
|
+
}
|
|
14
|
+
/**
|
|
15
|
+
* Encapsulates the logic of gradually enhancing the crawling context with additional information and utilities.
|
|
16
|
+
*
|
|
17
|
+
* The enhancement is done by a chain of middlewares that are added to the pipeline after its creation.
|
|
18
|
+
* This class provides a type-safe way to build a pipeline of context transformations where each step
|
|
19
|
+
* can enhance the context with additional properties or utilities.
|
|
20
|
+
*
|
|
21
|
+
* @template TContextBase - The base context type that serves as the starting point
|
|
22
|
+
* @template TCrawlingContext - The final context type after all middleware transformations
|
|
23
|
+
*/
|
|
24
|
+
export declare abstract class ContextPipeline<TContextBase, TCrawlingContext extends TContextBase> {
|
|
25
|
+
/**
|
|
26
|
+
* Creates a new empty context pipeline.
|
|
27
|
+
*
|
|
28
|
+
* @template TContextBase - The base context type for the pipeline
|
|
29
|
+
* @returns A new ContextPipeline instance with no transformations
|
|
30
|
+
*/
|
|
31
|
+
static create<TContextBase>(): ContextPipeline<TContextBase, TContextBase>;
|
|
32
|
+
/**
|
|
33
|
+
* Adds a middleware to the pipeline, creating a new pipeline instance.
|
|
34
|
+
*
|
|
35
|
+
* This method provides a fluent interface for building context transformation pipelines.
|
|
36
|
+
* Each middleware can enhance the context with additional properties or utilities.
|
|
37
|
+
*
|
|
38
|
+
* @template TCrawlingContextExtension - The enhanced context type produced by this middleware
|
|
39
|
+
* @param middleware - The middleware to add to the pipeline
|
|
40
|
+
* @returns A new ContextPipeline instance with the added middleware
|
|
41
|
+
*/
|
|
42
|
+
abstract compose<TCrawlingContextExtension>(middleware: ContextMiddleware<TCrawlingContext, TCrawlingContextExtension>): ContextPipeline<TContextBase, TCrawlingContext & TCrawlingContextExtension>;
|
|
43
|
+
/**
|
|
44
|
+
* Executes the middleware pipeline and passes the final context to a consumer function.
|
|
45
|
+
*
|
|
46
|
+
* This method runs the crawling context through the entire middleware chain, enhancing it
|
|
47
|
+
* at each step, and then passes the final enhanced context to the provided consumer function.
|
|
48
|
+
* Proper cleanup is performed even if exceptions occur during processing.
|
|
49
|
+
*
|
|
50
|
+
* @param crawlingContext - The initial context to process through the pipeline
|
|
51
|
+
* @param finalContextConsumer - The function that will receive the final enhanced context
|
|
52
|
+
*
|
|
53
|
+
* @throws {ContextPipelineInitializationError} When a middleware fails during initialization
|
|
54
|
+
* @throws {ContextPipelineInterruptedError} When the pipeline is intentionally interrupted during initialization
|
|
55
|
+
* @throws {RequestHandlerError} When the final context consumer throws an exception
|
|
56
|
+
* @throws {ContextPipelineCleanupError} When cleanup operations fail
|
|
57
|
+
* @throws {SessionError} Session errors are re-thrown as-is for special handling
|
|
58
|
+
*/
|
|
59
|
+
abstract call(crawlingContext: TContextBase, finalContextConsumer: (finalContext: TCrawlingContext) => Awaitable<unknown>): Promise<void>;
|
|
60
|
+
}
|
|
61
|
+
//# sourceMappingURL=context_pipeline.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"context_pipeline.d.ts","sourceRoot":"","sources":["../../src/crawlers/context_pipeline.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAUhD;;;;;GAKG;AACH,MAAM,WAAW,iBAAiB,CAAC,gBAAgB,EAAE,yBAAyB;IAC1E,6DAA6D;IAC7D,MAAM,EAAE,CAAC,OAAO,EAAE,gBAAgB,KAAK,SAAS,CAAC,yBAAyB,CAAC,CAAC;IAC5E,4EAA4E;IAC5E,OAAO,CAAC,EAAE,CAAC,OAAO,EAAE,gBAAgB,GAAG,yBAAyB,EAAE,KAAK,CAAC,EAAE,OAAO,KAAK,SAAS,CAAC,IAAI,CAAC,CAAC;CACzG;AAED;;;;;;;;;GASG;AACH,8BAAsB,eAAe,CAAC,YAAY,EAAE,gBAAgB,SAAS,YAAY;IACrF;;;;;OAKG;IACH,MAAM,CAAC,MAAM,CAAC,YAAY,KAAK,eAAe,CAAC,YAAY,EAAE,YAAY,CAAC;IAI1E;;;;;;;;;OASG;IACH,QAAQ,CAAC,OAAO,CAAC,yBAAyB,EACtC,UAAU,EAAE,iBAAiB,CAAC,gBAAgB,EAAE,yBAAyB,CAAC,GAC3E,eAAe,CAAC,YAAY,EAAE,gBAAgB,GAAG,yBAAyB,CAAC;IAE9E;;;;;;;;;;;;;;;OAeG;IACH,QAAQ,CAAC,IAAI,CACT,eAAe,EAAE,YAAY,EAC7B,oBAAoB,EAAE,CAAC,YAAY,EAAE,gBAAgB,KAAK,SAAS,CAAC,OAAO,CAAC,GAC7E,OAAO,CAAC,IAAI,CAAC;CACnB"}
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
import { ContextPipelineCleanupError, ContextPipelineInitializationError, ContextPipelineInterruptedError, RequestHandlerError, SessionError, } from '../errors.js';
|
|
2
|
+
/**
|
|
3
|
+
* Encapsulates the logic of gradually enhancing the crawling context with additional information and utilities.
|
|
4
|
+
*
|
|
5
|
+
* The enhancement is done by a chain of middlewares that are added to the pipeline after its creation.
|
|
6
|
+
* This class provides a type-safe way to build a pipeline of context transformations where each step
|
|
7
|
+
* can enhance the context with additional properties or utilities.
|
|
8
|
+
*
|
|
9
|
+
* @template TContextBase - The base context type that serves as the starting point
|
|
10
|
+
* @template TCrawlingContext - The final context type after all middleware transformations
|
|
11
|
+
*/
|
|
12
|
+
export class ContextPipeline {
|
|
13
|
+
/**
|
|
14
|
+
* Creates a new empty context pipeline.
|
|
15
|
+
*
|
|
16
|
+
* @template TContextBase - The base context type for the pipeline
|
|
17
|
+
* @returns A new ContextPipeline instance with no transformations
|
|
18
|
+
*/
|
|
19
|
+
static create() {
|
|
20
|
+
return new ContextPipelineImpl({ action: async (context) => context });
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* Implementation of the `ContextPipeline` logic. This hides implementation details such as the `middleware` and `parent`
|
|
25
|
+
* properties from the `ContextPipeline` interface, making type checking more reliable.
|
|
26
|
+
*/
|
|
27
|
+
class ContextPipelineImpl extends ContextPipeline {
|
|
28
|
+
middleware;
|
|
29
|
+
parent;
|
|
30
|
+
constructor(middleware, parent) {
|
|
31
|
+
super();
|
|
32
|
+
this.middleware = middleware;
|
|
33
|
+
this.parent = parent;
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* @inheritdoc
|
|
37
|
+
*/
|
|
38
|
+
compose(middleware) {
|
|
39
|
+
return new ContextPipelineImpl(middleware, this);
|
|
40
|
+
}
|
|
41
|
+
*middlewareChain() {
|
|
42
|
+
let step = this;
|
|
43
|
+
while (step !== undefined) {
|
|
44
|
+
yield step.middleware;
|
|
45
|
+
step = step.parent;
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
/**
|
|
49
|
+
* @inheritdoc
|
|
50
|
+
*/
|
|
51
|
+
async call(crawlingContext, finalContextConsumer) {
|
|
52
|
+
const middlewares = Array.from(this.middlewareChain()).reverse();
|
|
53
|
+
const cleanupStack = [];
|
|
54
|
+
let consumerException;
|
|
55
|
+
try {
|
|
56
|
+
for (const { action, cleanup } of middlewares) {
|
|
57
|
+
try {
|
|
58
|
+
const contextExtension = await action(crawlingContext);
|
|
59
|
+
Object.defineProperties(crawlingContext, Object.getOwnPropertyDescriptors(contextExtension));
|
|
60
|
+
if (cleanup) {
|
|
61
|
+
cleanupStack.push(cleanup);
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
catch (exception) {
|
|
65
|
+
if (exception instanceof SessionError) {
|
|
66
|
+
throw exception; // Session errors are re-thrown as-is
|
|
67
|
+
}
|
|
68
|
+
if (exception instanceof ContextPipelineInterruptedError) {
|
|
69
|
+
throw exception;
|
|
70
|
+
}
|
|
71
|
+
throw new ContextPipelineInitializationError(exception);
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
try {
|
|
75
|
+
await finalContextConsumer(crawlingContext);
|
|
76
|
+
}
|
|
77
|
+
catch (exception) {
|
|
78
|
+
if (exception instanceof SessionError) {
|
|
79
|
+
consumerException = exception;
|
|
80
|
+
throw exception; // Session errors are re-thrown as-is
|
|
81
|
+
}
|
|
82
|
+
consumerException = exception;
|
|
83
|
+
throw new RequestHandlerError(exception);
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
finally {
|
|
87
|
+
try {
|
|
88
|
+
for (const cleanup of cleanupStack.reverse()) {
|
|
89
|
+
await cleanup(crawlingContext, consumerException);
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
catch (exception) {
|
|
93
|
+
// eslint-disable-next-line no-unsafe-finally
|
|
94
|
+
throw new ContextPipelineCleanupError(exception);
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
//# sourceMappingURL=context_pipeline.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"context_pipeline.js","sourceRoot":"","sources":["../../src/crawlers/context_pipeline.ts"],"names":[],"mappings":"AAEA,OAAO,EACH,2BAA2B,EAC3B,kCAAkC,EAClC,+BAA+B,EAC/B,mBAAmB,EACnB,YAAY,GACf,MAAM,cAAc,CAAC;AAetB;;;;;;;;;GASG;AACH,MAAM,OAAgB,eAAe;IACjC;;;;;OAKG;IACH,MAAM,CAAC,MAAM;QACT,OAAO,IAAI,mBAAmB,CAA6B,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC;IACvG,CAAC;CAoCJ;AAED;;;GAGG;AACH,MAAM,mBAAyE,SAAQ,eAGtF;IAEe;IACA;IAFZ,YACY,UAA6D,EAC7D,MAAwD;QAEhE,KAAK,EAAE,CAAC;QAHA,eAAU,GAAV,UAAU,CAAmD;QAC7D,WAAM,GAAN,MAAM,CAAkD;IAGpE,CAAC;IAED;;OAEG;IACH,OAAO,CACH,UAA0E;QAE1E,OAAO,IAAI,mBAAmB,CAC1B,UAAiB,EACjB,IAAW,CACd,CAAC;IACN,CAAC;IAEO,CAAC,eAAe;QACpB,IAAI,IAAI,GAAgE,IAAW,CAAC;QAEpF,OAAO,IAAI,KAAK,SAAS,EAAE,CAAC;YACxB,MAAM,IAAI,CAAC,UAAU,CAAC;YACtB,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC;QACvB,CAAC;IACL,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,IAAI,CACN,eAA6B,EAC7B,oBAA0E;QAE1E,MAAM,WAAW,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC;QACjE,MAAM,YAAY,GAAG,EAAE,CAAC;QACxB,IAAI,iBAAsC,CAAC;QAE3C,IAAI,CAAC;YACD,KAAK,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,WAAW,EAAE,CAAC;gBAC5C,IAAI,CAAC;oBACD,MAAM,gBAAgB,GAAG,MAAM,MAAM,CAAC,eAAe,CAAC,CAAC;oBACvD,MAAM,CAAC,gBAAgB,CAAC,eAAe,EAAE,MAAM,CAAC,yBAAyB,CAAC,gBAAgB,CAAC,CAAC,CAAC;oBAE7F,IAAI,OAAO,EAAE,CAAC;wBACV,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;oBAC/B,CAAC;gBACL,CAAC;gBAAC,OAAO,SAAkB,EAAE,CAAC;oBAC1B,IAAI,SAAS,YAAY,YAAY,EAAE,CAAC;wBACpC,MAAM,SAAS,CAAC,CAAC,qCAAqC;oBAC1D,CAAC;oBACD,IAAI,SAAS,YAAY,+BAA+B,EAAE,CAAC;wBACvD,MAAM,SAAS,CAAC;oBACpB,CAAC;oBAED,MAAM,IAAI,kCAAkC,CAAC,SAAS,CAAC,CAAC;gBAC5D,CAAC;YACL,CAAC;YAED,IAAI,CAAC;gBACD,MAAM,oBAAoB,CAAC,eAAmC,CAAC,CAAC;YACpE,CAAC;YAAC,OAAO,SAAkB,EAAE,CAAC;gBAC1B,IAAI,SAAS,YAAY,YAAY,EAAE,CAAC;oBACpC,iBAAiB,GAAG,SAAS,CAAC;oBAC9B,MAAM,SAAS,CAAC,CAAC,qCAAqC;gBAC1D,CAAC;gBACD,iBAAiB,GAAG,SAAS,CAAC;gBAC9B,MAAM,IAAI,mBAAmB,CAAC,SAAS,CAAC,CAAC;YAC7C,CAAC;QACL,CAAC;gBAAS,CAAC;YACP,IAAI,CAAC;gBACD,KAAK,MAAM,OAAO,IAAI,YAAY,CAAC,OAAO,EAAE,EAAE,CAAC;oBAC3C,MAAM,OAAO,CAAC,eAAe,EAAE,iBAAiB,CAAC,CAAC;gBACtD,CAAC;YACL,CAAC;YAAC,OAAO,SAAkB,EAAE,CAAC;gBAC1B,6CAA6C;gBAC7C,MAAM,IAAI,2BAA2B,CAAC,SAAS,CAAC,CAAC;YACrD,CAAC;QACL,CAAC;IACL,CAAC;CACJ"}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type { Dictionary } from '@crawlee/types';
|
|
2
2
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
3
|
-
import type { OptionsInit
|
|
4
|
-
import type { ReadonlyDeep } from 'type-fest';
|
|
3
|
+
import type { OptionsInit } from 'got-scraping';
|
|
4
|
+
import type { ReadonlyDeep, SetRequired } from 'type-fest';
|
|
5
5
|
import type { Configuration } from '../configuration.js';
|
|
6
6
|
import type { EnqueueLinksOptions } from '../enqueue_links/enqueue_links.js';
|
|
7
7
|
import type { Log } from '../log.js';
|
|
@@ -22,7 +22,7 @@ export type LoadedRequest<R extends Request> = WithRequired<R, 'id' | 'loadedUrl
|
|
|
22
22
|
export type LoadedContext<Context extends RestrictedCrawlingContext> = IsAny<Context> extends true ? Context : {
|
|
23
23
|
request: LoadedRequest<Context['request']>;
|
|
24
24
|
} & Omit<Context, 'request'>;
|
|
25
|
-
export interface RestrictedCrawlingContext<UserData extends Dictionary = Dictionary>
|
|
25
|
+
export interface RestrictedCrawlingContext<UserData extends Dictionary = Dictionary> {
|
|
26
26
|
id: string;
|
|
27
27
|
session?: Session;
|
|
28
28
|
/**
|
|
@@ -66,7 +66,7 @@ export interface RestrictedCrawlingContext<UserData extends Dictionary = Diction
|
|
|
66
66
|
*
|
|
67
67
|
* @param [options] All `enqueueLinks()` parameters are passed via an options object.
|
|
68
68
|
*/
|
|
69
|
-
enqueueLinks: (options
|
|
69
|
+
enqueueLinks: (options: ReadonlyDeep<Omit<SetRequired<EnqueueLinksOptions, 'urls'>, 'requestQueue' | 'robotsTxtFile'>>) => Promise<unknown>;
|
|
70
70
|
/**
|
|
71
71
|
* Add requests directly to the request queue.
|
|
72
72
|
*
|
|
@@ -87,8 +87,7 @@ export interface RestrictedCrawlingContext<UserData extends Dictionary = Diction
|
|
|
87
87
|
*/
|
|
88
88
|
log: Log;
|
|
89
89
|
}
|
|
90
|
-
export interface CrawlingContext<
|
|
91
|
-
crawler: Crawler;
|
|
90
|
+
export interface CrawlingContext<UserData extends Dictionary = Dictionary> extends RestrictedCrawlingContext<UserData> {
|
|
92
91
|
/**
|
|
93
92
|
* This function automatically finds and enqueues links from the current page, adding them to the {@link RequestQueue}
|
|
94
93
|
* currently used by the crawler.
|
|
@@ -114,11 +113,7 @@ export interface CrawlingContext<Crawler = unknown, UserData extends Dictionary
|
|
|
114
113
|
* @param [options] All `enqueueLinks()` parameters are passed via an options object.
|
|
115
114
|
* @returns Promise that resolves to {@link BatchAddRequestsResult} object.
|
|
116
115
|
*/
|
|
117
|
-
enqueueLinks(options
|
|
118
|
-
/**
|
|
119
|
-
* Get a key-value store with given name or id, or the default one for the crawler.
|
|
120
|
-
*/
|
|
121
|
-
getKeyValueStore: (idOrName?: string) => Promise<KeyValueStore>;
|
|
116
|
+
enqueueLinks(options: ReadonlyDeep<Omit<SetRequired<EnqueueLinksOptions, 'urls'>, 'requestQueue' | 'robotsTxtFile'>> & Pick<EnqueueLinksOptions, 'requestQueue' | 'robotsTxtFile'>): Promise<unknown>;
|
|
122
117
|
/**
|
|
123
118
|
* Fires HTTP request via [`got-scraping`](https://crawlee.dev/js/docs/guides/got-scraping), allowing to override the request
|
|
124
119
|
* options on the fly.
|
|
@@ -136,7 +131,11 @@ export interface CrawlingContext<Crawler = unknown, UserData extends Dictionary
|
|
|
136
131
|
* },
|
|
137
132
|
* ```
|
|
138
133
|
*/
|
|
139
|
-
sendRequest
|
|
134
|
+
sendRequest(overrideOptions?: Partial<OptionsInit>): Promise<Response>;
|
|
135
|
+
/**
|
|
136
|
+
* Register a function to be called at the very end of the request handling process. This is useful for resources that should be accessible to error handlers, for instance.
|
|
137
|
+
*/
|
|
138
|
+
registerDeferredCleanup(cleanup: () => Promise<unknown>): void;
|
|
140
139
|
}
|
|
141
140
|
/**
|
|
142
141
|
* A partial implementation of {@link RestrictedCrawlingContext} that stores parameters of calls to context methods for later inspection.
|
|
@@ -149,7 +148,6 @@ export declare class RequestHandlerResult {
|
|
|
149
148
|
private _keyValueStoreChanges;
|
|
150
149
|
private pushDataCalls;
|
|
151
150
|
private addRequestsCalls;
|
|
152
|
-
private enqueueLinksCalls;
|
|
153
151
|
constructor(config: Configuration, crawleeStateKey: string);
|
|
154
152
|
/**
|
|
155
153
|
* A record of calls to {@link RestrictedCrawlingContext.pushData}, {@link RestrictedCrawlingContext.addRequests}, {@link RestrictedCrawlingContext.enqueueLinks} made by a request handler.
|
|
@@ -157,7 +155,6 @@ export declare class RequestHandlerResult {
|
|
|
157
155
|
get calls(): ReadonlyDeep<{
|
|
158
156
|
pushData: Parameters<RestrictedCrawlingContext['pushData']>[];
|
|
159
157
|
addRequests: Parameters<RestrictedCrawlingContext['addRequests']>[];
|
|
160
|
-
enqueueLinks: Parameters<RestrictedCrawlingContext['enqueueLinks']>[];
|
|
161
158
|
}>;
|
|
162
159
|
/**
|
|
163
160
|
* A record of changes made to key-value stores by a request handler.
|
|
@@ -188,7 +185,6 @@ export declare class RequestHandlerResult {
|
|
|
188
185
|
label?: string;
|
|
189
186
|
}[]>;
|
|
190
187
|
pushData: RestrictedCrawlingContext['pushData'];
|
|
191
|
-
enqueueLinks: RestrictedCrawlingContext['enqueueLinks'];
|
|
192
188
|
addRequests: RestrictedCrawlingContext['addRequests'];
|
|
193
189
|
useState: RestrictedCrawlingContext['useState'];
|
|
194
190
|
getKeyValueStore: RestrictedCrawlingContext['getKeyValueStore'];
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"crawler_commons.d.ts","sourceRoot":"","sources":["../../src/crawlers/crawler_commons.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,
|
|
1
|
+
{"version":3,"file":"crawler_commons.d.ts","sourceRoot":"","sources":["../../src/crawlers/crawler_commons.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AACjD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAChD,OAAO,KAAK,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAE3D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACzD,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,mCAAmC,CAAC;AAC7E,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,WAAW,CAAC;AACrC,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,2BAA2B,CAAC;AAC3D,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,eAAe,CAAC;AACrD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAC;AAC1D,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,wBAAwB,CAAC;AACtD,OAAO,EAAE,aAAa,EAAE,KAAK,aAAa,EAAE,MAAM,gCAAgC,CAAC;AACnF,OAAO,KAAK,EAAE,4BAA4B,EAAE,MAAM,iCAAiC,CAAC;AAEpF,gBAAgB;AAChB,MAAM,MAAM,KAAK,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,IAAI,GAAG,KAAK,CAAC;AAEtD,gBAAgB;AAChB,MAAM,MAAM,YAAY,CAAC,CAAC,EAAE,CAAC,SAAS,MAAM,CAAC,IAAI,CAAC,GAAG;KAAG,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;CAAE,CAAC;AAE1E,MAAM,MAAM,aAAa,CAAC,CAAC,SAAS,OAAO,IAAI,YAAY,CAAC,CAAC,EAAE,IAAI,GAAG,WAAW,CAAC,CAAC;AAEnF,gBAAgB;AAChB,MAAM,MAAM,aAAa,CAAC,OAAO,SAAS,yBAAyB,IAAI,KAAK,CAAC,OAAO,CAAC,SAAS,IAAI,GAC5F,OAAO,GACP;IACI,OAAO,EAAE,aAAa,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC;CAC9C,GAAG,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;AAEnC,MAAM,WAAW,yBAAyB,CAAC,QAAQ,SAAS,UAAU,GAAG,UAAU;IAC/E,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,CAAC,EAAE,OAAO,CAAC;IAElB;;;OAGG;IACH,SAAS,CAAC,EAAE,SAAS,CAAC;IAEtB;;OAEG;IACH,OAAO,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;IAE3B;;;;;;OAMG;IACH,QAAQ,CAAC,IAAI,EAAE,YAAY,CAAC,UAAU,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,eAAe,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAE1G;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACH,YAAY,EAAE,CACV,OAAO,EAAE,YAAY,CAAC,IAAI,CAAC,WAAW,CAAC,mBAAmB,EAAE,MAAM,CAAC,EAAE,cAAc,GAAG,eAAe,CAAC,CAAC,KACtG,OAAO,CAAC,OAAO,CAAC,CAAC;IAEtB;;;;;OAKG;IACH,WAAW,EAAE,CACT,YAAY,EAAE,YAAY,CAAC,CAAC,MAAM,GAAG,MAAM,CAAC,EAAE,CAAC,EAC/C,OAAO,CAAC,EAAE,YAAY,CAAC,4BAA4B,CAAC,KACnD,OAAO,CAAC,IAAI,CAAC,CAAC;IAEnB;;OAEG;IACH,QAAQ,EAAE,CAAC,KAAK,SAAS,UAAU,GAAG,UAAU,EAAE,YAAY,CAAC,EAAE,KAAK,KAAK,OAAO,CAAC,KAAK,CAAC,CAAC;IAE1F;;OAEG;IACH,gBAAgB,EAAE,CACd,QAAQ,CAAC,EAAE,MAAM,KAChB,OAAO,CAAC,IAAI,CAAC,aAAa,EAAE,IAAI,GAAG,MAAM,GAAG,UAAU,GAAG,mBAAmB,GAAG,UAAU,GAAG,cAAc,CAAC,CAAC,CAAC;IAElH;;OAEG;IACH,GAAG,EAAE,GAAG,CAAC;CACZ;AAED,MAAM,WAAW,eAAe,CAAC,QAAQ,SAAS,UAAU,GAAG,UAAU,CAAE,SAAQ,yBAAyB,CAAC,QAAQ,CAAC;IAClH;;;;;;;;;;;;;;;;;;;;;;;;OAwBG;IACH,YAAY,CACR,OAAO,EAAE,YAAY,CAAC,IAAI,CAAC,WAAW,CAAC,mBAAmB,EAAE,MAAM,CAAC,EAAE,cAAc,GAAG,eAAe,CAAC,CAAC,GACnG,IAAI,CAAC,mBAAmB,EAAE,cAAc,GAAG,eAAe,CAAC,GAChE,OAAO,CAAC,OAAO,CAAC,CAAC;IAEpB;;;;;;;;;;;;;;;;OAgBG;IACH,WAAW,CAAC,eAAe,CAAC,EAAE,OAAO,CAAC,WAAW,CAAC,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;IAEvE;;OAEG;IACH,uBAAuB,CAAC,OAAO,EAAE,MAAM,OAAO,CAAC,OAAO,CAAC,GAAG,IAAI,CAAC;CAClE;AAED;;;;GAIG;AACH,qBAAa,oBAAoB;IASzB,OAAO,CAAC,MAAM;IACd,OAAO,CAAC,eAAe;IAT3B,OAAO,CAAC,qBAAqB,CACtB;IAEP,OAAO,CAAC,aAAa,CAA2D;IAEhF,OAAO,CAAC,gBAAgB,CAA8D;gBAG1E,MAAM,EAAE,aAAa,EACrB,eAAe,EAAE,MAAM;IAGnC;;OAEG;IACH,IAAI,KAAK,IAAI,YAAY,CAAC;QACtB,QAAQ,EAAE,UAAU,CAAC,yBAAyB,CAAC,UAAU,CAAC,CAAC,EAAE,CAAC;QAC9D,WAAW,EAAE,UAAU,CAAC,yBAAyB,CAAC,aAAa,CAAC,CAAC,EAAE,CAAC;KACvE,CAAC,CAKD;IAED;;OAEG;IACH,IAAI,oBAAoB,IAAI,YAAY,CACpC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE;QAAE,YAAY,EAAE,OAAO,CAAC;QAAC,OAAO,CAAC,EAAE,aAAa,CAAA;KAAE,CAAC,CAAC,CACrF,CAEA;IAED;;OAEG;IACH,IAAI,YAAY,IAAI,YAAY,CAAC;QAAE,IAAI,EAAE,UAAU,CAAC;QAAC,eAAe,CAAC,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC,CAIjF;IAED;;OAEG;IACH,IAAI,YAAY,IAAI,YAAY,CAAC;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC,CAkBlE;IAED;;OAEG;IACH,IAAI,gBAAgB,IAAI,YAAY,CAAC;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC,CAgB1E;IAED,QAAQ,EAAE,yBAAyB,CAAC,UAAU,CAAC,CAE7C;IAEF,WAAW,EAAE,yBAAyB,CAAC,aAAa,CAAC,CAEnD;IAEF,QAAQ,EAAE,yBAAyB,CAAC,UAAU,CAAC,CAG7C;IAEF,gBAAgB,EAAE,yBAAyB,CAAC,kBAAkB,CAAC,CAa7D;IAEF,OAAO,CAAC,WAAW,CAAwF;IAE3G,OAAO,CAAC,4BAA4B,CAIlC;IAEF,OAAO,CAAC,4BAA4B,CASlC;CACL"}
|
|
@@ -10,7 +10,6 @@ export class RequestHandlerResult {
|
|
|
10
10
|
_keyValueStoreChanges = {};
|
|
11
11
|
pushDataCalls = [];
|
|
12
12
|
addRequestsCalls = [];
|
|
13
|
-
enqueueLinksCalls = [];
|
|
14
13
|
constructor(config, crawleeStateKey) {
|
|
15
14
|
this.config = config;
|
|
16
15
|
this.crawleeStateKey = crawleeStateKey;
|
|
@@ -22,7 +21,6 @@ export class RequestHandlerResult {
|
|
|
22
21
|
return {
|
|
23
22
|
pushData: this.pushDataCalls,
|
|
24
23
|
addRequests: this.addRequestsCalls,
|
|
25
|
-
enqueueLinks: this.enqueueLinksCalls,
|
|
26
24
|
};
|
|
27
25
|
}
|
|
28
26
|
/**
|
|
@@ -42,9 +40,6 @@ export class RequestHandlerResult {
|
|
|
42
40
|
*/
|
|
43
41
|
get enqueuedUrls() {
|
|
44
42
|
const result = [];
|
|
45
|
-
for (const [options] of this.enqueueLinksCalls) {
|
|
46
|
-
result.push(...(options?.urls?.map((url) => ({ url, label: options?.label })) ?? []));
|
|
47
|
-
}
|
|
48
43
|
for (const [requests] of this.addRequestsCalls) {
|
|
49
44
|
for (const request of requests) {
|
|
50
45
|
if (typeof request === 'object' &&
|
|
@@ -78,9 +73,6 @@ export class RequestHandlerResult {
|
|
|
78
73
|
pushData = async (data, datasetIdOrName) => {
|
|
79
74
|
this.pushDataCalls.push([data, datasetIdOrName]);
|
|
80
75
|
};
|
|
81
|
-
enqueueLinks = async (options) => {
|
|
82
|
-
this.enqueueLinksCalls.push([options]);
|
|
83
|
-
};
|
|
84
76
|
addRequests = async (requests, options = {}) => {
|
|
85
77
|
this.addRequestsCalls.push([requests, options]);
|
|
86
78
|
};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"crawler_commons.js","sourceRoot":"","sources":["../../src/crawlers/crawler_commons.ts"],"names":[],"mappings":"AAWA,OAAO,EAAE,aAAa,EAAsB,MAAM,gCAAgC,CAAC;
|
|
1
|
+
{"version":3,"file":"crawler_commons.js","sourceRoot":"","sources":["../../src/crawlers/crawler_commons.ts"],"names":[],"mappings":"AAWA,OAAO,EAAE,aAAa,EAAsB,MAAM,gCAAgC,CAAC;AA2JnF;;;;GAIG;AACH,MAAM,OAAO,oBAAoB;IASjB;IACA;IATJ,qBAAqB,GACzB,EAAE,CAAC;IAEC,aAAa,GAAwD,EAAE,CAAC;IAExE,gBAAgB,GAA2D,EAAE,CAAC;IAEtF,YACY,MAAqB,EACrB,eAAuB;QADvB,WAAM,GAAN,MAAM,CAAe;QACrB,oBAAe,GAAf,eAAe,CAAQ;IAChC,CAAC;IAEJ;;OAEG;IACH,IAAI,KAAK;QAIL,OAAO;YACH,QAAQ,EAAE,IAAI,CAAC,aAAa;YAC5B,WAAW,EAAE,IAAI,CAAC,gBAAgB;SACrC,CAAC;IACN,CAAC;IAED;;OAEG;IACH,IAAI,oBAAoB;QAGpB,OAAO,IAAI,CAAC,qBAAqB,CAAC;IACtC,CAAC;IAED;;OAEG;IACH,IAAI,YAAY;QACZ,OAAO,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,eAAe,CAAC,EAAE,EAAE,CAC1D,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,eAAe,EAAE,CAAC,CAAC,CACnF,CAAC;IACN,CAAC;IAED;;OAEG;IACH,IAAI,YAAY;QACZ,MAAM,MAAM,GAAsC,EAAE,CAAC;QAErD,KAAK,MAAM,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC7C,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;gBAC7B,IACI,OAAO,OAAO,KAAK,QAAQ;oBAC3B,CAAC,CAAC,CAAC,iBAAiB,IAAI,OAAO,CAAC,IAAI,OAAO,CAAC,eAAe,KAAK,SAAS,CAAC;oBAC1E,OAAO,CAAC,GAAG,KAAK,SAAS,EAC3B,CAAC;oBACC,MAAM,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC;gBAC5D,CAAC;qBAAM,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE,CAAC;oBACrC,MAAM,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,CAAC;gBAClC,CAAC;YACL,CAAC;QACL,CAAC;QAED,OAAO,MAAM,CAAC;IAClB,CAAC;IAED;;OAEG;IACH,IAAI,gBAAgB;QAChB,MAAM,MAAM,GAA0C,EAAE,CAAC;QAEzD,KAAK,MAAM,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC7C,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;gBAC7B,IACI,OAAO,OAAO,KAAK,QAAQ;oBAC3B,iBAAiB,IAAI,OAAO;oBAC5B,OAAO,CAAC,eAAe,KAAK,SAAS,EACvC,CAAC;oBACC,MAAM,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,eAAe,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC;gBAC5E,CAAC;YACL,CAAC;QACL,CAAC;QAED,OAAO,MAAM,CAAC;IAClB,CAAC;IAED,QAAQ,GAA0C,KAAK,EAAE,IAAI,EAAE,eAAe,EAAE,EAAE;QAC9E,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,eAAe,CAAC,CAAC,CAAC;IACrD,CAAC,CAAC;IAEF,WAAW,GAA6C,KAAK,EAAE,QAAQ,EAAE,OAAO,GAAG,EAAE,EAAE,EAAE;QACrF,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC;IACpD,CAAC,CAAC;IAEF,QAAQ,GAA0C,KAAK,EAAE,YAAY,EAAE,EAAE;QACrE,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC;QACrD,OAAO,MAAM,KAAK,CAAC,iBAAiB,CAAC,IAAI,CAAC,eAAe,EAAE,YAAY,CAAC,CAAC;IAC7E,CAAC,CAAC;IAEF,gBAAgB,GAAkD,KAAK,EAAE,QAAQ,EAAE,EAAE;QACjF,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;QAE1E,OAAO;YACH,EAAE,EAAE,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC;YAC9B,IAAI,EAAE,QAAQ;YACd,QAAQ,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,4BAA4B,CAAC,QAAQ,EAAE,GAAG,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;YACxG,QAAQ,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;gBACpC,IAAI,CAAC,4BAA4B,CAAC,QAAQ,EAAE,GAAG,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;YACrE,CAAC;YACD,iBAAiB,EAAE,KAAK,CAAC,iBAAiB,CAAC,IAAI,CAAC,KAAK,CAAC;YACtD,YAAY,EAAE,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC;SAC/C,CAAC;IACN,CAAC,CAAC;IAEM,WAAW,GAAG,CAAC,QAAiB,EAAU,EAAE,CAAC,QAAQ,IAAI,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,wBAAwB,CAAC,CAAC;IAEnG,4BAA4B,GAAG,CAAC,QAA4B,EAAE,GAAW,EAAE,EAAE;QACjF,MAAM,EAAE,GAAG,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;QACtC,IAAI,CAAC,qBAAqB,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC;QACtC,OAAO,IAAI,CAAC,oBAAoB,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,YAAY,IAAI,IAAI,CAAC;IACpE,CAAC,CAAC;IAEM,4BAA4B,GAAG,CACnC,QAA4B,EAC5B,GAAW,EACX,YAAqB,EACrB,OAAuB,EACzB,EAAE;QACA,MAAM,EAAE,GAAG,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;QACtC,IAAI,CAAC,qBAAqB,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC;QACtC,IAAI,CAAC,qBAAqB,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,GAAG,EAAE,YAAY,EAAE,OAAO,EAAE,CAAC;IACpE,CAAC,CAAC;CACL"}
|