@crawlee/core 3.0.0-beta.52 → 3.0.0-beta.55
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +38 -87
- package/autoscaling/snapshotter.d.ts +1 -1
- package/autoscaling/snapshotter.d.ts.map +1 -1
- package/autoscaling/snapshotter.js +11 -7
- package/autoscaling/snapshotter.js.map +1 -1
- package/configuration.d.ts +67 -57
- package/configuration.d.ts.map +1 -1
- package/configuration.js +71 -85
- package/configuration.js.map +1 -1
- package/events/event_manager.d.ts +6 -3
- package/events/event_manager.d.ts.map +1 -1
- package/events/event_manager.js +10 -2
- package/events/event_manager.js.map +1 -1
- package/package.json +7 -5
- package/storages/storage_manager.d.ts.map +1 -1
- package/storages/storage_manager.js +0 -9
- package/storages/storage_manager.js.map +1 -1
- package/tsconfig.build.tsbuildinfo +1 -1
package/configuration.js
CHANGED
|
@@ -1,60 +1,51 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.Configuration = void 0;
|
|
4
|
-
const consts_1 = require("@apify/consts");
|
|
5
4
|
const memory_storage_1 = require("@crawlee/memory-storage");
|
|
5
|
+
const fs_extra_1 = require("fs-extra");
|
|
6
|
+
const node_path_1 = require("node:path");
|
|
6
7
|
const node_async_hooks_1 = require("node:async_hooks");
|
|
7
8
|
const node_events_1 = require("node:events");
|
|
8
9
|
const typedefs_1 = require("./typedefs");
|
|
9
10
|
const events_1 = require("./events");
|
|
10
11
|
/**
|
|
11
|
-
* `Configuration` is a value object holding
|
|
12
|
+
* `Configuration` is a value object holding Crawlee configuration. By default, there is a
|
|
13
|
+
* global singleton instance of this class available via `Configuration.getGlobalConfig()`.
|
|
14
|
+
* Places that depend on a configurable behaviour depend on this class as have the global
|
|
15
|
+
* instance as the default value.
|
|
12
16
|
*
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
* import { Actor } from 'apify';
|
|
17
|
+
* ```js
|
|
18
|
+
* import { BasicCrawler, Configuration } from 'crawlee';
|
|
16
19
|
*
|
|
17
|
-
*
|
|
18
|
-
*
|
|
19
|
-
*
|
|
20
|
-
* 2. To get the global configuration (singleton instance). It will respect the environment variables.
|
|
21
|
-
* ```js
|
|
22
|
-
* import { Configuration } from 'crawlee';
|
|
20
|
+
* const config = new Configuration({ persistStateIntervalMillis: 30_000 });
|
|
21
|
+
* const crawler = new BasicCrawler({ ... }, config);
|
|
22
|
+
* ```
|
|
23
23
|
*
|
|
24
|
-
*
|
|
25
|
-
*
|
|
26
|
-
*
|
|
24
|
+
* The configuration provided via environment variables always takes precedence. We can also
|
|
25
|
+
* define the `crawlee.json` file in the project root directory which will serve as a baseline,
|
|
26
|
+
* so the options provided in constructor will override those.
|
|
27
|
+
*
|
|
28
|
+
* > In other words, the precedence is: crawlee.json < constructor options < environment variables.
|
|
27
29
|
*
|
|
28
30
|
* ## Supported Configuration Options
|
|
29
31
|
*
|
|
30
32
|
* Key | Environment Variable | Default Value
|
|
31
33
|
* ---|---|---
|
|
32
|
-
* `
|
|
33
|
-
* `
|
|
34
|
-
* `
|
|
35
|
-
* `
|
|
34
|
+
* `memoryMbytes` | `CRAWLEE_MEMORY_MBYTES` | -
|
|
35
|
+
* `headless` | `CRAWLEE_HEADLESS` | -
|
|
36
|
+
* `defaultDatasetId` | `CRAWLEE_DEFAULT_DATASET_ID` | `'default'`
|
|
37
|
+
* `defaultKeyValueStoreId` | `CRAWLEE_DEFAULT_KEY_VALUE_STORE_ID` | `'default'`
|
|
38
|
+
* `defaultRequestQueueId` | `CRAWLEE_DEFAULT_REQUEST_QUEUE_ID` | `'default'`
|
|
39
|
+
* `persistStateIntervalMillis` | `CRAWLEE_PERSIST_STATE_INTERVAL_MILLIS` | `60e3`
|
|
36
40
|
*
|
|
37
41
|
* ## Advanced Configuration Options
|
|
38
42
|
*
|
|
39
43
|
* Key | Environment Variable | Default Value
|
|
40
44
|
* ---|---|---
|
|
41
|
-
* `
|
|
42
|
-
* `
|
|
43
|
-
* `
|
|
44
|
-
* `
|
|
45
|
-
* `containerUrl` | `APIFY_CONTAINER_URL` | `'http://localhost:4321'`
|
|
46
|
-
* `inputKey` | `APIFY_INPUT_KEY` | `'INPUT'`
|
|
47
|
-
* `metamorphAfterSleepMillis` | `APIFY_METAMORPH_AFTER_SLEEP_MILLIS` | `300e3`
|
|
48
|
-
* `proxyHostname` | `APIFY_PROXY_HOSTNAME` | `'proxy.apify.com'`
|
|
49
|
-
* `proxyPassword` | `APIFY_PROXY_PASSWORD` | -
|
|
50
|
-
* `proxyPort` | `APIFY_PROXY_PORT` | `8000`
|
|
51
|
-
*
|
|
52
|
-
* ## Not Supported environment variables
|
|
53
|
-
*
|
|
54
|
-
* - `MEMORY_MBYTES`
|
|
55
|
-
* - `HEADLESS`
|
|
56
|
-
* - `XVFB`
|
|
57
|
-
* - `CHROME_EXECUTABLE_PATH`
|
|
45
|
+
* `inputKey` | `CRAWLEE_INPUT_KEY` | `'INPUT'`
|
|
46
|
+
* `xvfb` | `CRAWLEE_XVFB` | -
|
|
47
|
+
* `chromeExecutablePath` | `CRAWLEE_CHROME_EXECUTABLE_PATH` | -
|
|
48
|
+
* `defaultBrowserPath` | `CRAWLEE_DEFAULT_BROWSER_PATH` | -
|
|
58
49
|
*/
|
|
59
50
|
class Configuration {
|
|
60
51
|
/**
|
|
@@ -73,7 +64,7 @@ class Configuration {
|
|
|
73
64
|
writable: true,
|
|
74
65
|
value: new Map()
|
|
75
66
|
});
|
|
76
|
-
this.
|
|
67
|
+
this.buildOptions(options);
|
|
77
68
|
// Increase the global limit for event emitter memory leak warnings.
|
|
78
69
|
node_events_1.EventEmitter.defaultMaxListeners = 50;
|
|
79
70
|
}
|
|
@@ -83,9 +74,16 @@ class Configuration {
|
|
|
83
74
|
* in the above section.
|
|
84
75
|
*/
|
|
85
76
|
get(key, defaultValue) {
|
|
86
|
-
// prefer env vars
|
|
87
|
-
|
|
88
|
-
const
|
|
77
|
+
// prefer env vars, always iterate through the whole map as there might be duplicate env vars for the same option
|
|
78
|
+
let envValue;
|
|
79
|
+
for (const [k, v] of (0, typedefs_1.entries)(Configuration.ENV_MAP)) {
|
|
80
|
+
if (key === v) {
|
|
81
|
+
envValue = process.env[k];
|
|
82
|
+
if (envValue) {
|
|
83
|
+
break;
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
}
|
|
89
87
|
if (envValue != null) {
|
|
90
88
|
return this._castEnvValue(key, envValue);
|
|
91
89
|
}
|
|
@@ -118,7 +116,7 @@ class Configuration {
|
|
|
118
116
|
* this {@link Configuration} instance. Only first call of this method will create the client, following calls will
|
|
119
117
|
* return the same client instance.
|
|
120
118
|
*
|
|
121
|
-
* Caching works based on the
|
|
119
|
+
* Caching works based on the `storageClientOptions`, so calling this method with different options will return
|
|
122
120
|
* multiple instances, one for each variant of the options.
|
|
123
121
|
* @internal
|
|
124
122
|
*/
|
|
@@ -188,83 +186,71 @@ class Configuration {
|
|
|
188
186
|
static resetGlobalState() {
|
|
189
187
|
delete this.globalConfig;
|
|
190
188
|
}
|
|
189
|
+
buildOptions(options) {
|
|
190
|
+
// try to load configuration from crawlee.json as the baseline
|
|
191
|
+
const path = (0, node_path_1.join)(process.cwd(), 'crawlee.json');
|
|
192
|
+
if ((0, fs_extra_1.pathExistsSync)(path)) {
|
|
193
|
+
try {
|
|
194
|
+
const file = (0, fs_extra_1.readFileSync)(path);
|
|
195
|
+
const optionsFromFileConfig = JSON.parse(file.toString());
|
|
196
|
+
Object.assign(options, optionsFromFileConfig);
|
|
197
|
+
}
|
|
198
|
+
catch {
|
|
199
|
+
// ignore
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
this.options = new Map((0, typedefs_1.entries)(options));
|
|
203
|
+
}
|
|
191
204
|
}
|
|
192
205
|
exports.Configuration = Configuration;
|
|
193
206
|
/**
|
|
194
|
-
* Maps environment variables to config keys (e.g. `
|
|
207
|
+
* Maps environment variables to config keys (e.g. `CRAWLEE_MEMORY_MBYTES` to `memoryMbytes`)
|
|
195
208
|
*/
|
|
196
209
|
Object.defineProperty(Configuration, "ENV_MAP", {
|
|
197
210
|
enumerable: true,
|
|
198
211
|
configurable: true,
|
|
199
212
|
writable: true,
|
|
200
213
|
value: {
|
|
201
|
-
// TODO prefix once we have a package name
|
|
202
214
|
CRAWLEE_AVAILABLE_MEMORY_RATIO: 'availableMemoryRatio',
|
|
203
215
|
CRAWLEE_PURGE_ON_START: 'purgeOnStart',
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
APIFY_CONTAINER_URL: 'containerUrl',
|
|
216
|
-
APIFY_PROXY_HOSTNAME: 'proxyHostname',
|
|
217
|
-
APIFY_PROXY_PASSWORD: 'proxyPassword',
|
|
218
|
-
APIFY_PROXY_PORT: 'proxyPort',
|
|
219
|
-
// not supported, use env vars directly:
|
|
220
|
-
// APIFY_MEMORY_MBYTES: 'memoryMbytes',
|
|
221
|
-
// APIFY_HEADLESS: 'headless',
|
|
222
|
-
// APIFY_XVFB: 'xvfb',
|
|
223
|
-
// APIFY_CHROME_EXECUTABLE_PATH: 'chromeExecutablePath',
|
|
216
|
+
CRAWLEE_MEMORY_MBYTES: 'memoryMbytes',
|
|
217
|
+
CRAWLEE_DEFAULT_DATASET_ID: 'defaultDatasetId',
|
|
218
|
+
CRAWLEE_DEFAULT_KEY_VALUE_STORE_ID: 'defaultKeyValueStoreId',
|
|
219
|
+
CRAWLEE_DEFAULT_REQUEST_QUEUE_ID: 'defaultRequestQueueId',
|
|
220
|
+
CRAWLEE_INPUT_KEY: 'inputKey',
|
|
221
|
+
CRAWLEE_PERSIST_STATE_INTERVAL_MILLIS: 'persistStateIntervalMillis',
|
|
222
|
+
CRAWLEE_HEADLESS: 'headless',
|
|
223
|
+
CRAWLEE_XVFB: 'xvfb',
|
|
224
|
+
CRAWLEE_CHROME_EXECUTABLE_PATH: 'chromeExecutablePath',
|
|
225
|
+
CRAWLEE_DEFAULT_BROWSER_PATH: 'defaultBrowserPath',
|
|
226
|
+
CRAWLEE_DISABLE_BROWSER_SANDBOX: 'disableBrowserSandbox',
|
|
224
227
|
}
|
|
225
228
|
});
|
|
226
|
-
/**
|
|
227
|
-
* Maps config keys to environment variables (e.g. `proxyPort` to `APIFY_PROXY_PORT`).
|
|
228
|
-
*/
|
|
229
|
-
Object.defineProperty(Configuration, "ENV_MAP_REVERSED", {
|
|
230
|
-
enumerable: true,
|
|
231
|
-
configurable: true,
|
|
232
|
-
writable: true,
|
|
233
|
-
value: (0, typedefs_1.entries)(Configuration.ENV_MAP).reduce((obj, [key, value]) => {
|
|
234
|
-
obj[value] = key;
|
|
235
|
-
return obj;
|
|
236
|
-
}, {})
|
|
237
|
-
});
|
|
238
229
|
Object.defineProperty(Configuration, "BOOLEAN_VARS", {
|
|
239
230
|
enumerable: true,
|
|
240
231
|
configurable: true,
|
|
241
232
|
writable: true,
|
|
242
|
-
value: ['purgeOnStart']
|
|
233
|
+
value: ['purgeOnStart', 'headless', 'xvfb', 'disableBrowserSandbox']
|
|
243
234
|
});
|
|
244
235
|
Object.defineProperty(Configuration, "INTEGER_VARS", {
|
|
245
236
|
enumerable: true,
|
|
246
237
|
configurable: true,
|
|
247
238
|
writable: true,
|
|
248
|
-
value: ['
|
|
239
|
+
value: ['memoryMbytes', 'persistStateIntervalMillis', 'systemInfoIntervalMillis']
|
|
249
240
|
});
|
|
250
241
|
Object.defineProperty(Configuration, "DEFAULTS", {
|
|
251
242
|
enumerable: true,
|
|
252
243
|
configurable: true,
|
|
253
244
|
writable: true,
|
|
254
245
|
value: {
|
|
255
|
-
defaultKeyValueStoreId:
|
|
256
|
-
defaultDatasetId:
|
|
257
|
-
defaultRequestQueueId:
|
|
246
|
+
defaultKeyValueStoreId: 'default',
|
|
247
|
+
defaultDatasetId: 'default',
|
|
248
|
+
defaultRequestQueueId: 'default',
|
|
249
|
+
inputKey: 'INPUT',
|
|
258
250
|
maxUsedCpuRatio: 0.95,
|
|
259
251
|
availableMemoryRatio: 0.25,
|
|
260
252
|
storageClientOptions: {},
|
|
261
253
|
purgeOnStart: true,
|
|
262
|
-
inputKey: 'INPUT',
|
|
263
|
-
proxyHostname: consts_1.LOCAL_ENV_VARS[consts_1.ENV_VARS.PROXY_HOSTNAME],
|
|
264
|
-
proxyPort: +consts_1.LOCAL_ENV_VARS[consts_1.ENV_VARS.PROXY_PORT],
|
|
265
|
-
containerPort: +consts_1.LOCAL_ENV_VARS[consts_1.ENV_VARS.CONTAINER_PORT],
|
|
266
|
-
containerUrl: consts_1.LOCAL_ENV_VARS[consts_1.ENV_VARS.CONTAINER_URL],
|
|
267
|
-
metamorphAfterSleepMillis: 300000,
|
|
268
254
|
persistStateIntervalMillis: 60000,
|
|
269
255
|
systemInfoIntervalMillis: 60000,
|
|
270
256
|
}
|
package/configuration.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"configuration.js","sourceRoot":"","sources":["../src/configuration.ts"],"names":[],"mappings":";;;
|
|
1
|
+
{"version":3,"file":"configuration.js","sourceRoot":"","sources":["../src/configuration.ts"],"names":[],"mappings":";;;AACA,4DAAwD;AACxD,uCAAwD;AACxD,yCAAiC;AACjC,uDAAqD;AACrD,6CAA2C;AAE3C,yCAAqC;AAErC,qCAA6C;AAuB7C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsCG;AACH,MAAa,aAAa;IAiDtB;;OAEG;IACH,YAAY,UAAgC,EAAE;QAT9C;;;;;WAAsG;QACtG;;;;mBAAqB,IAAI,GAAG,EAAmB;WAAC;QAS5C,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;QAE3B,oEAAoE;QACpE,0BAAY,CAAC,mBAAmB,GAAG,EAAE,CAAC;IAC1C,CAAC;IAED;;;;OAIG;IACH,GAAG,CAA0E,GAAM,EAAE,YAAgB;QACjG,iHAAiH;QACjH,IAAI,QAA4B,CAAC;QAEjC,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,IAAA,kBAAO,EAAC,aAAa,CAAC,OAAO,CAAC,EAAE;YACjD,IAAI,GAAG,KAAK,CAAC,EAAE;gBACX,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;gBAE1B,IAAI,QAAQ,EAAE;oBACV,MAAM;iBACT;aACJ;SACJ;QAED,IAAI,QAAQ,IAAI,IAAI,EAAE;YAClB,OAAO,IAAI,CAAC,aAAa,CAAC,GAAG,EAAE,QAAQ,CAAM,CAAC;SACjD;QAED,+BAA+B;QAC/B,IAAI,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE;YACvB,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAM,CAAC;SACrC;QAED,uBAAuB;QACvB,OAAO,CAAC,YAAY,IAAI,aAAa,CAAC,QAAQ,CAAC,GAA0C,CAAC,IAAI,QAAQ,CAAM,CAAC;IACjH,CAAC;IAES,aAAa,CAAC,GAA+B,EAAE,KAAgC;QACrF,IAAI,aAAa,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE;YAC1C,OAAO,CAAC,KAAK,CAAC;SACjB;QAED,IAAI,aAAa,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE;YAC1C,wDAAwD;YACxD,OAAO,CAAC,CAAC,GAAG,EAAE,OAAO,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;SACpE;QAED,OAAO,KAAK,CAAC;IACjB,CAAC;IAED;;;OAGG;IACH,GAAG,CAAC,GAA+B,EAAE,KAAW;QAC5C,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;IACjC,CAAC;IAED;;;;;;;;OAQG;IACH,gBAAgB;QACZ,IAAI,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,EAAE;YACnC,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAkB,CAAC;SAC7D;QAED,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAe,CAAC;QACvE,OAAO,IAAI,CAAC,mBAAmB,CAAC,OAAO,CAAC,CAAC;IAC7C,CAAC;IAED,eAAe;QACX,IAAI,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,EAAE;YAClC,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAiB,CAAC;SAC3D;QAED,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,cAAc,CAAC,EAAE;YACnC,OAAO,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,cAAc,CAAiB,CAAC;SAC5D;QAED,MAAM,YAAY,GAAG,IAAI,0BAAiB,CAAC,IAAI,CAAC,CAAC;QACjD,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,cAAc,EAAE,YAAY,CAAC,CAAC;QAEhD,OAAO,YAAY,CAAC;IACxB,CAAC;IAED;;;OAGG;IACH,mBAAmB,CAAC,UAAgC,EAAE;QAClD,MAAM,QAAQ,GAAG,iBAAiB,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,CAAC;QAE5D,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE;YAC7B,OAAO,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAkB,CAAC;SACvD;QAED,MAAM,OAAO,GAAG,IAAI,8BAAa,CAAC,OAAO,CAAC,CAAC;QAC3C,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAErC,OAAO,OAAO,CAAC;IACnB,CAAC;IAED,gBAAgB,CAAC,MAAqB;QAClC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,EAAE,MAAM,CAAC,CAAC;IAC9C,CAAC;IAED,eAAe,CAAC,MAAoB;QAChC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,MAAM,CAAC,CAAC;IAC7C,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,eAAe;QAClB,IAAI,aAAa,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE;YAClC,OAAO,aAAa,CAAC,OAAO,CAAC,QAAQ,EAAG,CAAC;SAC5C;QAED,aAAa,CAAC,YAAY,KAA1B,aAAa,CAAC,YAAY,GAAK,IAAI,aAAa,EAAE,EAAC;QACnD,OAAO,aAAa,CAAC,YAAY,CAAC;IACtC,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,gBAAgB;QACnB,OAAO,IAAI,CAAC,eAAe,EAAE,CAAC,gBAAgB,EAAE,CAAC;IACrD,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,eAAe;QAClB,OAAO,IAAI,CAAC,eAAe,EAAE,CAAC,eAAe,EAAE,CAAC;IACpD,CAAC;IAED;;;OAGG;IACH,MAAM,CAAC,gBAAgB;QACnB,OAAO,IAAI,CAAC,YAAY,CAAC;IAC7B,CAAC;IAES,YAAY,CAAC,OAA6B;QAChD,8DAA8D;QAC9D,MAAM,IAAI,GAAG,IAAA,gBAAI,EAAC,OAAO,CAAC,GAAG,EAAE,EAAE,cAAc,CAAC,CAAC;QAEjD,IAAI,IAAA,yBAAc,EAAC,IAAI,CAAC,EAAE;YACtB,IAAI;gBACA,MAAM,IAAI,GAAG,IAAA,uBAAY,EAAC,IAAI,CAAC,CAAC;gBAChC,MAAM,qBAAqB,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;gBAC1D,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,qBAAqB,CAAC,CAAC;aACjD;YAAC,MAAM;gBACJ,SAAS;aACZ;SACJ;QAED,IAAI,CAAC,OAAO,GAAG,IAAI,GAAG,CAAC,IAAA,kBAAO,EAAC,OAAO,CAAC,CAAC,CAAC;IAC7C,CAAC;;AA3NL,sCA4NC;AA3NG;;GAEG;AACH;;;;WAA2B;QACvB,8BAA8B,EAAE,sBAAsB;QACtD,sBAAsB,EAAE,cAAc;QACtC,qBAAqB,EAAE,cAAc;QACrC,0BAA0B,EAAE,kBAAkB;QAC9C,kCAAkC,EAAE,wBAAwB;QAC5D,gCAAgC,EAAE,uBAAuB;QACzD,iBAAiB,EAAE,UAAU;QAC7B,qCAAqC,EAAE,4BAA4B;QACnE,gBAAgB,EAAE,UAAU;QAC5B,YAAY,EAAE,MAAM;QACpB,8BAA8B,EAAE,sBAAsB;QACtD,4BAA4B,EAAE,oBAAoB;QAClD,+BAA+B,EAAE,uBAAuB;KAC3D;GAAC;AAEF;;;;WAA0C,CAAC,cAAc,EAAE,UAAU,EAAE,MAAM,EAAE,uBAAuB,CAAC;GAAC;AAExG;;;;WAAgC,CAAC,cAAc,EAAE,4BAA4B,EAAE,0BAA0B,CAAC;GAAC;AAE3G;;;;WAA4B;QACxB,sBAAsB,EAAE,SAAS;QACjC,gBAAgB,EAAE,SAAS;QAC3B,qBAAqB,EAAE,SAAS;QAChC,QAAQ,EAAE,OAAO;QACjB,eAAe,EAAE,IAAI;QACrB,oBAAoB,EAAE,IAAI;QAC1B,oBAAoB,EAAE,EAAE;QACxB,YAAY,EAAE,IAAI;QAClB,0BAA0B,EAAE,KAAM;QAClC,wBAAwB,EAAE,KAAM;KACnC;GAAC;AAEF;;;GAGG;AACH;;;;WAAiB,IAAI,oCAAiB,EAAiB;GAAC"}
|
|
@@ -1,6 +1,5 @@
|
|
|
1
|
-
/// <reference types="node" />
|
|
2
|
-
import { EventEmitter } from 'node:events';
|
|
3
1
|
import type { BetterIntervalID } from '@apify/utilities';
|
|
2
|
+
import { AsyncEventEmitter } from '@vladfrangu/async_event_emitter';
|
|
4
3
|
import { Configuration } from '../configuration';
|
|
5
4
|
export declare const enum EventType {
|
|
6
5
|
PERSIST_STATE = "persistState",
|
|
@@ -16,7 +15,7 @@ interface Intervals {
|
|
|
16
15
|
}
|
|
17
16
|
export declare abstract class EventManager {
|
|
18
17
|
readonly config: Configuration;
|
|
19
|
-
protected events:
|
|
18
|
+
protected events: AsyncEventEmitter<Record<string | symbol, unknown[]> & import("@vladfrangu/async_event_emitter").AsyncEventEmitterPredefinedEvents>;
|
|
20
19
|
protected initialized: boolean;
|
|
21
20
|
protected intervals: Intervals;
|
|
22
21
|
protected log: import("@apify/log").Log;
|
|
@@ -43,6 +42,10 @@ export declare abstract class EventManager {
|
|
|
43
42
|
* @internal
|
|
44
43
|
*/
|
|
45
44
|
listeners(event: EventTypeName): (() => Promise<unknown>)[];
|
|
45
|
+
/**
|
|
46
|
+
* @internal
|
|
47
|
+
*/
|
|
48
|
+
waitForAllListenersToComplete(): Promise<boolean>;
|
|
46
49
|
}
|
|
47
50
|
export {};
|
|
48
51
|
//# sourceMappingURL=event_manager.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"event_manager.d.ts","sourceRoot":"","sources":["../../src/events/event_manager.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"event_manager.d.ts","sourceRoot":"","sources":["../../src/events/event_manager.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAEzD,OAAO,EAAE,iBAAiB,EAAE,MAAM,iCAAiC,CAAC;AACpE,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEjD,0BAAkB,SAAS;IACvB,aAAa,iBAAiB;IAC9B,WAAW,eAAe;IAC1B,SAAS,cAAc;IACvB,QAAQ,aAAa;IACrB,IAAI,SAAS;CAChB;AAED,oBAAY,aAAa,GAAG,SAAS,GAAG,YAAY,GAAG,cAAc,GAAG,WAAW,GAAG,UAAU,GAAG,MAAM,CAAC;AAE1G,UAAU,SAAS;IACf,YAAY,CAAC,EAAE,gBAAgB,CAAC;IAChC,UAAU,CAAC,EAAE,gBAAgB,CAAC;CACjC;AAED,8BAAsB,YAAY;IAMlB,QAAQ,CAAC,MAAM;IAL3B,SAAS,CAAC,MAAM,sIAA2B;IAC3C,SAAS,CAAC,WAAW,UAAS;IAC9B,SAAS,CAAC,SAAS,EAAE,SAAS,CAAM;IACpC,SAAS,CAAC,GAAG,2BAAmC;gBAE3B,MAAM,gBAAkC;IAE7D;;;OAGG;IACG,IAAI;IAaV;;;OAGG;IACG,KAAK;IAYX,EAAE,CAAC,KAAK,EAAE,aAAa,EAAE,QAAQ,EAAE,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,GAAG,GAAG,IAAI;IAIjE,GAAG,CAAC,KAAK,EAAE,aAAa,EAAE,QAAQ,CAAC,EAAE,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,GAAG,GAAG,IAAI;IAQnE,IAAI,CAAC,KAAK,EAAE,aAAa,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI;IAIpD,aAAa,IAAI,OAAO;IAIxB;;OAEG;IACH,aAAa,CAAC,KAAK,EAAE,aAAa,GAAG,MAAM;IAI3C;;OAEG;IACH,SAAS,CAAC,KAAK,EAAE,aAAa,GAAG,CAAC,MAAM,OAAO,CAAC,OAAO,CAAC,CAAC,EAAE;IAI3D;;OAEG;IACH,6BAA6B;CAGhC"}
|
package/events/event_manager.js
CHANGED
|
@@ -3,8 +3,8 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
3
3
|
exports.EventManager = exports.EventType = void 0;
|
|
4
4
|
const tslib_1 = require("tslib");
|
|
5
5
|
const log_1 = tslib_1.__importDefault(require("@apify/log"));
|
|
6
|
-
const node_events_1 = require("node:events");
|
|
7
6
|
const utilities_1 = require("@apify/utilities");
|
|
7
|
+
const async_event_emitter_1 = require("@vladfrangu/async_event_emitter");
|
|
8
8
|
const configuration_1 = require("../configuration");
|
|
9
9
|
var EventType;
|
|
10
10
|
(function (EventType) {
|
|
@@ -26,7 +26,7 @@ class EventManager {
|
|
|
26
26
|
enumerable: true,
|
|
27
27
|
configurable: true,
|
|
28
28
|
writable: true,
|
|
29
|
-
value: new
|
|
29
|
+
value: new async_event_emitter_1.AsyncEventEmitter()
|
|
30
30
|
});
|
|
31
31
|
Object.defineProperty(this, "initialized", {
|
|
32
32
|
enumerable: true,
|
|
@@ -72,6 +72,8 @@ class EventManager {
|
|
|
72
72
|
}
|
|
73
73
|
(0, utilities_1.betterClearInterval)(this.intervals.persistState);
|
|
74
74
|
this.initialized = false;
|
|
75
|
+
// Emit final PERSIST_STATE event
|
|
76
|
+
this.emit("persistState" /* EventType.PERSIST_STATE */, { isMigrating: false });
|
|
75
77
|
}
|
|
76
78
|
on(event, listener) {
|
|
77
79
|
this.events.on(event, listener);
|
|
@@ -102,6 +104,12 @@ class EventManager {
|
|
|
102
104
|
listeners(event) {
|
|
103
105
|
return this.events.listeners(event);
|
|
104
106
|
}
|
|
107
|
+
/**
|
|
108
|
+
* @internal
|
|
109
|
+
*/
|
|
110
|
+
waitForAllListenersToComplete() {
|
|
111
|
+
return this.events.waitForAllListenersToComplete();
|
|
112
|
+
}
|
|
105
113
|
}
|
|
106
114
|
exports.EventManager = EventManager;
|
|
107
115
|
//# sourceMappingURL=event_manager.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"event_manager.js","sourceRoot":"","sources":["../../src/events/event_manager.ts"],"names":[],"mappings":";;;;AAAA,6DAA6B;
|
|
1
|
+
{"version":3,"file":"event_manager.js","sourceRoot":"","sources":["../../src/events/event_manager.ts"],"names":[],"mappings":";;;;AAAA,6DAA6B;AAE7B,gDAA0E;AAC1E,yEAAoE;AACpE,oDAAiD;AAEjD,IAAkB,SAMjB;AAND,WAAkB,SAAS;IACvB,2CAA8B,CAAA;IAC9B,uCAA0B,CAAA;IAC1B,oCAAuB,CAAA;IACvB,kCAAqB,CAAA;IACrB,0BAAa,CAAA;AACjB,CAAC,EANiB,SAAS,GAAT,iBAAS,KAAT,iBAAS,QAM1B;AASD,MAAsB,YAAY;IAM9B,YAAqB,SAAS,6BAAa,CAAC,eAAe,EAAE;;;;;mBAAxC;;QALrB;;;;mBAAmB,IAAI,uCAAiB,EAAE;WAAC;QAC3C;;;;mBAAwB,KAAK;WAAC;QAC9B;;;;mBAAiC,EAAE;WAAC;QACpC;;;;mBAAgB,aAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC;WAAC;IAEgB,CAAC;IAEjE;;;OAGG;IACH,KAAK,CAAC,IAAI;QACN,IAAI,IAAI,CAAC,WAAW,EAAE;YAClB,OAAO;SACV;QAED,MAAM,0BAA0B,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,4BAA4B,CAAE,CAAC;QAClF,IAAI,CAAC,SAAS,CAAC,YAAY,GAAG,IAAA,6BAAiB,EAAC,CAAC,gBAA+B,EAAE,EAAE;YAChF,IAAI,CAAC,IAAI,+CAA0B,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC,CAAC;YAC3D,gBAAgB,EAAE,CAAC;QACvB,CAAC,EAAE,0BAA0B,CAAC,CAAC;QAC/B,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;IAC5B,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,KAAK;QACP,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE;YACnB,OAAO;SACV;QAED,IAAA,+BAAmB,EAAC,IAAI,CAAC,SAAS,CAAC,YAAa,CAAC,CAAC;QAClD,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;QAEzB,iCAAiC;QACjC,IAAI,CAAC,IAAI,+CAA0B,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC,CAAC;IAC/D,CAAC;IAED,EAAE,CAAC,KAAoB,EAAE,QAAiC;QACtD,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;IACpC,CAAC;IAED,GAAG,CAAC,KAAoB,EAAE,QAAkC;QACxD,IAAI,QAAQ,EAAE;YACV,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;SAC/C;aAAM;YACH,IAAI,CAAC,MAAM,CAAC,kBAAkB,CAAC,KAAK,CAAC,CAAC;SACzC;IACL,CAAC;IAED,IAAI,CAAC,KAAoB,EAAE,GAAG,IAAe;QACzC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,GAAG,IAAI,CAAC,CAAC;IACrC,CAAC;IAED,aAAa;QACT,OAAO,IAAI,CAAC,WAAW,CAAC;IAC5B,CAAC;IAED;;OAEG;IACH,aAAa,CAAC,KAAoB;QAC9B,OAAO,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;IAC5C,CAAC;IAED;;OAEG;IACH,SAAS,CAAC,KAAoB;QAC1B,OAAO,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAA+B,CAAC;IACtE,CAAC;IAED;;OAEG;IACH,6BAA6B;QACzB,OAAO,IAAI,CAAC,MAAM,CAAC,6BAA6B,EAAE,CAAC;IACvD,CAAC;CACJ;AAjFD,oCAiFC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/core",
|
|
3
|
-
"version": "3.0.0-beta.
|
|
3
|
+
"version": "3.0.0-beta.55",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=16.0.0"
|
|
@@ -58,11 +58,13 @@
|
|
|
58
58
|
"@apify/pseudo_url": "^2.0.0",
|
|
59
59
|
"@apify/timeout": "^0.3.0",
|
|
60
60
|
"@apify/utilities": "^2.0.0",
|
|
61
|
-
"@crawlee/browser-pool": "^3.0.0-beta.
|
|
62
|
-
"@crawlee/memory-storage": "^3.0.0-beta.
|
|
63
|
-
"@crawlee/types": "^3.0.0-beta.
|
|
64
|
-
"@crawlee/utils": "^3.0.0-beta.
|
|
61
|
+
"@crawlee/browser-pool": "^3.0.0-beta.55",
|
|
62
|
+
"@crawlee/memory-storage": "^3.0.0-beta.55",
|
|
63
|
+
"@crawlee/types": "^3.0.0-beta.55",
|
|
64
|
+
"@crawlee/utils": "^3.0.0-beta.55",
|
|
65
65
|
"@types/tough-cookie": "^4.0.2",
|
|
66
|
+
"@vladfrangu/async_event_emitter": "^2.0.0",
|
|
67
|
+
"fs-extra": "^10.1.0",
|
|
66
68
|
"minimatch": "^5.1.0",
|
|
67
69
|
"ow": "^0.28.1",
|
|
68
70
|
"stream-chain": "^2.2.5",
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"storage_manager.d.ts","sourceRoot":"","sources":["../../src/storages/storage_manager.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"storage_manager.d.ts","sourceRoot":"","sources":["../../src/storages/storage_manager.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAChE,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AACjD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAQ/C,MAAM,WAAW,QAAQ;IACrB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;;GAGG;AACH,qBAAa,cAAc,CAAC,CAAC,SAAS,QAAQ,GAAG,QAAQ;IAQjD,OAAO,CAAC,QAAQ,CAAC,MAAM;IAP3B,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,eAAe,CAA0C;IACjF,OAAO,CAAC,QAAQ,CAAC,IAAI,CAA+C;IACpE,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAoC;IACvE,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAwB;gBAG1C,kBAAkB,EAAE,WAAW,CAAC,CAAC,CAAC,EACjB,MAAM,gBAAkC;IAM7D,MAAM,CAAC,WAAW,CAAC,CAAC,SAAS,QAAQ,EACjC,YAAY,EAAE,WAAW,CAAC,CAAC,CAAC,EAC5B,QAAQ,CAAC,EAAE,MAAM,EACjB,MAAM,CAAC,EAAE,aAAa,EACtB,MAAM,gBAAkC,GACzC,OAAO,CAAC,CAAC,CAAC;IAIb,MAAM,CAAC,UAAU,CAAC,CAAC,SAAS,QAAQ,EAChC,YAAY,EAAE,WAAW,CAAC,CAAC,CAAC,EAC5B,MAAM,gBAAkC,GACzC,cAAc,CAAC,CAAC,CAAC;IASpB,gBAAgB;IAChB,MAAM,CAAC,UAAU,IAAI,IAAI;IAWnB,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,EAAE,MAAM,CAAC,EAAE,aAAa,GAAG,OAAO,CAAC,CAAC,CAAC;IAuB/E,YAAY,CAAC,OAAO,EAAE;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI;IAU1D;;OAEG;cACa,mBAAmB,CAAC,eAAe,EAAE,MAAM,EAAE,sBAAsB,EAAE,MAAM,EAAE,SAAS,EAAE,aAAa;IAcrH,SAAS,CAAC,0BAA0B,CAAC,MAAM,EAAE,aAAa,EAAE,sBAAsB,EAAE,MAAM;;;;IAY1F,SAAS,CAAC,kBAAkB,CAAC,OAAO,EAAE,CAAC,GAAG,IAAI;CASjD;AAKD,MAAM,WAAW,qBAAqB;IAClC;;OAEG;IACH,MAAM,CAAC,EAAE,aAAa,CAAC;CAC1B"}
|
|
@@ -1,13 +1,7 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.StorageManager = void 0;
|
|
4
|
-
const consts_1 = require("@apify/consts");
|
|
5
4
|
const configuration_1 = require("../configuration");
|
|
6
|
-
const DEFAULT_ID_ENV_VAR_NAMES = {
|
|
7
|
-
Dataset: consts_1.ENV_VARS.DEFAULT_DATASET_ID,
|
|
8
|
-
KeyValueStore: consts_1.ENV_VARS.DEFAULT_KEY_VALUE_STORE_ID,
|
|
9
|
-
RequestQueue: consts_1.ENV_VARS.DEFAULT_REQUEST_QUEUE_ID,
|
|
10
|
-
};
|
|
11
5
|
const DEFAULT_ID_CONFIG_KEYS = {
|
|
12
6
|
Dataset: 'defaultDatasetId',
|
|
13
7
|
KeyValueStore: 'defaultKeyValueStoreId',
|
|
@@ -69,11 +63,8 @@ class StorageManager {
|
|
|
69
63
|
}
|
|
70
64
|
async openStorage(idOrName, client) {
|
|
71
65
|
if (!idOrName) {
|
|
72
|
-
const defaultIdEnvVarName = DEFAULT_ID_ENV_VAR_NAMES[this.name];
|
|
73
66
|
const defaultIdConfigKey = DEFAULT_ID_CONFIG_KEYS[this.name];
|
|
74
67
|
idOrName = this.config.get(defaultIdConfigKey);
|
|
75
|
-
if (!idOrName)
|
|
76
|
-
throw new Error(`The '${defaultIdEnvVarName}' environment variable is not defined.`);
|
|
77
68
|
}
|
|
78
69
|
const cacheKey = idOrName;
|
|
79
70
|
let storage = this.cache.get(cacheKey);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"storage_manager.js","sourceRoot":"","sources":["../../src/storages/storage_manager.ts"],"names":[],"mappings":";;;
|
|
1
|
+
{"version":3,"file":"storage_manager.js","sourceRoot":"","sources":["../../src/storages/storage_manager.ts"],"names":[],"mappings":";;;AACA,oDAAiD;AAGjD,MAAM,sBAAsB,GAAG;IAC3B,OAAO,EAAE,kBAAkB;IAC3B,aAAa,EAAE,wBAAwB;IACvC,YAAY,EAAE,uBAAuB;CAC/B,CAAC;AAOX;;;GAGG;AACH,MAAa,cAAc;IAMvB,YACI,kBAAkC,EACjB,SAAS,6BAAa,CAAC,eAAe,EAAE;;;;;mBAAxC;;QANrB;;;;;WAAoE;QACpE;;;;;WAAuE;QACvE;;;;mBAAyB,IAAI,GAAG,EAAa;WAAC;QAM1C,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,kBAAkB,CAAC,IAAoD,CAAC;IAC7F,CAAC;IAED,MAAM,CAAC,WAAW,CACd,YAA4B,EAC5B,QAAiB,EACjB,MAAsB,EACtB,MAAM,GAAG,6BAAa,CAAC,eAAe,EAAE;QAExC,OAAO,IAAI,CAAC,UAAU,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC,WAAW,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IAC/E,CAAC;IAED,MAAM,CAAC,UAAU,CACb,YAA4B,EAC5B,MAAM,GAAG,6BAAa,CAAC,eAAe,EAAE;QAExC,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,YAAY,CAAC,EAAE;YACzC,MAAM,OAAO,GAAG,IAAI,cAAc,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;YACzD,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;SACnD;QAED,OAAO,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,YAAY,CAAsB,CAAC;IACvE,CAAC;IAED,gBAAgB;IAChB,MAAM,CAAC,UAAU;QACb,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;YACrC,IAAI,OAAO,CAAC,IAAI,KAAK,eAAe,EAAE;gBAClC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;oBAC1B,IAAmB,CAAC,UAAU,EAAE,EAAE,CAAC;gBACxC,CAAC,CAAC,CAAC;aACN;QACL,CAAC,CAAC,CAAC;QACH,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;IACjC,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,QAAwB,EAAE,MAAsB;QAC9D,IAAI,CAAC,QAAQ,EAAE;YACX,MAAM,kBAAkB,GAAG,sBAAsB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC7D,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,kBAAkB,CAAW,CAAC;SAC5D;QAED,MAAM,QAAQ,GAAG,QAAQ,CAAC;QAC1B,IAAI,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAEvC,IAAI,CAAC,OAAO,EAAE;YACV,MAAM,KAAN,MAAM,GAAK,IAAI,CAAC,MAAM,CAAC,gBAAgB,EAAE,EAAC;YAC1C,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,QAAQ,EAAE,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;YAClF,OAAO,GAAG,IAAI,IAAI,CAAC,kBAAkB,CAAC;gBAClC,EAAE,EAAE,aAAa,CAAC,EAAE;gBACpB,IAAI,EAAE,aAAa,CAAC,IAAI;gBACxB,MAAM;aACT,CAAC,CAAC;YACH,IAAI,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC;SACpC;QAED,OAAO,OAAO,CAAC;IACnB,CAAC;IAED,YAAY,CAAC,OAAsC;QAC/C,MAAM,KAAK,GAAG,OAAO,CAAC,EAAE,CAAC;QACzB,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAEzB,IAAI,OAAO,CAAC,IAAI,EAAE;YACd,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;YAC7B,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;SAC9B;IACL,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,mBAAmB,CAAC,eAAuB,EAAE,sBAA8B,EAAE,SAAwB;QACjH,MAAM,EACF,mBAAmB,EACnB,6BAA6B,GAChC,GAAG,IAAI,CAAC,0BAA0B,CAAC,SAAS,EAAE,sBAAsB,CAAC,CAAC;QAEvE,MAAM,aAAa,GAAG,mBAAmB,CAAC,eAAe,CAAC,CAAC;QAC3D,MAAM,eAAe,GAAG,MAAM,aAAa,CAAC,GAAG,EAAE,CAAC;QAClD,IAAI,eAAe;YAAE,OAAO,eAAe,CAAC;QAE5C,MAAM,uBAAuB,GAAG,6BAA6B,EAAE,CAAC;QAChE,OAAO,uBAAuB,CAAC,WAAW,CAAC,eAAe,CAAC,CAAC;IAChE,CAAC;IAES,0BAA0B,CAAC,MAAqB,EAAE,sBAA8B;QACtF,qBAAqB;QACrB,MAAM,UAAU,GAAG,sBAAsB,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,GAAG,sBAAsB,CAAC,KAAK,CAAC,CAAC,CAAgB,CAAC;QAC5G,sBAAsB;QACtB,MAAM,oBAAoB,GAAG,GAAG,UAAU,GAA4B,CAAC;QAEvE,OAAO;YACH,mBAAmB,EAAE,MAAM,CAAC,UAAW,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC;YACrD,6BAA6B,EAAE,MAAM,CAAC,oBAAqB,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC;SAC5E,CAAC;IACN,CAAC;IAES,kBAAkB,CAAC,OAAU;QACnC,MAAM,KAAK,GAAG,OAAO,CAAC,EAAE,CAAC;QACzB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;QAE/B,IAAI,OAAO,CAAC,IAAI,EAAE;YACd,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;YAC7B,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;SACpC;IACL,CAAC;;AArHL,wCAsHC;AArHG;;;;WAA0C,IAAI,GAAG,EAA+B;GAAC"}
|