@crawlee/core 3.0.5-beta.5 → 3.0.5-beta.51
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/configuration.d.ts +12 -2
- package/configuration.d.ts.map +1 -1
- package/configuration.js +13 -3
- package/configuration.js.map +1 -1
- package/crawlers/statistics.d.ts +16 -0
- package/crawlers/statistics.d.ts.map +1 -1
- package/crawlers/statistics.js +50 -1
- package/crawlers/statistics.js.map +1 -1
- package/enqueue_links/enqueue_links.d.ts.map +1 -1
- package/enqueue_links/enqueue_links.js +15 -4
- package/enqueue_links/enqueue_links.js.map +1 -1
- package/enqueue_links/shared.d.ts +1 -0
- package/enqueue_links/shared.d.ts.map +1 -1
- package/enqueue_links/shared.js +26 -5
- package/enqueue_links/shared.js.map +1 -1
- package/events/event_manager.d.ts.map +1 -1
- package/events/event_manager.js +2 -0
- package/events/event_manager.js.map +1 -1
- package/except.d.ts +7 -0
- package/except.d.ts.map +1 -0
- package/except.js +4 -0
- package/except.js.map +1 -0
- package/index.d.ts +1 -1
- package/index.d.ts.map +1 -1
- package/index.mjs +1 -0
- package/package.json +5 -4
- package/request.d.ts +1 -1
- package/request.js +1 -1
- package/router.d.ts +11 -2
- package/router.d.ts.map +1 -1
- package/router.js.map +1 -1
- package/storages/dataset.d.ts +4 -0
- package/storages/dataset.d.ts.map +1 -1
- package/storages/dataset.js +7 -0
- package/storages/dataset.js.map +1 -1
- package/storages/key_value_store.d.ts +63 -4
- package/storages/key_value_store.d.ts.map +1 -1
- package/storages/key_value_store.js +4 -3
- package/storages/key_value_store.js.map +1 -1
- package/storages/storage_manager.js.map +1 -1
- package/tsconfig.build.tsbuildinfo +1 -1
package/configuration.d.ts
CHANGED
|
@@ -125,10 +125,16 @@ export interface ConfigurationOptions {
|
|
|
125
125
|
/**
|
|
126
126
|
* Sets the log level to the given value.
|
|
127
127
|
*
|
|
128
|
-
* Alternative to `CRAWLEE_LOG_LEVEL
|
|
128
|
+
* Alternative to `CRAWLEE_LOG_LEVEL` environment variable.
|
|
129
129
|
* @default 'INFO'
|
|
130
130
|
*/
|
|
131
131
|
logLevel?: LogLevel | LogLevel[keyof LogLevel];
|
|
132
|
+
/**
|
|
133
|
+
* Defines whether the storage client used should persist the data it stores.
|
|
134
|
+
*
|
|
135
|
+
* Alternative to `CRAWLEE_PERSIST_STORAGE` environment variable.
|
|
136
|
+
*/
|
|
137
|
+
persistStorage?: boolean;
|
|
132
138
|
}
|
|
133
139
|
/**
|
|
134
140
|
* `Configuration` is a value object holding Crawlee configuration. By default, there is a
|
|
@@ -179,7 +185,9 @@ export interface ConfigurationOptions {
|
|
|
179
185
|
* `defaultDatasetId` | `CRAWLEE_DEFAULT_DATASET_ID` | `'default'`
|
|
180
186
|
* `defaultKeyValueStoreId` | `CRAWLEE_DEFAULT_KEY_VALUE_STORE_ID` | `'default'`
|
|
181
187
|
* `defaultRequestQueueId` | `CRAWLEE_DEFAULT_REQUEST_QUEUE_ID` | `'default'`
|
|
182
|
-
* `persistStateIntervalMillis` | `CRAWLEE_PERSIST_STATE_INTERVAL_MILLIS` | `
|
|
188
|
+
* `persistStateIntervalMillis` | `CRAWLEE_PERSIST_STATE_INTERVAL_MILLIS` | `60_000`
|
|
189
|
+
* `purgeOnStart` | `CRAWLEE_PURGE_ON_START` | `true`
|
|
190
|
+
* `persistStorage` | `CRAWLEE_PERSIST_STORAGE` | `true`
|
|
183
191
|
*
|
|
184
192
|
* ## Advanced Configuration Options
|
|
185
193
|
*
|
|
@@ -189,6 +197,8 @@ export interface ConfigurationOptions {
|
|
|
189
197
|
* `xvfb` | `CRAWLEE_XVFB` | -
|
|
190
198
|
* `chromeExecutablePath` | `CRAWLEE_CHROME_EXECUTABLE_PATH` | -
|
|
191
199
|
* `defaultBrowserPath` | `CRAWLEE_DEFAULT_BROWSER_PATH` | -
|
|
200
|
+
* `disableBrowserSandbox` | `CRAWLEE_DISABLE_BROWSER_SANDBOX` | -
|
|
201
|
+
* `availableMemoryRatio` | `CRAWLEE_AVAILABLE_MEMORY_RATIO` | `0.25`
|
|
192
202
|
*/
|
|
193
203
|
export declare class Configuration {
|
|
194
204
|
/**
|
package/configuration.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"configuration.d.ts","sourceRoot":"","sources":["../src/configuration.ts"],"names":[],"mappings":";AAAA,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AACpE,OAAO,EAAE,aAAa,EAAE,MAAM,yBAAyB,CAAC;AAGxD,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAErD,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAChE,OAAY,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAE3C,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAG7C,MAAM,WAAW,oBAAoB;IACjC;;;OAGG;IACH,aAAa,CAAC,EAAE,aAAa,CAAC;IAE9B;;;OAGG;IACH,YAAY,CAAC,EAAE,YAAY,CAAC;IAE5B;;;OAGG;IACH,oBAAoB,CAAC,EAAE,UAAU,CAAC;IAElC;;;;;OAKG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAE1B;;;;;OAKG;IACH,YAAY,CAAC,EAAE,OAAO,CAAC;IAEvB;;;;;OAKG;IACH,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAEhC;;;;;OAKG;IACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAE/B;;;;OAIG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IAEzB;;;;;;OAMG;IACH,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAE9B;;;;;OAKG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IAEtB;;;;;OAKG;IACH,0BAA0B,CAAC,EAAE,MAAM,CAAC;IAEpC;;;OAGG;IACH,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAElC;;;;;;OAMG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB;;;;;OAKG;IACH,QAAQ,CAAC,EAAE,OAAO,CAAC;IAEnB;;;;;OAKG;IACH,IAAI,CAAC,EAAE,OAAO,CAAC;IAEf;;;;OAIG;IACH,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAE9B;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAE5B;;;;OAIG;IACH,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAEhC;;;;;OAKG;IACH,QAAQ,CAAC,EAAE,QAAQ,GAAG,QAAQ,CAAC,MAAM,QAAQ,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"configuration.d.ts","sourceRoot":"","sources":["../src/configuration.ts"],"names":[],"mappings":";AAAA,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AACpE,OAAO,EAAE,aAAa,EAAE,MAAM,yBAAyB,CAAC;AAGxD,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAErD,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAChE,OAAY,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAE3C,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAG7C,MAAM,WAAW,oBAAoB;IACjC;;;OAGG;IACH,aAAa,CAAC,EAAE,aAAa,CAAC;IAE9B;;;OAGG;IACH,YAAY,CAAC,EAAE,YAAY,CAAC;IAE5B;;;OAGG;IACH,oBAAoB,CAAC,EAAE,UAAU,CAAC;IAElC;;;;;OAKG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAE1B;;;;;OAKG;IACH,YAAY,CAAC,EAAE,OAAO,CAAC;IAEvB;;;;;OAKG;IACH,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAEhC;;;;;OAKG;IACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAE/B;;;;OAIG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IAEzB;;;;;;OAMG;IACH,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAE9B;;;;;OAKG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IAEtB;;;;;OAKG;IACH,0BAA0B,CAAC,EAAE,MAAM,CAAC;IAEpC;;;OAGG;IACH,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAElC;;;;;;OAMG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB;;;;;OAKG;IACH,QAAQ,CAAC,EAAE,OAAO,CAAC;IAEnB;;;;;OAKG;IACH,IAAI,CAAC,EAAE,OAAO,CAAC;IAEf;;;;OAIG;IACH,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAE9B;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAE5B;;;;OAIG;IACH,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAEhC;;;;;OAKG;IACH,QAAQ,CAAC,EAAE,QAAQ,GAAG,QAAQ,CAAC,MAAM,QAAQ,CAAC,CAAC;IAE/C;;;;OAIG;IACH,cAAc,CAAC,EAAE,OAAO,CAAC;CAC5B;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+DG;AACH,qBAAa,aAAa;IACtB;;OAEG;IACH,SAAS,CAAC,MAAM,CAAC,OAAO,EAAE,UAAU,CAgBlC;IAEF,SAAS,CAAC,MAAM,CAAC,YAAY,WAAmF;IAEhH,SAAS,CAAC,MAAM,CAAC,YAAY,WAA8E;IAE3G,SAAS,CAAC,MAAM,CAAC,QAAQ,EAAE,UAAU,CAanC;IAEF;;;OAGG;IACH,MAAM,CAAC,OAAO,mCAA0C;IAExD,SAAS,CAAC,OAAO,EAAG,GAAG,CAAC,MAAM,oBAAoB,EAAE,oBAAoB,CAAC,MAAM,oBAAoB,CAAC,CAAC,CAAC;IACtG,SAAS,CAAC,QAAQ,uBAA8B;IAEhD,gBAAgB;IAChB,MAAM,CAAC,YAAY,CAAC,EAAE,aAAa,CAAC;IAEpC;;OAEG;gBACS,OAAO,GAAE,oBAAyB;IAe9C;;;;OAIG;IACH,GAAG,CAAC,CAAC,SAAS,MAAM,oBAAoB,EAAE,CAAC,SAAS,oBAAoB,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,YAAY,CAAC,EAAE,CAAC,GAAG,CAAC;IA2BzG,SAAS,CAAC,aAAa,CAAC,GAAG,EAAE,MAAM,oBAAoB,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO;IAazF;;;OAGG;IACH,GAAG,CAAC,GAAG,EAAE,MAAM,oBAAoB,EAAE,KAAK,CAAC,EAAE,GAAG,GAAG,IAAI;IAIvD;;;;;;;;OAQG;IACH,gBAAgB,IAAI,aAAa;IASjC,eAAe,IAAI,YAAY;IAe/B;;;OAGG;IACH,mBAAmB,CAAC,OAAO,GAAE,oBAAyB,GAAG,aAAa;IAiBtE,gBAAgB,CAAC,MAAM,EAAE,aAAa,GAAG,IAAI;IAI7C,eAAe,CAAC,MAAM,EAAE,YAAY,GAAG,IAAI;IAI3C;;OAEG;IACH,MAAM,CAAC,eAAe,IAAI,aAAa;IASvC;;OAEG;IACH,MAAM,CAAC,gBAAgB,IAAI,aAAa;IAIxC;;OAEG;IACH,MAAM,CAAC,eAAe,IAAI,YAAY;IAItC;;;OAGG;IACH,MAAM,CAAC,gBAAgB,IAAI,IAAI;IAI/B,SAAS,CAAC,YAAY,CAAC,OAAO,EAAE,oBAAoB;CAgBvD"}
|
package/configuration.js
CHANGED
|
@@ -59,7 +59,9 @@ const events_1 = require("./events");
|
|
|
59
59
|
* `defaultDatasetId` | `CRAWLEE_DEFAULT_DATASET_ID` | `'default'`
|
|
60
60
|
* `defaultKeyValueStoreId` | `CRAWLEE_DEFAULT_KEY_VALUE_STORE_ID` | `'default'`
|
|
61
61
|
* `defaultRequestQueueId` | `CRAWLEE_DEFAULT_REQUEST_QUEUE_ID` | `'default'`
|
|
62
|
-
* `persistStateIntervalMillis` | `CRAWLEE_PERSIST_STATE_INTERVAL_MILLIS` | `
|
|
62
|
+
* `persistStateIntervalMillis` | `CRAWLEE_PERSIST_STATE_INTERVAL_MILLIS` | `60_000`
|
|
63
|
+
* `purgeOnStart` | `CRAWLEE_PURGE_ON_START` | `true`
|
|
64
|
+
* `persistStorage` | `CRAWLEE_PERSIST_STORAGE` | `true`
|
|
63
65
|
*
|
|
64
66
|
* ## Advanced Configuration Options
|
|
65
67
|
*
|
|
@@ -69,6 +71,8 @@ const events_1 = require("./events");
|
|
|
69
71
|
* `xvfb` | `CRAWLEE_XVFB` | -
|
|
70
72
|
* `chromeExecutablePath` | `CRAWLEE_CHROME_EXECUTABLE_PATH` | -
|
|
71
73
|
* `defaultBrowserPath` | `CRAWLEE_DEFAULT_BROWSER_PATH` | -
|
|
74
|
+
* `disableBrowserSandbox` | `CRAWLEE_DISABLE_BROWSER_SANDBOX` | -
|
|
75
|
+
* `availableMemoryRatio` | `CRAWLEE_AVAILABLE_MEMORY_RATIO` | `0.25`
|
|
72
76
|
*/
|
|
73
77
|
class Configuration {
|
|
74
78
|
/**
|
|
@@ -176,7 +180,11 @@ class Configuration {
|
|
|
176
180
|
if (this.services.has(cacheKey)) {
|
|
177
181
|
return this.services.get(cacheKey);
|
|
178
182
|
}
|
|
179
|
-
const storage = new memory_storage_1.MemoryStorage(
|
|
183
|
+
const storage = new memory_storage_1.MemoryStorage({
|
|
184
|
+
persistStorage: this.get('persistStorage'),
|
|
185
|
+
// Override persistStorage if user provides it via storageClientOptions
|
|
186
|
+
...options,
|
|
187
|
+
});
|
|
180
188
|
this.services.set(cacheKey, storage);
|
|
181
189
|
return storage;
|
|
182
190
|
}
|
|
@@ -254,13 +262,14 @@ Object.defineProperty(Configuration, "ENV_MAP", {
|
|
|
254
262
|
CRAWLEE_DEFAULT_BROWSER_PATH: 'defaultBrowserPath',
|
|
255
263
|
CRAWLEE_DISABLE_BROWSER_SANDBOX: 'disableBrowserSandbox',
|
|
256
264
|
CRAWLEE_LOG_LEVEL: 'logLevel',
|
|
265
|
+
CRAWLEE_PERSIST_STORAGE: 'persistStorage',
|
|
257
266
|
}
|
|
258
267
|
});
|
|
259
268
|
Object.defineProperty(Configuration, "BOOLEAN_VARS", {
|
|
260
269
|
enumerable: true,
|
|
261
270
|
configurable: true,
|
|
262
271
|
writable: true,
|
|
263
|
-
value: ['purgeOnStart', 'headless', 'xvfb', 'disableBrowserSandbox']
|
|
272
|
+
value: ['purgeOnStart', 'headless', 'xvfb', 'disableBrowserSandbox', 'persistStorage']
|
|
264
273
|
});
|
|
265
274
|
Object.defineProperty(Configuration, "INTEGER_VARS", {
|
|
266
275
|
enumerable: true,
|
|
@@ -284,6 +293,7 @@ Object.defineProperty(Configuration, "DEFAULTS", {
|
|
|
284
293
|
headless: true,
|
|
285
294
|
persistStateIntervalMillis: 60000,
|
|
286
295
|
systemInfoIntervalMillis: 60000,
|
|
296
|
+
persistStorage: true,
|
|
287
297
|
}
|
|
288
298
|
});
|
|
289
299
|
/**
|
package/configuration.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"configuration.js","sourceRoot":"","sources":["../src/configuration.ts"],"names":[],"mappings":";;;;AACA,4DAAwD;AACxD,uCAAwD;AACxD,yCAAiC;AACjC,uDAAqD;AACrD,6CAA2C;AAE3C,0DAA2C;AAC3C,yCAAqC;AAErC,qCAA6C;
|
|
1
|
+
{"version":3,"file":"configuration.js","sourceRoot":"","sources":["../src/configuration.ts"],"names":[],"mappings":";;;;AACA,4DAAwD;AACxD,uCAAwD;AACxD,yCAAiC;AACjC,uDAAqD;AACrD,6CAA2C;AAE3C,0DAA2C;AAC3C,yCAAqC;AAErC,qCAA6C;AAyJ7C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+DG;AACH,MAAa,aAAa;IAqDtB;;OAEG;IACH,YAAY,UAAgC,EAAE;QAT9C;;;;;WAAsG;QACtG;;;;mBAAqB,IAAI,GAAG,EAAmB;WAAC;QAS5C,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;QAE3B,oEAAoE;QACpE,0BAAY,CAAC,mBAAmB,GAAG,EAAE,CAAC;QAEtC,6DAA6D;QAC7D,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QAEtC,IAAI,QAAQ,EAAE;YACV,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,cAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAyB,CAAC,CAAC;YACvH,aAAG,CAAC,QAAQ,CAAC,KAAiB,CAAC,CAAC;SACnC;IACL,CAAC;IAED;;;;OAIG;IACH,GAAG,CAA0E,GAAM,EAAE,YAAgB;QACjG,iHAAiH;QACjH,IAAI,QAA4B,CAAC;QAEjC,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,IAAA,kBAAO,EAAC,aAAa,CAAC,OAAO,CAAC,EAAE;YACjD,IAAI,GAAG,KAAK,CAAC,EAAE;gBACX,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,CAAW,CAAC,CAAC;gBAEpC,IAAI,QAAQ,EAAE;oBACV,MAAM;iBACT;aACJ;SACJ;QAED,IAAI,QAAQ,IAAI,IAAI,EAAE;YAClB,OAAO,IAAI,CAAC,aAAa,CAAC,GAAG,EAAE,QAAQ,CAAM,CAAC;SACjD;QAED,+BAA+B;QAC/B,IAAI,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE;YACvB,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAM,CAAC;SACrC;QAED,uBAAuB;QACvB,OAAO,CAAC,YAAY,IAAI,aAAa,CAAC,QAAQ,CAAC,GAA0C,CAAC,IAAI,QAAQ,CAAM,CAAC;IACjH,CAAC;IAES,aAAa,CAAC,GAA+B,EAAE,KAAgC;QACrF,IAAI,aAAa,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE;YAC1C,OAAO,CAAC,KAAK,CAAC;SACjB;QAED,IAAI,aAAa,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE;YAC1C,wDAAwD;YACxD,OAAO,CAAC,CAAC,GAAG,EAAE,OAAO,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;SACpE;QAED,OAAO,KAAK,CAAC;IACjB,CAAC;IAED;;;OAGG;IACH,GAAG,CAAC,GAA+B,EAAE,KAAW;QAC5C,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;IACjC,CAAC;IAED;;;;;;;;OAQG;IACH,gBAAgB;QACZ,IAAI,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,EAAE;YACnC,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAkB,CAAC;SAC7D;QAED,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAe,CAAC;QACvE,OAAO,IAAI,CAAC,mBAAmB,CAAC,OAAO,CAAC,CAAC;IAC7C,CAAC;IAED,eAAe;QACX,IAAI,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,EAAE;YAClC,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAiB,CAAC;SAC3D;QAED,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,cAAc,CAAC,EAAE;YACnC,OAAO,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,cAAc,CAAiB,CAAC;SAC5D;QAED,MAAM,YAAY,GAAG,IAAI,0BAAiB,CAAC,IAAI,CAAC,CAAC;QACjD,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,cAAc,EAAE,YAAY,CAAC,CAAC;QAEhD,OAAO,YAAY,CAAC;IACxB,CAAC;IAED;;;OAGG;IACH,mBAAmB,CAAC,UAAgC,EAAE;QAClD,MAAM,QAAQ,GAAG,iBAAiB,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,CAAC;QAE5D,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE;YAC7B,OAAO,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAkB,CAAC;SACvD;QAED,MAAM,OAAO,GAAG,IAAI,8BAAa,CAAC;YAC9B,cAAc,EAAE,IAAI,CAAC,GAAG,CAAC,gBAAgB,CAAC;YAC1C,uEAAuE;YACvE,GAAG,OAAO;SACb,CAAC,CAAC;QACH,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAErC,OAAO,OAAO,CAAC;IACnB,CAAC;IAED,gBAAgB,CAAC,MAAqB;QAClC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,EAAE,MAAM,CAAC,CAAC;IAC9C,CAAC;IAED,eAAe,CAAC,MAAoB;QAChC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,MAAM,CAAC,CAAC;IAC7C,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,eAAe;QAClB,IAAI,aAAa,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE;YAClC,OAAO,aAAa,CAAC,OAAO,CAAC,QAAQ,EAAG,CAAC;SAC5C;QAED,aAAa,CAAC,YAAY,KAA1B,aAAa,CAAC,YAAY,GAAK,IAAI,aAAa,EAAE,EAAC;QACnD,OAAO,aAAa,CAAC,YAAY,CAAC;IACtC,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,gBAAgB;QACnB,OAAO,IAAI,CAAC,eAAe,EAAE,CAAC,gBAAgB,EAAE,CAAC;IACrD,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,eAAe;QAClB,OAAO,IAAI,CAAC,eAAe,EAAE,CAAC,eAAe,EAAE,CAAC;IACpD,CAAC;IAED;;;OAGG;IACH,MAAM,CAAC,gBAAgB;QACnB,OAAO,IAAI,CAAC,YAAY,CAAC;IAC7B,CAAC;IAES,YAAY,CAAC,OAA6B;QAChD,8DAA8D;QAC9D,MAAM,IAAI,GAAG,IAAA,gBAAI,EAAC,OAAO,CAAC,GAAG,EAAE,EAAE,cAAc,CAAC,CAAC;QAEjD,IAAI,IAAA,yBAAc,EAAC,IAAI,CAAC,EAAE;YACtB,IAAI;gBACA,MAAM,IAAI,GAAG,IAAA,uBAAY,EAAC,IAAI,CAAC,CAAC;gBAChC,MAAM,qBAAqB,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;gBAC1D,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,qBAAqB,CAAC,CAAC;aACjD;YAAC,MAAM;gBACJ,SAAS;aACZ;SACJ;QAED,IAAI,CAAC,OAAO,GAAG,IAAI,GAAG,CAAC,IAAA,kBAAO,EAAC,OAAO,CAAC,CAAC,CAAC;IAC7C,CAAC;;AA3OL,sCA4OC;AA3OG;;GAEG;AACc;;;;WAAsB;QACnC,8BAA8B,EAAE,sBAAsB;QACtD,sBAAsB,EAAE,cAAc;QACtC,qBAAqB,EAAE,cAAc;QACrC,0BAA0B,EAAE,kBAAkB;QAC9C,kCAAkC,EAAE,wBAAwB;QAC5D,gCAAgC,EAAE,uBAAuB;QACzD,iBAAiB,EAAE,UAAU;QAC7B,qCAAqC,EAAE,4BAA4B;QACnE,gBAAgB,EAAE,UAAU;QAC5B,YAAY,EAAE,MAAM;QACpB,8BAA8B,EAAE,sBAAsB;QACtD,4BAA4B,EAAE,oBAAoB;QAClD,+BAA+B,EAAE,uBAAuB;QACxD,iBAAiB,EAAE,UAAU;QAC7B,uBAAuB,EAAE,gBAAgB;KAC5C;GAAC;AAEe;;;;WAAe,CAAC,cAAc,EAAE,UAAU,EAAE,MAAM,EAAE,uBAAuB,EAAE,gBAAgB,CAAC;GAAC;AAE/F;;;;WAAe,CAAC,cAAc,EAAE,4BAA4B,EAAE,0BAA0B,CAAC;GAAC;AAE1F;;;;WAAuB;QACpC,sBAAsB,EAAE,SAAS;QACjC,gBAAgB,EAAE,SAAS;QAC3B,qBAAqB,EAAE,SAAS;QAChC,QAAQ,EAAE,OAAO;QACjB,eAAe,EAAE,IAAI;QACrB,oBAAoB,EAAE,IAAI;QAC1B,oBAAoB,EAAE,EAAE;QACxB,YAAY,EAAE,IAAI;QAClB,QAAQ,EAAE,IAAI;QACd,0BAA0B,EAAE,KAAM;QAClC,wBAAwB,EAAE,KAAM;QAChC,cAAc,EAAE,IAAI;KACvB;GAAC;AAEF;;;GAGG;AACI;;;;WAAU,IAAI,oCAAiB,EAAiB;GAAC"}
|
package/crawlers/statistics.d.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { ErrorTracker } from '@crawlee/utils';
|
|
1
2
|
import { KeyValueStore } from '../storages/key_value_store';
|
|
2
3
|
import { Configuration } from '../configuration';
|
|
3
4
|
/**
|
|
@@ -23,6 +24,14 @@ declare class Job {
|
|
|
23
24
|
*/
|
|
24
25
|
export declare class Statistics {
|
|
25
26
|
private static id;
|
|
27
|
+
/**
|
|
28
|
+
* An error tracker for final retry errors.
|
|
29
|
+
*/
|
|
30
|
+
errorTracker: ErrorTracker;
|
|
31
|
+
/**
|
|
32
|
+
* An error tracker for retry errors prior to the final retry.
|
|
33
|
+
*/
|
|
34
|
+
errorTrackerRetry: ErrorTracker;
|
|
26
35
|
/**
|
|
27
36
|
* Statistic instance id.
|
|
28
37
|
*/
|
|
@@ -53,6 +62,10 @@ export declare class Statistics {
|
|
|
53
62
|
* Set the current statistic instance to pristine values
|
|
54
63
|
*/
|
|
55
64
|
reset(): void;
|
|
65
|
+
/**
|
|
66
|
+
* Increments the status code counter.
|
|
67
|
+
*/
|
|
68
|
+
registerStatusCode(code: number): void;
|
|
56
69
|
/**
|
|
57
70
|
* Starts a job
|
|
58
71
|
* @ignore
|
|
@@ -144,6 +157,9 @@ export interface StatisticState {
|
|
|
144
157
|
crawlerFinishedAt: Date | string | null;
|
|
145
158
|
crawlerRuntimeMillis: number;
|
|
146
159
|
statsPersistedAt: Date | string | null;
|
|
160
|
+
errors: Record<string, unknown>;
|
|
161
|
+
retryErrors: Record<string, unknown>;
|
|
162
|
+
requestsWithStatusCode: Record<string, number>;
|
|
147
163
|
}
|
|
148
164
|
export {};
|
|
149
165
|
//# sourceMappingURL=statistics.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"statistics.d.ts","sourceRoot":"","sources":["../../src/crawlers/statistics.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"statistics.d.ts","sourceRoot":"","sources":["../../src/crawlers/statistics.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAE9C,OAAO,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AAG5D,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEjD;;GAEG;AACH,cAAM,GAAG;IACL,OAAO,CAAC,SAAS,CAAuB;IACxC,OAAO,CAAC,IAAI,CAAK;IACjB,OAAO,CAAC,cAAc,CAAC,CAAS;IAEhC,GAAG;IAKH,MAAM;IAKN,UAAU;CAGb;AAWD;;;;;;;;;GASG;AACH,qBAAa,UAAU;IACnB,OAAO,CAAC,MAAM,CAAC,EAAE,CAAK;IAEtB;;OAEG;IACH,YAAY,eAAwC;IAEpD;;OAEG;IACH,iBAAiB,eAAwC;IAEzD;;OAEG;IACH,QAAQ,CAAC,EAAE,SAAmB;IAE9B;;OAEG;IACH,KAAK,EAAG,cAAc,CAAC;IAEvB;;OAEG;IACH,QAAQ,CAAC,qBAAqB,EAAE,MAAM,EAAE,CAAM;IAE9C,OAAO,CAAC,aAAa,CAAC,CAA4B;IAClD,OAAO,CAAC,eAAe,CAAuC;IAC9D,OAAO,CAAC,iBAAiB,CAAS;IAClC,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,QAAQ,CAAsB;IACtC,OAAO,CAAC,kBAAkB,CAAmC;IAC7D,OAAO,CAAC,QAAQ,CAAC,GAAG,CAA8C;IAClE,OAAO,CAAC,aAAa,CAAU;IAC/B,OAAO,CAAC,WAAW,CAAU;IAC7B,OAAO,CAAC,MAAM,CAAe;IAE7B;;OAEG;gBACS,OAAO,GAAE,iBAAsB;IAyB3C;;OAEG;IACH,KAAK;IA8BL;;OAEG;IACH,kBAAkB,CAAC,IAAI,EAAE,MAAM;IAU/B;;;OAGG;IACH,QAAQ,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM;IAO5B;;;OAGG;IACH,SAAS,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM;IAY7B;;;OAGG;IACH,OAAO,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM;IAS3B;;OAEG;IACH,SAAS;;;;;;;;;IAqBT;;;OAGG;IACG,cAAc;IAmBpB;;OAEG;IACG,aAAa;IAQnB,SAAS,CAAC,qBAAqB,CAAC,GAAG,EAAE,GAAG;IAQxC;;OAEG;IACG,YAAY;IAWlB;;OAEG;cACa,oBAAoB;IAwCpC,SAAS,CAAC,SAAS,IAAI,IAAI;IAU3B;;;OAGG;IACH,MAAM,IAAI,uBAAuB;CAyBpC;AAED,UAAU,iBAAiB;IACvB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,aAAa,CAAC,EAAE,aAAa,CAAC;IAC9B,MAAM,CAAC,EAAE,aAAa,CAAC;CAC1B;AAED;;GAEG;AACH,MAAM,WAAW,uBAAwB,SAAQ,IAAI,CAAC,cAAc,EAAE,kBAAkB,CAAC;IACrF,qBAAqB,EAAE,MAAM,EAAE,CAAC;IAChC,OAAO,EAAE,MAAM,CAAC;IAChB,8BAA8B,EAAE,MAAM,CAAC;IACvC,gCAAgC,EAAE,MAAM,CAAC;IACzC,yBAAyB,EAAE,MAAM,CAAC;IAClC,uBAAuB,EAAE,MAAM,CAAC;IAChC,0BAA0B,EAAE,MAAM,CAAC;IACnC,aAAa,EAAE,MAAM,CAAC;IACtB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,yBAAyB,EAAE,MAAM,CAAC;IAClC,gBAAgB,EAAE,MAAM,CAAC;CAC5B;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC3B,gBAAgB,EAAE,MAAM,CAAC;IACzB,cAAc,EAAE,MAAM,CAAC;IACvB,eAAe,EAAE,MAAM,CAAC;IACxB,uBAAuB,EAAE,MAAM,CAAC;IAChC,yBAAyB,EAAE,MAAM,CAAC;IAClC,wBAAwB,EAAE,MAAM,CAAC;IACjC,wBAAwB,EAAE,MAAM,CAAC;IACjC,gCAAgC,EAAE,MAAM,CAAC;IACzC,kCAAkC,EAAE,MAAM,CAAC;IAC3C,gBAAgB,EAAE,IAAI,GAAG,MAAM,GAAG,IAAI,CAAC;IACvC,iBAAiB,EAAE,IAAI,GAAG,MAAM,GAAG,IAAI,CAAC;IACxC,oBAAoB,EAAE,MAAM,CAAC;IAC7B,gBAAgB,EAAE,IAAI,GAAG,MAAM,GAAG,IAAI,CAAC;IACvC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAChC,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACrC,sBAAsB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAClD"}
|
package/crawlers/statistics.js
CHANGED
|
@@ -3,6 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
3
3
|
exports.Statistics = void 0;
|
|
4
4
|
const tslib_1 = require("tslib");
|
|
5
5
|
const ow_1 = tslib_1.__importDefault(require("ow"));
|
|
6
|
+
const utils_1 = require("@crawlee/utils");
|
|
6
7
|
const log_1 = require("../log");
|
|
7
8
|
const key_value_store_1 = require("../storages/key_value_store");
|
|
8
9
|
const configuration_1 = require("../configuration");
|
|
@@ -42,6 +43,14 @@ class Job {
|
|
|
42
43
|
return Math.max(0, this.runs - 1);
|
|
43
44
|
}
|
|
44
45
|
}
|
|
46
|
+
const errorTrackerConfig = {
|
|
47
|
+
showErrorCode: true,
|
|
48
|
+
showErrorName: true,
|
|
49
|
+
showStackTrace: true,
|
|
50
|
+
showFullStack: false,
|
|
51
|
+
showErrorMessage: true,
|
|
52
|
+
showFullMessage: false,
|
|
53
|
+
};
|
|
45
54
|
/**
|
|
46
55
|
* The statistics class provides an interface to collecting and logging run
|
|
47
56
|
* statistics for requests.
|
|
@@ -57,6 +66,24 @@ class Statistics {
|
|
|
57
66
|
* @internal
|
|
58
67
|
*/
|
|
59
68
|
constructor(options = {}) {
|
|
69
|
+
/**
|
|
70
|
+
* An error tracker for final retry errors.
|
|
71
|
+
*/
|
|
72
|
+
Object.defineProperty(this, "errorTracker", {
|
|
73
|
+
enumerable: true,
|
|
74
|
+
configurable: true,
|
|
75
|
+
writable: true,
|
|
76
|
+
value: new utils_1.ErrorTracker(errorTrackerConfig)
|
|
77
|
+
});
|
|
78
|
+
/**
|
|
79
|
+
* An error tracker for retry errors prior to the final retry.
|
|
80
|
+
*/
|
|
81
|
+
Object.defineProperty(this, "errorTrackerRetry", {
|
|
82
|
+
enumerable: true,
|
|
83
|
+
configurable: true,
|
|
84
|
+
writable: true,
|
|
85
|
+
value: new utils_1.ErrorTracker(errorTrackerConfig)
|
|
86
|
+
});
|
|
60
87
|
/**
|
|
61
88
|
* Statistic instance id.
|
|
62
89
|
*/
|
|
@@ -163,6 +190,8 @@ class Statistics {
|
|
|
163
190
|
* Set the current statistic instance to pristine values
|
|
164
191
|
*/
|
|
165
192
|
reset() {
|
|
193
|
+
this.errorTracker.reset();
|
|
194
|
+
this.errorTrackerRetry.reset();
|
|
166
195
|
this.state = {
|
|
167
196
|
requestsFinished: 0,
|
|
168
197
|
requestsFailed: 0,
|
|
@@ -177,12 +206,25 @@ class Statistics {
|
|
|
177
206
|
crawlerFinishedAt: null,
|
|
178
207
|
statsPersistedAt: null,
|
|
179
208
|
crawlerRuntimeMillis: 0,
|
|
209
|
+
requestsWithStatusCode: {},
|
|
210
|
+
errors: this.errorTracker.result,
|
|
211
|
+
retryErrors: this.errorTrackerRetry.result,
|
|
180
212
|
};
|
|
181
213
|
this.requestRetryHistogram.length = 0;
|
|
182
214
|
this.requestsInProgress.clear();
|
|
183
215
|
this.instanceStart = Date.now();
|
|
184
216
|
this._teardown();
|
|
185
217
|
}
|
|
218
|
+
/**
|
|
219
|
+
* Increments the status code counter.
|
|
220
|
+
*/
|
|
221
|
+
registerStatusCode(code) {
|
|
222
|
+
const s = String(code);
|
|
223
|
+
if (this.state.requestsWithStatusCode[s] === undefined) {
|
|
224
|
+
this.state.requestsWithStatusCode[s] = 0;
|
|
225
|
+
}
|
|
226
|
+
this.state.requestsWithStatusCode[s]++;
|
|
227
|
+
}
|
|
186
228
|
/**
|
|
187
229
|
* Starts a job
|
|
188
230
|
* @ignore
|
|
@@ -339,7 +381,7 @@ class Statistics {
|
|
|
339
381
|
// merge all the current state information that can be used from the outside
|
|
340
382
|
// without the need to reconstruct for the sake of stats.calculate()
|
|
341
383
|
// omit duplicated information
|
|
342
|
-
|
|
384
|
+
const result = {
|
|
343
385
|
...this.state,
|
|
344
386
|
crawlerLastStartTimestamp: this.instanceStart,
|
|
345
387
|
crawlerFinishedAt: this.state.crawlerFinishedAt ? new Date(this.state.crawlerFinishedAt).toISOString() : null,
|
|
@@ -349,6 +391,13 @@ class Statistics {
|
|
|
349
391
|
statsPersistedAt: new Date().toISOString(),
|
|
350
392
|
...this.calculate(),
|
|
351
393
|
};
|
|
394
|
+
Reflect.deleteProperty(result, 'requestsWithStatusCode');
|
|
395
|
+
Reflect.deleteProperty(result, 'errors');
|
|
396
|
+
Reflect.deleteProperty(result, 'retryErrors');
|
|
397
|
+
result.requestsWithStatusCode = this.state.requestsWithStatusCode;
|
|
398
|
+
result.errors = this.state.errors;
|
|
399
|
+
result.retryErrors = this.state.retryErrors;
|
|
400
|
+
return result;
|
|
352
401
|
}
|
|
353
402
|
}
|
|
354
403
|
exports.Statistics = Statistics;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"statistics.js","sourceRoot":"","sources":["../../src/crawlers/statistics.ts"],"names":[],"mappings":";;;;AAAA,oDAAoB;AACpB,gCAA2C;AAC3C,iEAA4D;AAG5D,oDAAiD;AAEjD;;GAEG;AACH,MAAM,GAAG;IAAT;QACI;;;;mBAAmC,IAAI;WAAC;QACxC;;;;mBAAe,CAAC;WAAC;QACjB;;;;;WAAgC;IAepC,CAAC;IAbG,GAAG;QACC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC5B,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC;IACvB,CAAC;IAED,MAAM;QACF,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,SAAU,CAAC;QACnD,OAAO,IAAI,CAAC,cAAc,CAAC;IAC/B,CAAC;IAED,UAAU;QACN,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC;IACtC,CAAC;CACJ;AAED;;;;;;;;;GASG;AACH,MAAa,UAAU;
|
|
1
|
+
{"version":3,"file":"statistics.js","sourceRoot":"","sources":["../../src/crawlers/statistics.ts"],"names":[],"mappings":";;;;AAAA,oDAAoB;AACpB,0CAA8C;AAC9C,gCAA2C;AAC3C,iEAA4D;AAG5D,oDAAiD;AAEjD;;GAEG;AACH,MAAM,GAAG;IAAT;QACI;;;;mBAAmC,IAAI;WAAC;QACxC;;;;mBAAe,CAAC;WAAC;QACjB;;;;;WAAgC;IAepC,CAAC;IAbG,GAAG;QACC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC5B,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC;IACvB,CAAC;IAED,MAAM;QACF,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,SAAU,CAAC;QACnD,OAAO,IAAI,CAAC,cAAc,CAAC;IAC/B,CAAC;IAED,UAAU;QACN,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC;IACtC,CAAC;CACJ;AAED,MAAM,kBAAkB,GAAG;IACvB,aAAa,EAAE,IAAI;IACnB,aAAa,EAAE,IAAI;IACnB,cAAc,EAAE,IAAI;IACpB,aAAa,EAAE,KAAK;IACpB,gBAAgB,EAAE,IAAI;IACtB,eAAe,EAAE,KAAK;CACzB,CAAC;AAEF;;;;;;;;;GASG;AACH,MAAa,UAAU;IAuCnB;;OAEG;IACH,YAAY,UAA6B,EAAE;QAvC3C;;WAEG;QACH;;;;mBAAe,IAAI,oBAAY,CAAC,kBAAkB,CAAC;WAAC;QAEpD;;WAEG;QACH;;;;mBAAoB,IAAI,oBAAY,CAAC,kBAAkB,CAAC;WAAC;QAEzD;;WAEG;QACH;;;;mBAAc,UAAU,CAAC,EAAE,EAAE;WAAC,CAAC,sEAAsE;QAErG;;WAEG;QACH;;;;;WAAuB;QAEvB;;WAEG;QACH;;;;mBAA2C,EAAE;WAAC;QAE9C;;;;mBAAwC,SAAS;WAAC;QAClD;;;;mBAA0B,0BAA0B,IAAI,CAAC,EAAE,EAAE;WAAC;QAC9D;;;;;WAAkC;QAClC;;;;;WAA2B;QAC3B;;;;;WAAsC;QACtC;;;;mBAA6B,IAAI,GAAG,EAAwB;WAAC;QAC7D;;;;mBAAuB,SAAU,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC;WAAC;QAClE;;;;;WAA+B;QAC/B;;;;;WAA6B;QAC7B;;;;;WAA6B;QAMzB,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7B,eAAe,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACnC,UAAU,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC9B,aAAa,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACjC,MAAM,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;SAC7B,CAAC,CAAC,CAAC;QAEJ,MAAM,EACF,eAAe,GAAG,EAAE,EACpB,UAAU,GAAG,YAAY,EACzB,aAAa,EACb,MAAM,GAAG,6BAAa,CAAC,eAAe,EAAE,GAC3C,GAAG,OAAO,CAAC;QAEZ,IAAI,CAAC,iBAAiB,GAAG,eAAe,GAAG,IAAI,CAAC;QAChD,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;QACnC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7C,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,eAAe,EAAE,CAAC;QAEvC,4BAA4B;QAC5B,IAAI,CAAC,KAAK,EAAE,CAAC;IACjB,CAAC;IAED;;OAEG;IACH,KAAK;QACD,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,CAAC;QAC1B,IAAI,CAAC,iBAAiB,CAAC,KAAK,EAAE,CAAC;QAE/B,IAAI,CAAC,KAAK,GAAG;YACT,gBAAgB,EAAE,CAAC;YACnB,cAAc,EAAE,CAAC;YACjB,eAAe,EAAE,CAAC;YAClB,uBAAuB,EAAE,CAAC;YAC1B,yBAAyB,EAAE,CAAC;YAC5B,wBAAwB,EAAE,QAAQ;YAClC,wBAAwB,EAAE,CAAC;YAC3B,gCAAgC,EAAE,CAAC;YACnC,kCAAkC,EAAE,CAAC;YACrC,gBAAgB,EAAE,IAAI;YACtB,iBAAiB,EAAE,IAAI;YACvB,gBAAgB,EAAE,IAAI;YACtB,oBAAoB,EAAE,CAAC;YACvB,sBAAsB,EAAE,EAAE;YAC1B,MAAM,EAAE,IAAI,CAAC,YAAY,CAAC,MAAM;YAChC,WAAW,EAAE,IAAI,CAAC,iBAAiB,CAAC,MAAM;SAC7C,CAAC;QAEF,IAAI,CAAC,qBAAqB,CAAC,MAAM,GAAG,CAAC,CAAC;QACtC,IAAI,CAAC,kBAAkB,CAAC,KAAK,EAAE,CAAC;QAChC,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAEhC,IAAI,CAAC,SAAS,EAAE,CAAC;IACrB,CAAC;IAED;;OAEG;IACH,kBAAkB,CAAC,IAAY;QAC3B,MAAM,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;QAEvB,IAAI,IAAI,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC,CAAC,KAAK,SAAS,EAAE;YACpD,IAAI,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;SAC5C;QAED,IAAI,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC,CAAC,EAAE,CAAC;IAC3C,CAAC;IAED;;;OAGG;IACH,QAAQ,CAAC,EAAmB;QACxB,IAAI,GAAG,GAAG,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC1C,IAAI,CAAC,GAAG;YAAE,GAAG,GAAG,IAAI,GAAG,EAAE,CAAC;QAC1B,GAAG,CAAC,GAAG,EAAE,CAAC;QACV,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;IACzC,CAAC;IAED;;;OAGG;IACH,SAAS,CAAC,EAAmB;QACzB,MAAM,GAAG,GAAG,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC5C,IAAI,CAAC,GAAG;YAAE,OAAO;QACjB,MAAM,iBAAiB,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC;QACvC,IAAI,CAAC,KAAK,CAAC,gBAAgB,EAAE,CAAC;QAC9B,IAAI,CAAC,KAAK,CAAC,kCAAkC,IAAI,iBAAiB,CAAC;QACnE,IAAI,CAAC,qBAAqB,CAAC,GAAG,CAAC,CAAC;QAChC,IAAI,iBAAiB,GAAG,IAAI,CAAC,KAAK,CAAC,wBAAwB;YAAE,IAAI,CAAC,KAAK,CAAC,wBAAwB,GAAG,iBAAiB,CAAC;QACrH,IAAI,iBAAiB,GAAG,IAAI,CAAC,KAAK,CAAC,wBAAwB;YAAE,IAAI,CAAC,KAAK,CAAC,wBAAwB,GAAG,iBAAiB,CAAC;QACrH,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IACvC,CAAC;IAED;;;OAGG;IACH,OAAO,CAAC,EAAmB;QACvB,MAAM,GAAG,GAAG,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC5C,IAAI,CAAC,GAAG;YAAE,OAAO;QACjB,IAAI,CAAC,KAAK,CAAC,gCAAgC,IAAI,GAAG,CAAC,MAAM,EAAE,CAAC;QAC5D,IAAI,CAAC,KAAK,CAAC,cAAc,EAAE,CAAC;QAC5B,IAAI,CAAC,qBAAqB,CAAC,GAAG,CAAC,CAAC;QAChC,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IACvC,CAAC;IAED;;OAEG;IACH,SAAS;QACL,MAAM,EACF,cAAc,EACd,gBAAgB,EAChB,gCAAgC,EAChC,kCAAkC,GACrC,GAAG,IAAI,CAAC,KAAK,CAAC;QACf,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,aAAa,CAAC;QACpD,MAAM,YAAY,GAAG,WAAW,GAAG,IAAI,GAAG,EAAE,CAAC;QAE7C,OAAO;YACH,8BAA8B,EAAE,IAAI,CAAC,KAAK,CAAC,gCAAgC,GAAG,cAAc,CAAC,IAAI,QAAQ;YACzG,gCAAgC,EAAE,IAAI,CAAC,KAAK,CAAC,kCAAkC,GAAG,gBAAgB,CAAC,IAAI,QAAQ;YAC/G,yBAAyB,EAAE,IAAI,CAAC,KAAK,CAAC,gBAAgB,GAAG,YAAY,CAAC,IAAI,CAAC;YAC3E,uBAAuB,EAAE,IAAI,CAAC,KAAK,CAAC,cAAc,GAAG,YAAY,CAAC,IAAI,CAAC;YACvE,0BAA0B,EAAE,kCAAkC,GAAG,gCAAgC;YACjG,aAAa,EAAE,cAAc,GAAG,gBAAgB;YAChD,oBAAoB,EAAE,WAAW;SACpC,CAAC;IACN,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,cAAc;QAChB,IAAI,CAAC,aAAa,KAAlB,IAAI,CAAC,aAAa,GAAK,MAAM,+BAAa,CAAC,IAAI,EAAE,EAAC;QAElD,MAAM,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAElC,IAAI,IAAI,CAAC,KAAK,CAAC,gBAAgB,KAAK,IAAI,EAAE;YACtC,IAAI,CAAC,KAAK,CAAC,gBAAgB,GAAG,IAAI,IAAI,EAAE,CAAC;SAC5C;QAED,IAAI,CAAC,MAAM,CAAC,EAAE,+CAA0B,IAAI,CAAC,QAAQ,CAAC,CAAC;QAEvD,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC,GAAG,EAAE;YAChC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE;gBAC3B,GAAG,IAAI,CAAC,SAAS,EAAE;gBACnB,cAAc,EAAE,IAAI,CAAC,qBAAqB;aAC7C,CAAC,CAAC;QACP,CAAC,EAAE,IAAI,CAAC,iBAAiB,CAAC,CAAC;IAC/B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,aAAa;QACf,IAAI,CAAC,SAAS,EAAE,CAAC;QAEjB,IAAI,CAAC,KAAK,CAAC,iBAAiB,GAAG,IAAI,IAAI,EAAE,CAAC;QAE1C,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;IAC9B,CAAC;IAES,qBAAqB,CAAC,GAAQ;QACpC,MAAM,UAAU,GAAG,GAAG,CAAC,UAAU,EAAE,CAAC;QACpC,IAAI,UAAU,GAAG,CAAC;YAAE,IAAI,CAAC,KAAK,CAAC,eAAe,EAAE,CAAC;QACjD,IAAI,CAAC,qBAAqB,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC,qBAAqB,CAAC,UAAU,CAAC;YAC3E,CAAC,CAAC,IAAI,CAAC,qBAAqB,CAAC,UAAU,CAAC,GAAG,CAAC;YAC5C,CAAC,CAAC,CAAC,CAAC;IACZ,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,YAAY;QACd,8FAA8F;QAC9F,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE;YACrB,OAAO;SACV;QAED,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,kBAAkB,EAAE,EAAE,eAAe,EAAE,IAAI,CAAC,eAAe,EAAE,CAAC,CAAC;QAE9E,MAAM,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,IAAI,CAAC,eAAe,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;IAC3E,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,oBAAoB;QAChC,8FAA8F;QAC9F,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE;YACrB,OAAO;SACV;QAED,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,QAAQ,CAA0B,IAAI,CAAC,eAAe,CAAC,CAAC;QAEpG,IAAI,CAAC,UAAU;YAAE,OAAO;QAExB,4EAA4E;QAC5E,0EAA0E;QAC1E,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,qBAAqB,CAAC,EAAE;YAClD,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,8CAA8C,EAAE;gBAC7D,eAAe,EAAE,IAAI,CAAC,eAAe;gBACrC,KAAK,EAAE,UAAU;aACpB,CAAC,CAAC;SACN;QAED,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,qCAAqC,EAAE,EAAE,eAAe,EAAE,IAAI,CAAC,eAAe,EAAE,CAAC,CAAC;QAEjG,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,qBAAqB,CAAC,CAAC;QACrE,IAAI,CAAC,KAAK,CAAC,gBAAgB,GAAG,UAAU,CAAC,gBAAgB,CAAC;QAC1D,IAAI,CAAC,KAAK,CAAC,cAAc,GAAG,UAAU,CAAC,cAAc,CAAC;QACtD,IAAI,CAAC,KAAK,CAAC,eAAe,GAAG,UAAU,CAAC,eAAe,CAAC;QAExD,IAAI,CAAC,KAAK,CAAC,gCAAgC,GAAG,UAAU,CAAC,gCAAgC,CAAC;QAC1F,IAAI,CAAC,KAAK,CAAC,kCAAkC,GAAG,UAAU,CAAC,kCAAkC,CAAC;QAC9F,IAAI,CAAC,KAAK,CAAC,wBAAwB,GAAG,UAAU,CAAC,wBAAwB,CAAC;QAC1E,IAAI,CAAC,KAAK,CAAC,wBAAwB,GAAG,UAAU,CAAC,wBAAwB,CAAC;QAC1E,wCAAwC;QACxC,IAAI,CAAC,KAAK,CAAC,iBAAiB,GAAG,UAAU,CAAC,iBAAiB,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAC5G,IAAI,CAAC,KAAK,CAAC,gBAAgB,GAAG,UAAU,CAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QACzG,IAAI,CAAC,KAAK,CAAC,gBAAgB,GAAG,UAAU,CAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QACzG,IAAI,CAAC,KAAK,CAAC,oBAAoB,GAAG,UAAU,CAAC,oBAAoB,CAAC;QAClE,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,gBAAiB,GAAG,UAAU,CAAC,yBAAyB,CAAC,CAAC;QAEzG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,2BAA2B,CAAC,CAAC;IAChD,CAAC;IAES,SAAS;QACf,uFAAuF;QACvF,IAAI,CAAC,MAAM,CAAC,GAAG,+CAA0B,IAAI,CAAC,QAAQ,CAAC,CAAC;QAExD,IAAI,IAAI,CAAC,WAAW,EAAE;YAClB,aAAa,CAAC,IAAI,CAAC,WAAqB,CAAC,CAAC;YAC1C,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;SAC3B;IACL,CAAC;IAED;;;OAGG;IACH,MAAM;QACF,4EAA4E;QAC5E,oEAAoE;QACpE,8BAA8B;QAC9B,MAAM,MAAM,GAAG;YACX,GAAG,IAAI,CAAC,KAAK;YACb,yBAAyB,EAAE,IAAI,CAAC,aAAa;YAC7C,iBAAiB,EAAE,IAAI,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,IAAI;YAC7G,gBAAgB,EAAE,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,IAAI;YAC1G,qBAAqB,EAAE,IAAI,CAAC,qBAAqB;YACjD,OAAO,EAAE,IAAI,CAAC,EAAE;YAChB,gBAAgB,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YAC1C,GAAG,IAAI,CAAC,SAAS,EAAE;SACtB,CAAC;QAEF,OAAO,CAAC,cAAc,CAAC,MAAM,EAAE,wBAAwB,CAAC,CAAC;QACzD,OAAO,CAAC,cAAc,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QACzC,OAAO,CAAC,cAAc,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;QAE9C,MAAM,CAAC,sBAAsB,GAAG,IAAI,CAAC,KAAK,CAAC,sBAAsB,CAAC;QAClE,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;QAClC,MAAM,CAAC,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC;QAE5C,OAAO,MAAM,CAAC;IAClB,CAAC;;AA1TL,gCA2TC;AA1TkB;;;;WAAK,CAAC;GAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"enqueue_links.d.ts","sourceRoot":"","sources":["../../src/enqueue_links/enqueue_links.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,sBAAsB,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AACzE,OAAO,KAAK,EAAE,SAAS,EAAE,cAAc,EAAE,WAAW,EAAE,gBAAgB,EAAoB,MAAM,UAAU,CAAC;
|
|
1
|
+
{"version":3,"file":"enqueue_links.d.ts","sourceRoot":"","sources":["../../src/enqueue_links/enqueue_links.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,sBAAsB,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AACzE,OAAO,KAAK,EAAE,SAAS,EAAE,cAAc,EAAE,WAAW,EAAE,gBAAgB,EAAoB,MAAM,UAAU,CAAC;AAS3G,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAC;AAG9D,MAAM,WAAW,mBAAmB;IAChC,sHAAsH;IACtH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf,mCAAmC;IACnC,IAAI,EAAE,MAAM,EAAE,CAAC;IAEf,0DAA0D;IAC1D,YAAY,EAAE,YAAY,CAAC;IAE3B,oDAAoD;IACpD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB,oEAAoE;IACpE,QAAQ,CAAC,EAAE,UAAU,CAAC;IAEtB,iEAAiE;IACjE,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;;;;;;;;;;;OAYG;IACH,KAAK,CAAC,EAAE,SAAS,EAAE,CAAC;IAEpB;;;;;;;;;OASG;IACH,OAAO,CAAC,EAAE,WAAW,EAAE,CAAC;IAExB;;;;;;;;;;;;;;;;;OAiBG;IACH,UAAU,CAAC,EAAE,cAAc,EAAE,CAAC;IAE9B;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACH,wBAAwB,CAAC,EAAE,gBAAgB,CAAC;IAE5C;;;OAGG;IACH,QAAQ,CAAC,EAAE,eAAe,GAAG,KAAK,GAAG,aAAa,GAAG,eAAe,CAAC;CACxE;AAED,oBAAY,eAAe;IACvB;;OAEG;IACH,GAAG,QAAQ;IAEX;;;;OAIG;IACH,YAAY,kBAAkB;IAE9B;;;;OAIG;IACH,UAAU,gBAAgB;CAC7B;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,wBAAsB,YAAY,CAAC,OAAO,EAAE,mBAAmB,GAAG,OAAO,CAAC,sBAAsB,CAAC,CAqHhG;AAED;;;;;;;GAOG;AACH,wBAAgB,sCAAsC,CAAC,EACnD,eAAe,EACf,eAAe,EACf,kBAAkB,EAClB,mBAAmB,GACtB,EAAE,cAAc,sBA+BhB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC3B,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,eAAe,CAAC,EAAE,mBAAmB,CAAC,UAAU,CAAC,CAAC;IAClD,kBAAkB,EAAE,MAAM,CAAC;IAC3B,eAAe,CAAC,EAAE,MAAM,CAAC;CAC5B"}
|
|
@@ -83,6 +83,7 @@ async function enqueueLinks(options) {
|
|
|
83
83
|
if (!urlPatternObjects.length) {
|
|
84
84
|
options.strategy ?? (options.strategy = EnqueueStrategy.SameHostname);
|
|
85
85
|
}
|
|
86
|
+
const enqueueStrategyPatterns = [];
|
|
86
87
|
if (options.baseUrl) {
|
|
87
88
|
const url = new URL(options.baseUrl);
|
|
88
89
|
switch (options.strategy) {
|
|
@@ -90,7 +91,7 @@ async function enqueueLinks(options) {
|
|
|
90
91
|
// We need to get the origin of the passed in domain in the event someone sets baseUrl
|
|
91
92
|
// to an url like https://example.com/deep/default/path and one of the found urls is an
|
|
92
93
|
// absolute relative path (/path/to/page)
|
|
93
|
-
|
|
94
|
+
enqueueStrategyPatterns.push({ glob: `${url.origin}/**` });
|
|
94
95
|
break;
|
|
95
96
|
case EnqueueStrategy.SameDomain: {
|
|
96
97
|
// Get the actual hostname from the base url
|
|
@@ -98,12 +99,12 @@ async function enqueueLinks(options) {
|
|
|
98
99
|
if (baseUrlHostname) {
|
|
99
100
|
// We have a hostname, so we can use it to match all links on the page that point to it and any subdomains of it
|
|
100
101
|
url.hostname = baseUrlHostname;
|
|
101
|
-
|
|
102
|
+
enqueueStrategyPatterns.push({ glob: `${url.origin.replace(baseUrlHostname, `*.${baseUrlHostname}`)}/**` }, { glob: `${url.origin}/**` });
|
|
102
103
|
}
|
|
103
104
|
else {
|
|
104
105
|
// We don't have a hostname (can happen for ips for instance), so reproduce the same behavior
|
|
105
106
|
// as SameDomainAndSubdomain
|
|
106
|
-
|
|
107
|
+
enqueueStrategyPatterns.push({ glob: `${url.origin}/**` });
|
|
107
108
|
}
|
|
108
109
|
break;
|
|
109
110
|
}
|
|
@@ -116,7 +117,17 @@ async function enqueueLinks(options) {
|
|
|
116
117
|
if (transformRequestFunction) {
|
|
117
118
|
requestOptions = requestOptions.map((request) => transformRequestFunction(request)).filter((r) => !!r);
|
|
118
119
|
}
|
|
119
|
-
|
|
120
|
+
function createFilteredRequests() {
|
|
121
|
+
// No user provided patterns means we can skip an extra filtering step
|
|
122
|
+
if (urlPatternObjects.length === 0) {
|
|
123
|
+
return (0, shared_1.createRequests)(requestOptions, enqueueStrategyPatterns);
|
|
124
|
+
}
|
|
125
|
+
// Generate requests based on the user patterns first
|
|
126
|
+
const generatedRequestsFromUserFilters = (0, shared_1.createRequests)(requestOptions, urlPatternObjects);
|
|
127
|
+
// ...then filter them by the enqueue links strategy (making this an AND check)
|
|
128
|
+
return (0, shared_1.filterRequestsByPatterns)(generatedRequestsFromUserFilters, enqueueStrategyPatterns);
|
|
129
|
+
}
|
|
130
|
+
let requests = createFilteredRequests();
|
|
120
131
|
if (limit)
|
|
121
132
|
requests = requests.slice(0, limit);
|
|
122
133
|
return requestQueue.addRequests(requests);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"enqueue_links.js","sourceRoot":"","sources":["../../src/enqueue_links/enqueue_links.ts"],"names":[],"mappings":";;;;AAAA,iCAAkC;AAClC,oDAAoB;AACpB,6DAA6B;AAG7B,
|
|
1
|
+
{"version":3,"file":"enqueue_links.js","sourceRoot":"","sources":["../../src/enqueue_links/enqueue_links.ts"],"names":[],"mappings":";;;;AAAA,iCAAkC;AAClC,oDAAoB;AACpB,6DAA6B;AAG7B,qCAOkB;AA6GlB,IAAY,eAmBX;AAnBD,WAAY,eAAe;IACvB;;OAEG;IACH,8BAAW,CAAA;IAEX;;;;OAIG;IACH,iDAA8B,CAAA;IAE9B;;;;OAIG;IACH,6CAA0B,CAAA;AAC9B,CAAC,EAnBW,eAAe,GAAf,uBAAe,KAAf,uBAAe,QAmB1B;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACI,KAAK,UAAU,YAAY,CAAC,OAA4B;IAC3D,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;QAC7B,IAAI,EAAE,YAAE,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,MAAM,CAAC;QAChC,YAAY,EAAE,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,kBAAkB,EAAE,YAAY,CAAC;QACjE,KAAK,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACzB,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC5B,OAAO,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC3B,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC5B,KAAK,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACzB,UAAU,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,GAAG,CACvC,YAAE,CAAC,MAAM,EACT,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAC5B,CAAC;QACF,KAAK,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,GAAG,CAClC,YAAE,CAAC,MAAM,EACT,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAC5B,CAAC;QACF,OAAO,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,GAAG,CACpC,YAAE,CAAC,MAAM,EACT,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAC9B,CAAC;QACF,wBAAwB,EAAE,YAAE,CAAC,QAAQ,CAAC,QAAQ;QAC9C,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC;YAC/B,eAAe,CAAC,GAAG;YACnB,eAAe,CAAC,YAAY;YAC5B,eAAe,CAAC,UAAU;SAC7B,CAAC;KACL,CAAC,CAAC,CAAC;IAEJ,MAAM,EACF,YAAY,EACZ,KAAK,EACL,IAAI,EACJ,UAAU,EACV,KAAK,EACL,OAAO,EACP,wBAAwB,GAC3B,GAAG,OAAO,CAAC;IAEZ,MAAM,iBAAiB,GAAuB,EAAE,CAAC;IAEjD,IAAI,UAAU,EAAE,MAAM,EAAE;QACpB,aAAG,CAAC,UAAU,CAAC,qEAAqE,CAAC,CAAC;QACtF,iBAAiB,CAAC,IAAI,CAAC,GAAG,IAAA,6CAAoC,EAAC,UAAU,CAAC,CAAC,CAAC;KAC/E;IAED,IAAI,KAAK,EAAE,MAAM,EAAE;QACf,iBAAiB,CAAC,IAAI,CAAC,GAAG,IAAA,sCAA6B,EAAC,KAAK,CAAC,CAAC,CAAC;KACnE;IAED,IAAI,OAAO,EAAE,MAAM,EAAE;QACjB,iBAAiB,CAAC,IAAI,CAAC,GAAG,IAAA,0CAAiC,EAAC,OAAO,CAAC,CAAC,CAAC;KACzE;IAED,IAAI,CAAC,iBAAiB,CAAC,MAAM,EAAE;QAC3B,OAAO,CAAC,QAAQ,KAAhB,OAAO,CAAC,QAAQ,GAAK,eAAe,CAAC,YAAY,EAAC;KACrD;IAED,MAAM,uBAAuB,GAAuB,EAAE,CAAC;IAEvD,IAAI,OAAO,CAAC,OAAO,EAAE;QACjB,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAErC,QAAQ,OAAO,CAAC,QAAQ,EAAE;YACtB,KAAK,eAAe,CAAC,YAAY;gBAC7B,sFAAsF;gBACtF,uFAAuF;gBACvF,yCAAyC;gBACzC,uBAAuB,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,GAAG,CAAC,MAAM,KAAK,EAAE,CAAC,CAAC;gBAC3D,MAAM;YACV,KAAK,eAAe,CAAC,UAAU,CAAC,CAAC;gBAC7B,4CAA4C;gBAC5C,MAAM,eAAe,GAAG,IAAA,iBAAS,EAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC,CAAC;gBAExE,IAAI,eAAe,EAAE;oBACjB,gHAAgH;oBAChH,GAAG,CAAC,QAAQ,GAAG,eAAe,CAAC;oBAC/B,uBAAuB,CAAC,IAAI,CACxB,EAAE,IAAI,EAAE,GAAG,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,eAAe,EAAE,KAAK,eAAe,EAAE,CAAC,KAAK,EAAE,EAC7E,EAAE,IAAI,EAAE,GAAG,GAAG,CAAC,MAAM,KAAK,EAAE,CAC/B,CAAC;iBACL;qBAAM;oBACH,6FAA6F;oBAC7F,4BAA4B;oBAC5B,uBAAuB,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,GAAG,CAAC,MAAM,KAAK,EAAE,CAAC,CAAC;iBAC9D;gBAED,MAAM;aACT;YACD,KAAK,eAAe,CAAC,GAAG,CAAC;YACzB;gBACI,MAAM;SACb;KACJ;IAED,IAAI,cAAc,GAAG,IAAA,6BAAoB,EAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAEzD,IAAI,wBAAwB,EAAE;QAC1B,cAAc,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,wBAAwB,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAqB,CAAC;KAC9H;IAED,SAAS,sBAAsB;QAC3B,sEAAsE;QACtE,IAAI,iBAAiB,CAAC,MAAM,KAAK,CAAC,EAAE;YAChC,OAAO,IAAA,uBAAc,EAAC,cAAc,EAAE,uBAAuB,CAAC,CAAC;SAClE;QAED,qDAAqD;QACrD,MAAM,gCAAgC,GAAG,IAAA,uBAAc,EAAC,cAAc,EAAE,iBAAiB,CAAC,CAAC;QAC3F,+EAA+E;QAC/E,OAAO,IAAA,iCAAwB,EAAC,gCAAgC,EAAE,uBAAuB,CAAC,CAAC;IAC/F,CAAC;IAED,IAAI,QAAQ,GAAG,sBAAsB,EAAE,CAAC;IACxC,IAAI,KAAK;QAAE,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;IAE/C,OAAO,YAAY,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;AAC9C,CAAC;AArHD,oCAqHC;AAED;;;;;;;GAOG;AACH,SAAgB,sCAAsC,CAAC,EACnD,eAAe,EACf,eAAe,EACf,kBAAkB,EAClB,mBAAmB,GACN;IACb,wCAAwC;IACxC,IAAI,mBAAmB,EAAE;QACrB,OAAO,mBAAmB,CAAC;KAC9B;IAED,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC,kBAAkB,CAAC,CAAC,MAAM,CAAC;IAC7D,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,eAAe,IAAI,kBAAkB,CAAC,CAAC,MAAM,CAAC;IAE7E,6DAA6D;IAC7D,IAAI,eAAe,KAAK,eAAe,CAAC,GAAG,EAAE;QACzC,OAAO,cAAc,CAAC;KACzB;IAED,0HAA0H;IAC1H,+IAA+I;IAC/I,4DAA4D;IAC5D,IAAI,eAAe,KAAK,eAAe,CAAC,UAAU,EAAE;QAChD,MAAM,gBAAgB,GAAG,IAAA,iBAAS,EAAC,iBAAiB,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,CAAE,CAAC;QAC/E,MAAM,aAAa,GAAG,IAAA,iBAAS,EAAC,cAAc,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,CAAE,CAAC;QAEzE,IAAI,gBAAgB,KAAK,aAAa,EAAE;YACpC,OAAO,cAAc,CAAC;SACzB;QAED,OAAO,SAAS,CAAC;KACpB;IAED,2JAA2J;IAC3J,mCAAmC;IACnC,OAAO,iBAAiB,CAAC;AAC7B,CAAC;AApCD,wFAoCC"}
|
|
@@ -47,6 +47,7 @@ export declare function constructRegExpObjectsFromRegExps(regexps: RegExpInput[]
|
|
|
47
47
|
* @ignore
|
|
48
48
|
*/
|
|
49
49
|
export declare function createRequests(requestOptions: (string | RequestOptions)[], urlPatternObjects?: UrlPatternObject[]): Request[];
|
|
50
|
+
export declare function filterRequestsByPatterns(requests: Request[], patterns?: UrlPatternObject[]): Request[];
|
|
50
51
|
/**
|
|
51
52
|
* @ignore
|
|
52
53
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"shared.d.ts","sourceRoot":"","sources":["../../src/enqueue_links/shared.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AACjD,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACrC,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AAY3D,oBAAY,gBAAgB,GAAG;IAC3B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;CACnB,GAAG,IAAI,CAAC,cAAc,EAAE,QAAQ,GAAG,SAAS,GAAG,OAAO,GAAG,UAAU,GAAG,SAAS,CAAC,CAAC;AAElF,oBAAY,eAAe,GAAG;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAAC,cAAc,EAAE,QAAQ,GAAG,SAAS,GAAG,OAAO,GAAG,UAAU,GAAG,SAAS,CAAC,CAAC;AAE/H,oBAAY,cAAc,GAAG,MAAM,GAAG,eAAe,CAAC;AAEtD,oBAAY,UAAU,GAAG;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAAC,cAAc,EAAE,QAAQ,GAAG,SAAS,GAAG,OAAO,GAAG,UAAU,GAAG,SAAS,CAAC,CAAC;AAE1H,oBAAY,SAAS,GAAG,MAAM,GAAG,UAAU,CAAC;AAE5C,oBAAY,YAAY,GAAG;IAAE,MAAM,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAAC,cAAc,EAAE,QAAQ,GAAG,SAAS,GAAG,OAAO,GAAG,UAAU,GAAG,SAAS,CAAC,CAAC;AAE9H,oBAAY,WAAW,GAAG,MAAM,GAAG,YAAY,CAAC;AAEhD;;GAEG;AACH,wBAAgB,8BAA8B,CAAC,IAAI,EAAE,SAAS,GAAG,WAAW,GAAG,cAAc,EAAE,OAAO,EAAE,YAAY,GAAG,UAAU,GAAG,IAAI,CAMvI;AAED;;;;GAIG;AACH,wBAAgB,oCAAoC,CAAC,UAAU,EAAE,cAAc,EAAE,GAAG,YAAY,EAAE,CAiBjG;AAED;;;;GAIG;AACH,wBAAgB,6BAA6B,CAAC,KAAK,EAAE,SAAS,EAAE,GAAG,UAAU,EAAE,CAiB9E;AAED;;GAEG;AACH,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAIxD;AAED;;;;GAIG;AACH,wBAAgB,iCAAiC,CAAC,OAAO,EAAE,WAAW,EAAE,GAAG,YAAY,EAAE,CAgBxF;AAED;;GAEG;AACH,wBAAgB,cAAc,CAAC,cAAc,EAAE,CAAC,MAAM,GAAG,cAAc,CAAC,EAAE,EAAE,iBAAiB,CAAC,EAAE,gBAAgB,EAAE,GAAG,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"shared.d.ts","sourceRoot":"","sources":["../../src/enqueue_links/shared.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AACjD,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACrC,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AAY3D,oBAAY,gBAAgB,GAAG;IAC3B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;CACnB,GAAG,IAAI,CAAC,cAAc,EAAE,QAAQ,GAAG,SAAS,GAAG,OAAO,GAAG,UAAU,GAAG,SAAS,CAAC,CAAC;AAElF,oBAAY,eAAe,GAAG;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAAC,cAAc,EAAE,QAAQ,GAAG,SAAS,GAAG,OAAO,GAAG,UAAU,GAAG,SAAS,CAAC,CAAC;AAE/H,oBAAY,cAAc,GAAG,MAAM,GAAG,eAAe,CAAC;AAEtD,oBAAY,UAAU,GAAG;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAAC,cAAc,EAAE,QAAQ,GAAG,SAAS,GAAG,OAAO,GAAG,UAAU,GAAG,SAAS,CAAC,CAAC;AAE1H,oBAAY,SAAS,GAAG,MAAM,GAAG,UAAU,CAAC;AAE5C,oBAAY,YAAY,GAAG;IAAE,MAAM,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAAC,cAAc,EAAE,QAAQ,GAAG,SAAS,GAAG,OAAO,GAAG,UAAU,GAAG,SAAS,CAAC,CAAC;AAE9H,oBAAY,WAAW,GAAG,MAAM,GAAG,YAAY,CAAC;AAEhD;;GAEG;AACH,wBAAgB,8BAA8B,CAAC,IAAI,EAAE,SAAS,GAAG,WAAW,GAAG,cAAc,EAAE,OAAO,EAAE,YAAY,GAAG,UAAU,GAAG,IAAI,CAMvI;AAED;;;;GAIG;AACH,wBAAgB,oCAAoC,CAAC,UAAU,EAAE,cAAc,EAAE,GAAG,YAAY,EAAE,CAiBjG;AAED;;;;GAIG;AACH,wBAAgB,6BAA6B,CAAC,KAAK,EAAE,SAAS,EAAE,GAAG,UAAU,EAAE,CAiB9E;AAED;;GAEG;AACH,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAIxD;AAED;;;;GAIG;AACH,wBAAgB,iCAAiC,CAAC,OAAO,EAAE,WAAW,EAAE,GAAG,YAAY,EAAE,CAgBxF;AAED;;GAEG;AACH,wBAAgB,cAAc,CAAC,cAAc,EAAE,CAAC,MAAM,GAAG,cAAc,CAAC,EAAE,EAAE,iBAAiB,CAAC,EAAE,gBAAgB,EAAE,GAAG,OAAO,EAAE,CA6B7H;AAED,wBAAgB,wBAAwB,CAAC,QAAQ,EAAE,OAAO,EAAE,EAAE,QAAQ,CAAC,EAAE,gBAAgB,EAAE,GAAG,OAAO,EAAE,CAuBtG;AAED;;GAEG;AACH,wBAAgB,oBAAoB,CAChC,OAAO,EAAE,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,EAAE,EAC7C,OAAO,GAAE,IAAI,CAAC,mBAAmB,EAAE,OAAO,GAAG,UAAU,CAAM,GAC9D,cAAc,EAAE,CAelB;AAED;;;GAGG;AACH,MAAM,WAAW,gBAAgB;IAC7B;;;OAGG;IACH,CAAC,QAAQ,EAAE,cAAc,GAAG,cAAc,GAAG,KAAK,GAAG,SAAS,GAAG,IAAI,CAAC;CACzE"}
|
package/enqueue_links/shared.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.createRequestOptions = exports.createRequests = exports.constructRegExpObjectsFromRegExps = exports.validateGlobPattern = exports.constructGlobObjectsFromGlobs = exports.constructRegExpObjectsFromPseudoUrls = exports.updateEnqueueLinksPatternCache = void 0;
|
|
3
|
+
exports.createRequestOptions = exports.filterRequestsByPatterns = exports.createRequests = exports.constructRegExpObjectsFromRegExps = exports.validateGlobPattern = exports.constructGlobObjectsFromGlobs = exports.constructRegExpObjectsFromPseudoUrls = exports.updateEnqueueLinksPatternCache = void 0;
|
|
4
4
|
const tslib_1 = require("tslib");
|
|
5
5
|
const url_1 = require("url");
|
|
6
6
|
const pseudo_url_1 = require("@apify/pseudo_url");
|
|
@@ -112,22 +112,43 @@ function createRequests(requestOptions, urlPatternObjects) {
|
|
|
112
112
|
.map((opts) => new request_1.Request(typeof opts === 'string' ? { url: opts } : opts));
|
|
113
113
|
}
|
|
114
114
|
const requests = [];
|
|
115
|
-
for (const
|
|
116
|
-
const
|
|
117
|
-
for (const
|
|
118
|
-
const
|
|
115
|
+
for (const opts of requestOptions) {
|
|
116
|
+
const urlToMatch = typeof opts === 'string' ? opts : opts.url;
|
|
117
|
+
for (const urlPatternObject of urlPatternObjects) {
|
|
118
|
+
const { regexp, glob, ...requestRegExpOptions } = urlPatternObject;
|
|
119
119
|
if ((regexp && urlToMatch.match(regexp)) || // eslint-disable-line
|
|
120
120
|
(glob && (0, minimatch_1.default)(urlToMatch, glob, { nocase: true }))) {
|
|
121
121
|
const request = typeof opts === 'string'
|
|
122
122
|
? { url: opts, ...requestRegExpOptions }
|
|
123
123
|
: { ...opts, ...requestRegExpOptions };
|
|
124
124
|
requests.push(new request_1.Request(request));
|
|
125
|
+
// Stop checking other patterns for this request option as it was already matched
|
|
126
|
+
break;
|
|
125
127
|
}
|
|
126
128
|
}
|
|
127
129
|
}
|
|
128
130
|
return requests;
|
|
129
131
|
}
|
|
130
132
|
exports.createRequests = createRequests;
|
|
133
|
+
function filterRequestsByPatterns(requests, patterns) {
|
|
134
|
+
if (!patterns?.length) {
|
|
135
|
+
return requests;
|
|
136
|
+
}
|
|
137
|
+
const filtered = [];
|
|
138
|
+
for (const request of requests) {
|
|
139
|
+
for (const urlPatternObject of patterns) {
|
|
140
|
+
const { regexp, glob } = urlPatternObject;
|
|
141
|
+
if ((regexp && request.url.match(regexp)) || // eslint-disable-line
|
|
142
|
+
(glob && (0, minimatch_1.default)(request.url, glob, { nocase: true }))) {
|
|
143
|
+
filtered.push(request);
|
|
144
|
+
// Break the pattern loop, as we already matched this request once
|
|
145
|
+
break;
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
return filtered;
|
|
150
|
+
}
|
|
151
|
+
exports.filterRequestsByPatterns = filterRequestsByPatterns;
|
|
131
152
|
/**
|
|
132
153
|
* @ignore
|
|
133
154
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"shared.js","sourceRoot":"","sources":["../../src/enqueue_links/shared.ts"],"names":[],"mappings":";;;;AAAA,6BAA0B;AAC1B,kDAAiD;AACjD,kEAAkC;AAElC,wCAAqC;AAGrC,MAAM,4BAA4B,GAAG,IAAI,CAAC;AAE1C;;;;;GAKG;AACH,MAAM,wBAAwB,GAAG,IAAI,GAAG,EAAE,CAAC;AAmB3C;;GAEG;AACH,SAAgB,8BAA8B,CAAC,IAA8C,EAAE,OAAkC;IAC7H,wBAAwB,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC5C,IAAI,wBAAwB,CAAC,IAAI,GAAG,4BAA4B,EAAE;QAC9D,MAAM,GAAG,GAAG,wBAAwB,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC;QACzD,wBAAwB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;KACxC;AACL,CAAC;AAND,wEAMC;AAED;;;;GAIG;AACH,SAAgB,oCAAoC,CAAC,UAA4B;IAC7E,OAAO,UAAU,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QAC3B,mCAAmC;QACnC,IAAI,YAAY,GAAG,wBAAwB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACtD,IAAI,YAAY;YAAE,OAAO,YAAY,CAAC;QAEtC,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE;YAC1B,YAAY,GAAG,EAAE,MAAM,EAAE,IAAA,yBAAY,EAAC,IAAI,CAAC,EAAE,CAAC;SACjD;aAAM;YACH,MAAM,EAAE,IAAI,EAAE,GAAG,cAAc,EAAE,GAAG,IAAI,CAAC;YACzC,YAAY,GAAG,EAAE,MAAM,EAAE,IAAA,yBAAY,EAAC,IAAI,CAAC,EAAE,GAAG,cAAc,EAAE,CAAC;SACpE;QAED,8BAA8B,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC;QAEnD,OAAO,YAAY,CAAC;IACxB,CAAC,CAAC,CAAC;AACP,CAAC;AAjBD,oFAiBC;AAED;;;;GAIG;AACH,SAAgB,6BAA6B,CAAC,KAAkB;IAC5D,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACtB,8BAA8B;QAC9B,IAAI,UAAU,GAAG,wBAAwB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACpD,IAAI,UAAU;YAAE,OAAO,UAAU,CAAC;QAElC,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE;YAC1B,UAAU,GAAG,EAAE,IAAI,EAAE,mBAAmB,CAAC,IAAI,CAAC,EAAE,CAAC;SACpD;aAAM;YACH,MAAM,EAAE,IAAI,EAAE,GAAG,cAAc,EAAE,GAAG,IAAI,CAAC;YACzC,UAAU,GAAG,EAAE,IAAI,EAAE,mBAAmB,CAAC,IAAI,CAAC,EAAE,GAAG,cAAc,EAAE,CAAC;SACvE;QAED,8BAA8B,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;QAEjD,OAAO,UAAU,CAAC;IACtB,CAAC,CAAC,CAAC;AACP,CAAC;AAjBD,sEAiBC;AAED;;GAEG;AACH,SAAgB,mBAAmB,CAAC,IAAY;IAC5C,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IAChC,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,8BAA8B,WAAW,mCAAmC,CAAC,CAAC;IAC5H,OAAO,WAAW,CAAC;AACvB,CAAC;AAJD,kDAIC;AAED;;;;GAIG;AACH,SAAgB,iCAAiC,CAAC,OAAsB;IACpE,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACxB,gCAAgC;QAChC,IAAI,YAAY,GAAG,wBAAwB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACtD,IAAI,YAAY;YAAE,OAAO,YAAY,CAAC;QAEtC,IAAI,IAAI,YAAY,MAAM,EAAE;YACxB,YAAY,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;SACnC;aAAM;YACH,YAAY,GAAG,IAAI,CAAC;SACvB;QAED,8BAA8B,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC;QAEnD,OAAO,YAAY,CAAC;IACxB,CAAC,CAAC,CAAC;AACP,CAAC;AAhBD,8EAgBC;AAED;;GAEG;AACH,SAAgB,cAAc,CAAC,cAA2C,EAAE,iBAAsC;IAC9G,IAAI,CAAC,iBAAiB,IAAI,CAAC,iBAAiB,CAAC,MAAM,EAAE;QACjD,OAAO,cAAc;aAChB,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,iBAAO,CAAC,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;KACpF;IAED,MAAM,QAAQ,GAAc,EAAE,CAAC;IAE/B,KAAK,MAAM,
|
|
1
|
+
{"version":3,"file":"shared.js","sourceRoot":"","sources":["../../src/enqueue_links/shared.ts"],"names":[],"mappings":";;;;AAAA,6BAA0B;AAC1B,kDAAiD;AACjD,kEAAkC;AAElC,wCAAqC;AAGrC,MAAM,4BAA4B,GAAG,IAAI,CAAC;AAE1C;;;;;GAKG;AACH,MAAM,wBAAwB,GAAG,IAAI,GAAG,EAAE,CAAC;AAmB3C;;GAEG;AACH,SAAgB,8BAA8B,CAAC,IAA8C,EAAE,OAAkC;IAC7H,wBAAwB,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC5C,IAAI,wBAAwB,CAAC,IAAI,GAAG,4BAA4B,EAAE;QAC9D,MAAM,GAAG,GAAG,wBAAwB,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC;QACzD,wBAAwB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;KACxC;AACL,CAAC;AAND,wEAMC;AAED;;;;GAIG;AACH,SAAgB,oCAAoC,CAAC,UAA4B;IAC7E,OAAO,UAAU,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QAC3B,mCAAmC;QACnC,IAAI,YAAY,GAAG,wBAAwB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACtD,IAAI,YAAY;YAAE,OAAO,YAAY,CAAC;QAEtC,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE;YAC1B,YAAY,GAAG,EAAE,MAAM,EAAE,IAAA,yBAAY,EAAC,IAAI,CAAC,EAAE,CAAC;SACjD;aAAM;YACH,MAAM,EAAE,IAAI,EAAE,GAAG,cAAc,EAAE,GAAG,IAAI,CAAC;YACzC,YAAY,GAAG,EAAE,MAAM,EAAE,IAAA,yBAAY,EAAC,IAAI,CAAC,EAAE,GAAG,cAAc,EAAE,CAAC;SACpE;QAED,8BAA8B,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC;QAEnD,OAAO,YAAY,CAAC;IACxB,CAAC,CAAC,CAAC;AACP,CAAC;AAjBD,oFAiBC;AAED;;;;GAIG;AACH,SAAgB,6BAA6B,CAAC,KAAkB;IAC5D,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACtB,8BAA8B;QAC9B,IAAI,UAAU,GAAG,wBAAwB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACpD,IAAI,UAAU;YAAE,OAAO,UAAU,CAAC;QAElC,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE;YAC1B,UAAU,GAAG,EAAE,IAAI,EAAE,mBAAmB,CAAC,IAAI,CAAC,EAAE,CAAC;SACpD;aAAM;YACH,MAAM,EAAE,IAAI,EAAE,GAAG,cAAc,EAAE,GAAG,IAAI,CAAC;YACzC,UAAU,GAAG,EAAE,IAAI,EAAE,mBAAmB,CAAC,IAAI,CAAC,EAAE,GAAG,cAAc,EAAE,CAAC;SACvE;QAED,8BAA8B,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;QAEjD,OAAO,UAAU,CAAC;IACtB,CAAC,CAAC,CAAC;AACP,CAAC;AAjBD,sEAiBC;AAED;;GAEG;AACH,SAAgB,mBAAmB,CAAC,IAAY;IAC5C,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IAChC,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,8BAA8B,WAAW,mCAAmC,CAAC,CAAC;IAC5H,OAAO,WAAW,CAAC;AACvB,CAAC;AAJD,kDAIC;AAED;;;;GAIG;AACH,SAAgB,iCAAiC,CAAC,OAAsB;IACpE,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACxB,gCAAgC;QAChC,IAAI,YAAY,GAAG,wBAAwB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACtD,IAAI,YAAY;YAAE,OAAO,YAAY,CAAC;QAEtC,IAAI,IAAI,YAAY,MAAM,EAAE;YACxB,YAAY,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;SACnC;aAAM;YACH,YAAY,GAAG,IAAI,CAAC;SACvB;QAED,8BAA8B,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC;QAEnD,OAAO,YAAY,CAAC;IACxB,CAAC,CAAC,CAAC;AACP,CAAC;AAhBD,8EAgBC;AAED;;GAEG;AACH,SAAgB,cAAc,CAAC,cAA2C,EAAE,iBAAsC;IAC9G,IAAI,CAAC,iBAAiB,IAAI,CAAC,iBAAiB,CAAC,MAAM,EAAE;QACjD,OAAO,cAAc;aAChB,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,iBAAO,CAAC,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;KACpF;IAED,MAAM,QAAQ,GAAc,EAAE,CAAC;IAE/B,KAAK,MAAM,IAAI,IAAI,cAAc,EAAE;QAC/B,MAAM,UAAU,GAAG,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC;QAE9D,KAAK,MAAM,gBAAgB,IAAI,iBAAiB,EAAE;YAC9C,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,oBAAoB,EAAE,GAAG,gBAAgB,CAAC;YACnE,IACI,CAAC,MAAM,IAAI,UAAU,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,sBAAsB;gBAC9D,CAAC,IAAI,IAAI,IAAA,mBAAS,EAAC,UAAU,EAAE,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,EACzD;gBACE,MAAM,OAAO,GAAG,OAAO,IAAI,KAAK,QAAQ;oBACpC,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,oBAAoB,EAAE;oBACxC,CAAC,CAAC,EAAE,GAAG,IAAI,EAAE,GAAG,oBAAoB,EAAE,CAAC;gBAC3C,QAAQ,CAAC,IAAI,CAAC,IAAI,iBAAO,CAAC,OAAO,CAAC,CAAC,CAAC;gBAEpC,iFAAiF;gBACjF,MAAM;aACT;SACJ;KACJ;IAED,OAAO,QAAQ,CAAC;AACpB,CAAC;AA7BD,wCA6BC;AAED,SAAgB,wBAAwB,CAAC,QAAmB,EAAE,QAA6B;IACvF,IAAI,CAAC,QAAQ,EAAE,MAAM,EAAE;QACnB,OAAO,QAAQ,CAAC;KACnB;IAED,MAAM,QAAQ,GAAc,EAAE,CAAC;IAE/B,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE;QAC5B,KAAK,MAAM,gBAAgB,IAAI,QAAQ,EAAE;YACrC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,gBAAgB,CAAC;YAE1C,IACI,CAAC,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,sBAAsB;gBAC/D,CAAC,IAAI,IAAI,IAAA,mBAAS,EAAC,OAAO,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,EAC1D;gBACE,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBACvB,kEAAkE;gBAClE,MAAM;aACT;SACJ;KACJ;IAED,OAAO,QAAQ,CAAC;AACpB,CAAC;AAvBD,4DAuBC;AAED;;GAEG;AACH,SAAgB,oBAAoB,CAChC,OAA6C,EAC7C,UAA2D,EAAE;IAE7D,OAAO,OAAO;SACT,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,OAAO,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,GAAgC,CAAC,CAAC;SACzF,MAAM,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE;QAChB,IAAI;YACA,OAAO,IAAI,SAAG,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;SAC5B;QAAC,OAAO,GAAG,EAAE;YACV,OAAO,KAAK,CAAC;SAChB;IACL,CAAC,CAAC;SACD,GAAG,CAAC,CAAC,cAAc,EAAE,EAAE;;QACpB,cAAc,CAAC,QAAQ,KAAvB,cAAc,CAAC,QAAQ,GAAK,OAAO,CAAC,QAAQ,IAAI,EAAE,EAAC;QACnD,MAAA,cAAc,CAAC,QAAS,EAAC,KAAK,QAAL,KAAK,GAAK,OAAO,CAAC,KAAK,EAAC;QACjD,OAAO,cAAc,CAAC;IAC1B,CAAC,CAAC,CAAC;AACX,CAAC;AAlBD,oDAkBC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"event_manager.d.ts","sourceRoot":"","sources":["../../src/events/event_manager.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAEzD,OAAO,EAAE,iBAAiB,EAAE,MAAM,iCAAiC,CAAC;AACpE,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEjD,0BAAkB,SAAS;IACvB,aAAa,iBAAiB;IAC9B,WAAW,eAAe;IAC1B,SAAS,cAAc;IACvB,QAAQ,aAAa;IACrB,IAAI,SAAS;CAChB;AAED,oBAAY,aAAa,GAAG,SAAS,GAAG,YAAY,GAAG,cAAc,GAAG,WAAW,GAAG,UAAU,GAAG,MAAM,CAAC;AAE1G,UAAU,SAAS;IACf,YAAY,CAAC,EAAE,gBAAgB,CAAC;IAChC,UAAU,CAAC,EAAE,gBAAgB,CAAC;CACjC;AAED,8BAAsB,YAAY;IAMlB,QAAQ,CAAC,MAAM;IAL3B,SAAS,CAAC,MAAM,sIAA2B;IAC3C,SAAS,CAAC,WAAW,UAAS;IAC9B,SAAS,CAAC,SAAS,EAAE,SAAS,CAAM;IACpC,SAAS,CAAC,GAAG,2BAAmC;gBAE3B,MAAM,gBAAkC;IAE7D;;;OAGG;IACG,IAAI;IAaV;;;OAGG;IACG,KAAK;
|
|
1
|
+
{"version":3,"file":"event_manager.d.ts","sourceRoot":"","sources":["../../src/events/event_manager.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAEzD,OAAO,EAAE,iBAAiB,EAAE,MAAM,iCAAiC,CAAC;AACpE,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEjD,0BAAkB,SAAS;IACvB,aAAa,iBAAiB;IAC9B,WAAW,eAAe;IAC1B,SAAS,cAAc;IACvB,QAAQ,aAAa;IACrB,IAAI,SAAS;CAChB;AAED,oBAAY,aAAa,GAAG,SAAS,GAAG,YAAY,GAAG,cAAc,GAAG,WAAW,GAAG,UAAU,GAAG,MAAM,CAAC;AAE1G,UAAU,SAAS;IACf,YAAY,CAAC,EAAE,gBAAgB,CAAC;IAChC,UAAU,CAAC,EAAE,gBAAgB,CAAC;CACjC;AAED,8BAAsB,YAAY;IAMlB,QAAQ,CAAC,MAAM;IAL3B,SAAS,CAAC,MAAM,sIAA2B;IAC3C,SAAS,CAAC,WAAW,UAAS;IAC9B,SAAS,CAAC,SAAS,EAAE,SAAS,CAAM;IACpC,SAAS,CAAC,GAAG,2BAAmC;gBAE3B,MAAM,gBAAkC;IAE7D;;;OAGG;IACG,IAAI;IAaV;;;OAGG;IACG,KAAK;IAeX,EAAE,CAAC,KAAK,EAAE,aAAa,EAAE,QAAQ,EAAE,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,GAAG,GAAG,IAAI;IAIjE,GAAG,CAAC,KAAK,EAAE,aAAa,EAAE,QAAQ,CAAC,EAAE,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,GAAG,GAAG,IAAI;IAQnE,IAAI,CAAC,KAAK,EAAE,aAAa,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI;IAIpD,aAAa,IAAI,OAAO;IAIxB;;OAEG;IACH,aAAa,CAAC,KAAK,EAAE,aAAa,GAAG,MAAM;IAI3C;;OAEG;IACH,SAAS,CAAC,KAAK,EAAE,aAAa,GAAG,CAAC,MAAM,OAAO,CAAC,OAAO,CAAC,CAAC,EAAE;IAI3D;;OAEG;IACH,6BAA6B;CAGhC"}
|
package/events/event_manager.js
CHANGED
|
@@ -74,6 +74,8 @@ class EventManager {
|
|
|
74
74
|
this.initialized = false;
|
|
75
75
|
// Emit final PERSIST_STATE event
|
|
76
76
|
this.emit("persistState" /* EventType.PERSIST_STATE */, { isMigrating: false });
|
|
77
|
+
// Wait for PERSIST_STATE to process
|
|
78
|
+
await this.waitForAllListenersToComplete();
|
|
77
79
|
}
|
|
78
80
|
on(event, listener) {
|
|
79
81
|
this.events.on(event, listener);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"event_manager.js","sourceRoot":"","sources":["../../src/events/event_manager.ts"],"names":[],"mappings":";;;;AAAA,6DAA6B;AAE7B,gDAA0E;AAC1E,yEAAoE;AACpE,oDAAiD;AAEjD,IAAkB,SAMjB;AAND,WAAkB,SAAS;IACvB,2CAA8B,CAAA;IAC9B,uCAA0B,CAAA;IAC1B,oCAAuB,CAAA;IACvB,kCAAqB,CAAA;IACrB,0BAAa,CAAA;AACjB,CAAC,EANiB,SAAS,GAAT,iBAAS,KAAT,iBAAS,QAM1B;AASD,MAAsB,YAAY;IAM9B,YAAqB,SAAS,6BAAa,CAAC,eAAe,EAAE;;;;;mBAAxC;;QALrB;;;;mBAAmB,IAAI,uCAAiB,EAAE;WAAC;QAC3C;;;;mBAAwB,KAAK;WAAC;QAC9B;;;;mBAAiC,EAAE;WAAC;QACpC;;;;mBAAgB,aAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC;WAAC;IAEgB,CAAC;IAEjE;;;OAGG;IACH,KAAK,CAAC,IAAI;QACN,IAAI,IAAI,CAAC,WAAW,EAAE;YAClB,OAAO;SACV;QAED,MAAM,0BAA0B,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,4BAA4B,CAAE,CAAC;QAClF,IAAI,CAAC,SAAS,CAAC,YAAY,GAAG,IAAA,6BAAiB,EAAC,CAAC,gBAA+B,EAAE,EAAE;YAChF,IAAI,CAAC,IAAI,+CAA0B,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC,CAAC;YAC3D,gBAAgB,EAAE,CAAC;QACvB,CAAC,EAAE,0BAA0B,CAAC,CAAC;QAC/B,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;IAC5B,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,KAAK;QACP,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE;YACnB,OAAO;SACV;QAED,IAAA,+BAAmB,EAAC,IAAI,CAAC,SAAS,CAAC,YAAa,CAAC,CAAC;QAClD,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;QAEzB,iCAAiC;QACjC,IAAI,CAAC,IAAI,+CAA0B,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC,CAAC;IAC/
|
|
1
|
+
{"version":3,"file":"event_manager.js","sourceRoot":"","sources":["../../src/events/event_manager.ts"],"names":[],"mappings":";;;;AAAA,6DAA6B;AAE7B,gDAA0E;AAC1E,yEAAoE;AACpE,oDAAiD;AAEjD,IAAkB,SAMjB;AAND,WAAkB,SAAS;IACvB,2CAA8B,CAAA;IAC9B,uCAA0B,CAAA;IAC1B,oCAAuB,CAAA;IACvB,kCAAqB,CAAA;IACrB,0BAAa,CAAA;AACjB,CAAC,EANiB,SAAS,GAAT,iBAAS,KAAT,iBAAS,QAM1B;AASD,MAAsB,YAAY;IAM9B,YAAqB,SAAS,6BAAa,CAAC,eAAe,EAAE;;;;;mBAAxC;;QALrB;;;;mBAAmB,IAAI,uCAAiB,EAAE;WAAC;QAC3C;;;;mBAAwB,KAAK;WAAC;QAC9B;;;;mBAAiC,EAAE;WAAC;QACpC;;;;mBAAgB,aAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC;WAAC;IAEgB,CAAC;IAEjE;;;OAGG;IACH,KAAK,CAAC,IAAI;QACN,IAAI,IAAI,CAAC,WAAW,EAAE;YAClB,OAAO;SACV;QAED,MAAM,0BAA0B,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,4BAA4B,CAAE,CAAC;QAClF,IAAI,CAAC,SAAS,CAAC,YAAY,GAAG,IAAA,6BAAiB,EAAC,CAAC,gBAA+B,EAAE,EAAE;YAChF,IAAI,CAAC,IAAI,+CAA0B,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC,CAAC;YAC3D,gBAAgB,EAAE,CAAC;QACvB,CAAC,EAAE,0BAA0B,CAAC,CAAC;QAC/B,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;IAC5B,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,KAAK;QACP,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE;YACnB,OAAO;SACV;QAED,IAAA,+BAAmB,EAAC,IAAI,CAAC,SAAS,CAAC,YAAa,CAAC,CAAC;QAClD,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;QAEzB,iCAAiC;QACjC,IAAI,CAAC,IAAI,+CAA0B,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC,CAAC;QAE3D,oCAAoC;QACpC,MAAM,IAAI,CAAC,6BAA6B,EAAE,CAAC;IAC/C,CAAC;IAED,EAAE,CAAC,KAAoB,EAAE,QAAiC;QACtD,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;IACpC,CAAC;IAED,GAAG,CAAC,KAAoB,EAAE,QAAkC;QACxD,IAAI,QAAQ,EAAE;YACV,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;SAC/C;aAAM;YACH,IAAI,CAAC,MAAM,CAAC,kBAAkB,CAAC,KAAK,CAAC,CAAC;SACzC;IACL,CAAC;IAED,IAAI,CAAC,KAAoB,EAAE,GAAG,IAAe;QACzC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,GAAG,IAAI,CAAC,CAAC;IACrC,CAAC;IAED,aAAa;QACT,OAAO,IAAI,CAAC,WAAW,CAAC;IAC5B,CAAC;IAED;;OAEG;IACH,aAAa,CAAC,KAAoB;QAC9B,OAAO,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;IAC5C,CAAC;IAED;;OAEG;IACH,SAAS,CAAC,KAAoB;QAC1B,OAAO,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAA+B,CAAC;IACtE,CAAC;IAED;;OAEG;IACH,6BAA6B;QACzB,OAAO,IAAI,CAAC,MAAM,CAAC,6BAA6B,EAAE,CAAC;IACvD,CAAC;CACJ;AApFD,oCAoFC"}
|
package/except.d.ts
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
declare type IsEqual<T, U> = (<G>() => G extends T ? 1 : 2) extends (<G>() => G extends U ? 1 : 2) ? true : false;
|
|
2
|
+
declare type Filter<KeyType, ExcludeType> = IsEqual<KeyType, ExcludeType> extends true ? never : (KeyType extends ExcludeType ? never : KeyType);
|
|
3
|
+
export declare type Except<ObjectType, KeysType extends keyof ObjectType> = {
|
|
4
|
+
[KeyType in keyof ObjectType as Filter<KeyType, KeysType>]: ObjectType[KeyType];
|
|
5
|
+
};
|
|
6
|
+
export {};
|
|
7
|
+
//# sourceMappingURL=except.d.ts.map
|
package/except.d.ts.map
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"except.d.ts","sourceRoot":"","sources":["../src/except.ts"],"names":[],"mappings":"AAEA,aAAK,OAAO,CAAC,CAAC,EAAE,CAAC,IACb,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,SAC9B,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,GACxB,IAAI,GACJ,KAAK,CAAC;AAEhB,aAAK,MAAM,CAAC,OAAO,EAAE,WAAW,IAAI,OAAO,CAAC,OAAO,EAAE,WAAW,CAAC,SAAS,IAAI,GAAG,KAAK,GAAG,CAAC,OAAO,SAAS,WAAW,GAAG,KAAK,GAAG,OAAO,CAAC,CAAC;AAEzI,oBAAY,MAAM,CAAC,UAAU,EAAE,QAAQ,SAAS,MAAM,UAAU,IAAI;KAC/D,OAAO,IAAI,MAAM,UAAU,IAAI,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,GAAG,UAAU,CAAC,OAAO,CAAC;CAClF,CAAC"}
|
package/except.js
ADDED
package/except.js.map
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"except.js","sourceRoot":"","sources":["../src/except.ts"],"names":[],"mappings":";AAAA,oCAAoC"}
|
package/index.d.ts
CHANGED
|
@@ -14,5 +14,5 @@ export * from './storages';
|
|
|
14
14
|
export * from './validators';
|
|
15
15
|
export * from './cookie_utils';
|
|
16
16
|
export { PseudoUrl } from '@apify/pseudo_url';
|
|
17
|
-
export { Dictionary, Awaitable, Constructor, StorageClient } from '@crawlee/types';
|
|
17
|
+
export { Dictionary, Awaitable, Constructor, StorageClient, Cookie } from '@crawlee/types';
|
|
18
18
|
//# sourceMappingURL=index.d.ts.map
|
package/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,UAAU,CAAC;AACzB,cAAc,eAAe,CAAC;AAC9B,cAAc,iBAAiB,CAAC;AAChC,cAAc,YAAY,CAAC;AAC3B,cAAc,iBAAiB,CAAC;AAChC,cAAc,UAAU,CAAC;AACzB,cAAc,OAAO,CAAC;AACtB,cAAc,uBAAuB,CAAC;AACtC,cAAc,WAAW,CAAC;AAC1B,cAAc,UAAU,CAAC;AACzB,cAAc,iBAAiB,CAAC;AAChC,cAAc,gBAAgB,CAAC;AAC/B,cAAc,YAAY,CAAC;AAC3B,cAAc,cAAc,CAAC;AAC7B,cAAc,gBAAgB,CAAC;AAC/B,OAAO,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAC9C,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,WAAW,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,UAAU,CAAC;AACzB,cAAc,eAAe,CAAC;AAC9B,cAAc,iBAAiB,CAAC;AAChC,cAAc,YAAY,CAAC;AAC3B,cAAc,iBAAiB,CAAC;AAChC,cAAc,UAAU,CAAC;AACzB,cAAc,OAAO,CAAC;AACtB,cAAc,uBAAuB,CAAC;AACtC,cAAc,WAAW,CAAC;AAC1B,cAAc,UAAU,CAAC;AACzB,cAAc,iBAAiB,CAAC;AAChC,cAAc,gBAAgB,CAAC;AAC/B,cAAc,YAAY,CAAC;AAC3B,cAAc,cAAc,CAAC;AAC7B,cAAc,gBAAgB,CAAC;AAC/B,OAAO,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAC9C,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,WAAW,EAAE,aAAa,EAAE,MAAM,EAAE,MAAM,gBAAgB,CAAC"}
|
package/index.mjs
CHANGED
|
@@ -53,6 +53,7 @@ export const createRequestOptions = mod.createRequestOptions;
|
|
|
53
53
|
export const createRequests = mod.createRequests;
|
|
54
54
|
export const deserializeArray = mod.deserializeArray;
|
|
55
55
|
export const enqueueLinks = mod.enqueueLinks;
|
|
56
|
+
export const filterRequestsByPatterns = mod.filterRequestsByPatterns;
|
|
56
57
|
export const getCookiesFromResponse = mod.getCookiesFromResponse;
|
|
57
58
|
export const getDefaultCookieExpirationDate = mod.getDefaultCookieExpirationDate;
|
|
58
59
|
export const getRequestId = mod.getRequestId;
|