@crawlee/core 3.0.0-beta.32 → 3.0.0-beta.35
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/events/event_manager.d.ts +1 -0
- package/events/event_manager.d.ts.map +1 -1
- package/events/event_manager.js +3 -0
- package/events/event_manager.js.map +1 -1
- package/index.d.ts +1 -0
- package/index.d.ts.map +1 -1
- package/package.json +5 -5
- package/router.js +2 -2
- package/router.js.map +1 -1
- package/storages/dataset.d.ts +3 -1
- package/storages/dataset.d.ts.map +1 -1
- package/storages/dataset.js +11 -4
- package/storages/dataset.js.map +1 -1
- package/storages/key_value_store.d.ts +14 -3
- package/storages/key_value_store.d.ts.map +1 -1
- package/storages/key_value_store.js +67 -7
- package/storages/key_value_store.js.map +1 -1
- package/storages/request_list.js +1 -1
- package/storages/request_list.js.map +1 -1
- package/storages/request_queue.d.ts +5 -2
- package/storages/request_queue.d.ts.map +1 -1
- package/storages/request_queue.js +14 -4
- package/storages/request_queue.js.map +1 -1
- package/storages/storage_manager.d.ts +8 -4
- package/storages/storage_manager.d.ts.map +1 -1
- package/storages/storage_manager.js +28 -9
- package/storages/storage_manager.js.map +1 -1
- package/tsconfig.build.tsbuildinfo +1 -1
|
@@ -34,6 +34,7 @@ export declare abstract class EventManager {
|
|
|
34
34
|
on(event: EventTypeName, listener: (...args: any[]) => any): void;
|
|
35
35
|
off(event: EventTypeName, listener?: (...args: any[]) => any): void;
|
|
36
36
|
emit(event: EventTypeName, ...args: unknown[]): void;
|
|
37
|
+
isInitialized(): boolean;
|
|
37
38
|
/**
|
|
38
39
|
* @internal
|
|
39
40
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"event_manager.d.ts","sourceRoot":"","sources":["../../src/events/event_manager.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EAAuB,gBAAgB,EAAqB,MAAM,kBAAkB,CAAC;AAC5F,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEjD,0BAAkB,SAAS;IACvB,aAAa,iBAAiB;IAC9B,WAAW,eAAe;IAC1B,SAAS,cAAc;IACvB,QAAQ,aAAa;IACrB,IAAI,SAAS;CAChB;AAED,oBAAY,aAAa,GAAG,SAAS,GAAG,YAAY,GAAG,cAAc,GAAG,WAAW,GAAG,UAAU,GAAG,MAAM,CAAC;AAE1G,UAAU,SAAS;IACf,YAAY,CAAC,EAAE,gBAAgB,CAAC;IAChC,UAAU,CAAC,EAAE,gBAAgB,CAAC;CACjC;AAED,8BAAsB,YAAY;IAMlB,QAAQ,CAAC,MAAM;IAL3B,SAAS,CAAC,MAAM,eAAsB;IACtC,SAAS,CAAC,WAAW,UAAS;IAC9B,SAAS,CAAC,SAAS,EAAE,SAAS,CAAM;IACpC,SAAS,CAAC,GAAG,2BAAmC;gBAE3B,MAAM,gBAAkC;IAE7D;;;OAGG;IACG,IAAI;IAaV;;;OAGG;IACG,KAAK;IASX,EAAE,CAAC,KAAK,EAAE,aAAa,EAAE,QAAQ,EAAE,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,GAAG,GAAG,IAAI;IAIjE,GAAG,CAAC,KAAK,EAAE,aAAa,EAAE,QAAQ,CAAC,EAAE,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,GAAG,GAAG,IAAI;IAQnE,IAAI,CAAC,KAAK,EAAE,aAAa,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI;IAIpD;;OAEG;IACH,aAAa,CAAC,KAAK,EAAE,aAAa,GAAG,MAAM;IAI3C;;OAEG;IACH,SAAS,CAAC,KAAK,EAAE,aAAa,GAAG,CAAC,MAAM,OAAO,CAAC,OAAO,CAAC,CAAC,EAAE;CAG9D"}
|
|
1
|
+
{"version":3,"file":"event_manager.d.ts","sourceRoot":"","sources":["../../src/events/event_manager.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EAAuB,gBAAgB,EAAqB,MAAM,kBAAkB,CAAC;AAC5F,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEjD,0BAAkB,SAAS;IACvB,aAAa,iBAAiB;IAC9B,WAAW,eAAe;IAC1B,SAAS,cAAc;IACvB,QAAQ,aAAa;IACrB,IAAI,SAAS;CAChB;AAED,oBAAY,aAAa,GAAG,SAAS,GAAG,YAAY,GAAG,cAAc,GAAG,WAAW,GAAG,UAAU,GAAG,MAAM,CAAC;AAE1G,UAAU,SAAS;IACf,YAAY,CAAC,EAAE,gBAAgB,CAAC;IAChC,UAAU,CAAC,EAAE,gBAAgB,CAAC;CACjC;AAED,8BAAsB,YAAY;IAMlB,QAAQ,CAAC,MAAM;IAL3B,SAAS,CAAC,MAAM,eAAsB;IACtC,SAAS,CAAC,WAAW,UAAS;IAC9B,SAAS,CAAC,SAAS,EAAE,SAAS,CAAM;IACpC,SAAS,CAAC,GAAG,2BAAmC;gBAE3B,MAAM,gBAAkC;IAE7D;;;OAGG;IACG,IAAI;IAaV;;;OAGG;IACG,KAAK;IASX,EAAE,CAAC,KAAK,EAAE,aAAa,EAAE,QAAQ,EAAE,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,GAAG,GAAG,IAAI;IAIjE,GAAG,CAAC,KAAK,EAAE,aAAa,EAAE,QAAQ,CAAC,EAAE,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,GAAG,GAAG,IAAI;IAQnE,IAAI,CAAC,KAAK,EAAE,aAAa,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI;IAIpD,aAAa,IAAI,OAAO;IAIxB;;OAEG;IACH,aAAa,CAAC,KAAK,EAAE,aAAa,GAAG,MAAM;IAI3C;;OAEG;IACH,SAAS,CAAC,KAAK,EAAE,aAAa,GAAG,CAAC,MAAM,OAAO,CAAC,OAAO,CAAC,CAAC,EAAE;CAG9D"}
|
package/events/event_manager.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"event_manager.js","sourceRoot":"","sources":["../../src/events/event_manager.ts"],"names":[],"mappings":";;;;AAAA,6DAA6B;AAC7B,6CAA2C;AAC3C,gDAA4F;AAC5F,oDAAiD;AAEjD,IAAkB,SAMjB;AAND,WAAkB,SAAS;IACvB,2CAA8B,CAAA;IAC9B,uCAA0B,CAAA;IAC1B,oCAAuB,CAAA;IACvB,kCAAqB,CAAA;IACrB,0BAAa,CAAA;AACjB,CAAC,EANiB,SAAS,GAAT,iBAAS,KAAT,iBAAS,QAM1B;AASD,MAAsB,YAAY;IAM9B,YAAqB,SAAS,6BAAa,CAAC,eAAe,EAAE;;;;;mBAAxC;;QALrB;;;;mBAAmB,IAAI,0BAAY,EAAE;WAAC;QACtC;;;;mBAAwB,KAAK;WAAC;QAC9B;;;;mBAAiC,EAAE;WAAC;QACpC;;;;mBAAgB,aAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC;WAAC;IAEgB,CAAC;IAEjE;;;OAGG;IACH,KAAK,CAAC,IAAI;QACN,IAAI,IAAI,CAAC,WAAW,EAAE;YAClB,OAAO;SACV;QAED,MAAM,0BAA0B,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,4BAA4B,CAAE,CAAC;QAClF,IAAI,CAAC,SAAS,CAAC,YAAY,GAAG,IAAA,6BAAiB,EAAC,CAAC,gBAA+B,EAAE,EAAE;YAChF,IAAI,CAAC,IAAI,+CAA0B,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC,CAAC;YAC3D,gBAAgB,EAAE,CAAC;QACvB,CAAC,EAAE,0BAA0B,CAAC,CAAC;QAC/B,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;IAC5B,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,KAAK;QACP,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE;YACnB,OAAO;SACV;QAED,IAAA,+BAAmB,EAAC,IAAI,CAAC,SAAS,CAAC,YAAa,CAAC,CAAC;QAClD,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;IAC7B,CAAC;IAED,EAAE,CAAC,KAAoB,EAAE,QAAiC;QACtD,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;IACpC,CAAC;IAED,GAAG,CAAC,KAAoB,EAAE,QAAkC;QACxD,IAAI,QAAQ,EAAE;YACV,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;SAC/C;aAAM;YACH,IAAI,CAAC,MAAM,CAAC,kBAAkB,CAAC,KAAK,CAAC,CAAC;SACzC;IACL,CAAC;IAED,IAAI,CAAC,KAAoB,EAAE,GAAG,IAAe;QACzC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,GAAG,IAAI,CAAC,CAAC;IACrC,CAAC;IAED;;OAEG;IACH,aAAa,CAAC,KAAoB;QAC9B,OAAO,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;IAC5C,CAAC;IAED;;OAEG;IACH,SAAS,CAAC,KAAoB;QAC1B,OAAO,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAA+B,CAAC;IACtE,CAAC;CACJ;
|
|
1
|
+
{"version":3,"file":"event_manager.js","sourceRoot":"","sources":["../../src/events/event_manager.ts"],"names":[],"mappings":";;;;AAAA,6DAA6B;AAC7B,6CAA2C;AAC3C,gDAA4F;AAC5F,oDAAiD;AAEjD,IAAkB,SAMjB;AAND,WAAkB,SAAS;IACvB,2CAA8B,CAAA;IAC9B,uCAA0B,CAAA;IAC1B,oCAAuB,CAAA;IACvB,kCAAqB,CAAA;IACrB,0BAAa,CAAA;AACjB,CAAC,EANiB,SAAS,GAAT,iBAAS,KAAT,iBAAS,QAM1B;AASD,MAAsB,YAAY;IAM9B,YAAqB,SAAS,6BAAa,CAAC,eAAe,EAAE;;;;;mBAAxC;;QALrB;;;;mBAAmB,IAAI,0BAAY,EAAE;WAAC;QACtC;;;;mBAAwB,KAAK;WAAC;QAC9B;;;;mBAAiC,EAAE;WAAC;QACpC;;;;mBAAgB,aAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC;WAAC;IAEgB,CAAC;IAEjE;;;OAGG;IACH,KAAK,CAAC,IAAI;QACN,IAAI,IAAI,CAAC,WAAW,EAAE;YAClB,OAAO;SACV;QAED,MAAM,0BAA0B,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,4BAA4B,CAAE,CAAC;QAClF,IAAI,CAAC,SAAS,CAAC,YAAY,GAAG,IAAA,6BAAiB,EAAC,CAAC,gBAA+B,EAAE,EAAE;YAChF,IAAI,CAAC,IAAI,+CAA0B,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC,CAAC;YAC3D,gBAAgB,EAAE,CAAC;QACvB,CAAC,EAAE,0BAA0B,CAAC,CAAC;QAC/B,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;IAC5B,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,KAAK;QACP,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE;YACnB,OAAO;SACV;QAED,IAAA,+BAAmB,EAAC,IAAI,CAAC,SAAS,CAAC,YAAa,CAAC,CAAC;QAClD,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;IAC7B,CAAC;IAED,EAAE,CAAC,KAAoB,EAAE,QAAiC;QACtD,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;IACpC,CAAC;IAED,GAAG,CAAC,KAAoB,EAAE,QAAkC;QACxD,IAAI,QAAQ,EAAE;YACV,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;SAC/C;aAAM;YACH,IAAI,CAAC,MAAM,CAAC,kBAAkB,CAAC,KAAK,CAAC,CAAC;SACzC;IACL,CAAC;IAED,IAAI,CAAC,KAAoB,EAAE,GAAG,IAAe;QACzC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,GAAG,IAAI,CAAC,CAAC;IACrC,CAAC;IAED,aAAa;QACT,OAAO,IAAI,CAAC,WAAW,CAAC;IAC5B,CAAC;IAED;;OAEG;IACH,aAAa,CAAC,KAAoB;QAC9B,OAAO,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;IAC5C,CAAC;IAED;;OAEG;IACH,SAAS,CAAC,KAAoB;QAC1B,OAAO,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAA+B,CAAC;IACtE,CAAC;CACJ;AAvED,oCAuEC"}
|
package/index.d.ts
CHANGED
package/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,eAAe,CAAC;AAC9B,cAAc,iBAAiB,CAAC;AAChC,cAAc,aAAa,CAAC;AAC5B,cAAc,YAAY,CAAC;AAC3B,cAAc,iBAAiB,CAAC;AAChC,cAAc,UAAU,CAAC;AACzB,cAAc,OAAO,CAAC;AACtB,cAAc,uBAAuB,CAAC;AACtC,cAAc,WAAW,CAAC;AAC1B,cAAc,UAAU,CAAC;AACzB,cAAc,iBAAiB,CAAC;AAChC,cAAc,gBAAgB,CAAC;AAC/B,cAAc,YAAY,CAAC;AAC3B,cAAc,cAAc,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,eAAe,CAAC;AAC9B,cAAc,iBAAiB,CAAC;AAChC,cAAc,aAAa,CAAC;AAC5B,cAAc,YAAY,CAAC;AAC3B,cAAc,iBAAiB,CAAC;AAChC,cAAc,UAAU,CAAC;AACzB,cAAc,OAAO,CAAC;AACtB,cAAc,uBAAuB,CAAC;AACtC,cAAc,WAAW,CAAC;AAC1B,cAAc,UAAU,CAAC;AACzB,cAAc,iBAAiB,CAAC;AAChC,cAAc,gBAAgB,CAAC;AAC/B,cAAc,YAAY,CAAC;AAC3B,cAAc,cAAc,CAAC;AAC7B,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/core",
|
|
3
|
-
"version": "3.0.0-beta.
|
|
3
|
+
"version": "3.0.0-beta.35",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=16.0.0"
|
|
@@ -55,10 +55,10 @@
|
|
|
55
55
|
"@apify/pseudo_url": "^1.1.4",
|
|
56
56
|
"@apify/timeout": "^0.2.1",
|
|
57
57
|
"@apify/utilities": "^1.2.12",
|
|
58
|
-
"@crawlee/browser-pool": "^3.0.0-beta.
|
|
59
|
-
"@crawlee/memory-storage": "^3.0.0-beta.
|
|
60
|
-
"@crawlee/types": "^3.0.0-beta.
|
|
61
|
-
"@crawlee/utils": "^3.0.0-beta.
|
|
58
|
+
"@crawlee/browser-pool": "^3.0.0-beta.35",
|
|
59
|
+
"@crawlee/memory-storage": "^3.0.0-beta.35",
|
|
60
|
+
"@crawlee/types": "^3.0.0-beta.35",
|
|
61
|
+
"@crawlee/utils": "^3.0.0-beta.35",
|
|
62
62
|
"@types/tough-cookie": "^4.0.2",
|
|
63
63
|
"iconv-lite": "^0.6.3",
|
|
64
64
|
"minimatch": "^5.1.0",
|
package/router.js
CHANGED
|
@@ -124,8 +124,8 @@ class Router {
|
|
|
124
124
|
obj.addDefaultHandler = router.addDefaultHandler.bind(router);
|
|
125
125
|
obj.getHandler = router.getHandler.bind(router);
|
|
126
126
|
const func = function (context) {
|
|
127
|
-
const { url, label } = context.request;
|
|
128
|
-
context.log.info('Page opened.', { label, url });
|
|
127
|
+
const { url, loadedUrl, label } = context.request;
|
|
128
|
+
context.log.info('Page opened.', { label, url: loadedUrl ?? url });
|
|
129
129
|
return router.getHandler(label)(context);
|
|
130
130
|
};
|
|
131
131
|
Object.setPrototypeOf(func, obj);
|
package/router.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"router.js","sourceRoot":"","sources":["../src/router.ts"],"names":[],"mappings":";;;AAGA,MAAM,YAAY,GAAG,MAAM,CAAC,eAAe,CAAC,CAAC;AAM7C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyCG;AACH,MAAa,MAAM;IAGf;;;OAGG;IACH;QANA;;;;mBAAmF,IAAI,GAAG,EAAE;WAAC;IAMpE,CAAC;IAE1B;;OAEG;IACH,UAAU,CAAC,KAAsB,EAAE,OAA0C;QACzE,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;QACrB,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;IACpC,CAAC;IAED;;OAEG;IACH,iBAAiB,CAAC,OAA0C;QACxD,IAAI,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;QAC5B,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;IAC3C,CAAC;IAED;;OAEG;IACH,UAAU,CAAC,KAAuB;QAC9B,IAAI,KAAK,IAAI,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE;YACjC,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAE,CAAC;SAClC;QAED,IAAI,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,YAAY,CAAC,EAAE;YAC/B,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,YAAY,CAAE,CAAC;SACzC;QAED,IAAI,CAAC,KAAK,EAAE;YACR,MAAM,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAC;SAC/C;QAED,MAAM,IAAI,KAAK,CAAC,8BAA8B,MAAM,CAAC,KAAK,CAAC,gCAAgC,CAAC,CAAC;IACjG,CAAC;IAED;;OAEG;IACK,QAAQ,CAAC,KAAsB;QACnC,IAAI,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE;YACxB,MAAM,OAAO,GAAG,KAAK,KAAK,YAAY;gBAClC,CAAC,CAAC,mCAAmC;gBACrC,CAAC,CAAC,oBAAoB,MAAM,CAAC,KAAK,CAAC,uBAAuB,CAAC;YAC/D,MAAM,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC;SAC5B;IACL,CAAC;IAED;;;;;;;;;;;;;;;;;;;OAmBG;IACH,MAAM,CAAC,MAAM;QACT,MAAM,MAAM,GAAG,IAAI,MAAM,EAAW,CAAC;QACrC,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;QAE9C,GAAG,CAAC,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAChD,GAAG,CAAC,iBAAiB,GAAG,MAAM,CAAC,iBAAiB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC9D,GAAG,CAAC,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAEhD,MAAM,IAAI,GAAG,UAAU,OAAgB;YACnC,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,OAAO,CAAC,OAAO,CAAC;
|
|
1
|
+
{"version":3,"file":"router.js","sourceRoot":"","sources":["../src/router.ts"],"names":[],"mappings":";;;AAGA,MAAM,YAAY,GAAG,MAAM,CAAC,eAAe,CAAC,CAAC;AAM7C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyCG;AACH,MAAa,MAAM;IAGf;;;OAGG;IACH;QANA;;;;mBAAmF,IAAI,GAAG,EAAE;WAAC;IAMpE,CAAC;IAE1B;;OAEG;IACH,UAAU,CAAC,KAAsB,EAAE,OAA0C;QACzE,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;QACrB,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;IACpC,CAAC;IAED;;OAEG;IACH,iBAAiB,CAAC,OAA0C;QACxD,IAAI,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;QAC5B,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;IAC3C,CAAC;IAED;;OAEG;IACH,UAAU,CAAC,KAAuB;QAC9B,IAAI,KAAK,IAAI,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE;YACjC,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAE,CAAC;SAClC;QAED,IAAI,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,YAAY,CAAC,EAAE;YAC/B,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,YAAY,CAAE,CAAC;SACzC;QAED,IAAI,CAAC,KAAK,EAAE;YACR,MAAM,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAC;SAC/C;QAED,MAAM,IAAI,KAAK,CAAC,8BAA8B,MAAM,CAAC,KAAK,CAAC,gCAAgC,CAAC,CAAC;IACjG,CAAC;IAED;;OAEG;IACK,QAAQ,CAAC,KAAsB;QACnC,IAAI,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE;YACxB,MAAM,OAAO,GAAG,KAAK,KAAK,YAAY;gBAClC,CAAC,CAAC,mCAAmC;gBACrC,CAAC,CAAC,oBAAoB,MAAM,CAAC,KAAK,CAAC,uBAAuB,CAAC;YAC/D,MAAM,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC;SAC5B;IACL,CAAC;IAED;;;;;;;;;;;;;;;;;;;OAmBG;IACH,MAAM,CAAC,MAAM;QACT,MAAM,MAAM,GAAG,IAAI,MAAM,EAAW,CAAC;QACrC,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;QAE9C,GAAG,CAAC,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAChD,GAAG,CAAC,iBAAiB,GAAG,MAAM,CAAC,iBAAiB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC9D,GAAG,CAAC,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAEhD,MAAM,IAAI,GAAG,UAAU,OAAgB;YACnC,MAAM,EAAE,GAAG,EAAE,SAAS,EAAE,KAAK,EAAE,GAAG,OAAO,CAAC,OAAO,CAAC;YAClD,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,cAAc,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,SAAS,IAAI,GAAG,EAAE,CAAC,CAAC;YACnE,OAAO,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,CAAC;QAC7C,CAAC,CAAC;QAEF,MAAM,CAAC,cAAc,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QAEjC,OAAO,IAAyC,CAAC;IACrD,CAAC;CACJ;AA9FD,wBA8FC"}
|
package/storages/dataset.d.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { DatasetClient, DatasetInfo, PaginatedList, StorageClient, Dictionary } from '@crawlee/types';
|
|
2
|
+
import { Configuration } from '../configuration';
|
|
2
3
|
import { Awaitable } from '../typedefs';
|
|
3
4
|
import { StorageManagerOptions } from './storage_manager';
|
|
4
5
|
/** @internal */
|
|
@@ -130,6 +131,7 @@ export interface DatasetIteratorOptions extends Omit<DatasetDataOptions, 'offset
|
|
|
130
131
|
* @category Result Stores
|
|
131
132
|
*/
|
|
132
133
|
export declare class Dataset<Data extends Dictionary = Dictionary> {
|
|
134
|
+
readonly config: Configuration;
|
|
133
135
|
id: string;
|
|
134
136
|
name?: string;
|
|
135
137
|
client: DatasetClient<Data>;
|
|
@@ -137,7 +139,7 @@ export declare class Dataset<Data extends Dictionary = Dictionary> {
|
|
|
137
139
|
/**
|
|
138
140
|
* @internal
|
|
139
141
|
*/
|
|
140
|
-
constructor(options: DatasetOptions);
|
|
142
|
+
constructor(options: DatasetOptions, config?: Configuration);
|
|
141
143
|
/**
|
|
142
144
|
* Stores an object or an array of objects to the dataset.
|
|
143
145
|
* The function returns a promise that resolves when the operation finishes.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"dataset.d.ts","sourceRoot":"","sources":["../../src/storages/dataset.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,aAAa,EAAE,WAAW,EAAE,aAAa,EAAE,aAAa,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;
|
|
1
|
+
{"version":3,"file":"dataset.d.ts","sourceRoot":"","sources":["../../src/storages/dataset.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,aAAa,EAAE,WAAW,EAAE,aAAa,EAAE,aAAa,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAC3G,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEjD,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AACxC,OAAO,EAAkB,qBAAqB,EAAE,MAAM,mBAAmB,CAAC;AAE1E,gBAAgB;AAChB,eAAO,MAAM,+BAA+B,QAAQ,CAAC;AAIrD;;;;;GAKG;AACH,wBAAgB,iBAAiB,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,UAAU,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,CAsBxF;AAED;;;;;;;;GAQG;AACH,wBAAgB,WAAW,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,UAAU,EAAE,MAAM,GAAG,MAAM,EAAE,CA8BzE;AAED,MAAM,WAAW,kBAAkB;IAC/B;;;OAGG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf;;;;OAIG;IACH,IAAI,CAAC,EAAE,OAAO,CAAC;IAEf;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;IAElB;;;OAGG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB;;;;OAIG;IACH,KAAK,CAAC,EAAE,OAAO,CAAC;IAEhB;;;OAGG;IACH,UAAU,CAAC,EAAE,OAAO,CAAC;IAErB;;;;OAIG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;CACvB;AAED,MAAM,WAAW,sBAAuB,SAAQ,IAAI,CAAC,kBAAkB,EAAE,QAAQ,GAAG,OAAO,GAAG,OAAO,GAAG,YAAY,GAAG,WAAW,CAAC;IAC/H,gBAAgB;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf,gBAAgB;IAChB,KAAK,CAAC,EAAE,OAAO,CAAC;IAEhB,gBAAgB;IAChB,UAAU,CAAC,EAAE,OAAO,CAAC;IAErB,gBAAgB;IAChB,SAAS,CAAC,EAAE,OAAO,CAAC;IAEpB,gBAAgB;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;AACH,qBAAa,OAAO,CAAC,IAAI,SAAS,UAAU,GAAG,UAAU;IAShB,QAAQ,CAAC,MAAM;IARpD,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,aAAa,CAAC,IAAI,CAAC,CAAC;IAC5B,GAAG,+BAAoC;IAEvC;;OAEG;gBACS,OAAO,EAAE,cAAc,EAAW,MAAM,gBAAkC;IAMtF;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACG,QAAQ,CAAC,IAAI,EAAE,IAAI,GAAG,IAAI,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAqBlD;;;;;;OAMG;IACG,OAAO,CAAC,OAAO,GAAE,kBAAuB,GAAG,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;IAY7E;;;;;;;;;;;;;;;;;;;;;;OAsBG;IACG,OAAO,IAAI,OAAO,CAAC,WAAW,GAAG,SAAS,CAAC;IAIjD;;;;;;;;;;;;;;;;;;;OAmBG;IACG,OAAO,CAAC,QAAQ,EAAE,eAAe,CAAC,IAAI,CAAC,EAAE,OAAO,GAAE,sBAA2B,EAAE,KAAK,SAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IAkB9G;;;;;;;;OAQG;IACG,GAAG,CAAC,CAAC,EAAE,QAAQ,EAAE,aAAa,CAAC,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,GAAE,sBAA2B,GAAG,OAAO,CAAC,CAAC,EAAE,CAAC;IAWlG;;;;;;;;;;;;;;OAcG;IACG,MAAM,CAAC,CAAC,EAAE,QAAQ,EAAE,cAAc,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,OAAO,GAAE,sBAA2B,GAAG,OAAO,CAAC,CAAC,CAAC;IAoB7G;;;OAGG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAM3B;;;;;;;;;;;;;OAaG;WACU,IAAI,CAAC,IAAI,SAAS,UAAU,GAAG,UAAU,EAAE,eAAe,CAAC,EAAE,MAAM,GAAG,IAAI,EAAE,OAAO,GAAE,qBAA0B,GAAG,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IAWrJ;;;;;;;;;;;;;;;;;;;;;;;OAuBG;WACU,QAAQ,CAAC,IAAI,SAAS,UAAU,GAAG,UAAU,EAAE,IAAI,EAAE,IAAI,GAAG,IAAI,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;CAIlG;AAED;;GAEG;AACH,MAAM,WAAW,eAAe,CAAC,IAAI;IAEjC;;;OAGG;IACH,CAAC,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;CAEhD;AAED;;GAEG;AACH,MAAM,WAAW,aAAa,CAAC,IAAI,EAAE,CAAC;IAElC;;;;OAIG;IACH,CAAC,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;CAE7C;AAED;;GAEG;AACH,MAAM,WAAW,cAAc,CAAC,CAAC,EAAE,IAAI;IAEnC;;;;OAIG;IACH,CAAC,IAAI,EAAE,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;CAEtD;AAED,MAAM,WAAW,cAAc;IAC3B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,aAAa,CAAC;CACzB;AAGD,MAAM,WAAW,cAAc,CAAC,IAAI;IAChC,6CAA6C;IAC7C,KAAK,EAAE,MAAM,CAAC;IACd,qDAAqD;IACrD,KAAK,EAAE,MAAM,CAAC;IACd,2DAA2D;IAC3D,MAAM,EAAE,MAAM,CAAC;IACf,mDAAmD;IACnD,KAAK,EAAE,MAAM,CAAC;IACd,iDAAiD;IACjD,IAAI,EAAE,OAAO,CAAC;IACd,wDAAwD;IACxD,KAAK,EAAE,IAAI,EAAE,CAAC;CACjB"}
|
package/storages/dataset.js
CHANGED
|
@@ -130,7 +130,13 @@ class Dataset {
|
|
|
130
130
|
/**
|
|
131
131
|
* @internal
|
|
132
132
|
*/
|
|
133
|
-
constructor(options) {
|
|
133
|
+
constructor(options, config = configuration_1.Configuration.getGlobalConfig()) {
|
|
134
|
+
Object.defineProperty(this, "config", {
|
|
135
|
+
enumerable: true,
|
|
136
|
+
configurable: true,
|
|
137
|
+
writable: true,
|
|
138
|
+
value: config
|
|
139
|
+
});
|
|
134
140
|
Object.defineProperty(this, "id", {
|
|
135
141
|
enumerable: true,
|
|
136
142
|
configurable: true,
|
|
@@ -337,7 +343,7 @@ class Dataset {
|
|
|
337
343
|
*/
|
|
338
344
|
async drop() {
|
|
339
345
|
await this.client.delete();
|
|
340
|
-
const manager =
|
|
346
|
+
const manager = storage_manager_1.StorageManager.getManager(Dataset, this.config);
|
|
341
347
|
manager.closeStorage(this);
|
|
342
348
|
}
|
|
343
349
|
/**
|
|
@@ -359,8 +365,9 @@ class Dataset {
|
|
|
359
365
|
(0, ow_1.default)(options, ow_1.default.object.exactShape({
|
|
360
366
|
config: ow_1.default.optional.object.instanceOf(configuration_1.Configuration),
|
|
361
367
|
}));
|
|
362
|
-
|
|
363
|
-
|
|
368
|
+
options.config ?? (options.config = configuration_1.Configuration.getGlobalConfig());
|
|
369
|
+
const manager = storage_manager_1.StorageManager.getManager(Dataset, options.config);
|
|
370
|
+
return manager.openStorage(datasetIdOrName, options.config.getStorageClient());
|
|
364
371
|
}
|
|
365
372
|
/**
|
|
366
373
|
* Stores an object or an array of objects to the default {@link Dataset} of the current crawler run.
|
package/storages/dataset.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"dataset.js","sourceRoot":"","sources":["../../src/storages/dataset.ts"],"names":[],"mappings":";;;;AAAA,0CAAuD;AACvD,oDAAoB;AAEpB,oDAAiD;AACjD,gCAA6B;AAE7B,uDAA0E;AAE1E,gBAAgB;AACH,QAAA,+BAA+B,GAAG,KAAK,CAAC;AAErD,MAAM,qBAAqB,GAAG,IAAI,GAAG,GAAG,CAAC,CAAC,QAAQ;AAElD;;;;;GAKG;AACH,SAAgB,iBAAiB,CAAI,IAAO,EAAE,UAAkB,EAAE,KAAc;IAC5E,MAAM,CAAC,GAAG,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,aAAa,KAAK,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;IAClE,MAAM,YAAY,GAAG,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IAE9E,IAAI,CAAC,YAAY,EAAE;QACf,MAAM,IAAI,KAAK,CAAC,YAAY,CAAC,6DAA6D,CAAC,CAAC;KAC/F;IAED,IAAI,OAAO,CAAC;IACZ,IAAI;QACA,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;KAClC;IAAC,OAAO,CAAC,EAAE;QACR,MAAM,GAAG,GAAG,CAAU,CAAC;QACvB,MAAM,IAAI,KAAK,CAAC,YAAY,CAAC,wCAAwC,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;KACvF;IAED,MAAM,KAAK,GAAG,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;IACzC,IAAI,KAAK,GAAG,UAAU,EAAE;QACpB,MAAM,IAAI,KAAK,CAAC,YAAY,CAAC,uBAAuB,KAAK,kBAAkB,UAAU,SAAS,CAAC,CAAC;KACnG;IAED,OAAO,OAAO,CAAC;AACnB,CAAC;AAtBD,8CAsBC;AAED;;;;;;;;GAQG;AACH,SAAgB,WAAW,CAAC,KAAe,EAAE,UAAkB;IAC3D,IAAI,CAAC,KAAK,CAAC,MAAM;QAAE,OAAO,EAAE,CAAC;IAC7B,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAErC,6CAA6C;IAC7C,IAAI,cAAc,GAAG,CAAC,CAAC,CAAC,8BAA8B;IACtD,MAAM,MAAM,GAA0B,EAAE,CAAC;IAEzC,KAAK,MAAM,OAAO,IAAI,KAAK,EAAE;QACzB,MAAM,KAAK,GAAG,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;QAEzC,IAAI,KAAK,IAAI,UAAU,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,GAAG,UAAU,EAAE;YACjD,2EAA2E;YAC3E,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACrB,cAAc,GAAG,KAAK,CAAC;SAC1B;aAAM,IAAI,cAAc,GAAG,KAAK,IAAI,UAAU,EAAE;YAC7C,eAAe;YACf,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,EAAE;gBAC3C,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;aACnB;YACA,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAc,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACtD,cAAc,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC,gCAAgC;SAChE;aAAM;YACH,MAAM,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;YACvB,cAAc,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,8BAA8B;SAC7D;KACJ;IAED,0BAA0B;IAC1B,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;AAC/F,CAAC;AA9BD,kCA8BC;AA6ED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;AACH,MAAa,OAAO;IAMhB;;OAEG;IACH,YAAY,OAAuB
|
|
1
|
+
{"version":3,"file":"dataset.js","sourceRoot":"","sources":["../../src/storages/dataset.ts"],"names":[],"mappings":";;;;AAAA,0CAAuD;AACvD,oDAAoB;AAEpB,oDAAiD;AACjD,gCAA6B;AAE7B,uDAA0E;AAE1E,gBAAgB;AACH,QAAA,+BAA+B,GAAG,KAAK,CAAC;AAErD,MAAM,qBAAqB,GAAG,IAAI,GAAG,GAAG,CAAC,CAAC,QAAQ;AAElD;;;;;GAKG;AACH,SAAgB,iBAAiB,CAAI,IAAO,EAAE,UAAkB,EAAE,KAAc;IAC5E,MAAM,CAAC,GAAG,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,aAAa,KAAK,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;IAClE,MAAM,YAAY,GAAG,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IAE9E,IAAI,CAAC,YAAY,EAAE;QACf,MAAM,IAAI,KAAK,CAAC,YAAY,CAAC,6DAA6D,CAAC,CAAC;KAC/F;IAED,IAAI,OAAO,CAAC;IACZ,IAAI;QACA,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;KAClC;IAAC,OAAO,CAAC,EAAE;QACR,MAAM,GAAG,GAAG,CAAU,CAAC;QACvB,MAAM,IAAI,KAAK,CAAC,YAAY,CAAC,wCAAwC,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;KACvF;IAED,MAAM,KAAK,GAAG,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;IACzC,IAAI,KAAK,GAAG,UAAU,EAAE;QACpB,MAAM,IAAI,KAAK,CAAC,YAAY,CAAC,uBAAuB,KAAK,kBAAkB,UAAU,SAAS,CAAC,CAAC;KACnG;IAED,OAAO,OAAO,CAAC;AACnB,CAAC;AAtBD,8CAsBC;AAED;;;;;;;;GAQG;AACH,SAAgB,WAAW,CAAC,KAAe,EAAE,UAAkB;IAC3D,IAAI,CAAC,KAAK,CAAC,MAAM;QAAE,OAAO,EAAE,CAAC;IAC7B,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAErC,6CAA6C;IAC7C,IAAI,cAAc,GAAG,CAAC,CAAC,CAAC,8BAA8B;IACtD,MAAM,MAAM,GAA0B,EAAE,CAAC;IAEzC,KAAK,MAAM,OAAO,IAAI,KAAK,EAAE;QACzB,MAAM,KAAK,GAAG,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;QAEzC,IAAI,KAAK,IAAI,UAAU,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,GAAG,UAAU,EAAE;YACjD,2EAA2E;YAC3E,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACrB,cAAc,GAAG,KAAK,CAAC;SAC1B;aAAM,IAAI,cAAc,GAAG,KAAK,IAAI,UAAU,EAAE;YAC7C,eAAe;YACf,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,EAAE;gBAC3C,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;aACnB;YACA,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAc,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACtD,cAAc,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC,gCAAgC;SAChE;aAAM;YACH,MAAM,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;YACvB,cAAc,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,8BAA8B;SAC7D;KACJ;IAED,0BAA0B;IAC1B,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;AAC/F,CAAC;AA9BD,kCA8BC;AA6ED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;AACH,MAAa,OAAO;IAMhB;;OAEG;IACH,YAAY,OAAuB,EAAW,SAAS,6BAAa,CAAC,eAAe,EAAE;;;;;mBAAxC;;QAR9C;;;;;WAAW;QACX;;;;;WAAc;QACd;;;;;WAA4B;QAC5B;;;;mBAAM,SAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;WAAC;QAMnC,IAAI,CAAC,EAAE,GAAG,OAAO,CAAC,EAAE,CAAC;QACrB,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;QACzB,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAwB,CAAC;IACzE,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACH,KAAK,CAAC,QAAQ,CAAC,IAAmB;QAC9B,IAAA,YAAE,EAAC,IAAI,EAAE,MAAM,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC;QAC5B,MAAM,QAAQ,GAAG,CAAC,OAAe,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QACrE,MAAM,KAAK,GAAG,+BAAsB,GAAG,IAAI,CAAC,IAAI,CAAC,+BAAsB,GAAG,qBAAqB,CAAC,CAAC;QAEjG,0BAA0B;QAC1B,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE;YACtB,MAAM,OAAO,GAAG,iBAAiB,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;YAC/C,OAAO,QAAQ,CAAC,OAAO,CAAC,CAAC;SAC5B;QAED,gBAAgB;QAChB,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,iBAAiB,CAAC,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC;QAClF,MAAM,MAAM,GAAG,WAAW,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;QAE5C,oDAAoD;QACpD,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE;YACxB,MAAM,QAAQ,CAAC,KAAK,CAAC,CAAC;SACzB;IACL,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,OAAO,CAAC,UAA8B,EAAE;QAC1C,IAAI;YACA,OAAO,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;SAC/C;QAAC,OAAO,CAAC,EAAE;YACR,MAAM,KAAK,GAAG,CAAU,CAAC;YACzB,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,oCAAoC,CAAC,EAAE;gBAC9D,MAAM,IAAI,KAAK,CAAC,4GAA4G,CAAC,CAAC;aACjI;YACD,MAAM,CAAC,CAAC;SACX;IACL,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;OAsBG;IACH,KAAK,CAAC,OAAO;QACT,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC;IAC7B,CAAC;IAED;;;;;;;;;;;;;;;;;;;OAmBG;IACH,KAAK,CAAC,OAAO,CAAC,QAA+B,EAAE,UAAkC,EAAE,EAAE,KAAK,GAAG,CAAC;QAC1F,IAAI,CAAC,OAAO,CAAC,MAAM;YAAE,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC;QACxC,IAAI,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,MAAM,KAAK,MAAM;YAAE,MAAM,IAAI,KAAK,CAAC,4DAA4D,CAAC,CAAC;QAC/H,IAAI,CAAC,OAAO,CAAC,KAAK;YAAE,OAAO,CAAC,KAAK,GAAG,uCAA+B,CAAC;QAEpE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAEpE,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE;YACtB,MAAM,QAAQ,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;SACjC;QAED,MAAM,SAAS,GAAG,MAAM,GAAG,KAAK,CAAC;QACjC,IAAI,SAAS,IAAI,KAAK;YAAE,OAAO;QAE/B,MAAM,OAAO,GAAG,EAAE,GAAG,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;QAClD,OAAO,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;IAClD,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,GAAG,CAAI,QAAgC,EAAE,UAAkC,EAAE;QAC/E,MAAM,MAAM,GAAQ,EAAE,CAAC;QAEvB,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE;YACrC,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;YACxC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACrB,CAAC,EAAE,OAAO,CAAC,CAAC;QAEZ,OAAO,MAAM,CAAC;IAClB,CAAC;IAED;;;;;;;;;;;;;;OAcG;IACH,KAAK,CAAC,MAAM,CAAI,QAAiC,EAAE,IAAO,EAAE,UAAkC,EAAE;QAC5F,IAAI,WAAW,GAAM,IAAI,CAAC;QAE1B,MAAM,WAAW,GAA0B,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;YACvD,OAAO,OAAO;iBACT,OAAO,EAAE;iBACT,IAAI,CAAC,GAAG,EAAE;gBACP,OAAO,CAAC,KAAK,IAAI,WAAW,KAAK,SAAS;oBACtC,CAAC,CAAC,IAAI;oBACN,CAAC,CAAC,QAAQ,CAAC,WAAW,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC;YAC7C,CAAC,CAAC;iBACD,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE;gBACd,WAAW,GAAG,OAAY,CAAC;YAC/B,CAAC,CAAC,CAAC;QACX,CAAC,CAAC;QAEF,MAAM,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;QACzC,OAAO,WAAW,CAAC;IACvB,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,IAAI;QACN,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QAC3B,MAAM,OAAO,GAAG,gCAAc,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QAChE,OAAO,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;IAC/B,CAAC;IAED;;;;;;;;;;;;;OAaG;IACH,MAAM,CAAC,KAAK,CAAC,IAAI,CAAuC,eAA+B,EAAE,UAAiC,EAAE;QACxH,IAAA,YAAE,EAAC,eAAe,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACxC,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7B,MAAM,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,6BAAa,CAAC;SACvD,CAAC,CAAC,CAAC;QACJ,OAAO,CAAC,MAAM,KAAd,OAAO,CAAC,MAAM,GAAK,6BAAa,CAAC,eAAe,EAAE,EAAC;QAEnD,MAAM,OAAO,GAAG,gCAAc,CAAC,UAAU,CAAgB,OAAO,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;QAClF,OAAO,OAAO,CAAC,WAAW,CAAC,eAAe,EAAE,OAAO,CAAC,MAAM,CAAC,gBAAgB,EAAE,CAAC,CAAC;IACnF,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACH,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAuC,IAAmB;QAC3E,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAClC,OAAO,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;IAClC,CAAC;CACJ;AAtQD,0BAsQC"}
|
|
@@ -80,7 +80,10 @@ export declare class KeyValueStore {
|
|
|
80
80
|
readonly config: Configuration;
|
|
81
81
|
readonly id: string;
|
|
82
82
|
readonly name?: string;
|
|
83
|
-
private client;
|
|
83
|
+
private readonly client;
|
|
84
|
+
private persistStateEventStarted;
|
|
85
|
+
/** Cache for persistent (auto-saved) values. When we try to set such value, the cache will be updated automatically. */
|
|
86
|
+
private readonly cache;
|
|
84
87
|
/**
|
|
85
88
|
* @internal
|
|
86
89
|
*/
|
|
@@ -110,12 +113,16 @@ export declare class KeyValueStore {
|
|
|
110
113
|
* @param key
|
|
111
114
|
* Unique key of the record. It can be at most 256 characters long and only consist
|
|
112
115
|
* of the following characters: `a`-`z`, `A`-`Z`, `0`-`9` and `!-_.'()`
|
|
116
|
+
* @param defaultValue
|
|
117
|
+
* Fallback that will be returned if no value if present in the storage.
|
|
113
118
|
* @returns
|
|
114
119
|
* Returns a promise that resolves to an object, string
|
|
115
120
|
* or [`Buffer`](https://nodejs.org/api/buffer.html), depending
|
|
116
121
|
* on the MIME content type of the record.
|
|
117
122
|
*/
|
|
118
|
-
getValue<T = unknown>(key: string): Promise<T | null>;
|
|
123
|
+
getValue<T = unknown>(key: string, defaultValue?: T): Promise<T | null>;
|
|
124
|
+
getAutoSavedValue<T extends Dictionary = Dictionary>(key: string, defaultValue?: T): Promise<T>;
|
|
125
|
+
private ensurePersistStateEvent;
|
|
119
126
|
/**
|
|
120
127
|
* Saves or deletes a record in the key-value store.
|
|
121
128
|
* The function returns a promise that resolves once the record has been saved or deleted.
|
|
@@ -170,6 +177,8 @@ export declare class KeyValueStore {
|
|
|
170
177
|
* access the value in the remote key-value store.
|
|
171
178
|
*/
|
|
172
179
|
getPublicUrl(key: string): string;
|
|
180
|
+
/** @internal */
|
|
181
|
+
clearCache(): void;
|
|
173
182
|
/**
|
|
174
183
|
* Iterates over key-value store keys, yielding each in turn to an `iteratee` function.
|
|
175
184
|
* Each invocation of `iteratee` is called with three arguments: `(key, index, info)`, where `key`
|
|
@@ -229,6 +238,7 @@ export declare class KeyValueStore {
|
|
|
229
238
|
* and {@link KeyValueStore.getValue}.
|
|
230
239
|
*
|
|
231
240
|
* @param key Unique record key.
|
|
241
|
+
* @param defaultValue Fallback that will be returned if no value if present in the storage.
|
|
232
242
|
* @returns
|
|
233
243
|
* Returns a promise that resolves to an object, string
|
|
234
244
|
* or [`Buffer`](https://nodejs.org/api/buffer.html), depending
|
|
@@ -236,7 +246,8 @@ export declare class KeyValueStore {
|
|
|
236
246
|
* if the record is missing.
|
|
237
247
|
* @ignore
|
|
238
248
|
*/
|
|
239
|
-
static getValue<T = unknown>(key: string): Promise<T | null>;
|
|
249
|
+
static getValue<T = unknown>(key: string, defaultValue?: T): Promise<T | null>;
|
|
250
|
+
static getAutoSavedValue<T extends Dictionary = Dictionary>(key: string, defaultValue?: T): Promise<T>;
|
|
240
251
|
/**
|
|
241
252
|
* Stores or deletes a value in the default {@link KeyValueStore} associated with the current crawler run.
|
|
242
253
|
*
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"key_value_store.d.ts","sourceRoot":"","sources":["../../src/storages/key_value_store.ts"],"names":[],"mappings":";AAGA,OAAO,EAAE,UAAU,EAAuB,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAChF,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AACjD,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AACxC,OAAO,EAAkB,qBAAqB,EAAE,MAAM,mBAAmB,CAAC;AAE1E;;;;GAIG;AACH,eAAO,MAAM,cAAc,yBAA0B;IAAE,WAAW,CAAC,EAAE,MAAM,CAAA;CAAE,MAwB5E,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgEG;AACH,qBAAa,aAAa;
|
|
1
|
+
{"version":3,"file":"key_value_store.d.ts","sourceRoot":"","sources":["../../src/storages/key_value_store.ts"],"names":[],"mappings":";AAGA,OAAO,EAAE,UAAU,EAAuB,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAChF,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AACjD,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AACxC,OAAO,EAAkB,qBAAqB,EAAE,MAAM,mBAAmB,CAAC;AAE1E;;;;GAIG;AACH,eAAO,MAAM,cAAc,yBAA0B;IAAE,WAAW,CAAC,EAAE,MAAM,CAAA;CAAE,MAwB5E,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgEG;AACH,qBAAa,aAAa;IAYqB,QAAQ,CAAC,MAAM;IAX1D,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IACvB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAsB;IAC7C,OAAO,CAAC,wBAAwB,CAAS;IAEzC,wHAAwH;IACxH,OAAO,CAAC,QAAQ,CAAC,KAAK,CAA8B;IAEpD;;OAEG;gBACS,OAAO,EAAE,oBAAoB,EAAW,MAAM,gBAAkC;IAM5F;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA+BG;IACG,QAAQ,CAAC,CAAC,GAAG,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,YAAY,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC;IAOvE,iBAAiB,CAAC,CAAC,SAAS,UAAU,GAAG,UAAU,EAAE,GAAG,EAAE,MAAM,EAAE,YAAY,IAAU,GAAG,OAAO,CAAC,CAAC,CAAC;IAY3G,OAAO,CAAC,uBAAuB;IAc/B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA0CG;IACG,QAAQ,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,GAAG,IAAI,EAAE,OAAO,GAAE,aAAkB,GAAG,OAAO,CAAC,IAAI,CAAC;IA8C3F;;;OAGG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAM3B;;;OAGG;IACH,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM;IAKjC,gBAAgB;IAChB,UAAU,IAAI,IAAI;IAIlB;;;;;;;;;;;;;;;;;;;;OAoBG;IACG,UAAU,CAAC,QAAQ,EAAE,WAAW,EAAE,OAAO,GAAE,4BAAiC,GAAG,OAAO,CAAC,IAAI,CAAC;YAIpF,WAAW;IAiBzB;;;;;;;;;;;;;OAaG;WACU,IAAI,CAAC,aAAa,CAAC,EAAE,MAAM,GAAG,IAAI,EAAE,OAAO,GAAE,qBAA0B,GAAG,OAAO,CAAC,aAAa,CAAC;IAU7G;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA4BG;WACU,QAAQ,CAAC,CAAC,GAAG,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,YAAY,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC;WAKvE,iBAAiB,CAAC,CAAC,SAAS,UAAU,GAAG,UAAU,EAAE,GAAG,EAAE,MAAM,EAAE,YAAY,IAAU,GAAG,OAAO,CAAC,CAAC,CAAC;IAKlH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA8BG;WACU,QAAQ,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,GAAG,IAAI,EAAE,OAAO,GAAE,aAAkB,GAAG,OAAO,CAAC,IAAI,CAAC;IAKlG;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA4BG;WACU,QAAQ,CAAC,CAAC,GAAG,UAAU,GAAG,MAAM,GAAG,MAAM,KAAK,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC;CAI/E;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IACxB;;;;;OAKG;IACH,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;CACzE;AAED,MAAM,WAAW,oBAAoB;IACjC,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,aAAa,CAAC;CACzB;AAED,MAAM,WAAW,aAAa;IAC1B;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,4BAA4B;IACzC;;OAEG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC9B"}
|
|
@@ -130,6 +130,19 @@ class KeyValueStore {
|
|
|
130
130
|
writable: true,
|
|
131
131
|
value: void 0
|
|
132
132
|
});
|
|
133
|
+
Object.defineProperty(this, "persistStateEventStarted", {
|
|
134
|
+
enumerable: true,
|
|
135
|
+
configurable: true,
|
|
136
|
+
writable: true,
|
|
137
|
+
value: false
|
|
138
|
+
});
|
|
139
|
+
/** Cache for persistent (auto-saved) values. When we try to set such value, the cache will be updated automatically. */
|
|
140
|
+
Object.defineProperty(this, "cache", {
|
|
141
|
+
enumerable: true,
|
|
142
|
+
configurable: true,
|
|
143
|
+
writable: true,
|
|
144
|
+
value: new Map()
|
|
145
|
+
});
|
|
133
146
|
this.id = options.id;
|
|
134
147
|
this.name = options.name;
|
|
135
148
|
this.client = options.client.keyValueStore(this.id);
|
|
@@ -159,15 +172,37 @@ class KeyValueStore {
|
|
|
159
172
|
* @param key
|
|
160
173
|
* Unique key of the record. It can be at most 256 characters long and only consist
|
|
161
174
|
* of the following characters: `a`-`z`, `A`-`Z`, `0`-`9` and `!-_.'()`
|
|
175
|
+
* @param defaultValue
|
|
176
|
+
* Fallback that will be returned if no value if present in the storage.
|
|
162
177
|
* @returns
|
|
163
178
|
* Returns a promise that resolves to an object, string
|
|
164
179
|
* or [`Buffer`](https://nodejs.org/api/buffer.html), depending
|
|
165
180
|
* on the MIME content type of the record.
|
|
166
181
|
*/
|
|
167
|
-
async getValue(key) {
|
|
182
|
+
async getValue(key, defaultValue) {
|
|
168
183
|
(0, ow_1.default)(key, ow_1.default.string.nonEmpty);
|
|
169
184
|
const record = await this.client.getRecord(key);
|
|
170
|
-
return record?.value ?? null;
|
|
185
|
+
return record?.value ?? defaultValue ?? null;
|
|
186
|
+
}
|
|
187
|
+
async getAutoSavedValue(key, defaultValue = {}) {
|
|
188
|
+
if (this.cache.has(key)) {
|
|
189
|
+
return this.cache.get(key);
|
|
190
|
+
}
|
|
191
|
+
const value = await this.getValue(key, defaultValue);
|
|
192
|
+
this.cache.set(key, value);
|
|
193
|
+
this.ensurePersistStateEvent();
|
|
194
|
+
return value;
|
|
195
|
+
}
|
|
196
|
+
ensurePersistStateEvent() {
|
|
197
|
+
if (this.persistStateEventStarted) {
|
|
198
|
+
return;
|
|
199
|
+
}
|
|
200
|
+
this.config.getEventManager().on('persistState', async () => {
|
|
201
|
+
for (const [key, value] of this.cache) {
|
|
202
|
+
await this.setValue(key, value);
|
|
203
|
+
}
|
|
204
|
+
});
|
|
205
|
+
this.persistStateEventStarted = true;
|
|
171
206
|
}
|
|
172
207
|
/**
|
|
173
208
|
* Saves or deletes a record in the key-value store.
|
|
@@ -227,6 +262,22 @@ class KeyValueStore {
|
|
|
227
262
|
}));
|
|
228
263
|
// Make copy of options, don't update what user passed.
|
|
229
264
|
const optionsCopy = { ...options };
|
|
265
|
+
// If we try to set the value of a cached state to a different reference, we need to update the cache accordingly.
|
|
266
|
+
const cachedValue = this.cache.get(key);
|
|
267
|
+
if (this.cache.has(key) && cachedValue !== value) {
|
|
268
|
+
if (value === null) {
|
|
269
|
+
// Cached state can be only object, so a propagation of `null` means removing all its properties.
|
|
270
|
+
Object.keys(cachedValue).forEach((k) => this.cache.delete(k));
|
|
271
|
+
}
|
|
272
|
+
else if (typeof value === 'object') {
|
|
273
|
+
// We need to remove the keys that are no longer present in the new value.
|
|
274
|
+
Object.keys(cachedValue)
|
|
275
|
+
.filter((k) => !(k in value))
|
|
276
|
+
.forEach((k) => this.cache.delete(k));
|
|
277
|
+
// And update the existing ones + add new ones.
|
|
278
|
+
Object.assign(cachedValue, value);
|
|
279
|
+
}
|
|
280
|
+
}
|
|
230
281
|
// In this case delete the record.
|
|
231
282
|
if (value === null)
|
|
232
283
|
return this.client.deleteRecord(key);
|
|
@@ -243,7 +294,7 @@ class KeyValueStore {
|
|
|
243
294
|
*/
|
|
244
295
|
async drop() {
|
|
245
296
|
await this.client.delete();
|
|
246
|
-
const manager =
|
|
297
|
+
const manager = storage_manager_1.StorageManager.getManager(KeyValueStore, this.config);
|
|
247
298
|
manager.closeStorage(this);
|
|
248
299
|
}
|
|
249
300
|
/**
|
|
@@ -254,6 +305,10 @@ class KeyValueStore {
|
|
|
254
305
|
// FIXME how should this work? should we remove this method or provide a way to configure the base url?
|
|
255
306
|
return `https://api.apify.com/v2/key-value-stores/${this.id}/records/${key}`;
|
|
256
307
|
}
|
|
308
|
+
/** @internal */
|
|
309
|
+
clearCache() {
|
|
310
|
+
this.cache.clear();
|
|
311
|
+
}
|
|
257
312
|
/**
|
|
258
313
|
* Iterates over key-value store keys, yielding each in turn to an `iteratee` function.
|
|
259
314
|
* Each invocation of `iteratee` is called with three arguments: `(key, index, info)`, where `key`
|
|
@@ -308,11 +363,11 @@ class KeyValueStore {
|
|
|
308
363
|
* @param [options] Storage manager options.
|
|
309
364
|
*/
|
|
310
365
|
static async open(storeIdOrName, options = {}) {
|
|
311
|
-
(0, ow_1.default)(storeIdOrName, ow_1.default.optional.string);
|
|
366
|
+
(0, ow_1.default)(storeIdOrName, ow_1.default.optional.any(ow_1.default.string, ow_1.default.null));
|
|
312
367
|
(0, ow_1.default)(options, ow_1.default.object.exactShape({
|
|
313
368
|
config: ow_1.default.optional.object.instanceOf(configuration_1.Configuration),
|
|
314
369
|
}));
|
|
315
|
-
const manager =
|
|
370
|
+
const manager = storage_manager_1.StorageManager.getManager(KeyValueStore, options.config);
|
|
316
371
|
return manager.openStorage(storeIdOrName);
|
|
317
372
|
}
|
|
318
373
|
/**
|
|
@@ -336,6 +391,7 @@ class KeyValueStore {
|
|
|
336
391
|
* and {@link KeyValueStore.getValue}.
|
|
337
392
|
*
|
|
338
393
|
* @param key Unique record key.
|
|
394
|
+
* @param defaultValue Fallback that will be returned if no value if present in the storage.
|
|
339
395
|
* @returns
|
|
340
396
|
* Returns a promise that resolves to an object, string
|
|
341
397
|
* or [`Buffer`](https://nodejs.org/api/buffer.html), depending
|
|
@@ -343,9 +399,13 @@ class KeyValueStore {
|
|
|
343
399
|
* if the record is missing.
|
|
344
400
|
* @ignore
|
|
345
401
|
*/
|
|
346
|
-
static async getValue(key) {
|
|
402
|
+
static async getValue(key, defaultValue) {
|
|
403
|
+
const store = await this.open();
|
|
404
|
+
return store.getValue(key, defaultValue);
|
|
405
|
+
}
|
|
406
|
+
static async getAutoSavedValue(key, defaultValue = {}) {
|
|
347
407
|
const store = await this.open();
|
|
348
|
-
return store.
|
|
408
|
+
return store.getAutoSavedValue(key, defaultValue);
|
|
349
409
|
}
|
|
350
410
|
/**
|
|
351
411
|
* Stores or deletes a value in the default {@link KeyValueStore} associated with the current crawler run.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"key_value_store.js","sourceRoot":"","sources":["../../src/storages/key_value_store.ts"],"names":[],"mappings":";;;;AAAA,0CAA0D;AAC1D,gDAAyD;AACzD,iDAAuC;AAEvC,oDAAiD;AAEjD,uDAA0E;AAE1E;;;;GAIG;AACI,MAAM,cAAc,GAAG,CAAI,KAAQ,EAAE,OAAiC,EAAE,EAAE;IAC7E,+DAA+D;IAC/D,IAAI,OAAO,CAAC,WAAW,KAAK,IAAI,IAAI,OAAO,CAAC,WAAW,KAAK,SAAS,EAAE;QACnE,OAAO,CAAC,WAAW,GAAG,iCAAiC,CAAC;QAExD,IAAI;YACA,kFAAkF;YAClF,KAAK,GAAG,IAAA,iCAAqB,EAAC,KAAmB,EAAE,IAAI,EAAE,CAAC,CAAiB,CAAC;SAC/E;QAAC,OAAO,CAAC,EAAE;YACR,MAAM,KAAK,GAAG,CAAU,CAAC;YACzB,qCAAqC;YACrC,IAAI,KAAK,CAAC,OAAO,EAAE,OAAO,CAAC,uBAAuB,CAAC,IAAI,CAAC,EAAE;gBACtD,KAAK,CAAC,OAAO,GAAG,qBAAqB,CAAC;aACzC;YACD,MAAM,IAAI,KAAK,CAAC,wDAAwD,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;SAC5F;QAED,IAAI,KAAK,KAAK,SAAS,EAAE;YACrB,MAAM,IAAI,KAAK,CAAC,wEAAwE;kBAClF,+DAA+D,CAAC,CAAC;SAC1E;KACJ;IAED,OAAO,KAAK,CAAC;AACjB,CAAC,CAAC;AAxBW,QAAA,cAAc,kBAwBzB;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgEG;AACH,MAAa,aAAa;
|
|
1
|
+
{"version":3,"file":"key_value_store.js","sourceRoot":"","sources":["../../src/storages/key_value_store.ts"],"names":[],"mappings":";;;;AAAA,0CAA0D;AAC1D,gDAAyD;AACzD,iDAAuC;AAEvC,oDAAiD;AAEjD,uDAA0E;AAE1E;;;;GAIG;AACI,MAAM,cAAc,GAAG,CAAI,KAAQ,EAAE,OAAiC,EAAE,EAAE;IAC7E,+DAA+D;IAC/D,IAAI,OAAO,CAAC,WAAW,KAAK,IAAI,IAAI,OAAO,CAAC,WAAW,KAAK,SAAS,EAAE;QACnE,OAAO,CAAC,WAAW,GAAG,iCAAiC,CAAC;QAExD,IAAI;YACA,kFAAkF;YAClF,KAAK,GAAG,IAAA,iCAAqB,EAAC,KAAmB,EAAE,IAAI,EAAE,CAAC,CAAiB,CAAC;SAC/E;QAAC,OAAO,CAAC,EAAE;YACR,MAAM,KAAK,GAAG,CAAU,CAAC;YACzB,qCAAqC;YACrC,IAAI,KAAK,CAAC,OAAO,EAAE,OAAO,CAAC,uBAAuB,CAAC,IAAI,CAAC,EAAE;gBACtD,KAAK,CAAC,OAAO,GAAG,qBAAqB,CAAC;aACzC;YACD,MAAM,IAAI,KAAK,CAAC,wDAAwD,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;SAC5F;QAED,IAAI,KAAK,KAAK,SAAS,EAAE;YACrB,MAAM,IAAI,KAAK,CAAC,wEAAwE;kBAClF,+DAA+D,CAAC,CAAC;SAC1E;KACJ;IAED,OAAO,KAAK,CAAC;AACjB,CAAC,CAAC;AAxBW,QAAA,cAAc,kBAwBzB;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgEG;AACH,MAAa,aAAa;IAStB;;OAEG;IACH,YAAY,OAA6B,EAAW,SAAS,6BAAa,CAAC,eAAe,EAAE;;;;;mBAAxC;;QAXpD;;;;;WAAoB;QACpB;;;;;WAAuB;QACvB;;;;;WAA6C;QAC7C;;;;mBAAmC,KAAK;WAAC;QAEzC,wHAAwH;QACxH;;;;mBAAyB,IAAI,GAAG,EAAmB;WAAC;QAMhD,IAAI,CAAC,EAAE,GAAG,OAAO,CAAC,EAAE,CAAC;QACrB,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;QACzB,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACxD,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA+BG;IACH,KAAK,CAAC,QAAQ,CAAc,GAAW,EAAE,YAAgB;QACrD,IAAA,YAAE,EAAC,GAAG,EAAE,YAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAC5B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;QAEhD,OAAO,MAAM,EAAE,KAAU,IAAI,YAAY,IAAI,IAAI,CAAC;IACtD,CAAC;IAED,KAAK,CAAC,iBAAiB,CAAoC,GAAW,EAAE,eAAe,EAAO;QAC1F,IAAI,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE;YACrB,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAM,CAAC;SACnC;QAED,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAI,GAAG,EAAE,YAAY,CAAC,CAAC;QACxD,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QAC3B,IAAI,CAAC,uBAAuB,EAAE,CAAC;QAE/B,OAAO,KAAM,CAAC;IAClB,CAAC;IAEO,uBAAuB;QAC3B,IAAI,IAAI,CAAC,wBAAwB,EAAE;YAC/B,OAAO;SACV;QAED,IAAI,CAAC,MAAM,CAAC,eAAe,EAAE,CAAC,EAAE,CAAC,cAAc,EAAE,KAAK,IAAI,EAAE;YACxD,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,KAAK,EAAE;gBACnC,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;aACnC;QACL,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,wBAAwB,GAAG,IAAI,CAAC;IACzC,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA0CG;IACH,KAAK,CAAC,QAAQ,CAAI,GAAW,EAAE,KAAe,EAAE,UAAyB,EAAE;QACvE,IAAA,YAAE,EAAC,GAAG,EAAE,KAAK,EAAE,YAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QACnC,IAAA,YAAE,EAAC,GAAG,EAAE,YAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAC/B,SAAS,EAAE,YAAE,CAAC,OAAO,CAAC,CAAC,EAAE,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,kCAAyB,CAAC,CAAC;YACtE,OAAO,EAAE,qHAAqH;SACjI,CAAC,CAAC,CAAC,CAAC;QACL,IAAI,OAAO,CAAC,WAAW;eACjB,CAAC,CAAC,YAAE,CAAC,OAAO,CAAC,KAAK,EAAE,YAAE,CAAC,GAAG,CAAC,YAAE,CAAC,MAAM,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,YAAE,CAAC,OAAO,CAAC,KAAK,EAAE,YAAE,CAAC,MAAM,CAAC,IAAI,OAAQ,KAAoB,CAAC,IAAI,KAAK,UAAU,CAAC,CAAC,EAAE;YAC5I,MAAM,IAAI,kBAAa,CAAC,mGAAmG,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;SAC/I;QACD,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7B,WAAW,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ;SAC3C,CAAC,CAAC,CAAC;QAEJ,uDAAuD;QACvD,MAAM,WAAW,GAAG,EAAE,GAAG,OAAO,EAAE,CAAC;QAEnC,kHAAkH;QAClH,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAM,CAAC;QAE7C,IAAI,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,WAAW,KAAK,KAAK,EAAE;YAC9C,IAAI,KAAK,KAAK,IAAI,EAAE;gBAChB,iGAAiG;gBACjG,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;aACjE;iBAAM,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE;gBAClC,0EAA0E;gBAC1E,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC;qBACnB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,KAAM,CAAC,CAAC;qBAC7B,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC1C,+CAA+C;gBAC/C,MAAM,CAAC,MAAM,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC;aACrC;SACJ;QAED,kCAAkC;QAClC,IAAI,KAAK,KAAK,IAAI;YAAE,OAAO,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;QAEzD,KAAK,GAAG,IAAA,sBAAc,EAAC,KAAK,EAAE,WAAW,CAAC,CAAC;QAE3C,OAAO,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC;YACzB,GAAG;YACH,KAAK;YACL,WAAW,EAAE,WAAW,CAAC,WAAW;SACvC,CAAC,CAAC;IACP,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,IAAI;QACN,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QAC3B,MAAM,OAAO,GAAG,gCAAc,CAAC,UAAU,CAAC,aAAa,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QACtE,OAAO,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;IAC/B,CAAC;IAED;;;OAGG;IACH,YAAY,CAAC,GAAW;QACpB,uGAAuG;QACvG,OAAO,6CAA6C,IAAI,CAAC,EAAE,YAAY,GAAG,EAAE,CAAC;IACjF,CAAC;IAED,gBAAgB;IAChB,UAAU;QACN,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;IACvB,CAAC;IAED;;;;;;;;;;;;;;;;;;;;OAoBG;IACH,KAAK,CAAC,UAAU,CAAC,QAAqB,EAAE,UAAwC,EAAE;QAC9E,OAAO,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC/C,CAAC;IAEO,KAAK,CAAC,WAAW,CAAC,QAAqB,EAAE,UAAwC,EAAE,EAAE,KAAK,GAAG,CAAC;QAClG,MAAM,EAAE,iBAAiB,EAAE,GAAG,OAAO,CAAC;QACtC,IAAA,YAAE,EAAC,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,CAAC;QAC1B,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7B,iBAAiB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;SACxC,CAAC,CAAC,CAAC;QAEJ,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,iBAAiB,EAAE,CAAC,CAAC;QACnE,MAAM,EAAE,qBAAqB,EAAE,WAAW,EAAE,KAAK,EAAE,GAAG,QAAQ,CAAC;QAC/D,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE;YACtB,MAAM,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;SAC1D;QACD,OAAO,WAAW;YACd,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,EAAE,iBAAiB,EAAE,qBAAqB,EAAE,EAAE,KAAK,CAAC;YACjF,CAAC,CAAC,SAAS,CAAC,CAAC,kCAAkC;IACvD,CAAC;IAED;;;;;;;;;;;;;OAaG;IACH,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,aAA6B,EAAE,UAAiC,EAAE;QAChF,IAAA,YAAE,EAAC,aAAa,EAAE,YAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,YAAE,CAAC,MAAM,EAAE,YAAE,CAAC,IAAI,CAAC,CAAC,CAAC;QACvD,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7B,MAAM,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,6BAAa,CAAC;SACvD,CAAC,CAAC,CAAC;QAEJ,MAAM,OAAO,GAAG,gCAAc,CAAC,UAAU,CAAC,aAAa,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;QACzE,OAAO,OAAO,CAAC,WAAW,CAAC,aAAa,CAAC,CAAC;IAC9C,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA4BG;IACH,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAc,GAAW,EAAE,YAAgB;QAC5D,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAChC,OAAO,KAAK,CAAC,QAAQ,CAAI,GAAG,EAAE,YAAY,CAAC,CAAC;IAChD,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,iBAAiB,CAAoC,GAAW,EAAE,eAAe,EAAO;QACjG,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAChC,OAAO,KAAK,CAAC,iBAAiB,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;IACtD,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA8BG;IACH,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAI,GAAW,EAAE,KAAe,EAAE,UAAyB,EAAE;QAC9E,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAChC,OAAO,KAAK,CAAC,QAAQ,CAAC,GAAG,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;IAC/C,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA4BG;IACH,MAAM,CAAC,KAAK,CAAC,QAAQ;QACjB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAChC,OAAO,KAAK,CAAC,QAAQ,CAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC;IAC3D,CAAC;CACJ;AAlXD,sCAkXC"}
|
package/storages/request_list.js
CHANGED
|
@@ -701,7 +701,7 @@ class RequestList {
|
|
|
701
701
|
* options and the `sources` parameter supersedes the {@link RequestListOptions.sources} option.
|
|
702
702
|
*/
|
|
703
703
|
static async open(listName, sources, options = {}) {
|
|
704
|
-
(0, ow_1.default)(listName, ow_1.default.any(ow_1.default.string, ow_1.default.null));
|
|
704
|
+
(0, ow_1.default)(listName, ow_1.default.optional.any(ow_1.default.string, ow_1.default.null));
|
|
705
705
|
(0, ow_1.default)(sources, ow_1.default.array);
|
|
706
706
|
(0, ow_1.default)(options, ow_1.default.object.is((v) => !Array.isArray(v)));
|
|
707
707
|
const rl = new RequestList({
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"request_list.js","sourceRoot":"","sources":["../../src/storages/request_list.ts"],"names":[],"mappings":";;;;AAAA,0CAAoD;AAEpD,iDAAuC;AAEvC,oDAAiD;AACjD,gCAA6B;AAC7B,wCAAqD;AACrD,oDAAqE;AACrE,iEAA4D;AAG5D,gBAAgB;AACH,QAAA,qBAAqB,GAAG,oBAAoB,CAAC;AAE1D,gBAAgB;AACH,QAAA,wBAAwB,GAAG,uBAAuB,CAAC;AAEhE,MAAM,mBAAmB,GAAG,0BAA0B,CAAC;AAwJvD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmEG;AACH,MAAa,WAAW;IAoDpB;;OAEG;IACH,YAAY,UAA8B,EAAE;QAtD5C;;;;mBAAc,SAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,aAAa,EAAE,CAAC;WAAC;QAEnD;;;;WAIG;QACH;;;;mBAAsB,EAAE;WAAC;QAEzB,kHAAkH;QAClH;;;;mBAAoB,CAAC;WAAC;QAEtB,gGAAgG;QAChG;;;;mBAAmD,EAAE;WAAC;QAEtD;;;WAGG;QACH;;;;mBAAa,IAAI,GAAG,EAAU;WAAC;QAE/B;;;WAGG;QACH;;;;mBAAY,IAAI,GAAG,EAAU;WAAC;QAE9B;;;WAGG;QACH;;;;mBAAmB,IAAI;WAAC;QAExB;;;WAGG;QACH;;;;mBAAuB,KAAK;WAAC;QAE7B;;;;mBAAoB,KAAK;WAAC;QAC1B;;;;mBAAwB,KAAK;WAAC;QAC9B;;;;;WAAiC;QACjC;;;;;WAAoC;QACpC;;;;;WAAwC;QACxC;;;;;WAA8B;QAC9B;;;;;WAAmC;QACnC;;;;;WAA0B;QAC1B;;;;;WAAqD;QACrD;;;;;WAAgD;QAChD;;;;;WAA6B;QAMzB,MAAM,EACF,OAAO,EACP,eAAe,EACf,eAAe,EACf,kBAAkB,EAClB,KAAK,EACL,kBAAkB,EAClB,iBAAiB,GAAG,KAAK,EACzB,MAAM,GAAG,6BAAa,CAAC,eAAe,EAAE,GAC3C,GAAG,OAAO,CAAC;QAEZ,IAAI,CAAC,CAAC,OAAO,IAAI,eAAe,CAAC,EAAE;YAC/B,MAAM,IAAI,kBAAa,CAAC,kEAAkE,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;SACjH;QACD,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7B,OAAO,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK;YAC1B,eAAe,EAAE,YAAE,CAAC,QAAQ,CAAC,QAAQ;YACrC,eAAe,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACnC,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACtC,KAAK,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC;gBACjC,SAAS,EAAE,YAAE,CAAC,MAAM;gBACpB,aAAa,EAAE,YAAE,CAAC,MAAM;gBACxB,UAAU,EAAE,YAAE,CAAC,MAAM;aACxB,CAAC;YACF,iBAAiB,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;YACtC,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;SACzC,CAAC,CAAC,CAAC;QAEJ,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC,CAAC,CAAC,OAAO,eAAe,EAAE,CAAC,CAAC,CAAC,eAAe,CAAC;QACpF,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC,CAAC,CAAC,OAAO,kBAAkB,EAAE,CAAC,CAAC,CAAC,kBAAkB,CAAC;QAChG,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC;QAC1B,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,eAAe,EAAE,CAAC;QAEvC,0HAA0H;QAC1H,IAAI,CAAC,iBAAiB,GAAG,iBAAiB,CAAC;QAE3C,qDAAqD;QACrD,IAAI,CAAC,OAAO,GAAG,OAAO,IAAI,EAAE,CAAC;QAC7B,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;QAEvC,gEAAgE;QAChE,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;IACjD,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,UAAU;QACZ,IAAI,IAAI,CAAC,SAAS,EAAE;YAChB,MAAM,IAAI,KAAK,CAAC,yDAAyD,CAAC,CAAC;SAC9E;QACD,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QAEtB,MAAM,CAAC,KAAK,EAAE,iBAAiB,CAAC,GAAG,MAAM,IAAI,CAAC,8BAA8B,EAAE,CAAC;QAE/E,mFAAmF;QACnF,8CAA8C;QAC9C,IAAI,iBAAiB,EAAE;YACnB,MAAM,IAAI,CAAC,qBAAqB,CAAC,iBAA2B,CAAC,CAAC;SACjE;aAAM;YACH,MAAM,IAAI,CAAC,uBAAuB,EAAE,CAAC;SACxC;QAED,IAAI,CAAC,aAAa,CAAC,KAAyB,CAAC,CAAC;QAC9C,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC;QAC1B,IAAI,IAAI,CAAC,kBAAkB,IAAI,CAAC,IAAI,CAAC,oBAAoB;YAAE,MAAM,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACzF,IAAI,IAAI,CAAC,eAAe,EAAE;YACtB,IAAI,CAAC,MAAM,CAAC,EAAE,+CAA0B,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;SACzE;IACL,CAAC;IAED;;;;OAIG;IACO,KAAK,CAAC,qBAAqB,CAAC,iBAAyB;QAC3D,gDAAgD;QAChD,uCAAuC;QACvC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;YAC1C,OAAO,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;SAC1B;QACD,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC;QAElB,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAC;QACjC,MAAM,aAAa,GAAG,IAAA,iCAAiB,EAAC,iBAAiB,CAAC,CAAC;QAC3D,IAAI,KAAK,EAAE,MAAM,OAAO,IAAI,aAAa,EAAE;YACvC,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;SAC7B;IACL,CAAC;IAED;;;;;OAKG;IACO,KAAK,CAAC,uBAAuB;QACnC,wFAAwF;QACxF,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;QACzC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,YAAY,EAAE,CAAC,EAAE,EAAE;YACnC,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;YAC/B,oEAAoE;YACpE,mDAAmD;YACnD,OAAO,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;YAEvB,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAK,MAAqB,CAAC,eAAe,EAAE;gBACtE,MAAM,eAAe,GAAG,MAAM,IAAI,CAAC,qBAAqB,CAAC,MAAwB,CAAC,CAAC;gBACnF,MAAM,IAAI,CAAC,mBAAmB,CAAC,MAAwB,EAAE,eAAe,CAAC,CAAC;aAC7E;iBAAM;gBACH,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;aAC5B;SACJ;QAED,iDAAiD;QACjD,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC;QAElB,IAAI,IAAI,CAAC,eAAe,EAAE;YACtB,IAAI;gBACA,MAAM,mBAAmB,GAAG,MAAM,IAAI,CAAC,eAAe,EAAE,CAAC;gBACzD,MAAM,wBAAwB,GAAG,mBAAmB,CAAC,MAAM,CAAC;gBAC5D,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,wBAAwB,EAAE,CAAC,EAAE,EAAE;oBAC/C,MAAM,MAAM,GAAG,mBAAmB,CAAC,KAAK,EAAE,CAAC;oBAC3C,IAAI,CAAC,WAAW,CAAC,MAAO,CAAC,CAAC;iBAC7B;aACJ;YAAC,OAAO,CAAC,EAAE;gBACR,MAAM,GAAG,GAAG,CAAU,CAAC;gBACvB,MAAM,IAAI,KAAK,CAAC,yDAAyD,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;aAC3F;SACJ;IACL,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,YAAY;QACd,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE;YACvB,MAAM,IAAI,KAAK,CAAC,2DAA2D,CAAC,CAAC;SAChF;QACD,IAAI,IAAI,CAAC,gBAAgB;YAAE,OAAO;QAClC,IAAI;YACA,IAAI,CAAC,KAAK,KAAV,IAAI,CAAC,KAAK,GAAK,MAAM,+BAAa,CAAC,IAAI,EAAE,EAAC;YAC1C,MAAM,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,eAAe,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;YACjE,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC;SAChC;QAAC,OAAO,CAAC,EAAE;YACR,MAAM,GAAG,GAAG,CAAU,CAAC;YACvB,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,yCAAyC,CAAC,CAAC;SACtE;IACL,CAAC;IAED;;;;OAIG;IACO,KAAK,CAAC,gBAAgB;QAC5B,MAAM,kBAAkB,GAAG,MAAM,IAAA,8BAAc,EAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC/D,IAAI,CAAC,KAAK,KAAV,IAAI,CAAC,KAAK,GAAK,MAAM,+BAAa,CAAC,IAAI,EAAE,EAAC;QAC1C,MAAM,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,kBAAmB,EAAE,kBAAkB,EAAE,EAAE,WAAW,EAAE,mBAAmB,EAAE,CAAC,CAAC;QAC9G,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAC;IACrC,CAAC;IAED;;OAEG;IACO,aAAa,CAAC,KAAwB;QAC5C,8DAA8D;QAC9D,IAAI,CAAC,KAAK;YAAE,OAAO;QACnB,0BAA0B;QAC1B,IAAI,OAAO,KAAK,CAAC,SAAS,KAAK,QAAQ,IAAI,KAAK,CAAC,SAAS,GAAG,CAAC,EAAE;YAC5D,MAAM,IAAI,KAAK,CAAC,uEAAuE,CAAC,CAAC;SAC5F;QACD,IAAI,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE;YACxC,MAAM,IAAI,KAAK,CAAC,8EAA8E,CAAC,CAAC;SACnG;QACD,IAAI,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM;eACnC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,SAAS,KAAK,KAAK,CAAC,aAAa,EAAE;YACrE,MAAM,IAAI,KAAK,CAAC,8FAA8F,CAAC,CAAC;SACnH;QAED,MAAM,oBAAoB,GAAa,EAAE,CAAC;QAC1C,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,SAAS,EAAE,EAAE;YACnC,MAAM,KAAK,GAAG,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC;YAC/C,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE;gBAC3B,MAAM,IAAI,KAAK,CAAC,iGAAiG,CAAC,CAAC;aACtH;YACD,IAAI,KAAK,IAAI,KAAK,CAAC,SAAS,EAAE;gBAC1B,oBAAoB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;aACxC;QACL,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC,SAAS,CAAC;QACjC,IAAI,CAAC,UAAU,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;QAE5C,cAAc;QACd,wEAAwE;QACxE,IAAI;QACJ,wBAAwB;QACxB,iDAAiD;QACjD,oBAAoB;QACpB,uCAAuC;QACvC,WAAW;QACX,2CAA2C;QAC3C,IAAI;QACJ,qGAAqG;QACrG,oFAAoF;QACpF,oCAAoC;QACpC,IAAI,oBAAoB,CAAC,MAAM,EAAE;YAC7B,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,0FAA0F,EAAE;gBACzG,oBAAoB;aACvB,CAAC,CAAC;YACH,KAAK,MAAM,SAAS,IAAI,oBAAoB,EAAE;gBAC1C,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;aACrC;SACJ;QAED,iDAAiD;QACjD,IAAI,CAAC,SAAS,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IAC9C,CAAC;IAED;;;OAGG;IACO,KAAK,CAAC,8BAA8B;QAC1C,IAAI,KAAwB,CAAC;QAC7B,IAAI,iBAA0B,CAAC;QAE/B,IAAI,IAAI,CAAC,YAAY,EAAE;YACnB,KAAK,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC;YAChC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,2CAA2C,CAAC,CAAC;SAC/D;aAAM,IAAI,IAAI,CAAC,eAAe,EAAE;YAC7B,KAAK,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;YAC5D,IAAI,KAAK;gBAAE,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,8DAA8D,CAAC,CAAC;SAC7F;QAED,IAAI,IAAI,CAAC,kBAAkB,EAAE;YACzB,iBAAiB,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;YAC3E,IAAI,iBAAiB;gBAAE,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,oEAAoE,CAAC,CAAC;SAC/G;QAED,OAAO,CAAC,KAAK,EAAE,iBAAiB,CAAC,CAAC;IACtC,CAAC;IAED;;;OAGG;IACH,QAAQ;QACJ,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAE5B,OAAO;YACH,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,aAAa,EAAE,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM;gBAChD,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,SAAS;gBACzC,CAAC,CAAC,IAAI;YACV,UAAU,EAAE,CAAC,GAAG,IAAI,CAAC,UAAU,CAAC;SACnC,CAAC;IACN,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,OAAO;QACT,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAE5B,OAAO,IAAI,CAAC,SAAS,CAAC,IAAI,KAAK,CAAC,IAAI,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;IAC/E,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU;QACZ,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAE5B,OAAO,IAAI,CAAC,UAAU,CAAC,IAAI,KAAK,CAAC,IAAI,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;IAChF,CAAC;IAED;;;;;;;OAOG;IACH,KAAK,CAAC,gBAAgB;QAClB,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAE5B,0CAA0C;QAC1C,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC;QACvD,IAAI,SAAS,EAAE;YACX,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;YACjC,MAAM,KAAK,GAAG,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC;YAC/C,OAAO,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;SAC/B;QAED,iCAAiC;QACjC,IAAI,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE;YACvC,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAC9C,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;YACvC,IAAI,CAAC,SAAS,EAAE,CAAC;YACjB,IAAI,CAAC,gBAAgB,GAAG,KAAK,CAAC;YAC9B,OAAO,OAAO,CAAC;SAClB;QAED,OAAO,IAAI,CAAC;IAChB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,kBAAkB,CAAC,OAAgB;QACrC,MAAM,EAAE,SAAS,EAAE,GAAG,OAAO,CAAC;QAE9B,IAAI,CAAC,qBAAqB,CAAC,SAAS,CAAC,CAAC;QACtC,IAAI,CAAC,gCAAgC,CAAC,SAAS,CAAC,CAAC;QACjD,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAE5B,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QAClC,IAAI,CAAC,gBAAgB,GAAG,KAAK,CAAC;IAClC,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,cAAc,CAAC,OAAgB;QACjC,MAAM,EAAE,SAAS,EAAE,GAAG,OAAO,CAAC;QAE9B,IAAI,CAAC,qBAAqB,CAAC,SAAS,CAAC,CAAC;QACtC,IAAI,CAAC,gCAAgC,CAAC,SAAS,CAAC,CAAC;QACjD,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAE5B,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;IAClC,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,mBAAmB,CAAC,MAAsB,EAAE,eAAiC;QACzF,MAAM,EAAE,eAAe,EAAE,KAAK,EAAE,GAAG,MAAM,CAAC;QAC1C,MAAM,cAAc,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;QAE5C,eAAe,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC;QAEhE,MAAM,YAAY,GAAG,eAAe,CAAC,MAAM,CAAC;QAC5C,MAAM,aAAa,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,cAAc,CAAC;QAE5D,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,qDAAqD,EAAE;YACjE,eAAe;YACf,KAAK;YACL,YAAY;YACZ,aAAa;YACb,cAAc,EAAE,YAAY,GAAG,aAAa;YAC5C,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;SACtD,CAAC,CAAC;IACP,CAAC;IAES,KAAK,CAAC,kBAAkB,CAAI,GAAW;QAC7C,IAAI,CAAC,KAAK,KAAV,IAAI,CAAC,KAAK,GAAK,MAAM,+BAAa,CAAC,IAAI,EAAE,EAAC;QAC1C,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAI,GAAG,CAAC,CAAC;QAEhD,OAAO,KAAM,CAAC;IAClB,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,qBAAqB,CAAC,MAAsB;QACxD,MAAM,EAAE,eAAe,EAAE,KAAK,EAAE,GAAG,UAAU,EAAE,GAAG,MAAM,CAAC;QAEzD,2CAA2C;QAC3C,IAAI,OAAO,CAAC;QACZ,IAAI;YACA,OAAO,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,EAAE,GAAG,EAAE,eAAe,EAAE,SAAS,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,IAAI,CAAC,kBAAkB,EAAE,MAAM,EAAE,EAAE,CAAC,CAAC;SAC3I;QAAC,OAAO,GAAG,EAAE;YACV,MAAM,IAAI,KAAK,CAAC,oCAAoC,eAAe,KAAK,GAAG,EAAE,CAAC,CAAC;SAClF;QAED,sCAAsC;QACtC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE;YACjB,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,gCAAgC,EAAE,EAAE,eAAe,EAAE,KAAK,EAAE,CAAC,CAAC;YAC/E,OAAO,EAAE,CAAC;SACb;QAED,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,GAAG,UAAU,EAAE,CAAC,CAAC,CAAC;IAC1D,CAAC;IAED;;;;OAIG;IACO,WAAW,CAAC,MAAc;QAChC,IAAI,OAAO,CAAC;QACZ,MAAM,IAAI,GAAG,OAAO,MAAM,CAAC;QAC3B,IAAI,IAAI,KAAK,QAAQ,EAAE;YACnB,OAAO,GAAG,IAAI,iBAAO,CAAC,EAAE,GAAG,EAAE,MAAgB,EAAE,CAAC,CAAC;SACpD;aAAM,IAAI,MAAM,YAAY,iBAAO,EAAE;YAClC,OAAO,GAAG,MAAM,CAAC;SACpB;aAAM,IAAI,MAAM,IAAI,IAAI,KAAK,QAAQ,EAAE;YACpC,OAAO,GAAG,IAAI,iBAAO,CAAC,MAAwB,CAAC,CAAC;SACnD;aAAM;YACH,MAAM,IAAI,KAAK,CAAC,oCAAoC,IAAI,EAAE,CAAC,CAAC;SAC/D;QAED,MAAM,YAAY,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,WAAW,CAAC,CAAC;QAE9D,sDAAsD;QACtD,IAAI,IAAI,CAAC,iBAAiB,IAAI,CAAC,YAAY,EAAE;YACzC,OAAO,CAAC,SAAS,IAAI,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;SACnD;QAED,MAAM,EAAE,SAAS,EAAE,GAAG,OAAO,CAAC;QAC9B,IAAI,CAAC,qBAAqB,CAAC,SAAS,CAAC,CAAC;QAEtC,yCAAyC;QACzC,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,cAAc,CAAC,SAAS,CAAC,EAAE;YAClD,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;YACxD,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;SAC/B;aAAM,IAAI,IAAI,CAAC,iBAAiB,EAAE;YAC/B,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,wBAAwB,SAAS,qFAAqF,CAAC,CAAC;SAC5I;IACL,CAAC;IAED;;;OAGG;IACO,qBAAqB,CAAC,SAAiB;QAC7C,IAAI,OAAO,SAAS,KAAK,QAAQ,IAAI,CAAC,SAAS,EAAE;YAC7C,MAAM,IAAI,KAAK,CAAC,wDAAwD,CAAC,CAAC;SAC7E;IACL,CAAC;IAED;;OAEG;IACO,gCAAgC,CAAC,SAAiB;QACxD,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE;YACjC,MAAM,IAAI,KAAK,CAAC,kDAAkD,SAAS,GAAG,CAAC,CAAC;SACnF;QACD,IAAI,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE;YAC/B,MAAM,IAAI,KAAK,CAAC,iDAAiD,SAAS,GAAG,CAAC,CAAC;SAClF;IACL,CAAC;IAED;;OAEG;IACO,oBAAoB;QAC1B,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE;YACrB,MAAM,IAAI,KAAK,CAAC,iGAAiG,CAAC,CAAC;SACtH;IACL,CAAC;IAED;;OAEG;IACH,MAAM;QACF,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAE5B,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;IAChC,CAAC;IAED;;OAEG;IACH,YAAY;QACR,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAE5B,OAAO,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;IACjD,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAoDG;IACH,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,QAAuB,EAAE,OAAiB,EAAE,UAA8B,EAAE;QAC1F,IAAA,YAAE,EAAC,QAAQ,EAAE,YAAE,CAAC,GAAG,CAAC,YAAE,CAAC,MAAM,EAAE,YAAE,CAAC,IAAI,CAAC,CAAC,CAAC;QACzC,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,KAAK,CAAC,CAAC;QACtB,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAEpD,MAAM,EAAE,GAAG,IAAI,WAAW,CAAC;YACvB,GAAG,OAAO;YACV,eAAe,EAAE,QAAQ,CAAC,CAAC,CAAC,GAAG,QAAQ,IAAI,6BAAqB,EAAE,CAAC,CAAC,CAAC,SAAS;YAC9E,kBAAkB,EAAE,QAAQ,CAAC,CAAC,CAAC,GAAG,QAAQ,IAAI,gCAAwB,EAAE,CAAC,CAAC,CAAC,SAAS;YACpF,OAAO;SACV,CAAC,CAAC;QACH,MAAM,EAAE,CAAC,UAAU,EAAE,CAAC;QAEtB,OAAO,EAAE,CAAC;IACd,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,mBAAmB,CAAC,OAA+D;QAC7F,OAAO,IAAA,0BAAkB,EAAC,OAAO,CAAC,CAAC;IACvC,CAAC;CACJ;AArmBD,kCAqmBC"}
|
|
1
|
+
{"version":3,"file":"request_list.js","sourceRoot":"","sources":["../../src/storages/request_list.ts"],"names":[],"mappings":";;;;AAAA,0CAAoD;AAEpD,iDAAuC;AAEvC,oDAAiD;AACjD,gCAA6B;AAC7B,wCAAqD;AACrD,oDAAqE;AACrE,iEAA4D;AAG5D,gBAAgB;AACH,QAAA,qBAAqB,GAAG,oBAAoB,CAAC;AAE1D,gBAAgB;AACH,QAAA,wBAAwB,GAAG,uBAAuB,CAAC;AAEhE,MAAM,mBAAmB,GAAG,0BAA0B,CAAC;AAwJvD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmEG;AACH,MAAa,WAAW;IAoDpB;;OAEG;IACH,YAAY,UAA8B,EAAE;QAtD5C;;;;mBAAc,SAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,aAAa,EAAE,CAAC;WAAC;QAEnD;;;;WAIG;QACH;;;;mBAAsB,EAAE;WAAC;QAEzB,kHAAkH;QAClH;;;;mBAAoB,CAAC;WAAC;QAEtB,gGAAgG;QAChG;;;;mBAAmD,EAAE;WAAC;QAEtD;;;WAGG;QACH;;;;mBAAa,IAAI,GAAG,EAAU;WAAC;QAE/B;;;WAGG;QACH;;;;mBAAY,IAAI,GAAG,EAAU;WAAC;QAE9B;;;WAGG;QACH;;;;mBAAmB,IAAI;WAAC;QAExB;;;WAGG;QACH;;;;mBAAuB,KAAK;WAAC;QAE7B;;;;mBAAoB,KAAK;WAAC;QAC1B;;;;mBAAwB,KAAK;WAAC;QAC9B;;;;;WAAiC;QACjC;;;;;WAAoC;QACpC;;;;;WAAwC;QACxC;;;;;WAA8B;QAC9B;;;;;WAAmC;QACnC;;;;;WAA0B;QAC1B;;;;;WAAqD;QACrD;;;;;WAAgD;QAChD;;;;;WAA6B;QAMzB,MAAM,EACF,OAAO,EACP,eAAe,EACf,eAAe,EACf,kBAAkB,EAClB,KAAK,EACL,kBAAkB,EAClB,iBAAiB,GAAG,KAAK,EACzB,MAAM,GAAG,6BAAa,CAAC,eAAe,EAAE,GAC3C,GAAG,OAAO,CAAC;QAEZ,IAAI,CAAC,CAAC,OAAO,IAAI,eAAe,CAAC,EAAE;YAC/B,MAAM,IAAI,kBAAa,CAAC,kEAAkE,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;SACjH;QACD,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7B,OAAO,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK;YAC1B,eAAe,EAAE,YAAE,CAAC,QAAQ,CAAC,QAAQ;YACrC,eAAe,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACnC,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACtC,KAAK,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC;gBACjC,SAAS,EAAE,YAAE,CAAC,MAAM;gBACpB,aAAa,EAAE,YAAE,CAAC,MAAM;gBACxB,UAAU,EAAE,YAAE,CAAC,MAAM;aACxB,CAAC;YACF,iBAAiB,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;YACtC,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;SACzC,CAAC,CAAC,CAAC;QAEJ,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC,CAAC,CAAC,OAAO,eAAe,EAAE,CAAC,CAAC,CAAC,eAAe,CAAC;QACpF,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC,CAAC,CAAC,OAAO,kBAAkB,EAAE,CAAC,CAAC,CAAC,kBAAkB,CAAC;QAChG,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC;QAC1B,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,eAAe,EAAE,CAAC;QAEvC,0HAA0H;QAC1H,IAAI,CAAC,iBAAiB,GAAG,iBAAiB,CAAC;QAE3C,qDAAqD;QACrD,IAAI,CAAC,OAAO,GAAG,OAAO,IAAI,EAAE,CAAC;QAC7B,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;QAEvC,gEAAgE;QAChE,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;IACjD,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,UAAU;QACZ,IAAI,IAAI,CAAC,SAAS,EAAE;YAChB,MAAM,IAAI,KAAK,CAAC,yDAAyD,CAAC,CAAC;SAC9E;QACD,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QAEtB,MAAM,CAAC,KAAK,EAAE,iBAAiB,CAAC,GAAG,MAAM,IAAI,CAAC,8BAA8B,EAAE,CAAC;QAE/E,mFAAmF;QACnF,8CAA8C;QAC9C,IAAI,iBAAiB,EAAE;YACnB,MAAM,IAAI,CAAC,qBAAqB,CAAC,iBAA2B,CAAC,CAAC;SACjE;aAAM;YACH,MAAM,IAAI,CAAC,uBAAuB,EAAE,CAAC;SACxC;QAED,IAAI,CAAC,aAAa,CAAC,KAAyB,CAAC,CAAC;QAC9C,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC;QAC1B,IAAI,IAAI,CAAC,kBAAkB,IAAI,CAAC,IAAI,CAAC,oBAAoB;YAAE,MAAM,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACzF,IAAI,IAAI,CAAC,eAAe,EAAE;YACtB,IAAI,CAAC,MAAM,CAAC,EAAE,+CAA0B,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;SACzE;IACL,CAAC;IAED;;;;OAIG;IACO,KAAK,CAAC,qBAAqB,CAAC,iBAAyB;QAC3D,gDAAgD;QAChD,uCAAuC;QACvC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;YAC1C,OAAO,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;SAC1B;QACD,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC;QAElB,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAC;QACjC,MAAM,aAAa,GAAG,IAAA,iCAAiB,EAAC,iBAAiB,CAAC,CAAC;QAC3D,IAAI,KAAK,EAAE,MAAM,OAAO,IAAI,aAAa,EAAE;YACvC,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;SAC7B;IACL,CAAC;IAED;;;;;OAKG;IACO,KAAK,CAAC,uBAAuB;QACnC,wFAAwF;QACxF,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;QACzC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,YAAY,EAAE,CAAC,EAAE,EAAE;YACnC,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;YAC/B,oEAAoE;YACpE,mDAAmD;YACnD,OAAO,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;YAEvB,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAK,MAAqB,CAAC,eAAe,EAAE;gBACtE,MAAM,eAAe,GAAG,MAAM,IAAI,CAAC,qBAAqB,CAAC,MAAwB,CAAC,CAAC;gBACnF,MAAM,IAAI,CAAC,mBAAmB,CAAC,MAAwB,EAAE,eAAe,CAAC,CAAC;aAC7E;iBAAM;gBACH,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;aAC5B;SACJ;QAED,iDAAiD;QACjD,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC;QAElB,IAAI,IAAI,CAAC,eAAe,EAAE;YACtB,IAAI;gBACA,MAAM,mBAAmB,GAAG,MAAM,IAAI,CAAC,eAAe,EAAE,CAAC;gBACzD,MAAM,wBAAwB,GAAG,mBAAmB,CAAC,MAAM,CAAC;gBAC5D,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,wBAAwB,EAAE,CAAC,EAAE,EAAE;oBAC/C,MAAM,MAAM,GAAG,mBAAmB,CAAC,KAAK,EAAE,CAAC;oBAC3C,IAAI,CAAC,WAAW,CAAC,MAAO,CAAC,CAAC;iBAC7B;aACJ;YAAC,OAAO,CAAC,EAAE;gBACR,MAAM,GAAG,GAAG,CAAU,CAAC;gBACvB,MAAM,IAAI,KAAK,CAAC,yDAAyD,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;aAC3F;SACJ;IACL,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,YAAY;QACd,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE;YACvB,MAAM,IAAI,KAAK,CAAC,2DAA2D,CAAC,CAAC;SAChF;QACD,IAAI,IAAI,CAAC,gBAAgB;YAAE,OAAO;QAClC,IAAI;YACA,IAAI,CAAC,KAAK,KAAV,IAAI,CAAC,KAAK,GAAK,MAAM,+BAAa,CAAC,IAAI,EAAE,EAAC;YAC1C,MAAM,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,eAAe,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;YACjE,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC;SAChC;QAAC,OAAO,CAAC,EAAE;YACR,MAAM,GAAG,GAAG,CAAU,CAAC;YACvB,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,yCAAyC,CAAC,CAAC;SACtE;IACL,CAAC;IAED;;;;OAIG;IACO,KAAK,CAAC,gBAAgB;QAC5B,MAAM,kBAAkB,GAAG,MAAM,IAAA,8BAAc,EAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC/D,IAAI,CAAC,KAAK,KAAV,IAAI,CAAC,KAAK,GAAK,MAAM,+BAAa,CAAC,IAAI,EAAE,EAAC;QAC1C,MAAM,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,kBAAmB,EAAE,kBAAkB,EAAE,EAAE,WAAW,EAAE,mBAAmB,EAAE,CAAC,CAAC;QAC9G,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAC;IACrC,CAAC;IAED;;OAEG;IACO,aAAa,CAAC,KAAwB;QAC5C,8DAA8D;QAC9D,IAAI,CAAC,KAAK;YAAE,OAAO;QACnB,0BAA0B;QAC1B,IAAI,OAAO,KAAK,CAAC,SAAS,KAAK,QAAQ,IAAI,KAAK,CAAC,SAAS,GAAG,CAAC,EAAE;YAC5D,MAAM,IAAI,KAAK,CAAC,uEAAuE,CAAC,CAAC;SAC5F;QACD,IAAI,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE;YACxC,MAAM,IAAI,KAAK,CAAC,8EAA8E,CAAC,CAAC;SACnG;QACD,IAAI,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM;eACnC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,SAAS,KAAK,KAAK,CAAC,aAAa,EAAE;YACrE,MAAM,IAAI,KAAK,CAAC,8FAA8F,CAAC,CAAC;SACnH;QAED,MAAM,oBAAoB,GAAa,EAAE,CAAC;QAC1C,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,SAAS,EAAE,EAAE;YACnC,MAAM,KAAK,GAAG,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC;YAC/C,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE;gBAC3B,MAAM,IAAI,KAAK,CAAC,iGAAiG,CAAC,CAAC;aACtH;YACD,IAAI,KAAK,IAAI,KAAK,CAAC,SAAS,EAAE;gBAC1B,oBAAoB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;aACxC;QACL,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC,SAAS,CAAC;QACjC,IAAI,CAAC,UAAU,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;QAE5C,cAAc;QACd,wEAAwE;QACxE,IAAI;QACJ,wBAAwB;QACxB,iDAAiD;QACjD,oBAAoB;QACpB,uCAAuC;QACvC,WAAW;QACX,2CAA2C;QAC3C,IAAI;QACJ,qGAAqG;QACrG,oFAAoF;QACpF,oCAAoC;QACpC,IAAI,oBAAoB,CAAC,MAAM,EAAE;YAC7B,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,0FAA0F,EAAE;gBACzG,oBAAoB;aACvB,CAAC,CAAC;YACH,KAAK,MAAM,SAAS,IAAI,oBAAoB,EAAE;gBAC1C,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;aACrC;SACJ;QAED,iDAAiD;QACjD,IAAI,CAAC,SAAS,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IAC9C,CAAC;IAED;;;OAGG;IACO,KAAK,CAAC,8BAA8B;QAC1C,IAAI,KAAwB,CAAC;QAC7B,IAAI,iBAA0B,CAAC;QAE/B,IAAI,IAAI,CAAC,YAAY,EAAE;YACnB,KAAK,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC;YAChC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,2CAA2C,CAAC,CAAC;SAC/D;aAAM,IAAI,IAAI,CAAC,eAAe,EAAE;YAC7B,KAAK,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;YAC5D,IAAI,KAAK;gBAAE,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,8DAA8D,CAAC,CAAC;SAC7F;QAED,IAAI,IAAI,CAAC,kBAAkB,EAAE;YACzB,iBAAiB,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;YAC3E,IAAI,iBAAiB;gBAAE,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,oEAAoE,CAAC,CAAC;SAC/G;QAED,OAAO,CAAC,KAAK,EAAE,iBAAiB,CAAC,CAAC;IACtC,CAAC;IAED;;;OAGG;IACH,QAAQ;QACJ,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAE5B,OAAO;YACH,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,aAAa,EAAE,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM;gBAChD,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,SAAS;gBACzC,CAAC,CAAC,IAAI;YACV,UAAU,EAAE,CAAC,GAAG,IAAI,CAAC,UAAU,CAAC;SACnC,CAAC;IACN,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,OAAO;QACT,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAE5B,OAAO,IAAI,CAAC,SAAS,CAAC,IAAI,KAAK,CAAC,IAAI,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;IAC/E,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU;QACZ,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAE5B,OAAO,IAAI,CAAC,UAAU,CAAC,IAAI,KAAK,CAAC,IAAI,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;IAChF,CAAC;IAED;;;;;;;OAOG;IACH,KAAK,CAAC,gBAAgB;QAClB,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAE5B,0CAA0C;QAC1C,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC;QACvD,IAAI,SAAS,EAAE;YACX,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;YACjC,MAAM,KAAK,GAAG,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC;YAC/C,OAAO,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;SAC/B;QAED,iCAAiC;QACjC,IAAI,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE;YACvC,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAC9C,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;YACvC,IAAI,CAAC,SAAS,EAAE,CAAC;YACjB,IAAI,CAAC,gBAAgB,GAAG,KAAK,CAAC;YAC9B,OAAO,OAAO,CAAC;SAClB;QAED,OAAO,IAAI,CAAC;IAChB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,kBAAkB,CAAC,OAAgB;QACrC,MAAM,EAAE,SAAS,EAAE,GAAG,OAAO,CAAC;QAE9B,IAAI,CAAC,qBAAqB,CAAC,SAAS,CAAC,CAAC;QACtC,IAAI,CAAC,gCAAgC,CAAC,SAAS,CAAC,CAAC;QACjD,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAE5B,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QAClC,IAAI,CAAC,gBAAgB,GAAG,KAAK,CAAC;IAClC,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,cAAc,CAAC,OAAgB;QACjC,MAAM,EAAE,SAAS,EAAE,GAAG,OAAO,CAAC;QAE9B,IAAI,CAAC,qBAAqB,CAAC,SAAS,CAAC,CAAC;QACtC,IAAI,CAAC,gCAAgC,CAAC,SAAS,CAAC,CAAC;QACjD,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAE5B,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;IAClC,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,mBAAmB,CAAC,MAAsB,EAAE,eAAiC;QACzF,MAAM,EAAE,eAAe,EAAE,KAAK,EAAE,GAAG,MAAM,CAAC;QAC1C,MAAM,cAAc,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;QAE5C,eAAe,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC;QAEhE,MAAM,YAAY,GAAG,eAAe,CAAC,MAAM,CAAC;QAC5C,MAAM,aAAa,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,cAAc,CAAC;QAE5D,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,qDAAqD,EAAE;YACjE,eAAe;YACf,KAAK;YACL,YAAY;YACZ,aAAa;YACb,cAAc,EAAE,YAAY,GAAG,aAAa;YAC5C,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;SACtD,CAAC,CAAC;IACP,CAAC;IAES,KAAK,CAAC,kBAAkB,CAAI,GAAW;QAC7C,IAAI,CAAC,KAAK,KAAV,IAAI,CAAC,KAAK,GAAK,MAAM,+BAAa,CAAC,IAAI,EAAE,EAAC;QAC1C,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAI,GAAG,CAAC,CAAC;QAEhD,OAAO,KAAM,CAAC;IAClB,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,qBAAqB,CAAC,MAAsB;QACxD,MAAM,EAAE,eAAe,EAAE,KAAK,EAAE,GAAG,UAAU,EAAE,GAAG,MAAM,CAAC;QAEzD,2CAA2C;QAC3C,IAAI,OAAO,CAAC;QACZ,IAAI;YACA,OAAO,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,EAAE,GAAG,EAAE,eAAe,EAAE,SAAS,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,IAAI,CAAC,kBAAkB,EAAE,MAAM,EAAE,EAAE,CAAC,CAAC;SAC3I;QAAC,OAAO,GAAG,EAAE;YACV,MAAM,IAAI,KAAK,CAAC,oCAAoC,eAAe,KAAK,GAAG,EAAE,CAAC,CAAC;SAClF;QAED,sCAAsC;QACtC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE;YACjB,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,gCAAgC,EAAE,EAAE,eAAe,EAAE,KAAK,EAAE,CAAC,CAAC;YAC/E,OAAO,EAAE,CAAC;SACb;QAED,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,GAAG,UAAU,EAAE,CAAC,CAAC,CAAC;IAC1D,CAAC;IAED;;;;OAIG;IACO,WAAW,CAAC,MAAc;QAChC,IAAI,OAAO,CAAC;QACZ,MAAM,IAAI,GAAG,OAAO,MAAM,CAAC;QAC3B,IAAI,IAAI,KAAK,QAAQ,EAAE;YACnB,OAAO,GAAG,IAAI,iBAAO,CAAC,EAAE,GAAG,EAAE,MAAgB,EAAE,CAAC,CAAC;SACpD;aAAM,IAAI,MAAM,YAAY,iBAAO,EAAE;YAClC,OAAO,GAAG,MAAM,CAAC;SACpB;aAAM,IAAI,MAAM,IAAI,IAAI,KAAK,QAAQ,EAAE;YACpC,OAAO,GAAG,IAAI,iBAAO,CAAC,MAAwB,CAAC,CAAC;SACnD;aAAM;YACH,MAAM,IAAI,KAAK,CAAC,oCAAoC,IAAI,EAAE,CAAC,CAAC;SAC/D;QAED,MAAM,YAAY,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,WAAW,CAAC,CAAC;QAE9D,sDAAsD;QACtD,IAAI,IAAI,CAAC,iBAAiB,IAAI,CAAC,YAAY,EAAE;YACzC,OAAO,CAAC,SAAS,IAAI,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;SACnD;QAED,MAAM,EAAE,SAAS,EAAE,GAAG,OAAO,CAAC;QAC9B,IAAI,CAAC,qBAAqB,CAAC,SAAS,CAAC,CAAC;QAEtC,yCAAyC;QACzC,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,cAAc,CAAC,SAAS,CAAC,EAAE;YAClD,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;YACxD,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;SAC/B;aAAM,IAAI,IAAI,CAAC,iBAAiB,EAAE;YAC/B,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,wBAAwB,SAAS,qFAAqF,CAAC,CAAC;SAC5I;IACL,CAAC;IAED;;;OAGG;IACO,qBAAqB,CAAC,SAAiB;QAC7C,IAAI,OAAO,SAAS,KAAK,QAAQ,IAAI,CAAC,SAAS,EAAE;YAC7C,MAAM,IAAI,KAAK,CAAC,wDAAwD,CAAC,CAAC;SAC7E;IACL,CAAC;IAED;;OAEG;IACO,gCAAgC,CAAC,SAAiB;QACxD,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE;YACjC,MAAM,IAAI,KAAK,CAAC,kDAAkD,SAAS,GAAG,CAAC,CAAC;SACnF;QACD,IAAI,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE;YAC/B,MAAM,IAAI,KAAK,CAAC,iDAAiD,SAAS,GAAG,CAAC,CAAC;SAClF;IACL,CAAC;IAED;;OAEG;IACO,oBAAoB;QAC1B,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE;YACrB,MAAM,IAAI,KAAK,CAAC,iGAAiG,CAAC,CAAC;SACtH;IACL,CAAC;IAED;;OAEG;IACH,MAAM;QACF,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAE5B,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;IAChC,CAAC;IAED;;OAEG;IACH,YAAY;QACR,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAE5B,OAAO,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;IACjD,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAoDG;IACH,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,QAAuB,EAAE,OAAiB,EAAE,UAA8B,EAAE;QAC1F,IAAA,YAAE,EAAC,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,YAAE,CAAC,MAAM,EAAE,YAAE,CAAC,IAAI,CAAC,CAAC,CAAC;QAClD,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,KAAK,CAAC,CAAC;QACtB,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAEpD,MAAM,EAAE,GAAG,IAAI,WAAW,CAAC;YACvB,GAAG,OAAO;YACV,eAAe,EAAE,QAAQ,CAAC,CAAC,CAAC,GAAG,QAAQ,IAAI,6BAAqB,EAAE,CAAC,CAAC,CAAC,SAAS;YAC9E,kBAAkB,EAAE,QAAQ,CAAC,CAAC,CAAC,GAAG,QAAQ,IAAI,gCAAwB,EAAE,CAAC,CAAC,CAAC,SAAS;YACpF,OAAO;SACV,CAAC,CAAC;QACH,MAAM,EAAE,CAAC,UAAU,EAAE,CAAC;QAEtB,OAAO,EAAE,CAAC;IACd,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,mBAAmB,CAAC,OAA+D;QAC7F,OAAO,IAAA,0BAAkB,EAAC,OAAO,CAAC,CAAC;IACvC,CAAC;CACJ;AArmBD,kCAqmBC"}
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import { LruCache } from '@apify/datastructures';
|
|
2
2
|
import { BatchAddRequestsResult, Dictionary, RequestQueueClient, RequestQueueInfo, StorageClient } from '@crawlee/types';
|
|
3
|
+
import { StorageManagerOptions } from './storage_manager';
|
|
3
4
|
import { Request, RequestOptions } from '../request';
|
|
5
|
+
import { Configuration } from '../configuration';
|
|
4
6
|
/**
|
|
5
7
|
* When requesting queue head we always fetch requestsInProgressCount * QUERY_HEAD_BUFFER number of requests.
|
|
6
8
|
* @internal
|
|
@@ -106,6 +108,7 @@ export interface RequestQueueOperationOptions {
|
|
|
106
108
|
* @category Sources
|
|
107
109
|
*/
|
|
108
110
|
export declare class RequestQueue {
|
|
111
|
+
readonly config: Configuration;
|
|
109
112
|
log: import("@apify/log/log").Log;
|
|
110
113
|
id: string;
|
|
111
114
|
name?: string;
|
|
@@ -138,7 +141,7 @@ export declare class RequestQueue {
|
|
|
138
141
|
/**
|
|
139
142
|
* @internal
|
|
140
143
|
*/
|
|
141
|
-
constructor(options: RequestQueueOptions);
|
|
144
|
+
constructor(options: RequestQueueOptions, config?: Configuration);
|
|
142
145
|
/**
|
|
143
146
|
* @ignore
|
|
144
147
|
*/
|
|
@@ -301,7 +304,7 @@ export declare class RequestQueue {
|
|
|
301
304
|
* the function returns the default request queue associated with the crawler run.
|
|
302
305
|
* @param [options] Open Request Queue options.
|
|
303
306
|
*/
|
|
304
|
-
static open(queueIdOrName?: string | null): Promise<RequestQueue>;
|
|
307
|
+
static open(queueIdOrName?: string | null, options?: StorageManagerOptions): Promise<RequestQueue>;
|
|
305
308
|
}
|
|
306
309
|
export interface RequestQueueOptions {
|
|
307
310
|
id: string;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"request_queue.d.ts","sourceRoot":"","sources":["../../src/storages/request_queue.ts"],"names":[],"mappings":"AACA,OAAO,EAAkB,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AAKjE,OAAO,EAAE,sBAAsB,EAAE,UAAU,EAAE,kBAAkB,EAAE,gBAAgB,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;
|
|
1
|
+
{"version":3,"file":"request_queue.d.ts","sourceRoot":"","sources":["../../src/storages/request_queue.ts"],"names":[],"mappings":"AACA,OAAO,EAAkB,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AAKjE,OAAO,EAAE,sBAAsB,EAAE,UAAU,EAAE,kBAAkB,EAAE,gBAAgB,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AACzH,OAAO,EAAkB,qBAAqB,EAAE,MAAM,mBAAmB,CAAC;AAE1E,OAAO,EAAE,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AACrD,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAIjD;;;GAGG;AACH,eAAO,MAAM,qBAAqB,MAAM,CAAC;AAEzC,gBAAgB;AAChB,eAAO,MAAM,iBAAiB,IAAI,CAAC;AAEnC;;;;GAIG;AACH,eAAO,MAAM,mCAAmC,QAAS,CAAC;AAE1D;;;GAGG;AACH,eAAO,MAAM,2BAA2B,IAAI,CAAC;AAS7C;;;;GAIG;AACH,eAAO,MAAM,gCAAgC,OAAO,CAAC;AAErD;;;;;;;;GAQG;AACH,wBAAgB,YAAY,CAAC,SAAS,EAAE,MAAM,UAQ7C;AAED;;;GAGG;AACH,MAAM,WAAW,kBAAkB;IAE/B,6DAA6D;IAC7D,iBAAiB,EAAE,OAAO,CAAC;IAE3B,0DAA0D;IAC1D,iBAAiB,EAAE,OAAO,CAAC;IAE3B,kCAAkC;IAClC,SAAS,EAAE,MAAM,CAAC;IAElB,SAAS,EAAE,MAAM,CAAC;CAErB;AAED,MAAM,WAAW,4BAA4B;IACzC;;;;;;;OAOG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;CACvB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6CG;AACH,qBAAa,YAAY;IAmDqB,QAAQ,CAAC,MAAM;IAlDzD,GAAG,+BAAyC;IAC5C,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,WAAW,SAAM;IACjB,SAAS,SAA0B;IACnC,MAAM,EAAE,kBAAkB,CAAC;IAE3B;;;;OAIG;IACH,OAAO,CAAC,aAAa,CAAgC;IACrD,qBAAqB,CAAC,EAAE,OAAO,CAAC;QAC5B,eAAe,EAAE,OAAO,CAAC;QACzB,SAAS,EAAE,MAAM,CAAC;QAClB,eAAe,EAAE,IAAI,CAAC;QACtB,cAAc,EAAE,IAAI,CAAC;QACrB,kBAAkB,CAAC,EAAE,OAAO,CAAC;KAChC,CAAC,GAAG,IAAI,CAAQ;IAIjB,UAAU,eAAa;IAKvB,YAAY,OAAc;IAC1B,qBAAqB,SAAY;IAKjC,eAAe,gBAA4D;IAI3E,iBAAiB,SAAK;IACtB,mBAAmB,SAAK;IAIxB,aAAa;mBACkE,OAAO;YAAM,MAAM;OAC5D;IAEtC;;OAEG;gBACS,OAAO,EAAE,mBAAmB,EAAW,MAAM,gBAAkC;IAS3F;;OAEG;IACH,eAAe;IAIf;;;;;;;;;;;;;OAaG;IACG,UAAU,CAAC,WAAW,EAAE,OAAO,GAAG,cAAc,EAAE,OAAO,GAAE,4BAAiC,GAAG,OAAO,CAAC,kBAAkB,CAAC;IAiDhI;;;;;;;;;;;OAWG;IACG,WAAW,CACb,YAAY,EAAE,CAAC,OAAO,GAAG,cAAc,CAAC,EAAE,EAC1C,OAAO,GAAE,4BAAiC,GAC3C,OAAO,CAAC,sBAAsB,CAAC;IAwFlC;;;;;OAKG;IACG,UAAU,CAAC,CAAC,SAAS,UAAU,GAAG,UAAU,EAAE,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;IAS3F;;;;;;;;;;;;;;;;OAgBG;IACG,gBAAgB,CAAC,CAAC,SAAS,UAAU,GAAG,UAAU,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;IA0DvF;;;;;OAKG;IACG,kBAAkB,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,kBAAkB,GAAG,IAAI,CAAC;IA8B9E;;;;;OAKG;IACG,cAAc,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,GAAE,4BAAiC,GAAG,OAAO,CAAC,kBAAkB,GAAG,IAAI,CAAC;IAwCtH;;;;;OAKG;IACG,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC;IAKjC;;;;;OAKG;IACG,UAAU,IAAI,OAAO,CAAC,OAAO,CAAC;IAapC,OAAO,CAAC,MAAM;IAWd;;OAEG;IACH,SAAS,CAAC,aAAa,CAAC,QAAQ,EAAE,MAAM,EAAE,kBAAkB,EAAE,kBAAkB,GAAG,IAAI;IASvF;;;;;;;;;;OAUG;cACa,qBAAqB,CACjC,iBAAiB,UAAQ,EACzB,KAAK,SAA8E,EACnF,SAAS,SAAI,GACd,OAAO,CAAC,OAAO,CAAC;IAmFnB;;OAEG;IACH,OAAO,CAAC,2BAA2B;IAQnC;;;OAGG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAM3B;;;;;;;;OAQG;IACG,YAAY,IAAI,OAAO,CAAC,MAAM,CAAC;IAMrC;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACG,OAAO,IAAI,OAAO,CAAC,gBAAgB,GAAG,SAAS,CAAC;IAItD;;;;;;;;;;;;;;;OAeG;WACU,IAAI,CAAC,aAAa,CAAC,EAAE,MAAM,GAAG,IAAI,EAAE,OAAO,GAAE,qBAA0B,GAAG,OAAO,CAAC,YAAY,CAAC;CAS/G;AAED,MAAM,WAAW,mBAAmB;IAChC,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,aAAa,CAAC;CACzB;AAED,MAAM,WAAW,yBAAyB;IACtC,SAAS,EAAE,MAAM,CAAC;IAClB,iBAAiB,EAAE,OAAO,CAAC;CAC9B"}
|