@crawlee/core 4.0.0-beta.30 → 4.0.0-beta.32
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/autoscaling/autoscaled_pool.d.ts +2 -2
- package/autoscaling/autoscaled_pool.d.ts.map +1 -1
- package/autoscaling/autoscaled_pool.js +2 -2
- package/autoscaling/autoscaled_pool.js.map +1 -1
- package/autoscaling/snapshotter.d.ts +3 -3
- package/autoscaling/snapshotter.d.ts.map +1 -1
- package/autoscaling/snapshotter.js +1 -2
- package/autoscaling/snapshotter.js.map +1 -1
- package/configuration.d.ts +1 -1
- package/configuration.d.ts.map +1 -1
- package/configuration.js +1 -1
- package/configuration.js.map +1 -1
- package/cookie_utils.d.ts.map +1 -1
- package/cookie_utils.js +4 -2
- package/cookie_utils.js.map +1 -1
- package/crawlers/context_pipeline.d.ts +10 -0
- package/crawlers/context_pipeline.d.ts.map +1 -1
- package/crawlers/context_pipeline.js +23 -1
- package/crawlers/context_pipeline.js.map +1 -1
- package/crawlers/crawler_commons.d.ts +2 -2
- package/crawlers/crawler_commons.d.ts.map +1 -1
- package/crawlers/statistics.d.ts +2 -2
- package/crawlers/statistics.d.ts.map +1 -1
- package/crawlers/statistics.js +1 -2
- package/crawlers/statistics.js.map +1 -1
- package/enqueue_links/enqueue_links.d.ts.map +1 -1
- package/enqueue_links/enqueue_links.js +2 -2
- package/enqueue_links/enqueue_links.js.map +1 -1
- package/events/event_manager.d.ts +1 -1
- package/events/event_manager.d.ts.map +1 -1
- package/events/event_manager.js +2 -2
- package/events/event_manager.js.map +1 -1
- package/events/local_event_manager.d.ts.map +1 -1
- package/events/local_event_manager.js +1 -2
- package/events/local_event_manager.js.map +1 -1
- package/log.d.ts +151 -0
- package/log.d.ts.map +1 -1
- package/log.js +111 -0
- package/log.js.map +1 -1
- package/package.json +5 -5
- package/proxy_configuration.d.ts +1 -1
- package/proxy_configuration.d.ts.map +1 -1
- package/proxy_configuration.js +3 -3
- package/proxy_configuration.js.map +1 -1
- package/recoverable_state.d.ts +2 -3
- package/recoverable_state.d.ts.map +1 -1
- package/recoverable_state.js +1 -2
- package/recoverable_state.js.map +1 -1
- package/request.d.ts.map +1 -1
- package/request.js +4 -3
- package/request.js.map +1 -1
- package/service_locator.d.ts +24 -33
- package/service_locator.d.ts.map +1 -1
- package/service_locator.js +44 -6
- package/service_locator.js.map +1 -1
- package/session_pool/session.d.ts +2 -2
- package/session_pool/session.d.ts.map +1 -1
- package/session_pool/session.js +2 -2
- package/session_pool/session.js.map +1 -1
- package/session_pool/session_pool.d.ts +3 -3
- package/session_pool/session_pool.d.ts.map +1 -1
- package/session_pool/session_pool.js +1 -2
- package/session_pool/session_pool.js.map +1 -1
- package/storages/dataset.d.ts +2 -2
- package/storages/dataset.d.ts.map +1 -1
- package/storages/dataset.js +2 -2
- package/storages/dataset.js.map +1 -1
- package/storages/key_value_store.d.ts.map +1 -1
- package/storages/key_value_store.js +1 -2
- package/storages/key_value_store.js.map +1 -1
- package/storages/request_list.d.ts.map +1 -1
- package/storages/request_list.js +1 -2
- package/storages/request_list.js.map +1 -1
- package/storages/request_manager_tandem.d.ts.map +1 -1
- package/storages/request_manager_tandem.js +2 -2
- package/storages/request_manager_tandem.js.map +1 -1
- package/storages/request_provider.d.ts +2 -2
- package/storages/request_provider.d.ts.map +1 -1
- package/storages/request_provider.js +3 -2
- package/storages/request_provider.js.map +1 -1
- package/storages/sitemap_request_list.d.ts.map +1 -1
- package/storages/sitemap_request_list.js +2 -2
- package/storages/sitemap_request_list.js.map +1 -1
package/request.js
CHANGED
|
@@ -2,10 +2,9 @@ import crypto from 'node:crypto';
|
|
|
2
2
|
import util from 'node:util';
|
|
3
3
|
import ow from 'ow';
|
|
4
4
|
import { normalizeUrl } from '@apify/utilities';
|
|
5
|
-
import {
|
|
5
|
+
import { serviceLocator } from './service_locator.js';
|
|
6
6
|
import { keys } from './typedefs.js';
|
|
7
7
|
// new properties on the Request object breaks serialization
|
|
8
|
-
const log = defaultLog.child({ prefix: 'Request' });
|
|
9
8
|
const requestOptionalPredicates = {
|
|
10
9
|
id: ow.optional.string,
|
|
11
10
|
loadedUrl: ow.optional.string.url,
|
|
@@ -370,7 +369,9 @@ class CrawleeRequest {
|
|
|
370
369
|
const normalizedUrl = normalizeUrl(url, keepUrlFragment) || url; // It returns null when url is invalid, causing weird errors.
|
|
371
370
|
if (!useExtendedUniqueKey) {
|
|
372
371
|
if (normalizedMethod !== 'GET' && payload) {
|
|
373
|
-
|
|
372
|
+
serviceLocator
|
|
373
|
+
.getLogger()
|
|
374
|
+
.warningOnce(`We've encountered a ${normalizedMethod} Request with a payload. ` +
|
|
374
375
|
'This is fine. Just letting you know that if your requests point to the same URL ' +
|
|
375
376
|
'and differ only in method and payload, you should see the "useExtendedUniqueKey" option of Request constructor.');
|
|
376
377
|
}
|
package/request.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"request.js","sourceRoot":"","sources":["../src/request.ts"],"names":[],"mappings":"AACA,OAAO,MAAM,MAAM,aAAa,CAAC;AACjC,OAAO,IAAI,MAAM,WAAW,CAAC;AAI7B,OAAO,EAAE,MAAM,IAAI,CAAC;AAEpB,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAIhD,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"request.js","sourceRoot":"","sources":["../src/request.ts"],"names":[],"mappings":"AACA,OAAO,MAAM,MAAM,aAAa,CAAC;AACjC,OAAO,IAAI,MAAM,WAAW,CAAC;AAI7B,OAAO,EAAE,MAAM,IAAI,CAAC;AAEpB,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAIhD,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAEtD,OAAO,EAAE,IAAI,EAAE,MAAM,eAAe,CAAC;AAErC,4DAA4D;AAC5D,MAAM,yBAAyB,GAAG;IAC9B,EAAE,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;IACtB,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG;IACjC,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;IAC7B,MAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;IAC1B,OAAO,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,UAAU,CAAC;IAClD,OAAO,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;IAC5B,UAAU,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;IAC9B,oBAAoB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;IACxC,UAAU,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;IAC9B,aAAa,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC;IAClD,OAAO,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;IAC3B,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;IAC5B,KAAK,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;IACzB,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC;IACnD,eAAe,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;IACpC,oBAAoB,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;IACzC,cAAc,EAAE,EAAE,CAAC,QAAQ,CAAC,OAAO;IACnC,UAAU,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,kBAAkB,CAAC,CAAC,CAAC;IACpD,KAAK,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC;CACrE,CAAC;AAEF,MAAM,CAAN,IAAY,YASX;AATD,WAAY,YAAY;IACpB,6DAAW,CAAA;IACX,2DAAU,CAAA;IACV,yDAAS,CAAA;IACT,qEAAe,CAAA;IACf,+CAAI,CAAA;IACJ,iEAAa,CAAA;IACb,iDAAK,CAAA;IACL,qDAAO,CAAA;AACX,CAAC,EATW,YAAY,KAAZ,YAAY,QASvB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AACH,MAAM,cAAc;IAChB,iBAAiB;IACjB,EAAE,CAAU;IAEZ,oCAAoC;IACpC,GAAG,CAAS;IAEZ;;;;;;;OAOG;IACH,SAAS,CAAU;IAEnB;;;OAGG;IACH,SAAS,CAAS;IAElB,yCAAyC;IACzC,MAAM,CAAqB;IAE3B,oDAAoD;IACpD,OAAO,CAAU;IAEjB,8FAA8F;IAC9F,OAAO,CAAU;IAEjB,2FAA2F;IAC3F,UAAU,CAAS;IAEnB,0DAA0D;IAC1D,aAAa,CAAW;IAExB,wEAAwE;IACxE,OAAO,CAA0B;IAEjC,sEAAsE;IAC9D,SAAS,GAAwB,EAAE,CAAC;IAE5C,gDAAgD;IAChD,QAAQ,GAAa,EAAc,CAAC;IAEpC;;;OAGG;IACH,SAAS,CAAU;IAEnB;;OAEG;IACH,YAAY,OAAiC;QACzC,EAAE,CAAC,OAAO,EAAE,gBAAgB,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC;QACzC,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,oBAAoB,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC;QACjD,4DAA4D;QAC5D,oDAAoD;QACpD,0DAA0D;QAC1D,sDAAsD;QACtD,oDAAoD;QACpD,IAAI,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;YAC3B,0CAA0C;YAC1C,IAAI,IAAI,KAAK,KAAK,EAAE,CAAC;gBACjB,OAAO;YACX,CAAC;YAED,MAAM,SAAS,GAAG,yBAAyB,CAAC,IAA8C,CAAC,CAAC;YAC5F,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;YAC5B,IAAI,SAAS,EAAE,CAAC;gBACZ,EAAE,CAAC,KAAK,EAAE,kBAAkB,IAAI,EAAE,EAAE,SAA0B,CAAC,CAAC;YACpE,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,MAAM,EACF,EAAE,EACF,GAAG,EACH,SAAS,EACT,SAAS,EACT,OAAO,EACP,OAAO,GAAG,KAAK,EACf,UAAU,GAAG,CAAC,EACd,oBAAoB,GAAG,CAAC,EACxB,UAAU,EACV,aAAa,GAAG,EAAE,EAClB,OAAO,GAAG,EAAE,EACZ,QAAQ,GAAG,EAAE,EACb,KAAK,EACL,SAAS,EACT,eAAe,GAAG,KAAK,EACvB,oBAAoB,GAAG,KAAK,EAC5B,cAAc,EACd,eAAe,EACf,UAAU,GACb,GAAG,OAMH,CAAC;QAEF,IAAI,EAAE,MAAM,GAAG,KAAK,EAAE,GAAG,OAAO,CAAC;QAEjC,MAAM,GAAG,MAAM,CAAC,WAAW,EAAwB,CAAC;QAEpD,IAAI,MAAM,KAAK,KAAK,IAAI,OAAO;YAAE,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;QAEnG,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC;QACf,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,SAAS;YACV,SAAS;gBACT,cAAc,CAAC,gBAAgB,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,eAAe,EAAE,oBAAoB,EAAE,CAAC,CAAC;QACrG,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,oBAAoB,GAAG,oBAAoB,CAAC;QACjD,IAAI,CAAC,aAAa,GAAG,CAAC,GAAG,aAAa,CAAC,CAAC;QACxC,IAAI,CAAC,OAAO,GAAG,EAAE,GAAG,OAAO,EAAE,CAAC;QAC9B,IAAI,CAAC,SAAS,GAAI,SAAqB,YAAY,IAAI,CAAC,CAAC,CAAE,SAAkB,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,SAAU,CAAC;QAEzG,IAAI,KAAK,EAAE,CAAC;YACR,QAAQ,CAAC,KAAK,GAAG,KAAK,CAAC;QAC3B,CAAC;QAED,MAAM,CAAC,gBAAgB,CAAC,IAAI,EAAE;YAC1B,SAAS,EAAE;gBACP,KAAK,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE,GAAG,QAAQ,EAAE;gBACrC,UAAU,EAAE,KAAK;gBACjB,QAAQ,EAAE,IAAI;aACjB;YACD,QAAQ,EAAE;gBACN,GAAG,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,SAAS;gBACzB,GAAG,EAAE,CAAC,KAA0B,EAAE,EAAE;oBAChC,MAAM,CAAC,gBAAgB,CAAC,KAAK,EAAE;wBAC3B,SAAS,EAAE;4BACP,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS;4BAC/B,UAAU,EAAE,KAAK;4BACjB,QAAQ,EAAE,IAAI;yBACjB;wBACD,MAAM,EAAE;4BACJ,KAAK,EAAE,GAAG,EAAE;gCACR,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oCACnD,OAAO;wCACH,GAAG,IAAI,CAAC,SAAS;wCACjB,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS;qCACtC,CAAC;gCACN,CAAC;gCAED,OAAO,IAAI,CAAC,SAAS,CAAC;4BAC1B,CAAC;4BACD,UAAU,EAAE,KAAK;4BACjB,QAAQ,EAAE,IAAI;yBACjB;qBACJ,CAAC,CAAC;oBACH,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;gBAC3B,CAAC;gBACD,UAAU,EAAE,IAAI;aACnB;SACJ,CAAC,CAAC;QAEH,4EAA4E;QAC5E,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QAEzB,IAAI,cAAc,IAAI,IAAI;YAAE,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC;QACjE,IAAI,UAAU,IAAI,IAAI;YAAE,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QACrD,IAAI,UAAU,IAAI,IAAI;YAAE,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,UAAU,KAAK,UAAU,CAAC;QAE1E,mFAAmF;QACnF,IAAI,eAAe,EAAE,CAAC;YAClB,IAAI,CAAC,eAAe,KAAK,eAAe,CAAC;QAC7C,CAAC;IACL,CAAC;IAED;;;OAGG;IACI,mBAAmB;QACtB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE;YACzB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,IAAI,EAAE,IAAI,CAAC,OAAO;SACrB,CAAC,CAAC;IACP,CAAC;IAED,yGAAyG;IACzG,IAAI,cAAc;QACd,OAAO,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,cAAc,IAAI,KAAK,CAAC;IAC5D,CAAC;IAED,yGAAyG;IACzG,IAAI,cAAc,CAAC,KAAc;QAC7B,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,CAAC;YAC1B,IAAI,CAAC,QAAuB,CAAC,SAAS,GAAG,EAAE,cAAc,EAAE,KAAK,EAAE,CAAC;QACxE,CAAC;aAAM,CAAC;YACJ,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,cAAc,GAAG,KAAK,CAAC;QACnD,CAAC;IACL,CAAC;IAED;;;OAGG;IACH,IAAI,UAAU;QACV,OAAO,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,UAAU,IAAI,CAAC,CAAC;IACpD,CAAC;IAED;;;OAGG;IACH,IAAI,UAAU,CAAC,KAAa;QACvB,IAAI,CAAC,QAAuB,CAAC,SAAS,KAAK,EAAE,CAAC;QAC/C,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,UAAU,GAAG,KAAK,CAAC;IAC/C,CAAC;IAED,2HAA2H;IAC3H,IAAI,oBAAoB;QACpB,OAAO,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,oBAAoB,IAAI,CAAC,CAAC;IAC9D,CAAC;IAED,2HAA2H;IAC3H,IAAI,oBAAoB,CAAC,KAAa;QAClC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,CAAC;YAC1B,IAAI,CAAC,QAAuB,CAAC,SAAS,GAAG,EAAE,oBAAoB,EAAE,KAAK,EAAE,CAAC;QAC9E,CAAC;aAAM,CAAC;YACJ,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,oBAAoB,GAAG,KAAK,CAAC;QACzD,CAAC;IACL,CAAC;IAED,oDAAoD;IACpD,IAAI,KAAK;QACL,OAAO,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;IAC/B,CAAC;IAED,oDAAoD;IACpD,IAAI,KAAK,CAAC,KAAyB;QAC9B,IAAI,CAAC,QAAuB,CAAC,KAAK,GAAG,KAAK,CAAC;IAChD,CAAC;IAED,8HAA8H;IAC9H,IAAI,UAAU;QACV,OAAO,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,UAAU,CAAC;IAC/C,CAAC;IAED,8HAA8H;IAC9H,IAAI,UAAU,CAAC,KAAyB;QACpC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,CAAC;YAC1B,IAAI,CAAC,QAAuB,CAAC,SAAS,GAAG,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC;QACpE,CAAC;aAAM,CAAC;YACJ,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,UAAU,GAAG,KAAK,CAAC;QAC/C,CAAC;IACL,CAAC;IAED,uDAAuD;IACvD,IAAI,KAAK;QACL,OAAO,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,KAAK,IAAI,YAAY,CAAC,WAAW,CAAC;IACtE,CAAC;IAED,uDAAuD;IACvD,IAAI,KAAK,CAAC,KAAmB;QACzB,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,CAAC;YAC1B,IAAI,CAAC,QAAuB,CAAC,SAAS,GAAG,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;QAC/D,CAAC;aAAM,CAAC;YACJ,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,KAAK,GAAG,KAAK,CAAC;QAC1C,CAAC;IACL,CAAC;IAED;;OAEG;IACH,IAAI,aAAa;QACb,OAAO,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,aAAa,CAAC;IAClD,CAAC;IAED;;OAEG;IACH,IAAI,aAAa,CAAC,KAAuC;QACrD,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,CAAC;YAC1B,IAAI,CAAC,QAAuB,CAAC,SAAS,GAAG,EAAE,aAAa,EAAE,KAAK,EAAE,CAAC;QACvE,CAAC;aAAM,CAAC;YACJ,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,aAAa,GAAG,KAAK,CAAC;QAClD,CAAC;IACL,CAAC;IAED,IAAY,eAAe;QACvB,OAAO,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,eAAe,CAAC;IACpD,CAAC;IAED,IAAY,eAAe,CAAC,KAAsC;QAC9D,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,CAAC;YAC1B,IAAI,CAAC,QAAuB,CAAC,SAAS,GAAG,EAAE,eAAe,EAAE,KAAK,EAAE,CAAC;QACzE,CAAC;aAAM,CAAC;YACJ,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,eAAe,GAAG,KAAK,CAAC;QACpD,CAAC;IACL,CAAC;IAED;;;;;;;;;;;;OAYG;IACH,gBAAgB,CAAC,cAAuB,EAAE,UAAmC,EAAE;QAC3E,MAAM,EAAE,SAAS,EAAE,GAAG,OAAO,CAAC;QAC9B,IAAI,OAAO,CAAC;QACZ,MAAM,IAAI,GAAG,OAAO,cAAc,CAAC;QACnC,IAAI,IAAI,KAAK,QAAQ,EAAE,CAAC;YACpB,IAAI,CAAC,cAAc,EAAE,CAAC;gBAClB,OAAO,GAAG,MAAM,CAAC;YACrB,CAAC;iBAAM,IAAI,cAAc,YAAY,KAAK,EAAE,CAAC;gBACzC,OAAO,GAAG,SAAS;oBACf,CAAC,CAAC,cAAc,CAAC,OAAO;oBACxB,CAAC,CAAC,8BAA8B;wBAC9B,cAAc,CAAC,KAAK,CAAC;YAC/B,CAAC;iBAAM,IAAI,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,cAAc,CAAC,EAAE,SAAS,CAAC,EAAE,CAAC;gBACxD,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,cAAc,CAAC,EAAE,SAAS,CAAC,CAAC;YAC7D,CAAC;iBAAM,IAAK,cAAyB,CAAC,QAAQ,EAAE,KAAK,iBAAiB,EAAE,CAAC;gBACrE,OAAO,GAAI,cAAyB,CAAC,QAAQ,EAAE,CAAC;YACpD,CAAC;iBAAM,CAAC;gBACJ,IAAI,CAAC;oBACD,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;gBAC3C,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACX,OAAO,GAAG,yDAAyD,CAAC;gBACxE,CAAC;YACL,CAAC;QACL,CAAC;aAAM,IAAI,IAAI,KAAK,WAAW,EAAE,CAAC;YAC9B,OAAO,GAAG,WAAW,CAAC;QAC1B,CAAC;aAAM,CAAC;YACJ,OAAO,GAAI,cAAyB,CAAC,QAAQ,EAAE,CAAC;QACpD,CAAC;QAED,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACrC,CAAC;IAED,gBAAgB;IAChB,MAAM,CAAC,gBAAgB,CAAC,EACpB,GAAG,EACH,MAAM,GAAG,KAAK,EACd,OAAO,EACP,eAAe,GAAG,KAAK,EACvB,oBAAoB,GAAG,KAAK,GACN;QACtB,MAAM,gBAAgB,GAAG,MAAM,CAAC,WAAW,EAAE,CAAC;QAC9C,MAAM,aAAa,GAAG,YAAY,CAAC,GAAG,EAAE,eAAe,CAAC,IAAI,GAAG,CAAC,CAAC,6DAA6D;QAC9H,IAAI,CAAC,oBAAoB,EAAE,CAAC;YACxB,IAAI,gBAAgB,KAAK,KAAK,IAAI,OAAO,EAAE,CAAC;gBACxC,cAAc;qBACT,SAAS,EAAE;qBACX,WAAW,CACR,uBAAuB,gBAAgB,2BAA2B;oBAC9D,kFAAkF;oBAClF,iHAAiH,CACxH,CAAC;YACV,CAAC;YACD,OAAO,aAAa,CAAC;QACzB,CAAC;QACD,MAAM,WAAW,GAAG,OAAO,CAAC,CAAC,CAAC,cAAc,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACvE,OAAO,GAAG,gBAAgB,IAAI,WAAW,KAAK,aAAa,EAAE,CAAC;IAClE,CAAC;IAED,gBAAgB;IAChB,MAAM,CAAC,WAAW,CAAC,OAAmB;QAClC,OAAO,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAC9G,CAAC;CACJ;AA8ID,OAAO,EAAE,cAAc,IAAI,OAAO,EAAE,CAAC"}
|
package/service_locator.d.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import type { StorageClient } from '@crawlee/types';
|
|
2
2
|
import { Configuration } from './configuration.js';
|
|
3
3
|
import type { EventManager } from './events/event_manager.js';
|
|
4
|
+
import type { CrawleeLogger } from './log.js';
|
|
4
5
|
import type { IStorage, StorageManager } from './storages/storage_manager.js';
|
|
5
6
|
import type { Constructor } from './typedefs.js';
|
|
6
7
|
interface ServiceLocatorInterface {
|
|
@@ -40,6 +41,23 @@ interface ServiceLocatorInterface {
|
|
|
40
41
|
* @throws {ServiceConflictError} If a different storage client has already been retrieved
|
|
41
42
|
*/
|
|
42
43
|
setStorageClient(storageClient: StorageClient): void;
|
|
44
|
+
/**
|
|
45
|
+
* Get the logger.
|
|
46
|
+
* Returns the default `@apify/log` logger if none has been set.
|
|
47
|
+
*/
|
|
48
|
+
getLogger(): CrawleeLogger;
|
|
49
|
+
/**
|
|
50
|
+
* Set the logger.
|
|
51
|
+
*
|
|
52
|
+
* @param logger The logger to set
|
|
53
|
+
* @throws {ServiceConflictError} If a different logger has already been retrieved
|
|
54
|
+
*/
|
|
55
|
+
setLogger(logger: CrawleeLogger): void;
|
|
56
|
+
/**
|
|
57
|
+
* Get a child logger with the given prefix.
|
|
58
|
+
* Equivalent to `getLogger().child({ prefix })`.
|
|
59
|
+
*/
|
|
60
|
+
getChildLog(prefix: string): CrawleeLogger;
|
|
43
61
|
getStorageManager(constructor: Constructor<IStorage>): StorageManager | undefined;
|
|
44
62
|
setStorageManager(constructor: Constructor<IStorage>, storageManager: StorageManager): void;
|
|
45
63
|
/**
|
|
@@ -54,42 +72,11 @@ interface ServiceLocatorInterface {
|
|
|
54
72
|
*/
|
|
55
73
|
reset(): void;
|
|
56
74
|
}
|
|
57
|
-
/**
|
|
58
|
-
* Service locator for managing the services used by Crawlee.
|
|
59
|
-
*
|
|
60
|
-
* All services are initialized to their default value lazily.
|
|
61
|
-
*
|
|
62
|
-
* There are two primary usage patterns:
|
|
63
|
-
*
|
|
64
|
-
* **1. Global service locator (for default services):**
|
|
65
|
-
* ```typescript
|
|
66
|
-
* import { serviceLocator, BasicCrawler } from 'crawlee';
|
|
67
|
-
*
|
|
68
|
-
* // Optionally configure global services before creating crawlers
|
|
69
|
-
* serviceLocator.setStorageClient(myCustomClient);
|
|
70
|
-
*
|
|
71
|
-
* // Crawler uses global services
|
|
72
|
-
* const crawler = new BasicCrawler({ ... });
|
|
73
|
-
* ```
|
|
74
|
-
*
|
|
75
|
-
* **2. Per-crawler services (recommended for isolation):**
|
|
76
|
-
* ```typescript
|
|
77
|
-
* import { BasicCrawler, Configuration, LocalEventManager } from 'crawlee';
|
|
78
|
-
* import { MemoryStorage } from '@crawlee/memory-storage';
|
|
79
|
-
*
|
|
80
|
-
* const crawler = new BasicCrawler({
|
|
81
|
-
* requestHandler: async ({ request }) => { ... },
|
|
82
|
-
* configuration: new Configuration({ ... }), // custom config
|
|
83
|
-
* storageClient: new MemoryStorage(), // custom storage
|
|
84
|
-
* eventManager: LocalEventManager.fromConfig(), // custom events
|
|
85
|
-
* });
|
|
86
|
-
* // Crawler has its own isolated ServiceLocator instance
|
|
87
|
-
* ```
|
|
88
|
-
*/
|
|
89
75
|
export declare class ServiceLocator implements ServiceLocatorInterface {
|
|
90
76
|
private configuration?;
|
|
91
77
|
private eventManager?;
|
|
92
78
|
private storageClient?;
|
|
79
|
+
private logger?;
|
|
93
80
|
/**
|
|
94
81
|
* Storage managers for Dataset, KeyValueStore, and RequestQueue.
|
|
95
82
|
* Manages caching and lifecycle of storage instances.
|
|
@@ -101,14 +88,18 @@ export declare class ServiceLocator implements ServiceLocatorInterface {
|
|
|
101
88
|
* @param configuration Optional configuration instance to use
|
|
102
89
|
* @param eventManager Optional event manager instance to use
|
|
103
90
|
* @param storageClient Optional storage client instance to use
|
|
91
|
+
* @param logger Optional logger instance to use
|
|
104
92
|
*/
|
|
105
|
-
constructor(configuration?: Configuration, eventManager?: EventManager, storageClient?: StorageClient);
|
|
93
|
+
constructor(configuration?: Configuration, eventManager?: EventManager, storageClient?: StorageClient, logger?: CrawleeLogger);
|
|
106
94
|
getConfiguration(): Configuration;
|
|
107
95
|
setConfiguration(configuration: Configuration): void;
|
|
108
96
|
getEventManager(): EventManager;
|
|
109
97
|
setEventManager(eventManager: EventManager): void;
|
|
110
98
|
getStorageClient(): StorageClient;
|
|
111
99
|
setStorageClient(storageClient: StorageClient): void;
|
|
100
|
+
getLogger(): CrawleeLogger;
|
|
101
|
+
setLogger(logger: CrawleeLogger): void;
|
|
102
|
+
getChildLog(prefix: string): CrawleeLogger;
|
|
112
103
|
getStorageManager(constructor: Constructor<IStorage>): StorageManager | undefined;
|
|
113
104
|
setStorageManager(constructor: Constructor<IStorage>, storageManager: StorageManager): void;
|
|
114
105
|
clearStorageManagerCache(): void;
|
package/service_locator.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"service_locator.d.ts","sourceRoot":"","sources":["../src/service_locator.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAIpD,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAEnD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAC;AAE9D,OAAO,KAAK,EAAE,QAAQ,EAAE,cAAc,EAAE,MAAM,+BAA+B,CAAC;AAC9E,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAEjD,UAAU,uBAAuB;IAC7B;;;OAGG;IACH,gBAAgB,IAAI,aAAa,CAAC;IAElC;;;;;OAKG;IACH,gBAAgB,CAAC,aAAa,EAAE,aAAa,GAAG,IAAI,CAAC;IAErD;;;OAGG;IACH,eAAe,IAAI,YAAY,CAAC;IAEhC;;;;;OAKG;IACH,eAAe,CAAC,YAAY,EAAE,YAAY,GAAG,IAAI,CAAC;IAElD;;;OAGG;IACH,gBAAgB,IAAI,aAAa,CAAC;IAElC;;;;;OAKG;IACH,gBAAgB,CAAC,aAAa,EAAE,aAAa,GAAG,IAAI,CAAC;IAErD,iBAAiB,CAAC,WAAW,EAAE,WAAW,CAAC,QAAQ,CAAC,GAAG,cAAc,GAAG,SAAS,CAAC;IAElF,iBAAiB,CAAC,WAAW,EAAE,WAAW,CAAC,QAAQ,CAAC,EAAE,cAAc,EAAE,cAAc,GAAG,IAAI,CAAC;IAE5F;;;OAGG;IACH,wBAAwB,IAAI,IAAI,CAAC;IAEjC;;;;OAIG;IACH,KAAK,IAAI,IAAI,CAAC;CACjB;
|
|
1
|
+
{"version":3,"file":"service_locator.d.ts","sourceRoot":"","sources":["../src/service_locator.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAIpD,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAEnD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAC;AAE9D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAE9C,OAAO,KAAK,EAAE,QAAQ,EAAE,cAAc,EAAE,MAAM,+BAA+B,CAAC;AAC9E,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAEjD,UAAU,uBAAuB;IAC7B;;;OAGG;IACH,gBAAgB,IAAI,aAAa,CAAC;IAElC;;;;;OAKG;IACH,gBAAgB,CAAC,aAAa,EAAE,aAAa,GAAG,IAAI,CAAC;IAErD;;;OAGG;IACH,eAAe,IAAI,YAAY,CAAC;IAEhC;;;;;OAKG;IACH,eAAe,CAAC,YAAY,EAAE,YAAY,GAAG,IAAI,CAAC;IAElD;;;OAGG;IACH,gBAAgB,IAAI,aAAa,CAAC;IAElC;;;;;OAKG;IACH,gBAAgB,CAAC,aAAa,EAAE,aAAa,GAAG,IAAI,CAAC;IAErD;;;OAGG;IACH,SAAS,IAAI,aAAa,CAAC;IAE3B;;;;;OAKG;IACH,SAAS,CAAC,MAAM,EAAE,aAAa,GAAG,IAAI,CAAC;IAEvC;;;OAGG;IACH,WAAW,CAAC,MAAM,EAAE,MAAM,GAAG,aAAa,CAAC;IAE3C,iBAAiB,CAAC,WAAW,EAAE,WAAW,CAAC,QAAQ,CAAC,GAAG,cAAc,GAAG,SAAS,CAAC;IAElF,iBAAiB,CAAC,WAAW,EAAE,WAAW,CAAC,QAAQ,CAAC,EAAE,cAAc,EAAE,cAAc,GAAG,IAAI,CAAC;IAE5F;;;OAGG;IACH,wBAAwB,IAAI,IAAI,CAAC;IAEjC;;;;OAIG;IACH,KAAK,IAAI,IAAI,CAAC;CACjB;AAsCD,qBAAa,cAAe,YAAW,uBAAuB;IAC1D,OAAO,CAAC,aAAa,CAAC,CAAgB;IACtC,OAAO,CAAC,YAAY,CAAC,CAAe;IACpC,OAAO,CAAC,aAAa,CAAC,CAAgB;IACtC,OAAO,CAAC,MAAM,CAAC,CAAgB;IAE/B;;;OAGG;IACH,OAAO,CAAC,eAAe,CAAoD;IAE3E;;;;;;;OAOG;gBAEC,aAAa,CAAC,EAAE,aAAa,EAC7B,YAAY,CAAC,EAAE,YAAY,EAC3B,aAAa,CAAC,EAAE,aAAa,EAC7B,MAAM,CAAC,EAAE,aAAa;IAQ1B,gBAAgB,IAAI,aAAa;IAUjC,gBAAgB,CAAC,aAAa,EAAE,aAAa,GAAG,IAAI;IAcpD,eAAe,IAAI,YAAY;IAgB/B,eAAe,CAAC,YAAY,EAAE,YAAY,GAAG,IAAI;IAcjD,gBAAgB,IAAI,aAAa;IAmBjC,gBAAgB,CAAC,aAAa,EAAE,aAAa,GAAG,IAAI;IAcpD,SAAS,IAAI,aAAa;IAO1B,SAAS,CAAC,MAAM,EAAE,aAAa,GAAG,IAAI;IAYtC,WAAW,CAAC,MAAM,EAAE,MAAM,GAAG,aAAa;IAI1C,iBAAiB,CAAC,WAAW,EAAE,WAAW,CAAC,QAAQ,CAAC,GAAG,cAAc,GAAG,SAAS;IAIjF,iBAAiB,CAAC,WAAW,EAAE,WAAW,CAAC,QAAQ,CAAC,EAAE,cAAc,EAAE,cAAc,GAAG,IAAI;IAY3F,wBAAwB,IAAI,IAAI;IAchC,KAAK,IAAI,IAAI;CAOhB;AASD;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,2BAA2B,CACvC,cAAc,EAAE,cAAc,EAC9B,MAAM,EAAE,EAAE,GACX;IAAE,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;IAAC,UAAU,EAAE,MAAM,IAAI,CAAC;IAAC,SAAS,EAAE,MAAM,IAAI,CAAA;CAAE,CA4C/E;AAED,eAAO,MAAM,cAAc,EAAE,uBAqD5B,CAAC"}
|
package/service_locator.js
CHANGED
|
@@ -4,6 +4,7 @@ import log from '@apify/log';
|
|
|
4
4
|
import { Configuration } from './configuration.js';
|
|
5
5
|
import { ServiceConflictError } from './errors.js';
|
|
6
6
|
import { LocalEventManager } from './events/local_event_manager.js';
|
|
7
|
+
import { ApifyLogAdapter } from './log.js';
|
|
7
8
|
/**
|
|
8
9
|
* Service locator for managing the services used by Crawlee.
|
|
9
10
|
*
|
|
@@ -36,10 +37,14 @@ import { LocalEventManager } from './events/local_event_manager.js';
|
|
|
36
37
|
* // Crawler has its own isolated ServiceLocator instance
|
|
37
38
|
* ```
|
|
38
39
|
*/
|
|
40
|
+
// Used as fallback in ServiceLocator methods that need to log before a logger is explicitly set,
|
|
41
|
+
// without implicitly locking the logger slot (which getLogger() would do).
|
|
42
|
+
const fallbackLog = new ApifyLogAdapter(log);
|
|
39
43
|
export class ServiceLocator {
|
|
40
44
|
configuration;
|
|
41
45
|
eventManager;
|
|
42
46
|
storageClient;
|
|
47
|
+
logger;
|
|
43
48
|
/**
|
|
44
49
|
* Storage managers for Dataset, KeyValueStore, and RequestQueue.
|
|
45
50
|
* Manages caching and lifecycle of storage instances.
|
|
@@ -51,15 +56,17 @@ export class ServiceLocator {
|
|
|
51
56
|
* @param configuration Optional configuration instance to use
|
|
52
57
|
* @param eventManager Optional event manager instance to use
|
|
53
58
|
* @param storageClient Optional storage client instance to use
|
|
59
|
+
* @param logger Optional logger instance to use
|
|
54
60
|
*/
|
|
55
|
-
constructor(configuration, eventManager, storageClient) {
|
|
61
|
+
constructor(configuration, eventManager, storageClient, logger) {
|
|
56
62
|
this.configuration = configuration;
|
|
57
63
|
this.eventManager = eventManager;
|
|
58
64
|
this.storageClient = storageClient;
|
|
65
|
+
this.logger = logger;
|
|
59
66
|
}
|
|
60
67
|
getConfiguration() {
|
|
61
68
|
if (!this.configuration) {
|
|
62
|
-
|
|
69
|
+
(this.logger ?? fallbackLog).debug('No configuration set, implicitly creating and using default Configuration.');
|
|
63
70
|
this.configuration = new Configuration();
|
|
64
71
|
}
|
|
65
72
|
return this.configuration;
|
|
@@ -77,9 +84,9 @@ export class ServiceLocator {
|
|
|
77
84
|
}
|
|
78
85
|
getEventManager() {
|
|
79
86
|
if (!this.eventManager) {
|
|
80
|
-
|
|
87
|
+
(this.logger ?? fallbackLog).debug('No event manager set, implicitly creating and using default LocalEventManager.');
|
|
81
88
|
if (!this.configuration) {
|
|
82
|
-
|
|
89
|
+
(this.logger ?? fallbackLog).warning('Implicit creation of event manager will implicitly set configuration as side effect. ' +
|
|
83
90
|
'It is advised to explicitly first set the configuration instead.');
|
|
84
91
|
}
|
|
85
92
|
this.eventManager = LocalEventManager.fromConfig(this.getConfiguration());
|
|
@@ -99,9 +106,9 @@ export class ServiceLocator {
|
|
|
99
106
|
}
|
|
100
107
|
getStorageClient() {
|
|
101
108
|
if (!this.storageClient) {
|
|
102
|
-
|
|
109
|
+
(this.logger ?? fallbackLog).debug('No storage client set, implicitly creating and using default MemoryStorage.');
|
|
103
110
|
if (!this.configuration) {
|
|
104
|
-
|
|
111
|
+
(this.logger ?? fallbackLog).warning('Implicit creation of storage client will implicitly set configuration as side effect. ' +
|
|
105
112
|
'It is advised to explicitly first set the configuration instead.');
|
|
106
113
|
}
|
|
107
114
|
const config = this.getConfiguration();
|
|
@@ -122,6 +129,24 @@ export class ServiceLocator {
|
|
|
122
129
|
}
|
|
123
130
|
this.storageClient = storageClient;
|
|
124
131
|
}
|
|
132
|
+
getLogger() {
|
|
133
|
+
if (!this.logger) {
|
|
134
|
+
this.logger = new ApifyLogAdapter(log);
|
|
135
|
+
}
|
|
136
|
+
return this.logger;
|
|
137
|
+
}
|
|
138
|
+
setLogger(logger) {
|
|
139
|
+
if (this.logger === logger) {
|
|
140
|
+
return;
|
|
141
|
+
}
|
|
142
|
+
if (this.logger) {
|
|
143
|
+
throw new ServiceConflictError('Logger', logger, this.logger);
|
|
144
|
+
}
|
|
145
|
+
this.logger = logger;
|
|
146
|
+
}
|
|
147
|
+
getChildLog(prefix) {
|
|
148
|
+
return this.getLogger().child({ prefix });
|
|
149
|
+
}
|
|
125
150
|
getStorageManager(constructor) {
|
|
126
151
|
return this.storageManagers.get(constructor);
|
|
127
152
|
}
|
|
@@ -148,6 +173,7 @@ export class ServiceLocator {
|
|
|
148
173
|
this.configuration = undefined;
|
|
149
174
|
this.eventManager = undefined;
|
|
150
175
|
this.storageClient = undefined;
|
|
176
|
+
this.logger = undefined;
|
|
151
177
|
this.clearStorageManagerCache();
|
|
152
178
|
}
|
|
153
179
|
}
|
|
@@ -232,6 +258,18 @@ export const serviceLocator = {
|
|
|
232
258
|
const currentServiceLocator = serviceLocatorStorage.getStore() ?? globalServiceLocator;
|
|
233
259
|
currentServiceLocator.setStorageClient(storageClient);
|
|
234
260
|
},
|
|
261
|
+
getLogger() {
|
|
262
|
+
const currentServiceLocator = serviceLocatorStorage.getStore() ?? globalServiceLocator;
|
|
263
|
+
return currentServiceLocator.getLogger();
|
|
264
|
+
},
|
|
265
|
+
setLogger(logger) {
|
|
266
|
+
const currentServiceLocator = serviceLocatorStorage.getStore() ?? globalServiceLocator;
|
|
267
|
+
currentServiceLocator.setLogger(logger);
|
|
268
|
+
},
|
|
269
|
+
getChildLog(prefix) {
|
|
270
|
+
const currentServiceLocator = serviceLocatorStorage.getStore() ?? globalServiceLocator;
|
|
271
|
+
return currentServiceLocator.getChildLog(prefix);
|
|
272
|
+
},
|
|
235
273
|
getStorageManager(constructor) {
|
|
236
274
|
const currentServiceLocator = serviceLocatorStorage.getStore() ?? globalServiceLocator;
|
|
237
275
|
return currentServiceLocator.getStorageManager(constructor);
|
package/service_locator.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"service_locator.js","sourceRoot":"","sources":["../src/service_locator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAErD,OAAO,EAAE,aAAa,EAAE,MAAM,yBAAyB,CAAC;AAGxD,OAAO,GAAG,MAAM,YAAY,CAAC;AAE7B,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAEnD,OAAO,EAAE,iBAAiB,EAAE,MAAM,iCAAiC,CAAC;
|
|
1
|
+
{"version":3,"file":"service_locator.js","sourceRoot":"","sources":["../src/service_locator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAErD,OAAO,EAAE,aAAa,EAAE,MAAM,yBAAyB,CAAC;AAGxD,OAAO,GAAG,MAAM,YAAY,CAAC;AAE7B,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAEnD,OAAO,EAAE,iBAAiB,EAAE,MAAM,iCAAiC,CAAC;AAEpE,OAAO,EAAE,eAAe,EAAE,MAAM,UAAU,CAAC;AAqF3C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AACH,iGAAiG;AACjG,2EAA2E;AAC3E,MAAM,WAAW,GAAG,IAAI,eAAe,CAAC,GAAG,CAAC,CAAC;AAE7C,MAAM,OAAO,cAAc;IACf,aAAa,CAAiB;IAC9B,YAAY,CAAgB;IAC5B,aAAa,CAAiB;IAC9B,MAAM,CAAiB;IAE/B;;;OAGG;IACK,eAAe,GAAG,IAAI,GAAG,EAAyC,CAAC;IAE3E;;;;;;;OAOG;IACH,YACI,aAA6B,EAC7B,YAA2B,EAC3B,aAA6B,EAC7B,MAAsB;QAEtB,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;QACnC,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;QACjC,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;QACnC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACzB,CAAC;IAED,gBAAgB;QACZ,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC;YACtB,CAAC,IAAI,CAAC,MAAM,IAAI,WAAW,CAAC,CAAC,KAAK,CAC9B,4EAA4E,CAC/E,CAAC;YACF,IAAI,CAAC,aAAa,GAAG,IAAI,aAAa,EAAE,CAAC;QAC7C,CAAC;QACD,OAAO,IAAI,CAAC,aAAa,CAAC;IAC9B,CAAC;IAED,gBAAgB,CAAC,aAA4B;QACzC,wCAAwC;QACxC,IAAI,IAAI,CAAC,aAAa,KAAK,aAAa,EAAE,CAAC;YACvC,OAAO;QACX,CAAC;QAED,4DAA4D;QAC5D,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YACrB,MAAM,IAAI,oBAAoB,CAAC,eAAe,EAAE,aAAa,EAAE,IAAI,CAAC,aAAa,CAAC,CAAC;QACvF,CAAC;QAED,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;IACvC,CAAC;IAED,eAAe;QACX,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;YACrB,CAAC,IAAI,CAAC,MAAM,IAAI,WAAW,CAAC,CAAC,KAAK,CAC9B,gFAAgF,CACnF,CAAC;YACF,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC;gBACtB,CAAC,IAAI,CAAC,MAAM,IAAI,WAAW,CAAC,CAAC,OAAO,CAChC,uFAAuF;oBACnF,kEAAkE,CACzE,CAAC;YACN,CAAC;YACD,IAAI,CAAC,YAAY,GAAG,iBAAiB,CAAC,UAAU,CAAC,IAAI,CAAC,gBAAgB,EAAE,CAAC,CAAC;QAC9E,CAAC;QACD,OAAO,IAAI,CAAC,YAAY,CAAC;IAC7B,CAAC;IAED,eAAe,CAAC,YAA0B;QACtC,wCAAwC;QACxC,IAAI,IAAI,CAAC,YAAY,KAAK,YAAY,EAAE,CAAC;YACrC,OAAO;QACX,CAAC;QAED,4DAA4D;QAC5D,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACpB,MAAM,IAAI,oBAAoB,CAAC,cAAc,EAAE,YAAY,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;QACpF,CAAC;QAED,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;IACrC,CAAC;IAED,gBAAgB;QACZ,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC;YACtB,CAAC,IAAI,CAAC,MAAM,IAAI,WAAW,CAAC,CAAC,KAAK,CAC9B,6EAA6E,CAChF,CAAC;YACF,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC;gBACtB,CAAC,IAAI,CAAC,MAAM,IAAI,WAAW,CAAC,CAAC,OAAO,CAChC,wFAAwF;oBACpF,kEAAkE,CACzE,CAAC;YACN,CAAC;YACD,MAAM,MAAM,GAAG,IAAI,CAAC,gBAAgB,EAAE,CAAC;YACvC,IAAI,CAAC,aAAa,GAAG,IAAI,aAAa,CAAC;gBACnC,cAAc,EAAE,MAAM,CAAC,GAAG,CAAC,gBAAgB,CAAC;aAC/C,CAAC,CAAC;QACP,CAAC;QACD,OAAO,IAAI,CAAC,aAAa,CAAC;IAC9B,CAAC;IAED,gBAAgB,CAAC,aAA4B;QACzC,wCAAwC;QACxC,IAAI,IAAI,CAAC,aAAa,KAAK,aAAa,EAAE,CAAC;YACvC,OAAO;QACX,CAAC;QAED,6DAA6D;QAC7D,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YACrB,MAAM,IAAI,oBAAoB,CAAC,eAAe,EAAE,aAAa,EAAE,IAAI,CAAC,aAAa,CAAC,CAAC;QACvF,CAAC;QAED,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;IACvC,CAAC;IAED,SAAS;QACL,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACf,IAAI,CAAC,MAAM,GAAG,IAAI,eAAe,CAAC,GAAG,CAAC,CAAC;QAC3C,CAAC;QACD,OAAO,IAAI,CAAC,MAAM,CAAC;IACvB,CAAC;IAED,SAAS,CAAC,MAAqB;QAC3B,IAAI,IAAI,CAAC,MAAM,KAAK,MAAM,EAAE,CAAC;YACzB,OAAO;QACX,CAAC;QAED,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YACd,MAAM,IAAI,oBAAoB,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QAClE,CAAC;QAED,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACzB,CAAC;IAED,WAAW,CAAC,MAAc;QACtB,OAAO,IAAI,CAAC,SAAS,EAAE,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;IAC9C,CAAC;IAED,iBAAiB,CAAC,WAAkC;QAChD,OAAO,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;IACjD,CAAC;IAED,iBAAiB,CAAC,WAAkC,EAAE,cAA8B;QAChF,IAAI,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,CAAC;YACxC,MAAM,IAAI,oBAAoB,CAC1B,kBAAkB,WAAW,CAAC,IAAI,GAAG,EACrC,cAAc,EACd,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,WAAW,CAAC,CACxC,CAAC;QACN,CAAC;QAED,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,WAAW,EAAE,cAAc,CAAC,CAAC;IAC1D,CAAC;IAED,wBAAwB;QACpB,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;YACrC,yDAAyD;YACzD,iGAAiG;YACjG,6GAA6G;YAC7G,IAAK,OAAe,CAAC,IAAI,KAAK,eAAe,EAAE,CAAC;gBAC3C,OAAe,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC,IAAS,EAAE,EAAE;oBAC1C,IAAI,CAAC,UAAU,EAAE,EAAE,CAAC;gBACxB,CAAC,CAAC,CAAC;YACP,CAAC;QACL,CAAC,CAAC,CAAC;QACH,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;IACjC,CAAC;IAED,KAAK;QACD,IAAI,CAAC,aAAa,GAAG,SAAS,CAAC;QAC/B,IAAI,CAAC,YAAY,GAAG,SAAS,CAAC;QAC9B,IAAI,CAAC,aAAa,GAAG,SAAS,CAAC;QAC/B,IAAI,CAAC,MAAM,GAAG,SAAS,CAAC;QACxB,IAAI,CAAC,wBAAwB,EAAE,CAAC;IACpC,CAAC;CACJ;AAED;;GAEG;AACH,MAAM,oBAAoB,GAAG,IAAI,cAAc,EAAE,CAAC;AAElD,MAAM,qBAAqB,GAAG,IAAI,iBAAiB,EAA2B,CAAC;AAE/E;;;;;;;;;;;;;;GAcG;AACH,MAAM,UAAU,2BAA2B,CACvC,cAA8B,EAC9B,MAAU;IAEV,IAAI,KAAK,GAAG,MAAM,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;IAE1C,OAAO,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,MAAM,CAAC,SAAS,EAAE,CAAC;QAClD,MAAM,YAAY,GAAG,CAAC,GAAG,MAAM,CAAC,mBAAmB,CAAC,KAAK,CAAC,EAAE,GAAG,MAAM,CAAC,qBAAqB,CAAC,KAAK,CAAC,CAAC,CAAC;QAEpG,KAAK,MAAM,WAAW,IAAI,YAAY,EAAE,CAAC;YACrC,MAAM,UAAU,GAAG,MAAM,CAAC,wBAAwB,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;YAEvE,kFAAkF;YAClF,sEAAsE;YACtE,uEAAuE;YACvE,IACI,WAAW,KAAK,aAAa;gBAC7B,CAAC,UAAU;gBACX,UAAU,CAAC,GAAG;gBACd,UAAU,CAAC,GAAG;gBACd,OAAO,UAAU,CAAC,KAAK,KAAK,UAAU;gBAEtC,SAAS;YAEb,MAAM,QAAQ,GAAG,UAAU,CAAC,KAAK,CAAC;YACjC,MAA2C,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,IAAW,EAAE,EAAE;gBAC3E,OAAO,qBAAqB,CAAC,GAAG,CAAC,cAAc,EAAE,GAAG,EAAE;oBAClD,OAAO,QAAQ,CAAC,KAAK,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;gBACxC,CAAC,CAAC,CAAC;YACP,CAAC,CAAC;QACN,CAAC;QAED,KAAK,GAAG,MAAM,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC;IACzC,CAAC;IAED,IAAI,aAAkD,CAAC;IAEvD,OAAO;QACH,GAAG,EAAE,CAAI,EAAW,EAAK,EAAE,CAAC,qBAAqB,CAAC,GAAG,CAAC,cAAc,EAAE,EAAE,CAAC;QACzE,UAAU,EAAE,GAAG,EAAE;YACb,aAAa,GAAG,qBAAqB,CAAC,QAAQ,EAAE,CAAC;YACjD,qBAAqB,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC;QACpD,CAAC;QACD,SAAS,EAAE,GAAG,EAAE;YACZ,qBAAqB,CAAC,SAAS,CAAC,aAAoB,CAAC,CAAC,CAAC,uFAAuF;QAClJ,CAAC;KACJ,CAAC;AACN,CAAC;AAED,MAAM,CAAC,MAAM,cAAc,GAA4B;IACnD,gBAAgB;QACZ,MAAM,qBAAqB,GAAG,qBAAqB,CAAC,QAAQ,EAAE,IAAI,oBAAoB,CAAC;QACvF,OAAO,qBAAqB,CAAC,gBAAgB,EAAE,CAAC;IACpD,CAAC;IACD,gBAAgB,CAAC,aAA4B;QACzC,MAAM,qBAAqB,GAAG,qBAAqB,CAAC,QAAQ,EAAE,IAAI,oBAAoB,CAAC;QACvF,qBAAqB,CAAC,gBAAgB,CAAC,aAAa,CAAC,CAAC;IAC1D,CAAC;IACD,eAAe;QACX,MAAM,qBAAqB,GAAG,qBAAqB,CAAC,QAAQ,EAAE,IAAI,oBAAoB,CAAC;QACvF,OAAO,qBAAqB,CAAC,eAAe,EAAE,CAAC;IACnD,CAAC;IACD,eAAe,CAAC,YAA0B;QACtC,MAAM,qBAAqB,GAAG,qBAAqB,CAAC,QAAQ,EAAE,IAAI,oBAAoB,CAAC;QACvF,qBAAqB,CAAC,eAAe,CAAC,YAAY,CAAC,CAAC;IACxD,CAAC;IACD,gBAAgB;QACZ,MAAM,qBAAqB,GAAG,qBAAqB,CAAC,QAAQ,EAAE,IAAI,oBAAoB,CAAC;QACvF,OAAO,qBAAqB,CAAC,gBAAgB,EAAE,CAAC;IACpD,CAAC;IACD,gBAAgB,CAAC,aAA4B;QACzC,MAAM,qBAAqB,GAAG,qBAAqB,CAAC,QAAQ,EAAE,IAAI,oBAAoB,CAAC;QACvF,qBAAqB,CAAC,gBAAgB,CAAC,aAAa,CAAC,CAAC;IAC1D,CAAC;IACD,SAAS;QACL,MAAM,qBAAqB,GAAG,qBAAqB,CAAC,QAAQ,EAAE,IAAI,oBAAoB,CAAC;QACvF,OAAO,qBAAqB,CAAC,SAAS,EAAE,CAAC;IAC7C,CAAC;IACD,SAAS,CAAC,MAAqB;QAC3B,MAAM,qBAAqB,GAAG,qBAAqB,CAAC,QAAQ,EAAE,IAAI,oBAAoB,CAAC;QACvF,qBAAqB,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;IAC5C,CAAC;IACD,WAAW,CAAC,MAAc;QACtB,MAAM,qBAAqB,GAAG,qBAAqB,CAAC,QAAQ,EAAE,IAAI,oBAAoB,CAAC;QACvF,OAAO,qBAAqB,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;IACrD,CAAC;IACD,iBAAiB,CAAC,WAAkC;QAChD,MAAM,qBAAqB,GAAG,qBAAqB,CAAC,QAAQ,EAAE,IAAI,oBAAoB,CAAC;QACvF,OAAO,qBAAqB,CAAC,iBAAiB,CAAC,WAAW,CAAC,CAAC;IAChE,CAAC;IACD,iBAAiB,CAAC,WAAkC,EAAE,cAA8B;QAChF,MAAM,qBAAqB,GAAG,qBAAqB,CAAC,QAAQ,EAAE,IAAI,oBAAoB,CAAC;QACvF,qBAAqB,CAAC,iBAAiB,CAAC,WAAW,EAAE,cAAc,CAAC,CAAC;IACzE,CAAC;IACD,wBAAwB;QACpB,MAAM,qBAAqB,GAAG,qBAAqB,CAAC,QAAQ,EAAE,IAAI,oBAAoB,CAAC;QACvF,qBAAqB,CAAC,wBAAwB,EAAE,CAAC;IACrD,CAAC;IACD,KAAK;QACD,MAAM,qBAAqB,GAAG,qBAAqB,CAAC,QAAQ,EAAE,IAAI,oBAAoB,CAAC;QACvF,qBAAqB,CAAC,KAAK,EAAE,CAAC;IAClC,CAAC;CACJ,CAAC"}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import type { Cookie as CookieObject, Dictionary, ISession, ProxyInfo, SessionState } from '@crawlee/types';
|
|
2
2
|
import type { Cookie } from 'tough-cookie';
|
|
3
3
|
import { CookieJar } from 'tough-cookie';
|
|
4
|
-
import type {
|
|
4
|
+
import type { CrawleeLogger } from '../log.js';
|
|
5
5
|
export interface SessionOptions {
|
|
6
6
|
/** Id of session used for generating fingerprints. It is used as proxy session name. */
|
|
7
7
|
id?: string;
|
|
@@ -44,7 +44,7 @@ export interface SessionOptions {
|
|
|
44
44
|
maxUsageCount?: number;
|
|
45
45
|
/** SessionPool instance. Session will emit the `sessionRetired` event on this instance. */
|
|
46
46
|
sessionPool?: import('./session_pool.js').SessionPool;
|
|
47
|
-
log?:
|
|
47
|
+
log?: CrawleeLogger;
|
|
48
48
|
errorScore?: number;
|
|
49
49
|
cookieJar?: CookieJar;
|
|
50
50
|
proxyInfo?: ProxyInfo;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session.d.ts","sourceRoot":"","sources":["../../src/session_pool/session.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,MAAM,IAAI,YAAY,EAAE,UAAU,EAAE,QAAQ,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAE5G,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAC3C,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;
|
|
1
|
+
{"version":3,"file":"session.d.ts","sourceRoot":"","sources":["../../src/session_pool/session.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,MAAM,IAAI,YAAY,EAAE,UAAU,EAAE,QAAQ,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAE5G,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAC3C,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAUzC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAI/C,MAAM,WAAW,cAAc;IAC3B,wFAAwF;IACxF,EAAE,CAAC,EAAE,MAAM,CAAC;IAEZ;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB,+EAA+E;IAC/E,QAAQ,CAAC,EAAE,UAAU,CAAC;IAEtB;;;;;;OAMG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IAEvB;;;;;OAKG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAE7B,wBAAwB;IACxB,SAAS,CAAC,EAAE,IAAI,CAAC;IAEjB,0BAA0B;IAC1B,SAAS,CAAC,EAAE,IAAI,CAAC;IAEjB;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB;;;;OAIG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IAEvB,2FAA2F;IAC3F,WAAW,CAAC,EAAE,OAAO,mBAAmB,EAAE,WAAW,CAAC;IAEtD,GAAG,CAAC,EAAE,aAAa,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,SAAS,CAAC;IACtB,SAAS,CAAC,EAAE,SAAS,CAAC;CACzB;AAED;;;;;GAKG;AACH,qBAAa,OAAQ,YAAW,QAAQ;IACpC,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,UAAU,CAAS;IAC3B,QAAQ,EAAE,UAAU,CAAC;IACrB,OAAO,CAAC,cAAc,CAAS;IAC/B,OAAO,CAAC,oBAAoB,CAAS;IACrC,OAAO,CAAC,UAAU,CAAO;IACzB,OAAO,CAAC,UAAU,CAAO;IACzB,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,cAAc,CAAS;IAC/B,OAAO,CAAC,WAAW,CAA0C;IAC7D,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,UAAU,CAAC,CAAY;IAC/B,OAAO,CAAC,UAAU,CAAY;IAC9B,OAAO,CAAC,GAAG,CAAgB;IAE3B,IAAI,UAAU,WAEb;IAED,IAAI,UAAU,WAEb;IAED,IAAI,aAAa,WAEhB;IAED,IAAI,mBAAmB,WAEtB;IAED,IAAI,SAAS,SAEZ;IAED,IAAI,SAAS,SAEZ;IAED,IAAI,aAAa,WAEhB;IAED,IAAI,SAAS,cAEZ;IAED,IAAI,SAAS,0BAEZ;IAED;;OAEG;gBACS,OAAO,EAAE,cAAc;IA0DnC;;;OAGG;IACH,SAAS,IAAI,OAAO;IAIpB;;;;OAIG;IACH,SAAS,IAAI,OAAO;IAIpB;;;OAGG;IACH,sBAAsB,IAAI,OAAO;IAIjC;;;OAGG;IACH,QAAQ,IAAI,OAAO;IAInB;;;OAGG;IACH,QAAQ;IAUR;;;OAGG;IACH,QAAQ,IAAI,YAAY;IAgBxB;;;;;;OAMG;IACH,MAAM;IASN;;;OAGG;IACH,OAAO;IAOP;;;;;;;OAOG;IACH,0BAA0B,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO;IASvD;;;;;;OAMG;IACH,sBAAsB,CAAC,QAAQ,EAAE,QAAQ;IAWzC;;;;;;;;;;;;OAYG;IACH,UAAU,CAAC,OAAO,EAAE,YAAY,EAAE,EAAE,GAAG,EAAE,MAAM;IAK/C;;;OAGG;IACH,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG,YAAY,EAAE;IAKvC;;;;;OAKG;IACH,eAAe,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM;IAIpC;;OAEG;IACH,SAAS,CAAC,SAAS,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,IAAI;IAQ/C;;OAEG;IACH,SAAS,CAAC,WAAW,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,MAAM,GAAG,IAAI;IAkB3D;;OAEG;IACH,SAAS,CAAC,gBAAgB,IAAI,IAAI;CAKrC"}
|
package/session_pool/session.js
CHANGED
|
@@ -3,7 +3,7 @@ import ow from 'ow';
|
|
|
3
3
|
import { CookieJar } from 'tough-cookie';
|
|
4
4
|
import { cryptoRandomObjectId } from '@apify/utilities';
|
|
5
5
|
import { browserPoolCookieToToughCookie, getCookiesFromResponse, getDefaultCookieExpirationDate, toughCookieToBrowserPoolCookie, } from '../cookie_utils.js';
|
|
6
|
-
import {
|
|
6
|
+
import { serviceLocator } from '../service_locator.js';
|
|
7
7
|
import { EVENT_SESSION_RETIRED } from './events.js';
|
|
8
8
|
/**
|
|
9
9
|
* Sessions are used to store information such as cookies and can be used for generating fingerprints and proxy sessions.
|
|
@@ -73,7 +73,7 @@ export class Session {
|
|
|
73
73
|
maxUsageCount: ow.optional.number,
|
|
74
74
|
log: ow.optional.object,
|
|
75
75
|
}));
|
|
76
|
-
const { sessionPool, id = `session_${cryptoRandomObjectId(10)}`, cookieJar = new CookieJar(), proxyInfo = undefined, maxAgeSecs = 3000, userData = {}, maxErrorScore = 3, errorScoreDecrement = 0.5, createdAt = new Date(), usageCount = 0, errorScore = 0, maxUsageCount = 50, log =
|
|
76
|
+
const { sessionPool, id = `session_${cryptoRandomObjectId(10)}`, cookieJar = new CookieJar(), proxyInfo = undefined, maxAgeSecs = 3000, userData = {}, maxErrorScore = 3, errorScoreDecrement = 0.5, createdAt = new Date(), usageCount = 0, errorScore = 0, maxUsageCount = 50, log = serviceLocator.getLogger(), } = options;
|
|
77
77
|
const { expiresAt = getDefaultCookieExpirationDate(maxAgeSecs) } = options;
|
|
78
78
|
this.log = log.child({ prefix: 'Session' });
|
|
79
79
|
this._cookieJar = cookieJar.setCookie ? cookieJar : CookieJar.fromJSON(JSON.stringify(cookieJar));
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session.js","sourceRoot":"","sources":["../../src/session_pool/session.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAG3C,OAAO,EAAE,MAAM,IAAI,CAAC;AAEpB,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;
|
|
1
|
+
{"version":3,"file":"session.js","sourceRoot":"","sources":["../../src/session_pool/session.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAG3C,OAAO,EAAE,MAAM,IAAI,CAAC;AAEpB,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAEzC,OAAO,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AAExD,OAAO,EACH,8BAA8B,EAC9B,sBAAsB,EACtB,8BAA8B,EAC9B,8BAA8B,GACjC,MAAM,oBAAoB,CAAC;AAE5B,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvD,OAAO,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;AA4DpD;;;;;GAKG;AACH,MAAM,OAAO,OAAO;IACP,EAAE,CAAS;IACZ,UAAU,CAAS;IAC3B,QAAQ,CAAa;IACb,cAAc,CAAS;IACvB,oBAAoB,CAAS;IAC7B,UAAU,CAAO;IACjB,UAAU,CAAO;IACjB,WAAW,CAAS;IACpB,cAAc,CAAS;IACvB,WAAW,CAA0C;IACrD,WAAW,CAAS;IACpB,UAAU,CAAa;IACvB,UAAU,CAAY;IACtB,GAAG,CAAgB;IAE3B,IAAI,UAAU;QACV,OAAO,IAAI,CAAC,WAAW,CAAC;IAC5B,CAAC;IAED,IAAI,UAAU;QACV,OAAO,IAAI,CAAC,WAAW,CAAC;IAC5B,CAAC;IAED,IAAI,aAAa;QACb,OAAO,IAAI,CAAC,cAAc,CAAC;IAC/B,CAAC;IAED,IAAI,mBAAmB;QACnB,OAAO,IAAI,CAAC,oBAAoB,CAAC;IACrC,CAAC;IAED,IAAI,SAAS;QACT,OAAO,IAAI,CAAC,UAAU,CAAC;IAC3B,CAAC;IAED,IAAI,SAAS;QACT,OAAO,IAAI,CAAC,UAAU,CAAC;IAC3B,CAAC;IAED,IAAI,aAAa;QACb,OAAO,IAAI,CAAC,cAAc,CAAC;IAC/B,CAAC;IAED,IAAI,SAAS;QACT,OAAO,IAAI,CAAC,UAAU,CAAC;IAC3B,CAAC;IAED,IAAI,SAAS;QACT,OAAO,IAAI,CAAC,UAAU,CAAC;IAC3B,CAAC;IAED;;OAEG;IACH,YAAY,OAAuB;QAC/B,EAAE,CACE,OAAO,EACP,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YACjB,WAAW,EAAE,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,YAAY,CAAC;YAC/C,EAAE,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACtB,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YAC7B,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YAC7B,UAAU,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YAC9B,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YAC5B,aAAa,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACjC,mBAAmB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACvC,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI;YAC3B,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI;YAC3B,UAAU,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YAC9B,UAAU,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YAC9B,aAAa,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACjC,GAAG,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;SAC1B,CAAC,CACL,CAAC;QAEF,MAAM,EACF,WAAW,EACX,EAAE,GAAG,WAAW,oBAAoB,CAAC,EAAE,CAAC,EAAE,EAC1C,SAAS,GAAG,IAAI,SAAS,EAAE,EAC3B,SAAS,GAAG,SAAS,EACrB,UAAU,GAAG,IAAI,EACjB,QAAQ,GAAG,EAAE,EACb,aAAa,GAAG,CAAC,EACjB,mBAAmB,GAAG,GAAG,EACzB,SAAS,GAAG,IAAI,IAAI,EAAE,EACtB,UAAU,GAAG,CAAC,EACd,UAAU,GAAG,CAAC,EACd,aAAa,GAAG,EAAE,EAClB,GAAG,GAAG,cAAc,CAAC,SAAS,EAAE,GACnC,GAAG,OAAO,CAAC;QAEZ,MAAM,EAAE,SAAS,GAAG,8BAA8B,CAAC,UAAU,CAAC,EAAE,GAAG,OAAO,CAAC;QAE3E,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;QAE5C,IAAI,CAAC,UAAU,GAAI,SAAS,CAAC,SAAqB,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC;QAC/G,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;QAC5B,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,cAAc,GAAG,aAAa,CAAC;QACpC,IAAI,CAAC,oBAAoB,GAAG,mBAAmB,CAAC;QAEhD,WAAW;QACX,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;QAC5B,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;QAC5B,IAAI,CAAC,WAAW,GAAG,UAAU,CAAC,CAAC,qDAAqD;QACpF,IAAI,CAAC,WAAW,GAAG,UAAU,CAAC,CAAC,yDAAyD;QACxF,IAAI,CAAC,cAAc,GAAG,aAAa,CAAC;QACpC,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;IACnC,CAAC;IAED;;;OAGG;IACH,SAAS;QACL,OAAO,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,aAAa,CAAC;IACjD,CAAC;IAED;;;;OAIG;IACH,SAAS;QACL,OAAO,IAAI,CAAC,SAAS,IAAI,IAAI,IAAI,EAAE,CAAC;IACxC,CAAC;IAED;;;OAGG;IACH,sBAAsB;QAClB,OAAO,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,aAAa,CAAC;IACjD,CAAC;IAED;;;OAGG;IACH,QAAQ;QACJ,OAAO,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,sBAAsB,EAAE,CAAC;IACpF,CAAC;IAED;;;OAGG;IACH,QAAQ;QACJ,IAAI,CAAC,WAAW,IAAI,CAAC,CAAC;QAEtB,IAAI,IAAI,CAAC,WAAW,GAAG,CAAC,EAAE,CAAC;YACvB,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,oBAAoB,CAAC;QAClD,CAAC;QAED,IAAI,CAAC,gBAAgB,EAAE,CAAC;IAC5B,CAAC;IAED;;;OAGG;IACH,QAAQ;QACJ,OAAO;YACH,EAAE,EAAE,IAAI,CAAC,EAAE;YACX,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAG;YACnC,SAAS,EAAE,IAAI,CAAC,UAAU;YAC1B,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,aAAa,EAAE,IAAI,CAAC,aAAa;YACjC,mBAAmB,EAAE,IAAI,CAAC,mBAAmB;YAC7C,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE;YACvC,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE;YACvC,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,aAAa,EAAE,IAAI,CAAC,aAAa;YACjC,UAAU,EAAE,IAAI,CAAC,UAAU;SAC9B,CAAC;IACN,CAAC;IAED;;;;;;OAMG;IACH,MAAM;QACF,6DAA6D;QAC7D,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,cAAc,CAAC;QACxC,IAAI,CAAC,WAAW,IAAI,CAAC,CAAC;QAEtB,wDAAwD;QACxD,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,qBAAqB,EAAE,IAAI,CAAC,CAAC;IACvD,CAAC;IAED;;;OAGG;IACH,OAAO;QACH,IAAI,CAAC,WAAW,IAAI,CAAC,CAAC;QACtB,IAAI,CAAC,WAAW,IAAI,CAAC,CAAC;QAEtB,IAAI,CAAC,gBAAgB,EAAE,CAAC;IAC5B,CAAC;IAED;;;;;;;OAOG;IACH,0BAA0B,CAAC,UAAkB;QACzC,sEAAsE;QACtE,MAAM,SAAS,GAAG,IAAI,CAAC,WAAW,CAAC,oBAAoB,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;QAC9E,IAAI,SAAS,EAAE,CAAC;YACZ,IAAI,CAAC,MAAM,EAAE,CAAC;QAClB,CAAC;QACD,OAAO,SAAS,CAAC;IACrB,CAAC;IAED;;;;;;OAMG;IACH,sBAAsB,CAAC,QAAkB;QACrC,IAAI,CAAC;YACD,MAAM,OAAO,GAAG,sBAAsB,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;YAClE,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC;QAC5C,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,MAAM,GAAG,GAAG,CAAU,CAAC;YACvB,+DAA+D;YAC/D,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,qCAAqC,CAAC,CAAC;QACnE,CAAC;IACL,CAAC;IAED;;;;;;;;;;;;OAYG;IACH,UAAU,CAAC,OAAuB,EAAE,GAAW;QAC3C,MAAM,iBAAiB,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,8BAA8B,CAAC,CAAC,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC;QACjG,IAAI,CAAC,WAAW,CAAC,iBAAiB,EAAE,GAAG,CAAC,CAAC;IAC7C,CAAC;IAED;;;OAGG;IACH,UAAU,CAAC,GAAW;QAClB,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;QACnD,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,8BAA8B,CAAC,CAAC,CAAC,CAAC,CAAC;IACjE,CAAC;IAED;;;;;OAKG;IACH,eAAe,CAAC,GAAW;QACvB,OAAO,IAAI,CAAC,SAAS,CAAC,mBAAmB,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;IACvD,CAAC;IAED;;OAEG;IACH,SAAS,CAAC,SAAiB,EAAE,GAAW;QACpC,IAAI,CAAC;YACD,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;QACjD,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,uBAAuB,EAAE,EAAE,GAAG,EAAE,KAAK,EAAG,CAAW,CAAC,OAAO,EAAE,CAAC,CAAC;QACpF,CAAC;IACL,CAAC;IAED;;OAEG;IACO,WAAW,CAAC,OAAiB,EAAE,GAAW;QAChD,MAAM,aAAa,GAAa,EAAE,CAAC;QAEnC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC3B,IAAI,CAAC;gBACD,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,MAAM,EAAE,GAAG,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC,CAAC;YACtE,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACT,MAAM,GAAG,GAAG,CAAU,CAAC;gBACvB,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YACpC,CAAC;QACL,CAAC;QAED,sGAAsG;QACtG,IAAI,aAAa,CAAC,MAAM,EAAE,CAAC;YACvB,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,wBAAwB,EAAE,EAAE,aAAa,EAAE,CAAC,CAAC;QAClE,CAAC;IACL,CAAC;IAED;;OAEG;IACO,gBAAgB;QACtB,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,CAAC;YACnB,IAAI,CAAC,MAAM,EAAE,CAAC;QAClB,CAAC;IACL,CAAC;CACJ"}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { EventEmitter } from 'node:events';
|
|
2
|
-
import type { Log } from '@apify/log';
|
|
3
2
|
import type { PersistenceOptions } from '../crawlers/statistics.js';
|
|
4
3
|
import type { EventManager } from '../events/event_manager.js';
|
|
4
|
+
import type { CrawleeLogger } from '../log.js';
|
|
5
5
|
import { KeyValueStore } from '../storages/key_value_store.js';
|
|
6
6
|
import type { SessionOptions } from './session.js';
|
|
7
7
|
import { Session } from './session.js';
|
|
@@ -45,7 +45,7 @@ export interface SessionPoolOptions {
|
|
|
45
45
|
*/
|
|
46
46
|
blockedStatusCodes?: number[];
|
|
47
47
|
/** @internal */
|
|
48
|
-
log?:
|
|
48
|
+
log?: CrawleeLogger;
|
|
49
49
|
/**
|
|
50
50
|
* Control how and when to persist the state of the session pool.
|
|
51
51
|
*/
|
|
@@ -117,7 +117,7 @@ export interface SessionPoolOptions {
|
|
|
117
117
|
* @category Scaling
|
|
118
118
|
*/
|
|
119
119
|
export declare class SessionPool extends EventEmitter {
|
|
120
|
-
protected log:
|
|
120
|
+
protected log: CrawleeLogger;
|
|
121
121
|
protected maxPoolSize: number;
|
|
122
122
|
protected createSessionFunction: CreateSession;
|
|
123
123
|
protected keyValueStore: KeyValueStore;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session_pool.d.ts","sourceRoot":"","sources":["../../src/session_pool/session_pool.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAM3C,OAAO,KAAK,EAAE,
|
|
1
|
+
{"version":3,"file":"session_pool.d.ts","sourceRoot":"","sources":["../../src/session_pool/session_pool.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAM3C,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,2BAA2B,CAAC;AACpE,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAE/D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAE/C,OAAO,EAAE,aAAa,EAAE,MAAM,gCAAgC,CAAC;AAE/D,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AACnD,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAEvC;;GAEG;AACH,MAAM,WAAW,aAAa;IAC1B;;;OAGG;IACH,CAAC,WAAW,EAAE,WAAW,EAAE,OAAO,CAAC,EAAE;QAAE,cAAc,CAAC,EAAE,cAAc,CAAA;KAAE,GAAG,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;CACzG;AAED,MAAM,WAAW,kBAAkB;IAC/B;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB,kEAAkE;IAClE,cAAc,CAAC,EAAE,cAAc,CAAC;IAEhC,6EAA6E;IAC7E,2BAA2B,CAAC,EAAE,MAAM,CAAC;IAErC;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IAEzB;;;;OAIG;IACH,qBAAqB,CAAC,EAAE,aAAa,CAAC;IAEtC;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,MAAM,EAAE,CAAC;IAE9B,gBAAgB;IAChB,GAAG,CAAC,EAAE,aAAa,CAAC;IAEpB;;OAEG;IACH,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;CAC3C;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgEG;AACH,qBAAa,WAAY,SAAQ,YAAY;IACzC,SAAS,CAAC,GAAG,EAAE,aAAa,CAAC;IAC7B,SAAS,CAAC,WAAW,EAAE,MAAM,CAAC;IAC9B,SAAS,CAAC,qBAAqB,EAAE,aAAa,CAAC;IAC/C,SAAS,CAAC,aAAa,EAAG,aAAa,CAAC;IACxC,SAAS,CAAC,QAAQ,EAAE,OAAO,EAAE,CAAM;IACnC,SAAS,CAAC,UAAU,uBAA8B;IAClD,SAAS,CAAC,cAAc,EAAE,cAAc,CAAC;IACzC,SAAS,CAAC,2BAA2B,CAAC,EAAE,MAAM,CAAC;IAC/C,SAAS,CAAC,eAAe,EAAE,MAAM,CAAC;IAClC,SAAS,CAAC,SAAS,EAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAC1C,SAAS,CAAC,MAAM,EAAE,YAAY,CAAC;IAC/B,SAAS,CAAC,QAAQ,CAAC,kBAAkB,EAAE,MAAM,EAAE,CAAC;IAChD,SAAS,CAAC,kBAAkB,EAAE,kBAAkB,CAAC;IACjD,SAAS,CAAC,aAAa,UAAS;IAEhC,OAAO,CAAC,KAAK,CAAoB;IAEjC;;OAEG;gBACS,OAAO,GAAE,kBAAuB;IAoD5C;;OAEG;IACH,IAAI,mBAAmB,IAAI,MAAM,CAEhC;IAED;;OAEG;IACH,IAAI,oBAAoB,IAAI,MAAM,CAEjC;IAED;;;OAGG;IACG,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IA4BjC;;;;;OAKG;IACG,UAAU,CAAC,OAAO,GAAE,OAAO,GAAG,cAAmB,GAAG,OAAO,CAAC,IAAI,CAAC;IAqBvE;;;;;OAKG;IACG,UAAU,CAAC,cAAc,CAAC,EAAE,cAAc,GAAG,OAAO,CAAC,OAAO,CAAC;IASnE;;;;;OAKG;IACG,UAAU,IAAI,OAAO,CAAC,OAAO,CAAC;IAEpC;;OAEG;IACG,UAAU,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAqCrD;;OAEG;IACG,UAAU,CAAC,OAAO,CAAC,EAAE,kBAAkB;IAQ7C;;;OAGG;IACH,QAAQ;;;;;IAQR;;;;OAIG;IACG,YAAY,CAAC,OAAO,CAAC,EAAE,kBAAkB,GAAG,OAAO,CAAC,IAAI,CAAC;IAuB/D;;;OAGG;IACG,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAK/B;;OAEG;IACH,SAAS,CAAC,sBAAsB;IAIhC;;OAEG;IACH,SAAS,CAAC,sBAAsB;IAWhC;;;OAGG;IACH,SAAS,CAAC,WAAW,CAAC,UAAU,EAAE,OAAO;IAKzC;;OAEG;IACH,SAAS,CAAC,eAAe,IAAI,MAAM;IAInC;;;;;;OAMG;cACa,6BAA6B,CACzC,WAAW,EAAE,WAAW,EACxB,OAAO,GAAE;QAAE,cAAc,CAAC,EAAE,cAAc,CAAA;KAAO,GAClD,OAAO,CAAC,OAAO,CAAC;IAWnB;;;OAGG;cACa,cAAc,IAAI,OAAO,CAAC,OAAO,CAAC;IAQlD;;OAEG;IACH,SAAS,CAAC,mBAAmB,IAAI,OAAO;IAIxC;;;OAGG;IACH,SAAS,CAAC,YAAY,IAAI,OAAO;IAIjC;;;OAGG;cACa,qBAAqB,IAAI,OAAO,CAAC,IAAI,CAAC;IAyBtD;;;;;OAKG;WACU,IAAI,CAAC,OAAO,CAAC,EAAE,kBAAkB,GAAG,OAAO,CAAC,WAAW,CAAC;CAKxE"}
|
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import { EventEmitter } from 'node:events';
|
|
2
2
|
import { AsyncQueue } from '@sapphire/async-queue';
|
|
3
3
|
import ow from 'ow';
|
|
4
|
-
import { log as defaultLog } from '../log.js';
|
|
5
4
|
import { serviceLocator } from '../service_locator.js';
|
|
6
5
|
import { KeyValueStore } from '../storages/key_value_store.js';
|
|
7
6
|
import { BLOCKED_STATUS_CODES, MAX_POOL_SIZE, PERSIST_STATE_KEY } from './consts.js';
|
|
@@ -102,7 +101,7 @@ export class SessionPool extends EventEmitter {
|
|
|
102
101
|
log: ow.optional.object,
|
|
103
102
|
persistenceOptions: ow.optional.object,
|
|
104
103
|
}));
|
|
105
|
-
const { maxPoolSize = MAX_POOL_SIZE, persistStateKeyValueStoreId, persistStateKey = PERSIST_STATE_KEY, createSessionFunction, sessionOptions = {}, blockedStatusCodes = BLOCKED_STATUS_CODES, log =
|
|
104
|
+
const { maxPoolSize = MAX_POOL_SIZE, persistStateKeyValueStoreId, persistStateKey = PERSIST_STATE_KEY, createSessionFunction, sessionOptions = {}, blockedStatusCodes = BLOCKED_STATUS_CODES, log = serviceLocator.getLogger(), persistenceOptions = {
|
|
106
105
|
enable: true,
|
|
107
106
|
}, } = options;
|
|
108
107
|
this.blockedStatusCodes = blockedStatusCodes;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session_pool.js","sourceRoot":"","sources":["../../src/session_pool/session_pool.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAG3C,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AACnD,OAAO,EAAE,MAAM,IAAI,CAAC;
|
|
1
|
+
{"version":3,"file":"session_pool.js","sourceRoot":"","sources":["../../src/session_pool/session_pool.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAG3C,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AACnD,OAAO,EAAE,MAAM,IAAI,CAAC;AAMpB,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvD,OAAO,EAAE,aAAa,EAAE,MAAM,gCAAgC,CAAC;AAC/D,OAAO,EAAE,oBAAoB,EAAE,aAAa,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAErF,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAuDvC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgEG;AACH,MAAM,OAAO,WAAY,SAAQ,YAAY;IAC/B,GAAG,CAAgB;IACnB,WAAW,CAAS;IACpB,qBAAqB,CAAgB;IACrC,aAAa,CAAiB;IAC9B,QAAQ,GAAc,EAAE,CAAC;IACzB,UAAU,GAAG,IAAI,GAAG,EAAmB,CAAC;IACxC,cAAc,CAAiB;IAC/B,2BAA2B,CAAU;IACrC,eAAe,CAAS;IACxB,SAAS,CAAuB;IAChC,MAAM,CAAe;IACZ,kBAAkB,CAAW;IACtC,kBAAkB,CAAqB;IACvC,aAAa,GAAG,KAAK,CAAC;IAExB,KAAK,GAAG,IAAI,UAAU,EAAE,CAAC;IAEjC;;OAEG;IACH,YAAY,UAA8B,EAAE;QACxC,KAAK,EAAE,CAAC;QAER,EAAE,CACE,OAAO,EACP,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YACjB,WAAW,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YAC/B,2BAA2B,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YAC/C,eAAe,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACnC,qBAAqB,EAAE,EAAE,CAAC,QAAQ,CAAC,QAAQ;YAC3C,cAAc,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YAClC,kBAAkB,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC;YACvD,GAAG,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACvB,kBAAkB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;SACzC,CAAC,CACL,CAAC;QAEF,MAAM,EACF,WAAW,GAAG,aAAa,EAC3B,2BAA2B,EAC3B,eAAe,GAAG,iBAAiB,EACnC,qBAAqB,EACrB,cAAc,GAAG,EAAE,EACnB,kBAAkB,GAAG,oBAAoB,EACzC,GAAG,GAAG,cAAc,CAAC,SAAS,EAAE,EAChC,kBAAkB,GAAG;YACjB,MAAM,EAAE,IAAI;SACf,GACJ,GAAG,OAAO,CAAC;QAEZ,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C,IAAI,CAAC,MAAM,GAAG,cAAc,CAAC,eAAe,EAAE,CAAC;QAC/C,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,aAAa,EAAE,CAAC,CAAC;QAChD,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAE7C,qBAAqB;QACrB,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAC/B,IAAI,CAAC,qBAAqB,GAAG,qBAAqB,IAAI,IAAI,CAAC,6BAA6B,CAAC;QAEzF,wBAAwB;QACxB,IAAI,CAAC,cAAc,GAAG;YAClB,GAAG,cAAc;YACjB,6GAA6G;YAC7G,oDAAoD;YACpD,GAAG,EAAE,IAAI,CAAC,GAAG;SAChB,CAAC;QAEF,wBAAwB;QACxB,IAAI,CAAC,2BAA2B,GAAG,2BAA2B,CAAC;QAC/D,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IAC3C,CAAC;IAED;;OAEG;IACH,IAAI,mBAAmB;QACnB,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC;IACxE,CAAC;IAED;;OAEG;IACH,IAAI,oBAAoB;QACpB,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC;IACzE,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,UAAU;QACZ,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YACrB,OAAO;QACX,CAAC;QAED,IAAI,CAAC,aAAa,GAAG,MAAM,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,2BAA2B,EAAE;YAC5E,MAAM,EAAE,cAAc,CAAC,gBAAgB,EAAE;SAC5C,CAAC,CAAC;QACH,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,MAAM,EAAE,CAAC;YAClC,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC;YAC1B,OAAO;QACX,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,2BAA2B,EAAE,CAAC;YACpC,IAAI,CAAC,GAAG,CAAC,KAAK,CACV,iIAAiI,IAAI,CAAC,aAAa,CAAC,EAAE,EAAE,CAC3J,CAAC;QACN,CAAC;QAED,iGAAiG;QACjG,MAAM,IAAI,CAAC,qBAAqB,EAAE,CAAC;QAEnC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAE9C,IAAI,CAAC,MAAM,CAAC,EAAE,+CAA0B,IAAI,CAAC,SAAS,CAAC,CAAC;QACxD,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC;IAC9B,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,UAAU,CAAC,UAAoC,EAAE;QACnD,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAC9B,MAAM,EAAE,EAAE,EAAE,GAAG,OAAO,CAAC;QACvB,IAAI,EAAE,EAAE,CAAC;YACL,MAAM,aAAa,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAC9C,IAAI,aAAa,EAAE,CAAC;gBAChB,MAAM,IAAI,KAAK,CAAC,+BAA+B,EAAE,oCAAoC,CAAC,CAAC;YAC3F,CAAC;QACL,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,mBAAmB,EAAE,EAAE,CAAC;YAC9B,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAClC,CAAC;QAED,MAAM,UAAU,GACZ,OAAO,YAAY,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,qBAAqB,CAAC,IAAI,EAAE,EAAE,cAAc,EAAE,OAAO,EAAE,CAAC,CAAC;QAC/G,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,wBAAwB,UAAU,CAAC,EAAE,EAAE,CAAC,CAAC;QAExD,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;IACjC,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,UAAU,CAAC,cAA+B;QAC5C,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAE9B,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,qBAAqB,CAAC,IAAI,EAAE,EAAE,cAAc,EAAE,CAAC,CAAC;QAC9E,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;QAE7B,OAAO,UAAU,CAAC;IACtB,CAAC;IAeD;;;;;;OAMG;IACH,KAAK,CAAC,UAAU,CAAC,SAAkB;QAC/B,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;QAExB,IAAI,CAAC;YACD,IAAI,CAAC,sBAAsB,EAAE,CAAC;YAE9B,IAAI,SAAS,EAAE,CAAC;gBACZ,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;gBAC/C,IAAI,OAAO,IAAI,OAAO,CAAC,QAAQ,EAAE;oBAAE,OAAO,OAAO,CAAC;gBAClD,OAAO,SAAS,CAAC;YACrB,CAAC;YAED,IAAI,IAAI,CAAC,mBAAmB,EAAE,EAAE,CAAC;gBAC7B,OAAO,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;YACvC,CAAC;YAED,MAAM,aAAa,GAAG,IAAI,CAAC,YAAY,EAAE,CAAC;YAC1C,IAAI,aAAa,CAAC,QAAQ,EAAE,EAAE,CAAC;gBAC3B,OAAO,aAAa,CAAC;YACzB,CAAC;YAED,IAAI,CAAC,sBAAsB,EAAE,CAAC;YAC9B,OAAO,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;QACvC,CAAC;gBAAS,CAAC;YACP,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QACvB,CAAC;IACL,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU,CAAC,OAA4B;QACzC,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,MAAM,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;YACtD,OAAO;QACX,CAAC;QAED,MAAM,IAAI,CAAC,aAAa,EAAE,QAAQ,CAAC,IAAI,CAAC,eAAe,EAAE,IAAI,CAAC,CAAC;IACnE,CAAC;IAED;;;OAGG;IACH,QAAQ;QACJ,OAAO;YACH,mBAAmB,EAAE,IAAI,CAAC,mBAAmB;YAC7C,oBAAoB,EAAE,IAAI,CAAC,oBAAoB;YAC/C,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC;SAC/D,CAAC;IACN,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,YAAY,CAAC,OAA4B;QAC3C,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,MAAM,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;YACtD,OAAO;QACX,CAAC;QAED,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,kBAAkB,EAAE;YAC/B,2BAA2B,EAAE,IAAI,CAAC,2BAA2B;YAC7D,eAAe,EAAE,IAAI,CAAC,eAAe;SACxC,CAAC,CAAC;QAEH,mEAAmE;QACnE,MAAM,0BAA0B,GAAG,cAAc,CAAC,gBAAgB,EAAE,CAAC,GAAG,CAAC,4BAA4B,CAAE,CAAC;QACxG,MAAM,WAAW,GAAG,0BAA0B,GAAG,KAAK,CAAC;QACvD,MAAM,IAAI,CAAC,aAAa;aACnB,QAAQ,CAAC,IAAI,CAAC,eAAe,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE;YAC7C,WAAW;YACX,kBAAkB,EAAE,IAAI;SAC3B,CAAC;aACD,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CACb,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,+CAA+C,IAAI,CAAC,eAAe,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,CACrG,CAAC;IACV,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,QAAQ;QACV,IAAI,CAAC,MAAM,CAAC,GAAG,+CAA0B,IAAI,CAAC,SAAS,CAAC,CAAC;QACzD,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;IAC9B,CAAC;IAED;;OAEG;IACO,sBAAsB;QAC5B,IAAI,CAAC,IAAI,CAAC,aAAa;YAAE,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC;IAChF,CAAC;IAED;;OAEG;IACO,sBAAsB;QAC5B,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,aAAa,EAAE,EAAE;YACnD,IAAI,aAAa,CAAC,QAAQ,EAAE;gBAAE,OAAO,IAAI,CAAC;YAE1C,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC;YACzC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,qBAAqB,aAAa,CAAC,EAAE,EAAE,CAAC,CAAC;YAExD,OAAO,KAAK,CAAC;QACjB,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;;OAGG;IACO,WAAW,CAAC,UAAmB;QACrC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAC/B,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,EAAE,UAAU,CAAC,CAAC;IACnD,CAAC;IAED;;OAEG;IACO,eAAe;QACrB,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IAC5D,CAAC;IAED;;;;;;OAMG;IACO,KAAK,CAAC,6BAA6B,CACzC,WAAwB,EACxB,UAA+C,EAAE;QAEjD,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,cAAc,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QAC1E,MAAM,EAAE,cAAc,GAAG,EAAE,EAAE,GAAG,OAAO,CAAC;QAExC,OAAO,IAAI,OAAO,CAAC;YACf,GAAG,IAAI,CAAC,cAAc;YACtB,GAAG,cAAc;YACjB,WAAW;SACd,CAAC,CAAC;IACP,CAAC;IAED;;;OAGG;IACO,KAAK,CAAC,cAAc;QAC1B,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,CAAC;QAC1D,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;QAC7B,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,yBAAyB,UAAU,CAAC,EAAE,EAAE,CAAC,CAAC;QAEzD,OAAO,UAAU,CAAC;IACtB,CAAC;IAED;;OAEG;IACO,mBAAmB;QACzB,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,IAAI,CAAC,WAAW,CAAC;IACnD,CAAC;IAED;;;OAGG;IACO,YAAY;QAClB,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC,0EAA0E;IAC5H,CAAC;IAED;;;OAGG;IACO,KAAK,CAAC,qBAAqB;QACjC,MAAM,iBAAiB,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,QAAQ,CAA6B,IAAI,CAAC,eAAe,CAAC,CAAC;QAE9G,IAAI,CAAC,iBAAiB;YAAE,OAAO;QAE/B,wDAAwD;QACxD,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,qCAAqC,EAAE;YAClD,2BAA2B,EAAE,IAAI,CAAC,2BAA2B;YAC7D,eAAe,EAAE,IAAI,CAAC,eAAe;SACxC,CAAC,CAAC;QAEH,KAAK,MAAM,aAAa,IAAI,iBAAiB,CAAC,QAAQ,EAAE,CAAC;YACrD,aAAa,CAAC,WAAW,GAAG,IAAI,CAAC;YACjC,aAAa,CAAC,SAAS,GAAG,IAAI,IAAI,CAAC,aAAa,CAAC,SAAmB,CAAC,CAAC;YACtE,aAAa,CAAC,SAAS,GAAG,IAAI,IAAI,CAAC,aAAa,CAAC,SAAmB,CAAC,CAAC;YACtE,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,qBAAqB,CAAC,IAAI,EAAE,EAAE,cAAc,EAAE,aAAa,EAAE,CAAC,CAAC;YAEnG,IAAI,gBAAgB,CAAC,QAAQ,EAAE,EAAE,CAAC;gBAC9B,IAAI,CAAC,WAAW,CAAC,gBAAgB,CAAC,CAAC;YACvC,CAAC;QACL,CAAC;QAED,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,mBAAmB,4CAA4C,CAAC,CAAC;IAC5F,CAAC;IAED;;;;;OAKG;IACH,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,OAA4B;QAC1C,MAAM,WAAW,GAAG,IAAI,WAAW,CAAC,OAAO,CAAC,CAAC;QAC7C,MAAM,WAAW,CAAC,UAAU,EAAE,CAAC;QAC/B,OAAO,WAAW,CAAC;IACvB,CAAC;CACJ"}
|
package/storages/dataset.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { DatasetClient, DatasetInfo, Dictionary, StorageClient } from '@crawlee/types';
|
|
2
2
|
import { Configuration } from '../configuration.js';
|
|
3
|
-
import {
|
|
3
|
+
import type { CrawleeLogger } from '../log.js';
|
|
4
4
|
import type { Awaitable } from '../typedefs.js';
|
|
5
5
|
import type { StorageManagerOptions } from './storage_manager.js';
|
|
6
6
|
/** @internal */
|
|
@@ -151,7 +151,7 @@ export declare class Dataset<Data extends Dictionary = Dictionary> {
|
|
|
151
151
|
name?: string;
|
|
152
152
|
client: DatasetClient<Data>;
|
|
153
153
|
readonly storageObject?: Record<string, unknown>;
|
|
154
|
-
log:
|
|
154
|
+
log: CrawleeLogger;
|
|
155
155
|
/**
|
|
156
156
|
* @internal
|
|
157
157
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"dataset.d.ts","sourceRoot":"","sources":["../../src/storages/dataset.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,WAAW,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAM5F,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"dataset.d.ts","sourceRoot":"","sources":["../../src/storages/dataset.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,WAAW,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAM5F,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAE/C,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAGhD,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC;AAIlE,gBAAgB;AAChB,eAAO,MAAM,+BAA+B,QAAQ,CAAC;AAIrD;;;;;GAKG;AACH,wBAAgB,iBAAiB,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,UAAU,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,CAsBxF;AAED;;;;;;;;GAQG;AACH,wBAAgB,WAAW,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,UAAU,EAAE,MAAM,GAAG,MAAM,EAAE,CA8BzE;AAED,MAAM,WAAW,kBAAkB;IAC/B;;;OAGG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf;;;;OAIG;IACH,IAAI,CAAC,EAAE,OAAO,CAAC;IAEf;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;IAElB;;;OAGG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB;;;;OAIG;IACH,KAAK,CAAC,EAAE,OAAO,CAAC;IAEhB;;;OAGG;IACH,UAAU,CAAC,EAAE,OAAO,CAAC;IAErB;;;;OAIG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;CACvB;AAED,MAAM,WAAW,oBAAqB,SAAQ,IAAI,CAAC,kBAAkB,EAAE,QAAQ,GAAG,OAAO,CAAC;IACtF;;;OAGG;IACH,cAAc,CAAC,EAAE,OAAO,CAAC;CAC5B;AAED,MAAM,WAAW,sBACb,SAAQ,IAAI,CAAC,kBAAkB,EAAE,QAAQ,GAAG,OAAO,GAAG,OAAO,GAAG,YAAY,GAAG,WAAW,CAAC;IAC3F,gBAAgB;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf,gBAAgB;IAChB,KAAK,CAAC,EAAE,OAAO,CAAC;IAEhB,gBAAgB;IAChB,UAAU,CAAC,EAAE,OAAO,CAAC;IAErB,gBAAgB;IAChB,SAAS,CAAC,EAAE,OAAO,CAAC;IAEpB,gBAAgB;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,sBAAuB,SAAQ,oBAAoB;IAChE,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;CAClB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkDG;AACH,qBAAa,OAAO,CAAC,IAAI,SAAS,UAAU,GAAG,UAAU;IAYjD,QAAQ,CAAC,MAAM;IAXnB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,aAAa,CAAC,IAAI,CAAC,CAAC;IAC5B,QAAQ,CAAC,aAAa,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACjD,GAAG,EAAE,aAAa,CAAC;IAEnB;;OAEG;gBAEC,OAAO,EAAE,cAAc,EACd,MAAM,gBAAkC;IASrD;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACG,QAAQ,CAAC,IAAI,EAAE,IAAI,GAAG,IAAI,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAwBlD;;OAEG;IACG,OAAO,CAAC,OAAO,GAAE,kBAAuB,GAAG,OAAO,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;IAgB9E;;;OAGG;IACG,MAAM,CAAC,OAAO,GAAE,oBAAyB,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;IAyBjE;;;;;;OAMG;IACG,QAAQ,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,sBAAsB,EAAE,WAAW,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;IAmCpG;;;;;OAKG;IACG,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,IAAI,CAAC,sBAAsB,EAAE,aAAa,CAAC;IAIrF;;;;;OAKG;IACG,WAAW,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,IAAI,CAAC,sBAAsB,EAAE,aAAa,CAAC;IAIpF;;;;;OAKG;WACU,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,sBAAsB;IAOvE;;;;;OAKG;WACU,WAAW,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,sBAAsB;IAOtE;;;;;;;;;;;;;;;;;;;;;OAqBG;IACG,OAAO,IAAI,OAAO,CAAC,WAAW,GAAG,SAAS,CAAC;IAMjD;;;;;;;;;;;;;;;;;;;OAmBG;IACG,OAAO,CAAC,QAAQ,EAAE,eAAe,CAAC,IAAI,CAAC,EAAE,OAAO,GAAE,sBAA2B,EAAE,KAAK,SAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IAqB9G;;;;;;;;OAQG;IACG,GAAG,CAAC,CAAC,EAAE,QAAQ,EAAE,aAAa,CAAC,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,GAAE,sBAA2B,GAAG,OAAO,CAAC,CAAC,EAAE,CAAC;IAalG;;;;;;;;;;;;;;;;OAgBG;IACG,MAAM,CAAC,QAAQ,EAAE,cAAc,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,GAAG,SAAS,CAAC;IAE7E;;;;;;;;;;;;;;;;;;OAkBG;IACG,MAAM,CACR,QAAQ,EAAE,cAAc,CAAC,IAAI,EAAE,IAAI,CAAC,EACpC,IAAI,EAAE,SAAS,EACf,OAAO,EAAE,sBAAsB,GAChC,OAAO,CAAC,IAAI,GAAG,SAAS,CAAC;IAE5B;;;;;;;;;;;;OAYG;IACG,MAAM,CAAC,CAAC,EAAE,QAAQ,EAAE,cAAc,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,OAAO,CAAC,EAAE,sBAAsB,GAAG,OAAO,CAAC,CAAC,CAAC;IAyBzG;;;OAGG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAQ3B;;;;;;;;;;;;;OAaG;WACU,IAAI,CAAC,IAAI,SAAS,UAAU,GAAG,UAAU,EAClD,eAAe,CAAC,EAAE,MAAM,GAAG,IAAI,EAC/B,OAAO,GAAE,qBAA0B,GACpC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IAsBzB;;;;;;;;;;;;;;;;;;;;;;;OAuBG;WACU,QAAQ,CAAC,IAAI,SAAS,UAAU,GAAG,UAAU,EAAE,IAAI,EAAE,IAAI,GAAG,IAAI,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAK/F;;OAEG;WACU,OAAO,CAAC,IAAI,SAAS,UAAU,GAAG,UAAU,EACrD,OAAO,GAAE,kBAAuB,GACjC,OAAO,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;CAInC;AAED;;GAEG;AACH,MAAM,WAAW,eAAe,CAAC,IAAI;IACjC;;;OAGG;IACH,CAAC,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;CAChD;AAED;;GAEG;AACH,MAAM,WAAW,aAAa,CAAC,IAAI,EAAE,CAAC;IAClC;;;;OAIG;IACH,CAAC,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;CAC7C;AAED;;GAEG;AACH,MAAM,WAAW,cAAc,CAAC,CAAC,EAAE,IAAI;IACnC;;;;OAIG;IACH,CAAC,IAAI,EAAE,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;CACtD;AAED,MAAM,WAAW,cAAc;IAC3B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,aAAa,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAC3C;AAED,MAAM,WAAW,cAAc,CAAC,IAAI;IAChC,6CAA6C;IAC7C,KAAK,EAAE,MAAM,CAAC;IACd,qDAAqD;IACrD,KAAK,EAAE,MAAM,CAAC;IACd,2DAA2D;IAC3D,MAAM,EAAE,MAAM,CAAC;IACf,mDAAmD;IACnD,KAAK,EAAE,MAAM,CAAC;IACd,wDAAwD;IACxD,KAAK,EAAE,IAAI,EAAE,CAAC;IACd,iDAAiD;IACjD,IAAI,CAAC,EAAE,OAAO,CAAC;CAClB"}
|
package/storages/dataset.js
CHANGED
|
@@ -2,7 +2,6 @@ import { stringify } from 'csv-stringify/sync';
|
|
|
2
2
|
import ow from 'ow';
|
|
3
3
|
import { MAX_PAYLOAD_SIZE_BYTES } from '@apify/consts';
|
|
4
4
|
import { Configuration } from '../configuration.js';
|
|
5
|
-
import { log } from '../log.js';
|
|
6
5
|
import { serviceLocator } from '../service_locator.js';
|
|
7
6
|
import { checkStorageAccess } from './access_checking.js';
|
|
8
7
|
import { KeyValueStore } from './key_value_store.js';
|
|
@@ -134,7 +133,7 @@ export class Dataset {
|
|
|
134
133
|
name;
|
|
135
134
|
client;
|
|
136
135
|
storageObject;
|
|
137
|
-
log
|
|
136
|
+
log;
|
|
138
137
|
/**
|
|
139
138
|
* @internal
|
|
140
139
|
*/
|
|
@@ -144,6 +143,7 @@ export class Dataset {
|
|
|
144
143
|
this.name = options.name;
|
|
145
144
|
this.client = options.client.dataset(this.id);
|
|
146
145
|
this.storageObject = options.storageObject;
|
|
146
|
+
this.log = serviceLocator.getLogger().child({ prefix: 'Dataset' });
|
|
147
147
|
}
|
|
148
148
|
/**
|
|
149
149
|
* Stores an object or an array of objects to the dataset.
|