@crawlee/core 3.13.5 → 4.0.0-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/autoscaling/autoscaled_pool.d.ts +3 -3
- package/autoscaling/autoscaled_pool.d.ts.map +1 -1
- package/autoscaling/autoscaled_pool.js +76 -225
- package/autoscaling/autoscaled_pool.js.map +1 -1
- package/autoscaling/index.d.ts +3 -3
- package/autoscaling/index.d.ts.map +1 -1
- package/autoscaling/index.js +3 -6
- package/autoscaling/index.js.map +1 -1
- package/autoscaling/snapshotter.d.ts +3 -3
- package/autoscaling/snapshotter.d.ts.map +1 -1
- package/autoscaling/snapshotter.js +42 -137
- package/autoscaling/snapshotter.js.map +1 -1
- package/autoscaling/system_status.d.ts +2 -2
- package/autoscaling/system_status.d.ts.map +1 -1
- package/autoscaling/system_status.js +20 -55
- package/autoscaling/system_status.js.map +1 -1
- package/configuration.d.ts +4 -4
- package/configuration.d.ts.map +1 -1
- package/configuration.js +79 -126
- package/configuration.js.map +1 -1
- package/cookie_utils.js +17 -25
- package/cookie_utils.js.map +1 -1
- package/crawlers/crawler_commons.d.ts +9 -8
- package/crawlers/crawler_commons.d.ts.map +1 -1
- package/crawlers/crawler_commons.js +47 -121
- package/crawlers/crawler_commons.js.map +1 -1
- package/crawlers/crawler_extension.d.ts +1 -1
- package/crawlers/crawler_extension.d.ts.map +1 -1
- package/crawlers/crawler_extension.js +4 -20
- package/crawlers/crawler_extension.js.map +1 -1
- package/crawlers/crawler_utils.d.ts +1 -1
- package/crawlers/crawler_utils.d.ts.map +1 -1
- package/crawlers/crawler_utils.js +3 -6
- package/crawlers/crawler_utils.js.map +1 -1
- package/crawlers/error_snapshotter.d.ts +3 -3
- package/crawlers/error_snapshotter.d.ts.map +1 -1
- package/crawlers/error_snapshotter.js +8 -38
- package/crawlers/error_snapshotter.js.map +1 -1
- package/crawlers/error_tracker.d.ts +2 -2
- package/crawlers/error_tracker.d.ts.map +1 -1
- package/crawlers/error_tracker.js +18 -40
- package/crawlers/error_tracker.js.map +1 -1
- package/crawlers/index.d.ts +6 -6
- package/crawlers/index.d.ts.map +1 -1
- package/crawlers/index.js +6 -9
- package/crawlers/index.js.map +1 -1
- package/crawlers/statistics.d.ts +3 -3
- package/crawlers/statistics.d.ts.map +1 -1
- package/crawlers/statistics.js +59 -167
- package/crawlers/statistics.js.map +1 -1
- package/enqueue_links/enqueue_links.d.ts +2 -2
- package/enqueue_links/enqueue_links.d.ts.map +1 -1
- package/enqueue_links/enqueue_links.js +41 -47
- package/enqueue_links/enqueue_links.js.map +1 -1
- package/enqueue_links/index.d.ts +2 -2
- package/enqueue_links/index.d.ts.map +1 -1
- package/enqueue_links/index.js +2 -5
- package/enqueue_links/index.js.map +1 -1
- package/enqueue_links/shared.d.ts +4 -4
- package/enqueue_links/shared.d.ts.map +1 -1
- package/enqueue_links/shared.js +23 -35
- package/enqueue_links/shared.js.map +1 -1
- package/errors.js +5 -13
- package/errors.js.map +1 -1
- package/events/event_manager.d.ts +1 -1
- package/events/event_manager.d.ts.map +1 -1
- package/events/event_manager.js +16 -45
- package/events/event_manager.js.map +1 -1
- package/events/index.d.ts +2 -2
- package/events/index.d.ts.map +1 -1
- package/events/index.js +2 -5
- package/events/index.js.map +1 -1
- package/events/local_event_manager.d.ts +1 -1
- package/events/local_event_manager.d.ts.map +1 -1
- package/events/local_event_manager.js +15 -28
- package/events/local_event_manager.js.map +1 -1
- package/http_clients/base-http-client.d.ts +1 -1
- package/http_clients/base-http-client.d.ts.map +1 -1
- package/http_clients/base-http-client.js +5 -8
- package/http_clients/base-http-client.js.map +1 -1
- package/http_clients/form-data-like.js +1 -2
- package/http_clients/form-data-like.js.map +1 -1
- package/http_clients/got-scraping-http-client.d.ts +1 -1
- package/http_clients/got-scraping-http-client.d.ts.map +1 -1
- package/http_clients/got-scraping-http-client.js +6 -11
- package/http_clients/got-scraping-http-client.js.map +1 -1
- package/http_clients/index.d.ts +2 -2
- package/http_clients/index.d.ts.map +1 -1
- package/http_clients/index.js +2 -5
- package/http_clients/index.js.map +1 -1
- package/index.d.ts +16 -16
- package/index.d.ts.map +1 -1
- package/index.js +17 -22
- package/index.js.map +1 -1
- package/log.js +2 -11
- package/log.js.map +1 -1
- package/package.json +27 -33
- package/proxy_configuration.d.ts +1 -1
- package/proxy_configuration.d.ts.map +1 -1
- package/proxy_configuration.js +22 -78
- package/proxy_configuration.js.map +1 -1
- package/request.d.ts +2 -2
- package/request.d.ts.map +1 -1
- package/request.js +76 -147
- package/request.js.map +1 -1
- package/router.d.ts +3 -3
- package/router.d.ts.map +1 -1
- package/router.js +6 -21
- package/router.js.map +1 -1
- package/serialization.d.ts.map +1 -1
- package/serialization.js +22 -44
- package/serialization.js.map +1 -1
- package/session_pool/consts.js +3 -6
- package/session_pool/consts.js.map +1 -1
- package/session_pool/errors.js +3 -11
- package/session_pool/errors.js.map +1 -1
- package/session_pool/events.js +1 -4
- package/session_pool/events.js.map +1 -1
- package/session_pool/index.d.ts +5 -5
- package/session_pool/index.d.ts.map +1 -1
- package/session_pool/index.js +5 -8
- package/session_pool/index.js.map +1 -1
- package/session_pool/session.d.ts +3 -17
- package/session_pool/session.d.ts.map +1 -1
- package/session_pool/session.js +52 -116
- package/session_pool/session.js.map +1 -1
- package/session_pool/session_pool.d.ts +7 -7
- package/session_pool/session_pool.d.ts.map +1 -1
- package/session_pool/session_pool.js +42 -126
- package/session_pool/session_pool.js.map +1 -1
- package/storages/access_checking.d.ts +1 -1
- package/storages/access_checking.d.ts.map +1 -1
- package/storages/access_checking.js +4 -9
- package/storages/access_checking.js.map +1 -1
- package/storages/dataset.d.ts +4 -4
- package/storages/dataset.d.ts.map +1 -1
- package/storages/dataset.js +45 -76
- package/storages/dataset.js.map +1 -1
- package/storages/index.d.ts +11 -11
- package/storages/index.d.ts.map +1 -1
- package/storages/index.js +11 -18
- package/storages/index.js.map +1 -1
- package/storages/key_value_store.d.ts +3 -3
- package/storages/key_value_store.d.ts.map +1 -1
- package/storages/key_value_store.js +62 -102
- package/storages/key_value_store.js.map +1 -1
- package/storages/request_list.d.ts +3 -3
- package/storages/request_list.d.ts.map +1 -1
- package/storages/request_list.js +83 -183
- package/storages/request_list.js.map +1 -1
- package/storages/request_provider.d.ts +5 -5
- package/storages/request_provider.d.ts.map +1 -1
- package/storages/request_provider.js +97 -201
- package/storages/request_provider.js.map +1 -1
- package/storages/request_queue.d.ts +4 -4
- package/storages/request_queue.d.ts.map +1 -1
- package/storages/request_queue.js +27 -40
- package/storages/request_queue.js.map +1 -1
- package/storages/request_queue_v2.d.ts +4 -4
- package/storages/request_queue_v2.d.ts.map +1 -1
- package/storages/request_queue_v2.js +18 -42
- package/storages/request_queue_v2.js.map +1 -1
- package/storages/sitemap_request_list.d.ts +4 -4
- package/storages/sitemap_request_list.d.ts.map +1 -1
- package/storages/sitemap_request_list.js +96 -181
- package/storages/sitemap_request_list.js.map +1 -1
- package/storages/storage_manager.d.ts +3 -3
- package/storages/storage_manager.d.ts.map +1 -1
- package/storages/storage_manager.js +14 -42
- package/storages/storage_manager.js.map +1 -1
- package/storages/utils.d.ts +1 -1
- package/storages/utils.d.ts.map +1 -1
- package/storages/utils.js +16 -23
- package/storages/utils.js.map +1 -1
- package/tsconfig.build.tsbuildinfo +1 -1
- package/typedefs.js +2 -6
- package/typedefs.js.map +1 -1
- package/validators.js +6 -10
- package/validators.js.map +1 -1
- package/index.mjs +0 -88
package/session_pool/index.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
export * from './errors';
|
|
2
|
-
export * from './events';
|
|
3
|
-
export * from './session';
|
|
4
|
-
export * from './session_pool';
|
|
5
|
-
export * from './consts';
|
|
1
|
+
export * from './errors.js';
|
|
2
|
+
export * from './events.js';
|
|
3
|
+
export * from './session.js';
|
|
4
|
+
export * from './session_pool.js';
|
|
5
|
+
export * from './consts.js';
|
|
6
6
|
//# sourceMappingURL=index.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/session_pool/index.ts"],"names":[],"mappings":"AAAA,cAAc,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/session_pool/index.ts"],"names":[],"mappings":"AAAA,cAAc,aAAa,CAAC;AAC5B,cAAc,aAAa,CAAC;AAC5B,cAAc,cAAc,CAAC;AAC7B,cAAc,mBAAmB,CAAC;AAClC,cAAc,aAAa,CAAC"}
|
package/session_pool/index.js
CHANGED
|
@@ -1,9 +1,6 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
tslib_1.__exportStar(require("./session"), exports);
|
|
7
|
-
tslib_1.__exportStar(require("./session_pool"), exports);
|
|
8
|
-
tslib_1.__exportStar(require("./consts"), exports);
|
|
1
|
+
export * from './errors.js';
|
|
2
|
+
export * from './events.js';
|
|
3
|
+
export * from './session.js';
|
|
4
|
+
export * from './session_pool.js';
|
|
5
|
+
export * from './consts.js';
|
|
9
6
|
//# sourceMappingURL=index.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/session_pool/index.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/session_pool/index.ts"],"names":[],"mappings":"AAAA,cAAc,aAAa,CAAC;AAC5B,cAAc,aAAa,CAAC;AAC5B,cAAc,cAAc,CAAC;AAC7B,cAAc,mBAAmB,CAAC;AAClC,cAAc,aAAa,CAAC"}
|
|
@@ -2,7 +2,7 @@ import type { Cookie as CookieObject, Dictionary } from '@crawlee/types';
|
|
|
2
2
|
import type { Cookie, SerializedCookieJar } from 'tough-cookie';
|
|
3
3
|
import { CookieJar } from 'tough-cookie';
|
|
4
4
|
import type { Log } from '@apify/log';
|
|
5
|
-
import type { ResponseLike } from '../cookie_utils';
|
|
5
|
+
import type { ResponseLike } from '../cookie_utils.js';
|
|
6
6
|
/**
|
|
7
7
|
* Persistable {@link Session} state.
|
|
8
8
|
*/
|
|
@@ -59,7 +59,7 @@ export interface SessionOptions {
|
|
|
59
59
|
*/
|
|
60
60
|
maxUsageCount?: number;
|
|
61
61
|
/** SessionPool instance. Session will emit the `sessionRetired` event on this instance. */
|
|
62
|
-
sessionPool?: import('./session_pool').SessionPool;
|
|
62
|
+
sessionPool?: import('./session_pool.js').SessionPool;
|
|
63
63
|
log?: Log;
|
|
64
64
|
errorScore?: number;
|
|
65
65
|
cookieJar?: CookieJar;
|
|
@@ -143,26 +143,12 @@ export declare class Session {
|
|
|
143
143
|
/**
|
|
144
144
|
* With certain status codes: `401`, `403` or `429` we can be certain
|
|
145
145
|
* that the target website is blocking us. This function helps to do this conveniently
|
|
146
|
-
* by retiring the session when such code is received. Optionally the default status
|
|
146
|
+
* by retiring the session when such code is received. Optionally, the default status
|
|
147
147
|
* codes can be extended in the second parameter.
|
|
148
148
|
* @param statusCode HTTP status code.
|
|
149
149
|
* @returns Whether the session was retired.
|
|
150
150
|
*/
|
|
151
151
|
retireOnBlockedStatusCodes(statusCode: number): boolean;
|
|
152
|
-
/**
|
|
153
|
-
* With certain status codes: `401`, `403` or `429` we can be certain
|
|
154
|
-
* that the target website is blocking us. This function helps to do this conveniently
|
|
155
|
-
* by retiring the session when such code is received. Optionally the default status
|
|
156
|
-
* codes can be extended in the second parameter.
|
|
157
|
-
* @param statusCode HTTP status code.
|
|
158
|
-
* @param [additionalBlockedStatusCodes]
|
|
159
|
-
* Custom HTTP status codes that means blocking on particular website.
|
|
160
|
-
*
|
|
161
|
-
* **This parameter is deprecated and will be removed in next major version.**
|
|
162
|
-
* @returns Whether the session was retired.
|
|
163
|
-
* @deprecated The parameter `additionalBlockedStatusCodes` is deprecated and will be removed in next major version.
|
|
164
|
-
*/
|
|
165
|
-
retireOnBlockedStatusCodes(statusCode: number, additionalBlockedStatusCodes?: number[]): boolean;
|
|
166
152
|
/**
|
|
167
153
|
* Saves cookies from an HTTP response to be used with the session.
|
|
168
154
|
* It expects an object with a `headers` property that's either an `Object`
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session.d.ts","sourceRoot":"","sources":["../../src/session_pool/session.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,MAAM,IAAI,YAAY,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAEzE,OAAO,KAAK,EAAE,MAAM,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAC;AAChE,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAEzC,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AAGtC,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,
|
|
1
|
+
{"version":3,"file":"session.d.ts","sourceRoot":"","sources":["../../src/session_pool/session.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,MAAM,IAAI,YAAY,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAEzE,OAAO,KAAK,EAAE,MAAM,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAC;AAChE,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAEzC,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AAGtC,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAUvD;;GAEG;AACH,MAAM,WAAW,YAAY;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,mBAAmB,CAAC;IAC/B,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,cAAc;IAC3B,wFAAwF;IACxF,EAAE,CAAC,EAAE,MAAM,CAAC;IAEZ;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB,+EAA+E;IAC/E,QAAQ,CAAC,EAAE,UAAU,CAAC;IAEtB;;;;;;OAMG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IAEvB;;;;;OAKG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAE7B,wBAAwB;IACxB,SAAS,CAAC,EAAE,IAAI,CAAC;IAEjB,0BAA0B;IAC1B,SAAS,CAAC,EAAE,IAAI,CAAC;IAEjB;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB;;;;OAIG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IAEvB,2FAA2F;IAC3F,WAAW,CAAC,EAAE,OAAO,mBAAmB,EAAE,WAAW,CAAC;IAEtD,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,SAAS,CAAC;CACzB;AAED;;;;;GAKG;AACH,qBAAa,OAAO;IAChB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,UAAU,CAAS;IAC3B,QAAQ,EAAE,UAAU,CAAC;IACrB,OAAO,CAAC,cAAc,CAAS;IAC/B,OAAO,CAAC,oBAAoB,CAAS;IACrC,OAAO,CAAC,UAAU,CAAO;IACzB,OAAO,CAAC,UAAU,CAAO;IACzB,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,cAAc,CAAS;IAC/B,OAAO,CAAC,WAAW,CAA0C;IAC7D,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,UAAU,CAAY;IAC9B,OAAO,CAAC,GAAG,CAAM;IAEjB,IAAI,UAAU,WAEb;IAED,IAAI,UAAU,WAEb;IAED,IAAI,aAAa,WAEhB;IAED,IAAI,mBAAmB,WAEtB;IAED,IAAI,SAAS,SAEZ;IAED,IAAI,SAAS,SAEZ;IAED,IAAI,aAAa,WAEhB;IAED,IAAI,SAAS,cAEZ;IAED;;OAEG;gBACS,OAAO,EAAE,cAAc;IAuDnC;;;OAGG;IACH,SAAS,IAAI,OAAO;IAIpB;;;;OAIG;IACH,SAAS,IAAI,OAAO;IAIpB;;;OAGG;IACH,sBAAsB,IAAI,OAAO;IAIjC;;;OAGG;IACH,QAAQ,IAAI,OAAO;IAInB;;;OAGG;IACH,QAAQ;IAUR;;;OAGG;IACH,QAAQ,IAAI,YAAY;IAexB;;;;;;OAMG;IACH,MAAM;IASN;;;OAGG;IACH,OAAO;IAOP;;;;;;;OAOG;IACH,0BAA0B,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO;IASvD;;;;;;OAMG;IACH,sBAAsB,CAAC,QAAQ,EAAE,YAAY;IAW7C;;;;;;;;;;;;OAYG;IACH,UAAU,CAAC,OAAO,EAAE,YAAY,EAAE,EAAE,GAAG,EAAE,MAAM;IAK/C;;;OAGG;IACH,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG,YAAY,EAAE;IAKvC;;;;;OAKG;IACH,eAAe,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM;IAIpC;;OAEG;IACH,SAAS,CAAC,SAAS,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,IAAI;IAI/C;;OAEG;IACH,SAAS,CAAC,WAAW,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,MAAM,GAAG,IAAI;IAkB3D;;OAEG;IACH,SAAS,CAAC,gBAAgB,IAAI,IAAI;CAKrC"}
|
package/session_pool/session.js
CHANGED
|
@@ -1,21 +1,30 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
const utilities_1 = require("@apify/utilities");
|
|
9
|
-
const cookie_utils_1 = require("../cookie_utils");
|
|
10
|
-
const log_1 = require("../log");
|
|
11
|
-
const events_1 = require("./events");
|
|
1
|
+
import { EventEmitter } from 'node:events';
|
|
2
|
+
import ow from 'ow';
|
|
3
|
+
import { CookieJar } from 'tough-cookie';
|
|
4
|
+
import { cryptoRandomObjectId } from '@apify/utilities';
|
|
5
|
+
import { browserPoolCookieToToughCookie, getCookiesFromResponse, getDefaultCookieExpirationDate, toughCookieToBrowserPoolCookie, } from '../cookie_utils.js';
|
|
6
|
+
import { log as defaultLog } from '../log.js';
|
|
7
|
+
import { EVENT_SESSION_RETIRED } from './events.js';
|
|
12
8
|
/**
|
|
13
9
|
* Sessions are used to store information such as cookies and can be used for generating fingerprints and proxy sessions.
|
|
14
10
|
* You can imagine each session as a specific user, with its own cookies, IP (via proxy) and potentially a unique browser fingerprint.
|
|
15
11
|
* Session internal state can be enriched with custom user data for example some authorization tokens and specific headers in general.
|
|
16
12
|
* @category Scaling
|
|
17
13
|
*/
|
|
18
|
-
class Session {
|
|
14
|
+
export class Session {
|
|
15
|
+
id;
|
|
16
|
+
maxAgeSecs;
|
|
17
|
+
userData;
|
|
18
|
+
_maxErrorScore;
|
|
19
|
+
_errorScoreDecrement;
|
|
20
|
+
_createdAt;
|
|
21
|
+
_expiresAt;
|
|
22
|
+
_usageCount;
|
|
23
|
+
_maxUsageCount;
|
|
24
|
+
sessionPool;
|
|
25
|
+
_errorScore;
|
|
26
|
+
_cookieJar;
|
|
27
|
+
log;
|
|
19
28
|
get errorScore() {
|
|
20
29
|
return this._errorScore;
|
|
21
30
|
}
|
|
@@ -44,103 +53,25 @@ class Session {
|
|
|
44
53
|
* Session configuration.
|
|
45
54
|
*/
|
|
46
55
|
constructor(options) {
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
configurable: true,
|
|
62
|
-
writable: true,
|
|
63
|
-
value: void 0
|
|
64
|
-
});
|
|
65
|
-
Object.defineProperty(this, "_maxErrorScore", {
|
|
66
|
-
enumerable: true,
|
|
67
|
-
configurable: true,
|
|
68
|
-
writable: true,
|
|
69
|
-
value: void 0
|
|
70
|
-
});
|
|
71
|
-
Object.defineProperty(this, "_errorScoreDecrement", {
|
|
72
|
-
enumerable: true,
|
|
73
|
-
configurable: true,
|
|
74
|
-
writable: true,
|
|
75
|
-
value: void 0
|
|
76
|
-
});
|
|
77
|
-
Object.defineProperty(this, "_createdAt", {
|
|
78
|
-
enumerable: true,
|
|
79
|
-
configurable: true,
|
|
80
|
-
writable: true,
|
|
81
|
-
value: void 0
|
|
82
|
-
});
|
|
83
|
-
Object.defineProperty(this, "_expiresAt", {
|
|
84
|
-
enumerable: true,
|
|
85
|
-
configurable: true,
|
|
86
|
-
writable: true,
|
|
87
|
-
value: void 0
|
|
88
|
-
});
|
|
89
|
-
Object.defineProperty(this, "_usageCount", {
|
|
90
|
-
enumerable: true,
|
|
91
|
-
configurable: true,
|
|
92
|
-
writable: true,
|
|
93
|
-
value: void 0
|
|
94
|
-
});
|
|
95
|
-
Object.defineProperty(this, "_maxUsageCount", {
|
|
96
|
-
enumerable: true,
|
|
97
|
-
configurable: true,
|
|
98
|
-
writable: true,
|
|
99
|
-
value: void 0
|
|
100
|
-
});
|
|
101
|
-
Object.defineProperty(this, "sessionPool", {
|
|
102
|
-
enumerable: true,
|
|
103
|
-
configurable: true,
|
|
104
|
-
writable: true,
|
|
105
|
-
value: void 0
|
|
106
|
-
});
|
|
107
|
-
Object.defineProperty(this, "_errorScore", {
|
|
108
|
-
enumerable: true,
|
|
109
|
-
configurable: true,
|
|
110
|
-
writable: true,
|
|
111
|
-
value: void 0
|
|
112
|
-
});
|
|
113
|
-
Object.defineProperty(this, "_cookieJar", {
|
|
114
|
-
enumerable: true,
|
|
115
|
-
configurable: true,
|
|
116
|
-
writable: true,
|
|
117
|
-
value: void 0
|
|
118
|
-
});
|
|
119
|
-
Object.defineProperty(this, "log", {
|
|
120
|
-
enumerable: true,
|
|
121
|
-
configurable: true,
|
|
122
|
-
writable: true,
|
|
123
|
-
value: void 0
|
|
124
|
-
});
|
|
125
|
-
(0, ow_1.default)(options, ow_1.default.object.exactShape({
|
|
126
|
-
sessionPool: ow_1.default.object.instanceOf(node_events_1.EventEmitter),
|
|
127
|
-
id: ow_1.default.optional.string,
|
|
128
|
-
cookieJar: ow_1.default.optional.object,
|
|
129
|
-
maxAgeSecs: ow_1.default.optional.number,
|
|
130
|
-
userData: ow_1.default.optional.object,
|
|
131
|
-
maxErrorScore: ow_1.default.optional.number,
|
|
132
|
-
errorScoreDecrement: ow_1.default.optional.number,
|
|
133
|
-
createdAt: ow_1.default.optional.date,
|
|
134
|
-
expiresAt: ow_1.default.optional.date,
|
|
135
|
-
usageCount: ow_1.default.optional.number,
|
|
136
|
-
errorScore: ow_1.default.optional.number,
|
|
137
|
-
maxUsageCount: ow_1.default.optional.number,
|
|
138
|
-
log: ow_1.default.optional.object,
|
|
56
|
+
ow(options, ow.object.exactShape({
|
|
57
|
+
sessionPool: ow.object.instanceOf(EventEmitter),
|
|
58
|
+
id: ow.optional.string,
|
|
59
|
+
cookieJar: ow.optional.object,
|
|
60
|
+
maxAgeSecs: ow.optional.number,
|
|
61
|
+
userData: ow.optional.object,
|
|
62
|
+
maxErrorScore: ow.optional.number,
|
|
63
|
+
errorScoreDecrement: ow.optional.number,
|
|
64
|
+
createdAt: ow.optional.date,
|
|
65
|
+
expiresAt: ow.optional.date,
|
|
66
|
+
usageCount: ow.optional.number,
|
|
67
|
+
errorScore: ow.optional.number,
|
|
68
|
+
maxUsageCount: ow.optional.number,
|
|
69
|
+
log: ow.optional.object,
|
|
139
70
|
}));
|
|
140
|
-
const { sessionPool, id = `session_${
|
|
141
|
-
const { expiresAt =
|
|
71
|
+
const { sessionPool, id = `session_${cryptoRandomObjectId(10)}`, cookieJar = new CookieJar(), maxAgeSecs = 3000, userData = {}, maxErrorScore = 3, errorScoreDecrement = 0.5, createdAt = new Date(), usageCount = 0, errorScore = 0, maxUsageCount = 50, log = defaultLog, } = options;
|
|
72
|
+
const { expiresAt = getDefaultCookieExpirationDate(maxAgeSecs) } = options;
|
|
142
73
|
this.log = log.child({ prefix: 'Session' });
|
|
143
|
-
this._cookieJar = cookieJar.setCookie ? cookieJar :
|
|
74
|
+
this._cookieJar = cookieJar.setCookie ? cookieJar : CookieJar.fromJSON(JSON.stringify(cookieJar));
|
|
144
75
|
this.id = id;
|
|
145
76
|
this.maxAgeSecs = maxAgeSecs;
|
|
146
77
|
this.userData = userData;
|
|
@@ -224,7 +155,7 @@ class Session {
|
|
|
224
155
|
this._errorScore += this._maxErrorScore;
|
|
225
156
|
this._usageCount += 1;
|
|
226
157
|
// emit event so we can retire browser in puppeteer pool
|
|
227
|
-
this.sessionPool.emit(
|
|
158
|
+
this.sessionPool.emit(EVENT_SESSION_RETIRED, this);
|
|
228
159
|
}
|
|
229
160
|
/**
|
|
230
161
|
* Increases usage and error count.
|
|
@@ -235,11 +166,17 @@ class Session {
|
|
|
235
166
|
this._usageCount += 1;
|
|
236
167
|
this._maybeSelfRetire();
|
|
237
168
|
}
|
|
238
|
-
|
|
169
|
+
/**
|
|
170
|
+
* With certain status codes: `401`, `403` or `429` we can be certain
|
|
171
|
+
* that the target website is blocking us. This function helps to do this conveniently
|
|
172
|
+
* by retiring the session when such code is received. Optionally, the default status
|
|
173
|
+
* codes can be extended in the second parameter.
|
|
174
|
+
* @param statusCode HTTP status code.
|
|
175
|
+
* @returns Whether the session was retired.
|
|
176
|
+
*/
|
|
177
|
+
retireOnBlockedStatusCodes(statusCode) {
|
|
239
178
|
// eslint-disable-next-line dot-notation -- accessing private property
|
|
240
|
-
const isBlocked = this.sessionPool['blockedStatusCodes']
|
|
241
|
-
.concat(additionalBlockedStatusCodes)
|
|
242
|
-
.includes(statusCode);
|
|
179
|
+
const isBlocked = this.sessionPool['blockedStatusCodes'].includes(statusCode);
|
|
243
180
|
if (isBlocked) {
|
|
244
181
|
this.retire();
|
|
245
182
|
}
|
|
@@ -254,7 +191,7 @@ class Session {
|
|
|
254
191
|
*/
|
|
255
192
|
setCookiesFromResponse(response) {
|
|
256
193
|
try {
|
|
257
|
-
const cookies =
|
|
194
|
+
const cookies = getCookiesFromResponse(response).filter((c) => c);
|
|
258
195
|
this._setCookies(cookies, typeof response.url === 'function' ? response.url() : response.url);
|
|
259
196
|
}
|
|
260
197
|
catch (e) {
|
|
@@ -277,7 +214,7 @@ class Session {
|
|
|
277
214
|
* ```
|
|
278
215
|
*/
|
|
279
216
|
setCookies(cookies, url) {
|
|
280
|
-
const normalizedCookies = cookies.map((c) =>
|
|
217
|
+
const normalizedCookies = cookies.map((c) => browserPoolCookieToToughCookie(c, this.maxAgeSecs));
|
|
281
218
|
this._setCookies(normalizedCookies, url);
|
|
282
219
|
}
|
|
283
220
|
/**
|
|
@@ -286,7 +223,7 @@ class Session {
|
|
|
286
223
|
*/
|
|
287
224
|
getCookies(url) {
|
|
288
225
|
const cookies = this.cookieJar.getCookiesSync(url);
|
|
289
|
-
return cookies.map((c) =>
|
|
226
|
+
return cookies.map((c) => toughCookieToBrowserPoolCookie(c));
|
|
290
227
|
}
|
|
291
228
|
/**
|
|
292
229
|
* Returns cookies saved with the session in the typical
|
|
@@ -331,5 +268,4 @@ class Session {
|
|
|
331
268
|
}
|
|
332
269
|
}
|
|
333
270
|
}
|
|
334
|
-
exports.Session = Session;
|
|
335
271
|
//# sourceMappingURL=session.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session.js","sourceRoot":"","sources":["../../src/session_pool/session.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"session.js","sourceRoot":"","sources":["../../src/session_pool/session.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAG3C,OAAO,EAAE,MAAM,IAAI,CAAC;AAEpB,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAGzC,OAAO,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AAGxD,OAAO,EACH,8BAA8B,EAC9B,sBAAsB,EACtB,8BAA8B,EAC9B,8BAA8B,GACjC,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,GAAG,IAAI,UAAU,EAAE,MAAM,WAAW,CAAC;AAC9C,OAAO,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;AA2EpD;;;;;GAKG;AACH,MAAM,OAAO,OAAO;IACP,EAAE,CAAS;IACZ,UAAU,CAAS;IAC3B,QAAQ,CAAa;IACb,cAAc,CAAS;IACvB,oBAAoB,CAAS;IAC7B,UAAU,CAAO;IACjB,UAAU,CAAO;IACjB,WAAW,CAAS;IACpB,cAAc,CAAS;IACvB,WAAW,CAA0C;IACrD,WAAW,CAAS;IACpB,UAAU,CAAY;IACtB,GAAG,CAAM;IAEjB,IAAI,UAAU;QACV,OAAO,IAAI,CAAC,WAAW,CAAC;IAC5B,CAAC;IAED,IAAI,UAAU;QACV,OAAO,IAAI,CAAC,WAAW,CAAC;IAC5B,CAAC;IAED,IAAI,aAAa;QACb,OAAO,IAAI,CAAC,cAAc,CAAC;IAC/B,CAAC;IAED,IAAI,mBAAmB;QACnB,OAAO,IAAI,CAAC,oBAAoB,CAAC;IACrC,CAAC;IAED,IAAI,SAAS;QACT,OAAO,IAAI,CAAC,UAAU,CAAC;IAC3B,CAAC;IAED,IAAI,SAAS;QACT,OAAO,IAAI,CAAC,UAAU,CAAC;IAC3B,CAAC;IAED,IAAI,aAAa;QACb,OAAO,IAAI,CAAC,cAAc,CAAC;IAC/B,CAAC;IAED,IAAI,SAAS;QACT,OAAO,IAAI,CAAC,UAAU,CAAC;IAC3B,CAAC;IAED;;OAEG;IACH,YAAY,OAAuB;QAC/B,EAAE,CACE,OAAO,EACP,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YACjB,WAAW,EAAE,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,YAAY,CAAC;YAC/C,EAAE,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACtB,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YAC7B,UAAU,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YAC9B,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YAC5B,aAAa,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACjC,mBAAmB,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACvC,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI;YAC3B,SAAS,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI;YAC3B,UAAU,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YAC9B,UAAU,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YAC9B,aAAa,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;YACjC,GAAG,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM;SAC1B,CAAC,CACL,CAAC;QAEF,MAAM,EACF,WAAW,EACX,EAAE,GAAG,WAAW,oBAAoB,CAAC,EAAE,CAAC,EAAE,EAC1C,SAAS,GAAG,IAAI,SAAS,EAAE,EAC3B,UAAU,GAAG,IAAI,EACjB,QAAQ,GAAG,EAAE,EACb,aAAa,GAAG,CAAC,EACjB,mBAAmB,GAAG,GAAG,EACzB,SAAS,GAAG,IAAI,IAAI,EAAE,EACtB,UAAU,GAAG,CAAC,EACd,UAAU,GAAG,CAAC,EACd,aAAa,GAAG,EAAE,EAClB,GAAG,GAAG,UAAU,GACnB,GAAG,OAAO,CAAC;QAEZ,MAAM,EAAE,SAAS,GAAG,8BAA8B,CAAC,UAAU,CAAC,EAAE,GAAG,OAAO,CAAC;QAE3E,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;QAE5C,IAAI,CAAC,UAAU,GAAI,SAAS,CAAC,SAAqB,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC;QAC/G,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,cAAc,GAAG,aAAa,CAAC;QACpC,IAAI,CAAC,oBAAoB,GAAG,mBAAmB,CAAC;QAEhD,WAAW;QACX,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;QAC5B,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;QAC5B,IAAI,CAAC,WAAW,GAAG,UAAU,CAAC,CAAC,qDAAqD;QACpF,IAAI,CAAC,WAAW,GAAG,UAAU,CAAC,CAAC,yDAAyD;QACxF,IAAI,CAAC,cAAc,GAAG,aAAa,CAAC;QACpC,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;IACnC,CAAC;IAED;;;OAGG;IACH,SAAS;QACL,OAAO,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,aAAa,CAAC;IACjD,CAAC;IAED;;;;OAIG;IACH,SAAS;QACL,OAAO,IAAI,CAAC,SAAS,IAAI,IAAI,IAAI,EAAE,CAAC;IACxC,CAAC;IAED;;;OAGG;IACH,sBAAsB;QAClB,OAAO,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,aAAa,CAAC;IACjD,CAAC;IAED;;;OAGG;IACH,QAAQ;QACJ,OAAO,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,sBAAsB,EAAE,CAAC;IACpF,CAAC;IAED;;;OAGG;IACH,QAAQ;QACJ,IAAI,CAAC,WAAW,IAAI,CAAC,CAAC;QAEtB,IAAI,IAAI,CAAC,WAAW,GAAG,CAAC,EAAE,CAAC;YACvB,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,oBAAoB,CAAC;QAClD,CAAC;QAED,IAAI,CAAC,gBAAgB,EAAE,CAAC;IAC5B,CAAC;IAED;;;OAGG;IACH,QAAQ;QACJ,OAAO;YACH,EAAE,EAAE,IAAI,CAAC,EAAE;YACX,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAG;YACnC,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,aAAa,EAAE,IAAI,CAAC,aAAa;YACjC,mBAAmB,EAAE,IAAI,CAAC,mBAAmB;YAC7C,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE;YACvC,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE;YACvC,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,aAAa,EAAE,IAAI,CAAC,aAAa;YACjC,UAAU,EAAE,IAAI,CAAC,UAAU;SAC9B,CAAC;IACN,CAAC;IAED;;;;;;OAMG;IACH,MAAM;QACF,6DAA6D;QAC7D,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,cAAc,CAAC;QACxC,IAAI,CAAC,WAAW,IAAI,CAAC,CAAC;QAEtB,wDAAwD;QACxD,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,qBAAqB,EAAE,IAAI,CAAC,CAAC;IACvD,CAAC;IAED;;;OAGG;IACH,OAAO;QACH,IAAI,CAAC,WAAW,IAAI,CAAC,CAAC;QACtB,IAAI,CAAC,WAAW,IAAI,CAAC,CAAC;QAEtB,IAAI,CAAC,gBAAgB,EAAE,CAAC;IAC5B,CAAC;IAED;;;;;;;OAOG;IACH,0BAA0B,CAAC,UAAkB;QACzC,sEAAsE;QACtE,MAAM,SAAS,GAAG,IAAI,CAAC,WAAW,CAAC,oBAAoB,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;QAC9E,IAAI,SAAS,EAAE,CAAC;YACZ,IAAI,CAAC,MAAM,EAAE,CAAC;QAClB,CAAC;QACD,OAAO,SAAS,CAAC;IACrB,CAAC;IAED;;;;;;OAMG;IACH,sBAAsB,CAAC,QAAsB;QACzC,IAAI,CAAC;YACD,MAAM,OAAO,GAAG,sBAAsB,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;YAClE,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,OAAO,QAAQ,CAAC,GAAG,KAAK,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAI,CAAC,CAAC;QACnG,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,MAAM,GAAG,GAAG,CAAU,CAAC;YACvB,+DAA+D;YAC/D,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,qCAAqC,CAAC,CAAC;QACnE,CAAC;IACL,CAAC;IAED;;;;;;;;;;;;OAYG;IACH,UAAU,CAAC,OAAuB,EAAE,GAAW;QAC3C,MAAM,iBAAiB,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,8BAA8B,CAAC,CAAC,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC;QACjG,IAAI,CAAC,WAAW,CAAC,iBAAiB,EAAE,GAAG,CAAC,CAAC;IAC7C,CAAC;IAED;;;OAGG;IACH,UAAU,CAAC,GAAW;QAClB,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;QACnD,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,8BAA8B,CAAC,CAAC,CAAC,CAAC,CAAC;IACjE,CAAC;IAED;;;;;OAKG;IACH,eAAe,CAAC,GAAW;QACvB,OAAO,IAAI,CAAC,SAAS,CAAC,mBAAmB,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;IACvD,CAAC;IAED;;OAEG;IACH,SAAS,CAAC,SAAiB,EAAE,GAAW;QACpC,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;IACjD,CAAC;IAED;;OAEG;IACO,WAAW,CAAC,OAAiB,EAAE,GAAW;QAChD,MAAM,aAAa,GAAa,EAAE,CAAC;QAEnC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC3B,IAAI,CAAC;gBACD,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,MAAM,EAAE,GAAG,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC,CAAC;YACtE,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACT,MAAM,GAAG,GAAG,CAAU,CAAC;gBACvB,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YACpC,CAAC;QACL,CAAC;QAED,sGAAsG;QACtG,IAAI,aAAa,CAAC,MAAM,EAAE,CAAC;YACvB,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,wBAAwB,EAAE,EAAE,aAAa,EAAE,CAAC,CAAC;QAChE,CAAC;IACL,CAAC;IAED;;OAEG;IACO,gBAAgB;QACtB,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,CAAC;YACnB,IAAI,CAAC,MAAM,EAAE,CAAC;QAClB,CAAC;IACL,CAAC;CACJ"}
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
import { EventEmitter } from 'node:events';
|
|
2
2
|
import type { Log } from '@apify/log';
|
|
3
|
-
import { Configuration } from '../configuration';
|
|
4
|
-
import type { PersistenceOptions } from '../crawlers/statistics';
|
|
5
|
-
import type { EventManager } from '../events/event_manager';
|
|
6
|
-
import { KeyValueStore } from '../storages/key_value_store';
|
|
7
|
-
import type { SessionOptions } from './session';
|
|
8
|
-
import { Session } from './session';
|
|
3
|
+
import { Configuration } from '../configuration.js';
|
|
4
|
+
import type { PersistenceOptions } from '../crawlers/statistics.js';
|
|
5
|
+
import type { EventManager } from '../events/event_manager.js';
|
|
6
|
+
import { KeyValueStore } from '../storages/key_value_store.js';
|
|
7
|
+
import type { SessionOptions } from './session.js';
|
|
8
|
+
import { Session } from './session.js';
|
|
9
9
|
/**
|
|
10
10
|
* Factory user-function which creates customized {@link Session} instances.
|
|
11
11
|
*/
|
|
@@ -181,7 +181,7 @@ export declare class SessionPool extends EventEmitter {
|
|
|
181
181
|
usableSessionsCount: number;
|
|
182
182
|
retiredSessionsCount: number;
|
|
183
183
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
184
|
-
sessions: import("./session").SessionState[];
|
|
184
|
+
sessions: import("./session.js").SessionState[];
|
|
185
185
|
};
|
|
186
186
|
/**
|
|
187
187
|
* Persists the current state of the `SessionPool` into the default {@link KeyValueStore}.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session_pool.d.ts","sourceRoot":"","sources":["../../src/session_pool/session_pool.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAM3C,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AAEtC,OAAO,EAAE,aAAa,EAAE,MAAM,
|
|
1
|
+
{"version":3,"file":"session_pool.d.ts","sourceRoot":"","sources":["../../src/session_pool/session_pool.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAM3C,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AAEtC,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,2BAA2B,CAAC;AACpE,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAG/D,OAAO,EAAE,aAAa,EAAE,MAAM,gCAAgC,CAAC;AAE/D,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AACnD,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAEvC;;GAEG;AACH,MAAM,WAAW,aAAa;IAC1B;;;OAGG;IACH,CAAC,WAAW,EAAE,WAAW,EAAE,OAAO,CAAC,EAAE;QAAE,cAAc,CAAC,EAAE,cAAc,CAAA;KAAE,GAAG,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;CACzG;AAED,MAAM,WAAW,kBAAkB;IAC/B;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB,kEAAkE;IAClE,cAAc,CAAC,EAAE,cAAc,CAAC;IAEhC,6EAA6E;IAC7E,2BAA2B,CAAC,EAAE,MAAM,CAAC;IAErC;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IAEzB;;;;OAIG;IACH,qBAAqB,CAAC,EAAE,aAAa,CAAC;IAEtC;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,MAAM,EAAE,CAAC;IAE9B,gBAAgB;IAChB,GAAG,CAAC,EAAE,GAAG,CAAC;IAEV;;OAEG;IACH,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;CAC3C;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgEG;AACH,qBAAa,WAAY,SAAQ,YAAY;IAuBrC,QAAQ,CAAC,MAAM;IAtBnB,SAAS,CAAC,GAAG,EAAE,GAAG,CAAC;IACnB,SAAS,CAAC,WAAW,EAAE,MAAM,CAAC;IAC9B,SAAS,CAAC,qBAAqB,EAAE,aAAa,CAAC;IAC/C,SAAS,CAAC,aAAa,EAAG,aAAa,CAAC;IACxC,SAAS,CAAC,QAAQ,EAAE,OAAO,EAAE,CAAM;IACnC,SAAS,CAAC,UAAU,uBAA8B;IAClD,SAAS,CAAC,cAAc,EAAE,cAAc,CAAC;IACzC,SAAS,CAAC,2BAA2B,CAAC,EAAE,MAAM,CAAC;IAC/C,SAAS,CAAC,eAAe,EAAE,MAAM,CAAC;IAClC,SAAS,CAAC,SAAS,EAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAC1C,SAAS,CAAC,MAAM,EAAE,YAAY,CAAC;IAC/B,SAAS,CAAC,QAAQ,CAAC,kBAAkB,EAAE,MAAM,EAAE,CAAC;IAChD,SAAS,CAAC,kBAAkB,EAAE,kBAAkB,CAAC;IACjD,SAAS,CAAC,aAAa,UAAS;IAEhC,OAAO,CAAC,KAAK,CAAoB;IAEjC;;OAEG;gBAEC,OAAO,GAAE,kBAAuB,EACvB,MAAM,gBAAkC;IAsDrD;;OAEG;IACH,IAAI,mBAAmB,IAAI,MAAM,CAEhC;IAED;;OAEG;IACH,IAAI,oBAAoB,IAAI,MAAM,CAEjC;IAED;;;OAGG;IACG,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IA0BjC;;;;;OAKG;IACG,UAAU,CAAC,OAAO,GAAE,OAAO,GAAG,cAAmB,GAAG,OAAO,CAAC,IAAI,CAAC;IAqBvE;;;;;OAKG;IACG,UAAU,IAAI,OAAO,CAAC,OAAO,CAAC;IAEpC;;OAEG;IACG,UAAU,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAqCrD;;OAEG;IACG,UAAU,CAAC,OAAO,CAAC,EAAE,kBAAkB;IAQ7C;;;OAGG;IACH,QAAQ;;;;;IAQR;;;;OAIG;IACG,YAAY,CAAC,OAAO,CAAC,EAAE,kBAAkB,GAAG,OAAO,CAAC,IAAI,CAAC;IAuB/D;;;OAGG;IACG,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAK/B;;OAEG;IACH,SAAS,CAAC,sBAAsB;IAIhC;;OAEG;IACH,SAAS,CAAC,sBAAsB;IAWhC;;;OAGG;IACH,SAAS,CAAC,WAAW,CAAC,UAAU,EAAE,OAAO;IAKzC;;OAEG;IACH,SAAS,CAAC,eAAe,IAAI,MAAM;IAInC;;;;;;OAMG;IACH,SAAS,CAAC,6BAA6B,CACnC,WAAW,EAAE,WAAW,EACxB,OAAO,GAAE;QAAE,cAAc,CAAC,EAAE,cAAc,CAAA;KAAO,GAClD,OAAO;IAUV;;;OAGG;cACa,cAAc,IAAI,OAAO,CAAC,OAAO,CAAC;IAQlD;;OAEG;IACH,SAAS,CAAC,mBAAmB,IAAI,OAAO;IAIxC;;;OAGG;IACH,SAAS,CAAC,YAAY,IAAI,OAAO;IAIjC;;;OAGG;cACa,qBAAqB,IAAI,OAAO,CAAC,IAAI,CAAC;IAyBtD;;;;;OAKG;WACU,IAAI,CAAC,OAAO,CAAC,EAAE,kBAAkB,EAAE,MAAM,CAAC,EAAE,aAAa,GAAG,OAAO,CAAC,WAAW,CAAC;CAKhG"}
|
|
@@ -1,15 +1,11 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
const log_1 = require("../log");
|
|
10
|
-
const key_value_store_1 = require("../storages/key_value_store");
|
|
11
|
-
const consts_1 = require("./consts");
|
|
12
|
-
const session_1 = require("./session");
|
|
1
|
+
import { EventEmitter } from 'node:events';
|
|
2
|
+
import { AsyncQueue } from '@sapphire/async-queue';
|
|
3
|
+
import ow from 'ow';
|
|
4
|
+
import { Configuration } from '../configuration.js';
|
|
5
|
+
import { log as defaultLog } from '../log.js';
|
|
6
|
+
import { KeyValueStore } from '../storages/key_value_store.js';
|
|
7
|
+
import { BLOCKED_STATUS_CODES, MAX_POOL_SIZE, PERSIST_STATE_KEY } from './consts.js';
|
|
8
|
+
import { Session } from './session.js';
|
|
13
9
|
/**
|
|
14
10
|
* Handles the rotation, creation and persistence of user-like sessions.
|
|
15
11
|
* Creates a pool of {@link Session} instances, that are randomly rotated.
|
|
@@ -75,119 +71,40 @@ const session_1 = require("./session");
|
|
|
75
71
|
*
|
|
76
72
|
* @category Scaling
|
|
77
73
|
*/
|
|
78
|
-
class SessionPool extends
|
|
74
|
+
export class SessionPool extends EventEmitter {
|
|
75
|
+
config;
|
|
76
|
+
log;
|
|
77
|
+
maxPoolSize;
|
|
78
|
+
createSessionFunction;
|
|
79
|
+
keyValueStore;
|
|
80
|
+
sessions = [];
|
|
81
|
+
sessionMap = new Map();
|
|
82
|
+
sessionOptions;
|
|
83
|
+
persistStateKeyValueStoreId;
|
|
84
|
+
persistStateKey;
|
|
85
|
+
_listener;
|
|
86
|
+
events;
|
|
87
|
+
blockedStatusCodes;
|
|
88
|
+
persistenceOptions;
|
|
89
|
+
isInitialized = false;
|
|
90
|
+
queue = new AsyncQueue();
|
|
79
91
|
/**
|
|
80
92
|
* @internal
|
|
81
93
|
*/
|
|
82
|
-
constructor(options = {}, config =
|
|
94
|
+
constructor(options = {}, config = Configuration.getGlobalConfig()) {
|
|
83
95
|
super();
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
value: void 0
|
|
95
|
-
});
|
|
96
|
-
Object.defineProperty(this, "maxPoolSize", {
|
|
97
|
-
enumerable: true,
|
|
98
|
-
configurable: true,
|
|
99
|
-
writable: true,
|
|
100
|
-
value: void 0
|
|
101
|
-
});
|
|
102
|
-
Object.defineProperty(this, "createSessionFunction", {
|
|
103
|
-
enumerable: true,
|
|
104
|
-
configurable: true,
|
|
105
|
-
writable: true,
|
|
106
|
-
value: void 0
|
|
107
|
-
});
|
|
108
|
-
Object.defineProperty(this, "keyValueStore", {
|
|
109
|
-
enumerable: true,
|
|
110
|
-
configurable: true,
|
|
111
|
-
writable: true,
|
|
112
|
-
value: void 0
|
|
113
|
-
});
|
|
114
|
-
Object.defineProperty(this, "sessions", {
|
|
115
|
-
enumerable: true,
|
|
116
|
-
configurable: true,
|
|
117
|
-
writable: true,
|
|
118
|
-
value: []
|
|
119
|
-
});
|
|
120
|
-
Object.defineProperty(this, "sessionMap", {
|
|
121
|
-
enumerable: true,
|
|
122
|
-
configurable: true,
|
|
123
|
-
writable: true,
|
|
124
|
-
value: new Map()
|
|
125
|
-
});
|
|
126
|
-
Object.defineProperty(this, "sessionOptions", {
|
|
127
|
-
enumerable: true,
|
|
128
|
-
configurable: true,
|
|
129
|
-
writable: true,
|
|
130
|
-
value: void 0
|
|
131
|
-
});
|
|
132
|
-
Object.defineProperty(this, "persistStateKeyValueStoreId", {
|
|
133
|
-
enumerable: true,
|
|
134
|
-
configurable: true,
|
|
135
|
-
writable: true,
|
|
136
|
-
value: void 0
|
|
137
|
-
});
|
|
138
|
-
Object.defineProperty(this, "persistStateKey", {
|
|
139
|
-
enumerable: true,
|
|
140
|
-
configurable: true,
|
|
141
|
-
writable: true,
|
|
142
|
-
value: void 0
|
|
143
|
-
});
|
|
144
|
-
Object.defineProperty(this, "_listener", {
|
|
145
|
-
enumerable: true,
|
|
146
|
-
configurable: true,
|
|
147
|
-
writable: true,
|
|
148
|
-
value: void 0
|
|
149
|
-
});
|
|
150
|
-
Object.defineProperty(this, "events", {
|
|
151
|
-
enumerable: true,
|
|
152
|
-
configurable: true,
|
|
153
|
-
writable: true,
|
|
154
|
-
value: void 0
|
|
155
|
-
});
|
|
156
|
-
Object.defineProperty(this, "blockedStatusCodes", {
|
|
157
|
-
enumerable: true,
|
|
158
|
-
configurable: true,
|
|
159
|
-
writable: true,
|
|
160
|
-
value: void 0
|
|
161
|
-
});
|
|
162
|
-
Object.defineProperty(this, "persistenceOptions", {
|
|
163
|
-
enumerable: true,
|
|
164
|
-
configurable: true,
|
|
165
|
-
writable: true,
|
|
166
|
-
value: void 0
|
|
167
|
-
});
|
|
168
|
-
Object.defineProperty(this, "isInitialized", {
|
|
169
|
-
enumerable: true,
|
|
170
|
-
configurable: true,
|
|
171
|
-
writable: true,
|
|
172
|
-
value: false
|
|
173
|
-
});
|
|
174
|
-
Object.defineProperty(this, "queue", {
|
|
175
|
-
enumerable: true,
|
|
176
|
-
configurable: true,
|
|
177
|
-
writable: true,
|
|
178
|
-
value: new async_queue_1.AsyncQueue()
|
|
179
|
-
});
|
|
180
|
-
(0, ow_1.default)(options, ow_1.default.object.exactShape({
|
|
181
|
-
maxPoolSize: ow_1.default.optional.number,
|
|
182
|
-
persistStateKeyValueStoreId: ow_1.default.optional.string,
|
|
183
|
-
persistStateKey: ow_1.default.optional.string,
|
|
184
|
-
createSessionFunction: ow_1.default.optional.function,
|
|
185
|
-
sessionOptions: ow_1.default.optional.object,
|
|
186
|
-
blockedStatusCodes: ow_1.default.optional.array.ofType(ow_1.default.number),
|
|
187
|
-
log: ow_1.default.optional.object,
|
|
188
|
-
persistenceOptions: ow_1.default.optional.object,
|
|
96
|
+
this.config = config;
|
|
97
|
+
ow(options, ow.object.exactShape({
|
|
98
|
+
maxPoolSize: ow.optional.number,
|
|
99
|
+
persistStateKeyValueStoreId: ow.optional.string,
|
|
100
|
+
persistStateKey: ow.optional.string,
|
|
101
|
+
createSessionFunction: ow.optional.function,
|
|
102
|
+
sessionOptions: ow.optional.object,
|
|
103
|
+
blockedStatusCodes: ow.optional.array.ofType(ow.number),
|
|
104
|
+
log: ow.optional.object,
|
|
105
|
+
persistenceOptions: ow.optional.object,
|
|
189
106
|
}));
|
|
190
|
-
const { maxPoolSize =
|
|
107
|
+
const { maxPoolSize = MAX_POOL_SIZE, persistStateKeyValueStoreId, persistStateKey = PERSIST_STATE_KEY, createSessionFunction, sessionOptions = {}, blockedStatusCodes = BLOCKED_STATUS_CODES, log = defaultLog, persistenceOptions = {
|
|
191
108
|
enable: true,
|
|
192
109
|
}, } = options;
|
|
193
110
|
this.config = config;
|
|
@@ -229,7 +146,7 @@ class SessionPool extends node_events_1.EventEmitter {
|
|
|
229
146
|
if (this.isInitialized) {
|
|
230
147
|
return;
|
|
231
148
|
}
|
|
232
|
-
this.keyValueStore = await
|
|
149
|
+
this.keyValueStore = await KeyValueStore.open(this.persistStateKeyValueStoreId, { config: this.config });
|
|
233
150
|
if (!this.persistenceOptions.enable) {
|
|
234
151
|
this.isInitialized = true;
|
|
235
152
|
return;
|
|
@@ -261,7 +178,7 @@ class SessionPool extends node_events_1.EventEmitter {
|
|
|
261
178
|
if (!this._hasSpaceForSession()) {
|
|
262
179
|
this._removeRetiredSessions();
|
|
263
180
|
}
|
|
264
|
-
const newSession = options instanceof
|
|
181
|
+
const newSession = options instanceof Session ? options : await this.createSessionFunction(this, { sessionOptions: options });
|
|
265
182
|
this.log.debug(`Adding new Session - ${newSession.id}`);
|
|
266
183
|
this._addSession(newSession);
|
|
267
184
|
}
|
|
@@ -331,7 +248,7 @@ class SessionPool extends node_events_1.EventEmitter {
|
|
|
331
248
|
});
|
|
332
249
|
// use half the interval of `persistState` to avoid race conditions
|
|
333
250
|
const persistStateIntervalMillis = this.config.get('persistStateIntervalMillis');
|
|
334
|
-
const timeoutSecs = persistStateIntervalMillis /
|
|
251
|
+
const timeoutSecs = persistStateIntervalMillis / 2_000;
|
|
335
252
|
await this.keyValueStore
|
|
336
253
|
.setValue(this.persistStateKey, this.getState(), {
|
|
337
254
|
timeoutSecs,
|
|
@@ -388,9 +305,9 @@ class SessionPool extends node_events_1.EventEmitter {
|
|
|
388
305
|
* @returns New session.
|
|
389
306
|
*/
|
|
390
307
|
_defaultCreateSessionFunction(sessionPool, options = {}) {
|
|
391
|
-
(
|
|
308
|
+
ow(options, ow.object.exactShape({ sessionOptions: ow.optional.object }));
|
|
392
309
|
const { sessionOptions = {} } = options;
|
|
393
|
-
return new
|
|
310
|
+
return new Session({
|
|
394
311
|
...this.sessionOptions,
|
|
395
312
|
...sessionOptions,
|
|
396
313
|
sessionPool,
|
|
@@ -455,5 +372,4 @@ class SessionPool extends node_events_1.EventEmitter {
|
|
|
455
372
|
return sessionPool;
|
|
456
373
|
}
|
|
457
374
|
}
|
|
458
|
-
exports.SessionPool = SessionPool;
|
|
459
375
|
//# sourceMappingURL=session_pool.js.map
|