@crawlee/core 4.0.0-beta.14 → 4.0.0-beta.140
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -14
- package/autoscaling/autoscaled_pool.d.ts +67 -172
- package/autoscaling/autoscaled_pool.js +182 -329
- package/autoscaling/concurrency_system.d.ts +269 -0
- package/autoscaling/concurrency_system.js +365 -0
- package/autoscaling/cpu_load_signal.d.ts +43 -0
- package/autoscaling/cpu_load_signal.js +47 -0
- package/autoscaling/event_loop_load_signal.d.ts +51 -0
- package/autoscaling/event_loop_load_signal.js +60 -0
- package/autoscaling/index.d.ts +6 -1
- package/autoscaling/index.js +6 -1
- package/autoscaling/load_signal.d.ts +100 -0
- package/autoscaling/load_signal.js +105 -0
- package/autoscaling/memory_load_signal.d.ts +47 -0
- package/autoscaling/memory_load_signal.js +106 -0
- package/autoscaling/snapshotter.d.ts +58 -163
- package/autoscaling/snapshotter.js +45 -263
- package/autoscaling/storage_backend_load_signal.d.ts +56 -0
- package/autoscaling/storage_backend_load_signal.js +73 -0
- package/autoscaling/system_status.d.ts +67 -89
- package/autoscaling/system_status.js +92 -122
- package/autoscaling/weighted_avg.d.ts +5 -0
- package/autoscaling/weighted_avg.js +14 -0
- package/byte_utils.d.ts +17 -0
- package/byte_utils.js +42 -0
- package/configuration.d.ts +96 -223
- package/configuration.js +170 -222
- package/cookie_utils.d.ts +3 -2
- package/cookie_utils.js +18 -7
- package/crawlers/context_pipeline.d.ts +10 -1
- package/crawlers/context_pipeline.js +31 -8
- package/crawlers/crawler_commons.d.ts +90 -126
- package/crawlers/crawler_commons.js +1 -108
- package/crawlers/error_snapshotter.d.ts +2 -5
- package/crawlers/error_snapshotter.js +7 -8
- package/crawlers/error_tracker.d.ts +0 -1
- package/crawlers/error_tracker.js +0 -1
- package/crawlers/index.d.ts +1 -3
- package/crawlers/index.js +0 -3
- package/crawlers/internals/types.d.ts +0 -1
- package/crawlers/internals/types.js +0 -1
- package/crawlers/statistics.d.ts +187 -63
- package/crawlers/statistics.js +354 -164
- package/debug.d.ts +36 -0
- package/debug.js +70 -0
- package/enqueue_links/enqueue_links.d.ts +60 -153
- package/enqueue_links/enqueue_links.js +38 -229
- package/enqueue_links/index.d.ts +0 -1
- package/enqueue_links/index.js +0 -1
- package/enqueue_links/shared.d.ts +49 -30
- package/enqueue_links/shared.js +94 -68
- package/errors.d.ts +72 -4
- package/errors.js +89 -5
- package/events/event_manager.d.ts +35 -9
- package/events/event_manager.js +12 -13
- package/events/index.d.ts +0 -1
- package/events/index.js +0 -1
- package/events/local_event_manager.d.ts +15 -3
- package/events/local_event_manager.js +39 -13
- package/http.d.ts +9 -0
- package/http.js +28 -0
- package/index.d.ts +7 -5
- package/index.js +8 -4
- package/iterables.d.ts +79 -0
- package/iterables.js +134 -0
- package/log.d.ts +78 -1
- package/log.js +106 -1
- package/memory-storage/consts.d.ts +4 -0
- package/memory-storage/consts.js +4 -0
- package/memory-storage/index.d.ts +1 -0
- package/memory-storage/index.js +1 -0
- package/memory-storage/memory-storage.d.ts +44 -0
- package/memory-storage/memory-storage.js +160 -0
- package/memory-storage/resource-clients/common/base-client.d.ts +4 -0
- package/memory-storage/resource-clients/common/base-client.js +6 -0
- package/memory-storage/resource-clients/dataset.d.ts +40 -0
- package/memory-storage/resource-clients/dataset.js +106 -0
- package/memory-storage/resource-clients/key-value-store.d.ts +63 -0
- package/memory-storage/resource-clients/key-value-store.js +199 -0
- package/memory-storage/resource-clients/request-queue.d.ts +77 -0
- package/memory-storage/resource-clients/request-queue.js +407 -0
- package/memory-storage/utils.d.ts +16 -0
- package/memory-storage/utils.js +41 -0
- package/owned_or_injected.d.ts +58 -0
- package/owned_or_injected.js +98 -0
- package/package.json +13 -12
- package/proxy_configuration.d.ts +22 -128
- package/proxy_configuration.js +32 -144
- package/recoverable_state.d.ts +83 -51
- package/recoverable_state.js +163 -72
- package/request.d.ts +57 -16
- package/request.js +130 -69
- package/router.d.ts +193 -21
- package/router.js +188 -43
- package/serialization.d.ts +0 -1
- package/serialization.js +15 -15
- package/service_locator.d.ts +165 -0
- package/service_locator.js +253 -0
- package/session_pool/consts.d.ts +1 -2
- package/session_pool/consts.js +1 -2
- package/session_pool/errors.d.ts +0 -1
- package/session_pool/errors.js +0 -1
- package/session_pool/fingerprint.d.ts +9 -0
- package/session_pool/fingerprint.js +30 -0
- package/session_pool/index.d.ts +0 -2
- package/session_pool/index.js +0 -2
- package/session_pool/session.d.ts +35 -88
- package/session_pool/session.js +101 -159
- package/session_pool/session_pool.d.ts +74 -91
- package/session_pool/session_pool.js +175 -165
- package/storages/batched_adds.d.ts +37 -0
- package/storages/batched_adds.js +73 -0
- package/storages/dataset.d.ts +109 -56
- package/storages/dataset.js +283 -149
- package/storages/index.d.ts +9 -9
- package/storages/index.js +7 -9
- package/storages/key_value_store.d.ts +183 -48
- package/storages/key_value_store.js +445 -171
- package/storages/key_value_store_codec.d.ts +32 -0
- package/storages/key_value_store_codec.js +113 -0
- package/storages/request_dedup_cache.d.ts +22 -0
- package/storages/request_dedup_cache.js +48 -0
- package/storages/request_list.d.ts +52 -109
- package/storages/request_list.js +183 -152
- package/storages/request_loader.d.ts +101 -0
- package/storages/request_loader.js +1 -0
- package/storages/request_manager.d.ts +33 -0
- package/storages/request_manager.js +1 -0
- package/storages/request_manager_tandem.d.ts +48 -19
- package/storages/request_manager_tandem.js +118 -45
- package/storages/request_queue.d.ts +290 -47
- package/storages/request_queue.js +762 -216
- package/storages/{sitemap_request_list.d.ts → sitemap_request_loader.d.ts} +45 -89
- package/storages/sitemap_request_loader.js +438 -0
- package/storages/storage_instance_manager.d.ts +87 -0
- package/storages/storage_instance_manager.js +256 -0
- package/storages/storage_stats.d.ts +48 -0
- package/storages/storage_stats.js +29 -0
- package/storages/throttling_request_manager.d.ts +239 -0
- package/storages/throttling_request_manager.js +646 -0
- package/storages/transaction.d.ts +252 -0
- package/storages/transaction.js +251 -0
- package/storages/utils.d.ts +59 -11
- package/storages/utils.js +75 -15
- package/system-info/cpu-info.d.ts +67 -0
- package/system-info/cpu-info.js +216 -0
- package/system-info/memory-info.d.ts +31 -0
- package/system-info/memory-info.js +115 -0
- package/system-info/ps-tree.d.ts +17 -0
- package/system-info/ps-tree.js +144 -0
- package/system-info/runtime.d.ts +14 -0
- package/system-info/runtime.js +80 -0
- package/typedefs.d.ts +0 -6
- package/typedefs.js +0 -1
- package/url.d.ts +9 -0
- package/url.js +11 -0
- package/validators.d.ts +22 -18
- package/validators.js +13 -18
- package/autoscaling/autoscaled_pool.d.ts.map +0 -1
- package/autoscaling/autoscaled_pool.js.map +0 -1
- package/autoscaling/index.d.ts.map +0 -1
- package/autoscaling/index.js.map +0 -1
- package/autoscaling/snapshotter.d.ts.map +0 -1
- package/autoscaling/snapshotter.js.map +0 -1
- package/autoscaling/system_status.d.ts.map +0 -1
- package/autoscaling/system_status.js.map +0 -1
- package/configuration.d.ts.map +0 -1
- package/configuration.js.map +0 -1
- package/cookie_utils.d.ts.map +0 -1
- package/cookie_utils.js.map +0 -1
- package/crawlers/context_pipeline.d.ts.map +0 -1
- package/crawlers/context_pipeline.js.map +0 -1
- package/crawlers/crawler_commons.d.ts.map +0 -1
- package/crawlers/crawler_commons.js.map +0 -1
- package/crawlers/crawler_utils.d.ts +0 -10
- package/crawlers/crawler_utils.d.ts.map +0 -1
- package/crawlers/crawler_utils.js +0 -12
- package/crawlers/crawler_utils.js.map +0 -1
- package/crawlers/error_snapshotter.d.ts.map +0 -1
- package/crawlers/error_snapshotter.js.map +0 -1
- package/crawlers/error_tracker.d.ts.map +0 -1
- package/crawlers/error_tracker.js.map +0 -1
- package/crawlers/index.d.ts.map +0 -1
- package/crawlers/index.js.map +0 -1
- package/crawlers/internals/types.d.ts.map +0 -1
- package/crawlers/internals/types.js.map +0 -1
- package/crawlers/statistics.d.ts.map +0 -1
- package/crawlers/statistics.js.map +0 -1
- package/enqueue_links/enqueue_links.d.ts.map +0 -1
- package/enqueue_links/enqueue_links.js.map +0 -1
- package/enqueue_links/index.d.ts.map +0 -1
- package/enqueue_links/index.js.map +0 -1
- package/enqueue_links/shared.d.ts.map +0 -1
- package/enqueue_links/shared.js.map +0 -1
- package/errors.d.ts.map +0 -1
- package/errors.js.map +0 -1
- package/events/event_manager.d.ts.map +0 -1
- package/events/event_manager.js.map +0 -1
- package/events/index.d.ts.map +0 -1
- package/events/index.js.map +0 -1
- package/events/local_event_manager.d.ts.map +0 -1
- package/events/local_event_manager.js.map +0 -1
- package/http_clients/base-http-client.d.ts +0 -140
- package/http_clients/base-http-client.d.ts.map +0 -1
- package/http_clients/base-http-client.js +0 -40
- package/http_clients/base-http-client.js.map +0 -1
- package/http_clients/form-data-like.d.ts +0 -67
- package/http_clients/form-data-like.d.ts.map +0 -1
- package/http_clients/form-data-like.js +0 -5
- package/http_clients/form-data-like.js.map +0 -1
- package/http_clients/got-scraping-http-client.d.ts +0 -20
- package/http_clients/got-scraping-http-client.d.ts.map +0 -1
- package/http_clients/got-scraping-http-client.js +0 -85
- package/http_clients/got-scraping-http-client.js.map +0 -1
- package/http_clients/index.d.ts +0 -3
- package/http_clients/index.d.ts.map +0 -1
- package/http_clients/index.js +0 -3
- package/http_clients/index.js.map +0 -1
- package/index.d.ts.map +0 -1
- package/index.js.map +0 -1
- package/log.d.ts.map +0 -1
- package/log.js.map +0 -1
- package/proxy_configuration.d.ts.map +0 -1
- package/proxy_configuration.js.map +0 -1
- package/recoverable_state.d.ts.map +0 -1
- package/recoverable_state.js.map +0 -1
- package/request.d.ts.map +0 -1
- package/request.js.map +0 -1
- package/router.d.ts.map +0 -1
- package/router.js.map +0 -1
- package/serialization.d.ts.map +0 -1
- package/serialization.js.map +0 -1
- package/session_pool/consts.d.ts.map +0 -1
- package/session_pool/consts.js.map +0 -1
- package/session_pool/errors.d.ts.map +0 -1
- package/session_pool/errors.js.map +0 -1
- package/session_pool/events.d.ts +0 -3
- package/session_pool/events.d.ts.map +0 -1
- package/session_pool/events.js +0 -3
- package/session_pool/events.js.map +0 -1
- package/session_pool/index.d.ts.map +0 -1
- package/session_pool/index.js.map +0 -1
- package/session_pool/session.d.ts.map +0 -1
- package/session_pool/session.js.map +0 -1
- package/session_pool/session_pool.d.ts.map +0 -1
- package/session_pool/session_pool.js.map +0 -1
- package/storages/access_checking.d.ts +0 -13
- package/storages/access_checking.d.ts.map +0 -1
- package/storages/access_checking.js +0 -14
- package/storages/access_checking.js.map +0 -1
- package/storages/dataset.d.ts.map +0 -1
- package/storages/dataset.js.map +0 -1
- package/storages/index.d.ts.map +0 -1
- package/storages/index.js.map +0 -1
- package/storages/key_value_store.d.ts.map +0 -1
- package/storages/key_value_store.js.map +0 -1
- package/storages/request_list.d.ts.map +0 -1
- package/storages/request_list.js.map +0 -1
- package/storages/request_list_adapter.d.ts +0 -58
- package/storages/request_list_adapter.d.ts.map +0 -1
- package/storages/request_list_adapter.js +0 -81
- package/storages/request_list_adapter.js.map +0 -1
- package/storages/request_manager_tandem.d.ts.map +0 -1
- package/storages/request_manager_tandem.js.map +0 -1
- package/storages/request_provider.d.ts +0 -371
- package/storages/request_provider.d.ts.map +0 -1
- package/storages/request_provider.js +0 -585
- package/storages/request_provider.js.map +0 -1
- package/storages/request_queue.d.ts.map +0 -1
- package/storages/request_queue.js.map +0 -1
- package/storages/request_queue_v2.d.ts +0 -87
- package/storages/request_queue_v2.d.ts.map +0 -1
- package/storages/request_queue_v2.js +0 -438
- package/storages/request_queue_v2.js.map +0 -1
- package/storages/sitemap_request_list.d.ts.map +0 -1
- package/storages/sitemap_request_list.js +0 -430
- package/storages/sitemap_request_list.js.map +0 -1
- package/storages/storage_manager.d.ts +0 -58
- package/storages/storage_manager.d.ts.map +0 -1
- package/storages/storage_manager.js +0 -105
- package/storages/storage_manager.js.map +0 -1
- package/storages/utils.d.ts.map +0 -1
- package/storages/utils.js.map +0 -1
- package/typedefs.d.ts.map +0 -1
- package/typedefs.js.map +0 -1
- package/validators.d.ts.map +0 -1
- package/validators.js.map +0 -1
package/session_pool/session.js
CHANGED
|
@@ -1,10 +1,28 @@
|
|
|
1
|
-
import { EventEmitter } from 'node:events';
|
|
2
|
-
import ow from 'ow';
|
|
3
1
|
import { CookieJar } from 'tough-cookie';
|
|
2
|
+
import { z } from 'zod';
|
|
4
3
|
import { cryptoRandomObjectId } from '@apify/utilities';
|
|
5
|
-
import {
|
|
6
|
-
import {
|
|
7
|
-
import {
|
|
4
|
+
import { getDefaultCookieExpirationDate } from '../cookie_utils.js';
|
|
5
|
+
import { serviceLocator } from '../service_locator.js';
|
|
6
|
+
import { parseArgument, schemas, validators } from '../validators.js';
|
|
7
|
+
// `schemas.anyObject` passes values through by reference (object schemas return a pruned plain
|
|
8
|
+
// copy), so class instances like cookie jars and loggers keep their prototype.
|
|
9
|
+
const sessionOptionsSchema = z.strictObject({
|
|
10
|
+
id: z.string().default(() => `session_${cryptoRandomObjectId(10)}`),
|
|
11
|
+
cookieJar: schemas.anyObject.default(() => new CookieJar()),
|
|
12
|
+
proxyInfo: schemas.anyObject.optional(),
|
|
13
|
+
maxAgeSecs: schemas.anyNumber.default(3000),
|
|
14
|
+
userData: schemas.anyObject.default(() => ({})),
|
|
15
|
+
maxErrorScore: schemas.anyNumber.default(3),
|
|
16
|
+
errorScoreDecrement: schemas.anyNumber.default(0.5),
|
|
17
|
+
createdAt: z.date().default(() => new Date()),
|
|
18
|
+
expiresAt: z.date().optional(),
|
|
19
|
+
usageCount: schemas.anyNumber.default(0),
|
|
20
|
+
errorScore: schemas.anyNumber.default(0),
|
|
21
|
+
maxUsageCount: schemas.anyNumber.default(50),
|
|
22
|
+
retired: z.boolean().default(false),
|
|
23
|
+
log: validators.logger.default(() => serviceLocator.getLogger()),
|
|
24
|
+
fingerprint: schemas.anyObject.optional(),
|
|
25
|
+
});
|
|
8
26
|
/**
|
|
9
27
|
* Sessions are used to store information such as cookies and can be used for generating fingerprints and proxy sessions.
|
|
10
28
|
* You can imagine each session as a specific user, with its own cookies, IP (via proxy) and potentially a unique browser fingerprint.
|
|
@@ -13,83 +31,79 @@ import { EVENT_SESSION_RETIRED } from './events.js';
|
|
|
13
31
|
*/
|
|
14
32
|
export class Session {
|
|
15
33
|
id;
|
|
16
|
-
maxAgeSecs;
|
|
17
34
|
userData;
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
35
|
+
#maxErrorScore;
|
|
36
|
+
#errorScoreDecrement;
|
|
37
|
+
#createdAt;
|
|
38
|
+
#expiresAt;
|
|
39
|
+
#usageCount;
|
|
40
|
+
#maxUsageCount;
|
|
41
|
+
#errorScore;
|
|
42
|
+
#retired = false;
|
|
43
|
+
#proxyInfo;
|
|
44
|
+
#cookieJar;
|
|
45
|
+
#fingerprint;
|
|
46
|
+
#log;
|
|
29
47
|
get errorScore() {
|
|
30
|
-
return this
|
|
48
|
+
return this.#errorScore;
|
|
31
49
|
}
|
|
32
50
|
get usageCount() {
|
|
33
|
-
return this
|
|
51
|
+
return this.#usageCount;
|
|
34
52
|
}
|
|
35
53
|
get maxErrorScore() {
|
|
36
|
-
return this
|
|
54
|
+
return this.#maxErrorScore;
|
|
37
55
|
}
|
|
38
56
|
get errorScoreDecrement() {
|
|
39
|
-
return this
|
|
57
|
+
return this.#errorScoreDecrement;
|
|
40
58
|
}
|
|
41
59
|
get expiresAt() {
|
|
42
|
-
return this
|
|
60
|
+
return this.#expiresAt;
|
|
43
61
|
}
|
|
44
62
|
get createdAt() {
|
|
45
|
-
return this
|
|
63
|
+
return this.#createdAt;
|
|
46
64
|
}
|
|
47
65
|
get maxUsageCount() {
|
|
48
|
-
return this
|
|
66
|
+
return this.#maxUsageCount;
|
|
49
67
|
}
|
|
50
68
|
get cookieJar() {
|
|
51
|
-
return this
|
|
69
|
+
return this.#cookieJar;
|
|
52
70
|
}
|
|
53
71
|
get proxyInfo() {
|
|
54
|
-
return this
|
|
72
|
+
return this.#proxyInfo;
|
|
73
|
+
}
|
|
74
|
+
get fingerprint() {
|
|
75
|
+
return this.#fingerprint;
|
|
76
|
+
}
|
|
77
|
+
set fingerprint(fingerprint) {
|
|
78
|
+
this.#fingerprint = fingerprint;
|
|
79
|
+
}
|
|
80
|
+
/**
|
|
81
|
+
* `true` once {@link Session.retire|`retire()`} has been called. Retirement is terminal:
|
|
82
|
+
* a retired session is never picked by the pool and cannot be revived via `markGood()`.
|
|
83
|
+
*/
|
|
84
|
+
get retired() {
|
|
85
|
+
return this.#retired;
|
|
55
86
|
}
|
|
56
87
|
/**
|
|
57
88
|
* Session configuration.
|
|
58
89
|
*/
|
|
59
|
-
constructor(options) {
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
maxAgeSecs: ow.optional.number,
|
|
66
|
-
userData: ow.optional.object,
|
|
67
|
-
maxErrorScore: ow.optional.number,
|
|
68
|
-
errorScoreDecrement: ow.optional.number,
|
|
69
|
-
createdAt: ow.optional.date,
|
|
70
|
-
expiresAt: ow.optional.date,
|
|
71
|
-
usageCount: ow.optional.number,
|
|
72
|
-
errorScore: ow.optional.number,
|
|
73
|
-
maxUsageCount: ow.optional.number,
|
|
74
|
-
log: ow.optional.object,
|
|
75
|
-
}));
|
|
76
|
-
const { sessionPool, id = `session_${cryptoRandomObjectId(10)}`, cookieJar = new CookieJar(), proxyInfo = undefined, maxAgeSecs = 3000, userData = {}, maxErrorScore = 3, errorScoreDecrement = 0.5, createdAt = new Date(), usageCount = 0, errorScore = 0, maxUsageCount = 50, log = defaultLog, } = options;
|
|
77
|
-
const { expiresAt = getDefaultCookieExpirationDate(maxAgeSecs) } = options;
|
|
78
|
-
this.log = log.child({ prefix: 'Session' });
|
|
79
|
-
this._cookieJar = cookieJar.setCookie ? cookieJar : CookieJar.fromJSON(JSON.stringify(cookieJar));
|
|
80
|
-
this._proxyInfo = proxyInfo;
|
|
90
|
+
constructor(options = {}) {
|
|
91
|
+
const { id, cookieJar, proxyInfo, maxAgeSecs, userData, maxErrorScore, errorScoreDecrement, createdAt, usageCount, errorScore, maxUsageCount, retired, log, fingerprint, expiresAt = getDefaultCookieExpirationDate(maxAgeSecs), } = parseArgument(options, sessionOptionsSchema);
|
|
92
|
+
this.#log = log.child({ prefix: 'Session' });
|
|
93
|
+
this.#cookieJar = cookieJar.setCookie ? cookieJar : CookieJar.fromJSON(JSON.stringify(cookieJar));
|
|
94
|
+
this.#proxyInfo = proxyInfo;
|
|
95
|
+
this.#fingerprint = fingerprint;
|
|
81
96
|
this.id = id;
|
|
82
|
-
this.maxAgeSecs = maxAgeSecs;
|
|
83
97
|
this.userData = userData;
|
|
84
|
-
this
|
|
85
|
-
this
|
|
98
|
+
this.#maxErrorScore = maxErrorScore;
|
|
99
|
+
this.#errorScoreDecrement = errorScoreDecrement;
|
|
86
100
|
// Internal
|
|
87
|
-
this
|
|
88
|
-
this
|
|
89
|
-
this
|
|
90
|
-
this
|
|
91
|
-
this
|
|
92
|
-
this
|
|
101
|
+
this.#expiresAt = expiresAt;
|
|
102
|
+
this.#createdAt = createdAt;
|
|
103
|
+
this.#usageCount = usageCount; // indicates how many times the session has been used
|
|
104
|
+
this.#errorScore = errorScore; // indicates number of markBaded request with the session
|
|
105
|
+
this.#maxUsageCount = maxUsageCount;
|
|
106
|
+
this.#retired = retired;
|
|
93
107
|
}
|
|
94
108
|
/**
|
|
95
109
|
* Indicates whether the session is blocked.
|
|
@@ -115,21 +129,21 @@ export class Session {
|
|
|
115
129
|
}
|
|
116
130
|
/**
|
|
117
131
|
* Indicates whether the session can be used for next requests.
|
|
118
|
-
* Session is usable when it is not expired, not blocked and the maximum usage count has not be reached.
|
|
132
|
+
* Session is usable when it is not retired, not expired, not blocked and the maximum usage count has not be reached.
|
|
119
133
|
*/
|
|
120
134
|
isUsable() {
|
|
121
|
-
return !this.isBlocked() && !this.isExpired() && !this.isMaxUsageCountReached();
|
|
135
|
+
return !this.#retired && !this.isBlocked() && !this.isExpired() && !this.isMaxUsageCountReached();
|
|
122
136
|
}
|
|
123
137
|
/**
|
|
124
138
|
* This method should be called after a successful session usage.
|
|
125
139
|
* It increases `usageCount` and potentially lowers the `errorScore` by the `errorScoreDecrement`.
|
|
126
140
|
*/
|
|
127
141
|
markGood() {
|
|
128
|
-
this
|
|
129
|
-
if (this
|
|
130
|
-
this
|
|
142
|
+
this.#usageCount += 1;
|
|
143
|
+
if (this.#errorScore > 0) {
|
|
144
|
+
this.#errorScore -= this.#errorScoreDecrement;
|
|
131
145
|
}
|
|
132
|
-
this.
|
|
146
|
+
this.maybeSelfRetire();
|
|
133
147
|
}
|
|
134
148
|
/**
|
|
135
149
|
* Gets session state for persistence in KeyValueStore.
|
|
@@ -139,8 +153,9 @@ export class Session {
|
|
|
139
153
|
return {
|
|
140
154
|
id: this.id,
|
|
141
155
|
cookieJar: this.cookieJar.toJSON(),
|
|
142
|
-
proxyInfo: this
|
|
156
|
+
proxyInfo: this.#proxyInfo,
|
|
143
157
|
userData: this.userData,
|
|
158
|
+
fingerprint: this.#fingerprint,
|
|
144
159
|
maxErrorScore: this.maxErrorScore,
|
|
145
160
|
errorScoreDecrement: this.errorScoreDecrement,
|
|
146
161
|
expiresAt: this.expiresAt.toISOString(),
|
|
@@ -148,89 +163,31 @@ export class Session {
|
|
|
148
163
|
usageCount: this.usageCount,
|
|
149
164
|
maxUsageCount: this.maxUsageCount,
|
|
150
165
|
errorScore: this.errorScore,
|
|
166
|
+
retired: this.#retired,
|
|
151
167
|
};
|
|
152
168
|
}
|
|
153
169
|
/**
|
|
154
|
-
*
|
|
155
|
-
*
|
|
156
|
-
*
|
|
157
|
-
*
|
|
158
|
-
*
|
|
170
|
+
* Permanently retires the session — `isUsable()` will return `false` from here on,
|
|
171
|
+
* and no `markGood()` / `markBad()` can revive it. Calling `retire()` again is a no-op.
|
|
172
|
+
*
|
|
173
|
+
* Use this when you're confident the session itself is the problem (e.g. a `403` response).
|
|
174
|
+
* For transient external failures (such as `5XX` responses), use `markBad()` instead.
|
|
159
175
|
*/
|
|
160
176
|
retire() {
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
this
|
|
164
|
-
|
|
165
|
-
this
|
|
177
|
+
if (this.#retired)
|
|
178
|
+
return;
|
|
179
|
+
this.#errorScore += this.#maxErrorScore;
|
|
180
|
+
this.#usageCount += 1;
|
|
181
|
+
this.#retired = true;
|
|
166
182
|
}
|
|
167
183
|
/**
|
|
168
184
|
* Increases usage and error count.
|
|
169
185
|
* Should be used when the session has been used unsuccessfully. For example because of timeouts.
|
|
170
186
|
*/
|
|
171
187
|
markBad() {
|
|
172
|
-
this
|
|
173
|
-
this
|
|
174
|
-
this.
|
|
175
|
-
}
|
|
176
|
-
/**
|
|
177
|
-
* With certain status codes: `401`, `403` or `429` we can be certain
|
|
178
|
-
* that the target website is blocking us. This function helps to do this conveniently
|
|
179
|
-
* by retiring the session when such code is received. Optionally, the default status
|
|
180
|
-
* codes can be extended in the second parameter.
|
|
181
|
-
* @param statusCode HTTP status code.
|
|
182
|
-
* @returns Whether the session was retired.
|
|
183
|
-
*/
|
|
184
|
-
retireOnBlockedStatusCodes(statusCode) {
|
|
185
|
-
// eslint-disable-next-line dot-notation -- accessing private property
|
|
186
|
-
const isBlocked = this.sessionPool['blockedStatusCodes'].includes(statusCode);
|
|
187
|
-
if (isBlocked) {
|
|
188
|
-
this.retire();
|
|
189
|
-
}
|
|
190
|
-
return isBlocked;
|
|
191
|
-
}
|
|
192
|
-
/**
|
|
193
|
-
* Saves cookies from an HTTP response to be used with the session.
|
|
194
|
-
* It expects an object with a `headers` property that's either an `Object`
|
|
195
|
-
* (typical Node.js responses) or a `Function` (Puppeteer Response).
|
|
196
|
-
*
|
|
197
|
-
* It then parses and saves the cookies from the `set-cookie` header, if available.
|
|
198
|
-
*/
|
|
199
|
-
setCookiesFromResponse(response) {
|
|
200
|
-
try {
|
|
201
|
-
const cookies = getCookiesFromResponse(response).filter((c) => c);
|
|
202
|
-
this._setCookies(cookies, response.url);
|
|
203
|
-
}
|
|
204
|
-
catch (e) {
|
|
205
|
-
const err = e;
|
|
206
|
-
// if invalid Cookie header is provided just log the exception.
|
|
207
|
-
this.log.exception(err, 'Could not get cookies from response');
|
|
208
|
-
}
|
|
209
|
-
}
|
|
210
|
-
/**
|
|
211
|
-
* Saves an array with cookie objects to be used with the session.
|
|
212
|
-
* The objects should be in the format that
|
|
213
|
-
* [Puppeteer uses](https://pptr.dev/#?product=Puppeteer&version=v2.0.0&show=api-pagecookiesurls),
|
|
214
|
-
* but you can also use this function to set cookies manually:
|
|
215
|
-
*
|
|
216
|
-
* ```
|
|
217
|
-
* [
|
|
218
|
-
* { name: 'cookie1', value: 'my-cookie' },
|
|
219
|
-
* { name: 'cookie2', value: 'your-cookie' }
|
|
220
|
-
* ]
|
|
221
|
-
* ```
|
|
222
|
-
*/
|
|
223
|
-
setCookies(cookies, url) {
|
|
224
|
-
const normalizedCookies = cookies.map((c) => browserPoolCookieToToughCookie(c, this.maxAgeSecs));
|
|
225
|
-
this._setCookies(normalizedCookies, url);
|
|
226
|
-
}
|
|
227
|
-
/**
|
|
228
|
-
* Returns cookies in a format compatible with puppeteer/playwright and ready to be used with `page.setCookie`.
|
|
229
|
-
* @param url website url. Only cookies stored for this url will be returned
|
|
230
|
-
*/
|
|
231
|
-
getCookies(url) {
|
|
232
|
-
const cookies = this.cookieJar.getCookiesSync(url);
|
|
233
|
-
return cookies.map((c) => toughCookieToBrowserPoolCookie(c));
|
|
188
|
+
this.#errorScore += 1;
|
|
189
|
+
this.#usageCount += 1;
|
|
190
|
+
this.maybeSelfRetire();
|
|
234
191
|
}
|
|
235
192
|
/**
|
|
236
193
|
* Returns cookies saved with the session in the typical
|
|
@@ -238,41 +195,26 @@ export class Session {
|
|
|
238
195
|
* a cookie header or elsewhere.
|
|
239
196
|
* @returns Represents `Cookie` header.
|
|
240
197
|
*/
|
|
241
|
-
getCookieString(url) {
|
|
242
|
-
return this.cookieJar.
|
|
198
|
+
async getCookieString(url) {
|
|
199
|
+
return this.cookieJar.getCookieString(url, {});
|
|
243
200
|
}
|
|
244
201
|
/**
|
|
245
202
|
* Sets a cookie within this session for the specific URL.
|
|
246
203
|
*/
|
|
247
|
-
setCookie(rawCookie, url) {
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
/**
|
|
251
|
-
* Sets cookies.
|
|
252
|
-
*/
|
|
253
|
-
_setCookies(cookies, url) {
|
|
254
|
-
const errorMessages = [];
|
|
255
|
-
for (const cookie of cookies) {
|
|
256
|
-
try {
|
|
257
|
-
this.cookieJar.setCookieSync(cookie, url, { ignoreError: false });
|
|
258
|
-
}
|
|
259
|
-
catch (e) {
|
|
260
|
-
const err = e;
|
|
261
|
-
errorMessages.push(err.message);
|
|
262
|
-
}
|
|
204
|
+
async setCookie(rawCookie, url) {
|
|
205
|
+
try {
|
|
206
|
+
await this.cookieJar.setCookie(rawCookie, url);
|
|
263
207
|
}
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
this.log.debug('Could not set cookies.', { errorMessages });
|
|
208
|
+
catch (e) {
|
|
209
|
+
this.#log.warning('Could not set cookie.', { url, error: e.message });
|
|
267
210
|
}
|
|
268
211
|
}
|
|
269
212
|
/**
|
|
270
213
|
* Checks if session is not usable. if it is not retires the session.
|
|
271
214
|
*/
|
|
272
|
-
|
|
215
|
+
maybeSelfRetire() {
|
|
273
216
|
if (!this.isUsable()) {
|
|
274
217
|
this.retire();
|
|
275
218
|
}
|
|
276
219
|
}
|
|
277
220
|
}
|
|
278
|
-
//# sourceMappingURL=session.js.map
|
|
@@ -1,24 +1,28 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import type { Log } from '@apify/log';
|
|
3
|
-
import { Configuration } from '../configuration.js';
|
|
1
|
+
import type { ISessionPool } from '@crawlee/types';
|
|
4
2
|
import type { PersistenceOptions } from '../crawlers/statistics.js';
|
|
5
|
-
import type {
|
|
6
|
-
import { KeyValueStore } from '../storages/key_value_store.js';
|
|
3
|
+
import type { CrawleeLogger } from '../log.js';
|
|
7
4
|
import type { SessionOptions } from './session.js';
|
|
8
5
|
import { Session } from './session.js';
|
|
6
|
+
declare const SESSION_REUSE_STRATEGIES: readonly ["random", "round-robin", "use-until-failure"];
|
|
7
|
+
export type SessionReuseStrategy = (typeof SESSION_REUSE_STRATEGIES)[number];
|
|
9
8
|
/**
|
|
10
9
|
* Factory user-function which creates customized {@link Session} instances.
|
|
11
10
|
*/
|
|
12
11
|
export interface CreateSession {
|
|
13
12
|
/**
|
|
14
|
-
* @param
|
|
15
|
-
* @param options
|
|
13
|
+
* @param options.sessionOptions Per-call session options already merged with the pool-wide defaults.
|
|
16
14
|
*/
|
|
17
|
-
(
|
|
15
|
+
(options?: {
|
|
18
16
|
sessionOptions?: SessionOptions;
|
|
19
17
|
}): Session | Promise<Session>;
|
|
20
18
|
}
|
|
21
19
|
export interface SessionPoolOptions {
|
|
20
|
+
/**
|
|
21
|
+
* Unique identifier for this session pool instance. Used to generate a unique
|
|
22
|
+
* persistence key when `persistStateKey` is not provided.
|
|
23
|
+
* If not provided, an auto-incrementing ID is used.
|
|
24
|
+
*/
|
|
25
|
+
id?: string | number;
|
|
22
26
|
/**
|
|
23
27
|
* Maximum size of the pool. Indicates how many sessions are rotated.
|
|
24
28
|
* @default 1000
|
|
@@ -29,24 +33,26 @@ export interface SessionPoolOptions {
|
|
|
29
33
|
/** Name or Id of `KeyValueStore` where is the `SessionPool` state stored. */
|
|
30
34
|
persistStateKeyValueStoreId?: string;
|
|
31
35
|
/**
|
|
32
|
-
* Session pool persists
|
|
33
|
-
* @default
|
|
36
|
+
* Session pool persists its state under this key in Key value store.
|
|
37
|
+
* @default CRAWLEE_SESSION_POOL_STATE_{id}
|
|
34
38
|
*/
|
|
35
39
|
persistStateKey?: string;
|
|
36
40
|
/**
|
|
37
41
|
* Custom function that should return a `Session` instance, or a promise resolving to such instance.
|
|
38
|
-
* Any error thrown from this function will terminate the process.
|
|
39
|
-
*
|
|
42
|
+
* Any error thrown from this function will terminate the process. Receives `{ sessionOptions }`
|
|
43
|
+
* already merged from the pool-wide defaults and the per-call overrides.
|
|
40
44
|
*/
|
|
41
45
|
createSessionFunction?: CreateSession;
|
|
42
46
|
/**
|
|
43
|
-
*
|
|
44
|
-
*
|
|
45
|
-
*
|
|
47
|
+
* Strategy for picking sessions from the pool.
|
|
48
|
+
* - `'random'` (default): fills the pool up to `maxPoolSize`, then picks a random usable session
|
|
49
|
+
* - `'round-robin'`: fills the pool up to `maxPoolSize`, then reuses sessions cycling through them in order
|
|
50
|
+
* - `'use-until-failure'`: always reuses the same session until it is retired, then moves to the next one
|
|
51
|
+
* @default 'random'
|
|
46
52
|
*/
|
|
47
|
-
|
|
53
|
+
sessionReuseStrategy?: SessionReuseStrategy;
|
|
48
54
|
/** @internal */
|
|
49
|
-
log?:
|
|
55
|
+
log?: CrawleeLogger;
|
|
50
56
|
/**
|
|
51
57
|
* Control how and when to persist the state of the session pool.
|
|
52
58
|
*/
|
|
@@ -58,20 +64,8 @@ export interface SessionPoolOptions {
|
|
|
58
64
|
* When some session is marked as blocked, it is removed and new one is created instead (the pool never returns an unusable session).
|
|
59
65
|
* Learn more in the {@doclink guides/session-management | Session management guide}.
|
|
60
66
|
*
|
|
61
|
-
*
|
|
62
|
-
*
|
|
63
|
-
* Session pool is already integrated into crawlers, and it can significantly improve your scraper
|
|
64
|
-
* performance with just 2 lines of code.
|
|
65
|
-
*
|
|
66
|
-
* **Example usage:**
|
|
67
|
-
*
|
|
68
|
-
* ```javascript
|
|
69
|
-
* const crawler = new CheerioCrawler({
|
|
70
|
-
* useSessionPool: true,
|
|
71
|
-
* persistCookiesPerSession: true,
|
|
72
|
-
* // ...
|
|
73
|
-
* })
|
|
74
|
-
* ```
|
|
67
|
+
* Session pool is already integrated into crawlers and is always active.
|
|
68
|
+
* All public methods are lazy-initialized — the pool initializes itself on first use.
|
|
75
69
|
*
|
|
76
70
|
* You can configure the pool with many options. See the {@link SessionPoolOptions}.
|
|
77
71
|
* Session pool is by default persisted in default {@link KeyValueStore}.
|
|
@@ -81,7 +75,7 @@ export interface SessionPoolOptions {
|
|
|
81
75
|
* **Advanced usage:**
|
|
82
76
|
*
|
|
83
77
|
* ```javascript
|
|
84
|
-
* const sessionPool =
|
|
78
|
+
* const sessionPool = new SessionPool({
|
|
85
79
|
* maxPoolSize: 25,
|
|
86
80
|
* sessionOptions:{
|
|
87
81
|
* maxAgeSecs: 10,
|
|
@@ -117,40 +111,31 @@ export interface SessionPoolOptions {
|
|
|
117
111
|
*
|
|
118
112
|
* @category Scaling
|
|
119
113
|
*/
|
|
120
|
-
export declare class SessionPool
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
protected _listener: () => Promise<void>;
|
|
132
|
-
protected events: EventManager;
|
|
133
|
-
protected readonly blockedStatusCodes: number[];
|
|
134
|
-
protected persistenceOptions: PersistenceOptions;
|
|
135
|
-
protected isInitialized: boolean;
|
|
136
|
-
private queue;
|
|
137
|
-
/**
|
|
138
|
-
* @internal
|
|
139
|
-
*/
|
|
140
|
-
constructor(options?: SessionPoolOptions, config?: Configuration);
|
|
114
|
+
export declare class SessionPool implements ISessionPool {
|
|
115
|
+
#private;
|
|
116
|
+
readonly id: string;
|
|
117
|
+
private maxPoolSize;
|
|
118
|
+
private createSessionFunction;
|
|
119
|
+
private keyValueStore?;
|
|
120
|
+
private sessionMap;
|
|
121
|
+
private sessionOptions;
|
|
122
|
+
private persistStateKeyValueStoreId?;
|
|
123
|
+
private persistStateKey;
|
|
124
|
+
constructor(options?: SessionPoolOptions);
|
|
141
125
|
/**
|
|
142
126
|
* Gets count of usable sessions in the pool.
|
|
143
127
|
*/
|
|
144
|
-
|
|
128
|
+
usableSessionsCount(): Promise<number>;
|
|
145
129
|
/**
|
|
146
130
|
* Gets count of retired sessions in the pool.
|
|
147
131
|
*/
|
|
148
|
-
|
|
132
|
+
retiredSessionsCount(): Promise<number>;
|
|
149
133
|
/**
|
|
150
134
|
* Starts periodic state persistence and potentially loads SessionPool state from {@link KeyValueStore}.
|
|
151
|
-
*
|
|
135
|
+
* Called automatically on first use of any public method.
|
|
152
136
|
*/
|
|
153
|
-
|
|
137
|
+
private ensureInitialized;
|
|
138
|
+
private setupPool;
|
|
154
139
|
/**
|
|
155
140
|
* Adds a new session to the session pool. The pool automatically creates sessions up to the maximum size of the pool,
|
|
156
141
|
* but this allows you to add more sessions once the max pool size is reached.
|
|
@@ -170,12 +155,9 @@ export declare class SessionPool extends EventEmitter {
|
|
|
170
155
|
* If there is space for new session, it creates and returns new session.
|
|
171
156
|
* If the session pool is full, it picks a session from the pool,
|
|
172
157
|
* If the picked session is usable it is returned, otherwise it creates and returns a new one.
|
|
158
|
+
* @param [sessionId] If provided, it returns the usable session with this id, `undefined` otherwise.
|
|
173
159
|
*/
|
|
174
|
-
getSession(): Promise<Session>;
|
|
175
|
-
/**
|
|
176
|
-
* Gets session based on the provided session id or `undefined.
|
|
177
|
-
*/
|
|
178
|
-
getSession(sessionId: string): Promise<Session>;
|
|
160
|
+
getSession(sessionId?: string): Promise<Session | undefined>;
|
|
179
161
|
/**
|
|
180
162
|
* @param options - Override the persistence options provided in the constructor
|
|
181
163
|
*/
|
|
@@ -184,75 +166,76 @@ export declare class SessionPool extends EventEmitter {
|
|
|
184
166
|
* Returns an object representing the internal state of the `SessionPool` instance.
|
|
185
167
|
* Note that the object's fields can change in future releases.
|
|
186
168
|
*/
|
|
187
|
-
getState(): {
|
|
169
|
+
getState(): Promise<{
|
|
188
170
|
usableSessionsCount: number;
|
|
189
171
|
retiredSessionsCount: number;
|
|
190
172
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
191
|
-
sessions: import("
|
|
192
|
-
}
|
|
173
|
+
sessions: import("@crawlee/types").SessionState[];
|
|
174
|
+
}>;
|
|
193
175
|
/**
|
|
194
176
|
* Persists the current state of the `SessionPool` into the default {@link KeyValueStore}.
|
|
195
177
|
* The state is persisted automatically in regular intervals.
|
|
196
178
|
* @param options - Override the persistence options provided in the constructor
|
|
197
179
|
*/
|
|
198
180
|
persistState(options?: PersistenceOptions): Promise<void>;
|
|
181
|
+
[Symbol.asyncDispose](): Promise<void>;
|
|
199
182
|
/**
|
|
200
183
|
* Removes listener from `persistState` event.
|
|
201
184
|
* This function should be called after you are done with using the `SessionPool` instance.
|
|
185
|
+
* @param options - Set `persistState` to false when the final state was already persisted by the event manager.
|
|
202
186
|
*/
|
|
203
|
-
teardown(
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
*/
|
|
207
|
-
protected _throwIfNotInitialized(): void;
|
|
187
|
+
teardown({ persistState }?: {
|
|
188
|
+
persistState?: boolean;
|
|
189
|
+
}): Promise<void>;
|
|
208
190
|
/**
|
|
209
191
|
* Removes retired `Session` instances from `SessionPool`.
|
|
210
192
|
*/
|
|
211
|
-
|
|
193
|
+
private removeRetiredSessions;
|
|
212
194
|
/**
|
|
213
195
|
* Adds `Session` instance to `SessionPool`.
|
|
214
196
|
* @param newSession `Session` instance to be added.
|
|
215
197
|
*/
|
|
216
|
-
|
|
198
|
+
private registerSession;
|
|
217
199
|
/**
|
|
218
200
|
* Gets random index.
|
|
219
201
|
*/
|
|
220
|
-
|
|
202
|
+
private getRandomIndex;
|
|
221
203
|
/**
|
|
222
204
|
* Creates new session without any extra behavior.
|
|
223
|
-
* @param sessionPool
|
|
224
205
|
* @param [options]
|
|
225
206
|
* @param [options.sessionOptions] The configuration options for the session being created.
|
|
226
207
|
* @returns New session.
|
|
227
208
|
*/
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
209
|
+
private defaultCreateSessionFunction;
|
|
210
|
+
/**
|
|
211
|
+
* Invokes `createSessionFunction` with `sessionOptions` already merged from pool-wide defaults and
|
|
212
|
+
* the supplied per-call overrides, so custom implementations don't need to spread `pool.sessionOptions` themselves.
|
|
213
|
+
*
|
|
214
|
+
* A default {@link SessionFingerprint} is generated up front (host OS as
|
|
215
|
+
* `platform`, a random valid `browser`/`device` for that platform). Pool-wide
|
|
216
|
+
* and per-call options override it, and a persisted fingerprint coming
|
|
217
|
+
* through `maybeLoadSessionPool` naturally wins because it arrives in
|
|
218
|
+
* `perCallOptions`.
|
|
219
|
+
*/
|
|
220
|
+
private invokeCreateSessionFunction;
|
|
231
221
|
/**
|
|
232
222
|
* Creates new session and adds it to the pool.
|
|
233
223
|
* @returns Newly created `Session` instance.
|
|
234
224
|
*/
|
|
235
|
-
|
|
225
|
+
private createSession;
|
|
236
226
|
/**
|
|
237
227
|
* Decides whether there is enough space for creating new session.
|
|
238
228
|
*/
|
|
239
|
-
|
|
229
|
+
private hasSpaceForSession;
|
|
240
230
|
/**
|
|
241
|
-
* Picks
|
|
242
|
-
*
|
|
231
|
+
* Picks a session from the `SessionPool` according to the configured `sessionReuseStrategy`.
|
|
232
|
+
* Returns `undefined` when no session should be reused and a new one should be created instead.
|
|
243
233
|
*/
|
|
244
|
-
|
|
234
|
+
private pickSession;
|
|
245
235
|
/**
|
|
246
236
|
* Potentially loads `SessionPool`.
|
|
247
237
|
* If the state was persisted it loads the `SessionPool` from the persisted state.
|
|
248
238
|
*/
|
|
249
|
-
|
|
250
|
-
/**
|
|
251
|
-
* Opens a SessionPool and returns a promise resolving to an instance
|
|
252
|
-
* of the {@link SessionPool} class that is already initialized.
|
|
253
|
-
*
|
|
254
|
-
* For more details and code examples, see the {@link SessionPool} class.
|
|
255
|
-
*/
|
|
256
|
-
static open(options?: SessionPoolOptions, config?: Configuration): Promise<SessionPool>;
|
|
239
|
+
private maybeLoadSessionPool;
|
|
257
240
|
}
|
|
258
|
-
|
|
241
|
+
export {};
|