@crawlee/core 4.0.0-beta.99 → 4.0.0-rc.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/configuration.d.ts +16 -47
- package/configuration.js +13 -25
- package/debug.js +4 -4
- package/errors.d.ts +28 -38
- package/errors.js +33 -47
- package/events/event_manager.d.ts +2 -2
- package/events/event_manager.js +7 -6
- package/events/index.d.ts +1 -0
- package/events/local_event_manager.d.ts +1 -8
- package/events/local_event_manager.js +13 -13
- package/events/system_info.d.ts +38 -0
- package/index.d.ts +2 -8
- package/index.js +4 -8
- package/internal.d.ts +8 -0
- package/internal.js +9 -0
- package/log.d.ts +10 -11
- package/log.js +52 -20
- package/memory-storage/memory-storage.d.ts +15 -18
- package/memory-storage/memory-storage.js +80 -58
- package/memory-storage/resource-clients/dataset.d.ts +1 -6
- package/memory-storage/resource-clients/dataset.js +23 -31
- package/memory-storage/resource-clients/key-value-store.d.ts +1 -10
- package/memory-storage/resource-clients/key-value-store.js +43 -67
- package/memory-storage/resource-clients/request-queue.d.ts +1 -42
- package/memory-storage/resource-clients/request-queue.js +109 -117
- package/owned_or_injected.d.ts +1 -3
- package/owned_or_injected.js +17 -17
- package/package.json +17 -20
- package/proxy_configuration.d.ts +21 -26
- package/proxy_configuration.js +35 -25
- package/recoverable_state.d.ts +104 -47
- package/recoverable_state.js +199 -74
- package/request.d.ts +20 -107
- package/request.js +78 -244
- package/serialization.js +17 -16
- package/service_locator.d.ts +22 -10
- package/service_locator.js +59 -48
- package/storages/batched_adds.d.ts +37 -0
- package/storages/batched_adds.js +73 -0
- package/storages/dataset.d.ts +13 -8
- package/storages/dataset.js +149 -40
- package/storages/index.d.ts +4 -4
- package/storages/index.js +2 -4
- package/storages/key_value_store.d.ts +16 -35
- package/storages/key_value_store.js +223 -110
- package/storages/key_value_store_codec.js +6 -11
- package/storages/request_dedup_cache.d.ts +1 -4
- package/storages/request_dedup_cache.js +15 -15
- package/storages/request_list.d.ts +9 -104
- package/storages/request_list.js +236 -233
- package/storages/request_loader.d.ts +49 -18
- package/storages/request_loader.js +36 -1
- package/storages/request_manager.d.ts +86 -0
- package/storages/request_manager_tandem.d.ts +14 -38
- package/storages/request_manager_tandem.js +67 -64
- package/storages/request_queue.d.ts +23 -50
- package/storages/request_queue.js +371 -226
- package/storages/storage_instance_manager.d.ts +2 -4
- package/storages/storage_instance_manager.js +21 -21
- package/storages/storage_stats.d.ts +1 -1
- package/storages/storage_stats.js +4 -4
- package/storages/transaction.d.ts +270 -0
- package/storages/transaction.js +296 -0
- package/storages/utils.d.ts +6 -3
- package/storages/utils.js +11 -2
- package/system-info/runtime.js +7 -7
- package/url.d.ts +9 -0
- package/url.js +11 -0
- package/validators.d.ts +23 -25
- package/validators.js +14 -25
- package/autoscaling/autoscaled_pool.d.ts +0 -213
- package/autoscaling/autoscaled_pool.js +0 -378
- package/autoscaling/client_load_signal.d.ts +0 -59
- package/autoscaling/client_load_signal.js +0 -73
- package/autoscaling/concurrency_system.d.ts +0 -283
- package/autoscaling/concurrency_system.js +0 -350
- package/autoscaling/cpu_load_signal.d.ts +0 -44
- package/autoscaling/cpu_load_signal.js +0 -46
- package/autoscaling/event_loop_load_signal.d.ts +0 -54
- package/autoscaling/event_loop_load_signal.js +0 -60
- package/autoscaling/index.d.ts +0 -9
- package/autoscaling/index.js +0 -9
- package/autoscaling/load_signal.d.ts +0 -99
- package/autoscaling/load_signal.js +0 -103
- package/autoscaling/memory_load_signal.d.ts +0 -56
- package/autoscaling/memory_load_signal.js +0 -106
- package/autoscaling/snapshotter.d.ts +0 -87
- package/autoscaling/snapshotter.js +0 -67
- package/autoscaling/system_status.d.ts +0 -161
- package/autoscaling/system_status.js +0 -139
- package/autoscaling/weighted_avg.d.ts +0 -5
- package/autoscaling/weighted_avg.js +0 -14
- package/cookie_utils.d.ts +0 -44
- package/cookie_utils.js +0 -122
- package/crawlers/context_pipeline.d.ts +0 -70
- package/crawlers/context_pipeline.js +0 -122
- package/crawlers/crawler_commons.d.ts +0 -257
- package/crawlers/crawler_commons.js +0 -107
- package/crawlers/error_snapshotter.d.ts +0 -59
- package/crawlers/error_snapshotter.js +0 -117
- package/crawlers/error_tracker.d.ts +0 -54
- package/crawlers/error_tracker.js +0 -308
- package/crawlers/index.d.ts +0 -5
- package/crawlers/index.js +0 -5
- package/crawlers/internals/types.d.ts +0 -7
- package/crawlers/statistics.d.ts +0 -209
- package/crawlers/statistics.js +0 -350
- package/enqueue_links/enqueue_links.d.ts +0 -264
- package/enqueue_links/enqueue_links.js +0 -271
- package/enqueue_links/index.d.ts +0 -2
- package/enqueue_links/index.js +0 -2
- package/enqueue_links/shared.d.ts +0 -83
- package/enqueue_links/shared.js +0 -221
- package/router.d.ts +0 -309
- package/router.js +0 -309
- package/session_pool/consts.d.ts +0 -3
- package/session_pool/consts.js +0 -3
- package/session_pool/errors.d.ts +0 -7
- package/session_pool/errors.js +0 -11
- package/session_pool/fingerprint.d.ts +0 -9
- package/session_pool/fingerprint.js +0 -30
- package/session_pool/index.d.ts +0 -4
- package/session_pool/index.js +0 -4
- package/session_pool/session.d.ts +0 -161
- package/session_pool/session.js +0 -218
- package/session_pool/session_pool.d.ts +0 -246
- package/session_pool/session_pool.js +0 -386
- package/storages/access_checking.d.ts +0 -12
- package/storages/access_checking.js +0 -17
- package/storages/sitemap_request_loader.d.ts +0 -249
- package/storages/sitemap_request_loader.js +0 -432
- /package/{crawlers/internals/types.js → events/system_info.js} +0 -0
|
@@ -1,30 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* (browser, platform, device) combinations that correspond to setups people
|
|
3
|
-
* actually run. Anything not listed here (e.g. `edge` on android, `safari` on
|
|
4
|
-
* windows, `desktop` mobile platforms) is left out so a randomized default
|
|
5
|
-
* never produces a fingerprint that would itself be a giveaway.
|
|
6
|
-
*/
|
|
7
|
-
const PROFILES_BY_PLATFORM = [
|
|
8
|
-
{ browser: 'chrome', platform: 'windows', device: 'desktop' },
|
|
9
|
-
{ browser: 'firefox', platform: 'windows', device: 'desktop' },
|
|
10
|
-
{ browser: 'edge', platform: 'windows', device: 'desktop' },
|
|
11
|
-
{ browser: 'chrome', platform: 'macos', device: 'desktop' },
|
|
12
|
-
{ browser: 'firefox', platform: 'macos', device: 'desktop' },
|
|
13
|
-
{ browser: 'safari', platform: 'macos', device: 'desktop' },
|
|
14
|
-
{ browser: 'edge', platform: 'macos', device: 'desktop' },
|
|
15
|
-
{ browser: 'chrome', platform: 'linux', device: 'desktop' },
|
|
16
|
-
{ browser: 'firefox', platform: 'linux', device: 'desktop' },
|
|
17
|
-
{ browser: 'chrome', platform: 'android', device: 'mobile' },
|
|
18
|
-
{ browser: 'firefox', platform: 'android', device: 'mobile' },
|
|
19
|
-
{ browser: 'safari', platform: 'ios', device: 'mobile' },
|
|
20
|
-
];
|
|
21
|
-
/**
|
|
22
|
-
* Build a {@link SessionFingerprint} whose `platform` matches the host OS
|
|
23
|
-
* and whose `browser`/`device` are randomized within the realistic profiles for
|
|
24
|
-
* that platform. Used by {@link SessionPool} as the default fingerprint for
|
|
25
|
-
* freshly created sessions; callers can override by passing their own
|
|
26
|
-
* `fingerprint` in `sessionOptions`.
|
|
27
|
-
*/
|
|
28
|
-
export function createDefaultSessionFingerprint() {
|
|
29
|
-
return { ...PROFILES_BY_PLATFORM[Math.floor(Math.random() * PROFILES_BY_PLATFORM.length)] };
|
|
30
|
-
}
|
package/session_pool/index.d.ts
DELETED
package/session_pool/index.js
DELETED
|
@@ -1,161 +0,0 @@
|
|
|
1
|
-
import type { Dictionary, ISession, ProxyInfo, SessionFingerprint, SessionState } from '@crawlee/types';
|
|
2
|
-
import { CookieJar } from 'tough-cookie';
|
|
3
|
-
import type { CrawleeLogger } from '../log.js';
|
|
4
|
-
export interface SessionOptions {
|
|
5
|
-
/** Id of session used for generating fingerprints. It is used as proxy session name. */
|
|
6
|
-
id?: string;
|
|
7
|
-
/**
|
|
8
|
-
* Number of seconds after which the session is considered as expired.
|
|
9
|
-
* @default 3000
|
|
10
|
-
*/
|
|
11
|
-
maxAgeSecs?: number;
|
|
12
|
-
/** Object where custom user data can be stored. For example custom headers. */
|
|
13
|
-
userData?: Dictionary;
|
|
14
|
-
/**
|
|
15
|
-
* Maximum number of marking session as blocked usage.
|
|
16
|
-
* If the `errorScore` reaches the `maxErrorScore` session is marked as block and it is thrown away.
|
|
17
|
-
* It starts at 0. Calling the `markBad` function increases the `errorScore` by 1.
|
|
18
|
-
* Calling the `markGood` will decrease the `errorScore` by `errorScoreDecrement`
|
|
19
|
-
* @default 3
|
|
20
|
-
*/
|
|
21
|
-
maxErrorScore?: number;
|
|
22
|
-
/**
|
|
23
|
-
* It is used for healing the session.
|
|
24
|
-
* For example: if your session is marked bad two times, but it is successful on the third attempt it's errorScore
|
|
25
|
-
* is decremented by this number.
|
|
26
|
-
* @default 0.5
|
|
27
|
-
*/
|
|
28
|
-
errorScoreDecrement?: number;
|
|
29
|
-
/** Date of creation. */
|
|
30
|
-
createdAt?: Date;
|
|
31
|
-
/** Date of expiration. */
|
|
32
|
-
expiresAt?: Date;
|
|
33
|
-
/**
|
|
34
|
-
* Indicates how many times the session has been used.
|
|
35
|
-
* @default 0
|
|
36
|
-
*/
|
|
37
|
-
usageCount?: number;
|
|
38
|
-
/**
|
|
39
|
-
* Session should be used only a limited amount of times.
|
|
40
|
-
* This number indicates how many times the session is going to be used, before it is thrown away.
|
|
41
|
-
* @default 50
|
|
42
|
-
*/
|
|
43
|
-
maxUsageCount?: number;
|
|
44
|
-
/**
|
|
45
|
-
* Marks the session as already retired. Used when restoring a previously persisted session
|
|
46
|
-
* so that `isUsable()` reflects the terminal state regardless of error score or usage count.
|
|
47
|
-
* @default false
|
|
48
|
-
*/
|
|
49
|
-
retired?: boolean;
|
|
50
|
-
log?: CrawleeLogger;
|
|
51
|
-
errorScore?: number;
|
|
52
|
-
cookieJar?: CookieJar;
|
|
53
|
-
proxyInfo?: ProxyInfo;
|
|
54
|
-
/**
|
|
55
|
-
* Browser / HTTP client fingerprint tied to this session. Backends use this to make
|
|
56
|
-
* repeated requests with the same session look consistent (same user-agent, headers,
|
|
57
|
-
* TLS profile). See {@link SessionFingerprint}.
|
|
58
|
-
*/
|
|
59
|
-
fingerprint?: SessionFingerprint;
|
|
60
|
-
}
|
|
61
|
-
/**
|
|
62
|
-
* Sessions are used to store information such as cookies and can be used for generating fingerprints and proxy sessions.
|
|
63
|
-
* You can imagine each session as a specific user, with its own cookies, IP (via proxy) and potentially a unique browser fingerprint.
|
|
64
|
-
* Session internal state can be enriched with custom user data for example some authorization tokens and specific headers in general.
|
|
65
|
-
* @category Scaling
|
|
66
|
-
*/
|
|
67
|
-
export declare class Session implements ISession {
|
|
68
|
-
readonly id: string;
|
|
69
|
-
readonly userData: Dictionary;
|
|
70
|
-
private _maxErrorScore;
|
|
71
|
-
private _errorScoreDecrement;
|
|
72
|
-
private _createdAt;
|
|
73
|
-
private _expiresAt;
|
|
74
|
-
private _usageCount;
|
|
75
|
-
private _maxUsageCount;
|
|
76
|
-
private _errorScore;
|
|
77
|
-
private _retired;
|
|
78
|
-
private _proxyInfo?;
|
|
79
|
-
private _cookieJar;
|
|
80
|
-
private _fingerprint?;
|
|
81
|
-
private log;
|
|
82
|
-
get errorScore(): number;
|
|
83
|
-
get usageCount(): number;
|
|
84
|
-
get maxErrorScore(): number;
|
|
85
|
-
get errorScoreDecrement(): number;
|
|
86
|
-
get expiresAt(): Date;
|
|
87
|
-
get createdAt(): Date;
|
|
88
|
-
get maxUsageCount(): number;
|
|
89
|
-
get cookieJar(): CookieJar;
|
|
90
|
-
get proxyInfo(): ProxyInfo | undefined;
|
|
91
|
-
get fingerprint(): SessionFingerprint | undefined;
|
|
92
|
-
set fingerprint(fingerprint: SessionFingerprint | undefined);
|
|
93
|
-
/**
|
|
94
|
-
* `true` once {@link Session.retire|`retire()`} has been called. Retirement is terminal:
|
|
95
|
-
* a retired session is never picked by the pool and cannot be revived via `markGood()`.
|
|
96
|
-
*/
|
|
97
|
-
get retired(): boolean;
|
|
98
|
-
/**
|
|
99
|
-
* Session configuration.
|
|
100
|
-
*/
|
|
101
|
-
constructor(options?: SessionOptions);
|
|
102
|
-
/**
|
|
103
|
-
* Indicates whether the session is blocked.
|
|
104
|
-
* Session is blocked once it reaches the `maxErrorScore`.
|
|
105
|
-
*/
|
|
106
|
-
isBlocked(): boolean;
|
|
107
|
-
/**
|
|
108
|
-
* Indicates whether the session is expired.
|
|
109
|
-
* Session expiration is determined by the `maxAgeSecs`.
|
|
110
|
-
* Once the session is older than `createdAt + maxAgeSecs` the session is considered expired.
|
|
111
|
-
*/
|
|
112
|
-
isExpired(): boolean;
|
|
113
|
-
/**
|
|
114
|
-
* Indicates whether the session is used maximum number of times.
|
|
115
|
-
* Session maximum usage count can be changed by `maxUsageCount` parameter.
|
|
116
|
-
*/
|
|
117
|
-
isMaxUsageCountReached(): boolean;
|
|
118
|
-
/**
|
|
119
|
-
* Indicates whether the session can be used for next requests.
|
|
120
|
-
* Session is usable when it is not retired, not expired, not blocked and the maximum usage count has not be reached.
|
|
121
|
-
*/
|
|
122
|
-
isUsable(): boolean;
|
|
123
|
-
/**
|
|
124
|
-
* This method should be called after a successful session usage.
|
|
125
|
-
* It increases `usageCount` and potentially lowers the `errorScore` by the `errorScoreDecrement`.
|
|
126
|
-
*/
|
|
127
|
-
markGood(): void;
|
|
128
|
-
/**
|
|
129
|
-
* Gets session state for persistence in KeyValueStore.
|
|
130
|
-
* @returns Represents session internal state.
|
|
131
|
-
*/
|
|
132
|
-
getState(): SessionState;
|
|
133
|
-
/**
|
|
134
|
-
* Permanently retires the session — `isUsable()` will return `false` from here on,
|
|
135
|
-
* and no `markGood()` / `markBad()` can revive it. Calling `retire()` again is a no-op.
|
|
136
|
-
*
|
|
137
|
-
* Use this when you're confident the session itself is the problem (e.g. a `403` response).
|
|
138
|
-
* For transient external failures (such as `5XX` responses), use `markBad()` instead.
|
|
139
|
-
*/
|
|
140
|
-
retire(): void;
|
|
141
|
-
/**
|
|
142
|
-
* Increases usage and error count.
|
|
143
|
-
* Should be used when the session has been used unsuccessfully. For example because of timeouts.
|
|
144
|
-
*/
|
|
145
|
-
markBad(): void;
|
|
146
|
-
/**
|
|
147
|
-
* Returns cookies saved with the session in the typical
|
|
148
|
-
* key1=value1; key2=value2 format, ready to be used in
|
|
149
|
-
* a cookie header or elsewhere.
|
|
150
|
-
* @returns Represents `Cookie` header.
|
|
151
|
-
*/
|
|
152
|
-
getCookieString(url: string): string;
|
|
153
|
-
/**
|
|
154
|
-
* Sets a cookie within this session for the specific URL.
|
|
155
|
-
*/
|
|
156
|
-
setCookie(rawCookie: string, url: string): void;
|
|
157
|
-
/**
|
|
158
|
-
* Checks if session is not usable. if it is not retires the session.
|
|
159
|
-
*/
|
|
160
|
-
private maybeSelfRetire;
|
|
161
|
-
}
|
package/session_pool/session.js
DELETED
|
@@ -1,218 +0,0 @@
|
|
|
1
|
-
import ow from 'ow';
|
|
2
|
-
import { CookieJar } from 'tough-cookie';
|
|
3
|
-
import { cryptoRandomObjectId } from '@apify/utilities';
|
|
4
|
-
import { getDefaultCookieExpirationDate } from '../cookie_utils.js';
|
|
5
|
-
import { serviceLocator } from '../service_locator.js';
|
|
6
|
-
/**
|
|
7
|
-
* Sessions are used to store information such as cookies and can be used for generating fingerprints and proxy sessions.
|
|
8
|
-
* You can imagine each session as a specific user, with its own cookies, IP (via proxy) and potentially a unique browser fingerprint.
|
|
9
|
-
* Session internal state can be enriched with custom user data for example some authorization tokens and specific headers in general.
|
|
10
|
-
* @category Scaling
|
|
11
|
-
*/
|
|
12
|
-
export class Session {
|
|
13
|
-
id;
|
|
14
|
-
userData;
|
|
15
|
-
_maxErrorScore;
|
|
16
|
-
_errorScoreDecrement;
|
|
17
|
-
_createdAt;
|
|
18
|
-
_expiresAt;
|
|
19
|
-
_usageCount;
|
|
20
|
-
_maxUsageCount;
|
|
21
|
-
_errorScore;
|
|
22
|
-
_retired = false;
|
|
23
|
-
_proxyInfo;
|
|
24
|
-
_cookieJar;
|
|
25
|
-
_fingerprint;
|
|
26
|
-
log;
|
|
27
|
-
get errorScore() {
|
|
28
|
-
return this._errorScore;
|
|
29
|
-
}
|
|
30
|
-
get usageCount() {
|
|
31
|
-
return this._usageCount;
|
|
32
|
-
}
|
|
33
|
-
get maxErrorScore() {
|
|
34
|
-
return this._maxErrorScore;
|
|
35
|
-
}
|
|
36
|
-
get errorScoreDecrement() {
|
|
37
|
-
return this._errorScoreDecrement;
|
|
38
|
-
}
|
|
39
|
-
get expiresAt() {
|
|
40
|
-
return this._expiresAt;
|
|
41
|
-
}
|
|
42
|
-
get createdAt() {
|
|
43
|
-
return this._createdAt;
|
|
44
|
-
}
|
|
45
|
-
get maxUsageCount() {
|
|
46
|
-
return this._maxUsageCount;
|
|
47
|
-
}
|
|
48
|
-
get cookieJar() {
|
|
49
|
-
return this._cookieJar;
|
|
50
|
-
}
|
|
51
|
-
get proxyInfo() {
|
|
52
|
-
return this._proxyInfo;
|
|
53
|
-
}
|
|
54
|
-
get fingerprint() {
|
|
55
|
-
return this._fingerprint;
|
|
56
|
-
}
|
|
57
|
-
set fingerprint(fingerprint) {
|
|
58
|
-
this._fingerprint = fingerprint;
|
|
59
|
-
}
|
|
60
|
-
/**
|
|
61
|
-
* `true` once {@link Session.retire|`retire()`} has been called. Retirement is terminal:
|
|
62
|
-
* a retired session is never picked by the pool and cannot be revived via `markGood()`.
|
|
63
|
-
*/
|
|
64
|
-
get retired() {
|
|
65
|
-
return this._retired;
|
|
66
|
-
}
|
|
67
|
-
/**
|
|
68
|
-
* Session configuration.
|
|
69
|
-
*/
|
|
70
|
-
constructor(options = {}) {
|
|
71
|
-
ow(options, ow.object.exactShape({
|
|
72
|
-
id: ow.optional.string,
|
|
73
|
-
cookieJar: ow.optional.object,
|
|
74
|
-
proxyInfo: ow.optional.object,
|
|
75
|
-
maxAgeSecs: ow.optional.number,
|
|
76
|
-
userData: ow.optional.object,
|
|
77
|
-
maxErrorScore: ow.optional.number,
|
|
78
|
-
errorScoreDecrement: ow.optional.number,
|
|
79
|
-
createdAt: ow.optional.date,
|
|
80
|
-
expiresAt: ow.optional.date,
|
|
81
|
-
usageCount: ow.optional.number,
|
|
82
|
-
errorScore: ow.optional.number,
|
|
83
|
-
maxUsageCount: ow.optional.number,
|
|
84
|
-
retired: ow.optional.boolean,
|
|
85
|
-
log: ow.optional.object,
|
|
86
|
-
fingerprint: ow.optional.object,
|
|
87
|
-
}));
|
|
88
|
-
const { id = `session_${cryptoRandomObjectId(10)}`, cookieJar = new CookieJar(), proxyInfo = undefined, maxAgeSecs = 3000, userData = {}, maxErrorScore = 3, errorScoreDecrement = 0.5, createdAt = new Date(), usageCount = 0, errorScore = 0, maxUsageCount = 50, retired = false, log = serviceLocator.getLogger(), fingerprint, } = options;
|
|
89
|
-
const { expiresAt = getDefaultCookieExpirationDate(maxAgeSecs) } = options;
|
|
90
|
-
this.log = log.child({ prefix: 'Session' });
|
|
91
|
-
this._cookieJar = cookieJar.setCookie ? cookieJar : CookieJar.fromJSON(JSON.stringify(cookieJar));
|
|
92
|
-
this._proxyInfo = proxyInfo;
|
|
93
|
-
this._fingerprint = fingerprint;
|
|
94
|
-
this.id = id;
|
|
95
|
-
this.userData = userData;
|
|
96
|
-
this._maxErrorScore = maxErrorScore;
|
|
97
|
-
this._errorScoreDecrement = errorScoreDecrement;
|
|
98
|
-
// Internal
|
|
99
|
-
this._expiresAt = expiresAt;
|
|
100
|
-
this._createdAt = createdAt;
|
|
101
|
-
this._usageCount = usageCount; // indicates how many times the session has been used
|
|
102
|
-
this._errorScore = errorScore; // indicates number of markBaded request with the session
|
|
103
|
-
this._maxUsageCount = maxUsageCount;
|
|
104
|
-
this._retired = retired;
|
|
105
|
-
}
|
|
106
|
-
/**
|
|
107
|
-
* Indicates whether the session is blocked.
|
|
108
|
-
* Session is blocked once it reaches the `maxErrorScore`.
|
|
109
|
-
*/
|
|
110
|
-
isBlocked() {
|
|
111
|
-
return this.errorScore >= this.maxErrorScore;
|
|
112
|
-
}
|
|
113
|
-
/**
|
|
114
|
-
* Indicates whether the session is expired.
|
|
115
|
-
* Session expiration is determined by the `maxAgeSecs`.
|
|
116
|
-
* Once the session is older than `createdAt + maxAgeSecs` the session is considered expired.
|
|
117
|
-
*/
|
|
118
|
-
isExpired() {
|
|
119
|
-
return this.expiresAt <= new Date();
|
|
120
|
-
}
|
|
121
|
-
/**
|
|
122
|
-
* Indicates whether the session is used maximum number of times.
|
|
123
|
-
* Session maximum usage count can be changed by `maxUsageCount` parameter.
|
|
124
|
-
*/
|
|
125
|
-
isMaxUsageCountReached() {
|
|
126
|
-
return this.usageCount >= this.maxUsageCount;
|
|
127
|
-
}
|
|
128
|
-
/**
|
|
129
|
-
* Indicates whether the session can be used for next requests.
|
|
130
|
-
* Session is usable when it is not retired, not expired, not blocked and the maximum usage count has not be reached.
|
|
131
|
-
*/
|
|
132
|
-
isUsable() {
|
|
133
|
-
return !this._retired && !this.isBlocked() && !this.isExpired() && !this.isMaxUsageCountReached();
|
|
134
|
-
}
|
|
135
|
-
/**
|
|
136
|
-
* This method should be called after a successful session usage.
|
|
137
|
-
* It increases `usageCount` and potentially lowers the `errorScore` by the `errorScoreDecrement`.
|
|
138
|
-
*/
|
|
139
|
-
markGood() {
|
|
140
|
-
this._usageCount += 1;
|
|
141
|
-
if (this._errorScore > 0) {
|
|
142
|
-
this._errorScore -= this._errorScoreDecrement;
|
|
143
|
-
}
|
|
144
|
-
this.maybeSelfRetire();
|
|
145
|
-
}
|
|
146
|
-
/**
|
|
147
|
-
* Gets session state for persistence in KeyValueStore.
|
|
148
|
-
* @returns Represents session internal state.
|
|
149
|
-
*/
|
|
150
|
-
getState() {
|
|
151
|
-
return {
|
|
152
|
-
id: this.id,
|
|
153
|
-
cookieJar: this.cookieJar.toJSON(),
|
|
154
|
-
proxyInfo: this._proxyInfo,
|
|
155
|
-
userData: this.userData,
|
|
156
|
-
fingerprint: this._fingerprint,
|
|
157
|
-
maxErrorScore: this.maxErrorScore,
|
|
158
|
-
errorScoreDecrement: this.errorScoreDecrement,
|
|
159
|
-
expiresAt: this.expiresAt.toISOString(),
|
|
160
|
-
createdAt: this.createdAt.toISOString(),
|
|
161
|
-
usageCount: this.usageCount,
|
|
162
|
-
maxUsageCount: this.maxUsageCount,
|
|
163
|
-
errorScore: this.errorScore,
|
|
164
|
-
retired: this._retired,
|
|
165
|
-
};
|
|
166
|
-
}
|
|
167
|
-
/**
|
|
168
|
-
* Permanently retires the session — `isUsable()` will return `false` from here on,
|
|
169
|
-
* and no `markGood()` / `markBad()` can revive it. Calling `retire()` again is a no-op.
|
|
170
|
-
*
|
|
171
|
-
* Use this when you're confident the session itself is the problem (e.g. a `403` response).
|
|
172
|
-
* For transient external failures (such as `5XX` responses), use `markBad()` instead.
|
|
173
|
-
*/
|
|
174
|
-
retire() {
|
|
175
|
-
if (this._retired)
|
|
176
|
-
return;
|
|
177
|
-
this._errorScore += this._maxErrorScore;
|
|
178
|
-
this._usageCount += 1;
|
|
179
|
-
this._retired = true;
|
|
180
|
-
}
|
|
181
|
-
/**
|
|
182
|
-
* Increases usage and error count.
|
|
183
|
-
* Should be used when the session has been used unsuccessfully. For example because of timeouts.
|
|
184
|
-
*/
|
|
185
|
-
markBad() {
|
|
186
|
-
this._errorScore += 1;
|
|
187
|
-
this._usageCount += 1;
|
|
188
|
-
this.maybeSelfRetire();
|
|
189
|
-
}
|
|
190
|
-
/**
|
|
191
|
-
* Returns cookies saved with the session in the typical
|
|
192
|
-
* key1=value1; key2=value2 format, ready to be used in
|
|
193
|
-
* a cookie header or elsewhere.
|
|
194
|
-
* @returns Represents `Cookie` header.
|
|
195
|
-
*/
|
|
196
|
-
getCookieString(url) {
|
|
197
|
-
return this.cookieJar.getCookieStringSync(url, {});
|
|
198
|
-
}
|
|
199
|
-
/**
|
|
200
|
-
* Sets a cookie within this session for the specific URL.
|
|
201
|
-
*/
|
|
202
|
-
setCookie(rawCookie, url) {
|
|
203
|
-
try {
|
|
204
|
-
this.cookieJar.setCookieSync(rawCookie, url);
|
|
205
|
-
}
|
|
206
|
-
catch (e) {
|
|
207
|
-
this.log.warning('Could not set cookie.', { url, error: e.message });
|
|
208
|
-
}
|
|
209
|
-
}
|
|
210
|
-
/**
|
|
211
|
-
* Checks if session is not usable. if it is not retires the session.
|
|
212
|
-
*/
|
|
213
|
-
maybeSelfRetire() {
|
|
214
|
-
if (!this.isUsable()) {
|
|
215
|
-
this.retire();
|
|
216
|
-
}
|
|
217
|
-
}
|
|
218
|
-
}
|
|
@@ -1,246 +0,0 @@
|
|
|
1
|
-
import type { ISessionPool } from '@crawlee/types';
|
|
2
|
-
import type { PersistenceOptions } from '../crawlers/statistics.js';
|
|
3
|
-
import type { CrawleeLogger } from '../log.js';
|
|
4
|
-
import type { SessionOptions } from './session.js';
|
|
5
|
-
import { Session } from './session.js';
|
|
6
|
-
declare const SESSION_REUSE_STRATEGIES: readonly ["random", "round-robin", "use-until-failure"];
|
|
7
|
-
export type SessionReuseStrategy = (typeof SESSION_REUSE_STRATEGIES)[number];
|
|
8
|
-
/**
|
|
9
|
-
* Factory user-function which creates customized {@link Session} instances.
|
|
10
|
-
*/
|
|
11
|
-
export interface CreateSession {
|
|
12
|
-
/**
|
|
13
|
-
* @param options.sessionOptions Per-call session options already merged with the pool-wide defaults.
|
|
14
|
-
*/
|
|
15
|
-
(options?: {
|
|
16
|
-
sessionOptions?: SessionOptions;
|
|
17
|
-
}): Session | Promise<Session>;
|
|
18
|
-
}
|
|
19
|
-
export interface SessionPoolOptions {
|
|
20
|
-
/**
|
|
21
|
-
* Unique identifier for this session pool instance. Used to generate a unique
|
|
22
|
-
* persistence key when `persistStateKey` is not provided.
|
|
23
|
-
* If not provided, an auto-incrementing ID is used.
|
|
24
|
-
*/
|
|
25
|
-
id?: string | number;
|
|
26
|
-
/**
|
|
27
|
-
* Maximum size of the pool. Indicates how many sessions are rotated.
|
|
28
|
-
* @default 1000
|
|
29
|
-
*/
|
|
30
|
-
maxPoolSize?: number;
|
|
31
|
-
/** The configuration options for {@link Session} instances. */
|
|
32
|
-
sessionOptions?: SessionOptions;
|
|
33
|
-
/** Name or Id of `KeyValueStore` where is the `SessionPool` state stored. */
|
|
34
|
-
persistStateKeyValueStoreId?: string;
|
|
35
|
-
/**
|
|
36
|
-
* Session pool persists its state under this key in Key value store.
|
|
37
|
-
* @default CRAWLEE_SESSION_POOL_STATE_{id}
|
|
38
|
-
*/
|
|
39
|
-
persistStateKey?: string;
|
|
40
|
-
/**
|
|
41
|
-
* Custom function that should return a `Session` instance, or a promise resolving to such instance.
|
|
42
|
-
* Any error thrown from this function will terminate the process. Receives `{ sessionOptions }`
|
|
43
|
-
* already merged from the pool-wide defaults and the per-call overrides.
|
|
44
|
-
*/
|
|
45
|
-
createSessionFunction?: CreateSession;
|
|
46
|
-
/**
|
|
47
|
-
* Strategy for picking sessions from the pool.
|
|
48
|
-
* - `'random'` (default): fills the pool up to `maxPoolSize`, then picks a random usable session
|
|
49
|
-
* - `'round-robin'`: fills the pool up to `maxPoolSize`, then reuses sessions cycling through them in order
|
|
50
|
-
* - `'use-until-failure'`: always reuses the same session until it is retired, then moves to the next one
|
|
51
|
-
* @default 'random'
|
|
52
|
-
*/
|
|
53
|
-
sessionReuseStrategy?: SessionReuseStrategy;
|
|
54
|
-
/** @internal */
|
|
55
|
-
log?: CrawleeLogger;
|
|
56
|
-
/**
|
|
57
|
-
* Control how and when to persist the state of the session pool.
|
|
58
|
-
*/
|
|
59
|
-
persistenceOptions?: PersistenceOptions;
|
|
60
|
-
}
|
|
61
|
-
/**
|
|
62
|
-
* Handles the rotation, creation and persistence of user-like sessions.
|
|
63
|
-
* Creates a pool of {@link Session} instances, that are randomly rotated.
|
|
64
|
-
* When some session is marked as blocked, it is removed and new one is created instead (the pool never returns an unusable session).
|
|
65
|
-
* Learn more in the {@doclink guides/session-management | Session management guide}.
|
|
66
|
-
*
|
|
67
|
-
* Session pool is already integrated into crawlers and is always active.
|
|
68
|
-
* All public methods are lazy-initialized — the pool initializes itself on first use.
|
|
69
|
-
*
|
|
70
|
-
* You can configure the pool with many options. See the {@link SessionPoolOptions}.
|
|
71
|
-
* Session pool is by default persisted in default {@link KeyValueStore}.
|
|
72
|
-
* If you want to have one pool for all runs you have to specify
|
|
73
|
-
* {@link SessionPoolOptions.persistStateKeyValueStoreId}.
|
|
74
|
-
*
|
|
75
|
-
* **Advanced usage:**
|
|
76
|
-
*
|
|
77
|
-
* ```javascript
|
|
78
|
-
* const sessionPool = new SessionPool({
|
|
79
|
-
* maxPoolSize: 25,
|
|
80
|
-
* sessionOptions:{
|
|
81
|
-
* maxAgeSecs: 10,
|
|
82
|
-
* maxUsageCount: 150, // for example when you know that the site blocks after 150 requests.
|
|
83
|
-
* },
|
|
84
|
-
* persistStateKeyValueStoreId: 'my-key-value-store-for-sessions',
|
|
85
|
-
* persistStateKey: 'my-session-pool',
|
|
86
|
-
* });
|
|
87
|
-
*
|
|
88
|
-
* // Get random session from the pool
|
|
89
|
-
* const session1 = await sessionPool.getSession();
|
|
90
|
-
* const session2 = await sessionPool.getSession();
|
|
91
|
-
* const session3 = await sessionPool.getSession();
|
|
92
|
-
*
|
|
93
|
-
* // Now you can mark the session either failed or successful
|
|
94
|
-
*
|
|
95
|
-
* // Marks session as bad after unsuccessful usage -> it increases error count (soft retire)
|
|
96
|
-
* session1.markBad()
|
|
97
|
-
*
|
|
98
|
-
* // Marks as successful.
|
|
99
|
-
* session2.markGood()
|
|
100
|
-
*
|
|
101
|
-
* // Retires session -> session is removed from the pool
|
|
102
|
-
* session3.retire()
|
|
103
|
-
*
|
|
104
|
-
* ```
|
|
105
|
-
*
|
|
106
|
-
* **Default session allocation flow:*
|
|
107
|
-
* 1. Until the `SessionPool` reaches `maxPoolSize`, new sessions are created, provided to the user and added to the pool
|
|
108
|
-
* 2. Blocked/retired sessions stay in the pool but are never provided to the user
|
|
109
|
-
* 3. Once the pool is full (live plus blocked session count reaches `maxPoolSize`), a random session from the pool is provided.
|
|
110
|
-
* 4. If a blocked session would be picked, instead all blocked sessions are evicted from the pool and a new session is created and provided
|
|
111
|
-
*
|
|
112
|
-
* @category Scaling
|
|
113
|
-
*/
|
|
114
|
-
export declare class SessionPool implements ISessionPool {
|
|
115
|
-
private static nextId;
|
|
116
|
-
readonly id: string;
|
|
117
|
-
private log;
|
|
118
|
-
private maxPoolSize;
|
|
119
|
-
private createSessionFunction;
|
|
120
|
-
private keyValueStore?;
|
|
121
|
-
private sessions;
|
|
122
|
-
private sessionMap;
|
|
123
|
-
private sessionOptions;
|
|
124
|
-
private persistStateKeyValueStoreId?;
|
|
125
|
-
private persistStateKey;
|
|
126
|
-
private listener?;
|
|
127
|
-
private events;
|
|
128
|
-
private persistenceOptions;
|
|
129
|
-
private sessionReuseStrategy;
|
|
130
|
-
private initPromise?;
|
|
131
|
-
private queue;
|
|
132
|
-
private roundRobinIndex;
|
|
133
|
-
constructor(options?: SessionPoolOptions);
|
|
134
|
-
/**
|
|
135
|
-
* Gets count of usable sessions in the pool.
|
|
136
|
-
*/
|
|
137
|
-
usableSessionsCount(): Promise<number>;
|
|
138
|
-
/**
|
|
139
|
-
* Gets count of retired sessions in the pool.
|
|
140
|
-
*/
|
|
141
|
-
retiredSessionsCount(): Promise<number>;
|
|
142
|
-
/**
|
|
143
|
-
* Starts periodic state persistence and potentially loads SessionPool state from {@link KeyValueStore}.
|
|
144
|
-
* Called automatically on first use of any public method.
|
|
145
|
-
*/
|
|
146
|
-
private ensureInitialized;
|
|
147
|
-
private setupPool;
|
|
148
|
-
/**
|
|
149
|
-
* Adds a new session to the session pool. The pool automatically creates sessions up to the maximum size of the pool,
|
|
150
|
-
* but this allows you to add more sessions once the max pool size is reached.
|
|
151
|
-
* This also allows you to add session with overridden session options (e.g. with specific session id).
|
|
152
|
-
* @param [options] The configuration options for the session being added to the session pool.
|
|
153
|
-
*/
|
|
154
|
-
addSession(options?: Session | SessionOptions): Promise<void>;
|
|
155
|
-
/**
|
|
156
|
-
* Adds a new session to the session pool. The pool automatically creates sessions up to the maximum size of the pool,
|
|
157
|
-
* but this allows you to add more sessions once the max pool size is reached.
|
|
158
|
-
* This also allows you to add session with overridden session options (e.g. with specific session id).
|
|
159
|
-
* @param [options] The configuration options for the session being added to the session pool.
|
|
160
|
-
*/
|
|
161
|
-
newSession(sessionOptions?: SessionOptions): Promise<Session>;
|
|
162
|
-
/**
|
|
163
|
-
* Gets session.
|
|
164
|
-
* If there is space for new session, it creates and returns new session.
|
|
165
|
-
* If the session pool is full, it picks a session from the pool,
|
|
166
|
-
* If the picked session is usable it is returned, otherwise it creates and returns a new one.
|
|
167
|
-
* @param [sessionId] If provided, it returns the usable session with this id, `undefined` otherwise.
|
|
168
|
-
*/
|
|
169
|
-
getSession(sessionId?: string): Promise<Session | undefined>;
|
|
170
|
-
/**
|
|
171
|
-
* @param options - Override the persistence options provided in the constructor
|
|
172
|
-
*/
|
|
173
|
-
resetStore(options?: PersistenceOptions): Promise<void>;
|
|
174
|
-
/**
|
|
175
|
-
* Returns an object representing the internal state of the `SessionPool` instance.
|
|
176
|
-
* Note that the object's fields can change in future releases.
|
|
177
|
-
*/
|
|
178
|
-
getState(): Promise<{
|
|
179
|
-
usableSessionsCount: number;
|
|
180
|
-
retiredSessionsCount: number;
|
|
181
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
182
|
-
sessions: import("@crawlee/types").SessionState[];
|
|
183
|
-
}>;
|
|
184
|
-
/**
|
|
185
|
-
* Persists the current state of the `SessionPool` into the default {@link KeyValueStore}.
|
|
186
|
-
* The state is persisted automatically in regular intervals.
|
|
187
|
-
* @param options - Override the persistence options provided in the constructor
|
|
188
|
-
*/
|
|
189
|
-
persistState(options?: PersistenceOptions): Promise<void>;
|
|
190
|
-
/**
|
|
191
|
-
* Removes listener from `persistState` event.
|
|
192
|
-
* This function should be called after you are done with using the `SessionPool` instance.
|
|
193
|
-
*/
|
|
194
|
-
teardown(): Promise<void>;
|
|
195
|
-
/**
|
|
196
|
-
* Removes retired `Session` instances from `SessionPool`.
|
|
197
|
-
*/
|
|
198
|
-
private removeRetiredSessions;
|
|
199
|
-
/**
|
|
200
|
-
* Adds `Session` instance to `SessionPool`.
|
|
201
|
-
* @param newSession `Session` instance to be added.
|
|
202
|
-
*/
|
|
203
|
-
private registerSession;
|
|
204
|
-
/**
|
|
205
|
-
* Gets random index.
|
|
206
|
-
*/
|
|
207
|
-
private getRandomIndex;
|
|
208
|
-
/**
|
|
209
|
-
* Creates new session without any extra behavior.
|
|
210
|
-
* @param [options]
|
|
211
|
-
* @param [options.sessionOptions] The configuration options for the session being created.
|
|
212
|
-
* @returns New session.
|
|
213
|
-
*/
|
|
214
|
-
private defaultCreateSessionFunction;
|
|
215
|
-
/**
|
|
216
|
-
* Invokes `createSessionFunction` with `sessionOptions` already merged from pool-wide defaults and
|
|
217
|
-
* the supplied per-call overrides, so custom implementations don't need to spread `pool.sessionOptions` themselves.
|
|
218
|
-
*
|
|
219
|
-
* A default {@link SessionFingerprint} is generated up front (host OS as
|
|
220
|
-
* `platform`, a random valid `browser`/`device` for that platform). Pool-wide
|
|
221
|
-
* and per-call options override it, and a persisted fingerprint coming
|
|
222
|
-
* through `maybeLoadSessionPool` naturally wins because it arrives in
|
|
223
|
-
* `perCallOptions`.
|
|
224
|
-
*/
|
|
225
|
-
private _invokeCreateSessionFunction;
|
|
226
|
-
/**
|
|
227
|
-
* Creates new session and adds it to the pool.
|
|
228
|
-
* @returns Newly created `Session` instance.
|
|
229
|
-
*/
|
|
230
|
-
private createSession;
|
|
231
|
-
/**
|
|
232
|
-
* Decides whether there is enough space for creating new session.
|
|
233
|
-
*/
|
|
234
|
-
private hasSpaceForSession;
|
|
235
|
-
/**
|
|
236
|
-
* Picks a session from the `SessionPool` according to the configured `sessionReuseStrategy`.
|
|
237
|
-
* Returns `undefined` when no session should be reused and a new one should be created instead.
|
|
238
|
-
*/
|
|
239
|
-
private pickSession;
|
|
240
|
-
/**
|
|
241
|
-
* Potentially loads `SessionPool`.
|
|
242
|
-
* If the state was persisted it loads the `SessionPool` from the persisted state.
|
|
243
|
-
*/
|
|
244
|
-
private maybeLoadSessionPool;
|
|
245
|
-
}
|
|
246
|
-
export {};
|