@crawlee/basic 4.0.0-beta.27 → 4.0.0-beta.271
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -15
- package/index.d.ts +11 -2
- package/index.js +11 -1
- package/internals/autoscaling/autoscaled-pool.d.ts +178 -0
- package/internals/autoscaling/autoscaled-pool.js +389 -0
- package/internals/autoscaling/concurrency-system.d.ts +235 -0
- package/internals/autoscaling/concurrency-system.js +357 -0
- package/internals/autoscaling/cpu-load-signal.d.ts +43 -0
- package/internals/autoscaling/cpu-load-signal.js +46 -0
- package/internals/autoscaling/event-loop-load-signal.d.ts +51 -0
- package/internals/autoscaling/event-loop-load-signal.js +60 -0
- package/internals/autoscaling/index.d.ts +9 -0
- package/internals/autoscaling/index.js +9 -0
- package/internals/autoscaling/load-signal.d.ts +98 -0
- package/internals/autoscaling/load-signal.js +103 -0
- package/internals/autoscaling/memory-load-signal.d.ts +45 -0
- package/internals/autoscaling/memory-load-signal.js +104 -0
- package/internals/autoscaling/snapshotter.d.ts +84 -0
- package/internals/autoscaling/snapshotter.js +67 -0
- package/internals/autoscaling/storage-backend-load-signal.d.ts +56 -0
- package/internals/autoscaling/storage-backend-load-signal.js +73 -0
- package/internals/autoscaling/system-status.d.ts +112 -0
- package/internals/autoscaling/system-status.js +139 -0
- package/internals/autoscaling/weighted-avg.d.ts +5 -0
- package/internals/autoscaling/weighted-avg.js +14 -0
- package/internals/basic-crawler.d.ts +383 -369
- package/internals/basic-crawler.js +1468 -890
- package/internals/cookie-utils.d.ts +40 -0
- package/internals/cookie-utils.js +122 -0
- package/internals/crawler-run.d.ts +76 -0
- package/internals/crawler-run.js +144 -0
- package/internals/crawlers/context-pipeline.d.ts +75 -0
- package/internals/crawlers/context-pipeline.js +109 -0
- package/internals/crawlers/crawler-commons.d.ts +216 -0
- package/internals/crawlers/crawler-commons.js +1 -0
- package/internals/crawlers/error-snapshotter.d.ts +29 -0
- package/internals/crawlers/error-snapshotter.js +116 -0
- package/internals/crawlers/error-tracker.d.ts +51 -0
- package/internals/crawlers/error-tracker.js +316 -0
- package/internals/crawlers/index.d.ts +4 -0
- package/internals/crawlers/index.js +3 -0
- package/internals/crawlers/internals/types.d.ts +7 -0
- package/internals/crawlers/internals/types.js +1 -0
- package/internals/crawlers/statistics.d.ts +326 -0
- package/internals/crawlers/statistics.js +559 -0
- package/internals/crawling-request.d.ts +86 -0
- package/internals/crawling-request.js +127 -0
- package/internals/enqueue-links/enqueue-links.d.ts +147 -0
- package/internals/enqueue-links/enqueue-links.js +35 -0
- package/internals/enqueue-links/index.d.ts +3 -0
- package/internals/enqueue-links/index.js +3 -0
- package/internals/enqueue-links/shared.d.ts +107 -0
- package/internals/enqueue-links/shared.js +228 -0
- package/internals/errors.d.ts +73 -0
- package/internals/errors.js +108 -0
- package/internals/http.d.ts +9 -0
- package/internals/http.js +28 -0
- package/internals/request-timeout.d.ts +54 -0
- package/internals/request-timeout.js +98 -0
- package/internals/router.d.ts +299 -0
- package/internals/router.js +312 -0
- package/internals/send-request.d.ts +4 -4
- package/internals/send-request.js +18 -8
- package/internals/session-pool/consts.d.ts +1 -0
- package/internals/session-pool/consts.js +1 -0
- package/internals/session-pool/errors.d.ts +7 -0
- package/internals/session-pool/errors.js +11 -0
- package/internals/session-pool/fingerprint.d.ts +9 -0
- package/internals/session-pool/fingerprint.js +30 -0
- package/internals/session-pool/index.d.ts +4 -0
- package/internals/session-pool/index.js +4 -0
- package/internals/session-pool/session-pool.d.ts +193 -0
- package/internals/session-pool/session-pool.js +453 -0
- package/internals/session-pool/session.d.ts +151 -0
- package/internals/session-pool/session.js +225 -0
- package/internals/sitemap-request-loader.d.ts +158 -0
- package/internals/sitemap-request-loader.js +427 -0
- package/internals/throttling-request-manager.d.ts +229 -0
- package/internals/throttling-request-manager.js +860 -0
- package/package.json +23 -17
- package/index.d.ts.map +0 -1
- package/index.js.map +0 -1
- package/internals/basic-crawler.d.ts.map +0 -1
- package/internals/basic-crawler.js.map +0 -1
- package/internals/send-request.d.ts.map +0 -1
- package/internals/send-request.js.map +0 -1
|
@@ -0,0 +1,453 @@
|
|
|
1
|
+
import { KeyValueStore, RecoverableState, serviceLocator, validators } from '@crawlee/core';
|
|
2
|
+
import { parseArgument, schemas } from '@crawlee/utils/internal';
|
|
3
|
+
import { AsyncQueue } from '@sapphire/async-queue';
|
|
4
|
+
import { z } from 'zod';
|
|
5
|
+
import { createDefaultSessionFingerprint } from './fingerprint.js';
|
|
6
|
+
import { Session } from './session.js';
|
|
7
|
+
/** Default upper bound on the number of sessions the pool keeps around. */
|
|
8
|
+
const MAX_POOL_SIZE = 1000;
|
|
9
|
+
/** Prefix of the default key the pool persists its state under; the pool id is appended to it. */
|
|
10
|
+
const PERSIST_STATE_KEY = 'CRAWLEE_SESSION_POOL_STATE';
|
|
11
|
+
// Runtime mirror of `SessionReuseStrategy` for the zod schema below. Declared this way round - and not as
|
|
12
|
+
// `typeof SESSION_REUSE_STRATEGIES[number]` - so the public type is a plain literal union that does not drag
|
|
13
|
+
// the array into the API surface. `satisfies` keeps the array from naming a strategy the type does not have.
|
|
14
|
+
const SESSION_REUSE_STRATEGIES = [
|
|
15
|
+
'random',
|
|
16
|
+
'round-robin',
|
|
17
|
+
'use-until-failure',
|
|
18
|
+
];
|
|
19
|
+
// `schemas.anyObject` passes values through by reference (object schemas return a pruned plain
|
|
20
|
+
// copy), so class instances like loggers keep their prototype.
|
|
21
|
+
const sessionPoolOptionsSchema = z.strictObject({
|
|
22
|
+
id: z.union([schemas.anyNumber, z.string()]).optional(),
|
|
23
|
+
maxPoolSize: schemas.anyNumber.default(MAX_POOL_SIZE),
|
|
24
|
+
persistStateKeyValueStoreId: z.string().optional(),
|
|
25
|
+
persistStateKey: z.string().optional(),
|
|
26
|
+
createSessionFunction: schemas.anyFunction.optional(),
|
|
27
|
+
sessionOptions: schemas.anyObject.default(() => ({})),
|
|
28
|
+
log: validators.logger.default(() => serviceLocator.getLogger()),
|
|
29
|
+
persistenceOptions: schemas.anyObject.default(() => ({ enable: true })),
|
|
30
|
+
sessionReuseStrategy: z.enum(SESSION_REUSE_STRATEGIES).default('random'),
|
|
31
|
+
});
|
|
32
|
+
const createSessionOptionsSchema = z.strictObject({
|
|
33
|
+
sessionOptions: schemas.anyObject.default(() => ({})),
|
|
34
|
+
});
|
|
35
|
+
// Only what the recreation reads is pinned down - the rest of a session's state is the session's own business.
|
|
36
|
+
const persistedSessionState = z.custom((value) => typeof value === 'object' &&
|
|
37
|
+
value !== null &&
|
|
38
|
+
typeof value.createdAt === 'string' &&
|
|
39
|
+
typeof value.expiresAt === 'string', 'Expected a persisted session with `createdAt` and `expiresAt` timestamps');
|
|
40
|
+
const persistedSessionPoolState = z.object({
|
|
41
|
+
usableSessionsCount: z.number(),
|
|
42
|
+
retiredSessionsCount: z.number(),
|
|
43
|
+
sessions: z.array(persistedSessionState),
|
|
44
|
+
});
|
|
45
|
+
const sessionPoolInternalState = z.object({
|
|
46
|
+
sessions: z.array(z.instanceof(Session)),
|
|
47
|
+
});
|
|
48
|
+
/**
|
|
49
|
+
* The conversion between the pool's sessions and its persisted record. Decoding recreates the sessions through
|
|
50
|
+
* the pool's factory and keeps only the usable ones, which is why the codec is built per pool.
|
|
51
|
+
*/
|
|
52
|
+
function buildSessionPoolStateCodec(pool) {
|
|
53
|
+
return z.codec(persistedSessionPoolState, sessionPoolInternalState, {
|
|
54
|
+
decode: async (record) => {
|
|
55
|
+
const sessions = [];
|
|
56
|
+
for (const sessionState of record.sessions) {
|
|
57
|
+
const session = await pool.recreateSession({
|
|
58
|
+
...sessionState,
|
|
59
|
+
createdAt: new Date(sessionState.createdAt),
|
|
60
|
+
expiresAt: new Date(sessionState.expiresAt),
|
|
61
|
+
});
|
|
62
|
+
if (session.isUsable()) {
|
|
63
|
+
sessions.push(session);
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
pool.log.debug(`${sessions.length} active sessions loaded from KeyValueStore`);
|
|
67
|
+
return { sessions };
|
|
68
|
+
},
|
|
69
|
+
encode: ({ sessions }) => pool.toRecord(sessions),
|
|
70
|
+
});
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* Handles the rotation, creation and persistence of user-like sessions.
|
|
74
|
+
* Creates a pool of {@link Session} instances, that are randomly rotated.
|
|
75
|
+
* When some session is marked as blocked, it is removed and new one is created instead (the pool never returns an unusable session).
|
|
76
|
+
* Learn more in the {@doclink guides/session-management | Session management guide}.
|
|
77
|
+
*
|
|
78
|
+
* Session pool is already integrated into crawlers and is always active.
|
|
79
|
+
* All public methods are lazy-initialized — the pool initializes itself on first use.
|
|
80
|
+
*
|
|
81
|
+
* You can configure the pool with many options. See the {@link SessionPoolOptions}.
|
|
82
|
+
* Session pool is by default persisted in default {@link KeyValueStore}.
|
|
83
|
+
* If you want to have one pool for all runs you have to specify
|
|
84
|
+
* {@link SessionPoolOptions.persistStateKeyValueStoreId}.
|
|
85
|
+
*
|
|
86
|
+
* **Advanced usage:**
|
|
87
|
+
*
|
|
88
|
+
* ```javascript
|
|
89
|
+
* const sessionPool = new SessionPool({
|
|
90
|
+
* maxPoolSize: 25,
|
|
91
|
+
* sessionOptions:{
|
|
92
|
+
* maxAgeSecs: 10,
|
|
93
|
+
* maxUsageCount: 150, // for example when you know that the site blocks after 150 requests.
|
|
94
|
+
* },
|
|
95
|
+
* persistStateKeyValueStoreId: 'my-key-value-store-for-sessions',
|
|
96
|
+
* persistStateKey: 'my-session-pool',
|
|
97
|
+
* });
|
|
98
|
+
*
|
|
99
|
+
* // Get random session from the pool
|
|
100
|
+
* const session1 = await sessionPool.getSession();
|
|
101
|
+
* const session2 = await sessionPool.getSession();
|
|
102
|
+
* const session3 = await sessionPool.getSession();
|
|
103
|
+
*
|
|
104
|
+
* // Now you can mark the session either failed or successful
|
|
105
|
+
*
|
|
106
|
+
* // Marks session as bad after unsuccessful usage -> it increases error count (soft retire)
|
|
107
|
+
* session1.markBad()
|
|
108
|
+
*
|
|
109
|
+
* // Marks as successful.
|
|
110
|
+
* session2.markGood()
|
|
111
|
+
*
|
|
112
|
+
* // Retires session -> session is removed from the pool
|
|
113
|
+
* session3.retire()
|
|
114
|
+
*
|
|
115
|
+
* ```
|
|
116
|
+
*
|
|
117
|
+
* **Default session allocation flow:*
|
|
118
|
+
* 1. Until the `SessionPool` reaches `maxPoolSize`, new sessions are created, provided to the user and added to the pool
|
|
119
|
+
* 2. Blocked/retired sessions stay in the pool but are never provided to the user
|
|
120
|
+
* 3. Once the pool is full (live plus blocked session count reaches `maxPoolSize`), a random session from the pool is provided.
|
|
121
|
+
* 4. If a blocked session would be picked, instead all blocked sessions are evicted from the pool and a new session is created and provided
|
|
122
|
+
*
|
|
123
|
+
* @category Scaling
|
|
124
|
+
*/
|
|
125
|
+
export class SessionPool {
|
|
126
|
+
static #nextId = 0;
|
|
127
|
+
id;
|
|
128
|
+
#log;
|
|
129
|
+
#maxPoolSize;
|
|
130
|
+
#createSessionFunction;
|
|
131
|
+
#sessionMap = new Map();
|
|
132
|
+
#sessionOptions;
|
|
133
|
+
#persistStateKeyValueStoreId;
|
|
134
|
+
#persistStateKey;
|
|
135
|
+
#persistenceOptions;
|
|
136
|
+
#sessionReuseStrategy;
|
|
137
|
+
/**
|
|
138
|
+
* The sessions live in a {@link RecoverableState} that handles the loading, recreation and periodic
|
|
139
|
+
* persistence of the record under `persistStateKey`.
|
|
140
|
+
*/
|
|
141
|
+
#state;
|
|
142
|
+
#stateCodec;
|
|
143
|
+
#initPromise;
|
|
144
|
+
#queue = new AsyncQueue();
|
|
145
|
+
#roundRobinIndex = 0;
|
|
146
|
+
constructor(options = {}) {
|
|
147
|
+
const { id, maxPoolSize, persistStateKeyValueStoreId, persistStateKey, createSessionFunction, sessionOptions, log, persistenceOptions, sessionReuseStrategy, } = parseArgument(options, sessionPoolOptionsSchema);
|
|
148
|
+
this.id = id != null ? String(id) : String(SessionPool.#nextId++);
|
|
149
|
+
this.#sessionReuseStrategy = sessionReuseStrategy;
|
|
150
|
+
this.#log = log.child({ prefix: 'SessionPool' });
|
|
151
|
+
this.#persistenceOptions = persistenceOptions;
|
|
152
|
+
// Pool Configuration
|
|
153
|
+
this.#maxPoolSize = maxPoolSize;
|
|
154
|
+
this.#createSessionFunction = createSessionFunction || this.#defaultCreateSessionFunction;
|
|
155
|
+
// Session configuration. The pool-scoped logger is merged into per-call sessionOptions inside
|
|
156
|
+
// `invokeCreateSessionFunction`, so every Session inherits it without custom createSessionFunctions
|
|
157
|
+
// having to know about it.
|
|
158
|
+
this.#sessionOptions = {
|
|
159
|
+
...sessionOptions,
|
|
160
|
+
log: this.#log,
|
|
161
|
+
};
|
|
162
|
+
// Session keyValueStore
|
|
163
|
+
this.#persistStateKeyValueStoreId = persistStateKeyValueStoreId;
|
|
164
|
+
this.#persistStateKey = persistStateKey ?? `${PERSIST_STATE_KEY}_${this.id}`;
|
|
165
|
+
this.#stateCodec = buildSessionPoolStateCodec({
|
|
166
|
+
recreateSession: (sessionState) => this.#invokeCreateSessionFunction(sessionState),
|
|
167
|
+
toRecord: (sessions) => this.#buildPersistedState(sessions),
|
|
168
|
+
log: this.#log,
|
|
169
|
+
});
|
|
170
|
+
this.#state = new RecoverableState({
|
|
171
|
+
defaultState: () => ({ sessions: [] }),
|
|
172
|
+
persistStateKey: this.#persistStateKey,
|
|
173
|
+
persistenceEnabled: !!this.#persistenceOptions.enable,
|
|
174
|
+
logger: this.#log,
|
|
175
|
+
keyValueStore: this.#persistenceOptions.enable && persistStateKeyValueStoreId
|
|
176
|
+
? KeyValueStore.open({ id: persistStateKeyValueStoreId }, { configuration: serviceLocator.getConfiguration() })
|
|
177
|
+
: undefined,
|
|
178
|
+
// The codec validates in the decode direction, so it is a Standard Schema as-is; encoding needs a call.
|
|
179
|
+
deserialize: this.#stateCodec,
|
|
180
|
+
serialize: (state) => this.#stateCodec.encode(state),
|
|
181
|
+
});
|
|
182
|
+
this.#state.reset();
|
|
183
|
+
}
|
|
184
|
+
/**
|
|
185
|
+
* Gets count of usable sessions in the pool.
|
|
186
|
+
*/
|
|
187
|
+
async usableSessionsCount() {
|
|
188
|
+
await this.#ensureInitialized();
|
|
189
|
+
return this.#state.currentValue.sessions.filter((session) => session.isUsable()).length;
|
|
190
|
+
}
|
|
191
|
+
/**
|
|
192
|
+
* Gets count of retired sessions in the pool.
|
|
193
|
+
*/
|
|
194
|
+
async retiredSessionsCount() {
|
|
195
|
+
await this.#ensureInitialized();
|
|
196
|
+
return this.#state.currentValue.sessions.filter((session) => !session.isUsable()).length;
|
|
197
|
+
}
|
|
198
|
+
/**
|
|
199
|
+
* Starts periodic state persistence and potentially loads SessionPool state from {@link KeyValueStore}.
|
|
200
|
+
* Called automatically on first use of any public method.
|
|
201
|
+
*/
|
|
202
|
+
async #ensureInitialized() {
|
|
203
|
+
if (!this.#initPromise) {
|
|
204
|
+
this.#initPromise = this.#setupPool();
|
|
205
|
+
}
|
|
206
|
+
return this.#initPromise;
|
|
207
|
+
}
|
|
208
|
+
async #setupPool() {
|
|
209
|
+
await this.#state.initialize();
|
|
210
|
+
for (const session of this.#state.currentValue.sessions) {
|
|
211
|
+
this.#sessionMap.set(session.id, session);
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
/**
|
|
215
|
+
* Adds a new session to the session pool. The pool automatically creates sessions up to the maximum size of the pool,
|
|
216
|
+
* but this allows you to add more sessions once the max pool size is reached.
|
|
217
|
+
* This also allows you to add session with overridden session options (e.g. with specific session id).
|
|
218
|
+
*
|
|
219
|
+
* If the pool already contains a session with the same id, it is replaced when it is no longer usable
|
|
220
|
+
* (e.g. retired), and an error is thrown otherwise.
|
|
221
|
+
* @param [options] The configuration options for the session being added to the session pool.
|
|
222
|
+
*/
|
|
223
|
+
async addSession(options = {}) {
|
|
224
|
+
await this.#ensureInitialized();
|
|
225
|
+
const { id } = options;
|
|
226
|
+
if (id) {
|
|
227
|
+
const existingSession = this.#sessionMap.get(id);
|
|
228
|
+
if (existingSession) {
|
|
229
|
+
if (existingSession.isUsable()) {
|
|
230
|
+
throw new Error(`Cannot add session with id '${id}' as it already exists in the pool`);
|
|
231
|
+
}
|
|
232
|
+
this.#removeSession(existingSession);
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
if (!this.#hasSpaceForSession()) {
|
|
236
|
+
this.#removeRetiredSessions();
|
|
237
|
+
}
|
|
238
|
+
const newSession = options instanceof Session ? options : await this.#invokeCreateSessionFunction(options);
|
|
239
|
+
this.#log.debug(`Adding new Session - ${newSession.id}`);
|
|
240
|
+
this.#registerSession(newSession);
|
|
241
|
+
}
|
|
242
|
+
/**
|
|
243
|
+
* Adds a new session to the session pool. The pool automatically creates sessions up to the maximum size of the pool,
|
|
244
|
+
* but this allows you to add more sessions once the max pool size is reached.
|
|
245
|
+
* This also allows you to add session with overridden session options (e.g. with specific session id).
|
|
246
|
+
* @param [options] The configuration options for the session being added to the session pool.
|
|
247
|
+
*/
|
|
248
|
+
async newSession(sessionOptions) {
|
|
249
|
+
await this.#ensureInitialized();
|
|
250
|
+
const newSession = await this.#invokeCreateSessionFunction(sessionOptions);
|
|
251
|
+
this.#registerSession(newSession);
|
|
252
|
+
return newSession;
|
|
253
|
+
}
|
|
254
|
+
/**
|
|
255
|
+
* Gets session.
|
|
256
|
+
* If there is space for new session, it creates and returns new session.
|
|
257
|
+
* If the session pool is full, it picks a session from the pool,
|
|
258
|
+
* If the picked session is usable it is returned, otherwise it creates and returns a new one.
|
|
259
|
+
* @param [sessionId] If provided, it returns the usable session with this id, `undefined` otherwise.
|
|
260
|
+
*/
|
|
261
|
+
async getSession(sessionId) {
|
|
262
|
+
await this.#ensureInitialized();
|
|
263
|
+
await this.#queue.wait();
|
|
264
|
+
try {
|
|
265
|
+
if (sessionId) {
|
|
266
|
+
const session = this.#sessionMap.get(sessionId);
|
|
267
|
+
if (session?.isUsable())
|
|
268
|
+
return session;
|
|
269
|
+
return undefined;
|
|
270
|
+
}
|
|
271
|
+
const pickedSession = this.#pickSession();
|
|
272
|
+
if (pickedSession)
|
|
273
|
+
return pickedSession;
|
|
274
|
+
if (this.#hasSpaceForSession()) {
|
|
275
|
+
return await this.#createSession();
|
|
276
|
+
}
|
|
277
|
+
this.#removeRetiredSessions();
|
|
278
|
+
return await this.#createSession();
|
|
279
|
+
}
|
|
280
|
+
finally {
|
|
281
|
+
this.#queue.shift();
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
/**
|
|
285
|
+
* Discards all sessions in the pool, both usable and retired, resetting it to a blank state.
|
|
286
|
+
*
|
|
287
|
+
* The persisted record is left alone - use {@link SessionPool.resetStore} to clear that as well.
|
|
288
|
+
*/
|
|
289
|
+
reset() {
|
|
290
|
+
this.#state.reset();
|
|
291
|
+
this.#sessionMap.clear();
|
|
292
|
+
this.#roundRobinIndex = 0;
|
|
293
|
+
}
|
|
294
|
+
/**
|
|
295
|
+
* Clears the persisted pool record, leaving the in-memory sessions alone.
|
|
296
|
+
*
|
|
297
|
+
* Throws while the state is still being persisted periodically - the next PERSIST_STATE event would write
|
|
298
|
+
* the record straight back. Call {@link SessionPool.teardown} first, or use {@link SessionPool.reset}
|
|
299
|
+
* to discard the sessions themselves. A no-op if persistence is disabled.
|
|
300
|
+
*/
|
|
301
|
+
async resetStore() {
|
|
302
|
+
await this.#state.resetStore();
|
|
303
|
+
}
|
|
304
|
+
/**
|
|
305
|
+
* Returns an object representing the internal state of the `SessionPool` instance.
|
|
306
|
+
* @internal
|
|
307
|
+
*/
|
|
308
|
+
async getState() {
|
|
309
|
+
await this.#ensureInitialized();
|
|
310
|
+
return this.#buildPersistedState(this.#state.currentValue.sessions);
|
|
311
|
+
}
|
|
312
|
+
/**
|
|
313
|
+
* Persists the current state of the `SessionPool` into the default {@link KeyValueStore}.
|
|
314
|
+
* The state is persisted automatically in regular intervals.
|
|
315
|
+
*/
|
|
316
|
+
async persistState() {
|
|
317
|
+
if (!this.#persistenceOptions.enable) {
|
|
318
|
+
return;
|
|
319
|
+
}
|
|
320
|
+
await this.#ensureInitialized();
|
|
321
|
+
this.#log.debug('Persisting state', {
|
|
322
|
+
persistStateKeyValueStoreId: this.#persistStateKeyValueStoreId,
|
|
323
|
+
persistStateKey: this.#persistStateKey,
|
|
324
|
+
});
|
|
325
|
+
await this.#state
|
|
326
|
+
.persistState()
|
|
327
|
+
.catch((error) => this.#log.warning(`Failed to persist the session pool stats to ${this.#persistStateKey}`, { error }));
|
|
328
|
+
}
|
|
329
|
+
async [Symbol.asyncDispose]() {
|
|
330
|
+
await this.teardown();
|
|
331
|
+
}
|
|
332
|
+
/**
|
|
333
|
+
* Stops the periodic state persistence and persists the state one last time.
|
|
334
|
+
* This function should be called after you are done with using the `SessionPool` instance.
|
|
335
|
+
* Using the pool again afterwards initializes it anew.
|
|
336
|
+
*/
|
|
337
|
+
async teardown() {
|
|
338
|
+
if (!this.#initPromise)
|
|
339
|
+
return;
|
|
340
|
+
await this.#ensureInitialized();
|
|
341
|
+
await this.#state.teardown();
|
|
342
|
+
// The next use initializes the pool again, restoring the record and resuming the periodic persistence -
|
|
343
|
+
// a crawler that is run twice gets a persisting pool both times.
|
|
344
|
+
this.#initPromise = undefined;
|
|
345
|
+
}
|
|
346
|
+
/**
|
|
347
|
+
* Removes retired `Session` instances from `SessionPool`.
|
|
348
|
+
*/
|
|
349
|
+
#removeRetiredSessions() {
|
|
350
|
+
this.#state.currentValue.sessions = this.#state.currentValue.sessions.filter((storedSession) => {
|
|
351
|
+
if (storedSession.isUsable())
|
|
352
|
+
return true;
|
|
353
|
+
this.#sessionMap.delete(storedSession.id);
|
|
354
|
+
this.#log.debug(`Removed Session - ${storedSession.id}`);
|
|
355
|
+
return false;
|
|
356
|
+
});
|
|
357
|
+
}
|
|
358
|
+
/**
|
|
359
|
+
* Removes a `Session` instance from `SessionPool`.
|
|
360
|
+
*/
|
|
361
|
+
#removeSession(session) {
|
|
362
|
+
this.#state.currentValue.sessions = this.#state.currentValue.sessions.filter((storedSession) => storedSession !== session);
|
|
363
|
+
this.#sessionMap.delete(session.id);
|
|
364
|
+
this.#log.debug(`Removed Session - ${session.id}`);
|
|
365
|
+
}
|
|
366
|
+
/**
|
|
367
|
+
* Adds `Session` instance to `SessionPool`.
|
|
368
|
+
* @param newSession `Session` instance to be added.
|
|
369
|
+
*/
|
|
370
|
+
#registerSession(newSession) {
|
|
371
|
+
this.#state.currentValue.sessions.push(newSession);
|
|
372
|
+
this.#sessionMap.set(newSession.id, newSession);
|
|
373
|
+
}
|
|
374
|
+
/**
|
|
375
|
+
* Gets random index.
|
|
376
|
+
*/
|
|
377
|
+
#getRandomIndex() {
|
|
378
|
+
return Math.floor(Math.random() * this.#state.currentValue.sessions.length);
|
|
379
|
+
}
|
|
380
|
+
/**
|
|
381
|
+
* Creates new session without any extra behavior.
|
|
382
|
+
* @param [options]
|
|
383
|
+
* @param [options.sessionOptions] The configuration options for the session being created.
|
|
384
|
+
* @returns New session.
|
|
385
|
+
*/
|
|
386
|
+
async #defaultCreateSessionFunction(options = {}) {
|
|
387
|
+
const { sessionOptions } = parseArgument(options, createSessionOptionsSchema);
|
|
388
|
+
return new Session(sessionOptions);
|
|
389
|
+
}
|
|
390
|
+
/**
|
|
391
|
+
* Invokes `createSessionFunction` with `sessionOptions` already merged from pool-wide defaults and
|
|
392
|
+
* the supplied per-call overrides, so custom implementations don't need to spread `pool.sessionOptions` themselves.
|
|
393
|
+
*
|
|
394
|
+
* A default {@link SessionFingerprint} is generated up front (host OS as
|
|
395
|
+
* `platform`, a random valid `browser`/`device` for that platform). Pool-wide
|
|
396
|
+
* and per-call options override it, and a persisted fingerprint coming
|
|
397
|
+
* through `maybeLoadSessionPool` naturally wins because it arrives in
|
|
398
|
+
* `perCallOptions`.
|
|
399
|
+
*/
|
|
400
|
+
async #invokeCreateSessionFunction(perCallOptions) {
|
|
401
|
+
const sessionOptions = {
|
|
402
|
+
fingerprint: createDefaultSessionFingerprint(),
|
|
403
|
+
...this.#sessionOptions,
|
|
404
|
+
...perCallOptions,
|
|
405
|
+
};
|
|
406
|
+
return this.#createSessionFunction({ sessionOptions });
|
|
407
|
+
}
|
|
408
|
+
/**
|
|
409
|
+
* Creates new session and adds it to the pool.
|
|
410
|
+
* @returns Newly created `Session` instance.
|
|
411
|
+
*/
|
|
412
|
+
async #createSession() {
|
|
413
|
+
const newSession = await this.#invokeCreateSessionFunction();
|
|
414
|
+
this.#registerSession(newSession);
|
|
415
|
+
this.#log.debug(`Created new Session - ${newSession.id}`);
|
|
416
|
+
return newSession;
|
|
417
|
+
}
|
|
418
|
+
/**
|
|
419
|
+
* Decides whether there is enough space for creating new session.
|
|
420
|
+
*/
|
|
421
|
+
#hasSpaceForSession() {
|
|
422
|
+
return this.#state.currentValue.sessions.length < this.#maxPoolSize;
|
|
423
|
+
}
|
|
424
|
+
/**
|
|
425
|
+
* Picks a session from the `SessionPool` according to the configured `sessionReuseStrategy`.
|
|
426
|
+
* Returns `undefined` when no session should be reused and a new one should be created instead.
|
|
427
|
+
*/
|
|
428
|
+
#pickSession() {
|
|
429
|
+
if (this.#sessionReuseStrategy !== 'use-until-failure' && this.#hasSpaceForSession())
|
|
430
|
+
return undefined;
|
|
431
|
+
if (this.#sessionReuseStrategy === 'use-until-failure') {
|
|
432
|
+
return this.#state.currentValue.sessions.find((session) => session.isUsable());
|
|
433
|
+
}
|
|
434
|
+
let picked;
|
|
435
|
+
if (this.#sessionReuseStrategy === 'round-robin') {
|
|
436
|
+
const index = this.#roundRobinIndex % this.#state.currentValue.sessions.length;
|
|
437
|
+
this.#roundRobinIndex = index + 1;
|
|
438
|
+
picked = this.#state.currentValue.sessions[index];
|
|
439
|
+
}
|
|
440
|
+
else {
|
|
441
|
+
picked = this.#state.currentValue.sessions[this.#getRandomIndex()];
|
|
442
|
+
}
|
|
443
|
+
return picked.isUsable() ? picked : undefined;
|
|
444
|
+
}
|
|
445
|
+
/** The persisted form of the pool, which is also what {@link SessionPool.getState} returns. */
|
|
446
|
+
#buildPersistedState(sessions) {
|
|
447
|
+
return {
|
|
448
|
+
usableSessionsCount: sessions.filter((session) => session.isUsable()).length,
|
|
449
|
+
retiredSessionsCount: sessions.filter((session) => !session.isUsable()).length,
|
|
450
|
+
sessions: sessions.map((session) => session.getState()),
|
|
451
|
+
};
|
|
452
|
+
}
|
|
453
|
+
}
|
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
import type { CrawleeLogger } from '@crawlee/core';
|
|
2
|
+
import type { Dictionary, ISession, ProxyInfo, SessionFingerprint, SessionState } from '@crawlee/types';
|
|
3
|
+
import { CookieJar } from 'tough-cookie';
|
|
4
|
+
export interface SessionOptions {
|
|
5
|
+
/** Id of session used for generating fingerprints. It is used as proxy session name. */
|
|
6
|
+
id?: string;
|
|
7
|
+
/**
|
|
8
|
+
* Number of seconds after which the session is considered as expired.
|
|
9
|
+
* @default 3000
|
|
10
|
+
*/
|
|
11
|
+
maxAgeSecs?: number;
|
|
12
|
+
/** Object where custom user data can be stored. For example custom headers. */
|
|
13
|
+
userData?: Dictionary;
|
|
14
|
+
/**
|
|
15
|
+
* Maximum number of marking session as blocked usage.
|
|
16
|
+
* If the `errorScore` reaches the `maxErrorScore` session is marked as block and it is thrown away.
|
|
17
|
+
* It starts at 0. Calling the `markBad` function increases the `errorScore` by 1.
|
|
18
|
+
* Calling the `markGood` will decrease the `errorScore` by `errorScoreDecrement`
|
|
19
|
+
* @default 3
|
|
20
|
+
*/
|
|
21
|
+
maxErrorScore?: number;
|
|
22
|
+
/**
|
|
23
|
+
* It is used for healing the session.
|
|
24
|
+
* For example: if your session is marked bad two times, but it is successful on the third attempt it's errorScore
|
|
25
|
+
* is decremented by this number.
|
|
26
|
+
* @default 0.5
|
|
27
|
+
*/
|
|
28
|
+
errorScoreDecrement?: number;
|
|
29
|
+
/** Date of creation. */
|
|
30
|
+
createdAt?: Date;
|
|
31
|
+
/**
|
|
32
|
+
* Date of expiration.
|
|
33
|
+
* @default createdAt + maxAgeSecs
|
|
34
|
+
*/
|
|
35
|
+
expiresAt?: Date;
|
|
36
|
+
/**
|
|
37
|
+
* Indicates how many times the session has been used.
|
|
38
|
+
* @default 0
|
|
39
|
+
*/
|
|
40
|
+
usageCount?: number;
|
|
41
|
+
/**
|
|
42
|
+
* Session should be used only a limited amount of times.
|
|
43
|
+
* This number indicates how many times the session is going to be used, before it is thrown away.
|
|
44
|
+
* @default 50
|
|
45
|
+
*/
|
|
46
|
+
maxUsageCount?: number;
|
|
47
|
+
/**
|
|
48
|
+
* Marks the session as already retired. Used when restoring a previously persisted session
|
|
49
|
+
* so that `isUsable()` reflects the terminal state regardless of error score or usage count.
|
|
50
|
+
* @default false
|
|
51
|
+
*/
|
|
52
|
+
retired?: boolean;
|
|
53
|
+
/** @internal */
|
|
54
|
+
log?: CrawleeLogger;
|
|
55
|
+
errorScore?: number;
|
|
56
|
+
cookieJar?: CookieJar;
|
|
57
|
+
proxyInfo?: ProxyInfo;
|
|
58
|
+
/**
|
|
59
|
+
* Browser / HTTP client fingerprint tied to this session. Backends use this to make
|
|
60
|
+
* repeated requests with the same session look consistent (same user-agent, headers,
|
|
61
|
+
* TLS profile). See {@link SessionFingerprint}.
|
|
62
|
+
*/
|
|
63
|
+
fingerprint?: SessionFingerprint;
|
|
64
|
+
}
|
|
65
|
+
/**
|
|
66
|
+
* Sessions are used to store information such as cookies and can be used for generating fingerprints and proxy sessions.
|
|
67
|
+
* You can imagine each session as a specific user, with its own cookies, IP (via proxy) and potentially a unique browser fingerprint.
|
|
68
|
+
* Session internal state can be enriched with custom user data for example some authorization tokens and specific headers in general.
|
|
69
|
+
* @category Scaling
|
|
70
|
+
*/
|
|
71
|
+
export declare class Session implements ISession {
|
|
72
|
+
#private;
|
|
73
|
+
readonly id: string;
|
|
74
|
+
readonly userData: Dictionary;
|
|
75
|
+
get errorScore(): number;
|
|
76
|
+
get usageCount(): number;
|
|
77
|
+
get maxErrorScore(): number;
|
|
78
|
+
get errorScoreDecrement(): number;
|
|
79
|
+
get expiresAt(): Date;
|
|
80
|
+
get createdAt(): Date;
|
|
81
|
+
get maxUsageCount(): number;
|
|
82
|
+
get cookieJar(): CookieJar;
|
|
83
|
+
get proxyInfo(): ProxyInfo | undefined;
|
|
84
|
+
get fingerprint(): SessionFingerprint | undefined;
|
|
85
|
+
set fingerprint(fingerprint: SessionFingerprint | undefined);
|
|
86
|
+
/**
|
|
87
|
+
* `true` once {@link Session.retire|`retire()`} has been called. Retirement is terminal:
|
|
88
|
+
* a retired session is never picked by the pool and cannot be revived via `markGood()`.
|
|
89
|
+
*/
|
|
90
|
+
get retired(): boolean;
|
|
91
|
+
/**
|
|
92
|
+
* Session configuration.
|
|
93
|
+
*/
|
|
94
|
+
constructor(options?: SessionOptions);
|
|
95
|
+
/**
|
|
96
|
+
* Indicates whether the session is blocked.
|
|
97
|
+
* Session is blocked once it reaches the `maxErrorScore`.
|
|
98
|
+
*/
|
|
99
|
+
isBlocked(): boolean;
|
|
100
|
+
/**
|
|
101
|
+
* Indicates whether the session is expired.
|
|
102
|
+
* Session expiration is determined by the `maxAgeSecs`.
|
|
103
|
+
* Once the session is older than `createdAt + maxAgeSecs` the session is considered expired.
|
|
104
|
+
*/
|
|
105
|
+
isExpired(): boolean;
|
|
106
|
+
/**
|
|
107
|
+
* Indicates whether the session is used maximum number of times.
|
|
108
|
+
* Session maximum usage count can be changed by `maxUsageCount` parameter.
|
|
109
|
+
*/
|
|
110
|
+
isMaxUsageCountReached(): boolean;
|
|
111
|
+
/**
|
|
112
|
+
* Indicates whether the session can be used for next requests.
|
|
113
|
+
* Session is usable when it is not retired, not expired, not blocked and the maximum usage count has not be reached.
|
|
114
|
+
*/
|
|
115
|
+
isUsable(): boolean;
|
|
116
|
+
/**
|
|
117
|
+
* This method should be called after a successful session usage.
|
|
118
|
+
* It increases `usageCount` and potentially lowers the `errorScore` by the `errorScoreDecrement`.
|
|
119
|
+
*/
|
|
120
|
+
markGood(): void;
|
|
121
|
+
/**
|
|
122
|
+
* Gets session state for persistence in KeyValueStore.
|
|
123
|
+
*
|
|
124
|
+
* @internal
|
|
125
|
+
*/
|
|
126
|
+
getState(): SessionState;
|
|
127
|
+
/**
|
|
128
|
+
* Permanently retires the session — `isUsable()` will return `false` from here on,
|
|
129
|
+
* and no `markGood()` / `markBad()` can revive it. Calling `retire()` again is a no-op.
|
|
130
|
+
*
|
|
131
|
+
* Use this when you're confident the session itself is the problem (e.g. a `403` response).
|
|
132
|
+
* For transient external failures (such as `5XX` responses), use `markBad()` instead.
|
|
133
|
+
*/
|
|
134
|
+
retire(): void;
|
|
135
|
+
/**
|
|
136
|
+
* Increases usage and error count.
|
|
137
|
+
* Should be used when the session has been used unsuccessfully. For example because of timeouts.
|
|
138
|
+
*/
|
|
139
|
+
markBad(): void;
|
|
140
|
+
/**
|
|
141
|
+
* Returns cookies saved with the session in the typical
|
|
142
|
+
* key1=value1; key2=value2 format, ready to be used in
|
|
143
|
+
* a cookie header or elsewhere.
|
|
144
|
+
* @returns Represents `Cookie` header.
|
|
145
|
+
*/
|
|
146
|
+
getCookieString(url: string): Promise<string>;
|
|
147
|
+
/**
|
|
148
|
+
* Sets a cookie within this session for the specific URL.
|
|
149
|
+
*/
|
|
150
|
+
setCookie(rawCookie: string, url: string): Promise<void>;
|
|
151
|
+
}
|