@crawlee/core 3.0.1-beta.14 → 3.0.1-beta.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/cookie_utils.d.ts +43 -0
- package/cookie_utils.d.ts.map +1 -0
- package/cookie_utils.js +118 -0
- package/cookie_utils.js.map +1 -0
- package/crawlers/crawler_utils.d.ts +0 -10
- package/crawlers/crawler_utils.d.ts.map +1 -1
- package/crawlers/crawler_utils.js +1 -47
- package/crawlers/crawler_utils.js.map +1 -1
- package/enqueue_links/enqueue_links.d.ts +5 -1
- package/enqueue_links/enqueue_links.d.ts.map +1 -1
- package/enqueue_links/enqueue_links.js +10 -14
- package/enqueue_links/enqueue_links.js.map +1 -1
- package/index.d.ts +1 -0
- package/index.d.ts.map +1 -1
- package/index.js +1 -0
- package/index.js.map +1 -1
- package/index.mjs +5 -2
- package/package.json +5 -5
- package/session_pool/index.d.ts +0 -1
- package/session_pool/index.d.ts.map +1 -1
- package/session_pool/index.js +0 -1
- package/session_pool/index.js.map +1 -1
- package/session_pool/session.d.ts +3 -19
- package/session_pool/session.d.ts.map +1 -1
- package/session_pool/session.js +6 -52
- package/session_pool/session.js.map +1 -1
- package/tsconfig.build.tsbuildinfo +1 -1
- package/session_pool/session_utils.d.ts +0 -11
- package/session_pool/session_utils.d.ts.map +0 -1
- package/session_pool/session_utils.js +0 -22
- package/session_pool/session_utils.js.map +0 -1
package/session_pool/session.js
CHANGED
|
@@ -9,7 +9,7 @@ const tough_cookie_1 = require("tough-cookie");
|
|
|
9
9
|
const constants_1 = require("../constants");
|
|
10
10
|
const log_1 = require("../log");
|
|
11
11
|
const events_1 = require("./events");
|
|
12
|
-
const
|
|
12
|
+
const cookie_utils_1 = require("../cookie_utils");
|
|
13
13
|
// CONSTANTS
|
|
14
14
|
const DEFAULT_SESSION_MAX_AGE_SECS = 3000;
|
|
15
15
|
/**
|
|
@@ -117,7 +117,7 @@ class Session {
|
|
|
117
117
|
log: ow_1.default.optional.object,
|
|
118
118
|
}));
|
|
119
119
|
const { sessionPool, id = `session_${(0, utilities_1.cryptoRandomObjectId)(10)}`, cookieJar = new tough_cookie_1.CookieJar(), maxAgeSecs = DEFAULT_SESSION_MAX_AGE_SECS, userData = {}, maxErrorScore = 3, errorScoreDecrement = 0.5, createdAt = new Date(), usageCount = 0, errorScore = 0, maxUsageCount = 50, log = log_1.log, } = options;
|
|
120
|
-
const { expiresAt =
|
|
120
|
+
const { expiresAt = (0, cookie_utils_1.getDefaultCookieExpirationDate)(maxAgeSecs) } = options;
|
|
121
121
|
this.log = log.child({ prefix: 'Session' });
|
|
122
122
|
this.cookieJar = cookieJar.setCookie ? cookieJar : tough_cookie_1.CookieJar.fromJSON(JSON.stringify(cookieJar));
|
|
123
123
|
this.id = id;
|
|
@@ -132,7 +132,6 @@ class Session {
|
|
|
132
132
|
this.errorScore = errorScore; // indicates number of markBaded request with the session
|
|
133
133
|
this.maxUsageCount = maxUsageCount;
|
|
134
134
|
this.sessionPool = sessionPool;
|
|
135
|
-
this._cookieObjectToTough = this._cookieObjectToTough.bind(this);
|
|
136
135
|
}
|
|
137
136
|
/**
|
|
138
137
|
* Indicates whether the session is blocked.
|
|
@@ -240,7 +239,7 @@ class Session {
|
|
|
240
239
|
*/
|
|
241
240
|
setCookiesFromResponse(response) {
|
|
242
241
|
try {
|
|
243
|
-
const cookies = (0,
|
|
242
|
+
const cookies = (0, cookie_utils_1.getCookiesFromResponse)(response).filter((c) => c);
|
|
244
243
|
this._setCookies(cookies, typeof response.url === 'function' ? response.url() : response.url);
|
|
245
244
|
}
|
|
246
245
|
catch (e) {
|
|
@@ -263,16 +262,16 @@ class Session {
|
|
|
263
262
|
* ```
|
|
264
263
|
*/
|
|
265
264
|
setCookies(cookies, url) {
|
|
266
|
-
const normalizedCookies = cookies.map(this.
|
|
265
|
+
const normalizedCookies = cookies.map((c) => (0, cookie_utils_1.browserPoolCookieToToughCookie)(c, this.maxAgeSecs));
|
|
267
266
|
this._setCookies(normalizedCookies, url);
|
|
268
267
|
}
|
|
269
268
|
/**
|
|
270
|
-
* Returns cookies in a format compatible with puppeteer and ready to be used with `page.setCookie`.
|
|
269
|
+
* Returns cookies in a format compatible with puppeteer/playwright and ready to be used with `page.setCookie`.
|
|
271
270
|
* @param url website url. Only cookies stored for this url will be returned
|
|
272
271
|
*/
|
|
273
272
|
getCookies(url) {
|
|
274
273
|
const cookies = this.cookieJar.getCookiesSync(url);
|
|
275
|
-
return cookies.map(
|
|
274
|
+
return cookies.map((c) => (0, cookie_utils_1.toughCookieToBrowserPoolCookie)(c));
|
|
276
275
|
}
|
|
277
276
|
/**
|
|
278
277
|
* Returns cookies saved with the session in the typical
|
|
@@ -289,43 +288,6 @@ class Session {
|
|
|
289
288
|
setCookie(rawCookie, url) {
|
|
290
289
|
this.cookieJar.setCookieSync(rawCookie, url);
|
|
291
290
|
}
|
|
292
|
-
/**
|
|
293
|
-
* Transforms puppeteer cookie to tough-cookie.
|
|
294
|
-
* @param cookieObject Cookie object (for instance from the `page.cookies` method).
|
|
295
|
-
*/
|
|
296
|
-
_cookieObjectToTough(cookieObject) {
|
|
297
|
-
const isExpiresValid = cookieObject.expires && typeof cookieObject.expires === 'number' && cookieObject.expires > 0;
|
|
298
|
-
const expires = isExpiresValid ? new Date(cookieObject.expires * 1000) : this._getDefaultCookieExpirationDate(this.maxAgeSecs);
|
|
299
|
-
const domain = typeof cookieObject.domain === 'string' && cookieObject.domain.startsWith('.')
|
|
300
|
-
? cookieObject.domain.slice(1)
|
|
301
|
-
: cookieObject.domain;
|
|
302
|
-
return new tough_cookie_1.Cookie({
|
|
303
|
-
key: cookieObject.name,
|
|
304
|
-
value: cookieObject.value,
|
|
305
|
-
expires,
|
|
306
|
-
domain,
|
|
307
|
-
path: cookieObject.path,
|
|
308
|
-
secure: cookieObject.secure,
|
|
309
|
-
httpOnly: cookieObject.httpOnly,
|
|
310
|
-
});
|
|
311
|
-
}
|
|
312
|
-
/**
|
|
313
|
-
* Transforms tough-cookie to puppeteer cookie.
|
|
314
|
-
* @param toughCookie Cookie from CookieJar
|
|
315
|
-
* @return Cookie from Puppeteer
|
|
316
|
-
*/
|
|
317
|
-
_toughCookieToPuppeteer(toughCookie) {
|
|
318
|
-
return {
|
|
319
|
-
name: toughCookie.key,
|
|
320
|
-
value: toughCookie.value,
|
|
321
|
-
// Puppeteer and Playwright expect 'expires' to be 'Unix time in seconds', not ms
|
|
322
|
-
expires: new Date(toughCookie.expires).getTime() / 1000,
|
|
323
|
-
domain: toughCookie.domain,
|
|
324
|
-
path: toughCookie.path,
|
|
325
|
-
secure: toughCookie.secure,
|
|
326
|
-
httpOnly: toughCookie.httpOnly,
|
|
327
|
-
};
|
|
328
|
-
}
|
|
329
291
|
/**
|
|
330
292
|
* Sets cookies.
|
|
331
293
|
*/
|
|
@@ -345,14 +307,6 @@ class Session {
|
|
|
345
307
|
this.log.debug('Could not set cookies.', { errorMessages });
|
|
346
308
|
}
|
|
347
309
|
}
|
|
348
|
-
/**
|
|
349
|
-
* Calculate cookie expiration date
|
|
350
|
-
* @param maxAgeSecs
|
|
351
|
-
* @returns Calculated date by session max age seconds.
|
|
352
|
-
*/
|
|
353
|
-
_getDefaultCookieExpirationDate(maxAgeSecs) {
|
|
354
|
-
return new Date(Date.now() + (maxAgeSecs * 1000));
|
|
355
|
-
}
|
|
356
310
|
/**
|
|
357
311
|
* Checks if session is not usable. if it is not retires the session.
|
|
358
312
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session.js","sourceRoot":"","sources":["../../src/session_pool/session.ts"],"names":[],"mappings":";;;;AACA,gDAAwD;AAGxD,6CAA2C;AAC3C,oDAAoB;
|
|
1
|
+
{"version":3,"file":"session.js","sourceRoot":"","sources":["../../src/session_pool/session.ts"],"names":[],"mappings":";;;;AACA,gDAAwD;AAGxD,6CAA2C;AAC3C,oDAAoB;AAEpB,+CAAyC;AACzC,4CAAoD;AACpD,gCAA2C;AAC3C,qCAAiD;AACjD,kDAAyJ;AAEzJ,YAAY;AACZ,MAAM,4BAA4B,GAAG,IAAI,CAAC;AA6E1C;;;;;GAKG;AACH,MAAa,OAAO;IAehB;;OAEG;IACH,YAAY,OAAuB;QAjBnC;;;;;WAAoB;QACpB;;;;;WAA2B;QAC3B;;;;;WAAqB;QACrB;;;;;WAA8B;QAC9B;;;;;WAAoC;QACpC;;;;;WAAwB;QACxB;;;;;WAAwB;QACxB;;;;;WAA2B;QAC3B;;;;;WAA8B;QAC9B;;;;;WAA0D;QAC1D;;;;;WAA2B;QAC3B;;;;;WAA6B;QAC7B;;;;;WAAiB;QAMb,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7B,WAAW,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC,0BAAY,CAAC;YAC/C,EAAE,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACtB,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC7B,UAAU,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC9B,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC5B,aAAa,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACjC,mBAAmB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACvC,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,IAAI;YAC3B,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,IAAI;YAC3B,UAAU,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC9B,UAAU,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC9B,aAAa,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACjC,GAAG,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;SAC1B,CAAC,CAAC,CAAC;QAEJ,MAAM,EACF,WAAW,EACX,EAAE,GAAG,WAAW,IAAA,gCAAoB,EAAC,EAAE,CAAC,EAAE,EAC1C,SAAS,GAAG,IAAI,wBAAS,EAAE,EAC3B,UAAU,GAAG,4BAA4B,EACzC,QAAQ,GAAG,EAAE,EACb,aAAa,GAAG,CAAC,EACjB,mBAAmB,GAAG,GAAG,EACzB,SAAS,GAAG,IAAI,IAAI,EAAE,EACtB,UAAU,GAAG,CAAC,EACd,UAAU,GAAG,CAAC,EACd,aAAa,GAAG,EAAE,EAClB,GAAG,GAAG,SAAU,GACnB,GAAG,OAAO,CAAC;QAEZ,MAAM,EAAE,SAAS,GAAG,IAAA,6CAA8B,EAAC,UAAU,CAAC,EAAE,GAAG,OAAO,CAAC;QAE3E,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;QAE5C,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC,SAAoB,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,wBAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC;QAC5G,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;QACnC,IAAI,CAAC,mBAAmB,GAAG,mBAAmB,CAAC;QAE/C,WAAW;QACX,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC,CAAC,qDAAqD;QACnF,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC,CAAC,yDAAyD;QACvF,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;QACnC,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;IACnC,CAAC;IAED;;;OAGG;IACH,SAAS;QACL,OAAO,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,aAAa,CAAC;IACjD,CAAC;IAED;;;;OAIG;IACH,SAAS;QACL,OAAO,IAAI,CAAC,SAAS,IAAI,IAAI,IAAI,EAAE,CAAC;IACxC,CAAC;IAED;;;OAGG;IACH,sBAAsB;QAClB,OAAO,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,aAAa,CAAC;IACjD,CAAC;IAED;;;OAGG;IACH,QAAQ;QACJ,OAAO,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,sBAAsB,EAAE,CAAC;IACpF,CAAC;IAED;;;OAGG;IACH,QAAQ;QACJ,IAAI,CAAC,UAAU,IAAI,CAAC,CAAC;QAErB,IAAI,IAAI,CAAC,UAAU,GAAG,CAAC,EAAE;YACrB,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,mBAAmB,CAAC;SAC/C;QAED,IAAI,CAAC,gBAAgB,EAAE,CAAC;IAC5B,CAAC;IAED;;;OAGG;IACH,QAAQ;QACJ,OAAO;YACH,EAAE,EAAE,IAAI,CAAC,EAAE;YACX,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE;YAClC,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,aAAa,EAAE,IAAI,CAAC,aAAa;YACjC,mBAAmB,EAAE,IAAI,CAAC,mBAAmB;YAC7C,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE;YACvC,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE;YACvC,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,aAAa,EAAE,IAAI,CAAC,aAAa;YACjC,UAAU,EAAE,IAAI,CAAC,UAAU;SAC9B,CAAC;IACN,CAAC;IAED;;;;;;OAMG;IACH,MAAM;QACF,6DAA6D;QAC7D,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,aAAa,CAAC;QACtC,IAAI,CAAC,UAAU,IAAI,CAAC,CAAC;QAErB,wDAAwD;QACxD,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,8BAAqB,EAAE,IAAI,CAAC,CAAC;IACvD,CAAC;IAED;;;OAGG;IACH,OAAO;QACH,IAAI,CAAC,UAAU,IAAI,CAAC,CAAC;QACrB,IAAI,CAAC,UAAU,IAAI,CAAC,CAAC;QAErB,IAAI,CAAC,gBAAgB,EAAE,CAAC;IAC5B,CAAC;IAED;;;;;;;;OAQG;IACH,0BAA0B,CAAC,UAAkB,EAAE,qBAA+B,EAAE;QAC5E,MAAM,SAAS,GAAG,gCAAoB,CAAC,MAAM,CAAC,kBAAkB,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;QACvF,IAAI,SAAS,EAAE;YACX,IAAI,CAAC,MAAM,EAAE,CAAC;SACjB;QACD,OAAO,SAAS,CAAC;IACrB,CAAC;IAED;;;;;;OAMG;IACH,sBAAsB,CAAC,QAAyG;QAC5H,IAAI;YACA,MAAM,OAAO,GAAG,IAAA,qCAAsB,EAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;YAClE,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,OAAO,QAAQ,CAAC,GAAG,KAAK,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAI,CAAC,CAAC;SAClG;QAAC,OAAO,CAAC,EAAE;YACR,MAAM,GAAG,GAAG,CAAU,CAAC;YACvB,+DAA+D;YAC/D,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,qCAAqC,CAAC,CAAC;SAClE;IACL,CAAC;IAED;;;;;;;;;;;;OAYG;IACH,UAAU,CAAC,OAAuB,EAAE,GAAW;QAC3C,MAAM,iBAAiB,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAA,6CAA8B,EAAC,CAAC,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC;QACjG,IAAI,CAAC,WAAW,CAAC,iBAAiB,EAAE,GAAG,CAAC,CAAC;IAC7C,CAAC;IAED;;;OAGG;IACH,UAAU,CAAC,GAAW;QAClB,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;QACnD,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAA,6CAA8B,EAAC,CAAC,CAAC,CAAC,CAAC;IACjE,CAAC;IAED;;;;;OAKG;IACH,eAAe,CAAC,GAAW;QACvB,OAAO,IAAI,CAAC,SAAS,CAAC,mBAAmB,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;IACvD,CAAC;IAED;;OAEG;IACH,SAAS,CAAC,SAAiB,EAAE,GAAW;QACpC,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;IACjD,CAAC;IAED;;OAEG;IACO,WAAW,CAAC,OAAiB,EAAE,GAAW;QAChD,MAAM,aAAa,GAAa,EAAE,CAAC;QAEnC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE;YAC1B,IAAI;gBACA,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,MAAM,EAAE,GAAG,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC,CAAC;aACrE;YAAC,OAAO,CAAC,EAAE;gBACR,MAAM,GAAG,GAAG,CAAU,CAAC;gBACvB,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;aACnC;SACJ;QAED,sGAAsG;QACtG,IAAI,aAAa,CAAC,MAAM,EAAE;YACtB,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,wBAAwB,EAAE,EAAE,aAAa,EAAE,CAAC,CAAC;SAC/D;IACL,CAAC;IAED;;OAEG;IACO,gBAAgB;QACtB,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE;YAClB,IAAI,CAAC,MAAM,EAAE,CAAC;SACjB;IACL,CAAC;CACJ;AA/QD,0BA+QC"}
|