@crawlee/core 3.0.1-beta.15 → 3.0.1-beta.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/cookie_utils.d.ts +43 -0
- package/cookie_utils.d.ts.map +1 -0
- package/cookie_utils.js +118 -0
- package/cookie_utils.js.map +1 -0
- package/crawlers/crawler_utils.d.ts +0 -10
- package/crawlers/crawler_utils.d.ts.map +1 -1
- package/crawlers/crawler_utils.js +1 -47
- package/crawlers/crawler_utils.js.map +1 -1
- package/index.d.ts +1 -0
- package/index.d.ts.map +1 -1
- package/index.js +1 -0
- package/index.js.map +1 -1
- package/index.mjs +4 -1
- package/package.json +5 -5
- package/session_pool/index.d.ts +0 -1
- package/session_pool/index.d.ts.map +1 -1
- package/session_pool/index.js +0 -1
- package/session_pool/index.js.map +1 -1
- package/session_pool/session.d.ts +3 -19
- package/session_pool/session.d.ts.map +1 -1
- package/session_pool/session.js +6 -52
- package/session_pool/session.js.map +1 -1
- package/tsconfig.build.tsbuildinfo +1 -1
- package/session_pool/session_utils.d.ts +0 -11
- package/session_pool/session_utils.d.ts.map +0 -1
- package/session_pool/session_utils.js +0 -22
- package/session_pool/session_utils.js.map +0 -1
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
/// <reference types="node" />
|
|
2
|
+
import type { BrowserLikeResponse, Dictionary, Cookie as CookieObject } from '@crawlee/types';
|
|
3
|
+
import type { IncomingMessage } from 'node:http';
|
|
4
|
+
import { Cookie } from 'tough-cookie';
|
|
5
|
+
/**
|
|
6
|
+
* @internal
|
|
7
|
+
*/
|
|
8
|
+
export declare function getCookiesFromResponse(response: IncomingMessage | BrowserLikeResponse | {
|
|
9
|
+
headers: Dictionary<string | string[]>;
|
|
10
|
+
}): Cookie[];
|
|
11
|
+
/**
|
|
12
|
+
* Calculate cookie expiration date
|
|
13
|
+
* @param maxAgeSecs
|
|
14
|
+
* @returns Calculated date by session max age seconds.
|
|
15
|
+
* @internal
|
|
16
|
+
*/
|
|
17
|
+
export declare function getDefaultCookieExpirationDate(maxAgeSecs: number): Date;
|
|
18
|
+
/**
|
|
19
|
+
* Transforms tough-cookie to puppeteer cookie.
|
|
20
|
+
* @param toughCookie Cookie from CookieJar
|
|
21
|
+
* @return Cookie compatible with browser pool
|
|
22
|
+
* @internal
|
|
23
|
+
*/
|
|
24
|
+
export declare function toughCookieToBrowserPoolCookie(toughCookie: Cookie): CookieObject;
|
|
25
|
+
/**
|
|
26
|
+
* Transforms browser-pool cookie to tough-cookie.
|
|
27
|
+
* @param cookieObject Cookie object (for instance from the `page.cookies` method).
|
|
28
|
+
* @internal
|
|
29
|
+
*/
|
|
30
|
+
export declare function browserPoolCookieToToughCookie(cookieObject: CookieObject, maxAgeSecs: number): Cookie;
|
|
31
|
+
/**
|
|
32
|
+
* @internal
|
|
33
|
+
* @param cookieString The cookie string to attempt parsing
|
|
34
|
+
* @returns Browser pool compatible cookie, or null if cookie cannot be parsed
|
|
35
|
+
*/
|
|
36
|
+
export declare function cookieStringToToughCookie(cookieString: string): CookieObject | null;
|
|
37
|
+
/**
|
|
38
|
+
* Merges multiple cookie strings. Keys are compared case-sensitively, warning will be logged
|
|
39
|
+
* if we see two cookies with same keys but different casing.
|
|
40
|
+
* @internal
|
|
41
|
+
*/
|
|
42
|
+
export declare function mergeCookies(url: string, sourceCookies: string[]): string;
|
|
43
|
+
//# sourceMappingURL=cookie_utils.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cookie_utils.d.ts","sourceRoot":"","sources":["../src/cookie_utils.ts"],"names":[],"mappings":";AAAA,OAAO,KAAK,EAAE,mBAAmB,EAAE,UAAU,EAAE,MAAM,IAAI,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9F,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AACjD,OAAO,EAAE,MAAM,EAAa,MAAM,cAAc,CAAC;AAIjD;;GAEG;AACH,wBAAgB,sBAAsB,CAAC,QAAQ,EAAE,eAAe,GAAG,mBAAmB,GAAG;IAAE,OAAO,EAAE,UAAU,CAAC,MAAM,GAAG,MAAM,EAAE,CAAC,CAAA;CAAE,GAAG,MAAM,EAAE,CAW7I;AAED;;;;;GAKG;AACH,wBAAgB,8BAA8B,CAAC,UAAU,EAAE,MAAM,QAEhE;AAED;;;;;GAKG;AACH,wBAAgB,8BAA8B,CAAC,WAAW,EAAE,MAAM,GAAG,YAAY,CAYhF;AAED;;;;GAIG;AACH,wBAAgB,8BAA8B,CAAC,YAAY,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,UAgB5F;AAED;;;;GAIG;AACH,wBAAgB,yBAAyB,CAAC,YAAY,EAAE,MAAM,uBAQ7D;AAED;;;;GAIG;AACH,wBAAgB,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,GAAG,MAAM,CA4BzE"}
|
package/cookie_utils.js
ADDED
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.mergeCookies = exports.cookieStringToToughCookie = exports.browserPoolCookieToToughCookie = exports.toughCookieToBrowserPoolCookie = exports.getDefaultCookieExpirationDate = exports.getCookiesFromResponse = void 0;
|
|
4
|
+
const tough_cookie_1 = require("tough-cookie");
|
|
5
|
+
const errors_1 = require("./session_pool/errors");
|
|
6
|
+
const log_1 = require("./log");
|
|
7
|
+
/**
|
|
8
|
+
* @internal
|
|
9
|
+
*/
|
|
10
|
+
function getCookiesFromResponse(response) {
|
|
11
|
+
const headers = typeof response.headers === 'function' ? response.headers() : response.headers;
|
|
12
|
+
const cookieHeader = headers['set-cookie'] || '';
|
|
13
|
+
try {
|
|
14
|
+
return Array.isArray(cookieHeader)
|
|
15
|
+
? cookieHeader.map((cookie) => tough_cookie_1.Cookie.parse(cookie))
|
|
16
|
+
: [tough_cookie_1.Cookie.parse(cookieHeader)];
|
|
17
|
+
}
|
|
18
|
+
catch (e) {
|
|
19
|
+
throw new errors_1.CookieParseError(cookieHeader);
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
exports.getCookiesFromResponse = getCookiesFromResponse;
|
|
23
|
+
/**
|
|
24
|
+
* Calculate cookie expiration date
|
|
25
|
+
* @param maxAgeSecs
|
|
26
|
+
* @returns Calculated date by session max age seconds.
|
|
27
|
+
* @internal
|
|
28
|
+
*/
|
|
29
|
+
function getDefaultCookieExpirationDate(maxAgeSecs) {
|
|
30
|
+
return new Date(Date.now() + (maxAgeSecs * 1000));
|
|
31
|
+
}
|
|
32
|
+
exports.getDefaultCookieExpirationDate = getDefaultCookieExpirationDate;
|
|
33
|
+
/**
|
|
34
|
+
* Transforms tough-cookie to puppeteer cookie.
|
|
35
|
+
* @param toughCookie Cookie from CookieJar
|
|
36
|
+
* @return Cookie compatible with browser pool
|
|
37
|
+
* @internal
|
|
38
|
+
*/
|
|
39
|
+
function toughCookieToBrowserPoolCookie(toughCookie) {
|
|
40
|
+
return {
|
|
41
|
+
name: toughCookie.key,
|
|
42
|
+
value: toughCookie.value,
|
|
43
|
+
// Puppeteer and Playwright expect 'expires' to be 'Unix time in seconds', not ms
|
|
44
|
+
// If there is no expires date (so defaults to Infinity), we don't provide it to the browsers
|
|
45
|
+
expires: toughCookie.expires === 'Infinity' ? undefined : new Date(toughCookie.expires).getTime() / 1000,
|
|
46
|
+
domain: toughCookie.domain ?? undefined,
|
|
47
|
+
path: toughCookie.path ?? undefined,
|
|
48
|
+
secure: toughCookie.secure,
|
|
49
|
+
httpOnly: toughCookie.httpOnly,
|
|
50
|
+
};
|
|
51
|
+
}
|
|
52
|
+
exports.toughCookieToBrowserPoolCookie = toughCookieToBrowserPoolCookie;
|
|
53
|
+
/**
|
|
54
|
+
* Transforms browser-pool cookie to tough-cookie.
|
|
55
|
+
* @param cookieObject Cookie object (for instance from the `page.cookies` method).
|
|
56
|
+
* @internal
|
|
57
|
+
*/
|
|
58
|
+
function browserPoolCookieToToughCookie(cookieObject, maxAgeSecs) {
|
|
59
|
+
const isExpiresValid = cookieObject.expires && typeof cookieObject.expires === 'number' && cookieObject.expires > 0;
|
|
60
|
+
const expires = isExpiresValid ? new Date(cookieObject.expires * 1000) : getDefaultCookieExpirationDate(maxAgeSecs);
|
|
61
|
+
const domain = typeof cookieObject.domain === 'string' && cookieObject.domain.startsWith('.')
|
|
62
|
+
? cookieObject.domain.slice(1)
|
|
63
|
+
: cookieObject.domain;
|
|
64
|
+
return new tough_cookie_1.Cookie({
|
|
65
|
+
key: cookieObject.name,
|
|
66
|
+
value: cookieObject.value,
|
|
67
|
+
expires,
|
|
68
|
+
domain,
|
|
69
|
+
path: cookieObject.path,
|
|
70
|
+
secure: cookieObject.secure,
|
|
71
|
+
httpOnly: cookieObject.httpOnly,
|
|
72
|
+
});
|
|
73
|
+
}
|
|
74
|
+
exports.browserPoolCookieToToughCookie = browserPoolCookieToToughCookie;
|
|
75
|
+
/**
|
|
76
|
+
* @internal
|
|
77
|
+
* @param cookieString The cookie string to attempt parsing
|
|
78
|
+
* @returns Browser pool compatible cookie, or null if cookie cannot be parsed
|
|
79
|
+
*/
|
|
80
|
+
function cookieStringToToughCookie(cookieString) {
|
|
81
|
+
const parsed = tough_cookie_1.Cookie.parse(cookieString);
|
|
82
|
+
if (parsed) {
|
|
83
|
+
return toughCookieToBrowserPoolCookie(parsed);
|
|
84
|
+
}
|
|
85
|
+
return null;
|
|
86
|
+
}
|
|
87
|
+
exports.cookieStringToToughCookie = cookieStringToToughCookie;
|
|
88
|
+
/**
|
|
89
|
+
* Merges multiple cookie strings. Keys are compared case-sensitively, warning will be logged
|
|
90
|
+
* if we see two cookies with same keys but different casing.
|
|
91
|
+
* @internal
|
|
92
|
+
*/
|
|
93
|
+
function mergeCookies(url, sourceCookies) {
|
|
94
|
+
const jar = new tough_cookie_1.CookieJar();
|
|
95
|
+
// ignore empty cookies
|
|
96
|
+
for (const sourceCookieString of sourceCookies) {
|
|
97
|
+
// ignore empty cookies
|
|
98
|
+
if (!sourceCookieString)
|
|
99
|
+
continue;
|
|
100
|
+
const cookies = sourceCookieString.split(/ *; */);
|
|
101
|
+
for (const cookieString of cookies) {
|
|
102
|
+
// ignore extra spaces
|
|
103
|
+
if (!cookieString)
|
|
104
|
+
continue;
|
|
105
|
+
const cookie = tough_cookie_1.Cookie.parse(cookieString);
|
|
106
|
+
const similarKeyCookie = jar.getCookiesSync(url).find((c) => {
|
|
107
|
+
return cookie.key !== c.key && cookie.key.toLowerCase() === c.key.toLowerCase();
|
|
108
|
+
});
|
|
109
|
+
if (similarKeyCookie) {
|
|
110
|
+
log_1.log.deprecated(`Found cookies with similar name during cookie merging: '${cookie.key}' and '${similarKeyCookie.key}'`);
|
|
111
|
+
}
|
|
112
|
+
jar.setCookieSync(cookie, url);
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
return jar.getCookieStringSync(url);
|
|
116
|
+
}
|
|
117
|
+
exports.mergeCookies = mergeCookies;
|
|
118
|
+
//# sourceMappingURL=cookie_utils.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cookie_utils.js","sourceRoot":"","sources":["../src/cookie_utils.ts"],"names":[],"mappings":";;;AAEA,+CAAiD;AACjD,kDAAyD;AACzD,+BAA4B;AAE5B;;GAEG;AACH,SAAgB,sBAAsB,CAAC,QAA4F;IAC/H,MAAM,OAAO,GAAG,OAAO,QAAQ,CAAC,OAAO,KAAK,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC;IAC/F,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC;IAEjD,IAAI;QACA,OAAO,KAAK,CAAC,OAAO,CAAC,YAAY,CAAC;YAC9B,CAAC,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,qBAAM,CAAC,KAAK,CAAC,MAAM,CAAE,CAAC;YACrD,CAAC,CAAC,CAAC,qBAAM,CAAC,KAAK,CAAC,YAAY,CAAE,CAAC,CAAC;KACvC;IAAC,OAAO,CAAC,EAAE;QACR,MAAM,IAAI,yBAAgB,CAAC,YAAY,CAAC,CAAC;KAC5C;AACL,CAAC;AAXD,wDAWC;AAED;;;;;GAKG;AACH,SAAgB,8BAA8B,CAAC,UAAkB;IAC7D,OAAO,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,UAAU,GAAG,IAAI,CAAC,CAAC,CAAC;AACtD,CAAC;AAFD,wEAEC;AAED;;;;;GAKG;AACH,SAAgB,8BAA8B,CAAC,WAAmB;IAC9D,OAAO;QACH,IAAI,EAAE,WAAW,CAAC,GAAG;QACrB,KAAK,EAAE,WAAW,CAAC,KAAK;QACxB,iFAAiF;QACjF,6FAA6F;QAC7F,OAAO,EAAE,WAAW,CAAC,OAAO,KAAK,UAAU,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,GAAG,IAAI;QACxG,MAAM,EAAE,WAAW,CAAC,MAAM,IAAI,SAAS;QACvC,IAAI,EAAE,WAAW,CAAC,IAAI,IAAI,SAAS;QACnC,MAAM,EAAE,WAAW,CAAC,MAAM;QAC1B,QAAQ,EAAE,WAAW,CAAC,QAAQ;KACjC,CAAC;AACN,CAAC;AAZD,wEAYC;AAED;;;;GAIG;AACH,SAAgB,8BAA8B,CAAC,YAA0B,EAAE,UAAkB;IACzF,MAAM,cAAc,GAAG,YAAY,CAAC,OAAO,IAAI,OAAO,YAAY,CAAC,OAAO,KAAK,QAAQ,IAAI,YAAY,CAAC,OAAO,GAAG,CAAC,CAAC;IACpH,MAAM,OAAO,GAAG,cAAc,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,YAAY,CAAC,OAAQ,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,8BAA8B,CAAC,UAAU,CAAC,CAAC;IACrH,MAAM,MAAM,GAAG,OAAO,YAAY,CAAC,MAAM,KAAK,QAAQ,IAAI,YAAY,CAAC,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC;QACzF,CAAC,CAAC,YAAY,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;QAC9B,CAAC,CAAC,YAAY,CAAC,MAAM,CAAC;IAE1B,OAAO,IAAI,qBAAM,CAAC;QACd,GAAG,EAAE,YAAY,CAAC,IAAI;QACtB,KAAK,EAAE,YAAY,CAAC,KAAK;QACzB,OAAO;QACP,MAAM;QACN,IAAI,EAAE,YAAY,CAAC,IAAI;QACvB,MAAM,EAAE,YAAY,CAAC,MAAM;QAC3B,QAAQ,EAAE,YAAY,CAAC,QAAQ;KAClC,CAAC,CAAC;AACP,CAAC;AAhBD,wEAgBC;AAED;;;;GAIG;AACH,SAAgB,yBAAyB,CAAC,YAAoB;IAC1D,MAAM,MAAM,GAAG,qBAAM,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;IAE1C,IAAI,MAAM,EAAE;QACR,OAAO,8BAA8B,CAAC,MAAM,CAAC,CAAC;KACjD;IAED,OAAO,IAAI,CAAC;AAChB,CAAC;AARD,8DAQC;AAED;;;;GAIG;AACH,SAAgB,YAAY,CAAC,GAAW,EAAE,aAAuB;IAC7D,MAAM,GAAG,GAAG,IAAI,wBAAS,EAAE,CAAC;IAE5B,uBAAuB;IACvB,KAAK,MAAM,kBAAkB,IAAI,aAAa,EAAE;QAC5C,uBAAuB;QACvB,IAAI,CAAC,kBAAkB;YAAE,SAAS;QAElC,MAAM,OAAO,GAAG,kBAAkB,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAElD,KAAK,MAAM,YAAY,IAAI,OAAO,EAAE;YAChC,sBAAsB;YACtB,IAAI,CAAC,YAAY;gBAAE,SAAS;YAE5B,MAAM,MAAM,GAAG,qBAAM,CAAC,KAAK,CAAC,YAAY,CAAE,CAAC;YAC3C,MAAM,gBAAgB,GAAG,GAAG,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE;gBACxD,OAAO,MAAM,CAAC,GAAG,KAAK,CAAC,CAAC,GAAG,IAAI,MAAM,CAAC,GAAG,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC;YACpF,CAAC,CAAC,CAAC;YAEH,IAAI,gBAAgB,EAAE;gBAClB,SAAG,CAAC,UAAU,CAAC,2DAA2D,MAAM,CAAC,GAAG,UAAU,gBAAgB,CAAC,GAAG,GAAG,CAAC,CAAC;aAC1H;YAED,GAAG,CAAC,aAAa,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;SAClC;KACJ;IAED,OAAO,GAAG,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAC;AACxC,CAAC;AA5BD,oCA4BC"}
|
|
@@ -7,14 +7,4 @@ export declare function handleRequestTimeout({ session, errorMessage }: {
|
|
|
7
7
|
session?: Session;
|
|
8
8
|
errorMessage: string;
|
|
9
9
|
}): void;
|
|
10
|
-
/**
|
|
11
|
-
* Merges multiple cookie strings. Keys are compared case-sensitively, warning will be logged
|
|
12
|
-
* if we see two cookies with same keys but different casing.
|
|
13
|
-
* @internal
|
|
14
|
-
*/
|
|
15
|
-
export declare function mergeCookies(url: string, sourceCookies: string[]): string;
|
|
16
|
-
/**
|
|
17
|
-
* @internal
|
|
18
|
-
*/
|
|
19
|
-
export declare function diffCookies(url: string, cookieString1?: string, cookieString2?: string): string;
|
|
20
10
|
//# sourceMappingURL=crawler_utils.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"crawler_utils.d.ts","sourceRoot":"","sources":["../../src/crawlers/crawler_utils.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"crawler_utils.d.ts","sourceRoot":"","sources":["../../src/crawlers/crawler_utils.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAEvD;;;GAGG;AACH,wBAAgB,oBAAoB,CAAC,EAAE,OAAO,EAAE,YAAY,EAAE,EAAE;IAAE,OAAO,CAAC,EAAE,OAAO,CAAC;IAAC,YAAY,EAAE,MAAM,CAAA;CAAE,QAK1G"}
|
|
@@ -1,9 +1,7 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.
|
|
3
|
+
exports.handleRequestTimeout = void 0;
|
|
4
4
|
const timeout_1 = require("@apify/timeout");
|
|
5
|
-
const tough_cookie_1 = require("tough-cookie");
|
|
6
|
-
const log_1 = require("../log");
|
|
7
5
|
/**
|
|
8
6
|
* Handles timeout request
|
|
9
7
|
* @internal
|
|
@@ -15,48 +13,4 @@ function handleRequestTimeout({ session, errorMessage }) {
|
|
|
15
13
|
throw new timeout_1.TimeoutError(`Navigation timed out after ${timeoutSecs} seconds.`);
|
|
16
14
|
}
|
|
17
15
|
exports.handleRequestTimeout = handleRequestTimeout;
|
|
18
|
-
/**
|
|
19
|
-
* Merges multiple cookie strings. Keys are compared case-sensitively, warning will be logged
|
|
20
|
-
* if we see two cookies with same keys but different casing.
|
|
21
|
-
* @internal
|
|
22
|
-
*/
|
|
23
|
-
function mergeCookies(url, sourceCookies) {
|
|
24
|
-
const jar = new tough_cookie_1.CookieJar();
|
|
25
|
-
// ignore empty cookies
|
|
26
|
-
for (const sourceCookieString of sourceCookies.filter((c) => c)) {
|
|
27
|
-
const cookies = sourceCookieString.split(/ *; */).filter((c) => c); // ignore extra spaces
|
|
28
|
-
for (const cookieString of cookies) {
|
|
29
|
-
const cookie = tough_cookie_1.Cookie.parse(cookieString);
|
|
30
|
-
const similarKeyCookie = jar.getCookiesSync(url).find((c) => {
|
|
31
|
-
return cookie.key !== c.key && cookie.key.toLowerCase() === c.key.toLowerCase();
|
|
32
|
-
});
|
|
33
|
-
if (similarKeyCookie) {
|
|
34
|
-
log_1.log.deprecated(`Found cookies with similar name during cookie merging: '${cookie.key}' and '${similarKeyCookie.key}'`);
|
|
35
|
-
}
|
|
36
|
-
jar.setCookieSync(cookie, url);
|
|
37
|
-
}
|
|
38
|
-
}
|
|
39
|
-
return jar.getCookieStringSync(url);
|
|
40
|
-
}
|
|
41
|
-
exports.mergeCookies = mergeCookies;
|
|
42
|
-
/**
|
|
43
|
-
* @internal
|
|
44
|
-
*/
|
|
45
|
-
function diffCookies(url, cookieString1 = '', cookieString2 = '') {
|
|
46
|
-
if (cookieString1 === cookieString2 || !cookieString2) {
|
|
47
|
-
return '';
|
|
48
|
-
}
|
|
49
|
-
if (!cookieString1) {
|
|
50
|
-
return cookieString2;
|
|
51
|
-
}
|
|
52
|
-
const cookies1 = cookieString1.split(/ *; */).filter((item) => Boolean(item)).map((cookie) => tough_cookie_1.Cookie.parse(cookie));
|
|
53
|
-
const cookies2 = cookieString2.split(/ *; */).filter((item) => Boolean(item)).map((cookie) => tough_cookie_1.Cookie.parse(cookie));
|
|
54
|
-
const added = cookies2.filter((newCookie) => {
|
|
55
|
-
return !cookies1.find((oldCookie) => newCookie.toString() === oldCookie.toString());
|
|
56
|
-
});
|
|
57
|
-
const jar = new tough_cookie_1.CookieJar();
|
|
58
|
-
added.forEach((cookie) => jar.setCookieSync(cookie, url));
|
|
59
|
-
return jar.getCookieStringSync(url);
|
|
60
|
-
}
|
|
61
|
-
exports.diffCookies = diffCookies;
|
|
62
16
|
//# sourceMappingURL=crawler_utils.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"crawler_utils.js","sourceRoot":"","sources":["../../src/crawlers/crawler_utils.ts"],"names":[],"mappings":";;;AAAA,4CAA8C;
|
|
1
|
+
{"version":3,"file":"crawler_utils.js","sourceRoot":"","sources":["../../src/crawlers/crawler_utils.ts"],"names":[],"mappings":";;;AAAA,4CAA8C;AAG9C;;;GAGG;AACH,SAAgB,oBAAoB,CAAC,EAAE,OAAO,EAAE,YAAY,EAA+C;IACvG,OAAO,EAAE,OAAO,EAAE,CAAC;IACnB,MAAM,aAAa,GAAG,YAAY,CAAC,KAAK,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,wBAAwB;IACrF,MAAM,WAAW,GAAG,MAAM,CAAC,aAAa,CAAC,GAAG,IAAI,CAAC;IACjD,MAAM,IAAI,sBAAY,CAAC,8BAA8B,WAAW,WAAW,CAAC,CAAC;AACjF,CAAC;AALD,oDAKC"}
|
package/index.d.ts
CHANGED
|
@@ -13,6 +13,7 @@ export * from './serialization';
|
|
|
13
13
|
export * from './session_pool';
|
|
14
14
|
export * from './storages';
|
|
15
15
|
export * from './validators';
|
|
16
|
+
export * from './cookie_utils';
|
|
16
17
|
export { PseudoUrl } from '@apify/pseudo_url';
|
|
17
18
|
export { Dictionary, Awaitable, Constructor, StorageClient } from '@crawlee/types';
|
|
18
19
|
//# sourceMappingURL=index.d.ts.map
|
package/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,UAAU,CAAC;AACzB,cAAc,eAAe,CAAC;AAC9B,cAAc,iBAAiB,CAAC;AAChC,cAAc,aAAa,CAAC;AAC5B,cAAc,YAAY,CAAC;AAC3B,cAAc,iBAAiB,CAAC;AAChC,cAAc,UAAU,CAAC;AACzB,cAAc,OAAO,CAAC;AACtB,cAAc,uBAAuB,CAAC;AACtC,cAAc,WAAW,CAAC;AAC1B,cAAc,UAAU,CAAC;AACzB,cAAc,iBAAiB,CAAC;AAChC,cAAc,gBAAgB,CAAC;AAC/B,cAAc,YAAY,CAAC;AAC3B,cAAc,cAAc,CAAC;AAC7B,OAAO,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAC9C,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,WAAW,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,UAAU,CAAC;AACzB,cAAc,eAAe,CAAC;AAC9B,cAAc,iBAAiB,CAAC;AAChC,cAAc,aAAa,CAAC;AAC5B,cAAc,YAAY,CAAC;AAC3B,cAAc,iBAAiB,CAAC;AAChC,cAAc,UAAU,CAAC;AACzB,cAAc,OAAO,CAAC;AACtB,cAAc,uBAAuB,CAAC;AACtC,cAAc,WAAW,CAAC;AAC1B,cAAc,UAAU,CAAC;AACzB,cAAc,iBAAiB,CAAC;AAChC,cAAc,gBAAgB,CAAC;AAC/B,cAAc,YAAY,CAAC;AAC3B,cAAc,cAAc,CAAC;AAC7B,cAAc,gBAAgB,CAAC;AAC/B,OAAO,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAC9C,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,WAAW,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC"}
|
package/index.js
CHANGED
|
@@ -17,6 +17,7 @@ tslib_1.__exportStar(require("./serialization"), exports);
|
|
|
17
17
|
tslib_1.__exportStar(require("./session_pool"), exports);
|
|
18
18
|
tslib_1.__exportStar(require("./storages"), exports);
|
|
19
19
|
tslib_1.__exportStar(require("./validators"), exports);
|
|
20
|
+
tslib_1.__exportStar(require("./cookie_utils"), exports);
|
|
20
21
|
var pseudo_url_1 = require("@apify/pseudo_url");
|
|
21
22
|
Object.defineProperty(exports, "PseudoUrl", { enumerable: true, get: function () { return pseudo_url_1.PseudoUrl; } });
|
|
22
23
|
//# sourceMappingURL=index.js.map
|
package/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;;AAAA,mDAAyB;AACzB,wDAA8B;AAC9B,0DAAgC;AAChC,sDAA4B;AAC5B,qDAA2B;AAC3B,0DAAgC;AAChC,mDAAyB;AACzB,gDAAsB;AACtB,gEAAsC;AACtC,oDAA0B;AAC1B,mDAAyB;AACzB,0DAAgC;AAChC,yDAA+B;AAC/B,qDAA2B;AAC3B,uDAA6B;AAC7B,gDAA8C;AAArC,uGAAA,SAAS,OAAA"}
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;;AAAA,mDAAyB;AACzB,wDAA8B;AAC9B,0DAAgC;AAChC,sDAA4B;AAC5B,qDAA2B;AAC3B,0DAAgC;AAChC,mDAAyB;AACzB,gDAAsB;AACtB,gEAAsC;AACtC,oDAA0B;AAC1B,mDAAyB;AACzB,0DAAgC;AAChC,yDAA+B;AAC/B,qDAA2B;AAC3B,uDAA6B;AAC7B,yDAA+B;AAC/B,gDAA8C;AAArC,uGAAA,SAAS,OAAA"}
|
package/index.mjs
CHANGED
|
@@ -41,18 +41,20 @@ export const Snapshotter = mod.Snapshotter;
|
|
|
41
41
|
export const Statistics = mod.Statistics;
|
|
42
42
|
export const StorageManager = mod.StorageManager;
|
|
43
43
|
export const SystemStatus = mod.SystemStatus;
|
|
44
|
+
export const browserPoolCookieToToughCookie = mod.browserPoolCookieToToughCookie;
|
|
44
45
|
export const checkAndSerialize = mod.checkAndSerialize;
|
|
45
46
|
export const chunkBySize = mod.chunkBySize;
|
|
46
47
|
export const constructGlobObjectsFromGlobs = mod.constructGlobObjectsFromGlobs;
|
|
47
48
|
export const constructRegExpObjectsFromPseudoUrls = mod.constructRegExpObjectsFromPseudoUrls;
|
|
48
49
|
export const constructRegExpObjectsFromRegExps = mod.constructRegExpObjectsFromRegExps;
|
|
50
|
+
export const cookieStringToToughCookie = mod.cookieStringToToughCookie;
|
|
49
51
|
export const createDeserialize = mod.createDeserialize;
|
|
50
52
|
export const createRequestOptions = mod.createRequestOptions;
|
|
51
53
|
export const createRequests = mod.createRequests;
|
|
52
54
|
export const deserializeArray = mod.deserializeArray;
|
|
53
|
-
export const diffCookies = mod.diffCookies;
|
|
54
55
|
export const enqueueLinks = mod.enqueueLinks;
|
|
55
56
|
export const getCookiesFromResponse = mod.getCookiesFromResponse;
|
|
57
|
+
export const getDefaultCookieExpirationDate = mod.getDefaultCookieExpirationDate;
|
|
56
58
|
export const getRequestId = mod.getRequestId;
|
|
57
59
|
export const handleRequestTimeout = mod.handleRequestTimeout;
|
|
58
60
|
export const log = mod.log;
|
|
@@ -61,6 +63,7 @@ export const mergeCookies = mod.mergeCookies;
|
|
|
61
63
|
export const purgeDefaultStorages = mod.purgeDefaultStorages;
|
|
62
64
|
export const resolveBaseUrl = mod.resolveBaseUrl;
|
|
63
65
|
export const serializeArray = mod.serializeArray;
|
|
66
|
+
export const toughCookieToBrowserPoolCookie = mod.toughCookieToBrowserPoolCookie;
|
|
64
67
|
export const updateEnqueueLinksPatternCache = mod.updateEnqueueLinksPatternCache;
|
|
65
68
|
export const validateGlobPattern = mod.validateGlobPattern;
|
|
66
69
|
export const validators = mod.validators;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/core",
|
|
3
|
-
"version": "3.0.1-beta.
|
|
3
|
+
"version": "3.0.1-beta.16",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=16.0.0"
|
|
@@ -58,10 +58,10 @@
|
|
|
58
58
|
"@apify/pseudo_url": "^2.0.0",
|
|
59
59
|
"@apify/timeout": "^0.3.0",
|
|
60
60
|
"@apify/utilities": "^2.0.0",
|
|
61
|
-
"@crawlee/browser-pool": "^3.0.1-beta.
|
|
62
|
-
"@crawlee/memory-storage": "^3.0.1-beta.
|
|
63
|
-
"@crawlee/types": "^3.0.1-beta.
|
|
64
|
-
"@crawlee/utils": "^3.0.1-beta.
|
|
61
|
+
"@crawlee/browser-pool": "^3.0.1-beta.16",
|
|
62
|
+
"@crawlee/memory-storage": "^3.0.1-beta.16",
|
|
63
|
+
"@crawlee/types": "^3.0.1-beta.16",
|
|
64
|
+
"@crawlee/utils": "^3.0.1-beta.16",
|
|
65
65
|
"@types/tough-cookie": "^4.0.2",
|
|
66
66
|
"@vladfrangu/async_event_emitter": "^2.0.0",
|
|
67
67
|
"fs-extra": "^10.1.0",
|
package/session_pool/index.d.ts
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/session_pool/index.ts"],"names":[],"mappings":"AAAA,cAAc,UAAU,CAAC;AACzB,cAAc,UAAU,CAAC;AACzB,cAAc,WAAW,CAAC;AAC1B,cAAc,gBAAgB,CAAC
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/session_pool/index.ts"],"names":[],"mappings":"AAAA,cAAc,UAAU,CAAC;AACzB,cAAc,UAAU,CAAC;AACzB,cAAc,WAAW,CAAC;AAC1B,cAAc,gBAAgB,CAAC"}
|
package/session_pool/index.js
CHANGED
|
@@ -5,5 +5,4 @@ tslib_1.__exportStar(require("./errors"), exports);
|
|
|
5
5
|
tslib_1.__exportStar(require("./events"), exports);
|
|
6
6
|
tslib_1.__exportStar(require("./session"), exports);
|
|
7
7
|
tslib_1.__exportStar(require("./session_pool"), exports);
|
|
8
|
-
tslib_1.__exportStar(require("./session_utils"), exports);
|
|
9
8
|
//# sourceMappingURL=index.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/session_pool/index.ts"],"names":[],"mappings":";;;AAAA,mDAAyB;AACzB,mDAAyB;AACzB,oDAA0B;AAC1B,yDAA+B
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/session_pool/index.ts"],"names":[],"mappings":";;;AAAA,mDAAyB;AACzB,mDAAyB;AACzB,oDAA0B;AAC1B,yDAA+B"}
|
|
@@ -2,7 +2,8 @@
|
|
|
2
2
|
import type { Log } from '@apify/log';
|
|
3
3
|
import type { Dictionary, Cookie as CookieObject, BrowserLikeResponse } from '@crawlee/types';
|
|
4
4
|
import type { IncomingMessage } from 'node:http';
|
|
5
|
-
import { Cookie
|
|
5
|
+
import type { Cookie } from 'tough-cookie';
|
|
6
|
+
import { CookieJar } from 'tough-cookie';
|
|
6
7
|
/**
|
|
7
8
|
* Persistable {@link Session} state.
|
|
8
9
|
*/
|
|
@@ -168,7 +169,7 @@ export declare class Session {
|
|
|
168
169
|
*/
|
|
169
170
|
setCookies(cookies: CookieObject[], url: string): void;
|
|
170
171
|
/**
|
|
171
|
-
* Returns cookies in a format compatible with puppeteer and ready to be used with `page.setCookie`.
|
|
172
|
+
* Returns cookies in a format compatible with puppeteer/playwright and ready to be used with `page.setCookie`.
|
|
172
173
|
* @param url website url. Only cookies stored for this url will be returned
|
|
173
174
|
*/
|
|
174
175
|
getCookies(url: string): CookieObject[];
|
|
@@ -183,27 +184,10 @@ export declare class Session {
|
|
|
183
184
|
* Sets a cookie within this session for the specific URL.
|
|
184
185
|
*/
|
|
185
186
|
setCookie(rawCookie: string, url: string): void;
|
|
186
|
-
/**
|
|
187
|
-
* Transforms puppeteer cookie to tough-cookie.
|
|
188
|
-
* @param cookieObject Cookie object (for instance from the `page.cookies` method).
|
|
189
|
-
*/
|
|
190
|
-
protected _cookieObjectToTough(cookieObject: CookieObject): Cookie;
|
|
191
|
-
/**
|
|
192
|
-
* Transforms tough-cookie to puppeteer cookie.
|
|
193
|
-
* @param toughCookie Cookie from CookieJar
|
|
194
|
-
* @return Cookie from Puppeteer
|
|
195
|
-
*/
|
|
196
|
-
protected _toughCookieToPuppeteer(toughCookie: Cookie): CookieObject;
|
|
197
187
|
/**
|
|
198
188
|
* Sets cookies.
|
|
199
189
|
*/
|
|
200
190
|
protected _setCookies(cookies: Cookie[], url: string): void;
|
|
201
|
-
/**
|
|
202
|
-
* Calculate cookie expiration date
|
|
203
|
-
* @param maxAgeSecs
|
|
204
|
-
* @returns Calculated date by session max age seconds.
|
|
205
|
-
*/
|
|
206
|
-
protected _getDefaultCookieExpirationDate(maxAgeSecs: number): Date;
|
|
207
191
|
/**
|
|
208
192
|
* Checks if session is not usable. if it is not retires the session.
|
|
209
193
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session.d.ts","sourceRoot":"","sources":["../../src/session_pool/session.ts"],"names":[],"mappings":";AAAA,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AAEtC,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,IAAI,YAAY,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AAC9F,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAGjD,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;
|
|
1
|
+
{"version":3,"file":"session.d.ts","sourceRoot":"","sources":["../../src/session_pool/session.ts"],"names":[],"mappings":";AAAA,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AAEtC,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,IAAI,YAAY,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AAC9F,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAGjD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAC3C,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AASzC;;GAEG;AACH,MAAM,WAAW,YAAY;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,SAAS,CAAC,UAAU,CAAC;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,cAAc;IAE3B,wFAAwF;IACxF,EAAE,CAAC,EAAE,MAAM,CAAC;IAEZ;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB,+EAA+E;IAC/E,QAAQ,CAAC,EAAE,UAAU,CAAC;IAEtB;;;;;;OAMG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IAEvB;;;;;OAKG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAE7B,wBAAwB;IACxB,SAAS,CAAC,EAAE,IAAI,CAAC;IAEjB,0BAA0B;IAC1B,SAAS,CAAC,EAAE,IAAI,CAAC;IAEjB;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB;;;;OAIG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IAEvB,2FAA2F;IAC3F,WAAW,CAAC,EAAE,OAAO,gBAAgB,EAAE,WAAW,CAAC;IAEnD,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,SAAS,CAAC;CAEzB;AAED;;;;;GAKG;AACH,qBAAa,OAAO;IAChB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,UAAU,CAAS;IAC3B,QAAQ,EAAE,UAAU,CAAC;IACrB,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,mBAAmB,CAAS;IACpC,OAAO,CAAC,SAAS,CAAO;IACxB,OAAO,CAAC,SAAS,CAAO;IACxB,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,WAAW,CAAuC;IAC1D,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,SAAS,CAAY;IAC7B,OAAO,CAAC,GAAG,CAAM;IAEjB;;OAEG;gBACS,OAAO,EAAE,cAAc;IAoDnC;;;OAGG;IACH,SAAS,IAAI,OAAO;IAIpB;;;;OAIG;IACH,SAAS,IAAI,OAAO;IAIpB;;;OAGG;IACH,sBAAsB,IAAI,OAAO;IAIjC;;;OAGG;IACH,QAAQ,IAAI,OAAO;IAInB;;;OAGG;IACH,QAAQ;IAUR;;;OAGG;IACH,QAAQ,IAAI,YAAY;IAexB;;;;;;OAMG;IACH,MAAM;IASN;;;OAGG;IACH,OAAO;IAOP;;;;;;;;OAQG;IACH,0BAA0B,CAAC,UAAU,EAAE,MAAM,EAAE,kBAAkB,GAAE,MAAM,EAAO,GAAG,OAAO;IAQ1F;;;;;;OAMG;IACH,sBAAsB,CAAC,QAAQ,EAAE,eAAe,GAAG,mBAAmB,GAAG;QAAE,OAAO,EAAE,UAAU,CAAC,MAAM,GAAG,MAAM,EAAE,CAAC,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE;IAWhI;;;;;;;;;;;;OAYG;IACH,UAAU,CAAC,OAAO,EAAE,YAAY,EAAE,EAAE,GAAG,EAAE,MAAM;IAK/C;;;OAGG;IACH,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG,YAAY,EAAE;IAKvC;;;;;OAKG;IACH,eAAe,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM;IAIpC;;OAEG;IACH,SAAS,CAAC,SAAS,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,IAAI;IAI/C;;OAEG;IACH,SAAS,CAAC,WAAW,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,MAAM,GAAG,IAAI;IAkB3D;;OAEG;IACH,SAAS,CAAC,gBAAgB,IAAI,IAAI;CAKrC"}
|
package/session_pool/session.js
CHANGED
|
@@ -9,7 +9,7 @@ const tough_cookie_1 = require("tough-cookie");
|
|
|
9
9
|
const constants_1 = require("../constants");
|
|
10
10
|
const log_1 = require("../log");
|
|
11
11
|
const events_1 = require("./events");
|
|
12
|
-
const
|
|
12
|
+
const cookie_utils_1 = require("../cookie_utils");
|
|
13
13
|
// CONSTANTS
|
|
14
14
|
const DEFAULT_SESSION_MAX_AGE_SECS = 3000;
|
|
15
15
|
/**
|
|
@@ -117,7 +117,7 @@ class Session {
|
|
|
117
117
|
log: ow_1.default.optional.object,
|
|
118
118
|
}));
|
|
119
119
|
const { sessionPool, id = `session_${(0, utilities_1.cryptoRandomObjectId)(10)}`, cookieJar = new tough_cookie_1.CookieJar(), maxAgeSecs = DEFAULT_SESSION_MAX_AGE_SECS, userData = {}, maxErrorScore = 3, errorScoreDecrement = 0.5, createdAt = new Date(), usageCount = 0, errorScore = 0, maxUsageCount = 50, log = log_1.log, } = options;
|
|
120
|
-
const { expiresAt =
|
|
120
|
+
const { expiresAt = (0, cookie_utils_1.getDefaultCookieExpirationDate)(maxAgeSecs) } = options;
|
|
121
121
|
this.log = log.child({ prefix: 'Session' });
|
|
122
122
|
this.cookieJar = cookieJar.setCookie ? cookieJar : tough_cookie_1.CookieJar.fromJSON(JSON.stringify(cookieJar));
|
|
123
123
|
this.id = id;
|
|
@@ -132,7 +132,6 @@ class Session {
|
|
|
132
132
|
this.errorScore = errorScore; // indicates number of markBaded request with the session
|
|
133
133
|
this.maxUsageCount = maxUsageCount;
|
|
134
134
|
this.sessionPool = sessionPool;
|
|
135
|
-
this._cookieObjectToTough = this._cookieObjectToTough.bind(this);
|
|
136
135
|
}
|
|
137
136
|
/**
|
|
138
137
|
* Indicates whether the session is blocked.
|
|
@@ -240,7 +239,7 @@ class Session {
|
|
|
240
239
|
*/
|
|
241
240
|
setCookiesFromResponse(response) {
|
|
242
241
|
try {
|
|
243
|
-
const cookies = (0,
|
|
242
|
+
const cookies = (0, cookie_utils_1.getCookiesFromResponse)(response).filter((c) => c);
|
|
244
243
|
this._setCookies(cookies, typeof response.url === 'function' ? response.url() : response.url);
|
|
245
244
|
}
|
|
246
245
|
catch (e) {
|
|
@@ -263,16 +262,16 @@ class Session {
|
|
|
263
262
|
* ```
|
|
264
263
|
*/
|
|
265
264
|
setCookies(cookies, url) {
|
|
266
|
-
const normalizedCookies = cookies.map(this.
|
|
265
|
+
const normalizedCookies = cookies.map((c) => (0, cookie_utils_1.browserPoolCookieToToughCookie)(c, this.maxAgeSecs));
|
|
267
266
|
this._setCookies(normalizedCookies, url);
|
|
268
267
|
}
|
|
269
268
|
/**
|
|
270
|
-
* Returns cookies in a format compatible with puppeteer and ready to be used with `page.setCookie`.
|
|
269
|
+
* Returns cookies in a format compatible with puppeteer/playwright and ready to be used with `page.setCookie`.
|
|
271
270
|
* @param url website url. Only cookies stored for this url will be returned
|
|
272
271
|
*/
|
|
273
272
|
getCookies(url) {
|
|
274
273
|
const cookies = this.cookieJar.getCookiesSync(url);
|
|
275
|
-
return cookies.map(
|
|
274
|
+
return cookies.map((c) => (0, cookie_utils_1.toughCookieToBrowserPoolCookie)(c));
|
|
276
275
|
}
|
|
277
276
|
/**
|
|
278
277
|
* Returns cookies saved with the session in the typical
|
|
@@ -289,43 +288,6 @@ class Session {
|
|
|
289
288
|
setCookie(rawCookie, url) {
|
|
290
289
|
this.cookieJar.setCookieSync(rawCookie, url);
|
|
291
290
|
}
|
|
292
|
-
/**
|
|
293
|
-
* Transforms puppeteer cookie to tough-cookie.
|
|
294
|
-
* @param cookieObject Cookie object (for instance from the `page.cookies` method).
|
|
295
|
-
*/
|
|
296
|
-
_cookieObjectToTough(cookieObject) {
|
|
297
|
-
const isExpiresValid = cookieObject.expires && typeof cookieObject.expires === 'number' && cookieObject.expires > 0;
|
|
298
|
-
const expires = isExpiresValid ? new Date(cookieObject.expires * 1000) : this._getDefaultCookieExpirationDate(this.maxAgeSecs);
|
|
299
|
-
const domain = typeof cookieObject.domain === 'string' && cookieObject.domain.startsWith('.')
|
|
300
|
-
? cookieObject.domain.slice(1)
|
|
301
|
-
: cookieObject.domain;
|
|
302
|
-
return new tough_cookie_1.Cookie({
|
|
303
|
-
key: cookieObject.name,
|
|
304
|
-
value: cookieObject.value,
|
|
305
|
-
expires,
|
|
306
|
-
domain,
|
|
307
|
-
path: cookieObject.path,
|
|
308
|
-
secure: cookieObject.secure,
|
|
309
|
-
httpOnly: cookieObject.httpOnly,
|
|
310
|
-
});
|
|
311
|
-
}
|
|
312
|
-
/**
|
|
313
|
-
* Transforms tough-cookie to puppeteer cookie.
|
|
314
|
-
* @param toughCookie Cookie from CookieJar
|
|
315
|
-
* @return Cookie from Puppeteer
|
|
316
|
-
*/
|
|
317
|
-
_toughCookieToPuppeteer(toughCookie) {
|
|
318
|
-
return {
|
|
319
|
-
name: toughCookie.key,
|
|
320
|
-
value: toughCookie.value,
|
|
321
|
-
// Puppeteer and Playwright expect 'expires' to be 'Unix time in seconds', not ms
|
|
322
|
-
expires: new Date(toughCookie.expires).getTime() / 1000,
|
|
323
|
-
domain: toughCookie.domain,
|
|
324
|
-
path: toughCookie.path,
|
|
325
|
-
secure: toughCookie.secure,
|
|
326
|
-
httpOnly: toughCookie.httpOnly,
|
|
327
|
-
};
|
|
328
|
-
}
|
|
329
291
|
/**
|
|
330
292
|
* Sets cookies.
|
|
331
293
|
*/
|
|
@@ -345,14 +307,6 @@ class Session {
|
|
|
345
307
|
this.log.debug('Could not set cookies.', { errorMessages });
|
|
346
308
|
}
|
|
347
309
|
}
|
|
348
|
-
/**
|
|
349
|
-
* Calculate cookie expiration date
|
|
350
|
-
* @param maxAgeSecs
|
|
351
|
-
* @returns Calculated date by session max age seconds.
|
|
352
|
-
*/
|
|
353
|
-
_getDefaultCookieExpirationDate(maxAgeSecs) {
|
|
354
|
-
return new Date(Date.now() + (maxAgeSecs * 1000));
|
|
355
|
-
}
|
|
356
310
|
/**
|
|
357
311
|
* Checks if session is not usable. if it is not retires the session.
|
|
358
312
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session.js","sourceRoot":"","sources":["../../src/session_pool/session.ts"],"names":[],"mappings":";;;;AACA,gDAAwD;AAGxD,6CAA2C;AAC3C,oDAAoB;
|
|
1
|
+
{"version":3,"file":"session.js","sourceRoot":"","sources":["../../src/session_pool/session.ts"],"names":[],"mappings":";;;;AACA,gDAAwD;AAGxD,6CAA2C;AAC3C,oDAAoB;AAEpB,+CAAyC;AACzC,4CAAoD;AACpD,gCAA2C;AAC3C,qCAAiD;AACjD,kDAAyJ;AAEzJ,YAAY;AACZ,MAAM,4BAA4B,GAAG,IAAI,CAAC;AA6E1C;;;;;GAKG;AACH,MAAa,OAAO;IAehB;;OAEG;IACH,YAAY,OAAuB;QAjBnC;;;;;WAAoB;QACpB;;;;;WAA2B;QAC3B;;;;;WAAqB;QACrB;;;;;WAA8B;QAC9B;;;;;WAAoC;QACpC;;;;;WAAwB;QACxB;;;;;WAAwB;QACxB;;;;;WAA2B;QAC3B;;;;;WAA8B;QAC9B;;;;;WAA0D;QAC1D;;;;;WAA2B;QAC3B;;;;;WAA6B;QAC7B;;;;;WAAiB;QAMb,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7B,WAAW,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC,0BAAY,CAAC;YAC/C,EAAE,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACtB,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC7B,UAAU,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC9B,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC5B,aAAa,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACjC,mBAAmB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACvC,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,IAAI;YAC3B,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,IAAI;YAC3B,UAAU,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC9B,UAAU,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC9B,aAAa,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACjC,GAAG,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;SAC1B,CAAC,CAAC,CAAC;QAEJ,MAAM,EACF,WAAW,EACX,EAAE,GAAG,WAAW,IAAA,gCAAoB,EAAC,EAAE,CAAC,EAAE,EAC1C,SAAS,GAAG,IAAI,wBAAS,EAAE,EAC3B,UAAU,GAAG,4BAA4B,EACzC,QAAQ,GAAG,EAAE,EACb,aAAa,GAAG,CAAC,EACjB,mBAAmB,GAAG,GAAG,EACzB,SAAS,GAAG,IAAI,IAAI,EAAE,EACtB,UAAU,GAAG,CAAC,EACd,UAAU,GAAG,CAAC,EACd,aAAa,GAAG,EAAE,EAClB,GAAG,GAAG,SAAU,GACnB,GAAG,OAAO,CAAC;QAEZ,MAAM,EAAE,SAAS,GAAG,IAAA,6CAA8B,EAAC,UAAU,CAAC,EAAE,GAAG,OAAO,CAAC;QAE3E,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;QAE5C,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC,SAAoB,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,wBAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC;QAC5G,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;QACnC,IAAI,CAAC,mBAAmB,GAAG,mBAAmB,CAAC;QAE/C,WAAW;QACX,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC,CAAC,qDAAqD;QACnF,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC,CAAC,yDAAyD;QACvF,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;QACnC,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;IACnC,CAAC;IAED;;;OAGG;IACH,SAAS;QACL,OAAO,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,aAAa,CAAC;IACjD,CAAC;IAED;;;;OAIG;IACH,SAAS;QACL,OAAO,IAAI,CAAC,SAAS,IAAI,IAAI,IAAI,EAAE,CAAC;IACxC,CAAC;IAED;;;OAGG;IACH,sBAAsB;QAClB,OAAO,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,aAAa,CAAC;IACjD,CAAC;IAED;;;OAGG;IACH,QAAQ;QACJ,OAAO,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,sBAAsB,EAAE,CAAC;IACpF,CAAC;IAED;;;OAGG;IACH,QAAQ;QACJ,IAAI,CAAC,UAAU,IAAI,CAAC,CAAC;QAErB,IAAI,IAAI,CAAC,UAAU,GAAG,CAAC,EAAE;YACrB,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,mBAAmB,CAAC;SAC/C;QAED,IAAI,CAAC,gBAAgB,EAAE,CAAC;IAC5B,CAAC;IAED;;;OAGG;IACH,QAAQ;QACJ,OAAO;YACH,EAAE,EAAE,IAAI,CAAC,EAAE;YACX,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE;YAClC,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,aAAa,EAAE,IAAI,CAAC,aAAa;YACjC,mBAAmB,EAAE,IAAI,CAAC,mBAAmB;YAC7C,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE;YACvC,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE;YACvC,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,aAAa,EAAE,IAAI,CAAC,aAAa;YACjC,UAAU,EAAE,IAAI,CAAC,UAAU;SAC9B,CAAC;IACN,CAAC;IAED;;;;;;OAMG;IACH,MAAM;QACF,6DAA6D;QAC7D,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,aAAa,CAAC;QACtC,IAAI,CAAC,UAAU,IAAI,CAAC,CAAC;QAErB,wDAAwD;QACxD,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,8BAAqB,EAAE,IAAI,CAAC,CAAC;IACvD,CAAC;IAED;;;OAGG;IACH,OAAO;QACH,IAAI,CAAC,UAAU,IAAI,CAAC,CAAC;QACrB,IAAI,CAAC,UAAU,IAAI,CAAC,CAAC;QAErB,IAAI,CAAC,gBAAgB,EAAE,CAAC;IAC5B,CAAC;IAED;;;;;;;;OAQG;IACH,0BAA0B,CAAC,UAAkB,EAAE,qBAA+B,EAAE;QAC5E,MAAM,SAAS,GAAG,gCAAoB,CAAC,MAAM,CAAC,kBAAkB,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;QACvF,IAAI,SAAS,EAAE;YACX,IAAI,CAAC,MAAM,EAAE,CAAC;SACjB;QACD,OAAO,SAAS,CAAC;IACrB,CAAC;IAED;;;;;;OAMG;IACH,sBAAsB,CAAC,QAAyG;QAC5H,IAAI;YACA,MAAM,OAAO,GAAG,IAAA,qCAAsB,EAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;YAClE,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,OAAO,QAAQ,CAAC,GAAG,KAAK,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAI,CAAC,CAAC;SAClG;QAAC,OAAO,CAAC,EAAE;YACR,MAAM,GAAG,GAAG,CAAU,CAAC;YACvB,+DAA+D;YAC/D,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,qCAAqC,CAAC,CAAC;SAClE;IACL,CAAC;IAED;;;;;;;;;;;;OAYG;IACH,UAAU,CAAC,OAAuB,EAAE,GAAW;QAC3C,MAAM,iBAAiB,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAA,6CAA8B,EAAC,CAAC,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC;QACjG,IAAI,CAAC,WAAW,CAAC,iBAAiB,EAAE,GAAG,CAAC,CAAC;IAC7C,CAAC;IAED;;;OAGG;IACH,UAAU,CAAC,GAAW;QAClB,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;QACnD,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAA,6CAA8B,EAAC,CAAC,CAAC,CAAC,CAAC;IACjE,CAAC;IAED;;;;;OAKG;IACH,eAAe,CAAC,GAAW;QACvB,OAAO,IAAI,CAAC,SAAS,CAAC,mBAAmB,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;IACvD,CAAC;IAED;;OAEG;IACH,SAAS,CAAC,SAAiB,EAAE,GAAW;QACpC,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;IACjD,CAAC;IAED;;OAEG;IACO,WAAW,CAAC,OAAiB,EAAE,GAAW;QAChD,MAAM,aAAa,GAAa,EAAE,CAAC;QAEnC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE;YAC1B,IAAI;gBACA,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,MAAM,EAAE,GAAG,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC,CAAC;aACrE;YAAC,OAAO,CAAC,EAAE;gBACR,MAAM,GAAG,GAAG,CAAU,CAAC;gBACvB,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;aACnC;SACJ;QAED,sGAAsG;QACtG,IAAI,aAAa,CAAC,MAAM,EAAE;YACtB,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,wBAAwB,EAAE,EAAE,aAAa,EAAE,CAAC,CAAC;SAC/D;IACL,CAAC;IAED;;OAEG;IACO,gBAAgB;QACtB,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE;YAClB,IAAI,CAAC,MAAM,EAAE,CAAC;SACjB;IACL,CAAC;CACJ;AA/QD,0BA+QC"}
|