@crawlee/core 3.10.6-beta.2 → 3.10.6-beta.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +0 -11
- package/configuration.d.ts +0 -2
- package/configuration.d.ts.map +1 -1
- package/cookie_utils.d.ts +0 -2
- package/cookie_utils.d.ts.map +1 -1
- package/cookie_utils.js +6 -7
- package/cookie_utils.js.map +1 -1
- package/crawlers/crawler_utils.js +1 -2
- package/crawlers/crawler_utils.js.map +1 -1
- package/enqueue_links/enqueue_links.d.ts +5 -5
- package/enqueue_links/enqueue_links.d.ts.map +1 -1
- package/enqueue_links/enqueue_links.js +3 -3
- package/enqueue_links/enqueue_links.js.map +1 -1
- package/enqueue_links/shared.d.ts +3 -3
- package/enqueue_links/shared.d.ts.map +1 -1
- package/enqueue_links/shared.js +9 -9
- package/enqueue_links/shared.js.map +1 -1
- package/events/event_manager.d.ts +1 -2
- package/events/event_manager.d.ts.map +1 -1
- package/package.json +5 -5
- package/request.d.ts +0 -4
- package/request.d.ts.map +1 -1
- package/serialization.d.ts +0 -4
- package/serialization.d.ts.map +1 -1
- package/serialization.js +3 -4
- package/serialization.js.map +1 -1
- package/session_pool/session.d.ts +0 -2
- package/session_pool/session.d.ts.map +1 -1
- package/session_pool/session_pool.d.ts +0 -2
- package/session_pool/session_pool.d.ts.map +1 -1
- package/storages/access_checking.d.ts.map +1 -1
- package/storages/dataset.js +3 -3
- package/storages/dataset.js.map +1 -1
- package/storages/key_value_store.d.ts +0 -2
- package/storages/key_value_store.d.ts.map +1 -1
- package/storages/request_list.d.ts +1 -3
- package/storages/request_list.d.ts.map +1 -1
- package/storages/storage_manager.d.ts +2 -2
- package/storages/storage_manager.d.ts.map +1 -1
- package/storages/utils.d.ts.map +1 -1
- package/storages/utils.js +4 -4
- package/storages/utils.js.map +1 -1
- package/tsconfig.build.tsbuildinfo +1 -1
- package/typedefs.d.ts.map +1 -1
- package/typedefs.js +2 -3
- package/typedefs.js.map +1 -1
package/README.md
CHANGED
|
@@ -16,17 +16,6 @@
|
|
|
16
16
|
<a href="https://github.com/apify/crawlee/actions/workflows/test-ci.yml"><img src="https://github.com/apify/crawlee/actions/workflows/test-ci.yml/badge.svg?branch=master" alt="Build Status" style="max-width: 100%;"></a>
|
|
17
17
|
</p>
|
|
18
18
|
|
|
19
|
-
<h1 align="center">
|
|
20
|
-
<a href="https://apify.com/resources/scraping-with-crawlee">
|
|
21
|
-
<picture>
|
|
22
|
-
<source media="(prefers-color-scheme: dark)" srcset="https://cdn-cms.apify.com/Scraping_with_Crawlee_101_de11e9de45.png">
|
|
23
|
-
<img alt="Crawlee" src="https://cdn-cms.apify.com/Scraping_with_Crawlee_101_de11e9de45.png" width="500">
|
|
24
|
-
</picture>
|
|
25
|
-
</a>
|
|
26
|
-
<br>
|
|
27
|
-
<small>Join us for a free webinar about Crawlee at <a href="https://apify.com/resources/scraping-with-crawlee"> 9 AM EST on June 12th, 2024</a> to learn how to build reliable web scrapers fast!</small>
|
|
28
|
-
</h1>
|
|
29
|
-
|
|
30
19
|
Crawlee covers your crawling and scraping end-to-end and **helps you build reliable scrapers. Fast.**
|
|
31
20
|
|
|
32
21
|
Your crawlers will appear human-like and fly under the radar of modern bot protections even with the default configuration. Crawlee gives you the tools to crawl the web for links, scrape data, and store it to disk or cloud while staying configurable to suit your project's needs.
|
package/configuration.d.ts
CHANGED
package/configuration.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"configuration.d.ts","sourceRoot":"","sources":["../src/configuration.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"configuration.d.ts","sourceRoot":"","sources":["../src/configuration.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAIrD,OAAY,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAC3C,OAAO,EAAE,aAAa,EAAE,MAAM,yBAAyB,CAAC;AACxD,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AACpE,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAGhE,OAAO,EAAqB,KAAK,YAAY,EAAE,MAAM,UAAU,CAAC;AAChE,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AACjD,OAAO,EAAW,KAAK,WAAW,EAAE,MAAM,YAAY,CAAC;AAEvD,MAAM,WAAW,oBAAoB;IACjC;;;OAGG;IACH,aAAa,CAAC,EAAE,aAAa,CAAC;IAE9B;;;OAGG;IACH,YAAY,CAAC,EAAE,YAAY,CAAC;IAE5B;;;OAGG;IACH,oBAAoB,CAAC,EAAE,UAAU,CAAC;IAElC;;;;;OAKG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAE1B;;;;;OAKG;IACH,YAAY,CAAC,EAAE,OAAO,CAAC;IAEvB;;;;;OAKG;IACH,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAEhC;;;;;OAKG;IACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAE/B;;;;OAIG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IAEzB;;;;;;OAMG;IACH,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAE9B;;;;;OAKG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IAEtB;;;;;OAKG;IACH,0BAA0B,CAAC,EAAE,MAAM,CAAC;IAEpC;;;OAGG;IACH,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAElC;;;;;;OAMG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB;;;;;OAKG;IACH,QAAQ,CAAC,EAAE,OAAO,CAAC;IAEnB;;;;;OAKG;IACH,IAAI,CAAC,EAAE,OAAO,CAAC;IAEf;;;;OAIG;IACH,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAE9B;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAE5B;;;;OAIG;IACH,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAEhC;;;;;OAKG;IACH,QAAQ,CAAC,EAAE,QAAQ,GAAG,QAAQ,CAAC,MAAM,QAAQ,CAAC,CAAC;IAE/C;;;;OAIG;IACH,cAAc,CAAC,EAAE,OAAO,CAAC;CAC5B;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+DG;AACH,qBAAa,aAAa;IACtB;;OAEG;IACH,SAAS,CAAC,MAAM,CAAC,OAAO,EAAE,UAAU,CAgBlC;IAEF,SAAS,CAAC,MAAM,CAAC,YAAY,WAAmF;IAEhH,SAAS,CAAC,MAAM,CAAC,YAAY,WAA8E;IAE3G,SAAS,CAAC,MAAM,CAAC,QAAQ,EAAE,UAAU,CAanC;IAEF;;;OAGG;IACH,MAAM,CAAC,OAAO,mCAA0C;IAExD,SAAS,CAAC,OAAO,EAAG,GAAG,CAAC,MAAM,oBAAoB,EAAE,oBAAoB,CAAC,MAAM,oBAAoB,CAAC,CAAC,CAAC;IACtG,SAAS,CAAC,QAAQ,uBAA8B;IAEhD,gBAAgB;IAChB,MAAM,CAAC,YAAY,CAAC,EAAE,aAAa,CAAC;IAEpC,SAAgB,eAAe,kEAA0C;IAEzE;;OAEG;gBACS,OAAO,GAAE,oBAAyB;IAiB9C;;;;OAIG;IACH,GAAG,CAAC,CAAC,SAAS,MAAM,oBAAoB,EAAE,CAAC,SAAS,oBAAoB,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,YAAY,CAAC,EAAE,CAAC,GAAG,CAAC;IA2BzG,SAAS,CAAC,aAAa,CAAC,GAAG,EAAE,MAAM,oBAAoB,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO;IAazF;;;OAGG;IACH,GAAG,CAAC,GAAG,EAAE,MAAM,oBAAoB,EAAE,KAAK,CAAC,EAAE,GAAG,GAAG,IAAI;IAIvD;;;OAGG;IACH,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,oBAAoB,EAAE,KAAK,CAAC,EAAE,GAAG,GAAG,IAAI;IAI9D;;;;;;;;OAQG;IACH,gBAAgB,IAAI,aAAa;IASjC,eAAe,IAAI,YAAY;IAe/B;;;OAGG;IACH,mBAAmB,CAAC,OAAO,GAAE,oBAAyB,GAAG,aAAa;IAiBtE,gBAAgB,CAAC,MAAM,EAAE,aAAa,GAAG,IAAI;IAI7C,MAAM,CAAC,gBAAgB,CAAC,MAAM,EAAE,aAAa,GAAG,IAAI;IAIpD,eAAe,CAAC,MAAM,EAAE,YAAY,GAAG,IAAI;IAI3C;;OAEG;IACH,MAAM,CAAC,eAAe,IAAI,aAAa;IASvC;;OAEG;IACH,MAAM,CAAC,gBAAgB,IAAI,aAAa;IAIxC;;OAEG;IACH,MAAM,CAAC,eAAe,IAAI,YAAY;IAItC;;;OAGG;IACH,MAAM,CAAC,gBAAgB,IAAI,IAAI;IAI/B,SAAS,CAAC,YAAY,CAAC,OAAO,EAAE,oBAAoB;CAgBvD"}
|
package/cookie_utils.d.ts
CHANGED
|
@@ -1,5 +1,3 @@
|
|
|
1
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
2
|
-
/// <reference types="node" />
|
|
3
1
|
import type { IncomingMessage } from 'node:http';
|
|
4
2
|
import type { BrowserLikeResponse, Dictionary, Cookie as CookieObject } from '@crawlee/types';
|
|
5
3
|
import { Cookie } from 'tough-cookie';
|
package/cookie_utils.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cookie_utils.d.ts","sourceRoot":"","sources":["../src/cookie_utils.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"cookie_utils.d.ts","sourceRoot":"","sources":["../src/cookie_utils.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAEjD,OAAO,KAAK,EAAE,mBAAmB,EAAE,UAAU,EAAE,MAAM,IAAI,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9F,OAAO,EAAE,MAAM,EAAa,MAAM,cAAc,CAAC;AAKjD;;GAEG;AACH,wBAAgB,sBAAsB,CAClC,QAAQ,EAAE,eAAe,GAAG,mBAAmB,GAAG;IAAE,OAAO,EAAE,UAAU,CAAC,MAAM,GAAG,MAAM,EAAE,CAAC,CAAA;CAAE,GAC7F,MAAM,EAAE,CAWV;AAED;;;;;GAKG;AACH,wBAAgB,8BAA8B,CAAC,UAAU,EAAE,MAAM,QAEhE;AAED;;;;;GAKG;AACH,wBAAgB,8BAA8B,CAAC,WAAW,EAAE,MAAM,GAAG,YAAY,CAYhF;AAED;;;;GAIG;AACH,wBAAgB,8BAA8B,CAAC,YAAY,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,UAiB5F;AAED;;;;GAIG;AACH,wBAAgB,yBAAyB,CAAC,YAAY,EAAE,MAAM,uBAQ7D;AAED;;;;GAIG;AACH,wBAAgB,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,GAAG,MAAM,CA8BzE"}
|
package/cookie_utils.js
CHANGED
|
@@ -1,6 +1,11 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.
|
|
3
|
+
exports.getCookiesFromResponse = getCookiesFromResponse;
|
|
4
|
+
exports.getDefaultCookieExpirationDate = getDefaultCookieExpirationDate;
|
|
5
|
+
exports.toughCookieToBrowserPoolCookie = toughCookieToBrowserPoolCookie;
|
|
6
|
+
exports.browserPoolCookieToToughCookie = browserPoolCookieToToughCookie;
|
|
7
|
+
exports.cookieStringToToughCookie = cookieStringToToughCookie;
|
|
8
|
+
exports.mergeCookies = mergeCookies;
|
|
4
9
|
const tough_cookie_1 = require("tough-cookie");
|
|
5
10
|
const log_1 = require("./log");
|
|
6
11
|
const errors_1 = require("./session_pool/errors");
|
|
@@ -19,7 +24,6 @@ function getCookiesFromResponse(response) {
|
|
|
19
24
|
throw new errors_1.CookieParseError(cookieHeader);
|
|
20
25
|
}
|
|
21
26
|
}
|
|
22
|
-
exports.getCookiesFromResponse = getCookiesFromResponse;
|
|
23
27
|
/**
|
|
24
28
|
* Calculate cookie expiration date
|
|
25
29
|
* @param maxAgeSecs
|
|
@@ -29,7 +33,6 @@ exports.getCookiesFromResponse = getCookiesFromResponse;
|
|
|
29
33
|
function getDefaultCookieExpirationDate(maxAgeSecs) {
|
|
30
34
|
return new Date(Date.now() + maxAgeSecs * 1000);
|
|
31
35
|
}
|
|
32
|
-
exports.getDefaultCookieExpirationDate = getDefaultCookieExpirationDate;
|
|
33
36
|
/**
|
|
34
37
|
* Transforms tough-cookie to puppeteer cookie.
|
|
35
38
|
* @param toughCookie Cookie from CookieJar
|
|
@@ -49,7 +52,6 @@ function toughCookieToBrowserPoolCookie(toughCookie) {
|
|
|
49
52
|
httpOnly: toughCookie.httpOnly,
|
|
50
53
|
};
|
|
51
54
|
}
|
|
52
|
-
exports.toughCookieToBrowserPoolCookie = toughCookieToBrowserPoolCookie;
|
|
53
55
|
/**
|
|
54
56
|
* Transforms browser-pool cookie to tough-cookie.
|
|
55
57
|
* @param cookieObject Cookie object (for instance from the `page.cookies` method).
|
|
@@ -73,7 +75,6 @@ function browserPoolCookieToToughCookie(cookieObject, maxAgeSecs) {
|
|
|
73
75
|
hostOnly: !domainHasLeadingDot,
|
|
74
76
|
});
|
|
75
77
|
}
|
|
76
|
-
exports.browserPoolCookieToToughCookie = browserPoolCookieToToughCookie;
|
|
77
78
|
/**
|
|
78
79
|
* @internal
|
|
79
80
|
* @param cookieString The cookie string to attempt parsing
|
|
@@ -86,7 +87,6 @@ function cookieStringToToughCookie(cookieString) {
|
|
|
86
87
|
}
|
|
87
88
|
return null;
|
|
88
89
|
}
|
|
89
|
-
exports.cookieStringToToughCookie = cookieStringToToughCookie;
|
|
90
90
|
/**
|
|
91
91
|
* Merges multiple cookie strings. Keys are compared case-sensitively, warning will be logged
|
|
92
92
|
* if we see two cookies with same keys but different casing.
|
|
@@ -116,5 +116,4 @@ function mergeCookies(url, sourceCookies) {
|
|
|
116
116
|
}
|
|
117
117
|
return jar.getCookieStringSync(url);
|
|
118
118
|
}
|
|
119
|
-
exports.mergeCookies = mergeCookies;
|
|
120
119
|
//# sourceMappingURL=cookie_utils.js.map
|
package/cookie_utils.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cookie_utils.js","sourceRoot":"","sources":["../src/cookie_utils.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"cookie_utils.js","sourceRoot":"","sources":["../src/cookie_utils.ts"],"names":[],"mappings":";;AAWA,wDAaC;AAQD,wEAEC;AAQD,wEAYC;AAOD,wEAiBC;AAOD,8DAQC;AAOD,oCA8BC;AA/HD,+CAAiD;AAEjD,+BAA4B;AAC5B,kDAAyD;AAEzD;;GAEG;AACH,SAAgB,sBAAsB,CAClC,QAA4F;IAE5F,MAAM,OAAO,GAAG,OAAO,QAAQ,CAAC,OAAO,KAAK,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC;IAC/F,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC;IAEjD,IAAI,CAAC;QACD,OAAO,KAAK,CAAC,OAAO,CAAC,YAAY,CAAC;YAC9B,CAAC,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,qBAAM,CAAC,KAAK,CAAC,MAAM,CAAE,CAAC;YACrD,CAAC,CAAC,CAAC,qBAAM,CAAC,KAAK,CAAC,YAAY,CAAE,CAAC,CAAC;IACxC,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACT,MAAM,IAAI,yBAAgB,CAAC,YAAY,CAAC,CAAC;IAC7C,CAAC;AACL,CAAC;AAED;;;;;GAKG;AACH,SAAgB,8BAA8B,CAAC,UAAkB;IAC7D,OAAO,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,UAAU,GAAG,IAAI,CAAC,CAAC;AACpD,CAAC;AAED;;;;;GAKG;AACH,SAAgB,8BAA8B,CAAC,WAAmB;IAC9D,OAAO;QACH,IAAI,EAAE,WAAW,CAAC,GAAG;QACrB,KAAK,EAAE,WAAW,CAAC,KAAK;QACxB,iFAAiF;QACjF,6FAA6F;QAC7F,OAAO,EAAE,WAAW,CAAC,OAAO,KAAK,UAAU,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,GAAG,IAAI;QACxG,MAAM,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,WAAW,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,GAAG,WAAW,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,SAAS;QAClG,IAAI,EAAE,WAAW,CAAC,IAAI,IAAI,SAAS;QACnC,MAAM,EAAE,WAAW,CAAC,MAAM;QAC1B,QAAQ,EAAE,WAAW,CAAC,QAAQ;KACjC,CAAC;AACN,CAAC;AAED;;;;GAIG;AACH,SAAgB,8BAA8B,CAAC,YAA0B,EAAE,UAAkB;IACzF,MAAM,cAAc,GAAG,YAAY,CAAC,OAAO,IAAI,OAAO,YAAY,CAAC,OAAO,KAAK,QAAQ,IAAI,YAAY,CAAC,OAAO,GAAG,CAAC,CAAC;IACpH,MAAM,OAAO,GAAG,cAAc;QAC1B,CAAC,CAAC,IAAI,IAAI,CAAC,YAAY,CAAC,OAAQ,GAAG,IAAI,CAAC;QACxC,CAAC,CAAC,8BAA8B,CAAC,UAAU,CAAC,CAAC;IACjD,MAAM,mBAAmB,GAAG,YAAY,CAAC,MAAM,EAAE,UAAU,EAAE,CAAC,GAAG,CAAC,CAAC;IACnE,MAAM,MAAM,GAAG,mBAAmB,CAAC,CAAC,CAAC,YAAY,CAAC,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,MAAM,CAAC;IAC3F,OAAO,IAAI,qBAAM,CAAC;QACd,GAAG,EAAE,YAAY,CAAC,IAAI;QACtB,KAAK,EAAE,YAAY,CAAC,KAAK;QACzB,OAAO;QACP,MAAM;QACN,IAAI,EAAE,YAAY,CAAC,IAAI;QACvB,MAAM,EAAE,YAAY,CAAC,MAAM;QAC3B,QAAQ,EAAE,YAAY,CAAC,QAAQ;QAC/B,QAAQ,EAAE,CAAC,mBAAmB;KACjC,CAAC,CAAC;AACP,CAAC;AAED;;;;GAIG;AACH,SAAgB,yBAAyB,CAAC,YAAoB;IAC1D,MAAM,MAAM,GAAG,qBAAM,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;IAE1C,IAAI,MAAM,EAAE,CAAC;QACT,OAAO,8BAA8B,CAAC,MAAM,CAAC,CAAC;IAClD,CAAC;IAED,OAAO,IAAI,CAAC;AAChB,CAAC;AAED;;;;GAIG;AACH,SAAgB,YAAY,CAAC,GAAW,EAAE,aAAuB;IAC7D,MAAM,GAAG,GAAG,IAAI,wBAAS,EAAE,CAAC;IAE5B,uBAAuB;IACvB,KAAK,MAAM,kBAAkB,IAAI,aAAa,EAAE,CAAC;QAC7C,uBAAuB;QACvB,IAAI,CAAC,kBAAkB;YAAE,SAAS;QAElC,MAAM,OAAO,GAAG,kBAAkB,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAElD,KAAK,MAAM,YAAY,IAAI,OAAO,EAAE,CAAC;YACjC,sBAAsB;YACtB,IAAI,CAAC,YAAY;gBAAE,SAAS;YAE5B,MAAM,MAAM,GAAG,qBAAM,CAAC,KAAK,CAAC,YAAY,CAAE,CAAC;YAC3C,MAAM,gBAAgB,GAAG,GAAG,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE;gBACxD,OAAO,MAAM,CAAC,GAAG,KAAK,CAAC,CAAC,GAAG,IAAI,MAAM,CAAC,GAAG,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC;YACpF,CAAC,CAAC,CAAC;YAEH,IAAI,gBAAgB,EAAE,CAAC;gBACnB,SAAG,CAAC,UAAU,CACV,2DAA2D,MAAM,CAAC,GAAG,UAAU,gBAAgB,CAAC,GAAG,GAAG,CACzG,CAAC;YACN,CAAC;YAED,GAAG,CAAC,aAAa,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QACnC,CAAC;IACL,CAAC;IAED,OAAO,GAAG,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAC;AACxC,CAAC"}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.handleRequestTimeout =
|
|
3
|
+
exports.handleRequestTimeout = handleRequestTimeout;
|
|
4
4
|
const timeout_1 = require("@apify/timeout");
|
|
5
5
|
/**
|
|
6
6
|
* Handles timeout request
|
|
@@ -12,5 +12,4 @@ function handleRequestTimeout({ session, errorMessage }) {
|
|
|
12
12
|
const timeoutSecs = Number(timeoutMillis) / 1000;
|
|
13
13
|
throw new timeout_1.TimeoutError(`Navigation timed out after ${timeoutSecs} seconds.`);
|
|
14
14
|
}
|
|
15
|
-
exports.handleRequestTimeout = handleRequestTimeout;
|
|
16
15
|
//# sourceMappingURL=crawler_utils.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"crawler_utils.js","sourceRoot":"","sources":["../../src/crawlers/crawler_utils.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"crawler_utils.js","sourceRoot":"","sources":["../../src/crawlers/crawler_utils.ts"],"names":[],"mappings":";;AAQA,oDAKC;AAbD,4CAA8C;AAI9C;;;GAGG;AACH,SAAgB,oBAAoB,CAAC,EAAE,OAAO,EAAE,YAAY,EAA+C;IACvG,OAAO,EAAE,OAAO,EAAE,CAAC;IACnB,MAAM,aAAa,GAAG,YAAY,CAAC,KAAK,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,wBAAwB;IACrF,MAAM,WAAW,GAAG,MAAM,CAAC,aAAa,CAAC,GAAG,IAAI,CAAC;IACjD,MAAM,IAAI,sBAAY,CAAC,8BAA8B,WAAW,WAAW,CAAC,CAAC;AACjF,CAAC"}
|
|
@@ -6,7 +6,7 @@ export interface EnqueueLinksOptions extends RequestQueueOperationOptions {
|
|
|
6
6
|
/** Limit the amount of actually enqueued URLs to this number. Useful for testing across the entire crawling scope. */
|
|
7
7
|
limit?: number;
|
|
8
8
|
/** An array of URLs to enqueue. */
|
|
9
|
-
urls?:
|
|
9
|
+
urls?: readonly string[];
|
|
10
10
|
/** A request queue to which the URLs will be enqueued. */
|
|
11
11
|
requestQueue?: RequestProvider;
|
|
12
12
|
/** A CSS selector matching links to be enqueued. */
|
|
@@ -38,7 +38,7 @@ export interface EnqueueLinksOptions extends RequestQueueOperationOptions {
|
|
|
38
38
|
* If `globs` is an empty array or `undefined`, and `regexps` are also not defined, then the function
|
|
39
39
|
* enqueues the links with the same subdomain.
|
|
40
40
|
*/
|
|
41
|
-
globs?:
|
|
41
|
+
globs?: readonly GlobInput[];
|
|
42
42
|
/**
|
|
43
43
|
* An array of glob pattern strings, regexp patterns or plain objects
|
|
44
44
|
* containing patterns matching URLs that will **never** be enqueued.
|
|
@@ -49,7 +49,7 @@ export interface EnqueueLinksOptions extends RequestQueueOperationOptions {
|
|
|
49
49
|
* Glob matching is always case-insensitive.
|
|
50
50
|
* If you need case-sensitive matching, provide a regexp.
|
|
51
51
|
*/
|
|
52
|
-
exclude?:
|
|
52
|
+
exclude?: readonly (GlobInput | RegExpInput)[];
|
|
53
53
|
/**
|
|
54
54
|
* An array of regular expressions or plain objects
|
|
55
55
|
* containing regular expressions matching the URLs to be enqueued.
|
|
@@ -60,7 +60,7 @@ export interface EnqueueLinksOptions extends RequestQueueOperationOptions {
|
|
|
60
60
|
* If `regexps` is an empty array or `undefined`, and `globs` are also not defined, then the function
|
|
61
61
|
* enqueues the links with the same subdomain.
|
|
62
62
|
*/
|
|
63
|
-
regexps?:
|
|
63
|
+
regexps?: readonly RegExpInput[];
|
|
64
64
|
/**
|
|
65
65
|
* *NOTE:* In future versions of SDK the options will be removed.
|
|
66
66
|
* Please use `globs` or `regexps` instead.
|
|
@@ -79,7 +79,7 @@ export interface EnqueueLinksOptions extends RequestQueueOperationOptions {
|
|
|
79
79
|
*
|
|
80
80
|
* @deprecated prefer using `globs` or `regexps` instead
|
|
81
81
|
*/
|
|
82
|
-
pseudoUrls?:
|
|
82
|
+
pseudoUrls?: readonly PseudoUrlInput[];
|
|
83
83
|
/**
|
|
84
84
|
* Just before a new {@apilink Request} is constructed and enqueued to the {@apilink RequestQueue}, this function can be used
|
|
85
85
|
* to remove it or modify its contents such as `userData`, `payload` or, most importantly `uniqueKey`. This is useful
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"enqueue_links.d.ts","sourceRoot":"","sources":["../../src/enqueue_links/enqueue_links.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,sBAAsB,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAGzE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAE7C,OAAO,KAAK,EAAE,SAAS,EAAE,cAAc,EAAE,WAAW,EAAE,gBAAgB,EAAoB,MAAM,UAAU,CAAC;AAU3G,OAAO,KAAK,EAAE,eAAe,EAAE,4BAA4B,EAAE,MAAM,aAAa,CAAC;AAEjF,MAAM,WAAW,mBAAoB,SAAQ,4BAA4B;IACrE,sHAAsH;IACtH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf,mCAAmC;IACnC,IAAI,CAAC,EAAE,
|
|
1
|
+
{"version":3,"file":"enqueue_links.d.ts","sourceRoot":"","sources":["../../src/enqueue_links/enqueue_links.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,sBAAsB,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAGzE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAE7C,OAAO,KAAK,EAAE,SAAS,EAAE,cAAc,EAAE,WAAW,EAAE,gBAAgB,EAAoB,MAAM,UAAU,CAAC;AAU3G,OAAO,KAAK,EAAE,eAAe,EAAE,4BAA4B,EAAE,MAAM,aAAa,CAAC;AAEjF,MAAM,WAAW,mBAAoB,SAAQ,4BAA4B;IACrE,sHAAsH;IACtH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf,mCAAmC;IACnC,IAAI,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAEzB,0DAA0D;IAC1D,YAAY,CAAC,EAAE,eAAe,CAAC;IAE/B,oDAAoD;IACpD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB,oEAAoE;IACpE,QAAQ,CAAC,EAAE,UAAU,CAAC;IAEtB,iEAAiE;IACjE,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf;;;OAGG;IACH,cAAc,CAAC,EAAE,OAAO,CAAC;IAEzB;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;;;;;;;;;;;OAYG;IACH,KAAK,CAAC,EAAE,SAAS,SAAS,EAAE,CAAC;IAE7B;;;;;;;;;OASG;IACH,OAAO,CAAC,EAAE,SAAS,CAAC,SAAS,GAAG,WAAW,CAAC,EAAE,CAAC;IAE/C;;;;;;;;;OASG;IACH,OAAO,CAAC,EAAE,SAAS,WAAW,EAAE,CAAC;IAEjC;;;;;;;;;;;;;;;;;OAiBG;IACH,UAAU,CAAC,EAAE,SAAS,cAAc,EAAE,CAAC;IAEvC;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACH,wBAAwB,CAAC,EAAE,gBAAgB,CAAC;IAE5C;;;;;;;;;;;;;;;;;OAiBG;IACH,QAAQ,CAAC,EAAE,eAAe,GAAG,KAAK,GAAG,aAAa,GAAG,eAAe,GAAG,aAAa,CAAC;IAErF;;;OAGG;IACH,2BAA2B,CAAC,EAAE,OAAO,CAAC;CACzC;AAED;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,oBAAY,eAAe;IACvB;;OAEG;IACH,GAAG,QAAQ;IAEX;;;;;;OAMG;IACH,YAAY,kBAAkB;IAE9B;;;;;;OAMG;IACH,UAAU,gBAAgB;IAE1B;;;;;;OAMG;IACH,UAAU,gBAAgB;CAC7B;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,wBAAsB,YAAY,CAC9B,OAAO,EAAE,WAAW,CAAC,mBAAmB,EAAE,cAAc,GAAG,MAAM,CAAC,GACnE,OAAO,CAAC,sBAAsB,CAAC,CA0JjC;AAED;;;;;;;GAOG;AACH,wBAAgB,sCAAsC,CAAC,EACnD,eAAe,EACf,eAAe,EACf,kBAAkB,EAClB,mBAAmB,GACtB,EAAE,cAAc,sBA+BhB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC3B,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,eAAe,CAAC,EAAE,mBAAmB,CAAC,UAAU,CAAC,CAAC;IAClD,kBAAkB,EAAE,MAAM,CAAC;IAC3B,eAAe,CAAC,EAAE,MAAM,CAAC;CAC5B"}
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.
|
|
3
|
+
exports.EnqueueStrategy = void 0;
|
|
4
|
+
exports.enqueueLinks = enqueueLinks;
|
|
5
|
+
exports.resolveBaseUrlForEnqueueLinksFiltering = resolveBaseUrlForEnqueueLinksFiltering;
|
|
4
6
|
const tslib_1 = require("tslib");
|
|
5
7
|
const log_1 = tslib_1.__importDefault(require("@apify/log"));
|
|
6
8
|
const ow_1 = tslib_1.__importDefault(require("ow"));
|
|
@@ -194,7 +196,6 @@ async function enqueueLinks(options) {
|
|
|
194
196
|
});
|
|
195
197
|
return { processedRequests: addedRequests, unprocessedRequests: [] };
|
|
196
198
|
}
|
|
197
|
-
exports.enqueueLinks = enqueueLinks;
|
|
198
199
|
/**
|
|
199
200
|
* @internal
|
|
200
201
|
* This method helps resolve the baseUrl that will be used for filtering in {@apilink enqueueLinks}.
|
|
@@ -229,7 +230,6 @@ function resolveBaseUrlForEnqueueLinksFiltering({ enqueueStrategy, finalRequestU
|
|
|
229
230
|
// before actually finding the urls
|
|
230
231
|
return originalUrlOrigin;
|
|
231
232
|
}
|
|
232
|
-
exports.resolveBaseUrlForEnqueueLinksFiltering = resolveBaseUrlForEnqueueLinksFiltering;
|
|
233
233
|
/**
|
|
234
234
|
* Internal function that changes the enqueue globs to match both http and https
|
|
235
235
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"enqueue_links.js","sourceRoot":"","sources":["../../src/enqueue_links/enqueue_links.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"enqueue_links.js","sourceRoot":"","sources":["../../src/enqueue_links/enqueue_links.ts"],"names":[],"mappings":";;;AAgPA,oCA4JC;AAUD,wFAoCC;;AA1bD,6DAA6B;AAE7B,oDAAoB;AACpB,iCAAkC;AAIlC,qCAOkB;AAmJlB;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,IAAY,eAgCX;AAhCD,WAAY,eAAe;IACvB;;OAEG;IACH,8BAAW,CAAA;IAEX;;;;;;OAMG;IACH,iDAA8B,CAAA;IAE9B;;;;;;OAMG;IACH,6CAA0B,CAAA;IAE1B;;;;;;OAMG;IACH,6CAA0B,CAAA;AAC9B,CAAC,EAhCW,eAAe,+BAAf,eAAe,QAgC1B;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACI,KAAK,UAAU,YAAY,CAC9B,OAAkE;IAElE,IAAI,CAAC,OAAO,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAChD,MAAM,IAAI,UAAU,CAChB;YACI,4JAA4J;YAC5J,+GAA+G;SAClH,CAAC,IAAI,CAAC,IAAI,CAAC,CACf,CAAC;IACN,CAAC;IAED,IAAA,YAAE,EACE,OAAO,EACP,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;QACjB,IAAI,EAAE,YAAE,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,MAAM,CAAC;QAChC,YAAY,EAAE,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,kBAAkB,EAAE,YAAY,CAAC;QACjE,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;QAC9B,cAAc,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;QACnC,KAAK,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACzB,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC5B,OAAO,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC3B,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC5B,KAAK,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACzB,UAAU,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,GAAG,CAAC,YAAE,CAAC,MAAM,EAAE,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;QAClF,KAAK,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,GAAG,CAAC,YAAE,CAAC,MAAM,EAAE,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;QAC7E,OAAO,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAC7B,YAAE,CAAC,GAAG,CAAC,YAAE,CAAC,MAAM,EAAE,YAAE,CAAC,MAAM,EAAE,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CACvF;QACD,OAAO,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,GAAG,CAAC,YAAE,CAAC,MAAM,EAAE,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;QACjF,wBAAwB,EAAE,YAAE,CAAC,QAAQ,CAAC,QAAQ;QAC9C,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;QAClE,2BAA2B,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;KACnD,CAAC,CACL,CAAC;IAEF,MAAM,EACF,YAAY,EACZ,KAAK,EACL,IAAI,EACJ,UAAU,EACV,OAAO,EACP,KAAK,EACL,OAAO,EACP,wBAAwB,EACxB,SAAS,EACT,2BAA2B,GAC9B,GAAG,OAAO,CAAC;IAEZ,MAAM,wBAAwB,GAAuB,EAAE,CAAC;IACxD,MAAM,iBAAiB,GAAuB,EAAE,CAAC;IAEjD,IAAI,OAAO,EAAE,MAAM,EAAE,CAAC;QAClB,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE,CAAC;YACzB,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;gBAC7C,wBAAwB,CAAC,IAAI,CAAC,GAAG,IAAA,sCAA6B,EAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAC5E,CAAC;iBAAM,IAAI,IAAI,YAAY,MAAM,IAAI,QAAQ,IAAI,IAAI,EAAE,CAAC;gBACpD,wBAAwB,CAAC,IAAI,CAAC,GAAG,IAAA,0CAAiC,EAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAChF,CAAC;QACL,CAAC;IACL,CAAC;IAED,IAAI,UAAU,EAAE,MAAM,EAAE,CAAC;QACrB,aAAG,CAAC,UAAU,CAAC,qEAAqE,CAAC,CAAC;QACtF,iBAAiB,CAAC,IAAI,CAAC,GAAG,IAAA,6CAAoC,EAAC,UAAU,CAAC,CAAC,CAAC;IAChF,CAAC;IAED,IAAI,KAAK,EAAE,MAAM,EAAE,CAAC;QAChB,iBAAiB,CAAC,IAAI,CAAC,GAAG,IAAA,sCAA6B,EAAC,KAAK,CAAC,CAAC,CAAC;IACpE,CAAC;IAED,IAAI,OAAO,EAAE,MAAM,EAAE,CAAC;QAClB,iBAAiB,CAAC,IAAI,CAAC,GAAG,IAAA,0CAAiC,EAAC,OAAO,CAAC,CAAC,CAAC;IAC1E,CAAC;IAED,IAAI,CAAC,iBAAiB,CAAC,MAAM,EAAE,CAAC;QAC5B,OAAO,CAAC,QAAQ,KAAhB,OAAO,CAAC,QAAQ,GAAK,eAAe,CAAC,YAAY,EAAC;IACtD,CAAC;IAED,MAAM,uBAAuB,GAAuB,EAAE,CAAC;IAEvD,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;QAClB,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAErC,QAAQ,OAAO,CAAC,QAAQ,EAAE,CAAC;YACvB,KAAK,eAAe,CAAC,YAAY;gBAC7B,sFAAsF;gBACtF,uFAAuF;gBACvF,yCAAyC;gBACzC,uBAAuB,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,gBAAgB,CAAC,GAAG,GAAG,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC,CAAC;gBAC7E,MAAM;YACV,KAAK,eAAe,CAAC,UAAU,CAAC,CAAC,CAAC;gBAC9B,4CAA4C;gBAC5C,MAAM,eAAe,GAAG,IAAA,iBAAS,EAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC,CAAC;gBAExE,IAAI,eAAe,EAAE,CAAC;oBAClB,gHAAgH;oBAChH,GAAG,CAAC,QAAQ,GAAG,eAAe,CAAC;oBAC/B,uBAAuB,CAAC,IAAI,CACxB,EAAE,IAAI,EAAE,gBAAgB,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,eAAe,EAAE,KAAK,eAAe,EAAE,CAAC,KAAK,CAAC,EAAE,EAC/F,EAAE,IAAI,EAAE,gBAAgB,CAAC,GAAG,GAAG,CAAC,MAAM,KAAK,CAAC,EAAE,CACjD,CAAC;gBACN,CAAC;qBAAM,CAAC;oBACJ,6FAA6F;oBAC7F,4BAA4B;oBAC5B,uBAAuB,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,gBAAgB,CAAC,GAAG,GAAG,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC,CAAC;gBACjF,CAAC;gBAED,MAAM;YACV,CAAC;YACD,KAAK,eAAe,CAAC,UAAU,CAAC,CAAC,CAAC;gBAC9B,4EAA4E;gBAC5E,uBAAuB,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,GAAG,CAAC,MAAM,KAAK,EAAE,CAAC,CAAC;gBAC3D,MAAM;YACV,CAAC;YACD,KAAK,eAAe,CAAC,GAAG,CAAC;YACzB;gBACI,uBAAuB,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,eAAe,EAAE,CAAC,CAAC;gBACxD,MAAM;QACd,CAAC;IACL,CAAC;IAED,IAAI,cAAc,GAAG,IAAA,6BAAoB,EAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAEzD,IAAI,wBAAwB,EAAE,CAAC;QAC3B,cAAc,GAAG,cAAc;aAC1B,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,wBAAwB,CAAC,OAAO,CAAC,CAAC;aACnD,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAqB,CAAC;IAChD,CAAC;IAED,SAAS,sBAAsB;QAC3B,sEAAsE;QACtE,IAAI,iBAAiB,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACjC,OAAO,IAAA,uBAAc,EAAC,cAAc,EAAE,uBAAuB,EAAE,wBAAwB,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;QAC/G,CAAC;QAED,qDAAqD;QACrD,MAAM,gCAAgC,GAAG,IAAA,uBAAc,EACnD,cAAc,EACd,iBAAiB,EACjB,wBAAwB,EACxB,OAAO,CAAC,QAAQ,CACnB,CAAC;QACF,+EAA+E;QAC/E,OAAO,IAAA,iCAAwB,EAAC,gCAAgC,EAAE,uBAAuB,CAAC,CAAC;IAC/F,CAAC;IAED,IAAI,QAAQ,GAAG,sBAAsB,EAAE,CAAC;IACxC,IAAI,KAAK;QAAE,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;IAE/C,MAAM,EAAE,aAAa,EAAE,GAAG,MAAM,YAAY,CAAC,kBAAkB,CAAC,QAAQ,EAAE;QACtE,SAAS;QACT,2BAA2B;KAC9B,CAAC,CAAC;IAEH,OAAO,EAAE,iBAAiB,EAAE,aAAa,EAAE,mBAAmB,EAAE,EAAE,EAAE,CAAC;AACzE,CAAC;AAED;;;;;;;GAOG;AACH,SAAgB,sCAAsC,CAAC,EACnD,eAAe,EACf,eAAe,EACf,kBAAkB,EAClB,mBAAmB,GACN;IACb,wCAAwC;IACxC,IAAI,mBAAmB,EAAE,CAAC;QACtB,OAAO,mBAAmB,CAAC;IAC/B,CAAC;IAED,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC,kBAAkB,CAAC,CAAC,MAAM,CAAC;IAC7D,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,eAAe,IAAI,kBAAkB,CAAC,CAAC,MAAM,CAAC;IAE7E,6DAA6D;IAC7D,IAAI,eAAe,KAAK,eAAe,CAAC,GAAG,EAAE,CAAC;QAC1C,OAAO,cAAc,CAAC;IAC1B,CAAC;IAED,0HAA0H;IAC1H,+IAA+I;IAC/I,4DAA4D;IAC5D,IAAI,eAAe,KAAK,eAAe,CAAC,UAAU,EAAE,CAAC;QACjD,MAAM,gBAAgB,GAAG,IAAA,iBAAS,EAAC,iBAAiB,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,CAAE,CAAC;QAC/E,MAAM,aAAa,GAAG,IAAA,iBAAS,EAAC,cAAc,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,CAAE,CAAC;QAEzE,IAAI,gBAAgB,KAAK,aAAa,EAAE,CAAC;YACrC,OAAO,cAAc,CAAC;QAC1B,CAAC;QAED,OAAO,SAAS,CAAC;IACrB,CAAC;IAED,2JAA2J;IAC3J,mCAAmC;IACnC,OAAO,iBAAiB,CAAC;AAC7B,CAAC;AAYD;;GAEG;AACH,SAAS,gBAAgB,CAAC,OAAe;IACrC,OAAO,OAAO,CAAC,OAAO,CAAC,gBAAgB,EAAE,aAAa,CAAC,CAAC;AAC5D,CAAC"}
|
|
@@ -27,13 +27,13 @@ export declare function updateEnqueueLinksPatternCache(item: GlobInput | RegExpI
|
|
|
27
27
|
* to construct RegExps from PseudoUrl strings.
|
|
28
28
|
* @ignore
|
|
29
29
|
*/
|
|
30
|
-
export declare function constructRegExpObjectsFromPseudoUrls(pseudoUrls:
|
|
30
|
+
export declare function constructRegExpObjectsFromPseudoUrls(pseudoUrls: readonly PseudoUrlInput[]): RegExpObject[];
|
|
31
31
|
/**
|
|
32
32
|
* Helper factory used in the `enqueueLinks()` and enqueueLinksByClickingElements() function
|
|
33
33
|
* to construct Glob objects from Glob pattern strings.
|
|
34
34
|
* @ignore
|
|
35
35
|
*/
|
|
36
|
-
export declare function constructGlobObjectsFromGlobs(globs:
|
|
36
|
+
export declare function constructGlobObjectsFromGlobs(globs: readonly GlobInput[]): GlobObject[];
|
|
37
37
|
/**
|
|
38
38
|
* @internal
|
|
39
39
|
*/
|
|
@@ -43,7 +43,7 @@ export declare function validateGlobPattern(glob: string): string;
|
|
|
43
43
|
* to check RegExps input and return valid RegExps.
|
|
44
44
|
* @ignore
|
|
45
45
|
*/
|
|
46
|
-
export declare function constructRegExpObjectsFromRegExps(regexps:
|
|
46
|
+
export declare function constructRegExpObjectsFromRegExps(regexps: readonly RegExpInput[]): RegExpObject[];
|
|
47
47
|
/**
|
|
48
48
|
* @ignore
|
|
49
49
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"shared.d.ts","sourceRoot":"","sources":["../../src/enqueue_links/shared.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AAC3D,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AACjD,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAErC,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAYhD,MAAM,MAAM,gBAAgB,GAAG;IAC3B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;CACnB,GAAG,IAAI,CAAC,cAAc,EAAE,QAAQ,GAAG,SAAS,GAAG,OAAO,GAAG,UAAU,GAAG,SAAS,CAAC,CAAC;AAElF,MAAM,MAAM,eAAe,GAAG;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CACjD,cAAc,EACd,QAAQ,GAAG,SAAS,GAAG,OAAO,GAAG,UAAU,GAAG,SAAS,CAC1D,CAAC;AAEF,MAAM,MAAM,cAAc,GAAG,MAAM,GAAG,eAAe,CAAC;AAEtD,MAAM,MAAM,UAAU,GAAG;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAC5C,cAAc,EACd,QAAQ,GAAG,SAAS,GAAG,OAAO,GAAG,UAAU,GAAG,SAAS,CAC1D,CAAC;AAEF,MAAM,MAAM,SAAS,GAAG,MAAM,GAAG,UAAU,CAAC;AAE5C,MAAM,MAAM,YAAY,GAAG;IAAE,MAAM,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAChD,cAAc,EACd,QAAQ,GAAG,SAAS,GAAG,OAAO,GAAG,UAAU,GAAG,SAAS,CAC1D,CAAC;AAEF,MAAM,MAAM,WAAW,GAAG,MAAM,GAAG,YAAY,CAAC;AAEhD;;GAEG;AACH,wBAAgB,8BAA8B,CAC1C,IAAI,EAAE,SAAS,GAAG,WAAW,GAAG,cAAc,EAC9C,OAAO,EAAE,YAAY,GAAG,UAAU,GACnC,IAAI,CAMN;AAED;;;;GAIG;AACH,wBAAgB,oCAAoC,CAAC,UAAU,EAAE,
|
|
1
|
+
{"version":3,"file":"shared.d.ts","sourceRoot":"","sources":["../../src/enqueue_links/shared.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AAC3D,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AACjD,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAErC,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAYhD,MAAM,MAAM,gBAAgB,GAAG;IAC3B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;CACnB,GAAG,IAAI,CAAC,cAAc,EAAE,QAAQ,GAAG,SAAS,GAAG,OAAO,GAAG,UAAU,GAAG,SAAS,CAAC,CAAC;AAElF,MAAM,MAAM,eAAe,GAAG;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CACjD,cAAc,EACd,QAAQ,GAAG,SAAS,GAAG,OAAO,GAAG,UAAU,GAAG,SAAS,CAC1D,CAAC;AAEF,MAAM,MAAM,cAAc,GAAG,MAAM,GAAG,eAAe,CAAC;AAEtD,MAAM,MAAM,UAAU,GAAG;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAC5C,cAAc,EACd,QAAQ,GAAG,SAAS,GAAG,OAAO,GAAG,UAAU,GAAG,SAAS,CAC1D,CAAC;AAEF,MAAM,MAAM,SAAS,GAAG,MAAM,GAAG,UAAU,CAAC;AAE5C,MAAM,MAAM,YAAY,GAAG;IAAE,MAAM,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAChD,cAAc,EACd,QAAQ,GAAG,SAAS,GAAG,OAAO,GAAG,UAAU,GAAG,SAAS,CAC1D,CAAC;AAEF,MAAM,MAAM,WAAW,GAAG,MAAM,GAAG,YAAY,CAAC;AAEhD;;GAEG;AACH,wBAAgB,8BAA8B,CAC1C,IAAI,EAAE,SAAS,GAAG,WAAW,GAAG,cAAc,EAC9C,OAAO,EAAE,YAAY,GAAG,UAAU,GACnC,IAAI,CAMN;AAED;;;;GAIG;AACH,wBAAgB,oCAAoC,CAAC,UAAU,EAAE,SAAS,cAAc,EAAE,GAAG,YAAY,EAAE,CAiB1G;AAED;;;;GAIG;AACH,wBAAgB,6BAA6B,CAAC,KAAK,EAAE,SAAS,SAAS,EAAE,GAAG,UAAU,EAAE,CAkCvF;AAED;;GAEG;AACH,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAKxD;AAED;;;;GAIG;AACH,wBAAgB,iCAAiC,CAAC,OAAO,EAAE,SAAS,WAAW,EAAE,GAAG,YAAY,EAAE,CAgBjG;AAED;;GAEG;AACH,wBAAgB,cAAc,CAC1B,cAAc,EAAE,CAAC,MAAM,GAAG,cAAc,CAAC,EAAE,EAC3C,iBAAiB,CAAC,EAAE,gBAAgB,EAAE,EACtC,qBAAqB,GAAE,gBAAgB,EAAO,EAC9C,QAAQ,CAAC,EAAE,mBAAmB,CAAC,UAAU,CAAC,GAC3C,OAAO,EAAE,CA8BX;AAED,wBAAgB,wBAAwB,CAAC,QAAQ,EAAE,OAAO,EAAE,EAAE,QAAQ,CAAC,EAAE,gBAAgB,EAAE,GAAG,OAAO,EAAE,CAoBtG;AAED;;GAEG;AACH,wBAAgB,oBAAoB,CAChC,OAAO,EAAE,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,EAAE,EAC7C,OAAO,GAAE,IAAI,CAAC,mBAAmB,EAAE,OAAO,GAAG,UAAU,GAAG,SAAS,GAAG,gBAAgB,GAAG,UAAU,CAAM,GAC1G,cAAc,EAAE,CA+BlB;AAED;;;GAGG;AACH,MAAM,WAAW,gBAAgB;IAC7B;;;OAGG;IACH,CAAC,QAAQ,EAAE,cAAc,GAAG,cAAc,GAAG,KAAK,GAAG,SAAS,GAAG,IAAI,CAAC;CACzE"}
|
package/enqueue_links/shared.js
CHANGED
|
@@ -1,6 +1,14 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.
|
|
3
|
+
exports.tryAbsoluteURL = void 0;
|
|
4
|
+
exports.updateEnqueueLinksPatternCache = updateEnqueueLinksPatternCache;
|
|
5
|
+
exports.constructRegExpObjectsFromPseudoUrls = constructRegExpObjectsFromPseudoUrls;
|
|
6
|
+
exports.constructGlobObjectsFromGlobs = constructGlobObjectsFromGlobs;
|
|
7
|
+
exports.validateGlobPattern = validateGlobPattern;
|
|
8
|
+
exports.constructRegExpObjectsFromRegExps = constructRegExpObjectsFromRegExps;
|
|
9
|
+
exports.createRequests = createRequests;
|
|
10
|
+
exports.filterRequestsByPatterns = filterRequestsByPatterns;
|
|
11
|
+
exports.createRequestOptions = createRequestOptions;
|
|
4
12
|
const url_1 = require("url");
|
|
5
13
|
const pseudo_url_1 = require("@apify/pseudo_url");
|
|
6
14
|
const minimatch_1 = require("minimatch");
|
|
@@ -25,7 +33,6 @@ function updateEnqueueLinksPatternCache(item, pattern) {
|
|
|
25
33
|
enqueueLinksPatternCache.delete(key);
|
|
26
34
|
}
|
|
27
35
|
}
|
|
28
|
-
exports.updateEnqueueLinksPatternCache = updateEnqueueLinksPatternCache;
|
|
29
36
|
/**
|
|
30
37
|
* Helper factory used in the `enqueueLinks()` and enqueueLinksByClickingElements() function
|
|
31
38
|
* to construct RegExps from PseudoUrl strings.
|
|
@@ -48,7 +55,6 @@ function constructRegExpObjectsFromPseudoUrls(pseudoUrls) {
|
|
|
48
55
|
return regexpObject;
|
|
49
56
|
});
|
|
50
57
|
}
|
|
51
|
-
exports.constructRegExpObjectsFromPseudoUrls = constructRegExpObjectsFromPseudoUrls;
|
|
52
58
|
/**
|
|
53
59
|
* Helper factory used in the `enqueueLinks()` and enqueueLinksByClickingElements() function
|
|
54
60
|
* to construct Glob objects from Glob pattern strings.
|
|
@@ -85,7 +91,6 @@ function constructGlobObjectsFromGlobs(globs) {
|
|
|
85
91
|
return globObject;
|
|
86
92
|
});
|
|
87
93
|
}
|
|
88
|
-
exports.constructGlobObjectsFromGlobs = constructGlobObjectsFromGlobs;
|
|
89
94
|
/**
|
|
90
95
|
* @internal
|
|
91
96
|
*/
|
|
@@ -95,7 +100,6 @@ function validateGlobPattern(glob) {
|
|
|
95
100
|
throw new Error(`Cannot parse Glob pattern '${globTrimmed}': it must be an non-empty string`);
|
|
96
101
|
return globTrimmed;
|
|
97
102
|
}
|
|
98
|
-
exports.validateGlobPattern = validateGlobPattern;
|
|
99
103
|
/**
|
|
100
104
|
* Helper factory used in the `enqueueLinks()` and enqueueLinksByClickingElements() function
|
|
101
105
|
* to check RegExps input and return valid RegExps.
|
|
@@ -117,7 +121,6 @@ function constructRegExpObjectsFromRegExps(regexps) {
|
|
|
117
121
|
return regexpObject;
|
|
118
122
|
});
|
|
119
123
|
}
|
|
120
|
-
exports.constructRegExpObjectsFromRegExps = constructRegExpObjectsFromRegExps;
|
|
121
124
|
/**
|
|
122
125
|
* @ignore
|
|
123
126
|
*/
|
|
@@ -148,7 +151,6 @@ function createRequests(requestOptions, urlPatternObjects, excludePatternObjects
|
|
|
148
151
|
})
|
|
149
152
|
.filter((request) => request);
|
|
150
153
|
}
|
|
151
|
-
exports.createRequests = createRequests;
|
|
152
154
|
function filterRequestsByPatterns(requests, patterns) {
|
|
153
155
|
if (!patterns?.length) {
|
|
154
156
|
return requests;
|
|
@@ -166,7 +168,6 @@ function filterRequestsByPatterns(requests, patterns) {
|
|
|
166
168
|
}
|
|
167
169
|
return filtered;
|
|
168
170
|
}
|
|
169
|
-
exports.filterRequestsByPatterns = filterRequestsByPatterns;
|
|
170
171
|
/**
|
|
171
172
|
* @ignore
|
|
172
173
|
*/
|
|
@@ -198,5 +199,4 @@ function createRequestOptions(sources, options = {}) {
|
|
|
198
199
|
return requestOptions;
|
|
199
200
|
});
|
|
200
201
|
}
|
|
201
|
-
exports.createRequestOptions = createRequestOptions;
|
|
202
202
|
//# sourceMappingURL=shared.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"shared.js","sourceRoot":"","sources":["../../src/enqueue_links/shared.ts"],"names":[],"mappings":";;;
|
|
1
|
+
{"version":3,"file":"shared.js","sourceRoot":"","sources":["../../src/enqueue_links/shared.ts"],"names":[],"mappings":";;;AAkDA,wEASC;AAOD,oFAiBC;AAOD,sEAkCC;AAKD,kDAKC;AAOD,8EAgBC;AAKD,wCAmCC;AAED,4DAoBC;AAKD,oDAkCC;AAlQD,6BAA0B;AAE1B,kDAAiD;AACjD,yCAAsC;AAItC,wCAAqC;AAErC,wCAAgD;AAAvC,uGAAA,cAAc,OAAA;AAEvB,MAAM,4BAA4B,GAAG,IAAI,CAAC;AAE1C;;;;;GAKG;AACH,MAAM,wBAAwB,GAAG,IAAI,GAAG,EAAE,CAAC;AA4B3C;;GAEG;AACH,SAAgB,8BAA8B,CAC1C,IAA8C,EAC9C,OAAkC;IAElC,wBAAwB,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC5C,IAAI,wBAAwB,CAAC,IAAI,GAAG,4BAA4B,EAAE,CAAC;QAC/D,MAAM,GAAG,GAAG,wBAAwB,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC;QACzD,wBAAwB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IACzC,CAAC;AACL,CAAC;AAED;;;;GAIG;AACH,SAAgB,oCAAoC,CAAC,UAAqC;IACtF,OAAO,UAAU,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QAC3B,mCAAmC;QACnC,IAAI,YAAY,GAAG,wBAAwB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACtD,IAAI,YAAY;YAAE,OAAO,YAAY,CAAC;QAEtC,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC3B,YAAY,GAAG,EAAE,MAAM,EAAE,IAAA,yBAAY,EAAC,IAAI,CAAC,EAAE,CAAC;QAClD,CAAC;aAAM,CAAC;YACJ,MAAM,EAAE,IAAI,EAAE,GAAG,cAAc,EAAE,GAAG,IAAI,CAAC;YACzC,YAAY,GAAG,EAAE,MAAM,EAAE,IAAA,yBAAY,EAAC,IAAI,CAAC,EAAE,GAAG,cAAc,EAAE,CAAC;QACrE,CAAC;QAED,8BAA8B,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC;QAEnD,OAAO,YAAY,CAAC;IACxB,CAAC,CAAC,CAAC;AACP,CAAC;AAED;;;;GAIG;AACH,SAAgB,6BAA6B,CAAC,KAA2B;IACrE,OAAO,KAAK;SACP,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE;QACb,uCAAuC;QACvC,IAAI,CAAC,IAAI,EAAE,CAAC;YACR,OAAO,KAAK,CAAC;QACjB,CAAC;QAED,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC3B,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC;QAClC,CAAC;QAED,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YACZ,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC;QACvC,CAAC;QAED,OAAO,KAAK,CAAC;IACjB,CAAC,CAAC;SACD,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACV,8BAA8B;QAC9B,IAAI,UAAU,GAAG,wBAAwB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACpD,IAAI,UAAU;YAAE,OAAO,UAAU,CAAC;QAElC,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC3B,UAAU,GAAG,EAAE,IAAI,EAAE,mBAAmB,CAAC,IAAI,CAAC,EAAE,CAAC;QACrD,CAAC;aAAM,CAAC;YACJ,MAAM,EAAE,IAAI,EAAE,GAAG,cAAc,EAAE,GAAG,IAAI,CAAC;YACzC,UAAU,GAAG,EAAE,IAAI,EAAE,mBAAmB,CAAC,IAAI,CAAC,EAAE,GAAG,cAAc,EAAE,CAAC;QACxE,CAAC;QAED,8BAA8B,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;QAEjD,OAAO,UAAU,CAAC;IACtB,CAAC,CAAC,CAAC;AACX,CAAC;AAED;;GAEG;AACH,SAAgB,mBAAmB,CAAC,IAAY;IAC5C,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IAChC,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC;QACxB,MAAM,IAAI,KAAK,CAAC,8BAA8B,WAAW,mCAAmC,CAAC,CAAC;IAClG,OAAO,WAAW,CAAC;AACvB,CAAC;AAED;;;;GAIG;AACH,SAAgB,iCAAiC,CAAC,OAA+B;IAC7E,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACxB,gCAAgC;QAChC,IAAI,YAAY,GAAG,wBAAwB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACtD,IAAI,YAAY;YAAE,OAAO,YAAY,CAAC;QAEtC,IAAI,IAAI,YAAY,MAAM,EAAE,CAAC;YACzB,YAAY,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;QACpC,CAAC;aAAM,CAAC;YACJ,YAAY,GAAG,IAAI,CAAC;QACxB,CAAC;QAED,8BAA8B,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC;QAEnD,OAAO,YAAY,CAAC;IACxB,CAAC,CAAC,CAAC;AACP,CAAC;AAED;;GAEG;AACH,SAAgB,cAAc,CAC1B,cAA2C,EAC3C,iBAAsC,EACtC,wBAA4C,EAAE,EAC9C,QAA0C;IAE1C,OAAO,cAAc;SAChB,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC;SAC1E,MAAM,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE;QAChB,OAAO,CAAC,qBAAqB,CAAC,IAAI,CAAC,CAAC,oBAAoB,EAAE,EAAE;YACxD,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,oBAAoB,CAAC;YAC9C,OAAO,CAAC,MAAM,IAAI,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,IAAI,IAAA,qBAAS,EAAC,GAAG,EAAE,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;QAC7F,CAAC,CAAC,CAAC;IACP,CAAC,CAAC;SACD,GAAG,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE,EAAE;QACnB,IAAI,CAAC,iBAAiB,IAAI,CAAC,iBAAiB,CAAC,MAAM,EAAE,CAAC;YAClD,OAAO,IAAI,iBAAO,CAAC,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,eAAe,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,IAAI,EAAE,CAAC,CAAC;QAC1G,CAAC;QAED,KAAK,MAAM,gBAAgB,IAAI,iBAAiB,EAAE,CAAC;YAC/C,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,oBAAoB,EAAE,GAAG,gBAAgB,CAAC;YACnE,IAAI,CAAC,MAAM,IAAI,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,IAAI,IAAA,qBAAS,EAAC,GAAG,EAAE,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC;gBACpF,MAAM,OAAO,GACT,OAAO,IAAI,KAAK,QAAQ;oBACpB,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,oBAAoB,EAAE,eAAe,EAAE,QAAQ,EAAE;oBACnE,CAAC,CAAC,EAAE,GAAG,IAAI,EAAE,GAAG,oBAAoB,EAAE,eAAe,EAAE,QAAQ,EAAE,CAAC;gBAE1E,OAAO,IAAI,iBAAO,CAAC,OAAO,CAAC,CAAC;YAChC,CAAC;QACL,CAAC;QAED,oCAAoC;QACpC,OAAO,IAAI,CAAC;IAChB,CAAC,CAAC;SACD,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAc,CAAC;AACnD,CAAC;AAED,SAAgB,wBAAwB,CAAC,QAAmB,EAAE,QAA6B;IACvF,IAAI,CAAC,QAAQ,EAAE,MAAM,EAAE,CAAC;QACpB,OAAO,QAAQ,CAAC;IACpB,CAAC;IAED,MAAM,QAAQ,GAAc,EAAE,CAAC;IAE/B,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC7B,KAAK,MAAM,gBAAgB,IAAI,QAAQ,EAAE,CAAC;YACtC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,gBAAgB,CAAC;YAE1C,IAAI,CAAC,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,IAAI,IAAA,qBAAS,EAAC,OAAO,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC;gBACpG,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBACvB,kEAAkE;gBAClE,MAAM;YACV,CAAC;QACL,CAAC;IACL,CAAC;IAED,OAAO,QAAQ,CAAC;AACpB,CAAC;AAED;;GAEG;AACH,SAAgB,oBAAoB,CAChC,OAA6C,EAC7C,UAAuG,EAAE;IAEzG,OAAO,OAAO;SACT,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CACT,OAAO,GAAG,KAAK,QAAQ;QACnB,CAAC,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,eAAe,EAAE,OAAO,CAAC,QAAQ,EAAE;QACjD,CAAC,CAAE,EAAE,GAAG,GAAG,EAAE,eAAe,EAAE,OAAO,CAAC,QAAQ,EAAqB,CAC1E;SACA,MAAM,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE;QAChB,IAAI,CAAC;YACD,OAAO,IAAI,SAAG,CAAC,GAAG,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC;QAC9C,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,OAAO,KAAK,CAAC;QACjB,CAAC;IACL,CAAC,CAAC;SACD,GAAG,CAAC,CAAC,cAAc,EAAE,EAAE;QACpB,cAAc,CAAC,GAAG,GAAG,IAAI,SAAG,CAAC,cAAc,CAAC,GAAG,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC;QACvE,cAAc,CAAC,QAAQ,KAAvB,cAAc,CAAC,QAAQ,GAAK,OAAO,CAAC,QAAQ,IAAI,EAAE,EAAC;QAEnD,IAAI,OAAO,OAAO,CAAC,KAAK,KAAK,QAAQ,EAAE,CAAC;YACpC,cAAc,CAAC,QAAQ,GAAG;gBACtB,GAAG,cAAc,CAAC,QAAQ;gBAC1B,KAAK,EAAE,OAAO,CAAC,KAAK;aACvB,CAAC;QACN,CAAC;QAED,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;YACzB,cAAc,CAAC,cAAc,GAAG,IAAI,CAAC;QACzC,CAAC;QAED,OAAO,cAAc,CAAC;IAC1B,CAAC,CAAC,CAAC;AACX,CAAC"}
|
|
@@ -15,8 +15,7 @@ interface Intervals {
|
|
|
15
15
|
}
|
|
16
16
|
export declare abstract class EventManager {
|
|
17
17
|
readonly config: Configuration;
|
|
18
|
-
|
|
19
|
-
protected events: AsyncEventEmitter<Record<PropertyKey, unknown[]> & import("@vladfrangu/async_event_emitter").AsyncEventEmitterPredefinedEvents, Record<PropertyKey, unknown[]> & import("@vladfrangu/async_event_emitter").AsyncEventEmitterPredefinedEvents>;
|
|
18
|
+
protected events: AsyncEventEmitter<{}>;
|
|
20
19
|
protected initialized: boolean;
|
|
21
20
|
protected intervals: Intervals;
|
|
22
21
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"event_manager.d.ts","sourceRoot":"","sources":["../../src/events/event_manager.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAEzD,OAAO,EAAE,iBAAiB,EAAE,MAAM,iCAAiC,CAAC;AAEpE,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEjD,0BAAkB,SAAS;IACvB,aAAa,iBAAiB;IAC9B,WAAW,eAAe;IAC1B,SAAS,cAAc;IACvB,QAAQ,aAAa;IACrB,IAAI,SAAS;CAChB;AAED,MAAM,MAAM,aAAa,GAAG,SAAS,GAAG,YAAY,GAAG,cAAc,GAAG,WAAW,GAAG,UAAU,GAAG,MAAM,CAAC;AAE1G,UAAU,SAAS;IACf,YAAY,CAAC,EAAE,gBAAgB,CAAC;IAChC,UAAU,CAAC,EAAE,gBAAgB,CAAC;CACjC;AAED,8BAAsB,YAAY;IAMlB,QAAQ,CAAC,MAAM;IAL3B,SAAS,CAAC,MAAM,
|
|
1
|
+
{"version":3,"file":"event_manager.d.ts","sourceRoot":"","sources":["../../src/events/event_manager.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAEzD,OAAO,EAAE,iBAAiB,EAAE,MAAM,iCAAiC,CAAC;AAEpE,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEjD,0BAAkB,SAAS;IACvB,aAAa,iBAAiB;IAC9B,WAAW,eAAe;IAC1B,SAAS,cAAc;IACvB,QAAQ,aAAa;IACrB,IAAI,SAAS;CAChB;AAED,MAAM,MAAM,aAAa,GAAG,SAAS,GAAG,YAAY,GAAG,cAAc,GAAG,WAAW,GAAG,UAAU,GAAG,MAAM,CAAC;AAE1G,UAAU,SAAS;IACf,YAAY,CAAC,EAAE,gBAAgB,CAAC;IAChC,UAAU,CAAC,EAAE,gBAAgB,CAAC;CACjC;AAED,8BAAsB,YAAY;IAMlB,QAAQ,CAAC,MAAM;IAL3B,SAAS,CAAC,MAAM,wBAA2B;IAC3C,SAAS,CAAC,WAAW,UAAS;IAC9B,SAAS,CAAC,SAAS,EAAE,SAAS,CAAM;IACpC,SAAS,CAAC,GAAG,2BAAmC;gBAE3B,MAAM,gBAAkC;IAI7D;;;OAGG;IACG,IAAI;IAaV;;;OAGG;IACG,KAAK;IAeX,EAAE,CAAC,KAAK,EAAE,aAAa,EAAE,QAAQ,EAAE,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,GAAG,GAAG,IAAI;IAIjE,GAAG,CAAC,KAAK,EAAE,aAAa,EAAE,QAAQ,CAAC,EAAE,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,GAAG,GAAG,IAAI;IAQnE,IAAI,CAAC,KAAK,EAAE,aAAa,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI;IAIpD,aAAa,IAAI,OAAO;IAIxB;;OAEG;IACH,aAAa,CAAC,KAAK,EAAE,aAAa,GAAG,MAAM;IAI3C;;OAEG;IACH,SAAS,CAAC,KAAK,EAAE,aAAa,GAAG,CAAC,MAAM,OAAO,CAAC,OAAO,CAAC,CAAC,EAAE;IAI3D;;OAEG;IACG,6BAA6B;CAGtC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/core",
|
|
3
|
-
"version": "3.10.6-beta.
|
|
3
|
+
"version": "3.10.6-beta.21",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=16.0.0"
|
|
@@ -59,9 +59,9 @@
|
|
|
59
59
|
"@apify/pseudo_url": "^2.0.30",
|
|
60
60
|
"@apify/timeout": "^0.3.0",
|
|
61
61
|
"@apify/utilities": "^2.7.10",
|
|
62
|
-
"@crawlee/memory-storage": "3.10.6-beta.
|
|
63
|
-
"@crawlee/types": "3.10.6-beta.
|
|
64
|
-
"@crawlee/utils": "3.10.6-beta.
|
|
62
|
+
"@crawlee/memory-storage": "3.10.6-beta.21",
|
|
63
|
+
"@crawlee/types": "3.10.6-beta.21",
|
|
64
|
+
"@crawlee/utils": "3.10.6-beta.21",
|
|
65
65
|
"@sapphire/async-queue": "^1.5.1",
|
|
66
66
|
"@types/tough-cookie": "^4.0.2",
|
|
67
67
|
"@vladfrangu/async_event_emitter": "^2.2.2",
|
|
@@ -85,5 +85,5 @@
|
|
|
85
85
|
}
|
|
86
86
|
}
|
|
87
87
|
},
|
|
88
|
-
"gitHead": "
|
|
88
|
+
"gitHead": "25496c44bbf97dd3f1986915a60fb5bbb754b79f"
|
|
89
89
|
}
|
package/request.d.ts
CHANGED
|
@@ -1,7 +1,3 @@
|
|
|
1
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
2
|
-
/// <reference types="node" />
|
|
3
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
4
|
-
/// <reference types="node" />
|
|
5
1
|
import type { BinaryLike } from 'node:crypto';
|
|
6
2
|
import type { Dictionary } from '@crawlee/types';
|
|
7
3
|
import type { EnqueueLinksOptions } from './enqueue_links/enqueue_links';
|
package/request.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"request.d.ts","sourceRoot":"","sources":["../src/request.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"request.d.ts","sourceRoot":"","sources":["../src/request.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAK9C,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAIjD,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,+BAA+B,CAAC;AAEzE,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AA2BrD,oBAAY,YAAY;IACpB,WAAW,IAAA;IACX,UAAU,IAAA;IACV,SAAS,IAAA;IACT,eAAe,IAAA;IACf,IAAI,IAAA;IACJ,aAAa,IAAA;IACb,KAAK,IAAA;IACL,OAAO,IAAA;CACV;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AACH,qBAAa,OAAO,CAAC,QAAQ,SAAS,UAAU,GAAG,UAAU;IACzD,iBAAiB;IACjB,EAAE,CAAC,EAAE,MAAM,CAAC;IAEZ,oCAAoC;IACpC,GAAG,EAAE,MAAM,CAAC;IAEZ;;;;;;;OAOG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;;OAGG;IACH,SAAS,EAAE,MAAM,CAAC;IAElB,yCAAyC;IACzC,MAAM,EAAE,kBAAkB,CAAC;IAE3B,oDAAoD;IACpD,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB,8FAA8F;IAC9F,OAAO,EAAE,OAAO,CAAC;IAEjB,2FAA2F;IAC3F,UAAU,EAAE,MAAM,CAAC;IAEnB,0DAA0D;IAC1D,aAAa,EAAE,MAAM,EAAE,CAAC;IAExB,wEAAwE;IACxE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAEjC,sEAAsE;IACtE,OAAO,CAAC,SAAS,CAA2B;IAE5C,gDAAgD;IAChD,QAAQ,EAAE,QAAQ,CAAkB;IAEpC;;;OAGG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;OAEG;gBACS,OAAO,EAAE,cAAc,CAAC,QAAQ,CAAC;IAwH7C,yGAAyG;IACzG,IAAI,cAAc,IAAI,OAAO,CAE5B;IAED,yGAAyG;IACzG,IAAI,cAAc,CAAC,KAAK,EAAE,OAAO,EAMhC;IAED,2HAA2H;IAC3H,IAAI,oBAAoB,IAAI,MAAM,CAEjC;IAED,2HAA2H;IAC3H,IAAI,oBAAoB,CAAC,KAAK,EAAE,MAAM,EAMrC;IAED,oDAAoD;IACpD,IAAI,KAAK,IAAI,MAAM,GAAG,SAAS,CAE9B;IAED,oDAAoD;IACpD,IAAI,KAAK,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,EAElC;IAED,8HAA8H;IAC9H,IAAI,UAAU,IAAI,MAAM,GAAG,SAAS,CAEnC;IAED,8HAA8H;IAC9H,IAAI,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,EAMvC;IAED,uDAAuD;IACvD,IAAI,KAAK,IAAI,YAAY,CAExB;IAED,uDAAuD;IACvD,IAAI,KAAK,CAAC,KAAK,EAAE,YAAY,EAM5B;IAED,OAAO,KAAK,eAAe,GAE1B;IAED,OAAO,KAAK,eAAe,QAM1B;IAED;;;;;;;;;;;;OAYG;IACH,gBAAgB,CAAC,cAAc,EAAE,OAAO,EAAE,OAAO,GAAE,uBAA4B,GAAG,IAAI;IAiCtF,SAAS,CAAC,iBAAiB,CAAC,OAAO,EAAE,uBAAuB;IAK5D,SAAS,CAAC,YAAY,CAAC,OAAO,EAAE,UAAU,GAAG,MAAM;IAInD,gBAAgB;IAChB,MAAM,CAAC,gBAAgB,CAAC,EACpB,GAAG,EACH,MAAc,EACd,OAAO,EACP,eAAuB,EACvB,oBAA4B,GAC/B,EAAE,uBAAuB;IAkB1B,gBAAgB;IAChB,MAAM,CAAC,WAAW,CAAC,OAAO,EAAE,UAAU,GAAG,MAAM;CAGlD;AAED;;GAEG;AACH,MAAM,WAAW,cAAc,CAAC,QAAQ,SAAS,UAAU,GAAG,UAAU;IACpE,mEAAmE;IACnE,GAAG,EAAE,MAAM,CAAC;IAEZ;;;;;;;;;;;;;;;;OAgBG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB,qBAAqB;IACrB,MAAM,CAAC,EAAE,kBAAkB,GAAG,SAAS,CAAC,kBAAkB,CAAC,CAAC;IAE5D,oDAAoD;IACpD,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;;;;;;;OAQG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAEjC;;;OAGG;IACH,QAAQ,CAAC,EAAE,QAAQ,CAAC;IAEpB;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf;;;;;;OAMG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;IAE1B;;;;;OAKG;IACH,oBAAoB,CAAC,EAAE,OAAO,CAAC;IAE/B;;;OAGG;IACH,OAAO,CAAC,EAAE,OAAO,CAAC;IAElB;;;;OAIG;IACH,cAAc,CAAC,EAAE,OAAO,CAAC;IAEzB;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB,gBAAgB;IAChB,EAAE,CAAC,EAAE,MAAM,CAAC;IAEZ,gBAAgB;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB,gBAAgB;IAChB,aAAa,CAAC,EAAE,IAAI,CAAC;IAErB,gBAAgB;IAChB,eAAe,CAAC,EAAE,mBAAmB,CAAC,UAAU,CAAC,CAAC;CACrD;AAED,MAAM,WAAW,uBAAuB;IACpC;;;OAGG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;CACvB;AAED,UAAU,uBAAuB;IAC7B,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,kBAAkB,CAAC;IAC3B,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IAC1B,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,oBAAoB,CAAC,EAAE,OAAO,CAAC;CAClC;AAED,MAAM,MAAM,MAAM,GAAG,CAAC,OAAO,CAAC,cAAc,CAAC,GAAG;IAAE,eAAe,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC,GAAG,OAAO,CAAC;AAExG,gBAAgB;AAChB,MAAM,WAAW,cAAc;IAC3B,eAAe,EAAE,MAAM,CAAC;IACxB,KAAK,CAAC,EAAE,MAAM,CAAC;CAClB"}
|
package/serialization.d.ts
CHANGED
|
@@ -1,7 +1,3 @@
|
|
|
1
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
2
|
-
/// <reference types="node" />
|
|
3
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
4
|
-
/// <reference types="node" />
|
|
5
1
|
import { Readable } from 'stream';
|
|
6
2
|
/**
|
|
7
3
|
* Uses Gzip compression to take an array of values, which can be anything
|
package/serialization.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"serialization.d.ts","sourceRoot":"","sources":["../src/serialization.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"serialization.d.ts","sourceRoot":"","sources":["../src/serialization.ts"],"names":[],"mappings":"AAEA,OAAO,EAA8B,QAAQ,EAAY,MAAM,QAAQ,CAAC;AAqDxE;;;;;;;;GAQG;AACH,wBAAsB,cAAc,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAMlE;AAED;;;;;;;;GAQG;AACH,wBAAsB,gBAAgB,CAAC,CAAC,SAAS,MAAM,GAAG,MAAM,EAAE,cAAc,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC,EAAE,CAAC,CAMtG;AAED;;;;;;;;GAQG;AACH,wBAAgB,iBAAiB,CAAC,cAAc,EAAE,MAAM,GAAG,QAAQ,CAclE"}
|
package/serialization.js
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.
|
|
3
|
+
exports.serializeArray = serializeArray;
|
|
4
|
+
exports.deserializeArray = deserializeArray;
|
|
5
|
+
exports.createDeserialize = createDeserialize;
|
|
4
6
|
const tslib_1 = require("tslib");
|
|
5
7
|
const node_util_1 = tslib_1.__importDefault(require("node:util"));
|
|
6
8
|
const node_zlib_1 = tslib_1.__importDefault(require("node:zlib"));
|
|
@@ -81,7 +83,6 @@ async function serializeArray(data) {
|
|
|
81
83
|
await pipeline(new ArrayToJson(data), node_zlib_1.default.createGzip(), collector);
|
|
82
84
|
return Buffer.concat(chunks);
|
|
83
85
|
}
|
|
84
|
-
exports.serializeArray = serializeArray;
|
|
85
86
|
/**
|
|
86
87
|
* Decompresses a Buffer previously created with compressData (technically,
|
|
87
88
|
* any JSON that is an Array) and collects it into an Array of values
|
|
@@ -97,7 +98,6 @@ async function deserializeArray(compressedData) {
|
|
|
97
98
|
await pipeline(stream_1.Readable.from([compressedData]), node_zlib_1.default.createGunzip(), StreamArray_1.default.withParser(), collector);
|
|
98
99
|
return chunks;
|
|
99
100
|
}
|
|
100
|
-
exports.deserializeArray = deserializeArray;
|
|
101
101
|
/**
|
|
102
102
|
* Creates a stream that decompresses a Buffer previously created with
|
|
103
103
|
* compressData (technically, any JSON that is an Array) and collects it
|
|
@@ -116,7 +116,6 @@ function createDeserialize(compressedData) {
|
|
|
116
116
|
(err) => destination.emit(err));
|
|
117
117
|
return destination;
|
|
118
118
|
}
|
|
119
|
-
exports.createDeserialize = createDeserialize;
|
|
120
119
|
function createChunkCollector(options = {}) {
|
|
121
120
|
const { fromValuesStream = false } = options;
|
|
122
121
|
const chunks = [];
|
package/serialization.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"serialization.js","sourceRoot":"","sources":["../src/serialization.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"serialization.js","sourceRoot":"","sources":["../src/serialization.ts"],"names":[],"mappings":";;AAgEA,wCAMC;AAWD,4CAMC;AAWD,8CAcC;;AAhHD,kEAA6B;AAC7B,kEAA6B;AAC7B,mCAAwE;AAExE,oDAAoB;AAEpB,4FAA4D;AAE5D,MAAM,QAAQ,GAAG,mBAAI,CAAC,SAAS,CAAC,iBAAc,CAAC,CAAC;AAEhD;;;;;GAKG;AACH,MAAM,WAAe,SAAQ,iBAAQ;IAIjC,YACY,IAAS,EACjB,UAAkC,EAAE;QAEpC,KAAK,CAAC;YACF,GAAG,OAAO;YACV,WAAW,EAAE,IAAI;YACjB,SAAS,EAAE,IAAI;SAClB,CAAC,CAAC;QAPH;;;;mBAAQ,IAAI;WAAK;QAJb;;;;mBAAS,CAAC;WAAC;QACF;;;;;WAAkB;QAW/B,MAAM,EAAE,SAAS,GAAG,KAAK,EAAE,GAAG,OAAO,CAAC;QACtC,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACnB,CAAC;IAEQ,KAAK;QACV,IAAI,CAAC;YACD,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC;YACzE,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;gBACf,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;gBACnC,uCAAuC;gBACvC,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBACtD,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC;oBAAE,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;gBAC5C,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;gBAC9B,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,SAAS,CAAC;YAClC,CAAC;iBAAM,CAAC;gBACJ,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACf,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACpB,CAAC;QACL,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;QAC5B,CAAC;IACL,CAAC;CACJ;AAED;;;;;;;;GAQG;AACI,KAAK,UAAU,cAAc,CAAI,IAAS;IAC7C,IAAA,YAAE,EAAC,IAAI,EAAE,YAAE,CAAC,KAAK,CAAC,CAAC;IACnB,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,oBAAoB,EAAE,CAAC;IACrD,MAAM,QAAQ,CAAC,IAAI,WAAW,CAAC,IAAI,CAAC,EAAE,mBAAI,CAAC,UAAU,EAAE,EAAE,SAAS,CAAC,CAAC;IAEpE,OAAO,MAAM,CAAC,MAAM,CAAC,MAAkB,CAAC,CAAC;AAC7C,CAAC;AAED;;;;;;;;GAQG;AACI,KAAK,UAAU,gBAAgB,CAA4B,cAAsB;IACpF,IAAA,YAAE,EAAC,cAAc,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC;IAC9B,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,oBAAoB,CAAI,EAAE,gBAAgB,EAAE,IAAI,EAAE,CAAC,CAAC;IAClF,MAAM,QAAQ,CAAC,iBAAQ,CAAC,IAAI,CAAC,CAAC,cAAc,CAAC,CAAC,EAAE,mBAAI,CAAC,YAAY,EAAE,EAAE,qBAAW,CAAC,UAAU,EAAE,EAAE,SAAS,CAAC,CAAC;IAE1G,OAAO,MAAa,CAAC;AACzB,CAAC;AAED;;;;;;;;GAQG;AACH,SAAgB,iBAAiB,CAAC,cAAsB;IACpD,IAAA,YAAE,EAAC,cAAc,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC;IAC9B,MAAM,WAAW,GAAG,qBAAW,CAAC,UAAU,EAAE,CAAC;IAC7C,MAAM,WAAW,GAAG,UAAU,CAAC,WAAW,CAAC,CAAC;IAE5C,IAAA,iBAAc,EACV,iBAAQ,CAAC,IAAI,CAAC,CAAC,cAAc,CAAC,CAAC,EAC/B,mBAAI,CAAC,YAAY,EAAE,EACnB,WAAW;IACX,wHAAwH;IACxH,CAAC,GAAG,EAAE,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CACjC,CAAC;IAEF,OAAO,WAAW,CAAC;AACvB,CAAC;AAED,SAAS,oBAAoB,CACzB,UAA0C,EAAE;IAE5C,MAAM,EAAE,gBAAgB,GAAG,KAAK,EAAE,GAAG,OAAO,CAAC;IAC7C,MAAM,MAAM,GAAQ,EAAE,CAAC;IACvB,MAAM,SAAS,GAAG,IAAI,iBAAQ,CAAC;QAC3B,aAAa,EAAE,KAAK;QACpB,UAAU,EAAE,gBAAgB;QAC5B,KAAK,CAAC,KAAK,EAAE,IAAI,EAAE,QAAQ;YACvB,MAAM,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;YACpD,QAAQ,EAAE,CAAC;QACf,CAAC;QACD,MAAM,CAAC,YAAY,EAAE,QAAQ;YACzB,MAAM,OAAO,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE;gBAC3C,OAAO,gBAAgB,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC;YAClD,CAAC,CAAC,CAAC;YACH,YAAY,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,CAAC;YAC9B,QAAQ,EAAE,CAAC;QACf,CAAC;KACJ,CAAC,CAAC;IAEH,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC;AACjC,CAAC;AAED,SAAS,UAAU,CAAC,WAAkB;IAClC,MAAM,QAAQ,GAAG,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IACpD,WAAW,CAAC,IAAI,GAAG,CAAC,GAAG,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC;IACvD,OAAO,WAAW,CAAC;AACvB,CAAC"}
|
|
@@ -1,5 +1,3 @@
|
|
|
1
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
2
|
-
/// <reference types="node" />
|
|
3
1
|
import type { IncomingMessage } from 'node:http';
|
|
4
2
|
import type { Log } from '@apify/log';
|
|
5
3
|
import type { BrowserLikeResponse, Cookie as CookieObject, Dictionary } from '@crawlee/types';
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session.d.ts","sourceRoot":"","sources":["../../src/session_pool/session.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"session.d.ts","sourceRoot":"","sources":["../../src/session_pool/session.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAEjD,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AAEtC,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,IAAI,YAAY,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAE9F,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAC3C,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAWzC;;GAEG;AACH,MAAM,WAAW,YAAY;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,SAAS,CAAC,UAAU,CAAC;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,cAAc;IAC3B,wFAAwF;IACxF,EAAE,CAAC,EAAE,MAAM,CAAC;IAEZ;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB,+EAA+E;IAC/E,QAAQ,CAAC,EAAE,UAAU,CAAC;IAEtB;;;;;;OAMG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IAEvB;;;;;OAKG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAE7B,wBAAwB;IACxB,SAAS,CAAC,EAAE,IAAI,CAAC;IAEjB,0BAA0B;IAC1B,SAAS,CAAC,EAAE,IAAI,CAAC;IAEjB;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB;;;;OAIG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IAEvB,2FAA2F;IAC3F,WAAW,CAAC,EAAE,OAAO,gBAAgB,EAAE,WAAW,CAAC;IAEnD,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,SAAS,CAAC;CACzB;AAED;;;;;GAKG;AACH,qBAAa,OAAO;IAChB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,UAAU,CAAS;IAC3B,QAAQ,EAAE,UAAU,CAAC;IACrB,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,mBAAmB,CAAS;IACpC,OAAO,CAAC,SAAS,CAAO;IACxB,OAAO,CAAC,SAAS,CAAO;IACxB,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,WAAW,CAAuC;IAC1D,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,SAAS,CAAY;IAC7B,OAAO,CAAC,GAAG,CAAM;IAEjB;;OAEG;gBACS,OAAO,EAAE,cAAc;IAuDnC;;;OAGG;IACH,SAAS,IAAI,OAAO;IAIpB;;;;OAIG;IACH,SAAS,IAAI,OAAO;IAIpB;;;OAGG;IACH,sBAAsB,IAAI,OAAO;IAIjC;;;OAGG;IACH,QAAQ,IAAI,OAAO;IAInB;;;OAGG;IACH,QAAQ;IAUR;;;OAGG;IACH,QAAQ,IAAI,YAAY;IAexB;;;;;;OAMG;IACH,MAAM;IASN;;;OAGG;IACH,OAAO;IAOP;;;;;;;OAOG;IACH,0BAA0B,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO;IAEvD;;;;;;;;;;;;OAYG;IACH,0BAA0B,CAAC,UAAU,EAAE,MAAM,EAAE,4BAA4B,CAAC,EAAE,MAAM,EAAE,GAAG,OAAO;IAahG;;;;;;OAMG;IACH,sBAAsB,CAClB,QAAQ,EAAE,eAAe,GAAG,mBAAmB,GAAG;QAAE,OAAO,EAAE,UAAU,CAAC,MAAM,GAAG,MAAM,EAAE,CAAC,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE;IAY7G;;;;;;;;;;;;OAYG;IACH,UAAU,CAAC,OAAO,EAAE,YAAY,EAAE,EAAE,GAAG,EAAE,MAAM;IAK/C;;;OAGG;IACH,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG,YAAY,EAAE;IAKvC;;;;;OAKG;IACH,eAAe,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM;IAIpC;;OAEG;IACH,SAAS,CAAC,SAAS,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,IAAI;IAI/C;;OAEG;IACH,SAAS,CAAC,WAAW,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,MAAM,GAAG,IAAI;IAkB3D;;OAEG;IACH,SAAS,CAAC,gBAAgB,IAAI,IAAI;CAKrC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session_pool.d.ts","sourceRoot":"","sources":["../../src/session_pool/session_pool.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"session_pool.d.ts","sourceRoot":"","sources":["../../src/session_pool/session_pool.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAE3C,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AAMtC,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAChD,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AACjD,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAC;AACjE,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AAG5D,OAAO,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AAE5D;;GAEG;AACH,MAAM,WAAW,aAAa;IAC1B;;;OAGG;IACH,CAAC,WAAW,EAAE,WAAW,EAAE,OAAO,CAAC,EAAE;QAAE,cAAc,CAAC,EAAE,cAAc,CAAA;KAAE,GAAG,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;CACzG;AAED,MAAM,WAAW,kBAAkB;IAC/B;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB,kEAAkE;IAClE,cAAc,CAAC,EAAE,cAAc,CAAC;IAEhC,6EAA6E;IAC7E,2BAA2B,CAAC,EAAE,MAAM,CAAC;IAErC;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IAEzB;;;;OAIG;IACH,qBAAqB,CAAC,EAAE,aAAa,CAAC;IAEtC;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,MAAM,EAAE,CAAC;IAE9B,gBAAgB;IAChB,GAAG,CAAC,EAAE,GAAG,CAAC;IAEV;;OAEG;IACH,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;CAC3C;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgEG;AACH,qBAAa,WAAY,SAAQ,YAAY;IAuBrC,QAAQ,CAAC,MAAM;IAtBnB,SAAS,CAAC,GAAG,EAAE,GAAG,CAAC;IACnB,SAAS,CAAC,WAAW,EAAE,MAAM,CAAC;IAC9B,SAAS,CAAC,qBAAqB,EAAE,aAAa,CAAC;IAC/C,SAAS,CAAC,aAAa,EAAG,aAAa,CAAC;IACxC,SAAS,CAAC,QAAQ,EAAE,OAAO,EAAE,CAAM;IACnC,SAAS,CAAC,UAAU,uBAA8B;IAClD,SAAS,CAAC,cAAc,EAAE,cAAc,CAAC;IACzC,SAAS,CAAC,2BAA2B,CAAC,EAAE,MAAM,CAAC;IAC/C,SAAS,CAAC,eAAe,EAAE,MAAM,CAAC;IAClC,SAAS,CAAC,SAAS,EAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAC1C,SAAS,CAAC,MAAM,EAAE,YAAY,CAAC;IAC/B,SAAS,CAAC,QAAQ,CAAC,kBAAkB,EAAE,MAAM,EAAE,CAAC;IAChD,SAAS,CAAC,kBAAkB,EAAE,kBAAkB,CAAC;IACjD,SAAS,CAAC,aAAa,UAAS;IAEhC,OAAO,CAAC,KAAK,CAAoB;IAEjC;;OAEG;gBAEC,OAAO,GAAE,kBAAuB,EACvB,MAAM,gBAAkC;IAsDrD;;OAEG;IACH,IAAI,mBAAmB,IAAI,MAAM,CAEhC;IAED;;OAEG;IACH,IAAI,oBAAoB,IAAI,MAAM,CAEjC;IAED;;;OAGG;IACG,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IA0BjC;;;;;OAKG;IACG,UAAU,CAAC,OAAO,GAAE,OAAO,GAAG,cAAmB,GAAG,OAAO,CAAC,IAAI,CAAC;IAqBvE;;;;;OAKG;IACG,UAAU,IAAI,OAAO,CAAC,OAAO,CAAC;IAEpC;;OAEG;IACG,UAAU,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAqCrD;;OAEG;IACG,UAAU,CAAC,OAAO,CAAC,EAAE,kBAAkB;IAQ7C;;;OAGG;IACH,QAAQ;;;;;IAQR;;;;OAIG;IACG,YAAY,CAAC,OAAO,CAAC,EAAE,kBAAkB,GAAG,OAAO,CAAC,IAAI,CAAC;IAY/D;;;OAGG;IACG,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAK/B;;OAEG;IACH,SAAS,CAAC,sBAAsB;IAIhC;;OAEG;IACH,SAAS,CAAC,sBAAsB;IAWhC;;;OAGG;IACH,SAAS,CAAC,WAAW,CAAC,UAAU,EAAE,OAAO;IAKzC;;OAEG;IACH,SAAS,CAAC,eAAe,IAAI,MAAM;IAInC;;;;;;OAMG;IACH,SAAS,CAAC,6BAA6B,CACnC,WAAW,EAAE,WAAW,EACxB,OAAO,GAAE;QAAE,cAAc,CAAC,EAAE,cAAc,CAAA;KAAO,GAClD,OAAO;IAUV;;;OAGG;cACa,cAAc,IAAI,OAAO,CAAC,OAAO,CAAC;IAQlD;;OAEG;IACH,SAAS,CAAC,mBAAmB,IAAI,OAAO;IAIxC;;;OAGG;IACH,SAAS,CAAC,YAAY,IAAI,OAAO;IAIjC;;;OAGG;cACa,qBAAqB,IAAI,OAAO,CAAC,IAAI,CAAC;IAyBtD;;;;;OAKG;WACU,IAAI,CAAC,OAAO,CAAC,EAAE,kBAAkB,EAAE,MAAM,CAAC,EAAE,aAAa,GAAG,OAAO,CAAC,WAAW,CAAC;CAKhG"}
|