crawlee 3.17.1-beta.9 → 3.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.mjs +6 -0
- package/package.json +15 -14
package/index.mjs
CHANGED
|
@@ -83,6 +83,7 @@ export const RequestQueue = mod.RequestQueue;
|
|
|
83
83
|
export const RequestQueueV1 = mod.RequestQueueV1;
|
|
84
84
|
export const RequestQueueV2 = mod.RequestQueueV2;
|
|
85
85
|
export const RequestState = mod.RequestState;
|
|
86
|
+
export const RequestValidationError = mod.RequestValidationError;
|
|
86
87
|
export const RetryRequestError = mod.RetryRequestError;
|
|
87
88
|
export const RobotsFile = mod.RobotsFile;
|
|
88
89
|
export const RobotsTxtFile = mod.RobotsTxtFile;
|
|
@@ -99,6 +100,7 @@ export const Snapshotter = mod.Snapshotter;
|
|
|
99
100
|
export const Statistics = mod.Statistics;
|
|
100
101
|
export const StorageManager = mod.StorageManager;
|
|
101
102
|
export const SystemStatus = mod.SystemStatus;
|
|
103
|
+
export const UNSUPPORTED_SCHEME_MESSAGE = mod.UNSUPPORTED_SCHEME_MESSAGE;
|
|
102
104
|
export const URL_NO_COMMAS_REGEX = mod.URL_NO_COMMAS_REGEX;
|
|
103
105
|
export const URL_WITH_COMMAS_REGEX = mod.URL_WITH_COMMAS_REGEX;
|
|
104
106
|
export const anonymizeProxySugar = mod.anonymizeProxySugar;
|
|
@@ -132,6 +134,7 @@ export const createPuppeteerRouter = mod.createPuppeteerRouter;
|
|
|
132
134
|
export const createRequestDebugInfo = mod.createRequestDebugInfo;
|
|
133
135
|
export const createRequestOptions = mod.createRequestOptions;
|
|
134
136
|
export const createRequests = mod.createRequests;
|
|
137
|
+
export const defaultRoute = mod.defaultRoute;
|
|
135
138
|
export const deserializeArray = mod.deserializeArray;
|
|
136
139
|
export const discoverValidSitemaps = mod.discoverValidSitemaps;
|
|
137
140
|
export const domCrawlerEnqueueLinks = mod.domCrawlerEnqueueLinks;
|
|
@@ -144,6 +147,7 @@ export const extractUrls = mod.extractUrls;
|
|
|
144
147
|
export const extractUrlsFromCheerio = mod.extractUrlsFromCheerio;
|
|
145
148
|
export const extractUrlsFromPage = mod.extractUrlsFromPage;
|
|
146
149
|
export const filterRequestsByPatterns = mod.filterRequestsByPatterns;
|
|
150
|
+
export const filterUrl = mod.filterUrl;
|
|
147
151
|
export const getCgroupsVersion = mod.getCgroupsVersion;
|
|
148
152
|
export const getCookiesFromResponse = mod.getCookiesFromResponse;
|
|
149
153
|
export const getCurrentCpuTicksV2 = mod.getCurrentCpuTicksV2;
|
|
@@ -165,6 +169,7 @@ export const launchPlaywright = mod.launchPlaywright;
|
|
|
165
169
|
export const launchPuppeteer = mod.launchPuppeteer;
|
|
166
170
|
export const linkedomCrawlerEnqueueLinks = mod.linkedomCrawlerEnqueueLinks;
|
|
167
171
|
export const log = mod.log;
|
|
172
|
+
export const matchesEnqueueStrategy = mod.matchesEnqueueStrategy;
|
|
168
173
|
export const maybeStringify = mod.maybeStringify;
|
|
169
174
|
export const mergeAsyncIterables = mod.mergeAsyncIterables;
|
|
170
175
|
export const mergeCookies = mod.mergeCookies;
|
|
@@ -189,6 +194,7 @@ export const updateEnqueueLinksPatternCache = mod.updateEnqueueLinksPatternCache
|
|
|
189
194
|
export const useState = mod.useState;
|
|
190
195
|
export const utils = mod.utils;
|
|
191
196
|
export const validateGlobPattern = mod.validateGlobPattern;
|
|
197
|
+
export const validateUserData = mod.validateUserData;
|
|
192
198
|
export const validators = mod.validators;
|
|
193
199
|
export const weightedAvg = mod.weightedAvg;
|
|
194
200
|
export const withCheckedStorageAccess = mod.withCheckedStorageAccess;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "crawlee",
|
|
3
|
-
"version": "3.
|
|
3
|
+
"version": "3.18.0",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=16.0.0"
|
|
@@ -54,18 +54,19 @@
|
|
|
54
54
|
"access": "public"
|
|
55
55
|
},
|
|
56
56
|
"dependencies": {
|
|
57
|
-
"@crawlee/basic": "3.
|
|
58
|
-
"@crawlee/browser": "3.
|
|
59
|
-
"@crawlee/browser-pool": "3.
|
|
60
|
-
"@crawlee/cheerio": "3.
|
|
61
|
-
"@crawlee/cli": "3.
|
|
62
|
-
"@crawlee/core": "3.
|
|
63
|
-
"@crawlee/http": "3.
|
|
64
|
-
"@crawlee/jsdom": "3.
|
|
65
|
-
"@crawlee/linkedom": "3.
|
|
66
|
-
"@crawlee/
|
|
67
|
-
"@crawlee/
|
|
68
|
-
"@crawlee/
|
|
57
|
+
"@crawlee/basic": "3.18.0",
|
|
58
|
+
"@crawlee/browser": "3.18.0",
|
|
59
|
+
"@crawlee/browser-pool": "3.18.0",
|
|
60
|
+
"@crawlee/cheerio": "3.18.0",
|
|
61
|
+
"@crawlee/cli": "3.18.0",
|
|
62
|
+
"@crawlee/core": "3.18.0",
|
|
63
|
+
"@crawlee/http": "3.18.0",
|
|
64
|
+
"@crawlee/jsdom": "3.18.0",
|
|
65
|
+
"@crawlee/linkedom": "3.18.0",
|
|
66
|
+
"@crawlee/memory-storage": "3.18.0",
|
|
67
|
+
"@crawlee/playwright": "3.18.0",
|
|
68
|
+
"@crawlee/puppeteer": "3.18.0",
|
|
69
|
+
"@crawlee/utils": "3.18.0",
|
|
69
70
|
"import-local": "^3.1.0",
|
|
70
71
|
"tslib": "^2.4.0"
|
|
71
72
|
},
|
|
@@ -92,5 +93,5 @@
|
|
|
92
93
|
}
|
|
93
94
|
}
|
|
94
95
|
},
|
|
95
|
-
"gitHead": "
|
|
96
|
+
"gitHead": "49c115e1ce3b3fbf2bef61f16bffeadfdcf5bd19"
|
|
96
97
|
}
|