crawlee 3.15.4-beta.8 → 3.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -4
- package/index.mjs +3 -0
- package/package.json +18 -14
package/README.md
CHANGED
|
@@ -89,7 +89,7 @@ By default, Crawlee stores data to `./storage` in the current working directory.
|
|
|
89
89
|
We provide automated beta builds for every merged code change in Crawlee. You can find them in the npm [list of releases](https://www.npmjs.com/package/crawlee?activeTab=versions). If you want to test new features or bug fixes before we release them, feel free to install a beta build like this:
|
|
90
90
|
|
|
91
91
|
```bash
|
|
92
|
-
npm install crawlee@
|
|
92
|
+
npm install crawlee@next
|
|
93
93
|
```
|
|
94
94
|
|
|
95
95
|
If you also use the [Apify SDK](https://github.com/apify/apify-sdk-js), you need to specify dependency overrides in your `package.json` file so that you don't end up with multiple versions of Crawlee installed:
|
|
@@ -98,9 +98,9 @@ If you also use the [Apify SDK](https://github.com/apify/apify-sdk-js), you need
|
|
|
98
98
|
{
|
|
99
99
|
"overrides": {
|
|
100
100
|
"apify": {
|
|
101
|
-
"@crawlee/core": "
|
|
102
|
-
"@crawlee/types": "
|
|
103
|
-
"@crawlee/utils": "
|
|
101
|
+
"@crawlee/core": "$crawlee",
|
|
102
|
+
"@crawlee/types": "$crawlee",
|
|
103
|
+
"@crawlee/utils": "$crawlee"
|
|
104
104
|
}
|
|
105
105
|
}
|
|
106
106
|
}
|
package/index.mjs
CHANGED
|
@@ -99,6 +99,7 @@ export const StorageManager = mod.StorageManager;
|
|
|
99
99
|
export const SystemStatus = mod.SystemStatus;
|
|
100
100
|
export const URL_NO_COMMAS_REGEX = mod.URL_NO_COMMAS_REGEX;
|
|
101
101
|
export const URL_WITH_COMMAS_REGEX = mod.URL_WITH_COMMAS_REGEX;
|
|
102
|
+
export const anonymizeProxySugar = mod.anonymizeProxySugar;
|
|
102
103
|
export const applySearchParams = mod.applySearchParams;
|
|
103
104
|
export const asyncifyIterable = mod.asyncifyIterable;
|
|
104
105
|
export const browserCrawlerEnqueueLinks = mod.browserCrawlerEnqueueLinks;
|
|
@@ -127,6 +128,7 @@ export const createRequestDebugInfo = mod.createRequestDebugInfo;
|
|
|
127
128
|
export const createRequestOptions = mod.createRequestOptions;
|
|
128
129
|
export const createRequests = mod.createRequests;
|
|
129
130
|
export const deserializeArray = mod.deserializeArray;
|
|
131
|
+
export const discoverValidSitemaps = mod.discoverValidSitemaps;
|
|
130
132
|
export const domCrawlerEnqueueLinks = mod.domCrawlerEnqueueLinks;
|
|
131
133
|
export const downloadListOfUrls = mod.downloadListOfUrls;
|
|
132
134
|
export const enqueueLinks = mod.enqueueLinks;
|
|
@@ -158,6 +160,7 @@ export const launchPuppeteer = mod.launchPuppeteer;
|
|
|
158
160
|
export const linkedomCrawlerEnqueueLinks = mod.linkedomCrawlerEnqueueLinks;
|
|
159
161
|
export const log = mod.log;
|
|
160
162
|
export const maybeStringify = mod.maybeStringify;
|
|
163
|
+
export const mergeAsyncIterables = mod.mergeAsyncIterables;
|
|
161
164
|
export const mergeCookies = mod.mergeCookies;
|
|
162
165
|
export const parseOpenGraph = mod.parseOpenGraph;
|
|
163
166
|
export const parseSitemap = mod.parseSitemap;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "crawlee",
|
|
3
|
-
"version": "3.
|
|
3
|
+
"version": "3.16.0",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=16.0.0"
|
|
@@ -54,26 +54,30 @@
|
|
|
54
54
|
"access": "public"
|
|
55
55
|
},
|
|
56
56
|
"dependencies": {
|
|
57
|
-
"@crawlee/basic": "3.
|
|
58
|
-
"@crawlee/browser": "3.
|
|
59
|
-
"@crawlee/browser-pool": "3.
|
|
60
|
-
"@crawlee/cheerio": "3.
|
|
61
|
-
"@crawlee/cli": "3.
|
|
62
|
-
"@crawlee/core": "3.
|
|
63
|
-
"@crawlee/http": "3.
|
|
64
|
-
"@crawlee/jsdom": "3.
|
|
65
|
-
"@crawlee/linkedom": "3.
|
|
66
|
-
"@crawlee/playwright": "3.
|
|
67
|
-
"@crawlee/puppeteer": "3.
|
|
68
|
-
"@crawlee/utils": "3.
|
|
57
|
+
"@crawlee/basic": "3.16.0",
|
|
58
|
+
"@crawlee/browser": "3.16.0",
|
|
59
|
+
"@crawlee/browser-pool": "3.16.0",
|
|
60
|
+
"@crawlee/cheerio": "3.16.0",
|
|
61
|
+
"@crawlee/cli": "3.16.0",
|
|
62
|
+
"@crawlee/core": "3.16.0",
|
|
63
|
+
"@crawlee/http": "3.16.0",
|
|
64
|
+
"@crawlee/jsdom": "3.16.0",
|
|
65
|
+
"@crawlee/linkedom": "3.16.0",
|
|
66
|
+
"@crawlee/playwright": "3.16.0",
|
|
67
|
+
"@crawlee/puppeteer": "3.16.0",
|
|
68
|
+
"@crawlee/utils": "3.16.0",
|
|
69
69
|
"import-local": "^3.1.0",
|
|
70
70
|
"tslib": "^2.4.0"
|
|
71
71
|
},
|
|
72
72
|
"peerDependencies": {
|
|
73
|
+
"idcac-playwright": "*",
|
|
73
74
|
"playwright": "*",
|
|
74
75
|
"puppeteer": "*"
|
|
75
76
|
},
|
|
76
77
|
"peerDependenciesMeta": {
|
|
78
|
+
"idcac-playwright": {
|
|
79
|
+
"optional": true
|
|
80
|
+
},
|
|
77
81
|
"playwright": {
|
|
78
82
|
"optional": true
|
|
79
83
|
},
|
|
@@ -88,5 +92,5 @@
|
|
|
88
92
|
}
|
|
89
93
|
}
|
|
90
94
|
},
|
|
91
|
-
"gitHead": "
|
|
95
|
+
"gitHead": "e6451749f838744d539c81bf9d969c1cfcc9e86b"
|
|
92
96
|
}
|