@crawlee/basic 4.0.0-beta.196 → 4.0.0-beta.197
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -1669,13 +1669,13 @@ export class BasicCrawler {
|
|
|
1669
1669
|
// oxlint-disable-next-line crawlee/prefer-private-fields -- patched by @crawlee/otel
|
|
1670
1670
|
async requestFunctionErrorHandler(error, crawlingContext, request, source) {
|
|
1671
1671
|
if (error instanceof RequestThrottledError) {
|
|
1672
|
-
// The domain told us to come back later, so the request was never really attempted. Put it back
|
|
1673
|
-
// without recording a failure - it costs neither a retry nor session reputation.
|
|
1672
|
+
// The domain told us to come back later, so the request was never really attempted. Put it back where
|
|
1673
|
+
// it was, without recording a failure - it costs neither a retry nor session reputation.
|
|
1674
1674
|
this.log.debug(`Deferring request because its domain is rate-limiting us. ${error.message}`, {
|
|
1675
1675
|
id: request.id,
|
|
1676
1676
|
url: request.url,
|
|
1677
1677
|
});
|
|
1678
|
-
await source.reclaimRequest(request, { forefront:
|
|
1678
|
+
await source.reclaimRequest(request, { forefront: true });
|
|
1679
1679
|
return;
|
|
1680
1680
|
}
|
|
1681
1681
|
request.pushErrorMessage(error);
|
|
@@ -1701,7 +1701,7 @@ export class BasicCrawler {
|
|
|
1701
1701
|
url,
|
|
1702
1702
|
retryCount,
|
|
1703
1703
|
});
|
|
1704
|
-
await source.reclaimRequest(request
|
|
1704
|
+
await source.reclaimRequest(request);
|
|
1705
1705
|
return;
|
|
1706
1706
|
}
|
|
1707
1707
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/basic",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.197",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.0.0"
|
|
@@ -42,10 +42,10 @@
|
|
|
42
42
|
"@apify/datastructures": "^3.0.1",
|
|
43
43
|
"@apify/timeout": "^1.0.1",
|
|
44
44
|
"@apify/utilities": "^3.0.1",
|
|
45
|
-
"@crawlee/core": "4.0.0-beta.
|
|
46
|
-
"@crawlee/http-client": "4.0.0-beta.
|
|
47
|
-
"@crawlee/types": "4.0.0-beta.
|
|
48
|
-
"@crawlee/utils": "4.0.0-beta.
|
|
45
|
+
"@crawlee/core": "4.0.0-beta.197",
|
|
46
|
+
"@crawlee/http-client": "4.0.0-beta.197",
|
|
47
|
+
"@crawlee/types": "4.0.0-beta.197",
|
|
48
|
+
"@crawlee/utils": "4.0.0-beta.197",
|
|
49
49
|
"@sapphire/async-queue": "^1.5.5",
|
|
50
50
|
"@standard-schema/spec": "^1.0.0",
|
|
51
51
|
"csv-stringify": "^6.5.2",
|
|
@@ -57,7 +57,7 @@
|
|
|
57
57
|
"zod": "^4.5.4"
|
|
58
58
|
},
|
|
59
59
|
"optionalDependencies": {
|
|
60
|
-
"@crawlee/impit-client": "4.0.0-beta.
|
|
60
|
+
"@crawlee/impit-client": "4.0.0-beta.197"
|
|
61
61
|
},
|
|
62
62
|
"lerna": {
|
|
63
63
|
"command": {
|
|
@@ -66,5 +66,5 @@
|
|
|
66
66
|
}
|
|
67
67
|
}
|
|
68
68
|
},
|
|
69
|
-
"gitHead": "
|
|
69
|
+
"gitHead": "6dfbf74898b378341626fbe0c5ff7655b57ac305"
|
|
70
70
|
}
|