@crawlee/http 4.0.0-beta.124 → 4.0.0-beta.126
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -20,7 +20,7 @@ export declare const HTTP_OPTIMIZED_CONCURRENCY_SYSTEM_OPTIONS: ConcurrencySyste
|
|
|
20
20
|
export type HttpErrorHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
|
|
21
21
|
JSONData extends JsonValue = any, // with default to Dictionary we cant use a typed router in untyped crawler
|
|
22
22
|
ContextExtension = Dictionary<never>> = ErrorHandler<CrawlingContext, HttpCrawlingContext<UserData, JSONData> & ContextExtension>;
|
|
23
|
-
export interface HttpCrawlerOptions<Context extends InternalHttpCrawlingContext = InternalHttpCrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']
|
|
23
|
+
export interface HttpCrawlerOptions<Context extends InternalHttpCrawlingContext = InternalHttpCrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>, StatisticStateExtension extends object = {}> extends BasicCrawlerOptions<Context, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
|
|
24
24
|
/**
|
|
25
25
|
* Timeout for the whole navigation phase, given in seconds. A single window shared by the
|
|
26
26
|
* `preNavigationHooks`, the navigation (the HTTP request to the resource), and the `postNavigationHooks` -
|
|
@@ -247,7 +247,7 @@ JSONData extends JsonValue = any> = RequestHandler<HttpCrawlingContext<UserData,
|
|
|
247
247
|
* ```
|
|
248
248
|
* @category Crawlers
|
|
249
249
|
*/
|
|
250
|
-
export declare class HttpCrawler<Context extends InternalHttpCrawlingContext<any, any> = InternalHttpCrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']
|
|
250
|
+
export declare class HttpCrawler<Context extends InternalHttpCrawlingContext<any, any> = InternalHttpCrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>, StatisticStateExtension extends object = {}> extends BasicCrawler<Context, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
|
|
251
251
|
#private;
|
|
252
252
|
protected static optionsShape: {
|
|
253
253
|
navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
@@ -362,7 +362,7 @@ export declare class HttpCrawler<Context extends InternalHttpCrawlingContext<any
|
|
|
362
362
|
/**
|
|
363
363
|
* All `HttpCrawlerOptions` parameters are passed via an options object.
|
|
364
364
|
*/
|
|
365
|
-
constructor(options?: HttpCrawlerOptions<Context, ContextExtension, ExtendedContext> & RequireContextPipeline<InternalHttpCrawlingContext, Context>);
|
|
365
|
+
constructor(options?: HttpCrawlerOptions<Context, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> & RequireContextPipeline<InternalHttpCrawlingContext, Context>);
|
|
366
366
|
protected getNavigationTimeoutMillis(): number;
|
|
367
367
|
/**
|
|
368
368
|
* Folds {@link HTTP_OPTIMIZED_CONCURRENCY_SYSTEM_OPTIONS} into the default system, keeping the user's
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/http",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.126",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.0.0"
|
|
@@ -49,11 +49,11 @@
|
|
|
49
49
|
"dependencies": {
|
|
50
50
|
"@apify/timeout": "^0.4.4",
|
|
51
51
|
"@apify/utilities": "^2.15.5",
|
|
52
|
-
"@crawlee/basic": "4.0.0-beta.
|
|
53
|
-
"@crawlee/core": "4.0.0-beta.
|
|
54
|
-
"@crawlee/http-client": "4.0.0-beta.
|
|
55
|
-
"@crawlee/types": "4.0.0-beta.
|
|
56
|
-
"@crawlee/utils": "4.0.0-beta.
|
|
52
|
+
"@crawlee/basic": "4.0.0-beta.126",
|
|
53
|
+
"@crawlee/core": "4.0.0-beta.126",
|
|
54
|
+
"@crawlee/http-client": "4.0.0-beta.126",
|
|
55
|
+
"@crawlee/types": "4.0.0-beta.126",
|
|
56
|
+
"@crawlee/utils": "4.0.0-beta.126",
|
|
57
57
|
"@types/content-type": "^1.1.8",
|
|
58
58
|
"cheerio": "^1.0.0",
|
|
59
59
|
"content-type": "^1.0.5",
|
|
@@ -70,5 +70,5 @@
|
|
|
70
70
|
}
|
|
71
71
|
}
|
|
72
72
|
},
|
|
73
|
-
"gitHead": "
|
|
73
|
+
"gitHead": "5f0a8c1a43d4587f8640881fc55836a23e1c4f30"
|
|
74
74
|
}
|