@crawlee/http 4.0.0-rc.1 → 4.0.0-rc.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/internals/dom-crawler.d.ts +26 -12
- package/internals/dom-crawler.js +18 -9
- package/internals/file-download.d.ts +123 -6
- package/internals/file-download.js +21 -6
- package/internals/http-crawler.d.ts +16 -2
- package/internals/http-crawler.js +38 -27
- package/internals/utils.d.ts +0 -5
- package/internals/utils.js +1 -36
- package/package.json +12 -11
|
@@ -31,7 +31,7 @@ export interface DOMParseResult {
|
|
|
31
31
|
* });
|
|
32
32
|
* ```
|
|
33
33
|
*/
|
|
34
|
-
export interface DOMParser<Parsed extends DOMParseResult> {
|
|
34
|
+
export interface DOMParser<Parsed extends DOMParseResult, Selection extends ArrayLike<unknown> = ArrayLike<unknown>, Single = unknown> {
|
|
35
35
|
/**
|
|
36
36
|
* The context members {@link DOMParser.parse|`parse`} contributes, mapped to `true`. Used to build the
|
|
37
37
|
* placeholders that report a helpful error when the members are accessed after `skipNavigation` - the `Record`
|
|
@@ -45,10 +45,15 @@ export interface DOMParser<Parsed extends DOMParseResult> {
|
|
|
45
45
|
*/
|
|
46
46
|
extractLinks(parsed: Parsed, selector: string, baseUrl: string): Awaitable<string[]>;
|
|
47
47
|
/**
|
|
48
|
-
* Returns the current matches of `selector
|
|
49
|
-
* {@link DOMCrawlingContext.
|
|
48
|
+
* Returns the current matches of `selector`, in whatever form the DOM implementation uses - this is what
|
|
49
|
+
* {@link DOMCrawlingContext.querySelectorAll|`querySelectorAll`} hands to the request handler.
|
|
50
50
|
*/
|
|
51
|
-
|
|
51
|
+
querySelectorAll(parsed: Parsed, selector: string): Awaitable<Selection>;
|
|
52
|
+
/**
|
|
53
|
+
* Returns the first match of `selector`, or `undefined` when there is none - this is what
|
|
54
|
+
* {@link DOMCrawlingContext.querySelectorOne|`querySelectorOne`} hands to the request handler.
|
|
55
|
+
*/
|
|
56
|
+
querySelectorOne(parsed: Parsed, selector: string): Awaitable<Single | undefined>;
|
|
52
57
|
/**
|
|
53
58
|
* Whether the parse result can change after {@link DOMParser.parse|`parse`} returned - the case when the DOM
|
|
54
59
|
* implementation runs the page scripts. If it can, {@link DOMCrawlingContext.waitForSelector|`waitForSelector`}
|
|
@@ -67,7 +72,7 @@ export interface DOMParser<Parsed extends DOMParseResult> {
|
|
|
67
72
|
*/
|
|
68
73
|
cleanup?(parsed: Parsed): Awaitable<void>;
|
|
69
74
|
}
|
|
70
|
-
export interface DOMCrawlingHelpers {
|
|
75
|
+
export interface DOMCrawlingHelpers<Selection extends ArrayLike<unknown> = ArrayLike<unknown>, Single = unknown> {
|
|
71
76
|
/**
|
|
72
77
|
* Extracts URLs from the parsed DOM, without adding them to the request queue.
|
|
73
78
|
*/
|
|
@@ -92,6 +97,15 @@ export interface DOMCrawlingHelpers {
|
|
|
92
97
|
* ```
|
|
93
98
|
*/
|
|
94
99
|
waitForSelector(selector: string, timeoutMs?: number): Promise<void>;
|
|
100
|
+
/**
|
|
101
|
+
* Like {@link DOMCrawlingHelpers.waitForSelector|`waitForSelector`}, but returns the matches in the form
|
|
102
|
+
* the parser produces them - a Cheerio object in {@link CheerioCrawler}. Throws when nothing matches.
|
|
103
|
+
*/
|
|
104
|
+
querySelectorAll(selector: string, timeoutMs?: number): Promise<Selection>;
|
|
105
|
+
/**
|
|
106
|
+
* Like {@link DOMCrawlingHelpers.querySelectorAll|`querySelectorAll`}, but returns only the first match.
|
|
107
|
+
*/
|
|
108
|
+
querySelectorOne(selector: string, timeoutMs?: number): Promise<Single>;
|
|
95
109
|
/**
|
|
96
110
|
* Returns Cheerio handle, allowing to work with the data same way as with {@link CheerioCrawler}.
|
|
97
111
|
* When provided with the `selector` argument, it will throw if it's not available.
|
|
@@ -106,14 +120,14 @@ export interface DOMCrawlingHelpers {
|
|
|
106
120
|
*/
|
|
107
121
|
parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioAPI>;
|
|
108
122
|
}
|
|
109
|
-
export type DOMCrawlingContext<Parsed extends DOMParseResult = DOMParseResult, UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
|
|
110
|
-
JSONData extends Dictionary = any> = InternalHttpCrawlingContext<UserData, JSONData> & Parsed & DOMCrawlingHelpers
|
|
111
|
-
export interface DOMCrawlerOptions<Parsed extends DOMParseResult = DOMParseResult, ContextExtension = Dictionary<never>, ExtendedContext extends DOMCrawlingContext<Parsed> = DOMCrawlingContext<Parsed> & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, any>, StatisticStateExtension extends object = {}> extends Omit<HttpCrawlerOptions<DOMCrawlingContext<Parsed>, ContextExtension, ExtendedContext, Routes, StatisticStateExtension>, 'contextPipelineBuilder'> {
|
|
123
|
+
export type DOMCrawlingContext<Parsed extends DOMParseResult = DOMParseResult, Selection extends ArrayLike<unknown> = ArrayLike<unknown>, Single = unknown, UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
|
|
124
|
+
JSONData extends Dictionary = any> = Omit<InternalHttpCrawlingContext<UserData, JSONData>, keyof DOMCrawlingHelpers> & Parsed & DOMCrawlingHelpers<Selection, Single>;
|
|
125
|
+
export interface DOMCrawlerOptions<Parsed extends DOMParseResult = DOMParseResult, Selection extends ArrayLike<unknown> = ArrayLike<unknown>, Single = unknown, ContextExtension = Dictionary<never>, ExtendedContext extends DOMCrawlingContext<Parsed, Selection, Single> = DOMCrawlingContext<Parsed, Selection, Single> & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, any>, StatisticStateExtension extends object = {}> extends Omit<HttpCrawlerOptions<DOMCrawlingContext<Parsed, Selection, Single>, ContextExtension, ExtendedContext, Routes, StatisticStateExtension>, 'contextPipelineBuilder'> {
|
|
112
126
|
/**
|
|
113
127
|
* The DOM implementation to parse the response bodies with. Its parse result becomes part of the crawling
|
|
114
128
|
* context, so the members the request handler receives follow from the parser you pass.
|
|
115
129
|
*/
|
|
116
|
-
parser: DOMParser<Parsed>;
|
|
130
|
+
parser: DOMParser<Parsed, Selection, Single>;
|
|
117
131
|
}
|
|
118
132
|
/**
|
|
119
133
|
* An {@link HttpCrawler} that parses each response into a DOM using the {@link DOMCrawlerOptions.parser|`parser`}
|
|
@@ -124,8 +138,8 @@ export interface DOMCrawlerOptions<Parsed extends DOMParseResult = DOMParseResul
|
|
|
124
138
|
*
|
|
125
139
|
* @category Crawlers
|
|
126
140
|
*/
|
|
127
|
-
export declare class DOMCrawler<Parsed extends DOMParseResult = DOMParseResult, ContextExtension = Dictionary<never>, ExtendedContext extends DOMCrawlingContext<Parsed> = DOMCrawlingContext<Parsed> & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<DOMCrawlingContext<Parsed>['request']>>, StatisticStateExtension extends object = {}> extends HttpCrawler<DOMCrawlingContext<Parsed>, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
|
|
141
|
+
export declare class DOMCrawler<Parsed extends DOMParseResult = DOMParseResult, Selection extends ArrayLike<unknown> = ArrayLike<unknown>, Single = unknown, ContextExtension = Dictionary<never>, ExtendedContext extends DOMCrawlingContext<Parsed, Selection, Single> = DOMCrawlingContext<Parsed, Selection, Single> & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<DOMCrawlingContext<Parsed, Selection, Single>['request']>>, StatisticStateExtension extends object = {}> extends HttpCrawler<DOMCrawlingContext<Parsed, Selection, Single>, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
|
|
128
142
|
#private;
|
|
129
|
-
constructor(options: DOMCrawlerOptions<Parsed, ContextExtension, ExtendedContext, Routes, StatisticStateExtension>);
|
|
130
|
-
protected buildContextPipeline(): ContextPipeline<CrawlingContext, DOMCrawlingContext<Parsed>>;
|
|
143
|
+
constructor(options: DOMCrawlerOptions<Parsed, Selection, Single, ContextExtension, ExtendedContext, Routes, StatisticStateExtension>);
|
|
144
|
+
protected buildContextPipeline(): ContextPipeline<CrawlingContext, DOMCrawlingContext<Parsed, Selection, Single>>;
|
|
131
145
|
}
|
package/internals/dom-crawler.js
CHANGED
|
@@ -50,19 +50,29 @@ export class DOMCrawler extends HttpCrawler {
|
|
|
50
50
|
const { addRequests } = context;
|
|
51
51
|
const parser = this.#parser;
|
|
52
52
|
const extractLinks = async (options) => parser.extractLinks(context, options?.selector ?? 'a', options?.baseUrl ?? context.request.loadedUrl ?? context.request.url);
|
|
53
|
-
|
|
53
|
+
// Polls `query` while `isEmpty` holds, for as long as a mutable parser allows; a static one gets one shot.
|
|
54
|
+
const poll = async (selector, timeoutMs, query, isEmpty) => {
|
|
54
55
|
let remaining = parser.mutable ? timeoutMs : 0;
|
|
55
|
-
|
|
56
|
+
let result = await query();
|
|
57
|
+
while (isEmpty(result)) {
|
|
56
58
|
if (remaining <= 0) {
|
|
57
59
|
throw new Error(`Selector '${selector}' not found.`);
|
|
58
60
|
}
|
|
59
61
|
await sleep(50);
|
|
60
62
|
remaining -= 50;
|
|
63
|
+
result = await query();
|
|
61
64
|
}
|
|
65
|
+
return result;
|
|
66
|
+
};
|
|
67
|
+
const querySelectorAll = async (selector, timeoutMs = 5_000) => poll(selector, timeoutMs, () => parser.querySelectorAll(context, selector), (matches) => matches.length === 0);
|
|
68
|
+
const waitForSelector = async (selector, timeoutMs) => {
|
|
69
|
+
await querySelectorAll(selector, timeoutMs);
|
|
62
70
|
};
|
|
63
71
|
return {
|
|
64
72
|
extractLinks,
|
|
65
73
|
waitForSelector,
|
|
74
|
+
querySelectorAll,
|
|
75
|
+
querySelectorOne: async (selector, timeoutMs = 5_000) => poll(selector, timeoutMs, () => parser.querySelectorOne(context, selector), (match) => match === undefined),
|
|
66
76
|
enqueueLinks: async (options = {}) => {
|
|
67
77
|
const baseUrl = resolveBaseUrlForEnqueueLinksFiltering({
|
|
68
78
|
enqueueStrategy: options.strategy,
|
|
@@ -77,16 +87,15 @@ export class DOMCrawler extends HttpCrawler {
|
|
|
77
87
|
strategy: options.strategy ?? EnqueueStrategy.SameHostname,
|
|
78
88
|
});
|
|
79
89
|
},
|
|
80
|
-
async parseWithCheerio(selector,
|
|
90
|
+
async parseWithCheerio(selector, timeoutMs) {
|
|
91
|
+
if (selector) {
|
|
92
|
+
await waitForSelector(selector, timeoutMs);
|
|
93
|
+
}
|
|
81
94
|
// Import full cheerio (not cheerio/slim) to be browser-compliant for DOM Crawlers (not CheerioCrawler).
|
|
82
|
-
|
|
95
|
+
return ((await parser.toCheerio?.(context)) ??
|
|
83
96
|
(await import('cheerio')).load(context.body, {
|
|
84
97
|
xmlMode: context.contentType.type.includes('xml'),
|
|
85
|
-
});
|
|
86
|
-
if (selector && $(selector).get().length === 0) {
|
|
87
|
-
throw new Error(`Selector '${selector}' not found.`);
|
|
88
|
-
}
|
|
89
|
-
return $;
|
|
98
|
+
}));
|
|
90
99
|
},
|
|
91
100
|
};
|
|
92
101
|
}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import type { BasicCrawlerOptions, CrawlingContext, CrawlingRequest, LoadedRequest } from '@crawlee/basic';
|
|
2
2
|
import { BasicCrawler } from '@crawlee/basic';
|
|
3
3
|
import type { Dictionary } from '@crawlee/types';
|
|
4
|
+
import { z } from 'zod';
|
|
4
5
|
import type { ErrorHandler, GetUserDataFromRequest, RequestHandler, RouterHandler, RouterRoutes, RouteSchemas, RoutesFromSchemas } from '../index.js';
|
|
5
6
|
export type FileDownloadErrorHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
|
|
6
7
|
ContextExtension = Dictionary<never>> = ErrorHandler<CrawlingContext, FileDownloadCrawlingContext<UserData> & ContextExtension>;
|
|
@@ -13,6 +14,20 @@ export interface FileDownloadCrawlingContext<UserData extends Dictionary = any>
|
|
|
13
14
|
};
|
|
14
15
|
}
|
|
15
16
|
export type FileDownloadRequestHandler<UserData extends Dictionary = any> = RequestHandler<FileDownloadCrawlingContext<UserData>>;
|
|
17
|
+
export interface FileDownloadOptions extends Omit<BasicCrawlerOptions<FileDownloadCrawlingContext>, 'contextPipelineBuilder'> {
|
|
18
|
+
/**
|
|
19
|
+
* If set to `true`, TLS/SSL certificate errors are ignored. Forwarded to the HTTP client as
|
|
20
|
+
* {@link SendRequestOptions.ignoreTlsErrors|`ignoreTlsErrors`} on every download, so custom
|
|
21
|
+
* {@link BaseHttpClient} implementations should honor that flag (the built-in impit and got-scraping
|
|
22
|
+
* clients do; the native fetch fallback cannot disable TLS verification and warns instead).
|
|
23
|
+
*
|
|
24
|
+
* Certificates are verified by default. Sessions whose proxy is a MITM proxy
|
|
25
|
+
* (`session.proxyInfo.ignoreTlsErrors`) skip verification regardless of this option.
|
|
26
|
+
*
|
|
27
|
+
* @default false
|
|
28
|
+
*/
|
|
29
|
+
ignoreTlsErrors?: boolean;
|
|
30
|
+
}
|
|
16
31
|
/**
|
|
17
32
|
* Provides a framework for downloading files in parallel using plain HTTP requests. The URLs to download are fed either from a static list of URLs or they can be added on the fly from another crawler.
|
|
18
33
|
*
|
|
@@ -20,7 +35,7 @@ export type FileDownloadRequestHandler<UserData extends Dictionary = any> = Requ
|
|
|
20
35
|
* However, it doesn't parse the content - if you need to e.g. extract data from the downloaded files,
|
|
21
36
|
* you might need to use {@link CheerioCrawler}, {@link PuppeteerCrawler} or {@link PlaywrightCrawler} instead.
|
|
22
37
|
*
|
|
23
|
-
* `
|
|
38
|
+
* `FileDownload` downloads each URL using a plain HTTP request and then invokes the user-provided {@link BasicCrawlerOptions.requestHandler} where the user can specify what to do with the downloaded data.
|
|
24
39
|
*
|
|
25
40
|
* The source URLs are represented using {@link Request} objects that are fed from the {@link IRequestManager|request manager} provided via the {@link BasicCrawlerOptions.requestManager|`requestManager`} constructor option (a {@link RequestQueue} is itself a request manager). To read from a read-only source such as a {@link RequestList} while still being able to enqueue new requests, combine it with a queue into a {@link RequestManagerTandem} via {@link IRequestLoader.toTandem|`requestLoader.toTandem()`} and pass the result as `requestManager`.
|
|
26
41
|
*
|
|
@@ -28,14 +43,14 @@ export type FileDownloadRequestHandler<UserData extends Dictionary = any> = Requ
|
|
|
28
43
|
*
|
|
29
44
|
* The crawler finishes when there are no more {@link Request} objects to crawl.
|
|
30
45
|
*
|
|
31
|
-
* New requests are only dispatched when there is enough free CPU and memory available, as judged by the crawler's {@link ConcurrencySystem}. Concurrency is tuned via the `minConcurrency`, `maxConcurrency` and `maxRequestsPerMinute` options of the `
|
|
46
|
+
* New requests are only dispatched when there is enough free CPU and memory available, as judged by the crawler's {@link ConcurrencySystem}. Concurrency is tuned via the `minConcurrency`, `maxConcurrency` and `maxRequestsPerMinute` options of the `FileDownload` constructor, or, for finer control, by injecting a pre-configured {@link ConcurrencySystem|`concurrencySystem`}.
|
|
32
47
|
*
|
|
33
48
|
* ## Example usage
|
|
34
49
|
*
|
|
35
50
|
* ```ts
|
|
36
|
-
* const crawler = new
|
|
37
|
-
* requestHandler({
|
|
38
|
-
*
|
|
51
|
+
* const crawler = new FileDownload({
|
|
52
|
+
* async requestHandler({ response, request }) {
|
|
53
|
+
* await writeFile(request.url.replace(/[^a-z0-9\.]/gi, '_'), Buffer.from(await response.arrayBuffer()));
|
|
39
54
|
* },
|
|
40
55
|
* });
|
|
41
56
|
*
|
|
@@ -48,7 +63,109 @@ export type FileDownloadRequestHandler<UserData extends Dictionary = any> = Requ
|
|
|
48
63
|
*/
|
|
49
64
|
export declare class FileDownload extends BasicCrawler<FileDownloadCrawlingContext> {
|
|
50
65
|
#private;
|
|
51
|
-
|
|
66
|
+
/**
|
|
67
|
+
* @internal
|
|
68
|
+
*/
|
|
69
|
+
protected static optionsShape: {
|
|
70
|
+
contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
71
|
+
extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
72
|
+
requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
73
|
+
requestQueue: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
74
|
+
requestManager: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
75
|
+
requestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
76
|
+
requestHandlerTimeoutSecs: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
77
|
+
errorHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
78
|
+
failedRequestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
79
|
+
maxRequestRetries: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
80
|
+
sameDomainDelaySecs: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
81
|
+
maxRequestsPerCrawl: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
82
|
+
maxCrawlDepth: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
83
|
+
taskLoopOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
84
|
+
concurrencySystem: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
85
|
+
sessionPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
86
|
+
proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
87
|
+
statusMessageLoggingInterval: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
88
|
+
statusMessageCallback: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
89
|
+
additionalHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
|
|
90
|
+
ignoreHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
|
|
91
|
+
blockedStatusCodes: z.ZodOptional<z.ZodArray<z.ZodCustom<number, number>>>;
|
|
92
|
+
retryOnBlocked: z.ZodDefault<z.ZodBoolean>;
|
|
93
|
+
respectRobotsTxtFile: z.ZodDefault<z.ZodUnion<readonly [z.ZodBoolean, z.ZodCustom<Dictionary, Dictionary>]>>;
|
|
94
|
+
transactionalStorage: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodObject<{
|
|
95
|
+
requestQueue: z.ZodOptional<z.ZodEnum<{
|
|
96
|
+
deferred: "deferred";
|
|
97
|
+
writeThrough: "writeThrough";
|
|
98
|
+
}>>;
|
|
99
|
+
}, z.core.$strict>]>>;
|
|
100
|
+
onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
101
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
102
|
+
httpClient: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/http-client").BaseHttpClient>>;
|
|
103
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
104
|
+
configuration: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/basic").Configuration>>;
|
|
105
|
+
storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
106
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
107
|
+
eventManager: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/basic").EventManager>>;
|
|
108
|
+
logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
109
|
+
minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
110
|
+
maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
111
|
+
initialConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
112
|
+
maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
113
|
+
keepAlive: z.ZodOptional<z.ZodBoolean>;
|
|
114
|
+
statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
115
|
+
id: z.ZodOptional<z.ZodString>;
|
|
116
|
+
ignoreTlsErrors: z.ZodDefault<z.ZodBoolean>;
|
|
117
|
+
};
|
|
118
|
+
/** @internal */
|
|
119
|
+
protected static optionsSchema: z.ZodObject<{
|
|
120
|
+
contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
121
|
+
extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
122
|
+
requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
123
|
+
requestQueue: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
124
|
+
requestManager: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
125
|
+
requestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
126
|
+
requestHandlerTimeoutSecs: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
127
|
+
errorHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
128
|
+
failedRequestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
129
|
+
maxRequestRetries: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
130
|
+
sameDomainDelaySecs: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
131
|
+
maxRequestsPerCrawl: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
132
|
+
maxCrawlDepth: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
133
|
+
taskLoopOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
134
|
+
concurrencySystem: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
135
|
+
sessionPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
136
|
+
proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
137
|
+
statusMessageLoggingInterval: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
138
|
+
statusMessageCallback: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
139
|
+
additionalHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
|
|
140
|
+
ignoreHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
|
|
141
|
+
blockedStatusCodes: z.ZodOptional<z.ZodArray<z.ZodCustom<number, number>>>;
|
|
142
|
+
retryOnBlocked: z.ZodDefault<z.ZodBoolean>;
|
|
143
|
+
respectRobotsTxtFile: z.ZodDefault<z.ZodUnion<readonly [z.ZodBoolean, z.ZodCustom<Dictionary, Dictionary>]>>;
|
|
144
|
+
transactionalStorage: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodObject<{
|
|
145
|
+
requestQueue: z.ZodOptional<z.ZodEnum<{
|
|
146
|
+
deferred: "deferred";
|
|
147
|
+
writeThrough: "writeThrough";
|
|
148
|
+
}>>;
|
|
149
|
+
}, z.core.$strict>]>>;
|
|
150
|
+
onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
151
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
152
|
+
httpClient: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/http-client").BaseHttpClient>>;
|
|
153
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
154
|
+
configuration: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/basic").Configuration>>;
|
|
155
|
+
storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
156
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
157
|
+
eventManager: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/basic").EventManager>>;
|
|
158
|
+
logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
159
|
+
minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
160
|
+
maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
161
|
+
initialConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
162
|
+
maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
163
|
+
keepAlive: z.ZodOptional<z.ZodBoolean>;
|
|
164
|
+
statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
165
|
+
id: z.ZodOptional<z.ZodString>;
|
|
166
|
+
ignoreTlsErrors: z.ZodDefault<z.ZodBoolean>;
|
|
167
|
+
}, z.core.$strict>;
|
|
168
|
+
constructor(options?: FileDownloadOptions);
|
|
52
169
|
}
|
|
53
170
|
/**
|
|
54
171
|
* Creates new {@link Router} instance that works based on request labels.
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
import { BasicCrawler, ContextPipeline } from '@crawlee/basic';
|
|
2
2
|
import { ResponseWithUrl } from '@crawlee/http-client';
|
|
3
|
+
import { parseArgument } from '@crawlee/utils/internal';
|
|
4
|
+
import { z } from 'zod';
|
|
3
5
|
import { Router } from '../index.js';
|
|
4
6
|
import { parseContentTypeFromResponse } from './utils.js';
|
|
5
7
|
/**
|
|
@@ -9,7 +11,7 @@ import { parseContentTypeFromResponse } from './utils.js';
|
|
|
9
11
|
* However, it doesn't parse the content - if you need to e.g. extract data from the downloaded files,
|
|
10
12
|
* you might need to use {@link CheerioCrawler}, {@link PuppeteerCrawler} or {@link PlaywrightCrawler} instead.
|
|
11
13
|
*
|
|
12
|
-
* `
|
|
14
|
+
* `FileDownload` downloads each URL using a plain HTTP request and then invokes the user-provided {@link BasicCrawlerOptions.requestHandler} where the user can specify what to do with the downloaded data.
|
|
13
15
|
*
|
|
14
16
|
* The source URLs are represented using {@link Request} objects that are fed from the {@link IRequestManager|request manager} provided via the {@link BasicCrawlerOptions.requestManager|`requestManager`} constructor option (a {@link RequestQueue} is itself a request manager). To read from a read-only source such as a {@link RequestList} while still being able to enqueue new requests, combine it with a queue into a {@link RequestManagerTandem} via {@link IRequestLoader.toTandem|`requestLoader.toTandem()`} and pass the result as `requestManager`.
|
|
15
17
|
*
|
|
@@ -17,14 +19,14 @@ import { parseContentTypeFromResponse } from './utils.js';
|
|
|
17
19
|
*
|
|
18
20
|
* The crawler finishes when there are no more {@link Request} objects to crawl.
|
|
19
21
|
*
|
|
20
|
-
* New requests are only dispatched when there is enough free CPU and memory available, as judged by the crawler's {@link ConcurrencySystem}. Concurrency is tuned via the `minConcurrency`, `maxConcurrency` and `maxRequestsPerMinute` options of the `
|
|
22
|
+
* New requests are only dispatched when there is enough free CPU and memory available, as judged by the crawler's {@link ConcurrencySystem}. Concurrency is tuned via the `minConcurrency`, `maxConcurrency` and `maxRequestsPerMinute` options of the `FileDownload` constructor, or, for finer control, by injecting a pre-configured {@link ConcurrencySystem|`concurrencySystem`}.
|
|
21
23
|
*
|
|
22
24
|
* ## Example usage
|
|
23
25
|
*
|
|
24
26
|
* ```ts
|
|
25
|
-
* const crawler = new
|
|
26
|
-
* requestHandler({
|
|
27
|
-
*
|
|
27
|
+
* const crawler = new FileDownload({
|
|
28
|
+
* async requestHandler({ response, request }) {
|
|
29
|
+
* await writeFile(request.url.replace(/[^a-z0-9\.]/gi, '_'), Buffer.from(await response.arrayBuffer()));
|
|
28
30
|
* },
|
|
29
31
|
* });
|
|
30
32
|
*
|
|
@@ -36,12 +38,24 @@ import { parseContentTypeFromResponse } from './utils.js';
|
|
|
36
38
|
* ```
|
|
37
39
|
*/
|
|
38
40
|
export class FileDownload extends BasicCrawler {
|
|
41
|
+
#ignoreTlsErrors;
|
|
42
|
+
/**
|
|
43
|
+
* @internal
|
|
44
|
+
*/
|
|
45
|
+
static optionsShape = {
|
|
46
|
+
...BasicCrawler.optionsShape,
|
|
47
|
+
ignoreTlsErrors: z.boolean().default(false),
|
|
48
|
+
};
|
|
49
|
+
/** @internal */
|
|
50
|
+
static optionsSchema = z.strictObject(FileDownload.optionsShape);
|
|
39
51
|
// TODO hooks
|
|
40
52
|
constructor(options = {}) {
|
|
53
|
+
const { ignoreTlsErrors, ...basicCrawlerOptions } = parseArgument(options, FileDownload.optionsSchema, 'FileDownloadOptions');
|
|
41
54
|
super({
|
|
42
|
-
...
|
|
55
|
+
...basicCrawlerOptions,
|
|
43
56
|
contextPipelineBuilder: () => this.#buildContextPipeline(),
|
|
44
57
|
});
|
|
58
|
+
this.#ignoreTlsErrors = ignoreTlsErrors;
|
|
45
59
|
}
|
|
46
60
|
#buildContextPipeline() {
|
|
47
61
|
return ContextPipeline.create().compose(this.#initiateDownload.bind(this));
|
|
@@ -49,6 +63,7 @@ export class FileDownload extends BasicCrawler {
|
|
|
49
63
|
async #initiateDownload(context, onCleanup) {
|
|
50
64
|
const response = await this.httpClient.sendRequest(context.request.intoFetchAPIRequest(), {
|
|
51
65
|
session: context.session,
|
|
66
|
+
ignoreTlsErrors: this.#ignoreTlsErrors,
|
|
52
67
|
});
|
|
53
68
|
const { type, charset: encoding } = parseContentTypeFromResponse(response);
|
|
54
69
|
context.request.url = response.url;
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import type { BasicCrawlerOptions, ConcurrencySystem, ConcurrencySystemOptions, CrawlingContext, ErrorHandler, GetUserDataFromRequest, LoadedRequest, CrawlingRequest, RequireContextPipeline, RequestHandler, RouterHandler, RouterRoutes, RouteSchemas, RoutesFromSchemas } from '@crawlee/basic';
|
|
2
2
|
import { BasicCrawler, ContextPipeline } from '@crawlee/basic';
|
|
3
3
|
import type { Awaitable, Dictionary } from '@crawlee/types';
|
|
4
|
-
import type { CheerioAPI } from 'cheerio';
|
|
4
|
+
import type { Cheerio, CheerioAPI } from 'cheerio';
|
|
5
|
+
import type { AnyNode } from 'domhandler';
|
|
5
6
|
import type { JsonValue } from 'type-fest';
|
|
6
7
|
import { z } from 'zod';
|
|
7
8
|
/**
|
|
@@ -35,7 +36,7 @@ export interface HttpCrawlerOptions<Context extends InternalHttpCrawlingContext
|
|
|
35
36
|
* {@link BaseHttpClient} implementations should honor that flag (the built-in impit and got-scraping
|
|
36
37
|
* clients do; the native fetch fallback cannot disable TLS verification and warns instead).
|
|
37
38
|
*
|
|
38
|
-
* @default
|
|
39
|
+
* @default false
|
|
39
40
|
*/
|
|
40
41
|
ignoreTlsErrors?: boolean;
|
|
41
42
|
/**
|
|
@@ -162,6 +163,17 @@ JSONData extends JsonValue = any> extends CrawlingContextWithResponse<UserData>
|
|
|
162
163
|
* ```
|
|
163
164
|
*/
|
|
164
165
|
waitForSelector(selector: string, timeoutMs?: number): Promise<void>;
|
|
166
|
+
/**
|
|
167
|
+
* Like {@link InternalHttpCrawlingContext.waitForSelector|`waitForSelector`}, but returns the matched
|
|
168
|
+
* elements - a Cheerio object here, whatever the parser produces in {@link DOMCrawler}. Throws when nothing
|
|
169
|
+
* matches.
|
|
170
|
+
*/
|
|
171
|
+
querySelectorAll(selector: string, timeoutMs?: number): Promise<ArrayLike<unknown>>;
|
|
172
|
+
/**
|
|
173
|
+
* Like {@link InternalHttpCrawlingContext.querySelectorAll|`querySelectorAll`}, but returns only the first
|
|
174
|
+
* match.
|
|
175
|
+
*/
|
|
176
|
+
querySelectorOne(selector: string, timeoutMs?: number): Promise<unknown>;
|
|
165
177
|
/**
|
|
166
178
|
* Returns Cheerio handle for `page.content()`, allowing to work with the data same way as with {@link CheerioCrawler}.
|
|
167
179
|
* This is here to unify the crawler API, so they all have this handy method - in {@link CheerioCrawler} it has
|
|
@@ -182,6 +194,8 @@ JSONData extends JsonValue = any> extends CrawlingContextWithResponse<UserData>
|
|
|
182
194
|
parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioAPI>;
|
|
183
195
|
}
|
|
184
196
|
export interface HttpCrawlingContext<UserData extends Dictionary = any, JSONData extends JsonValue = any> extends InternalHttpCrawlingContext<UserData, JSONData> {
|
|
197
|
+
querySelectorAll(selector: string, timeoutMs?: number): Promise<Cheerio<AnyNode>>;
|
|
198
|
+
querySelectorOne(selector: string, timeoutMs?: number): Promise<Cheerio<AnyNode>>;
|
|
185
199
|
}
|
|
186
200
|
export type HttpRequestHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
|
|
187
201
|
JSONData extends JsonValue = any> = RequestHandler<HttpCrawlingContext<UserData, JSONData>>;
|
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
import { Readable } from 'node:stream';
|
|
2
2
|
import util from 'node:util';
|
|
3
|
-
import { BasicCrawler, ContextPipeline, getCookiesFromResponse, NavigationSkippedError, remainingNavigationWindowMillis, RequestState, RequestThrottledError, Router, SessionError, } from '@crawlee/basic';
|
|
3
|
+
import { BasicCrawler, ContextPipeline, getCookiesFromResponse, NavigationSkippedError, skippedNavigationRequest, remainingNavigationWindowMillis, RequestState, RequestThrottledError, Router, SessionError, } from '@crawlee/basic';
|
|
4
4
|
import { ResponseWithUrl } from '@crawlee/http-client';
|
|
5
|
-
import { parseArgument, RETRY_CSS_SELECTORS, schemas } from '@crawlee/utils/internal';
|
|
5
|
+
import { parseArgument, processHttpRequestOptions, RETRY_CSS_SELECTORS, schemas } from '@crawlee/utils/internal';
|
|
6
6
|
import { isTypeValid, parse as parseContentType } from 'content-type';
|
|
7
7
|
import iconv from 'iconv-lite';
|
|
8
8
|
import { z } from 'zod';
|
|
9
9
|
import { addTimeoutToPromise, storage, TimeoutError, tryCancel } from '@apify/timeout';
|
|
10
|
-
import { extractCharsetFromHtmlBytes, parseContentTypeFromResponse
|
|
10
|
+
import { extractCharsetFromHtmlBytes, parseContentTypeFromResponse } from './utils.js';
|
|
11
11
|
/**
|
|
12
12
|
* Default mime types, which HttpScraper supports.
|
|
13
13
|
*/
|
|
@@ -117,13 +117,14 @@ export class HttpCrawler extends BasicCrawler {
|
|
|
117
117
|
#suggestResponseEncoding;
|
|
118
118
|
#forceResponseEncoding;
|
|
119
119
|
#supportedMimeTypes;
|
|
120
|
+
#downloadAbortControllers = new WeakMap();
|
|
120
121
|
/**
|
|
121
122
|
* @internal
|
|
122
123
|
*/
|
|
123
124
|
static optionsShape = {
|
|
124
125
|
...BasicCrawler.optionsShape,
|
|
125
126
|
navigationTimeoutSecs: schemas.anyNumber.default(30),
|
|
126
|
-
ignoreTlsErrors: z.boolean().default(
|
|
127
|
+
ignoreTlsErrors: z.boolean().default(false),
|
|
127
128
|
additionalMimeTypes: schemas.arrayOf(z.string(), 'strings').default(() => []),
|
|
128
129
|
suggestResponseEncoding: z.string().optional(),
|
|
129
130
|
forceResponseEncoding: z.string().optional(),
|
|
@@ -207,17 +208,7 @@ export class HttpCrawler extends BasicCrawler {
|
|
|
207
208
|
const { request } = crawlingContext;
|
|
208
209
|
if (request.skipNavigation) {
|
|
209
210
|
return {
|
|
210
|
-
request:
|
|
211
|
-
get(target, propertyName) {
|
|
212
|
-
if (propertyName === 'loadedUrl') {
|
|
213
|
-
throw new NavigationSkippedError('The `request.loadedUrl` property is not available - `skipNavigation` was used');
|
|
214
|
-
}
|
|
215
|
-
// `target` as receiver and bound methods, or `#` members throw on the proxy
|
|
216
|
-
const value = Reflect.get(target, propertyName, target);
|
|
217
|
-
return typeof value === 'function' ? value.bind(target) : value;
|
|
218
|
-
},
|
|
219
|
-
set: (target, propertyName, value) => Reflect.set(target, propertyName, value, target),
|
|
220
|
-
}),
|
|
211
|
+
request: skippedNavigationRequest(request),
|
|
221
212
|
get response() {
|
|
222
213
|
throw new NavigationSkippedError('The `response` property is not available - `skipNavigation` was used');
|
|
223
214
|
},
|
|
@@ -231,10 +222,12 @@ export class HttpCrawler extends BasicCrawler {
|
|
|
231
222
|
tryCancel();
|
|
232
223
|
const { request, session } = crawlingContext;
|
|
233
224
|
const proxyUrl = crawlingContext.proxyInfo?.url;
|
|
225
|
+
const downloadAbortController = new AbortController();
|
|
226
|
+
this.#downloadAbortControllers.set(request, downloadAbortController);
|
|
234
227
|
// Bound the request by whatever is left of the shared navigation window (the pre-navigation hooks may
|
|
235
228
|
// have already spent part of it), so it produces a clean navigation-timeout error rather than the raw
|
|
236
229
|
// client abort.
|
|
237
|
-
const httpResponse = await addTimeoutToPromise(async () => this.#requestFunction({ request, session, proxyUrl }), Math.max(1, remainingNavigationWindowMillis(crawlingContext, this.#navigationTimeoutMillis)), `Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);
|
|
230
|
+
const httpResponse = await addTimeoutToPromise(async () => this.#requestFunction({ request, session, proxyUrl, signal: downloadAbortController.signal }), Math.max(1, remainingNavigationWindowMillis(crawlingContext, this.#navigationTimeoutMillis)), `Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);
|
|
238
231
|
tryCancel();
|
|
239
232
|
request.loadedUrl = httpResponse?.url;
|
|
240
233
|
request.state = RequestState.AFTER_NAV;
|
|
@@ -258,6 +251,12 @@ export class HttpCrawler extends BasicCrawler {
|
|
|
258
251
|
get parseWithCheerio() {
|
|
259
252
|
throw new NavigationSkippedError('The `parseWithCheerio` method is not available - `skipNavigation` was used');
|
|
260
253
|
},
|
|
254
|
+
get querySelectorAll() {
|
|
255
|
+
throw new NavigationSkippedError('The `querySelectorAll` method is not available - `skipNavigation` was used');
|
|
256
|
+
},
|
|
257
|
+
get querySelectorOne() {
|
|
258
|
+
throw new NavigationSkippedError('The `querySelectorOne` method is not available - `skipNavigation` was used');
|
|
259
|
+
},
|
|
261
260
|
};
|
|
262
261
|
}
|
|
263
262
|
tryCancel();
|
|
@@ -279,7 +278,11 @@ export class HttpCrawler extends BasicCrawler {
|
|
|
279
278
|
if (remaining <= 0) {
|
|
280
279
|
throw new TimeoutError(`Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);
|
|
281
280
|
}
|
|
282
|
-
const parsed = await addTimeoutToPromise(async () => this.#parseResponse(crawlingContext.request, crawlingContext.response), remaining, `Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`)
|
|
281
|
+
const parsed = await addTimeoutToPromise(async () => this.#parseResponse(crawlingContext.request, crawlingContext.response), remaining, `Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`).catch((error) => {
|
|
282
|
+
// Stop the download along with the read, so a body that never ends doesn't outlive the request.
|
|
283
|
+
this.#downloadAbortControllers.get(crawlingContext.request)?.abort();
|
|
284
|
+
throw error;
|
|
285
|
+
});
|
|
283
286
|
tryCancel();
|
|
284
287
|
const response = parsed.response;
|
|
285
288
|
const contentType = parsed.contentType;
|
|
@@ -287,11 +290,16 @@ export class HttpCrawler extends BasicCrawler {
|
|
|
287
290
|
const { load } = await import('cheerio/slim');
|
|
288
291
|
return load(parsed.body.toString(), { xmlMode: contentType.type.includes('xml') });
|
|
289
292
|
};
|
|
290
|
-
const
|
|
291
|
-
const
|
|
292
|
-
if (
|
|
293
|
+
const querySelectorAll = async (selector, _timeoutMs) => {
|
|
294
|
+
const matches = (await loadBody())(selector);
|
|
295
|
+
if (matches.length === 0) {
|
|
293
296
|
throw new Error(`Selector '${selector}' not found.`);
|
|
294
297
|
}
|
|
298
|
+
return matches;
|
|
299
|
+
};
|
|
300
|
+
const querySelectorOne = async (selector, timeoutMs) => (await querySelectorAll(selector, timeoutMs)).first();
|
|
301
|
+
const waitForSelector = async (selector, timeoutMs) => {
|
|
302
|
+
await querySelectorAll(selector, timeoutMs);
|
|
295
303
|
};
|
|
296
304
|
const parseWithCheerio = async (selector, timeoutMs) => {
|
|
297
305
|
const $ = await loadBody();
|
|
@@ -328,6 +336,8 @@ export class HttpCrawler extends BasicCrawler {
|
|
|
328
336
|
return JSON.parse(jsonString);
|
|
329
337
|
},
|
|
330
338
|
waitForSelector,
|
|
339
|
+
querySelectorOne,
|
|
340
|
+
querySelectorAll,
|
|
331
341
|
parseWithCheerio,
|
|
332
342
|
contentType,
|
|
333
343
|
body: parsed.body,
|
|
@@ -359,10 +369,10 @@ export class HttpCrawler extends BasicCrawler {
|
|
|
359
369
|
* on the request such as only downloading the request body if the
|
|
360
370
|
* received content type matches text/html, application/xml, application/xhtml+xml.
|
|
361
371
|
*/
|
|
362
|
-
async #requestFunction({ request, session, proxyUrl }) {
|
|
372
|
+
async #requestFunction({ request, session, proxyUrl, signal }) {
|
|
363
373
|
const opts = this.#getRequestOptions(request, proxyUrl);
|
|
364
374
|
try {
|
|
365
|
-
return await this.#requestAsBrowser(opts, session);
|
|
375
|
+
return await this.#requestAsBrowser(opts, session, signal);
|
|
366
376
|
}
|
|
367
377
|
catch (e) {
|
|
368
378
|
if (e instanceof Error && e.constructor.name === 'TimeoutError') {
|
|
@@ -515,7 +525,7 @@ export class HttpCrawler extends BasicCrawler {
|
|
|
515
525
|
/**
|
|
516
526
|
* @internal wraps public utility for mocking purposes
|
|
517
527
|
*/
|
|
518
|
-
#requestAsBrowser = async (options, session) => {
|
|
528
|
+
#requestAsBrowser = async (options, session, signal) => {
|
|
519
529
|
const opts = processHttpRequestOptions({
|
|
520
530
|
...options,
|
|
521
531
|
responseType: 'text',
|
|
@@ -526,10 +536,11 @@ export class HttpCrawler extends BasicCrawler {
|
|
|
526
536
|
const cookieJar = this.#saveResponseCookies ? session.cookieJar : await session.cookieJar.clone();
|
|
527
537
|
// Bind the request to the shared navigation window instead of a fixed per-request timeout, so
|
|
528
538
|
// `extendTimeout()` can push the deadline and a fixed `AbortSignal.timeout` won't fire on its own and
|
|
529
|
-
// kill a lazily-read body mid-extension.
|
|
530
|
-
//
|
|
531
|
-
// still fails cleanly with a navigation timeout
|
|
539
|
+
// kill a lazily-read body mid-extension. The window's signal only covers the header phase; the body read is
|
|
540
|
+
// bounded separately (see `processHttpResponse`), which aborts `signal` when it fails, so a slow-streaming
|
|
541
|
+
// body still fails cleanly with a navigation timeout and its download stops too.
|
|
532
542
|
const cancelSignal = storage.getStore()?.cancelTask.signal;
|
|
543
|
+
const requestSignal = cancelSignal ? AbortSignal.any([cancelSignal, signal]) : signal;
|
|
533
544
|
const response = await this.httpClient.sendRequest(new Request(opts.url, {
|
|
534
545
|
body: opts.body ? Readable.toWeb(opts.body) : undefined,
|
|
535
546
|
headers: new Headers(opts.headers),
|
|
@@ -540,7 +551,7 @@ export class HttpCrawler extends BasicCrawler {
|
|
|
540
551
|
session,
|
|
541
552
|
cookieJar,
|
|
542
553
|
proxyUrl: opts.proxyUrl,
|
|
543
|
-
signal:
|
|
554
|
+
signal: requestSignal,
|
|
544
555
|
timeoutMillis: cancelSignal ? undefined : opts.timeout,
|
|
545
556
|
ignoreTlsErrors: this.#ignoreTlsErrors,
|
|
546
557
|
});
|
package/internals/utils.d.ts
CHANGED
|
@@ -1,8 +1,3 @@
|
|
|
1
|
-
import type { HttpRequest, HttpRequestOptions } from '@crawlee/types';
|
|
2
|
-
/**
|
|
3
|
-
* Converts {@link HttpRequestOptions} to a {@link HttpRequest}.
|
|
4
|
-
*/
|
|
5
|
-
export declare function processHttpRequestOptions({ searchParams, form, json, username, password, ...request }: HttpRequestOptions): HttpRequest;
|
|
6
1
|
/**
|
|
7
2
|
* Scans the first 1024 bytes of an HTML document (as latin1) to extract the charset
|
|
8
3
|
* declared via `<meta charset>` or `<meta http-equiv="Content-Type" content="...;charset=...">`.
|
package/internals/utils.js
CHANGED
|
@@ -1,44 +1,9 @@
|
|
|
1
1
|
import { extname } from 'node:path';
|
|
2
|
-
import {
|
|
3
|
-
import { applySearchParams, parseArgument } from '@crawlee/utils/internal';
|
|
2
|
+
import { parseArgument } from '@crawlee/utils/internal';
|
|
4
3
|
import { isTypeValid, parse as parseContentType } from 'content-type';
|
|
5
4
|
import mime from 'mime-types';
|
|
6
5
|
import { z } from 'zod';
|
|
7
6
|
const responseWithUrlSchema = z.looseObject({ url: z.url(), headers: z.looseObject({}) });
|
|
8
|
-
/**
|
|
9
|
-
* Converts {@link HttpRequestOptions} to a {@link HttpRequest}.
|
|
10
|
-
*/
|
|
11
|
-
export function processHttpRequestOptions({ searchParams, form, json, username, password, ...request }) {
|
|
12
|
-
const url = new URL(request.url);
|
|
13
|
-
const headers = new Headers(request.headers);
|
|
14
|
-
applySearchParams(url, searchParams);
|
|
15
|
-
if ([request.body, form, json].filter((value) => value !== undefined).length > 1) {
|
|
16
|
-
throw new Error('At most one of `body`, `form` and `json` may be specified in sendRequest arguments');
|
|
17
|
-
}
|
|
18
|
-
const body = (() => {
|
|
19
|
-
if (form !== undefined) {
|
|
20
|
-
return Readable.from(new URLSearchParams(form).toString());
|
|
21
|
-
}
|
|
22
|
-
if (json !== undefined) {
|
|
23
|
-
return Readable.from(JSON.stringify(json));
|
|
24
|
-
}
|
|
25
|
-
if (request.body !== undefined) {
|
|
26
|
-
return Readable.from(request.body);
|
|
27
|
-
}
|
|
28
|
-
return undefined;
|
|
29
|
-
})();
|
|
30
|
-
if (form !== undefined && !headers.has('content-type')) {
|
|
31
|
-
headers.set('content-type', 'application/x-www-form-urlencoded');
|
|
32
|
-
}
|
|
33
|
-
if (json !== undefined && !headers.has('content-type')) {
|
|
34
|
-
headers.set('content-type', 'application/json');
|
|
35
|
-
}
|
|
36
|
-
if (username !== undefined || password !== undefined) {
|
|
37
|
-
const encodedAuth = Buffer.from(`${username ?? ''}:${password ?? ''}`).toString('base64');
|
|
38
|
-
headers.set('authorization', `Basic ${encodedAuth}`);
|
|
39
|
-
}
|
|
40
|
-
return { ...request, body, url, headers };
|
|
41
|
-
}
|
|
42
7
|
/**
|
|
43
8
|
* Scans the first 1024 bytes of an HTML document (as latin1) to extract the charset
|
|
44
9
|
* declared via `<meta charset>` or `<meta http-equiv="Content-Type" content="...;charset=...">`.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/http",
|
|
3
|
-
"version": "4.0.0-rc.
|
|
3
|
+
"version": "4.0.0-rc.2",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.13.0"
|
|
@@ -48,17 +48,18 @@
|
|
|
48
48
|
},
|
|
49
49
|
"dependencies": {
|
|
50
50
|
"@apify/timeout": "^1.0.1",
|
|
51
|
-
"@crawlee/basic": "4.0.0-rc.
|
|
52
|
-
"@crawlee/http-client": "4.0.0-rc.
|
|
53
|
-
"@crawlee/types": "4.0.0-rc.
|
|
54
|
-
"@crawlee/utils": "4.0.0-rc.
|
|
55
|
-
"cheerio": "^1.
|
|
51
|
+
"@crawlee/basic": "4.0.0-rc.2",
|
|
52
|
+
"@crawlee/http-client": "4.0.0-rc.2",
|
|
53
|
+
"@crawlee/types": "4.0.0-rc.2",
|
|
54
|
+
"@crawlee/utils": "4.0.0-rc.2",
|
|
55
|
+
"cheerio": "^1.2.0",
|
|
56
56
|
"content-type": "^3.1.1",
|
|
57
|
-
"
|
|
58
|
-
"
|
|
59
|
-
"
|
|
57
|
+
"domhandler": "^6.0.1",
|
|
58
|
+
"iconv-lite": "^0.7.3",
|
|
59
|
+
"mime-types": "^3.0.2",
|
|
60
|
+
"tslib": "^2.9.1",
|
|
60
61
|
"type-fest": "^4.41.0",
|
|
61
|
-
"zod": "^4.
|
|
62
|
+
"zod": "^4.6.7"
|
|
62
63
|
},
|
|
63
64
|
"lerna": {
|
|
64
65
|
"command": {
|
|
@@ -67,5 +68,5 @@
|
|
|
67
68
|
}
|
|
68
69
|
}
|
|
69
70
|
},
|
|
70
|
-
"gitHead": "
|
|
71
|
+
"gitHead": "a9b623229cd7d1bff5063fc82940efebaea47165"
|
|
71
72
|
}
|