@crawlee/http 4.0.0-rc.1 → 4.0.0-rc.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -31,7 +31,7 @@ export interface DOMParseResult {
31
31
  * });
32
32
  * ```
33
33
  */
34
- export interface DOMParser<Parsed extends DOMParseResult> {
34
+ export interface DOMParser<Parsed extends DOMParseResult, Selection extends ArrayLike<unknown> = ArrayLike<unknown>, Single = unknown> {
35
35
  /**
36
36
  * The context members {@link DOMParser.parse|`parse`} contributes, mapped to `true`. Used to build the
37
37
  * placeholders that report a helpful error when the members are accessed after `skipNavigation` - the `Record`
@@ -45,10 +45,15 @@ export interface DOMParser<Parsed extends DOMParseResult> {
45
45
  */
46
46
  extractLinks(parsed: Parsed, selector: string, baseUrl: string): Awaitable<string[]>;
47
47
  /**
48
- * Returns the current matches of `selector`. Only the count is used, by
49
- * {@link DOMCrawlingContext.waitForSelector|`waitForSelector`}.
48
+ * Returns the current matches of `selector`, in whatever form the DOM implementation uses - this is what
49
+ * {@link DOMCrawlingContext.querySelectorAll|`querySelectorAll`} hands to the request handler.
50
50
  */
51
- select(parsed: Parsed, selector: string): Awaitable<ArrayLike<unknown>>;
51
+ querySelectorAll(parsed: Parsed, selector: string): Awaitable<Selection>;
52
+ /**
53
+ * Returns the first match of `selector`, or `undefined` when there is none - this is what
54
+ * {@link DOMCrawlingContext.querySelectorOne|`querySelectorOne`} hands to the request handler.
55
+ */
56
+ querySelectorOne(parsed: Parsed, selector: string): Awaitable<Single | undefined>;
52
57
  /**
53
58
  * Whether the parse result can change after {@link DOMParser.parse|`parse`} returned - the case when the DOM
54
59
  * implementation runs the page scripts. If it can, {@link DOMCrawlingContext.waitForSelector|`waitForSelector`}
@@ -67,7 +72,7 @@ export interface DOMParser<Parsed extends DOMParseResult> {
67
72
  */
68
73
  cleanup?(parsed: Parsed): Awaitable<void>;
69
74
  }
70
- export interface DOMCrawlingHelpers {
75
+ export interface DOMCrawlingHelpers<Selection extends ArrayLike<unknown> = ArrayLike<unknown>, Single = unknown> {
71
76
  /**
72
77
  * Extracts URLs from the parsed DOM, without adding them to the request queue.
73
78
  */
@@ -92,6 +97,15 @@ export interface DOMCrawlingHelpers {
92
97
  * ```
93
98
  */
94
99
  waitForSelector(selector: string, timeoutMs?: number): Promise<void>;
100
+ /**
101
+ * Like {@link DOMCrawlingHelpers.waitForSelector|`waitForSelector`}, but returns the matches in the form
102
+ * the parser produces them - a Cheerio object in {@link CheerioCrawler}. Throws when nothing matches.
103
+ */
104
+ querySelectorAll(selector: string, timeoutMs?: number): Promise<Selection>;
105
+ /**
106
+ * Like {@link DOMCrawlingHelpers.querySelectorAll|`querySelectorAll`}, but returns only the first match.
107
+ */
108
+ querySelectorOne(selector: string, timeoutMs?: number): Promise<Single>;
95
109
  /**
96
110
  * Returns Cheerio handle, allowing to work with the data same way as with {@link CheerioCrawler}.
97
111
  * When provided with the `selector` argument, it will throw if it's not available.
@@ -106,14 +120,14 @@ export interface DOMCrawlingHelpers {
106
120
  */
107
121
  parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioAPI>;
108
122
  }
109
- export type DOMCrawlingContext<Parsed extends DOMParseResult = DOMParseResult, UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
110
- JSONData extends Dictionary = any> = InternalHttpCrawlingContext<UserData, JSONData> & Parsed & DOMCrawlingHelpers;
111
- export interface DOMCrawlerOptions<Parsed extends DOMParseResult = DOMParseResult, ContextExtension = Dictionary<never>, ExtendedContext extends DOMCrawlingContext<Parsed> = DOMCrawlingContext<Parsed> & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, any>, StatisticStateExtension extends object = {}> extends Omit<HttpCrawlerOptions<DOMCrawlingContext<Parsed>, ContextExtension, ExtendedContext, Routes, StatisticStateExtension>, 'contextPipelineBuilder'> {
123
+ export type DOMCrawlingContext<Parsed extends DOMParseResult = DOMParseResult, Selection extends ArrayLike<unknown> = ArrayLike<unknown>, Single = unknown, UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
124
+ JSONData extends Dictionary = any> = Omit<InternalHttpCrawlingContext<UserData, JSONData>, keyof DOMCrawlingHelpers> & Parsed & DOMCrawlingHelpers<Selection, Single>;
125
+ export interface DOMCrawlerOptions<Parsed extends DOMParseResult = DOMParseResult, Selection extends ArrayLike<unknown> = ArrayLike<unknown>, Single = unknown, ContextExtension = Dictionary<never>, ExtendedContext extends DOMCrawlingContext<Parsed, Selection, Single> = DOMCrawlingContext<Parsed, Selection, Single> & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, any>, StatisticStateExtension extends object = {}> extends Omit<HttpCrawlerOptions<DOMCrawlingContext<Parsed, Selection, Single>, ContextExtension, ExtendedContext, Routes, StatisticStateExtension>, 'contextPipelineBuilder'> {
112
126
  /**
113
127
  * The DOM implementation to parse the response bodies with. Its parse result becomes part of the crawling
114
128
  * context, so the members the request handler receives follow from the parser you pass.
115
129
  */
116
- parser: DOMParser<Parsed>;
130
+ parser: DOMParser<Parsed, Selection, Single>;
117
131
  }
118
132
  /**
119
133
  * An {@link HttpCrawler} that parses each response into a DOM using the {@link DOMCrawlerOptions.parser|`parser`}
@@ -124,8 +138,8 @@ export interface DOMCrawlerOptions<Parsed extends DOMParseResult = DOMParseResul
124
138
  *
125
139
  * @category Crawlers
126
140
  */
127
- export declare class DOMCrawler<Parsed extends DOMParseResult = DOMParseResult, ContextExtension = Dictionary<never>, ExtendedContext extends DOMCrawlingContext<Parsed> = DOMCrawlingContext<Parsed> & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<DOMCrawlingContext<Parsed>['request']>>, StatisticStateExtension extends object = {}> extends HttpCrawler<DOMCrawlingContext<Parsed>, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
141
+ export declare class DOMCrawler<Parsed extends DOMParseResult = DOMParseResult, Selection extends ArrayLike<unknown> = ArrayLike<unknown>, Single = unknown, ContextExtension = Dictionary<never>, ExtendedContext extends DOMCrawlingContext<Parsed, Selection, Single> = DOMCrawlingContext<Parsed, Selection, Single> & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<DOMCrawlingContext<Parsed, Selection, Single>['request']>>, StatisticStateExtension extends object = {}> extends HttpCrawler<DOMCrawlingContext<Parsed, Selection, Single>, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
128
142
  #private;
129
- constructor(options: DOMCrawlerOptions<Parsed, ContextExtension, ExtendedContext, Routes, StatisticStateExtension>);
130
- protected buildContextPipeline(): ContextPipeline<CrawlingContext, DOMCrawlingContext<Parsed>>;
143
+ constructor(options: DOMCrawlerOptions<Parsed, Selection, Single, ContextExtension, ExtendedContext, Routes, StatisticStateExtension>);
144
+ protected buildContextPipeline(): ContextPipeline<CrawlingContext, DOMCrawlingContext<Parsed, Selection, Single>>;
131
145
  }
@@ -50,19 +50,29 @@ export class DOMCrawler extends HttpCrawler {
50
50
  const { addRequests } = context;
51
51
  const parser = this.#parser;
52
52
  const extractLinks = async (options) => parser.extractLinks(context, options?.selector ?? 'a', options?.baseUrl ?? context.request.loadedUrl ?? context.request.url);
53
- const waitForSelector = async (selector, timeoutMs = 5_000) => {
53
+ // Polls `query` while `isEmpty` holds, for as long as a mutable parser allows; a static one gets one shot.
54
+ const poll = async (selector, timeoutMs, query, isEmpty) => {
54
55
  let remaining = parser.mutable ? timeoutMs : 0;
55
- while ((await parser.select(context, selector)).length === 0) {
56
+ let result = await query();
57
+ while (isEmpty(result)) {
56
58
  if (remaining <= 0) {
57
59
  throw new Error(`Selector '${selector}' not found.`);
58
60
  }
59
61
  await sleep(50);
60
62
  remaining -= 50;
63
+ result = await query();
61
64
  }
65
+ return result;
66
+ };
67
+ const querySelectorAll = async (selector, timeoutMs = 5_000) => poll(selector, timeoutMs, () => parser.querySelectorAll(context, selector), (matches) => matches.length === 0);
68
+ const waitForSelector = async (selector, timeoutMs) => {
69
+ await querySelectorAll(selector, timeoutMs);
62
70
  };
63
71
  return {
64
72
  extractLinks,
65
73
  waitForSelector,
74
+ querySelectorAll,
75
+ querySelectorOne: async (selector, timeoutMs = 5_000) => poll(selector, timeoutMs, () => parser.querySelectorOne(context, selector), (match) => match === undefined),
66
76
  enqueueLinks: async (options = {}) => {
67
77
  const baseUrl = resolveBaseUrlForEnqueueLinksFiltering({
68
78
  enqueueStrategy: options.strategy,
@@ -77,16 +87,15 @@ export class DOMCrawler extends HttpCrawler {
77
87
  strategy: options.strategy ?? EnqueueStrategy.SameHostname,
78
88
  });
79
89
  },
80
- async parseWithCheerio(selector, _timeoutMs = 5_000) {
90
+ async parseWithCheerio(selector, timeoutMs) {
91
+ if (selector) {
92
+ await waitForSelector(selector, timeoutMs);
93
+ }
81
94
  // Import full cheerio (not cheerio/slim) to be browser-compliant for DOM Crawlers (not CheerioCrawler).
82
- const $ = (await parser.toCheerio?.(context)) ??
95
+ return ((await parser.toCheerio?.(context)) ??
83
96
  (await import('cheerio')).load(context.body, {
84
97
  xmlMode: context.contentType.type.includes('xml'),
85
- });
86
- if (selector && $(selector).get().length === 0) {
87
- throw new Error(`Selector '${selector}' not found.`);
88
- }
89
- return $;
98
+ }));
90
99
  },
91
100
  };
92
101
  }
@@ -1,6 +1,7 @@
1
1
  import type { BasicCrawlerOptions, CrawlingContext, CrawlingRequest, LoadedRequest } from '@crawlee/basic';
2
2
  import { BasicCrawler } from '@crawlee/basic';
3
3
  import type { Dictionary } from '@crawlee/types';
4
+ import { z } from 'zod';
4
5
  import type { ErrorHandler, GetUserDataFromRequest, RequestHandler, RouterHandler, RouterRoutes, RouteSchemas, RoutesFromSchemas } from '../index.js';
5
6
  export type FileDownloadErrorHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
6
7
  ContextExtension = Dictionary<never>> = ErrorHandler<CrawlingContext, FileDownloadCrawlingContext<UserData> & ContextExtension>;
@@ -13,6 +14,20 @@ export interface FileDownloadCrawlingContext<UserData extends Dictionary = any>
13
14
  };
14
15
  }
15
16
  export type FileDownloadRequestHandler<UserData extends Dictionary = any> = RequestHandler<FileDownloadCrawlingContext<UserData>>;
17
+ export interface FileDownloadOptions extends Omit<BasicCrawlerOptions<FileDownloadCrawlingContext>, 'contextPipelineBuilder'> {
18
+ /**
19
+ * If set to `true`, TLS/SSL certificate errors are ignored. Forwarded to the HTTP client as
20
+ * {@link SendRequestOptions.ignoreTlsErrors|`ignoreTlsErrors`} on every download, so custom
21
+ * {@link BaseHttpClient} implementations should honor that flag (the built-in impit and got-scraping
22
+ * clients do; the native fetch fallback cannot disable TLS verification and warns instead).
23
+ *
24
+ * Certificates are verified by default. Sessions whose proxy is a MITM proxy
25
+ * (`session.proxyInfo.ignoreTlsErrors`) skip verification regardless of this option.
26
+ *
27
+ * @default false
28
+ */
29
+ ignoreTlsErrors?: boolean;
30
+ }
16
31
  /**
17
32
  * Provides a framework for downloading files in parallel using plain HTTP requests. The URLs to download are fed either from a static list of URLs or they can be added on the fly from another crawler.
18
33
  *
@@ -20,7 +35,7 @@ export type FileDownloadRequestHandler<UserData extends Dictionary = any> = Requ
20
35
  * However, it doesn't parse the content - if you need to e.g. extract data from the downloaded files,
21
36
  * you might need to use {@link CheerioCrawler}, {@link PuppeteerCrawler} or {@link PlaywrightCrawler} instead.
22
37
  *
23
- * `FileCrawler` downloads each URL using a plain HTTP request and then invokes the user-provided {@link BasicCrawlerOptions.requestHandler} where the user can specify what to do with the downloaded data.
38
+ * `FileDownload` downloads each URL using a plain HTTP request and then invokes the user-provided {@link BasicCrawlerOptions.requestHandler} where the user can specify what to do with the downloaded data.
24
39
  *
25
40
  * The source URLs are represented using {@link Request} objects that are fed from the {@link IRequestManager|request manager} provided via the {@link BasicCrawlerOptions.requestManager|`requestManager`} constructor option (a {@link RequestQueue} is itself a request manager). To read from a read-only source such as a {@link RequestList} while still being able to enqueue new requests, combine it with a queue into a {@link RequestManagerTandem} via {@link IRequestLoader.toTandem|`requestLoader.toTandem()`} and pass the result as `requestManager`.
26
41
  *
@@ -28,14 +43,14 @@ export type FileDownloadRequestHandler<UserData extends Dictionary = any> = Requ
28
43
  *
29
44
  * The crawler finishes when there are no more {@link Request} objects to crawl.
30
45
  *
31
- * New requests are only dispatched when there is enough free CPU and memory available, as judged by the crawler's {@link ConcurrencySystem}. Concurrency is tuned via the `minConcurrency`, `maxConcurrency` and `maxRequestsPerMinute` options of the `FileCrawler` constructor, or, for finer control, by injecting a pre-configured {@link ConcurrencySystem|`concurrencySystem`}.
46
+ * New requests are only dispatched when there is enough free CPU and memory available, as judged by the crawler's {@link ConcurrencySystem}. Concurrency is tuned via the `minConcurrency`, `maxConcurrency` and `maxRequestsPerMinute` options of the `FileDownload` constructor, or, for finer control, by injecting a pre-configured {@link ConcurrencySystem|`concurrencySystem`}.
32
47
  *
33
48
  * ## Example usage
34
49
  *
35
50
  * ```ts
36
- * const crawler = new FileDownloader({
37
- * requestHandler({ body, request }) {
38
- * writeFileSync(request.url.replace(/[^a-z0-9\.]/gi, '_'), body);
51
+ * const crawler = new FileDownload({
52
+ * async requestHandler({ response, request }) {
53
+ * await writeFile(request.url.replace(/[^a-z0-9\.]/gi, '_'), Buffer.from(await response.arrayBuffer()));
39
54
  * },
40
55
  * });
41
56
  *
@@ -48,7 +63,109 @@ export type FileDownloadRequestHandler<UserData extends Dictionary = any> = Requ
48
63
  */
49
64
  export declare class FileDownload extends BasicCrawler<FileDownloadCrawlingContext> {
50
65
  #private;
51
- constructor(options?: Omit<BasicCrawlerOptions<FileDownloadCrawlingContext>, 'contextPipelineBuilder'>);
66
+ /**
67
+ * @internal
68
+ */
69
+ protected static optionsShape: {
70
+ contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
71
+ extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
72
+ requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
73
+ requestQueue: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
74
+ requestManager: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
75
+ requestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
76
+ requestHandlerTimeoutSecs: z.ZodOptional<z.ZodCustom<number, number>>;
77
+ errorHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
78
+ failedRequestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
79
+ maxRequestRetries: z.ZodDefault<z.ZodCustom<number, number>>;
80
+ sameDomainDelaySecs: z.ZodDefault<z.ZodCustom<number, number>>;
81
+ maxRequestsPerCrawl: z.ZodOptional<z.ZodCustom<number, number>>;
82
+ maxCrawlDepth: z.ZodOptional<z.ZodCustom<number, number>>;
83
+ taskLoopOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
84
+ concurrencySystem: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
85
+ sessionPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
86
+ proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
87
+ statusMessageLoggingInterval: z.ZodDefault<z.ZodCustom<number, number>>;
88
+ statusMessageCallback: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
89
+ additionalHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
90
+ ignoreHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
91
+ blockedStatusCodes: z.ZodOptional<z.ZodArray<z.ZodCustom<number, number>>>;
92
+ retryOnBlocked: z.ZodDefault<z.ZodBoolean>;
93
+ respectRobotsTxtFile: z.ZodDefault<z.ZodUnion<readonly [z.ZodBoolean, z.ZodCustom<Dictionary, Dictionary>]>>;
94
+ transactionalStorage: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodObject<{
95
+ requestQueue: z.ZodOptional<z.ZodEnum<{
96
+ deferred: "deferred";
97
+ writeThrough: "writeThrough";
98
+ }>>;
99
+ }, z.core.$strict>]>>;
100
+ onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
101
+ // @ts-ignore optional peer dependency or compatibility with es2022
102
+ httpClient: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/http-client").BaseHttpClient>>;
103
+ // @ts-ignore optional peer dependency or compatibility with es2022
104
+ configuration: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/basic").Configuration>>;
105
+ storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
106
+ // @ts-ignore optional peer dependency or compatibility with es2022
107
+ eventManager: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/basic").EventManager>>;
108
+ logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
109
+ minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
110
+ maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
111
+ initialConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
112
+ maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;
113
+ keepAlive: z.ZodOptional<z.ZodBoolean>;
114
+ statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
115
+ id: z.ZodOptional<z.ZodString>;
116
+ ignoreTlsErrors: z.ZodDefault<z.ZodBoolean>;
117
+ };
118
+ /** @internal */
119
+ protected static optionsSchema: z.ZodObject<{
120
+ contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
121
+ extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
122
+ requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
123
+ requestQueue: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
124
+ requestManager: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
125
+ requestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
126
+ requestHandlerTimeoutSecs: z.ZodOptional<z.ZodCustom<number, number>>;
127
+ errorHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
128
+ failedRequestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
129
+ maxRequestRetries: z.ZodDefault<z.ZodCustom<number, number>>;
130
+ sameDomainDelaySecs: z.ZodDefault<z.ZodCustom<number, number>>;
131
+ maxRequestsPerCrawl: z.ZodOptional<z.ZodCustom<number, number>>;
132
+ maxCrawlDepth: z.ZodOptional<z.ZodCustom<number, number>>;
133
+ taskLoopOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
134
+ concurrencySystem: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
135
+ sessionPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
136
+ proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
137
+ statusMessageLoggingInterval: z.ZodDefault<z.ZodCustom<number, number>>;
138
+ statusMessageCallback: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
139
+ additionalHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
140
+ ignoreHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
141
+ blockedStatusCodes: z.ZodOptional<z.ZodArray<z.ZodCustom<number, number>>>;
142
+ retryOnBlocked: z.ZodDefault<z.ZodBoolean>;
143
+ respectRobotsTxtFile: z.ZodDefault<z.ZodUnion<readonly [z.ZodBoolean, z.ZodCustom<Dictionary, Dictionary>]>>;
144
+ transactionalStorage: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodObject<{
145
+ requestQueue: z.ZodOptional<z.ZodEnum<{
146
+ deferred: "deferred";
147
+ writeThrough: "writeThrough";
148
+ }>>;
149
+ }, z.core.$strict>]>>;
150
+ onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
151
+ // @ts-ignore optional peer dependency or compatibility with es2022
152
+ httpClient: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/http-client").BaseHttpClient>>;
153
+ // @ts-ignore optional peer dependency or compatibility with es2022
154
+ configuration: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/basic").Configuration>>;
155
+ storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
156
+ // @ts-ignore optional peer dependency or compatibility with es2022
157
+ eventManager: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/basic").EventManager>>;
158
+ logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
159
+ minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
160
+ maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
161
+ initialConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
162
+ maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;
163
+ keepAlive: z.ZodOptional<z.ZodBoolean>;
164
+ statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
165
+ id: z.ZodOptional<z.ZodString>;
166
+ ignoreTlsErrors: z.ZodDefault<z.ZodBoolean>;
167
+ }, z.core.$strict>;
168
+ constructor(options?: FileDownloadOptions);
52
169
  }
53
170
  /**
54
171
  * Creates new {@link Router} instance that works based on request labels.
@@ -1,5 +1,7 @@
1
1
  import { BasicCrawler, ContextPipeline } from '@crawlee/basic';
2
2
  import { ResponseWithUrl } from '@crawlee/http-client';
3
+ import { parseArgument } from '@crawlee/utils/internal';
4
+ import { z } from 'zod';
3
5
  import { Router } from '../index.js';
4
6
  import { parseContentTypeFromResponse } from './utils.js';
5
7
  /**
@@ -9,7 +11,7 @@ import { parseContentTypeFromResponse } from './utils.js';
9
11
  * However, it doesn't parse the content - if you need to e.g. extract data from the downloaded files,
10
12
  * you might need to use {@link CheerioCrawler}, {@link PuppeteerCrawler} or {@link PlaywrightCrawler} instead.
11
13
  *
12
- * `FileCrawler` downloads each URL using a plain HTTP request and then invokes the user-provided {@link BasicCrawlerOptions.requestHandler} where the user can specify what to do with the downloaded data.
14
+ * `FileDownload` downloads each URL using a plain HTTP request and then invokes the user-provided {@link BasicCrawlerOptions.requestHandler} where the user can specify what to do with the downloaded data.
13
15
  *
14
16
  * The source URLs are represented using {@link Request} objects that are fed from the {@link IRequestManager|request manager} provided via the {@link BasicCrawlerOptions.requestManager|`requestManager`} constructor option (a {@link RequestQueue} is itself a request manager). To read from a read-only source such as a {@link RequestList} while still being able to enqueue new requests, combine it with a queue into a {@link RequestManagerTandem} via {@link IRequestLoader.toTandem|`requestLoader.toTandem()`} and pass the result as `requestManager`.
15
17
  *
@@ -17,14 +19,14 @@ import { parseContentTypeFromResponse } from './utils.js';
17
19
  *
18
20
  * The crawler finishes when there are no more {@link Request} objects to crawl.
19
21
  *
20
- * New requests are only dispatched when there is enough free CPU and memory available, as judged by the crawler's {@link ConcurrencySystem}. Concurrency is tuned via the `minConcurrency`, `maxConcurrency` and `maxRequestsPerMinute` options of the `FileCrawler` constructor, or, for finer control, by injecting a pre-configured {@link ConcurrencySystem|`concurrencySystem`}.
22
+ * New requests are only dispatched when there is enough free CPU and memory available, as judged by the crawler's {@link ConcurrencySystem}. Concurrency is tuned via the `minConcurrency`, `maxConcurrency` and `maxRequestsPerMinute` options of the `FileDownload` constructor, or, for finer control, by injecting a pre-configured {@link ConcurrencySystem|`concurrencySystem`}.
21
23
  *
22
24
  * ## Example usage
23
25
  *
24
26
  * ```ts
25
- * const crawler = new FileDownloader({
26
- * requestHandler({ body, request }) {
27
- * writeFileSync(request.url.replace(/[^a-z0-9\.]/gi, '_'), body);
27
+ * const crawler = new FileDownload({
28
+ * async requestHandler({ response, request }) {
29
+ * await writeFile(request.url.replace(/[^a-z0-9\.]/gi, '_'), Buffer.from(await response.arrayBuffer()));
28
30
  * },
29
31
  * });
30
32
  *
@@ -36,12 +38,24 @@ import { parseContentTypeFromResponse } from './utils.js';
36
38
  * ```
37
39
  */
38
40
  export class FileDownload extends BasicCrawler {
41
+ #ignoreTlsErrors;
42
+ /**
43
+ * @internal
44
+ */
45
+ static optionsShape = {
46
+ ...BasicCrawler.optionsShape,
47
+ ignoreTlsErrors: z.boolean().default(false),
48
+ };
49
+ /** @internal */
50
+ static optionsSchema = z.strictObject(FileDownload.optionsShape);
39
51
  // TODO hooks
40
52
  constructor(options = {}) {
53
+ const { ignoreTlsErrors, ...basicCrawlerOptions } = parseArgument(options, FileDownload.optionsSchema, 'FileDownloadOptions');
41
54
  super({
42
- ...options,
55
+ ...basicCrawlerOptions,
43
56
  contextPipelineBuilder: () => this.#buildContextPipeline(),
44
57
  });
58
+ this.#ignoreTlsErrors = ignoreTlsErrors;
45
59
  }
46
60
  #buildContextPipeline() {
47
61
  return ContextPipeline.create().compose(this.#initiateDownload.bind(this));
@@ -49,6 +63,7 @@ export class FileDownload extends BasicCrawler {
49
63
  async #initiateDownload(context, onCleanup) {
50
64
  const response = await this.httpClient.sendRequest(context.request.intoFetchAPIRequest(), {
51
65
  session: context.session,
66
+ ignoreTlsErrors: this.#ignoreTlsErrors,
52
67
  });
53
68
  const { type, charset: encoding } = parseContentTypeFromResponse(response);
54
69
  context.request.url = response.url;
@@ -1,7 +1,8 @@
1
1
  import type { BasicCrawlerOptions, ConcurrencySystem, ConcurrencySystemOptions, CrawlingContext, ErrorHandler, GetUserDataFromRequest, LoadedRequest, CrawlingRequest, RequireContextPipeline, RequestHandler, RouterHandler, RouterRoutes, RouteSchemas, RoutesFromSchemas } from '@crawlee/basic';
2
2
  import { BasicCrawler, ContextPipeline } from '@crawlee/basic';
3
3
  import type { Awaitable, Dictionary } from '@crawlee/types';
4
- import type { CheerioAPI } from 'cheerio';
4
+ import type { Cheerio, CheerioAPI } from 'cheerio';
5
+ import type { AnyNode } from 'domhandler';
5
6
  import type { JsonValue } from 'type-fest';
6
7
  import { z } from 'zod';
7
8
  /**
@@ -35,7 +36,7 @@ export interface HttpCrawlerOptions<Context extends InternalHttpCrawlingContext
35
36
  * {@link BaseHttpClient} implementations should honor that flag (the built-in impit and got-scraping
36
37
  * clients do; the native fetch fallback cannot disable TLS verification and warns instead).
37
38
  *
38
- * @default true
39
+ * @default false
39
40
  */
40
41
  ignoreTlsErrors?: boolean;
41
42
  /**
@@ -162,6 +163,17 @@ JSONData extends JsonValue = any> extends CrawlingContextWithResponse<UserData>
162
163
  * ```
163
164
  */
164
165
  waitForSelector(selector: string, timeoutMs?: number): Promise<void>;
166
+ /**
167
+ * Like {@link InternalHttpCrawlingContext.waitForSelector|`waitForSelector`}, but returns the matched
168
+ * elements - a Cheerio object here, whatever the parser produces in {@link DOMCrawler}. Throws when nothing
169
+ * matches.
170
+ */
171
+ querySelectorAll(selector: string, timeoutMs?: number): Promise<ArrayLike<unknown>>;
172
+ /**
173
+ * Like {@link InternalHttpCrawlingContext.querySelectorAll|`querySelectorAll`}, but returns only the first
174
+ * match.
175
+ */
176
+ querySelectorOne(selector: string, timeoutMs?: number): Promise<unknown>;
165
177
  /**
166
178
  * Returns Cheerio handle for `page.content()`, allowing to work with the data same way as with {@link CheerioCrawler}.
167
179
  * This is here to unify the crawler API, so they all have this handy method - in {@link CheerioCrawler} it has
@@ -182,6 +194,8 @@ JSONData extends JsonValue = any> extends CrawlingContextWithResponse<UserData>
182
194
  parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioAPI>;
183
195
  }
184
196
  export interface HttpCrawlingContext<UserData extends Dictionary = any, JSONData extends JsonValue = any> extends InternalHttpCrawlingContext<UserData, JSONData> {
197
+ querySelectorAll(selector: string, timeoutMs?: number): Promise<Cheerio<AnyNode>>;
198
+ querySelectorOne(selector: string, timeoutMs?: number): Promise<Cheerio<AnyNode>>;
185
199
  }
186
200
  export type HttpRequestHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
187
201
  JSONData extends JsonValue = any> = RequestHandler<HttpCrawlingContext<UserData, JSONData>>;
@@ -1,13 +1,13 @@
1
1
  import { Readable } from 'node:stream';
2
2
  import util from 'node:util';
3
- import { BasicCrawler, ContextPipeline, getCookiesFromResponse, NavigationSkippedError, remainingNavigationWindowMillis, RequestState, RequestThrottledError, Router, SessionError, } from '@crawlee/basic';
3
+ import { BasicCrawler, ContextPipeline, getCookiesFromResponse, NavigationSkippedError, skippedNavigationRequest, remainingNavigationWindowMillis, RequestState, RequestThrottledError, Router, SessionError, } from '@crawlee/basic';
4
4
  import { ResponseWithUrl } from '@crawlee/http-client';
5
- import { parseArgument, RETRY_CSS_SELECTORS, schemas } from '@crawlee/utils/internal';
5
+ import { parseArgument, processHttpRequestOptions, RETRY_CSS_SELECTORS, schemas } from '@crawlee/utils/internal';
6
6
  import { isTypeValid, parse as parseContentType } from 'content-type';
7
7
  import iconv from 'iconv-lite';
8
8
  import { z } from 'zod';
9
9
  import { addTimeoutToPromise, storage, TimeoutError, tryCancel } from '@apify/timeout';
10
- import { extractCharsetFromHtmlBytes, parseContentTypeFromResponse, processHttpRequestOptions } from './utils.js';
10
+ import { extractCharsetFromHtmlBytes, parseContentTypeFromResponse } from './utils.js';
11
11
  /**
12
12
  * Default mime types, which HttpScraper supports.
13
13
  */
@@ -117,13 +117,14 @@ export class HttpCrawler extends BasicCrawler {
117
117
  #suggestResponseEncoding;
118
118
  #forceResponseEncoding;
119
119
  #supportedMimeTypes;
120
+ #downloadAbortControllers = new WeakMap();
120
121
  /**
121
122
  * @internal
122
123
  */
123
124
  static optionsShape = {
124
125
  ...BasicCrawler.optionsShape,
125
126
  navigationTimeoutSecs: schemas.anyNumber.default(30),
126
- ignoreTlsErrors: z.boolean().default(true),
127
+ ignoreTlsErrors: z.boolean().default(false),
127
128
  additionalMimeTypes: schemas.arrayOf(z.string(), 'strings').default(() => []),
128
129
  suggestResponseEncoding: z.string().optional(),
129
130
  forceResponseEncoding: z.string().optional(),
@@ -207,17 +208,7 @@ export class HttpCrawler extends BasicCrawler {
207
208
  const { request } = crawlingContext;
208
209
  if (request.skipNavigation) {
209
210
  return {
210
- request: new Proxy(request, {
211
- get(target, propertyName) {
212
- if (propertyName === 'loadedUrl') {
213
- throw new NavigationSkippedError('The `request.loadedUrl` property is not available - `skipNavigation` was used');
214
- }
215
- // `target` as receiver and bound methods, or `#` members throw on the proxy
216
- const value = Reflect.get(target, propertyName, target);
217
- return typeof value === 'function' ? value.bind(target) : value;
218
- },
219
- set: (target, propertyName, value) => Reflect.set(target, propertyName, value, target),
220
- }),
211
+ request: skippedNavigationRequest(request),
221
212
  get response() {
222
213
  throw new NavigationSkippedError('The `response` property is not available - `skipNavigation` was used');
223
214
  },
@@ -231,10 +222,12 @@ export class HttpCrawler extends BasicCrawler {
231
222
  tryCancel();
232
223
  const { request, session } = crawlingContext;
233
224
  const proxyUrl = crawlingContext.proxyInfo?.url;
225
+ const downloadAbortController = new AbortController();
226
+ this.#downloadAbortControllers.set(request, downloadAbortController);
234
227
  // Bound the request by whatever is left of the shared navigation window (the pre-navigation hooks may
235
228
  // have already spent part of it), so it produces a clean navigation-timeout error rather than the raw
236
229
  // client abort.
237
- const httpResponse = await addTimeoutToPromise(async () => this.#requestFunction({ request, session, proxyUrl }), Math.max(1, remainingNavigationWindowMillis(crawlingContext, this.#navigationTimeoutMillis)), `Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);
230
+ const httpResponse = await addTimeoutToPromise(async () => this.#requestFunction({ request, session, proxyUrl, signal: downloadAbortController.signal }), Math.max(1, remainingNavigationWindowMillis(crawlingContext, this.#navigationTimeoutMillis)), `Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);
238
231
  tryCancel();
239
232
  request.loadedUrl = httpResponse?.url;
240
233
  request.state = RequestState.AFTER_NAV;
@@ -258,6 +251,12 @@ export class HttpCrawler extends BasicCrawler {
258
251
  get parseWithCheerio() {
259
252
  throw new NavigationSkippedError('The `parseWithCheerio` method is not available - `skipNavigation` was used');
260
253
  },
254
+ get querySelectorAll() {
255
+ throw new NavigationSkippedError('The `querySelectorAll` method is not available - `skipNavigation` was used');
256
+ },
257
+ get querySelectorOne() {
258
+ throw new NavigationSkippedError('The `querySelectorOne` method is not available - `skipNavigation` was used');
259
+ },
261
260
  };
262
261
  }
263
262
  tryCancel();
@@ -279,7 +278,11 @@ export class HttpCrawler extends BasicCrawler {
279
278
  if (remaining <= 0) {
280
279
  throw new TimeoutError(`Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);
281
280
  }
282
- const parsed = await addTimeoutToPromise(async () => this.#parseResponse(crawlingContext.request, crawlingContext.response), remaining, `Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);
281
+ const parsed = await addTimeoutToPromise(async () => this.#parseResponse(crawlingContext.request, crawlingContext.response), remaining, `Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`).catch((error) => {
282
+ // Stop the download along with the read, so a body that never ends doesn't outlive the request.
283
+ this.#downloadAbortControllers.get(crawlingContext.request)?.abort();
284
+ throw error;
285
+ });
283
286
  tryCancel();
284
287
  const response = parsed.response;
285
288
  const contentType = parsed.contentType;
@@ -287,11 +290,16 @@ export class HttpCrawler extends BasicCrawler {
287
290
  const { load } = await import('cheerio/slim');
288
291
  return load(parsed.body.toString(), { xmlMode: contentType.type.includes('xml') });
289
292
  };
290
- const waitForSelector = async (selector, _timeoutMs) => {
291
- const $ = await loadBody();
292
- if ($(selector).get().length === 0) {
293
+ const querySelectorAll = async (selector, _timeoutMs) => {
294
+ const matches = (await loadBody())(selector);
295
+ if (matches.length === 0) {
293
296
  throw new Error(`Selector '${selector}' not found.`);
294
297
  }
298
+ return matches;
299
+ };
300
+ const querySelectorOne = async (selector, timeoutMs) => (await querySelectorAll(selector, timeoutMs)).first();
301
+ const waitForSelector = async (selector, timeoutMs) => {
302
+ await querySelectorAll(selector, timeoutMs);
295
303
  };
296
304
  const parseWithCheerio = async (selector, timeoutMs) => {
297
305
  const $ = await loadBody();
@@ -328,6 +336,8 @@ export class HttpCrawler extends BasicCrawler {
328
336
  return JSON.parse(jsonString);
329
337
  },
330
338
  waitForSelector,
339
+ querySelectorOne,
340
+ querySelectorAll,
331
341
  parseWithCheerio,
332
342
  contentType,
333
343
  body: parsed.body,
@@ -359,10 +369,10 @@ export class HttpCrawler extends BasicCrawler {
359
369
  * on the request such as only downloading the request body if the
360
370
  * received content type matches text/html, application/xml, application/xhtml+xml.
361
371
  */
362
- async #requestFunction({ request, session, proxyUrl }) {
372
+ async #requestFunction({ request, session, proxyUrl, signal }) {
363
373
  const opts = this.#getRequestOptions(request, proxyUrl);
364
374
  try {
365
- return await this.#requestAsBrowser(opts, session);
375
+ return await this.#requestAsBrowser(opts, session, signal);
366
376
  }
367
377
  catch (e) {
368
378
  if (e instanceof Error && e.constructor.name === 'TimeoutError') {
@@ -515,7 +525,7 @@ export class HttpCrawler extends BasicCrawler {
515
525
  /**
516
526
  * @internal wraps public utility for mocking purposes
517
527
  */
518
- #requestAsBrowser = async (options, session) => {
528
+ #requestAsBrowser = async (options, session, signal) => {
519
529
  const opts = processHttpRequestOptions({
520
530
  ...options,
521
531
  responseType: 'text',
@@ -526,10 +536,11 @@ export class HttpCrawler extends BasicCrawler {
526
536
  const cookieJar = this.#saveResponseCookies ? session.cookieJar : await session.cookieJar.clone();
527
537
  // Bind the request to the shared navigation window instead of a fixed per-request timeout, so
528
538
  // `extendTimeout()` can push the deadline and a fixed `AbortSignal.timeout` won't fire on its own and
529
- // kill a lazily-read body mid-extension. This aborts the socket only during the header phase; the body
530
- // read is bounded separately at the promise level (see `processHttpResponse`), so a slow-streaming body
531
- // still fails cleanly with a navigation timeout, though the socket is left to close on its own.
539
+ // kill a lazily-read body mid-extension. The window's signal only covers the header phase; the body read is
540
+ // bounded separately (see `processHttpResponse`), which aborts `signal` when it fails, so a slow-streaming
541
+ // body still fails cleanly with a navigation timeout and its download stops too.
532
542
  const cancelSignal = storage.getStore()?.cancelTask.signal;
543
+ const requestSignal = cancelSignal ? AbortSignal.any([cancelSignal, signal]) : signal;
533
544
  const response = await this.httpClient.sendRequest(new Request(opts.url, {
534
545
  body: opts.body ? Readable.toWeb(opts.body) : undefined,
535
546
  headers: new Headers(opts.headers),
@@ -540,7 +551,7 @@ export class HttpCrawler extends BasicCrawler {
540
551
  session,
541
552
  cookieJar,
542
553
  proxyUrl: opts.proxyUrl,
543
- signal: cancelSignal,
554
+ signal: requestSignal,
544
555
  timeoutMillis: cancelSignal ? undefined : opts.timeout,
545
556
  ignoreTlsErrors: this.#ignoreTlsErrors,
546
557
  });
@@ -1,8 +1,3 @@
1
- import type { HttpRequest, HttpRequestOptions } from '@crawlee/types';
2
- /**
3
- * Converts {@link HttpRequestOptions} to a {@link HttpRequest}.
4
- */
5
- export declare function processHttpRequestOptions({ searchParams, form, json, username, password, ...request }: HttpRequestOptions): HttpRequest;
6
1
  /**
7
2
  * Scans the first 1024 bytes of an HTML document (as latin1) to extract the charset
8
3
  * declared via `<meta charset>` or `<meta http-equiv="Content-Type" content="...;charset=...">`.
@@ -1,44 +1,9 @@
1
1
  import { extname } from 'node:path';
2
- import { Readable } from 'node:stream';
3
- import { applySearchParams, parseArgument } from '@crawlee/utils/internal';
2
+ import { parseArgument } from '@crawlee/utils/internal';
4
3
  import { isTypeValid, parse as parseContentType } from 'content-type';
5
4
  import mime from 'mime-types';
6
5
  import { z } from 'zod';
7
6
  const responseWithUrlSchema = z.looseObject({ url: z.url(), headers: z.looseObject({}) });
8
- /**
9
- * Converts {@link HttpRequestOptions} to a {@link HttpRequest}.
10
- */
11
- export function processHttpRequestOptions({ searchParams, form, json, username, password, ...request }) {
12
- const url = new URL(request.url);
13
- const headers = new Headers(request.headers);
14
- applySearchParams(url, searchParams);
15
- if ([request.body, form, json].filter((value) => value !== undefined).length > 1) {
16
- throw new Error('At most one of `body`, `form` and `json` may be specified in sendRequest arguments');
17
- }
18
- const body = (() => {
19
- if (form !== undefined) {
20
- return Readable.from(new URLSearchParams(form).toString());
21
- }
22
- if (json !== undefined) {
23
- return Readable.from(JSON.stringify(json));
24
- }
25
- if (request.body !== undefined) {
26
- return Readable.from(request.body);
27
- }
28
- return undefined;
29
- })();
30
- if (form !== undefined && !headers.has('content-type')) {
31
- headers.set('content-type', 'application/x-www-form-urlencoded');
32
- }
33
- if (json !== undefined && !headers.has('content-type')) {
34
- headers.set('content-type', 'application/json');
35
- }
36
- if (username !== undefined || password !== undefined) {
37
- const encodedAuth = Buffer.from(`${username ?? ''}:${password ?? ''}`).toString('base64');
38
- headers.set('authorization', `Basic ${encodedAuth}`);
39
- }
40
- return { ...request, body, url, headers };
41
- }
42
7
  /**
43
8
  * Scans the first 1024 bytes of an HTML document (as latin1) to extract the charset
44
9
  * declared via `<meta charset>` or `<meta http-equiv="Content-Type" content="...;charset=...">`.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@crawlee/http",
3
- "version": "4.0.0-rc.1",
3
+ "version": "4.0.0-rc.2",
4
4
  "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
5
5
  "engines": {
6
6
  "node": ">=22.13.0"
@@ -48,17 +48,18 @@
48
48
  },
49
49
  "dependencies": {
50
50
  "@apify/timeout": "^1.0.1",
51
- "@crawlee/basic": "4.0.0-rc.1",
52
- "@crawlee/http-client": "4.0.0-rc.1",
53
- "@crawlee/types": "4.0.0-rc.1",
54
- "@crawlee/utils": "4.0.0-rc.1",
55
- "cheerio": "^1.0.0",
51
+ "@crawlee/basic": "4.0.0-rc.2",
52
+ "@crawlee/http-client": "4.0.0-rc.2",
53
+ "@crawlee/types": "4.0.0-rc.2",
54
+ "@crawlee/utils": "4.0.0-rc.2",
55
+ "cheerio": "^1.2.0",
56
56
  "content-type": "^3.1.1",
57
- "iconv-lite": "^0.7.2",
58
- "mime-types": "^3.0.1",
59
- "tslib": "^2.8.1",
57
+ "domhandler": "^6.0.1",
58
+ "iconv-lite": "^0.7.3",
59
+ "mime-types": "^3.0.2",
60
+ "tslib": "^2.9.1",
60
61
  "type-fest": "^4.41.0",
61
- "zod": "^4.5.4"
62
+ "zod": "^4.6.7"
62
63
  },
63
64
  "lerna": {
64
65
  "command": {
@@ -67,5 +68,5 @@
67
68
  }
68
69
  }
69
70
  },
70
- "gitHead": "f354ca5e943bed1a5c657a1fce1974c053f9fff0"
71
+ "gitHead": "a9b623229cd7d1bff5063fc82940efebaea47165"
71
72
  }