@crawlee/http 4.0.0-beta.277 → 4.0.0-beta.278

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,7 @@
1
1
  import type { BasicCrawlerOptions, CrawlingContext, CrawlingRequest, LoadedRequest } from '@crawlee/basic';
2
2
  import { BasicCrawler } from '@crawlee/basic';
3
3
  import type { Dictionary } from '@crawlee/types';
4
+ import { z } from 'zod';
4
5
  import type { ErrorHandler, GetUserDataFromRequest, RequestHandler, RouterHandler, RouterRoutes, RouteSchemas, RoutesFromSchemas } from '../index.js';
5
6
  export type FileDownloadErrorHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
6
7
  ContextExtension = Dictionary<never>> = ErrorHandler<CrawlingContext, FileDownloadCrawlingContext<UserData> & ContextExtension>;
@@ -13,6 +14,20 @@ export interface FileDownloadCrawlingContext<UserData extends Dictionary = any>
13
14
  };
14
15
  }
15
16
  export type FileDownloadRequestHandler<UserData extends Dictionary = any> = RequestHandler<FileDownloadCrawlingContext<UserData>>;
17
+ export interface FileDownloadOptions extends Omit<BasicCrawlerOptions<FileDownloadCrawlingContext>, 'contextPipelineBuilder'> {
18
+ /**
19
+ * If set to `true`, TLS/SSL certificate errors are ignored. Forwarded to the HTTP client as
20
+ * {@link SendRequestOptions.ignoreTlsErrors|`ignoreTlsErrors`} on every download, so custom
21
+ * {@link BaseHttpClient} implementations should honor that flag (the built-in impit and got-scraping
22
+ * clients do; the native fetch fallback cannot disable TLS verification and warns instead).
23
+ *
24
+ * Certificates are verified by default. Sessions whose proxy is a MITM proxy
25
+ * (`session.proxyInfo.ignoreTlsErrors`) skip verification regardless of this option.
26
+ *
27
+ * @default false
28
+ */
29
+ ignoreTlsErrors?: boolean;
30
+ }
16
31
  /**
17
32
  * Provides a framework for downloading files in parallel using plain HTTP requests. The URLs to download are fed either from a static list of URLs or they can be added on the fly from another crawler.
18
33
  *
@@ -20,7 +35,7 @@ export type FileDownloadRequestHandler<UserData extends Dictionary = any> = Requ
20
35
  * However, it doesn't parse the content - if you need to e.g. extract data from the downloaded files,
21
36
  * you might need to use {@link CheerioCrawler}, {@link PuppeteerCrawler} or {@link PlaywrightCrawler} instead.
22
37
  *
23
- * `FileCrawler` downloads each URL using a plain HTTP request and then invokes the user-provided {@link BasicCrawlerOptions.requestHandler} where the user can specify what to do with the downloaded data.
38
+ * `FileDownload` downloads each URL using a plain HTTP request and then invokes the user-provided {@link BasicCrawlerOptions.requestHandler} where the user can specify what to do with the downloaded data.
24
39
  *
25
40
  * The source URLs are represented using {@link Request} objects that are fed from the {@link IRequestManager|request manager} provided via the {@link BasicCrawlerOptions.requestManager|`requestManager`} constructor option (a {@link RequestQueue} is itself a request manager). To read from a read-only source such as a {@link RequestList} while still being able to enqueue new requests, combine it with a queue into a {@link RequestManagerTandem} via {@link IRequestLoader.toTandem|`requestLoader.toTandem()`} and pass the result as `requestManager`.
26
41
  *
@@ -28,14 +43,14 @@ export type FileDownloadRequestHandler<UserData extends Dictionary = any> = Requ
28
43
  *
29
44
  * The crawler finishes when there are no more {@link Request} objects to crawl.
30
45
  *
31
- * New requests are only dispatched when there is enough free CPU and memory available, as judged by the crawler's {@link ConcurrencySystem}. Concurrency is tuned via the `minConcurrency`, `maxConcurrency` and `maxRequestsPerMinute` options of the `FileCrawler` constructor, or, for finer control, by injecting a pre-configured {@link ConcurrencySystem|`concurrencySystem`}.
46
+ * New requests are only dispatched when there is enough free CPU and memory available, as judged by the crawler's {@link ConcurrencySystem}. Concurrency is tuned via the `minConcurrency`, `maxConcurrency` and `maxRequestsPerMinute` options of the `FileDownload` constructor, or, for finer control, by injecting a pre-configured {@link ConcurrencySystem|`concurrencySystem`}.
32
47
  *
33
48
  * ## Example usage
34
49
  *
35
50
  * ```ts
36
- * const crawler = new FileDownloader({
37
- * requestHandler({ body, request }) {
38
- * writeFileSync(request.url.replace(/[^a-z0-9\.]/gi, '_'), body);
51
+ * const crawler = new FileDownload({
52
+ * async requestHandler({ response, request }) {
53
+ * await writeFile(request.url.replace(/[^a-z0-9\.]/gi, '_'), Buffer.from(await response.arrayBuffer()));
39
54
  * },
40
55
  * });
41
56
  *
@@ -48,7 +63,109 @@ export type FileDownloadRequestHandler<UserData extends Dictionary = any> = Requ
48
63
  */
49
64
  export declare class FileDownload extends BasicCrawler<FileDownloadCrawlingContext> {
50
65
  #private;
51
- constructor(options?: Omit<BasicCrawlerOptions<FileDownloadCrawlingContext>, 'contextPipelineBuilder'>);
66
+ /**
67
+ * @internal
68
+ */
69
+ protected static optionsShape: {
70
+ contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
71
+ extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
72
+ requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
73
+ requestQueue: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
74
+ requestManager: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
75
+ requestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
76
+ requestHandlerTimeoutSecs: z.ZodOptional<z.ZodCustom<number, number>>;
77
+ errorHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
78
+ failedRequestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
79
+ maxRequestRetries: z.ZodDefault<z.ZodCustom<number, number>>;
80
+ sameDomainDelaySecs: z.ZodDefault<z.ZodCustom<number, number>>;
81
+ maxRequestsPerCrawl: z.ZodOptional<z.ZodCustom<number, number>>;
82
+ maxCrawlDepth: z.ZodOptional<z.ZodCustom<number, number>>;
83
+ taskLoopOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
84
+ concurrencySystem: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
85
+ sessionPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
86
+ proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
87
+ statusMessageLoggingInterval: z.ZodDefault<z.ZodCustom<number, number>>;
88
+ statusMessageCallback: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
89
+ additionalHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
90
+ ignoreHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
91
+ blockedStatusCodes: z.ZodOptional<z.ZodArray<z.ZodCustom<number, number>>>;
92
+ retryOnBlocked: z.ZodDefault<z.ZodBoolean>;
93
+ respectRobotsTxtFile: z.ZodDefault<z.ZodUnion<readonly [z.ZodBoolean, z.ZodCustom<Dictionary, Dictionary>]>>;
94
+ transactionalStorage: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodObject<{
95
+ requestQueue: z.ZodOptional<z.ZodEnum<{
96
+ deferred: "deferred";
97
+ writeThrough: "writeThrough";
98
+ }>>;
99
+ }, z.core.$strict>]>>;
100
+ onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
101
+ // @ts-ignore optional peer dependency or compatibility with es2022
102
+ httpClient: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/http-client").BaseHttpClient>>;
103
+ // @ts-ignore optional peer dependency or compatibility with es2022
104
+ configuration: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/basic").Configuration>>;
105
+ storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
106
+ // @ts-ignore optional peer dependency or compatibility with es2022
107
+ eventManager: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/basic").EventManager>>;
108
+ logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
109
+ minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
110
+ maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
111
+ initialConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
112
+ maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;
113
+ keepAlive: z.ZodOptional<z.ZodBoolean>;
114
+ statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
115
+ id: z.ZodOptional<z.ZodString>;
116
+ ignoreTlsErrors: z.ZodDefault<z.ZodBoolean>;
117
+ };
118
+ /** @internal */
119
+ protected static optionsSchema: z.ZodObject<{
120
+ contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
121
+ extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
122
+ requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
123
+ requestQueue: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
124
+ requestManager: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
125
+ requestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
126
+ requestHandlerTimeoutSecs: z.ZodOptional<z.ZodCustom<number, number>>;
127
+ errorHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
128
+ failedRequestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
129
+ maxRequestRetries: z.ZodDefault<z.ZodCustom<number, number>>;
130
+ sameDomainDelaySecs: z.ZodDefault<z.ZodCustom<number, number>>;
131
+ maxRequestsPerCrawl: z.ZodOptional<z.ZodCustom<number, number>>;
132
+ maxCrawlDepth: z.ZodOptional<z.ZodCustom<number, number>>;
133
+ taskLoopOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
134
+ concurrencySystem: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
135
+ sessionPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
136
+ proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
137
+ statusMessageLoggingInterval: z.ZodDefault<z.ZodCustom<number, number>>;
138
+ statusMessageCallback: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
139
+ additionalHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
140
+ ignoreHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
141
+ blockedStatusCodes: z.ZodOptional<z.ZodArray<z.ZodCustom<number, number>>>;
142
+ retryOnBlocked: z.ZodDefault<z.ZodBoolean>;
143
+ respectRobotsTxtFile: z.ZodDefault<z.ZodUnion<readonly [z.ZodBoolean, z.ZodCustom<Dictionary, Dictionary>]>>;
144
+ transactionalStorage: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodObject<{
145
+ requestQueue: z.ZodOptional<z.ZodEnum<{
146
+ deferred: "deferred";
147
+ writeThrough: "writeThrough";
148
+ }>>;
149
+ }, z.core.$strict>]>>;
150
+ onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
151
+ // @ts-ignore optional peer dependency or compatibility with es2022
152
+ httpClient: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/http-client").BaseHttpClient>>;
153
+ // @ts-ignore optional peer dependency or compatibility with es2022
154
+ configuration: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/basic").Configuration>>;
155
+ storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
156
+ // @ts-ignore optional peer dependency or compatibility with es2022
157
+ eventManager: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/basic").EventManager>>;
158
+ logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
159
+ minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
160
+ maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
161
+ initialConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
162
+ maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;
163
+ keepAlive: z.ZodOptional<z.ZodBoolean>;
164
+ statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
165
+ id: z.ZodOptional<z.ZodString>;
166
+ ignoreTlsErrors: z.ZodDefault<z.ZodBoolean>;
167
+ }, z.core.$strict>;
168
+ constructor(options?: FileDownloadOptions);
52
169
  }
53
170
  /**
54
171
  * Creates new {@link Router} instance that works based on request labels.
@@ -1,5 +1,7 @@
1
1
  import { BasicCrawler, ContextPipeline } from '@crawlee/basic';
2
2
  import { ResponseWithUrl } from '@crawlee/http-client';
3
+ import { parseArgument } from '@crawlee/utils/internal';
4
+ import { z } from 'zod';
3
5
  import { Router } from '../index.js';
4
6
  import { parseContentTypeFromResponse } from './utils.js';
5
7
  /**
@@ -9,7 +11,7 @@ import { parseContentTypeFromResponse } from './utils.js';
9
11
  * However, it doesn't parse the content - if you need to e.g. extract data from the downloaded files,
10
12
  * you might need to use {@link CheerioCrawler}, {@link PuppeteerCrawler} or {@link PlaywrightCrawler} instead.
11
13
  *
12
- * `FileCrawler` downloads each URL using a plain HTTP request and then invokes the user-provided {@link BasicCrawlerOptions.requestHandler} where the user can specify what to do with the downloaded data.
14
+ * `FileDownload` downloads each URL using a plain HTTP request and then invokes the user-provided {@link BasicCrawlerOptions.requestHandler} where the user can specify what to do with the downloaded data.
13
15
  *
14
16
  * The source URLs are represented using {@link Request} objects that are fed from the {@link IRequestManager|request manager} provided via the {@link BasicCrawlerOptions.requestManager|`requestManager`} constructor option (a {@link RequestQueue} is itself a request manager). To read from a read-only source such as a {@link RequestList} while still being able to enqueue new requests, combine it with a queue into a {@link RequestManagerTandem} via {@link IRequestLoader.toTandem|`requestLoader.toTandem()`} and pass the result as `requestManager`.
15
17
  *
@@ -17,14 +19,14 @@ import { parseContentTypeFromResponse } from './utils.js';
17
19
  *
18
20
  * The crawler finishes when there are no more {@link Request} objects to crawl.
19
21
  *
20
- * New requests are only dispatched when there is enough free CPU and memory available, as judged by the crawler's {@link ConcurrencySystem}. Concurrency is tuned via the `minConcurrency`, `maxConcurrency` and `maxRequestsPerMinute` options of the `FileCrawler` constructor, or, for finer control, by injecting a pre-configured {@link ConcurrencySystem|`concurrencySystem`}.
22
+ * New requests are only dispatched when there is enough free CPU and memory available, as judged by the crawler's {@link ConcurrencySystem}. Concurrency is tuned via the `minConcurrency`, `maxConcurrency` and `maxRequestsPerMinute` options of the `FileDownload` constructor, or, for finer control, by injecting a pre-configured {@link ConcurrencySystem|`concurrencySystem`}.
21
23
  *
22
24
  * ## Example usage
23
25
  *
24
26
  * ```ts
25
- * const crawler = new FileDownloader({
26
- * requestHandler({ body, request }) {
27
- * writeFileSync(request.url.replace(/[^a-z0-9\.]/gi, '_'), body);
27
+ * const crawler = new FileDownload({
28
+ * async requestHandler({ response, request }) {
29
+ * await writeFile(request.url.replace(/[^a-z0-9\.]/gi, '_'), Buffer.from(await response.arrayBuffer()));
28
30
  * },
29
31
  * });
30
32
  *
@@ -36,12 +38,24 @@ import { parseContentTypeFromResponse } from './utils.js';
36
38
  * ```
37
39
  */
38
40
  export class FileDownload extends BasicCrawler {
41
+ #ignoreTlsErrors;
42
+ /**
43
+ * @internal
44
+ */
45
+ static optionsShape = {
46
+ ...BasicCrawler.optionsShape,
47
+ ignoreTlsErrors: z.boolean().default(false),
48
+ };
49
+ /** @internal */
50
+ static optionsSchema = z.strictObject(FileDownload.optionsShape);
39
51
  // TODO hooks
40
52
  constructor(options = {}) {
53
+ const { ignoreTlsErrors, ...basicCrawlerOptions } = parseArgument(options, FileDownload.optionsSchema, 'FileDownloadOptions');
41
54
  super({
42
- ...options,
55
+ ...basicCrawlerOptions,
43
56
  contextPipelineBuilder: () => this.#buildContextPipeline(),
44
57
  });
58
+ this.#ignoreTlsErrors = ignoreTlsErrors;
45
59
  }
46
60
  #buildContextPipeline() {
47
61
  return ContextPipeline.create().compose(this.#initiateDownload.bind(this));
@@ -49,6 +63,7 @@ export class FileDownload extends BasicCrawler {
49
63
  async #initiateDownload(context, onCleanup) {
50
64
  const response = await this.httpClient.sendRequest(context.request.intoFetchAPIRequest(), {
51
65
  session: context.session,
66
+ ignoreTlsErrors: this.#ignoreTlsErrors,
52
67
  });
53
68
  const { type, charset: encoding } = parseContentTypeFromResponse(response);
54
69
  context.request.url = response.url;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@crawlee/http",
3
- "version": "4.0.0-beta.277",
3
+ "version": "4.0.0-beta.278",
4
4
  "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
5
5
  "engines": {
6
6
  "node": ">=22.13.0"
@@ -48,10 +48,10 @@
48
48
  },
49
49
  "dependencies": {
50
50
  "@apify/timeout": "^1.0.1",
51
- "@crawlee/basic": "4.0.0-beta.277",
52
- "@crawlee/http-client": "4.0.0-beta.277",
53
- "@crawlee/types": "4.0.0-beta.277",
54
- "@crawlee/utils": "4.0.0-beta.277",
51
+ "@crawlee/basic": "4.0.0-beta.278",
52
+ "@crawlee/http-client": "4.0.0-beta.278",
53
+ "@crawlee/types": "4.0.0-beta.278",
54
+ "@crawlee/utils": "4.0.0-beta.278",
55
55
  "cheerio": "^1.2.0",
56
56
  "content-type": "^3.1.1",
57
57
  "domhandler": "^6.0.1",
@@ -68,5 +68,5 @@
68
68
  }
69
69
  }
70
70
  },
71
- "gitHead": "086a9f966a05dd1159f37cc25c4589b886fc5c93"
71
+ "gitHead": "745f27ebf5c8307d949416c9c884ab941eef1559"
72
72
  }