@crawlee/http 4.0.0-beta.277 → 4.0.0-beta.279
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/internals/file-download.d.ts +123 -6
- package/internals/file-download.js +21 -6
- package/package.json +6 -6
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import type { BasicCrawlerOptions, CrawlingContext, CrawlingRequest, LoadedRequest } from '@crawlee/basic';
|
|
2
2
|
import { BasicCrawler } from '@crawlee/basic';
|
|
3
3
|
import type { Dictionary } from '@crawlee/types';
|
|
4
|
+
import { z } from 'zod';
|
|
4
5
|
import type { ErrorHandler, GetUserDataFromRequest, RequestHandler, RouterHandler, RouterRoutes, RouteSchemas, RoutesFromSchemas } from '../index.js';
|
|
5
6
|
export type FileDownloadErrorHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
|
|
6
7
|
ContextExtension = Dictionary<never>> = ErrorHandler<CrawlingContext, FileDownloadCrawlingContext<UserData> & ContextExtension>;
|
|
@@ -13,6 +14,20 @@ export interface FileDownloadCrawlingContext<UserData extends Dictionary = any>
|
|
|
13
14
|
};
|
|
14
15
|
}
|
|
15
16
|
export type FileDownloadRequestHandler<UserData extends Dictionary = any> = RequestHandler<FileDownloadCrawlingContext<UserData>>;
|
|
17
|
+
export interface FileDownloadOptions extends Omit<BasicCrawlerOptions<FileDownloadCrawlingContext>, 'contextPipelineBuilder'> {
|
|
18
|
+
/**
|
|
19
|
+
* If set to `true`, TLS/SSL certificate errors are ignored. Forwarded to the HTTP client as
|
|
20
|
+
* {@link SendRequestOptions.ignoreTlsErrors|`ignoreTlsErrors`} on every download, so custom
|
|
21
|
+
* {@link BaseHttpClient} implementations should honor that flag (the built-in impit and got-scraping
|
|
22
|
+
* clients do; the native fetch fallback cannot disable TLS verification and warns instead).
|
|
23
|
+
*
|
|
24
|
+
* Certificates are verified by default. Sessions whose proxy is a MITM proxy
|
|
25
|
+
* (`session.proxyInfo.ignoreTlsErrors`) skip verification regardless of this option.
|
|
26
|
+
*
|
|
27
|
+
* @default false
|
|
28
|
+
*/
|
|
29
|
+
ignoreTlsErrors?: boolean;
|
|
30
|
+
}
|
|
16
31
|
/**
|
|
17
32
|
* Provides a framework for downloading files in parallel using plain HTTP requests. The URLs to download are fed either from a static list of URLs or they can be added on the fly from another crawler.
|
|
18
33
|
*
|
|
@@ -20,7 +35,7 @@ export type FileDownloadRequestHandler<UserData extends Dictionary = any> = Requ
|
|
|
20
35
|
* However, it doesn't parse the content - if you need to e.g. extract data from the downloaded files,
|
|
21
36
|
* you might need to use {@link CheerioCrawler}, {@link PuppeteerCrawler} or {@link PlaywrightCrawler} instead.
|
|
22
37
|
*
|
|
23
|
-
* `
|
|
38
|
+
* `FileDownload` downloads each URL using a plain HTTP request and then invokes the user-provided {@link BasicCrawlerOptions.requestHandler} where the user can specify what to do with the downloaded data.
|
|
24
39
|
*
|
|
25
40
|
* The source URLs are represented using {@link Request} objects that are fed from the {@link IRequestManager|request manager} provided via the {@link BasicCrawlerOptions.requestManager|`requestManager`} constructor option (a {@link RequestQueue} is itself a request manager). To read from a read-only source such as a {@link RequestList} while still being able to enqueue new requests, combine it with a queue into a {@link RequestManagerTandem} via {@link IRequestLoader.toTandem|`requestLoader.toTandem()`} and pass the result as `requestManager`.
|
|
26
41
|
*
|
|
@@ -28,14 +43,14 @@ export type FileDownloadRequestHandler<UserData extends Dictionary = any> = Requ
|
|
|
28
43
|
*
|
|
29
44
|
* The crawler finishes when there are no more {@link Request} objects to crawl.
|
|
30
45
|
*
|
|
31
|
-
* New requests are only dispatched when there is enough free CPU and memory available, as judged by the crawler's {@link ConcurrencySystem}. Concurrency is tuned via the `minConcurrency`, `maxConcurrency` and `maxRequestsPerMinute` options of the `
|
|
46
|
+
* New requests are only dispatched when there is enough free CPU and memory available, as judged by the crawler's {@link ConcurrencySystem}. Concurrency is tuned via the `minConcurrency`, `maxConcurrency` and `maxRequestsPerMinute` options of the `FileDownload` constructor, or, for finer control, by injecting a pre-configured {@link ConcurrencySystem|`concurrencySystem`}.
|
|
32
47
|
*
|
|
33
48
|
* ## Example usage
|
|
34
49
|
*
|
|
35
50
|
* ```ts
|
|
36
|
-
* const crawler = new
|
|
37
|
-
* requestHandler({
|
|
38
|
-
*
|
|
51
|
+
* const crawler = new FileDownload({
|
|
52
|
+
* async requestHandler({ response, request }) {
|
|
53
|
+
* await writeFile(request.url.replace(/[^a-z0-9\.]/gi, '_'), Buffer.from(await response.arrayBuffer()));
|
|
39
54
|
* },
|
|
40
55
|
* });
|
|
41
56
|
*
|
|
@@ -48,7 +63,109 @@ export type FileDownloadRequestHandler<UserData extends Dictionary = any> = Requ
|
|
|
48
63
|
*/
|
|
49
64
|
export declare class FileDownload extends BasicCrawler<FileDownloadCrawlingContext> {
|
|
50
65
|
#private;
|
|
51
|
-
|
|
66
|
+
/**
|
|
67
|
+
* @internal
|
|
68
|
+
*/
|
|
69
|
+
protected static optionsShape: {
|
|
70
|
+
contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
71
|
+
extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
72
|
+
requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
73
|
+
requestQueue: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
74
|
+
requestManager: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
75
|
+
requestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
76
|
+
requestHandlerTimeoutSecs: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
77
|
+
errorHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
78
|
+
failedRequestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
79
|
+
maxRequestRetries: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
80
|
+
sameDomainDelaySecs: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
81
|
+
maxRequestsPerCrawl: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
82
|
+
maxCrawlDepth: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
83
|
+
taskLoopOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
84
|
+
concurrencySystem: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
85
|
+
sessionPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
86
|
+
proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
87
|
+
statusMessageLoggingInterval: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
88
|
+
statusMessageCallback: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
89
|
+
additionalHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
|
|
90
|
+
ignoreHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
|
|
91
|
+
blockedStatusCodes: z.ZodOptional<z.ZodArray<z.ZodCustom<number, number>>>;
|
|
92
|
+
retryOnBlocked: z.ZodDefault<z.ZodBoolean>;
|
|
93
|
+
respectRobotsTxtFile: z.ZodDefault<z.ZodUnion<readonly [z.ZodBoolean, z.ZodCustom<Dictionary, Dictionary>]>>;
|
|
94
|
+
transactionalStorage: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodObject<{
|
|
95
|
+
requestQueue: z.ZodOptional<z.ZodEnum<{
|
|
96
|
+
deferred: "deferred";
|
|
97
|
+
writeThrough: "writeThrough";
|
|
98
|
+
}>>;
|
|
99
|
+
}, z.core.$strict>]>>;
|
|
100
|
+
onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
101
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
102
|
+
httpClient: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/http-client").BaseHttpClient>>;
|
|
103
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
104
|
+
configuration: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/basic").Configuration>>;
|
|
105
|
+
storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
106
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
107
|
+
eventManager: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/basic").EventManager>>;
|
|
108
|
+
logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
109
|
+
minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
110
|
+
maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
111
|
+
initialConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
112
|
+
maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
113
|
+
keepAlive: z.ZodOptional<z.ZodBoolean>;
|
|
114
|
+
statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
115
|
+
id: z.ZodOptional<z.ZodString>;
|
|
116
|
+
ignoreTlsErrors: z.ZodDefault<z.ZodBoolean>;
|
|
117
|
+
};
|
|
118
|
+
/** @internal */
|
|
119
|
+
protected static optionsSchema: z.ZodObject<{
|
|
120
|
+
contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
121
|
+
extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
122
|
+
requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
123
|
+
requestQueue: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
124
|
+
requestManager: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
125
|
+
requestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
126
|
+
requestHandlerTimeoutSecs: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
127
|
+
errorHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
128
|
+
failedRequestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
129
|
+
maxRequestRetries: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
130
|
+
sameDomainDelaySecs: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
131
|
+
maxRequestsPerCrawl: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
132
|
+
maxCrawlDepth: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
133
|
+
taskLoopOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
134
|
+
concurrencySystem: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
135
|
+
sessionPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
136
|
+
proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
137
|
+
statusMessageLoggingInterval: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
138
|
+
statusMessageCallback: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
139
|
+
additionalHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
|
|
140
|
+
ignoreHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
|
|
141
|
+
blockedStatusCodes: z.ZodOptional<z.ZodArray<z.ZodCustom<number, number>>>;
|
|
142
|
+
retryOnBlocked: z.ZodDefault<z.ZodBoolean>;
|
|
143
|
+
respectRobotsTxtFile: z.ZodDefault<z.ZodUnion<readonly [z.ZodBoolean, z.ZodCustom<Dictionary, Dictionary>]>>;
|
|
144
|
+
transactionalStorage: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodObject<{
|
|
145
|
+
requestQueue: z.ZodOptional<z.ZodEnum<{
|
|
146
|
+
deferred: "deferred";
|
|
147
|
+
writeThrough: "writeThrough";
|
|
148
|
+
}>>;
|
|
149
|
+
}, z.core.$strict>]>>;
|
|
150
|
+
onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
151
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
152
|
+
httpClient: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/http-client").BaseHttpClient>>;
|
|
153
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
154
|
+
configuration: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/basic").Configuration>>;
|
|
155
|
+
storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
156
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
157
|
+
eventManager: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/basic").EventManager>>;
|
|
158
|
+
logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
159
|
+
minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
160
|
+
maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
161
|
+
initialConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
162
|
+
maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
163
|
+
keepAlive: z.ZodOptional<z.ZodBoolean>;
|
|
164
|
+
statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
165
|
+
id: z.ZodOptional<z.ZodString>;
|
|
166
|
+
ignoreTlsErrors: z.ZodDefault<z.ZodBoolean>;
|
|
167
|
+
}, z.core.$strict>;
|
|
168
|
+
constructor(options?: FileDownloadOptions);
|
|
52
169
|
}
|
|
53
170
|
/**
|
|
54
171
|
* Creates new {@link Router} instance that works based on request labels.
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
import { BasicCrawler, ContextPipeline } from '@crawlee/basic';
|
|
2
2
|
import { ResponseWithUrl } from '@crawlee/http-client';
|
|
3
|
+
import { parseArgument } from '@crawlee/utils/internal';
|
|
4
|
+
import { z } from 'zod';
|
|
3
5
|
import { Router } from '../index.js';
|
|
4
6
|
import { parseContentTypeFromResponse } from './utils.js';
|
|
5
7
|
/**
|
|
@@ -9,7 +11,7 @@ import { parseContentTypeFromResponse } from './utils.js';
|
|
|
9
11
|
* However, it doesn't parse the content - if you need to e.g. extract data from the downloaded files,
|
|
10
12
|
* you might need to use {@link CheerioCrawler}, {@link PuppeteerCrawler} or {@link PlaywrightCrawler} instead.
|
|
11
13
|
*
|
|
12
|
-
* `
|
|
14
|
+
* `FileDownload` downloads each URL using a plain HTTP request and then invokes the user-provided {@link BasicCrawlerOptions.requestHandler} where the user can specify what to do with the downloaded data.
|
|
13
15
|
*
|
|
14
16
|
* The source URLs are represented using {@link Request} objects that are fed from the {@link IRequestManager|request manager} provided via the {@link BasicCrawlerOptions.requestManager|`requestManager`} constructor option (a {@link RequestQueue} is itself a request manager). To read from a read-only source such as a {@link RequestList} while still being able to enqueue new requests, combine it with a queue into a {@link RequestManagerTandem} via {@link IRequestLoader.toTandem|`requestLoader.toTandem()`} and pass the result as `requestManager`.
|
|
15
17
|
*
|
|
@@ -17,14 +19,14 @@ import { parseContentTypeFromResponse } from './utils.js';
|
|
|
17
19
|
*
|
|
18
20
|
* The crawler finishes when there are no more {@link Request} objects to crawl.
|
|
19
21
|
*
|
|
20
|
-
* New requests are only dispatched when there is enough free CPU and memory available, as judged by the crawler's {@link ConcurrencySystem}. Concurrency is tuned via the `minConcurrency`, `maxConcurrency` and `maxRequestsPerMinute` options of the `
|
|
22
|
+
* New requests are only dispatched when there is enough free CPU and memory available, as judged by the crawler's {@link ConcurrencySystem}. Concurrency is tuned via the `minConcurrency`, `maxConcurrency` and `maxRequestsPerMinute` options of the `FileDownload` constructor, or, for finer control, by injecting a pre-configured {@link ConcurrencySystem|`concurrencySystem`}.
|
|
21
23
|
*
|
|
22
24
|
* ## Example usage
|
|
23
25
|
*
|
|
24
26
|
* ```ts
|
|
25
|
-
* const crawler = new
|
|
26
|
-
* requestHandler({
|
|
27
|
-
*
|
|
27
|
+
* const crawler = new FileDownload({
|
|
28
|
+
* async requestHandler({ response, request }) {
|
|
29
|
+
* await writeFile(request.url.replace(/[^a-z0-9\.]/gi, '_'), Buffer.from(await response.arrayBuffer()));
|
|
28
30
|
* },
|
|
29
31
|
* });
|
|
30
32
|
*
|
|
@@ -36,12 +38,24 @@ import { parseContentTypeFromResponse } from './utils.js';
|
|
|
36
38
|
* ```
|
|
37
39
|
*/
|
|
38
40
|
export class FileDownload extends BasicCrawler {
|
|
41
|
+
#ignoreTlsErrors;
|
|
42
|
+
/**
|
|
43
|
+
* @internal
|
|
44
|
+
*/
|
|
45
|
+
static optionsShape = {
|
|
46
|
+
...BasicCrawler.optionsShape,
|
|
47
|
+
ignoreTlsErrors: z.boolean().default(false),
|
|
48
|
+
};
|
|
49
|
+
/** @internal */
|
|
50
|
+
static optionsSchema = z.strictObject(FileDownload.optionsShape);
|
|
39
51
|
// TODO hooks
|
|
40
52
|
constructor(options = {}) {
|
|
53
|
+
const { ignoreTlsErrors, ...basicCrawlerOptions } = parseArgument(options, FileDownload.optionsSchema, 'FileDownloadOptions');
|
|
41
54
|
super({
|
|
42
|
-
...
|
|
55
|
+
...basicCrawlerOptions,
|
|
43
56
|
contextPipelineBuilder: () => this.#buildContextPipeline(),
|
|
44
57
|
});
|
|
58
|
+
this.#ignoreTlsErrors = ignoreTlsErrors;
|
|
45
59
|
}
|
|
46
60
|
#buildContextPipeline() {
|
|
47
61
|
return ContextPipeline.create().compose(this.#initiateDownload.bind(this));
|
|
@@ -49,6 +63,7 @@ export class FileDownload extends BasicCrawler {
|
|
|
49
63
|
async #initiateDownload(context, onCleanup) {
|
|
50
64
|
const response = await this.httpClient.sendRequest(context.request.intoFetchAPIRequest(), {
|
|
51
65
|
session: context.session,
|
|
66
|
+
ignoreTlsErrors: this.#ignoreTlsErrors,
|
|
52
67
|
});
|
|
53
68
|
const { type, charset: encoding } = parseContentTypeFromResponse(response);
|
|
54
69
|
context.request.url = response.url;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/http",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.279",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.13.0"
|
|
@@ -48,10 +48,10 @@
|
|
|
48
48
|
},
|
|
49
49
|
"dependencies": {
|
|
50
50
|
"@apify/timeout": "^1.0.1",
|
|
51
|
-
"@crawlee/basic": "4.0.0-beta.
|
|
52
|
-
"@crawlee/http-client": "4.0.0-beta.
|
|
53
|
-
"@crawlee/types": "4.0.0-beta.
|
|
54
|
-
"@crawlee/utils": "4.0.0-beta.
|
|
51
|
+
"@crawlee/basic": "4.0.0-beta.279",
|
|
52
|
+
"@crawlee/http-client": "4.0.0-beta.279",
|
|
53
|
+
"@crawlee/types": "4.0.0-beta.279",
|
|
54
|
+
"@crawlee/utils": "4.0.0-beta.279",
|
|
55
55
|
"cheerio": "^1.2.0",
|
|
56
56
|
"content-type": "^3.1.1",
|
|
57
57
|
"domhandler": "^6.0.1",
|
|
@@ -68,5 +68,5 @@
|
|
|
68
68
|
}
|
|
69
69
|
}
|
|
70
70
|
},
|
|
71
|
-
"gitHead": "
|
|
71
|
+
"gitHead": "c48476d4a812b8c65f3316979a8eb59f17975a24"
|
|
72
72
|
}
|