@crawlee/http 4.0.0-beta.120 → 4.0.0-beta.122
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/internals/http-crawler.d.ts +109 -90
- package/internals/http-crawler.js +13 -13
- package/internals/utils.js +4 -6
- package/package.json +9 -9
|
@@ -4,6 +4,7 @@ import { type LoadedRequest } from '@crawlee/core';
|
|
|
4
4
|
import type { Awaitable, Dictionary } from '@crawlee/types';
|
|
5
5
|
import { type CheerioRoot } from '@crawlee/utils/internal';
|
|
6
6
|
import type { JsonValue } from 'type-fest';
|
|
7
|
+
import { z } from 'zod';
|
|
7
8
|
/**
|
|
8
9
|
* A higher starting concurrency and a relaxed event loop signal, since HTTP-only crawling barely touches the event
|
|
9
10
|
* loop. {@link HttpCrawler} folds these into the {@link ConcurrencySystem} it builds by default.
|
|
@@ -249,97 +250,115 @@ JSONData extends JsonValue = any> = RequestHandler<HttpCrawlingContext<UserData,
|
|
|
249
250
|
export declare class HttpCrawler<Context extends InternalHttpCrawlingContext<any, any> = InternalHttpCrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>> extends BasicCrawler<Context, ContextExtension, ExtendedContext, Routes> {
|
|
250
251
|
#private;
|
|
251
252
|
protected static optionsShape: {
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
statusMessageCallback: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
|
|
306
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
307
|
-
additionalHttpErrorStatusCodes: import("ow").ArrayPredicate<number>;
|
|
308
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
309
|
-
ignoreHttpErrorStatusCodes: import("ow").ArrayPredicate<number>;
|
|
310
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
311
|
-
blockedStatusCodes: import("ow").ArrayPredicate<number>;
|
|
312
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
313
|
-
retryOnBlocked: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
|
|
314
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
315
|
-
respectRobotsTxtFile: import("ow").AnyPredicate<boolean | object>;
|
|
316
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
317
|
-
transactionalStorage: import("ow").BasePredicate<boolean | Partial<import("@crawlee/basic").StorageWritePolicy> | undefined>;
|
|
318
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
319
|
-
onSkippedRequest: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
|
|
320
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
321
|
-
httpClient: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
322
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
323
|
-
configuration: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
324
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
325
|
-
storageBackend: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
326
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
327
|
-
eventManager: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
328
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
329
|
-
logger: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
330
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
331
|
-
minConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
|
|
332
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
333
|
-
maxConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
|
|
334
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
335
|
-
maxRequestsPerMinute: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
|
|
336
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
337
|
-
keepAlive: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
|
|
338
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
339
|
-
statistics: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
340
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
341
|
-
id: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
|
|
253
|
+
navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
254
|
+
ignoreTlsErrors: z.ZodDefault<z.ZodBoolean>;
|
|
255
|
+
additionalMimeTypes: z.ZodDefault<z.ZodArray<z.ZodString>>;
|
|
256
|
+
suggestResponseEncoding: z.ZodOptional<z.ZodString>;
|
|
257
|
+
forceResponseEncoding: z.ZodOptional<z.ZodString>;
|
|
258
|
+
saveResponseCookies: z.ZodDefault<z.ZodBoolean>;
|
|
259
|
+
preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
|
|
260
|
+
postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
|
|
261
|
+
contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
262
|
+
extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
263
|
+
requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
264
|
+
requestQueue: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
265
|
+
requestManager: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
266
|
+
requestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
267
|
+
requestHandlerTimeoutSecs: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
268
|
+
errorHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
269
|
+
failedRequestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
270
|
+
maxRequestRetries: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
271
|
+
sameDomainDelaySecs: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
272
|
+
maxRequestsPerCrawl: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
273
|
+
maxCrawlDepth: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
274
|
+
taskLoopOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
275
|
+
concurrencySystem: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
276
|
+
sessionPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
277
|
+
proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
278
|
+
statusMessageLoggingInterval: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
279
|
+
statusMessageCallback: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
280
|
+
additionalHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
|
|
281
|
+
ignoreHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
|
|
282
|
+
blockedStatusCodes: z.ZodOptional<z.ZodArray<z.ZodCustom<number, number>>>;
|
|
283
|
+
retryOnBlocked: z.ZodDefault<z.ZodBoolean>;
|
|
284
|
+
respectRobotsTxtFile: z.ZodDefault<z.ZodUnion<readonly [z.ZodBoolean, z.ZodCustom<Dictionary, Dictionary>]>>;
|
|
285
|
+
transactionalStorage: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodObject<{
|
|
286
|
+
requestQueue: z.ZodOptional<z.ZodEnum<{
|
|
287
|
+
deferred: "deferred";
|
|
288
|
+
writeThrough: "writeThrough";
|
|
289
|
+
}>>;
|
|
290
|
+
}, z.core.$strict>]>>;
|
|
291
|
+
onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
292
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
293
|
+
httpClient: z.ZodOptional<z.ZodCustom<import("@crawlee/http-client").BaseHttpClient, import("@crawlee/http-client").BaseHttpClient>>;
|
|
294
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
295
|
+
configuration: z.ZodOptional<z.ZodCustom<import("@crawlee/basic").Configuration, import("@crawlee/basic").Configuration>>;
|
|
296
|
+
storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
297
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
298
|
+
eventManager: z.ZodOptional<z.ZodCustom<import("@crawlee/basic").EventManager, import("@crawlee/basic").EventManager>>;
|
|
299
|
+
logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
300
|
+
minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
301
|
+
maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
302
|
+
maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
303
|
+
keepAlive: z.ZodOptional<z.ZodBoolean>;
|
|
304
|
+
statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
305
|
+
id: z.ZodOptional<z.ZodString>;
|
|
342
306
|
};
|
|
307
|
+
protected static optionsSchema: z.ZodObject<{
|
|
308
|
+
navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
309
|
+
ignoreTlsErrors: z.ZodDefault<z.ZodBoolean>;
|
|
310
|
+
additionalMimeTypes: z.ZodDefault<z.ZodArray<z.ZodString>>;
|
|
311
|
+
suggestResponseEncoding: z.ZodOptional<z.ZodString>;
|
|
312
|
+
forceResponseEncoding: z.ZodOptional<z.ZodString>;
|
|
313
|
+
saveResponseCookies: z.ZodDefault<z.ZodBoolean>;
|
|
314
|
+
preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
|
|
315
|
+
postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
|
|
316
|
+
contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
317
|
+
extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
318
|
+
requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
319
|
+
requestQueue: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
320
|
+
requestManager: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
321
|
+
requestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
322
|
+
requestHandlerTimeoutSecs: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
323
|
+
errorHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
324
|
+
failedRequestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
325
|
+
maxRequestRetries: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
326
|
+
sameDomainDelaySecs: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
327
|
+
maxRequestsPerCrawl: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
328
|
+
maxCrawlDepth: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
329
|
+
taskLoopOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
330
|
+
concurrencySystem: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
331
|
+
sessionPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
332
|
+
proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
333
|
+
statusMessageLoggingInterval: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
334
|
+
statusMessageCallback: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
335
|
+
additionalHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
|
|
336
|
+
ignoreHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
|
|
337
|
+
blockedStatusCodes: z.ZodOptional<z.ZodArray<z.ZodCustom<number, number>>>;
|
|
338
|
+
retryOnBlocked: z.ZodDefault<z.ZodBoolean>;
|
|
339
|
+
respectRobotsTxtFile: z.ZodDefault<z.ZodUnion<readonly [z.ZodBoolean, z.ZodCustom<Dictionary, Dictionary>]>>;
|
|
340
|
+
transactionalStorage: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodObject<{
|
|
341
|
+
requestQueue: z.ZodOptional<z.ZodEnum<{
|
|
342
|
+
deferred: "deferred";
|
|
343
|
+
writeThrough: "writeThrough";
|
|
344
|
+
}>>;
|
|
345
|
+
}, z.core.$strict>]>>;
|
|
346
|
+
onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
347
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
348
|
+
httpClient: z.ZodOptional<z.ZodCustom<import("@crawlee/http-client").BaseHttpClient, import("@crawlee/http-client").BaseHttpClient>>;
|
|
349
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
350
|
+
configuration: z.ZodOptional<z.ZodCustom<import("@crawlee/basic").Configuration, import("@crawlee/basic").Configuration>>;
|
|
351
|
+
storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
352
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
353
|
+
eventManager: z.ZodOptional<z.ZodCustom<import("@crawlee/basic").EventManager, import("@crawlee/basic").EventManager>>;
|
|
354
|
+
logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
355
|
+
minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
356
|
+
maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
357
|
+
maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
358
|
+
keepAlive: z.ZodOptional<z.ZodBoolean>;
|
|
359
|
+
statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
360
|
+
id: z.ZodOptional<z.ZodString>;
|
|
361
|
+
}, z.core.$strict>;
|
|
343
362
|
/**
|
|
344
363
|
* All `HttpCrawlerOptions` parameters are passed via an options object.
|
|
345
364
|
*/
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
import { Readable } from 'node:stream';
|
|
2
2
|
import util from 'node:util';
|
|
3
3
|
import { BasicCrawler, ContextPipeline, NavigationSkippedError, remainingNavigationWindowMillis, RequestState, Router, SessionError, } from '@crawlee/basic';
|
|
4
|
-
import { RequestThrottledError, getCookiesFromResponse } from '@crawlee/core';
|
|
4
|
+
import { RequestThrottledError, getCookiesFromResponse, parseArgument, schemas, } from '@crawlee/core';
|
|
5
5
|
import { ResponseWithUrl } from '@crawlee/http-client';
|
|
6
6
|
import { RETRY_CSS_SELECTORS } from '@crawlee/utils/internal';
|
|
7
7
|
import contentTypeParser from 'content-type';
|
|
8
8
|
import iconv from 'iconv-lite';
|
|
9
|
-
import
|
|
9
|
+
import { z } from 'zod';
|
|
10
10
|
import { addTimeoutToPromise, storage, TimeoutError, tryCancel } from '@apify/timeout';
|
|
11
11
|
import { extractCharsetFromHtmlBytes, parseContentTypeFromResponse, processHttpRequestOptions } from './utils.js';
|
|
12
12
|
/**
|
|
@@ -119,23 +119,23 @@ export class HttpCrawler extends BasicCrawler {
|
|
|
119
119
|
#supportedMimeTypes;
|
|
120
120
|
static optionsShape = {
|
|
121
121
|
...BasicCrawler.optionsShape,
|
|
122
|
-
navigationTimeoutSecs:
|
|
123
|
-
ignoreTlsErrors:
|
|
124
|
-
additionalMimeTypes:
|
|
125
|
-
suggestResponseEncoding:
|
|
126
|
-
forceResponseEncoding:
|
|
127
|
-
saveResponseCookies:
|
|
128
|
-
preNavigationHooks:
|
|
129
|
-
postNavigationHooks:
|
|
122
|
+
navigationTimeoutSecs: schemas.anyNumber.default(30),
|
|
123
|
+
ignoreTlsErrors: z.boolean().default(true),
|
|
124
|
+
additionalMimeTypes: schemas.arrayOf(z.string(), 'strings').default(() => []),
|
|
125
|
+
suggestResponseEncoding: z.string().optional(),
|
|
126
|
+
forceResponseEncoding: z.string().optional(),
|
|
127
|
+
saveResponseCookies: z.boolean().default(true),
|
|
128
|
+
preNavigationHooks: schemas.anyArray.default(() => []),
|
|
129
|
+
postNavigationHooks: schemas.anyArray.default(() => []),
|
|
130
130
|
};
|
|
131
|
+
static optionsSchema = z.strictObject(HttpCrawler.optionsShape);
|
|
131
132
|
/**
|
|
132
133
|
* All `HttpCrawlerOptions` parameters are passed via an options object.
|
|
133
134
|
*/
|
|
134
135
|
constructor(options = {}) {
|
|
135
|
-
|
|
136
|
-
const { navigationTimeoutSecs = 30, ignoreTlsErrors = true, additionalMimeTypes = [], suggestResponseEncoding, forceResponseEncoding, saveResponseCookies = true, preNavigationHooks = [], postNavigationHooks = [],
|
|
136
|
+
const { navigationTimeoutSecs, ignoreTlsErrors, additionalMimeTypes, suggestResponseEncoding, forceResponseEncoding, saveResponseCookies, preNavigationHooks, postNavigationHooks,
|
|
137
137
|
// BasicCrawler
|
|
138
|
-
contextPipelineBuilder, ...basicCrawlerOptions } = options;
|
|
138
|
+
contextPipelineBuilder, ...basicCrawlerOptions } = parseArgument(options, HttpCrawler.optionsSchema, 'HttpCrawlerOptions');
|
|
139
139
|
super({
|
|
140
140
|
...basicCrawlerOptions,
|
|
141
141
|
contextPipelineBuilder: contextPipelineBuilder ??
|
package/internals/utils.js
CHANGED
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
import { extname } from 'node:path';
|
|
2
2
|
import { Readable } from 'node:stream';
|
|
3
|
-
import { applySearchParams } from '@crawlee/utils/internal';
|
|
3
|
+
import { applySearchParams, parseArgument } from '@crawlee/utils/internal';
|
|
4
4
|
import contentTypeParser from 'content-type';
|
|
5
5
|
import mime from 'mime-types';
|
|
6
|
-
import
|
|
6
|
+
import { z } from 'zod';
|
|
7
|
+
const responseWithUrlSchema = z.looseObject({ url: z.url(), headers: z.looseObject({}) });
|
|
7
8
|
/**
|
|
8
9
|
* Converts {@link HttpRequestOptions} to a {@link HttpRequest}.
|
|
9
10
|
*/
|
|
@@ -54,10 +55,7 @@ export function extractCharsetFromHtmlBytes(bytes) {
|
|
|
54
55
|
* @param response HTTP response object
|
|
55
56
|
*/
|
|
56
57
|
export function parseContentTypeFromResponse(response) {
|
|
57
|
-
|
|
58
|
-
url: ow.string.url,
|
|
59
|
-
headers: new ObjectPredicate(),
|
|
60
|
-
}));
|
|
58
|
+
parseArgument(response, responseWithUrlSchema);
|
|
61
59
|
const { url, headers } = response;
|
|
62
60
|
let parsedContentType;
|
|
63
61
|
if (headers.get('content-type')) {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/http",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.122",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.0.0"
|
|
@@ -49,19 +49,19 @@
|
|
|
49
49
|
"dependencies": {
|
|
50
50
|
"@apify/timeout": "^0.4.4",
|
|
51
51
|
"@apify/utilities": "^2.15.5",
|
|
52
|
-
"@crawlee/basic": "4.0.0-beta.
|
|
53
|
-
"@crawlee/core": "4.0.0-beta.
|
|
54
|
-
"@crawlee/http-client": "4.0.0-beta.
|
|
55
|
-
"@crawlee/types": "4.0.0-beta.
|
|
56
|
-
"@crawlee/utils": "4.0.0-beta.
|
|
52
|
+
"@crawlee/basic": "4.0.0-beta.122",
|
|
53
|
+
"@crawlee/core": "4.0.0-beta.122",
|
|
54
|
+
"@crawlee/http-client": "4.0.0-beta.122",
|
|
55
|
+
"@crawlee/types": "4.0.0-beta.122",
|
|
56
|
+
"@crawlee/utils": "4.0.0-beta.122",
|
|
57
57
|
"@types/content-type": "^1.1.8",
|
|
58
58
|
"cheerio": "^1.0.0",
|
|
59
59
|
"content-type": "^1.0.5",
|
|
60
60
|
"iconv-lite": "^0.7.2",
|
|
61
61
|
"mime-types": "^3.0.1",
|
|
62
|
-
"ow": "^2.0.0",
|
|
63
62
|
"tslib": "^2.8.1",
|
|
64
|
-
"type-fest": "^4.41.0"
|
|
63
|
+
"type-fest": "^4.41.0",
|
|
64
|
+
"zod": "^4.4.3"
|
|
65
65
|
},
|
|
66
66
|
"lerna": {
|
|
67
67
|
"command": {
|
|
@@ -70,5 +70,5 @@
|
|
|
70
70
|
}
|
|
71
71
|
}
|
|
72
72
|
},
|
|
73
|
-
"gitHead": "
|
|
73
|
+
"gitHead": "2c3e87fefdb9e1fca4c144f03167d8524cfdc2e5"
|
|
74
74
|
}
|