@crawlee/http 4.0.0-beta.121 → 4.0.0-beta.122

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -4,6 +4,7 @@ import { type LoadedRequest } from '@crawlee/core';
4
4
  import type { Awaitable, Dictionary } from '@crawlee/types';
5
5
  import { type CheerioRoot } from '@crawlee/utils/internal';
6
6
  import type { JsonValue } from 'type-fest';
7
+ import { z } from 'zod';
7
8
  /**
8
9
  * A higher starting concurrency and a relaxed event loop signal, since HTTP-only crawling barely touches the event
9
10
  * loop. {@link HttpCrawler} folds these into the {@link ConcurrencySystem} it builds by default.
@@ -249,97 +250,115 @@ JSONData extends JsonValue = any> = RequestHandler<HttpCrawlingContext<UserData,
249
250
  export declare class HttpCrawler<Context extends InternalHttpCrawlingContext<any, any> = InternalHttpCrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>> extends BasicCrawler<Context, ContextExtension, ExtendedContext, Routes> {
250
251
  #private;
251
252
  protected static optionsShape: {
252
- // @ts-ignore optional peer dependency or compatibility with es2022
253
- navigationTimeoutSecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
254
- // @ts-ignore optional peer dependency or compatibility with es2022
255
- ignoreTlsErrors: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
256
- // @ts-ignore optional peer dependency or compatibility with es2022
257
- additionalMimeTypes: import("ow").ArrayPredicate<string>;
258
- // @ts-ignore optional peer dependency or compatibility with es2022
259
- suggestResponseEncoding: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
260
- // @ts-ignore optional peer dependency or compatibility with es2022
261
- forceResponseEncoding: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
262
- // @ts-ignore optional peer dependency or compatibility with es2022
263
- saveResponseCookies: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
264
- // @ts-ignore optional peer dependency or compatibility with es2022
265
- preNavigationHooks: import("ow").ArrayPredicate<unknown> & import("ow").BasePredicate<unknown[] | undefined>;
266
- // @ts-ignore optional peer dependency or compatibility with es2022
267
- postNavigationHooks: import("ow").ArrayPredicate<unknown> & import("ow").BasePredicate<unknown[] | undefined>;
268
- // @ts-ignore optional peer dependency or compatibility with es2022
269
- contextPipelineBuilder: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
270
- // @ts-ignore optional peer dependency or compatibility with es2022
271
- extendContext: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
272
- // @ts-ignore optional peer dependency or compatibility with es2022
273
- requestList: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
274
- // @ts-ignore optional peer dependency or compatibility with es2022
275
- requestQueue: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
276
- // @ts-ignore optional peer dependency or compatibility with es2022
277
- requestManager: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
278
- // @ts-ignore optional peer dependency or compatibility with es2022
279
- requestHandler: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
280
- // @ts-ignore optional peer dependency or compatibility with es2022
281
- requestHandlerTimeoutSecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
282
- // @ts-ignore optional peer dependency or compatibility with es2022
283
- errorHandler: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
284
- // @ts-ignore optional peer dependency or compatibility with es2022
285
- failedRequestHandler: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
286
- // @ts-ignore optional peer dependency or compatibility with es2022
287
- maxRequestRetries: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
288
- // @ts-ignore optional peer dependency or compatibility with es2022
289
- sameDomainDelaySecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
290
- // @ts-ignore optional peer dependency or compatibility with es2022
291
- maxRequestsPerCrawl: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
292
- // @ts-ignore optional peer dependency or compatibility with es2022
293
- maxCrawlDepth: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
294
- // @ts-ignore optional peer dependency or compatibility with es2022
295
- taskLoopOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
296
- // @ts-ignore optional peer dependency or compatibility with es2022
297
- concurrencySystem: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
298
- // @ts-ignore optional peer dependency or compatibility with es2022
299
- sessionPool: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
300
- // @ts-ignore optional peer dependency or compatibility with es2022
301
- proxyConfiguration: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
302
- // @ts-ignore optional peer dependency or compatibility with es2022
303
- statusMessageLoggingInterval: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
304
- // @ts-ignore optional peer dependency or compatibility with es2022
305
- statusMessageCallback: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
306
- // @ts-ignore optional peer dependency or compatibility with es2022
307
- additionalHttpErrorStatusCodes: import("ow").ArrayPredicate<number>;
308
- // @ts-ignore optional peer dependency or compatibility with es2022
309
- ignoreHttpErrorStatusCodes: import("ow").ArrayPredicate<number>;
310
- // @ts-ignore optional peer dependency or compatibility with es2022
311
- blockedStatusCodes: import("ow").ArrayPredicate<number>;
312
- // @ts-ignore optional peer dependency or compatibility with es2022
313
- retryOnBlocked: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
314
- // @ts-ignore optional peer dependency or compatibility with es2022
315
- respectRobotsTxtFile: import("ow").AnyPredicate<boolean | object>;
316
- // @ts-ignore optional peer dependency or compatibility with es2022
317
- transactionalStorage: import("ow").BasePredicate<boolean | Partial<import("@crawlee/basic").StorageWritePolicy> | undefined>;
318
- // @ts-ignore optional peer dependency or compatibility with es2022
319
- onSkippedRequest: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
320
- // @ts-ignore optional peer dependency or compatibility with es2022
321
- httpClient: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
322
- // @ts-ignore optional peer dependency or compatibility with es2022
323
- configuration: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
324
- // @ts-ignore optional peer dependency or compatibility with es2022
325
- storageBackend: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
326
- // @ts-ignore optional peer dependency or compatibility with es2022
327
- eventManager: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
328
- // @ts-ignore optional peer dependency or compatibility with es2022
329
- logger: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
330
- // @ts-ignore optional peer dependency or compatibility with es2022
331
- minConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
332
- // @ts-ignore optional peer dependency or compatibility with es2022
333
- maxConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
334
- // @ts-ignore optional peer dependency or compatibility with es2022
335
- maxRequestsPerMinute: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
336
- // @ts-ignore optional peer dependency or compatibility with es2022
337
- keepAlive: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
338
- // @ts-ignore optional peer dependency or compatibility with es2022
339
- statistics: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
340
- // @ts-ignore optional peer dependency or compatibility with es2022
341
- id: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
253
+ navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;
254
+ ignoreTlsErrors: z.ZodDefault<z.ZodBoolean>;
255
+ additionalMimeTypes: z.ZodDefault<z.ZodArray<z.ZodString>>;
256
+ suggestResponseEncoding: z.ZodOptional<z.ZodString>;
257
+ forceResponseEncoding: z.ZodOptional<z.ZodString>;
258
+ saveResponseCookies: z.ZodDefault<z.ZodBoolean>;
259
+ preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
260
+ postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
261
+ contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
262
+ extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
263
+ requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
264
+ requestQueue: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
265
+ requestManager: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
266
+ requestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
267
+ requestHandlerTimeoutSecs: z.ZodOptional<z.ZodCustom<number, number>>;
268
+ errorHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
269
+ failedRequestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
270
+ maxRequestRetries: z.ZodDefault<z.ZodCustom<number, number>>;
271
+ sameDomainDelaySecs: z.ZodDefault<z.ZodCustom<number, number>>;
272
+ maxRequestsPerCrawl: z.ZodOptional<z.ZodCustom<number, number>>;
273
+ maxCrawlDepth: z.ZodOptional<z.ZodCustom<number, number>>;
274
+ taskLoopOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
275
+ concurrencySystem: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
276
+ sessionPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
277
+ proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
278
+ statusMessageLoggingInterval: z.ZodDefault<z.ZodCustom<number, number>>;
279
+ statusMessageCallback: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
280
+ additionalHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
281
+ ignoreHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
282
+ blockedStatusCodes: z.ZodOptional<z.ZodArray<z.ZodCustom<number, number>>>;
283
+ retryOnBlocked: z.ZodDefault<z.ZodBoolean>;
284
+ respectRobotsTxtFile: z.ZodDefault<z.ZodUnion<readonly [z.ZodBoolean, z.ZodCustom<Dictionary, Dictionary>]>>;
285
+ transactionalStorage: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodObject<{
286
+ requestQueue: z.ZodOptional<z.ZodEnum<{
287
+ deferred: "deferred";
288
+ writeThrough: "writeThrough";
289
+ }>>;
290
+ }, z.core.$strict>]>>;
291
+ onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
292
+ // @ts-ignore optional peer dependency or compatibility with es2022
293
+ httpClient: z.ZodOptional<z.ZodCustom<import("@crawlee/http-client").BaseHttpClient, import("@crawlee/http-client").BaseHttpClient>>;
294
+ // @ts-ignore optional peer dependency or compatibility with es2022
295
+ configuration: z.ZodOptional<z.ZodCustom<import("@crawlee/basic").Configuration, import("@crawlee/basic").Configuration>>;
296
+ storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
297
+ // @ts-ignore optional peer dependency or compatibility with es2022
298
+ eventManager: z.ZodOptional<z.ZodCustom<import("@crawlee/basic").EventManager, import("@crawlee/basic").EventManager>>;
299
+ logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
300
+ minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
301
+ maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
302
+ maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;
303
+ keepAlive: z.ZodOptional<z.ZodBoolean>;
304
+ statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
305
+ id: z.ZodOptional<z.ZodString>;
342
306
  };
307
+ protected static optionsSchema: z.ZodObject<{
308
+ navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;
309
+ ignoreTlsErrors: z.ZodDefault<z.ZodBoolean>;
310
+ additionalMimeTypes: z.ZodDefault<z.ZodArray<z.ZodString>>;
311
+ suggestResponseEncoding: z.ZodOptional<z.ZodString>;
312
+ forceResponseEncoding: z.ZodOptional<z.ZodString>;
313
+ saveResponseCookies: z.ZodDefault<z.ZodBoolean>;
314
+ preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
315
+ postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
316
+ contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
317
+ extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
318
+ requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
319
+ requestQueue: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
320
+ requestManager: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
321
+ requestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
322
+ requestHandlerTimeoutSecs: z.ZodOptional<z.ZodCustom<number, number>>;
323
+ errorHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
324
+ failedRequestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
325
+ maxRequestRetries: z.ZodDefault<z.ZodCustom<number, number>>;
326
+ sameDomainDelaySecs: z.ZodDefault<z.ZodCustom<number, number>>;
327
+ maxRequestsPerCrawl: z.ZodOptional<z.ZodCustom<number, number>>;
328
+ maxCrawlDepth: z.ZodOptional<z.ZodCustom<number, number>>;
329
+ taskLoopOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
330
+ concurrencySystem: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
331
+ sessionPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
332
+ proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
333
+ statusMessageLoggingInterval: z.ZodDefault<z.ZodCustom<number, number>>;
334
+ statusMessageCallback: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
335
+ additionalHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
336
+ ignoreHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
337
+ blockedStatusCodes: z.ZodOptional<z.ZodArray<z.ZodCustom<number, number>>>;
338
+ retryOnBlocked: z.ZodDefault<z.ZodBoolean>;
339
+ respectRobotsTxtFile: z.ZodDefault<z.ZodUnion<readonly [z.ZodBoolean, z.ZodCustom<Dictionary, Dictionary>]>>;
340
+ transactionalStorage: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodObject<{
341
+ requestQueue: z.ZodOptional<z.ZodEnum<{
342
+ deferred: "deferred";
343
+ writeThrough: "writeThrough";
344
+ }>>;
345
+ }, z.core.$strict>]>>;
346
+ onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
347
+ // @ts-ignore optional peer dependency or compatibility with es2022
348
+ httpClient: z.ZodOptional<z.ZodCustom<import("@crawlee/http-client").BaseHttpClient, import("@crawlee/http-client").BaseHttpClient>>;
349
+ // @ts-ignore optional peer dependency or compatibility with es2022
350
+ configuration: z.ZodOptional<z.ZodCustom<import("@crawlee/basic").Configuration, import("@crawlee/basic").Configuration>>;
351
+ storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
352
+ // @ts-ignore optional peer dependency or compatibility with es2022
353
+ eventManager: z.ZodOptional<z.ZodCustom<import("@crawlee/basic").EventManager, import("@crawlee/basic").EventManager>>;
354
+ logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
355
+ minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
356
+ maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
357
+ maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;
358
+ keepAlive: z.ZodOptional<z.ZodBoolean>;
359
+ statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
360
+ id: z.ZodOptional<z.ZodString>;
361
+ }, z.core.$strict>;
343
362
  /**
344
363
  * All `HttpCrawlerOptions` parameters are passed via an options object.
345
364
  */
@@ -1,12 +1,12 @@
1
1
  import { Readable } from 'node:stream';
2
2
  import util from 'node:util';
3
3
  import { BasicCrawler, ContextPipeline, NavigationSkippedError, remainingNavigationWindowMillis, RequestState, Router, SessionError, } from '@crawlee/basic';
4
- import { RequestThrottledError, getCookiesFromResponse } from '@crawlee/core';
4
+ import { RequestThrottledError, getCookiesFromResponse, parseArgument, schemas, } from '@crawlee/core';
5
5
  import { ResponseWithUrl } from '@crawlee/http-client';
6
6
  import { RETRY_CSS_SELECTORS } from '@crawlee/utils/internal';
7
7
  import contentTypeParser from 'content-type';
8
8
  import iconv from 'iconv-lite';
9
- import ow from 'ow';
9
+ import { z } from 'zod';
10
10
  import { addTimeoutToPromise, storage, TimeoutError, tryCancel } from '@apify/timeout';
11
11
  import { extractCharsetFromHtmlBytes, parseContentTypeFromResponse, processHttpRequestOptions } from './utils.js';
12
12
  /**
@@ -119,23 +119,23 @@ export class HttpCrawler extends BasicCrawler {
119
119
  #supportedMimeTypes;
120
120
  static optionsShape = {
121
121
  ...BasicCrawler.optionsShape,
122
- navigationTimeoutSecs: ow.optional.number,
123
- ignoreTlsErrors: ow.optional.boolean,
124
- additionalMimeTypes: ow.optional.array.ofType(ow.string),
125
- suggestResponseEncoding: ow.optional.string,
126
- forceResponseEncoding: ow.optional.string,
127
- saveResponseCookies: ow.optional.boolean,
128
- preNavigationHooks: ow.optional.array,
129
- postNavigationHooks: ow.optional.array,
122
+ navigationTimeoutSecs: schemas.anyNumber.default(30),
123
+ ignoreTlsErrors: z.boolean().default(true),
124
+ additionalMimeTypes: schemas.arrayOf(z.string(), 'strings').default(() => []),
125
+ suggestResponseEncoding: z.string().optional(),
126
+ forceResponseEncoding: z.string().optional(),
127
+ saveResponseCookies: z.boolean().default(true),
128
+ preNavigationHooks: schemas.anyArray.default(() => []),
129
+ postNavigationHooks: schemas.anyArray.default(() => []),
130
130
  };
131
+ static optionsSchema = z.strictObject(HttpCrawler.optionsShape);
131
132
  /**
132
133
  * All `HttpCrawlerOptions` parameters are passed via an options object.
133
134
  */
134
135
  constructor(options = {}) {
135
- ow(options, 'HttpCrawlerOptions', ow.object.exactShape(HttpCrawler.optionsShape));
136
- const { navigationTimeoutSecs = 30, ignoreTlsErrors = true, additionalMimeTypes = [], suggestResponseEncoding, forceResponseEncoding, saveResponseCookies = true, preNavigationHooks = [], postNavigationHooks = [],
136
+ const { navigationTimeoutSecs, ignoreTlsErrors, additionalMimeTypes, suggestResponseEncoding, forceResponseEncoding, saveResponseCookies, preNavigationHooks, postNavigationHooks,
137
137
  // BasicCrawler
138
- contextPipelineBuilder, ...basicCrawlerOptions } = options;
138
+ contextPipelineBuilder, ...basicCrawlerOptions } = parseArgument(options, HttpCrawler.optionsSchema, 'HttpCrawlerOptions');
139
139
  super({
140
140
  ...basicCrawlerOptions,
141
141
  contextPipelineBuilder: contextPipelineBuilder ??
@@ -1,9 +1,10 @@
1
1
  import { extname } from 'node:path';
2
2
  import { Readable } from 'node:stream';
3
- import { applySearchParams } from '@crawlee/utils/internal';
3
+ import { applySearchParams, parseArgument } from '@crawlee/utils/internal';
4
4
  import contentTypeParser from 'content-type';
5
5
  import mime from 'mime-types';
6
- import ow, { ObjectPredicate } from 'ow';
6
+ import { z } from 'zod';
7
+ const responseWithUrlSchema = z.looseObject({ url: z.url(), headers: z.looseObject({}) });
7
8
  /**
8
9
  * Converts {@link HttpRequestOptions} to a {@link HttpRequest}.
9
10
  */
@@ -54,10 +55,7 @@ export function extractCharsetFromHtmlBytes(bytes) {
54
55
  * @param response HTTP response object
55
56
  */
56
57
  export function parseContentTypeFromResponse(response) {
57
- ow(response, ow.object.partialShape({
58
- url: ow.string.url,
59
- headers: new ObjectPredicate(),
60
- }));
58
+ parseArgument(response, responseWithUrlSchema);
61
59
  const { url, headers } = response;
62
60
  let parsedContentType;
63
61
  if (headers.get('content-type')) {
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@crawlee/http",
3
- "version": "4.0.0-beta.121",
3
+ "version": "4.0.0-beta.122",
4
4
  "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
5
5
  "engines": {
6
6
  "node": ">=22.0.0"
@@ -49,19 +49,19 @@
49
49
  "dependencies": {
50
50
  "@apify/timeout": "^0.4.4",
51
51
  "@apify/utilities": "^2.15.5",
52
- "@crawlee/basic": "4.0.0-beta.121",
53
- "@crawlee/core": "4.0.0-beta.121",
54
- "@crawlee/http-client": "4.0.0-beta.121",
55
- "@crawlee/types": "4.0.0-beta.121",
56
- "@crawlee/utils": "4.0.0-beta.121",
52
+ "@crawlee/basic": "4.0.0-beta.122",
53
+ "@crawlee/core": "4.0.0-beta.122",
54
+ "@crawlee/http-client": "4.0.0-beta.122",
55
+ "@crawlee/types": "4.0.0-beta.122",
56
+ "@crawlee/utils": "4.0.0-beta.122",
57
57
  "@types/content-type": "^1.1.8",
58
58
  "cheerio": "^1.0.0",
59
59
  "content-type": "^1.0.5",
60
60
  "iconv-lite": "^0.7.2",
61
61
  "mime-types": "^3.0.1",
62
- "ow": "^2.0.0",
63
62
  "tslib": "^2.8.1",
64
- "type-fest": "^4.41.0"
63
+ "type-fest": "^4.41.0",
64
+ "zod": "^4.4.3"
65
65
  },
66
66
  "lerna": {
67
67
  "command": {
@@ -70,5 +70,5 @@
70
70
  }
71
71
  }
72
72
  },
73
- "gitHead": "5027317de626f5ba6de5047ae9341a898258cc5a"
73
+ "gitHead": "2c3e87fefdb9e1fca4c144f03167d8524cfdc2e5"
74
74
  }