apify-client 3.0.0-beta.22 → 3.0.0-beta.23
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/apify_client.d.ts +9 -0
- package/dist/apify_client.js +4 -1
- package/dist/bundle.js +11 -11
- package/dist/bundle.js.map +1 -1
- package/dist/http_client.d.ts +7 -0
- package/dist/http_client.js +8 -2
- package/dist/http_compressors/base.d.ts +40 -0
- package/dist/http_compressors/base.js +9 -0
- package/dist/http_compressors/brotli.d.ts +33 -0
- package/dist/http_compressors/brotli.js +39 -0
- package/dist/http_compressors/gzip.d.ts +33 -0
- package/dist/http_compressors/gzip.js +39 -0
- package/dist/http_compressors/index.d.ts +6 -0
- package/dist/http_compressors/index.js +2 -0
- package/dist/http_compressors/resolve.d.ts +23 -0
- package/dist/http_compressors/resolve.js +27 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1 -0
- package/dist/interceptors.d.ts +8 -1
- package/dist/interceptors.js +43 -13
- package/dist/runtime/node.js +10 -19
- package/dist/runtime/types.d.ts +12 -10
- package/dist/runtime/web.js +4 -3
- package/dist/utils.d.ts +13 -6
- package/dist/utils.js +9 -15
- package/package.json +1 -1
package/dist/http_client.d.ts
CHANGED
|
@@ -2,6 +2,7 @@ import type http from 'node:http';
|
|
|
2
2
|
import type https from 'node:https';
|
|
3
3
|
import type { AxiosInstance, AxiosRequestConfig, AxiosResponse, InternalAxiosRequestConfig } from 'axios';
|
|
4
4
|
import type { Log } from '@apify/log';
|
|
5
|
+
import type { HttpCompressor } from './http_compressors/base.js';
|
|
5
6
|
import type { RequestInterceptorFunction } from './interceptors.js';
|
|
6
7
|
import type { Statistics } from './statistics.js';
|
|
7
8
|
import type { Timeout, TimeoutTier } from './timeouts.js';
|
|
@@ -11,6 +12,11 @@ export declare class HttpClient {
|
|
|
11
12
|
maxRetries: number;
|
|
12
13
|
minDelayBetweenRetriesMillis: number;
|
|
13
14
|
userProvidedRequestInterceptors: RequestInterceptorFunction[];
|
|
15
|
+
/**
|
|
16
|
+
* Compressor applied to request bodies that are worth compressing. The request interceptor captures it at
|
|
17
|
+
* construction, so it is fixed for the lifetime of the client.
|
|
18
|
+
*/
|
|
19
|
+
readonly httpCompressor: HttpCompressor;
|
|
14
20
|
logger: Log;
|
|
15
21
|
/** Duration of each timeout tier, in milliseconds. */
|
|
16
22
|
timeoutMillis: Record<TimeoutTier, number>;
|
|
@@ -43,6 +49,7 @@ export interface HttpClientOptions {
|
|
|
43
49
|
maxRetries: number;
|
|
44
50
|
minDelayBetweenRetriesMillis: number;
|
|
45
51
|
requestInterceptors: RequestInterceptorFunction[];
|
|
52
|
+
httpCompressor: HttpCompressor;
|
|
46
53
|
timeoutShortSecs: number;
|
|
47
54
|
timeoutMediumSecs: number;
|
|
48
55
|
timeoutLongSecs: number;
|
package/dist/http_client.js
CHANGED
|
@@ -3,7 +3,7 @@ import axios, { AxiosHeaders } from 'axios';
|
|
|
3
3
|
import { APIFY_ENV_VARS } from '@apify/consts';
|
|
4
4
|
import { concatStreamToBuffer } from '@apify/utilities';
|
|
5
5
|
import { ApifyApiError } from './apify_api_error.js';
|
|
6
|
-
import {
|
|
6
|
+
import { createRequestInterceptors, InvalidResponseBodyError, responseInterceptors } from './interceptors.js';
|
|
7
7
|
import { runtime } from '#runtime';
|
|
8
8
|
import { asArray, cast, getEnv, isStream, version } from './utils.js';
|
|
9
9
|
const RATE_LIMIT_EXCEEDED_STATUS_CODE = 429;
|
|
@@ -12,6 +12,11 @@ export class HttpClient {
|
|
|
12
12
|
maxRetries;
|
|
13
13
|
minDelayBetweenRetriesMillis;
|
|
14
14
|
userProvidedRequestInterceptors;
|
|
15
|
+
/**
|
|
16
|
+
* Compressor applied to request bodies that are worth compressing. The request interceptor captures it at
|
|
17
|
+
* construction, so it is fixed for the lifetime of the client.
|
|
18
|
+
*/
|
|
19
|
+
httpCompressor;
|
|
15
20
|
logger;
|
|
16
21
|
/** Duration of each timeout tier, in milliseconds. */
|
|
17
22
|
timeoutMillis;
|
|
@@ -28,6 +33,7 @@ export class HttpClient {
|
|
|
28
33
|
this.maxRetries = options.maxRetries;
|
|
29
34
|
this.minDelayBetweenRetriesMillis = options.minDelayBetweenRetriesMillis;
|
|
30
35
|
this.userProvidedRequestInterceptors = options.requestInterceptors;
|
|
36
|
+
this.httpCompressor = options.httpCompressor;
|
|
31
37
|
this.timeoutMillis = {
|
|
32
38
|
short: options.timeoutShortSecs * 1000,
|
|
33
39
|
medium: options.timeoutMediumSecs * 1000,
|
|
@@ -86,7 +92,7 @@ export class HttpClient {
|
|
|
86
92
|
}
|
|
87
93
|
this.axios.defaults.headers['User-Agent'] = userAgent;
|
|
88
94
|
}
|
|
89
|
-
|
|
95
|
+
createRequestInterceptors(this.httpCompressor).forEach((i) => this.axios.interceptors.request.use(i));
|
|
90
96
|
this.userProvidedRequestInterceptors.forEach((i) => this.axios.interceptors.request.use(i));
|
|
91
97
|
responseInterceptors.forEach((i) => this.axios.interceptors.response.use(i));
|
|
92
98
|
}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
/**
|
|
3
|
+
* Strategy for compressing HTTP request bodies.
|
|
4
|
+
*
|
|
5
|
+
* Implement this interface to create a custom compressor. Set `contentEncoding` to the value that should be sent in
|
|
6
|
+
* the `Content-Encoding` header and implement `compress()`. The client calls it only for bodies that are large
|
|
7
|
+
* enough to benefit from compression, whose content type does not already carry its own compression, and which the
|
|
8
|
+
* caller did not send with a `Content-Encoding` of their own.
|
|
9
|
+
*
|
|
10
|
+
* @example
|
|
11
|
+
* ```javascript
|
|
12
|
+
* import { ApifyClient } from 'apify-client';
|
|
13
|
+
*
|
|
14
|
+
* const identityCompressor = {
|
|
15
|
+
* contentEncoding: 'identity',
|
|
16
|
+
*
|
|
17
|
+
* async compress(data) {
|
|
18
|
+
* return data;
|
|
19
|
+
* },
|
|
20
|
+
* };
|
|
21
|
+
*
|
|
22
|
+
* const client = new ApifyClient({ token: 'my-token', compression: identityCompressor });
|
|
23
|
+
* ```
|
|
24
|
+
*/
|
|
25
|
+
export interface HttpCompressor {
|
|
26
|
+
/** Value sent in the `Content-Encoding` header, for example `gzip` or `br`. */
|
|
27
|
+
readonly contentEncoding: string;
|
|
28
|
+
/**
|
|
29
|
+
* Compresses a request body.
|
|
30
|
+
*
|
|
31
|
+
* @param data - The raw bytes to compress.
|
|
32
|
+
* @returns The compressed bytes.
|
|
33
|
+
*/
|
|
34
|
+
compress(data: Buffer): Promise<Buffer>;
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* Schema accepting anything shaped like an {@link HttpCompressor}.
|
|
38
|
+
* @internal
|
|
39
|
+
*/
|
|
40
|
+
export declare const httpCompressorSchema: z.ZodCustom<HttpCompressor, HttpCompressor>;
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
/**
|
|
3
|
+
* Schema accepting anything shaped like an {@link HttpCompressor}.
|
|
4
|
+
* @internal
|
|
5
|
+
*/
|
|
6
|
+
export const httpCompressorSchema = z.custom((value) => typeof value === 'object' &&
|
|
7
|
+
value !== null &&
|
|
8
|
+
typeof value.contentEncoding === 'string' &&
|
|
9
|
+
typeof value.compress === 'function', { error: 'Invalid input: expected an HttpCompressor' });
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import type { HttpCompressor } from './base.js';
|
|
2
|
+
/**
|
|
3
|
+
* Compresses request bodies using brotli.
|
|
4
|
+
*
|
|
5
|
+
* Built on the `node:zlib` module, so it works wherever the client runs on Node.js.
|
|
6
|
+
*
|
|
7
|
+
* @example
|
|
8
|
+
* ```javascript
|
|
9
|
+
* import { ApifyClient, BrotliHttpCompressor } from 'apify-client';
|
|
10
|
+
*
|
|
11
|
+
* const client = new ApifyClient({ token: 'my-token', compression: new BrotliHttpCompressor({ quality: 11 }) });
|
|
12
|
+
* ```
|
|
13
|
+
*/
|
|
14
|
+
export declare class BrotliHttpCompressor implements HttpCompressor {
|
|
15
|
+
#private;
|
|
16
|
+
readonly contentEncoding = "br";
|
|
17
|
+
/**
|
|
18
|
+
* @param options - Compressor options.
|
|
19
|
+
* @throws {ArgumentValidationError} If `quality` is out of the valid range.
|
|
20
|
+
*/
|
|
21
|
+
constructor(options?: BrotliHttpCompressorOptions);
|
|
22
|
+
compress(data: Buffer): Promise<Buffer>;
|
|
23
|
+
}
|
|
24
|
+
/**
|
|
25
|
+
* Options for {@link BrotliHttpCompressor}.
|
|
26
|
+
*/
|
|
27
|
+
export interface BrotliHttpCompressorOptions {
|
|
28
|
+
/**
|
|
29
|
+
* Compression level, from `0` (the fastest) to `11` (the best compression).
|
|
30
|
+
* @default 6
|
|
31
|
+
*/
|
|
32
|
+
quality?: number;
|
|
33
|
+
}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
import { runtime } from '#runtime';
|
|
3
|
+
import { parseArgument } from '../utils.js';
|
|
4
|
+
/** Lowest valid brotli quality, the fastest with the least compression. */
|
|
5
|
+
const MIN_QUALITY = 0;
|
|
6
|
+
/** Highest valid brotli quality, the slowest with the best compression. */
|
|
7
|
+
const MAX_QUALITY = 11;
|
|
8
|
+
/** Middle of the range, where brotli already beats gzip at a comparable CPU cost. */
|
|
9
|
+
const DEFAULT_QUALITY = 6;
|
|
10
|
+
const optionsSchema = z.strictObject({
|
|
11
|
+
quality: z.int().min(MIN_QUALITY).max(MAX_QUALITY).default(DEFAULT_QUALITY),
|
|
12
|
+
});
|
|
13
|
+
/**
|
|
14
|
+
* Compresses request bodies using brotli.
|
|
15
|
+
*
|
|
16
|
+
* Built on the `node:zlib` module, so it works wherever the client runs on Node.js.
|
|
17
|
+
*
|
|
18
|
+
* @example
|
|
19
|
+
* ```javascript
|
|
20
|
+
* import { ApifyClient, BrotliHttpCompressor } from 'apify-client';
|
|
21
|
+
*
|
|
22
|
+
* const client = new ApifyClient({ token: 'my-token', compression: new BrotliHttpCompressor({ quality: 11 }) });
|
|
23
|
+
* ```
|
|
24
|
+
*/
|
|
25
|
+
export class BrotliHttpCompressor {
|
|
26
|
+
contentEncoding = 'br';
|
|
27
|
+
#quality;
|
|
28
|
+
/**
|
|
29
|
+
* @param options - Compressor options.
|
|
30
|
+
* @throws {ArgumentValidationError} If `quality` is out of the valid range.
|
|
31
|
+
*/
|
|
32
|
+
constructor(options = {}) {
|
|
33
|
+
const { quality } = parseArgument(options, optionsSchema, 'BrotliHttpCompressorOptions');
|
|
34
|
+
this.#quality = quality;
|
|
35
|
+
}
|
|
36
|
+
async compress(data) {
|
|
37
|
+
return runtime.compress(data, { algorithm: 'br', quality: this.#quality });
|
|
38
|
+
}
|
|
39
|
+
}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import type { HttpCompressor } from './base.js';
|
|
2
|
+
/**
|
|
3
|
+
* Compresses request bodies using gzip.
|
|
4
|
+
*
|
|
5
|
+
* Built on the `node:zlib` module, so it works wherever the client runs on Node.js.
|
|
6
|
+
*
|
|
7
|
+
* @example
|
|
8
|
+
* ```javascript
|
|
9
|
+
* import { ApifyClient, GzipHttpCompressor } from 'apify-client';
|
|
10
|
+
*
|
|
11
|
+
* const client = new ApifyClient({ token: 'my-token', compression: new GzipHttpCompressor({ quality: 1 }) });
|
|
12
|
+
* ```
|
|
13
|
+
*/
|
|
14
|
+
export declare class GzipHttpCompressor implements HttpCompressor {
|
|
15
|
+
#private;
|
|
16
|
+
readonly contentEncoding = "gzip";
|
|
17
|
+
/**
|
|
18
|
+
* @param options - Compressor options.
|
|
19
|
+
* @throws {ArgumentValidationError} If `quality` is out of the valid range.
|
|
20
|
+
*/
|
|
21
|
+
constructor(options?: GzipHttpCompressorOptions);
|
|
22
|
+
compress(data: Buffer): Promise<Buffer>;
|
|
23
|
+
}
|
|
24
|
+
/**
|
|
25
|
+
* Options for {@link GzipHttpCompressor}.
|
|
26
|
+
*/
|
|
27
|
+
export interface GzipHttpCompressorOptions {
|
|
28
|
+
/**
|
|
29
|
+
* Compression level, from `1` (the fastest) to `9` (the best compression).
|
|
30
|
+
* @default 6
|
|
31
|
+
*/
|
|
32
|
+
quality?: number;
|
|
33
|
+
}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
import { runtime } from '#runtime';
|
|
3
|
+
import { parseArgument } from '../utils.js';
|
|
4
|
+
/** Lowest valid gzip quality, the fastest with the least compression. */
|
|
5
|
+
const MIN_QUALITY = 1;
|
|
6
|
+
/** Highest valid gzip quality, the slowest with the best compression. */
|
|
7
|
+
const MAX_QUALITY = 9;
|
|
8
|
+
/** zlib's own default. The levels above it roughly double the CPU cost for about a percent fewer bytes. */
|
|
9
|
+
const DEFAULT_QUALITY = 6;
|
|
10
|
+
const optionsSchema = z.strictObject({
|
|
11
|
+
quality: z.int().min(MIN_QUALITY).max(MAX_QUALITY).default(DEFAULT_QUALITY),
|
|
12
|
+
});
|
|
13
|
+
/**
|
|
14
|
+
* Compresses request bodies using gzip.
|
|
15
|
+
*
|
|
16
|
+
* Built on the `node:zlib` module, so it works wherever the client runs on Node.js.
|
|
17
|
+
*
|
|
18
|
+
* @example
|
|
19
|
+
* ```javascript
|
|
20
|
+
* import { ApifyClient, GzipHttpCompressor } from 'apify-client';
|
|
21
|
+
*
|
|
22
|
+
* const client = new ApifyClient({ token: 'my-token', compression: new GzipHttpCompressor({ quality: 1 }) });
|
|
23
|
+
* ```
|
|
24
|
+
*/
|
|
25
|
+
export class GzipHttpCompressor {
|
|
26
|
+
contentEncoding = 'gzip';
|
|
27
|
+
#quality;
|
|
28
|
+
/**
|
|
29
|
+
* @param options - Compressor options.
|
|
30
|
+
* @throws {ArgumentValidationError} If `quality` is out of the valid range.
|
|
31
|
+
*/
|
|
32
|
+
constructor(options = {}) {
|
|
33
|
+
const { quality } = parseArgument(options, optionsSchema, 'GzipHttpCompressorOptions');
|
|
34
|
+
this.#quality = quality;
|
|
35
|
+
}
|
|
36
|
+
async compress(data) {
|
|
37
|
+
return runtime.compress(data, { algorithm: 'gzip', quality: this.#quality });
|
|
38
|
+
}
|
|
39
|
+
}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
export type { HttpCompressor } from './base.js';
|
|
2
|
+
export { BrotliHttpCompressor } from './brotli.js';
|
|
3
|
+
export type { BrotliHttpCompressorOptions } from './brotli.js';
|
|
4
|
+
export { GzipHttpCompressor } from './gzip.js';
|
|
5
|
+
export type { GzipHttpCompressorOptions } from './gzip.js';
|
|
6
|
+
export type { HttpCompressionAlgorithm } from './resolve.js';
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
import type { HttpCompressor } from './base.js';
|
|
3
|
+
/**
|
|
4
|
+
* Compression algorithms the `compression` option of {@link ApifyClient} accepts by name. Each name selects the
|
|
5
|
+
* matching built-in compressor with its default quality: `'brotli'` a {@link BrotliHttpCompressor}, `'gzip'` a
|
|
6
|
+
* {@link GzipHttpCompressor}. The Apify API also accepts `deflate` and `identity`, which a custom
|
|
7
|
+
* {@link HttpCompressor} covers.
|
|
8
|
+
*/
|
|
9
|
+
export type HttpCompressionAlgorithm = 'brotli' | 'gzip';
|
|
10
|
+
/**
|
|
11
|
+
* Schema of the `compression` option: an algorithm name or an {@link HttpCompressor}.
|
|
12
|
+
* @internal
|
|
13
|
+
*/
|
|
14
|
+
export declare const compressionSchema: z.ZodUnion<readonly [z.ZodEnum<{
|
|
15
|
+
gzip: "gzip";
|
|
16
|
+
brotli: "brotli";
|
|
17
|
+
}>, z.ZodCustom<HttpCompressor, HttpCompressor>]>;
|
|
18
|
+
/**
|
|
19
|
+
* Turns the `compression` option into a ready-to-use {@link HttpCompressor}: a name into the matching built-in
|
|
20
|
+
* compressor with its default quality, a compressor into itself.
|
|
21
|
+
* @internal
|
|
22
|
+
*/
|
|
23
|
+
export declare function resolveCompressor(compression: HttpCompressionAlgorithm | HttpCompressor): HttpCompressor;
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
import { httpCompressorSchema } from './base.js';
|
|
3
|
+
import { BrotliHttpCompressor } from './brotli.js';
|
|
4
|
+
import { GzipHttpCompressor } from './gzip.js';
|
|
5
|
+
/**
|
|
6
|
+
* Schema of the `compression` option: an algorithm name or an {@link HttpCompressor}.
|
|
7
|
+
* @internal
|
|
8
|
+
*/
|
|
9
|
+
export const compressionSchema = z.union([
|
|
10
|
+
z.enum(['brotli', 'gzip']),
|
|
11
|
+
httpCompressorSchema,
|
|
12
|
+
]);
|
|
13
|
+
/**
|
|
14
|
+
* Turns the `compression` option into a ready-to-use {@link HttpCompressor}: a name into the matching built-in
|
|
15
|
+
* compressor with its default quality, a compressor into itself.
|
|
16
|
+
* @internal
|
|
17
|
+
*/
|
|
18
|
+
export function resolveCompressor(compression) {
|
|
19
|
+
if (typeof compression !== 'string')
|
|
20
|
+
return compression;
|
|
21
|
+
switch (compression) {
|
|
22
|
+
case 'gzip':
|
|
23
|
+
return new GzipHttpCompressor();
|
|
24
|
+
case 'brotli':
|
|
25
|
+
return new BrotliHttpCompressor();
|
|
26
|
+
}
|
|
27
|
+
}
|
package/dist/index.d.ts
CHANGED
|
@@ -30,5 +30,6 @@ export * from './apify_api_error.js';
|
|
|
30
30
|
export { ArgumentValidationError } from '@apify/validations';
|
|
31
31
|
export * from './response_validation_error.js';
|
|
32
32
|
export { InvalidResponseBodyError } from './interceptors.js';
|
|
33
|
+
export * from './http_compressors/index.js';
|
|
33
34
|
export type { PaginatedList, Dictionary } from './utils.js';
|
|
34
35
|
export type { Timeout, TimeoutOptions, TimeoutTier } from './timeouts.js';
|
package/dist/index.js
CHANGED
package/dist/interceptors.d.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import type { AxiosInterceptorManager, AxiosResponse } from 'axios';
|
|
2
2
|
import type { ApifyRequestConfig, ApifyResponse } from './http_client.js';
|
|
3
|
+
import type { HttpCompressor } from './http_compressors/base.js';
|
|
3
4
|
/**
|
|
4
5
|
* This error exists for the quite common situation, where only a partial JSON response is received and
|
|
5
6
|
* an attempt to parse the JSON throws an error. In most cases this can be resolved by retrying the
|
|
@@ -15,5 +16,11 @@ export declare class InvalidResponseBodyError extends Error {
|
|
|
15
16
|
}
|
|
16
17
|
export type RequestInterceptorFunction = Parameters<AxiosInterceptorManager<ApifyRequestConfig>['use']>[0];
|
|
17
18
|
export type ResponseInterceptorFunction = Parameters<AxiosInterceptorManager<ApifyResponse>['use']>[0];
|
|
18
|
-
|
|
19
|
+
/**
|
|
20
|
+
* The client's own request interceptors, in registration order. Axios runs request interceptors in the reverse
|
|
21
|
+
* order of registration, so the body is serialized before it is compressed, and interceptors registered later,
|
|
22
|
+
* such as the user-provided ones, run before both.
|
|
23
|
+
* @internal
|
|
24
|
+
*/
|
|
25
|
+
export declare function createRequestInterceptors(compressor: HttpCompressor): RequestInterceptorFunction[];
|
|
19
26
|
export declare const responseInterceptors: ResponseInterceptorFunction[];
|
package/dist/interceptors.js
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import axios, { AxiosHeaders } from 'axios';
|
|
2
2
|
import contentTypeParser from 'content-type';
|
|
3
3
|
import { maybeParseBody } from './body_parser.js';
|
|
4
|
-
import {
|
|
4
|
+
import { runtime } from '#runtime';
|
|
5
|
+
import { isCompressibleContentType, MIN_COMPRESS_BYTES, toBytes } from './utils.js';
|
|
5
6
|
/**
|
|
6
7
|
* This error exists for the quite common situation, where only a partial JSON response is received and
|
|
7
8
|
* an attempt to parse the JSON throws an error. In most cases this can be resolved by retrying the
|
|
@@ -30,6 +31,17 @@ function getHeader(config, name) {
|
|
|
30
31
|
const value = key === undefined ? undefined : config.headers?.[key];
|
|
31
32
|
return typeof value === 'string' ? value : undefined;
|
|
32
33
|
}
|
|
34
|
+
/** Removes a request header regardless of the casing it was set with. */
|
|
35
|
+
function deleteHeader(config, name) {
|
|
36
|
+
const { headers } = config;
|
|
37
|
+
if (!headers)
|
|
38
|
+
return;
|
|
39
|
+
const wanted = name.toLowerCase();
|
|
40
|
+
for (const key of Object.keys(headers)) {
|
|
41
|
+
if (key.toLowerCase() === wanted)
|
|
42
|
+
delete headers[key];
|
|
43
|
+
}
|
|
44
|
+
}
|
|
33
45
|
function serializeRequest(config) {
|
|
34
46
|
// A string body with an explicit content type is already serialized and goes out as it is. The axios default
|
|
35
47
|
// transform would otherwise parse a JSON one in full just to check that it is valid, which for a body assembled
|
|
@@ -79,18 +91,32 @@ function stringifyWithFunctions(obj) {
|
|
|
79
91
|
return typeof value === 'function' ? value.toString() : value;
|
|
80
92
|
});
|
|
81
93
|
}
|
|
82
|
-
|
|
94
|
+
/**
|
|
95
|
+
* Compresses the request body with the client's compressor and labels it with the compressor's `Content-Encoding`.
|
|
96
|
+
*
|
|
97
|
+
* Runs after `serializeRequest`, so a JSON body is already a string here. A caller-set `Content-Encoding` is
|
|
98
|
+
* forwarded verbatim, which is how a pre-encoded body is uploaded, and `Content-Encoding: identity` opts a single
|
|
99
|
+
* request out of compression. The built-in compressors need `node:zlib`, so a body sent from a browser or an edge
|
|
100
|
+
* runtime goes out as it is.
|
|
101
|
+
*
|
|
102
|
+
* Compressing changes the body length, so any `Content-Length` the caller set describes the wrong body and has to
|
|
103
|
+
* go. Axios keeps a caller-set one over the size it computes, which would stall the request until it times out.
|
|
104
|
+
*/
|
|
105
|
+
async function maybeCompressRequest(config, compressor) {
|
|
106
|
+
if (!runtime.isNode)
|
|
107
|
+
return config;
|
|
108
|
+
const bytes = toBytes(config.data);
|
|
109
|
+
if (!bytes || bytes.byteLength < MIN_COMPRESS_BYTES)
|
|
110
|
+
return config;
|
|
83
111
|
// A caller-supplied encoding means the body is already encoded and the header describes it, so leave both alone.
|
|
84
112
|
if (getHeader(config, 'content-encoding'))
|
|
85
113
|
return config;
|
|
86
114
|
if (!isCompressibleContentType(getHeader(config, 'content-type')))
|
|
87
115
|
return config;
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
config.data = maybeCompressed.data;
|
|
93
|
-
}
|
|
116
|
+
config.data = await compressor.compress(Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength));
|
|
117
|
+
config.headers ??= {};
|
|
118
|
+
config.headers['content-encoding'] = compressor.contentEncoding;
|
|
119
|
+
deleteHeader(config, 'content-length');
|
|
94
120
|
return config;
|
|
95
121
|
}
|
|
96
122
|
function parseResponseData(response) {
|
|
@@ -115,9 +141,13 @@ function parseResponseData(response) {
|
|
|
115
141
|
}
|
|
116
142
|
return response;
|
|
117
143
|
}
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
144
|
+
/**
|
|
145
|
+
* The client's own request interceptors, in registration order. Axios runs request interceptors in the reverse
|
|
146
|
+
* order of registration, so the body is serialized before it is compressed, and interceptors registered later,
|
|
147
|
+
* such as the user-provided ones, run before both.
|
|
148
|
+
* @internal
|
|
149
|
+
*/
|
|
150
|
+
export function createRequestInterceptors(compressor) {
|
|
151
|
+
return [async (config) => maybeCompressRequest(config, compressor), serializeRequest, ensureHeadersPrototype];
|
|
152
|
+
}
|
|
123
153
|
export const responseInterceptors = [parseResponseData];
|
package/dist/runtime/node.js
CHANGED
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import os from 'node:os';
|
|
2
|
-
import { promisify } from 'node:util';
|
|
3
2
|
import { brotliCompress, constants, gzip } from 'node:zlib';
|
|
4
3
|
/**
|
|
5
4
|
* The Node.js implementation of {@link Runtime}. Deno and Bun resolve the `node` condition too and
|
|
@@ -8,24 +7,16 @@ import { brotliCompress, constants, gzip } from 'node:zlib';
|
|
|
8
7
|
export const runtime = {
|
|
9
8
|
isNode: true,
|
|
10
9
|
platform: `${os.platform()}; Node/${process.version}`,
|
|
11
|
-
async compress(data) {
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
// brotli, but usually do implement gzip.
|
|
22
|
-
}
|
|
23
|
-
try {
|
|
24
|
-
return { data: await promisify(gzip)(data), encoding: 'gzip' };
|
|
25
|
-
}
|
|
26
|
-
catch {
|
|
27
|
-
return undefined;
|
|
28
|
-
}
|
|
10
|
+
async compress(data, { algorithm, quality }) {
|
|
11
|
+
return new Promise((resolve, reject) => {
|
|
12
|
+
const done = (error, result) => (error ? reject(error) : resolve(result));
|
|
13
|
+
if (algorithm === 'br') {
|
|
14
|
+
brotliCompress(data, { params: { [constants.BROTLI_PARAM_QUALITY]: quality } }, done);
|
|
15
|
+
}
|
|
16
|
+
else {
|
|
17
|
+
gzip(data, { level: quality }, done);
|
|
18
|
+
}
|
|
19
|
+
});
|
|
29
20
|
},
|
|
30
21
|
async createHttpAgents({ timeoutMillis }) {
|
|
31
22
|
// Loaded on the first request rather than with the client: the proxy support pulls in a sizeable
|
package/dist/runtime/types.d.ts
CHANGED
|
@@ -1,12 +1,14 @@
|
|
|
1
1
|
import type http from 'node:http';
|
|
2
2
|
import type https from 'node:https';
|
|
3
3
|
/**
|
|
4
|
-
*
|
|
4
|
+
* What to compress a request body with, as the built-in `HttpCompressor` implementations ask for it.
|
|
5
5
|
* @internal
|
|
6
6
|
*/
|
|
7
|
-
export interface
|
|
8
|
-
|
|
9
|
-
|
|
7
|
+
export interface CompressionOptions {
|
|
8
|
+
/** Algorithm to compress with, named by the `Content-Encoding` value it produces. */
|
|
9
|
+
algorithm: 'br' | 'gzip';
|
|
10
|
+
/** Compression level, in the range the algorithm defines. */
|
|
11
|
+
quality: number;
|
|
10
12
|
}
|
|
11
13
|
/**
|
|
12
14
|
* The agents axios's Node.js adapter sends requests through.
|
|
@@ -25,9 +27,8 @@ export interface HttpAgents {
|
|
|
25
27
|
export interface Runtime {
|
|
26
28
|
/**
|
|
27
29
|
* Whether the Node.js implementation was selected, which the `node` condition settles when `#runtime` is
|
|
28
|
-
* resolved - so a Node.js application bundled for a browser or a neutral target reports `false`.
|
|
29
|
-
*
|
|
30
|
-
* gated on it.
|
|
30
|
+
* resolved - so a Node.js application bundled for a browser or a neutral target reports `false`. Log
|
|
31
|
+
* streaming, the `stream` record option and request body compression are gated on it.
|
|
31
32
|
*/
|
|
32
33
|
isNode: boolean;
|
|
33
34
|
/**
|
|
@@ -36,10 +37,11 @@ export interface Runtime {
|
|
|
36
37
|
*/
|
|
37
38
|
platform: string | undefined;
|
|
38
39
|
/**
|
|
39
|
-
* Compresses a request body,
|
|
40
|
-
*
|
|
40
|
+
* Compresses a request body. The built-in compressors go through it, so a bundle for a browser or an edge
|
|
41
|
+
* runtime carries no `node:zlib`. Throws where the runtime has no compression, which the client never
|
|
42
|
+
* reaches, since it compresses only where {@link isNode} holds.
|
|
41
43
|
*/
|
|
42
|
-
compress(data: Uint8Array): Promise<
|
|
44
|
+
compress(data: Uint8Array, options: CompressionOptions): Promise<Buffer>;
|
|
43
45
|
/**
|
|
44
46
|
* Creates the agents axios's Node.js adapter sends requests through, or resolves to `undefined` where
|
|
45
47
|
* axios does not use agents.
|
package/dist/runtime/web.js
CHANGED
|
@@ -5,10 +5,11 @@ export const runtime = {
|
|
|
5
5
|
isNode: false,
|
|
6
6
|
// Browsers do not let a page set the `User-Agent` header.
|
|
7
7
|
platform: undefined,
|
|
8
|
-
// No request compression: brotli
|
|
9
|
-
// list of headers the Apify API allows in a cross-origin request, so a browser would fail the
|
|
8
|
+
// No request compression: brotli and gzip have no Web API, and a `content-encoding` request header is not
|
|
9
|
+
// on the list of headers the Apify API allows in a cross-origin request, so a browser would fail the
|
|
10
|
+
// preflight. The client compresses in Node.js alone, so nothing reaches this.
|
|
10
11
|
async compress() {
|
|
11
|
-
|
|
12
|
+
throw new Error('Request body compression is only available in Node.js.');
|
|
12
13
|
},
|
|
13
14
|
// The XHR and fetch adapters of axios do not use agents.
|
|
14
15
|
async createHttpAgents() {
|
package/dist/utils.d.ts
CHANGED
|
@@ -4,9 +4,15 @@ import { z } from 'zod';
|
|
|
4
4
|
import type { ApifyApiError } from './apify_api_error.js';
|
|
5
5
|
import { parseArgument } from '@apify/validations';
|
|
6
6
|
import type { ApifyResponse } from './http_client.js';
|
|
7
|
-
import type { CompressedValue } from './runtime/types.js';
|
|
8
7
|
import type { RequestQueueClientListRequestsOptions, RequestQueueClientListRequestsResult } from './resource_clients/request_queue.js';
|
|
9
8
|
import type { WebhookUpdateData } from './resource_clients/webhook.js';
|
|
9
|
+
/**
|
|
10
|
+
* Smallest request body, in bytes, that is worth compressing. A smaller body already fits in a single network
|
|
11
|
+
* packet, so compressing it costs CPU time without saving a round trip, and the compression format itself can
|
|
12
|
+
* make it larger.
|
|
13
|
+
* @internal
|
|
14
|
+
*/
|
|
15
|
+
export declare const MIN_COMPRESS_BYTES = 1024;
|
|
10
16
|
export declare const version: any;
|
|
11
17
|
export { parseArgument };
|
|
12
18
|
/**
|
|
@@ -71,6 +77,12 @@ export declare function bytesToBase64(bytes: Uint8Array): string;
|
|
|
71
77
|
* Concatenates byte chunks into one array.
|
|
72
78
|
*/
|
|
73
79
|
export declare function concatBytes(chunks: Uint8Array[]): Uint8Array;
|
|
80
|
+
/**
|
|
81
|
+
* Views a request body as bytes: a string is UTF-8 encoded, binary values are viewed in place. Anything else
|
|
82
|
+
* - a stream, a `Blob`, form data - is `undefined`.
|
|
83
|
+
* @internal
|
|
84
|
+
*/
|
|
85
|
+
export declare function toBytes(value: unknown): Uint8Array | undefined;
|
|
74
86
|
/**
|
|
75
87
|
* Decides whether a request body with the given content type is worth compressing.
|
|
76
88
|
*
|
|
@@ -81,11 +93,6 @@ export declare function concatBytes(chunks: Uint8Array[]): Uint8Array;
|
|
|
81
93
|
* @internal
|
|
82
94
|
*/
|
|
83
95
|
export declare function isCompressibleContentType(contentType?: string): boolean;
|
|
84
|
-
/**
|
|
85
|
-
* Compresses the passed value with the runtime's best available algorithm. Returns `undefined` if the data
|
|
86
|
-
* is too small, is not a string or binary value, or if the runtime does not offer compression.
|
|
87
|
-
*/
|
|
88
|
-
export declare function maybeCompressValue(value: unknown): Promise<CompressedValue | undefined>;
|
|
89
96
|
/**
|
|
90
97
|
* Reads an environment variable, on runtimes that have them.
|
|
91
98
|
*/
|
package/dist/utils.js
CHANGED
|
@@ -1,11 +1,16 @@
|
|
|
1
1
|
import { z } from 'zod';
|
|
2
2
|
import { NotFoundError } from './apify_api_error.js';
|
|
3
3
|
import { parseArgument } from '@apify/validations';
|
|
4
|
-
import { runtime } from '#runtime';
|
|
5
4
|
import { ResponseValidationError } from './response_validation_error.js';
|
|
6
5
|
// @ts-ignore if we enable `resolveJsonModule`, we end up with a `src` folder in `dist`
|
|
7
6
|
import packageJson from '../package.json' with { type: 'json' };
|
|
8
|
-
|
|
7
|
+
/**
|
|
8
|
+
* Smallest request body, in bytes, that is worth compressing. A smaller body already fits in a single network
|
|
9
|
+
* packet, so compressing it costs CPU time without saving a round trip, and the compression format itself can
|
|
10
|
+
* make it larger.
|
|
11
|
+
* @internal
|
|
12
|
+
*/
|
|
13
|
+
export const MIN_COMPRESS_BYTES = 1024;
|
|
9
14
|
const textEncoder = new TextEncoder();
|
|
10
15
|
// Only the version, so a bundler can drop the rest of the manifest.
|
|
11
16
|
export const { version } = packageJson;
|
|
@@ -160,8 +165,9 @@ export function concatBytes(chunks) {
|
|
|
160
165
|
/**
|
|
161
166
|
* Views a request body as bytes: a string is UTF-8 encoded, binary values are viewed in place. Anything else
|
|
162
167
|
* - a stream, a `Blob`, form data - is `undefined`.
|
|
168
|
+
* @internal
|
|
163
169
|
*/
|
|
164
|
-
function toBytes(value) {
|
|
170
|
+
export function toBytes(value) {
|
|
165
171
|
if (typeof value === 'string')
|
|
166
172
|
return textEncoder.encode(value);
|
|
167
173
|
if (!isBuffer(value))
|
|
@@ -192,18 +198,6 @@ export function isCompressibleContentType(contentType) {
|
|
|
192
198
|
return false;
|
|
193
199
|
return !ALREADY_COMPRESSED_MEDIA_TYPE_PREFIXES.some((prefix) => mediaType.startsWith(prefix));
|
|
194
200
|
}
|
|
195
|
-
/**
|
|
196
|
-
* Compresses the passed value with the runtime's best available algorithm. Returns `undefined` if the data
|
|
197
|
-
* is too small, is not a string or binary value, or if the runtime does not offer compression.
|
|
198
|
-
*/
|
|
199
|
-
export async function maybeCompressValue(value) {
|
|
200
|
-
// Request compression is not that important so let's
|
|
201
|
-
// skip it instead of throwing for unsupported types.
|
|
202
|
-
const bytes = toBytes(value);
|
|
203
|
-
if (!bytes || bytes.byteLength < MIN_COMPRESS_BYTES)
|
|
204
|
-
return undefined;
|
|
205
|
-
return runtime.compress(bytes);
|
|
206
|
-
}
|
|
207
201
|
/**
|
|
208
202
|
* Reads an environment variable, on runtimes that have them.
|
|
209
203
|
*/
|