apify-client 3.0.0-beta.21 → 3.0.0-beta.23
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/apify_client.d.ts +9 -0
- package/dist/apify_client.js +4 -1
- package/dist/bundle.js +11 -11
- package/dist/bundle.js.map +1 -1
- package/dist/http_client.d.ts +7 -0
- package/dist/http_client.js +8 -2
- package/dist/http_compressors/base.d.ts +40 -0
- package/dist/http_compressors/base.js +9 -0
- package/dist/http_compressors/brotli.d.ts +33 -0
- package/dist/http_compressors/brotli.js +39 -0
- package/dist/http_compressors/gzip.d.ts +33 -0
- package/dist/http_compressors/gzip.js +39 -0
- package/dist/http_compressors/index.d.ts +6 -0
- package/dist/http_compressors/index.js +2 -0
- package/dist/http_compressors/resolve.d.ts +23 -0
- package/dist/http_compressors/resolve.js +27 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1 -0
- package/dist/interceptors.d.ts +8 -1
- package/dist/interceptors.js +43 -13
- package/dist/resource_clients/dataset.d.ts +8 -10
- package/dist/resource_clients/dataset.js +8 -10
- package/dist/runtime/node.js +10 -19
- package/dist/runtime/types.d.ts +12 -10
- package/dist/runtime/web.js +4 -3
- package/dist/utils.d.ts +13 -6
- package/dist/utils.js +9 -15
- package/package.json +1 -1
package/dist/http_client.d.ts
CHANGED
|
@@ -2,6 +2,7 @@ import type http from 'node:http';
|
|
|
2
2
|
import type https from 'node:https';
|
|
3
3
|
import type { AxiosInstance, AxiosRequestConfig, AxiosResponse, InternalAxiosRequestConfig } from 'axios';
|
|
4
4
|
import type { Log } from '@apify/log';
|
|
5
|
+
import type { HttpCompressor } from './http_compressors/base.js';
|
|
5
6
|
import type { RequestInterceptorFunction } from './interceptors.js';
|
|
6
7
|
import type { Statistics } from './statistics.js';
|
|
7
8
|
import type { Timeout, TimeoutTier } from './timeouts.js';
|
|
@@ -11,6 +12,11 @@ export declare class HttpClient {
|
|
|
11
12
|
maxRetries: number;
|
|
12
13
|
minDelayBetweenRetriesMillis: number;
|
|
13
14
|
userProvidedRequestInterceptors: RequestInterceptorFunction[];
|
|
15
|
+
/**
|
|
16
|
+
* Compressor applied to request bodies that are worth compressing. The request interceptor captures it at
|
|
17
|
+
* construction, so it is fixed for the lifetime of the client.
|
|
18
|
+
*/
|
|
19
|
+
readonly httpCompressor: HttpCompressor;
|
|
14
20
|
logger: Log;
|
|
15
21
|
/** Duration of each timeout tier, in milliseconds. */
|
|
16
22
|
timeoutMillis: Record<TimeoutTier, number>;
|
|
@@ -43,6 +49,7 @@ export interface HttpClientOptions {
|
|
|
43
49
|
maxRetries: number;
|
|
44
50
|
minDelayBetweenRetriesMillis: number;
|
|
45
51
|
requestInterceptors: RequestInterceptorFunction[];
|
|
52
|
+
httpCompressor: HttpCompressor;
|
|
46
53
|
timeoutShortSecs: number;
|
|
47
54
|
timeoutMediumSecs: number;
|
|
48
55
|
timeoutLongSecs: number;
|
package/dist/http_client.js
CHANGED
|
@@ -3,7 +3,7 @@ import axios, { AxiosHeaders } from 'axios';
|
|
|
3
3
|
import { APIFY_ENV_VARS } from '@apify/consts';
|
|
4
4
|
import { concatStreamToBuffer } from '@apify/utilities';
|
|
5
5
|
import { ApifyApiError } from './apify_api_error.js';
|
|
6
|
-
import {
|
|
6
|
+
import { createRequestInterceptors, InvalidResponseBodyError, responseInterceptors } from './interceptors.js';
|
|
7
7
|
import { runtime } from '#runtime';
|
|
8
8
|
import { asArray, cast, getEnv, isStream, version } from './utils.js';
|
|
9
9
|
const RATE_LIMIT_EXCEEDED_STATUS_CODE = 429;
|
|
@@ -12,6 +12,11 @@ export class HttpClient {
|
|
|
12
12
|
maxRetries;
|
|
13
13
|
minDelayBetweenRetriesMillis;
|
|
14
14
|
userProvidedRequestInterceptors;
|
|
15
|
+
/**
|
|
16
|
+
* Compressor applied to request bodies that are worth compressing. The request interceptor captures it at
|
|
17
|
+
* construction, so it is fixed for the lifetime of the client.
|
|
18
|
+
*/
|
|
19
|
+
httpCompressor;
|
|
15
20
|
logger;
|
|
16
21
|
/** Duration of each timeout tier, in milliseconds. */
|
|
17
22
|
timeoutMillis;
|
|
@@ -28,6 +33,7 @@ export class HttpClient {
|
|
|
28
33
|
this.maxRetries = options.maxRetries;
|
|
29
34
|
this.minDelayBetweenRetriesMillis = options.minDelayBetweenRetriesMillis;
|
|
30
35
|
this.userProvidedRequestInterceptors = options.requestInterceptors;
|
|
36
|
+
this.httpCompressor = options.httpCompressor;
|
|
31
37
|
this.timeoutMillis = {
|
|
32
38
|
short: options.timeoutShortSecs * 1000,
|
|
33
39
|
medium: options.timeoutMediumSecs * 1000,
|
|
@@ -86,7 +92,7 @@ export class HttpClient {
|
|
|
86
92
|
}
|
|
87
93
|
this.axios.defaults.headers['User-Agent'] = userAgent;
|
|
88
94
|
}
|
|
89
|
-
|
|
95
|
+
createRequestInterceptors(this.httpCompressor).forEach((i) => this.axios.interceptors.request.use(i));
|
|
90
96
|
this.userProvidedRequestInterceptors.forEach((i) => this.axios.interceptors.request.use(i));
|
|
91
97
|
responseInterceptors.forEach((i) => this.axios.interceptors.response.use(i));
|
|
92
98
|
}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
/**
|
|
3
|
+
* Strategy for compressing HTTP request bodies.
|
|
4
|
+
*
|
|
5
|
+
* Implement this interface to create a custom compressor. Set `contentEncoding` to the value that should be sent in
|
|
6
|
+
* the `Content-Encoding` header and implement `compress()`. The client calls it only for bodies that are large
|
|
7
|
+
* enough to benefit from compression, whose content type does not already carry its own compression, and which the
|
|
8
|
+
* caller did not send with a `Content-Encoding` of their own.
|
|
9
|
+
*
|
|
10
|
+
* @example
|
|
11
|
+
* ```javascript
|
|
12
|
+
* import { ApifyClient } from 'apify-client';
|
|
13
|
+
*
|
|
14
|
+
* const identityCompressor = {
|
|
15
|
+
* contentEncoding: 'identity',
|
|
16
|
+
*
|
|
17
|
+
* async compress(data) {
|
|
18
|
+
* return data;
|
|
19
|
+
* },
|
|
20
|
+
* };
|
|
21
|
+
*
|
|
22
|
+
* const client = new ApifyClient({ token: 'my-token', compression: identityCompressor });
|
|
23
|
+
* ```
|
|
24
|
+
*/
|
|
25
|
+
export interface HttpCompressor {
|
|
26
|
+
/** Value sent in the `Content-Encoding` header, for example `gzip` or `br`. */
|
|
27
|
+
readonly contentEncoding: string;
|
|
28
|
+
/**
|
|
29
|
+
* Compresses a request body.
|
|
30
|
+
*
|
|
31
|
+
* @param data - The raw bytes to compress.
|
|
32
|
+
* @returns The compressed bytes.
|
|
33
|
+
*/
|
|
34
|
+
compress(data: Buffer): Promise<Buffer>;
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* Schema accepting anything shaped like an {@link HttpCompressor}.
|
|
38
|
+
* @internal
|
|
39
|
+
*/
|
|
40
|
+
export declare const httpCompressorSchema: z.ZodCustom<HttpCompressor, HttpCompressor>;
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
/**
|
|
3
|
+
* Schema accepting anything shaped like an {@link HttpCompressor}.
|
|
4
|
+
* @internal
|
|
5
|
+
*/
|
|
6
|
+
export const httpCompressorSchema = z.custom((value) => typeof value === 'object' &&
|
|
7
|
+
value !== null &&
|
|
8
|
+
typeof value.contentEncoding === 'string' &&
|
|
9
|
+
typeof value.compress === 'function', { error: 'Invalid input: expected an HttpCompressor' });
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import type { HttpCompressor } from './base.js';
|
|
2
|
+
/**
|
|
3
|
+
* Compresses request bodies using brotli.
|
|
4
|
+
*
|
|
5
|
+
* Built on the `node:zlib` module, so it works wherever the client runs on Node.js.
|
|
6
|
+
*
|
|
7
|
+
* @example
|
|
8
|
+
* ```javascript
|
|
9
|
+
* import { ApifyClient, BrotliHttpCompressor } from 'apify-client';
|
|
10
|
+
*
|
|
11
|
+
* const client = new ApifyClient({ token: 'my-token', compression: new BrotliHttpCompressor({ quality: 11 }) });
|
|
12
|
+
* ```
|
|
13
|
+
*/
|
|
14
|
+
export declare class BrotliHttpCompressor implements HttpCompressor {
|
|
15
|
+
#private;
|
|
16
|
+
readonly contentEncoding = "br";
|
|
17
|
+
/**
|
|
18
|
+
* @param options - Compressor options.
|
|
19
|
+
* @throws {ArgumentValidationError} If `quality` is out of the valid range.
|
|
20
|
+
*/
|
|
21
|
+
constructor(options?: BrotliHttpCompressorOptions);
|
|
22
|
+
compress(data: Buffer): Promise<Buffer>;
|
|
23
|
+
}
|
|
24
|
+
/**
|
|
25
|
+
* Options for {@link BrotliHttpCompressor}.
|
|
26
|
+
*/
|
|
27
|
+
export interface BrotliHttpCompressorOptions {
|
|
28
|
+
/**
|
|
29
|
+
* Compression level, from `0` (the fastest) to `11` (the best compression).
|
|
30
|
+
* @default 6
|
|
31
|
+
*/
|
|
32
|
+
quality?: number;
|
|
33
|
+
}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
import { runtime } from '#runtime';
|
|
3
|
+
import { parseArgument } from '../utils.js';
|
|
4
|
+
/** Lowest valid brotli quality, the fastest with the least compression. */
|
|
5
|
+
const MIN_QUALITY = 0;
|
|
6
|
+
/** Highest valid brotli quality, the slowest with the best compression. */
|
|
7
|
+
const MAX_QUALITY = 11;
|
|
8
|
+
/** Middle of the range, where brotli already beats gzip at a comparable CPU cost. */
|
|
9
|
+
const DEFAULT_QUALITY = 6;
|
|
10
|
+
const optionsSchema = z.strictObject({
|
|
11
|
+
quality: z.int().min(MIN_QUALITY).max(MAX_QUALITY).default(DEFAULT_QUALITY),
|
|
12
|
+
});
|
|
13
|
+
/**
|
|
14
|
+
* Compresses request bodies using brotli.
|
|
15
|
+
*
|
|
16
|
+
* Built on the `node:zlib` module, so it works wherever the client runs on Node.js.
|
|
17
|
+
*
|
|
18
|
+
* @example
|
|
19
|
+
* ```javascript
|
|
20
|
+
* import { ApifyClient, BrotliHttpCompressor } from 'apify-client';
|
|
21
|
+
*
|
|
22
|
+
* const client = new ApifyClient({ token: 'my-token', compression: new BrotliHttpCompressor({ quality: 11 }) });
|
|
23
|
+
* ```
|
|
24
|
+
*/
|
|
25
|
+
export class BrotliHttpCompressor {
|
|
26
|
+
contentEncoding = 'br';
|
|
27
|
+
#quality;
|
|
28
|
+
/**
|
|
29
|
+
* @param options - Compressor options.
|
|
30
|
+
* @throws {ArgumentValidationError} If `quality` is out of the valid range.
|
|
31
|
+
*/
|
|
32
|
+
constructor(options = {}) {
|
|
33
|
+
const { quality } = parseArgument(options, optionsSchema, 'BrotliHttpCompressorOptions');
|
|
34
|
+
this.#quality = quality;
|
|
35
|
+
}
|
|
36
|
+
async compress(data) {
|
|
37
|
+
return runtime.compress(data, { algorithm: 'br', quality: this.#quality });
|
|
38
|
+
}
|
|
39
|
+
}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import type { HttpCompressor } from './base.js';
|
|
2
|
+
/**
|
|
3
|
+
* Compresses request bodies using gzip.
|
|
4
|
+
*
|
|
5
|
+
* Built on the `node:zlib` module, so it works wherever the client runs on Node.js.
|
|
6
|
+
*
|
|
7
|
+
* @example
|
|
8
|
+
* ```javascript
|
|
9
|
+
* import { ApifyClient, GzipHttpCompressor } from 'apify-client';
|
|
10
|
+
*
|
|
11
|
+
* const client = new ApifyClient({ token: 'my-token', compression: new GzipHttpCompressor({ quality: 1 }) });
|
|
12
|
+
* ```
|
|
13
|
+
*/
|
|
14
|
+
export declare class GzipHttpCompressor implements HttpCompressor {
|
|
15
|
+
#private;
|
|
16
|
+
readonly contentEncoding = "gzip";
|
|
17
|
+
/**
|
|
18
|
+
* @param options - Compressor options.
|
|
19
|
+
* @throws {ArgumentValidationError} If `quality` is out of the valid range.
|
|
20
|
+
*/
|
|
21
|
+
constructor(options?: GzipHttpCompressorOptions);
|
|
22
|
+
compress(data: Buffer): Promise<Buffer>;
|
|
23
|
+
}
|
|
24
|
+
/**
|
|
25
|
+
* Options for {@link GzipHttpCompressor}.
|
|
26
|
+
*/
|
|
27
|
+
export interface GzipHttpCompressorOptions {
|
|
28
|
+
/**
|
|
29
|
+
* Compression level, from `1` (the fastest) to `9` (the best compression).
|
|
30
|
+
* @default 6
|
|
31
|
+
*/
|
|
32
|
+
quality?: number;
|
|
33
|
+
}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
import { runtime } from '#runtime';
|
|
3
|
+
import { parseArgument } from '../utils.js';
|
|
4
|
+
/** Lowest valid gzip quality, the fastest with the least compression. */
|
|
5
|
+
const MIN_QUALITY = 1;
|
|
6
|
+
/** Highest valid gzip quality, the slowest with the best compression. */
|
|
7
|
+
const MAX_QUALITY = 9;
|
|
8
|
+
/** zlib's own default. The levels above it roughly double the CPU cost for about a percent fewer bytes. */
|
|
9
|
+
const DEFAULT_QUALITY = 6;
|
|
10
|
+
const optionsSchema = z.strictObject({
|
|
11
|
+
quality: z.int().min(MIN_QUALITY).max(MAX_QUALITY).default(DEFAULT_QUALITY),
|
|
12
|
+
});
|
|
13
|
+
/**
|
|
14
|
+
* Compresses request bodies using gzip.
|
|
15
|
+
*
|
|
16
|
+
* Built on the `node:zlib` module, so it works wherever the client runs on Node.js.
|
|
17
|
+
*
|
|
18
|
+
* @example
|
|
19
|
+
* ```javascript
|
|
20
|
+
* import { ApifyClient, GzipHttpCompressor } from 'apify-client';
|
|
21
|
+
*
|
|
22
|
+
* const client = new ApifyClient({ token: 'my-token', compression: new GzipHttpCompressor({ quality: 1 }) });
|
|
23
|
+
* ```
|
|
24
|
+
*/
|
|
25
|
+
export class GzipHttpCompressor {
|
|
26
|
+
contentEncoding = 'gzip';
|
|
27
|
+
#quality;
|
|
28
|
+
/**
|
|
29
|
+
* @param options - Compressor options.
|
|
30
|
+
* @throws {ArgumentValidationError} If `quality` is out of the valid range.
|
|
31
|
+
*/
|
|
32
|
+
constructor(options = {}) {
|
|
33
|
+
const { quality } = parseArgument(options, optionsSchema, 'GzipHttpCompressorOptions');
|
|
34
|
+
this.#quality = quality;
|
|
35
|
+
}
|
|
36
|
+
async compress(data) {
|
|
37
|
+
return runtime.compress(data, { algorithm: 'gzip', quality: this.#quality });
|
|
38
|
+
}
|
|
39
|
+
}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
export type { HttpCompressor } from './base.js';
|
|
2
|
+
export { BrotliHttpCompressor } from './brotli.js';
|
|
3
|
+
export type { BrotliHttpCompressorOptions } from './brotli.js';
|
|
4
|
+
export { GzipHttpCompressor } from './gzip.js';
|
|
5
|
+
export type { GzipHttpCompressorOptions } from './gzip.js';
|
|
6
|
+
export type { HttpCompressionAlgorithm } from './resolve.js';
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
import type { HttpCompressor } from './base.js';
|
|
3
|
+
/**
|
|
4
|
+
* Compression algorithms the `compression` option of {@link ApifyClient} accepts by name. Each name selects the
|
|
5
|
+
* matching built-in compressor with its default quality: `'brotli'` a {@link BrotliHttpCompressor}, `'gzip'` a
|
|
6
|
+
* {@link GzipHttpCompressor}. The Apify API also accepts `deflate` and `identity`, which a custom
|
|
7
|
+
* {@link HttpCompressor} covers.
|
|
8
|
+
*/
|
|
9
|
+
export type HttpCompressionAlgorithm = 'brotli' | 'gzip';
|
|
10
|
+
/**
|
|
11
|
+
* Schema of the `compression` option: an algorithm name or an {@link HttpCompressor}.
|
|
12
|
+
* @internal
|
|
13
|
+
*/
|
|
14
|
+
export declare const compressionSchema: z.ZodUnion<readonly [z.ZodEnum<{
|
|
15
|
+
gzip: "gzip";
|
|
16
|
+
brotli: "brotli";
|
|
17
|
+
}>, z.ZodCustom<HttpCompressor, HttpCompressor>]>;
|
|
18
|
+
/**
|
|
19
|
+
* Turns the `compression` option into a ready-to-use {@link HttpCompressor}: a name into the matching built-in
|
|
20
|
+
* compressor with its default quality, a compressor into itself.
|
|
21
|
+
* @internal
|
|
22
|
+
*/
|
|
23
|
+
export declare function resolveCompressor(compression: HttpCompressionAlgorithm | HttpCompressor): HttpCompressor;
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
import { httpCompressorSchema } from './base.js';
|
|
3
|
+
import { BrotliHttpCompressor } from './brotli.js';
|
|
4
|
+
import { GzipHttpCompressor } from './gzip.js';
|
|
5
|
+
/**
|
|
6
|
+
* Schema of the `compression` option: an algorithm name or an {@link HttpCompressor}.
|
|
7
|
+
* @internal
|
|
8
|
+
*/
|
|
9
|
+
export const compressionSchema = z.union([
|
|
10
|
+
z.enum(['brotli', 'gzip']),
|
|
11
|
+
httpCompressorSchema,
|
|
12
|
+
]);
|
|
13
|
+
/**
|
|
14
|
+
* Turns the `compression` option into a ready-to-use {@link HttpCompressor}: a name into the matching built-in
|
|
15
|
+
* compressor with its default quality, a compressor into itself.
|
|
16
|
+
* @internal
|
|
17
|
+
*/
|
|
18
|
+
export function resolveCompressor(compression) {
|
|
19
|
+
if (typeof compression !== 'string')
|
|
20
|
+
return compression;
|
|
21
|
+
switch (compression) {
|
|
22
|
+
case 'gzip':
|
|
23
|
+
return new GzipHttpCompressor();
|
|
24
|
+
case 'brotli':
|
|
25
|
+
return new BrotliHttpCompressor();
|
|
26
|
+
}
|
|
27
|
+
}
|
package/dist/index.d.ts
CHANGED
|
@@ -30,5 +30,6 @@ export * from './apify_api_error.js';
|
|
|
30
30
|
export { ArgumentValidationError } from '@apify/validations';
|
|
31
31
|
export * from './response_validation_error.js';
|
|
32
32
|
export { InvalidResponseBodyError } from './interceptors.js';
|
|
33
|
+
export * from './http_compressors/index.js';
|
|
33
34
|
export type { PaginatedList, Dictionary } from './utils.js';
|
|
34
35
|
export type { Timeout, TimeoutOptions, TimeoutTier } from './timeouts.js';
|
package/dist/index.js
CHANGED
package/dist/interceptors.d.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import type { AxiosInterceptorManager, AxiosResponse } from 'axios';
|
|
2
2
|
import type { ApifyRequestConfig, ApifyResponse } from './http_client.js';
|
|
3
|
+
import type { HttpCompressor } from './http_compressors/base.js';
|
|
3
4
|
/**
|
|
4
5
|
* This error exists for the quite common situation, where only a partial JSON response is received and
|
|
5
6
|
* an attempt to parse the JSON throws an error. In most cases this can be resolved by retrying the
|
|
@@ -15,5 +16,11 @@ export declare class InvalidResponseBodyError extends Error {
|
|
|
15
16
|
}
|
|
16
17
|
export type RequestInterceptorFunction = Parameters<AxiosInterceptorManager<ApifyRequestConfig>['use']>[0];
|
|
17
18
|
export type ResponseInterceptorFunction = Parameters<AxiosInterceptorManager<ApifyResponse>['use']>[0];
|
|
18
|
-
|
|
19
|
+
/**
|
|
20
|
+
* The client's own request interceptors, in registration order. Axios runs request interceptors in the reverse
|
|
21
|
+
* order of registration, so the body is serialized before it is compressed, and interceptors registered later,
|
|
22
|
+
* such as the user-provided ones, run before both.
|
|
23
|
+
* @internal
|
|
24
|
+
*/
|
|
25
|
+
export declare function createRequestInterceptors(compressor: HttpCompressor): RequestInterceptorFunction[];
|
|
19
26
|
export declare const responseInterceptors: ResponseInterceptorFunction[];
|
package/dist/interceptors.js
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import axios, { AxiosHeaders } from 'axios';
|
|
2
2
|
import contentTypeParser from 'content-type';
|
|
3
3
|
import { maybeParseBody } from './body_parser.js';
|
|
4
|
-
import {
|
|
4
|
+
import { runtime } from '#runtime';
|
|
5
|
+
import { isCompressibleContentType, MIN_COMPRESS_BYTES, toBytes } from './utils.js';
|
|
5
6
|
/**
|
|
6
7
|
* This error exists for the quite common situation, where only a partial JSON response is received and
|
|
7
8
|
* an attempt to parse the JSON throws an error. In most cases this can be resolved by retrying the
|
|
@@ -30,6 +31,17 @@ function getHeader(config, name) {
|
|
|
30
31
|
const value = key === undefined ? undefined : config.headers?.[key];
|
|
31
32
|
return typeof value === 'string' ? value : undefined;
|
|
32
33
|
}
|
|
34
|
+
/** Removes a request header regardless of the casing it was set with. */
|
|
35
|
+
function deleteHeader(config, name) {
|
|
36
|
+
const { headers } = config;
|
|
37
|
+
if (!headers)
|
|
38
|
+
return;
|
|
39
|
+
const wanted = name.toLowerCase();
|
|
40
|
+
for (const key of Object.keys(headers)) {
|
|
41
|
+
if (key.toLowerCase() === wanted)
|
|
42
|
+
delete headers[key];
|
|
43
|
+
}
|
|
44
|
+
}
|
|
33
45
|
function serializeRequest(config) {
|
|
34
46
|
// A string body with an explicit content type is already serialized and goes out as it is. The axios default
|
|
35
47
|
// transform would otherwise parse a JSON one in full just to check that it is valid, which for a body assembled
|
|
@@ -79,18 +91,32 @@ function stringifyWithFunctions(obj) {
|
|
|
79
91
|
return typeof value === 'function' ? value.toString() : value;
|
|
80
92
|
});
|
|
81
93
|
}
|
|
82
|
-
|
|
94
|
+
/**
|
|
95
|
+
* Compresses the request body with the client's compressor and labels it with the compressor's `Content-Encoding`.
|
|
96
|
+
*
|
|
97
|
+
* Runs after `serializeRequest`, so a JSON body is already a string here. A caller-set `Content-Encoding` is
|
|
98
|
+
* forwarded verbatim, which is how a pre-encoded body is uploaded, and `Content-Encoding: identity` opts a single
|
|
99
|
+
* request out of compression. The built-in compressors need `node:zlib`, so a body sent from a browser or an edge
|
|
100
|
+
* runtime goes out as it is.
|
|
101
|
+
*
|
|
102
|
+
* Compressing changes the body length, so any `Content-Length` the caller set describes the wrong body and has to
|
|
103
|
+
* go. Axios keeps a caller-set one over the size it computes, which would stall the request until it times out.
|
|
104
|
+
*/
|
|
105
|
+
async function maybeCompressRequest(config, compressor) {
|
|
106
|
+
if (!runtime.isNode)
|
|
107
|
+
return config;
|
|
108
|
+
const bytes = toBytes(config.data);
|
|
109
|
+
if (!bytes || bytes.byteLength < MIN_COMPRESS_BYTES)
|
|
110
|
+
return config;
|
|
83
111
|
// A caller-supplied encoding means the body is already encoded and the header describes it, so leave both alone.
|
|
84
112
|
if (getHeader(config, 'content-encoding'))
|
|
85
113
|
return config;
|
|
86
114
|
if (!isCompressibleContentType(getHeader(config, 'content-type')))
|
|
87
115
|
return config;
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
config.data = maybeCompressed.data;
|
|
93
|
-
}
|
|
116
|
+
config.data = await compressor.compress(Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength));
|
|
117
|
+
config.headers ??= {};
|
|
118
|
+
config.headers['content-encoding'] = compressor.contentEncoding;
|
|
119
|
+
deleteHeader(config, 'content-length');
|
|
94
120
|
return config;
|
|
95
121
|
}
|
|
96
122
|
function parseResponseData(response) {
|
|
@@ -115,9 +141,13 @@ function parseResponseData(response) {
|
|
|
115
141
|
}
|
|
116
142
|
return response;
|
|
117
143
|
}
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
144
|
+
/**
|
|
145
|
+
* The client's own request interceptors, in registration order. Axios runs request interceptors in the reverse
|
|
146
|
+
* order of registration, so the body is serialized before it is compressed, and interceptors registered later,
|
|
147
|
+
* such as the user-provided ones, run before both.
|
|
148
|
+
* @internal
|
|
149
|
+
*/
|
|
150
|
+
export function createRequestInterceptors(compressor) {
|
|
151
|
+
return [async (config) => maybeCompressRequest(config, compressor), serializeRequest, ensureHeadersPrototype];
|
|
152
|
+
}
|
|
123
153
|
export const responseInterceptors = [parseResponseData];
|
|
@@ -158,13 +158,14 @@ export declare class DatasetClient<Data extends Record<string | number, any> = R
|
|
|
158
158
|
/**
|
|
159
159
|
* Stores one or more items into the dataset.
|
|
160
160
|
*
|
|
161
|
-
*
|
|
162
|
-
*
|
|
163
|
-
*
|
|
164
|
-
*
|
|
161
|
+
* Each item will be stored as a separate record in the dataset. Objects are automatically
|
|
162
|
+
* serialized to JSON. If you provide an array, all items will be stored in order. This method
|
|
163
|
+
* is idempotent - calling it multiple times with the same data will not create duplicates, but
|
|
164
|
+
* will append items each time.
|
|
165
165
|
*
|
|
166
|
-
* @param items - A single item
|
|
167
|
-
*
|
|
166
|
+
* @param items - A single item, an array of items, or a string that is the JSON serialization of
|
|
167
|
+
* either - the API only accepts an object or an array of objects, so a plain string
|
|
168
|
+
* is not a valid item on its own.
|
|
168
169
|
* @param options - Request options
|
|
169
170
|
* @param options.timeoutSecs - Timeout for the API request. Default is `'medium'`.
|
|
170
171
|
* @see https://docs.apify.com/api/v2/dataset-items-post
|
|
@@ -184,12 +185,9 @@ export declare class DatasetClient<Data extends Record<string | number, any> = R
|
|
|
184
185
|
* { url: 'https://test.com', title: 'Test' },
|
|
185
186
|
* { url: 'https://demo.com', title: 'Demo' }
|
|
186
187
|
* ]);
|
|
187
|
-
*
|
|
188
|
-
* // Store string items
|
|
189
|
-
* await client.dataset('my-dataset').pushItems(['item1', 'item2', 'item3']);
|
|
190
188
|
* ```
|
|
191
189
|
*/
|
|
192
|
-
pushItems(items: Data | Data[] | string
|
|
190
|
+
pushItems(items: Data | Data[] | string, options?: TimeoutOptions): Promise<void>;
|
|
193
191
|
/**
|
|
194
192
|
* Gets statistical information about the dataset.
|
|
195
193
|
*
|
|
@@ -43,7 +43,7 @@ const downloadItemsOptionsSchema = z.strictObject({
|
|
|
43
43
|
signature: z.string().optional(),
|
|
44
44
|
...timeoutOptionsShape,
|
|
45
45
|
});
|
|
46
|
-
const pushItemsSchema = z.union([itemSchema, z.string(), z.array(
|
|
46
|
+
const pushItemsSchema = z.union([itemSchema, z.string(), z.array(itemSchema)]);
|
|
47
47
|
// Apart from `timeoutSecs` and `expiresInSecs`, every option becomes a query parameter of the generated URL, so
|
|
48
48
|
// `chunkSize` (client-side only) and `signature` (which this method produces) are left out. The options type
|
|
49
49
|
// omits both to match.
|
|
@@ -255,13 +255,14 @@ export class DatasetClient extends ResourceClient {
|
|
|
255
255
|
/**
|
|
256
256
|
* Stores one or more items into the dataset.
|
|
257
257
|
*
|
|
258
|
-
*
|
|
259
|
-
*
|
|
260
|
-
*
|
|
261
|
-
*
|
|
258
|
+
* Each item will be stored as a separate record in the dataset. Objects are automatically
|
|
259
|
+
* serialized to JSON. If you provide an array, all items will be stored in order. This method
|
|
260
|
+
* is idempotent - calling it multiple times with the same data will not create duplicates, but
|
|
261
|
+
* will append items each time.
|
|
262
262
|
*
|
|
263
|
-
* @param items - A single item
|
|
264
|
-
*
|
|
263
|
+
* @param items - A single item, an array of items, or a string that is the JSON serialization of
|
|
264
|
+
* either - the API only accepts an object or an array of objects, so a plain string
|
|
265
|
+
* is not a valid item on its own.
|
|
265
266
|
* @param options - Request options
|
|
266
267
|
* @param options.timeoutSecs - Timeout for the API request. Default is `'medium'`.
|
|
267
268
|
* @see https://docs.apify.com/api/v2/dataset-items-post
|
|
@@ -281,9 +282,6 @@ export class DatasetClient extends ResourceClient {
|
|
|
281
282
|
* { url: 'https://test.com', title: 'Test' },
|
|
282
283
|
* { url: 'https://demo.com', title: 'Demo' }
|
|
283
284
|
* ]);
|
|
284
|
-
*
|
|
285
|
-
* // Store string items
|
|
286
|
-
* await client.dataset('my-dataset').pushItems(['item1', 'item2', 'item3']);
|
|
287
285
|
* ```
|
|
288
286
|
*/
|
|
289
287
|
async pushItems(items, options = {}) {
|
package/dist/runtime/node.js
CHANGED
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import os from 'node:os';
|
|
2
|
-
import { promisify } from 'node:util';
|
|
3
2
|
import { brotliCompress, constants, gzip } from 'node:zlib';
|
|
4
3
|
/**
|
|
5
4
|
* The Node.js implementation of {@link Runtime}. Deno and Bun resolve the `node` condition too and
|
|
@@ -8,24 +7,16 @@ import { brotliCompress, constants, gzip } from 'node:zlib';
|
|
|
8
7
|
export const runtime = {
|
|
9
8
|
isNode: true,
|
|
10
9
|
platform: `${os.platform()}; Node/${process.version}`,
|
|
11
|
-
async compress(data) {
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
// brotli, but usually do implement gzip.
|
|
22
|
-
}
|
|
23
|
-
try {
|
|
24
|
-
return { data: await promisify(gzip)(data), encoding: 'gzip' };
|
|
25
|
-
}
|
|
26
|
-
catch {
|
|
27
|
-
return undefined;
|
|
28
|
-
}
|
|
10
|
+
async compress(data, { algorithm, quality }) {
|
|
11
|
+
return new Promise((resolve, reject) => {
|
|
12
|
+
const done = (error, result) => (error ? reject(error) : resolve(result));
|
|
13
|
+
if (algorithm === 'br') {
|
|
14
|
+
brotliCompress(data, { params: { [constants.BROTLI_PARAM_QUALITY]: quality } }, done);
|
|
15
|
+
}
|
|
16
|
+
else {
|
|
17
|
+
gzip(data, { level: quality }, done);
|
|
18
|
+
}
|
|
19
|
+
});
|
|
29
20
|
},
|
|
30
21
|
async createHttpAgents({ timeoutMillis }) {
|
|
31
22
|
// Loaded on the first request rather than with the client: the proxy support pulls in a sizeable
|
package/dist/runtime/types.d.ts
CHANGED
|
@@ -1,12 +1,14 @@
|
|
|
1
1
|
import type http from 'node:http';
|
|
2
2
|
import type https from 'node:https';
|
|
3
3
|
/**
|
|
4
|
-
*
|
|
4
|
+
* What to compress a request body with, as the built-in `HttpCompressor` implementations ask for it.
|
|
5
5
|
* @internal
|
|
6
6
|
*/
|
|
7
|
-
export interface
|
|
8
|
-
|
|
9
|
-
|
|
7
|
+
export interface CompressionOptions {
|
|
8
|
+
/** Algorithm to compress with, named by the `Content-Encoding` value it produces. */
|
|
9
|
+
algorithm: 'br' | 'gzip';
|
|
10
|
+
/** Compression level, in the range the algorithm defines. */
|
|
11
|
+
quality: number;
|
|
10
12
|
}
|
|
11
13
|
/**
|
|
12
14
|
* The agents axios's Node.js adapter sends requests through.
|
|
@@ -25,9 +27,8 @@ export interface HttpAgents {
|
|
|
25
27
|
export interface Runtime {
|
|
26
28
|
/**
|
|
27
29
|
* Whether the Node.js implementation was selected, which the `node` condition settles when `#runtime` is
|
|
28
|
-
* resolved - so a Node.js application bundled for a browser or a neutral target reports `false`.
|
|
29
|
-
*
|
|
30
|
-
* gated on it.
|
|
30
|
+
* resolved - so a Node.js application bundled for a browser or a neutral target reports `false`. Log
|
|
31
|
+
* streaming, the `stream` record option and request body compression are gated on it.
|
|
31
32
|
*/
|
|
32
33
|
isNode: boolean;
|
|
33
34
|
/**
|
|
@@ -36,10 +37,11 @@ export interface Runtime {
|
|
|
36
37
|
*/
|
|
37
38
|
platform: string | undefined;
|
|
38
39
|
/**
|
|
39
|
-
* Compresses a request body,
|
|
40
|
-
*
|
|
40
|
+
* Compresses a request body. The built-in compressors go through it, so a bundle for a browser or an edge
|
|
41
|
+
* runtime carries no `node:zlib`. Throws where the runtime has no compression, which the client never
|
|
42
|
+
* reaches, since it compresses only where {@link isNode} holds.
|
|
41
43
|
*/
|
|
42
|
-
compress(data: Uint8Array): Promise<
|
|
44
|
+
compress(data: Uint8Array, options: CompressionOptions): Promise<Buffer>;
|
|
43
45
|
/**
|
|
44
46
|
* Creates the agents axios's Node.js adapter sends requests through, or resolves to `undefined` where
|
|
45
47
|
* axios does not use agents.
|
package/dist/runtime/web.js
CHANGED
|
@@ -5,10 +5,11 @@ export const runtime = {
|
|
|
5
5
|
isNode: false,
|
|
6
6
|
// Browsers do not let a page set the `User-Agent` header.
|
|
7
7
|
platform: undefined,
|
|
8
|
-
// No request compression: brotli
|
|
9
|
-
// list of headers the Apify API allows in a cross-origin request, so a browser would fail the
|
|
8
|
+
// No request compression: brotli and gzip have no Web API, and a `content-encoding` request header is not
|
|
9
|
+
// on the list of headers the Apify API allows in a cross-origin request, so a browser would fail the
|
|
10
|
+
// preflight. The client compresses in Node.js alone, so nothing reaches this.
|
|
10
11
|
async compress() {
|
|
11
|
-
|
|
12
|
+
throw new Error('Request body compression is only available in Node.js.');
|
|
12
13
|
},
|
|
13
14
|
// The XHR and fetch adapters of axios do not use agents.
|
|
14
15
|
async createHttpAgents() {
|