apify-client 3.0.0-beta.22 → 3.0.0-beta.24
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -0
- package/dist/apify_api_error.d.ts +3 -3
- package/dist/apify_api_error.js +11 -4
- package/dist/apify_client.d.ts +79 -5
- package/dist/apify_client.js +92 -16
- package/dist/base/api_client.d.ts +1 -1
- package/dist/bundle.js +14 -14
- package/dist/bundle.js.map +1 -1
- package/dist/generated/schemas.d.ts +110 -110
- package/dist/http_clients/axios.d.ts +60 -0
- package/dist/http_clients/axios.js +117 -0
- package/dist/http_clients/base.d.ts +333 -0
- package/dist/http_clients/base.js +583 -0
- package/dist/http_clients/index.d.ts +4 -0
- package/dist/http_clients/index.js +2 -0
- package/dist/http_compressors/base.d.ts +40 -0
- package/dist/http_compressors/base.js +9 -0
- package/dist/http_compressors/brotli.d.ts +33 -0
- package/dist/http_compressors/brotli.js +39 -0
- package/dist/http_compressors/gzip.d.ts +33 -0
- package/dist/http_compressors/gzip.js +39 -0
- package/dist/http_compressors/index.d.ts +6 -0
- package/dist/http_compressors/index.js +2 -0
- package/dist/http_compressors/resolve.d.ts +23 -0
- package/dist/http_compressors/resolve.js +27 -0
- package/dist/index.d.ts +3 -1
- package/dist/index.js +3 -1
- package/dist/invalid_response_body_error.d.ts +13 -0
- package/dist/invalid_response_body_error.js +17 -0
- package/dist/resource_clients/actor.js +2 -4
- package/dist/resource_clients/dataset.js +4 -3
- package/dist/resource_clients/key_value_store.js +1 -1
- package/dist/resource_clients/run.d.ts +1 -1
- package/dist/resource_clients/run.js +1 -2
- package/dist/resource_clients/task.js +1 -2
- package/dist/runtime/node.js +10 -19
- package/dist/runtime/types.d.ts +12 -10
- package/dist/runtime/web.js +4 -3
- package/dist/utils.d.ts +14 -7
- package/dist/utils.js +9 -15
- package/package.json +1 -3
- package/dist/http_client.d.ts +0 -55
- package/dist/http_client.js +0 -260
- package/dist/interceptors.d.ts +0 -19
- package/dist/interceptors.js +0 -123
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
import { runtime } from '#runtime';
|
|
3
|
+
import { parseArgument } from '../utils.js';
|
|
4
|
+
/** Lowest valid brotli quality, the fastest with the least compression. */
|
|
5
|
+
const MIN_QUALITY = 0;
|
|
6
|
+
/** Highest valid brotli quality, the slowest with the best compression. */
|
|
7
|
+
const MAX_QUALITY = 11;
|
|
8
|
+
/** Middle of the range, where brotli already beats gzip at a comparable CPU cost. */
|
|
9
|
+
const DEFAULT_QUALITY = 6;
|
|
10
|
+
const optionsSchema = z.strictObject({
|
|
11
|
+
quality: z.int().min(MIN_QUALITY).max(MAX_QUALITY).default(DEFAULT_QUALITY),
|
|
12
|
+
});
|
|
13
|
+
/**
|
|
14
|
+
* Compresses request bodies using brotli.
|
|
15
|
+
*
|
|
16
|
+
* Built on the `node:zlib` module, so it works wherever the client runs on Node.js.
|
|
17
|
+
*
|
|
18
|
+
* @example
|
|
19
|
+
* ```javascript
|
|
20
|
+
* import { ApifyClient, BrotliHttpCompressor } from 'apify-client';
|
|
21
|
+
*
|
|
22
|
+
* const client = new ApifyClient({ token: 'my-token', compression: new BrotliHttpCompressor({ quality: 11 }) });
|
|
23
|
+
* ```
|
|
24
|
+
*/
|
|
25
|
+
export class BrotliHttpCompressor {
|
|
26
|
+
contentEncoding = 'br';
|
|
27
|
+
#quality;
|
|
28
|
+
/**
|
|
29
|
+
* @param options - Compressor options.
|
|
30
|
+
* @throws {ArgumentValidationError} If `quality` is out of the valid range.
|
|
31
|
+
*/
|
|
32
|
+
constructor(options = {}) {
|
|
33
|
+
const { quality } = parseArgument(options, optionsSchema, 'BrotliHttpCompressorOptions');
|
|
34
|
+
this.#quality = quality;
|
|
35
|
+
}
|
|
36
|
+
async compress(data) {
|
|
37
|
+
return runtime.compress(data, { algorithm: 'br', quality: this.#quality });
|
|
38
|
+
}
|
|
39
|
+
}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import type { HttpCompressor } from './base.js';
|
|
2
|
+
/**
|
|
3
|
+
* Compresses request bodies using gzip.
|
|
4
|
+
*
|
|
5
|
+
* Built on the `node:zlib` module, so it works wherever the client runs on Node.js.
|
|
6
|
+
*
|
|
7
|
+
* @example
|
|
8
|
+
* ```javascript
|
|
9
|
+
* import { ApifyClient, GzipHttpCompressor } from 'apify-client';
|
|
10
|
+
*
|
|
11
|
+
* const client = new ApifyClient({ token: 'my-token', compression: new GzipHttpCompressor({ quality: 1 }) });
|
|
12
|
+
* ```
|
|
13
|
+
*/
|
|
14
|
+
export declare class GzipHttpCompressor implements HttpCompressor {
|
|
15
|
+
#private;
|
|
16
|
+
readonly contentEncoding = "gzip";
|
|
17
|
+
/**
|
|
18
|
+
* @param options - Compressor options.
|
|
19
|
+
* @throws {ArgumentValidationError} If `quality` is out of the valid range.
|
|
20
|
+
*/
|
|
21
|
+
constructor(options?: GzipHttpCompressorOptions);
|
|
22
|
+
compress(data: Buffer): Promise<Buffer>;
|
|
23
|
+
}
|
|
24
|
+
/**
|
|
25
|
+
* Options for {@link GzipHttpCompressor}.
|
|
26
|
+
*/
|
|
27
|
+
export interface GzipHttpCompressorOptions {
|
|
28
|
+
/**
|
|
29
|
+
* Compression level, from `1` (the fastest) to `9` (the best compression).
|
|
30
|
+
* @default 6
|
|
31
|
+
*/
|
|
32
|
+
quality?: number;
|
|
33
|
+
}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
import { runtime } from '#runtime';
|
|
3
|
+
import { parseArgument } from '../utils.js';
|
|
4
|
+
/** Lowest valid gzip quality, the fastest with the least compression. */
|
|
5
|
+
const MIN_QUALITY = 1;
|
|
6
|
+
/** Highest valid gzip quality, the slowest with the best compression. */
|
|
7
|
+
const MAX_QUALITY = 9;
|
|
8
|
+
/** zlib's own default. The levels above it roughly double the CPU cost for about a percent fewer bytes. */
|
|
9
|
+
const DEFAULT_QUALITY = 6;
|
|
10
|
+
const optionsSchema = z.strictObject({
|
|
11
|
+
quality: z.int().min(MIN_QUALITY).max(MAX_QUALITY).default(DEFAULT_QUALITY),
|
|
12
|
+
});
|
|
13
|
+
/**
|
|
14
|
+
* Compresses request bodies using gzip.
|
|
15
|
+
*
|
|
16
|
+
* Built on the `node:zlib` module, so it works wherever the client runs on Node.js.
|
|
17
|
+
*
|
|
18
|
+
* @example
|
|
19
|
+
* ```javascript
|
|
20
|
+
* import { ApifyClient, GzipHttpCompressor } from 'apify-client';
|
|
21
|
+
*
|
|
22
|
+
* const client = new ApifyClient({ token: 'my-token', compression: new GzipHttpCompressor({ quality: 1 }) });
|
|
23
|
+
* ```
|
|
24
|
+
*/
|
|
25
|
+
export class GzipHttpCompressor {
|
|
26
|
+
contentEncoding = 'gzip';
|
|
27
|
+
#quality;
|
|
28
|
+
/**
|
|
29
|
+
* @param options - Compressor options.
|
|
30
|
+
* @throws {ArgumentValidationError} If `quality` is out of the valid range.
|
|
31
|
+
*/
|
|
32
|
+
constructor(options = {}) {
|
|
33
|
+
const { quality } = parseArgument(options, optionsSchema, 'GzipHttpCompressorOptions');
|
|
34
|
+
this.#quality = quality;
|
|
35
|
+
}
|
|
36
|
+
async compress(data) {
|
|
37
|
+
return runtime.compress(data, { algorithm: 'gzip', quality: this.#quality });
|
|
38
|
+
}
|
|
39
|
+
}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
export type { HttpCompressor } from './base.js';
|
|
2
|
+
export { BrotliHttpCompressor } from './brotli.js';
|
|
3
|
+
export type { BrotliHttpCompressorOptions } from './brotli.js';
|
|
4
|
+
export { GzipHttpCompressor } from './gzip.js';
|
|
5
|
+
export type { GzipHttpCompressorOptions } from './gzip.js';
|
|
6
|
+
export type { HttpCompressionAlgorithm } from './resolve.js';
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
import type { HttpCompressor } from './base.js';
|
|
3
|
+
/**
|
|
4
|
+
* Compression algorithms the `compression` option of {@link ApifyClient} accepts by name. Each name selects the
|
|
5
|
+
* matching built-in compressor with its default quality: `'brotli'` a {@link BrotliHttpCompressor}, `'gzip'` a
|
|
6
|
+
* {@link GzipHttpCompressor}. The Apify API also accepts `deflate` and `identity`, which a custom
|
|
7
|
+
* {@link HttpCompressor} covers.
|
|
8
|
+
*/
|
|
9
|
+
export type HttpCompressionAlgorithm = 'brotli' | 'gzip';
|
|
10
|
+
/**
|
|
11
|
+
* Schema of the `compression` option: an algorithm name or an {@link HttpCompressor}.
|
|
12
|
+
* @internal
|
|
13
|
+
*/
|
|
14
|
+
export declare const compressionSchema: z.ZodUnion<readonly [z.ZodEnum<{
|
|
15
|
+
gzip: "gzip";
|
|
16
|
+
brotli: "brotli";
|
|
17
|
+
}>, z.ZodCustom<HttpCompressor, HttpCompressor>]>;
|
|
18
|
+
/**
|
|
19
|
+
* Turns the `compression` option into a ready-to-use {@link HttpCompressor}: a name into the matching built-in
|
|
20
|
+
* compressor with its default quality, a compressor into itself.
|
|
21
|
+
* @internal
|
|
22
|
+
*/
|
|
23
|
+
export declare function resolveCompressor(compression: HttpCompressionAlgorithm | HttpCompressor): HttpCompressor;
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
import { httpCompressorSchema } from './base.js';
|
|
3
|
+
import { BrotliHttpCompressor } from './brotli.js';
|
|
4
|
+
import { GzipHttpCompressor } from './gzip.js';
|
|
5
|
+
/**
|
|
6
|
+
* Schema of the `compression` option: an algorithm name or an {@link HttpCompressor}.
|
|
7
|
+
* @internal
|
|
8
|
+
*/
|
|
9
|
+
export const compressionSchema = z.union([
|
|
10
|
+
z.enum(['brotli', 'gzip']),
|
|
11
|
+
httpCompressorSchema,
|
|
12
|
+
]);
|
|
13
|
+
/**
|
|
14
|
+
* Turns the `compression` option into a ready-to-use {@link HttpCompressor}: a name into the matching built-in
|
|
15
|
+
* compressor with its default quality, a compressor into itself.
|
|
16
|
+
* @internal
|
|
17
|
+
*/
|
|
18
|
+
export function resolveCompressor(compression) {
|
|
19
|
+
if (typeof compression !== 'string')
|
|
20
|
+
return compression;
|
|
21
|
+
switch (compression) {
|
|
22
|
+
case 'gzip':
|
|
23
|
+
return new GzipHttpCompressor();
|
|
24
|
+
case 'brotli':
|
|
25
|
+
return new BrotliHttpCompressor();
|
|
26
|
+
}
|
|
27
|
+
}
|
package/dist/index.d.ts
CHANGED
|
@@ -29,6 +29,8 @@ export * from './resource_clients/webhook_dispatch_collection.js';
|
|
|
29
29
|
export * from './apify_api_error.js';
|
|
30
30
|
export { ArgumentValidationError } from '@apify/validations';
|
|
31
31
|
export * from './response_validation_error.js';
|
|
32
|
-
export
|
|
32
|
+
export * from './invalid_response_body_error.js';
|
|
33
|
+
export * from './http_clients/index.js';
|
|
34
|
+
export * from './http_compressors/index.js';
|
|
33
35
|
export type { PaginatedList, Dictionary } from './utils.js';
|
|
34
36
|
export type { Timeout, TimeoutOptions, TimeoutTier } from './timeouts.js';
|
package/dist/index.js
CHANGED
|
@@ -29,4 +29,6 @@ export * from './resource_clients/webhook_dispatch_collection.js';
|
|
|
29
29
|
export * from './apify_api_error.js';
|
|
30
30
|
export { ArgumentValidationError } from '@apify/validations';
|
|
31
31
|
export * from './response_validation_error.js';
|
|
32
|
-
export
|
|
32
|
+
export * from './invalid_response_body_error.js';
|
|
33
|
+
export * from './http_clients/index.js';
|
|
34
|
+
export * from './http_compressors/index.js';
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import type { HttpResponse } from './http_clients/base.js';
|
|
2
|
+
/**
|
|
3
|
+
* Thrown when a response body does not parse as its content type claims, typically a JSON document cut short by a
|
|
4
|
+
* connection dropped mid-response. The client retries such requests, so the error surfaces once the retries are
|
|
5
|
+
* exhausted.
|
|
6
|
+
*/
|
|
7
|
+
export declare class InvalidResponseBodyError extends Error {
|
|
8
|
+
code: string;
|
|
9
|
+
/** The response whose body did not parse, with the body as the transport returned it. */
|
|
10
|
+
response: HttpResponse;
|
|
11
|
+
cause: Error;
|
|
12
|
+
constructor(response: HttpResponse, cause: Error);
|
|
13
|
+
}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Thrown when a response body does not parse as its content type claims, typically a JSON document cut short by a
|
|
3
|
+
* connection dropped mid-response. The client retries such requests, so the error surfaces once the retries are
|
|
4
|
+
* exhausted.
|
|
5
|
+
*/
|
|
6
|
+
export class InvalidResponseBodyError extends Error {
|
|
7
|
+
code;
|
|
8
|
+
/** The response whose body did not parse, with the body as the transport returned it. */
|
|
9
|
+
response;
|
|
10
|
+
constructor(response, cause) {
|
|
11
|
+
super(`Response body could not be parsed.\nCause:${cause.message}`);
|
|
12
|
+
this.name = this.constructor.name;
|
|
13
|
+
this.code = 'invalid-response-body';
|
|
14
|
+
this.response = response;
|
|
15
|
+
this.cause = cause;
|
|
16
|
+
}
|
|
17
|
+
}
|
|
@@ -178,8 +178,7 @@ export class ActorClient extends ResourceClient {
|
|
|
178
178
|
method: 'POST',
|
|
179
179
|
data: input,
|
|
180
180
|
params: this.buildParams(params),
|
|
181
|
-
//
|
|
182
|
-
// to stringify functions to JSON, instead of omitting them.
|
|
181
|
+
// Actor input may carry page functions, which plain JSON serialization would drop.
|
|
183
182
|
stringifyFunctions: true,
|
|
184
183
|
timeoutSecs: this.timeoutForWaitForFinish(timeoutSecs, 'medium', waitForFinish),
|
|
185
184
|
};
|
|
@@ -281,8 +280,7 @@ export class ActorClient extends ResourceClient {
|
|
|
281
280
|
method: 'POST',
|
|
282
281
|
data: input,
|
|
283
282
|
params: this.buildParams({ build: parsed.build }),
|
|
284
|
-
//
|
|
285
|
-
// to stringify functions to JSON, instead of omitting them.
|
|
283
|
+
// Actor input may carry page functions, which plain JSON serialization would drop.
|
|
286
284
|
stringifyFunctions: true,
|
|
287
285
|
timeoutSecs: parsed.timeoutSecs ?? 'short',
|
|
288
286
|
};
|
|
@@ -247,7 +247,7 @@ export class DatasetClient extends ResourceClient {
|
|
|
247
247
|
format,
|
|
248
248
|
...query,
|
|
249
249
|
}),
|
|
250
|
-
|
|
250
|
+
responseType: 'buffer',
|
|
251
251
|
timeoutSecs,
|
|
252
252
|
});
|
|
253
253
|
return cast(data);
|
|
@@ -295,7 +295,7 @@ export class DatasetClient extends ResourceClient {
|
|
|
295
295
|
},
|
|
296
296
|
data: items,
|
|
297
297
|
params: this.buildParams(),
|
|
298
|
-
doNotRetryTimeouts: true,
|
|
298
|
+
doNotRetryTimeouts: true,
|
|
299
299
|
timeoutSecs,
|
|
300
300
|
});
|
|
301
301
|
}
|
|
@@ -370,6 +370,7 @@ export class DatasetClient extends ResourceClient {
|
|
|
370
370
|
return createdItemsPublicUrl.toString();
|
|
371
371
|
}
|
|
372
372
|
#createPaginationList(response, userProvidedDesc) {
|
|
373
|
+
const descHeader = response.headers['x-apify-pagination-desc'];
|
|
373
374
|
const page = {
|
|
374
375
|
items: response.data,
|
|
375
376
|
total: Number(response.headers['x-apify-pagination-total']),
|
|
@@ -379,7 +380,7 @@ export class DatasetClient extends ResourceClient {
|
|
|
379
380
|
count: response.data.length,
|
|
380
381
|
limit: Number(response.headers['x-apify-pagination-limit']), // API returns 999999999999 when no limit is used
|
|
381
382
|
// TODO: Replace this once https://github.com/apify/apify-core/issues/3503 is solved
|
|
382
|
-
desc: JSON.parse(
|
|
383
|
+
desc: typeof descHeader === 'string' ? JSON.parse(descHeader) : userProvidedDesc,
|
|
383
384
|
};
|
|
384
385
|
// The offset iterator paginates by the scanned number, so it travels with the page outside its public shape.
|
|
385
386
|
return Object.defineProperty(page, SCANNED_COUNT, {
|
|
@@ -328,7 +328,7 @@ export class KeyValueStoreClient extends ResourceClient {
|
|
|
328
328
|
timeoutSecs: parsed.timeoutSecs ?? 'long',
|
|
329
329
|
};
|
|
330
330
|
if (parsed.buffer)
|
|
331
|
-
requestOpts.
|
|
331
|
+
requestOpts.responseType = 'buffer';
|
|
332
332
|
if (parsed.stream)
|
|
333
333
|
requestOpts.responseType = 'stream';
|
|
334
334
|
try {
|
|
@@ -2,7 +2,7 @@ import type { RUN_GENERAL_ACCESS } from '@apify/consts';
|
|
|
2
2
|
import { Log } from '@apify/log';
|
|
3
3
|
import type { ApiClientOptionsWithOptionalResourcePath } from '../base/api_client.js';
|
|
4
4
|
import { ResourceClient } from '../base/resource_client.js';
|
|
5
|
-
import type { ApifyResponse } from '../
|
|
5
|
+
import type { ApifyResponse } from '../http_clients/index.js';
|
|
6
6
|
import type { TimeoutOptions } from '../timeouts.js';
|
|
7
7
|
import type { ActorInput, ActorRun } from './actor.js';
|
|
8
8
|
import { DatasetClient } from './dataset.js';
|
|
@@ -169,8 +169,7 @@ export class RunClient extends ResourceClient {
|
|
|
169
169
|
method: 'POST',
|
|
170
170
|
data: input,
|
|
171
171
|
params: this.buildParams(params),
|
|
172
|
-
//
|
|
173
|
-
// to stringify functions to JSON, instead of omitting them.
|
|
172
|
+
// Actor input may carry page functions, which plain JSON serialization would drop.
|
|
174
173
|
stringifyFunctions: true,
|
|
175
174
|
timeoutSecs: parsed.timeoutSecs ?? 'medium',
|
|
176
175
|
};
|
|
@@ -173,8 +173,7 @@ export class TaskClient extends ResourceClient {
|
|
|
173
173
|
method: 'POST',
|
|
174
174
|
data: input,
|
|
175
175
|
params: this.buildParams(params),
|
|
176
|
-
//
|
|
177
|
-
// to stringify functions to JSON, instead of omitting them.
|
|
176
|
+
// Actor input may carry page functions, which plain JSON serialization would drop.
|
|
178
177
|
stringifyFunctions: true,
|
|
179
178
|
headers: {
|
|
180
179
|
'Content-Type': 'application/json',
|
package/dist/runtime/node.js
CHANGED
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import os from 'node:os';
|
|
2
|
-
import { promisify } from 'node:util';
|
|
3
2
|
import { brotliCompress, constants, gzip } from 'node:zlib';
|
|
4
3
|
/**
|
|
5
4
|
* The Node.js implementation of {@link Runtime}. Deno and Bun resolve the `node` condition too and
|
|
@@ -8,24 +7,16 @@ import { brotliCompress, constants, gzip } from 'node:zlib';
|
|
|
8
7
|
export const runtime = {
|
|
9
8
|
isNode: true,
|
|
10
9
|
platform: `${os.platform()}; Node/${process.version}`,
|
|
11
|
-
async compress(data) {
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
// brotli, but usually do implement gzip.
|
|
22
|
-
}
|
|
23
|
-
try {
|
|
24
|
-
return { data: await promisify(gzip)(data), encoding: 'gzip' };
|
|
25
|
-
}
|
|
26
|
-
catch {
|
|
27
|
-
return undefined;
|
|
28
|
-
}
|
|
10
|
+
async compress(data, { algorithm, quality }) {
|
|
11
|
+
return new Promise((resolve, reject) => {
|
|
12
|
+
const done = (error, result) => (error ? reject(error) : resolve(result));
|
|
13
|
+
if (algorithm === 'br') {
|
|
14
|
+
brotliCompress(data, { params: { [constants.BROTLI_PARAM_QUALITY]: quality } }, done);
|
|
15
|
+
}
|
|
16
|
+
else {
|
|
17
|
+
gzip(data, { level: quality }, done);
|
|
18
|
+
}
|
|
19
|
+
});
|
|
29
20
|
},
|
|
30
21
|
async createHttpAgents({ timeoutMillis }) {
|
|
31
22
|
// Loaded on the first request rather than with the client: the proxy support pulls in a sizeable
|
package/dist/runtime/types.d.ts
CHANGED
|
@@ -1,12 +1,14 @@
|
|
|
1
1
|
import type http from 'node:http';
|
|
2
2
|
import type https from 'node:https';
|
|
3
3
|
/**
|
|
4
|
-
*
|
|
4
|
+
* What to compress a request body with, as the built-in `HttpCompressor` implementations ask for it.
|
|
5
5
|
* @internal
|
|
6
6
|
*/
|
|
7
|
-
export interface
|
|
8
|
-
|
|
9
|
-
|
|
7
|
+
export interface CompressionOptions {
|
|
8
|
+
/** Algorithm to compress with, named by the `Content-Encoding` value it produces. */
|
|
9
|
+
algorithm: 'br' | 'gzip';
|
|
10
|
+
/** Compression level, in the range the algorithm defines. */
|
|
11
|
+
quality: number;
|
|
10
12
|
}
|
|
11
13
|
/**
|
|
12
14
|
* The agents axios's Node.js adapter sends requests through.
|
|
@@ -25,9 +27,8 @@ export interface HttpAgents {
|
|
|
25
27
|
export interface Runtime {
|
|
26
28
|
/**
|
|
27
29
|
* Whether the Node.js implementation was selected, which the `node` condition settles when `#runtime` is
|
|
28
|
-
* resolved - so a Node.js application bundled for a browser or a neutral target reports `false`.
|
|
29
|
-
*
|
|
30
|
-
* gated on it.
|
|
30
|
+
* resolved - so a Node.js application bundled for a browser or a neutral target reports `false`. Log
|
|
31
|
+
* streaming, the `stream` record option and request body compression are gated on it.
|
|
31
32
|
*/
|
|
32
33
|
isNode: boolean;
|
|
33
34
|
/**
|
|
@@ -36,10 +37,11 @@ export interface Runtime {
|
|
|
36
37
|
*/
|
|
37
38
|
platform: string | undefined;
|
|
38
39
|
/**
|
|
39
|
-
* Compresses a request body,
|
|
40
|
-
*
|
|
40
|
+
* Compresses a request body. The built-in compressors go through it, so a bundle for a browser or an edge
|
|
41
|
+
* runtime carries no `node:zlib`. Throws where the runtime has no compression, which the client never
|
|
42
|
+
* reaches, since it compresses only where {@link isNode} holds.
|
|
41
43
|
*/
|
|
42
|
-
compress(data: Uint8Array): Promise<
|
|
44
|
+
compress(data: Uint8Array, options: CompressionOptions): Promise<Buffer>;
|
|
43
45
|
/**
|
|
44
46
|
* Creates the agents axios's Node.js adapter sends requests through, or resolves to `undefined` where
|
|
45
47
|
* axios does not use agents.
|
package/dist/runtime/web.js
CHANGED
|
@@ -5,10 +5,11 @@ export const runtime = {
|
|
|
5
5
|
isNode: false,
|
|
6
6
|
// Browsers do not let a page set the `User-Agent` header.
|
|
7
7
|
platform: undefined,
|
|
8
|
-
// No request compression: brotli
|
|
9
|
-
// list of headers the Apify API allows in a cross-origin request, so a browser would fail the
|
|
8
|
+
// No request compression: brotli and gzip have no Web API, and a `content-encoding` request header is not
|
|
9
|
+
// on the list of headers the Apify API allows in a cross-origin request, so a browser would fail the
|
|
10
|
+
// preflight. The client compresses in Node.js alone, so nothing reaches this.
|
|
10
11
|
async compress() {
|
|
11
|
-
|
|
12
|
+
throw new Error('Request body compression is only available in Node.js.');
|
|
12
13
|
},
|
|
13
14
|
// The XHR and fetch adapters of axios do not use agents.
|
|
14
15
|
async createHttpAgents() {
|
package/dist/utils.d.ts
CHANGED
|
@@ -3,10 +3,16 @@ import type { TypedArray } from 'type-fest';
|
|
|
3
3
|
import { z } from 'zod';
|
|
4
4
|
import type { ApifyApiError } from './apify_api_error.js';
|
|
5
5
|
import { parseArgument } from '@apify/validations';
|
|
6
|
-
import type { ApifyResponse } from './
|
|
7
|
-
import type { CompressedValue } from './runtime/types.js';
|
|
6
|
+
import type { ApifyResponse } from './http_clients/index.js';
|
|
8
7
|
import type { RequestQueueClientListRequestsOptions, RequestQueueClientListRequestsResult } from './resource_clients/request_queue.js';
|
|
9
8
|
import type { WebhookUpdateData } from './resource_clients/webhook.js';
|
|
9
|
+
/**
|
|
10
|
+
* Smallest request body, in bytes, that is worth compressing. A smaller body already fits in a single network
|
|
11
|
+
* packet, so compressing it costs CPU time without saving a round trip, and the compression format itself can
|
|
12
|
+
* make it larger.
|
|
13
|
+
* @internal
|
|
14
|
+
*/
|
|
15
|
+
export declare const MIN_COMPRESS_BYTES = 1024;
|
|
10
16
|
export declare const version: any;
|
|
11
17
|
export { parseArgument };
|
|
12
18
|
/**
|
|
@@ -71,6 +77,12 @@ export declare function bytesToBase64(bytes: Uint8Array): string;
|
|
|
71
77
|
* Concatenates byte chunks into one array.
|
|
72
78
|
*/
|
|
73
79
|
export declare function concatBytes(chunks: Uint8Array[]): Uint8Array;
|
|
80
|
+
/**
|
|
81
|
+
* Views a request body as bytes: a string is UTF-8 encoded, binary values are viewed in place. Anything else
|
|
82
|
+
* - a stream, a `Blob`, form data - is `undefined`.
|
|
83
|
+
* @internal
|
|
84
|
+
*/
|
|
85
|
+
export declare function toBytes(value: unknown): Uint8Array | undefined;
|
|
74
86
|
/**
|
|
75
87
|
* Decides whether a request body with the given content type is worth compressing.
|
|
76
88
|
*
|
|
@@ -81,11 +93,6 @@ export declare function concatBytes(chunks: Uint8Array[]): Uint8Array;
|
|
|
81
93
|
* @internal
|
|
82
94
|
*/
|
|
83
95
|
export declare function isCompressibleContentType(contentType?: string): boolean;
|
|
84
|
-
/**
|
|
85
|
-
* Compresses the passed value with the runtime's best available algorithm. Returns `undefined` if the data
|
|
86
|
-
* is too small, is not a string or binary value, or if the runtime does not offer compression.
|
|
87
|
-
*/
|
|
88
|
-
export declare function maybeCompressValue(value: unknown): Promise<CompressedValue | undefined>;
|
|
89
96
|
/**
|
|
90
97
|
* Reads an environment variable, on runtimes that have them.
|
|
91
98
|
*/
|
package/dist/utils.js
CHANGED
|
@@ -1,11 +1,16 @@
|
|
|
1
1
|
import { z } from 'zod';
|
|
2
2
|
import { NotFoundError } from './apify_api_error.js';
|
|
3
3
|
import { parseArgument } from '@apify/validations';
|
|
4
|
-
import { runtime } from '#runtime';
|
|
5
4
|
import { ResponseValidationError } from './response_validation_error.js';
|
|
6
5
|
// @ts-ignore if we enable `resolveJsonModule`, we end up with a `src` folder in `dist`
|
|
7
6
|
import packageJson from '../package.json' with { type: 'json' };
|
|
8
|
-
|
|
7
|
+
/**
|
|
8
|
+
* Smallest request body, in bytes, that is worth compressing. A smaller body already fits in a single network
|
|
9
|
+
* packet, so compressing it costs CPU time without saving a round trip, and the compression format itself can
|
|
10
|
+
* make it larger.
|
|
11
|
+
* @internal
|
|
12
|
+
*/
|
|
13
|
+
export const MIN_COMPRESS_BYTES = 1024;
|
|
9
14
|
const textEncoder = new TextEncoder();
|
|
10
15
|
// Only the version, so a bundler can drop the rest of the manifest.
|
|
11
16
|
export const { version } = packageJson;
|
|
@@ -160,8 +165,9 @@ export function concatBytes(chunks) {
|
|
|
160
165
|
/**
|
|
161
166
|
* Views a request body as bytes: a string is UTF-8 encoded, binary values are viewed in place. Anything else
|
|
162
167
|
* - a stream, a `Blob`, form data - is `undefined`.
|
|
168
|
+
* @internal
|
|
163
169
|
*/
|
|
164
|
-
function toBytes(value) {
|
|
170
|
+
export function toBytes(value) {
|
|
165
171
|
if (typeof value === 'string')
|
|
166
172
|
return textEncoder.encode(value);
|
|
167
173
|
if (!isBuffer(value))
|
|
@@ -192,18 +198,6 @@ export function isCompressibleContentType(contentType) {
|
|
|
192
198
|
return false;
|
|
193
199
|
return !ALREADY_COMPRESSED_MEDIA_TYPE_PREFIXES.some((prefix) => mediaType.startsWith(prefix));
|
|
194
200
|
}
|
|
195
|
-
/**
|
|
196
|
-
* Compresses the passed value with the runtime's best available algorithm. Returns `undefined` if the data
|
|
197
|
-
* is too small, is not a string or binary value, or if the runtime does not offer compression.
|
|
198
|
-
*/
|
|
199
|
-
export async function maybeCompressValue(value) {
|
|
200
|
-
// Request compression is not that important so let's
|
|
201
|
-
// skip it instead of throwing for unsupported types.
|
|
202
|
-
const bytes = toBytes(value);
|
|
203
|
-
if (!bytes || bytes.byteLength < MIN_COMPRESS_BYTES)
|
|
204
|
-
return undefined;
|
|
205
|
-
return runtime.compress(bytes);
|
|
206
|
-
}
|
|
207
201
|
/**
|
|
208
202
|
* Reads an environment variable, on runtimes that have them.
|
|
209
203
|
*/
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "apify-client",
|
|
3
|
-
"version": "3.0.0-beta.
|
|
3
|
+
"version": "3.0.0-beta.24",
|
|
4
4
|
"description": "Apify API client for JavaScript",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.0.0"
|
|
@@ -69,7 +69,6 @@
|
|
|
69
69
|
"@apify/validations": "^1.1.0",
|
|
70
70
|
"@crawlee/types": "^3.18.1",
|
|
71
71
|
"ansi-colors": "^4.1.3",
|
|
72
|
-
"async-retry": "^1.3.3",
|
|
73
72
|
"axios": "^1.20.0",
|
|
74
73
|
"content-type": "^1.0.5",
|
|
75
74
|
"proxy-agent": "^8.0.2",
|
|
@@ -85,7 +84,6 @@
|
|
|
85
84
|
"@rsbuild/plugin-node-polyfill": "^1.4.6",
|
|
86
85
|
"@rspack/cli": "^2.2.2",
|
|
87
86
|
"@rspack/core": "^2.2.2",
|
|
88
|
-
"@types/async-retry": "^1.4.9",
|
|
89
87
|
"@types/compression": "^1.8.1",
|
|
90
88
|
"@types/content-type": "^1.1.9",
|
|
91
89
|
"@types/express": "^5.0.6",
|
package/dist/http_client.d.ts
DELETED
|
@@ -1,55 +0,0 @@
|
|
|
1
|
-
import type http from 'node:http';
|
|
2
|
-
import type https from 'node:https';
|
|
3
|
-
import type { AxiosInstance, AxiosRequestConfig, AxiosResponse, InternalAxiosRequestConfig } from 'axios';
|
|
4
|
-
import type { Log } from '@apify/log';
|
|
5
|
-
import type { RequestInterceptorFunction } from './interceptors.js';
|
|
6
|
-
import type { Statistics } from './statistics.js';
|
|
7
|
-
import type { Timeout, TimeoutTier } from './timeouts.js';
|
|
8
|
-
export declare class HttpClient {
|
|
9
|
-
#private;
|
|
10
|
-
stats: Statistics;
|
|
11
|
-
maxRetries: number;
|
|
12
|
-
minDelayBetweenRetriesMillis: number;
|
|
13
|
-
userProvidedRequestInterceptors: RequestInterceptorFunction[];
|
|
14
|
-
logger: Log;
|
|
15
|
-
/** Duration of each timeout tier, in milliseconds. */
|
|
16
|
-
timeoutMillis: Record<TimeoutTier, number>;
|
|
17
|
-
/** Cap on the timeout of a single request attempt, in milliseconds. */
|
|
18
|
-
timeoutMaxMillis: number;
|
|
19
|
-
httpAgent?: http.Agent;
|
|
20
|
-
httpsAgent?: https.Agent;
|
|
21
|
-
axios: AxiosInstance;
|
|
22
|
-
workflowKey?: string;
|
|
23
|
-
constructor(options: HttpClientOptions);
|
|
24
|
-
call<T = any>(config: ApifyRequestConfig): Promise<ApifyResponse<T>>;
|
|
25
|
-
}
|
|
26
|
-
export interface ApifyRequestConfig extends Omit<AxiosRequestConfig, 'timeout'> {
|
|
27
|
-
stringifyFunctions?: boolean;
|
|
28
|
-
forceBuffer?: boolean;
|
|
29
|
-
doNotRetryTimeouts?: boolean;
|
|
30
|
-
/**
|
|
31
|
-
* Timeout of the request: a tier name, a number of seconds, or `'noTimeout'`. The client resolves it to
|
|
32
|
-
* the axios `timeout` in milliseconds for each attempt, before axios runs its interceptors, so a request
|
|
33
|
-
* interceptor already sees a number of milliseconds.
|
|
34
|
-
* @default 'medium'
|
|
35
|
-
*/
|
|
36
|
-
timeoutSecs?: Timeout;
|
|
37
|
-
}
|
|
38
|
-
export interface ApifyResponse<T = any> extends AxiosResponse<T> {
|
|
39
|
-
config: ApifyRequestConfig & InternalAxiosRequestConfig;
|
|
40
|
-
}
|
|
41
|
-
export interface HttpClientOptions {
|
|
42
|
-
apifyClientStats: Statistics;
|
|
43
|
-
maxRetries: number;
|
|
44
|
-
minDelayBetweenRetriesMillis: number;
|
|
45
|
-
requestInterceptors: RequestInterceptorFunction[];
|
|
46
|
-
timeoutShortSecs: number;
|
|
47
|
-
timeoutMediumSecs: number;
|
|
48
|
-
timeoutLongSecs: number;
|
|
49
|
-
timeoutMaxSecs: number;
|
|
50
|
-
logger: Log;
|
|
51
|
-
token?: string;
|
|
52
|
-
workflowKey?: string;
|
|
53
|
-
/** @internal */
|
|
54
|
-
userAgentSuffix?: string | string[];
|
|
55
|
-
}
|