apify-client 3.0.0-beta.20 → 3.0.0-beta.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -2
- package/dist/apify_api_error.js +1 -2
- package/dist/apify_client.js +2 -2
- package/dist/body_parser.d.ts +3 -4
- package/dist/body_parser.js +17 -24
- package/dist/bundle.js +13 -13
- package/dist/bundle.js.map +1 -1
- package/dist/generated/schemas.d.ts +110 -110
- package/dist/http_client.js +22 -54
- package/dist/interceptors.js +3 -3
- package/dist/resource_clients/key_value_store.js +3 -2
- package/dist/resource_clients/log.js +5 -4
- package/dist/resource_clients/run.js +3 -2
- package/dist/runtime/node.d.ts +6 -0
- package/dist/runtime/node.js +58 -0
- package/dist/runtime/types.d.ts +50 -0
- package/dist/runtime/types.js +1 -0
- package/dist/runtime/web.d.ts +5 -0
- package/dist/runtime/web.js +17 -0
- package/dist/utils.d.ts +18 -14
- package/dist/utils.js +47 -60
- package/package.json +7 -1
package/dist/http_client.js
CHANGED
|
@@ -4,8 +4,8 @@ import { APIFY_ENV_VARS } from '@apify/consts';
|
|
|
4
4
|
import { concatStreamToBuffer } from '@apify/utilities';
|
|
5
5
|
import { ApifyApiError } from './apify_api_error.js';
|
|
6
6
|
import { InvalidResponseBodyError, requestInterceptors, responseInterceptors } from './interceptors.js';
|
|
7
|
-
import {
|
|
8
|
-
|
|
7
|
+
import { runtime } from '#runtime';
|
|
8
|
+
import { asArray, cast, getEnv, isStream, version } from './utils.js';
|
|
9
9
|
const RATE_LIMIT_EXCEEDED_STATUS_CODE = 429;
|
|
10
10
|
export class HttpClient {
|
|
11
11
|
stats;
|
|
@@ -21,8 +21,7 @@ export class HttpClient {
|
|
|
21
21
|
httpsAgent;
|
|
22
22
|
axios;
|
|
23
23
|
workflowKey;
|
|
24
|
-
#
|
|
25
|
-
#userAgentSuffix;
|
|
24
|
+
#httpAgentsPromise;
|
|
26
25
|
constructor(options) {
|
|
27
26
|
const { token } = options;
|
|
28
27
|
this.stats = options.apifyClientStats;
|
|
@@ -36,8 +35,7 @@ export class HttpClient {
|
|
|
36
35
|
};
|
|
37
36
|
this.timeoutMaxMillis = options.timeoutMaxSecs * 1000;
|
|
38
37
|
this.logger = options.logger;
|
|
39
|
-
this.workflowKey = options.workflowKey ||
|
|
40
|
-
this.#userAgentSuffix = options.userAgentSuffix;
|
|
38
|
+
this.workflowKey = options.workflowKey || getEnv(APIFY_ENV_VARS.WORKFLOW_KEY);
|
|
41
39
|
this.axios = axios.create({
|
|
42
40
|
// Disable axios's built-in proxy handling since we're using custom agents
|
|
43
41
|
proxy: false,
|
|
@@ -79,64 +77,34 @@ export class HttpClient {
|
|
|
79
77
|
if (token) {
|
|
80
78
|
this.axios.defaults.headers.Authorization = `Bearer ${token}`;
|
|
81
79
|
}
|
|
80
|
+
// Browsers do not let a page set the header, so it is only sent where the runtime describes its platform.
|
|
81
|
+
if (runtime.platform) {
|
|
82
|
+
const isAtHome = !!getEnv(APIFY_ENV_VARS.IS_AT_HOME);
|
|
83
|
+
let userAgent = `ApifyClient/${version} (${runtime.platform}); isAtHome/${isAtHome}`;
|
|
84
|
+
if (options.userAgentSuffix) {
|
|
85
|
+
userAgent += `; ${asArray(options.userAgentSuffix).join('; ')}`;
|
|
86
|
+
}
|
|
87
|
+
this.axios.defaults.headers['User-Agent'] = userAgent;
|
|
88
|
+
}
|
|
82
89
|
requestInterceptors.forEach((i) => this.axios.interceptors.request.use(i));
|
|
83
90
|
this.userProvidedRequestInterceptors.forEach((i) => this.axios.interceptors.request.use(i));
|
|
84
91
|
responseInterceptors.forEach((i) => this.axios.interceptors.response.use(i));
|
|
85
92
|
}
|
|
86
|
-
async #
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
this.#nodeInitPromise ??= this.#initNode();
|
|
90
|
-
return this.#nodeInitPromise;
|
|
93
|
+
async #ensureHttpAgents() {
|
|
94
|
+
this.#httpAgentsPromise ??= this.#initHttpAgents();
|
|
95
|
+
return this.#httpAgentsPromise;
|
|
91
96
|
}
|
|
92
|
-
async #
|
|
93
|
-
|
|
97
|
+
async #initHttpAgents() {
|
|
98
|
+
const agents = await runtime.createHttpAgents({ timeoutMillis: this.timeoutMaxMillis });
|
|
99
|
+
if (!agents)
|
|
94
100
|
return;
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
// Enhanced agent configuration based on agentkeepalive best practices:
|
|
98
|
-
// - Nagle's algorithm disabled for lower latency
|
|
99
|
-
// - Free socket timeout to prevent socket leaks
|
|
100
|
-
// - LIFO scheduling to reuse recent sockets
|
|
101
|
-
// - Socket TTL for connection freshness
|
|
102
|
-
const agentOptions = {
|
|
103
|
-
keepAlive: true,
|
|
104
|
-
// Timeout for inactive sockets
|
|
105
|
-
// Prevents socket leaks from idle connections
|
|
106
|
-
timeout: this.timeoutMaxMillis,
|
|
107
|
-
// Keep alive timeout for free sockets (15 seconds)
|
|
108
|
-
// Node.js will close unused sockets after this period
|
|
109
|
-
keepAliveMsecs: 15_000,
|
|
110
|
-
// Maximum number of sockets per host
|
|
111
|
-
maxSockets: 256,
|
|
112
|
-
maxFreeSockets: 256,
|
|
113
|
-
// LIFO scheduling - reuse most recently used sockets for better performance
|
|
114
|
-
scheduling: 'lifo',
|
|
115
|
-
};
|
|
116
|
-
// Use ProxyAgent which automatically detects proxy from environment variables
|
|
117
|
-
// and supports CONNECT tunneling
|
|
118
|
-
const proxyAgent = new ProxyAgent(agentOptions);
|
|
119
|
-
this.httpAgent = proxyAgent;
|
|
120
|
-
this.httpsAgent = proxyAgent;
|
|
121
|
-
// Disable Nagle's algorithm for lower latency
|
|
122
|
-
// This sends data immediately instead of buffering small packets
|
|
123
|
-
const setNoDelay = (socket) => {
|
|
124
|
-
socket.setNoDelay(true);
|
|
125
|
-
};
|
|
126
|
-
this.httpAgent.on('socket', setNoDelay);
|
|
127
|
-
this.httpsAgent.on('socket', setNoDelay);
|
|
101
|
+
this.httpAgent = agents.httpAgent;
|
|
102
|
+
this.httpsAgent = agents.httpsAgent;
|
|
128
103
|
this.axios.defaults.httpAgent = this.httpAgent;
|
|
129
104
|
this.axios.defaults.httpsAgent = this.httpsAgent;
|
|
130
|
-
// Works only in Node. Cannot be set in browser
|
|
131
|
-
const isAtHome = !!process.env[APIFY_ENV_VARS.IS_AT_HOME];
|
|
132
|
-
let userAgent = `ApifyClient/${version} (${os.platform()}; Node/${process.version}); isAtHome/${isAtHome}`;
|
|
133
|
-
if (this.#userAgentSuffix) {
|
|
134
|
-
userAgent += `; ${asArray(this.#userAgentSuffix).join('; ')}`;
|
|
135
|
-
}
|
|
136
|
-
this.axios.defaults.headers['User-Agent'] = userAgent;
|
|
137
105
|
}
|
|
138
106
|
async call(config) {
|
|
139
|
-
await this.#
|
|
107
|
+
await this.#ensureHttpAgents();
|
|
140
108
|
this.stats.calls++;
|
|
141
109
|
const makeRequest = this.#createRequestHandler(config);
|
|
142
110
|
return retry(makeRequest, {
|
package/dist/interceptors.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import axios, { AxiosHeaders } from 'axios';
|
|
2
2
|
import contentTypeParser from 'content-type';
|
|
3
3
|
import { maybeParseBody } from './body_parser.js';
|
|
4
|
-
import { isCompressibleContentType,
|
|
4
|
+
import { isCompressibleContentType, maybeCompressValue } from './utils.js';
|
|
5
5
|
/**
|
|
6
6
|
* This error exists for the quite common situation, where only a partial JSON response is received and
|
|
7
7
|
* an attempt to parse the JSON throws an error. In most cases this can be resolved by retrying the
|
|
@@ -100,8 +100,8 @@ function parseResponseData(response) {
|
|
|
100
100
|
) {
|
|
101
101
|
return response;
|
|
102
102
|
}
|
|
103
|
-
|
|
104
|
-
if (
|
|
103
|
+
// A `Buffer` from the Node.js adapter and an `ArrayBuffer` from the browser ones both carry `byteLength`.
|
|
104
|
+
if (!response.data.byteLength) {
|
|
105
105
|
// undefined is better than an empty buffer
|
|
106
106
|
response.data = undefined;
|
|
107
107
|
return response;
|
|
@@ -4,7 +4,8 @@ import { createHmacSignatureAsync, createStorageContentSignatureAsync } from '@a
|
|
|
4
4
|
import { ResourceClient } from '../base/resource_client.js';
|
|
5
5
|
import * as schemas from '../schemas.js';
|
|
6
6
|
import { timeoutOptionsSchema, timeoutOptionsShape } from '../timeouts.js';
|
|
7
|
-
import {
|
|
7
|
+
import { runtime } from '#runtime';
|
|
8
|
+
import { anyObjectSchema, applyQueryParamsToUrl, catchNotFoundOrThrow, isBuffer, isStream, parseArgument, parseResponse, } from '../utils.js';
|
|
8
9
|
const listKeysOptionsSchema = z.strictObject({
|
|
9
10
|
limit: z.number().min(0).optional(),
|
|
10
11
|
exclusiveStartKey: z.string().optional(),
|
|
@@ -310,7 +311,7 @@ export class KeyValueStoreClient extends ResourceClient {
|
|
|
310
311
|
async getRecord(key, options = {}) {
|
|
311
312
|
parseArgument(key, keySchema);
|
|
312
313
|
const parsed = parseArgument(options, getRecordOptionsSchema, 'KeyValueClientGetRecordOptions');
|
|
313
|
-
if (parsed.stream && !isNode
|
|
314
|
+
if (parsed.stream && !runtime.isNode) {
|
|
314
315
|
throw new Error('The stream option can only be used in Node.js environment.');
|
|
315
316
|
}
|
|
316
317
|
if ('disableRedirect' in parsed) {
|
|
@@ -3,7 +3,7 @@ import { z } from 'zod';
|
|
|
3
3
|
import log, { Logger, LogLevel } from '@apify/log';
|
|
4
4
|
import { ResourceClient } from '../base/resource_client.js';
|
|
5
5
|
import { timeoutOptionsShape } from '../timeouts.js';
|
|
6
|
-
import { cast, catchNotFoundForResourceOrThrow, parseArgument } from '../utils.js';
|
|
6
|
+
import { cast, catchNotFoundForResourceOrThrow, concatBytes, parseArgument } from '../utils.js';
|
|
7
7
|
const logOptionsSchema = z.strictObject({ raw: z.boolean().optional(), ...timeoutOptionsShape });
|
|
8
8
|
/**
|
|
9
9
|
* Client for accessing Actor run or build logs.
|
|
@@ -132,6 +132,7 @@ export class LoggerActorRedirect extends Logger {
|
|
|
132
132
|
export class StreamedLog {
|
|
133
133
|
#destinationLog;
|
|
134
134
|
#streamBuffer = [];
|
|
135
|
+
#decoder = new TextDecoder();
|
|
135
136
|
#splitMarker = /(?:\n|^)(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z)/g;
|
|
136
137
|
#relevancyTimeLimit;
|
|
137
138
|
#logClient;
|
|
@@ -184,7 +185,7 @@ export class StreamedLog {
|
|
|
184
185
|
}
|
|
185
186
|
const lastChunkRemainder = await this.#logStreamChunks(logStream);
|
|
186
187
|
// Process whatever is left when exiting. Maybe it is incomplete, maybe it is last log without EOL.
|
|
187
|
-
const lastMessage =
|
|
188
|
+
const lastMessage = this.#decoder.decode(lastChunkRemainder).trim();
|
|
188
189
|
if (lastMessage.length) {
|
|
189
190
|
this.#destinationLog.info(lastMessage);
|
|
190
191
|
}
|
|
@@ -205,7 +206,7 @@ export class StreamedLog {
|
|
|
205
206
|
const lastCompleteMessageIndex = chunkWithPreviousRemainder.lastIndexOf(0x0a);
|
|
206
207
|
previousChunkRemainder = chunkWithPreviousRemainder.slice(lastCompleteMessageIndex);
|
|
207
208
|
// Push complete part of the chunk to the buffer
|
|
208
|
-
this.#streamBuffer.push(
|
|
209
|
+
this.#streamBuffer.push(chunkWithPreviousRemainder.slice(0, lastCompleteMessageIndex));
|
|
209
210
|
this.#logBufferContent();
|
|
210
211
|
// Keep processing the new data until stopped
|
|
211
212
|
if (this.#stopLogging) {
|
|
@@ -218,7 +219,7 @@ export class StreamedLog {
|
|
|
218
219
|
* Parse the buffer and log complete messages.
|
|
219
220
|
*/
|
|
220
221
|
#logBufferContent() {
|
|
221
|
-
const allParts =
|
|
222
|
+
const allParts = this.#decoder.decode(concatBytes(this.#streamBuffer)).split(this.#splitMarker).slice(1);
|
|
222
223
|
// Parse the buffer parts into complete messages
|
|
223
224
|
const messageMarkers = allParts.filter((_, i) => i % 2 === 0);
|
|
224
225
|
const messageContents = allParts.filter((_, i) => i % 2 !== 0);
|
|
@@ -3,7 +3,8 @@ import { LEVELS, Log } from '@apify/log';
|
|
|
3
3
|
import { ResourceClient } from '../base/resource_client.js';
|
|
4
4
|
import * as schemas from '../schemas.js';
|
|
5
5
|
import { optionalTimeoutSchema, timeoutOptionsSchema, timeoutOptionsShape } from '../timeouts.js';
|
|
6
|
-
import {
|
|
6
|
+
import { runtime } from '#runtime';
|
|
7
|
+
import { anyObjectSchema, parseArgument, parseResponse } from '../utils.js';
|
|
7
8
|
import { DatasetClient } from './dataset.js';
|
|
8
9
|
import { KeyValueStoreClient } from './key_value_store.js';
|
|
9
10
|
import { LogClient, LoggerActorRedirect, StreamedLog } from './log.js';
|
|
@@ -426,7 +427,7 @@ export class RunClient extends ResourceClient {
|
|
|
426
427
|
parseArgument(options.timeoutSecs, optionalTimeoutSchema);
|
|
427
428
|
const { fromStart = true, timeoutSecs = 'long' } = options;
|
|
428
429
|
let { toLog } = options;
|
|
429
|
-
if (toLog === null || !isNode
|
|
430
|
+
if (toLog === null || !runtime.isNode) {
|
|
430
431
|
// Explicitly no logging or not in Node.js
|
|
431
432
|
return undefined;
|
|
432
433
|
}
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
import os from 'node:os';
|
|
2
|
+
import { promisify } from 'node:util';
|
|
3
|
+
import { brotliCompress, constants, gzip } from 'node:zlib';
|
|
4
|
+
/**
|
|
5
|
+
* The Node.js implementation of {@link Runtime}. Deno and Bun resolve the `node` condition too and
|
|
6
|
+
* provide the built-ins it uses.
|
|
7
|
+
*/
|
|
8
|
+
export const runtime = {
|
|
9
|
+
isNode: true,
|
|
10
|
+
platform: `${os.platform()}; Node/${process.version}`,
|
|
11
|
+
async compress(data) {
|
|
12
|
+
try {
|
|
13
|
+
// `promisify()` belongs inside the fallback chain: a partial `node:zlib` can export
|
|
14
|
+
// `brotliCompress` without implementing it, and `promisify(undefined)` throws. At module scope
|
|
15
|
+
// that would fail the whole import instead of falling through to gzip.
|
|
16
|
+
const options = { params: { [constants.BROTLI_PARAM_QUALITY]: 6 } };
|
|
17
|
+
return { data: await promisify(brotliCompress)(data, options), encoding: 'br' };
|
|
18
|
+
}
|
|
19
|
+
catch {
|
|
20
|
+
// Runtimes that only provide a partial `node:zlib` (Node.js compatibility shims) may not implement
|
|
21
|
+
// brotli, but usually do implement gzip.
|
|
22
|
+
}
|
|
23
|
+
try {
|
|
24
|
+
return { data: await promisify(gzip)(data), encoding: 'gzip' };
|
|
25
|
+
}
|
|
26
|
+
catch {
|
|
27
|
+
return undefined;
|
|
28
|
+
}
|
|
29
|
+
},
|
|
30
|
+
async createHttpAgents({ timeoutMillis }) {
|
|
31
|
+
// Loaded on the first request rather than with the client: the proxy support pulls in a sizeable
|
|
32
|
+
// dependency tree that a client which never sends a request should not pay for.
|
|
33
|
+
const { ProxyAgent } = await import('proxy-agent');
|
|
34
|
+
// We want to keep sockets alive for better performance.
|
|
35
|
+
const agentOptions = {
|
|
36
|
+
keepAlive: true,
|
|
37
|
+
// Timeout for inactive sockets
|
|
38
|
+
// Prevents socket leaks from idle connections
|
|
39
|
+
timeout: timeoutMillis,
|
|
40
|
+
// Keep alive timeout for free sockets (15 seconds)
|
|
41
|
+
// Node.js will close unused sockets after this period
|
|
42
|
+
keepAliveMsecs: 15_000,
|
|
43
|
+
// Maximum number of sockets per host
|
|
44
|
+
maxSockets: 256,
|
|
45
|
+
maxFreeSockets: 256,
|
|
46
|
+
// LIFO scheduling - reuse most recently used sockets for better performance
|
|
47
|
+
scheduling: 'lifo',
|
|
48
|
+
};
|
|
49
|
+
// ProxyAgent picks the proxy up from the environment variables and supports CONNECT tunneling.
|
|
50
|
+
const agent = new ProxyAgent(agentOptions);
|
|
51
|
+
// Disable Nagle's algorithm for lower latency
|
|
52
|
+
// This sends data immediately instead of buffering small packets
|
|
53
|
+
agent.on('socket', (socket) => {
|
|
54
|
+
socket.setNoDelay(true);
|
|
55
|
+
});
|
|
56
|
+
return { httpAgent: agent, httpsAgent: agent };
|
|
57
|
+
},
|
|
58
|
+
};
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
import type http from 'node:http';
|
|
2
|
+
import type https from 'node:https';
|
|
3
|
+
/**
|
|
4
|
+
* A request body compressed for transport, with the `content-encoding` value that declares it.
|
|
5
|
+
* @internal
|
|
6
|
+
*/
|
|
7
|
+
export interface CompressedValue {
|
|
8
|
+
data: Uint8Array;
|
|
9
|
+
encoding: 'br' | 'gzip';
|
|
10
|
+
}
|
|
11
|
+
/**
|
|
12
|
+
* The agents axios's Node.js adapter sends requests through.
|
|
13
|
+
* @internal
|
|
14
|
+
*/
|
|
15
|
+
export interface HttpAgents {
|
|
16
|
+
httpAgent: http.Agent;
|
|
17
|
+
httpsAgent: https.Agent;
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* The runtime-specific part of the client. `#runtime` resolves to the Node.js implementation under the `node`
|
|
21
|
+
* condition and to the Web API one everywhere else (see the `imports` field of `package.json`), so a bundler
|
|
22
|
+
* targeting a browser or an edge runtime never sees the Node.js built-ins the Node.js one uses.
|
|
23
|
+
* @internal
|
|
24
|
+
*/
|
|
25
|
+
export interface Runtime {
|
|
26
|
+
/**
|
|
27
|
+
* Whether the Node.js implementation was selected, which the `node` condition settles when `#runtime` is
|
|
28
|
+
* resolved - so a Node.js application bundled for a browser or a neutral target reports `false`. The
|
|
29
|
+
* features that need a Node.js `Readable` response - log streaming and the `stream` record option - are
|
|
30
|
+
* gated on it.
|
|
31
|
+
*/
|
|
32
|
+
isNode: boolean;
|
|
33
|
+
/**
|
|
34
|
+
* The platform part of the `User-Agent` header, or `undefined` where the runtime does not let a client
|
|
35
|
+
* set that header (browsers).
|
|
36
|
+
*/
|
|
37
|
+
platform: string | undefined;
|
|
38
|
+
/**
|
|
39
|
+
* Compresses a request body, or resolves to `undefined` where compression is not available. Compression
|
|
40
|
+
* is a best-effort optimization, so it never throws.
|
|
41
|
+
*/
|
|
42
|
+
compress(data: Uint8Array): Promise<CompressedValue | undefined>;
|
|
43
|
+
/**
|
|
44
|
+
* Creates the agents axios's Node.js adapter sends requests through, or resolves to `undefined` where
|
|
45
|
+
* axios does not use agents.
|
|
46
|
+
*/
|
|
47
|
+
createHttpAgents(options: {
|
|
48
|
+
timeoutMillis: number;
|
|
49
|
+
}): Promise<HttpAgents | undefined>;
|
|
50
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The {@link Runtime} for browsers and edge runtimes, built on Web APIs only.
|
|
3
|
+
*/
|
|
4
|
+
export const runtime = {
|
|
5
|
+
isNode: false,
|
|
6
|
+
// Browsers do not let a page set the `User-Agent` header.
|
|
7
|
+
platform: undefined,
|
|
8
|
+
// No request compression: brotli has no Web API, and a `content-encoding` request header is not on the
|
|
9
|
+
// list of headers the Apify API allows in a cross-origin request, so a browser would fail the preflight.
|
|
10
|
+
async compress() {
|
|
11
|
+
return undefined;
|
|
12
|
+
},
|
|
13
|
+
// The XHR and fetch adapters of axios do not use agents.
|
|
14
|
+
async createHttpAgents() {
|
|
15
|
+
return undefined;
|
|
16
|
+
},
|
|
17
|
+
};
|
package/dist/utils.d.ts
CHANGED
|
@@ -4,8 +4,10 @@ import { z } from 'zod';
|
|
|
4
4
|
import type { ApifyApiError } from './apify_api_error.js';
|
|
5
5
|
import { parseArgument } from '@apify/validations';
|
|
6
6
|
import type { ApifyResponse } from './http_client.js';
|
|
7
|
+
import type { CompressedValue } from './runtime/types.js';
|
|
7
8
|
import type { RequestQueueClientListRequestsOptions, RequestQueueClientListRequestsResult } from './resource_clients/request_queue.js';
|
|
8
9
|
import type { WebhookUpdateData } from './resource_clients/webhook.js';
|
|
10
|
+
export declare const version: any;
|
|
9
11
|
export { parseArgument };
|
|
10
12
|
/**
|
|
11
13
|
* Accepts any non-null, non-array object as a predicate for `z.custom()`.
|
|
@@ -59,10 +61,16 @@ export declare function catchNotFoundForResourceOrThrow(err: ApifyApiError, reso
|
|
|
59
61
|
* Helper function that converts array of webhooks to base64 string
|
|
60
62
|
*/
|
|
61
63
|
export declare function stringifyWebhooksToBase64(webhooks?: readonly WebhookUpdateData[]): string | undefined;
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
64
|
+
/**
|
|
65
|
+
* Encodes bytes as base64. `btoa()` takes a binary string, and the bytes are turned into one in slices,
|
|
66
|
+
* because spreading them all into a single `String.fromCharCode()` call overflows the argument limit on
|
|
67
|
+
* inputs of a few tens of kilobytes.
|
|
68
|
+
*/
|
|
69
|
+
export declare function bytesToBase64(bytes: Uint8Array): string;
|
|
70
|
+
/**
|
|
71
|
+
* Concatenates byte chunks into one array.
|
|
72
|
+
*/
|
|
73
|
+
export declare function concatBytes(chunks: Uint8Array[]): Uint8Array;
|
|
66
74
|
/**
|
|
67
75
|
* Decides whether a request body with the given content type is worth compressing.
|
|
68
76
|
*
|
|
@@ -74,10 +82,14 @@ export interface CompressedValue {
|
|
|
74
82
|
*/
|
|
75
83
|
export declare function isCompressibleContentType(contentType?: string): boolean;
|
|
76
84
|
/**
|
|
77
|
-
*
|
|
78
|
-
* too small
|
|
85
|
+
* Compresses the passed value with the runtime's best available algorithm. Returns `undefined` if the data
|
|
86
|
+
* is too small, is not a string or binary value, or if the runtime does not offer compression.
|
|
79
87
|
*/
|
|
80
88
|
export declare function maybeCompressValue(value: unknown): Promise<CompressedValue | undefined>;
|
|
89
|
+
/**
|
|
90
|
+
* Reads an environment variable, on runtimes that have them.
|
|
91
|
+
*/
|
|
92
|
+
export declare function getEnv(name: string): string | undefined;
|
|
81
93
|
/**
|
|
82
94
|
* Returns the UTF-8 byte length of a string.
|
|
83
95
|
*/
|
|
@@ -94,12 +106,8 @@ export declare function splitIntoJsonArrayBatches<T extends {
|
|
|
94
106
|
maxCount: number;
|
|
95
107
|
maxByteLength: number;
|
|
96
108
|
}): T[][];
|
|
97
|
-
export declare function isNode(): boolean;
|
|
98
109
|
export declare function isBuffer(value: unknown): value is Buffer | ArrayBuffer | TypedArray;
|
|
99
110
|
export declare function isStream(value: unknown): value is Readable;
|
|
100
|
-
export declare function getVersionData(): {
|
|
101
|
-
version: string;
|
|
102
|
-
};
|
|
103
111
|
/**
|
|
104
112
|
* Helper class to create async iterators from paginated list endpoints.
|
|
105
113
|
*/
|
|
@@ -108,10 +116,6 @@ export declare class RequestQueuePaginationIterator {
|
|
|
108
116
|
constructor(options: RequestQueuePaginationIteratorOptions);
|
|
109
117
|
[Symbol.asyncIterator](): AsyncIterator<RequestQueueClientListRequestsResult>;
|
|
110
118
|
}
|
|
111
|
-
declare global {
|
|
112
|
-
export const BROWSER_BUILD: boolean | undefined;
|
|
113
|
-
export const VERSION: string | undefined;
|
|
114
|
-
}
|
|
115
119
|
/**
|
|
116
120
|
* Options for creating a pagination iterator.
|
|
117
121
|
*/
|
package/dist/utils.js
CHANGED
|
@@ -1,10 +1,14 @@
|
|
|
1
1
|
import { z } from 'zod';
|
|
2
2
|
import { NotFoundError } from './apify_api_error.js';
|
|
3
3
|
import { parseArgument } from '@apify/validations';
|
|
4
|
+
import { runtime } from '#runtime';
|
|
4
5
|
import { ResponseValidationError } from './response_validation_error.js';
|
|
5
6
|
// @ts-ignore if we enable `resolveJsonModule`, we end up with a `src` folder in `dist`
|
|
6
7
|
import packageJson from '../package.json' with { type: 'json' };
|
|
7
8
|
const MIN_COMPRESS_BYTES = 1024;
|
|
9
|
+
const textEncoder = new TextEncoder();
|
|
10
|
+
// Only the version, so a bundler can drop the rest of the manifest.
|
|
11
|
+
export const { version } = packageJson;
|
|
8
12
|
/** Media type prefixes whose payloads carry their own compression, so compressing the request body is wasted work. */
|
|
9
13
|
const ALREADY_COMPRESSED_MEDIA_TYPE_PREFIXES = ['audio/', 'image/', 'video/'];
|
|
10
14
|
/** Exact media types whose payloads carry their own compression. */
|
|
@@ -126,39 +130,45 @@ export function catchNotFoundForResourceOrThrow(err, resourceId) {
|
|
|
126
130
|
export function stringifyWebhooksToBase64(webhooks) {
|
|
127
131
|
if (!webhooks)
|
|
128
132
|
return;
|
|
129
|
-
|
|
130
|
-
if (isNode()) {
|
|
131
|
-
return Buffer.from(webhooksJson, 'utf8').toString('base64');
|
|
132
|
-
}
|
|
133
|
-
const encoder = new TextEncoder();
|
|
134
|
-
const uint8Array = encoder.encode(webhooksJson);
|
|
135
|
-
return btoa(String.fromCharCode(...uint8Array));
|
|
133
|
+
return bytesToBase64(textEncoder.encode(JSON.stringify(webhooks)));
|
|
136
134
|
}
|
|
137
|
-
let brotliCompressPromisified;
|
|
138
135
|
/**
|
|
139
|
-
*
|
|
136
|
+
* Encodes bytes as base64. `btoa()` takes a binary string, and the bytes are turned into one in slices,
|
|
137
|
+
* because spreading them all into a single `String.fromCharCode()` call overflows the argument limit on
|
|
138
|
+
* inputs of a few tens of kilobytes.
|
|
140
139
|
*/
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
const options = { params: { [constants.BROTLI_PARAM_QUALITY]: 6 } };
|
|
147
|
-
brotliCompressPromisified = async (input) => compress(input, options);
|
|
140
|
+
export function bytesToBase64(bytes) {
|
|
141
|
+
const SLICE_LENGTH = 0x8000;
|
|
142
|
+
let binary = '';
|
|
143
|
+
for (let i = 0; i < bytes.length; i += SLICE_LENGTH) {
|
|
144
|
+
binary += String.fromCharCode(...bytes.subarray(i, i + SLICE_LENGTH));
|
|
148
145
|
}
|
|
149
|
-
return
|
|
146
|
+
return btoa(binary);
|
|
150
147
|
}
|
|
151
|
-
let gzipPromisified;
|
|
152
148
|
/**
|
|
153
|
-
*
|
|
149
|
+
* Concatenates byte chunks into one array.
|
|
154
150
|
*/
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
151
|
+
export function concatBytes(chunks) {
|
|
152
|
+
const result = new Uint8Array(chunks.reduce((length, chunk) => length + chunk.length, 0));
|
|
153
|
+
let offset = 0;
|
|
154
|
+
for (const chunk of chunks) {
|
|
155
|
+
result.set(chunk, offset);
|
|
156
|
+
offset += chunk.length;
|
|
160
157
|
}
|
|
161
|
-
return
|
|
158
|
+
return result;
|
|
159
|
+
}
|
|
160
|
+
/**
|
|
161
|
+
* Views a request body as bytes: a string is UTF-8 encoded, binary values are viewed in place. Anything else
|
|
162
|
+
* - a stream, a `Blob`, form data - is `undefined`.
|
|
163
|
+
*/
|
|
164
|
+
function toBytes(value) {
|
|
165
|
+
if (typeof value === 'string')
|
|
166
|
+
return textEncoder.encode(value);
|
|
167
|
+
if (!isBuffer(value))
|
|
168
|
+
return undefined;
|
|
169
|
+
return ArrayBuffer.isView(value)
|
|
170
|
+
? new Uint8Array(value.buffer, value.byteOffset, value.byteLength)
|
|
171
|
+
: new Uint8Array(value);
|
|
162
172
|
}
|
|
163
173
|
/**
|
|
164
174
|
* Decides whether a request body with the given content type is worth compressing.
|
|
@@ -183,40 +193,28 @@ export function isCompressibleContentType(contentType) {
|
|
|
183
193
|
return !ALREADY_COMPRESSED_MEDIA_TYPE_PREFIXES.some((prefix) => mediaType.startsWith(prefix));
|
|
184
194
|
}
|
|
185
195
|
/**
|
|
186
|
-
*
|
|
187
|
-
* too small
|
|
196
|
+
* Compresses the passed value with the runtime's best available algorithm. Returns `undefined` if the data
|
|
197
|
+
* is too small, is not a string or binary value, or if the runtime does not offer compression.
|
|
188
198
|
*/
|
|
189
199
|
export async function maybeCompressValue(value) {
|
|
190
|
-
if (!isNode())
|
|
191
|
-
return undefined;
|
|
192
200
|
// Request compression is not that important so let's
|
|
193
201
|
// skip it instead of throwing for unsupported types.
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
const areDataLargeEnough = Buffer.byteLength(value) >= MIN_COMPRESS_BYTES;
|
|
197
|
-
if (!areDataLargeEnough)
|
|
202
|
+
const bytes = toBytes(value);
|
|
203
|
+
if (!bytes || bytes.byteLength < MIN_COMPRESS_BYTES)
|
|
198
204
|
return undefined;
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
try {
|
|
207
|
-
return { data: await gzipValue(value), encoding: 'gzip' };
|
|
208
|
-
}
|
|
209
|
-
catch {
|
|
210
|
-
// Same reasoning as above: compression is a best-effort optimization, so skip it instead
|
|
211
|
-
// of failing the request.
|
|
212
|
-
return undefined;
|
|
213
|
-
}
|
|
205
|
+
return runtime.compress(bytes);
|
|
206
|
+
}
|
|
207
|
+
/**
|
|
208
|
+
* Reads an environment variable, on runtimes that have them.
|
|
209
|
+
*/
|
|
210
|
+
export function getEnv(name) {
|
|
211
|
+
return typeof process !== 'undefined' ? process.env?.[name] : undefined;
|
|
214
212
|
}
|
|
215
213
|
/**
|
|
216
214
|
* Returns the UTF-8 byte length of a string.
|
|
217
215
|
*/
|
|
218
216
|
export function utf8ByteLength(value) {
|
|
219
|
-
return
|
|
217
|
+
return textEncoder.encode(value).byteLength;
|
|
220
218
|
}
|
|
221
219
|
/**
|
|
222
220
|
* Splits JSON-serialized items into consecutive batches of at most `maxCount` items, each of which fits into a JSON
|
|
@@ -243,11 +241,6 @@ export function splitIntoJsonArrayBatches(items, { maxCount, maxByteLength }) {
|
|
|
243
241
|
batches.push(batch);
|
|
244
242
|
return batches;
|
|
245
243
|
}
|
|
246
|
-
export function isNode() {
|
|
247
|
-
if (typeof BROWSER_BUILD !== 'undefined')
|
|
248
|
-
return false;
|
|
249
|
-
return !!(typeof process !== 'undefined' && process.versions && process.versions.node);
|
|
250
|
-
}
|
|
251
244
|
export function isBuffer(value) {
|
|
252
245
|
// Tag checks rather than `instanceof`, to also match buffers from another realm. `isView()`
|
|
253
246
|
// additionally covers `DataView`, which is not raw binary content.
|
|
@@ -264,12 +257,6 @@ export function isStream(value) {
|
|
|
264
257
|
const { on, pipe } = value;
|
|
265
258
|
return typeof on === 'function' && typeof pipe === 'function';
|
|
266
259
|
}
|
|
267
|
-
export function getVersionData() {
|
|
268
|
-
if (typeof BROWSER_BUILD !== 'undefined') {
|
|
269
|
-
return { version: VERSION };
|
|
270
|
-
}
|
|
271
|
-
return packageJson;
|
|
272
|
-
}
|
|
273
260
|
/**
|
|
274
261
|
* Helper class to create async iterators from paginated list endpoints.
|
|
275
262
|
*/
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "apify-client",
|
|
3
|
-
"version": "3.0.0-beta.
|
|
3
|
+
"version": "3.0.0-beta.21",
|
|
4
4
|
"description": "Apify API client for JavaScript",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.0.0"
|
|
@@ -10,6 +10,12 @@
|
|
|
10
10
|
"types": "dist/index.d.ts",
|
|
11
11
|
"browser": "dist/bundle.js",
|
|
12
12
|
"unpkg": "dist/bundle.js",
|
|
13
|
+
"imports": {
|
|
14
|
+
"#runtime": {
|
|
15
|
+
"node": "./dist/runtime/node.js",
|
|
16
|
+
"default": "./dist/runtime/web.js"
|
|
17
|
+
}
|
|
18
|
+
},
|
|
13
19
|
"exports": {
|
|
14
20
|
"./package.json": "./package.json",
|
|
15
21
|
"./browser": "./dist/bundle.js",
|