apify-client 3.0.0-beta.23 → 3.0.0-beta.24
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -0
- package/dist/apify_api_error.d.ts +3 -3
- package/dist/apify_api_error.js +11 -4
- package/dist/apify_client.d.ts +70 -5
- package/dist/apify_client.js +91 -18
- package/dist/base/api_client.d.ts +1 -1
- package/dist/bundle.js +14 -14
- package/dist/bundle.js.map +1 -1
- package/dist/generated/schemas.d.ts +110 -110
- package/dist/http_clients/axios.d.ts +60 -0
- package/dist/http_clients/axios.js +117 -0
- package/dist/http_clients/base.d.ts +333 -0
- package/dist/http_clients/base.js +583 -0
- package/dist/http_clients/index.d.ts +4 -0
- package/dist/http_clients/index.js +2 -0
- package/dist/index.d.ts +2 -1
- package/dist/index.js +2 -1
- package/dist/invalid_response_body_error.d.ts +13 -0
- package/dist/invalid_response_body_error.js +17 -0
- package/dist/resource_clients/actor.js +2 -4
- package/dist/resource_clients/dataset.js +4 -3
- package/dist/resource_clients/key_value_store.js +1 -1
- package/dist/resource_clients/run.d.ts +1 -1
- package/dist/resource_clients/run.js +1 -2
- package/dist/resource_clients/task.js +1 -2
- package/dist/utils.d.ts +1 -1
- package/package.json +1 -3
- package/dist/http_client.d.ts +0 -62
- package/dist/http_client.js +0 -266
- package/dist/interceptors.d.ts +0 -26
- package/dist/interceptors.js +0 -153
|
@@ -247,7 +247,7 @@ export class DatasetClient extends ResourceClient {
|
|
|
247
247
|
format,
|
|
248
248
|
...query,
|
|
249
249
|
}),
|
|
250
|
-
|
|
250
|
+
responseType: 'buffer',
|
|
251
251
|
timeoutSecs,
|
|
252
252
|
});
|
|
253
253
|
return cast(data);
|
|
@@ -295,7 +295,7 @@ export class DatasetClient extends ResourceClient {
|
|
|
295
295
|
},
|
|
296
296
|
data: items,
|
|
297
297
|
params: this.buildParams(),
|
|
298
|
-
doNotRetryTimeouts: true,
|
|
298
|
+
doNotRetryTimeouts: true,
|
|
299
299
|
timeoutSecs,
|
|
300
300
|
});
|
|
301
301
|
}
|
|
@@ -370,6 +370,7 @@ export class DatasetClient extends ResourceClient {
|
|
|
370
370
|
return createdItemsPublicUrl.toString();
|
|
371
371
|
}
|
|
372
372
|
#createPaginationList(response, userProvidedDesc) {
|
|
373
|
+
const descHeader = response.headers['x-apify-pagination-desc'];
|
|
373
374
|
const page = {
|
|
374
375
|
items: response.data,
|
|
375
376
|
total: Number(response.headers['x-apify-pagination-total']),
|
|
@@ -379,7 +380,7 @@ export class DatasetClient extends ResourceClient {
|
|
|
379
380
|
count: response.data.length,
|
|
380
381
|
limit: Number(response.headers['x-apify-pagination-limit']), // API returns 999999999999 when no limit is used
|
|
381
382
|
// TODO: Replace this once https://github.com/apify/apify-core/issues/3503 is solved
|
|
382
|
-
desc: JSON.parse(
|
|
383
|
+
desc: typeof descHeader === 'string' ? JSON.parse(descHeader) : userProvidedDesc,
|
|
383
384
|
};
|
|
384
385
|
// The offset iterator paginates by the scanned number, so it travels with the page outside its public shape.
|
|
385
386
|
return Object.defineProperty(page, SCANNED_COUNT, {
|
|
@@ -328,7 +328,7 @@ export class KeyValueStoreClient extends ResourceClient {
|
|
|
328
328
|
timeoutSecs: parsed.timeoutSecs ?? 'long',
|
|
329
329
|
};
|
|
330
330
|
if (parsed.buffer)
|
|
331
|
-
requestOpts.
|
|
331
|
+
requestOpts.responseType = 'buffer';
|
|
332
332
|
if (parsed.stream)
|
|
333
333
|
requestOpts.responseType = 'stream';
|
|
334
334
|
try {
|
|
@@ -2,7 +2,7 @@ import type { RUN_GENERAL_ACCESS } from '@apify/consts';
|
|
|
2
2
|
import { Log } from '@apify/log';
|
|
3
3
|
import type { ApiClientOptionsWithOptionalResourcePath } from '../base/api_client.js';
|
|
4
4
|
import { ResourceClient } from '../base/resource_client.js';
|
|
5
|
-
import type { ApifyResponse } from '../
|
|
5
|
+
import type { ApifyResponse } from '../http_clients/index.js';
|
|
6
6
|
import type { TimeoutOptions } from '../timeouts.js';
|
|
7
7
|
import type { ActorInput, ActorRun } from './actor.js';
|
|
8
8
|
import { DatasetClient } from './dataset.js';
|
|
@@ -169,8 +169,7 @@ export class RunClient extends ResourceClient {
|
|
|
169
169
|
method: 'POST',
|
|
170
170
|
data: input,
|
|
171
171
|
params: this.buildParams(params),
|
|
172
|
-
//
|
|
173
|
-
// to stringify functions to JSON, instead of omitting them.
|
|
172
|
+
// Actor input may carry page functions, which plain JSON serialization would drop.
|
|
174
173
|
stringifyFunctions: true,
|
|
175
174
|
timeoutSecs: parsed.timeoutSecs ?? 'medium',
|
|
176
175
|
};
|
|
@@ -173,8 +173,7 @@ export class TaskClient extends ResourceClient {
|
|
|
173
173
|
method: 'POST',
|
|
174
174
|
data: input,
|
|
175
175
|
params: this.buildParams(params),
|
|
176
|
-
//
|
|
177
|
-
// to stringify functions to JSON, instead of omitting them.
|
|
176
|
+
// Actor input may carry page functions, which plain JSON serialization would drop.
|
|
178
177
|
stringifyFunctions: true,
|
|
179
178
|
headers: {
|
|
180
179
|
'Content-Type': 'application/json',
|
package/dist/utils.d.ts
CHANGED
|
@@ -3,7 +3,7 @@ import type { TypedArray } from 'type-fest';
|
|
|
3
3
|
import { z } from 'zod';
|
|
4
4
|
import type { ApifyApiError } from './apify_api_error.js';
|
|
5
5
|
import { parseArgument } from '@apify/validations';
|
|
6
|
-
import type { ApifyResponse } from './
|
|
6
|
+
import type { ApifyResponse } from './http_clients/index.js';
|
|
7
7
|
import type { RequestQueueClientListRequestsOptions, RequestQueueClientListRequestsResult } from './resource_clients/request_queue.js';
|
|
8
8
|
import type { WebhookUpdateData } from './resource_clients/webhook.js';
|
|
9
9
|
/**
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "apify-client",
|
|
3
|
-
"version": "3.0.0-beta.
|
|
3
|
+
"version": "3.0.0-beta.24",
|
|
4
4
|
"description": "Apify API client for JavaScript",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.0.0"
|
|
@@ -69,7 +69,6 @@
|
|
|
69
69
|
"@apify/validations": "^1.1.0",
|
|
70
70
|
"@crawlee/types": "^3.18.1",
|
|
71
71
|
"ansi-colors": "^4.1.3",
|
|
72
|
-
"async-retry": "^1.3.3",
|
|
73
72
|
"axios": "^1.20.0",
|
|
74
73
|
"content-type": "^1.0.5",
|
|
75
74
|
"proxy-agent": "^8.0.2",
|
|
@@ -85,7 +84,6 @@
|
|
|
85
84
|
"@rsbuild/plugin-node-polyfill": "^1.4.6",
|
|
86
85
|
"@rspack/cli": "^2.2.2",
|
|
87
86
|
"@rspack/core": "^2.2.2",
|
|
88
|
-
"@types/async-retry": "^1.4.9",
|
|
89
87
|
"@types/compression": "^1.8.1",
|
|
90
88
|
"@types/content-type": "^1.1.9",
|
|
91
89
|
"@types/express": "^5.0.6",
|
package/dist/http_client.d.ts
DELETED
|
@@ -1,62 +0,0 @@
|
|
|
1
|
-
import type http from 'node:http';
|
|
2
|
-
import type https from 'node:https';
|
|
3
|
-
import type { AxiosInstance, AxiosRequestConfig, AxiosResponse, InternalAxiosRequestConfig } from 'axios';
|
|
4
|
-
import type { Log } from '@apify/log';
|
|
5
|
-
import type { HttpCompressor } from './http_compressors/base.js';
|
|
6
|
-
import type { RequestInterceptorFunction } from './interceptors.js';
|
|
7
|
-
import type { Statistics } from './statistics.js';
|
|
8
|
-
import type { Timeout, TimeoutTier } from './timeouts.js';
|
|
9
|
-
export declare class HttpClient {
|
|
10
|
-
#private;
|
|
11
|
-
stats: Statistics;
|
|
12
|
-
maxRetries: number;
|
|
13
|
-
minDelayBetweenRetriesMillis: number;
|
|
14
|
-
userProvidedRequestInterceptors: RequestInterceptorFunction[];
|
|
15
|
-
/**
|
|
16
|
-
* Compressor applied to request bodies that are worth compressing. The request interceptor captures it at
|
|
17
|
-
* construction, so it is fixed for the lifetime of the client.
|
|
18
|
-
*/
|
|
19
|
-
readonly httpCompressor: HttpCompressor;
|
|
20
|
-
logger: Log;
|
|
21
|
-
/** Duration of each timeout tier, in milliseconds. */
|
|
22
|
-
timeoutMillis: Record<TimeoutTier, number>;
|
|
23
|
-
/** Cap on the timeout of a single request attempt, in milliseconds. */
|
|
24
|
-
timeoutMaxMillis: number;
|
|
25
|
-
httpAgent?: http.Agent;
|
|
26
|
-
httpsAgent?: https.Agent;
|
|
27
|
-
axios: AxiosInstance;
|
|
28
|
-
workflowKey?: string;
|
|
29
|
-
constructor(options: HttpClientOptions);
|
|
30
|
-
call<T = any>(config: ApifyRequestConfig): Promise<ApifyResponse<T>>;
|
|
31
|
-
}
|
|
32
|
-
export interface ApifyRequestConfig extends Omit<AxiosRequestConfig, 'timeout'> {
|
|
33
|
-
stringifyFunctions?: boolean;
|
|
34
|
-
forceBuffer?: boolean;
|
|
35
|
-
doNotRetryTimeouts?: boolean;
|
|
36
|
-
/**
|
|
37
|
-
* Timeout of the request: a tier name, a number of seconds, or `'noTimeout'`. The client resolves it to
|
|
38
|
-
* the axios `timeout` in milliseconds for each attempt, before axios runs its interceptors, so a request
|
|
39
|
-
* interceptor already sees a number of milliseconds.
|
|
40
|
-
* @default 'medium'
|
|
41
|
-
*/
|
|
42
|
-
timeoutSecs?: Timeout;
|
|
43
|
-
}
|
|
44
|
-
export interface ApifyResponse<T = any> extends AxiosResponse<T> {
|
|
45
|
-
config: ApifyRequestConfig & InternalAxiosRequestConfig;
|
|
46
|
-
}
|
|
47
|
-
export interface HttpClientOptions {
|
|
48
|
-
apifyClientStats: Statistics;
|
|
49
|
-
maxRetries: number;
|
|
50
|
-
minDelayBetweenRetriesMillis: number;
|
|
51
|
-
requestInterceptors: RequestInterceptorFunction[];
|
|
52
|
-
httpCompressor: HttpCompressor;
|
|
53
|
-
timeoutShortSecs: number;
|
|
54
|
-
timeoutMediumSecs: number;
|
|
55
|
-
timeoutLongSecs: number;
|
|
56
|
-
timeoutMaxSecs: number;
|
|
57
|
-
logger: Log;
|
|
58
|
-
token?: string;
|
|
59
|
-
workflowKey?: string;
|
|
60
|
-
/** @internal */
|
|
61
|
-
userAgentSuffix?: string | string[];
|
|
62
|
-
}
|
package/dist/http_client.js
DELETED
|
@@ -1,266 +0,0 @@
|
|
|
1
|
-
import retry from 'async-retry';
|
|
2
|
-
import axios, { AxiosHeaders } from 'axios';
|
|
3
|
-
import { APIFY_ENV_VARS } from '@apify/consts';
|
|
4
|
-
import { concatStreamToBuffer } from '@apify/utilities';
|
|
5
|
-
import { ApifyApiError } from './apify_api_error.js';
|
|
6
|
-
import { createRequestInterceptors, InvalidResponseBodyError, responseInterceptors } from './interceptors.js';
|
|
7
|
-
import { runtime } from '#runtime';
|
|
8
|
-
import { asArray, cast, getEnv, isStream, version } from './utils.js';
|
|
9
|
-
const RATE_LIMIT_EXCEEDED_STATUS_CODE = 429;
|
|
10
|
-
export class HttpClient {
|
|
11
|
-
stats;
|
|
12
|
-
maxRetries;
|
|
13
|
-
minDelayBetweenRetriesMillis;
|
|
14
|
-
userProvidedRequestInterceptors;
|
|
15
|
-
/**
|
|
16
|
-
* Compressor applied to request bodies that are worth compressing. The request interceptor captures it at
|
|
17
|
-
* construction, so it is fixed for the lifetime of the client.
|
|
18
|
-
*/
|
|
19
|
-
httpCompressor;
|
|
20
|
-
logger;
|
|
21
|
-
/** Duration of each timeout tier, in milliseconds. */
|
|
22
|
-
timeoutMillis;
|
|
23
|
-
/** Cap on the timeout of a single request attempt, in milliseconds. */
|
|
24
|
-
timeoutMaxMillis;
|
|
25
|
-
httpAgent;
|
|
26
|
-
httpsAgent;
|
|
27
|
-
axios;
|
|
28
|
-
workflowKey;
|
|
29
|
-
#httpAgentsPromise;
|
|
30
|
-
constructor(options) {
|
|
31
|
-
const { token } = options;
|
|
32
|
-
this.stats = options.apifyClientStats;
|
|
33
|
-
this.maxRetries = options.maxRetries;
|
|
34
|
-
this.minDelayBetweenRetriesMillis = options.minDelayBetweenRetriesMillis;
|
|
35
|
-
this.userProvidedRequestInterceptors = options.requestInterceptors;
|
|
36
|
-
this.httpCompressor = options.httpCompressor;
|
|
37
|
-
this.timeoutMillis = {
|
|
38
|
-
short: options.timeoutShortSecs * 1000,
|
|
39
|
-
medium: options.timeoutMediumSecs * 1000,
|
|
40
|
-
long: options.timeoutLongSecs * 1000,
|
|
41
|
-
};
|
|
42
|
-
this.timeoutMaxMillis = options.timeoutMaxSecs * 1000;
|
|
43
|
-
this.logger = options.logger;
|
|
44
|
-
this.workflowKey = options.workflowKey || getEnv(APIFY_ENV_VARS.WORKFLOW_KEY);
|
|
45
|
-
this.axios = axios.create({
|
|
46
|
-
// Disable axios's built-in proxy handling since we're using custom agents
|
|
47
|
-
proxy: false,
|
|
48
|
-
paramsSerializer: (params) => {
|
|
49
|
-
const formattedParams = Object.entries(params)
|
|
50
|
-
.filter(([, value]) => value !== undefined)
|
|
51
|
-
.map(([key, value]) => {
|
|
52
|
-
if (value instanceof Date) {
|
|
53
|
-
return [key, value.toISOString()];
|
|
54
|
-
}
|
|
55
|
-
const updatedValue = typeof value === 'boolean' ? Number(value) : value;
|
|
56
|
-
return [key, String(updatedValue)];
|
|
57
|
-
});
|
|
58
|
-
return new URLSearchParams(formattedParams).toString();
|
|
59
|
-
},
|
|
60
|
-
validateStatus: null,
|
|
61
|
-
// Interceptors serialize requests and parse responses instead. Empty arrays rather than `undefined`,
|
|
62
|
-
// which axios fills in with its default transforms.
|
|
63
|
-
transformRequest: [],
|
|
64
|
-
transformResponse: [],
|
|
65
|
-
responseType: 'arraybuffer',
|
|
66
|
-
// Every request sets its own timeout in `createRequestHandler`, so the default only backs a raw
|
|
67
|
-
// `axios.request()` call.
|
|
68
|
-
timeout: this.timeoutMaxMillis,
|
|
69
|
-
// maxBodyLength needs to be Infinity, because -1 falls back to a 10 MB default
|
|
70
|
-
// from an axios subdependency - 'follow-redirects'
|
|
71
|
-
maxBodyLength: Infinity,
|
|
72
|
-
// maxContentLength must be -1, because Infinity will cause axios to run super slow
|
|
73
|
-
// thanks to a bug that's now fixed, but not released yet https://github.com/axios/axios/pull/3738
|
|
74
|
-
maxContentLength: -1,
|
|
75
|
-
});
|
|
76
|
-
// Clean all default headers because they only make a mess and their merging is difficult to understand and buggy.
|
|
77
|
-
this.axios.defaults.headers = new AxiosHeaders();
|
|
78
|
-
// If workflow key is available, pass it as a header
|
|
79
|
-
if (this.workflowKey) {
|
|
80
|
-
this.axios.defaults.headers['X-Apify-Workflow-Key'] = this.workflowKey;
|
|
81
|
-
}
|
|
82
|
-
// Attach Authorization header for all requests if token was provided
|
|
83
|
-
if (token) {
|
|
84
|
-
this.axios.defaults.headers.Authorization = `Bearer ${token}`;
|
|
85
|
-
}
|
|
86
|
-
// Browsers do not let a page set the header, so it is only sent where the runtime describes its platform.
|
|
87
|
-
if (runtime.platform) {
|
|
88
|
-
const isAtHome = !!getEnv(APIFY_ENV_VARS.IS_AT_HOME);
|
|
89
|
-
let userAgent = `ApifyClient/${version} (${runtime.platform}); isAtHome/${isAtHome}`;
|
|
90
|
-
if (options.userAgentSuffix) {
|
|
91
|
-
userAgent += `; ${asArray(options.userAgentSuffix).join('; ')}`;
|
|
92
|
-
}
|
|
93
|
-
this.axios.defaults.headers['User-Agent'] = userAgent;
|
|
94
|
-
}
|
|
95
|
-
createRequestInterceptors(this.httpCompressor).forEach((i) => this.axios.interceptors.request.use(i));
|
|
96
|
-
this.userProvidedRequestInterceptors.forEach((i) => this.axios.interceptors.request.use(i));
|
|
97
|
-
responseInterceptors.forEach((i) => this.axios.interceptors.response.use(i));
|
|
98
|
-
}
|
|
99
|
-
async #ensureHttpAgents() {
|
|
100
|
-
this.#httpAgentsPromise ??= this.#initHttpAgents();
|
|
101
|
-
return this.#httpAgentsPromise;
|
|
102
|
-
}
|
|
103
|
-
async #initHttpAgents() {
|
|
104
|
-
const agents = await runtime.createHttpAgents({ timeoutMillis: this.timeoutMaxMillis });
|
|
105
|
-
if (!agents)
|
|
106
|
-
return;
|
|
107
|
-
this.httpAgent = agents.httpAgent;
|
|
108
|
-
this.httpsAgent = agents.httpsAgent;
|
|
109
|
-
this.axios.defaults.httpAgent = this.httpAgent;
|
|
110
|
-
this.axios.defaults.httpsAgent = this.httpsAgent;
|
|
111
|
-
}
|
|
112
|
-
async call(config) {
|
|
113
|
-
await this.#ensureHttpAgents();
|
|
114
|
-
this.stats.calls++;
|
|
115
|
-
const makeRequest = this.#createRequestHandler(config);
|
|
116
|
-
return retry(makeRequest, {
|
|
117
|
-
retries: this.maxRetries,
|
|
118
|
-
minTimeout: this.minDelayBetweenRetriesMillis,
|
|
119
|
-
onRetry: (error, attempt) => this.#onRequestRetry(error, attempt),
|
|
120
|
-
});
|
|
121
|
-
}
|
|
122
|
-
#informAboutStreamNoRetry() {
|
|
123
|
-
this.logger.warningOnce('Request body was a stream - retrying will not work, as part of it was already consumed.');
|
|
124
|
-
this.logger.warningOnce('If you want Apify client to handle retries for you, collect the stream into a buffer before sending it.');
|
|
125
|
-
}
|
|
126
|
-
/**
|
|
127
|
-
* Successful responses are returned, errors and unsuccessful
|
|
128
|
-
* status codes are retried. See the following functions for the
|
|
129
|
-
* retrying logic.
|
|
130
|
-
*/
|
|
131
|
-
#createRequestHandler(config) {
|
|
132
|
-
const { timeoutSecs = 'medium', ...axiosConfig } = config;
|
|
133
|
-
const makeRequest = async (stopTrying, attempt) => {
|
|
134
|
-
this.stats.requests++;
|
|
135
|
-
let response;
|
|
136
|
-
const requestIsStream = isStream(config.data);
|
|
137
|
-
try {
|
|
138
|
-
if (requestIsStream) {
|
|
139
|
-
// Handling redirects is not possible without buffering - part of the stream has already been sent and can't be recovered
|
|
140
|
-
// when server sends the redirect. Therefore we need to override this in Axios config to prevent it from buffering the body.
|
|
141
|
-
// see also axios/axios#1045
|
|
142
|
-
axiosConfig.maxRedirects = 0;
|
|
143
|
-
}
|
|
144
|
-
response = await this.axios.request({
|
|
145
|
-
...axiosConfig,
|
|
146
|
-
timeout: this.#computeTimeoutMillis(timeoutSecs, attempt),
|
|
147
|
-
});
|
|
148
|
-
if (this.#isStatusOk(response.status))
|
|
149
|
-
return response;
|
|
150
|
-
// A failed request with `responseType: 'stream'` carries the API error body in the stream. Read
|
|
151
|
-
// it so that `ApifyApiError` can parse it like any other error body. A body that cannot be read
|
|
152
|
-
// leaves the error without a message, which beats losing the status code to a stream error.
|
|
153
|
-
if (isStream(response.data)) {
|
|
154
|
-
response.data = await concatStreamToBuffer(response.data).catch(() => undefined);
|
|
155
|
-
}
|
|
156
|
-
}
|
|
157
|
-
catch (err) {
|
|
158
|
-
return cast(this.#handleRequestError(err, config, stopTrying));
|
|
159
|
-
}
|
|
160
|
-
if (response.status === RATE_LIMIT_EXCEEDED_STATUS_CODE) {
|
|
161
|
-
this.stats.addRateLimitError(attempt);
|
|
162
|
-
}
|
|
163
|
-
const apiError = ApifyApiError.fromResponse(response, attempt);
|
|
164
|
-
if (this.#isStatusCodeRetryable(response.status)) {
|
|
165
|
-
if (requestIsStream) {
|
|
166
|
-
this.#informAboutStreamNoRetry();
|
|
167
|
-
}
|
|
168
|
-
else {
|
|
169
|
-
// allow a retry
|
|
170
|
-
throw apiError;
|
|
171
|
-
}
|
|
172
|
-
}
|
|
173
|
-
stopTrying(apiError);
|
|
174
|
-
return response;
|
|
175
|
-
};
|
|
176
|
-
return makeRequest;
|
|
177
|
-
}
|
|
178
|
-
#isStatusOk(statusCode) {
|
|
179
|
-
return statusCode < 300;
|
|
180
|
-
}
|
|
181
|
-
/**
|
|
182
|
-
* Resolves `timeoutSecs` to the number of milliseconds the given attempt gets. A tier name resolves to
|
|
183
|
-
* its configured duration, a number is taken as seconds, and `'noTimeout'` becomes `0`, which axios reads
|
|
184
|
-
* as no timeout. The result doubles with each attempt and is capped at `timeoutMaxMillis`. A requested
|
|
185
|
-
* value above the cap is capped too, which warns once, since it does not take effect in full.
|
|
186
|
-
*/
|
|
187
|
-
#computeTimeoutMillis(timeoutSecs, attempt) {
|
|
188
|
-
if (timeoutSecs === 'noTimeout')
|
|
189
|
-
return 0;
|
|
190
|
-
const requestedMillis = typeof timeoutSecs === 'number' ? timeoutSecs * 1000 : this.timeoutMillis[timeoutSecs];
|
|
191
|
-
if (requestedMillis > this.timeoutMaxMillis) {
|
|
192
|
-
// `warningOnce` keys by message, so each requested value warns once.
|
|
193
|
-
this.logger.warningOnce(`The requested timeout of ${requestedMillis / 1000}s exceeds timeoutMaxSecs ` +
|
|
194
|
-
`(${this.timeoutMaxMillis / 1000}s) and is capped at it. ` +
|
|
195
|
-
'Raise timeoutMaxSecs on the client to allow longer request timeouts.');
|
|
196
|
-
}
|
|
197
|
-
return Math.min(requestedMillis * 2 ** (attempt - 1), this.timeoutMaxMillis);
|
|
198
|
-
}
|
|
199
|
-
/**
|
|
200
|
-
* Handles all unexpected errors that can happen, but are not
|
|
201
|
-
* Apify API typed errors. E.g. network errors, timeouts and so on.
|
|
202
|
-
*/
|
|
203
|
-
#handleRequestError(err, config, stopTrying) {
|
|
204
|
-
if (this.#isTimeoutError(err) && config.doNotRetryTimeouts) {
|
|
205
|
-
return stopTrying(err);
|
|
206
|
-
}
|
|
207
|
-
if (this.#isRetryableError(err)) {
|
|
208
|
-
if (isStream(config.data)) {
|
|
209
|
-
this.#informAboutStreamNoRetry();
|
|
210
|
-
}
|
|
211
|
-
else {
|
|
212
|
-
throw err;
|
|
213
|
-
}
|
|
214
|
-
}
|
|
215
|
-
return stopTrying(err);
|
|
216
|
-
}
|
|
217
|
-
/**
|
|
218
|
-
* Axios calls req.abort() on timeouts so timeout errors will
|
|
219
|
-
* have a code ECONNABORTED.
|
|
220
|
-
*/
|
|
221
|
-
#isTimeoutError(err) {
|
|
222
|
-
return err.code === 'ECONNABORTED';
|
|
223
|
-
}
|
|
224
|
-
/**
|
|
225
|
-
* We don't want to retry every exception thrown from Axios.
|
|
226
|
-
* The common denominator for retryable errors are network issues.
|
|
227
|
-
* @param {Error} err
|
|
228
|
-
* @private
|
|
229
|
-
*/
|
|
230
|
-
#isRetryableError(err) {
|
|
231
|
-
return this.#isNetworkError(err) || this.#isResponseBodyInvalid(err);
|
|
232
|
-
}
|
|
233
|
-
/**
|
|
234
|
-
* When a network connection to our API is interrupted in the middle of streaming
|
|
235
|
-
* a response, the request often does not fail, but simply contains
|
|
236
|
-
* an incomplete response. This can often be fixed by retrying.
|
|
237
|
-
*/
|
|
238
|
-
#isResponseBodyInvalid(err) {
|
|
239
|
-
return err instanceof InvalidResponseBodyError;
|
|
240
|
-
}
|
|
241
|
-
/**
|
|
242
|
-
* When a network request is attempted by axios and fails,
|
|
243
|
-
* it throws an AxiosError, which will have the request
|
|
244
|
-
* and config (and other) properties.
|
|
245
|
-
*/
|
|
246
|
-
#isNetworkError(err) {
|
|
247
|
-
const hasRequest = err.request && typeof err.request === 'object';
|
|
248
|
-
const hasConfig = err.config && typeof err.config === 'object';
|
|
249
|
-
return hasRequest && hasConfig;
|
|
250
|
-
}
|
|
251
|
-
/**
|
|
252
|
-
* We retry 429 (rate limit) and 500+.
|
|
253
|
-
* For status codes 300-499 (except 429) we do not retry the request,
|
|
254
|
-
* because it's probably caused by invalid url (redirect 3xx) or invalid user input (4xx).
|
|
255
|
-
*/
|
|
256
|
-
#isStatusCodeRetryable(statusCode) {
|
|
257
|
-
const isRateLimitError = statusCode === RATE_LIMIT_EXCEEDED_STATUS_CODE;
|
|
258
|
-
const isInternalError = statusCode >= 500;
|
|
259
|
-
return isRateLimitError || isInternalError;
|
|
260
|
-
}
|
|
261
|
-
#onRequestRetry(error, attempt) {
|
|
262
|
-
if (attempt === Math.round(this.maxRetries / 2)) {
|
|
263
|
-
this.logger.warning(`API request failed ${attempt} times. Max attempts: ${this.maxRetries + 1}.\nCause:${error.stack}`);
|
|
264
|
-
}
|
|
265
|
-
}
|
|
266
|
-
}
|
package/dist/interceptors.d.ts
DELETED
|
@@ -1,26 +0,0 @@
|
|
|
1
|
-
import type { AxiosInterceptorManager, AxiosResponse } from 'axios';
|
|
2
|
-
import type { ApifyRequestConfig, ApifyResponse } from './http_client.js';
|
|
3
|
-
import type { HttpCompressor } from './http_compressors/base.js';
|
|
4
|
-
/**
|
|
5
|
-
* This error exists for the quite common situation, where only a partial JSON response is received and
|
|
6
|
-
* an attempt to parse the JSON throws an error. In most cases this can be resolved by retrying the
|
|
7
|
-
* request. We do that by identifying this error in HttpClient.
|
|
8
|
-
*
|
|
9
|
-
* The properties mimic AxiosError for easier integration in HttpClient error handling.
|
|
10
|
-
*/
|
|
11
|
-
export declare class InvalidResponseBodyError extends Error {
|
|
12
|
-
code: string;
|
|
13
|
-
response: AxiosResponse;
|
|
14
|
-
cause: Error;
|
|
15
|
-
constructor(response: AxiosResponse, cause: Error);
|
|
16
|
-
}
|
|
17
|
-
export type RequestInterceptorFunction = Parameters<AxiosInterceptorManager<ApifyRequestConfig>['use']>[0];
|
|
18
|
-
export type ResponseInterceptorFunction = Parameters<AxiosInterceptorManager<ApifyResponse>['use']>[0];
|
|
19
|
-
/**
|
|
20
|
-
* The client's own request interceptors, in registration order. Axios runs request interceptors in the reverse
|
|
21
|
-
* order of registration, so the body is serialized before it is compressed, and interceptors registered later,
|
|
22
|
-
* such as the user-provided ones, run before both.
|
|
23
|
-
* @internal
|
|
24
|
-
*/
|
|
25
|
-
export declare function createRequestInterceptors(compressor: HttpCompressor): RequestInterceptorFunction[];
|
|
26
|
-
export declare const responseInterceptors: ResponseInterceptorFunction[];
|
package/dist/interceptors.js
DELETED
|
@@ -1,153 +0,0 @@
|
|
|
1
|
-
import axios, { AxiosHeaders } from 'axios';
|
|
2
|
-
import contentTypeParser from 'content-type';
|
|
3
|
-
import { maybeParseBody } from './body_parser.js';
|
|
4
|
-
import { runtime } from '#runtime';
|
|
5
|
-
import { isCompressibleContentType, MIN_COMPRESS_BYTES, toBytes } from './utils.js';
|
|
6
|
-
/**
|
|
7
|
-
* This error exists for the quite common situation, where only a partial JSON response is received and
|
|
8
|
-
* an attempt to parse the JSON throws an error. In most cases this can be resolved by retrying the
|
|
9
|
-
* request. We do that by identifying this error in HttpClient.
|
|
10
|
-
*
|
|
11
|
-
* The properties mimic AxiosError for easier integration in HttpClient error handling.
|
|
12
|
-
*/
|
|
13
|
-
export class InvalidResponseBodyError extends Error {
|
|
14
|
-
code;
|
|
15
|
-
response;
|
|
16
|
-
constructor(response, cause) {
|
|
17
|
-
super(`Response body could not be parsed.\nCause:${cause.message}`);
|
|
18
|
-
this.name = this.constructor.name;
|
|
19
|
-
this.code = 'invalid-response-body';
|
|
20
|
-
this.response = response;
|
|
21
|
-
this.cause = cause;
|
|
22
|
-
}
|
|
23
|
-
}
|
|
24
|
-
/**
|
|
25
|
-
* Reads a request header regardless of the casing it was set with, since HTTP header names are case-insensitive
|
|
26
|
-
* while the config keeps whatever casing the caller used.
|
|
27
|
-
*/
|
|
28
|
-
function getHeader(config, name) {
|
|
29
|
-
const wanted = name.toLowerCase();
|
|
30
|
-
const key = Object.keys(config.headers ?? {}).find((candidate) => candidate.toLowerCase() === wanted);
|
|
31
|
-
const value = key === undefined ? undefined : config.headers?.[key];
|
|
32
|
-
return typeof value === 'string' ? value : undefined;
|
|
33
|
-
}
|
|
34
|
-
/** Removes a request header regardless of the casing it was set with. */
|
|
35
|
-
function deleteHeader(config, name) {
|
|
36
|
-
const { headers } = config;
|
|
37
|
-
if (!headers)
|
|
38
|
-
return;
|
|
39
|
-
const wanted = name.toLowerCase();
|
|
40
|
-
for (const key of Object.keys(headers)) {
|
|
41
|
-
if (key.toLowerCase() === wanted)
|
|
42
|
-
delete headers[key];
|
|
43
|
-
}
|
|
44
|
-
}
|
|
45
|
-
function serializeRequest(config) {
|
|
46
|
-
// A string body with an explicit content type is already serialized and goes out as it is. The axios default
|
|
47
|
-
// transform would otherwise parse a JSON one in full just to check that it is valid, which for a body assembled
|
|
48
|
-
// from thousands of pre-serialized requests costs about as much as serializing them did.
|
|
49
|
-
if (typeof config.data === 'string' && getHeader(config, 'content-type'))
|
|
50
|
-
return config;
|
|
51
|
-
const [defaultTransform] = axios.defaults.transformRequest;
|
|
52
|
-
// The function not only serializes data, but it also adds correct headers.
|
|
53
|
-
const data = defaultTransform(config.data, config.headers);
|
|
54
|
-
// Actor inputs can include functions and we don't want to omit those,
|
|
55
|
-
// because it's convenient for users. JSON.stringify removes them.
|
|
56
|
-
// It's a bit inefficient that we serialize the JSON twice, but I feel
|
|
57
|
-
// it's a small price to pay. The axios default transform does a lot
|
|
58
|
-
// of body type checks and we would have to copy all of them to the resource clients.
|
|
59
|
-
if (config.stringifyFunctions) {
|
|
60
|
-
const contentTypeHeader = getHeader(config, 'content-type');
|
|
61
|
-
try {
|
|
62
|
-
const type = contentTypeHeader ? contentTypeParser.parse(contentTypeHeader).type : undefined;
|
|
63
|
-
if (type === 'application/json' && typeof config.data === 'object') {
|
|
64
|
-
config.data = stringifyWithFunctions(config.data);
|
|
65
|
-
}
|
|
66
|
-
else {
|
|
67
|
-
config.data = data;
|
|
68
|
-
}
|
|
69
|
-
}
|
|
70
|
-
catch {
|
|
71
|
-
config.data = data;
|
|
72
|
-
}
|
|
73
|
-
}
|
|
74
|
-
else {
|
|
75
|
-
config.data = data;
|
|
76
|
-
}
|
|
77
|
-
return config;
|
|
78
|
-
}
|
|
79
|
-
function ensureHeadersPrototype(config) {
|
|
80
|
-
if (config.headers && !(config.headers instanceof AxiosHeaders)) {
|
|
81
|
-
Object.setPrototypeOf(config.headers, AxiosHeaders.prototype);
|
|
82
|
-
}
|
|
83
|
-
return config;
|
|
84
|
-
}
|
|
85
|
-
/**
|
|
86
|
-
* JSON.stringify() that serializes functions to string instead
|
|
87
|
-
* of replacing them with null or removing them.
|
|
88
|
-
*/
|
|
89
|
-
function stringifyWithFunctions(obj) {
|
|
90
|
-
return JSON.stringify(obj, (_key, value) => {
|
|
91
|
-
return typeof value === 'function' ? value.toString() : value;
|
|
92
|
-
});
|
|
93
|
-
}
|
|
94
|
-
/**
|
|
95
|
-
* Compresses the request body with the client's compressor and labels it with the compressor's `Content-Encoding`.
|
|
96
|
-
*
|
|
97
|
-
* Runs after `serializeRequest`, so a JSON body is already a string here. A caller-set `Content-Encoding` is
|
|
98
|
-
* forwarded verbatim, which is how a pre-encoded body is uploaded, and `Content-Encoding: identity` opts a single
|
|
99
|
-
* request out of compression. The built-in compressors need `node:zlib`, so a body sent from a browser or an edge
|
|
100
|
-
* runtime goes out as it is.
|
|
101
|
-
*
|
|
102
|
-
* Compressing changes the body length, so any `Content-Length` the caller set describes the wrong body and has to
|
|
103
|
-
* go. Axios keeps a caller-set one over the size it computes, which would stall the request until it times out.
|
|
104
|
-
*/
|
|
105
|
-
async function maybeCompressRequest(config, compressor) {
|
|
106
|
-
if (!runtime.isNode)
|
|
107
|
-
return config;
|
|
108
|
-
const bytes = toBytes(config.data);
|
|
109
|
-
if (!bytes || bytes.byteLength < MIN_COMPRESS_BYTES)
|
|
110
|
-
return config;
|
|
111
|
-
// A caller-supplied encoding means the body is already encoded and the header describes it, so leave both alone.
|
|
112
|
-
if (getHeader(config, 'content-encoding'))
|
|
113
|
-
return config;
|
|
114
|
-
if (!isCompressibleContentType(getHeader(config, 'content-type')))
|
|
115
|
-
return config;
|
|
116
|
-
config.data = await compressor.compress(Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength));
|
|
117
|
-
config.headers ??= {};
|
|
118
|
-
config.headers['content-encoding'] = compressor.contentEncoding;
|
|
119
|
-
deleteHeader(config, 'content-length');
|
|
120
|
-
return config;
|
|
121
|
-
}
|
|
122
|
-
function parseResponseData(response) {
|
|
123
|
-
if (!response.data || // Nothing to do here.
|
|
124
|
-
response.config.responseType !== 'arraybuffer' || // We don't want to parse custom response types.
|
|
125
|
-
response.config.forceBuffer // Apify custom property to prevent parsing of buffer.
|
|
126
|
-
) {
|
|
127
|
-
return response;
|
|
128
|
-
}
|
|
129
|
-
// A `Buffer` from the Node.js adapter and an `ArrayBuffer` from the browser ones both carry `byteLength`.
|
|
130
|
-
if (!response.data.byteLength) {
|
|
131
|
-
// undefined is better than an empty buffer
|
|
132
|
-
response.data = undefined;
|
|
133
|
-
return response;
|
|
134
|
-
}
|
|
135
|
-
const contentTypeHeader = response.headers['content-type'];
|
|
136
|
-
try {
|
|
137
|
-
response.data = maybeParseBody(response.data, contentTypeHeader);
|
|
138
|
-
}
|
|
139
|
-
catch (err) {
|
|
140
|
-
throw new InvalidResponseBodyError(response, err);
|
|
141
|
-
}
|
|
142
|
-
return response;
|
|
143
|
-
}
|
|
144
|
-
/**
|
|
145
|
-
* The client's own request interceptors, in registration order. Axios runs request interceptors in the reverse
|
|
146
|
-
* order of registration, so the body is serialized before it is compressed, and interceptors registered later,
|
|
147
|
-
* such as the user-provided ones, run before both.
|
|
148
|
-
* @internal
|
|
149
|
-
*/
|
|
150
|
-
export function createRequestInterceptors(compressor) {
|
|
151
|
-
return [async (config) => maybeCompressRequest(config, compressor), serializeRequest, ensureHeadersPrototype];
|
|
152
|
-
}
|
|
153
|
-
export const responseInterceptors = [parseResponseData];
|