apify-client 3.0.0-beta.23 → 3.0.0-beta.25

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -247,7 +247,7 @@ export class DatasetClient extends ResourceClient {
247
247
  format,
248
248
  ...query,
249
249
  }),
250
- forceBuffer: true,
250
+ responseType: 'buffer',
251
251
  timeoutSecs,
252
252
  });
253
253
  return cast(data);
@@ -295,7 +295,7 @@ export class DatasetClient extends ResourceClient {
295
295
  },
296
296
  data: items,
297
297
  params: this.buildParams(),
298
- doNotRetryTimeouts: true, // see timeoutSecs handling in http-client
298
+ doNotRetryTimeouts: true,
299
299
  timeoutSecs,
300
300
  });
301
301
  }
@@ -370,6 +370,7 @@ export class DatasetClient extends ResourceClient {
370
370
  return createdItemsPublicUrl.toString();
371
371
  }
372
372
  #createPaginationList(response, userProvidedDesc) {
373
+ const descHeader = response.headers['x-apify-pagination-desc'];
373
374
  const page = {
374
375
  items: response.data,
375
376
  total: Number(response.headers['x-apify-pagination-total']),
@@ -379,7 +380,7 @@ export class DatasetClient extends ResourceClient {
379
380
  count: response.data.length,
380
381
  limit: Number(response.headers['x-apify-pagination-limit']), // API returns 999999999999 when no limit is used
381
382
  // TODO: Replace this once https://github.com/apify/apify-core/issues/3503 is solved
382
- desc: JSON.parse(response.headers['x-apify-pagination-desc'] ?? userProvidedDesc),
383
+ desc: typeof descHeader === 'string' ? JSON.parse(descHeader) : userProvidedDesc,
383
384
  };
384
385
  // The offset iterator paginates by the scanned number, so it travels with the page outside its public shape.
385
386
  return Object.defineProperty(page, SCANNED_COUNT, {
@@ -174,8 +174,14 @@ export declare class KeyValueStoreClient extends ResourceClient {
174
174
  */
175
175
  recordExists(key: string, options?: TimeoutOptions): Promise<boolean>;
176
176
  /**
177
+ * By default, the value is parsed by the record's content type: `application/json` is parsed into a value,
178
+ * `text/*` and `application/*xml` are decoded to a string, and anything else is returned as a Buffer (Node.js)
179
+ * or ArrayBuffer (browser). A record whose body does not parse as its content type claims, such as a non-JSON
180
+ * string stored under `application/json`, makes the call throw an {@link InvalidResponseBodyError}, which
181
+ * carries the raw body in `error.response`.
182
+ *
177
183
  * You can use the `buffer` option to get the value in a Buffer (Node.js)
178
- * or ArrayBuffer (browser) format. In Node.js (not in browser) you can also
184
+ * or ArrayBuffer (browser) format, which skips the parsing. In Node.js (not in browser) you can also
179
185
  * use the `stream` option to get a Readable stream.
180
186
  *
181
187
  * When the record does not exist, the function resolves to `undefined`. It does
@@ -198,6 +204,9 @@ export declare class KeyValueStoreClient extends ResourceClient {
198
204
  * The content type is automatically determined based on the value type, but can be
199
205
  * overridden using the `contentType` property.
200
206
  *
207
+ * A string is sent as it is, even under `application/json`, so a string stored under a JSON content type has
208
+ * to be valid JSON already, or {@link getRecord} fails to parse it back.
209
+ *
201
210
  * **Note about streams:** If the value is a stream object (has `.pipe` and `.on` methods),
202
211
  * the upload cannot be retried on failure or follow redirects. For reliable uploads,
203
212
  * buffer the entire stream into memory first.
@@ -328,7 +328,7 @@ export class KeyValueStoreClient extends ResourceClient {
328
328
  timeoutSecs: parsed.timeoutSecs ?? 'long',
329
329
  };
330
330
  if (parsed.buffer)
331
- requestOpts.forceBuffer = true;
331
+ requestOpts.responseType = 'buffer';
332
332
  if (parsed.stream)
333
333
  requestOpts.responseType = 'stream';
334
334
  try {
@@ -351,6 +351,9 @@ export class KeyValueStoreClient extends ResourceClient {
351
351
  * The content type is automatically determined based on the value type, but can be
352
352
  * overridden using the `contentType` property.
353
353
  *
354
+ * A string is sent as it is, even under `application/json`, so a string stored under a JSON content type has
355
+ * to be valid JSON already, or {@link getRecord} fails to parse it back.
356
+ *
354
357
  * **Note about streams:** If the value is a stream object (has `.pipe` and `.on` methods),
355
358
  * the upload cannot be retried on failure or follow redirects. For reliable uploads,
356
359
  * buffer the entire stream into memory first.
@@ -2,7 +2,7 @@ import type { RUN_GENERAL_ACCESS } from '@apify/consts';
2
2
  import { Log } from '@apify/log';
3
3
  import type { ApiClientOptionsWithOptionalResourcePath } from '../base/api_client.js';
4
4
  import { ResourceClient } from '../base/resource_client.js';
5
- import type { ApifyResponse } from '../http_client.js';
5
+ import type { ApifyResponse } from '../http_clients/index.js';
6
6
  import type { TimeoutOptions } from '../timeouts.js';
7
7
  import type { ActorInput, ActorRun } from './actor.js';
8
8
  import { DatasetClient } from './dataset.js';
@@ -169,8 +169,7 @@ export class RunClient extends ResourceClient {
169
169
  method: 'POST',
170
170
  data: input,
171
171
  params: this.buildParams(params),
172
- // Apify internal property. Tells the request serialization interceptor
173
- // to stringify functions to JSON, instead of omitting them.
172
+ // Actor input may carry page functions, which plain JSON serialization would drop.
174
173
  stringifyFunctions: true,
175
174
  timeoutSecs: parsed.timeoutSecs ?? 'medium',
176
175
  };
@@ -173,8 +173,7 @@ export class TaskClient extends ResourceClient {
173
173
  method: 'POST',
174
174
  data: input,
175
175
  params: this.buildParams(params),
176
- // Apify internal property. Tells the request serialization interceptor
177
- // to stringify functions to JSON, instead of omitting them.
176
+ // Actor input may carry page functions, which plain JSON serialization would drop.
178
177
  stringifyFunctions: true,
179
178
  headers: {
180
179
  'Content-Type': 'application/json',
package/dist/utils.d.ts CHANGED
@@ -3,7 +3,7 @@ import type { TypedArray } from 'type-fest';
3
3
  import { z } from 'zod';
4
4
  import type { ApifyApiError } from './apify_api_error.js';
5
5
  import { parseArgument } from '@apify/validations';
6
- import type { ApifyResponse } from './http_client.js';
6
+ import type { ApifyResponse } from './http_clients/index.js';
7
7
  import type { RequestQueueClientListRequestsOptions, RequestQueueClientListRequestsResult } from './resource_clients/request_queue.js';
8
8
  import type { WebhookUpdateData } from './resource_clients/webhook.js';
9
9
  /**
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "apify-client",
3
- "version": "3.0.0-beta.23",
3
+ "version": "3.0.0-beta.25",
4
4
  "description": "Apify API client for JavaScript",
5
5
  "engines": {
6
6
  "node": ">=22.0.0"
@@ -69,7 +69,6 @@
69
69
  "@apify/validations": "^1.1.0",
70
70
  "@crawlee/types": "^3.18.1",
71
71
  "ansi-colors": "^4.1.3",
72
- "async-retry": "^1.3.3",
73
72
  "axios": "^1.20.0",
74
73
  "content-type": "^1.0.5",
75
74
  "proxy-agent": "^8.0.2",
@@ -85,7 +84,6 @@
85
84
  "@rsbuild/plugin-node-polyfill": "^1.4.6",
86
85
  "@rspack/cli": "^2.2.2",
87
86
  "@rspack/core": "^2.2.2",
88
- "@types/async-retry": "^1.4.9",
89
87
  "@types/compression": "^1.8.1",
90
88
  "@types/content-type": "^1.1.9",
91
89
  "@types/express": "^5.0.6",
@@ -1,62 +0,0 @@
1
- import type http from 'node:http';
2
- import type https from 'node:https';
3
- import type { AxiosInstance, AxiosRequestConfig, AxiosResponse, InternalAxiosRequestConfig } from 'axios';
4
- import type { Log } from '@apify/log';
5
- import type { HttpCompressor } from './http_compressors/base.js';
6
- import type { RequestInterceptorFunction } from './interceptors.js';
7
- import type { Statistics } from './statistics.js';
8
- import type { Timeout, TimeoutTier } from './timeouts.js';
9
- export declare class HttpClient {
10
- #private;
11
- stats: Statistics;
12
- maxRetries: number;
13
- minDelayBetweenRetriesMillis: number;
14
- userProvidedRequestInterceptors: RequestInterceptorFunction[];
15
- /**
16
- * Compressor applied to request bodies that are worth compressing. The request interceptor captures it at
17
- * construction, so it is fixed for the lifetime of the client.
18
- */
19
- readonly httpCompressor: HttpCompressor;
20
- logger: Log;
21
- /** Duration of each timeout tier, in milliseconds. */
22
- timeoutMillis: Record<TimeoutTier, number>;
23
- /** Cap on the timeout of a single request attempt, in milliseconds. */
24
- timeoutMaxMillis: number;
25
- httpAgent?: http.Agent;
26
- httpsAgent?: https.Agent;
27
- axios: AxiosInstance;
28
- workflowKey?: string;
29
- constructor(options: HttpClientOptions);
30
- call<T = any>(config: ApifyRequestConfig): Promise<ApifyResponse<T>>;
31
- }
32
- export interface ApifyRequestConfig extends Omit<AxiosRequestConfig, 'timeout'> {
33
- stringifyFunctions?: boolean;
34
- forceBuffer?: boolean;
35
- doNotRetryTimeouts?: boolean;
36
- /**
37
- * Timeout of the request: a tier name, a number of seconds, or `'noTimeout'`. The client resolves it to
38
- * the axios `timeout` in milliseconds for each attempt, before axios runs its interceptors, so a request
39
- * interceptor already sees a number of milliseconds.
40
- * @default 'medium'
41
- */
42
- timeoutSecs?: Timeout;
43
- }
44
- export interface ApifyResponse<T = any> extends AxiosResponse<T> {
45
- config: ApifyRequestConfig & InternalAxiosRequestConfig;
46
- }
47
- export interface HttpClientOptions {
48
- apifyClientStats: Statistics;
49
- maxRetries: number;
50
- minDelayBetweenRetriesMillis: number;
51
- requestInterceptors: RequestInterceptorFunction[];
52
- httpCompressor: HttpCompressor;
53
- timeoutShortSecs: number;
54
- timeoutMediumSecs: number;
55
- timeoutLongSecs: number;
56
- timeoutMaxSecs: number;
57
- logger: Log;
58
- token?: string;
59
- workflowKey?: string;
60
- /** @internal */
61
- userAgentSuffix?: string | string[];
62
- }
@@ -1,266 +0,0 @@
1
- import retry from 'async-retry';
2
- import axios, { AxiosHeaders } from 'axios';
3
- import { APIFY_ENV_VARS } from '@apify/consts';
4
- import { concatStreamToBuffer } from '@apify/utilities';
5
- import { ApifyApiError } from './apify_api_error.js';
6
- import { createRequestInterceptors, InvalidResponseBodyError, responseInterceptors } from './interceptors.js';
7
- import { runtime } from '#runtime';
8
- import { asArray, cast, getEnv, isStream, version } from './utils.js';
9
- const RATE_LIMIT_EXCEEDED_STATUS_CODE = 429;
10
- export class HttpClient {
11
- stats;
12
- maxRetries;
13
- minDelayBetweenRetriesMillis;
14
- userProvidedRequestInterceptors;
15
- /**
16
- * Compressor applied to request bodies that are worth compressing. The request interceptor captures it at
17
- * construction, so it is fixed for the lifetime of the client.
18
- */
19
- httpCompressor;
20
- logger;
21
- /** Duration of each timeout tier, in milliseconds. */
22
- timeoutMillis;
23
- /** Cap on the timeout of a single request attempt, in milliseconds. */
24
- timeoutMaxMillis;
25
- httpAgent;
26
- httpsAgent;
27
- axios;
28
- workflowKey;
29
- #httpAgentsPromise;
30
- constructor(options) {
31
- const { token } = options;
32
- this.stats = options.apifyClientStats;
33
- this.maxRetries = options.maxRetries;
34
- this.minDelayBetweenRetriesMillis = options.minDelayBetweenRetriesMillis;
35
- this.userProvidedRequestInterceptors = options.requestInterceptors;
36
- this.httpCompressor = options.httpCompressor;
37
- this.timeoutMillis = {
38
- short: options.timeoutShortSecs * 1000,
39
- medium: options.timeoutMediumSecs * 1000,
40
- long: options.timeoutLongSecs * 1000,
41
- };
42
- this.timeoutMaxMillis = options.timeoutMaxSecs * 1000;
43
- this.logger = options.logger;
44
- this.workflowKey = options.workflowKey || getEnv(APIFY_ENV_VARS.WORKFLOW_KEY);
45
- this.axios = axios.create({
46
- // Disable axios's built-in proxy handling since we're using custom agents
47
- proxy: false,
48
- paramsSerializer: (params) => {
49
- const formattedParams = Object.entries(params)
50
- .filter(([, value]) => value !== undefined)
51
- .map(([key, value]) => {
52
- if (value instanceof Date) {
53
- return [key, value.toISOString()];
54
- }
55
- const updatedValue = typeof value === 'boolean' ? Number(value) : value;
56
- return [key, String(updatedValue)];
57
- });
58
- return new URLSearchParams(formattedParams).toString();
59
- },
60
- validateStatus: null,
61
- // Interceptors serialize requests and parse responses instead. Empty arrays rather than `undefined`,
62
- // which axios fills in with its default transforms.
63
- transformRequest: [],
64
- transformResponse: [],
65
- responseType: 'arraybuffer',
66
- // Every request sets its own timeout in `createRequestHandler`, so the default only backs a raw
67
- // `axios.request()` call.
68
- timeout: this.timeoutMaxMillis,
69
- // maxBodyLength needs to be Infinity, because -1 falls back to a 10 MB default
70
- // from an axios subdependency - 'follow-redirects'
71
- maxBodyLength: Infinity,
72
- // maxContentLength must be -1, because Infinity will cause axios to run super slow
73
- // thanks to a bug that's now fixed, but not released yet https://github.com/axios/axios/pull/3738
74
- maxContentLength: -1,
75
- });
76
- // Clean all default headers because they only make a mess and their merging is difficult to understand and buggy.
77
- this.axios.defaults.headers = new AxiosHeaders();
78
- // If workflow key is available, pass it as a header
79
- if (this.workflowKey) {
80
- this.axios.defaults.headers['X-Apify-Workflow-Key'] = this.workflowKey;
81
- }
82
- // Attach Authorization header for all requests if token was provided
83
- if (token) {
84
- this.axios.defaults.headers.Authorization = `Bearer ${token}`;
85
- }
86
- // Browsers do not let a page set the header, so it is only sent where the runtime describes its platform.
87
- if (runtime.platform) {
88
- const isAtHome = !!getEnv(APIFY_ENV_VARS.IS_AT_HOME);
89
- let userAgent = `ApifyClient/${version} (${runtime.platform}); isAtHome/${isAtHome}`;
90
- if (options.userAgentSuffix) {
91
- userAgent += `; ${asArray(options.userAgentSuffix).join('; ')}`;
92
- }
93
- this.axios.defaults.headers['User-Agent'] = userAgent;
94
- }
95
- createRequestInterceptors(this.httpCompressor).forEach((i) => this.axios.interceptors.request.use(i));
96
- this.userProvidedRequestInterceptors.forEach((i) => this.axios.interceptors.request.use(i));
97
- responseInterceptors.forEach((i) => this.axios.interceptors.response.use(i));
98
- }
99
- async #ensureHttpAgents() {
100
- this.#httpAgentsPromise ??= this.#initHttpAgents();
101
- return this.#httpAgentsPromise;
102
- }
103
- async #initHttpAgents() {
104
- const agents = await runtime.createHttpAgents({ timeoutMillis: this.timeoutMaxMillis });
105
- if (!agents)
106
- return;
107
- this.httpAgent = agents.httpAgent;
108
- this.httpsAgent = agents.httpsAgent;
109
- this.axios.defaults.httpAgent = this.httpAgent;
110
- this.axios.defaults.httpsAgent = this.httpsAgent;
111
- }
112
- async call(config) {
113
- await this.#ensureHttpAgents();
114
- this.stats.calls++;
115
- const makeRequest = this.#createRequestHandler(config);
116
- return retry(makeRequest, {
117
- retries: this.maxRetries,
118
- minTimeout: this.minDelayBetweenRetriesMillis,
119
- onRetry: (error, attempt) => this.#onRequestRetry(error, attempt),
120
- });
121
- }
122
- #informAboutStreamNoRetry() {
123
- this.logger.warningOnce('Request body was a stream - retrying will not work, as part of it was already consumed.');
124
- this.logger.warningOnce('If you want Apify client to handle retries for you, collect the stream into a buffer before sending it.');
125
- }
126
- /**
127
- * Successful responses are returned, errors and unsuccessful
128
- * status codes are retried. See the following functions for the
129
- * retrying logic.
130
- */
131
- #createRequestHandler(config) {
132
- const { timeoutSecs = 'medium', ...axiosConfig } = config;
133
- const makeRequest = async (stopTrying, attempt) => {
134
- this.stats.requests++;
135
- let response;
136
- const requestIsStream = isStream(config.data);
137
- try {
138
- if (requestIsStream) {
139
- // Handling redirects is not possible without buffering - part of the stream has already been sent and can't be recovered
140
- // when server sends the redirect. Therefore we need to override this in Axios config to prevent it from buffering the body.
141
- // see also axios/axios#1045
142
- axiosConfig.maxRedirects = 0;
143
- }
144
- response = await this.axios.request({
145
- ...axiosConfig,
146
- timeout: this.#computeTimeoutMillis(timeoutSecs, attempt),
147
- });
148
- if (this.#isStatusOk(response.status))
149
- return response;
150
- // A failed request with `responseType: 'stream'` carries the API error body in the stream. Read
151
- // it so that `ApifyApiError` can parse it like any other error body. A body that cannot be read
152
- // leaves the error without a message, which beats losing the status code to a stream error.
153
- if (isStream(response.data)) {
154
- response.data = await concatStreamToBuffer(response.data).catch(() => undefined);
155
- }
156
- }
157
- catch (err) {
158
- return cast(this.#handleRequestError(err, config, stopTrying));
159
- }
160
- if (response.status === RATE_LIMIT_EXCEEDED_STATUS_CODE) {
161
- this.stats.addRateLimitError(attempt);
162
- }
163
- const apiError = ApifyApiError.fromResponse(response, attempt);
164
- if (this.#isStatusCodeRetryable(response.status)) {
165
- if (requestIsStream) {
166
- this.#informAboutStreamNoRetry();
167
- }
168
- else {
169
- // allow a retry
170
- throw apiError;
171
- }
172
- }
173
- stopTrying(apiError);
174
- return response;
175
- };
176
- return makeRequest;
177
- }
178
- #isStatusOk(statusCode) {
179
- return statusCode < 300;
180
- }
181
- /**
182
- * Resolves `timeoutSecs` to the number of milliseconds the given attempt gets. A tier name resolves to
183
- * its configured duration, a number is taken as seconds, and `'noTimeout'` becomes `0`, which axios reads
184
- * as no timeout. The result doubles with each attempt and is capped at `timeoutMaxMillis`. A requested
185
- * value above the cap is capped too, which warns once, since it does not take effect in full.
186
- */
187
- #computeTimeoutMillis(timeoutSecs, attempt) {
188
- if (timeoutSecs === 'noTimeout')
189
- return 0;
190
- const requestedMillis = typeof timeoutSecs === 'number' ? timeoutSecs * 1000 : this.timeoutMillis[timeoutSecs];
191
- if (requestedMillis > this.timeoutMaxMillis) {
192
- // `warningOnce` keys by message, so each requested value warns once.
193
- this.logger.warningOnce(`The requested timeout of ${requestedMillis / 1000}s exceeds timeoutMaxSecs ` +
194
- `(${this.timeoutMaxMillis / 1000}s) and is capped at it. ` +
195
- 'Raise timeoutMaxSecs on the client to allow longer request timeouts.');
196
- }
197
- return Math.min(requestedMillis * 2 ** (attempt - 1), this.timeoutMaxMillis);
198
- }
199
- /**
200
- * Handles all unexpected errors that can happen, but are not
201
- * Apify API typed errors. E.g. network errors, timeouts and so on.
202
- */
203
- #handleRequestError(err, config, stopTrying) {
204
- if (this.#isTimeoutError(err) && config.doNotRetryTimeouts) {
205
- return stopTrying(err);
206
- }
207
- if (this.#isRetryableError(err)) {
208
- if (isStream(config.data)) {
209
- this.#informAboutStreamNoRetry();
210
- }
211
- else {
212
- throw err;
213
- }
214
- }
215
- return stopTrying(err);
216
- }
217
- /**
218
- * Axios calls req.abort() on timeouts so timeout errors will
219
- * have a code ECONNABORTED.
220
- */
221
- #isTimeoutError(err) {
222
- return err.code === 'ECONNABORTED';
223
- }
224
- /**
225
- * We don't want to retry every exception thrown from Axios.
226
- * The common denominator for retryable errors are network issues.
227
- * @param {Error} err
228
- * @private
229
- */
230
- #isRetryableError(err) {
231
- return this.#isNetworkError(err) || this.#isResponseBodyInvalid(err);
232
- }
233
- /**
234
- * When a network connection to our API is interrupted in the middle of streaming
235
- * a response, the request often does not fail, but simply contains
236
- * an incomplete response. This can often be fixed by retrying.
237
- */
238
- #isResponseBodyInvalid(err) {
239
- return err instanceof InvalidResponseBodyError;
240
- }
241
- /**
242
- * When a network request is attempted by axios and fails,
243
- * it throws an AxiosError, which will have the request
244
- * and config (and other) properties.
245
- */
246
- #isNetworkError(err) {
247
- const hasRequest = err.request && typeof err.request === 'object';
248
- const hasConfig = err.config && typeof err.config === 'object';
249
- return hasRequest && hasConfig;
250
- }
251
- /**
252
- * We retry 429 (rate limit) and 500+.
253
- * For status codes 300-499 (except 429) we do not retry the request,
254
- * because it's probably caused by invalid url (redirect 3xx) or invalid user input (4xx).
255
- */
256
- #isStatusCodeRetryable(statusCode) {
257
- const isRateLimitError = statusCode === RATE_LIMIT_EXCEEDED_STATUS_CODE;
258
- const isInternalError = statusCode >= 500;
259
- return isRateLimitError || isInternalError;
260
- }
261
- #onRequestRetry(error, attempt) {
262
- if (attempt === Math.round(this.maxRetries / 2)) {
263
- this.logger.warning(`API request failed ${attempt} times. Max attempts: ${this.maxRetries + 1}.\nCause:${error.stack}`);
264
- }
265
- }
266
- }
@@ -1,26 +0,0 @@
1
- import type { AxiosInterceptorManager, AxiosResponse } from 'axios';
2
- import type { ApifyRequestConfig, ApifyResponse } from './http_client.js';
3
- import type { HttpCompressor } from './http_compressors/base.js';
4
- /**
5
- * This error exists for the quite common situation, where only a partial JSON response is received and
6
- * an attempt to parse the JSON throws an error. In most cases this can be resolved by retrying the
7
- * request. We do that by identifying this error in HttpClient.
8
- *
9
- * The properties mimic AxiosError for easier integration in HttpClient error handling.
10
- */
11
- export declare class InvalidResponseBodyError extends Error {
12
- code: string;
13
- response: AxiosResponse;
14
- cause: Error;
15
- constructor(response: AxiosResponse, cause: Error);
16
- }
17
- export type RequestInterceptorFunction = Parameters<AxiosInterceptorManager<ApifyRequestConfig>['use']>[0];
18
- export type ResponseInterceptorFunction = Parameters<AxiosInterceptorManager<ApifyResponse>['use']>[0];
19
- /**
20
- * The client's own request interceptors, in registration order. Axios runs request interceptors in the reverse
21
- * order of registration, so the body is serialized before it is compressed, and interceptors registered later,
22
- * such as the user-provided ones, run before both.
23
- * @internal
24
- */
25
- export declare function createRequestInterceptors(compressor: HttpCompressor): RequestInterceptorFunction[];
26
- export declare const responseInterceptors: ResponseInterceptorFunction[];
@@ -1,153 +0,0 @@
1
- import axios, { AxiosHeaders } from 'axios';
2
- import contentTypeParser from 'content-type';
3
- import { maybeParseBody } from './body_parser.js';
4
- import { runtime } from '#runtime';
5
- import { isCompressibleContentType, MIN_COMPRESS_BYTES, toBytes } from './utils.js';
6
- /**
7
- * This error exists for the quite common situation, where only a partial JSON response is received and
8
- * an attempt to parse the JSON throws an error. In most cases this can be resolved by retrying the
9
- * request. We do that by identifying this error in HttpClient.
10
- *
11
- * The properties mimic AxiosError for easier integration in HttpClient error handling.
12
- */
13
- export class InvalidResponseBodyError extends Error {
14
- code;
15
- response;
16
- constructor(response, cause) {
17
- super(`Response body could not be parsed.\nCause:${cause.message}`);
18
- this.name = this.constructor.name;
19
- this.code = 'invalid-response-body';
20
- this.response = response;
21
- this.cause = cause;
22
- }
23
- }
24
- /**
25
- * Reads a request header regardless of the casing it was set with, since HTTP header names are case-insensitive
26
- * while the config keeps whatever casing the caller used.
27
- */
28
- function getHeader(config, name) {
29
- const wanted = name.toLowerCase();
30
- const key = Object.keys(config.headers ?? {}).find((candidate) => candidate.toLowerCase() === wanted);
31
- const value = key === undefined ? undefined : config.headers?.[key];
32
- return typeof value === 'string' ? value : undefined;
33
- }
34
- /** Removes a request header regardless of the casing it was set with. */
35
- function deleteHeader(config, name) {
36
- const { headers } = config;
37
- if (!headers)
38
- return;
39
- const wanted = name.toLowerCase();
40
- for (const key of Object.keys(headers)) {
41
- if (key.toLowerCase() === wanted)
42
- delete headers[key];
43
- }
44
- }
45
- function serializeRequest(config) {
46
- // A string body with an explicit content type is already serialized and goes out as it is. The axios default
47
- // transform would otherwise parse a JSON one in full just to check that it is valid, which for a body assembled
48
- // from thousands of pre-serialized requests costs about as much as serializing them did.
49
- if (typeof config.data === 'string' && getHeader(config, 'content-type'))
50
- return config;
51
- const [defaultTransform] = axios.defaults.transformRequest;
52
- // The function not only serializes data, but it also adds correct headers.
53
- const data = defaultTransform(config.data, config.headers);
54
- // Actor inputs can include functions and we don't want to omit those,
55
- // because it's convenient for users. JSON.stringify removes them.
56
- // It's a bit inefficient that we serialize the JSON twice, but I feel
57
- // it's a small price to pay. The axios default transform does a lot
58
- // of body type checks and we would have to copy all of them to the resource clients.
59
- if (config.stringifyFunctions) {
60
- const contentTypeHeader = getHeader(config, 'content-type');
61
- try {
62
- const type = contentTypeHeader ? contentTypeParser.parse(contentTypeHeader).type : undefined;
63
- if (type === 'application/json' && typeof config.data === 'object') {
64
- config.data = stringifyWithFunctions(config.data);
65
- }
66
- else {
67
- config.data = data;
68
- }
69
- }
70
- catch {
71
- config.data = data;
72
- }
73
- }
74
- else {
75
- config.data = data;
76
- }
77
- return config;
78
- }
79
- function ensureHeadersPrototype(config) {
80
- if (config.headers && !(config.headers instanceof AxiosHeaders)) {
81
- Object.setPrototypeOf(config.headers, AxiosHeaders.prototype);
82
- }
83
- return config;
84
- }
85
- /**
86
- * JSON.stringify() that serializes functions to string instead
87
- * of replacing them with null or removing them.
88
- */
89
- function stringifyWithFunctions(obj) {
90
- return JSON.stringify(obj, (_key, value) => {
91
- return typeof value === 'function' ? value.toString() : value;
92
- });
93
- }
94
- /**
95
- * Compresses the request body with the client's compressor and labels it with the compressor's `Content-Encoding`.
96
- *
97
- * Runs after `serializeRequest`, so a JSON body is already a string here. A caller-set `Content-Encoding` is
98
- * forwarded verbatim, which is how a pre-encoded body is uploaded, and `Content-Encoding: identity` opts a single
99
- * request out of compression. The built-in compressors need `node:zlib`, so a body sent from a browser or an edge
100
- * runtime goes out as it is.
101
- *
102
- * Compressing changes the body length, so any `Content-Length` the caller set describes the wrong body and has to
103
- * go. Axios keeps a caller-set one over the size it computes, which would stall the request until it times out.
104
- */
105
- async function maybeCompressRequest(config, compressor) {
106
- if (!runtime.isNode)
107
- return config;
108
- const bytes = toBytes(config.data);
109
- if (!bytes || bytes.byteLength < MIN_COMPRESS_BYTES)
110
- return config;
111
- // A caller-supplied encoding means the body is already encoded and the header describes it, so leave both alone.
112
- if (getHeader(config, 'content-encoding'))
113
- return config;
114
- if (!isCompressibleContentType(getHeader(config, 'content-type')))
115
- return config;
116
- config.data = await compressor.compress(Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength));
117
- config.headers ??= {};
118
- config.headers['content-encoding'] = compressor.contentEncoding;
119
- deleteHeader(config, 'content-length');
120
- return config;
121
- }
122
- function parseResponseData(response) {
123
- if (!response.data || // Nothing to do here.
124
- response.config.responseType !== 'arraybuffer' || // We don't want to parse custom response types.
125
- response.config.forceBuffer // Apify custom property to prevent parsing of buffer.
126
- ) {
127
- return response;
128
- }
129
- // A `Buffer` from the Node.js adapter and an `ArrayBuffer` from the browser ones both carry `byteLength`.
130
- if (!response.data.byteLength) {
131
- // undefined is better than an empty buffer
132
- response.data = undefined;
133
- return response;
134
- }
135
- const contentTypeHeader = response.headers['content-type'];
136
- try {
137
- response.data = maybeParseBody(response.data, contentTypeHeader);
138
- }
139
- catch (err) {
140
- throw new InvalidResponseBodyError(response, err);
141
- }
142
- return response;
143
- }
144
- /**
145
- * The client's own request interceptors, in registration order. Axios runs request interceptors in the reverse
146
- * order of registration, so the body is serialized before it is compressed, and interceptors registered later,
147
- * such as the user-provided ones, run before both.
148
- * @internal
149
- */
150
- export function createRequestInterceptors(compressor) {
151
- return [async (config) => maybeCompressRequest(config, compressor), serializeRequest, ensureHeadersPrototype];
152
- }
153
- export const responseInterceptors = [parseResponseData];