apify-client 3.0.0-beta.9 → 3.0.0-rc.0

package/dist/base/resource_client.js

@@ -1,5 +1,5 @@
 import { ACT_JOB_TERMINAL_STATUSES } from '@apify/consts';
-import { catchNotFoundOrThrow, parseResponse } from '../utils.js';
+import { catchNotFoundForResourceOrThrow, catchNotFoundOrThrow, parseResponse, sleep } from '../utils.js';
 import { ApiClient } from './api_client.js';
 /**
  * We need to supply some number for the API,
@@ -7,65 +7,97 @@ import { ApiClient } from './api_client.js';
  * 999999 seconds is more than 10 days.
  */
 const MAX_WAIT_FOR_FINISH = 999999;
-export const SMALL_TIMEOUT_MILLIS = 5 * 1000; // For fast and common actions. Suitable for idempotent actions.
-export const MEDIUM_TIMEOUT_MILLIS = 30 * 1000; // For actions that may take longer.
-export const DEFAULT_TIMEOUT_MILLIS = 360 * 1000; // 6 minutes
+/** The API holds a `waitForFinish` response for at most a minute, however long the parameter asks for. */
+const MAX_WAIT_FOR_FINISH_HOLD_SECS = 60;
+/** How long `waitForFinish()` keeps polling a job that returns 404, to ride out replica lag, before it gives up. */
+const MAX_WAIT_WHEN_JOB_NOT_EXIST_MILLIS = 3000;
 /**
  * Resource client.
  * @private
  */
 export class ResourceClient extends ApiClient {
-    async _get(schema, options = {}, timeoutMillis) {
+    /**
+     * Picks the timeout of a request that asks the API to hold its response with `waitForFinish`. The request
+     * gets the hold the caller asked for plus the round trip its tier allows, so the client does not abort a
+     * request while the API is still holding it - which for `start()` and `build()` would retry a call that
+     * creates a resource. An explicit per-call `timeoutSecs` is used as given.
+     */
+    timeoutForWaitForFinish(timeoutSecs, tier, waitForFinishSecs) {
+        if (timeoutSecs !== undefined)
+            return timeoutSecs;
+        if (waitForFinishSecs === undefined)
+            return tier;
+        const holdSecs = Math.min(waitForFinishSecs, MAX_WAIT_FOR_FINISH_HOLD_SECS);
+        if (holdSecs <= 0)
+            return tier;
+        return holdSecs + this.httpClient.timeoutMillis[tier] / 1000;
+    }
+    /**
+     * A 404 resolves to `undefined` only when the client names its resource by ID. A chained client without one, such
+     * as `run.dataset()`, throws it instead (see `catchNotFoundForResourceOrThrow()`).
+     */
+    async getResource(schema, options, timeoutSecs, signal) {
         const requestOpts = {
-            url: this._url(),
+            url: this.buildUrl(),
             method: 'GET',
-            params: this._params(options),
-            timeout: timeoutMillis,
+            params: this.buildParams(options),
+            timeoutSecs,
+            signal,
         };
         try {
             const response = await this.httpClient.call(requestOpts);
             return parseResponse(response, schema);
         }
         catch (err) {
-            catchNotFoundOrThrow(err);
+            catchNotFoundForResourceOrThrow(err, this.id);
         }
         return undefined;
     }
-    async _update(schema, newFields, timeoutMillis) {
+    async updateResource(schema, newFields, timeoutSecs, signal) {
         const response = await this.httpClient.call({
-            url: this._url(),
+            url: this.buildUrl(),
             method: 'PUT',
-            params: this._params(),
+            params: this.buildParams(),
             data: newFields,
-            timeout: timeoutMillis,
+            timeoutSecs,
+            signal,
         });
         return parseResponse(response, schema);
     }
-    async _delete(timeoutMillis) {
+    /**
+     * A 404 is swallowed, keeping the DELETE idempotent, only when the client names its resource by ID. A chained client
+     * without one throws it instead (see `catchNotFoundForResourceOrThrow()`).
+     */
+    async deleteResource(timeoutSecs, signal) {
         try {
             await this.httpClient.call({
-                url: this._url(),
+                url: this.buildUrl(),
                 method: 'DELETE',
-                params: this._params(),
-                timeout: timeoutMillis,
+                params: this.buildParams(),
+                timeoutSecs,
+                signal,
             });
         }
         catch (err) {
-            catchNotFoundOrThrow(err);
+            catchNotFoundForResourceOrThrow(err, this.id);
         }
     }
     /**
      * This function is used in Build and Run endpoints so it's kept
      * here to stay DRY.
      */
-    async _waitForFinish(schema, options = {}) {
-        const { waitSecs = MAX_WAIT_FOR_FINISH } = options;
+    async waitForJobFinish(schema, options = {}) {
+        const { waitSecs = MAX_WAIT_FOR_FINISH, timeoutSecs = 'noTimeout', signal } = options;
         const waitMillis = waitSecs * 1000;
         let job;
+        let notFoundSince;
+        let notFoundError;
         const startedAt = Date.now();
         const shouldRepeat = () => {
-            const millisSinceStart = Date.now() - startedAt;
-            if (millisSinceStart >= waitMillis)
+            const now = Date.now();
+            if (now - startedAt >= waitMillis)
+                return false;
+            if (notFoundSince !== undefined && now - notFoundSince >= MAX_WAIT_WHEN_JOB_NOT_EXIST_MILLIS)
                 return false;
             const hasJobEnded = job && ACT_JOB_TERMINAL_STATUSES.includes(job.status);
             return !hasJobEnded;
@@ -75,30 +107,32 @@ export class ResourceClient extends ApiClient {
             const remainingWaitSeconds = Math.round((waitMillis - millisSinceStart) / 1000);
             const waitForFinish = Math.max(0, remainingWaitSeconds);
             const requestOpts = {
-                url: this._url(),
+                url: this.buildUrl(),
                 method: 'GET',
-                params: this._params({ waitForFinish }),
+                params: this.buildParams({ waitForFinish }),
+                timeoutSecs,
+                signal,
             };
             try {
                 const response = await this.httpClient.call(requestOpts);
                 job = parseResponse(response, schema);
+                notFoundSince = undefined;
             }
             catch (err) {
                 catchNotFoundOrThrow(err);
                 job = undefined;
+                notFoundSince ??= Date.now();
+                notFoundError = err;
             }
             // It might take some time for database replicas to get up-to-date,
             // so getRun() might return null. Wait a little bit and try it again.
-            if (!job)
-                await new Promise((resolve) => {
-                    setTimeout(resolve, 250);
-                });
+            if (!job) {
+                await sleep(250, signal);
+                signal?.throwIfAborted();
+            }
         } while (shouldRepeat());
-        if (!job) {
-            const constructorName = this.constructor.name;
-            const jobName = constructorName.match(/(\w+)Client/)[1].toLowerCase();
-            throw new Error(`Waiting for ${jobName} to finish failed. Cannot fetch actor ${jobName} details from the server.`);
-        }
+        if (!job)
+            throw notFoundError;
         return job;
     }
 }

package/dist/base/resource_collection_client.d.ts

@@ -1,4 +1,5 @@
 import type { z } from 'zod';
+import type { Timeout, TimeoutOptions } from '../timeouts.js';
 import type { PaginatedResponse, PaginationOptions } from '../utils.js';
 import { ApiClient } from './api_client.js';
 /**
@@ -9,11 +10,18 @@ export declare class ResourceCollectionClient extends ApiClient {
     /**
      * @private
      */
-    protected _list<T, R>(schema: z.ZodType, options?: T): Promise<R>;
+    protected listResources<T, R>(schema: z.ZodType, options: T | undefined, timeoutSecs: Timeout, signal?: AbortSignal): Promise<R>;
+    /**
+     * Lists an endpoint that returns every item in one response and ignores `offset` and `limit`. The result can be
+     * awaited as that response or iterated over its items, with a single request either way.
+     */
+    protected listUnpaginatedResources<Data, R extends PaginatedResponse<Data>>(schema: z.ZodType, timeoutSecs: Timeout, signal?: AbortSignal): AsyncIterable<Data> & Promise<R>;
     /**
      * Returns async iterator to iterate through all items and Promise that can be awaited to get first page of results.
+     * `defaultTimeoutSecs` applies to every page request unless `options.timeoutSecs` overrides it, and
+     * `options.signal` goes to every page request.
      */
-    protected _listPaginated<T extends PaginationOptions, Data, R extends PaginatedResponse<Data>>(schema: z.ZodType, options?: T): AsyncIterable<Data> & Promise<R>;
-    protected _create<D, R>(schema: z.ZodType, resource: D): Promise<R>;
-    protected _getOrCreate<D, R>(schema: z.ZodType, name?: string, resource?: D): Promise<R>;
+    protected listResourcesPaginated<T extends PaginationOptions & TimeoutOptions, Data, R extends PaginatedResponse<Data>>(schema: z.ZodType, options: T, defaultTimeoutSecs: Timeout): AsyncIterable<Data> & Promise<R>;
+    protected createResource<D, R>(schema: z.ZodType, resource: D, timeoutSecs: Timeout, signal?: AbortSignal): Promise<R>;
+    protected getOrCreateResource<D, R>(schema: z.ZodType, name: string | undefined, resource: D | undefined, timeoutSecs: Timeout, signal?: AbortSignal): Promise<R>;
 }

package/dist/base/resource_collection_client.js

@@ -8,35 +8,58 @@ export class ResourceCollectionClient extends ApiClient {
     /**
      * @private
      */
-    async _list(schema, options = {}) {
+    async listResources(schema, options, timeoutSecs, signal) {
         const response = await this.httpClient.call({
-            url: this._url(),
+            url: this.buildUrl(),
             method: 'GET',
-            params: this._params(options),
+            params: this.buildParams(options),
+            timeoutSecs,
+            signal,
         });
         return parseResponse(response, schema);
     }
+    /**
+     * Lists an endpoint that returns every item in one response and ignores `offset` and `limit`. The result can be
+     * awaited as that response or iterated over its items, with a single request either way.
+     */
+    listUnpaginatedResources(schema, timeoutSecs, signal) {
+        const listPromise = this.listResources(schema, undefined, timeoutSecs, signal);
+        return Object.defineProperty(listPromise, Symbol.asyncIterator, {
+            async *value() {
+                yield* (await listPromise).items;
+            },
+        });
+    }
     /**
      * Returns async iterator to iterate through all items and Promise that can be awaited to get first page of results.
+     * `defaultTimeoutSecs` applies to every page request unless `options.timeoutSecs` overrides it, and
+     * `options.signal` goes to every page request.
      */
-    _listPaginated(schema, options = {}) {
-        return this._listPaginatedFromCallback(async (listOptions) => this._list(schema, listOptions), options);
+    listResourcesPaginated(schema, options, defaultTimeoutSecs) {
+        // `timeoutSecs` and `signal` only apply to the page requests; they are not API parameters, so they must not
+        // reach the query string.
+        const { timeoutSecs = defaultTimeoutSecs, signal, ...listOptions } = options;
+        return this.listPaginatedFromCallback(async (pageOptions) => this.listResources(schema, pageOptions, timeoutSecs, signal), listOptions);
     }
-    async _create(schema, resource) {
+    async createResource(schema, resource, timeoutSecs, signal) {
         const response = await this.httpClient.call({
-            url: this._url(),
+            url: this.buildUrl(),
             method: 'POST',
-            params: this._params(),
+            params: this.buildParams(),
             data: resource,
+            timeoutSecs,
+            signal,
         });
         return parseResponse(response, schema);
     }
-    async _getOrCreate(schema, name, resource) {
+    async getOrCreateResource(schema, name, resource, timeoutSecs, signal) {
         const response = await this.httpClient.call({
-            url: this._url(),
+            url: this.buildUrl(),
             method: 'POST',
-            params: this._params({ name }),
+            params: this.buildParams({ name }),
             data: resource,
+            timeoutSecs,
+            signal,
         });
         return parseResponse(response, schema);
     }

package/dist/body_parser.d.ts

@@ -1,13 +1,12 @@
 import type { JsonArray, JsonObject } from 'type-fest';
 /**
- * Parses a Buffer or ArrayBuffer using the provided content type header.
+ * Parses a binary response body using the provided content type header.
  *
  * - application/json is returned as a parsed object.
  * - application/*xml and text/* are returned as strings.
  * - everything else is returned as original body.
  *
  * If the header includes a charset, the body will be stringified only
- * if the charset represents a known encoding to Node.js or Browser.
+ * if the charset is an encoding `TextDecoder` knows.
  */
-export declare function maybeParseBody(body: Buffer | ArrayBuffer, contentTypeHeader: string): string | Buffer | ArrayBuffer | JsonObject | JsonArray;
-export declare function isomorphicBufferToString(buffer: Buffer | ArrayBuffer, encoding: BufferEncoding): string;
+export declare function maybeParseBody(body: ArrayBuffer | ArrayBufferView, contentTypeHeader: string): string | ArrayBuffer | ArrayBufferView | JsonObject | JsonArray;

package/dist/body_parser.js

@@ -1,22 +1,21 @@
-import contentTypeParser from 'content-type';
-import { isNode } from './utils.js';
+import { parse as parseContentType } from 'content-type';
 const CONTENT_TYPE_JSON = 'application/json';
 const STRINGIFIABLE_CONTENT_TYPE_RXS = [new RegExp(`^${CONTENT_TYPE_JSON}`, 'i'), /^application\/.*xml$/i, /^text\//i];
 /**
- * Parses a Buffer or ArrayBuffer using the provided content type header.
+ * Parses a binary response body using the provided content type header.
  *
  * - application/json is returned as a parsed object.
  * - application/*xml and text/* are returned as strings.
  * - everything else is returned as original body.
  *
  * If the header includes a charset, the body will be stringified only
- * if the charset represents a known encoding to Node.js or Browser.
+ * if the charset is an encoding `TextDecoder` knows.
  */
 export function maybeParseBody(body, contentTypeHeader) {
     let contentType;
     let charset;
     try {
-        const result = contentTypeParser.parse(contentTypeHeader);
+        const result = parseContentType(contentTypeHeader);
         contentType = result.type;
         charset = result.parameters.charset;
     }
@@ -24,35 +23,29 @@ export function maybeParseBody(body, contentTypeHeader) {
         // can't parse, keep original body
         return body;
     }
-    // If we can't successfully parse it, we return
+    if (!isContentTypeStringifiable(contentType))
+        return body;
+    // If we can't successfully decode it, we return
     // the original buffer rather than a mangled string.
-    if (!areDataStringifiable(contentType, charset))
+    const decoder = createDecoder(charset);
+    if (!decoder)
         return body;
-    const dataString = isomorphicBufferToString(body, charset);
+    const dataString = decoder.decode(body);
     return contentType === CONTENT_TYPE_JSON ? JSON.parse(dataString) : dataString;
 }
-export function isomorphicBufferToString(buffer, encoding) {
-    if (buffer.constructor.name !== ArrayBuffer.name) {
-        return buffer.toString(encoding);
+function createDecoder(charset) {
+    try {
+        // No charset defaults to utf-8. A leading BOM is kept, so a BOM-prefixed CSV stored in a key-value store
+        // round-trips through `getRecord()` unchanged.
+        return new TextDecoder(charset, { ignoreBOM: true });
+    }
+    catch {
+        // `TextDecoder` throws a `RangeError` for a label it does not know.
+        return undefined;
     }
-    // Browser decoding only works with UTF-8.
-    const utf8decoder = new TextDecoder();
-    return utf8decoder.decode(new Uint8Array(buffer));
-}
-function isCharsetStringifiable(charset) {
-    if (!charset)
-        return true; // hope that it's utf-8
-    if (isNode())
-        return Buffer.isEncoding(charset);
-    const normalizedCharset = charset.toLowerCase().replace('-', '');
-    // Browsers only support decoding utf-8 buffers.
-    return normalizedCharset === 'utf8';
 }
 function isContentTypeStringifiable(contentType) {
     if (!contentType)
         return false; // keep buffer
     return STRINGIFIABLE_CONTENT_TYPE_RXS.some((rx) => rx.test(contentType));
 }
-function areDataStringifiable(contentType, charset) {
-    return isContentTypeStringifiable(contentType) && isCharsetStringifiable(charset);
-}