apify-client 3.0.0-beta.2 → 3.0.0-beta.21

package/dist/base/api_client.js

@@ -1,4 +1,4 @@
-import { toPath, toPathSegment } from '../utils.js';
+import { SCANNED_COUNT, toPath, toPathSegment } from '../utils.js';
 /** @private */
 export class ApiClient {
     id;
@@ -16,7 +16,7 @@ export class ApiClient {
     constructor(options) {
         const { baseUrl, publicBaseUrl, apifyClient, httpClient, resourcePath, id, params = {} } = options;
         this.id = id;
-        this.safeId = id && this._toSafeId(id);
+        this.safeId = id && this.toSafeId(id);
         this.baseUrl = baseUrl;
         this.publicBaseUrl = publicBaseUrl;
         this.resourcePath = resourcePath;
@@ -25,35 +25,35 @@ export class ApiClient {
         this.httpClient = httpClient;
         this.params = params;
     }
-    _subResourceOptions(moreOptions) {
+    subResourceOptions(moreOptions) {
         const baseOptions = {
-            baseUrl: this._url(),
+            baseUrl: this.buildUrl(),
             publicBaseUrl: this.publicBaseUrl,
             apifyClient: this.apifyClient,
             httpClient: this.httpClient,
-            params: this._params(),
+            params: this.buildParams(),
         };
         return { ...baseOptions, ...moreOptions };
     }
-    _url(path) {
+    buildUrl(path) {
         return path ? `${this.url}/${toPath(path)}` : this.url;
     }
-    _publicUrl(path) {
+    buildPublicUrl(path) {
         const url = this.id
             ? `${this.publicBaseUrl}/${this.resourcePath}/${toPathSegment(this.safeId)}`
             : `${this.publicBaseUrl}/${this.resourcePath}`;
         return path ? `${url}/${toPath(path)}` : url;
     }
-    _params(endpointParams) {
+    buildParams(endpointParams) {
         return { ...this.params, ...endpointParams };
     }
-    _toSafeId(id) {
+    toSafeId(id) {
         return id.replaceAll('/', '~');
     }
     /**
      * Returns async iterator to iterate through all items and Promise that can be awaited to get first page of results.
      */
-    _listPaginatedFromCallback(getPaginatedList, options = {}) {
+    listPaginatedFromCallback(getPaginatedList, options = {}) {
         const minForLimitParam = (a, b) => {
             // API treats 0 as undefined for limit parameter
             if (a === 0)
@@ -67,20 +67,27 @@ export class ApiClient {
             return Math.min(a, b);
         };
         // `chunkSize` only sizes this loop's requests; it is not an API parameter, so it must not reach
-        // `_params()` and the query string.
+        // `buildParams()` and the query string. The same goes for `timeoutSecs`, which callers take out before
+        // calling this, since it also picks the timeout of every page request.
         const { chunkSize, ...listOptions } = options;
         const paginatedListPromise = getPaginatedList({
             ...listOptions,
             limit: minForLimitParam(options.limit, chunkSize),
         });
+        // A page can return more or fewer items than the rows it scanned (see `SCANNED_COUNT`). The next offset and
+        // the stop condition follow the scanned number alone: advancing by `items.length` would re-scan rows after a
+        // filter dropped some and skip rows after `unwind` multiplied them, and stopping at an empty page would end
+        // the iteration in front of rows a filter hid.
+        const scannedRows = (page) => page[SCANNED_COUNT] ?? page.items.length;
         async function* asyncGenerator() {
             let currentPage = await paginatedListPromise;
             yield* currentPage.items;
             const offset = options.offset ?? 0;
             const limit = Math.min(options.limit || currentPage.total, currentPage.total);
-            let currentOffset = offset + currentPage.items.length;
-            let remainingItems = Math.min(currentPage.total - offset, limit) - currentPage.items.length;
-            while (currentPage.items.length > 0 && // Continue only if at least some items were returned in the last page.
+            let pageScanned = scannedRows(currentPage);
+            let currentOffset = offset + pageScanned;
+            let remainingItems = Math.min(currentPage.total - offset, limit) - pageScanned;
+            while (pageScanned > 0 && // Continue only if the last page scanned some rows.
                 remainingItems > 0) {
                 const newOptions = {
                     ...listOptions,
@@ -89,8 +96,9 @@ export class ApiClient {
                 };
                 currentPage = await getPaginatedList(newOptions);
                 yield* currentPage.items;
-                currentOffset += currentPage.items.length;
-                remainingItems -= currentPage.items.length;
+                pageScanned = scannedRows(currentPage);
+                currentOffset += pageScanned;
+                remainingItems -= pageScanned;
             }
         }
         return Object.defineProperty(paginatedListPromise, Symbol.asyncIterator, {

package/dist/base/resource_client.d.ts

@@ -1,24 +1,38 @@
 import type { ACT_JOB_STATUSES } from '@apify/consts';
+import type { z } from 'zod';
+import type { Timeout, TimeoutOptions, TimeoutTier } from '../timeouts.js';
 import { ApiClient } from './api_client.js';
-export declare const SMALL_TIMEOUT_MILLIS: number;
-export declare const MEDIUM_TIMEOUT_MILLIS: number;
-export declare const DEFAULT_TIMEOUT_MILLIS: number;
 /**
  * Resource client.
  * @private
  */
 export declare class ResourceClient extends ApiClient {
-    protected _get<T, R>(options?: T, timeoutMillis?: number): Promise<R | undefined>;
-    protected _update<T, R>(newFields: T, timeoutMillis?: number): Promise<R>;
-    protected _delete(timeoutMillis?: number): Promise<void>;
+    /**
+     * Picks the timeout of a request that asks the API to hold its response with `waitForFinish`. The request
+     * gets the hold the caller asked for plus the round trip its tier allows, so the client does not abort a
+     * request while the API is still holding it - which for `start()` and `build()` would retry a call that
+     * creates a resource. An explicit per-call `timeoutSecs` is used as given.
+     */
+    protected timeoutForWaitForFinish(timeoutSecs: Timeout | undefined, tier: TimeoutTier, waitForFinishSecs: number | undefined): Timeout;
+    /**
+     * A 404 resolves to `undefined` only when the client names its resource by ID. A chained client without one, such
+     * as `run.dataset()`, throws it instead (see `catchNotFoundForResourceOrThrow()`).
+     */
+    protected getResource<T, R>(schema: z.ZodType, options: T, timeoutSecs: Timeout): Promise<R | undefined>;
+    protected updateResource<T, R>(schema: z.ZodType, newFields: T, timeoutSecs: Timeout): Promise<R>;
+    /**
+     * A 404 is swallowed, keeping the DELETE idempotent, only when the client names its resource by ID. A chained client
+     * without one throws it instead (see `catchNotFoundForResourceOrThrow()`).
+     */
+    protected deleteResource(timeoutSecs: Timeout): Promise<void>;
     /**
      * This function is used in Build and Run endpoints so it's kept
      * here to stay DRY.
      */
-    protected _waitForFinish<R extends {
+    protected waitForJobFinish<R extends {
         status: (typeof ACT_JOB_STATUSES)[keyof typeof ACT_JOB_STATUSES];
-    }>(options?: WaitForFinishOptions): Promise<R>;
+    }>(schema: z.ZodType, options?: WaitForFinishOptions): Promise<R>;
 }
-export interface WaitForFinishOptions {
+export interface WaitForFinishOptions extends TimeoutOptions {
     waitSecs?: number;
 }

package/dist/base/resource_client.js

@@ -1,5 +1,5 @@
 import { ACT_JOB_TERMINAL_STATUSES } from '@apify/consts';
-import { catchNotFoundOrThrow, parseDateFields, pluckData } from '../utils.js';
+import { catchNotFoundForResourceOrThrow, catchNotFoundOrThrow, parseResponse } from '../utils.js';
 import { ApiClient } from './api_client.js';
 /**
  * We need to supply some number for the API,
@@ -7,59 +7,82 @@ import { ApiClient } from './api_client.js';
  * 999999 seconds is more than 10 days.
  */
 const MAX_WAIT_FOR_FINISH = 999999;
-export const SMALL_TIMEOUT_MILLIS = 5 * 1000; // For fast and common actions. Suitable for idempotent actions.
-export const MEDIUM_TIMEOUT_MILLIS = 30 * 1000; // For actions that may take longer.
-export const DEFAULT_TIMEOUT_MILLIS = 360 * 1000; // 6 minutes
+/** The API holds a `waitForFinish` response for at most a minute, however long the parameter asks for. */
+const MAX_WAIT_FOR_FINISH_HOLD_SECS = 60;
 /**
  * Resource client.
  * @private
  */
 export class ResourceClient extends ApiClient {
-    async _get(options = {}, timeoutMillis) {
+    /**
+     * Picks the timeout of a request that asks the API to hold its response with `waitForFinish`. The request
+     * gets the hold the caller asked for plus the round trip its tier allows, so the client does not abort a
+     * request while the API is still holding it - which for `start()` and `build()` would retry a call that
+     * creates a resource. An explicit per-call `timeoutSecs` is used as given.
+     */
+    timeoutForWaitForFinish(timeoutSecs, tier, waitForFinishSecs) {
+        if (timeoutSecs !== undefined)
+            return timeoutSecs;
+        if (waitForFinishSecs === undefined)
+            return tier;
+        const holdSecs = Math.min(waitForFinishSecs, MAX_WAIT_FOR_FINISH_HOLD_SECS);
+        if (holdSecs <= 0)
+            return tier;
+        return holdSecs + this.httpClient.timeoutMillis[tier] / 1000;
+    }
+    /**
+     * A 404 resolves to `undefined` only when the client names its resource by ID. A chained client without one, such
+     * as `run.dataset()`, throws it instead (see `catchNotFoundForResourceOrThrow()`).
+     */
+    async getResource(schema, options, timeoutSecs) {
         const requestOpts = {
-            url: this._url(),
+            url: this.buildUrl(),
             method: 'GET',
-            params: this._params(options),
-            timeout: timeoutMillis,
+            params: this.buildParams(options),
+            timeoutSecs,
         };
         try {
             const response = await this.httpClient.call(requestOpts);
-            return parseDateFields(pluckData(response.data));
+            return parseResponse(response, schema);
         }
         catch (err) {
-            catchNotFoundOrThrow(err);
+            catchNotFoundForResourceOrThrow(err, this.id);
         }
         return undefined;
     }
-    async _update(newFields, timeoutMillis) {
+    async updateResource(schema, newFields, timeoutSecs) {
         const response = await this.httpClient.call({
-            url: this._url(),
+            url: this.buildUrl(),
             method: 'PUT',
-            params: this._params(),
+            params: this.buildParams(),
             data: newFields,
-            timeout: timeoutMillis,
+            timeoutSecs,
         });
-        return parseDateFields(pluckData(response.data));
+        return parseResponse(response, schema);
     }
-    async _delete(timeoutMillis) {
+    /**
+     * A 404 is swallowed, keeping the DELETE idempotent, only when the client names its resource by ID. A chained client
+     * without one throws it instead (see `catchNotFoundForResourceOrThrow()`).
+     */
+    async deleteResource(timeoutSecs) {
         try {
             await this.httpClient.call({
-                url: this._url(),
+                url: this.buildUrl(),
                 method: 'DELETE',
-                params: this._params(),
-                timeout: timeoutMillis,
+                params: this.buildParams(),
+                timeoutSecs,
             });
         }
         catch (err) {
-            catchNotFoundOrThrow(err);
+            catchNotFoundForResourceOrThrow(err, this.id);
         }
     }
     /**
      * This function is used in Build and Run endpoints so it's kept
      * here to stay DRY.
      */
-    async _waitForFinish(options = {}) {
-        const { waitSecs = MAX_WAIT_FOR_FINISH } = options;
+    async waitForJobFinish(schema, options = {}) {
+        const { waitSecs = MAX_WAIT_FOR_FINISH, timeoutSecs = 'noTimeout' } = options;
         const waitMillis = waitSecs * 1000;
         let job;
         const startedAt = Date.now();
@@ -75,13 +98,14 @@ export class ResourceClient extends ApiClient {
             const remainingWaitSeconds = Math.round((waitMillis - millisSinceStart) / 1000);
             const waitForFinish = Math.max(0, remainingWaitSeconds);
             const requestOpts = {
-                url: this._url(),
+                url: this.buildUrl(),
                 method: 'GET',
-                params: this._params({ waitForFinish }),
+                params: this.buildParams({ waitForFinish }),
+                timeoutSecs,
             };
             try {
                 const response = await this.httpClient.call(requestOpts);
-                job = parseDateFields(pluckData(response.data));
+                job = parseResponse(response, schema);
             }
             catch (err) {
                 catchNotFoundOrThrow(err);

package/dist/base/resource_collection_client.d.ts

@@ -1,3 +1,5 @@
+import type { z } from 'zod';
+import type { Timeout, TimeoutOptions } from '../timeouts.js';
 import type { PaginatedResponse, PaginationOptions } from '../utils.js';
 import { ApiClient } from './api_client.js';
 /**
@@ -8,11 +10,12 @@ export declare class ResourceCollectionClient extends ApiClient {
     /**
      * @private
      */
-    protected _list<T, R>(options?: T): Promise<R>;
+    protected listResources<T, R>(schema: z.ZodType, options: T | undefined, timeoutSecs: Timeout): Promise<R>;
     /**
      * Returns async iterator to iterate through all items and Promise that can be awaited to get first page of results.
+     * `defaultTimeoutSecs` applies to every page request unless `options.timeoutSecs` overrides it.
      */
-    protected _listPaginated<T extends PaginationOptions, Data, R extends PaginatedResponse<Data>>(options?: T): AsyncIterable<Data> & Promise<R>;
-    protected _create<D, R>(resource: D): Promise<R>;
-    protected _getOrCreate<D, R>(name?: string, resource?: D): Promise<R>;
+    protected listResourcesPaginated<T extends PaginationOptions & TimeoutOptions, Data, R extends PaginatedResponse<Data>>(schema: z.ZodType, options: T, defaultTimeoutSecs: Timeout): AsyncIterable<Data> & Promise<R>;
+    protected createResource<D, R>(schema: z.ZodType, resource: D, timeoutSecs: Timeout): Promise<R>;
+    protected getOrCreateResource<D, R>(schema: z.ZodType, name: string | undefined, resource: D | undefined, timeoutSecs: Timeout): Promise<R>;
 }

package/dist/base/resource_collection_client.js

@@ -1,4 +1,4 @@
-import { parseDateFields, pluckData } from '../utils.js';
+import { parseResponse } from '../utils.js';
 import { ApiClient } from './api_client.js';
 /**
  * Resource collection client.
@@ -8,36 +8,42 @@ export class ResourceCollectionClient extends ApiClient {
     /**
      * @private
      */
-    async _list(options = {}) {
+    async listResources(schema, options, timeoutSecs) {
         const response = await this.httpClient.call({
-            url: this._url(),
+            url: this.buildUrl(),
             method: 'GET',
-            params: this._params(options),
+            params: this.buildParams(options),
+            timeoutSecs,
         });
-        return parseDateFields(pluckData(response.data));
+        return parseResponse(response, schema);
     }
     /**
      * Returns async iterator to iterate through all items and Promise that can be awaited to get first page of results.
+     * `defaultTimeoutSecs` applies to every page request unless `options.timeoutSecs` overrides it.
      */
-    _listPaginated(options = {}) {
-        return this._listPaginatedFromCallback((this._list.bind(this)), options);
+    listResourcesPaginated(schema, options, defaultTimeoutSecs) {
+        // `timeoutSecs` only times the page requests; it is not an API parameter, so it must not reach the query string.
+        const { timeoutSecs = defaultTimeoutSecs, ...listOptions } = options;
+        return this.listPaginatedFromCallback(async (pageOptions) => this.listResources(schema, pageOptions, timeoutSecs), listOptions);
     }
-    async _create(resource) {
+    async createResource(schema, resource, timeoutSecs) {
         const response = await this.httpClient.call({
-            url: this._url(),
+            url: this.buildUrl(),
             method: 'POST',
-            params: this._params(),
+            params: this.buildParams(),
             data: resource,
+            timeoutSecs,
         });
-        return parseDateFields(pluckData(response.data));
+        return parseResponse(response, schema);
     }
-    async _getOrCreate(name, resource) {
+    async getOrCreateResource(schema, name, resource, timeoutSecs) {
         const response = await this.httpClient.call({
-            url: this._url(),
+            url: this.buildUrl(),
             method: 'POST',
-            params: this._params({ name }),
+            params: this.buildParams({ name }),
             data: resource,
+            timeoutSecs,
         });
-        return parseDateFields(pluckData(response.data));
+        return parseResponse(response, schema);
     }
 }

package/dist/body_parser.d.ts

@@ -1,13 +1,12 @@
 import type { JsonArray, JsonObject } from 'type-fest';
 /**
- * Parses a Buffer or ArrayBuffer using the provided content type header.
+ * Parses a binary response body using the provided content type header.
  *
  * - application/json is returned as a parsed object.
  * - application/*xml and text/* are returned as strings.
  * - everything else is returned as original body.
  *
  * If the header includes a charset, the body will be stringified only
- * if the charset represents a known encoding to Node.js or Browser.
+ * if the charset is an encoding `TextDecoder` knows.
  */
-export declare function maybeParseBody(body: Buffer | ArrayBuffer, contentTypeHeader: string): string | Buffer | ArrayBuffer | JsonObject | JsonArray;
-export declare function isomorphicBufferToString(buffer: Buffer | ArrayBuffer, encoding: BufferEncoding): string;
+export declare function maybeParseBody(body: ArrayBuffer | ArrayBufferView, contentTypeHeader: string): string | ArrayBuffer | ArrayBufferView | JsonObject | JsonArray;

package/dist/body_parser.js

@@ -1,16 +1,15 @@
 import contentTypeParser from 'content-type';
-import { isNode } from './utils.js';
 const CONTENT_TYPE_JSON = 'application/json';
 const STRINGIFIABLE_CONTENT_TYPE_RXS = [new RegExp(`^${CONTENT_TYPE_JSON}`, 'i'), /^application\/.*xml$/i, /^text\//i];
 /**
- * Parses a Buffer or ArrayBuffer using the provided content type header.
+ * Parses a binary response body using the provided content type header.
  *
  * - application/json is returned as a parsed object.
  * - application/*xml and text/* are returned as strings.
  * - everything else is returned as original body.
  *
  * If the header includes a charset, the body will be stringified only
- * if the charset represents a known encoding to Node.js or Browser.
+ * if the charset is an encoding `TextDecoder` knows.
  */
 export function maybeParseBody(body, contentTypeHeader) {
     let contentType;
@@ -24,35 +23,29 @@ export function maybeParseBody(body, contentTypeHeader) {
         // can't parse, keep original body
         return body;
     }
-    // If we can't successfully parse it, we return
+    if (!isContentTypeStringifiable(contentType))
+        return body;
+    // If we can't successfully decode it, we return
     // the original buffer rather than a mangled string.
-    if (!areDataStringifiable(contentType, charset))
+    const decoder = createDecoder(charset);
+    if (!decoder)
         return body;
-    const dataString = isomorphicBufferToString(body, charset);
+    const dataString = decoder.decode(body);
     return contentType === CONTENT_TYPE_JSON ? JSON.parse(dataString) : dataString;
 }
-export function isomorphicBufferToString(buffer, encoding) {
-    if (buffer.constructor.name !== ArrayBuffer.name) {
-        return buffer.toString(encoding);
+function createDecoder(charset) {
+    try {
+        // No charset defaults to utf-8. A leading BOM is kept, so a BOM-prefixed CSV stored in a key-value store
+        // round-trips through `getRecord()` unchanged.
+        return new TextDecoder(charset, { ignoreBOM: true });
+    }
+    catch {
+        // `TextDecoder` throws a `RangeError` for a label it does not know.
+        return undefined;
     }
-    // Browser decoding only works with UTF-8.
-    const utf8decoder = new TextDecoder();
-    return utf8decoder.decode(new Uint8Array(buffer));
-}
-function isCharsetStringifiable(charset) {
-    if (!charset)
-        return true; // hope that it's utf-8
-    if (isNode())
-        return Buffer.isEncoding(charset);
-    const normalizedCharset = charset.toLowerCase().replace('-', '');
-    // Browsers only support decoding utf-8 buffers.
-    return normalizedCharset === 'utf8';
 }
 function isContentTypeStringifiable(contentType) {
     if (!contentType)
         return false; // keep buffer
     return STRINGIFIABLE_CONTENT_TYPE_RXS.some((rx) => rx.test(contentType));
 }
-function areDataStringifiable(contentType, charset) {
-    return isContentTypeStringifiable(contentType) && isCharsetStringifiable(charset);
-}