apify-client 3.0.0-beta.2 → 3.0.0-beta.20

package/dist/base/api_client.js

@@ -1,4 +1,4 @@
-import { toPath, toPathSegment } from '../utils.js';
+import { SCANNED_COUNT, toPath, toPathSegment } from '../utils.js';
 /** @private */
 export class ApiClient {
     id;
@@ -16,7 +16,7 @@ export class ApiClient {
     constructor(options) {
         const { baseUrl, publicBaseUrl, apifyClient, httpClient, resourcePath, id, params = {} } = options;
         this.id = id;
-        this.safeId = id && this._toSafeId(id);
+        this.safeId = id && this.toSafeId(id);
         this.baseUrl = baseUrl;
         this.publicBaseUrl = publicBaseUrl;
         this.resourcePath = resourcePath;
@@ -25,35 +25,35 @@ export class ApiClient {
         this.httpClient = httpClient;
         this.params = params;
     }
-    _subResourceOptions(moreOptions) {
+    subResourceOptions(moreOptions) {
         const baseOptions = {
-            baseUrl: this._url(),
+            baseUrl: this.buildUrl(),
             publicBaseUrl: this.publicBaseUrl,
             apifyClient: this.apifyClient,
             httpClient: this.httpClient,
-            params: this._params(),
+            params: this.buildParams(),
         };
         return { ...baseOptions, ...moreOptions };
     }
-    _url(path) {
+    buildUrl(path) {
         return path ? `${this.url}/${toPath(path)}` : this.url;
     }
-    _publicUrl(path) {
+    buildPublicUrl(path) {
         const url = this.id
             ? `${this.publicBaseUrl}/${this.resourcePath}/${toPathSegment(this.safeId)}`
             : `${this.publicBaseUrl}/${this.resourcePath}`;
         return path ? `${url}/${toPath(path)}` : url;
     }
-    _params(endpointParams) {
+    buildParams(endpointParams) {
         return { ...this.params, ...endpointParams };
     }
-    _toSafeId(id) {
+    toSafeId(id) {
         return id.replaceAll('/', '~');
     }
     /**
      * Returns async iterator to iterate through all items and Promise that can be awaited to get first page of results.
      */
-    _listPaginatedFromCallback(getPaginatedList, options = {}) {
+    listPaginatedFromCallback(getPaginatedList, options = {}) {
         const minForLimitParam = (a, b) => {
             // API treats 0 as undefined for limit parameter
             if (a === 0)
@@ -67,20 +67,27 @@ export class ApiClient {
             return Math.min(a, b);
         };
         // `chunkSize` only sizes this loop's requests; it is not an API parameter, so it must not reach
-        // `_params()` and the query string.
+        // `buildParams()` and the query string. The same goes for `timeoutSecs`, which callers take out before
+        // calling this, since it also picks the timeout of every page request.
         const { chunkSize, ...listOptions } = options;
         const paginatedListPromise = getPaginatedList({
             ...listOptions,
             limit: minForLimitParam(options.limit, chunkSize),
         });
+        // A page can return more or fewer items than the rows it scanned (see `SCANNED_COUNT`). The next offset and
+        // the stop condition follow the scanned number alone: advancing by `items.length` would re-scan rows after a
+        // filter dropped some and skip rows after `unwind` multiplied them, and stopping at an empty page would end
+        // the iteration in front of rows a filter hid.
+        const scannedRows = (page) => page[SCANNED_COUNT] ?? page.items.length;
         async function* asyncGenerator() {
             let currentPage = await paginatedListPromise;
             yield* currentPage.items;
             const offset = options.offset ?? 0;
             const limit = Math.min(options.limit || currentPage.total, currentPage.total);
-            let currentOffset = offset + currentPage.items.length;
-            let remainingItems = Math.min(currentPage.total - offset, limit) - currentPage.items.length;
-            while (currentPage.items.length > 0 && // Continue only if at least some items were returned in the last page.
+            let pageScanned = scannedRows(currentPage);
+            let currentOffset = offset + pageScanned;
+            let remainingItems = Math.min(currentPage.total - offset, limit) - pageScanned;
+            while (pageScanned > 0 && // Continue only if the last page scanned some rows.
                 remainingItems > 0) {
                 const newOptions = {
                     ...listOptions,
@@ -89,8 +96,9 @@ export class ApiClient {
                 };
                 currentPage = await getPaginatedList(newOptions);
                 yield* currentPage.items;
-                currentOffset += currentPage.items.length;
-                remainingItems -= currentPage.items.length;
+                pageScanned = scannedRows(currentPage);
+                currentOffset += pageScanned;
+                remainingItems -= pageScanned;
             }
         }
         return Object.defineProperty(paginatedListPromise, Symbol.asyncIterator, {

package/dist/base/resource_client.d.ts

@@ -1,24 +1,38 @@
 import type { ACT_JOB_STATUSES } from '@apify/consts';
+import type { z } from 'zod';
+import type { Timeout, TimeoutOptions, TimeoutTier } from '../timeouts.js';
 import { ApiClient } from './api_client.js';
-export declare const SMALL_TIMEOUT_MILLIS: number;
-export declare const MEDIUM_TIMEOUT_MILLIS: number;
-export declare const DEFAULT_TIMEOUT_MILLIS: number;
 /**
  * Resource client.
  * @private
  */
 export declare class ResourceClient extends ApiClient {
-    protected _get<T, R>(options?: T, timeoutMillis?: number): Promise<R | undefined>;
-    protected _update<T, R>(newFields: T, timeoutMillis?: number): Promise<R>;
-    protected _delete(timeoutMillis?: number): Promise<void>;
+    /**
+     * Picks the timeout of a request that asks the API to hold its response with `waitForFinish`. The request
+     * gets the hold the caller asked for plus the round trip its tier allows, so the client does not abort a
+     * request while the API is still holding it - which for `start()` and `build()` would retry a call that
+     * creates a resource. An explicit per-call `timeoutSecs` is used as given.
+     */
+    protected timeoutForWaitForFinish(timeoutSecs: Timeout | undefined, tier: TimeoutTier, waitForFinishSecs: number | undefined): Timeout;
+    /**
+     * A 404 resolves to `undefined` only when the client names its resource by ID. A chained client without one, such
+     * as `run.dataset()`, throws it instead (see `catchNotFoundForResourceOrThrow()`).
+     */
+    protected getResource<T, R>(schema: z.ZodType, options: T, timeoutSecs: Timeout): Promise<R | undefined>;
+    protected updateResource<T, R>(schema: z.ZodType, newFields: T, timeoutSecs: Timeout): Promise<R>;
+    /**
+     * A 404 is swallowed, keeping the DELETE idempotent, only when the client names its resource by ID. A chained client
+     * without one throws it instead (see `catchNotFoundForResourceOrThrow()`).
+     */
+    protected deleteResource(timeoutSecs: Timeout): Promise<void>;
     /**
      * This function is used in Build and Run endpoints so it's kept
      * here to stay DRY.
      */
-    protected _waitForFinish<R extends {
+    protected waitForJobFinish<R extends {
         status: (typeof ACT_JOB_STATUSES)[keyof typeof ACT_JOB_STATUSES];
-    }>(options?: WaitForFinishOptions): Promise<R>;
+    }>(schema: z.ZodType, options?: WaitForFinishOptions): Promise<R>;
 }
-export interface WaitForFinishOptions {
+export interface WaitForFinishOptions extends TimeoutOptions {
     waitSecs?: number;
 }

package/dist/base/resource_client.js

@@ -1,5 +1,5 @@
 import { ACT_JOB_TERMINAL_STATUSES } from '@apify/consts';
-import { catchNotFoundOrThrow, parseDateFields, pluckData } from '../utils.js';
+import { catchNotFoundForResourceOrThrow, catchNotFoundOrThrow, parseResponse } from '../utils.js';
 import { ApiClient } from './api_client.js';
 /**
  * We need to supply some number for the API,
@@ -7,59 +7,82 @@ import { ApiClient } from './api_client.js';
  * 999999 seconds is more than 10 days.
  */
 const MAX_WAIT_FOR_FINISH = 999999;
-export const SMALL_TIMEOUT_MILLIS = 5 * 1000; // For fast and common actions. Suitable for idempotent actions.
-export const MEDIUM_TIMEOUT_MILLIS = 30 * 1000; // For actions that may take longer.
-export const DEFAULT_TIMEOUT_MILLIS = 360 * 1000; // 6 minutes
+/** The API holds a `waitForFinish` response for at most a minute, however long the parameter asks for. */
+const MAX_WAIT_FOR_FINISH_HOLD_SECS = 60;
 /**
  * Resource client.
  * @private
  */
 export class ResourceClient extends ApiClient {
-    async _get(options = {}, timeoutMillis) {
+    /**
+     * Picks the timeout of a request that asks the API to hold its response with `waitForFinish`. The request
+     * gets the hold the caller asked for plus the round trip its tier allows, so the client does not abort a
+     * request while the API is still holding it - which for `start()` and `build()` would retry a call that
+     * creates a resource. An explicit per-call `timeoutSecs` is used as given.
+     */
+    timeoutForWaitForFinish(timeoutSecs, tier, waitForFinishSecs) {
+        if (timeoutSecs !== undefined)
+            return timeoutSecs;
+        if (waitForFinishSecs === undefined)
+            return tier;
+        const holdSecs = Math.min(waitForFinishSecs, MAX_WAIT_FOR_FINISH_HOLD_SECS);
+        if (holdSecs <= 0)
+            return tier;
+        return holdSecs + this.httpClient.timeoutMillis[tier] / 1000;
+    }
+    /**
+     * A 404 resolves to `undefined` only when the client names its resource by ID. A chained client without one, such
+     * as `run.dataset()`, throws it instead (see `catchNotFoundForResourceOrThrow()`).
+     */
+    async getResource(schema, options, timeoutSecs) {
         const requestOpts = {
-            url: this._url(),
+            url: this.buildUrl(),
             method: 'GET',
-            params: this._params(options),
-            timeout: timeoutMillis,
+            params: this.buildParams(options),
+            timeoutSecs,
         };
         try {
             const response = await this.httpClient.call(requestOpts);
-            return parseDateFields(pluckData(response.data));
+            return parseResponse(response, schema);
         }
         catch (err) {
-            catchNotFoundOrThrow(err);
+            catchNotFoundForResourceOrThrow(err, this.id);
         }
         return undefined;
     }
-    async _update(newFields, timeoutMillis) {
+    async updateResource(schema, newFields, timeoutSecs) {
         const response = await this.httpClient.call({
-            url: this._url(),
+            url: this.buildUrl(),
             method: 'PUT',
-            params: this._params(),
+            params: this.buildParams(),
             data: newFields,
-            timeout: timeoutMillis,
+            timeoutSecs,
         });
-        return parseDateFields(pluckData(response.data));
+        return parseResponse(response, schema);
     }
-    async _delete(timeoutMillis) {
+    /**
+     * A 404 is swallowed, keeping the DELETE idempotent, only when the client names its resource by ID. A chained client
+     * without one throws it instead (see `catchNotFoundForResourceOrThrow()`).
+     */
+    async deleteResource(timeoutSecs) {
         try {
             await this.httpClient.call({
-                url: this._url(),
+                url: this.buildUrl(),
                 method: 'DELETE',
-                params: this._params(),
-                timeout: timeoutMillis,
+                params: this.buildParams(),
+                timeoutSecs,
             });
         }
         catch (err) {
-            catchNotFoundOrThrow(err);
+            catchNotFoundForResourceOrThrow(err, this.id);
         }
     }
     /**
      * This function is used in Build and Run endpoints so it's kept
      * here to stay DRY.
      */
-    async _waitForFinish(options = {}) {
-        const { waitSecs = MAX_WAIT_FOR_FINISH } = options;
+    async waitForJobFinish(schema, options = {}) {
+        const { waitSecs = MAX_WAIT_FOR_FINISH, timeoutSecs = 'noTimeout' } = options;
         const waitMillis = waitSecs * 1000;
         let job;
         const startedAt = Date.now();
@@ -75,13 +98,14 @@ export class ResourceClient extends ApiClient {
             const remainingWaitSeconds = Math.round((waitMillis - millisSinceStart) / 1000);
             const waitForFinish = Math.max(0, remainingWaitSeconds);
             const requestOpts = {
-                url: this._url(),
+                url: this.buildUrl(),
                 method: 'GET',
-                params: this._params({ waitForFinish }),
+                params: this.buildParams({ waitForFinish }),
+                timeoutSecs,
             };
             try {
                 const response = await this.httpClient.call(requestOpts);
-                job = parseDateFields(pluckData(response.data));
+                job = parseResponse(response, schema);
             }
             catch (err) {
                 catchNotFoundOrThrow(err);

package/dist/base/resource_collection_client.d.ts

@@ -1,3 +1,5 @@
+import type { z } from 'zod';
+import type { Timeout, TimeoutOptions } from '../timeouts.js';
 import type { PaginatedResponse, PaginationOptions } from '../utils.js';
 import { ApiClient } from './api_client.js';
 /**
@@ -8,11 +10,12 @@ export declare class ResourceCollectionClient extends ApiClient {
     /**
      * @private
      */
-    protected _list<T, R>(options?: T): Promise<R>;
+    protected listResources<T, R>(schema: z.ZodType, options: T | undefined, timeoutSecs: Timeout): Promise<R>;
     /**
      * Returns async iterator to iterate through all items and Promise that can be awaited to get first page of results.
+     * `defaultTimeoutSecs` applies to every page request unless `options.timeoutSecs` overrides it.
      */
-    protected _listPaginated<T extends PaginationOptions, Data, R extends PaginatedResponse<Data>>(options?: T): AsyncIterable<Data> & Promise<R>;
-    protected _create<D, R>(resource: D): Promise<R>;
-    protected _getOrCreate<D, R>(name?: string, resource?: D): Promise<R>;
+    protected listResourcesPaginated<T extends PaginationOptions & TimeoutOptions, Data, R extends PaginatedResponse<Data>>(schema: z.ZodType, options: T, defaultTimeoutSecs: Timeout): AsyncIterable<Data> & Promise<R>;
+    protected createResource<D, R>(schema: z.ZodType, resource: D, timeoutSecs: Timeout): Promise<R>;
+    protected getOrCreateResource<D, R>(schema: z.ZodType, name: string | undefined, resource: D | undefined, timeoutSecs: Timeout): Promise<R>;
 }

package/dist/base/resource_collection_client.js

@@ -1,4 +1,4 @@
-import { parseDateFields, pluckData } from '../utils.js';
+import { parseResponse } from '../utils.js';
 import { ApiClient } from './api_client.js';
 /**
  * Resource collection client.
@@ -8,36 +8,42 @@ export class ResourceCollectionClient extends ApiClient {
     /**
      * @private
      */
-    async _list(options = {}) {
+    async listResources(schema, options, timeoutSecs) {
         const response = await this.httpClient.call({
-            url: this._url(),
+            url: this.buildUrl(),
             method: 'GET',
-            params: this._params(options),
+            params: this.buildParams(options),
+            timeoutSecs,
         });
-        return parseDateFields(pluckData(response.data));
+        return parseResponse(response, schema);
     }
     /**
      * Returns async iterator to iterate through all items and Promise that can be awaited to get first page of results.
+     * `defaultTimeoutSecs` applies to every page request unless `options.timeoutSecs` overrides it.
      */
-    _listPaginated(options = {}) {
-        return this._listPaginatedFromCallback((this._list.bind(this)), options);
+    listResourcesPaginated(schema, options, defaultTimeoutSecs) {
+        // `timeoutSecs` only times the page requests; it is not an API parameter, so it must not reach the query string.
+        const { timeoutSecs = defaultTimeoutSecs, ...listOptions } = options;
+        return this.listPaginatedFromCallback(async (pageOptions) => this.listResources(schema, pageOptions, timeoutSecs), listOptions);
     }
-    async _create(resource) {
+    async createResource(schema, resource, timeoutSecs) {
         const response = await this.httpClient.call({
-            url: this._url(),
+            url: this.buildUrl(),
             method: 'POST',
-            params: this._params(),
+            params: this.buildParams(),
             data: resource,
+            timeoutSecs,
         });
-        return parseDateFields(pluckData(response.data));
+        return parseResponse(response, schema);
     }
-    async _getOrCreate(name, resource) {
+    async getOrCreateResource(schema, name, resource, timeoutSecs) {
         const response = await this.httpClient.call({
-            url: this._url(),
+            url: this.buildUrl(),
             method: 'POST',
-            params: this._params({ name }),
+            params: this.buildParams({ name }),
             data: resource,
+            timeoutSecs,
         });
-        return parseDateFields(pluckData(response.data));
+        return parseResponse(response, schema);
     }
 }