apify-client 3.0.0-beta.9 → 3.0.0-beta.90

package/dist/resource_clients/dataset.js

@@ -1,8 +1,24 @@
 import { z } from 'zod';
 import { createStorageContentSignatureAsync } from '@apify/utilities';
-import { DEFAULT_TIMEOUT_MILLIS, MEDIUM_TIMEOUT_MILLIS, ResourceClient, SMALL_TIMEOUT_MILLIS, } from '../base/resource_client.js';
+import { ResourceClient } from '../base/resource_client.js';
 import * as schemas from '../schemas.js';
-import { anyObjectSchema, applyQueryParamsToUrl, cast, catchNotFoundOrThrow, isNonArrayObject, paginationOptionsShape, parseArgument, parseResponse, } from '../utils.js';
+import { timeoutOptionsSchema, timeoutOptionsShape } from '../timeouts.js';
+import { runtime } from '#runtime';
+import { anyObjectSchema, applyQueryParamsToUrl, cast, isNonArrayObject, paginationOptionsShape, parseArgument, parseResponse, } from '../utils.js';
+/**
+ * Supported formats for downloading dataset items.
+ */
+export var DownloadItemsFormat;
+(function (DownloadItemsFormat) {
+    DownloadItemsFormat["JSON"] = "json";
+    DownloadItemsFormat["JSONL"] = "jsonl";
+    DownloadItemsFormat["XML"] = "xml";
+    DownloadItemsFormat["HTML"] = "html";
+    DownloadItemsFormat["CSV"] = "csv";
+    DownloadItemsFormat["XLSX"] = "xlsx";
+    DownloadItemsFormat["RSS"] = "rss";
+})(DownloadItemsFormat || (DownloadItemsFormat = {}));
+const itemFormatSchema = z.enum(DownloadItemsFormat);
 // A predicate, not a `z.object()` arm, which would walk and copy every key of every pushed item.
 const itemSchema = z.custom(isNonArrayObject, 'Expected an object');
 const listItemsOptionsSchema = z.strictObject({
@@ -14,9 +30,10 @@ const listItemsOptionsSchema = z.strictObject({
     ...paginationOptionsShape,
     skipEmpty: z.boolean().optional(),
     skipHidden: z.boolean().optional(),
-    unwind: z.union([z.string(), z.array(z.string())]).optional(),
+    unwind: z.array(z.string()).optional(),
     view: z.string().optional(),
     signature: z.string().optional(),
+    ...timeoutOptionsShape,
 });
 // Spells out `limit` / `offset` instead of spreading `paginationOptionsShape`: a download is one
 // request, so `chunkSize` has nothing to size. The options type omits it to match.
@@ -34,28 +51,33 @@ const downloadItemsOptionsSchema = z.strictObject({
     skipEmpty: z.boolean().optional(),
     skipHeaderRow: z.boolean().optional(),
     skipHidden: z.boolean().optional(),
-    unwind: z.union([z.string(), z.array(z.string())]).optional(),
+    unwind: z.array(z.string()).optional(),
     view: z.string().optional(),
     xmlRoot: z.string().optional(),
     xmlRow: z.string().optional(),
     signature: z.string().optional(),
+    stream: z.boolean().optional(),
+    ...timeoutOptionsShape,
 });
-const pushItemsSchema = z.union([itemSchema, z.string(), z.array(z.union([itemSchema, z.string()]))]);
-// Every option becomes a query parameter of the generated URL, so `chunkSize` (client-side only) and
-// `signature` (which this method produces) are left out. The options type omits both to match.
+const pushItemsSchema = z.union([itemSchema, z.string(), z.array(itemSchema)]);
+// Apart from `timeoutSecs`, `signal` and `expiresInSecs`, every option becomes a query parameter of the generated
+// URL, so `chunkSize` (client-side only) and `signature` (which this method produces) are left out. The options type
+// omits both to match.
 const createItemsPublicUrlOptionsSchema = z.strictObject({
     clean: z.boolean().optional(),
     desc: z.boolean().optional(),
     flatten: z.array(z.string()).optional(),
     fields: z.array(z.string()).optional(),
+    format: itemFormatSchema.optional(),
     omit: z.array(z.string()).optional(),
     limit: z.number().min(0).optional(),
     offset: z.number().min(0).optional(),
     skipEmpty: z.boolean().optional(),
     skipHidden: z.boolean().optional(),
-    unwind: z.union([z.string(), z.array(z.string())]).optional(),
+    unwind: z.array(z.string()).optional(),
     view: z.string().optional(),
-    expiresInSecs: z.number().optional(),
+    expiresInSecs: z.number().positive().optional(),
+    ...timeoutOptionsShape,
 });
 /**
  * Client for managing a specific Dataset.
@@ -99,30 +121,39 @@ export class DatasetClient extends ResourceClient {
     /**
      * Gets the dataset object from the Apify API.
      *
+     * @param options - Request options
+     * @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
      * @returns The Dataset object, or `undefined` if it does not exist
      * @see https://docs.apify.com/api/v2/dataset-get
      */
-    async get() {
-        return this._get(schemas.Dataset(), {}, SMALL_TIMEOUT_MILLIS);
+    async get(options = {}) {
+        const { timeoutSecs = 'short', signal } = parseArgument(options, timeoutOptionsSchema, 'TimeoutOptions');
+        return this.getResource(schemas.DatasetResource(), {}, timeoutSecs, signal);
     }
     /**
      * Updates the dataset with specified fields.
      *
      * @param newFields - Fields to update in the dataset
+     * @param options - Request options
+     * @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
      * @returns The updated Dataset object
      * @see https://docs.apify.com/api/v2/dataset-put
      */
-    async update(newFields) {
+    async update(newFields, options = {}) {
         parseArgument(newFields, anyObjectSchema);
-        return this._update(schemas.Dataset(), newFields, SMALL_TIMEOUT_MILLIS);
+        const { timeoutSecs = 'short', signal } = parseArgument(options, timeoutOptionsSchema, 'TimeoutOptions');
+        return this.updateResource(schemas.DatasetResource(), newFields, timeoutSecs, signal);
     }
     /**
      * Deletes the dataset.
      *
+     * @param options - Request options
+     * @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
      * @see https://docs.apify.com/api/v2/dataset-delete
      */
-    async delete() {
-        return this._delete(SMALL_TIMEOUT_MILLIS);
+    async delete(options = {}) {
+        const { timeoutSecs = 'short', signal } = parseArgument(options, timeoutOptionsSchema, 'TimeoutOptions');
+        return this.deleteResource(timeoutSecs, signal);
     }
     /**
      * Lists items in the dataset.
@@ -142,8 +173,9 @@ export class DatasetClient extends ResourceClient {
      * @param options.skipEmpty - If `true`, skips empty items. Default is `false`.
      * @param options.skipHidden - If `true`, skips hidden fields (fields starting with `#`). Default is `false`.
      * @param options.flatten - Array of field names to flatten. Nested objects are converted to dot notation (e.g., `obj.field`).
-     * @param options.unwind - Field name or array of field names to unwind. Each array value creates a separate item.
+     * @param options.unwind - Array of field names to unwind. Each array value creates a separate item.
      * @param options.view - Name of a predefined view to use for field selection.
+     * @param options.timeoutSecs - Timeout for each API request. Default is `'long'`.
      * @returns A paginated list with `items`, `total` count, `offset`, `count`, and `limit`
      * @see https://docs.apify.com/api/v2/dataset-items-get
      *
@@ -169,83 +201,51 @@ export class DatasetClient extends ResourceClient {
      * ```
      */
     listItems(options = {}) {
-        const parsed = parseArgument(options, listItemsOptionsSchema, 'DatasetClientListItemOptions');
+        const { timeoutSecs = 'long', signal, ...listOptions } = parseArgument(options, listItemsOptionsSchema, 'DatasetClientListItemOptions');
         const fetchItems = async (datasetListOptions = {}) => {
             const response = await this.httpClient.call({
-                url: this._url('items'),
+                url: this.buildUrl('items'),
                 method: 'GET',
-                params: this._params(datasetListOptions),
-                timeout: DEFAULT_TIMEOUT_MILLIS,
+                params: this.buildParams(datasetListOptions),
+                timeoutSecs,
+                signal,
             });
-            return this._createPaginationList(response, datasetListOptions.desc ?? false);
+            return this.#createPaginationList(response, datasetListOptions.desc ?? false);
         };
-        return this._listPaginatedFromCallback(fetchItems, parsed);
+        return this.listPaginatedFromCallback(fetchItems, listOptions);
     }
-    /**
-     * Downloads dataset items in a specific format.
-     *
-     * Unlike {@link listItems} which returns a {@link PaginatedList} with an array of individual
-     * dataset items, this method returns the items serialized to the provided format
-     * (JSON, CSV, Excel, etc.) as a Buffer. Useful for exporting data for further processing.
-     *
-     * @param format - Output format: `'json'`, `'jsonl'`, `'csv'`, `'xlsx'`, `'xml'`, `'rss'`, or `'html'`
-     * @param options - Download and formatting options (extends all options from {@link listItems})
-     * @param options.attachment - If `true`, the response will have `Content-Disposition: attachment` header.
-     * @param options.bom - If `true`, adds UTF-8 BOM to the beginning of the file (useful for Excel compatibility).
-     * @param options.delimiter - CSV delimiter character. Default is `,` (comma).
-     * @param options.skipHeaderRow - If `true`, CSV export will not include the header row with field names.
-     * @param options.xmlRoot - Name of the root XML element. Default is `'items'`.
-     * @param options.xmlRow - Name of the XML element for each item. Default is `'item'`.
-     * @param options.fields - Array of field names to include in the export.
-     * @param options.omit - Array of field names to exclude from the export.
-     * @returns Buffer containing the serialized data in the specified format
-     * @see https://docs.apify.com/api/v2/dataset-items-get
-     *
-     * @example
-     * ```javascript
-     * // Download as CSV with BOM for Excel compatibility
-     * const csvBuffer = await client.dataset('my-dataset').downloadItems('csv', { bom: true });
-     * require('fs').writeFileSync('output.csv', csvBuffer);
-     *
-     * // Download as Excel with custom options
-     * const xlsxBuffer = await client.dataset('my-dataset').downloadItems('xlsx', {
-     *   fields: ['url', 'title', 'price'],
-     *   skipEmpty: true,
-     *   limit: 1000
-     * });
-     *
-     * // Download as XML with custom element names
-     * const xmlBuffer = await client.dataset('my-dataset').downloadItems('xml', {
-     *   xmlRoot: 'products',
-     *   xmlRow: 'product'
-     * });
-     * ```
-     */
     async downloadItems(format, options = {}) {
         parseArgument(format, itemFormatSchema);
-        const parsed = parseArgument(options, downloadItemsOptionsSchema, 'DatasetClientDownloadItemsOptions');
+        const { timeoutSecs = 'long', signal, stream, ...query } = parseArgument(options, downloadItemsOptionsSchema, 'DatasetClientDownloadItemsOptions');
+        if (stream && !runtime.isNode) {
+            throw new Error('The stream option can only be used in Node.js environment.');
+        }
         const { data } = await this.httpClient.call({
-            url: this._url('items'),
+            url: this.buildUrl('items'),
             method: 'GET',
-            params: this._params({
+            params: this.buildParams({
                 format,
-                ...parsed,
+                ...query,
             }),
-            forceBuffer: true,
-            timeout: DEFAULT_TIMEOUT_MILLIS,
+            responseType: stream ? 'stream' : 'buffer',
+            timeoutSecs,
+            signal,
         });
         return cast(data);
     }
     /**
      * Stores one or more items into the dataset.
      *
-     * Items can be objects, strings, or arrays thereof. Each item will be stored as a separate
-     * record in the dataset. Objects are automatically serialized to JSON. If you provide an array,
-     * all items will be stored in order. This method is idempotent - calling it multiple times
-     * with the same data will not create duplicates, but will append items each time.
+     * Each item will be stored as a separate record in the dataset. Objects are automatically
+     * serialized to JSON. If you provide an array, all items will be stored in order. This method
+     * is idempotent - calling it multiple times with the same data will not create duplicates, but
+     * will append items each time.
      *
-     * @param items - A single item (object or string) or an array of items to store.
-     *                Objects are automatically stringified to JSON. Strings are stored as-is.
+     * @param items - A single item, an array of items, or a string that is the JSON serialization of
+     *                either - the API only accepts an object or an array of objects, so a plain string
+     *                is not a valid item on its own.
+     * @param options - Request options
+     * @param options.timeoutSecs - Timeout for the API request. Default is `'medium'`.
      * @see https://docs.apify.com/api/v2/dataset-items-post
      *
      * @example
@@ -263,23 +263,22 @@ export class DatasetClient extends ResourceClient {
      *   { url: 'https://test.com', title: 'Test' },
      *   { url: 'https://demo.com', title: 'Demo' }
      * ]);
-     *
-     * // Store string items
-     * await client.dataset('my-dataset').pushItems(['item1', 'item2', 'item3']);
      * ```
      */
-    async pushItems(items) {
+    async pushItems(items, options = {}) {
         parseArgument(items, pushItemsSchema);
+        const { timeoutSecs = 'medium', signal } = parseArgument(options, timeoutOptionsSchema, 'TimeoutOptions');
         await this.httpClient.call({
-            url: this._url('items'),
+            url: this.buildUrl('items'),
             method: 'POST',
             headers: {
                 'content-type': 'application/json; charset=utf-8',
             },
             data: items,
-            params: this._params(),
-            doNotRetryTimeouts: true, // see timeout handling in http-client
-            timeout: MEDIUM_TIMEOUT_MILLIS,
+            params: this.buildParams(),
+            doNotRetryTimeouts: true,
+            timeoutSecs,
+            signal,
         });
     }
     /**
@@ -288,25 +287,22 @@ export class DatasetClient extends ResourceClient {
      * Returns statistics for each field in the dataset, including information about
      * data types, null counts, and value ranges.
      *
-     * @returns Dataset statistics, or `undefined` if not available
+     * @param options - Request options
+     * @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
+     * @returns Dataset statistics
      * @see https://docs.apify.com/api/v2/dataset-statistics-get
      * @since Added in 2.11.2
      */
-    async getStatistics() {
-        const requestOpts = {
-            url: this._url('statistics'),
+    async getStatistics(options = {}) {
+        const { timeoutSecs = 'short', signal } = parseArgument(options, timeoutOptionsSchema, 'TimeoutOptions');
+        const response = await this.httpClient.call({
+            url: this.buildUrl('statistics'),
             method: 'GET',
-            params: this._params(),
-            timeout: SMALL_TIMEOUT_MILLIS,
-        };
-        try {
-            const response = await this.httpClient.call(requestOpts);
-            return parseResponse(response, schemas.DatasetStatistics());
-        }
-        catch (err) {
-            catchNotFoundOrThrow(err);
-        }
-        return undefined;
+            params: this.buildParams(),
+            timeoutSecs,
+            signal,
+        });
+        return parseResponse(response, schemas.DatasetStatistics());
     }
     /**
      * Generates a public URL for accessing dataset items.
@@ -316,10 +312,13 @@ export class DatasetClient extends ResourceClient {
      * This is useful for sharing dataset results with external services or users.
      *
      * @param options - URL generation options (extends all options from {@link listItems})
-     * @param options.expiresInSecs - Number of seconds until the signed URL expires. If omitted, the URL never expires.
+     * @param options.expiresInSecs - Number of seconds until the signed URL expires. Must be positive. If omitted, the URL
+     * never expires.
      * @param options.fields - Array of field names to include in the response.
+     * @param options.format - Format of the items served by the URL. Defaults to `json`.
      * @param options.limit - Maximum number of items to return.
      * @param options.offset - Number of items to skip.
+     * @param options.timeoutSecs - Timeout for the API request that fetches the dataset. Default is `'long'`.
      * @returns A public URL string for accessing the dataset items
      *
      * @example
@@ -341,10 +340,9 @@ export class DatasetClient extends ResourceClient {
      * @since Added in 2.13.0
      */
     async createItemsPublicUrl(options = {}) {
-        const parsed = parseArgument(options, createItemsPublicUrlOptionsSchema, 'DatasetClientCreateItemsUrlOptions');
-        const dataset = await this.get();
-        const { expiresInSecs, ...queryOptions } = parsed;
-        let createdItemsPublicUrl = new URL(this._publicUrl('items'));
+        const { timeoutSecs = 'long', signal, expiresInSecs, ...queryOptions } = parseArgument(options, createItemsPublicUrlOptionsSchema, 'DatasetClientCreateItemsUrlOptions');
+        const dataset = await this.get({ timeoutSecs, signal });
+        let createdItemsPublicUrl = new URL(this.buildPublicUrl('items'));
         if (dataset?.urlSigningSecretKey) {
             const signature = await createStorageContentSignatureAsync({
                 resourceId: dataset.id,
@@ -356,31 +354,19 @@ export class DatasetClient extends ResourceClient {
         createdItemsPublicUrl = applyQueryParamsToUrl(createdItemsPublicUrl, queryOptions);
         return createdItemsPublicUrl.toString();
     }
-    _createPaginationList(response, userProvidedDesc) {
+    #createPaginationList(response, userProvidedDesc) {
+        const descHeader = response.headers['x-apify-pagination-desc'];
         return {
             items: response.data,
             total: Number(response.headers['x-apify-pagination-total']),
             offset: Number(response.headers['x-apify-pagination-offset']),
-            count: response.data.length, // because x-apify-pagination-count returns invalid values when hidden/empty items are skipped
+            // `x-apify-pagination-count` reports the rows scanned, which filters and `unwind` can leave above or below
+            // the items returned. It is derived from the dataset's item count, which lags a fresh push, so it can also
+            // fall below the items returned on an unfiltered page.
+            count: Math.max(Number(response.headers['x-apify-pagination-count'] ?? 0), response.data.length),
             limit: Number(response.headers['x-apify-pagination-limit']), // API returns 999999999999 when no limit is used
             // TODO: Replace this once https://github.com/apify/apify-core/issues/3503 is solved
-            desc: JSON.parse(response.headers['x-apify-pagination-desc'] ?? userProvidedDesc),
+            desc: typeof descHeader === 'string' ? JSON.parse(descHeader) : userProvidedDesc,
         };
     }
 }
-/**
- * Supported formats for downloading dataset items.
- */
-export var DownloadItemsFormat;
-(function (DownloadItemsFormat) {
-    DownloadItemsFormat["JSON"] = "json";
-    DownloadItemsFormat["JSONL"] = "jsonl";
-    DownloadItemsFormat["XML"] = "xml";
-    DownloadItemsFormat["HTML"] = "html";
-    DownloadItemsFormat["CSV"] = "csv";
-    DownloadItemsFormat["XLSX"] = "xlsx";
-    DownloadItemsFormat["RSS"] = "rss";
-})(DownloadItemsFormat || (DownloadItemsFormat = {}));
-const validItemFormats = [...new Set(Object.values(DownloadItemsFormat).map((item) => item.toLowerCase()))];
-// Declared below `validItemFormats` because it references it at module load time.
-const itemFormatSchema = z.enum(validItemFormats);

package/dist/resource_clients/dataset_collection.d.ts

@@ -1,8 +1,9 @@
 import { STORAGE_OWNERSHIP_FILTER } from '@apify/consts';
 import type { ApiClientSubResourceOptions } from '../base/api_client.js';
 import { ResourceCollectionClient } from '../base/resource_collection_client.js';
+import type { TimeoutOptions } from '../timeouts.js';
 import type { PaginatedList, PaginationOptions } from '../utils.js';
-import type { Dataset } from './dataset.js';
+import type { DatasetResource } from './dataset.js';
 /**
  * Client for managing the collection of datasets in your account.
  *
@@ -45,21 +46,24 @@ export declare class DatasetCollectionClient extends ResourceCollectionClient {
      * ```
      *
      * @param options - Pagination options.
+     * @param options.timeoutSecs - Timeout for each API request. Default is `'medium'`.
      * @returns A paginated iterator of Datasets.
      * @see https://docs.apify.com/api/v2/datasets-get
      */
-    list(options?: DatasetCollectionClientListOptions): Promise<DatasetCollectionClientListResult> & AsyncIterable<Dataset>;
+    list(options?: DatasetCollectionClientListOptions): Promise<DatasetCollectionClientListResult> & AsyncIterable<DatasetResource>;
     /**
      * Gets or creates a dataset with the specified name.
      *
      * @param name - Name of the dataset. If not provided, a default dataset is used.
      * @param options - Additional options like schema.
+     * @param options.schema - Schema of the dataset.
+     * @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
      * @returns The dataset object.
      * @see https://docs.apify.com/api/v2/datasets-post
      */
-    getOrCreate(name?: string, options?: DatasetCollectionClientGetOrCreateOptions): Promise<Dataset>;
+    getOrCreate(name?: string, options?: DatasetCollectionClientGetOrCreateOptions): Promise<DatasetResource>;
 }
-export interface DatasetCollectionClientListOptions extends PaginationOptions {
+export interface DatasetCollectionClientListOptions extends PaginationOptions, TimeoutOptions {
     unnamed?: boolean;
     desc?: boolean;
     /**
@@ -71,9 +75,9 @@ export interface DatasetCollectionClientListOptions extends PaginationOptions {
 /**
  * @since Added in 2.3.0
  */
-export interface DatasetCollectionClientGetOrCreateOptions {
+export interface DatasetCollectionClientGetOrCreateOptions extends TimeoutOptions {
     schema?: Record<string, unknown>;
 }
-export type DatasetCollectionClientListResult = PaginatedList<Dataset> & {
+export type DatasetCollectionClientListResult = PaginatedList<DatasetResource> & {
     unnamed: boolean;
 };

package/dist/resource_clients/dataset_collection.js

@@ -2,12 +2,14 @@ import { z } from 'zod';
 import { STORAGE_OWNERSHIP_FILTER } from '@apify/consts';
 import { ResourceCollectionClient } from '../base/resource_collection_client.js';
 import * as schemas from '../schemas.js';
+import { optionalSignalSchema, optionalTimeoutSchema, timeoutOptionsShape } from '../timeouts.js';
 import { anyObjectSchema, paginationOptionsShape, parseArgument } from '../utils.js';
 const listOptionsSchema = z.strictObject({
     unnamed: z.boolean().optional(),
     ...paginationOptionsShape,
     desc: z.boolean().optional(),
     ownership: z.enum(STORAGE_OWNERSHIP_FILTER).optional(),
+    ...timeoutOptionsShape,
 });
 const nameSchema = z.string().optional();
 const schemaSchema = anyObjectSchema.optional();
@@ -58,24 +60,33 @@ export class DatasetCollectionClient extends ResourceCollectionClient {
      * ```
      *
      * @param options - Pagination options.
+     * @param options.timeoutSecs - Timeout for each API request. Default is `'medium'`.
      * @returns A paginated iterator of Datasets.
      * @see https://docs.apify.com/api/v2/datasets-get
      */
     list(options = {}) {
         const parsed = parseArgument(options, listOptionsSchema, 'DatasetCollectionClientListOptions');
-        return this._listPaginated(schemas.ListOfDatasets(), parsed);
+        return this.listResourcesPaginated(schemas.ListOfDatasets(), parsed, 'medium');
     }
     /**
      * Gets or creates a dataset with the specified name.
      *
      * @param name - Name of the dataset. If not provided, a default dataset is used.
      * @param options - Additional options like schema.
+     * @param options.schema - Schema of the dataset.
+     * @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
      * @returns The dataset object.
      * @see https://docs.apify.com/api/v2/datasets-post
      */
     async getOrCreate(name, options) {
         parseArgument(name, nameSchema);
         parseArgument(options?.schema, schemaSchema); // TODO: Add schema validation
-        return this._getOrCreate(schemas.Dataset(), name, options);
+        parseArgument(options?.timeoutSecs, optionalTimeoutSchema);
+        parseArgument(options?.signal, optionalSignalSchema);
+        // `timeoutSecs` and `signal` are not part of the resource, so the body carries only the rest, and stays
+        // absent when there is nothing else to send.
+        const { timeoutSecs = 'short', signal, ...resource } = options ?? {};
+        const hasResource = Object.keys(resource).length > 0;
+        return this.getOrCreateResource(schemas.DatasetResource(), name, hasResource ? resource : undefined, timeoutSecs, signal);
     }
 }