apify-client 3.0.0-beta.42 → 3.0.0-beta.43

package/dist/resource_clients/run.d.ts

@@ -4,6 +4,7 @@ import type { ApiClientOptionsWithOptionalResourcePath } from '../base/api_clien
 import { ResourceClient } from '../base/resource_client.js';
 import type { ApifyResponse } from '../http_clients/index.js';
 import type { TimeoutOptions } from '../timeouts.js';
+import type { PaginationOptions } from '../utils.js';
 import type { ActorInput, ActorRun } from './actor.js';
 import { DatasetClient } from './dataset.js';
 import { KeyValueStoreClient } from './key_value_store.js';
@@ -294,6 +295,41 @@ export declare class RunClient extends ResourceClient {
      * @since Added in 2.20.0
      */
     getStreamedLog(options?: GetStreamedLogOptions): Promise<StreamedLog | undefined>;
+    /**
+     * Iterates over the items of the run's default dataset while the run is still producing them.
+     *
+     * While the run has not finished, each poll yields the rows below the dataset's `itemCount` and then waits up to
+     * `pollIntervalSecs` for the run to finish, so the last rows are read as soon as it does. Each page is requested
+     * with a `limit` that ends at `itemCount`, so it covers exactly the rows it asks for, whatever the filters or
+     * `unwind` do to the items. `itemCount` lags a few seconds behind
+     * the pushed items, so once the run reaches a terminal status, the rows past it are read a page at a time until
+     * none are left, and the iterator returns. On a `lastRun()` client, the iterator sticks to the run that its first
+     * request resolves to.
+     *
+     * @param options - Iteration options
+     * @param options.offset - Number of rows to skip from the beginning. Default is 0.
+     * @param options.limit - Maximum number of dataset rows to scan. Fewer items are yielded when filters drop some,
+     * more when `unwind` splits a row into several. Default is no limit.
+     * @param options.clean - If `true`, returns only non-empty items and skips hidden fields. Default is `false`.
+     * @param options.fields - Array of field names to include in the results. Omits all other fields.
+     * @param options.omit - Array of field names to exclude from the results.
+     * @param options.unwind - Array of field names to unwind. Each array value creates a separate item.
+     * @param options.skipEmpty - If `true`, skips empty items. Default is `false`.
+     * @param options.skipHidden - If `true`, skips hidden fields (fields starting with `#`). Default is `false`.
+     * @param options.chunkSize - Maximum number of dataset rows requested per API call. Default is 1000.
+     * @param options.pollIntervalSecs - How long to wait for the run to finish between polls, in seconds. Default is 5.
+     * @param options.timeoutSecs - Timeout for each API request. Default is `'long'`.
+     * @returns An async iterable of the dataset items
+     * @see https://docs.apify.com/api/v2/dataset-items-get
+     *
+     * @example
+     * ```javascript
+     * for await (const item of client.run('run-id').iterateDatasetItems()) {
+     *     console.log(item);
+     * }
+     * ```
+     */
+    iterateDatasetItems<Data extends Record<string | number, any> = Record<string | number, unknown>>(options?: RunIterateDatasetItemsOptions): AsyncGenerator<Data, void, undefined>;
 }
 /**
  * Options for getting a streamed log.
@@ -386,3 +422,22 @@ export interface RunWaitForFinishOptions extends TimeoutOptions {
      */
     waitSecs?: number;
 }
+/**
+ * Options for iterating over the items of a run's default dataset while the run is still producing them.
+ */
+export interface RunIterateDatasetItemsOptions extends PaginationOptions, TimeoutOptions {
+    /** If `true`, returns only non-empty items and skips hidden fields. */
+    clean?: boolean;
+    /** Field names to include in the items. Omits all other fields. */
+    fields?: string[];
+    /** Field names to exclude from the items. */
+    omit?: string[];
+    /** If `true`, skips empty items. */
+    skipEmpty?: boolean;
+    /** If `true`, skips hidden fields (fields starting with `#`). */
+    skipHidden?: boolean;
+    /** Field names to unwind. Each array value creates a separate item. */
+    unwind?: string[];
+    /** How long to wait for the run to finish between polls, in seconds. Default is 5. */
+    pollIntervalSecs?: number;
+}

package/dist/resource_clients/run.js

@@ -1,10 +1,11 @@
 import { z } from 'zod';
+import { ACT_JOB_TERMINAL_STATUSES } from '@apify/consts';
 import { LEVELS, Log } from '@apify/log';
 import { ResourceClient } from '../base/resource_client.js';
 import * as schemas from '../schemas.js';
 import { optionalSignalSchema, optionalTimeoutSchema, timeoutOptionsSchema, timeoutOptionsShape } from '../timeouts.js';
 import { runtime } from '#runtime';
-import { anyObjectSchema, parseArgument, parseResponse } from '../utils.js';
+import { anyObjectSchema, paginationOptionsShape, parseArgument, parseResponse, SCANNED_COUNT } from '../utils.js';
 import { DatasetClient } from './dataset.js';
 import { KeyValueStoreClient } from './key_value_store.js';
 import { LogClient, LoggerActorRedirect, StreamedLog } from './log.js';
@@ -34,6 +35,18 @@ const chargeOptionsSchema = z.strictObject({
     ...timeoutOptionsShape,
 });
 const waitForFinishOptionsSchema = z.strictObject({ waitSecs: z.number().optional(), ...timeoutOptionsShape });
+const iterateDatasetItemsOptionsSchema = z.strictObject({
+    clean: z.boolean().optional(),
+    fields: z.array(z.string()).optional(),
+    omit: z.array(z.string()).optional(),
+    ...paginationOptionsShape,
+    skipEmpty: z.boolean().optional(),
+    skipHidden: z.boolean().optional(),
+    unwind: z.array(z.string()).optional(),
+    pollIntervalSecs: z.number().min(0).optional(),
+    ...timeoutOptionsShape,
+});
+const DEFAULT_ITERATE_CHUNK_SIZE = 1000;
 /**
  * Client for managing a specific Actor run.
  *
@@ -451,4 +464,85 @@ export class RunClient extends ResourceClient {
         }
         return new StreamedLog({ logClient: this.log(), toLog, fromStart, signal });
     }
+    /**
+     * Iterates over the items of the run's default dataset while the run is still producing them.
+     *
+     * While the run has not finished, each poll yields the rows below the dataset's `itemCount` and then waits up to
+     * `pollIntervalSecs` for the run to finish, so the last rows are read as soon as it does. Each page is requested
+     * with a `limit` that ends at `itemCount`, so it covers exactly the rows it asks for, whatever the filters or
+     * `unwind` do to the items. `itemCount` lags a few seconds behind
+     * the pushed items, so once the run reaches a terminal status, the rows past it are read a page at a time until
+     * none are left, and the iterator returns. On a `lastRun()` client, the iterator sticks to the run that its first
+     * request resolves to.
+     *
+     * @param options - Iteration options
+     * @param options.offset - Number of rows to skip from the beginning. Default is 0.
+     * @param options.limit - Maximum number of dataset rows to scan. Fewer items are yielded when filters drop some,
+     * more when `unwind` splits a row into several. Default is no limit.
+     * @param options.clean - If `true`, returns only non-empty items and skips hidden fields. Default is `false`.
+     * @param options.fields - Array of field names to include in the results. Omits all other fields.
+     * @param options.omit - Array of field names to exclude from the results.
+     * @param options.unwind - Array of field names to unwind. Each array value creates a separate item.
+     * @param options.skipEmpty - If `true`, skips empty items. Default is `false`.
+     * @param options.skipHidden - If `true`, skips hidden fields (fields starting with `#`). Default is `false`.
+     * @param options.chunkSize - Maximum number of dataset rows requested per API call. Default is 1000.
+     * @param options.pollIntervalSecs - How long to wait for the run to finish between polls, in seconds. Default is 5.
+     * @param options.timeoutSecs - Timeout for each API request. Default is `'long'`.
+     * @returns An async iterable of the dataset items
+     * @see https://docs.apify.com/api/v2/dataset-items-get
+     *
+     * @example
+     * ```javascript
+     * for await (const item of client.run('run-id').iterateDatasetItems()) {
+     *     console.log(item);
+     * }
+     * ```
+     */
+    async *iterateDatasetItems(options = {}) {
+        const { offset, limit, chunkSize, pollIntervalSecs = 5, timeoutSecs = 'long', signal, ...itemOptions } = parseArgument(options, iterateDatasetItemsOptionsSchema, 'RunIterateDatasetItemsOptions');
+        const pageSize = chunkSize || DEFAULT_ITERATE_CHUNK_SIZE;
+        let position = offset ?? 0;
+        const end = limit ? position + limit : undefined;
+        let run = await this.get({ timeoutSecs, signal });
+        // A `lastRun()` client resolves `runs/last` on each request, so a newer run would swap the dataset mid-iteration.
+        const runClient = run && run.id !== this.id ? this.apifyClient.run(run.id) : this;
+        const datasetClient = runClient.dataset();
+        const listPage = async (pageOffset, pageLimit) => datasetClient.listItems({ ...itemOptions, offset: pageOffset, limit: pageLimit, timeoutSecs, signal });
+        while (true) {
+            const isFinished = !run || ACT_JOB_TERMINAL_STATUSES.includes(run.status);
+            const dataset = await datasetClient.get({ timeoutSecs, signal });
+            let itemCount = dataset?.itemCount ?? 0;
+            if (end !== undefined)
+                itemCount = Math.min(itemCount, end);
+            while (position < itemCount) {
+                const pageLimit = Math.min(pageSize, itemCount - position);
+                const page = await listPage(position, pageLimit);
+                yield* page.items;
+                position += pageLimit;
+            }
+            if (end !== undefined && position >= end)
+                return;
+            if (isFinished)
+                break;
+            run = await runClient.waitForFinish({ waitSecs: pollIntervalSecs, timeoutSecs, signal });
+        }
+        const { clean, skipEmpty, unwind } = itemOptions;
+        while (true) {
+            const pageLimit = end !== undefined ? Math.min(pageSize, end - position) : pageSize;
+            const page = await listPage(position, pageLimit);
+            yield* page.items;
+            // Only an empty page marks the end, as filters can shorten a full one. A page that `clean`, `skipEmpty` or
+            // `unwind` emptied past a lagging `itemCount` reports no scanned rows either, so a plain read checks.
+            const isEmpty = page.items.length === 0 && !page[SCANNED_COUNT];
+            if (isEmpty &&
+                (!(clean || skipEmpty || unwind?.length) ||
+                    (await datasetClient.listItems({ offset: position, limit: 1, timeoutSecs, signal })).items
+                        .length === 0)) {
+                return;
+            }
+            position += pageLimit;
+            if (end !== undefined && position >= end)
+                return;
+        }
+    }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "apify-client",
-  "version": "3.0.0-beta.42",
+  "version": "3.0.0-beta.43",
   "description": "Apify API client for JavaScript",
   "engines": {
     "node": ">=22.0.0"