apify-client 3.0.0-beta.46 → 3.0.0-beta.48

package/dist/base/api_client.js

@@ -70,35 +70,39 @@ export class ApiClient {
         // `buildParams()` and the query string. The same goes for `timeoutSecs` and `signal`, which callers take out
         // before calling this, since they also apply to every page request.
         const { chunkSize, ...listOptions } = options;
-        const paginatedListPromise = getPaginatedList({
-            ...listOptions,
-            limit: minForLimitParam(options.limit, chunkSize),
-        });
+        const firstPageLimit = minForLimitParam(options.limit, chunkSize);
+        const paginatedListPromise = getPaginatedList({ ...listOptions, limit: firstPageLimit });
         // A page can return more or fewer items than the rows it scanned (see `SCANNED_COUNT`). The next offset and
-        // the stop condition follow the scanned number alone: advancing by `items.length` would re-scan rows after a
-        // filter dropped some and skip rows after `unwind` multiplied them, and stopping at an empty page would end
-        // the iteration in front of rows a filter hid.
-        const scannedRows = (page) => page[SCANNED_COUNT] ?? page.items.length;
+        // the stop condition follow the scanned rows: advancing by `items.length` would re-scan rows after a filter
+        // dropped some and skip rows after `unwind` multiplied them, and stopping at an empty page would end the
+        // iteration in front of rows a filter hid. The scanned number is derived from a dataset's item count, which
+        // lags a fresh push, so the larger of it and `items.length` is taken, capped at the rows the request asked for
+        // so that an unwound page does not advance past rows the next request would then never read.
+        const scannedRows = (page, requestedLimit) => {
+            const scanned = Math.max(page[SCANNED_COUNT] ?? 0, page.items.length);
+            return requestedLimit ? Math.min(scanned, requestedLimit) : scanned;
+        };
+        // `total` is not consulted, as it changes when the listed resource grows during the iteration.
         async function* asyncGenerator() {
             let currentPage = await paginatedListPromise;
             yield* currentPage.items;
-            const offset = options.offset ?? 0;
-            const limit = Math.min(options.limit || currentPage.total, currentPage.total);
-            let pageScanned = scannedRows(currentPage);
-            let currentOffset = offset + pageScanned;
-            let remainingItems = Math.min(currentPage.total - offset, limit) - pageScanned;
+            let pageScanned = scannedRows(currentPage, firstPageLimit);
+            let currentOffset = (options.offset ?? 0) + pageScanned;
+            let remainingItems = options.limit ? options.limit - pageScanned : undefined;
             while (pageScanned > 0 && // Continue only if the last page scanned some rows.
-                remainingItems > 0) {
+                (remainingItems === undefined || remainingItems > 0)) {
+                const requestedLimit = minForLimitParam(remainingItems, chunkSize);
                 const newOptions = {
                     ...listOptions,
-                    limit: minForLimitParam(remainingItems, chunkSize),
+                    limit: requestedLimit,
                     offset: currentOffset,
                 };
                 currentPage = await getPaginatedList(newOptions);
                 yield* currentPage.items;
-                pageScanned = scannedRows(currentPage);
+                pageScanned = scannedRows(currentPage, requestedLimit);
                 currentOffset += pageScanned;
-                remainingItems -= pageScanned;
+                if (remainingItems !== undefined)
+                    remainingItems -= pageScanned;
             }
         }
         return Object.defineProperty(paginatedListPromise, Symbol.asyncIterator, {

package/dist/base/resource_collection_client.d.ts

@@ -11,6 +11,11 @@ export declare class ResourceCollectionClient extends ApiClient {
      * @private
      */
     protected listResources<T, R>(schema: z.ZodType, options: T | undefined, timeoutSecs: Timeout, signal?: AbortSignal): Promise<R>;
+    /**
+     * Lists an endpoint that returns every item in one response and ignores `offset` and `limit`. The result can be
+     * awaited as that response or iterated over its items, with a single request either way.
+     */
+    protected listUnpaginatedResources<Data, R extends PaginatedResponse<Data>>(schema: z.ZodType, timeoutSecs: Timeout, signal?: AbortSignal): AsyncIterable<Data> & Promise<R>;
     /**
      * Returns async iterator to iterate through all items and Promise that can be awaited to get first page of results.
      * `defaultTimeoutSecs` applies to every page request unless `options.timeoutSecs` overrides it, and

package/dist/base/resource_collection_client.js

@@ -18,6 +18,18 @@ export class ResourceCollectionClient extends ApiClient {
         });
         return parseResponse(response, schema);
     }
+    /**
+     * Lists an endpoint that returns every item in one response and ignores `offset` and `limit`. The result can be
+     * awaited as that response or iterated over its items, with a single request either way.
+     */
+    listUnpaginatedResources(schema, timeoutSecs, signal) {
+        const listPromise = this.listResources(schema, undefined, timeoutSecs, signal);
+        return Object.defineProperty(listPromise, Symbol.asyncIterator, {
+            async *value() {
+                yield* (await listPromise).items;
+            },
+        });
+    }
     /**
      * Returns async iterator to iterate through all items and Promise that can be awaited to get first page of results.
      * `defaultTimeoutSecs` applies to every page request unless `options.timeoutSecs` overrides it, and