apify-client 3.0.0-beta.8 → 3.0.0-rc.0

package/dist/resource_clients/key_value_store_collection.js

@@ -2,12 +2,14 @@ import { z } from 'zod';
 import { STORAGE_OWNERSHIP_FILTER } from '@apify/consts';
 import { ResourceCollectionClient } from '../base/resource_collection_client.js';
 import * as schemas from '../schemas.js';
+import { optionalSignalSchema, optionalTimeoutSchema, timeoutOptionsShape } from '../timeouts.js';
 import { anyObjectSchema, paginationOptionsShape, parseArgument } from '../utils.js';
 const listOptionsSchema = z.strictObject({
     unnamed: z.boolean().optional(),
     ...paginationOptionsShape,
     desc: z.boolean().optional(),
     ownership: z.enum(STORAGE_OWNERSHIP_FILTER).optional(),
+    ...timeoutOptionsShape,
 });
 const nameSchema = z.string().optional();
 const schemaSchema = anyObjectSchema.optional();
@@ -58,24 +60,33 @@ export class KeyValueStoreCollectionClient extends ResourceCollectionClient {
      * ```
      *
      * @param options - Pagination options.
+     * @param options.timeoutSecs - Timeout for each API request. Default is `'medium'`.
      * @returns A paginated iterator of Key-value stores.
      * @see https://docs.apify.com/api/v2/key-value-stores-get
      */
     list(options = {}) {
         const parsed = parseArgument(options, listOptionsSchema, 'KeyValueStoreCollectionClientListOptions');
-        return this._listPaginated(schemas.ListOfKeyValueStores(), parsed);
+        return this.listResourcesPaginated(schemas.ListOfKeyValueStores(), parsed, 'medium');
     }
     /**
      * Gets or creates a key-value store with the specified name.
      *
      * @param name - Name of the key-value store. If not provided, a default store is used.
      * @param options - Additional options like schema.
+     * @param options.schema - Schema of the key-value store.
+     * @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
      * @returns The key-value store object.
      * @see https://docs.apify.com/api/v2/key-value-stores-post
      */
     async getOrCreate(name, options) {
         parseArgument(name, nameSchema);
         parseArgument(options?.schema, schemaSchema); // TODO: Add schema validation
-        return this._getOrCreate(schemas.KeyValueStore(), name, options);
+        parseArgument(options?.timeoutSecs, optionalTimeoutSchema);
+        parseArgument(options?.signal, optionalSignalSchema);
+        // `timeoutSecs` and `signal` are not part of the resource, so the body carries only the rest, and stays
+        // absent when there is nothing else to send.
+        const { timeoutSecs = 'short', signal, ...resource } = options ?? {};
+        const hasResource = Object.keys(resource).length > 0;
+        return this.getOrCreateResource(schemas.KeyValueStoreResource(), name, hasResource ? resource : undefined, timeoutSecs, signal);
     }
 }

package/dist/resource_clients/log.d.ts

@@ -3,6 +3,7 @@ import type { Log } from '@apify/log';
 import { Logger, LogLevel } from '@apify/log';
 import type { ApiClientSubResourceOptions } from '../base/api_client.js';
 import { ResourceClient } from '../base/resource_client.js';
+import type { TimeoutOptions } from '../timeouts.js';
 /**
  * Client for accessing Actor run or build logs.
  *
@@ -35,7 +36,9 @@ export declare class LogClient extends ResourceClient {
      *
      * @param options - Log retrieval options.
      * @param options.raw - If `true`, returns raw log content without any processing. Default is `false`.
-     * @returns The log content as a string, or `undefined` if it does not exist.
+     * @param options.timeoutSecs - Timeout for the API request. Default is `'long'`.
+     * @returns The log content as a string, or `undefined` if it does not exist. A chained client such as
+     * `run.log()` throws an `ApifyApiError` on a 404, since the run itself may be what is missing.
      * @see https://docs.apify.com/api/v2/log-get
      */
     get(options?: LogOptions): Promise<string | undefined>;
@@ -44,7 +47,9 @@ export declare class LogClient extends ResourceClient {
      *
      * @param options - Log retrieval options.
      * @param options.raw - If `true`, returns raw log content without any processing. Default is `false`.
-     * @returns The log content as a Readable stream, or `undefined` if it does not exist.
+     * @param options.timeoutSecs - Timeout for the API request. Default is `'long'`.
+     * @returns The log content as a Readable stream, or `undefined` if it does not exist. A chained client such as
+     * `run.log()` throws an `ApifyApiError` on a 404, since the run itself may be what is missing.
      * @see https://docs.apify.com/api/v2/log-get
      */
     stream(options?: LogOptions): Promise<Readable | undefined>;
@@ -52,7 +57,7 @@ export declare class LogClient extends ResourceClient {
 /**
  * @since Added in 2.20.0
  */
-export interface LogOptions {
+export interface LogOptions extends TimeoutOptions {
     /** @default false */
     raw?: boolean;
 }
@@ -69,31 +74,17 @@ export declare class LoggerActorRedirect extends Logger {
  * @since Added in 2.20.0
  */
 export declare class StreamedLog {
-    private destinationLog;
-    private streamBuffer;
-    private splitMarker;
-    private relevancyTimeLimit;
-    private logClient;
-    private streamingTask;
-    private stopLogging;
+    #private;
     constructor(options: StreamedLogOptions);
     /**
      * Start log redirection.
      */
     start(): void;
     /**
-     * Stop log redirection.
+     * Stop log redirection. Waits up to one second for lines already in flight, then aborts the log stream request. If
+     * no stream has delivered anything yet, reads the whole log in one request instead.
      */
     stop(): Promise<void>;
-    /**
-     * Get log stream from response and redirect it to another log.
-     */
-    private streamLog;
-    private logStreamChunks;
-    /**
-     * Parse the buffer and log complete messages.
-     */
-    private logBufferContent;
 }
 /**
  * @since Added in 2.20.0
@@ -105,4 +96,6 @@ export interface StreamedLogOptions {
     toLog: Log;
     /** Whether to redirect all logs from Actor run start (even logs from the past). */
     fromStart?: boolean;
+    /** Ends the log stream once it aborts. */
+    signal?: AbortSignal;
 }

package/dist/resource_clients/log.js

@@ -1,7 +1,20 @@
 import c from 'ansi-colors';
+import { z } from 'zod';
 import log, { Logger, LogLevel } from '@apify/log';
 import { ResourceClient } from '../base/resource_client.js';
-import { cast, catchNotFoundOrThrow } from '../utils.js';
+import { timeoutOptionsShape } from '../timeouts.js';
+import { cast, catchNotFoundForResourceOrThrow, concatBytes, parseArgument, sleep } from '../utils.js';
+/**
+ * Pause before the first reopen of a log stream that ended before the run logged anything. Each further reopen doubles
+ * the pause up to `EMPTY_LOG_STREAM_MAX_RETRY_MILLIS`, which bounds the request rate while a run waits long to start,
+ * for example for free memory.
+ */
+const EMPTY_LOG_STREAM_RETRY_MILLIS = 500;
+/** Upper bound on the pause between reopens of an empty log stream. */
+const EMPTY_LOG_STREAM_MAX_RETRY_MILLIS = 5_000;
+const logOptionsSchema = z.strictObject({ raw: z.boolean().optional(), ...timeoutOptionsShape });
+/** How long `StreamedLog.stop()` waits for lines already in flight before aborting the log stream request. */
+const STOP_GRACE_MILLIS = 1000;
 /**
  * Client for accessing Actor run or build logs.
  *
@@ -39,21 +52,26 @@ export class LogClient extends ResourceClient {
      *
      * @param options - Log retrieval options.
      * @param options.raw - If `true`, returns raw log content without any processing. Default is `false`.
-     * @returns The log content as a string, or `undefined` if it does not exist.
+     * @param options.timeoutSecs - Timeout for the API request. Default is `'long'`.
+     * @returns The log content as a string, or `undefined` if it does not exist. A chained client such as
+     * `run.log()` throws an `ApifyApiError` on a 404, since the run itself may be what is missing.
      * @see https://docs.apify.com/api/v2/log-get
      */
     async get(options = {}) {
+        const { timeoutSecs = 'long', signal, ...params } = parseArgument(options, logOptionsSchema, 'LogOptions');
         const requestOpts = {
-            url: this._url(),
+            url: this.buildUrl(),
             method: 'GET',
-            params: this._params(options),
+            params: this.buildParams(params),
+            timeoutSecs,
+            signal,
         };
         try {
             const response = await this.httpClient.call(requestOpts);
             return cast(response.data);
         }
         catch (err) {
-            catchNotFoundOrThrow(err);
+            catchNotFoundForResourceOrThrow(err, this.id);
         }
         return undefined;
     }
@@ -62,26 +80,31 @@ export class LogClient extends ResourceClient {
      *
      * @param options - Log retrieval options.
      * @param options.raw - If `true`, returns raw log content without any processing. Default is `false`.
-     * @returns The log content as a Readable stream, or `undefined` if it does not exist.
+     * @param options.timeoutSecs - Timeout for the API request. Default is `'long'`.
+     * @returns The log content as a Readable stream, or `undefined` if it does not exist. A chained client such as
+     * `run.log()` throws an `ApifyApiError` on a 404, since the run itself may be what is missing.
      * @see https://docs.apify.com/api/v2/log-get
      */
     async stream(options = {}) {
+        const { timeoutSecs = 'long', signal, raw } = parseArgument(options, logOptionsSchema, 'LogOptions');
         const params = {
             stream: true,
-            raw: options.raw,
+            raw,
         };
         const requestOpts = {
-            url: this._url(),
+            url: this.buildUrl(),
             method: 'GET',
-            params: this._params(params),
+            params: this.buildParams(params),
             responseType: 'stream',
+            timeoutSecs,
+            signal,
         };
         try {
             const response = await this.httpClient.call(requestOpts);
             return cast(response.data);
         }
         catch (err) {
-            catchNotFoundOrThrow(err);
+            catchNotFoundForResourceOrThrow(err, this.id);
         }
         return undefined;
     }
@@ -119,39 +142,50 @@ export class LoggerActorRedirect extends Logger {
  * @since Added in 2.20.0
  */
 export class StreamedLog {
-    destinationLog;
-    streamBuffer = [];
-    splitMarker = /(?:\n|^)(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z)/g;
-    relevancyTimeLimit;
-    logClient;
-    streamingTask = null;
-    stopLogging = false;
+    #destinationLog;
+    #streamBuffer = [];
+    #decoder = new TextDecoder();
+    #splitMarker = /(?:\n|^)(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z)/g;
+    #relevancyTimeLimit;
+    #logClient;
+    #signal;
+    #streamingTask = null;
+    #stopLogging = false;
+    #stopController = new AbortController();
+    #wakeController = new AbortController();
     constructor(options) {
-        const { toLog, logClient, fromStart = true } = options;
-        this.destinationLog = toLog;
-        this.logClient = logClient;
-        this.relevancyTimeLimit = fromStart ? null : new Date();
+        const { toLog, logClient, fromStart = true, signal } = options;
+        this.#destinationLog = toLog;
+        this.#logClient = logClient;
+        this.#signal = signal;
+        this.#relevancyTimeLimit = fromStart ? null : new Date();
     }
     /**
      * Start log redirection.
      */
     start() {
-        if (this.streamingTask) {
+        if (this.#streamingTask) {
             throw new Error('Streaming task already active');
         }
-        this.stopLogging = false;
-        this.streamingTask = this.streamLog();
+        this.#stopLogging = false;
+        this.#stopController = new AbortController();
+        this.#wakeController = new AbortController();
+        this.#streamingTask = this.#streamLog();
     }
     /**
-     * Stop log redirection.
+     * Stop log redirection. Waits up to one second for lines already in flight, then aborts the log stream request. If
+     * no stream has delivered anything yet, reads the whole log in one request instead.
      */
     async stop() {
-        if (!this.streamingTask) {
+        if (!this.#streamingTask) {
             throw new Error('Streaming task is not active');
         }
-        this.stopLogging = true;
+        this.#stopLogging = true;
+        this.#wakeController.abort();
+        const stopController = this.#stopController;
+        const abortTimeout = setTimeout(() => stopController.abort(), STOP_GRACE_MILLIS);
         try {
-            await this.streamingTask;
+            await this.#streamingTask;
         }
         catch (err) {
             if (!(err instanceof Error && err.name === 'AbortError')) {
@@ -159,73 +193,121 @@ export class StreamedLog {
             }
         }
         finally {
-            this.streamingTask = null;
+            clearTimeout(abortTimeout);
+            this.#streamingTask = null;
         }
     }
     /**
      * Get log stream from response and redirect it to another log.
      */
-    async streamLog() {
-        const logStream = await this.logClient.stream({ raw: true });
-        if (!logStream) {
-            return;
-        }
+    async #streamLog() {
+        const signal = this.#signal
+            ? AbortSignal.any([this.#signal, this.#stopController.signal])
+            : this.#stopController.signal;
         try {
-            const lastChunkRemainder = await this.logStreamChunks(logStream);
+            let lastChunkRemainder;
+            let retryMillis = EMPTY_LOG_STREAM_RETRY_MILLIS;
+            // The API serves the log of a run that has not logged anything yet as an empty stream that ends at once,
+            // so reopen it until the first bytes arrive. If stopped before any bytes arrive, fetch it in one request.
+            while (!lastChunkRemainder) {
+                if (this.#stopLogging) {
+                    // `signal` is aborted once the stop grace period ends, which would cut this read short.
+                    const logContent = await this.#logClient.get({ raw: true, signal: this.#signal });
+                    lastChunkRemainder = await this.#logStreamChunks([new TextEncoder().encode(logContent ?? '')], signal);
+                    break;
+                }
+                let logStream;
+                try {
+                    logStream = await this.#logClient.stream({ raw: true, signal });
+                }
+                catch (err) {
+                    // The stop grace period ended while the stream was connecting, so read the log in one request.
+                    if (this.#stopLogging && !this.#signal?.aborted)
+                        continue;
+                    throw err;
+                }
+                if (!logStream) {
+                    return;
+                }
+                // A stream opened during stop() would be cut after its first chunk, so read the log in one request.
+                if (this.#stopLogging) {
+                    logStream.destroy();
+                    continue;
+                }
+                lastChunkRemainder = await this.#logStreamChunks(logStream, signal);
+                if (!lastChunkRemainder) {
+                    await sleep(retryMillis, AbortSignal.any([signal, this.#wakeController.signal]));
+                    retryMillis = Math.min(retryMillis * 2, EMPTY_LOG_STREAM_MAX_RETRY_MILLIS);
+                }
+            }
             // Process whatever is left when exiting. Maybe it is incomplete, maybe it is last log without EOL.
-            const lastMessage = Buffer.from(lastChunkRemainder).toString().trim();
+            const lastMessage = this.#decoder.decode(lastChunkRemainder).trim();
             if (lastMessage.length) {
-                this.destinationLog.info(lastMessage);
+                this.#destinationLog.info(lastMessage);
             }
         }
         catch (err) {
+            if (signal.aborted)
+                return;
             log.warning(`Log redirection stopped due to error`, err);
         }
     }
-    async logStreamChunks(logStream) {
+    /**
+     * Redirect every complete message in the chunks and return the incomplete rest, or `undefined` when there were no
+     * chunks at all.
+     */
+    async #logStreamChunks(logStream, signal) {
         // Chunk may be incomplete. Keep remainder for next chunk.
-        let previousChunkRemainder = new Uint8Array();
-        for await (const chunk of logStream) {
-            // Handle possible leftover incomplete line from previous chunk.
-            // Everything before last end of line is complete.
-            const chunkWithPreviousRemainder = new Uint8Array(previousChunkRemainder.length + chunk.length);
-            chunkWithPreviousRemainder.set(previousChunkRemainder, 0);
-            chunkWithPreviousRemainder.set(chunk, previousChunkRemainder.length);
-            const lastCompleteMessageIndex = chunkWithPreviousRemainder.lastIndexOf(0x0a);
-            previousChunkRemainder = chunkWithPreviousRemainder.slice(lastCompleteMessageIndex);
-            // Push complete part of the chunk to the buffer
-            this.streamBuffer.push(Buffer.from(chunkWithPreviousRemainder.slice(0, lastCompleteMessageIndex)));
-            this.logBufferContent();
-            // Keep processing the new data until stopped
-            if (this.stopLogging) {
-                break;
+        let previousChunkRemainder;
+        try {
+            for await (const chunk of logStream) {
+                // Handle possible leftover incomplete line from previous chunk.
+                // Everything before last end of line is complete.
+                previousChunkRemainder ??= new Uint8Array();
+                const chunkWithPreviousRemainder = new Uint8Array(previousChunkRemainder.length + chunk.length);
+                chunkWithPreviousRemainder.set(previousChunkRemainder, 0);
+                chunkWithPreviousRemainder.set(chunk, previousChunkRemainder.length);
+                const lastCompleteMessageIndex = chunkWithPreviousRemainder.lastIndexOf(0x0a);
+                previousChunkRemainder = chunkWithPreviousRemainder.slice(lastCompleteMessageIndex);
+                // Push complete part of the chunk to the buffer
+                this.#streamBuffer.push(chunkWithPreviousRemainder.slice(0, lastCompleteMessageIndex));
+                this.#logBufferContent();
+                // Keep processing the new data until stopped
+                if (this.#stopLogging) {
+                    break;
+                }
             }
         }
+        catch (err) {
+            // An aborted stream still hands back its unterminated last line, so the caller can flush it.
+            if (!signal.aborted)
+                throw err;
+        }
         return previousChunkRemainder;
     }
     /**
      * Parse the buffer and log complete messages.
      */
-    logBufferContent() {
-        const allParts = Buffer.concat(this.streamBuffer).toString().split(this.splitMarker).slice(1);
+    #logBufferContent() {
+        const allParts = this.#decoder.decode(concatBytes(this.#streamBuffer)).split(this.#splitMarker).slice(1);
         // Parse the buffer parts into complete messages
         const messageMarkers = allParts.filter((_, i) => i % 2 === 0);
         const messageContents = allParts.filter((_, i) => i % 2 !== 0);
-        this.streamBuffer = [];
+        this.#streamBuffer = [];
         messageMarkers.forEach((marker, index) => {
             const decodedMarker = marker;
             const decodedContent = messageContents[index];
-            if (this.relevancyTimeLimit) {
+            if (this.#relevancyTimeLimit) {
                 // Log only relevant messages. Ignore too old log messages.
                 const logTime = new Date(decodedMarker);
-                if (logTime < this.relevancyTimeLimit) {
+                if (logTime < this.#relevancyTimeLimit) {
                     return;
                 }
             }
             const message = decodedMarker + decodedContent;
             // Original log level information is not available. Log all on info level. Log level could be guessed for
             // some logs, but for any multiline logs such guess would be probably correct only for the first line.
-            this.destinationLog.info(message.trim());
+            this.#destinationLog.info(message.trim());
         });
     }
 }