apify-client 3.0.0-beta.7 → 3.0.0-beta.71

package/dist/resource_clients/log.js

@@ -1,7 +1,21 @@
 import c from 'ansi-colors';
+import { z } from 'zod';
 import log, { Logger, LogLevel } from '@apify/log';
 import { ResourceClient } from '../base/resource_client.js';
-import { cast, catchNotFoundOrThrow } from '../utils.js';
+import { timeoutOptionsShape } from '../timeouts.js';
+import { cast, catchNotFoundForResourceOrThrow, concatBytes, parseArgument, sleep } from '../utils.js';
+/**
+ * Pause before the first reopen of a log stream that ended before the run logged anything. Each further reopen doubles
+ * the pause up to `EMPTY_LOG_STREAM_MAX_RETRY_MILLIS`, which bounds the request rate while a run waits long to start,
+ * for example for free memory.
+ */
+const EMPTY_LOG_STREAM_RETRY_MILLIS = 500;
+/** Upper bound on the pause between reopens of an empty log stream. */
+const EMPTY_LOG_STREAM_MAX_RETRY_MILLIS = 5_000;
+const logOptionsSchema = z.strictObject({ raw: z.boolean().optional(), ...timeoutOptionsShape });
+const logGetOptionsSchema = logOptionsSchema.extend({ buffer: z.boolean().optional() });
+/** How long `StreamedLog.stop()` waits for lines already in flight before aborting the log stream request. */
+const STOP_GRACE_MILLIS = 1000;
 /**
  * Client for accessing Actor run or build logs.
  *
@@ -17,6 +31,9 @@ import { cast, catchNotFoundOrThrow } from '../utils.js';
  * const log = await runClient.log().get();
  * console.log(log);
  *
+ * // Get the log content as raw bytes
+ * const logBuffer = await runClient.log().get({ buffer: true });
+ *
  * // Stream the log in real-time
  * const stream = await runClient.log().stream();
  * stream.on('line', (line) => console.log(line));
@@ -34,26 +51,23 @@ export class LogClient extends ResourceClient {
             ...options,
         });
     }
-    /**
-     * Retrieves the log as a string.
-     *
-     * @param options - Log retrieval options.
-     * @param options.raw - If `true`, returns raw log content without any processing. Default is `false`.
-     * @returns The log content as a string, or `undefined` if it does not exist.
-     * @see https://docs.apify.com/api/v2/log-get
-     */
     async get(options = {}) {
+        const { timeoutSecs = 'long', signal, buffer, ...params } = parseArgument(options, logGetOptionsSchema, 'LogGetOptions');
         const requestOpts = {
-            url: this._url(),
+            url: this.buildUrl(),
             method: 'GET',
-            params: this._params(options),
+            params: this.buildParams(params),
+            timeoutSecs,
+            signal,
         };
+        if (buffer)
+            requestOpts.responseType = 'buffer';
         try {
             const response = await this.httpClient.call(requestOpts);
             return cast(response.data);
         }
         catch (err) {
-            catchNotFoundOrThrow(err);
+            catchNotFoundForResourceOrThrow(err, this.id);
         }
         return undefined;
     }
@@ -62,26 +76,31 @@ export class LogClient extends ResourceClient {
      *
      * @param options - Log retrieval options.
      * @param options.raw - If `true`, returns raw log content without any processing. Default is `false`.
-     * @returns The log content as a Readable stream, or `undefined` if it does not exist.
+     * @param options.timeoutSecs - Timeout for the API request. Default is `'long'`.
+     * @returns The log content as a Readable stream, or `undefined` if it does not exist. A chained client such as
+     * `run.log()` throws an `ApifyApiError` on a 404, since the run itself may be what is missing.
      * @see https://docs.apify.com/api/v2/log-get
      */
     async stream(options = {}) {
+        const { timeoutSecs = 'long', signal, raw } = parseArgument(options, logOptionsSchema, 'LogOptions');
         const params = {
             stream: true,
-            raw: options.raw,
+            raw,
         };
         const requestOpts = {
-            url: this._url(),
+            url: this.buildUrl(),
             method: 'GET',
-            params: this._params(params),
+            params: this.buildParams(params),
             responseType: 'stream',
+            timeoutSecs,
+            signal,
         };
         try {
             const response = await this.httpClient.call(requestOpts);
             return cast(response.data);
         }
         catch (err) {
-            catchNotFoundOrThrow(err);
+            catchNotFoundForResourceOrThrow(err, this.id);
         }
         return undefined;
     }
@@ -119,39 +138,50 @@ export class LoggerActorRedirect extends Logger {
  * @since Added in 2.20.0
  */
 export class StreamedLog {
-    destinationLog;
-    streamBuffer = [];
-    splitMarker = /(?:\n|^)(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z)/g;
-    relevancyTimeLimit;
-    logClient;
-    streamingTask = null;
-    stopLogging = false;
+    #destinationLog;
+    #streamBuffer = [];
+    #decoder = new TextDecoder();
+    #splitMarker = /(?:\n|^)(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z)/g;
+    #relevancyTimeLimit;
+    #logClient;
+    #signal;
+    #streamingTask = null;
+    #stopLogging = false;
+    #stopController = new AbortController();
+    #wakeController = new AbortController();
     constructor(options) {
-        const { toLog, logClient, fromStart = true } = options;
-        this.destinationLog = toLog;
-        this.logClient = logClient;
-        this.relevancyTimeLimit = fromStart ? null : new Date();
+        const { toLog, logClient, fromStart = true, signal } = options;
+        this.#destinationLog = toLog;
+        this.#logClient = logClient;
+        this.#signal = signal;
+        this.#relevancyTimeLimit = fromStart ? null : new Date();
     }
     /**
      * Start log redirection.
      */
     start() {
-        if (this.streamingTask) {
+        if (this.#streamingTask) {
             throw new Error('Streaming task already active');
         }
-        this.stopLogging = false;
-        this.streamingTask = this.streamLog();
+        this.#stopLogging = false;
+        this.#stopController = new AbortController();
+        this.#wakeController = new AbortController();
+        this.#streamingTask = this.#streamLog();
     }
     /**
-     * Stop log redirection.
+     * Stop log redirection. Waits up to one second for lines already in flight, then aborts the log stream request. If
+     * no stream has delivered anything yet, reads the whole log in one request instead.
      */
     async stop() {
-        if (!this.streamingTask) {
+        if (!this.#streamingTask) {
             throw new Error('Streaming task is not active');
         }
-        this.stopLogging = true;
+        this.#stopLogging = true;
+        this.#wakeController.abort();
+        const stopController = this.#stopController;
+        const abortTimeout = setTimeout(() => stopController.abort(), STOP_GRACE_MILLIS);
         try {
-            await this.streamingTask;
+            await this.#streamingTask;
         }
         catch (err) {
             if (!(err instanceof Error && err.name === 'AbortError')) {
@@ -159,73 +189,121 @@ export class StreamedLog {
             }
         }
         finally {
-            this.streamingTask = null;
+            clearTimeout(abortTimeout);
+            this.#streamingTask = null;
         }
     }
     /**
      * Get log stream from response and redirect it to another log.
      */
-    async streamLog() {
-        const logStream = await this.logClient.stream({ raw: true });
-        if (!logStream) {
-            return;
-        }
+    async #streamLog() {
+        const signal = this.#signal
+            ? AbortSignal.any([this.#signal, this.#stopController.signal])
+            : this.#stopController.signal;
         try {
-            const lastChunkRemainder = await this.logStreamChunks(logStream);
+            let lastChunkRemainder;
+            let retryMillis = EMPTY_LOG_STREAM_RETRY_MILLIS;
+            // The API serves the log of a run that has not logged anything yet as an empty stream that ends at once,
+            // so reopen it until the first bytes arrive. If stopped before any bytes arrive, fetch it in one request.
+            while (!lastChunkRemainder) {
+                if (this.#stopLogging) {
+                    // `signal` is aborted once the stop grace period ends, which would cut this read short.
+                    const logContent = await this.#logClient.get({ raw: true, signal: this.#signal });
+                    lastChunkRemainder = await this.#logStreamChunks([new TextEncoder().encode(logContent ?? '')], signal);
+                    break;
+                }
+                let logStream;
+                try {
+                    logStream = await this.#logClient.stream({ raw: true, signal });
+                }
+                catch (err) {
+                    // The stop grace period ended while the stream was connecting, so read the log in one request.
+                    if (this.#stopLogging && !this.#signal?.aborted)
+                        continue;
+                    throw err;
+                }
+                if (!logStream) {
+                    return;
+                }
+                // A stream opened during stop() would be cut after its first chunk, so read the log in one request.
+                if (this.#stopLogging) {
+                    logStream.destroy();
+                    continue;
+                }
+                lastChunkRemainder = await this.#logStreamChunks(logStream, signal);
+                if (!lastChunkRemainder) {
+                    await sleep(retryMillis, AbortSignal.any([signal, this.#wakeController.signal]));
+                    retryMillis = Math.min(retryMillis * 2, EMPTY_LOG_STREAM_MAX_RETRY_MILLIS);
+                }
+            }
             // Process whatever is left when exiting. Maybe it is incomplete, maybe it is last log without EOL.
-            const lastMessage = Buffer.from(lastChunkRemainder).toString().trim();
+            const lastMessage = this.#decoder.decode(lastChunkRemainder).trim();
             if (lastMessage.length) {
-                this.destinationLog.info(lastMessage);
+                this.#destinationLog.info(lastMessage);
             }
         }
         catch (err) {
+            if (signal.aborted)
+                return;
             log.warning(`Log redirection stopped due to error`, err);
         }
     }
-    async logStreamChunks(logStream) {
+    /**
+     * Redirect every complete message in the chunks and return the incomplete rest, or `undefined` when there were no
+     * chunks at all.
+     */
+    async #logStreamChunks(logStream, signal) {
         // Chunk may be incomplete. Keep remainder for next chunk.
-        let previousChunkRemainder = new Uint8Array();
-        for await (const chunk of logStream) {
-            // Handle possible leftover incomplete line from previous chunk.
-            // Everything before last end of line is complete.
-            const chunkWithPreviousRemainder = new Uint8Array(previousChunkRemainder.length + chunk.length);
-            chunkWithPreviousRemainder.set(previousChunkRemainder, 0);
-            chunkWithPreviousRemainder.set(chunk, previousChunkRemainder.length);
-            const lastCompleteMessageIndex = chunkWithPreviousRemainder.lastIndexOf(0x0a);
-            previousChunkRemainder = chunkWithPreviousRemainder.slice(lastCompleteMessageIndex);
-            // Push complete part of the chunk to the buffer
-            this.streamBuffer.push(Buffer.from(chunkWithPreviousRemainder.slice(0, lastCompleteMessageIndex)));
-            this.logBufferContent();
-            // Keep processing the new data until stopped
-            if (this.stopLogging) {
-                break;
+        let previousChunkRemainder;
+        try {
+            for await (const chunk of logStream) {
+                // Handle possible leftover incomplete line from previous chunk.
+                // Everything before last end of line is complete.
+                previousChunkRemainder ??= new Uint8Array();
+                const chunkWithPreviousRemainder = new Uint8Array(previousChunkRemainder.length + chunk.length);
+                chunkWithPreviousRemainder.set(previousChunkRemainder, 0);
+                chunkWithPreviousRemainder.set(chunk, previousChunkRemainder.length);
+                const lastCompleteMessageIndex = chunkWithPreviousRemainder.lastIndexOf(0x0a);
+                previousChunkRemainder = chunkWithPreviousRemainder.slice(lastCompleteMessageIndex);
+                // Push complete part of the chunk to the buffer
+                this.#streamBuffer.push(chunkWithPreviousRemainder.slice(0, lastCompleteMessageIndex));
+                this.#logBufferContent();
+                // Keep processing the new data until stopped
+                if (this.#stopLogging) {
+                    break;
+                }
             }
         }
+        catch (err) {
+            // An aborted stream still hands back its unterminated last line, so the caller can flush it.
+            if (!signal.aborted)
+                throw err;
+        }
         return previousChunkRemainder;
     }
     /**
      * Parse the buffer and log complete messages.
      */
-    logBufferContent() {
-        const allParts = Buffer.concat(this.streamBuffer).toString().split(this.splitMarker).slice(1);
+    #logBufferContent() {
+        const allParts = this.#decoder.decode(concatBytes(this.#streamBuffer)).split(this.#splitMarker).slice(1);
         // Parse the buffer parts into complete messages
         const messageMarkers = allParts.filter((_, i) => i % 2 === 0);
         const messageContents = allParts.filter((_, i) => i % 2 !== 0);
-        this.streamBuffer = [];
+        this.#streamBuffer = [];
         messageMarkers.forEach((marker, index) => {
             const decodedMarker = marker;
             const decodedContent = messageContents[index];
-            if (this.relevancyTimeLimit) {
+            if (this.#relevancyTimeLimit) {
                 // Log only relevant messages. Ignore too old log messages.
                 const logTime = new Date(decodedMarker);
-                if (logTime < this.relevancyTimeLimit) {
+                if (logTime < this.#relevancyTimeLimit) {
                     return;
                 }
             }
             const message = decodedMarker + decodedContent;
             // Original log level information is not available. Log all on info level. Log level could be guessed for
             // some logs, but for any multiline logs such guess would be probably correct only for the first line.
-            this.destinationLog.info(message.trim());
+            this.#destinationLog.info(message.trim());
         });
     }
 }