apify-client 3.0.0-beta.5 → 3.0.0-beta.51
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +13 -3
- package/dist/apify_api_error.d.ts +62 -21
- package/dist/apify_api_error.js +96 -23
- package/dist/apify_client.d.ts +125 -10
- package/dist/apify_client.js +133 -48
- package/dist/base/api_client.d.ts +7 -7
- package/dist/base/api_client.js +36 -24
- package/dist/base/resource_client.d.ts +21 -8
- package/dist/base/resource_client.js +54 -26
- package/dist/base/resource_collection_client.d.ts +12 -4
- package/dist/base/resource_collection_client.js +34 -11
- package/dist/body_parser.d.ts +3 -4
- package/dist/body_parser.js +19 -26
- package/dist/bundle.js +27 -22
- package/dist/bundle.js.map +1 -1
- package/dist/generated/api.d.ts +610 -112
- package/dist/generated/schemas.d.ts +726 -703
- package/dist/generated/schemas.js +92 -85
- package/dist/http_clients/axios.d.ts +60 -0
- package/dist/http_clients/axios.js +117 -0
- package/dist/http_clients/base.d.ts +333 -0
- package/dist/http_clients/base.js +563 -0
- package/dist/http_clients/index.d.ts +4 -0
- package/dist/http_clients/index.js +2 -0
- package/dist/http_compressors/base.d.ts +40 -0
- package/dist/http_compressors/base.js +9 -0
- package/dist/http_compressors/brotli.d.ts +33 -0
- package/dist/http_compressors/brotli.js +39 -0
- package/dist/http_compressors/gzip.d.ts +33 -0
- package/dist/http_compressors/gzip.js +39 -0
- package/dist/http_compressors/index.d.ts +6 -0
- package/dist/http_compressors/index.js +2 -0
- package/dist/http_compressors/resolve.d.ts +23 -0
- package/dist/http_compressors/resolve.js +27 -0
- package/dist/index.d.ts +4 -1
- package/dist/index.js +3 -1
- package/dist/invalid_response_body_error.d.ts +13 -0
- package/dist/invalid_response_body_error.js +17 -0
- package/dist/models.d.ts +19 -22
- package/dist/resource_clients/actor.d.ts +72 -32
- package/dist/resource_clients/actor.js +87 -56
- package/dist/resource_clients/actor_collection.d.ts +9 -5
- package/dist/resource_clients/actor_collection.js +11 -3
- package/dist/resource_clients/actor_env_var.d.ts +10 -3
- package/dist/resource_clients/actor_env_var.js +16 -6
- package/dist/resource_clients/actor_env_var_collection.d.ts +14 -18
- package/dist/resource_clients/actor_env_var_collection.js +17 -11
- package/dist/resource_clients/actor_version.d.ts +18 -3
- package/dist/resource_clients/actor_version.js +19 -9
- package/dist/resource_clients/actor_version_collection.d.ts +16 -17
- package/dist/resource_clients/actor_version_collection.js +19 -11
- package/dist/resource_clients/build.d.ts +18 -6
- package/dist/resource_clients/build.js +34 -15
- package/dist/resource_clients/build_collection.d.ts +3 -1
- package/dist/resource_clients/build_collection.js +4 -1
- package/dist/resource_clients/dataset.d.ts +51 -33
- package/dist/resource_clients/dataset.js +105 -80
- package/dist/resource_clients/dataset_collection.d.ts +6 -2
- package/dist/resource_clients/dataset_collection.js +13 -2
- package/dist/resource_clients/key_value_store.d.ts +58 -16
- package/dist/resource_clients/key_value_store.js +78 -40
- package/dist/resource_clients/key_value_store_collection.d.ts +6 -2
- package/dist/resource_clients/key_value_store_collection.js +13 -2
- package/dist/resource_clients/log.d.ts +12 -20
- package/dist/resource_clients/log.js +93 -58
- package/dist/resource_clients/request_queue.d.ts +56 -26
- package/dist/resource_clients/request_queue.js +176 -103
- package/dist/resource_clients/request_queue_collection.d.ts +6 -2
- package/dist/resource_clients/request_queue_collection.js +9 -3
- package/dist/resource_clients/run.d.ts +108 -18
- package/dist/resource_clients/run.js +183 -43
- package/dist/resource_clients/run_collection.d.ts +3 -1
- package/dist/resource_clients/run_collection.js +4 -1
- package/dist/resource_clients/schedule.d.ts +14 -5
- package/dist/resource_clients/schedule.js +29 -21
- package/dist/resource_clients/schedule_collection.d.ts +6 -2
- package/dist/resource_clients/schedule_collection.js +9 -3
- package/dist/resource_clients/store_collection.d.ts +3 -1
- package/dist/resource_clients/store_collection.js +4 -1
- package/dist/resource_clients/task.d.ts +33 -10
- package/dist/resource_clients/task.js +79 -46
- package/dist/resource_clients/task_collection.d.ts +6 -2
- package/dist/resource_clients/task_collection.js +9 -3
- package/dist/resource_clients/user.d.ts +17 -8
- package/dist/resource_clients/user.js +42 -38
- package/dist/resource_clients/webhook.d.ts +14 -5
- package/dist/resource_clients/webhook.js +30 -22
- package/dist/resource_clients/webhook_collection.d.ts +6 -2
- package/dist/resource_clients/webhook_collection.js +9 -3
- package/dist/resource_clients/webhook_dispatch.d.ts +4 -1
- package/dist/resource_clients/webhook_dispatch.js +7 -2
- package/dist/resource_clients/webhook_dispatch_collection.d.ts +3 -1
- package/dist/resource_clients/webhook_dispatch_collection.js +4 -1
- package/dist/runtime/node.d.ts +6 -0
- package/dist/runtime/node.js +49 -0
- package/dist/runtime/types.d.ts +52 -0
- package/dist/runtime/types.js +1 -0
- package/dist/runtime/web.d.ts +5 -0
- package/dist/runtime/web.js +18 -0
- package/dist/schemas.d.ts +1 -1
- package/dist/schemas.js +1 -1
- package/dist/statistics.d.ts +1 -6
- package/dist/statistics.js +2 -2
- package/dist/timeouts.d.ts +84 -0
- package/dist/timeouts.js +37 -0
- package/dist/timezones.d.ts +1 -1
- package/dist/utils.d.ts +81 -52
- package/dist/utils.js +201 -170
- package/dist/wait_for_resources.d.ts +8 -0
- package/dist/wait_for_resources.js +39 -0
- package/package.json +32 -29
- package/dist/http_client.d.ts +0 -88
- package/dist/http_client.js +0 -255
- package/dist/interceptors.d.ts +0 -19
- package/dist/interceptors.js +0 -105
|
@@ -3,6 +3,7 @@ import type { Log } from '@apify/log';
|
|
|
3
3
|
import { Logger, LogLevel } from '@apify/log';
|
|
4
4
|
import type { ApiClientSubResourceOptions } from '../base/api_client.js';
|
|
5
5
|
import { ResourceClient } from '../base/resource_client.js';
|
|
6
|
+
import type { TimeoutOptions } from '../timeouts.js';
|
|
6
7
|
/**
|
|
7
8
|
* Client for accessing Actor run or build logs.
|
|
8
9
|
*
|
|
@@ -35,7 +36,9 @@ export declare class LogClient extends ResourceClient {
|
|
|
35
36
|
*
|
|
36
37
|
* @param options - Log retrieval options.
|
|
37
38
|
* @param options.raw - If `true`, returns raw log content without any processing. Default is `false`.
|
|
38
|
-
* @
|
|
39
|
+
* @param options.timeoutSecs - Timeout for the API request. Default is `'long'`.
|
|
40
|
+
* @returns The log content as a string, or `undefined` if it does not exist. A chained client such as
|
|
41
|
+
* `run.log()` throws an `ApifyApiError` on a 404, since the run itself may be what is missing.
|
|
39
42
|
* @see https://docs.apify.com/api/v2/log-get
|
|
40
43
|
*/
|
|
41
44
|
get(options?: LogOptions): Promise<string | undefined>;
|
|
@@ -44,7 +47,9 @@ export declare class LogClient extends ResourceClient {
|
|
|
44
47
|
*
|
|
45
48
|
* @param options - Log retrieval options.
|
|
46
49
|
* @param options.raw - If `true`, returns raw log content without any processing. Default is `false`.
|
|
47
|
-
* @
|
|
50
|
+
* @param options.timeoutSecs - Timeout for the API request. Default is `'long'`.
|
|
51
|
+
* @returns The log content as a Readable stream, or `undefined` if it does not exist. A chained client such as
|
|
52
|
+
* `run.log()` throws an `ApifyApiError` on a 404, since the run itself may be what is missing.
|
|
48
53
|
* @see https://docs.apify.com/api/v2/log-get
|
|
49
54
|
*/
|
|
50
55
|
stream(options?: LogOptions): Promise<Readable | undefined>;
|
|
@@ -52,7 +57,7 @@ export declare class LogClient extends ResourceClient {
|
|
|
52
57
|
/**
|
|
53
58
|
* @since Added in 2.20.0
|
|
54
59
|
*/
|
|
55
|
-
export interface LogOptions {
|
|
60
|
+
export interface LogOptions extends TimeoutOptions {
|
|
56
61
|
/** @default false */
|
|
57
62
|
raw?: boolean;
|
|
58
63
|
}
|
|
@@ -69,31 +74,16 @@ export declare class LoggerActorRedirect extends Logger {
|
|
|
69
74
|
* @since Added in 2.20.0
|
|
70
75
|
*/
|
|
71
76
|
export declare class StreamedLog {
|
|
72
|
-
private
|
|
73
|
-
private streamBuffer;
|
|
74
|
-
private splitMarker;
|
|
75
|
-
private relevancyTimeLimit;
|
|
76
|
-
private logClient;
|
|
77
|
-
private streamingTask;
|
|
78
|
-
private stopLogging;
|
|
77
|
+
#private;
|
|
79
78
|
constructor(options: StreamedLogOptions);
|
|
80
79
|
/**
|
|
81
80
|
* Start log redirection.
|
|
82
81
|
*/
|
|
83
82
|
start(): void;
|
|
84
83
|
/**
|
|
85
|
-
* Stop log redirection.
|
|
84
|
+
* Stop log redirection. Waits up to one second for lines already in flight, then aborts the log stream request.
|
|
86
85
|
*/
|
|
87
86
|
stop(): Promise<void>;
|
|
88
|
-
/**
|
|
89
|
-
* Get log stream from response and redirect it to another log.
|
|
90
|
-
*/
|
|
91
|
-
private streamLog;
|
|
92
|
-
private logStreamChunks;
|
|
93
|
-
/**
|
|
94
|
-
* Parse the buffer and log complete messages.
|
|
95
|
-
*/
|
|
96
|
-
private logBufferContent;
|
|
97
87
|
}
|
|
98
88
|
/**
|
|
99
89
|
* @since Added in 2.20.0
|
|
@@ -105,4 +95,6 @@ export interface StreamedLogOptions {
|
|
|
105
95
|
toLog: Log;
|
|
106
96
|
/** Whether to redirect all logs from Actor run start (even logs from the past). */
|
|
107
97
|
fromStart?: boolean;
|
|
98
|
+
/** Ends the log stream once it aborts. */
|
|
99
|
+
signal?: AbortSignal;
|
|
108
100
|
}
|
|
@@ -1,7 +1,12 @@
|
|
|
1
1
|
import c from 'ansi-colors';
|
|
2
|
+
import { z } from 'zod';
|
|
2
3
|
import log, { Logger, LogLevel } from '@apify/log';
|
|
3
4
|
import { ResourceClient } from '../base/resource_client.js';
|
|
4
|
-
import {
|
|
5
|
+
import { timeoutOptionsShape } from '../timeouts.js';
|
|
6
|
+
import { cast, catchNotFoundForResourceOrThrow, concatBytes, parseArgument } from '../utils.js';
|
|
7
|
+
const logOptionsSchema = z.strictObject({ raw: z.boolean().optional(), ...timeoutOptionsShape });
|
|
8
|
+
/** How long `StreamedLog.stop()` waits for lines already in flight before aborting the log stream request. */
|
|
9
|
+
const STOP_GRACE_MILLIS = 1000;
|
|
5
10
|
/**
|
|
6
11
|
* Client for accessing Actor run or build logs.
|
|
7
12
|
*
|
|
@@ -39,21 +44,26 @@ export class LogClient extends ResourceClient {
|
|
|
39
44
|
*
|
|
40
45
|
* @param options - Log retrieval options.
|
|
41
46
|
* @param options.raw - If `true`, returns raw log content without any processing. Default is `false`.
|
|
42
|
-
* @
|
|
47
|
+
* @param options.timeoutSecs - Timeout for the API request. Default is `'long'`.
|
|
48
|
+
* @returns The log content as a string, or `undefined` if it does not exist. A chained client such as
|
|
49
|
+
* `run.log()` throws an `ApifyApiError` on a 404, since the run itself may be what is missing.
|
|
43
50
|
* @see https://docs.apify.com/api/v2/log-get
|
|
44
51
|
*/
|
|
45
52
|
async get(options = {}) {
|
|
53
|
+
const { timeoutSecs = 'long', signal, ...params } = parseArgument(options, logOptionsSchema, 'LogOptions');
|
|
46
54
|
const requestOpts = {
|
|
47
|
-
url: this.
|
|
55
|
+
url: this.buildUrl(),
|
|
48
56
|
method: 'GET',
|
|
49
|
-
params: this.
|
|
57
|
+
params: this.buildParams(params),
|
|
58
|
+
timeoutSecs,
|
|
59
|
+
signal,
|
|
50
60
|
};
|
|
51
61
|
try {
|
|
52
62
|
const response = await this.httpClient.call(requestOpts);
|
|
53
63
|
return cast(response.data);
|
|
54
64
|
}
|
|
55
65
|
catch (err) {
|
|
56
|
-
|
|
66
|
+
catchNotFoundForResourceOrThrow(err, this.id);
|
|
57
67
|
}
|
|
58
68
|
return undefined;
|
|
59
69
|
}
|
|
@@ -62,26 +72,31 @@ export class LogClient extends ResourceClient {
|
|
|
62
72
|
*
|
|
63
73
|
* @param options - Log retrieval options.
|
|
64
74
|
* @param options.raw - If `true`, returns raw log content without any processing. Default is `false`.
|
|
65
|
-
* @
|
|
75
|
+
* @param options.timeoutSecs - Timeout for the API request. Default is `'long'`.
|
|
76
|
+
* @returns The log content as a Readable stream, or `undefined` if it does not exist. A chained client such as
|
|
77
|
+
* `run.log()` throws an `ApifyApiError` on a 404, since the run itself may be what is missing.
|
|
66
78
|
* @see https://docs.apify.com/api/v2/log-get
|
|
67
79
|
*/
|
|
68
80
|
async stream(options = {}) {
|
|
81
|
+
const { timeoutSecs = 'long', signal, raw } = parseArgument(options, logOptionsSchema, 'LogOptions');
|
|
69
82
|
const params = {
|
|
70
83
|
stream: true,
|
|
71
|
-
raw
|
|
84
|
+
raw,
|
|
72
85
|
};
|
|
73
86
|
const requestOpts = {
|
|
74
|
-
url: this.
|
|
87
|
+
url: this.buildUrl(),
|
|
75
88
|
method: 'GET',
|
|
76
|
-
params: this.
|
|
89
|
+
params: this.buildParams(params),
|
|
77
90
|
responseType: 'stream',
|
|
91
|
+
timeoutSecs,
|
|
92
|
+
signal,
|
|
78
93
|
};
|
|
79
94
|
try {
|
|
80
95
|
const response = await this.httpClient.call(requestOpts);
|
|
81
96
|
return cast(response.data);
|
|
82
97
|
}
|
|
83
98
|
catch (err) {
|
|
84
|
-
|
|
99
|
+
catchNotFoundForResourceOrThrow(err, this.id);
|
|
85
100
|
}
|
|
86
101
|
return undefined;
|
|
87
102
|
}
|
|
@@ -119,39 +134,46 @@ export class LoggerActorRedirect extends Logger {
|
|
|
119
134
|
* @since Added in 2.20.0
|
|
120
135
|
*/
|
|
121
136
|
export class StreamedLog {
|
|
122
|
-
destinationLog;
|
|
123
|
-
streamBuffer = [];
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
137
|
+
#destinationLog;
|
|
138
|
+
#streamBuffer = [];
|
|
139
|
+
#decoder = new TextDecoder();
|
|
140
|
+
#splitMarker = /(?:\n|^)(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z)/g;
|
|
141
|
+
#relevancyTimeLimit;
|
|
142
|
+
#logClient;
|
|
143
|
+
#signal;
|
|
144
|
+
#streamingTask = null;
|
|
145
|
+
#stopLogging = false;
|
|
146
|
+
#stopController = new AbortController();
|
|
129
147
|
constructor(options) {
|
|
130
|
-
const { toLog, logClient, fromStart = true } = options;
|
|
131
|
-
this
|
|
132
|
-
this
|
|
133
|
-
this
|
|
148
|
+
const { toLog, logClient, fromStart = true, signal } = options;
|
|
149
|
+
this.#destinationLog = toLog;
|
|
150
|
+
this.#logClient = logClient;
|
|
151
|
+
this.#signal = signal;
|
|
152
|
+
this.#relevancyTimeLimit = fromStart ? null : new Date();
|
|
134
153
|
}
|
|
135
154
|
/**
|
|
136
155
|
* Start log redirection.
|
|
137
156
|
*/
|
|
138
157
|
start() {
|
|
139
|
-
if (this
|
|
158
|
+
if (this.#streamingTask) {
|
|
140
159
|
throw new Error('Streaming task already active');
|
|
141
160
|
}
|
|
142
|
-
this
|
|
143
|
-
this
|
|
161
|
+
this.#stopLogging = false;
|
|
162
|
+
this.#stopController = new AbortController();
|
|
163
|
+
this.#streamingTask = this.#streamLog();
|
|
144
164
|
}
|
|
145
165
|
/**
|
|
146
|
-
* Stop log redirection.
|
|
166
|
+
* Stop log redirection. Waits up to one second for lines already in flight, then aborts the log stream request.
|
|
147
167
|
*/
|
|
148
168
|
async stop() {
|
|
149
|
-
if (!this
|
|
169
|
+
if (!this.#streamingTask) {
|
|
150
170
|
throw new Error('Streaming task is not active');
|
|
151
171
|
}
|
|
152
|
-
this
|
|
172
|
+
this.#stopLogging = true;
|
|
173
|
+
const stopController = this.#stopController;
|
|
174
|
+
const abortTimeout = setTimeout(() => stopController.abort(), STOP_GRACE_MILLIS);
|
|
153
175
|
try {
|
|
154
|
-
await this
|
|
176
|
+
await this.#streamingTask;
|
|
155
177
|
}
|
|
156
178
|
catch (err) {
|
|
157
179
|
if (!(err instanceof Error && err.name === 'AbortError')) {
|
|
@@ -159,73 +181,86 @@ export class StreamedLog {
|
|
|
159
181
|
}
|
|
160
182
|
}
|
|
161
183
|
finally {
|
|
162
|
-
|
|
184
|
+
clearTimeout(abortTimeout);
|
|
185
|
+
this.#streamingTask = null;
|
|
163
186
|
}
|
|
164
187
|
}
|
|
165
188
|
/**
|
|
166
189
|
* Get log stream from response and redirect it to another log.
|
|
167
190
|
*/
|
|
168
|
-
async streamLog() {
|
|
169
|
-
const
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
}
|
|
191
|
+
async #streamLog() {
|
|
192
|
+
const signal = this.#signal
|
|
193
|
+
? AbortSignal.any([this.#signal, this.#stopController.signal])
|
|
194
|
+
: this.#stopController.signal;
|
|
173
195
|
try {
|
|
174
|
-
const
|
|
196
|
+
const logStream = await this.#logClient.stream({ raw: true, signal });
|
|
197
|
+
if (!logStream) {
|
|
198
|
+
return;
|
|
199
|
+
}
|
|
200
|
+
const lastChunkRemainder = await this.#logStreamChunks(logStream, signal);
|
|
175
201
|
// Process whatever is left when exiting. Maybe it is incomplete, maybe it is last log without EOL.
|
|
176
|
-
const lastMessage =
|
|
202
|
+
const lastMessage = this.#decoder.decode(lastChunkRemainder).trim();
|
|
177
203
|
if (lastMessage.length) {
|
|
178
|
-
this
|
|
204
|
+
this.#destinationLog.info(lastMessage);
|
|
179
205
|
}
|
|
180
206
|
}
|
|
181
207
|
catch (err) {
|
|
208
|
+
if (signal.aborted)
|
|
209
|
+
return;
|
|
182
210
|
log.warning(`Log redirection stopped due to error`, err);
|
|
183
211
|
}
|
|
184
212
|
}
|
|
185
|
-
async logStreamChunks(logStream) {
|
|
213
|
+
async #logStreamChunks(logStream, signal) {
|
|
186
214
|
// Chunk may be incomplete. Keep remainder for next chunk.
|
|
187
215
|
let previousChunkRemainder = new Uint8Array();
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
216
|
+
try {
|
|
217
|
+
for await (const chunk of logStream) {
|
|
218
|
+
// Handle possible leftover incomplete line from previous chunk.
|
|
219
|
+
// Everything before last end of line is complete.
|
|
220
|
+
const chunkWithPreviousRemainder = new Uint8Array(previousChunkRemainder.length + chunk.length);
|
|
221
|
+
chunkWithPreviousRemainder.set(previousChunkRemainder, 0);
|
|
222
|
+
chunkWithPreviousRemainder.set(chunk, previousChunkRemainder.length);
|
|
223
|
+
const lastCompleteMessageIndex = chunkWithPreviousRemainder.lastIndexOf(0x0a);
|
|
224
|
+
previousChunkRemainder = chunkWithPreviousRemainder.slice(lastCompleteMessageIndex);
|
|
225
|
+
// Push complete part of the chunk to the buffer
|
|
226
|
+
this.#streamBuffer.push(chunkWithPreviousRemainder.slice(0, lastCompleteMessageIndex));
|
|
227
|
+
this.#logBufferContent();
|
|
228
|
+
// Keep processing the new data until stopped
|
|
229
|
+
if (this.#stopLogging) {
|
|
230
|
+
break;
|
|
231
|
+
}
|
|
202
232
|
}
|
|
203
233
|
}
|
|
234
|
+
catch (err) {
|
|
235
|
+
// An aborted stream still hands back its unterminated last line, so the caller can flush it.
|
|
236
|
+
if (!signal.aborted)
|
|
237
|
+
throw err;
|
|
238
|
+
}
|
|
204
239
|
return previousChunkRemainder;
|
|
205
240
|
}
|
|
206
241
|
/**
|
|
207
242
|
* Parse the buffer and log complete messages.
|
|
208
243
|
*/
|
|
209
|
-
logBufferContent() {
|
|
210
|
-
const allParts =
|
|
244
|
+
#logBufferContent() {
|
|
245
|
+
const allParts = this.#decoder.decode(concatBytes(this.#streamBuffer)).split(this.#splitMarker).slice(1);
|
|
211
246
|
// Parse the buffer parts into complete messages
|
|
212
247
|
const messageMarkers = allParts.filter((_, i) => i % 2 === 0);
|
|
213
248
|
const messageContents = allParts.filter((_, i) => i % 2 !== 0);
|
|
214
|
-
this
|
|
249
|
+
this.#streamBuffer = [];
|
|
215
250
|
messageMarkers.forEach((marker, index) => {
|
|
216
251
|
const decodedMarker = marker;
|
|
217
252
|
const decodedContent = messageContents[index];
|
|
218
|
-
if (this
|
|
253
|
+
if (this.#relevancyTimeLimit) {
|
|
219
254
|
// Log only relevant messages. Ignore too old log messages.
|
|
220
255
|
const logTime = new Date(decodedMarker);
|
|
221
|
-
if (logTime < this
|
|
256
|
+
if (logTime < this.#relevancyTimeLimit) {
|
|
222
257
|
return;
|
|
223
258
|
}
|
|
224
259
|
}
|
|
225
260
|
const message = decodedMarker + decodedContent;
|
|
226
261
|
// Original log level information is not available. Log all on info level. Log level could be guessed for
|
|
227
262
|
// some logs, but for any multiline logs such guess would be probably correct only for the first line.
|
|
228
|
-
this
|
|
263
|
+
this.#destinationLog.info(message.trim());
|
|
229
264
|
});
|
|
230
265
|
}
|
|
231
266
|
}
|
|
@@ -2,6 +2,16 @@ import type { STORAGE_GENERAL_ACCESS } from '@apify/consts';
|
|
|
2
2
|
import type { ApiClientSubResourceOptions } from '../base/api_client.js';
|
|
3
3
|
import { ResourceClient } from '../base/resource_client.js';
|
|
4
4
|
import type { RequestQueue, RequestQueueClientAddRequestResult, RequestQueueClientBatchDeleteRequestsResult, RequestQueueClientBatchRequestsOperationResult, RequestQueueClientListAndLockHeadResult, RequestQueueClientListHeadResult, RequestQueueClientListRequestsResult, RequestQueueClientProlongRequestLockResult, RequestQueueClientRequestSchema, RequestQueueClientRequestToAdd, RequestQueueClientRequestToUpdate, RequestQueueClientUnlockRequestsResult } from '../models.js';
|
|
5
|
+
import type { TimeoutOptions } from '../timeouts.js';
|
|
6
|
+
/**
|
|
7
|
+
* A request to add, serialized once up front: `byteLength` decides which batch it goes into, `json` is what the body of
|
|
8
|
+
* that batch is assembled from, and `request` is what the result bookkeeping needs.
|
|
9
|
+
*/
|
|
10
|
+
interface SerializedRequestToAdd {
|
|
11
|
+
request: RequestQueueClientRequestToAdd;
|
|
12
|
+
json: string;
|
|
13
|
+
byteLength: number;
|
|
14
|
+
}
|
|
5
15
|
export type { AllowedHttpMethods, RequestQueue, RequestQueueClientAddRequestResult, RequestQueueClientBatchDeleteRequestsResult, RequestQueueClientBatchRequestsOperationResult, RequestQueueClientListAndLockHeadResult, RequestQueueClientListHeadResult, RequestQueueClientListItem, RequestQueueClientListRequestsResult, RequestQueueClientLockedListItem, RequestQueueClientProlongRequestLockResult, RequestQueueClientRequestSchema, RequestQueueClientRequestToAdd, RequestQueueClientRequestToUpdate, RequestQueueClientUnlockRequestsResult, RequestQueueStats, } from '../models.js';
|
|
6
16
|
/**
|
|
7
17
|
* Client for managing a specific Request queue.
|
|
@@ -34,8 +44,7 @@ export type { AllowedHttpMethods, RequestQueue, RequestQueueClientAddRequestResu
|
|
|
34
44
|
* @see https://docs.apify.com/platform/storage/request-queue
|
|
35
45
|
*/
|
|
36
46
|
export declare class RequestQueueClient extends ResourceClient {
|
|
37
|
-
private
|
|
38
|
-
private timeoutMillis?;
|
|
47
|
+
#private;
|
|
39
48
|
/**
|
|
40
49
|
* @hidden
|
|
41
50
|
*/
|
|
@@ -43,24 +52,30 @@ export declare class RequestQueueClient extends ResourceClient {
|
|
|
43
52
|
/**
|
|
44
53
|
* Gets the Request queue object from the Apify API.
|
|
45
54
|
*
|
|
55
|
+
* @param options - Request options
|
|
56
|
+
* @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
|
|
46
57
|
* @returns The RequestQueue object, or `undefined` if it does not exist
|
|
47
58
|
* @see https://docs.apify.com/api/v2/request-queue-get
|
|
48
59
|
*/
|
|
49
|
-
get(): Promise<RequestQueue | undefined>;
|
|
60
|
+
get(options?: TimeoutOptions): Promise<RequestQueue | undefined>;
|
|
50
61
|
/**
|
|
51
62
|
* Updates the Request queue with specified fields.
|
|
52
63
|
*
|
|
53
64
|
* @param newFields - Fields to update in the Request queue
|
|
65
|
+
* @param options - Request options
|
|
66
|
+
* @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
|
|
54
67
|
* @returns The updated RequestQueue object
|
|
55
68
|
* @see https://docs.apify.com/api/v2/request-queue-put
|
|
56
69
|
*/
|
|
57
|
-
update(newFields: RequestQueueClientUpdateOptions): Promise<RequestQueue>;
|
|
70
|
+
update(newFields: RequestQueueClientUpdateOptions, options?: TimeoutOptions): Promise<RequestQueue>;
|
|
58
71
|
/**
|
|
59
72
|
* Deletes the Request queue.
|
|
60
73
|
*
|
|
74
|
+
* @param options - Request options
|
|
75
|
+
* @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
|
|
61
76
|
* @see https://docs.apify.com/api/v2/request-queue-delete
|
|
62
77
|
*/
|
|
63
|
-
delete(): Promise<void>;
|
|
78
|
+
delete(options?: TimeoutOptions): Promise<void>;
|
|
64
79
|
/**
|
|
65
80
|
* Lists requests from the beginning of the queue (head).
|
|
66
81
|
*
|
|
@@ -68,6 +83,8 @@ export declare class RequestQueueClient extends ResourceClient {
|
|
|
68
83
|
* inspecting what requests are waiting to be processed.
|
|
69
84
|
*
|
|
70
85
|
* @param options - Options for listing (e.g., limit)
|
|
86
|
+
* @param options.limit - Maximum number of requests to return.
|
|
87
|
+
* @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
|
|
71
88
|
* @returns List of requests from the queue head
|
|
72
89
|
* @see https://docs.apify.com/api/v2/request-queue-head-get
|
|
73
90
|
*/
|
|
@@ -84,6 +101,7 @@ export declare class RequestQueueClient extends ResourceClient {
|
|
|
84
101
|
* @param options - Lock configuration
|
|
85
102
|
* @param options.lockSecs - **Required.** Duration in seconds to lock the requests. After this time, the locks expire and requests can be retrieved by other clients.
|
|
86
103
|
* @param options.limit - Maximum number of requests to return. Default is 25.
|
|
104
|
+
* @param options.timeoutSecs - Timeout for the API request. Default is `'medium'`.
|
|
87
105
|
* @returns Object containing `items` (locked requests), `queueModifiedAt`, `hadMultipleClients`, and lock information
|
|
88
106
|
* @see https://docs.apify.com/api/v2/request-queue-head-lock-post
|
|
89
107
|
*
|
|
@@ -122,6 +140,7 @@ export declare class RequestQueueClient extends ResourceClient {
|
|
|
122
140
|
* @param request.payload - HTTP payload for POST/PUT requests (string).
|
|
123
141
|
* @param options - Additional options
|
|
124
142
|
* @param options.forefront - If `true`, adds the request to the beginning of the queue. Default is `false` (adds to the end).
|
|
143
|
+
* @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
|
|
125
144
|
* @returns Object with `requestId`, `wasAlreadyPresent`, and `wasAlreadyHandled` flags
|
|
126
145
|
* @see https://docs.apify.com/api/v2/request-queue-requests-post
|
|
127
146
|
*
|
|
@@ -150,8 +169,8 @@ export declare class RequestQueueClient extends ResourceClient {
|
|
|
150
169
|
*
|
|
151
170
|
* @private
|
|
152
171
|
*/
|
|
153
|
-
protected
|
|
154
|
-
protected
|
|
172
|
+
protected addRequestBatch(requests: SerializedRequestToAdd[], options?: RequestQueueClientAddRequestOptions): Promise<RequestQueueClientBatchRequestsOperationResult>;
|
|
173
|
+
protected addRequestBatchWithRetries(requests: SerializedRequestToAdd[], options?: RequestQueueClientBatchAddRequestWithRetriesOptions): Promise<RequestQueueClientBatchRequestsOperationResult>;
|
|
155
174
|
/**
|
|
156
175
|
* Adds multiple requests to the queue in a single operation.
|
|
157
176
|
*
|
|
@@ -167,6 +186,7 @@ export declare class RequestQueueClient extends ResourceClient {
|
|
|
167
186
|
* @param options.maxUnprocessedRequestsRetries - Maximum number of retry attempts for rate-limited requests. Default is 3.
|
|
168
187
|
* @param options.maxParallel - Maximum number of parallel batch API calls. Default is 5.
|
|
169
188
|
* @param options.minDelayBetweenUnprocessedRequestsRetriesMillis - Minimum delay before retrying rate-limited requests. Default is 500ms.
|
|
189
|
+
* @param options.timeoutSecs - Timeout for each batch API request. Default is `'medium'`.
|
|
170
190
|
* @returns Object with `processedRequests` (successfully added) and `unprocessedRequests` (failed after all retries)
|
|
171
191
|
* @see https://docs.apify.com/api/v2/request-queue-requests-batch-post
|
|
172
192
|
*
|
|
@@ -197,24 +217,30 @@ export declare class RequestQueueClient extends ResourceClient {
|
|
|
197
217
|
* Requests can be identified by either their ID or unique key.
|
|
198
218
|
*
|
|
199
219
|
* @param requests - Array of requests to delete (by id or uniqueKey)
|
|
220
|
+
* @param options - Request options
|
|
221
|
+
* @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
|
|
200
222
|
* @returns Result containing processed and unprocessed requests
|
|
201
223
|
* @see https://docs.apify.com/api/v2/request-queue-requests-batch-delete
|
|
202
224
|
* @since Added in 2.3.0
|
|
203
225
|
*/
|
|
204
|
-
batchDeleteRequests(requests: RequestQueueClientRequestToDelete[]): Promise<RequestQueueClientBatchDeleteRequestsResult>;
|
|
226
|
+
batchDeleteRequests(requests: RequestQueueClientRequestToDelete[], options?: TimeoutOptions): Promise<RequestQueueClientBatchDeleteRequestsResult>;
|
|
205
227
|
/**
|
|
206
228
|
* Gets a specific request from the queue by its ID.
|
|
207
229
|
*
|
|
208
230
|
* @param id - Request ID
|
|
231
|
+
* @param options - Request options
|
|
232
|
+
* @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
|
|
209
233
|
* @returns The request object, or `undefined` if not found
|
|
210
234
|
* @see https://docs.apify.com/api/v2/request-queue-request-get
|
|
211
235
|
*/
|
|
212
|
-
getRequest(id: string): Promise<RequestQueueClientGetRequestResult | undefined>;
|
|
236
|
+
getRequest(id: string, options?: TimeoutOptions): Promise<RequestQueueClientGetRequestResult | undefined>;
|
|
213
237
|
/**
|
|
214
238
|
* Updates a request in the queue.
|
|
215
239
|
*
|
|
216
240
|
* @param request - The updated request object (must include id)
|
|
217
241
|
* @param options - Update options such as whether to move to front
|
|
242
|
+
* @param options.forefront - If `true`, moves the request to the beginning of the queue. Default is `false`.
|
|
243
|
+
* @param options.timeoutSecs - Timeout for the API request. Default is `'medium'`.
|
|
218
244
|
* @returns Information about the updated request
|
|
219
245
|
* @see https://docs.apify.com/api/v2/request-queue-request-put
|
|
220
246
|
*/
|
|
@@ -223,8 +249,10 @@ export declare class RequestQueueClient extends ResourceClient {
|
|
|
223
249
|
* Deletes a specific request from the queue.
|
|
224
250
|
*
|
|
225
251
|
* @param id - Request ID
|
|
252
|
+
* @param options - Request options
|
|
253
|
+
* @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
|
|
226
254
|
*/
|
|
227
|
-
deleteRequest(id: string): Promise<void>;
|
|
255
|
+
deleteRequest(id: string, options?: TimeoutOptions): Promise<void>;
|
|
228
256
|
/**
|
|
229
257
|
* Prolongs the lock on a request to prevent it from being returned to other clients.
|
|
230
258
|
*
|
|
@@ -236,6 +264,7 @@ export declare class RequestQueueClient extends ResourceClient {
|
|
|
236
264
|
* @param options - Lock extension options
|
|
237
265
|
* @param options.lockSecs - **Required.** New lock duration in seconds from now.
|
|
238
266
|
* @param options.forefront - If `true`, moves the request to the beginning of the queue when the lock expires. Default is `false`.
|
|
267
|
+
* @param options.timeoutSecs - Timeout for the API request. Default is `'medium'`.
|
|
239
268
|
* @returns Object with new `lockExpiresAt` timestamp
|
|
240
269
|
* @see https://docs.apify.com/api/v2/request-queue-request-lock-put
|
|
241
270
|
*
|
|
@@ -259,6 +288,8 @@ export declare class RequestQueueClient extends ResourceClient {
|
|
|
259
288
|
*
|
|
260
289
|
* @param id - Request ID
|
|
261
290
|
* @param options - Options such as whether to move to front
|
|
291
|
+
* @param options.forefront - If `true`, moves the request to the beginning of the queue. Default is `false`.
|
|
292
|
+
* @param options.timeoutSecs - Timeout for the API request. Default is `'short'`.
|
|
262
293
|
* @see https://docs.apify.com/api/v2/request-queue-request-lock-delete
|
|
263
294
|
* @since Added in 2.4.1
|
|
264
295
|
*/
|
|
@@ -270,6 +301,7 @@ export declare class RequestQueueClient extends ResourceClient {
|
|
|
270
301
|
* queue contents.
|
|
271
302
|
*
|
|
272
303
|
* @param options - Pagination options
|
|
304
|
+
* @param options.timeoutSecs - Timeout for each API request. Default is `'medium'`.
|
|
273
305
|
* @returns List of requests with pagination information
|
|
274
306
|
* @see https://docs.apify.com/api/v2/request-queue-requests-get
|
|
275
307
|
* @since Added in 2.5.1
|
|
@@ -281,11 +313,13 @@ export declare class RequestQueueClient extends ResourceClient {
|
|
|
281
313
|
* This is useful for releasing all locks at once, for example when shutting down
|
|
282
314
|
* a crawler gracefully.
|
|
283
315
|
*
|
|
316
|
+
* @param options - Request options
|
|
317
|
+
* @param options.timeoutSecs - Timeout for the API request. Default is `'long'`.
|
|
284
318
|
* @returns Number of requests that were unlocked
|
|
285
319
|
* @see https://docs.apify.com/api/v2/request-queue-requests-unlock-post
|
|
286
320
|
* @since Added in 2.12.5
|
|
287
321
|
*/
|
|
288
|
-
unlockRequests(): Promise<RequestQueueClientUnlockRequestsResult>;
|
|
322
|
+
unlockRequests(options?: TimeoutOptions): Promise<RequestQueueClientUnlockRequestsResult>;
|
|
289
323
|
/**
|
|
290
324
|
* Returns an async iterable for paginating through all requests in the queue.
|
|
291
325
|
*
|
|
@@ -293,6 +327,7 @@ export declare class RequestQueueClient extends ResourceClient {
|
|
|
293
327
|
* automatically handling pagination behind the scenes.
|
|
294
328
|
*
|
|
295
329
|
* @param options - Pagination options
|
|
330
|
+
* @param options.timeoutSecs - Timeout for each API request. Default is `'medium'`.
|
|
296
331
|
* @returns An async iterable of request pages
|
|
297
332
|
* @see https://docs.apify.com/api/v2/request-queue-requests-get
|
|
298
333
|
*
|
|
@@ -312,6 +347,8 @@ export declare class RequestQueueClient extends ResourceClient {
|
|
|
312
347
|
export interface RequestQueueUserOptions {
|
|
313
348
|
clientKey?: string;
|
|
314
349
|
/**
|
|
350
|
+
* Cap, in seconds, on the default timeout tier of every request this client sends. An explicit per-call
|
|
351
|
+
* `timeoutSecs` is not capped.
|
|
315
352
|
* @since Added in 2.2.0
|
|
316
353
|
*/
|
|
317
354
|
timeoutSecs?: number;
|
|
@@ -333,7 +370,7 @@ export interface RequestQueueClientUpdateOptions {
|
|
|
333
370
|
/**
|
|
334
371
|
* Options for listing requests from the queue head.
|
|
335
372
|
*/
|
|
336
|
-
export interface RequestQueueClientListHeadOptions {
|
|
373
|
+
export interface RequestQueueClientListHeadOptions extends TimeoutOptions {
|
|
337
374
|
limit?: number;
|
|
338
375
|
}
|
|
339
376
|
/**
|
|
@@ -344,13 +381,8 @@ export type RequestQueueListRequestsFilter = 'locked' | 'pending';
|
|
|
344
381
|
* Options for listing all requests in the queue.
|
|
345
382
|
* @since Added in 2.5.1
|
|
346
383
|
*/
|
|
347
|
-
export interface RequestQueueClientListRequestsOptions {
|
|
384
|
+
export interface RequestQueueClientListRequestsOptions extends TimeoutOptions {
|
|
348
385
|
limit?: number;
|
|
349
|
-
/**
|
|
350
|
-
* Using id of request that does not exist in request queue leads to unpredictable results.
|
|
351
|
-
* @deprecated Use `cursor` for pagination instead.
|
|
352
|
-
*/
|
|
353
|
-
exclusiveStartId?: string;
|
|
354
386
|
/**
|
|
355
387
|
* @since Added in 2.23.2
|
|
356
388
|
*/
|
|
@@ -364,11 +396,9 @@ export interface RequestQueueClientListRequestsOptions {
|
|
|
364
396
|
* Options for paginating through requests in the queue.
|
|
365
397
|
* @since Added in 2.5.1
|
|
366
398
|
*/
|
|
367
|
-
export interface RequestQueueClientPaginateRequestsOptions {
|
|
399
|
+
export interface RequestQueueClientPaginateRequestsOptions extends TimeoutOptions {
|
|
368
400
|
limit?: number;
|
|
369
401
|
maxPageLimit?: number;
|
|
370
|
-
/** @deprecated Use `cursor` for pagination instead. */
|
|
371
|
-
exclusiveStartId?: string;
|
|
372
402
|
/**
|
|
373
403
|
* @since Added in 2.23.2
|
|
374
404
|
*/
|
|
@@ -382,30 +412,30 @@ export interface RequestQueueClientPaginateRequestsOptions {
|
|
|
382
412
|
* Options for listing and locking requests from the queue head.
|
|
383
413
|
* @since Added in 2.4.1
|
|
384
414
|
*/
|
|
385
|
-
export interface RequestQueueClientListAndLockHeadOptions {
|
|
415
|
+
export interface RequestQueueClientListAndLockHeadOptions extends TimeoutOptions {
|
|
386
416
|
lockSecs: number;
|
|
387
417
|
limit?: number;
|
|
388
418
|
}
|
|
389
|
-
export interface RequestQueueClientAddRequestOptions {
|
|
419
|
+
export interface RequestQueueClientAddRequestOptions extends TimeoutOptions {
|
|
390
420
|
forefront?: boolean;
|
|
391
421
|
}
|
|
392
422
|
/**
|
|
393
423
|
* @since Added in 2.4.1
|
|
394
424
|
*/
|
|
395
|
-
export interface RequestQueueClientProlongRequestLockOptions {
|
|
425
|
+
export interface RequestQueueClientProlongRequestLockOptions extends TimeoutOptions {
|
|
396
426
|
forefront?: boolean;
|
|
397
427
|
lockSecs: number;
|
|
398
428
|
}
|
|
399
429
|
/**
|
|
400
430
|
* @since Added in 2.4.1
|
|
401
431
|
*/
|
|
402
|
-
export interface RequestQueueClientDeleteRequestLockOptions {
|
|
432
|
+
export interface RequestQueueClientDeleteRequestLockOptions extends TimeoutOptions {
|
|
403
433
|
forefront?: boolean;
|
|
404
434
|
}
|
|
405
435
|
/**
|
|
406
436
|
* @since Added in 2.3.0
|
|
407
437
|
*/
|
|
408
|
-
export interface RequestQueueClientBatchAddRequestWithRetriesOptions {
|
|
438
|
+
export interface RequestQueueClientBatchAddRequestWithRetriesOptions extends TimeoutOptions {
|
|
409
439
|
forefront?: boolean;
|
|
410
440
|
maxUnprocessedRequestsRetries?: number;
|
|
411
441
|
maxParallel?: number;
|