apify 4.0.0-beta.30 → 4.0.0-beta.32
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/actor.d.ts +21 -43
- package/dist/actor.js +74 -74
- package/dist/apify_dataset_backend.d.ts +1 -1
- package/dist/apify_dataset_backend.js +6 -6
- package/dist/apify_key_value_store_backend.d.ts +1 -1
- package/dist/apify_key_value_store_backend.js +10 -10
- package/dist/apify_request_queue_backend.d.ts +1 -1
- package/dist/apify_request_queue_backend.js +3 -3
- package/dist/apify_request_queue_shared_backend.d.ts +1 -14
- package/dist/apify_request_queue_shared_backend.js +38 -38
- package/dist/apify_request_queue_single_backend.d.ts +1 -10
- package/dist/apify_request_queue_single_backend.js +44 -44
- package/dist/apify_storage_backend.d.ts +23 -12
- package/dist/apify_storage_backend.js +104 -51
- package/dist/charging.d.ts +1 -14
- package/dist/charging.js +84 -81
- package/dist/index.d.ts +1 -1
- package/dist/platform_event_manager.d.ts +1 -2
- package/dist/platform_event_manager.js +7 -7
- package/dist/proxy_configuration.d.ts +7 -18
- package/dist/proxy_configuration.js +46 -57
- package/dist/storage.d.ts +6 -26
- package/dist/storage.js +1 -79
- package/package.json +1 -1
package/dist/actor.d.ts
CHANGED
|
@@ -11,7 +11,7 @@ import type { ConfigurationOptions } from './configuration.js';
|
|
|
11
11
|
import { Configuration } from './configuration.js';
|
|
12
12
|
import type { ProxyConfigurationOptions } from './proxy_configuration.js';
|
|
13
13
|
import { ProxyConfiguration } from './proxy_configuration.js';
|
|
14
|
-
import type { OpenStorageOptions, StorageIdentifier
|
|
14
|
+
import type { OpenStorageOptions, StorageIdentifier } from './storage.js';
|
|
15
15
|
export interface InitOptions {
|
|
16
16
|
storage?: StorageBackend;
|
|
17
17
|
/**
|
|
@@ -319,8 +319,6 @@ export declare const EXIT_CODES: {
|
|
|
319
319
|
*/
|
|
320
320
|
export declare class Actor<Data extends Dictionary = Dictionary> {
|
|
321
321
|
#private;
|
|
322
|
-
/** @internal */
|
|
323
|
-
static _instance: Actor;
|
|
324
322
|
/**
|
|
325
323
|
* Configuration of this SDK instance (provided to its constructor). See {@link Configuration} for details.
|
|
326
324
|
* @internal
|
|
@@ -340,32 +338,6 @@ export declare class Actor<Data extends Dictionary = Dictionary> {
|
|
|
340
338
|
* Whether the Actor instance was initialized. This is set by calling {@link Actor.init}.
|
|
341
339
|
*/
|
|
342
340
|
initialized: boolean;
|
|
343
|
-
/**
|
|
344
|
-
* Set if the Actor called a method that requires the instance to be initialized, but did not do so.
|
|
345
|
-
* A call to `init` after this warning is emitted is considered an invalid state and will throw an error.
|
|
346
|
-
*/
|
|
347
|
-
private warnedAboutMissingInitCall;
|
|
348
|
-
/**
|
|
349
|
-
* Set if the Actor is currently rebooting.
|
|
350
|
-
*/
|
|
351
|
-
private isRebooting;
|
|
352
|
-
/**
|
|
353
|
-
* Set if the Actor is currently exiting. Prevents double-exit from graceful shutdown handlers.
|
|
354
|
-
*/
|
|
355
|
-
private isExiting;
|
|
356
|
-
/**
|
|
357
|
-
* References to graceful shutdown handlers so they can be removed during cleanup.
|
|
358
|
-
*/
|
|
359
|
-
private gracefulShutdownHandlers;
|
|
360
|
-
private chargingManager;
|
|
361
|
-
/**
|
|
362
|
-
* Tracks which aliased storages have been purged during this session,
|
|
363
|
-
* so we only purge them once (on first open) when running locally.
|
|
364
|
-
* @internal
|
|
365
|
-
*/
|
|
366
|
-
purgedStorageAliases: Set<string>;
|
|
367
|
-
/** How Apify platform request queues are consumed; set from {@link InitOptions.requestQueueAccess}. */
|
|
368
|
-
private requestQueueAccess;
|
|
369
341
|
constructor(options?: ActorOptions);
|
|
370
342
|
/**
|
|
371
343
|
* Runs the main user function that performs the job of the Actor
|
|
@@ -623,8 +595,8 @@ export declare class Actor<Data extends Dictionary = Dictionary> {
|
|
|
623
595
|
* @param [datasetIdOrName]
|
|
624
596
|
* ID, name, or alias of the dataset to be opened. If `null` or `undefined`,
|
|
625
597
|
* the function returns the default dataset associated with the Actor run.
|
|
626
|
-
* You can also pass `{ alias: '
|
|
627
|
-
*
|
|
598
|
+
* You can also pass `{ alias: 'abc' }` to open a run-scoped storage, `{ id: 'abc' }` to open by
|
|
599
|
+
* explicit ID, or `{ name: 'abc' }` to open by explicit name.
|
|
628
600
|
* @param [options]
|
|
629
601
|
* @ignore
|
|
630
602
|
*/
|
|
@@ -735,13 +707,14 @@ export declare class Actor<Data extends Dictionary = Dictionary> {
|
|
|
735
707
|
* For more details and code examples, see the {@link KeyValueStore} class.
|
|
736
708
|
*
|
|
737
709
|
* @param [storeIdOrName]
|
|
738
|
-
* ID or
|
|
710
|
+
* ID, name, or alias of the key-value store to be opened. If `null` or `undefined`,
|
|
739
711
|
* the function returns the default key-value store associated with the Actor run.
|
|
740
|
-
* You can also pass `{
|
|
712
|
+
* You can also pass `{ alias: 'abc' }` to open a run-scoped storage, `{ id: 'abc' }` to open by
|
|
713
|
+
* explicit ID, or `{ name: 'abc' }` to open by explicit name.
|
|
741
714
|
* @param [options]
|
|
742
715
|
* @ignore
|
|
743
716
|
*/
|
|
744
|
-
openKeyValueStore(storeIdOrName?:
|
|
717
|
+
openKeyValueStore(storeIdOrName?: StorageIdentifier | null, options?: OpenStorageOptions): Promise<KeyValueStore>;
|
|
745
718
|
/**
|
|
746
719
|
* Opens a request queue and returns a promise resolving to an instance
|
|
747
720
|
* of the {@link RequestQueue} class.
|
|
@@ -754,13 +727,14 @@ export declare class Actor<Data extends Dictionary = Dictionary> {
|
|
|
754
727
|
* For more details and code examples, see the {@link RequestQueue} class.
|
|
755
728
|
*
|
|
756
729
|
* @param [queueIdOrName]
|
|
757
|
-
* ID or
|
|
730
|
+
* ID, name, or alias of the request queue to be opened. If `null` or `undefined`,
|
|
758
731
|
* the function returns the default request queue associated with the Actor run.
|
|
759
|
-
* You can also pass `{
|
|
732
|
+
* You can also pass `{ alias: 'abc' }` to open a run-scoped storage, `{ id: 'abc' }` to open by
|
|
733
|
+
* explicit ID, or `{ name: 'abc' }` to open by explicit name.
|
|
760
734
|
* @param [options]
|
|
761
735
|
* @ignore
|
|
762
736
|
*/
|
|
763
|
-
openRequestQueue(queueIdOrName?:
|
|
737
|
+
openRequestQueue(queueIdOrName?: StorageIdentifier | null, options?: OpenStorageOptions): Promise<RequestQueue>;
|
|
764
738
|
/**
|
|
765
739
|
* Creates a proxy configuration and returns a promise resolving to an instance
|
|
766
740
|
* of the {@link ProxyConfiguration} class that is already initialized.
|
|
@@ -1168,8 +1142,8 @@ export declare class Actor<Data extends Dictionary = Dictionary> {
|
|
|
1168
1142
|
* @param [datasetIdOrName]
|
|
1169
1143
|
* ID, name, or alias of the dataset to be opened. If `null` or `undefined`,
|
|
1170
1144
|
* the function returns the default dataset associated with the Actor run.
|
|
1171
|
-
* You can also pass `{ alias: '
|
|
1172
|
-
*
|
|
1145
|
+
* You can also pass `{ alias: 'abc' }` to open a run-scoped storage, `{ id: 'abc' }` to open by
|
|
1146
|
+
* explicit ID, or `{ name: 'abc' }` to open by explicit name.
|
|
1173
1147
|
* @param [options]
|
|
1174
1148
|
*/
|
|
1175
1149
|
static openDataset<Data extends Dictionary = Dictionary>(datasetIdOrName?: StorageIdentifier | null, options?: OpenStorageOptions): Promise<Dataset<Data>>;
|
|
@@ -1280,7 +1254,7 @@ export declare class Actor<Data extends Dictionary = Dictionary> {
|
|
|
1280
1254
|
* You can also pass `{ id: 'abc' }` to open by explicit ID, or `{ name: 'abc' }` to open by explicit name.
|
|
1281
1255
|
* @param [options]
|
|
1282
1256
|
*/
|
|
1283
|
-
static openKeyValueStore(storeIdOrName?:
|
|
1257
|
+
static openKeyValueStore(storeIdOrName?: StorageIdentifier | null, options?: OpenStorageOptions): Promise<KeyValueStore>;
|
|
1284
1258
|
/**
|
|
1285
1259
|
* Opens a request queue and returns a promise resolving to an instance
|
|
1286
1260
|
* of the {@link RequestQueue} class.
|
|
@@ -1298,7 +1272,7 @@ export declare class Actor<Data extends Dictionary = Dictionary> {
|
|
|
1298
1272
|
* You can also pass `{ id: 'abc' }` to open by explicit ID, or `{ name: 'abc' }` to open by explicit name.
|
|
1299
1273
|
* @param [options]
|
|
1300
1274
|
*/
|
|
1301
|
-
static openRequestQueue(queueIdOrName?:
|
|
1275
|
+
static openRequestQueue(queueIdOrName?: StorageIdentifier | null, options?: OpenStorageOptions): Promise<RequestQueue>;
|
|
1302
1276
|
/**
|
|
1303
1277
|
* Creates a proxy configuration and returns a promise resolving to an instance
|
|
1304
1278
|
* of the {@link ProxyConfiguration} class that is already initialized.
|
|
@@ -1387,12 +1361,16 @@ export declare class Actor<Data extends Dictionary = Dictionary> {
|
|
|
1387
1361
|
static get configuration(): Configuration;
|
|
1388
1362
|
/** @internal */
|
|
1389
1363
|
static getDefaultInstance(): Actor;
|
|
1364
|
+
/**
|
|
1365
|
+
* Replaces or clears the cached default instance returned by {@link Actor.getDefaultInstance}.
|
|
1366
|
+
* @internal
|
|
1367
|
+
*/
|
|
1368
|
+
static setDefaultInstance(instance?: Actor): void;
|
|
1390
1369
|
private usesPushDataInterception;
|
|
1391
1370
|
private pushDataViaInterceptedClient;
|
|
1392
1371
|
private pushDataWithExplicitCharging;
|
|
1393
|
-
private _openStorage;
|
|
1394
1372
|
private createApifyStorageBackend;
|
|
1395
|
-
private
|
|
1373
|
+
private ensureActorInit;
|
|
1396
1374
|
/**
|
|
1397
1375
|
* Get time remaining from the Actor run timeout in seconds, rounded up to whole seconds with minimum value of 1 second.
|
|
1398
1376
|
*
|
package/dist/actor.js
CHANGED
|
@@ -14,7 +14,6 @@ import { Configuration } from './configuration.js';
|
|
|
14
14
|
import { getDefaultsFromInputSchema, noActorInputSchemaDefinedMarker, readInputSchema } from './input-schemas.js';
|
|
15
15
|
import { PlatformEventManager } from './platform_event_manager.js';
|
|
16
16
|
import { ProxyConfiguration } from './proxy_configuration.js';
|
|
17
|
-
import { openStorage } from './storage.js';
|
|
18
17
|
import { checkCrawleeVersion, getSystemInfo, isNonEmptyObject, printOutdatedSdkWarning, snakeCaseToCamelCase, } from './utils.js';
|
|
19
18
|
/**
|
|
20
19
|
* Exit codes for the Actor process.
|
|
@@ -34,7 +33,7 @@ export const EXIT_CODES = {
|
|
|
34
33
|
*/
|
|
35
34
|
export class Actor {
|
|
36
35
|
/** @internal */
|
|
37
|
-
static
|
|
36
|
+
static #instance;
|
|
38
37
|
/**
|
|
39
38
|
* Configuration of this SDK instance (provided to its constructor). See {@link Configuration} for details.
|
|
40
39
|
* @internal
|
|
@@ -58,32 +57,26 @@ export class Actor {
|
|
|
58
57
|
* Set if the Actor called a method that requires the instance to be initialized, but did not do so.
|
|
59
58
|
* A call to `init` after this warning is emitted is considered an invalid state and will throw an error.
|
|
60
59
|
*/
|
|
61
|
-
warnedAboutMissingInitCall = false;
|
|
60
|
+
#warnedAboutMissingInitCall = false;
|
|
62
61
|
/**
|
|
63
62
|
* Set if the Actor is currently rebooting.
|
|
64
63
|
*/
|
|
65
|
-
isRebooting = false;
|
|
64
|
+
#isRebooting = false;
|
|
66
65
|
/**
|
|
67
66
|
* Set if the Actor is currently exiting. Prevents double-exit from graceful shutdown handlers.
|
|
68
67
|
*/
|
|
69
|
-
isExiting = false;
|
|
68
|
+
#isExiting = false;
|
|
70
69
|
/**
|
|
71
70
|
* References to graceful shutdown handlers so they can be removed during cleanup.
|
|
72
71
|
*/
|
|
73
|
-
gracefulShutdownHandlers = {};
|
|
72
|
+
#gracefulShutdownHandlers = {};
|
|
74
73
|
/**
|
|
75
74
|
* Reference to the crawlee status message forwarder, so it can be removed during cleanup.
|
|
76
75
|
*/
|
|
77
76
|
#statusMessageForwarder;
|
|
78
|
-
chargingManager;
|
|
79
|
-
/**
|
|
80
|
-
* Tracks which aliased storages have been purged during this session,
|
|
81
|
-
* so we only purge them once (on first open) when running locally.
|
|
82
|
-
* @internal
|
|
83
|
-
*/
|
|
84
|
-
purgedStorageAliases = new Set();
|
|
77
|
+
#chargingManager;
|
|
85
78
|
/** How Apify platform request queues are consumed; set from {@link InitOptions.requestQueueAccess}. */
|
|
86
|
-
requestQueueAccess = 'single';
|
|
79
|
+
#requestQueueAccess = 'single';
|
|
87
80
|
constructor(options = {}) {
|
|
88
81
|
const { configuration, ...configOptions } = options;
|
|
89
82
|
if (configuration) {
|
|
@@ -108,7 +101,7 @@ export class Actor {
|
|
|
108
101
|
}
|
|
109
102
|
this.apifyClient = this.newClient();
|
|
110
103
|
this.eventManager = new PlatformEventManager(this.configuration);
|
|
111
|
-
this
|
|
104
|
+
this.#chargingManager = new ChargingManager(this.configuration, this.apifyClient);
|
|
112
105
|
}
|
|
113
106
|
/**
|
|
114
107
|
* Runs the main user function that performs the job of the Actor
|
|
@@ -199,7 +192,7 @@ export class Actor {
|
|
|
199
192
|
return;
|
|
200
193
|
}
|
|
201
194
|
// If the warning about forgotten init call was emitted, we will not continue the init procedure.
|
|
202
|
-
if (this
|
|
195
|
+
if (this.#warnedAboutMissingInitCall) {
|
|
203
196
|
throw new Error([
|
|
204
197
|
'Actor.init() was called after a method that would access a storage client was used.',
|
|
205
198
|
'This in an invalid state. Please make sure to call Actor.init() before such methods are called.',
|
|
@@ -213,7 +206,7 @@ export class Actor {
|
|
|
213
206
|
// the event manager resolve the same instance (`availableMemoryRatio` /
|
|
214
207
|
// `disableBrowserSandbox` at-home defaults now live in `Configuration`).
|
|
215
208
|
serviceLocator.setConfiguration(this.configuration);
|
|
216
|
-
this
|
|
209
|
+
this.#requestQueueAccess = options.requestQueueAccess ?? 'single';
|
|
217
210
|
if (this.isAtHome()) {
|
|
218
211
|
serviceLocator.setStorageBackend(this.createApifyStorageBackend());
|
|
219
212
|
serviceLocator.setEventManager(this.eventManager);
|
|
@@ -230,22 +223,22 @@ export class Actor {
|
|
|
230
223
|
// Using setTimeout to avoid deadlock with waitForAllListenersToComplete() in exit()/reboot()
|
|
231
224
|
if (options.gracefulShutdown !== false) {
|
|
232
225
|
const delay = options.gracefulShutdownDelayMillis ?? 0;
|
|
233
|
-
this
|
|
226
|
+
this.#gracefulShutdownHandlers.aborting = () => {
|
|
234
227
|
setTimeout(() => {
|
|
235
228
|
this.exit().catch((err) => {
|
|
236
229
|
log.exception(err, 'Failed to exit gracefully');
|
|
237
230
|
});
|
|
238
231
|
}, delay);
|
|
239
232
|
};
|
|
240
|
-
this.on(ACTOR_EVENT_NAMES.ABORTING, this
|
|
241
|
-
this
|
|
233
|
+
this.on(ACTOR_EVENT_NAMES.ABORTING, this.#gracefulShutdownHandlers.aborting);
|
|
234
|
+
this.#gracefulShutdownHandlers.migrating = () => {
|
|
242
235
|
setTimeout(() => {
|
|
243
236
|
this.reboot().catch((err) => {
|
|
244
237
|
log.exception(err, 'Failed to reboot on migration');
|
|
245
238
|
});
|
|
246
239
|
}, delay);
|
|
247
240
|
};
|
|
248
|
-
this.on(ACTOR_EVENT_NAMES.MIGRATING, this
|
|
241
|
+
this.on(ACTOR_EVENT_NAMES.MIGRATING, this.#gracefulShutdownHandlers.migrating);
|
|
249
242
|
}
|
|
250
243
|
// Crawlee crawlers, for instance, broadcast their status messages as `statusMessage` events.
|
|
251
244
|
this.#statusMessageForwarder = async ({ message, isStatusMessageTerminal }) => this.#updateRunStatusMessage(message, isStatusMessageTerminal);
|
|
@@ -255,19 +248,19 @@ export class Actor {
|
|
|
255
248
|
onlyPurgeOnce: true,
|
|
256
249
|
});
|
|
257
250
|
log.debug(`Default storages purged`);
|
|
258
|
-
await this
|
|
259
|
-
log.debug(`ChargingManager initialized`, this
|
|
251
|
+
await this.#chargingManager.init();
|
|
252
|
+
log.debug(`ChargingManager initialized`, this.#chargingManager.getPricingInfo());
|
|
260
253
|
}
|
|
261
254
|
/**
|
|
262
255
|
* @ignore
|
|
263
256
|
*/
|
|
264
257
|
async exit(messageOrOptions, options = {}) {
|
|
265
258
|
// Prevent double-exit from graceful shutdown handlers
|
|
266
|
-
if (this
|
|
259
|
+
if (this.#isExiting) {
|
|
267
260
|
log.debug('Actor.exit() called while already exiting, skipping');
|
|
268
261
|
return;
|
|
269
262
|
}
|
|
270
|
-
this
|
|
263
|
+
this.#isExiting = true;
|
|
271
264
|
options =
|
|
272
265
|
typeof messageOrOptions === 'string'
|
|
273
266
|
? { ...options, statusMessage: messageOrOptions }
|
|
@@ -275,15 +268,15 @@ export class Actor {
|
|
|
275
268
|
options.exit ??= true;
|
|
276
269
|
options.exitCode ??= EXIT_CODES.SUCCESS;
|
|
277
270
|
options.timeoutSecs ??= 30;
|
|
278
|
-
this.
|
|
271
|
+
this.ensureActorInit('exit');
|
|
279
272
|
const client = serviceLocator.getStorageBackend();
|
|
280
273
|
const events = serviceLocator.getEventManager();
|
|
281
274
|
// Remove graceful shutdown handlers to prevent them from interfering with exit
|
|
282
|
-
if (this
|
|
283
|
-
this.off(ACTOR_EVENT_NAMES.ABORTING, this
|
|
275
|
+
if (this.#gracefulShutdownHandlers.aborting) {
|
|
276
|
+
this.off(ACTOR_EVENT_NAMES.ABORTING, this.#gracefulShutdownHandlers.aborting);
|
|
284
277
|
}
|
|
285
|
-
if (this
|
|
286
|
-
this.off(ACTOR_EVENT_NAMES.MIGRATING, this
|
|
278
|
+
if (this.#gracefulShutdownHandlers.migrating) {
|
|
279
|
+
this.off(ACTOR_EVENT_NAMES.MIGRATING, this.#gracefulShutdownHandlers.migrating);
|
|
287
280
|
}
|
|
288
281
|
// Close the event manager and emit the final PERSIST_STATE event
|
|
289
282
|
await events.close();
|
|
@@ -331,7 +324,7 @@ export class Actor {
|
|
|
331
324
|
});
|
|
332
325
|
// Reset the flag so the instance can be reused (e.g., in tests or when exit is false).
|
|
333
326
|
// When process.exit() actually terminates the process, this line is never reached - which is fine.
|
|
334
|
-
this
|
|
327
|
+
this.#isExiting = false;
|
|
335
328
|
if (!options.exit) {
|
|
336
329
|
return;
|
|
337
330
|
}
|
|
@@ -505,16 +498,16 @@ export class Actor {
|
|
|
505
498
|
* @ignore
|
|
506
499
|
*/
|
|
507
500
|
async reboot(options = {}) {
|
|
508
|
-
this.
|
|
501
|
+
this.ensureActorInit('reboot');
|
|
509
502
|
if (!this.isAtHome()) {
|
|
510
503
|
log.warning('Actor.reboot() is only supported when running on the Apify platform.');
|
|
511
504
|
return;
|
|
512
505
|
}
|
|
513
|
-
if (this
|
|
506
|
+
if (this.#isRebooting) {
|
|
514
507
|
log.debug('Actor is already rebooting, skipping the additional reboot call.');
|
|
515
508
|
return;
|
|
516
509
|
}
|
|
517
|
-
this
|
|
510
|
+
this.#isRebooting = true;
|
|
518
511
|
// Waiting for all the listeners to finish, as `.reboot()` kills the container.
|
|
519
512
|
await Promise.all([
|
|
520
513
|
// `persistState` for individual RequestLists, RequestQueue... instances to be persisted
|
|
@@ -588,7 +581,7 @@ export class Actor {
|
|
|
588
581
|
const { isStatusMessageTerminal, level } = options || {};
|
|
589
582
|
parseArgument(statusMessage, z.string());
|
|
590
583
|
parseArgument(isStatusMessageTerminal, z.boolean().optional());
|
|
591
|
-
this.
|
|
584
|
+
this.ensureActorInit('setStatusMessage');
|
|
592
585
|
const loggedStatusMessage = `[Status message]: ${statusMessage}`;
|
|
593
586
|
switch (level) {
|
|
594
587
|
case 'DEBUG':
|
|
@@ -646,7 +639,7 @@ export class Actor {
|
|
|
646
639
|
* @ignore
|
|
647
640
|
*/
|
|
648
641
|
async pushData(item, eventName) {
|
|
649
|
-
this.
|
|
642
|
+
this.ensureActorInit('pushData');
|
|
650
643
|
if (eventName?.startsWith('apify-')) {
|
|
651
644
|
throw new Error(`Cannot charge for synthetic event '${eventName}' manually`);
|
|
652
645
|
}
|
|
@@ -674,15 +667,17 @@ export class Actor {
|
|
|
674
667
|
* @param [datasetIdOrName]
|
|
675
668
|
* ID, name, or alias of the dataset to be opened. If `null` or `undefined`,
|
|
676
669
|
* the function returns the default dataset associated with the Actor run.
|
|
677
|
-
* You can also pass `{ alias: '
|
|
678
|
-
*
|
|
670
|
+
* You can also pass `{ alias: 'abc' }` to open a run-scoped storage, `{ id: 'abc' }` to open by
|
|
671
|
+
* explicit ID, or `{ name: 'abc' }` to open by explicit name.
|
|
679
672
|
* @param [options]
|
|
680
673
|
* @ignore
|
|
681
674
|
*/
|
|
682
675
|
async openDataset(datasetIdOrName, options = {}) {
|
|
683
676
|
parseArgument(options, z.object({ forceCloud: z.boolean().optional() }).strict());
|
|
684
|
-
this.
|
|
685
|
-
return
|
|
677
|
+
this.ensureActorInit('openDataset');
|
|
678
|
+
return Dataset.open(datasetIdOrName ?? null, {
|
|
679
|
+
storageBackend: options.forceCloud ? this.createApifyStorageBackend() : undefined,
|
|
680
|
+
});
|
|
686
681
|
}
|
|
687
682
|
/**
|
|
688
683
|
* Gets a value from the default {@link KeyValueStore} associated with the current Actor run.
|
|
@@ -713,7 +708,7 @@ export class Actor {
|
|
|
713
708
|
* @ignore
|
|
714
709
|
*/
|
|
715
710
|
async getValue(key) {
|
|
716
|
-
this.
|
|
711
|
+
this.ensureActorInit('getValue');
|
|
717
712
|
const store = await this.openKeyValueStore();
|
|
718
713
|
return store.getValue(key);
|
|
719
714
|
}
|
|
@@ -749,7 +744,7 @@ export class Actor {
|
|
|
749
744
|
* @ignore
|
|
750
745
|
*/
|
|
751
746
|
async setValue(key, value, options = {}) {
|
|
752
|
-
this.
|
|
747
|
+
this.ensureActorInit('setValue');
|
|
753
748
|
const store = await this.openKeyValueStore();
|
|
754
749
|
return store.setValue(key, value, options);
|
|
755
750
|
}
|
|
@@ -783,7 +778,7 @@ export class Actor {
|
|
|
783
778
|
* @ignore
|
|
784
779
|
*/
|
|
785
780
|
async getInput() {
|
|
786
|
-
this.
|
|
781
|
+
this.ensureActorInit('getInput');
|
|
787
782
|
const { inputSecretsPrivateKeyFile, inputSecretsPrivateKeyPassphrase } = this.configuration;
|
|
788
783
|
const rawInput = await this.getValue(this.configuration.inputKey);
|
|
789
784
|
let input = rawInput;
|
|
@@ -820,16 +815,19 @@ export class Actor {
|
|
|
820
815
|
* For more details and code examples, see the {@link KeyValueStore} class.
|
|
821
816
|
*
|
|
822
817
|
* @param [storeIdOrName]
|
|
823
|
-
* ID or
|
|
818
|
+
* ID, name, or alias of the key-value store to be opened. If `null` or `undefined`,
|
|
824
819
|
* the function returns the default key-value store associated with the Actor run.
|
|
825
|
-
* You can also pass `{
|
|
820
|
+
* You can also pass `{ alias: 'abc' }` to open a run-scoped storage, `{ id: 'abc' }` to open by
|
|
821
|
+
* explicit ID, or `{ name: 'abc' }` to open by explicit name.
|
|
826
822
|
* @param [options]
|
|
827
823
|
* @ignore
|
|
828
824
|
*/
|
|
829
825
|
async openKeyValueStore(storeIdOrName, options = {}) {
|
|
830
826
|
parseArgument(options, z.object({ forceCloud: z.boolean().optional() }).strict());
|
|
831
|
-
this.
|
|
832
|
-
return
|
|
827
|
+
this.ensureActorInit('openKeyValueStore');
|
|
828
|
+
return KeyValueStore.open(storeIdOrName ?? null, {
|
|
829
|
+
storageBackend: options.forceCloud ? this.createApifyStorageBackend() : undefined,
|
|
830
|
+
});
|
|
833
831
|
}
|
|
834
832
|
/**
|
|
835
833
|
* Opens a request queue and returns a promise resolving to an instance
|
|
@@ -843,17 +841,19 @@ export class Actor {
|
|
|
843
841
|
* For more details and code examples, see the {@link RequestQueue} class.
|
|
844
842
|
*
|
|
845
843
|
* @param [queueIdOrName]
|
|
846
|
-
* ID or
|
|
844
|
+
* ID, name, or alias of the request queue to be opened. If `null` or `undefined`,
|
|
847
845
|
* the function returns the default request queue associated with the Actor run.
|
|
848
|
-
* You can also pass `{
|
|
846
|
+
* You can also pass `{ alias: 'abc' }` to open a run-scoped storage, `{ id: 'abc' }` to open by
|
|
847
|
+
* explicit ID, or `{ name: 'abc' }` to open by explicit name.
|
|
849
848
|
* @param [options]
|
|
850
849
|
* @ignore
|
|
851
850
|
*/
|
|
852
851
|
async openRequestQueue(queueIdOrName, options = {}) {
|
|
853
852
|
parseArgument(options, z.object({ forceCloud: z.boolean().optional() }).strict());
|
|
854
|
-
this.
|
|
855
|
-
|
|
856
|
-
|
|
853
|
+
this.ensureActorInit('openRequestQueue');
|
|
854
|
+
return RequestQueue.open(queueIdOrName ?? null, {
|
|
855
|
+
storageBackend: options.forceCloud ? this.createApifyStorageBackend() : undefined,
|
|
856
|
+
});
|
|
857
857
|
}
|
|
858
858
|
/**
|
|
859
859
|
* Creates a proxy configuration and returns a promise resolving to an instance
|
|
@@ -928,16 +928,16 @@ export class Actor {
|
|
|
928
928
|
* @ignore
|
|
929
929
|
*/
|
|
930
930
|
async charge(options) {
|
|
931
|
-
this.
|
|
932
|
-
return this
|
|
931
|
+
this.ensureActorInit('charge');
|
|
932
|
+
return this.#chargingManager.charge(options);
|
|
933
933
|
}
|
|
934
934
|
/**
|
|
935
935
|
* Retrieve the charging manager to access granular pricing information.
|
|
936
936
|
* @ignore
|
|
937
937
|
*/
|
|
938
938
|
getChargingManager() {
|
|
939
|
-
this.
|
|
940
|
-
return this
|
|
939
|
+
this.ensureActorInit('getChargingManager');
|
|
940
|
+
return this.#chargingManager;
|
|
941
941
|
}
|
|
942
942
|
/**
|
|
943
943
|
* Modifies Actor env vars so parsing respects the structure of {@link ApifyEnv} interface.
|
|
@@ -1022,7 +1022,7 @@ export class Actor {
|
|
|
1022
1022
|
* @param options An optional object parameter where a custom `keyValueStoreName` and `config` can be passed in.
|
|
1023
1023
|
*/
|
|
1024
1024
|
async useState(name, defaultValue = {}, options) {
|
|
1025
|
-
this.
|
|
1025
|
+
this.ensureActorInit('useState');
|
|
1026
1026
|
const kvStore = await KeyValueStore.open(options?.keyValueStoreName, {
|
|
1027
1027
|
configuration: options?.configuration || Configuration.getGlobalConfiguration(),
|
|
1028
1028
|
});
|
|
@@ -1343,8 +1343,8 @@ export class Actor {
|
|
|
1343
1343
|
* @param [datasetIdOrName]
|
|
1344
1344
|
* ID, name, or alias of the dataset to be opened. If `null` or `undefined`,
|
|
1345
1345
|
* the function returns the default dataset associated with the Actor run.
|
|
1346
|
-
* You can also pass `{ alias: '
|
|
1347
|
-
*
|
|
1346
|
+
* You can also pass `{ alias: 'abc' }` to open a run-scoped storage, `{ id: 'abc' }` to open by
|
|
1347
|
+
* explicit ID, or `{ name: 'abc' }` to open by explicit name.
|
|
1348
1348
|
* @param [options]
|
|
1349
1349
|
*/
|
|
1350
1350
|
static async openDataset(datasetIdOrName, options = {}) {
|
|
@@ -1590,8 +1590,15 @@ export class Actor {
|
|
|
1590
1590
|
}
|
|
1591
1591
|
/** @internal */
|
|
1592
1592
|
static getDefaultInstance() {
|
|
1593
|
-
|
|
1594
|
-
return
|
|
1593
|
+
Actor.#instance ??= new Actor();
|
|
1594
|
+
return Actor.#instance;
|
|
1595
|
+
}
|
|
1596
|
+
/**
|
|
1597
|
+
* Replaces or clears the cached default instance returned by {@link Actor.getDefaultInstance}.
|
|
1598
|
+
* @internal
|
|
1599
|
+
*/
|
|
1600
|
+
static setDefaultInstance(instance) {
|
|
1601
|
+
Actor.#instance = instance;
|
|
1595
1602
|
}
|
|
1596
1603
|
usesPushDataInterception(dataset) {
|
|
1597
1604
|
return Boolean(dataset.backend[USES_PUSH_DATA_INTERCEPTION]);
|
|
@@ -1624,36 +1631,29 @@ export class Actor {
|
|
|
1624
1631
|
}
|
|
1625
1632
|
const isDefaultDataset = dataset.id === this.configuration.defaultDatasetId;
|
|
1626
1633
|
return pushDataAndCharge({
|
|
1627
|
-
chargingManager: this
|
|
1634
|
+
chargingManager: this.#chargingManager,
|
|
1628
1635
|
items,
|
|
1629
1636
|
eventName: explicitEventName,
|
|
1630
1637
|
isDefaultDataset,
|
|
1631
1638
|
pushFn: async (limitedItems) => dataset.pushData(limitedItems),
|
|
1632
1639
|
});
|
|
1633
1640
|
}
|
|
1634
|
-
async _openStorage(storageClass, identifier, options = {}) {
|
|
1635
|
-
return openStorage(storageClass, identifier, {
|
|
1636
|
-
config: this.configuration,
|
|
1637
|
-
backend: options.forceCloud ? this.createApifyStorageBackend() : undefined,
|
|
1638
|
-
purgedStorageAliases: this.purgedStorageAliases,
|
|
1639
|
-
});
|
|
1640
|
-
}
|
|
1641
1641
|
createApifyStorageBackend() {
|
|
1642
1642
|
return new ApifyStorageBackend(this.apifyClient, {
|
|
1643
1643
|
configuration: this.configuration,
|
|
1644
|
-
requestQueueAccess: this
|
|
1645
|
-
getChargingManager: () => this
|
|
1644
|
+
requestQueueAccess: this.#requestQueueAccess,
|
|
1645
|
+
getChargingManager: () => this.#chargingManager,
|
|
1646
1646
|
});
|
|
1647
1647
|
}
|
|
1648
|
-
|
|
1648
|
+
ensureActorInit(methodCalled) {
|
|
1649
1649
|
// If we already warned the user once, don't do it again to prevent spam
|
|
1650
|
-
if (this
|
|
1650
|
+
if (this.#warnedAboutMissingInitCall) {
|
|
1651
1651
|
return;
|
|
1652
1652
|
}
|
|
1653
1653
|
if (this.initialized) {
|
|
1654
1654
|
return;
|
|
1655
1655
|
}
|
|
1656
|
-
this
|
|
1656
|
+
this.#warnedAboutMissingInitCall = true;
|
|
1657
1657
|
log.warning([
|
|
1658
1658
|
`Actor.${methodCalled}() was called but the Actor instance was not initialized.`,
|
|
1659
1659
|
'Did you forget to call Actor.init()?',
|
|
@@ -9,7 +9,7 @@ import type { DatasetClient } from 'apify-client';
|
|
|
9
9
|
* @internal
|
|
10
10
|
*/
|
|
11
11
|
export declare class ApifyDatasetBackend implements DatasetBackend {
|
|
12
|
-
private
|
|
12
|
+
#private;
|
|
13
13
|
constructor(client: DatasetClient);
|
|
14
14
|
getMetadata(): Promise<DatasetInfo>;
|
|
15
15
|
drop(): Promise<void>;
|
|
@@ -13,19 +13,19 @@ const MAX_ITEM_BYTES = EFFECTIVE_LIMIT_BYTES - 2;
|
|
|
13
13
|
* @internal
|
|
14
14
|
*/
|
|
15
15
|
export class ApifyDatasetBackend {
|
|
16
|
-
client;
|
|
16
|
+
#client;
|
|
17
17
|
constructor(client) {
|
|
18
|
-
this
|
|
18
|
+
this.#client = client;
|
|
19
19
|
}
|
|
20
20
|
async getMetadata() {
|
|
21
|
-
const metadata = await this
|
|
21
|
+
const metadata = await this.#client.get();
|
|
22
22
|
if (!metadata) {
|
|
23
23
|
throw new Error('Dataset not found or has been deleted.');
|
|
24
24
|
}
|
|
25
25
|
return metadata;
|
|
26
26
|
}
|
|
27
27
|
async drop() {
|
|
28
|
-
await this
|
|
28
|
+
await this.#client.delete();
|
|
29
29
|
}
|
|
30
30
|
async purge() {
|
|
31
31
|
throw new Error('Purging a dataset is not supported on the Apify platform. ' +
|
|
@@ -36,11 +36,11 @@ export class ApifyDatasetBackend {
|
|
|
36
36
|
// that fit, pushed sequentially to preserve item order.
|
|
37
37
|
const payloads = items.map((item, index) => serializeToSizeLimit(item, index));
|
|
38
38
|
for (const chunk of chunkBySize(payloads, EFFECTIVE_LIMIT_BYTES)) {
|
|
39
|
-
await this
|
|
39
|
+
await this.#client.pushItems(chunk);
|
|
40
40
|
}
|
|
41
41
|
}
|
|
42
42
|
async getData(options) {
|
|
43
|
-
return await this
|
|
43
|
+
return await this.#client.listItems(options);
|
|
44
44
|
}
|
|
45
45
|
}
|
|
46
46
|
/** Serializes a dataset item, throwing if it alone exceeds the payload size limit. */
|
|
@@ -9,7 +9,7 @@ import type { KeyValueStoreClient } from 'apify-client';
|
|
|
9
9
|
* @internal
|
|
10
10
|
*/
|
|
11
11
|
export declare class ApifyKeyValueStoreBackend implements KeyValueStoreBackend {
|
|
12
|
-
private
|
|
12
|
+
#private;
|
|
13
13
|
constructor(client: KeyValueStoreClient);
|
|
14
14
|
getMetadata(): Promise<KeyValueStoreInfo>;
|
|
15
15
|
drop(): Promise<void>;
|
|
@@ -7,19 +7,19 @@
|
|
|
7
7
|
* @internal
|
|
8
8
|
*/
|
|
9
9
|
export class ApifyKeyValueStoreBackend {
|
|
10
|
-
client;
|
|
10
|
+
#client;
|
|
11
11
|
constructor(client) {
|
|
12
|
-
this
|
|
12
|
+
this.#client = client;
|
|
13
13
|
}
|
|
14
14
|
async getMetadata() {
|
|
15
|
-
const metadata = await this
|
|
15
|
+
const metadata = await this.#client.get();
|
|
16
16
|
if (!metadata) {
|
|
17
17
|
throw new Error('Key-value store not found or has been deleted.');
|
|
18
18
|
}
|
|
19
19
|
return metadata;
|
|
20
20
|
}
|
|
21
21
|
async drop() {
|
|
22
|
-
await this
|
|
22
|
+
await this.#client.delete();
|
|
23
23
|
}
|
|
24
24
|
async purge() {
|
|
25
25
|
throw new Error('Purging a key-value store is not supported on the Apify platform. ' +
|
|
@@ -28,16 +28,16 @@ export class ApifyKeyValueStoreBackend {
|
|
|
28
28
|
async getValue(key) {
|
|
29
29
|
// Storage backends are byte transports — the KeyValueStore frontend parses values
|
|
30
30
|
// according to their content type, so the record must be returned unparsed.
|
|
31
|
-
return this
|
|
31
|
+
return this.#client.getRecord(key, { buffer: true });
|
|
32
32
|
}
|
|
33
33
|
async setValue(record) {
|
|
34
|
-
await this
|
|
34
|
+
await this.#client.setRecord(record);
|
|
35
35
|
}
|
|
36
36
|
async deleteValue(key) {
|
|
37
|
-
await this
|
|
37
|
+
await this.#client.deleteRecord(key);
|
|
38
38
|
}
|
|
39
39
|
async listKeys(options) {
|
|
40
|
-
const result = await this
|
|
40
|
+
const result = await this.#client.listKeys(options);
|
|
41
41
|
// The API does not report a content type for listed keys; crawlee's item shape
|
|
42
42
|
// requires the field, so it is left undefined via the cast.
|
|
43
43
|
return {
|
|
@@ -46,9 +46,9 @@ export class ApifyKeyValueStoreBackend {
|
|
|
46
46
|
};
|
|
47
47
|
}
|
|
48
48
|
async getPublicUrl(key) {
|
|
49
|
-
return this
|
|
49
|
+
return this.#client.getRecordPublicUrl(key);
|
|
50
50
|
}
|
|
51
51
|
async recordExists(key) {
|
|
52
|
-
return this
|
|
52
|
+
return this.#client.recordExists(key);
|
|
53
53
|
}
|
|
54
54
|
}
|