apify 4.0.0-beta.30 → 4.0.0-beta.31
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/actor.d.ts +15 -20
- package/dist/actor.js +19 -26
- package/dist/apify_storage_backend.d.ts +27 -5
- package/dist/apify_storage_backend.js +71 -18
- package/dist/index.d.ts +1 -1
- package/dist/storage.d.ts +6 -26
- package/dist/storage.js +1 -79
- package/package.json +1 -1
package/dist/actor.d.ts
CHANGED
|
@@ -11,7 +11,7 @@ import type { ConfigurationOptions } from './configuration.js';
|
|
|
11
11
|
import { Configuration } from './configuration.js';
|
|
12
12
|
import type { ProxyConfigurationOptions } from './proxy_configuration.js';
|
|
13
13
|
import { ProxyConfiguration } from './proxy_configuration.js';
|
|
14
|
-
import type { OpenStorageOptions, StorageIdentifier
|
|
14
|
+
import type { OpenStorageOptions, StorageIdentifier } from './storage.js';
|
|
15
15
|
export interface InitOptions {
|
|
16
16
|
storage?: StorageBackend;
|
|
17
17
|
/**
|
|
@@ -358,12 +358,6 @@ export declare class Actor<Data extends Dictionary = Dictionary> {
|
|
|
358
358
|
*/
|
|
359
359
|
private gracefulShutdownHandlers;
|
|
360
360
|
private chargingManager;
|
|
361
|
-
/**
|
|
362
|
-
* Tracks which aliased storages have been purged during this session,
|
|
363
|
-
* so we only purge them once (on first open) when running locally.
|
|
364
|
-
* @internal
|
|
365
|
-
*/
|
|
366
|
-
purgedStorageAliases: Set<string>;
|
|
367
361
|
/** How Apify platform request queues are consumed; set from {@link InitOptions.requestQueueAccess}. */
|
|
368
362
|
private requestQueueAccess;
|
|
369
363
|
constructor(options?: ActorOptions);
|
|
@@ -623,8 +617,8 @@ export declare class Actor<Data extends Dictionary = Dictionary> {
|
|
|
623
617
|
* @param [datasetIdOrName]
|
|
624
618
|
* ID, name, or alias of the dataset to be opened. If `null` or `undefined`,
|
|
625
619
|
* the function returns the default dataset associated with the Actor run.
|
|
626
|
-
* You can also pass `{ alias: '
|
|
627
|
-
*
|
|
620
|
+
* You can also pass `{ alias: 'abc' }` to open a run-scoped storage, `{ id: 'abc' }` to open by
|
|
621
|
+
* explicit ID, or `{ name: 'abc' }` to open by explicit name.
|
|
628
622
|
* @param [options]
|
|
629
623
|
* @ignore
|
|
630
624
|
*/
|
|
@@ -735,13 +729,14 @@ export declare class Actor<Data extends Dictionary = Dictionary> {
|
|
|
735
729
|
* For more details and code examples, see the {@link KeyValueStore} class.
|
|
736
730
|
*
|
|
737
731
|
* @param [storeIdOrName]
|
|
738
|
-
* ID or
|
|
732
|
+
* ID, name, or alias of the key-value store to be opened. If `null` or `undefined`,
|
|
739
733
|
* the function returns the default key-value store associated with the Actor run.
|
|
740
|
-
* You can also pass `{
|
|
734
|
+
* You can also pass `{ alias: 'abc' }` to open a run-scoped storage, `{ id: 'abc' }` to open by
|
|
735
|
+
* explicit ID, or `{ name: 'abc' }` to open by explicit name.
|
|
741
736
|
* @param [options]
|
|
742
737
|
* @ignore
|
|
743
738
|
*/
|
|
744
|
-
openKeyValueStore(storeIdOrName?:
|
|
739
|
+
openKeyValueStore(storeIdOrName?: StorageIdentifier | null, options?: OpenStorageOptions): Promise<KeyValueStore>;
|
|
745
740
|
/**
|
|
746
741
|
* Opens a request queue and returns a promise resolving to an instance
|
|
747
742
|
* of the {@link RequestQueue} class.
|
|
@@ -754,13 +749,14 @@ export declare class Actor<Data extends Dictionary = Dictionary> {
|
|
|
754
749
|
* For more details and code examples, see the {@link RequestQueue} class.
|
|
755
750
|
*
|
|
756
751
|
* @param [queueIdOrName]
|
|
757
|
-
* ID or
|
|
752
|
+
* ID, name, or alias of the request queue to be opened. If `null` or `undefined`,
|
|
758
753
|
* the function returns the default request queue associated with the Actor run.
|
|
759
|
-
* You can also pass `{
|
|
754
|
+
* You can also pass `{ alias: 'abc' }` to open a run-scoped storage, `{ id: 'abc' }` to open by
|
|
755
|
+
* explicit ID, or `{ name: 'abc' }` to open by explicit name.
|
|
760
756
|
* @param [options]
|
|
761
757
|
* @ignore
|
|
762
758
|
*/
|
|
763
|
-
openRequestQueue(queueIdOrName?:
|
|
759
|
+
openRequestQueue(queueIdOrName?: StorageIdentifier | null, options?: OpenStorageOptions): Promise<RequestQueue>;
|
|
764
760
|
/**
|
|
765
761
|
* Creates a proxy configuration and returns a promise resolving to an instance
|
|
766
762
|
* of the {@link ProxyConfiguration} class that is already initialized.
|
|
@@ -1168,8 +1164,8 @@ export declare class Actor<Data extends Dictionary = Dictionary> {
|
|
|
1168
1164
|
* @param [datasetIdOrName]
|
|
1169
1165
|
* ID, name, or alias of the dataset to be opened. If `null` or `undefined`,
|
|
1170
1166
|
* the function returns the default dataset associated with the Actor run.
|
|
1171
|
-
* You can also pass `{ alias: '
|
|
1172
|
-
*
|
|
1167
|
+
* You can also pass `{ alias: 'abc' }` to open a run-scoped storage, `{ id: 'abc' }` to open by
|
|
1168
|
+
* explicit ID, or `{ name: 'abc' }` to open by explicit name.
|
|
1173
1169
|
* @param [options]
|
|
1174
1170
|
*/
|
|
1175
1171
|
static openDataset<Data extends Dictionary = Dictionary>(datasetIdOrName?: StorageIdentifier | null, options?: OpenStorageOptions): Promise<Dataset<Data>>;
|
|
@@ -1280,7 +1276,7 @@ export declare class Actor<Data extends Dictionary = Dictionary> {
|
|
|
1280
1276
|
* You can also pass `{ id: 'abc' }` to open by explicit ID, or `{ name: 'abc' }` to open by explicit name.
|
|
1281
1277
|
* @param [options]
|
|
1282
1278
|
*/
|
|
1283
|
-
static openKeyValueStore(storeIdOrName?:
|
|
1279
|
+
static openKeyValueStore(storeIdOrName?: StorageIdentifier | null, options?: OpenStorageOptions): Promise<KeyValueStore>;
|
|
1284
1280
|
/**
|
|
1285
1281
|
* Opens a request queue and returns a promise resolving to an instance
|
|
1286
1282
|
* of the {@link RequestQueue} class.
|
|
@@ -1298,7 +1294,7 @@ export declare class Actor<Data extends Dictionary = Dictionary> {
|
|
|
1298
1294
|
* You can also pass `{ id: 'abc' }` to open by explicit ID, or `{ name: 'abc' }` to open by explicit name.
|
|
1299
1295
|
* @param [options]
|
|
1300
1296
|
*/
|
|
1301
|
-
static openRequestQueue(queueIdOrName?:
|
|
1297
|
+
static openRequestQueue(queueIdOrName?: StorageIdentifier | null, options?: OpenStorageOptions): Promise<RequestQueue>;
|
|
1302
1298
|
/**
|
|
1303
1299
|
* Creates a proxy configuration and returns a promise resolving to an instance
|
|
1304
1300
|
* of the {@link ProxyConfiguration} class that is already initialized.
|
|
@@ -1390,7 +1386,6 @@ export declare class Actor<Data extends Dictionary = Dictionary> {
|
|
|
1390
1386
|
private usesPushDataInterception;
|
|
1391
1387
|
private pushDataViaInterceptedClient;
|
|
1392
1388
|
private pushDataWithExplicitCharging;
|
|
1393
|
-
private _openStorage;
|
|
1394
1389
|
private createApifyStorageBackend;
|
|
1395
1390
|
private _ensureActorInit;
|
|
1396
1391
|
/**
|
package/dist/actor.js
CHANGED
|
@@ -14,7 +14,6 @@ import { Configuration } from './configuration.js';
|
|
|
14
14
|
import { getDefaultsFromInputSchema, noActorInputSchemaDefinedMarker, readInputSchema } from './input-schemas.js';
|
|
15
15
|
import { PlatformEventManager } from './platform_event_manager.js';
|
|
16
16
|
import { ProxyConfiguration } from './proxy_configuration.js';
|
|
17
|
-
import { openStorage } from './storage.js';
|
|
18
17
|
import { checkCrawleeVersion, getSystemInfo, isNonEmptyObject, printOutdatedSdkWarning, snakeCaseToCamelCase, } from './utils.js';
|
|
19
18
|
/**
|
|
20
19
|
* Exit codes for the Actor process.
|
|
@@ -76,12 +75,6 @@ export class Actor {
|
|
|
76
75
|
*/
|
|
77
76
|
#statusMessageForwarder;
|
|
78
77
|
chargingManager;
|
|
79
|
-
/**
|
|
80
|
-
* Tracks which aliased storages have been purged during this session,
|
|
81
|
-
* so we only purge them once (on first open) when running locally.
|
|
82
|
-
* @internal
|
|
83
|
-
*/
|
|
84
|
-
purgedStorageAliases = new Set();
|
|
85
78
|
/** How Apify platform request queues are consumed; set from {@link InitOptions.requestQueueAccess}. */
|
|
86
79
|
requestQueueAccess = 'single';
|
|
87
80
|
constructor(options = {}) {
|
|
@@ -674,15 +667,17 @@ export class Actor {
|
|
|
674
667
|
* @param [datasetIdOrName]
|
|
675
668
|
* ID, name, or alias of the dataset to be opened. If `null` or `undefined`,
|
|
676
669
|
* the function returns the default dataset associated with the Actor run.
|
|
677
|
-
* You can also pass `{ alias: '
|
|
678
|
-
*
|
|
670
|
+
* You can also pass `{ alias: 'abc' }` to open a run-scoped storage, `{ id: 'abc' }` to open by
|
|
671
|
+
* explicit ID, or `{ name: 'abc' }` to open by explicit name.
|
|
679
672
|
* @param [options]
|
|
680
673
|
* @ignore
|
|
681
674
|
*/
|
|
682
675
|
async openDataset(datasetIdOrName, options = {}) {
|
|
683
676
|
parseArgument(options, z.object({ forceCloud: z.boolean().optional() }).strict());
|
|
684
677
|
this._ensureActorInit('openDataset');
|
|
685
|
-
return
|
|
678
|
+
return Dataset.open(datasetIdOrName ?? null, {
|
|
679
|
+
storageBackend: options.forceCloud ? this.createApifyStorageBackend() : undefined,
|
|
680
|
+
});
|
|
686
681
|
}
|
|
687
682
|
/**
|
|
688
683
|
* Gets a value from the default {@link KeyValueStore} associated with the current Actor run.
|
|
@@ -820,16 +815,19 @@ export class Actor {
|
|
|
820
815
|
* For more details and code examples, see the {@link KeyValueStore} class.
|
|
821
816
|
*
|
|
822
817
|
* @param [storeIdOrName]
|
|
823
|
-
* ID or
|
|
818
|
+
* ID, name, or alias of the key-value store to be opened. If `null` or `undefined`,
|
|
824
819
|
* the function returns the default key-value store associated with the Actor run.
|
|
825
|
-
* You can also pass `{
|
|
820
|
+
* You can also pass `{ alias: 'abc' }` to open a run-scoped storage, `{ id: 'abc' }` to open by
|
|
821
|
+
* explicit ID, or `{ name: 'abc' }` to open by explicit name.
|
|
826
822
|
* @param [options]
|
|
827
823
|
* @ignore
|
|
828
824
|
*/
|
|
829
825
|
async openKeyValueStore(storeIdOrName, options = {}) {
|
|
830
826
|
parseArgument(options, z.object({ forceCloud: z.boolean().optional() }).strict());
|
|
831
827
|
this._ensureActorInit('openKeyValueStore');
|
|
832
|
-
return
|
|
828
|
+
return KeyValueStore.open(storeIdOrName ?? null, {
|
|
829
|
+
storageBackend: options.forceCloud ? this.createApifyStorageBackend() : undefined,
|
|
830
|
+
});
|
|
833
831
|
}
|
|
834
832
|
/**
|
|
835
833
|
* Opens a request queue and returns a promise resolving to an instance
|
|
@@ -843,17 +841,19 @@ export class Actor {
|
|
|
843
841
|
* For more details and code examples, see the {@link RequestQueue} class.
|
|
844
842
|
*
|
|
845
843
|
* @param [queueIdOrName]
|
|
846
|
-
* ID or
|
|
844
|
+
* ID, name, or alias of the request queue to be opened. If `null` or `undefined`,
|
|
847
845
|
* the function returns the default request queue associated with the Actor run.
|
|
848
|
-
* You can also pass `{
|
|
846
|
+
* You can also pass `{ alias: 'abc' }` to open a run-scoped storage, `{ id: 'abc' }` to open by
|
|
847
|
+
* explicit ID, or `{ name: 'abc' }` to open by explicit name.
|
|
849
848
|
* @param [options]
|
|
850
849
|
* @ignore
|
|
851
850
|
*/
|
|
852
851
|
async openRequestQueue(queueIdOrName, options = {}) {
|
|
853
852
|
parseArgument(options, z.object({ forceCloud: z.boolean().optional() }).strict());
|
|
854
853
|
this._ensureActorInit('openRequestQueue');
|
|
855
|
-
|
|
856
|
-
|
|
854
|
+
return RequestQueue.open(queueIdOrName ?? null, {
|
|
855
|
+
storageBackend: options.forceCloud ? this.createApifyStorageBackend() : undefined,
|
|
856
|
+
});
|
|
857
857
|
}
|
|
858
858
|
/**
|
|
859
859
|
* Creates a proxy configuration and returns a promise resolving to an instance
|
|
@@ -1343,8 +1343,8 @@ export class Actor {
|
|
|
1343
1343
|
* @param [datasetIdOrName]
|
|
1344
1344
|
* ID, name, or alias of the dataset to be opened. If `null` or `undefined`,
|
|
1345
1345
|
* the function returns the default dataset associated with the Actor run.
|
|
1346
|
-
* You can also pass `{ alias: '
|
|
1347
|
-
*
|
|
1346
|
+
* You can also pass `{ alias: 'abc' }` to open a run-scoped storage, `{ id: 'abc' }` to open by
|
|
1347
|
+
* explicit ID, or `{ name: 'abc' }` to open by explicit name.
|
|
1348
1348
|
* @param [options]
|
|
1349
1349
|
*/
|
|
1350
1350
|
static async openDataset(datasetIdOrName, options = {}) {
|
|
@@ -1631,13 +1631,6 @@ export class Actor {
|
|
|
1631
1631
|
pushFn: async (limitedItems) => dataset.pushData(limitedItems),
|
|
1632
1632
|
});
|
|
1633
1633
|
}
|
|
1634
|
-
async _openStorage(storageClass, identifier, options = {}) {
|
|
1635
|
-
return openStorage(storageClass, identifier, {
|
|
1636
|
-
config: this.configuration,
|
|
1637
|
-
backend: options.forceCloud ? this.createApifyStorageBackend() : undefined,
|
|
1638
|
-
purgedStorageAliases: this.purgedStorageAliases,
|
|
1639
|
-
});
|
|
1640
|
-
}
|
|
1641
1634
|
createApifyStorageBackend() {
|
|
1642
1635
|
return new ApifyStorageBackend(this.apifyClient, {
|
|
1643
1636
|
configuration: this.configuration,
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { AsyncLocalStorage } from 'node:async_hooks';
|
|
2
2
|
import type { DatasetBackend, KeyValueStoreBackend, RequestQueueBackend, StorageBackend, StorageIdentifier } from '@crawlee/types';
|
|
3
3
|
import type { ApifyClient } from 'apify-client';
|
|
4
|
-
import type
|
|
4
|
+
import { type RequestQueueAccessMode } from './apify_request_queue_backend.js';
|
|
5
5
|
import { type ChargeResult, type ChargingManager } from './charging.js';
|
|
6
6
|
import type { Configuration } from './configuration.js';
|
|
7
7
|
type StorageType = 'Dataset' | 'KeyValueStore' | 'RequestQueue';
|
|
@@ -66,8 +66,12 @@ export declare class ApifyStorageBackend implements StorageBackend {
|
|
|
66
66
|
private readonly config?;
|
|
67
67
|
private readonly requestQueueAccess;
|
|
68
68
|
private readonly getChargingManager?;
|
|
69
|
-
/** Unnamed storages
|
|
69
|
+
/** Unnamed storages resolved for aliases in this process, keyed like {@link AliasMapping}. */
|
|
70
70
|
private readonly aliasIdCache;
|
|
71
|
+
/** The alias mapping read from the run's default key-value store; `undefined` until first read. */
|
|
72
|
+
private persistedAliasIds?;
|
|
73
|
+
/** Serializes alias resolution — see `resolveAliasId`. */
|
|
74
|
+
private readonly aliasLock;
|
|
71
75
|
/** Fallback request queue client key when the run id is unavailable — one per backend. */
|
|
72
76
|
private fallbackClientKey?;
|
|
73
77
|
constructor(client: ApifyClient, options?: ApifyStorageBackendOptions);
|
|
@@ -78,6 +82,8 @@ export declare class ApifyStorageBackend implements StorageBackend {
|
|
|
78
82
|
* and `shared` mode at once is not supported, and whichever backend opens it first wins.
|
|
79
83
|
*/
|
|
80
84
|
getStorageBackendCacheKey(): string;
|
|
85
|
+
/** Short digest of the API base URL and token — identifies the credentials a storage was opened with. */
|
|
86
|
+
private credentialsHash;
|
|
81
87
|
storageExists(id: string, type: StorageType): Promise<boolean>;
|
|
82
88
|
createDatasetBackend(options?: StorageIdentifier): Promise<DatasetBackend>;
|
|
83
89
|
createKeyValueStoreBackend(options?: StorageIdentifier): Promise<KeyValueStoreBackend>;
|
|
@@ -97,11 +103,27 @@ export declare class ApifyStorageBackend implements StorageBackend {
|
|
|
97
103
|
* Resolves a crawlee {@link StorageIdentifier} to a platform storage id.
|
|
98
104
|
*
|
|
99
105
|
* Aliases resolve to unnamed storages: the reserved `__default__` alias maps to the run's
|
|
100
|
-
* default storage, and
|
|
101
|
-
* `ACTOR_STORAGES_JSON` environment variable
|
|
102
|
-
*
|
|
106
|
+
* default storage, and an alias declared in the Actor's schema to the storage the platform
|
|
107
|
+
* created for it (via the `ACTOR_STORAGES_JSON` environment variable). Any other alias gets an
|
|
108
|
+
* unnamed storage of its own — crawlee mints aliases at runtime, one per extra crawler instance
|
|
109
|
+
* and one per throttled domain, so an undeclared alias is not an error.
|
|
103
110
|
*/
|
|
104
111
|
private resolveId;
|
|
112
|
+
/**
|
|
113
|
+
* Returns the unnamed storage backing `alias`, creating it on first use.
|
|
114
|
+
*
|
|
115
|
+
* On the platform the mapping is persisted, so a migrated run reopens the same storages rather
|
|
116
|
+
* than empty ones — aliased request queues hold live requests. Serialized, so one alias means
|
|
117
|
+
* one storage and the mapping's read-modify-write cannot drop entries.
|
|
118
|
+
*/
|
|
119
|
+
private resolveAliasId;
|
|
120
|
+
/**
|
|
121
|
+
* Re-reads the record first, so a second backend in this process does not drop its entries.
|
|
122
|
+
* Logged rather than thrown: a lost entry only costs a re-created storage after a migration.
|
|
123
|
+
*/
|
|
124
|
+
private persistAliasId;
|
|
125
|
+
/** The run's default key-value store, where the mapping lives — `undefined` off the platform. */
|
|
126
|
+
private aliasMappingStore;
|
|
105
127
|
/** Looks an alias up in the Actor's schema storages (the `ACTOR_STORAGES_JSON` env var). */
|
|
106
128
|
private aliasFromActorStorages;
|
|
107
129
|
private resourceClient;
|
|
@@ -2,14 +2,22 @@
|
|
|
2
2
|
import { AsyncLocalStorage } from 'node:async_hooks';
|
|
3
3
|
import { createHash } from 'node:crypto';
|
|
4
4
|
import { DatasetClient as ApifyDatasetClient } from 'apify-client';
|
|
5
|
+
import log from '@apify/log';
|
|
5
6
|
import { cryptoRandomObjectId } from '@apify/utilities';
|
|
6
7
|
import { ApifyDatasetBackend } from './apify_dataset_backend.js';
|
|
7
8
|
import { ApifyKeyValueStoreBackend } from './apify_key_value_store_backend.js';
|
|
9
|
+
import { AsyncLock } from './apify_request_queue_backend.js';
|
|
8
10
|
import { ApifyRequestQueueSharedBackend } from './apify_request_queue_shared_backend.js';
|
|
9
11
|
import { ApifyRequestQueueSingleBackend } from './apify_request_queue_single_backend.js';
|
|
10
12
|
import { DEFAULT_DATASET_ITEM_EVENT, mergeChargeResults, pushDataAndCharge, } from './charging.js';
|
|
11
13
|
/** The reserved alias crawlee uses for the default (unnamed) storage. */
|
|
12
14
|
const DEFAULT_STORAGE_ALIAS = '__default__';
|
|
15
|
+
/** The key of the default key-value store record holding this run's alias -> storage id mapping. */
|
|
16
|
+
const ALIAS_MAPPING_RECORD_KEY = '__STORAGE_ALIASES_MAPPING';
|
|
17
|
+
async function readAliasMapping(store) {
|
|
18
|
+
const record = await store.getRecord(ALIAS_MAPPING_RECORD_KEY);
|
|
19
|
+
return record?.value ?? {};
|
|
20
|
+
}
|
|
13
21
|
/** The maximum clientKey length accepted by the request queue API. */
|
|
14
22
|
const MAX_CLIENT_KEY_LENGTH = 32;
|
|
15
23
|
const DEFAULT_ID_CONFIG_KEY = {
|
|
@@ -95,8 +103,12 @@ export class ApifyStorageBackend {
|
|
|
95
103
|
config;
|
|
96
104
|
requestQueueAccess;
|
|
97
105
|
getChargingManager;
|
|
98
|
-
/** Unnamed storages
|
|
106
|
+
/** Unnamed storages resolved for aliases in this process, keyed like {@link AliasMapping}. */
|
|
99
107
|
aliasIdCache = new Map();
|
|
108
|
+
/** The alias mapping read from the run's default key-value store; `undefined` until first read. */
|
|
109
|
+
persistedAliasIds;
|
|
110
|
+
/** Serializes alias resolution — see `resolveAliasId`. */
|
|
111
|
+
aliasLock = new AsyncLock();
|
|
100
112
|
/** Fallback request queue client key when the run id is unavailable — one per backend. */
|
|
101
113
|
fallbackClientKey;
|
|
102
114
|
constructor(client, options = {}) {
|
|
@@ -112,11 +124,14 @@ export class ApifyStorageBackend {
|
|
|
112
124
|
* and `shared` mode at once is not supported, and whichever backend opens it first wins.
|
|
113
125
|
*/
|
|
114
126
|
getStorageBackendCacheKey() {
|
|
115
|
-
|
|
127
|
+
return `ApifyStorageBackend:${this.credentialsHash()}`;
|
|
128
|
+
}
|
|
129
|
+
/** Short digest of the API base URL and token — identifies the credentials a storage was opened with. */
|
|
130
|
+
credentialsHash() {
|
|
131
|
+
return createHash('sha256')
|
|
116
132
|
.update(`${this.client.publicBaseUrl}${this.client.token ?? ''}`)
|
|
117
133
|
.digest('hex')
|
|
118
134
|
.slice(0, 8);
|
|
119
|
-
return `ApifyStorageBackend:${hash}`;
|
|
120
135
|
}
|
|
121
136
|
async storageExists(id, type) {
|
|
122
137
|
// Lets `Dataset.open(idOrName)` and friends resolve a string to an id first (when one
|
|
@@ -183,9 +198,10 @@ export class ApifyStorageBackend {
|
|
|
183
198
|
* Resolves a crawlee {@link StorageIdentifier} to a platform storage id.
|
|
184
199
|
*
|
|
185
200
|
* Aliases resolve to unnamed storages: the reserved `__default__` alias maps to the run's
|
|
186
|
-
* default storage, and
|
|
187
|
-
* `ACTOR_STORAGES_JSON` environment variable
|
|
188
|
-
*
|
|
201
|
+
* default storage, and an alias declared in the Actor's schema to the storage the platform
|
|
202
|
+
* created for it (via the `ACTOR_STORAGES_JSON` environment variable). Any other alias gets an
|
|
203
|
+
* unnamed storage of its own — crawlee mints aliases at runtime, one per extra crawler instance
|
|
204
|
+
* and one per throttled domain, so an undeclared alias is not an error.
|
|
189
205
|
*/
|
|
190
206
|
async resolveId(options, type) {
|
|
191
207
|
if (options?.id)
|
|
@@ -203,20 +219,57 @@ export class ApifyStorageBackend {
|
|
|
203
219
|
const declaredId = this.aliasFromActorStorages(alias, type);
|
|
204
220
|
if (declaredId)
|
|
205
221
|
return declaredId;
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
222
|
+
}
|
|
223
|
+
return this.resolveAliasId(alias, type);
|
|
224
|
+
}
|
|
225
|
+
/**
|
|
226
|
+
* Returns the unnamed storage backing `alias`, creating it on first use.
|
|
227
|
+
*
|
|
228
|
+
* On the platform the mapping is persisted, so a migrated run reopens the same storages rather
|
|
229
|
+
* than empty ones — aliased request queues hold live requests. Serialized, so one alias means
|
|
230
|
+
* one storage and the mapping's read-modify-write cannot drop entries.
|
|
231
|
+
*/
|
|
232
|
+
async resolveAliasId(alias, type) {
|
|
233
|
+
// The credentials are part of the key, so the same alias opened through two
|
|
234
|
+
// differently-authenticated backends maps to two storages.
|
|
235
|
+
const key = [type, alias, this.credentialsHash()].join(',');
|
|
236
|
+
return this.aliasLock.runExclusive(async () => {
|
|
237
|
+
const knownId = this.aliasIdCache.get(key);
|
|
238
|
+
if (knownId)
|
|
239
|
+
return knownId;
|
|
240
|
+
const store = this.aliasMappingStore();
|
|
241
|
+
this.persistedAliasIds ??= store ? await readAliasMapping(store) : {};
|
|
242
|
+
// A persisted id can point at a storage the user has since deleted.
|
|
243
|
+
const persistedId = this.persistedAliasIds[key];
|
|
244
|
+
if (persistedId && (await this.resourceClient(persistedId, type).get())) {
|
|
245
|
+
this.aliasIdCache.set(key, persistedId);
|
|
246
|
+
return persistedId;
|
|
209
247
|
}
|
|
248
|
+
const { id } = await this.collectionClient(type).getOrCreate();
|
|
249
|
+
this.aliasIdCache.set(key, id);
|
|
250
|
+
if (store)
|
|
251
|
+
await this.persistAliasId(store, key, id);
|
|
252
|
+
return id;
|
|
253
|
+
});
|
|
254
|
+
}
|
|
255
|
+
/**
|
|
256
|
+
* Re-reads the record first, so a second backend in this process does not drop its entries.
|
|
257
|
+
* Logged rather than thrown: a lost entry only costs a re-created storage after a migration.
|
|
258
|
+
*/
|
|
259
|
+
async persistAliasId(store, key, id) {
|
|
260
|
+
try {
|
|
261
|
+
const mapping = await readAliasMapping(store);
|
|
262
|
+
mapping[key] = id;
|
|
263
|
+
await store.setRecord({ key: ALIAS_MAPPING_RECORD_KEY, value: mapping });
|
|
264
|
+
this.persistedAliasIds = mapping;
|
|
265
|
+
}
|
|
266
|
+
catch (error) {
|
|
267
|
+
log.warning(`Failed to persist the storage alias mapping: ${error.message}`);
|
|
210
268
|
}
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
if (cachedId)
|
|
216
|
-
return cachedId;
|
|
217
|
-
const created = await this.collectionClient(type).getOrCreate();
|
|
218
|
-
this.aliasIdCache.set(cacheKey, created.id);
|
|
219
|
-
return created.id;
|
|
269
|
+
}
|
|
270
|
+
/** The run's default key-value store, where the mapping lives — `undefined` off the platform. */
|
|
271
|
+
aliasMappingStore() {
|
|
272
|
+
return this.config?.isAtHome ? this.client.keyValueStore(this.config.defaultKeyValueStoreId) : undefined;
|
|
220
273
|
}
|
|
221
274
|
/** Looks an alias up in the Actor's schema storages (the `ACTOR_STORAGES_JSON` env var). */
|
|
222
275
|
aliasFromActorStorages(alias, type) {
|
package/dist/index.d.ts
CHANGED
|
@@ -3,7 +3,7 @@ export { ApifyStorageBackend, type ApifyStorageBackendOptions } from './apify_st
|
|
|
3
3
|
export type { RequestQueueAccessMode } from './apify_request_queue_backend.js';
|
|
4
4
|
export { ArgumentValidationError } from './utils.js';
|
|
5
5
|
export { createTransformRequestFunction, type GlobInput, type PseudoUrlInput, type UrlPatternFilters, type UrlPatternRequestOptions, } from './enqueue_links_filters.js';
|
|
6
|
-
export type { OpenStorageOptions, StorageAlias, StorageId, StorageName, StorageIdentifier
|
|
6
|
+
export type { OpenStorageOptions, StorageAlias, StorageId, StorageName, StorageIdentifier } from './storage.js';
|
|
7
7
|
export { ChargeOptions, ChargeResult, ActorPricingInfo, ChargingManager } from './charging.js';
|
|
8
8
|
export * from './configuration.js';
|
|
9
9
|
export * from './proxy_configuration.js';
|
package/dist/storage.d.ts
CHANGED
|
@@ -1,6 +1,3 @@
|
|
|
1
|
-
import type { IStorage, StorageOpenOptions } from '@crawlee/core';
|
|
2
|
-
import type { Constructor, StorageBackend } from '@crawlee/types';
|
|
3
|
-
import type { Configuration } from './configuration.js';
|
|
4
1
|
export interface OpenStorageOptions {
|
|
5
2
|
/**
|
|
6
3
|
* If set to `true` then the cloud storage is used even if the `CRAWLEE_STORAGE_DIR`
|
|
@@ -10,8 +7,11 @@ export interface OpenStorageOptions {
|
|
|
10
7
|
forceCloud?: boolean;
|
|
11
8
|
}
|
|
12
9
|
/**
|
|
13
|
-
* Identifies a storage by its alias
|
|
14
|
-
*
|
|
10
|
+
* Identifies a run-scoped storage by its alias.
|
|
11
|
+
*
|
|
12
|
+
* An alias declared in the Actor's schema storages (the `ACTOR_STORAGES_JSON` environment variable)
|
|
13
|
+
* resolves to the storage the platform created for it; any other alias gets an unnamed storage of
|
|
14
|
+
* its own, for this run only.
|
|
15
15
|
*/
|
|
16
16
|
export interface StorageAlias {
|
|
17
17
|
alias: string;
|
|
@@ -31,28 +31,8 @@ export interface StorageName {
|
|
|
31
31
|
/**
|
|
32
32
|
* Identifies a storage to open. Can be:
|
|
33
33
|
* - A plain `string` for backward compatibility (treated as ID or name)
|
|
34
|
-
* - `{ alias: string }` to
|
|
34
|
+
* - `{ alias: string }` to open a run-scoped storage — see {@link StorageAlias}
|
|
35
35
|
* - `{ id: string }` to open by explicit platform ID
|
|
36
36
|
* - `{ name: string }` to open by explicit name
|
|
37
37
|
*/
|
|
38
38
|
export type StorageIdentifier = string | StorageAlias | StorageId | StorageName;
|
|
39
|
-
/**
|
|
40
|
-
* Identifies a storage to open, without alias support.
|
|
41
|
-
* Used for key-value stores and request queues, which do not support aliases.
|
|
42
|
-
* Can be:
|
|
43
|
-
* - A plain `string` for backward compatibility (treated as ID or name)
|
|
44
|
-
* - `{ id: string }` to open by explicit platform ID
|
|
45
|
-
* - `{ name: string }` to open by explicit name
|
|
46
|
-
*/
|
|
47
|
-
export type StorageIdentifierWithoutAlias = string | StorageId | StorageName;
|
|
48
|
-
export interface OpenStorageContext {
|
|
49
|
-
config: Configuration;
|
|
50
|
-
backend?: StorageBackend;
|
|
51
|
-
purgedStorageAliases: Set<string>;
|
|
52
|
-
}
|
|
53
|
-
/**
|
|
54
|
-
* Opens a storage by its identifier, handling Apify alias resolution and local purging.
|
|
55
|
-
*/
|
|
56
|
-
export declare function openStorage<T extends IStorage>(storageClass: Constructor<T> & {
|
|
57
|
-
open(id?: string | null, options?: StorageOpenOptions): Promise<T>;
|
|
58
|
-
}, identifier: StorageIdentifier | null | undefined, context: OpenStorageContext): Promise<T>;
|
package/dist/storage.js
CHANGED
|
@@ -1,79 +1 @@
|
|
|
1
|
-
|
|
2
|
-
const STORAGE_TYPE_KEYS = {
|
|
3
|
-
Dataset: 'datasets',
|
|
4
|
-
KeyValueStore: 'keyValueStores',
|
|
5
|
-
RequestQueue: 'requestQueues',
|
|
6
|
-
};
|
|
7
|
-
const parsedStoragesJson = new Map();
|
|
8
|
-
/**
|
|
9
|
-
* Resolves a {@link StorageIdentifier} to a plain string ID or name
|
|
10
|
-
* that can be passed to crawlee v4's `<Storage>.open()`.
|
|
11
|
-
*/
|
|
12
|
-
function resolveStorageIdentifier(storageType, identifier, config) {
|
|
13
|
-
if (identifier === null || identifier === undefined) {
|
|
14
|
-
return undefined;
|
|
15
|
-
}
|
|
16
|
-
if (typeof identifier === 'string') {
|
|
17
|
-
return identifier;
|
|
18
|
-
}
|
|
19
|
-
if ('id' in identifier) {
|
|
20
|
-
return identifier.id;
|
|
21
|
-
}
|
|
22
|
-
if ('name' in identifier) {
|
|
23
|
-
return identifier.name;
|
|
24
|
-
}
|
|
25
|
-
// { alias: string }
|
|
26
|
-
const storagesJson = config.actorStoragesJson;
|
|
27
|
-
if (config.isAtHome && storagesJson) {
|
|
28
|
-
let storages;
|
|
29
|
-
try {
|
|
30
|
-
if (!parsedStoragesJson.has(storagesJson)) {
|
|
31
|
-
parsedStoragesJson.set(storagesJson, JSON.parse(storagesJson));
|
|
32
|
-
}
|
|
33
|
-
storages = parsedStoragesJson.get(storagesJson);
|
|
34
|
-
}
|
|
35
|
-
catch {
|
|
36
|
-
throw new Error(`Failed to parse ACTOR_STORAGES_JSON environment variable: ${storagesJson}`);
|
|
37
|
-
}
|
|
38
|
-
const typeKey = STORAGE_TYPE_KEYS[storageType];
|
|
39
|
-
const resolvedId = storages[typeKey]?.[identifier.alias];
|
|
40
|
-
if (resolvedId) {
|
|
41
|
-
return resolvedId;
|
|
42
|
-
}
|
|
43
|
-
throw new Error(`Storage alias "${identifier.alias}" not found in ACTOR_STORAGES_JSON for storage type "${storageType}". ` +
|
|
44
|
-
`Available aliases: ${Object.keys(storages[typeKey] ?? {}).join(', ') || '(none)'}`);
|
|
45
|
-
}
|
|
46
|
-
// When using local storage, just use the alias as a name.
|
|
47
|
-
// When using platform storage, we can't just make up a name — the alias must be
|
|
48
|
-
// in ACTOR_STORAGES_JSON.
|
|
49
|
-
if (config.isAtHome) {
|
|
50
|
-
throw new Error(`Storage alias "${identifier.alias}" cannot be resolved because ACTOR_STORAGES_JSON is not set. ` +
|
|
51
|
-
`Aliases are only available for storages declared in the Actor's schema.`);
|
|
52
|
-
}
|
|
53
|
-
return identifier.alias;
|
|
54
|
-
}
|
|
55
|
-
/**
|
|
56
|
-
* Opens a storage by its identifier, handling Apify alias resolution and local purging.
|
|
57
|
-
*/
|
|
58
|
-
export async function openStorage(storageClass, identifier, context) {
|
|
59
|
-
const isAlias = identifier !== null && identifier !== undefined && typeof identifier === 'object' && 'alias' in identifier;
|
|
60
|
-
if (isAlias && !context.config.isAtHome && context.backend instanceof ApifyStorageBackend) {
|
|
61
|
-
throw new Error('The `alias` option is not allowed for Apify-based storages running outside of Apify');
|
|
62
|
-
}
|
|
63
|
-
const resolvedIdOrName = resolveStorageIdentifier(storageClass.name, identifier, context.config);
|
|
64
|
-
// When running locally, purge aliased storages on first open
|
|
65
|
-
// (similar to how crawlee purges default storages on start).
|
|
66
|
-
if (isAlias &&
|
|
67
|
-
!context.config.isAtHome &&
|
|
68
|
-
context.config.purgeOnStart &&
|
|
69
|
-
!context.purgedStorageAliases.has(identifier.alias)) {
|
|
70
|
-
context.purgedStorageAliases.add(identifier.alias);
|
|
71
|
-
const existingStorage = await storageClass.open(resolvedIdOrName ?? null, {
|
|
72
|
-
storageBackend: context.backend,
|
|
73
|
-
});
|
|
74
|
-
await existingStorage.drop();
|
|
75
|
-
}
|
|
76
|
-
return storageClass.open(resolvedIdOrName ?? null, {
|
|
77
|
-
storageBackend: context.backend,
|
|
78
|
-
});
|
|
79
|
-
}
|
|
1
|
+
export {};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "apify",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.31",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.0.0"
|