apify 4.0.0-beta.30 → 4.0.0-beta.31

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/actor.d.ts CHANGED
@@ -11,7 +11,7 @@ import type { ConfigurationOptions } from './configuration.js';
11
11
  import { Configuration } from './configuration.js';
12
12
  import type { ProxyConfigurationOptions } from './proxy_configuration.js';
13
13
  import { ProxyConfiguration } from './proxy_configuration.js';
14
- import type { OpenStorageOptions, StorageIdentifier, StorageIdentifierWithoutAlias } from './storage.js';
14
+ import type { OpenStorageOptions, StorageIdentifier } from './storage.js';
15
15
  export interface InitOptions {
16
16
  storage?: StorageBackend;
17
17
  /**
@@ -358,12 +358,6 @@ export declare class Actor<Data extends Dictionary = Dictionary> {
358
358
  */
359
359
  private gracefulShutdownHandlers;
360
360
  private chargingManager;
361
- /**
362
- * Tracks which aliased storages have been purged during this session,
363
- * so we only purge them once (on first open) when running locally.
364
- * @internal
365
- */
366
- purgedStorageAliases: Set<string>;
367
361
  /** How Apify platform request queues are consumed; set from {@link InitOptions.requestQueueAccess}. */
368
362
  private requestQueueAccess;
369
363
  constructor(options?: ActorOptions);
@@ -623,8 +617,8 @@ export declare class Actor<Data extends Dictionary = Dictionary> {
623
617
  * @param [datasetIdOrName]
624
618
  * ID, name, or alias of the dataset to be opened. If `null` or `undefined`,
625
619
  * the function returns the default dataset associated with the Actor run.
626
- * You can also pass `{ alias: 'name' }` to open a dataset defined in the Actor's schema storages,
627
- * `{ id: 'abc' }` to open by explicit ID, or `{ name: 'abc' }` to open by explicit name.
620
+ * You can also pass `{ alias: 'abc' }` to open a run-scoped storage, `{ id: 'abc' }` to open by
621
+ * explicit ID, or `{ name: 'abc' }` to open by explicit name.
628
622
  * @param [options]
629
623
  * @ignore
630
624
  */
@@ -735,13 +729,14 @@ export declare class Actor<Data extends Dictionary = Dictionary> {
735
729
  * For more details and code examples, see the {@link KeyValueStore} class.
736
730
  *
737
731
  * @param [storeIdOrName]
738
- * ID or name of the key-value store to be opened. If `null` or `undefined`,
732
+ * ID, name, or alias of the key-value store to be opened. If `null` or `undefined`,
739
733
  * the function returns the default key-value store associated with the Actor run.
740
- * You can also pass `{ id: 'abc' }` to open by explicit ID, or `{ name: 'abc' }` to open by explicit name.
734
+ * You can also pass `{ alias: 'abc' }` to open a run-scoped storage, `{ id: 'abc' }` to open by
735
+ * explicit ID, or `{ name: 'abc' }` to open by explicit name.
741
736
  * @param [options]
742
737
  * @ignore
743
738
  */
744
- openKeyValueStore(storeIdOrName?: StorageIdentifierWithoutAlias | null, options?: OpenStorageOptions): Promise<KeyValueStore>;
739
+ openKeyValueStore(storeIdOrName?: StorageIdentifier | null, options?: OpenStorageOptions): Promise<KeyValueStore>;
745
740
  /**
746
741
  * Opens a request queue and returns a promise resolving to an instance
747
742
  * of the {@link RequestQueue} class.
@@ -754,13 +749,14 @@ export declare class Actor<Data extends Dictionary = Dictionary> {
754
749
  * For more details and code examples, see the {@link RequestQueue} class.
755
750
  *
756
751
  * @param [queueIdOrName]
757
- * ID or name of the request queue to be opened. If `null` or `undefined`,
752
+ * ID, name, or alias of the request queue to be opened. If `null` or `undefined`,
758
753
  * the function returns the default request queue associated with the Actor run.
759
- * You can also pass `{ id: 'abc' }` to open by explicit ID, or `{ name: 'abc' }` to open by explicit name.
754
+ * You can also pass `{ alias: 'abc' }` to open a run-scoped storage, `{ id: 'abc' }` to open by
755
+ * explicit ID, or `{ name: 'abc' }` to open by explicit name.
760
756
  * @param [options]
761
757
  * @ignore
762
758
  */
763
- openRequestQueue(queueIdOrName?: StorageIdentifierWithoutAlias | null, options?: OpenStorageOptions): Promise<RequestQueue>;
759
+ openRequestQueue(queueIdOrName?: StorageIdentifier | null, options?: OpenStorageOptions): Promise<RequestQueue>;
764
760
  /**
765
761
  * Creates a proxy configuration and returns a promise resolving to an instance
766
762
  * of the {@link ProxyConfiguration} class that is already initialized.
@@ -1168,8 +1164,8 @@ export declare class Actor<Data extends Dictionary = Dictionary> {
1168
1164
  * @param [datasetIdOrName]
1169
1165
  * ID, name, or alias of the dataset to be opened. If `null` or `undefined`,
1170
1166
  * the function returns the default dataset associated with the Actor run.
1171
- * You can also pass `{ alias: 'name' }` to open a dataset defined in the Actor's schema storages,
1172
- * `{ id: 'abc' }` to open by explicit ID, or `{ name: 'abc' }` to open by explicit name.
1167
+ * You can also pass `{ alias: 'abc' }` to open a run-scoped storage, `{ id: 'abc' }` to open by
1168
+ * explicit ID, or `{ name: 'abc' }` to open by explicit name.
1173
1169
  * @param [options]
1174
1170
  */
1175
1171
  static openDataset<Data extends Dictionary = Dictionary>(datasetIdOrName?: StorageIdentifier | null, options?: OpenStorageOptions): Promise<Dataset<Data>>;
@@ -1280,7 +1276,7 @@ export declare class Actor<Data extends Dictionary = Dictionary> {
1280
1276
  * You can also pass `{ id: 'abc' }` to open by explicit ID, or `{ name: 'abc' }` to open by explicit name.
1281
1277
  * @param [options]
1282
1278
  */
1283
- static openKeyValueStore(storeIdOrName?: StorageIdentifierWithoutAlias | null, options?: OpenStorageOptions): Promise<KeyValueStore>;
1279
+ static openKeyValueStore(storeIdOrName?: StorageIdentifier | null, options?: OpenStorageOptions): Promise<KeyValueStore>;
1284
1280
  /**
1285
1281
  * Opens a request queue and returns a promise resolving to an instance
1286
1282
  * of the {@link RequestQueue} class.
@@ -1298,7 +1294,7 @@ export declare class Actor<Data extends Dictionary = Dictionary> {
1298
1294
  * You can also pass `{ id: 'abc' }` to open by explicit ID, or `{ name: 'abc' }` to open by explicit name.
1299
1295
  * @param [options]
1300
1296
  */
1301
- static openRequestQueue(queueIdOrName?: StorageIdentifierWithoutAlias | null, options?: OpenStorageOptions): Promise<RequestQueue>;
1297
+ static openRequestQueue(queueIdOrName?: StorageIdentifier | null, options?: OpenStorageOptions): Promise<RequestQueue>;
1302
1298
  /**
1303
1299
  * Creates a proxy configuration and returns a promise resolving to an instance
1304
1300
  * of the {@link ProxyConfiguration} class that is already initialized.
@@ -1390,7 +1386,6 @@ export declare class Actor<Data extends Dictionary = Dictionary> {
1390
1386
  private usesPushDataInterception;
1391
1387
  private pushDataViaInterceptedClient;
1392
1388
  private pushDataWithExplicitCharging;
1393
- private _openStorage;
1394
1389
  private createApifyStorageBackend;
1395
1390
  private _ensureActorInit;
1396
1391
  /**
package/dist/actor.js CHANGED
@@ -14,7 +14,6 @@ import { Configuration } from './configuration.js';
14
14
  import { getDefaultsFromInputSchema, noActorInputSchemaDefinedMarker, readInputSchema } from './input-schemas.js';
15
15
  import { PlatformEventManager } from './platform_event_manager.js';
16
16
  import { ProxyConfiguration } from './proxy_configuration.js';
17
- import { openStorage } from './storage.js';
18
17
  import { checkCrawleeVersion, getSystemInfo, isNonEmptyObject, printOutdatedSdkWarning, snakeCaseToCamelCase, } from './utils.js';
19
18
  /**
20
19
  * Exit codes for the Actor process.
@@ -76,12 +75,6 @@ export class Actor {
76
75
  */
77
76
  #statusMessageForwarder;
78
77
  chargingManager;
79
- /**
80
- * Tracks which aliased storages have been purged during this session,
81
- * so we only purge them once (on first open) when running locally.
82
- * @internal
83
- */
84
- purgedStorageAliases = new Set();
85
78
  /** How Apify platform request queues are consumed; set from {@link InitOptions.requestQueueAccess}. */
86
79
  requestQueueAccess = 'single';
87
80
  constructor(options = {}) {
@@ -674,15 +667,17 @@ export class Actor {
674
667
  * @param [datasetIdOrName]
675
668
  * ID, name, or alias of the dataset to be opened. If `null` or `undefined`,
676
669
  * the function returns the default dataset associated with the Actor run.
677
- * You can also pass `{ alias: 'name' }` to open a dataset defined in the Actor's schema storages,
678
- * `{ id: 'abc' }` to open by explicit ID, or `{ name: 'abc' }` to open by explicit name.
670
+ * You can also pass `{ alias: 'abc' }` to open a run-scoped storage, `{ id: 'abc' }` to open by
671
+ * explicit ID, or `{ name: 'abc' }` to open by explicit name.
679
672
  * @param [options]
680
673
  * @ignore
681
674
  */
682
675
  async openDataset(datasetIdOrName, options = {}) {
683
676
  parseArgument(options, z.object({ forceCloud: z.boolean().optional() }).strict());
684
677
  this._ensureActorInit('openDataset');
685
- return this._openStorage(Dataset, datasetIdOrName, options);
678
+ return Dataset.open(datasetIdOrName ?? null, {
679
+ storageBackend: options.forceCloud ? this.createApifyStorageBackend() : undefined,
680
+ });
686
681
  }
687
682
  /**
688
683
  * Gets a value from the default {@link KeyValueStore} associated with the current Actor run.
@@ -820,16 +815,19 @@ export class Actor {
820
815
  * For more details and code examples, see the {@link KeyValueStore} class.
821
816
  *
822
817
  * @param [storeIdOrName]
823
- * ID or name of the key-value store to be opened. If `null` or `undefined`,
818
+ * ID, name, or alias of the key-value store to be opened. If `null` or `undefined`,
824
819
  * the function returns the default key-value store associated with the Actor run.
825
- * You can also pass `{ id: 'abc' }` to open by explicit ID, or `{ name: 'abc' }` to open by explicit name.
820
+ * You can also pass `{ alias: 'abc' }` to open a run-scoped storage, `{ id: 'abc' }` to open by
821
+ * explicit ID, or `{ name: 'abc' }` to open by explicit name.
826
822
  * @param [options]
827
823
  * @ignore
828
824
  */
829
825
  async openKeyValueStore(storeIdOrName, options = {}) {
830
826
  parseArgument(options, z.object({ forceCloud: z.boolean().optional() }).strict());
831
827
  this._ensureActorInit('openKeyValueStore');
832
- return this._openStorage(KeyValueStore, storeIdOrName, options);
828
+ return KeyValueStore.open(storeIdOrName ?? null, {
829
+ storageBackend: options.forceCloud ? this.createApifyStorageBackend() : undefined,
830
+ });
833
831
  }
834
832
  /**
835
833
  * Opens a request queue and returns a promise resolving to an instance
@@ -843,17 +841,19 @@ export class Actor {
843
841
  * For more details and code examples, see the {@link RequestQueue} class.
844
842
  *
845
843
  * @param [queueIdOrName]
846
- * ID or name of the request queue to be opened. If `null` or `undefined`,
844
+ * ID, name, or alias of the request queue to be opened. If `null` or `undefined`,
847
845
  * the function returns the default request queue associated with the Actor run.
848
- * You can also pass `{ id: 'abc' }` to open by explicit ID, or `{ name: 'abc' }` to open by explicit name.
846
+ * You can also pass `{ alias: 'abc' }` to open a run-scoped storage, `{ id: 'abc' }` to open by
847
+ * explicit ID, or `{ name: 'abc' }` to open by explicit name.
849
848
  * @param [options]
850
849
  * @ignore
851
850
  */
852
851
  async openRequestQueue(queueIdOrName, options = {}) {
853
852
  parseArgument(options, z.object({ forceCloud: z.boolean().optional() }).strict());
854
853
  this._ensureActorInit('openRequestQueue');
855
- const queue = await this._openStorage(RequestQueue, queueIdOrName, options);
856
- return queue;
854
+ return RequestQueue.open(queueIdOrName ?? null, {
855
+ storageBackend: options.forceCloud ? this.createApifyStorageBackend() : undefined,
856
+ });
857
857
  }
858
858
  /**
859
859
  * Creates a proxy configuration and returns a promise resolving to an instance
@@ -1343,8 +1343,8 @@ export class Actor {
1343
1343
  * @param [datasetIdOrName]
1344
1344
  * ID, name, or alias of the dataset to be opened. If `null` or `undefined`,
1345
1345
  * the function returns the default dataset associated with the Actor run.
1346
- * You can also pass `{ alias: 'name' }` to open a dataset defined in the Actor's schema storages,
1347
- * `{ id: 'abc' }` to open by explicit ID, or `{ name: 'abc' }` to open by explicit name.
1346
+ * You can also pass `{ alias: 'abc' }` to open a run-scoped storage, `{ id: 'abc' }` to open by
1347
+ * explicit ID, or `{ name: 'abc' }` to open by explicit name.
1348
1348
  * @param [options]
1349
1349
  */
1350
1350
  static async openDataset(datasetIdOrName, options = {}) {
@@ -1631,13 +1631,6 @@ export class Actor {
1631
1631
  pushFn: async (limitedItems) => dataset.pushData(limitedItems),
1632
1632
  });
1633
1633
  }
1634
- async _openStorage(storageClass, identifier, options = {}) {
1635
- return openStorage(storageClass, identifier, {
1636
- config: this.configuration,
1637
- backend: options.forceCloud ? this.createApifyStorageBackend() : undefined,
1638
- purgedStorageAliases: this.purgedStorageAliases,
1639
- });
1640
- }
1641
1634
  createApifyStorageBackend() {
1642
1635
  return new ApifyStorageBackend(this.apifyClient, {
1643
1636
  configuration: this.configuration,
@@ -1,7 +1,7 @@
1
1
  import { AsyncLocalStorage } from 'node:async_hooks';
2
2
  import type { DatasetBackend, KeyValueStoreBackend, RequestQueueBackend, StorageBackend, StorageIdentifier } from '@crawlee/types';
3
3
  import type { ApifyClient } from 'apify-client';
4
- import type { RequestQueueAccessMode } from './apify_request_queue_backend.js';
4
+ import { type RequestQueueAccessMode } from './apify_request_queue_backend.js';
5
5
  import { type ChargeResult, type ChargingManager } from './charging.js';
6
6
  import type { Configuration } from './configuration.js';
7
7
  type StorageType = 'Dataset' | 'KeyValueStore' | 'RequestQueue';
@@ -66,8 +66,12 @@ export declare class ApifyStorageBackend implements StorageBackend {
66
66
  private readonly config?;
67
67
  private readonly requestQueueAccess;
68
68
  private readonly getChargingManager?;
69
- /** Unnamed storages created for aliases in this process, so an alias maps to one storage. */
69
+ /** Unnamed storages resolved for aliases in this process, keyed like {@link AliasMapping}. */
70
70
  private readonly aliasIdCache;
71
+ /** The alias mapping read from the run's default key-value store; `undefined` until first read. */
72
+ private persistedAliasIds?;
73
+ /** Serializes alias resolution — see `resolveAliasId`. */
74
+ private readonly aliasLock;
71
75
  /** Fallback request queue client key when the run id is unavailable — one per backend. */
72
76
  private fallbackClientKey?;
73
77
  constructor(client: ApifyClient, options?: ApifyStorageBackendOptions);
@@ -78,6 +82,8 @@ export declare class ApifyStorageBackend implements StorageBackend {
78
82
  * and `shared` mode at once is not supported, and whichever backend opens it first wins.
79
83
  */
80
84
  getStorageBackendCacheKey(): string;
85
+ /** Short digest of the API base URL and token — identifies the credentials a storage was opened with. */
86
+ private credentialsHash;
81
87
  storageExists(id: string, type: StorageType): Promise<boolean>;
82
88
  createDatasetBackend(options?: StorageIdentifier): Promise<DatasetBackend>;
83
89
  createKeyValueStoreBackend(options?: StorageIdentifier): Promise<KeyValueStoreBackend>;
@@ -97,11 +103,27 @@ export declare class ApifyStorageBackend implements StorageBackend {
97
103
  * Resolves a crawlee {@link StorageIdentifier} to a platform storage id.
98
104
  *
99
105
  * Aliases resolve to unnamed storages: the reserved `__default__` alias maps to the run's
100
- * default storage, and other aliases to the storages declared in the Actor's schema (via the
101
- * `ACTOR_STORAGES_JSON` environment variable, maintained by the platform). Outside the
102
- * platform, an unnamed storage is created per alias instead (remembered for this process only).
106
+ * default storage, and an alias declared in the Actor's schema to the storage the platform
107
+ * created for it (via the `ACTOR_STORAGES_JSON` environment variable). Any other alias gets an
108
+ * unnamed storage of its own crawlee mints aliases at runtime, one per extra crawler instance
109
+ * and one per throttled domain, so an undeclared alias is not an error.
103
110
  */
104
111
  private resolveId;
112
+ /**
113
+ * Returns the unnamed storage backing `alias`, creating it on first use.
114
+ *
115
+ * On the platform the mapping is persisted, so a migrated run reopens the same storages rather
116
+ * than empty ones — aliased request queues hold live requests. Serialized, so one alias means
117
+ * one storage and the mapping's read-modify-write cannot drop entries.
118
+ */
119
+ private resolveAliasId;
120
+ /**
121
+ * Re-reads the record first, so a second backend in this process does not drop its entries.
122
+ * Logged rather than thrown: a lost entry only costs a re-created storage after a migration.
123
+ */
124
+ private persistAliasId;
125
+ /** The run's default key-value store, where the mapping lives — `undefined` off the platform. */
126
+ private aliasMappingStore;
105
127
  /** Looks an alias up in the Actor's schema storages (the `ACTOR_STORAGES_JSON` env var). */
106
128
  private aliasFromActorStorages;
107
129
  private resourceClient;
@@ -2,14 +2,22 @@
2
2
  import { AsyncLocalStorage } from 'node:async_hooks';
3
3
  import { createHash } from 'node:crypto';
4
4
  import { DatasetClient as ApifyDatasetClient } from 'apify-client';
5
+ import log from '@apify/log';
5
6
  import { cryptoRandomObjectId } from '@apify/utilities';
6
7
  import { ApifyDatasetBackend } from './apify_dataset_backend.js';
7
8
  import { ApifyKeyValueStoreBackend } from './apify_key_value_store_backend.js';
9
+ import { AsyncLock } from './apify_request_queue_backend.js';
8
10
  import { ApifyRequestQueueSharedBackend } from './apify_request_queue_shared_backend.js';
9
11
  import { ApifyRequestQueueSingleBackend } from './apify_request_queue_single_backend.js';
10
12
  import { DEFAULT_DATASET_ITEM_EVENT, mergeChargeResults, pushDataAndCharge, } from './charging.js';
11
13
  /** The reserved alias crawlee uses for the default (unnamed) storage. */
12
14
  const DEFAULT_STORAGE_ALIAS = '__default__';
15
+ /** The key of the default key-value store record holding this run's alias -> storage id mapping. */
16
+ const ALIAS_MAPPING_RECORD_KEY = '__STORAGE_ALIASES_MAPPING';
17
+ async function readAliasMapping(store) {
18
+ const record = await store.getRecord(ALIAS_MAPPING_RECORD_KEY);
19
+ return record?.value ?? {};
20
+ }
13
21
  /** The maximum clientKey length accepted by the request queue API. */
14
22
  const MAX_CLIENT_KEY_LENGTH = 32;
15
23
  const DEFAULT_ID_CONFIG_KEY = {
@@ -95,8 +103,12 @@ export class ApifyStorageBackend {
95
103
  config;
96
104
  requestQueueAccess;
97
105
  getChargingManager;
98
- /** Unnamed storages created for aliases in this process, so an alias maps to one storage. */
106
+ /** Unnamed storages resolved for aliases in this process, keyed like {@link AliasMapping}. */
99
107
  aliasIdCache = new Map();
108
+ /** The alias mapping read from the run's default key-value store; `undefined` until first read. */
109
+ persistedAliasIds;
110
+ /** Serializes alias resolution — see `resolveAliasId`. */
111
+ aliasLock = new AsyncLock();
100
112
  /** Fallback request queue client key when the run id is unavailable — one per backend. */
101
113
  fallbackClientKey;
102
114
  constructor(client, options = {}) {
@@ -112,11 +124,14 @@ export class ApifyStorageBackend {
112
124
  * and `shared` mode at once is not supported, and whichever backend opens it first wins.
113
125
  */
114
126
  getStorageBackendCacheKey() {
115
- const hash = createHash('sha256')
127
+ return `ApifyStorageBackend:${this.credentialsHash()}`;
128
+ }
129
+ /** Short digest of the API base URL and token — identifies the credentials a storage was opened with. */
130
+ credentialsHash() {
131
+ return createHash('sha256')
116
132
  .update(`${this.client.publicBaseUrl}${this.client.token ?? ''}`)
117
133
  .digest('hex')
118
134
  .slice(0, 8);
119
- return `ApifyStorageBackend:${hash}`;
120
135
  }
121
136
  async storageExists(id, type) {
122
137
  // Lets `Dataset.open(idOrName)` and friends resolve a string to an id first (when one
@@ -183,9 +198,10 @@ export class ApifyStorageBackend {
183
198
  * Resolves a crawlee {@link StorageIdentifier} to a platform storage id.
184
199
  *
185
200
  * Aliases resolve to unnamed storages: the reserved `__default__` alias maps to the run's
186
- * default storage, and other aliases to the storages declared in the Actor's schema (via the
187
- * `ACTOR_STORAGES_JSON` environment variable, maintained by the platform). Outside the
188
- * platform, an unnamed storage is created per alias instead (remembered for this process only).
201
+ * default storage, and an alias declared in the Actor's schema to the storage the platform
202
+ * created for it (via the `ACTOR_STORAGES_JSON` environment variable). Any other alias gets an
203
+ * unnamed storage of its own crawlee mints aliases at runtime, one per extra crawler instance
204
+ * and one per throttled domain, so an undeclared alias is not an error.
189
205
  */
190
206
  async resolveId(options, type) {
191
207
  if (options?.id)
@@ -203,20 +219,57 @@ export class ApifyStorageBackend {
203
219
  const declaredId = this.aliasFromActorStorages(alias, type);
204
220
  if (declaredId)
205
221
  return declaredId;
206
- if (this.config?.isAtHome) {
207
- throw new Error(`Storage alias "${alias}" cannot be resolved because it is not declared in the Actor's schema storages. ` +
208
- `Declare it in the Actor schema, or open the storage by name instead.`);
222
+ }
223
+ return this.resolveAliasId(alias, type);
224
+ }
225
+ /**
226
+ * Returns the unnamed storage backing `alias`, creating it on first use.
227
+ *
228
+ * On the platform the mapping is persisted, so a migrated run reopens the same storages rather
229
+ * than empty ones — aliased request queues hold live requests. Serialized, so one alias means
230
+ * one storage and the mapping's read-modify-write cannot drop entries.
231
+ */
232
+ async resolveAliasId(alias, type) {
233
+ // The credentials are part of the key, so the same alias opened through two
234
+ // differently-authenticated backends maps to two storages.
235
+ const key = [type, alias, this.credentialsHash()].join(',');
236
+ return this.aliasLock.runExclusive(async () => {
237
+ const knownId = this.aliasIdCache.get(key);
238
+ if (knownId)
239
+ return knownId;
240
+ const store = this.aliasMappingStore();
241
+ this.persistedAliasIds ??= store ? await readAliasMapping(store) : {};
242
+ // A persisted id can point at a storage the user has since deleted.
243
+ const persistedId = this.persistedAliasIds[key];
244
+ if (persistedId && (await this.resourceClient(persistedId, type).get())) {
245
+ this.aliasIdCache.set(key, persistedId);
246
+ return persistedId;
209
247
  }
248
+ const { id } = await this.collectionClient(type).getOrCreate();
249
+ this.aliasIdCache.set(key, id);
250
+ if (store)
251
+ await this.persistAliasId(store, key, id);
252
+ return id;
253
+ });
254
+ }
255
+ /**
256
+ * Re-reads the record first, so a second backend in this process does not drop its entries.
257
+ * Logged rather than thrown: a lost entry only costs a re-created storage after a migration.
258
+ */
259
+ async persistAliasId(store, key, id) {
260
+ try {
261
+ const mapping = await readAliasMapping(store);
262
+ mapping[key] = id;
263
+ await store.setRecord({ key: ALIAS_MAPPING_RECORD_KEY, value: mapping });
264
+ this.persistedAliasIds = mapping;
265
+ }
266
+ catch (error) {
267
+ log.warning(`Failed to persist the storage alias mapping: ${error.message}`);
210
268
  }
211
- // No platform-provided id for this alias (e.g. cloud storage used locally via an API
212
- // token) create an unnamed storage for it, one per alias per process.
213
- const cacheKey = `${type}:${alias}`;
214
- const cachedId = this.aliasIdCache.get(cacheKey);
215
- if (cachedId)
216
- return cachedId;
217
- const created = await this.collectionClient(type).getOrCreate();
218
- this.aliasIdCache.set(cacheKey, created.id);
219
- return created.id;
269
+ }
270
+ /** The run's default key-value store, where the mapping lives `undefined` off the platform. */
271
+ aliasMappingStore() {
272
+ return this.config?.isAtHome ? this.client.keyValueStore(this.config.defaultKeyValueStoreId) : undefined;
220
273
  }
221
274
  /** Looks an alias up in the Actor's schema storages (the `ACTOR_STORAGES_JSON` env var). */
222
275
  aliasFromActorStorages(alias, type) {
package/dist/index.d.ts CHANGED
@@ -3,7 +3,7 @@ export { ApifyStorageBackend, type ApifyStorageBackendOptions } from './apify_st
3
3
  export type { RequestQueueAccessMode } from './apify_request_queue_backend.js';
4
4
  export { ArgumentValidationError } from './utils.js';
5
5
  export { createTransformRequestFunction, type GlobInput, type PseudoUrlInput, type UrlPatternFilters, type UrlPatternRequestOptions, } from './enqueue_links_filters.js';
6
- export type { OpenStorageOptions, StorageAlias, StorageId, StorageName, StorageIdentifier, StorageIdentifierWithoutAlias, } from './storage.js';
6
+ export type { OpenStorageOptions, StorageAlias, StorageId, StorageName, StorageIdentifier } from './storage.js';
7
7
  export { ChargeOptions, ChargeResult, ActorPricingInfo, ChargingManager } from './charging.js';
8
8
  export * from './configuration.js';
9
9
  export * from './proxy_configuration.js';
package/dist/storage.d.ts CHANGED
@@ -1,6 +1,3 @@
1
- import type { IStorage, StorageOpenOptions } from '@crawlee/core';
2
- import type { Constructor, StorageBackend } from '@crawlee/types';
3
- import type { Configuration } from './configuration.js';
4
1
  export interface OpenStorageOptions {
5
2
  /**
6
3
  * If set to `true` then the cloud storage is used even if the `CRAWLEE_STORAGE_DIR`
@@ -10,8 +7,11 @@ export interface OpenStorageOptions {
10
7
  forceCloud?: boolean;
11
8
  }
12
9
  /**
13
- * Identifies a storage by its alias from the Actor's schema storages
14
- * (resolved via the `ACTOR_STORAGES_JSON` environment variable).
10
+ * Identifies a run-scoped storage by its alias.
11
+ *
12
+ * An alias declared in the Actor's schema storages (the `ACTOR_STORAGES_JSON` environment variable)
13
+ * resolves to the storage the platform created for it; any other alias gets an unnamed storage of
14
+ * its own, for this run only.
15
15
  */
16
16
  export interface StorageAlias {
17
17
  alias: string;
@@ -31,28 +31,8 @@ export interface StorageName {
31
31
  /**
32
32
  * Identifies a storage to open. Can be:
33
33
  * - A plain `string` for backward compatibility (treated as ID or name)
34
- * - `{ alias: string }` to resolve from the Actor's schema storages (`ACTOR_STORAGES_JSON`)
34
+ * - `{ alias: string }` to open a run-scoped storage see {@link StorageAlias}
35
35
  * - `{ id: string }` to open by explicit platform ID
36
36
  * - `{ name: string }` to open by explicit name
37
37
  */
38
38
  export type StorageIdentifier = string | StorageAlias | StorageId | StorageName;
39
- /**
40
- * Identifies a storage to open, without alias support.
41
- * Used for key-value stores and request queues, which do not support aliases.
42
- * Can be:
43
- * - A plain `string` for backward compatibility (treated as ID or name)
44
- * - `{ id: string }` to open by explicit platform ID
45
- * - `{ name: string }` to open by explicit name
46
- */
47
- export type StorageIdentifierWithoutAlias = string | StorageId | StorageName;
48
- export interface OpenStorageContext {
49
- config: Configuration;
50
- backend?: StorageBackend;
51
- purgedStorageAliases: Set<string>;
52
- }
53
- /**
54
- * Opens a storage by its identifier, handling Apify alias resolution and local purging.
55
- */
56
- export declare function openStorage<T extends IStorage>(storageClass: Constructor<T> & {
57
- open(id?: string | null, options?: StorageOpenOptions): Promise<T>;
58
- }, identifier: StorageIdentifier | null | undefined, context: OpenStorageContext): Promise<T>;
package/dist/storage.js CHANGED
@@ -1,79 +1 @@
1
- import { ApifyStorageBackend } from './apify_storage_backend.js';
2
- const STORAGE_TYPE_KEYS = {
3
- Dataset: 'datasets',
4
- KeyValueStore: 'keyValueStores',
5
- RequestQueue: 'requestQueues',
6
- };
7
- const parsedStoragesJson = new Map();
8
- /**
9
- * Resolves a {@link StorageIdentifier} to a plain string ID or name
10
- * that can be passed to crawlee v4's `<Storage>.open()`.
11
- */
12
- function resolveStorageIdentifier(storageType, identifier, config) {
13
- if (identifier === null || identifier === undefined) {
14
- return undefined;
15
- }
16
- if (typeof identifier === 'string') {
17
- return identifier;
18
- }
19
- if ('id' in identifier) {
20
- return identifier.id;
21
- }
22
- if ('name' in identifier) {
23
- return identifier.name;
24
- }
25
- // { alias: string }
26
- const storagesJson = config.actorStoragesJson;
27
- if (config.isAtHome && storagesJson) {
28
- let storages;
29
- try {
30
- if (!parsedStoragesJson.has(storagesJson)) {
31
- parsedStoragesJson.set(storagesJson, JSON.parse(storagesJson));
32
- }
33
- storages = parsedStoragesJson.get(storagesJson);
34
- }
35
- catch {
36
- throw new Error(`Failed to parse ACTOR_STORAGES_JSON environment variable: ${storagesJson}`);
37
- }
38
- const typeKey = STORAGE_TYPE_KEYS[storageType];
39
- const resolvedId = storages[typeKey]?.[identifier.alias];
40
- if (resolvedId) {
41
- return resolvedId;
42
- }
43
- throw new Error(`Storage alias "${identifier.alias}" not found in ACTOR_STORAGES_JSON for storage type "${storageType}". ` +
44
- `Available aliases: ${Object.keys(storages[typeKey] ?? {}).join(', ') || '(none)'}`);
45
- }
46
- // When using local storage, just use the alias as a name.
47
- // When using platform storage, we can't just make up a name — the alias must be
48
- // in ACTOR_STORAGES_JSON.
49
- if (config.isAtHome) {
50
- throw new Error(`Storage alias "${identifier.alias}" cannot be resolved because ACTOR_STORAGES_JSON is not set. ` +
51
- `Aliases are only available for storages declared in the Actor's schema.`);
52
- }
53
- return identifier.alias;
54
- }
55
- /**
56
- * Opens a storage by its identifier, handling Apify alias resolution and local purging.
57
- */
58
- export async function openStorage(storageClass, identifier, context) {
59
- const isAlias = identifier !== null && identifier !== undefined && typeof identifier === 'object' && 'alias' in identifier;
60
- if (isAlias && !context.config.isAtHome && context.backend instanceof ApifyStorageBackend) {
61
- throw new Error('The `alias` option is not allowed for Apify-based storages running outside of Apify');
62
- }
63
- const resolvedIdOrName = resolveStorageIdentifier(storageClass.name, identifier, context.config);
64
- // When running locally, purge aliased storages on first open
65
- // (similar to how crawlee purges default storages on start).
66
- if (isAlias &&
67
- !context.config.isAtHome &&
68
- context.config.purgeOnStart &&
69
- !context.purgedStorageAliases.has(identifier.alias)) {
70
- context.purgedStorageAliases.add(identifier.alias);
71
- const existingStorage = await storageClass.open(resolvedIdOrName ?? null, {
72
- storageBackend: context.backend,
73
- });
74
- await existingStorage.drop();
75
- }
76
- return storageClass.open(resolvedIdOrName ?? null, {
77
- storageBackend: context.backend,
78
- });
79
- }
1
+ export {};
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "apify",
3
- "version": "4.0.0-beta.30",
3
+ "version": "4.0.0-beta.31",
4
4
  "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
5
5
  "engines": {
6
6
  "node": ">=22.0.0"