apify 4.0.0-beta.30 → 4.0.0-beta.32

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -2,14 +2,22 @@
2
2
  import { AsyncLocalStorage } from 'node:async_hooks';
3
3
  import { createHash } from 'node:crypto';
4
4
  import { DatasetClient as ApifyDatasetClient } from 'apify-client';
5
+ import log from '@apify/log';
5
6
  import { cryptoRandomObjectId } from '@apify/utilities';
6
7
  import { ApifyDatasetBackend } from './apify_dataset_backend.js';
7
8
  import { ApifyKeyValueStoreBackend } from './apify_key_value_store_backend.js';
9
+ import { AsyncLock } from './apify_request_queue_backend.js';
8
10
  import { ApifyRequestQueueSharedBackend } from './apify_request_queue_shared_backend.js';
9
11
  import { ApifyRequestQueueSingleBackend } from './apify_request_queue_single_backend.js';
10
12
  import { DEFAULT_DATASET_ITEM_EVENT, mergeChargeResults, pushDataAndCharge, } from './charging.js';
11
13
  /** The reserved alias crawlee uses for the default (unnamed) storage. */
12
14
  const DEFAULT_STORAGE_ALIAS = '__default__';
15
+ /** The key of the default key-value store record holding this run's alias -> storage id mapping. */
16
+ const ALIAS_MAPPING_RECORD_KEY = '__STORAGE_ALIASES_MAPPING';
17
+ async function readAliasMapping(store) {
18
+ const record = await store.getRecord(ALIAS_MAPPING_RECORD_KEY);
19
+ return record?.value ?? {};
20
+ }
13
21
  /** The maximum clientKey length accepted by the request queue API. */
14
22
  const MAX_CLIENT_KEY_LENGTH = 32;
15
23
  const DEFAULT_ID_CONFIG_KEY = {
@@ -32,10 +40,10 @@ export const pushDataChargingContext = new AsyncLocalStorage();
32
40
  * used.
33
41
  */
34
42
  class PpeAwareDatasetClient extends ApifyDatasetClient {
35
- getChargingManager;
43
+ #getChargingManager;
36
44
  constructor(options, getChargingManager) {
37
45
  super(options);
38
- this.getChargingManager = getChargingManager;
46
+ this.#getChargingManager = getChargingManager;
39
47
  }
40
48
  normalizeItems(items) {
41
49
  if (typeof items === 'string') {
@@ -54,7 +62,7 @@ class PpeAwareDatasetClient extends ApifyDatasetClient {
54
62
  // each logical item is counted individually.
55
63
  const normalizedItems = this.normalizeItems(items);
56
64
  const result = await pushDataAndCharge({
57
- chargingManager: this.getChargingManager(),
65
+ chargingManager: this.#getChargingManager(),
58
66
  items: normalizedItems,
59
67
  eventName: context?.eventName,
60
68
  isDefaultDataset: true,
@@ -91,19 +99,23 @@ class PpeAwareDatasetClient extends ApifyDatasetClient {
91
99
  * ```
92
100
  */
93
101
  export class ApifyStorageBackend {
94
- client;
95
- config;
96
- requestQueueAccess;
97
- getChargingManager;
98
- /** Unnamed storages created for aliases in this process, so an alias maps to one storage. */
99
- aliasIdCache = new Map();
102
+ #client;
103
+ #config;
104
+ #requestQueueAccess;
105
+ #getChargingManager;
106
+ /** Unnamed storages resolved for aliases in this process, keyed like {@link AliasMapping}. */
107
+ #aliasIdCache = new Map();
108
+ /** The alias mapping read from the run's default key-value store; `undefined` until first read. */
109
+ #persistedAliasIds;
110
+ /** Serializes alias resolution — see `resolveAliasId`. */
111
+ #aliasLock = new AsyncLock();
100
112
  /** Fallback request queue client key when the run id is unavailable — one per backend. */
101
- fallbackClientKey;
113
+ #fallbackClientKey;
102
114
  constructor(client, options = {}) {
103
- this.client = client;
104
- this.config = options.configuration;
105
- this.requestQueueAccess = options.requestQueueAccess ?? 'single';
106
- this.getChargingManager = options.getChargingManager;
115
+ this.#client = client;
116
+ this.#config = options.configuration;
117
+ this.#requestQueueAccess = options.requestQueueAccess ?? 'single';
118
+ this.#getChargingManager = options.getChargingManager;
107
119
  }
108
120
  /**
109
121
  * Partitions crawlee's storage-instance cache by API base URL and token, so the same storage
@@ -112,11 +124,14 @@ export class ApifyStorageBackend {
112
124
  * and `shared` mode at once is not supported, and whichever backend opens it first wins.
113
125
  */
114
126
  getStorageBackendCacheKey() {
115
- const hash = createHash('sha256')
116
- .update(`${this.client.publicBaseUrl}${this.client.token ?? ''}`)
127
+ return `ApifyStorageBackend:${this.credentialsHash()}`;
128
+ }
129
+ /** Short digest of the API base URL and token — identifies the credentials a storage was opened with. */
130
+ credentialsHash() {
131
+ return createHash('sha256')
132
+ .update(`${this.#client.publicBaseUrl}${this.#client.token ?? ''}`)
117
133
  .digest('hex')
118
134
  .slice(0, 8);
119
- return `ApifyStorageBackend:${hash}`;
120
135
  }
121
136
  async storageExists(id, type) {
122
137
  // Lets `Dataset.open(idOrName)` and friends resolve a string to an id first (when one
@@ -130,7 +145,7 @@ export class ApifyStorageBackend {
130
145
  async createDatasetBackend(options) {
131
146
  const id = await this.resolveId(options, 'Dataset');
132
147
  const chargingClient = this.chargingDatasetClient(id);
133
- const backend = new ApifyDatasetBackend(chargingClient ?? this.client.dataset(id));
148
+ const backend = new ApifyDatasetBackend(chargingClient ?? this.#client.dataset(id));
134
149
  if (chargingClient) {
135
150
  // `Actor.pushData()` looks for this marker on the dataset's backend to know the
136
151
  // pay-per-event charging happens inside the intercepted `pushItems()` calls.
@@ -140,12 +155,12 @@ export class ApifyStorageBackend {
140
155
  }
141
156
  async createKeyValueStoreBackend(options) {
142
157
  const id = await this.resolveId(options, 'KeyValueStore');
143
- return new ApifyKeyValueStoreBackend(this.client.keyValueStore(id));
158
+ return new ApifyKeyValueStoreBackend(this.#client.keyValueStore(id));
144
159
  }
145
160
  async createRequestQueueBackend(options) {
146
161
  const id = await this.resolveId(options, 'RequestQueue');
147
- const client = this.client.requestQueue(id, { clientKey: this.requestQueueClientKey() });
148
- return this.requestQueueAccess === 'shared'
162
+ const client = this.#client.requestQueue(id, { clientKey: this.requestQueueClientKey() });
163
+ return this.#requestQueueAccess === 'shared'
149
164
  ? new ApifyRequestQueueSharedBackend(client)
150
165
  : new ApifyRequestQueueSingleBackend(client);
151
166
  }
@@ -154,7 +169,7 @@ export class ApifyStorageBackend {
154
169
  * migrated or resurrected run re-acquire the request locks of its previous incarnation.
155
170
  */
156
171
  requestQueueClientKey() {
157
- const key = this.config?.actorRunId ?? (this.fallbackClientKey ??= cryptoRandomObjectId(MAX_CLIENT_KEY_LENGTH));
172
+ const key = this.#config?.actorRunId ?? (this.#fallbackClientKey ??= cryptoRandomObjectId(MAX_CLIENT_KEY_LENGTH));
158
173
  return key.slice(0, MAX_CLIENT_KEY_LENGTH);
159
174
  }
160
175
  /**
@@ -163,29 +178,30 @@ export class ApifyStorageBackend {
163
178
  * `undefined` (caller uses the plain client).
164
179
  */
165
180
  chargingDatasetClient(id) {
166
- const { getChargingManager } = this;
181
+ const getChargingManager = this.#getChargingManager;
167
182
  if (!getChargingManager)
168
183
  return undefined;
169
- if (id !== this.config?.defaultDatasetId)
184
+ if (id !== this.#config?.defaultDatasetId)
170
185
  return undefined;
171
186
  const hasDefaultDatasetItemEvent = DEFAULT_DATASET_ITEM_EVENT in getChargingManager().getPricingInfo().perEventPrices;
172
187
  if (!hasDefaultDatasetItemEvent)
173
188
  return undefined;
174
189
  return new PpeAwareDatasetClient({
175
190
  id,
176
- baseUrl: this.client.baseUrl,
177
- publicBaseUrl: this.client.publicBaseUrl,
178
- apifyClient: this.client,
179
- httpClient: this.client.httpClient,
191
+ baseUrl: this.#client.baseUrl,
192
+ publicBaseUrl: this.#client.publicBaseUrl,
193
+ apifyClient: this.#client,
194
+ httpClient: this.#client.httpClient,
180
195
  }, getChargingManager);
181
196
  }
182
197
  /**
183
198
  * Resolves a crawlee {@link StorageIdentifier} to a platform storage id.
184
199
  *
185
200
  * Aliases resolve to unnamed storages: the reserved `__default__` alias maps to the run's
186
- * default storage, and other aliases to the storages declared in the Actor's schema (via the
187
- * `ACTOR_STORAGES_JSON` environment variable, maintained by the platform). Outside the
188
- * platform, an unnamed storage is created per alias instead (remembered for this process only).
201
+ * default storage, and an alias declared in the Actor's schema to the storage the platform
202
+ * created for it (via the `ACTOR_STORAGES_JSON` environment variable). Any other alias gets an
203
+ * unnamed storage of its own crawlee mints aliases at runtime, one per extra crawler instance
204
+ * and one per throttled domain, so an undeclared alias is not an error.
189
205
  */
190
206
  async resolveId(options, type) {
191
207
  if (options?.id)
@@ -195,7 +211,7 @@ export class ApifyStorageBackend {
195
211
  }
196
212
  const alias = (options && 'alias' in options && options.alias) || DEFAULT_STORAGE_ALIAS;
197
213
  if (alias === DEFAULT_STORAGE_ALIAS) {
198
- const defaultId = this.config?.[DEFAULT_ID_CONFIG_KEY[type]];
214
+ const defaultId = this.#config?.[DEFAULT_ID_CONFIG_KEY[type]];
199
215
  if (defaultId)
200
216
  return defaultId;
201
217
  }
@@ -203,24 +219,61 @@ export class ApifyStorageBackend {
203
219
  const declaredId = this.aliasFromActorStorages(alias, type);
204
220
  if (declaredId)
205
221
  return declaredId;
206
- if (this.config?.isAtHome) {
207
- throw new Error(`Storage alias "${alias}" cannot be resolved because it is not declared in the Actor's schema storages. ` +
208
- `Declare it in the Actor schema, or open the storage by name instead.`);
222
+ }
223
+ return this.resolveAliasId(alias, type);
224
+ }
225
+ /**
226
+ * Returns the unnamed storage backing `alias`, creating it on first use.
227
+ *
228
+ * On the platform the mapping is persisted, so a migrated run reopens the same storages rather
229
+ * than empty ones — aliased request queues hold live requests. Serialized, so one alias means
230
+ * one storage and the mapping's read-modify-write cannot drop entries.
231
+ */
232
+ async resolveAliasId(alias, type) {
233
+ // The credentials are part of the key, so the same alias opened through two
234
+ // differently-authenticated backends maps to two storages.
235
+ const key = [type, alias, this.credentialsHash()].join(',');
236
+ return this.#aliasLock.runExclusive(async () => {
237
+ const knownId = this.#aliasIdCache.get(key);
238
+ if (knownId)
239
+ return knownId;
240
+ const store = this.aliasMappingStore();
241
+ this.#persistedAliasIds ??= store ? await readAliasMapping(store) : {};
242
+ // A persisted id can point at a storage the user has since deleted.
243
+ const persistedId = this.#persistedAliasIds[key];
244
+ if (persistedId && (await this.resourceClient(persistedId, type).get())) {
245
+ this.#aliasIdCache.set(key, persistedId);
246
+ return persistedId;
209
247
  }
248
+ const { id } = await this.collectionClient(type).getOrCreate();
249
+ this.#aliasIdCache.set(key, id);
250
+ if (store)
251
+ await this.persistAliasId(store, key, id);
252
+ return id;
253
+ });
254
+ }
255
+ /**
256
+ * Re-reads the record first, so a second backend in this process does not drop its entries.
257
+ * Logged rather than thrown: a lost entry only costs a re-created storage after a migration.
258
+ */
259
+ async persistAliasId(store, key, id) {
260
+ try {
261
+ const mapping = await readAliasMapping(store);
262
+ mapping[key] = id;
263
+ await store.setRecord({ key: ALIAS_MAPPING_RECORD_KEY, value: mapping });
264
+ this.#persistedAliasIds = mapping;
265
+ }
266
+ catch (error) {
267
+ log.warning(`Failed to persist the storage alias mapping: ${error.message}`);
210
268
  }
211
- // No platform-provided id for this alias (e.g. cloud storage used locally via an API
212
- // token) create an unnamed storage for it, one per alias per process.
213
- const cacheKey = `${type}:${alias}`;
214
- const cachedId = this.aliasIdCache.get(cacheKey);
215
- if (cachedId)
216
- return cachedId;
217
- const created = await this.collectionClient(type).getOrCreate();
218
- this.aliasIdCache.set(cacheKey, created.id);
219
- return created.id;
269
+ }
270
+ /** The run's default key-value store, where the mapping lives `undefined` off the platform. */
271
+ aliasMappingStore() {
272
+ return this.#config?.isAtHome ? this.#client.keyValueStore(this.#config.defaultKeyValueStoreId) : undefined;
220
273
  }
221
274
  /** Looks an alias up in the Actor's schema storages (the `ACTOR_STORAGES_JSON` env var). */
222
275
  aliasFromActorStorages(alias, type) {
223
- const storagesJson = this.config?.actorStoragesJson;
276
+ const storagesJson = this.#config?.actorStoragesJson;
224
277
  if (!storagesJson)
225
278
  return undefined;
226
279
  let storages;
@@ -234,16 +287,16 @@ export class ApifyStorageBackend {
234
287
  }
235
288
  resourceClient(id, type) {
236
289
  if (type === 'Dataset')
237
- return this.client.dataset(id);
290
+ return this.#client.dataset(id);
238
291
  if (type === 'KeyValueStore')
239
- return this.client.keyValueStore(id);
240
- return this.client.requestQueue(id);
292
+ return this.#client.keyValueStore(id);
293
+ return this.#client.requestQueue(id);
241
294
  }
242
295
  collectionClient(type) {
243
296
  if (type === 'Dataset')
244
- return this.client.datasets();
297
+ return this.#client.datasets();
245
298
  if (type === 'KeyValueStore')
246
- return this.client.keyValueStores();
247
- return this.client.requestQueues();
299
+ return this.#client.keyValueStores();
300
+ return this.#client.requestQueues();
248
301
  }
249
302
  }
@@ -54,20 +54,7 @@ export declare function mergeChargeResults(a: ChargeResult, b: ChargeResult): Ch
54
54
  * Handles pay-per-event charging.
55
55
  */
56
56
  export declare class ChargingManager {
57
- private configuration;
58
- private readonly LOCAL_CHARGING_LOG_DATASET_NAME;
59
- private readonly PLATFORM_CHARGING_LOG_DATASET_ID_KEY;
60
- private maxTotalChargeUsd;
61
- private isAtHome;
62
- private actorRunId?;
63
- private pricingModel?;
64
- private purgeChargingLogDataset;
65
- private useChargingLogDataset;
66
- private notPpeWarningPrinted;
67
- private pricingInfo;
68
- private chargingState?;
69
- private chargingLogDataset?;
70
- private apifyClient;
57
+ #private;
71
58
  constructor(configuration: Configuration, apifyClient: ApifyClient);
72
59
  private get isPayPerEvent();
73
60
  private fetchPricingInfo;
package/dist/charging.js CHANGED
@@ -15,45 +15,45 @@ export function mergeChargeResults(a, b) {
15
15
  * Handles pay-per-event charging.
16
16
  */
17
17
  export class ChargingManager {
18
- configuration;
19
- LOCAL_CHARGING_LOG_DATASET_NAME = 'charging_log';
20
- PLATFORM_CHARGING_LOG_DATASET_ID_KEY = 'CHARGING_LOG_DATASET_ID';
21
- maxTotalChargeUsd;
22
- isAtHome;
23
- actorRunId;
24
- pricingModel;
25
- purgeChargingLogDataset;
26
- useChargingLogDataset;
27
- notPpeWarningPrinted = false;
28
- pricingInfo = {};
29
- chargingState;
30
- chargingLogDataset;
31
- apifyClient;
18
+ #LOCAL_CHARGING_LOG_DATASET_NAME = 'charging_log';
19
+ #PLATFORM_CHARGING_LOG_DATASET_ID_KEY = 'CHARGING_LOG_DATASET_ID';
20
+ #maxTotalChargeUsd;
21
+ #isAtHome;
22
+ #actorRunId;
23
+ #pricingModel;
24
+ #purgeChargingLogDataset;
25
+ #useChargingLogDataset;
26
+ #notPpeWarningPrinted = false;
27
+ #pricingInfo = {};
28
+ #chargingState;
29
+ #chargingLogDataset;
30
+ #apifyClient;
31
+ #configuration;
32
32
  constructor(configuration, apifyClient) {
33
- this.configuration = configuration;
34
- this.maxTotalChargeUsd = configuration.maxTotalChargeUsd || Infinity; // convert `0` to `Infinity` in case the value is an empty string
35
- this.isAtHome = configuration.isAtHome;
36
- this.actorRunId = configuration.actorRunId;
37
- this.purgeChargingLogDataset = configuration.purgeOnStart;
38
- this.useChargingLogDataset = configuration.useChargingLogDataset;
39
- this.apifyClient = apifyClient;
33
+ this.#configuration = configuration;
34
+ this.#maxTotalChargeUsd = configuration.maxTotalChargeUsd || Infinity; // convert `0` to `Infinity` in case the value is an empty string
35
+ this.#isAtHome = configuration.isAtHome;
36
+ this.#actorRunId = configuration.actorRunId;
37
+ this.#purgeChargingLogDataset = configuration.purgeOnStart;
38
+ this.#useChargingLogDataset = configuration.useChargingLogDataset;
39
+ this.#apifyClient = apifyClient;
40
40
  }
41
41
  get isPayPerEvent() {
42
- return this.pricingModel === 'PAY_PER_EVENT';
42
+ return this.#pricingModel === 'PAY_PER_EVENT';
43
43
  }
44
44
  async fetchPricingInfo() {
45
- if (this.configuration.actorPricingInfo && this.configuration.chargedEventCounts) {
45
+ if (this.#configuration.actorPricingInfo && this.#configuration.chargedEventCounts) {
46
46
  return {
47
- pricingInfo: JSON.parse(this.configuration.actorPricingInfo),
48
- chargedEventCounts: JSON.parse(this.configuration.chargedEventCounts),
49
- maxTotalChargeUsd: this.configuration.maxTotalChargeUsd || Infinity,
47
+ pricingInfo: JSON.parse(this.#configuration.actorPricingInfo),
48
+ chargedEventCounts: JSON.parse(this.#configuration.chargedEventCounts),
49
+ maxTotalChargeUsd: this.#configuration.maxTotalChargeUsd || Infinity,
50
50
  };
51
51
  }
52
- if (this.isAtHome) {
53
- if (this.actorRunId === undefined) {
52
+ if (this.#isAtHome) {
53
+ if (this.#actorRunId === undefined) {
54
54
  throw new Error('Actor run ID not found even though the Actor is running on Apify');
55
55
  }
56
- const run = await this.apifyClient.run(this.actorRunId).get();
56
+ const run = await this.#apifyClient.run(this.#actorRunId).get();
57
57
  if (run === undefined) {
58
58
  throw new Error('Actor run not found');
59
59
  }
@@ -66,7 +66,7 @@ export class ChargingManager {
66
66
  return {
67
67
  pricingInfo: undefined,
68
68
  chargedEventCounts: {},
69
- maxTotalChargeUsd: this.configuration.maxTotalChargeUsd || Infinity,
69
+ maxTotalChargeUsd: this.#configuration.maxTotalChargeUsd || Infinity,
70
70
  };
71
71
  }
72
72
  /**
@@ -74,63 +74,63 @@ export class ChargingManager {
74
74
  */
75
75
  async init() {
76
76
  // Validate config - it may have changed since the instantiation
77
- if (this.useChargingLogDataset && this.isAtHome) {
77
+ if (this.#useChargingLogDataset && this.#isAtHome) {
78
78
  throw new Error('Using the ACTOR_USE_CHARGING_LOG_DATASET environment variable is only supported in a local development environment');
79
79
  }
80
- if (this.configuration.testPayPerEvent) {
81
- if (this.isAtHome) {
80
+ if (this.#configuration.testPayPerEvent) {
81
+ if (this.#isAtHome) {
82
82
  throw new Error('Using the ACTOR_TEST_PAY_PER_EVENT environment variable is only supported in a local development environment');
83
83
  }
84
84
  }
85
85
  // Retrieve pricing information
86
86
  const { pricingInfo, chargedEventCounts, maxTotalChargeUsd } = await this.fetchPricingInfo();
87
- if (this.configuration.testPayPerEvent) {
88
- this.pricingModel = 'PAY_PER_EVENT';
87
+ if (this.#configuration.testPayPerEvent) {
88
+ this.#pricingModel = 'PAY_PER_EVENT';
89
89
  }
90
90
  else {
91
- this.pricingModel ??= pricingInfo?.pricingModel;
91
+ this.#pricingModel ??= pricingInfo?.pricingModel;
92
92
  }
93
93
  // Load per-event pricing information
94
94
  if (pricingInfo?.pricingModel === 'PAY_PER_EVENT') {
95
95
  for (const [eventName, eventPricing] of Object.entries(pricingInfo.pricingPerEvent.actorChargeEvents)) {
96
- this.pricingInfo[eventName] = {
96
+ this.#pricingInfo[eventName] = {
97
97
  price: eventPricing.eventPriceUsd,
98
98
  title: eventPricing.eventTitle,
99
99
  };
100
100
  }
101
- this.maxTotalChargeUsd = maxTotalChargeUsd;
101
+ this.#maxTotalChargeUsd = maxTotalChargeUsd;
102
102
  }
103
- this.chargingState = {};
103
+ this.#chargingState = {};
104
104
  for (const [eventName, chargeCount] of Object.entries(chargedEventCounts ?? {})) {
105
- this.chargingState[eventName] = {
105
+ this.#chargingState[eventName] = {
106
106
  chargeCount,
107
- totalChargedAmount: chargeCount * (this.pricingInfo[eventName]?.price ?? 0),
107
+ totalChargedAmount: chargeCount * (this.#pricingInfo[eventName]?.price ?? 0),
108
108
  };
109
109
  }
110
- if (!this.isPayPerEvent || !this.useChargingLogDataset) {
110
+ if (!this.isPayPerEvent || !this.#useChargingLogDataset) {
111
111
  return;
112
112
  }
113
113
  // Set up charging log dataset
114
- if (this.isAtHome) {
114
+ if (this.#isAtHome) {
115
115
  const datasetId = await this.ensureChargingLogDatasetOnPlatform();
116
- this.chargingLogDataset = await Dataset.open(datasetId);
116
+ this.#chargingLogDataset = await Dataset.open(datasetId);
117
117
  }
118
118
  else {
119
- if (this.purgeChargingLogDataset) {
120
- const dataset = await Dataset.open(this.LOCAL_CHARGING_LOG_DATASET_NAME);
119
+ if (this.#purgeChargingLogDataset) {
120
+ const dataset = await Dataset.open(this.#LOCAL_CHARGING_LOG_DATASET_NAME);
121
121
  await dataset.drop();
122
122
  }
123
- this.chargingLogDataset = await Dataset.open(this.LOCAL_CHARGING_LOG_DATASET_NAME);
123
+ this.#chargingLogDataset = await Dataset.open(this.#LOCAL_CHARGING_LOG_DATASET_NAME);
124
124
  }
125
125
  }
126
126
  async ensureChargingLogDatasetOnPlatform() {
127
127
  const defaultStore = await KeyValueStore.open();
128
- const storedDatasetId = await defaultStore.getValue(this.PLATFORM_CHARGING_LOG_DATASET_ID_KEY);
128
+ const storedDatasetId = await defaultStore.getValue(this.#PLATFORM_CHARGING_LOG_DATASET_ID_KEY);
129
129
  if (storedDatasetId !== null) {
130
130
  return storedDatasetId;
131
131
  }
132
- const dataset = await this.apifyClient.datasets().getOrCreate();
133
- await defaultStore.setValue(this.PLATFORM_CHARGING_LOG_DATASET_ID_KEY, dataset.id);
132
+ const dataset = await this.#apifyClient.datasets().getOrCreate();
133
+ await defaultStore.setValue(this.#PLATFORM_CHARGING_LOG_DATASET_ID_KEY, dataset.id);
134
134
  return dataset.id;
135
135
  }
136
136
  /**
@@ -138,20 +138,20 @@ export class ChargingManager {
138
138
  * {@link ChargingManager.getPricingInfo}) require an initialized manager.
139
139
  */
140
140
  get isInitialized() {
141
- return this.chargingState !== undefined;
141
+ return this.#chargingState !== undefined;
142
142
  }
143
143
  /**
144
144
  * Get information about the pricing for this Actor.
145
145
  */
146
146
  getPricingInfo() {
147
- if (this.chargingState === undefined) {
147
+ if (this.#chargingState === undefined) {
148
148
  throw new Error('ChargingManager is not initialized');
149
149
  }
150
150
  return {
151
- pricingModel: this.pricingModel,
151
+ pricingModel: this.#pricingModel,
152
152
  isPayPerEvent: this.isPayPerEvent,
153
- maxTotalChargeUsd: this.maxTotalChargeUsd,
154
- perEventPrices: Object.fromEntries(Object.entries(this.pricingInfo).map(([eventName, { price }]) => [eventName, price])),
153
+ maxTotalChargeUsd: this.#maxTotalChargeUsd,
154
+ perEventPrices: Object.fromEntries(Object.entries(this.#pricingInfo).map(([eventName, { price }]) => [eventName, price])),
155
155
  };
156
156
  }
157
157
  /**
@@ -171,11 +171,14 @@ export class ChargingManager {
171
171
  * @param options The name of the event to charge for and the number of events to be charged.
172
172
  */
173
173
  async charge({ eventName, count = 1 }) {
174
- const calculateChargeableWithinLimit = () => Object.fromEntries(Object.keys(this.pricingInfo).map((name) => [name, this.calculateMaxEventChargeCountWithinLimit(name)]));
174
+ const calculateChargeableWithinLimit = () => Object.fromEntries(Object.keys(this.#pricingInfo).map((name) => [
175
+ name,
176
+ this.calculateMaxEventChargeCountWithinLimit(name),
177
+ ]));
175
178
  if (!this.isPayPerEvent) {
176
- if (!this.notPpeWarningPrinted) {
179
+ if (!this.#notPpeWarningPrinted) {
177
180
  log.warning('Ignored attempt to charge for an event - the Actor does not use the pay-per-event pricing');
178
- this.notPpeWarningPrinted = true;
181
+ this.#notPpeWarningPrinted = true;
179
182
  }
180
183
  return {
181
184
  eventChargeLimitReached: false,
@@ -183,7 +186,7 @@ export class ChargingManager {
183
186
  chargeableWithinLimit: calculateChargeableWithinLimit(),
184
187
  };
185
188
  }
186
- if (this.chargingState === undefined) {
189
+ if (this.#chargingState === undefined) {
187
190
  throw new Error('ChargingManager is not initialized');
188
191
  }
189
192
  /* START OF CRITICAL SECTION - no awaits here */
@@ -195,7 +198,7 @@ export class ChargingManager {
195
198
  // If the caller tries to charge more than the budget allows, overcharge by one event
196
199
  // so that the Actor is detected by the platform and terminated.
197
200
  // But don't do this if already strictly over the budget - no point piling on charges.
198
- if (this.calculateTotalChargedAmount() <= this.maxTotalChargeUsd) {
201
+ if (this.calculateTotalChargedAmount() <= this.#maxTotalChargeUsd) {
199
202
  return maxEventChargeCount + 1;
200
203
  }
201
204
  return 0;
@@ -207,32 +210,32 @@ export class ChargingManager {
207
210
  chargeableWithinLimit: calculateChargeableWithinLimit(),
208
211
  };
209
212
  }
210
- const pricingInfo = this.pricingInfo[eventName] ?? {
211
- price: this.isAtHome ? 0 : 1, // Use a nonzero price for local development so that the maximum budget can be reached
213
+ const pricingInfo = this.#pricingInfo[eventName] ?? {
214
+ price: this.#isAtHome ? 0 : 1, // Use a nonzero price for local development so that the maximum budget can be reached
212
215
  title: `Unknown event '${eventName}'`,
213
216
  };
214
- this.chargingState[eventName] ??= {
217
+ this.#chargingState[eventName] ??= {
215
218
  chargeCount: 0,
216
219
  totalChargedAmount: 0,
217
220
  };
218
- this.chargingState[eventName].chargeCount += chargedCount;
219
- this.chargingState[eventName].totalChargedAmount += chargedCount * pricingInfo.price;
221
+ this.#chargingState[eventName].chargeCount += chargedCount;
222
+ this.#chargingState[eventName].totalChargedAmount += chargedCount * pricingInfo.price;
220
223
  /* END OF CRITICAL SECTION */
221
- if (this.isAtHome) {
224
+ if (this.#isAtHome) {
222
225
  if (eventName.startsWith('apify-')) {
223
226
  // Synthetic events (e.g. apify-default-dataset-item) are tracked locally only,
224
227
  // the platform handles them automatically based on dataset writes.
225
228
  }
226
- else if (this.pricingInfo[eventName] !== undefined) {
227
- await this.apifyClient.run(this.actorRunId).charge({ eventName, count: chargedCount });
229
+ else if (this.#pricingInfo[eventName] !== undefined) {
230
+ await this.#apifyClient.run(this.#actorRunId).charge({ eventName, count: chargedCount });
228
231
  }
229
232
  else {
230
233
  log.warning(`Attempting to charge for an unknown event '${eventName}'`);
231
234
  }
232
235
  }
233
236
  const timestamp = new Date().toISOString();
234
- if (this.chargingLogDataset !== undefined) {
235
- await this.chargingLogDataset.pushData({
237
+ if (this.#chargingLogDataset !== undefined) {
238
+ await this.#chargingLogDataset.pushData({
236
239
  eventName,
237
240
  eventTitle: pricingInfo.title,
238
241
  eventPriceUsd: pricingInfo.price,
@@ -254,25 +257,25 @@ export class ChargingManager {
254
257
  * Get the number of events with given name that the Actor has charged for so far.
255
258
  */
256
259
  getChargedEventCount(eventName) {
257
- if (this.chargingState === undefined) {
260
+ if (this.#chargingState === undefined) {
258
261
  throw new Error('ChargingManager is not initialized');
259
262
  }
260
- return this.chargingState[eventName]?.chargeCount ?? 0;
263
+ return this.#chargingState[eventName]?.chargeCount ?? 0;
261
264
  }
262
265
  /**
263
266
  * Get the maximum amount of money that the Actor is allowed to charge.
264
267
  */
265
268
  getMaxTotalChargeUsd() {
266
- if (this.chargingState === undefined) {
269
+ if (this.#chargingState === undefined) {
267
270
  throw new Error('ChargingManager is not initialized');
268
271
  }
269
- return this.maxTotalChargeUsd;
272
+ return this.#maxTotalChargeUsd;
270
273
  }
271
274
  calculateTotalChargedAmount() {
272
- if (this.chargingState === undefined) {
275
+ if (this.#chargingState === undefined) {
273
276
  throw new Error('ChargingManager is not initialized');
274
277
  }
275
- const result = Object.values(this.chargingState)
278
+ const result = Object.values(this.#chargingState)
276
279
  .map(({ totalChargedAmount }) => totalChargedAmount)
277
280
  .reduce((sum, inc) => sum + inc, 0);
278
281
  // Keeping float precision issues at bay
@@ -283,7 +286,7 @@ export class ChargingManager {
283
286
  * If the event is not registered, returns Infinity (free of charge)
284
287
  */
285
288
  calculateMaxEventChargeCountWithinLimit(eventName) {
286
- if (this.chargingState === undefined) {
289
+ if (this.#chargingState === undefined) {
287
290
  throw new Error('ChargingManager is not initialized');
288
291
  }
289
292
  const price = this.calculateEventPrice(eventName);
@@ -293,11 +296,11 @@ export class ChargingManager {
293
296
  return this.calculateMaxChargesByPrice(price);
294
297
  }
295
298
  calculateEventPrice(eventName) {
296
- return this.isAtHome ? this.pricingInfo[eventName]?.price : 1; // Use a nonzero price for local development so that the maximum budget can be reached
299
+ return this.#isAtHome ? this.#pricingInfo[eventName]?.price : 1; // Use a nonzero price for local development so that the maximum budget can be reached
297
300
  }
298
301
  calculateMaxChargesByPrice(price) {
299
302
  // The raw number of events allowed by the budget
300
- const unroundedResult = (this.maxTotalChargeUsd - this.calculateTotalChargedAmount()) / price;
303
+ const unroundedResult = (this.#maxTotalChargeUsd - this.calculateTotalChargedAmount()) / price;
301
304
  // First round as Math.floor(4.9999999999999999) will incorrectly return 5
302
305
  const roundedResult = Math.floor(Number(unroundedResult.toFixed(4)));
303
306
  return Math.max(0, roundedResult);
@@ -307,7 +310,7 @@ export class ChargingManager {
307
310
  * Returns the limited items and count to charge.
308
311
  */
309
312
  calculatePushDataLimits({ items, eventName, isDefaultDataset, }) {
310
- if (this.chargingState === undefined) {
313
+ if (this.#chargingState === undefined) {
311
314
  throw new Error('ChargingManager is not initialized');
312
315
  }
313
316
  const itemsArray = Array.isArray(items) ? items : [items];
@@ -329,7 +332,7 @@ export class ChargingManager {
329
332
  // But don't do this if already strictly over the budget - no point piling on charges.
330
333
  if (itemsArray.length > 0 &&
331
334
  maxChargedCount === 0 &&
332
- this.calculateTotalChargedAmount() <= this.maxTotalChargeUsd) {
335
+ this.calculateTotalChargedAmount() <= this.#maxTotalChargeUsd) {
333
336
  return 1;
334
337
  }
335
338
  return maxChargedCount;
package/dist/index.d.ts CHANGED
@@ -3,7 +3,7 @@ export { ApifyStorageBackend, type ApifyStorageBackendOptions } from './apify_st
3
3
  export type { RequestQueueAccessMode } from './apify_request_queue_backend.js';
4
4
  export { ArgumentValidationError } from './utils.js';
5
5
  export { createTransformRequestFunction, type GlobInput, type PseudoUrlInput, type UrlPatternFilters, type UrlPatternRequestOptions, } from './enqueue_links_filters.js';
6
- export type { OpenStorageOptions, StorageAlias, StorageId, StorageName, StorageIdentifier, StorageIdentifierWithoutAlias, } from './storage.js';
6
+ export type { OpenStorageOptions, StorageAlias, StorageId, StorageName, StorageIdentifier } from './storage.js';
7
7
  export { ChargeOptions, ChargeResult, ActorPricingInfo, ChargingManager } from './charging.js';
8
8
  export * from './configuration.js';
9
9
  export * from './proxy_configuration.js';