apify 4.0.0-beta.30 → 4.0.0-beta.32
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/actor.d.ts +21 -43
- package/dist/actor.js +74 -74
- package/dist/apify_dataset_backend.d.ts +1 -1
- package/dist/apify_dataset_backend.js +6 -6
- package/dist/apify_key_value_store_backend.d.ts +1 -1
- package/dist/apify_key_value_store_backend.js +10 -10
- package/dist/apify_request_queue_backend.d.ts +1 -1
- package/dist/apify_request_queue_backend.js +3 -3
- package/dist/apify_request_queue_shared_backend.d.ts +1 -14
- package/dist/apify_request_queue_shared_backend.js +38 -38
- package/dist/apify_request_queue_single_backend.d.ts +1 -10
- package/dist/apify_request_queue_single_backend.js +44 -44
- package/dist/apify_storage_backend.d.ts +23 -12
- package/dist/apify_storage_backend.js +104 -51
- package/dist/charging.d.ts +1 -14
- package/dist/charging.js +84 -81
- package/dist/index.d.ts +1 -1
- package/dist/platform_event_manager.d.ts +1 -2
- package/dist/platform_event_manager.js +7 -7
- package/dist/proxy_configuration.d.ts +7 -18
- package/dist/proxy_configuration.js +46 -57
- package/dist/storage.d.ts +6 -26
- package/dist/storage.js +1 -79
- package/package.json +1 -1
|
@@ -2,14 +2,22 @@
|
|
|
2
2
|
import { AsyncLocalStorage } from 'node:async_hooks';
|
|
3
3
|
import { createHash } from 'node:crypto';
|
|
4
4
|
import { DatasetClient as ApifyDatasetClient } from 'apify-client';
|
|
5
|
+
import log from '@apify/log';
|
|
5
6
|
import { cryptoRandomObjectId } from '@apify/utilities';
|
|
6
7
|
import { ApifyDatasetBackend } from './apify_dataset_backend.js';
|
|
7
8
|
import { ApifyKeyValueStoreBackend } from './apify_key_value_store_backend.js';
|
|
9
|
+
import { AsyncLock } from './apify_request_queue_backend.js';
|
|
8
10
|
import { ApifyRequestQueueSharedBackend } from './apify_request_queue_shared_backend.js';
|
|
9
11
|
import { ApifyRequestQueueSingleBackend } from './apify_request_queue_single_backend.js';
|
|
10
12
|
import { DEFAULT_DATASET_ITEM_EVENT, mergeChargeResults, pushDataAndCharge, } from './charging.js';
|
|
11
13
|
/** The reserved alias crawlee uses for the default (unnamed) storage. */
|
|
12
14
|
const DEFAULT_STORAGE_ALIAS = '__default__';
|
|
15
|
+
/** The key of the default key-value store record holding this run's alias -> storage id mapping. */
|
|
16
|
+
const ALIAS_MAPPING_RECORD_KEY = '__STORAGE_ALIASES_MAPPING';
|
|
17
|
+
async function readAliasMapping(store) {
|
|
18
|
+
const record = await store.getRecord(ALIAS_MAPPING_RECORD_KEY);
|
|
19
|
+
return record?.value ?? {};
|
|
20
|
+
}
|
|
13
21
|
/** The maximum clientKey length accepted by the request queue API. */
|
|
14
22
|
const MAX_CLIENT_KEY_LENGTH = 32;
|
|
15
23
|
const DEFAULT_ID_CONFIG_KEY = {
|
|
@@ -32,10 +40,10 @@ export const pushDataChargingContext = new AsyncLocalStorage();
|
|
|
32
40
|
* used.
|
|
33
41
|
*/
|
|
34
42
|
class PpeAwareDatasetClient extends ApifyDatasetClient {
|
|
35
|
-
getChargingManager;
|
|
43
|
+
#getChargingManager;
|
|
36
44
|
constructor(options, getChargingManager) {
|
|
37
45
|
super(options);
|
|
38
|
-
this
|
|
46
|
+
this.#getChargingManager = getChargingManager;
|
|
39
47
|
}
|
|
40
48
|
normalizeItems(items) {
|
|
41
49
|
if (typeof items === 'string') {
|
|
@@ -54,7 +62,7 @@ class PpeAwareDatasetClient extends ApifyDatasetClient {
|
|
|
54
62
|
// each logical item is counted individually.
|
|
55
63
|
const normalizedItems = this.normalizeItems(items);
|
|
56
64
|
const result = await pushDataAndCharge({
|
|
57
|
-
chargingManager: this
|
|
65
|
+
chargingManager: this.#getChargingManager(),
|
|
58
66
|
items: normalizedItems,
|
|
59
67
|
eventName: context?.eventName,
|
|
60
68
|
isDefaultDataset: true,
|
|
@@ -91,19 +99,23 @@ class PpeAwareDatasetClient extends ApifyDatasetClient {
|
|
|
91
99
|
* ```
|
|
92
100
|
*/
|
|
93
101
|
export class ApifyStorageBackend {
|
|
94
|
-
client;
|
|
95
|
-
config;
|
|
96
|
-
requestQueueAccess;
|
|
97
|
-
getChargingManager;
|
|
98
|
-
/** Unnamed storages
|
|
99
|
-
aliasIdCache = new Map();
|
|
102
|
+
#client;
|
|
103
|
+
#config;
|
|
104
|
+
#requestQueueAccess;
|
|
105
|
+
#getChargingManager;
|
|
106
|
+
/** Unnamed storages resolved for aliases in this process, keyed like {@link AliasMapping}. */
|
|
107
|
+
#aliasIdCache = new Map();
|
|
108
|
+
/** The alias mapping read from the run's default key-value store; `undefined` until first read. */
|
|
109
|
+
#persistedAliasIds;
|
|
110
|
+
/** Serializes alias resolution — see `resolveAliasId`. */
|
|
111
|
+
#aliasLock = new AsyncLock();
|
|
100
112
|
/** Fallback request queue client key when the run id is unavailable — one per backend. */
|
|
101
|
-
fallbackClientKey;
|
|
113
|
+
#fallbackClientKey;
|
|
102
114
|
constructor(client, options = {}) {
|
|
103
|
-
this
|
|
104
|
-
this
|
|
105
|
-
this
|
|
106
|
-
this
|
|
115
|
+
this.#client = client;
|
|
116
|
+
this.#config = options.configuration;
|
|
117
|
+
this.#requestQueueAccess = options.requestQueueAccess ?? 'single';
|
|
118
|
+
this.#getChargingManager = options.getChargingManager;
|
|
107
119
|
}
|
|
108
120
|
/**
|
|
109
121
|
* Partitions crawlee's storage-instance cache by API base URL and token, so the same storage
|
|
@@ -112,11 +124,14 @@ export class ApifyStorageBackend {
|
|
|
112
124
|
* and `shared` mode at once is not supported, and whichever backend opens it first wins.
|
|
113
125
|
*/
|
|
114
126
|
getStorageBackendCacheKey() {
|
|
115
|
-
|
|
116
|
-
|
|
127
|
+
return `ApifyStorageBackend:${this.credentialsHash()}`;
|
|
128
|
+
}
|
|
129
|
+
/** Short digest of the API base URL and token — identifies the credentials a storage was opened with. */
|
|
130
|
+
credentialsHash() {
|
|
131
|
+
return createHash('sha256')
|
|
132
|
+
.update(`${this.#client.publicBaseUrl}${this.#client.token ?? ''}`)
|
|
117
133
|
.digest('hex')
|
|
118
134
|
.slice(0, 8);
|
|
119
|
-
return `ApifyStorageBackend:${hash}`;
|
|
120
135
|
}
|
|
121
136
|
async storageExists(id, type) {
|
|
122
137
|
// Lets `Dataset.open(idOrName)` and friends resolve a string to an id first (when one
|
|
@@ -130,7 +145,7 @@ export class ApifyStorageBackend {
|
|
|
130
145
|
async createDatasetBackend(options) {
|
|
131
146
|
const id = await this.resolveId(options, 'Dataset');
|
|
132
147
|
const chargingClient = this.chargingDatasetClient(id);
|
|
133
|
-
const backend = new ApifyDatasetBackend(chargingClient ?? this
|
|
148
|
+
const backend = new ApifyDatasetBackend(chargingClient ?? this.#client.dataset(id));
|
|
134
149
|
if (chargingClient) {
|
|
135
150
|
// `Actor.pushData()` looks for this marker on the dataset's backend to know the
|
|
136
151
|
// pay-per-event charging happens inside the intercepted `pushItems()` calls.
|
|
@@ -140,12 +155,12 @@ export class ApifyStorageBackend {
|
|
|
140
155
|
}
|
|
141
156
|
async createKeyValueStoreBackend(options) {
|
|
142
157
|
const id = await this.resolveId(options, 'KeyValueStore');
|
|
143
|
-
return new ApifyKeyValueStoreBackend(this
|
|
158
|
+
return new ApifyKeyValueStoreBackend(this.#client.keyValueStore(id));
|
|
144
159
|
}
|
|
145
160
|
async createRequestQueueBackend(options) {
|
|
146
161
|
const id = await this.resolveId(options, 'RequestQueue');
|
|
147
|
-
const client = this
|
|
148
|
-
return this
|
|
162
|
+
const client = this.#client.requestQueue(id, { clientKey: this.requestQueueClientKey() });
|
|
163
|
+
return this.#requestQueueAccess === 'shared'
|
|
149
164
|
? new ApifyRequestQueueSharedBackend(client)
|
|
150
165
|
: new ApifyRequestQueueSingleBackend(client);
|
|
151
166
|
}
|
|
@@ -154,7 +169,7 @@ export class ApifyStorageBackend {
|
|
|
154
169
|
* migrated or resurrected run re-acquire the request locks of its previous incarnation.
|
|
155
170
|
*/
|
|
156
171
|
requestQueueClientKey() {
|
|
157
|
-
const key = this
|
|
172
|
+
const key = this.#config?.actorRunId ?? (this.#fallbackClientKey ??= cryptoRandomObjectId(MAX_CLIENT_KEY_LENGTH));
|
|
158
173
|
return key.slice(0, MAX_CLIENT_KEY_LENGTH);
|
|
159
174
|
}
|
|
160
175
|
/**
|
|
@@ -163,29 +178,30 @@ export class ApifyStorageBackend {
|
|
|
163
178
|
* `undefined` (caller uses the plain client).
|
|
164
179
|
*/
|
|
165
180
|
chargingDatasetClient(id) {
|
|
166
|
-
const
|
|
181
|
+
const getChargingManager = this.#getChargingManager;
|
|
167
182
|
if (!getChargingManager)
|
|
168
183
|
return undefined;
|
|
169
|
-
if (id !== this
|
|
184
|
+
if (id !== this.#config?.defaultDatasetId)
|
|
170
185
|
return undefined;
|
|
171
186
|
const hasDefaultDatasetItemEvent = DEFAULT_DATASET_ITEM_EVENT in getChargingManager().getPricingInfo().perEventPrices;
|
|
172
187
|
if (!hasDefaultDatasetItemEvent)
|
|
173
188
|
return undefined;
|
|
174
189
|
return new PpeAwareDatasetClient({
|
|
175
190
|
id,
|
|
176
|
-
baseUrl: this
|
|
177
|
-
publicBaseUrl: this
|
|
178
|
-
apifyClient: this
|
|
179
|
-
httpClient: this
|
|
191
|
+
baseUrl: this.#client.baseUrl,
|
|
192
|
+
publicBaseUrl: this.#client.publicBaseUrl,
|
|
193
|
+
apifyClient: this.#client,
|
|
194
|
+
httpClient: this.#client.httpClient,
|
|
180
195
|
}, getChargingManager);
|
|
181
196
|
}
|
|
182
197
|
/**
|
|
183
198
|
* Resolves a crawlee {@link StorageIdentifier} to a platform storage id.
|
|
184
199
|
*
|
|
185
200
|
* Aliases resolve to unnamed storages: the reserved `__default__` alias maps to the run's
|
|
186
|
-
* default storage, and
|
|
187
|
-
* `ACTOR_STORAGES_JSON` environment variable
|
|
188
|
-
*
|
|
201
|
+
* default storage, and an alias declared in the Actor's schema to the storage the platform
|
|
202
|
+
* created for it (via the `ACTOR_STORAGES_JSON` environment variable). Any other alias gets an
|
|
203
|
+
* unnamed storage of its own — crawlee mints aliases at runtime, one per extra crawler instance
|
|
204
|
+
* and one per throttled domain, so an undeclared alias is not an error.
|
|
189
205
|
*/
|
|
190
206
|
async resolveId(options, type) {
|
|
191
207
|
if (options?.id)
|
|
@@ -195,7 +211,7 @@ export class ApifyStorageBackend {
|
|
|
195
211
|
}
|
|
196
212
|
const alias = (options && 'alias' in options && options.alias) || DEFAULT_STORAGE_ALIAS;
|
|
197
213
|
if (alias === DEFAULT_STORAGE_ALIAS) {
|
|
198
|
-
const defaultId = this
|
|
214
|
+
const defaultId = this.#config?.[DEFAULT_ID_CONFIG_KEY[type]];
|
|
199
215
|
if (defaultId)
|
|
200
216
|
return defaultId;
|
|
201
217
|
}
|
|
@@ -203,24 +219,61 @@ export class ApifyStorageBackend {
|
|
|
203
219
|
const declaredId = this.aliasFromActorStorages(alias, type);
|
|
204
220
|
if (declaredId)
|
|
205
221
|
return declaredId;
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
222
|
+
}
|
|
223
|
+
return this.resolveAliasId(alias, type);
|
|
224
|
+
}
|
|
225
|
+
/**
|
|
226
|
+
* Returns the unnamed storage backing `alias`, creating it on first use.
|
|
227
|
+
*
|
|
228
|
+
* On the platform the mapping is persisted, so a migrated run reopens the same storages rather
|
|
229
|
+
* than empty ones — aliased request queues hold live requests. Serialized, so one alias means
|
|
230
|
+
* one storage and the mapping's read-modify-write cannot drop entries.
|
|
231
|
+
*/
|
|
232
|
+
async resolveAliasId(alias, type) {
|
|
233
|
+
// The credentials are part of the key, so the same alias opened through two
|
|
234
|
+
// differently-authenticated backends maps to two storages.
|
|
235
|
+
const key = [type, alias, this.credentialsHash()].join(',');
|
|
236
|
+
return this.#aliasLock.runExclusive(async () => {
|
|
237
|
+
const knownId = this.#aliasIdCache.get(key);
|
|
238
|
+
if (knownId)
|
|
239
|
+
return knownId;
|
|
240
|
+
const store = this.aliasMappingStore();
|
|
241
|
+
this.#persistedAliasIds ??= store ? await readAliasMapping(store) : {};
|
|
242
|
+
// A persisted id can point at a storage the user has since deleted.
|
|
243
|
+
const persistedId = this.#persistedAliasIds[key];
|
|
244
|
+
if (persistedId && (await this.resourceClient(persistedId, type).get())) {
|
|
245
|
+
this.#aliasIdCache.set(key, persistedId);
|
|
246
|
+
return persistedId;
|
|
209
247
|
}
|
|
248
|
+
const { id } = await this.collectionClient(type).getOrCreate();
|
|
249
|
+
this.#aliasIdCache.set(key, id);
|
|
250
|
+
if (store)
|
|
251
|
+
await this.persistAliasId(store, key, id);
|
|
252
|
+
return id;
|
|
253
|
+
});
|
|
254
|
+
}
|
|
255
|
+
/**
|
|
256
|
+
* Re-reads the record first, so a second backend in this process does not drop its entries.
|
|
257
|
+
* Logged rather than thrown: a lost entry only costs a re-created storage after a migration.
|
|
258
|
+
*/
|
|
259
|
+
async persistAliasId(store, key, id) {
|
|
260
|
+
try {
|
|
261
|
+
const mapping = await readAliasMapping(store);
|
|
262
|
+
mapping[key] = id;
|
|
263
|
+
await store.setRecord({ key: ALIAS_MAPPING_RECORD_KEY, value: mapping });
|
|
264
|
+
this.#persistedAliasIds = mapping;
|
|
265
|
+
}
|
|
266
|
+
catch (error) {
|
|
267
|
+
log.warning(`Failed to persist the storage alias mapping: ${error.message}`);
|
|
210
268
|
}
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
if (cachedId)
|
|
216
|
-
return cachedId;
|
|
217
|
-
const created = await this.collectionClient(type).getOrCreate();
|
|
218
|
-
this.aliasIdCache.set(cacheKey, created.id);
|
|
219
|
-
return created.id;
|
|
269
|
+
}
|
|
270
|
+
/** The run's default key-value store, where the mapping lives — `undefined` off the platform. */
|
|
271
|
+
aliasMappingStore() {
|
|
272
|
+
return this.#config?.isAtHome ? this.#client.keyValueStore(this.#config.defaultKeyValueStoreId) : undefined;
|
|
220
273
|
}
|
|
221
274
|
/** Looks an alias up in the Actor's schema storages (the `ACTOR_STORAGES_JSON` env var). */
|
|
222
275
|
aliasFromActorStorages(alias, type) {
|
|
223
|
-
const storagesJson = this
|
|
276
|
+
const storagesJson = this.#config?.actorStoragesJson;
|
|
224
277
|
if (!storagesJson)
|
|
225
278
|
return undefined;
|
|
226
279
|
let storages;
|
|
@@ -234,16 +287,16 @@ export class ApifyStorageBackend {
|
|
|
234
287
|
}
|
|
235
288
|
resourceClient(id, type) {
|
|
236
289
|
if (type === 'Dataset')
|
|
237
|
-
return this
|
|
290
|
+
return this.#client.dataset(id);
|
|
238
291
|
if (type === 'KeyValueStore')
|
|
239
|
-
return this
|
|
240
|
-
return this
|
|
292
|
+
return this.#client.keyValueStore(id);
|
|
293
|
+
return this.#client.requestQueue(id);
|
|
241
294
|
}
|
|
242
295
|
collectionClient(type) {
|
|
243
296
|
if (type === 'Dataset')
|
|
244
|
-
return this
|
|
297
|
+
return this.#client.datasets();
|
|
245
298
|
if (type === 'KeyValueStore')
|
|
246
|
-
return this
|
|
247
|
-
return this
|
|
299
|
+
return this.#client.keyValueStores();
|
|
300
|
+
return this.#client.requestQueues();
|
|
248
301
|
}
|
|
249
302
|
}
|
package/dist/charging.d.ts
CHANGED
|
@@ -54,20 +54,7 @@ export declare function mergeChargeResults(a: ChargeResult, b: ChargeResult): Ch
|
|
|
54
54
|
* Handles pay-per-event charging.
|
|
55
55
|
*/
|
|
56
56
|
export declare class ChargingManager {
|
|
57
|
-
private
|
|
58
|
-
private readonly LOCAL_CHARGING_LOG_DATASET_NAME;
|
|
59
|
-
private readonly PLATFORM_CHARGING_LOG_DATASET_ID_KEY;
|
|
60
|
-
private maxTotalChargeUsd;
|
|
61
|
-
private isAtHome;
|
|
62
|
-
private actorRunId?;
|
|
63
|
-
private pricingModel?;
|
|
64
|
-
private purgeChargingLogDataset;
|
|
65
|
-
private useChargingLogDataset;
|
|
66
|
-
private notPpeWarningPrinted;
|
|
67
|
-
private pricingInfo;
|
|
68
|
-
private chargingState?;
|
|
69
|
-
private chargingLogDataset?;
|
|
70
|
-
private apifyClient;
|
|
57
|
+
#private;
|
|
71
58
|
constructor(configuration: Configuration, apifyClient: ApifyClient);
|
|
72
59
|
private get isPayPerEvent();
|
|
73
60
|
private fetchPricingInfo;
|
package/dist/charging.js
CHANGED
|
@@ -15,45 +15,45 @@ export function mergeChargeResults(a, b) {
|
|
|
15
15
|
* Handles pay-per-event charging.
|
|
16
16
|
*/
|
|
17
17
|
export class ChargingManager {
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
18
|
+
#LOCAL_CHARGING_LOG_DATASET_NAME = 'charging_log';
|
|
19
|
+
#PLATFORM_CHARGING_LOG_DATASET_ID_KEY = 'CHARGING_LOG_DATASET_ID';
|
|
20
|
+
#maxTotalChargeUsd;
|
|
21
|
+
#isAtHome;
|
|
22
|
+
#actorRunId;
|
|
23
|
+
#pricingModel;
|
|
24
|
+
#purgeChargingLogDataset;
|
|
25
|
+
#useChargingLogDataset;
|
|
26
|
+
#notPpeWarningPrinted = false;
|
|
27
|
+
#pricingInfo = {};
|
|
28
|
+
#chargingState;
|
|
29
|
+
#chargingLogDataset;
|
|
30
|
+
#apifyClient;
|
|
31
|
+
#configuration;
|
|
32
32
|
constructor(configuration, apifyClient) {
|
|
33
|
-
this
|
|
34
|
-
this
|
|
35
|
-
this
|
|
36
|
-
this
|
|
37
|
-
this
|
|
38
|
-
this
|
|
39
|
-
this
|
|
33
|
+
this.#configuration = configuration;
|
|
34
|
+
this.#maxTotalChargeUsd = configuration.maxTotalChargeUsd || Infinity; // convert `0` to `Infinity` in case the value is an empty string
|
|
35
|
+
this.#isAtHome = configuration.isAtHome;
|
|
36
|
+
this.#actorRunId = configuration.actorRunId;
|
|
37
|
+
this.#purgeChargingLogDataset = configuration.purgeOnStart;
|
|
38
|
+
this.#useChargingLogDataset = configuration.useChargingLogDataset;
|
|
39
|
+
this.#apifyClient = apifyClient;
|
|
40
40
|
}
|
|
41
41
|
get isPayPerEvent() {
|
|
42
|
-
return this
|
|
42
|
+
return this.#pricingModel === 'PAY_PER_EVENT';
|
|
43
43
|
}
|
|
44
44
|
async fetchPricingInfo() {
|
|
45
|
-
if (this
|
|
45
|
+
if (this.#configuration.actorPricingInfo && this.#configuration.chargedEventCounts) {
|
|
46
46
|
return {
|
|
47
|
-
pricingInfo: JSON.parse(this
|
|
48
|
-
chargedEventCounts: JSON.parse(this
|
|
49
|
-
maxTotalChargeUsd: this
|
|
47
|
+
pricingInfo: JSON.parse(this.#configuration.actorPricingInfo),
|
|
48
|
+
chargedEventCounts: JSON.parse(this.#configuration.chargedEventCounts),
|
|
49
|
+
maxTotalChargeUsd: this.#configuration.maxTotalChargeUsd || Infinity,
|
|
50
50
|
};
|
|
51
51
|
}
|
|
52
|
-
if (this
|
|
53
|
-
if (this
|
|
52
|
+
if (this.#isAtHome) {
|
|
53
|
+
if (this.#actorRunId === undefined) {
|
|
54
54
|
throw new Error('Actor run ID not found even though the Actor is running on Apify');
|
|
55
55
|
}
|
|
56
|
-
const run = await this
|
|
56
|
+
const run = await this.#apifyClient.run(this.#actorRunId).get();
|
|
57
57
|
if (run === undefined) {
|
|
58
58
|
throw new Error('Actor run not found');
|
|
59
59
|
}
|
|
@@ -66,7 +66,7 @@ export class ChargingManager {
|
|
|
66
66
|
return {
|
|
67
67
|
pricingInfo: undefined,
|
|
68
68
|
chargedEventCounts: {},
|
|
69
|
-
maxTotalChargeUsd: this
|
|
69
|
+
maxTotalChargeUsd: this.#configuration.maxTotalChargeUsd || Infinity,
|
|
70
70
|
};
|
|
71
71
|
}
|
|
72
72
|
/**
|
|
@@ -74,63 +74,63 @@ export class ChargingManager {
|
|
|
74
74
|
*/
|
|
75
75
|
async init() {
|
|
76
76
|
// Validate config - it may have changed since the instantiation
|
|
77
|
-
if (this
|
|
77
|
+
if (this.#useChargingLogDataset && this.#isAtHome) {
|
|
78
78
|
throw new Error('Using the ACTOR_USE_CHARGING_LOG_DATASET environment variable is only supported in a local development environment');
|
|
79
79
|
}
|
|
80
|
-
if (this
|
|
81
|
-
if (this
|
|
80
|
+
if (this.#configuration.testPayPerEvent) {
|
|
81
|
+
if (this.#isAtHome) {
|
|
82
82
|
throw new Error('Using the ACTOR_TEST_PAY_PER_EVENT environment variable is only supported in a local development environment');
|
|
83
83
|
}
|
|
84
84
|
}
|
|
85
85
|
// Retrieve pricing information
|
|
86
86
|
const { pricingInfo, chargedEventCounts, maxTotalChargeUsd } = await this.fetchPricingInfo();
|
|
87
|
-
if (this
|
|
88
|
-
this
|
|
87
|
+
if (this.#configuration.testPayPerEvent) {
|
|
88
|
+
this.#pricingModel = 'PAY_PER_EVENT';
|
|
89
89
|
}
|
|
90
90
|
else {
|
|
91
|
-
this
|
|
91
|
+
this.#pricingModel ??= pricingInfo?.pricingModel;
|
|
92
92
|
}
|
|
93
93
|
// Load per-event pricing information
|
|
94
94
|
if (pricingInfo?.pricingModel === 'PAY_PER_EVENT') {
|
|
95
95
|
for (const [eventName, eventPricing] of Object.entries(pricingInfo.pricingPerEvent.actorChargeEvents)) {
|
|
96
|
-
this
|
|
96
|
+
this.#pricingInfo[eventName] = {
|
|
97
97
|
price: eventPricing.eventPriceUsd,
|
|
98
98
|
title: eventPricing.eventTitle,
|
|
99
99
|
};
|
|
100
100
|
}
|
|
101
|
-
this
|
|
101
|
+
this.#maxTotalChargeUsd = maxTotalChargeUsd;
|
|
102
102
|
}
|
|
103
|
-
this
|
|
103
|
+
this.#chargingState = {};
|
|
104
104
|
for (const [eventName, chargeCount] of Object.entries(chargedEventCounts ?? {})) {
|
|
105
|
-
this
|
|
105
|
+
this.#chargingState[eventName] = {
|
|
106
106
|
chargeCount,
|
|
107
|
-
totalChargedAmount: chargeCount * (this
|
|
107
|
+
totalChargedAmount: chargeCount * (this.#pricingInfo[eventName]?.price ?? 0),
|
|
108
108
|
};
|
|
109
109
|
}
|
|
110
|
-
if (!this.isPayPerEvent || !this
|
|
110
|
+
if (!this.isPayPerEvent || !this.#useChargingLogDataset) {
|
|
111
111
|
return;
|
|
112
112
|
}
|
|
113
113
|
// Set up charging log dataset
|
|
114
|
-
if (this
|
|
114
|
+
if (this.#isAtHome) {
|
|
115
115
|
const datasetId = await this.ensureChargingLogDatasetOnPlatform();
|
|
116
|
-
this
|
|
116
|
+
this.#chargingLogDataset = await Dataset.open(datasetId);
|
|
117
117
|
}
|
|
118
118
|
else {
|
|
119
|
-
if (this
|
|
120
|
-
const dataset = await Dataset.open(this
|
|
119
|
+
if (this.#purgeChargingLogDataset) {
|
|
120
|
+
const dataset = await Dataset.open(this.#LOCAL_CHARGING_LOG_DATASET_NAME);
|
|
121
121
|
await dataset.drop();
|
|
122
122
|
}
|
|
123
|
-
this
|
|
123
|
+
this.#chargingLogDataset = await Dataset.open(this.#LOCAL_CHARGING_LOG_DATASET_NAME);
|
|
124
124
|
}
|
|
125
125
|
}
|
|
126
126
|
async ensureChargingLogDatasetOnPlatform() {
|
|
127
127
|
const defaultStore = await KeyValueStore.open();
|
|
128
|
-
const storedDatasetId = await defaultStore.getValue(this
|
|
128
|
+
const storedDatasetId = await defaultStore.getValue(this.#PLATFORM_CHARGING_LOG_DATASET_ID_KEY);
|
|
129
129
|
if (storedDatasetId !== null) {
|
|
130
130
|
return storedDatasetId;
|
|
131
131
|
}
|
|
132
|
-
const dataset = await this
|
|
133
|
-
await defaultStore.setValue(this
|
|
132
|
+
const dataset = await this.#apifyClient.datasets().getOrCreate();
|
|
133
|
+
await defaultStore.setValue(this.#PLATFORM_CHARGING_LOG_DATASET_ID_KEY, dataset.id);
|
|
134
134
|
return dataset.id;
|
|
135
135
|
}
|
|
136
136
|
/**
|
|
@@ -138,20 +138,20 @@ export class ChargingManager {
|
|
|
138
138
|
* {@link ChargingManager.getPricingInfo}) require an initialized manager.
|
|
139
139
|
*/
|
|
140
140
|
get isInitialized() {
|
|
141
|
-
return this
|
|
141
|
+
return this.#chargingState !== undefined;
|
|
142
142
|
}
|
|
143
143
|
/**
|
|
144
144
|
* Get information about the pricing for this Actor.
|
|
145
145
|
*/
|
|
146
146
|
getPricingInfo() {
|
|
147
|
-
if (this
|
|
147
|
+
if (this.#chargingState === undefined) {
|
|
148
148
|
throw new Error('ChargingManager is not initialized');
|
|
149
149
|
}
|
|
150
150
|
return {
|
|
151
|
-
pricingModel: this
|
|
151
|
+
pricingModel: this.#pricingModel,
|
|
152
152
|
isPayPerEvent: this.isPayPerEvent,
|
|
153
|
-
maxTotalChargeUsd: this
|
|
154
|
-
perEventPrices: Object.fromEntries(Object.entries(this
|
|
153
|
+
maxTotalChargeUsd: this.#maxTotalChargeUsd,
|
|
154
|
+
perEventPrices: Object.fromEntries(Object.entries(this.#pricingInfo).map(([eventName, { price }]) => [eventName, price])),
|
|
155
155
|
};
|
|
156
156
|
}
|
|
157
157
|
/**
|
|
@@ -171,11 +171,14 @@ export class ChargingManager {
|
|
|
171
171
|
* @param options The name of the event to charge for and the number of events to be charged.
|
|
172
172
|
*/
|
|
173
173
|
async charge({ eventName, count = 1 }) {
|
|
174
|
-
const calculateChargeableWithinLimit = () => Object.fromEntries(Object.keys(this
|
|
174
|
+
const calculateChargeableWithinLimit = () => Object.fromEntries(Object.keys(this.#pricingInfo).map((name) => [
|
|
175
|
+
name,
|
|
176
|
+
this.calculateMaxEventChargeCountWithinLimit(name),
|
|
177
|
+
]));
|
|
175
178
|
if (!this.isPayPerEvent) {
|
|
176
|
-
if (!this
|
|
179
|
+
if (!this.#notPpeWarningPrinted) {
|
|
177
180
|
log.warning('Ignored attempt to charge for an event - the Actor does not use the pay-per-event pricing');
|
|
178
|
-
this
|
|
181
|
+
this.#notPpeWarningPrinted = true;
|
|
179
182
|
}
|
|
180
183
|
return {
|
|
181
184
|
eventChargeLimitReached: false,
|
|
@@ -183,7 +186,7 @@ export class ChargingManager {
|
|
|
183
186
|
chargeableWithinLimit: calculateChargeableWithinLimit(),
|
|
184
187
|
};
|
|
185
188
|
}
|
|
186
|
-
if (this
|
|
189
|
+
if (this.#chargingState === undefined) {
|
|
187
190
|
throw new Error('ChargingManager is not initialized');
|
|
188
191
|
}
|
|
189
192
|
/* START OF CRITICAL SECTION - no awaits here */
|
|
@@ -195,7 +198,7 @@ export class ChargingManager {
|
|
|
195
198
|
// If the caller tries to charge more than the budget allows, overcharge by one event
|
|
196
199
|
// so that the Actor is detected by the platform and terminated.
|
|
197
200
|
// But don't do this if already strictly over the budget - no point piling on charges.
|
|
198
|
-
if (this.calculateTotalChargedAmount() <= this
|
|
201
|
+
if (this.calculateTotalChargedAmount() <= this.#maxTotalChargeUsd) {
|
|
199
202
|
return maxEventChargeCount + 1;
|
|
200
203
|
}
|
|
201
204
|
return 0;
|
|
@@ -207,32 +210,32 @@ export class ChargingManager {
|
|
|
207
210
|
chargeableWithinLimit: calculateChargeableWithinLimit(),
|
|
208
211
|
};
|
|
209
212
|
}
|
|
210
|
-
const pricingInfo = this
|
|
211
|
-
price: this
|
|
213
|
+
const pricingInfo = this.#pricingInfo[eventName] ?? {
|
|
214
|
+
price: this.#isAtHome ? 0 : 1, // Use a nonzero price for local development so that the maximum budget can be reached
|
|
212
215
|
title: `Unknown event '${eventName}'`,
|
|
213
216
|
};
|
|
214
|
-
this
|
|
217
|
+
this.#chargingState[eventName] ??= {
|
|
215
218
|
chargeCount: 0,
|
|
216
219
|
totalChargedAmount: 0,
|
|
217
220
|
};
|
|
218
|
-
this
|
|
219
|
-
this
|
|
221
|
+
this.#chargingState[eventName].chargeCount += chargedCount;
|
|
222
|
+
this.#chargingState[eventName].totalChargedAmount += chargedCount * pricingInfo.price;
|
|
220
223
|
/* END OF CRITICAL SECTION */
|
|
221
|
-
if (this
|
|
224
|
+
if (this.#isAtHome) {
|
|
222
225
|
if (eventName.startsWith('apify-')) {
|
|
223
226
|
// Synthetic events (e.g. apify-default-dataset-item) are tracked locally only,
|
|
224
227
|
// the platform handles them automatically based on dataset writes.
|
|
225
228
|
}
|
|
226
|
-
else if (this
|
|
227
|
-
await this
|
|
229
|
+
else if (this.#pricingInfo[eventName] !== undefined) {
|
|
230
|
+
await this.#apifyClient.run(this.#actorRunId).charge({ eventName, count: chargedCount });
|
|
228
231
|
}
|
|
229
232
|
else {
|
|
230
233
|
log.warning(`Attempting to charge for an unknown event '${eventName}'`);
|
|
231
234
|
}
|
|
232
235
|
}
|
|
233
236
|
const timestamp = new Date().toISOString();
|
|
234
|
-
if (this
|
|
235
|
-
await this
|
|
237
|
+
if (this.#chargingLogDataset !== undefined) {
|
|
238
|
+
await this.#chargingLogDataset.pushData({
|
|
236
239
|
eventName,
|
|
237
240
|
eventTitle: pricingInfo.title,
|
|
238
241
|
eventPriceUsd: pricingInfo.price,
|
|
@@ -254,25 +257,25 @@ export class ChargingManager {
|
|
|
254
257
|
* Get the number of events with given name that the Actor has charged for so far.
|
|
255
258
|
*/
|
|
256
259
|
getChargedEventCount(eventName) {
|
|
257
|
-
if (this
|
|
260
|
+
if (this.#chargingState === undefined) {
|
|
258
261
|
throw new Error('ChargingManager is not initialized');
|
|
259
262
|
}
|
|
260
|
-
return this
|
|
263
|
+
return this.#chargingState[eventName]?.chargeCount ?? 0;
|
|
261
264
|
}
|
|
262
265
|
/**
|
|
263
266
|
* Get the maximum amount of money that the Actor is allowed to charge.
|
|
264
267
|
*/
|
|
265
268
|
getMaxTotalChargeUsd() {
|
|
266
|
-
if (this
|
|
269
|
+
if (this.#chargingState === undefined) {
|
|
267
270
|
throw new Error('ChargingManager is not initialized');
|
|
268
271
|
}
|
|
269
|
-
return this
|
|
272
|
+
return this.#maxTotalChargeUsd;
|
|
270
273
|
}
|
|
271
274
|
calculateTotalChargedAmount() {
|
|
272
|
-
if (this
|
|
275
|
+
if (this.#chargingState === undefined) {
|
|
273
276
|
throw new Error('ChargingManager is not initialized');
|
|
274
277
|
}
|
|
275
|
-
const result = Object.values(this
|
|
278
|
+
const result = Object.values(this.#chargingState)
|
|
276
279
|
.map(({ totalChargedAmount }) => totalChargedAmount)
|
|
277
280
|
.reduce((sum, inc) => sum + inc, 0);
|
|
278
281
|
// Keeping float precision issues at bay
|
|
@@ -283,7 +286,7 @@ export class ChargingManager {
|
|
|
283
286
|
* If the event is not registered, returns Infinity (free of charge)
|
|
284
287
|
*/
|
|
285
288
|
calculateMaxEventChargeCountWithinLimit(eventName) {
|
|
286
|
-
if (this
|
|
289
|
+
if (this.#chargingState === undefined) {
|
|
287
290
|
throw new Error('ChargingManager is not initialized');
|
|
288
291
|
}
|
|
289
292
|
const price = this.calculateEventPrice(eventName);
|
|
@@ -293,11 +296,11 @@ export class ChargingManager {
|
|
|
293
296
|
return this.calculateMaxChargesByPrice(price);
|
|
294
297
|
}
|
|
295
298
|
calculateEventPrice(eventName) {
|
|
296
|
-
return this
|
|
299
|
+
return this.#isAtHome ? this.#pricingInfo[eventName]?.price : 1; // Use a nonzero price for local development so that the maximum budget can be reached
|
|
297
300
|
}
|
|
298
301
|
calculateMaxChargesByPrice(price) {
|
|
299
302
|
// The raw number of events allowed by the budget
|
|
300
|
-
const unroundedResult = (this
|
|
303
|
+
const unroundedResult = (this.#maxTotalChargeUsd - this.calculateTotalChargedAmount()) / price;
|
|
301
304
|
// First round as Math.floor(4.9999999999999999) will incorrectly return 5
|
|
302
305
|
const roundedResult = Math.floor(Number(unroundedResult.toFixed(4)));
|
|
303
306
|
return Math.max(0, roundedResult);
|
|
@@ -307,7 +310,7 @@ export class ChargingManager {
|
|
|
307
310
|
* Returns the limited items and count to charge.
|
|
308
311
|
*/
|
|
309
312
|
calculatePushDataLimits({ items, eventName, isDefaultDataset, }) {
|
|
310
|
-
if (this
|
|
313
|
+
if (this.#chargingState === undefined) {
|
|
311
314
|
throw new Error('ChargingManager is not initialized');
|
|
312
315
|
}
|
|
313
316
|
const itemsArray = Array.isArray(items) ? items : [items];
|
|
@@ -329,7 +332,7 @@ export class ChargingManager {
|
|
|
329
332
|
// But don't do this if already strictly over the budget - no point piling on charges.
|
|
330
333
|
if (itemsArray.length > 0 &&
|
|
331
334
|
maxChargedCount === 0 &&
|
|
332
|
-
this.calculateTotalChargedAmount() <= this
|
|
335
|
+
this.calculateTotalChargedAmount() <= this.#maxTotalChargeUsd) {
|
|
333
336
|
return 1;
|
|
334
337
|
}
|
|
335
338
|
return maxChargedCount;
|
package/dist/index.d.ts
CHANGED
|
@@ -3,7 +3,7 @@ export { ApifyStorageBackend, type ApifyStorageBackendOptions } from './apify_st
|
|
|
3
3
|
export type { RequestQueueAccessMode } from './apify_request_queue_backend.js';
|
|
4
4
|
export { ArgumentValidationError } from './utils.js';
|
|
5
5
|
export { createTransformRequestFunction, type GlobInput, type PseudoUrlInput, type UrlPatternFilters, type UrlPatternRequestOptions, } from './enqueue_links_filters.js';
|
|
6
|
-
export type { OpenStorageOptions, StorageAlias, StorageId, StorageName, StorageIdentifier
|
|
6
|
+
export type { OpenStorageOptions, StorageAlias, StorageId, StorageName, StorageIdentifier } from './storage.js';
|
|
7
7
|
export { ChargeOptions, ChargeResult, ActorPricingInfo, ChargingManager } from './charging.js';
|
|
8
8
|
export * from './configuration.js';
|
|
9
9
|
export * from './proxy_configuration.js';
|