apify 4.0.0-beta.33 → 4.0.0-beta.34
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/actor.d.ts +36 -3
- package/dist/actor.js +100 -50
- package/dist/apify_storage_backend.d.ts +2 -32
- package/dist/apify_storage_backend.js +2 -87
- package/dist/charging.d.ts +39 -30
- package/dist/charging.js +128 -146
- package/dist/charging_storage_backend.d.ts +53 -0
- package/dist/charging_storage_backend.js +105 -0
- package/package.json +1 -1
package/dist/actor.d.ts
CHANGED
|
@@ -13,6 +13,14 @@ import type { ProxyConfigurationOptions } from './proxy_configuration.js';
|
|
|
13
13
|
import { ProxyConfiguration } from './proxy_configuration.js';
|
|
14
14
|
import type { OpenStorageOptions, StorageIdentifier } from './storage.js';
|
|
15
15
|
export interface InitOptions {
|
|
16
|
+
/**
|
|
17
|
+
* Storage backend to use, in place of the Apify platform storage (on the platform) or crawlee's
|
|
18
|
+
* local storage (outside of it).
|
|
19
|
+
*
|
|
20
|
+
* Items pushed to the default dataset are then not charged for under the pay-per-event pricing
|
|
21
|
+
* model: the platform counts an `apify-default-dataset-item` per item it stores itself, and a
|
|
22
|
+
* backend of your own stores them elsewhere.
|
|
23
|
+
*/
|
|
16
24
|
storage?: StorageBackend;
|
|
17
25
|
/**
|
|
18
26
|
* Determines how request queues opened on the Apify platform are consumed.
|
|
@@ -577,6 +585,9 @@ export declare class Actor<Data extends Dictionary = Dictionary> {
|
|
|
577
585
|
* **IMPORTANT**: Make sure to use the `await` keyword when calling `pushData()`,
|
|
578
586
|
* otherwise the Actor process might finish before the data are stored!
|
|
579
587
|
*
|
|
588
|
+
* Inside a crawlee storage transaction, the items are stored - and charged for - at commit time,
|
|
589
|
+
* so the returned counts are zero and charging for an explicit `eventName` is rejected outright.
|
|
590
|
+
*
|
|
580
591
|
* @param item Object or array of objects containing data to be stored in the default dataset.
|
|
581
592
|
* The objects must be serializable to JSON and the JSON representation of each object must be smaller than 9MB.
|
|
582
593
|
* @param eventName If provided, the method will attempt to charge for the event for each pushed item.
|
|
@@ -1125,6 +1136,10 @@ export declare class Actor<Data extends Dictionary = Dictionary> {
|
|
|
1125
1136
|
* **IMPORTANT**: Make sure to use the `await` keyword when calling `pushData()`,
|
|
1126
1137
|
* otherwise the Actor process might finish before the data are stored!
|
|
1127
1138
|
*
|
|
1139
|
+
* Charging is rejected inside a crawlee storage transaction, where the items would only be stored
|
|
1140
|
+
* at commit while the charge happens immediately - use `withDirectStorageAccess()` to opt out of
|
|
1141
|
+
* the transaction, or push without an event name and charge separately.
|
|
1142
|
+
*
|
|
1128
1143
|
* @param item Object or array of objects containing data to be stored in the default dataset.
|
|
1129
1144
|
* The objects must be serializable to JSON and the JSON representation of each object must be smaller than 9MB.
|
|
1130
1145
|
* @param eventName If provided, the method will attempt to charge for the event for each pushed item.
|
|
@@ -1366,10 +1381,28 @@ export declare class Actor<Data extends Dictionary = Dictionary> {
|
|
|
1366
1381
|
* @internal
|
|
1367
1382
|
*/
|
|
1368
1383
|
static setDefaultInstance(instance?: Actor): void;
|
|
1369
|
-
|
|
1370
|
-
|
|
1371
|
-
|
|
1384
|
+
/**
|
|
1385
|
+
* The backend the Actor installs: the caller's, the platform's, or crawlee's local default.
|
|
1386
|
+
*
|
|
1387
|
+
* Only the last two are wrapped for dataset-item charging. The platform counts an
|
|
1388
|
+
* `apify-default-dataset-item` per item written to the run's default dataset through Apify
|
|
1389
|
+
* storage, so a caller-supplied backend is billed nothing and must not be accounted for -
|
|
1390
|
+
* charging for it would spend a budget nobody is consuming and trim the caller's items to fit
|
|
1391
|
+
* it. crawlee's local default stands in for Apify storage, so it is wrapped to keep local
|
|
1392
|
+
* pay-per-event testing faithful.
|
|
1393
|
+
*
|
|
1394
|
+
* Wrapping means owning the instance, which is why the local default is constructed here
|
|
1395
|
+
* rather than left to be created lazily on first use.
|
|
1396
|
+
*/
|
|
1397
|
+
private createStorageBackend;
|
|
1398
|
+
/** Wrapped here rather than at the `init()` call site so that `forceCloud` storages are charged too. */
|
|
1372
1399
|
private createApifyStorageBackend;
|
|
1400
|
+
/**
|
|
1401
|
+
* crawlee's service locator is set-once, so a backend registered before `init()` wins and the
|
|
1402
|
+
* Actor cannot install its own over it. Replaces the resulting `ServiceConflictError` with the
|
|
1403
|
+
* way out.
|
|
1404
|
+
*/
|
|
1405
|
+
private installStorageBackend;
|
|
1373
1406
|
private ensureActorInit;
|
|
1374
1407
|
/**
|
|
1375
1408
|
* Get time remaining from the Actor run timeout in seconds, rounded up to whole seconds with minimum value of 1 second.
|
package/dist/actor.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { createPrivateKey } from 'node:crypto';
|
|
2
|
-
import { Dataset, EventType, KeyValueStore, purgeDefaultStorages, RequestQueue, serviceLocator } from '@crawlee/core';
|
|
2
|
+
import { Dataset, EventType, KeyValueStore, purgeDefaultStorages, rejectOperationInTransaction, RequestQueue, ServiceLocator, serviceLocator, } from '@crawlee/core';
|
|
3
3
|
import { sleep } from '@crawlee/utils';
|
|
4
4
|
import { ApifyClient } from 'apify-client';
|
|
5
5
|
import { z } from 'zod';
|
|
@@ -8,8 +8,9 @@ import { decryptInputSecrets } from '@apify/input_secrets';
|
|
|
8
8
|
import log from '@apify/log';
|
|
9
9
|
import { addTimeoutToPromise } from '@apify/timeout';
|
|
10
10
|
import { parseArgument } from '@apify/validations';
|
|
11
|
-
import { ApifyStorageBackend
|
|
12
|
-
import { ChargingManager,
|
|
11
|
+
import { ApifyStorageBackend } from './apify_storage_backend.js';
|
|
12
|
+
import { ChargingManager, DEFAULT_DATASET_ITEM_EVENT } from './charging.js';
|
|
13
|
+
import { ChargingDatasetBackend, ChargingStorageBackend } from './charging_storage_backend.js';
|
|
13
14
|
import { Configuration } from './configuration.js';
|
|
14
15
|
import { getDefaultsFromInputSchema, noActorInputSchemaDefinedMarker, readInputSchema } from './input-schemas.js';
|
|
15
16
|
import { PlatformEventManager } from './platform_event_manager.js';
|
|
@@ -208,11 +209,13 @@ export class Actor {
|
|
|
208
209
|
serviceLocator.setConfiguration(this.configuration);
|
|
209
210
|
this.#requestQueueAccess = options.requestQueueAccess ?? 'single';
|
|
210
211
|
if (this.isAtHome()) {
|
|
211
|
-
serviceLocator.setStorageBackend(this.createApifyStorageBackend());
|
|
212
212
|
serviceLocator.setEventManager(this.eventManager);
|
|
213
213
|
}
|
|
214
|
+
if (!serviceLocator.getServicesIfSet().storageBackend) {
|
|
215
|
+
this.installStorageBackend(this.createStorageBackend(options.storage));
|
|
216
|
+
}
|
|
214
217
|
else if (options.storage) {
|
|
215
|
-
|
|
218
|
+
this.installStorageBackend(options.storage);
|
|
216
219
|
}
|
|
217
220
|
// Init the event manager the config uses
|
|
218
221
|
await serviceLocator.getEventManager().init();
|
|
@@ -250,6 +253,13 @@ export class Actor {
|
|
|
250
253
|
log.debug(`Default storages purged`);
|
|
251
254
|
await this.#chargingManager.init();
|
|
252
255
|
log.debug(`ChargingManager initialized`, this.#chargingManager.getPricingInfo());
|
|
256
|
+
// Only knowable once the pricing is loaded. Reachable with a caller-supplied backend or one
|
|
257
|
+
// registered with crawlee directly; the storages the Actor installs itself are wrapped.
|
|
258
|
+
if (this.#chargingManager.isPayPerEvent &&
|
|
259
|
+
!(serviceLocator.getStorageBackend() instanceof ChargingStorageBackend)) {
|
|
260
|
+
log.warning('Items pushed to the default dataset will not be charged for, because this run does not use Apify ' +
|
|
261
|
+
'storage - the platform only counts items it stores itself.');
|
|
262
|
+
}
|
|
253
263
|
}
|
|
254
264
|
/**
|
|
255
265
|
* @ignore
|
|
@@ -633,6 +643,9 @@ export class Actor {
|
|
|
633
643
|
* **IMPORTANT**: Make sure to use the `await` keyword when calling `pushData()`,
|
|
634
644
|
* otherwise the Actor process might finish before the data are stored!
|
|
635
645
|
*
|
|
646
|
+
* Inside a crawlee storage transaction, the items are stored - and charged for - at commit time,
|
|
647
|
+
* so the returned counts are zero and charging for an explicit `eventName` is rejected outright.
|
|
648
|
+
*
|
|
636
649
|
* @param item Object or array of objects containing data to be stored in the default dataset.
|
|
637
650
|
* The objects must be serializable to JSON and the JSON representation of each object must be smaller than 9MB.
|
|
638
651
|
* @param eventName If provided, the method will attempt to charge for the event for each pushed item.
|
|
@@ -643,17 +656,49 @@ export class Actor {
|
|
|
643
656
|
if (eventName?.startsWith('apify-')) {
|
|
644
657
|
throw new Error(`Cannot charge for synthetic event '${eventName}' manually`);
|
|
645
658
|
}
|
|
659
|
+
if (eventName !== undefined) {
|
|
660
|
+
rejectOperationInTransaction(`Actor.pushData() with the event name '${eventName}'`, 'the items are stored when the transaction commits, but the charge happens right away, ' +
|
|
661
|
+
'so a rolled-back request would be billed for items that were never stored.');
|
|
662
|
+
}
|
|
646
663
|
const dataset = await this.openDataset();
|
|
647
|
-
|
|
648
|
-
//
|
|
649
|
-
//
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
if (this.usesPushDataInterception(dataset)) {
|
|
654
|
-
return await this.pushDataViaInterceptedClient(dataset, item, eventName);
|
|
664
|
+
const items = Array.isArray(item) ? item : [item];
|
|
665
|
+
// `Actor.pushData()` historically worked even without calling `Actor.init()`.
|
|
666
|
+
// In that case, charging isn't configured, so just push the data through.
|
|
667
|
+
if (!this.#chargingManager.isInitialized) {
|
|
668
|
+
await dataset.pushData(items);
|
|
669
|
+
return { eventChargeLimitReached: false, chargedCount: 0, chargeableWithinLimit: {} };
|
|
655
670
|
}
|
|
656
|
-
|
|
671
|
+
// The reservation below and the charge that acts on it have to stay under one lock, so that
|
|
672
|
+
// a concurrent push cannot charge in between. The dataset backend re-enters the same lock to
|
|
673
|
+
// charge the synthetic per-item event.
|
|
674
|
+
return await this.#chargingManager.withChargeLock(async () => {
|
|
675
|
+
if (eventName === undefined) {
|
|
676
|
+
// Nothing to charge here - the dataset backend charges the synthetic event for
|
|
677
|
+
// whatever it stores, so the result is read back off the charging state.
|
|
678
|
+
const chargedBefore = this.#chargingManager.getChargedEventCount(DEFAULT_DATASET_ITEM_EVENT);
|
|
679
|
+
await dataset.pushData(items);
|
|
680
|
+
return {
|
|
681
|
+
eventChargeLimitReached: this.#chargingManager.isEventChargeLimitReached(DEFAULT_DATASET_ITEM_EVENT),
|
|
682
|
+
chargedCount: this.#chargingManager.getChargedEventCount(DEFAULT_DATASET_ITEM_EVENT) - chargedBefore,
|
|
683
|
+
chargeableWithinLimit: this.#chargingManager.calculateChargeableWithinLimit(),
|
|
684
|
+
};
|
|
685
|
+
}
|
|
686
|
+
const limit = this.#chargingManager.calculatePushDataLimit(items.length, {
|
|
687
|
+
eventName,
|
|
688
|
+
// Pushing one item charges the synthetic per-item event too when the dataset backend
|
|
689
|
+
// is the charging one, so that price is part of what an item costs here.
|
|
690
|
+
isDefaultDataset: dataset.backend instanceof ChargingDatasetBackend,
|
|
691
|
+
});
|
|
692
|
+
if (limit === 0) {
|
|
693
|
+
return {
|
|
694
|
+
eventChargeLimitReached: items.length > 0,
|
|
695
|
+
chargedCount: 0,
|
|
696
|
+
chargeableWithinLimit: this.#chargingManager.calculateChargeableWithinLimit(),
|
|
697
|
+
};
|
|
698
|
+
}
|
|
699
|
+
await dataset.pushData(limit < items.length ? items.slice(0, limit) : items);
|
|
700
|
+
return await this.#chargingManager.charge({ eventName, count: limit });
|
|
701
|
+
});
|
|
657
702
|
}
|
|
658
703
|
/**
|
|
659
704
|
* Opens a dataset and returns a promise resolving to an instance of the {@link Dataset} class.
|
|
@@ -1600,51 +1645,56 @@ export class Actor {
|
|
|
1600
1645
|
static setDefaultInstance(instance) {
|
|
1601
1646
|
Actor.#instance = instance;
|
|
1602
1647
|
}
|
|
1603
|
-
|
|
1604
|
-
|
|
1605
|
-
|
|
1606
|
-
|
|
1607
|
-
|
|
1608
|
-
|
|
1609
|
-
|
|
1610
|
-
|
|
1611
|
-
|
|
1612
|
-
|
|
1613
|
-
|
|
1614
|
-
|
|
1615
|
-
|
|
1616
|
-
|
|
1617
|
-
|
|
1618
|
-
|
|
1619
|
-
}
|
|
1620
|
-
|
|
1621
|
-
|
|
1622
|
-
// `Actor.pushData()` historically worked even without calling `Actor.init()`.
|
|
1623
|
-
// In that case, charging isn't configured, so just push the data through.
|
|
1624
|
-
if (!this.initialized && explicitEventName === undefined) {
|
|
1625
|
-
await dataset.pushData(items);
|
|
1626
|
-
return {
|
|
1627
|
-
eventChargeLimitReached: false,
|
|
1628
|
-
chargedCount: 0,
|
|
1629
|
-
chargeableWithinLimit: {},
|
|
1630
|
-
};
|
|
1648
|
+
/**
|
|
1649
|
+
* The backend the Actor installs: the caller's, the platform's, or crawlee's local default.
|
|
1650
|
+
*
|
|
1651
|
+
* Only the last two are wrapped for dataset-item charging. The platform counts an
|
|
1652
|
+
* `apify-default-dataset-item` per item written to the run's default dataset through Apify
|
|
1653
|
+
* storage, so a caller-supplied backend is billed nothing and must not be accounted for -
|
|
1654
|
+
* charging for it would spend a budget nobody is consuming and trim the caller's items to fit
|
|
1655
|
+
* it. crawlee's local default stands in for Apify storage, so it is wrapped to keep local
|
|
1656
|
+
* pay-per-event testing faithful.
|
|
1657
|
+
*
|
|
1658
|
+
* Wrapping means owning the instance, which is why the local default is constructed here
|
|
1659
|
+
* rather than left to be created lazily on first use.
|
|
1660
|
+
*/
|
|
1661
|
+
createStorageBackend(storage) {
|
|
1662
|
+
if (storage) {
|
|
1663
|
+
return storage;
|
|
1664
|
+
}
|
|
1665
|
+
if (this.isAtHome()) {
|
|
1666
|
+
return this.createApifyStorageBackend();
|
|
1631
1667
|
}
|
|
1632
|
-
|
|
1633
|
-
|
|
1634
|
-
|
|
1635
|
-
items,
|
|
1636
|
-
eventName: explicitEventName,
|
|
1637
|
-
isDefaultDataset,
|
|
1638
|
-
pushFn: async (limitedItems) => dataset.pushData(limitedItems),
|
|
1668
|
+
return new ChargingStorageBackend(new ServiceLocator(this.configuration).getStorageBackend(), {
|
|
1669
|
+
configuration: this.configuration,
|
|
1670
|
+
getChargingManager: () => this.#chargingManager,
|
|
1639
1671
|
});
|
|
1640
1672
|
}
|
|
1673
|
+
/** Wrapped here rather than at the `init()` call site so that `forceCloud` storages are charged too. */
|
|
1641
1674
|
createApifyStorageBackend() {
|
|
1642
|
-
|
|
1675
|
+
const backend = new ApifyStorageBackend(this.apifyClient, {
|
|
1643
1676
|
configuration: this.configuration,
|
|
1644
1677
|
requestQueueAccess: this.#requestQueueAccess,
|
|
1678
|
+
});
|
|
1679
|
+
return new ChargingStorageBackend(backend, {
|
|
1680
|
+
configuration: this.configuration,
|
|
1645
1681
|
getChargingManager: () => this.#chargingManager,
|
|
1646
1682
|
});
|
|
1647
1683
|
}
|
|
1684
|
+
/**
|
|
1685
|
+
* crawlee's service locator is set-once, so a backend registered before `init()` wins and the
|
|
1686
|
+
* Actor cannot install its own over it. Replaces the resulting `ServiceConflictError` with the
|
|
1687
|
+
* way out.
|
|
1688
|
+
*/
|
|
1689
|
+
installStorageBackend(backend) {
|
|
1690
|
+
try {
|
|
1691
|
+
serviceLocator.setStorageBackend(backend);
|
|
1692
|
+
}
|
|
1693
|
+
catch (error) {
|
|
1694
|
+
throw new Error('A storage backend is already registered with crawlee, so the Actor cannot install its own. Pass it ' +
|
|
1695
|
+
'as `Actor.init({ storage })` instead of calling `serviceLocator.setStorageBackend()`.', { cause: error });
|
|
1696
|
+
}
|
|
1697
|
+
}
|
|
1648
1698
|
ensureActorInit(methodCalled) {
|
|
1649
1699
|
// If we already warned the user once, don't do it again to prevent spam
|
|
1650
1700
|
if (this.#warnedAboutMissingInitCall) {
|
|
@@ -1,24 +1,10 @@
|
|
|
1
|
-
import { AsyncLocalStorage } from 'node:async_hooks';
|
|
2
1
|
import type { DatasetBackend, KeyValueStoreBackend, RequestQueueBackend, StorageBackend, StorageIdentifier } from '@crawlee/types';
|
|
3
2
|
import type { ApifyClient } from 'apify-client';
|
|
4
3
|
import { type RequestQueueAccessMode } from './apify_request_queue_backend.js';
|
|
5
|
-
import { type ChargeResult, type ChargingManager } from './charging.js';
|
|
6
4
|
import type { Configuration } from './configuration.js';
|
|
7
5
|
type StorageType = 'Dataset' | 'KeyValueStore' | 'RequestQueue';
|
|
8
|
-
/**
|
|
9
|
-
export declare const
|
|
10
|
-
/**
|
|
11
|
-
* Context of a single `Actor.pushData()` call, shared with the intercepted
|
|
12
|
-
* `pushItems()` calls so they can (1) know which event to charge and
|
|
13
|
-
* (2) aggregate the {@link ChargeResult} across the multiple `pushItems()`
|
|
14
|
-
* calls a single `pushData()` may trigger (the backend splits pushes exceeding
|
|
15
|
-
* the API's payload size limit).
|
|
16
|
-
*/
|
|
17
|
-
export interface PpeAwarePushDataContext {
|
|
18
|
-
eventName: string | undefined;
|
|
19
|
-
chargeResult?: ChargeResult;
|
|
20
|
-
}
|
|
21
|
-
export declare const pushDataChargingContext: AsyncLocalStorage<PpeAwarePushDataContext>;
|
|
6
|
+
/** The reserved alias crawlee uses for the default (unnamed) storage. */
|
|
7
|
+
export declare const DEFAULT_STORAGE_ALIAS = "__default__";
|
|
22
8
|
export interface ApifyStorageBackendOptions {
|
|
23
9
|
/**
|
|
24
10
|
* SDK configuration providing the run's default storage ids and related environment values.
|
|
@@ -32,12 +18,6 @@ export interface ApifyStorageBackendOptions {
|
|
|
32
18
|
* consumers can process the same queue safely.
|
|
33
19
|
*/
|
|
34
20
|
requestQueueAccess?: RequestQueueAccessMode;
|
|
35
|
-
/**
|
|
36
|
-
* Supplies the charging manager for pay-per-event runs, enabling the charging-aware default
|
|
37
|
-
* dataset client.
|
|
38
|
-
* @internal
|
|
39
|
-
*/
|
|
40
|
-
getChargingManager?: () => ChargingManager;
|
|
41
21
|
}
|
|
42
22
|
/**
|
|
43
23
|
* Bridges `apify-client`'s synchronous resource accessors (`dataset(id)`,
|
|
@@ -45,10 +25,6 @@ export interface ApifyStorageBackendOptions {
|
|
|
45
25
|
* `StorageBackend` interface (async factory methods accepting an `id`,
|
|
46
26
|
* a `name`, or an `alias`).
|
|
47
27
|
*
|
|
48
|
-
* For the run's default dataset it transparently swaps in a charging-aware
|
|
49
|
-
* dataset client (pay-per-event on `Actor.pushData()`), provided a charging
|
|
50
|
-
* manager is supplied and a default-dataset-item price is configured.
|
|
51
|
-
*
|
|
52
28
|
* `Actor` wires this up automatically; construct it directly only to use Apify
|
|
53
29
|
* platform storage with crawlee's storage classes outside of `Actor` — e.g. to
|
|
54
30
|
* read another run's output with an explicit token:
|
|
@@ -82,12 +58,6 @@ export declare class ApifyStorageBackend implements StorageBackend {
|
|
|
82
58
|
* migrated or resurrected run re-acquire the request locks of its previous incarnation.
|
|
83
59
|
*/
|
|
84
60
|
private requestQueueClientKey;
|
|
85
|
-
/**
|
|
86
|
-
* Returns a charging-aware dataset client when `id` is the run's default
|
|
87
|
-
* dataset and a default-dataset-item price is configured; otherwise
|
|
88
|
-
* `undefined` (caller uses the plain client).
|
|
89
|
-
*/
|
|
90
|
-
private chargingDatasetClient;
|
|
91
61
|
/**
|
|
92
62
|
* Resolves a crawlee {@link StorageIdentifier} to a platform storage id.
|
|
93
63
|
*
|
|
@@ -1,7 +1,4 @@
|
|
|
1
|
-
/* eslint-disable max-classes-per-file */
|
|
2
|
-
import { AsyncLocalStorage } from 'node:async_hooks';
|
|
3
1
|
import { createHash } from 'node:crypto';
|
|
4
|
-
import { DatasetClient as ApifyDatasetClient } from 'apify-client';
|
|
5
2
|
import log from '@apify/log';
|
|
6
3
|
import { cryptoRandomObjectId } from '@apify/utilities';
|
|
7
4
|
import { ApifyDatasetBackend } from './apify_dataset_backend.js';
|
|
@@ -9,9 +6,8 @@ import { ApifyKeyValueStoreBackend } from './apify_key_value_store_backend.js';
|
|
|
9
6
|
import { AsyncLock } from './apify_request_queue_backend.js';
|
|
10
7
|
import { ApifyRequestQueueSharedBackend } from './apify_request_queue_shared_backend.js';
|
|
11
8
|
import { ApifyRequestQueueSingleBackend } from './apify_request_queue_single_backend.js';
|
|
12
|
-
import { DEFAULT_DATASET_ITEM_EVENT, mergeChargeResults, pushDataAndCharge, } from './charging.js';
|
|
13
9
|
/** The reserved alias crawlee uses for the default (unnamed) storage. */
|
|
14
|
-
const DEFAULT_STORAGE_ALIAS = '__default__';
|
|
10
|
+
export const DEFAULT_STORAGE_ALIAS = '__default__';
|
|
15
11
|
/** The key of the default key-value store record holding this run's alias -> storage id mapping. */
|
|
16
12
|
const ALIAS_MAPPING_RECORD_KEY = '__STORAGE_ALIASES_MAPPING';
|
|
17
13
|
async function readAliasMapping(store) {
|
|
@@ -30,62 +26,12 @@ const ACTOR_STORAGES_TYPE_KEY = {
|
|
|
30
26
|
KeyValueStore: 'keyValueStores',
|
|
31
27
|
RequestQueue: 'requestQueues',
|
|
32
28
|
};
|
|
33
|
-
/** Marks a dataset backend whose underlying client charges for pushed items (pay-per-event). @internal */
|
|
34
|
-
export const USES_PUSH_DATA_INTERCEPTION = Symbol('apify:uses-push-data-interception');
|
|
35
|
-
export const pushDataChargingContext = new AsyncLocalStorage();
|
|
36
|
-
/**
|
|
37
|
-
* Default `DatasetClient` that charges for pushed items (pay-per-event). Used
|
|
38
|
-
* only for the run's default dataset when a `apify-default-dataset-item` price
|
|
39
|
-
* is configured; for everything else the plain `apify-client` dataset client is
|
|
40
|
-
* used.
|
|
41
|
-
*/
|
|
42
|
-
class PpeAwareDatasetClient extends ApifyDatasetClient {
|
|
43
|
-
#getChargingManager;
|
|
44
|
-
constructor(options, getChargingManager) {
|
|
45
|
-
super(options);
|
|
46
|
-
this.#getChargingManager = getChargingManager;
|
|
47
|
-
}
|
|
48
|
-
normalizeItems(items) {
|
|
49
|
-
if (typeof items === 'string') {
|
|
50
|
-
const parsed = JSON.parse(items);
|
|
51
|
-
return Array.isArray(parsed) ? parsed : [parsed];
|
|
52
|
-
}
|
|
53
|
-
if (Array.isArray(items)) {
|
|
54
|
-
return items.flatMap((item) => typeof item === 'string' ? JSON.parse(item) : item);
|
|
55
|
-
}
|
|
56
|
-
return [items];
|
|
57
|
-
}
|
|
58
|
-
async pushItems(items) {
|
|
59
|
-
const context = pushDataChargingContext.getStore();
|
|
60
|
-
// A single JSON string may encode multiple items (e.g. '[{...},{...}]'),
|
|
61
|
-
// which the charging logic would miscount — parse strings into arrays so
|
|
62
|
-
// each logical item is counted individually.
|
|
63
|
-
const normalizedItems = this.normalizeItems(items);
|
|
64
|
-
const result = await pushDataAndCharge({
|
|
65
|
-
chargingManager: this.#getChargingManager(),
|
|
66
|
-
items: normalizedItems,
|
|
67
|
-
eventName: context?.eventName,
|
|
68
|
-
isDefaultDataset: true,
|
|
69
|
-
// stringify for faster validation in the Apify client
|
|
70
|
-
pushFn: async (limitedItems) => super.pushItems(JSON.stringify(limitedItems)),
|
|
71
|
-
});
|
|
72
|
-
if (!context)
|
|
73
|
-
return;
|
|
74
|
-
// One `Actor.pushData()` may map to several `pushItems()` calls — aggregate.
|
|
75
|
-
context.chargeResult =
|
|
76
|
-
context.chargeResult === undefined ? result : mergeChargeResults(context.chargeResult, result);
|
|
77
|
-
}
|
|
78
|
-
}
|
|
79
29
|
/**
|
|
80
30
|
* Bridges `apify-client`'s synchronous resource accessors (`dataset(id)`,
|
|
81
31
|
* `keyValueStore(id)`, `requestQueue(id, options?)`) to crawlee v4's
|
|
82
32
|
* `StorageBackend` interface (async factory methods accepting an `id`,
|
|
83
33
|
* a `name`, or an `alias`).
|
|
84
34
|
*
|
|
85
|
-
* For the run's default dataset it transparently swaps in a charging-aware
|
|
86
|
-
* dataset client (pay-per-event on `Actor.pushData()`), provided a charging
|
|
87
|
-
* manager is supplied and a default-dataset-item price is configured.
|
|
88
|
-
*
|
|
89
35
|
* `Actor` wires this up automatically; construct it directly only to use Apify
|
|
90
36
|
* platform storage with crawlee's storage classes outside of `Actor` — e.g. to
|
|
91
37
|
* read another run's output with an explicit token:
|
|
@@ -102,7 +48,6 @@ export class ApifyStorageBackend {
|
|
|
102
48
|
#client;
|
|
103
49
|
#config;
|
|
104
50
|
#requestQueueAccess;
|
|
105
|
-
#getChargingManager;
|
|
106
51
|
/** Unnamed storages resolved for aliases in this process, keyed like {@link AliasMapping}. */
|
|
107
52
|
#aliasIdCache = new Map();
|
|
108
53
|
/** The alias mapping read from the run's default key-value store; `undefined` until first read. */
|
|
@@ -115,7 +60,6 @@ export class ApifyStorageBackend {
|
|
|
115
60
|
this.#client = client;
|
|
116
61
|
this.#config = options.configuration;
|
|
117
62
|
this.#requestQueueAccess = options.requestQueueAccess ?? 'single';
|
|
118
|
-
this.#getChargingManager = options.getChargingManager;
|
|
119
63
|
}
|
|
120
64
|
/**
|
|
121
65
|
* Partitions crawlee's storage-instance cache by API base URL and token, so the same storage
|
|
@@ -144,14 +88,7 @@ export class ApifyStorageBackend {
|
|
|
144
88
|
}
|
|
145
89
|
async createDatasetBackend(options) {
|
|
146
90
|
const id = await this.resolveId(options, 'Dataset');
|
|
147
|
-
|
|
148
|
-
const backend = new ApifyDatasetBackend(chargingClient ?? this.#client.dataset(id));
|
|
149
|
-
if (chargingClient) {
|
|
150
|
-
// `Actor.pushData()` looks for this marker on the dataset's backend to know the
|
|
151
|
-
// pay-per-event charging happens inside the intercepted `pushItems()` calls.
|
|
152
|
-
Object.assign(backend, { [USES_PUSH_DATA_INTERCEPTION]: true });
|
|
153
|
-
}
|
|
154
|
-
return backend;
|
|
91
|
+
return new ApifyDatasetBackend(this.#client.dataset(id));
|
|
155
92
|
}
|
|
156
93
|
async createKeyValueStoreBackend(options) {
|
|
157
94
|
const id = await this.resolveId(options, 'KeyValueStore');
|
|
@@ -172,28 +109,6 @@ export class ApifyStorageBackend {
|
|
|
172
109
|
const key = this.#config?.actorRunId ?? (this.#fallbackClientKey ??= cryptoRandomObjectId(MAX_CLIENT_KEY_LENGTH));
|
|
173
110
|
return key.slice(0, MAX_CLIENT_KEY_LENGTH);
|
|
174
111
|
}
|
|
175
|
-
/**
|
|
176
|
-
* Returns a charging-aware dataset client when `id` is the run's default
|
|
177
|
-
* dataset and a default-dataset-item price is configured; otherwise
|
|
178
|
-
* `undefined` (caller uses the plain client).
|
|
179
|
-
*/
|
|
180
|
-
chargingDatasetClient(id) {
|
|
181
|
-
const getChargingManager = this.#getChargingManager;
|
|
182
|
-
if (!getChargingManager)
|
|
183
|
-
return undefined;
|
|
184
|
-
if (id !== this.#config?.defaultDatasetId)
|
|
185
|
-
return undefined;
|
|
186
|
-
const hasDefaultDatasetItemEvent = DEFAULT_DATASET_ITEM_EVENT in getChargingManager().getPricingInfo().perEventPrices;
|
|
187
|
-
if (!hasDefaultDatasetItemEvent)
|
|
188
|
-
return undefined;
|
|
189
|
-
return new PpeAwareDatasetClient({
|
|
190
|
-
id,
|
|
191
|
-
baseUrl: this.#client.baseUrl,
|
|
192
|
-
publicBaseUrl: this.#client.publicBaseUrl,
|
|
193
|
-
apifyClient: this.#client,
|
|
194
|
-
httpClient: this.#client.httpClient,
|
|
195
|
-
}, getChargingManager);
|
|
196
|
-
}
|
|
197
112
|
/**
|
|
198
113
|
* Resolves a crawlee {@link StorageIdentifier} to a platform storage id.
|
|
199
114
|
*
|
package/dist/charging.d.ts
CHANGED
|
@@ -12,6 +12,11 @@ export interface ChargeOptions {
|
|
|
12
12
|
* @default 1
|
|
13
13
|
*/
|
|
14
14
|
count?: number;
|
|
15
|
+
/**
|
|
16
|
+
* Repeated charges sharing a key are billed once - e.g. a request handler rerun by a retry.
|
|
17
|
+
* Local accounting still counts every call, so the remaining budget comes out underestimated.
|
|
18
|
+
*/
|
|
19
|
+
idempotencyKey?: string;
|
|
15
20
|
}
|
|
16
21
|
export interface ChargeResult {
|
|
17
22
|
/**
|
|
@@ -49,14 +54,14 @@ export interface ActorPricingInfo {
|
|
|
49
54
|
isPayPerEvent: boolean;
|
|
50
55
|
perEventPrices: Record<string, number>;
|
|
51
56
|
}
|
|
52
|
-
export declare function mergeChargeResults(a: ChargeResult, b: ChargeResult): ChargeResult;
|
|
53
57
|
/**
|
|
54
58
|
* Handles pay-per-event charging.
|
|
55
59
|
*/
|
|
56
60
|
export declare class ChargingManager {
|
|
57
61
|
#private;
|
|
58
62
|
constructor(configuration: Configuration, apifyClient: ApifyClient);
|
|
59
|
-
|
|
63
|
+
/** A shortcut - true if the Actor runs with the pay-per-event pricing model. */
|
|
64
|
+
get isPayPerEvent(): boolean;
|
|
60
65
|
private fetchPricingInfo;
|
|
61
66
|
/**
|
|
62
67
|
* Initialize the ChargingManager by loading pricing information and charging state via Apify API.
|
|
@@ -88,7 +93,7 @@ export declare class ChargingManager {
|
|
|
88
93
|
*
|
|
89
94
|
* @param options The name of the event to charge for and the number of events to be charged.
|
|
90
95
|
*/
|
|
91
|
-
charge({ eventName, count }: ChargeOptions): Promise<ChargeResult>;
|
|
96
|
+
charge({ eventName, count, idempotencyKey }: ChargeOptions): Promise<ChargeResult>;
|
|
92
97
|
/**
|
|
93
98
|
* Get the number of events with given name that the Actor has charged for so far.
|
|
94
99
|
*/
|
|
@@ -106,31 +111,35 @@ export declare class ChargingManager {
|
|
|
106
111
|
private calculateEventPrice;
|
|
107
112
|
private calculateMaxChargesByPrice;
|
|
108
113
|
/**
|
|
109
|
-
*
|
|
110
|
-
*
|
|
111
|
-
*/
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
114
|
+
* Whether the remaining budget is insufficient to charge even a single event of the given type.
|
|
115
|
+
* Always false when nothing is charged in the first place.
|
|
116
|
+
*/
|
|
117
|
+
isEventChargeLimitReached(eventName: string): boolean;
|
|
118
|
+
/**
|
|
119
|
+
* How many events of each known type can still be charged within the limit.
|
|
120
|
+
*/
|
|
121
|
+
calculateChargeableWithinLimit(): Record<string, number>;
|
|
122
|
+
/**
|
|
123
|
+
* How many of `itemsCount` items may be pushed to a dataset within the remaining budget.
|
|
124
|
+
*
|
|
125
|
+
* The budget is measured against the combined per-item price of the explicit `eventName` and - on
|
|
126
|
+
* the default dataset - the synthetic {@link DEFAULT_DATASET_ITEM_EVENT}, since pushing one item
|
|
127
|
+
* charges both.
|
|
128
|
+
*
|
|
129
|
+
* When the budget cannot cover a single item, one item is still allowed through so that the
|
|
130
|
+
* resulting overcharge makes the platform terminate the run - unless the run is already strictly
|
|
131
|
+
* over budget, in which case nothing is allowed through.
|
|
132
|
+
*/
|
|
133
|
+
calculatePushDataLimit(itemsCount: number, { eventName, isDefaultDataset }?: {
|
|
134
|
+
eventName?: string;
|
|
135
|
+
isDefaultDataset?: boolean;
|
|
136
|
+
}): number;
|
|
137
|
+
/**
|
|
138
|
+
* Runs `fn` under the charge lock, which keeps a limit reservation and the charge that acts on it
|
|
139
|
+
* atomic against concurrent pushes. Only pay-per-event runs charge anything, so for every other
|
|
140
|
+
* run there is nothing to serialize and the lock is skipped.
|
|
141
|
+
*
|
|
142
|
+
* @internal
|
|
143
|
+
*/
|
|
144
|
+
withChargeLock<T>(fn: () => Promise<T>): Promise<T>;
|
|
120
145
|
}
|
|
121
|
-
/**
|
|
122
|
-
* Helper for PPE-aware pushing of data to the dataset.
|
|
123
|
-
*
|
|
124
|
-
* 1. Calculate limits based on budget
|
|
125
|
-
* 2. Push limited items via the provided callback
|
|
126
|
-
* 3. Charge for the events
|
|
127
|
-
*
|
|
128
|
-
* @internal
|
|
129
|
-
*/
|
|
130
|
-
export declare function pushDataAndCharge<T>({ chargingManager, items, eventName, isDefaultDataset, pushFn, }: {
|
|
131
|
-
chargingManager: ChargingManager;
|
|
132
|
-
items: T | T[];
|
|
133
|
-
eventName: string | undefined;
|
|
134
|
-
isDefaultDataset: boolean;
|
|
135
|
-
pushFn: (limitedItems: T | T[]) => Promise<void>;
|
|
136
|
-
}): Promise<ChargeResult>;
|
package/dist/charging.js
CHANGED
|
@@ -1,15 +1,23 @@
|
|
|
1
|
+
import { AsyncLocalStorage } from 'node:async_hooks';
|
|
1
2
|
import { Dataset, KeyValueStore } from '@crawlee/core';
|
|
2
3
|
import log from '@apify/log';
|
|
3
4
|
export const DEFAULT_DATASET_ITEM_EVENT = 'apify-default-dataset-item';
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
5
|
+
/**
|
|
6
|
+
* A FIFO mutex that a critical section may re-enter from a nested call — the charge lock is taken by
|
|
7
|
+
* `Actor.pushData()` and again, one level down, by the dataset backend it pushes through.
|
|
8
|
+
*/
|
|
9
|
+
class ReentrantAsyncLock {
|
|
10
|
+
#tail = Promise.resolve();
|
|
11
|
+
#held = new AsyncLocalStorage();
|
|
12
|
+
async runExclusive(fn) {
|
|
13
|
+
if (this.#held.getStore()) {
|
|
14
|
+
return await fn();
|
|
15
|
+
}
|
|
16
|
+
const run = this.#tail.then(async () => this.#held.run(true, fn));
|
|
17
|
+
// Keep the chain alive even when the critical section throws.
|
|
18
|
+
this.#tail = run.catch(() => { });
|
|
19
|
+
return await run;
|
|
20
|
+
}
|
|
13
21
|
}
|
|
14
22
|
/**
|
|
15
23
|
* Handles pay-per-event charging.
|
|
@@ -27,6 +35,7 @@ export class ChargingManager {
|
|
|
27
35
|
#pricingInfo = {};
|
|
28
36
|
#chargingState;
|
|
29
37
|
#chargingLogDataset;
|
|
38
|
+
#chargeLock = new ReentrantAsyncLock();
|
|
30
39
|
#apifyClient;
|
|
31
40
|
#configuration;
|
|
32
41
|
constructor(configuration, apifyClient) {
|
|
@@ -38,6 +47,7 @@ export class ChargingManager {
|
|
|
38
47
|
this.#useChargingLogDataset = configuration.useChargingLogDataset;
|
|
39
48
|
this.#apifyClient = apifyClient;
|
|
40
49
|
}
|
|
50
|
+
/** A shortcut - true if the Actor runs with the pay-per-event pricing model. */
|
|
41
51
|
get isPayPerEvent() {
|
|
42
52
|
return this.#pricingModel === 'PAY_PER_EVENT';
|
|
43
53
|
}
|
|
@@ -170,11 +180,7 @@ export class ChargingManager {
|
|
|
170
180
|
*
|
|
171
181
|
* @param options The name of the event to charge for and the number of events to be charged.
|
|
172
182
|
*/
|
|
173
|
-
async charge({ eventName, count = 1 }) {
|
|
174
|
-
const calculateChargeableWithinLimit = () => Object.fromEntries(Object.keys(this.#pricingInfo).map((name) => [
|
|
175
|
-
name,
|
|
176
|
-
this.calculateMaxEventChargeCountWithinLimit(name),
|
|
177
|
-
]));
|
|
183
|
+
async charge({ eventName, count = 1, idempotencyKey }) {
|
|
178
184
|
if (!this.isPayPerEvent) {
|
|
179
185
|
if (!this.#notPpeWarningPrinted) {
|
|
180
186
|
log.warning('Ignored attempt to charge for an event - the Actor does not use the pay-per-event pricing');
|
|
@@ -183,75 +189,79 @@ export class ChargingManager {
|
|
|
183
189
|
return {
|
|
184
190
|
eventChargeLimitReached: false,
|
|
185
191
|
chargedCount: 0,
|
|
186
|
-
chargeableWithinLimit: calculateChargeableWithinLimit(),
|
|
192
|
+
chargeableWithinLimit: this.calculateChargeableWithinLimit(),
|
|
187
193
|
};
|
|
188
194
|
}
|
|
189
|
-
|
|
195
|
+
const chargingState = this.#chargingState;
|
|
196
|
+
if (chargingState === undefined) {
|
|
190
197
|
throw new Error('ChargingManager is not initialized');
|
|
191
198
|
}
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
199
|
+
return await this.withChargeLock(async () => {
|
|
200
|
+
const maxEventChargeCount = this.calculateMaxEventChargeCountWithinLimit(eventName);
|
|
201
|
+
const chargedCount = (() => {
|
|
202
|
+
if (count <= maxEventChargeCount) {
|
|
203
|
+
return count;
|
|
204
|
+
}
|
|
205
|
+
// If the caller tries to charge more than the budget allows, overcharge by one event
|
|
206
|
+
// so that the Actor is detected by the platform and terminated.
|
|
207
|
+
// But don't do this if already strictly over the budget - no point piling on charges.
|
|
208
|
+
if (this.calculateTotalChargedAmount() <= this.#maxTotalChargeUsd) {
|
|
209
|
+
return maxEventChargeCount + 1;
|
|
210
|
+
}
|
|
211
|
+
return 0;
|
|
212
|
+
})();
|
|
213
|
+
if (chargedCount === 0) {
|
|
214
|
+
return {
|
|
215
|
+
eventChargeLimitReached: count > 0, // Only true if user wanted to charge but couldn't
|
|
216
|
+
chargedCount: 0,
|
|
217
|
+
chargeableWithinLimit: this.calculateChargeableWithinLimit(),
|
|
218
|
+
};
|
|
203
219
|
}
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
return {
|
|
208
|
-
eventChargeLimitReached: count > 0, // Only true if user wanted to charge but couldn't
|
|
209
|
-
chargedCount: 0,
|
|
210
|
-
chargeableWithinLimit: calculateChargeableWithinLimit(),
|
|
220
|
+
const pricingInfo = this.#pricingInfo[eventName] ?? {
|
|
221
|
+
price: this.#isAtHome ? 0 : 1, // Use a nonzero price for local development so that the maximum budget can be reached
|
|
222
|
+
title: `Unknown event '${eventName}'`,
|
|
211
223
|
};
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
224
|
+
chargingState[eventName] ??= {
|
|
225
|
+
chargeCount: 0,
|
|
226
|
+
totalChargedAmount: 0,
|
|
227
|
+
};
|
|
228
|
+
chargingState[eventName].chargeCount += chargedCount;
|
|
229
|
+
chargingState[eventName].totalChargedAmount += chargedCount * pricingInfo.price;
|
|
230
|
+
if (this.#isAtHome) {
|
|
231
|
+
if (eventName.startsWith('apify-')) {
|
|
232
|
+
// Synthetic events (e.g. apify-default-dataset-item) are tracked locally only,
|
|
233
|
+
// the platform handles them automatically based on dataset writes.
|
|
234
|
+
}
|
|
235
|
+
else if (this.#pricingInfo[eventName] !== undefined) {
|
|
236
|
+
await this.#apifyClient.run(this.#actorRunId).charge({
|
|
237
|
+
eventName,
|
|
238
|
+
count: chargedCount,
|
|
239
|
+
idempotencyKey,
|
|
240
|
+
});
|
|
241
|
+
}
|
|
242
|
+
else {
|
|
243
|
+
log.warning(`Attempting to charge for an unknown event '${eventName}'`);
|
|
244
|
+
}
|
|
228
245
|
}
|
|
229
|
-
|
|
230
|
-
await this.#
|
|
246
|
+
if (this.#chargingLogDataset !== undefined) {
|
|
247
|
+
await this.#chargingLogDataset.pushData({
|
|
248
|
+
eventName,
|
|
249
|
+
eventTitle: pricingInfo.title,
|
|
250
|
+
eventPriceUsd: pricingInfo.price,
|
|
251
|
+
chargedCount,
|
|
252
|
+
timestamp: new Date().toISOString(),
|
|
253
|
+
});
|
|
231
254
|
}
|
|
232
|
-
|
|
233
|
-
|
|
255
|
+
if (chargedCount < count) {
|
|
256
|
+
const subject = count === 1 ? 'instance' : 'instances';
|
|
257
|
+
log.info(`Charging ${count} ${subject} of '${eventName}' event would exceed maxTotalChargeUsd - only ${chargedCount} events were charged`);
|
|
234
258
|
}
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
if (this.#chargingLogDataset !== undefined) {
|
|
238
|
-
await this.#chargingLogDataset.pushData({
|
|
239
|
-
eventName,
|
|
240
|
-
eventTitle: pricingInfo.title,
|
|
241
|
-
eventPriceUsd: pricingInfo.price,
|
|
259
|
+
return {
|
|
260
|
+
eventChargeLimitReached: this.isEventChargeLimitReached(eventName),
|
|
242
261
|
chargedCount,
|
|
243
|
-
|
|
244
|
-
}
|
|
245
|
-
}
|
|
246
|
-
if (chargedCount < count) {
|
|
247
|
-
const subject = count === 1 ? 'instance' : 'instances';
|
|
248
|
-
log.info(`Charging ${count} ${subject} of '${eventName}' event would exceed maxTotalChargeUsd - only ${chargedCount} events were charged`);
|
|
249
|
-
}
|
|
250
|
-
return {
|
|
251
|
-
eventChargeLimitReached: this.calculateMaxEventChargeCountWithinLimit(eventName) <= 0,
|
|
252
|
-
chargedCount,
|
|
253
|
-
chargeableWithinLimit: calculateChargeableWithinLimit(),
|
|
254
|
-
};
|
|
262
|
+
chargeableWithinLimit: this.calculateChargeableWithinLimit(),
|
|
263
|
+
};
|
|
264
|
+
});
|
|
255
265
|
}
|
|
256
266
|
/**
|
|
257
267
|
* Get the number of events with given name that the Actor has charged for so far.
|
|
@@ -306,86 +316,58 @@ export class ChargingManager {
|
|
|
306
316
|
return Math.max(0, roundedResult);
|
|
307
317
|
}
|
|
308
318
|
/**
|
|
309
|
-
*
|
|
310
|
-
*
|
|
319
|
+
* Whether the remaining budget is insufficient to charge even a single event of the given type.
|
|
320
|
+
* Always false when nothing is charged in the first place.
|
|
311
321
|
*/
|
|
312
|
-
|
|
322
|
+
isEventChargeLimitReached(eventName) {
|
|
323
|
+
return this.isPayPerEvent && this.calculateMaxEventChargeCountWithinLimit(eventName) <= 0;
|
|
324
|
+
}
|
|
325
|
+
/**
|
|
326
|
+
* How many events of each known type can still be charged within the limit.
|
|
327
|
+
*/
|
|
328
|
+
calculateChargeableWithinLimit() {
|
|
329
|
+
return Object.fromEntries(Object.keys(this.#pricingInfo).map((name) => [name, this.calculateMaxEventChargeCountWithinLimit(name)]));
|
|
330
|
+
}
|
|
331
|
+
/**
|
|
332
|
+
* How many of `itemsCount` items may be pushed to a dataset within the remaining budget.
|
|
333
|
+
*
|
|
334
|
+
* The budget is measured against the combined per-item price of the explicit `eventName` and - on
|
|
335
|
+
* the default dataset - the synthetic {@link DEFAULT_DATASET_ITEM_EVENT}, since pushing one item
|
|
336
|
+
* charges both.
|
|
337
|
+
*
|
|
338
|
+
* When the budget cannot cover a single item, one item is still allowed through so that the
|
|
339
|
+
* resulting overcharge makes the platform terminate the run - unless the run is already strictly
|
|
340
|
+
* over budget, in which case nothing is allowed through.
|
|
341
|
+
*/
|
|
342
|
+
calculatePushDataLimit(itemsCount, { eventName, isDefaultDataset = false } = {}) {
|
|
313
343
|
if (this.#chargingState === undefined) {
|
|
314
344
|
throw new Error('ChargingManager is not initialized');
|
|
315
345
|
}
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
return {
|
|
319
|
-
limitedItems: itemsArray,
|
|
320
|
-
eventsToCharge: {},
|
|
321
|
-
};
|
|
346
|
+
if (!this.isPayPerEvent || itemsCount === 0) {
|
|
347
|
+
return itemsCount;
|
|
322
348
|
}
|
|
323
|
-
const itemPrice = (
|
|
324
|
-
(
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
if (maxChargedCount >= itemsArray.length) {
|
|
328
|
-
return itemsArray.length;
|
|
329
|
-
}
|
|
330
|
-
// If the caller tries to push items even though the limit is depleted, overcharge by one
|
|
331
|
-
// so that the Platform terminates the run.
|
|
332
|
-
// But don't do this if already strictly over the budget - no point piling on charges.
|
|
333
|
-
if (itemsArray.length > 0 &&
|
|
334
|
-
maxChargedCount === 0 &&
|
|
335
|
-
this.calculateTotalChargedAmount() <= this.#maxTotalChargeUsd) {
|
|
336
|
-
return 1;
|
|
337
|
-
}
|
|
338
|
-
return maxChargedCount;
|
|
339
|
-
})();
|
|
340
|
-
const eventsToCharge = {};
|
|
341
|
-
if (eventName !== undefined && itemsToKeep > 0) {
|
|
342
|
-
eventsToCharge[eventName] = itemsToKeep;
|
|
349
|
+
const itemPrice = (eventName === undefined ? 0 : (this.calculateEventPrice(eventName) ?? 0)) +
|
|
350
|
+
(isDefaultDataset ? (this.calculateEventPrice(DEFAULT_DATASET_ITEM_EVENT) ?? 0) : 0);
|
|
351
|
+
if (itemPrice === 0) {
|
|
352
|
+
return itemsCount;
|
|
343
353
|
}
|
|
344
|
-
|
|
345
|
-
|
|
354
|
+
const maxChargedCount = this.calculateMaxChargesByPrice(itemPrice);
|
|
355
|
+
if (maxChargedCount >= itemsCount) {
|
|
356
|
+
return itemsCount;
|
|
346
357
|
}
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
}
|
|
352
|
-
}
|
|
353
|
-
/**
|
|
354
|
-
* Helper for PPE-aware pushing of data to the dataset.
|
|
355
|
-
*
|
|
356
|
-
* 1. Calculate limits based on budget
|
|
357
|
-
* 2. Push limited items via the provided callback
|
|
358
|
-
* 3. Charge for the events
|
|
359
|
-
*
|
|
360
|
-
* @internal
|
|
361
|
-
*/
|
|
362
|
-
export async function pushDataAndCharge({ chargingManager, items, eventName, isDefaultDataset, pushFn, }) {
|
|
363
|
-
const { limitedItems, eventsToCharge } = chargingManager.calculatePushDataLimits({
|
|
364
|
-
items,
|
|
365
|
-
eventName,
|
|
366
|
-
isDefaultDataset,
|
|
367
|
-
});
|
|
368
|
-
if (limitedItems.length > 0) {
|
|
369
|
-
// Preserve original call shape for single items
|
|
370
|
-
await pushFn(Array.isArray(items) ? limitedItems : limitedItems[0]);
|
|
358
|
+
if (maxChargedCount > 0) {
|
|
359
|
+
return maxChargedCount;
|
|
360
|
+
}
|
|
361
|
+
return this.calculateTotalChargedAmount() <= this.#maxTotalChargeUsd ? 1 : 0;
|
|
371
362
|
}
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
// whether ANY of the charged events hit their limit.
|
|
382
|
-
return Object.values(results).reduce(mergeChargeResults);
|
|
363
|
+
/**
|
|
364
|
+
* Runs `fn` under the charge lock, which keeps a limit reservation and the charge that acts on it
|
|
365
|
+
* atomic against concurrent pushes. Only pay-per-event runs charge anything, so for every other
|
|
366
|
+
* run there is nothing to serialize and the lock is skipped.
|
|
367
|
+
*
|
|
368
|
+
* @internal
|
|
369
|
+
*/
|
|
370
|
+
async withChargeLock(fn) {
|
|
371
|
+
return this.isPayPerEvent ? await this.#chargeLock.runExclusive(fn) : await fn();
|
|
383
372
|
}
|
|
384
|
-
const itemsArray = Array.isArray(items) ? items : [items];
|
|
385
|
-
const allItemsTrimmed = itemsArray.length > 0 && limitedItems.length === 0;
|
|
386
|
-
return {
|
|
387
|
-
eventChargeLimitReached: allItemsTrimmed,
|
|
388
|
-
chargedCount: 0,
|
|
389
|
-
chargeableWithinLimit: {},
|
|
390
|
-
};
|
|
391
373
|
}
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import type { DatasetBackend, DatasetBackendListOptions, DatasetInfo, Dictionary, KeyValueStoreBackend, PaginatedList, RequestQueueBackend, StorageBackend, StorageIdentifier } from '@crawlee/types';
|
|
2
|
+
import type { ChargingManager } from './charging.js';
|
|
3
|
+
import type { Configuration } from './configuration.js';
|
|
4
|
+
export interface ChargingStorageBackendOptions {
|
|
5
|
+
/** Supplies the run's default dataset id, which is the only dataset charged for. */
|
|
6
|
+
configuration: Configuration;
|
|
7
|
+
/** Resolved per push, as the charging manager only knows the pricing once `Actor.init()` has run. */
|
|
8
|
+
getChargingManager: () => ChargingManager;
|
|
9
|
+
}
|
|
10
|
+
/**
|
|
11
|
+
* Charges the synthetic `apify-default-dataset-item` event for the items it stores, trimming a push
|
|
12
|
+
* that the remaining budget cannot cover.
|
|
13
|
+
*
|
|
14
|
+
* It wraps the whole dataset backend rather than its API calls, so that a push split into several
|
|
15
|
+
* payload-sized requests is still counted and charged exactly once.
|
|
16
|
+
*
|
|
17
|
+
* @internal
|
|
18
|
+
*/
|
|
19
|
+
export declare class ChargingDatasetBackend implements DatasetBackend {
|
|
20
|
+
#private;
|
|
21
|
+
constructor(inner: DatasetBackend, getChargingManager: () => ChargingManager);
|
|
22
|
+
getMetadata(): Promise<DatasetInfo>;
|
|
23
|
+
drop(): Promise<void>;
|
|
24
|
+
purge(): Promise<void>;
|
|
25
|
+
getData(options?: DatasetBackendListOptions): Promise<PaginatedList<Dictionary>>;
|
|
26
|
+
pushData(items: Dictionary[]): Promise<void>;
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* Wraps another storage backend so that items pushed to the run's default dataset are charged for
|
|
30
|
+
* under the pay-per-event pricing model.
|
|
31
|
+
*
|
|
32
|
+
* Charging belongs here rather than in `Actor.pushData()` because `Dataset.pushData()` is also
|
|
33
|
+
* called directly - by user code and by crawlee itself (`context.pushData()`) - and every item that
|
|
34
|
+
* reaches the default dataset is billed by the platform regardless of who pushed it.
|
|
35
|
+
*
|
|
36
|
+
* @internal
|
|
37
|
+
*/
|
|
38
|
+
export declare class ChargingStorageBackend implements StorageBackend {
|
|
39
|
+
#private;
|
|
40
|
+
storageExists?: StorageBackend['storageExists'];
|
|
41
|
+
purge?: StorageBackend['purge'];
|
|
42
|
+
teardown?: StorageBackend['teardown'];
|
|
43
|
+
constructor(inner: StorageBackend, options: ChargingStorageBackendOptions);
|
|
44
|
+
get stats(): {
|
|
45
|
+
rateLimitErrors: number[];
|
|
46
|
+
} | undefined;
|
|
47
|
+
/** Repeats crawlee's own fallback, so that wrapping a backend does not change how storages are cached. */
|
|
48
|
+
getStorageBackendCacheKey(): string;
|
|
49
|
+
createDatasetBackend(options?: StorageIdentifier): Promise<DatasetBackend>;
|
|
50
|
+
createKeyValueStoreBackend(options?: StorageIdentifier): Promise<KeyValueStoreBackend>;
|
|
51
|
+
createRequestQueueBackend(options?: StorageIdentifier): Promise<RequestQueueBackend>;
|
|
52
|
+
private isDefaultDataset;
|
|
53
|
+
}
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
import { DEFAULT_DATASET_ITEM_EVENT } from './charging.js';
|
|
2
|
+
import { DEFAULT_STORAGE_ALIAS } from './apify_storage_backend.js';
|
|
3
|
+
/**
|
|
4
|
+
* Charges the synthetic `apify-default-dataset-item` event for the items it stores, trimming a push
|
|
5
|
+
* that the remaining budget cannot cover.
|
|
6
|
+
*
|
|
7
|
+
* It wraps the whole dataset backend rather than its API calls, so that a push split into several
|
|
8
|
+
* payload-sized requests is still counted and charged exactly once.
|
|
9
|
+
*
|
|
10
|
+
* @internal
|
|
11
|
+
*/
|
|
12
|
+
export class ChargingDatasetBackend {
|
|
13
|
+
#inner;
|
|
14
|
+
#getChargingManager;
|
|
15
|
+
constructor(inner, getChargingManager) {
|
|
16
|
+
this.#inner = inner;
|
|
17
|
+
this.#getChargingManager = getChargingManager;
|
|
18
|
+
}
|
|
19
|
+
async getMetadata() {
|
|
20
|
+
return await this.#inner.getMetadata();
|
|
21
|
+
}
|
|
22
|
+
async drop() {
|
|
23
|
+
await this.#inner.drop();
|
|
24
|
+
}
|
|
25
|
+
async purge() {
|
|
26
|
+
await this.#inner.purge();
|
|
27
|
+
}
|
|
28
|
+
async getData(options) {
|
|
29
|
+
return await this.#inner.getData(options);
|
|
30
|
+
}
|
|
31
|
+
async pushData(items) {
|
|
32
|
+
const chargingManager = this.#getChargingManager();
|
|
33
|
+
// `Dataset.pushData()` also works without `Actor.init()`, where there is no charging state yet.
|
|
34
|
+
if (items.length === 0 || !chargingManager.isInitialized || !chargingManager.isPayPerEvent) {
|
|
35
|
+
await this.#inner.pushData(items);
|
|
36
|
+
return;
|
|
37
|
+
}
|
|
38
|
+
await chargingManager.withChargeLock(async () => {
|
|
39
|
+
const limit = chargingManager.calculatePushDataLimit(items.length, { isDefaultDataset: true });
|
|
40
|
+
if (limit === 0) {
|
|
41
|
+
return;
|
|
42
|
+
}
|
|
43
|
+
await this.#inner.pushData(limit < items.length ? items.slice(0, limit) : items);
|
|
44
|
+
await chargingManager.charge({ eventName: DEFAULT_DATASET_ITEM_EVENT, count: limit });
|
|
45
|
+
});
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
/**
|
|
49
|
+
* Wraps another storage backend so that items pushed to the run's default dataset are charged for
|
|
50
|
+
* under the pay-per-event pricing model.
|
|
51
|
+
*
|
|
52
|
+
* Charging belongs here rather than in `Actor.pushData()` because `Dataset.pushData()` is also
|
|
53
|
+
* called directly - by user code and by crawlee itself (`context.pushData()`) - and every item that
|
|
54
|
+
* reaches the default dataset is billed by the platform regardless of who pushed it.
|
|
55
|
+
*
|
|
56
|
+
* @internal
|
|
57
|
+
*/
|
|
58
|
+
export class ChargingStorageBackend {
|
|
59
|
+
storageExists;
|
|
60
|
+
purge;
|
|
61
|
+
teardown;
|
|
62
|
+
#inner;
|
|
63
|
+
#options;
|
|
64
|
+
constructor(inner, options) {
|
|
65
|
+
this.#inner = inner;
|
|
66
|
+
this.#options = options;
|
|
67
|
+
// Crawlee branches on the presence of these, so they have to stay absent when `inner` lacks them.
|
|
68
|
+
const { storageExists, purge, teardown } = inner;
|
|
69
|
+
if (storageExists)
|
|
70
|
+
this.storageExists = async (id, type) => storageExists.call(inner, id, type);
|
|
71
|
+
if (purge)
|
|
72
|
+
this.purge = async () => purge.call(inner);
|
|
73
|
+
if (teardown)
|
|
74
|
+
this.teardown = async () => teardown.call(inner);
|
|
75
|
+
}
|
|
76
|
+
get stats() {
|
|
77
|
+
return this.#inner.stats;
|
|
78
|
+
}
|
|
79
|
+
/** Repeats crawlee's own fallback, so that wrapping a backend does not change how storages are cached. */
|
|
80
|
+
getStorageBackendCacheKey() {
|
|
81
|
+
return this.#inner.getStorageBackendCacheKey?.() ?? this.#inner.constructor.name;
|
|
82
|
+
}
|
|
83
|
+
async createDatasetBackend(options) {
|
|
84
|
+
const backend = await this.#inner.createDatasetBackend(options);
|
|
85
|
+
if (!this.isDefaultDataset(options)) {
|
|
86
|
+
return backend;
|
|
87
|
+
}
|
|
88
|
+
return new ChargingDatasetBackend(backend, this.#options.getChargingManager);
|
|
89
|
+
}
|
|
90
|
+
async createKeyValueStoreBackend(options) {
|
|
91
|
+
return await this.#inner.createKeyValueStoreBackend(options);
|
|
92
|
+
}
|
|
93
|
+
async createRequestQueueBackend(options) {
|
|
94
|
+
return await this.#inner.createRequestQueueBackend(options);
|
|
95
|
+
}
|
|
96
|
+
isDefaultDataset(options) {
|
|
97
|
+
if (!options)
|
|
98
|
+
return true;
|
|
99
|
+
if (options.alias !== undefined)
|
|
100
|
+
return options.alias === DEFAULT_STORAGE_ALIAS;
|
|
101
|
+
if (options.id !== undefined)
|
|
102
|
+
return options.id === this.#options.configuration.defaultDatasetId;
|
|
103
|
+
return false;
|
|
104
|
+
}
|
|
105
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "apify",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.34",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.0.0"
|