apify 4.0.0-beta.31 → 4.0.0-beta.32

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -21,23 +21,23 @@ const HEAD_LOCK_LIMIT = 25;
21
21
  */
22
22
  export class ApifyRequestQueueSharedBackend extends ApifyRequestQueueBackend {
23
23
  /** Ids of requests locked by this client and waiting to be handed out by `fetchNextRequest`. */
24
- headIds = [];
24
+ #headIds = [];
25
25
  /** Dedup records for requests known to exist on the platform, keyed by id. */
26
- cachedRequestInfo = new LruCache({ maxLength: MAX_CACHED_REQUESTS });
26
+ #cachedRequestInfo = new LruCache({ maxLength: MAX_CACHED_REQUESTS });
27
27
  /** Ids of requests currently being processed by this client. */
28
- inProgressIds = new Set();
28
+ #inProgressIds = new Set();
29
29
  /** Whether the last head read reported any locked requests left in the queue (any client's). */
30
- queueHasLockedRequests;
30
+ #queueHasLockedRequests;
31
31
  /** Set after a forefront insert — the next head read starts fresh so the insert is honored. */
32
- shouldCheckForefrontRequests = false;
32
+ #shouldCheckForefrontRequests = false;
33
33
  /** Lock duration applied to fetched requests; raised via `setExpectedRequestProcessingTimeSecs`. */
34
- lockSecs = DEFAULT_REQUEST_LOCK_SECS;
34
+ #lockSecs = DEFAULT_REQUEST_LOCK_SECS;
35
35
  /** Serializes head reads and reclaims — both reorder the shared head state. */
36
- headLock = new AsyncLock();
36
+ #headLock = new AsyncLock();
37
37
  async setExpectedRequestProcessingTimeSecs(secs) {
38
38
  // Only ever raise the lock duration — several consumers may share this client, and a
39
39
  // short-lived one must not cut the reservation of a long-running one short.
40
- this.lockSecs = Math.max(this.lockSecs, secs);
40
+ this.#lockSecs = Math.max(this.#lockSecs, secs);
41
41
  }
42
42
  async addBatchOfRequests(requests, options = {}) {
43
43
  const { forefront = false } = options;
@@ -48,7 +48,7 @@ export class ApifyRequestQueueSharedBackend extends ApifyRequestQueueBackend {
48
48
  const newRequests = [];
49
49
  for (const request of requests) {
50
50
  const id = this.requestIdFromUniqueKey(request.uniqueKey);
51
- const cached = this.cachedRequestInfo.get(id);
51
+ const cached = this.#cachedRequestInfo.get(id);
52
52
  if (cached) {
53
53
  alreadyPresent.push({
54
54
  requestId: id,
@@ -70,7 +70,7 @@ export class ApifyRequestQueueSharedBackend extends ApifyRequestQueueBackend {
70
70
  // A forefront insert changes the head order — have the next head read re-fetch the
71
71
  // front of the queue instead of draining the local buffer first.
72
72
  if (forefront) {
73
- this.shouldCheckForefrontRequests = true;
73
+ this.#shouldCheckForefrontRequests = true;
74
74
  }
75
75
  }
76
76
  result.processedRequests.push(...alreadyPresent);
@@ -83,9 +83,9 @@ export class ApifyRequestQueueSharedBackend extends ApifyRequestQueueBackend {
83
83
  return this.getRequestById(this.requestIdFromUniqueKey(uniqueKey));
84
84
  }
85
85
  async fetchNextRequest() {
86
- const id = await this.headLock.runExclusive(async () => {
86
+ const id = await this.#headLock.runExclusive(async () => {
87
87
  await this.ensureHeadIsNonEmpty();
88
- return this.headIds.shift();
88
+ return this.#headIds.shift();
89
89
  });
90
90
  if (!id)
91
91
  return undefined;
@@ -103,7 +103,7 @@ export class ApifyRequestQueueSharedBackend extends ApifyRequestQueueBackend {
103
103
  this.cacheRequestInfo(id, { wasAlreadyHandled: true });
104
104
  return undefined;
105
105
  }
106
- this.inProgressIds.add(id);
106
+ this.#inProgressIds.add(id);
107
107
  return request;
108
108
  }
109
109
  async markRequestAsHandled(request) {
@@ -111,12 +111,12 @@ export class ApifyRequestQueueSharedBackend extends ApifyRequestQueueBackend {
111
111
  // Contract: marking a request that does not exist in the queue is a no-op — it must not be
112
112
  // added as a side effect (the platform update endpoint would upsert it).
113
113
  if (!(await this.isKnownOrExists(id))) {
114
- this.inProgressIds.delete(id);
114
+ this.#inProgressIds.delete(id);
115
115
  return undefined;
116
116
  }
117
117
  const handledAt = request.handledAt ?? new Date().toISOString();
118
118
  const info = await this.updateRequestOnPlatform({ ...request, id, handledAt });
119
- this.inProgressIds.delete(id);
119
+ this.#inProgressIds.delete(id);
120
120
  this.cacheRequestInfo(id, { wasAlreadyHandled: true });
121
121
  if (!info.wasAlreadyHandled) {
122
122
  this.estimatedHandledRequestCount += 1;
@@ -128,10 +128,10 @@ export class ApifyRequestQueueSharedBackend extends ApifyRequestQueueBackend {
128
128
  const id = this.requestIdFromUniqueKey(request.uniqueKey);
129
129
  // Same contract as `markRequestAsHandled` — never insert as a side effect.
130
130
  if (!(await this.isKnownOrExists(id))) {
131
- this.inProgressIds.delete(id);
131
+ this.#inProgressIds.delete(id);
132
132
  return undefined;
133
133
  }
134
- return this.headLock.runExclusive(async () => {
134
+ return this.#headLock.runExclusive(async () => {
135
135
  const info = await this.updateRequestOnPlatform({ ...request, id, handledAt: undefined }, forefront);
136
136
  // Release the server-side lock so the request becomes fetchable again immediately —
137
137
  // by any consumer — rather than only after the lock expires.
@@ -141,10 +141,10 @@ export class ApifyRequestQueueSharedBackend extends ApifyRequestQueueBackend {
141
141
  catch (err) {
142
142
  log.debug(`Failed to delete the lock of a reclaimed request (id: ${id}): ${err.message}`);
143
143
  }
144
- this.inProgressIds.delete(id);
144
+ this.#inProgressIds.delete(id);
145
145
  this.cacheRequestInfo(id, { wasAlreadyHandled: false });
146
146
  if (forefront) {
147
- this.shouldCheckForefrontRequests = true;
147
+ this.#shouldCheckForefrontRequests = true;
148
148
  }
149
149
  if (info.wasAlreadyHandled) {
150
150
  this.estimatedHandledRequestCount -= 1;
@@ -153,25 +153,25 @@ export class ApifyRequestQueueSharedBackend extends ApifyRequestQueueBackend {
153
153
  });
154
154
  }
155
155
  async isEmpty() {
156
- return this.headLock.runExclusive(async () => {
157
- if (this.headIds.length > 0)
156
+ return this.#headLock.runExclusive(async () => {
157
+ if (this.#headIds.length > 0)
158
158
  return false;
159
159
  await this.listAndLockHead(1);
160
- return this.headIds.length === 0;
160
+ return this.#headIds.length === 0;
161
161
  });
162
162
  }
163
163
  async isFinished() {
164
- return this.headLock.runExclusive(async () => {
165
- if (this.headIds.length > 0)
164
+ return this.#headLock.runExclusive(async () => {
165
+ if (this.#headIds.length > 0)
166
166
  return false;
167
167
  // The head read also refreshes `queueHasLockedRequests`, so the order matters here.
168
168
  await this.listAndLockHead(1);
169
- return this.headIds.length === 0 && !this.queueHasLockedRequests;
169
+ return this.#headIds.length === 0 && !this.#queueHasLockedRequests;
170
170
  });
171
171
  }
172
172
  /** Must be called with the head lock held. */
173
173
  async ensureHeadIsNonEmpty() {
174
- if (this.headIds.length > 1 && !this.shouldCheckForefrontRequests) {
174
+ if (this.#headIds.length > 1 && !this.#shouldCheckForefrontRequests) {
175
175
  return;
176
176
  }
177
177
  await this.listAndLockHead(HEAD_LOCK_LIMIT);
@@ -181,31 +181,31 @@ export class ApifyRequestQueueSharedBackend extends ApifyRequestQueueBackend {
181
181
  // After a forefront insert the local buffer no longer starts at the true front of the
182
182
  // queue — re-fetch the front and keep the already-locked leftovers for afterwards.
183
183
  let leftoverIds = [];
184
- if (this.shouldCheckForefrontRequests) {
185
- leftoverIds = this.headIds.splice(0);
186
- this.shouldCheckForefrontRequests = false;
184
+ if (this.#shouldCheckForefrontRequests) {
185
+ leftoverIds = this.#headIds.splice(0);
186
+ this.#shouldCheckForefrontRequests = false;
187
187
  }
188
- const head = await this.client.listAndLockHead({ limit, lockSecs: this.lockSecs });
189
- this.queueHasLockedRequests = head.queueHasLockedRequests;
188
+ const head = await this.client.listAndLockHead({ limit, lockSecs: this.#lockSecs });
189
+ this.#queueHasLockedRequests = head.queueHasLockedRequests;
190
190
  for (const item of head.items) {
191
- if (this.inProgressIds.has(item.id))
191
+ if (this.#inProgressIds.has(item.id))
192
192
  continue;
193
- if (this.headIds.includes(item.id) || leftoverIds.includes(item.id))
193
+ if (this.#headIds.includes(item.id) || leftoverIds.includes(item.id))
194
194
  continue;
195
195
  this.cacheRequestInfo(item.id, { wasAlreadyHandled: false });
196
- this.headIds.push(item.id);
196
+ this.#headIds.push(item.id);
197
197
  }
198
- this.headIds.push(...leftoverIds);
198
+ this.#headIds.push(...leftoverIds);
199
199
  }
200
200
  async isKnownOrExists(id) {
201
- if (this.inProgressIds.has(id) || this.cachedRequestInfo.get(id)) {
201
+ if (this.#inProgressIds.has(id) || this.#cachedRequestInfo.get(id)) {
202
202
  return true;
203
203
  }
204
204
  return (await this.getRequestById(id)) !== undefined;
205
205
  }
206
206
  cacheRequestInfo(id, info) {
207
207
  // `LruCache.add` does not overwrite existing entries, so remove first.
208
- this.cachedRequestInfo.remove(id);
209
- this.cachedRequestInfo.add(id, info);
208
+ this.#cachedRequestInfo.remove(id);
209
+ this.#cachedRequestInfo.add(id, info);
210
210
  }
211
211
  }
@@ -22,16 +22,7 @@ import { ApifyRequestQueueBackend } from './apify_request_queue_backend.js';
22
22
  * @internal
23
23
  */
24
24
  export declare class ApifyRequestQueueSingleBackend extends ApifyRequestQueueBackend {
25
- /** Local estimate of the queue head — request ids in the order they should be fetched. */
26
- private readonly headIds;
27
- /** Unhandled full request objects added by (or fetched through) this client, keyed by id. */
28
- private readonly cachedRequests;
29
- /** Ids of requests known to be already handled — cheap dedup without caching full objects. */
30
- private readonly handledIds;
31
- /** Ids of requests currently being processed by this client. */
32
- private readonly inProgressIds;
33
- /** Memoized one-time prefetch of existing queue contents into the local caches. */
34
- private initCachesPromise?;
25
+ #private;
35
26
  addBatchOfRequests(requests: RequestSchema[], options?: RequestQueueOperationOptions): Promise<BatchAddRequestsResult>;
36
27
  getRequest(uniqueKey: string): Promise<UpdateRequestSchema | undefined>;
37
28
  fetchNextRequest(): Promise<UpdateRequestSchema | undefined>;
@@ -32,25 +32,25 @@ const INIT_CACHES_REQUEST_LIMIT = 10_000;
32
32
  */
33
33
  export class ApifyRequestQueueSingleBackend extends ApifyRequestQueueBackend {
34
34
  /** Local estimate of the queue head — request ids in the order they should be fetched. */
35
- headIds = [];
35
+ #headIds = [];
36
36
  /** Unhandled full request objects added by (or fetched through) this client, keyed by id. */
37
- cachedRequests = new LruCache({ maxLength: MAX_CACHED_REQUESTS });
37
+ #cachedRequests = new LruCache({ maxLength: MAX_CACHED_REQUESTS });
38
38
  /** Ids of requests known to be already handled — cheap dedup without caching full objects. */
39
- handledIds = new Set();
39
+ #handledIds = new Set();
40
40
  /** Ids of requests currently being processed by this client. */
41
- inProgressIds = new Set();
41
+ #inProgressIds = new Set();
42
42
  /** Memoized one-time prefetch of existing queue contents into the local caches. */
43
- initCachesPromise;
43
+ #initCachesPromise;
44
44
  async addBatchOfRequests(requests, options = {}) {
45
45
  const { forefront = false } = options;
46
- await (this.initCachesPromise ??= this.initCaches());
46
+ await (this.#initCachesPromise ??= this.initCaches());
47
47
  // Split the batch into requests we already know about (dedup them locally — a platform
48
48
  // write costs an API call and a paid write operation) and genuinely new ones.
49
49
  const alreadyPresent = [];
50
50
  const newRequests = [];
51
51
  for (const request of requests) {
52
52
  const id = this.requestIdFromUniqueKey(request.uniqueKey);
53
- if (this.handledIds.has(id)) {
53
+ if (this.#handledIds.has(id)) {
54
54
  alreadyPresent.push({
55
55
  requestId: id,
56
56
  uniqueKey: request.uniqueKey,
@@ -58,7 +58,7 @@ export class ApifyRequestQueueSingleBackend extends ApifyRequestQueueBackend {
58
58
  wasAlreadyHandled: true,
59
59
  });
60
60
  }
61
- else if (this.cachedRequests.get(id)) {
61
+ else if (this.#cachedRequests.get(id)) {
62
62
  alreadyPresent.push({
63
63
  requestId: id,
64
64
  uniqueKey: request.uniqueKey,
@@ -83,15 +83,15 @@ export class ApifyRequestQueueSingleBackend extends ApifyRequestQueueBackend {
83
83
  if (!processed)
84
84
  continue; // rejected by the platform, reported in `unprocessedRequests`
85
85
  if (processed.wasAlreadyHandled) {
86
- this.handledIds.add(processed.requestId);
86
+ this.#handledIds.add(processed.requestId);
87
87
  continue;
88
88
  }
89
89
  this.cacheRequest({ ...request, id: processed.requestId });
90
90
  if (forefront) {
91
- this.headIds.unshift(processed.requestId);
91
+ this.#headIds.unshift(processed.requestId);
92
92
  }
93
93
  else {
94
- this.headIds.push(processed.requestId);
94
+ this.#headIds.push(processed.requestId);
95
95
  }
96
96
  }
97
97
  }
@@ -101,16 +101,16 @@ export class ApifyRequestQueueSingleBackend extends ApifyRequestQueueBackend {
101
101
  }
102
102
  async getRequest(uniqueKey) {
103
103
  const id = this.requestIdFromUniqueKey(uniqueKey);
104
- const cached = this.cachedRequests.get(id);
104
+ const cached = this.#cachedRequests.get(id);
105
105
  if (cached)
106
106
  return cached;
107
107
  const request = await this.getRequestById(id);
108
108
  if (!request)
109
109
  return undefined;
110
110
  // Requests already in progress are ones the client knows about — no caching needed.
111
- if (!this.inProgressIds.has(id)) {
111
+ if (!this.#inProgressIds.has(id)) {
112
112
  if (request.handledAt) {
113
- this.handledIds.add(id);
113
+ this.#handledIds.add(id);
114
114
  }
115
115
  else {
116
116
  this.cacheRequest(request);
@@ -120,24 +120,24 @@ export class ApifyRequestQueueSingleBackend extends ApifyRequestQueueBackend {
120
120
  }
121
121
  async fetchNextRequest() {
122
122
  await this.ensureHeadIsNonEmpty();
123
- while (this.headIds.length > 0) {
124
- const id = this.headIds.shift();
125
- if (this.inProgressIds.has(id) || this.handledIds.has(id)) {
123
+ while (this.#headIds.length > 0) {
124
+ const id = this.#headIds.shift();
125
+ if (this.#inProgressIds.has(id) || this.#handledIds.has(id)) {
126
126
  continue;
127
127
  }
128
- this.inProgressIds.add(id);
128
+ this.#inProgressIds.add(id);
129
129
  // Requests added by this client are served straight from the cache; only requests
130
130
  // discovered via `listHead` (added by another producer) need a round-trip.
131
- const request = this.cachedRequests.get(id) ?? (await this.getRequestById(id));
131
+ const request = this.#cachedRequests.get(id) ?? (await this.getRequestById(id));
132
132
  if (!request) {
133
- this.inProgressIds.delete(id);
133
+ this.#inProgressIds.delete(id);
134
134
  continue;
135
135
  }
136
136
  if (request.handledAt) {
137
137
  // Handled elsewhere in the meantime — skip it and remember the outcome.
138
- this.inProgressIds.delete(id);
139
- this.handledIds.add(id);
140
- this.cachedRequests.remove(id);
138
+ this.#inProgressIds.delete(id);
139
+ this.#handledIds.add(id);
140
+ this.#cachedRequests.remove(id);
141
141
  continue;
142
142
  }
143
143
  return request;
@@ -149,14 +149,14 @@ export class ApifyRequestQueueSingleBackend extends ApifyRequestQueueBackend {
149
149
  // Contract: marking a request that does not exist in the queue is a no-op — it must not be
150
150
  // added as a side effect (the platform update endpoint would upsert it).
151
151
  if (!(await this.isKnownOrExists(id))) {
152
- this.inProgressIds.delete(id);
152
+ this.#inProgressIds.delete(id);
153
153
  return undefined;
154
154
  }
155
155
  const handledAt = request.handledAt ?? new Date().toISOString();
156
156
  const info = await this.updateRequestOnPlatform({ ...request, id, handledAt });
157
- this.inProgressIds.delete(id);
158
- this.handledIds.add(id);
159
- this.cachedRequests.remove(id);
157
+ this.#inProgressIds.delete(id);
158
+ this.#handledIds.add(id);
159
+ this.#cachedRequests.remove(id);
160
160
  if (!info.wasAlreadyHandled) {
161
161
  this.estimatedHandledRequestCount += 1;
162
162
  }
@@ -167,24 +167,24 @@ export class ApifyRequestQueueSingleBackend extends ApifyRequestQueueBackend {
167
167
  const id = this.requestIdFromUniqueKey(request.uniqueKey);
168
168
  // Same contract as `markRequestAsHandled` — never insert as a side effect.
169
169
  if (!(await this.isKnownOrExists(id))) {
170
- this.inProgressIds.delete(id);
170
+ this.#inProgressIds.delete(id);
171
171
  return undefined;
172
172
  }
173
173
  // Reclaiming returns the request to the queue for reprocessing.
174
174
  const reclaimed = { ...request, id, handledAt: undefined };
175
175
  const info = await this.updateRequestOnPlatform(reclaimed, forefront);
176
- this.inProgressIds.delete(id);
177
- this.handledIds.delete(id);
176
+ this.#inProgressIds.delete(id);
177
+ this.#handledIds.delete(id);
178
178
  this.cacheRequest(reclaimed);
179
179
  // Return the id to the local head estimate right away — the platform head read can lag a
180
180
  // few seconds behind the update, and `isFinished` must never report `true` while a
181
181
  // reclaimed request is still waiting to be reprocessed.
182
- if (!this.headIds.includes(id)) {
182
+ if (!this.#headIds.includes(id)) {
183
183
  if (forefront) {
184
- this.headIds.unshift(id);
184
+ this.#headIds.unshift(id);
185
185
  }
186
186
  else {
187
- this.headIds.push(id);
187
+ this.#headIds.push(id);
188
188
  }
189
189
  }
190
190
  if (info.wasAlreadyHandled) {
@@ -194,28 +194,28 @@ export class ApifyRequestQueueSingleBackend extends ApifyRequestQueueBackend {
194
194
  }
195
195
  async isEmpty() {
196
196
  await this.ensureHeadIsNonEmpty();
197
- return this.headIds.length === 0;
197
+ return this.#headIds.length === 0;
198
198
  }
199
199
  async isFinished() {
200
- return (await this.isEmpty()) && this.inProgressIds.size === 0;
200
+ return (await this.isEmpty()) && this.#inProgressIds.size === 0;
201
201
  }
202
202
  async ensureHeadIsNonEmpty() {
203
- if (this.headIds.length <= 1) {
203
+ if (this.#headIds.length <= 1) {
204
204
  await this.listHead();
205
205
  }
206
206
  }
207
207
  async listHead() {
208
208
  // The head read returns in-progress requests too, so fetch enough to find new ones.
209
- const limit = Math.min(MAX_HEAD_ITEMS, DESIRED_NEW_HEAD_ITEMS + this.inProgressIds.size);
209
+ const limit = Math.min(MAX_HEAD_ITEMS, DESIRED_NEW_HEAD_ITEMS + this.#inProgressIds.size);
210
210
  const head = await this.client.listHead({ limit });
211
211
  for (const item of head.items) {
212
- if (this.inProgressIds.has(item.id) || this.handledIds.has(item.id)) {
212
+ if (this.#inProgressIds.has(item.id) || this.#handledIds.has(item.id)) {
213
213
  continue;
214
214
  }
215
215
  // `headIds` is nearly drained whenever this runs (see `ensureHeadIsNonEmpty`), so the
216
216
  // linear dedup scan stays cheap.
217
- if (!this.headIds.includes(item.id)) {
218
- this.headIds.push(item.id);
217
+ if (!this.#headIds.includes(item.id)) {
218
+ this.#headIds.push(item.id);
219
219
  }
220
220
  }
221
221
  }
@@ -229,7 +229,7 @@ export class ApifyRequestQueueSingleBackend extends ApifyRequestQueueBackend {
229
229
  const response = await this.client.listRequests({ limit: INIT_CACHES_REQUEST_LIMIT });
230
230
  for (const request of response.items) {
231
231
  if (request.handledAt) {
232
- this.handledIds.add(request.id);
232
+ this.#handledIds.add(request.id);
233
233
  }
234
234
  else {
235
235
  this.cacheRequest(request);
@@ -243,14 +243,14 @@ export class ApifyRequestQueueSingleBackend extends ApifyRequestQueueBackend {
243
243
  }
244
244
  }
245
245
  async isKnownOrExists(id) {
246
- if (this.inProgressIds.has(id) || this.handledIds.has(id) || this.cachedRequests.get(id)) {
246
+ if (this.#inProgressIds.has(id) || this.#handledIds.has(id) || this.#cachedRequests.get(id)) {
247
247
  return true;
248
248
  }
249
249
  return (await this.getRequestById(id)) !== undefined;
250
250
  }
251
251
  cacheRequest(request) {
252
252
  // `LruCache.add` does not overwrite existing entries, so remove first.
253
- this.cachedRequests.remove(request.id);
254
- this.cachedRequests.add(request.id, request);
253
+ this.#cachedRequests.remove(request.id);
254
+ this.#cachedRequests.add(request.id, request);
255
255
  }
256
256
  }
@@ -62,18 +62,7 @@ export interface ApifyStorageBackendOptions {
62
62
  * ```
63
63
  */
64
64
  export declare class ApifyStorageBackend implements StorageBackend {
65
- private readonly client;
66
- private readonly config?;
67
- private readonly requestQueueAccess;
68
- private readonly getChargingManager?;
69
- /** Unnamed storages resolved for aliases in this process, keyed like {@link AliasMapping}. */
70
- private readonly aliasIdCache;
71
- /** The alias mapping read from the run's default key-value store; `undefined` until first read. */
72
- private persistedAliasIds?;
73
- /** Serializes alias resolution — see `resolveAliasId`. */
74
- private readonly aliasLock;
75
- /** Fallback request queue client key when the run id is unavailable — one per backend. */
76
- private fallbackClientKey?;
65
+ #private;
77
66
  constructor(client: ApifyClient, options?: ApifyStorageBackendOptions);
78
67
  /**
79
68
  * Partitions crawlee's storage-instance cache by API base URL and token, so the same storage
@@ -40,10 +40,10 @@ export const pushDataChargingContext = new AsyncLocalStorage();
40
40
  * used.
41
41
  */
42
42
  class PpeAwareDatasetClient extends ApifyDatasetClient {
43
- getChargingManager;
43
+ #getChargingManager;
44
44
  constructor(options, getChargingManager) {
45
45
  super(options);
46
- this.getChargingManager = getChargingManager;
46
+ this.#getChargingManager = getChargingManager;
47
47
  }
48
48
  normalizeItems(items) {
49
49
  if (typeof items === 'string') {
@@ -62,7 +62,7 @@ class PpeAwareDatasetClient extends ApifyDatasetClient {
62
62
  // each logical item is counted individually.
63
63
  const normalizedItems = this.normalizeItems(items);
64
64
  const result = await pushDataAndCharge({
65
- chargingManager: this.getChargingManager(),
65
+ chargingManager: this.#getChargingManager(),
66
66
  items: normalizedItems,
67
67
  eventName: context?.eventName,
68
68
  isDefaultDataset: true,
@@ -99,23 +99,23 @@ class PpeAwareDatasetClient extends ApifyDatasetClient {
99
99
  * ```
100
100
  */
101
101
  export class ApifyStorageBackend {
102
- client;
103
- config;
104
- requestQueueAccess;
105
- getChargingManager;
102
+ #client;
103
+ #config;
104
+ #requestQueueAccess;
105
+ #getChargingManager;
106
106
  /** Unnamed storages resolved for aliases in this process, keyed like {@link AliasMapping}. */
107
- aliasIdCache = new Map();
107
+ #aliasIdCache = new Map();
108
108
  /** The alias mapping read from the run's default key-value store; `undefined` until first read. */
109
- persistedAliasIds;
109
+ #persistedAliasIds;
110
110
  /** Serializes alias resolution — see `resolveAliasId`. */
111
- aliasLock = new AsyncLock();
111
+ #aliasLock = new AsyncLock();
112
112
  /** Fallback request queue client key when the run id is unavailable — one per backend. */
113
- fallbackClientKey;
113
+ #fallbackClientKey;
114
114
  constructor(client, options = {}) {
115
- this.client = client;
116
- this.config = options.configuration;
117
- this.requestQueueAccess = options.requestQueueAccess ?? 'single';
118
- this.getChargingManager = options.getChargingManager;
115
+ this.#client = client;
116
+ this.#config = options.configuration;
117
+ this.#requestQueueAccess = options.requestQueueAccess ?? 'single';
118
+ this.#getChargingManager = options.getChargingManager;
119
119
  }
120
120
  /**
121
121
  * Partitions crawlee's storage-instance cache by API base URL and token, so the same storage
@@ -129,7 +129,7 @@ export class ApifyStorageBackend {
129
129
  /** Short digest of the API base URL and token — identifies the credentials a storage was opened with. */
130
130
  credentialsHash() {
131
131
  return createHash('sha256')
132
- .update(`${this.client.publicBaseUrl}${this.client.token ?? ''}`)
132
+ .update(`${this.#client.publicBaseUrl}${this.#client.token ?? ''}`)
133
133
  .digest('hex')
134
134
  .slice(0, 8);
135
135
  }
@@ -145,7 +145,7 @@ export class ApifyStorageBackend {
145
145
  async createDatasetBackend(options) {
146
146
  const id = await this.resolveId(options, 'Dataset');
147
147
  const chargingClient = this.chargingDatasetClient(id);
148
- const backend = new ApifyDatasetBackend(chargingClient ?? this.client.dataset(id));
148
+ const backend = new ApifyDatasetBackend(chargingClient ?? this.#client.dataset(id));
149
149
  if (chargingClient) {
150
150
  // `Actor.pushData()` looks for this marker on the dataset's backend to know the
151
151
  // pay-per-event charging happens inside the intercepted `pushItems()` calls.
@@ -155,12 +155,12 @@ export class ApifyStorageBackend {
155
155
  }
156
156
  async createKeyValueStoreBackend(options) {
157
157
  const id = await this.resolveId(options, 'KeyValueStore');
158
- return new ApifyKeyValueStoreBackend(this.client.keyValueStore(id));
158
+ return new ApifyKeyValueStoreBackend(this.#client.keyValueStore(id));
159
159
  }
160
160
  async createRequestQueueBackend(options) {
161
161
  const id = await this.resolveId(options, 'RequestQueue');
162
- const client = this.client.requestQueue(id, { clientKey: this.requestQueueClientKey() });
163
- return this.requestQueueAccess === 'shared'
162
+ const client = this.#client.requestQueue(id, { clientKey: this.requestQueueClientKey() });
163
+ return this.#requestQueueAccess === 'shared'
164
164
  ? new ApifyRequestQueueSharedBackend(client)
165
165
  : new ApifyRequestQueueSingleBackend(client);
166
166
  }
@@ -169,7 +169,7 @@ export class ApifyStorageBackend {
169
169
  * migrated or resurrected run re-acquire the request locks of its previous incarnation.
170
170
  */
171
171
  requestQueueClientKey() {
172
- const key = this.config?.actorRunId ?? (this.fallbackClientKey ??= cryptoRandomObjectId(MAX_CLIENT_KEY_LENGTH));
172
+ const key = this.#config?.actorRunId ?? (this.#fallbackClientKey ??= cryptoRandomObjectId(MAX_CLIENT_KEY_LENGTH));
173
173
  return key.slice(0, MAX_CLIENT_KEY_LENGTH);
174
174
  }
175
175
  /**
@@ -178,20 +178,20 @@ export class ApifyStorageBackend {
178
178
  * `undefined` (caller uses the plain client).
179
179
  */
180
180
  chargingDatasetClient(id) {
181
- const { getChargingManager } = this;
181
+ const getChargingManager = this.#getChargingManager;
182
182
  if (!getChargingManager)
183
183
  return undefined;
184
- if (id !== this.config?.defaultDatasetId)
184
+ if (id !== this.#config?.defaultDatasetId)
185
185
  return undefined;
186
186
  const hasDefaultDatasetItemEvent = DEFAULT_DATASET_ITEM_EVENT in getChargingManager().getPricingInfo().perEventPrices;
187
187
  if (!hasDefaultDatasetItemEvent)
188
188
  return undefined;
189
189
  return new PpeAwareDatasetClient({
190
190
  id,
191
- baseUrl: this.client.baseUrl,
192
- publicBaseUrl: this.client.publicBaseUrl,
193
- apifyClient: this.client,
194
- httpClient: this.client.httpClient,
191
+ baseUrl: this.#client.baseUrl,
192
+ publicBaseUrl: this.#client.publicBaseUrl,
193
+ apifyClient: this.#client,
194
+ httpClient: this.#client.httpClient,
195
195
  }, getChargingManager);
196
196
  }
197
197
  /**
@@ -211,7 +211,7 @@ export class ApifyStorageBackend {
211
211
  }
212
212
  const alias = (options && 'alias' in options && options.alias) || DEFAULT_STORAGE_ALIAS;
213
213
  if (alias === DEFAULT_STORAGE_ALIAS) {
214
- const defaultId = this.config?.[DEFAULT_ID_CONFIG_KEY[type]];
214
+ const defaultId = this.#config?.[DEFAULT_ID_CONFIG_KEY[type]];
215
215
  if (defaultId)
216
216
  return defaultId;
217
217
  }
@@ -233,20 +233,20 @@ export class ApifyStorageBackend {
233
233
  // The credentials are part of the key, so the same alias opened through two
234
234
  // differently-authenticated backends maps to two storages.
235
235
  const key = [type, alias, this.credentialsHash()].join(',');
236
- return this.aliasLock.runExclusive(async () => {
237
- const knownId = this.aliasIdCache.get(key);
236
+ return this.#aliasLock.runExclusive(async () => {
237
+ const knownId = this.#aliasIdCache.get(key);
238
238
  if (knownId)
239
239
  return knownId;
240
240
  const store = this.aliasMappingStore();
241
- this.persistedAliasIds ??= store ? await readAliasMapping(store) : {};
241
+ this.#persistedAliasIds ??= store ? await readAliasMapping(store) : {};
242
242
  // A persisted id can point at a storage the user has since deleted.
243
- const persistedId = this.persistedAliasIds[key];
243
+ const persistedId = this.#persistedAliasIds[key];
244
244
  if (persistedId && (await this.resourceClient(persistedId, type).get())) {
245
- this.aliasIdCache.set(key, persistedId);
245
+ this.#aliasIdCache.set(key, persistedId);
246
246
  return persistedId;
247
247
  }
248
248
  const { id } = await this.collectionClient(type).getOrCreate();
249
- this.aliasIdCache.set(key, id);
249
+ this.#aliasIdCache.set(key, id);
250
250
  if (store)
251
251
  await this.persistAliasId(store, key, id);
252
252
  return id;
@@ -261,7 +261,7 @@ export class ApifyStorageBackend {
261
261
  const mapping = await readAliasMapping(store);
262
262
  mapping[key] = id;
263
263
  await store.setRecord({ key: ALIAS_MAPPING_RECORD_KEY, value: mapping });
264
- this.persistedAliasIds = mapping;
264
+ this.#persistedAliasIds = mapping;
265
265
  }
266
266
  catch (error) {
267
267
  log.warning(`Failed to persist the storage alias mapping: ${error.message}`);
@@ -269,11 +269,11 @@ export class ApifyStorageBackend {
269
269
  }
270
270
  /** The run's default key-value store, where the mapping lives — `undefined` off the platform. */
271
271
  aliasMappingStore() {
272
- return this.config?.isAtHome ? this.client.keyValueStore(this.config.defaultKeyValueStoreId) : undefined;
272
+ return this.#config?.isAtHome ? this.#client.keyValueStore(this.#config.defaultKeyValueStoreId) : undefined;
273
273
  }
274
274
  /** Looks an alias up in the Actor's schema storages (the `ACTOR_STORAGES_JSON` env var). */
275
275
  aliasFromActorStorages(alias, type) {
276
- const storagesJson = this.config?.actorStoragesJson;
276
+ const storagesJson = this.#config?.actorStoragesJson;
277
277
  if (!storagesJson)
278
278
  return undefined;
279
279
  let storages;
@@ -287,16 +287,16 @@ export class ApifyStorageBackend {
287
287
  }
288
288
  resourceClient(id, type) {
289
289
  if (type === 'Dataset')
290
- return this.client.dataset(id);
290
+ return this.#client.dataset(id);
291
291
  if (type === 'KeyValueStore')
292
- return this.client.keyValueStore(id);
293
- return this.client.requestQueue(id);
292
+ return this.#client.keyValueStore(id);
293
+ return this.#client.requestQueue(id);
294
294
  }
295
295
  collectionClient(type) {
296
296
  if (type === 'Dataset')
297
- return this.client.datasets();
297
+ return this.#client.datasets();
298
298
  if (type === 'KeyValueStore')
299
- return this.client.keyValueStores();
300
- return this.client.requestQueues();
299
+ return this.#client.keyValueStores();
300
+ return this.#client.requestQueues();
301
301
  }
302
302
  }