apify 4.0.0-beta.31 → 4.0.0-beta.32
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/actor.d.ts +6 -23
- package/dist/actor.js +55 -48
- package/dist/apify_dataset_backend.d.ts +1 -1
- package/dist/apify_dataset_backend.js +6 -6
- package/dist/apify_key_value_store_backend.d.ts +1 -1
- package/dist/apify_key_value_store_backend.js +10 -10
- package/dist/apify_request_queue_backend.d.ts +1 -1
- package/dist/apify_request_queue_backend.js +3 -3
- package/dist/apify_request_queue_shared_backend.d.ts +1 -14
- package/dist/apify_request_queue_shared_backend.js +38 -38
- package/dist/apify_request_queue_single_backend.d.ts +1 -10
- package/dist/apify_request_queue_single_backend.js +44 -44
- package/dist/apify_storage_backend.d.ts +1 -12
- package/dist/apify_storage_backend.js +43 -43
- package/dist/charging.d.ts +1 -14
- package/dist/charging.js +84 -81
- package/dist/platform_event_manager.d.ts +1 -2
- package/dist/platform_event_manager.js +7 -7
- package/dist/proxy_configuration.d.ts +7 -18
- package/dist/proxy_configuration.js +46 -57
- package/package.json +1 -1
|
@@ -21,23 +21,23 @@ const HEAD_LOCK_LIMIT = 25;
|
|
|
21
21
|
*/
|
|
22
22
|
export class ApifyRequestQueueSharedBackend extends ApifyRequestQueueBackend {
|
|
23
23
|
/** Ids of requests locked by this client and waiting to be handed out by `fetchNextRequest`. */
|
|
24
|
-
headIds = [];
|
|
24
|
+
#headIds = [];
|
|
25
25
|
/** Dedup records for requests known to exist on the platform, keyed by id. */
|
|
26
|
-
cachedRequestInfo = new LruCache({ maxLength: MAX_CACHED_REQUESTS });
|
|
26
|
+
#cachedRequestInfo = new LruCache({ maxLength: MAX_CACHED_REQUESTS });
|
|
27
27
|
/** Ids of requests currently being processed by this client. */
|
|
28
|
-
inProgressIds = new Set();
|
|
28
|
+
#inProgressIds = new Set();
|
|
29
29
|
/** Whether the last head read reported any locked requests left in the queue (any client's). */
|
|
30
|
-
queueHasLockedRequests;
|
|
30
|
+
#queueHasLockedRequests;
|
|
31
31
|
/** Set after a forefront insert — the next head read starts fresh so the insert is honored. */
|
|
32
|
-
shouldCheckForefrontRequests = false;
|
|
32
|
+
#shouldCheckForefrontRequests = false;
|
|
33
33
|
/** Lock duration applied to fetched requests; raised via `setExpectedRequestProcessingTimeSecs`. */
|
|
34
|
-
lockSecs = DEFAULT_REQUEST_LOCK_SECS;
|
|
34
|
+
#lockSecs = DEFAULT_REQUEST_LOCK_SECS;
|
|
35
35
|
/** Serializes head reads and reclaims — both reorder the shared head state. */
|
|
36
|
-
headLock = new AsyncLock();
|
|
36
|
+
#headLock = new AsyncLock();
|
|
37
37
|
async setExpectedRequestProcessingTimeSecs(secs) {
|
|
38
38
|
// Only ever raise the lock duration — several consumers may share this client, and a
|
|
39
39
|
// short-lived one must not cut the reservation of a long-running one short.
|
|
40
|
-
this
|
|
40
|
+
this.#lockSecs = Math.max(this.#lockSecs, secs);
|
|
41
41
|
}
|
|
42
42
|
async addBatchOfRequests(requests, options = {}) {
|
|
43
43
|
const { forefront = false } = options;
|
|
@@ -48,7 +48,7 @@ export class ApifyRequestQueueSharedBackend extends ApifyRequestQueueBackend {
|
|
|
48
48
|
const newRequests = [];
|
|
49
49
|
for (const request of requests) {
|
|
50
50
|
const id = this.requestIdFromUniqueKey(request.uniqueKey);
|
|
51
|
-
const cached = this
|
|
51
|
+
const cached = this.#cachedRequestInfo.get(id);
|
|
52
52
|
if (cached) {
|
|
53
53
|
alreadyPresent.push({
|
|
54
54
|
requestId: id,
|
|
@@ -70,7 +70,7 @@ export class ApifyRequestQueueSharedBackend extends ApifyRequestQueueBackend {
|
|
|
70
70
|
// A forefront insert changes the head order — have the next head read re-fetch the
|
|
71
71
|
// front of the queue instead of draining the local buffer first.
|
|
72
72
|
if (forefront) {
|
|
73
|
-
this
|
|
73
|
+
this.#shouldCheckForefrontRequests = true;
|
|
74
74
|
}
|
|
75
75
|
}
|
|
76
76
|
result.processedRequests.push(...alreadyPresent);
|
|
@@ -83,9 +83,9 @@ export class ApifyRequestQueueSharedBackend extends ApifyRequestQueueBackend {
|
|
|
83
83
|
return this.getRequestById(this.requestIdFromUniqueKey(uniqueKey));
|
|
84
84
|
}
|
|
85
85
|
async fetchNextRequest() {
|
|
86
|
-
const id = await this
|
|
86
|
+
const id = await this.#headLock.runExclusive(async () => {
|
|
87
87
|
await this.ensureHeadIsNonEmpty();
|
|
88
|
-
return this
|
|
88
|
+
return this.#headIds.shift();
|
|
89
89
|
});
|
|
90
90
|
if (!id)
|
|
91
91
|
return undefined;
|
|
@@ -103,7 +103,7 @@ export class ApifyRequestQueueSharedBackend extends ApifyRequestQueueBackend {
|
|
|
103
103
|
this.cacheRequestInfo(id, { wasAlreadyHandled: true });
|
|
104
104
|
return undefined;
|
|
105
105
|
}
|
|
106
|
-
this
|
|
106
|
+
this.#inProgressIds.add(id);
|
|
107
107
|
return request;
|
|
108
108
|
}
|
|
109
109
|
async markRequestAsHandled(request) {
|
|
@@ -111,12 +111,12 @@ export class ApifyRequestQueueSharedBackend extends ApifyRequestQueueBackend {
|
|
|
111
111
|
// Contract: marking a request that does not exist in the queue is a no-op — it must not be
|
|
112
112
|
// added as a side effect (the platform update endpoint would upsert it).
|
|
113
113
|
if (!(await this.isKnownOrExists(id))) {
|
|
114
|
-
this
|
|
114
|
+
this.#inProgressIds.delete(id);
|
|
115
115
|
return undefined;
|
|
116
116
|
}
|
|
117
117
|
const handledAt = request.handledAt ?? new Date().toISOString();
|
|
118
118
|
const info = await this.updateRequestOnPlatform({ ...request, id, handledAt });
|
|
119
|
-
this
|
|
119
|
+
this.#inProgressIds.delete(id);
|
|
120
120
|
this.cacheRequestInfo(id, { wasAlreadyHandled: true });
|
|
121
121
|
if (!info.wasAlreadyHandled) {
|
|
122
122
|
this.estimatedHandledRequestCount += 1;
|
|
@@ -128,10 +128,10 @@ export class ApifyRequestQueueSharedBackend extends ApifyRequestQueueBackend {
|
|
|
128
128
|
const id = this.requestIdFromUniqueKey(request.uniqueKey);
|
|
129
129
|
// Same contract as `markRequestAsHandled` — never insert as a side effect.
|
|
130
130
|
if (!(await this.isKnownOrExists(id))) {
|
|
131
|
-
this
|
|
131
|
+
this.#inProgressIds.delete(id);
|
|
132
132
|
return undefined;
|
|
133
133
|
}
|
|
134
|
-
return this
|
|
134
|
+
return this.#headLock.runExclusive(async () => {
|
|
135
135
|
const info = await this.updateRequestOnPlatform({ ...request, id, handledAt: undefined }, forefront);
|
|
136
136
|
// Release the server-side lock so the request becomes fetchable again immediately —
|
|
137
137
|
// by any consumer — rather than only after the lock expires.
|
|
@@ -141,10 +141,10 @@ export class ApifyRequestQueueSharedBackend extends ApifyRequestQueueBackend {
|
|
|
141
141
|
catch (err) {
|
|
142
142
|
log.debug(`Failed to delete the lock of a reclaimed request (id: ${id}): ${err.message}`);
|
|
143
143
|
}
|
|
144
|
-
this
|
|
144
|
+
this.#inProgressIds.delete(id);
|
|
145
145
|
this.cacheRequestInfo(id, { wasAlreadyHandled: false });
|
|
146
146
|
if (forefront) {
|
|
147
|
-
this
|
|
147
|
+
this.#shouldCheckForefrontRequests = true;
|
|
148
148
|
}
|
|
149
149
|
if (info.wasAlreadyHandled) {
|
|
150
150
|
this.estimatedHandledRequestCount -= 1;
|
|
@@ -153,25 +153,25 @@ export class ApifyRequestQueueSharedBackend extends ApifyRequestQueueBackend {
|
|
|
153
153
|
});
|
|
154
154
|
}
|
|
155
155
|
async isEmpty() {
|
|
156
|
-
return this
|
|
157
|
-
if (this
|
|
156
|
+
return this.#headLock.runExclusive(async () => {
|
|
157
|
+
if (this.#headIds.length > 0)
|
|
158
158
|
return false;
|
|
159
159
|
await this.listAndLockHead(1);
|
|
160
|
-
return this
|
|
160
|
+
return this.#headIds.length === 0;
|
|
161
161
|
});
|
|
162
162
|
}
|
|
163
163
|
async isFinished() {
|
|
164
|
-
return this
|
|
165
|
-
if (this
|
|
164
|
+
return this.#headLock.runExclusive(async () => {
|
|
165
|
+
if (this.#headIds.length > 0)
|
|
166
166
|
return false;
|
|
167
167
|
// The head read also refreshes `queueHasLockedRequests`, so the order matters here.
|
|
168
168
|
await this.listAndLockHead(1);
|
|
169
|
-
return this
|
|
169
|
+
return this.#headIds.length === 0 && !this.#queueHasLockedRequests;
|
|
170
170
|
});
|
|
171
171
|
}
|
|
172
172
|
/** Must be called with the head lock held. */
|
|
173
173
|
async ensureHeadIsNonEmpty() {
|
|
174
|
-
if (this
|
|
174
|
+
if (this.#headIds.length > 1 && !this.#shouldCheckForefrontRequests) {
|
|
175
175
|
return;
|
|
176
176
|
}
|
|
177
177
|
await this.listAndLockHead(HEAD_LOCK_LIMIT);
|
|
@@ -181,31 +181,31 @@ export class ApifyRequestQueueSharedBackend extends ApifyRequestQueueBackend {
|
|
|
181
181
|
// After a forefront insert the local buffer no longer starts at the true front of the
|
|
182
182
|
// queue — re-fetch the front and keep the already-locked leftovers for afterwards.
|
|
183
183
|
let leftoverIds = [];
|
|
184
|
-
if (this
|
|
185
|
-
leftoverIds = this
|
|
186
|
-
this
|
|
184
|
+
if (this.#shouldCheckForefrontRequests) {
|
|
185
|
+
leftoverIds = this.#headIds.splice(0);
|
|
186
|
+
this.#shouldCheckForefrontRequests = false;
|
|
187
187
|
}
|
|
188
|
-
const head = await this.client.listAndLockHead({ limit, lockSecs: this
|
|
189
|
-
this
|
|
188
|
+
const head = await this.client.listAndLockHead({ limit, lockSecs: this.#lockSecs });
|
|
189
|
+
this.#queueHasLockedRequests = head.queueHasLockedRequests;
|
|
190
190
|
for (const item of head.items) {
|
|
191
|
-
if (this
|
|
191
|
+
if (this.#inProgressIds.has(item.id))
|
|
192
192
|
continue;
|
|
193
|
-
if (this
|
|
193
|
+
if (this.#headIds.includes(item.id) || leftoverIds.includes(item.id))
|
|
194
194
|
continue;
|
|
195
195
|
this.cacheRequestInfo(item.id, { wasAlreadyHandled: false });
|
|
196
|
-
this
|
|
196
|
+
this.#headIds.push(item.id);
|
|
197
197
|
}
|
|
198
|
-
this
|
|
198
|
+
this.#headIds.push(...leftoverIds);
|
|
199
199
|
}
|
|
200
200
|
async isKnownOrExists(id) {
|
|
201
|
-
if (this
|
|
201
|
+
if (this.#inProgressIds.has(id) || this.#cachedRequestInfo.get(id)) {
|
|
202
202
|
return true;
|
|
203
203
|
}
|
|
204
204
|
return (await this.getRequestById(id)) !== undefined;
|
|
205
205
|
}
|
|
206
206
|
cacheRequestInfo(id, info) {
|
|
207
207
|
// `LruCache.add` does not overwrite existing entries, so remove first.
|
|
208
|
-
this
|
|
209
|
-
this
|
|
208
|
+
this.#cachedRequestInfo.remove(id);
|
|
209
|
+
this.#cachedRequestInfo.add(id, info);
|
|
210
210
|
}
|
|
211
211
|
}
|
|
@@ -22,16 +22,7 @@ import { ApifyRequestQueueBackend } from './apify_request_queue_backend.js';
|
|
|
22
22
|
* @internal
|
|
23
23
|
*/
|
|
24
24
|
export declare class ApifyRequestQueueSingleBackend extends ApifyRequestQueueBackend {
|
|
25
|
-
|
|
26
|
-
private readonly headIds;
|
|
27
|
-
/** Unhandled full request objects added by (or fetched through) this client, keyed by id. */
|
|
28
|
-
private readonly cachedRequests;
|
|
29
|
-
/** Ids of requests known to be already handled — cheap dedup without caching full objects. */
|
|
30
|
-
private readonly handledIds;
|
|
31
|
-
/** Ids of requests currently being processed by this client. */
|
|
32
|
-
private readonly inProgressIds;
|
|
33
|
-
/** Memoized one-time prefetch of existing queue contents into the local caches. */
|
|
34
|
-
private initCachesPromise?;
|
|
25
|
+
#private;
|
|
35
26
|
addBatchOfRequests(requests: RequestSchema[], options?: RequestQueueOperationOptions): Promise<BatchAddRequestsResult>;
|
|
36
27
|
getRequest(uniqueKey: string): Promise<UpdateRequestSchema | undefined>;
|
|
37
28
|
fetchNextRequest(): Promise<UpdateRequestSchema | undefined>;
|
|
@@ -32,25 +32,25 @@ const INIT_CACHES_REQUEST_LIMIT = 10_000;
|
|
|
32
32
|
*/
|
|
33
33
|
export class ApifyRequestQueueSingleBackend extends ApifyRequestQueueBackend {
|
|
34
34
|
/** Local estimate of the queue head — request ids in the order they should be fetched. */
|
|
35
|
-
headIds = [];
|
|
35
|
+
#headIds = [];
|
|
36
36
|
/** Unhandled full request objects added by (or fetched through) this client, keyed by id. */
|
|
37
|
-
cachedRequests = new LruCache({ maxLength: MAX_CACHED_REQUESTS });
|
|
37
|
+
#cachedRequests = new LruCache({ maxLength: MAX_CACHED_REQUESTS });
|
|
38
38
|
/** Ids of requests known to be already handled — cheap dedup without caching full objects. */
|
|
39
|
-
handledIds = new Set();
|
|
39
|
+
#handledIds = new Set();
|
|
40
40
|
/** Ids of requests currently being processed by this client. */
|
|
41
|
-
inProgressIds = new Set();
|
|
41
|
+
#inProgressIds = new Set();
|
|
42
42
|
/** Memoized one-time prefetch of existing queue contents into the local caches. */
|
|
43
|
-
initCachesPromise;
|
|
43
|
+
#initCachesPromise;
|
|
44
44
|
async addBatchOfRequests(requests, options = {}) {
|
|
45
45
|
const { forefront = false } = options;
|
|
46
|
-
await (this
|
|
46
|
+
await (this.#initCachesPromise ??= this.initCaches());
|
|
47
47
|
// Split the batch into requests we already know about (dedup them locally — a platform
|
|
48
48
|
// write costs an API call and a paid write operation) and genuinely new ones.
|
|
49
49
|
const alreadyPresent = [];
|
|
50
50
|
const newRequests = [];
|
|
51
51
|
for (const request of requests) {
|
|
52
52
|
const id = this.requestIdFromUniqueKey(request.uniqueKey);
|
|
53
|
-
if (this
|
|
53
|
+
if (this.#handledIds.has(id)) {
|
|
54
54
|
alreadyPresent.push({
|
|
55
55
|
requestId: id,
|
|
56
56
|
uniqueKey: request.uniqueKey,
|
|
@@ -58,7 +58,7 @@ export class ApifyRequestQueueSingleBackend extends ApifyRequestQueueBackend {
|
|
|
58
58
|
wasAlreadyHandled: true,
|
|
59
59
|
});
|
|
60
60
|
}
|
|
61
|
-
else if (this
|
|
61
|
+
else if (this.#cachedRequests.get(id)) {
|
|
62
62
|
alreadyPresent.push({
|
|
63
63
|
requestId: id,
|
|
64
64
|
uniqueKey: request.uniqueKey,
|
|
@@ -83,15 +83,15 @@ export class ApifyRequestQueueSingleBackend extends ApifyRequestQueueBackend {
|
|
|
83
83
|
if (!processed)
|
|
84
84
|
continue; // rejected by the platform, reported in `unprocessedRequests`
|
|
85
85
|
if (processed.wasAlreadyHandled) {
|
|
86
|
-
this
|
|
86
|
+
this.#handledIds.add(processed.requestId);
|
|
87
87
|
continue;
|
|
88
88
|
}
|
|
89
89
|
this.cacheRequest({ ...request, id: processed.requestId });
|
|
90
90
|
if (forefront) {
|
|
91
|
-
this
|
|
91
|
+
this.#headIds.unshift(processed.requestId);
|
|
92
92
|
}
|
|
93
93
|
else {
|
|
94
|
-
this
|
|
94
|
+
this.#headIds.push(processed.requestId);
|
|
95
95
|
}
|
|
96
96
|
}
|
|
97
97
|
}
|
|
@@ -101,16 +101,16 @@ export class ApifyRequestQueueSingleBackend extends ApifyRequestQueueBackend {
|
|
|
101
101
|
}
|
|
102
102
|
async getRequest(uniqueKey) {
|
|
103
103
|
const id = this.requestIdFromUniqueKey(uniqueKey);
|
|
104
|
-
const cached = this
|
|
104
|
+
const cached = this.#cachedRequests.get(id);
|
|
105
105
|
if (cached)
|
|
106
106
|
return cached;
|
|
107
107
|
const request = await this.getRequestById(id);
|
|
108
108
|
if (!request)
|
|
109
109
|
return undefined;
|
|
110
110
|
// Requests already in progress are ones the client knows about — no caching needed.
|
|
111
|
-
if (!this
|
|
111
|
+
if (!this.#inProgressIds.has(id)) {
|
|
112
112
|
if (request.handledAt) {
|
|
113
|
-
this
|
|
113
|
+
this.#handledIds.add(id);
|
|
114
114
|
}
|
|
115
115
|
else {
|
|
116
116
|
this.cacheRequest(request);
|
|
@@ -120,24 +120,24 @@ export class ApifyRequestQueueSingleBackend extends ApifyRequestQueueBackend {
|
|
|
120
120
|
}
|
|
121
121
|
async fetchNextRequest() {
|
|
122
122
|
await this.ensureHeadIsNonEmpty();
|
|
123
|
-
while (this
|
|
124
|
-
const id = this
|
|
125
|
-
if (this
|
|
123
|
+
while (this.#headIds.length > 0) {
|
|
124
|
+
const id = this.#headIds.shift();
|
|
125
|
+
if (this.#inProgressIds.has(id) || this.#handledIds.has(id)) {
|
|
126
126
|
continue;
|
|
127
127
|
}
|
|
128
|
-
this
|
|
128
|
+
this.#inProgressIds.add(id);
|
|
129
129
|
// Requests added by this client are served straight from the cache; only requests
|
|
130
130
|
// discovered via `listHead` (added by another producer) need a round-trip.
|
|
131
|
-
const request = this
|
|
131
|
+
const request = this.#cachedRequests.get(id) ?? (await this.getRequestById(id));
|
|
132
132
|
if (!request) {
|
|
133
|
-
this
|
|
133
|
+
this.#inProgressIds.delete(id);
|
|
134
134
|
continue;
|
|
135
135
|
}
|
|
136
136
|
if (request.handledAt) {
|
|
137
137
|
// Handled elsewhere in the meantime — skip it and remember the outcome.
|
|
138
|
-
this
|
|
139
|
-
this
|
|
140
|
-
this
|
|
138
|
+
this.#inProgressIds.delete(id);
|
|
139
|
+
this.#handledIds.add(id);
|
|
140
|
+
this.#cachedRequests.remove(id);
|
|
141
141
|
continue;
|
|
142
142
|
}
|
|
143
143
|
return request;
|
|
@@ -149,14 +149,14 @@ export class ApifyRequestQueueSingleBackend extends ApifyRequestQueueBackend {
|
|
|
149
149
|
// Contract: marking a request that does not exist in the queue is a no-op — it must not be
|
|
150
150
|
// added as a side effect (the platform update endpoint would upsert it).
|
|
151
151
|
if (!(await this.isKnownOrExists(id))) {
|
|
152
|
-
this
|
|
152
|
+
this.#inProgressIds.delete(id);
|
|
153
153
|
return undefined;
|
|
154
154
|
}
|
|
155
155
|
const handledAt = request.handledAt ?? new Date().toISOString();
|
|
156
156
|
const info = await this.updateRequestOnPlatform({ ...request, id, handledAt });
|
|
157
|
-
this
|
|
158
|
-
this
|
|
159
|
-
this
|
|
157
|
+
this.#inProgressIds.delete(id);
|
|
158
|
+
this.#handledIds.add(id);
|
|
159
|
+
this.#cachedRequests.remove(id);
|
|
160
160
|
if (!info.wasAlreadyHandled) {
|
|
161
161
|
this.estimatedHandledRequestCount += 1;
|
|
162
162
|
}
|
|
@@ -167,24 +167,24 @@ export class ApifyRequestQueueSingleBackend extends ApifyRequestQueueBackend {
|
|
|
167
167
|
const id = this.requestIdFromUniqueKey(request.uniqueKey);
|
|
168
168
|
// Same contract as `markRequestAsHandled` — never insert as a side effect.
|
|
169
169
|
if (!(await this.isKnownOrExists(id))) {
|
|
170
|
-
this
|
|
170
|
+
this.#inProgressIds.delete(id);
|
|
171
171
|
return undefined;
|
|
172
172
|
}
|
|
173
173
|
// Reclaiming returns the request to the queue for reprocessing.
|
|
174
174
|
const reclaimed = { ...request, id, handledAt: undefined };
|
|
175
175
|
const info = await this.updateRequestOnPlatform(reclaimed, forefront);
|
|
176
|
-
this
|
|
177
|
-
this
|
|
176
|
+
this.#inProgressIds.delete(id);
|
|
177
|
+
this.#handledIds.delete(id);
|
|
178
178
|
this.cacheRequest(reclaimed);
|
|
179
179
|
// Return the id to the local head estimate right away — the platform head read can lag a
|
|
180
180
|
// few seconds behind the update, and `isFinished` must never report `true` while a
|
|
181
181
|
// reclaimed request is still waiting to be reprocessed.
|
|
182
|
-
if (!this
|
|
182
|
+
if (!this.#headIds.includes(id)) {
|
|
183
183
|
if (forefront) {
|
|
184
|
-
this
|
|
184
|
+
this.#headIds.unshift(id);
|
|
185
185
|
}
|
|
186
186
|
else {
|
|
187
|
-
this
|
|
187
|
+
this.#headIds.push(id);
|
|
188
188
|
}
|
|
189
189
|
}
|
|
190
190
|
if (info.wasAlreadyHandled) {
|
|
@@ -194,28 +194,28 @@ export class ApifyRequestQueueSingleBackend extends ApifyRequestQueueBackend {
|
|
|
194
194
|
}
|
|
195
195
|
async isEmpty() {
|
|
196
196
|
await this.ensureHeadIsNonEmpty();
|
|
197
|
-
return this
|
|
197
|
+
return this.#headIds.length === 0;
|
|
198
198
|
}
|
|
199
199
|
async isFinished() {
|
|
200
|
-
return (await this.isEmpty()) && this
|
|
200
|
+
return (await this.isEmpty()) && this.#inProgressIds.size === 0;
|
|
201
201
|
}
|
|
202
202
|
async ensureHeadIsNonEmpty() {
|
|
203
|
-
if (this
|
|
203
|
+
if (this.#headIds.length <= 1) {
|
|
204
204
|
await this.listHead();
|
|
205
205
|
}
|
|
206
206
|
}
|
|
207
207
|
async listHead() {
|
|
208
208
|
// The head read returns in-progress requests too, so fetch enough to find new ones.
|
|
209
|
-
const limit = Math.min(MAX_HEAD_ITEMS, DESIRED_NEW_HEAD_ITEMS + this
|
|
209
|
+
const limit = Math.min(MAX_HEAD_ITEMS, DESIRED_NEW_HEAD_ITEMS + this.#inProgressIds.size);
|
|
210
210
|
const head = await this.client.listHead({ limit });
|
|
211
211
|
for (const item of head.items) {
|
|
212
|
-
if (this
|
|
212
|
+
if (this.#inProgressIds.has(item.id) || this.#handledIds.has(item.id)) {
|
|
213
213
|
continue;
|
|
214
214
|
}
|
|
215
215
|
// `headIds` is nearly drained whenever this runs (see `ensureHeadIsNonEmpty`), so the
|
|
216
216
|
// linear dedup scan stays cheap.
|
|
217
|
-
if (!this
|
|
218
|
-
this
|
|
217
|
+
if (!this.#headIds.includes(item.id)) {
|
|
218
|
+
this.#headIds.push(item.id);
|
|
219
219
|
}
|
|
220
220
|
}
|
|
221
221
|
}
|
|
@@ -229,7 +229,7 @@ export class ApifyRequestQueueSingleBackend extends ApifyRequestQueueBackend {
|
|
|
229
229
|
const response = await this.client.listRequests({ limit: INIT_CACHES_REQUEST_LIMIT });
|
|
230
230
|
for (const request of response.items) {
|
|
231
231
|
if (request.handledAt) {
|
|
232
|
-
this
|
|
232
|
+
this.#handledIds.add(request.id);
|
|
233
233
|
}
|
|
234
234
|
else {
|
|
235
235
|
this.cacheRequest(request);
|
|
@@ -243,14 +243,14 @@ export class ApifyRequestQueueSingleBackend extends ApifyRequestQueueBackend {
|
|
|
243
243
|
}
|
|
244
244
|
}
|
|
245
245
|
async isKnownOrExists(id) {
|
|
246
|
-
if (this
|
|
246
|
+
if (this.#inProgressIds.has(id) || this.#handledIds.has(id) || this.#cachedRequests.get(id)) {
|
|
247
247
|
return true;
|
|
248
248
|
}
|
|
249
249
|
return (await this.getRequestById(id)) !== undefined;
|
|
250
250
|
}
|
|
251
251
|
cacheRequest(request) {
|
|
252
252
|
// `LruCache.add` does not overwrite existing entries, so remove first.
|
|
253
|
-
this
|
|
254
|
-
this
|
|
253
|
+
this.#cachedRequests.remove(request.id);
|
|
254
|
+
this.#cachedRequests.add(request.id, request);
|
|
255
255
|
}
|
|
256
256
|
}
|
|
@@ -62,18 +62,7 @@ export interface ApifyStorageBackendOptions {
|
|
|
62
62
|
* ```
|
|
63
63
|
*/
|
|
64
64
|
export declare class ApifyStorageBackend implements StorageBackend {
|
|
65
|
-
private
|
|
66
|
-
private readonly config?;
|
|
67
|
-
private readonly requestQueueAccess;
|
|
68
|
-
private readonly getChargingManager?;
|
|
69
|
-
/** Unnamed storages resolved for aliases in this process, keyed like {@link AliasMapping}. */
|
|
70
|
-
private readonly aliasIdCache;
|
|
71
|
-
/** The alias mapping read from the run's default key-value store; `undefined` until first read. */
|
|
72
|
-
private persistedAliasIds?;
|
|
73
|
-
/** Serializes alias resolution — see `resolveAliasId`. */
|
|
74
|
-
private readonly aliasLock;
|
|
75
|
-
/** Fallback request queue client key when the run id is unavailable — one per backend. */
|
|
76
|
-
private fallbackClientKey?;
|
|
65
|
+
#private;
|
|
77
66
|
constructor(client: ApifyClient, options?: ApifyStorageBackendOptions);
|
|
78
67
|
/**
|
|
79
68
|
* Partitions crawlee's storage-instance cache by API base URL and token, so the same storage
|
|
@@ -40,10 +40,10 @@ export const pushDataChargingContext = new AsyncLocalStorage();
|
|
|
40
40
|
* used.
|
|
41
41
|
*/
|
|
42
42
|
class PpeAwareDatasetClient extends ApifyDatasetClient {
|
|
43
|
-
getChargingManager;
|
|
43
|
+
#getChargingManager;
|
|
44
44
|
constructor(options, getChargingManager) {
|
|
45
45
|
super(options);
|
|
46
|
-
this
|
|
46
|
+
this.#getChargingManager = getChargingManager;
|
|
47
47
|
}
|
|
48
48
|
normalizeItems(items) {
|
|
49
49
|
if (typeof items === 'string') {
|
|
@@ -62,7 +62,7 @@ class PpeAwareDatasetClient extends ApifyDatasetClient {
|
|
|
62
62
|
// each logical item is counted individually.
|
|
63
63
|
const normalizedItems = this.normalizeItems(items);
|
|
64
64
|
const result = await pushDataAndCharge({
|
|
65
|
-
chargingManager: this
|
|
65
|
+
chargingManager: this.#getChargingManager(),
|
|
66
66
|
items: normalizedItems,
|
|
67
67
|
eventName: context?.eventName,
|
|
68
68
|
isDefaultDataset: true,
|
|
@@ -99,23 +99,23 @@ class PpeAwareDatasetClient extends ApifyDatasetClient {
|
|
|
99
99
|
* ```
|
|
100
100
|
*/
|
|
101
101
|
export class ApifyStorageBackend {
|
|
102
|
-
client;
|
|
103
|
-
config;
|
|
104
|
-
requestQueueAccess;
|
|
105
|
-
getChargingManager;
|
|
102
|
+
#client;
|
|
103
|
+
#config;
|
|
104
|
+
#requestQueueAccess;
|
|
105
|
+
#getChargingManager;
|
|
106
106
|
/** Unnamed storages resolved for aliases in this process, keyed like {@link AliasMapping}. */
|
|
107
|
-
aliasIdCache = new Map();
|
|
107
|
+
#aliasIdCache = new Map();
|
|
108
108
|
/** The alias mapping read from the run's default key-value store; `undefined` until first read. */
|
|
109
|
-
persistedAliasIds;
|
|
109
|
+
#persistedAliasIds;
|
|
110
110
|
/** Serializes alias resolution — see `resolveAliasId`. */
|
|
111
|
-
aliasLock = new AsyncLock();
|
|
111
|
+
#aliasLock = new AsyncLock();
|
|
112
112
|
/** Fallback request queue client key when the run id is unavailable — one per backend. */
|
|
113
|
-
fallbackClientKey;
|
|
113
|
+
#fallbackClientKey;
|
|
114
114
|
constructor(client, options = {}) {
|
|
115
|
-
this
|
|
116
|
-
this
|
|
117
|
-
this
|
|
118
|
-
this
|
|
115
|
+
this.#client = client;
|
|
116
|
+
this.#config = options.configuration;
|
|
117
|
+
this.#requestQueueAccess = options.requestQueueAccess ?? 'single';
|
|
118
|
+
this.#getChargingManager = options.getChargingManager;
|
|
119
119
|
}
|
|
120
120
|
/**
|
|
121
121
|
* Partitions crawlee's storage-instance cache by API base URL and token, so the same storage
|
|
@@ -129,7 +129,7 @@ export class ApifyStorageBackend {
|
|
|
129
129
|
/** Short digest of the API base URL and token — identifies the credentials a storage was opened with. */
|
|
130
130
|
credentialsHash() {
|
|
131
131
|
return createHash('sha256')
|
|
132
|
-
.update(`${this
|
|
132
|
+
.update(`${this.#client.publicBaseUrl}${this.#client.token ?? ''}`)
|
|
133
133
|
.digest('hex')
|
|
134
134
|
.slice(0, 8);
|
|
135
135
|
}
|
|
@@ -145,7 +145,7 @@ export class ApifyStorageBackend {
|
|
|
145
145
|
async createDatasetBackend(options) {
|
|
146
146
|
const id = await this.resolveId(options, 'Dataset');
|
|
147
147
|
const chargingClient = this.chargingDatasetClient(id);
|
|
148
|
-
const backend = new ApifyDatasetBackend(chargingClient ?? this
|
|
148
|
+
const backend = new ApifyDatasetBackend(chargingClient ?? this.#client.dataset(id));
|
|
149
149
|
if (chargingClient) {
|
|
150
150
|
// `Actor.pushData()` looks for this marker on the dataset's backend to know the
|
|
151
151
|
// pay-per-event charging happens inside the intercepted `pushItems()` calls.
|
|
@@ -155,12 +155,12 @@ export class ApifyStorageBackend {
|
|
|
155
155
|
}
|
|
156
156
|
async createKeyValueStoreBackend(options) {
|
|
157
157
|
const id = await this.resolveId(options, 'KeyValueStore');
|
|
158
|
-
return new ApifyKeyValueStoreBackend(this
|
|
158
|
+
return new ApifyKeyValueStoreBackend(this.#client.keyValueStore(id));
|
|
159
159
|
}
|
|
160
160
|
async createRequestQueueBackend(options) {
|
|
161
161
|
const id = await this.resolveId(options, 'RequestQueue');
|
|
162
|
-
const client = this
|
|
163
|
-
return this
|
|
162
|
+
const client = this.#client.requestQueue(id, { clientKey: this.requestQueueClientKey() });
|
|
163
|
+
return this.#requestQueueAccess === 'shared'
|
|
164
164
|
? new ApifyRequestQueueSharedBackend(client)
|
|
165
165
|
: new ApifyRequestQueueSingleBackend(client);
|
|
166
166
|
}
|
|
@@ -169,7 +169,7 @@ export class ApifyStorageBackend {
|
|
|
169
169
|
* migrated or resurrected run re-acquire the request locks of its previous incarnation.
|
|
170
170
|
*/
|
|
171
171
|
requestQueueClientKey() {
|
|
172
|
-
const key = this
|
|
172
|
+
const key = this.#config?.actorRunId ?? (this.#fallbackClientKey ??= cryptoRandomObjectId(MAX_CLIENT_KEY_LENGTH));
|
|
173
173
|
return key.slice(0, MAX_CLIENT_KEY_LENGTH);
|
|
174
174
|
}
|
|
175
175
|
/**
|
|
@@ -178,20 +178,20 @@ export class ApifyStorageBackend {
|
|
|
178
178
|
* `undefined` (caller uses the plain client).
|
|
179
179
|
*/
|
|
180
180
|
chargingDatasetClient(id) {
|
|
181
|
-
const
|
|
181
|
+
const getChargingManager = this.#getChargingManager;
|
|
182
182
|
if (!getChargingManager)
|
|
183
183
|
return undefined;
|
|
184
|
-
if (id !== this
|
|
184
|
+
if (id !== this.#config?.defaultDatasetId)
|
|
185
185
|
return undefined;
|
|
186
186
|
const hasDefaultDatasetItemEvent = DEFAULT_DATASET_ITEM_EVENT in getChargingManager().getPricingInfo().perEventPrices;
|
|
187
187
|
if (!hasDefaultDatasetItemEvent)
|
|
188
188
|
return undefined;
|
|
189
189
|
return new PpeAwareDatasetClient({
|
|
190
190
|
id,
|
|
191
|
-
baseUrl: this
|
|
192
|
-
publicBaseUrl: this
|
|
193
|
-
apifyClient: this
|
|
194
|
-
httpClient: this
|
|
191
|
+
baseUrl: this.#client.baseUrl,
|
|
192
|
+
publicBaseUrl: this.#client.publicBaseUrl,
|
|
193
|
+
apifyClient: this.#client,
|
|
194
|
+
httpClient: this.#client.httpClient,
|
|
195
195
|
}, getChargingManager);
|
|
196
196
|
}
|
|
197
197
|
/**
|
|
@@ -211,7 +211,7 @@ export class ApifyStorageBackend {
|
|
|
211
211
|
}
|
|
212
212
|
const alias = (options && 'alias' in options && options.alias) || DEFAULT_STORAGE_ALIAS;
|
|
213
213
|
if (alias === DEFAULT_STORAGE_ALIAS) {
|
|
214
|
-
const defaultId = this
|
|
214
|
+
const defaultId = this.#config?.[DEFAULT_ID_CONFIG_KEY[type]];
|
|
215
215
|
if (defaultId)
|
|
216
216
|
return defaultId;
|
|
217
217
|
}
|
|
@@ -233,20 +233,20 @@ export class ApifyStorageBackend {
|
|
|
233
233
|
// The credentials are part of the key, so the same alias opened through two
|
|
234
234
|
// differently-authenticated backends maps to two storages.
|
|
235
235
|
const key = [type, alias, this.credentialsHash()].join(',');
|
|
236
|
-
return this
|
|
237
|
-
const knownId = this
|
|
236
|
+
return this.#aliasLock.runExclusive(async () => {
|
|
237
|
+
const knownId = this.#aliasIdCache.get(key);
|
|
238
238
|
if (knownId)
|
|
239
239
|
return knownId;
|
|
240
240
|
const store = this.aliasMappingStore();
|
|
241
|
-
this
|
|
241
|
+
this.#persistedAliasIds ??= store ? await readAliasMapping(store) : {};
|
|
242
242
|
// A persisted id can point at a storage the user has since deleted.
|
|
243
|
-
const persistedId = this
|
|
243
|
+
const persistedId = this.#persistedAliasIds[key];
|
|
244
244
|
if (persistedId && (await this.resourceClient(persistedId, type).get())) {
|
|
245
|
-
this
|
|
245
|
+
this.#aliasIdCache.set(key, persistedId);
|
|
246
246
|
return persistedId;
|
|
247
247
|
}
|
|
248
248
|
const { id } = await this.collectionClient(type).getOrCreate();
|
|
249
|
-
this
|
|
249
|
+
this.#aliasIdCache.set(key, id);
|
|
250
250
|
if (store)
|
|
251
251
|
await this.persistAliasId(store, key, id);
|
|
252
252
|
return id;
|
|
@@ -261,7 +261,7 @@ export class ApifyStorageBackend {
|
|
|
261
261
|
const mapping = await readAliasMapping(store);
|
|
262
262
|
mapping[key] = id;
|
|
263
263
|
await store.setRecord({ key: ALIAS_MAPPING_RECORD_KEY, value: mapping });
|
|
264
|
-
this
|
|
264
|
+
this.#persistedAliasIds = mapping;
|
|
265
265
|
}
|
|
266
266
|
catch (error) {
|
|
267
267
|
log.warning(`Failed to persist the storage alias mapping: ${error.message}`);
|
|
@@ -269,11 +269,11 @@ export class ApifyStorageBackend {
|
|
|
269
269
|
}
|
|
270
270
|
/** The run's default key-value store, where the mapping lives — `undefined` off the platform. */
|
|
271
271
|
aliasMappingStore() {
|
|
272
|
-
return this
|
|
272
|
+
return this.#config?.isAtHome ? this.#client.keyValueStore(this.#config.defaultKeyValueStoreId) : undefined;
|
|
273
273
|
}
|
|
274
274
|
/** Looks an alias up in the Actor's schema storages (the `ACTOR_STORAGES_JSON` env var). */
|
|
275
275
|
aliasFromActorStorages(alias, type) {
|
|
276
|
-
const storagesJson = this
|
|
276
|
+
const storagesJson = this.#config?.actorStoragesJson;
|
|
277
277
|
if (!storagesJson)
|
|
278
278
|
return undefined;
|
|
279
279
|
let storages;
|
|
@@ -287,16 +287,16 @@ export class ApifyStorageBackend {
|
|
|
287
287
|
}
|
|
288
288
|
resourceClient(id, type) {
|
|
289
289
|
if (type === 'Dataset')
|
|
290
|
-
return this
|
|
290
|
+
return this.#client.dataset(id);
|
|
291
291
|
if (type === 'KeyValueStore')
|
|
292
|
-
return this
|
|
293
|
-
return this
|
|
292
|
+
return this.#client.keyValueStore(id);
|
|
293
|
+
return this.#client.requestQueue(id);
|
|
294
294
|
}
|
|
295
295
|
collectionClient(type) {
|
|
296
296
|
if (type === 'Dataset')
|
|
297
|
-
return this
|
|
297
|
+
return this.#client.datasets();
|
|
298
298
|
if (type === 'KeyValueStore')
|
|
299
|
-
return this
|
|
300
|
-
return this
|
|
299
|
+
return this.#client.keyValueStores();
|
|
300
|
+
return this.#client.requestQueues();
|
|
301
301
|
}
|
|
302
302
|
}
|