apify 4.0.0-beta.30 → 4.0.0-beta.32
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/actor.d.ts +21 -43
- package/dist/actor.js +74 -74
- package/dist/apify_dataset_backend.d.ts +1 -1
- package/dist/apify_dataset_backend.js +6 -6
- package/dist/apify_key_value_store_backend.d.ts +1 -1
- package/dist/apify_key_value_store_backend.js +10 -10
- package/dist/apify_request_queue_backend.d.ts +1 -1
- package/dist/apify_request_queue_backend.js +3 -3
- package/dist/apify_request_queue_shared_backend.d.ts +1 -14
- package/dist/apify_request_queue_shared_backend.js +38 -38
- package/dist/apify_request_queue_single_backend.d.ts +1 -10
- package/dist/apify_request_queue_single_backend.js +44 -44
- package/dist/apify_storage_backend.d.ts +23 -12
- package/dist/apify_storage_backend.js +104 -51
- package/dist/charging.d.ts +1 -14
- package/dist/charging.js +84 -81
- package/dist/index.d.ts +1 -1
- package/dist/platform_event_manager.d.ts +1 -2
- package/dist/platform_event_manager.js +7 -7
- package/dist/proxy_configuration.d.ts +7 -18
- package/dist/proxy_configuration.js +46 -57
- package/dist/storage.d.ts +6 -26
- package/dist/storage.js +1 -79
- package/package.json +1 -1
|
@@ -104,11 +104,11 @@ export class ApifyRequestQueueBackend {
|
|
|
104
104
|
* @internal
|
|
105
105
|
*/
|
|
106
106
|
export class AsyncLock {
|
|
107
|
-
tail = Promise.resolve();
|
|
107
|
+
#tail = Promise.resolve();
|
|
108
108
|
async runExclusive(fn) {
|
|
109
|
-
const run = this
|
|
109
|
+
const run = this.#tail.then(fn);
|
|
110
110
|
// Keep the chain alive even when the critical section throws.
|
|
111
|
-
this
|
|
111
|
+
this.#tail = run.catch(() => { });
|
|
112
112
|
return run;
|
|
113
113
|
}
|
|
114
114
|
}
|
|
@@ -13,20 +13,7 @@ import { ApifyRequestQueueBackend } from './apify_request_queue_backend.js';
|
|
|
13
13
|
* @internal
|
|
14
14
|
*/
|
|
15
15
|
export declare class ApifyRequestQueueSharedBackend extends ApifyRequestQueueBackend {
|
|
16
|
-
|
|
17
|
-
private readonly headIds;
|
|
18
|
-
/** Dedup records for requests known to exist on the platform, keyed by id. */
|
|
19
|
-
private readonly cachedRequestInfo;
|
|
20
|
-
/** Ids of requests currently being processed by this client. */
|
|
21
|
-
private readonly inProgressIds;
|
|
22
|
-
/** Whether the last head read reported any locked requests left in the queue (any client's). */
|
|
23
|
-
private queueHasLockedRequests?;
|
|
24
|
-
/** Set after a forefront insert — the next head read starts fresh so the insert is honored. */
|
|
25
|
-
private shouldCheckForefrontRequests;
|
|
26
|
-
/** Lock duration applied to fetched requests; raised via `setExpectedRequestProcessingTimeSecs`. */
|
|
27
|
-
private lockSecs;
|
|
28
|
-
/** Serializes head reads and reclaims — both reorder the shared head state. */
|
|
29
|
-
private readonly headLock;
|
|
16
|
+
#private;
|
|
30
17
|
setExpectedRequestProcessingTimeSecs(secs: number): Promise<void>;
|
|
31
18
|
addBatchOfRequests(requests: RequestSchema[], options?: RequestQueueOperationOptions): Promise<BatchAddRequestsResult>;
|
|
32
19
|
getRequest(uniqueKey: string): Promise<UpdateRequestSchema | undefined>;
|
|
@@ -21,23 +21,23 @@ const HEAD_LOCK_LIMIT = 25;
|
|
|
21
21
|
*/
|
|
22
22
|
export class ApifyRequestQueueSharedBackend extends ApifyRequestQueueBackend {
|
|
23
23
|
/** Ids of requests locked by this client and waiting to be handed out by `fetchNextRequest`. */
|
|
24
|
-
headIds = [];
|
|
24
|
+
#headIds = [];
|
|
25
25
|
/** Dedup records for requests known to exist on the platform, keyed by id. */
|
|
26
|
-
cachedRequestInfo = new LruCache({ maxLength: MAX_CACHED_REQUESTS });
|
|
26
|
+
#cachedRequestInfo = new LruCache({ maxLength: MAX_CACHED_REQUESTS });
|
|
27
27
|
/** Ids of requests currently being processed by this client. */
|
|
28
|
-
inProgressIds = new Set();
|
|
28
|
+
#inProgressIds = new Set();
|
|
29
29
|
/** Whether the last head read reported any locked requests left in the queue (any client's). */
|
|
30
|
-
queueHasLockedRequests;
|
|
30
|
+
#queueHasLockedRequests;
|
|
31
31
|
/** Set after a forefront insert — the next head read starts fresh so the insert is honored. */
|
|
32
|
-
shouldCheckForefrontRequests = false;
|
|
32
|
+
#shouldCheckForefrontRequests = false;
|
|
33
33
|
/** Lock duration applied to fetched requests; raised via `setExpectedRequestProcessingTimeSecs`. */
|
|
34
|
-
lockSecs = DEFAULT_REQUEST_LOCK_SECS;
|
|
34
|
+
#lockSecs = DEFAULT_REQUEST_LOCK_SECS;
|
|
35
35
|
/** Serializes head reads and reclaims — both reorder the shared head state. */
|
|
36
|
-
headLock = new AsyncLock();
|
|
36
|
+
#headLock = new AsyncLock();
|
|
37
37
|
async setExpectedRequestProcessingTimeSecs(secs) {
|
|
38
38
|
// Only ever raise the lock duration — several consumers may share this client, and a
|
|
39
39
|
// short-lived one must not cut the reservation of a long-running one short.
|
|
40
|
-
this
|
|
40
|
+
this.#lockSecs = Math.max(this.#lockSecs, secs);
|
|
41
41
|
}
|
|
42
42
|
async addBatchOfRequests(requests, options = {}) {
|
|
43
43
|
const { forefront = false } = options;
|
|
@@ -48,7 +48,7 @@ export class ApifyRequestQueueSharedBackend extends ApifyRequestQueueBackend {
|
|
|
48
48
|
const newRequests = [];
|
|
49
49
|
for (const request of requests) {
|
|
50
50
|
const id = this.requestIdFromUniqueKey(request.uniqueKey);
|
|
51
|
-
const cached = this
|
|
51
|
+
const cached = this.#cachedRequestInfo.get(id);
|
|
52
52
|
if (cached) {
|
|
53
53
|
alreadyPresent.push({
|
|
54
54
|
requestId: id,
|
|
@@ -70,7 +70,7 @@ export class ApifyRequestQueueSharedBackend extends ApifyRequestQueueBackend {
|
|
|
70
70
|
// A forefront insert changes the head order — have the next head read re-fetch the
|
|
71
71
|
// front of the queue instead of draining the local buffer first.
|
|
72
72
|
if (forefront) {
|
|
73
|
-
this
|
|
73
|
+
this.#shouldCheckForefrontRequests = true;
|
|
74
74
|
}
|
|
75
75
|
}
|
|
76
76
|
result.processedRequests.push(...alreadyPresent);
|
|
@@ -83,9 +83,9 @@ export class ApifyRequestQueueSharedBackend extends ApifyRequestQueueBackend {
|
|
|
83
83
|
return this.getRequestById(this.requestIdFromUniqueKey(uniqueKey));
|
|
84
84
|
}
|
|
85
85
|
async fetchNextRequest() {
|
|
86
|
-
const id = await this
|
|
86
|
+
const id = await this.#headLock.runExclusive(async () => {
|
|
87
87
|
await this.ensureHeadIsNonEmpty();
|
|
88
|
-
return this
|
|
88
|
+
return this.#headIds.shift();
|
|
89
89
|
});
|
|
90
90
|
if (!id)
|
|
91
91
|
return undefined;
|
|
@@ -103,7 +103,7 @@ export class ApifyRequestQueueSharedBackend extends ApifyRequestQueueBackend {
|
|
|
103
103
|
this.cacheRequestInfo(id, { wasAlreadyHandled: true });
|
|
104
104
|
return undefined;
|
|
105
105
|
}
|
|
106
|
-
this
|
|
106
|
+
this.#inProgressIds.add(id);
|
|
107
107
|
return request;
|
|
108
108
|
}
|
|
109
109
|
async markRequestAsHandled(request) {
|
|
@@ -111,12 +111,12 @@ export class ApifyRequestQueueSharedBackend extends ApifyRequestQueueBackend {
|
|
|
111
111
|
// Contract: marking a request that does not exist in the queue is a no-op — it must not be
|
|
112
112
|
// added as a side effect (the platform update endpoint would upsert it).
|
|
113
113
|
if (!(await this.isKnownOrExists(id))) {
|
|
114
|
-
this
|
|
114
|
+
this.#inProgressIds.delete(id);
|
|
115
115
|
return undefined;
|
|
116
116
|
}
|
|
117
117
|
const handledAt = request.handledAt ?? new Date().toISOString();
|
|
118
118
|
const info = await this.updateRequestOnPlatform({ ...request, id, handledAt });
|
|
119
|
-
this
|
|
119
|
+
this.#inProgressIds.delete(id);
|
|
120
120
|
this.cacheRequestInfo(id, { wasAlreadyHandled: true });
|
|
121
121
|
if (!info.wasAlreadyHandled) {
|
|
122
122
|
this.estimatedHandledRequestCount += 1;
|
|
@@ -128,10 +128,10 @@ export class ApifyRequestQueueSharedBackend extends ApifyRequestQueueBackend {
|
|
|
128
128
|
const id = this.requestIdFromUniqueKey(request.uniqueKey);
|
|
129
129
|
// Same contract as `markRequestAsHandled` — never insert as a side effect.
|
|
130
130
|
if (!(await this.isKnownOrExists(id))) {
|
|
131
|
-
this
|
|
131
|
+
this.#inProgressIds.delete(id);
|
|
132
132
|
return undefined;
|
|
133
133
|
}
|
|
134
|
-
return this
|
|
134
|
+
return this.#headLock.runExclusive(async () => {
|
|
135
135
|
const info = await this.updateRequestOnPlatform({ ...request, id, handledAt: undefined }, forefront);
|
|
136
136
|
// Release the server-side lock so the request becomes fetchable again immediately —
|
|
137
137
|
// by any consumer — rather than only after the lock expires.
|
|
@@ -141,10 +141,10 @@ export class ApifyRequestQueueSharedBackend extends ApifyRequestQueueBackend {
|
|
|
141
141
|
catch (err) {
|
|
142
142
|
log.debug(`Failed to delete the lock of a reclaimed request (id: ${id}): ${err.message}`);
|
|
143
143
|
}
|
|
144
|
-
this
|
|
144
|
+
this.#inProgressIds.delete(id);
|
|
145
145
|
this.cacheRequestInfo(id, { wasAlreadyHandled: false });
|
|
146
146
|
if (forefront) {
|
|
147
|
-
this
|
|
147
|
+
this.#shouldCheckForefrontRequests = true;
|
|
148
148
|
}
|
|
149
149
|
if (info.wasAlreadyHandled) {
|
|
150
150
|
this.estimatedHandledRequestCount -= 1;
|
|
@@ -153,25 +153,25 @@ export class ApifyRequestQueueSharedBackend extends ApifyRequestQueueBackend {
|
|
|
153
153
|
});
|
|
154
154
|
}
|
|
155
155
|
async isEmpty() {
|
|
156
|
-
return this
|
|
157
|
-
if (this
|
|
156
|
+
return this.#headLock.runExclusive(async () => {
|
|
157
|
+
if (this.#headIds.length > 0)
|
|
158
158
|
return false;
|
|
159
159
|
await this.listAndLockHead(1);
|
|
160
|
-
return this
|
|
160
|
+
return this.#headIds.length === 0;
|
|
161
161
|
});
|
|
162
162
|
}
|
|
163
163
|
async isFinished() {
|
|
164
|
-
return this
|
|
165
|
-
if (this
|
|
164
|
+
return this.#headLock.runExclusive(async () => {
|
|
165
|
+
if (this.#headIds.length > 0)
|
|
166
166
|
return false;
|
|
167
167
|
// The head read also refreshes `queueHasLockedRequests`, so the order matters here.
|
|
168
168
|
await this.listAndLockHead(1);
|
|
169
|
-
return this
|
|
169
|
+
return this.#headIds.length === 0 && !this.#queueHasLockedRequests;
|
|
170
170
|
});
|
|
171
171
|
}
|
|
172
172
|
/** Must be called with the head lock held. */
|
|
173
173
|
async ensureHeadIsNonEmpty() {
|
|
174
|
-
if (this
|
|
174
|
+
if (this.#headIds.length > 1 && !this.#shouldCheckForefrontRequests) {
|
|
175
175
|
return;
|
|
176
176
|
}
|
|
177
177
|
await this.listAndLockHead(HEAD_LOCK_LIMIT);
|
|
@@ -181,31 +181,31 @@ export class ApifyRequestQueueSharedBackend extends ApifyRequestQueueBackend {
|
|
|
181
181
|
// After a forefront insert the local buffer no longer starts at the true front of the
|
|
182
182
|
// queue — re-fetch the front and keep the already-locked leftovers for afterwards.
|
|
183
183
|
let leftoverIds = [];
|
|
184
|
-
if (this
|
|
185
|
-
leftoverIds = this
|
|
186
|
-
this
|
|
184
|
+
if (this.#shouldCheckForefrontRequests) {
|
|
185
|
+
leftoverIds = this.#headIds.splice(0);
|
|
186
|
+
this.#shouldCheckForefrontRequests = false;
|
|
187
187
|
}
|
|
188
|
-
const head = await this.client.listAndLockHead({ limit, lockSecs: this
|
|
189
|
-
this
|
|
188
|
+
const head = await this.client.listAndLockHead({ limit, lockSecs: this.#lockSecs });
|
|
189
|
+
this.#queueHasLockedRequests = head.queueHasLockedRequests;
|
|
190
190
|
for (const item of head.items) {
|
|
191
|
-
if (this
|
|
191
|
+
if (this.#inProgressIds.has(item.id))
|
|
192
192
|
continue;
|
|
193
|
-
if (this
|
|
193
|
+
if (this.#headIds.includes(item.id) || leftoverIds.includes(item.id))
|
|
194
194
|
continue;
|
|
195
195
|
this.cacheRequestInfo(item.id, { wasAlreadyHandled: false });
|
|
196
|
-
this
|
|
196
|
+
this.#headIds.push(item.id);
|
|
197
197
|
}
|
|
198
|
-
this
|
|
198
|
+
this.#headIds.push(...leftoverIds);
|
|
199
199
|
}
|
|
200
200
|
async isKnownOrExists(id) {
|
|
201
|
-
if (this
|
|
201
|
+
if (this.#inProgressIds.has(id) || this.#cachedRequestInfo.get(id)) {
|
|
202
202
|
return true;
|
|
203
203
|
}
|
|
204
204
|
return (await this.getRequestById(id)) !== undefined;
|
|
205
205
|
}
|
|
206
206
|
cacheRequestInfo(id, info) {
|
|
207
207
|
// `LruCache.add` does not overwrite existing entries, so remove first.
|
|
208
|
-
this
|
|
209
|
-
this
|
|
208
|
+
this.#cachedRequestInfo.remove(id);
|
|
209
|
+
this.#cachedRequestInfo.add(id, info);
|
|
210
210
|
}
|
|
211
211
|
}
|
|
@@ -22,16 +22,7 @@ import { ApifyRequestQueueBackend } from './apify_request_queue_backend.js';
|
|
|
22
22
|
* @internal
|
|
23
23
|
*/
|
|
24
24
|
export declare class ApifyRequestQueueSingleBackend extends ApifyRequestQueueBackend {
|
|
25
|
-
|
|
26
|
-
private readonly headIds;
|
|
27
|
-
/** Unhandled full request objects added by (or fetched through) this client, keyed by id. */
|
|
28
|
-
private readonly cachedRequests;
|
|
29
|
-
/** Ids of requests known to be already handled — cheap dedup without caching full objects. */
|
|
30
|
-
private readonly handledIds;
|
|
31
|
-
/** Ids of requests currently being processed by this client. */
|
|
32
|
-
private readonly inProgressIds;
|
|
33
|
-
/** Memoized one-time prefetch of existing queue contents into the local caches. */
|
|
34
|
-
private initCachesPromise?;
|
|
25
|
+
#private;
|
|
35
26
|
addBatchOfRequests(requests: RequestSchema[], options?: RequestQueueOperationOptions): Promise<BatchAddRequestsResult>;
|
|
36
27
|
getRequest(uniqueKey: string): Promise<UpdateRequestSchema | undefined>;
|
|
37
28
|
fetchNextRequest(): Promise<UpdateRequestSchema | undefined>;
|
|
@@ -32,25 +32,25 @@ const INIT_CACHES_REQUEST_LIMIT = 10_000;
|
|
|
32
32
|
*/
|
|
33
33
|
export class ApifyRequestQueueSingleBackend extends ApifyRequestQueueBackend {
|
|
34
34
|
/** Local estimate of the queue head — request ids in the order they should be fetched. */
|
|
35
|
-
headIds = [];
|
|
35
|
+
#headIds = [];
|
|
36
36
|
/** Unhandled full request objects added by (or fetched through) this client, keyed by id. */
|
|
37
|
-
cachedRequests = new LruCache({ maxLength: MAX_CACHED_REQUESTS });
|
|
37
|
+
#cachedRequests = new LruCache({ maxLength: MAX_CACHED_REQUESTS });
|
|
38
38
|
/** Ids of requests known to be already handled — cheap dedup without caching full objects. */
|
|
39
|
-
handledIds = new Set();
|
|
39
|
+
#handledIds = new Set();
|
|
40
40
|
/** Ids of requests currently being processed by this client. */
|
|
41
|
-
inProgressIds = new Set();
|
|
41
|
+
#inProgressIds = new Set();
|
|
42
42
|
/** Memoized one-time prefetch of existing queue contents into the local caches. */
|
|
43
|
-
initCachesPromise;
|
|
43
|
+
#initCachesPromise;
|
|
44
44
|
async addBatchOfRequests(requests, options = {}) {
|
|
45
45
|
const { forefront = false } = options;
|
|
46
|
-
await (this
|
|
46
|
+
await (this.#initCachesPromise ??= this.initCaches());
|
|
47
47
|
// Split the batch into requests we already know about (dedup them locally — a platform
|
|
48
48
|
// write costs an API call and a paid write operation) and genuinely new ones.
|
|
49
49
|
const alreadyPresent = [];
|
|
50
50
|
const newRequests = [];
|
|
51
51
|
for (const request of requests) {
|
|
52
52
|
const id = this.requestIdFromUniqueKey(request.uniqueKey);
|
|
53
|
-
if (this
|
|
53
|
+
if (this.#handledIds.has(id)) {
|
|
54
54
|
alreadyPresent.push({
|
|
55
55
|
requestId: id,
|
|
56
56
|
uniqueKey: request.uniqueKey,
|
|
@@ -58,7 +58,7 @@ export class ApifyRequestQueueSingleBackend extends ApifyRequestQueueBackend {
|
|
|
58
58
|
wasAlreadyHandled: true,
|
|
59
59
|
});
|
|
60
60
|
}
|
|
61
|
-
else if (this
|
|
61
|
+
else if (this.#cachedRequests.get(id)) {
|
|
62
62
|
alreadyPresent.push({
|
|
63
63
|
requestId: id,
|
|
64
64
|
uniqueKey: request.uniqueKey,
|
|
@@ -83,15 +83,15 @@ export class ApifyRequestQueueSingleBackend extends ApifyRequestQueueBackend {
|
|
|
83
83
|
if (!processed)
|
|
84
84
|
continue; // rejected by the platform, reported in `unprocessedRequests`
|
|
85
85
|
if (processed.wasAlreadyHandled) {
|
|
86
|
-
this
|
|
86
|
+
this.#handledIds.add(processed.requestId);
|
|
87
87
|
continue;
|
|
88
88
|
}
|
|
89
89
|
this.cacheRequest({ ...request, id: processed.requestId });
|
|
90
90
|
if (forefront) {
|
|
91
|
-
this
|
|
91
|
+
this.#headIds.unshift(processed.requestId);
|
|
92
92
|
}
|
|
93
93
|
else {
|
|
94
|
-
this
|
|
94
|
+
this.#headIds.push(processed.requestId);
|
|
95
95
|
}
|
|
96
96
|
}
|
|
97
97
|
}
|
|
@@ -101,16 +101,16 @@ export class ApifyRequestQueueSingleBackend extends ApifyRequestQueueBackend {
|
|
|
101
101
|
}
|
|
102
102
|
async getRequest(uniqueKey) {
|
|
103
103
|
const id = this.requestIdFromUniqueKey(uniqueKey);
|
|
104
|
-
const cached = this
|
|
104
|
+
const cached = this.#cachedRequests.get(id);
|
|
105
105
|
if (cached)
|
|
106
106
|
return cached;
|
|
107
107
|
const request = await this.getRequestById(id);
|
|
108
108
|
if (!request)
|
|
109
109
|
return undefined;
|
|
110
110
|
// Requests already in progress are ones the client knows about — no caching needed.
|
|
111
|
-
if (!this
|
|
111
|
+
if (!this.#inProgressIds.has(id)) {
|
|
112
112
|
if (request.handledAt) {
|
|
113
|
-
this
|
|
113
|
+
this.#handledIds.add(id);
|
|
114
114
|
}
|
|
115
115
|
else {
|
|
116
116
|
this.cacheRequest(request);
|
|
@@ -120,24 +120,24 @@ export class ApifyRequestQueueSingleBackend extends ApifyRequestQueueBackend {
|
|
|
120
120
|
}
|
|
121
121
|
async fetchNextRequest() {
|
|
122
122
|
await this.ensureHeadIsNonEmpty();
|
|
123
|
-
while (this
|
|
124
|
-
const id = this
|
|
125
|
-
if (this
|
|
123
|
+
while (this.#headIds.length > 0) {
|
|
124
|
+
const id = this.#headIds.shift();
|
|
125
|
+
if (this.#inProgressIds.has(id) || this.#handledIds.has(id)) {
|
|
126
126
|
continue;
|
|
127
127
|
}
|
|
128
|
-
this
|
|
128
|
+
this.#inProgressIds.add(id);
|
|
129
129
|
// Requests added by this client are served straight from the cache; only requests
|
|
130
130
|
// discovered via `listHead` (added by another producer) need a round-trip.
|
|
131
|
-
const request = this
|
|
131
|
+
const request = this.#cachedRequests.get(id) ?? (await this.getRequestById(id));
|
|
132
132
|
if (!request) {
|
|
133
|
-
this
|
|
133
|
+
this.#inProgressIds.delete(id);
|
|
134
134
|
continue;
|
|
135
135
|
}
|
|
136
136
|
if (request.handledAt) {
|
|
137
137
|
// Handled elsewhere in the meantime — skip it and remember the outcome.
|
|
138
|
-
this
|
|
139
|
-
this
|
|
140
|
-
this
|
|
138
|
+
this.#inProgressIds.delete(id);
|
|
139
|
+
this.#handledIds.add(id);
|
|
140
|
+
this.#cachedRequests.remove(id);
|
|
141
141
|
continue;
|
|
142
142
|
}
|
|
143
143
|
return request;
|
|
@@ -149,14 +149,14 @@ export class ApifyRequestQueueSingleBackend extends ApifyRequestQueueBackend {
|
|
|
149
149
|
// Contract: marking a request that does not exist in the queue is a no-op — it must not be
|
|
150
150
|
// added as a side effect (the platform update endpoint would upsert it).
|
|
151
151
|
if (!(await this.isKnownOrExists(id))) {
|
|
152
|
-
this
|
|
152
|
+
this.#inProgressIds.delete(id);
|
|
153
153
|
return undefined;
|
|
154
154
|
}
|
|
155
155
|
const handledAt = request.handledAt ?? new Date().toISOString();
|
|
156
156
|
const info = await this.updateRequestOnPlatform({ ...request, id, handledAt });
|
|
157
|
-
this
|
|
158
|
-
this
|
|
159
|
-
this
|
|
157
|
+
this.#inProgressIds.delete(id);
|
|
158
|
+
this.#handledIds.add(id);
|
|
159
|
+
this.#cachedRequests.remove(id);
|
|
160
160
|
if (!info.wasAlreadyHandled) {
|
|
161
161
|
this.estimatedHandledRequestCount += 1;
|
|
162
162
|
}
|
|
@@ -167,24 +167,24 @@ export class ApifyRequestQueueSingleBackend extends ApifyRequestQueueBackend {
|
|
|
167
167
|
const id = this.requestIdFromUniqueKey(request.uniqueKey);
|
|
168
168
|
// Same contract as `markRequestAsHandled` — never insert as a side effect.
|
|
169
169
|
if (!(await this.isKnownOrExists(id))) {
|
|
170
|
-
this
|
|
170
|
+
this.#inProgressIds.delete(id);
|
|
171
171
|
return undefined;
|
|
172
172
|
}
|
|
173
173
|
// Reclaiming returns the request to the queue for reprocessing.
|
|
174
174
|
const reclaimed = { ...request, id, handledAt: undefined };
|
|
175
175
|
const info = await this.updateRequestOnPlatform(reclaimed, forefront);
|
|
176
|
-
this
|
|
177
|
-
this
|
|
176
|
+
this.#inProgressIds.delete(id);
|
|
177
|
+
this.#handledIds.delete(id);
|
|
178
178
|
this.cacheRequest(reclaimed);
|
|
179
179
|
// Return the id to the local head estimate right away — the platform head read can lag a
|
|
180
180
|
// few seconds behind the update, and `isFinished` must never report `true` while a
|
|
181
181
|
// reclaimed request is still waiting to be reprocessed.
|
|
182
|
-
if (!this
|
|
182
|
+
if (!this.#headIds.includes(id)) {
|
|
183
183
|
if (forefront) {
|
|
184
|
-
this
|
|
184
|
+
this.#headIds.unshift(id);
|
|
185
185
|
}
|
|
186
186
|
else {
|
|
187
|
-
this
|
|
187
|
+
this.#headIds.push(id);
|
|
188
188
|
}
|
|
189
189
|
}
|
|
190
190
|
if (info.wasAlreadyHandled) {
|
|
@@ -194,28 +194,28 @@ export class ApifyRequestQueueSingleBackend extends ApifyRequestQueueBackend {
|
|
|
194
194
|
}
|
|
195
195
|
async isEmpty() {
|
|
196
196
|
await this.ensureHeadIsNonEmpty();
|
|
197
|
-
return this
|
|
197
|
+
return this.#headIds.length === 0;
|
|
198
198
|
}
|
|
199
199
|
async isFinished() {
|
|
200
|
-
return (await this.isEmpty()) && this
|
|
200
|
+
return (await this.isEmpty()) && this.#inProgressIds.size === 0;
|
|
201
201
|
}
|
|
202
202
|
async ensureHeadIsNonEmpty() {
|
|
203
|
-
if (this
|
|
203
|
+
if (this.#headIds.length <= 1) {
|
|
204
204
|
await this.listHead();
|
|
205
205
|
}
|
|
206
206
|
}
|
|
207
207
|
async listHead() {
|
|
208
208
|
// The head read returns in-progress requests too, so fetch enough to find new ones.
|
|
209
|
-
const limit = Math.min(MAX_HEAD_ITEMS, DESIRED_NEW_HEAD_ITEMS + this
|
|
209
|
+
const limit = Math.min(MAX_HEAD_ITEMS, DESIRED_NEW_HEAD_ITEMS + this.#inProgressIds.size);
|
|
210
210
|
const head = await this.client.listHead({ limit });
|
|
211
211
|
for (const item of head.items) {
|
|
212
|
-
if (this
|
|
212
|
+
if (this.#inProgressIds.has(item.id) || this.#handledIds.has(item.id)) {
|
|
213
213
|
continue;
|
|
214
214
|
}
|
|
215
215
|
// `headIds` is nearly drained whenever this runs (see `ensureHeadIsNonEmpty`), so the
|
|
216
216
|
// linear dedup scan stays cheap.
|
|
217
|
-
if (!this
|
|
218
|
-
this
|
|
217
|
+
if (!this.#headIds.includes(item.id)) {
|
|
218
|
+
this.#headIds.push(item.id);
|
|
219
219
|
}
|
|
220
220
|
}
|
|
221
221
|
}
|
|
@@ -229,7 +229,7 @@ export class ApifyRequestQueueSingleBackend extends ApifyRequestQueueBackend {
|
|
|
229
229
|
const response = await this.client.listRequests({ limit: INIT_CACHES_REQUEST_LIMIT });
|
|
230
230
|
for (const request of response.items) {
|
|
231
231
|
if (request.handledAt) {
|
|
232
|
-
this
|
|
232
|
+
this.#handledIds.add(request.id);
|
|
233
233
|
}
|
|
234
234
|
else {
|
|
235
235
|
this.cacheRequest(request);
|
|
@@ -243,14 +243,14 @@ export class ApifyRequestQueueSingleBackend extends ApifyRequestQueueBackend {
|
|
|
243
243
|
}
|
|
244
244
|
}
|
|
245
245
|
async isKnownOrExists(id) {
|
|
246
|
-
if (this
|
|
246
|
+
if (this.#inProgressIds.has(id) || this.#handledIds.has(id) || this.#cachedRequests.get(id)) {
|
|
247
247
|
return true;
|
|
248
248
|
}
|
|
249
249
|
return (await this.getRequestById(id)) !== undefined;
|
|
250
250
|
}
|
|
251
251
|
cacheRequest(request) {
|
|
252
252
|
// `LruCache.add` does not overwrite existing entries, so remove first.
|
|
253
|
-
this
|
|
254
|
-
this
|
|
253
|
+
this.#cachedRequests.remove(request.id);
|
|
254
|
+
this.#cachedRequests.add(request.id, request);
|
|
255
255
|
}
|
|
256
256
|
}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { AsyncLocalStorage } from 'node:async_hooks';
|
|
2
2
|
import type { DatasetBackend, KeyValueStoreBackend, RequestQueueBackend, StorageBackend, StorageIdentifier } from '@crawlee/types';
|
|
3
3
|
import type { ApifyClient } from 'apify-client';
|
|
4
|
-
import type
|
|
4
|
+
import { type RequestQueueAccessMode } from './apify_request_queue_backend.js';
|
|
5
5
|
import { type ChargeResult, type ChargingManager } from './charging.js';
|
|
6
6
|
import type { Configuration } from './configuration.js';
|
|
7
7
|
type StorageType = 'Dataset' | 'KeyValueStore' | 'RequestQueue';
|
|
@@ -62,14 +62,7 @@ export interface ApifyStorageBackendOptions {
|
|
|
62
62
|
* ```
|
|
63
63
|
*/
|
|
64
64
|
export declare class ApifyStorageBackend implements StorageBackend {
|
|
65
|
-
private
|
|
66
|
-
private readonly config?;
|
|
67
|
-
private readonly requestQueueAccess;
|
|
68
|
-
private readonly getChargingManager?;
|
|
69
|
-
/** Unnamed storages created for aliases in this process, so an alias maps to one storage. */
|
|
70
|
-
private readonly aliasIdCache;
|
|
71
|
-
/** Fallback request queue client key when the run id is unavailable — one per backend. */
|
|
72
|
-
private fallbackClientKey?;
|
|
65
|
+
#private;
|
|
73
66
|
constructor(client: ApifyClient, options?: ApifyStorageBackendOptions);
|
|
74
67
|
/**
|
|
75
68
|
* Partitions crawlee's storage-instance cache by API base URL and token, so the same storage
|
|
@@ -78,6 +71,8 @@ export declare class ApifyStorageBackend implements StorageBackend {
|
|
|
78
71
|
* and `shared` mode at once is not supported, and whichever backend opens it first wins.
|
|
79
72
|
*/
|
|
80
73
|
getStorageBackendCacheKey(): string;
|
|
74
|
+
/** Short digest of the API base URL and token — identifies the credentials a storage was opened with. */
|
|
75
|
+
private credentialsHash;
|
|
81
76
|
storageExists(id: string, type: StorageType): Promise<boolean>;
|
|
82
77
|
createDatasetBackend(options?: StorageIdentifier): Promise<DatasetBackend>;
|
|
83
78
|
createKeyValueStoreBackend(options?: StorageIdentifier): Promise<KeyValueStoreBackend>;
|
|
@@ -97,11 +92,27 @@ export declare class ApifyStorageBackend implements StorageBackend {
|
|
|
97
92
|
* Resolves a crawlee {@link StorageIdentifier} to a platform storage id.
|
|
98
93
|
*
|
|
99
94
|
* Aliases resolve to unnamed storages: the reserved `__default__` alias maps to the run's
|
|
100
|
-
* default storage, and
|
|
101
|
-
* `ACTOR_STORAGES_JSON` environment variable
|
|
102
|
-
*
|
|
95
|
+
* default storage, and an alias declared in the Actor's schema to the storage the platform
|
|
96
|
+
* created for it (via the `ACTOR_STORAGES_JSON` environment variable). Any other alias gets an
|
|
97
|
+
* unnamed storage of its own — crawlee mints aliases at runtime, one per extra crawler instance
|
|
98
|
+
* and one per throttled domain, so an undeclared alias is not an error.
|
|
103
99
|
*/
|
|
104
100
|
private resolveId;
|
|
101
|
+
/**
|
|
102
|
+
* Returns the unnamed storage backing `alias`, creating it on first use.
|
|
103
|
+
*
|
|
104
|
+
* On the platform the mapping is persisted, so a migrated run reopens the same storages rather
|
|
105
|
+
* than empty ones — aliased request queues hold live requests. Serialized, so one alias means
|
|
106
|
+
* one storage and the mapping's read-modify-write cannot drop entries.
|
|
107
|
+
*/
|
|
108
|
+
private resolveAliasId;
|
|
109
|
+
/**
|
|
110
|
+
* Re-reads the record first, so a second backend in this process does not drop its entries.
|
|
111
|
+
* Logged rather than thrown: a lost entry only costs a re-created storage after a migration.
|
|
112
|
+
*/
|
|
113
|
+
private persistAliasId;
|
|
114
|
+
/** The run's default key-value store, where the mapping lives — `undefined` off the platform. */
|
|
115
|
+
private aliasMappingStore;
|
|
105
116
|
/** Looks an alias up in the Actor's schema storages (the `ACTOR_STORAGES_JSON` env var). */
|
|
106
117
|
private aliasFromActorStorages;
|
|
107
118
|
private resourceClient;
|