apify 4.0.0-beta.50 → 4.0.0-beta.52
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -28,8 +28,6 @@ export class ApifyRequestQueueSharedBackend extends ApifyRequestQueueBackend {
|
|
|
28
28
|
#inProgressIds = new Set();
|
|
29
29
|
/** Whether the last head read reported any locked requests left in the queue (any client's). */
|
|
30
30
|
#queueHasLockedRequests;
|
|
31
|
-
/** Set after a forefront insert — the next head read starts fresh so the insert is honored. */
|
|
32
|
-
#shouldCheckForefrontRequests = false;
|
|
33
31
|
/** Lock duration applied to fetched requests; raised via `setExpectedRequestProcessingTimeSecs`. */
|
|
34
32
|
#lockSecs = DEFAULT_REQUEST_LOCK_SECS;
|
|
35
33
|
/** Serializes head reads and reclaims — both reorder the shared head state. */
|
|
@@ -67,11 +65,6 @@ export class ApifyRequestQueueSharedBackend extends ApifyRequestQueueBackend {
|
|
|
67
65
|
for (const processed of result.processedRequests) {
|
|
68
66
|
this.cacheRequestInfo(processed.requestId, { wasAlreadyHandled: processed.wasAlreadyHandled });
|
|
69
67
|
}
|
|
70
|
-
// A forefront insert changes the head order — have the next head read re-fetch the
|
|
71
|
-
// front of the queue instead of draining the local buffer first.
|
|
72
|
-
if (forefront) {
|
|
73
|
-
this.#shouldCheckForefrontRequests = true;
|
|
74
|
-
}
|
|
75
68
|
}
|
|
76
69
|
result.processedRequests.push(...alreadyPresent);
|
|
77
70
|
this.recordAddedRequests(result);
|
|
@@ -143,9 +136,6 @@ export class ApifyRequestQueueSharedBackend extends ApifyRequestQueueBackend {
|
|
|
143
136
|
}
|
|
144
137
|
this.#inProgressIds.delete(id);
|
|
145
138
|
this.cacheRequestInfo(id, { wasAlreadyHandled: false });
|
|
146
|
-
if (forefront) {
|
|
147
|
-
this.#shouldCheckForefrontRequests = true;
|
|
148
|
-
}
|
|
149
139
|
if (info.wasAlreadyHandled) {
|
|
150
140
|
this.estimatedHandledRequestCount -= 1;
|
|
151
141
|
}
|
|
@@ -171,31 +161,23 @@ export class ApifyRequestQueueSharedBackend extends ApifyRequestQueueBackend {
|
|
|
171
161
|
}
|
|
172
162
|
/** Must be called with the head lock held. */
|
|
173
163
|
async ensureHeadIsNonEmpty() {
|
|
174
|
-
if (this.#headIds.length > 1
|
|
164
|
+
if (this.#headIds.length > 1) {
|
|
175
165
|
return;
|
|
176
166
|
}
|
|
177
167
|
await this.listAndLockHead(HEAD_LOCK_LIMIT);
|
|
178
168
|
}
|
|
179
169
|
/** Must be called with the head lock held. */
|
|
180
170
|
async listAndLockHead(limit) {
|
|
181
|
-
// After a forefront insert the local buffer no longer starts at the true front of the
|
|
182
|
-
// queue — re-fetch the front and keep the already-locked leftovers for afterwards.
|
|
183
|
-
let leftoverIds = [];
|
|
184
|
-
if (this.#shouldCheckForefrontRequests) {
|
|
185
|
-
leftoverIds = this.#headIds.splice(0);
|
|
186
|
-
this.#shouldCheckForefrontRequests = false;
|
|
187
|
-
}
|
|
188
171
|
const head = await this.client.listAndLockHead({ limit, lockSecs: this.#lockSecs });
|
|
189
172
|
this.#queueHasLockedRequests = head.queueHasLockedRequests;
|
|
190
173
|
for (const item of head.items) {
|
|
191
174
|
if (this.#inProgressIds.has(item.id))
|
|
192
175
|
continue;
|
|
193
|
-
if (this.#headIds.includes(item.id)
|
|
176
|
+
if (this.#headIds.includes(item.id))
|
|
194
177
|
continue;
|
|
195
178
|
this.cacheRequestInfo(item.id, { wasAlreadyHandled: false });
|
|
196
179
|
this.#headIds.push(item.id);
|
|
197
180
|
}
|
|
198
|
-
this.#headIds.push(...leftoverIds);
|
|
199
181
|
}
|
|
200
182
|
async isKnownOrExists(id) {
|
|
201
183
|
if (this.#inProgressIds.has(id) || this.#cachedRequestInfo.get(id)) {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "apify",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.52",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.13.0"
|
|
@@ -97,7 +97,7 @@
|
|
|
97
97
|
"devEngines": {
|
|
98
98
|
"packageManager": {
|
|
99
99
|
"name": "pnpm",
|
|
100
|
-
"version": "12.
|
|
100
|
+
"version": "12.10.1",
|
|
101
101
|
"onFail": "warn"
|
|
102
102
|
}
|
|
103
103
|
},
|