@crawlee/core 4.0.0-beta.105 → 4.0.0-beta.106
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/autoscaling/autoscaled_pool.d.ts +3 -21
- package/autoscaling/autoscaled_pool.js +85 -85
- package/autoscaling/client_load_signal.d.ts +1 -5
- package/autoscaling/client_load_signal.js +20 -20
- package/autoscaling/concurrency_system.d.ts +5 -20
- package/autoscaling/concurrency_system.js +81 -80
- package/autoscaling/cpu_load_signal.d.ts +1 -2
- package/autoscaling/cpu_load_signal.js +10 -10
- package/autoscaling/event_loop_load_signal.d.ts +1 -4
- package/autoscaling/event_loop_load_signal.js +18 -18
- package/autoscaling/load_signal.d.ts +1 -1
- package/autoscaling/load_signal.js +12 -11
- package/autoscaling/memory_load_signal.d.ts +3 -12
- package/autoscaling/memory_load_signal.js +40 -41
- package/autoscaling/snapshotter.d.ts +1 -4
- package/autoscaling/snapshotter.js +12 -12
- package/autoscaling/system_status.d.ts +1 -3
- package/autoscaling/system_status.js +11 -11
- package/configuration.d.ts +1 -1
- package/configuration.js +3 -3
- package/crawlers/context_pipeline.js +6 -6
- package/crawlers/statistics.d.ts +1 -8
- package/crawlers/statistics.js +45 -44
- package/events/event_manager.d.ts +1 -1
- package/events/event_manager.js +3 -3
- package/events/local_event_manager.d.ts +1 -1
- package/events/local_event_manager.js +3 -3
- package/log.js +5 -1
- package/memory-storage/memory-storage.d.ts +1 -5
- package/memory-storage/memory-storage.js +2 -2
- package/memory-storage/resource-clients/dataset.d.ts +1 -1
- package/memory-storage/resource-clients/dataset.js +6 -5
- package/memory-storage/resource-clients/key-value-store.d.ts +1 -1
- package/memory-storage/resource-clients/key-value-store.js +13 -12
- package/memory-storage/resource-clients/request-queue.d.ts +4 -23
- package/memory-storage/resource-clients/request-queue.js +59 -58
- package/owned_or_injected.d.ts +1 -3
- package/owned_or_injected.js +17 -17
- package/package.json +5 -5
- package/proxy_configuration.d.ts +1 -3
- package/proxy_configuration.js +8 -8
- package/recoverable_state.d.ts +1 -10
- package/recoverable_state.js +41 -41
- package/request.d.ts +1 -2
- package/request.js +10 -13
- package/router.d.ts +1 -4
- package/router.js +23 -23
- package/serialization.js +8 -9
- package/service_locator.d.ts +1 -10
- package/service_locator.js +48 -48
- package/session_pool/session.d.ts +1 -12
- package/session_pool/session.js +50 -50
- package/session_pool/session_pool.d.ts +2 -11
- package/session_pool/session_pool.js +59 -58
- package/storages/dataset.d.ts +1 -1
- package/storages/dataset.js +5 -5
- package/storages/key_value_store.d.ts +1 -4
- package/storages/key_value_store.js +21 -20
- package/storages/request_dedup_cache.d.ts +1 -2
- package/storages/request_dedup_cache.js +9 -9
- package/storages/request_list.d.ts +2 -22
- package/storages/request_list.js +74 -73
- package/storages/request_manager_tandem.d.ts +1 -10
- package/storages/request_manager_tandem.js +27 -27
- package/storages/request_queue.d.ts +2 -18
- package/storages/request_queue.js +37 -35
- package/storages/sitemap_request_loader.d.ts +1 -44
- package/storages/sitemap_request_loader.js +87 -87
- package/storages/storage_instance_manager.d.ts +1 -2
- package/storages/storage_instance_manager.js +17 -17
- package/storages/storage_stats.d.ts +1 -1
- package/storages/storage_stats.js +4 -4
- package/storages/transaction.d.ts +1 -3
- package/storages/transaction.js +17 -17
- package/system-info/runtime.js +7 -7
|
@@ -2,10 +2,10 @@ import { betterClearInterval, betterSetInterval } from '@apify/utilities';
|
|
|
2
2
|
import { serviceLocator } from '../service_locator.js';
|
|
3
3
|
import { EventManager } from './event_manager.js';
|
|
4
4
|
export class LocalEventManager extends EventManager {
|
|
5
|
-
systemInfoIntervalMillis;
|
|
5
|
+
#systemInfoIntervalMillis;
|
|
6
6
|
constructor(options) {
|
|
7
7
|
super(options);
|
|
8
|
-
this
|
|
8
|
+
this.#systemInfoIntervalMillis = options.systemInfoIntervalMillis;
|
|
9
9
|
}
|
|
10
10
|
/**
|
|
11
11
|
* Creates a new `LocalEventManager` based on the provided `Configuration`.
|
|
@@ -28,7 +28,7 @@ export class LocalEventManager extends EventManager {
|
|
|
28
28
|
}
|
|
29
29
|
await super.init();
|
|
30
30
|
this.emitSystemInfoEvent = this.emitSystemInfoEvent.bind(this);
|
|
31
|
-
this.intervals.systemInfo = betterSetInterval(this.emitSystemInfoEvent.bind(this), this
|
|
31
|
+
this.intervals.systemInfo = betterSetInterval(this.emitSystemInfoEvent.bind(this), this.#systemInfoIntervalMillis);
|
|
32
32
|
}
|
|
33
33
|
/**
|
|
34
34
|
* @inheritDoc
|
package/log.js
CHANGED
|
@@ -30,6 +30,8 @@ import log, { Log, Logger, LoggerJson, LoggerText, LogLevel } from '@apify/log';
|
|
|
30
30
|
* ```
|
|
31
31
|
*/
|
|
32
32
|
export class BaseCrawleeLogger {
|
|
33
|
+
// kept as TS-private: the adaptive crawler's log proxy calls non-intercepted methods with `this === proxy`,
|
|
34
|
+
// where `#`-field access would throw at runtime
|
|
33
35
|
options;
|
|
34
36
|
warningsLogged = new Set();
|
|
35
37
|
constructor(options = {}) {
|
|
@@ -87,7 +89,9 @@ export class BaseCrawleeLogger {
|
|
|
87
89
|
*/
|
|
88
90
|
export class ApifyLogAdapter extends BaseCrawleeLogger {
|
|
89
91
|
apifyLog;
|
|
90
|
-
constructor(
|
|
92
|
+
constructor(
|
|
93
|
+
// kept as a TS-private parameter property: reached through the adaptive crawler's log proxy, see above
|
|
94
|
+
apifyLog, options) {
|
|
91
95
|
super(options ?? {});
|
|
92
96
|
this.apifyLog = apifyLog;
|
|
93
97
|
}
|
|
@@ -10,12 +10,8 @@ export interface MemoryStorageOptions {
|
|
|
10
10
|
logger?: CrawleeLogger;
|
|
11
11
|
}
|
|
12
12
|
export declare class MemoryStorageBackend implements storage.StorageBackend {
|
|
13
|
+
#private;
|
|
13
14
|
readonly logger?: CrawleeLogger;
|
|
14
|
-
/**
|
|
15
|
-
* Unique per-instance cache partition key. Mirrors the way `FileSystemStorageBackend` partitions its
|
|
16
|
-
* cache by storage directory: two distinct `MemoryStorageBackend` instances must not share cached backends.
|
|
17
|
-
*/
|
|
18
|
-
private readonly instanceCacheKey;
|
|
19
15
|
readonly keyValueStoreBackendCache: KeyValueStoreBackend[];
|
|
20
16
|
readonly datasetBackendCache: DatasetBackend[];
|
|
21
17
|
readonly requestQueueBackendCache: RequestQueueBackend[];
|
|
@@ -8,7 +8,7 @@ export class MemoryStorageBackend {
|
|
|
8
8
|
* Unique per-instance cache partition key. Mirrors the way `FileSystemStorageBackend` partitions its
|
|
9
9
|
* cache by storage directory: two distinct `MemoryStorageBackend` instances must not share cached backends.
|
|
10
10
|
*/
|
|
11
|
-
instanceCacheKey = `MemoryStorageBackend:${randomUUID()}`;
|
|
11
|
+
#instanceCacheKey = `MemoryStorageBackend:${randomUUID()}`;
|
|
12
12
|
keyValueStoreBackendCache = [];
|
|
13
13
|
datasetBackendCache = [];
|
|
14
14
|
requestQueueBackendCache = [];
|
|
@@ -20,7 +20,7 @@ export class MemoryStorageBackend {
|
|
|
20
20
|
* cache partitions in the storage backend cache.
|
|
21
21
|
*/
|
|
22
22
|
getStorageBackendCacheKey() {
|
|
23
|
-
return this
|
|
23
|
+
return this.#instanceCacheKey;
|
|
24
24
|
}
|
|
25
25
|
static resolveStorageKey(options) {
|
|
26
26
|
const isAlias = 'alias' in options && !!options.alias;
|
|
@@ -14,6 +14,7 @@ export interface DatasetBackendOptions {
|
|
|
14
14
|
storageBackend: MemoryStorageBackend;
|
|
15
15
|
}
|
|
16
16
|
export declare class DatasetBackend<Data extends Dictionary = Dictionary> extends BaseClient implements storage.DatasetBackend<Data> {
|
|
17
|
+
#private;
|
|
17
18
|
name?: string;
|
|
18
19
|
/**
|
|
19
20
|
* The key used for cache lookup. For named storages, this equals the name. For alias (unnamed)
|
|
@@ -24,7 +25,6 @@ export declare class DatasetBackend<Data extends Dictionary = Dictionary> extend
|
|
|
24
25
|
accessedAt: Date;
|
|
25
26
|
modifiedAt: Date;
|
|
26
27
|
itemCount: number;
|
|
27
|
-
private readonly datasetEntries;
|
|
28
28
|
private readonly storageBackend;
|
|
29
29
|
constructor(options: DatasetBackendOptions);
|
|
30
30
|
getMetadata(): Promise<storage.DatasetInfo>;
|
|
@@ -22,7 +22,8 @@ export class DatasetBackend extends BaseClient {
|
|
|
22
22
|
accessedAt = new Date();
|
|
23
23
|
modifiedAt = new Date();
|
|
24
24
|
itemCount = 0;
|
|
25
|
-
datasetEntries = new Map();
|
|
25
|
+
#datasetEntries = new Map();
|
|
26
|
+
// kept as TS-private: storage-backend tests read this field at runtime
|
|
26
27
|
storageBackend;
|
|
27
28
|
constructor(options) {
|
|
28
29
|
super(options.id ?? randomUUID());
|
|
@@ -39,12 +40,12 @@ export class DatasetBackend extends BaseClient {
|
|
|
39
40
|
if (storeIndex !== -1) {
|
|
40
41
|
const [oldBackend] = this.storageBackend.datasetBackendCache.splice(storeIndex, 1);
|
|
41
42
|
oldBackend.itemCount = 0;
|
|
42
|
-
oldBackend
|
|
43
|
+
oldBackend.#datasetEntries.clear();
|
|
43
44
|
}
|
|
44
45
|
}
|
|
45
46
|
async purge() {
|
|
46
47
|
this.itemCount = 0;
|
|
47
|
-
this
|
|
48
|
+
this.#datasetEntries.clear();
|
|
48
49
|
this.updateTimestamps(true);
|
|
49
50
|
}
|
|
50
51
|
getData(options = {}) {
|
|
@@ -67,7 +68,7 @@ export class DatasetBackend extends BaseClient {
|
|
|
67
68
|
const items = [];
|
|
68
69
|
for (let idx = start; idx < end; idx++) {
|
|
69
70
|
const entryNumber = this.generateLocalEntryName(idx);
|
|
70
|
-
items.push(this
|
|
71
|
+
items.push(this.#datasetEntries.get(entryNumber));
|
|
71
72
|
}
|
|
72
73
|
this.updateTimestamps(false);
|
|
73
74
|
return {
|
|
@@ -82,7 +83,7 @@ export class DatasetBackend extends BaseClient {
|
|
|
82
83
|
async pushData(items) {
|
|
83
84
|
for (const entry of items) {
|
|
84
85
|
const idx = this.generateLocalEntryName(++this.itemCount);
|
|
85
|
-
this
|
|
86
|
+
this.#datasetEntries.set(idx, JSON.parse(JSON.stringify(entry)));
|
|
86
87
|
}
|
|
87
88
|
this.updateTimestamps(true);
|
|
88
89
|
}
|
|
@@ -19,6 +19,7 @@ export interface InternalKeyRecord {
|
|
|
19
19
|
extension: string;
|
|
20
20
|
}
|
|
21
21
|
export declare class KeyValueStoreBackend extends BaseClient implements storage.KeyValueStoreBackend {
|
|
22
|
+
#private;
|
|
22
23
|
name?: string;
|
|
23
24
|
/**
|
|
24
25
|
* The key used for cache lookup. For named storages, this equals the name. For alias (unnamed)
|
|
@@ -28,7 +29,6 @@ export declare class KeyValueStoreBackend extends BaseClient implements storage.
|
|
|
28
29
|
createdAt: Date;
|
|
29
30
|
accessedAt: Date;
|
|
30
31
|
modifiedAt: Date;
|
|
31
|
-
private readonly keyValueEntries;
|
|
32
32
|
private readonly storageBackend;
|
|
33
33
|
constructor(options: KeyValueStoreBackendOptions);
|
|
34
34
|
getMetadata(): Promise<storage.KeyValueStoreInfo>;
|
|
@@ -19,7 +19,8 @@ export class KeyValueStoreBackend extends BaseClient {
|
|
|
19
19
|
createdAt = new Date();
|
|
20
20
|
accessedAt = new Date();
|
|
21
21
|
modifiedAt = new Date();
|
|
22
|
-
keyValueEntries = new Map();
|
|
22
|
+
#keyValueEntries = new Map();
|
|
23
|
+
// kept as TS-private: storage-backend tests read this field at runtime
|
|
23
24
|
storageBackend;
|
|
24
25
|
constructor(options) {
|
|
25
26
|
super(options.id ?? randomUUID());
|
|
@@ -35,11 +36,11 @@ export class KeyValueStoreBackend extends BaseClient {
|
|
|
35
36
|
const storeIndex = this.storageBackend.keyValueStoreBackendCache.findIndex((store) => store.id === this.id);
|
|
36
37
|
if (storeIndex !== -1) {
|
|
37
38
|
const [oldBackend] = this.storageBackend.keyValueStoreBackendCache.splice(storeIndex, 1);
|
|
38
|
-
oldBackend
|
|
39
|
+
oldBackend.#keyValueEntries.clear();
|
|
39
40
|
}
|
|
40
41
|
}
|
|
41
42
|
async purge() {
|
|
42
|
-
this
|
|
43
|
+
this.#keyValueEntries.clear();
|
|
43
44
|
this.updateTimestamps(true);
|
|
44
45
|
}
|
|
45
46
|
/**
|
|
@@ -49,9 +50,9 @@ export class KeyValueStoreBackend extends BaseClient {
|
|
|
49
50
|
* preserve the bare `INPUT` key only.
|
|
50
51
|
*/
|
|
51
52
|
async purgeExceptInput() {
|
|
52
|
-
for (const key of this
|
|
53
|
+
for (const key of this.#keyValueEntries.keys()) {
|
|
53
54
|
if (key !== KEY_VALUE_STORE_INPUT_KEY) {
|
|
54
|
-
this
|
|
55
|
+
this.#keyValueEntries.delete(key);
|
|
55
56
|
}
|
|
56
57
|
}
|
|
57
58
|
this.updateTimestamps(true);
|
|
@@ -65,7 +66,7 @@ export class KeyValueStoreBackend extends BaseClient {
|
|
|
65
66
|
})
|
|
66
67
|
.parse(options);
|
|
67
68
|
const items = [];
|
|
68
|
-
for (const record of this
|
|
69
|
+
for (const record of this.#keyValueEntries.values()) {
|
|
69
70
|
const size = Buffer.byteLength(record.value);
|
|
70
71
|
items.push({
|
|
71
72
|
key: record.key,
|
|
@@ -114,11 +115,11 @@ export class KeyValueStoreBackend extends BaseClient {
|
|
|
114
115
|
*/
|
|
115
116
|
async recordExists(key) {
|
|
116
117
|
s.string().parse(key);
|
|
117
|
-
return this
|
|
118
|
+
return this.#keyValueEntries.has(key);
|
|
118
119
|
}
|
|
119
120
|
async getValue(key) {
|
|
120
121
|
s.string().parse(key);
|
|
121
|
-
const entry = this
|
|
122
|
+
const entry = this.#keyValueEntries.get(key);
|
|
122
123
|
if (!entry) {
|
|
123
124
|
return undefined;
|
|
124
125
|
}
|
|
@@ -169,19 +170,19 @@ export class KeyValueStoreBackend extends BaseClient {
|
|
|
169
170
|
const normalizedValue = typeof value === 'string'
|
|
170
171
|
? Buffer.from(value, 'utf-8')
|
|
171
172
|
: toBuffer(value);
|
|
172
|
-
const
|
|
173
|
+
const normalizedRecord = {
|
|
173
174
|
extension,
|
|
174
175
|
key,
|
|
175
176
|
value: normalizedValue,
|
|
176
177
|
contentType,
|
|
177
178
|
};
|
|
178
|
-
this
|
|
179
|
+
this.#keyValueEntries.set(key, normalizedRecord);
|
|
179
180
|
this.updateTimestamps(true);
|
|
180
181
|
}
|
|
181
182
|
async deleteValue(key) {
|
|
182
183
|
s.string().parse(key);
|
|
183
|
-
if (this
|
|
184
|
-
this
|
|
184
|
+
if (this.#keyValueEntries.has(key)) {
|
|
185
|
+
this.#keyValueEntries.delete(key);
|
|
185
186
|
this.updateTimestamps(true);
|
|
186
187
|
}
|
|
187
188
|
}
|
|
@@ -22,6 +22,7 @@ export interface InternalRequest {
|
|
|
22
22
|
json: string;
|
|
23
23
|
}
|
|
24
24
|
export declare class RequestQueueBackend extends BaseClient implements storage.RequestQueueBackend {
|
|
25
|
+
#private;
|
|
25
26
|
name?: string;
|
|
26
27
|
/**
|
|
27
28
|
* The key used for cache lookup. For named storages, this equals the name. For alias (unnamed)
|
|
@@ -33,26 +34,6 @@ export declare class RequestQueueBackend extends BaseClient implements storage.R
|
|
|
33
34
|
modifiedAt: Date;
|
|
34
35
|
handledRequestCount: number;
|
|
35
36
|
pendingRequestCount: number;
|
|
36
|
-
/**
|
|
37
|
-
* Serializes every operation that reads-then-writes this backend's shared queue state — the
|
|
38
|
-
* `requests` map, the `forefrontRequestIds` array, the `inProgressRequestIds` set and the request
|
|
39
|
-
* counts. Those mutations span `await` points, so without this mutex a concurrent operation could
|
|
40
|
-
* interleave and corrupt them (e.g. a head scan pruning `forefrontRequestIds` while
|
|
41
|
-
* `addBatchOfRequests` pushes to it). Held by every mutating method as well as by `isEmpty`/
|
|
42
|
-
* `isFinished`, whose head scan also prunes `forefrontRequestIds`.
|
|
43
|
-
*/
|
|
44
|
-
private readonly queueStateMutex;
|
|
45
|
-
private forefrontRequestIds;
|
|
46
|
-
/**
|
|
47
|
-
* IDs of requests currently fetched but not yet handled or reclaimed. A request in this set is
|
|
48
|
-
* "in progress" and will not be handed out again by {@link fetchNextRequest}.
|
|
49
|
-
*
|
|
50
|
-
* Unlike the file-system / platform clients, the in-memory queue lives entirely within a single
|
|
51
|
-
* process and is never shared with another consumer, so there is no need for an expiring,
|
|
52
|
-
* cross-process-visible lock — tracking in-progress requests in this set is enough.
|
|
53
|
-
*/
|
|
54
|
-
private readonly inProgressRequestIds;
|
|
55
|
-
private readonly requests;
|
|
56
37
|
private readonly storageBackend;
|
|
57
38
|
constructor(options: RequestQueueBackendOptions);
|
|
58
39
|
getMetadata(): Promise<storage.RequestQueueInfo>;
|
|
@@ -90,7 +71,7 @@ export declare class RequestQueueBackend extends BaseClient implements storage.R
|
|
|
90
71
|
listItems(): Promise<storage.UpdateRequestSchema[]>;
|
|
91
72
|
toRequestQueueInfo(): storage.RequestQueueInfo;
|
|
92
73
|
private updateTimestamps;
|
|
93
|
-
private
|
|
94
|
-
private
|
|
95
|
-
private
|
|
74
|
+
private jsonToRequest;
|
|
75
|
+
private createInternalRequest;
|
|
76
|
+
private calculateOrderNo;
|
|
96
77
|
}
|
|
@@ -38,8 +38,8 @@ export class RequestQueueBackend extends BaseClient {
|
|
|
38
38
|
* `addBatchOfRequests` pushes to it). Held by every mutating method as well as by `isEmpty`/
|
|
39
39
|
* `isFinished`, whose head scan also prunes `forefrontRequestIds`.
|
|
40
40
|
*/
|
|
41
|
-
queueStateMutex = new AsyncQueue();
|
|
42
|
-
forefrontRequestIds = [];
|
|
41
|
+
#queueStateMutex = new AsyncQueue();
|
|
42
|
+
#forefrontRequestIds = [];
|
|
43
43
|
/**
|
|
44
44
|
* IDs of requests currently fetched but not yet handled or reclaimed. A request in this set is
|
|
45
45
|
* "in progress" and will not be handed out again by {@link fetchNextRequest}.
|
|
@@ -48,8 +48,9 @@ export class RequestQueueBackend extends BaseClient {
|
|
|
48
48
|
* process and is never shared with another consumer, so there is no need for an expiring,
|
|
49
49
|
* cross-process-visible lock — tracking in-progress requests in this set is enough.
|
|
50
50
|
*/
|
|
51
|
-
inProgressRequestIds = new Set();
|
|
52
|
-
requests = new Map();
|
|
51
|
+
#inProgressRequestIds = new Set();
|
|
52
|
+
#requests = new Map();
|
|
53
|
+
// kept as TS-private: storage-backend tests read this field at runtime
|
|
53
54
|
storageBackend;
|
|
54
55
|
constructor(options) {
|
|
55
56
|
super(options.id ?? randomUUID());
|
|
@@ -65,7 +66,7 @@ export class RequestQueueBackend extends BaseClient {
|
|
|
65
66
|
// Serialize against other mutators (and the head scans in `isEmpty`/`isFinished`) so a concurrent
|
|
66
67
|
// operation cannot observe half-cleared state — e.g. a forefront id whose request has already been
|
|
67
68
|
// removed, which `listPendingHead` would then dereference as `undefined`.
|
|
68
|
-
await this
|
|
69
|
+
await this.#queueStateMutex.wait();
|
|
69
70
|
try {
|
|
70
71
|
const storeIndex = this.storageBackend.requestQueueBackendCache.findIndex((queue) => queue.id === this.id);
|
|
71
72
|
if (storeIndex !== -1) {
|
|
@@ -74,37 +75,37 @@ export class RequestQueueBackend extends BaseClient {
|
|
|
74
75
|
// Clear all in-memory state, consistent with `purge`. Clearing `requests` alone would
|
|
75
76
|
// leave dangling ids in `forefrontRequestIds`/`inProgressRequestIds`, which a later head
|
|
76
77
|
// scan would resolve to a missing request and dereference.
|
|
77
|
-
oldBackend
|
|
78
|
-
oldBackend
|
|
79
|
-
oldBackend
|
|
78
|
+
oldBackend.#requests.clear();
|
|
79
|
+
oldBackend.#forefrontRequestIds = [];
|
|
80
|
+
oldBackend.#inProgressRequestIds.clear();
|
|
80
81
|
}
|
|
81
82
|
}
|
|
82
83
|
finally {
|
|
83
|
-
this
|
|
84
|
+
this.#queueStateMutex.shift();
|
|
84
85
|
}
|
|
85
86
|
}
|
|
86
87
|
async purge() {
|
|
87
88
|
// Serialize against other mutators (and the head scans in `isEmpty`/`isFinished`) so a concurrent
|
|
88
89
|
// operation cannot observe or repopulate half-cleared state across the `await` below.
|
|
89
|
-
await this
|
|
90
|
+
await this.#queueStateMutex.wait();
|
|
90
91
|
try {
|
|
91
92
|
// Clear all in-memory state
|
|
92
|
-
this
|
|
93
|
-
this
|
|
94
|
-
this
|
|
93
|
+
this.#requests.clear();
|
|
94
|
+
this.#forefrontRequestIds = [];
|
|
95
|
+
this.#inProgressRequestIds.clear();
|
|
95
96
|
this.handledRequestCount = 0;
|
|
96
97
|
this.pendingRequestCount = 0;
|
|
97
98
|
this.updateTimestamps(true);
|
|
98
99
|
}
|
|
99
100
|
finally {
|
|
100
|
-
this
|
|
101
|
+
this.#queueStateMutex.shift();
|
|
101
102
|
}
|
|
102
103
|
}
|
|
103
104
|
*requestKeyIterator() {
|
|
104
|
-
for (let i = this
|
|
105
|
-
yield this
|
|
105
|
+
for (let i = this.#forefrontRequestIds.length - 1; i >= 0; i--) {
|
|
106
|
+
yield this.#forefrontRequestIds[i];
|
|
106
107
|
}
|
|
107
|
-
for (const key of this
|
|
108
|
+
for (const key of this.#requests.keys()) {
|
|
108
109
|
yield key;
|
|
109
110
|
}
|
|
110
111
|
}
|
|
@@ -140,17 +141,17 @@ export class RequestQueueBackend extends BaseClient {
|
|
|
140
141
|
continue;
|
|
141
142
|
}
|
|
142
143
|
seenRequestIds.add(requestId);
|
|
143
|
-
const request = this
|
|
144
|
+
const request = this.#requests.get(requestId);
|
|
144
145
|
// Permanently-handled requests (`orderNo === null`) are in a terminal state and can be skipped.
|
|
145
146
|
if (request.orderNo === null) {
|
|
146
|
-
if (this
|
|
147
|
+
if (this.#forefrontRequestIds.includes(requestId)) {
|
|
147
148
|
handledForefrontIds.add(requestId);
|
|
148
149
|
}
|
|
149
150
|
continue;
|
|
150
151
|
}
|
|
151
152
|
// In progress (fetched but not yet handled or reclaimed) — skip it, but remember that the
|
|
152
153
|
// queue is not truly empty.
|
|
153
|
-
if (this
|
|
154
|
+
if (this.#inProgressRequestIds.has(requestId)) {
|
|
154
155
|
hasInProgressRequests = true;
|
|
155
156
|
continue;
|
|
156
157
|
}
|
|
@@ -158,7 +159,7 @@ export class RequestQueueBackend extends BaseClient {
|
|
|
158
159
|
items.push(request);
|
|
159
160
|
}
|
|
160
161
|
}
|
|
161
|
-
this
|
|
162
|
+
this.#forefrontRequestIds = this.#forefrontRequestIds.filter((id) => !handledForefrontIds.has(id));
|
|
162
163
|
return {
|
|
163
164
|
items: items.sort((a, b) => a.orderNo - b.orderNo),
|
|
164
165
|
hasInProgressRequests: detectInProgressRequests ? hasInProgressRequests : undefined,
|
|
@@ -166,7 +167,7 @@ export class RequestQueueBackend extends BaseClient {
|
|
|
166
167
|
}
|
|
167
168
|
async fetchNextRequest() {
|
|
168
169
|
this.updateTimestamps(false);
|
|
169
|
-
await this
|
|
170
|
+
await this.#queueStateMutex.wait();
|
|
170
171
|
try {
|
|
171
172
|
const { items: [head], } = await this.listPendingHead(1);
|
|
172
173
|
if (!head) {
|
|
@@ -174,11 +175,11 @@ export class RequestQueueBackend extends BaseClient {
|
|
|
174
175
|
}
|
|
175
176
|
// Mark the request as in progress so it is not handed out again until it is handled or
|
|
176
177
|
// reclaimed. The request keeps its `orderNo` (and thus its forefront / normal ordering).
|
|
177
|
-
this
|
|
178
|
-
return this.
|
|
178
|
+
this.#inProgressRequestIds.add(head.id);
|
|
179
|
+
return this.jsonToRequest(head.json) ?? undefined;
|
|
179
180
|
}
|
|
180
181
|
finally {
|
|
181
|
-
this
|
|
182
|
+
this.#queueStateMutex.shift();
|
|
182
183
|
}
|
|
183
184
|
}
|
|
184
185
|
async addBatchOfRequests(requests, options = {}) {
|
|
@@ -187,15 +188,15 @@ export class RequestQueueBackend extends BaseClient {
|
|
|
187
188
|
// Serialize against other mutators (and the head scans in `isEmpty`/`isFinished`) so that the
|
|
188
189
|
// shared `requests` map, `forefrontRequestIds` array and request counts are not corrupted by a
|
|
189
190
|
// concurrent operation interleaving at one of the `await` points below.
|
|
190
|
-
await this
|
|
191
|
+
await this.#queueStateMutex.wait();
|
|
191
192
|
try {
|
|
192
193
|
const result = {
|
|
193
194
|
processedRequests: [],
|
|
194
195
|
unprocessedRequests: [],
|
|
195
196
|
};
|
|
196
197
|
for (const model of requests) {
|
|
197
|
-
const requestModel = this.
|
|
198
|
-
const existingRequestWithId = this
|
|
198
|
+
const requestModel = this.createInternalRequest(model, options.forefront);
|
|
199
|
+
const existingRequestWithId = this.#requests.get(requestModel.id);
|
|
199
200
|
if (existingRequestWithId) {
|
|
200
201
|
result.processedRequests.push({
|
|
201
202
|
requestId: existingRequestWithId.id,
|
|
@@ -205,7 +206,7 @@ export class RequestQueueBackend extends BaseClient {
|
|
|
205
206
|
});
|
|
206
207
|
continue;
|
|
207
208
|
}
|
|
208
|
-
this
|
|
209
|
+
this.#requests.set(requestModel.id, requestModel);
|
|
209
210
|
if (requestModel.orderNo) {
|
|
210
211
|
this.pendingRequestCount += 1;
|
|
211
212
|
}
|
|
@@ -213,7 +214,7 @@ export class RequestQueueBackend extends BaseClient {
|
|
|
213
214
|
this.handledRequestCount += 1;
|
|
214
215
|
}
|
|
215
216
|
if (options.forefront) {
|
|
216
|
-
this
|
|
217
|
+
this.#forefrontRequestIds.push(requestModel.id);
|
|
217
218
|
}
|
|
218
219
|
result.processedRequests.push({
|
|
219
220
|
requestId: requestModel.id,
|
|
@@ -228,15 +229,15 @@ export class RequestQueueBackend extends BaseClient {
|
|
|
228
229
|
return result;
|
|
229
230
|
}
|
|
230
231
|
finally {
|
|
231
|
-
this
|
|
232
|
+
this.#queueStateMutex.shift();
|
|
232
233
|
}
|
|
233
234
|
}
|
|
234
235
|
async getRequest(uniqueKey) {
|
|
235
236
|
s.string().parse(uniqueKey);
|
|
236
237
|
this.updateTimestamps(false);
|
|
237
238
|
const id = uniqueKeyToRequestId(uniqueKey);
|
|
238
|
-
const json = this
|
|
239
|
-
return this.
|
|
239
|
+
const json = this.#requests.get(id)?.json;
|
|
240
|
+
return this.jsonToRequest(json);
|
|
240
241
|
}
|
|
241
242
|
async markRequestAsHandled(request) {
|
|
242
243
|
requestShape.parse(request);
|
|
@@ -244,10 +245,10 @@ export class RequestQueueBackend extends BaseClient {
|
|
|
244
245
|
// Serialize against other mutators (and the head scans in `isEmpty`/`isFinished`) so the shared
|
|
245
246
|
// `requests` map, `inProgressRequestIds` set and request counts stay consistent across the
|
|
246
247
|
// `await` points below.
|
|
247
|
-
await this
|
|
248
|
+
await this.#queueStateMutex.wait();
|
|
248
249
|
try {
|
|
249
250
|
const id = uniqueKeyToRequestId(request.uniqueKey);
|
|
250
|
-
const existingRequest = this
|
|
251
|
+
const existingRequest = this.#requests.get(id);
|
|
251
252
|
// The request must exist to be marked as handled. We intentionally do NOT require it to still
|
|
252
253
|
// be in progress: marking an already-released request handled must still succeed, otherwise
|
|
253
254
|
// the request could be handed out again and the queue would never finish.
|
|
@@ -257,10 +258,10 @@ export class RequestQueueBackend extends BaseClient {
|
|
|
257
258
|
// A handled request has `orderNo === null`. Marking it again is an idempotent no-op.
|
|
258
259
|
const wasAlreadyHandled = existingRequest.orderNo === null;
|
|
259
260
|
const handledAt = request.handledAt ?? new Date().toISOString();
|
|
260
|
-
const requestModel = this.
|
|
261
|
-
this
|
|
261
|
+
const requestModel = this.createInternalRequest({ ...request, handledAt }, false);
|
|
262
|
+
this.#requests.set(id, requestModel);
|
|
262
263
|
// The request is no longer in progress for this client.
|
|
263
|
-
this
|
|
264
|
+
this.#inProgressRequestIds.delete(id);
|
|
264
265
|
if (!wasAlreadyHandled) {
|
|
265
266
|
this.pendingRequestCount -= 1;
|
|
266
267
|
this.handledRequestCount += 1;
|
|
@@ -273,7 +274,7 @@ export class RequestQueueBackend extends BaseClient {
|
|
|
273
274
|
};
|
|
274
275
|
}
|
|
275
276
|
finally {
|
|
276
|
-
this
|
|
277
|
+
this.#queueStateMutex.shift();
|
|
277
278
|
}
|
|
278
279
|
}
|
|
279
280
|
async reclaimRequest(request, options = {}) {
|
|
@@ -283,10 +284,10 @@ export class RequestQueueBackend extends BaseClient {
|
|
|
283
284
|
// Serialize against other mutators (and the head scans in `isEmpty`/`isFinished`) so the shared
|
|
284
285
|
// `requests` map, `forefrontRequestIds` array and `inProgressRequestIds` set stay consistent
|
|
285
286
|
// across the `await` points below.
|
|
286
|
-
await this
|
|
287
|
+
await this.#queueStateMutex.wait();
|
|
287
288
|
try {
|
|
288
289
|
const id = uniqueKeyToRequestId(request.uniqueKey);
|
|
289
|
-
const existingRequest = this
|
|
290
|
+
const existingRequest = this.#requests.get(id);
|
|
290
291
|
// The request must exist and not already be handled to be reclaimed. As with
|
|
291
292
|
// `markRequestAsHandled`, we do NOT require it to still be in progress — returning an
|
|
292
293
|
// already-released request to the queue (e.g. to honor a `forefront` reorder) must still
|
|
@@ -296,12 +297,12 @@ export class RequestQueueBackend extends BaseClient {
|
|
|
296
297
|
}
|
|
297
298
|
// Reclaiming resets the `orderNo` to a fresh timestamp, restoring the request to the queue
|
|
298
299
|
// (at the front if `forefront`).
|
|
299
|
-
const requestModel = this.
|
|
300
|
-
this
|
|
300
|
+
const requestModel = this.createInternalRequest(request, options.forefront);
|
|
301
|
+
this.#requests.set(id, requestModel);
|
|
301
302
|
// The request is no longer in progress for this client.
|
|
302
|
-
this
|
|
303
|
+
this.#inProgressRequestIds.delete(id);
|
|
303
304
|
if (options.forefront) {
|
|
304
|
-
this
|
|
305
|
+
this.#forefrontRequestIds.push(id);
|
|
305
306
|
}
|
|
306
307
|
this.updateTimestamps(true);
|
|
307
308
|
return {
|
|
@@ -311,7 +312,7 @@ export class RequestQueueBackend extends BaseClient {
|
|
|
311
312
|
};
|
|
312
313
|
}
|
|
313
314
|
finally {
|
|
314
|
-
this
|
|
315
|
+
this.#queueStateMutex.shift();
|
|
315
316
|
}
|
|
316
317
|
}
|
|
317
318
|
async isEmpty() {
|
|
@@ -324,13 +325,13 @@ export class RequestQueueBackend extends BaseClient {
|
|
|
324
325
|
//
|
|
325
326
|
// `listPendingHead` prunes `forefrontRequestIds` as it scans, so we must hold the queue-state mutex to avoid
|
|
326
327
|
// racing a concurrent mutator (e.g. `addBatchOfRequests`) at its `await` points.
|
|
327
|
-
await this
|
|
328
|
+
await this.#queueStateMutex.wait();
|
|
328
329
|
try {
|
|
329
330
|
const { items } = await this.listPendingHead(1);
|
|
330
331
|
return items.length === 0;
|
|
331
332
|
}
|
|
332
333
|
finally {
|
|
333
|
-
this
|
|
334
|
+
this.#queueStateMutex.shift();
|
|
334
335
|
}
|
|
335
336
|
}
|
|
336
337
|
async isFinished() {
|
|
@@ -344,13 +345,13 @@ export class RequestQueueBackend extends BaseClient {
|
|
|
344
345
|
//
|
|
345
346
|
// `listPendingHead` prunes `forefrontRequestIds` as it scans, so we must hold the queue-state mutex to avoid
|
|
346
347
|
// racing a concurrent mutator (e.g. `addBatchOfRequests`) at its `await` points.
|
|
347
|
-
await this
|
|
348
|
+
await this.#queueStateMutex.wait();
|
|
348
349
|
try {
|
|
349
350
|
const { items, hasInProgressRequests } = await this.listPendingHead(1, true);
|
|
350
351
|
return items.length === 0 && !hasInProgressRequests;
|
|
351
352
|
}
|
|
352
353
|
finally {
|
|
353
|
-
this
|
|
354
|
+
this.#queueStateMutex.shift();
|
|
354
355
|
}
|
|
355
356
|
}
|
|
356
357
|
/**
|
|
@@ -362,13 +363,13 @@ export class RequestQueueBackend extends BaseClient {
|
|
|
362
363
|
this.updateTimestamps(false);
|
|
363
364
|
// `listPendingHead` prunes `forefrontRequestIds` as it scans, so we must hold the queue-state
|
|
364
365
|
// mutex to avoid racing a concurrent mutator at its `await` points.
|
|
365
|
-
await this
|
|
366
|
+
await this.#queueStateMutex.wait();
|
|
366
367
|
try {
|
|
367
368
|
const { items } = await this.listPendingHead(Number.POSITIVE_INFINITY);
|
|
368
|
-
return items.map((request) => this.
|
|
369
|
+
return items.map((request) => this.jsonToRequest(request.json));
|
|
369
370
|
}
|
|
370
371
|
finally {
|
|
371
|
-
this
|
|
372
|
+
this.#queueStateMutex.shift();
|
|
372
373
|
}
|
|
373
374
|
}
|
|
374
375
|
toRequestQueueInfo() {
|
|
@@ -380,7 +381,7 @@ export class RequestQueueBackend extends BaseClient {
|
|
|
380
381
|
modifiedAt: this.modifiedAt,
|
|
381
382
|
name: this.name,
|
|
382
383
|
pendingRequestCount: this.pendingRequestCount,
|
|
383
|
-
totalRequestCount: this
|
|
384
|
+
totalRequestCount: this.#requests.size,
|
|
384
385
|
};
|
|
385
386
|
}
|
|
386
387
|
updateTimestamps(hasBeenModified) {
|
|
@@ -389,14 +390,14 @@ export class RequestQueueBackend extends BaseClient {
|
|
|
389
390
|
this.modifiedAt = new Date();
|
|
390
391
|
}
|
|
391
392
|
}
|
|
392
|
-
|
|
393
|
+
jsonToRequest(requestJson) {
|
|
393
394
|
if (!requestJson)
|
|
394
395
|
return undefined;
|
|
395
396
|
const request = JSON.parse(requestJson);
|
|
396
397
|
return purgeNullsFromObject(request);
|
|
397
398
|
}
|
|
398
|
-
|
|
399
|
-
const orderNo = this.
|
|
399
|
+
createInternalRequest(request, forefront) {
|
|
400
|
+
const orderNo = this.calculateOrderNo(request, forefront);
|
|
400
401
|
const id = uniqueKeyToRequestId(request.uniqueKey);
|
|
401
402
|
if (request.id && request.id !== id) {
|
|
402
403
|
throw new Error('Request ID does not match its uniqueKey.');
|
|
@@ -412,7 +413,7 @@ export class RequestQueueBackend extends BaseClient {
|
|
|
412
413
|
url: request.url,
|
|
413
414
|
};
|
|
414
415
|
}
|
|
415
|
-
|
|
416
|
+
calculateOrderNo(request, forefront) {
|
|
416
417
|
if (request.handledAt)
|
|
417
418
|
return null;
|
|
418
419
|
const timestamp = Date.now();
|
package/owned_or_injected.d.ts
CHANGED
|
@@ -10,9 +10,7 @@
|
|
|
10
10
|
* @internal
|
|
11
11
|
*/
|
|
12
12
|
export declare class OwnedOrInjected<Injected, Owned extends Injected = Injected> {
|
|
13
|
-
private
|
|
14
|
-
private readonly _owned;
|
|
15
|
-
private _present;
|
|
13
|
+
#private;
|
|
16
14
|
private constructor();
|
|
17
15
|
/**
|
|
18
16
|
* Resolves a collaborator from an optionally-injected instance. `Injected` is the public/borrowed type exposed via
|