@crawlee/core 4.0.0-beta.86 → 4.0.0-beta.87
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/autoscaling/cpu_load_signal.d.ts +1 -1
- package/autoscaling/memory_load_signal.d.ts +2 -2
- package/autoscaling/memory_load_signal.js +5 -5
- package/autoscaling/snapshotter.d.ts +2 -2
- package/autoscaling/snapshotter.js +6 -6
- package/configuration.d.ts +3 -3
- package/configuration.js +3 -3
- package/crawlers/crawler_commons.d.ts +2 -2
- package/crawlers/crawler_commons.js +7 -7
- package/crawlers/statistics.js +1 -1
- package/events/local_event_manager.d.ts +1 -1
- package/events/local_event_manager.js +4 -4
- package/package.json +5 -5
- package/recoverable_state.d.ts +1 -1
- package/recoverable_state.js +3 -1
- package/service_locator.d.ts +2 -2
- package/service_locator.js +6 -6
- package/session_pool/session_pool.js +1 -1
- package/storages/dataset.d.ts +2 -2
- package/storages/dataset.js +7 -7
- package/storages/key_value_store.d.ts +2 -2
- package/storages/key_value_store.js +7 -7
- package/storages/request_list.d.ts +1 -1
- package/storages/request_queue.js +3 -3
- package/storages/sitemap_request_loader.js +0 -1
- package/storages/utils.d.ts +5 -5
- package/storages/utils.js +8 -8
|
@@ -12,7 +12,7 @@ export interface CpuSnapshot extends LoadSnapshot {
|
|
|
12
12
|
export interface CpuLoadSignalOptions {
|
|
13
13
|
overloadedRatio?: number;
|
|
14
14
|
snapshotHistoryMillis?: number;
|
|
15
|
-
|
|
15
|
+
configuration: Configuration;
|
|
16
16
|
}
|
|
17
17
|
/**
|
|
18
18
|
* Tracks CPU usage via `SYSTEM_INFO` events and reports overload when
|
|
@@ -9,7 +9,7 @@ export interface MemoryLoadSignalOptions {
|
|
|
9
9
|
maxUsedMemoryRatio?: number;
|
|
10
10
|
overloadedRatio?: number;
|
|
11
11
|
snapshotHistoryMillis?: number;
|
|
12
|
-
|
|
12
|
+
configuration: Configuration;
|
|
13
13
|
log?: CrawleeLogger;
|
|
14
14
|
}
|
|
15
15
|
/**
|
|
@@ -20,7 +20,7 @@ export declare class MemoryLoadSignal implements LoadSignal {
|
|
|
20
20
|
readonly name = "memInfo";
|
|
21
21
|
readonly overloadedRatio: number;
|
|
22
22
|
private readonly store;
|
|
23
|
-
private readonly
|
|
23
|
+
private readonly configuration;
|
|
24
24
|
private readonly events;
|
|
25
25
|
private readonly log;
|
|
26
26
|
private readonly maxUsedMemoryRatio;
|
|
@@ -12,7 +12,7 @@ export class MemoryLoadSignal {
|
|
|
12
12
|
name = 'memInfo';
|
|
13
13
|
overloadedRatio;
|
|
14
14
|
store;
|
|
15
|
-
|
|
15
|
+
configuration;
|
|
16
16
|
events;
|
|
17
17
|
log;
|
|
18
18
|
maxUsedMemoryRatio;
|
|
@@ -21,7 +21,7 @@ export class MemoryLoadSignal {
|
|
|
21
21
|
lastLoggedCriticalMemoryOverloadAt = null;
|
|
22
22
|
constructor(options) {
|
|
23
23
|
this.store = new SnapshotStore(options.snapshotHistoryMillis);
|
|
24
|
-
this.
|
|
24
|
+
this.configuration = options.configuration;
|
|
25
25
|
this.events = serviceLocator.getEventManager();
|
|
26
26
|
this.log = options.log ?? serviceLocator.getLogger().child({ prefix: 'MemoryLoadSignal' });
|
|
27
27
|
this.maxUsedMemoryRatio = options.maxUsedMemoryRatio ?? 0.9;
|
|
@@ -29,12 +29,12 @@ export class MemoryLoadSignal {
|
|
|
29
29
|
this._onSystemInfo = this._onSystemInfo.bind(this);
|
|
30
30
|
}
|
|
31
31
|
async start() {
|
|
32
|
-
const memoryMbytes = this.
|
|
32
|
+
const memoryMbytes = this.configuration.memoryMbytes ?? 0;
|
|
33
33
|
if (memoryMbytes > 0) {
|
|
34
34
|
this.maxMemoryBytes = memoryMbytes * 1024 * 1024;
|
|
35
35
|
}
|
|
36
36
|
else {
|
|
37
|
-
this.maxMemoryRatio = this.
|
|
37
|
+
this.maxMemoryRatio = this.configuration.availableMemoryRatio;
|
|
38
38
|
if (!this.maxMemoryRatio) {
|
|
39
39
|
throw new Error('availableMemoryRatio is not set in configuration.');
|
|
40
40
|
}
|
|
@@ -96,7 +96,7 @@ export class MemoryLoadSignal {
|
|
|
96
96
|
}
|
|
97
97
|
}
|
|
98
98
|
async _getTotalMemoryBytes() {
|
|
99
|
-
const containerized = this.
|
|
99
|
+
const containerized = this.configuration.containerized ?? (await isContainerized());
|
|
100
100
|
return (await getMemoryInfo({ containerized, logger: serviceLocator.getLogger() })).totalBytes;
|
|
101
101
|
}
|
|
102
102
|
}
|
|
@@ -47,7 +47,7 @@ export interface SnapshotterOptions {
|
|
|
47
47
|
/** @internal */
|
|
48
48
|
client?: StorageBackend;
|
|
49
49
|
/** @internal */
|
|
50
|
-
|
|
50
|
+
configuration?: Configuration;
|
|
51
51
|
}
|
|
52
52
|
/**
|
|
53
53
|
* Creates snapshots of system resources at given intervals and marks the resource
|
|
@@ -78,7 +78,7 @@ export interface SnapshotterOptions {
|
|
|
78
78
|
export declare class Snapshotter {
|
|
79
79
|
readonly log: CrawleeLogger;
|
|
80
80
|
readonly client: StorageBackend;
|
|
81
|
-
readonly
|
|
81
|
+
readonly configuration: Configuration;
|
|
82
82
|
private readonly memorySignal;
|
|
83
83
|
private readonly eventLoopSignal;
|
|
84
84
|
private readonly cpuSignal;
|
|
@@ -33,7 +33,7 @@ import { MemoryLoadSignal } from './memory_load_signal.js';
|
|
|
33
33
|
export class Snapshotter {
|
|
34
34
|
log;
|
|
35
35
|
client;
|
|
36
|
-
|
|
36
|
+
configuration;
|
|
37
37
|
memorySignal;
|
|
38
38
|
eventLoopSignal;
|
|
39
39
|
cpuSignal;
|
|
@@ -71,17 +71,17 @@ export class Snapshotter {
|
|
|
71
71
|
maxClientErrors: ow.optional.number,
|
|
72
72
|
log: ow.optional.object,
|
|
73
73
|
client: ow.optional.object,
|
|
74
|
-
|
|
74
|
+
configuration: ow.optional.object,
|
|
75
75
|
}));
|
|
76
|
-
const { eventLoopSnapshotIntervalSecs = 0.5, clientSnapshotIntervalSecs = 1, snapshotHistorySecs = 30, maxBlockedMillis = 50, maxUsedMemoryRatio = 0.9, maxClientErrors = 3, log = serviceLocator.getLogger(),
|
|
76
|
+
const { eventLoopSnapshotIntervalSecs = 0.5, clientSnapshotIntervalSecs = 1, snapshotHistorySecs = 30, maxBlockedMillis = 50, maxUsedMemoryRatio = 0.9, maxClientErrors = 3, log = serviceLocator.getLogger(), configuration = serviceLocator.getConfiguration(), client = serviceLocator.getStorageBackend(), } = options;
|
|
77
77
|
this.log = log.child({ prefix: 'Snapshotter' });
|
|
78
78
|
this.client = client;
|
|
79
|
-
this.
|
|
79
|
+
this.configuration = configuration;
|
|
80
80
|
const snapshotHistoryMillis = snapshotHistorySecs * 1000;
|
|
81
81
|
this.memorySignal = new MemoryLoadSignal({
|
|
82
82
|
maxUsedMemoryRatio,
|
|
83
83
|
snapshotHistoryMillis,
|
|
84
|
-
|
|
84
|
+
configuration: this.configuration,
|
|
85
85
|
log: this.log,
|
|
86
86
|
});
|
|
87
87
|
this.eventLoopSignal = createEventLoopLoadSignal({
|
|
@@ -91,7 +91,7 @@ export class Snapshotter {
|
|
|
91
91
|
});
|
|
92
92
|
this.cpuSignal = createCpuLoadSignal({
|
|
93
93
|
snapshotHistoryMillis,
|
|
94
|
-
|
|
94
|
+
configuration: this.configuration,
|
|
95
95
|
});
|
|
96
96
|
this.clientSignal = createClientLoadSignal({
|
|
97
97
|
client: this.client,
|
package/configuration.d.ts
CHANGED
|
@@ -57,7 +57,7 @@ export interface Configuration extends ResolvedConfigValues {
|
|
|
57
57
|
}
|
|
58
58
|
/**
|
|
59
59
|
* `Configuration` is a value object holding Crawlee configuration. By default, there is a
|
|
60
|
-
* global singleton instance of this class available via `Configuration.
|
|
60
|
+
* global singleton instance of this class available via `Configuration.getGlobalConfiguration()`.
|
|
61
61
|
* Places that depend on a configurable behaviour depend on this class, as they have the global
|
|
62
62
|
* instance as the default value.
|
|
63
63
|
*
|
|
@@ -66,7 +66,7 @@ export interface Configuration extends ResolvedConfigValues {
|
|
|
66
66
|
* import { BasicCrawler, Configuration } from 'crawlee';
|
|
67
67
|
*
|
|
68
68
|
* // Get the global configuration
|
|
69
|
-
* const config = Configuration.
|
|
69
|
+
* const config = Configuration.getGlobalConfiguration();
|
|
70
70
|
* // Access configuration values directly as properties
|
|
71
71
|
* console.log(config.headless);
|
|
72
72
|
* console.log(config.persistStateIntervalMillis);
|
|
@@ -134,7 +134,7 @@ export declare class Configuration {
|
|
|
134
134
|
*
|
|
135
135
|
* Delegates to the global ServiceLocator, making it the single source of truth for service management.
|
|
136
136
|
*/
|
|
137
|
-
static
|
|
137
|
+
static getGlobalConfiguration(): Configuration;
|
|
138
138
|
/**
|
|
139
139
|
* Resolves all field values once using the priority chain:
|
|
140
140
|
* constructor options > env vars > crawlee.json > schema defaults.
|
package/configuration.js
CHANGED
|
@@ -75,7 +75,7 @@ export const crawleeConfigFields = {
|
|
|
75
75
|
};
|
|
76
76
|
/**
|
|
77
77
|
* `Configuration` is a value object holding Crawlee configuration. By default, there is a
|
|
78
|
-
* global singleton instance of this class available via `Configuration.
|
|
78
|
+
* global singleton instance of this class available via `Configuration.getGlobalConfiguration()`.
|
|
79
79
|
* Places that depend on a configurable behaviour depend on this class, as they have the global
|
|
80
80
|
* instance as the default value.
|
|
81
81
|
*
|
|
@@ -84,7 +84,7 @@ export const crawleeConfigFields = {
|
|
|
84
84
|
* import { BasicCrawler, Configuration } from 'crawlee';
|
|
85
85
|
*
|
|
86
86
|
* // Get the global configuration
|
|
87
|
-
* const config = Configuration.
|
|
87
|
+
* const config = Configuration.getGlobalConfiguration();
|
|
88
88
|
* // Access configuration values directly as properties
|
|
89
89
|
* console.log(config.headless);
|
|
90
90
|
* console.log(config.persistStateIntervalMillis);
|
|
@@ -163,7 +163,7 @@ export class Configuration {
|
|
|
163
163
|
*
|
|
164
164
|
* Delegates to the global ServiceLocator, making it the single source of truth for service management.
|
|
165
165
|
*/
|
|
166
|
-
static
|
|
166
|
+
static getGlobalConfiguration() {
|
|
167
167
|
return serviceLocator.getConfiguration();
|
|
168
168
|
}
|
|
169
169
|
/**
|
|
@@ -191,12 +191,12 @@ export interface CrawlingContext<UserData extends Dictionary = Dictionary> exten
|
|
|
191
191
|
* @experimental
|
|
192
192
|
*/
|
|
193
193
|
export declare class RequestHandlerResult {
|
|
194
|
-
private
|
|
194
|
+
private configuration;
|
|
195
195
|
private crawleeStateKey;
|
|
196
196
|
private _keyValueStoreChanges;
|
|
197
197
|
private pushDataCalls;
|
|
198
198
|
private addRequestsCalls;
|
|
199
|
-
constructor(
|
|
199
|
+
constructor(configuration: Configuration, crawleeStateKey: string);
|
|
200
200
|
/**
|
|
201
201
|
* A record of calls to {@link RestrictedCrawlingContext.pushData}, {@link RestrictedCrawlingContext.addRequests}, {@link RestrictedCrawlingContext.enqueueLinks} made by a request handler.
|
|
202
202
|
*/
|
|
@@ -5,13 +5,13 @@ import { KeyValueStore } from '../storages/key_value_store.js';
|
|
|
5
5
|
* @experimental
|
|
6
6
|
*/
|
|
7
7
|
export class RequestHandlerResult {
|
|
8
|
-
|
|
8
|
+
configuration;
|
|
9
9
|
crawleeStateKey;
|
|
10
10
|
_keyValueStoreChanges = {};
|
|
11
11
|
pushDataCalls = [];
|
|
12
12
|
addRequestsCalls = [];
|
|
13
|
-
constructor(
|
|
14
|
-
this.
|
|
13
|
+
constructor(configuration, crawleeStateKey) {
|
|
14
|
+
this.configuration = configuration;
|
|
15
15
|
this.crawleeStateKey = crawleeStateKey;
|
|
16
16
|
}
|
|
17
17
|
/**
|
|
@@ -81,10 +81,10 @@ export class RequestHandlerResult {
|
|
|
81
81
|
return await store.getAutoSavedValue(this.crawleeStateKey, defaultValue);
|
|
82
82
|
};
|
|
83
83
|
getKeyValueStore = async (identifier) => {
|
|
84
|
-
const store = await KeyValueStore.open(identifier, {
|
|
84
|
+
const store = await KeyValueStore.open(identifier, { configuration: this.configuration });
|
|
85
85
|
const storeId = store.id;
|
|
86
86
|
return {
|
|
87
|
-
id: storeId ?? this.
|
|
87
|
+
id: storeId ?? this.configuration.defaultKeyValueStoreId,
|
|
88
88
|
name: store.name,
|
|
89
89
|
getValue: async (key) => this.getKeyValueStoreChangedValue(storeId, key) ?? (await store.getValue(key)),
|
|
90
90
|
setValue: async (key, value, options) => {
|
|
@@ -95,12 +95,12 @@ export class RequestHandlerResult {
|
|
|
95
95
|
};
|
|
96
96
|
};
|
|
97
97
|
getKeyValueStoreChangedValue = (storeKey, key) => {
|
|
98
|
-
const id = storeKey ?? this.
|
|
98
|
+
const id = storeKey ?? this.configuration.defaultKeyValueStoreId;
|
|
99
99
|
this._keyValueStoreChanges[id] ??= {};
|
|
100
100
|
return this.keyValueStoreChanges[id][key]?.changedValue ?? null;
|
|
101
101
|
};
|
|
102
102
|
setKeyValueStoreChangedValue = (storeKey, key, changedValue, options) => {
|
|
103
|
-
const id = storeKey ?? this.
|
|
103
|
+
const id = storeKey ?? this.configuration.defaultKeyValueStoreId;
|
|
104
104
|
this._keyValueStoreChanges[id] ??= {};
|
|
105
105
|
this._keyValueStoreChanges[id][key] = { changedValue, options };
|
|
106
106
|
};
|
package/crawlers/statistics.js
CHANGED
|
@@ -225,7 +225,7 @@ export class Statistics {
|
|
|
225
225
|
* displaying the current state in predefined intervals
|
|
226
226
|
*/
|
|
227
227
|
async startCapturing() {
|
|
228
|
-
this.keyValueStore ??= await KeyValueStore.open(null, {
|
|
228
|
+
this.keyValueStore ??= await KeyValueStore.open(null, { configuration: serviceLocator.getConfiguration() });
|
|
229
229
|
if (this.state.crawlerStartedAt === null) {
|
|
230
230
|
this.state.crawlerStartedAt = new Date();
|
|
231
231
|
}
|
|
@@ -11,7 +11,7 @@ export declare class LocalEventManager extends EventManager {
|
|
|
11
11
|
* Creates a new `LocalEventManager` based on the provided `Configuration`.
|
|
12
12
|
* Uses the global configuration from the service locator if none is provided.
|
|
13
13
|
*/
|
|
14
|
-
static
|
|
14
|
+
static fromConfiguration(configuration?: Configuration): LocalEventManager;
|
|
15
15
|
/**
|
|
16
16
|
* Initializes the EventManager and sets up periodic `systemInfo` events.
|
|
17
17
|
* This is automatically called at the beginning of `crawler.run()`.
|
|
@@ -11,11 +11,11 @@ export class LocalEventManager extends EventManager {
|
|
|
11
11
|
* Creates a new `LocalEventManager` based on the provided `Configuration`.
|
|
12
12
|
* Uses the global configuration from the service locator if none is provided.
|
|
13
13
|
*/
|
|
14
|
-
static
|
|
15
|
-
const
|
|
14
|
+
static fromConfiguration(configuration) {
|
|
15
|
+
const resolvedConfiguration = configuration ?? serviceLocator.getConfiguration();
|
|
16
16
|
return new LocalEventManager({
|
|
17
|
-
persistStateIntervalMillis:
|
|
18
|
-
systemInfoIntervalMillis:
|
|
17
|
+
persistStateIntervalMillis: resolvedConfiguration.persistStateIntervalMillis,
|
|
18
|
+
systemInfoIntervalMillis: resolvedConfiguration.systemInfoIntervalMillis,
|
|
19
19
|
});
|
|
20
20
|
}
|
|
21
21
|
/**
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/core",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.87",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.0.0"
|
|
@@ -53,9 +53,9 @@
|
|
|
53
53
|
"@apify/pseudo_url": "^2.0.59",
|
|
54
54
|
"@apify/timeout": "^0.3.2",
|
|
55
55
|
"@apify/utilities": "^2.15.5",
|
|
56
|
-
"@crawlee/fs-storage": "4.0.0-beta.
|
|
57
|
-
"@crawlee/types": "4.0.0-beta.
|
|
58
|
-
"@crawlee/utils": "4.0.0-beta.
|
|
56
|
+
"@crawlee/fs-storage": "4.0.0-beta.87",
|
|
57
|
+
"@crawlee/types": "4.0.0-beta.87",
|
|
58
|
+
"@crawlee/utils": "4.0.0-beta.87",
|
|
59
59
|
"@sapphire/async-queue": "^1.5.5",
|
|
60
60
|
"@sapphire/shapeshift": "^4.0.0",
|
|
61
61
|
"@vladfrangu/async_event_emitter": "^2.4.6",
|
|
@@ -79,5 +79,5 @@
|
|
|
79
79
|
}
|
|
80
80
|
}
|
|
81
81
|
},
|
|
82
|
-
"gitHead": "
|
|
82
|
+
"gitHead": "1b7604cd77b694460aac4448b4555444fa1b2bdb"
|
|
83
83
|
}
|
package/recoverable_state.d.ts
CHANGED
|
@@ -35,7 +35,7 @@ export interface RecoverableStateOptions<TStateModel = Record<string, unknown>>
|
|
|
35
35
|
/**
|
|
36
36
|
* Configuration instance to use
|
|
37
37
|
*/
|
|
38
|
-
|
|
38
|
+
configuration?: Configuration;
|
|
39
39
|
/**
|
|
40
40
|
* Optional function to transform the state to a JSON string before persistence.
|
|
41
41
|
* If not provided, JSON.stringify will be used.
|
package/recoverable_state.js
CHANGED
|
@@ -59,7 +59,9 @@ export class RecoverableState {
|
|
|
59
59
|
else if (this.persistStateKvsId) {
|
|
60
60
|
kvsIdentifier = { id: this.persistStateKvsId };
|
|
61
61
|
}
|
|
62
|
-
this.keyValueStore = await KeyValueStore.open(kvsIdentifier, {
|
|
62
|
+
this.keyValueStore = await KeyValueStore.open(kvsIdentifier, {
|
|
63
|
+
configuration: serviceLocator.getConfiguration(),
|
|
64
|
+
});
|
|
63
65
|
await this.loadSavedState();
|
|
64
66
|
// Register for persist state events
|
|
65
67
|
const eventManager = serviceLocator.getEventManager();
|
package/service_locator.d.ts
CHANGED
|
@@ -93,9 +93,9 @@ interface ServiceLocatorInterface {
|
|
|
93
93
|
*
|
|
94
94
|
* const crawler = new BasicCrawler({
|
|
95
95
|
* requestHandler: async ({ request }) => { ... },
|
|
96
|
-
* configuration: new Configuration({ ... }),
|
|
96
|
+
* configuration: new Configuration({ ... }), // custom configuration
|
|
97
97
|
* storageBackend: new MemoryStorageBackend(), // custom storage
|
|
98
|
-
* eventManager: LocalEventManager.
|
|
98
|
+
* eventManager: LocalEventManager.fromConfiguration(), // custom events
|
|
99
99
|
* });
|
|
100
100
|
* // Crawler has its own isolated ServiceLocator instance
|
|
101
101
|
* ```
|
package/service_locator.js
CHANGED
|
@@ -31,9 +31,9 @@ import { StorageInstanceManager } from './storages/storage_instance_manager.js';
|
|
|
31
31
|
*
|
|
32
32
|
* const crawler = new BasicCrawler({
|
|
33
33
|
* requestHandler: async ({ request }) => { ... },
|
|
34
|
-
* configuration: new Configuration({ ... }),
|
|
34
|
+
* configuration: new Configuration({ ... }), // custom configuration
|
|
35
35
|
* storageBackend: new MemoryStorageBackend(), // custom storage
|
|
36
|
-
* eventManager: LocalEventManager.
|
|
36
|
+
* eventManager: LocalEventManager.fromConfiguration(), // custom events
|
|
37
37
|
* });
|
|
38
38
|
* // Crawler has its own isolated ServiceLocator instance
|
|
39
39
|
* ```
|
|
@@ -88,7 +88,7 @@ export class ServiceLocator {
|
|
|
88
88
|
this.getLogger().warning('Implicit creation of event manager will implicitly set configuration as side effect. ' +
|
|
89
89
|
'It is advised to explicitly first set the configuration instead.');
|
|
90
90
|
}
|
|
91
|
-
this.eventManager = LocalEventManager.
|
|
91
|
+
this.eventManager = LocalEventManager.fromConfiguration(this.getConfiguration());
|
|
92
92
|
}
|
|
93
93
|
return this.eventManager;
|
|
94
94
|
}
|
|
@@ -111,10 +111,10 @@ export class ServiceLocator {
|
|
|
111
111
|
this.getLogger().warning('Implicit creation of storage backend will implicitly set configuration as side effect. ' +
|
|
112
112
|
'It is advised to explicitly first set the configuration instead.');
|
|
113
113
|
}
|
|
114
|
-
const
|
|
115
|
-
this.storageBackend =
|
|
114
|
+
const configuration = this.getConfiguration();
|
|
115
|
+
this.storageBackend = configuration.persistStorage
|
|
116
116
|
? new FileSystemStorageBackend({
|
|
117
|
-
localDataDirectory:
|
|
117
|
+
localDataDirectory: configuration.storageDir,
|
|
118
118
|
logger: this.getLogger().child({ prefix: 'FileSystemStorageBackend' }),
|
|
119
119
|
})
|
|
120
120
|
: new MemoryStorageBackend({
|
|
@@ -141,7 +141,7 @@ export class SessionPool {
|
|
|
141
141
|
return;
|
|
142
142
|
}
|
|
143
143
|
this.keyValueStore = await KeyValueStore.open(this.persistStateKeyValueStoreId ? { id: this.persistStateKeyValueStoreId } : null, {
|
|
144
|
-
|
|
144
|
+
configuration: serviceLocator.getConfiguration(),
|
|
145
145
|
});
|
|
146
146
|
if (!this.persistStateKeyValueStoreId) {
|
|
147
147
|
this.log.debug(`No 'persistStateKeyValueStoreId' options specified, this session pool's data has been saved in the KeyValueStore with the id: ${this.keyValueStore.id}`);
|
package/storages/dataset.d.ts
CHANGED
|
@@ -140,7 +140,7 @@ export interface DatasetExportToOptions extends DatasetExportOptions {
|
|
|
140
140
|
* @category Result Stores
|
|
141
141
|
*/
|
|
142
142
|
export declare class Dataset<Data extends Dictionary = Dictionary> {
|
|
143
|
-
readonly
|
|
143
|
+
readonly configuration: Configuration;
|
|
144
144
|
id: string;
|
|
145
145
|
name?: string;
|
|
146
146
|
backend: DatasetBackend<Data>;
|
|
@@ -149,7 +149,7 @@ export declare class Dataset<Data extends Dictionary = Dictionary> {
|
|
|
149
149
|
/**
|
|
150
150
|
* @internal
|
|
151
151
|
*/
|
|
152
|
-
constructor(options: DatasetOptions,
|
|
152
|
+
constructor(options: DatasetOptions, configuration?: Configuration);
|
|
153
153
|
/**
|
|
154
154
|
* Backend-independent usage counters tracked for this dataset (read / write operations issued to
|
|
155
155
|
* the underlying storage backend). Counted per backend call.
|
package/storages/dataset.js
CHANGED
|
@@ -83,7 +83,7 @@ export function assertJsonSerializable(item, index) {
|
|
|
83
83
|
* @category Result Stores
|
|
84
84
|
*/
|
|
85
85
|
export class Dataset {
|
|
86
|
-
|
|
86
|
+
configuration;
|
|
87
87
|
id;
|
|
88
88
|
name;
|
|
89
89
|
backend;
|
|
@@ -95,8 +95,8 @@ export class Dataset {
|
|
|
95
95
|
/**
|
|
96
96
|
* @internal
|
|
97
97
|
*/
|
|
98
|
-
constructor(options,
|
|
99
|
-
this.
|
|
98
|
+
constructor(options, configuration = Configuration.getGlobalConfiguration()) {
|
|
99
|
+
this.configuration = configuration;
|
|
100
100
|
this.id = options.metadata.id;
|
|
101
101
|
this.name = options.metadata.name;
|
|
102
102
|
this.backend = options.backend;
|
|
@@ -168,7 +168,7 @@ export class Dataset {
|
|
|
168
168
|
* @param [contentType] Only JSON and CSV are supported currently, defaults to JSON.
|
|
169
169
|
*/
|
|
170
170
|
async exportTo(key, options, contentType) {
|
|
171
|
-
const kvStore = await KeyValueStore.open(options?.toKVS ?? null, {
|
|
171
|
+
const kvStore = await KeyValueStore.open(options?.toKVS ?? null, { configuration: this.configuration });
|
|
172
172
|
const items = await this.export(options);
|
|
173
173
|
if (contentType === 'text/csv') {
|
|
174
174
|
// To handle empty dataset exports gracefully.
|
|
@@ -455,12 +455,12 @@ export class Dataset {
|
|
|
455
455
|
static async open(identifier, options = {}) {
|
|
456
456
|
checkStorageAccess();
|
|
457
457
|
ow(options, ow.object.exactShape({
|
|
458
|
-
|
|
458
|
+
configuration: ow.optional.object.instanceOf(Configuration),
|
|
459
459
|
storageBackend: ow.optional.object,
|
|
460
460
|
}));
|
|
461
|
-
options.
|
|
461
|
+
options.configuration ??= Configuration.getGlobalConfiguration();
|
|
462
462
|
const storageBackend = options.storageBackend ?? serviceLocator.getStorageBackend();
|
|
463
|
-
await purgeDefaultStorages({ onlyPurgeOnce: true, storageBackend,
|
|
463
|
+
await purgeDefaultStorages({ onlyPurgeOnce: true, storageBackend, configuration: options.configuration });
|
|
464
464
|
const resolved = await resolveStorageIdentifier(identifier, storageBackend, 'Dataset');
|
|
465
465
|
return serviceLocator.getStorageInstanceManager().openStorage(this, {
|
|
466
466
|
...resolved,
|
|
@@ -61,7 +61,7 @@ import type { StorageOpenOptions } from './utils.js';
|
|
|
61
61
|
* @category Result Stores
|
|
62
62
|
*/
|
|
63
63
|
export declare class KeyValueStore {
|
|
64
|
-
readonly
|
|
64
|
+
readonly configuration: Configuration;
|
|
65
65
|
readonly id: string;
|
|
66
66
|
readonly name?: string;
|
|
67
67
|
private readonly backend;
|
|
@@ -72,7 +72,7 @@ export declare class KeyValueStore {
|
|
|
72
72
|
/**
|
|
73
73
|
* @internal
|
|
74
74
|
*/
|
|
75
|
-
constructor(options: KeyValueStoreOptions,
|
|
75
|
+
constructor(options: KeyValueStoreOptions, configuration?: Configuration);
|
|
76
76
|
/**
|
|
77
77
|
* Backend-independent usage counters tracked for this key-value store (read / write / delete /
|
|
78
78
|
* list operations issued to the underlying storage backend). Counted per backend call.
|
|
@@ -67,7 +67,7 @@ const KVS_KEYS_DEFAULT_LIMIT = 1000;
|
|
|
67
67
|
* @category Result Stores
|
|
68
68
|
*/
|
|
69
69
|
export class KeyValueStore {
|
|
70
|
-
|
|
70
|
+
configuration;
|
|
71
71
|
id;
|
|
72
72
|
name;
|
|
73
73
|
backend;
|
|
@@ -83,8 +83,8 @@ export class KeyValueStore {
|
|
|
83
83
|
/**
|
|
84
84
|
* @internal
|
|
85
85
|
*/
|
|
86
|
-
constructor(options,
|
|
87
|
-
this.
|
|
86
|
+
constructor(options, configuration = Configuration.getGlobalConfiguration()) {
|
|
87
|
+
this.configuration = configuration;
|
|
88
88
|
this.id = options.metadata.id;
|
|
89
89
|
this.name = options.metadata.name;
|
|
90
90
|
this.backend = options.backend;
|
|
@@ -517,12 +517,12 @@ export class KeyValueStore {
|
|
|
517
517
|
static async open(identifier, options = {}) {
|
|
518
518
|
checkStorageAccess();
|
|
519
519
|
ow(options, ow.object.exactShape({
|
|
520
|
-
|
|
520
|
+
configuration: ow.optional.object.instanceOf(Configuration),
|
|
521
521
|
storageBackend: ow.optional.object,
|
|
522
522
|
}));
|
|
523
|
-
options.
|
|
523
|
+
options.configuration ??= Configuration.getGlobalConfiguration();
|
|
524
524
|
const storageBackend = options.storageBackend ?? serviceLocator.getStorageBackend();
|
|
525
|
-
await purgeDefaultStorages({ onlyPurgeOnce: true, storageBackend,
|
|
525
|
+
await purgeDefaultStorages({ onlyPurgeOnce: true, storageBackend, configuration: options.configuration });
|
|
526
526
|
const resolved = await resolveStorageIdentifier(identifier, storageBackend, 'KeyValueStore');
|
|
527
527
|
return serviceLocator.getStorageInstanceManager().openStorage(this, {
|
|
528
528
|
...resolved,
|
|
@@ -649,6 +649,6 @@ export class KeyValueStore {
|
|
|
649
649
|
*/
|
|
650
650
|
static async getInput() {
|
|
651
651
|
const store = await this.open();
|
|
652
|
-
return store.getValue(store.
|
|
652
|
+
return store.getValue(store.configuration.inputKey);
|
|
653
653
|
}
|
|
654
654
|
}
|
|
@@ -711,14 +711,14 @@ export class RequestQueue {
|
|
|
711
711
|
static async open(identifier, options = {}) {
|
|
712
712
|
checkStorageAccess();
|
|
713
713
|
ow(options, ow.object.exactShape({
|
|
714
|
-
|
|
714
|
+
configuration: ow.optional.object.instanceOf(Configuration),
|
|
715
715
|
storageBackend: ow.optional.object,
|
|
716
716
|
proxyConfiguration: ow.optional.object,
|
|
717
717
|
httpClient: ow.optional.object,
|
|
718
718
|
}));
|
|
719
719
|
const storageBackend = options.storageBackend ?? serviceLocator.getStorageBackend();
|
|
720
|
-
const
|
|
721
|
-
await purgeDefaultStorages({ onlyPurgeOnce: true, storageBackend,
|
|
720
|
+
const configuration = options.configuration ?? serviceLocator.getConfiguration();
|
|
721
|
+
await purgeDefaultStorages({ onlyPurgeOnce: true, storageBackend, configuration });
|
|
722
722
|
const resolved = await resolveStorageIdentifier(identifier, storageBackend, 'RequestQueue');
|
|
723
723
|
const queue = await serviceLocator
|
|
724
724
|
.getStorageInstanceManager()
|
|
@@ -90,7 +90,6 @@ export class SitemapRequestLoader {
|
|
|
90
90
|
globs: ow.optional.array.ofType(ow.any(ow.string, ow.object.hasKeys('glob'))),
|
|
91
91
|
exclude: ow.optional.array.ofType(ow.any(ow.string, ow.regExp, ow.object.hasKeys('glob'), ow.object.hasKeys('regexp'))),
|
|
92
92
|
regexps: ow.optional.array.ofType(ow.any(ow.regExp, ow.object.hasKeys('regexp'))),
|
|
93
|
-
config: ow.optional.object,
|
|
94
93
|
persistenceOptions: ow.optional.object,
|
|
95
94
|
}));
|
|
96
95
|
const { globs, exclude, regexps } = options;
|
package/storages/utils.d.ts
CHANGED
|
@@ -9,7 +9,7 @@ interface PurgeDefaultStorageOptions {
|
|
|
9
9
|
* If set to `true`, calling multiple times will only have effect at the first time.
|
|
10
10
|
*/
|
|
11
11
|
onlyPurgeOnce?: boolean;
|
|
12
|
-
|
|
12
|
+
configuration?: Configuration;
|
|
13
13
|
storageBackend?: StorageBackend;
|
|
14
14
|
}
|
|
15
15
|
/**
|
|
@@ -37,9 +37,9 @@ export declare function purgeDefaultStorages(options?: PurgeDefaultStorageOption
|
|
|
37
37
|
* This is a shortcut for running (optional) `purge` method on the StorageBackend interface, in other words
|
|
38
38
|
* it will call the `purge` method of the underlying storage implementation we are currently using.
|
|
39
39
|
*/
|
|
40
|
-
export declare function purgeDefaultStorages(
|
|
40
|
+
export declare function purgeDefaultStorages(configuration?: Configuration, storageBackend?: StorageBackend): Promise<void>;
|
|
41
41
|
export interface UseStateOptions {
|
|
42
|
-
|
|
42
|
+
configuration?: Configuration;
|
|
43
43
|
/**
|
|
44
44
|
* The name of the key-value store you'd like the state to be stored in.
|
|
45
45
|
* If not provided, the default store will be used.
|
|
@@ -53,7 +53,7 @@ export interface UseStateOptions {
|
|
|
53
53
|
*
|
|
54
54
|
* @param name The name of the store to use.
|
|
55
55
|
* @param defaultValue If the store does not yet have a value in it, the value will be initialized with the `defaultValue` you provide.
|
|
56
|
-
* @param options An optional object parameter where a custom `keyValueStoreName` and `
|
|
56
|
+
* @param options An optional object parameter where a custom `keyValueStoreName` and `configuration` can be passed in.
|
|
57
57
|
*/
|
|
58
58
|
export declare function useState<State extends Dictionary = Dictionary>(name?: string, defaultValue?: State, options?: UseStateOptions): Promise<State>;
|
|
59
59
|
/**
|
|
@@ -119,7 +119,7 @@ export interface StorageOpenOptions {
|
|
|
119
119
|
/**
|
|
120
120
|
* SDK configuration instance, defaults to the static register.
|
|
121
121
|
*/
|
|
122
|
-
|
|
122
|
+
configuration?: Configuration;
|
|
123
123
|
/**
|
|
124
124
|
* Optional storage backend that should be used to open storages.
|
|
125
125
|
*/
|
package/storages/utils.js
CHANGED
|
@@ -2,18 +2,18 @@ import crypto from 'node:crypto';
|
|
|
2
2
|
import { Configuration } from '../configuration.js';
|
|
3
3
|
import { serviceLocator } from '../service_locator.js';
|
|
4
4
|
import { KeyValueStore } from './key_value_store.js';
|
|
5
|
-
export async function purgeDefaultStorages(
|
|
6
|
-
const options =
|
|
5
|
+
export async function purgeDefaultStorages(configurationOrOptions, storageBackend) {
|
|
6
|
+
const options = configurationOrOptions instanceof Configuration
|
|
7
7
|
? {
|
|
8
8
|
storageBackend,
|
|
9
|
-
|
|
9
|
+
configuration: configurationOrOptions,
|
|
10
10
|
}
|
|
11
|
-
: (
|
|
12
|
-
const {
|
|
11
|
+
: (configurationOrOptions ?? {});
|
|
12
|
+
const { configuration = serviceLocator.getConfiguration(), onlyPurgeOnce = false } = options;
|
|
13
13
|
({ storageBackend = serviceLocator.getStorageBackend() } = options);
|
|
14
14
|
const casted = storageBackend;
|
|
15
15
|
// if `onlyPurgeOnce` is true, will purge anytime this function is called, otherwise - only on start
|
|
16
|
-
if (!onlyPurgeOnce || (
|
|
16
|
+
if (!onlyPurgeOnce || (configuration.purgeOnStart && !casted.__purged)) {
|
|
17
17
|
casted.__purged = true;
|
|
18
18
|
await casted.purge?.();
|
|
19
19
|
}
|
|
@@ -25,11 +25,11 @@ export async function purgeDefaultStorages(configOrOptions, storageBackend) {
|
|
|
25
25
|
*
|
|
26
26
|
* @param name The name of the store to use.
|
|
27
27
|
* @param defaultValue If the store does not yet have a value in it, the value will be initialized with the `defaultValue` you provide.
|
|
28
|
-
* @param options An optional object parameter where a custom `keyValueStoreName` and `
|
|
28
|
+
* @param options An optional object parameter where a custom `keyValueStoreName` and `configuration` can be passed in.
|
|
29
29
|
*/
|
|
30
30
|
export async function useState(name, defaultValue = {}, options) {
|
|
31
31
|
const kvStore = await KeyValueStore.open(options?.keyValueStoreName ? { name: options.keyValueStoreName } : null, {
|
|
32
|
-
|
|
32
|
+
configuration: options?.configuration || serviceLocator.getConfiguration(),
|
|
33
33
|
});
|
|
34
34
|
return kvStore.getAutoSavedValue(name || 'CRAWLEE_GLOBAL_STATE', defaultValue);
|
|
35
35
|
}
|