@crawlee/core 3.7.1-beta.4 → 3.7.1-beta.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/configuration.d.ts +2 -0
- package/cookie_utils.d.ts +1 -0
- package/crawlers/crawler_commons.d.ts +1 -1
- package/events/event_manager.d.ts +2 -0
- package/package.json +5 -5
- package/proxy_configuration.d.ts +1 -0
- package/request.d.ts +2 -0
- package/serialization.d.ts +2 -0
- package/session_pool/session.d.ts +1 -0
- package/session_pool/session_pool.d.ts +2 -0
- package/storages/key_value_store.d.ts +1 -0
- package/storages/request_list.d.ts +1 -0
- package/storages/request_queue.d.ts +5 -0
- package/storages/storage_manager.d.ts +3 -0
package/configuration.d.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
1
2
|
/// <reference types="node" />
|
|
2
3
|
import { AsyncLocalStorage } from 'node:async_hooks';
|
|
3
4
|
import { LogLevel } from '@apify/log';
|
|
@@ -219,6 +220,7 @@ export declare class Configuration {
|
|
|
219
220
|
protected services: Map<string, unknown>;
|
|
220
221
|
/** @internal */
|
|
221
222
|
static globalConfig?: Configuration;
|
|
223
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
222
224
|
readonly storageManagers: Map<Constructor, StorageManager<import("./storages").IStorage>>;
|
|
223
225
|
/**
|
|
224
226
|
* Creates new `Configuration` instance with provided options. Env vars will have precedence over those.
|
package/cookie_utils.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { Dictionary, BatchAddRequestsResult } from '@crawlee/types';
|
|
2
|
-
// @ts-ignore optional peer dependency
|
|
2
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
3
3
|
import type { Response as GotResponse, OptionsInit } from 'got-scraping';
|
|
4
4
|
import type { EnqueueLinksOptions } from '../enqueue_links/enqueue_links';
|
|
5
5
|
import type { Log } from '../log';
|
|
@@ -15,9 +15,11 @@ interface Intervals {
|
|
|
15
15
|
}
|
|
16
16
|
export declare abstract class EventManager {
|
|
17
17
|
readonly config: Configuration;
|
|
18
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
18
19
|
protected events: AsyncEventEmitter<Record<PropertyKey, unknown[]> & import("@vladfrangu/async_event_emitter").AsyncEventEmitterPredefinedEvents, Record<PropertyKey, unknown[]> & import("@vladfrangu/async_event_emitter").AsyncEventEmitterPredefinedEvents>;
|
|
19
20
|
protected initialized: boolean;
|
|
20
21
|
protected intervals: Intervals;
|
|
22
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
21
23
|
protected log: import("@apify/log").Log;
|
|
22
24
|
constructor(config?: Configuration);
|
|
23
25
|
/**
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/core",
|
|
3
|
-
"version": "3.7.1-beta.
|
|
3
|
+
"version": "3.7.1-beta.6",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=16.0.0"
|
|
@@ -59,9 +59,9 @@
|
|
|
59
59
|
"@apify/pseudo_url": "^2.0.30",
|
|
60
60
|
"@apify/timeout": "^0.3.0",
|
|
61
61
|
"@apify/utilities": "^2.7.10",
|
|
62
|
-
"@crawlee/memory-storage": "3.7.1-beta.
|
|
63
|
-
"@crawlee/types": "3.7.1-beta.
|
|
64
|
-
"@crawlee/utils": "3.7.1-beta.
|
|
62
|
+
"@crawlee/memory-storage": "3.7.1-beta.6",
|
|
63
|
+
"@crawlee/types": "3.7.1-beta.6",
|
|
64
|
+
"@crawlee/utils": "3.7.1-beta.6",
|
|
65
65
|
"@sapphire/async-queue": "^1.5.1",
|
|
66
66
|
"@types/tough-cookie": "^4.0.2",
|
|
67
67
|
"@vladfrangu/async_event_emitter": "^2.2.2",
|
|
@@ -85,5 +85,5 @@
|
|
|
85
85
|
}
|
|
86
86
|
}
|
|
87
87
|
},
|
|
88
|
-
"gitHead": "
|
|
88
|
+
"gitHead": "01664e58799374a6acc2aa9db397566c1acc69d4"
|
|
89
89
|
}
|
package/proxy_configuration.d.ts
CHANGED
|
@@ -105,6 +105,7 @@ export declare class ProxyConfiguration {
|
|
|
105
105
|
protected proxyUrls?: string[];
|
|
106
106
|
protected usedProxyUrls: Map<string, string>;
|
|
107
107
|
protected newUrlFunction?: ProxyConfigurationFunction;
|
|
108
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
108
109
|
protected log: import("@apify/log").Log;
|
|
109
110
|
/**
|
|
110
111
|
* Creates a {@apilink ProxyConfiguration} instance based on the provided options. Proxy servers are used to prevent target websites from
|
package/request.d.ts
CHANGED
|
@@ -1,4 +1,6 @@
|
|
|
1
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
1
2
|
/// <reference types="node" />
|
|
3
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
2
4
|
/// <reference types="node" />
|
|
3
5
|
import type { BinaryLike } from 'node:crypto';
|
|
4
6
|
import type { Dictionary } from '@crawlee/types';
|
package/serialization.d.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
1
2
|
/// <reference types="node" />
|
|
2
3
|
import { EventEmitter } from 'node:events';
|
|
3
4
|
import type { Log } from '@apify/log';
|
|
@@ -181,6 +182,7 @@ export declare class SessionPool extends EventEmitter {
|
|
|
181
182
|
getState(): {
|
|
182
183
|
usableSessionsCount: number;
|
|
183
184
|
retiredSessionsCount: number;
|
|
185
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
184
186
|
sessions: import("./session").SessionState[];
|
|
185
187
|
};
|
|
186
188
|
/**
|
|
@@ -88,10 +88,15 @@ export declare class RequestQueue extends RequestProvider {
|
|
|
88
88
|
*/
|
|
89
89
|
protected _ensureHeadIsNonEmpty(ensureConsistency?: boolean, limit?: number, iteration?: number): Promise<boolean>;
|
|
90
90
|
isFinished(): Promise<boolean>;
|
|
91
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
91
92
|
addRequest(...args: Parameters<RequestProvider['addRequest']>): Promise<import("./request_provider").RequestQueueOperationInfo>;
|
|
93
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
92
94
|
addRequests(...args: Parameters<RequestProvider['addRequests']>): Promise<import("@crawlee/types").BatchAddRequestsResult>;
|
|
95
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
93
96
|
addRequestsBatched(...args: Parameters<RequestProvider['addRequestsBatched']>): Promise<import("./request_provider").AddRequestsBatchedResult>;
|
|
97
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
94
98
|
markRequestHandled(...args: Parameters<RequestProvider['markRequestHandled']>): Promise<import("./request_provider").RequestQueueOperationInfo | null>;
|
|
99
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
95
100
|
reclaimRequest(...args: Parameters<RequestProvider['reclaimRequest']>): Promise<import("./request_provider").RequestQueueOperationInfo | null>;
|
|
96
101
|
protected _reset(): void;
|
|
97
102
|
static open(...args: Parameters<typeof RequestProvider.open>): Promise<RequestQueue>;
|
|
@@ -29,9 +29,12 @@ export declare class StorageManager<T extends IStorage = IStorage> {
|
|
|
29
29
|
/**
|
|
30
30
|
* Helper function that first requests storage by ID and if storage doesn't exist then gets it by name.
|
|
31
31
|
*/
|
|
32
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
32
33
|
protected _getOrCreateStorage(storageIdOrName: string, storageConstructorName: string, apiClient: StorageClient): Promise<import("@crawlee/types").DatasetCollectionData>;
|
|
33
34
|
protected _getStorageClientFactories(client: StorageClient, storageConstructorName: string): {
|
|
35
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
34
36
|
createStorageClient: ((id: string) => import("@crawlee/types").DatasetClient<Dictionary>) | ((id: string) => import("@crawlee/types").KeyValueStoreClient) | ((id: string, options?: import("@crawlee/types").RequestQueueOptions | undefined) => import("@crawlee/types").RequestQueueClient);
|
|
37
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
35
38
|
createStorageCollectionClient: (() => import("@crawlee/types").DatasetCollectionClient) | (() => import("@crawlee/types").KeyValueStoreCollectionClient) | (() => import("@crawlee/types").RequestQueueCollectionClient);
|
|
36
39
|
};
|
|
37
40
|
protected _addStorageToCache(storage: T): void;
|