@crawlee/core 4.0.0-beta.90 → 4.0.0-beta.91
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/crawlers/crawler_commons.d.ts +1 -1
- package/index.d.ts +1 -1
- package/package.json +5 -5
- package/request.d.ts +1 -2
- package/router.d.ts +1 -2
- package/storages/access_checking.d.ts +1 -1
- package/storages/dataset.d.ts +2 -3
- package/storages/key_value_store.d.ts +2 -3
- package/storages/storage_instance_manager.d.ts +1 -2
- package/typedefs.d.ts +0 -5
|
@@ -4,10 +4,10 @@ import type { Configuration } from '../configuration.js';
|
|
|
4
4
|
import type { EnqueueLinksOptions } from '../enqueue_links/enqueue_links.js';
|
|
5
5
|
import type { CrawleeLogger } from '../log.js';
|
|
6
6
|
import type { Request, RequestOptions, Source } from '../request.js';
|
|
7
|
+
import type { StorageIdentifier } from '../storages/storage_instance_manager.js';
|
|
7
8
|
import type { Dataset } from '../storages/dataset.js';
|
|
8
9
|
import { KeyValueStore, type RecordOptions } from '../storages/key_value_store.js';
|
|
9
10
|
import type { RequestQueueOperationOptions } from '../storages/request_queue.js';
|
|
10
|
-
import type { StorageIdentifier } from '../storages/storage_instance_manager.js';
|
|
11
11
|
/** @internal */
|
|
12
12
|
export type IsAny<T> = 0 extends 1 & T ? true : false;
|
|
13
13
|
/**
|
package/index.d.ts
CHANGED
|
@@ -19,4 +19,4 @@ export * from './validators.js';
|
|
|
19
19
|
export * from './cookie_utils.js';
|
|
20
20
|
export * from './recoverable_state.js';
|
|
21
21
|
export { PseudoUrl } from '@apify/pseudo_url';
|
|
22
|
-
export type {
|
|
22
|
+
export type { StorageBackend } from '@crawlee/types';
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/core",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.91",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.0.0"
|
|
@@ -53,9 +53,9 @@
|
|
|
53
53
|
"@apify/pseudo_url": "^2.0.59",
|
|
54
54
|
"@apify/timeout": "^0.3.2",
|
|
55
55
|
"@apify/utilities": "^2.15.5",
|
|
56
|
-
"@crawlee/fs-storage": "4.0.0-beta.
|
|
57
|
-
"@crawlee/types": "4.0.0-beta.
|
|
58
|
-
"@crawlee/utils": "4.0.0-beta.
|
|
56
|
+
"@crawlee/fs-storage": "4.0.0-beta.91",
|
|
57
|
+
"@crawlee/types": "4.0.0-beta.91",
|
|
58
|
+
"@crawlee/utils": "4.0.0-beta.91",
|
|
59
59
|
"@sapphire/async-queue": "^1.5.5",
|
|
60
60
|
"@sapphire/shapeshift": "^4.0.0",
|
|
61
61
|
"@vladfrangu/async_event_emitter": "^2.4.6",
|
|
@@ -79,5 +79,5 @@
|
|
|
79
79
|
}
|
|
80
80
|
}
|
|
81
81
|
},
|
|
82
|
-
"gitHead": "
|
|
82
|
+
"gitHead": "1e3e1ca10f24be53e1d527b3ea25f456baebecbf"
|
|
83
83
|
}
|
package/request.d.ts
CHANGED
|
@@ -1,8 +1,7 @@
|
|
|
1
1
|
import type { BinaryLike } from 'node:crypto';
|
|
2
|
-
import type { Dictionary } from '@crawlee/types';
|
|
2
|
+
import type { AllowedHttpMethods, Dictionary } from '@crawlee/types';
|
|
3
3
|
import type { EnqueueLinksOptions } from './enqueue_links/enqueue_links.js';
|
|
4
4
|
import type { SkippedRequestReason } from './enqueue_links/shared.js';
|
|
5
|
-
import type { AllowedHttpMethods } from './typedefs.js';
|
|
6
5
|
export declare enum RequestState {
|
|
7
6
|
UNPROCESSED = 0,
|
|
8
7
|
BEFORE_NAV = 1,
|
package/router.d.ts
CHANGED
|
@@ -1,8 +1,7 @@
|
|
|
1
|
-
import type { Dictionary } from '@crawlee/types';
|
|
1
|
+
import type { Awaitable, Dictionary } from '@crawlee/types';
|
|
2
2
|
import type { StandardSchemaV1 } from '@standard-schema/spec';
|
|
3
3
|
import type { CrawlingContext, LoadedRequest, RestrictedCrawlingContext, TypedContextAddRequests, TypedContextEnqueueLinks } from './crawlers/crawler_commons.js';
|
|
4
4
|
import type { Request } from './request.js';
|
|
5
|
-
import type { Awaitable } from './typedefs.js';
|
|
6
5
|
/**
|
|
7
6
|
* The key of the default route — the fallback handler registered via {@link Router.addDefaultHandler}.
|
|
8
7
|
* Use it in a {@link RouteSchemas} map to register a schema that validates the `userData` of every request
|
package/storages/dataset.d.ts
CHANGED
|
@@ -1,10 +1,9 @@
|
|
|
1
|
-
import type { DatasetBackend, DatasetInfo, Dictionary } from '@crawlee/types';
|
|
1
|
+
import type { Awaitable, DatasetBackend, DatasetInfo, Dictionary } from '@crawlee/types';
|
|
2
2
|
import { Configuration } from '../configuration.js';
|
|
3
3
|
import type { CrawleeLogger } from '../log.js';
|
|
4
|
-
import type { Awaitable } from '../typedefs.js';
|
|
5
4
|
import type { DatasetStats } from './storage_stats.js';
|
|
6
|
-
import type { StorageIdentifier } from './storage_instance_manager.js';
|
|
7
5
|
import type { StorageOpenOptions } from './utils.js';
|
|
6
|
+
import type { StorageIdentifier } from './storage_instance_manager.js';
|
|
8
7
|
/** @internal */
|
|
9
8
|
export declare const DATASET_ITERATORS_DEFAULT_LIMIT = 10000;
|
|
10
9
|
/**
|
|
@@ -1,9 +1,8 @@
|
|
|
1
|
-
import type { Dictionary, KeyValueStoreBackend, KeyValueStoreInfo } from '@crawlee/types';
|
|
1
|
+
import type { Awaitable, Dictionary, KeyValueStoreBackend, KeyValueStoreInfo } from '@crawlee/types';
|
|
2
2
|
import { Configuration } from '../configuration.js';
|
|
3
|
-
import type { Awaitable } from '../typedefs.js';
|
|
4
3
|
import type { KeyValueStoreStats } from './storage_stats.js';
|
|
5
|
-
import type { StorageIdentifier } from './storage_instance_manager.js';
|
|
6
4
|
import type { StorageOpenOptions } from './utils.js';
|
|
5
|
+
import type { StorageIdentifier } from './storage_instance_manager.js';
|
|
7
6
|
/**
|
|
8
7
|
* The `KeyValueStore` class represents a key-value store, a simple data storage that is used
|
|
9
8
|
* for saving and reading data records or files. Each data record is
|
|
@@ -1,5 +1,4 @@
|
|
|
1
|
-
import type { DatasetBackend, KeyValueStoreBackend, RequestQueueBackend, StorageBackend, StorageIdentifier } from '@crawlee/types';
|
|
2
|
-
import type { Constructor } from '../typedefs.js';
|
|
1
|
+
import type { Constructor, DatasetBackend, KeyValueStoreBackend, RequestQueueBackend, StorageBackend, StorageIdentifier } from '@crawlee/types';
|
|
3
2
|
export type { StorageIdentifier } from '@crawlee/types';
|
|
4
3
|
/**
|
|
5
4
|
* Matches an `IStorage` – a storage "frontend" (Dataset, KeyValueStore, RequestQueue).
|
package/typedefs.d.ts
CHANGED
|
@@ -1,9 +1,4 @@
|
|
|
1
1
|
/** @ignore */
|
|
2
|
-
export type Constructor<T = unknown> = new (...args: any[]) => T;
|
|
3
|
-
/** @ignore */
|
|
4
|
-
export type Awaitable<T> = T | PromiseLike<T>;
|
|
5
|
-
/** @ignore */
|
|
6
2
|
export declare function entries<T extends {}>(obj: T): [keyof T, T[keyof T]][];
|
|
7
3
|
/** @ignore */
|
|
8
4
|
export declare function keys<T extends {}>(obj: T): (keyof T)[];
|
|
9
|
-
export type { AllowedHttpMethods } from '@crawlee/types';
|