@crawlee/core 4.0.0-beta.223 → 4.0.0-beta.225
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/configuration.js +1 -1
- package/events/{event_manager.js → event-manager.js} +1 -1
- package/events/index.d.ts +3 -3
- package/events/index.js +2 -2
- package/events/{local_event_manager.d.ts → local-event-manager.d.ts} +1 -1
- package/events/{local_event_manager.js → local-event-manager.js} +2 -2
- package/index.d.ts +4 -4
- package/index.js +4 -4
- package/internal.d.ts +2 -2
- package/internal.js +2 -2
- package/memory-storage/utils.d.ts +1 -1
- package/memory-storage/utils.js +1 -1
- package/package.json +6 -6
- package/{recoverable_state.d.ts → recoverable-state.d.ts} +1 -1
- package/{recoverable_state.js → recoverable-state.js} +3 -3
- package/request.js +1 -1
- package/{service_locator.d.ts → service-locator.d.ts} +2 -2
- package/{service_locator.js → service-locator.js} +2 -2
- package/storages/{batched_adds.d.ts → batched-adds.d.ts} +1 -1
- package/storages/dataset.d.ts +2 -2
- package/storages/dataset.js +4 -4
- package/storages/index.d.ts +10 -10
- package/storages/index.js +6 -6
- package/storages/{key_value_store_codec.js → key-value-store-codec.js} +1 -1
- package/storages/{key_value_store.d.ts → key-value-store.d.ts} +2 -2
- package/storages/{key_value_store.js → key-value-store.js} +5 -5
- package/storages/{request_list.d.ts → request-list.d.ts} +3 -3
- package/storages/{request_list.js → request-list.js} +5 -5
- package/storages/{request_loader.d.ts → request-loader.d.ts} +2 -2
- package/storages/{request_manager_tandem.d.ts → request-manager-tandem.d.ts} +3 -3
- package/storages/{request_manager_tandem.js → request-manager-tandem.js} +2 -2
- package/storages/{request_manager.d.ts → request-manager.d.ts} +2 -2
- package/storages/{request_queue.d.ts → request-queue.d.ts} +5 -5
- package/storages/{request_queue.js → request-queue.js} +6 -6
- package/storages/transaction.d.ts +1 -1
- package/storages/transaction.js +1 -1
- package/storages/utils.d.ts +1 -1
- package/storages/utils.js +2 -2
- /package/{byte_utils.d.ts → byte-utils.d.ts} +0 -0
- /package/{byte_utils.js → byte-utils.js} +0 -0
- /package/events/{event_manager.d.ts → event-manager.d.ts} +0 -0
- /package/events/{system_info.d.ts → system-info.d.ts} +0 -0
- /package/events/{system_info.js → system-info.js} +0 -0
- /package/{owned_or_injected.d.ts → owned-or-injected.d.ts} +0 -0
- /package/{owned_or_injected.js → owned-or-injected.js} +0 -0
- /package/{proxy_configuration.d.ts → proxy-configuration.d.ts} +0 -0
- /package/{proxy_configuration.js → proxy-configuration.js} +0 -0
- /package/storages/{batched_adds.js → batched-adds.js} +0 -0
- /package/storages/{key_value_store_codec.d.ts → key-value-store-codec.d.ts} +0 -0
- /package/storages/{request_dedup_cache.d.ts → request-dedup-cache.d.ts} +0 -0
- /package/storages/{request_dedup_cache.js → request-dedup-cache.js} +0 -0
- /package/storages/{request_loader.js → request-loader.js} +0 -0
- /package/storages/{request_manager.js → request-manager.js} +0 -0
- /package/storages/{storage_instance_manager.d.ts → storage-instance-manager.d.ts} +0 -0
- /package/storages/{storage_instance_manager.js → storage-instance-manager.js} +0 -0
- /package/storages/{storage_stats.d.ts → storage-stats.d.ts} +0 -0
- /package/storages/{storage_stats.js → storage-stats.js} +0 -0
package/configuration.js
CHANGED
|
@@ -3,7 +3,7 @@ import { readFileSync } from 'node:fs';
|
|
|
3
3
|
import { join } from 'node:path';
|
|
4
4
|
import { z } from 'zod';
|
|
5
5
|
import { log, LogLevel } from './log.js';
|
|
6
|
-
import { serviceLocator } from './
|
|
6
|
+
import { serviceLocator } from './service-locator.js';
|
|
7
7
|
// Crawlee attaches many listeners to shared EventEmitters (one per crawler/session/autoscaled pool),
|
|
8
8
|
// which can exceed Node's default limit of 10 and trigger spurious MaxListenersExceededWarning logs.
|
|
9
9
|
// Raising the global default avoids false positives; real leaks will still manifest as unbounded growth.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { AsyncEventEmitter } from '@vladfrangu/async_event_emitter';
|
|
2
2
|
import { betterClearInterval, betterSetInterval } from '@apify/utilities';
|
|
3
|
-
import { serviceLocator } from '../
|
|
3
|
+
import { serviceLocator } from '../service-locator.js';
|
|
4
4
|
export var EventType;
|
|
5
5
|
(function (EventType) {
|
|
6
6
|
EventType["PERSIST_STATE"] = "persistState";
|
package/events/index.d.ts
CHANGED
|
@@ -1,3 +1,3 @@
|
|
|
1
|
-
export * from './
|
|
2
|
-
export * from './
|
|
3
|
-
export type * from './
|
|
1
|
+
export * from './event-manager.js';
|
|
2
|
+
export * from './local-event-manager.js';
|
|
3
|
+
export type * from './system-info.js';
|
package/events/index.js
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
export * from './
|
|
2
|
-
export * from './
|
|
1
|
+
export * from './event-manager.js';
|
|
2
|
+
export * from './local-event-manager.js';
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { Configuration } from '../configuration.js';
|
|
2
|
-
import { EventManager, type EventManagerOptions } from './
|
|
2
|
+
import { EventManager, type EventManagerOptions } from './event-manager.js';
|
|
3
3
|
export interface LocalEventManagerOptions extends EventManagerOptions {
|
|
4
4
|
/** Interval between emitted `systemInfo` events in milliseconds. */
|
|
5
5
|
systemInfoIntervalMillis: number;
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { betterClearInterval, betterSetInterval } from '@apify/utilities';
|
|
2
|
-
import { serviceLocator } from '../
|
|
3
|
-
import { EventManager, EventType } from './
|
|
2
|
+
import { serviceLocator } from '../service-locator.js';
|
|
3
|
+
import { EventManager, EventType } from './event-manager.js';
|
|
4
4
|
export class LocalEventManager extends EventManager {
|
|
5
5
|
#systemInfoIntervalMillis;
|
|
6
6
|
constructor(options) {
|
package/index.d.ts
CHANGED
|
@@ -1,16 +1,16 @@
|
|
|
1
1
|
export * from './debug.js';
|
|
2
2
|
export * from './errors.js';
|
|
3
3
|
export * from './configuration.js';
|
|
4
|
-
export * from './
|
|
4
|
+
export * from './service-locator.js';
|
|
5
5
|
export * from './events/index.js';
|
|
6
6
|
export * from './log.js';
|
|
7
|
-
export * from './
|
|
8
|
-
export * from './
|
|
7
|
+
export * from './owned-or-injected.js';
|
|
8
|
+
export * from './proxy-configuration.js';
|
|
9
9
|
export * from './request.js';
|
|
10
10
|
export * from './serialization.js';
|
|
11
11
|
export * from './storages/index.js';
|
|
12
12
|
export * from './memory-storage/index.js';
|
|
13
13
|
export { ArgumentValidationError, validators } from './validators.js';
|
|
14
|
-
export * from './
|
|
14
|
+
export * from './recoverable-state.js';
|
|
15
15
|
export type { StorageBackend } from '@crawlee/types';
|
|
16
16
|
export { EnqueueStrategy } from '@crawlee/utils';
|
package/index.js
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
export * from './debug.js';
|
|
2
2
|
export * from './errors.js';
|
|
3
3
|
export * from './configuration.js';
|
|
4
|
-
export * from './
|
|
4
|
+
export * from './service-locator.js';
|
|
5
5
|
export * from './events/index.js';
|
|
6
6
|
export * from './log.js';
|
|
7
|
-
export * from './
|
|
8
|
-
export * from './
|
|
7
|
+
export * from './owned-or-injected.js';
|
|
8
|
+
export * from './proxy-configuration.js';
|
|
9
9
|
export * from './request.js';
|
|
10
10
|
export * from './serialization.js';
|
|
11
11
|
export * from './storages/index.js';
|
|
@@ -13,5 +13,5 @@ export * from './memory-storage/index.js';
|
|
|
13
13
|
// Not `export *`: the rest of the module re-exports `@crawlee/utils/internal` symbols, which carry no
|
|
14
14
|
// semver guarantees and must not reach the public surface. Internal consumers import them directly.
|
|
15
15
|
export { ArgumentValidationError, validators } from './validators.js';
|
|
16
|
-
export * from './
|
|
16
|
+
export * from './recoverable-state.js';
|
|
17
17
|
export { EnqueueStrategy } from '@crawlee/utils';
|
package/internal.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
export * from './iterables.js';
|
|
2
|
-
export * from './storages/
|
|
3
|
-
export { joinRequestSourceStatuses } from './storages/
|
|
2
|
+
export * from './storages/batched-adds.js';
|
|
3
|
+
export { joinRequestSourceStatuses } from './storages/request-loader.js';
|
|
4
4
|
export * from './system-info/cpu-info.js';
|
|
5
5
|
export * from './system-info/memory-info.js';
|
|
6
6
|
export * from './system-info/ps-tree.js';
|
package/internal.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// Not part of the public API: internals that other Crawlee packages need but that carry no semver guarantees.
|
|
2
2
|
export * from './iterables.js';
|
|
3
|
-
export * from './storages/
|
|
4
|
-
export { joinRequestSourceStatuses } from './storages/
|
|
3
|
+
export * from './storages/batched-adds.js';
|
|
4
|
+
export { joinRequestSourceStatuses } from './storages/request-loader.js';
|
|
5
5
|
export * from './system-info/cpu-info.js';
|
|
6
6
|
export * from './system-info/memory-info.js';
|
|
7
7
|
export * from './system-info/ps-tree.js';
|
|
@@ -13,4 +13,4 @@ export declare function purgeNullsFromObject<T>(object: T): T;
|
|
|
13
13
|
* Creates a standard request ID (same as Platform).
|
|
14
14
|
*/
|
|
15
15
|
export declare function uniqueKeyToRequestId(uniqueKey: string): string;
|
|
16
|
-
export { isBuffer, isStream, toBuffer } from '../
|
|
16
|
+
export { isBuffer, isStream, toBuffer } from '../byte-utils.js';
|
package/memory-storage/utils.js
CHANGED
|
@@ -38,4 +38,4 @@ export function uniqueKeyToRequestId(uniqueKey) {
|
|
|
38
38
|
.replace(/(\+|\/|=)/g, '');
|
|
39
39
|
return str.length > REQUEST_ID_LENGTH ? str.slice(0, REQUEST_ID_LENGTH) : str;
|
|
40
40
|
}
|
|
41
|
-
export { isBuffer, isStream, toBuffer } from '../
|
|
41
|
+
export { isBuffer, isStream, toBuffer } from '../byte-utils.js';
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/core",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.225",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.13.0"
|
|
@@ -53,10 +53,10 @@
|
|
|
53
53
|
"@apify/log": "^3.0.1",
|
|
54
54
|
"@apify/timeout": "^1.0.1",
|
|
55
55
|
"@apify/utilities": "^3.0.1",
|
|
56
|
-
"@crawlee/fs-storage": "4.0.0-beta.
|
|
57
|
-
"@crawlee/http-client": "4.0.0-beta.
|
|
58
|
-
"@crawlee/types": "4.0.0-beta.
|
|
59
|
-
"@crawlee/utils": "4.0.0-beta.
|
|
56
|
+
"@crawlee/fs-storage": "4.0.0-beta.225",
|
|
57
|
+
"@crawlee/http-client": "4.0.0-beta.225",
|
|
58
|
+
"@crawlee/types": "4.0.0-beta.225",
|
|
59
|
+
"@crawlee/utils": "4.0.0-beta.225",
|
|
60
60
|
"@sapphire/async-queue": "^1.5.5",
|
|
61
61
|
"@standard-schema/spec": "^1.0.0",
|
|
62
62
|
"@vladfrangu/async_event_emitter": "^2.4.6",
|
|
@@ -76,5 +76,5 @@
|
|
|
76
76
|
}
|
|
77
77
|
}
|
|
78
78
|
},
|
|
79
|
-
"gitHead": "
|
|
79
|
+
"gitHead": "5d71fa0f785d7dc5736d95bdf5f4ec907e5a1c42"
|
|
80
80
|
}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { Configuration } from './configuration.js';
|
|
2
2
|
import type { CrawleeLogger } from './log.js';
|
|
3
|
-
import { KeyValueStore } from './storages/
|
|
3
|
+
import { KeyValueStore } from './storages/key-value-store.js';
|
|
4
4
|
import type { Awaitable } from '@crawlee/types';
|
|
5
5
|
import type { StandardSchemaV1 } from '@standard-schema/spec';
|
|
6
6
|
/**
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import { Readable } from 'node:stream';
|
|
2
2
|
import { addTimeoutToPromise, storage as timeoutStorage } from '@apify/timeout';
|
|
3
3
|
import { StateValidationError } from './errors.js';
|
|
4
|
-
import { EventType } from './events/
|
|
5
|
-
import { serviceLocator } from './
|
|
6
|
-
import { KeyValueStore } from './storages/
|
|
4
|
+
import { EventType } from './events/event-manager.js';
|
|
5
|
+
import { serviceLocator } from './service-locator.js';
|
|
6
|
+
import { KeyValueStore } from './storages/key-value-store.js';
|
|
7
7
|
const DEFAULT_PERSISTENCE_TIMEOUT_MILLIS = 60_000;
|
|
8
8
|
/**
|
|
9
9
|
* Applies a {@link SyncStateConversion}, throwing a {@link StateValidationError} for a schema that rejects
|
package/request.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import crypto from 'node:crypto';
|
|
2
2
|
import { z } from 'zod';
|
|
3
3
|
import { cryptoRandomObjectId, normalizeUrl } from '@apify/utilities';
|
|
4
|
-
import { serviceLocator } from './
|
|
4
|
+
import { serviceLocator } from './service-locator.js';
|
|
5
5
|
import { parseArgument, schemas } from './validators.js';
|
|
6
6
|
// new properties on the Request object breaks serialization
|
|
7
7
|
// Compiled once: every `Request` runs it, and the generated fast path skips zod's interpreter.
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import type { StorageBackend } from '@crawlee/types';
|
|
2
2
|
import { Configuration } from './configuration.js';
|
|
3
|
-
import type { EventManager } from './events/
|
|
3
|
+
import type { EventManager } from './events/event-manager.js';
|
|
4
4
|
import type { CrawleeLogger } from './log.js';
|
|
5
|
-
import { StorageInstanceManager } from './storages/
|
|
5
|
+
import { StorageInstanceManager } from './storages/storage-instance-manager.js';
|
|
6
6
|
interface ServiceLocatorInterface {
|
|
7
7
|
/**
|
|
8
8
|
* Get the configuration.
|
|
@@ -3,10 +3,10 @@ import { FileSystemStorageBackend } from '@crawlee/fs-storage';
|
|
|
3
3
|
import log from '@apify/log';
|
|
4
4
|
import { Configuration } from './configuration.js';
|
|
5
5
|
import { ServiceConflictError } from './errors.js';
|
|
6
|
-
import { LocalEventManager } from './events/
|
|
6
|
+
import { LocalEventManager } from './events/local-event-manager.js';
|
|
7
7
|
import { ApifyLogAdapter } from './log.js';
|
|
8
8
|
import { MemoryStorageBackend } from './memory-storage/index.js';
|
|
9
|
-
import { StorageInstanceManager } from './storages/
|
|
9
|
+
import { StorageInstanceManager } from './storages/storage-instance-manager.js';
|
|
10
10
|
/**
|
|
11
11
|
* Service locator for managing the services used by Crawlee.
|
|
12
12
|
*
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { ProcessedRequest } from '@crawlee/types';
|
|
2
2
|
import type { Source } from '../request.js';
|
|
3
|
-
import type { AddRequestsBatchedResult } from './
|
|
3
|
+
import type { AddRequestsBatchedResult } from './request-queue.js';
|
|
4
4
|
export interface DrainRequestBatchesOptions<TItem extends Source> {
|
|
5
5
|
/**
|
|
6
6
|
* The requests to add, already normalized by the caller. Consumed lazily: an unbounded or expensive
|
package/storages/dataset.d.ts
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import type { Awaitable, DatasetBackend, DatasetInfo, Dictionary } from '@crawlee/types';
|
|
2
2
|
import { Configuration } from '../configuration.js';
|
|
3
3
|
import type { JournalEntry } from './transaction.js';
|
|
4
|
-
import type { DatasetStats } from './
|
|
4
|
+
import type { DatasetStats } from './storage-stats.js';
|
|
5
5
|
import type { StorageOpenOptions } from './utils.js';
|
|
6
|
-
import type { StorageIdentifier } from './
|
|
6
|
+
import type { StorageIdentifier } from './storage-instance-manager.js';
|
|
7
7
|
/** @internal */
|
|
8
8
|
export declare const DATASET_ITERATORS_DEFAULT_LIMIT = 10000;
|
|
9
9
|
/**
|
package/storages/dataset.js
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
import { z } from 'zod';
|
|
2
2
|
import { tryCancel } from '@apify/timeout';
|
|
3
3
|
import { Configuration } from '../configuration.js';
|
|
4
|
-
import { serviceLocator } from '../
|
|
4
|
+
import { serviceLocator } from '../service-locator.js';
|
|
5
5
|
import { parseArgument, schemas, validators } from '../validators.js';
|
|
6
6
|
import { activeStorageTransaction, rejectOperationInTransaction, snapshotValue } from './transaction.js';
|
|
7
|
-
import { KeyValueStore } from './
|
|
8
|
-
import { StorageStatsTracker } from './
|
|
9
|
-
import { resolveStorageIdentifier } from './
|
|
7
|
+
import { KeyValueStore } from './key-value-store.js';
|
|
8
|
+
import { StorageStatsTracker } from './storage-stats.js';
|
|
9
|
+
import { resolveStorageIdentifier } from './storage-instance-manager.js';
|
|
10
10
|
import { createDualIterable, purgeDefaultStorages } from './utils.js';
|
|
11
11
|
const openOptionsSchema = z.strictObject({
|
|
12
12
|
configuration: z.instanceof(Configuration).optional(),
|
package/storages/index.d.ts
CHANGED
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
export * from './dataset.js';
|
|
2
|
-
export * from './
|
|
3
|
-
export * from './
|
|
4
|
-
export * from './
|
|
5
|
-
export type * from './
|
|
6
|
-
export type * from './
|
|
7
|
-
export * from './
|
|
8
|
-
export type { DefaultStorageIdentifier, ExplicitStorageIdentifier, IStorage, StorageIdentifier, } from './
|
|
9
|
-
export { StorageInstanceManager } from './
|
|
10
|
-
export type { DatasetStats, KeyValueStoreStats, RequestQueueStats } from './
|
|
2
|
+
export * from './key-value-store.js';
|
|
3
|
+
export * from './key-value-store-codec.js';
|
|
4
|
+
export * from './request-list.js';
|
|
5
|
+
export type * from './request-loader.js';
|
|
6
|
+
export type * from './request-manager.js';
|
|
7
|
+
export * from './request-queue.js';
|
|
8
|
+
export type { DefaultStorageIdentifier, ExplicitStorageIdentifier, IStorage, StorageIdentifier, } from './storage-instance-manager.js';
|
|
9
|
+
export { StorageInstanceManager } from './storage-instance-manager.js';
|
|
10
|
+
export type { DatasetStats, KeyValueStoreStats, RequestQueueStats } from './storage-stats.js';
|
|
11
11
|
export * from './utils.js';
|
|
12
12
|
export * from './transaction.js';
|
|
13
|
-
export * from './
|
|
13
|
+
export * from './request-manager-tandem.js';
|
package/storages/index.js
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
export * from './dataset.js';
|
|
2
|
-
export * from './
|
|
3
|
-
export * from './
|
|
4
|
-
export * from './
|
|
5
|
-
export * from './
|
|
6
|
-
export { StorageInstanceManager } from './
|
|
2
|
+
export * from './key-value-store.js';
|
|
3
|
+
export * from './key-value-store-codec.js';
|
|
4
|
+
export * from './request-list.js';
|
|
5
|
+
export * from './request-queue.js';
|
|
6
|
+
export { StorageInstanceManager } from './storage-instance-manager.js';
|
|
7
7
|
export * from './utils.js';
|
|
8
8
|
export * from './transaction.js';
|
|
9
|
-
export * from './
|
|
9
|
+
export * from './request-manager-tandem.js';
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { isTypeValid, parse as parseContentType } from 'content-type';
|
|
2
2
|
import JSON5 from 'json5';
|
|
3
3
|
import { jsonStringifyExtended } from '@apify/utilities';
|
|
4
|
-
import { isBuffer, isStream } from '../
|
|
4
|
+
import { isBuffer, isStream } from '../byte-utils.js';
|
|
5
5
|
const CONTENT_TYPE_JSON = 'application/json';
|
|
6
6
|
const STRINGIFIABLE_CONTENT_TYPE_RXS = [new RegExp(`^${CONTENT_TYPE_JSON}$`, 'i'), /^application\/.*xml$/i, /^text\//i];
|
|
7
7
|
/**
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import type { Awaitable, Dictionary, KeyValueStoreBackend, KeyValueStoreInfo } from '@crawlee/types';
|
|
2
2
|
import { Configuration } from '../configuration.js';
|
|
3
3
|
import type { JournalEntry } from './transaction.js';
|
|
4
|
-
import type { KeyValueStoreStats } from './
|
|
4
|
+
import type { KeyValueStoreStats } from './storage-stats.js';
|
|
5
5
|
import type { StorageOpenOptions } from './utils.js';
|
|
6
|
-
import type { StorageIdentifier } from './
|
|
6
|
+
import type { StorageIdentifier } from './storage-instance-manager.js';
|
|
7
7
|
/**
|
|
8
8
|
* The `KeyValueStore` class represents a key-value store, a simple data storage that is used
|
|
9
9
|
* for saving and reading data records or files. Each data record is
|
|
@@ -2,14 +2,14 @@ import { z } from 'zod';
|
|
|
2
2
|
import { KEY_VALUE_STORE_KEY_REGEX } from '@apify/consts';
|
|
3
3
|
import { tryCancel } from '@apify/timeout';
|
|
4
4
|
import { Configuration } from '../configuration.js';
|
|
5
|
-
import { serviceLocator } from '../
|
|
5
|
+
import { serviceLocator } from '../service-locator.js';
|
|
6
6
|
import { parseArgument, schemas, validators } from '../validators.js';
|
|
7
7
|
import { activeStorageTransaction, operationRejectedInTransaction, rejectOperationInTransaction, snapshotValue, withDirectStorageAccess, } from './transaction.js';
|
|
8
|
-
import { parseValue, serializeValue } from './
|
|
9
|
-
import { StorageStatsTracker } from './
|
|
10
|
-
import { resolveStorageIdentifier } from './
|
|
8
|
+
import { parseValue, serializeValue } from './key-value-store-codec.js';
|
|
9
|
+
import { StorageStatsTracker } from './storage-stats.js';
|
|
10
|
+
import { resolveStorageIdentifier } from './storage-instance-manager.js';
|
|
11
11
|
import { createDualIterable, purgeDefaultStorages } from './utils.js';
|
|
12
|
-
import { isBuffer, isStream } from '../
|
|
12
|
+
import { isBuffer, isStream } from '../byte-utils.js';
|
|
13
13
|
/** @internal */
|
|
14
14
|
const KVS_KEYS_DEFAULT_LIMIT = 1000;
|
|
15
15
|
const keySchema = z.string().nonempty();
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import type { BaseHttpClient } from '@crawlee/http-client';
|
|
2
2
|
import type { Dictionary } from '@crawlee/types';
|
|
3
3
|
import type { Configuration } from '../configuration.js';
|
|
4
|
-
import type { IProxyConfiguration } from '../
|
|
4
|
+
import type { IProxyConfiguration } from '../proxy-configuration.js';
|
|
5
5
|
import { Request, type RequestOptions, type Source } from '../request.js';
|
|
6
|
-
import type { IRequestLoader, RequestLoaderStatus } from './
|
|
7
|
-
import type { IRequestManager } from './
|
|
6
|
+
import type { IRequestLoader, RequestLoaderStatus } from './request-loader.js';
|
|
7
|
+
import type { IRequestManager } from './request-manager.js';
|
|
8
8
|
/** @internal */
|
|
9
9
|
export declare const STATE_PERSISTENCE_KEY = "REQUEST_LIST_STATE";
|
|
10
10
|
/** @internal */
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
import { downloadListOfUrls } from '@crawlee/utils';
|
|
2
2
|
import { z } from 'zod';
|
|
3
|
-
import { convertStateSync, RecoverableState } from '../
|
|
3
|
+
import { convertStateSync, RecoverableState } from '../recoverable-state.js';
|
|
4
4
|
import { Request } from '../request.js';
|
|
5
5
|
import { createDeserialize, serializeArray } from '../serialization.js';
|
|
6
|
-
import { serviceLocator } from '../
|
|
6
|
+
import { serviceLocator } from '../service-locator.js';
|
|
7
7
|
import { parseArgument, schemas, validators } from '../validators.js';
|
|
8
|
-
import { KeyValueStore } from './
|
|
8
|
+
import { KeyValueStore } from './key-value-store.js';
|
|
9
9
|
import { purgeDefaultStorages } from './utils.js';
|
|
10
10
|
/** @internal */
|
|
11
11
|
export const STATE_PERSISTENCE_KEY = 'REQUEST_LIST_STATE';
|
|
@@ -570,8 +570,8 @@ export class RequestList {
|
|
|
570
570
|
*/
|
|
571
571
|
async toTandem(requestManager) {
|
|
572
572
|
// Import here to avoid circular imports.
|
|
573
|
-
const { RequestManagerTandem } = await import('./
|
|
574
|
-
const { RequestQueue } = await import('./
|
|
573
|
+
const { RequestManagerTandem } = await import('./request-manager-tandem.js');
|
|
574
|
+
const { RequestQueue } = await import('./request-queue.js');
|
|
575
575
|
return new RequestManagerTandem(this, requestManager ?? (await RequestQueue.open()));
|
|
576
576
|
}
|
|
577
577
|
/**
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import type { Dictionary } from '@crawlee/types';
|
|
2
2
|
import type { Request } from '../request.js';
|
|
3
|
-
import type { IRequestManager } from './
|
|
4
|
-
import type { RequestQueueOperationInfo } from './
|
|
3
|
+
import type { IRequestManager } from './request-manager.js';
|
|
4
|
+
import type { RequestQueueOperationInfo } from './request-queue.js';
|
|
5
5
|
/**
|
|
6
6
|
* A request source's own availability, in a single answer.
|
|
7
7
|
*
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import type { Dictionary } from '@crawlee/types';
|
|
2
2
|
import type { Request, Source } from '../request.js';
|
|
3
|
-
import type { IRequestLoader, RequestSourceStatus } from './
|
|
4
|
-
import type { IRequestManager, PacingSignal, RequestsLike } from './
|
|
5
|
-
import type { AddRequestsBatchedOptions, AddRequestsBatchedResult, RequestQueueOperationInfo, RequestQueueOperationOptions } from './
|
|
3
|
+
import type { IRequestLoader, RequestSourceStatus } from './request-loader.js';
|
|
4
|
+
import type { IRequestManager, PacingSignal, RequestsLike } from './request-manager.js';
|
|
5
|
+
import type { AddRequestsBatchedOptions, AddRequestsBatchedResult, RequestQueueOperationInfo, RequestQueueOperationOptions } from './request-queue.js';
|
|
6
6
|
/**
|
|
7
7
|
* A request manager that combines a {@link IRequestLoader} (such as a `RequestList`) with a writable
|
|
8
8
|
* {@link IRequestManager} (such as a `RequestQueue`).
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { serviceLocator } from '../
|
|
2
|
-
import { joinRequestSourceStatuses } from './
|
|
1
|
+
import { serviceLocator } from '../service-locator.js';
|
|
2
|
+
import { joinRequestSourceStatuses } from './request-loader.js';
|
|
3
3
|
/**
|
|
4
4
|
* A request manager that combines a {@link IRequestLoader} (such as a `RequestList`) with a writable
|
|
5
5
|
* {@link IRequestManager} (such as a `RequestQueue`).
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import type { LiteralUnion } from 'type-fest';
|
|
2
2
|
import type { Request, Source } from '../request.js';
|
|
3
|
-
import type { IRequestLoader } from './
|
|
4
|
-
import type { AddRequestsBatchedOptions, AddRequestsBatchedResult, RequestQueueOperationInfo, RequestQueueOperationOptions } from './
|
|
3
|
+
import type { IRequestLoader } from './request-loader.js';
|
|
4
|
+
import type { AddRequestsBatchedOptions, AddRequestsBatchedResult, RequestQueueOperationInfo, RequestQueueOperationOptions } from './request-queue.js';
|
|
5
5
|
export type RequestsLike = AsyncIterable<Source | string> | Iterable<Source | string> | (Source | string)[];
|
|
6
6
|
/**
|
|
7
7
|
* Extends the read-only {@link IRequestLoader} interface with the capability to enqueue new requests
|
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
import type { BatchAddRequestsResult, Dictionary, ProcessedRequest, QueueOperationInfo, RequestQueueBackend, RequestQueueInfo } from '@crawlee/types';
|
|
2
2
|
import type { ReadonlyDeep } from 'type-fest';
|
|
3
3
|
import type { CrawleeLogger } from '../log.js';
|
|
4
|
-
import type { IProxyConfiguration } from '../
|
|
4
|
+
import type { IProxyConfiguration } from '../proxy-configuration.js';
|
|
5
5
|
import type { Source } from '../request.js';
|
|
6
6
|
import { Request } from '../request.js';
|
|
7
7
|
import type { JournalEntry } from './transaction.js';
|
|
8
|
-
import type { RequestLoaderStatus } from './
|
|
9
|
-
import type { IRequestManager, PacingSignal, RequestsLike } from './
|
|
10
|
-
import type { RequestQueueStats } from './
|
|
11
|
-
import type { IStorage, StorageIdentifier } from './
|
|
8
|
+
import type { RequestLoaderStatus } from './request-loader.js';
|
|
9
|
+
import type { IRequestManager, PacingSignal, RequestsLike } from './request-manager.js';
|
|
10
|
+
import type { RequestQueueStats } from './storage-stats.js';
|
|
11
|
+
import type { IStorage, StorageIdentifier } from './storage-instance-manager.js';
|
|
12
12
|
import type { StorageOpenOptions } from './utils.js';
|
|
13
13
|
/**
|
|
14
14
|
* Represents a queue of URLs to crawl, which is used for deep crawling of websites
|
|
@@ -6,16 +6,16 @@ import { LruCache } from '@apify/datastructures';
|
|
|
6
6
|
import { tryCancel } from '@apify/timeout';
|
|
7
7
|
import { Configuration } from '../configuration.js';
|
|
8
8
|
import { getObjectType } from '../debug.js';
|
|
9
|
-
import { EventType } from '../events/
|
|
9
|
+
import { EventType } from '../events/event-manager.js';
|
|
10
10
|
import { Request } from '../request.js';
|
|
11
|
-
import { serviceLocator } from '../
|
|
11
|
+
import { serviceLocator } from '../service-locator.js';
|
|
12
12
|
import { parseArgument, schemas, validators } from '../validators.js';
|
|
13
13
|
import { activeStorageTransaction, rejectOperationInTransaction } from './transaction.js';
|
|
14
|
-
import { drainRequestBatches } from './
|
|
15
|
-
import { StorageStatsTracker } from './
|
|
16
|
-
import { resolveStorageIdentifier } from './
|
|
14
|
+
import { drainRequestBatches } from './batched-adds.js';
|
|
15
|
+
import { StorageStatsTracker } from './storage-stats.js';
|
|
16
|
+
import { resolveStorageIdentifier } from './storage-instance-manager.js';
|
|
17
17
|
import { getRequestId, purgeDefaultStorages } from './utils.js';
|
|
18
|
-
import { RequestDeduplicationCache } from './
|
|
18
|
+
import { RequestDeduplicationCache } from './request-dedup-cache.js';
|
|
19
19
|
/**
|
|
20
20
|
* The maximum number of requests cached locally to avoid redundant calls to the storage backend.
|
|
21
21
|
* @internal
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { Awaitable, Dictionary, RequestSchema } from '@crawlee/types';
|
|
2
|
-
import type { RecordOptions } from './
|
|
2
|
+
import type { RecordOptions } from './key-value-store.js';
|
|
3
3
|
/**
|
|
4
4
|
* Governs whether writes of a given storage type performed inside a {@link StorageTransaction} are
|
|
5
5
|
* applied immediately (`writeThrough`) or recorded and replayed on commit (`deferred`).
|
package/storages/transaction.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { AsyncLocalStorage } from 'node:async_hooks';
|
|
2
2
|
import { addTimeoutToPromise, storage as timeoutStorage, tryCancel } from '@apify/timeout';
|
|
3
|
-
import { serviceLocator } from '../
|
|
3
|
+
import { serviceLocator } from '../service-locator.js';
|
|
4
4
|
import { AfterCommitError, NonRetryableError } from '../errors.js';
|
|
5
5
|
const DEFAULT_STORAGE_WRITE_POLICY = { requestQueue: 'writeThrough' };
|
|
6
6
|
const DEFAULT_COMMIT_TIMEOUT_MILLIS = 300_000;
|
package/storages/utils.d.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import type { BaseHttpClient } from '@crawlee/http-client';
|
|
2
2
|
import type { Dictionary, StorageBackend } from '@crawlee/types';
|
|
3
3
|
import { Configuration } from '../configuration.js';
|
|
4
|
-
import type { IProxyConfiguration } from '../
|
|
4
|
+
import type { IProxyConfiguration } from '../proxy-configuration.js';
|
|
5
5
|
/**
|
|
6
6
|
* Options for purging default storage.
|
|
7
7
|
*/
|
package/storages/utils.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import crypto from 'node:crypto';
|
|
2
2
|
import { Configuration } from '../configuration.js';
|
|
3
|
-
import { serviceLocator } from '../
|
|
4
|
-
import { KeyValueStore } from './
|
|
3
|
+
import { serviceLocator } from '../service-locator.js';
|
|
4
|
+
import { KeyValueStore } from './key-value-store.js';
|
|
5
5
|
export async function purgeDefaultStorages(configurationOrOptions, storageBackend) {
|
|
6
6
|
const options = configurationOrOptions instanceof Configuration
|
|
7
7
|
? {
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|