@crawlee/core 4.0.0-beta.211 → 4.0.0-beta.213

package/proxy_configuration.d.ts

@@ -38,8 +38,12 @@ export interface IProxyConfiguration {
     /**
      * Creates a new {@link ProxyInfo} object describing the proxy to use.
      * Returns `undefined` when no proxy should be used.
+     *
+     * @param proxyInfo A previously created `ProxyInfo`, e.g. one restored with a persisted session. Implementations
+     *  should return an equivalent `ProxyInfo` that is usable in the current environment, or the argument itself when
+     *  there is nothing to refresh.
      */
-    newProxyInfo(): Promise<ProxyInfo | undefined>;
+    newProxyInfo(proxyInfo?: ProxyInfo): Promise<ProxyInfo | undefined>;
 }
 /**
  * Configures connection to a proxy server with the provided options. Proxy servers are used to prevent target websites from blocking
@@ -100,9 +104,10 @@ export declare class ProxyConfiguration implements IProxyConfiguration {
      * Use it if you want to work with a rich representation of a proxy URL.
      * If you need the URL string only, use {@link ProxyConfiguration.newUrl}.
      *
+     * @param proxyInfo A previously created `ProxyInfo`, returned unchanged.
      * @return Represents information about used proxy and its configuration.
      */
-    newProxyInfo(): Promise<ProxyInfo | undefined>;
+    newProxyInfo(proxyInfo?: ProxyInfo): Promise<ProxyInfo | undefined>;
     /**
      * Returns a new proxy URL based on provided configuration options.
      *
@@ -110,11 +115,4 @@ export declare class ProxyConfiguration implements IProxyConfiguration {
      *  For example, `http://bob:password123@proxy.example.com:8000`
      */
     newUrl(): Promise<string | undefined>;
-    private handleProxyUrlsList;
-    /**
-     * Calls the custom newUrlFunction and checks format of its return value
-     */
-    private callNewUrlFunction;
-    private throwCannotCombineCustomMethods;
-    private throwNoOptionsProvided;
 }

package/proxy_configuration.js

@@ -71,9 +71,9 @@ export class ProxyConfiguration {
         }
         const { proxyUrls, newUrlFunction } = parseArgument(rest, proxyConfigurationOptionsSchema);
         if (proxyUrls && newUrlFunction)
-            this.throwCannotCombineCustomMethods();
+            this.#throwCannotCombineCustomMethods();
         if (!proxyUrls && !newUrlFunction && validateRequired)
-            this.throwNoOptionsProvided();
+            this.#throwNoOptionsProvided();
         this.#proxyUrls = proxyUrls;
         this.#newUrlFunction = newUrlFunction;
     }
@@ -84,9 +84,12 @@ export class ProxyConfiguration {
      * Use it if you want to work with a rich representation of a proxy URL.
      * If you need the URL string only, use {@link ProxyConfiguration.newUrl}.
      *
+     * @param proxyInfo A previously created `ProxyInfo`, returned unchanged.
      * @return Represents information about used proxy and its configuration.
      */
-    async newProxyInfo() {
+    async newProxyInfo(proxyInfo) {
+        if (proxyInfo)
+            return proxyInfo;
         const url = await this.newUrl();
         if (!url)
             return undefined;
@@ -107,17 +110,17 @@ export class ProxyConfiguration {
      */
     async newUrl() {
         if (this.#newUrlFunction) {
-            return (await this.callNewUrlFunction()) ?? undefined;
+            return (await this.#callNewUrlFunction()) ?? undefined;
         }
-        return this.handleProxyUrlsList() ?? undefined;
+        return this.#handleProxyUrlsList() ?? undefined;
     }
-    handleProxyUrlsList() {
+    #handleProxyUrlsList() {
         return this.#proxyUrls[this.#nextCustomUrlIndex++ % this.#proxyUrls.length];
     }
     /**
      * Calls the custom newUrlFunction and checks format of its return value
      */
-    async callNewUrlFunction() {
+    async #callNewUrlFunction() {
         const proxyUrl = await this.#newUrlFunction();
         try {
             if (proxyUrl) {
@@ -129,10 +132,10 @@ export class ProxyConfiguration {
             throw new Error(`The provided newUrlFunction did not return a valid URL.\nCause: ${err.message}`);
         }
     }
-    throwCannotCombineCustomMethods() {
+    #throwCannotCombineCustomMethods() {
         throw new Error('Cannot combine custom proxies "options.proxyUrls" with custom generating function "options.newUrlFunction".');
     }
-    throwNoOptionsProvided() {
+    #throwNoOptionsProvided() {
         throw new Error('One of "options.proxyUrls" or "options.newUrlFunction" needs to be provided.');
     }
 }

package/storages/dataset.d.ts

@@ -169,14 +169,6 @@ export declare class Dataset<Data extends Dictionary = Dictionary> {
      * Returns {@link DatasetContent} object holding the items in the dataset based on the provided parameters.
      */
     getData(options?: DatasetDataOptions): Promise<DatasetContent<Data>>;
-    /**
-     * The single transaction-aware page read all dataset read paths go through — both `getData()` and
-     * the private `fetchPages()`. Returns the real page concatenated with the current transaction's
-     * buffered items, with `offset` / `limit` / `desc` windowing applied across the concatenation.
-     */
-    private readPage;
-    /** The active transaction's buffered writes to this dataset, derived from its journal. */
-    private bufferedJournalEntries;
     /** @internal */
     commitJournalEntries(entries: JournalEntry[]): Promise<void>;
     /**
@@ -321,8 +313,6 @@ export declare class Dataset<Data extends Dictionary = Dictionary> {
      * @param [options] An object containing extra options for `reduce()`
      */
     reduce<T>(iteratee: DatasetReducer<T, Data>, memo: T, options?: DatasetIteratorOptions): Promise<T>;
-    private fetchEntryPages;
-    private fetchPages;
     /**
      * Returns dataset items.
      *

package/storages/dataset.js

@@ -91,7 +91,7 @@ export class Dataset {
     configuration;
     id;
     name;
-    // kept as TS-private: dataset tests spy on the backend directly
+    // oxlint-disable-next-line crawlee/prefer-private-fields -- tests spy on the backend directly
     backend;
     #statsTracker = new StorageStatsTracker({
         readCount: 0,
@@ -152,7 +152,7 @@ export class Dataset {
      */
     async getData(options = {}) {
         try {
-            return await this.readPage(options);
+            return await this.#readPage(options);
         }
         catch (e) {
             const error = e;
@@ -167,8 +167,8 @@ export class Dataset {
      * the private `fetchPages()`. Returns the real page concatenated with the current transaction's
      * buffered items, with `offset` / `limit` / `desc` windowing applied across the concatenation.
      */
-    async readPage(options) {
-        const buffered = this.bufferedJournalEntries()?.flatMap((entry) => entry.items);
+    async #readPage(options) {
+        const buffered = this.#bufferedJournalEntries()?.flatMap((entry) => entry.items);
         // Every branch below hits the backend exactly once.
         this.#statsTracker.add('readCount');
         if (!buffered?.length) {
@@ -224,7 +224,7 @@ export class Dataset {
         };
     }
     /** The active transaction's buffered writes to this dataset, derived from its journal. */
-    bufferedJournalEntries() {
+    #bufferedJournalEntries() {
         const transaction = activeStorageTransaction();
         return transaction?.journal.filter((entry) => entry.type === 'dataset' && entry.participant === this);
     }
@@ -250,7 +250,7 @@ export class Dataset {
     async export(options = {}) {
         tryCancel();
         const items = [];
-        for await (const page of this.fetchPages(options)) {
+        for await (const page of this.#fetchPages(options)) {
             items.push(...page.items);
         }
         return items;
@@ -348,7 +348,7 @@ export class Dataset {
      * @throws If the underlying storage no longer exists (e.g. it was deleted externally).
      */
     async getInfo() {
-        const buffered = this.bufferedJournalEntries();
+        const buffered = this.#bufferedJournalEntries();
         const metadata = await this.backend.getMetadata();
         if (buffered?.length) {
             const lastWriteAt = buffered[buffered.length - 1].recordedAt;
@@ -432,16 +432,16 @@ export class Dataset {
         await this.forEach(wrappedFunc, options);
         return currentMemo;
     }
-    async *fetchEntryPages(options) {
+    async *#fetchEntryPages(options) {
         let index = options.offset ?? 0;
-        for await (const page of this.fetchPages(options)) {
+        for await (const page of this.#fetchPages(options)) {
             yield {
                 ...page,
                 items: page.items.map((item) => [index++, item]),
             };
         }
     }
-    async *fetchPages(options, pageSize = DATASET_ITERATORS_DEFAULT_LIMIT) {
+    async *#fetchPages(options, pageSize = DATASET_ITERATORS_DEFAULT_LIMIT) {
         let offset = options.offset ?? 0;
         const totalLimit = options.limit;
         let yielded = 0;
@@ -449,7 +449,7 @@ export class Dataset {
             const fetchLimit = totalLimit !== undefined ? Math.min(pageSize, totalLimit - yielded) : pageSize;
             if (fetchLimit <= 0)
                 break;
-            const page = await this.readPage({ ...options, offset, limit: fetchLimit });
+            const page = await this.#readPage({ ...options, offset, limit: fetchLimit });
             yield page;
             yielded += page.items.length;
             if (page.items.length < fetchLimit || offset + page.items.length >= page.total)
@@ -483,7 +483,7 @@ export class Dataset {
     values(options = {}) {
         tryCancel();
         return createDualIterable({
-            createPages: () => this.fetchPages(options),
+            createPages: () => this.#fetchPages(options),
             extractItems: (page) => page.items,
         });
     }
@@ -513,7 +513,7 @@ export class Dataset {
     entries(options = {}) {
         tryCancel();
         return createDualIterable({
-            createPages: () => this.fetchEntryPages(options),
+            createPages: () => this.#fetchEntryPages(options),
             extractItems: (page) => page.items,
         });
     }

package/storages/key_value_store.d.ts

@@ -136,21 +136,6 @@ export declare class KeyValueStore {
      *   on the MIME content type of the record, or the default value if the key is missing from the store.
      */
     getValue<T = unknown>(key: string, defaultValue: T): Promise<T>;
-    /**
-     * The active transaction's last buffered write per key for this store, derived from its journal.
-     * An entry with a `null` value is a tombstone (an in-transaction deletion).
-     */
-    private bufferedJournalEntries;
-    /**
-     * The single transaction-aware record read shared by `getValue`, `getRecord`, `recordExists` and the
-     * listing paths: buffered key → serialized through the standard codec (same fidelity as a real
-     * round-trip); tombstoned key → `null`; otherwise the backend.
-     *
-     * The per-key buffered lookup requires the whole journal to be reduced to a last-write-per-key map,
-     * which is O(journal). Single-record callers let it default (rebuilt per call); the listing paths,
-     * which read many keys, pass a map built once so the read stays O(1) per key instead of O(journal).
-     */
-    private readRecord;
     /**
      * Reads a record from the key-value store without parsing the value.
      *
@@ -186,9 +171,6 @@ export declare class KeyValueStore {
      */
     recordExists(key: string): Promise<boolean>;
     getAutoSavedValue<T extends Dictionary = Dictionary>(key: string, defaultValue?: T): Promise<T>;
-    private ensurePersistStateEvent;
-    private fetchKeyValuePages;
-    private fetchKeyPages;
     /**
      * Saves or deletes a record in the key-value store.
      * The function returns a promise that resolves once the record has been saved or deleted.

package/storages/key_value_store.js

@@ -144,7 +144,7 @@ export class KeyValueStore {
     async getValue(key, defaultValue) {
         tryCancel();
         parseArgument(key, keySchema);
-        const record = await this.readRecord(key);
+        const record = await this.#readRecord(key);
         // A missing record falls back to the default; a record that parses to a falsy value (including
         // a stored literal `null`) is returned verbatim, so callers can tell "stored null" from "absent".
         if (!record) {
@@ -157,7 +157,7 @@ export class KeyValueStore {
      * The active transaction's last buffered write per key for this store, derived from its journal.
      * An entry with a `null` value is a tombstone (an in-transaction deletion).
      */
-    bufferedJournalEntries() {
+    #bufferedJournalEntries() {
         const transaction = activeStorageTransaction();
         if (!transaction)
             return undefined;
@@ -178,7 +178,7 @@ export class KeyValueStore {
      * which is O(journal). Single-record callers let it default (rebuilt per call); the listing paths,
      * which read many keys, pass a map built once so the read stays O(1) per key instead of O(journal).
      */
-    async readRecord(key, buffered = this.bufferedJournalEntries()) {
+    async #readRecord(key, buffered = this.#bufferedJournalEntries()) {
         const entry = buffered?.get(key);
         if (entry) {
             if (entry.value === null) {
@@ -228,7 +228,7 @@ export class KeyValueStore {
     async getRecord(key) {
         tryCancel();
         parseArgument(key, keySchema);
-        return this.readRecord(key);
+        return this.#readRecord(key);
     }
     /**
      * Tests whether a record with the given key exists in the key-value store without retrieving its value.
@@ -239,7 +239,7 @@ export class KeyValueStore {
     async recordExists(key) {
         tryCancel();
         parseArgument(key, keySchema);
-        const entry = this.bufferedJournalEntries()?.get(key);
+        const entry = this.#bufferedJournalEntries()?.get(key);
         if (entry) {
             return entry.value !== null;
         }
@@ -261,10 +261,10 @@ export class KeyValueStore {
             return this.#cache.get(key);
         }
         this.#cache.set(key, value);
-        this.ensurePersistStateEvent();
+        this.#ensurePersistStateEvent();
         return value;
     }
-    ensurePersistStateEvent() {
+    #ensurePersistStateEvent() {
         if (this.#persistStateEventStarted) {
             return;
         }
@@ -277,15 +277,15 @@ export class KeyValueStore {
         });
         this.#persistStateEventStarted = true;
     }
-    async *fetchKeyValuePages(options, mapRecord) {
+    async *#fetchKeyValuePages(options, mapRecord) {
         // Reduce the journal once for the whole iteration, not once per key inside `readRecord`.
-        const buffered = this.bufferedJournalEntries();
-        for await (const page of this.fetchKeyPages(options, buffered)) {
+        const buffered = this.#bufferedJournalEntries();
+        for await (const page of this.#fetchKeyPages(options, buffered)) {
             const results = [];
             for (const item of page) {
                 // The shared transaction-aware read, so a key that exists only in the transaction resolves
                 // here instead of being dropped (`values()` would disagree with `keys()` on length).
-                const record = await this.readRecord(item.key, buffered);
+                const record = await this.#readRecord(item.key, buffered);
                 if (record) {
                     const parsed = parseValue(record.value, record.contentType ?? null);
                     results.push(mapRecord(item.key, parsed));
@@ -294,7 +294,7 @@ export class KeyValueStore {
             yield results;
         }
     }
-    async *fetchKeyPages(options, buffered = this.bufferedJournalEntries(), limit = KVS_KEYS_DEFAULT_LIMIT) {
+    async *#fetchKeyPages(options, buffered = this.#bufferedJournalEntries(), limit = KVS_KEYS_DEFAULT_LIMIT) {
         // Buffered keys are emitted first, then the real pages with any buffered (or tombstoned) key
         // skipped - a merge-join is not an option, since `listKeys` promises no sort order.
         const shadowedKeys = new Set();
@@ -497,7 +497,7 @@ export class KeyValueStore {
         parseArgument(iteratee, schemas.anyFunction);
         const parsedOptions = parseArgument(options, iteratorOptionsSchema);
         let index = 0;
-        for await (const page of this.fetchKeyPages(parsedOptions)) {
+        for await (const page of this.#fetchKeyPages(parsedOptions)) {
             for (const item of page) {
                 await iteratee(item.key, index++, { size: item.size });
             }
@@ -529,7 +529,7 @@ export class KeyValueStore {
     keys(options = {}) {
         tryCancel();
         return createDualIterable({
-            createPages: () => this.fetchKeyPages(options),
+            createPages: () => this.#fetchKeyPages(options),
             extractItems: (page) => page.map((item) => item.key),
         });
     }
@@ -559,7 +559,7 @@ export class KeyValueStore {
     values(options = {}) {
         tryCancel();
         return createDualIterable({
-            createPages: () => this.fetchKeyValuePages(options, (_key, value) => value),
+            createPages: () => this.#fetchKeyValuePages(options, (_key, value) => value),
             extractItems: (page) => page,
         });
     }
@@ -589,7 +589,7 @@ export class KeyValueStore {
     entries(options = {}) {
         tryCancel();
         return createDualIterable({
-            createPages: () => this.fetchKeyValuePages(options, (key, value) => [key, value]),
+            createPages: () => this.#fetchKeyValuePages(options, (key, value) => [key, value]),
             extractItems: (page) => page,
         });
     }

package/storages/request_dedup_cache.d.ts

@@ -13,10 +13,8 @@
  */
 export declare class RequestDeduplicationCache {
     #private;
-    private readonly size;
     constructor(size?: number);
     get(cacheKey: string): string | null;
     add(cacheKey: string, requestId: string): void;
     clear(): void;
-    private indexOf;
 }

package/storages/request_dedup_cache.js

@@ -12,37 +12,37 @@
  * @internal
  */
 export class RequestDeduplicationCache {
-    size;
     #keys;
     #ids;
+    #size;
     // The slot count is the same for every queue, so it's a fixed default rather than a per-consumer option.
     constructor(size = 1_000_000) {
-        this.size = size;
+        this.#size = size;
         this.#keys = new Array(size);
         this.#ids = new Array(size);
     }
     get(cacheKey) {
-        const index = this.indexOf(cacheKey);
+        const index = this.#indexOf(cacheKey);
         return this.#keys[index] === cacheKey ? this.#ids[index] : null;
     }
     add(cacheKey, requestId) {
-        const index = this.indexOf(cacheKey);
+        const index = this.#indexOf(cacheKey);
         this.#keys[index] = cacheKey;
         this.#ids[index] = requestId;
     }
     clear() {
-        this.#keys = new Array(this.size);
-        this.#ids = new Array(this.size);
+        this.#keys = new Array(this.#size);
+        this.#ids = new Array(this.#size);
     }
     // A cheap FNV-1a hash of the cache key — avoids pulling in a dedicated hashing dependency.
-    indexOf(cacheKey) {
+    #indexOf(cacheKey) {
         /* eslint-disable no-bitwise */
         let hash = 0x811c9dc5;
         for (let i = 0; i < cacheKey.length; i++) {
             hash ^= cacheKey.charCodeAt(i);
             hash = Math.imul(hash, 0x01000193);
         }
-        return (hash >>> 0) % this.size;
+        return (hash >>> 0) % this.#size;
         /* eslint-enable no-bitwise */
     }
 }

package/storages/request_list.d.ts

@@ -241,24 +241,6 @@ export declare class RequestList implements IRequestLoader {
      * @internal
      */
     private constructor();
-    /**
-     * Loads all remote sources of URLs and potentially starts periodic state persistence.
-     * This function must be called before you can start using the instance in a meaningful way.
-     */
-    private initialize;
-    /**
-     * Adds previously persisted Requests, as retrieved from the key-value store.
-     * This needs to be done in a memory efficient way. We should update the input
-     * to a Stream once apify-client supports streams.
-     */
-    private addPersistedRequests;
-    /**
-     * Add Requests from both options.sources and options.sourcesFunction.
-     * This function is called only when persisted sources were not loaded.
-     * We need to avoid keeping both sources and requests in memory
-     * to reduce memory footprint with very large sources.
-     */
-    private addRequestsFromSources;
     /**
      * @inheritDoc
      */
@@ -269,12 +251,6 @@ export declare class RequestList implements IRequestLoader {
      * leaking the listener (and the requests it retains) on the shared event manager.
      */
     teardown(): Promise<void>;
-    /**
-     * Unlike persistState(), this is used only internally, since the sources
-     * are automatically persisted at RequestList initialization (if the persistRequestsKey is set),
-     * but there's no reason to persist it again afterwards, because RequestList is immutable.
-     */
-    private persistRequests;
     /**
      * Returns an object representing the internal state of the `RequestList` instance.
      * Note that the object's fields can change in future releases.
@@ -292,39 +268,10 @@ export declare class RequestList implements IRequestLoader {
      * @inheritDoc
      */
     [Symbol.asyncIterator](): AsyncGenerator<Request<Dictionary>, void, unknown>;
-    private ensureRequest;
     /**
      * @inheritDoc
      */
     markRequestAsHandled(request: Request): Promise<void>;
-    /**
-     * Adds all fetched requests from a URL from a remote resource.
-     */
-    private addFetchedRequests;
-    private getPersistedState;
-    /**
-     * Fetches URLs from requestsFromUrl and returns them in format of list of requests
-     */
-    private fetchRequestsFromUrl;
-    /**
-     * Adds given request.
-     * If the `source` parameter is a string or plain object and not an instance
-     * of a `Request`, then the function creates a `Request` instance.
-     */
-    private addRequest;
-    /**
-     * Helper function that validates unique key.
-     * Throws an error if uniqueKey is not a non-empty string.
-     */
-    private ensureUniqueKeyValid;
-    /**
-     * Checks that a request is currently being processed and throws an error if not.
-     */
-    private ensureInProgress;
-    /**
-     * Throws an error if request list wasn't initialized.
-     */
-    private ensureIsInitialized;
     /**
      * Returns the total number of unique requests present in the `RequestList`.
      */
@@ -397,10 +344,6 @@ export declare class RequestList implements IRequestLoader {
      *   options and the `sources` parameter supersedes the {@link RequestListOptions.sources} option.
      */
     static open(listNameOrOptions: string | null | RequestListOptions, sources?: RequestListSource[], options?: RequestListOptions): Promise<RequestList>;
-    /**
-     * @internal wraps public utility for mocking purposes
-     */
-    private downloadListOfUrls;
 }
 /**
  * Represents state of a {@link RequestList}. It can be used to resume a {@link RequestList} which has been previously processed.