@crawlee/core 4.0.0-beta.13 → 4.0.0-beta.130

package/log.js

@@ -1,3 +1,108 @@
 import log, { Log, Logger, LoggerJson, LoggerText, LogLevel } from '@apify/log';
+/**
+ * Abstract base class for custom Crawlee logger implementations.
+ *
+ * Subclasses must implement two methods:
+ * - {@link BaseCrawleeLogger.logWithLevel} — the core logging dispatch
+ * - {@link BaseCrawleeLogger.createChild} — how to create a child logger instance
+ *
+ * All other `CrawleeLogger` methods (`error`, `warning`, `info`, `debug`, etc.)
+ * are derived automatically. Level filtering is entirely the responsibility of the
+ * underlying library — `logWithLevel()` is called for every message.
+ *
+ * **Example — Winston adapter:**
+ * ```typescript
+ * const CRAWLEE_TO_WINSTON = { 1: 'error', 2: 'warn', 3: 'warn', 4: 'info', 5: 'debug', 6: 'debug' };
+ *
+ * class WinstonAdapter extends BaseCrawleeLogger {
+ *     constructor(private logger: winston.Logger, options?: Partial<CrawleeLoggerOptions>) {
+ *         super(options);
+ *     }
+ *
+ *     logWithLevel(level: number, message: string, data?: Record<string, unknown>): void {
+ *         this.logger.log(CRAWLEE_TO_WINSTON[level] ?? 'info', message, data);
+ *     }
+ *
+ *     protected createChild(options: Partial<CrawleeLoggerOptions>): CrawleeLogger {
+ *         return new WinstonAdapter(this.logger.child({ prefix: options.prefix }), { ...this.getOptions(), ...options });
+ *     }
+ * }
+ * ```
+ */
+export class BaseCrawleeLogger {
+    // kept as TS-private: the adaptive crawler's log proxy calls non-intercepted methods with `this === proxy`,
+    // where `#`-field access would throw at runtime
+    options;
+    warningsLogged = new Set();
+    constructor(options = {}) {
+        this.options = options;
+    }
+    getOptions() {
+        return this.options;
+    }
+    setOptions(options) {
+        this.options = { ...this.options, ...options };
+    }
+    child(options) {
+        return this.createChild(options);
+    }
+    error(message, data) {
+        this.logWithLevel(LogLevel.ERROR, message, data);
+    }
+    exception(exception, message, data) {
+        this.logWithLevel(LogLevel.ERROR, `${message}: ${exception.message}`, {
+            ...data,
+            stack: exception.stack,
+            exception,
+        });
+    }
+    softFail(message, data) {
+        this.logWithLevel(LogLevel.SOFT_FAIL, message, data);
+    }
+    warning(message, data) {
+        this.logWithLevel(LogLevel.WARNING, message, data);
+    }
+    warningOnce(message) {
+        if (!this.warningsLogged.has(message)) {
+            this.warningsLogged.add(message);
+            this.warning(message);
+        }
+    }
+    info(message, data) {
+        this.logWithLevel(LogLevel.INFO, message, data);
+    }
+    debug(message, data) {
+        this.logWithLevel(LogLevel.DEBUG, message, data);
+    }
+    perf(message, data) {
+        this.logWithLevel(LogLevel.PERF, `[PERF] ${message}`, data);
+    }
+    deprecated(message) {
+        this.warningOnce(`[DEPRECATED] ${message}`);
+    }
+}
+/**
+ * Adapter that wraps `@apify/log`'s {@link Log} instance to implement the {@link CrawleeLogger} interface.
+ *
+ * This is the default logger used by Crawlee when no custom logger is configured.
+ * Users who want to use a different logging library should implement {@link BaseCrawleeLogger} directly.
+ */
+export class ApifyLogAdapter extends BaseCrawleeLogger {
+    apifyLog;
+    constructor(
+    // kept as a TS-private parameter property: reached through the adaptive crawler's log proxy, see above
+    apifyLog, options) {
+        super(options ?? {});
+        this.apifyLog = apifyLog;
+    }
+    logWithLevel(level, message, data) {
+        this.apifyLog.internal(level, message, data);
+    }
+    createChild(options) {
+        return new ApifyLogAdapter(this.apifyLog.child({ prefix: options.prefix ?? null }), {
+            ...this.getOptions(),
+            ...options,
+        });
+    }
+}
 export { log, Log, LogLevel, Logger, LoggerJson, LoggerText };
-//# sourceMappingURL=log.js.map

package/memory-storage/consts.d.ts

@@ -0,0 +1,4 @@
+/**
+ * Length of id property of a Request instance in characters.
+ */
+export declare const REQUEST_ID_LENGTH = 15;

package/memory-storage/consts.js

@@ -0,0 +1,4 @@
+/**
+ * Length of id property of a Request instance in characters.
+ */
+export const REQUEST_ID_LENGTH = 15;

package/memory-storage/index.d.ts

@@ -0,0 +1 @@
+export * from './memory-storage.js';

package/memory-storage/index.js

@@ -0,0 +1 @@
+export * from './memory-storage.js';

package/memory-storage/memory-storage.d.ts

@@ -0,0 +1,38 @@
+import type * as storage from '@crawlee/types';
+import type { CrawleeLogger } from '@crawlee/types';
+import { DatasetBackend } from './resource-clients/dataset.js';
+import { KeyValueStoreBackend } from './resource-clients/key-value-store.js';
+import { RequestQueueBackend } from './resource-clients/request-queue.js';
+export interface MemoryStorageOptions {
+    /**
+     * Optional logger for MemoryStorageBackend warnings.
+     */
+    logger?: CrawleeLogger;
+}
+export declare class MemoryStorageBackend implements storage.StorageBackend {
+    #private;
+    readonly logger?: CrawleeLogger;
+    readonly keyValueStoreBackendCache: KeyValueStoreBackend[];
+    readonly datasetBackendCache: DatasetBackend[];
+    readonly requestQueueBackendCache: RequestQueueBackend[];
+    constructor(options?: MemoryStorageOptions);
+    /**
+     * Return a per-instance unique cache key so that distinct `MemoryStorageBackend` instances get separate
+     * cache partitions in the storage backend cache.
+     */
+    getStorageBackendCacheKey(): string;
+    createDatasetBackend(options?: storage.StorageIdentifier): Promise<storage.DatasetBackend>;
+    createKeyValueStoreBackend(options?: storage.StorageIdentifier): Promise<storage.KeyValueStoreBackend>;
+    createRequestQueueBackend(options?: storage.StorageIdentifier): Promise<RequestQueueBackend>;
+    storageExists(id: string, type: 'Dataset' | 'KeyValueStore' | 'RequestQueue'): Promise<boolean>;
+    /**
+     * Cleans up the run-scoped storages before the run starts. For the in-memory storage this simply
+     * resets the in-memory state of the cached backends.
+     */
+    purge(): Promise<void>;
+    /**
+     * This method should be called at the end of the process. The in-memory storage holds no resources
+     * that outlive the process (no file handles, no cross-process locks), so there is nothing to do.
+     */
+    teardown(): Promise<void>;
+}

package/memory-storage/memory-storage.js

@@ -0,0 +1,130 @@
+import { randomUUID } from 'node:crypto';
+import { DatasetBackend } from './resource-clients/dataset.js';
+import { KeyValueStoreBackend } from './resource-clients/key-value-store.js';
+import { RequestQueueBackend } from './resource-clients/request-queue.js';
+/** The alias the default (unnamed) storage is opened under. */
+const DEFAULT_STORAGE_ALIAS = '__default__';
+export class MemoryStorageBackend {
+    logger;
+    /**
+     * Unique per-instance cache partition key. Mirrors the way `FileSystemStorageBackend` partitions its
+     * cache by storage directory: two distinct `MemoryStorageBackend` instances must not share cached backends.
+     */
+    #instanceCacheKey = `MemoryStorageBackend:${randomUUID()}`;
+    keyValueStoreBackendCache = [];
+    datasetBackendCache = [];
+    requestQueueBackendCache = [];
+    constructor(options = {}) {
+        this.logger = options.logger;
+    }
+    /**
+     * Return a per-instance unique cache key so that distinct `MemoryStorageBackend` instances get separate
+     * cache partitions in the storage backend cache.
+     */
+    getStorageBackendCacheKey() {
+        return this.#instanceCacheKey;
+    }
+    static #resolveStorageKey(options) {
+        // No identifier at all means the default storage, which is opened under the reserved alias —
+        // same rule as `resolveStorageIdentifier` in the storage frontends, so that a backend used
+        // directly lands on the very storage the frontends would have opened.
+        const alias = options.alias || (!options.id && !options.name ? DEFAULT_STORAGE_ALIAS : undefined);
+        // `alias` covers the identifier-less case, so one of the three is always set.
+        const rawKey = alias ?? options.name ?? options.id;
+        // Normalize the internal __default__ alias to the user-facing 'default' name.
+        const cacheKey = rawKey === DEFAULT_STORAGE_ALIAS ? 'default' : rawKey;
+        return { isAlias: alias !== undefined, cacheKey };
+    }
+    async createDatasetBackend(options = {}) {
+        const { isAlias, cacheKey } = MemoryStorageBackend.#resolveStorageKey(options);
+        const found = this.datasetBackendCache.find((store) => store.id === cacheKey ||
+            store.name?.toLowerCase() === cacheKey.toLowerCase() ||
+            store.cacheKey.toLowerCase() === cacheKey.toLowerCase());
+        if (found) {
+            return found;
+        }
+        const newStore = new DatasetBackend({
+            name: isAlias ? undefined : cacheKey,
+            cacheKey,
+            storageBackend: this,
+        });
+        this.datasetBackendCache.push(newStore);
+        return newStore;
+    }
+    async createKeyValueStoreBackend(options = {}) {
+        const { isAlias, cacheKey } = MemoryStorageBackend.#resolveStorageKey(options);
+        const found = this.keyValueStoreBackendCache.find((store) => store.id === cacheKey ||
+            store.name?.toLowerCase() === cacheKey.toLowerCase() ||
+            store.cacheKey.toLowerCase() === cacheKey.toLowerCase());
+        if (found) {
+            return found;
+        }
+        const newStore = new KeyValueStoreBackend({
+            name: isAlias ? undefined : cacheKey,
+            cacheKey,
+            storageBackend: this,
+        });
+        this.keyValueStoreBackendCache.push(newStore);
+        return newStore;
+    }
+    async createRequestQueueBackend(options = {}) {
+        const { isAlias, cacheKey } = MemoryStorageBackend.#resolveStorageKey(options);
+        const found = this.requestQueueBackendCache.find((queue) => queue.id === cacheKey ||
+            queue.name?.toLowerCase() === cacheKey.toLowerCase() ||
+            queue.cacheKey.toLowerCase() === cacheKey.toLowerCase());
+        if (found) {
+            return found;
+        }
+        const newStore = new RequestQueueBackend({
+            name: isAlias ? undefined : cacheKey,
+            cacheKey,
+            storageBackend: this,
+        });
+        this.requestQueueBackendCache.push(newStore);
+        return newStore;
+    }
+    async storageExists(id, type) {
+        let backends;
+        switch (type) {
+            case 'Dataset':
+                backends = this.datasetBackendCache;
+                break;
+            case 'KeyValueStore':
+                backends = this.keyValueStoreBackendCache;
+                break;
+            case 'RequestQueue':
+                backends = this.requestQueueBackendCache;
+                break;
+            default:
+                return false;
+        }
+        // In-memory storage only knows about backends in its cache.
+        return backends.some((store) => store.id === id);
+    }
+    /**
+     * Cleans up the run-scoped storages before the run starts. For the in-memory storage this simply
+     * resets the in-memory state of the cached backends.
+     */
+    async purge() {
+        // `#resolveStorageKey` leaves `name` unset for the default and alias-keyed storages, which is what
+        // marks them as run-scoped. `'default'` is the exception — it collapses onto the default storage.
+        const isRunScoped = (store) => store.name === undefined || store.name === 'default';
+        const isDefault = (store) => store.name === 'default' || store.cacheKey === 'default';
+        const purgeRunScoped = async (cache, purgeStore) => {
+            await Promise.all(cache.filter(isRunScoped).map(async (store) => purgeStore(store)));
+        };
+        await Promise.all([
+            // Only the default store holds the run input, so it is the only one that keeps `INPUT`.
+            purgeRunScoped(this.keyValueStoreBackendCache, async (store) => isDefault(store) ? store.purgeExceptInput() : store.purge()),
+            purgeRunScoped(this.datasetBackendCache, async (store) => store.purge()),
+            purgeRunScoped(this.requestQueueBackendCache, async (store) => store.purge()),
+        ]);
+    }
+    /**
+     * This method should be called at the end of the process. The in-memory storage holds no resources
+     * that outlive the process (no file handles, no cross-process locks), so there is nothing to do.
+     */
+    async teardown() {
+        // Nothing to tear down for in-memory storage.
+    }
+}

package/memory-storage/resource-clients/common/base-client.d.ts

@@ -0,0 +1,4 @@
+export declare class BaseClient {
+    id: string;
+    constructor(id: string);
+}

package/memory-storage/resource-clients/common/base-client.js

@@ -0,0 +1,6 @@
+export class BaseClient {
+    id;
+    constructor(id) {
+        this.id = id;
+    }
+}

package/memory-storage/resource-clients/dataset.d.ts

@@ -0,0 +1,40 @@
+import type * as storage from '@crawlee/types';
+import type { Dictionary } from '@crawlee/types';
+import type { MemoryStorageBackend } from '../memory-storage.js';
+import { BaseClient } from './common/base-client.js';
+export interface DatasetBackendOptions {
+    id?: string;
+    name?: string;
+    /**
+     * The key used for cache lookup. When provided, takes precedence over `name` and `id`.
+     * This allows alias-opened storages to have a cache key that differs from their
+     * metadata `name` (which is `undefined` for unnamed storages).
+     */
+    cacheKey?: string;
+    storageBackend: MemoryStorageBackend;
+}
+export declare class DatasetBackend<Data extends Dictionary = Dictionary> extends BaseClient implements storage.DatasetBackend<Data> {
+    #private;
+    name?: string;
+    /**
+     * The key used for cache lookup. For named storages, this equals the name. For alias (unnamed)
+     * storages, this is the alias string. Falls back to id.
+     */
+    cacheKey: string;
+    createdAt: Date;
+    accessedAt: Date;
+    modifiedAt: Date;
+    itemCount: number;
+    private readonly storageBackend;
+    constructor(options: DatasetBackendOptions);
+    getMetadata(): Promise<storage.DatasetInfo>;
+    drop(): Promise<void>;
+    purge(): Promise<void>;
+    getData(options?: storage.DatasetBackendListOptions): Promise<storage.PaginatedList<Data>>;
+    private getDataPage;
+    pushData(items: Data[]): Promise<void>;
+    toDatasetInfo(): storage.DatasetInfo;
+    private generateLocalEntryName;
+    private getStartAndEndIndexes;
+    private updateTimestamps;
+}

package/memory-storage/resource-clients/dataset.js

@@ -0,0 +1,108 @@
+import { randomUUID } from 'node:crypto';
+import { parseArgument, schemas } from '@crawlee/utils/internal';
+import { BaseClient } from './common/base-client.js';
+/**
+ * This is what API returns in the x-apify-pagination-limit
+ * header when no limit query parameter is used.
+ */
+const LIST_ITEMS_LIMIT = 999_999_999_999;
+/**
+ * Number of characters of the dataset item entry names.
+ * E.g.: 000000019 - 9 digits
+ */
+const LOCAL_ENTRY_NAME_DIGITS = 9;
+export class DatasetBackend extends BaseClient {
+    name;
+    /**
+     * The key used for cache lookup. For named storages, this equals the name. For alias (unnamed)
+     * storages, this is the alias string. Falls back to id.
+     */
+    cacheKey;
+    createdAt = new Date();
+    accessedAt = new Date();
+    modifiedAt = new Date();
+    itemCount = 0;
+    #datasetEntries = new Map();
+    // kept as TS-private: storage-backend tests read this field at runtime
+    storageBackend;
+    constructor(options) {
+        super(options.id ?? randomUUID());
+        this.name = options.name;
+        this.cacheKey = options.cacheKey ?? this.name ?? this.id;
+        this.storageBackend = options.storageBackend;
+    }
+    async getMetadata() {
+        this.updateTimestamps(false);
+        return this.toDatasetInfo();
+    }
+    async drop() {
+        const storeIndex = this.storageBackend.datasetBackendCache.findIndex((store) => store.id === this.id);
+        if (storeIndex !== -1) {
+            const [oldBackend] = this.storageBackend.datasetBackendCache.splice(storeIndex, 1);
+            oldBackend.itemCount = 0;
+            oldBackend.#datasetEntries.clear();
+        }
+    }
+    async purge() {
+        this.itemCount = 0;
+        this.#datasetEntries.clear();
+        this.updateTimestamps(true);
+    }
+    getData(options = {}) {
+        const { desc, limit, offset } = parseArgument(options, schemas.datasetListItemsOptions);
+        return this.getDataPage({
+            desc,
+            offset: offset ?? 0,
+            limit: Math.min(limit ?? LIST_ITEMS_LIMIT, LIST_ITEMS_LIMIT),
+        });
+    }
+    async getDataPage(options = {}) {
+        const { limit = LIST_ITEMS_LIMIT, offset = 0, desc } = options;
+        const [start, end] = this.getStartAndEndIndexes(desc ? Math.max(this.itemCount - offset - limit, 0) : offset, limit);
+        const items = [];
+        for (let idx = start; idx < end; idx++) {
+            const entryNumber = this.generateLocalEntryName(idx);
+            items.push(this.#datasetEntries.get(entryNumber));
+        }
+        this.updateTimestamps(false);
+        return {
+            count: items.length,
+            desc: desc ?? false,
+            items: desc ? items.reverse() : items,
+            limit,
+            offset,
+            total: this.itemCount,
+        };
+    }
+    async pushData(items) {
+        for (const entry of items) {
+            const idx = this.generateLocalEntryName(++this.itemCount);
+            this.#datasetEntries.set(idx, JSON.parse(JSON.stringify(entry)));
+        }
+        this.updateTimestamps(true);
+    }
+    toDatasetInfo() {
+        return {
+            id: this.id,
+            accessedAt: this.accessedAt,
+            createdAt: this.createdAt,
+            itemCount: this.itemCount,
+            modifiedAt: this.modifiedAt,
+            name: this.name,
+        };
+    }
+    generateLocalEntryName(idx) {
+        return idx.toString().padStart(LOCAL_ENTRY_NAME_DIGITS, '0');
+    }
+    getStartAndEndIndexes(offset, limit = this.itemCount) {
+        const start = offset + 1;
+        const end = Math.min(offset + limit, this.itemCount) + 1;
+        return [start, end];
+    }
+    updateTimestamps(hasBeenModified) {
+        this.accessedAt = new Date();
+        if (hasBeenModified) {
+            this.modifiedAt = new Date();
+        }
+    }
+}

package/memory-storage/resource-clients/key-value-store.d.ts

@@ -0,0 +1,63 @@
+import type * as storage from '@crawlee/types';
+import type { MemoryStorageBackend } from '../memory-storage.js';
+import { BaseClient } from './common/base-client.js';
+export interface KeyValueStoreBackendOptions {
+    name?: string;
+    id?: string;
+    /**
+     * The key used for cache lookup. When provided, takes precedence over `name` and `id`.
+     * This allows alias-opened storages to have a cache key that differs from their
+     * metadata `name` (which is `undefined` for unnamed storages).
+     */
+    cacheKey?: string;
+    storageBackend: MemoryStorageBackend;
+}
+export interface InternalKeyRecord {
+    key: string;
+    value: Buffer;
+    contentType?: string;
+    extension: string;
+}
+export declare class KeyValueStoreBackend extends BaseClient implements storage.KeyValueStoreBackend {
+    #private;
+    name?: string;
+    /**
+     * The key used for cache lookup. For named storages, this equals the name. For alias (unnamed)
+     * storages, this is the alias string. Falls back to id.
+     */
+    cacheKey: string;
+    createdAt: Date;
+    accessedAt: Date;
+    modifiedAt: Date;
+    private readonly storageBackend;
+    constructor(options: KeyValueStoreBackendOptions);
+    getMetadata(): Promise<storage.KeyValueStoreInfo>;
+    drop(): Promise<void>;
+    purge(): Promise<void>;
+    /**
+     * Purges every record except the run's input. Used by {@link MemoryStorageBackend.purge} for the
+     * default key-value store, mirroring `FileSystemStorageBackend`, which preserves `INPUT` (and its
+     * extension variants) when purging the default store. The in-memory key has no extension, so we
+     * preserve the bare `INPUT` key only.
+     */
+    purgeExceptInput(): Promise<void>;
+    listKeys(options?: storage.KeyValueStoreListKeysOptions): Promise<storage.KeyValueStoreListKeysResult>;
+    /**
+     * In-memory records are not file-backed, so there is no public file URL to return.
+     * Always resolves to `undefined`.
+     * @param key The key of the record to generate the public URL for.
+     */
+    getPublicUrl(key: string): Promise<string | undefined>;
+    /**
+     * Tests whether a record with the given key exists in the key-value store without retrieving its value.
+     *
+     * @param key The queried record key.
+     * @returns `true` if the record exists, `false` if it does not.
+     */
+    recordExists(key: string): Promise<boolean>;
+    getValue(key: string): Promise<storage.KeyValueStoreRecord | undefined>;
+    setValue(record: storage.KeyValueStoreInputRecord): Promise<void>;
+    deleteValue(key: string): Promise<void>;
+    toKeyValueStoreInfo(): storage.KeyValueStoreInfo;
+    private updateTimestamps;
+}