@crawlee/core 4.0.0-beta.99 → 4.0.0-rc.1

package/storages/dataset.js

@@ -1,12 +1,17 @@
-import { stringify } from 'csv-stringify/sync';
-import ow from 'ow';
+import { z } from 'zod';
+import { tryCancel } from '@apify/timeout';
 import { Configuration } from '../configuration.js';
 import { serviceLocator } from '../service_locator.js';
-import { checkStorageAccess } from './access_checking.js';
+import { parseArgument, schemas, validators } from '../validators.js';
+import { activeStorageTransaction, rejectOperationInTransaction, snapshotValue } from './transaction.js';
 import { KeyValueStore } from './key_value_store.js';
 import { StorageStatsTracker } from './storage_stats.js';
 import { resolveStorageIdentifier } from './storage_instance_manager.js';
 import { createDualIterable, purgeDefaultStorages } from './utils.js';
+const openOptionsSchema = z.strictObject({
+    configuration: z.instanceof(Configuration).optional(),
+    storageBackend: validators.storageBackend.optional(),
+});
 /** @internal */
 export const DATASET_ITERATORS_DEFAULT_LIMIT = 10000;
 /**
@@ -86,9 +91,9 @@ export class Dataset {
     configuration;
     id;
     name;
+    // oxlint-disable-next-line crawlee/prefer-private-fields -- tests spy on the backend directly
     backend;
-    log;
-    statsTracker = new StorageStatsTracker({
+    #statsTracker = new StorageStatsTracker({
         readCount: 0,
         writeCount: 0,
     });
@@ -100,14 +105,13 @@ export class Dataset {
         this.id = options.metadata.id;
         this.name = options.metadata.name;
         this.backend = options.backend;
-        this.log = serviceLocator.getLogger().child({ prefix: 'Dataset' });
     }
     /**
      * Backend-independent usage counters tracked for this dataset (read / write operations issued to
      * the underlying storage backend). Counted per backend call.
      */
     get stats() {
-        return this.statsTracker.current;
+        return this.#statsTracker.current;
     }
     /**
      * Stores an object or an array of objects to the dataset.
@@ -121,24 +125,34 @@ export class Dataset {
      *   The objects must be serializable to JSON.
      */
     async pushData(data) {
-        checkStorageAccess();
-        ow(data, 'data', ow.object);
+        tryCancel();
+        const transaction = activeStorageTransaction();
+        parseArgument(data, schemas.anyObject);
         // Normalize to array and validate each item
         const items = Array.isArray(data) ? data : [data];
         for (let i = 0; i < items.length; i++) {
             assertJsonSerializable(items[i], i);
         }
-        this.statsTracker.add('writeCount');
+        if (transaction) {
+            // One snapshot serves both the reads and the commit replay, so the two cannot disagree.
+            transaction.recordJournalEntry({
+                type: 'dataset',
+                participant: this,
+                storageId: this.id,
+                items: snapshotValue(items),
+                recordedAt: new Date(),
+            });
+            return;
+        }
+        this.#statsTracker.add('writeCount');
         await this.backend.pushData(items);
     }
     /**
      * Returns {@link DatasetContent} object holding the items in the dataset based on the provided parameters.
      */
     async getData(options = {}) {
-        checkStorageAccess();
         try {
-            this.statsTracker.add('readCount');
-            return await this.backend.getData(options);
+            return await this.#readPage(options);
         }
         catch (e) {
             const error = e;
@@ -148,14 +162,95 @@ export class Dataset {
             throw e;
         }
     }
+    /**
+     * The single transaction-aware page read all dataset read paths go through — both `getData()` and
+     * the private `fetchPages()`. Returns the real page concatenated with the current transaction's
+     * buffered items, with `offset` / `limit` / `desc` windowing applied across the concatenation.
+     */
+    async #readPage(options) {
+        const buffered = this.#bufferedJournalEntries()?.flatMap((entry) => entry.items);
+        // Every branch below hits the backend exactly once.
+        this.#statsTracker.add('readCount');
+        if (!buffered?.length) {
+            return this.backend.getData(options);
+        }
+        const { offset = 0, limit, desc = false } = options;
+        if (!desc) {
+            const realPage = await this.backend.getData(options);
+            // Buffered items sit past `realPage.total`, so the window bounds must come from that - not
+            // from the page's shortfall, which `skipEmpty` produces without exhausting the real items.
+            const bufferedStart = Math.max(0, offset - realPage.total);
+            const bufferedEnd = limit === undefined ? buffered.length : Math.max(0, offset + limit - realPage.total);
+            const items = [...realPage.items, ...buffered.slice(bufferedStart, bufferedEnd)];
+            return {
+                items,
+                total: realPage.total + buffered.length,
+                offset,
+                // A caller that passed no limit wants everything, so the backend-reported `limit` is
+                // passed through - backends are free to report a page size or a sentinel there.
+                limit: limit ?? realPage.limit,
+                count: items.length,
+                desc,
+            };
+        }
+        // Descending order: the buffered items are the newest, so they come first, reversed.
+        const reversedBuffer = [...buffered].reverse();
+        const fromBuffer = limit === undefined ? reversedBuffer.slice(offset) : reversedBuffer.slice(offset, offset + limit);
+        const needed = limit === undefined ? Infinity : limit - fromBuffer.length;
+        if (needed <= 0) {
+            // The whole window is served from the buffer; only the real total is missing.
+            const { itemCount } = await this.backend.getMetadata();
+            return {
+                items: fromBuffer,
+                total: itemCount + buffered.length,
+                offset,
+                limit: limit,
+                count: fromBuffer.length,
+                desc,
+            };
+        }
+        const realPage = await this.backend.getData({
+            ...options,
+            offset: Math.max(0, offset - buffered.length),
+            ...(limit === undefined ? {} : { limit: needed }),
+        });
+        return {
+            items: [...fromBuffer, ...realPage.items],
+            total: realPage.total + buffered.length,
+            offset,
+            limit: limit ?? realPage.limit,
+            count: fromBuffer.length + realPage.items.length,
+            desc,
+        };
+    }
+    /** The active transaction's buffered writes to this dataset, derived from its journal. */
+    #bufferedJournalEntries() {
+        const transaction = activeStorageTransaction();
+        return transaction?.journal.filter((entry) => entry.type === 'dataset' && entry.participant === this);
+    }
+    /** @internal */
+    async commitJournalEntries(entries) {
+        const items = [];
+        for (const entry of entries) {
+            if (entry.type === 'dataset') {
+                items.push(...entry.items);
+            }
+        }
+        // One backend call with all journaled items, in order - as close to atomic as the backend allows.
+        // Straight to the backend: the items were validated and snapshotted at write time.
+        if (items.length > 0) {
+            this.#statsTracker.add('writeCount');
+            await this.backend.pushData(items);
+        }
+    }
     /**
      * Returns all the data from the dataset. This will iterate through the whole dataset
      * via the `listItems()` client method, which gives you only paginated results.
      */
     async export(options = {}) {
-        checkStorageAccess();
+        tryCancel();
         const items = [];
-        for await (const page of this.fetchPages(options)) {
+        for await (const page of this.#fetchPages(options)) {
             items.push(...page.items);
         }
         return items;
@@ -179,6 +274,7 @@ export class Dataset {
             const keys = options?.collectAllKeys
                 ? Array.from(new Set(items.flatMap(Object.keys)))
                 : Object.keys(items[0]);
+            const { stringify } = await import('csv-stringify/sync');
             const value = stringify([
                 keys,
                 ...items.map((item) => {
@@ -219,7 +315,7 @@ export class Dataset {
      * @param [options] An optional options object where you can provide the dataset and target KVS name.
      */
     static async exportToJSON(key, options) {
-        checkStorageAccess();
+        tryCancel();
         const dataset = await this.open(options?.fromDataset);
         await dataset.exportToJSON(key, options);
     }
@@ -230,7 +326,7 @@ export class Dataset {
      * @param [options] An optional options object where you can provide the dataset and target KVS name.
      */
     static async exportToCSV(key, options) {
-        checkStorageAccess();
+        tryCancel();
         const dataset = await this.open(options?.fromDataset);
         await dataset.exportToCSV(key, options);
     }
@@ -252,8 +348,17 @@ export class Dataset {
      * @throws If the underlying storage no longer exists (e.g. it was deleted externally).
      */
     async getInfo() {
-        checkStorageAccess();
-        return this.backend.getMetadata();
+        const buffered = this.#bufferedJournalEntries();
+        const metadata = await this.backend.getMetadata();
+        if (buffered?.length) {
+            const lastWriteAt = buffered[buffered.length - 1].recordedAt;
+            return {
+                ...metadata,
+                itemCount: metadata.itemCount + buffered.reduce((sum, entry) => sum + entry.items.length, 0),
+                modifiedAt: metadata.modifiedAt > lastWriteAt ? metadata.modifiedAt : lastWriteAt,
+            };
+        }
+        return metadata;
     }
     /**
      * Iterates over dataset items, yielding each in turn to an `iteratee` function.
@@ -276,7 +381,7 @@ export class Dataset {
      * @default 0
      */
     async forEach(iteratee, options = {}, index = 0) {
-        checkStorageAccess();
+        tryCancel();
         if (!options.offset)
             options.offset = 0;
         if (options.format && options.format !== 'json')
@@ -303,7 +408,7 @@ export class Dataset {
      * @param [options] All `map()` parameters.
      */
     async map(iteratee, options = {}) {
-        checkStorageAccess();
+        tryCancel();
         const result = [];
         await this.forEach(async (item, index) => {
             const res = await iteratee(item, index);
@@ -312,7 +417,7 @@ export class Dataset {
         return result;
     }
     async reduce(iteratee, memo, options = {}) {
-        checkStorageAccess();
+        tryCancel();
         let currentMemo = memo;
         const wrappedFunc = async (item, index) => {
             if (index === 0 && currentMemo === undefined) {
@@ -327,16 +432,16 @@ export class Dataset {
         await this.forEach(wrappedFunc, options);
         return currentMemo;
     }
-    async *fetchEntryPages(options) {
+    async *#fetchEntryPages(options) {
         let index = options.offset ?? 0;
-        for await (const page of this.fetchPages(options)) {
+        for await (const page of this.#fetchPages(options)) {
             yield {
                 ...page,
                 items: page.items.map((item) => [index++, item]),
             };
         }
     }
-    async *fetchPages(options, pageSize = DATASET_ITERATORS_DEFAULT_LIMIT) {
+    async *#fetchPages(options, pageSize = DATASET_ITERATORS_DEFAULT_LIMIT) {
         let offset = options.offset ?? 0;
         const totalLimit = options.limit;
         let yielded = 0;
@@ -344,8 +449,7 @@ export class Dataset {
             const fetchLimit = totalLimit !== undefined ? Math.min(pageSize, totalLimit - yielded) : pageSize;
             if (fetchLimit <= 0)
                 break;
-            this.statsTracker.add('readCount');
-            const page = await this.backend.getData({ ...options, offset, limit: fetchLimit });
+            const page = await this.#readPage({ ...options, offset, limit: fetchLimit });
             yield page;
             yielded += page.items.length;
             if (page.items.length < fetchLimit || offset + page.items.length >= page.total)
@@ -377,9 +481,9 @@ export class Dataset {
      * @param options Options for the iteration.
      */
     values(options = {}) {
-        checkStorageAccess();
+        tryCancel();
         return createDualIterable({
-            createPages: () => this.fetchPages(options),
+            createPages: () => this.#fetchPages(options),
             extractItems: (page) => page.items,
         });
     }
@@ -407,9 +511,9 @@ export class Dataset {
      * @param options Options for the iteration.
      */
     entries(options = {}) {
-        checkStorageAccess();
+        tryCancel();
         return createDualIterable({
-            createPages: () => this.fetchEntryPages(options),
+            createPages: () => this.#fetchEntryPages(options),
             extractItems: (page) => page.items,
         });
     }
@@ -433,10 +537,18 @@ export class Dataset {
      * depending on the mode of operation.
      */
     async drop() {
-        checkStorageAccess();
+        rejectOperationInTransaction('Dataset.drop()');
         await this.backend.drop();
         serviceLocator.getStorageInstanceManager().removeFromCache(this);
     }
+    /**
+     * Removes all items from the dataset but keeps the dataset itself, along with its
+     * {@link Dataset.id|`id`} and {@link Dataset.name|`name`}.
+     */
+    async purge() {
+        rejectOperationInTransaction('Dataset.purge()');
+        await this.backend.purge();
+    }
     /**
      * Opens a dataset and returns a promise resolving to an instance of the {@link Dataset} class.
      *
@@ -453,14 +565,11 @@ export class Dataset {
      * @param [options] Storage manager options.
      */
     static async open(identifier, options = {}) {
-        checkStorageAccess();
-        ow(options, ow.object.exactShape({
-            configuration: ow.optional.object.instanceOf(Configuration),
-            storageBackend: ow.optional.object,
-        }));
-        options.configuration ??= Configuration.getGlobalConfiguration();
-        const storageBackend = options.storageBackend ?? serviceLocator.getStorageBackend();
-        await purgeDefaultStorages({ onlyPurgeOnce: true, storageBackend, configuration: options.configuration });
+        tryCancel();
+        const parsedOptions = parseArgument(options, openOptionsSchema);
+        const configuration = parsedOptions.configuration ?? Configuration.getGlobalConfiguration();
+        const storageBackend = parsedOptions.storageBackend ?? serviceLocator.getStorageBackend();
+        await purgeDefaultStorages({ onlyPurgeOnce: true, storageBackend, configuration });
         const resolved = await resolveStorageIdentifier(identifier, storageBackend, 'Dataset');
         return serviceLocator.getStorageInstanceManager().openStorage(this, {
             ...resolved,

package/storages/index.d.ts

@@ -5,9 +5,9 @@ export * from './request_list.js';
 export type * from './request_loader.js';
 export type * from './request_manager.js';
 export * from './request_queue.js';
-export * from './storage_instance_manager.js';
-export * from './storage_stats.js';
+export type { DefaultStorageIdentifier, ExplicitStorageIdentifier, IStorage, StorageIdentifier, } from './storage_instance_manager.js';
+export { StorageInstanceManager } from './storage_instance_manager.js';
+export type { DatasetStats, KeyValueStoreStats, RequestQueueStats } from './storage_stats.js';
 export * from './utils.js';
-export * from './access_checking.js';
-export * from './sitemap_request_loader.js';
+export * from './transaction.js';
 export * from './request_manager_tandem.js';

package/storages/index.js

@@ -3,9 +3,7 @@ export * from './key_value_store.js';
 export * from './key_value_store_codec.js';
 export * from './request_list.js';
 export * from './request_queue.js';
-export * from './storage_instance_manager.js';
-export * from './storage_stats.js';
+export { StorageInstanceManager } from './storage_instance_manager.js';
 export * from './utils.js';
-export * from './access_checking.js';
-export * from './sitemap_request_loader.js';
+export * from './transaction.js';
 export * from './request_manager_tandem.js';

package/storages/key_value_store.d.ts

@@ -1,5 +1,6 @@
 import type { Awaitable, Dictionary, KeyValueStoreBackend, KeyValueStoreInfo } from '@crawlee/types';
 import { Configuration } from '../configuration.js';
+import type { JournalEntry } from './transaction.js';
 import type { KeyValueStoreStats } from './storage_stats.js';
 import type { StorageOpenOptions } from './utils.js';
 import type { StorageIdentifier } from './storage_instance_manager.js';
@@ -19,8 +20,6 @@ import type { StorageIdentifier } from './storage_instance_manager.js';
  * To access the default key-value store directly, you can use the
  * {@link KeyValueStore.getValue} and {@link KeyValueStore.setValue} convenience functions.
  *
- * To access the input, you can also use the {@link KeyValueStore.getInput} convenience function.
- *
  * `KeyValueStore` stores its data on a local disk.
  *
  * If the `CRAWLEE_STORAGE_DIR` environment variable is set, the data is stored in
@@ -28,15 +27,14 @@ import type { StorageIdentifier } from './storage_instance_manager.js';
  * ```
  * {CRAWLEE_STORAGE_DIR}/key_value_stores/{STORE_ID}/{INDEX}.{EXT}
  * ```
- * Note that `{STORE_ID}` is the name or ID of the key-value store. The default key-value store has ID: `default`,
- * unless you override it by setting the `CRAWLEE_DEFAULT_KEY_VALUE_STORE_ID` environment variable.
+ * Note that `{STORE_ID}` is the name or ID of the key-value store. The default key-value store has ID: `default`.
  * The `{KEY}` is the key of the record and `{EXT}` corresponds to the MIME content type of the data value.
  *
  * **Example usage:**
  *
  * ```javascript
  * // Get crawler input from the default key-value store.
- * const input = await KeyValueStore.getInput();
+ * const input = await KeyValueStore.getValue('INPUT');
  * // Get some value from the default key-value store.
  * const otherValue = await KeyValueStore.getValue('my-key');
  *
@@ -60,14 +58,11 @@ import type { StorageIdentifier } from './storage_instance_manager.js';
  * @category Result Stores
  */
 export declare class KeyValueStore {
+    #private;
     readonly configuration: Configuration;
     readonly id: string;
     readonly name?: string;
-    private readonly backend;
-    private persistStateEventStarted;
-    /** Cache for persistent (auto-saved) values. When we try to set such value, the cache will be updated automatically. */
-    private readonly cache;
-    private readonly statsTracker;
+    readonly backend: KeyValueStoreBackend;
     /**
      * @internal
      */
@@ -176,9 +171,6 @@ export declare class KeyValueStore {
      */
     recordExists(key: string): Promise<boolean>;
     getAutoSavedValue<T extends Dictionary = Dictionary>(key: string, defaultValue?: T): Promise<T>;
-    private ensurePersistStateEvent;
-    private fetchKeyValuePages;
-    private fetchKeyPages;
     /**
      * Saves or deletes a record in the key-value store.
      * The function returns a promise that resolves once the record has been saved or deleted.
@@ -223,11 +215,18 @@ export declare class KeyValueStore {
      * @param [options] Record options.
      */
     setValue<T>(key: string, value: T | null, options?: RecordOptions): Promise<void>;
+    /** @internal */
+    commitJournalEntries(entries: JournalEntry[]): Promise<void>;
     /**
      * Removes the key-value store either from the Apify cloud storage or from the local directory,
      * depending on the mode of operation.
      */
     drop(): Promise<void>;
+    /**
+     * Removes all records from the store but keeps the store itself, along with its
+     * {@link KeyValueStore.id|`id`} and {@link KeyValueStore.name|`name`}.
+     */
+    purge(): Promise<void>;
     /** @internal */
     clearCache(): void;
     /**
@@ -340,7 +339,9 @@ export declare class KeyValueStore {
     /**
      * Returns a file URL for the given key.
      *
-     * If the record does not exist or has no associated file path (i.e., it is not stored as a file), returns `undefined`.
+     * The URL is derived from the key, so it is also returned for a record that does not exist (yet) —
+     * including one written earlier in an uncommitted storage transaction. Returns `undefined` only when
+     * the storage has no file URLs at all (e.g. the in-memory storage).
      *
      * @param key The key of the record to generate the public URL for.
      */
@@ -471,27 +472,6 @@ export declare class KeyValueStore {
      * @ignore
      */
     static setValue<T>(key: string, value: T | null, options?: RecordOptions): Promise<void>;
-    /**
-     * Gets the crawler input value from the default {@link KeyValueStore} associated with the current crawler run.
-     *
-     * The input is read from the default {@link KeyValueStore} under the configured input key
-     * (`CRAWLEE_INPUT_KEY`, default `INPUT`).
-     *
-     * Note that the `getInput()` function does not cache the value read from the key-value store.
-     * If you need to use the input multiple times in your crawler,
-     * it is far more efficient to read it once and store it locally.
-     *
-     * For more information, see {@link KeyValueStore.open}
-     * and {@link KeyValueStore.getValue}.
-     *
-     * @returns
-     *   Returns a promise that resolves to an object, string
-     *   or [`Buffer`](https://nodejs.org/api/buffer.html), depending
-     *   on the MIME content type of the record, or `null`
-     *   if the record is missing.
-     * @ignore
-     */
-    static getInput<T = Dictionary | string | Buffer>(): Promise<T | null>;
 }
 /**
  * User-function used in the  {@link KeyValueStore.forEachKey} method.
@@ -507,6 +487,7 @@ export interface KeyConsumer {
         size: number;
     }): Awaitable<void>;
 }
+/** @internal */
 export interface KeyValueStoreOptions {
     /** Resolved metadata for the key-value store, as returned by the backend's `getMetadata()`. */
     metadata: KeyValueStoreInfo;