@crawlee/core 4.0.0-beta.99 → 4.0.0-rc.0

package/storages/dataset.js

@@ -1,12 +1,17 @@
-import { stringify } from 'csv-stringify/sync';
-import ow from 'ow';
+import { z } from 'zod';
+import { tryCancel } from '@apify/timeout';
 import { Configuration } from '../configuration.js';
 import { serviceLocator } from '../service_locator.js';
-import { checkStorageAccess } from './access_checking.js';
+import { parseArgument, schemas, validators } from '../validators.js';
+import { activeStorageTransaction, rejectOperationInTransaction, snapshotValue } from './transaction.js';
 import { KeyValueStore } from './key_value_store.js';
 import { StorageStatsTracker } from './storage_stats.js';
 import { resolveStorageIdentifier } from './storage_instance_manager.js';
 import { createDualIterable, purgeDefaultStorages } from './utils.js';
+const openOptionsSchema = z.strictObject({
+    configuration: z.instanceof(Configuration).optional(),
+    storageBackend: validators.storageBackend.optional(),
+});
 /** @internal */
 export const DATASET_ITERATORS_DEFAULT_LIMIT = 10000;
 /**
@@ -88,7 +93,7 @@ export class Dataset {
     name;
     backend;
     log;
-    statsTracker = new StorageStatsTracker({
+    #statsTracker = new StorageStatsTracker({
         readCount: 0,
         writeCount: 0,
     });
@@ -107,7 +112,7 @@ export class Dataset {
      * the underlying storage backend). Counted per backend call.
      */
     get stats() {
-        return this.statsTracker.current;
+        return this.#statsTracker.current;
     }
     /**
      * Stores an object or an array of objects to the dataset.
@@ -121,24 +126,33 @@ export class Dataset {
      *   The objects must be serializable to JSON.
      */
     async pushData(data) {
-        checkStorageAccess();
-        ow(data, 'data', ow.object);
+        const transaction = activeStorageTransaction();
+        parseArgument(data, schemas.anyObject);
         // Normalize to array and validate each item
         const items = Array.isArray(data) ? data : [data];
         for (let i = 0; i < items.length; i++) {
             assertJsonSerializable(items[i], i);
         }
-        this.statsTracker.add('writeCount');
+        if (transaction) {
+            // One snapshot serves both the reads and the commit replay, so the two cannot disagree.
+            transaction.recordJournalEntry({
+                type: 'dataset',
+                participant: this,
+                storageId: this.id,
+                items: snapshotValue(items),
+                recordedAt: new Date(),
+            });
+            return;
+        }
+        this.#statsTracker.add('writeCount');
         await this.backend.pushData(items);
     }
     /**
      * Returns {@link DatasetContent} object holding the items in the dataset based on the provided parameters.
      */
     async getData(options = {}) {
-        checkStorageAccess();
         try {
-            this.statsTracker.add('readCount');
-            return await this.backend.getData(options);
+            return await this.readPage(options);
         }
         catch (e) {
             const error = e;
@@ -148,12 +162,93 @@ export class Dataset {
             throw e;
         }
     }
+    /**
+     * The single transaction-aware page read all dataset read paths go through — both `getData()` and
+     * the private `fetchPages()`. Returns the real page concatenated with the current transaction's
+     * buffered items, with `offset` / `limit` / `desc` windowing applied across the concatenation.
+     */
+    async readPage(options) {
+        const buffered = this.bufferedJournalEntries()?.flatMap((entry) => entry.items);
+        // Every branch below hits the backend exactly once.
+        this.#statsTracker.add('readCount');
+        if (!buffered?.length) {
+            return this.backend.getData(options);
+        }
+        const { offset = 0, limit, desc = false } = options;
+        if (!desc) {
+            const realPage = await this.backend.getData(options);
+            // Buffered items sit past `realPage.total`, so the window bounds must come from that - not
+            // from the page's shortfall, which `skipEmpty` produces without exhausting the real items.
+            const bufferedStart = Math.max(0, offset - realPage.total);
+            const bufferedEnd = limit === undefined ? buffered.length : Math.max(0, offset + limit - realPage.total);
+            const items = [...realPage.items, ...buffered.slice(bufferedStart, bufferedEnd)];
+            return {
+                items,
+                total: realPage.total + buffered.length,
+                offset,
+                // A caller that passed no limit wants everything, so the backend-reported `limit` is
+                // passed through - backends are free to report a page size or a sentinel there.
+                limit: limit ?? realPage.limit,
+                count: items.length,
+                desc,
+            };
+        }
+        // Descending order: the buffered items are the newest, so they come first, reversed.
+        const reversedBuffer = [...buffered].reverse();
+        const fromBuffer = limit === undefined ? reversedBuffer.slice(offset) : reversedBuffer.slice(offset, offset + limit);
+        const needed = limit === undefined ? Infinity : limit - fromBuffer.length;
+        if (needed <= 0) {
+            // The whole window is served from the buffer; only the real total is missing.
+            const { itemCount } = await this.backend.getMetadata();
+            return {
+                items: fromBuffer,
+                total: itemCount + buffered.length,
+                offset,
+                limit: limit,
+                count: fromBuffer.length,
+                desc,
+            };
+        }
+        const realPage = await this.backend.getData({
+            ...options,
+            offset: Math.max(0, offset - buffered.length),
+            ...(limit === undefined ? {} : { limit: needed }),
+        });
+        return {
+            items: [...fromBuffer, ...realPage.items],
+            total: realPage.total + buffered.length,
+            offset,
+            limit: limit ?? realPage.limit,
+            count: fromBuffer.length + realPage.items.length,
+            desc,
+        };
+    }
+    /** The active transaction's buffered writes to this dataset, derived from its journal. */
+    bufferedJournalEntries() {
+        const transaction = activeStorageTransaction();
+        return transaction?.journal.filter((entry) => entry.type === 'dataset' && entry.participant === this);
+    }
+    /** @internal */
+    async commitJournalEntries(entries) {
+        const items = [];
+        for (const entry of entries) {
+            if (entry.type === 'dataset') {
+                items.push(...entry.items);
+            }
+        }
+        // One backend call with all journaled items, in order - as close to atomic as the backend allows.
+        // Straight to the backend: the items were validated and snapshotted at write time.
+        if (items.length > 0) {
+            this.#statsTracker.add('writeCount');
+            await this.backend.pushData(items);
+        }
+    }
     /**
      * Returns all the data from the dataset. This will iterate through the whole dataset
      * via the `listItems()` client method, which gives you only paginated results.
      */
     async export(options = {}) {
-        checkStorageAccess();
+        tryCancel();
         const items = [];
         for await (const page of this.fetchPages(options)) {
             items.push(...page.items);
@@ -179,6 +274,7 @@ export class Dataset {
             const keys = options?.collectAllKeys
                 ? Array.from(new Set(items.flatMap(Object.keys)))
                 : Object.keys(items[0]);
+            const { stringify } = await import('csv-stringify/sync');
             const value = stringify([
                 keys,
                 ...items.map((item) => {
@@ -219,7 +315,7 @@ export class Dataset {
      * @param [options] An optional options object where you can provide the dataset and target KVS name.
      */
     static async exportToJSON(key, options) {
-        checkStorageAccess();
+        tryCancel();
         const dataset = await this.open(options?.fromDataset);
         await dataset.exportToJSON(key, options);
     }
@@ -230,7 +326,7 @@ export class Dataset {
      * @param [options] An optional options object where you can provide the dataset and target KVS name.
      */
     static async exportToCSV(key, options) {
-        checkStorageAccess();
+        tryCancel();
         const dataset = await this.open(options?.fromDataset);
         await dataset.exportToCSV(key, options);
     }
@@ -252,8 +348,17 @@ export class Dataset {
      * @throws If the underlying storage no longer exists (e.g. it was deleted externally).
      */
     async getInfo() {
-        checkStorageAccess();
-        return this.backend.getMetadata();
+        const buffered = this.bufferedJournalEntries();
+        const metadata = await this.backend.getMetadata();
+        if (buffered?.length) {
+            const lastWriteAt = buffered[buffered.length - 1].recordedAt;
+            return {
+                ...metadata,
+                itemCount: metadata.itemCount + buffered.reduce((sum, entry) => sum + entry.items.length, 0),
+                modifiedAt: metadata.modifiedAt > lastWriteAt ? metadata.modifiedAt : lastWriteAt,
+            };
+        }
+        return metadata;
     }
     /**
      * Iterates over dataset items, yielding each in turn to an `iteratee` function.
@@ -276,7 +381,7 @@ export class Dataset {
      * @default 0
      */
     async forEach(iteratee, options = {}, index = 0) {
-        checkStorageAccess();
+        tryCancel();
         if (!options.offset)
             options.offset = 0;
         if (options.format && options.format !== 'json')
@@ -303,7 +408,7 @@ export class Dataset {
      * @param [options] All `map()` parameters.
      */
     async map(iteratee, options = {}) {
-        checkStorageAccess();
+        tryCancel();
         const result = [];
         await this.forEach(async (item, index) => {
             const res = await iteratee(item, index);
@@ -312,7 +417,7 @@ export class Dataset {
         return result;
     }
     async reduce(iteratee, memo, options = {}) {
-        checkStorageAccess();
+        tryCancel();
         let currentMemo = memo;
         const wrappedFunc = async (item, index) => {
             if (index === 0 && currentMemo === undefined) {
@@ -344,8 +449,7 @@ export class Dataset {
             const fetchLimit = totalLimit !== undefined ? Math.min(pageSize, totalLimit - yielded) : pageSize;
             if (fetchLimit <= 0)
                 break;
-            this.statsTracker.add('readCount');
-            const page = await this.backend.getData({ ...options, offset, limit: fetchLimit });
+            const page = await this.readPage({ ...options, offset, limit: fetchLimit });
             yield page;
             yielded += page.items.length;
             if (page.items.length < fetchLimit || offset + page.items.length >= page.total)
@@ -377,7 +481,7 @@ export class Dataset {
      * @param options Options for the iteration.
      */
     values(options = {}) {
-        checkStorageAccess();
+        tryCancel();
         return createDualIterable({
             createPages: () => this.fetchPages(options),
             extractItems: (page) => page.items,
@@ -407,7 +511,7 @@ export class Dataset {
      * @param options Options for the iteration.
      */
     entries(options = {}) {
-        checkStorageAccess();
+        tryCancel();
         return createDualIterable({
             createPages: () => this.fetchEntryPages(options),
             extractItems: (page) => page.items,
@@ -433,7 +537,7 @@ export class Dataset {
      * depending on the mode of operation.
      */
     async drop() {
-        checkStorageAccess();
+        rejectOperationInTransaction('Dataset.drop()');
         await this.backend.drop();
         serviceLocator.getStorageInstanceManager().removeFromCache(this);
     }
@@ -453,14 +557,11 @@ export class Dataset {
      * @param [options] Storage manager options.
      */
     static async open(identifier, options = {}) {
-        checkStorageAccess();
-        ow(options, ow.object.exactShape({
-            configuration: ow.optional.object.instanceOf(Configuration),
-            storageBackend: ow.optional.object,
-        }));
-        options.configuration ??= Configuration.getGlobalConfiguration();
-        const storageBackend = options.storageBackend ?? serviceLocator.getStorageBackend();
-        await purgeDefaultStorages({ onlyPurgeOnce: true, storageBackend, configuration: options.configuration });
+        tryCancel();
+        const parsedOptions = parseArgument(options, openOptionsSchema);
+        const configuration = parsedOptions.configuration ?? Configuration.getGlobalConfiguration();
+        const storageBackend = parsedOptions.storageBackend ?? serviceLocator.getStorageBackend();
+        await purgeDefaultStorages({ onlyPurgeOnce: true, storageBackend, configuration });
         const resolved = await resolveStorageIdentifier(identifier, storageBackend, 'Dataset');
         return serviceLocator.getStorageInstanceManager().openStorage(this, {
             ...resolved,

package/storages/index.d.ts

@@ -8,6 +8,7 @@ export * from './request_queue.js';
 export * from './storage_instance_manager.js';
 export * from './storage_stats.js';
 export * from './utils.js';
-export * from './access_checking.js';
+export * from './transaction.js';
 export * from './sitemap_request_loader.js';
 export * from './request_manager_tandem.js';
+export * from './throttling_request_manager.js';

package/storages/index.js

@@ -6,6 +6,7 @@ export * from './request_queue.js';
 export * from './storage_instance_manager.js';
 export * from './storage_stats.js';
 export * from './utils.js';
-export * from './access_checking.js';
+export * from './transaction.js';
 export * from './sitemap_request_loader.js';
 export * from './request_manager_tandem.js';
+export * from './throttling_request_manager.js';

package/storages/key_value_store.d.ts

@@ -1,5 +1,6 @@
 import type { Awaitable, Dictionary, KeyValueStoreBackend, KeyValueStoreInfo } from '@crawlee/types';
 import { Configuration } from '../configuration.js';
+import type { JournalEntry } from './transaction.js';
 import type { KeyValueStoreStats } from './storage_stats.js';
 import type { StorageOpenOptions } from './utils.js';
 import type { StorageIdentifier } from './storage_instance_manager.js';
@@ -60,14 +61,11 @@ import type { StorageIdentifier } from './storage_instance_manager.js';
  * @category Result Stores
  */
 export declare class KeyValueStore {
+    #private;
     readonly configuration: Configuration;
     readonly id: string;
     readonly name?: string;
     private readonly backend;
-    private persistStateEventStarted;
-    /** Cache for persistent (auto-saved) values. When we try to set such value, the cache will be updated automatically. */
-    private readonly cache;
-    private readonly statsTracker;
     /**
      * @internal
      */
@@ -141,6 +139,21 @@ export declare class KeyValueStore {
      *   on the MIME content type of the record, or the default value if the key is missing from the store.
      */
     getValue<T = unknown>(key: string, defaultValue: T): Promise<T>;
+    /**
+     * The active transaction's last buffered write per key for this store, derived from its journal.
+     * An entry with a `null` value is a tombstone (an in-transaction deletion).
+     */
+    private bufferedJournalEntries;
+    /**
+     * The single transaction-aware record read shared by `getValue`, `getRecord`, `recordExists` and the
+     * listing paths: buffered key → serialized through the standard codec (same fidelity as a real
+     * round-trip); tombstoned key → `null`; otherwise the backend.
+     *
+     * The per-key buffered lookup requires the whole journal to be reduced to a last-write-per-key map,
+     * which is O(journal). Single-record callers let it default (rebuilt per call); the listing paths,
+     * which read many keys, pass a map built once so the read stays O(1) per key instead of O(journal).
+     */
+    private readRecord;
     /**
      * Reads a record from the key-value store without parsing the value.
      *
@@ -223,6 +236,8 @@ export declare class KeyValueStore {
      * @param [options] Record options.
      */
     setValue<T>(key: string, value: T | null, options?: RecordOptions): Promise<void>;
+    /** @internal */
+    commitJournalEntries(entries: JournalEntry[]): Promise<void>;
     /**
      * Removes the key-value store either from the Apify cloud storage or from the local directory,
      * depending on the mode of operation.