@crawlee/core 4.0.0-beta.103 → 4.0.0-beta.105

package/crawlers/crawler_commons.d.ts

@@ -1,12 +1,11 @@
 import type { Dictionary, HttpRequestOptions, ISession, ProxyInfo, SendRequestOptions } from '@crawlee/types';
 import type { ReadonlyDeep, SetRequired } from 'type-fest';
-import type { Configuration } from '../configuration.js';
 import type { EnqueueLinksOptions } from '../enqueue_links/enqueue_links.js';
 import type { CrawleeLogger } from '../log.js';
 import type { Request, RequestOptions, Source } from '../request.js';
 import type { StorageIdentifier } from '../storages/storage_instance_manager.js';
 import type { Dataset } from '../storages/dataset.js';
-import { KeyValueStore, type RecordOptions } from '../storages/key_value_store.js';
+import type { KeyValueStore } from '../storages/key_value_store.js';
 import type { RequestQueueOperationOptions } from '../storages/request_queue.js';
 /** @internal */
 export type IsAny<T> = 0 extends 1 & T ? true : false;
@@ -176,6 +175,11 @@ export interface CrawlingContext<UserData extends Dictionary = Dictionary> exten
     sendRequest: (requestOverrides?: Partial<HttpRequestOptions>, optionsOverrides?: SendRequestOptions) => Promise<Response>;
     /**
      * Register a function to be called at the very end of the request handling process. This is useful for resources that should be accessible to error handlers, for instance.
+     *
+     * The callback runs *outside* the request's storage transaction, so storage writes made here are
+     * applied immediately and are **not** rolled back when the request fails. In
+     * {@link AdaptivePlaywrightCrawler} it also runs once per request handler attempt, so a write
+     * here can land more than once for a single request. Push results from the request handler itself.
      */
     registerDeferredCleanup(cleanup: () => Promise<unknown>): void;
     /**
@@ -198,58 +202,4 @@ export interface CrawlingContext<UserData extends Dictionary = Dictionary> exten
      */
     extendTimeout(secs: number): void;
 }
-/**
- * A partial implementation of {@link RestrictedCrawlingContext} that stores parameters of calls to context methods for later inspection.
- *
- * @experimental
- */
-export declare class RequestHandlerResult {
-    private configuration;
-    private crawleeStateKey;
-    private _keyValueStoreChanges;
-    private pushDataCalls;
-    private addRequestsCalls;
-    constructor(configuration: Configuration, crawleeStateKey: string);
-    /**
-     * A record of calls to {@link RestrictedCrawlingContext.pushData}, {@link RestrictedCrawlingContext.addRequests}, {@link RestrictedCrawlingContext.enqueueLinks} made by a request handler.
-     */
-    get calls(): ReadonlyDeep<{
-        pushData: Parameters<RestrictedCrawlingContext['pushData']>[];
-        addRequests: Parameters<RestrictedCrawlingContext['addRequests']>[];
-    }>;
-    /**
-     * A record of changes made to key-value stores by a request handler.
-     */
-    get keyValueStoreChanges(): ReadonlyDeep<Record<string, Record<string, {
-        changedValue: unknown;
-        options?: RecordOptions;
-    }>>>;
-    /**
-     * Items added to datasets by a request handler.
-     */
-    get datasetItems(): ReadonlyDeep<{
-        item: Dictionary;
-        datasetIdentifier?: string | StorageIdentifier;
-    }[]>;
-    /**
-     * URLs enqueued to the request queue by a request handler, either via {@link RestrictedCrawlingContext.addRequests} or {@link RestrictedCrawlingContext.enqueueLinks}
-     */
-    get enqueuedUrls(): ReadonlyDeep<{
-        url: string;
-        label?: string;
-    }[]>;
-    /**
-     * URL lists enqueued to the request queue by a request handler via {@link RestrictedCrawlingContext.addRequests} using the `requestsFromUrl` option.
-     */
-    get enqueuedUrlLists(): ReadonlyDeep<{
-        listUrl: string;
-        label?: string;
-    }[]>;
-    pushData: RestrictedCrawlingContext['pushData'];
-    addRequests: RestrictedCrawlingContext['addRequests'];
-    useState: RestrictedCrawlingContext['useState'];
-    getKeyValueStore: RestrictedCrawlingContext['getKeyValueStore'];
-    private getKeyValueStoreChangedValue;
-    private setKeyValueStoreChangedValue;
-}
 export {};

package/crawlers/crawler_commons.js

@@ -1,107 +1 @@
-import { KeyValueStore } from '../storages/key_value_store.js';
-/**
- * A partial implementation of {@link RestrictedCrawlingContext} that stores parameters of calls to context methods for later inspection.
- *
- * @experimental
- */
-export class RequestHandlerResult {
-    configuration;
-    crawleeStateKey;
-    _keyValueStoreChanges = {};
-    pushDataCalls = [];
-    addRequestsCalls = [];
-    constructor(configuration, crawleeStateKey) {
-        this.configuration = configuration;
-        this.crawleeStateKey = crawleeStateKey;
-    }
-    /**
-     * A record of calls to {@link RestrictedCrawlingContext.pushData}, {@link RestrictedCrawlingContext.addRequests}, {@link RestrictedCrawlingContext.enqueueLinks} made by a request handler.
-     */
-    get calls() {
-        return {
-            pushData: this.pushDataCalls,
-            addRequests: this.addRequestsCalls,
-        };
-    }
-    /**
-     * A record of changes made to key-value stores by a request handler.
-     */
-    get keyValueStoreChanges() {
-        return this._keyValueStoreChanges;
-    }
-    /**
-     * Items added to datasets by a request handler.
-     */
-    get datasetItems() {
-        return this.pushDataCalls.flatMap(([data, datasetIdentifier]) => (Array.isArray(data) ? data : [data]).map((item) => ({ item, datasetIdentifier })));
-    }
-    /**
-     * URLs enqueued to the request queue by a request handler, either via {@link RestrictedCrawlingContext.addRequests} or {@link RestrictedCrawlingContext.enqueueLinks}
-     */
-    get enqueuedUrls() {
-        const result = [];
-        for (const [requests] of this.addRequestsCalls) {
-            for (const request of requests) {
-                if (typeof request === 'object' &&
-                    (!('requestsFromUrl' in request) || request.requestsFromUrl !== undefined) &&
-                    request.url !== undefined) {
-                    result.push({ url: request.url, label: request.label });
-                }
-                else if (typeof request === 'string') {
-                    result.push({ url: request });
-                }
-            }
-        }
-        return result;
-    }
-    /**
-     * URL lists enqueued to the request queue by a request handler via {@link RestrictedCrawlingContext.addRequests} using the `requestsFromUrl` option.
-     */
-    get enqueuedUrlLists() {
-        const result = [];
-        for (const [requests] of this.addRequestsCalls) {
-            for (const request of requests) {
-                if (typeof request === 'object' &&
-                    'requestsFromUrl' in request &&
-                    request.requestsFromUrl !== undefined) {
-                    result.push({ listUrl: request.requestsFromUrl, label: request.label });
-                }
-            }
-        }
-        return result;
-    }
-    pushData = async (data, datasetIdOrName) => {
-        this.pushDataCalls.push([data, datasetIdOrName]);
-    };
-    addRequests = async (requests, options = {}) => {
-        this.addRequestsCalls.push([requests, options]);
-    };
-    useState = async (defaultValue) => {
-        const store = await this.getKeyValueStore(undefined);
-        return await store.getAutoSavedValue(this.crawleeStateKey, defaultValue);
-    };
-    getKeyValueStore = async (identifier) => {
-        const store = await KeyValueStore.open(identifier, { configuration: this.configuration });
-        const storeId = store.id;
-        return {
-            id: storeId ?? this.configuration.defaultKeyValueStoreId,
-            name: store.name,
-            getValue: async (key) => this.getKeyValueStoreChangedValue(storeId, key) ?? (await store.getValue(key)),
-            setValue: async (key, value, options) => {
-                this.setKeyValueStoreChangedValue(storeId, key, value, options);
-            },
-            getAutoSavedValue: store.getAutoSavedValue.bind(store),
-            getPublicUrl: store.getPublicUrl.bind(store),
-        };
-    };
-    getKeyValueStoreChangedValue = (storeKey, key) => {
-        const id = storeKey ?? this.configuration.defaultKeyValueStoreId;
-        this._keyValueStoreChanges[id] ??= {};
-        return this.keyValueStoreChanges[id][key]?.changedValue ?? null;
-    };
-    setKeyValueStoreChangedValue = (storeKey, key, changedValue, options) => {
-        const id = storeKey ?? this.configuration.defaultKeyValueStoreId;
-        this._keyValueStoreChanges[id] ??= {};
-        this._keyValueStoreChanges[id][key] = { changedValue, options };
-    };
-}
+export {};

package/crawlers/index.d.ts

@@ -1,5 +1,5 @@
 export * from './context_pipeline.js';
-export * from './crawler_commons.js';
+export type * from './crawler_commons.js';
 export * from './statistics.js';
 export * from './error_tracker.js';
 export * from './error_snapshotter.js';

package/crawlers/index.js

@@ -1,5 +1,4 @@
 export * from './context_pipeline.js';
-export * from './crawler_commons.js';
 export * from './statistics.js';
 export * from './error_tracker.js';
 export * from './error_snapshotter.js';

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/core",
-    "version": "4.0.0-beta.103",
+    "version": "4.0.0-beta.105",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -52,9 +52,9 @@
         "@apify/log": "^2.5.18",
         "@apify/timeout": "^0.4.4",
         "@apify/utilities": "^2.15.5",
-        "@crawlee/fs-storage": "4.0.0-beta.103",
-        "@crawlee/types": "4.0.0-beta.103",
-        "@crawlee/utils": "4.0.0-beta.103",
+        "@crawlee/fs-storage": "4.0.0-beta.105",
+        "@crawlee/types": "4.0.0-beta.105",
+        "@crawlee/utils": "4.0.0-beta.105",
         "@sapphire/async-queue": "^1.5.5",
         "@sapphire/shapeshift": "^4.0.0",
         "@vladfrangu/async_event_emitter": "^2.4.6",
@@ -78,5 +78,5 @@
             }
         }
     },
-    "gitHead": "fe5d0ae11067683e27a2d17b4edd226ccf112cf5"
+    "gitHead": "26073a822c7699ac487931383a39192bc3daae7a"
 }

package/storages/dataset.d.ts

@@ -1,6 +1,7 @@
 import type { Awaitable, DatasetBackend, DatasetInfo, Dictionary } from '@crawlee/types';
 import { Configuration } from '../configuration.js';
 import type { CrawleeLogger } from '../log.js';
+import type { JournalEntry } from './transaction.js';
 import type { DatasetStats } from './storage_stats.js';
 import type { StorageOpenOptions } from './utils.js';
 import type { StorageIdentifier } from './storage_instance_manager.js';
@@ -170,6 +171,16 @@ export declare class Dataset<Data extends Dictionary = Dictionary> {
      * Returns {@link DatasetContent} object holding the items in the dataset based on the provided parameters.
      */
     getData(options?: DatasetDataOptions): Promise<DatasetContent<Data>>;
+    /**
+     * The single transaction-aware page read all dataset read paths go through — both `getData()` and
+     * the private `fetchPages()`. Returns the real page concatenated with the current transaction's
+     * buffered items, with `offset` / `limit` / `desc` windowing applied across the concatenation.
+     */
+    private readPage;
+    /** The active transaction's buffered writes to this dataset, derived from its journal. */
+    private bufferedJournalEntries;
+    /** @internal */
+    commitJournalEntries(entries: JournalEntry[]): Promise<void>;
     /**
      * Returns all the data from the dataset. This will iterate through the whole dataset
      * via the `listItems()` client method, which gives you only paginated results.

package/storages/dataset.js

@@ -1,8 +1,9 @@
 import { stringify } from 'csv-stringify/sync';
 import ow from 'ow';
+import { tryCancel } from '@apify/timeout';
 import { Configuration } from '../configuration.js';
 import { serviceLocator } from '../service_locator.js';
-import { checkStorageAccess } from './access_checking.js';
+import { activeStorageTransaction, rejectOperationInTransaction, snapshotValue } from './transaction.js';
 import { KeyValueStore } from './key_value_store.js';
 import { StorageStatsTracker } from './storage_stats.js';
 import { resolveStorageIdentifier } from './storage_instance_manager.js';
@@ -121,13 +122,24 @@ export class Dataset {
      *   The objects must be serializable to JSON.
      */
     async pushData(data) {
-        checkStorageAccess();
+        const transaction = activeStorageTransaction();
         ow(data, 'data', ow.object);
         // Normalize to array and validate each item
         const items = Array.isArray(data) ? data : [data];
         for (let i = 0; i < items.length; i++) {
             assertJsonSerializable(items[i], i);
         }
+        if (transaction) {
+            // One snapshot serves both the reads and the commit replay, so the two cannot disagree.
+            transaction.recordJournalEntry({
+                type: 'dataset',
+                participant: this,
+                storageId: this.id,
+                items: snapshotValue(items),
+                recordedAt: new Date(),
+            });
+            return;
+        }
         this.statsTracker.add('writeCount');
         await this.backend.pushData(items);
     }
@@ -135,10 +147,8 @@ export class Dataset {
      * Returns {@link DatasetContent} object holding the items in the dataset based on the provided parameters.
      */
     async getData(options = {}) {
-        checkStorageAccess();
         try {
-            this.statsTracker.add('readCount');
-            return await this.backend.getData(options);
+            return await this.readPage(options);
         }
         catch (e) {
             const error = e;
@@ -148,12 +158,93 @@ export class Dataset {
             throw e;
         }
     }
+    /**
+     * The single transaction-aware page read all dataset read paths go through — both `getData()` and
+     * the private `fetchPages()`. Returns the real page concatenated with the current transaction's
+     * buffered items, with `offset` / `limit` / `desc` windowing applied across the concatenation.
+     */
+    async readPage(options) {
+        const buffered = this.bufferedJournalEntries()?.flatMap((entry) => entry.items);
+        // Every branch below hits the backend exactly once.
+        this.statsTracker.add('readCount');
+        if (!buffered?.length) {
+            return this.backend.getData(options);
+        }
+        const { offset = 0, limit, desc = false } = options;
+        if (!desc) {
+            const realPage = await this.backend.getData(options);
+            // Buffered items sit past `realPage.total`, so the window bounds must come from that - not
+            // from the page's shortfall, which `skipEmpty` produces without exhausting the real items.
+            const bufferedStart = Math.max(0, offset - realPage.total);
+            const bufferedEnd = limit === undefined ? buffered.length : Math.max(0, offset + limit - realPage.total);
+            const items = [...realPage.items, ...buffered.slice(bufferedStart, bufferedEnd)];
+            return {
+                items,
+                total: realPage.total + buffered.length,
+                offset,
+                // A caller that passed no limit wants everything, so the backend-reported `limit` is
+                // passed through - backends are free to report a page size or a sentinel there.
+                limit: limit ?? realPage.limit,
+                count: items.length,
+                desc,
+            };
+        }
+        // Descending order: the buffered items are the newest, so they come first, reversed.
+        const reversedBuffer = [...buffered].reverse();
+        const fromBuffer = limit === undefined ? reversedBuffer.slice(offset) : reversedBuffer.slice(offset, offset + limit);
+        const needed = limit === undefined ? Infinity : limit - fromBuffer.length;
+        if (needed <= 0) {
+            // The whole window is served from the buffer; only the real total is missing.
+            const { itemCount } = await this.backend.getMetadata();
+            return {
+                items: fromBuffer,
+                total: itemCount + buffered.length,
+                offset,
+                limit: limit,
+                count: fromBuffer.length,
+                desc,
+            };
+        }
+        const realPage = await this.backend.getData({
+            ...options,
+            offset: Math.max(0, offset - buffered.length),
+            ...(limit === undefined ? {} : { limit: needed }),
+        });
+        return {
+            items: [...fromBuffer, ...realPage.items],
+            total: realPage.total + buffered.length,
+            offset,
+            limit: limit ?? realPage.limit,
+            count: fromBuffer.length + realPage.items.length,
+            desc,
+        };
+    }
+    /** The active transaction's buffered writes to this dataset, derived from its journal. */
+    bufferedJournalEntries() {
+        const transaction = activeStorageTransaction();
+        return transaction?.journal.filter((entry) => entry.type === 'dataset' && entry.participant === this);
+    }
+    /** @internal */
+    async commitJournalEntries(entries) {
+        const items = [];
+        for (const entry of entries) {
+            if (entry.type === 'dataset') {
+                items.push(...entry.items);
+            }
+        }
+        // One backend call with all journaled items, in order - as close to atomic as the backend allows.
+        // Straight to the backend: the items were validated and snapshotted at write time.
+        if (items.length > 0) {
+            this.statsTracker.add('writeCount');
+            await this.backend.pushData(items);
+        }
+    }
     /**
      * Returns all the data from the dataset. This will iterate through the whole dataset
      * via the `listItems()` client method, which gives you only paginated results.
      */
     async export(options = {}) {
-        checkStorageAccess();
+        tryCancel();
         const items = [];
         for await (const page of this.fetchPages(options)) {
             items.push(...page.items);
@@ -219,7 +310,7 @@ export class Dataset {
      * @param [options] An optional options object where you can provide the dataset and target KVS name.
      */
     static async exportToJSON(key, options) {
-        checkStorageAccess();
+        tryCancel();
         const dataset = await this.open(options?.fromDataset);
         await dataset.exportToJSON(key, options);
     }
@@ -230,7 +321,7 @@ export class Dataset {
      * @param [options] An optional options object where you can provide the dataset and target KVS name.
      */
     static async exportToCSV(key, options) {
-        checkStorageAccess();
+        tryCancel();
         const dataset = await this.open(options?.fromDataset);
         await dataset.exportToCSV(key, options);
     }
@@ -252,8 +343,17 @@ export class Dataset {
      * @throws If the underlying storage no longer exists (e.g. it was deleted externally).
      */
     async getInfo() {
-        checkStorageAccess();
-        return this.backend.getMetadata();
+        const buffered = this.bufferedJournalEntries();
+        const metadata = await this.backend.getMetadata();
+        if (buffered?.length) {
+            const lastWriteAt = buffered[buffered.length - 1].recordedAt;
+            return {
+                ...metadata,
+                itemCount: metadata.itemCount + buffered.reduce((sum, entry) => sum + entry.items.length, 0),
+                modifiedAt: metadata.modifiedAt > lastWriteAt ? metadata.modifiedAt : lastWriteAt,
+            };
+        }
+        return metadata;
     }
     /**
      * Iterates over dataset items, yielding each in turn to an `iteratee` function.
@@ -276,7 +376,7 @@ export class Dataset {
      * @default 0
      */
     async forEach(iteratee, options = {}, index = 0) {
-        checkStorageAccess();
+        tryCancel();
         if (!options.offset)
             options.offset = 0;
         if (options.format && options.format !== 'json')
@@ -303,7 +403,7 @@ export class Dataset {
      * @param [options] All `map()` parameters.
      */
     async map(iteratee, options = {}) {
-        checkStorageAccess();
+        tryCancel();
         const result = [];
         await this.forEach(async (item, index) => {
             const res = await iteratee(item, index);
@@ -312,7 +412,7 @@ export class Dataset {
         return result;
     }
     async reduce(iteratee, memo, options = {}) {
-        checkStorageAccess();
+        tryCancel();
         let currentMemo = memo;
         const wrappedFunc = async (item, index) => {
             if (index === 0 && currentMemo === undefined) {
@@ -344,8 +444,7 @@ export class Dataset {
             const fetchLimit = totalLimit !== undefined ? Math.min(pageSize, totalLimit - yielded) : pageSize;
             if (fetchLimit <= 0)
                 break;
-            this.statsTracker.add('readCount');
-            const page = await this.backend.getData({ ...options, offset, limit: fetchLimit });
+            const page = await this.readPage({ ...options, offset, limit: fetchLimit });
             yield page;
             yielded += page.items.length;
             if (page.items.length < fetchLimit || offset + page.items.length >= page.total)
@@ -377,7 +476,7 @@ export class Dataset {
      * @param options Options for the iteration.
      */
     values(options = {}) {
-        checkStorageAccess();
+        tryCancel();
         return createDualIterable({
             createPages: () => this.fetchPages(options),
             extractItems: (page) => page.items,
@@ -407,7 +506,7 @@ export class Dataset {
      * @param options Options for the iteration.
      */
     entries(options = {}) {
-        checkStorageAccess();
+        tryCancel();
         return createDualIterable({
             createPages: () => this.fetchEntryPages(options),
             extractItems: (page) => page.items,
@@ -433,7 +532,7 @@ export class Dataset {
      * depending on the mode of operation.
      */
     async drop() {
-        checkStorageAccess();
+        rejectOperationInTransaction('Dataset.drop()');
         await this.backend.drop();
         serviceLocator.getStorageInstanceManager().removeFromCache(this);
     }
@@ -453,7 +552,7 @@ export class Dataset {
      * @param [options] Storage manager options.
      */
     static async open(identifier, options = {}) {
-        checkStorageAccess();
+        tryCancel();
         ow(options, ow.object.exactShape({
             configuration: ow.optional.object.instanceOf(Configuration),
             storageBackend: ow.optional.object,

package/storages/index.d.ts

@@ -8,6 +8,6 @@ export * from './request_queue.js';
 export * from './storage_instance_manager.js';
 export * from './storage_stats.js';
 export * from './utils.js';
-export * from './access_checking.js';
+export * from './transaction.js';
 export * from './sitemap_request_loader.js';
 export * from './request_manager_tandem.js';

package/storages/index.js

@@ -6,6 +6,6 @@ export * from './request_queue.js';
 export * from './storage_instance_manager.js';
 export * from './storage_stats.js';
 export * from './utils.js';
-export * from './access_checking.js';
+export * from './transaction.js';
 export * from './sitemap_request_loader.js';
 export * from './request_manager_tandem.js';

package/storages/key_value_store.d.ts

@@ -1,5 +1,6 @@
 import type { Awaitable, Dictionary, KeyValueStoreBackend, KeyValueStoreInfo } from '@crawlee/types';
 import { Configuration } from '../configuration.js';
+import type { JournalEntry } from './transaction.js';
 import type { KeyValueStoreStats } from './storage_stats.js';
 import type { StorageOpenOptions } from './utils.js';
 import type { StorageIdentifier } from './storage_instance_manager.js';
@@ -141,6 +142,21 @@ export declare class KeyValueStore {
      *   on the MIME content type of the record, or the default value if the key is missing from the store.
      */
     getValue<T = unknown>(key: string, defaultValue: T): Promise<T>;
+    /**
+     * The active transaction's last buffered write per key for this store, derived from its journal.
+     * An entry with a `null` value is a tombstone (an in-transaction deletion).
+     */
+    private bufferedJournalEntries;
+    /**
+     * The single transaction-aware record read shared by `getValue`, `getRecord`, `recordExists` and the
+     * listing paths: buffered key → serialized through the standard codec (same fidelity as a real
+     * round-trip); tombstoned key → `null`; otherwise the backend.
+     *
+     * The per-key buffered lookup requires the whole journal to be reduced to a last-write-per-key map,
+     * which is O(journal). Single-record callers let it default (rebuilt per call); the listing paths,
+     * which read many keys, pass a map built once so the read stays O(1) per key instead of O(journal).
+     */
+    private readRecord;
     /**
      * Reads a record from the key-value store without parsing the value.
      *
@@ -223,6 +239,8 @@ export declare class KeyValueStore {
      * @param [options] Record options.
      */
     setValue<T>(key: string, value: T | null, options?: RecordOptions): Promise<void>;
+    /** @internal */
+    commitJournalEntries(entries: JournalEntry[]): Promise<void>;
     /**
      * Removes the key-value store either from the Apify cloud storage or from the local directory,
      * depending on the mode of operation.