crawlee-one 1.0.0

package/dist/cjs/lib/io/pushData.d.ts

@@ -0,0 +1,124 @@
+import type { CrawlingContext } from 'crawlee';
+import type { CrawleeOneIO } from '../integrations/types';
+/** Functions that generates a "redacted" version of a value */
+export type PrivateValueGen<V, K, O> = (val: V, key: K, obj: O) => any;
+/**
+ * Given a property value (and its position) this function
+ * determines if the property is considered private (and
+ * hence should be hidden for privacy reasons).
+ *
+ * Property is private if the function returns truthy value.
+ */
+export type PrivacyFilter<V, K, O> = (val: V, key: K, obj: O, options?: {
+    setCustomPrivateValue: (val: V) => any;
+    privateValueGen: PrivateValueGen<V, K, O>;
+}) => any;
+/**
+ * PrivacyMask determines which (potentally nested) properties
+ * of an object are considered private.
+ *
+ * PrivacyMask copies the structure of another object, but each
+ * non-object property on PrivacyMask is a PrivacyFilter - function
+ * that determines if the property is considered private.
+ *
+ * Property is private if the function returns truthy value.
+ */
+export type PrivacyMask<T extends object> = {
+    [Key in keyof T]?: T[Key] extends Date | any[] ? PrivacyFilter<T[Key], Key, T> : T[Key] extends object ? PrivacyMask<T[Key]> : PrivacyFilter<T[Key], Key, T>;
+};
+export interface PushDataOptions<T extends object> {
+    io?: CrawleeOneIO<any, any>;
+    /**
+     * If set, only at most this many entries will be scraped.
+     *
+     * The count is determined from the Dataset that's used for the crawler run.
+     *
+     * This means that if `maxCount` is set to 50, but the
+     * associated Dataset already has 40 items in it, then only 10 new entries
+     * will be saved.
+     */
+    maxCount?: number;
+    /**
+     * Whether items should be enriched with request and run metadata.
+     *
+     * If truthy, the metadata is set under the `metadata` property.
+     */
+    includeMetadata?: boolean;
+    /**
+     * Whether properties that are considered personal data should be shown as is.
+     *
+     * If falsy or not set, these properties are redacted to hide the actual information.
+     *
+     * Which properties are personal data is determined by `privacyMask`.
+     */
+    showPrivate?: boolean;
+    /**
+     * Determine which properties are considered personal data.
+     *
+     * See {@link PrivacyMask}.
+     **/
+    privacyMask: PrivacyMask<T>;
+    /**
+     * Option to select which keys (fields) of an entry to keep (discarding the rest)
+     * before pushing the entries to the dataset.
+     *
+     * This serves mainly to allow users to select the keys from actor input UI.
+     *
+     * This is done before `remapKeys`.
+     *
+     * Keys can be nested, e.g. `"someProp.value[0]"`. Nested path is
+     * resolved using Lodash.get().
+     */
+    pickKeys?: string[];
+    /**
+     * Option to remap the keys before pushing the entries to the dataset.
+     *
+     * This serves mainly to allow users to remap the keys from actor input UI.
+     *
+     * Keys can be nested, e.g. `"someProp.value[0]"`. Nested path is
+     * resolved using Lodash.get().
+     */
+    remapKeys?: Record<string, string>;
+    /**
+     * Option to freely transform an entry before pushing it to the dataset.
+     *
+     * This serves mainly to allow users to transform the entries from actor input UI.
+     */
+    transform?: (item: any) => any;
+    /**
+     * Option to filter an entry before pushing it to the dataset.
+     *
+     * This serves mainly to allow users to filter the entries from actor input UI.
+     */
+    filter?: (item: any) => any;
+    /** ID or name of the dataset to which the data should be pushed */
+    datasetId?: string;
+    /** ID of the RequestQueue that stores remaining requests */
+    requestQueueId?: string;
+    /** ID or name of the key-value store used as cache */
+    cacheStoreId?: string;
+    /** Define fields that uniquely identify entries for caching */
+    cachePrimaryKeys?: string[];
+    /** Define whether we want to add, remove, or overwrite cached entries with results from the actor run */
+    cacheActionOnResult?: 'add' | 'remove' | 'overwrite' | null;
+}
+/**
+ * Serialize dataset item to fixed-length hash.
+ *
+ * NOTE: Apify (around which this lib is designed) allows the key-value store key
+ *       to be max 256 char long.
+ *       https://docs.apify.com/sdk/js/reference/class/KeyValueStore#setValue
+ */
+export declare const itemCacheKey: (item: any, primaryKeys?: string[]) => string;
+/**
+ * Apify's `Actor.pushData` with extra features:
+ *
+ * - Data can be sent elsewhere, not just to Apify. This is set by the `io` options. By default data is sent using Apify (cloud/local).
+ * - Limit the max size of the Dataset. No entries are added when Dataset is at or above the limit.
+ * - Redact "private" fields
+ * - Add metadata to entries before they are pushed to dataset.
+ * - Select and rename (nested) properties
+ * - Transform and filter entries. Entries that did not pass the filter are not added to the dataset.
+ * - Add/remove entries to/from KeyValueStore. Entries are saved to the store by hash generated from entry fields set by `cachePrimaryKeys`.
+ */
+export declare const pushData: <Ctx extends CrawlingContext<unknown, import("crawlee").Dictionary>, T extends Record<any, any> = Record<any, any>>(oneOrManyItems: T | T[], ctx: Ctx, options: PushDataOptions<T>) => Promise<unknown[]>;

package/dist/cjs/lib/io/pushData.js

@@ -0,0 +1,193 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.pushData = exports.itemCacheKey = void 0;
+const lodash_1 = require("lodash");
+const async_1 = require("../../utils/async");
+const apify_1 = require("../integrations/apify");
+const dataset_1 = require("./dataset");
+const createMetadataMapper = (ctx, options) => __awaiter(void 0, void 0, void 0, function* () {
+    const { io = apify_1.apifyIO } = options !== null && options !== void 0 ? options : {};
+    const metadata = yield io.generateEntryMetadata(ctx);
+    const addMetadataToData = (item) => (Object.assign(Object.assign({}, item), { metadata }));
+    return addMetadataToData;
+});
+const applyPrivacyMask = (item, options) => {
+    const { showPrivate, privacyMask, privateValueGen = (_, key) => `<Redacted property "${key}">`, } = options;
+    const resolvePrivateValue = (key, val) => {
+        // Allow to set custom "redacted" value by calling
+        // `setCustomPrivateValue` from inside the filter function.
+        let customPrivateValue;
+        let setCustomPrivateValueCalled = false;
+        const setCustomPrivateValue = (val) => {
+            customPrivateValue = val;
+            setCustomPrivateValueCalled = true;
+        };
+        const privacyFilter = privacyMask[key];
+        const isPrivate = privacyFilter
+            ? privacyFilter(val, key, item, { setCustomPrivateValue, privateValueGen })
+            : false;
+        // prettier-ignore
+        const privateValue = (
+        // Don't redact anything if we're asked to show private data
+        showPrivate ? val
+            // Otherwise, if custom value was given, use that
+            : setCustomPrivateValueCalled ? customPrivateValue
+                // Otherwise, decide based on filter truthiness
+                : isPrivate ? privateValueGen(val, key, item) : val);
+        return privateValue;
+    };
+    const redactedObj = Object.entries(item).reduce((agg, [key, val]) => {
+        var _a;
+        const isNestedObj = typeof val === 'object' && val != null && !(val instanceof Date) && !Array.isArray(val);
+        if (isNestedObj) {
+            // Recursively process nested objects
+            const subObj = applyPrivacyMask(val, {
+                showPrivate,
+                privacyMask: ((_a = privacyMask[key]) !== null && _a !== void 0 ? _a : {}),
+                privateValueGen,
+            });
+            agg[key] = subObj;
+        }
+        else {
+            agg[key] = resolvePrivateValue(key, val);
+        }
+        return agg;
+    }, {});
+    return redactedObj;
+};
+/** Rename object properties in place */
+const renameKeys = (item, keyNameMap) => {
+    Object.entries(keyNameMap || {}).forEach(([oldPath, newPath]) => {
+        if (oldPath === newPath)
+            return;
+        const val = (0, lodash_1.get)(item, oldPath);
+        (0, lodash_1.set)(item, newPath, val);
+        (0, lodash_1.unset)(item, oldPath);
+    });
+    return item;
+};
+const sortObjectKeys = (obj) => (0, lodash_1.fromPairs)((0, lodash_1.sortBy)(Object.keys(obj)).map((key) => [key, obj[key]]));
+/**
+ * Serialize dataset item to fixed-length hash.
+ *
+ * NOTE: Apify (around which this lib is designed) allows the key-value store key
+ *       to be max 256 char long.
+ *       https://docs.apify.com/sdk/js/reference/class/KeyValueStore#setValue
+ */
+const itemCacheKey = (item, primaryKeys) => {
+    const thePrimaryKeys = primaryKeys
+        ? (0, lodash_1.sortBy)((0, lodash_1.uniq)(primaryKeys.map((s) => s === null || s === void 0 ? void 0 : s.trim()).filter(Boolean)))
+        : null;
+    const serializedItem = thePrimaryKeys
+        ? thePrimaryKeys.map((k) => item === null || item === void 0 ? void 0 : item[k]).join(':')
+        : item && (0, lodash_1.isPlainObject)(item)
+            ? JSON.stringify(sortObjectKeys(item)) // If possible sort the object's keys
+            : JSON.stringify(item);
+    const cacheId = cyrb53(serializedItem);
+    return cacheId.toString();
+};
+exports.itemCacheKey = itemCacheKey;
+/**
+ * Hashing function used when calculating cache ID hash from entries.
+ *
+ * See https://stackoverflow.com/a/52171480/9788634.
+ */
+const cyrb53 = (str, seed = 0) => {
+    let h1 = 0xdeadbeef ^ seed, h2 = 0x41c6ce57 ^ seed;
+    for (let i = 0, ch; i < str.length; i++) {
+        ch = str.charCodeAt(i);
+        h1 = Math.imul(h1 ^ ch, 2654435761);
+        h2 = Math.imul(h2 ^ ch, 1597334677);
+    }
+    h1 = Math.imul(h1 ^ (h1 >>> 16), 2246822507);
+    h1 ^= Math.imul(h2 ^ (h2 >>> 13), 3266489909);
+    h2 = Math.imul(h2 ^ (h2 >>> 16), 2246822507);
+    h2 ^= Math.imul(h1 ^ (h1 >>> 13), 3266489909);
+    return 4294967296 * (2097151 & h2) + (h1 >>> 0);
+};
+const shortenToSize = (entries, maxCount, options) => __awaiter(void 0, void 0, void 0, function* () {
+    const { io, datasetId, requestQueueId, log } = options !== null && options !== void 0 ? options : {};
+    const datasetName = datasetId ? `"${datasetId}"` : 'DEFAULT';
+    const sizeMonitor = (0, dataset_1.datasetSizeMonitor)(maxCount, { datasetId, requestQueueId, io });
+    // Ignore incoming entries if the dataset is already full
+    const isDatasetFull = yield sizeMonitor.isFull();
+    if (isDatasetFull) {
+        log === null || log === void 0 ? void 0 : log.warning(`Dataset (${datasetName}) is already full (${maxCount} entries), ${entries.length} entries will be discarded.`);
+        return [];
+    } // prettier-ignore
+    // Show warning when only part of the incoming data made it into the dataset
+    const slicedEntries = yield sizeMonitor.shortenToSize(entries);
+    if (slicedEntries.length !== entries.length) {
+        log === null || log === void 0 ? void 0 : log.warning(`Dataset (${datasetName}) has become full (${maxCount} entries), ${entries.length} entries will be discarded.`);
+        return [];
+    } // prettier-ignore
+    return slicedEntries;
+});
+/**
+ * Apify's `Actor.pushData` with extra features:
+ *
+ * - Data can be sent elsewhere, not just to Apify. This is set by the `io` options. By default data is sent using Apify (cloud/local).
+ * - Limit the max size of the Dataset. No entries are added when Dataset is at or above the limit.
+ * - Redact "private" fields
+ * - Add metadata to entries before they are pushed to dataset.
+ * - Select and rename (nested) properties
+ * - Transform and filter entries. Entries that did not pass the filter are not added to the dataset.
+ * - Add/remove entries to/from KeyValueStore. Entries are saved to the store by hash generated from entry fields set by `cachePrimaryKeys`.
+ */
+const pushData = (oneOrManyItems, ctx, options) => __awaiter(void 0, void 0, void 0, function* () {
+    const { io = apify_1.apifyIO, maxCount, includeMetadata, showPrivate, privacyMask, remapKeys, pickKeys, transform, filter, datasetId, requestQueueId, cacheStoreId, cachePrimaryKeys, cacheActionOnResult, } = options;
+    const manyItems = Array.isArray(oneOrManyItems) ? oneOrManyItems : [oneOrManyItems];
+    const items = maxCount != null
+        ? yield shortenToSize(manyItems, maxCount, { io, datasetId, requestQueueId, log: ctx.log })
+        : manyItems;
+    ctx.log.debug(`Preparing to push ${items.length} entries to dataset`); // prettier-ignore
+    const addMetadataToData = yield createMetadataMapper(ctx, { io });
+    const adjustedItems = yield items.reduce((aggPromise, item) => __awaiter(void 0, void 0, void 0, function* () {
+        const agg = yield aggPromise;
+        const itemWithMetadata = includeMetadata ? addMetadataToData(item) : item;
+        const maskedItem = applyPrivacyMask(itemWithMetadata, {
+            showPrivate,
+            privacyMask,
+            privateValueGen: (val, key) => `<Redacted property "${key}". To include the actual value, toggle ON the input option "Include personal data">`,
+        });
+        const renamedItem = remapKeys ? renameKeys(maskedItem, remapKeys) : maskedItem;
+        const pickedItem = pickKeys ? (0, lodash_1.pick)(renamedItem, pickKeys) : renamedItem;
+        const transformedItem = transform ? yield transform(pickedItem) : pickedItem;
+        const passedFilter = filter ? yield filter(transformedItem) : true;
+        if (passedFilter)
+            agg.push(transformedItem);
+        return agg;
+    }), Promise.resolve([]));
+    // Push entries to primary dataset
+    ctx.log.info(`Pushing ${adjustedItems.length} entries to dataset`);
+    const dataset = yield io.openDataset(datasetId);
+    yield dataset.pushData(adjustedItems);
+    ctx.log.info(`Done pushing ${adjustedItems.length} entries to dataset`);
+    // Update entries in cache
+    if (cacheStoreId && cacheActionOnResult) {
+        ctx.log.info(`Update ${adjustedItems.length} entries in cache`);
+        const store = yield io.openKeyValueStore(cacheStoreId);
+        yield (0, async_1.serialAsyncMap)(adjustedItems, (item) => __awaiter(void 0, void 0, void 0, function* () {
+            const cacheId = (0, exports.itemCacheKey)(item, cachePrimaryKeys);
+            if (['add', 'overwrite'].includes(cacheActionOnResult)) {
+                yield store.setValue(cacheId, item);
+            }
+            else if (cacheActionOnResult === 'remove') {
+                yield store.setValue(cacheId, null);
+            }
+        }));
+        ctx.log.info(`Done updating ${adjustedItems.length} entries in cache`);
+    }
+    return adjustedItems;
+});
+exports.pushData = pushData;
+//# sourceMappingURL=pushData.js.map

package/dist/cjs/lib/io/pushData.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"pushData.js","sourceRoot":"","sources":["../../../../src/lib/io/pushData.ts"],"names":[],"mappings":";;;;;;;;;;;;AACA,mCAAuF;AAEvF,6CAAmD;AAEnD,iDAAmE;AACnE,uCAA+C;AAqH/C,MAAM,oBAAoB,GAAG,CAI3B,GAAQ,EACR,OAAoB,EACpB,EAAE;IACF,MAAM,EAAE,EAAE,GAAG,eAAO,EAAE,GAAG,OAAO,aAAP,OAAO,cAAP,OAAO,GAAI,EAAE,CAAC;IAEvC,MAAM,QAAQ,GAAG,MAAM,EAAE,CAAC,qBAAqB,CAAC,GAAG,CAAC,CAAC;IACrD,MAAM,iBAAiB,GAAG,CAAmB,IAAO,EAAE,EAAE,CAAC,iCAAM,IAAI,KAAE,QAAQ,IAAG,CAAC;IACjF,OAAO,iBAAiB,CAAC;AAC3B,CAAC,CAAA,CAAC;AAEF,MAAM,gBAAgB,GAAG,CACvB,IAAO,EACP,OAIC,EACD,EAAE;IACF,MAAM,EACJ,WAAW,EACX,WAAW,EACX,eAAe,GAAG,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,CAAC,uBAAuB,GAAG,IAAI,GAC7D,GAAG,OAAO,CAAC;IAEZ,MAAM,mBAAmB,GAAG,CAAC,GAAW,EAAE,GAAQ,EAAE,EAAE;QACpD,kDAAkD;QAClD,2DAA2D;QAC3D,IAAI,kBAAkB,CAAC;QACvB,IAAI,2BAA2B,GAAG,KAAK,CAAC;QACxC,MAAM,qBAAqB,GAAG,CAAC,GAAQ,EAAE,EAAE;YACzC,kBAAkB,GAAG,GAAG,CAAC;YACzB,2BAA2B,GAAG,IAAI,CAAC;QACrC,CAAC,CAAC;QAEF,MAAM,aAAa,GAAG,WAAW,CAAC,GAAG,CAA6C,CAAC;QACnF,MAAM,SAAS,GAAG,aAAa;YAC7B,CAAC,CAAC,aAAa,CAAC,GAAG,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE,qBAAqB,EAAE,eAAe,EAAE,CAAC;YAC3E,CAAC,CAAC,KAAK,CAAC;QAEV,kBAAkB;QAClB,MAAM,YAAY,GAAG;QACnB,4DAA4D;QAC5D,WAAW,CAAC,CAAC,CAAC,GAAG;YACjB,iDAAiD;YACjD,CAAC,CAAC,2BAA2B,CAAC,CAAC,CAAC,kBAAkB;gBAClD,+CAA+C;gBAC/C,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,eAAe,CAAC,GAAG,EAAE,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CACpD,CAAC;QACF,OAAO,YAAY,CAAC;IACtB,CAAC,CAAC;IAEF,MAAM,WAAW,GAAG,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,EAAE;;QAClE,MAAM,WAAW,GACf,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,IAAI,IAAI,IAAI,CAAC,CAAC,GAAG,YAAY,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAE1F,IAAI,WAAW,EAAE;YACf,qCAAqC;YACrC,MAAM,MAAM,GAAG,gBAAgB,CAAC,GAAG,EAAE;gBACnC,WAAW;gBACX,WAAW,EAAE,CAAC,MAAA,WAAW,CAAC,GAAG,CAAC,mCAAI,EAAE,CAAQ;gBAC5C,eAAe;aAChB,CAAC,CAAC;YACH,GAAG,CAAC,GAAc,CAAC,GAAG,MAAa,CAAC;SACrC;aAAM;YACL,GAAG,CAAC,GAAc,CAAC,GAAG,mBAAmB,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;SACrD;QACD,OAAO,GAAG,CAAC;IACb,CAAC,EAAE,EAAO,CAAC,CAAC;IAEZ,OAAO,WAAW,CAAC;AACrB,CAAC,CAAC;AAEF,wCAAwC;AACxC,MAAM,UAAU,GAAG,CAAmB,IAAO,EAAE,UAAkC,EAAE,EAAE;IACnF,MAAM,CAAC,OAAO,CAAC,UAAU,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,EAAE,OAAO,CAAC,EAAE,EAAE;QAC9D,IAAI,OAAO,KAAK,OAAO;YAAE,OAAO;QAChC,MAAM,GAAG,GAAG,IAAA,YAAG,EAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QAC/B,IAAA,YAAG,EAAC,IAAI,EAAE,OAAiB,EAAE,GAAG,CAAC,CAAC;QAClC,IAAA,cAAK,EAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IACvB,CAAC,CAAC,CAAC;IACH,OAAO,IAAI,CAAC;AACd,CAAC,CAAC;AAEF,MAAM,cAAc,GAAG,CAAmB,GAAM,EAAE,EAAE,CAClD,IAAA,kBAAS,EAAC,IAAA,eAAM,EAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;AAEpE;;;;;;GAMG;AACI,MAAM,YAAY,GAAG,CAAC,IAAS,EAAE,WAAsB,EAAE,EAAE;IAChE,MAAM,cAAc,GAAG,WAAW;QAChC,CAAC,CAAC,IAAA,eAAM,EAAC,IAAA,aAAI,EAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,aAAD,CAAC,uBAAD,CAAC,CAAE,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC;QACjE,CAAC,CAAC,IAAI,CAAC;IAET,MAAM,cAAc,GAAG,cAAc;QACnC,CAAC,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,aAAJ,IAAI,uBAAJ,IAAI,CAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC;QAChD,CAAC,CAAC,IAAI,IAAI,IAAA,sBAAa,EAAC,IAAI,CAAC;YAC7B,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,qCAAqC;YAC5E,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;IAEzB,MAAM,OAAO,GAAG,MAAM,CAAC,cAAc,CAAC,CAAC;IACvC,OAAO,OAAO,CAAC,QAAQ,EAAE,CAAC;AAC5B,CAAC,CAAC;AAbW,QAAA,YAAY,gBAavB;AAEF;;;;GAIG;AACH,MAAM,MAAM,GAAG,CAAC,GAAG,EAAE,IAAI,GAAG,CAAC,EAAE,EAAE;IAC/B,IAAI,EAAE,GAAG,UAAU,GAAG,IAAI,EACxB,EAAE,GAAG,UAAU,GAAG,IAAI,CAAC;IACzB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;QACvC,EAAE,GAAG,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QACvB,EAAE,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,EAAE,UAAU,CAAC,CAAC;QACpC,EAAE,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,EAAE,UAAU,CAAC,CAAC;KACrC;IACD,EAAE,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,GAAG,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,UAAU,CAAC,CAAC;IAC7C,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC,EAAE,GAAG,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,UAAU,CAAC,CAAC;IAC9C,EAAE,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,GAAG,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,UAAU,CAAC,CAAC;IAC7C,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC,EAAE,GAAG,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,UAAU,CAAC,CAAC;IAE9C,OAAO,UAAU,GAAG,CAAC,OAAO,GAAG,EAAE,CAAC,GAAG,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC;AAClD,CAAC,CAAC;AAEF,MAAM,aAAa,GAAG,CACpB,OAAY,EACZ,QAAgB,EAChB,OAAsF,EACtF,EAAE;IACF,MAAM,EAAE,EAAE,EAAE,SAAS,EAAE,cAAc,EAAE,GAAG,EAAE,GAAG,OAAO,aAAP,OAAO,cAAP,OAAO,GAAI,EAAE,CAAC;IAC7D,MAAM,WAAW,GAAG,SAAS,CAAC,CAAC,CAAC,IAAI,SAAS,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC;IAE7D,MAAM,WAAW,GAAG,IAAA,4BAAkB,EAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,cAAc,EAAE,EAAE,EAAE,CAAC,CAAC;IAEpF,yDAAyD;IACzD,MAAM,aAAa,GAAG,MAAM,WAAW,CAAC,MAAM,EAAE,CAAC;IACjD,IAAI,aAAa,EAAE;QACjB,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,OAAO,CAAC,YAAY,WAAW,sBAAsB,QAAQ,cAAc,OAAO,CAAC,MAAM,6BAA6B,CAAC,CAAC;QAC7H,OAAO,EAAE,CAAC;KACX,CAAC,kBAAkB;IAEpB,4EAA4E;IAC5E,MAAM,aAAa,GAAG,MAAM,WAAW,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;IAC/D,IAAI,aAAa,CAAC,MAAM,KAAK,OAAO,CAAC,MAAM,EAAE;QAC3C,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,OAAO,CAAC,YAAY,WAAW,sBAAsB,QAAQ,cAAc,OAAO,CAAC,MAAM,6BAA6B,CAAC,CAAC;QAC7H,OAAO,EAAE,CAAC;KACX,CAAC,kBAAkB;IAEpB,OAAO,aAAa,CAAC;AACvB,CAAC,CAAA,CAAC;AAEF;;;;;;;;;;GAUG;AACI,MAAM,QAAQ,GAAG,CAItB,cAAuB,EACvB,GAAQ,EACR,OAA2B,EAC3B,EAAE;IACF,MAAM,EACJ,EAAE,GAAG,eAAuB,EAC5B,QAAQ,EACR,eAAe,EACf,WAAW,EACX,WAAW,EACX,SAAS,EACT,QAAQ,EACR,SAAS,EACT,MAAM,EACN,SAAS,EACT,cAAc,EACd,YAAY,EACZ,gBAAgB,EAChB,mBAAmB,GACpB,GAAG,OAAO,CAAC;IAEZ,MAAM,SAAS,GAAG,KAAK,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC;IACpF,MAAM,KAAK,GACT,QAAQ,IAAI,IAAI;QACd,CAAC,CAAC,MAAM,aAAa,CAAC,SAAS,EAAE,QAAQ,EAAE,EAAE,EAAE,EAAE,SAAS,EAAE,cAAc,EAAE,GAAG,EAAE,GAAG,CAAC,GAAG,EAAE,CAAC;QAC3F,CAAC,CAAC,SAAS,CAAC;IAEhB,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,qBAAqB,KAAK,CAAC,MAAM,qBAAqB,CAAC,CAAC,CAAC,kBAAkB;IACzF,MAAM,iBAAiB,GAAG,MAAM,oBAAoB,CAAC,GAAG,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC;IAElE,MAAM,aAAa,GAAG,MAAM,KAAK,CAAC,MAAM,CAAC,CAAO,UAAU,EAAE,IAAI,EAAE,EAAE;QAClE,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC;QAE7B,MAAM,gBAAgB,GAAG,eAAe,CAAC,CAAC,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAC1E,MAAM,UAAU,GAAG,gBAAgB,CAAC,gBAAgB,EAAE;YACpD,WAAW;YACX,WAAW;YACX,eAAe,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,CAC5B,uBAAuB,GAAG,qFAAqF;SAClH,CAAC,CAAC;QAEH,MAAM,WAAW,GAAG,SAAS,CAAC,CAAC,CAAC,UAAU,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC;QAC/E,MAAM,UAAU,GAAG,QAAQ,CAAC,CAAC,CAAC,IAAA,aAAI,EAAC,WAAW,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC;QACxE,MAAM,eAAe,GAAG,SAAS,CAAC,CAAC,CAAC,MAAM,SAAS,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC;QAC7E,MAAM,YAAY,GAAG,MAAM,CAAC,CAAC,CAAC,MAAM,MAAM,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAEnE,IAAI,YAAY;YAAE,GAAG,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QAE5C,OAAO,GAAG,CAAC;IACb,CAAC,CAAA,EAAE,OAAO,CAAC,OAAO,CAAC,EAAe,CAAC,CAAC,CAAC;IAErC,kCAAkC;IAClC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,aAAa,CAAC,MAAM,qBAAqB,CAAC,CAAC;IACnE,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC;IAChD,MAAM,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC;IACtC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,gBAAgB,aAAa,CAAC,MAAM,qBAAqB,CAAC,CAAC;IAExE,0BAA0B;IAC1B,IAAI,YAAY,IAAI,mBAAmB,EAAE;QACvC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,aAAa,CAAC,MAAM,mBAAmB,CAAC,CAAC;QAChE,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC,iBAAiB,CAAC,YAAY,CAAC,CAAC;QACvD,MAAM,IAAA,sBAAc,EAAC,aAAa,EAAE,CAAO,IAAS,EAAE,EAAE;YACtD,MAAM,OAAO,GAAG,IAAA,oBAAY,EAAC,IAAI,EAAE,gBAAgB,CAAC,CAAC;YAErD,IAAI,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC,QAAQ,CAAC,mBAAmB,CAAC,EAAE;gBACtD,MAAM,KAAK,CAAC,QAAQ,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;aACrC;iBAAM,IAAI,mBAAmB,KAAK,QAAQ,EAAE;gBAC3C,MAAM,KAAK,CAAC,QAAQ,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;aACrC;QACH,CAAC,CAAA,CAAC,CAAC;QACH,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,iBAAiB,aAAa,CAAC,MAAM,mBAAmB,CAAC,CAAC;KACxE;IAED,OAAO,aAAa,CAAC;AACvB,CAAC,CAAA,CAAC;AA9EW,QAAA,QAAQ,YA8EnB","sourcesContent":["import type { CrawlingContext, Log } from 'crawlee';\nimport { get, pick, set, unset, uniq, sortBy, isPlainObject, fromPairs } from 'lodash';\n\nimport { serialAsyncMap } from '../../utils/async';\nimport type { CrawleeOneIO } from '../integrations/types';\nimport { ApifyCrawleeOneIO, apifyIO } from '../integrations/apify';\nimport { datasetSizeMonitor } from './dataset';\n\n/** Functions that generates a \"redacted\" version of a value */\nexport type PrivateValueGen<V, K, O> = (val: V, key: K, obj: O) => any;\n\n/**\n * Given a property value (and its position) this function\n * determines if the property is considered private (and\n * hence should be hidden for privacy reasons).\n *\n * Property is private if the function returns truthy value.\n */\nexport type PrivacyFilter<V, K, O> = (\n  val: V,\n  key: K,\n  obj: O,\n  options?: {\n    setCustomPrivateValue: (val: V) => any;\n    privateValueGen: PrivateValueGen<V, K, O>;\n  }\n) => any;\n\n/**\n * PrivacyMask determines which (potentally nested) properties\n * of an object are considered private.\n *\n * PrivacyMask copies the structure of another object, but each\n * non-object property on PrivacyMask is a PrivacyFilter - function\n * that determines if the property is considered private.\n *\n * Property is private if the function returns truthy value.\n */\nexport type PrivacyMask<T extends object> = {\n  [Key in keyof T]?: T[Key] extends Date | any[] // Consider Data and Array as non-objects\n    ? PrivacyFilter<T[Key], Key, T>\n    : T[Key] extends object\n    ? PrivacyMask<T[Key]>\n    : PrivacyFilter<T[Key], Key, T>;\n};\n\nexport interface PushDataOptions<T extends object> {\n  io?: CrawleeOneIO<any, any>;\n  /**\n   * If set, only at most this many entries will be scraped.\n   *\n   * The count is determined from the Dataset that's used for the crawler run.\n   *\n   * This means that if `maxCount` is set to 50, but the\n   * associated Dataset already has 40 items in it, then only 10 new entries\n   * will be saved.\n   */\n  maxCount?: number;\n  /**\n   * Whether items should be enriched with request and run metadata.\n   *\n   * If truthy, the metadata is set under the `metadata` property.\n   */\n  includeMetadata?: boolean;\n  /**\n   * Whether properties that are considered personal data should be shown as is.\n   *\n   * If falsy or not set, these properties are redacted to hide the actual information.\n   *\n   * Which properties are personal data is determined by `privacyMask`.\n   */\n  showPrivate?: boolean;\n  /**\n   * Determine which properties are considered personal data.\n   *\n   * See {@link PrivacyMask}.\n   **/\n  privacyMask: PrivacyMask<T>;\n  /**\n   * Option to select which keys (fields) of an entry to keep (discarding the rest)\n   * before pushing the entries to the dataset.\n   *\n   * This serves mainly to allow users to select the keys from actor input UI.\n   *\n   * This is done before `remapKeys`.\n   *\n   * Keys can be nested, e.g. `\"someProp.value[0]\"`. Nested path is\n   * resolved using Lodash.get().\n   */\n  pickKeys?: string[];\n  /**\n   * Option to remap the keys before pushing the entries to the dataset.\n   *\n   * This serves mainly to allow users to remap the keys from actor input UI.\n   *\n   * Keys can be nested, e.g. `\"someProp.value[0]\"`. Nested path is\n   * resolved using Lodash.get().\n   */\n  remapKeys?: Record<string, string>;\n  /**\n   * Option to freely transform an entry before pushing it to the dataset.\n   *\n   * This serves mainly to allow users to transform the entries from actor input UI.\n   */\n  transform?: (item: any) => any;\n  /**\n   * Option to filter an entry before pushing it to the dataset.\n   *\n   * This serves mainly to allow users to filter the entries from actor input UI.\n   */\n  filter?: (item: any) => any;\n  /** ID or name of the dataset to which the data should be pushed */\n  datasetId?: string;\n  /** ID of the RequestQueue that stores remaining requests */\n  requestQueueId?: string;\n  /** ID or name of the key-value store used as cache */\n  cacheStoreId?: string;\n  /** Define fields that uniquely identify entries for caching */\n  cachePrimaryKeys?: string[];\n  /** Define whether we want to add, remove, or overwrite cached entries with results from the actor run */\n  cacheActionOnResult?: 'add' | 'remove' | 'overwrite' | null;\n}\n\nconst createMetadataMapper = async <\n  Ctx extends CrawlingContext,\n  TIO extends CrawleeOneIO<any, any> = ApifyCrawleeOneIO\n>(\n  ctx: Ctx,\n  options: { io: TIO }\n) => {\n  const { io = apifyIO } = options ?? {};\n\n  const metadata = await io.generateEntryMetadata(ctx);\n  const addMetadataToData = <T extends object>(item: T) => ({ ...item, metadata });\n  return addMetadataToData;\n};\n\nconst applyPrivacyMask = <T extends Record<any, any> = Record<any, any>>(\n  item: T,\n  options: {\n    showPrivate?: boolean;\n    privacyMask: PrivacyMask<T>;\n    privateValueGen?: (val: any, key: string, item: T) => any;\n  }\n) => {\n  const {\n    showPrivate,\n    privacyMask,\n    privateValueGen = (_, key) => `<Redacted property \"${key}\">`,\n  } = options;\n\n  const resolvePrivateValue = (key: string, val: any) => {\n    // Allow to set custom \"redacted\" value by calling\n    // `setCustomPrivateValue` from inside the filter function.\n    let customPrivateValue;\n    let setCustomPrivateValueCalled = false;\n    const setCustomPrivateValue = (val: any) => {\n      customPrivateValue = val;\n      setCustomPrivateValueCalled = true;\n    };\n\n    const privacyFilter = privacyMask[key] as PrivacyFilter<any, any, any> | undefined;\n    const isPrivate = privacyFilter\n      ? privacyFilter(val, key, item, { setCustomPrivateValue, privateValueGen })\n      : false;\n\n    // prettier-ignore\n    const privateValue = (\n      // Don't redact anything if we're asked to show private data\n      showPrivate ? val\n      // Otherwise, if custom value was given, use that\n      : setCustomPrivateValueCalled ? customPrivateValue\n      // Otherwise, decide based on filter truthiness\n      : isPrivate ? privateValueGen(val, key, item) : val\n    );\n    return privateValue;\n  };\n\n  const redactedObj = Object.entries(item).reduce((agg, [key, val]) => {\n    const isNestedObj =\n      typeof val === 'object' && val != null && !(val instanceof Date) && !Array.isArray(val);\n\n    if (isNestedObj) {\n      // Recursively process nested objects\n      const subObj = applyPrivacyMask(val, {\n        showPrivate,\n        privacyMask: (privacyMask[key] ?? {}) as any,\n        privateValueGen,\n      });\n      agg[key as keyof T] = subObj as any;\n    } else {\n      agg[key as keyof T] = resolvePrivateValue(key, val);\n    }\n    return agg;\n  }, {} as T);\n\n  return redactedObj;\n};\n\n/** Rename object properties in place */\nconst renameKeys = <T extends object>(item: T, keyNameMap: Record<string, string>) => {\n  Object.entries(keyNameMap || {}).forEach(([oldPath, newPath]) => {\n    if (oldPath === newPath) return;\n    const val = get(item, oldPath);\n    set(item, newPath as string, val);\n    unset(item, oldPath);\n  });\n  return item;\n};\n\nconst sortObjectKeys = <T extends object>(obj: T) =>\n  fromPairs(sortBy(Object.keys(obj)).map((key) => [key, obj[key]]));\n\n/**\n * Serialize dataset item to fixed-length hash.\n *\n * NOTE: Apify (around which this lib is designed) allows the key-value store key\n *       to be max 256 char long.\n *       https://docs.apify.com/sdk/js/reference/class/KeyValueStore#setValue\n */\nexport const itemCacheKey = (item: any, primaryKeys?: string[]) => {\n  const thePrimaryKeys = primaryKeys\n    ? sortBy(uniq(primaryKeys.map((s) => s?.trim()).filter(Boolean)))\n    : null;\n\n  const serializedItem = thePrimaryKeys\n    ? thePrimaryKeys.map((k) => item?.[k]).join(':')\n    : item && isPlainObject(item)\n    ? JSON.stringify(sortObjectKeys(item)) // If possible sort the object's keys\n    : JSON.stringify(item);\n\n  const cacheId = cyrb53(serializedItem);\n  return cacheId.toString();\n};\n\n/**\n * Hashing function used when calculating cache ID hash from entries.\n *\n * See https://stackoverflow.com/a/52171480/9788634.\n */\nconst cyrb53 = (str, seed = 0) => {\n  let h1 = 0xdeadbeef ^ seed,\n    h2 = 0x41c6ce57 ^ seed;\n  for (let i = 0, ch; i < str.length; i++) {\n    ch = str.charCodeAt(i);\n    h1 = Math.imul(h1 ^ ch, 2654435761);\n    h2 = Math.imul(h2 ^ ch, 1597334677);\n  }\n  h1 = Math.imul(h1 ^ (h1 >>> 16), 2246822507);\n  h1 ^= Math.imul(h2 ^ (h2 >>> 13), 3266489909);\n  h2 = Math.imul(h2 ^ (h2 >>> 16), 2246822507);\n  h2 ^= Math.imul(h1 ^ (h1 >>> 13), 3266489909);\n\n  return 4294967296 * (2097151 & h2) + (h1 >>> 0);\n};\n\nconst shortenToSize = async <T>(\n  entries: T[],\n  maxCount: number,\n  options?: { io?: CrawleeOneIO; datasetId?: string; requestQueueId?: string; log: Log }\n) => {\n  const { io, datasetId, requestQueueId, log } = options ?? {};\n  const datasetName = datasetId ? `\"${datasetId}\"` : 'DEFAULT';\n\n  const sizeMonitor = datasetSizeMonitor(maxCount, { datasetId, requestQueueId, io });\n\n  // Ignore incoming entries if the dataset is already full\n  const isDatasetFull = await sizeMonitor.isFull();\n  if (isDatasetFull) {\n    log?.warning(`Dataset (${datasetName}) is already full (${maxCount} entries), ${entries.length} entries will be discarded.`);\n    return [];\n  } // prettier-ignore\n\n  // Show warning when only part of the incoming data made it into the dataset\n  const slicedEntries = await sizeMonitor.shortenToSize(entries);\n  if (slicedEntries.length !== entries.length) {\n    log?.warning(`Dataset (${datasetName}) has become full (${maxCount} entries), ${entries.length} entries will be discarded.`);\n    return [];\n  } // prettier-ignore\n\n  return slicedEntries;\n};\n\n/**\n * Apify's `Actor.pushData` with extra features:\n *\n * - Data can be sent elsewhere, not just to Apify. This is set by the `io` options. By default data is sent using Apify (cloud/local).\n * - Limit the max size of the Dataset. No entries are added when Dataset is at or above the limit.\n * - Redact \"private\" fields\n * - Add metadata to entries before they are pushed to dataset.\n * - Select and rename (nested) properties\n * - Transform and filter entries. Entries that did not pass the filter are not added to the dataset.\n * - Add/remove entries to/from KeyValueStore. Entries are saved to the store by hash generated from entry fields set by `cachePrimaryKeys`.\n */\nexport const pushData = async <\n  Ctx extends CrawlingContext,\n  T extends Record<any, any> = Record<any, any>\n>(\n  oneOrManyItems: T | T[],\n  ctx: Ctx,\n  options: PushDataOptions<T>\n) => {\n  const {\n    io = apifyIO as CrawleeOneIO,\n    maxCount,\n    includeMetadata,\n    showPrivate,\n    privacyMask,\n    remapKeys,\n    pickKeys,\n    transform,\n    filter,\n    datasetId,\n    requestQueueId,\n    cacheStoreId,\n    cachePrimaryKeys,\n    cacheActionOnResult,\n  } = options;\n\n  const manyItems = Array.isArray(oneOrManyItems) ? oneOrManyItems : [oneOrManyItems];\n  const items =\n    maxCount != null\n      ? await shortenToSize(manyItems, maxCount, { io, datasetId, requestQueueId, log: ctx.log })\n      : manyItems;\n\n  ctx.log.debug(`Preparing to push ${items.length} entries to dataset`); // prettier-ignore\n  const addMetadataToData = await createMetadataMapper(ctx, { io });\n\n  const adjustedItems = await items.reduce(async (aggPromise, item) => {\n    const agg = await aggPromise;\n\n    const itemWithMetadata = includeMetadata ? addMetadataToData(item) : item;\n    const maskedItem = applyPrivacyMask(itemWithMetadata, {\n      showPrivate,\n      privacyMask,\n      privateValueGen: (val, key) =>\n        `<Redacted property \"${key}\". To include the actual value, toggle ON the input option \"Include personal data\">`,\n    });\n\n    const renamedItem = remapKeys ? renameKeys(maskedItem, remapKeys) : maskedItem;\n    const pickedItem = pickKeys ? pick(renamedItem, pickKeys) : renamedItem;\n    const transformedItem = transform ? await transform(pickedItem) : pickedItem;\n    const passedFilter = filter ? await filter(transformedItem) : true;\n\n    if (passedFilter) agg.push(transformedItem);\n\n    return agg;\n  }, Promise.resolve([] as unknown[]));\n\n  // Push entries to primary dataset\n  ctx.log.info(`Pushing ${adjustedItems.length} entries to dataset`);\n  const dataset = await io.openDataset(datasetId);\n  await dataset.pushData(adjustedItems);\n  ctx.log.info(`Done pushing ${adjustedItems.length} entries to dataset`);\n\n  // Update entries in cache\n  if (cacheStoreId && cacheActionOnResult) {\n    ctx.log.info(`Update ${adjustedItems.length} entries in cache`);\n    const store = await io.openKeyValueStore(cacheStoreId);\n    await serialAsyncMap(adjustedItems, async (item: any) => {\n      const cacheId = itemCacheKey(item, cachePrimaryKeys);\n\n      if (['add', 'overwrite'].includes(cacheActionOnResult)) {\n        await store.setValue(cacheId, item);\n      } else if (cacheActionOnResult === 'remove') {\n        await store.setValue(cacheId, null);\n      }\n    });\n    ctx.log.info(`Done updating ${adjustedItems.length} entries in cache`);\n  }\n\n  return adjustedItems;\n};\n"]}

package/dist/cjs/lib/io/pushRequests.d.ts

@@ -0,0 +1,38 @@
+import type { CrawlingContext, Request as CrawleeRequest, RequestQueueOperationOptions } from 'crawlee';
+import type { CrawleeOneIO } from '../integrations/types';
+export interface PushRequestsOptions<T extends CrawleeRequest = CrawleeRequest> {
+    io?: CrawleeOneIO<any, any>;
+    /**
+     * If set, only at most this many requests will be added to the RequestQueue.
+     *
+     * The count is determined from the RequestQueue that's used for the crawler run.
+     *
+     * This means that if `maxCount` is set to 50, but the
+     * associated RequestQueue already handled 40 requests, then only 10 new requests
+     * will be processed.
+     */
+    maxCount?: number;
+    /**
+     * Option to freely transform a request before pushing it to the RequestQueue.
+     *
+     * This serves mainly to allow users to transform the requests from actor input UI.
+     */
+    transform?: (req: T) => any;
+    /**
+     * Option to filter a request before pushing it to the RequestQueue.
+     *
+     * This serves mainly to allow users to filter the requests from actor input UI.
+     */
+    filter?: (req: T) => any;
+    /** ID of the RequestQueue to which the data should be pushed */
+    requestQueueId?: string;
+    queueOptions?: RequestQueueOperationOptions;
+}
+/**
+ * Similar to `Actor.openRequestQueue().addRequests`, but with extra features:
+ *
+ * - Data can be sent elsewhere, not just to Apify. This is set by the `io` options. By default data is sent using Apify (cloud/local).
+ * - Limit the max size of the RequestQueue. No requests are added when RequestQueue is at or above the limit.
+ * - Transform and filter requests. Requests that did not pass the filter are not added to the RequestQueue.
+ */
+export declare const pushRequests: <Ctx extends CrawlingContext<unknown, import("crawlee").Dictionary>, T extends CrawleeRequest<import("crawlee").Dictionary> = CrawleeRequest<import("crawlee").Dictionary>>(oneOrManyItems: T | T[], ctx: Ctx, options: PushRequestsOptions<T>) => Promise<unknown[]>;

package/dist/cjs/lib/io/pushRequests.js

@@ -0,0 +1,63 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.pushRequests = void 0;
+const requestQueue_1 = require("./requestQueue");
+const apify_1 = require("../integrations/apify");
+const shortenToSize = (entries, maxCount, options) => __awaiter(void 0, void 0, void 0, function* () {
+    const { requestQueueId, log } = options !== null && options !== void 0 ? options : {};
+    const queueName = requestQueueId ? `"${requestQueueId}"` : 'DEFAULT';
+    const sizeMonitor = (0, requestQueue_1.requestQueueSizeMonitor)(maxCount, options);
+    // Ignore incoming entries if the queue is already full
+    const isFull = yield sizeMonitor.isFull();
+    if (isFull) {
+        log === null || log === void 0 ? void 0 : log.warning(`RequestQueue (${queueName}) is already full (${maxCount} entries), ${entries.length} entries will be discarded.`);
+        return [];
+    } // prettier-ignore
+    // Show warning when only part of the incoming requests made it into the queue
+    const slicedEntries = yield sizeMonitor.shortenToSize(entries);
+    if (slicedEntries.length !== entries.length) {
+        log === null || log === void 0 ? void 0 : log.warning(`RequestQueue (${queueName}) has become full (${maxCount} entries), ${entries.length} entries will be discarded.`);
+        return [];
+    } // prettier-ignore
+    return slicedEntries;
+});
+/**
+ * Similar to `Actor.openRequestQueue().addRequests`, but with extra features:
+ *
+ * - Data can be sent elsewhere, not just to Apify. This is set by the `io` options. By default data is sent using Apify (cloud/local).
+ * - Limit the max size of the RequestQueue. No requests are added when RequestQueue is at or above the limit.
+ * - Transform and filter requests. Requests that did not pass the filter are not added to the RequestQueue.
+ */
+const pushRequests = (oneOrManyItems, ctx, options) => __awaiter(void 0, void 0, void 0, function* () {
+    const { io = apify_1.apifyIO, maxCount, transform, filter, requestQueueId, queueOptions, } = options;
+    const manyItems = Array.isArray(oneOrManyItems) ? oneOrManyItems : [oneOrManyItems];
+    const items = maxCount != null
+        ? yield shortenToSize(manyItems, maxCount, { io, requestQueueId, log: ctx.log })
+        : manyItems;
+    ctx.log.debug(`Preparing to push ${items.length} requests to queue`); // prettier-ignore
+    const adjustedItems = yield items.reduce((aggPromise, item) => __awaiter(void 0, void 0, void 0, function* () {
+        const agg = yield aggPromise;
+        const transformedItem = transform ? yield transform(item) : item;
+        const passedFilter = filter ? yield filter(transformedItem) : true;
+        if (passedFilter)
+            agg.push(transformedItem);
+        return agg;
+    }), Promise.resolve([]));
+    // Push requests to primary RequestQueue
+    ctx.log.info(`Pushing ${adjustedItems.length} requests to queue`);
+    const reqQueue = yield io.openRequestQueue(requestQueueId);
+    yield reqQueue.addRequests(adjustedItems, queueOptions);
+    ctx.log.info(`Done pushing ${adjustedItems.length} requests to queue`);
+    return adjustedItems;
+});
+exports.pushRequests = pushRequests;
+//# sourceMappingURL=pushRequests.js.map

package/dist/cjs/lib/io/pushRequests.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"pushRequests.js","sourceRoot":"","sources":["../../../../src/lib/io/pushRequests.ts"],"names":[],"mappings":";;;;;;;;;;;;AAOA,iDAAyD;AAEzD,iDAAgD;AAiChD,MAAM,aAAa,GAAG,CACpB,OAAY,EACZ,QAAgB,EAChB,OAAkE,EAClE,EAAE;IACF,MAAM,EAAE,cAAc,EAAE,GAAG,EAAE,GAAG,OAAO,aAAP,OAAO,cAAP,OAAO,GAAI,EAAE,CAAC;IAE9C,MAAM,SAAS,GAAG,cAAc,CAAC,CAAC,CAAC,IAAI,cAAc,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC;IAErE,MAAM,WAAW,GAAG,IAAA,sCAAuB,EAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAE/D,uDAAuD;IACvD,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,MAAM,EAAE,CAAC;IAC1C,IAAI,MAAM,EAAE;QACV,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,OAAO,CAAC,iBAAiB,SAAS,sBAAsB,QAAQ,cAAc,OAAO,CAAC,MAAM,6BAA6B,CAAC,CAAC;QAChI,OAAO,EAAE,CAAC;KACX,CAAC,kBAAkB;IAEpB,8EAA8E;IAC9E,MAAM,aAAa,GAAG,MAAM,WAAW,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;IAC/D,IAAI,aAAa,CAAC,MAAM,KAAK,OAAO,CAAC,MAAM,EAAE;QAC3C,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,OAAO,CAAC,iBAAiB,SAAS,sBAAsB,QAAQ,cAAc,OAAO,CAAC,MAAM,6BAA6B,CAAC,CAAC;QAChI,OAAO,EAAE,CAAC;KACX,CAAC,kBAAkB;IAEpB,OAAO,aAAa,CAAC;AACvB,CAAC,CAAA,CAAC;AAEF;;;;;;GAMG;AACI,MAAM,YAAY,GAAG,CAI1B,cAAuB,EACvB,GAAQ,EACR,OAA+B,EAC/B,EAAE;IACF,MAAM,EACJ,EAAE,GAAG,eAAuB,EAC5B,QAAQ,EACR,SAAS,EACT,MAAM,EACN,cAAc,EACd,YAAY,GACb,GAAG,OAAO,CAAC;IAEZ,MAAM,SAAS,GAAG,KAAK,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC;IACpF,MAAM,KAAK,GACT,QAAQ,IAAI,IAAI;QACd,CAAC,CAAC,MAAM,aAAa,CAAC,SAAS,EAAE,QAAQ,EAAE,EAAE,EAAE,EAAE,cAAc,EAAE,GAAG,EAAE,GAAG,CAAC,GAAG,EAAE,CAAC;QAChF,CAAC,CAAC,SAAS,CAAC;IAEhB,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,qBAAqB,KAAK,CAAC,MAAM,oBAAoB,CAAC,CAAC,CAAC,kBAAkB;IAExF,MAAM,aAAa,GAAG,MAAM,KAAK,CAAC,MAAM,CAAC,CAAO,UAAU,EAAE,IAAI,EAAE,EAAE;QAClE,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC;QAE7B,MAAM,eAAe,GAAG,SAAS,CAAC,CAAC,CAAC,MAAM,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QACjE,MAAM,YAAY,GAAG,MAAM,CAAC,CAAC,CAAC,MAAM,MAAM,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAEnE,IAAI,YAAY;YAAE,GAAG,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QAE5C,OAAO,GAAG,CAAC;IACb,CAAC,CAAA,EAAE,OAAO,CAAC,OAAO,CAAC,EAAe,CAAC,CAAC,CAAC;IAErC,wCAAwC;IACxC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,aAAa,CAAC,MAAM,oBAAoB,CAAC,CAAC;IAClE,MAAM,QAAQ,GAAG,MAAM,EAAE,CAAC,gBAAgB,CAAC,cAAc,CAAC,CAAC;IAC3D,MAAM,QAAQ,CAAC,WAAW,CAAC,aAAsB,EAAE,YAAY,CAAC,CAAC;IACjE,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,gBAAgB,aAAa,CAAC,MAAM,oBAAoB,CAAC,CAAC;IAEvE,OAAO,aAAa,CAAC;AACvB,CAAC,CAAA,CAAC;AA3CW,QAAA,YAAY,gBA2CvB","sourcesContent":["import type {\n  CrawlingContext,\n  Log,\n  Request as CrawleeRequest,\n  RequestQueueOperationOptions,\n} from 'crawlee';\n\nimport { requestQueueSizeMonitor } from './requestQueue';\nimport type { CrawleeOneIO } from '../integrations/types';\nimport { apifyIO } from '../integrations/apify';\n\nexport interface PushRequestsOptions<T extends CrawleeRequest = CrawleeRequest> {\n  io?: CrawleeOneIO<any, any>;\n  /**\n   * If set, only at most this many requests will be added to the RequestQueue.\n   *\n   * The count is determined from the RequestQueue that's used for the crawler run.\n   *\n   * This means that if `maxCount` is set to 50, but the\n   * associated RequestQueue already handled 40 requests, then only 10 new requests\n   * will be processed.\n   */\n  maxCount?: number;\n  /**\n   * Option to freely transform a request before pushing it to the RequestQueue.\n   *\n   * This serves mainly to allow users to transform the requests from actor input UI.\n   */\n  transform?: (req: T) => any;\n  /**\n   * Option to filter a request before pushing it to the RequestQueue.\n   *\n   * This serves mainly to allow users to filter the requests from actor input UI.\n   */\n  filter?: (req: T) => any;\n  /** ID of the RequestQueue to which the data should be pushed */\n  requestQueueId?: string;\n\n  // Pass-through options\n  queueOptions?: RequestQueueOperationOptions;\n}\n\nconst shortenToSize = async <T>(\n  entries: T[],\n  maxCount: number,\n  options?: { io?: CrawleeOneIO; requestQueueId?: string; log: Log }\n) => {\n  const { requestQueueId, log } = options ?? {};\n\n  const queueName = requestQueueId ? `\"${requestQueueId}\"` : 'DEFAULT';\n\n  const sizeMonitor = requestQueueSizeMonitor(maxCount, options);\n\n  // Ignore incoming entries if the queue is already full\n  const isFull = await sizeMonitor.isFull();\n  if (isFull) {\n    log?.warning(`RequestQueue (${queueName}) is already full (${maxCount} entries), ${entries.length} entries will be discarded.`);\n    return [];\n  } // prettier-ignore\n\n  // Show warning when only part of the incoming requests made it into the queue\n  const slicedEntries = await sizeMonitor.shortenToSize(entries);\n  if (slicedEntries.length !== entries.length) {\n    log?.warning(`RequestQueue (${queueName}) has become full (${maxCount} entries), ${entries.length} entries will be discarded.`);\n    return [];\n  } // prettier-ignore\n\n  return slicedEntries;\n};\n\n/**\n * Similar to `Actor.openRequestQueue().addRequests`, but with extra features:\n *\n * - Data can be sent elsewhere, not just to Apify. This is set by the `io` options. By default data is sent using Apify (cloud/local).\n * - Limit the max size of the RequestQueue. No requests are added when RequestQueue is at or above the limit.\n * - Transform and filter requests. Requests that did not pass the filter are not added to the RequestQueue.\n */\nexport const pushRequests = async <\n  Ctx extends CrawlingContext,\n  T extends CrawleeRequest = CrawleeRequest\n>(\n  oneOrManyItems: T | T[],\n  ctx: Ctx,\n  options: PushRequestsOptions<T>\n) => {\n  const {\n    io = apifyIO as CrawleeOneIO,\n    maxCount,\n    transform,\n    filter,\n    requestQueueId,\n    queueOptions,\n  } = options;\n\n  const manyItems = Array.isArray(oneOrManyItems) ? oneOrManyItems : [oneOrManyItems];\n  const items =\n    maxCount != null\n      ? await shortenToSize(manyItems, maxCount, { io, requestQueueId, log: ctx.log })\n      : manyItems;\n\n  ctx.log.debug(`Preparing to push ${items.length} requests to queue`); // prettier-ignore\n\n  const adjustedItems = await items.reduce(async (aggPromise, item) => {\n    const agg = await aggPromise;\n\n    const transformedItem = transform ? await transform(item) : item;\n    const passedFilter = filter ? await filter(transformedItem) : true;\n\n    if (passedFilter) agg.push(transformedItem);\n\n    return agg;\n  }, Promise.resolve([] as unknown[]));\n\n  // Push requests to primary RequestQueue\n  ctx.log.info(`Pushing ${adjustedItems.length} requests to queue`);\n  const reqQueue = await io.openRequestQueue(requestQueueId);\n  await reqQueue.addRequests(adjustedItems as any[], queueOptions);\n  ctx.log.info(`Done pushing ${adjustedItems.length} requests to queue`);\n\n  return adjustedItems;\n};\n"]}

package/dist/cjs/lib/io/requestQueue.d.ts

@@ -0,0 +1,28 @@
+import { type ValueMonitorOptions } from '../../utils/valueMonitor';
+import type { CrawleeOneIO } from '../integrations/types';
+export interface RequestQueueSizeMonitorOptions extends ValueMonitorOptions {
+    io?: CrawleeOneIO<any, any>;
+    /**
+     * ID of the RequestQueue that's monitored for size.
+     *
+     * If omitted, the default RequestQueue is used.
+     */
+    requestQueueId?: string;
+}
+/**
+ * Semi-automatic monitoring of RequestQueue size. This is used for limiting the total of
+ * entries scraped per run / RequestQueue:
+ * - When RequestQueue reaches `maxSize`, then all remaining Requests are removed.
+ * - Pass an array of items to `shortenToSize` to shorten the array to the size
+ *   that still fits the RequestQueue.
+ *
+ * By default uses Apify RequestQueue.
+ */
+export declare const requestQueueSizeMonitor: (maxSize: number, options?: RequestQueueSizeMonitorOptions) => {
+    shortenToSize: <T>(arr: T[]) => Promise<T[]>;
+    isFull: () => Promise<boolean>;
+    value: () => number | Promise<number> | null;
+    isStale: () => boolean;
+    refresh: () => Promise<number>;
+    onValue: (callback: import("../../utils/valueMonitor").ValueCallback<number>) => () => void;
+};

package/dist/cjs/lib/io/requestQueue.js

@@ -0,0 +1,40 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.requestQueueSizeMonitor = void 0;
+const valueMonitor_1 = require("../../utils/valueMonitor");
+const apify_1 = require("../integrations/apify");
+/**
+ * Semi-automatic monitoring of RequestQueue size. This is used for limiting the total of
+ * entries scraped per run / RequestQueue:
+ * - When RequestQueue reaches `maxSize`, then all remaining Requests are removed.
+ * - Pass an array of items to `shortenToSize` to shorten the array to the size
+ *   that still fits the RequestQueue.
+ *
+ * By default uses Apify RequestQueue.
+ */
+const requestQueueSizeMonitor = (maxSize, options) => {
+    const { io = apify_1.apifyIO } = options !== null && options !== void 0 ? options : {};
+    const getSize = () => __awaiter(void 0, void 0, void 0, function* () {
+        var _a;
+        const reqQueue = yield io.openRequestQueue(options === null || options === void 0 ? void 0 : options.requestQueueId);
+        const count = (_a = (yield reqQueue.handledCount())) !== null && _a !== void 0 ? _a : 0;
+        return count;
+    });
+    // When we've reached the RequestQueue's max size, then remove all remaining Requests
+    const onMaxSizeReached = () => __awaiter(void 0, void 0, void 0, function* () {
+        const reqQueue = yield io.openRequestQueue(options === null || options === void 0 ? void 0 : options.requestQueueId);
+        yield reqQueue.drop();
+    });
+    return (0, valueMonitor_1.createSizeMonitor)(maxSize, getSize, onMaxSizeReached, options);
+};
+exports.requestQueueSizeMonitor = requestQueueSizeMonitor;
+//# sourceMappingURL=requestQueue.js.map

package/dist/cjs/lib/io/requestQueue.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"requestQueue.js","sourceRoot":"","sources":["../../../../src/lib/io/requestQueue.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,2DAAuF;AAEvF,iDAAgD;AAYhD;;;;;;;;GAQG;AACI,MAAM,uBAAuB,GAAG,CACrC,OAAe,EACf,OAAwC,EACxC,EAAE;IACF,MAAM,EAAE,EAAE,GAAG,eAAO,EAAE,GAAG,OAAO,aAAP,OAAO,cAAP,OAAO,GAAI,EAAE,CAAC;IAEvC,MAAM,OAAO,GAAG,GAAS,EAAE;;QACzB,MAAM,QAAQ,GAAG,MAAM,EAAE,CAAC,gBAAgB,CAAC,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,cAAc,CAAC,CAAC;QACpE,MAAM,KAAK,GAAG,MAAA,CAAC,MAAM,QAAQ,CAAC,YAAY,EAAE,CAAC,mCAAI,CAAC,CAAC;QACnD,OAAO,KAAK,CAAC;IACf,CAAC,CAAA,CAAC;IAEF,qFAAqF;IACrF,MAAM,gBAAgB,GAAG,GAAS,EAAE;QAClC,MAAM,QAAQ,GAAG,MAAM,EAAE,CAAC,gBAAgB,CAAC,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,cAAc,CAAC,CAAC;QACpE,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;IACxB,CAAC,CAAA,CAAC;IAEF,OAAO,IAAA,gCAAiB,EAAC,OAAO,EAAE,OAAO,EAAE,gBAAgB,EAAE,OAAO,CAAC,CAAC;AACxE,CAAC,CAAC;AAnBW,QAAA,uBAAuB,2BAmBlC","sourcesContent":["import { createSizeMonitor, type ValueMonitorOptions } from '../../utils/valueMonitor';\nimport type { CrawleeOneIO } from '../integrations/types';\nimport { apifyIO } from '../integrations/apify';\n\nexport interface RequestQueueSizeMonitorOptions extends ValueMonitorOptions {\n  io?: CrawleeOneIO<any, any>;\n  /**\n   * ID of the RequestQueue that's monitored for size.\n   *\n   * If omitted, the default RequestQueue is used.\n   */\n  requestQueueId?: string;\n}\n\n/**\n * Semi-automatic monitoring of RequestQueue size. This is used for limiting the total of\n * entries scraped per run / RequestQueue:\n * - When RequestQueue reaches `maxSize`, then all remaining Requests are removed.\n * - Pass an array of items to `shortenToSize` to shorten the array to the size\n *   that still fits the RequestQueue.\n *\n * By default uses Apify RequestQueue.\n */\nexport const requestQueueSizeMonitor = (\n  maxSize: number,\n  options?: RequestQueueSizeMonitorOptions\n) => {\n  const { io = apifyIO } = options ?? {};\n\n  const getSize = async () => {\n    const reqQueue = await io.openRequestQueue(options?.requestQueueId);\n    const count = (await reqQueue.handledCount()) ?? 0;\n    return count;\n  };\n\n  // When we've reached the RequestQueue's max size, then remove all remaining Requests\n  const onMaxSizeReached = async () => {\n    const reqQueue = await io.openRequestQueue(options?.requestQueueId);\n    await reqQueue.drop();\n  };\n\n  return createSizeMonitor(maxSize, getSize, onMaxSizeReached, options);\n};\n"]}

package/dist/cjs/lib/log.d.ts

@@ -0,0 +1,38 @@
+import { type CrawlingContext, LogLevel as CrawleeLogLevel } from 'crawlee';
+import type { ArrVal } from '../utils/types';
+import type { CrawlerRouterWrapper } from './router';
+export declare const LOG_LEVEL: readonly ["debug", "info", "warn", "error", "off"];
+export type LogLevel = ArrVal<typeof LOG_LEVEL>;
+/** Map log levels of `crawlee-one` to log levels of `crawlee` */
+export declare const logLevelToCrawlee: Record<LogLevel, CrawleeLogLevel>;
+/**
+ * Wrapper for Crawlee route handler that configures log level.
+ *
+ *
+ * Usage with Crawlee's `RouterHandler.addDefaultHandler`
+ * ```ts
+ * const wrappedHandler = logLevelHandlerWrapper('debug')(handler)
+ * await router.addDefaultHandler<Ctx>(wrappedHandler);
+ * ```
+ *
+ * Usage with Crawlee's `RouterHandler.addHandler`
+ * ```ts
+ * const wrappedHandler = logLevelHandlerWrapper('error')(handler)
+ * await router.addHandler<Ctx>(wrappedHandler);
+ * ```
+ *
+ * Usage with `createCrawleeOne`
+ * ```ts
+ * const actor = await createCrawleeOne<CheerioCrawlingContext>({
+ *   validateInput,
+ *   router: createCheerioRouter(),
+ *   routes,
+ *   routeHandlers: ({ input }) => createHandlers(input!),
+ *   routerWrappers: ({ input }) => [
+ *     logLevelHandlerWrapper<CheerioCrawlingContext<any, any>>(input?.logLevel ?? 'info'),
+ *   ],
+ *   createCrawler: ({ router, input }) => createCrawler({ router, input, crawlerConfig }),
+ * });
+ * ```
+ */
+export declare const logLevelHandlerWrapper: <T extends CrawlingContext<unknown, import("crawlee").Dictionary>, RouterCtx extends Record<string, any> = Record<string, any>>(logLevel: LogLevel) => CrawlerRouterWrapper<T, RouterCtx>;