@crawlee/utils 4.0.0-beta.99 → 4.0.0-rc.1

package/README.md

@@ -34,7 +34,7 @@ Crawlee is available as the [`crawlee`](https://www.npmjs.com/package/crawlee) N
 
 We recommend visiting the [Introduction tutorial](https://crawlee.dev/js/docs/introduction) in Crawlee documentation for more information.
 
-> Crawlee requires **Node.js 16 or higher**.
+> Crawlee requires **Node.js 22.13 or higher**.
 
 ### With Crawlee CLI
 

package/index.d.ts

@@ -1,10 +1,12 @@
-export * from './internals/blocked.js';
-export * from './internals/cheerio.js';
-export * from './internals/extract-urls.js';
-export * from './internals/general.js';
+export { htmlToText } from './internals/cheerio.js';
+export { downloadListOfUrls, extractUrls } from './internals/extract-urls.js';
+export { EnqueueStrategy } from './internals/url.js';
+export type { DownloadListOfUrlsOptions, ExtractUrlsOptions } from './internals/extract-urls.js';
+export { sleep } from './internals/general.js';
 export * as social from './internals/social.js';
+export * from './internals/extract-microdata.js';
 export * from './internals/open_graph_parser.js';
 export * from './internals/robots.js';
-export * from './internals/sitemap.js';
-export * from './internals/iterables.js';
-export * from './internals/url.js';
+export { discoverValidSitemaps, Sitemap } from './internals/sitemap.js';
+export type { ParseSitemapOptions } from './internals/sitemap.js';
+export { ArgumentValidationError } from './internals/validation.js';

package/index.js

@@ -1,10 +1,10 @@
-export * from './internals/blocked.js';
-export * from './internals/cheerio.js';
-export * from './internals/extract-urls.js';
-export * from './internals/general.js';
+export { htmlToText } from './internals/cheerio.js';
+export { downloadListOfUrls, extractUrls } from './internals/extract-urls.js';
+export { EnqueueStrategy } from './internals/url.js';
+export { sleep } from './internals/general.js';
 export * as social from './internals/social.js';
+export * from './internals/extract-microdata.js';
 export * from './internals/open_graph_parser.js';
 export * from './internals/robots.js';
-export * from './internals/sitemap.js';
-export * from './internals/iterables.js';
-export * from './internals/url.js';
+export { discoverValidSitemaps, Sitemap } from './internals/sitemap.js';
+export { ArgumentValidationError } from './internals/validation.js';

package/internal.d.ts

@@ -0,0 +1,10 @@
+export * from './internals/blocked.js';
+export type { CheerioAPI, Cheerio, Element } from './internals/cheerio.js';
+export { extractUrlsFromCheerio } from './internals/cheerio.js';
+export { tryAbsoluteURL } from './internals/extract-urls.js';
+export { expandShadowRoots, URL_NO_COMMAS_REGEX, URL_WITH_COMMAS_REGEX } from './internals/general.js';
+export * from './internals/iterables.js';
+export * from './internals/url.js';
+export * from './internals/validation.js';
+export * as schemas from './internals/schemas.js';
+export { parseSitemap, type SitemapUrl } from './internals/sitemap.js';

package/internal.js

@@ -0,0 +1,9 @@
+export * from './internals/blocked.js';
+export { extractUrlsFromCheerio } from './internals/cheerio.js';
+export { tryAbsoluteURL } from './internals/extract-urls.js';
+export { expandShadowRoots, URL_NO_COMMAS_REGEX, URL_WITH_COMMAS_REGEX } from './internals/general.js';
+export * from './internals/iterables.js';
+export * from './internals/url.js';
+export * from './internals/validation.js';
+export * as schemas from './internals/schemas.js';
+export { parseSitemap } from './internals/sitemap.js';

package/internals/cheerio.d.ts

@@ -1,5 +1,4 @@
 import type { CheerioAPI } from 'cheerio';
-export type CheerioRoot = CheerioAPI;
 export type { CheerioAPI, Cheerio } from 'cheerio';
 export type { Element } from 'domhandler';
 /**
@@ -29,7 +28,7 @@ export type { Element } from 'domhandler';
  * @param htmlOrCheerioElement HTML text or parsed HTML represented using a [cheerio](https://www.npmjs.com/package/cheerio) function.
  * @return Plain text
  */
-export declare function htmlToText(htmlOrCheerioElement: string | CheerioRoot): Promise<string>;
+export declare function htmlToText(htmlOrCheerioElement: string | CheerioAPI): Promise<string>;
 /**
  * Extracts URLs from a given Cheerio object.
  *

package/internals/cheerio.js

@@ -30,7 +30,7 @@ const BLOCK_TAGS_REGEX = /^(p|h1|h2|h3|h4|h5|h6|ol|ul|li|pre|address|blockquote|
  * @return Plain text
  */
 export async function htmlToText(htmlOrCheerioElement) {
-    const { load } = await import('cheerio');
+    const { load } = await import('cheerio/slim');
     if (!htmlOrCheerioElement)
         return '';
     const $ = typeof htmlOrCheerioElement === 'function' ? htmlOrCheerioElement : load(htmlOrCheerioElement);
@@ -42,8 +42,10 @@ export async function htmlToText(htmlOrCheerioElement) {
             if (elem.type === 'text') {
                 // Compress spaces, unless we're inside <pre> element
                 let compr;
-                if (elem.parent?.tagName === 'pre')
-                    compr = elem.data;
+                if (elem.parent?.tagName === 'pre') {
+                    // Strip single newline right after `<pre>` that htmlparser2 keeps
+                    compr = elem.parent.children?.[0] === elem ? elem.data.replace(/^\r?\n/, '') : elem.data;
+                }
                 else
                     compr = elem.data.replace(/\s+/g, ' ');
                 // If text is empty or ends with a whitespace, don't add the leading whitespace

package/internals/extract-microdata.d.ts

@@ -0,0 +1,24 @@
+import type { CheerioAPI } from 'cheerio';
+/** The value of a microdata property: either text or a nested item. */
+export type MicrodataValue = string | MicrodataItem;
+/** A single schema.org item extracted from a document. */
+export interface MicrodataItem {
+    /** Tokens of the item's `itemtype` attribute. */
+    type?: string[];
+    /** The item's `itemid` attribute. */
+    id?: string;
+    /** Values keyed by `itemprop` name, an array where the property repeats. */
+    properties: Record<string, MicrodataValue | MicrodataValue[]>;
+}
+/**
+ * Easily parse all schema.org microdata from a page with just a `CheerioAPI` object or raw HTML,
+ * following the [microdata processing model](https://html.spec.whatwg.org/multipage/microdata.html#microdata).
+ *
+ * Text values are trimmed and their inner whitespace collapsed. URL-valued attributes are returned
+ * verbatim rather than resolved against the document's base URL.
+ *
+ * @param htmlOrCheerioElement A `CheerioAPI` object, or a string of raw HTML.
+ * @returns The document's top-level items. Nested items are the property values of their parent.
+ */
+export declare function extractMicrodata(raw: string): Promise<MicrodataItem[]>;
+export declare function extractMicrodata($: CheerioAPI): Promise<MicrodataItem[]>;

package/internals/extract-microdata.js

@@ -0,0 +1,118 @@
+import { isTag } from 'domhandler';
+export async function extractMicrodata(htmlOrCheerioElement) {
+    // Dynamic so that importing `@crawlee/utils` does not pull in cheerio - see #3836.
+    const { load } = await import('cheerio/slim');
+    const $ = typeof htmlOrCheerioElement === 'string' ? load(htmlOrCheerioElement) : htmlOrCheerioElement;
+    const context = { $ };
+    return $('[itemscope]')
+        .toArray()
+        .filter((element) => !('itemprop' in element.attribs))
+        .map((element) => parseItem(context, element, new Set()));
+}
+function parseItem(context, element, ancestors) {
+    const item = { properties: {} };
+    const type = uniqueTokens(element.attribs.itemtype);
+    const id = element.attribs.itemid;
+    if (type.length > 0) {
+        item.type = type;
+    }
+    if (id) {
+        item.id = id.trim();
+    }
+    ancestors.add(element);
+    for (const propertyElement of collectPropertyElements(context, element)) {
+        const value = getPropertyValue(context, propertyElement, ancestors);
+        for (const name of uniqueTokens(propertyElement.attribs.itemprop)) {
+            addProperty(item.properties, name, value);
+        }
+    }
+    ancestors.delete(element);
+    return item;
+}
+function collectPropertyElements(context, scope) {
+    const elements = [];
+    collectFromNodes(scope.children, elements);
+    for (const id of uniqueTokens(scope.attribs.itemref)) {
+        const referenced = (context.idIndex ??= indexIds(context.$)).get(id);
+        if (referenced) {
+            collectFromNodes([referenced], elements);
+        }
+    }
+    return elements;
+}
+function collectFromNodes(nodes, elements) {
+    for (const node of nodes) {
+        if (!isTag(node)) {
+            continue;
+        }
+        if ('itemprop' in node.attribs) {
+            elements.push(node);
+        }
+        // A nested item owns everything below it, so its subtree is not part of the enclosing item.
+        if (!('itemscope' in node.attribs)) {
+            collectFromNodes(node.children, elements);
+        }
+    }
+}
+function indexIds($) {
+    const index = new Map();
+    for (const element of $('[id]').toArray()) {
+        // Duplicate ids are invalid HTML; `getElementById` resolves them to the first element.
+        if (!index.has(element.attribs.id)) {
+            index.set(element.attribs.id, element);
+        }
+    }
+    return index;
+}
+function getPropertyValue(context, element, ancestors) {
+    if ('itemscope' in element.attribs) {
+        // `itemref` can point back at an enclosing item, which the spec treats as an error.
+        return ancestors.has(element) ? { properties: {} } : parseItem(context, element, ancestors);
+    }
+    const { attribs } = element;
+    switch (element.tagName.toLowerCase()) {
+        case 'meta':
+            return attribs.content ?? '';
+        case 'audio':
+        case 'embed':
+        case 'iframe':
+        case 'img':
+        case 'source':
+        case 'track':
+        case 'video':
+            return attribs.src ?? '';
+        case 'a':
+        case 'area':
+        case 'link':
+            return attribs.href ?? '';
+        case 'object':
+            return attribs.data ?? '';
+        case 'data':
+        case 'meter':
+            return attribs.value ?? '';
+        case 'time':
+            return attribs.datetime ?? context.$(element).text().replace(/\s+/g, ' ').trim();
+        default:
+            return context.$(element).text().replace(/\s+/g, ' ').trim();
+    }
+}
+function addProperty(properties, name, value) {
+    const existing = properties[name];
+    if (existing === undefined) {
+        properties[name] = value;
+    }
+    else if (Array.isArray(existing)) {
+        existing.push(value);
+    }
+    else {
+        properties[name] = [existing, value];
+    }
+}
+/** `itemtype`, `itemprop` and `itemref` are all unordered sets of unique space-separated tokens. */
+function uniqueTokens(value) {
+    if (!value) {
+        return [];
+    }
+    const tokens = value.split(/\s+/).filter(Boolean);
+    return tokens.length > 1 ? [...new Set(tokens)] : tokens;
+}

package/internals/extract-urls.d.ts

@@ -1,4 +1,4 @@
-import type { BaseHttpClient } from '@crawlee/types';
+import type { BaseHttpClient } from '@crawlee/http-client';
 export interface DownloadListOfUrlsOptions {
     /**
      * URL to the file

package/internals/extract-urls.js

@@ -1,19 +1,25 @@
 import { FetchHttpClient } from '@crawlee/http-client';
-import ow from 'ow';
+import { z } from 'zod';
 import { URL_NO_COMMAS_REGEX } from './general.js';
+import { httpClient as httpClientSchema } from './schemas.js';
+import { parseArgument } from './validation.js';
+const downloadListOfUrlsOptionsSchema = z.strictObject({
+    url: z.url(),
+    encoding: z.string().default('utf8'),
+    urlRegExp: z.instanceof(RegExp).default(URL_NO_COMMAS_REGEX),
+    proxyUrl: z.string().optional(),
+    httpClient: httpClientSchema.default(() => new FetchHttpClient()),
+});
+const extractUrlsOptionsSchema = z.strictObject({
+    string: z.string(),
+    urlRegExp: z.instanceof(RegExp).default(URL_NO_COMMAS_REGEX),
+});
 /**
  * Returns a promise that resolves to an array of urls parsed from the resource available at the provided url.
  * Optionally, custom regular expression and encoding may be provided.
  */
 export async function downloadListOfUrls(options) {
-    ow(options, ow.object.exactShape({
-        url: ow.string.url,
-        encoding: ow.optional.string,
-        urlRegExp: ow.optional.regExp,
-        proxyUrl: ow.optional.string,
-        httpClient: ow.optional.object,
-    }));
-    const { url, encoding = 'utf8', urlRegExp = URL_NO_COMMAS_REGEX, proxyUrl, httpClient = new FetchHttpClient(), } = options;
+    const { url, encoding, urlRegExp, proxyUrl, httpClient } = parseArgument(options, downloadListOfUrlsOptionsSchema);
     // Try to detect wrong urls and fix them. Currently, detects only sharing url instead of csv download one.
     const match = /^(https:\/\/docs\.google\.com\/spreadsheets\/d\/(?:\w|-)+)\/?/.exec(url);
     let fixedUrl = url;
@@ -30,13 +36,9 @@ export async function downloadListOfUrls(options) {
  * Collects all URLs in an arbitrary string to an array, optionally using a custom regular expression.
  */
 export function extractUrls(options) {
-    ow(options, ow.object.exactShape({
-        string: ow.string,
-        urlRegExp: ow.optional.regExp,
-    }));
-    const lines = options.string.split('\n');
+    const { string, urlRegExp } = parseArgument(options, extractUrlsOptionsSchema);
+    const lines = string.split('\n');
     const result = [];
-    const urlRegExp = options.urlRegExp ?? URL_NO_COMMAS_REGEX;
     for (const line of lines) {
         result.push(...(line.match(urlRegExp) ?? []));
     }

package/internals/open_graph_parser.js

@@ -366,7 +366,7 @@ const parseOpenGraphProperty = (property, $) => {
     };
 };
 export async function parseOpenGraph(item, additionalProperties) {
-    const { load } = await import('cheerio');
+    const { load } = await import('cheerio/slim');
     const $ = typeof item === 'string' ? load(item) : item;
     return [...(additionalProperties || []), ...OPEN_GRAPH_PROPERTIES].reduce((acc, curr) => {
         return {

package/internals/robots.d.ts

@@ -1,5 +1,15 @@
-import type { BaseHttpClient, CrawleeLogger } from '@crawlee/types';
+import type { BaseHttpClient } from '@crawlee/http-client';
+import type { CrawleeLogger } from '@crawlee/types';
 import { Sitemap } from './sitemap.js';
+import { type EnqueueStrategy } from './url.js';
+export interface RobotsTxtFileSitemapsOptions {
+    /**
+     * Keep only sitemap URLs matching this strategy relative to the robots.txt host; non-`http(s)` schemes
+     * are always dropped. Pass `'all'` to disable host filtering.
+     * @default 'same-hostname'
+     */
+    enqueueStrategy?: EnqueueStrategy | `${EnqueueStrategy}`;
+}
 /**
  * Loads and queries information from a [robots.txt file](https://en.wikipedia.org/wiki/Robots.txt).
  *
@@ -19,9 +29,7 @@ import { Sitemap } from './sitemap.js';
  * ```
  */
 export declare class RobotsTxtFile {
-    private robots;
-    private proxyUrl?;
-    private logger?;
+    #private;
     private constructor();
     /**
      * Determine the location of a robots.txt file for a URL and fetch it.
@@ -45,7 +53,11 @@ export declare class RobotsTxtFile {
      * @param [proxyUrl] a proxy to be used for fetching the robots.txt file
      */
     static from(url: string, content: string, proxyUrl?: string): RobotsTxtFile;
-    private static load;
+    /**
+     * Get crawl delay for a given user agent.
+     * @param [userAgent] relevant user agent, default to `*`
+     */
+    getCrawlDelay(userAgent?: string): number | undefined;
     /**
      * Check if a URL should be crawled by robots.
      * @param url the URL to check against the rules in robots.txt
@@ -53,15 +65,19 @@ export declare class RobotsTxtFile {
      */
     isAllowed(url: string, userAgent?: string): boolean;
     /**
-     * Get URLs of sitemaps referenced in the robots file.
+     * Get URLs of sitemaps referenced in the robots file, filtered by `options.enqueueStrategy` relative to
+     * the robots.txt host (default `'same-hostname'`; pass `'all'` to disable). Non-`http(s)` schemes are
+     * always dropped.
      */
-    getSitemaps(): string[];
+    getSitemaps(options?: RobotsTxtFileSitemapsOptions): string[];
     /**
-     * Parse all the sitemaps referenced in the robots file.
+     * Parse all the sitemaps referenced in the robots file. `options` are forwarded to `getSitemaps`
+     * and the sitemap parser.
      */
-    parseSitemaps(): Promise<Sitemap>;
+    parseSitemaps(options?: RobotsTxtFileSitemapsOptions): Promise<Sitemap>;
     /**
      * Get all URLs from all the sitemaps referenced in the robots file. A shorthand for `(await robots.parseSitemaps()).urls`.
+     * `options` are forwarded to `parseSitemaps`.
      */
-    parseUrlsFromSitemaps(): Promise<string[]>;
+    parseUrlsFromSitemaps(options?: RobotsTxtFileSitemapsOptions): Promise<string[]>;
 }

package/internals/robots.js

@@ -1,6 +1,7 @@
 import { FetchHttpClient } from '@crawlee/http-client';
 import robotsParser from 'robots-parser';
 import { Sitemap } from './sitemap.js';
+import { filterUrl } from './url.js';
 /**
  * Loads and queries information from a [robots.txt file](https://en.wikipedia.org/wiki/Robots.txt).
  *
@@ -20,13 +21,15 @@ import { Sitemap } from './sitemap.js';
  * ```
  */
 export class RobotsTxtFile {
-    robots;
-    proxyUrl;
-    logger;
-    constructor(robots, proxyUrl, logger) {
-        this.robots = robots;
-        this.proxyUrl = proxyUrl;
-        this.logger = logger;
+    #url;
+    #robots;
+    #proxyUrl;
+    #logger;
+    constructor(url, robots, proxyUrl, logger) {
+        this.#url = url;
+        this.#robots = robots;
+        this.#proxyUrl = proxyUrl;
+        this.#logger = logger;
     }
     /**
      * Determine the location of a robots.txt file for a URL and fetch it.
@@ -40,7 +43,7 @@ export class RobotsTxtFile {
         const robotsTxtFileUrl = new URL(url);
         robotsTxtFileUrl.pathname = '/robots.txt';
         robotsTxtFileUrl.search = '';
-        return RobotsTxtFile.load(robotsTxtFileUrl.toString(), options);
+        return RobotsTxtFile.#load(robotsTxtFileUrl.toString(), options);
     }
     /**
      * Allows providing the URL and robots.txt content explicitly instead of loading it from the target site.
@@ -50,30 +53,42 @@ export class RobotsTxtFile {
      */
     static from(url, content, proxyUrl) {
         // @ts-ignore
-        return new RobotsTxtFile(robotsParser(url, content), proxyUrl);
+        return new RobotsTxtFile(url, robotsParser(url, content), proxyUrl);
     }
-    static async load(url, options) {
+    static async #load(url, options) {
         const { proxyUrl, logger, httpClient = new FetchHttpClient() } = options || {};
         const response = await httpClient.sendRequest(new Request(url, { method: 'GET' }), {
             proxyUrl,
             timeoutMillis: options?.timeoutMillis,
             signal: options?.signal,
         });
-        if (response.status < 200 || response.status >= 300) {
-            throw new Error(`Failed to load robots.txt from ${url}: HTTP ${response.status}`);
-        }
-        if (response.status === 404) {
-            return new RobotsTxtFile({
+        // RFC 9309: an unavailable (4xx) robots.txt allows everything, an unreachable (5xx) one disallows everything.
+        if (response.status >= 400 && response.status < 600) {
+            const allowed = response.status < 500;
+            return new RobotsTxtFile(url, {
                 isAllowed() {
-                    return true;
+                    return allowed;
                 },
                 getSitemaps() {
                     return [];
                 },
+                getCrawlDelay() {
+                    return undefined;
+                },
             }, proxyUrl, logger);
         }
+        if (response.status < 200 || response.status >= 300) {
+            throw new Error(`Failed to load robots.txt from ${url}: HTTP ${response.status}`);
+        }
         // @ts-ignore
-        return new RobotsTxtFile(robotsParser(url.toString(), await response.text()), proxyUrl, logger);
+        return new RobotsTxtFile(url, robotsParser(url.toString(), await response.text()), proxyUrl, logger);
+    }
+    /**
+     * Get crawl delay for a given user agent.
+     * @param [userAgent] relevant user agent, default to `*`
+     */
+    getCrawlDelay(userAgent = '*') {
+        return this.#robots.getCrawlDelay(userAgent);
     }
     /**
      * Check if a URL should be crawled by robots.
@@ -81,24 +96,39 @@ export class RobotsTxtFile {
      * @param [userAgent] relevant user agent, default to `*`
      */
     isAllowed(url, userAgent = '*') {
-        return this.robots.isAllowed(url, userAgent) ?? true; // `undefined` means that there is no explicit rule for the requested URL - assume it's allowed
+        return this.#robots.isAllowed(url, userAgent) ?? true; // `undefined` means that there is no explicit rule for the requested URL - assume it's allowed
     }
     /**
-     * Get URLs of sitemaps referenced in the robots file.
+     * Get URLs of sitemaps referenced in the robots file, filtered by `options.enqueueStrategy` relative to
+     * the robots.txt host (default `'same-hostname'`; pass `'all'` to disable). Non-`http(s)` schemes are
+     * always dropped.
      */
-    getSitemaps() {
-        return this.robots.getSitemaps();
+    getSitemaps(options = {}) {
+        const { enqueueStrategy = 'same-hostname' } = options;
+        const sitemaps = [];
+        for (const sitemapUrl of this.#robots.getSitemaps()) {
+            // `filterUrl` tolerates an unparseable origin (returns not-allowed) rather than throwing.
+            const { allowed, reason } = filterUrl(sitemapUrl, this.#url, enqueueStrategy);
+            if (!allowed) {
+                this.#logger?.warning(`Skipping sitemap ${sitemapUrl} listed in robots.txt at ${this.#url}: ${reason}.`);
+                continue;
+            }
+            sitemaps.push(sitemapUrl);
+        }
+        return sitemaps;
     }
     /**
-     * Parse all the sitemaps referenced in the robots file.
+     * Parse all the sitemaps referenced in the robots file. `options` are forwarded to `getSitemaps`
+     * and the sitemap parser.
      */
-    async parseSitemaps() {
-        return Sitemap.load(this.robots.getSitemaps(), this.proxyUrl, { logger: this.logger });
+    async parseSitemaps(options = {}) {
+        return Sitemap.load(this.getSitemaps(options), this.#proxyUrl, { ...options, logger: this.#logger });
     }
     /**
      * Get all URLs from all the sitemaps referenced in the robots file. A shorthand for `(await robots.parseSitemaps()).urls`.
+     * `options` are forwarded to `parseSitemaps`.
      */
-    async parseUrlsFromSitemaps() {
-        return (await this.parseSitemaps()).urls;
+    async parseUrlsFromSitemaps(options = {}) {
+        return (await this.parseSitemaps(options)).urls;
     }
 }

package/internals/schemas.d.ts

@@ -0,0 +1,114 @@
+import { BaseHttpClient } from '@crawlee/http-client';
+import type { Dictionary } from '@crawlee/types';
+import { z } from 'zod';
+/**
+ * Accepts any object (including arrays and functions).
+ * @internal
+ */
+export declare const anyObject: z.ZodCustom<Dictionary, Dictionary>;
+/**
+ * Accepts any array without validating its items (cheap for huge arrays).
+ * @internal
+ */
+export declare const anyArray: z.ZodCustom<unknown[], unknown[]>;
+/**
+ * Accepts any function.
+ * @internal
+ */
+export declare const anyFunction: z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>;
+/**
+ * Mirrors `ow.number`: `Infinity` is a valid number, `NaN` is not.
+ * @internal
+ */
+export declare const anyNumber: z.ZodCustom<number, number>;
+/**
+ * Accepts any object (including functions) that has all the given keys, own or inherited.
+ * @internal
+ */
+export declare function objectWithKeys(keys: string[], message?: string): z.ZodType<Dictionary>;
+/**
+ * Accepts only instances of {@link BaseHttpClient} (all Crawlee HTTP clients extend it).
+ * @internal
+ */
+export declare const httpClient: z.ZodInstanceOf<BaseHttpClient>;
+/**
+ * Accepts any object implementing the CrawleeLogger interface.
+ * @internal
+ */
+export declare const logger: z.ZodType<Dictionary, unknown, z.core.$ZodTypeInternals<Dictionary, unknown>>;
+/**
+ * Accepts any typed array (`Uint8Array`, `Float64Array`, ...), but not a `DataView`.
+ * @internal
+ */
+export declare const typedArray: z.ZodCustom<NodeJS.TypedArray<ArrayBufferLike>, NodeJS.TypedArray<ArrayBufferLike>>;
+/**
+ * Accepts any non-null, non-array object.
+ * @internal
+ */
+export declare const plainObject: z.ZodCustom<Record<string, unknown>, Record<string, unknown>>;
+/**
+ * Shape of a request stored in a request queue. Compiled: storage clients validate every request with it.
+ * @internal
+ */
+export declare const storageRequest: z.ZodObject<{
+    id: z.ZodString;
+    url: z.ZodURL;
+    uniqueKey: z.ZodString;
+    method: z.ZodOptional<z.ZodString>;
+    retryCount: z.ZodOptional<z.ZodNumber>;
+    handledAt: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodDate]>>;
+}, z.core.$loose>;
+/**
+ * {@link storageRequest} before an id is assigned.
+ * @internal
+ */
+export declare const storageRequestWithoutId: z.ZodObject<{
+    url: z.ZodURL;
+    uniqueKey: z.ZodString;
+    method: z.ZodOptional<z.ZodString>;
+    retryCount: z.ZodOptional<z.ZodNumber>;
+    handledAt: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodDate]>>;
+}, z.core.$loose>;
+/**
+ * `z.array(item)` whose top-level type error names the element type — ``expected an array of numbers`` —
+ * instead of zod's bare `expected array`. Element failures keep zod's per-index messages, and `elements`
+ * is a human-readable plural (`'numbers'`, `'URL patterns'`), since element types cannot be introspected.
+ * @internal
+ */
+export declare function arrayOf<TItem extends z.ZodType>(item: TItem, elements: string): z.ZodArray<TItem>;
+/**
+ * Batch of {@link storageRequestWithoutId}.
+ * @internal
+ */
+export declare const storageRequestBatch: z.ZodArray<z.ZodObject<{
+    url: z.ZodURL;
+    uniqueKey: z.ZodString;
+    method: z.ZodOptional<z.ZodString>;
+    retryCount: z.ZodOptional<z.ZodNumber>;
+    handledAt: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodDate]>>;
+}, z.core.$loose>>;
+/**
+ * Options of request queue add/update operations.
+ * @internal
+ */
+export declare const requestQueueOperationOptions: z.ZodObject<{
+    forefront: z.ZodOptional<z.ZodBoolean>;
+}, z.core.$strip>;
+/**
+ * Options of key-value store `listKeys`.
+ * @internal
+ */
+export declare const keyValueStoreListKeysOptions: z.ZodObject<{
+    prefix: z.ZodOptional<z.ZodString>;
+    exclusiveStartKey: z.ZodOptional<z.ZodString>;
+    limit: z.ZodOptional<z.ZodNumber>;
+}, z.core.$strip>;
+/**
+ * Options of dataset item listing.
+ * @internal
+ */
+export declare const datasetListItemsOptions: z.ZodObject<{
+    desc: z.ZodOptional<z.ZodBoolean>;
+    limit: z.ZodOptional<z.ZodNumber>;
+    offset: z.ZodOptional<z.ZodNumber>;
+}, z.core.$strip>;