@crawlee/core 4.0.0-rc.0 → 4.0.0-rc.1

package/enqueue_links/enqueue_links.js

@@ -1,78 +0,0 @@
-import { EnqueueStrategy } from '@crawlee/utils';
-import { getDomain } from 'tldts';
-export { EnqueueStrategy };
-/**
- * @internal
- * This method helps resolve the baseUrl that will be used for filtering in {@link enqueueLinks}.
- * - If a user provides a base url, we always return it
- * - If a user specifies {@link EnqueueStrategy.All} strategy, they do not care if the newly found urls are on the original
- *   request domain, or a redirected one
- * - In all other cases, we return the domain of the original request as that's the one we need to use for filtering
- */
-export function resolveBaseUrlForEnqueueLinksFiltering({ enqueueStrategy, finalRequestUrl, originalRequestUrl, userProvidedBaseUrl, }) {
-    // User provided base url takes priority
-    if (userProvidedBaseUrl) {
-        return userProvidedBaseUrl;
-    }
-    const originalUrlOrigin = new URL(originalRequestUrl).origin;
-    const finalUrlOrigin = new URL(finalRequestUrl ?? originalRequestUrl).origin;
-    // We can assume users want to go off the domain in this case
-    if (enqueueStrategy === EnqueueStrategy.All) {
-        return finalUrlOrigin;
-    }
-    // If the user wants to ensure the same domain is accessed, regardless of subdomains, we check to ensure the domains match
-    // Returning undefined here is intentional! If the domains don't match, having no baseUrl in enqueueLinks will cause it to not enqueue anything
-    // which is the intended behavior (since we went off domain)
-    if (enqueueStrategy === EnqueueStrategy.SameDomain) {
-        const originalHostname = getDomain(originalUrlOrigin, { mixedInputs: false });
-        const finalHostname = getDomain(finalUrlOrigin, { mixedInputs: false });
-        if (originalHostname === finalHostname) {
-            return finalUrlOrigin;
-        }
-        return undefined;
-    }
-    // Always enqueue urls that are from the same origin in all other cases, as the filtering happens on the original request url, even if there was a redirect
-    // before actually finding the urls
-    return originalUrlOrigin;
-}
-/**
- * @internal
- * Builds the glob patterns a URL must match to satisfy the given enqueue `strategy`, anchored at `baseUrl`.
- */
-export function buildEnqueueStrategyPatterns(baseUrl, strategy) {
-    const url = new URL(baseUrl);
-    switch (strategy) {
-        case EnqueueStrategy.SameHostname:
-            // We need to get the origin of the passed in domain in the event someone sets baseUrl
-            // to an url like https://example.com/deep/default/path and one of the found urls is an
-            // absolute relative path (/path/to/page)
-            return [{ glob: ignoreHttpSchema(`${url.origin}/**`) }];
-        case EnqueueStrategy.SameDomain: {
-            // Get the actual hostname from the base url
-            const baseUrlHostname = getDomain(url.hostname, { mixedInputs: false });
-            if (baseUrlHostname) {
-                // We have a hostname, so we can use it to match all links on the page that point to it and any subdomains of it
-                url.hostname = baseUrlHostname;
-                return [
-                    { glob: ignoreHttpSchema(`${url.origin.replace(baseUrlHostname, `*.${baseUrlHostname}`)}/**`) },
-                    { glob: ignoreHttpSchema(`${url.origin}/**`) },
-                ];
-            }
-            // We don't have a hostname (can happen for ips for instance), so reproduce the same behavior
-            // as SameDomainAndSubdomain
-            return [{ glob: ignoreHttpSchema(`${url.origin}/**`) }];
-        }
-        case EnqueueStrategy.SameOrigin:
-            // The same behavior as SameHostname, but respecting the protocol of the URL
-            return [{ glob: `${url.origin}/**` }];
-        case EnqueueStrategy.All:
-        default:
-            return [{ glob: `http{s,}://**` }];
-    }
-}
-/**
- * Internal function that changes the enqueue glob patterns to match both http and https
- */
-function ignoreHttpSchema(pattern) {
-    return pattern.replace(/^(https?):\/\//, 'http{s,}://');
-}

package/enqueue_links/index.d.ts

@@ -1,2 +0,0 @@
-export * from './enqueue_links.js';
-export * from './shared.js';

package/enqueue_links/index.js

@@ -1,2 +0,0 @@
-export * from './enqueue_links.js';
-export * from './shared.js';

package/enqueue_links/shared.d.ts

@@ -1,93 +0,0 @@
-import type { Awaitable, Dictionary } from '@crawlee/types';
-import { z } from 'zod';
-import type { RequestOptions } from '../request.js';
-import type { EnqueueStrategyOption } from './enqueue_links.js';
-export { tryAbsoluteURL } from '@crawlee/utils/internal';
-export interface UrlPatternObject {
-    glob?: string;
-    regexp?: RegExp;
-}
-export interface GlobObject {
-    glob: string;
-}
-export type GlobInput = string | GlobObject;
-export interface RegExpObject {
-    regexp: RegExp;
-}
-export type RegExpInput = RegExp | RegExpObject;
-/** Unified URL pattern input — accepts glob strings, glob objects, RegExp instances, or regexp objects. */
-export type UrlPatternInput = GlobInput | RegExpInput;
-/**
- * Accepts one {@link UrlPatternInput} — a glob string, a RegExp instance, or a `{ glob }` / `{ regexp }` object.
- * @internal
- */
-export declare const urlPatternSchema: z.ZodType<UrlPatternInput>;
-export type SkippedRequestReason = 'robotsTxt' | 'limit' | 'enqueueLimit' | 'filters' | 'transform' | 'redirect' | 'depth';
-export type SkippedRequestCallback = (args: {
-    url: string;
-    reason: SkippedRequestReason;
-}) => Awaitable<void>;
-/**
- * @ignore
- */
-export declare function updateEnqueueLinksPatternCache(item: GlobInput | RegExpInput, pattern: RegExpObject | GlobObject): void;
-/**
- * Helper factory used in the `enqueueLinks()` and enqueueLinksByClickingElements() function
- * to construct Glob objects from Glob pattern strings.
- * @ignore
- */
-export declare function constructGlobObjectsFromGlobs(globs: readonly GlobInput[]): GlobObject[];
-/**
- * @internal
- */
-export declare function validateGlobPattern(glob: string): string;
-/**
- * Helper factory used in the `enqueueLinks()` and enqueueLinksByClickingElements() function
- * to check RegExps input and return valid RegExps.
- * @ignore
- */
-export declare function constructRegExpObjectsFromRegExps(regexps: readonly RegExpInput[]): RegExpObject[];
-/**
- * Helper factory used in the `enqueueLinks()` function to construct UrlPatternObjects
- * from a mixed array of glob strings, glob objects, RegExp instances, and regexp objects.
- * @ignore
- */
-export declare function constructUrlPatternObjects(patterns: readonly UrlPatternInput[]): UrlPatternObject[];
-/**
- * Filters request options by URL patterns.
- *
- * When `includePatterns` is empty/undefined, all options pass through (only exclude filtering applies).
- * @ignore
- */
-export declare function filterRequestOptionsByPatterns(requestOptions: RequestOptions[], includePatterns: UrlPatternObject[] | undefined, excludePatterns?: UrlPatternObject[], strategy?: EnqueueStrategyOption, onSkippedUrl?: (url: string) => void): RequestOptions[];
-/**
- * @ignore
- */
-export declare function createRequestOptions(sources: readonly (string | Record<string, unknown>)[], options?: {
-    label?: string;
-    userData?: Dictionary;
-    baseUrl?: string;
-    skipNavigation?: boolean;
-    sessionId?: string;
-    strategy?: EnqueueStrategyOption;
-}): RequestOptions[];
-/**
- * Takes a {@link RequestOptions} object and changes its attributes in a desired way. This user-function is used
- * by {@link enqueueLinks} to modify request options before they are converted to {@link Request} instances.
- */
-export interface RequestTransform {
-    /**
-     * @param original Request options to be modified.
-     * @returns The modified request options to enqueue, `'unchanged'` to keep the original options as-is,
-     *   or a falsy value / `'skip'` to exclude the request from the queue.
-     */
-    (original: RequestOptions): RequestOptions | false | undefined | null | 'skip' | 'unchanged';
-}
-/**
- * Applies a {@link RequestTransform} function to a list of request options.
- * Options for which the transform returns a falsy value are removed from the list.
- * @param onSkipped Called with the original request options when the transform returns a falsy value (i.e. the request is skipped).
- * @ignore
- * @internal
- */
-export declare function applyRequestTransform(requestOptions: RequestOptions[], transformFn: RequestTransform, onSkipped?: (requestOptions: RequestOptions) => void): RequestOptions[];

package/enqueue_links/shared.js

@@ -1,239 +0,0 @@
-import { URL } from 'node:url';
-import { Minimatch } from 'minimatch';
-import { z } from 'zod';
-import { schemas } from '../validators.js';
-export { tryAbsoluteURL } from '@crawlee/utils/internal';
-const MAX_ENQUEUE_LINKS_CACHE_SIZE = 1000;
-/**
- * To keep high performance when the same patterns are passed on every `enqueueLinks()` call,
- * each glob/regexp is only compiled once and then kept in a cache.
- * @ignore
- */
-const enqueueLinksPatternCache = new Map();
-/**
- * Accepts one {@link UrlPatternInput} — a glob string, a RegExp instance, or a `{ glob }` / `{ regexp }` object.
- * @internal
- */
-export const urlPatternSchema = z.union([
-    z.string(),
-    z.instanceof(RegExp),
-    schemas.objectWithKeys(['glob']),
-    schemas.objectWithKeys(['regexp']),
-]);
-/**
- * @ignore
- */
-export function updateEnqueueLinksPatternCache(item, pattern) {
-    enqueueLinksPatternCache.set(item, pattern);
-    if (enqueueLinksPatternCache.size > MAX_ENQUEUE_LINKS_CACHE_SIZE) {
-        const key = enqueueLinksPatternCache.keys().next().value;
-        enqueueLinksPatternCache.delete(key);
-    }
-}
-/**
- * Helper factory used in the `enqueueLinks()` and enqueueLinksByClickingElements() function
- * to construct Glob objects from Glob pattern strings.
- * @ignore
- */
-export function constructGlobObjectsFromGlobs(globs) {
-    return globs
-        .filter((glob) => {
-        // Skip possibly nullish, empty strings
-        if (!glob) {
-            return false;
-        }
-        if (typeof glob === 'string') {
-            return glob.trim().length > 0;
-        }
-        if (glob.glob) {
-            return glob.glob.trim().length > 0;
-        }
-        return false;
-    })
-        .map((item) => {
-        // Get glob object from cache.
-        let globObject = enqueueLinksPatternCache.get(item);
-        if (globObject)
-            return globObject;
-        if (typeof item === 'string') {
-            globObject = { glob: validateGlobPattern(item) };
-        }
-        else {
-            globObject = { glob: validateGlobPattern(item.glob) };
-        }
-        updateEnqueueLinksPatternCache(item, globObject);
-        return globObject;
-    });
-}
-/**
- * @internal
- */
-export function validateGlobPattern(glob) {
-    const globTrimmed = glob.trim();
-    if (globTrimmed.length === 0)
-        throw new Error(`Cannot parse Glob pattern '${globTrimmed}': it must be an non-empty string`);
-    return globTrimmed;
-}
-/**
- * Helper factory used in the `enqueueLinks()` and enqueueLinksByClickingElements() function
- * to check RegExps input and return valid RegExps.
- * @ignore
- */
-export function constructRegExpObjectsFromRegExps(regexps) {
-    return regexps.map((item) => {
-        // Get regexp object from cache.
-        let regexpObject = enqueueLinksPatternCache.get(item);
-        if (regexpObject)
-            return regexpObject;
-        if (item instanceof RegExp) {
-            regexpObject = { regexp: item };
-        }
-        else {
-            regexpObject = { regexp: item.regexp };
-        }
-        updateEnqueueLinksPatternCache(item, regexpObject);
-        return regexpObject;
-    });
-}
-/**
- * Helper factory used in the `enqueueLinks()` function to construct UrlPatternObjects
- * from a mixed array of glob strings, glob objects, RegExp instances, and regexp objects.
- * @ignore
- */
-export function constructUrlPatternObjects(patterns) {
-    const result = [];
-    for (const item of patterns) {
-        if (typeof item === 'string' || 'glob' in item) {
-            result.push(...constructGlobObjectsFromGlobs([item]));
-        }
-        else if (item instanceof RegExp || 'regexp' in item) {
-            result.push(...constructRegExpObjectsFromRegExps([item]));
-        }
-    }
-    return result;
-}
-/**
- * Filters request options by URL patterns.
- *
- * When `includePatterns` is empty/undefined, all options pass through (only exclude filtering applies).
- * @ignore
- */
-export function filterRequestOptionsByPatterns(requestOptions, includePatterns, excludePatterns = [], strategy, onSkippedUrl) {
-    const excludeMatchers = excludePatterns.map(createPatternObjectMatcher);
-    const includeMatchers = includePatterns?.length ? includePatterns.map(createPatternObjectMatcher) : undefined;
-    return requestOptions
-        .filter(({ url }) => {
-        const matchesExclude = excludeMatchers.some(({ match }) => match(url));
-        if (matchesExclude) {
-            onSkippedUrl?.(url);
-        }
-        return !matchesExclude;
-    })
-        .map((opts) => {
-        if (!includeMatchers) {
-            return { ...opts, enqueueStrategy: strategy };
-        }
-        for (const { match } of includeMatchers) {
-            if (match(opts.url)) {
-                return { ...opts, enqueueStrategy: strategy };
-            }
-        }
-        // didn't match any positive pattern
-        onSkippedUrl?.(opts.url);
-        return null;
-    })
-        .filter((opts) => opts !== null);
-}
-function isAbsoluteUrl(url) {
-    try {
-        // eslint-disable-next-line no-new
-        new URL(url);
-        return true;
-    }
-    catch {
-        return false;
-    }
-}
-/**
- * @ignore
- */
-export function createRequestOptions(sources, options = {}) {
-    return sources
-        .map((src) => typeof src === 'string'
-        ? { url: src, enqueueStrategy: options.strategy }
-        : { ...src, enqueueStrategy: options.strategy })
-        .filter(({ url }) => {
-        try {
-            return new URL(url, options.baseUrl).href;
-        }
-        catch (err) {
-            return false;
-        }
-    })
-        .map((requestOptions) => {
-        // Leave already-absolute URLs untouched - re-deriving them via `new URL()` would normalize them
-        // (e.g. adding a trailing slash to a bare domain), which is surprising for URLs that didn't need
-        // resolving against `baseUrl` in the first place.
-        if (!isAbsoluteUrl(requestOptions.url)) {
-            requestOptions.url = new URL(requestOptions.url, options.baseUrl).href;
-        }
-        requestOptions.userData ??= options.userData ?? {};
-        if (typeof options.label === 'string') {
-            requestOptions.userData = {
-                ...requestOptions.userData,
-                label: options.label,
-            };
-        }
-        if (options.skipNavigation) {
-            requestOptions.skipNavigation = true;
-        }
-        if (options.sessionId) {
-            requestOptions.sessionId = options.sessionId;
-        }
-        return requestOptions;
-    });
-}
-/**
- * @ignore
- */
-function createPatternObjectMatcher(urlPatternObject) {
-    const { regexp, glob } = urlPatternObject;
-    let match;
-    if (regexp) {
-        match = (url) => regexp.test(url);
-    }
-    else if (glob) {
-        const m = new Minimatch(glob, { nocase: true });
-        match = (url) => m.match(url);
-    }
-    else {
-        match = () => false;
-    }
-    return { match };
-}
-/**
- * Applies a {@link RequestTransform} function to a list of request options.
- * Options for which the transform returns a falsy value are removed from the list.
- * @param onSkipped Called with the original request options when the transform returns a falsy value (i.e. the request is skipped).
- * @ignore
- * @internal
- */
-export function applyRequestTransform(requestOptions, transformFn, onSkipped) {
-    return requestOptions
-        .map((opts) => {
-        const transformed = transformFn(opts);
-        if (transformed === 'skip') {
-            onSkipped?.(opts);
-            return null;
-        }
-        if (transformed === 'unchanged') {
-            return opts;
-        }
-        if (!transformed) {
-            onSkipped?.(opts);
-            return null;
-        }
-        return transformed;
-    })
-        .filter((r) => r !== null);
-}

package/http.d.ts

@@ -1,9 +0,0 @@
-/**
- * Parses a `Retry-After` response header into a delay in milliseconds.
- *
- * The header holds either a non-negative number of seconds or an HTTP-date.
- * See [MDN](https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Headers/Retry-After).
- *
- * @returns The delay in milliseconds, or `null` if the header is absent, unparseable, or already elapsed.
- */
-export declare function parseRetryAfterHeader(value?: string | null): number | null;

package/http.js

@@ -1,28 +0,0 @@
-/**
- * Parses a `Retry-After` response header into a delay in milliseconds.
- *
- * The header holds either a non-negative number of seconds or an HTTP-date.
- * See [MDN](https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Headers/Retry-After).
- *
- * @returns The delay in milliseconds, or `null` if the header is absent, unparseable, or already elapsed.
- */
-export function parseRetryAfterHeader(value) {
-    if (!value) {
-        return null;
-    }
-    const trimmed = value.trim();
-    // Per the spec this is a `delay-seconds`: digits only, so a negative or fractional value is not one.
-    if (/^\d+$/.test(trimmed)) {
-        // `Retry-After: 0` names no future deadline, same as an HTTP-date that has already passed. Reporting it
-        // as a zero delay would leave the domain unthrottled while still counting as a rate-limit event, so the
-        // caller would defer the request for free and re-send it immediately.
-        const delayMs = Number(trimmed) * 1000;
-        return delayMs > 0 ? delayMs : null;
-    }
-    const date = Date.parse(trimmed);
-    if (!Number.isNaN(date)) {
-        const delayMs = date - Date.now();
-        return delayMs > 0 ? delayMs : null;
-    }
-    return null;
-}