@crawlee/core 4.0.0-beta.149 → 4.0.0-beta.150

package/enqueue_links/enqueue_links.d.ts

@@ -139,7 +139,7 @@ export type EnqueueStrategyOption = EnqueueStrategy | 'all' | 'same-domain' | 's
  *   request domain, or a redirected one
  * - In all other cases, we return the domain of the original request as that's the one we need to use for filtering
  */
-export declare function resolveBaseUrlForEnqueueLinksFiltering({ enqueueStrategy, finalRequestUrl, originalRequestUrl, userProvidedBaseUrl, }: ResolveBaseUrl): string | undefined;
+export declare function resolveBaseUrlForEnqueueLinksFiltering({ enqueueStrategy, finalRequestUrl, originalRequestUrl, userProvidedBaseUrl, }: ResolveBaseUrl): string;
 /**
  * @internal
  */

package/enqueue_links/enqueue_links.js

@@ -21,15 +21,14 @@ export function resolveBaseUrlForEnqueueLinksFiltering({ enqueueStrategy, finalR
         return finalUrlOrigin;
     }
     // If the user wants to ensure the same domain is accessed, regardless of subdomains, we check to ensure the domains match
-    // Returning undefined here is intentional! If the domains don't match, having no baseUrl in enqueueLinks will cause it to not enqueue anything
-    // which is the intended behavior (since we went off domain)
+    // If they don't (we went off domain via a redirect), we keep filtering against the original domain - returning undefined would disable the filtering entirely
     if (enqueueStrategy === EnqueueStrategy.SameDomain) {
         const originalHostname = getDomain(originalUrlOrigin, { mixedInputs: false });
         const finalHostname = getDomain(finalUrlOrigin, { mixedInputs: false });
         if (originalHostname === finalHostname) {
             return finalUrlOrigin;
         }
-        return undefined;
+        return originalUrlOrigin;
     }
     // Always enqueue urls that are from the same origin in all other cases, as the filtering happens on the original request url, even if there was a redirect
     // before actually finding the urls

package/enqueue_links/shared.d.ts

@@ -1,6 +1,7 @@
 import type { Awaitable, Dictionary } from '@crawlee/types';
 import { z } from 'zod';
-import type { RequestOptions } from '../request.js';
+import type { RequestOptions, Source } from '../request.js';
+import { Request } from '../request.js';
 import type { EnqueueStrategyOption } from './enqueue_links.js';
 export interface UrlPatternObject {
     glob?: string;
@@ -23,9 +24,16 @@ export type UrlPatternInput = GlobInput | RegExpInput;
 export declare const urlPatternSchema: z.ZodType<UrlPatternInput>;
 export type SkippedRequestReason = 'robotsTxt' | 'limit' | 'enqueueLimit' | 'filters' | 'transform' | 'redirect' | 'depth';
 export type SkippedRequestCallback = (args: {
-    url: string;
+    request: Request;
     reason: SkippedRequestReason;
 }) => Awaitable<void>;
+/**
+ * Builds the `{ request, reason }` argument passed to a {@link SkippedRequestCallback}, constructing the
+ * `Request` lazily on first access to `request` (and caching it) since most skips are never observed by a
+ * real callback and building a `Request` isn't free.
+ * @ignore
+ */
+export declare function createSkippedRequestArgs(source: string | Source, reason: SkippedRequestReason): Parameters<SkippedRequestCallback>[0];
 /**
  * @ignore
  */
@@ -58,7 +66,7 @@ export declare function constructUrlPatternObjects(patterns: readonly UrlPattern
  * When `includePatterns` is empty/undefined, all options pass through (only exclude filtering applies).
  * @ignore
  */
-export declare function filterRequestOptionsByPatterns(requestOptions: RequestOptions[], includePatterns: UrlPatternObject[] | undefined, excludePatterns?: UrlPatternObject[], strategy?: EnqueueStrategyOption, onSkippedUrl?: (url: string) => void): RequestOptions[];
+export declare function filterRequestOptionsByPatterns(requestOptions: RequestOptions[], includePatterns: UrlPatternObject[] | undefined, excludePatterns?: UrlPatternObject[], strategy?: EnqueueStrategyOption, onSkippedRequestOptions?: (options: RequestOptions) => void): RequestOptions[];
 /**
  * @ignore
  */

package/enqueue_links/shared.js

@@ -1,6 +1,7 @@
 import { URL } from 'node:url';
 import { Minimatch } from 'minimatch';
 import { z } from 'zod';
+import { Request } from '../request.js';
 import { schemas } from '../validators.js';
 const MAX_ENQUEUE_LINKS_CACHE_SIZE = 1000;
 /**
@@ -19,6 +20,26 @@ export const urlPatternSchema = z.union([
     schemas.objectWithKeys(['glob']),
     schemas.objectWithKeys(['regexp']),
 ]);
+/**
+ * Builds the `{ request, reason }` argument passed to a {@link SkippedRequestCallback}, constructing the
+ * `Request` lazily on first access to `request` (and caching it) since most skips are never observed by a
+ * real callback and building a `Request` isn't free.
+ * @ignore
+ */
+export function createSkippedRequestArgs(source, reason) {
+    const sourceReason = typeof source === 'string' ? undefined : source.skippedReason;
+    let request;
+    return {
+        reason: sourceReason ?? reason,
+        get request() {
+            request ??=
+                source instanceof Request
+                    ? source
+                    : new Request(typeof source === 'string' ? { url: source } : source);
+            return request;
+        },
+    };
+}
 /**
  * @ignore
  */
@@ -117,14 +138,14 @@ export function constructUrlPatternObjects(patterns) {
  * When `includePatterns` is empty/undefined, all options pass through (only exclude filtering applies).
  * @ignore
  */
-export function filterRequestOptionsByPatterns(requestOptions, includePatterns, excludePatterns = [], strategy, onSkippedUrl) {
+export function filterRequestOptionsByPatterns(requestOptions, includePatterns, excludePatterns = [], strategy, onSkippedRequestOptions) {
     const excludeMatchers = excludePatterns.map(createPatternObjectMatcher);
     const includeMatchers = includePatterns?.length ? includePatterns.map(createPatternObjectMatcher) : undefined;
     return requestOptions
-        .filter(({ url }) => {
-        const matchesExclude = excludeMatchers.some(({ match }) => match(url));
+        .filter((opts) => {
+        const matchesExclude = excludeMatchers.some(({ match }) => match(opts.url));
         if (matchesExclude) {
-            onSkippedUrl?.(url);
+            onSkippedRequestOptions?.(opts);
         }
         return !matchesExclude;
     })
@@ -138,7 +159,7 @@ export function filterRequestOptionsByPatterns(requestOptions, includePatterns,
             }
         }
         // didn't match any positive pattern
-        onSkippedUrl?.(opts.url);
+        onSkippedRequestOptions?.(opts);
         return null;
     })
         .filter((opts) => opts !== null);

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/core",
-    "version": "4.0.0-beta.149",
+    "version": "4.0.0-beta.150",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -52,10 +52,10 @@
         "@apify/log": "^2.5.18",
         "@apify/timeout": "^0.4.4",
         "@apify/utilities": "^2.15.5",
-        "@crawlee/fs-storage": "4.0.0-beta.149",
-        "@crawlee/http-client": "4.0.0-beta.149",
-        "@crawlee/types": "4.0.0-beta.149",
-        "@crawlee/utils": "4.0.0-beta.149",
+        "@crawlee/fs-storage": "4.0.0-beta.150",
+        "@crawlee/http-client": "4.0.0-beta.150",
+        "@crawlee/types": "4.0.0-beta.150",
+        "@crawlee/utils": "4.0.0-beta.150",
         "@sapphire/async-queue": "^1.5.5",
         "@standard-schema/spec": "^1.0.0",
         "@vladfrangu/async_event_emitter": "^2.4.6",
@@ -78,5 +78,5 @@
             }
         }
     },
-    "gitHead": "c88124c43b4c366d4b0ade01da5e90843d59c582"
+    "gitHead": "ef7e9fb2e2fcaa0f74e60a30fa8ae61c60c64626"
 }