apify 4.0.0-beta.25 → 4.0.0-beta.26

package/dist/enqueue_links_filters.d.ts

@@ -0,0 +1,29 @@
+import type { RequestOptions, RequestTransform } from '@crawlee/core';
+/** Request options that the `globs` and `pseudoUrls` input editors can attach to a single URL pattern. */
+export interface UrlPatternRequestOptions extends Pick<RequestOptions, 'method' | 'payload' | 'label' | 'userData' | 'headers'> {
+}
+/** A single item of an input schema array field using the `globs` editor. */
+export interface GlobInput extends UrlPatternRequestOptions {
+    glob: string;
+}
+/** A single item of an input schema array field using the `pseudoUrls` editor. */
+export interface PseudoUrlInput extends UrlPatternRequestOptions {
+    purl: string;
+}
+export interface UrlPatternFilters {
+    /** Value of an input schema field using the `globs` editor. */
+    globs?: readonly (string | GlobInput)[];
+    /** Value of an input schema field using the `pseudoUrls` editor. */
+    pseudoUrls?: readonly (string | PseudoUrlInput)[];
+}
+/**
+ * Turns the `globs` and `pseudoUrls` input editor values into a `transformRequestFunction` that skips requests
+ * matching no pattern (all requests pass when no pattern is given) and applies the options of the matched pattern.
+ * When a URL matches multiple patterns, the first one wins, with globs checked before pseudo-URLs. The matched
+ * pattern's options shallowly override the request's fields, including `userData` (no deep merge, like in v3).
+ *
+ * ```ts
+ * await enqueueLinks({ transformRequestFunction: createTransformRequestFunction({ globs: input.globs }) });
+ * ```
+ */
+export declare function createTransformRequestFunction({ globs, pseudoUrls }: UrlPatternFilters): RequestTransform;

package/dist/enqueue_links_filters.js

@@ -0,0 +1,52 @@
+import { Minimatch } from 'minimatch';
+import { purlToRegExp } from '@apify/pseudo_url';
+/**
+ * Turns the `globs` and `pseudoUrls` input editor values into a `transformRequestFunction` that skips requests
+ * matching no pattern (all requests pass when no pattern is given) and applies the options of the matched pattern.
+ * When a URL matches multiple patterns, the first one wins, with globs checked before pseudo-URLs. The matched
+ * pattern's options shallowly override the request's fields, including `userData` (no deep merge, like in v3).
+ *
+ * ```ts
+ * await enqueueLinks({ transformRequestFunction: createTransformRequestFunction({ globs: input.globs }) });
+ * ```
+ */
+export function createTransformRequestFunction({ globs = [], pseudoUrls = [] }) {
+    const patterns = [
+        ...globs.flatMap((item) => {
+            if (item == null) {
+                return [];
+            }
+            const { glob, ...options } = typeof item === 'string' ? { glob: item } : item;
+            const trimmedGlob = typeof glob === 'string' ? glob.trim() : '';
+            if (trimmedGlob.length === 0) {
+                return [];
+            }
+            const minimatch = new Minimatch(trimmedGlob, { nocase: true });
+            return [{ matches: (url) => minimatch.match(url), options }];
+        }),
+        ...pseudoUrls.flatMap((item) => {
+            if (item == null) {
+                return [];
+            }
+            const { purl, ...options } = typeof item === 'string' ? { purl: item } : item;
+            if (typeof purl !== 'string' || purl.trim().length === 0) {
+                return [];
+            }
+            let regexp;
+            try {
+                regexp = purlToRegExp(purl);
+            }
+            catch (cause) {
+                throw new Error(`Invalid pseudoUrl pattern '${purl}': ${cause.message}`, { cause });
+            }
+            return [{ matches: (url) => regexp.test(url), options }];
+        }),
+    ];
+    if (patterns.length === 0) {
+        return () => 'unchanged';
+    }
+    return (request) => {
+        const matched = patterns.find(({ matches }) => matches(request.url));
+        return matched ? { ...request, ...matched.options } : false;
+    };
+}

package/dist/index.d.ts

@@ -2,6 +2,7 @@ export * from './actor.js';
 export { ApifyStorageBackend, type ApifyStorageBackendOptions } from './apify_storage_backend.js';
 export type { RequestQueueAccessMode } from './apify_request_queue_backend.js';
 export { ArgumentValidationError } from './utils.js';
+export { createTransformRequestFunction, type GlobInput, type PseudoUrlInput, type UrlPatternFilters, type UrlPatternRequestOptions, } from './enqueue_links_filters.js';
 export type { OpenStorageOptions, StorageAlias, StorageId, StorageName, StorageIdentifier, StorageIdentifierWithoutAlias, } from './storage.js';
 export { ChargeOptions, ChargeResult, ActorPricingInfo, ChargingManager } from './charging.js';
 export * from './configuration.js';

package/dist/index.js

@@ -1,6 +1,7 @@
 export * from './actor.js';
 export { ApifyStorageBackend } from './apify_storage_backend.js';
 export { ArgumentValidationError } from './utils.js';
+export { createTransformRequestFunction, } from './enqueue_links_filters.js';
 export { ChargingManager } from './charging.js';
 export * from './configuration.js';
 export * from './proxy_configuration.js';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "apify",
-  "version": "4.0.0-beta.25",
+  "version": "4.0.0-beta.26",
   "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
   "engines": {
     "node": ">=22.0.0"
@@ -53,16 +53,18 @@
     ]
   },
   "dependencies": {
-    "@apify/consts": "^2.51.0",
+    "@apify/consts": "^2.57.0",
     "@apify/datastructures": "^2.0.3",
     "@apify/input_secrets": "^1.2.0",
-    "@apify/log": "^2.4.3",
+    "@apify/log": "^2.5.48",
+    "@apify/pseudo_url": "^2.0.89",
     "@apify/timeout": "^0.4.0",
     "@apify/utilities": "^2.13.0",
     "@crawlee/core": "^4.0.0-beta.128",
     "@crawlee/types": "^4.0.0-beta.128",
     "@crawlee/utils": "^4.0.0-beta.128",
     "apify-client": "^2.23.4",
+    "minimatch": "^10.0.1",
     "semver": "^7.5.4",
     "tslib": "^2.6.2",
     "ws": "^8.18.0",