@crawlee/basic 4.0.0-beta.259 → 4.0.0-beta.260

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -2,9 +2,8 @@ import { mkdir, writeFile } from 'node:fs/promises';
2
2
  import { dirname } from 'node:path';
3
3
  import { bindMethodsToServiceLocator, Configuration, createStorageTransaction, Request, CriticalError, currentStorageTransaction, Dataset, EnqueueStrategy, EventManager, EventType, getObjectType, KeyValueStore, log, NonRetryableError, OwnedOrInjected, purgeDefaultStorages, RequestManagerTandem, RequestQueue, ServiceLocator, serviceLocator, SessionError, validators, withDirectStorageAccess, } from '@crawlee/core';
4
4
  import { BaseHttpClient, FetchHttpClient } from '@crawlee/http-client';
5
- import { isAsyncIterable, isIterable, parseArgument, ROTATE_PROXY_ERRORS, schemas } from '@crawlee/utils/internal';
5
+ import { filterUrl, isAsyncIterable, isIterable, matchesEnqueueStrategy, parseArgument, ROTATE_PROXY_ERRORS, schemas, } from '@crawlee/utils/internal';
6
6
  import { RobotsTxtFile } from '@crawlee/utils';
7
- import { getDomain } from 'tldts';
8
7
  import { z } from 'zod';
9
8
  import { LruCache } from '@apify/datastructures';
10
9
  import { addTimeoutToPromise, extendTimeout, storage as timeoutStorage, TimeoutError, tryCancel } from '@apify/timeout';
@@ -15,7 +14,7 @@ import { ContextPipeline } from './crawlers/context-pipeline.js';
15
14
  import { CrawlingRequest, RequestState } from './crawling-request.js';
16
15
  import { ContextPipelineCleanupError, ContextPipelineInterruptedError, MissingSessionError, NavigationSkippedError, PersistentRateLimitError, ContextPipelineInitializationError, RequestHandlerError, RequestThrottledError, RetryRequestError, } from './errors.js';
17
16
  import { Statistics } from './crawlers/statistics.js';
18
- import { applyRequestTransform, buildEnqueueStrategyPatterns, constructUrlPatternObjects, createRequestOptions, createSkippedRequestArgs, filterRequestOptionsByPatterns, } from './enqueue-links/index.js';
17
+ import { applyRequestTransform, constructUrlPatternObjects, createRequestOptions, createSkippedRequestArgs, filterRequestOptionsByPatterns, } from './enqueue-links/index.js';
19
18
  import { parseRetryAfterHeader } from './http.js';
20
19
  import { CrawlerRun } from './crawler-run.js';
21
20
  import { extendTimeoutKey, navigationDeadlineKey, raceWithTimeout, suspendTimeoutKey, timeoutExpiredKey, } from './request-timeout.js';
@@ -1116,7 +1115,7 @@ export class BasicCrawler {
1116
1115
  const urlPatternObjects = options.include?.length ? constructUrlPatternObjects(options.include) : [];
1117
1116
  // The strategy always applies, even when `include` patterns are provided - the two are AND-ed together
1118
1117
  // (a URL must match an `include` pattern *and* satisfy the strategy). This mirrors crawlee-python.
1119
- const enqueueStrategyPatterns = options.baseUrl ? buildEnqueueStrategyPatterns(options.baseUrl, strategy) : [];
1118
+ const strategyBaseUrl = options.baseUrl ? new URL(options.baseUrl) : undefined;
1120
1119
  const isAllowedBasedOnRobotsTxtFile = this.#isAllowedBasedOnRobotsTxtFile.bind(this);
1121
1120
  const maxCrawlDepth = this.#maxCrawlDepth;
1122
1121
  const validateRequestUserData = this.#validateRequestUserData.bind(this);
@@ -1167,7 +1166,14 @@ export class BasicCrawler {
1167
1166
  // Filter by user patterns first (with exclude)...
1168
1167
  let filtered = filterRequestOptionsByPatterns([requestOptions], urlPatternObjects.length > 0 ? urlPatternObjects : undefined, urlExcludePatternObjects, strategy, onSkippedByFilter);
1169
1168
  // ...then filter by the enqueue strategy (making this an AND check)
1170
- filtered = filterRequestOptionsByPatterns(filtered, enqueueStrategyPatterns.length > 0 ? enqueueStrategyPatterns : undefined, [], strategy, onSkippedByFilter);
1169
+ if (strategyBaseUrl) {
1170
+ filtered = filtered.filter((opts) => {
1171
+ const { allowed } = filterUrl(opts.url, strategyBaseUrl, strategy);
1172
+ if (!allowed)
1173
+ onSkippedByFilter(opts);
1174
+ return allowed;
1175
+ });
1176
+ }
1171
1177
  if (filtered.length === 0) {
1172
1178
  continue;
1173
1179
  }
@@ -1879,30 +1885,7 @@ export class BasicCrawler {
1879
1885
  if (!loadedUrl) {
1880
1886
  return true;
1881
1887
  }
1882
- const baseUrl = new URL(url);
1883
- const loadedBaseUrl = new URL(loadedUrl);
1884
- switch (strategy) {
1885
- case EnqueueStrategy.SameHostname: {
1886
- return baseUrl.hostname === loadedBaseUrl.hostname;
1887
- }
1888
- case EnqueueStrategy.SameDomain: {
1889
- const baseUrlHostname = getDomain(baseUrl.hostname, { mixedInputs: false });
1890
- if (baseUrlHostname) {
1891
- const loadedBaseUrlHostname = getDomain(loadedBaseUrl.hostname, { mixedInputs: false });
1892
- return baseUrlHostname === loadedBaseUrlHostname;
1893
- }
1894
- // Can happen for IPs, we just check like same origin
1895
- return baseUrl.origin === loadedBaseUrl.origin;
1896
- }
1897
- case EnqueueStrategy.SameOrigin: {
1898
- // Same as hostname, but also checks protocol
1899
- return baseUrl.origin === loadedBaseUrl.origin;
1900
- }
1901
- case EnqueueStrategy.All:
1902
- default: {
1903
- return baseUrl.protocol === 'http:' || baseUrl.protocol === 'https:';
1904
- }
1905
- }
1888
+ return matchesEnqueueStrategy(strategy, new URL(loadedUrl), new URL(url));
1906
1889
  }
1907
1890
  }
1908
1891
  /** The hostname of `url`, falling back to the whole string when it is not parseable - for log messages only. */
@@ -1,6 +1,6 @@
1
1
  import type { Dictionary } from '@crawlee/types';
2
2
  import type { EnqueueStrategyOption, RequestQueueOperationOptions } from '@crawlee/core';
3
- import type { RequestTransform, SkippedRequestCallback, UrlPatternInput, UrlPatternObject } from './shared.js';
3
+ import type { RequestTransform, SkippedRequestCallback, UrlPatternInput } from './shared.js';
4
4
  /**
5
5
  * Options shared by the `extractLinks()` context helper across crawler types.
6
6
  */
@@ -145,8 +145,3 @@ export interface ResolveBaseUrl {
145
145
  originalRequestUrl: string;
146
146
  finalRequestUrl?: string;
147
147
  }
148
- /**
149
- * @internal
150
- * Builds the glob patterns a URL must match to satisfy the given enqueue `strategy`, anchored at `baseUrl`.
151
- */
152
- export declare function buildEnqueueStrategyPatterns(baseUrl: string, strategy: EnqueueStrategyOption): UrlPatternObject[];
@@ -33,44 +33,3 @@ export function resolveBaseUrlForEnqueueLinksFiltering({ enqueueStrategy, finalR
33
33
  // before actually finding the urls
34
34
  return originalUrlOrigin;
35
35
  }
36
- /**
37
- * @internal
38
- * Builds the glob patterns a URL must match to satisfy the given enqueue `strategy`, anchored at `baseUrl`.
39
- */
40
- export function buildEnqueueStrategyPatterns(baseUrl, strategy) {
41
- const url = new URL(baseUrl);
42
- switch (strategy) {
43
- case EnqueueStrategy.SameHostname:
44
- // We need to get the origin of the passed in domain in the event someone sets baseUrl
45
- // to an url like https://example.com/deep/default/path and one of the found urls is an
46
- // absolute relative path (/path/to/page)
47
- return [{ glob: ignoreHttpSchema(`${url.origin}/**`) }];
48
- case EnqueueStrategy.SameDomain: {
49
- // Get the actual hostname from the base url
50
- const baseUrlHostname = getDomain(url.hostname, { mixedInputs: false });
51
- if (baseUrlHostname) {
52
- // We have a hostname, so we can use it to match all links on the page that point to it and any subdomains of it
53
- url.hostname = baseUrlHostname;
54
- return [
55
- { glob: ignoreHttpSchema(`${url.origin.replace(baseUrlHostname, `*.${baseUrlHostname}`)}/**`) },
56
- { glob: ignoreHttpSchema(`${url.origin}/**`) },
57
- ];
58
- }
59
- // We don't have a hostname (can happen for ips for instance), so reproduce the same behavior
60
- // as SameDomainAndSubdomain
61
- return [{ glob: ignoreHttpSchema(`${url.origin}/**`) }];
62
- }
63
- case EnqueueStrategy.SameOrigin:
64
- // The same behavior as SameHostname, but respecting the protocol of the URL
65
- return [{ glob: `${url.origin}/**` }];
66
- case EnqueueStrategy.All:
67
- default:
68
- return [{ glob: `http{s,}://**` }];
69
- }
70
- }
71
- /**
72
- * Internal function that changes the enqueue glob patterns to match both http and https
73
- */
74
- function ignoreHttpSchema(pattern) {
75
- return pattern.replace(/^(https?):\/\//, 'http{s,}://');
76
- }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@crawlee/basic",
3
- "version": "4.0.0-beta.259",
3
+ "version": "4.0.0-beta.260",
4
4
  "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
5
5
  "engines": {
6
6
  "node": ">=22.13.0"
@@ -42,10 +42,10 @@
42
42
  "@apify/datastructures": "^3.0.1",
43
43
  "@apify/timeout": "^1.0.1",
44
44
  "@apify/utilities": "^3.1.0",
45
- "@crawlee/core": "4.0.0-beta.259",
46
- "@crawlee/http-client": "4.0.0-beta.259",
47
- "@crawlee/types": "4.0.0-beta.259",
48
- "@crawlee/utils": "4.0.0-beta.259",
45
+ "@crawlee/core": "4.0.0-beta.260",
46
+ "@crawlee/http-client": "4.0.0-beta.260",
47
+ "@crawlee/types": "4.0.0-beta.260",
48
+ "@crawlee/utils": "4.0.0-beta.260",
49
49
  "@sapphire/async-queue": "^1.5.5",
50
50
  "@standard-schema/spec": "^1.1.0",
51
51
  "csv-stringify": "^6.9.0",
@@ -57,7 +57,7 @@
57
57
  "zod": "^4.6.7"
58
58
  },
59
59
  "optionalDependencies": {
60
- "@crawlee/impit-client": "4.0.0-beta.259"
60
+ "@crawlee/impit-client": "4.0.0-beta.260"
61
61
  },
62
62
  "lerna": {
63
63
  "command": {
@@ -66,5 +66,5 @@
66
66
  }
67
67
  }
68
68
  },
69
- "gitHead": "8b6c21145fd782c4618503b633fd45f6ce8e2385"
69
+ "gitHead": "c96c87d09eb5df1bcdf6365563ea99739f612a35"
70
70
  }