@crawlee/basic 4.0.0-beta.267 → 4.0.0-beta.269

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1191,7 +1191,7 @@ export class BasicCrawler {
1191
1191
  }
1192
1192
  const result = await this.requestManager.addRequestsBatched(filteredRequests(), {
1193
1193
  forefront: options.forefront,
1194
- waitForAllRequestsToBeAdded: options.waitForAllRequestsToBeAdded,
1194
+ waitForAllRequestsToBeAdded: options.waitForAllRequestsToBeAdded || requestLimit !== undefined,
1195
1195
  batchSize: options.batchSize,
1196
1196
  waitBetweenBatchesMillis: options.waitBetweenBatchesMillis,
1197
1197
  maxNewRequests: requestLimit,
@@ -34,13 +34,13 @@ export declare function createSkippedRequestArgs(source: string | Source, reason
34
34
  /**
35
35
  * @ignore
36
36
  */
37
- export declare function updateEnqueueLinksPatternCache(item: GlobInput | RegExpInput, pattern: RegExpObject | GlobObject): void;
37
+ export declare function updateEnqueueLinksPatternCache(item: UrlPatternInput, pattern: UrlPatternObject): void;
38
38
  /**
39
39
  * Helper factory used in the `enqueueLinks()` and enqueueLinksByClickingElements() function
40
40
  * to construct Glob objects from Glob pattern strings.
41
41
  * @ignore
42
42
  */
43
- export declare function constructGlobObjectsFromGlobs(globs: readonly GlobInput[]): GlobObject[];
43
+ export declare function constructGlobObjectsFromGlobs(globs: readonly GlobInput[]): (GlobObject & UrlPatternObject)[];
44
44
  /**
45
45
  * @internal
46
46
  */
@@ -50,7 +50,7 @@ export declare function validateGlobPattern(glob: string): string;
50
50
  * to check RegExps input and return valid RegExps.
51
51
  * @ignore
52
52
  */
53
- export declare function constructRegExpObjectsFromRegExps(regexps: readonly RegExpInput[]): RegExpObject[];
53
+ export declare function constructRegExpObjectsFromRegExps(regexps: readonly RegExpInput[]): (RegExpObject & UrlPatternObject)[];
54
54
  /**
55
55
  * The compiled form of a {@link UrlPatternInput}, produced by {@link constructUrlPatternObjects}. Deliberately
56
56
  * not re-exported from the package entry point — it is an implementation detail of the pattern matching helpers.
@@ -59,6 +59,7 @@ export declare function constructRegExpObjectsFromRegExps(regexps: readonly RegE
59
59
  export interface UrlPatternObject {
60
60
  glob?: string;
61
61
  regexp?: RegExp;
62
+ match: (url: string) => boolean;
62
63
  }
63
64
  /**
64
65
  * Helper factory used in the `enqueueLinks()` function to construct UrlPatternObjects
@@ -45,8 +45,7 @@ export function createSkippedRequestArgs(source, reason) {
45
45
  export function updateEnqueueLinksPatternCache(item, pattern) {
46
46
  enqueueLinksPatternCache.set(item, pattern);
47
47
  if (enqueueLinksPatternCache.size > MAX_ENQUEUE_LINKS_CACHE_SIZE) {
48
- const key = enqueueLinksPatternCache.keys().next().value;
49
- enqueueLinksPatternCache.delete(key);
48
+ enqueueLinksPatternCache.delete(enqueueLinksPatternCache.keys().next().value);
50
49
  }
51
50
  }
52
51
  /**
@@ -70,16 +69,12 @@ export function constructGlobObjectsFromGlobs(globs) {
70
69
  return false;
71
70
  })
72
71
  .map((item) => {
73
- // Get glob object from cache.
74
- let globObject = enqueueLinksPatternCache.get(item);
75
- if (globObject)
76
- return globObject;
77
- if (typeof item === 'string') {
78
- globObject = { glob: validateGlobPattern(item) };
79
- }
80
- else {
81
- globObject = { glob: validateGlobPattern(item.glob) };
82
- }
72
+ const cached = enqueueLinksPatternCache.get(item);
73
+ if (cached)
74
+ return cached;
75
+ const glob = validateGlobPattern(typeof item === 'string' ? item : item.glob);
76
+ const minimatch = new Minimatch(glob, { nocase: true });
77
+ const globObject = { glob, match: (url) => minimatch.match(url) };
83
78
  updateEnqueueLinksPatternCache(item, globObject);
84
79
  return globObject;
85
80
  });
@@ -100,16 +95,11 @@ export function validateGlobPattern(glob) {
100
95
  */
101
96
  export function constructRegExpObjectsFromRegExps(regexps) {
102
97
  return regexps.map((item) => {
103
- // Get regexp object from cache.
104
- let regexpObject = enqueueLinksPatternCache.get(item);
105
- if (regexpObject)
106
- return regexpObject;
107
- if (item instanceof RegExp) {
108
- regexpObject = { regexp: item };
109
- }
110
- else {
111
- regexpObject = { regexp: item.regexp };
112
- }
98
+ const cached = enqueueLinksPatternCache.get(item);
99
+ if (cached)
100
+ return cached;
101
+ const regexp = item instanceof RegExp ? item : item.regexp;
102
+ const regexpObject = { regexp, match: (url) => regexp.test(url) };
113
103
  updateEnqueueLinksPatternCache(item, regexpObject);
114
104
  return regexpObject;
115
105
  });
@@ -138,21 +128,19 @@ export function constructUrlPatternObjects(patterns) {
138
128
  * @ignore
139
129
  */
140
130
  export function filterRequestOptionsByPatterns(requestOptions, includePatterns, excludePatterns = [], strategy, onSkippedRequestOptions) {
141
- const excludeMatchers = excludePatterns.map(createPatternObjectMatcher);
142
- const includeMatchers = includePatterns?.length ? includePatterns.map(createPatternObjectMatcher) : undefined;
143
131
  return requestOptions
144
132
  .filter((opts) => {
145
- const matchesExclude = excludeMatchers.some(({ match }) => match(opts.url));
133
+ const matchesExclude = excludePatterns.some(({ match }) => match(opts.url));
146
134
  if (matchesExclude) {
147
135
  onSkippedRequestOptions?.(opts);
148
136
  }
149
137
  return !matchesExclude;
150
138
  })
151
139
  .map((opts) => {
152
- if (!includeMatchers) {
140
+ if (!includePatterns?.length) {
153
141
  return { ...opts, enqueueStrategy: strategy };
154
142
  }
155
- for (const { match } of includeMatchers) {
143
+ for (const { match } of includePatterns) {
156
144
  if (match(opts.url)) {
157
145
  return { ...opts, enqueueStrategy: strategy };
158
146
  }
@@ -212,24 +200,6 @@ export function createRequestOptions(sources, options = {}) {
212
200
  return requestOptions;
213
201
  });
214
202
  }
215
- /**
216
- * @ignore
217
- */
218
- function createPatternObjectMatcher(urlPatternObject) {
219
- const { regexp, glob } = urlPatternObject;
220
- let match;
221
- if (regexp) {
222
- match = (url) => regexp.test(url);
223
- }
224
- else if (glob) {
225
- const m = new Minimatch(glob, { nocase: true });
226
- match = (url) => m.match(url);
227
- }
228
- else {
229
- match = () => false;
230
- }
231
- return { match };
232
- }
233
203
  /**
234
204
  * Applies a {@link RequestTransform} function to a list of request options.
235
205
  * Options for which the transform returns a falsy value are removed from the list.
@@ -1,7 +1,6 @@
1
1
  import { Transform } from 'node:stream';
2
2
  import { EnqueueStrategy } from '@crawlee/utils';
3
3
  import { parseArgument, parseSitemap, schemas } from '@crawlee/utils/internal';
4
- import { minimatch } from 'minimatch';
5
4
  import { z } from 'zod';
6
5
  import { EventType, KeyValueStore, purgeDefaultStorages, Request, RequestManagerTandem, RequestQueue, serviceLocator, } from '@crawlee/core';
7
6
  import { constructUrlPatternObjects, urlPatternSchema } from './enqueue-links/index.js';
@@ -123,27 +122,14 @@ export class SitemapRequestLoader {
123
122
  highWaterMark,
124
123
  }).pause();
125
124
  }
126
- /**
127
- * Returns a function that checks whether the provided pattern matches the closure URL.
128
- * @param url URL to be checked.
129
- * @returns A matcher function that checks whether the pattern matches the closure URL.
130
- */
131
- #matchesUrl(url) {
132
- return (patternObject) => {
133
- const { regexp, glob } = patternObject;
134
- const matchesRegex = (regexp && url.match(regexp)) || false;
135
- const matchesGlob = (glob && minimatch(url, glob, { nocase: true })) || false;
136
- return Boolean(matchesRegex || matchesGlob);
137
- };
138
- }
139
125
  /**
140
126
  * Checks whether the URL matches the `include` / `exclude` patterns provided in the `options`.
141
127
  * @param url URL to be checked.
142
128
  * @returns `true` if the URL matches the patterns, `false` otherwise.
143
129
  */
144
130
  #isUrlMatchingPatterns(url) {
145
- return (!this.#urlExcludePatternObjects.some(this.#matchesUrl(url)) &&
146
- (this.#urlPatternObjects.length === 0 || this.#urlPatternObjects.some(this.#matchesUrl(url))));
131
+ return (!this.#urlExcludePatternObjects.some(({ match }) => match(url)) &&
132
+ (this.#urlPatternObjects.length === 0 || this.#urlPatternObjects.some(({ match }) => match(url))));
147
133
  }
148
134
  /**
149
135
  * Adds a URL to the queue of parsed URLs.
@@ -188,9 +188,8 @@ export declare class ThrottlingRequestManager<T extends IRequestManager = IReque
188
188
  /**
189
189
  * Reports whether anything can be dispatched right now, and if not, when — or why never.
190
190
  *
191
- * One traversal of the domain clocks answers all of it: only domains whose delays have run out are probed,
192
- * the rest merely contribute the moment they come due. Throttled requests count as outstanding work, so a
193
- * crawler gated on this idles for the backoff instead of concluding it is done.
191
+ * Dispatchable domains are probed first. When nothing can be dispatched, cooling domains contribute their
192
+ * deadlines only while they still have outstanding work, so a finished crawl does not wait out a cooldown.
194
193
  *
195
194
  * `ready` from anywhere else outranks a stalling domain and is returned without looking at the stall clocks,
196
195
  * so one hopeless domain never ends a crawl making progress elsewhere. It cannot outrank itself, though -
@@ -618,9 +618,8 @@ export class ThrottlingRequestManager {
618
618
  /**
619
619
  * Reports whether anything can be dispatched right now, and if not, when — or why never.
620
620
  *
621
- * One traversal of the domain clocks answers all of it: only domains whose delays have run out are probed,
622
- * the rest merely contribute the moment they come due. Throttled requests count as outstanding work, so a
623
- * crawler gated on this idles for the backoff instead of concluding it is done.
621
+ * Dispatchable domains are probed first. When nothing can be dispatched, cooling domains contribute their
622
+ * deadlines only while they still have outstanding work, so a finished crawl does not wait out a cooldown.
624
623
  *
625
624
  * `ready` from anywhere else outranks a stalling domain and is returned without looking at the stall clocks,
626
625
  * so one hopeless domain never ends a crawl making progress elsewhere. It cannot outrank itself, though -
@@ -631,24 +630,25 @@ export class ThrottlingRequestManager {
631
630
  await this.#ensureSubManagers();
632
631
  const now = Date.now();
633
632
  const dispatchable = [];
634
- let readyAt;
633
+ const cooling = [];
635
634
  let stallCandidates;
636
635
  for (const state of this.#domainStates.values()) {
637
636
  // A `Crawl-delay` can give a domain a clock before its first request gives it a queue - nothing
638
637
  // to fetch from and nothing to wait for until then.
639
- if (!this.#subManagers.has(state.domain)) {
638
+ const subManager = this.#subManagers.get(state.domain);
639
+ if (!subManager) {
640
640
  continue;
641
641
  }
642
642
  if (isStalling(state, now, this.#maxDomainStallMs)) {
643
643
  (stallCandidates ??= []).push(state);
644
644
  continue;
645
645
  }
646
- const until = throttledUntil(state);
647
- if (now >= until) {
648
- dispatchable.push(this.#subManagers.get(state.domain));
646
+ const readyAt = throttledUntil(state);
647
+ if (now >= readyAt) {
648
+ dispatchable.push(subManager);
649
649
  }
650
- else if (readyAt === undefined || until < readyAt) {
651
- readyAt = until;
650
+ else {
651
+ cooling.push({ subManager, readyAt });
652
652
  }
653
653
  }
654
654
  const inner = await this.#getInner();
@@ -660,8 +660,14 @@ export class ThrottlingRequestManager {
660
660
  if (probed.status === 'ready') {
661
661
  return probed;
662
662
  }
663
- if (readyAt !== undefined) {
664
- return joinRequestSourceStatuses(probed, { status: 'waiting', readyAt });
663
+ // A cooling domain with nothing left to dispatch is nothing to wait for; in deadline order, the first
664
+ // with work left is the earliest wake-up.
665
+ cooling.sort((a, b) => a.readyAt - b.readyAt);
666
+ for (const { subManager, readyAt } of cooling) {
667
+ const { status } = await (await subManager).checkReadiness();
668
+ if (status !== 'finished') {
669
+ return joinRequestSourceStatuses(probed, { status: 'waiting', readyAt });
670
+ }
665
671
  }
666
672
  // Batches still landing in the background are work nobody can see in a queue yet.
667
673
  if (this.#inProgressBatchCount > 0 && probed.status === 'finished') {
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@crawlee/basic",
3
- "version": "4.0.0-beta.267",
3
+ "version": "4.0.0-beta.269",
4
4
  "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
5
5
  "engines": {
6
6
  "node": ">=22.13.0"
@@ -42,10 +42,10 @@
42
42
  "@apify/datastructures": "^3.0.1",
43
43
  "@apify/timeout": "^1.0.1",
44
44
  "@apify/utilities": "^3.1.0",
45
- "@crawlee/core": "4.0.0-beta.267",
46
- "@crawlee/http-client": "4.0.0-beta.267",
47
- "@crawlee/types": "4.0.0-beta.267",
48
- "@crawlee/utils": "4.0.0-beta.267",
45
+ "@crawlee/core": "4.0.0-beta.269",
46
+ "@crawlee/http-client": "4.0.0-beta.269",
47
+ "@crawlee/types": "4.0.0-beta.269",
48
+ "@crawlee/utils": "4.0.0-beta.269",
49
49
  "@sapphire/async-queue": "^1.5.5",
50
50
  "@standard-schema/spec": "^1.1.0",
51
51
  "csv-stringify": "^6.9.0",
@@ -57,7 +57,7 @@
57
57
  "zod": "^4.6.7"
58
58
  },
59
59
  "optionalDependencies": {
60
- "@crawlee/impit-client": "4.0.0-beta.267"
60
+ "@crawlee/impit-client": "4.0.0-beta.269"
61
61
  },
62
62
  "lerna": {
63
63
  "command": {
@@ -66,5 +66,5 @@
66
66
  }
67
67
  }
68
68
  },
69
- "gitHead": "a4e49e8a8117acdb808c6cba5b05c3d56bf2b123"
69
+ "gitHead": "ee60f6c7d6aefb5a275a199a118244f75f4675e6"
70
70
  }